diff --git a/deps/graph-parser/src/logseq/graph_parser.cljs b/deps/graph-parser/src/logseq/graph_parser.cljs index b733ee3084..ffad73c5ed 100644 --- a/deps/graph-parser/src/logseq/graph_parser.cljs +++ b/deps/graph-parser/src/logseq/graph_parser.cljs @@ -8,20 +8,13 @@ [clojure.string :as string] [clojure.set :as set])) -(defn- db-set-file-content! - "Modified copy of frontend.db.model/db-set-file-content!" - [conn path content] - (let [tx-data {:file/path path - :file/content content}] - (d/transact! conn [tx-data] {:skip-refresh? true}))) - (defn parse-file "Parse file and save parsed data to the given db. Main parse fn used by logseq app" - [conn file content {:keys [new? delete-blocks-fn extract-options] + [conn file content {:keys [new? delete-blocks-fn extract-options skip-db-transact?] :or {new? true - delete-blocks-fn (constantly [])} + delete-blocks-fn (constantly []) + skip-db-transact? false} :as options}] - (db-set-file-content! conn file content) (let [format (gp-util/get-format file) file-content [{:file/path file}] {:keys [tx ast]} @@ -48,12 +41,15 @@ {:tx (concat file-content pages-index delete-blocks pages block-ids blocks) :ast ast}) {:tx file-content}) - tx (concat tx [(cond-> {:file/path file} + tx (concat tx [(cond-> {:file/path file + :file/content content} new? ;; TODO: use file system timestamp? (assoc :file/created-at (date-time-util/time-ms)))]) tx' (gp-util/remove-nils tx) - result (d/transact! conn tx' (select-keys options [:new-graph? :from-disk?]))] + result (if skip-db-transact? + tx' + (d/transact! conn tx' (select-keys options [:new-graph? :from-disk?])))] {:tx result :ast ast})) diff --git a/src/main/frontend/handler/common/file.cljs b/src/main/frontend/handler/common/file.cljs index cd8a567fc0..8fa4f9006c 100644 --- a/src/main/frontend/handler/common/file.cljs +++ b/src/main/frontend/handler/common/file.cljs @@ -62,22 +62,18 @@ :else file) file (gp-util/path-normalize file) - new? (nil? (db/entity [:file/path file]))] - (:tx - (graph-parser/parse-file - (db/get-db repo-url false) - file - content - (merge (dissoc options :verbose) - {:new? new? - :delete-blocks-fn (partial get-delete-blocks repo-url) - :extract-options (merge - {:user-config (state/get-config) - :date-formatter (state/get-date-formatter) - :page-name-order (state/page-name-order) - :block-pattern (config/get-block-pattern (gp-util/get-format file)) - :supported-formats (gp-config/supported-formats)} - (when (some? verbose) {:verbose verbose}))})))) + new? (nil? (db/entity [:file/path file])) + options (merge (dissoc options :verbose) + {:new? new? + :delete-blocks-fn (partial get-delete-blocks repo-url) + :extract-options (merge + {:user-config (state/get-config) + :date-formatter (state/get-date-formatter) + :page-name-order (state/page-name-order) + :block-pattern (config/get-block-pattern (gp-util/get-format file)) + :supported-formats (gp-config/supported-formats)} + (when (some? verbose) {:verbose verbose}))})] + (:tx (graph-parser/parse-file (db/get-db repo-url false) file content options))) (catch :default e (prn "Reset file failed " {:file file}) (log/error :exception e))))) diff --git a/src/main/frontend/handler/file.cljs b/src/main/frontend/handler/file.cljs index bb30f185b4..ba4e6a6655 100644 --- a/src/main/frontend/handler/file.cljs +++ b/src/main/frontend/handler/file.cljs @@ -86,7 +86,8 @@ ;; TODO: Remove this function in favor of `alter-files` (defn alter-file - [repo path content {:keys [reset? re-render-root? from-disk? skip-compare? new-graph? verbose] + [repo path content {:keys [reset? re-render-root? from-disk? skip-compare? new-graph? verbose + skip-db-transact?] :or {reset? true re-render-root? false from-disk? false @@ -101,17 +102,19 @@ (assoc (when original-content {:old-content original-content}) :skip-compare? skip-compare?)))) opts {:new-graph? new-graph? - :from-disk? from-disk?}] - (if reset? - (do - (when-let [page-id (db/get-file-page-id path)] - (db/transact! repo - [[:db/retract page-id :block/alias] - [:db/retract page-id :block/tags]] - opts)) - (file-common-handler/reset-file! repo path content (merge opts - (when (some? verbose) {:verbose verbose})))) - (db/set-file-content! repo path content opts)) + :from-disk? from-disk? + :skip-db-transact? skip-db-transact?} + result (if reset? + (do + (when-not skip-db-transact? + (when-let [page-id (db/get-file-page-id path)] + (db/transact! repo + [[:db/retract page-id :block/alias] + [:db/retract page-id :block/tags]] + opts))) + (file-common-handler/reset-file! repo path content (merge opts + (when (some? verbose) {:verbose verbose})))) + (db/set-file-content! repo path content opts))] (util/p-handle (write-file!) (fn [_] (cond @@ -134,7 +137,8 @@ :status :error}])) (println "Write file failed, path: " path ", content: " content) - (log/error :write/failed error))))) + (log/error :write/failed error))) + result)) (defn set-file-content! [repo path new-content] diff --git a/src/main/frontend/handler/repo.cljs b/src/main/frontend/handler/repo.cljs index 400bdd513b..3124b9d6fb 100644 --- a/src/main/frontend/handler/repo.cljs +++ b/src/main/frontend/handler/repo.cljs @@ -31,7 +31,8 @@ [cljs-bean.core :as bean] [clojure.core.async :as async] [frontend.encrypt :as encrypt] - [frontend.mobile.util :as mobile-util])) + [frontend.mobile.util :as mobile-util] + [medley.core :as medley])) ;; Project settings should be checked in two situations: ;; 1. User changes the config.edn directly in logseq.com (fn: alter-file) @@ -174,21 +175,27 @@ file-paths [path]] (load-pages-metadata! repo file-paths files force?))) +(defonce *file-tx (atom nil)) + (defn- parse-and-load-file! - [repo-url file {:keys [new-graph? verbose]}] + [repo-url file {:keys [new-graph? verbose skip-db-transact?] + :or {skip-db-transact? true}}] (try - (file-handler/alter-file repo-url - (:file/path file) - (:file/content file) - (merge {:new-graph? new-graph? - :re-render-root? false - :from-disk? true} - (when (some? verbose) {:verbose verbose}))) + (reset! *file-tx + (file-handler/alter-file repo-url + (:file/path file) + (:file/content file) + (merge {:new-graph? new-graph? + :re-render-root? false + :from-disk? true + :skip-db-transact? skip-db-transact?} + (when (some? verbose) {:verbose verbose})))) (catch :default e (state/set-parsing-state! (fn [m] (update m :failed-parsing-files conj [(:file/path file) e]))))) (state/set-parsing-state! (fn [m] - (update m :finished inc)))) + (update m :finished inc))) + @*file-tx) (defn- after-parse [repo-url files file-paths db-encrypted? re-render? re-render-opts opts graph-added-chan] @@ -210,8 +217,11 @@ (let [supported-files (graph-parser/filter-files files) delete-data (->> (concat delete-files delete-blocks) (remove nil?)) - chan (async/to-chan! supported-files) - graph-added-chan (async/promise-chan)] + indexed-files (medley/indexed supported-files) + chan (async/to-chan! indexed-files) + graph-added-chan (async/promise-chan) + total (count supported-files) + large-graph? (> total 1000)] (when (seq delete-data) (db/transact! repo-url delete-data)) (state/set-current-repo! repo-url) (state/set-parsing-state! {:total (count supported-files)}) @@ -220,18 +230,33 @@ (do (doseq [file supported-files] (state/set-parsing-state! (fn [m] - (assoc m :current-parsing-file (:file/path file)))) - (parse-and-load-file! repo-url file (select-keys opts [:new-graph? :verbose]))) + (assoc m + :current-parsing-file (:file/path file)))) + (parse-and-load-file! repo-url file (assoc + (select-keys opts [:new-graph? :verbose]) + :skip-db-transact? false))) (after-parse repo-url files file-paths db-encrypted? re-render? re-render-opts opts graph-added-chan)) - (async/go-loop [] - (if-let [file (async/