perf: importing graph

There're two things makes the importing slow:
1. db transact! for each file, now it's batched for 100 files.
2. block waiting for 10s before parsing so that the progress bar ui
can be changed, this is batched too for 10 files.

src/main/frontend/handler/common/file.cljs

@@ -62,22 +62,18 @@
                   :else
                   file)
            file (gp-util/path-normalize file)
-           new? (nil? (db/entity [:file/path file]))]
-       (:tx
-        (graph-parser/parse-file
-         (db/get-db repo-url false)
-         file
-         content
-         (merge (dissoc options :verbose)
-                {:new? new?
-                 :delete-blocks-fn (partial get-delete-blocks repo-url)
-                 :extract-options (merge
-                                   {:user-config (state/get-config)
-                                    :date-formatter (state/get-date-formatter)
-                                    :page-name-order (state/page-name-order)
-                                    :block-pattern (config/get-block-pattern (gp-util/get-format file))
-                                    :supported-formats (gp-config/supported-formats)}
-                                   (when (some? verbose) {:verbose verbose}))}))))
+           new? (nil? (db/entity [:file/path file]))
+           options (merge (dissoc options :verbose)
+                          {:new? new?
+                           :delete-blocks-fn (partial get-delete-blocks repo-url)
+                           :extract-options (merge
+                                             {:user-config (state/get-config)
+                                              :date-formatter (state/get-date-formatter)
+                                              :page-name-order (state/page-name-order)
+                                              :block-pattern (config/get-block-pattern (gp-util/get-format file))
+                                              :supported-formats (gp-config/supported-formats)}
+                                             (when (some? verbose) {:verbose verbose}))})]
+       (:tx (graph-parser/parse-file (db/get-db repo-url false) file content options)))
      (catch :default e
        (prn "Reset file failed " {:file file})
        (log/error :exception e)))))

src/main/frontend/handler/file.cljs

@@ -86,7 +86,8 @@
 
 ;; TODO: Remove this function in favor of `alter-files`
 (defn alter-file
-  [repo path content {:keys [reset? re-render-root? from-disk? skip-compare? new-graph? verbose]
+  [repo path content {:keys [reset? re-render-root? from-disk? skip-compare? new-graph? verbose
+                             skip-db-transact?]
                       :or {reset? true
                            re-render-root? false
                            from-disk? false
@@ -101,17 +102,19 @@
                                         (assoc (when original-content {:old-content original-content})
                                                :skip-compare? skip-compare?))))
         opts {:new-graph? new-graph?
-              :from-disk? from-disk?}]
-    (if reset?
-      (do
-        (when-let [page-id (db/get-file-page-id path)]
-          (db/transact! repo
-                        [[:db/retract page-id :block/alias]
-                         [:db/retract page-id :block/tags]]
-                        opts))
-        (file-common-handler/reset-file! repo path content (merge opts
-                                                                  (when (some? verbose) {:verbose verbose}))))
-      (db/set-file-content! repo path content opts))
+              :from-disk? from-disk?
+              :skip-db-transact? skip-db-transact?}
+        result (if reset?
+                 (do
+                   (when-not skip-db-transact?
+                     (when-let [page-id (db/get-file-page-id path)]
+                       (db/transact! repo
+                         [[:db/retract page-id :block/alias]
+                          [:db/retract page-id :block/tags]]
+                         opts)))
+                   (file-common-handler/reset-file! repo path content (merge opts
+                                                         (when (some? verbose) {:verbose verbose}))))
+                 (db/set-file-content! repo path content opts))]
     (util/p-handle (write-file!)
                    (fn [_]
                      (cond
@@ -134,7 +137,8 @@
                                           :status :error}]))
 
                      (println "Write file failed, path: " path ", content: " content)
-                     (log/error :write/failed error)))))
+                     (log/error :write/failed error)))
+    result))
 
 (defn set-file-content!
   [repo path new-content]

src/main/frontend/handler/repo.cljs

@@ -31,7 +31,8 @@
             [cljs-bean.core :as bean]
             [clojure.core.async :as async]
             [frontend.encrypt :as encrypt]
-            [frontend.mobile.util :as mobile-util]))
+            [frontend.mobile.util :as mobile-util]
+            [medley.core :as medley]))
 
 ;; Project settings should be checked in two situations:
 ;; 1. User changes the config.edn directly in logseq.com (fn: alter-file)
@@ -174,21 +175,27 @@
         file-paths [path]]
     (load-pages-metadata! repo file-paths files force?)))
 
+(defonce *file-tx (atom nil))
+
 (defn- parse-and-load-file!
-  [repo-url file {:keys [new-graph? verbose]}]
+  [repo-url file {:keys [new-graph? verbose skip-db-transact?]
+                  :or {skip-db-transact? true}}]
   (try
-    (file-handler/alter-file repo-url
-                             (:file/path file)
-                             (:file/content file)
-                             (merge {:new-graph? new-graph?
-                                     :re-render-root? false
-                                     :from-disk? true}
-                                    (when (some? verbose) {:verbose verbose})))
+    (reset! *file-tx
+            (file-handler/alter-file repo-url
+                                     (:file/path file)
+                                     (:file/content file)
+                                     (merge {:new-graph? new-graph?
+                                             :re-render-root? false
+                                             :from-disk? true
+                                             :skip-db-transact? skip-db-transact?}
+                                            (when (some? verbose) {:verbose verbose}))))
     (catch :default e
       (state/set-parsing-state! (fn [m]
                                   (update m :failed-parsing-files conj [(:file/path file) e])))))
   (state/set-parsing-state! (fn [m]
-                              (update m :finished inc))))
+                              (update m :finished inc)))
+  @*file-tx)
 
 (defn- after-parse
   [repo-url files file-paths db-encrypted? re-render? re-render-opts opts graph-added-chan]
@@ -210,8 +217,11 @@
   (let [supported-files (graph-parser/filter-files files)
         delete-data (->> (concat delete-files delete-blocks)
                          (remove nil?))
-        chan (async/to-chan! supported-files)
-        graph-added-chan (async/promise-chan)]
+        indexed-files (medley/indexed supported-files)
+        chan (async/to-chan! indexed-files)
+        graph-added-chan (async/promise-chan)
+        total (count supported-files)
+        large-graph? (> total 1000)]
     (when (seq delete-data) (db/transact! repo-url delete-data))
     (state/set-current-repo! repo-url)
     (state/set-parsing-state! {:total (count supported-files)})
@@ -220,18 +230,33 @@
       (do
         (doseq [file supported-files]
           (state/set-parsing-state! (fn [m]
-                                      (assoc m :current-parsing-file (:file/path file))))
-          (parse-and-load-file! repo-url file (select-keys opts [:new-graph? :verbose])))
+                                      (assoc m
+                                             :current-parsing-file (:file/path file))))
+          (parse-and-load-file! repo-url file (assoc
+                                               (select-keys opts [:new-graph? :verbose])
+                                               :skip-db-transact? false)))
         (after-parse repo-url files file-paths db-encrypted? re-render? re-render-opts opts graph-added-chan))
-      (async/go-loop []
-        (if-let [file (async/<! chan)]
-          (do
+      (async/go-loop [tx []]
+        (if-let [item (async/<! chan)]
+          (let [[idx file] item
+                yield-for-ui? (or (not large-graph?)
+                                  (zero? (rem idx 10))
+                                  (<= (- total idx) 10))]
             (state/set-parsing-state! (fn [m]
                                         (assoc m :current-parsing-file (:file/path file))))
-            (async/<! (async/timeout 10))
-            (parse-and-load-file! repo-url file (select-keys opts [:new-graph? :verbose]))
-            (recur))
-          (after-parse repo-url files file-paths db-encrypted? re-render? re-render-opts opts graph-added-chan))))
+
+            (when yield-for-ui? (async/<! (async/timeout 1)))
+
+            (let [result (parse-and-load-file! repo-url file (select-keys opts [:new-graph? :verbose]))
+                  tx' (concat tx result)
+                  tx' (if (zero? (rem (inc idx) 100))
+                        (do (db/transact! repo-url tx' {:from-disk? true})
+                            [])
+                        tx')]
+              (recur tx')))
+          (do
+            (when (seq tx) (db/transact! repo-url tx {:from-disk? true}))
+            (after-parse repo-url files file-paths db-encrypted? re-render? re-render-opts opts graph-added-chan)))))
     graph-added-chan))
 
 (defn- parse-files-and-create-default-files!