fix: debounced store shouldn't be used for batch-transact!

deps/db/src/logseq/db.cljs

@@ -99,8 +99,9 @@
 (defn debounced-store-db
   [conn]
   (when-some [_storage (storage/storage @conn)]
-    (let [f (or @*debounce-fn d/store)]
-      (f @conn))))
+    (when-not (:batch-tx? @conn)
+      (let [f (or @*debounce-fn d/store)]
+        (f @conn)))))
 
 (defn- transact-sync
   [conn tx-data tx-meta]
@@ -112,12 +113,13 @@
                 (or (:rtc-download-graph? tx-meta)
                     (:reset-conn! tx-meta)
                     (:initial-db? tx-meta)
-                    (:skip-validate-db? db)
                     (:skip-validate-db? tx-meta false)
+                    ;; used by `batch-transact-with-temp-conn!`
+                    (:skip-validate-db? @conn)
                     (:logseq.graph-parser.exporter/new-graph? tx-meta))))
         (let [tx-report* (d/with db tx-data tx-meta)
               pipeline-f @*transact-pipeline-fn
-              tx-report (if-let [f pipeline-f] (f tx-report*) tx-report*)
+              tx-report (if pipeline-f (pipeline-f tx-report*) tx-report*)
               _ (throw-if-page-has-block-parent! (:db-after tx-report) (:tx-data tx-report))
               [validate-result errors] (db-validate/validate-tx-report tx-report nil)]
           (cond
@@ -126,11 +128,8 @@
                        (seq (:tx-data tx-report)))
               ;; perf enhancement: avoid repeated call on `d/with`
               (reset! conn (:db-after tx-report))
-              (if (:batch-tx? @conn)
-                (dc/run-callbacks conn tx-report)
-                (do
-                  (debounced-store-db conn)
-                  (dc/run-callbacks conn tx-report))))
+              (debounced-store-db conn)
+              (dc/run-callbacks conn tx-report))
 
             :else
             (do
@@ -237,7 +236,8 @@
 
       (swap! conn dissoc :skip-store? :batch-tx?)
 
-      (debounced-store-db conn)
+      (when-some [_storage (storage/storage @conn)]
+       (d/store @conn))
 
       (let [batch-tx-data @*tx-data
             _ (reset! *tx-data nil)

deps/db/src/logseq/db/common/normalize.cljs

@@ -101,6 +101,36 @@
        ;; sort by :tx, use nth to make this fn works on both vector and datom
        (sort-by #(nth % 3))))
 
+(defn- retract-entity-op?
+  [item]
+  (and (= 2 (count item))
+       (= :db/retractEntity (first item))))
+
+(defn- retract-entity-match-keys
+  [e]
+  (if (and (vector? e) (= :block/uuid (first e)))
+    (let [uuid (second e)]
+      #{e uuid (str uuid)})
+    #{e}))
+
+(defn- reorder-retract-entity-first
+  [tx-data]
+  (let [retract-ops (filter retract-entity-op? tx-data)
+        retract-keys (->> retract-ops
+                          (map second)
+                          (mapcat retract-entity-match-keys)
+                          set)
+        datom-for-retracted-eid?
+        (fn [item]
+          (and (= 5 (count item))
+               (contains? retract-keys (second item))))
+        datoms-for-retracted-eids (filter datom-for-retracted-eid? tx-data)
+        others (remove (fn [item]
+                         (or (retract-entity-op? item)
+                             (datom-for-retracted-eid? item)))
+                       tx-data)]
+    (concat retract-ops datoms-for-retracted-eids others)))
+
 (defn normalize-tx-data
   [db-after db-before tx-data]
   (let [title-updated-entities
@@ -150,4 +180,5 @@
                                   e)]
                   [op e'])))))
          (remove-retract-entity-ref db-after)
+         reorder-retract-entity-first
          distinct)))

deps/db/src/logseq/db/frontend/malli_schema.cljs

@@ -290,7 +290,8 @@
    [:block/refs {:optional true} [:set :int]]
    [:block/tx-id {:optional true} :int]
    [:block/collapsed? {:optional true} :boolean]
-   [:block/warning {:optional true} [:keyword]]])
+   [:block/warning {:optional true} [:keyword]]
+   [:logseq.property/created-by-ref {:optional true} :int]])
 
 (def page-attrs
   "Common attributes for pages"
@@ -441,13 +442,13 @@
     [:block/uuid :uuid]
     [:logseq.property.reaction/emoji-id :string]
     [:logseq.property.reaction/target :int]
-    [:block/properties {:optional true} block-properties]
     [:block/created-at :int]
     [:block/tx-id {:optional true} :int]
+    [:logseq.property/created-by-ref {:optional true} :int]
     [:block/refs {:optional true} [:set :int]]]))
 
 (def property-history-block*
-  [:map
+  [:map {:error/path ["property-history-block"]}
    [:block/uuid :uuid]
    [:block/created-at :int]
    [:block/updated-at {:optional true} :int]
@@ -455,6 +456,7 @@
    [:logseq.property.history/property :int]
    [:logseq.property.history/ref-value {:optional true} :int]
    [:logseq.property.history/scalar-value {:optional true} :any]
+   [:block/properties {:optional true} block-properties]
    [:block/tx-id {:optional true} :int]])
 
 (def property-history-block

deps/db/src/logseq/db/frontend/validate.cljs

@@ -82,7 +82,7 @@
                  :dispatch-key (->> (dissoc ent :db/id) (db-malli-schema/entity-dispatch-key db))
                  :errors errors'})))))
 
-(defn validate-db!
+(defn validate-db
   "Validates all the entities of the given db using :eavt datoms. Returns a map
   with info about db being validated. If there are errors, they are placed on
   :errors and grouped by entity"

src/main/frontend/worker/db/validate.cljs

@@ -231,7 +231,7 @@
    (fix-num-prefix-db-idents! conn)
 
    (let [db @conn
-         {:keys [errors datom-count entities]} (db-validate/validate-db! db)
+         {:keys [errors datom-count entities]} (db-validate/validate-db db)
          invalid-entity-ids (distinct (map (fn [e] (:db/id (:entity e))) errors))]
 
      (doseq [error errors]

src/main/frontend/worker/sync/apply_txs.cljs

@@ -206,6 +206,10 @@
           {:keys [forward-outliner-ops inverse-outliner-ops]}
           (derive-history-outliner-ops db-before db-after tx-data tx-meta)
           inferred-outliner-ops?' (inferred-outliner-ops? tx-meta)]
+      ;; (prn :debug :forward-outliner-ops)
+      ;; (cljs.pprint/pprint forward-outliner-ops)
+      ;; (prn :debug :inverse-outliner-ops)
+      ;; (cljs.pprint/pprint inverse-outliner-ops)
       (ldb/transact! conn [{:db-sync/tx-id tx-id
                             :db-sync/normalized-tx-data normalized-tx-data
                             :db-sync/reversed-tx-data reversed-datoms
@@ -469,38 +473,17 @@
                         (p/catch (fn [error]
                                    (js/console.error error))))))))))))))
 
-(defn- remote-tx-debug-meta
-  [temp-tx-meta remote-txs index {:keys [t outliner-op]}]
-  (cond-> (assoc temp-tx-meta
-                 :op :transact-remote-tx-data
-                 :skip-validate-db? true
-                 :remote-tx-index (inc index)
-                 :remote-tx-count (count remote-txs))
-    (number? t) (assoc :remote-t t)
-    outliner-op (assoc :outliner-op outliner-op)))
-
-(defn- local-tx-debug-meta
-  [tx-meta local-txs index local-tx op]
-  (cond-> (assoc tx-meta
-                 :op op
-                 :local-tx-index (inc index)
-                 :local-tx-count (count local-txs))
-    (:tx-id local-tx) (assoc :local-tx-id (:tx-id local-tx))
-    (:outliner-op local-tx) (assoc :outliner-op (:outliner-op local-tx))))
-
 (defn- reverse-history-action!
-  [conn local-txs index local-tx temp-tx-meta]
+  [conn local-tx]
   (if-let [tx-data (seq (:reversed-tx local-tx))]
-    (d/transact! conn
-                 tx-data
-                 (local-tx-debug-meta temp-tx-meta local-txs index local-tx :reverse))
+    (ldb/transact! conn tx-data {:reverse? true})
     (invalid-rebase-op! :reverse-history-action
                         {:reason :missing-reversed-tx-data
                          :tx-id (:tx-id local-tx)
                          :outliner-op (:outliner-op local-tx)})))
 
 (defn- transact-remote-txs!
-  [conn remote-txs temp-tx-meta]
+  [conn remote-txs]
   (loop [remaining remote-txs
          index 0
          results []]
@@ -508,9 +491,7 @@
       (let [tx-data (->> (:tx-data remote-tx)
                          seq)
             report (try
-                     (ldb/transact! conn
-                                    tx-data
-                                    (remote-tx-debug-meta temp-tx-meta remote-txs index remote-tx))
+                     (ldb/transact! conn tx-data {:transact-remote? true})
                      (catch :default e
                        (js/console.error e)
                        (log/error ::transact-remote-txs! {:remote-tx remote-tx
@@ -525,7 +506,7 @@
       results)))
 
 (defn reverse-local-txs!
-  [conn local-txs temp-tx-meta]
+  [conn local-txs]
   ;; (prn :debug :local-txs local-txs)
   (doall
    (->> local-txs
@@ -533,7 +514,7 @@
         (map-indexed
          (fn [index local-tx]
            (try
-             (reverse-history-action! conn local-txs index local-tx temp-tx-meta)
+             (reverse-history-action! conn local-tx)
              (catch :default e
                (log/error ::reverse-local-tx-error
                           {:index index
@@ -635,7 +616,7 @@
           block-uuid (:block/uuid block)
           block-ent (when block-uuid
                       (d/entity db [:block/uuid block-uuid]))
-          block-base (dissoc block :db/id :block/order)
+          block-base (dissoc block :db/id)
           block' (merge block-base
                         (op-construct/rewrite-block-title-with-retracted-refs db block-base))]
       (if (some? block-ent)
@@ -912,17 +893,19 @@
   (let [tx-meta {:rtc-tx? true
                  :with-local-changes? true}
         *rebase-tx-reports (atom [])]
+    ;; (prn :debug :apply-remote-tx (first remote-txs))
     (try
       (ldb/batch-transact!
        conn
        tx-meta
        (fn [conn]
-         (reverse-local-txs! conn local-txs {:rtc-tx? true})
+         (reverse-local-txs! conn local-txs)
 
-         (transact-remote-txs! conn remote-txs tx-meta)
+         (transact-remote-txs! conn remote-txs)
 
          (let [rebase-tx-report (rebase-local-txs! repo conn local-txs)]
            (fix-tx! conn rebase-tx-report {:outliner-op :rebase})))
+
        {:listen-db (fn [{:keys [tx-meta tx-data] :as tx-report}]
                      (when (and (= :rebase (:outliner-op tx-meta))
                                 (seq tx-data))
@@ -941,13 +924,13 @@
         (worker-undo-redo/clear-history! repo)))))
 
 (defn- apply-remote-tx-without-local-changes!
-  [{:keys [conn remote-txs temp-tx-meta]}]
+  [{:keys [conn remote-txs]}]
   (ldb/batch-transact-with-temp-conn!
    conn
    {:rtc-tx? true
     :without-local-changes? true}
    (fn [conn]
-     (transact-remote-txs! conn remote-txs temp-tx-meta))))
+     (transact-remote-txs! conn remote-txs))))
 
 (defn apply-remote-txs!
   [repo client remote-txs]

src/test/frontend/worker/db_sync_test.cljs

@@ -1212,7 +1212,7 @@
                     :reversed-tx [[:db/add child-id :block/title "raw reverse"]]}]
       (with-datascript-conns conn client-ops-conn
         (fn []
-          (let [reports (#'sync-apply/reverse-local-txs! conn [local-tx] {:rtc-tx? true})]
+          (let [reports (#'sync-apply/reverse-local-txs! conn [local-tx])]
             (is (= 1 (count reports)))
             (is (= "raw reverse"
                    (:block/title (d/entity @conn [:block/uuid child-uuid]))))))))))