From ba64df8c08842012335a40fc271399d7cc3c1144 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Wed, 25 Mar 2026 12:54:54 +0800 Subject: [PATCH] fix(db-sync): stabilize checksum parity for e2ee graph init --- deps/db-sync/src/logseq/db_sync/checksum.cljs | 55 ++++--- deps/db-sync/src/logseq/db_sync/storage.cljs | 8 +- .../logseq/db_sync/worker/handler/sync.cljs | 18 ++- .../test/logseq/db_sync/checksum_test.cljs | 137 +++++++++++++----- .../db_sync/worker_handler_sync_test.cljs | 19 +++ src/main/frontend/worker/sync.cljs | 3 +- .../frontend/worker/sync/handle_message.cljs | 3 +- 7 files changed, 174 insertions(+), 69 deletions(-) diff --git a/deps/db-sync/src/logseq/db_sync/checksum.cljs b/deps/db-sync/src/logseq/db_sync/checksum.cljs index f0d59a7d60..58b6950c94 100644 --- a/deps/db-sync/src/logseq/db_sync/checksum.cljs +++ b/deps/db-sync/src/logseq/db_sync/checksum.cljs @@ -55,6 +55,12 @@ (or (parse-hex32 (subs checksum 8 16)) 0)] [0 0])) +(defn- valid-checksum? + [checksum] + (boolean + (and (string? checksum) + (re-matches #"[0-9a-fA-F]{16}" checksum)))) + (defn- state->checksum [[fnv djb]] (str (unsigned-hex fnv) @@ -98,9 +104,9 @@ (not e2ee?) (hash-code field-separator) (not e2ee?) (digest-string name) (not e2ee?) (hash-code field-separator) - true (digest-string (some-> parent str)) + true (digest-string (some-> parent :block/uuid str)) true (hash-code field-separator) - true (digest-string (some-> page str)))))) + true (digest-string (some-> page :block/uuid str)))))) (defn recompute-checksum [db] @@ -122,23 +128,28 @@ (defn update-checksum [checksum {:keys [db-before db-after tx-data]}] - (let [db (or db-after db-before) - e2ee? (ldb/get-graph-rtc-e2ee? db) - changed-eids (->> tx-data (keep :e) distinct) - initial-state (if (string? checksum) - (checksum->state checksum) - (checksum->state (when db-before (recompute-checksum db-before))))] - (->> changed-eids - (reduce (fn [[sum-fnv sum-djb] eid] - (let [old-digest (when db-before (entity-digest db-before eid e2ee?)) - new-digest (when db-after (entity-digest db-after eid e2ee?)) - [sum-fnv sum-djb] (if old-digest - [(sub-step sum-fnv (first old-digest)) - (sub-step sum-djb (second old-digest))] - [sum-fnv sum-djb])] - (if new-digest - [(add-step sum-fnv (first new-digest)) - (add-step sum-djb (second new-digest))] - [sum-fnv sum-djb]))) - initial-state) - state->checksum))) + (let [before-e2ee? (ldb/get-graph-rtc-e2ee? db-before) + after-e2ee? (ldb/get-graph-rtc-e2ee? db-after)] + (if (not= before-e2ee? after-e2ee?) + ;; E2EE mode changes the global digest semantics, so incremental deltas are invalid. + (recompute-checksum db-after) + (let [changed-eids (->> tx-data + (remove (fn [d] + (contains? #{:block/tx-id} (:a d)))) + (keep :e) + distinct) + initial-state (if (valid-checksum? checksum) + (checksum->state checksum) + (checksum->state (recompute-checksum db-before)))] + (->> changed-eids + (reduce (fn [[sum-fnv sum-djb] eid] + (let [old-digest (entity-digest db-before eid after-e2ee?) + new-digest (entity-digest db-after eid after-e2ee?)] + [(cond-> sum-fnv + old-digest (sub-step (first old-digest)) + new-digest (add-step (first new-digest))) + (cond-> sum-djb + old-digest (sub-step (second old-digest)) + new-digest (add-step (second new-digest)))])) + initial-state) + state->checksum))))) diff --git a/deps/db-sync/src/logseq/db_sync/storage.cljs b/deps/db-sync/src/logseq/db_sync/storage.cljs index 708d657b44..858d7b4667 100644 --- a/deps/db-sync/src/logseq/db_sync/storage.cljs +++ b/deps/db-sync/src/logseq/db_sync/storage.cljs @@ -144,13 +144,11 @@ (restore-data-from-addr sql addr)))) (defn- append-tx-for-tx-report - [sql {:keys [db-after db-before tx-data tx-meta]}] + [sql {:keys [db-after db-before tx-data tx-meta] :as tx-report}] (let [new-t (next-t! sql) created-at (common/now-ms) - checksum (sync-checksum/update-checksum (get-checksum sql) - {:db-before db-before - :db-after db-after - :tx-data tx-data}) + prev-checksum (get-checksum sql) + checksum (sync-checksum/update-checksum prev-checksum tx-report) normalized-data (->> tx-data (db-normalize/normalize-tx-data db-after db-before)) ;; _ (prn :debug :tx-data tx-data) diff --git a/deps/db-sync/src/logseq/db_sync/worker/handler/sync.cljs b/deps/db-sync/src/logseq/db_sync/worker/handler/sync.cljs index d502d2f82e..10b936b69c 100644 --- a/deps/db-sync/src/logseq/db_sync/worker/handler/sync.cljs +++ b/deps/db-sync/src/logseq/db_sync/worker/handler/sync.cljs @@ -13,7 +13,8 @@ [logseq.db-sync.worker.routes.sync :as sync-routes] [logseq.db-sync.worker.ws :as ws] [logseq.db.frontend.schema :as db-schema] - [promesa.core :as p])) + [promesa.core :as p] + [logseq.db-sync.checksum :as checksum])) (def ^:private snapshot-download-batch-size 10000) (def ^:private snapshot-cache-control "private, max-age=300") @@ -55,9 +56,18 @@ (defn current-checksum [^js self] (ensure-conn! self) - (let [db @(.-conn self)] - (when-not (ldb/get-graph-rtc-e2ee? db) - (storage/get-checksum (.-sql self))))) + (let [db @(.-conn self) + full-checksum (checksum/recompute-checksum db) + current-checksum (storage/get-checksum (.-sql self))] + (if (or (nil? current-checksum) + (= full-checksum current-checksum)) + current-checksum + (do + (log/error :db-sync/server-checksum-mismatch + {:full-checksum full-checksum + :current-checksum current-checksum}) + (storage/set-checksum! (.-sql self) full-checksum) + full-checksum)))) (defn snapshot-upload-finished? [^js self] (ensure-schema! self) diff --git a/deps/db-sync/test/logseq/db_sync/checksum_test.cljs b/deps/db-sync/test/logseq/db_sync/checksum_test.cljs index 4ebd0b19ec..3ab875ef24 100644 --- a/deps/db-sync/test/logseq/db_sync/checksum_test.cljs +++ b/deps/db-sync/test/logseq/db_sync/checksum_test.cljs @@ -4,40 +4,109 @@ [logseq.db-sync.checksum :as checksum] [logseq.db.frontend.schema :as db-schema])) -(deftest checksum-ignores-unrelated-datoms-test - (testing "checksum only depends on uuid, title, parent, and page" - (let [page-uuid (random-uuid) - block-uuid (random-uuid) - base-db (-> (d/empty-db db-schema/schema) - (d/db-with [{:db/id 1 - :block/uuid page-uuid - :block/name "page" - :block/title "Page"} - {:db/id 2 - :block/uuid block-uuid - :block/title "Block" - :block/parent 1 - :block/page 1}])) - db-with-unrelated (d/db-with base-db [[:db/add 2 :block/updated-at 1773661308002] - [:db/add 2 :logseq.property/created-by-ref 99]])] - (is (= (checksum/recompute-checksum base-db) - (checksum/recompute-checksum db-with-unrelated)))))) +(defn- sample-db + [] + (let [page-a-uuid (random-uuid) + page-b-uuid (random-uuid) + parent-uuid (random-uuid) + child-uuid (random-uuid)] + (-> (d/empty-db db-schema/schema) + (d/db-with [{:db/id 1 + :block/uuid page-a-uuid + :block/name "page-a" + :block/title "Page A"} + {:db/id 2 + :block/uuid page-b-uuid + :block/name "page-b" + :block/title "Page B"} + {:db/id 3 + :block/uuid parent-uuid + :block/title "Parent" + :block/parent 1 + :block/page 1} + {:db/id 4 + :block/uuid child-uuid + :block/title "Child" + :block/parent 3 + :block/page 1}])))) -(deftest incremental-checksum-matches-recompute-test - (testing "incremental checksum matches full recompute after a tx" - (let [page-uuid (random-uuid) - block-uuid (random-uuid) - db-before (-> (d/empty-db db-schema/schema) - (d/db-with [{:db/id 1 - :block/uuid page-uuid - :block/name "page" - :block/title "Page"} - {:db/id 2 - :block/uuid block-uuid - :block/title "Block" - :block/parent 1 - :block/page 1}])) - tx-report (d/with db-before [[:db/add 2 :block/title "Updated"] - [:db/add 1 :block/name "page-updated"]])] +(defn- assert-incremental=full! + [db-before checksum-before tx-data] + (let [tx-report (d/with db-before tx-data) + full (checksum/recompute-checksum (:db-after tx-report)) + incremental (checksum/update-checksum checksum-before tx-report)] + (is (= full incremental) + (str "Expected checksum parity for tx-data: " (pr-str tx-data))) + {:db (:db-after tx-report) + :checksum incremental})) + +(deftest checksum-ignores-unrelated-datoms-test + (testing "recompute and incremental checksums ignore unrelated datoms" + (let [db-before (sample-db) + checksum-before (checksum/recompute-checksum db-before) + tx-data [[:db/add 4 :block/updated-at 1773661308002] + [:db/add 4 :logseq.property/created-by-ref 99]] + tx-report (d/with db-before tx-data)] + (is (= checksum-before + (checksum/recompute-checksum (:db-after tx-report)))) + (is (= checksum-before + (checksum/update-checksum checksum-before tx-report)))))) + +(deftest incremental-checksum-matches-recompute-on-replace-datom-test + (testing "incremental checksum matches full recompute when replacing existing values" + (let [db-before (sample-db) + tx-report (d/with db-before [[:db/add 4 :block/title "Child updated"] + [:db/add 1 :block/name "page-a-updated"]])] + (is (= (checksum/recompute-checksum (:db-after tx-report)) + (checksum/update-checksum (checksum/recompute-checksum db-before) tx-report)))))) + +(deftest incremental-checksum-matches-recompute-across-mixed-mutations-test + (testing "incremental checksum stays equal to full recompute across typical tx sequences" + (let [db0 (sample-db) + new-block-uuid (random-uuid) + {:keys [db checksum]} (reduce + (fn [{:keys [db checksum]} {:keys [tx-data]}] + (assert-incremental=full! db checksum tx-data)) + {:db db0 + :checksum (checksum/recompute-checksum db0)} + [{:tx-data [[:db/add 4 :block/title "Child edited"]]} + {:tx-data [[:db/add 1 :block/name "page-a-renamed"] + [:db/add 1 :block/title "Page A Renamed"]]} + {:tx-data [[:db/add 4 :block/parent 2] + [:db/add 4 :block/page 2]]} + {:tx-data [[:db/add -1 :block/uuid new-block-uuid] + [:db/add -1 :block/title "New block"] + [:db/add -1 :block/parent 2] + [:db/add -1 :block/page 2]]} + {:tx-data [[:db/retract 3 :block/title "Parent"]]} + {:tx-data [[:db/retractEntity [:block/uuid new-block-uuid]]]} + {:tx-data [[:db/add 4 :block/updated-at 1773661308002]]}])] + (is (= checksum (checksum/recompute-checksum db)))))) + +(deftest incremental-checksum-uses-recompute-when-initial-checksum-missing-test + (testing "nil initial checksum uses db-before recompute as baseline" + (let [db-before (sample-db) + tx-report (d/with db-before [[:db/add 4 :block/title "Child updated"]])] + (is (= (checksum/recompute-checksum (:db-after tx-report)) + (checksum/update-checksum nil tx-report)))))) + +(deftest checksum-e2ee-ignores-title-and-name-test + (testing "with E2EE enabled, checksum ignores title/name changes for both modes" + (let [db-before (-> (sample-db) + (d/db-with [{:db/ident :logseq.kv/graph-rtc-e2ee? + :kv/value true}])) + checksum-before (checksum/recompute-checksum db-before) + tx-report (d/with db-before [[:db/add 4 :block/title "Encrypted title update"] + [:db/add 1 :block/name "encrypted-name-update"]])] + (is (= checksum-before + (checksum/recompute-checksum (:db-after tx-report)))) + (is (= checksum-before + (checksum/update-checksum checksum-before tx-report)))))) + +(deftest incremental-checksum-recomputes-when-e2ee-mode-toggles-test + (testing "incremental checksum falls back to full recompute when E2EE mode changes" + (let [db-before (sample-db) + tx-report (d/with db-before [{:db/ident :logseq.kv/graph-rtc-e2ee? + :kv/value true}])] (is (= (checksum/recompute-checksum (:db-after tx-report)) (checksum/update-checksum (checksum/recompute-checksum db-before) tx-report)))))) diff --git a/deps/db-sync/test/logseq/db_sync/worker_handler_sync_test.cljs b/deps/db-sync/test/logseq/db_sync/worker_handler_sync_test.cljs index 35368ae967..2863077c33 100644 --- a/deps/db-sync/test/logseq/db_sync/worker_handler_sync_test.cljs +++ b/deps/db-sync/test/logseq/db_sync/worker_handler_sync_test.cljs @@ -226,6 +226,25 @@ (is false (str error)) (done))))))) +(deftest current-checksum-heals-stale-stored-checksum-test + (testing "server recomputes and persists checksum when stored checksum is stale" + (let [sql (test-sql/make-sql) + conn (storage/open-conn sql) + self #js {:sql sql + :conn conn + :schema-ready true} + stale-checksum "0000000000000000" + block-uuid (random-uuid)] + (d/transact! conn [{:block/uuid block-uuid + :block/title "hello"}]) + (is (string? (storage/get-checksum sql))) + (storage/set-checksum! sql stale-checksum) + (let [healed (sync-handler/current-checksum self)] + (is (string? healed)) + (is (not= stale-checksum healed)) + (is (= healed (storage/get-checksum sql))) + (is (= healed (sync-handler/current-checksum self))))))) + (deftest tx-batch-rejects-with-the-exact-failed-tx-entry-test (testing "db transact failure replies with the specific rejected tx entry" (let [sql (test-sql/make-sql) diff --git a/src/main/frontend/worker/sync.cljs b/src/main/frontend/worker/sync.cljs index 0d549433cd..53c00ee386 100644 --- a/src/main/frontend/worker/sync.cljs +++ b/src/main/frontend/worker/sync.cljs @@ -49,8 +49,7 @@ (defn update-local-sync-checksum! [repo tx-report] - (when (and (worker-state/get-client-ops-conn repo) - (not (sync-crypt/graph-e2ee? repo))) + (when (worker-state/get-client-ops-conn repo) (client-op/update-local-checksum repo (sync-checksum/update-checksum (client-op/get-local-checksum repo) tx-report)))) diff --git a/src/main/frontend/worker/sync/handle_message.cljs b/src/main/frontend/worker/sync/handle_message.cljs index 262709312a..d4798234f8 100644 --- a/src/main/frontend/worker/sync/handle_message.cljs +++ b/src/main/frontend/worker/sync/handle_message.cljs @@ -120,8 +120,7 @@ (defn- verify-sync-checksum! [repo client local-tx remote-tx remote-checksum context] - (when (and (not (sync-crypt/graph-e2ee? repo)) - (string? remote-checksum) + (when (and (string? remote-checksum) (checksum-compare-ready? repo client local-tx remote-tx)) (let [local-checksum (local-sync-checksum repo)] (when-not (= local-checksum remote-checksum)