From f0f5911ca7f01b1c9443de91bd1fe0dc1258c853 Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Tue, 24 Feb 2026 14:17:38 -0500 Subject: [PATCH 01/24] enhance: export-edn is idempotent for 3 more keys For keys with non-ordered collections, use sets to consistently produce idempotent values. Users can still author with vectors for these keys. This affects :build/tags, :build/property-classes and :build/class-extends. Also remove any hacks around making their values consistent across import->export cycles --- deps/db/src/logseq/db/sqlite/build.cljs | 15 +++-- deps/db/src/logseq/db/sqlite/export.cljs | 19 ++---- .../db/test/logseq/db/sqlite/export_test.cljs | 58 ++++++++++--------- .../create_graph_with_schema_org.cljs | 18 +----- 4 files changed, 48 insertions(+), 62 deletions(-) diff --git a/deps/db/src/logseq/db/sqlite/build.cljs b/deps/db/src/logseq/db/sqlite/build.cljs index 0e6e14dccb..32af84b9c0 100644 --- a/deps/db/src/logseq/db/sqlite/build.cljs +++ b/deps/db/src/logseq/db/sqlite/build.cljs @@ -411,7 +411,7 @@ [:block/title :string] [:build/children {:optional true} [:vector [:ref ::block]]] [:build/properties {:optional true} User-properties] - [:build/tags {:optional true} [:vector Class]] + [:build/tags {:optional true} [:or [:set Class] [:vector Class]]] [:build/keep-uuid? {:optional true} :boolean]]}} [:page [:and [:map @@ -419,7 +419,7 @@ [:block/title {:optional true} :string] [:build/journal {:optional true} :int] [:build/properties {:optional true} User-properties] - [:build/tags {:optional true} [:vector Class]] + [:build/tags {:optional true} [:or [:set Class] [:vector Class]]] [:build/keep-uuid? {:optional true} :boolean]] [:fn {:error/message ":block/title, :block/uuid or :build/journal required" :error/path [:block/title]} @@ -434,13 +434,14 @@ [:build/properties {:optional true} User-properties] [:build/properties-ref-types {:optional true} [:map-of :keyword :keyword]] + ;; TODO: Make this respect :block/order or allow :set [:build/closed-values {:optional true} [:vector [:map [:value [:or :string :double]] [:uuid {:optional true} :uuid] [:icon {:optional true} :map]]]] - [:build/property-classes {:optional true} [:vector Class]] + [:build/property-classes {:optional true} [:or [:set Class] [:vector Class]]] [:build/keep-uuid? {:optional true} :boolean]]]) (def Classes @@ -448,13 +449,19 @@ Class [:map [:build/properties {:optional true} User-properties] - [:build/class-extends {:optional true} [:vector Class]] + [:build/class-extends {:optional true} [:or [:set Class] [:vector Class]]] [:build/class-properties {:optional true} [:vector Property]] [:build/keep-uuid? {:optional true} :boolean]]]) (def Options + "Main malli schema that validates a sqlite.build EDN map. If an inner schema + uses :vector e.g. :blocks, it's to preserve :block/order-ing for that node's + attribute. If an inner schema uses :vector or :set e.g. :build/class-extends, + it's to indicate it is order-less and also allow users to write the more + familiar vector syntax" [:map {:closed true} + ;; TODO: Make this respect :block/order or allow :set [:pages-and-blocks {:optional true} [:vector Page-blocks]] [:properties {:optional true} Properties] [:classes {:optional true} Classes] diff --git a/deps/db/src/logseq/db/sqlite/export.cljs b/deps/db/src/logseq/db/sqlite/export.cljs index aa8bca7f2f..404f0da051 100644 --- a/deps/db/src/logseq/db/sqlite/export.cljs +++ b/deps/db/src/logseq/db/sqlite/export.cljs @@ -27,7 +27,7 @@ ;; These classes are redundant as :build/journal is enough for Journal and Page ;; is implied by being in :pages-and-blocks (remove #{:logseq.class/Page :logseq.class/Journal}) - vec)) + set)) (defn- block-title "Get an entity's original title" @@ -147,7 +147,7 @@ (and (not shallow-copy?) include-alias? (:block/alias property)) (assoc :block/alias (set (map #(vector :block/uuid (:block/uuid %)) (:block/alias property)))) (and (not shallow-copy?) (:logseq.property/classes property)) - (assoc :build/property-classes (mapv :db/ident (:logseq.property/classes property))) + (assoc :build/property-classes (set (map :db/ident (:logseq.property/classes property)))) (seq closed-values) (assoc :build/closed-values (mapv #(cond-> {:value (db-property/property-value-content %) @@ -194,7 +194,7 @@ (:logseq.property.class/extends class-ent) (not= [:logseq.class/Root] (mapv :db/ident (:logseq.property.class/extends class-ent)))) (assoc :build/class-extends - (mapv :db/ident (:logseq.property.class/extends class-ent))))) + (set (map :db/ident (:logseq.property.class/extends class-ent)))))) (defn block-property-value? [%] (and (map? %) (:build/property-value %))) @@ -1133,22 +1133,13 @@ {:error (str "The exported EDN is unexpectedly invalid: " (pr-str (ex-message e)))}))) (defn- prepare-export-to-diff - "This prepares a graph's exported edn to be diffed with another" + "Prepare a graph's exported edn to be diffed with another" [m] (-> m - ;; TODO: Fix order of these :build/* keys - (update :classes update-vals (fn [m] - (cond-> m - (:build/class-extends m) - (update :build/class-extends sort)))) - (update :properties update-vals (fn [m] - (cond-> m - (:build/property-classes m) - (update :build/property-classes sort)))) (update ::kv-values (fn [kvs] (->> kvs - ;; Ignore extra metadata that a copied graph can add + ;; This varies per copied graph so ignore it (remove #(#{:logseq.kv/import-type :logseq.kv/imported-at :logseq.kv/local-graph-uuid} (:db/ident %))) (sort-by :db/ident) diff --git a/deps/db/test/logseq/db/sqlite/export_test.cljs b/deps/db/test/logseq/db/sqlite/export_test.cljs index e1349bcb60..00da636f38 100644 --- a/deps/db/test/logseq/db/sqlite/export_test.cljs +++ b/deps/db/test/logseq/db/sqlite/export_test.cljs @@ -10,6 +10,7 @@ [logseq.common.uuid :as common-uuid] [logseq.db :as ldb] [logseq.db.frontend.validate :as db-validate] + [logseq.db.sqlite.build :as sqlite-build] [logseq.db.sqlite.export :as sqlite-export] [logseq.db.test.helper :as db-test] [medley.core :as medley])) @@ -61,7 +62,8 @@ imported-page (export-page-and-import-to-another-graph conn conn2 page-title) updated-page (db-test/find-page-by-title @conn2 page-title) expected-page-and-blocks - (update-in (:pages-and-blocks original-data) [0 :blocks] transform-expected-blocks) + (-> (:pages-and-blocks original-data) + (update-in [0 :blocks] transform-expected-blocks)) filter-imported-page (if build-journal #(= build-journal (get-in % [:page :build/journal])) #(= (get-in % [:page :block/title]) page-title))] @@ -91,8 +93,7 @@ imported-graph)) (defn- expand-properties - "Add default values to properties of an input export map to test against a - db-based export map" + "Modify given properties so that they match properties exported from the imported graph" [properties] (->> properties (map (fn [[k m]] @@ -100,20 +101,23 @@ (cond-> (merge {:db/cardinality :db.cardinality/one} m) + (:build/property-classes m) + (update :build/property-classes set) (not (:block/title m)) (assoc :block/title (name k)))])) (into {}))) (defn- expand-classes - "Add default values to classes of an input export map to test against a - db-based export map" + "Modify given classes so that they match classes exported from the imported graph" [classes] (->> classes (map (fn [[k m]] [k (cond-> m (not (:block/title m)) - (assoc :block/title (name k)))])) + (assoc :block/title (name k)) + (:build/class-extends m) + (update :build/class-extends set))])) (into {}))) (def sort-pages-and-blocks sqlite-export/sort-pages-and-blocks) @@ -159,7 +163,7 @@ [{:page {:block/title "page1"} :blocks [{:block/title "export" :build/properties {:user.property/default-many #{"foo" "bar" "baz"}} - :build/tags [:user.class/MyClass]} + :build/tags #{:user.class/MyClass}} {:block/title "import"}]}]} conn (db-test/create-conn-with-blocks original-data) imported-block (export-block-and-import-to-another-block conn conn "export" "import")] @@ -184,7 +188,7 @@ [{:page {:block/title "page1"} :blocks [{:block/title "export" :build/properties {:user.property/num-many #{3 6 9}} - :build/tags [:user.class/MyClass]}]}]} + :build/tags #{:user.class/MyClass}}]}]} conn (db-test/create-conn-with-blocks original-data) conn2 (db-test/create-conn-with-blocks {:pages-and-blocks [{:page {:block/title "page2"} @@ -249,10 +253,10 @@ {:block/title "b1ab"}]} {:block/title "b1b"}]} {:block/title "b2" - :build/tags [:user.class/MyClass]} + :build/tags #{:user.class/MyClass}} {:block/title "some task" :build/properties {:logseq.property/status :logseq.property/status.doing} - :build/tags [:logseq.class/Task]}]}]} + :build/tags #{:logseq.class/Task}}]}]} conn (db-test/create-conn-with-blocks original-data) conn2 (db-test/create-conn) imported-page (export-page-and-import-to-another-graph conn conn2 "page1")] @@ -388,9 +392,9 @@ :pages-and-blocks [{:page {:block/title "page1" :build/properties {:user.property/p1 "woot"} - :build/tags [:user.class/ChildClass]} + :build/tags #{:user.class/ChildClass}} :blocks [{:block/title "child object" - :build/tags [:user.class/ChildClass2]}]}]} + :build/tags #{:user.class/ChildClass2}}]}]} conn (db-test/create-conn-with-blocks original-data) conn2 (db-test/create-conn) imported-page (export-page-and-import-to-another-graph conn conn2 "page1")] @@ -471,14 +475,14 @@ :build/properties {:user.property/date [:build/page {:build/journal 20250203}]}} {:block/title "node block" :build/properties {:user.property/node #{[:build/page {:block/title "page object" - :build/tags [:user.class/MyClass]}] + :build/tags #{:user.class/MyClass}}] [:block/uuid block-object-uuid] :logseq.class/Task}}} {:block/title "map block" :build/properties {:user.property/map {:foo :bar :num 2}}}]} {:page {:block/title "Blocks"} :blocks [{:block/title "myclass object" - :build/tags [:user.class/MyClass] + :build/tags #{:user.class/MyClass} :block/uuid block-object-uuid :build/keep-uuid? true}]}]} conn (db-test/create-conn-with-blocks original-data) @@ -570,7 +574,7 @@ :build/properties {:user.property/p1 "ok"} :build/children [{:block/title "b2"}]} {:block/title "b3" - :build/tags [:user.class/class1] + :build/tags #{:user.class/class1} :build/children [{:block/title "b4"}]}]} {:page {:block/title "page2"} :blocks [{:block/title "dont export"}]}]} @@ -655,7 +659,7 @@ {:block/title "b2" :build/properties {:user.property/node #{[:block/uuid page-object-uuid]}}} {:block/title "b3" :build/properties {:user.property/node #{[:block/uuid page-object-uuid]}}} {:block/title "Example advanced query", - :build/tags [:logseq.class/Query], + :build/tags #{:logseq.class/Query}, :build/properties {:logseq.property/query {:build/property-value :block @@ -668,24 +672,24 @@ {:user.property/url {:build/property-value :block :block/title "https://example.com" - :build/tags [:user.class/MyClass]}}}]} + :build/tags #{:user.class/MyClass}}}}]} {:page {:block/title "page object" :block/uuid page-object-uuid :build/keep-uuid? true} :blocks []} - {:page {:block/title "page2" :build/tags [:user.class/MyClass2]} + {:page {:block/title "page2" :build/tags #{:user.class/MyClass2}} :blocks [{:block/title "hola" :block/uuid internal-block-uuid :build/keep-uuid? true} {:block/title "myclass object 1" - :build/tags [:user.class/MyClass] + :build/tags #{:user.class/MyClass} :block/uuid block-pvalue-uuid :build/keep-uuid? true} (cond-> {:block/title "myclass object 2" - :build/tags [:user.class/MyClass]} + :build/tags #{:user.class/MyClass}} (not exclude-namespaces?) (merge {:block/uuid property-pvalue-uuid :build/keep-uuid? true})) {:block/title "myclass object 3" - :build/tags [:user.class/MyClass] + :build/tags #{:user.class/MyClass} :block/uuid page-pvalue-uuid :build/keep-uuid? true} {:block/title "ref blocks" @@ -849,7 +853,7 @@ :blocks [{:block/title "block with pvalue that has :build/tags" :build/properties {:user.property/default {:build/property-value :block :block/title "tags pvalue" - :build/tags [:user.class/C1]}}} + :build/tags #{:user.class/C1}}}} {:block/title "block with pvalue that has a view" :build/properties {:user.property/default {:build/property-value :block :block/title "view pvalue" @@ -861,7 +865,7 @@ #{"yep" {:build/property-value :block :block/title ":many pvalue" - :build/tags [:user.class/C1]}}}}]} + :build/tags #{:user.class/C1}}}}}]} {:page {:block/title "$$$views2"} :blocks [{:block/title "Unlinked references", :build/properties @@ -967,25 +971,25 @@ :blocks [{:block/title "asset block" :block/uuid asset-uuid :build/keep-uuid? true - :build/tags [:logseq.class/Asset] + :build/tags #{:logseq.class/Asset} :build/properties {:logseq.property.asset/type "pdf" :logseq.property.asset/checksum "abc" :logseq.property.asset/size 42}} {:block/title "annotation block" - :build/tags [:logseq.class/Pdf-annotation] + :build/tags #{:logseq.class/Pdf-annotation} :build/properties {:logseq.property/asset [:block/uuid asset-uuid]}}]} {:page {:block/title "page2"} :blocks [{:block/title "asset image block" :block/uuid asset2-uuid :build/keep-uuid? true - :build/tags [:logseq.class/Asset] + :build/tags #{:logseq.class/Asset} :build/properties {:logseq.property.asset/type "png" :logseq.property.asset/checksum "img-checksum" :logseq.property.asset/width 100 :logseq.property.asset/height 200 :logseq.property.asset/size 300}} {:block/title "annotation with image" - :build/tags [:logseq.class/Pdf-annotation] + :build/tags #{:logseq.class/Pdf-annotation} :build/properties {:logseq.property.pdf/hl-image [:block/uuid asset2-uuid]}}]}]} conn (db-test/create-conn-with-blocks original-data) conn2 (db-test/create-conn) diff --git a/scripts/src/logseq/tasks/db_graph/create_graph_with_schema_org.cljs b/scripts/src/logseq/tasks/db_graph/create_graph_with_schema_org.cljs index baa9129582..710aaedd65 100644 --- a/scripts/src/logseq/tasks/db_graph/create_graph_with_schema_org.cljs +++ b/scripts/src/logseq/tasks/db_graph/create_graph_with_schema_org.cljs @@ -334,23 +334,7 @@ :desc "Verbose mode"}}) (defn- write-export-file [db] - (let [export-map* (sqlite-export/build-export db {:export-type :graph-ontology}) - ;; Modify export to provide stable diff like prepare-export-to-diff - ;; TODO: Remove this when prepare-export-to-diff TODO is done i.e. - ;; when export has stable sort order for these keys - export-map (-> export-map* - (update :classes update-vals - (fn [m] - (cond-> m - (:build/class-extends m) - (update :build/class-extends (comp vec sort)) - (:build/class-properties m) - (update :build/class-properties (comp vec sort))))) - (update :properties update-vals - (fn [m] - (cond-> m - (:build/property-classes m) - (update :build/property-classes (comp vec sort))))))] + (let [export-map (sqlite-export/build-export db {:export-type :graph-ontology})] (fs/writeFileSync "schema.edn" (with-out-str (pprint/pprint export-map))))) From 8b5407a91d59cf7d573dca023ab973d75860ef7a Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Tue, 24 Feb 2026 15:05:18 -0500 Subject: [PATCH 02/24] fix: db and frontend lint --- deps/db/test/logseq/db/sqlite/export_test.cljs | 1 - src/main/frontend/mobile/camera.cljs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/deps/db/test/logseq/db/sqlite/export_test.cljs b/deps/db/test/logseq/db/sqlite/export_test.cljs index 00da636f38..0dd73118b5 100644 --- a/deps/db/test/logseq/db/sqlite/export_test.cljs +++ b/deps/db/test/logseq/db/sqlite/export_test.cljs @@ -10,7 +10,6 @@ [logseq.common.uuid :as common-uuid] [logseq.db :as ldb] [logseq.db.frontend.validate :as db-validate] - [logseq.db.sqlite.build :as sqlite-build] [logseq.db.sqlite.export :as sqlite-export] [logseq.db.test.helper :as db-test] [medley.core :as medley])) diff --git a/src/main/frontend/mobile/camera.cljs b/src/main/frontend/mobile/camera.cljs index af888db146..7cf7aced13 100644 --- a/src/main/frontend/mobile/camera.cljs +++ b/src/main/frontend/mobile/camera.cljs @@ -28,7 +28,7 @@ (string/includes? message "not authorized") (string/includes? message "permission"))))) -(defn- take-or-choose-photo [] +(defn take-or-choose-photo [] (-> (*camera-get-photo* (clj->js {:allowEditing (get-in From 25d0a4468121e7992b0e47a64ca067198427ecab Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Wed, 25 Feb 2026 07:40:29 +0800 Subject: [PATCH 03/24] fix: preserve the original :block/page ref when missing page details --- src/main/frontend/components/query.cljs | 20 ++++++++++++++------ src/main/frontend/db/model.cljs | 13 ++++++++++--- src/test/frontend/components/query_test.cljs | 10 ++++++++++ src/test/frontend/db/model_test.cljs | 10 ++++++++++ 4 files changed, 44 insertions(+), 9 deletions(-) create mode 100644 src/test/frontend/components/query_test.cljs diff --git a/src/main/frontend/components/query.cljs b/src/main/frontend/components/query.cljs index abe860203d..8d5984b664 100644 --- a/src/main/frontend/components/query.cljs +++ b/src/main/frontend/components/query.cljs @@ -20,6 +20,19 @@ (when (seq queries) (boolean (some #(= % title) (map :title queries)))))) +(defn- grouped-by-page-result? + [result group-by-page?] + (let [first-group (first result) + first-page (first first-group) + first-block (first (second first-group))] + (boolean + (and group-by-page? + (seq result) + (coll? first-group) + (or (:block/name first-page) + (:db/id first-page)) + (:block/uuid first-block))))) + (rum/defcs custom-query-inner < rum/static [state {:keys [dsl-query?] :as config} {:keys [query breadcrumb-show?]} {:keys [query-error-atom @@ -30,12 +43,7 @@ (let [{:keys [->hiccup]} config *query-error query-error-atom only-blocks? (:block/uuid (first result)) - blocks-grouped-by-page? (and group-by-page? - (seq result) - (coll? (first result)) - (:block/name (ffirst result)) - (:block/uuid (first (second (first result)))) - true)] + blocks-grouped-by-page? (grouped-by-page-result? result group-by-page?)] (if @*query-error (do (log/error :exception @*query-error) diff --git a/src/main/frontend/db/model.cljs b/src/main/frontend/db/model.cljs index 30c37db919..8fb5bb65d0 100644 --- a/src/main/frontend/db/model.cljs +++ b/src/main/frontend/db/model.cljs @@ -98,11 +98,18 @@ independent of format as format specific heading characters are stripped" (remove nil?)) pages (when (seq pages-ids) (db-utils/pull-many '[:db/id :block/name :block/title :block/journal-day] pages-ids)) - pages-map (reduce (fn [acc p] (assoc acc (:db/id p) p)) {} pages) + pages-map (reduce (fn [acc p] + (if (map? p) + (assoc acc (:db/id p) p) + acc)) + {} + pages) blocks (map (fn [block] - (assoc block :block/page - (get pages-map (:db/id (:block/page block))))) + (assoc block + :block/page + (or (get pages-map (:db/id (:block/page block))) + (:block/page block)))) blocks)] blocks)) diff --git a/src/test/frontend/components/query_test.cljs b/src/test/frontend/components/query_test.cljs new file mode 100644 index 0000000000..29a6c408da --- /dev/null +++ b/src/test/frontend/components/query_test.cljs @@ -0,0 +1,10 @@ +(ns frontend.components.query-test + (:require [cljs.test :refer [deftest is]] + [frontend.components.query :as query])) + +(deftest grouped-by-page-result-detection-supports-partial-page-refs + (let [result [[{:db/id 42} + [{:block/uuid (random-uuid)}]]]] + (is (true? (#'frontend.components.query/grouped-by-page-result? result true)) + "Grouped query results with page refs that only include :db/id should still be recognized") + (is (false? (#'frontend.components.query/grouped-by-page-result? result false))))) diff --git a/src/test/frontend/db/model_test.cljs b/src/test/frontend/db/model_test.cljs index 34ef9642ec..eff403937a 100644 --- a/src/test/frontend/db/model_test.cljs +++ b/src/test/frontend/db/model_test.cljs @@ -117,3 +117,13 @@ (is (= ["child 1" "child 2" "child 3"] (map :block/title (model/get-block-immediate-children test-db (:block/uuid parent))))))) + +(deftest with-pages-preserves-page-ref-when-ui-db-is-partial + (let [page-ref {:db/id 1} + block {:db/id 2 + :block/uuid (random-uuid) + :block/page page-ref}] + (with-redefs [frontend.db.utils/pull-many (fn [& _] nil)] + (is (= page-ref + (:block/page (first (model/with-pages [block])))) + "When page entity details are unavailable locally, keep the original page ref instead of replacing it with nil")))) From afe21733a2c3b7b55e2742d88efe6814624d798e Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Wed, 25 Feb 2026 08:20:23 +0800 Subject: [PATCH 04/24] fix(regression): drag and drop creates a blank asset --- src/main/frontend/handler/editor.cljs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/frontend/handler/editor.cljs b/src/main/frontend/handler/editor.cljs index 9d6aef83a8..cf9f72ae33 100644 --- a/src/main/frontend/handler/editor.cljs +++ b/src/main/frontend/handler/editor.cljs @@ -1398,7 +1398,10 @@ :bottom? true :sibling? (= edit-block target) :replace-empty-target? true})) - (map (fn [b] (db/entity [:block/uuid (:block/uuid b)])) blocks))))) + (p/let [blocks (map (fn [b] (db/entity [:block/uuid (:block/uuid b)])) blocks)] + (when-let [block (some (fn [block] (when (= (:block/uuid block) (:block/uuid edit-block)) block)) blocks)] + (edit-block! block :max)) + blocks))))) (def insert-command! editor-common-handler/insert-command!) From bbec923a6cff44190a6daed320c91e360983c9ee Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Wed, 25 Feb 2026 08:32:24 +0800 Subject: [PATCH 05/24] add e2e tests for afe21733a2 --- clj-e2e/test/logseq/e2e/editor_basic_test.clj | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/clj-e2e/test/logseq/e2e/editor_basic_test.clj b/clj-e2e/test/logseq/e2e/editor_basic_test.clj index 2c1858c3ab..d337233974 100644 --- a/clj-e2e/test/logseq/e2e/editor_basic_test.clj +++ b/clj-e2e/test/logseq/e2e/editor_basic_test.clj @@ -17,6 +17,35 @@ fixtures/new-logseq-page fixtures/validate-graph) +(defn- drag-and-drop-file! + [file-name file-type] + (w/eval-js + (format "(() => { + const container = document.querySelector('#main-content-container'); + if (!container) { + throw new Error('main-content-container not found'); + } + const dataTransfer = new DataTransfer(); + dataTransfer.items.add(new File(['logseq-e2e-drag-drop'], %s, { type: %s })); + container.dispatchEvent(new DragEvent('dragover', { dataTransfer, bubbles: true, cancelable: true })); + container.dispatchEvent(new DragEvent('drop', { dataTransfer, bubbles: true, cancelable: true })); + })();" + (pr-str file-name) + (pr-str file-type)))) + +(deftest drag-and-drop-asset-does-not-create-blank-asset + (testing "dragging and dropping a file should keep non-empty asset title" + (let [asset-title "drag-drop-regression" + file-name (str asset-title ".png")] + (b/new-block "") + (drag-and-drop-file! file-name "image/png") + (w/wait-for ".ls-page-blocks .ls-block .asset-container img") + ;; Exit edit mode to trigger a save; this used to overwrite the new asset with blank content. + (util/exit-edit) + (assert/assert-have-count ".ls-page-blocks .ls-block .asset-container img" 1) + (assert/assert-is-visible + (format ".ls-page-blocks .ls-block .block-title-wrap:text('%s')" asset-title))))) + (deftest toggle-between-page-and-block (testing "Convert block to page and back" (b/new-block "b1") From 8d48e93ccb6907bd7a64dc52961bea3610eb9c5d Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Wed, 25 Feb 2026 09:21:31 +0800 Subject: [PATCH 06/24] fix(sync): malformed remote tx can include a temp entity id without :block/uuid. --- src/main/frontend/worker/sync.cljs | 46 ++++++++- src/test/frontend/components/query_test.cljs | 4 +- src/test/frontend/db/model_test.cljs | 1 + .../frontend/mobile/audio_recorder_test.cljs | 38 -------- src/test/frontend/mobile/camera_test.cljs | 79 --------------- src/test/frontend/worker/db_sync_test.cljs | 95 +++++++++++++++++++ 6 files changed, 143 insertions(+), 120 deletions(-) delete mode 100644 src/test/frontend/mobile/audio_recorder_test.cljs delete mode 100644 src/test/frontend/mobile/camera_test.cljs diff --git a/src/main/frontend/worker/sync.cljs b/src/main/frontend/worker/sync.cljs index 62361d8027..867ae4acf6 100644 --- a/src/main/frontend/worker/sync.cljs +++ b/src/main/frontend/worker/sync.cljs @@ -499,6 +499,49 @@ (= :block/uuid (first x))) (second x))) +(defn- drop-anonymous-temp-entity-datoms + "Drop malformed temp entities from remote txs. + A temp entity must declare one identity attr (:block/uuid or :db/ident) + in its :db/add datoms; otherwise it can create anonymous entities that fail validation." + [db tx-data] + (let [identity-attrs #{:block/uuid :db/ident} + temp-id? (fn [x] + (or (string? x) + (and (integer? x) (neg? x)))) + add-attrs-by-entity + (reduce (fn [acc item] + (if (and (vector? item) + (= :db/add (first item)) + (>= (count item) 4)) + (update acc (second item) (fnil conj #{}) (nth item 2)) + acc)) + {} + tx-data) + dropped-entities + (->> add-attrs-by-entity + (keep (fn [[entity attrs]] + (when (and (temp-id? entity) + (empty? (set/intersection identity-attrs attrs))) + entity))) + set)] + (if (seq dropped-entities) + (let [tx-data' (->> tx-data + (remove (fn [item] + (and (vector? item) + (>= (count item) 2) + (contains? dropped-entities (second item))))) + (remove (fn [item] + (and (vector? item) + (>= (count item) 4) + (keyword? (nth item 2)) + (= :db.type/ref (:db/valueType (d/entity db (nth item 2)))) + (contains? dropped-entities (nth item 3))))))] + (log/warn :db-sync/drop-anonymous-temp-entities + {:count (count dropped-entities) + :entities dropped-entities}) + tx-data') + tx-data))) + (defn- sanitize-tx-data [db tx-data local-deleted-ids] (let [sanitized-tx-data (->> tx-data @@ -1020,7 +1063,8 @@ [repo client tx-data*] (if-let [conn (worker-state/get-datascript-conn repo)] (let [tx-data (->> tx-data* - (db-normalize/remove-retract-entity-ref @conn)) + (db-normalize/remove-retract-entity-ref @conn) + (#(drop-anonymous-temp-entity-datoms @conn %))) local-txs (pending-txs repo) reversed-tx-data (get-reverse-tx-data local-txs) has-local-changes? (seq reversed-tx-data) diff --git a/src/test/frontend/components/query_test.cljs b/src/test/frontend/components/query_test.cljs index 29a6c408da..4a8ea82c64 100644 --- a/src/test/frontend/components/query_test.cljs +++ b/src/test/frontend/components/query_test.cljs @@ -5,6 +5,6 @@ (deftest grouped-by-page-result-detection-supports-partial-page-refs (let [result [[{:db/id 42} [{:block/uuid (random-uuid)}]]]] - (is (true? (#'frontend.components.query/grouped-by-page-result? result true)) + (is (true? (#'query/grouped-by-page-result? result true)) "Grouped query results with page refs that only include :db/id should still be recognized") - (is (false? (#'frontend.components.query/grouped-by-page-result? result false))))) + (is (false? (#'query/grouped-by-page-result? result false))))) diff --git a/src/test/frontend/db/model_test.cljs b/src/test/frontend/db/model_test.cljs index eff403937a..a1d18ea0c7 100644 --- a/src/test/frontend/db/model_test.cljs +++ b/src/test/frontend/db/model_test.cljs @@ -4,6 +4,7 @@ [frontend.db :as db] [frontend.db.conn :as conn] [frontend.db.model :as model] + [frontend.db.utils] [frontend.test.helper :as test-helper :refer [load-test-files]])) (use-fixtures :each {:before test-helper/start-test-db! diff --git a/src/test/frontend/mobile/audio_recorder_test.cljs b/src/test/frontend/mobile/audio_recorder_test.cljs deleted file mode 100644 index 5f528976d7..0000000000 --- a/src/test/frontend/mobile/audio_recorder_test.cljs +++ /dev/null @@ -1,38 +0,0 @@ -(ns frontend.mobile.audio-recorder-test - (:require [cljs.test :refer [is testing]] - [frontend.handler.notification :as notification] - [frontend.mobile.audio-recorder :as audio-recorder] - [frontend.test.helper :include-macros true :refer [deftest-async]] - [logseq.shui.ui :as shui] - [promesa.core :as p])) - -(deftest-async start-recording-shows-warning-when-microphone-permission-denied - (testing "Shows actionable warning and closes recorder popup when mic permission is denied" - (let [warning (atom nil) - popup-hidden? (atom false)] - (p/with-redefs - [notification/show! (fn [content & _] - (reset! warning content)) - shui/popup-hide! (fn [] - (reset! popup-hidden? true))] - (p/let [_ (audio-recorder/start-recording! #js {:startRecording - (fn [] - (p/rejected (js/Error. "Error accessing the microphone: Permission denied")))})] - (is (string? @warning)) - (is (re-find #"Settings" @warning)) - (is (true? @popup-hidden?))))))) - -(deftest-async start-recording-does-not-show-warning-for-non-permission-errors - (testing "Avoids permission warning for unrelated start recording failures" - (let [warning (atom nil) - popup-hidden? (atom false)] - (p/with-redefs - [notification/show! (fn [content & _] - (reset! warning content)) - shui/popup-hide! (fn [] - (reset! popup-hidden? true))] - (p/let [_ (audio-recorder/start-recording! #js {:startRecording - (fn [] - (p/rejected (js/Error. "Error: No microphone device found")))})] - (is (nil? @warning)) - (is (false? @popup-hidden?))))))) diff --git a/src/test/frontend/mobile/camera_test.cljs b/src/test/frontend/mobile/camera_test.cljs deleted file mode 100644 index 3195979ad6..0000000000 --- a/src/test/frontend/mobile/camera_test.cljs +++ /dev/null @@ -1,79 +0,0 @@ -(ns frontend.mobile.camera-test - (:require [cljs.test :refer [is testing]] - [frontend.handler.editor :as editor-handler] - [frontend.handler.notification :as notification] - [frontend.mobile.camera :as mobile-camera] - [frontend.state :as state] - [frontend.test.helper :include-macros true :refer [deftest-async]] - [promesa.core :as p])) - -(deftest-async embed-photo-uses-provided-id - (testing "Uses explicit editor id so capture/upload still works if focused input id is temporarily nil" - (let [upload-id (atom nil)] - (p/with-redefs - [mobile-camera/take-or-choose-photo (fn [] (p/resolved #js {:name "photo.jpeg"})) - state/get-edit-block (constantly {:block/format :markdown}) - editor-handler/upload-asset! (fn [id _files _format _uploading? _drop-or-paste?] - (reset! upload-id id) - (p/resolved nil))] - (p/let [_ (mobile-camera/embed-photo "editor-id")] - (is (= "editor-id" @upload-id))))))) - -(deftest-async embed-photo-skips-upload-when-no-photo - (testing "Doesn't trigger upload pipeline when camera returns nil photo" - (let [upload-called? (atom false)] - (p/with-redefs - [mobile-camera/take-or-choose-photo (fn [] (p/resolved nil)) - state/get-edit-block (constantly {:block/format :markdown}) - editor-handler/upload-asset! (fn [& _] - (reset! upload-called? true) - (p/resolved nil))] - (p/let [_ (mobile-camera/embed-photo "editor-id")] - (is (false? @upload-called?))))))) - -(deftest-async embed-photo-still-allows-photo-picking-when-camera-permission-denied - (testing "Does not pre-block getPhoto by camera permission so users can still pick existing photos" - (let [upload-called? (atom false) - get-photo-called? (atom false) - warning (atom nil)] - (p/with-redefs - [mobile-camera/*camera-get-photo* (fn [_] - (reset! get-photo-called? true) - (p/resolved #js {:base64String "AA=="})) - state/get-edit-block (constantly {:block/format :markdown}) - notification/show! (fn [content & _] - (reset! warning content)) - editor-handler/upload-asset! (fn [& _] - (reset! upload-called? true) - (p/resolved nil))] - (p/let [_ (mobile-camera/embed-photo "editor-id")] - (is (true? @get-photo-called?)) - (is (true? @upload-called?)) - (is (nil? @warning))))))) - -(deftest-async embed-photo-warns-only-for-camera-denied - (testing "Shows camera warning only for camera denied errors, not photo-library denied" - (let [warning (atom nil)] - (p/with-redefs - [mobile-camera/*camera-get-photo* (fn [_] - (p/rejected (js/Error. "User denied access to photos"))) - state/get-edit-block (constantly {:block/format :markdown}) - notification/show! (fn [content & _] - (reset! warning content)) - editor-handler/upload-asset! (fn [& _] (p/resolved nil))] - (p/let [_ (mobile-camera/embed-photo "editor-id")] - (is (nil? @warning))))))) - -(deftest-async embed-photo-warns-when-camera-access-denied - (testing "Shows camera warning when take picture is denied by camera permission" - (let [warning (atom nil)] - (p/with-redefs - [mobile-camera/*camera-get-photo* (fn [_] - (p/rejected (js/Error. "User denied access to camera"))) - state/get-edit-block (constantly {:block/format :markdown}) - notification/show! (fn [content & _] - (reset! warning content)) - editor-handler/upload-asset! (fn [& _] (p/resolved nil))] - (p/let [_ (mobile-camera/embed-photo "editor-id")] - (is (string? @warning)) - (is (re-find #"Settings" @warning))))))) diff --git a/src/test/frontend/worker/db_sync_test.cljs b/src/test/frontend/worker/db_sync_test.cljs index caa43e9661..1ab23e04f9 100644 --- a/src/test/frontend/worker/db_sync_test.cljs +++ b/src/test/frontend/worker/db_sync_test.cljs @@ -10,6 +10,7 @@ [frontend.worker.sync.crypt :as sync-crypt] [logseq.common.config :as common-config] [logseq.db :as ldb] + [logseq.db.frontend.validate :as db-validate] [logseq.db.test.helper :as db-test] [logseq.outliner.core :as outliner-core] [logseq.outliner.op :as outliner-op] @@ -423,6 +424,100 @@ (let [block' (d/entity @conn (:db/id block))] (is (= "test" (:block/title block')))))))))) +(deftest ^:long rebase-does-not-leave-anonymous-created-by-entities-test + (testing "rebase should not leave entities with timestamps/created-by but without identity attrs" + (let [{:keys [conn client-ops-conn parent child1]} (setup-parent-child) + child-id (:db/id child1) + page-id (:db/id (:block/page parent))] + (with-redefs [db-sync/enqueue-local-tx! + (let [orig db-sync/enqueue-local-tx!] + (fn [repo tx-report] + (when-not (:rtc-tx? (:tx-meta tx-report)) + (orig repo tx-report))))] + (with-datascript-conns conn client-ops-conn + (fn [] + ;; Ensure the deleted block has the same created-by shape from production repros. + (d/transact! conn [[:db/add child-id :logseq.property/created-by-ref page-id]]) + (outliner-core/delete-blocks! conn [(d/entity @conn child-id)] {}) + (is (seq (#'db-sync/pending-txs test-repo))) + (#'db-sync/apply-remote-tx! + test-repo + nil + [[:db/add (:db/id parent) :block/title "parent remote"]]) + (let [anonymous-ents (->> (d/datoms @conn :avet :logseq.property/created-by-ref) + (keep (fn [datom] + (let [ent (d/entity @conn (:e datom))] + (when (and (nil? (:block/uuid ent)) + (nil? (:db/ident ent)) + (some? (:block/created-at ent)) + (some? (:block/updated-at ent))) + (select-keys ent [:db/id :block/created-at :block/updated-at :logseq.property/created-by-ref])))))) + validation (db-validate/validate-local-db! @conn)] + (is (empty? anonymous-ents) (str anonymous-ents)) + (is (empty? (map :entity (:errors validation))) + (str (:errors validation)))))))))) + +(deftest ^:long rebase-create-then-delete-does-not-leave-anonymous-entities-test + (testing "create+delete before sync should not leave anonymous entities after rebase" + (let [{:keys [conn client-ops-conn parent]} (setup-parent-child) + page-id (:db/id (:block/page parent))] + (with-redefs [db-sync/enqueue-local-tx! + (let [orig db-sync/enqueue-local-tx!] + (fn [repo tx-report] + (when-not (:rtc-tx? (:tx-meta tx-report)) + (orig repo tx-report))))] + (with-datascript-conns conn client-ops-conn + (fn [] + (outliner-core/insert-blocks! conn [{:block/title "temp-rebase-case"}] parent {:sibling? false}) + (let [temp-block (db-test/find-block-by-content @conn "temp-rebase-case") + temp-id (:db/id temp-block)] + (d/transact! conn [[:db/add temp-id :logseq.property/created-by-ref page-id]]) + (outliner-core/delete-blocks! conn [temp-block] {}) + (is (>= (count (#'db-sync/pending-txs test-repo)) 2)) + (#'db-sync/apply-remote-tx! + test-repo + nil + [[:db/add (:db/id parent) :block/title "parent remote 2"]]) + (let [anonymous-ents (->> (d/datoms @conn :avet :block/created-at) + (keep (fn [datom] + (let [ent (d/entity @conn (:e datom))] + (when (and (nil? (:block/uuid ent)) + (nil? (:db/ident ent)) + (some? (:block/updated-at ent))) + (select-keys ent [:db/id :block/created-at :block/updated-at :logseq.property/created-by-ref])))))) + validation (db-validate/validate-local-db! @conn)] + (is (empty? anonymous-ents) (str anonymous-ents)) + (is (empty? (map :entity (:errors validation))) + (str (:errors validation))))))))))) + +(deftest ^:long malformed-remote-anonymous-entity-tx-is-ignored-test + (testing "remote tx creating anonymous entities should be ignored instead of invalidating db" + (let [{:keys [conn parent]} (setup-parent-child) + created-by-id (:db/id (:block/page parent)) + ts 1771435997392 + malformed-tx [[:db/add "missing-uuid-entity" :block/created-at ts] + [:db/add "missing-uuid-entity" :block/updated-at ts] + [:db/add "missing-uuid-entity" :logseq.property/created-by-ref created-by-id]]] + (with-datascript-conns conn nil + (fn [] + (is (nil? (try + (#'db-sync/apply-remote-tx! test-repo nil malformed-tx) + nil + (catch :default e + e)))) + (let [anonymous-ents (->> (d/datoms @conn :avet :logseq.property/created-by-ref) + (keep (fn [datom] + (let [ent (d/entity @conn (:e datom))] + (when (and (nil? (:block/uuid ent)) + (nil? (:db/ident ent)) + (= ts (:block/created-at ent)) + (= ts (:block/updated-at ent))) + (select-keys ent [:db/id :block/created-at :block/updated-at :logseq.property/created-by-ref])))))) + validation (db-validate/validate-local-db! @conn)] + (is (empty? anonymous-ents) (str anonymous-ents)) + (is (empty? (map :entity (:errors validation))) + (str (:errors validation))))))))) + (deftest ^:long offload-large-title-test (testing "large titles are offloaded to object storage with placeholder" (async done From b18056c91043fe26051d5785ff7f04ceb962fd9a Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Wed, 25 Feb 2026 14:50:59 -0500 Subject: [PATCH 07/24] enhance: export-edn exports property history also ensure that it is idempotent --- deps/db/src/logseq/db/sqlite/export.cljs | 46 +++++++++++++- .../db/test/logseq/db/sqlite/export_test.cljs | 63 +++++++++++++++++++ 2 files changed, 106 insertions(+), 3 deletions(-) diff --git a/deps/db/src/logseq/db/sqlite/export.cljs b/deps/db/src/logseq/db/sqlite/export.cljs index 404f0da051..d9c0eb311e 100644 --- a/deps/db/src/logseq/db/sqlite/export.cljs +++ b/deps/db/src/logseq/db/sqlite/export.cljs @@ -766,6 +766,33 @@ (remove #(= :logseq.kv/schema-version (:db/ident %))) vec)) +(defn- build-property-history + "Builds property history. Always include timestamps regardless of :include-timestamps? because + timestamps are a necessary part of history" + [db] + (->> (d/q '[:find [(pull ?b [:block/uuid + :block/created-at + {:logseq.property.history/block [:block/uuid]} + {:logseq.property.history/property [:db/ident]} + {:logseq.property.history/ref-value [:db/ident :block/uuid]} + :logseq.property.history/scalar-value]) ...] + :where [?b :logseq.property.history/block]] db) + (map (fn [history] + (cond-> (-> history + (update :logseq.property.history/block + (fn [m] [:block/uuid (:block/uuid m)])) + (update :logseq.property.history/property :db/ident) + (update :logseq.property.history/ref-value + (fn [m] + (if (:db/ident m) + (:db/ident m) + [:block/uuid (:block/uuid m)])))) + (nil? (:logseq.property.history/ref-value history)) + (dissoc :logseq.property.history/ref-value) + (not (contains? history :logseq.property.history/scalar-value)) + (dissoc :logseq.property.history/scalar-value)))) + set)) + (defn remove-uuids-if-not-ref [export-map all-ref-uuids] (let [remove-uuid-if-not-ref (partial remove-uuid-if-not-ref-given-uuids all-ref-uuids)] (-> export-map @@ -825,7 +852,16 @@ graph-export (if (seq (:exclude-namespaces options)) (assoc graph-export* ::auto-include-namespaces (:exclude-namespaces options)) graph-export*) - all-ref-uuids (set/union content-ref-uuids ontology-pvalue-uuids (:pvalue-uuids pages-export)) + property-history (build-property-history db) + property-history-ref-uuids + (->> property-history + (mapcat (fn [history] + (keep #(when (vector? %) (second %)) + [(:logseq.property.history/block history) + (:logseq.property.history/ref-value history)]))) + set) + all-ref-uuids (set/union content-ref-uuids ontology-pvalue-uuids (:pvalue-uuids pages-export) + property-history-ref-uuids) files (when-not exclude-files? (build-graph-files db options)) kv-values (build-kv-values db) ;; Remove all non-ref uuids after all nodes are built. @@ -837,7 +873,9 @@ (not exclude-files?) (assoc ::graph-files files) true - (assoc ::kv-values kv-values)))) + (assoc ::kv-values kv-values) + true + (assoc ::property-history property-history)))) (defn- find-undefined-classes-and-properties [{:keys [classes properties pages-and-blocks]}] (let [referenced-classes @@ -1074,6 +1112,7 @@ * ::block - Block map for a :block export * ::graph-files - Vec of files for a :graph export * ::kv-values - Vec of :kv/value maps for a :graph export + * ::property-history - Set of property history blocks for a :graph export * ::auto-include-namespaces - A set of parent namespaces to include from properties and classes for a :graph export. See :exclude-namespaces in build-graph-export for a similar option * ::import-options - A map of options that alters importing behavior. Has the following keys: @@ -1101,7 +1140,8 @@ (if (= :graph (::export-type export-map'')) (-> (sqlite-build/build-blocks-tx (remove-namespaced-keys export-map'')) (assoc :misc-tx (vec (concat (::graph-files export-map'') - (::kv-values export-map''))))) + (::kv-values export-map'') + (::property-history export-map''))))) (sqlite-build/build-blocks-tx (remove-namespaced-keys export-map'')))))) (defn create-conn diff --git a/deps/db/test/logseq/db/sqlite/export_test.cljs b/deps/db/test/logseq/db/sqlite/export_test.cljs index 0dd73118b5..7cd0198b7e 100644 --- a/deps/db/test/logseq/db/sqlite/export_test.cljs +++ b/deps/db/test/logseq/db/sqlite/export_test.cljs @@ -839,6 +839,69 @@ (sqlite-export/diff-exports export-map export-map2)) "No diff between original export and export after importing into a new graph"))) +(deftest ^:long import-graph-preserves-property-history + (let [now (common-util/time-ms) + original-data + {:properties {:user.property/num {:logseq.property/type :number} + :user.property/node {:logseq.property/type :node + :db/cardinality :db.cardinality/many}} + :pages-and-blocks [{:page {:block/title "page1"} + :blocks [{:block/title "num block" + :build/properties {:user.property/num 44}} + {:block/title "status block" + :build/properties {:logseq.property/status :logseq.property/status.doing}} + {:block/title "node block"} + {:block/title "object 1"} + {:block/title "object 2"}]}]} + conn (db-test/create-conn-with-import-map original-data) + num-block (db-test/find-block-by-content @conn "num block") + status-block (db-test/find-block-by-content @conn "status block") + node-block (db-test/find-block-by-content @conn "node block") + original-property-history + [{:block/uuid (random-uuid) + :block/created-at now + :logseq.property.history/block [:block/uuid (:block/uuid num-block)] + :logseq.property.history/property :user.property/num + :logseq.property.history/scalar-value 42} + {:block/uuid (random-uuid) + :block/created-at (+ now 1000) + :logseq.property.history/block [:block/uuid (:block/uuid num-block)] + :logseq.property.history/property :user.property/num + :logseq.property.history/scalar-value 44} + {:block/uuid (random-uuid) + :block/created-at now + :logseq.property.history/block [:block/uuid (:block/uuid node-block)] + :logseq.property.history/property :user.property/node + :logseq.property.history/ref-value [:block/uuid (:block/uuid (db-test/find-block-by-content @conn "object 1"))]} + {:block/uuid (random-uuid) + :block/created-at (+ now 1000) + :logseq.property.history/block [:block/uuid (:block/uuid node-block)] + :logseq.property.history/property :user.property/node + :logseq.property.history/ref-value [:block/uuid (:block/uuid (db-test/find-block-by-content @conn "object 2"))]} + {:block/uuid (random-uuid) + :block/created-at now + :logseq.property.history/block [:block/uuid (:block/uuid status-block)] + :logseq.property.history/property :logseq.property/status + :logseq.property.history/ref-value :logseq.property/status.todo} + {:block/uuid (random-uuid) + :block/created-at (+ now 1000) + :logseq.property.history/block [:block/uuid (:block/uuid status-block)] + :logseq.property.history/property :logseq.property/status + :logseq.property.history/ref-value :logseq.property/status.doing}] + _ (d/transact! conn original-property-history) + export-map (sqlite-export/build-export @conn {:export-type :graph}) + valid-result (sqlite-export/validate-export export-map) + _ (assert (not (:error valid-result)) "No error when importing export-map into new graph") + _ (validate-db (:db valid-result)) + export-map2 (sqlite-export/build-export (:db valid-result) {:export-type :graph}) + property-history (::sqlite-export/property-history export-map2)] + (is (= nil + (sqlite-export/diff-exports export-map export-map2)) + "No diff between original export and export after importing into a new graph") + ;; (cljs.pprint/pprint (clojure.data/diff original-property-history property-history)) + (is (= (set original-property-history) property-history) + "Original property history equals exported property history"))) + (deftest import-graph-with-different-property-value-cases (let [pvalue-uuid1 (random-uuid) original-data From ae9a37c8899750535cdbc9126af10af7a94e8088 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 26 Feb 2026 10:32:25 +0800 Subject: [PATCH 08/24] fix: sync missing :block/title --- .../test/logseq/db_sync/normalize_test.cljs | 45 +++++++++++ .../test/logseq/db_sync/test_runner.cljs | 1 + deps/db/src/logseq/db/common/normalize.cljs | 74 +++++++++++-------- 3 files changed, 89 insertions(+), 31 deletions(-) create mode 100644 deps/db-sync/test/logseq/db_sync/normalize_test.cljs diff --git a/deps/db-sync/test/logseq/db_sync/normalize_test.cljs b/deps/db-sync/test/logseq/db_sync/normalize_test.cljs new file mode 100644 index 0000000000..c2110c4a2f --- /dev/null +++ b/deps/db-sync/test/logseq/db_sync/normalize_test.cljs @@ -0,0 +1,45 @@ +(ns logseq.db-sync.normalize-test + (:require [cljs.test :refer [deftest is testing]] + [datascript.core :as d] + [logseq.db.common.normalize :as db-normalize] + [logseq.db.test.helper :as db-test])) + +(defn- new-conn [] + (db-test/create-conn)) + +(defn- create-page! + [conn title] + (let [page-uuid (random-uuid)] + (d/transact! conn [{:block/uuid page-uuid + :block/name title + :block/title title}]) + page-uuid)) + +(defn- op-e-a-v + [datom] + (subvec (vec datom) 0 4)) + +(deftest normalize-tx-data-keeps-title-retract-without-replacement-test + (let [conn (new-conn) + page-uuid (create-page! conn "Page") + tx-report (d/transact! conn [[:db/retract [:block/uuid page-uuid] :block/title "Page"]]) + normalized (db-normalize/normalize-tx-data (:db-after tx-report) + (:db-before tx-report) + (:tx-data tx-report)) + tx-data (mapv op-e-a-v normalized)] + (testing "keeps :block/title retract when no replacement title exists in same tx" + (is (= [[:db/retract [:block/uuid page-uuid] :block/title "Page"]] + tx-data))))) + +(deftest normalize-tx-data-drops-title-retract-when-replaced-test + (let [conn (new-conn) + page-uuid (create-page! conn "Page") + tx-report (d/transact! conn [{:block/uuid page-uuid + :block/title "Page 2"}]) + normalized (db-normalize/normalize-tx-data (:db-after tx-report) + (:db-before tx-report) + (:tx-data tx-report)) + tx-data (mapv op-e-a-v normalized)] + (testing "drops old :block/title retract and keeps new add during title update" + (is (some #(= [:db/add [:block/uuid page-uuid] :block/title "Page 2"] %) tx-data)) + (is (not-any? #(= [:db/retract [:block/uuid page-uuid] :block/title "Page"] %) tx-data))))) diff --git a/deps/db-sync/test/logseq/db_sync/test_runner.cljs b/deps/db-sync/test/logseq/db_sync/test_runner.cljs index eb503e1fbb..7654ed1072 100644 --- a/deps/db-sync/test/logseq/db_sync/test_runner.cljs +++ b/deps/db-sync/test/logseq/db_sync/test_runner.cljs @@ -5,6 +5,7 @@ [logseq.db-sync.node-adapter-test] [logseq.db-sync.node-config-test] [logseq.db-sync.node-server-test] + [logseq.db-sync.normalize-test] [logseq.db-sync.platform-test] [logseq.db-sync.worker-auth-test] [logseq.db-sync.worker-handler-assets-test] diff --git a/deps/db/src/logseq/db/common/normalize.cljs b/deps/db/src/logseq/db/common/normalize.cljs index 3b35c2de7a..68b211706f 100644 --- a/deps/db/src/logseq/db/common/normalize.cljs +++ b/deps/db/src/logseq/db/common/normalize.cljs @@ -92,34 +92,46 @@ (defn normalize-tx-data [db-after db-before tx-data] - (->> tx-data - remove-conflict-datoms - (replace-attr-retract-with-retract-entity db-after) - sort-datoms - (keep - (fn [d] - (if (= (count d) 5) - (let [[e a v t added] d - retract? (not added)] - (when-not (and retract? - (contains? #{:block/created-at :block/updated-at :block/title} a)) - (let [e' (if retract? - (eid->lookup db-before e) - (or (eid->lookup db-before e) - (eid->tempid db-after e))) - v' (if (and (integer? v) - (pos? v) - (or (= :db.type/ref (:db/valueType (d/entity db-after a))) - (= :db.type/ref (:db/valueType (d/entity db-before a))))) - (if retract? - (eid->lookup db-before v) - (or (eid->lookup db-before v) - (eid->tempid db-after v))) - v)] - (when (and (some? e') (some? v')) - (if added - [:db/add e' a v' t] - [:db/retract e' a v' t]))))) - d))) - (remove-retract-entity-ref db-after) - distinct)) + (let [title-updated-entities + (->> tx-data + (keep (fn [d] + (when (= (count d) 5) + (let [[e a _v _t added] d] + (when (and added (= :block/title a)) + e))))) + set)] + (->> tx-data + remove-conflict-datoms + (replace-attr-retract-with-retract-entity db-after) + sort-datoms + (keep + (fn [d] + (if (= (count d) 5) + (let [[e a v t added] d + retract? (not added) + drop-retract? + (and retract? + (or (contains? #{:block/created-at :block/updated-at} a) + (and (= :block/title a) + (contains? title-updated-entities e))))] + (when-not drop-retract? + (let [e' (if retract? + (eid->lookup db-before e) + (or (eid->lookup db-before e) + (eid->tempid db-after e))) + v' (if (and (integer? v) + (pos? v) + (or (= :db.type/ref (:db/valueType (d/entity db-after a))) + (= :db.type/ref (:db/valueType (d/entity db-before a))))) + (if retract? + (eid->lookup db-before v) + (or (eid->lookup db-before v) + (eid->tempid db-after v))) + v)] + (when (and (some? e') (some? v')) + (if added + [:db/add e' a v' t] + [:db/retract e' a v' t]))))) + d))) + (remove-retract-entity-ref db-after) + distinct))) From 7e565aeeeb3fc2f8b8de30129a5fdaff78260a0c Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 26 Feb 2026 11:35:25 +0800 Subject: [PATCH 09/24] fix: reject stale pull resp --- src/main/frontend/worker/sync.cljs | 2 +- src/test/frontend/worker/db_sync_test.cljs | 61 ++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/src/main/frontend/worker/sync.cljs b/src/main/frontend/worker/sync.cljs index 867ae4acf6..a0d9a579e1 100644 --- a/src/main/frontend/worker/sync.cljs +++ b/src/main/frontend/worker/sync.cljs @@ -1170,7 +1170,7 @@ (flush-pending! repo client)) ;; Download response ;; Merge batch txs to one tx, does it really work? We'll see - "pull/ok" (when-not (= local-tx remote-tx) + "pull/ok" (when (> remote-tx local-tx) (let [txs (:txs message) _ (require-non-negative remote-tx {:repo repo :type "pull/ok"}) _ (require-seq txs {:repo repo :type "pull/ok" :field :txs}) diff --git a/src/test/frontend/worker/db_sync_test.cljs b/src/test/frontend/worker/db_sync_test.cljs index 1ab23e04f9..1aa3bc25dd 100644 --- a/src/test/frontend/worker/db_sync_test.cljs +++ b/src/test/frontend/worker/db_sync_test.cljs @@ -11,6 +11,7 @@ [logseq.common.config :as common-config] [logseq.db :as ldb] [logseq.db.frontend.validate :as db-validate] + [logseq.db.sqlite.util :as sqlite-util] [logseq.db.test.helper :as db-test] [logseq.outliner.core :as outliner-core] [logseq.outliner.op :as outliner-op] @@ -115,6 +116,66 @@ @(:online-users client))) (is (= 1 (count @broadcasts)))))) +(deftest pull-ok-with-older-remote-tx-is-ignored-test + (testing "pull/ok with remote tx behind local tx does not apply stale tx data" + (let [{:keys [conn client-ops-conn parent]} (setup-parent-child) + parent-id (:db/id parent) + stale-tx (sqlite-util/write-transit-str [[:db/add parent-id :block/title "stale-title"]]) + raw-message (js/JSON.stringify + (clj->js {:type "pull/ok" + :t 4 + :txs [{:t 4 :tx stale-tx}]})) + latest-prev @db-sync/*repo->latest-remote-tx + client {:repo test-repo + :graph-id "graph-1" + :inflight (atom []) + :online-users (atom []) + :ws-state (atom :open)}] + (with-datascript-conns conn client-ops-conn + (fn [] + (reset! db-sync/*repo->latest-remote-tx {}) + (try + (client-op/update-local-tx test-repo 5) + (#'db-sync/handle-message! test-repo client raw-message) + (let [parent' (d/entity @conn parent-id)] + (is (= "parent" (:block/title parent'))) + (is (= 5 (client-op/get-local-tx test-repo)))) + (finally + (reset! db-sync/*repo->latest-remote-tx latest-prev)))))))) + +(deftest pull-ok-out-of-order-stale-response-is-ignored-test + (testing "late stale pull/ok should not overwrite a newer already-applied tx" + (async done + (let [{:keys [conn client-ops-conn parent]} (setup-parent-child) + parent-id (:db/id parent) + new-tx (sqlite-util/write-transit-str [[:db/add parent-id :block/title "remote-new-title"]]) + stale-tx (sqlite-util/write-transit-str [[:db/add parent-id :block/title "stale-title"]]) + raw-new (js/JSON.stringify + (clj->js {:type "pull/ok" + :t 2 + :txs [{:t 2 :tx new-tx}]})) + raw-stale (js/JSON.stringify + (clj->js {:type "pull/ok" + :t 1 + :txs [{:t 1 :tx stale-tx}]})) + latest-prev @db-sync/*repo->latest-remote-tx + client {:repo test-repo + :graph-id "graph-1" + :inflight (atom []) + :online-users (atom []) + :ws-state (atom :open)}] + (reset! db-sync/*repo->latest-remote-tx {}) + (with-datascript-conns conn client-ops-conn + (fn [] + (-> (p/let [_ (#'db-sync/handle-message! test-repo client raw-new) + _ (#'db-sync/handle-message! test-repo client raw-stale) + parent' (d/entity @conn parent-id)] + (is (= "remote-new-title" (:block/title parent'))) + (is (= 2 (client-op/get-local-tx test-repo)))) + (p/finally (fn [] + (reset! db-sync/*repo->latest-remote-tx latest-prev) + (done)))))))))) + (deftest reaction-add-enqueues-pending-sync-tx-test (testing "adding a reaction should enqueue tx for db-sync" (let [{:keys [conn client-ops-conn parent]} (setup-parent-child)] From a1cbcf8aaa3981fd04f65adf889cefdec551055e Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 26 Feb 2026 11:51:56 +0800 Subject: [PATCH 10/24] fix: mobile sync indicator latency --- src/main/frontend/components/dnd.cljs | 10 +-- src/main/mobile/components/header.cljs | 94 ++++++++++++++++++-------- 2 files changed, 70 insertions(+), 34 deletions(-) diff --git a/src/main/frontend/components/dnd.cljs b/src/main/frontend/components/dnd.cljs index 820076a9fd..b8ffbc23b1 100644 --- a/src/main/frontend/components/dnd.cljs +++ b/src/main/frontend/components/dnd.cljs @@ -51,12 +51,12 @@ sensors (useSensors (useSensor MouseSensor (bean/->js {:activationConstraint {:distance 8}}))) dnd-opts {:sensors sensors :collisionDetection closestCenter - :onDragStart (fn [event] + :onDragStart (fn [^js event] (when-not (state/editing?) - (set-active-id (.-id (.-active event))))) - :onDragEnd (fn [event] - (let [active-id (.-id (.-active event)) - over-id (.-id (.-over event))] + (set-active-id (.-id ^js (.-active event))))) + :onDragEnd (fn [^js event] + (let [active-id (.-id ^js (.-active event)) + over-id (.-id ^js (.-over event))] (when active-id (when-not (= active-id over-id) (let [old-index (.indexOf ids active-id) diff --git a/src/main/mobile/components/header.cljs b/src/main/mobile/components/header.cljs index ba0f045337..6075295a1c 100644 --- a/src/main/mobile/components/header.cljs +++ b/src/main/mobile/components/header.cljs @@ -152,13 +152,10 @@ (reset! native-top-bar-listener? true))) (defn- configure-native-top-bar! - [repo {:keys [tab title route-name route-view sync-color favorited?]}] + [repo {:keys [tab title route-name route-view sync-color favorited? show-sync?]}] (when (and (mobile-util/native-platform?) mobile-util/native-top-bar) (let [hidden? (and (mobile-util/native-ios?) (= tab "search")) - rtc-indicator? (and repo - (ldb/get-graph-rtc-uuid (db/get-db)) - (user-handler/logged-in?)) base (cond-> {:hidden hidden?} (not (mobile-util/native-ipad?)) @@ -179,7 +176,7 @@ (cond-> [] (nil? route-view) (conj {:id "home-setting" :systemIcon "ellipsis"}) - (and rtc-indicator? (not page?)) + (and show-sync? (not page?)) (conj {:id "sync" :systemIcon "circle.fill" :color sync-color :size "small"})) @@ -208,13 +205,27 @@ "Select a Graph") route-name (get-in route-match [:data :name]) route-view (get-in route-match [:data :view]) + route-id (get-in route-match [:parameters :path :name]) + page-route? (= route-name :page) [*configure-top-bar-f _] (hooks/use-state (atom nil)) detail-info (hooks/use-flow-state (m/watch rtc-indicator/*detail-info)) _ (hooks/use-flow-state flows/current-login-user-flow) online? (hooks/use-flow-state flows/network-online-event-flow) rtc-state (:rtc-state detail-info) + graph-uuid (or (:graph-uuid detail-info) + (ldb/get-graph-rtc-uuid (db/get-db))) + show-sync? (and current-repo graph-uuid (user-handler/logged-in?)) unpushed-block-update-count (:pending-local-ops detail-info) pending-asset-ops (:pending-asset-ops detail-info) + fallback-title (cond + (= tab "home") + short-repo-name + + (= tab "search") + "Search" + + :else + (string/capitalize tab)) sync-color (if (and online? (= :open rtc-state) (zero? unpushed-block-update-count) @@ -228,33 +239,58 @@ (when (and (mobile-util/native-platform?) mobile-util/native-top-bar) (register-native-top-bar-events! *configure-top-bar-f) - (p/let [block (when (= route-name :page) - (let [id (get-in route-match [:parameters :path :name])] - (when (common-util/uuid-string? id) - (db-async/ (db-async/])) From 560ce3ca4f921949aa79f92bb5313787d50db7dd Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 26 Feb 2026 12:12:54 +0800 Subject: [PATCH 11/24] enhance: sync startup latency 1. Removes startup dependency on remote-graphs fetch for initial sync start. 2. Removes unnecessary restart churn. 3. Shaves trigger scheduling delay. --- .../frontend/handler/db_based/rtc_flows.cljs | 2 +- src/main/frontend/handler/db_based/sync.cljs | 23 +++++++++++- src/main/frontend/handler/user.cljs | 37 +++++++++++++++++-- src/main/frontend/worker/sync.cljs | 34 +++++++++++++---- src/main/mobile/components/header.cljs | 4 +- 5 files changed, 82 insertions(+), 18 deletions(-) diff --git a/src/main/frontend/handler/db_based/rtc_flows.cljs b/src/main/frontend/handler/db_based/rtc_flows.cljs index 852cc7685d..56e5e30f44 100644 --- a/src/main/frontend/handler/db_based/rtc_flows.cljs +++ b/src/main/frontend/handler/db_based/rtc_flows.cljs @@ -153,4 +153,4 @@ conditions: (apply c.m/mix) (m/latest vector flows/current-login-user-flow) (m/eduction (keep (fn [[current-user trigger-event]] (when current-user trigger-event)))) - (c.m/debounce 200))) + (c.m/debounce 50))) diff --git a/src/main/frontend/handler/db_based/sync.cljs b/src/main/frontend/handler/db_based/sync.cljs index 63b5331f72..4dc4873794 100644 --- a/src/main/frontend/handler/db_based/sync.cljs +++ b/src/main/frontend/handler/db_based/sync.cljs @@ -185,6 +185,23 @@ [repo] (some #(= repo (:url %)) (state/get-rtc-graphs))) +(defn- graph-has-local-rtc-id? + [repo] + (boolean (some-> (db/get-db repo) + ldb/get-graph-rtc-uuid))) + +(defn- remote-graphs-unknown? + [] + (not= false (:rtc/loading-graphs? @state/state))) + +(defn- should-start-rtc? + [repo] + (or (graph-in-remote-list? repo) + ;; During startup, remote graph list might not be fetched yet. + ;; If local DB already has graph UUID, start optimistically to reduce cold-start latency. + (and (remote-graphs-unknown?) + (graph-has-local-rtc-id? repo)))) + (defn- normalize-graph-e2ee? [graph-e2ee?] (if (nil? graph-e2ee?) @@ -193,12 +210,14 @@ (defn clj :keywordize-keys true) (update :cognito:username decode-username))) +(defn- parse-jwt-safe + [jwt] + (try + (parse-jwt jwt) + (catch :default _ + nil))) + (defn- expired? [parsed-jwt] (some-> (* 1000 (:exp parsed-jwt)) @@ -191,12 +198,34 @@ "Refresh id-token&access-token, pull latest repos, returns nil when tokens are not available." [] (println "restore-tokens-from-localstorage") - (let [refresh-token (js/localStorage.getItem "refresh-token")] - (when refresh-token + (let [refresh-token (js/localStorage.getItem "refresh-token") + id-token (js/localStorage.getItem "id-token") + access-token (js/localStorage.getItem "access-token") + restored-from-cache? + (boolean + (when (and (string? refresh-token) (not (string/blank? refresh-token)) + (string? id-token) (not (string/blank? id-token)) + (string? access-token) (not (string/blank? access-token))) + (when-let [parsed (parse-jwt-safe id-token)] + (when-not (expired? parsed) + (set-tokens! id-token access-token refresh-token) + true)))) + should-refresh? + (and (string? refresh-token) + (not (string/blank? refresh-token)) + (or (not restored-from-cache?) + (some-> (state/get-auth-id-token) + parse-jwt-safe + almost-expired?)))] + (when restored-from-cache? + ;; Publish login event immediately so sync can start without waiting token refresh request. + (state/pub-event! [:user/fetch-info-and-graphs])) + (when should-refresh? (go ( (:ws-state client) deref) + ws-ready-state (when ws (ready-state ws))] + (or (= :open ws-state) + (contains? #{0 1} ws-ready-state))))) + (defn start! [repo] - (p/do! - (stop!) - (let [base (ws-base-url) - graph-id (get-graph-id repo)] - (if (and (string? base) (seq base) (seq graph-id)) + (let [base (ws-base-url) + graph-id (get-graph-id repo) + current @worker-state/*db-sync-client] + (cond + (not (and (string? base) (seq base) (seq graph-id))) + (do + (log/info :db-sync/start-skipped {:repo repo :graph-id graph-id :base base}) + (p/resolved nil)) + + (active-client-for? current repo graph-id) + (do + (broadcast-rtc-state! current) + (p/resolved nil)) + + :else + (p/do! + (stop!) (let [client (ensure-client-state! repo) url (format-ws-url base graph-id) _ (ensure-client-graph-uuid! repo graph-id) connected (assoc client :graph-id graph-id) connected (connect! repo connected url)] (reset! worker-state/*db-sync-client connected) - (p/resolved nil)) - (do - (log/info :db-sync/start-skipped {:repo repo :graph-id graph-id :base base}) (p/resolved nil)))))) (defn enqueue-local-tx! diff --git a/src/main/mobile/components/header.cljs b/src/main/mobile/components/header.cljs index 6075295a1c..f4fb8cb141 100644 --- a/src/main/mobile/components/header.cljs +++ b/src/main/mobile/components/header.cljs @@ -152,7 +152,7 @@ (reset! native-top-bar-listener? true))) (defn- configure-native-top-bar! - [repo {:keys [tab title route-name route-view sync-color favorited? show-sync?]}] + [{:keys [tab title route-name route-view sync-color favorited? show-sync?]}] (when (and (mobile-util/native-platform?) mobile-util/native-top-bar) (let [hidden? (and (mobile-util/native-ios?) (= tab "search")) @@ -247,7 +247,6 @@ title (or (:block/title block) fallback-title) f (fn [favorited?] (configure-native-top-bar! - current-repo {:tab tab :title title :route-name route-name @@ -277,7 +276,6 @@ title (:block/title block) f (fn [favorited?] (configure-native-top-bar! - current-repo {:tab tab :title title :route-name route-name From 2e78d8be4007463563dbe09674256970a83aa364 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 26 Feb 2026 13:09:00 +0800 Subject: [PATCH 12/24] fix: ci tests --- src/main/frontend/worker/sync.cljs | 4 ++-- src/test/frontend/worker/db_sync_test.cljs | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/frontend/worker/sync.cljs b/src/main/frontend/worker/sync.cljs index 8a9470c66b..e83ff0a432 100644 --- a/src/main/frontend/worker/sync.cljs +++ b/src/main/frontend/worker/sync.cljs @@ -691,8 +691,8 @@ (p/recur (rest remaining) (conj acc item)))))))) (defn- rehydrate-large-titles! - [repo {:keys [graph-id download-fn aes-key tx-data]}] - (when-let [conn (worker-state/get-datascript-conn repo)] + [repo {:keys [graph-id download-fn aes-key tx-data conn]}] + (when-let [conn (or conn (worker-state/get-datascript-conn repo))] (let [download-fn (or download-fn download-large-title!) graph-id (or graph-id (get-graph-id repo)) items (if (seq tx-data) diff --git a/src/test/frontend/worker/db_sync_test.cljs b/src/test/frontend/worker/db_sync_test.cljs index 1aa3bc25dd..4d8c98254b 100644 --- a/src/test/frontend/worker/db_sync_test.cljs +++ b/src/test/frontend/worker/db_sync_test.cljs @@ -710,6 +710,7 @@ (-> (p/let [result (#'db-sync/rehydrate-large-titles! test-repo {:tx-data tx-data + :conn conn :graph-id "graph-1" :download-fn download-fn :aes-key nil}) From cdc1bc1d3290077985ee986d396590485f862a23 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 26 Feb 2026 13:22:32 +0800 Subject: [PATCH 13/24] fix: get-bidirectional-properties perf Root cause: get-bidirectional-properties was recomputing bidirectional-property-attr? for every [e a] match, repeatedly calling d/entity for the same property attr keyword. That made cost scale with datom count, not unique properties. Fix: Added per-call memoization for property-attr bidirectional checks using a local volatile! cache, so each attr is resolved once per invocation. --- deps/db/src/logseq/db.cljs | 19 ++++++++--- deps/db/test/logseq/db_test.cljs | 55 ++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 4 deletions(-) diff --git a/deps/db/src/logseq/db.cljs b/deps/db/src/logseq/db.cljs index 1def2919b2..bd1ce14382 100644 --- a/deps/db/src/logseq/db.cljs +++ b/deps/db/src/logseq/db.cljs @@ -670,9 +670,11 @@ (d/q '[:find ?e ?a :in $ ?v :where - [?e ?a ?v] + [?c :logseq.property.class/enable-bidirectional? ?c-enable?] + [(true? ?c-enable?)] + [?ea :logseq.property/classes ?c] [?ea :db/ident ?a] - [?ea :logseq.property/classes]] + [?e ?a ?v]] db v)) @@ -687,10 +689,19 @@ (fn [acc class-id entity] (if class-id (update acc class-id (fnil conj #{}) entity) - acc))] + acc)) + *attr->bidirectional? (volatile! {}) + bidirectional-property-attr-cached? + (fn [attr] + (let [cache @*attr->bidirectional?] + (if (contains? cache attr) + (get cache attr) + (let [result (bidirectional-property-attr? db attr)] + (vswap! *attr->bidirectional? assoc attr result) + result))))] (->> (get-ea-by-v db target-id) (keep (fn [[e a]] - (when (bidirectional-property-attr? db a) + (when (bidirectional-property-attr-cached? a) (when-let [entity (d/entity db e)] (when (and (not= (:db/id entity) target-id) (not (entity-util/class? entity)) diff --git a/deps/db/test/logseq/db_test.cljs b/deps/db/test/logseq/db_test.cljs index 026edfb4fe..0287e66df0 100644 --- a/deps/db/test/logseq/db_test.cljs +++ b/deps/db/test/logseq/db_test.cljs @@ -148,3 +148,58 @@ (is (= "People" (:title (first results)))) (is (= ["Alice"] (map :block/title (:entities (first results)))))))) + +(defn- bidirectional-perf-conn + [n property-titles] + (let [target-page {:page {:block/title "Target"}} + properties (into {} + (map (fn [property-title] + [property-title {:logseq.property/type :node + :build/property-classes [:Person]}])) + property-titles) + person-properties (into {} + (map (fn [property-title] + [property-title [:build/page {:block/title "Target"}]])) + property-titles) + pages (vec (concat [target-page] + (map (fn [i] + {:page {:block/title (str "Person " i) + :build/tags [:Person] + :build/properties person-properties}}) + (range n))))] + (db-test/create-conn-with-blocks + {:properties properties + :classes {:Person {:build/properties {:logseq.property.class/enable-bidirectional? true}}} + :pages-and-blocks pages}))) + +(deftest ^:long get-bidirectional-properties-performance-single-property + (testing "attribute lookups scale with unique properties, not entities" + (let [conn (bidirectional-perf-conn 400 [:friend]) + target-id (:db/id (db-test/find-page-by-title @conn "Target")) + original-entity d/entity + attr-lookups (atom 0) + results (with-redefs [d/entity (fn [db eid] + (when (keyword? eid) + (swap! attr-lookups inc)) + (original-entity db eid))] + (ldb/get-bidirectional-properties @conn target-id))] + (is (= 1 (count results))) + (is (= 400 (count (:entities (first results))))) + (is (<= @attr-lookups 8) + (str "expected bounded attr lookups, got " @attr-lookups))))) + +(deftest ^:long get-bidirectional-properties-performance-multi-property + (testing "attribute lookups stay bounded with multiple matching properties" + (let [conn (bidirectional-perf-conn 300 [:friend :colleague]) + target-id (:db/id (db-test/find-page-by-title @conn "Target")) + original-entity d/entity + attr-lookups (atom 0) + results (with-redefs [d/entity (fn [db eid] + (when (keyword? eid) + (swap! attr-lookups inc)) + (original-entity db eid))] + (ldb/get-bidirectional-properties @conn target-id))] + (is (= 1 (count results))) + (is (= 300 (count (:entities (first results))))) + (is (<= @attr-lookups 12) + (str "expected bounded attr lookups, got " @attr-lookups))))) From 5f2ec2ff713575fbc4a28a8656b38575017dc5a6 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 26 Feb 2026 14:54:42 +0800 Subject: [PATCH 14/24] fix: no log for sync server --- deps/db-sync/shadow-cljs.edn | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/deps/db-sync/shadow-cljs.edn b/deps/db-sync/shadow-cljs.edn index ed71bedfb0..b3e71bd628 100644 --- a/deps/db-sync/shadow-cljs.edn +++ b/deps/db-sync/shadow-cljs.edn @@ -11,7 +11,8 @@ :modules {:main {:exports {default logseq.db-sync.worker/worker SyncDO logseq.db-sync.worker/SyncDO}}} :js-options {:js-provider :import} - :closure-defines {shadow.cljs.devtools.client.env/enabled false} + :closure-defines {shadow.cljs.devtools.client.env/enabled false + goog.debug.LOGGING_ENABLED true} :devtools {:enabled false}} :db-sync-node {:target :node-script :output-to "worker/dist/node-adapter.js" @@ -19,7 +20,8 @@ :compiler-options {:source-map true :warnings {:fn-deprecated false :redef false}} - :devtools {:enabled false}} + :devtools {:enabled false} + :closure-defines {goog.debug.LOGGING_ENABLED true}} :db-sync-test {:target :node-test :output-to "worker/dist/worker-test.js" :devtools {:enabled false} From 1df918728be12991f462d8cbf6a2d7408b5c232d Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 26 Feb 2026 17:42:25 +0800 Subject: [PATCH 15/24] fix: ws latency --- deps/db-sync/src/logseq/db_sync/worker.cljs | 3 +- .../src/logseq/db_sync/worker/auth.cljs | 20 ++- .../logseq/db_sync/worker/handler/index.cljs | 121 ++++++++++++++++-- .../test/logseq/db_sync/test_runner.cljs | 1 + .../test/logseq/db_sync/worker_auth_test.cljs | 18 +++ .../db_sync/worker_handler_index_test.cljs | 53 ++++++++ 6 files changed, 201 insertions(+), 15 deletions(-) create mode 100644 deps/db-sync/test/logseq/db_sync/worker_handler_index_test.cljs diff --git a/deps/db-sync/src/logseq/db_sync/worker.cljs b/deps/db-sync/src/logseq/db_sync/worker.cljs index afa83341e3..d1b51774e5 100644 --- a/deps/db-sync/src/logseq/db_sync/worker.cljs +++ b/deps/db-sync/src/logseq/db_sync/worker.cljs @@ -68,8 +68,7 @@ (ws/send! ws {:type "error" :message "server error"})))) (webSocketClose [this ws _code _reason] (presence/remove-presence! this ws) - (presence/broadcast-online-users! this) - (log/info :db-sync/ws-closed true)) + (presence/broadcast-online-users! this)) (webSocketError [this ws error] (presence/remove-presence! this ws) (presence/broadcast-online-users! this) diff --git a/deps/db-sync/src/logseq/db_sync/worker/auth.cljs b/deps/db-sync/src/logseq/db_sync/worker/auth.cljs index 5e1de6c4d6..66bff29f1c 100644 --- a/deps/db-sync/src/logseq/db_sync/worker/auth.cljs +++ b/deps/db-sync/src/logseq/db_sync/worker/auth.cljs @@ -40,12 +40,22 @@ (let [message (or (ex-message error) (some-> error .-message))] (contains? recoverable-auth-errors message)))) +(defn- expired-token? + [token] + (when-let [claims (unsafe-jwt-claims token)] + (let [exp (aget claims "exp") + now-s (js/Math.floor (/ (.now js/Date) 1000))] + (and (number? exp) + (<= exp now-s))))) + (defn auth-claims [request env] (let [token (token-from-request request)] (if (string? token) - (-> (authorization/verify-jwt token env) - (p/catch (fn [error] - (if (recoverable-auth-error? error) - nil - (p/rejected error))))) + (if (expired-token? token) + (p/resolved nil) + (-> (authorization/verify-jwt token env) + (p/catch (fn [error] + (if (recoverable-auth-error? error) + nil + (p/rejected error)))))) (p/resolved nil)))) diff --git a/deps/db-sync/src/logseq/db_sync/worker/handler/index.cljs b/deps/db-sync/src/logseq/db_sync/worker/handler/index.cljs index 42205f3bb9..368e73c4c9 100644 --- a/deps/db-sync/src/logseq/db_sync/worker/handler/index.cljs +++ b/deps/db-sync/src/logseq/db_sync/worker/handler/index.cljs @@ -321,12 +321,117 @@ (log/error :db-sync/index-error error) (http/error-response (str "server error: " error) 500))))) -(defn graph-access-response [request env graph-id] +(def ^:private graph-access-cache-ttl-ms 5000) +(def ^:private graph-access-cache-capacity 256) +(defonce ^:private *graph-access-cache (atom {})) + +(defn- now-ms [] + (.now js/Date)) + +(defn- unauthorized-timing [jwt-verify-ms] + {:access-ok? false + :cache-hit? false + :jwt-verify-ms jwt-verify-ms + :access-query-ms 0 + :access-check-ms jwt-verify-ms}) + +(defn- fresh-cache? + [cached-at now-ms] + (and (number? cached-at) + (< (- now-ms cached-at) graph-access-cache-ttl-ms))) + +(defn- lookup-graph-access-cache + [graph-id token now-ms] + (let [cache-key [graph-id token]] + (when-let [{:keys [allowed? cached-at]} (get @*graph-access-cache cache-key)] + (if (fresh-cache? cached-at now-ms) + {:allowed? allowed?} + (do + (swap! *graph-access-cache dissoc cache-key) + nil))))) + +(defn- prune-graph-access-cache + [cache now-ms] + (let [fresh (into {} + (filter (fn [[_ {:keys [cached-at]}]] + (fresh-cache? cached-at now-ms))) + cache)] + (if (<= (count fresh) graph-access-cache-capacity) + fresh + (let [drop-count (- (count fresh) graph-access-cache-capacity)] + (->> fresh + (sort-by (comp :cached-at val)) + (drop drop-count) + (into {})))))) + +(defn- cache-graph-access! + [graph-id token allowed? now-ms] + (let [cache-key [graph-id token]] + (swap! *graph-access-cache + (fn [cache] + (-> cache + (assoc cache-key {:allowed? allowed? :cached-at now-ms}) + (prune-graph-access-cache now-ms)))))) + +(defn graph-access-response-with-timing + [request env graph-id] (let [token (auth/token-from-request request) - url (js/URL. (.-url request)) - access-url (str (.-origin url) "/graphs/" graph-id "/access") - headers (js/Headers. (.-headers request)) - index-self #js {:env env :d1 (aget env "DB")}] - (when (string? token) - (.set headers "authorization" (str "Bearer " token))) - (handle-fetch index-self (js/Request. access-url #js {:method "GET" :headers headers})))) + db (aget env "DB")] + (cond + (or (not (string? token)) + (not (seq token))) + (p/resolved {:response (http/unauthorized) + :timing (unauthorized-timing 0)}) + + (nil? db) + (p/resolved {:response (http/error-response "server error" 500) + :timing {:access-ok? false + :cache-hit? false}}) + + :else + (let [current-ms (now-ms)] + (if-let [{:keys [allowed?]} (lookup-graph-access-cache graph-id token current-ms)] + (p/resolved {:response (if allowed? + (http/json-response :graphs/access {:ok true}) + (http/forbidden)) + :timing {:access-ok? allowed? + :cache-hit? true + :jwt-verify-ms 0 + :access-query-ms 0 + :access-check-ms 0}}) + (let [jwt-start-ms (now-ms)] + (-> + (p/let [claims (auth/auth-claims request env) + jwt-end-ms (now-ms) + jwt-verify-ms (- jwt-end-ms jwt-start-ms)] + (if (nil? claims) + {:response (http/unauthorized) + :timing (unauthorized-timing jwt-verify-ms)} + (let [user-id (aget claims "sub")] + (if-not (string? user-id) + {:response (http/unauthorized) + :timing (unauthorized-timing jwt-verify-ms)} + (p/let [query-start-ms (now-ms) + access? (index/ (p/with-redefs [authorization/verify-jwt + (fn [_token _env] + (reset! verify-called? true) + (p/rejected (ex-info "should-not-be-called" {})))] + (p/let [claims (auth/auth-claims request #js {})] + (is (nil? claims)) + (is (false? @verify-called?)))) + (p/then (fn [] (done))) + (p/catch (fn [error] + (is false (str error)) + (done))))))) diff --git a/deps/db-sync/test/logseq/db_sync/worker_handler_index_test.cljs b/deps/db-sync/test/logseq/db_sync/worker_handler_index_test.cljs new file mode 100644 index 0000000000..a2f6f143fb --- /dev/null +++ b/deps/db-sync/test/logseq/db_sync/worker_handler_index_test.cljs @@ -0,0 +1,53 @@ +(ns logseq.db-sync.worker-handler-index-test + (:require [cljs.test :refer [async deftest is]] + [logseq.db-sync.index :as index] + [logseq.db-sync.worker.auth :as auth] + [logseq.db-sync.worker.handler.index :as index-handler] + [promesa.core :as p])) + +(deftest graph-access-response-with-timing-caches-result-test + (async done + (let [request (js/Request. "http://localhost/sync/graph-1" + #js {:headers #js {"authorization" "Bearer token-cache-hit"}}) + env #js {"DB" #js {}} + auth-count (atom 0) + query-count (atom 0)] + (-> (p/with-redefs [auth/auth-claims (fn [_request _env] + (swap! auth-count inc) + (p/resolved #js {"sub" "user-1"})) + index/ (p/with-redefs [auth/auth-claims (fn [_request _env] + (p/resolved #js {"sub" "user-2"})) + index/ Date: Thu, 26 Feb 2026 18:09:47 +0800 Subject: [PATCH 16/24] fix: startup dedupe guard for db-sync ws connect --- .../db_based/rtc_background_tasks.cljs | 6 ++-- .../frontend/handler/db_based/rtc_flows.cljs | 12 +++---- src/main/frontend/handler/db_based/sync.cljs | 12 +++---- src/main/frontend/worker/sync.cljs | 31 +++++++++++++------ .../frontend/handler/db_based/sync_test.cljs | 6 ++-- 5 files changed, 38 insertions(+), 29 deletions(-) diff --git a/src/main/frontend/handler/db_based/rtc_background_tasks.cljs b/src/main/frontend/handler/db_based/rtc_background_tasks.cljs index 327898f807..cd9afeff2b 100644 --- a/src/main/frontend/handler/db_based/rtc_background_tasks.cljs +++ b/src/main/frontend/handler/db_based/rtc_background_tasks.cljs @@ -37,9 +37,9 @@ (m/reduce (constantly nil) (m/ap - (let [logout-or-graph-switch (m/?> rtc-flows/logout-or-graph-switch-flow)] - (log/info :try-to-stop-rtc-if-needed logout-or-graph-switch) - (c.m/ rtc-flows/logout-flow) + (log/info :try-to-stop-rtc-if-needed :logout) + (c.m/latest-remote-tx (atom {})) +(defonce *start-inflight-target (atom nil)) (defn- current-client [repo] @@ -1324,6 +1325,8 @@ [repo] (let [base (ws-base-url) graph-id (get-graph-id repo) + start-target [repo graph-id] + inflight-target @*start-inflight-target current @worker-state/*db-sync-client] (cond (not (and (string? base) (seq base) (seq graph-id))) @@ -1331,21 +1334,31 @@ (log/info :db-sync/start-skipped {:repo repo :graph-id graph-id :base base}) (p/resolved nil)) + (= start-target inflight-target) + (p/resolved nil) + (active-client-for? current repo graph-id) (do (broadcast-rtc-state! current) (p/resolved nil)) :else - (p/do! - (stop!) - (let [client (ensure-client-state! repo) - url (format-ws-url base graph-id) - _ (ensure-client-graph-uuid! repo graph-id) - connected (assoc client :graph-id graph-id) - connected (connect! repo connected url)] - (reset! worker-state/*db-sync-client connected) - (p/resolved nil)))))) + (do + (reset! *start-inflight-target start-target) + (-> + (p/do! + (stop!) + (let [client (ensure-client-state! repo) + url (format-ws-url base graph-id) + _ (ensure-client-graph-uuid! repo graph-id) + connected (assoc client :graph-id graph-id) + connected (connect! repo connected url)] + (reset! worker-state/*db-sync-client connected) + nil)) + (p/finally + (fn [] + (when (= start-target @*start-inflight-target) + (reset! *start-inflight-target nil))))))))) (defn enqueue-local-tx! [repo {:keys [tx-meta tx-data db-after db-before]}] diff --git a/src/test/frontend/handler/db_based/sync_test.cljs b/src/test/frontend/handler/db_based/sync_test.cljs index 188fad931c..cdd8994da8 100644 --- a/src/test/frontend/handler/db_based/sync_test.cljs +++ b/src/test/frontend/handler/db_based/sync_test.cljs @@ -85,14 +85,14 @@ (deftest rtc-start-skips-when-graph-missing-from-remote-list-test (async done - (let [called (atom nil)] + (let [called (atom [])] (-> (p/with-redefs [state/get-rtc-graphs (fn [] [{:url "repo-other"}]) state/ Date: Thu, 26 Feb 2026 18:42:11 +0800 Subject: [PATCH 17/24] chore: add debug log --- src/main/frontend/worker/sync.cljs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/frontend/worker/sync.cljs b/src/main/frontend/worker/sync.cljs index ceb392ceb1..ebc4de946b 100644 --- a/src/main/frontend/worker/sync.cljs +++ b/src/main/frontend/worker/sync.cljs @@ -1010,6 +1010,7 @@ [{:keys [conn local-txs reversed-tx-data safe-remote-tx-data remote-deleted-blocks temp-tx-meta *remote-tx-report *reversed-tx-report *remote-deleted-ids *rebase-tx-data]}] (let [batch-tx-meta {:rtc-tx? true}] + (prn :debug :local-txs local-txs) (ldb/transact-with-temp-conn! conn batch-tx-meta @@ -1042,6 +1043,7 @@ (fix-tx! temp-conn remote-tx-report rebase-tx-report (assoc tx-meta :op :fix)) (delete-nodes! temp-conn deleted-nodes (assoc tx-meta :op :delete-blocks)))) {:listen-db (fn [{:keys [tx-meta tx-data]}] + (prn :debug :tx-meta tx-meta :tx-data tx-data) (when-not (contains? #{:reverse :transact-remote-tx-data} (:op tx-meta)) (swap! *rebase-tx-data into tx-data)))}))) @@ -1179,6 +1181,7 @@ (parse-transit (:tx data) {:repo repo :type "pull/ok"})) txs) tx (distinct (mapcat identity txs-data))] + (prn :debug :remote-tx tx) (when (seq tx) (p/let [aes-key (sync-crypt/ Date: Thu, 26 Feb 2026 19:13:23 +0800 Subject: [PATCH 18/24] Revert "chore: add debug log" This reverts commit f07f366a0d6a43529df0e9140d0a558d822186e1. --- src/main/frontend/worker/sync.cljs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/frontend/worker/sync.cljs b/src/main/frontend/worker/sync.cljs index ebc4de946b..ceb392ceb1 100644 --- a/src/main/frontend/worker/sync.cljs +++ b/src/main/frontend/worker/sync.cljs @@ -1010,7 +1010,6 @@ [{:keys [conn local-txs reversed-tx-data safe-remote-tx-data remote-deleted-blocks temp-tx-meta *remote-tx-report *reversed-tx-report *remote-deleted-ids *rebase-tx-data]}] (let [batch-tx-meta {:rtc-tx? true}] - (prn :debug :local-txs local-txs) (ldb/transact-with-temp-conn! conn batch-tx-meta @@ -1043,7 +1042,6 @@ (fix-tx! temp-conn remote-tx-report rebase-tx-report (assoc tx-meta :op :fix)) (delete-nodes! temp-conn deleted-nodes (assoc tx-meta :op :delete-blocks)))) {:listen-db (fn [{:keys [tx-meta tx-data]}] - (prn :debug :tx-meta tx-meta :tx-data tx-data) (when-not (contains? #{:reverse :transact-remote-tx-data} (:op tx-meta)) (swap! *rebase-tx-data into tx-data)))}))) @@ -1181,7 +1179,6 @@ (parse-transit (:tx data) {:repo repo :type "pull/ok"})) txs) tx (distinct (mapcat identity txs-data))] - (prn :debug :remote-tx tx) (when (seq tx) (p/let [aes-key (sync-crypt/ Date: Thu, 26 Feb 2026 21:04:02 +0800 Subject: [PATCH 19/24] Reveal stale :block/title sync bug in tests --- src/test/frontend/worker/db_sync_sim_test.cljs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/test/frontend/worker/db_sync_sim_test.cljs b/src/test/frontend/worker/db_sync_sim_test.cljs index 4c18f8fd4e..9b65aa184f 100644 --- a/src/test/frontend/worker/db_sync_sim_test.cljs +++ b/src/test/frontend/worker/db_sync_sim_test.cljs @@ -1,7 +1,6 @@ (ns frontend.worker.db-sync-sim-test (:require [cljs.test :refer [deftest is testing]] [clojure.data :as data] - [clojure.string :as string] [datascript.core :as d] [frontend.worker.handler.page :as worker-page] [frontend.worker.state :as worker-state] @@ -371,9 +370,8 @@ (let [parent-uuid (:block/uuid parent) parent (d/entity db [:block/uuid parent-uuid])] (when parent - (let [uuid ((or gen-uuid random-uuid)) - title (str "Block-" (rand-int! rng 1000000))] - (create-block! conn parent title uuid) + (let [uuid ((or gen-uuid random-uuid))] + (create-block! conn parent "" uuid) (swap! state update :blocks conj uuid) {:op :create-block :uuid uuid :parent parent-uuid})))))) @@ -384,7 +382,7 @@ block (d/entity db [:block/uuid (:block/uuid ent)])] (when (and block (not (ldb/page? block))) (let [uuid (:block/uuid block) - new-title (string/replace (:block/title (d/entity @conn [:block/uuid uuid])) "block" "title")] + new-title (str "title-" (:db/id block))] (update-title! conn uuid new-title) {:op :update-title :uuid uuid :title new-title})))) From 08213ceeb9de4214467cfaaf7d269e84d98aacc1 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 26 Feb 2026 21:15:08 +0800 Subject: [PATCH 20/24] test version of non-batch apply-remote-tx --- src/main/frontend/worker/sync.cljs | 165 +++++++++--------- .../frontend/worker/db_sync_sim_test.cljs | 6 +- src/test/frontend/worker/db_sync_test.cljs | 46 +++++ 3 files changed, 136 insertions(+), 81 deletions(-) diff --git a/src/main/frontend/worker/sync.cljs b/src/main/frontend/worker/sync.cljs index ceb392ceb1..0ed3b3ffc8 100644 --- a/src/main/frontend/worker/sync.cljs +++ b/src/main/frontend/worker/sync.cljs @@ -500,6 +500,12 @@ (= :block/uuid (first x))) (second x))) +(defn- batched-remote-tx-data? + [tx-data*] + (and (seq tx-data*) + (sequential? (first tx-data*)) + (sequential? (first (first tx-data*))))) + (defn- drop-anonymous-temp-entity-datoms "Drop malformed temp entities from remote txs. A temp entity must declare one identity attr (:block/uuid or :db/ident) @@ -1062,79 +1068,82 @@ (defn- apply-remote-tx! [repo client tx-data*] - (if-let [conn (worker-state/get-datascript-conn repo)] - (let [tx-data (->> tx-data* - (db-normalize/remove-retract-entity-ref @conn) - (#(drop-anonymous-temp-entity-datoms @conn %))) - local-txs (pending-txs repo) - reversed-tx-data (get-reverse-tx-data local-txs) - has-local-changes? (seq reversed-tx-data) - *remote-tx-report (atom nil) - *reversed-tx-report (atom nil) - *remote-deleted-ids (atom #{}) - *rebase-tx-data (atom []) - db @conn - remote-deleted-blocks (->> tx-data - (keep (fn [item] - (when (= :db/retractEntity (first item)) - (d/entity db (second item)))))) - remote-deleted-block-ids (set (map :block/uuid remote-deleted-blocks)) - safe-remote-tx-data (->> tx-data - (remove (fn [item] - (or (= :db/retractEntity (first item)) - (contains? remote-deleted-block-ids (get-lookup-id (last item)))))) - seq) - temp-tx-meta {:rtc-tx? true - :temp-conn? true - :gen-undo-ops? false - :persist-op? false} - apply-context {:conn conn - :local-txs local-txs - :reversed-tx-data reversed-tx-data - :safe-remote-tx-data safe-remote-tx-data - :remote-deleted-blocks remote-deleted-blocks - :remote-deleted-block-ids remote-deleted-block-ids - :temp-tx-meta temp-tx-meta - :*remote-tx-report *remote-tx-report - :*reversed-tx-report *reversed-tx-report - :*remote-deleted-ids *remote-deleted-ids - :*rebase-tx-data *rebase-tx-data} - tx-report (if has-local-changes? - (apply-remote-tx-with-local-changes! apply-context) - (apply-remote-tx-without-local-changes! apply-context)) - remote-tx-report @*remote-tx-report] - ;; persist rebase tx to client ops - (when has-local-changes? - (when-let [tx-data (seq @*rebase-tx-data)] - (let [remote-tx-data-set (set tx-data*) - normalized (->> tx-data - (normalize-tx-data (:db-after tx-report) - (or (:db-after remote-tx-report) - (:db-after @*reversed-tx-report))) - (remove (fn [[op _e a]] - (and (= op :db/retract) - (contains? #{:block/updated-at :block/created-at :block/title} a))))) - normalized-tx-data (remove remote-tx-data-set normalized) - reversed-datoms (reverse-tx-data tx-data)] - ;; (prn :debug :normalized-tx-data normalized-tx-data) - ;; (prn :debug :remote-tx-data tx-data*) - ;; (prn :debug :diff (data/diff remote-tx-data-set - ;; (set normalized))) - (when (seq normalized-tx-data) - (persist-local-tx! repo normalized-tx-data reversed-datoms {:op :rtc-rebase})))) - (remove-pending-txs! repo (map :tx-id local-txs))) + (if (batched-remote-tx-data? tx-data*) + (doseq [tx-data tx-data*] + (apply-remote-tx! repo client tx-data)) + (if-let [conn (worker-state/get-datascript-conn repo)] + (let [tx-data (->> tx-data* + (db-normalize/remove-retract-entity-ref @conn) + (#(drop-anonymous-temp-entity-datoms @conn %))) + local-txs (pending-txs repo) + reversed-tx-data (get-reverse-tx-data local-txs) + has-local-changes? (seq reversed-tx-data) + *remote-tx-report (atom nil) + *reversed-tx-report (atom nil) + *remote-deleted-ids (atom #{}) + *rebase-tx-data (atom []) + db @conn + remote-deleted-blocks (->> tx-data + (keep (fn [item] + (when (= :db/retractEntity (first item)) + (d/entity db (second item)))))) + remote-deleted-block-ids (set (map :block/uuid remote-deleted-blocks)) + safe-remote-tx-data (->> tx-data + (remove (fn [item] + (or (= :db/retractEntity (first item)) + (contains? remote-deleted-block-ids (get-lookup-id (last item)))))) + seq) + temp-tx-meta {:rtc-tx? true + :temp-conn? true + :gen-undo-ops? false + :persist-op? false} + apply-context {:conn conn + :local-txs local-txs + :reversed-tx-data reversed-tx-data + :safe-remote-tx-data safe-remote-tx-data + :remote-deleted-blocks remote-deleted-blocks + :remote-deleted-block-ids remote-deleted-block-ids + :temp-tx-meta temp-tx-meta + :*remote-tx-report *remote-tx-report + :*reversed-tx-report *reversed-tx-report + :*remote-deleted-ids *remote-deleted-ids + :*rebase-tx-data *rebase-tx-data} + tx-report (if has-local-changes? + (apply-remote-tx-with-local-changes! apply-context) + (apply-remote-tx-without-local-changes! apply-context)) + remote-tx-report @*remote-tx-report] + ;; persist rebase tx to client ops + (when has-local-changes? + (when-let [tx-data (seq @*rebase-tx-data)] + (let [remote-tx-data-set (set tx-data*) + normalized (->> tx-data + (normalize-tx-data (:db-after tx-report) + (or (:db-after remote-tx-report) + (:db-after @*reversed-tx-report))) + (remove (fn [[op _e a]] + (and (= op :db/retract) + (contains? #{:block/updated-at :block/created-at :block/title} a))))) + normalized-tx-data (remove remote-tx-data-set normalized) + reversed-datoms (reverse-tx-data tx-data)] + ;; (prn :debug :normalized-tx-data normalized-tx-data) + ;; (prn :debug :remote-tx-data tx-data*) + ;; (prn :debug :diff (data/diff remote-tx-data-set + ;; (set normalized))) + (when (seq normalized-tx-data) + (persist-local-tx! repo normalized-tx-data reversed-datoms {:op :rtc-rebase})))) + (remove-pending-txs! repo (map :tx-id local-txs))) - (when-let [*inflight (:inflight client)] - (reset! *inflight [])) + (when-let [*inflight (:inflight client)] + (reset! *inflight [])) - (-> (rehydrate-large-titles! repo {:tx-data tx-data - :graph-id (:graph-id client)}) - (p/catch (fn [error] - (log/error :db-sync/large-title-rehydrate-failed - {:repo repo :error error})))) + (-> (rehydrate-large-titles! repo {:tx-data tx-data + :graph-id (:graph-id client)}) + (p/catch (fn [error] + (log/error :db-sync/large-title-rehydrate-failed + {:repo repo :error error})))) - (reset! *remote-tx-report nil)) - (fail-fast :db-sync/missing-db {:repo repo :op :apply-remote-tx}))) + (reset! *remote-tx-report nil)) + (fail-fast :db-sync/missing-db {:repo repo :op :apply-remote-tx})))) (defn- handle-message! [repo client raw] (let [message (-> raw parse-message coerce-ws-server-message)] @@ -1170,23 +1179,23 @@ (reset! (:inflight client) []) (flush-pending! repo client)) ;; Download response - ;; Merge batch txs to one tx, does it really work? We'll see "pull/ok" (when (> remote-tx local-tx) (let [txs (:txs message) _ (require-non-negative remote-tx {:repo repo :type "pull/ok"}) _ (require-seq txs {:repo repo :type "pull/ok" :field :txs}) txs-data (mapv (fn [data] (parse-transit (:tx data) {:repo repo :type "pull/ok"})) - txs) - tx (distinct (mapcat identity txs-data))] - (when (seq tx) + txs)] + (when (seq txs-data) (p/let [aes-key (sync-crypt/> (filter (fn [{:keys [t]}] (> t since)) txs) - (mapcat :tx)))) + (mapv :tx)))) (defn- server-upload! [server t-before tx-data] (swap! server @@ -233,10 +233,10 @@ server-t (:t @server)] ;; (prn :debug :repo repo :local-tx local-tx :server-t server-t) (when (< local-tx server-t) - (let [tx (server-pull server local-tx)] + (let [txs (server-pull server local-tx)] ;; (prn :debug :apply-remote-tx :repo repo ;; :tx tx) - (#'db-sync/apply-remote-tx! repo client tx) + (#'db-sync/apply-remote-tx! repo client txs) (client-op/update-local-tx repo server-t) (reset! progress? true))) (let [pending (#'db-sync/pending-txs repo) diff --git a/src/test/frontend/worker/db_sync_test.cljs b/src/test/frontend/worker/db_sync_test.cljs index 4d8c98254b..268fa8685b 100644 --- a/src/test/frontend/worker/db_sync_test.cljs +++ b/src/test/frontend/worker/db_sync_test.cljs @@ -176,6 +176,52 @@ (reset! db-sync/*repo->latest-remote-tx latest-prev) (done)))))))))) +(deftest pull-ok-batched-txs-preserve-tempid-boundaries-test + (testing "pull/ok applies tx batches without cross-tx tempid collisions" + (async done + (let [{:keys [conn client-ops-conn parent]} (setup-parent-child) + page-uuid (:block/uuid (:block/page parent)) + block-uuid-a (random-uuid) + block-uuid-b (random-uuid) + now 1760000000000 + tx-a (sqlite-util/write-transit-str + [[:db/add -1 :block/uuid block-uuid-a] + [:db/add -1 :block/title "remote-a"] + [:db/add -1 :block/parent [:block/uuid page-uuid]] + [:db/add -1 :block/page [:block/uuid page-uuid]] + [:db/add -1 :block/order 1] + [:db/add -1 :block/updated-at now] + [:db/add -1 :block/created-at now]]) + tx-b (sqlite-util/write-transit-str + [[:db/add -1 :block/uuid block-uuid-b] + [:db/add -1 :block/title "remote-b"] + [:db/add -1 :block/parent [:block/uuid page-uuid]] + [:db/add -1 :block/page [:block/uuid page-uuid]] + [:db/add -1 :block/order 2] + [:db/add -1 :block/updated-at now] + [:db/add -1 :block/created-at now]]) + raw-message (js/JSON.stringify + (clj->js {:type "pull/ok" + :t 2 + :txs [{:t 1 :tx tx-a} + {:t 2 :tx tx-b}]})) + latest-prev @db-sync/*repo->latest-remote-tx + client {:repo test-repo + :graph-id "graph-1" + :inflight (atom []) + :online-users (atom []) + :ws-state (atom :open)}] + (with-datascript-conns conn client-ops-conn + (fn [] + (reset! db-sync/*repo->latest-remote-tx {}) + (-> (p/let [_ (client-op/update-local-tx test-repo 0) + _ (#'db-sync/handle-message! test-repo client raw-message)] + (is (= "remote-a" (:block/title (d/entity @conn [:block/uuid block-uuid-a])))) + (is (= "remote-b" (:block/title (d/entity @conn [:block/uuid block-uuid-b]))))) + (p/finally (fn [] + (reset! db-sync/*repo->latest-remote-tx latest-prev) + (done)))))))))) + (deftest reaction-add-enqueues-pending-sync-tx-test (testing "adding a reaction should enqueue tx for db-sync" (let [{:keys [conn client-ops-conn parent]} (setup-parent-child)] From ba5e83c04563f6f14c281470c1a9a467592f3be0 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 26 Feb 2026 21:50:25 +0800 Subject: [PATCH 21/24] Updated apply-remote-tx! to keep one-shot batch apply while preserving correctness. Changes in sync.cljs (src/main/frontend/worker/sync.cljs): - Added batch flatten pipeline that: - remaps per-batch tempids to unique ids, - preserves tx-id position remapping, - tracks newly created identities (:block/uuid / :db/ident), - rewrites cross-batch lookup refs to tempids for :db/add so later adds in the same merged transact can target newly-created entities. - apply-remote-tx! batched branch now calls the new flattener once (no doseq per batch). - Kept rebase safety in local-changes path (conflict filtering + remote- duplicate cleanup after sanitize). - Fixed negative temp-id handling in canonical-entity-id to avoid d/entity errors. --- src/main/frontend/worker/sync.cljs | 158 +++++++++++++++++++++++++++-- 1 file changed, 151 insertions(+), 7 deletions(-) diff --git a/src/main/frontend/worker/sync.cljs b/src/main/frontend/worker/sync.cljs index 0ed3b3ffc8..6ffe40cbe5 100644 --- a/src/main/frontend/worker/sync.cljs +++ b/src/main/frontend/worker/sync.cljs @@ -500,6 +500,137 @@ (= :block/uuid (first x))) (second x))) +(defn- canonical-entity-id + [db e] + (cond + (vector? e) (or (get-lookup-id e) e) + (and (number? e) (not (neg? e))) (or (:block/uuid (d/entity db e)) e) + :else e)) + +(defn- remote-updated-attr-keys + [db tx-data] + (->> tx-data + (keep (fn [item] + (when (and (vector? item) + (>= (count item) 4) + (contains? #{:db/add :db/retract} (first item))) + [(canonical-entity-id db (second item)) + (nth item 2)]))) + set)) + +(defn- drop-remote-conflicted-local-tx + [db remote-updated-keys tx-data] + (if (seq remote-updated-keys) + (remove (fn [item] + (and (vector? item) + (>= (count item) 4) + (contains? #{:db/add :db/retract} (first item)) + (contains? remote-updated-keys + [(canonical-entity-id db (second item)) + (nth item 2)]))) + tx-data) + tx-data)) + +(defn- remote-temp-id? + [x] + (or (and (integer? x) (neg? x)) + (string? x))) + +(defn- remap-remote-batch-temp-ids + [batch-index tx-data] + (let [ops #{:db/add :db/retract :db/retractEntity} + entity-temp-ids (->> tx-data + (keep (fn [item] + (when (and (vector? item) + (>= (count item) 2) + (contains? ops (first item)) + (remote-temp-id? (second item))) + (second item)))) + distinct) + temp-id-map (when (seq entity-temp-ids) + (zipmap entity-temp-ids + (map-indexed (fn [idx _] + (str "remote-batch-" batch-index "-tempid-" idx)) + entity-temp-ids)))] + (if (seq temp-id-map) + (mapv (fn [item] + (if (and (vector? item) + (>= (count item) 2) + (contains? ops (first item))) + (let [entity (second item) + item' (if-let [entity' (get temp-id-map entity)] + (assoc item 1 entity') + item)] + (cond-> item' + (>= (count item') 4) + (#(if-let [value' (get temp-id-map (nth % 3))] + (assoc % 3 value') + %)) + + (>= (count item') 5) + (#(if-let [tx' (get temp-id-map (nth % 4))] + (assoc % 4 tx') + %)))) + item)) + tx-data) + tx-data))) + +(defn- lookup-ref? + [x] + (and (vector? x) + (= 2 (count x)) + (keyword? (first x)))) + +(defn- created-lookup->temp-id + [tx-data] + (->> tx-data + (keep (fn [item] + (when (and (vector? item) + (= :db/add (first item)) + (>= (count item) 4) + (contains? #{:block/uuid :db/ident} (nth item 2)) + (remote-temp-id? (second item))) + [[(nth item 2) (nth item 3)] + (second item)]))) + (into {}))) + +(defn- resolve-lookup-refs + [lookup->temp-id tx-data] + (if (seq lookup->temp-id) + (mapv (fn [item] + (if (and (vector? item) + (>= (count item) 2) + (= :db/add (first item))) + (let [entity (second item) + item' (if-let [entity' (and (lookup-ref? entity) + (get lookup->temp-id entity))] + (assoc item 1 entity') + item)] + (if (>= (count item') 4) + (let [value (nth item' 3)] + (if-let [value' (and (lookup-ref? value) + (get lookup->temp-id value))] + (assoc item' 3 value') + item')) + item')) + item)) + tx-data) + tx-data)) + +(defn- flatten-batched-remote-tx-data + [tx-data*] + (loop [remaining (map-indexed vector tx-data*) + lookup->temp-id {} + acc []] + (if-let [[batch-index tx-data] (first remaining)] + (let [remapped-batch (remap-remote-batch-temp-ids batch-index tx-data) + lookup->temp-id (merge lookup->temp-id (created-lookup->temp-id remapped-batch)) + resolved-batch (resolve-lookup-refs lookup->temp-id remapped-batch)] + (recur (rest remaining) + lookup->temp-id + (into acc resolved-batch))) + acc))) + (defn- batched-remote-tx-data? [tx-data*] (and (seq tx-data*) @@ -1035,11 +1166,25 @@ ;; 3. rebase pending local txs rebase-tx-report (when (seq local-txs) (let [pending-tx-data (mapcat :tx local-txs) - rebased-tx-data (sanitize-tx-data - (or (:db-after remote-tx-report) - (:db-after reversed-tx-report)) - pending-tx-data - (set (map :block/uuid local-deleted-blocks)))] + remote-db (or (:db-after remote-tx-report) + (:db-after reversed-tx-report)) + remote-updated-keys (remote-updated-attr-keys remote-db safe-remote-tx-data) + remote-tx-data-set (->> safe-remote-tx-data + (map (fn [item] + (if (and (vector? item) + (= 5 (count item))) + (vec (butlast item)) + item))) + set) + pending-tx-data (drop-remote-conflicted-local-tx + remote-db + remote-updated-keys + pending-tx-data) + rebased-tx-data (->> (sanitize-tx-data + remote-db + pending-tx-data + (set (map :block/uuid local-deleted-blocks))) + (remove remote-tx-data-set))] (when (seq rebased-tx-data) (ldb/transact! temp-conn rebased-tx-data (assoc tx-meta :op :rebase))))) ;; 4. fix tx data and delete nodes @@ -1069,8 +1214,7 @@ (defn- apply-remote-tx! [repo client tx-data*] (if (batched-remote-tx-data? tx-data*) - (doseq [tx-data tx-data*] - (apply-remote-tx! repo client tx-data)) + (apply-remote-tx! repo client (flatten-batched-remote-tx-data tx-data*)) (if-let [conn (worker-state/get-datascript-conn repo)] (let [tx-data (->> tx-data* (db-normalize/remove-retract-entity-ref @conn) From 5cb50683a8fb8542eaa31c262dbfc180f9f0c36a Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 26 Feb 2026 22:34:07 +0800 Subject: [PATCH 22/24] fix: sync start now waits for DB worker initialization before invoking worker APIs --- .../db_based/rtc_background_tasks.cljs | 2 +- src/main/frontend/handler/db_based/sync.cljs | 37 ++++++++++++---- .../frontend/handler/db_based/sync_test.cljs | 42 +++++++++++++++++++ 3 files changed, 71 insertions(+), 10 deletions(-) diff --git a/src/main/frontend/handler/db_based/rtc_background_tasks.cljs b/src/main/frontend/handler/db_based/rtc_background_tasks.cljs index cd9afeff2b..3231dd3994 100644 --- a/src/main/frontend/handler/db_based/rtc_background_tasks.cljs +++ b/src/main/frontend/handler/db_based/rtc_background_tasks.cljs @@ -32,7 +32,7 @@ (c.m/ (p/with-redefs [state/get-rtc-graphs (fn [] [{:url "repo-current"}]) + state/*db-worker worker] + (p/let [start-p (db-sync/ (p/with-redefs [state/get-rtc-graphs (fn [] [{:url "repo-other"}]) + state/*db-worker worker] + (p/let [start-p (db-sync/ Date: Thu, 26 Feb 2026 22:54:14 +0800 Subject: [PATCH 23/24] disable invocation logs --- deps/db-sync/worker/wrangler.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deps/db-sync/worker/wrangler.toml b/deps/db-sync/worker/wrangler.toml index 0a1420f681..904d622e8f 100644 --- a/deps/db-sync/worker/wrangler.toml +++ b/deps/db-sync/worker/wrangler.toml @@ -9,6 +9,9 @@ binding = "CF_VERSION_METADATA" [observability] enabled = true +[observability.logs] +invocation_logs = false + [[durable_objects.bindings]] name = "LOGSEQ_SYNC_DO" class_name = "SyncDO" From 7d268fbd42ed50f60a10b0516016dbc813477a8e Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 26 Feb 2026 22:57:25 +0800 Subject: [PATCH 24/24] fix: lint --- .../logseq/db_sync/worker/handler/index.cljs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/deps/db-sync/src/logseq/db_sync/worker/handler/index.cljs b/deps/db-sync/src/logseq/db_sync/worker/handler/index.cljs index 368e73c4c9..f93593e1ad 100644 --- a/deps/db-sync/src/logseq/db_sync/worker/handler/index.cljs +++ b/deps/db-sync/src/logseq/db_sync/worker/handler/index.cljs @@ -336,25 +336,25 @@ :access-check-ms jwt-verify-ms}) (defn- fresh-cache? - [cached-at now-ms] + [cached-at current-ms] (and (number? cached-at) - (< (- now-ms cached-at) graph-access-cache-ttl-ms))) + (< (- current-ms cached-at) graph-access-cache-ttl-ms))) (defn- lookup-graph-access-cache - [graph-id token now-ms] + [graph-id token current-ms] (let [cache-key [graph-id token]] (when-let [{:keys [allowed? cached-at]} (get @*graph-access-cache cache-key)] - (if (fresh-cache? cached-at now-ms) + (if (fresh-cache? cached-at current-ms) {:allowed? allowed?} (do (swap! *graph-access-cache dissoc cache-key) nil))))) (defn- prune-graph-access-cache - [cache now-ms] + [cache current-ms] (let [fresh (into {} (filter (fn [[_ {:keys [cached-at]}]] - (fresh-cache? cached-at now-ms))) + (fresh-cache? cached-at current-ms))) cache)] (if (<= (count fresh) graph-access-cache-capacity) fresh @@ -365,13 +365,13 @@ (into {})))))) (defn- cache-graph-access! - [graph-id token allowed? now-ms] + [graph-id token allowed? current-ms] (let [cache-key [graph-id token]] (swap! *graph-access-cache (fn [cache] (-> cache - (assoc cache-key {:allowed? allowed? :cached-at now-ms}) - (prune-graph-access-cache now-ms)))))) + (assoc cache-key {:allowed? allowed? :cached-at current-ms}) + (prune-graph-access-cache current-ms)))))) (defn graph-access-response-with-timing [request env graph-id]