From 83a716f907eef7f021ec1d8035fd95847fc79e18 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Wed, 11 Mar 2026 14:49:12 +0800 Subject: [PATCH 01/21] chore: bump datascript --- deps.edn | 2 +- deps/db-sync/deps.edn | 2 +- deps/db/deps.edn | 2 +- deps/outliner/deps.edn | 2 +- deps/publish/deps.edn | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deps.edn b/deps.edn index 9bfc69417c..23c11b6f17 100644 --- a/deps.edn +++ b/deps.edn @@ -5,7 +5,7 @@ :sha "5d672bf84ed944414b9f61eeb83808ead7be9127"} datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork - :sha "ff5a7d5326e2546f40146e4a489343f557519bc3"} + :sha "f91fec561ee2c11d6bf323feae365e9033585411"} ;; datascript/datascript {:local/root "../../datascript"} datascript-transit/datascript-transit {:mvn/version "0.3.0"} diff --git a/deps/db-sync/deps.edn b/deps/db-sync/deps.edn index 8915499b8d..14f589bac9 100644 --- a/deps/db-sync/deps.edn +++ b/deps/db-sync/deps.edn @@ -2,7 +2,7 @@ :deps {org.clojure/clojure {:mvn/version "1.11.1"} datascript/datascript {:git/url "https://github.com/logseq/datascript" - :sha "ff5a7d5326e2546f40146e4a489343f557519bc3"} + :sha "f91fec561ee2c11d6bf323feae365e9033585411"} datascript-transit/datascript-transit {:mvn/version "0.3.0" :exclusions [datascript/datascript]} com.cognitect/transit-cljs {:mvn/version "0.8.280"} diff --git a/deps/db/deps.edn b/deps/db/deps.edn index 4d4de6a2f6..d255710dd8 100644 --- a/deps/db/deps.edn +++ b/deps/db/deps.edn @@ -1,7 +1,7 @@ {:deps ;; These nbb-logseq deps are kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn {datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork - :sha "ff5a7d5326e2546f40146e4a489343f557519bc3"} + :sha "f91fec561ee2c11d6bf323feae365e9033585411"} ;; datascript/datascript {:local/root "../../../../datascript"} datascript-transit/datascript-transit {:mvn/version "0.3.0" :exclusions [datascript/datascript]} diff --git a/deps/outliner/deps.edn b/deps/outliner/deps.edn index 3cf51bdd39..f168846a37 100644 --- a/deps/outliner/deps.edn +++ b/deps/outliner/deps.edn @@ -1,7 +1,7 @@ {:deps ;; These nbb-logseq deps are kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn {datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork - :sha "ff5a7d5326e2546f40146e4a489343f557519bc3"} + :sha "f91fec561ee2c11d6bf323feae365e9033585411"} ;; datascript/datascript {:local/root "../../../../datascript"} com.cognitect/transit-cljs {:mvn/version "0.8.280"} diff --git a/deps/publish/deps.edn b/deps/publish/deps.edn index d0d209a905..7089766b52 100644 --- a/deps/publish/deps.edn +++ b/deps/publish/deps.edn @@ -5,7 +5,7 @@ :sha "5d672bf84ed944414b9f61eeb83808ead7be9127"} datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork - :sha "ff5a7d5326e2546f40146e4a489343f557519bc3"} + :sha "f91fec561ee2c11d6bf323feae365e9033585411"} datascript-transit/datascript-transit {:mvn/version "0.3.0" :exclusions [datascript/datascript]} funcool/promesa {:mvn/version "11.0.678"} From c1a9bee798ab663ef8ac32e51f8ad91ec467465b Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Wed, 11 Mar 2026 17:36:21 +0800 Subject: [PATCH 02/21] fix: undo should skip conflicted move instead of clearing stack --- src/main/frontend/undo_redo.cljs | 6 +- src/test/frontend/undo_redo_test.cljs | 27 +++++++ .../frontend/worker/db_sync_sim_test.cljs | 72 +++++++++++++++++++ 3 files changed, 103 insertions(+), 2 deletions(-) diff --git a/src/main/frontend/undo_redo.cljs b/src/main/frontend/undo_redo.cljs index 49a8f983d1..0a487b1aa8 100644 --- a/src/main/frontend/undo_redo.cljs +++ b/src/main/frontend/undo_redo.cljs @@ -391,8 +391,10 @@ (log/error ::undo-redo-failed e) (clear-history! repo))))) (do - (clear-history! repo) - (if undo? ::empty-undo-stack ::empty-redo-stack)))))))) + (log/warn ::undo-redo-skip-conflicted-op + {:undo? undo? + :outliner-op (:outliner-op tx-meta)}) + (undo-redo-aux repo undo?)))))))) (when ((if undo? empty-undo-stack? empty-redo-stack?) repo) (prn (str "No further " (if undo? "undo" "redo") " information")) diff --git a/src/test/frontend/undo_redo_test.cljs b/src/test/frontend/undo_redo_test.cljs index ae3408f1e5..cf6f4b8edb 100644 --- a/src/test/frontend/undo_redo_test.cljs +++ b/src/test/frontend/undo_redo_test.cljs @@ -348,6 +348,33 @@ (is (= child-uuid (:block/uuid (:block/parent parent)))) (is (= page-uuid (:block/uuid (:block/parent child)))))))) +(deftest undo-skips-conflicted-move-and-keeps-earlier-history-test + (testing "undo drops a conflicting move op but still undoes earlier safe ops" + (undo-redo/clear-history! test-db) + (let [conn (db/get-db test-db false) + {:keys [parent-a-uuid parent-b-uuid child-uuid]} (seed-page-two-parents-child!)] + (d/transact! conn + [[:db/add [:block/uuid child-uuid] :block/title "local-title"]] + {:outliner-op :save-block + :local-tx? true}) + (d/transact! conn + [[:db/add [:block/uuid child-uuid] :block/parent [:block/uuid parent-b-uuid]]] + {:outliner-op :move-blocks + :local-tx? true}) + (d/transact! conn + [[:db/retractEntity [:block/uuid parent-a-uuid]]] + {:outliner-op :delete-blocks + :local-tx? false}) + (let [undo-result (undo-redo/undo test-db) + child (d/entity @conn [:block/uuid child-uuid])] + (is (not= :frontend.undo-redo/empty-undo-stack undo-result)) + (is (= "child" (:block/title child))) + (is (= parent-b-uuid + (:block/uuid (:block/parent child)))) + (is (empty? (db-issues @conn)))) + (is (= :frontend.undo-redo/empty-undo-stack + (undo-redo/undo test-db)))))) + (deftest undo-validation-fast-path-skips-db-issues-for-non-structural-tx-test (testing "undo validation skips db-issues for non-structural tx-data" (let [conn (db/get-db test-db false) diff --git a/src/test/frontend/worker/db_sync_sim_test.cljs b/src/test/frontend/worker/db_sync_sim_test.cljs index 8838364da0..16afcf1a08 100644 --- a/src/test/frontend/worker/db_sync_sim_test.cljs +++ b/src/test/frontend/worker/db_sync_sim_test.cljs @@ -1266,6 +1266,78 @@ (sync-loop! server [{:repo repo-a :conn conn-a :client client-a :online? true}]) (is (= "test" (:block/title (d/entity @conn-a [:block/uuid block-uuid]))))))))) +(deftest ^:long two-clients-undo-skips-conflicted-move-but-keeps-db-valid-test + (testing "undo skips a conflicted move while syncing the remaining safe history" + (let [base-uuid (uuid "31111111-1111-1111-1111-111111111111") + parent-a-uuid (uuid "32222222-2222-2222-2222-222222222222") + parent-b-uuid (uuid "33333333-3333-3333-3333-333333333333") + child-uuid (uuid "34444444-4444-4444-4444-444444444444") + conn-a (db-test/create-conn) + conn-b (db-test/create-conn) + ops-a (d/create-conn client-op/schema-in-db) + ops-b (d/create-conn client-op/schema-in-db) + client-a (make-client repo-a) + client-b (make-client repo-b) + server (make-server) + seed 20260311 + history (atom [])] + (with-test-repos {repo-a {:conn conn-a :ops-conn ops-a} + repo-b {:conn conn-b :ops-conn ops-b}} + (fn [] + (let [{:keys [repro restore]} (install-invalid-tx-repro! seed history)] + (try + (reset! db-sync/*repo->latest-remote-tx {}) + (client-op/update-local-tx repo-a 0) + (client-op/update-local-tx repo-b 0) + (ensure-base-page! conn-a base-uuid) + (let [base-a (d/entity @conn-a [:block/uuid base-uuid])] + (create-block! conn-a base-a "parent-a" parent-a-uuid) + (create-block! conn-a base-a "parent-b" parent-b-uuid) + (let [parent-a (d/entity @conn-a [:block/uuid parent-a-uuid])] + (create-block! conn-a parent-a "seed-child" child-uuid))) + (sync-until-idle! server [{:repo repo-a :conn conn-a :client client-a :online? true} + {:repo repo-b :conn conn-b :client client-b :online? true}] + 20) + + (update-title! conn-a child-uuid "local-title") + (move-block! conn-a + {:block/uuid child-uuid} + {:block/uuid parent-b-uuid}) + (sync-until-idle! server [{:repo repo-a :conn conn-a :client client-a :online? true} + {:repo repo-b :conn conn-b :client client-b :online? true}] + 50) + + (delete-block! conn-b parent-a-uuid) + (sync-until-idle! server [{:repo repo-a :conn conn-a :client client-a :online? true} + {:repo repo-b :conn conn-b :client client-b :online? true}] + 50) + + (is (not= :frontend.undo-redo/empty-undo-stack + (undo-redo/undo repo-a))) + + (let [rounds (sync-until-idle! server [{:repo repo-a :conn conn-a :client client-a :online? true} + {:repo repo-b :conn conn-b :client client-b :online? true}] + 50) + child-a (d/entity @conn-a [:block/uuid child-uuid]) + child-b (d/entity @conn-b [:block/uuid child-uuid]) + attrs-a (block-attr-map @conn-a) + attrs-b (block-attr-map @conn-b) + issues-a (db-issues @conn-a) + issues-b (db-issues @conn-b)] + (is (< rounds 50) (str "sync did not become idle rounds=" rounds)) + (is (= "seed-child" (:block/title child-a))) + (is (= "seed-child" (:block/title child-b))) + (is (= parent-b-uuid + (:block/uuid (:block/parent child-a)))) + (is (= parent-b-uuid + (:block/uuid (:block/parent child-b)))) + (is (empty? issues-a) (str "db A issues " (pr-str issues-a))) + (is (empty? issues-b) (str "db B issues " (pr-str issues-b))) + (assert-synced-attrs! seed history attrs-a attrs-b attrs-b) + (assert-no-invalid-tx! seed history repro)) + (finally + (restore))))))))) + (defonce op-runs 200) (defn- run-random-ops! From 1c351aebff0ea1220e0de3ebb05eecc1f8ec3227 Mon Sep 17 00:00:00 2001 From: Mega Yu Date: Thu, 5 Mar 2026 19:42:09 +0800 Subject: [PATCH 03/21] feat: add code snippet extraction functionality in exporter and UI options --- .../src/logseq/graph_parser/exporter.cljs | 85 +++++++++++++- .../logseq/graph_parser/exporter_test.cljs | 104 +++++++++++++++++- .../journals/2026_03_01.md | 33 ++++++ src/main/frontend/components/imports.cljs | 10 ++ 4 files changed, 229 insertions(+), 3 deletions(-) create mode 100644 deps/graph-parser/test/resources/exporter-test-graph/journals/2026_03_01.md diff --git a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs index 6d7e1a997d..c08ffdf758 100644 --- a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs @@ -1390,6 +1390,59 @@ :block/tags [:logseq.class/Quote-block]}) block)) +(defn- split-title-by-code-fences + "Parses a block title string line-by-line, splitting into non-code text parts + and code fence segments. All code fences are extracted regardless of whether + they have a language tag; :lang is nil when not specified. + Returns {:text-parts [...] :code-segs [{:text ... :lang ...}]}." + [title] + (let [lines (string/split-lines title)] + (loop [remaining lines + in-code? false + lang nil + current [] + text-parts [] + code-segs []] + (if (empty? remaining) + {:text-parts (if (seq current) + (conj text-parts (string/join "\n" current)) + text-parts) + :code-segs code-segs} + (let [line (first remaining) + fence-start? (and (not in-code?) (re-matches #"```.*" line)) + fence-end? (and in-code? (= line "```"))] + (cond + fence-start? + (recur (rest remaining) true (not-empty (subs line 3)) [] + (if (seq current) + (conj text-parts (string/join "\n" current)) + text-parts) + code-segs) + fence-end? + (recur (rest remaining) false nil [] + text-parts + (conj code-segs {:text (string/join "\n" current) :lang lang})) + :else + (recur (rest remaining) in-code? lang (conj current line) + text-parts code-segs))))))) + +(defn- build-code-snippet-child-blocks + "Builds child block tx maps for extracted code snippets, tagging each as a + Code-block with its detected language." + [parent-block code-segs] + (mapv (fn [{:keys [text lang]}] + (cond-> (sqlite-util/block-with-timestamps + {:block/uuid (d/squuid) + :block/title text + :block/parent [:block/uuid (:block/uuid parent-block)] + :block/page (:block/page parent-block) + :block/order (db-order/gen-key) + :block/tags [:logseq.class/Code-block] + :logseq.property.node/display-type :code}) + lang + (assoc :logseq.property.code/lang lang))) + code-segs)) + (defn- handle-embeds "If a block contains page or block embeds, converts block to a :block/link based embed" [block page-names-to-uuids {:keys [embeds]} {:keys [log-fn] :or {log-fn prn}}] @@ -1445,8 +1498,36 @@ (dissoc :block/format :block.temp/ast-blocks) ;; ((fn [x] (prn ::block-out x) x)) )] - ;; Order matters as previous txs are referenced in block - (concat properties-tx deadline-properties-tx asset-blocks-tx [block']))) + ;; Code snippet extraction: when enabled, tag standalone code blocks in-place + ;; and extract code fences from mixed-content blocks into independent children. + (let [extract? (get-in options [:user-options :extract-code-snippets?]) + {:keys [text-parts code-segs]} (when extract? + (split-title-by-code-fences (:block/title block'))) + pure-single-code? (and (= 1 (count code-segs)) + (every? string/blank? text-parts)) + has-mixed-content? (and (seq code-segs) + (some #(not (string/blank? %)) text-parts)) + [final-block code-children-tx] + (cond + pure-single-code? + (let [{:keys [text lang]} (first code-segs)] + [(cond-> (assoc block' + :block/title text + :block/tags [:logseq.class/Code-block] + :logseq.property.node/display-type :code) + lang (assoc :logseq.property.code/lang lang)) + []]) + has-mixed-content? + (let [remaining-title (-> (string/join "\n" text-parts) + (string/replace #"\n{2,}" "\n") + string/trim) + updated-block (assoc block' :block/title remaining-title) + code-children (build-code-snippet-child-blocks updated-block code-segs)] + [updated-block code-children]) + :else + [block' []])] + ;; Order matters as previous txs are referenced in block + (concat properties-tx deadline-properties-tx asset-blocks-tx [final-block] code-children-tx)))) (defn- update-page-alias [m page-names-to-uuids] diff --git a/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs b/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs index cddc713875..eafef188eb 100644 --- a/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs +++ b/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs @@ -216,7 +216,7 @@ ;; Counts ;; Includes journals as property values e.g. :logseq.property/deadline - (is (= 33 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Journal]] @conn)))) + (is (= 34 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Journal]] @conn)))) (is (= 9 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Asset]] @conn)))) (is (= 5 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Task]] @conn)))) @@ -948,3 +948,105 @@ (is (= "yyyy-MM-dd" (:logseq.property.journal/title-format (d/entity @conn :logseq.class/Journal))) "title format set correctly by config"))) + +(deftest-async export-docs-graph-with-extract-code-snippet + (p/let [file-graph-dir "test/resources/exporter-test-graph" + conn (db-test/create-conn) + _ (db-pipeline/add-listener conn) + _ (import-file-graph-to-db file-graph-dir conn {:extract-code-snippets? true}) + journal-page-eid (d/q '[:find ?p . :where [?p :block/journal-day 20260301]] @conn) + top-blocks (->> (d/q '[:find [?b ...] + :in $ ?page + :where + [?b :block/page ?page] + [?b :block/parent ?page]] + @conn journal-page-eid) + (map #(d/entity @conn %)) + (sort-by :block/order) + vec) + get-direct-children (fn [block] + (->> (d/q '[:find [?c ...] + :in $ ?parent + :where [?c :block/parent ?parent]] + @conn (:db/id block)) + (map #(d/entity @conn %))))] + + (testing "standalone code block with language tag" + (let [b (nth top-blocks 0)] + (is (= "it's a individual code snippet with language tag" (:block/title b)) + "Standalone code block title has fences stripped") + (is (= 0 (count (get-direct-children b))) + "Standalone code block has no children") + (is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags b)))) + "Standalone code block is tagged as Code-block") + (is (= "markdown" (:logseq.property.code/lang b)) + "Standalone code block has markdown language property"))) + + (testing "standalone code block without language tag" + (let [b (nth top-blocks 1)] + (is (= "it's a individual code snippet without language tag" (:block/title b)) + "Standalone code block title has fences stripped") + (is (= 0 (count (get-direct-children b))) + "Standalone code block has no children") + (is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags b)))) + "Standalone code block is tagged as Code-block") + (is (= nil (:logseq.property.code/lang b)) + "Standalone code block has no language property"))) + + (testing "text before code snippet" + (let [b (nth top-blocks 2) + children (get-direct-children b)] + (is (= "before code snippet" (:block/title b)) + "Block title has text only without code") + (is (= 1 (count children)) + "Block has 1 code child") + (is (= "echo \"ok\"\nexit" (:block/title (first children))) + "Child code block has correct content without fence markers") + (is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags (first children))))) + "Child block is tagged as Code-block") + (is (= nil (:logseq.property.code/lang (first children))) + "Child block has no language property"))) + + (testing "text before and after code snippet" + (let [b (nth top-blocks 3) + children (get-direct-children b)] + (is (= "before code snippet\nafter code snippet" (:block/title b)) + "Block title has text only without code") + (is (= 1 (count children)) + "Block has 1 code child") + (is (= "echo \"ok\"\nexit" (:block/title (first children))) + "Child code block has correct content without fence markers") + (is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags (first children))))) + "Child block is tagged as Code-block") + (is (= "bash" (:logseq.property.code/lang (first children))) + "Child block has bash language property"))) + + (testing "code snippet before text" + (let [b (nth top-blocks 4) + children (get-direct-children b)] + (is (= "after code snippet" (:block/title b)) + "Block title has text only without code") + (is (= 1 (count children)) + "Block has 1 code child") + (is (= "echo \"ok\"\nexit" (:block/title (first children))) + "Child code block has correct content without fence markers") + (is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags (first children))))) + "Child block is tagged as Code-block") + (is (= "bash" (:logseq.property.code/lang (first children))) + "Child block has bash language property"))) + + (testing "multiple code snippets mixed with text" + (let [b (nth top-blocks 5) + children (sort-by :block/order (get-direct-children b))] + (is (= "before code snippet\nmiddle\nafter code snippet" (:block/title b)) + "Block title has all text parts without code") + (is (= 2 (count children)) + "Block has 2 code children") + (is (= "echo \"ok\"\nexit" (:block/title (first children))) + "First child code block has correct content without fence markers") + (is (= "echo \"bye\"\nexit" (:block/title (second children))) + "Second child code block has correct content without fence markers") + (is (every? #(= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags %)))) children) + "Both child blocks are tagged as Code-block") + (is (every? #(= "bash" (:logseq.property.code/lang %)) children) + "Both child blocks have bash language property"))))) diff --git a/deps/graph-parser/test/resources/exporter-test-graph/journals/2026_03_01.md b/deps/graph-parser/test/resources/exporter-test-graph/journals/2026_03_01.md new file mode 100644 index 0000000000..e2aa7a0f3c --- /dev/null +++ b/deps/graph-parser/test/resources/exporter-test-graph/journals/2026_03_01.md @@ -0,0 +1,33 @@ +- ```markdown + it's a individual code snippet with language tag + ``` +- ``` + it's a individual code snippet without language tag + ``` +- before code snippet + ``` + echo "ok" + exit + ``` +- before code snippet + ```bash + echo "ok" + exit + ``` + after code snippet +- ```bash + echo "ok" + exit + ``` + after code snippet +- before code snippet + ```bash + echo "ok" + exit + ``` + middle + ```bash + echo "bye" + exit + ``` + after code snippet \ No newline at end of file diff --git a/src/main/frontend/components/imports.cljs b/src/main/frontend/components/imports.cljs index 54239bb7a7..55db6af356 100644 --- a/src/main/frontend/components/imports.cljs +++ b/src/main/frontend/components/imports.cljs @@ -182,6 +182,7 @@ [:div.border.p-6.rounded.bg-gray-01.mt-4 (let [form-ctx (form-core/use-form {:defaultValues {:graph-name initial-name + :extract-code-snippets? false :convert-all-tags? true :tag-classes "" :remove-inline-tags? true @@ -212,6 +213,15 @@ (shui/form-description [:b.text-red-800 (:message error)]))))) + (shui/form-field {:name "extract-code-snippets?"} + (fn [field] + (shui/form-item + {:class "pt-3 flex justify-start items-center space-x-3 space-y-0 my-3 pr-3"} + (shui/form-label "Extract inline code snippets as child blocks") + (shui/form-control + (shui/checkbox {:checked (:value field) + :on-checked-change (:onChange field)}))))) + (shui/form-field {:name "convert-all-tags?"} (fn [field] (shui/form-item From e2f98a15079cba98e08265f696abfcf1ee94b3bb Mon Sep 17 00:00:00 2001 From: Mega Yu Date: Thu, 5 Mar 2026 20:14:58 +0800 Subject: [PATCH 04/21] standalone code blocks and tag them in-place during exporting --- .../src/logseq/graph_parser/exporter.cljs | 12 ++-- .../logseq/graph_parser/exporter_test.cljs | 58 +++++++++++++++++++ 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs index c08ffdf758..6070b472d2 100644 --- a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs @@ -1498,14 +1498,14 @@ (dissoc :block/format :block.temp/ast-blocks) ;; ((fn [x] (prn ::block-out x) x)) )] - ;; Code snippet extraction: when enabled, tag standalone code blocks in-place - ;; and extract code fences from mixed-content blocks into independent children. - (let [extract? (get-in options [:user-options :extract-code-snippets?]) - {:keys [text-parts code-segs]} (when extract? - (split-title-by-code-fences (:block/title block'))) + ;; Always detect standalone code blocks and tag them in-place. + ;; Extracting code fences from mixed-content blocks into children requires extract-code-snippets?. + (let [{:keys [text-parts code-segs]} (split-title-by-code-fences (:block/title block')) pure-single-code? (and (= 1 (count code-segs)) (every? string/blank? text-parts)) - has-mixed-content? (and (seq code-segs) + extract? (get-in options [:user-options :extract-code-snippets?]) + has-mixed-content? (and extract? + (seq code-segs) (some #(not (string/blank? %)) text-parts)) [final-block code-children-tx] (cond diff --git a/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs b/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs index eafef188eb..abd899abe3 100644 --- a/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs +++ b/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs @@ -1050,3 +1050,61 @@ "Both child blocks are tagged as Code-block") (is (every? #(= "bash" (:logseq.property.code/lang %)) children) "Both child blocks have bash language property"))))) + +(deftest-async export-docs-graph-without-extract-code-snippet + (p/let [file-graph-dir "test/resources/exporter-test-graph" + conn (db-test/create-conn) + _ (db-pipeline/add-listener conn) + _ (import-file-graph-to-db file-graph-dir conn {}) + journal-page-eid (d/q '[:find ?p . :where [?p :block/journal-day 20260301]] @conn) + top-blocks (->> (d/q '[:find [?b ...] + :in $ ?page + :where + [?b :block/page ?page] + [?b :block/parent ?page]] + @conn journal-page-eid) + (map #(d/entity @conn %)) + (sort-by :block/order) + vec) + get-direct-children (fn [block] + (->> (d/q '[:find [?c ...] + :in $ ?parent + :where [?c :block/parent ?parent]] + @conn (:db/id block)) + (map #(d/entity @conn %))))] + + (testing "standalone code block with language tag is still tagged as Code-block" + (let [b (nth top-blocks 0)] + (is (= "it's a individual code snippet with language tag" (:block/title b)) + "Standalone code block title has fences stripped") + (is (= 0 (count (get-direct-children b))) + "Standalone code block has no children") + (is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags b)))) + "Standalone code block is tagged as Code-block") + (is (= "markdown" (:logseq.property.code/lang b)) + "Standalone code block has markdown language property"))) + + (testing "standalone code block without language tag is still tagged as Code-block" + (let [b (nth top-blocks 1)] + (is (= "it's a individual code snippet without language tag" (:block/title b)) + "Standalone code block title has fences stripped") + (is (= 0 (count (get-direct-children b))) + "Standalone code block has no children") + (is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags b)))) + "Standalone code block is tagged as Code-block") + (is (= nil (:logseq.property.code/lang b)) + "Standalone code block has no language property"))) + + (testing "mixed-content block is NOT extracted into children when extract-code-snippets? is false" + (let [b (nth top-blocks 2)] + (is (= 0 (count (get-direct-children b))) + "Block with text before code has no children extracted") + (is (string/includes? (:block/title b) "```") + "Block title retains raw code fence markup"))) + + (testing "another mixed-content block is NOT extracted when extract-code-snippets? is false" + (let [b (nth top-blocks 3)] + (is (= 0 (count (get-direct-children b))) + "Block with text surrounding code has no children extracted") + (is (string/includes? (:block/title b) "```") + "Block title retains raw code fence markup"))))) From 309070fda00efeafe0406d8535612d20aa53822a Mon Sep 17 00:00:00 2001 From: Mega Yu Date: Thu, 5 Mar 2026 21:05:49 +0800 Subject: [PATCH 05/21] fix typo --- .../test/logseq/graph_parser/exporter_test.cljs | 8 ++++---- .../resources/exporter-test-graph/journals/2026_03_01.md | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs b/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs index abd899abe3..4386038c02 100644 --- a/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs +++ b/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs @@ -973,7 +973,7 @@ (testing "standalone code block with language tag" (let [b (nth top-blocks 0)] - (is (= "it's a individual code snippet with language tag" (:block/title b)) + (is (= "it's an individual code snippet with language tag" (:block/title b)) "Standalone code block title has fences stripped") (is (= 0 (count (get-direct-children b))) "Standalone code block has no children") @@ -984,7 +984,7 @@ (testing "standalone code block without language tag" (let [b (nth top-blocks 1)] - (is (= "it's a individual code snippet without language tag" (:block/title b)) + (is (= "it's an individual code snippet without language tag" (:block/title b)) "Standalone code block title has fences stripped") (is (= 0 (count (get-direct-children b))) "Standalone code block has no children") @@ -1075,7 +1075,7 @@ (testing "standalone code block with language tag is still tagged as Code-block" (let [b (nth top-blocks 0)] - (is (= "it's a individual code snippet with language tag" (:block/title b)) + (is (= "it's an individual code snippet with language tag" (:block/title b)) "Standalone code block title has fences stripped") (is (= 0 (count (get-direct-children b))) "Standalone code block has no children") @@ -1086,7 +1086,7 @@ (testing "standalone code block without language tag is still tagged as Code-block" (let [b (nth top-blocks 1)] - (is (= "it's a individual code snippet without language tag" (:block/title b)) + (is (= "it's an individual code snippet without language tag" (:block/title b)) "Standalone code block title has fences stripped") (is (= 0 (count (get-direct-children b))) "Standalone code block has no children") diff --git a/deps/graph-parser/test/resources/exporter-test-graph/journals/2026_03_01.md b/deps/graph-parser/test/resources/exporter-test-graph/journals/2026_03_01.md index e2aa7a0f3c..242739f931 100644 --- a/deps/graph-parser/test/resources/exporter-test-graph/journals/2026_03_01.md +++ b/deps/graph-parser/test/resources/exporter-test-graph/journals/2026_03_01.md @@ -1,8 +1,8 @@ - ```markdown - it's a individual code snippet with language tag + it's an individual code snippet with language tag ``` - ``` - it's a individual code snippet without language tag + it's an individual code snippet without language tag ``` - before code snippet ``` From ecad45531d285089f2ed221e777eba6496034fdd Mon Sep 17 00:00:00 2001 From: Mega Yu Date: Thu, 5 Mar 2026 22:39:47 +0800 Subject: [PATCH 06/21] feat: add support for Template, Math blocks and Cards in exporter --- .../src/logseq/graph_parser/exporter.cljs | 41 ++++++++++++++++++- .../logseq/graph_parser/exporter_test.cljs | 29 ++++++++++--- .../journals/2026_03_01.md | 4 +- 3 files changed, 66 insertions(+), 8 deletions(-) diff --git a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs index 6070b472d2..e8468d0859 100644 --- a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs @@ -693,7 +693,6 @@ (get-page-uuid page-names-to-uuids ((some-fn ::original-name :block/name) block) {:block block}) (:block/uuid block)) properties-text-values)) - ;; TODO: Add import support for :template. Ignore for now as they cause invalid property types (if (contains? props :template) {} (let [props' (-> (update-built-in-property-values @@ -782,6 +781,8 @@ (cond-> block true (merge block-properties) + (:template properties') + (update :block/tags (fnil conj []) :logseq.class/Template) (seq classes-from-properties) ;; Add a map of {:block.temp/new-class TAG} to be processed later (update :block/tags @@ -944,6 +945,7 @@ use in build-block-tx. This walk is only done once for perf reasons" [config ast-blocks] (let [results (atom {:simple-queries [] + :cards [] :asset-links [] :embeds [] :zotero-imported-files {} @@ -980,6 +982,10 @@ parsed-path (common-util/safe-read-string relative-path)] (when (string? parsed-path) (swap! results update :zotero-linked-files conj parsed-path))) + (and (vector? x) + (= "Macro" (first x)) + (= "cards" (:name (second x)))) + (swap! results update :cards conj x) (and (vector? x) (= "Macro" (first x)) (= "query" (:name (second x)))) @@ -1032,7 +1038,20 @@ (assoc :block/collapsed? true)))] {:block block' :pvalues-tx pvalues-tx'}) - {:block block}))) + (if-let [cards-macro (first (:cards walked-ast-blocks))] + (let [query (-> (:arguments (second cards-macro)) first string/trim) + props {:logseq.property/query query} + {:keys [block-properties pvalues-tx]} + (build-properties-and-values props db page-names-to-uuids + (select-keys block [:block/properties-text-values :block/name :block/title :block/uuid]) + options) + block' + (-> (update block :block/tags (fnil conj []) :logseq.class/Cards) + (merge block-properties + {:block/title (string/trim (string/replace-first title #"\{\{cards(.*)\}\}" ""))}))] + {:block block' + :pvalues-tx pvalues-tx}) + {:block block})))) (defn- handle-block-properties "Does everything page properties does and updates a couple of block specific attributes" @@ -1390,6 +1409,23 @@ :block/tags [:logseq.class/Quote-block]}) block)) +(defn- handle-math + "If a block's entire content is a single displayed math formula, convert to #Math-block node. + Detects blocks whose title is entirely delimited by $$ markers." + [block _opts] + (let [title (string/trim (:block/title block))] + (if (and (string/starts-with? title "$$") + (string/ends-with? title "$$") + (> (count title) 4) + ;; ensure there's no nested $$ pair (i.e. not two separate inline formulas) + (not (string/includes? (subs title 2 (- (count title) 2)) "$$"))) + (let [math-content (string/trim (subs title 2 (- (count title) 2)))] + (merge block + {:block/title math-content + :logseq.property.node/display-type :math + :block/tags [:logseq.class/Math-block]})) + block))) + (defn- split-title-by-code-fences "Parses a block title string line-by-line, splitting into non-code text parts and code fence segments. All code fences are extracted regardless of whether @@ -1492,6 +1528,7 @@ (update-block-tags db (:user-options options) per-file-state (:all-idents import-state)) (handle-embeds page-names-to-uuids walked-ast-blocks (select-keys options [:log-fn])) (handle-quotes (select-keys options [:log-fn])) + (handle-math (select-keys options [:log-fn])) (update-block-marker options) (update-block-priority options) add-missing-timestamps diff --git a/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs b/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs index 4386038c02..19a7be0cab 100644 --- a/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs +++ b/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs @@ -222,6 +222,10 @@ (is (= 5 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Task]] @conn)))) (is (= 4 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Query]] @conn)))) (is (= 2 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Card]] @conn)))) + (is (= 1 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Cards]] @conn)))) + (is (= 2 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Code-block]] @conn)))) + (is (= 1 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Math-block]] @conn)))) + (is (= 1 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Template]] @conn)))) (is (= 5 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Quote-block]] @conn)))) (is (= 7 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Pdf-annotation]] @conn)))) @@ -331,11 +335,6 @@ (mapv :db/id (:block/refs (db-test/find-block-by-content @conn #"ref to")))) "block with a block-ref has correct :block/refs") - (let [b (db-test/find-block-by-content @conn #"MEETING TITLE")] - (is (= {} - (and b (db-test/readable-properties b))) - ":template properties are ignored to not invalidate its property types")) - (is (= 20221126 (-> (db-test/readable-properties (db-test/find-block-by-content @conn "only deadline")) :logseq.property/deadline @@ -418,6 +417,26 @@ (db-test/readable-properties (db-test/find-block-by-content @conn "card 1"))) "None of the card properties are imported since they are deprecated") + ;; Cards (flashcard browser) + (is (= {:block/tags [:logseq.class/Cards] + :logseq.property/query "(tags #Card)"} + (db-test/readable-properties (find-block-by-property-value @conn :logseq.property/query "(tags #Card)"))) + "cards macro block has correct Cards class and query property") + + ;; Math blocks + (is (= {:block/tags [:logseq.class/Math-block] + :logseq.property.node/display-type :math} + (db-test/readable-properties (db-test/find-block-by-content @conn "E=mc^2"))) + "Math block has correct Math-block class and display-type") + (is (= "E=mc^2" (:block/title (db-test/find-block-by-content @conn "E=mc^2"))) + "Math block title has delimiters stripped") + + ;; Templates + (let [b (db-test/find-block-by-content @conn #"MEETING TITLE")] + (is (= {:block/tags [:logseq.class/Template]} + (and b (db-test/readable-properties b))) + "Template block is tagged as Template class")) + ;; Assets (is (= {:block/tags [:logseq.class/Asset] :logseq.property.asset/type "png" diff --git a/deps/graph-parser/test/resources/exporter-test-graph/journals/2026_03_01.md b/deps/graph-parser/test/resources/exporter-test-graph/journals/2026_03_01.md index 242739f931..a7d0b41a98 100644 --- a/deps/graph-parser/test/resources/exporter-test-graph/journals/2026_03_01.md +++ b/deps/graph-parser/test/resources/exporter-test-graph/journals/2026_03_01.md @@ -30,4 +30,6 @@ echo "bye" exit ``` - after code snippet \ No newline at end of file + after code snippet +- $$E=mc^2$$ +- {{cards (tags #Card)}} \ No newline at end of file From d8b85171e97d45559fc6bce0a238f48ee5d64a87 Mon Sep 17 00:00:00 2001 From: megayu Date: Fri, 6 Mar 2026 09:56:33 +0800 Subject: [PATCH 07/21] add guard against missing/blank args of cards macro Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../src/logseq/graph_parser/exporter.cljs | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs index e8468d0859..53f41b1343 100644 --- a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs @@ -1039,18 +1039,22 @@ {:block block' :pvalues-tx pvalues-tx'}) (if-let [cards-macro (first (:cards walked-ast-blocks))] - (let [query (-> (:arguments (second cards-macro)) first string/trim) - props {:logseq.property/query query} - {:keys [block-properties pvalues-tx]} - (build-properties-and-values props db page-names-to-uuids - (select-keys block [:block/properties-text-values :block/name :block/title :block/uuid]) - options) - block' - (-> (update block :block/tags (fnil conj []) :logseq.class/Cards) - (merge block-properties - {:block/title (string/trim (string/replace-first title #"\{\{cards(.*)\}\}" ""))}))] - {:block block' - :pvalues-tx pvalues-tx}) + (if-let [raw-query (some-> cards-macro second :arguments first)] + (let [query (string/trim raw-query)] + (if (string/blank? query) + {:block block} + (let [props {:logseq.property/query query} + {:keys [block-properties pvalues-tx]} + (build-properties-and-values props db page-names-to-uuids + (select-keys block [:block/properties-text-values :block/name :block/title :block/uuid]) + options) + block' + (-> (update block :block/tags (fnil conj []) :logseq.class/Cards) + (merge block-properties + {:block/title (string/trim (string/replace-first title #"\{\{cards(.*)\}\}" ""))))]] + {:block block' + :pvalues-tx pvalues-tx}))) + {:block block}) {:block block})))) (defn- handle-block-properties From 304bc7c9af2b5f08976f4abd7b78ad7664e22505 Mon Sep 17 00:00:00 2001 From: Mega Yu Date: Fri, 6 Mar 2026 10:12:51 +0800 Subject: [PATCH 08/21] add trim to the lang in the code snippet --- .../src/logseq/graph_parser/exporter.cljs | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs index 53f41b1343..6f17aad87f 100644 --- a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs @@ -953,9 +953,9 @@ (walk/prewalk (fn [x] (cond - (and (vector? x) - (= "Link" (first x)) - (let [path-or-map (second (:url (second x)))] + (and (vector? x) + (= "Link" (first x)) + (let [path-or-map (second (:url (second x)))] (cond (string? path-or-map) (or (common-config/local-relative-asset? path-or-map) @@ -969,19 +969,19 @@ (= "Macro" (first x)) (= "embed" (:name (second x)))) (swap! results update :embeds conj x) - (and (vector? x) - (= "Macro" (first x)) - (= "zotero-imported-file" (:name (second x)))) - (let [[item-key filename] (:arguments (second x))] - (when (and item-key filename) - (swap! results update :zotero-imported-files assoc item-key (common-util/safe-read-string filename)))) - (and (vector? x) - (= "Macro" (first x)) - (= "zotero-linked-file" (:name (second x)))) - (let [[relative-path] (:arguments (second x)) - parsed-path (common-util/safe-read-string relative-path)] - (when (string? parsed-path) - (swap! results update :zotero-linked-files conj parsed-path))) + (and (vector? x) + (= "Macro" (first x)) + (= "zotero-imported-file" (:name (second x)))) + (let [[item-key filename] (:arguments (second x))] + (when (and item-key filename) + (swap! results update :zotero-imported-files assoc item-key (common-util/safe-read-string filename)))) + (and (vector? x) + (= "Macro" (first x)) + (= "zotero-linked-file" (:name (second x)))) + (let [[relative-path] (:arguments (second x)) + parsed-path (common-util/safe-read-string relative-path)] + (when (string? parsed-path) + (swap! results update :zotero-linked-files conj parsed-path))) (and (vector? x) (= "Macro" (first x)) (= "cards" (:name (second x)))) @@ -1051,7 +1051,7 @@ block' (-> (update block :block/tags (fnil conj []) :logseq.class/Cards) (merge block-properties - {:block/title (string/trim (string/replace-first title #"\{\{cards(.*)\}\}" ""))))]] + {:block/title (string/trim (string/replace-first title #"\{\{cards(.*)\}\}" ""))}))] {:block block' :pvalues-tx pvalues-tx}))) {:block block}) @@ -1449,11 +1449,12 @@ text-parts) :code-segs code-segs} (let [line (first remaining) - fence-start? (and (not in-code?) (re-matches #"```.*" line)) - fence-end? (and in-code? (= line "```"))] + trimmed-line (string/trim line) + fence-start? (and (not in-code?) (re-matches #"```.*" trimmed-line)) + fence-end? (and in-code? (= trimmed-line "```"))] (cond fence-start? - (recur (rest remaining) true (not-empty (subs line 3)) [] + (recur (rest remaining) true (not-empty (subs trimmed-line 3)) [] (if (seq current) (conj text-parts (string/join "\n" current)) text-parts) From 7522a879396ab0fb44285ff2d968626feab0c433 Mon Sep 17 00:00:00 2001 From: Mega Yu Date: Fri, 6 Mar 2026 11:26:55 +0800 Subject: [PATCH 09/21] skip heavier parsing if code fences in the title add split-title-by-code-fences test --- .../src/logseq/graph_parser/exporter.cljs | 61 ++++++++++------- .../logseq/graph_parser/exporter_test.cljs | 65 +++++++++++++++++++ 2 files changed, 103 insertions(+), 23 deletions(-) diff --git a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs index 6f17aad87f..4027d24ee1 100644 --- a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs @@ -1508,6 +1508,17 @@ block)) block)) +(defn- at-least-two? + [s substr] + (if (empty? substr) + false + (loop [start 0 cnt 0] + (let [idx (string/index-of s substr start)] + (cond + (>= cnt 2) true + (nil? idx) false + :else (recur (+ idx (count substr)) (inc cnt))))))) + (defn- (assoc block' - :block/title text - :block/tags [:logseq.class/Code-block] - :logseq.property.node/display-type :code) - lang (assoc :logseq.property.code/lang lang)) - []]) - has-mixed-content? - (let [remaining-title (-> (string/join "\n" text-parts) - (string/replace #"\n{2,}" "\n") - string/trim) - updated-block (assoc block' :block/title remaining-title) - code-children (build-code-snippet-child-blocks updated-block code-segs)] - [updated-block code-children]) - :else + (if has-fence? + (let [{:keys [text-parts code-segs]} (split-title-by-code-fences title) + pure-single-code? (and (= 1 (count code-segs)) + (every? string/blank? text-parts)) + has-mixed-content? (and extract? + (seq code-segs) + (some #(not (string/blank? %)) text-parts))] + (cond + pure-single-code? + (let [{:keys [text lang]} (first code-segs)] + [(cond-> (assoc block' + :block/title text + :block/tags [:logseq.class/Code-block] + :logseq.property.node/display-type :code) + lang (assoc :logseq.property.code/lang lang)) + []]) + has-mixed-content? + (let [remaining-title (-> (string/join "\n" text-parts) + (string/replace #"\n{2,}" "\n") + string/trim) + updated-block (assoc block' :block/title remaining-title) + code-children (build-code-snippet-child-blocks updated-block code-segs)] + [updated-block code-children]) + :else + [block' []])) [block' []])] ;; Order matters as previous txs are referenced in block (concat properties-tx deadline-properties-tx asset-blocks-tx [final-block] code-children-tx)))) diff --git a/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs b/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs index 19a7be0cab..f82b0d1e50 100644 --- a/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs +++ b/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs @@ -968,6 +968,71 @@ (:logseq.property.journal/title-format (d/entity @conn :logseq.class/Journal))) "title format set correctly by config"))) +(deftest split-title-by-code-fences + (let [split-fn #'gp-exporter/split-title-by-code-fences] + (testing "standalone code fence with language" + (is (= {:text-parts [] + :code-segs [{:text "it's an individual code snippet with language tag" + :lang "markdown"}]} + (split-fn "```markdown\nit's an individual code snippet with language tag\n```")))) + + (testing "standalone code fence without language" + (is (= {:text-parts [] + :code-segs [{:text "it's an individual code snippet without language tag" + :lang nil}]} + (split-fn "```\nit's an individual code snippet without language tag\n```")))) + + (testing "one code fence with leading text" + (is (= {:text-parts ["before code snippet"] + :code-segs [{:text "echo \"ok\"\nexit" + :lang nil}]} + (split-fn "before code snippet\n```\necho \"ok\"\nexit\n```")))) + + (testing "one code fence with leading and trailing text" + (is (= {:text-parts ["before code snippet" "after code snippet"] + :code-segs [{:text "echo \"ok\"\nexit" + :lang "bash"}]} + (split-fn "before code snippet\n```bash\necho \"ok\"\nexit\n```\nafter code snippet")))) + + (testing "one code fence followed by trailing text" + (is (= {:text-parts ["after code snippet"] + :code-segs [{:text "echo \"ok\"\nexit" + :lang "bash"}]} + (split-fn "```bash\necho \"ok\"\nexit\n```\nafter code snippet")))) + + (testing "multiple code fences mixed with text" + (is (= {:text-parts ["before code snippet" "middle" "after code snippet"] + :code-segs [{:text "echo \"ok\"\nexit" + :lang "bash"} + {:text "echo \"bye\"\nexit" + :lang "bash"}]} + (split-fn "before code snippet\n```bash\necho \"ok\"\nexit\n```\nmiddle\n```bash\necho \"bye\"\nexit\n```\nafter code snippet")))) + + (testing "edge: one code fence followed by opening fence without closing fence" + (is (= {:text-parts ["echo \"missing end fence\""] ;; no "```bash" ahead, it's fine as is; let's leave it + :code-segs [{:text "echo \"ok\"\nexit" + :lang "bash"}]} + (split-fn "```bash\necho \"ok\"\nexit\n```\n```bash\necho \"missing end fence\"")))) + + (testing "edge: pure multiple code fences with no extra text" + (let [{:keys [text-parts code-segs]} (split-fn "```markdown\n1st code snippet with language tag\n```\n```\n2nd code snippet without language tag\n```")] + (is (and (empty? text-parts) (> (count code-segs) 1)) "not pure single code and no mixed content"))) + + (testing "edge: opening fence without closing fence" + (let [title "```bash\necho \"missing end fence\"" + {:keys [text-parts code-segs]} (split-fn title)] + (is (and (= (count text-parts) 1) (not= (first text-parts) title) (empty? code-segs)) "not pure single code and no mixed content"))) + + (testing "edge: plain text without any code fence" + (is (= {:text-parts ["plain text only"] + :code-segs []} + (split-fn "plain text only")))) + + (testing "edge: empty title" + (is (= {:text-parts [""] + :code-segs []} + (split-fn "")))))) + (deftest-async export-docs-graph-with-extract-code-snippet (p/let [file-graph-dir "test/resources/exporter-test-graph" conn (db-test/create-conn) From 5f7885dff52c3be3d6b04fe3079abcc9010040b6 Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Fri, 6 Mar 2026 16:29:26 -0500 Subject: [PATCH 10/21] fix: outdated docs and misc cleanup Also added the new option to the import CLI, improved tests that only test one file to only import 1 file and cleaned up some hard to read code --- deps/graph-parser/script/db_import.cljs | 3 + .../src/logseq/graph_parser/exporter.cljs | 109 +++++++++--------- .../logseq/graph_parser/exporter_test.cljs | 14 +-- 3 files changed, 67 insertions(+), 59 deletions(-) diff --git a/deps/graph-parser/script/db_import.cljs b/deps/graph-parser/script/db_import.cljs index ecfe1a4d32..851665cd05 100644 --- a/deps/graph-parser/script/db_import.cljs +++ b/deps/graph-parser/script/db_import.cljs @@ -193,6 +193,9 @@ {:alias :P :coerce [] :desc "List of properties whose values convert to a parent class"} + :extract-code-snippets? + {:alias :C + :desc "Extract code fence(s) to #Code"} :validate {:alias :V :desc "Validate db after creation"}}) diff --git a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs index 4027d24ee1..4ac664c197 100644 --- a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs @@ -995,7 +995,8 @@ @results)) (defn- handle-queries - "If a block contains a simple or advanced queries, converts block to a #Query node" + "If a block contains a simple or advanced queries, converts block to a #Query node. If a block + contains a cards query converts to a #Cards node" [{:block/keys [title] :as block} db page-names-to-uuids walked-ast-blocks options] (if-let [query (some-> (first (:simple-queries walked-ast-blocks)) (ast->text (select-keys options [:log-fn])) @@ -1039,21 +1040,18 @@ {:block block' :pvalues-tx pvalues-tx'}) (if-let [cards-macro (first (:cards walked-ast-blocks))] - (if-let [raw-query (some-> cards-macro second :arguments first)] - (let [query (string/trim raw-query)] - (if (string/blank? query) - {:block block} - (let [props {:logseq.property/query query} - {:keys [block-properties pvalues-tx]} - (build-properties-and-values props db page-names-to-uuids - (select-keys block [:block/properties-text-values :block/name :block/title :block/uuid]) - options) - block' - (-> (update block :block/tags (fnil conj []) :logseq.class/Cards) - (merge block-properties - {:block/title (string/trim (string/replace-first title #"\{\{cards(.*)\}\}" ""))}))] - {:block block' - :pvalues-tx pvalues-tx}))) + (if-let [query (some-> cards-macro second :arguments first string/trim not-empty)] + (let [props {:logseq.property/query query} + {:keys [block-properties pvalues-tx]} + (build-properties-and-values props db page-names-to-uuids + (select-keys block [:block/properties-text-values :block/name :block/title :block/uuid]) + options) + block' + (-> (update block :block/tags (fnil conj []) :logseq.class/Cards) + (merge block-properties + {:block/title (string/trim (string/replace-first title #"\{\{cards(.*)\}\}" ""))}))] + {:block block' + :pvalues-tx pvalues-tx}) {:block block}) {:block block})))) @@ -1414,9 +1412,9 @@ block)) (defn- handle-math - "If a block's entire content is a single displayed math formula, convert to #Math-block node. + "If a block's entire content is a single displayed math formula, convert to #Math node. Detects blocks whose title is entirely delimited by $$ markers." - [block _opts] + [block] (let [title (string/trim (:block/title block))] (if (and (string/starts-with? title "$$") (string/ends-with? title "$$") @@ -1519,6 +1517,44 @@ (nil? idx) false :else (recur (+ idx (count substr)) (inc cnt))))))) +(defn- handle-code-blocks + "Returns a vector of block and optional block children tx. If a block + contains code fence(s) i.e. ```, converts block to a #Code node. If user + enables :extract-code-snippets? option, multiple code fences are extracted out + of text and put into children blocks in the order they appear" + [block' options] + (let [title (:block/title block') + has-fence? (and (string? title) (at-least-two? title "```")) + extract? (get-in options [:user-options :extract-code-snippets?]) + [final-block code-children-tx] + (if has-fence? + (let [{:keys [text-parts code-segs]} (split-title-by-code-fences title) + pure-single-code? (and (= 1 (count code-segs)) + (every? string/blank? text-parts)) + has-mixed-content? (and extract? + (seq code-segs) + (some #(not (string/blank? %)) text-parts))] + (cond + pure-single-code? + (let [{:keys [text lang]} (first code-segs)] + [(cond-> (assoc block' + :block/title text + :block/tags [:logseq.class/Code-block] + :logseq.property.node/display-type :code) + lang (assoc :logseq.property.code/lang lang)) + []]) + has-mixed-content? + (let [remaining-title (-> (string/join "\n" text-parts) + (string/replace #"\n{2,}" "\n") + string/trim) + updated-block (assoc block' :block/title remaining-title) + code-children (build-code-snippet-child-blocks updated-block code-segs)] + [updated-block code-children]) + :else + [block' []])) + [block' []])] + [final-block code-children-tx])) + (defn- (assoc block' - :block/title text - :block/tags [:logseq.class/Code-block] - :logseq.property.node/display-type :code) - lang (assoc :logseq.property.code/lang lang)) - []]) - has-mixed-content? - (let [remaining-title (-> (string/join "\n" text-parts) - (string/replace #"\n{2,}" "\n") - string/trim) - updated-block (assoc block' :block/title remaining-title) - code-children (build-code-snippet-child-blocks updated-block code-segs)] - [updated-block code-children]) - :else - [block' []])) - [block' []])] + (let [[final-block code-children-tx] (handle-code-blocks block' options)] ;; Order matters as previous txs are referenced in block (concat properties-tx deadline-properties-tx asset-blocks-tx [final-block] code-children-tx)))) @@ -2050,7 +2055,7 @@ * :extract-options - Options map to pass to extract/extract * :user-options - User provided options maps that alter how a file is converted to db graph. Current options are: :tag-classes (set), :property-classes (set), :property-parent-classes (set), :convert-all-tags? (boolean) - and :remove-inline-tags? (boolean) + :remove-inline-tags? (boolean), :extract-code-snippets? (boolean) * :import-state - useful import state to maintain across files e.g. property schemas or ignored properties * :macros - map of macros for use with macro expansion * :notify-user - Displays warnings to user without failing the import. Fn receives a map with :msg diff --git a/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs b/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs index f82b0d1e50..bd34265ecd 100644 --- a/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs +++ b/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs @@ -412,7 +412,7 @@ (:block/title (db-test/find-block-by-content @conn #"tasks with todo"))) "Advanced query has custom title migrated") - ;; Cards + ;; Card (is (= {:block/tags [:logseq.class/Card]} (db-test/readable-properties (db-test/find-block-by-content @conn "card 1"))) "None of the card properties are imported since they are deprecated") @@ -1033,11 +1033,11 @@ :code-segs []} (split-fn "")))))) -(deftest-async export-docs-graph-with-extract-code-snippet +(deftest-async export-files-with-extract-code-snippet (p/let [file-graph-dir "test/resources/exporter-test-graph" + files (mapv #(node-path/join file-graph-dir %) ["journals/2026_03_01.md"]) conn (db-test/create-conn) - _ (db-pipeline/add-listener conn) - _ (import-file-graph-to-db file-graph-dir conn {:extract-code-snippets? true}) + _ (import-files-to-db files conn {:extract-code-snippets? true}) journal-page-eid (d/q '[:find ?p . :where [?p :block/journal-day 20260301]] @conn) top-blocks (->> (d/q '[:find [?b ...] :in $ ?page @@ -1135,11 +1135,11 @@ (is (every? #(= "bash" (:logseq.property.code/lang %)) children) "Both child blocks have bash language property"))))) -(deftest-async export-docs-graph-without-extract-code-snippet +(deftest-async export-files-without-extract-code-snippet (p/let [file-graph-dir "test/resources/exporter-test-graph" + files (mapv #(node-path/join file-graph-dir %) ["journals/2026_03_01.md"]) conn (db-test/create-conn) - _ (db-pipeline/add-listener conn) - _ (import-file-graph-to-db file-graph-dir conn {}) + _ (import-files-to-db files conn {:extract-code-snippets? false}) journal-page-eid (d/q '[:find ?p . :where [?p :block/journal-day 20260301]] @conn) top-blocks (->> (d/q '[:find [?b ...] :in $ ?page From 935673a86078ccb5e790166bd9ce5065ec171c26 Mon Sep 17 00:00:00 2001 From: Mega Yu Date: Tue, 10 Mar 2026 13:40:42 +0800 Subject: [PATCH 11/21] update path handling in exporter tests to make it compatible with Windows path --- .../logseq/graph_parser/exporter_test.cljs | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs b/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs index bd34265ecd..1909a18e74 100644 --- a/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs +++ b/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs @@ -7,6 +7,7 @@ [datascript.core :as d] [logseq.common.config :as common-config] [logseq.common.graph :as common-graph] + [logseq.common.path :as path] [logseq.common.util.date-time :as date-time-util] [logseq.db :as ldb] [logseq.db.common.entity-plus :as entity-plus] @@ -54,8 +55,8 @@ [dir*] (let [dir (node-path/resolve dir*)] (->> (common-graph/get-files dir) - (concat (when (fs/existsSync (node-path/join dir* "assets")) - (common-graph/readdir (node-path/join dir* "assets")))) + (concat (when (fs/existsSync (path/path-join dir* "assets")) + (common-graph/readdir (path/path-join dir* "assets")))) (mapv #(hash-map :path % ::rpath (node-path/relative dir* %)))))) @@ -826,7 +827,7 @@ (deftest-async export-files-with-tag-classes-option (p/let [file-graph-dir "test/resources/exporter-test-graph" - files (mapv #(node-path/join file-graph-dir %) ["journals/2024_02_07.md" "pages/Interstellar.md"]) + files (mapv #(path/path-join file-graph-dir %) ["journals/2024_02_07.md" "pages/Interstellar.md"]) conn (db-test/create-conn) _ (import-files-to-db files conn {:tag-classes ["movie"]})] (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn)))) @@ -852,7 +853,7 @@ (deftest-async export-files-with-property-classes-option (p/let [file-graph-dir "test/resources/exporter-test-graph" - files (mapv #(node-path/join file-graph-dir %) + files (mapv #(path/path-join file-graph-dir %) ["journals/2024_02_23.md" "pages/url.md" "pages/Whiteboard___Tool.md" "pages/Whiteboard___Arrow_head_toggle.md" "pages/Library.md"]) @@ -899,7 +900,7 @@ (deftest-async export-files-with-remove-inline-tags (p/let [file-graph-dir "test/resources/exporter-test-graph" - files (mapv #(node-path/join file-graph-dir %) ["journals/2024_02_07.md" + files (mapv #(path/path-join file-graph-dir %) ["journals/2024_02_07.md" "journals/2026_01_27.md"]) conn (db-test/create-conn) _ (import-files-to-db files conn {:remove-inline-tags? false :convert-all-tags? true})] @@ -915,7 +916,7 @@ (deftest-async export-files-with-ignored-properties (p/let [file-graph-dir "test/resources/exporter-test-graph" - files (mapv #(node-path/join file-graph-dir %) ["ignored/icon-page.md"]) + files (mapv #(path/path-join file-graph-dir %) ["ignored/icon-page.md"]) conn (db-test/create-conn) {:keys [import-state]} (import-files-to-db files conn {})] (is (= 2 @@ -924,7 +925,7 @@ (deftest-async export-files-with-property-parent-classes-option (p/let [file-graph-dir "test/resources/exporter-test-graph" - files (mapv #(node-path/join file-graph-dir %) ["journals/2024_11_26.md" + files (mapv #(path/path-join file-graph-dir %) ["journals/2024_11_26.md" "pages/CreativeWork.md" "pages/Movie.md" "pages/type.md" "pages/Whiteboard___Tool.md" "pages/Whiteboard___Arrow_head_toggle.md" "pages/Property.md" "pages/url.md"]) @@ -952,7 +953,7 @@ (deftest-async export-files-with-property-pages-disabled (p/let [file-graph-dir "test/resources/exporter-test-graph" ;; any page with properties - files (mapv #(node-path/join file-graph-dir %) ["journals/2024_01_17.md"]) + files (mapv #(path/path-join file-graph-dir %) ["journals/2024_01_17.md"]) conn (db-test/create-conn) _ (import-files-to-db files conn {:user-config {:property-pages/enabled? false :property-pages/excludelist #{:prop-string}}})] @@ -1035,7 +1036,7 @@ (deftest-async export-files-with-extract-code-snippet (p/let [file-graph-dir "test/resources/exporter-test-graph" - files (mapv #(node-path/join file-graph-dir %) ["journals/2026_03_01.md"]) + files (mapv #(path/path-join file-graph-dir %) ["journals/2026_03_01.md"]) conn (db-test/create-conn) _ (import-files-to-db files conn {:extract-code-snippets? true}) journal-page-eid (d/q '[:find ?p . :where [?p :block/journal-day 20260301]] @conn) @@ -1137,7 +1138,7 @@ (deftest-async export-files-without-extract-code-snippet (p/let [file-graph-dir "test/resources/exporter-test-graph" - files (mapv #(node-path/join file-graph-dir %) ["journals/2026_03_01.md"]) + files (mapv #(path/path-join file-graph-dir %) ["journals/2026_03_01.md"]) conn (db-test/create-conn) _ (import-files-to-db files conn {:extract-code-snippets? false}) journal-page-eid (d/q '[:find ?p . :where [?p :block/journal-day 20260301]] @conn) From 5bc154aacbbf214542f807d2070ee7861d1e953a Mon Sep 17 00:00:00 2001 From: Mega Yu Date: Tue, 10 Mar 2026 13:41:10 +0800 Subject: [PATCH 12/21] chore: update test helpers in config for async tests --- .clj-kondo/config.edn | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index dd67810e36..5d9b45593a 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -228,8 +228,10 @@ clojure.test.check.properties/for-all clojure.core/for ;; src/main frontend.namespaces/import-vars potemkin/import-vars - ;; src/test + ;; src/test and deps tests frontend.test.helper/deftest-async clojure.test/deftest + logseq.graph-parser.test.helper/deftest-async clojure.test/deftest + logseq.publishing.test.helper/deftest-async clojure.test/deftest frontend.worker.rtc.idb-keyval-mock/with-reset-idb-keyval-mock cljs.test/async frontend.react/defc clojure.core/defn logseq.common.defkeywords/defkeyword cljs.spec.alpha/def From 371b1d5301f777a439b3969a4ade9444c9ba71b9 Mon Sep 17 00:00:00 2001 From: Mega Yu Date: Tue, 10 Mar 2026 18:57:44 +0800 Subject: [PATCH 13/21] feat(exporter): support template import --- .../src/logseq/graph_parser/exporter.cljs | 177 +++++++++++-- .../logseq/graph_parser/exporter_test.cljs | 239 +++++++++++++++++- .../journals/2024_02_16.md | 35 +++ 3 files changed, 416 insertions(+), 35 deletions(-) diff --git a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs index 4ac664c197..0af21987cd 100644 --- a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs @@ -57,6 +57,126 @@ :block/title new-title :block/name (common-util/page-name-sanity-lc new-title)}))) +(def template-file-property-names #{:template :template-including-parent}) + +(defn- get-template-name + [block] + (let [template-name (get-in block [:block/properties :template])] + (when (string? template-name) + (not-empty (string/trim template-name))))) + +(defn- template-including-parent? + [block] + (not= false (get-in block [:block/properties :template-including-parent]))) + +(defn- remove-template-property-lines + [title] + (if (string? title) + (->> (string/split-lines title) + (remove (fn [line] + (let [trimmed-line (string/triml line)] + (or (string/starts-with? trimmed-line "template::") + (string/starts-with? trimmed-line "template-including-parent::"))))) + (string/join "\n")) + title)) + +(defn- strip-template-properties + [block] + (cond-> block + (:block/title block) + (update :block/title remove-template-property-lines) + + (seq (:block/properties block)) + (update :block/properties #(apply dissoc % template-file-property-names)) + + (seq (:block/properties-text-values block)) + (update :block/properties-text-values #(apply dissoc % template-file-property-names)) + + (seq (:block/properties-order block)) + (update :block/properties-order (fn [props-order] (vec (remove template-file-property-names props-order)))))) + +(defn- group-block-children-by-parent + [blocks] + (reduce (fn [result {:block/keys [parent uuid]}] + (if (and (vector? parent) + (= :block/uuid (first parent))) + (update result (second parent) (fnil conj []) uuid) + result)) + {} + blocks)) + +(defn- get-block-subtree-uuids + [block-children root-uuid] + (loop [queue [root-uuid] + result []] + (if-let [current-uuid (first queue)] + (recur (into (vec (rest queue)) (get block-children current-uuid)) + (conj result current-uuid)) + result))) + +(defn- clone-template-content-blocks + [blocks-by-uuid block-children template-page-uuid template-root-uuid content-root-uuids] + (let [*cloned-uuids (atom #{}) + clone-block + (fn clone-block [source-uuid parent-uuid] + (let [source-block (get blocks-by-uuid source-uuid) + cloned-uuid (common-uuid/gen-uuid) + cloned-block (-> source-block + strip-template-properties + (assoc :block/uuid cloned-uuid + :block/page [:block/uuid template-page-uuid] + :block/parent [:block/uuid parent-uuid]) + (dissoc :db/id))] + (swap! *cloned-uuids conj cloned-uuid) + (cons cloned-block + (mapcat #(clone-block % cloned-uuid) + (get block-children source-uuid)))))] + {:blocks (vec (mapcat #(clone-block % template-root-uuid) content-root-uuids)) + :preserve-empty-properties-uuids @*cloned-uuids})) + +(defn- extract-template-blocks + [db blocks] + (let [template-page (d/entity db :logseq.class/Template) + template-page-uuid (:block/uuid template-page) + blocks-by-uuid (into {} (map (juxt :block/uuid identity) blocks)) + block-children (group-block-children-by-parent blocks)] + (reduce + (fn [{:keys [blocks template-blocks preserve-empty-properties-uuids]} block] + (let [template-name (get-template-name block) + cleaned-block (strip-template-properties block) + source-preserve-empty-properties-uuids + (if template-name + (set (get-block-subtree-uuids block-children (:block/uuid block))) + #{}) + content-root-uuids (if (template-including-parent? block) + [(:block/uuid block)] + (get block-children (:block/uuid block))) + template-root-block + (when template-name + {:block/uuid (common-uuid/gen-uuid) + :block/title template-name + :block/page [:block/uuid template-page-uuid] + :block/parent [:block/uuid template-page-uuid] + :block/order (db-order/gen-key) + :block/format (:block/format block) + :block/tags [:logseq.class/Template]}) + template-content + (if template-root-block + (clone-template-content-blocks blocks-by-uuid block-children template-page-uuid (:block/uuid template-root-block) content-root-uuids) + {:blocks [] + :preserve-empty-properties-uuids #{}})] + {:blocks (conj blocks cleaned-block) + :template-blocks (cond-> template-blocks + template-root-block + (into (into [template-root-block] (:blocks template-content)))) + :preserve-empty-properties-uuids (set/union preserve-empty-properties-uuids + source-preserve-empty-properties-uuids + (:preserve-empty-properties-uuids template-content))})) + {:blocks [] + :template-blocks [] + :preserve-empty-properties-uuids #{}} + blocks))) + (defn- get-page-uuid [page-names-to-uuids page-name ex-data'] (or (get @page-names-to-uuids (some-> (if (string/includes? (str page-name) "#") (string/lower-case (gp-block/sanitize-hashtag-name page-name)) @@ -684,7 +804,7 @@ {:keys [import-state user-options] :as options}] (let [{:keys [all-idents property-schemas]} import-state get-ident' #(get-ident @all-idents %) - user-properties (apply dissoc props file-built-in-property-names)] + user-properties (apply dissoc props (concat file-built-in-property-names template-file-property-names))] (when (seq user-properties) (swap! (:block-properties-text-values import-state) assoc @@ -693,20 +813,18 @@ (get-page-uuid page-names-to-uuids ((some-fn ::original-name :block/name) block) {:block block}) (:block/uuid block)) properties-text-values)) - (if (contains? props :template) - {} - (let [props' (-> (update-built-in-property-values - (select-keys props file-built-in-property-names) - page-names-to-uuids - (select-keys import-state [:ignored-properties :all-idents]) - (select-keys block [:block/name :block/title]) - (select-keys user-options [:property-classes])) - (merge (update-user-property-values user-properties page-names-to-uuids properties-text-values import-state options))) - pvalue-tx-m (->property-value-tx-m block props' #(get-property-schema @property-schemas %) @all-idents) - block-properties (-> (merge props' (db-property-build/build-properties-with-ref-values pvalue-tx-m)) - (update-keys get-ident'))] - {:block-properties block-properties - :pvalues-tx (into (mapcat #(if (set? %) % [%]) (vals pvalue-tx-m)))})))) + (let [props' (-> (update-built-in-property-values + (select-keys props file-built-in-property-names) + page-names-to-uuids + (select-keys import-state [:ignored-properties :all-idents]) + (select-keys block [:block/name :block/title]) + (select-keys user-options [:property-classes])) + (merge (update-user-property-values user-properties page-names-to-uuids properties-text-values import-state options))) + pvalue-tx-m (->property-value-tx-m block props' #(get-property-schema @property-schemas %) @all-idents) + block-properties (-> (merge props' (db-property-build/build-properties-with-ref-values pvalue-tx-m)) + (update-keys get-ident'))] + {:block-properties block-properties + :pvalues-tx (into (mapcat #(if (set? %) % [%]) (vals pvalue-tx-m)))}))) (def ignored-built-in-properties "Ignore built-in properties that are already imported or not supported in db graphs" @@ -725,8 +843,9 @@ (defn- pre-update-properties "Updates page and block properties before their property types are inferred" - [properties class-related-properties] + [properties class-related-properties {:keys [preserve-empty-properties?]}] (let [dissoced-props (concat ignored-built-in-properties + template-file-property-names ;; TODO: Deal with these dissoced built-in properties [:title :created-at :updated-at] class-related-properties)] @@ -735,8 +854,9 @@ (if (not (contains? file-built-in-property-names prop)) ;; only update user properties (if (string? val) - ;; Ignore blank values as they were usually generated by templates - (when-not (string/blank? val) + ;; Ignore blank values outside template-related blocks to preserve existing import behavior + (when (or preserve-empty-properties? + (not (string/blank? val))) [prop ;; handle float strings b/c graph-parser doesn't (or (parse-double val) val)]) @@ -756,14 +876,15 @@ :keys [import-state macros] :as options}] (-> (if (seq properties) - (let [classes-from-properties (->> (select-keys properties property-classes) + (let [preserve-empty-properties? (contains? (or (:preserve-empty-property-block-uuids options) #{}) + (:block/uuid block)) + classes-from-properties (->> (select-keys properties property-classes) (mapcat (fn [[_k v]] (if (coll? v) v [v]))) distinct) - properties' (pre-update-properties properties (into property-classes property-parent-classes)) - properties-to-infer (if (:template properties') - ;; Ignore template properties as they don't consistently have representative property values - {} - (apply dissoc properties' file-built-in-property-names)) + properties' (pre-update-properties properties + (into property-classes property-parent-classes) + {:preserve-empty-properties? preserve-empty-properties?}) + properties-to-infer (apply dissoc properties' file-built-in-property-names) property-changes (->> properties-to-infer (keep (fn [[prop val]] @@ -781,8 +902,6 @@ (cond-> block true (merge block-properties) - (:template properties') - (update :block/tags (fnil conj []) :logseq.class/Template) (seq classes-from-properties) ;; Add a map of {:block.temp/new-class TAG} to be processed later (update :block/tags @@ -2066,8 +2185,12 @@ :as *options}] (p/let [options (assoc *options :notify-user notify-user :log-fn log-fn :file file) {:keys [pages blocks]} (extract-pages-and-blocks @conn file content options) + {:keys [blocks template-blocks preserve-empty-properties-uuids]} + (extract-template-blocks @conn blocks) + blocks (into blocks template-blocks) tx-options (merge (build-tx-options options) - {:journal-created-ats (build-journal-created-ats pages)}) + {:journal-created-ats (build-journal-created-ats pages) + :preserve-empty-property-block-uuids preserve-empty-properties-uuids}) old-properties (keys @(get-in options [:import-state :property-schemas])) ;; Build page and block txs {:keys [pages-tx page-properties-tx per-file-state existing-pages]} (build-pages-tx conn pages blocks tx-options) diff --git a/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs b/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs index 1909a18e74..d984cc0132 100644 --- a/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs +++ b/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs @@ -48,6 +48,42 @@ first (d/entity db))) +(defn- ordered-children + [block] + (->> (:block/_parent block) + (remove :logseq.property/created-from-property) + (sort-by :block/order) + vec)) + +(defn- block-tree-with-properties + [block] + {:title (:block/title block) + :properties (dissoc (db-test/readable-properties block) :block/tags) + :children (mapv block-tree-with-properties (ordered-children block))}) + +(defn- find-template-by-title + [db title] + (some->> (d/q '[:find [?b ...] + :in $ ?title + :where + [?b :block/title ?title] + [?b :block/tags :logseq.class/Template]] + db title) + first + (d/entity db))) + +(defn- template-content-trees + [db title] + (some->> (find-template-by-title db title) + ordered-children + (mapv block-tree-with-properties))) + +(defn- journal-top-level-trees + [db journal-day] + (some->> (db-test/find-journal-by-journal-day db journal-day) + ordered-children + (mapv block-tree-with-properties))) + (defn- build-graph-files "Given a file graph directory, return all files including assets and adds relative paths on ::rpath since paths are absolute by default and exporter needs relative paths for @@ -174,6 +210,85 @@ "assets/subdir/partydino.gif"] "[[FIRST UUID]] and [[UUID]]")) +(deftest extract-template-blocks + (let [conn (db-test/create-conn) + page-uuid (random-uuid) + parent-uuid (random-uuid) + child-uuid (random-uuid) + include-children-only-uuid (random-uuid) + child-only-1-uuid (random-uuid) + child-only-2-uuid (random-uuid) + blocks [{:block/uuid parent-uuid + :block/title "source parent" + :block/page [:block/uuid page-uuid] + :block/parent {:block/uuid page-uuid} + :block/order "a" + :block/properties {:template " trimmed template " + :name ""} + :block/properties-text-values {:template " trimmed template " + :name ""} + :block/properties-order [:template :name]} + {:block/uuid child-uuid + :block/title "child" + :block/page [:block/uuid page-uuid] + :block/parent [:block/uuid parent-uuid] + :block/order "b" + :block/properties {:template "nested child" + :name "child default"} + :block/properties-text-values {:template "nested child" + :name "child default"} + :block/properties-order [:template :name]} + {:block/uuid include-children-only-uuid + :block/title "exclude source block" + :block/page [:block/uuid page-uuid] + :block/parent {:block/uuid page-uuid} + :block/order "c" + :block/properties {:template "children only" + :template-including-parent false} + :block/properties-text-values {:template "children only" + :template-including-parent "false"} + :block/properties-order [:template :template-including-parent]} + {:block/uuid child-only-1-uuid + :block/title "first child" + :block/page [:block/uuid page-uuid] + :block/parent [:block/uuid include-children-only-uuid] + :block/order "d"} + {:block/uuid child-only-2-uuid + :block/title "second child" + :block/page [:block/uuid page-uuid] + :block/parent [:block/uuid include-children-only-uuid] + :block/order "e"}] + {:keys [blocks template-blocks preserve-empty-properties-uuids]} + (#'gp-exporter/extract-template-blocks @conn blocks)] + (testing "source blocks remove template metadata and mark template subtrees to preserve empty properties" + (is (= [{:title "source parent" :properties {:name ""}} + {:title "child" :properties {:name "child default"}} + {:title "exclude source block" :properties {}} + {:title "first child" :properties nil} + {:title "second child" :properties nil}] + (mapv (fn [block] + {:title (:block/title block) + :properties (:block/properties block)}) + blocks))) + (is (= #{parent-uuid child-uuid include-children-only-uuid child-only-1-uuid child-only-2-uuid} + (set/intersection preserve-empty-properties-uuids + #{parent-uuid child-uuid include-children-only-uuid child-only-1-uuid child-only-2-uuid})))) + + (testing "template roots use trimmed names and clone the correct content" + (is (= #{"trimmed template" "nested child" "children only"} + (->> template-blocks + (filter #(some #{:logseq.class/Template} (:block/tags %))) + (map :block/title) + set))) + (is (= ["source parent" "child" "child" "first child" "second child"] + (->> template-blocks + (remove #(some #{:logseq.class/Template} (:block/tags %))) + (map :block/title)))) + (is (= 5 + (count (set/difference preserve-empty-properties-uuids + #{parent-uuid child-uuid include-children-only-uuid child-only-1-uuid child-only-2-uuid}))) + "cloned template content blocks are also marked to preserve empty properties")))) + (deftest-async ^:integration export-docs-graph-with-convert-all-tags (p/let [file-graph-dir "test/resources/docs-0.10.12" start-time (cljs.core/system-time) @@ -226,7 +341,7 @@ (is (= 1 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Cards]] @conn)))) (is (= 2 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Code-block]] @conn)))) (is (= 1 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Math-block]] @conn)))) - (is (= 1 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Template]] @conn)))) + (is (= 9 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Template]] @conn)))) (is (= 5 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Quote-block]] @conn)))) (is (= 7 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Pdf-annotation]] @conn)))) @@ -269,7 +384,7 @@ set)))) (testing "user properties" - (is (= 21 + (is (= 23 (->> @conn (d/q '[:find [(pull ?b [:db/ident]) ...] :where [?b :block/tags :logseq.class/Property]]) @@ -433,10 +548,115 @@ "Math block title has delimiters stripped") ;; Templates - (let [b (db-test/find-block-by-content @conn #"MEETING TITLE")] - (is (= {:block/tags [:logseq.class/Template]} - (and b (db-test/readable-properties b))) - "Template block is tagged as Template class")) + (is (= #{"meeting" + "title-only-no-children" + "properties-only-no-children" + "title-only-with-children" + "empty-title-with-children" + "children-only" + "nested-father" + "nested-child-1" + "nested-child-2"} + (->> (d/q '[:find [?title ...] + :where + [?b :block/tags :logseq.class/Template] + [?b :block/title ?title]] + @conn) + set)) + "All template definitions are imported as Template blocks") + (is (= [{:title "MEETING TITLE" + :properties {:user.property/participants #{"TODO"}} + :children []}] + (template-content-trees @conn "meeting"))) + (is (= [{:title "TITLE" + :properties {} + :children []}] + (template-content-trees @conn "title-only-no-children"))) + (is (= [{:title "" + :properties {:user.property/name "" + :user.property/author ""} + :children []}] + (template-content-trees @conn "properties-only-no-children"))) + (is (= [{:title "TITLE" + :properties {} + :children [{:title "intro" :properties {} :children []} + {:title "notes" :properties {} :children []}]}] + (template-content-trees @conn "title-only-with-children"))) + (is (= [{:title "" + :properties {} + :children [{:title "intro" :properties {} :children []} + {:title "notes" :properties {} :children []}]}] + (template-content-trees @conn "empty-title-with-children"))) + (is (= [{:title "intro" :properties {} :children []} + {:title "notes" :properties {} :children []}] + (template-content-trees @conn "children-only"))) + (is (= [{:title "it's a template with nested templates" + :properties {:user.property/name "you named it"} + :children [{:title "child-1" + :properties {:user.property/name ""} + :children [{:title "child-1-1" + :properties {:user.property/name ""} + :children []}]} + {:title "child-2" + :properties {:user.property/name ""} + :children [{:title "child-2-1" + :properties {:user.property/name ""} + :children []}]} + {:title "child-3" + :properties {:user.property/name ""} + :children []}]}] + (template-content-trees @conn "nested-father"))) + (is (= [{:title "child-1" + :properties {:user.property/name ""} + :children [{:title "child-1-1" + :properties {:user.property/name ""} + :children []}]}] + (template-content-trees @conn "nested-child-1"))) + (is (= [{:title "child-2-1" + :properties {:user.property/name ""} + :children []}] + (template-content-trees @conn "nested-child-2"))) + (is (= [{:title "MEETING TITLE" + :properties {:user.property/participants #{"TODO"}} + :children []} + {:title "TITLE" + :properties {} + :children []} + {:title "" + :properties {:user.property/name "" + :user.property/author ""} + :children []} + {:title "TITLE" + :properties {} + :children [{:title "intro" :properties {} :children []} + {:title "notes" :properties {} :children []}]} + {:title "" + :properties {} + :children [{:title "intro" :properties {} :children []} + {:title "notes" :properties {} :children []}]} + {:title "it should not be included in the template" + :properties {} + :children [{:title "intro" :properties {} :children []} + {:title "notes" :properties {} :children []}]} + {:title "it's a template with nested templates" + :properties {:user.property/name "you named it"} + :children [{:title "child-1" + :properties {:user.property/name ""} + :children [{:title "child-1-1" + :properties {:user.property/name ""} + :children []}]} + {:title "child-2" + :properties {:user.property/name ""} + :children [{:title "child-2-1" + :properties {:user.property/name ""} + :children []}]} + {:title "child-3" + :properties {:user.property/name ""} + :children []}]}] + (->> (journal-top-level-trees @conn 20240216) + (drop 1) + (take 7))) + "Template metadata is removed from the journal while user properties and children remain") ;; Assets (is (= {:block/tags [:logseq.class/Asset] @@ -672,9 +892,12 @@ (is (= :node (:logseq.property/type (d/entity @conn :user.property/finishedat))) ":date property to :node value changes to :node") - (is (= :node + (is (= :default (:logseq.property/type (d/entity @conn :user.property/participants))) - ":node property to :date value remains :node") + "template values cause participants to remain a :default property") + (is (= #{"[[Feb 7th, 2024]]"} + (:user.property/participants (db-test/readable-properties (db-test/find-block-by-content @conn #"test :node -> :date")))) + ":default participants property keeps the imported text value") (is (= :default (:logseq.property/type (d/entity @conn :user.property/description))) diff --git a/deps/graph-parser/test/resources/exporter-test-graph/journals/2024_02_16.md b/deps/graph-parser/test/resources/exporter-test-graph/journals/2024_02_16.md index e6c3a810dc..64a804d7cf 100644 --- a/deps/graph-parser/test/resources/exporter-test-graph/journals/2024_02_16.md +++ b/deps/graph-parser/test/resources/exporter-test-graph/journals/2024_02_16.md @@ -4,6 +4,41 @@ - MEETING TITLE template:: meeting participants:: TODO +- TITLE + template:: title-only-no-children +- template:: properties-only-no-children + name:: + author:: +- TITLE + template:: title-only-with-children + - intro + - notes +- + template:: empty-title-with-children + - intro + - notes +- it should not be included in the template + template:: children-only + template-including-parent:: false + - intro + - notes +- it's a template with nested templates + template:: nested-father + template-including-parent:: true + name:: you named it + - child-1 + template:: nested-child-1 + name:: + - child-1-1 + name:: + - child-2 + template:: nested-child-2 + template-including-parent:: false + name:: + - child-2-1 + name:: + - child-3 + name:: - pending block for :number to :default duration:: 10 - test :number to :default From df5fb72ed9a43fcf50301c055b79d1b752a0fade Mon Sep 17 00:00:00 2001 From: Mega Yu Date: Tue, 10 Mar 2026 20:01:16 +0800 Subject: [PATCH 14/21] fix lint Shadowed var --- deps/graph-parser/src/logseq/graph_parser/exporter.cljs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs index 0af21987cd..59d8e426ad 100644 --- a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs @@ -97,10 +97,11 @@ (defn- group-block-children-by-parent [blocks] - (reduce (fn [result {:block/keys [parent uuid]}] + (reduce (fn [result {parent :block/parent + child-uuid :block/uuid}] (if (and (vector? parent) (= :block/uuid (first parent))) - (update result (second parent) (fnil conj []) uuid) + (update result (second parent) (fnil conj []) child-uuid) result)) {} blocks)) From fdf1a8f8cca8fe72823920d14e38c46212c0fa31 Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Wed, 11 Mar 2026 13:55:35 -0400 Subject: [PATCH 15/21] fix: imported Template blocks Template blocks were being copied instead of updated in place. Would feel buggy to user as Template blocks would show up twice in search and would not be where they had defined them --- .../src/logseq/graph_parser/exporter.cljs | 129 +++++++++--------- .../logseq/graph_parser/exporter_test.cljs | 98 +++++-------- 2 files changed, 101 insertions(+), 126 deletions(-) diff --git a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs index 59d8e426ad..62e609c54e 100644 --- a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs @@ -115,68 +115,77 @@ (conj result current-uuid)) result))) -(defn- clone-template-content-blocks - [blocks-by-uuid block-children template-page-uuid template-root-uuid content-root-uuids] - (let [*cloned-uuids (atom #{}) - clone-block - (fn clone-block [source-uuid parent-uuid] - (let [source-block (get blocks-by-uuid source-uuid) - cloned-uuid (common-uuid/gen-uuid) - cloned-block (-> source-block - strip-template-properties - (assoc :block/uuid cloned-uuid - :block/page [:block/uuid template-page-uuid] - :block/parent [:block/uuid parent-uuid]) - (dissoc :db/id))] - (swap! *cloned-uuids conj cloned-uuid) - (cons cloned-block - (mapcat #(clone-block % cloned-uuid) - (get block-children source-uuid)))))] - {:blocks (vec (mapcat #(clone-block % template-root-uuid) content-root-uuids)) - :preserve-empty-properties-uuids @*cloned-uuids})) +(defn- get-parent-uuid [parent] + (cond + (and (vector? parent) (= :block/uuid (first parent))) + (second parent) + (and (map? parent) (:block/uuid parent)) + (:block/uuid parent) + :else + nil)) -(defn- extract-template-blocks - [db blocks] - (let [template-page (d/entity db :logseq.class/Template) - template-page-uuid (:block/uuid template-page) - blocks-by-uuid (into {} (map (juxt :block/uuid identity) blocks)) - block-children (group-block-children-by-parent blocks)] +(defn- handle-template-blocks + "Handles creating #Template blocks and their children and calculates + :preserve-empty-properties-uuids for use later" + [blocks*] + (let [include-parent-template-uuids (->> blocks* + (filter (fn [block] + (and (get-template-name block) + (template-including-parent? block)))) + (map :block/uuid) + set) + content-uuids-by-template (into {} + (map (fn [template-uuid] + [template-uuid (common-uuid/gen-uuid)])) + include-parent-template-uuids)] (reduce - (fn [{:keys [blocks template-blocks preserve-empty-properties-uuids]} block] - (let [template-name (get-template-name block) - cleaned-block (strip-template-properties block) - source-preserve-empty-properties-uuids - (if template-name - (set (get-block-subtree-uuids block-children (:block/uuid block))) - #{}) - content-root-uuids (if (template-including-parent? block) - [(:block/uuid block)] - (get block-children (:block/uuid block))) - template-root-block - (when template-name - {:block/uuid (common-uuid/gen-uuid) - :block/title template-name - :block/page [:block/uuid template-page-uuid] - :block/parent [:block/uuid template-page-uuid] - :block/order (db-order/gen-key) - :block/format (:block/format block) - :block/tags [:logseq.class/Template]}) - template-content - (if template-root-block - (clone-template-content-blocks blocks-by-uuid block-children template-page-uuid (:block/uuid template-root-block) content-root-uuids) - {:blocks [] - :preserve-empty-properties-uuids #{}})] - {:blocks (conj blocks cleaned-block) - :template-blocks (cond-> template-blocks - template-root-block - (into (into [template-root-block] (:blocks template-content)))) - :preserve-empty-properties-uuids (set/union preserve-empty-properties-uuids - source-preserve-empty-properties-uuids - (:preserve-empty-properties-uuids template-content))})) + (fn [{:keys [blocks preserve-empty-properties-uuids]} block] + (if-let [template-name (get-template-name block)] + (let [block-children (group-block-children-by-parent blocks*) + base-block (strip-template-properties block) + parent (:block/parent base-block) + parent-uuid (get-parent-uuid parent) + content-uuid (when parent-uuid + (get content-uuids-by-template parent-uuid)) + updated-parent (if content-uuid + [:block/uuid content-uuid] + parent) + cleaned-block' (if content-uuid + (assoc base-block :block/parent updated-parent) + base-block) + source-preserve-empty-properties-uuids (set (get-block-subtree-uuids block-children (:block/uuid block))) + template-root-block (-> cleaned-block' + (assoc :block/title template-name) + (update :block/tags (fnil conj []) :logseq.class/Template) + (dissoc :block/refs + :block/properties + :block/properties-text-values + :block/properties-order)) + template-content-block (when (template-including-parent? block) + (-> base-block + (assoc :block/uuid (get content-uuids-by-template (:block/uuid block)) + :block/parent [:block/uuid (:block/uuid base-block)] + :block/order (db-order/gen-key)) + (dissoc :db/id)))] + {:blocks (cond-> blocks + true + (conj template-root-block) + template-content-block + (conj template-content-block)) + :preserve-empty-properties-uuids (set/union preserve-empty-properties-uuids + source-preserve-empty-properties-uuids + (cond-> #{} + template-content-block + (conj (:block/uuid template-content-block))))}) + {:blocks (if-let [content-uuid (some->> (get-parent-uuid (:block/parent block)) + (get content-uuids-by-template))] + (conj blocks + (assoc block :block/parent [:block/uuid content-uuid])) + (conj blocks block)) + :preserve-empty-properties-uuids preserve-empty-properties-uuids})) {:blocks [] - :template-blocks [] :preserve-empty-properties-uuids #{}} - blocks))) + blocks*))) (defn- get-page-uuid [page-names-to-uuids page-name ex-data'] (or (get @page-names-to-uuids (some-> (if (string/includes? (str page-name) "#") @@ -2186,9 +2195,7 @@ :as *options}] (p/let [options (assoc *options :notify-user notify-user :log-fn log-fn :file file) {:keys [pages blocks]} (extract-pages-and-blocks @conn file content options) - {:keys [blocks template-blocks preserve-empty-properties-uuids]} - (extract-template-blocks @conn blocks) - blocks (into blocks template-blocks) + {:keys [blocks preserve-empty-properties-uuids]} (handle-template-blocks blocks) tx-options (merge (build-tx-options options) {:journal-created-ats (build-journal-created-ats pages) :preserve-empty-property-block-uuids preserve-empty-properties-uuids}) diff --git a/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs b/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs index d984cc0132..35505475f4 100644 --- a/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs +++ b/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs @@ -78,11 +78,6 @@ ordered-children (mapv block-tree-with-properties))) -(defn- journal-top-level-trees - [db journal-day] - (some->> (db-test/find-journal-by-journal-day db journal-day) - ordered-children - (mapv block-tree-with-properties))) (defn- build-graph-files "Given a file graph directory, return all files including assets and adds relative paths @@ -211,8 +206,7 @@ "[[FIRST UUID]] and [[UUID]]")) (deftest extract-template-blocks - (let [conn (db-test/create-conn) - page-uuid (random-uuid) + (let [page-uuid (random-uuid) parent-uuid (random-uuid) child-uuid (random-uuid) include-children-only-uuid (random-uuid) @@ -258,12 +252,14 @@ :block/page [:block/uuid page-uuid] :block/parent [:block/uuid include-children-only-uuid] :block/order "e"}] - {:keys [blocks template-blocks preserve-empty-properties-uuids]} - (#'gp-exporter/extract-template-blocks @conn blocks)] - (testing "source blocks remove template metadata and mark template subtrees to preserve empty properties" - (is (= [{:title "source parent" :properties {:name ""}} + {:keys [blocks preserve-empty-properties-uuids]} + (#'gp-exporter/handle-template-blocks blocks)] + (testing "template roots replace source blocks and strip template metadata" + (is (= [{:title "trimmed template" :properties nil} + {:title "source parent" :properties {:name ""}} + {:title "nested child" :properties nil} {:title "child" :properties {:name "child default"}} - {:title "exclude source block" :properties {}} + {:title "children only" :properties nil} {:title "first child" :properties nil} {:title "second child" :properties nil}] (mapv (fn [block] @@ -274,20 +270,21 @@ (set/intersection preserve-empty-properties-uuids #{parent-uuid child-uuid include-children-only-uuid child-only-1-uuid child-only-2-uuid})))) - (testing "template roots use trimmed names and clone the correct content" + (testing "template roots use trimmed names and include parent content when configured" (is (= #{"trimmed template" "nested child" "children only"} - (->> template-blocks + (->> blocks (filter #(some #{:logseq.class/Template} (:block/tags %))) (map :block/title) set))) - (is (= ["source parent" "child" "child" "first child" "second child"] - (->> template-blocks + (is (= ["source parent"] + (->> blocks (remove #(some #{:logseq.class/Template} (:block/tags %))) + (filter #(= [:block/uuid parent-uuid] (:block/parent %))) (map :block/title)))) - (is (= 5 + (is (= 2 (count (set/difference preserve-empty-properties-uuids #{parent-uuid child-uuid include-children-only-uuid child-only-1-uuid child-only-2-uuid}))) - "cloned template content blocks are also marked to preserve empty properties")))) + "in-place template content blocks are marked to preserve empty properties")))) (deftest-async ^:integration export-docs-graph-with-convert-all-tags (p/let [file-graph-dir "test/resources/docs-0.10.12" @@ -564,6 +561,16 @@ @conn) set)) "All template definitions are imported as Template blocks") + (let [journal-uuid (:block/uuid (db-test/find-journal-by-journal-day @conn 20240216)) + template-page-uuids (->> (d/q '[:find [?page-uuid ...] + :where + [?b :block/tags :logseq.class/Template] + [?b :block/page ?page] + [?page :block/uuid ?page-uuid]] + @conn) + set)] + (is (= #{journal-uuid} template-page-uuids) + "All template blocks are created on their source journal page")) (is (= [{:title "MEETING TITLE" :properties {:user.property/participants #{"TODO"}} :children []}] @@ -592,13 +599,15 @@ (template-content-trees @conn "children-only"))) (is (= [{:title "it's a template with nested templates" :properties {:user.property/name "you named it"} - :children [{:title "child-1" - :properties {:user.property/name ""} - :children [{:title "child-1-1" + :children [{:title "nested-child-1" + :properties {} + :children [{:title "child-1" :properties {:user.property/name ""} - :children []}]} - {:title "child-2" - :properties {:user.property/name ""} + :children [{:title "child-1-1" + :properties {:user.property/name ""} + :children []}]}]} + {:title "nested-child-2" + :properties {} :children [{:title "child-2-1" :properties {:user.property/name ""} :children []}]} @@ -616,47 +625,6 @@ :properties {:user.property/name ""} :children []}] (template-content-trees @conn "nested-child-2"))) - (is (= [{:title "MEETING TITLE" - :properties {:user.property/participants #{"TODO"}} - :children []} - {:title "TITLE" - :properties {} - :children []} - {:title "" - :properties {:user.property/name "" - :user.property/author ""} - :children []} - {:title "TITLE" - :properties {} - :children [{:title "intro" :properties {} :children []} - {:title "notes" :properties {} :children []}]} - {:title "" - :properties {} - :children [{:title "intro" :properties {} :children []} - {:title "notes" :properties {} :children []}]} - {:title "it should not be included in the template" - :properties {} - :children [{:title "intro" :properties {} :children []} - {:title "notes" :properties {} :children []}]} - {:title "it's a template with nested templates" - :properties {:user.property/name "you named it"} - :children [{:title "child-1" - :properties {:user.property/name ""} - :children [{:title "child-1-1" - :properties {:user.property/name ""} - :children []}]} - {:title "child-2" - :properties {:user.property/name ""} - :children [{:title "child-2-1" - :properties {:user.property/name ""} - :children []}]} - {:title "child-3" - :properties {:user.property/name ""} - :children []}]}] - (->> (journal-top-level-trees @conn 20240216) - (drop 1) - (take 7))) - "Template metadata is removed from the journal while user properties and children remain") ;; Assets (is (= {:block/tags [:logseq.class/Asset] From ae003f146911183871bcb0b377500c7dc37b8136 Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Wed, 11 Mar 2026 15:18:37 -0400 Subject: [PATCH 16/21] fix: clean up template property handling When new built-in properties are supported in exporter, all-built-in-names and file-built-in-property-names should be updated. No need to handle block/properties-order and block/properties-text-values as those are ignored --- .../src/logseq/graph_parser/exporter.cljs | 47 ++++++------------- .../logseq/graph_parser/exporter_test.cljs | 21 ++++----- 2 files changed, 23 insertions(+), 45 deletions(-) diff --git a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs index 62e609c54e..1373883a9d 100644 --- a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs @@ -80,21 +80,6 @@ (string/join "\n")) title)) -(defn- strip-template-properties - [block] - (cond-> block - (:block/title block) - (update :block/title remove-template-property-lines) - - (seq (:block/properties block)) - (update :block/properties #(apply dissoc % template-file-property-names)) - - (seq (:block/properties-text-values block)) - (update :block/properties-text-values #(apply dissoc % template-file-property-names)) - - (seq (:block/properties-order block)) - (update :block/properties-order (fn [props-order] (vec (remove template-file-property-names props-order)))))) - (defn- group-block-children-by-parent [blocks] (reduce (fn [result {parent :block/parent @@ -142,29 +127,24 @@ (fn [{:keys [blocks preserve-empty-properties-uuids]} block] (if-let [template-name (get-template-name block)] (let [block-children (group-block-children-by-parent blocks*) - base-block (strip-template-properties block) - parent (:block/parent base-block) - parent-uuid (get-parent-uuid parent) + parent-uuid (get-parent-uuid (:block/parent block)) content-uuid (when parent-uuid (get content-uuids-by-template parent-uuid)) - updated-parent (if content-uuid - [:block/uuid content-uuid] - parent) cleaned-block' (if content-uuid - (assoc base-block :block/parent updated-parent) - base-block) + (assoc block :block/parent [:block/uuid content-uuid]) + block) source-preserve-empty-properties-uuids (set (get-block-subtree-uuids block-children (:block/uuid block))) template-root-block (-> cleaned-block' (assoc :block/title template-name) (update :block/tags (fnil conj []) :logseq.class/Template) - (dissoc :block/refs - :block/properties - :block/properties-text-values - :block/properties-order)) + (dissoc :block/properties)) template-content-block (when (template-including-parent? block) - (-> base-block + (-> (cond-> block + (seq (:block/properties block)) + (update :block/properties #(apply dissoc % template-file-property-names))) + (update :block/title remove-template-property-lines) (assoc :block/uuid (get content-uuids-by-template (:block/uuid block)) - :block/parent [:block/uuid (:block/uuid base-block)] + :block/parent [:block/uuid (:block/uuid block)] :block/order (db-order/gen-key)) (dissoc :db/id)))] {:blocks (cond-> blocks @@ -593,7 +573,8 @@ "All built-in property file ids as a set of keywords" (-> built-in-property-file-to-db-idents keys set ;; built-in-properties that map to new properties - (set/union #{:filters :query-table :query-properties :query-sort-by :query-sort-desc :hl-stamp :file :file-path}))) + (set/union #{:filters :query-table :query-properties :query-sort-by :query-sort-desc :hl-stamp :file :file-path}) + (set/union template-file-property-names))) ;; TODO: Review whether this should be using :block/title instead of file graph ids (def all-built-in-names @@ -611,7 +592,8 @@ #{:alias :tags :background-color :heading :query-table :query-properties :query-sort-by :query-sort-desc :ls-type :hl-type :hl-color :hl-page :hl-stamp :hl-value :file :file-path - :logseq.order-list-type :icon :public :exclude-from-graph-view :filters}) + :logseq.order-list-type :icon :public :exclude-from-graph-view :filters + :template :template-including-parent}) (assert (set/subset? file-built-in-property-names all-built-in-property-file-ids) "All file-built-in properties are used in db graph") @@ -814,7 +796,7 @@ {:keys [import-state user-options] :as options}] (let [{:keys [all-idents property-schemas]} import-state get-ident' #(get-ident @all-idents %) - user-properties (apply dissoc props (concat file-built-in-property-names template-file-property-names))] + user-properties (apply dissoc props file-built-in-property-names)] (when (seq user-properties) (swap! (:block-properties-text-values import-state) assoc @@ -855,7 +837,6 @@ "Updates page and block properties before their property types are inferred" [properties class-related-properties {:keys [preserve-empty-properties?]}] (let [dissoced-props (concat ignored-built-in-properties - template-file-property-names ;; TODO: Deal with these dissoced built-in properties [:title :created-at :updated-at] class-related-properties)] diff --git a/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs b/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs index 35505475f4..01088dfb81 100644 --- a/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs +++ b/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs @@ -254,18 +254,15 @@ :block/order "e"}] {:keys [blocks preserve-empty-properties-uuids]} (#'gp-exporter/handle-template-blocks blocks)] - (testing "template roots replace source blocks and strip template metadata" - (is (= [{:title "trimmed template" :properties nil} - {:title "source parent" :properties {:name ""}} - {:title "nested child" :properties nil} - {:title "child" :properties {:name "child default"}} - {:title "children only" :properties nil} - {:title "first child" :properties nil} - {:title "second child" :properties nil}] - (mapv (fn [block] - {:title (:block/title block) - :properties (:block/properties block)}) - blocks))) + (testing "template roots replace source blocks" + (is (= ["trimmed template" + "source parent" + "nested child" + "child" + "children only" + "first child" + "second child"] + (mapv :block/title blocks))) (is (= #{parent-uuid child-uuid include-children-only-uuid child-only-1-uuid child-only-2-uuid} (set/intersection preserve-empty-properties-uuids #{parent-uuid child-uuid include-children-only-uuid child-only-1-uuid child-only-2-uuid})))) From 6e08f7b1872c91086540b8613bf58f40c335b13c Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 12 Mar 2026 08:03:07 +0800 Subject: [PATCH 17/21] fix: remote block create split across batches --- src/main/frontend/worker/sync.cljs | 16 ++++++----- src/test/frontend/worker/db_sync_test.cljs | 31 ++++++++++++++++++++++ 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/src/main/frontend/worker/sync.cljs b/src/main/frontend/worker/sync.cljs index b1209c5269..15d0e8da19 100644 --- a/src/main/frontend/worker/sync.cljs +++ b/src/main/frontend/worker/sync.cljs @@ -648,8 +648,12 @@ [x] (and (integer? x) (neg? x))) +(defn- remote-batch-temp-id + [temp-id] + (str "remote-batch-tempid-" temp-id)) + (defn- remap-remote-batch-temp-ids - [batch-index tx-data] + [tx-data] (let [ops #{:db/add :db/retract :db/retractEntity} entity-temp-ids (->> tx-data (keep (fn [item] @@ -661,9 +665,7 @@ distinct) temp-id-map (when (seq entity-temp-ids) (zipmap entity-temp-ids - (map-indexed (fn [idx _] - (str "remote-batch-" batch-index "-tempid-" idx)) - entity-temp-ids)))] + (map remote-batch-temp-id entity-temp-ids)))] (if (seq temp-id-map) (mapv (fn [item] (if (and (vector? item) @@ -731,11 +733,11 @@ (defn- flatten-batched-remote-tx-data [tx-data*] - (loop [remaining (map-indexed vector tx-data*) + (loop [remaining tx-data* lookup->temp-id {} acc []] - (if-let [[batch-index tx-data] (first remaining)] - (let [remapped-batch (remap-remote-batch-temp-ids batch-index tx-data) + (if-let [tx-data (first remaining)] + (let [remapped-batch (remap-remote-batch-temp-ids tx-data) lookup->temp-id (merge lookup->temp-id (created-lookup->temp-id remapped-batch)) resolved-batch (resolve-lookup-refs lookup->temp-id remapped-batch)] (recur (rest remaining) diff --git a/src/test/frontend/worker/db_sync_test.cljs b/src/test/frontend/worker/db_sync_test.cljs index b91ab834b3..075fc02391 100644 --- a/src/test/frontend/worker/db_sync_test.cljs +++ b/src/test/frontend/worker/db_sync_test.cljs @@ -740,6 +740,37 @@ vec)] (is (empty? sanitized))))) +(deftest apply-remote-batched-create-reuses-tempid-across-batches-test + (testing "a remote block create split across batches should still resolve to one valid block" + (let [{:keys [conn parent]} (setup-parent-child) + parent-uuid (:block/uuid parent) + page-uuid (:block/uuid (:block/page parent)) + remote-uuid (random-uuid) + batched-tx-data [[[:db/add -1 :block/uuid remote-uuid] + [:db/add -1 :block/title "remote batched child"] + [:db/add -1 :block/page [:block/uuid page-uuid]] + [:db/add -1 :block/created-at 1760000000000] + [:db/add -1 :block/updated-at 1760000000000]] + [[:db/add -1 :block/parent [:block/uuid parent-uuid]] + [:db/add -1 :block/order "a4"]]]] + (with-datascript-conns conn nil + (fn [] + (let [error (try + (#'db-sync/apply-remote-tx! test-repo nil batched-tx-data) + nil + (catch :default e + e))] + (is (nil? error) + (when error + (str (ex-message error) " " (pr-str (ex-data error))))) + (when-not error + (let [block (d/entity @conn [:block/uuid remote-uuid]) + validation (db-validate/validate-local-db! @conn)] + (is (= "remote batched child" (:block/title block))) + (is (= (:db/id parent) (:db/id (:block/parent block)))) + (is (empty? (map :entity (:errors validation))) + (str (:errors validation))))))))))) + (deftest ^:long sanitize-tx-data-drops-numeric-entity-datoms-for-deleted-block-test (testing "deleted-block-ids should also drop datoms when entity is numeric id" (let [{:keys [conn child1]} (setup-parent-child) From 5d602c9d5861b9cb03f0c58bc0b8983375fdc384 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 12 Mar 2026 08:18:52 +0800 Subject: [PATCH 18/21] fix(db-sync): drop stale lookup-ref tx datoms --- .../logseq/db_sync/worker/handler/sync.cljs | 32 ++++++++++++++++++- .../db_sync/worker_handler_sync_test.cljs | 26 ++++++++++++++- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/deps/db-sync/src/logseq/db_sync/worker/handler/sync.cljs b/deps/db-sync/src/logseq/db_sync/worker/handler/sync.cljs index 29f60b860b..2158e104eb 100644 --- a/deps/db-sync/src/logseq/db_sync/worker/handler/sync.cljs +++ b/deps/db-sync/src/logseq/db_sync/worker/handler/sync.cljs @@ -1,5 +1,6 @@ (ns logseq.db-sync.worker.handler.sync (:require [clojure.string :as string] + [datascript.core :as d] [lambdaisland.glogi :as log] [logseq.db :as ldb] [logseq.db-sync.batch :as batch] @@ -10,6 +11,7 @@ [logseq.db-sync.worker.http :as http] [logseq.db-sync.worker.routes.sync :as sync-routes] [logseq.db-sync.worker.ws :as ws] + [logseq.db.frontend.schema :as db-schema] [promesa.core :as p])) (def ^:private snapshot-download-batch-size 5000) @@ -264,7 +266,35 @@ (let [sql (.-sql self)] (ensure-conn! self) (let [conn (.-conn self) - tx-data (protocol/transit->tx txs)] + lookup-id (fn [x] + (when (and (vector? x) + (= 2 (count x)) + (= :block/uuid (first x))) + (second x))) + tx-data* (protocol/transit->tx txs) + created-block-uuids (->> tx-data* + (keep (fn [item] + (when (and (vector? item) + (= :db/add (first item)) + (>= (count item) 4) + (= :block/uuid (nth item 2))) + (nth item 3)))) + set) + missing-lookup-ref? (fn [x] + (when-let [block-uuid (lookup-id x)] + (and (not (contains? created-block-uuids block-uuid)) + (nil? (d/entity @conn x))))) + tx-data (remove (fn [item] + (when (vector? item) + (let [op (first item) + attr (nth item 2 nil) + value (when (>= (count item) 4) (nth item 3))] + (or (and (contains? #{:db/add :db/retract :db/retractEntity} op) + (missing-lookup-ref? (second item))) + (and (contains? #{:db/add :db/retract} op) + (contains? db-schema/ref-type-attributes attr) + (missing-lookup-ref? value)))))) + tx-data*)] (ldb/transact! conn tx-data {:op :apply-client-tx}) (let [new-t (storage/get-t sql)] ;; FIXME: no need to broadcast if client tx is less than remote tx diff --git a/deps/db-sync/test/logseq/db_sync/worker_handler_sync_test.cljs b/deps/db-sync/test/logseq/db_sync/worker_handler_sync_test.cljs index bc60f0ee33..2fee2699c6 100644 --- a/deps/db-sync/test/logseq/db_sync/worker_handler_sync_test.cljs +++ b/deps/db-sync/test/logseq/db_sync/worker_handler_sync_test.cljs @@ -1,8 +1,13 @@ (ns logseq.db-sync.worker-handler-sync-test - (:require [cljs.test :refer [async deftest is]] + (:require [cljs.test :refer [async deftest is testing]] + [datascript.core :as d] [logseq.db-sync.common :as common] + [logseq.db-sync.protocol :as protocol] [logseq.db-sync.storage :as storage] + [logseq.db-sync.test-sql :as test-sql] [logseq.db-sync.worker.handler.sync :as sync-handler] + [logseq.db-sync.worker.ws :as ws] + [logseq.db.frontend.schema :as db-schema] [promesa.core :as p])) (defn- empty-sql [] @@ -127,3 +132,22 @@ (p/catch (fn [error] (is false (str error)) (done))))))) + +(deftest tx-batch-drops-stale-lookup-entity-updates-test + (testing "stale lookup-ref entity updates should not reject the whole tx batch" + (let [sql (test-sql/make-sql) + conn (d/create-conn db-schema/schema) + self #js {:sql sql + :conn conn + :schema-ready true} + missing-uuid (random-uuid) + created-uuid (random-uuid) + tx-data [[:db/add [:block/uuid missing-uuid] :block/title "stale" 1] + [:db/add [:block/uuid missing-uuid] :block/updated-at 1773188050934 1] + [:db/add "temp-1" :block/uuid created-uuid 2] + [:db/add "temp-1" :block/title "ok" 2]] + response (with-redefs [ws/broadcast! (fn [& _] nil)] + (sync-handler/handle-tx-batch! self nil (protocol/tx->transit tx-data) 0))] + (is (= "tx/batch/ok" (:type response))) + (is (= "ok" (:block/title (d/entity @conn [:block/uuid created-uuid])))) + (is (nil? (d/entity @conn [:block/uuid missing-uuid])))))) From 7386eec6dbac58d0b84e986b32c15f85c09e5f63 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 12 Mar 2026 08:28:38 +0800 Subject: [PATCH 19/21] fix(db-sync): server shouldn't delete addrs --- deps/db-sync/src/logseq/db_sync/batch.cljs | 2 +- deps/db-sync/src/logseq/db_sync/storage.cljs | 11 +---------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/deps/db-sync/src/logseq/db_sync/batch.cljs b/deps/db-sync/src/logseq/db_sync/batch.cljs index 2ba8bb13ce..cd791ad9d9 100644 --- a/deps/db-sync/src/logseq/db_sync/batch.cljs +++ b/deps/db-sync/src/logseq/db_sync/batch.cljs @@ -1,7 +1,7 @@ (ns logseq.db-sync.batch (:require [clojure.string :as string])) -(def ^:private max-sql-params 99) +(def max-sql-params 99) (def ^:private row-param-count 3) (defn rows->insert-batches diff --git a/deps/db-sync/src/logseq/db_sync/storage.cljs b/deps/db-sync/src/logseq/db_sync/storage.cljs index 51de51072a..26c41afa35 100644 --- a/deps/db-sync/src/logseq/db_sync/storage.cljs +++ b/deps/db-sync/src/logseq/db_sync/storage.cljs @@ -1,6 +1,5 @@ (ns logseq.db-sync.storage (:require [cljs-bean.core :as bean] - [clojure.string :as string] [datascript.core :as d] [datascript.storage :refer [IStorage]] [logseq.db-sync.common :as common] @@ -70,13 +69,6 @@ :tx (aget row "tx")}) rows))) -(defn- delete-addrs! [sql addrs] - (when (seq addrs) - (let [placeholders (->> addrs (map (constantly "?")) (string/join ","))] - (apply common/sql-exec sql - (str "delete from kvs where addr in (" placeholders ")") - addrs)))) - (defn- upsert-addr-content! [sql data] (doseq [item data] (common/sql-exec sql @@ -97,7 +89,7 @@ (defn new-sqlite-storage [sql] (reify IStorage - (-store [_ addr+data-seq delete-addrs] + (-store [_ addr+data-seq _delete-addrs] (let [data (map (fn [[addr data]] (let [data' (if (map? data) (dissoc data :addresses) data) @@ -108,7 +100,6 @@ "content" (common/write-transit data') "addresses" addresses})) addr+data-seq)] - (delete-addrs! sql delete-addrs) (upsert-addr-content! sql data))) (-restore [_ addr] (restore-data-from-addr sql addr)))) From b1ceaa1216ac37a1e38a510d541eb2f2b1832238 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 12 Mar 2026 10:41:41 +0800 Subject: [PATCH 20/21] enhance: able to generate large graphs with >1M blocks --- .../create_graph_with_large_sizes.cljs | 141 +++++++++++------- .../create_graph_with_large_sizes_test.cljs | 69 +++++++++ scripts/test/logseq/tasks/test_runner.cljs | 8 + 3 files changed, 168 insertions(+), 50 deletions(-) create mode 100644 scripts/test/logseq/tasks/db_graph/create_graph_with_large_sizes_test.cljs create mode 100644 scripts/test/logseq/tasks/test_runner.cljs diff --git a/scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs b/scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs index 65659e124c..b27d12b2ff 100644 --- a/scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs +++ b/scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs @@ -9,41 +9,80 @@ [nbb.classpath :as cp] [nbb.core :as nbb])) -(def *ids (atom #{})) -(defn get-next-id - [] - (let [id (random-uuid)] - (if (@*ids id) - (get-next-id) - (do - (swap! *ids conj id) - id)))) +(def ^:private default-block-title "Block") +(def ^:private target-entities-per-batch 25000) +(def ^:private max-pages-per-batch 1000) -(defn build-pages - [start-idx n] - (let [ids (repeatedly n get-next-id)] - (map-indexed - (fn [idx id] - {:block/uuid id - :block/title (str "Page-" (+ start-idx idx))}) - ids))) +(defn- parse-long-option + [value] + (if (string? value) + (js/parseInt value 10) + value)) -(defn build-blocks - [size] - (vec (repeatedly size - (fn [] - (let [id (get-next-id)] - {:block/uuid id - :block/title (str id)}))))) - -(defn- create-init-data +(defn- normalize-options [options] - (let [pages (build-pages 0 (:pages options))] - {:pages-and-blocks - (mapv #(hash-map :page % :blocks (build-blocks (:blocks options))) - pages) - ;; Custom id fn because transaction chunks may separate blocks and pages from each other - :page-id-fn (fn [b] [:block/uuid (:block/uuid b)])})) + (update-vals options parse-long-option)) + +(defn default-batch-pages + [blocks-per-page] + (-> (quot target-entities-per-batch (max 1 (inc blocks-per-page))) + (max 1) + (min max-pages-per-batch))) + +(defn- build-blocks + [blocks-per-page next-id] + (loop [block-idx 0 + blocks (transient [])] + (if (= block-idx blocks-per-page) + (persistent! blocks) + (recur (inc block-idx) + (conj! blocks + {:block/uuid (next-id) + :block/title default-block-title}))))) + +(defn build-page-and-blocks-batch + ([start-idx page-count blocks-per-page] + (build-page-and-blocks-batch start-idx page-count blocks-per-page random-uuid)) + ([start-idx page-count blocks-per-page next-id] + (loop [page-idx 0 + pages-and-blocks (transient [])] + (if (= page-idx page-count) + (persistent! pages-and-blocks) + (recur (inc page-idx) + (conj! pages-and-blocks + {:page {:block/uuid (next-id) + :block/title (str "Page-" (+ start-idx page-idx))} + :blocks (build-blocks blocks-per-page next-id)})))))) + +(defn page-and-block-batches + ([{:keys [pages blocks batch-pages]}] + (page-and-block-batches {:pages pages + :blocks blocks + :batch-pages batch-pages} + random-uuid)) + ([{:keys [pages blocks batch-pages]} next-id] + (let [batch-pages' (or batch-pages (default-batch-pages blocks))] + ((fn step [start-idx] + (lazy-seq + (when (< start-idx pages) + (cons (build-page-and-blocks-batch start-idx + (min batch-pages' (- pages start-idx)) + blocks + next-id) + (step (+ start-idx batch-pages')))))) + 0)))) + +(defn- transact-batch! + [conn pages-and-blocks] + (let [{:keys [init-tx block-props-tx]} (outliner-cli/build-blocks-tx {:pages-and-blocks pages-and-blocks})] + (d/transact! conn init-tx) + (when (seq block-props-tx) + (d/transact! conn block-props-tx)))) + +(defn- total-batches + [{:keys [pages blocks batch-pages]}] + (let [batch-pages' (or batch-pages (default-batch-pages blocks))] + (js/Math.ceil (/ pages batch-pages')))) (def spec "Options spec" @@ -54,34 +93,36 @@ :desc "Number of pages to create"} :blocks {:alias :b :default 20 - :desc "Number of blocks to create"}}) + :desc "Number of blocks to create per page"} + :batch-pages {:alias :t + :desc "Number of pages to build and transact per batch"}}) + +(defn parse-args + [args] + {:graph-dir (first args) + :options (normalize-options (cli/parse-opts (rest args) {:spec spec}))}) (defn -main [args] - (let [graph-dir (first args) - options (cli/parse-opts args {:spec spec}) + (let [{:keys [graph-dir options]} (parse-args args) _ (when (or (nil? graph-dir) (:help options)) (println (str "Usage: $0 GRAPH-NAME [OPTIONS]\nOptions:\n" (cli/format-opts {:spec spec}))) (js/process.exit 1)) + {:keys [pages blocks batch-pages]} options [dir db-name] (if (string/includes? graph-dir "/") ((juxt node-path/dirname node-path/basename) graph-dir) [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]) conn (outliner-cli/init-conn dir db-name {:classpath (cp/get-classpath)}) - _ (println "Building tx ...") - {:keys [init-tx]} (outliner-cli/build-blocks-tx (create-init-data options))] - (println "Built" (count init-tx) "tx," (count (filter :block/title init-tx)) "pages and" - (count (filter :block/title init-tx)) "blocks ...") - ;; Vary the chunking with page size up to a max to avoid OOM - (let [tx-chunks (partition-all (min (:pages options) 30000) init-tx)] - (loop [chunks tx-chunks - chunk-num 1] - (when-let [chunk (first chunks)] - (println "Transacting chunk" chunk-num "of" (count tx-chunks) - "starting with block:" (pr-str (select-keys (first chunk) [:block/title :block/title]))) - (d/transact! conn chunk) - (recur (rest chunks) (inc chunk-num))))) - #_(d/transact! conn blocks-tx) - (println "Created graph" (str db-name " with " (count (d/datoms @conn :eavt)) " datoms!")))) + total-batches' (total-batches options) + pages-per-batch (or batch-pages (default-batch-pages blocks)) + total-blocks (* pages blocks)] + (println "Creating graph with" pages "pages and" total-blocks "blocks" + "using" total-batches' "batch(es) of up to" pages-per-batch "pages ...") + (doseq [[batch-num pages-and-blocks] (map-indexed vector (page-and-block-batches options))] + (println "Transacting batch" (inc batch-num) "of" total-batches' + "with" (count pages-and-blocks) "pages") + (transact-batch! conn pages-and-blocks)) + (println "Created graph" db-name "with" pages "pages and" total-blocks "blocks."))) (when (= nbb/*file* (nbb/invoked-file)) (-main *command-line-args*)) diff --git a/scripts/test/logseq/tasks/db_graph/create_graph_with_large_sizes_test.cljs b/scripts/test/logseq/tasks/db_graph/create_graph_with_large_sizes_test.cljs new file mode 100644 index 0000000000..bd897e07d1 --- /dev/null +++ b/scripts/test/logseq/tasks/db_graph/create_graph_with_large_sizes_test.cljs @@ -0,0 +1,69 @@ +(ns logseq.tasks.db-graph.create-graph-with-large-sizes-test + (:require [cljs.test :refer [deftest is testing]] + [logseq.tasks.db-graph.create-graph-with-large-sizes :as sut])) + +(deftest build-page-and-blocks-batch-builds-the-requested-graph-slice + (let [id-seq (map #(str "id-" %) (range)) + next-id (let [ids (atom id-seq)] + (fn [] + (let [id (first @ids)] + (swap! ids rest) + id))) + batch (#'sut/build-page-and-blocks-batch 10 2 3 next-id)] + (is (= 2 (count batch))) + (is (= ["Page-10" "Page-11"] + (map (comp :block/title :page) batch))) + (is (= ["id-0" "id-4"] + (map (comp :block/uuid :page) batch))) + (is (= [["Block" "Block" "Block"] + ["Block" "Block" "Block"]] + (map (fn [{:keys [blocks]}] + (mapv :block/title blocks)) + batch))) + (is (= [["id-1" "id-2" "id-3"] + ["id-5" "id-6" "id-7"]] + (map (fn [{:keys [blocks]}] + (mapv :block/uuid blocks)) + batch))))) + +(deftest page-and-block-batches-only-realize-requested-batches + (let [calls (atom 0) + next-id (fn [] + (swap! calls inc) + (str "id-" @calls)) + batches (#'sut/page-and-block-batches {:pages 50000 + :blocks 50 + :batch-pages 100} + next-id) + first-batch (first batches)] + (is (= 100 (count first-batch))) + (is (= (* 100 51) @calls) + "Only the first batch should be realized") + (is (= "Page-0" (get-in first-batch [0 :page :block/title]))) + (is (= "Page-99" (get-in first-batch [99 :page :block/title]))))) + +(deftest default-batching-keeps-large-graphs-bounded + (testing "50k pages with 50 blocks are split into many batches instead of one giant tx" + (let [batch-pages (#'sut/default-batch-pages 50)] + (is (< batch-pages 50000)) + (is (pos? batch-pages)) + (is (= batch-pages + (count (first (#'sut/page-and-block-batches {:pages 50000 + :blocks 50} + (constantly "id"))))))))) + +(deftest page-and-block-batches-handle-empty-input + (is (= [] + (into [] (#'sut/page-and-block-batches {:pages 0 + :blocks 50} + (constantly "id")))))) + +(deftest parse-args-keeps-the-graph-name-separate-from-cli-options + (let [{:keys [graph-dir options]} (sut/parse-args ["large-graph" + "-p" "3" + "-b" "2" + "-t" "1"])] + (is (= "large-graph" graph-dir)) + (is (= 3 (:pages options))) + (is (= 2 (:blocks options))) + (is (= 1 (:batch-pages options))))) diff --git a/scripts/test/logseq/tasks/test_runner.cljs b/scripts/test/logseq/tasks/test_runner.cljs new file mode 100644 index 0000000000..b4d65f4e24 --- /dev/null +++ b/scripts/test/logseq/tasks/test_runner.cljs @@ -0,0 +1,8 @@ +(ns logseq.tasks.test-runner + (:require [cljs.test :as test] + [logseq.tasks.db-graph.create-graph-with-large-sizes-test])) + +(defn -main [& _] + (let [{:keys [fail error]} (test/run-tests 'logseq.tasks.db-graph.create-graph-with-large-sizes-test)] + (when (pos? (+ fail error)) + (js/process.exit 1)))) From 1b0db680986c1956cab3e8a64e2e11670c2ab907 Mon Sep 17 00:00:00 2001 From: Charlie Date: Thu, 12 Mar 2026 11:18:51 +0800 Subject: [PATCH 21/21] enhance(ux): align for the image asset block container (#12425) * feat(asset): add alignment options for images * enhance(ux): select block for the action of double click in the image asset container * enhance(asset): change alignment property type from string to keyword and normalize alignment handling --- .../src/logseq/db/frontend/malli_schema.cljs | 3 +- deps/db/src/logseq/db/frontend/property.cljs | 5 + deps/db/src/logseq/db/frontend/schema.cljs | 2 +- src/main/frontend/components/block.cljs | 110 ++++++++++++++---- src/main/frontend/components/block.css | 13 +++ src/main/frontend/worker/db/migrate.cljs | 3 +- src/resources/dicts/en.edn | 4 + 7 files changed, 112 insertions(+), 28 deletions(-) diff --git a/deps/db/src/logseq/db/frontend/malli_schema.cljs b/deps/db/src/logseq/db/frontend/malli_schema.cljs index 16d6ee6f1f..2efcf516b1 100644 --- a/deps/db/src/logseq/db/frontend/malli_schema.cljs +++ b/deps/db/src/logseq/db/frontend/malli_schema.cljs @@ -510,7 +510,8 @@ [:logseq.property.asset/checksum :string] [:logseq.property.asset/size :int] [:logseq.property.asset/width {:optional true} :int] - [:logseq.property.asset/height {:optional true} :int]] + [:logseq.property.asset/height {:optional true} :int] + [:logseq.property.asset/align {:optional true} :keyword]] block-attrs page-or-block-attrs))) diff --git a/deps/db/src/logseq/db/frontend/property.cljs b/deps/db/src/logseq/db/frontend/property.cljs index 13ee76927a..29f285ff0d 100644 --- a/deps/db/src/logseq/db/frontend/property.cljs +++ b/deps/db/src/logseq/db/frontend/property.cljs @@ -567,6 +567,11 @@ :schema {:type :map :hide? true :public? false}} + :logseq.property.asset/align {:title "Asset alignment" + :schema {:type :keyword + :hide? true + :public? false} + :queryable? false} :logseq.property.fsrs/due {:title "Due" :schema {:type :datetime diff --git a/deps/db/src/logseq/db/frontend/schema.cljs b/deps/db/src/logseq/db/frontend/schema.cljs index 14586b399d..c6b2582114 100644 --- a/deps/db/src/logseq/db/frontend/schema.cljs +++ b/deps/db/src/logseq/db/frontend/schema.cljs @@ -30,7 +30,7 @@ (map (juxt :major :minor) [(parse-schema-version x) (parse-schema-version y)]))) -(def version (parse-schema-version "65.22")) +(def version (parse-schema-version "65.23")) (defn major-version "Return a number. diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index cb4fcf1995..a25b0f097e 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -202,11 +202,24 @@ (on-dimensions (.-naturalWidth img) (.-naturalHeight img)))) (set! (.-src img) url))) +(defn- normalize-asset-align + [asset-align] + (cond + (keyword? asset-align) asset-align + (string? asset-align) (case asset-align + "left" :left + "center" :center + "right" :right + nil) + :else nil)) + (defonce *resizing-image? (atom false)) + (rum/defc ^:large-vars/cleanup-todo asset-container [asset-block src title metadata {:keys [breadcrumb? positioned? local? full-text]}] (let [asset-width (:logseq.property.asset/width asset-block) - asset-height (:logseq.property.asset/height asset-block)] + asset-height (:logseq.property.asset/height asset-block) + asset-align (normalize-asset-align (:logseq.property.asset/align asset-block))] (hooks/use-effect! (fn [] (when (:block/uuid asset-block) @@ -275,7 +288,13 @@ :repo (state/get-current-repo) :href src :title title - :full-text full-text})))))))] + :full-text full-text}))))))) + handle-set-align! + (fn [align] + (when-let [asset-id (:block/uuid asset-block)] + (property-handler/set-block-property! asset-id + :logseq.property.asset/align + align)))] (when asset-block [:.asset-action-bar {:aria-hidden "true"} (shui/dropdown-menu @@ -288,6 +307,33 @@ :class "h-6 w-6"} (shui/tabler-icon "dots-vertical"))) (shui/dropdown-menu-content + (shui/dropdown-menu-sub + (shui/dropdown-menu-sub-trigger + [:span.flex.items-center.gap-1 + (ui/icon "layout-align-left") (t :asset/align)]) + (shui/dropdown-menu-sub-content + (shui/dropdown-menu-item + {:on-click #(handle-set-align! :left)} + [:span.flex.items-center.gap-2 + (ui/icon "layout-align-left") + (t :asset/align-left) + (when (or (nil? asset-align) (= asset-align :left)) + (ui/icon "check"))]) + (shui/dropdown-menu-item + {:on-click #(handle-set-align! :center)} + [:span.flex.items-center.gap-2 + (ui/icon "layout-align-center") + (t :asset/align-center) + (when (= asset-align :center) + (ui/icon "check"))]) + (shui/dropdown-menu-item + {:on-click #(handle-set-align! :right)} + [:span.flex.items-center.gap-2 + (ui/icon "layout-align-right") + (t :asset/align-right) + (when (= asset-align :right) + (ui/icon "check"))]))) + (shui/dropdown-menu-item {:on-click handle-copy!} [:span.flex.items-center.gap-1 @@ -301,6 +347,7 @@ (js/window.apis.openExternal image-src)))} [:span.flex.items-center.gap-1 (ui/icon "folder-pin") (t (if local? :asset/show-in-folder :asset/open-in-browser))])) + (when-not config/publishing? [:<> (shui/dropdown-menu-separator) @@ -318,6 +365,7 @@ (let [breadcrumb? (:breadcrumb? config) positioned? (:property-position config) asset-block (:asset-block config) + asset-align (normalize-asset-align (:logseq.property.asset/align asset-block)) width (:width metadata) *width (get state ::size) width (or @*width width) @@ -334,30 +382,42 @@ (:table-view? config) (not resizable?)) asset-container-cp - [:div.ls-resize-image.rounded-md - asset-container-cp - (resize-image-handles - (fn [k ^js event] - (let [dx (.-dx event) - ^js target (.-target event)] + [:div.ls-resize-inner.w-full.select-none + {:on-double-click (fn [^js e] + (let [^js target (.-target e) + ^js container (.closest target ".ls-resize-inner")] + (when (or container (= target container)) + (when-let [block-uuid (or (:block/uuid config) + (some-> config :block :block/uuid))] + (editor-handler/select-block! block-uuid)))))} + [:div.ls-resize-image.rounded-md + {:class (case asset-align + :center "align-center" + :right "align-right" + "align-left")} + asset-container-cp + (resize-image-handles + (fn [k ^js event] + (let [dx (.-dx event) + ^js target (.-target event)] - (case k - :start - (let [c (.closest target ".ls-resize-image")] - (reset! *width (.-offsetWidth c)) - (reset! *resizing-image? true)) - :move - (let [width' (+ @*width dx)] - (when (or (> width' 60) - (not (neg? dx))) - (reset! *width width'))) - :end - (let [width' @*width] - (when (and width' @*resizing-image?) - (when-let [block-id (or (:block/uuid config) - (some-> config :block (:block/uuid)))] - (editor-handler/resize-image! config block-id metadata full-text {:width width'}))) - (reset! *resizing-image? false))))))]))) + (case k + :start + (let [c (.closest target ".ls-resize-image")] + (reset! *width (.-offsetWidth c)) + (reset! *resizing-image? true)) + :move + (let [width' (+ @*width dx)] + (when (or (> width' 60) + (not (neg? dx))) + (reset! *width width'))) + :end + (let [width' @*width] + (when (and width' @*resizing-image?) + (when-let [block-id (or (:block/uuid config) + (some-> config :block (:block/uuid)))] + (editor-handler/resize-image! config block-id metadata full-text {:width width'}))) + (reset! *resizing-image? false))))))]]))) (rum/defc audio-cp ([src] (audio-cp src nil)) diff --git a/src/main/frontend/components/block.css b/src/main/frontend/components/block.css index d37edfb414..53e76622d0 100644 --- a/src/main/frontend/components/block.css +++ b/src/main/frontend/components/block.css @@ -1194,6 +1194,19 @@ html.is-mac { .ls-resize-image { @apply flex relative w-fit cursor-pointer; + &.align-left { + margin-right: auto; + } + + &.align-center { + margin-left: auto; + margin-right: auto; + } + + &.align-right { + margin-left: auto; + } + .handle-left, .handle-right { @apply absolute w-[6px] h-[15%] min-h-[30px] bg-black/30 hover:bg-black/70 top-[50%] left-[5px] rounded-full cursor-col-resize select-none diff --git a/src/main/frontend/worker/db/migrate.cljs b/src/main/frontend/worker/db/migrate.cljs index 8240a60f78..83a0233c9f 100644 --- a/src/main/frontend/worker/db/migrate.cljs +++ b/src/main/frontend/worker/db/migrate.cljs @@ -75,7 +75,8 @@ ["65.20" {:properties [:logseq.property.class/bidirectional-property-title :logseq.property.class/enable-bidirectional?]}] ["65.21" {:properties [:logseq.property.sync/large-title-object]}] ["65.22" {:properties [:logseq.property.reaction/emoji-id - :logseq.property.reaction/target]}]]) + :logseq.property.reaction/target]}] + ["65.23" {:properties [:logseq.property.asset/align]}]]) (let [[major minor] (last (sort (map (comp (juxt :major :minor) db-schema/parse-schema-version first) schema-version->updates)))] diff --git a/src/resources/dicts/en.edn b/src/resources/dicts/en.edn index 9656eb4462..07f433a4c0 100644 --- a/src/resources/dicts/en.edn +++ b/src/resources/dicts/en.edn @@ -135,6 +135,10 @@ :asset/ref-block "Asset ref block" :asset/confirm-delete "Are you sure you want to delete this {1}?" :asset/physical-delete "Remove the file too (notice it can't be restored)" + :asset/align "Align" + :asset/align-left "Align left" + :asset/align-center "Align center" + :asset/align-right "Align right" :color/gray "Gray" :color/red "Red" :color/yellow "Yellow"