diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index dd67810e36..5d9b45593a 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -228,8 +228,10 @@ clojure.test.check.properties/for-all clojure.core/for ;; src/main frontend.namespaces/import-vars potemkin/import-vars - ;; src/test + ;; src/test and deps tests frontend.test.helper/deftest-async clojure.test/deftest + logseq.graph-parser.test.helper/deftest-async clojure.test/deftest + logseq.publishing.test.helper/deftest-async clojure.test/deftest frontend.worker.rtc.idb-keyval-mock/with-reset-idb-keyval-mock cljs.test/async frontend.react/defc clojure.core/defn logseq.common.defkeywords/defkeyword cljs.spec.alpha/def diff --git a/deps.edn b/deps.edn index aaede5d60c..427a084d4f 100644 --- a/deps.edn +++ b/deps.edn @@ -5,7 +5,7 @@ :sha "5d672bf84ed944414b9f61eeb83808ead7be9127"} datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork - :sha "ff5a7d5326e2546f40146e4a489343f557519bc3"} + :sha "f91fec561ee2c11d6bf323feae365e9033585411"} ;; datascript/datascript {:local/root "../../datascript"} datascript-transit/datascript-transit {:mvn/version "0.3.0"} diff --git a/deps/db-sync/deps.edn b/deps/db-sync/deps.edn index 8915499b8d..14f589bac9 100644 --- a/deps/db-sync/deps.edn +++ b/deps/db-sync/deps.edn @@ -2,7 +2,7 @@ :deps {org.clojure/clojure {:mvn/version "1.11.1"} datascript/datascript {:git/url "https://github.com/logseq/datascript" - :sha "ff5a7d5326e2546f40146e4a489343f557519bc3"} + :sha "f91fec561ee2c11d6bf323feae365e9033585411"} datascript-transit/datascript-transit {:mvn/version "0.3.0" :exclusions [datascript/datascript]} com.cognitect/transit-cljs {:mvn/version "0.8.280"} diff --git a/deps/db-sync/src/logseq/db_sync/batch.cljs b/deps/db-sync/src/logseq/db_sync/batch.cljs index 2ba8bb13ce..cd791ad9d9 100644 --- a/deps/db-sync/src/logseq/db_sync/batch.cljs +++ b/deps/db-sync/src/logseq/db_sync/batch.cljs @@ -1,7 +1,7 @@ (ns logseq.db-sync.batch (:require [clojure.string :as string])) -(def ^:private max-sql-params 99) +(def max-sql-params 99) (def ^:private row-param-count 3) (defn rows->insert-batches diff --git a/deps/db-sync/src/logseq/db_sync/storage.cljs b/deps/db-sync/src/logseq/db_sync/storage.cljs index 51de51072a..26c41afa35 100644 --- a/deps/db-sync/src/logseq/db_sync/storage.cljs +++ b/deps/db-sync/src/logseq/db_sync/storage.cljs @@ -1,6 +1,5 @@ (ns logseq.db-sync.storage (:require [cljs-bean.core :as bean] - [clojure.string :as string] [datascript.core :as d] [datascript.storage :refer [IStorage]] [logseq.db-sync.common :as common] @@ -70,13 +69,6 @@ :tx (aget row "tx")}) rows))) -(defn- delete-addrs! [sql addrs] - (when (seq addrs) - (let [placeholders (->> addrs (map (constantly "?")) (string/join ","))] - (apply common/sql-exec sql - (str "delete from kvs where addr in (" placeholders ")") - addrs)))) - (defn- upsert-addr-content! [sql data] (doseq [item data] (common/sql-exec sql @@ -97,7 +89,7 @@ (defn new-sqlite-storage [sql] (reify IStorage - (-store [_ addr+data-seq delete-addrs] + (-store [_ addr+data-seq _delete-addrs] (let [data (map (fn [[addr data]] (let [data' (if (map? data) (dissoc data :addresses) data) @@ -108,7 +100,6 @@ "content" (common/write-transit data') "addresses" addresses})) addr+data-seq)] - (delete-addrs! sql delete-addrs) (upsert-addr-content! sql data))) (-restore [_ addr] (restore-data-from-addr sql addr)))) diff --git a/deps/db-sync/src/logseq/db_sync/worker/handler/sync.cljs b/deps/db-sync/src/logseq/db_sync/worker/handler/sync.cljs index 29f60b860b..2158e104eb 100644 --- a/deps/db-sync/src/logseq/db_sync/worker/handler/sync.cljs +++ b/deps/db-sync/src/logseq/db_sync/worker/handler/sync.cljs @@ -1,5 +1,6 @@ (ns logseq.db-sync.worker.handler.sync (:require [clojure.string :as string] + [datascript.core :as d] [lambdaisland.glogi :as log] [logseq.db :as ldb] [logseq.db-sync.batch :as batch] @@ -10,6 +11,7 @@ [logseq.db-sync.worker.http :as http] [logseq.db-sync.worker.routes.sync :as sync-routes] [logseq.db-sync.worker.ws :as ws] + [logseq.db.frontend.schema :as db-schema] [promesa.core :as p])) (def ^:private snapshot-download-batch-size 5000) @@ -264,7 +266,35 @@ (let [sql (.-sql self)] (ensure-conn! self) (let [conn (.-conn self) - tx-data (protocol/transit->tx txs)] + lookup-id (fn [x] + (when (and (vector? x) + (= 2 (count x)) + (= :block/uuid (first x))) + (second x))) + tx-data* (protocol/transit->tx txs) + created-block-uuids (->> tx-data* + (keep (fn [item] + (when (and (vector? item) + (= :db/add (first item)) + (>= (count item) 4) + (= :block/uuid (nth item 2))) + (nth item 3)))) + set) + missing-lookup-ref? (fn [x] + (when-let [block-uuid (lookup-id x)] + (and (not (contains? created-block-uuids block-uuid)) + (nil? (d/entity @conn x))))) + tx-data (remove (fn [item] + (when (vector? item) + (let [op (first item) + attr (nth item 2 nil) + value (when (>= (count item) 4) (nth item 3))] + (or (and (contains? #{:db/add :db/retract :db/retractEntity} op) + (missing-lookup-ref? (second item))) + (and (contains? #{:db/add :db/retract} op) + (contains? db-schema/ref-type-attributes attr) + (missing-lookup-ref? value)))))) + tx-data*)] (ldb/transact! conn tx-data {:op :apply-client-tx}) (let [new-t (storage/get-t sql)] ;; FIXME: no need to broadcast if client tx is less than remote tx diff --git a/deps/db-sync/test/logseq/db_sync/worker_handler_sync_test.cljs b/deps/db-sync/test/logseq/db_sync/worker_handler_sync_test.cljs index bc60f0ee33..2fee2699c6 100644 --- a/deps/db-sync/test/logseq/db_sync/worker_handler_sync_test.cljs +++ b/deps/db-sync/test/logseq/db_sync/worker_handler_sync_test.cljs @@ -1,8 +1,13 @@ (ns logseq.db-sync.worker-handler-sync-test - (:require [cljs.test :refer [async deftest is]] + (:require [cljs.test :refer [async deftest is testing]] + [datascript.core :as d] [logseq.db-sync.common :as common] + [logseq.db-sync.protocol :as protocol] [logseq.db-sync.storage :as storage] + [logseq.db-sync.test-sql :as test-sql] [logseq.db-sync.worker.handler.sync :as sync-handler] + [logseq.db-sync.worker.ws :as ws] + [logseq.db.frontend.schema :as db-schema] [promesa.core :as p])) (defn- empty-sql [] @@ -127,3 +132,22 @@ (p/catch (fn [error] (is false (str error)) (done))))))) + +(deftest tx-batch-drops-stale-lookup-entity-updates-test + (testing "stale lookup-ref entity updates should not reject the whole tx batch" + (let [sql (test-sql/make-sql) + conn (d/create-conn db-schema/schema) + self #js {:sql sql + :conn conn + :schema-ready true} + missing-uuid (random-uuid) + created-uuid (random-uuid) + tx-data [[:db/add [:block/uuid missing-uuid] :block/title "stale" 1] + [:db/add [:block/uuid missing-uuid] :block/updated-at 1773188050934 1] + [:db/add "temp-1" :block/uuid created-uuid 2] + [:db/add "temp-1" :block/title "ok" 2]] + response (with-redefs [ws/broadcast! (fn [& _] nil)] + (sync-handler/handle-tx-batch! self nil (protocol/tx->transit tx-data) 0))] + (is (= "tx/batch/ok" (:type response))) + (is (= "ok" (:block/title (d/entity @conn [:block/uuid created-uuid])))) + (is (nil? (d/entity @conn [:block/uuid missing-uuid])))))) diff --git a/deps/db/deps.edn b/deps/db/deps.edn index 5112ccdaf9..c54dff3713 100644 --- a/deps/db/deps.edn +++ b/deps/db/deps.edn @@ -1,7 +1,7 @@ {:deps ;; These nbb-logseq deps are kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn {datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork - :sha "ff5a7d5326e2546f40146e4a489343f557519bc3"} + :sha "f91fec561ee2c11d6bf323feae365e9033585411"} ;; datascript/datascript {:local/root "../../../../datascript"} datascript-transit/datascript-transit {:mvn/version "0.3.0" :exclusions [datascript/datascript]} diff --git a/deps/db/src/logseq/db/frontend/malli_schema.cljs b/deps/db/src/logseq/db/frontend/malli_schema.cljs index 16d6ee6f1f..2efcf516b1 100644 --- a/deps/db/src/logseq/db/frontend/malli_schema.cljs +++ b/deps/db/src/logseq/db/frontend/malli_schema.cljs @@ -510,7 +510,8 @@ [:logseq.property.asset/checksum :string] [:logseq.property.asset/size :int] [:logseq.property.asset/width {:optional true} :int] - [:logseq.property.asset/height {:optional true} :int]] + [:logseq.property.asset/height {:optional true} :int] + [:logseq.property.asset/align {:optional true} :keyword]] block-attrs page-or-block-attrs))) diff --git a/deps/db/src/logseq/db/frontend/property.cljs b/deps/db/src/logseq/db/frontend/property.cljs index 13ee76927a..29f285ff0d 100644 --- a/deps/db/src/logseq/db/frontend/property.cljs +++ b/deps/db/src/logseq/db/frontend/property.cljs @@ -567,6 +567,11 @@ :schema {:type :map :hide? true :public? false}} + :logseq.property.asset/align {:title "Asset alignment" + :schema {:type :keyword + :hide? true + :public? false} + :queryable? false} :logseq.property.fsrs/due {:title "Due" :schema {:type :datetime diff --git a/deps/db/src/logseq/db/frontend/schema.cljs b/deps/db/src/logseq/db/frontend/schema.cljs index 14586b399d..c6b2582114 100644 --- a/deps/db/src/logseq/db/frontend/schema.cljs +++ b/deps/db/src/logseq/db/frontend/schema.cljs @@ -30,7 +30,7 @@ (map (juxt :major :minor) [(parse-schema-version x) (parse-schema-version y)]))) -(def version (parse-schema-version "65.22")) +(def version (parse-schema-version "65.23")) (defn major-version "Return a number. diff --git a/deps/graph-parser/script/db_import.cljs b/deps/graph-parser/script/db_import.cljs index ecfe1a4d32..851665cd05 100644 --- a/deps/graph-parser/script/db_import.cljs +++ b/deps/graph-parser/script/db_import.cljs @@ -193,6 +193,9 @@ {:alias :P :coerce [] :desc "List of properties whose values convert to a parent class"} + :extract-code-snippets? + {:alias :C + :desc "Extract code fence(s) to #Code"} :validate {:alias :V :desc "Validate db after creation"}}) diff --git a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs index 6d7e1a997d..1373883a9d 100644 --- a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs @@ -57,6 +57,116 @@ :block/title new-title :block/name (common-util/page-name-sanity-lc new-title)}))) +(def template-file-property-names #{:template :template-including-parent}) + +(defn- get-template-name + [block] + (let [template-name (get-in block [:block/properties :template])] + (when (string? template-name) + (not-empty (string/trim template-name))))) + +(defn- template-including-parent? + [block] + (not= false (get-in block [:block/properties :template-including-parent]))) + +(defn- remove-template-property-lines + [title] + (if (string? title) + (->> (string/split-lines title) + (remove (fn [line] + (let [trimmed-line (string/triml line)] + (or (string/starts-with? trimmed-line "template::") + (string/starts-with? trimmed-line "template-including-parent::"))))) + (string/join "\n")) + title)) + +(defn- group-block-children-by-parent + [blocks] + (reduce (fn [result {parent :block/parent + child-uuid :block/uuid}] + (if (and (vector? parent) + (= :block/uuid (first parent))) + (update result (second parent) (fnil conj []) child-uuid) + result)) + {} + blocks)) + +(defn- get-block-subtree-uuids + [block-children root-uuid] + (loop [queue [root-uuid] + result []] + (if-let [current-uuid (first queue)] + (recur (into (vec (rest queue)) (get block-children current-uuid)) + (conj result current-uuid)) + result))) + +(defn- get-parent-uuid [parent] + (cond + (and (vector? parent) (= :block/uuid (first parent))) + (second parent) + (and (map? parent) (:block/uuid parent)) + (:block/uuid parent) + :else + nil)) + +(defn- handle-template-blocks + "Handles creating #Template blocks and their children and calculates + :preserve-empty-properties-uuids for use later" + [blocks*] + (let [include-parent-template-uuids (->> blocks* + (filter (fn [block] + (and (get-template-name block) + (template-including-parent? block)))) + (map :block/uuid) + set) + content-uuids-by-template (into {} + (map (fn [template-uuid] + [template-uuid (common-uuid/gen-uuid)])) + include-parent-template-uuids)] + (reduce + (fn [{:keys [blocks preserve-empty-properties-uuids]} block] + (if-let [template-name (get-template-name block)] + (let [block-children (group-block-children-by-parent blocks*) + parent-uuid (get-parent-uuid (:block/parent block)) + content-uuid (when parent-uuid + (get content-uuids-by-template parent-uuid)) + cleaned-block' (if content-uuid + (assoc block :block/parent [:block/uuid content-uuid]) + block) + source-preserve-empty-properties-uuids (set (get-block-subtree-uuids block-children (:block/uuid block))) + template-root-block (-> cleaned-block' + (assoc :block/title template-name) + (update :block/tags (fnil conj []) :logseq.class/Template) + (dissoc :block/properties)) + template-content-block (when (template-including-parent? block) + (-> (cond-> block + (seq (:block/properties block)) + (update :block/properties #(apply dissoc % template-file-property-names))) + (update :block/title remove-template-property-lines) + (assoc :block/uuid (get content-uuids-by-template (:block/uuid block)) + :block/parent [:block/uuid (:block/uuid block)] + :block/order (db-order/gen-key)) + (dissoc :db/id)))] + {:blocks (cond-> blocks + true + (conj template-root-block) + template-content-block + (conj template-content-block)) + :preserve-empty-properties-uuids (set/union preserve-empty-properties-uuids + source-preserve-empty-properties-uuids + (cond-> #{} + template-content-block + (conj (:block/uuid template-content-block))))}) + {:blocks (if-let [content-uuid (some->> (get-parent-uuid (:block/parent block)) + (get content-uuids-by-template))] + (conj blocks + (assoc block :block/parent [:block/uuid content-uuid])) + (conj blocks block)) + :preserve-empty-properties-uuids preserve-empty-properties-uuids})) + {:blocks [] + :preserve-empty-properties-uuids #{}} + blocks*))) + (defn- get-page-uuid [page-names-to-uuids page-name ex-data'] (or (get @page-names-to-uuids (some-> (if (string/includes? (str page-name) "#") (string/lower-case (gp-block/sanitize-hashtag-name page-name)) @@ -463,7 +573,8 @@ "All built-in property file ids as a set of keywords" (-> built-in-property-file-to-db-idents keys set ;; built-in-properties that map to new properties - (set/union #{:filters :query-table :query-properties :query-sort-by :query-sort-desc :hl-stamp :file :file-path}))) + (set/union #{:filters :query-table :query-properties :query-sort-by :query-sort-desc :hl-stamp :file :file-path}) + (set/union template-file-property-names))) ;; TODO: Review whether this should be using :block/title instead of file graph ids (def all-built-in-names @@ -481,7 +592,8 @@ #{:alias :tags :background-color :heading :query-table :query-properties :query-sort-by :query-sort-desc :ls-type :hl-type :hl-color :hl-page :hl-stamp :hl-value :file :file-path - :logseq.order-list-type :icon :public :exclude-from-graph-view :filters}) + :logseq.order-list-type :icon :public :exclude-from-graph-view :filters + :template :template-including-parent}) (assert (set/subset? file-built-in-property-names all-built-in-property-file-ids) "All file-built-in properties are used in db graph") @@ -693,21 +805,18 @@ (get-page-uuid page-names-to-uuids ((some-fn ::original-name :block/name) block) {:block block}) (:block/uuid block)) properties-text-values)) - ;; TODO: Add import support for :template. Ignore for now as they cause invalid property types - (if (contains? props :template) - {} - (let [props' (-> (update-built-in-property-values - (select-keys props file-built-in-property-names) - page-names-to-uuids - (select-keys import-state [:ignored-properties :all-idents]) - (select-keys block [:block/name :block/title]) - (select-keys user-options [:property-classes])) - (merge (update-user-property-values user-properties page-names-to-uuids properties-text-values import-state options))) - pvalue-tx-m (->property-value-tx-m block props' #(get-property-schema @property-schemas %) @all-idents) - block-properties (-> (merge props' (db-property-build/build-properties-with-ref-values pvalue-tx-m)) - (update-keys get-ident'))] - {:block-properties block-properties - :pvalues-tx (into (mapcat #(if (set? %) % [%]) (vals pvalue-tx-m)))})))) + (let [props' (-> (update-built-in-property-values + (select-keys props file-built-in-property-names) + page-names-to-uuids + (select-keys import-state [:ignored-properties :all-idents]) + (select-keys block [:block/name :block/title]) + (select-keys user-options [:property-classes])) + (merge (update-user-property-values user-properties page-names-to-uuids properties-text-values import-state options))) + pvalue-tx-m (->property-value-tx-m block props' #(get-property-schema @property-schemas %) @all-idents) + block-properties (-> (merge props' (db-property-build/build-properties-with-ref-values pvalue-tx-m)) + (update-keys get-ident'))] + {:block-properties block-properties + :pvalues-tx (into (mapcat #(if (set? %) % [%]) (vals pvalue-tx-m)))}))) (def ignored-built-in-properties "Ignore built-in properties that are already imported or not supported in db graphs" @@ -726,7 +835,7 @@ (defn- pre-update-properties "Updates page and block properties before their property types are inferred" - [properties class-related-properties] + [properties class-related-properties {:keys [preserve-empty-properties?]}] (let [dissoced-props (concat ignored-built-in-properties ;; TODO: Deal with these dissoced built-in properties [:title :created-at :updated-at] @@ -736,8 +845,9 @@ (if (not (contains? file-built-in-property-names prop)) ;; only update user properties (if (string? val) - ;; Ignore blank values as they were usually generated by templates - (when-not (string/blank? val) + ;; Ignore blank values outside template-related blocks to preserve existing import behavior + (when (or preserve-empty-properties? + (not (string/blank? val))) [prop ;; handle float strings b/c graph-parser doesn't (or (parse-double val) val)]) @@ -757,14 +867,15 @@ :keys [import-state macros] :as options}] (-> (if (seq properties) - (let [classes-from-properties (->> (select-keys properties property-classes) + (let [preserve-empty-properties? (contains? (or (:preserve-empty-property-block-uuids options) #{}) + (:block/uuid block)) + classes-from-properties (->> (select-keys properties property-classes) (mapcat (fn [[_k v]] (if (coll? v) v [v]))) distinct) - properties' (pre-update-properties properties (into property-classes property-parent-classes)) - properties-to-infer (if (:template properties') - ;; Ignore template properties as they don't consistently have representative property values - {} - (apply dissoc properties' file-built-in-property-names)) + properties' (pre-update-properties properties + (into property-classes property-parent-classes) + {:preserve-empty-properties? preserve-empty-properties?}) + properties-to-infer (apply dissoc properties' file-built-in-property-names) property-changes (->> properties-to-infer (keep (fn [[prop val]] @@ -944,6 +1055,7 @@ use in build-block-tx. This walk is only done once for perf reasons" [config ast-blocks] (let [results (atom {:simple-queries [] + :cards [] :asset-links [] :embeds [] :zotero-imported-files {} @@ -951,9 +1063,9 @@ (walk/prewalk (fn [x] (cond - (and (vector? x) - (= "Link" (first x)) - (let [path-or-map (second (:url (second x)))] + (and (vector? x) + (= "Link" (first x)) + (let [path-or-map (second (:url (second x)))] (cond (string? path-or-map) (or (common-config/local-relative-asset? path-or-map) @@ -967,19 +1079,23 @@ (= "Macro" (first x)) (= "embed" (:name (second x)))) (swap! results update :embeds conj x) - (and (vector? x) - (= "Macro" (first x)) - (= "zotero-imported-file" (:name (second x)))) - (let [[item-key filename] (:arguments (second x))] - (when (and item-key filename) - (swap! results update :zotero-imported-files assoc item-key (common-util/safe-read-string filename)))) - (and (vector? x) - (= "Macro" (first x)) - (= "zotero-linked-file" (:name (second x)))) - (let [[relative-path] (:arguments (second x)) - parsed-path (common-util/safe-read-string relative-path)] - (when (string? parsed-path) - (swap! results update :zotero-linked-files conj parsed-path))) + (and (vector? x) + (= "Macro" (first x)) + (= "zotero-imported-file" (:name (second x)))) + (let [[item-key filename] (:arguments (second x))] + (when (and item-key filename) + (swap! results update :zotero-imported-files assoc item-key (common-util/safe-read-string filename)))) + (and (vector? x) + (= "Macro" (first x)) + (= "zotero-linked-file" (:name (second x)))) + (let [[relative-path] (:arguments (second x)) + parsed-path (common-util/safe-read-string relative-path)] + (when (string? parsed-path) + (swap! results update :zotero-linked-files conj parsed-path))) + (and (vector? x) + (= "Macro" (first x)) + (= "cards" (:name (second x)))) + (swap! results update :cards conj x) (and (vector? x) (= "Macro" (first x)) (= "query" (:name (second x)))) @@ -989,7 +1105,8 @@ @results)) (defn- handle-queries - "If a block contains a simple or advanced queries, converts block to a #Query node" + "If a block contains a simple or advanced queries, converts block to a #Query node. If a block + contains a cards query converts to a #Cards node" [{:block/keys [title] :as block} db page-names-to-uuids walked-ast-blocks options] (if-let [query (some-> (first (:simple-queries walked-ast-blocks)) (ast->text (select-keys options [:log-fn])) @@ -1032,7 +1149,21 @@ (assoc :block/collapsed? true)))] {:block block' :pvalues-tx pvalues-tx'}) - {:block block}))) + (if-let [cards-macro (first (:cards walked-ast-blocks))] + (if-let [query (some-> cards-macro second :arguments first string/trim not-empty)] + (let [props {:logseq.property/query query} + {:keys [block-properties pvalues-tx]} + (build-properties-and-values props db page-names-to-uuids + (select-keys block [:block/properties-text-values :block/name :block/title :block/uuid]) + options) + block' + (-> (update block :block/tags (fnil conj []) :logseq.class/Cards) + (merge block-properties + {:block/title (string/trim (string/replace-first title #"\{\{cards(.*)\}\}" ""))}))] + {:block block' + :pvalues-tx pvalues-tx}) + {:block block}) + {:block block})))) (defn- handle-block-properties "Does everything page properties does and updates a couple of block specific attributes" @@ -1390,6 +1521,77 @@ :block/tags [:logseq.class/Quote-block]}) block)) +(defn- handle-math + "If a block's entire content is a single displayed math formula, convert to #Math node. + Detects blocks whose title is entirely delimited by $$ markers." + [block] + (let [title (string/trim (:block/title block))] + (if (and (string/starts-with? title "$$") + (string/ends-with? title "$$") + (> (count title) 4) + ;; ensure there's no nested $$ pair (i.e. not two separate inline formulas) + (not (string/includes? (subs title 2 (- (count title) 2)) "$$"))) + (let [math-content (string/trim (subs title 2 (- (count title) 2)))] + (merge block + {:block/title math-content + :logseq.property.node/display-type :math + :block/tags [:logseq.class/Math-block]})) + block))) + +(defn- split-title-by-code-fences + "Parses a block title string line-by-line, splitting into non-code text parts + and code fence segments. All code fences are extracted regardless of whether + they have a language tag; :lang is nil when not specified. + Returns {:text-parts [...] :code-segs [{:text ... :lang ...}]}." + [title] + (let [lines (string/split-lines title)] + (loop [remaining lines + in-code? false + lang nil + current [] + text-parts [] + code-segs []] + (if (empty? remaining) + {:text-parts (if (seq current) + (conj text-parts (string/join "\n" current)) + text-parts) + :code-segs code-segs} + (let [line (first remaining) + trimmed-line (string/trim line) + fence-start? (and (not in-code?) (re-matches #"```.*" trimmed-line)) + fence-end? (and in-code? (= trimmed-line "```"))] + (cond + fence-start? + (recur (rest remaining) true (not-empty (subs trimmed-line 3)) [] + (if (seq current) + (conj text-parts (string/join "\n" current)) + text-parts) + code-segs) + fence-end? + (recur (rest remaining) false nil [] + text-parts + (conj code-segs {:text (string/join "\n" current) :lang lang})) + :else + (recur (rest remaining) in-code? lang (conj current line) + text-parts code-segs))))))) + +(defn- build-code-snippet-child-blocks + "Builds child block tx maps for extracted code snippets, tagging each as a + Code-block with its detected language." + [parent-block code-segs] + (mapv (fn [{:keys [text lang]}] + (cond-> (sqlite-util/block-with-timestamps + {:block/uuid (d/squuid) + :block/title text + :block/parent [:block/uuid (:block/uuid parent-block)] + :block/page (:block/page parent-block) + :block/order (db-order/gen-key) + :block/tags [:logseq.class/Code-block] + :logseq.property.node/display-type :code}) + lang + (assoc :logseq.property.code/lang lang))) + code-segs)) + (defn- handle-embeds "If a block contains page or block embeds, converts block to a :block/link based embed" [block page-names-to-uuids {:keys [embeds]} {:keys [log-fn] :or {log-fn prn}}] @@ -1414,6 +1616,55 @@ block)) block)) +(defn- at-least-two? + [s substr] + (if (empty? substr) + false + (loop [start 0 cnt 0] + (let [idx (string/index-of s substr start)] + (cond + (>= cnt 2) true + (nil? idx) false + :else (recur (+ idx (count substr)) (inc cnt))))))) + +(defn- handle-code-blocks + "Returns a vector of block and optional block children tx. If a block + contains code fence(s) i.e. ```, converts block to a #Code node. If user + enables :extract-code-snippets? option, multiple code fences are extracted out + of text and put into children blocks in the order they appear" + [block' options] + (let [title (:block/title block') + has-fence? (and (string? title) (at-least-two? title "```")) + extract? (get-in options [:user-options :extract-code-snippets?]) + [final-block code-children-tx] + (if has-fence? + (let [{:keys [text-parts code-segs]} (split-title-by-code-fences title) + pure-single-code? (and (= 1 (count code-segs)) + (every? string/blank? text-parts)) + has-mixed-content? (and extract? + (seq code-segs) + (some #(not (string/blank? %)) text-parts))] + (cond + pure-single-code? + (let [{:keys [text lang]} (first code-segs)] + [(cond-> (assoc block' + :block/title text + :block/tags [:logseq.class/Code-block] + :logseq.property.node/display-type :code) + lang (assoc :logseq.property.code/lang lang)) + []]) + has-mixed-content? + (let [remaining-title (-> (string/join "\n" text-parts) + (string/replace #"\n{2,}" "\n") + string/trim) + updated-block (assoc block' :block/title remaining-title) + code-children (build-code-snippet-child-blocks updated-block code-segs)] + [updated-block code-children]) + :else + [block' []])) + [block' []])] + [final-block code-children-tx])) + (defn- > (:block/_parent block) + (remove :logseq.property/created-from-property) + (sort-by :block/order) + vec)) + +(defn- block-tree-with-properties + [block] + {:title (:block/title block) + :properties (dissoc (db-test/readable-properties block) :block/tags) + :children (mapv block-tree-with-properties (ordered-children block))}) + +(defn- find-template-by-title + [db title] + (some->> (d/q '[:find [?b ...] + :in $ ?title + :where + [?b :block/title ?title] + [?b :block/tags :logseq.class/Template]] + db title) + first + (d/entity db))) + +(defn- template-content-trees + [db title] + (some->> (find-template-by-title db title) + ordered-children + (mapv block-tree-with-properties))) + + (defn- build-graph-files "Given a file graph directory, return all files including assets and adds relative paths on ::rpath since paths are absolute by default and exporter needs relative paths for @@ -54,8 +86,8 @@ [dir*] (let [dir (node-path/resolve dir*)] (->> (common-graph/get-files dir) - (concat (when (fs/existsSync (node-path/join dir* "assets")) - (common-graph/readdir (node-path/join dir* "assets")))) + (concat (when (fs/existsSync (path/path-join dir* "assets")) + (common-graph/readdir (path/path-join dir* "assets")))) (mapv #(hash-map :path % ::rpath (node-path/relative dir* %)))))) @@ -173,6 +205,84 @@ "assets/subdir/partydino.gif"] "[[FIRST UUID]] and [[UUID]]")) +(deftest extract-template-blocks + (let [page-uuid (random-uuid) + parent-uuid (random-uuid) + child-uuid (random-uuid) + include-children-only-uuid (random-uuid) + child-only-1-uuid (random-uuid) + child-only-2-uuid (random-uuid) + blocks [{:block/uuid parent-uuid + :block/title "source parent" + :block/page [:block/uuid page-uuid] + :block/parent {:block/uuid page-uuid} + :block/order "a" + :block/properties {:template " trimmed template " + :name ""} + :block/properties-text-values {:template " trimmed template " + :name ""} + :block/properties-order [:template :name]} + {:block/uuid child-uuid + :block/title "child" + :block/page [:block/uuid page-uuid] + :block/parent [:block/uuid parent-uuid] + :block/order "b" + :block/properties {:template "nested child" + :name "child default"} + :block/properties-text-values {:template "nested child" + :name "child default"} + :block/properties-order [:template :name]} + {:block/uuid include-children-only-uuid + :block/title "exclude source block" + :block/page [:block/uuid page-uuid] + :block/parent {:block/uuid page-uuid} + :block/order "c" + :block/properties {:template "children only" + :template-including-parent false} + :block/properties-text-values {:template "children only" + :template-including-parent "false"} + :block/properties-order [:template :template-including-parent]} + {:block/uuid child-only-1-uuid + :block/title "first child" + :block/page [:block/uuid page-uuid] + :block/parent [:block/uuid include-children-only-uuid] + :block/order "d"} + {:block/uuid child-only-2-uuid + :block/title "second child" + :block/page [:block/uuid page-uuid] + :block/parent [:block/uuid include-children-only-uuid] + :block/order "e"}] + {:keys [blocks preserve-empty-properties-uuids]} + (#'gp-exporter/handle-template-blocks blocks)] + (testing "template roots replace source blocks" + (is (= ["trimmed template" + "source parent" + "nested child" + "child" + "children only" + "first child" + "second child"] + (mapv :block/title blocks))) + (is (= #{parent-uuid child-uuid include-children-only-uuid child-only-1-uuid child-only-2-uuid} + (set/intersection preserve-empty-properties-uuids + #{parent-uuid child-uuid include-children-only-uuid child-only-1-uuid child-only-2-uuid})))) + + (testing "template roots use trimmed names and include parent content when configured" + (is (= #{"trimmed template" "nested child" "children only"} + (->> blocks + (filter #(some #{:logseq.class/Template} (:block/tags %))) + (map :block/title) + set))) + (is (= ["source parent"] + (->> blocks + (remove #(some #{:logseq.class/Template} (:block/tags %))) + (filter #(= [:block/uuid parent-uuid] (:block/parent %))) + (map :block/title)))) + (is (= 2 + (count (set/difference preserve-empty-properties-uuids + #{parent-uuid child-uuid include-children-only-uuid child-only-1-uuid child-only-2-uuid}))) + "in-place template content blocks are marked to preserve empty properties")))) + (deftest-async ^:integration export-docs-graph-with-convert-all-tags (p/let [file-graph-dir "test/resources/docs-0.10.12" start-time (cljs.core/system-time) @@ -216,12 +326,16 @@ ;; Counts ;; Includes journals as property values e.g. :logseq.property/deadline - (is (= 33 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Journal]] @conn)))) + (is (= 34 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Journal]] @conn)))) (is (= 9 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Asset]] @conn)))) (is (= 5 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Task]] @conn)))) (is (= 4 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Query]] @conn)))) (is (= 2 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Card]] @conn)))) + (is (= 1 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Cards]] @conn)))) + (is (= 2 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Code-block]] @conn)))) + (is (= 1 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Math-block]] @conn)))) + (is (= 9 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Template]] @conn)))) (is (= 5 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Quote-block]] @conn)))) (is (= 7 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Pdf-annotation]] @conn)))) @@ -264,7 +378,7 @@ set)))) (testing "user properties" - (is (= 21 + (is (= 23 (->> @conn (d/q '[:find [(pull ?b [:db/ident]) ...] :where [?b :block/tags :logseq.class/Property]]) @@ -331,11 +445,6 @@ (mapv :db/id (:block/refs (db-test/find-block-by-content @conn #"ref to")))) "block with a block-ref has correct :block/refs") - (let [b (db-test/find-block-by-content @conn #"MEETING TITLE")] - (is (= {} - (and b (db-test/readable-properties b))) - ":template properties are ignored to not invalidate its property types")) - (is (= 20221126 (-> (db-test/readable-properties (db-test/find-block-by-content @conn "only deadline")) :logseq.property/deadline @@ -413,11 +522,107 @@ (:block/title (db-test/find-block-by-content @conn #"tasks with todo"))) "Advanced query has custom title migrated") - ;; Cards + ;; Card (is (= {:block/tags [:logseq.class/Card]} (db-test/readable-properties (db-test/find-block-by-content @conn "card 1"))) "None of the card properties are imported since they are deprecated") + ;; Cards (flashcard browser) + (is (= {:block/tags [:logseq.class/Cards] + :logseq.property/query "(tags #Card)"} + (db-test/readable-properties (find-block-by-property-value @conn :logseq.property/query "(tags #Card)"))) + "cards macro block has correct Cards class and query property") + + ;; Math blocks + (is (= {:block/tags [:logseq.class/Math-block] + :logseq.property.node/display-type :math} + (db-test/readable-properties (db-test/find-block-by-content @conn "E=mc^2"))) + "Math block has correct Math-block class and display-type") + (is (= "E=mc^2" (:block/title (db-test/find-block-by-content @conn "E=mc^2"))) + "Math block title has delimiters stripped") + + ;; Templates + (is (= #{"meeting" + "title-only-no-children" + "properties-only-no-children" + "title-only-with-children" + "empty-title-with-children" + "children-only" + "nested-father" + "nested-child-1" + "nested-child-2"} + (->> (d/q '[:find [?title ...] + :where + [?b :block/tags :logseq.class/Template] + [?b :block/title ?title]] + @conn) + set)) + "All template definitions are imported as Template blocks") + (let [journal-uuid (:block/uuid (db-test/find-journal-by-journal-day @conn 20240216)) + template-page-uuids (->> (d/q '[:find [?page-uuid ...] + :where + [?b :block/tags :logseq.class/Template] + [?b :block/page ?page] + [?page :block/uuid ?page-uuid]] + @conn) + set)] + (is (= #{journal-uuid} template-page-uuids) + "All template blocks are created on their source journal page")) + (is (= [{:title "MEETING TITLE" + :properties {:user.property/participants #{"TODO"}} + :children []}] + (template-content-trees @conn "meeting"))) + (is (= [{:title "TITLE" + :properties {} + :children []}] + (template-content-trees @conn "title-only-no-children"))) + (is (= [{:title "" + :properties {:user.property/name "" + :user.property/author ""} + :children []}] + (template-content-trees @conn "properties-only-no-children"))) + (is (= [{:title "TITLE" + :properties {} + :children [{:title "intro" :properties {} :children []} + {:title "notes" :properties {} :children []}]}] + (template-content-trees @conn "title-only-with-children"))) + (is (= [{:title "" + :properties {} + :children [{:title "intro" :properties {} :children []} + {:title "notes" :properties {} :children []}]}] + (template-content-trees @conn "empty-title-with-children"))) + (is (= [{:title "intro" :properties {} :children []} + {:title "notes" :properties {} :children []}] + (template-content-trees @conn "children-only"))) + (is (= [{:title "it's a template with nested templates" + :properties {:user.property/name "you named it"} + :children [{:title "nested-child-1" + :properties {} + :children [{:title "child-1" + :properties {:user.property/name ""} + :children [{:title "child-1-1" + :properties {:user.property/name ""} + :children []}]}]} + {:title "nested-child-2" + :properties {} + :children [{:title "child-2-1" + :properties {:user.property/name ""} + :children []}]} + {:title "child-3" + :properties {:user.property/name ""} + :children []}]}] + (template-content-trees @conn "nested-father"))) + (is (= [{:title "child-1" + :properties {:user.property/name ""} + :children [{:title "child-1-1" + :properties {:user.property/name ""} + :children []}]}] + (template-content-trees @conn "nested-child-1"))) + (is (= [{:title "child-2-1" + :properties {:user.property/name ""} + :children []}] + (template-content-trees @conn "nested-child-2"))) + ;; Assets (is (= {:block/tags [:logseq.class/Asset] :logseq.property.asset/type "png" @@ -652,9 +857,12 @@ (is (= :node (:logseq.property/type (d/entity @conn :user.property/finishedat))) ":date property to :node value changes to :node") - (is (= :node + (is (= :default (:logseq.property/type (d/entity @conn :user.property/participants))) - ":node property to :date value remains :node") + "template values cause participants to remain a :default property") + (is (= #{"[[Feb 7th, 2024]]"} + (:user.property/participants (db-test/readable-properties (db-test/find-block-by-content @conn #"test :node -> :date")))) + ":default participants property keeps the imported text value") (is (= :default (:logseq.property/type (d/entity @conn :user.property/description))) @@ -807,7 +1015,7 @@ (deftest-async export-files-with-tag-classes-option (p/let [file-graph-dir "test/resources/exporter-test-graph" - files (mapv #(node-path/join file-graph-dir %) ["journals/2024_02_07.md" "pages/Interstellar.md"]) + files (mapv #(path/path-join file-graph-dir %) ["journals/2024_02_07.md" "pages/Interstellar.md"]) conn (db-test/create-conn) _ (import-files-to-db files conn {:tag-classes ["movie"]})] (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn)))) @@ -833,7 +1041,7 @@ (deftest-async export-files-with-property-classes-option (p/let [file-graph-dir "test/resources/exporter-test-graph" - files (mapv #(node-path/join file-graph-dir %) + files (mapv #(path/path-join file-graph-dir %) ["journals/2024_02_23.md" "pages/url.md" "pages/Whiteboard___Tool.md" "pages/Whiteboard___Arrow_head_toggle.md" "pages/Library.md"]) @@ -880,7 +1088,7 @@ (deftest-async export-files-with-remove-inline-tags (p/let [file-graph-dir "test/resources/exporter-test-graph" - files (mapv #(node-path/join file-graph-dir %) ["journals/2024_02_07.md" + files (mapv #(path/path-join file-graph-dir %) ["journals/2024_02_07.md" "journals/2026_01_27.md"]) conn (db-test/create-conn) _ (import-files-to-db files conn {:remove-inline-tags? false :convert-all-tags? true})] @@ -896,7 +1104,7 @@ (deftest-async export-files-with-ignored-properties (p/let [file-graph-dir "test/resources/exporter-test-graph" - files (mapv #(node-path/join file-graph-dir %) ["ignored/icon-page.md"]) + files (mapv #(path/path-join file-graph-dir %) ["ignored/icon-page.md"]) conn (db-test/create-conn) {:keys [import-state]} (import-files-to-db files conn {})] (is (= 2 @@ -905,7 +1113,7 @@ (deftest-async export-files-with-property-parent-classes-option (p/let [file-graph-dir "test/resources/exporter-test-graph" - files (mapv #(node-path/join file-graph-dir %) ["journals/2024_11_26.md" + files (mapv #(path/path-join file-graph-dir %) ["journals/2024_11_26.md" "pages/CreativeWork.md" "pages/Movie.md" "pages/type.md" "pages/Whiteboard___Tool.md" "pages/Whiteboard___Arrow_head_toggle.md" "pages/Property.md" "pages/url.md"]) @@ -933,7 +1141,7 @@ (deftest-async export-files-with-property-pages-disabled (p/let [file-graph-dir "test/resources/exporter-test-graph" ;; any page with properties - files (mapv #(node-path/join file-graph-dir %) ["journals/2024_01_17.md"]) + files (mapv #(path/path-join file-graph-dir %) ["journals/2024_01_17.md"]) conn (db-test/create-conn) _ (import-files-to-db files conn {:user-config {:property-pages/enabled? false :property-pages/excludelist #{:prop-string}}})] @@ -948,3 +1156,228 @@ (is (= "yyyy-MM-dd" (:logseq.property.journal/title-format (d/entity @conn :logseq.class/Journal))) "title format set correctly by config"))) + +(deftest split-title-by-code-fences + (let [split-fn #'gp-exporter/split-title-by-code-fences] + (testing "standalone code fence with language" + (is (= {:text-parts [] + :code-segs [{:text "it's an individual code snippet with language tag" + :lang "markdown"}]} + (split-fn "```markdown\nit's an individual code snippet with language tag\n```")))) + + (testing "standalone code fence without language" + (is (= {:text-parts [] + :code-segs [{:text "it's an individual code snippet without language tag" + :lang nil}]} + (split-fn "```\nit's an individual code snippet without language tag\n```")))) + + (testing "one code fence with leading text" + (is (= {:text-parts ["before code snippet"] + :code-segs [{:text "echo \"ok\"\nexit" + :lang nil}]} + (split-fn "before code snippet\n```\necho \"ok\"\nexit\n```")))) + + (testing "one code fence with leading and trailing text" + (is (= {:text-parts ["before code snippet" "after code snippet"] + :code-segs [{:text "echo \"ok\"\nexit" + :lang "bash"}]} + (split-fn "before code snippet\n```bash\necho \"ok\"\nexit\n```\nafter code snippet")))) + + (testing "one code fence followed by trailing text" + (is (= {:text-parts ["after code snippet"] + :code-segs [{:text "echo \"ok\"\nexit" + :lang "bash"}]} + (split-fn "```bash\necho \"ok\"\nexit\n```\nafter code snippet")))) + + (testing "multiple code fences mixed with text" + (is (= {:text-parts ["before code snippet" "middle" "after code snippet"] + :code-segs [{:text "echo \"ok\"\nexit" + :lang "bash"} + {:text "echo \"bye\"\nexit" + :lang "bash"}]} + (split-fn "before code snippet\n```bash\necho \"ok\"\nexit\n```\nmiddle\n```bash\necho \"bye\"\nexit\n```\nafter code snippet")))) + + (testing "edge: one code fence followed by opening fence without closing fence" + (is (= {:text-parts ["echo \"missing end fence\""] ;; no "```bash" ahead, it's fine as is; let's leave it + :code-segs [{:text "echo \"ok\"\nexit" + :lang "bash"}]} + (split-fn "```bash\necho \"ok\"\nexit\n```\n```bash\necho \"missing end fence\"")))) + + (testing "edge: pure multiple code fences with no extra text" + (let [{:keys [text-parts code-segs]} (split-fn "```markdown\n1st code snippet with language tag\n```\n```\n2nd code snippet without language tag\n```")] + (is (and (empty? text-parts) (> (count code-segs) 1)) "not pure single code and no mixed content"))) + + (testing "edge: opening fence without closing fence" + (let [title "```bash\necho \"missing end fence\"" + {:keys [text-parts code-segs]} (split-fn title)] + (is (and (= (count text-parts) 1) (not= (first text-parts) title) (empty? code-segs)) "not pure single code and no mixed content"))) + + (testing "edge: plain text without any code fence" + (is (= {:text-parts ["plain text only"] + :code-segs []} + (split-fn "plain text only")))) + + (testing "edge: empty title" + (is (= {:text-parts [""] + :code-segs []} + (split-fn "")))))) + +(deftest-async export-files-with-extract-code-snippet + (p/let [file-graph-dir "test/resources/exporter-test-graph" + files (mapv #(path/path-join file-graph-dir %) ["journals/2026_03_01.md"]) + conn (db-test/create-conn) + _ (import-files-to-db files conn {:extract-code-snippets? true}) + journal-page-eid (d/q '[:find ?p . :where [?p :block/journal-day 20260301]] @conn) + top-blocks (->> (d/q '[:find [?b ...] + :in $ ?page + :where + [?b :block/page ?page] + [?b :block/parent ?page]] + @conn journal-page-eid) + (map #(d/entity @conn %)) + (sort-by :block/order) + vec) + get-direct-children (fn [block] + (->> (d/q '[:find [?c ...] + :in $ ?parent + :where [?c :block/parent ?parent]] + @conn (:db/id block)) + (map #(d/entity @conn %))))] + + (testing "standalone code block with language tag" + (let [b (nth top-blocks 0)] + (is (= "it's an individual code snippet with language tag" (:block/title b)) + "Standalone code block title has fences stripped") + (is (= 0 (count (get-direct-children b))) + "Standalone code block has no children") + (is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags b)))) + "Standalone code block is tagged as Code-block") + (is (= "markdown" (:logseq.property.code/lang b)) + "Standalone code block has markdown language property"))) + + (testing "standalone code block without language tag" + (let [b (nth top-blocks 1)] + (is (= "it's an individual code snippet without language tag" (:block/title b)) + "Standalone code block title has fences stripped") + (is (= 0 (count (get-direct-children b))) + "Standalone code block has no children") + (is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags b)))) + "Standalone code block is tagged as Code-block") + (is (= nil (:logseq.property.code/lang b)) + "Standalone code block has no language property"))) + + (testing "text before code snippet" + (let [b (nth top-blocks 2) + children (get-direct-children b)] + (is (= "before code snippet" (:block/title b)) + "Block title has text only without code") + (is (= 1 (count children)) + "Block has 1 code child") + (is (= "echo \"ok\"\nexit" (:block/title (first children))) + "Child code block has correct content without fence markers") + (is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags (first children))))) + "Child block is tagged as Code-block") + (is (= nil (:logseq.property.code/lang (first children))) + "Child block has no language property"))) + + (testing "text before and after code snippet" + (let [b (nth top-blocks 3) + children (get-direct-children b)] + (is (= "before code snippet\nafter code snippet" (:block/title b)) + "Block title has text only without code") + (is (= 1 (count children)) + "Block has 1 code child") + (is (= "echo \"ok\"\nexit" (:block/title (first children))) + "Child code block has correct content without fence markers") + (is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags (first children))))) + "Child block is tagged as Code-block") + (is (= "bash" (:logseq.property.code/lang (first children))) + "Child block has bash language property"))) + + (testing "code snippet before text" + (let [b (nth top-blocks 4) + children (get-direct-children b)] + (is (= "after code snippet" (:block/title b)) + "Block title has text only without code") + (is (= 1 (count children)) + "Block has 1 code child") + (is (= "echo \"ok\"\nexit" (:block/title (first children))) + "Child code block has correct content without fence markers") + (is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags (first children))))) + "Child block is tagged as Code-block") + (is (= "bash" (:logseq.property.code/lang (first children))) + "Child block has bash language property"))) + + (testing "multiple code snippets mixed with text" + (let [b (nth top-blocks 5) + children (sort-by :block/order (get-direct-children b))] + (is (= "before code snippet\nmiddle\nafter code snippet" (:block/title b)) + "Block title has all text parts without code") + (is (= 2 (count children)) + "Block has 2 code children") + (is (= "echo \"ok\"\nexit" (:block/title (first children))) + "First child code block has correct content without fence markers") + (is (= "echo \"bye\"\nexit" (:block/title (second children))) + "Second child code block has correct content without fence markers") + (is (every? #(= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags %)))) children) + "Both child blocks are tagged as Code-block") + (is (every? #(= "bash" (:logseq.property.code/lang %)) children) + "Both child blocks have bash language property"))))) + +(deftest-async export-files-without-extract-code-snippet + (p/let [file-graph-dir "test/resources/exporter-test-graph" + files (mapv #(path/path-join file-graph-dir %) ["journals/2026_03_01.md"]) + conn (db-test/create-conn) + _ (import-files-to-db files conn {:extract-code-snippets? false}) + journal-page-eid (d/q '[:find ?p . :where [?p :block/journal-day 20260301]] @conn) + top-blocks (->> (d/q '[:find [?b ...] + :in $ ?page + :where + [?b :block/page ?page] + [?b :block/parent ?page]] + @conn journal-page-eid) + (map #(d/entity @conn %)) + (sort-by :block/order) + vec) + get-direct-children (fn [block] + (->> (d/q '[:find [?c ...] + :in $ ?parent + :where [?c :block/parent ?parent]] + @conn (:db/id block)) + (map #(d/entity @conn %))))] + + (testing "standalone code block with language tag is still tagged as Code-block" + (let [b (nth top-blocks 0)] + (is (= "it's an individual code snippet with language tag" (:block/title b)) + "Standalone code block title has fences stripped") + (is (= 0 (count (get-direct-children b))) + "Standalone code block has no children") + (is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags b)))) + "Standalone code block is tagged as Code-block") + (is (= "markdown" (:logseq.property.code/lang b)) + "Standalone code block has markdown language property"))) + + (testing "standalone code block without language tag is still tagged as Code-block" + (let [b (nth top-blocks 1)] + (is (= "it's an individual code snippet without language tag" (:block/title b)) + "Standalone code block title has fences stripped") + (is (= 0 (count (get-direct-children b))) + "Standalone code block has no children") + (is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags b)))) + "Standalone code block is tagged as Code-block") + (is (= nil (:logseq.property.code/lang b)) + "Standalone code block has no language property"))) + + (testing "mixed-content block is NOT extracted into children when extract-code-snippets? is false" + (let [b (nth top-blocks 2)] + (is (= 0 (count (get-direct-children b))) + "Block with text before code has no children extracted") + (is (string/includes? (:block/title b) "```") + "Block title retains raw code fence markup"))) + + (testing "another mixed-content block is NOT extracted when extract-code-snippets? is false" + (let [b (nth top-blocks 3)] + (is (= 0 (count (get-direct-children b))) + "Block with text surrounding code has no children extracted") + (is (string/includes? (:block/title b) "```") + "Block title retains raw code fence markup"))))) diff --git a/deps/graph-parser/test/resources/exporter-test-graph/journals/2024_02_16.md b/deps/graph-parser/test/resources/exporter-test-graph/journals/2024_02_16.md index e6c3a810dc..64a804d7cf 100644 --- a/deps/graph-parser/test/resources/exporter-test-graph/journals/2024_02_16.md +++ b/deps/graph-parser/test/resources/exporter-test-graph/journals/2024_02_16.md @@ -4,6 +4,41 @@ - MEETING TITLE template:: meeting participants:: TODO +- TITLE + template:: title-only-no-children +- template:: properties-only-no-children + name:: + author:: +- TITLE + template:: title-only-with-children + - intro + - notes +- + template:: empty-title-with-children + - intro + - notes +- it should not be included in the template + template:: children-only + template-including-parent:: false + - intro + - notes +- it's a template with nested templates + template:: nested-father + template-including-parent:: true + name:: you named it + - child-1 + template:: nested-child-1 + name:: + - child-1-1 + name:: + - child-2 + template:: nested-child-2 + template-including-parent:: false + name:: + - child-2-1 + name:: + - child-3 + name:: - pending block for :number to :default duration:: 10 - test :number to :default diff --git a/deps/graph-parser/test/resources/exporter-test-graph/journals/2026_03_01.md b/deps/graph-parser/test/resources/exporter-test-graph/journals/2026_03_01.md new file mode 100644 index 0000000000..a7d0b41a98 --- /dev/null +++ b/deps/graph-parser/test/resources/exporter-test-graph/journals/2026_03_01.md @@ -0,0 +1,35 @@ +- ```markdown + it's an individual code snippet with language tag + ``` +- ``` + it's an individual code snippet without language tag + ``` +- before code snippet + ``` + echo "ok" + exit + ``` +- before code snippet + ```bash + echo "ok" + exit + ``` + after code snippet +- ```bash + echo "ok" + exit + ``` + after code snippet +- before code snippet + ```bash + echo "ok" + exit + ``` + middle + ```bash + echo "bye" + exit + ``` + after code snippet +- $$E=mc^2$$ +- {{cards (tags #Card)}} \ No newline at end of file diff --git a/deps/outliner/deps.edn b/deps/outliner/deps.edn index 3cf51bdd39..f168846a37 100644 --- a/deps/outliner/deps.edn +++ b/deps/outliner/deps.edn @@ -1,7 +1,7 @@ {:deps ;; These nbb-logseq deps are kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn {datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork - :sha "ff5a7d5326e2546f40146e4a489343f557519bc3"} + :sha "f91fec561ee2c11d6bf323feae365e9033585411"} ;; datascript/datascript {:local/root "../../../../datascript"} com.cognitect/transit-cljs {:mvn/version "0.8.280"} diff --git a/deps/publish/deps.edn b/deps/publish/deps.edn index d0d209a905..7089766b52 100644 --- a/deps/publish/deps.edn +++ b/deps/publish/deps.edn @@ -5,7 +5,7 @@ :sha "5d672bf84ed944414b9f61eeb83808ead7be9127"} datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork - :sha "ff5a7d5326e2546f40146e4a489343f557519bc3"} + :sha "f91fec561ee2c11d6bf323feae365e9033585411"} datascript-transit/datascript-transit {:mvn/version "0.3.0" :exclusions [datascript/datascript]} funcool/promesa {:mvn/version "11.0.678"} diff --git a/scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs b/scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs index 65659e124c..b27d12b2ff 100644 --- a/scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs +++ b/scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs @@ -9,41 +9,80 @@ [nbb.classpath :as cp] [nbb.core :as nbb])) -(def *ids (atom #{})) -(defn get-next-id - [] - (let [id (random-uuid)] - (if (@*ids id) - (get-next-id) - (do - (swap! *ids conj id) - id)))) +(def ^:private default-block-title "Block") +(def ^:private target-entities-per-batch 25000) +(def ^:private max-pages-per-batch 1000) -(defn build-pages - [start-idx n] - (let [ids (repeatedly n get-next-id)] - (map-indexed - (fn [idx id] - {:block/uuid id - :block/title (str "Page-" (+ start-idx idx))}) - ids))) +(defn- parse-long-option + [value] + (if (string? value) + (js/parseInt value 10) + value)) -(defn build-blocks - [size] - (vec (repeatedly size - (fn [] - (let [id (get-next-id)] - {:block/uuid id - :block/title (str id)}))))) - -(defn- create-init-data +(defn- normalize-options [options] - (let [pages (build-pages 0 (:pages options))] - {:pages-and-blocks - (mapv #(hash-map :page % :blocks (build-blocks (:blocks options))) - pages) - ;; Custom id fn because transaction chunks may separate blocks and pages from each other - :page-id-fn (fn [b] [:block/uuid (:block/uuid b)])})) + (update-vals options parse-long-option)) + +(defn default-batch-pages + [blocks-per-page] + (-> (quot target-entities-per-batch (max 1 (inc blocks-per-page))) + (max 1) + (min max-pages-per-batch))) + +(defn- build-blocks + [blocks-per-page next-id] + (loop [block-idx 0 + blocks (transient [])] + (if (= block-idx blocks-per-page) + (persistent! blocks) + (recur (inc block-idx) + (conj! blocks + {:block/uuid (next-id) + :block/title default-block-title}))))) + +(defn build-page-and-blocks-batch + ([start-idx page-count blocks-per-page] + (build-page-and-blocks-batch start-idx page-count blocks-per-page random-uuid)) + ([start-idx page-count blocks-per-page next-id] + (loop [page-idx 0 + pages-and-blocks (transient [])] + (if (= page-idx page-count) + (persistent! pages-and-blocks) + (recur (inc page-idx) + (conj! pages-and-blocks + {:page {:block/uuid (next-id) + :block/title (str "Page-" (+ start-idx page-idx))} + :blocks (build-blocks blocks-per-page next-id)})))))) + +(defn page-and-block-batches + ([{:keys [pages blocks batch-pages]}] + (page-and-block-batches {:pages pages + :blocks blocks + :batch-pages batch-pages} + random-uuid)) + ([{:keys [pages blocks batch-pages]} next-id] + (let [batch-pages' (or batch-pages (default-batch-pages blocks))] + ((fn step [start-idx] + (lazy-seq + (when (< start-idx pages) + (cons (build-page-and-blocks-batch start-idx + (min batch-pages' (- pages start-idx)) + blocks + next-id) + (step (+ start-idx batch-pages')))))) + 0)))) + +(defn- transact-batch! + [conn pages-and-blocks] + (let [{:keys [init-tx block-props-tx]} (outliner-cli/build-blocks-tx {:pages-and-blocks pages-and-blocks})] + (d/transact! conn init-tx) + (when (seq block-props-tx) + (d/transact! conn block-props-tx)))) + +(defn- total-batches + [{:keys [pages blocks batch-pages]}] + (let [batch-pages' (or batch-pages (default-batch-pages blocks))] + (js/Math.ceil (/ pages batch-pages')))) (def spec "Options spec" @@ -54,34 +93,36 @@ :desc "Number of pages to create"} :blocks {:alias :b :default 20 - :desc "Number of blocks to create"}}) + :desc "Number of blocks to create per page"} + :batch-pages {:alias :t + :desc "Number of pages to build and transact per batch"}}) + +(defn parse-args + [args] + {:graph-dir (first args) + :options (normalize-options (cli/parse-opts (rest args) {:spec spec}))}) (defn -main [args] - (let [graph-dir (first args) - options (cli/parse-opts args {:spec spec}) + (let [{:keys [graph-dir options]} (parse-args args) _ (when (or (nil? graph-dir) (:help options)) (println (str "Usage: $0 GRAPH-NAME [OPTIONS]\nOptions:\n" (cli/format-opts {:spec spec}))) (js/process.exit 1)) + {:keys [pages blocks batch-pages]} options [dir db-name] (if (string/includes? graph-dir "/") ((juxt node-path/dirname node-path/basename) graph-dir) [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]) conn (outliner-cli/init-conn dir db-name {:classpath (cp/get-classpath)}) - _ (println "Building tx ...") - {:keys [init-tx]} (outliner-cli/build-blocks-tx (create-init-data options))] - (println "Built" (count init-tx) "tx," (count (filter :block/title init-tx)) "pages and" - (count (filter :block/title init-tx)) "blocks ...") - ;; Vary the chunking with page size up to a max to avoid OOM - (let [tx-chunks (partition-all (min (:pages options) 30000) init-tx)] - (loop [chunks tx-chunks - chunk-num 1] - (when-let [chunk (first chunks)] - (println "Transacting chunk" chunk-num "of" (count tx-chunks) - "starting with block:" (pr-str (select-keys (first chunk) [:block/title :block/title]))) - (d/transact! conn chunk) - (recur (rest chunks) (inc chunk-num))))) - #_(d/transact! conn blocks-tx) - (println "Created graph" (str db-name " with " (count (d/datoms @conn :eavt)) " datoms!")))) + total-batches' (total-batches options) + pages-per-batch (or batch-pages (default-batch-pages blocks)) + total-blocks (* pages blocks)] + (println "Creating graph with" pages "pages and" total-blocks "blocks" + "using" total-batches' "batch(es) of up to" pages-per-batch "pages ...") + (doseq [[batch-num pages-and-blocks] (map-indexed vector (page-and-block-batches options))] + (println "Transacting batch" (inc batch-num) "of" total-batches' + "with" (count pages-and-blocks) "pages") + (transact-batch! conn pages-and-blocks)) + (println "Created graph" db-name "with" pages "pages and" total-blocks "blocks."))) (when (= nbb/*file* (nbb/invoked-file)) (-main *command-line-args*)) diff --git a/scripts/test/logseq/tasks/db_graph/create_graph_with_large_sizes_test.cljs b/scripts/test/logseq/tasks/db_graph/create_graph_with_large_sizes_test.cljs new file mode 100644 index 0000000000..bd897e07d1 --- /dev/null +++ b/scripts/test/logseq/tasks/db_graph/create_graph_with_large_sizes_test.cljs @@ -0,0 +1,69 @@ +(ns logseq.tasks.db-graph.create-graph-with-large-sizes-test + (:require [cljs.test :refer [deftest is testing]] + [logseq.tasks.db-graph.create-graph-with-large-sizes :as sut])) + +(deftest build-page-and-blocks-batch-builds-the-requested-graph-slice + (let [id-seq (map #(str "id-" %) (range)) + next-id (let [ids (atom id-seq)] + (fn [] + (let [id (first @ids)] + (swap! ids rest) + id))) + batch (#'sut/build-page-and-blocks-batch 10 2 3 next-id)] + (is (= 2 (count batch))) + (is (= ["Page-10" "Page-11"] + (map (comp :block/title :page) batch))) + (is (= ["id-0" "id-4"] + (map (comp :block/uuid :page) batch))) + (is (= [["Block" "Block" "Block"] + ["Block" "Block" "Block"]] + (map (fn [{:keys [blocks]}] + (mapv :block/title blocks)) + batch))) + (is (= [["id-1" "id-2" "id-3"] + ["id-5" "id-6" "id-7"]] + (map (fn [{:keys [blocks]}] + (mapv :block/uuid blocks)) + batch))))) + +(deftest page-and-block-batches-only-realize-requested-batches + (let [calls (atom 0) + next-id (fn [] + (swap! calls inc) + (str "id-" @calls)) + batches (#'sut/page-and-block-batches {:pages 50000 + :blocks 50 + :batch-pages 100} + next-id) + first-batch (first batches)] + (is (= 100 (count first-batch))) + (is (= (* 100 51) @calls) + "Only the first batch should be realized") + (is (= "Page-0" (get-in first-batch [0 :page :block/title]))) + (is (= "Page-99" (get-in first-batch [99 :page :block/title]))))) + +(deftest default-batching-keeps-large-graphs-bounded + (testing "50k pages with 50 blocks are split into many batches instead of one giant tx" + (let [batch-pages (#'sut/default-batch-pages 50)] + (is (< batch-pages 50000)) + (is (pos? batch-pages)) + (is (= batch-pages + (count (first (#'sut/page-and-block-batches {:pages 50000 + :blocks 50} + (constantly "id"))))))))) + +(deftest page-and-block-batches-handle-empty-input + (is (= [] + (into [] (#'sut/page-and-block-batches {:pages 0 + :blocks 50} + (constantly "id")))))) + +(deftest parse-args-keeps-the-graph-name-separate-from-cli-options + (let [{:keys [graph-dir options]} (sut/parse-args ["large-graph" + "-p" "3" + "-b" "2" + "-t" "1"])] + (is (= "large-graph" graph-dir)) + (is (= 3 (:pages options))) + (is (= 2 (:blocks options))) + (is (= 1 (:batch-pages options))))) diff --git a/scripts/test/logseq/tasks/test_runner.cljs b/scripts/test/logseq/tasks/test_runner.cljs new file mode 100644 index 0000000000..b4d65f4e24 --- /dev/null +++ b/scripts/test/logseq/tasks/test_runner.cljs @@ -0,0 +1,8 @@ +(ns logseq.tasks.test-runner + (:require [cljs.test :as test] + [logseq.tasks.db-graph.create-graph-with-large-sizes-test])) + +(defn -main [& _] + (let [{:keys [fail error]} (test/run-tests 'logseq.tasks.db-graph.create-graph-with-large-sizes-test)] + (when (pos? (+ fail error)) + (js/process.exit 1)))) diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index cb4fcf1995..a25b0f097e 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -202,11 +202,24 @@ (on-dimensions (.-naturalWidth img) (.-naturalHeight img)))) (set! (.-src img) url))) +(defn- normalize-asset-align + [asset-align] + (cond + (keyword? asset-align) asset-align + (string? asset-align) (case asset-align + "left" :left + "center" :center + "right" :right + nil) + :else nil)) + (defonce *resizing-image? (atom false)) + (rum/defc ^:large-vars/cleanup-todo asset-container [asset-block src title metadata {:keys [breadcrumb? positioned? local? full-text]}] (let [asset-width (:logseq.property.asset/width asset-block) - asset-height (:logseq.property.asset/height asset-block)] + asset-height (:logseq.property.asset/height asset-block) + asset-align (normalize-asset-align (:logseq.property.asset/align asset-block))] (hooks/use-effect! (fn [] (when (:block/uuid asset-block) @@ -275,7 +288,13 @@ :repo (state/get-current-repo) :href src :title title - :full-text full-text})))))))] + :full-text full-text}))))))) + handle-set-align! + (fn [align] + (when-let [asset-id (:block/uuid asset-block)] + (property-handler/set-block-property! asset-id + :logseq.property.asset/align + align)))] (when asset-block [:.asset-action-bar {:aria-hidden "true"} (shui/dropdown-menu @@ -288,6 +307,33 @@ :class "h-6 w-6"} (shui/tabler-icon "dots-vertical"))) (shui/dropdown-menu-content + (shui/dropdown-menu-sub + (shui/dropdown-menu-sub-trigger + [:span.flex.items-center.gap-1 + (ui/icon "layout-align-left") (t :asset/align)]) + (shui/dropdown-menu-sub-content + (shui/dropdown-menu-item + {:on-click #(handle-set-align! :left)} + [:span.flex.items-center.gap-2 + (ui/icon "layout-align-left") + (t :asset/align-left) + (when (or (nil? asset-align) (= asset-align :left)) + (ui/icon "check"))]) + (shui/dropdown-menu-item + {:on-click #(handle-set-align! :center)} + [:span.flex.items-center.gap-2 + (ui/icon "layout-align-center") + (t :asset/align-center) + (when (= asset-align :center) + (ui/icon "check"))]) + (shui/dropdown-menu-item + {:on-click #(handle-set-align! :right)} + [:span.flex.items-center.gap-2 + (ui/icon "layout-align-right") + (t :asset/align-right) + (when (= asset-align :right) + (ui/icon "check"))]))) + (shui/dropdown-menu-item {:on-click handle-copy!} [:span.flex.items-center.gap-1 @@ -301,6 +347,7 @@ (js/window.apis.openExternal image-src)))} [:span.flex.items-center.gap-1 (ui/icon "folder-pin") (t (if local? :asset/show-in-folder :asset/open-in-browser))])) + (when-not config/publishing? [:<> (shui/dropdown-menu-separator) @@ -318,6 +365,7 @@ (let [breadcrumb? (:breadcrumb? config) positioned? (:property-position config) asset-block (:asset-block config) + asset-align (normalize-asset-align (:logseq.property.asset/align asset-block)) width (:width metadata) *width (get state ::size) width (or @*width width) @@ -334,30 +382,42 @@ (:table-view? config) (not resizable?)) asset-container-cp - [:div.ls-resize-image.rounded-md - asset-container-cp - (resize-image-handles - (fn [k ^js event] - (let [dx (.-dx event) - ^js target (.-target event)] + [:div.ls-resize-inner.w-full.select-none + {:on-double-click (fn [^js e] + (let [^js target (.-target e) + ^js container (.closest target ".ls-resize-inner")] + (when (or container (= target container)) + (when-let [block-uuid (or (:block/uuid config) + (some-> config :block :block/uuid))] + (editor-handler/select-block! block-uuid)))))} + [:div.ls-resize-image.rounded-md + {:class (case asset-align + :center "align-center" + :right "align-right" + "align-left")} + asset-container-cp + (resize-image-handles + (fn [k ^js event] + (let [dx (.-dx event) + ^js target (.-target event)] - (case k - :start - (let [c (.closest target ".ls-resize-image")] - (reset! *width (.-offsetWidth c)) - (reset! *resizing-image? true)) - :move - (let [width' (+ @*width dx)] - (when (or (> width' 60) - (not (neg? dx))) - (reset! *width width'))) - :end - (let [width' @*width] - (when (and width' @*resizing-image?) - (when-let [block-id (or (:block/uuid config) - (some-> config :block (:block/uuid)))] - (editor-handler/resize-image! config block-id metadata full-text {:width width'}))) - (reset! *resizing-image? false))))))]))) + (case k + :start + (let [c (.closest target ".ls-resize-image")] + (reset! *width (.-offsetWidth c)) + (reset! *resizing-image? true)) + :move + (let [width' (+ @*width dx)] + (when (or (> width' 60) + (not (neg? dx))) + (reset! *width width'))) + :end + (let [width' @*width] + (when (and width' @*resizing-image?) + (when-let [block-id (or (:block/uuid config) + (some-> config :block (:block/uuid)))] + (editor-handler/resize-image! config block-id metadata full-text {:width width'}))) + (reset! *resizing-image? false))))))]]))) (rum/defc audio-cp ([src] (audio-cp src nil)) diff --git a/src/main/frontend/components/block.css b/src/main/frontend/components/block.css index d37edfb414..53e76622d0 100644 --- a/src/main/frontend/components/block.css +++ b/src/main/frontend/components/block.css @@ -1194,6 +1194,19 @@ html.is-mac { .ls-resize-image { @apply flex relative w-fit cursor-pointer; + &.align-left { + margin-right: auto; + } + + &.align-center { + margin-left: auto; + margin-right: auto; + } + + &.align-right { + margin-left: auto; + } + .handle-left, .handle-right { @apply absolute w-[6px] h-[15%] min-h-[30px] bg-black/30 hover:bg-black/70 top-[50%] left-[5px] rounded-full cursor-col-resize select-none diff --git a/src/main/frontend/components/imports.cljs b/src/main/frontend/components/imports.cljs index 54239bb7a7..55db6af356 100644 --- a/src/main/frontend/components/imports.cljs +++ b/src/main/frontend/components/imports.cljs @@ -182,6 +182,7 @@ [:div.border.p-6.rounded.bg-gray-01.mt-4 (let [form-ctx (form-core/use-form {:defaultValues {:graph-name initial-name + :extract-code-snippets? false :convert-all-tags? true :tag-classes "" :remove-inline-tags? true @@ -212,6 +213,15 @@ (shui/form-description [:b.text-red-800 (:message error)]))))) + (shui/form-field {:name "extract-code-snippets?"} + (fn [field] + (shui/form-item + {:class "pt-3 flex justify-start items-center space-x-3 space-y-0 my-3 pr-3"} + (shui/form-label "Extract inline code snippets as child blocks") + (shui/form-control + (shui/checkbox {:checked (:value field) + :on-checked-change (:onChange field)}))))) + (shui/form-field {:name "convert-all-tags?"} (fn [field] (shui/form-item diff --git a/src/main/frontend/undo_redo.cljs b/src/main/frontend/undo_redo.cljs index 49a8f983d1..0a487b1aa8 100644 --- a/src/main/frontend/undo_redo.cljs +++ b/src/main/frontend/undo_redo.cljs @@ -391,8 +391,10 @@ (log/error ::undo-redo-failed e) (clear-history! repo))))) (do - (clear-history! repo) - (if undo? ::empty-undo-stack ::empty-redo-stack)))))))) + (log/warn ::undo-redo-skip-conflicted-op + {:undo? undo? + :outliner-op (:outliner-op tx-meta)}) + (undo-redo-aux repo undo?)))))))) (when ((if undo? empty-undo-stack? empty-redo-stack?) repo) (prn (str "No further " (if undo? "undo" "redo") " information")) diff --git a/src/main/frontend/worker/db/migrate.cljs b/src/main/frontend/worker/db/migrate.cljs index 8240a60f78..83a0233c9f 100644 --- a/src/main/frontend/worker/db/migrate.cljs +++ b/src/main/frontend/worker/db/migrate.cljs @@ -75,7 +75,8 @@ ["65.20" {:properties [:logseq.property.class/bidirectional-property-title :logseq.property.class/enable-bidirectional?]}] ["65.21" {:properties [:logseq.property.sync/large-title-object]}] ["65.22" {:properties [:logseq.property.reaction/emoji-id - :logseq.property.reaction/target]}]]) + :logseq.property.reaction/target]}] + ["65.23" {:properties [:logseq.property.asset/align]}]]) (let [[major minor] (last (sort (map (comp (juxt :major :minor) db-schema/parse-schema-version first) schema-version->updates)))] diff --git a/src/main/frontend/worker/sync.cljs b/src/main/frontend/worker/sync.cljs index b1209c5269..15d0e8da19 100644 --- a/src/main/frontend/worker/sync.cljs +++ b/src/main/frontend/worker/sync.cljs @@ -648,8 +648,12 @@ [x] (and (integer? x) (neg? x))) +(defn- remote-batch-temp-id + [temp-id] + (str "remote-batch-tempid-" temp-id)) + (defn- remap-remote-batch-temp-ids - [batch-index tx-data] + [tx-data] (let [ops #{:db/add :db/retract :db/retractEntity} entity-temp-ids (->> tx-data (keep (fn [item] @@ -661,9 +665,7 @@ distinct) temp-id-map (when (seq entity-temp-ids) (zipmap entity-temp-ids - (map-indexed (fn [idx _] - (str "remote-batch-" batch-index "-tempid-" idx)) - entity-temp-ids)))] + (map remote-batch-temp-id entity-temp-ids)))] (if (seq temp-id-map) (mapv (fn [item] (if (and (vector? item) @@ -731,11 +733,11 @@ (defn- flatten-batched-remote-tx-data [tx-data*] - (loop [remaining (map-indexed vector tx-data*) + (loop [remaining tx-data* lookup->temp-id {} acc []] - (if-let [[batch-index tx-data] (first remaining)] - (let [remapped-batch (remap-remote-batch-temp-ids batch-index tx-data) + (if-let [tx-data (first remaining)] + (let [remapped-batch (remap-remote-batch-temp-ids tx-data) lookup->temp-id (merge lookup->temp-id (created-lookup->temp-id remapped-batch)) resolved-batch (resolve-lookup-refs lookup->temp-id remapped-batch)] (recur (rest remaining) diff --git a/src/resources/dicts/en.edn b/src/resources/dicts/en.edn index 9656eb4462..07f433a4c0 100644 --- a/src/resources/dicts/en.edn +++ b/src/resources/dicts/en.edn @@ -135,6 +135,10 @@ :asset/ref-block "Asset ref block" :asset/confirm-delete "Are you sure you want to delete this {1}?" :asset/physical-delete "Remove the file too (notice it can't be restored)" + :asset/align "Align" + :asset/align-left "Align left" + :asset/align-center "Align center" + :asset/align-right "Align right" :color/gray "Gray" :color/red "Red" :color/yellow "Yellow" diff --git a/src/test/frontend/undo_redo_test.cljs b/src/test/frontend/undo_redo_test.cljs index ae3408f1e5..cf6f4b8edb 100644 --- a/src/test/frontend/undo_redo_test.cljs +++ b/src/test/frontend/undo_redo_test.cljs @@ -348,6 +348,33 @@ (is (= child-uuid (:block/uuid (:block/parent parent)))) (is (= page-uuid (:block/uuid (:block/parent child)))))))) +(deftest undo-skips-conflicted-move-and-keeps-earlier-history-test + (testing "undo drops a conflicting move op but still undoes earlier safe ops" + (undo-redo/clear-history! test-db) + (let [conn (db/get-db test-db false) + {:keys [parent-a-uuid parent-b-uuid child-uuid]} (seed-page-two-parents-child!)] + (d/transact! conn + [[:db/add [:block/uuid child-uuid] :block/title "local-title"]] + {:outliner-op :save-block + :local-tx? true}) + (d/transact! conn + [[:db/add [:block/uuid child-uuid] :block/parent [:block/uuid parent-b-uuid]]] + {:outliner-op :move-blocks + :local-tx? true}) + (d/transact! conn + [[:db/retractEntity [:block/uuid parent-a-uuid]]] + {:outliner-op :delete-blocks + :local-tx? false}) + (let [undo-result (undo-redo/undo test-db) + child (d/entity @conn [:block/uuid child-uuid])] + (is (not= :frontend.undo-redo/empty-undo-stack undo-result)) + (is (= "child" (:block/title child))) + (is (= parent-b-uuid + (:block/uuid (:block/parent child)))) + (is (empty? (db-issues @conn)))) + (is (= :frontend.undo-redo/empty-undo-stack + (undo-redo/undo test-db)))))) + (deftest undo-validation-fast-path-skips-db-issues-for-non-structural-tx-test (testing "undo validation skips db-issues for non-structural tx-data" (let [conn (db/get-db test-db false) diff --git a/src/test/frontend/worker/db_sync_sim_test.cljs b/src/test/frontend/worker/db_sync_sim_test.cljs index 8838364da0..16afcf1a08 100644 --- a/src/test/frontend/worker/db_sync_sim_test.cljs +++ b/src/test/frontend/worker/db_sync_sim_test.cljs @@ -1266,6 +1266,78 @@ (sync-loop! server [{:repo repo-a :conn conn-a :client client-a :online? true}]) (is (= "test" (:block/title (d/entity @conn-a [:block/uuid block-uuid]))))))))) +(deftest ^:long two-clients-undo-skips-conflicted-move-but-keeps-db-valid-test + (testing "undo skips a conflicted move while syncing the remaining safe history" + (let [base-uuid (uuid "31111111-1111-1111-1111-111111111111") + parent-a-uuid (uuid "32222222-2222-2222-2222-222222222222") + parent-b-uuid (uuid "33333333-3333-3333-3333-333333333333") + child-uuid (uuid "34444444-4444-4444-4444-444444444444") + conn-a (db-test/create-conn) + conn-b (db-test/create-conn) + ops-a (d/create-conn client-op/schema-in-db) + ops-b (d/create-conn client-op/schema-in-db) + client-a (make-client repo-a) + client-b (make-client repo-b) + server (make-server) + seed 20260311 + history (atom [])] + (with-test-repos {repo-a {:conn conn-a :ops-conn ops-a} + repo-b {:conn conn-b :ops-conn ops-b}} + (fn [] + (let [{:keys [repro restore]} (install-invalid-tx-repro! seed history)] + (try + (reset! db-sync/*repo->latest-remote-tx {}) + (client-op/update-local-tx repo-a 0) + (client-op/update-local-tx repo-b 0) + (ensure-base-page! conn-a base-uuid) + (let [base-a (d/entity @conn-a [:block/uuid base-uuid])] + (create-block! conn-a base-a "parent-a" parent-a-uuid) + (create-block! conn-a base-a "parent-b" parent-b-uuid) + (let [parent-a (d/entity @conn-a [:block/uuid parent-a-uuid])] + (create-block! conn-a parent-a "seed-child" child-uuid))) + (sync-until-idle! server [{:repo repo-a :conn conn-a :client client-a :online? true} + {:repo repo-b :conn conn-b :client client-b :online? true}] + 20) + + (update-title! conn-a child-uuid "local-title") + (move-block! conn-a + {:block/uuid child-uuid} + {:block/uuid parent-b-uuid}) + (sync-until-idle! server [{:repo repo-a :conn conn-a :client client-a :online? true} + {:repo repo-b :conn conn-b :client client-b :online? true}] + 50) + + (delete-block! conn-b parent-a-uuid) + (sync-until-idle! server [{:repo repo-a :conn conn-a :client client-a :online? true} + {:repo repo-b :conn conn-b :client client-b :online? true}] + 50) + + (is (not= :frontend.undo-redo/empty-undo-stack + (undo-redo/undo repo-a))) + + (let [rounds (sync-until-idle! server [{:repo repo-a :conn conn-a :client client-a :online? true} + {:repo repo-b :conn conn-b :client client-b :online? true}] + 50) + child-a (d/entity @conn-a [:block/uuid child-uuid]) + child-b (d/entity @conn-b [:block/uuid child-uuid]) + attrs-a (block-attr-map @conn-a) + attrs-b (block-attr-map @conn-b) + issues-a (db-issues @conn-a) + issues-b (db-issues @conn-b)] + (is (< rounds 50) (str "sync did not become idle rounds=" rounds)) + (is (= "seed-child" (:block/title child-a))) + (is (= "seed-child" (:block/title child-b))) + (is (= parent-b-uuid + (:block/uuid (:block/parent child-a)))) + (is (= parent-b-uuid + (:block/uuid (:block/parent child-b)))) + (is (empty? issues-a) (str "db A issues " (pr-str issues-a))) + (is (empty? issues-b) (str "db B issues " (pr-str issues-b))) + (assert-synced-attrs! seed history attrs-a attrs-b attrs-b) + (assert-no-invalid-tx! seed history repro)) + (finally + (restore))))))))) + (defonce op-runs 200) (defn- run-random-ops! diff --git a/src/test/frontend/worker/db_sync_test.cljs b/src/test/frontend/worker/db_sync_test.cljs index b91ab834b3..075fc02391 100644 --- a/src/test/frontend/worker/db_sync_test.cljs +++ b/src/test/frontend/worker/db_sync_test.cljs @@ -740,6 +740,37 @@ vec)] (is (empty? sanitized))))) +(deftest apply-remote-batched-create-reuses-tempid-across-batches-test + (testing "a remote block create split across batches should still resolve to one valid block" + (let [{:keys [conn parent]} (setup-parent-child) + parent-uuid (:block/uuid parent) + page-uuid (:block/uuid (:block/page parent)) + remote-uuid (random-uuid) + batched-tx-data [[[:db/add -1 :block/uuid remote-uuid] + [:db/add -1 :block/title "remote batched child"] + [:db/add -1 :block/page [:block/uuid page-uuid]] + [:db/add -1 :block/created-at 1760000000000] + [:db/add -1 :block/updated-at 1760000000000]] + [[:db/add -1 :block/parent [:block/uuid parent-uuid]] + [:db/add -1 :block/order "a4"]]]] + (with-datascript-conns conn nil + (fn [] + (let [error (try + (#'db-sync/apply-remote-tx! test-repo nil batched-tx-data) + nil + (catch :default e + e))] + (is (nil? error) + (when error + (str (ex-message error) " " (pr-str (ex-data error))))) + (when-not error + (let [block (d/entity @conn [:block/uuid remote-uuid]) + validation (db-validate/validate-local-db! @conn)] + (is (= "remote batched child" (:block/title block))) + (is (= (:db/id parent) (:db/id (:block/parent block)))) + (is (empty? (map :entity (:errors validation))) + (str (:errors validation))))))))))) + (deftest ^:long sanitize-tx-data-drops-numeric-entity-datoms-for-deleted-block-test (testing "deleted-block-ids should also drop datoms when entity is numeric id" (let [{:keys [conn child1]} (setup-parent-child)