clone/logseq
1
0
Fork 0
mirror of https://github.com/logseq/logseq.git synced 2026-05-17 17:32:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into refactor/tech-stack-upgrade

Browse Source
This commit is contained in:
Mega Yu
2026-03-12 17:10:35 +08:00
parent f51eb43561 1b0db68098
commit 8a50be33a9
31 changed files with 1326 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.clj-kondo/config.edn
View File

@@ -228,8 +228,10 @@
clojure.test.check.properties/for-all clojure.core/for
;; src/main
frontend.namespaces/import-vars potemkin/import-vars
;; src/test
;; src/test and deps tests
frontend.test.helper/deftest-async clojure.test/deftest
logseq.graph-parser.test.helper/deftest-async clojure.test/deftest
logseq.publishing.test.helper/deftest-async clojure.test/deftest
frontend.worker.rtc.idb-keyval-mock/with-reset-idb-keyval-mock cljs.test/async
frontend.react/defc clojure.core/defn
logseq.common.defkeywords/defkeyword cljs.spec.alpha/def

2
deps.edn
View File

@@ -5,7 +5,7 @@
:sha "5d672bf84ed944414b9f61eeb83808ead7be9127"}
datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
:sha "ff5a7d5326e2546f40146e4a489343f557519bc3"}
:sha "f91fec561ee2c11d6bf323feae365e9033585411"}
;; datascript/datascript {:local/root "../../datascript"}
datascript-transit/datascript-transit {:mvn/version "0.3.0"}

2
deps/db-sync/deps.edn vendored
View File

@@ -2,7 +2,7 @@
:deps
{org.clojure/clojure {:mvn/version "1.11.1"}
datascript/datascript {:git/url "https://github.com/logseq/datascript"
:sha "ff5a7d5326e2546f40146e4a489343f557519bc3"}
:sha "f91fec561ee2c11d6bf323feae365e9033585411"}
datascript-transit/datascript-transit {:mvn/version "0.3.0"
:exclusions [datascript/datascript]}
com.cognitect/transit-cljs {:mvn/version "0.8.280"}

2
deps/db-sync/src/logseq/db_sync/batch.cljs vendored
View File

@@ -1,7 +1,7 @@
(ns logseq.db-sync.batch
(:require [clojure.string :as string]))
(def ^:private max-sql-params 99)
(def max-sql-params 99)
(def ^:private row-param-count 3)
(defn rows->insert-batches

11
deps/db-sync/src/logseq/db_sync/storage.cljs vendored
View File

@@ -1,6 +1,5 @@
(ns logseq.db-sync.storage
(:require [cljs-bean.core :as bean]
[clojure.string :as string]
[datascript.core :as d]
[datascript.storage :refer [IStorage]]
[logseq.db-sync.common :as common]
@@ -70,13 +69,6 @@
:tx (aget row "tx")})
rows)))
(defn- delete-addrs! [sql addrs]
(when (seq addrs)
(let [placeholders (->> addrs (map (constantly "?")) (string/join ","))]
(apply common/sql-exec sql
(str "delete from kvs where addr in (" placeholders ")")
addrs))))
(defn- upsert-addr-content! [sql data]
(doseq [item data]
(common/sql-exec sql
@@ -97,7 +89,7 @@
(defn new-sqlite-storage [sql]
(reify IStorage
(-store [_ addr+data-seq delete-addrs]
(-store [_ addr+data-seq _delete-addrs]
(let [data (map
(fn [[addr data]]
(let [data' (if (map? data) (dissoc data :addresses) data)
@@ -108,7 +100,6 @@
"content" (common/write-transit data')
"addresses" addresses}))
addr+data-seq)]
(delete-addrs! sql delete-addrs)
(upsert-addr-content! sql data)))
(-restore [_ addr]
(restore-data-from-addr sql addr))))

32
deps/db-sync/src/logseq/db_sync/worker/handler/sync.cljs vendored
View File

@@ -1,5 +1,6 @@
(ns logseq.db-sync.worker.handler.sync
(:require [clojure.string :as string]
[datascript.core :as d]
[lambdaisland.glogi :as log]
[logseq.db :as ldb]
[logseq.db-sync.batch :as batch]
@@ -10,6 +11,7 @@
[logseq.db-sync.worker.http :as http]
[logseq.db-sync.worker.routes.sync :as sync-routes]
[logseq.db-sync.worker.ws :as ws]
[logseq.db.frontend.schema :as db-schema]
[promesa.core :as p]))
(def ^:private snapshot-download-batch-size 5000)
@@ -264,7 +266,35 @@
(let [sql (.-sql self)]
(ensure-conn! self)
(let [conn (.-conn self)
tx-data (protocol/transit->tx txs)]
lookup-id (fn [x]
(when (and (vector? x)
(= 2 (count x))
(= :block/uuid (first x)))
(second x)))
tx-data* (protocol/transit->tx txs)
created-block-uuids (->> tx-data*
(keep (fn [item]
(when (and (vector? item)
(= :db/add (first item))
(>= (count item) 4)
(= :block/uuid (nth item 2)))
(nth item 3))))
set)
missing-lookup-ref? (fn [x]
(when-let [block-uuid (lookup-id x)]
(and (not (contains? created-block-uuids block-uuid))
(nil? (d/entity @conn x)))))
tx-data (remove (fn [item]
(when (vector? item)
(let [op (first item)
attr (nth item 2 nil)
value (when (>= (count item) 4) (nth item 3))]
(or (and (contains? #{:db/add :db/retract :db/retractEntity} op)
(missing-lookup-ref? (second item)))
(and (contains? #{:db/add :db/retract} op)
(contains? db-schema/ref-type-attributes attr)
(missing-lookup-ref? value))))))
tx-data*)]
(ldb/transact! conn tx-data {:op :apply-client-tx})
(let [new-t (storage/get-t sql)]
;; FIXME: no need to broadcast if client tx is less than remote tx

26
deps/db-sync/test/logseq/db_sync/worker_handler_sync_test.cljs vendored
View File

@@ -1,8 +1,13 @@
(ns logseq.db-sync.worker-handler-sync-test
(:require [cljs.test :refer [async deftest is]]
(:require [cljs.test :refer [async deftest is testing]]
[datascript.core :as d]
[logseq.db-sync.common :as common]
[logseq.db-sync.protocol :as protocol]
[logseq.db-sync.storage :as storage]
[logseq.db-sync.test-sql :as test-sql]
[logseq.db-sync.worker.handler.sync :as sync-handler]
[logseq.db-sync.worker.ws :as ws]
[logseq.db.frontend.schema :as db-schema]
[promesa.core :as p]))
(defn- empty-sql []
@@ -127,3 +132,22 @@
(p/catch (fn [error]
(is false (str error))
(done)))))))
(deftest tx-batch-drops-stale-lookup-entity-updates-test
(testing "stale lookup-ref entity updates should not reject the whole tx batch"
(let [sql (test-sql/make-sql)
conn (d/create-conn db-schema/schema)
self #js {:sql sql
:conn conn
:schema-ready true}
missing-uuid (random-uuid)
created-uuid (random-uuid)
tx-data [[:db/add [:block/uuid missing-uuid] :block/title "stale" 1]
[:db/add [:block/uuid missing-uuid] :block/updated-at 1773188050934 1]
[:db/add "temp-1" :block/uuid created-uuid 2]
[:db/add "temp-1" :block/title "ok" 2]]
response (with-redefs [ws/broadcast! (fn [& _] nil)]
(sync-handler/handle-tx-batch! self nil (protocol/tx->transit tx-data) 0))]
(is (= "tx/batch/ok" (:type response)))
(is (= "ok" (:block/title (d/entity @conn [:block/uuid created-uuid]))))
(is (nil? (d/entity @conn [:block/uuid missing-uuid]))))))

2
deps/db/deps.edn vendored
View File

@@ -1,7 +1,7 @@
{:deps
;; These nbb-logseq deps are kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn
{datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
:sha "ff5a7d5326e2546f40146e4a489343f557519bc3"}
:sha "f91fec561ee2c11d6bf323feae365e9033585411"}
;; datascript/datascript {:local/root "../../../../datascript"}
datascript-transit/datascript-transit {:mvn/version "0.3.0"
:exclusions [datascript/datascript]}

3
deps/db/src/logseq/db/frontend/malli_schema.cljs vendored
View File

@@ -510,7 +510,8 @@
[:logseq.property.asset/checksum :string]
[:logseq.property.asset/size :int]
[:logseq.property.asset/width {:optional true} :int]
[:logseq.property.asset/height {:optional true} :int]]
[:logseq.property.asset/height {:optional true} :int]
[:logseq.property.asset/align {:optional true} :keyword]]
block-attrs
page-or-block-attrs)))

5
deps/db/src/logseq/db/frontend/property.cljs vendored
View File

@@ -567,6 +567,11 @@
:schema {:type :map
:hide? true
:public? false}}
:logseq.property.asset/align {:title "Asset alignment"
:schema {:type :keyword
:hide? true
:public? false}
:queryable? false}
:logseq.property.fsrs/due {:title "Due"
:schema
{:type :datetime

2
deps/db/src/logseq/db/frontend/schema.cljs vendored
View File

@@ -30,7 +30,7 @@
(map (juxt :major :minor)
[(parse-schema-version x) (parse-schema-version y)])))
(def version (parse-schema-version "65.22"))
(def version (parse-schema-version "65.23"))
(defn major-version
"Return a number.

3
deps/graph-parser/script/db_import.cljs vendored
View File

@@ -193,6 +193,9 @@
{:alias :P
:coerce []
:desc "List of properties whose values convert to a parent class"}
:extract-code-snippets?
{:alias :C
:desc "Extract code fence(s) to #Code"}
:validate
{:alias :V
:desc "Validate db after creation"}})

351
deps/graph-parser/src/logseq/graph_parser/exporter.cljs vendored
View File

@@ -57,6 +57,116 @@
:block/title new-title
:block/name (common-util/page-name-sanity-lc new-title)})))
(def template-file-property-names #{:template :template-including-parent})
(defn- get-template-name
[block]
(let [template-name (get-in block [:block/properties :template])]
(when (string? template-name)
(not-empty (string/trim template-name)))))
(defn- template-including-parent?
[block]
(not= false (get-in block [:block/properties :template-including-parent])))
(defn- remove-template-property-lines
[title]
(if (string? title)
(->> (string/split-lines title)
(remove (fn [line]
(let [trimmed-line (string/triml line)]
(or (string/starts-with? trimmed-line "template::")
(string/starts-with? trimmed-line "template-including-parent::")))))
(string/join "\n"))
title))
(defn- group-block-children-by-parent
[blocks]
(reduce (fn [result {parent :block/parent
child-uuid :block/uuid}]
(if (and (vector? parent)
(= :block/uuid (first parent)))
(update result (second parent) (fnil conj []) child-uuid)
result))
{}
blocks))
(defn- get-block-subtree-uuids
[block-children root-uuid]
(loop [queue [root-uuid]
result []]
(if-let [current-uuid (first queue)]
(recur (into (vec (rest queue)) (get block-children current-uuid))
(conj result current-uuid))
result)))
(defn- get-parent-uuid [parent]
(cond
(and (vector? parent) (= :block/uuid (first parent)))
(second parent)
(and (map? parent) (:block/uuid parent))
(:block/uuid parent)
:else
nil))
(defn- handle-template-blocks
"Handles creating #Template blocks and their children and calculates
:preserve-empty-properties-uuids for use later"
[blocks*]
(let [include-parent-template-uuids (->> blocks*
(filter (fn [block]
(and (get-template-name block)
(template-including-parent? block))))
(map :block/uuid)
set)
content-uuids-by-template (into {}
(map (fn [template-uuid]
[template-uuid (common-uuid/gen-uuid)]))
include-parent-template-uuids)]
(reduce
(fn [{:keys [blocks preserve-empty-properties-uuids]} block]
(if-let [template-name (get-template-name block)]
(let [block-children (group-block-children-by-parent blocks*)
parent-uuid (get-parent-uuid (:block/parent block))
content-uuid (when parent-uuid
(get content-uuids-by-template parent-uuid))
cleaned-block' (if content-uuid
(assoc block :block/parent [:block/uuid content-uuid])
block)
source-preserve-empty-properties-uuids (set (get-block-subtree-uuids block-children (:block/uuid block)))
template-root-block (-> cleaned-block'
(assoc :block/title template-name)
(update :block/tags (fnil conj []) :logseq.class/Template)
(dissoc :block/properties))
template-content-block (when (template-including-parent? block)
(-> (cond-> block
(seq (:block/properties block))
(update :block/properties #(apply dissoc % template-file-property-names)))
(update :block/title remove-template-property-lines)
(assoc :block/uuid (get content-uuids-by-template (:block/uuid block))
:block/parent [:block/uuid (:block/uuid block)]
:block/order (db-order/gen-key))
(dissoc :db/id)))]
{:blocks (cond-> blocks
true
(conj template-root-block)
template-content-block
(conj template-content-block))
:preserve-empty-properties-uuids (set/union preserve-empty-properties-uuids
source-preserve-empty-properties-uuids
(cond-> #{}
template-content-block
(conj (:block/uuid template-content-block))))})
{:blocks (if-let [content-uuid (some->> (get-parent-uuid (:block/parent block))
(get content-uuids-by-template))]
(conj blocks
(assoc block :block/parent [:block/uuid content-uuid]))
(conj blocks block))
:preserve-empty-properties-uuids preserve-empty-properties-uuids}))
{:blocks []
:preserve-empty-properties-uuids #{}}
blocks*)))
(defn- get-page-uuid [page-names-to-uuids page-name ex-data']
(or (get @page-names-to-uuids (some-> (if (string/includes? (str page-name) "#")
(string/lower-case (gp-block/sanitize-hashtag-name page-name))
@@ -463,7 +573,8 @@
"All built-in property file ids as a set of keywords"
(-> built-in-property-file-to-db-idents keys set
;; built-in-properties that map to new properties
(set/union #{:filters :query-table :query-properties :query-sort-by :query-sort-desc :hl-stamp :file :file-path})))
(set/union #{:filters :query-table :query-properties :query-sort-by :query-sort-desc :hl-stamp :file :file-path})
(set/union template-file-property-names)))
;; TODO: Review whether this should be using :block/title instead of file graph ids
(def all-built-in-names
@@ -481,7 +592,8 @@
#{:alias :tags :background-color :heading
:query-table :query-properties :query-sort-by :query-sort-desc
:ls-type :hl-type :hl-color :hl-page :hl-stamp :hl-value :file :file-path
:logseq.order-list-type :icon :public :exclude-from-graph-view :filters})
:logseq.order-list-type :icon :public :exclude-from-graph-view :filters
:template :template-including-parent})
(assert (set/subset? file-built-in-property-names all-built-in-property-file-ids)
"All file-built-in properties are used in db graph")
@@ -693,21 +805,18 @@
(get-page-uuid page-names-to-uuids ((some-fn ::original-name :block/name) block) {:block block})
(:block/uuid block))
properties-text-values))
;; TODO: Add import support for :template. Ignore for now as they cause invalid property types
(if (contains? props :template)
{}
(let [props' (-> (update-built-in-property-values
(select-keys props file-built-in-property-names)
page-names-to-uuids
(select-keys import-state [:ignored-properties :all-idents])
(select-keys block [:block/name :block/title])
(select-keys user-options [:property-classes]))
(merge (update-user-property-values user-properties page-names-to-uuids properties-text-values import-state options)))
pvalue-tx-m (->property-value-tx-m block props' #(get-property-schema @property-schemas %) @all-idents)
block-properties (-> (merge props' (db-property-build/build-properties-with-ref-values pvalue-tx-m))
(update-keys get-ident'))]
{:block-properties block-properties
:pvalues-tx (into (mapcat #(if (set? %) % [%]) (vals pvalue-tx-m)))}))))
(let [props' (-> (update-built-in-property-values
(select-keys props file-built-in-property-names)
page-names-to-uuids
(select-keys import-state [:ignored-properties :all-idents])
(select-keys block [:block/name :block/title])
(select-keys user-options [:property-classes]))
(merge (update-user-property-values user-properties page-names-to-uuids properties-text-values import-state options)))
pvalue-tx-m (->property-value-tx-m block props' #(get-property-schema @property-schemas %) @all-idents)
block-properties (-> (merge props' (db-property-build/build-properties-with-ref-values pvalue-tx-m))
(update-keys get-ident'))]
{:block-properties block-properties
:pvalues-tx (into (mapcat #(if (set? %) % [%]) (vals pvalue-tx-m)))})))
(def ignored-built-in-properties
"Ignore built-in properties that are already imported or not supported in db graphs"
@@ -726,7 +835,7 @@
(defn- pre-update-properties
"Updates page and block properties before their property types are inferred"
[properties class-related-properties]
[properties class-related-properties {:keys [preserve-empty-properties?]}]
(let [dissoced-props (concat ignored-built-in-properties
;; TODO: Deal with these dissoced built-in properties
[:title :created-at :updated-at]
@@ -736,8 +845,9 @@
(if (not (contains? file-built-in-property-names prop))
;; only update user properties
(if (string? val)
;; Ignore blank values as they were usually generated by templates
(when-not (string/blank? val)
;; Ignore blank values outside template-related blocks to preserve existing import behavior
(when (or preserve-empty-properties?
(not (string/blank? val)))
[prop
;; handle float strings b/c graph-parser doesn't
(or (parse-double val) val)])
@@ -757,14 +867,15 @@
:keys [import-state macros]
:as options}]
(-> (if (seq properties)
(let [classes-from-properties (->> (select-keys properties property-classes)
(let [preserve-empty-properties? (contains? (or (:preserve-empty-property-block-uuids options) #{})
(:block/uuid block))
classes-from-properties (->> (select-keys properties property-classes)
(mapcat (fn [[_k v]] (if (coll? v) v [v])))
distinct)
properties' (pre-update-properties properties (into property-classes property-parent-classes))
properties-to-infer (if (:template properties')
;; Ignore template properties as they don't consistently have representative property values
{}
(apply dissoc properties' file-built-in-property-names))
properties' (pre-update-properties properties
(into property-classes property-parent-classes)
{:preserve-empty-properties? preserve-empty-properties?})
properties-to-infer (apply dissoc properties' file-built-in-property-names)
property-changes
(->> properties-to-infer
(keep (fn [[prop val]]
@@ -944,6 +1055,7 @@
use in build-block-tx. This walk is only done once for perf reasons"
[config ast-blocks]
(let [results (atom {:simple-queries []
:cards []
:asset-links []
:embeds []
:zotero-imported-files {}
@@ -951,9 +1063,9 @@
(walk/prewalk
(fn [x]
(cond
(and (vector? x)
(= "Link" (first x))
(let [path-or-map (second (:url (second x)))]
(and (vector? x)
(= "Link" (first x))
(let [path-or-map (second (:url (second x)))]
(cond
(string? path-or-map)
(or (common-config/local-relative-asset? path-or-map)
@@ -967,19 +1079,23 @@
(= "Macro" (first x))
(= "embed" (:name (second x))))
(swap! results update :embeds conj x)
(and (vector? x)
(= "Macro" (first x))
(= "zotero-imported-file" (:name (second x))))
(let [[item-key filename] (:arguments (second x))]
(when (and item-key filename)
(swap! results update :zotero-imported-files assoc item-key (common-util/safe-read-string filename))))
(and (vector? x)
(= "Macro" (first x))
(= "zotero-linked-file" (:name (second x))))
(let [[relative-path] (:arguments (second x))
parsed-path (common-util/safe-read-string relative-path)]
(when (string? parsed-path)
(swap! results update :zotero-linked-files conj parsed-path)))
(and (vector? x)
(= "Macro" (first x))
(= "zotero-imported-file" (:name (second x))))
(let [[item-key filename] (:arguments (second x))]
(when (and item-key filename)
(swap! results update :zotero-imported-files assoc item-key (common-util/safe-read-string filename))))
(and (vector? x)
(= "Macro" (first x))
(= "zotero-linked-file" (:name (second x))))
(let [[relative-path] (:arguments (second x))
parsed-path (common-util/safe-read-string relative-path)]
(when (string? parsed-path)
(swap! results update :zotero-linked-files conj parsed-path)))
(and (vector? x)
(= "Macro" (first x))
(= "cards" (:name (second x))))
(swap! results update :cards conj x)
(and (vector? x)
(= "Macro" (first x))
(= "query" (:name (second x))))
@@ -989,7 +1105,8 @@
@results))
(defn- handle-queries
"If a block contains a simple or advanced queries, converts block to a #Query node"
"If a block contains a simple or advanced queries, converts block to a #Query node. If a block
contains a cards query converts to a #Cards node"
[{:block/keys [title] :as block} db page-names-to-uuids walked-ast-blocks options]
(if-let [query (some-> (first (:simple-queries walked-ast-blocks))
(ast->text (select-keys options [:log-fn]))
@@ -1032,7 +1149,21 @@
(assoc :block/collapsed? true)))]
{:block block'
:pvalues-tx pvalues-tx'})
{:block block})))
(if-let [cards-macro (first (:cards walked-ast-blocks))]
(if-let [query (some-> cards-macro second :arguments first string/trim not-empty)]
(let [props {:logseq.property/query query}
{:keys [block-properties pvalues-tx]}
(build-properties-and-values props db page-names-to-uuids
(select-keys block [:block/properties-text-values :block/name :block/title :block/uuid])
options)
block'
(-> (update block :block/tags (fnil conj []) :logseq.class/Cards)
(merge block-properties
{:block/title (string/trim (string/replace-first title #"\{\{cards(.*)\}\}" ""))}))]
{:block block'
:pvalues-tx pvalues-tx})
{:block block})
{:block block}))))
(defn- handle-block-properties
"Does everything page properties does and updates a couple of block specific attributes"
@@ -1390,6 +1521,77 @@
:block/tags [:logseq.class/Quote-block]})
block))
(defn- handle-math
"If a block's entire content is a single displayed math formula, convert to #Math node.
Detects blocks whose title is entirely delimited by $$ markers."
[block]
(let [title (string/trim (:block/title block))]
(if (and (string/starts-with? title "$$")
(string/ends-with? title "$$")
(> (count title) 4)
;; ensure there's no nested $$ pair (i.e. not two separate inline formulas)
(not (string/includes? (subs title 2 (- (count title) 2)) "$$")))
(let [math-content (string/trim (subs title 2 (- (count title) 2)))]
(merge block
{:block/title math-content
:logseq.property.node/display-type :math
:block/tags [:logseq.class/Math-block]}))
block)))
(defn- split-title-by-code-fences
"Parses a block title string line-by-line, splitting into non-code text parts
and code fence segments. All code fences are extracted regardless of whether
they have a language tag; :lang is nil when not specified.
Returns {:text-parts [...] :code-segs [{:text ... :lang ...}]}."
[title]
(let [lines (string/split-lines title)]
(loop [remaining lines
in-code? false
lang nil
current []
text-parts []
code-segs []]
(if (empty? remaining)
{:text-parts (if (seq current)
(conj text-parts (string/join "\n" current))
text-parts)
:code-segs code-segs}
(let [line (first remaining)
trimmed-line (string/trim line)
fence-start? (and (not in-code?) (re-matches #"```.*" trimmed-line))
fence-end? (and in-code? (= trimmed-line "```"))]
(cond
fence-start?
(recur (rest remaining) true (not-empty (subs trimmed-line 3)) []
(if (seq current)
(conj text-parts (string/join "\n" current))
text-parts)
code-segs)
fence-end?
(recur (rest remaining) false nil []
text-parts
(conj code-segs {:text (string/join "\n" current) :lang lang}))
:else
(recur (rest remaining) in-code? lang (conj current line)
text-parts code-segs)))))))
(defn- build-code-snippet-child-blocks
"Builds child block tx maps for extracted code snippets, tagging each as a
Code-block with its detected language."
[parent-block code-segs]
(mapv (fn [{:keys [text lang]}]
(cond-> (sqlite-util/block-with-timestamps
{:block/uuid (d/squuid)
:block/title text
:block/parent [:block/uuid (:block/uuid parent-block)]
:block/page (:block/page parent-block)
:block/order (db-order/gen-key)
:block/tags [:logseq.class/Code-block]
:logseq.property.node/display-type :code})
lang
(assoc :logseq.property.code/lang lang)))
code-segs))
(defn- handle-embeds
"If a block contains page or block embeds, converts block to a :block/link based embed"
[block page-names-to-uuids {:keys [embeds]} {:keys [log-fn] :or {log-fn prn}}]
@@ -1414,6 +1616,55 @@
block))
block))
(defn- at-least-two?
[s substr]
(if (empty? substr)
false
(loop [start 0 cnt 0]
(let [idx (string/index-of s substr start)]
(cond
(>= cnt 2) true
(nil? idx) false
:else (recur (+ idx (count substr)) (inc cnt)))))))
(defn- handle-code-blocks
"Returns a vector of block and optional block children tx. If a block
contains code fence(s) i.e. ```, converts block to a #Code node. If user
enables :extract-code-snippets? option, multiple code fences are extracted out
of text and put into children blocks in the order they appear"
[block' options]
(let [title (:block/title block')
has-fence? (and (string? title) (at-least-two? title "```"))
extract? (get-in options [:user-options :extract-code-snippets?])
[final-block code-children-tx]
(if has-fence?
(let [{:keys [text-parts code-segs]} (split-title-by-code-fences title)
pure-single-code? (and (= 1 (count code-segs))
(every? string/blank? text-parts))
has-mixed-content? (and extract?
(seq code-segs)
(some #(not (string/blank? %)) text-parts))]
(cond
pure-single-code?
(let [{:keys [text lang]} (first code-segs)]
[(cond-> (assoc block'
:block/title text
:block/tags [:logseq.class/Code-block]
:logseq.property.node/display-type :code)
lang (assoc :logseq.property.code/lang lang))
[]])
has-mixed-content?
(let [remaining-title (-> (string/join "\n" text-parts)
(string/replace #"\n{2,}" "\n")
string/trim)
updated-block (assoc block' :block/title remaining-title)
code-children (build-code-snippet-child-blocks updated-block code-segs)]
[updated-block code-children])
:else
[block' []]))
[block' []])]
[final-block code-children-tx]))
(defn- <build-block-tx
[db block* pre-blocks {:keys [page-names-to-uuids] :as per-file-state}
{:keys [import-state journal-created-ats user-config] :as options}]
@@ -1439,14 +1690,16 @@
(update-block-tags db (:user-options options) per-file-state (:all-idents import-state))
(handle-embeds page-names-to-uuids walked-ast-blocks (select-keys options [:log-fn]))
(handle-quotes (select-keys options [:log-fn]))
(handle-math)
(update-block-marker options)
(update-block-priority options)
add-missing-timestamps
(dissoc :block/format :block.temp/ast-blocks)
;; ((fn [x] (prn ::block-out x) x))
)]
;; Order matters as previous txs are referenced in block
(concat properties-tx deadline-properties-tx asset-blocks-tx [block'])))
(let [[final-block code-children-tx] (handle-code-blocks block' options)]
;; Order matters as previous txs are referenced in block
(concat properties-tx deadline-properties-tx asset-blocks-tx [final-block] code-children-tx))))
(defn- update-page-alias
[m page-names-to-uuids]
@@ -1912,7 +2165,7 @@
* :extract-options - Options map to pass to extract/extract
* :user-options - User provided options maps that alter how a file is converted to db graph. Current options
are: :tag-classes (set), :property-classes (set), :property-parent-classes (set), :convert-all-tags? (boolean)
and :remove-inline-tags? (boolean)
:remove-inline-tags? (boolean), :extract-code-snippets? (boolean)
* :import-state - useful import state to maintain across files e.g. property schemas or ignored properties
* :macros - map of macros for use with macro expansion
* :notify-user - Displays warnings to user without failing the import. Fn receives a map with :msg
@@ -1923,8 +2176,10 @@
:as *options}]
(p/let [options (assoc *options :notify-user notify-user :log-fn log-fn :file file)
{:keys [pages blocks]} (extract-pages-and-blocks @conn file content options)
{:keys [blocks preserve-empty-properties-uuids]} (handle-template-blocks blocks)
tx-options (merge (build-tx-options options)
{:journal-created-ats (build-journal-created-ats pages)})
{:journal-created-ats (build-journal-created-ats pages)
:preserve-empty-property-block-uuids preserve-empty-properties-uuids})
old-properties (keys @(get-in options [:import-state :property-schemas]))
;; Build page and block txs
{:keys [pages-tx page-properties-tx per-file-state existing-pages]} (build-pages-tx conn pages blocks tx-options)

469
deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs vendored
View File

@@ -7,6 +7,7 @@
[datascript.core :as d]
[logseq.common.config :as common-config]
[logseq.common.graph :as common-graph]
[logseq.common.path :as path]
[logseq.common.util.date-time :as date-time-util]
[logseq.db :as ldb]
[logseq.db.common.entity-plus :as entity-plus]
@@ -47,6 +48,37 @@
first
(d/entity db)))
(defn- ordered-children
[block]
(->> (:block/_parent block)
(remove :logseq.property/created-from-property)
(sort-by :block/order)
vec))
(defn- block-tree-with-properties
[block]
{:title (:block/title block)
:properties (dissoc (db-test/readable-properties block) :block/tags)
:children (mapv block-tree-with-properties (ordered-children block))})
(defn- find-template-by-title
[db title]
(some->> (d/q '[:find [?b ...]
:in $ ?title
:where
[?b :block/title ?title]
[?b :block/tags :logseq.class/Template]]
db title)
first
(d/entity db)))
(defn- template-content-trees
[db title]
(some->> (find-template-by-title db title)
ordered-children
(mapv block-tree-with-properties)))
(defn- build-graph-files
"Given a file graph directory, return all files including assets and adds relative paths
on ::rpath since paths are absolute by default and exporter needs relative paths for
@@ -54,8 +86,8 @@
[dir*]
(let [dir (node-path/resolve dir*)]
(->> (common-graph/get-files dir)
(concat (when (fs/existsSync (node-path/join dir* "assets"))
(common-graph/readdir (node-path/join dir* "assets"))))
(concat (when (fs/existsSync (path/path-join dir* "assets"))
(common-graph/readdir (path/path-join dir* "assets"))))
(mapv #(hash-map :path %
::rpath (node-path/relative dir* %))))))
@@ -173,6 +205,84 @@
"assets/subdir/partydino.gif"]
"[[FIRST UUID]] and [[UUID]]"))
(deftest extract-template-blocks
(let [page-uuid (random-uuid)
parent-uuid (random-uuid)
child-uuid (random-uuid)
include-children-only-uuid (random-uuid)
child-only-1-uuid (random-uuid)
child-only-2-uuid (random-uuid)
blocks [{:block/uuid parent-uuid
:block/title "source parent"
:block/page [:block/uuid page-uuid]
:block/parent {:block/uuid page-uuid}
:block/order "a"
:block/properties {:template " trimmed template "
:name ""}
:block/properties-text-values {:template " trimmed template "
:name ""}
:block/properties-order [:template :name]}
{:block/uuid child-uuid
:block/title "child"
:block/page [:block/uuid page-uuid]
:block/parent [:block/uuid parent-uuid]
:block/order "b"
:block/properties {:template "nested child"
:name "child default"}
:block/properties-text-values {:template "nested child"
:name "child default"}
:block/properties-order [:template :name]}
{:block/uuid include-children-only-uuid
:block/title "exclude source block"
:block/page [:block/uuid page-uuid]
:block/parent {:block/uuid page-uuid}
:block/order "c"
:block/properties {:template "children only"
:template-including-parent false}
:block/properties-text-values {:template "children only"
:template-including-parent "false"}
:block/properties-order [:template :template-including-parent]}
{:block/uuid child-only-1-uuid
:block/title "first child"
:block/page [:block/uuid page-uuid]
:block/parent [:block/uuid include-children-only-uuid]
:block/order "d"}
{:block/uuid child-only-2-uuid
:block/title "second child"
:block/page [:block/uuid page-uuid]
:block/parent [:block/uuid include-children-only-uuid]
:block/order "e"}]
{:keys [blocks preserve-empty-properties-uuids]}
(#'gp-exporter/handle-template-blocks blocks)]
(testing "template roots replace source blocks"
(is (= ["trimmed template"
"source parent"
"nested child"
"child"
"children only"
"first child"
"second child"]
(mapv :block/title blocks)))
(is (= #{parent-uuid child-uuid include-children-only-uuid child-only-1-uuid child-only-2-uuid}
(set/intersection preserve-empty-properties-uuids
#{parent-uuid child-uuid include-children-only-uuid child-only-1-uuid child-only-2-uuid}))))
(testing "template roots use trimmed names and include parent content when configured"
(is (= #{"trimmed template" "nested child" "children only"}
(->> blocks
(filter #(some #{:logseq.class/Template} (:block/tags %)))
(map :block/title)
set)))
(is (= ["source parent"]
(->> blocks
(remove #(some #{:logseq.class/Template} (:block/tags %)))
(filter #(= [:block/uuid parent-uuid] (:block/parent %)))
(map :block/title))))
(is (= 2
(count (set/difference preserve-empty-properties-uuids
#{parent-uuid child-uuid include-children-only-uuid child-only-1-uuid child-only-2-uuid})))
"in-place template content blocks are marked to preserve empty properties"))))
(deftest-async ^:integration export-docs-graph-with-convert-all-tags
(p/let [file-graph-dir "test/resources/docs-0.10.12"
start-time (cljs.core/system-time)
@@ -216,12 +326,16 @@
;; Counts
;; Includes journals as property values e.g. :logseq.property/deadline
(is (= 33 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Journal]] @conn))))
(is (= 34 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Journal]] @conn))))
(is (= 9 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Asset]] @conn))))
(is (= 5 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Task]] @conn))))
(is (= 4 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Query]] @conn))))
(is (= 2 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Card]] @conn))))
(is (= 1 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Cards]] @conn))))
(is (= 2 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Code-block]] @conn))))
(is (= 1 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Math-block]] @conn))))
(is (= 9 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Template]] @conn))))
(is (= 5 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Quote-block]] @conn))))
(is (= 7 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Pdf-annotation]] @conn))))
@@ -264,7 +378,7 @@
set))))
(testing "user properties"
(is (= 21
(is (= 23
(->> @conn
(d/q '[:find [(pull ?b [:db/ident]) ...]
:where [?b :block/tags :logseq.class/Property]])
@@ -331,11 +445,6 @@
(mapv :db/id (:block/refs (db-test/find-block-by-content @conn #"ref to"))))
"block with a block-ref has correct :block/refs")
(let [b (db-test/find-block-by-content @conn #"MEETING TITLE")]
(is (= {}
(and b (db-test/readable-properties b)))
":template properties are ignored to not invalidate its property types"))
(is (= 20221126
(-> (db-test/readable-properties (db-test/find-block-by-content @conn "only deadline"))
:logseq.property/deadline
@@ -413,11 +522,107 @@
(:block/title (db-test/find-block-by-content @conn #"tasks with todo")))
"Advanced query has custom title migrated")
;; Cards
;; Card
(is (= {:block/tags [:logseq.class/Card]}
(db-test/readable-properties (db-test/find-block-by-content @conn "card 1")))
"None of the card properties are imported since they are deprecated")
;; Cards (flashcard browser)
(is (= {:block/tags [:logseq.class/Cards]
:logseq.property/query "(tags #Card)"}
(db-test/readable-properties (find-block-by-property-value @conn :logseq.property/query "(tags #Card)")))
"cards macro block has correct Cards class and query property")
;; Math blocks
(is (= {:block/tags [:logseq.class/Math-block]
:logseq.property.node/display-type :math}
(db-test/readable-properties (db-test/find-block-by-content @conn "E=mc^2")))
"Math block has correct Math-block class and display-type")
(is (= "E=mc^2" (:block/title (db-test/find-block-by-content @conn "E=mc^2")))
"Math block title has delimiters stripped")
;; Templates
(is (= #{"meeting"
"title-only-no-children"
"properties-only-no-children"
"title-only-with-children"
"empty-title-with-children"
"children-only"
"nested-father"
"nested-child-1"
"nested-child-2"}
(->> (d/q '[:find [?title ...]
:where
[?b :block/tags :logseq.class/Template]
[?b :block/title ?title]]
@conn)
set))
"All template definitions are imported as Template blocks")
(let [journal-uuid (:block/uuid (db-test/find-journal-by-journal-day @conn 20240216))
template-page-uuids (->> (d/q '[:find [?page-uuid ...]
:where
[?b :block/tags :logseq.class/Template]
[?b :block/page ?page]
[?page :block/uuid ?page-uuid]]
@conn)
set)]
(is (= #{journal-uuid} template-page-uuids)
"All template blocks are created on their source journal page"))
(is (= [{:title "MEETING TITLE"
:properties {:user.property/participants #{"TODO"}}
:children []}]
(template-content-trees @conn "meeting")))
(is (= [{:title "TITLE"
:properties {}
:children []}]
(template-content-trees @conn "title-only-no-children")))
(is (= [{:title ""
:properties {:user.property/name ""
:user.property/author ""}
:children []}]
(template-content-trees @conn "properties-only-no-children")))
(is (= [{:title "TITLE"
:properties {}
:children [{:title "intro" :properties {} :children []}
{:title "notes" :properties {} :children []}]}]
(template-content-trees @conn "title-only-with-children")))
(is (= [{:title ""
:properties {}
:children [{:title "intro" :properties {} :children []}
{:title "notes" :properties {} :children []}]}]
(template-content-trees @conn "empty-title-with-children")))
(is (= [{:title "intro" :properties {} :children []}
{:title "notes" :properties {} :children []}]
(template-content-trees @conn "children-only")))
(is (= [{:title "it's a template with nested templates"
:properties {:user.property/name "you named it"}
:children [{:title "nested-child-1"
:properties {}
:children [{:title "child-1"
:properties {:user.property/name ""}
:children [{:title "child-1-1"
:properties {:user.property/name ""}
:children []}]}]}
{:title "nested-child-2"
:properties {}
:children [{:title "child-2-1"
:properties {:user.property/name ""}
:children []}]}
{:title "child-3"
:properties {:user.property/name ""}
:children []}]}]
(template-content-trees @conn "nested-father")))
(is (= [{:title "child-1"
:properties {:user.property/name ""}
:children [{:title "child-1-1"
:properties {:user.property/name ""}
:children []}]}]
(template-content-trees @conn "nested-child-1")))
(is (= [{:title "child-2-1"
:properties {:user.property/name ""}
:children []}]
(template-content-trees @conn "nested-child-2")))
;; Assets
(is (= {:block/tags [:logseq.class/Asset]
:logseq.property.asset/type "png"
@@ -652,9 +857,12 @@
(is (= :node
(:logseq.property/type (d/entity @conn :user.property/finishedat)))
":date property to :node value changes to :node")
(is (= :node
(is (= :default
(:logseq.property/type (d/entity @conn :user.property/participants)))
":node property to :date value remains :node")
"template values cause participants to remain a :default property")
(is (= #{"[[Feb 7th, 2024]]"}
(:user.property/participants (db-test/readable-properties (db-test/find-block-by-content @conn #"test :node -> :date"))))
":default participants property keeps the imported text value")
(is (= :default
(:logseq.property/type (d/entity @conn :user.property/description)))
@@ -807,7 +1015,7 @@
(deftest-async export-files-with-tag-classes-option
(p/let [file-graph-dir "test/resources/exporter-test-graph"
files (mapv #(node-path/join file-graph-dir %) ["journals/2024_02_07.md" "pages/Interstellar.md"])
files (mapv #(path/path-join file-graph-dir %) ["journals/2024_02_07.md" "pages/Interstellar.md"])
conn (db-test/create-conn)
_ (import-files-to-db files conn {:tag-classes ["movie"]})]
(is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
@@ -833,7 +1041,7 @@
(deftest-async export-files-with-property-classes-option
(p/let [file-graph-dir "test/resources/exporter-test-graph"
files (mapv #(node-path/join file-graph-dir %)
files (mapv #(path/path-join file-graph-dir %)
["journals/2024_02_23.md" "pages/url.md" "pages/Whiteboard___Tool.md"
"pages/Whiteboard___Arrow_head_toggle.md"
"pages/Library.md"])
@@ -880,7 +1088,7 @@
(deftest-async export-files-with-remove-inline-tags
(p/let [file-graph-dir "test/resources/exporter-test-graph"
files (mapv #(node-path/join file-graph-dir %) ["journals/2024_02_07.md"
files (mapv #(path/path-join file-graph-dir %) ["journals/2024_02_07.md"
"journals/2026_01_27.md"])
conn (db-test/create-conn)
_ (import-files-to-db files conn {:remove-inline-tags? false :convert-all-tags? true})]
@@ -896,7 +1104,7 @@
(deftest-async export-files-with-ignored-properties
(p/let [file-graph-dir "test/resources/exporter-test-graph"
files (mapv #(node-path/join file-graph-dir %) ["ignored/icon-page.md"])
files (mapv #(path/path-join file-graph-dir %) ["ignored/icon-page.md"])
conn (db-test/create-conn)
{:keys [import-state]} (import-files-to-db files conn {})]
(is (= 2
@@ -905,7 +1113,7 @@
(deftest-async export-files-with-property-parent-classes-option
(p/let [file-graph-dir "test/resources/exporter-test-graph"
files (mapv #(node-path/join file-graph-dir %) ["journals/2024_11_26.md"
files (mapv #(path/path-join file-graph-dir %) ["journals/2024_11_26.md"
"pages/CreativeWork.md" "pages/Movie.md" "pages/type.md"
"pages/Whiteboard___Tool.md" "pages/Whiteboard___Arrow_head_toggle.md"
"pages/Property.md" "pages/url.md"])
@@ -933,7 +1141,7 @@
(deftest-async export-files-with-property-pages-disabled
(p/let [file-graph-dir "test/resources/exporter-test-graph"
;; any page with properties
files (mapv #(node-path/join file-graph-dir %) ["journals/2024_01_17.md"])
files (mapv #(path/path-join file-graph-dir %) ["journals/2024_01_17.md"])
conn (db-test/create-conn)
_ (import-files-to-db files conn {:user-config {:property-pages/enabled? false
:property-pages/excludelist #{:prop-string}}})]
@@ -948,3 +1156,228 @@
(is (= "yyyy-MM-dd"
(:logseq.property.journal/title-format (d/entity @conn :logseq.class/Journal)))
"title format set correctly by config")))
(deftest split-title-by-code-fences
(let [split-fn #'gp-exporter/split-title-by-code-fences]
(testing "standalone code fence with language"
(is (= {:text-parts []
:code-segs [{:text "it's an individual code snippet with language tag"
:lang "markdown"}]}
(split-fn "```markdown\nit's an individual code snippet with language tag\n```"))))
(testing "standalone code fence without language"
(is (= {:text-parts []
:code-segs [{:text "it's an individual code snippet without language tag"
:lang nil}]}
(split-fn "```\nit's an individual code snippet without language tag\n```"))))
(testing "one code fence with leading text"
(is (= {:text-parts ["before code snippet"]
:code-segs [{:text "echo \"ok\"\nexit"
:lang nil}]}
(split-fn "before code snippet\n```\necho \"ok\"\nexit\n```"))))
(testing "one code fence with leading and trailing text"
(is (= {:text-parts ["before code snippet" "after code snippet"]
:code-segs [{:text "echo \"ok\"\nexit"
:lang "bash"}]}
(split-fn "before code snippet\n```bash\necho \"ok\"\nexit\n```\nafter code snippet"))))
(testing "one code fence followed by trailing text"
(is (= {:text-parts ["after code snippet"]
:code-segs [{:text "echo \"ok\"\nexit"
:lang "bash"}]}
(split-fn "```bash\necho \"ok\"\nexit\n```\nafter code snippet"))))
(testing "multiple code fences mixed with text"
(is (= {:text-parts ["before code snippet" "middle" "after code snippet"]
:code-segs [{:text "echo \"ok\"\nexit"
:lang "bash"}
{:text "echo \"bye\"\nexit"
:lang "bash"}]}
(split-fn "before code snippet\n```bash\necho \"ok\"\nexit\n```\nmiddle\n```bash\necho \"bye\"\nexit\n```\nafter code snippet"))))
(testing "edge: one code fence followed by opening fence without closing fence"
(is (= {:text-parts ["echo \"missing end fence\""] ;; no "```bash" ahead, it's fine as is; let's leave it
:code-segs [{:text "echo \"ok\"\nexit"
:lang "bash"}]}
(split-fn "```bash\necho \"ok\"\nexit\n```\n```bash\necho \"missing end fence\""))))
(testing "edge: pure multiple code fences with no extra text"
(let [{:keys [text-parts code-segs]} (split-fn "```markdown\n1st code snippet with language tag\n```\n```\n2nd code snippet without language tag\n```")]
(is (and (empty? text-parts) (> (count code-segs) 1)) "not pure single code and no mixed content")))
(testing "edge: opening fence without closing fence"
(let [title "```bash\necho \"missing end fence\""
{:keys [text-parts code-segs]} (split-fn title)]
(is (and (= (count text-parts) 1) (not= (first text-parts) title) (empty? code-segs)) "not pure single code and no mixed content")))
(testing "edge: plain text without any code fence"
(is (= {:text-parts ["plain text only"]
:code-segs []}
(split-fn "plain text only"))))
(testing "edge: empty title"
(is (= {:text-parts [""]
:code-segs []}
(split-fn ""))))))
(deftest-async export-files-with-extract-code-snippet
(p/let [file-graph-dir "test/resources/exporter-test-graph"
files (mapv #(path/path-join file-graph-dir %) ["journals/2026_03_01.md"])
conn (db-test/create-conn)
_ (import-files-to-db files conn {:extract-code-snippets? true})
journal-page-eid (d/q '[:find ?p . :where [?p :block/journal-day 20260301]] @conn)
top-blocks (->> (d/q '[:find [?b ...]
:in $ ?page
:where
[?b :block/page ?page]
[?b :block/parent ?page]]
@conn journal-page-eid)
(map #(d/entity @conn %))
(sort-by :block/order)
vec)
get-direct-children (fn [block]
(->> (d/q '[:find [?c ...]
:in $ ?parent
:where [?c :block/parent ?parent]]
@conn (:db/id block))
(map #(d/entity @conn %))))]
(testing "standalone code block with language tag"
(let [b (nth top-blocks 0)]
(is (= "it's an individual code snippet with language tag" (:block/title b))
"Standalone code block title has fences stripped")
(is (= 0 (count (get-direct-children b)))
"Standalone code block has no children")
(is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags b))))
"Standalone code block is tagged as Code-block")
(is (= "markdown" (:logseq.property.code/lang b))
"Standalone code block has markdown language property")))
(testing "standalone code block without language tag"
(let [b (nth top-blocks 1)]
(is (= "it's an individual code snippet without language tag" (:block/title b))
"Standalone code block title has fences stripped")
(is (= 0 (count (get-direct-children b)))
"Standalone code block has no children")
(is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags b))))
"Standalone code block is tagged as Code-block")
(is (= nil (:logseq.property.code/lang b))
"Standalone code block has no language property")))
(testing "text before code snippet"
(let [b (nth top-blocks 2)
children (get-direct-children b)]
(is (= "before code snippet" (:block/title b))
"Block title has text only without code")
(is (= 1 (count children))
"Block has 1 code child")
(is (= "echo \"ok\"\nexit" (:block/title (first children)))
"Child code block has correct content without fence markers")
(is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags (first children)))))
"Child block is tagged as Code-block")
(is (= nil (:logseq.property.code/lang (first children)))
"Child block has no language property")))
(testing "text before and after code snippet"
(let [b (nth top-blocks 3)
children (get-direct-children b)]
(is (= "before code snippet\nafter code snippet" (:block/title b))
"Block title has text only without code")
(is (= 1 (count children))
"Block has 1 code child")
(is (= "echo \"ok\"\nexit" (:block/title (first children)))
"Child code block has correct content without fence markers")
(is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags (first children)))))
"Child block is tagged as Code-block")
(is (= "bash" (:logseq.property.code/lang (first children)))
"Child block has bash language property")))
(testing "code snippet before text"
(let [b (nth top-blocks 4)
children (get-direct-children b)]
(is (= "after code snippet" (:block/title b))
"Block title has text only without code")
(is (= 1 (count children))
"Block has 1 code child")
(is (= "echo \"ok\"\nexit" (:block/title (first children)))
"Child code block has correct content without fence markers")
(is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags (first children)))))
"Child block is tagged as Code-block")
(is (= "bash" (:logseq.property.code/lang (first children)))
"Child block has bash language property")))
(testing "multiple code snippets mixed with text"
(let [b (nth top-blocks 5)
children (sort-by :block/order (get-direct-children b))]
(is (= "before code snippet\nmiddle\nafter code snippet" (:block/title b))
"Block title has all text parts without code")
(is (= 2 (count children))
"Block has 2 code children")
(is (= "echo \"ok\"\nexit" (:block/title (first children)))
"First child code block has correct content without fence markers")
(is (= "echo \"bye\"\nexit" (:block/title (second children)))
"Second child code block has correct content without fence markers")
(is (every? #(= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags %)))) children)
"Both child blocks are tagged as Code-block")
(is (every? #(= "bash" (:logseq.property.code/lang %)) children)
"Both child blocks have bash language property")))))
(deftest-async export-files-without-extract-code-snippet
(p/let [file-graph-dir "test/resources/exporter-test-graph"
files (mapv #(path/path-join file-graph-dir %) ["journals/2026_03_01.md"])
conn (db-test/create-conn)
_ (import-files-to-db files conn {:extract-code-snippets? false})
journal-page-eid (d/q '[:find ?p . :where [?p :block/journal-day 20260301]] @conn)
top-blocks (->> (d/q '[:find [?b ...]
:in $ ?page
:where
[?b :block/page ?page]
[?b :block/parent ?page]]
@conn journal-page-eid)
(map #(d/entity @conn %))
(sort-by :block/order)
vec)
get-direct-children (fn [block]
(->> (d/q '[:find [?c ...]
:in $ ?parent
:where [?c :block/parent ?parent]]
@conn (:db/id block))
(map #(d/entity @conn %))))]
(testing "standalone code block with language tag is still tagged as Code-block"
(let [b (nth top-blocks 0)]
(is (= "it's an individual code snippet with language tag" (:block/title b))
"Standalone code block title has fences stripped")
(is (= 0 (count (get-direct-children b)))
"Standalone code block has no children")
(is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags b))))
"Standalone code block is tagged as Code-block")
(is (= "markdown" (:logseq.property.code/lang b))
"Standalone code block has markdown language property")))
(testing "standalone code block without language tag is still tagged as Code-block"
(let [b (nth top-blocks 1)]
(is (= "it's an individual code snippet without language tag" (:block/title b))
"Standalone code block title has fences stripped")
(is (= 0 (count (get-direct-children b)))
"Standalone code block has no children")
(is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags b))))
"Standalone code block is tagged as Code-block")
(is (= nil (:logseq.property.code/lang b))
"Standalone code block has no language property")))
(testing "mixed-content block is NOT extracted into children when extract-code-snippets? is false"
(let [b (nth top-blocks 2)]
(is (= 0 (count (get-direct-children b)))
"Block with text before code has no children extracted")
(is (string/includes? (:block/title b) "```")
"Block title retains raw code fence markup")))
(testing "another mixed-content block is NOT extracted when extract-code-snippets? is false"
(let [b (nth top-blocks 3)]
(is (= 0 (count (get-direct-children b)))
"Block with text surrounding code has no children extracted")
(is (string/includes? (:block/title b) "```")
"Block title retains raw code fence markup")))))

35
deps/graph-parser/test/resources/exporter-test-graph/journals/2024_02_16.md vendored
View File

@@ -4,6 +4,41 @@
- MEETING TITLE
template:: meeting
participants:: TODO
- TITLE
template:: title-only-no-children
- template:: properties-only-no-children
name::
author::
- TITLE
template:: title-only-with-children
- intro
- notes
-
template:: empty-title-with-children
- intro
- notes
- it should not be included in the template
template:: children-only
template-including-parent:: false
- intro
- notes
- it's a template with nested templates
template:: nested-father
template-including-parent:: true
name:: you named it
- child-1
template:: nested-child-1
name::
- child-1-1
name::
- child-2
template:: nested-child-2
template-including-parent:: false
name::
- child-2-1
name::
- child-3
name::
- pending block for :number to :default
duration:: 10
- test :number to :default

35
deps/graph-parser/test/resources/exporter-test-graph/journals/2026_03_01.md vendored Normal file
View File

@@ -0,0 +1,35 @@
- ```markdown
it's an individual code snippet with language tag
```
- ```
it's an individual code snippet without language tag
```
- before code snippet
```
echo "ok"
exit
```
- before code snippet
```bash
echo "ok"
exit
```
after code snippet
- ```bash
echo "ok"
exit
```
after code snippet
- before code snippet
```bash
echo "ok"
exit
```
middle
```bash
echo "bye"
exit
```
after code snippet
- $$E=mc^2$$
- {{cards (tags #Card)}}

2
deps/outliner/deps.edn vendored
View File

@@ -1,7 +1,7 @@
{:deps
;; These nbb-logseq deps are kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn
{datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
:sha "ff5a7d5326e2546f40146e4a489343f557519bc3"}
:sha "f91fec561ee2c11d6bf323feae365e9033585411"}
;; datascript/datascript {:local/root "../../../../datascript"}
com.cognitect/transit-cljs {:mvn/version "0.8.280"}

2
deps/publish/deps.edn vendored
View File

@@ -5,7 +5,7 @@
:sha "5d672bf84ed944414b9f61eeb83808ead7be9127"}
datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
:sha "ff5a7d5326e2546f40146e4a489343f557519bc3"}
:sha "f91fec561ee2c11d6bf323feae365e9033585411"}
datascript-transit/datascript-transit {:mvn/version "0.3.0"
:exclusions [datascript/datascript]}
funcool/promesa {:mvn/version "11.0.678"}

141
scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs
View File

@@ -9,41 +9,80 @@
[nbb.classpath :as cp]
[nbb.core :as nbb]))
(def *ids (atom #{}))
(defn get-next-id
[]
(let [id (random-uuid)]
(if (@*ids id)
(get-next-id)
(do
(swap! *ids conj id)
id))))
(def ^:private default-block-title "Block")
(def ^:private target-entities-per-batch 25000)
(def ^:private max-pages-per-batch 1000)
(defn build-pages
[start-idx n]
(let [ids (repeatedly n get-next-id)]
(map-indexed
(fn [idx id]
{:block/uuid id
:block/title (str "Page-" (+ start-idx idx))})
ids)))
(defn- parse-long-option
[value]
(if (string? value)
(js/parseInt value 10)
value))
(defn build-blocks
[size]
(vec (repeatedly size
(fn []
(let [id (get-next-id)]
{:block/uuid id
:block/title (str id)})))))
(defn- create-init-data
(defn- normalize-options
[options]
(let [pages (build-pages 0 (:pages options))]
{:pages-and-blocks
(mapv #(hash-map :page % :blocks (build-blocks (:blocks options)))
pages)
;; Custom id fn because transaction chunks may separate blocks and pages from each other
:page-id-fn (fn [b] [:block/uuid (:block/uuid b)])}))
(update-vals options parse-long-option))
(defn default-batch-pages
[blocks-per-page]
(-> (quot target-entities-per-batch (max 1 (inc blocks-per-page)))
(max 1)
(min max-pages-per-batch)))
(defn- build-blocks
[blocks-per-page next-id]
(loop [block-idx 0
blocks (transient [])]
(if (= block-idx blocks-per-page)
(persistent! blocks)
(recur (inc block-idx)
(conj! blocks
{:block/uuid (next-id)
:block/title default-block-title})))))
(defn build-page-and-blocks-batch
([start-idx page-count blocks-per-page]
(build-page-and-blocks-batch start-idx page-count blocks-per-page random-uuid))
([start-idx page-count blocks-per-page next-id]
(loop [page-idx 0
pages-and-blocks (transient [])]
(if (= page-idx page-count)
(persistent! pages-and-blocks)
(recur (inc page-idx)
(conj! pages-and-blocks
{:page {:block/uuid (next-id)
:block/title (str "Page-" (+ start-idx page-idx))}
:blocks (build-blocks blocks-per-page next-id)}))))))
(defn page-and-block-batches
([{:keys [pages blocks batch-pages]}]
(page-and-block-batches {:pages pages
:blocks blocks
:batch-pages batch-pages}
random-uuid))
([{:keys [pages blocks batch-pages]} next-id]
(let [batch-pages' (or batch-pages (default-batch-pages blocks))]
((fn step [start-idx]
(lazy-seq
(when (< start-idx pages)
(cons (build-page-and-blocks-batch start-idx
(min batch-pages' (- pages start-idx))
blocks
next-id)
(step (+ start-idx batch-pages'))))))
0))))
(defn- transact-batch!
[conn pages-and-blocks]
(let [{:keys [init-tx block-props-tx]} (outliner-cli/build-blocks-tx {:pages-and-blocks pages-and-blocks})]
(d/transact! conn init-tx)
(when (seq block-props-tx)
(d/transact! conn block-props-tx))))
(defn- total-batches
[{:keys [pages blocks batch-pages]}]
(let [batch-pages' (or batch-pages (default-batch-pages blocks))]
(js/Math.ceil (/ pages batch-pages'))))
(def spec
"Options spec"
@@ -54,34 +93,36 @@
:desc "Number of pages to create"}
:blocks {:alias :b
:default 20
:desc "Number of blocks to create"}})
:desc "Number of blocks to create per page"}
:batch-pages {:alias :t
:desc "Number of pages to build and transact per batch"}})
(defn parse-args
[args]
{:graph-dir (first args)
:options (normalize-options (cli/parse-opts (rest args) {:spec spec}))})
(defn -main [args]
(let [graph-dir (first args)
options (cli/parse-opts args {:spec spec})
(let [{:keys [graph-dir options]} (parse-args args)
_ (when (or (nil? graph-dir) (:help options))
(println (str "Usage: $0 GRAPH-NAME [OPTIONS]\nOptions:\n"
(cli/format-opts {:spec spec})))
(js/process.exit 1))
{:keys [pages blocks batch-pages]} options
[dir db-name] (if (string/includes? graph-dir "/")
((juxt node-path/dirname node-path/basename) graph-dir)
[(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
conn (outliner-cli/init-conn dir db-name {:classpath (cp/get-classpath)})
_ (println "Building tx ...")
{:keys [init-tx]} (outliner-cli/build-blocks-tx (create-init-data options))]
(println "Built" (count init-tx) "tx," (count (filter :block/title init-tx)) "pages and"
(count (filter :block/title init-tx)) "blocks ...")
;; Vary the chunking with page size up to a max to avoid OOM
(let [tx-chunks (partition-all (min (:pages options) 30000) init-tx)]
(loop [chunks tx-chunks
chunk-num 1]
(when-let [chunk (first chunks)]
(println "Transacting chunk" chunk-num "of" (count tx-chunks)
"starting with block:" (pr-str (select-keys (first chunk) [:block/title :block/title])))
(d/transact! conn chunk)
(recur (rest chunks) (inc chunk-num)))))
#_(d/transact! conn blocks-tx)
(println "Created graph" (str db-name " with " (count (d/datoms @conn :eavt)) " datoms!"))))
total-batches' (total-batches options)
pages-per-batch (or batch-pages (default-batch-pages blocks))
total-blocks (* pages blocks)]
(println "Creating graph with" pages "pages and" total-blocks "blocks"
"using" total-batches' "batch(es) of up to" pages-per-batch "pages ...")
(doseq [[batch-num pages-and-blocks] (map-indexed vector (page-and-block-batches options))]
(println "Transacting batch" (inc batch-num) "of" total-batches'
"with" (count pages-and-blocks) "pages")
(transact-batch! conn pages-and-blocks))
(println "Created graph" db-name "with" pages "pages and" total-blocks "blocks.")))
(when (= nbb/*file* (nbb/invoked-file))
(-main *command-line-args*))

69
scripts/test/logseq/tasks/db_graph/create_graph_with_large_sizes_test.cljs Normal file
View File

@@ -0,0 +1,69 @@
(ns logseq.tasks.db-graph.create-graph-with-large-sizes-test
(:require [cljs.test :refer [deftest is testing]]
[logseq.tasks.db-graph.create-graph-with-large-sizes :as sut]))
(deftest build-page-and-blocks-batch-builds-the-requested-graph-slice
(let [id-seq (map #(str "id-" %) (range))
next-id (let [ids (atom id-seq)]
(fn []
(let [id (first @ids)]
(swap! ids rest)
id)))
batch (#'sut/build-page-and-blocks-batch 10 2 3 next-id)]
(is (= 2 (count batch)))
(is (= ["Page-10" "Page-11"]
(map (comp :block/title :page) batch)))
(is (= ["id-0" "id-4"]
(map (comp :block/uuid :page) batch)))
(is (= [["Block" "Block" "Block"]
["Block" "Block" "Block"]]
(map (fn [{:keys [blocks]}]
(mapv :block/title blocks))
batch)))
(is (= [["id-1" "id-2" "id-3"]
["id-5" "id-6" "id-7"]]
(map (fn [{:keys [blocks]}]
(mapv :block/uuid blocks))
batch)))))
(deftest page-and-block-batches-only-realize-requested-batches
(let [calls (atom 0)
next-id (fn []
(swap! calls inc)
(str "id-" @calls))
batches (#'sut/page-and-block-batches {:pages 50000
:blocks 50
:batch-pages 100}
next-id)
first-batch (first batches)]
(is (= 100 (count first-batch)))
(is (= (* 100 51) @calls)
"Only the first batch should be realized")
(is (= "Page-0" (get-in first-batch [0 :page :block/title])))
(is (= "Page-99" (get-in first-batch [99 :page :block/title])))))
(deftest default-batching-keeps-large-graphs-bounded
(testing "50k pages with 50 blocks are split into many batches instead of one giant tx"
(let [batch-pages (#'sut/default-batch-pages 50)]
(is (< batch-pages 50000))
(is (pos? batch-pages))
(is (= batch-pages
(count (first (#'sut/page-and-block-batches {:pages 50000
:blocks 50}
(constantly "id")))))))))
(deftest page-and-block-batches-handle-empty-input
(is (= []
(into [] (#'sut/page-and-block-batches {:pages 0
:blocks 50}
(constantly "id"))))))
(deftest parse-args-keeps-the-graph-name-separate-from-cli-options
(let [{:keys [graph-dir options]} (sut/parse-args ["large-graph"
"-p" "3"
"-b" "2"
"-t" "1"])]
(is (= "large-graph" graph-dir))
(is (= 3 (:pages options)))
(is (= 2 (:blocks options)))
(is (= 1 (:batch-pages options)))))

8
scripts/test/logseq/tasks/test_runner.cljs Normal file
View File

@@ -0,0 +1,8 @@
(ns logseq.tasks.test-runner
(:require [cljs.test :as test]
[logseq.tasks.db-graph.create-graph-with-large-sizes-test]))
(defn -main [& _]
(let [{:keys [fail error]} (test/run-tests 'logseq.tasks.db-graph.create-graph-with-large-sizes-test)]
(when (pos? (+ fail error))
(js/process.exit 1))))

110
src/main/frontend/components/block.cljs
View File

@@ -202,11 +202,24 @@
(on-dimensions (.-naturalWidth img) (.-naturalHeight img))))
(set! (.-src img) url)))
(defn- normalize-asset-align
[asset-align]
(cond
(keyword? asset-align) asset-align
(string? asset-align) (case asset-align
"left" :left
"center" :center
"right" :right
nil)
:else nil))
(defonce *resizing-image? (atom false))
(rum/defc ^:large-vars/cleanup-todo asset-container
[asset-block src title metadata {:keys [breadcrumb? positioned? local? full-text]}]
(let [asset-width (:logseq.property.asset/width asset-block)
asset-height (:logseq.property.asset/height asset-block)]
asset-height (:logseq.property.asset/height asset-block)
asset-align (normalize-asset-align (:logseq.property.asset/align asset-block))]
(hooks/use-effect!
(fn []
(when (:block/uuid asset-block)
@@ -275,7 +288,13 @@
:repo (state/get-current-repo)
:href src
:title title
:full-text full-text})))))))]
:full-text full-text})))))))
handle-set-align!
(fn [align]
(when-let [asset-id (:block/uuid asset-block)]
(property-handler/set-block-property! asset-id
:logseq.property.asset/align
align)))]
(when asset-block
[:.asset-action-bar {:aria-hidden "true"}
(shui/dropdown-menu
@@ -288,6 +307,33 @@
:class "h-6 w-6"}
(shui/tabler-icon "dots-vertical")))
(shui/dropdown-menu-content
(shui/dropdown-menu-sub
(shui/dropdown-menu-sub-trigger
[:span.flex.items-center.gap-1
(ui/icon "layout-align-left") (t :asset/align)])
(shui/dropdown-menu-sub-content
(shui/dropdown-menu-item
{:on-click #(handle-set-align! :left)}
[:span.flex.items-center.gap-2
(ui/icon "layout-align-left")
(t :asset/align-left)
(when (or (nil? asset-align) (= asset-align :left))
(ui/icon "check"))])
(shui/dropdown-menu-item
{:on-click #(handle-set-align! :center)}
[:span.flex.items-center.gap-2
(ui/icon "layout-align-center")
(t :asset/align-center)
(when (= asset-align :center)
(ui/icon "check"))])
(shui/dropdown-menu-item
{:on-click #(handle-set-align! :right)}
[:span.flex.items-center.gap-2
(ui/icon "layout-align-right")
(t :asset/align-right)
(when (= asset-align :right)
(ui/icon "check"))])))
(shui/dropdown-menu-item
{:on-click handle-copy!}
[:span.flex.items-center.gap-1
@@ -301,6 +347,7 @@
(js/window.apis.openExternal image-src)))}
[:span.flex.items-center.gap-1
(ui/icon "folder-pin") (t (if local? :asset/show-in-folder :asset/open-in-browser))]))
(when-not config/publishing?
[:<>
(shui/dropdown-menu-separator)
@@ -318,6 +365,7 @@
(let [breadcrumb? (:breadcrumb? config)
positioned? (:property-position config)
asset-block (:asset-block config)
asset-align (normalize-asset-align (:logseq.property.asset/align asset-block))
width (:width metadata)
*width (get state ::size)
width (or @*width width)
@@ -334,30 +382,42 @@
(:table-view? config)
(not resizable?))
asset-container-cp
[:div.ls-resize-image.rounded-md
asset-container-cp
(resize-image-handles
(fn [k ^js event]
(let [dx (.-dx event)
^js target (.-target event)]
[:div.ls-resize-inner.w-full.select-none
{:on-double-click (fn [^js e]
(let [^js target (.-target e)
^js container (.closest target ".ls-resize-inner")]
(when (or container (= target container))
(when-let [block-uuid (or (:block/uuid config)
(some-> config :block :block/uuid))]
(editor-handler/select-block! block-uuid)))))}
[:div.ls-resize-image.rounded-md
{:class (case asset-align
:center "align-center"
:right "align-right"
"align-left")}
asset-container-cp
(resize-image-handles
(fn [k ^js event]
(let [dx (.-dx event)
^js target (.-target event)]
(case k
:start
(let [c (.closest target ".ls-resize-image")]
(reset! *width (.-offsetWidth c))
(reset! *resizing-image? true))
:move
(let [width' (+ @*width dx)]
(when (or (> width' 60)
(not (neg? dx)))
(reset! *width width')))
:end
(let [width' @*width]
(when (and width' @*resizing-image?)
(when-let [block-id (or (:block/uuid config)
(some-> config :block (:block/uuid)))]
(editor-handler/resize-image! config block-id metadata full-text {:width width'})))
(reset! *resizing-image? false))))))])))
(case k
:start
(let [c (.closest target ".ls-resize-image")]
(reset! *width (.-offsetWidth c))
(reset! *resizing-image? true))
:move
(let [width' (+ @*width dx)]
(when (or (> width' 60)
(not (neg? dx)))
(reset! *width width')))
:end
(let [width' @*width]
(when (and width' @*resizing-image?)
(when-let [block-id (or (:block/uuid config)
(some-> config :block (:block/uuid)))]
(editor-handler/resize-image! config block-id metadata full-text {:width width'})))
(reset! *resizing-image? false))))))]])))
(rum/defc audio-cp
([src] (audio-cp src nil))

13
src/main/frontend/components/block.css
View File

@@ -1194,6 +1194,19 @@ html.is-mac {
.ls-resize-image {
@apply flex relative w-fit cursor-pointer;
&.align-left {
margin-right: auto;
}
&.align-center {
margin-left: auto;
margin-right: auto;
}
&.align-right {
margin-left: auto;
}
.handle-left, .handle-right {
@apply absolute w-[6px] h-[15%] min-h-[30px] bg-black/30 hover:bg-black/70
top-[50%] left-[5px] rounded-full cursor-col-resize select-none

10
src/main/frontend/components/imports.cljs
View File

@@ -182,6 +182,7 @@
[:div.border.p-6.rounded.bg-gray-01.mt-4
(let [form-ctx (form-core/use-form
{:defaultValues {:graph-name initial-name
:extract-code-snippets? false
:convert-all-tags? true
:tag-classes ""
:remove-inline-tags? true
@@ -212,6 +213,15 @@
(shui/form-description
[:b.text-red-800 (:message error)])))))
(shui/form-field {:name "extract-code-snippets?"}
(fn [field]
(shui/form-item
{:class "pt-3 flex justify-start items-center space-x-3 space-y-0 my-3 pr-3"}
(shui/form-label "Extract inline code snippets as child blocks")
(shui/form-control
(shui/checkbox {:checked (:value field)
:on-checked-change (:onChange field)})))))
(shui/form-field {:name "convert-all-tags?"}
(fn [field]
(shui/form-item

6
src/main/frontend/undo_redo.cljs
View File

@@ -391,8 +391,10 @@
(log/error ::undo-redo-failed e)
(clear-history! repo)))))
(do
(clear-history! repo)
(if undo? ::empty-undo-stack ::empty-redo-stack))))))))
(log/warn ::undo-redo-skip-conflicted-op
{:undo? undo?
:outliner-op (:outliner-op tx-meta)})
(undo-redo-aux repo undo?))))))))
(when ((if undo? empty-undo-stack? empty-redo-stack?) repo)
(prn (str "No further " (if undo? "undo" "redo") " information"))

3
src/main/frontend/worker/db/migrate.cljs
View File

@@ -75,7 +75,8 @@
["65.20" {:properties [:logseq.property.class/bidirectional-property-title :logseq.property.class/enable-bidirectional?]}]
["65.21" {:properties [:logseq.property.sync/large-title-object]}]
["65.22" {:properties [:logseq.property.reaction/emoji-id
:logseq.property.reaction/target]}]])
:logseq.property.reaction/target]}]
["65.23" {:properties [:logseq.property.asset/align]}]])
(let [[major minor] (last (sort (map (comp (juxt :major :minor) db-schema/parse-schema-version first)
schema-version->updates)))]

16
src/main/frontend/worker/sync.cljs
View File

@@ -648,8 +648,12 @@
[x]
(and (integer? x) (neg? x)))
(defn- remote-batch-temp-id
[temp-id]
(str "remote-batch-tempid-" temp-id))
(defn- remap-remote-batch-temp-ids
[batch-index tx-data]
[tx-data]
(let [ops #{:db/add :db/retract :db/retractEntity}
entity-temp-ids (->> tx-data
(keep (fn [item]
@@ -661,9 +665,7 @@
distinct)
temp-id-map (when (seq entity-temp-ids)
(zipmap entity-temp-ids
(map-indexed (fn [idx _]
(str "remote-batch-" batch-index "-tempid-" idx))
entity-temp-ids)))]
(map remote-batch-temp-id entity-temp-ids)))]
(if (seq temp-id-map)
(mapv (fn [item]
(if (and (vector? item)
@@ -731,11 +733,11 @@
(defn- flatten-batched-remote-tx-data
[tx-data*]
(loop [remaining (map-indexed vector tx-data*)
(loop [remaining tx-data*
lookup->temp-id {}
acc []]
(if-let [[batch-index tx-data] (first remaining)]
(let [remapped-batch (remap-remote-batch-temp-ids batch-index tx-data)
(if-let [tx-data (first remaining)]
(let [remapped-batch (remap-remote-batch-temp-ids tx-data)
lookup->temp-id (merge lookup->temp-id (created-lookup->temp-id remapped-batch))
resolved-batch (resolve-lookup-refs lookup->temp-id remapped-batch)]
(recur (rest remaining)

4
src/resources/dicts/en.edn
View File

@@ -135,6 +135,10 @@
:asset/ref-block "Asset ref block"
:asset/confirm-delete "Are you sure you want to delete this {1}?"
:asset/physical-delete "Remove the file too (notice it can't be restored)"
:asset/align "Align"
:asset/align-left "Align left"
:asset/align-center "Align center"
:asset/align-right "Align right"
:color/gray "Gray"
:color/red "Red"
:color/yellow "Yellow"

27
src/test/frontend/undo_redo_test.cljs
View File

@@ -348,6 +348,33 @@
(is (= child-uuid (:block/uuid (:block/parent parent))))
(is (= page-uuid (:block/uuid (:block/parent child))))))))
(deftest undo-skips-conflicted-move-and-keeps-earlier-history-test
(testing "undo drops a conflicting move op but still undoes earlier safe ops"
(undo-redo/clear-history! test-db)
(let [conn (db/get-db test-db false)
{:keys [parent-a-uuid parent-b-uuid child-uuid]} (seed-page-two-parents-child!)]
(d/transact! conn
[[:db/add [:block/uuid child-uuid] :block/title "local-title"]]
{:outliner-op :save-block
:local-tx? true})
(d/transact! conn
[[:db/add [:block/uuid child-uuid] :block/parent [:block/uuid parent-b-uuid]]]
{:outliner-op :move-blocks
:local-tx? true})
(d/transact! conn
[[:db/retractEntity [:block/uuid parent-a-uuid]]]
{:outliner-op :delete-blocks
:local-tx? false})
(let [undo-result (undo-redo/undo test-db)
child (d/entity @conn [:block/uuid child-uuid])]
(is (not= :frontend.undo-redo/empty-undo-stack undo-result))
(is (= "child" (:block/title child)))
(is (= parent-b-uuid
(:block/uuid (:block/parent child))))
(is (empty? (db-issues @conn))))
(is (= :frontend.undo-redo/empty-undo-stack
(undo-redo/undo test-db))))))
(deftest undo-validation-fast-path-skips-db-issues-for-non-structural-tx-test
(testing "undo validation skips db-issues for non-structural tx-data"
(let [conn (db/get-db test-db false)

72
src/test/frontend/worker/db_sync_sim_test.cljs
View File

@@ -1266,6 +1266,78 @@
(sync-loop! server [{:repo repo-a :conn conn-a :client client-a :online? true}])
(is (= "test" (:block/title (d/entity @conn-a [:block/uuid block-uuid])))))))))
(deftest ^:long two-clients-undo-skips-conflicted-move-but-keeps-db-valid-test
(testing "undo skips a conflicted move while syncing the remaining safe history"
(let [base-uuid (uuid "31111111-1111-1111-1111-111111111111")
parent-a-uuid (uuid "32222222-2222-2222-2222-222222222222")
parent-b-uuid (uuid "33333333-3333-3333-3333-333333333333")
child-uuid (uuid "34444444-4444-4444-4444-444444444444")
conn-a (db-test/create-conn)
conn-b (db-test/create-conn)
ops-a (d/create-conn client-op/schema-in-db)
ops-b (d/create-conn client-op/schema-in-db)
client-a (make-client repo-a)
client-b (make-client repo-b)
server (make-server)
seed 20260311
history (atom [])]
(with-test-repos {repo-a {:conn conn-a :ops-conn ops-a}
repo-b {:conn conn-b :ops-conn ops-b}}
(fn []
(let [{:keys [repro restore]} (install-invalid-tx-repro! seed history)]
(try
(reset! db-sync/*repo->latest-remote-tx {})
(client-op/update-local-tx repo-a 0)
(client-op/update-local-tx repo-b 0)
(ensure-base-page! conn-a base-uuid)
(let [base-a (d/entity @conn-a [:block/uuid base-uuid])]
(create-block! conn-a base-a "parent-a" parent-a-uuid)
(create-block! conn-a base-a "parent-b" parent-b-uuid)
(let [parent-a (d/entity @conn-a [:block/uuid parent-a-uuid])]
(create-block! conn-a parent-a "seed-child" child-uuid)))
(sync-until-idle! server [{:repo repo-a :conn conn-a :client client-a :online? true}
{:repo repo-b :conn conn-b :client client-b :online? true}]
20)
(update-title! conn-a child-uuid "local-title")
(move-block! conn-a
{:block/uuid child-uuid}
{:block/uuid parent-b-uuid})
(sync-until-idle! server [{:repo repo-a :conn conn-a :client client-a :online? true}
{:repo repo-b :conn conn-b :client client-b :online? true}]
50)
(delete-block! conn-b parent-a-uuid)
(sync-until-idle! server [{:repo repo-a :conn conn-a :client client-a :online? true}
{:repo repo-b :conn conn-b :client client-b :online? true}]
50)
(is (not= :frontend.undo-redo/empty-undo-stack
(undo-redo/undo repo-a)))
(let [rounds (sync-until-idle! server [{:repo repo-a :conn conn-a :client client-a :online? true}
{:repo repo-b :conn conn-b :client client-b :online? true}]
50)
child-a (d/entity @conn-a [:block/uuid child-uuid])
child-b (d/entity @conn-b [:block/uuid child-uuid])
attrs-a (block-attr-map @conn-a)
attrs-b (block-attr-map @conn-b)
issues-a (db-issues @conn-a)
issues-b (db-issues @conn-b)]
(is (< rounds 50) (str "sync did not become idle rounds=" rounds))
(is (= "seed-child" (:block/title child-a)))
(is (= "seed-child" (:block/title child-b)))
(is (= parent-b-uuid
(:block/uuid (:block/parent child-a))))
(is (= parent-b-uuid
(:block/uuid (:block/parent child-b))))
(is (empty? issues-a) (str "db A issues " (pr-str issues-a)))
(is (empty? issues-b) (str "db B issues " (pr-str issues-b)))
(assert-synced-attrs! seed history attrs-a attrs-b attrs-b)
(assert-no-invalid-tx! seed history repro))
(finally
(restore)))))))))
(defonce op-runs 200)
(defn- run-random-ops!

31
src/test/frontend/worker/db_sync_test.cljs
View File

@@ -740,6 +740,37 @@
vec)]
(is (empty? sanitized)))))
(deftest apply-remote-batched-create-reuses-tempid-across-batches-test
(testing "a remote block create split across batches should still resolve to one valid block"
(let [{:keys [conn parent]} (setup-parent-child)
parent-uuid (:block/uuid parent)
page-uuid (:block/uuid (:block/page parent))
remote-uuid (random-uuid)
batched-tx-data [[[:db/add -1 :block/uuid remote-uuid]
[:db/add -1 :block/title "remote batched child"]
[:db/add -1 :block/page [:block/uuid page-uuid]]
[:db/add -1 :block/created-at 1760000000000]
[:db/add -1 :block/updated-at 1760000000000]]
[[:db/add -1 :block/parent [:block/uuid parent-uuid]]
[:db/add -1 :block/order "a4"]]]]
(with-datascript-conns conn nil
(fn []
(let [error (try
(#'db-sync/apply-remote-tx! test-repo nil batched-tx-data)
nil
(catch :default e
e))]
(is (nil? error)
(when error
(str (ex-message error) " " (pr-str (ex-data error)))))
(when-not error
(let [block (d/entity @conn [:block/uuid remote-uuid])
validation (db-validate/validate-local-db! @conn)]
(is (= "remote batched child" (:block/title block)))
(is (= (:db/id parent) (:db/id (:block/parent block))))
(is (empty? (map :entity (:errors validation)))
(str (:errors validation)))))))))))
(deftest ^:long sanitize-tx-data-drops-numeric-entity-datoms-for-deleted-block-test
(testing "deleted-block-ids should also drop datoms when entity is numeric id"
(let [{:keys [conn child1]} (setup-parent-child)