From 02beff3ec172c56cc8fea62c3164ec96e6eb9df0 Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Thu, 6 Feb 2025 18:29:30 -0500 Subject: [PATCH] enhance: export+import page handles most ref types in blocks All ref types handled except class ref. Also fixed most block types not persisting uuid when :build-existing-tx? set --- deps/db/src/logseq/db/sqlite/build.cljs | 85 ++++++++++--------- deps/db/src/logseq/db/sqlite/export.cljs | 67 ++++++++++----- deps/db/test/logseq/db/sqlite/build_test.cljs | 9 +- .../db/test/logseq/db/sqlite/export_test.cljs | 51 +++++++++-- 4 files changed, 142 insertions(+), 70 deletions(-) diff --git a/deps/db/src/logseq/db/sqlite/build.cljs b/deps/db/src/logseq/db/sqlite/build.cljs index 362d99851b..6a25661a53 100644 --- a/deps/db/src/logseq/db/sqlite/build.cljs +++ b/deps/db/src/logseq/db/sqlite/build.cljs @@ -121,9 +121,11 @@ [property-map v]))))) (db-property-build/build-property-values-tx-m new-block))) -(defn- extract-content-refs - "Extracts basic refs from :block/title like `[[foo]]`. Adding more ref support would - require parsing each block with mldoc and extracting with text/extract-refs-from-mldoc-ast" +(defn extract-content-refs + "Extracts basic refs from :block/title like `[[foo]]` or `[[UUID]]`. Can't + use db-content/get-matched-ids because of named ref support. Adding more ref + support would require parsing each block with mldoc and extracting with + text/extract-refs-from-mldoc-ast" [s] ;; FIXME: Better way to ignore refs inside a macro (if (string/starts-with? s "{{") @@ -208,10 +210,10 @@ (defn- build-properties-tx [properties page-uuids all-idents {:keys [build-existing-tx?]}] (let [properties' (if build-existing-tx? - (->> properties - (remove #(:block/uuid (val %))) - (into {})) - properties) + (->> properties + (remove (fn [[_ v]] (and (:block/uuid v) (not (:build/new-property? v))))) + (into {})) + properties) property-db-ids (->> (keys properties') (map #(vector % (new-db-id))) (into {})) @@ -222,10 +224,10 @@ (defn- build-classes-tx [classes properties-config uuid-maps all-idents {:keys [build-existing-tx?]}] (let [classes' (if build-existing-tx? - (->> classes - (remove #(:block/uuid (val %))) - (into {})) - classes) + (->> classes + (remove (fn [[_ v]] (and (:block/uuid v) (not (:build/new-class? v))))) + (into {})) + classes) class-db-ids (->> (keys classes') (map #(vector % (new-db-id))) (into {})) @@ -384,6 +386,26 @@ "Class and property db-idents have no overlap") all-idents)) +(defn- build-page-tx [page all-idents page-uuids properties] + (let [page' (dissoc page :build/tags :build/properties) + pvalue-tx-m (->property-value-tx-m page' (:build/properties page) properties all-idents)] + (cond-> [] + (seq pvalue-tx-m) + (into (mapcat #(if (set? %) % [%]) (vals pvalue-tx-m))) + true + (conj + (block-with-timestamps + (merge + page' + (when (seq (:build/properties page)) + (->block-properties (merge (:build/properties page) (db-property-build/build-properties-with-ref-values pvalue-tx-m)) + page-uuids + all-idents)) + (when-let [tags (:build/tags page)] + {:block/tags (-> (mapv #(hash-map :db/ident (get-ident all-idents %)) + tags) + (conj :logseq.class/Page))}))))))) + (defn- build-pages-and-blocks-tx [pages-and-blocks all-idents page-uuids {:keys [page-id-fn properties build-existing-tx?] :or {page-id-fn :db/id} @@ -391,7 +413,11 @@ (vec (mapcat (fn [{:keys [page blocks]}] - (let [page' (if (and build-existing-tx? (not (::new-page? (meta page)))) + (let [;; For a page to be ignored it's important that it only sends a {:block/uuid UUID} map. + ;; This allows import processes to append blocks to an existing page or to create a page + ;; that is referenced by another page + ignore-page-tx? (and build-existing-tx? (not (::new-page? (meta page))) (= '(:block/uuid) (keys page))) + page' (if ignore-page-tx? page (merge ;; TODO: Use sqlite-util/build-new-page @@ -399,33 +425,16 @@ :block/title (or (:block/title page) (string/capitalize (:block/name page))) :block/name (or (:block/name page) (common-util/page-name-sanity-lc (:block/title page))) :block/tags #{:logseq.class/Page}} - (dissoc page :build/properties :db/id :block/name :block/title :build/tags))) + (dissoc page :db/id :block/name :block/title))) page-id-fn' (if (and build-existing-tx? (not (::new-page? (meta page)))) - #(vector :block/uuid (:block/uuid %)) - page-id-fn) + #(vector :block/uuid (:block/uuid %)) + page-id-fn) opts' (assoc opts :existing-page? (and build-existing-tx? (not (::new-page? (meta page)))))] (into ;; page tx - (if (and build-existing-tx? (not (::new-page? (meta page)))) - ;; Ignore existing pages until there's a use case for updating them + (if ignore-page-tx? [] - (let [pvalue-tx-m (->property-value-tx-m page' (:build/properties page) properties all-idents)] - (cond-> [] - (seq pvalue-tx-m) - (into (mapcat #(if (set? %) % [%]) (vals pvalue-tx-m))) - true - (conj - (block-with-timestamps - (merge - page' - (when (seq (:build/properties page)) - (->block-properties (merge (:build/properties page) (db-property-build/build-properties-with-ref-values pvalue-tx-m)) - page-uuids - all-idents)) - (when-let [tags (:build/tags page)] - {:block/tags (-> (mapv #(hash-map :db/ident (get-ident all-idents %)) - tags) - (conj :logseq.class/Page))}))))))) + (build-page-tx page' all-idents page-uuids properties)) ;; blocks tx (reduce (fn [acc m] (into acc @@ -503,10 +512,10 @@ (with-meta block {::existing-block? true}) (assoc block :block/uuid (random-uuid))) block'' (cond-> block' - true - (dissoc :build/children) - parent-id - (assoc :block/parent {:db/id [:block/uuid parent-id]})) + true + (dissoc :build/children) + parent-id + (assoc :block/parent {:db/id [:block/uuid parent-id]})) children (:build/children block) child-maps (when children (expand-build-children children (:block/uuid block'')))] (cons block'' child-maps))) diff --git a/deps/db/src/logseq/db/sqlite/export.cljs b/deps/db/src/logseq/db/sqlite/export.cljs index 62ed3f311d..45f32ce826 100644 --- a/deps/db/src/logseq/db/sqlite/export.cljs +++ b/deps/db/src/logseq/db/sqlite/export.cljs @@ -5,6 +5,7 @@ [datascript.impl.entity :as de] [logseq.db :as ldb] [logseq.db.frontend.class :as db-class] + [logseq.db.frontend.content :as db-content] [logseq.db.frontend.entity-plus :as entity-plus] [logseq.db.frontend.property :as db-property] [logseq.db.sqlite.build :as sqlite-build])) @@ -56,9 +57,9 @@ (into {}))) (defn- build-export-properties - [db user-defined-properties {:keys [include-properties?]}] + [db user-property-idents {:keys [include-properties? include-uuid?]}] (let [properties-config-by-ent - (->> user-defined-properties + (->> user-property-idents (map (fn [ident] (let [property (d/entity db ident) closed-values (entity-plus/lookup-kv-then-entity property :property/closed-values)] @@ -66,6 +67,8 @@ (cond-> (select-keys property (-> (disj db-property/schema-properties :logseq.property/classes) (conj :block/title))) + include-uuid? + (assoc :block/uuid (:block/uuid property)) (:logseq.property/classes property) (assoc :build/property-classes (mapv :db/ident (:logseq.property/classes property))) (seq closed-values) @@ -120,7 +123,7 @@ (defn- build-entity-export "Given entity and optional existing properties, build an EDN export map" - [db entity {:keys [properties include-uuid?]}] + [db entity {:keys [properties include-uuid-fn] :or {include-uuid-fn (constantly false)}}] (let [ent-properties (dissoc (db-property/properties entity) :block/tags) new-user-property-ids (->> (keys ent-properties) (concat (->> (:block/tags entity) @@ -132,7 +135,8 @@ new-properties (build-export-properties db new-user-property-ids {}) build-tags (when (seq (:block/tags entity)) (->build-tags (:block/tags entity))) build-block (cond-> (select-keys entity - (cond-> [:block/title] include-uuid? (conj :block/uuid))) + (cond-> [:block/title] + (include-uuid-fn (:block/uuid entity)) (conj :block/uuid))) (seq build-tags) (assoc :build/tags build-tags) (seq ent-properties) @@ -169,7 +173,7 @@ (defn- build-blocks-tree "Given a page's block entities, returns the blocks in a sqlite.build EDN format and all properties and classes used in these blocks" - [db blocks & {:keys [include-uuid?]}] + [db blocks {:keys [include-uuid-fn]}] (let [*properties (atom {}) *classes (atom {}) *pvalue-uuids (atom #{}) @@ -178,7 +182,7 @@ build-block (fn build-block [block*] (let [child-nodes (mapv build-block (get children (:db/id block*) [])) {:build/keys [block] :keys [properties classes]} - (build-entity-export db block* {:properties @*properties :include-uuid? include-uuid?}) + (build-entity-export db block* {:properties @*properties :include-uuid-fn include-uuid-fn}) new-pvalue-uuids (get-pvalue-uuids block)] (when (seq properties) (swap! *properties merge properties)) (when (seq classes) (swap! *classes merge classes)) @@ -192,6 +196,20 @@ :classes @*classes :pvalue-uuids @*pvalue-uuids})) +(defn- get-uuid-block-pages [db pvalue-uuids content-ref-ents page-entity] + (let [uuid-block-ents-to-export (concat (map #(d/entity db [:block/uuid %]) pvalue-uuids) + (remove ldb/page? content-ref-ents))] + (when (seq uuid-block-ents-to-export) + (->> uuid-block-ents-to-export + (group-by :block/parent) + (map (fn [[parent-page-ent blocks]] + ;; Not a common case but can support later if needed + (when (= parent-page-ent page-entity) + (throw (ex-info "Can't export a uuid block from exported page" {}))) + ;; Don't export pvalue-uuids of pvalue blocks as it's too excessive for now + (merge (build-blocks-tree db (sort-by :block/order blocks) {:include-uuid-fn (constantly true)}) + {:page (select-keys parent-page-ent [:block/title])}))))))) + (defn build-page-export [db eid] (let [page-entity (d/entity db eid) ;; TODO: Fetch unloaded page datoms @@ -202,18 +220,16 @@ (sort-by :block/order) ;; Remove property value blocks as they are included in the block they belong to (remove #(:logseq.property/created-from-property %))) - {:keys [blocks properties classes pvalue-uuids]} (build-blocks-tree db page-blocks) - pvalue-pages (when (seq pvalue-uuids) - (->> pvalue-uuids - (map #(d/entity db [:block/uuid %])) - (group-by :block/parent) - (map (fn [[parent-page-ent blocks]] - ;; Not a common case but can support later if needed - (when (= parent-page-ent page-entity) - (throw (ex-info "Can't export a block object from exported page" {}))) - ;; Don't export pvalue-uuids of pvalue blocks as it's too excessive for now - (merge (build-blocks-tree db (sort-by :block/order blocks) {:include-uuid? true}) - {:page (select-keys parent-page-ent [:block/title])}))))) + content-ref-uuids (set (mapcat (comp db-content/get-matched-ids :block/title) page-blocks)) + content-ref-ents (map #(d/entity db [:block/uuid %]) content-ref-uuids) + content-ref-pages (filter ldb/internal-page? content-ref-ents) + content-ref-properties* (map :db/ident (filter ldb/property? content-ref-ents)) + content-ref-properties (when content-ref-properties* + (update-vals (build-export-properties db content-ref-properties* {:include-uuid? true}) + #(merge % {:build/new-property? true}))) + {:keys [blocks properties classes pvalue-uuids]} + (build-blocks-tree db page-blocks {:include-uuid-fn content-ref-uuids}) + uuid-block-pages (get-uuid-block-pages db pvalue-uuids content-ref-ents page-entity) page-ent-export (build-entity-export db page-entity {:properties properties}) page (merge (dissoc (:build/block page-ent-export) :block/title) (if (ldb/journal? page-entity) @@ -221,10 +237,17 @@ (select-keys page-entity [:block/title]))) pages-and-blocks (cond-> [{:page page :blocks blocks}] - (seq pvalue-pages) - (into (map #(select-keys % [:page :blocks]) pvalue-pages))) - properties' (apply merge properties (:properties page-ent-export) (map :properties pvalue-pages)) - classes' (apply merge classes (:classes page-ent-export) (map :classes pvalue-pages)) + ;; pages from uuid blocks or content-ref-pages are shallow copies e.g. only :block/title + (seq uuid-block-pages) + (into (map #(select-keys % [:page :blocks]) uuid-block-pages)) + (seq content-ref-pages) + (into (map #(hash-map :page (select-keys % [:block/title :block/uuid])) content-ref-pages))) + ;; Use merge-with to preserve new-property? + properties' (apply merge-with merge properties + (:properties page-ent-export) + content-ref-properties + (map :properties uuid-block-pages)) + classes' (apply merge classes (:classes page-ent-export) (map :classes uuid-block-pages)) page-export (cond-> {:pages-and-blocks pages-and-blocks} (seq properties') diff --git a/deps/db/test/logseq/db/sqlite/build_test.cljs b/deps/db/test/logseq/db/sqlite/build_test.cljs index f0317d30c5..51ef3681f5 100644 --- a/deps/db/test/logseq/db/sqlite/build_test.cljs +++ b/deps/db/test/logseq/db/sqlite/build_test.cljs @@ -82,7 +82,7 @@ block2 (db-test/find-block-by-content @conn "block 2") {:keys [init-tx block-props-tx]} (sqlite-build/build-blocks-tx - {:pages-and-blocks [{:page (select-keys (:block/page block) [:block/uuid :block/title]) + {:pages-and-blocks [{:page (select-keys (:block/page block) [:block/uuid]) :blocks [(merge {:block/title "imported task" :block/uuid (:block/uuid block)} {:build/properties {:logseq.task/status :logseq.task/status.todo} :build/tags [:logseq.class/Task]})]}] @@ -92,7 +92,7 @@ updated-block (d/entity @conn [:block/uuid (:block/uuid block)]) {init-tx2 :init-tx block-props-tx2 :block-props-tx :as _tx} (sqlite-build/build-blocks-tx - {:pages-and-blocks [{:page (select-keys (:block/page block2) [:block/uuid :block/title]) + {:pages-and-blocks [{:page (select-keys (:block/page block2) [:block/uuid]) :blocks [(merge {:block/title "imported block" :block/uuid (:block/uuid block2)} {:build/properties {:user.property/p1 "foo"} :build/tags [:user.class/MyClass]})]}] @@ -127,8 +127,9 @@ page-uuid (random-uuid) property-uuid (random-uuid) conn (db-test/create-conn-with-blocks - {:classes {:C1 {:block/uuid class-uuid}} - :properties {:p1 {:block/uuid property-uuid}} + {:classes {:C1 {:block/uuid class-uuid :build/new-class? true}} + :properties {:p1 {:block/uuid property-uuid :build/new-property? true}} + :build-existing-tx? true :pages-and-blocks [{:page {:block/title "page 1"} :blocks [{:block/title "named page ref to [[named page]]"} diff --git a/deps/db/test/logseq/db/sqlite/export_test.cljs b/deps/db/test/logseq/db/sqlite/export_test.cljs index 03bbc1a084..6911a1ae2b 100644 --- a/deps/db/test/logseq/db/sqlite/export_test.cljs +++ b/deps/db/test/logseq/db/sqlite/export_test.cljs @@ -1,6 +1,7 @@ (ns logseq.db.sqlite.export-test (:require [cljs.test :refer [deftest is testing]] [datascript.core :as d] + [logseq.common.util.page-ref :as page-ref] [logseq.db :as ldb] [logseq.db.frontend.property :as db-property] [logseq.db.sqlite.export :as sqlite-export] @@ -103,8 +104,8 @@ (db-test/readable-properties import-block)) "imported block properties equals exported one"))))) -;; Tests a variety of blocks including block children with new properties, blocks with new classes -;; and blocks with built-in properties +;; Tests a variety of blocks including block children with new properties, blocks with users classes +;; and blocks with built-in properties and classes (deftest import-page-with-different-blocks (let [original-data {:properties {:user.property/default {:logseq.property/type :default @@ -135,11 +136,9 @@ {:keys [init-tx block-props-tx] :as _txs} (->> (sqlite-export/build-page-export @conn (:db/id page)) (sqlite-export/build-import @conn2 {})) - _ (assert (nil? (d/entity @conn2 :user.property/default))) - _ (assert (nil? (d/entity @conn2 :user.class/MyClass))) + ;; _ (cljs.pprint/pprint _txs) _ (d/transact! conn2 init-tx) _ (d/transact! conn2 block-props-tx) - ;; _ (cljs.pprint/pprint _txs) page2 (db-test/find-page-by-title @conn2 "page1") full-imported-page (sqlite-export/build-page-export @conn2 (:db/id page2))] @@ -163,6 +162,46 @@ (fn [blocks] (into blocks blocks)))] (is (= expected-page-and-blocks (:pages-and-blocks full-imported-page))))))) +(deftest import-page-with-different-ref-types + (let [block-uuid (random-uuid) + ;; class-uuid (random-uuid) + page-uuid (random-uuid) + property-uuid (random-uuid) + original-data + {;:classes {:C1 {:block/uuid class-uuid}} + :properties {:user.property/p1 + {:db/cardinality :db.cardinality/one, :logseq.property/type :default + :block/uuid property-uuid :block/title "p1" :build/new-property? true}} + :pages-and-blocks + [{:page {:block/title "page1"} + :blocks [{:block/title (str "page ref to " (page-ref/->page-ref page-uuid))} + {:block/title (str "block ref to " (page-ref/->page-ref block-uuid))} + #_{:block/title (str "class ref to " (page-ref/->page-ref class-uuid))} + #_{:block/title (str "inline class ref to #" (page-ref/->page-ref class-uuid))} + {:block/title (str "property ref to " (page-ref/->page-ref property-uuid))}]} + {:page {:block/title "page with block ref"} + :blocks [{:block/title "hi" :block/uuid block-uuid}]} + {:page {:block/title "another page" :block/uuid page-uuid}}]} + conn (db-test/create-conn-with-blocks original-data) + page (db-test/find-page-by-title @conn "page1") + conn2 (db-test/create-conn) + {:keys [init-tx block-props-tx] :as _txs} + (->> (sqlite-export/build-page-export @conn (:db/id page)) + (sqlite-export/build-import @conn2 {})) + ;; _ (cljs.pprint/pprint _txs) + _ (d/transact! conn2 init-tx) + _ (d/transact! conn2 block-props-tx) + page2 (db-test/find-page-by-title @conn2 "page1") + full-imported-page (sqlite-export/build-page-export @conn2 (:db/id page2))] + + (is (= (:properties original-data) (:properties full-imported-page)) + "Page's properties are imported") + (is (= (:classes original-data) (:classes full-imported-page)) + "Page's classes are imported") + ;; (cljs.pprint/pprint (:pages-and-blocks full-imported-page)) + (is (= (:pages-and-blocks original-data) (:pages-and-blocks full-imported-page)) + "Page's blocks are imported"))) + (deftest import-page-with-different-page-and-classes (let [original-data {:properties {:user.property/p1 {:db/cardinality :db.cardinality/one, :logseq.property/type :default, :block/title "p1"} @@ -180,7 +219,7 @@ {:keys [init-tx block-props-tx] :as _txs} (->> (sqlite-export/build-page-export @conn (:db/id page)) (sqlite-export/build-import @conn2 {})) - _ (assert (nil? (d/entity @conn2 :user.property/default))) + _ (assert (nil? (d/entity @conn2 :user.property/p1))) _ (assert (nil? (d/entity @conn2 :user.class/MyClass))) _ (d/transact! conn2 init-tx) _ (d/transact! conn2 block-props-tx)