diff --git a/deps/db/src/logseq/db/sqlite/export.cljs b/deps/db/src/logseq/db/sqlite/export.cljs index 3a4446b0e4..989dc452ef 100644 --- a/deps/db/src/logseq/db/sqlite/export.cljs +++ b/deps/db/src/logseq/db/sqlite/export.cljs @@ -177,14 +177,53 @@ (if (set? val-or-vals) val-or-vals [val-or-vals])))) set)) +(defn- merge-export-maps [& export-maps] + (let [pages-and-blocks (reduce into [] (keep :pages-and-blocks export-maps)) + ;; Use merge-with to preserve new-property? + properties (apply merge-with merge (keep :properties export-maps)) + classes (apply merge-with merge (keep :classes export-maps))] + (cond-> {:pages-and-blocks pages-and-blocks} + (seq properties) + (assoc :properties properties) + (seq classes) + (assoc :classes classes)))) + +(defn- build-content-ref-export + "Builds an export config (and additional info) for refs in the given blocks. All the exported + entities found in block refs include their uuid in order to preserve the relationship to the blocks" + [db page-blocks] + (let [content-ref-uuids (set (mapcat (comp db-content/get-matched-ids block-title) page-blocks)) + content-ref-ents (map #(d/entity db [:block/uuid %]) content-ref-uuids) + content-ref-pages (filter #(or (ldb/internal-page? %) (ldb/journal? %)) content-ref-ents) + content-ref-properties (when-let [prop-ids (seq (map :db/ident (filter ldb/property? content-ref-ents)))] + (update-vals (build-export-properties db prop-ids {:include-uuid? true}) + #(merge % {:build/new-property? true}))) + content-ref-classes (when-let [class-ents (seq (filter ldb/class? content-ref-ents))] + (->> class-ents + ;; TODO: Export class parents when there's ability to control granularity of export + (map #(vector (:db/ident %) + (assoc (build-export-class % {:include-parents? false :include-uuid? true}) + :build/new-class? true))) + (into {})))] + {:content-ref-uuids content-ref-uuids + :content-ref-ents content-ref-ents + :properties content-ref-properties + :classes content-ref-classes + :pages-and-blocks (mapv #(hash-map :page (assoc (shallow-copy-page %) :block/uuid (:block/uuid %))) + content-ref-pages)})) + (defn build-block-export [db eid] - (let [export-map (build-entity-export db (d/entity db eid) {}) - pvalue-uuids (get-pvalue-uuids (:build/block export-map))] + (let [block-entity (d/entity db eid) + {:keys [content-ref-uuids _content-ref-ents] :as content-ref-export} (build-content-ref-export db [block-entity]) + block-export* (build-entity-export db block-entity {:include-uuid-fn content-ref-uuids}) + pvalue-uuids (get-pvalue-uuids (:build/block block-export*)) + block-export (assoc (merge-export-maps block-export* content-ref-export) + :build/block (:build/block block-export*))] ;; Maybe add support for this later (when (seq pvalue-uuids) (throw (ex-info "Exporting a block with :node block objects is not supported" {}))) - export-map)) + block-export)) (defn- build-blocks-tree "Given a page's block entities, returns the blocks in a sqlite.build EDN format @@ -230,30 +269,6 @@ :classes (apply merge (map :classes uuid-block-pages)) :pages-and-blocks (mapv #(select-keys % [:page :blocks]) uuid-block-pages)})) -(defn- build-content-ref-export - "Builds an export config (and additional info) for refs in the given blocks. All the exported - entities found in block refs include their uuid in order to preserve the relationship to the blocks" - [db page-blocks] - (let [content-ref-uuids (set (mapcat (comp db-content/get-matched-ids block-title) page-blocks)) - content-ref-ents (map #(d/entity db [:block/uuid %]) content-ref-uuids) - content-ref-pages (filter #(or (ldb/internal-page? %) (ldb/journal? %)) content-ref-ents) - content-ref-properties (when-let [prop-ids (seq (map :db/ident (filter ldb/property? content-ref-ents)))] - (update-vals (build-export-properties db prop-ids {:include-uuid? true}) - #(merge % {:build/new-property? true}))) - content-ref-classes (when-let [class-ents (seq (filter ldb/class? content-ref-ents))] - (->> class-ents - ;; TODO: Export class parents when there's ability to control granularity of export - (map #(vector (:db/ident %) - (assoc (build-export-class % {:include-parents? false :include-uuid? true}) - :build/new-class? true))) - (into {})))] - {:content-ref-uuids content-ref-uuids - :content-ref-ents content-ref-ents - :properties content-ref-properties - :classes content-ref-classes - :pages-and-blocks (mapv #(hash-map :page (assoc (shallow-copy-page %) :block/uuid (:block/uuid %))) - content-ref-pages)})) - (defn build-page-export [db eid] (let [page-entity (d/entity db eid) ;; TODO: Fetch unloaded page datoms @@ -271,27 +286,10 @@ page-ent-export (build-entity-export db page-entity {:properties properties}) page (merge (dissoc (:build/block page-ent-export) :block/title) (shallow-copy-page page-entity)) - pages-and-blocks - (cond-> [{:page page :blocks blocks}] - (seq (:pages-and-blocks uuid-block-export)) - (into (:pages-and-blocks uuid-block-export)) - (seq (:pages-and-blocks content-ref-export)) - (into (:pages-and-blocks content-ref-export))) - ;; Use merge-with to preserve new-property? - properties' (merge-with merge properties - (:properties page-ent-export) - (:properties content-ref-export) - (:properties uuid-block-export)) - classes' (merge-with merge classes - (:classes page-ent-export) - (:classes content-ref-export) - (:classes uuid-block-export)) - page-export - (cond-> {:pages-and-blocks pages-and-blocks} - (seq properties') - (assoc :properties properties') - (seq classes') - (assoc :classes classes'))] + page-blocks-export {:pages-and-blocks [{:page page :blocks blocks}] + :properties properties + :classes classes} + page-export (merge-export-maps page-blocks-export page-ent-export uuid-block-export content-ref-export)] page-export)) (defn build-graph-ontology-export @@ -356,16 +354,13 @@ (defn- build-block-import-options "Builds options for sqlite-build to import into current-block" [current-block export-map] - (let [{:build/keys [block]} - (merge-with merge - export-map - {:build/block + (let [block (merge (:build/block export-map) {:block/uuid (:block/uuid current-block) - :block/page (select-keys (:block/page current-block) [:block/uuid])}}) + :block/page (select-keys (:block/page current-block) [:block/uuid])}) pages-and-blocks [{:page (select-keys (:block/page block) [:block/uuid]) :blocks [(dissoc block :block/page)]}]] - (assoc export-map :pages-and-blocks pages-and-blocks))) + (merge-export-maps export-map {:pages-and-blocks pages-and-blocks}))) (defn- build-page-import-options [db export-map] diff --git a/deps/db/test/logseq/db/sqlite/export_test.cljs b/deps/db/test/logseq/db/sqlite/export_test.cljs index f89bd82e06..24591a521f 100644 --- a/deps/db/test/logseq/db/sqlite/export_test.cljs +++ b/deps/db/test/logseq/db/sqlite/export_test.cljs @@ -2,107 +2,103 @@ (:require [cljs.test :refer [deftest is testing]] [datascript.core :as d] [logseq.common.util.page-ref :as page-ref] - [logseq.db :as ldb] - [logseq.db.frontend.property :as db-property] [logseq.db.sqlite.export :as sqlite-export] [logseq.db.test.helper :as db-test])) -(deftest import-block-in-same-graph - (let [conn (db-test/create-conn-with-blocks - {:properties {:default-many {:logseq.property/type :default :db/cardinality :many}} - :classes {:MyClass {:build/class-properties [:default-many]}} - :pages-and-blocks - [{:page {:block/title "page1"} - :blocks [{:block/title "export" - :build/properties {:default-many #{"foo" "bar" "baz"}} - :build/tags [:MyClass]} - {:block/title "import"}]}]}) - export-block (db-test/find-block-by-content @conn "export") - import-block* (db-test/find-block-by-content @conn "import") - {:keys [init-tx block-props-tx]} - (->> (sqlite-export/build-block-export @conn [:block/uuid (:block/uuid export-block)]) - (sqlite-export/build-import @conn {:current-block import-block*})) - _ (assert (empty? block-props-tx) "This is empty for properties that already exist and thus no transacted") - _ (d/transact! conn init-tx) - import-block (d/entity @conn (:db/id import-block*))] - (is (= [] - (filter #(or (:db/id %) (:db/ident %)) init-tx)) - "Tx doesn't try to create new blocks or modify existing idents") +(defn- export-block-and-import-to-another-block + "Exports given block from one graph/conn, imports it to a 2nd block and then + exports the 2nd block. The two blocks do not have to be in the same graph" + [export-conn import-conn export-block-content import-block-content] + (let [export-block (db-test/find-block-by-content @export-conn export-block-content) + import-block (db-test/find-block-by-content @import-conn import-block-content) + {:keys [init-tx block-props-tx] :as _txs} + (->> (sqlite-export/build-block-export @export-conn [:block/uuid (:block/uuid export-block)]) + (sqlite-export/build-import @import-conn {:current-block import-block})) + ;; _ (cljs.pprint/pprint _txs) + _ (d/transact! import-conn init-tx) + _ (d/transact! import-conn block-props-tx)] + (sqlite-export/build-block-export @import-conn (:db/id import-block)))) - (is (= "export" (:block/title import-block)) - "imported block title equals exported one") - (is (= {:user.property/default-many #{"foo" "bar" "baz"} - :block/tags [:user.class/MyClass]} - (db-test/readable-properties import-block)) - "imported block properties and tags equals exported one"))) +(deftest import-block-in-same-graph + (let [original-data + {:properties {:user.property/default-many + {:block/title "default-many" :logseq.property/type :default :db/cardinality :db.cardinality/many}} + :classes {:user.class/MyClass + {:block/title "MyClass" :build/class-properties [:user.property/default-many]}} + :pages-and-blocks + [{:page {:block/title "page1"} + :blocks [{:block/title "export" + :build/properties {:user.property/default-many #{"foo" "bar" "baz"}} + :build/tags [:user.class/MyClass]} + {:block/title "import"}]}]} + conn (db-test/create-conn-with-blocks original-data) + imported-block (export-block-and-import-to-another-block conn conn "export" "import")] + + (is (= (get-in original-data [:pages-and-blocks 0 :blocks 0]) + (:build/block imported-block)) + "Imported block equals exported block") + (is (= (:properties original-data) (:properties imported-block))) + (is (= (:classes original-data) (:classes imported-block))))) (deftest import-block-in-different-graph - (let [conn (db-test/create-conn-with-blocks - {:properties {:num-many {:logseq.property/type :number - :db/cardinality :many - :block/title "Num Many" - :logseq.property/hide? true}} - :classes {:MyClass {:block/title "My Class" - :build/class-properties [:default-many :p1]}} - :pages-and-blocks - [{:page {:block/title "page1"} - :blocks [{:block/title "export" - :build/properties {:num-many #{3 6 9}} - :build/tags [:MyClass]}]}]}) + (let [original-data + {:properties {:user.property/num-many + {:logseq.property/type :number + :db/cardinality :db.cardinality/many + :block/title "Num Many" + :logseq.property/hide? true} + :user.property/p1 + {:db/cardinality :db.cardinality/one, + :logseq.property/type :default, + :block/title "p1"}} + :classes {:user.class/MyClass + {:block/title "My Class" + :build/class-properties [:user.property/num-many :user.property/p1]}} + :pages-and-blocks + [{:page {:block/title "page1"} + :blocks [{:block/title "export" + :build/properties {:user.property/num-many #{3 6 9}} + :build/tags [:user.class/MyClass]}]}]} + conn (db-test/create-conn-with-blocks original-data) conn2 (db-test/create-conn-with-blocks {:pages-and-blocks [{:page {:block/title "page2"} :blocks [{:block/title "import"} {:block/title "import2"}]}]}) - export-block (db-test/find-block-by-content @conn "export") - import-block* (db-test/find-block-by-content @conn2 "import") - {:keys [init-tx block-props-tx] :as _txs} - (->> (sqlite-export/build-block-export @conn [:block/uuid (:block/uuid export-block)]) - (sqlite-export/build-import @conn2 {:current-block import-block*})) - _ (assert (nil? (d/entity @conn2 :user.property/num-many)) "Does not have imported property") - _ (d/transact! conn2 init-tx) - _ (d/transact! conn2 block-props-tx) - ;; _ (cljs.pprint/pprint _txs) - import-block (d/entity @conn2 (:db/id import-block*))] + imported-block (export-block-and-import-to-another-block conn conn2 "export" "import")] - (is (ldb/property? (d/entity @conn2 :user.property/num-many)) - "New user property is imported") - (is (= "Num Many" - (:block/title (d/entity @conn2 :user.property/num-many)))) - (is (= {:db/cardinality :db.cardinality/many, :logseq.property/type :number, :logseq.property/hide? true} - (db-property/get-property-schema (d/entity @conn2 :user.property/num-many))) - "Imported property has correct schema properties") + (is (= (get-in original-data [:pages-and-blocks 0 :blocks 0]) + (:build/block imported-block)) + "Imported block equals exported block") + (is (= (:properties original-data) (:properties imported-block))) + (is (= (:classes original-data) (:classes imported-block))) - (is (= "My Class" - (:block/title (d/entity @conn2 :user.class/MyClass)))) - (is (= {:logseq.property.class/properties #{"default-many" "p1"} - :block/tags [:logseq.class/Tag] - :logseq.property/parent :logseq.class/Root} - (db-test/readable-properties (d/entity @conn2 :user.class/MyClass))) - "New user class has correct tag and properties") - (is (ldb/property? (d/entity @conn2 :user.property/p1)) - "New class property is property") + (testing "same import in another block" + (let [imported-block (export-block-and-import-to-another-block conn conn2 "export" "import2")] + (is (= (get-in original-data [:pages-and-blocks 0 :blocks 0]) + (:build/block imported-block)) + "Imported block equals exported block") + (is (= (:properties original-data) (:properties imported-block))) + (is (= (:classes original-data) (:classes imported-block))))))) - (is (= "export" (:block/title import-block)) - "imported block title equals exported one") - (is (= {:user.property/num-many #{3 6 9} - :block/tags [:user.class/MyClass]} - (db-test/readable-properties import-block)) - "imported block properties equals exported one") +(deftest import-block-with-block-ref + (let [page-uuid (random-uuid) + original-data + {:pages-and-blocks + [{:page {:block/title "page1"} + :blocks [{:block/title (str "page ref to " (page-ref/->page-ref page-uuid))}]} + {:page {:block/title "another page" :block/uuid page-uuid}}]} + conn (db-test/create-conn-with-blocks original-data) + conn2 (db-test/create-conn-with-blocks + {:pages-and-blocks [{:page {:block/title "page2"} + :blocks [{:block/title "import"}]}]}) + imported-block (export-block-and-import-to-another-block conn conn2 #"page ref" "import")] - (testing "importing a 2nd time is idempotent" - (let [import-block2* (db-test/find-block-by-content @conn2 "import2") - {:keys [init-tx block-props-tx] :as _txs} - (->> (sqlite-export/build-block-export @conn [:block/uuid (:block/uuid export-block)]) - (sqlite-export/build-import @conn2 {:current-block import-block2*})) - _ (assert (empty? block-props-tx) "This is empty for properties that already exist and thus no transacted") - _ (d/transact! conn2 init-tx) - import-block2 (d/entity @conn2 (:db/id import-block2*))] - (is (= "export" (:block/title import-block2)) - "imported block title equals exported one") - (is (= {:user.property/num-many #{3 6 9} - :block/tags [:user.class/MyClass]} - (db-test/readable-properties import-block)) - "imported block properties equals exported one"))))) + (is (= (get-in original-data [:pages-and-blocks 0 :blocks 0]) + (:build/block imported-block)) + "Imported block equals exported block") + (is (= (second (:pages-and-blocks original-data)) + (first (:pages-and-blocks imported-block))) + "Imported page equals exported page of page ref"))) (defn- export-page-and-import-to-another-graph "Exports given page from one graph/conn, imports it to a 2nd graph and then @@ -165,7 +161,7 @@ (import-second-time-appends-blocks conn conn2 "page1" original-data))) -(deftest ^:focus import-page-with-different-ref-types +(deftest import-page-with-different-ref-types (let [block-uuid (random-uuid) class-uuid (random-uuid) page-uuid (random-uuid) @@ -196,7 +192,6 @@ "Page's properties are imported") (is (= (:classes original-data) (:classes full-imported-page)) "Page's classes are imported") - ;; (cljs.pprint/pprint (:pages-and-blocks full-imported-page)) (is (= (:pages-and-blocks original-data) (:pages-and-blocks full-imported-page)) "Page's blocks are imported")