mirror of
https://github.com/logseq/logseq.git
synced 2026-05-21 03:12:38 +00:00
enhance: block export supports content refs
Also refactor block tests and how export-maps are built and
This commit is contained in:
Gabriel Horner
103
deps/db/src/logseq/db/sqlite/export.cljs
vendored
103
deps/db/src/logseq/db/sqlite/export.cljs
vendored
@@ -177,14 +177,53 @@
|
||||
(if (set? val-or-vals) val-or-vals [val-or-vals]))))
|
||||
set))
|
||||
|
||||
(defn- merge-export-maps [& export-maps]
|
||||
(let [pages-and-blocks (reduce into [] (keep :pages-and-blocks export-maps))
|
||||
;; Use merge-with to preserve new-property?
|
||||
properties (apply merge-with merge (keep :properties export-maps))
|
||||
classes (apply merge-with merge (keep :classes export-maps))]
|
||||
(cond-> {:pages-and-blocks pages-and-blocks}
|
||||
(seq properties)
|
||||
(assoc :properties properties)
|
||||
(seq classes)
|
||||
(assoc :classes classes))))
|
||||
|
||||
(defn- build-content-ref-export
|
||||
"Builds an export config (and additional info) for refs in the given blocks. All the exported
|
||||
entities found in block refs include their uuid in order to preserve the relationship to the blocks"
|
||||
[db page-blocks]
|
||||
(let [content-ref-uuids (set (mapcat (comp db-content/get-matched-ids block-title) page-blocks))
|
||||
content-ref-ents (map #(d/entity db [:block/uuid %]) content-ref-uuids)
|
||||
content-ref-pages (filter #(or (ldb/internal-page? %) (ldb/journal? %)) content-ref-ents)
|
||||
content-ref-properties (when-let [prop-ids (seq (map :db/ident (filter ldb/property? content-ref-ents)))]
|
||||
(update-vals (build-export-properties db prop-ids {:include-uuid? true})
|
||||
#(merge % {:build/new-property? true})))
|
||||
content-ref-classes (when-let [class-ents (seq (filter ldb/class? content-ref-ents))]
|
||||
(->> class-ents
|
||||
;; TODO: Export class parents when there's ability to control granularity of export
|
||||
(map #(vector (:db/ident %)
|
||||
(assoc (build-export-class % {:include-parents? false :include-uuid? true})
|
||||
:build/new-class? true)))
|
||||
(into {})))]
|
||||
{:content-ref-uuids content-ref-uuids
|
||||
:content-ref-ents content-ref-ents
|
||||
:properties content-ref-properties
|
||||
:classes content-ref-classes
|
||||
:pages-and-blocks (mapv #(hash-map :page (assoc (shallow-copy-page %) :block/uuid (:block/uuid %)))
|
||||
content-ref-pages)}))
|
||||
|
||||
(defn build-block-export
|
||||
[db eid]
|
||||
(let [export-map (build-entity-export db (d/entity db eid) {})
|
||||
pvalue-uuids (get-pvalue-uuids (:build/block export-map))]
|
||||
(let [block-entity (d/entity db eid)
|
||||
{:keys [content-ref-uuids _content-ref-ents] :as content-ref-export} (build-content-ref-export db [block-entity])
|
||||
block-export* (build-entity-export db block-entity {:include-uuid-fn content-ref-uuids})
|
||||
pvalue-uuids (get-pvalue-uuids (:build/block block-export*))
|
||||
block-export (assoc (merge-export-maps block-export* content-ref-export)
|
||||
:build/block (:build/block block-export*))]
|
||||
;; Maybe add support for this later
|
||||
(when (seq pvalue-uuids)
|
||||
(throw (ex-info "Exporting a block with :node block objects is not supported" {})))
|
||||
export-map))
|
||||
block-export))
|
||||
|
||||
(defn- build-blocks-tree
|
||||
"Given a page's block entities, returns the blocks in a sqlite.build EDN format
|
||||
@@ -230,30 +269,6 @@
|
||||
:classes (apply merge (map :classes uuid-block-pages))
|
||||
:pages-and-blocks (mapv #(select-keys % [:page :blocks]) uuid-block-pages)}))
|
||||
|
||||
(defn- build-content-ref-export
|
||||
"Builds an export config (and additional info) for refs in the given blocks. All the exported
|
||||
entities found in block refs include their uuid in order to preserve the relationship to the blocks"
|
||||
[db page-blocks]
|
||||
(let [content-ref-uuids (set (mapcat (comp db-content/get-matched-ids block-title) page-blocks))
|
||||
content-ref-ents (map #(d/entity db [:block/uuid %]) content-ref-uuids)
|
||||
content-ref-pages (filter #(or (ldb/internal-page? %) (ldb/journal? %)) content-ref-ents)
|
||||
content-ref-properties (when-let [prop-ids (seq (map :db/ident (filter ldb/property? content-ref-ents)))]
|
||||
(update-vals (build-export-properties db prop-ids {:include-uuid? true})
|
||||
#(merge % {:build/new-property? true})))
|
||||
content-ref-classes (when-let [class-ents (seq (filter ldb/class? content-ref-ents))]
|
||||
(->> class-ents
|
||||
;; TODO: Export class parents when there's ability to control granularity of export
|
||||
(map #(vector (:db/ident %)
|
||||
(assoc (build-export-class % {:include-parents? false :include-uuid? true})
|
||||
:build/new-class? true)))
|
||||
(into {})))]
|
||||
{:content-ref-uuids content-ref-uuids
|
||||
:content-ref-ents content-ref-ents
|
||||
:properties content-ref-properties
|
||||
:classes content-ref-classes
|
||||
:pages-and-blocks (mapv #(hash-map :page (assoc (shallow-copy-page %) :block/uuid (:block/uuid %)))
|
||||
content-ref-pages)}))
|
||||
|
||||
(defn build-page-export [db eid]
|
||||
(let [page-entity (d/entity db eid)
|
||||
;; TODO: Fetch unloaded page datoms
|
||||
@@ -271,27 +286,10 @@
|
||||
page-ent-export (build-entity-export db page-entity {:properties properties})
|
||||
page (merge (dissoc (:build/block page-ent-export) :block/title)
|
||||
(shallow-copy-page page-entity))
|
||||
pages-and-blocks
|
||||
(cond-> [{:page page :blocks blocks}]
|
||||
(seq (:pages-and-blocks uuid-block-export))
|
||||
(into (:pages-and-blocks uuid-block-export))
|
||||
(seq (:pages-and-blocks content-ref-export))
|
||||
(into (:pages-and-blocks content-ref-export)))
|
||||
;; Use merge-with to preserve new-property?
|
||||
properties' (merge-with merge properties
|
||||
(:properties page-ent-export)
|
||||
(:properties content-ref-export)
|
||||
(:properties uuid-block-export))
|
||||
classes' (merge-with merge classes
|
||||
(:classes page-ent-export)
|
||||
(:classes content-ref-export)
|
||||
(:classes uuid-block-export))
|
||||
page-export
|
||||
(cond-> {:pages-and-blocks pages-and-blocks}
|
||||
(seq properties')
|
||||
(assoc :properties properties')
|
||||
(seq classes')
|
||||
(assoc :classes classes'))]
|
||||
page-blocks-export {:pages-and-blocks [{:page page :blocks blocks}]
|
||||
:properties properties
|
||||
:classes classes}
|
||||
page-export (merge-export-maps page-blocks-export page-ent-export uuid-block-export content-ref-export)]
|
||||
page-export))
|
||||
|
||||
(defn build-graph-ontology-export
|
||||
@@ -356,16 +354,13 @@
|
||||
(defn- build-block-import-options
|
||||
"Builds options for sqlite-build to import into current-block"
|
||||
[current-block export-map]
|
||||
(let [{:build/keys [block]}
|
||||
(merge-with merge
|
||||
export-map
|
||||
{:build/block
|
||||
(let [block (merge (:build/block export-map)
|
||||
{:block/uuid (:block/uuid current-block)
|
||||
:block/page (select-keys (:block/page current-block) [:block/uuid])}})
|
||||
:block/page (select-keys (:block/page current-block) [:block/uuid])})
|
||||
pages-and-blocks
|
||||
[{:page (select-keys (:block/page block) [:block/uuid])
|
||||
:blocks [(dissoc block :block/page)]}]]
|
||||
(assoc export-map :pages-and-blocks pages-and-blocks)))
|
||||
(merge-export-maps export-map {:pages-and-blocks pages-and-blocks})))
|
||||
|
||||
(defn- build-page-import-options
|
||||
[db export-map]
|
||||
|
||||
173
deps/db/test/logseq/db/sqlite/export_test.cljs
vendored
173
deps/db/test/logseq/db/sqlite/export_test.cljs
vendored
@@ -2,107 +2,103 @@
|
||||
(:require [cljs.test :refer [deftest is testing]]
|
||||
[datascript.core :as d]
|
||||
[logseq.common.util.page-ref :as page-ref]
|
||||
[logseq.db :as ldb]
|
||||
[logseq.db.frontend.property :as db-property]
|
||||
[logseq.db.sqlite.export :as sqlite-export]
|
||||
[logseq.db.test.helper :as db-test]))
|
||||
|
||||
(deftest import-block-in-same-graph
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:properties {:default-many {:logseq.property/type :default :db/cardinality :many}}
|
||||
:classes {:MyClass {:build/class-properties [:default-many]}}
|
||||
:pages-and-blocks
|
||||
[{:page {:block/title "page1"}
|
||||
:blocks [{:block/title "export"
|
||||
:build/properties {:default-many #{"foo" "bar" "baz"}}
|
||||
:build/tags [:MyClass]}
|
||||
{:block/title "import"}]}]})
|
||||
export-block (db-test/find-block-by-content @conn "export")
|
||||
import-block* (db-test/find-block-by-content @conn "import")
|
||||
{:keys [init-tx block-props-tx]}
|
||||
(->> (sqlite-export/build-block-export @conn [:block/uuid (:block/uuid export-block)])
|
||||
(sqlite-export/build-import @conn {:current-block import-block*}))
|
||||
_ (assert (empty? block-props-tx) "This is empty for properties that already exist and thus no transacted")
|
||||
_ (d/transact! conn init-tx)
|
||||
import-block (d/entity @conn (:db/id import-block*))]
|
||||
(is (= []
|
||||
(filter #(or (:db/id %) (:db/ident %)) init-tx))
|
||||
"Tx doesn't try to create new blocks or modify existing idents")
|
||||
(defn- export-block-and-import-to-another-block
|
||||
"Exports given block from one graph/conn, imports it to a 2nd block and then
|
||||
exports the 2nd block. The two blocks do not have to be in the same graph"
|
||||
[export-conn import-conn export-block-content import-block-content]
|
||||
(let [export-block (db-test/find-block-by-content @export-conn export-block-content)
|
||||
import-block (db-test/find-block-by-content @import-conn import-block-content)
|
||||
{:keys [init-tx block-props-tx] :as _txs}
|
||||
(->> (sqlite-export/build-block-export @export-conn [:block/uuid (:block/uuid export-block)])
|
||||
(sqlite-export/build-import @import-conn {:current-block import-block}))
|
||||
;; _ (cljs.pprint/pprint _txs)
|
||||
_ (d/transact! import-conn init-tx)
|
||||
_ (d/transact! import-conn block-props-tx)]
|
||||
(sqlite-export/build-block-export @import-conn (:db/id import-block))))
|
||||
|
||||
(is (= "export" (:block/title import-block))
|
||||
"imported block title equals exported one")
|
||||
(is (= {:user.property/default-many #{"foo" "bar" "baz"}
|
||||
:block/tags [:user.class/MyClass]}
|
||||
(db-test/readable-properties import-block))
|
||||
"imported block properties and tags equals exported one")))
|
||||
(deftest import-block-in-same-graph
|
||||
(let [original-data
|
||||
{:properties {:user.property/default-many
|
||||
{:block/title "default-many" :logseq.property/type :default :db/cardinality :db.cardinality/many}}
|
||||
:classes {:user.class/MyClass
|
||||
{:block/title "MyClass" :build/class-properties [:user.property/default-many]}}
|
||||
:pages-and-blocks
|
||||
[{:page {:block/title "page1"}
|
||||
:blocks [{:block/title "export"
|
||||
:build/properties {:user.property/default-many #{"foo" "bar" "baz"}}
|
||||
:build/tags [:user.class/MyClass]}
|
||||
{:block/title "import"}]}]}
|
||||
conn (db-test/create-conn-with-blocks original-data)
|
||||
imported-block (export-block-and-import-to-another-block conn conn "export" "import")]
|
||||
|
||||
(is (= (get-in original-data [:pages-and-blocks 0 :blocks 0])
|
||||
(:build/block imported-block))
|
||||
"Imported block equals exported block")
|
||||
(is (= (:properties original-data) (:properties imported-block)))
|
||||
(is (= (:classes original-data) (:classes imported-block)))))
|
||||
|
||||
(deftest import-block-in-different-graph
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:properties {:num-many {:logseq.property/type :number
|
||||
:db/cardinality :many
|
||||
:block/title "Num Many"
|
||||
:logseq.property/hide? true}}
|
||||
:classes {:MyClass {:block/title "My Class"
|
||||
:build/class-properties [:default-many :p1]}}
|
||||
:pages-and-blocks
|
||||
[{:page {:block/title "page1"}
|
||||
:blocks [{:block/title "export"
|
||||
:build/properties {:num-many #{3 6 9}}
|
||||
:build/tags [:MyClass]}]}]})
|
||||
(let [original-data
|
||||
{:properties {:user.property/num-many
|
||||
{:logseq.property/type :number
|
||||
:db/cardinality :db.cardinality/many
|
||||
:block/title "Num Many"
|
||||
:logseq.property/hide? true}
|
||||
:user.property/p1
|
||||
{:db/cardinality :db.cardinality/one,
|
||||
:logseq.property/type :default,
|
||||
:block/title "p1"}}
|
||||
:classes {:user.class/MyClass
|
||||
{:block/title "My Class"
|
||||
:build/class-properties [:user.property/num-many :user.property/p1]}}
|
||||
:pages-and-blocks
|
||||
[{:page {:block/title "page1"}
|
||||
:blocks [{:block/title "export"
|
||||
:build/properties {:user.property/num-many #{3 6 9}}
|
||||
:build/tags [:user.class/MyClass]}]}]}
|
||||
conn (db-test/create-conn-with-blocks original-data)
|
||||
conn2 (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks [{:page {:block/title "page2"}
|
||||
:blocks [{:block/title "import"}
|
||||
{:block/title "import2"}]}]})
|
||||
export-block (db-test/find-block-by-content @conn "export")
|
||||
import-block* (db-test/find-block-by-content @conn2 "import")
|
||||
{:keys [init-tx block-props-tx] :as _txs}
|
||||
(->> (sqlite-export/build-block-export @conn [:block/uuid (:block/uuid export-block)])
|
||||
(sqlite-export/build-import @conn2 {:current-block import-block*}))
|
||||
_ (assert (nil? (d/entity @conn2 :user.property/num-many)) "Does not have imported property")
|
||||
_ (d/transact! conn2 init-tx)
|
||||
_ (d/transact! conn2 block-props-tx)
|
||||
;; _ (cljs.pprint/pprint _txs)
|
||||
import-block (d/entity @conn2 (:db/id import-block*))]
|
||||
imported-block (export-block-and-import-to-another-block conn conn2 "export" "import")]
|
||||
|
||||
(is (ldb/property? (d/entity @conn2 :user.property/num-many))
|
||||
"New user property is imported")
|
||||
(is (= "Num Many"
|
||||
(:block/title (d/entity @conn2 :user.property/num-many))))
|
||||
(is (= {:db/cardinality :db.cardinality/many, :logseq.property/type :number, :logseq.property/hide? true}
|
||||
(db-property/get-property-schema (d/entity @conn2 :user.property/num-many)))
|
||||
"Imported property has correct schema properties")
|
||||
(is (= (get-in original-data [:pages-and-blocks 0 :blocks 0])
|
||||
(:build/block imported-block))
|
||||
"Imported block equals exported block")
|
||||
(is (= (:properties original-data) (:properties imported-block)))
|
||||
(is (= (:classes original-data) (:classes imported-block)))
|
||||
|
||||
(is (= "My Class"
|
||||
(:block/title (d/entity @conn2 :user.class/MyClass))))
|
||||
(is (= {:logseq.property.class/properties #{"default-many" "p1"}
|
||||
:block/tags [:logseq.class/Tag]
|
||||
:logseq.property/parent :logseq.class/Root}
|
||||
(db-test/readable-properties (d/entity @conn2 :user.class/MyClass)))
|
||||
"New user class has correct tag and properties")
|
||||
(is (ldb/property? (d/entity @conn2 :user.property/p1))
|
||||
"New class property is property")
|
||||
(testing "same import in another block"
|
||||
(let [imported-block (export-block-and-import-to-another-block conn conn2 "export" "import2")]
|
||||
(is (= (get-in original-data [:pages-and-blocks 0 :blocks 0])
|
||||
(:build/block imported-block))
|
||||
"Imported block equals exported block")
|
||||
(is (= (:properties original-data) (:properties imported-block)))
|
||||
(is (= (:classes original-data) (:classes imported-block)))))))
|
||||
|
||||
(is (= "export" (:block/title import-block))
|
||||
"imported block title equals exported one")
|
||||
(is (= {:user.property/num-many #{3 6 9}
|
||||
:block/tags [:user.class/MyClass]}
|
||||
(db-test/readable-properties import-block))
|
||||
"imported block properties equals exported one")
|
||||
(deftest import-block-with-block-ref
|
||||
(let [page-uuid (random-uuid)
|
||||
original-data
|
||||
{:pages-and-blocks
|
||||
[{:page {:block/title "page1"}
|
||||
:blocks [{:block/title (str "page ref to " (page-ref/->page-ref page-uuid))}]}
|
||||
{:page {:block/title "another page" :block/uuid page-uuid}}]}
|
||||
conn (db-test/create-conn-with-blocks original-data)
|
||||
conn2 (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks [{:page {:block/title "page2"}
|
||||
:blocks [{:block/title "import"}]}]})
|
||||
imported-block (export-block-and-import-to-another-block conn conn2 #"page ref" "import")]
|
||||
|
||||
(testing "importing a 2nd time is idempotent"
|
||||
(let [import-block2* (db-test/find-block-by-content @conn2 "import2")
|
||||
{:keys [init-tx block-props-tx] :as _txs}
|
||||
(->> (sqlite-export/build-block-export @conn [:block/uuid (:block/uuid export-block)])
|
||||
(sqlite-export/build-import @conn2 {:current-block import-block2*}))
|
||||
_ (assert (empty? block-props-tx) "This is empty for properties that already exist and thus no transacted")
|
||||
_ (d/transact! conn2 init-tx)
|
||||
import-block2 (d/entity @conn2 (:db/id import-block2*))]
|
||||
(is (= "export" (:block/title import-block2))
|
||||
"imported block title equals exported one")
|
||||
(is (= {:user.property/num-many #{3 6 9}
|
||||
:block/tags [:user.class/MyClass]}
|
||||
(db-test/readable-properties import-block))
|
||||
"imported block properties equals exported one")))))
|
||||
(is (= (get-in original-data [:pages-and-blocks 0 :blocks 0])
|
||||
(:build/block imported-block))
|
||||
"Imported block equals exported block")
|
||||
(is (= (second (:pages-and-blocks original-data))
|
||||
(first (:pages-and-blocks imported-block)))
|
||||
"Imported page equals exported page of page ref")))
|
||||
|
||||
(defn- export-page-and-import-to-another-graph
|
||||
"Exports given page from one graph/conn, imports it to a 2nd graph and then
|
||||
@@ -165,7 +161,7 @@
|
||||
|
||||
(import-second-time-appends-blocks conn conn2 "page1" original-data)))
|
||||
|
||||
(deftest ^:focus import-page-with-different-ref-types
|
||||
(deftest import-page-with-different-ref-types
|
||||
(let [block-uuid (random-uuid)
|
||||
class-uuid (random-uuid)
|
||||
page-uuid (random-uuid)
|
||||
@@ -196,7 +192,6 @@
|
||||
"Page's properties are imported")
|
||||
(is (= (:classes original-data) (:classes full-imported-page))
|
||||
"Page's classes are imported")
|
||||
;; (cljs.pprint/pprint (:pages-and-blocks full-imported-page))
|
||||
(is (= (:pages-and-blocks original-data) (:pages-and-blocks full-imported-page))
|
||||
"Page's blocks are imported")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user