From 8b8055dd7236fed97a66c6ff8b12d174e9e8ebdd Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Mon, 10 Feb 2025 15:14:10 -0500 Subject: [PATCH] refactor: use namespaced keyword for block export instead of confusing :build/block. Also improved related tests, simplified import steps, added some missing sqlite.build docs, and fixed :build/uuid not working for some existing journals. --- deps/db/script/create_graph.cljs | 3 +- deps/db/src/logseq/db/sqlite/build.cljs | 4 +- deps/db/src/logseq/db/sqlite/export.cljs | 81 ++++++++-------- deps/db/src/logseq/db/test/helper.cljs | 12 +++ .../db/test/logseq/db/sqlite/export_test.cljs | 92 ++++++++++++------- .../frontend/handler/common/developer.cljs | 8 +- 6 files changed, 118 insertions(+), 82 deletions(-) diff --git a/deps/db/script/create_graph.cljs b/deps/db/script/create_graph.cljs index 24e1b00af6..80223e92da 100644 --- a/deps/db/script/create_graph.cljs +++ b/deps/db/script/create_graph.cljs @@ -28,9 +28,10 @@ sqlite-build-edn (merge {:auto-create-ontology? true} (-> (resolve-path edn-path) fs/readFileSync str edn/read-string)) conn (outliner-cli/init-conn dir db-name {:classpath (cp/get-classpath) :import-type :cli/create-graph}) - {:keys [init-tx block-props-tx]} (outliner-cli/build-blocks-tx sqlite-build-edn)] + {:keys [init-tx block-props-tx] :as _txs} (outliner-cli/build-blocks-tx sqlite-build-edn)] (println "Generating" (count (filter :block/name init-tx)) "pages and" (count (filter :block/title init-tx)) "blocks ...") + ;; (cljs.pprint/pprint _txs) (d/transact! conn init-tx) (d/transact! conn block-props-tx) (println "Created graph" (str db-name "!")))) diff --git a/deps/db/src/logseq/db/sqlite/build.cljs b/deps/db/src/logseq/db/sqlite/build.cljs index cf55fc2b29..46376272c3 100644 --- a/deps/db/src/logseq/db/sqlite/build.cljs +++ b/deps/db/src/logseq/db/sqlite/build.cljs @@ -544,7 +544,7 @@ :block/uuid (or (:block/uuid page) (common-uuid/gen-uuid :journal-page-uuid date-int)) :block/tags :logseq.class/Journal}) - (with-meta {::new-page? true}))))) + (with-meta {::new-page? (not (:block/uuid page))}))))) m))] ;; Order matters as some steps depend on previous step having prepared blocks or pages in a certain way (->> pages-and-blocks @@ -641,12 +641,14 @@ * :build/journal - Define a journal pages as an integer e.g. 20240101 is Jan 1, 2024. :block/title is not required if using this since it generates one * :build/properties - Defines properties on a page + * :build/tags - Defines tags on a page * :build/keep-uuid? - Keeps :block/uuid because another block depends on it * :blocks - This is a vec of datascript attribute maps for blocks with :block/title required. e.g. `{:block/title \"bar\"}`. Additional keys available: * :build/children - A vec of blocks that are nested (indented) under the current block. Allows for outlines to be expressed to whatever depth * :build/properties - Defines properties on a block + * :build/tags - Defines tags on a block * :build/keep-uuid? - Keeps :block/uuid because another block depends on it * :properties - This is a map to configure properties where the keys are property name keywords and the values are maps of datascript attributes e.g. `{:logseq.property/type :checkbox}`. diff --git a/deps/db/src/logseq/db/sqlite/export.cljs b/deps/db/src/logseq/db/sqlite/export.cljs index 169adf2fa3..e499534feb 100644 --- a/deps/db/src/logseq/db/sqlite/export.cljs +++ b/deps/db/src/logseq/db/sqlite/export.cljs @@ -137,8 +137,9 @@ (map #(vector (:db/ident %) (build-export-class % {:include-parents? false}))) (into {})))) -(defn- build-entity-export - "Given entity and optional existing properties, build an EDN export map" +(defn- build-node-export + "Given a block/page entity and optional existing properties, build an export map of its + tags and properties" [db entity {:keys [properties include-uuid-fn keep-uuid?] :or {include-uuid-fn (constantly false)}}] (let [ent-properties (dissoc (db-property/properties entity) :block/tags) new-user-property-ids (->> (keys ent-properties) @@ -150,7 +151,7 @@ (remove #(get properties %))) new-properties (build-export-properties db new-user-property-ids {}) build-tags (when (seq (:block/tags entity)) (->build-tags (:block/tags entity))) - build-block (cond-> {:block/title (block-title entity)} + build-node (cond-> {:block/title (block-title entity)} (include-uuid-fn (:block/uuid entity)) (assoc :block/uuid (:block/uuid entity)) keep-uuid? @@ -160,8 +161,8 @@ (seq ent-properties) (assoc :build/properties (buildable-properties db ent-properties (merge properties new-properties)))) - new-classes (build-export-block-classes db build-block (:block/tags entity))] - (cond-> {:build/block build-block} + new-classes (build-export-block-classes db build-node (:block/tags entity))] + (cond-> {:node build-node} (seq new-classes) (assoc :classes new-classes) (seq new-properties) @@ -219,10 +220,10 @@ [db eid] (let [block-entity (d/entity db eid) {:keys [content-ref-uuids _content-ref-ents] :as content-ref-export} (build-content-ref-export db [block-entity]) - block-export* (build-entity-export db block-entity {:include-uuid-fn content-ref-uuids}) - pvalue-uuids (get-pvalue-uuids (:build/block block-export*)) - block-export (assoc (merge-export-maps block-export* content-ref-export) - :build/block (:build/block block-export*))] + node-export (build-node-export db block-entity {:include-uuid-fn content-ref-uuids}) + pvalue-uuids (get-pvalue-uuids (:node node-export)) + block-export (merge {::block (:node node-export)} + (merge-export-maps node-export content-ref-export))] ;; Maybe add support for this later (when (seq pvalue-uuids) (throw (ex-info "Exporting a block with :node block objects is not supported" {}))) @@ -239,13 +240,13 @@ children (group-by #(get-in % [:block/parent :db/id]) blocks) build-block (fn build-block [block*] (let [child-nodes (mapv build-block (get children (:db/id block*) [])) - {:build/keys [block] :keys [properties classes]} - (build-entity-export db block* (assoc opts :properties @*properties)) - new-pvalue-uuids (get-pvalue-uuids block)] + {:keys [node properties classes]} + (build-node-export db block* (assoc opts :properties @*properties)) + new-pvalue-uuids (get-pvalue-uuids node)] (when (seq properties) (swap! *properties merge properties)) (when (seq classes) (swap! *classes merge classes)) (when (seq new-pvalue-uuids) (swap! *pvalue-uuids into new-pvalue-uuids)) - (cond-> block + (cond-> node (seq child-nodes) (assoc :build/children child-nodes)))) roots (remove #(contains? id-map (get-in % [:block/parent :db/id])) blocks) exported-blocks (mapv build-block roots)] @@ -288,8 +289,8 @@ {:keys [blocks properties classes pvalue-uuids]} (build-blocks-tree db page-blocks {:include-uuid-fn content-ref-uuids}) uuid-block-export (build-uuid-block-export db pvalue-uuids content-ref-ents page-entity) - page-ent-export (build-entity-export db page-entity {:properties properties}) - page (merge (dissoc (:build/block page-ent-export) :block/title) + page-ent-export (build-node-export db page-entity {:properties properties}) + page (merge (dissoc (:node page-ent-export) :block/title) (shallow-copy-page page-entity)) page-blocks-export {:pages-and-blocks [{:page page :blocks blocks}] :properties properties @@ -327,11 +328,25 @@ ;; Import fns ;; ========== -(defn- ->sqlite-build-options +(defn- check-for-existing-entities + "Checks export map for existing entities and adds :block/uuid to them if they exist in graph to import. + Also checks for property conflicts between existing properties and properties to be imported" [db {:keys [pages-and-blocks classes properties]} property-conflicts] (cond-> {:build-existing-tx? true} (seq pages-and-blocks) - (assoc :pages-and-blocks pages-and-blocks) + (assoc :pages-and-blocks + (mapv (fn [m] + (if-let [ent (some->> (get-in m [:page :build/journal]) + (d/datoms db :avet :block/journal-day) + first + :e + (d/entity db))] + (assoc-in m [:page :block/uuid] (:block/uuid ent)) + ;; For now only check page uniqueness by title. Could handle more uniqueness checks later + (if-let [ent (some->> (get-in m [:page :block/title]) (ldb/get-case-page db))] + (assoc-in m [:page :block/uuid] (:block/uuid ent)) + m))) + pages-and-blocks)) (seq classes) (assoc :classes (->> classes @@ -359,7 +374,7 @@ (defn- build-block-import-options "Builds options for sqlite-build to import into current-block" [current-block export-map] - (let [block (merge (:build/block export-map) + (let [block (merge (::block export-map) {:block/uuid (:block/uuid current-block) :block/page (select-keys (:block/page current-block) [:block/uuid])}) pages-and-blocks @@ -367,35 +382,17 @@ :blocks [(dissoc block :block/page)]}]] (merge-export-maps export-map {:pages-and-blocks pages-and-blocks}))) -(defn- build-page-import-options - [db export-map] - (assert (map? (get-in export-map [:pages-and-blocks 0 :page])) "page export exists") - (if-let [ent (some->> (get-in export-map [:pages-and-blocks 0 :page :build/journal]) - (d/datoms db :avet :block/journal-day) - first - :e - (d/entity db))] - (assoc-in export-map [:pages-and-blocks 0 :page] (select-keys ent [:block/uuid])) - ;; FIXME: Find an existing page more reliably than :block/title, :block/uuid? - (if-let [ent (some->> (get-in export-map [:pages-and-blocks 0 :page :block/title]) - (ldb/get-case-page db))] - (assoc-in export-map [:pages-and-blocks 0 :page] (select-keys ent [:block/uuid])) - export-map))) - (defn build-import "Given an entity's export map, build the import tx to create it" - [db {:keys [current-block]} export-map*] - (let [export-map (cond current-block - (build-block-import-options current-block export-map*) - (:pages-and-blocks export-map*) - (build-page-import-options db export-map*) - :else - export-map*) + [export-map* db {:keys [current-block]}] + (let [export-map (if (and (::block export-map*) current-block) + (build-block-import-options current-block export-map*) + export-map*) property-conflicts (atom []) - opts (->sqlite-build-options db export-map property-conflicts)] + export-map' (check-for-existing-entities db export-map property-conflicts)] (if (seq @property-conflicts) (do (js/console.error :property-conflicts @property-conflicts) {:error (str "The following imported properties conflict with the current graph: " (pr-str (mapv :property-id @property-conflicts)))}) - (sqlite-build/build-blocks-tx opts)))) \ No newline at end of file + (sqlite-build/build-blocks-tx export-map')))) \ No newline at end of file diff --git a/deps/db/src/logseq/db/test/helper.cljs b/deps/db/src/logseq/db/test/helper.cljs index 09d4a49f93..b607d03bfb 100644 --- a/deps/db/src/logseq/db/test/helper.cljs +++ b/deps/db/src/logseq/db/test/helper.cljs @@ -43,6 +43,18 @@ first (d/entity db))) +(defn find-journal-by-journal-day + [db journal-day] + (->> journal-day + (d/q + '[:find [?page ...] + :in $ ?journal-day + :where + [?page :block/journal-day ?journal-day]] + db) + first + (d/entity db))) + (defn readable-properties "Returns an entity's properties and tags in readable form for assertions. tags are included here since they behave like properties on an ent" diff --git a/deps/db/test/logseq/db/sqlite/export_test.cljs b/deps/db/test/logseq/db/sqlite/export_test.cljs index 6124d85464..ed120118da 100644 --- a/deps/db/test/logseq/db/sqlite/export_test.cljs +++ b/deps/db/test/logseq/db/sqlite/export_test.cljs @@ -3,7 +3,8 @@ [datascript.core :as d] [logseq.common.util.page-ref :as page-ref] [logseq.db.sqlite.export :as sqlite-export] - [logseq.db.test.helper :as db-test])) + [logseq.db.test.helper :as db-test] + [logseq.common.util.date-time :as date-time-util])) (defn- export-block-and-import-to-another-block "Exports given block from one graph/conn, imports it to a 2nd block and then @@ -12,8 +13,8 @@ (let [export-block (db-test/find-block-by-content @export-conn export-block-content) import-block (db-test/find-block-by-content @import-conn import-block-content) {:keys [init-tx block-props-tx] :as _txs} - (->> (sqlite-export/build-block-export @export-conn [:block/uuid (:block/uuid export-block)]) - (sqlite-export/build-import @import-conn {:current-block import-block})) + (-> (sqlite-export/build-block-export @export-conn [:block/uuid (:block/uuid export-block)]) + (sqlite-export/build-import @import-conn {:current-block import-block})) ;; _ (cljs.pprint/pprint _txs) _ (d/transact! import-conn init-tx) _ (d/transact! import-conn block-props-tx)] @@ -35,7 +36,7 @@ imported-block (export-block-and-import-to-another-block conn conn "export" "import")] (is (= (get-in original-data [:pages-and-blocks 0 :blocks 0]) - (:build/block imported-block)) + (::sqlite-export/block imported-block)) "Imported block equals exported block") (is (= (:properties original-data) (:properties imported-block))) (is (= (:classes original-data) (:classes imported-block))))) @@ -67,7 +68,7 @@ imported-block (export-block-and-import-to-another-block conn conn2 "export" "import")] (is (= (get-in original-data [:pages-and-blocks 0 :blocks 0]) - (:build/block imported-block)) + (::sqlite-export/block imported-block)) "Imported block equals exported block") (is (= (:properties original-data) (:properties imported-block))) (is (= (:classes original-data) (:classes imported-block))) @@ -75,7 +76,7 @@ (testing "same import in another block" (let [imported-block (export-block-and-import-to-another-block conn conn2 "export" "import2")] (is (= (get-in original-data [:pages-and-blocks 0 :blocks 0]) - (:build/block imported-block)) + (::sqlite-export/block imported-block)) "Imported block equals exported block") (is (= (:properties original-data) (:properties imported-block))) (is (= (:classes original-data) (:classes imported-block))))))) @@ -94,7 +95,7 @@ imported-block (export-block-and-import-to-another-block conn conn2 #"page ref" "import")] (is (= (get-in original-data [:pages-and-blocks 0 :blocks 0]) - (:build/block imported-block)) + (::sqlite-export/block imported-block)) "Imported block equals exported block") (is (= (second (:pages-and-blocks original-data)) (first (:pages-and-blocks imported-block))) @@ -106,21 +107,29 @@ [export-conn import-conn page-title] (let [page (db-test/find-page-by-title @export-conn page-title) {:keys [init-tx block-props-tx] :as _txs} - (->> (sqlite-export/build-page-export @export-conn (:db/id page)) - ;; ((fn [x] (cljs.pprint/pprint {:export x}) x)) - (sqlite-export/build-import @import-conn {})) + (-> (sqlite-export/build-page-export @export-conn (:db/id page)) + ;; ((fn [x] (cljs.pprint/pprint {:export x}) x)) + (sqlite-export/build-import @import-conn {})) ;; _ (cljs.pprint/pprint _txs) _ (d/transact! import-conn init-tx) _ (d/transact! import-conn block-props-tx) page2 (db-test/find-page-by-title @import-conn page-title)] (sqlite-export/build-page-export @import-conn (:db/id page2)))) -(defn- import-second-time-appends-blocks [conn conn2 page-title original-data] - (let [full-imported-page (export-page-and-import-to-another-graph conn conn2 page-title) +(defn- import-second-time-assertions [conn conn2 page-title original-data] + (let [page (db-test/find-page-by-title @conn2 page-title) + imported-page (export-page-and-import-to-another-graph conn conn2 page-title) + updated-page (db-test/find-page-by-title @conn2 page-title) expected-page-and-blocks (update-in (:pages-and-blocks original-data) [0 :blocks] (fn [blocks] (into blocks blocks)))] - (is (= expected-page-and-blocks (:pages-and-blocks full-imported-page))))) + + (is (= expected-page-and-blocks (:pages-and-blocks imported-page)) + "Blocks are appended to existing page blocks") + (is (= (:block/created-at page) (:block/created-at updated-page)) + "Existing page didn't get re-created") + (is (= (:block/updated-at page) (:block/updated-at updated-page)) + "Existing page didn't get updated"))) ;; Tests a variety of blocks including block children with new properties, blocks with users classes ;; and blocks with built-in properties and classes @@ -150,16 +159,16 @@ :build/tags [:logseq.class/Task]}]}]} conn (db-test/create-conn-with-blocks original-data) conn2 (db-test/create-conn) - full-imported-page (export-page-and-import-to-another-graph conn conn2 "page1")] + imported-page (export-page-and-import-to-another-graph conn conn2 "page1")] - (is (= (:properties original-data) (:properties full-imported-page)) + (is (= (:properties original-data) (:properties imported-page)) "Page's properties are imported") - (is (= (:classes original-data) (:classes full-imported-page)) + (is (= (:classes original-data) (:classes imported-page)) "Page's classes are imported") - (is (= (:pages-and-blocks original-data) (:pages-and-blocks full-imported-page)) + (is (= (:pages-and-blocks original-data) (:pages-and-blocks imported-page)) "Page's blocks are imported") - (import-second-time-appends-blocks conn conn2 "page1" original-data))) + (import-second-time-assertions conn conn2 "page1" original-data))) (deftest import-page-with-different-ref-types (let [block-uuid (random-uuid) @@ -186,16 +195,16 @@ {:page {:build/journal 20250207 :block/uuid journal-uuid :build/keep-uuid? true}}]} conn (db-test/create-conn-with-blocks original-data) conn2 (db-test/create-conn) - full-imported-page (export-page-and-import-to-another-graph conn conn2 "page1")] + imported-page (export-page-and-import-to-another-graph conn conn2 "page1")] - (is (= (:properties original-data) (:properties full-imported-page)) + (is (= (:properties original-data) (:properties imported-page)) "Page's properties are imported") - (is (= (:classes original-data) (:classes full-imported-page)) + (is (= (:classes original-data) (:classes imported-page)) "Page's classes are imported") - (is (= (:pages-and-blocks original-data) (:pages-and-blocks full-imported-page)) + (is (= (:pages-and-blocks original-data) (:pages-and-blocks imported-page)) "Page's blocks are imported") - (import-second-time-appends-blocks conn conn2 "page1" original-data))) + (import-second-time-assertions conn conn2 "page1" original-data))) (deftest import-page-with-different-page-and-classes (let [original-data @@ -210,16 +219,31 @@ :blocks []}]} conn (db-test/create-conn-with-blocks original-data) conn2 (db-test/create-conn) - full-imported-page (export-page-and-import-to-another-graph conn conn2 "page1")] + imported-page (export-page-and-import-to-another-graph conn conn2 "page1")] - (is (= (:properties original-data) (:properties full-imported-page)) + (is (= (:properties original-data) (:properties imported-page)) "Page's properties are imported") - (is (= (:classes original-data) (:classes full-imported-page)) + (is (= (:classes original-data) (:classes imported-page)) "Page's classes are imported") - (is (= (:pages-and-blocks original-data) (:pages-and-blocks full-imported-page)) + (is (= (:pages-and-blocks original-data) (:pages-and-blocks imported-page)) "Page's blocks are imported") - (import-second-time-appends-blocks conn conn2 "page1" original-data))) + (import-second-time-assertions conn conn2 "page1" original-data))) + +(deftest import-journal-page + (let [original-data + {:pages-and-blocks + [{:page {:build/journal 20250210 :build/tags [:logseq.class/Journal]} + :blocks [{:block/title "b1"} {:block/title "b2"}]}]} + conn (db-test/create-conn-with-blocks original-data) + conn2 (db-test/create-conn) + journal-title (date-time-util/int->journal-title 20250210 "MMM do, yyyy") + imported-page (export-page-and-import-to-another-graph conn conn2 journal-title)] + + (is (= (:pages-and-blocks original-data) (:pages-and-blocks imported-page)) + "Page's blocks are imported") + + (import-second-time-assertions conn conn2 journal-title original-data))) (deftest import-page-with-different-property-types (let [block-object-uuid (random-uuid) @@ -257,13 +281,13 @@ :build/keep-uuid? true}]}]} conn (db-test/create-conn-with-blocks original-data) conn2 (db-test/create-conn) - full-imported-page (export-page-and-import-to-another-graph conn conn2 "page1")] + imported-page (export-page-and-import-to-another-graph conn conn2 "page1")] - (is (= (:properties original-data) (:properties full-imported-page)) + (is (= (:properties original-data) (:properties imported-page)) "Page's properties are imported") - (is (= (:classes original-data) (:classes full-imported-page)) + (is (= (:classes original-data) (:classes imported-page)) "Page's classes are imported") - (is (= (:pages-and-blocks original-data) (:pages-and-blocks full-imported-page)) + (is (= (:pages-and-blocks original-data) (:pages-and-blocks imported-page)) "Page's blocks are imported"))) (deftest import-graph-ontology @@ -292,8 +316,8 @@ conn (db-test/create-conn-with-blocks original-data) conn2 (db-test/create-conn) {:keys [init-tx block-props-tx] :as _txs} - (->> (sqlite-export/build-graph-ontology-export @conn) - (sqlite-export/build-import @conn2 {})) + (-> (sqlite-export/build-graph-ontology-export @conn) + (sqlite-export/build-import @conn2 {})) ;; _ (cljs.pprint/pprint _txs) _ (d/transact! conn2 init-tx) _ (d/transact! conn2 block-props-tx) diff --git a/src/main/frontend/handler/common/developer.cljs b/src/main/frontend/handler/common/developer.cljs index 015d7f3f83..0205f32fc7 100644 --- a/src/main/frontend/handler/common/developer.cljs +++ b/src/main/frontend/handler/common/developer.cljs @@ -69,7 +69,7 @@ (defn- import-submit [import-inputs _e] (let [export-map (try (edn/read-string (:import-data @import-inputs)) (catch :default _err ::invalid-import)) - import-block? (:build/block export-map) + import-block? (::sqlite-export/block export-map) block (when import-block? (if-let [eid (:block-id (first (state/get-editor-args)))] (db/entity [:block/uuid eid]) @@ -77,9 +77,9 @@ (if (= ::invalid-import export-map) (notification/show! "The submitted EDN data is invalid! Fix and try again." :warning) (let [{:keys [init-tx block-props-tx error] :as txs} - (sqlite-export/build-import (db/get-db) - (when block {:current-block block}) - export-map)] + (sqlite-export/build-import export-map + (db/get-db) + (when block {:current-block block}))] (pprint/pprint txs) (if error (notification/show! error :error)