enhance: export+import page handles most ref types in blocks

All ref types handled except class ref. Also fixed most block types not
persisting uuid when :build-existing-tx? set

deps/db/src/logseq/db/sqlite/build.cljs

@@ -121,9 +121,11 @@
                      [property-map v])))))
        (db-property-build/build-property-values-tx-m new-block)))
 
-(defn- extract-content-refs
-  "Extracts basic refs from :block/title like `[[foo]]`. Adding more ref support would
-  require parsing each block with mldoc and extracting with text/extract-refs-from-mldoc-ast"
+(defn extract-content-refs
+  "Extracts basic refs from :block/title like `[[foo]]` or `[[UUID]]`. Can't
+  use db-content/get-matched-ids because of named ref support.  Adding more ref
+  support would require parsing each block with mldoc and extracting with
+  text/extract-refs-from-mldoc-ast"
   [s]
   ;; FIXME: Better way to ignore refs inside a macro
   (if (string/starts-with? s "{{")
@@ -208,10 +210,10 @@
 
 (defn- build-properties-tx [properties page-uuids all-idents {:keys [build-existing-tx?]}]
   (let [properties' (if build-existing-tx?
-                     (->> properties
-                          (remove #(:block/uuid (val %)))
-                          (into {}))
-                     properties)
+                      (->> properties
+                           (remove (fn [[_ v]] (and (:block/uuid v) (not (:build/new-property? v)))))
+                           (into {}))
+                      properties)
         property-db-ids (->> (keys properties')
                              (map #(vector % (new-db-id)))
                              (into {}))
@@ -222,10 +224,10 @@
 
 (defn- build-classes-tx [classes properties-config uuid-maps all-idents {:keys [build-existing-tx?]}]
   (let [classes' (if build-existing-tx?
-                  (->> classes
-                       (remove #(:block/uuid (val %)))
-                       (into {}))
-                  classes)
+                   (->> classes
+                        (remove (fn [[_ v]] (and (:block/uuid v) (not (:build/new-class? v)))))
+                        (into {}))
+                   classes)
         class-db-ids (->> (keys classes')
                           (map #(vector % (new-db-id)))
                           (into {}))
@@ -384,6 +386,26 @@
             "Class and property db-idents have no overlap")
     all-idents))
 
+(defn- build-page-tx [page all-idents page-uuids properties]
+  (let [page' (dissoc page :build/tags :build/properties)
+        pvalue-tx-m (->property-value-tx-m page' (:build/properties page) properties all-idents)]
+    (cond-> []
+      (seq pvalue-tx-m)
+      (into (mapcat #(if (set? %) % [%]) (vals pvalue-tx-m)))
+      true
+      (conj
+       (block-with-timestamps
+        (merge
+         page'
+         (when (seq (:build/properties page))
+           (->block-properties (merge (:build/properties page) (db-property-build/build-properties-with-ref-values pvalue-tx-m))
+                               page-uuids
+                               all-idents))
+         (when-let [tags (:build/tags page)]
+           {:block/tags (-> (mapv #(hash-map :db/ident (get-ident all-idents %))
+                                  tags)
+                            (conj :logseq.class/Page))})))))))
+
 (defn- build-pages-and-blocks-tx
   [pages-and-blocks all-idents page-uuids {:keys [page-id-fn properties build-existing-tx?]
                                            :or {page-id-fn :db/id}
@@ -391,7 +413,11 @@
   (vec
    (mapcat
     (fn [{:keys [page blocks]}]
-      (let [page' (if (and build-existing-tx? (not (::new-page? (meta page))))
+      (let [;; For a page to be ignored it's important that it only sends a {:block/uuid UUID} map.
+            ;; This allows import processes to append blocks to an existing page or to create a page
+            ;; that is referenced by another page
+            ignore-page-tx? (and build-existing-tx? (not (::new-page? (meta page))) (= '(:block/uuid) (keys page)))
+            page' (if ignore-page-tx?
                     page
                     (merge
                      ;; TODO: Use sqlite-util/build-new-page
@@ -399,33 +425,16 @@
                       :block/title (or (:block/title page) (string/capitalize (:block/name page)))
                       :block/name (or (:block/name page) (common-util/page-name-sanity-lc (:block/title page)))
                       :block/tags #{:logseq.class/Page}}
-                     (dissoc page :build/properties :db/id :block/name :block/title :build/tags)))
+                     (dissoc page :db/id :block/name :block/title)))
             page-id-fn' (if (and build-existing-tx? (not (::new-page? (meta page))))
-                         #(vector :block/uuid (:block/uuid %))
-                         page-id-fn)
+                          #(vector :block/uuid (:block/uuid %))
+                          page-id-fn)
             opts' (assoc opts :existing-page? (and build-existing-tx? (not (::new-page? (meta page)))))]
         (into
          ;; page tx
-         (if (and build-existing-tx? (not (::new-page? (meta page))))
-           ;; Ignore existing pages until there's a use case for updating them
+         (if ignore-page-tx?
            []
-           (let [pvalue-tx-m (->property-value-tx-m page' (:build/properties page) properties all-idents)]
-             (cond-> []
-               (seq pvalue-tx-m)
-               (into (mapcat #(if (set? %) % [%]) (vals pvalue-tx-m)))
-               true
-               (conj
-                (block-with-timestamps
-                 (merge
-                  page'
-                  (when (seq (:build/properties page))
-                    (->block-properties (merge (:build/properties page) (db-property-build/build-properties-with-ref-values pvalue-tx-m))
-                                        page-uuids
-                                        all-idents))
-                  (when-let [tags (:build/tags page)]
-                    {:block/tags (-> (mapv #(hash-map :db/ident (get-ident all-idents %))
-                                           tags)
-                                     (conj :logseq.class/Page))})))))))
+           (build-page-tx page' all-idents page-uuids properties))
          ;; blocks tx
          (reduce (fn [acc m]
                    (into acc
@@ -503,10 +512,10 @@
                       (with-meta block {::existing-block? true})
                       (assoc block :block/uuid (random-uuid)))
              block'' (cond-> block'
-                      true
-                      (dissoc :build/children)
-                      parent-id
-                      (assoc :block/parent {:db/id [:block/uuid parent-id]}))
+                       true
+                       (dissoc :build/children)
+                       parent-id
+                       (assoc :block/parent {:db/id [:block/uuid parent-id]}))
              children (:build/children block)
              child-maps (when children (expand-build-children children (:block/uuid block'')))]
          (cons block'' child-maps)))

deps/db/src/logseq/db/sqlite/export.cljs

@@ -5,6 +5,7 @@
             [datascript.impl.entity :as de]
             [logseq.db :as ldb]
             [logseq.db.frontend.class :as db-class]
+            [logseq.db.frontend.content :as db-content]
             [logseq.db.frontend.entity-plus :as entity-plus]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.sqlite.build :as sqlite-build]))
@@ -56,9 +57,9 @@
        (into {})))
 
 (defn- build-export-properties
-  [db user-defined-properties {:keys [include-properties?]}]
+  [db user-property-idents {:keys [include-properties? include-uuid?]}]
   (let [properties-config-by-ent
-        (->> user-defined-properties
+        (->> user-property-idents
              (map (fn [ident]
                     (let [property (d/entity db ident)
                           closed-values (entity-plus/lookup-kv-then-entity property :property/closed-values)]
@@ -66,6 +67,8 @@
                        (cond-> (select-keys property
                                             (-> (disj db-property/schema-properties :logseq.property/classes)
                                                 (conj :block/title)))
+                         include-uuid?
+                         (assoc :block/uuid (:block/uuid property))
                          (:logseq.property/classes property)
                          (assoc :build/property-classes (mapv :db/ident (:logseq.property/classes property)))
                          (seq closed-values)
@@ -120,7 +123,7 @@
 
 (defn- build-entity-export
   "Given entity and optional existing properties, build an EDN export map"
-  [db entity {:keys [properties include-uuid?]}]
+  [db entity {:keys [properties include-uuid-fn] :or {include-uuid-fn (constantly false)}}]
   (let [ent-properties (dissoc (db-property/properties entity) :block/tags)
         new-user-property-ids (->> (keys ent-properties)
                                    (concat (->> (:block/tags entity)
@@ -132,7 +135,8 @@
         new-properties (build-export-properties db new-user-property-ids {})
         build-tags (when (seq (:block/tags entity)) (->build-tags (:block/tags entity)))
         build-block (cond-> (select-keys entity
-                                         (cond-> [:block/title] include-uuid? (conj :block/uuid)))
+                                         (cond-> [:block/title]
+                                           (include-uuid-fn (:block/uuid entity)) (conj :block/uuid)))
                       (seq build-tags)
                       (assoc :build/tags build-tags)
                       (seq ent-properties)
@@ -169,7 +173,7 @@
 (defn- build-blocks-tree
   "Given a page's block entities, returns the blocks in a sqlite.build EDN format
    and all properties and classes used in these blocks"
-  [db blocks & {:keys [include-uuid?]}]
+  [db blocks {:keys [include-uuid-fn]}]
   (let [*properties (atom {})
         *classes (atom {})
         *pvalue-uuids (atom #{})
@@ -178,7 +182,7 @@
         build-block (fn build-block [block*]
                       (let [child-nodes (mapv build-block (get children (:db/id block*) []))
                             {:build/keys [block] :keys [properties classes]}
-                            (build-entity-export db block* {:properties @*properties :include-uuid? include-uuid?})
+                            (build-entity-export db block* {:properties @*properties :include-uuid-fn include-uuid-fn})
                             new-pvalue-uuids (get-pvalue-uuids block)]
                         (when (seq properties) (swap! *properties merge properties))
                         (when (seq classes) (swap! *classes merge classes))
@@ -192,6 +196,20 @@
      :classes @*classes
      :pvalue-uuids @*pvalue-uuids}))
 
+(defn- get-uuid-block-pages [db pvalue-uuids content-ref-ents page-entity]
+  (let [uuid-block-ents-to-export (concat (map #(d/entity db [:block/uuid %]) pvalue-uuids)
+                                          (remove ldb/page? content-ref-ents))]
+    (when (seq uuid-block-ents-to-export)
+      (->> uuid-block-ents-to-export
+           (group-by :block/parent)
+           (map (fn [[parent-page-ent blocks]]
+                  ;; Not a common case but can support later if needed
+                  (when (= parent-page-ent page-entity)
+                    (throw (ex-info "Can't export a uuid block from exported page" {})))
+                  ;; Don't export pvalue-uuids of pvalue blocks as it's too excessive for now
+                  (merge (build-blocks-tree db (sort-by :block/order blocks) {:include-uuid-fn (constantly true)})
+                         {:page (select-keys parent-page-ent [:block/title])})))))))
+
 (defn build-page-export [db eid]
   (let [page-entity (d/entity db eid)
         ;; TODO: Fetch unloaded page datoms
@@ -202,18 +220,16 @@
                          (sort-by :block/order)
                          ;; Remove property value blocks as they are included in the block they belong to
                          (remove #(:logseq.property/created-from-property %)))
-        {:keys [blocks properties classes pvalue-uuids]} (build-blocks-tree db page-blocks)
-        pvalue-pages (when (seq pvalue-uuids)
-                       (->> pvalue-uuids
-                            (map #(d/entity db [:block/uuid %]))
-                            (group-by :block/parent)
-                            (map (fn [[parent-page-ent blocks]]
-                                   ;; Not a common case but can support later if needed
-                                   (when (= parent-page-ent page-entity)
-                                     (throw (ex-info "Can't export a block object from exported page" {})))
-                                   ;; Don't export pvalue-uuids of pvalue blocks as it's too excessive for now
-                                   (merge (build-blocks-tree db (sort-by :block/order blocks) {:include-uuid? true})
-                                          {:page (select-keys parent-page-ent [:block/title])})))))
+        content-ref-uuids (set (mapcat (comp db-content/get-matched-ids :block/title) page-blocks))
+        content-ref-ents (map #(d/entity db [:block/uuid %]) content-ref-uuids)
+        content-ref-pages (filter ldb/internal-page? content-ref-ents)
+        content-ref-properties* (map :db/ident (filter ldb/property? content-ref-ents))
+        content-ref-properties (when content-ref-properties*
+                                 (update-vals (build-export-properties db content-ref-properties* {:include-uuid? true})
+                                              #(merge % {:build/new-property? true})))
+        {:keys [blocks properties classes pvalue-uuids]}
+        (build-blocks-tree db page-blocks {:include-uuid-fn content-ref-uuids})
+        uuid-block-pages (get-uuid-block-pages db pvalue-uuids content-ref-ents page-entity)
         page-ent-export (build-entity-export db page-entity {:properties properties})
         page (merge (dissoc (:build/block page-ent-export) :block/title)
                     (if (ldb/journal? page-entity)
@@ -221,10 +237,17 @@
                       (select-keys page-entity [:block/title])))
         pages-and-blocks
         (cond-> [{:page page :blocks blocks}]
-          (seq pvalue-pages)
-          (into (map #(select-keys % [:page :blocks]) pvalue-pages)))
-        properties' (apply merge properties (:properties page-ent-export) (map :properties pvalue-pages))
-        classes' (apply merge classes (:classes page-ent-export) (map :classes pvalue-pages))
+          ;; pages from uuid blocks or content-ref-pages are shallow copies e.g. only :block/title
+          (seq uuid-block-pages)
+          (into (map #(select-keys % [:page :blocks]) uuid-block-pages))
+          (seq content-ref-pages)
+          (into (map #(hash-map :page (select-keys % [:block/title :block/uuid])) content-ref-pages)))
+        ;; Use merge-with to preserve new-property?
+        properties' (apply merge-with merge properties
+                           (:properties page-ent-export)
+                           content-ref-properties
+                           (map :properties uuid-block-pages))
+        classes' (apply merge classes (:classes page-ent-export) (map :classes uuid-block-pages))
         page-export
         (cond-> {:pages-and-blocks pages-and-blocks}
           (seq properties')

deps/db/test/logseq/db/sqlite/build_test.cljs

@@ -82,7 +82,7 @@
         block2 (db-test/find-block-by-content @conn "block 2")
         {:keys [init-tx block-props-tx]}
         (sqlite-build/build-blocks-tx
-         {:pages-and-blocks [{:page (select-keys (:block/page block) [:block/uuid :block/title])
+         {:pages-and-blocks [{:page (select-keys (:block/page block) [:block/uuid])
                               :blocks [(merge {:block/title "imported task" :block/uuid (:block/uuid block)}
                                               {:build/properties {:logseq.task/status :logseq.task/status.todo}
                                                :build/tags [:logseq.class/Task]})]}]
@@ -92,7 +92,7 @@
         updated-block (d/entity @conn [:block/uuid (:block/uuid block)])
         {init-tx2 :init-tx block-props-tx2 :block-props-tx :as _tx}
         (sqlite-build/build-blocks-tx
-         {:pages-and-blocks [{:page (select-keys (:block/page block2) [:block/uuid :block/title])
+         {:pages-and-blocks [{:page (select-keys (:block/page block2) [:block/uuid])
                               :blocks [(merge {:block/title "imported block" :block/uuid (:block/uuid block2)}
                                               {:build/properties {:user.property/p1 "foo"}
                                                :build/tags [:user.class/MyClass]})]}]
@@ -127,8 +127,9 @@
         page-uuid (random-uuid)
         property-uuid (random-uuid)
         conn (db-test/create-conn-with-blocks
-              {:classes {:C1 {:block/uuid class-uuid}}
-               :properties {:p1 {:block/uuid property-uuid}}
+              {:classes {:C1 {:block/uuid class-uuid :build/new-class? true}}
+               :properties {:p1 {:block/uuid property-uuid :build/new-property? true}}
+               :build-existing-tx? true
                :pages-and-blocks
                [{:page {:block/title "page 1"}
                  :blocks [{:block/title "named page ref to [[named page]]"}

deps/db/test/logseq/db/sqlite/export_test.cljs

@@ -1,6 +1,7 @@
 (ns logseq.db.sqlite.export-test
   (:require [cljs.test :refer [deftest is testing]]
             [datascript.core :as d]
+            [logseq.common.util.page-ref :as page-ref]
             [logseq.db :as ldb]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.sqlite.export :as sqlite-export]
@@ -103,8 +104,8 @@
                (db-test/readable-properties import-block))
             "imported block properties equals exported one")))))
 
-;; Tests a variety of blocks including block children with new properties, blocks with new classes
-;; and blocks with built-in properties
+;; Tests a variety of blocks including block children with new properties, blocks with users classes
+;; and blocks with built-in properties and classes
 (deftest import-page-with-different-blocks
   (let [original-data
         {:properties {:user.property/default {:logseq.property/type :default
@@ -135,11 +136,9 @@
         {:keys [init-tx block-props-tx] :as _txs}
         (->> (sqlite-export/build-page-export @conn (:db/id page))
              (sqlite-export/build-import @conn2 {}))
-        _ (assert (nil? (d/entity @conn2 :user.property/default)))
-        _ (assert (nil? (d/entity @conn2 :user.class/MyClass)))
+        ;; _ (cljs.pprint/pprint _txs)
         _ (d/transact! conn2 init-tx)
         _ (d/transact! conn2 block-props-tx)
-        ;; _ (cljs.pprint/pprint _txs)
         page2 (db-test/find-page-by-title @conn2 "page1")
         full-imported-page (sqlite-export/build-page-export @conn2 (:db/id page2))]
 
@@ -163,6 +162,46 @@
                        (fn [blocks] (into blocks blocks)))]
         (is (= expected-page-and-blocks (:pages-and-blocks full-imported-page)))))))
 
+(deftest import-page-with-different-ref-types
+  (let [block-uuid (random-uuid)
+        ;; class-uuid (random-uuid)
+        page-uuid (random-uuid)
+        property-uuid (random-uuid)
+        original-data
+        {;:classes {:C1 {:block/uuid class-uuid}}
+         :properties {:user.property/p1
+                      {:db/cardinality :db.cardinality/one, :logseq.property/type :default
+                       :block/uuid property-uuid :block/title "p1" :build/new-property? true}}
+         :pages-and-blocks
+         [{:page {:block/title "page1"}
+           :blocks [{:block/title (str "page ref to " (page-ref/->page-ref page-uuid))}
+                    {:block/title (str "block ref to " (page-ref/->page-ref block-uuid))}
+                    #_{:block/title (str "class ref to " (page-ref/->page-ref class-uuid))}
+                    #_{:block/title (str "inline class ref to #" (page-ref/->page-ref class-uuid))}
+                    {:block/title (str "property ref to " (page-ref/->page-ref property-uuid))}]}
+          {:page {:block/title "page with block ref"}
+           :blocks [{:block/title "hi" :block/uuid block-uuid}]}
+          {:page {:block/title "another page" :block/uuid page-uuid}}]}
+        conn (db-test/create-conn-with-blocks original-data)
+        page (db-test/find-page-by-title @conn "page1")
+        conn2 (db-test/create-conn)
+        {:keys [init-tx block-props-tx] :as _txs}
+        (->> (sqlite-export/build-page-export @conn (:db/id page))
+             (sqlite-export/build-import @conn2 {}))
+        ;; _ (cljs.pprint/pprint _txs)
+        _ (d/transact! conn2 init-tx)
+        _ (d/transact! conn2 block-props-tx)
+        page2 (db-test/find-page-by-title @conn2 "page1")
+        full-imported-page (sqlite-export/build-page-export @conn2 (:db/id page2))]
+
+    (is (= (:properties original-data) (:properties full-imported-page))
+        "Page's properties are imported")
+    (is (= (:classes original-data) (:classes full-imported-page))
+        "Page's classes are imported")
+    ;; (cljs.pprint/pprint (:pages-and-blocks full-imported-page))
+    (is (= (:pages-and-blocks original-data) (:pages-and-blocks full-imported-page))
+        "Page's blocks are imported")))
+
 (deftest import-page-with-different-page-and-classes
   (let [original-data
         {:properties {:user.property/p1 {:db/cardinality :db.cardinality/one, :logseq.property/type :default, :block/title "p1"}
@@ -180,7 +219,7 @@
         {:keys [init-tx block-props-tx] :as _txs}
         (->> (sqlite-export/build-page-export @conn (:db/id page))
              (sqlite-export/build-import @conn2 {}))
-        _ (assert (nil? (d/entity @conn2 :user.property/default)))
+        _ (assert (nil? (d/entity @conn2 :user.property/p1)))
         _ (assert (nil? (d/entity @conn2 :user.class/MyClass)))
         _ (d/transact! conn2 init-tx)
         _ (d/transact! conn2 block-props-tx)