fix: db graph tests with children, created-at properties

and journal pages

src/test/frontend/db/query_dsl_test.cljs

@@ -465,17 +465,24 @@ tags: [[other]]
                 (set)))
         "NOT query")))
 
-(deftest nested-page-ref-queries
-  (load-test-files [{:file/path "pages/page1.md"
-                     :file/content "foo:: bar
+(deftest ^:done nested-page-ref-queries
+  (load-test-files (if js/process.env.DB_GRAPH
+                     [{:page {:block/original-name "page1"}
+                       :blocks [{:block/content "p1 [[Parent page]]"
+                                 :build/children [{:block/content "[[Child page]]"}]}
+                                {:block/content "p2 [[Parent page]]"
+                                 :build/children [{:block/content "Non linked content"}]}]}]
+                     [{:file/path "pages/page1.md"
+                       :file/content "foo:: bar
 - p1 [[Parent page]]
   - [[Child page]]
 - p2 [[Parent page]]
-  - Non linked content"}])
-  (is (= ["Non linked content"
-          "p2 [[Parent page]]"
-          "p1 [[Parent page]]"]
-         (map :block/content
+  - Non linked content"}]))
+  (is (= ["Non"
+          "p2"
+          "p1"]
+         ;; Just test for start of blocks as content varies for db graphs
+         (map #(re-find #"\w+" (:block/content %))
               (dsl-query "(and [[Parent page]] (not [[Child page]]))")))))
 
 (deftest between-queries
@@ -491,20 +498,21 @@ created-at:: 1608968448115
 created-at:: 1608968448116
 "}])
 
-  (are [x y] (= (count (dsl-query x)) y)
-       "(and (task now later done) (between [[Dec 26th, 2020]] tomorrow))"
-       3
+  (let [task-filter (if js/process.env.DB_GRAPH "(task todo done)" "(task later done)")]
+    (are [x y] (= (count (dsl-query x)) y)
+      (str "(and " task-filter " (between [[Dec 26th, 2020]] tomorrow))")
+      3
 
        ;; between with journal pages
-       "(and (task now later done) (between [[Dec 26th, 2020]] [[Dec 27th, 2020]]))"
-       3
+      (str "(and " task-filter " (between [[Dec 26th, 2020]] [[Dec 27th, 2020]]))")
+      3
 
        ;; ;; between with created-at
        ;; "(and (task now later done) (between created-at [[Dec 26th, 2020]] tomorrow))"
        ;; 3
-       ))
+      )))
 
-(deftest custom-query-test
+(deftest ^:done custom-query-test
   (load-test-files [{:file/path "pages/page1.md"
                      :file/content "foo:: bar
 - NOW b1
@@ -512,12 +520,15 @@ created-at:: 1608968448116
 - LATER b3
 - b3"}])
 
-  (is (= ["LATER b3"]
-         (map :block/content (custom-query {:query '(task later)}))))
+  (let [task-query (if js/process.env.DB_GRAPH
+                     '(task doing)
+                     '(task now))]
+    (is (= ["NOW b1"]
+           (map :block/content (custom-query {:query task-query}))))
 
-  (is (= ["LATER b3"]
-         (map :block/content (custom-query {:query (list 'and '(task later) "b")})))
-      "Query with rule that can't be derived from the form itself"))
+    (is (= ["NOW b1"]
+           (map :block/content (custom-query {:query (list 'and task-query "b")})))
+        "Query with rule that can't be derived from the form itself")))
 
 (if js/process.env.DB_GRAPH
   (def get-property-value query-dsl/get-db-property-value)

src/test/frontend/test/helper.cljs

@@ -4,10 +4,8 @@
             [frontend.state :as state]
             [frontend.db.conn :as conn]
             [clojure.string :as string]
-            [clojure.set :as set]
             [logseq.db.sqlite.util :as sqlite-util]
             [frontend.db :as db]
-            [frontend.date :as date]
             [frontend.handler.editor :as editor-handler]
             [frontend.handler.page :as page-handler]
             [datascript.core :as d]
@@ -18,7 +16,8 @@
             [logseq.db.frontend.order :as db-order]
             [logseq.db.sqlite.build :as sqlite-build]
             [frontend.handler.file-based.status :as status]
-            [logseq.db.frontend.property :as db-property]))
+            [logseq.db.frontend.property :as db-property]
+            [logseq.outliner.db-pipeline :as db-pipeline]))
 
 (def node? (exists? js/process))
 
@@ -35,6 +34,7 @@
     (conn/start! test-db opts)
     (let [conn (conn/get-db test-db false)]
       (when db-graph?
+        (db-pipeline/add-listener conn)
         (d/transact! conn (sqlite-create-graph/build-db-initial-data "")))
       (d/listen! conn ::listen-db-changes!
                  (fn [tx-report]
@@ -65,7 +65,7 @@
                   [(keyword k) (parse-property-value v)])))
        (into {})))
 
-(defn- build-block-properties
+#_(defn- build-block-properties
   "Parses out properties from a file's content and associates it with the page name
    or block content"
   [file-content]
@@ -84,17 +84,7 @@
           string/split-lines
           property-lines->properties)}))
 
-(defn- file-path->page-name
-  [file-path]
-  (or (if (string/starts-with? file-path "journals")
-        (some-> (second (re-find #"([^/]+).md" file-path))
-                date/normalize-date
-                date/journal-name
-                string/lower-case)
-        (second (re-find #"([^/]+).md" file-path)))
-      (throw (ex-info "No page found" {}))))
-
-(defn- update-file-for-db-graph
+#_(defn- update-file-for-db-graph
   "Adds properties by block/page for a file and updates block content"
   [file]
   (let [{:keys [block-properties page-properties]}
@@ -122,8 +112,7 @@
                {:file/content (string/join "\n"
                                            (map (fn [x] (str "- " (:name-or-content x))) block-properties))})))))
 
-#_:clj-kondo/ignore
-(defn- load-test-files-for-db-graph-old
+#_(defn- load-test-files-for-db-graph-old
   [files*]
   (let [files (mapv update-file-for-db-graph files*)]
     ;; TODO: Use sqlite instead of file graph to create client db
@@ -221,10 +210,14 @@
                       (mapv (fn [s]
                               (let [[content & props] (string/split-lines s)]
                                 (cond-> {:block/content content}
-                      ;; If no property chars may accidentally parse child blocks
-                      ;; so don't do property parsing
+                                  ;; If no property chars may accidentally parse child blocks
+                                  ;; so don't do property parsing
                                   (and (string/includes? s ":: ") props)
-                                  (assoc :build/properties (property-lines->properties props)))))))
+                                  ((fn [x]
+                                     (let [props' (property-lines->properties props)]
+                                       (merge x (cond-> {:build/properties (dissoc props' :created-at)}
+                                                  (:created-at props')
+                                                  (assoc :block/created-at (:created-at props'))))))))))))
         [page-props blocks*]
         (if (string/includes? (:block/content (first blocks**)) "::")
           [(property-lines->properties (string/split-lines (:block/content (first blocks**))))
@@ -247,15 +240,22 @@
     ;; TODO: handle different page properties
     (parse-content* content)))
 
-(defn load-test-files-for-db-graph
-  [options*]
+(defn- build-blocks-tx-options [options*]
   (let [pages-and-blocks
         (mapv (fn [{:file/keys [path content]}]
                 (let [{:keys [blocks page-properties]} (parse-content content)
-                      _ (prn :blocks content blocks)
-                      page-name (or (second (re-find #"/([^/]+)\." path))
-                                    (throw (ex-info (str "Can't detect page name of file: " (pr-str path)) {})))]
-                  {:page (cond-> {:block/original-name page-name}
+                      ;; _ (prn :parse-content content blocks)
+                      unique-page-attrs
+                      (if (string/starts-with? path "journals")
+                        {:build/journal
+                         (or (some-> (second (re-find #"/([^/]+)\." path))
+                                     (string/replace "_" "")
+                                     parse-double)
+                             (throw (ex-info (str "Can't detect page name of file: " (pr-str path)) {})))}
+                        {:block/original-name
+                         (or (second (re-find #"/([^/]+)\." path))
+                             (throw (ex-info (str "Can't detect page name of file: " (pr-str path)) {})))})]
+                  {:page (cond-> unique-page-attrs
                            (seq page-properties)
                            (assoc :build/properties page-properties))
                    :blocks blocks}))
@@ -270,7 +270,13 @@
                         (into {}))
         options (cond-> {:pages-and-blocks pages-and-blocks}
                   (seq properties)
-                  (assoc :properties properties))
+                  (assoc :properties properties))]
+    options))
+
+(defn load-test-files-for-db-graph
+  [options*]
+  (let [;; Builds options from markdown :file/content unless given explicit build-blocks config
+        options (if (:page (first options*)) {:pages-and-blocks options*} (build-blocks-tx-options options*))
         _ (prn :opt options)
         {:keys [init-tx block-props-tx]} (sqlite-build/build-blocks-tx options)]
     (db/transact! test-db init-tx)