enhance: bb task for creating graphs from EDN file

Converted inferred graph to an EDN file now that this task exists. Also
merge last of tasks.create-graph to relevant ns so that external users
can also create such tasks

bb.edn

@@ -76,11 +76,17 @@
   {:doc "Transact against a DB graph's datascript db"
    :task (apply shell {:dir "deps/outliner"} "yarn -s nbb-logseq script/transact.cljs" *command-line-args*)}
 
+  dev:db-create
+  {:doc "Create a DB graph given a sqlite.build EDN file"
+   :requires ([babashka.fs :as fs])
+   :task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
+                "yarn -s nbb-logseq -cp src:../outliner/src script/create_graph.cljs" *command-line-args*)}
+
   dev:db-import
   {:doc "Import a file graph to db graph"
    :requires ([babashka.fs :as fs])
    :task (apply shell {:dir "deps/graph-parser" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
-                "yarn -s nbb-logseq -cp src:../outliner/src:../../scripts/src script/db_import.cljs" *command-line-args*)}
+                "yarn -s nbb-logseq -cp src:../outliner/src script/db_import.cljs" *command-line-args*)}
 
   dev:db-datoms
   {:doc "Write db's datoms to a file"

deps/db/script/create_graph.cljs

@@ -0,0 +1,38 @@
+(ns create-graph
+  "An example script that creates a DB graph given a sqlite.build EDN file"
+  (:require [logseq.outliner.db-pipeline :as db-pipeline]
+            [clojure.string :as string]
+            [clojure.edn :as edn]
+            [datascript.core :as d]
+            ["path" :as node-path]
+            ["os" :as os]
+            ["fs" :as fs]
+            [nbb.classpath :as cp]
+            [nbb.core :as nbb]))
+
+(defn- resolve-path
+  "If relative path, resolve with $ORIGINAL_PWD"
+  [path]
+  (if (node-path/isAbsolute path)
+    path
+    (node-path/join (or js/process.env.ORIGINAL_PWD ".") path)))
+
+(defn -main [args]
+  (when (not= 2 (count args))
+    (println "Usage: $0 GRAPH-DIR EDN-PATH")
+    (js/process.exit 1))
+  (let [[graph-dir edn-path] args
+        [dir db-name] (if (string/includes? graph-dir "/")
+                        ((juxt node-path/dirname node-path/basename) graph-dir)
+                        [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
+        sqlite-build-edn (-> (resolve-path edn-path) fs/readFileSync str edn/read-string)
+        conn (db-pipeline/init-conn dir db-name {:classpath (cp/get-classpath)})
+        {:keys [init-tx block-props-tx]} (db-pipeline/build-blocks-tx sqlite-build-edn)]
+    (println "Generating" (count (filter :block/name init-tx)) "pages and"
+             (count (filter :block/content init-tx)) "blocks ...")
+    (d/transact! conn init-tx)
+    (d/transact! conn block-props-tx)
+    (println "Created graph" (str db-name "!"))))
+
+(when (= nbb/*file* (:file (meta #'-main)))
+  (-main *command-line-args*))

deps/db/script/create_graph/inferred.edn

@@ -0,0 +1,21 @@
+;; Script that generates classes and properties for a demo of inferring properties.
+;; To generate this graph:
+;; bb dev:db-create inferred deps/db/create_graph/inferred.edn
+;;
+;; To try the demo in the UI, in any page type:
+;; - Good Will Hunting #Movie #Ben-Affleck
+;; or
+;; - DB 3 #Meeting #Tienson
+{:auto-create-ontology? true
+ :classes {:Movie {:build/schema-properties [:actor :comment]}
+           :Meeting {:build/schema-properties [:attendee :duration]}}
+ :properties
+ {:actor {:block/schema {:type :object :cardinality :many}
+          :build/schema-classes [:Person]}
+  :attendee {:block/schema {:type :object :cardinality :many}
+             :build/schema-classes [:Person]}}
+ :pages-and-blocks
+ [{:page {:block/original-name "Matt-Damon" :build/tags [:Person]}}
+  {:page {:block/original-name "Ben-Affleck" :build/tags [:Person]}}
+  {:page {:block/original-name "Tienson" :build/tags [:Person]}}
+  {:page {:block/original-name "Zhiyuan" :build/tags [:Person]}}]}

deps/graph-parser/script/db_import.cljs

@@ -12,7 +12,7 @@
             [babashka.cli :as cli]
             [logseq.graph-parser.exporter :as gp-exporter]
             [logseq.common.graph :as common-graph]
-            [logseq.tasks.db-graph.create-graph :as create-graph]
+            [logseq.outliner.db-pipeline :as db-pipeline]
             [promesa.core :as p]))
 
 (defn- build-graph-files
@@ -107,7 +107,7 @@
                           ((juxt node-path/dirname node-path/basename) graph-dir'))
                         [(node-path/join (os/homedir) "logseq" "graphs") db-graph-dir])
         file-graph' (resolve-path file-graph)
-        conn (create-graph/init-conn dir db-name)
+        conn (db-pipeline/init-conn dir db-name)
         directory? (.isDirectory (fs/statSync file-graph'))]
     (p/do!
      (if directory?

deps/outliner/src/logseq/outliner/db_pipeline.cljs

@@ -1,12 +1,18 @@
 (ns ^:node-only logseq.outliner.db-pipeline
-  "This ns provides a datascript listener for DB graphs to add additional changes
+  "This ns provides a datascript listener for DB graphs and helper fns that
-   that the frontend also adds per transact.
+  build on top of it.  The listener adds additional changes that the frontend
-   Missing features from frontend.worker.pipeline including:
+  also adds per transact.  Missing features from frontend.worker.pipeline including:
    * Deleted blocks don't update effected :block/tx-id
    * Delete empty property parent"
-  (:require [datascript.core :as d]
+  (:require [clojure.string :as string]
+            [datascript.core :as d]
+            [logseq.db.sqlite.create-graph :as sqlite-create-graph]
+            [logseq.db.sqlite.build :as sqlite-build]
+            [logseq.db.sqlite.db :as sqlite-db]
+            [logseq.outliner.datascript-report :as ds-report]
             [logseq.outliner.pipeline :as outliner-pipeline]
-            [logseq.outliner.datascript-report :as ds-report]))
+            ["fs" :as fs]
+            ["path" :as node-path]))
 
 
 (defn- rebuild-block-refs
@@ -40,3 +46,41 @@
   [conn]
   (d/listen! conn :pipeline-updates (fn pipeline-updates [tx-report]
                                       (invoke-hooks conn tx-report))))
+
+(defn- find-on-classpath [classpath rel-path]
+  (some (fn [dir]
+          (let [f (node-path/join dir rel-path)]
+            (when (fs/existsSync f) f)))
+        (string/split classpath #":")))
+
+(defn- setup-init-data
+  "Setup initial data same as frontend.handler.repo/create-db"
+  [conn {:keys [additional-config classpath]}]
+  (let [config-content
+        (cond-> (or (some-> (find-on-classpath classpath "templates/config.edn") fs/readFileSync str)
+                    (do (println "Setting graph's config to empty since no templates/config.edn was found.")
+                        "{}"))
+          additional-config
+          ;; TODO: Replace with rewrite-clj when it's available
+          (string/replace-first #"(:file/name-format :triple-lowbar)"
+                                (str "$1 "
+                                     (string/replace-first (str additional-config) #"^\{(.*)\}$" "$1"))))]
+    (d/transact! conn (sqlite-create-graph/build-db-initial-data config-content))))
+
+(defn init-conn
+  "Create sqlite DB, initialize datascript connection and sync listener and then
+  transacts initial data. Takes the following options:
+   * :additional-config - Additional config map to merge into repo config.edn
+   * :classpath - A java classpath string i.e. paths delimited by ':'. Used to find default config.edn
+     that comes with Logseq"
+  [dir db-name & [opts]]
+  (fs/mkdirSync (node-path/join dir db-name) #js {:recursive true})
+  ;; Same order as frontend.db.conn/start!
+  (let [conn (sqlite-db/open-db! dir db-name)]
+    (add-listener conn)
+    (setup-init-data conn opts)
+    conn))
+
+(def build-blocks-tx
+  "An alias for build-blocks-tx to specify default options for this ns"
+  sqlite-build/build-blocks-tx)

docs/dev-practices.md

@@ -387,6 +387,20 @@ These tasks are specific to database graphs. For these tasks there is a one time
   Updated 16 block(s) for graph test-db!
   ```
 
+* `dev:db-create` - Create a DB graph given a `sqlite.build` EDN file
+
+First in Electron, create the name of the graph you want create e.g. `inferred`.
+Then:
+
+```sh
+bb dev:db-create inferred deps/db/script/create_graph/inferred.edn
+Generating 11 pages and 0 blocks ...
+Created graph inferred!
+```
+
+Finally, upload this created graph with the dev command: `Replace graph with`
+... Switch to the graph and you can use the created graph!
+
 * `dev:db-datoms` and `dev:diff-datoms` - Save a db's datoms to file and diff two datom files
 
   ```sh

scripts/README.md

@@ -13,11 +13,11 @@ Before running [nbb-logseq](https://github.com/logseq/nbb-logseq) scripts, be su
 #### Create graph scripts
 
 For database graphs, it is possible to create graphs with the
-[logseq.tasks.db-graph.create-graph](src/logseq/tasks/db_graph/create_graph.cljs)
+[logseq.outliner.db-pipeline](deps/outliner/src/logseq/outliner/db_pipeline.cljs)
-ns. This ns makes it easy to write scripts that create graphs by supporting a
+and [logseq.db.sqlite.build](deps/db/src/logseq/db/sqlite/build.cljs). These
-concise EDN map for graph generation. For example, the
+namespaces makes it easy to write scripts to create graphs with a concise EDN
-`create_graph_with_properties.cljs` script uses this ns to create a graph with
+map. For example, the `create_graph_with_properties.cljs` script uses this ns to
-a variety of properties:
+create a graph with a variety of properties:
 
 ```
 $ yarn nbb-logseq src/logseq/tasks/db_graph/create_graph_with_properties.cljs woot
@@ -28,7 +28,7 @@ Created graph woot!
 This script creates a DB graph with blocks containing several property types for
 both single and many cardinality. It also includes queries for most of these
 properties. Read the docs in
-[logseq.tasks.db-graph.create-graph](src/logseq/tasks/db_graph/create_graph.cljs)
+[logseq.db.sqlite.build](deps/db/src/logseq/db/sqlite/build.cljs)
 for specifics on the EDN map.
 
 To create large graphs with varying size:

scripts/src/logseq/tasks/db_graph/create_graph.cljs

@@ -1,47 +0,0 @@
-(ns logseq.tasks.db-graph.create-graph
-  "This ns provides fns to create a DB graph using EDN. See `init-conn` for
-  initializing a DB graph with a datascript connection that syncs to a sqlite DB
-  at the given directory. See `build-blocks-tx` for the EDN format to create a
-  graph and current limitations"
-  (:require [logseq.db.sqlite.db :as sqlite-db]
-            [logseq.db.sqlite.create-graph :as sqlite-create-graph]
-            [logseq.outliner.db-pipeline :as db-pipeline]
-            [clojure.string :as string]
-            [datascript.core :as d]
-            ["fs" :as fs]
-            ["path" :as node-path]
-            [nbb.classpath :as cp]
-            [logseq.db.sqlite.build :as sqlite-build]))
-
-(defn- find-on-classpath [rel-path]
-  (some (fn [dir]
-          (let [f (node-path/join dir rel-path)]
-            (when (fs/existsSync f) f)))
-        (string/split (cp/get-classpath) #":")))
-
-(defn- setup-init-data
-  "Setup initial data same as frontend.handler.repo/create-db"
-  [conn additional-config]
-  (let [config-content
-        (cond-> (or (some-> (find-on-classpath "templates/config.edn") fs/readFileSync str)
-                    (do (println "Setting graph's config to empty since no templates/config.edn was found.")
-                        "{}"))
-          additional-config
-          ;; TODO: Replace with rewrite-clj when it's available
-          (string/replace-first #"(:file/name-format :triple-lowbar)"
-                                (str "$1 "
-                                     (string/replace-first (str additional-config) #"^\{(.*)\}$" "$1"))))]
-    (d/transact! conn (sqlite-create-graph/build-db-initial-data config-content))))
-
-(defn init-conn
-  "Create sqlite DB, initialize datascript connection and sync listener and then
-  transacts initial data"
-  [dir db-name & {:keys [additional-config]}]
-  (fs/mkdirSync (node-path/join dir db-name) #js {:recursive true})
-  ;; Same order as frontend.db.conn/start!
-  (let [conn (sqlite-db/open-db! dir db-name)]
-    (db-pipeline/add-listener conn)
-    (setup-init-data conn additional-config)
-    conn))
-
-(def build-blocks-tx sqlite-build/build-blocks-tx)

scripts/src/logseq/tasks/db_graph/create_graph_with_inferred_properties.cljs

@@ -1,47 +0,0 @@
-(ns logseq.tasks.db-graph.create-graph-with-inferred-properties
-  "Script that generates classes and properties for a demo of inferring properties.
-   To try the demo, in any page type:
-   - Good Will Hunting #Movie #Ben-Affleck
-   or
-   - DB 3 #Meeting #Tienson"
-  (:require [logseq.tasks.db-graph.create-graph :as create-graph]
-            [logseq.db.sqlite.build :as sqlite-build]
-            [clojure.string :as string]
-            [datascript.core :as d]
-            ["path" :as node-path]
-            ["os" :as os]
-            [nbb.core :as nbb]))
-
-(defn- create-init-data []
-  {:auto-create-ontology? true
-   :classes {:Movie {:build/schema-properties [:actor :comment]}
-             :Meeting {:build/schema-properties [:attendee :duration]}}
-   :properties
-   {:actor {:block/schema {:type :object :cardinality :many}
-            :build/schema-classes [:Person]}
-    :attendee {:block/schema {:type :object :cardinality :many}
-               :build/schema-classes [:Person]}}
-   :pages-and-blocks
-   [{:page {:block/original-name "Matt-Damon" :build/tags [:Person]}}
-    {:page {:block/original-name "Ben-Affleck" :build/tags [:Person]}}
-    {:page {:block/original-name "Tienson" :build/tags [:Person]}}
-    {:page {:block/original-name "Zhiyuan" :build/tags [:Person]}}]})
-
-(defn -main [args]
-  (when (not= 1 (count args))
-    (println "Usage: $0 GRAPH-DIR")
-    (js/process.exit 1))
-  (let [graph-dir (first args)
-        [dir db-name] (if (string/includes? graph-dir "/")
-                        ((juxt node-path/dirname node-path/basename) graph-dir)
-                        [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
-        conn (create-graph/init-conn dir db-name)
-        {:keys [init-tx block-props-tx]} (sqlite-build/build-blocks-tx (create-init-data))]
-    (println "Generating" (count (filter :block/name init-tx)) "pages and"
-             (count (filter :block/content init-tx)) "blocks ...")
-    (d/transact! conn init-tx)
-    (d/transact! conn block-props-tx)
-    (println "Created graph" (str db-name "!"))))
-
-(when (= nbb/*file* (:file (meta #'-main)))
-  (-main *command-line-args*))

scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs

@@ -1,11 +1,12 @@
 (ns logseq.tasks.db-graph.create-graph-with-large-sizes
   "Script that generates graphs at large sizes"
-  (:require [logseq.tasks.db-graph.create-graph :as create-graph]
+  (:require [logseq.outliner.db-pipeline :as db-pipeline]
             [clojure.string :as string]
             [datascript.core :as d]
             [babashka.cli :as cli]
             ["path" :as node-path]
             ["os" :as os]
+            [nbb.classpath :as cp]
             [nbb.core :as nbb]))
 
 (def *ids (atom #{}))
@@ -65,9 +66,9 @@
         [dir db-name] (if (string/includes? graph-dir "/")
                         ((juxt node-path/dirname node-path/basename) graph-dir)
                         [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
-        conn (create-graph/init-conn dir db-name)
+        conn (db-pipeline/init-conn dir db-name {:classpath (cp/get-classpath)})
         _ (println "Building tx ...")
-        {:keys [init-tx]} (create-graph/build-blocks-tx (create-init-data options))]
+        {:keys [init-tx]} (db-pipeline/build-blocks-tx (create-init-data options))]
     (println "Built" (count init-tx) "tx," (count (filter :block/original-name init-tx)) "pages and"
              (count (filter :block/content init-tx)) "blocks ...")
     ;; Vary the chunking with page size up to a max to avoid OOM

scripts/src/logseq/tasks/db_graph/create_graph_with_properties.cljs

@@ -2,7 +2,7 @@
   "Script that generates all the permutations of property types and cardinality.
    Also creates a page of queries that exercises most properties
    NOTE: This script is also used in CI to confirm graph creation works"
-  (:require [logseq.tasks.db-graph.create-graph :as create-graph]
+  (:require [logseq.outliner.db-pipeline :as db-pipeline]
             [logseq.common.util.date-time :as date-time-util]
             [logseq.common.util.page-ref :as page-ref]
             [logseq.db.frontend.property.type :as db-property-type]
@@ -13,6 +13,7 @@
             ["path" :as node-path]
             ["os" :as os]
             [babashka.cli :as cli]
+            [nbb.classpath :as cp]
             [nbb.core :as nbb]))
 
 (defn- date-journal-title [date]
@@ -181,8 +182,9 @@
         [dir db-name] (if (string/includes? graph-dir "/")
                         ((juxt node-path/dirname node-path/basename) graph-dir)
                         [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
-        conn (create-graph/init-conn dir db-name {:additional-config (:config options)})
+        conn (db-pipeline/init-conn dir db-name {:additional-config (:config options)
-        {:keys [init-tx block-props-tx]} (create-graph/build-blocks-tx (create-init-data))
+                                                 :classpath (cp/get-classpath)})
+        {:keys [init-tx block-props-tx]} (db-pipeline/build-blocks-tx (create-init-data))
         existing-names (set (map :v (d/datoms @conn :avet :block/original-name)))
         conflicting-names (set/intersection existing-names (set (keep :block/original-name init-tx)))]
     (when (seq conflicting-names)

scripts/src/logseq/tasks/db_graph/create_graph_with_schema_org.cljs

@@ -10,7 +10,7 @@
      * Some properties are skipped because they are superseded/deprecated or because they have a property
        type logseq doesnt' support yet
      * schema.org assumes no cardinality. For now, only :page properties are given a :cardinality :many"
-  (:require [logseq.tasks.db-graph.create-graph :as create-graph]
+  (:require [logseq.outliner.db-pipeline :as db-pipeline]
             [logseq.common.util :as common-util]
             [logseq.db.frontend.property :as db-property]
             [clojure.string :as string]
@@ -19,6 +19,7 @@
             ["path" :as node-path]
             ["os" :as os]
             ["fs" :as fs]
+            [nbb.classpath :as cp]
             [nbb.core :as nbb]
             [clojure.set :as set]
             [clojure.walk :as w]
@@ -395,10 +396,11 @@
         [dir db-name] (if (string/includes? graph-dir "/")
                         ((juxt node-path/dirname node-path/basename) graph-dir)
                         [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
-        conn (create-graph/init-conn dir db-name {:additional-config (:config options)})
+        conn (db-pipeline/init-conn dir db-name {:additional-config (:config options)
+                                                 :classpath (cp/get-classpath)})
         init-data (create-init-data (d/q '[:find [?name ...] :where [?b :block/name ?name]] @conn)
                                     options)
-        {:keys [init-tx block-props-tx]} (create-graph/build-blocks-tx init-data)]
+        {:keys [init-tx block-props-tx]} (db-pipeline/build-blocks-tx init-data)]
     (println "Generating" (str (count (filter :block/name init-tx)) " pages with "
                                (count (:classes init-data)) " classes and "
                                (count (:properties init-data)) " properties ..."))