enhance: bb task for creating graphs from EDN file

Converted inferred graph to an EDN file now that this task exists. Also
merge last of tasks.create-graph to relevant ns so that external users
can also create such tasks

scripts/src/logseq/tasks/db_graph/create_graph.cljs

@@ -1,47 +0,0 @@
-(ns logseq.tasks.db-graph.create-graph
-  "This ns provides fns to create a DB graph using EDN. See `init-conn` for
-  initializing a DB graph with a datascript connection that syncs to a sqlite DB
-  at the given directory. See `build-blocks-tx` for the EDN format to create a
-  graph and current limitations"
-  (:require [logseq.db.sqlite.db :as sqlite-db]
-            [logseq.db.sqlite.create-graph :as sqlite-create-graph]
-            [logseq.outliner.db-pipeline :as db-pipeline]
-            [clojure.string :as string]
-            [datascript.core :as d]
-            ["fs" :as fs]
-            ["path" :as node-path]
-            [nbb.classpath :as cp]
-            [logseq.db.sqlite.build :as sqlite-build]))
-
-(defn- find-on-classpath [rel-path]
-  (some (fn [dir]
-          (let [f (node-path/join dir rel-path)]
-            (when (fs/existsSync f) f)))
-        (string/split (cp/get-classpath) #":")))
-
-(defn- setup-init-data
-  "Setup initial data same as frontend.handler.repo/create-db"
-  [conn additional-config]
-  (let [config-content
-        (cond-> (or (some-> (find-on-classpath "templates/config.edn") fs/readFileSync str)
-                    (do (println "Setting graph's config to empty since no templates/config.edn was found.")
-                        "{}"))
-          additional-config
-          ;; TODO: Replace with rewrite-clj when it's available
-          (string/replace-first #"(:file/name-format :triple-lowbar)"
-                                (str "$1 "
-                                     (string/replace-first (str additional-config) #"^\{(.*)\}$" "$1"))))]
-    (d/transact! conn (sqlite-create-graph/build-db-initial-data config-content))))
-
-(defn init-conn
-  "Create sqlite DB, initialize datascript connection and sync listener and then
-  transacts initial data"
-  [dir db-name & {:keys [additional-config]}]
-  (fs/mkdirSync (node-path/join dir db-name) #js {:recursive true})
-  ;; Same order as frontend.db.conn/start!
-  (let [conn (sqlite-db/open-db! dir db-name)]
-    (db-pipeline/add-listener conn)
-    (setup-init-data conn additional-config)
-    conn))
-
-(def build-blocks-tx sqlite-build/build-blocks-tx)

scripts/src/logseq/tasks/db_graph/create_graph_with_inferred_properties.cljs

@@ -1,47 +0,0 @@
-(ns logseq.tasks.db-graph.create-graph-with-inferred-properties
-  "Script that generates classes and properties for a demo of inferring properties.
-   To try the demo, in any page type:
-   - Good Will Hunting #Movie #Ben-Affleck
-   or
-   - DB 3 #Meeting #Tienson"
-  (:require [logseq.tasks.db-graph.create-graph :as create-graph]
-            [logseq.db.sqlite.build :as sqlite-build]
-            [clojure.string :as string]
-            [datascript.core :as d]
-            ["path" :as node-path]
-            ["os" :as os]
-            [nbb.core :as nbb]))
-
-(defn- create-init-data []
-  {:auto-create-ontology? true
-   :classes {:Movie {:build/schema-properties [:actor :comment]}
-             :Meeting {:build/schema-properties [:attendee :duration]}}
-   :properties
-   {:actor {:block/schema {:type :object :cardinality :many}
-            :build/schema-classes [:Person]}
-    :attendee {:block/schema {:type :object :cardinality :many}
-               :build/schema-classes [:Person]}}
-   :pages-and-blocks
-   [{:page {:block/original-name "Matt-Damon" :build/tags [:Person]}}
-    {:page {:block/original-name "Ben-Affleck" :build/tags [:Person]}}
-    {:page {:block/original-name "Tienson" :build/tags [:Person]}}
-    {:page {:block/original-name "Zhiyuan" :build/tags [:Person]}}]})
-
-(defn -main [args]
-  (when (not= 1 (count args))
-    (println "Usage: $0 GRAPH-DIR")
-    (js/process.exit 1))
-  (let [graph-dir (first args)
-        [dir db-name] (if (string/includes? graph-dir "/")
-                        ((juxt node-path/dirname node-path/basename) graph-dir)
-                        [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
-        conn (create-graph/init-conn dir db-name)
-        {:keys [init-tx block-props-tx]} (sqlite-build/build-blocks-tx (create-init-data))]
-    (println "Generating" (count (filter :block/name init-tx)) "pages and"
-             (count (filter :block/content init-tx)) "blocks ...")
-    (d/transact! conn init-tx)
-    (d/transact! conn block-props-tx)
-    (println "Created graph" (str db-name "!"))))
-
-(when (= nbb/*file* (:file (meta #'-main)))
-  (-main *command-line-args*))

scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs

@@ -1,11 +1,12 @@
 (ns logseq.tasks.db-graph.create-graph-with-large-sizes
   "Script that generates graphs at large sizes"
-  (:require [logseq.tasks.db-graph.create-graph :as create-graph]
+  (:require [logseq.outliner.db-pipeline :as db-pipeline]
             [clojure.string :as string]
             [datascript.core :as d]
             [babashka.cli :as cli]
             ["path" :as node-path]
             ["os" :as os]
+            [nbb.classpath :as cp]
             [nbb.core :as nbb]))
 
 (def *ids (atom #{}))
@@ -65,9 +66,9 @@
         [dir db-name] (if (string/includes? graph-dir "/")
                         ((juxt node-path/dirname node-path/basename) graph-dir)
                         [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
-        conn (create-graph/init-conn dir db-name)
+        conn (db-pipeline/init-conn dir db-name {:classpath (cp/get-classpath)})
         _ (println "Building tx ...")
-        {:keys [init-tx]} (create-graph/build-blocks-tx (create-init-data options))]
+        {:keys [init-tx]} (db-pipeline/build-blocks-tx (create-init-data options))]
     (println "Built" (count init-tx) "tx," (count (filter :block/original-name init-tx)) "pages and"
              (count (filter :block/content init-tx)) "blocks ...")
     ;; Vary the chunking with page size up to a max to avoid OOM

scripts/src/logseq/tasks/db_graph/create_graph_with_properties.cljs

@@ -2,7 +2,7 @@
   "Script that generates all the permutations of property types and cardinality.
    Also creates a page of queries that exercises most properties
    NOTE: This script is also used in CI to confirm graph creation works"
-  (:require [logseq.tasks.db-graph.create-graph :as create-graph]
+  (:require [logseq.outliner.db-pipeline :as db-pipeline]
             [logseq.common.util.date-time :as date-time-util]
             [logseq.common.util.page-ref :as page-ref]
             [logseq.db.frontend.property.type :as db-property-type]
@@ -13,6 +13,7 @@
             ["path" :as node-path]
             ["os" :as os]
             [babashka.cli :as cli]
+            [nbb.classpath :as cp]
             [nbb.core :as nbb]))
 
 (defn- date-journal-title [date]
@@ -181,8 +182,9 @@
         [dir db-name] (if (string/includes? graph-dir "/")
                         ((juxt node-path/dirname node-path/basename) graph-dir)
                         [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
-        conn (create-graph/init-conn dir db-name {:additional-config (:config options)})
-        {:keys [init-tx block-props-tx]} (create-graph/build-blocks-tx (create-init-data))
+        conn (db-pipeline/init-conn dir db-name {:additional-config (:config options)
+                                                 :classpath (cp/get-classpath)})
+        {:keys [init-tx block-props-tx]} (db-pipeline/build-blocks-tx (create-init-data))
         existing-names (set (map :v (d/datoms @conn :avet :block/original-name)))
         conflicting-names (set/intersection existing-names (set (keep :block/original-name init-tx)))]
     (when (seq conflicting-names)

scripts/src/logseq/tasks/db_graph/create_graph_with_schema_org.cljs

@@ -10,7 +10,7 @@
      * Some properties are skipped because they are superseded/deprecated or because they have a property
        type logseq doesnt' support yet
      * schema.org assumes no cardinality. For now, only :page properties are given a :cardinality :many"
-  (:require [logseq.tasks.db-graph.create-graph :as create-graph]
+  (:require [logseq.outliner.db-pipeline :as db-pipeline]
             [logseq.common.util :as common-util]
             [logseq.db.frontend.property :as db-property]
             [clojure.string :as string]
@@ -19,6 +19,7 @@
             ["path" :as node-path]
             ["os" :as os]
             ["fs" :as fs]
+            [nbb.classpath :as cp]
             [nbb.core :as nbb]
             [clojure.set :as set]
             [clojure.walk :as w]
@@ -395,10 +396,11 @@
         [dir db-name] (if (string/includes? graph-dir "/")
                         ((juxt node-path/dirname node-path/basename) graph-dir)
                         [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
-        conn (create-graph/init-conn dir db-name {:additional-config (:config options)})
+        conn (db-pipeline/init-conn dir db-name {:additional-config (:config options)
+                                                 :classpath (cp/get-classpath)})
         init-data (create-init-data (d/q '[:find [?name ...] :where [?b :block/name ?name]] @conn)
                                     options)
-        {:keys [init-tx block-props-tx]} (create-graph/build-blocks-tx init-data)]
+        {:keys [init-tx block-props-tx]} (db-pipeline/build-blocks-tx init-data)]
     (println "Generating" (str (count (filter :block/name init-tx)) " pages with "
                                (count (:classes init-data)) " classes and "
                                (count (:properties init-data)) " properties ..."))