From 0e1ada2ef6aee35dd3f7deaec40cd4c16abd29de Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Tue, 11 Jun 2024 10:03:02 -0400 Subject: [PATCH] enhance: bb task for creating graphs from EDN file Converted inferred graph to an EDN file now that this task exists. Also merge last of tasks.create-graph to relevant ns so that external users can also create such tasks --- bb.edn | 8 ++- deps/db/script/create_graph.cljs | 38 +++++++++++++ deps/db/script/create_graph/inferred.edn | 21 ++++++++ deps/graph-parser/script/db_import.cljs | 4 +- .../src/logseq/outliner/db_pipeline.cljs | 54 +++++++++++++++++-- docs/dev-practices.md | 14 +++++ scripts/README.md | 12 ++--- .../logseq/tasks/db_graph/create_graph.cljs | 47 ---------------- ...create_graph_with_inferred_properties.cljs | 47 ---------------- .../create_graph_with_large_sizes.cljs | 7 +-- .../create_graph_with_properties.cljs | 8 +-- .../create_graph_with_schema_org.cljs | 8 +-- 12 files changed, 151 insertions(+), 117 deletions(-) create mode 100644 deps/db/script/create_graph.cljs create mode 100644 deps/db/script/create_graph/inferred.edn delete mode 100644 scripts/src/logseq/tasks/db_graph/create_graph.cljs delete mode 100644 scripts/src/logseq/tasks/db_graph/create_graph_with_inferred_properties.cljs diff --git a/bb.edn b/bb.edn index 08019ec2f9..2f00bcfdac 100644 --- a/bb.edn +++ b/bb.edn @@ -76,11 +76,17 @@ {:doc "Transact against a DB graph's datascript db" :task (apply shell {:dir "deps/outliner"} "yarn -s nbb-logseq script/transact.cljs" *command-line-args*)} + dev:db-create + {:doc "Create a DB graph given a sqlite.build EDN file" + :requires ([babashka.fs :as fs]) + :task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}} + "yarn -s nbb-logseq -cp src:../outliner/src script/create_graph.cljs" *command-line-args*)} + dev:db-import {:doc "Import a file graph to db graph" :requires ([babashka.fs :as fs]) :task (apply shell {:dir "deps/graph-parser" :extra-env {"ORIGINAL_PWD" (fs/cwd)}} - "yarn -s nbb-logseq -cp src:../outliner/src:../../scripts/src script/db_import.cljs" *command-line-args*)} + "yarn -s nbb-logseq -cp src:../outliner/src script/db_import.cljs" *command-line-args*)} dev:db-datoms {:doc "Write db's datoms to a file" diff --git a/deps/db/script/create_graph.cljs b/deps/db/script/create_graph.cljs new file mode 100644 index 0000000000..1215c1e53b --- /dev/null +++ b/deps/db/script/create_graph.cljs @@ -0,0 +1,38 @@ +(ns create-graph + "An example script that creates a DB graph given a sqlite.build EDN file" + (:require [logseq.outliner.db-pipeline :as db-pipeline] + [clojure.string :as string] + [clojure.edn :as edn] + [datascript.core :as d] + ["path" :as node-path] + ["os" :as os] + ["fs" :as fs] + [nbb.classpath :as cp] + [nbb.core :as nbb])) + +(defn- resolve-path + "If relative path, resolve with $ORIGINAL_PWD" + [path] + (if (node-path/isAbsolute path) + path + (node-path/join (or js/process.env.ORIGINAL_PWD ".") path))) + +(defn -main [args] + (when (not= 2 (count args)) + (println "Usage: $0 GRAPH-DIR EDN-PATH") + (js/process.exit 1)) + (let [[graph-dir edn-path] args + [dir db-name] (if (string/includes? graph-dir "/") + ((juxt node-path/dirname node-path/basename) graph-dir) + [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]) + sqlite-build-edn (-> (resolve-path edn-path) fs/readFileSync str edn/read-string) + conn (db-pipeline/init-conn dir db-name {:classpath (cp/get-classpath)}) + {:keys [init-tx block-props-tx]} (db-pipeline/build-blocks-tx sqlite-build-edn)] + (println "Generating" (count (filter :block/name init-tx)) "pages and" + (count (filter :block/content init-tx)) "blocks ...") + (d/transact! conn init-tx) + (d/transact! conn block-props-tx) + (println "Created graph" (str db-name "!")))) + +(when (= nbb/*file* (:file (meta #'-main))) + (-main *command-line-args*)) diff --git a/deps/db/script/create_graph/inferred.edn b/deps/db/script/create_graph/inferred.edn new file mode 100644 index 0000000000..7b292a3ee8 --- /dev/null +++ b/deps/db/script/create_graph/inferred.edn @@ -0,0 +1,21 @@ +;; Script that generates classes and properties for a demo of inferring properties. +;; To generate this graph: +;; bb dev:db-create inferred deps/db/create_graph/inferred.edn +;; +;; To try the demo in the UI, in any page type: +;; - Good Will Hunting #Movie #Ben-Affleck +;; or +;; - DB 3 #Meeting #Tienson +{:auto-create-ontology? true + :classes {:Movie {:build/schema-properties [:actor :comment]} + :Meeting {:build/schema-properties [:attendee :duration]}} + :properties + {:actor {:block/schema {:type :object :cardinality :many} + :build/schema-classes [:Person]} + :attendee {:block/schema {:type :object :cardinality :many} + :build/schema-classes [:Person]}} + :pages-and-blocks + [{:page {:block/original-name "Matt-Damon" :build/tags [:Person]}} + {:page {:block/original-name "Ben-Affleck" :build/tags [:Person]}} + {:page {:block/original-name "Tienson" :build/tags [:Person]}} + {:page {:block/original-name "Zhiyuan" :build/tags [:Person]}}]} \ No newline at end of file diff --git a/deps/graph-parser/script/db_import.cljs b/deps/graph-parser/script/db_import.cljs index 8353923cab..38364688f1 100644 --- a/deps/graph-parser/script/db_import.cljs +++ b/deps/graph-parser/script/db_import.cljs @@ -12,7 +12,7 @@ [babashka.cli :as cli] [logseq.graph-parser.exporter :as gp-exporter] [logseq.common.graph :as common-graph] - [logseq.tasks.db-graph.create-graph :as create-graph] + [logseq.outliner.db-pipeline :as db-pipeline] [promesa.core :as p])) (defn- build-graph-files @@ -107,7 +107,7 @@ ((juxt node-path/dirname node-path/basename) graph-dir')) [(node-path/join (os/homedir) "logseq" "graphs") db-graph-dir]) file-graph' (resolve-path file-graph) - conn (create-graph/init-conn dir db-name) + conn (db-pipeline/init-conn dir db-name) directory? (.isDirectory (fs/statSync file-graph'))] (p/do! (if directory? diff --git a/deps/outliner/src/logseq/outliner/db_pipeline.cljs b/deps/outliner/src/logseq/outliner/db_pipeline.cljs index c3836671c5..0c48ef04e1 100644 --- a/deps/outliner/src/logseq/outliner/db_pipeline.cljs +++ b/deps/outliner/src/logseq/outliner/db_pipeline.cljs @@ -1,12 +1,18 @@ (ns ^:node-only logseq.outliner.db-pipeline - "This ns provides a datascript listener for DB graphs to add additional changes - that the frontend also adds per transact. - Missing features from frontend.worker.pipeline including: + "This ns provides a datascript listener for DB graphs and helper fns that + build on top of it. The listener adds additional changes that the frontend + also adds per transact. Missing features from frontend.worker.pipeline including: * Deleted blocks don't update effected :block/tx-id * Delete empty property parent" - (:require [datascript.core :as d] + (:require [clojure.string :as string] + [datascript.core :as d] + [logseq.db.sqlite.create-graph :as sqlite-create-graph] + [logseq.db.sqlite.build :as sqlite-build] + [logseq.db.sqlite.db :as sqlite-db] + [logseq.outliner.datascript-report :as ds-report] [logseq.outliner.pipeline :as outliner-pipeline] - [logseq.outliner.datascript-report :as ds-report])) + ["fs" :as fs] + ["path" :as node-path])) (defn- rebuild-block-refs @@ -40,3 +46,41 @@ [conn] (d/listen! conn :pipeline-updates (fn pipeline-updates [tx-report] (invoke-hooks conn tx-report)))) + +(defn- find-on-classpath [classpath rel-path] + (some (fn [dir] + (let [f (node-path/join dir rel-path)] + (when (fs/existsSync f) f))) + (string/split classpath #":"))) + +(defn- setup-init-data + "Setup initial data same as frontend.handler.repo/create-db" + [conn {:keys [additional-config classpath]}] + (let [config-content + (cond-> (or (some-> (find-on-classpath classpath "templates/config.edn") fs/readFileSync str) + (do (println "Setting graph's config to empty since no templates/config.edn was found.") + "{}")) + additional-config + ;; TODO: Replace with rewrite-clj when it's available + (string/replace-first #"(:file/name-format :triple-lowbar)" + (str "$1 " + (string/replace-first (str additional-config) #"^\{(.*)\}$" "$1"))))] + (d/transact! conn (sqlite-create-graph/build-db-initial-data config-content)))) + +(defn init-conn + "Create sqlite DB, initialize datascript connection and sync listener and then + transacts initial data. Takes the following options: + * :additional-config - Additional config map to merge into repo config.edn + * :classpath - A java classpath string i.e. paths delimited by ':'. Used to find default config.edn + that comes with Logseq" + [dir db-name & [opts]] + (fs/mkdirSync (node-path/join dir db-name) #js {:recursive true}) + ;; Same order as frontend.db.conn/start! + (let [conn (sqlite-db/open-db! dir db-name)] + (add-listener conn) + (setup-init-data conn opts) + conn)) + +(def build-blocks-tx + "An alias for build-blocks-tx to specify default options for this ns" + sqlite-build/build-blocks-tx) \ No newline at end of file diff --git a/docs/dev-practices.md b/docs/dev-practices.md index 04df55b925..74a0041d66 100644 --- a/docs/dev-practices.md +++ b/docs/dev-practices.md @@ -387,6 +387,20 @@ These tasks are specific to database graphs. For these tasks there is a one time Updated 16 block(s) for graph test-db! ``` +* `dev:db-create` - Create a DB graph given a `sqlite.build` EDN file + +First in Electron, create the name of the graph you want create e.g. `inferred`. +Then: + +```sh +bb dev:db-create inferred deps/db/script/create_graph/inferred.edn +Generating 11 pages and 0 blocks ... +Created graph inferred! +``` + +Finally, upload this created graph with the dev command: `Replace graph with` +... Switch to the graph and you can use the created graph! + * `dev:db-datoms` and `dev:diff-datoms` - Save a db's datoms to file and diff two datom files ```sh diff --git a/scripts/README.md b/scripts/README.md index 3b43110213..86eb98f64a 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -13,11 +13,11 @@ Before running [nbb-logseq](https://github.com/logseq/nbb-logseq) scripts, be su #### Create graph scripts For database graphs, it is possible to create graphs with the -[logseq.tasks.db-graph.create-graph](src/logseq/tasks/db_graph/create_graph.cljs) -ns. This ns makes it easy to write scripts that create graphs by supporting a -concise EDN map for graph generation. For example, the -`create_graph_with_properties.cljs` script uses this ns to create a graph with -a variety of properties: +[logseq.outliner.db-pipeline](deps/outliner/src/logseq/outliner/db_pipeline.cljs) +and [logseq.db.sqlite.build](deps/db/src/logseq/db/sqlite/build.cljs). These +namespaces makes it easy to write scripts to create graphs with a concise EDN +map. For example, the `create_graph_with_properties.cljs` script uses this ns to +create a graph with a variety of properties: ``` $ yarn nbb-logseq src/logseq/tasks/db_graph/create_graph_with_properties.cljs woot @@ -28,7 +28,7 @@ Created graph woot! This script creates a DB graph with blocks containing several property types for both single and many cardinality. It also includes queries for most of these properties. Read the docs in -[logseq.tasks.db-graph.create-graph](src/logseq/tasks/db_graph/create_graph.cljs) +[logseq.db.sqlite.build](deps/db/src/logseq/db/sqlite/build.cljs) for specifics on the EDN map. To create large graphs with varying size: diff --git a/scripts/src/logseq/tasks/db_graph/create_graph.cljs b/scripts/src/logseq/tasks/db_graph/create_graph.cljs deleted file mode 100644 index c4f55dd1c0..0000000000 --- a/scripts/src/logseq/tasks/db_graph/create_graph.cljs +++ /dev/null @@ -1,47 +0,0 @@ -(ns logseq.tasks.db-graph.create-graph - "This ns provides fns to create a DB graph using EDN. See `init-conn` for - initializing a DB graph with a datascript connection that syncs to a sqlite DB - at the given directory. See `build-blocks-tx` for the EDN format to create a - graph and current limitations" - (:require [logseq.db.sqlite.db :as sqlite-db] - [logseq.db.sqlite.create-graph :as sqlite-create-graph] - [logseq.outliner.db-pipeline :as db-pipeline] - [clojure.string :as string] - [datascript.core :as d] - ["fs" :as fs] - ["path" :as node-path] - [nbb.classpath :as cp] - [logseq.db.sqlite.build :as sqlite-build])) - -(defn- find-on-classpath [rel-path] - (some (fn [dir] - (let [f (node-path/join dir rel-path)] - (when (fs/existsSync f) f))) - (string/split (cp/get-classpath) #":"))) - -(defn- setup-init-data - "Setup initial data same as frontend.handler.repo/create-db" - [conn additional-config] - (let [config-content - (cond-> (or (some-> (find-on-classpath "templates/config.edn") fs/readFileSync str) - (do (println "Setting graph's config to empty since no templates/config.edn was found.") - "{}")) - additional-config - ;; TODO: Replace with rewrite-clj when it's available - (string/replace-first #"(:file/name-format :triple-lowbar)" - (str "$1 " - (string/replace-first (str additional-config) #"^\{(.*)\}$" "$1"))))] - (d/transact! conn (sqlite-create-graph/build-db-initial-data config-content)))) - -(defn init-conn - "Create sqlite DB, initialize datascript connection and sync listener and then - transacts initial data" - [dir db-name & {:keys [additional-config]}] - (fs/mkdirSync (node-path/join dir db-name) #js {:recursive true}) - ;; Same order as frontend.db.conn/start! - (let [conn (sqlite-db/open-db! dir db-name)] - (db-pipeline/add-listener conn) - (setup-init-data conn additional-config) - conn)) - -(def build-blocks-tx sqlite-build/build-blocks-tx) \ No newline at end of file diff --git a/scripts/src/logseq/tasks/db_graph/create_graph_with_inferred_properties.cljs b/scripts/src/logseq/tasks/db_graph/create_graph_with_inferred_properties.cljs deleted file mode 100644 index 467608146b..0000000000 --- a/scripts/src/logseq/tasks/db_graph/create_graph_with_inferred_properties.cljs +++ /dev/null @@ -1,47 +0,0 @@ -(ns logseq.tasks.db-graph.create-graph-with-inferred-properties - "Script that generates classes and properties for a demo of inferring properties. - To try the demo, in any page type: - - Good Will Hunting #Movie #Ben-Affleck - or - - DB 3 #Meeting #Tienson" - (:require [logseq.tasks.db-graph.create-graph :as create-graph] - [logseq.db.sqlite.build :as sqlite-build] - [clojure.string :as string] - [datascript.core :as d] - ["path" :as node-path] - ["os" :as os] - [nbb.core :as nbb])) - -(defn- create-init-data [] - {:auto-create-ontology? true - :classes {:Movie {:build/schema-properties [:actor :comment]} - :Meeting {:build/schema-properties [:attendee :duration]}} - :properties - {:actor {:block/schema {:type :object :cardinality :many} - :build/schema-classes [:Person]} - :attendee {:block/schema {:type :object :cardinality :many} - :build/schema-classes [:Person]}} - :pages-and-blocks - [{:page {:block/original-name "Matt-Damon" :build/tags [:Person]}} - {:page {:block/original-name "Ben-Affleck" :build/tags [:Person]}} - {:page {:block/original-name "Tienson" :build/tags [:Person]}} - {:page {:block/original-name "Zhiyuan" :build/tags [:Person]}}]}) - -(defn -main [args] - (when (not= 1 (count args)) - (println "Usage: $0 GRAPH-DIR") - (js/process.exit 1)) - (let [graph-dir (first args) - [dir db-name] (if (string/includes? graph-dir "/") - ((juxt node-path/dirname node-path/basename) graph-dir) - [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]) - conn (create-graph/init-conn dir db-name) - {:keys [init-tx block-props-tx]} (sqlite-build/build-blocks-tx (create-init-data))] - (println "Generating" (count (filter :block/name init-tx)) "pages and" - (count (filter :block/content init-tx)) "blocks ...") - (d/transact! conn init-tx) - (d/transact! conn block-props-tx) - (println "Created graph" (str db-name "!")))) - -(when (= nbb/*file* (:file (meta #'-main))) - (-main *command-line-args*)) diff --git a/scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs b/scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs index 0673080dcf..abd780c49c 100644 --- a/scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs +++ b/scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs @@ -1,11 +1,12 @@ (ns logseq.tasks.db-graph.create-graph-with-large-sizes "Script that generates graphs at large sizes" - (:require [logseq.tasks.db-graph.create-graph :as create-graph] + (:require [logseq.outliner.db-pipeline :as db-pipeline] [clojure.string :as string] [datascript.core :as d] [babashka.cli :as cli] ["path" :as node-path] ["os" :as os] + [nbb.classpath :as cp] [nbb.core :as nbb])) (def *ids (atom #{})) @@ -65,9 +66,9 @@ [dir db-name] (if (string/includes? graph-dir "/") ((juxt node-path/dirname node-path/basename) graph-dir) [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]) - conn (create-graph/init-conn dir db-name) + conn (db-pipeline/init-conn dir db-name {:classpath (cp/get-classpath)}) _ (println "Building tx ...") - {:keys [init-tx]} (create-graph/build-blocks-tx (create-init-data options))] + {:keys [init-tx]} (db-pipeline/build-blocks-tx (create-init-data options))] (println "Built" (count init-tx) "tx," (count (filter :block/original-name init-tx)) "pages and" (count (filter :block/content init-tx)) "blocks ...") ;; Vary the chunking with page size up to a max to avoid OOM diff --git a/scripts/src/logseq/tasks/db_graph/create_graph_with_properties.cljs b/scripts/src/logseq/tasks/db_graph/create_graph_with_properties.cljs index 47f7fa4ea9..ccd0863750 100644 --- a/scripts/src/logseq/tasks/db_graph/create_graph_with_properties.cljs +++ b/scripts/src/logseq/tasks/db_graph/create_graph_with_properties.cljs @@ -2,7 +2,7 @@ "Script that generates all the permutations of property types and cardinality. Also creates a page of queries that exercises most properties NOTE: This script is also used in CI to confirm graph creation works" - (:require [logseq.tasks.db-graph.create-graph :as create-graph] + (:require [logseq.outliner.db-pipeline :as db-pipeline] [logseq.common.util.date-time :as date-time-util] [logseq.common.util.page-ref :as page-ref] [logseq.db.frontend.property.type :as db-property-type] @@ -13,6 +13,7 @@ ["path" :as node-path] ["os" :as os] [babashka.cli :as cli] + [nbb.classpath :as cp] [nbb.core :as nbb])) (defn- date-journal-title [date] @@ -181,8 +182,9 @@ [dir db-name] (if (string/includes? graph-dir "/") ((juxt node-path/dirname node-path/basename) graph-dir) [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]) - conn (create-graph/init-conn dir db-name {:additional-config (:config options)}) - {:keys [init-tx block-props-tx]} (create-graph/build-blocks-tx (create-init-data)) + conn (db-pipeline/init-conn dir db-name {:additional-config (:config options) + :classpath (cp/get-classpath)}) + {:keys [init-tx block-props-tx]} (db-pipeline/build-blocks-tx (create-init-data)) existing-names (set (map :v (d/datoms @conn :avet :block/original-name))) conflicting-names (set/intersection existing-names (set (keep :block/original-name init-tx)))] (when (seq conflicting-names) diff --git a/scripts/src/logseq/tasks/db_graph/create_graph_with_schema_org.cljs b/scripts/src/logseq/tasks/db_graph/create_graph_with_schema_org.cljs index ea5773c72e..21ecfa352e 100644 --- a/scripts/src/logseq/tasks/db_graph/create_graph_with_schema_org.cljs +++ b/scripts/src/logseq/tasks/db_graph/create_graph_with_schema_org.cljs @@ -10,7 +10,7 @@ * Some properties are skipped because they are superseded/deprecated or because they have a property type logseq doesnt' support yet * schema.org assumes no cardinality. For now, only :page properties are given a :cardinality :many" - (:require [logseq.tasks.db-graph.create-graph :as create-graph] + (:require [logseq.outliner.db-pipeline :as db-pipeline] [logseq.common.util :as common-util] [logseq.db.frontend.property :as db-property] [clojure.string :as string] @@ -19,6 +19,7 @@ ["path" :as node-path] ["os" :as os] ["fs" :as fs] + [nbb.classpath :as cp] [nbb.core :as nbb] [clojure.set :as set] [clojure.walk :as w] @@ -395,10 +396,11 @@ [dir db-name] (if (string/includes? graph-dir "/") ((juxt node-path/dirname node-path/basename) graph-dir) [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]) - conn (create-graph/init-conn dir db-name {:additional-config (:config options)}) + conn (db-pipeline/init-conn dir db-name {:additional-config (:config options) + :classpath (cp/get-classpath)}) init-data (create-init-data (d/q '[:find [?name ...] :where [?b :block/name ?name]] @conn) options) - {:keys [init-tx block-props-tx]} (create-graph/build-blocks-tx init-data)] + {:keys [init-tx block-props-tx]} (db-pipeline/build-blocks-tx init-data)] (println "Generating" (str (count (filter :block/name init-tx)) " pages with " (count (:classes init-data)) " classes and " (count (:properties init-data)) " properties ..."))