From fc9b0d7b2ce59db19136a91973c07e3ef2d4914d Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Mon, 21 Aug 2023 14:54:29 -0400 Subject: [PATCH] Add example update script Split out persist-graph as it is useful outside of just creating graphs. Also updated docstring with more known limitations --- scripts/README.md | 22 ++++++++-- .../logseq/tasks/db_graph/create_graph.cljs | 42 ++++++------------- .../create_graph_with_properties.cljs | 6 ++- .../logseq/tasks/db_graph/persist_graph.cljs | 38 +++++++++++++++++ .../db_graph/update_graph_to_add_todos.cljs | 42 +++++++++++++++++++ 5 files changed, 117 insertions(+), 33 deletions(-) create mode 100644 scripts/src/logseq/tasks/db_graph/persist_graph.cljs create mode 100644 scripts/src/logseq/tasks/db_graph/update_graph_to_add_todos.cljs diff --git a/scripts/README.md b/scripts/README.md index c062aa5b4a..566f293d59 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -8,7 +8,7 @@ Most bb scripts live under `src/` and are defined as bb tasks. See [babashka tas ### Nbb scripts -Before running any [nbb-logseq](https://github.com/logseq/nbb-logseq) scripts, be sure to have node >= 18.14 installed as well as a recent [babashka](https://github.com/babashka/babashka) for managing the dependencies in `nbb.edn`. Then `yarn install` to install dependencies +Before running [nbb-logseq](https://github.com/logseq/nbb-logseq) scripts, be sure to have node >= 18.14 installed as well as a recent [babashka](https://github.com/babashka/babashka) for managing the dependencies in `nbb.edn`. Then `yarn install` to install dependencies #### Create graph scripts @@ -20,7 +20,7 @@ concise EDN map for graph generation. For example, the a variety of properties: ``` -$ yarn nbb-logseq src/logseq/tasks/db_graph/create_graph_with_properties.cljs ~/logseq/graphs/woot +$ yarn nbb-logseq src/logseq/tasks/db_graph/create_graph_with_properties.cljs woot Generating 16 pages and 24 blocks ... Created graph woot! ``` @@ -29,4 +29,20 @@ This script creates a DB graph with blocks containing several property types for both single and many cardinality. It also includes queries for most of these properties. Read the docs in [logseq.tasks.db-graph.create-graph](src/logseq/tasks/db_graph/create_graph.cljs) -for specifics on the EDN map. \ No newline at end of file +for specifics on the EDN map. + +#### Update graph scripts + +For database graphs, it is possible to update graphs with the +[logseq.tasks.db-graph.persist-graph](src/logseq/tasks/db_graph/persist_graph.cljs) +ns. This ns makes it easy to write scripts that update graphs using datascript +and logseq's schema. For example, the `update_graph_to_add_todos.cljs` script +uses this ns to update blocks with a specified query with a `TODO` task marker: + +``` +$ yarn -s nbb-logseq src/logseq/tasks/db_graph/update_graph_to_add_todos.cljs woot '[:find (pull ?b [*]) :where (has-property ?b :url-many)]' +Updated 1 block(s) with a 'TODO' for graph woot! +``` + +This script updates a DB graph by finding all blocks that match the given +property query and marking them as `TODO`. \ No newline at end of file diff --git a/scripts/src/logseq/tasks/db_graph/create_graph.cljs b/scripts/src/logseq/tasks/db_graph/create_graph.cljs index 70d866bef4..c1ad3416ad 100644 --- a/scripts/src/logseq/tasks/db_graph/create_graph.cljs +++ b/scripts/src/logseq/tasks/db_graph/create_graph.cljs @@ -2,41 +2,19 @@ "This ns provides fns to create a DB graph using EDN. See `init-conn` for initializing a DB graph with a datascript connection that syncs to a sqlite DB at the given directory. See `create-blocks-tx` for the EDN format to create a - graph. Note that block creation is limited to top level blocks for now. This - ns can likely be used to also update graphs" + graph and current limitations" (:require [logseq.db.sqlite.db :as sqlite-db] [logseq.db.sqlite.util :as sqlite-util] - [cljs-bean.core :as bean] + [logseq.tasks.db-graph.persist-graph :as persist-graph] [logseq.db :as ldb] [clojure.string :as string] [datascript.core :as d] ["fs" :as fs] ["path" :as node-path] [nbb.classpath :as cp] - ;; TODO: Move these namespaces to more stable deps/ namespaces - [frontend.modules.datascript-report.core :as ds-report] - [frontend.modules.outliner.pipeline-util :as pipeline-util] + ;; TODO: Move this namespace to more stable deps/ namespaces [frontend.handler.common.repo :as repo-common-handler])) -(defn- invoke-hooks - "Modified copy frontend.modules.outliner.pipeline/invoke-hooks that doesn't - handle :block/path-refs recalculation" - [{:keys [db-after] :as tx-report}] - (let [{:keys [blocks]} (ds-report/get-blocks-and-pages tx-report) - deleted-block-uuids (set (pipeline-util/filter-deleted-blocks (:tx-data tx-report))) - upsert-blocks (pipeline-util/build-upsert-blocks blocks deleted-block-uuids db-after)] - {:blocks upsert-blocks - :deleted-block-uuids deleted-block-uuids})) - -(defn- update-sqlite-db - "Modified copy of :db-transact-data defmethod in electron.handler" - [db-name {:keys [blocks deleted-block-uuids]}] - (when (seq deleted-block-uuids) - (sqlite-db/delete-blocks! db-name deleted-block-uuids)) - (when (seq blocks) - (let [blocks' (mapv sqlite-util/ds->sqlite-block blocks)] - (sqlite-db/upsert-blocks! db-name (bean/->js blocks'))))) - (defn- find-on-classpath [rel-path] (some (fn [dir] (let [f (node-path/join dir rel-path)] @@ -61,8 +39,7 @@ (sqlite-db/open-db! dir db-name) ;; Same order as frontend.db.conn/start! (let [conn (ldb/start-conn :create-default-pages? false)] - (d/listen! conn :persist-to-sqlite (fn persist-to-sqlite [tx-report] - (update-sqlite-db db-name (invoke-hooks tx-report)))) + (persist-graph/add-listener conn db-name) (ldb/create-default-pages! conn) (setup-init-data conn) conn)) @@ -145,8 +122,15 @@ (defn create-blocks-tx "Given an EDN map for defining pages, blocks and properties, this creates a - vector of transactable data for use with d/transact!. The EDN map is basic and - only supports defining blocks at the top level. The EDN map has the following keys: + vector of transactable data for use with d/transact!. The blocks that can be created + have the following limitations: + + * Only top level blocks can be easily defined. Other level blocks can be + defined but they require explicit setting of attributes like :block/left and :block/parent + * Block content containing page refs or tags is not supported yet + * Property types :object and :date aren't supported yet + + The EDN map has the following keys: * :pages-and-blocks - This is a vector of maps containing a :page key and optionally a :blocks key when defining a page's blocks. More about each key: diff --git a/scripts/src/logseq/tasks/db_graph/create_graph_with_properties.cljs b/scripts/src/logseq/tasks/db_graph/create_graph_with_properties.cljs index 15032d2b40..659f2629aa 100644 --- a/scripts/src/logseq/tasks/db_graph/create_graph_with_properties.cljs +++ b/scripts/src/logseq/tasks/db_graph/create_graph_with_properties.cljs @@ -5,6 +5,7 @@ [clojure.string :as string] [datascript.core :as d] ["path" :as node-path] + ["os" :as os] [nbb.core :as nbb])) (defn- date-journal-title [date] @@ -67,7 +68,10 @@ (when (not= 1 (count args)) (println "Usage: $0 GRAPH-DIR") (js/process.exit 1)) - (let [[dir db-name] ((juxt node-path/dirname node-path/basename) (first args)) + (let [graph-dir (first args) + [dir db-name] (if (string/includes? graph-dir "/") + ((juxt node-path/dirname node-path/basename) graph-dir) + [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]) conn (create-graph/init-conn dir db-name) blocks-tx (create-graph/create-blocks-tx (create-init-data))] (println "Generating" (count (filter :block/name blocks-tx)) "pages and" diff --git a/scripts/src/logseq/tasks/db_graph/persist_graph.cljs b/scripts/src/logseq/tasks/db_graph/persist_graph.cljs new file mode 100644 index 0000000000..f3100833bd --- /dev/null +++ b/scripts/src/logseq/tasks/db_graph/persist_graph.cljs @@ -0,0 +1,38 @@ +(ns logseq.tasks.db-graph.persist-graph + "This ns allows DB graphs to persist datascript changes to their respective + sqlite db. Since changes are persisted, this can be used to create or update graphs. + Known limitations: + * Changes to block references don't update :block/path-refs" + (:require [datascript.core :as d] + [logseq.db.sqlite.db :as sqlite-db] + [logseq.db.sqlite.util :as sqlite-util] + [cljs-bean.core :as bean] + ;; TODO: Move these namespaces to more stable deps/ namespaces + [frontend.modules.datascript-report.core :as ds-report] + [frontend.modules.outliner.pipeline-util :as pipeline-util])) + +(defn- invoke-hooks + "Modified copy frontend.modules.outliner.pipeline/invoke-hooks that doesn't + handle :block/path-refs recalculation" + [{:keys [db-after] :as tx-report}] + (let [{:keys [blocks]} (ds-report/get-blocks-and-pages tx-report) + deleted-block-uuids (set (pipeline-util/filter-deleted-blocks (:tx-data tx-report))) + upsert-blocks (pipeline-util/build-upsert-blocks blocks deleted-block-uuids db-after)] + {:blocks upsert-blocks + :deleted-block-uuids deleted-block-uuids})) + +(defn- update-sqlite-db + "Modified copy of :db-transact-data defmethod in electron.handler" + [db-name {:keys [blocks deleted-block-uuids]}] + (when (seq deleted-block-uuids) + (sqlite-db/delete-blocks! db-name deleted-block-uuids)) + (when (seq blocks) + (let [blocks' (mapv sqlite-util/ds->sqlite-block blocks)] + (sqlite-db/upsert-blocks! db-name (bean/->js blocks'))))) + +(defn add-listener + "Adds a listener to the datascript connection to persist changes to the given + sqlite db name" + [conn db-name] + (d/listen! conn :persist-to-sqlite (fn persist-to-sqlite [tx-report] + (update-sqlite-db db-name (invoke-hooks tx-report))))) diff --git a/scripts/src/logseq/tasks/db_graph/update_graph_to_add_todos.cljs b/scripts/src/logseq/tasks/db_graph/update_graph_to_add_todos.cljs new file mode 100644 index 0000000000..837a4a2c40 --- /dev/null +++ b/scripts/src/logseq/tasks/db_graph/update_graph_to_add_todos.cljs @@ -0,0 +1,42 @@ +(ns logseq.tasks.db-graph.update-graph-to-add-todos + "This script updates blocks that match the given query and turns them into TODOs" + (:require [logseq.tasks.db-graph.persist-graph :as persist-graph] + [logseq.db.sqlite.cli :as sqlite-cli] + [logseq.db.sqlite.db :as sqlite-db] + [logseq.db.rules :as rules] + [datascript.core :as d] + [clojure.edn :as edn] + [clojure.string :as string] + [nbb.core :as nbb] + ["path" :as node-path] + ["os" :as os])) + +(defn -main [args] + (when (not= 2 (count args)) + (println "Usage: $0 GRAPH-DIR QUERY") + (js/process.exit 1)) + (let [[graph-dir query*] args + [dir db-name] (if (string/includes? graph-dir "/") + ((juxt node-path/dirname node-path/basename) graph-dir) + [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]) + _ (sqlite-db/open-db! dir db-name) + conn (sqlite-cli/read-graph db-name) + ;; find blocks to update + query (into (edn/read-string query*) [:in '$ '%]) ;; assumes no :in are in queries + blocks-to-update (mapv first (d/q query @conn (rules/extract-rules rules/db-query-dsl-rules))) + ;; update + todo-id (or (:db/id (d/entity @conn [:block/name "todo"])) + (throw (ex-info "No :db/id for TODO" {}))) + update-tx (vec (keep #(when-not (:block/marker %) + (hash-map :db/id (:db/id %) + :block/content (str "TODO " (:block/content %)) + :block/marker "TODO" + :block/refs (into [{:db/id todo-id}] (:block/refs %)) + :block/path-refs (into [{:db/id todo-id}] (:block/path-refs %)))) + blocks-to-update))] + (persist-graph/add-listener conn db-name) + (d/transact! conn update-tx) + (println "Updated" (count update-tx) "block(s) with a 'TODO' for graph" (str db-name "!")))) + +(when (= nbb/*file* (:file (meta #'-main))) + (-main *command-line-args*)) \ No newline at end of file