From ed805f039ffec8f3533d6dda686e173546d139a1 Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Mon, 19 Jan 2026 11:23:51 -0500 Subject: [PATCH] chore: cleanup bb tasks after file graph removal Remove needless 'db-' prefix for several dev tasks. Remove linter for file and db graph separation. It served us well :) --- .github/workflows/build.yml | 3 - bb.edn | 17 +- deps/cli/README.md | 4 +- deps/db/script/create_graph/inferred.edn | 2 +- docs/dev-practices.md | 28 +-- scripts/README.md | 2 +- scripts/src/logseq/tasks/dev.clj | 4 +- .../logseq/tasks/dev/db_and_file_graphs.clj | 189 ------------------ 8 files changed, 27 insertions(+), 222 deletions(-) delete mode 100644 scripts/src/logseq/tasks/dev/db_and_file_graphs.clj diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c0fdfd36db..7e67b3b3d7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -121,9 +121,6 @@ jobs: - name: Lint to keep worker independent of frontend run: bb lint:worker-and-frontend-separate - - name: Lint to keep db and file graph code separate - run: bb lint:db-and-file-graphs-separate - db-graph-test: strategy: matrix: diff --git a/bb.edn b/bb.edn index d6165e086c..360ef27d3b 100644 --- a/bb.edn +++ b/bb.edn @@ -71,41 +71,41 @@ {:doc "Run CLI with current deps/db code. Commands with JS deps are not usable e.g. mcp-server" :task (apply shell {:dir "deps/db"} "yarn nbb-logseq -cp src:../cli/src:../graph-parser/src:../outliner/src -m logseq.cli" *command-line-args*)} - dev:db-query + dev:query {:doc "Query a DB graph's datascript db" :requires ([babashka.fs :as fs]) :task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}} "yarn -s nbb-logseq script/query.cljs" *command-line-args*)} - dev:db-transact + dev:transact {:doc "Transact against a DB graph's datascript db" :requires ([babashka.fs :as fs]) :task (apply shell {:dir "deps/outliner" :extra-env {"ORIGINAL_PWD" (fs/cwd)}} "yarn -s nbb-logseq script/transact.cljs" *command-line-args*)} - dev:db-create + dev:create {:doc "Create a DB graph given a sqlite.build EDN file" :requires ([babashka.fs :as fs]) :task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}} "yarn -s nbb-logseq -cp src:../outliner/src:script script/create_graph.cljs" *command-line-args*)} - dev:db-diff + dev:diff-graphs {:doc "Diffs two DB graphs" :requires ([babashka.fs :as fs]) :task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}} "yarn -s nbb-logseq script/diff_graphs.cljs" *command-line-args*)} - dev:db-import + dev:import {:doc "Import a file graph to db graph" :requires ([babashka.fs :as fs]) :task (apply shell {:dir "deps/graph-parser" :extra-env {"ORIGINAL_PWD" (fs/cwd)}} "yarn -s nbb-logseq -cp src:../outliner/src script/db_import.cljs" *command-line-args*)} - dev:db-import-many + dev:import-many {:doc "Import multiple file graphs to db graphs" :task logseq.tasks.dev/db-import-many} - dev:db-datoms + dev:datoms {:doc "Write db's datoms to a file" :requires ([babashka.fs :as fs]) :task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}} @@ -191,9 +191,6 @@ lint:ns-docstrings logseq.bb-tasks.lint.ns-docstrings/-main - lint:db-and-file-graphs-separate - logseq.tasks.dev.db-and-file-graphs/-main - lint:worker-and-frontend-separate logseq.tasks.dev.lint/worker-and-frontend-separate diff --git a/deps/cli/README.md b/deps/cli/README.md index 1ef9cfe271..9226c8635e 100644 --- a/deps/cli/README.md +++ b/deps/cli/README.md @@ -60,8 +60,8 @@ To run a command against the current desktop graph, set `$LOGSEQ_API_SERVER_TOKE # Search your current graph and print highlighted results one per line like grep $ logseq search woot -a my-token Search found 100 results: -dev:db-export woot woot.edn && dev:db-create woot2 woot.edn -dev:db-diff woot woot2 +dev:db-export woot woot.edn && dev:create woot2 woot.edn +dev:diff-graphs woot woot2 ... # Can also authenticate api with $LOGSEQ_API_SERVER_TOKEN $ export LOGSEQ_API_SERVER_TOKEN=my-token diff --git a/deps/db/script/create_graph/inferred.edn b/deps/db/script/create_graph/inferred.edn index ab98b42499..e86d8c2874 100644 --- a/deps/db/script/create_graph/inferred.edn +++ b/deps/db/script/create_graph/inferred.edn @@ -1,6 +1,6 @@ ;; Script that generates classes and properties for a demo of inferring properties. ;; To generate this graph: -;; bb dev:db-create inferred deps/db/create_graph/inferred.edn +;; bb dev:create inferred deps/db/create_graph/inferred.edn ;; ;; To try the demo in the UI, in any page type: ;; - Good Will Hunting #Movie #Ben-Affleck diff --git a/docs/dev-practices.md b/docs/dev-practices.md index fde4492fb5..f861a5c153 100644 --- a/docs/dev-practices.md +++ b/docs/dev-practices.md @@ -337,42 +337,42 @@ These tasks are specific to database graphs. For these tasks there is a one time $ cd deps/db && yarn install && cd ../outliner && yarn install && cd ../graph-parser && yarn install && cd ../.. ``` * `dev:db-cli` - Run a CLI command from deps/db using latest deps/db code -* `dev:db-query` - Query a DB graph +* `dev:query` - Query a DB graph ```sh - $ bb dev:db-query woot '[:find (pull ?b [*]) :where (block-content ?b "Dogma")]' + $ bb dev:query woot '[:find (pull ?b [*]) :where (block-content ?b "Dogma")]' DB contains 833 datoms [{:block/tx-id 536870923, :block/link #:db{:id 100065}, :block/uuid #uuid "65565c26-f972-4400-bce4-a15df488784d", :block/updated-at 1700158508564, :block/order "a0", :block/refs [#:db{:id 100064}], :block/created-at 1700158502056, :block/tags [#:db{:id 100064}], :block/title "Dogma #[[65565c2a-b1c5-4dc8-a0f0-81b786bc5c6d]]", :db/id 100090, :block/parent #:db{:id 100051}, :block/page #:db{:id 100051}}] ``` -* `dev:db-transact` - Run a `d/transact!` against the queried results of a DB graph +* `dev:transact` - Run a `d/transact!` against the queried results of a DB graph ```sh # The second arg is a datascript like with db-query. The third arg is a fn that is applied to each query result to generate transact data - $ bb dev:db-transact + $ bb dev:transact Usage: $0 GRAPH-DIR QUERY TRANSACT-FN # First use the -n flag to see a dry-run of what would happen - $ bb dev:db-transact test-db '[:find ?b :where [?b :block/title "say wut"]]' '(fn [id] (vector :db/add id :block/title "say woot!"))' -n + $ bb dev:transact test-db '[:find ?b :where [?b :block/title "say wut"]]' '(fn [id] (vector :db/add id :block/title "say woot!"))' -n Would update 1 blocks with the following tx: [[:db/add 169 :block/title "say woot!"]] With the following blocks updated: (#:block{:title "say wut"}) # When the transact looks good, run it without the flag - $ bb dev:db-transact test-db '[:find ?b :where [?b :block/title "say wut"]]' '(fn [id] (vector :db/add id :block/title "say woot!"))' + $ bb dev:transact test-db '[:find ?b :where [?b :block/title "say wut"]]' '(fn [id] (vector :db/add id :block/title "say woot!"))' Updated 1 block(s) for graph test-db! ``` Run the dev command `Replace graph with its db.sqlite file` to use the updated graph in the desktop app. -* `dev:db-create` - Create a DB graph given a `sqlite.build` EDN file +* `dev:create` - Create a DB graph given a `sqlite.build` EDN file First in Electron, create the name of the graph you want create e.g. `inferred`. Then: ```sh - bb dev:db-create inferred deps/db/script/create_graph/inferred.edn + bb dev:create inferred deps/db/script/create_graph/inferred.edn Generating 11 pages and 0 blocks ... Created graph inferred! ``` @@ -380,16 +380,16 @@ These tasks are specific to database graphs. For these tasks there is a one time Finally, upload this created graph with the dev command: `Replace graph with its db.sqlite file`. You'll be switched to the graph and you can use it! -* `dev:db-import` and `dev:db-import-many` - Imports a file graph to DB graph, for one or many graphs +* `dev:import` and `dev:import-many` - Imports a file graph to DB graph, for one or many graphs ```sh # Import the local test graph with the debug option - $ bb dev:db-import deps/graph-parser/test/resources/exporter-test-graph test-file-graph -d + $ bb dev:import deps/graph-parser/test/resources/exporter-test-graph test-file-graph -d Importing 43 files ... ... # Import and validate multiple file graphs and write them to ./out/ - $ bb dev:db-import-many /path/to/foo /path/to/bar -d + $ bb dev:import-many /path/to/foo /path/to/bar -d Importing ./out/foo ... Importing 321 files ... Valid! @@ -398,13 +398,13 @@ These tasks are specific to database graphs. For these tasks there is a one time Valid! ``` -* `dev:db-datoms` and `dev:diff-datoms` - Save a db's datoms to file and diff two datom files +* `dev:datoms` and `dev:diff-datoms` - Save a db's datoms to file and diff two datom files ```sh # Save a current datoms snapshot of a graph - $ bb dev:db-datoms woot w2.edn + $ bb dev:datoms woot w2.edn # After some edits, save another datoms snapshot - $ bb dev:db-datoms woot w3.edn + $ bb dev:datoms woot w3.edn # Diff the two datom snapshots # This snapshot correctly shows an added block with content "b7" and a property using a closed :default value diff --git a/scripts/README.md b/scripts/README.md index 50af3a01e0..0b76a8ef8f 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -14,7 +14,7 @@ Before running [nbb-logseq](https://github.com/logseq/nbb-logseq) scripts, be su These scripts generate custom database graphs written with nbb-logseq. If this is your first time generating a DB graph, first try the -`dev:db-create` bb task in [db graph +`dev:create` bb task in [db graph tasks](../docs/dev-practices.md#db-graph-tasks) as it only requires writing EDN. Creating graphs from the commandline using scripts and a concise EDN map is diff --git a/scripts/src/logseq/tasks/dev.clj b/scripts/src/logseq/tasks/dev.clj index b8fa77226c..ce01dd9478 100644 --- a/scripts/src/logseq/tasks/dev.clj +++ b/scripts/src/logseq/tasks/dev.clj @@ -52,7 +52,7 @@ (spit config-edn config)))) (defn diff-datoms - "Runs data/diff on two edn files written by dev:db-datoms" + "Runs data/diff on two edn files written by dev:datoms" [file1 file2 & args] (let [spec {:ignored-attributes ;; Ignores some attributes by default that are expected to change often @@ -109,4 +109,4 @@ (doseq [file-graph file-graphs] (let [db-graph (fs/path parent-graph-dir (fs/file-name file-graph))] (println "Importing" (str db-graph) "...") - (apply shell "bb" "dev:db-import" file-graph db-graph (concat import-options ["--validate"])))))) + (apply shell "bb" "dev:import" file-graph db-graph (concat import-options ["--validate"])))))) diff --git a/scripts/src/logseq/tasks/dev/db_and_file_graphs.clj b/scripts/src/logseq/tasks/dev/db_and_file_graphs.clj deleted file mode 100644 index e6ae3f0615..0000000000 --- a/scripts/src/logseq/tasks/dev/db_and_file_graphs.clj +++ /dev/null @@ -1,189 +0,0 @@ -(ns logseq.tasks.dev.db-and-file-graphs - (:require [babashka.process :refer [shell]] - [clojure.set :as set] - [clojure.string :as string])) - -(defn- escape-shell-regex - [s] - (reduce (fn [acc escape-char] - (string/replace acc escape-char (str "\\" escape-char))) - s - ["." "?"])) - -(def db-graph-ns - "Namespaces or parent namespaces _only_ for DB graphs. Use a '.' at end of a namespace for parent namespaces" - (mapv escape-shell-regex - ["logseq.db.sqlite." "logseq.db.frontend." - "logseq.outliner.property" "logseq.outliner.validate" "logseq.outliner.page" "logseq.outliner.cli" "logseq.outliner.db-pipeline" - "logseq.api.db-based" - "logseq.cli" - "electron.db" - "frontend.handler.db-based." - "frontend.inference-worker" - "frontend.components.property" "frontend.components.class" "frontend.components.quick-add" "frontend.components.vector-search" - "frontend.components.db-based" "frontend.components.objects" "frontend.components.query.view" - "mobile.core" "mobile.events" "mobile.externals" "mobile.init" "mobile.state" - "mobile.components"])) - -(def file-graph-ns - "Namespaces or parent namespaces _only_ for file graphs" - (mapv escape-shell-regex - ["logseq.graph-parser.db" "logseq.graph-parser.property" "logseq.graph-parser.extract" - "logseq.api.file-based" - "frontend.handler.file-based" "frontend.handler.file-sync" - "frontend.db.file-based" - "frontend.util.file-based" - "frontend.common.file-based" - "frontend.worker.handler.page.file-based" - ;; Want to only specify this ns and not the ones under it but don't have a way yet - "frontend.worker.file" - "frontend.fs" - "frontend.components.file-sync" - "frontend.components.file-based"])) - -(def block-name-db-graph-paths - "DB graph paths with :block/name" - ["deps/db/src/logseq/db/frontend" - "deps/db/src/logseq/db/sqlite" - "deps/outliner/src/logseq/outliner/property.cljs" - "deps/outliner/src/logseq/outliner/page.cljs"]) - -(def db-graph-paths - "Paths _only_ for DB graphs" - (into block-name-db-graph-paths - ["deps/outliner/src/logseq/outliner/cli.cljs" - "deps/outliner/src/logseq/outliner/db_pipeline.cljs" - "deps/outliner/src/logseq/outliner/validate.cljs" - "deps/outliner/src/logseq/outliner/page.cljs" - ;; TODO: change to deps/cli/src when :block/name no longer in other cli namespaces - "deps/cli/src/logseq/cli/commands" - "src/main/frontend/handler/db_based" - "src/main/frontend/components/class.cljs" - "src/main/frontend/components/property.cljs" - "src/main/frontend/components/property" - "src/main/frontend/components/objects.cljs" - "src/main/frontend/components/quick_add.cljs" - "src/main/frontend/components/vector_search" - "src/main/frontend/components/db_based" - "src/main/frontend/components/query/view.cljs" - "src/main/frontend/inference_worker" - "src/main/logseq/api/db_based.cljs" - "src/main/logseq/api/db_based" - "src/electron/electron/db.cljs" - "src/main/mobile"])) - -(def file-graph-paths - "Paths _only_ for file graphs" - ["deps/graph-parser/src/logseq/graph_parser/extract.cljc" - "deps/graph-parser/src/logseq/graph_parser/property.cljs" - "src/main/frontend/fs"]) - -(defn- grep-many - "Git greps a coll of patterns for given paths. Returns result from process/shell" - [patterns paths] - (apply shell {:out :string :continue true} - "git grep -E" (str "(" (string/join "|" patterns) ")") - paths)) - -(defn- validate-db-ns-not-in-file - [] - (let [res (grep-many db-graph-ns file-graph-paths)] - (when-not (or (and (= 1 (:exit res)) (= "" (:out res))) - ;; TODO: Refactor logseq.cli.common.file to not have file-based code - (= (:out res) "src/main/frontend/worker/file.cljs: [logseq.cli.common.file :as common-file]\n")) - (println "The following db graph namespaces should not be in file graph files:") - (println (:out res)) - (System/exit 1)))) - -(defn- validate-file-ns-not-in-db - [] - (let [res (grep-many file-graph-ns db-graph-paths)] - (when-not (and (= 1 (:exit res)) (= "" (:out res))) - (println "The following file graph namespaces should not be in db graph files:") - (println (:out res)) - (System/exit 1)))) - -(defn- validate-multi-graph-fns-not-in-file-or-db - [] - (let [multi-graph-fns ["/db-based-graph\\?" "\\(db-based-graph\\?" - ;; from frontend.handler.property.util - "/get-pid" - "logseq.db.common.property-util" - ;; Use file-entity-util and entity-util when in a single graph context - "ldb/whiteboard\\?" "ldb/journal\\?" "ldb/page\\?"] - res (grep-many multi-graph-fns (into file-graph-paths db-graph-paths))] - (when-not (and (= 1 (:exit res)) (= "" (:out res))) - (println "The following files should not have fns meant to be used in multi-graph contexts:") - (println (:out res)) - (System/exit 1)))) - -(defn- validate-file-concepts-not-in-db - [] - (let [file-concepts (->> - ;; from logseq.db.file-based.schema - [:block/namespace :block/properties-text-values :block/pre-block :recent/pages :block/file :block/properties-order - :block/repeated :block/deadline :block/scheduled :block/priority :block/marker :block/macros - :block/type :block/format] - (map str) - (into [;; e.g. block/properties :title - "block/properties :" - ;; anything org mode except for org.babashka or urls like schema.org - "[^\\.]org[^\\.]" - "file-based" - "#+BEGIN_" - "#+END_" - "pre-block"])) - ;; For now use the whole code line. If this is too brittle can make this smaller - allowed-exceptions #{":block/pre-block? :block/scheduled :block/deadline :block/type :block/name :block/marker" - "(dissoc :block/format))]" - "{:block/name page-title})" - ;; TODO: Mv this file-based ns out of db file - "[logseq.db.file-based.schema :as file-schema]))" - ;; :block/name ones from src/main/mobile - "(if-let [journal (db/get-page page-name)]" - "(p/then #(mobile-state/open-block-modal! (db/get-page page-name)))))))]"} - res (grep-many file-concepts db-graph-paths) - invalid-lines (when (= 0 (:exit res)) - (remove #(some->> (string/split % #":\s+") second string/trim (contains? allowed-exceptions)) - (string/split-lines (:out res)))) - _ (when (> (:exit res) 1) (System/exit 1)) - _ (when (and (= 0 (:exit res)) (seq invalid-lines)) - (println "The following files should not have contained file specific concepts:") - (println (string/join "\n" invalid-lines)) - (System/exit 1)) - - ;; :block/name isn't used in db graphs except for fns with journal or internal-page - block-name-file-concepts #{"block/name" "/page-name-sanity-lc" "db/get-page "} - no-block-name-db-graph-paths (set/difference (set db-graph-paths) (set block-name-db-graph-paths)) - block-name-res (grep-many block-name-file-concepts no-block-name-db-graph-paths) - block-name-invalid-lines (when (= 0 (:exit block-name-res)) - (remove #(some->> (string/split % #":\s+") second string/trim (contains? allowed-exceptions)) - (string/split-lines (:out block-name-res))))] - - (when (> (:exit block-name-res) 1) (System/exit 1)) - (when (and (= 0 (:exit block-name-res)) (seq block-name-invalid-lines)) - (println "The following files should not have contained file specific concepts:") - (println (string/join "\n" block-name-invalid-lines)) - (System/exit 1)))) - -(defn- validate-db-concepts-not-in-file - [] - (let [db-concepts - ;; from logseq.db.frontend.schema - ["closed-value" "class/properties" "classes" "property/parent" - "logseq.property" "logseq.class" "db-based" "library" "quick-add"] - res (grep-many db-concepts file-graph-paths)] - (when-not (and (= 1 (:exit res)) (= "" (:out res))) - (println "The following files should not have contained db specific concepts:") - (println (:out res)) - (System/exit 1)))) - -(defn -main - "Check that file and db graph specific namespaces and concepts are separate" - [] - (validate-db-ns-not-in-file) - (validate-file-ns-not-in-db) - (validate-file-concepts-not-in-db) - (validate-db-concepts-not-in-file) - (validate-multi-graph-fns-not-in-file-or-db) - (println "✅ All checks passed!"))