clone/logseq
1
0
Fork 0
mirror of https://github.com/logseq/logseq.git synced 2026-02-01 14:43:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: cleanup bb tasks after file graph removal

Browse Source 
Remove needless 'db-' prefix for several dev tasks. Remove linter for
file and db graph separation. It served us well :)
This commit is contained in:
Gabriel Horner
2026-01-19 11:23:51 -05:00
parent 208b4bc205
commit ed805f039f
8 changed files with 27 additions and 222 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.github/workflows/build.yml vendored
View File

@@ -121,9 +121,6 @@ jobs:
- name: Lint to keep worker independent of frontend - name: Lint to keep worker independent of frontend
run: bb lint:worker-and-frontend-separate run: bb lint:worker-and-frontend-separate
- name: Lint to keep db and file graph code separate
run: bb lint:db-and-file-graphs-separate
db-graph-test: db-graph-test:
strategy: strategy:
matrix: matrix:

17
bb.edn
View File

@@ -71,41 +71,41 @@
{:doc "Run CLI with current deps/db code. Commands with JS deps are not usable e.g. mcp-server" {:doc "Run CLI with current deps/db code. Commands with JS deps are not usable e.g. mcp-server"
:task (apply shell {:dir "deps/db"} :task (apply shell {:dir "deps/db"}
"yarn nbb-logseq -cp src:../cli/src:../graph-parser/src:../outliner/src -m logseq.cli" *command-line-args*)} "yarn nbb-logseq -cp src:../cli/src:../graph-parser/src:../outliner/src -m logseq.cli" *command-line-args*)}
dev:db-query dev:query
{:doc "Query a DB graph's datascript db" {:doc "Query a DB graph's datascript db"
:requires ([babashka.fs :as fs]) :requires ([babashka.fs :as fs])
:task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}} :task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
"yarn -s nbb-logseq script/query.cljs" *command-line-args*)} "yarn -s nbb-logseq script/query.cljs" *command-line-args*)}
dev:db-transact dev:transact
{:doc "Transact against a DB graph's datascript db" {:doc "Transact against a DB graph's datascript db"
:requires ([babashka.fs :as fs]) :requires ([babashka.fs :as fs])
:task (apply shell {:dir "deps/outliner" :extra-env {"ORIGINAL_PWD" (fs/cwd)}} :task (apply shell {:dir "deps/outliner" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
"yarn -s nbb-logseq script/transact.cljs" *command-line-args*)} "yarn -s nbb-logseq script/transact.cljs" *command-line-args*)}
dev:db-create dev:create
{:doc "Create a DB graph given a sqlite.build EDN file" {:doc "Create a DB graph given a sqlite.build EDN file"
:requires ([babashka.fs :as fs]) :requires ([babashka.fs :as fs])
:task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}} :task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
"yarn -s nbb-logseq -cp src:../outliner/src:script script/create_graph.cljs" *command-line-args*)} "yarn -s nbb-logseq -cp src:../outliner/src:script script/create_graph.cljs" *command-line-args*)}
dev:db-diff dev:diff-graphs
{:doc "Diffs two DB graphs" {:doc "Diffs two DB graphs"
:requires ([babashka.fs :as fs]) :requires ([babashka.fs :as fs])
:task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}} :task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
"yarn -s nbb-logseq script/diff_graphs.cljs" *command-line-args*)} "yarn -s nbb-logseq script/diff_graphs.cljs" *command-line-args*)}
dev:db-import dev:import
{:doc "Import a file graph to db graph" {:doc "Import a file graph to db graph"
:requires ([babashka.fs :as fs]) :requires ([babashka.fs :as fs])
:task (apply shell {:dir "deps/graph-parser" :extra-env {"ORIGINAL_PWD" (fs/cwd)}} :task (apply shell {:dir "deps/graph-parser" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
"yarn -s nbb-logseq -cp src:../outliner/src script/db_import.cljs" *command-line-args*)} "yarn -s nbb-logseq -cp src:../outliner/src script/db_import.cljs" *command-line-args*)}
dev:db-import-many dev:import-many
{:doc "Import multiple file graphs to db graphs" {:doc "Import multiple file graphs to db graphs"
:task logseq.tasks.dev/db-import-many} :task logseq.tasks.dev/db-import-many}
dev:db-datoms dev:datoms
{:doc "Write db's datoms to a file" {:doc "Write db's datoms to a file"
:requires ([babashka.fs :as fs]) :requires ([babashka.fs :as fs])
:task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}} :task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
@@ -191,9 +191,6 @@
lint:ns-docstrings lint:ns-docstrings
logseq.bb-tasks.lint.ns-docstrings/-main logseq.bb-tasks.lint.ns-docstrings/-main
lint:db-and-file-graphs-separate
logseq.tasks.dev.db-and-file-graphs/-main
lint:worker-and-frontend-separate lint:worker-and-frontend-separate
logseq.tasks.dev.lint/worker-and-frontend-separate logseq.tasks.dev.lint/worker-and-frontend-separate

4
deps/cli/README.md vendored
View File

@@ -60,8 +60,8 @@ To run a command against the current desktop graph, set `$LOGSEQ_API_SERVER_TOKE
# Search your current graph and print highlighted results one per line like grep # Search your current graph and print highlighted results one per line like grep
$ logseq search woot -a my-token $ logseq search woot -a my-token
Search found 100 results: Search found 100 results:
dev:db-export woot woot.edn && dev:db-create woot2 woot.edn dev:db-export woot woot.edn && dev:create woot2 woot.edn
dev:db-diff woot woot2 dev:diff-graphs woot woot2
... ...
# Can also authenticate api with $LOGSEQ_API_SERVER_TOKEN # Can also authenticate api with $LOGSEQ_API_SERVER_TOKEN
$ export LOGSEQ_API_SERVER_TOKEN=my-token $ export LOGSEQ_API_SERVER_TOKEN=my-token

2
deps/db/script/create_graph/inferred.edn vendored
View File

@@ -1,6 +1,6 @@
;; Script that generates classes and properties for a demo of inferring properties. ;; Script that generates classes and properties for a demo of inferring properties.
;; To generate this graph: ;; To generate this graph:
;; bb dev:db-create inferred deps/db/create_graph/inferred.edn ;; bb dev:create inferred deps/db/create_graph/inferred.edn
;; ;;
;; To try the demo in the UI, in any page type: ;; To try the demo in the UI, in any page type:
;; - Good Will Hunting #Movie #Ben-Affleck ;; - Good Will Hunting #Movie #Ben-Affleck

28
docs/dev-practices.md
View File

@@ -337,42 +337,42 @@ These tasks are specific to database graphs. For these tasks there is a one time
$ cd deps/db && yarn install && cd ../outliner && yarn install && cd ../graph-parser && yarn install && cd ../.. $ cd deps/db && yarn install && cd ../outliner && yarn install && cd ../graph-parser && yarn install && cd ../..
``` ```
* `dev:db-cli` - Run a CLI command from deps/db using latest deps/db code * `dev:db-cli` - Run a CLI command from deps/db using latest deps/db code
* `dev:db-query` - Query a DB graph * `dev:query` - Query a DB graph
```sh ```sh
$ bb dev:db-query woot '[:find (pull ?b [*]) :where (block-content ?b "Dogma")]' $ bb dev:query woot '[:find (pull ?b [*]) :where (block-content ?b "Dogma")]'
DB contains 833 datoms DB contains 833 datoms
[{:block/tx-id 536870923, :block/link #:db{:id 100065}, :block/uuid #uuid "65565c26-f972-4400-bce4-a15df488784d", :block/updated-at 1700158508564, :block/order "a0", :block/refs [#:db{:id 100064}], :block/created-at 1700158502056, :block/tags [#:db{:id 100064}], :block/title "Dogma #[[65565c2a-b1c5-4dc8-a0f0-81b786bc5c6d]]", :db/id 100090, :block/parent #:db{:id 100051}, :block/page #:db{:id 100051}}] [{:block/tx-id 536870923, :block/link #:db{:id 100065}, :block/uuid #uuid "65565c26-f972-4400-bce4-a15df488784d", :block/updated-at 1700158508564, :block/order "a0", :block/refs [#:db{:id 100064}], :block/created-at 1700158502056, :block/tags [#:db{:id 100064}], :block/title "Dogma #[[65565c2a-b1c5-4dc8-a0f0-81b786bc5c6d]]", :db/id 100090, :block/parent #:db{:id 100051}, :block/page #:db{:id 100051}}]
``` ```
* `dev:db-transact` - Run a `d/transact!` against the queried results of a DB graph * `dev:transact` - Run a `d/transact!` against the queried results of a DB graph
```sh ```sh
# The second arg is a datascript like with db-query. The third arg is a fn that is applied to each query result to generate transact data # The second arg is a datascript like with db-query. The third arg is a fn that is applied to each query result to generate transact data
$ bb dev:db-transact $ bb dev:transact
Usage: $0 GRAPH-DIR QUERY TRANSACT-FN Usage: $0 GRAPH-DIR QUERY TRANSACT-FN
# First use the -n flag to see a dry-run of what would happen # First use the -n flag to see a dry-run of what would happen
$ bb dev:db-transact test-db '[:find ?b :where [?b :block/title "say wut"]]' '(fn [id] (vector :db/add id :block/title "say woot!"))' -n $ bb dev:transact test-db '[:find ?b :where [?b :block/title "say wut"]]' '(fn [id] (vector :db/add id :block/title "say woot!"))' -n
Would update 1 blocks with the following tx: Would update 1 blocks with the following tx:
[[:db/add 169 :block/title "say woot!"]] [[:db/add 169 :block/title "say woot!"]]
With the following blocks updated: With the following blocks updated:
(#:block{:title "say wut"}) (#:block{:title "say wut"})
# When the transact looks good, run it without the flag # When the transact looks good, run it without the flag
$ bb dev:db-transact test-db '[:find ?b :where [?b :block/title "say wut"]]' '(fn [id] (vector :db/add id :block/title "say woot!"))' $ bb dev:transact test-db '[:find ?b :where [?b :block/title "say wut"]]' '(fn [id] (vector :db/add id :block/title "say woot!"))'
Updated 1 block(s) for graph test-db! Updated 1 block(s) for graph test-db!
``` ```
Run the dev command `Replace graph with its db.sqlite file` to use the updated graph in the desktop app. Run the dev command `Replace graph with its db.sqlite file` to use the updated graph in the desktop app.
* `dev:db-create` - Create a DB graph given a `sqlite.build` EDN file * `dev:create` - Create a DB graph given a `sqlite.build` EDN file
First in Electron, create the name of the graph you want create e.g. `inferred`. First in Electron, create the name of the graph you want create e.g. `inferred`.
Then: Then:
```sh ```sh
bb dev:db-create inferred deps/db/script/create_graph/inferred.edn bb dev:create inferred deps/db/script/create_graph/inferred.edn
Generating 11 pages and 0 blocks ... Generating 11 pages and 0 blocks ...
Created graph inferred! Created graph inferred!
``` ```
@@ -380,16 +380,16 @@ These tasks are specific to database graphs. For these tasks there is a one time
Finally, upload this created graph with the dev command: `Replace graph with Finally, upload this created graph with the dev command: `Replace graph with
its db.sqlite file`. You'll be switched to the graph and you can use it! its db.sqlite file`. You'll be switched to the graph and you can use it!
* `dev:db-import` and `dev:db-import-many` - Imports a file graph to DB graph, for one or many graphs * `dev:import` and `dev:import-many` - Imports a file graph to DB graph, for one or many graphs
```sh ```sh
# Import the local test graph with the debug option # Import the local test graph with the debug option
$ bb dev:db-import deps/graph-parser/test/resources/exporter-test-graph test-file-graph -d $ bb dev:import deps/graph-parser/test/resources/exporter-test-graph test-file-graph -d
Importing 43 files ... Importing 43 files ...
... ...
# Import and validate multiple file graphs and write them to ./out/ # Import and validate multiple file graphs and write them to ./out/
$ bb dev:db-import-many /path/to/foo /path/to/bar -d $ bb dev:import-many /path/to/foo /path/to/bar -d
Importing ./out/foo ... Importing ./out/foo ...
Importing 321 files ... Importing 321 files ...
Valid! Valid!
@@ -398,13 +398,13 @@ These tasks are specific to database graphs. For these tasks there is a one time
Valid! Valid!
``` ```
* `dev:db-datoms` and `dev:diff-datoms` - Save a db's datoms to file and diff two datom files * `dev:datoms` and `dev:diff-datoms` - Save a db's datoms to file and diff two datom files
```sh ```sh
# Save a current datoms snapshot of a graph # Save a current datoms snapshot of a graph
$ bb dev:db-datoms woot w2.edn $ bb dev:datoms woot w2.edn
# After some edits, save another datoms snapshot # After some edits, save another datoms snapshot
$ bb dev:db-datoms woot w3.edn $ bb dev:datoms woot w3.edn
# Diff the two datom snapshots # Diff the two datom snapshots
# This snapshot correctly shows an added block with content "b7" and a property using a closed :default value # This snapshot correctly shows an added block with content "b7" and a property using a closed :default value

2
scripts/README.md
View File

@@ -14,7 +14,7 @@ Before running [nbb-logseq](https://github.com/logseq/nbb-logseq) scripts, be su
These scripts generate custom database graphs written with nbb-logseq. If this is These scripts generate custom database graphs written with nbb-logseq. If this is
your first time generating a DB graph, first try the your first time generating a DB graph, first try the
`dev:db-create` bb task in [db graph `dev:create` bb task in [db graph
tasks](../docs/dev-practices.md#db-graph-tasks) as it only requires writing EDN. tasks](../docs/dev-practices.md#db-graph-tasks) as it only requires writing EDN.
Creating graphs from the commandline using scripts and a concise EDN map is Creating graphs from the commandline using scripts and a concise EDN map is

4
scripts/src/logseq/tasks/dev.clj
View File

@@ -52,7 +52,7 @@
(spit config-edn config)))) (spit config-edn config))))
(defn diff-datoms (defn diff-datoms
"Runs data/diff on two edn files written by dev:db-datoms" "Runs data/diff on two edn files written by dev:datoms"
[file1 file2 & args] [file1 file2 & args]
(let [spec {:ignored-attributes (let [spec {:ignored-attributes
;; Ignores some attributes by default that are expected to change often ;; Ignores some attributes by default that are expected to change often
@@ -109,4 +109,4 @@
(doseq [file-graph file-graphs] (doseq [file-graph file-graphs]
(let [db-graph (fs/path parent-graph-dir (fs/file-name file-graph))] (let [db-graph (fs/path parent-graph-dir (fs/file-name file-graph))]
(println "Importing" (str db-graph) "...") (println "Importing" (str db-graph) "...")
(apply shell "bb" "dev:db-import" file-graph db-graph (concat import-options ["--validate"])))))) (apply shell "bb" "dev:import" file-graph db-graph (concat import-options ["--validate"]))))))

189
scripts/src/logseq/tasks/dev/db_and_file_graphs.clj
View File

@@ -1,189 +0,0 @@
(ns logseq.tasks.dev.db-and-file-graphs
(:require [babashka.process :refer [shell]]
[clojure.set :as set]
[clojure.string :as string]))
(defn- escape-shell-regex
[s]
(reduce (fn [acc escape-char]
(string/replace acc escape-char (str "\\" escape-char)))
s
["." "?"]))
(def db-graph-ns
"Namespaces or parent namespaces _only_ for DB graphs. Use a '.' at end of a namespace for parent namespaces"
(mapv escape-shell-regex
["logseq.db.sqlite." "logseq.db.frontend."
"logseq.outliner.property" "logseq.outliner.validate" "logseq.outliner.page" "logseq.outliner.cli" "logseq.outliner.db-pipeline"
"logseq.api.db-based"
"logseq.cli"
"electron.db"
"frontend.handler.db-based."
"frontend.inference-worker"
"frontend.components.property" "frontend.components.class" "frontend.components.quick-add" "frontend.components.vector-search"
"frontend.components.db-based" "frontend.components.objects" "frontend.components.query.view"
"mobile.core" "mobile.events" "mobile.externals" "mobile.init" "mobile.state"
"mobile.components"]))
(def file-graph-ns
"Namespaces or parent namespaces _only_ for file graphs"
(mapv escape-shell-regex
["logseq.graph-parser.db" "logseq.graph-parser.property" "logseq.graph-parser.extract"
"logseq.api.file-based"
"frontend.handler.file-based" "frontend.handler.file-sync"
"frontend.db.file-based"
"frontend.util.file-based"
"frontend.common.file-based"
"frontend.worker.handler.page.file-based"
;; Want to only specify this ns and not the ones under it but don't have a way yet
"frontend.worker.file"
"frontend.fs"
"frontend.components.file-sync"
"frontend.components.file-based"]))
(def block-name-db-graph-paths
"DB graph paths with :block/name"
["deps/db/src/logseq/db/frontend"
"deps/db/src/logseq/db/sqlite"
"deps/outliner/src/logseq/outliner/property.cljs"
"deps/outliner/src/logseq/outliner/page.cljs"])
(def db-graph-paths
"Paths _only_ for DB graphs"
(into block-name-db-graph-paths
["deps/outliner/src/logseq/outliner/cli.cljs"
"deps/outliner/src/logseq/outliner/db_pipeline.cljs"
"deps/outliner/src/logseq/outliner/validate.cljs"
"deps/outliner/src/logseq/outliner/page.cljs"
;; TODO: change to deps/cli/src when :block/name no longer in other cli namespaces
"deps/cli/src/logseq/cli/commands"
"src/main/frontend/handler/db_based"
"src/main/frontend/components/class.cljs"
"src/main/frontend/components/property.cljs"
"src/main/frontend/components/property"
"src/main/frontend/components/objects.cljs"
"src/main/frontend/components/quick_add.cljs"
"src/main/frontend/components/vector_search"
"src/main/frontend/components/db_based"
"src/main/frontend/components/query/view.cljs"
"src/main/frontend/inference_worker"
"src/main/logseq/api/db_based.cljs"
"src/main/logseq/api/db_based"
"src/electron/electron/db.cljs"
"src/main/mobile"]))
(def file-graph-paths
"Paths _only_ for file graphs"
["deps/graph-parser/src/logseq/graph_parser/extract.cljc"
"deps/graph-parser/src/logseq/graph_parser/property.cljs"
"src/main/frontend/fs"])
(defn- grep-many
"Git greps a coll of patterns for given paths. Returns result from process/shell"
[patterns paths]
(apply shell {:out :string :continue true}
"git grep -E" (str "(" (string/join "|" patterns) ")")
paths))
(defn- validate-db-ns-not-in-file
[]
(let [res (grep-many db-graph-ns file-graph-paths)]
(when-not (or (and (= 1 (:exit res)) (= "" (:out res)))
;; TODO: Refactor logseq.cli.common.file to not have file-based code
(= (:out res) "src/main/frontend/worker/file.cljs: [logseq.cli.common.file :as common-file]\n"))
(println "The following db graph namespaces should not be in file graph files:")
(println (:out res))
(System/exit 1))))
(defn- validate-file-ns-not-in-db
[]
(let [res (grep-many file-graph-ns db-graph-paths)]
(when-not (and (= 1 (:exit res)) (= "" (:out res)))
(println "The following file graph namespaces should not be in db graph files:")
(println (:out res))
(System/exit 1))))
(defn- validate-multi-graph-fns-not-in-file-or-db
[]
(let [multi-graph-fns ["/db-based-graph\\?" "\\(db-based-graph\\?"
;; from frontend.handler.property.util
"/get-pid"
"logseq.db.common.property-util"
;; Use file-entity-util and entity-util when in a single graph context
"ldb/whiteboard\\?" "ldb/journal\\?" "ldb/page\\?"]
res (grep-many multi-graph-fns (into file-graph-paths db-graph-paths))]
(when-not (and (= 1 (:exit res)) (= "" (:out res)))
(println "The following files should not have fns meant to be used in multi-graph contexts:")
(println (:out res))
(System/exit 1))))
(defn- validate-file-concepts-not-in-db
[]
(let [file-concepts (->>
;; from logseq.db.file-based.schema
[:block/namespace :block/properties-text-values :block/pre-block :recent/pages :block/file :block/properties-order
:block/repeated :block/deadline :block/scheduled :block/priority :block/marker :block/macros
:block/type :block/format]
(map str)
(into [;; e.g. block/properties :title
"block/properties :"
;; anything org mode except for org.babashka or urls like schema.org
"[^\\.]org[^\\.]"
"file-based"
"#+BEGIN_"
"#+END_"
"pre-block"]))
;; For now use the whole code line. If this is too brittle can make this smaller
allowed-exceptions #{":block/pre-block? :block/scheduled :block/deadline :block/type :block/name :block/marker"
"(dissoc :block/format))]"
"{:block/name page-title})"
;; TODO: Mv this file-based ns out of db file
"[logseq.db.file-based.schema :as file-schema]))"
;; :block/name ones from src/main/mobile
"(if-let [journal (db/get-page page-name)]"
"(p/then #(mobile-state/open-block-modal! (db/get-page page-name)))))))]"}
res (grep-many file-concepts db-graph-paths)
invalid-lines (when (= 0 (:exit res))
(remove #(some->> (string/split % #":\s+") second string/trim (contains? allowed-exceptions))
(string/split-lines (:out res))))
_ (when (> (:exit res) 1) (System/exit 1))
_ (when (and (= 0 (:exit res)) (seq invalid-lines))
(println "The following files should not have contained file specific concepts:")
(println (string/join "\n" invalid-lines))
(System/exit 1))
;; :block/name isn't used in db graphs except for fns with journal or internal-page
block-name-file-concepts #{"block/name" "/page-name-sanity-lc" "db/get-page "}
no-block-name-db-graph-paths (set/difference (set db-graph-paths) (set block-name-db-graph-paths))
block-name-res (grep-many block-name-file-concepts no-block-name-db-graph-paths)
block-name-invalid-lines (when (= 0 (:exit block-name-res))
(remove #(some->> (string/split % #":\s+") second string/trim (contains? allowed-exceptions))
(string/split-lines (:out block-name-res))))]
(when (> (:exit block-name-res) 1) (System/exit 1))
(when (and (= 0 (:exit block-name-res)) (seq block-name-invalid-lines))
(println "The following files should not have contained file specific concepts:")
(println (string/join "\n" block-name-invalid-lines))
(System/exit 1))))
(defn- validate-db-concepts-not-in-file
[]
(let [db-concepts
;; from logseq.db.frontend.schema
["closed-value" "class/properties" "classes" "property/parent"
"logseq.property" "logseq.class" "db-based" "library" "quick-add"]
res (grep-many db-concepts file-graph-paths)]
(when-not (and (= 1 (:exit res)) (= "" (:out res)))
(println "The following files should not have contained db specific concepts:")
(println (:out res))
(System/exit 1))))
(defn -main
"Check that file and db graph specific namespaces and concepts are separate"
[]
(validate-db-ns-not-in-file)
(validate-file-ns-not-in-db)
(validate-file-concepts-not-in-db)
(validate-db-concepts-not-in-file)
(validate-multi-graph-fns-not-in-file-or-db)
(println "✅ All checks passed!"))