mirror of
https://github.com/logseq/logseq.git
synced 2026-04-24 22:25:01 +00:00
chore: cleanup bb tasks after file graph removal
Remove needless 'db-' prefix for several dev tasks. Remove linter for file and db graph separation. It served us well :)
This commit is contained in:
Gabriel Horner
@@ -14,7 +14,7 @@ Before running [nbb-logseq](https://github.com/logseq/nbb-logseq) scripts, be su
|
||||
|
||||
These scripts generate custom database graphs written with nbb-logseq. If this is
|
||||
your first time generating a DB graph, first try the
|
||||
`dev:db-create` bb task in [db graph
|
||||
`dev:create` bb task in [db graph
|
||||
tasks](../docs/dev-practices.md#db-graph-tasks) as it only requires writing EDN.
|
||||
|
||||
Creating graphs from the commandline using scripts and a concise EDN map is
|
||||
|
||||
@@ -52,7 +52,7 @@
|
||||
(spit config-edn config))))
|
||||
|
||||
(defn diff-datoms
|
||||
"Runs data/diff on two edn files written by dev:db-datoms"
|
||||
"Runs data/diff on two edn files written by dev:datoms"
|
||||
[file1 file2 & args]
|
||||
(let [spec {:ignored-attributes
|
||||
;; Ignores some attributes by default that are expected to change often
|
||||
@@ -109,4 +109,4 @@
|
||||
(doseq [file-graph file-graphs]
|
||||
(let [db-graph (fs/path parent-graph-dir (fs/file-name file-graph))]
|
||||
(println "Importing" (str db-graph) "...")
|
||||
(apply shell "bb" "dev:db-import" file-graph db-graph (concat import-options ["--validate"]))))))
|
||||
(apply shell "bb" "dev:import" file-graph db-graph (concat import-options ["--validate"]))))))
|
||||
|
||||
@@ -1,189 +0,0 @@
|
||||
(ns logseq.tasks.dev.db-and-file-graphs
|
||||
(:require [babashka.process :refer [shell]]
|
||||
[clojure.set :as set]
|
||||
[clojure.string :as string]))
|
||||
|
||||
(defn- escape-shell-regex
|
||||
[s]
|
||||
(reduce (fn [acc escape-char]
|
||||
(string/replace acc escape-char (str "\\" escape-char)))
|
||||
s
|
||||
["." "?"]))
|
||||
|
||||
(def db-graph-ns
|
||||
"Namespaces or parent namespaces _only_ for DB graphs. Use a '.' at end of a namespace for parent namespaces"
|
||||
(mapv escape-shell-regex
|
||||
["logseq.db.sqlite." "logseq.db.frontend."
|
||||
"logseq.outliner.property" "logseq.outliner.validate" "logseq.outliner.page" "logseq.outliner.cli" "logseq.outliner.db-pipeline"
|
||||
"logseq.api.db-based"
|
||||
"logseq.cli"
|
||||
"electron.db"
|
||||
"frontend.handler.db-based."
|
||||
"frontend.inference-worker"
|
||||
"frontend.components.property" "frontend.components.class" "frontend.components.quick-add" "frontend.components.vector-search"
|
||||
"frontend.components.db-based" "frontend.components.objects" "frontend.components.query.view"
|
||||
"mobile.core" "mobile.events" "mobile.externals" "mobile.init" "mobile.state"
|
||||
"mobile.components"]))
|
||||
|
||||
(def file-graph-ns
|
||||
"Namespaces or parent namespaces _only_ for file graphs"
|
||||
(mapv escape-shell-regex
|
||||
["logseq.graph-parser.db" "logseq.graph-parser.property" "logseq.graph-parser.extract"
|
||||
"logseq.api.file-based"
|
||||
"frontend.handler.file-based" "frontend.handler.file-sync"
|
||||
"frontend.db.file-based"
|
||||
"frontend.util.file-based"
|
||||
"frontend.common.file-based"
|
||||
"frontend.worker.handler.page.file-based"
|
||||
;; Want to only specify this ns and not the ones under it but don't have a way yet
|
||||
"frontend.worker.file"
|
||||
"frontend.fs"
|
||||
"frontend.components.file-sync"
|
||||
"frontend.components.file-based"]))
|
||||
|
||||
(def block-name-db-graph-paths
|
||||
"DB graph paths with :block/name"
|
||||
["deps/db/src/logseq/db/frontend"
|
||||
"deps/db/src/logseq/db/sqlite"
|
||||
"deps/outliner/src/logseq/outliner/property.cljs"
|
||||
"deps/outliner/src/logseq/outliner/page.cljs"])
|
||||
|
||||
(def db-graph-paths
|
||||
"Paths _only_ for DB graphs"
|
||||
(into block-name-db-graph-paths
|
||||
["deps/outliner/src/logseq/outliner/cli.cljs"
|
||||
"deps/outliner/src/logseq/outliner/db_pipeline.cljs"
|
||||
"deps/outliner/src/logseq/outliner/validate.cljs"
|
||||
"deps/outliner/src/logseq/outliner/page.cljs"
|
||||
;; TODO: change to deps/cli/src when :block/name no longer in other cli namespaces
|
||||
"deps/cli/src/logseq/cli/commands"
|
||||
"src/main/frontend/handler/db_based"
|
||||
"src/main/frontend/components/class.cljs"
|
||||
"src/main/frontend/components/property.cljs"
|
||||
"src/main/frontend/components/property"
|
||||
"src/main/frontend/components/objects.cljs"
|
||||
"src/main/frontend/components/quick_add.cljs"
|
||||
"src/main/frontend/components/vector_search"
|
||||
"src/main/frontend/components/db_based"
|
||||
"src/main/frontend/components/query/view.cljs"
|
||||
"src/main/frontend/inference_worker"
|
||||
"src/main/logseq/api/db_based.cljs"
|
||||
"src/main/logseq/api/db_based"
|
||||
"src/electron/electron/db.cljs"
|
||||
"src/main/mobile"]))
|
||||
|
||||
(def file-graph-paths
|
||||
"Paths _only_ for file graphs"
|
||||
["deps/graph-parser/src/logseq/graph_parser/extract.cljc"
|
||||
"deps/graph-parser/src/logseq/graph_parser/property.cljs"
|
||||
"src/main/frontend/fs"])
|
||||
|
||||
(defn- grep-many
|
||||
"Git greps a coll of patterns for given paths. Returns result from process/shell"
|
||||
[patterns paths]
|
||||
(apply shell {:out :string :continue true}
|
||||
"git grep -E" (str "(" (string/join "|" patterns) ")")
|
||||
paths))
|
||||
|
||||
(defn- validate-db-ns-not-in-file
|
||||
[]
|
||||
(let [res (grep-many db-graph-ns file-graph-paths)]
|
||||
(when-not (or (and (= 1 (:exit res)) (= "" (:out res)))
|
||||
;; TODO: Refactor logseq.cli.common.file to not have file-based code
|
||||
(= (:out res) "src/main/frontend/worker/file.cljs: [logseq.cli.common.file :as common-file]\n"))
|
||||
(println "The following db graph namespaces should not be in file graph files:")
|
||||
(println (:out res))
|
||||
(System/exit 1))))
|
||||
|
||||
(defn- validate-file-ns-not-in-db
|
||||
[]
|
||||
(let [res (grep-many file-graph-ns db-graph-paths)]
|
||||
(when-not (and (= 1 (:exit res)) (= "" (:out res)))
|
||||
(println "The following file graph namespaces should not be in db graph files:")
|
||||
(println (:out res))
|
||||
(System/exit 1))))
|
||||
|
||||
(defn- validate-multi-graph-fns-not-in-file-or-db
|
||||
[]
|
||||
(let [multi-graph-fns ["/db-based-graph\\?" "\\(db-based-graph\\?"
|
||||
;; from frontend.handler.property.util
|
||||
"/get-pid"
|
||||
"logseq.db.common.property-util"
|
||||
;; Use file-entity-util and entity-util when in a single graph context
|
||||
"ldb/whiteboard\\?" "ldb/journal\\?" "ldb/page\\?"]
|
||||
res (grep-many multi-graph-fns (into file-graph-paths db-graph-paths))]
|
||||
(when-not (and (= 1 (:exit res)) (= "" (:out res)))
|
||||
(println "The following files should not have fns meant to be used in multi-graph contexts:")
|
||||
(println (:out res))
|
||||
(System/exit 1))))
|
||||
|
||||
(defn- validate-file-concepts-not-in-db
|
||||
[]
|
||||
(let [file-concepts (->>
|
||||
;; from logseq.db.file-based.schema
|
||||
[:block/namespace :block/properties-text-values :block/pre-block :recent/pages :block/file :block/properties-order
|
||||
:block/repeated :block/deadline :block/scheduled :block/priority :block/marker :block/macros
|
||||
:block/type :block/format]
|
||||
(map str)
|
||||
(into [;; e.g. block/properties :title
|
||||
"block/properties :"
|
||||
;; anything org mode except for org.babashka or urls like schema.org
|
||||
"[^\\.]org[^\\.]"
|
||||
"file-based"
|
||||
"#+BEGIN_"
|
||||
"#+END_"
|
||||
"pre-block"]))
|
||||
;; For now use the whole code line. If this is too brittle can make this smaller
|
||||
allowed-exceptions #{":block/pre-block? :block/scheduled :block/deadline :block/type :block/name :block/marker"
|
||||
"(dissoc :block/format))]"
|
||||
"{:block/name page-title})"
|
||||
;; TODO: Mv this file-based ns out of db file
|
||||
"[logseq.db.file-based.schema :as file-schema]))"
|
||||
;; :block/name ones from src/main/mobile
|
||||
"(if-let [journal (db/get-page page-name)]"
|
||||
"(p/then #(mobile-state/open-block-modal! (db/get-page page-name)))))))]"}
|
||||
res (grep-many file-concepts db-graph-paths)
|
||||
invalid-lines (when (= 0 (:exit res))
|
||||
(remove #(some->> (string/split % #":\s+") second string/trim (contains? allowed-exceptions))
|
||||
(string/split-lines (:out res))))
|
||||
_ (when (> (:exit res) 1) (System/exit 1))
|
||||
_ (when (and (= 0 (:exit res)) (seq invalid-lines))
|
||||
(println "The following files should not have contained file specific concepts:")
|
||||
(println (string/join "\n" invalid-lines))
|
||||
(System/exit 1))
|
||||
|
||||
;; :block/name isn't used in db graphs except for fns with journal or internal-page
|
||||
block-name-file-concepts #{"block/name" "/page-name-sanity-lc" "db/get-page "}
|
||||
no-block-name-db-graph-paths (set/difference (set db-graph-paths) (set block-name-db-graph-paths))
|
||||
block-name-res (grep-many block-name-file-concepts no-block-name-db-graph-paths)
|
||||
block-name-invalid-lines (when (= 0 (:exit block-name-res))
|
||||
(remove #(some->> (string/split % #":\s+") second string/trim (contains? allowed-exceptions))
|
||||
(string/split-lines (:out block-name-res))))]
|
||||
|
||||
(when (> (:exit block-name-res) 1) (System/exit 1))
|
||||
(when (and (= 0 (:exit block-name-res)) (seq block-name-invalid-lines))
|
||||
(println "The following files should not have contained file specific concepts:")
|
||||
(println (string/join "\n" block-name-invalid-lines))
|
||||
(System/exit 1))))
|
||||
|
||||
(defn- validate-db-concepts-not-in-file
|
||||
[]
|
||||
(let [db-concepts
|
||||
;; from logseq.db.frontend.schema
|
||||
["closed-value" "class/properties" "classes" "property/parent"
|
||||
"logseq.property" "logseq.class" "db-based" "library" "quick-add"]
|
||||
res (grep-many db-concepts file-graph-paths)]
|
||||
(when-not (and (= 1 (:exit res)) (= "" (:out res)))
|
||||
(println "The following files should not have contained db specific concepts:")
|
||||
(println (:out res))
|
||||
(System/exit 1))))
|
||||
|
||||
(defn -main
|
||||
"Check that file and db graph specific namespaces and concepts are separate"
|
||||
[]
|
||||
(validate-db-ns-not-in-file)
|
||||
(validate-file-ns-not-in-db)
|
||||
(validate-file-concepts-not-in-db)
|
||||
(validate-db-concepts-not-in-file)
|
||||
(validate-multi-graph-fns-not-in-file-or-db)
|
||||
(println "✅ All checks passed!"))
|
||||
Reference in New Issue
Block a user