From 79e75060d6c5be2fcc4e332f0fa8cc46e90f56b9 Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Wed, 19 Nov 2025 17:51:45 -0500 Subject: [PATCH] enhance(cli): Add validate command for local graph(s) Improves :errors of invalid errors to be more user friendly. Also enable closed-value validation optionally as it is useful to have higher data quality for non RTC graphs --- deps/cli/.carve/ignore | 1 + deps/cli/README.md | 3 +- deps/cli/src/logseq/cli.cljs | 6 ++- .../cli/src/logseq/cli/commands/validate.cljs | 53 +++++++++++++++++++ deps/cli/src/logseq/cli/spec.cljs | 8 ++- deps/db/script/validate_db.cljs | 5 +- .../src/logseq/db/frontend/malli_schema.cljs | 34 +++++++++--- deps/db/src/logseq/db/frontend/validate.cljs | 2 +- src/main/logseq/api/db_based/cli.cljs | 2 +- 9 files changed, 101 insertions(+), 13 deletions(-) create mode 100644 deps/cli/src/logseq/cli/commands/validate.cljs diff --git a/deps/cli/.carve/ignore b/deps/cli/.carve/ignore index 68eb86ac6b..9a690f58e1 100644 --- a/deps/cli/.carve/ignore +++ b/deps/cli/.carve/ignore @@ -8,3 +8,4 @@ logseq.cli.commands.export/export logseq.cli.commands.append/append logseq.cli.commands.mcp-server/start logseq.cli.commands.import-edn/import-edn +logseq.cli.commands.validate/validate diff --git a/deps/cli/README.md b/deps/cli/README.md index ebfd528e80..733204db39 100644 --- a/deps/cli/README.md +++ b/deps/cli/README.md @@ -12,7 +12,7 @@ This section assumes you have installed the CLI from npm or via the [dev setup](#setup). If you haven't, substitute `node cli.mjs` for `logseq` e.g. `node.cli.mjs -h`. -All commands work with both local graphs and the current in-app graph except for `append` (in-app graph only) and `export` (local graph only). For a command to work with an in-app graph, the [HTTP API Server](https://docs.logseq.com/#/page/local%20http%20server) must be turned on. +All commands work with both local graphs and the current in-app graph except for `append` (in-app graph only), `validate` (local graph only) and `export` (local graph only). For a command to work with an in-app graph, the [HTTP API Server](https://docs.logseq.com/#/page/local%20http%20server) must be turned on. Now let's use the CLI! @@ -33,6 +33,7 @@ export-edn [options] Export DB graph as EDN import-edn [options] Import into DB graph with EDN append [options] Appends text to current page mcp-server [options] Run a MCP server +validate [options] Validate DB graph help Print a command's help $ logseq list diff --git a/deps/cli/src/logseq/cli.cljs b/deps/cli/src/logseq/cli.cljs index 59a28e8905..b381222d87 100644 --- a/deps/cli/src/logseq/cli.cljs +++ b/deps/cli/src/logseq/cli.cljs @@ -97,7 +97,7 @@ :description "Import with EDN into a local graph or the current in-app graph if --api-server-token is given. See https://github.com/logseq/docs/blob/master/db-version.md#edn-data-export for more about this import type." :fn (lazy-load-fn 'logseq.cli.commands.import-edn/import-edn) :spec cli-spec/import-edn} - {:cmds ["append"] :desc "Appends text to current page" + {:cmds ["append"] :desc "Append text to current page" :description "Append text to current page of current in-app graph." :fn (lazy-load-fn 'logseq.cli.commands.append/append) :args->opts [:args] :require [:args] :coerce {:args []} @@ -106,6 +106,10 @@ :description "Run a MCP server against a local graph if --graph is given or against the current in-app graph. By default the MCP server runs as a HTTP Streamable server. Use --stdio to run it as a stdio server." :fn (lazy-load-fn 'logseq.cli.commands.mcp-server/start) :spec cli-spec/mcp-server} + {:cmds ["validate"] :desc "Validate DB graph" + :description "Validate a local DB graph. Exit 1 if there are validation errors" + :fn (lazy-load-fn 'logseq.cli.commands.validate/validate) + :spec cli-spec/validate} {:cmds ["help"] :fn help-command :desc "Print a command's help" :args->opts [:command] :require [:command]} {:cmds [] diff --git a/deps/cli/src/logseq/cli/commands/validate.cljs b/deps/cli/src/logseq/cli/commands/validate.cljs new file mode 100644 index 0000000000..81d693ada6 --- /dev/null +++ b/deps/cli/src/logseq/cli/commands/validate.cljs @@ -0,0 +1,53 @@ +(ns logseq.cli.commands.validate + "Validate graph command" + (:require ["fs" :as fs] + [cljs.pprint :as pprint] + [datascript.core :as d] + [logseq.cli.util :as cli-util] + [logseq.db.common.sqlite-cli :as sqlite-cli] + [logseq.db.frontend.malli-schema :as db-malli-schema] + [logseq.db.frontend.validate :as db-validate] + [malli.error :as me])) + +(defn- validate-db* + "Validate datascript db as a vec of entity maps" + [db ent-maps* _options] + (let [ent-maps (db-malli-schema/update-properties-in-ents db ent-maps*) + explainer db-validate/closed-db-schema-explainer] + (if-let [explanation (binding [db-malli-schema/*db-for-validate-fns* db + db-malli-schema/*closed-values-validate?* true] + (->> (map (fn [e] (dissoc e :db/id)) ent-maps) explainer not-empty))] + (let [ent-errors + (->> (db-validate/group-errors-by-entity db ent-maps (:errors explanation)) + (map #(-> (dissoc % :errors-by-type) + (update :errors (fn [errs] + ;; errs looks like: {178 {:logseq.property/hide? ["disallowed key"]}} + ;; map is indexed by :in which is unused since all errors are for the same map + (->> (me/humanize {:errors errs}) + vals + (apply merge-with into)))))))] + (println "Found" (count ent-errors) + (if (= 1 (count ent-errors)) "entity" "entities") + "with errors:") + (pprint/pprint ent-errors) + (js/process.exit 1)) + (println "Valid!")))) + +(defn- validate-db [db db-name options] + (let [datoms (d/datoms db :eavt) + ent-maps (db-malli-schema/datoms->entities datoms)] + (println "Read graph" (str db-name " with counts: " + (pr-str (assoc (db-validate/graph-counts db ent-maps) + :datoms (count datoms))))) + (validate-db* db ent-maps options))) + +(defn- validate-graph [graph options] + (if (fs/existsSync (cli-util/get-graph-path graph)) + (let [conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph)) + _ (cli-util/ensure-db-graph-for-command @conn)] + (validate-db @conn graph options)) + (cli-util/error "Graph" (pr-str graph) "does not exist"))) + +(defn validate [{{:keys [graphs] :as opts} :opts}] + (doseq [graph graphs] + (validate-graph graph opts))) diff --git a/deps/cli/src/logseq/cli/spec.cljs b/deps/cli/src/logseq/cli/spec.cljs index 4257ebbf22..e69fe1ecd9 100644 --- a/deps/cli/src/logseq/cli/spec.cljs +++ b/deps/cli/src/logseq/cli/spec.cljs @@ -82,4 +82,10 @@ :desc "Host for streamable HTTP server"} :debug-tool {:alias :t :coerce :keyword - :desc "Debug mcp tool with direct invocation"}}) \ No newline at end of file + :desc "Debug mcp tool with direct invocation"}}) + +(def validate + {:graphs {:alias :g + :coerce [] + :require true + :desc "Local graph(s) to validate"}}) \ No newline at end of file diff --git a/deps/db/script/validate_db.cljs b/deps/db/script/validate_db.cljs index f3738e91ac..71c4373c9c 100644 --- a/deps/db/script/validate_db.cljs +++ b/deps/db/script/validate_db.cljs @@ -27,7 +27,10 @@ (pprint/pprint ent-errors) humanize (pprint/pprint (map #(-> (dissoc % :errors-by-type) - (update :errors (fn [errs] (me/humanize {:errors errs})))) + (update :errors (fn [errs] + (->> (me/humanize {:errors errs}) + vals + (apply merge-with into))))) ent-errors)) :else (pprint/pprint (map :entity ent-errors)))) diff --git a/deps/db/src/logseq/db/frontend/malli_schema.cljs b/deps/db/src/logseq/db/frontend/malli_schema.cljs index 735f54dd39..7ff476f6da 100644 --- a/deps/db/src/logseq/db/frontend/malli_schema.cljs +++ b/deps/db/src/logseq/db/frontend/malli_schema.cljs @@ -89,18 +89,31 @@ expected to be a coll if the property has a :many cardinality. validate-fn is a fn that is called directly on each value to return a truthy value. validate-fn varies by property type" - [db validate-fn [property property-val] & {:keys [_skip-strict-url-validate?] - :as validate-option}] + [db validate-fn [property property-val] & {:keys [new-closed-value? :closed-values-validate? _skip-strict-url-validate?] + :as validate-options}] ;; For debugging ;; (when (not (internal-ident? (:db/ident property))) (prn :validate-val (dissoc property :property/closed-values) property-val)) (let [validate-fn' (if (db-property-type/property-types-with-db (:logseq.property/type property)) (fn [value] - (validate-fn db value validate-option)) - validate-fn)] + (validate-fn db value validate-options)) + validate-fn) + validate-fn'' (if (and closed-values-validate? + (db-property-type/closed-value-property-types (:logseq.property/type property)) + ;; new closed values aren't associated with the property yet + (not new-closed-value?) + (seq (:property/closed-values property))) + (fn closed-value-valid? [val] + (and (validate-fn' val) + (let [ids (set (map :db/id (:property/closed-values property))) + result (contains? ids val)] + (when-not result + (js/console.error (str "Error: not a closed value, id: " val ", existing choices: " ids ", property: " (:db/ident property)))) + result))) + validate-fn')] (if (db-property/many? property) - (or (every? validate-fn' property-val) + (or (every? validate-fn'' property-val) (empty-placeholder-value? db property (first property-val))) - (or (validate-fn' property-val) + (or (validate-fn'' property-val) ;; also valid if value is empty-placeholder (empty-placeholder-value? db property property-val))))) @@ -214,6 +227,12 @@ "`true` allows updating a block's other property when it has invalid URL value" false) +(def ^:dynamic *closed-values-validate?* + "By default this is false because we can't ensure this when merging updates from server. + `true` allows for non RTC graphs to have higher data quality and avoid + possible UX bugs related to closed values." + false) + (def property-tuple "A tuple of a property map and a property value" (into @@ -229,7 +248,8 @@ {:error/message error-message}) (fn [tuple] (validate-property-value *db-for-validate-fns* schema-fn tuple - {:skip-strict-url-validate? *skip-strict-url-validate?*}))])]) + {:skip-strict-url-validate? *skip-strict-url-validate?* + :closed-values-validate? *closed-values-validate?*}))])]) db-property-type/built-in-validation-schemas))) (def block-properties diff --git a/deps/db/src/logseq/db/frontend/validate.cljs b/deps/db/src/logseq/db/frontend/validate.cljs index 7705eda78a..6067cab604 100644 --- a/deps/db/src/logseq/db/frontend/validate.cljs +++ b/deps/db/src/logseq/db/frontend/validate.cljs @@ -11,7 +11,7 @@ (def ^:private db-schema-validator (m/validator db-malli-schema/DB)) (def ^:private db-schema-explainer (m/explainer db-malli-schema/DB)) (def ^:private closed-db-schema-validator (m/validator (mu/closed-schema db-malli-schema/DB))) -(def ^:private closed-db-schema-explainer (m/explainer (mu/closed-schema db-malli-schema/DB))) +(def closed-db-schema-explainer (m/explainer (mu/closed-schema db-malli-schema/DB))) (defn get-schema-validator [closed-schema?] diff --git a/src/main/logseq/api/db_based/cli.cljs b/src/main/logseq/api/db_based/cli.cljs index 2cf2963e12..ed5f38a7c8 100644 --- a/src/main/logseq/api/db_based/cli.cljs +++ b/src/main/logseq/api/db_based/cli.cljs @@ -1,5 +1,5 @@ (ns logseq.api.db-based.cli - "API fns for CLI. DB graph checks are either at the top level or in cli-common-mcp-tools ns." + "API fns for CLI" (:require [clojure.string :as string] [frontend.handler.ui :as ui-handler] [frontend.modules.outliner.op :as outliner-op]