enhance(cli): Add validate command for local graph(s)

Improves :errors of invalid errors to be more user friendly.
Also enable closed-value validation optionally as it is useful
to have higher data quality for non RTC graphs

deps/cli/.carve/ignore

@@ -8,3 +8,4 @@ logseq.cli.commands.export/export
 logseq.cli.commands.append/append
 logseq.cli.commands.mcp-server/start
 logseq.cli.commands.import-edn/import-edn
+logseq.cli.commands.validate/validate

deps/cli/README.md

@@ -12,7 +12,7 @@ This section assumes you have installed the CLI from npm or via the [dev
 setup](#setup). If you haven't, substitute `node cli.mjs` for `logseq` e.g.
 `node.cli.mjs -h`.
 
-All commands work with both local graphs and the current in-app graph except for `append` (in-app graph only) and `export` (local graph only). For a command to work with an in-app graph, the [HTTP API Server](https://docs.logseq.com/#/page/local%20http%20server) must be turned on.
+All commands work with both local graphs and the current in-app graph except for `append` (in-app graph only), `validate` (local graph only) and `export` (local graph only). For a command to work with an in-app graph, the [HTTP API Server](https://docs.logseq.com/#/page/local%20http%20server) must be turned on.
 
 Now let's use the CLI!
 
@@ -33,6 +33,7 @@ export-edn [options] Export DB graph as EDN
 import-edn [options] Import into DB graph with EDN
 append [options]     Appends text to current page
 mcp-server [options] Run a MCP server
+validate [options]   Validate DB graph
 help                 Print a command's help
 
 $ logseq list

deps/cli/src/logseq/cli.cljs

@@ -97,7 +97,7 @@
     :description "Import with EDN into a local graph or the current in-app graph if --api-server-token is given. See https://github.com/logseq/docs/blob/master/db-version.md#edn-data-export for more about this import type."
     :fn (lazy-load-fn 'logseq.cli.commands.import-edn/import-edn)
     :spec cli-spec/import-edn}
-   {:cmds ["append"] :desc "Appends text to current page"
+   {:cmds ["append"] :desc "Append text to current page"
     :description "Append text to current page of current in-app graph."
     :fn (lazy-load-fn 'logseq.cli.commands.append/append)
     :args->opts [:args] :require [:args] :coerce {:args []}
@@ -106,6 +106,10 @@
     :description "Run a MCP server against a local graph if --graph is given or against the current in-app graph. By default the MCP server runs as a HTTP Streamable server. Use --stdio to run it as a stdio server."
     :fn (lazy-load-fn 'logseq.cli.commands.mcp-server/start)
     :spec cli-spec/mcp-server}
+   {:cmds ["validate"] :desc "Validate DB graph"
+    :description "Validate a local DB graph. Exit 1 if there are validation errors"
+    :fn (lazy-load-fn 'logseq.cli.commands.validate/validate)
+    :spec cli-spec/validate}
    {:cmds ["help"] :fn help-command :desc "Print a command's help"
     :args->opts [:command] :require [:command]}
    {:cmds []

deps/cli/src/logseq/cli/commands/validate.cljs

@@ -0,0 +1,53 @@
+(ns logseq.cli.commands.validate
+  "Validate graph command"
+  (:require ["fs" :as fs]
+            [cljs.pprint :as pprint]
+            [datascript.core :as d]
+            [logseq.cli.util :as cli-util]
+            [logseq.db.common.sqlite-cli :as sqlite-cli]
+            [logseq.db.frontend.malli-schema :as db-malli-schema]
+            [logseq.db.frontend.validate :as db-validate]
+            [malli.error :as me]))
+
+(defn- validate-db*
+  "Validate datascript db as a vec of entity maps"
+  [db ent-maps* _options]
+  (let [ent-maps (db-malli-schema/update-properties-in-ents db ent-maps*)
+        explainer db-validate/closed-db-schema-explainer]
+    (if-let [explanation (binding [db-malli-schema/*db-for-validate-fns* db
+                                   db-malli-schema/*closed-values-validate?* true]
+                           (->> (map (fn [e] (dissoc e :db/id)) ent-maps) explainer not-empty))]
+      (let [ent-errors
+            (->> (db-validate/group-errors-by-entity db ent-maps (:errors explanation))
+                 (map #(-> (dissoc % :errors-by-type)
+                           (update :errors (fn [errs]
+                                             ;; errs looks like: {178 {:logseq.property/hide? ["disallowed key"]}}
+                                             ;; map is indexed by :in which is unused since all errors are for the same map
+                                             (->> (me/humanize {:errors errs})
+                                                  vals
+                                                  (apply merge-with into)))))))]
+        (println "Found" (count ent-errors)
+                 (if (= 1 (count ent-errors)) "entity" "entities")
+                 "with errors:")
+        (pprint/pprint ent-errors)
+        (js/process.exit 1))
+      (println "Valid!"))))
+
+(defn- validate-db [db db-name options]
+  (let [datoms (d/datoms db :eavt)
+        ent-maps (db-malli-schema/datoms->entities datoms)]
+    (println "Read graph" (str db-name " with counts: "
+                               (pr-str (assoc (db-validate/graph-counts db ent-maps)
+                                              :datoms (count datoms)))))
+    (validate-db* db ent-maps options)))
+
+(defn- validate-graph [graph options]
+  (if (fs/existsSync (cli-util/get-graph-path graph))
+    (let [conn (apply sqlite-cli/open-db! (cli-util/->open-db-args graph))
+          _ (cli-util/ensure-db-graph-for-command @conn)]
+      (validate-db @conn graph options))
+    (cli-util/error "Graph" (pr-str graph) "does not exist")))
+
+(defn validate [{{:keys [graphs] :as opts} :opts}]
+  (doseq [graph graphs]
+    (validate-graph graph opts)))

deps/cli/src/logseq/cli/spec.cljs

@@ -82,4 +82,10 @@
           :desc "Host for streamable HTTP server"}
    :debug-tool {:alias :t
                 :coerce :keyword
-                :desc "Debug mcp tool with direct invocation"}})
+                :desc "Debug mcp tool with direct invocation"}})
+
+(def validate
+  {:graphs {:alias :g
+            :coerce []
+            :require true
+            :desc "Local graph(s) to validate"}})

deps/db/script/validate_db.cljs

@@ -27,7 +27,10 @@
               (pprint/pprint ent-errors)
               humanize
               (pprint/pprint (map #(-> (dissoc % :errors-by-type)
-                                       (update :errors (fn [errs] (me/humanize {:errors errs}))))
+                                       (update :errors (fn [errs]
+                                                         (->> (me/humanize {:errors errs})
+                                                              vals
+                                                              (apply merge-with into)))))
                                   ent-errors))
               :else
               (pprint/pprint (map :entity ent-errors))))

deps/db/src/logseq/db/frontend/malli_schema.cljs

@@ -89,18 +89,31 @@
   expected to be a coll if the property has a :many cardinality. validate-fn is
   a fn that is called directly on each value to return a truthy value.
   validate-fn varies by property type"
-  [db validate-fn [property property-val] & {:keys [_skip-strict-url-validate?]
-                                             :as validate-option}]
+  [db validate-fn [property property-val] & {:keys [new-closed-value? :closed-values-validate? _skip-strict-url-validate?]
+                                             :as validate-options}]
   ;; For debugging
   ;; (when (not (internal-ident? (:db/ident property))) (prn :validate-val (dissoc property :property/closed-values) property-val))
   (let [validate-fn' (if (db-property-type/property-types-with-db (:logseq.property/type property))
                        (fn [value]
-                         (validate-fn db value validate-option))
-                       validate-fn)]
+                         (validate-fn db value validate-options))
+                       validate-fn)
+        validate-fn'' (if (and closed-values-validate?
+                               (db-property-type/closed-value-property-types (:logseq.property/type property))
+                               ;; new closed values aren't associated with the property yet
+                               (not new-closed-value?)
+                               (seq (:property/closed-values property)))
+                        (fn closed-value-valid? [val]
+                          (and (validate-fn' val)
+                               (let [ids (set (map :db/id (:property/closed-values property)))
+                                     result (contains? ids val)]
+                                 (when-not result
+                                   (js/console.error (str "Error: not a closed value, id: " val ", existing choices: " ids ", property: " (:db/ident property))))
+                                 result)))
+                        validate-fn')]
     (if (db-property/many? property)
-      (or (every? validate-fn' property-val)
+      (or (every? validate-fn'' property-val)
           (empty-placeholder-value? db property (first property-val)))
-      (or (validate-fn' property-val)
+      (or (validate-fn'' property-val)
           ;; also valid if value is empty-placeholder
           (empty-placeholder-value? db property property-val)))))
 
@@ -214,6 +227,12 @@
   "`true` allows updating a block's other property when it has invalid URL value"
   false)
 
+(def ^:dynamic *closed-values-validate?*
+  "By default this is false because we can't ensure this when merging updates from server.
+   `true` allows for non RTC graphs to have higher data quality and avoid
+   possible UX bugs related to closed values."
+  false)
+
 (def property-tuple
   "A tuple of a property map and a property value"
   (into
@@ -229,7 +248,8 @@
                 {:error/message error-message})
               (fn [tuple]
                 (validate-property-value *db-for-validate-fns* schema-fn tuple
-                                         {:skip-strict-url-validate? *skip-strict-url-validate?*}))])])
+                                         {:skip-strict-url-validate? *skip-strict-url-validate?*
+                                          :closed-values-validate? *closed-values-validate?*}))])])
         db-property-type/built-in-validation-schemas)))
 
 (def block-properties

deps/db/src/logseq/db/frontend/validate.cljs

@@ -11,7 +11,7 @@
 (def ^:private db-schema-validator (m/validator db-malli-schema/DB))
 (def ^:private db-schema-explainer (m/explainer db-malli-schema/DB))
 (def ^:private closed-db-schema-validator (m/validator (mu/closed-schema db-malli-schema/DB)))
-(def ^:private closed-db-schema-explainer (m/explainer (mu/closed-schema db-malli-schema/DB)))
+(def closed-db-schema-explainer (m/explainer (mu/closed-schema db-malli-schema/DB)))
 
 (defn get-schema-validator
   [closed-schema?]

src/main/logseq/api/db_based/cli.cljs

@@ -1,5 +1,5 @@
 (ns logseq.api.db-based.cli
-  "API fns for CLI. DB graph checks are either at the top level or in cli-common-mcp-tools ns."
+  "API fns for CLI"
   (:require [clojure.string :as string]
             [frontend.handler.ui :as ui-handler]
             [frontend.modules.outliner.op :as outliner-op]