From fe7a46eac9962a19a737343f02198a9aff630f5b Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Wed, 11 Oct 2023 08:37:49 -0400 Subject: [PATCH] Move db malli schema into db dep since it's stable Also add a validate-db task. Part of LOG-2739 --- bb.edn | 7 + deps/db/.carve/config.edn | 1 + deps/db/nbb.edn | 4 +- deps/db/script/validate_client_db.cljs | 133 +++++++ deps/db/src/logseq/db/malli_schema.cljs | 236 ++++++++++++ deps/db/src/logseq/db/schema.cljs | 2 +- docs/dev-practices.md | 13 + scripts/nbb.edn | 4 +- .../tasks/db_graph/validate_client_db.cljs | 345 ------------------ 9 files changed, 395 insertions(+), 350 deletions(-) create mode 100644 deps/db/script/validate_client_db.cljs create mode 100644 deps/db/src/logseq/db/malli_schema.cljs delete mode 100644 scripts/src/logseq/tasks/db_graph/validate_client_db.cljs diff --git a/bb.edn b/bb.edn index 1eba4c1400..8adb2ff74d 100644 --- a/bb.edn +++ b/bb.edn @@ -42,6 +42,13 @@ "yarn -s nbb-logseq -cp src -m logseq.tasks.dev.publishing" (into ["static"] *command-line-args*))} + dev:validate-db + {:doc "Validate a DB graph's datascript schema" + :requires ([babashka.fs :as fs]) + :task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}} + "yarn -s nbb-logseq script/validate_client_db.cljs" + *command-line-args*)} + dev:npx-cap-run-ios logseq.tasks.dev.mobile/npx-cap-run-ios diff --git a/deps/db/.carve/config.edn b/deps/db/.carve/config.edn index 8faeb33903..f2ccd45934 100644 --- a/deps/db/.carve/config.edn +++ b/deps/db/.carve/config.edn @@ -5,6 +5,7 @@ logseq.db.sqlite.util logseq.db.sqlite.cli logseq.db.property + logseq.db.malli-schema ;; Some fns are used by frontend but not worth moving over yet logseq.db.schema] :report {:format :ignore}} diff --git a/deps/db/nbb.edn b/deps/db/nbb.edn index 3774f52e5a..6796dd6353 100644 --- a/deps/db/nbb.edn +++ b/deps/db/nbb.edn @@ -1,4 +1,6 @@ {:paths ["src"] :deps - {io.github.nextjournal/nbb-test-runner + {metosin/malli + {:mvn/version "0.10.0"} + io.github.nextjournal/nbb-test-runner {:git/sha "60ed57aa04bca8d604f5ba6b28848bd887109347"}}} diff --git a/deps/db/script/validate_client_db.cljs b/deps/db/script/validate_client_db.cljs new file mode 100644 index 0000000000..b789d43e0e --- /dev/null +++ b/deps/db/script/validate_client_db.cljs @@ -0,0 +1,133 @@ +(ns validate-client-db + "Script that validates the datascript db of a DB graph" + (:require [logseq.db.sqlite.cli :as sqlite-cli] + [logseq.db.sqlite.db :as sqlite-db] + [logseq.db.schema :as db-schema] + [logseq.db.malli-schema :as db-malli-schema] + [datascript.core :as d] + [clojure.string :as string] + [nbb.core :as nbb] + [clojure.walk :as walk] + [malli.core :as m] + [babashka.cli :as cli] + ["path" :as node-path] + ["os" :as os] + [cljs.pprint :as pprint])) + +(defn- build-grouped-errors [db full-maps errors] + (->> errors + (group-by #(-> % :in first)) + (map (fn [[idx errors']] + {:entity (cond-> (get full-maps idx) + ;; Provide additional page info for debugging + (:block/page (get full-maps idx)) + (update :block/page + (fn [id] (select-keys (d/entity db id) + [:block/name :block/type :db/id :block/created-at])))) + ;; Group by type to reduce verbosity + :errors-by-type + (->> (group-by :type errors') + (map (fn [[type' type-errors]] + [type' + {:in-value-distinct (->> type-errors + (map #(select-keys % [:in :value])) + distinct + vec) + :schema-distinct (->> (map :schema type-errors) + (map m/form) + distinct + vec)}])) + (into {}))})))) + +(defn- update-schema + "Updates the db schema to add a datascript db for property validations + and to optionally close maps" + [db-schema db {:keys [closed-maps]}] + (let [db-schema-with-property-vals (db-malli-schema/update-properties-in-schema db-schema db)] + (if closed-maps + (walk/postwalk (fn [e] + (if (and (vector? e) + (= :map (first e)) + (contains? (second e) :closed)) + (assoc e 1 (assoc (second e) :closed true)) + e)) + db-schema-with-property-vals) + db-schema-with-property-vals))) + +(defn validate-client-db + "Validate datascript db as a vec of entity maps" + [db ent-maps* {:keys [verbose group-errors] :as options}] + (let [ent-maps (vec (db-malli-schema/update-properties-in-ents (vals ent-maps*))) + schema (update-schema db-malli-schema/DB db options)] + (if-let [errors (->> ent-maps + (m/explain schema) + :errors)] + (do + (if group-errors + (let [ent-errors (build-grouped-errors db ent-maps errors)] + (println "Found" (count ent-errors) "entities in errors:") + (if verbose + (pprint/pprint ent-errors) + (pprint/pprint (map :entity ent-errors)))) + (do + (println "Found" (count errors) "errors:") + (if verbose + (pprint/pprint + (map #(assoc % + :entity (get ent-maps (-> % :in first)) + :schema (m/form (:schema %))) + errors)) + (pprint/pprint errors)))) + (js/process.exit 1)) + (println "Valid!")))) + +(defn- datoms->entity-maps + "Returns entity maps for given :eavt datoms" + [datoms] + (->> datoms + (reduce (fn [acc m] + (if (contains? db-schema/card-many-attributes (:a m)) + (update acc (:e m) update (:a m) (fnil conj #{}) (:v m)) + (update acc (:e m) assoc (:a m) (:v m)))) + {}))) + +(def spec + "Options spec" + {:help {:alias :h + :desc "Print help"} + :verbose {:alias :v + :desc "Print more info"} + :closed-maps {:alias :c + :desc "Validate maps marked with closed as :closed"} + :group-errors {:alias :g + :desc "Groups errors by their entity id"}}) + +(defn- validate-graph [graph-dir options] + (let [[dir db-name] (if (string/includes? graph-dir "/") + (let [graph-dir' + (node-path/join (or js/process.env.ORIGINAL_PWD ".") graph-dir)] + ((juxt node-path/dirname node-path/basename) graph-dir')) + [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]) + _ (try (sqlite-db/open-db! dir db-name) + (catch :default e + (println "Error: For graph" (str (pr-str graph-dir) ":") (str e)) + (js/process.exit 1))) + conn (sqlite-cli/read-graph db-name) + datoms (d/datoms @conn :eavt) + ent-maps (datoms->entity-maps datoms)] + (println "Read graph" (str db-name " with " (count datoms) " datoms, " + (count ent-maps) " entities and " + (count (mapcat :block/properties (vals ent-maps))) " properties")) + (validate-client-db @conn ent-maps options))) + +(defn -main [argv] + (let [{:keys [args opts]} (cli/parse-args argv {:spec spec}) + _ (when (or (empty? args) (:help opts)) + (println (str "Usage: $0 GRAPH-NAME [& ADDITIONAL-GRAPHS] [OPTIONS]\nOptions:\n" + (cli/format-opts {:spec spec}))) + (js/process.exit 1))] + (doseq [graph-dir args] + (validate-graph graph-dir opts)))) + +(when (= nbb/*file* (:file (meta #'-main))) + (-main *command-line-args*)) \ No newline at end of file diff --git a/deps/db/src/logseq/db/malli_schema.cljs b/deps/db/src/logseq/db/malli_schema.cljs new file mode 100644 index 0000000000..817ee14504 --- /dev/null +++ b/deps/db/src/logseq/db/malli_schema.cljs @@ -0,0 +1,236 @@ +(ns logseq.db.malli-schema + "Malli schemas and fns for logseq.db.*" + (:require [clojure.walk :as walk] + [datascript.core :as d] + [logseq.db.property :as db-property] + [logseq.db.property.type :as db-property-type])) + +;; Helper fns +;; ========== +(defn validate-property-value + "Validates the value in a property tuple. The property value can be one or + many of a value to validated" + [prop-type schema-fn val] + (if (and (or (sequential? val) (set? val)) + (not= :coll prop-type)) + (every? schema-fn val) + (schema-fn val))) + +(defn update-properties-in-schema + "Needs to be called on the DB schema to add the datascript db to it" + [db-schema db] + (walk/postwalk (fn [e] + (let [meta' (meta e)] + (cond + (:add-db meta') + (partial e db) + (:property-value meta') + (let [[property-type schema-fn] e + schema-fn' (if (db-property-type/property-types-with-db property-type) (partial schema-fn db) schema-fn) + validation-fn #(validate-property-value property-type schema-fn' %)] + [property-type [:tuple :uuid [:fn validation-fn]]]) + :else + e))) + db-schema)) + +(defn update-properties-in-ents + "Prepares entities to be validated by DB schema" + [ents] + (map #(if (:block/properties %) + (update % :block/properties (fn [x] (mapv identity x))) + %) + ents)) + +;; Malli schemas +;; ============= +;; These schemas should be data vars to remain as simple and reusable as possible +(def property-tuple + "Represents a tuple of a property and its property value. This schema + has 2 metadata hooks which are used to inject a datascript db later" + (into + [:multi {:dispatch ^:add-db (fn [db property-tuple] + (get-in (d/entity db [:block/uuid (first property-tuple)]) + [:block/schema :type]))}] + (map (fn [[prop-type value-schema]] + ^:property-value [prop-type (if (vector? value-schema) (last value-schema) value-schema)]) + db-property-type/builtin-schema-types))) + +(def block-properties + "Validates a slightly modified verson of :block/properties. Properties are + expected to be a vector of tuples instead of a map in order to validate each + property with its property value that is valid for its type" + [:sequential property-tuple]) + +(def page-or-block-attrs + "Common attributes for page and normal blocks" + [[:block/uuid :uuid] + [:block/created-at :int] + [:block/updated-at :int] + [:block/properties {:optional true} + block-properties] + [:block/refs {:optional true} [:set :int]] + [:block/tags {:optional true} [:set :int]] + [:block/tx-id {:optional true} :int]]) + +(def page-attrs + "Common attributes for pages" + [[:block/name :string] + [:block/original-name :string] + [:block/type {:optional true} [:enum #{"property"} #{"class"} #{"object"} #{"whiteboard"} #{"hidden"}]] + [:block/journal? :boolean] + ;; TODO: Consider moving to just normal and class after figuring out journal attributes + [:block/format {:optional true} [:enum :markdown]] + ;; TODO: Should this be here or in common? + [:block/path-refs {:optional true} [:set :int]]]) + +(def normal-page + (vec + (concat + [:map {:closed false}] + page-attrs + ;; journal-day is only set for journal pages + [[:block/journal-day {:optional true} :int] + [:block/namespace {:optional true} :int]] + page-or-block-attrs))) + +(def object-page + (vec + (concat + [:map {:closed false}] + [[:block/collapsed? {:optional true} :boolean] + [:block/tags [:set :int]]] + page-attrs + (remove #(= :block/tags (first %)) page-or-block-attrs)))) + +(def class-page + (vec + (concat + [:map {:closed false}] + [[:block/namespace {:optional true} :int] + ;; TODO: Require :block/schema + [:block/schema + {:optional true} + [:map + {:closed false} + [:properties {:optional true} [:vector :uuid]]]]] + page-attrs + page-or-block-attrs))) + +(def internal-property + (vec + (concat + [:map {:closed false}] + [[:block/schema + [:map + {:closed false} + [:type (apply vector :enum (into db-property-type/internal-builtin-schema-types + db-property-type/user-builtin-schema-types))] + [:hide? {:optional true} :boolean] + [:cardinality {:optional true} [:enum :one :many]]]]] + page-attrs + page-or-block-attrs))) + +(def user-property + (vec + (concat + [:map {:closed false}] + [[:block/schema + [:map + {:closed false} + [:type (apply vector :enum db-property-type/user-builtin-schema-types)] + [:hide? {:optional true} :boolean] + [:description {:optional true} :string] + ;; For any types except for :checkbox :default :template :enum + [:cardinality {:optional true} [:enum :one :many]] + ;; Just for :enum type + [:enum-config {:optional true} :map] + ;; :template uses :sequential and :page uses :set. + ;; Should :template should use :set? + [:classes {:optional true} [:or + [:set :uuid] + [:sequential :uuid]]]]]] + page-attrs + page-or-block-attrs))) + +(def property-page + [:multi {:dispatch + (fn [m] (contains? db-property/built-in-properties-keys-str (:block/name m)))} + [true internal-property] + [:malli.core/default user-property]]) + +(def page + [:multi {:dispatch :block/type} + [#{"property"} property-page] + [#{"class"} class-page] + [#{"object"} object-page] + [:malli.core/default normal-page]]) + +(def block-attrs + "Common attributes for normal blocks" + [[:block/content :string] + [:block/left :int] + [:block/parent :int] + [:block/metadata {:optional true} + [:map {:closed false} + [:created-from-block :uuid] + [:created-from-property :uuid] + [:created-from-template {:optional true} :uuid]]] + ;; refs + [:block/page :int] + [:block/path-refs {:optional true} [:set :int]] + [:block/link {:optional true} :int] + ;; other + [:block/format [:enum :markdown]] + [:block/marker {:optional true} :string] + [:block/priority {:optional true} :string] + [:block/collapsed? {:optional true} :boolean]]) + +(def object-block + "A normal block with tags" + (vec + (concat + [:map {:closed false}] + [[:block/type [:= #{"object"}]] + [:block/tags [:set :int]]] + block-attrs + (remove #(= :block/tags (first %)) page-or-block-attrs)))) + +(def normal-block + "A block with content and no special type or tag behavior" + (vec + (concat + [:map {:closed false}] + block-attrs + page-or-block-attrs))) + +(def block + "A block has content and a page" + [:or + normal-block + object-block]) + +;; TODO: Figure out where this is coming from +(def unknown-empty-block + [:map {:closed true} + [:block/uuid :uuid]]) + +(def file-block + [:map {:closed true} + [:block/uuid :uuid] + [:block/tx-id {:optional true} :int] + [:file/content :string] + [:file/path :string] + ;; TODO: Remove when bug is fixed + [:file/last-modified-at {:optional true} :any]]) + +(def DB + "Malli schema for entities from schema/schema-for-db-based-graph. In order to + thoroughly validate properties, the entities and this schema should be + prepared with update-properties-in-ents and update-properties-in-schema + respectively" + [:sequential + [:or + page + block + file-block + unknown-empty-block]]) \ No newline at end of file diff --git a/deps/db/src/logseq/db/schema.cljs b/deps/db/src/logseq/db/schema.cljs index 871513c4a7..5d9761e894 100644 --- a/deps/db/src/logseq/db/schema.cljs +++ b/deps/db/src/logseq/db/schema.cljs @@ -1,5 +1,5 @@ (ns logseq.db.schema - "Main db schemas for the Logseq app" + "Main datascript schemas for the Logseq app" (:require [clojure.set :as set])) (defonce version 2) diff --git a/docs/dev-practices.md b/docs/dev-practices.md index 5e7d2a1d18..454ca89ebb 100644 --- a/docs/dev-practices.md +++ b/docs/dev-practices.md @@ -301,6 +301,19 @@ point out: bb dev:validate-repo-config-edn deps/common/resources/templates/config.edn ``` +* `dev:validate-db` - Validates a DB graph's datascript schema + + ```sh + # One time setup + $ cd deps/db && yarn install && cd - + # One or more graphs can be validated e.g. + $ bb dev:validate-db test-db schema -c -g + Read graph test-db with 1572 datoms, 220 entities and 13 properties + Valid! + Read graph schema with 26105 datoms, 2320 entities and 3168 properties + Valid! + ``` + * `dev:publishing` - Build a publishing app for a given graph dir. If the publishing frontend is out of date, it builds that first which takes time. Subsequent runs are quick. diff --git a/scripts/nbb.edn b/scripts/nbb.edn index 3c652129d2..f72a2ffe7a 100644 --- a/scripts/nbb.edn +++ b/scripts/nbb.edn @@ -1,8 +1,6 @@ {:paths ["src"] :deps - {metosin/malli - {:mvn/version "0.10.0"} - logseq/graph-parser + {logseq/graph-parser {:local/root "../deps/graph-parser"} logseq/outliner {:local/root "../deps/outliner"} diff --git a/scripts/src/logseq/tasks/db_graph/validate_client_db.cljs b/scripts/src/logseq/tasks/db_graph/validate_client_db.cljs deleted file mode 100644 index b4e9b86fce..0000000000 --- a/scripts/src/logseq/tasks/db_graph/validate_client_db.cljs +++ /dev/null @@ -1,345 +0,0 @@ -(ns logseq.tasks.db-graph.validate-client-db - "Script that validates the datascript db of a db graph" - (:require [logseq.db.sqlite.cli :as sqlite-cli] - [logseq.db.sqlite.db :as sqlite-db] - [logseq.db.schema :as db-schema] - [logseq.db.property :as db-property] - [logseq.db.property.type :as db-property-type] - [datascript.core :as d] - [clojure.string :as string] - [nbb.core :as nbb] - [clojure.pprint :as pprint] - [clojure.walk :as walk] - [malli.core :as m] - [babashka.cli :as cli] - ["path" :as node-path] - ["os" :as os] - [cljs.pprint :as pprint])) - -(defn- validate-property-value - "Validates the value in a property tuple. The property value can be one or - many of a value to validated" - [prop-type schema-fn val] - (if (and (or (sequential? val) (set? val)) - (not= :coll prop-type)) - (every? schema-fn val) - (schema-fn val))) - -(def property-tuple - "Represents a tuple of a property and its property value. This schema - has 2 metadata hooks which are used to inject a datascript db later" - (into - [:multi {:dispatch ^:add-db (fn [db property-tuple] - (get-in (d/entity db [:block/uuid (first property-tuple)]) - [:block/schema :type]))}] - (map (fn [[prop-type value-schema]] - ^:property-value [prop-type (if (vector? value-schema) (last value-schema) value-schema)]) - db-property-type/builtin-schema-types))) - -(def block-properties - "Validates a slightly modified verson of :block/properties. Properties are - expected to be a vector of tuples instead of a map in order to validate each - property with its property value that is valid for its type" - [:sequential property-tuple]) - -(def page-or-block-attrs - "Common attributes for page and normal blocks" - [[:block/uuid :uuid] - [:block/created-at :int] - [:block/updated-at :int] - [:block/properties {:optional true} - block-properties] - [:block/refs {:optional true} [:set :int]] - [:block/tags {:optional true} [:set :int]] - [:block/tx-id {:optional true} :int]]) - -(def page-attrs - "Common attributes for pages" - [[:block/name :string] - [:block/original-name :string] - [:block/type {:optional true} [:enum #{"property"} #{"class"} #{"object"} #{"whiteboard"} #{"hidden"}]] - [:block/journal? :boolean] - ;; TODO: Consider moving to just normal and class after figuring out journal attributes - [:block/format {:optional true} [:enum :markdown]] - ;; TODO: Should this be here or in common? - [:block/path-refs {:optional true} [:set :int]]]) - -(def normal-page - (vec - (concat - [:map {:closed false}] - page-attrs - ;; journal-day is only set for journal pages - [[:block/journal-day {:optional true} :int] - [:block/namespace {:optional true} :int]] - page-or-block-attrs))) - -(def object-page - (vec - (concat - [:map {:closed false}] - [[:block/collapsed? {:optional true} :boolean] - [:block/tags [:set :int]]] - page-attrs - (remove #(= :block/tags (first %)) page-or-block-attrs)))) - -(def class-page - (vec - (concat - [:map {:closed false}] - [[:block/namespace {:optional true} :int] - ;; TODO: Require :block/schema - [:block/schema - {:optional true} - [:map - {:closed false} - [:properties {:optional true} [:vector :uuid]]]]] - page-attrs - page-or-block-attrs))) - -(def internal-property - (vec - (concat - [:map {:closed false}] - [[:block/schema - [:map - {:closed false} - [:type (apply vector :enum (into db-property-type/internal-builtin-schema-types - db-property-type/user-builtin-schema-types))] - [:hide? {:optional true} :boolean] - [:cardinality {:optional true} [:enum :one :many]]]]] - page-attrs - page-or-block-attrs))) - -(def user-property - (vec - (concat - [:map {:closed false}] - [[:block/schema - [:map - {:closed false} - [:type (apply vector :enum db-property-type/user-builtin-schema-types)] - [:hide? {:optional true} :boolean] - [:description {:optional true} :string] - ;; For any types except for :checkbox :default :template :enum - [:cardinality {:optional true} [:enum :one :many]] - ;; Just for :enum type - [:enum-config {:optional true} :map] - ;; :template uses :sequential and :page uses :set. - ;; Should :template should use :set? - [:classes {:optional true} [:or - [:set :uuid] - [:sequential :uuid]]]]]] - page-attrs - page-or-block-attrs))) - -(def property-page - [:multi {:dispatch - (fn [m] (contains? db-property/built-in-properties-keys-str (:block/name m)))} - [true internal-property] - [::m/default user-property]]) - -(def page - [:multi {:dispatch :block/type} - [#{"property"} property-page] - [#{"class"} class-page] - [#{"object"} object-page] - [::m/default normal-page]]) - -(def block-attrs - "Common attributes for normal blocks" - [[:block/content :string] - [:block/left :int] - [:block/parent :int] - [:block/metadata {:optional true} - [:map {:closed false} - [:created-from-block :uuid] - [:created-from-property :uuid] - [:created-from-template {:optional true} :uuid]]] - ;; refs - [:block/page :int] - [:block/path-refs {:optional true} [:set :int]] - [:block/link {:optional true} :int] - ;; other - [:block/format [:enum :markdown]] - [:block/marker {:optional true} :string] - [:block/priority {:optional true} :string] - [:block/collapsed? {:optional true} :boolean]]) - -(def object-block - "A normal block with tags" - (vec - (concat - [:map {:closed false}] - [[:block/type [:= #{"object"}]] - [:block/tags [:set :int]]] - block-attrs - (remove #(= :block/tags (first %)) page-or-block-attrs)))) - -(def normal-block - "A block with content and no special type or tag behavior" - (vec - (concat - [:map {:closed false}] - block-attrs - page-or-block-attrs))) - -(def block - "A block has content and a page" - [:or - normal-block - object-block]) - -;; TODO: Figure out where this is coming from -(def unknown-empty-block - [:map {:closed true} - [:block/uuid :uuid]]) - -(def file-block - [:map {:closed true} - [:block/uuid :uuid] - [:block/tx-id {:optional true} :int] - [:file/content :string] - [:file/path :string] - ;; TODO: Remove when bug is fixed - [:file/last-modified-at {:optional true} :any]]) - -(def client-db-schema - [:sequential - [:or - page - block - file-block - unknown-empty-block]]) - -(defn- build-grouped-errors [db full-maps errors] - (->> errors - (group-by #(-> % :in first)) - (map (fn [[idx errors']] - {:entity (cond-> (get full-maps idx) - ;; Provide additional page info for debugging - (:block/page (get full-maps idx)) - (update :block/page - (fn [id] (select-keys (d/entity db id) - [:block/name :block/type :db/id :block/created-at])))) - ;; Group by type to reduce verbosity - :errors-by-type - (->> (group-by :type errors') - (map (fn [[type' type-errors]] - [type' - {:in-value-distinct (->> type-errors - (map #(select-keys % [:in :value])) - distinct - vec) - :schema-distinct (->> (map :schema type-errors) - (map m/form) - distinct - vec)}])) - (into {}))})))) - -(defn- update-schema - "Updates the db schema to add a datascript db for property validations - and to optionally close maps" - [db-schema db {:keys [closed-maps]}] - (let [db-schema-with-property-vals - (walk/postwalk (fn [e] - (let [meta' (meta e)] - (cond - (:add-db meta') - (partial e db) - (:property-value meta') - (let [[property-type schema-fn] e - schema-fn' (if (db-property-type/property-types-with-db property-type) (partial schema-fn db) schema-fn) - validation-fn #(validate-property-value property-type schema-fn' %)] - [property-type [:tuple :uuid [:fn validation-fn]]]) - :else - e))) - db-schema)] - (if closed-maps - (walk/postwalk (fn [e] - (if (and (vector? e) - (= :map (first e)) - (contains? (second e) :closed)) - (assoc e 1 (assoc (second e) :closed true)) - e)) - db-schema-with-property-vals) - db-schema-with-property-vals))) - -(defn validate-client-db - "Validate datascript db as a vec of entity maps" - [db ent-maps* {:keys [verbose group-errors] :as options}] - (let [ent-maps (vec (map #(if (:block/properties %) - (update % :block/properties (fn [x] (mapv identity x))) - %) - (vals ent-maps*))) - schema (update-schema client-db-schema db options)] - (if-let [errors (->> ent-maps - (m/explain schema) - :errors)] - (do - (if group-errors - (let [ent-errors (build-grouped-errors db ent-maps errors)] - (println "Found" (count ent-errors) "entities in errors:") - (if verbose - (pprint/pprint ent-errors) - (pprint/pprint (map :entity ent-errors)))) - (do - (println "Found" (count errors) "errors:") - (if verbose - (pprint/pprint - (map #(assoc % - :entity (get ent-maps (-> % :in first)) - :schema (m/form (:schema %))) - errors)) - (pprint/pprint errors)))) - (js/process.exit 1)) - (println "Valid!")))) - -(defn- datoms->entity-maps - "Returns entity maps for given :eavt datoms" - [datoms] - (->> datoms - (reduce (fn [acc m] - (if (contains? db-schema/card-many-attributes (:a m)) - (update acc (:e m) update (:a m) (fnil conj #{}) (:v m)) - (update acc (:e m) assoc (:a m) (:v m)))) - {}))) - -(def spec - "Options spec" - {:help {:alias :h - :desc "Print help"} - :verbose {:alias :v - :desc "Print more info"} - :closed-maps {:alias :c - :desc "Validate maps marked with closed as :closed"} - :group-errors {:alias :g - :desc "Groups errors by their entity id"}}) - -(defn- validate-graph [graph-dir options] - (let [[dir db-name] (if (string/includes? graph-dir "/") - ((juxt node-path/dirname node-path/basename) graph-dir) - [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]) - _ (try (sqlite-db/open-db! dir db-name) - (catch :default e - (println "Error: For graph" (str (pr-str graph-dir) ":") (str e)) - (js/process.exit 1))) - conn (sqlite-cli/read-graph db-name) - datoms (d/datoms @conn :eavt) - ent-maps (datoms->entity-maps datoms)] - (println "Read graph" (str db-name " with " (count datoms) " datoms, " - (count ent-maps) " entities and " - (count (mapcat :block/properties (vals ent-maps))) " properties")) - (validate-client-db @conn ent-maps options))) - -(defn -main [argv] - (let [{:keys [args opts]} (cli/parse-args argv {:spec spec}) - _ (when (or (empty? args) (:help opts)) - (println (str "Usage: $0 GRAPH-NAME [& ADDITIONAL-GRAPHS] [OPTIONS]\nOptions:\n" - (cli/format-opts {:spec spec}))) - (js/process.exit 1))] - (doseq [graph-dir args] - (validate-graph graph-dir opts)))) - -(when (= nbb/*file* (:file (meta #'-main))) - (-main *command-line-args*))