diff --git a/deps/db/script/validate_client_db.cljs b/deps/db/script/validate_client_db.cljs index d79794d13d..cf341c6669 100644 --- a/deps/db/script/validate_client_db.cljs +++ b/deps/db/script/validate_client_db.cljs @@ -4,7 +4,6 @@ (:require [logseq.db.sqlite.cli :as sqlite-cli] [logseq.db.frontend.malli-schema :as db-malli-schema] [logseq.db.frontend.validate :as db-validate] - [logseq.db.frontend.property :as db-property] [datascript.core :as d] [clojure.string :as string] [nbb.core :as nbb] @@ -13,8 +12,7 @@ ["path" :as node-path] ["os" :as os] [cljs.pprint :as pprint] - [malli.error :as me] - [logseq.db :as ldb])) + [malli.error :as me])) (defn validate-client-db "Validate datascript db as a vec of entity maps" @@ -57,12 +55,15 @@ {:help {:alias :h :desc "Print help"} :humanize {:alias :H + :default true :desc "Humanize errors as an alternative to -v"} :verbose {:alias :v :desc "Print more info"} :closed-maps {:alias :c + :default true :desc "Validate maps marked with closed as :closed"} :group-errors {:alias :g + :default true :desc "Groups errors by their entity id"}}) (defn- validate-graph [graph-dir options] @@ -77,14 +78,9 @@ (js/process.exit 1))) datoms (d/datoms @conn :eavt) ent-maps (db-malli-schema/datoms->entities datoms)] - (println "Read graph" (str db-name " with " (count datoms) " datoms, " - (count ent-maps) " entities, " - (count (filter :block/name ent-maps)) " pages, " - (count (filter :block/title ent-maps)) " blocks, " - (count (filter ldb/class? ent-maps)) " tags, " - (count (filter #(seq (:block/tags %)) ent-maps)) " objects, " - (count (filter ldb/property? ent-maps)) " properties and " - (count (mapcat db-property/properties ent-maps)) " property pairs")) + (println "Read graph" (str db-name " with counts: " + (pr-str (assoc (db-validate/graph-counts @conn ent-maps) + :datoms (count datoms))))) (validate-client-db @conn ent-maps options))) (defn -main [argv] diff --git a/deps/db/src/logseq/db/frontend/validate.cljs b/deps/db/src/logseq/db/frontend/validate.cljs index 365a291ae3..833a182b05 100644 --- a/deps/db/src/logseq/db/frontend/validate.cljs +++ b/deps/db/src/logseq/db/frontend/validate.cljs @@ -2,6 +2,7 @@ "Validate frontend db for DB graphs" (:require [datascript.core :as d] [logseq.db.frontend.malli-schema :as db-malli-schema] + [logseq.db.frontend.property :as db-property] [malli.core :as m] [malli.error :as me] [malli.util :as mu] @@ -103,8 +104,23 @@ errors (binding [db-malli-schema/*db-for-validate-fns* db] (-> ent-maps closed-db-schema-explainer :errors))] (cond-> {:datom-count (count datoms) - :entities ent-maps} + :entities ent-maps*} (some? errors) (assoc :errors (map #(-> (dissoc % :errors-by-type) (update :errors (fn [errs] (me/humanize {:errors errs})))) (group-errors-by-entity db ent-maps errors)))))) + +(defn graph-counts + "Calculates graph-wide counts given a graph's db and its entities from :eavt datoms" + [db entities] + (let [classes-count (count (d/datoms db :avet :block/tags :logseq.class/Tag)) + properties-count (count (d/datoms db :avet :block/tags :logseq.class/Property))] + {:entities (count entities) + :pages (count (filter :block/name entities)) + ;; Nodes that aren't pages + :blocks (count (filter :block/parent entities)) + :classes classes-count + :properties properties-count + ;; Objects that aren't classes or properties + :objects (- (count (d/datoms db :avet :block/tags)) classes-count properties-count) + :property-pairs (count (mapcat #(-> % db-property/properties (dissoc :block/tags)) entities))})) \ No newline at end of file diff --git a/docs/dev-practices.md b/docs/dev-practices.md index 0564ef230c..e9b607bbf9 100644 --- a/docs/dev-practices.md +++ b/docs/dev-practices.md @@ -365,7 +365,7 @@ These tasks are specific to database graphs. For these tasks there is a one time ```sh # One or more graphs can be validated e.g. - $ bb dev:validate-db test-db schema -c -g + $ bb dev:validate-db test-db schema Read graph test-db with 1572 datoms, 220 entities and 13 properties Valid! Read graph schema with 26105 datoms, 2320 entities and 3168 properties diff --git a/src/main/frontend/components/imports.cljs b/src/main/frontend/components/imports.cljs index b5fd04ef6e..9641ea6fed 100644 --- a/src/main/frontend/components/imports.cljs +++ b/src/main/frontend/components/imports.cljs @@ -31,8 +31,7 @@ [logseq.shui.dialog.core :as shui-dialog] [logseq.shui.form.core :as form-core] [lambdaisland.glogi :as log] - [logseq.db.frontend.validate :as db-validate] - [logseq.db :as ldb])) + [logseq.db.frontend.validate :as db-validate])) ;; Can't name this component as `frontend.components.import` since shadow-cljs ;; will complain about it. @@ -278,16 +277,6 @@ (shui/button {:type "submit" :class "right-0 mt-3"} "Submit")]))]) -(defn- counts-from-entities - [entities] - {:entities (count entities) - :pages (count (filter :block/name entities)) - :blocks (count (filter :block/title entities)) - :classes (count (filter ldb/class? entities)) - :objects (count (filter #(seq (:block/tags %)) entities)) - :properties (count (filter ldb/property? entities)) - :property-values (count (mapcat :block/properties entities))}) - (defn- validate-imported-data [db import-state files] (when-let [org-files (seq (filter #(= "org" (path/file-ext (:path %))) files))] @@ -320,12 +309,12 @@ (if errors (do (log/error :import-errors {:msg (str "Import detected " (count errors) " invalid block(s):") - :counts (assoc (counts-from-entities entities) :datoms datom-count)}) + :counts (assoc (db-validate/graph-counts db entities) :datoms datom-count)}) (pprint/pprint errors) (notification/show! (str "Import detected " (count errors) " invalid block(s). These blocks may be buggy when you interact with them. See the javascript console for more.") :warning false)) (log/info :import-valid {:msg "Valid import!" - :counts (assoc (counts-from-entities entities) :datoms datom-count)})))) + :counts (assoc (db-validate/graph-counts db entities) :datoms datom-count)})))) (defn- show-notification [{:keys [msg level ex-data]}] (if (= :error level) diff --git a/src/main/frontend/worker/db/validate.cljs b/src/main/frontend/worker/db/validate.cljs index 3b7e57eca5..f6ec5370f2 100644 --- a/src/main/frontend/worker/db/validate.cljs +++ b/src/main/frontend/worker/db/validate.cljs @@ -2,19 +2,8 @@ "Validate db" (:require [cljs.pprint :as pprint] [frontend.worker.util :as worker-util] - [logseq.db :as ldb] [logseq.db.frontend.validate :as db-validate])) -(defn- counts-from-entities - [entities] - {:entities (count entities) - :pages (count (filter :block/name entities)) - :blocks (count (filter :block/title entities)) - :classes (count (filter ldb/class? entities)) - :objects (count (filter #(seq (:block/tags %)) entities)) - :properties (count (filter ldb/property? entities)) - :property-values (count (mapcat :block/properties entities))}) - (defn validate-db [db] (let [{:keys [errors datom-count entities]} (db-validate/validate-db! db)] @@ -22,12 +11,12 @@ (do (worker-util/post-message :log [:db-invalid :error {:msg (str "Validation detected " (count errors) " invalid block(s):") - :counts (assoc (counts-from-entities entities) :datoms datom-count)}]) + :counts (assoc (db-validate/graph-counts db entities) :datoms datom-count)}]) (pprint/pprint errors) (worker-util/post-message :notification [(str "Validation detected " (count errors) " invalid block(s). These blocks may be buggy when you interact with them. See the javascript console for more.") :warning false])) (worker-util/post-message :notification - [(str "Your graph is valid! " (assoc (counts-from-entities entities) :datoms datom-count)) - :success false])))) + [(str "Your graph is valid! " (assoc (db-validate/graph-counts db entities) :datoms datom-count)) + :success false])))) \ No newline at end of file