clone/logseq
1
0
Fork 0
mirror of https://github.com/logseq/logseq.git synced 2026-04-24 22:25:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: graph counts after validation are wrong

Browse Source 
after type->tags migration. Also update recommended validate cli options
to default to true. Also clean up graph counts spread across two
validators and importer. The property-pairs count in the UI were way off
because they were only counting pairs that are validated by malli-schema
This commit is contained in:
Gabriel Horner
2025-01-04 02:11:38 -05:00
parent f7e150b642
commit 1bba10c266
5 changed files with 31 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
deps/db/script/validate_client_db.cljs vendored
View File

@@ -4,7 +4,6 @@
(:require [logseq.db.sqlite.cli :as sqlite-cli]
[logseq.db.frontend.malli-schema :as db-malli-schema]
[logseq.db.frontend.validate :as db-validate]
[logseq.db.frontend.property :as db-property]
[datascript.core :as d]
[clojure.string :as string]
[nbb.core :as nbb]
@@ -13,8 +12,7 @@
["path" :as node-path]
["os" :as os]
[cljs.pprint :as pprint]
[malli.error :as me]
[logseq.db :as ldb]))
[malli.error :as me]))
(defn validate-client-db
"Validate datascript db as a vec of entity maps"
@@ -57,12 +55,15 @@
{:help {:alias :h
:desc "Print help"}
:humanize {:alias :H
:default true
:desc "Humanize errors as an alternative to -v"}
:verbose {:alias :v
:desc "Print more info"}
:closed-maps {:alias :c
:default true
:desc "Validate maps marked with closed as :closed"}
:group-errors {:alias :g
:default true
:desc "Groups errors by their entity id"}})
(defn- validate-graph [graph-dir options]
@@ -77,14 +78,9 @@
(js/process.exit 1)))
datoms (d/datoms @conn :eavt)
ent-maps (db-malli-schema/datoms->entities datoms)]
(println "Read graph" (str db-name " with " (count datoms) " datoms, "
(count ent-maps) " entities, "
(count (filter :block/name ent-maps)) " pages, "
(count (filter :block/title ent-maps)) " blocks, "
(count (filter ldb/class? ent-maps)) " tags, "
(count (filter #(seq (:block/tags %)) ent-maps)) " objects, "
(count (filter ldb/property? ent-maps)) " properties and "
(count (mapcat db-property/properties ent-maps)) " property pairs"))
(println "Read graph" (str db-name " with counts: "
(pr-str (assoc (db-validate/graph-counts @conn ent-maps)
:datoms (count datoms)))))
(validate-client-db @conn ent-maps options)))
(defn -main [argv]

18
deps/db/src/logseq/db/frontend/validate.cljs vendored
View File

@@ -2,6 +2,7 @@
"Validate frontend db for DB graphs"
(:require [datascript.core :as d]
[logseq.db.frontend.malli-schema :as db-malli-schema]
[logseq.db.frontend.property :as db-property]
[malli.core :as m]
[malli.error :as me]
[malli.util :as mu]
@@ -103,8 +104,23 @@
errors (binding [db-malli-schema/*db-for-validate-fns* db]
(-> ent-maps closed-db-schema-explainer :errors))]
(cond-> {:datom-count (count datoms)
:entities ent-maps}
:entities ent-maps*}
(some? errors)
(assoc :errors (map #(-> (dissoc % :errors-by-type)
(update :errors (fn [errs] (me/humanize {:errors errs}))))
(group-errors-by-entity db ent-maps errors))))))
(defn graph-counts
"Calculates graph-wide counts given a graph's db and its entities from :eavt datoms"
[db entities]
(let [classes-count (count (d/datoms db :avet :block/tags :logseq.class/Tag))
properties-count (count (d/datoms db :avet :block/tags :logseq.class/Property))]
{:entities (count entities)
:pages (count (filter :block/name entities))
;; Nodes that aren't pages
:blocks (count (filter :block/parent entities))
:classes classes-count
:properties properties-count
;; Objects that aren't classes or properties
:objects (- (count (d/datoms db :avet :block/tags)) classes-count properties-count)
:property-pairs (count (mapcat #(-> % db-property/properties (dissoc :block/tags)) entities))}))

2
docs/dev-practices.md
View File

@@ -365,7 +365,7 @@ These tasks are specific to database graphs. For these tasks there is a one time
```sh
# One or more graphs can be validated e.g.
$ bb dev:validate-db test-db schema -c -g
$ bb dev:validate-db test-db schema
Read graph test-db with 1572 datoms, 220 entities and 13 properties
Valid!
Read graph schema with 26105 datoms, 2320 entities and 3168 properties

17
src/main/frontend/components/imports.cljs
View File

@@ -31,8 +31,7 @@
[logseq.shui.dialog.core :as shui-dialog]
[logseq.shui.form.core :as form-core]
[lambdaisland.glogi :as log]
[logseq.db.frontend.validate :as db-validate]
[logseq.db :as ldb]))
[logseq.db.frontend.validate :as db-validate]))
;; Can't name this component as `frontend.components.import` since shadow-cljs
;; will complain about it.
@@ -278,16 +277,6 @@
(shui/button {:type "submit" :class "right-0 mt-3"} "Submit")]))])
(defn- counts-from-entities
[entities]
{:entities (count entities)
:pages (count (filter :block/name entities))
:blocks (count (filter :block/title entities))
:classes (count (filter ldb/class? entities))
:objects (count (filter #(seq (:block/tags %)) entities))
:properties (count (filter ldb/property? entities))
:property-values (count (mapcat :block/properties entities))})
(defn- validate-imported-data
[db import-state files]
(when-let [org-files (seq (filter #(= "org" (path/file-ext (:path %))) files))]
@@ -320,12 +309,12 @@
(if errors
(do
(log/error :import-errors {:msg (str "Import detected " (count errors) " invalid block(s):")
:counts (assoc (counts-from-entities entities) :datoms datom-count)})
:counts (assoc (db-validate/graph-counts db entities) :datoms datom-count)})
(pprint/pprint errors)
(notification/show! (str "Import detected " (count errors) " invalid block(s). These blocks may be buggy when you interact with them. See the javascript console for more.")
:warning false))
(log/info :import-valid {:msg "Valid import!"
:counts (assoc (counts-from-entities entities) :datoms datom-count)}))))
:counts (assoc (db-validate/graph-counts db entities) :datoms datom-count)}))))
(defn- show-notification [{:keys [msg level ex-data]}]
(if (= :error level)

17
src/main/frontend/worker/db/validate.cljs
View File

@@ -2,19 +2,8 @@
"Validate db"
(:require [cljs.pprint :as pprint]
[frontend.worker.util :as worker-util]
[logseq.db :as ldb]
[logseq.db.frontend.validate :as db-validate]))
(defn- counts-from-entities
[entities]
{:entities (count entities)
:pages (count (filter :block/name entities))
:blocks (count (filter :block/title entities))
:classes (count (filter ldb/class? entities))
:objects (count (filter #(seq (:block/tags %)) entities))
:properties (count (filter ldb/property? entities))
:property-values (count (mapcat :block/properties entities))})
(defn validate-db
[db]
(let [{:keys [errors datom-count entities]} (db-validate/validate-db! db)]
@@ -22,12 +11,12 @@
(do
(worker-util/post-message :log [:db-invalid :error
{:msg (str "Validation detected " (count errors) " invalid block(s):")
:counts (assoc (counts-from-entities entities) :datoms datom-count)}])
:counts (assoc (db-validate/graph-counts db entities) :datoms datom-count)}])
(pprint/pprint errors)
(worker-util/post-message :notification
[(str "Validation detected " (count errors) " invalid block(s). These blocks may be buggy when you interact with them. See the javascript console for more.")
:warning false]))
(worker-util/post-message :notification
[(str "Your graph is valid! " (assoc (counts-from-entities entities) :datoms datom-count))
:success false]))))
[(str "Your graph is valid! " (assoc (db-validate/graph-counts db entities) :datoms datom-count))
:success false]))))