refactor: Update importer to use validate-local-db

Also update validation in DB scripts:
* replace alternative validate-db impl with standard CLI one
* Add validation to import script
* Choose to decouple other DB scripts from validate-db as it may go away
  later

deps/cli/src/logseq/cli/commands/validate.cljs

@@ -7,9 +7,10 @@
             [logseq.db.frontend.validate :as db-validate]))
 
 (defn- validate-db [db db-name options]
-  (if-let [errors (:errors (db-validate/validate-local-db!
-                            db
-                            (merge options {:db-name db-name :verbose true})))]
+  (if-let [errors (:errors
+                   (db-validate/validate-local-db!
+                    db
+                    (merge options {:db-name db-name :verbose true})))]
     (do
       (println "Found" (count errors)
                (if (= 1 (count errors)) "entity" "entities")

deps/db/script/create_graph.cljs

@@ -7,11 +7,12 @@
             [clojure.edn :as edn]
             [logseq.db :as ldb]
             [logseq.db.common.sqlite-cli :as sqlite-cli]
+            [logseq.db.frontend.validate :as db-validate]
             [logseq.db.sqlite.export :as sqlite-export]
             [logseq.outliner.cli :as outliner-cli]
+            [clojure.pprint :as pprint]
             [nbb.classpath :as cp]
-            [nbb.core :as nbb]
-            [validate-db]))
+            [nbb.core :as nbb]))
 
 (defn- resolve-path
   "If relative path, resolve with $ORIGINAL_PWD"
@@ -20,6 +21,19 @@
     path
     (node-path/join (or js/process.env.ORIGINAL_PWD ".") path)))
 
+(defn- validate-db [db db-name options]
+  (if-let [errors (:errors
+                   (db-validate/validate-local-db!
+                    db
+                    (merge options {:db-name db-name :verbose true})))]
+    (do
+      (println "Found" (count errors)
+               (if (= 1 (count errors)) "entity" "entities")
+               "with errors:")
+      (pprint/pprint errors)
+      (js/process.exit 1))
+    (println "Valid!")))
+
 (def spec
   "Options spec"
   {:help {:alias :h
@@ -56,7 +70,7 @@
     (when (seq misc-tx) (ldb/transact! conn misc-tx))
     (println (if graph-exists? "Updated graph" "Created graph") (str db-name "!"))
     (when (:validate options)
-      (validate-db/validate-db @conn db-name {:group-errors true :closed-maps true :humanize true}))))
+      (validate-db @conn db-name {}))))
 
 (when (= nbb/*file* (nbb/invoked-file))
   (-main *command-line-args*))

deps/db/script/validate_db.cljs

@@ -11,67 +11,23 @@
             [malli.error :as me]
             [nbb.core :as nbb]))
 
-(defn validate-db*
-  "Validate datascript db as a vec of entity maps"
-  [db ent-maps* {:keys [verbose group-errors humanize closed-maps]}]
-  (let [ent-maps (db-malli-schema/update-properties-in-ents db ent-maps*)
-        explainer (db-validate/get-schema-explainer closed-maps)]
-    (if-let [explanation (binding [db-malli-schema/*db-for-validate-fns* db]
-                           (->> (map (fn [e] (dissoc e :db/id)) ent-maps) explainer not-empty))]
-      (do
-        (if group-errors
-          (let [ent-errors (db-validate/group-errors-by-entity db ent-maps (:errors explanation))]
-            (println "Found" (count ent-errors) "entities in errors:")
-            (cond
-              verbose
-              (pprint/pprint ent-errors)
-              humanize
-              (pprint/pprint (map #(update % :errors (fn [errs]
-                                                       (->> (me/humanize {:errors errs})
-                                                            vals
-                                                            (apply merge-with into))))
-                                  ent-errors))
-              :else
-              (pprint/pprint (map :entity ent-errors))))
-          (let [errors (:errors explanation)]
-            (println "Found" (count errors) "errors:")
-            (cond
-              verbose
-              (pprint/pprint
-               (map #(assoc %
-                            :entity (get ent-maps (-> % :in first))
-                            :schema (m/form (:schema %)))
-                    errors))
-              humanize
-              (pprint/pprint (me/humanize {:errors errors}))
-              :else
-              (pprint/pprint errors))))
-        (js/process.exit 1))
-      (println "Valid!"))))
-
 (def spec
   "Options spec"
   {:help {:alias :h
-          :desc "Print help"}
-   :humanize {:alias :H
-              :default true
-              :desc "Humanize errors as an alternative to -v"}
-   :verbose {:alias :v
-             :desc "Print more info"}
-   :closed-maps {:alias :c
-                 :default true
-                 :desc "Validate maps marked with closed as :closed"}
-   :group-errors {:alias :g
-                  :default true
-                  :desc "Groups errors by their entity id"}})
+          :desc "Print help"}})
 
-(defn validate-db [db db-name options]
-  (let [datoms (d/datoms db :eavt)
-        ent-maps (db-malli-schema/datoms->entities datoms)]
-    (println "Read graph" (str db-name " with counts: "
-                               (pr-str (assoc (db-validate/graph-counts db ent-maps)
-                                              :datoms (count datoms)))))
-    (validate-db* db ent-maps options)))
+(defn- validate-db [db db-name options]
+  (if-let [errors (:errors
+                   (db-validate/validate-local-db!
+                    db
+                    (merge options {:db-name db-name :verbose true})))]
+    (do
+      (println "Found" (count errors)
+               (if (= 1 (count errors)) "entity" "entities")
+               "with errors:")
+      (pprint/pprint errors)
+      (js/process.exit 1))
+    (println "Valid!")))
 
 (defn- validate-graph [graph-dir options]
   (let [open-db-args (sqlite-cli/->open-db-args graph-dir)

deps/db/src/logseq/db/frontend/validate.cljs

@@ -62,7 +62,7 @@
                 [false errors]))
             [true nil]))))))
 
-(defn group-errors-by-entity
+(defn- group-errors-by-entity
   "Groups malli errors by entities. db is used for providing more debugging info"
   [db ent-maps errors]
   (assert (vector? ent-maps) "Must be a vec for grouping to work")
@@ -117,7 +117,7 @@
 
 (defn validate-local-db!
   "Validates a local (non-RTC) DB like validate-db! but with default behavior,
-  options and logging specific to a local DB. Used by CLI and tests"
+  options and logging specific to a local DB. Used by CLI, importer and tests"
   [db & {:keys [db-name open-schema verbose]}]
   (let [datoms (d/datoms db :eavt)
         ent-maps (db-malli-schema/datoms->entities datoms)

deps/graph-parser/script/db_import.cljs

@@ -14,9 +14,9 @@
             [logseq.common.graph :as common-graph]
             [logseq.db.common.sqlite-cli :as sqlite-cli]
             [logseq.db.frontend.asset :as db-asset]
+            [logseq.db.frontend.validate :as db-validate]
             [logseq.graph-parser.exporter :as gp-exporter]
             [logseq.outliner.cli :as outliner-cli]
-            [logseq.outliner.pipeline :as outliner-pipeline]
             [nbb.classpath :as cp]
             [nbb.core :as nbb]
             [promesa.core :as p]))
@@ -145,6 +145,19 @@
       (p/let [_ (gp-exporter/export-doc-files conn files' <read-file doc-options)]
         {:import-state (:import-state doc-options)}))))
 
+(defn- validate-db [db db-name options]
+  (if-let [errors (:errors
+                   (db-validate/validate-local-db!
+                    db
+                    (merge options {:db-name db-name :verbose true})))]
+    (do
+      (println "Found" (count errors)
+               (if (= 1 (count errors)) "entity" "entities")
+               "with errors:")
+      (pprint/pprint errors)
+      (js/process.exit 1))
+    (println "Valid!")))
+
 (def spec
   "Options spec"
   {:help {:alias :h
@@ -171,7 +184,10 @@
    :property-parent-classes
    {:alias :P
     :coerce []
-    :desc "List of properties whose values convert to a parent class"}})
+    :desc "List of properties whose values convert to a parent class"}
+   :validate
+   {:alias :V
+    :desc "Validate db after creation"}})
 
 (defn -main [args]
   (let [[file-graph db-graph-dir] args
@@ -197,7 +213,7 @@
                        (set/rename-keys {:all-tags :convert-all-tags? :remove-inline-tags :remove-inline-tags?}))
         _ (when (:verbose options) (prn :options user-options))
         options' (merge {:user-options user-options}
-                        (select-keys options [:files :verbose :continue :debug]))]
+                        (select-keys options [:files :verbose :continue :debug :validate]))]
     (p/let [{:keys [import-state]}
             (if directory?
               (import-file-graph-to-db file-graph' db-full-dir conn options')
@@ -210,7 +226,8 @@
       (when-let [ignored-files (seq @(:ignored-files import-state))]
         (println (count ignored-files) "ignored file(s):" (pr-str (vec ignored-files))))
       (when (:verbose options') (println "Transacted" (count (d/datoms @conn :eavt)) "datoms"))
-      (println "Created graph" (str db-name "!")))))
+      (println "Created graph" (str db-name "!"))
+      (when (:validate options') (validate-db @conn db-name {})))))
 
 (when (= nbb/*file* (nbb/invoked-file))
   (-main *command-line-args*))

deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs

@@ -182,7 +182,7 @@
       (is (< (-> end-time (- start-time) (/ 1000)) max-time)
           (str "Importing large graph takes less than " max-time "s")))
 
-    (is (empty? (map :entity (:errors (db-validate/validate-db! @conn))))
+    (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
         "Created graph has no validation errors")
     (is (= 0 (count @(:ignored-properties import-state))) "No ignored properties")
     (is (= 0 (count @(:ignored-assets import-state))) "No ignored assets")
@@ -205,7 +205,7 @@
 
     (testing "whole graph"
 
-      (is (empty? (map :entity (:errors (db-validate/validate-db! @conn))))
+      (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
           "Created graph has no validation errors")
 
       ;; Counts
@@ -617,7 +617,7 @@
           {:keys [import-state]}
           (import-file-graph-to-db file-graph-dir conn {:convert-all-tags? false})]
 
-    (is (empty? (map :entity (:errors (db-validate/validate-db! @conn))))
+    (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
         "Created graph has no validation errors")
     (is (= 0 (count @(:ignored-properties import-state))) "No ignored properties")
     (is (= 0 (->> @conn
@@ -672,7 +672,7 @@
           files (mapv #(node-path/join file-graph-dir %) ["journals/2024_02_07.md" "pages/Interstellar.md"])
           conn (db-test/create-conn)
           _ (import-files-to-db files conn {:tag-classes ["movie"]})]
-    (is (empty? (map :entity (:errors (db-validate/validate-db! @conn))))
+    (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
         "Created graph has no validation errors")
 
     (let [block (db-test/find-block-by-content @conn #"Inception")
@@ -703,7 +703,7 @@
           _ (import-files-to-db files conn {:property-classes ["type"]})
           _ (@#'gp-exporter/export-class-properties conn conn)]
 
-    (is (empty? (map :entity (:errors (db-validate/validate-db! @conn))))
+    (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
         "Created graph has no validation errors")
 
     (is (= #{:user.class/Property :user.class/Movie :user.class/Class :user.class/Tool}
@@ -746,7 +746,7 @@
           conn (db-test/create-conn)
           _ (import-files-to-db files conn {:remove-inline-tags? false :convert-all-tags? true})]
 
-    (is (empty? (map :entity (:errors (db-validate/validate-db! @conn))))
+    (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
         "Created graph has no validation errors")
     (is (string/starts-with? (:block/title (db-test/find-block-by-content @conn #"Inception"))
                              "Inception #Movie")
@@ -772,7 +772,7 @@
                                             ;; Also add this option to trigger some edge cases with namespace pages
                                             :property-classes ["type"]})]
 
-    (is (empty? (map :entity (:errors (db-validate/validate-db! @conn))))
+    (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
         "Created graph has no validation errors")
 
     (is (= #{:user.class/Movie :user.class/CreativeWork :user.class/Thing :user.class/Feature
@@ -796,7 +796,7 @@
           _ (import-files-to-db files conn {:user-config {:property-pages/enabled? false
                                                           :property-pages/excludelist #{:prop-string}}})]
 
-    (is (empty? (map :entity (:errors (db-validate/validate-db! @conn))))
+    (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
         "Created graph has no validation errors")))
 
 (deftest-async export-config-file-sets-title-format

scripts/src/logseq/tasks/dev.clj

@@ -109,5 +109,4 @@
     (doseq [file-graph file-graphs]
       (let [db-graph (fs/path parent-graph-dir (fs/file-name file-graph))]
         (println "Importing" (str db-graph) "...")
-        (apply shell "bb" "dev:db-import" file-graph db-graph import-options)
-        (shell "bb" "dev:validate-db" db-graph "-gHc")))))
+        (apply shell "bb" "dev:db-import" file-graph db-graph (concat import-options ["--validate"]))))))

src/main/frontend/components/imports.cljs

@@ -330,16 +330,14 @@
                    [:dt.m-0 [:strong (str k)]]
                    [:dd {:class "text-warning"} v]])))]
      :warning false))
-  (let [{:keys [errors datom-count entities]} (db-validate/validate-db! db)]
+  (let [{:keys [errors]} (db-validate/validate-local-db! db {:verbose true})]
     (if errors
       (do
-        (log/error :import-errors {:msg (str "Import detected " (count errors) " invalid block(s):")
-                                   :counts (assoc (db-validate/graph-counts db entities) :datoms datom-count)})
+        (log/error :import-errors {:msg (str "Import detected " (count errors) " invalid block(s):")})
         (pprint/pprint errors)
         (notification/show! (str "Import detected " (count errors) " invalid block(s). These blocks may be buggy when you interact with them. See the javascript console for more.")
                             :warning false))
-      (log/info :import-valid {:msg "Valid import!"
-                               :counts (assoc (db-validate/graph-counts db entities) :datoms datom-count)}))))
+      (log/info :import-valid {:msg "Valid import!"}))))
 
 (defn- show-notification [{:keys [msg level ex-data]}]
   (if (= :error level)