dev: add tasks for saving and diffing datoms

part of LOG-2974

bb.edn

@@ -57,6 +57,16 @@
   {:doc "Transact against a DB graph's datascript db"
    :task (apply shell {:dir "deps/outliner"} "yarn -s nbb-logseq script/transact.cljs" *command-line-args*)}
 
+  dev:db-datoms
+  {:doc "Write db's datoms to a file"
+   :requires ([babashka.fs :as fs])
+   :task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
+                "yarn -s nbb-logseq script/dump_datoms.cljs"
+                *command-line-args*)}
+
+  dev:diff-datoms
+  logseq.tasks.dev/diff-datoms
+
   dev:npx-cap-run-ios
   logseq.tasks.dev.mobile/npx-cap-run-ios
 

deps/db/script/dump_datoms.cljs

@@ -0,0 +1,32 @@
+  (ns dump-datoms
+    "An example script that dumps all eavt datoms to a specified edn file
+
+     $ yarn -s nbb-logseq script/dump_datoms.cljs db-name datoms.edn"
+    (:require [datascript.core :as d]
+              [clojure.pprint :as pprint]
+              [logseq.db.sqlite.db :as sqlite-db]
+              [nbb.core :as nbb]
+              ["path" :as path]
+              ["os" :as os]
+              ["fs" :as fs]))
+
+(defn read-graph
+  "The db graph bare version of gp-cli/parse-graph"
+  [graph-name]
+  (let [graphs-dir (path/join (os/homedir) "logseq/graphs")]
+    (sqlite-db/open-db! graphs-dir graph-name)))
+
+(defn -main [args]
+  (when (< (count args) 2)
+    (println "Usage: $0 GRAPH FILE")
+    (js/process.exit 1))
+  (let [[graph-name file*] args
+        conn (read-graph graph-name)
+        datoms (mapv #(vec %) (d/datoms @conn :eavt))
+        parent-dir (or js/process.env.ORIGINAL_PWD ".")
+        file (path/join parent-dir file*)]
+    (println "Writing" (count datoms) "datoms to" file)
+    (fs/writeFileSync file (with-out-str (pprint/pprint datoms)))))
+
+(when (= nbb/*file* (:file (meta #'-main)))
+  (-main *command-line-args*))

docs/dev-practices.md

@@ -377,6 +377,81 @@ These tasks are specific to database graphs. For these tasks there is a one time
   Updated 16 block(s) for graph test-db!
   ```
 
+* `dev:db-datoms` and `dev:diff-datoms` - Save a db's datoms to file and diff two datom files
+
+  ```sh
+  # Save a current datoms snapshot of a graph
+  $ bb dev:db-datoms woot w2.edn
+  # After some edits, save another datoms snapshot
+  $ bb dev:db-datoms woot w3.edn
+
+  # Diff the two datom snapshots
+  # This snapshot correctly shows an added block with content "b7" and a property using a closed :default value
+  $  bb dev:diff-datoms w2.edn w3.edn
+  [[]
+  [[162 :block/content "b7" 536871039 true]
+    [162 :block/created-at 1703004379103 536871037 true]
+    [162 :block/format :markdown 536871037 true]
+    [162 :block/page 149 536871037 true]
+    [162 :block/parent 149 536871037 true]
+    [162 :block/path-refs 108 536871044 true]
+    [162 :block/path-refs 149 536871044 true]
+    [162 :block/path-refs 160 536871044 true]
+    [162
+    :block/properties
+    {#uuid "21be4275-bba9-48b8-9351-c9ca27883159"
+      #uuid "6581b09e-8b9c-4dca-a938-c900aedc8275"}
+    536871043
+    true]
+    [162 :block/refs 108 536871043 true]
+    [162 :block/refs 160 536871043 true]
+    [162
+    :block/uuid
+    #uuid "6581c8db-a2a2-4e09-b30d-cdea6ad69512"
+    536871037
+    true]]]
+  
+  # By default this task ignores commonly changing datascript attributes.
+  # To see all changed attributes, tell the task to ignore a nonexistent attribute:
+  $ bb dev:diff-datoms w2.edn w3.edn -i a
+  [[[nil nil 536871029 536871030]
+    [nil nil 1702998192728 536871029]
+    [nil nil 536871035 536871036]
+    [nil nil 1703000139716 536871035]
+    [nil nil 149 536871033]
+    [nil nil 536871035 536871036]]
+  [[nil nil 536871041 536871042]
+    [nil nil 1703004384793 536871041]
+    [nil nil 536871039 536871040]
+    [nil nil 1703004380918 536871039]
+    [nil nil 162 536871037]
+    [nil nil 536871037 536871038]
+    [162 :block/content "b7" 536871039 true]
+    [162 :block/created-at 1703004379103 536871037 true]
+    [162 :block/format :markdown 536871037 true]
+    [162 :block/left 149 536871037 true]
+    [162 :block/page 149 536871037 true]
+    [162 :block/parent 149 536871037 true]
+    [162 :block/path-refs 108 536871044 true]
+    [162 :block/path-refs 149 536871044 true]
+    [162 :block/path-refs 160 536871044 true]
+    [162
+    :block/properties
+    {#uuid "21be4275-bba9-48b8-9351-c9ca27883159"
+      #uuid "6581b09e-8b9c-4dca-a938-c900aedc8275"}
+    536871043
+    true]
+    [162 :block/refs 108 536871043 true]
+    [162 :block/refs 160 536871043 true]
+    [162 :block/tx-id 536871043 536871044 true]
+    [162 :block/updated-at 1703004380918 536871039 true]
+    [162
+    :block/uuid
+    #uuid "6581c8db-a2a2-4e09-b30d-cdea6ad69512"
+    536871037
+    true]]]
+  ```
+
 ### Dev Commands
 
 In the app, you can enable Dev commands under `Settings > Advanced > Developer

scripts/src/logseq/tasks/dev.clj

@@ -3,9 +3,11 @@
   namespaces"
   (:require [babashka.process :refer [shell]]
             [babashka.fs :as fs]
+            [babashka.cli :as cli]
             [clojure.java.io :as io]
             [clojure.pprint :as pp]
-            [clojure.edn :as edn]))
+            [clojure.edn :as edn]
+            [clojure.data :as data]))
 
 (defn lint
   "Run all lint tasks
@@ -49,3 +51,19 @@
       (println "Building publishing js asset...")
       (shell "clojure -M:cljs release publishing"))
     (println "Publishing js asset is up to date")))
+
+(defn diff-datoms
+  "Runs data/diff on two edn files written by dev:db-datoms"
+  [file1 file2 & args]
+  (let [spec {:ignored-attributes
+              ;; Ignores some attributes by default that are expected to change often
+              {:alias :i :coerce #{:keyword} :default #{:block/tx-id :block/left :block/updated-at}}}
+        {{:keys [ignored-attributes]} :opts} (cli/parse-args args {:spec spec})
+        datom-filter (fn [[e a _ _ _]] (contains? ignored-attributes a))
+        data-diff* (apply data/diff (map (fn [x] (->> x slurp edn/read-string (remove datom-filter))) [file1 file2]))
+        data-diff (->> data-diff*
+                       ;; Drop common as we're only interested in differences
+                       drop-last
+                       ;; Remove nils as we're only interested in diffs
+                       (mapv #(vec (remove nil? %))))]
+    (pp/pprint data-diff)))