fix: remove unused db.sqlite related fns

- remove sqlite-db/close! as its not used and didn't make sense to be
  used in other electron ns
- remove sqlite-db/connections atom as its not needed
- remove read-graph as its not necessary
- rename cli.persist-graph as its primary purpose has changed
- update effected scripts

deps/db/.carve/config.edn

@@ -1,5 +1,6 @@
 {:paths ["src"]
  :api-namespaces [logseq.db.sqlite.db
+                  logseq.db.sqlite.common-db
                   logseq.db.sqlite.rtc
                   logseq.db.sqlite.util
                   logseq.db.sqlite.cli

deps/db/script/query.cljs

@@ -5,7 +5,6 @@
     (:require [datascript.core :as d]
               [clojure.edn :as edn]
               [logseq.db.sqlite.db :as sqlite-db]
-              [logseq.db.sqlite.cli :as sqlite-cli]
               [logseq.db.frontend.rules :as rules]
               [nbb.core :as nbb]
               ["path" :as path]
@@ -15,18 +14,17 @@
   "The db graph bare version of gp-cli/parse-graph"
   [graph-name]
   (let [graphs-dir (path/join (os/homedir) "logseq/graphs")]
-    (sqlite-db/open-db! graphs-dir graph-name)
-    (sqlite-cli/read-graph graph-name)))
+    (sqlite-db/open-db! graphs-dir graph-name)))
 
 (defn -main [args]
-  (when (not= 2 (count args))
+  (when (< (count args) 2)
     (println "Usage: $0 GRAPH QUERY")
     (js/process.exit 1))
   (let [[graph-name query*] args
         conn (read-graph graph-name)
         query (into (edn/read-string query*) [:in '$ '%]) ;; assumes no :in are in queries
         results (mapv first (d/q query @conn (rules/extract-rules rules/db-query-dsl-rules)))]
-    #_(println "DB contains" (count (d/datoms @conn :eavt)) "datoms")
+    (when ((set args) "-v") (println "DB contains" (count (d/datoms @conn :eavt)) "datoms"))
     (prn results)))
 
 (when (= nbb/*file* (:file (meta #'-main)))

deps/db/script/validate_client_db.cljs

@@ -1,8 +1,7 @@
 (ns validate-client-db
   "Script that validates the datascript db of a DB graph
    NOTE: This script is also used in CI to confirm our db's schema is up to date"
-  (:require [logseq.db.sqlite.cli :as sqlite-cli]
-            [logseq.db.sqlite.db :as sqlite-db]
+  (:require [logseq.db.sqlite.db :as sqlite-db]
             [logseq.db.frontend.malli-schema :as db-malli-schema]
             [datascript.core :as d]
             [clojure.string :as string]
@@ -93,11 +92,10 @@
                               (node-path/join (or js/process.env.ORIGINAL_PWD ".") graph-dir)]
                           ((juxt node-path/dirname node-path/basename) graph-dir'))
                         [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
-        _ (try (sqlite-db/open-db! dir db-name)
-               (catch :default e
-                 (println "Error: For graph" (str (pr-str graph-dir) ":") (str e))
-                 (js/process.exit 1)))
-        conn (sqlite-cli/read-graph db-name)
+        conn (try (sqlite-db/open-db! dir db-name)
+                  (catch :default e
+                    (println "Error: For graph" (str (pr-str graph-dir) ":") (str e))
+                    (js/process.exit 1)))
         datoms (d/datoms @conn :eavt)
         ent-maps (db-malli-schema/datoms->entity-maps datoms)]
     (println "Read graph" (str db-name " with " (count datoms) " datoms, "

deps/db/src/logseq/db/sqlite/cli.cljs

@@ -1,20 +1,9 @@
 (ns ^:node-only logseq.db.sqlite.cli
   "Primary ns to interact with DB graphs with node.js based CLIs"
-  (:require [logseq.db.sqlite.db :as sqlite-db]
-            [logseq.db.sqlite.common-db :as sqlite-common-db]
-            ["fs" :as fs]
+  (:require ["fs" :as fs]
             ["path" :as node-path]))
 
 (defn db-graph-directory?
   "Returns boolean indicating if the given directory is a DB graph"
   [graph-dir]
-  (fs/existsSync (node-path/join graph-dir "db.sqlite")))
-
-(defn read-graph
-  "Reads a given sqlite db and returns a datascript connection of its contents.
-   The sqlite db is assumed to have already been opened by sqlite-db/open-db!"
-  [db-name]
-  (-> (sqlite-db/get-conn db-name)
-      deref
-      sqlite-common-db/get-initial-data
-      sqlite-common-db/restore-initial-data))
+  (fs/existsSync (node-path/join graph-dir "db.sqlite")))

deps/db/src/logseq/db/sqlite/db.cljs

@@ -6,26 +6,15 @@
             [logseq.db.sqlite.common-db :as sqlite-common-db]
             [logseq.db.sqlite.util :as sqlite-util]
             ;; FIXME: datascript.core has to come before datascript.storage or else nbb fails
+            #_:clj-kondo/ignore
             [datascript.core :as d]
             [datascript.storage :refer [IStorage]]
             [goog.object :as gobj]
             [clojure.edn :as edn]))
 
-;; sqlite databases
-(defonce databases (atom nil))
-;; datascript conns
-(defonce conns (atom nil))
-
 ;; Reference same sqlite default class in cljs + nbb without needing .cljc
 (def sqlite (if (find-ns 'nbb.core) (aget sqlite3 "default") sqlite3))
 
-(defn close!
-  []
-  (when @databases
-    (doseq [[_ database] @databases]
-      (.close database))
-    (reset! databases nil)))
-
 (defn sanitize-db-name
   [db-name]
   (-> db-name
@@ -34,10 +23,6 @@
       (string/replace "\\" "_")
       (string/replace ":" "_"))) ;; windows
 
-(defn get-conn
-  [repo]
-  (get @conns (sanitize-db-name repo)))
-
 (defn get-db-full-path
   [graphs-dir db-name]
   (let [db-name' (sanitize-db-name db-name)
@@ -83,22 +68,13 @@
         (edn/read-string content)))))
 
 (defn open-db!
+  "For a given database name, opens a sqlite db connection for it, creates
+  needed sqlite tables if not created and returns a datascript connection that's
+  connected to the sqlite db"
   [graphs-dir db-name]
-  (let [[db-sanitized-name db-full-path] (get-db-full-path graphs-dir db-name)
+  (let [[_db-sanitized-name db-full-path] (get-db-full-path graphs-dir db-name)
         db (new sqlite db-full-path nil)]
     (sqlite-common-db/create-kvs-table! db)
-    (swap! databases assoc db-sanitized-name db)
     (let [storage (new-sqlite-storage db)
           conn (sqlite-common-db/get-storage-conn storage)]
-      (swap! conns assoc db-sanitized-name conn)))
-  nil)
-
-;; TODO: Remove as it looks unused
-(defn transact!
-  [repo tx-data tx-meta]
-  (when-let [conn (get-conn repo)]
-    (try
-      (d/transact! conn tx-data tx-meta)
-      (catch :default e
-        (prn :debug :error)
-        (js/console.error e)))))
+      conn)))

deps/db/test/logseq/db/sqlite/db_test.cljs

@@ -24,11 +24,11 @@
 (deftest get-initial-data
   (testing "Fetches a defined block"
     (create-graph-dir "tmp/graphs" "test-db")
-    (sqlite-db/open-db! "tmp/graphs" "test-db")
-    (let [blocks [{:block/uuid (random-uuid)
+    
+    (let [conn* (sqlite-db/open-db! "tmp/graphs" "test-db")
+          blocks [{:block/uuid (random-uuid)
                    :file/path "logseq/config.edn"
                    :file/content "{:foo :bar}"}]
-          conn* (sqlite-db/get-conn "test-db")
           _ (d/transact! conn* blocks)
           ;; Simulate getting data from sqlite and restoring it for frontend
           conn (-> (sqlite-common-db/get-initial-data @conn*)
@@ -42,8 +42,8 @@
 (deftest restore-initial-data
   (testing "Restore a journal page with its block"
     (create-graph-dir "tmp/graphs" "test-db")
-    (sqlite-db/open-db! "tmp/graphs" "test-db")
-    (let [page-uuid (random-uuid)
+    (let [conn* (sqlite-db/open-db! "tmp/graphs" "test-db")
+          page-uuid (random-uuid)
           block-uuid (random-uuid)
           created-at (js/Date.now)
           blocks [{:db/id 100001
@@ -58,7 +58,6 @@
                    :block/page {:db/id 100001}
                    :block/created-at created-at
                    :block/updated-at created-at}]
-          conn* (sqlite-db/get-conn "test-db")
           _ (d/transact! conn* blocks)
           ;; Simulate getting data from sqlite and restoring it for frontend
           conn (-> (sqlite-common-db/get-initial-data @conn*)

deps/graph-parser/test/logseq/graph_parser/cli_test.cljs

@@ -5,7 +5,6 @@
             [clojure.string :as string]
             [datascript.core :as d]
             [logseq.db.sqlite.db :as sqlite-db]
-            [logseq.db.sqlite.cli :as sqlite-cli]
             [logseq.db :as ldb]
             [clojure.set :as set]
             ["fs" :as fs]
@@ -127,10 +126,9 @@
   "Creates a sqlite-based db graph given a map of pages to blocks and returns a datascript db.
    Blocks in a map can only be top-level blocks with no referenced content"
   [dir db-name pages-to-blocks]
-  (sqlite-db/open-db! dir db-name)
-  (let [frontend-blocks (create-frontend-blocks pages-to-blocks)
-        _ (sqlite-db/transact! db-name frontend-blocks {})
-        conn (sqlite-cli/read-graph db-name)]
+  (let [conn (sqlite-db/open-db! dir db-name)
+        frontend-blocks (create-frontend-blocks pages-to-blocks)
+        _ (d/transact! conn frontend-blocks)]
     (ldb/create-default-pages! conn {:db-graph? true})
     @conn))
 

deps/outliner/.carve/ignore

@@ -1,2 +1,2 @@
 ;; API fn
-logseq.outliner.cli.persist-graph/add-listener
+logseq.outliner.cli.pipeline/add-listener

deps/outliner/script/transact.cljs

@@ -1,7 +1,6 @@
 (ns transact
   "This script generically runs transactions against the queried blocks"
-  (:require [logseq.outliner.cli.persist-graph :as persist-graph]
-            [logseq.db.sqlite.cli :as sqlite-cli]
+  (:require [logseq.outliner.cli.pipeline :as cli-pipeline]
             [logseq.db.sqlite.db :as sqlite-db]
             [logseq.db.frontend.rules :as rules]
             [datascript.core :as d]
@@ -20,8 +19,7 @@
         [dir db-name] (if (string/includes? graph-dir "/")
                         ((juxt node-path/dirname node-path/basename) graph-dir)
                         [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
-        _ (sqlite-db/open-db! dir db-name)
-        conn (sqlite-cli/read-graph db-name)
+        conn (sqlite-db/open-db! dir db-name)
         ;; find blocks to update
         query (into (edn/read-string query*) [:in '$ '%]) ;; assumes no :in are in queries
         transact-fn (edn/read-string transact-fn*)
@@ -35,7 +33,7 @@
           (println "With the following blocks updated:")
           (prn (map #(select-keys (d/entity @conn %) [:block/name :block/content]) blocks-to-update)))
       (do
-        (persist-graph/add-listener conn db-name)
+        (cli-pipeline/add-listener conn)
         (d/transact! conn update-tx)
         (println "Updated" (count update-tx) "block(s) for graph" (str db-name "!"))))))
 

deps/outliner/src/logseq/outliner/cli/persist_graph.cljs

@@ -1,32 +0,0 @@
-(ns ^:node-only logseq.outliner.cli.persist-graph
-  "This ns allows DB graphs to persist datascript changes to their respective
-  sqlite db. Since changes are persisted, this can be used to create or update graphs.
-   Known limitations:
-   * Deleted blocks don't update effected :block/tx-id"
-  (:require [datascript.core :as d]
-            [logseq.db.sqlite.db :as sqlite-db]
-            [logseq.outliner.datascript-report :as ds-report]
-            [logseq.outliner.pipeline :as outliner-pipeline]))
-
-(defn- invoke-hooks
-  "Modified copy of frontend.modules.outliner.pipeline/invoke-hooks that doesn't
-  handle :block/tx-id"
-  [conn tx-report]
-  (when (not (get-in tx-report [:tx-meta :replace?]))
-    (let [{:keys [blocks]} (ds-report/get-blocks-and-pages tx-report)
-          block-path-refs-tx (outliner-pipeline/compute-block-path-refs-tx tx-report blocks)]
-      (d/transact! conn block-path-refs-tx {:replace? true})
-      ;; frontend also passes original tx-report
-      tx-report)))
-
-(defn- update-sqlite-db
-  "Same as :db-transact-data defmethod in electron.handler"
-  [db-name tx-report]
-  (sqlite-db/transact! db-name (:tx-data tx-report) (:tx-meta tx-report)))
-
-(defn add-listener
-  "Adds a listener to the datascript connection to persist changes to the given
-  sqlite db name"
-  [conn db-name]
-  (d/listen! conn :persist-to-sqlite (fn persist-to-sqlite [tx-report]
-                                       (update-sqlite-db db-name (invoke-hooks conn tx-report)))))

deps/outliner/src/logseq/outliner/cli/pipeline.cljs

@@ -0,0 +1,23 @@
+(ns ^:node-only logseq.outliner.cli.pipeline
+  "This ns provides a datascript listener for DB graphs to add additional changes
+   that the frontend also adds per transact.
+   Known limitations:
+   * Deleted blocks don't update effected :block/tx-id"
+  (:require [datascript.core :as d]
+            [logseq.outliner.datascript-report :as ds-report]
+            [logseq.outliner.pipeline :as outliner-pipeline]))
+
+(defn- invoke-hooks
+  "Modified copy of frontend.modules.outliner.pipeline/invoke-hooks that doesn't
+  handle :block/tx-id"
+  [conn tx-report]
+  (when (not (get-in tx-report [:tx-meta :replace?]))
+    (let [{:keys [blocks]} (ds-report/get-blocks-and-pages tx-report)
+          block-path-refs-tx (outliner-pipeline/compute-block-path-refs-tx tx-report blocks)]
+      (d/transact! conn block-path-refs-tx {:replace? true}))))
+
+(defn add-listener
+  "Adds a listener to the datascript connection to add additional changes from outliner.pipeline"
+  [conn]
+  (d/listen! conn :pipeline-updates (fn pipeline-updates [tx-report]
+                                      (invoke-hooks conn tx-report))))

scripts/README.md

@@ -45,7 +45,5 @@ Created graph schema!
 
 #### Update graph scripts
 
-For database graphs, it is possible to update graphs with the
-`logseq.outliner.cli.persist-graph`.
-ns. This ns makes it easy to write scripts that update graphs using datascript
-and logseq's schema. TODO
+For database graphs, it is recommended to use
+`logseq.outliner.cli.pipeline/add-listener!` when updating graphs.  TODO

scripts/src/logseq/tasks/db_graph/create_graph.cljs

@@ -7,7 +7,7 @@
             [logseq.db.sqlite.util :as sqlite-util]
             [logseq.db.sqlite.create-graph :as sqlite-create-graph]
             [logseq.db.frontend.property.util :as db-property-util]
-            [logseq.outliner.cli.persist-graph :as persist-graph]
+            [logseq.outliner.cli.pipeline :as cli-pipeline]
             [logseq.db :as ldb]
             [clojure.string :as string]
             [datascript.core :as d]
@@ -34,10 +34,9 @@
   transacts initial data"
   [dir db-name]
   (fs/mkdirSync (node-path/join dir db-name) #js {:recursive true})
-  (sqlite-db/open-db! dir db-name)
   ;; Same order as frontend.db.conn/start!
-  (let [conn (ldb/start-conn :create-default-pages? false)]
-    (persist-graph/add-listener conn db-name)
+  (let [conn (sqlite-db/open-db! dir db-name)]
+    (cli-pipeline/add-listener conn)
     (ldb/create-default-pages! conn {:db-graph? true})
     (setup-init-data conn)
     conn))

scripts/src/logseq/tasks/dev/publishing.cljs

@@ -23,8 +23,7 @@
 
 (defn- publish-db-graph [static-dir graph-dir output-path]
   (let [db-name (node-path/basename graph-dir)
-        _ (sqlite-db/open-db! (node-path/dirname graph-dir) db-name)
-        conn (sqlite-cli/read-graph db-name)
+        conn (sqlite-db/open-db! (node-path/dirname graph-dir) db-name)
         repo-config (-> (d/q '[:find ?content
                                :where [?b :file/path "logseq/config.edn"] [?b :file/content ?content]]
                              @conn)

src/electron/electron/core.cljs

@@ -308,7 +308,6 @@
 (defn main []
   (if-not (.requestSingleInstanceLock app)
     (do
-      (db/close!)
       (search/close!)
       (.quit app))
     (let [privileges {:standard        true
@@ -338,7 +337,6 @@
                                      (logger/debug "window-all-closed" "Quitting...")
                                      (try
                                        (fs-watcher/close-watcher!)
-                                       (db/close!)
                                        (search/close!)
                                        (catch :default e
                                          (logger/error "window-all-closed" e)))

src/electron/electron/db.cljs

@@ -7,8 +7,6 @@
             [logseq.db.sqlite.db :as sqlite-db]
             [electron.backup-file :as backup-file]))
 
-(def close! sqlite-db/close!)
-
 (defn get-graphs-dir
   []
   (let [path (.getPath ^object app "home")]