Merge pull request #10726 from logseq/enhance/write-to-disk

Save transactions to disk db to avoid data-loss

deps/db/src/logseq/db/sqlite/db.cljs

@@ -15,13 +15,29 @@
 ;; Reference same sqlite default class in cljs + nbb without needing .cljc
 (def sqlite (if (find-ns 'nbb.core) (aget sqlite3 "default") sqlite3))
 
+;; sqlite databases
+(defonce databases (atom nil))
+;; datascript conns
+(defonce conns (atom nil))
+
+(defn close!
+  []
+  (when @databases
+    (doseq [[_ database] @databases]
+      (.close database))
+    (reset! databases nil)))
+
 (defn sanitize-db-name
   [db-name]
   (-> db-name
       (string/replace sqlite-util/db-version-prefix "")
       (string/replace "/" "_")
       (string/replace "\\" "_")
-      (string/replace ":" "_"))) ;; windows
+      (string/replace ":" "_")))
+
+(defn get-conn
+  [repo]
+  (get @conns (sanitize-db-name repo)))
 
 (defn get-db-full-path
   [graphs-dir db-name]
@@ -76,9 +92,16 @@
   needed sqlite tables if not created and returns a datascript connection that's
   connected to the sqlite db"
   [graphs-dir db-name]
-  (let [[_db-sanitized-name db-full-path] (get-db-full-path graphs-dir db-name)
+  (let [[db-sanitized-name db-full-path] (get-db-full-path graphs-dir db-name)
         db (new sqlite db-full-path nil)]
     (sqlite-common-db/create-kvs-table! db)
+    (swap! databases assoc db-sanitized-name db)
     (let [storage (new-sqlite-storage db)
           conn (sqlite-common-db/get-storage-conn storage)]
+      (swap! conns assoc db-sanitized-name conn)
       conn)))
+
+(defn transact!
+  [repo tx-data tx-meta]
+  (when-let [conn (get-conn repo)]
+    (d/transact! conn tx-data tx-meta)))

src/electron/electron/core.cljs

@@ -308,6 +308,7 @@
 (defn main []
   (if-not (.requestSingleInstanceLock app)
     (do
+      (db/close!)
       (search/close!)
       (.quit app))
     (let [privileges {:standard        true
@@ -337,6 +338,7 @@
                                      (logger/debug "window-all-closed" "Quitting...")
                                      (try
                                        (fs-watcher/close-watcher!)
+                                       (db/close!)
                                        (search/close!)
                                        (catch :default e
                                          (logger/error "window-all-closed" e)))

src/electron/electron/db.cljs

@@ -3,10 +3,12 @@
   (:require ["path" :as node-path]
             ["fs-extra" :as fs]
             ["electron" :refer [app]]
-            ;; [electron.logger :as logger]
+            [electron.logger :as logger]
             [logseq.db.sqlite.db :as sqlite-db]
             [electron.backup-file :as backup-file]))
 
+(def close! sqlite-db/close!)
+
 (defn get-graphs-dir
   []
   (let [path (.getPath ^object app "home")]
@@ -23,6 +25,14 @@
     (fs/ensureDirSync graph-dir)
     graph-dir))
 
+(defn open-db!
+  [db-name]
+  (let [graphs-dir (get-graphs-dir)]
+    (try (sqlite-db/open-db! graphs-dir db-name)
+         (catch :default e
+           (js/console.error e)
+           (logger/error (str e ": " db-name))))))
+
 (defn save-db!
   [db-name data]
   (let [graph-dir (ensure-graph-dir! db-name)
@@ -50,4 +60,4 @@
                     new-path)]
     (when (fs/existsSync path)
       (fs/ensureDirSync unlinked)
-      (fs/moveSync path new-path'))))
+      (fs/moveSync path new-path'))))

src/electron/electron/handler.cljs

@@ -32,8 +32,10 @@
             [electron.window :as win]
             [electron.handler-interface :refer [handle]]
             [logseq.db.sqlite.util :as sqlite-util]
+            [logseq.db.sqlite.db :as sqlite-db]
             [logseq.common.graph :as common-graph]
-            [promesa.core :as p]))
+            [promesa.core :as p]
+            [clojure.edn :as edn]))
 
 (defmethod handle :mkdir [_window [_ dir]]
   (fs/mkdirSync dir))
@@ -225,14 +227,6 @@
     (fs-extra/ensureDirSync dir)
     dir))
 
-(defn- get-db-based-graphs-dir
-  []
-  (let [dir (if utils/ci?
-              (.resolve node-path js/__dirname "../tmp/graphs")
-              (.join node-path (.homedir os) "logseq" "graphs"))]
-    (fs-extra/ensureDirSync dir)
-    dir))
-
 ;; TODO: move file based graphs to "~/logseq/graphs" too
 (defn- get-file-based-graphs
   "Returns all graph names in the cache directory (starting with `logseq_local_`)"
@@ -243,20 +237,9 @@
          (map #(node-path/basename % ".transit"))
          (map graph-name->path))))
 
-(defn- get-db-based-graphs
-  "Returns all graph names in the cache directory"
-  []
-  (let [dir (get-db-based-graphs-dir)]
-    (->> (common-graph/read-directories dir)
-         (remove (fn [s] (= s db/unlinked-graphs-dir)))
-         (map graph-name->path)
-         (map (fn [s] (str sqlite-util/db-version-prefix s))))))
-
 (defn- get-graphs
   []
-  (concat
-   (get-file-based-graphs)
-   (get-db-based-graphs)))
+  (get-file-based-graphs))
 
 ;; TODO support alias mechanism
 (defn get-graph-name
@@ -366,6 +349,18 @@
   (db/ensure-graph-dir! repo)
   (db/save-db! repo data))
 
+(defmethod handle :db-open [_window [_ repo]]
+  (db/ensure-graph-dir! repo)
+  (db/open-db! repo)
+  nil)
+
+(defmethod handle :db-transact [_window [_ repo tx-data-str tx-meta-str]]
+  (when-let [conn (sqlite-db/get-conn repo)]
+    (let [tx-data (edn/read-string tx-data-str)
+          tx-meta (edn/read-string tx-meta-str)]
+      (sqlite-db/transact! repo tx-data tx-meta)
+      (:max-tx @conn))))
+
 ;; DB related IPCs End
 
 (defn clear-cache!

src/main/frontend/db_worker.cljs

@@ -18,6 +18,14 @@
 (defonce *datascript-conns (atom nil))
 (defonce *opfs-pools (atom nil))
 
+(defn sanitize-db-name
+  [db-name]
+  (-> db-name
+      (string/replace " " "_")
+      (string/replace "/" "_")
+      (string/replace "\\" "_")
+      (string/replace ":" "_")))
+
 (defn- get-sqlite-conn
   [repo]
   (get @*sqlite-conns repo))
@@ -33,7 +41,7 @@
 (defn- <get-opfs-pool
   [graph]
   (or (get-opfs-pool graph)
-      (p/let [^js pool (.installOpfsSAHPoolVfs @*sqlite #js {:name (str "logseq-pool-" graph)
+      (p/let [^js pool (.installOpfsSAHPoolVfs @*sqlite #js {:name (str "logseq-pool-" (sanitize-db-name graph))
                                                              :initialCapacity 10})]
         (swap! *opfs-pools assoc graph pool)
         pool)))
@@ -54,7 +62,7 @@
 
 (defn- get-repo-path
   [repo]
-  (str "/" repo ".sqlite"))
+  (str "/" (sanitize-db-name repo) ".sqlite"))
 
 (defn- <export-db-file
   [repo]
@@ -212,11 +220,11 @@
                           (string/replace-first (.-name file) ".logseq-pool-" "")))
                       all-files)
                 distinct)]
-     (prn :debug :all-files (map #(.-name %) all-files))
-     (prn :debug :all-files-count (count (filter
-                                          #(= (.-kind %) "file")
-                                          all-files)))
-     (prn :dbs dbs)
+     ;; (prn :debug :all-files (map #(.-name %) all-files))
+     ;; (prn :debug :all-files-count (count (filter
+     ;;                                      #(= (.-kind %) "file")
+     ;;                                      all-files)))
+     ;; (prn :dbs dbs)
      (bean/->js dbs)))
 
   (createOrOpenDB
@@ -224,6 +232,11 @@
    (p/let [_ (close-other-dbs! repo)]
      (create-or-open-db! repo)))
 
+  (getMaxTx
+   [_this repo]
+   (when-let [conn (get-datascript-conn repo)]
+     (:max-tx @conn)))
+
   (transact
    [_this repo tx-data tx-meta]
    (when-let [conn (get-datascript-conn repo)]

src/main/frontend/handler.cljs

@@ -49,8 +49,7 @@
             [cljs-bean.core :as bean]
             [frontend.handler.test :as test]
             [frontend.db.rtc.op-mem-layer :as op-mem-layer]
-            [frontend.persist-db.browser :as db-browser]
-            [frontend.persist-db :as persist-db]))
+            [frontend.persist-db.browser :as db-browser]))
 
 (defn- set-global-error-notification!
   []
@@ -247,7 +246,6 @@
                _ (mobile-util/hide-splash) ;; hide splash as early as ui is stable
                repo (or (state/get-current-repo) (:url (first repos)))
                _ (restore-and-setup! repo repos)]
-         (persist-db/run-export-periodically!)
          (when (mobile-util/native-platform?)
            (state/restore-mobile-theme!)))
        (p/catch (fn [e]

src/main/frontend/persist_db.cljs

@@ -1,11 +1,9 @@
 (ns frontend.persist-db
-   "Backend of DB based graph"
-   (:require [frontend.persist-db.browser :as browser]
-             [frontend.persist-db.protocol :as protocol]
-             [promesa.core :as p]
-             [frontend.config :as config]
-             [frontend.state :as state]
-             [frontend.util :as util]))
+  "Backend of DB based graph"
+  (:require [frontend.persist-db.browser :as browser]
+            [frontend.persist-db.protocol :as protocol]
+            [promesa.core :as p]
+            [electron.ipc :as ipc]))
 
 (defonce opfs-db (browser/->InBrowser))
 
@@ -35,28 +33,30 @@
   ([repo]
    (<fetch-init-data repo {}))
   ([repo opts]
-   (p/let [ret (protocol/<fetch-initial-data (get-impl) repo opts)]
-     (js/console.log "fetch-initial-data" ret)
-     ret)))
+   (p/do!
+    (ipc/ipc :db-open repo)
+    (protocol/<fetch-initial-data (get-impl) repo opts))))
 
 ;; FIXME: limit repo name's length
 ;; @shuyu Do we still need this?
 (defn <new [repo]
   {:pre [(<= (count repo) 56)]}
-  (p/let [_ (protocol/<new (get-impl) repo)]
-    (<export-db repo {})))
+  (p/let [_ (protocol/<new (get-impl) repo)
+          _ (<export-db repo {})]
+    (ipc/ipc :db-open repo)))
 
 (defn <release-access-handles
   [repo]
   (protocol/<release-access-handles (get-impl) repo))
 
-(defn run-export-periodically!
-  []
-  (js/setInterval
-   (fn []
-     (when-let [repo (state/get-current-repo)]
-       (when (and (util/electron?) (config/db-based-graph? repo))
-         (println :debug :save-db-to-disk repo)
-         (<export-db repo {}))))
+(comment
+  (defn run-export-periodically!
+    []
+    (js/setInterval
+     (fn []
+       (when-let [repo (state/get-current-repo)]
+         (when (and (util/electron?) (config/db-based-graph? repo))
+           (println :debug :save-db-to-disk repo)
+           (<export-db repo {}))))
    ;; every 10 minutes
-   (* 10 60 1000)))
+     (* 10 60 1000))))

src/main/frontend/persist_db/browser.cljs

@@ -44,7 +44,7 @@
     (ipc/ipc :db-export repo data)
 
     ;; TODO: browser nfs-supported? auto backup
-    
+
     ;;
     :else
     nil))
@@ -77,9 +77,13 @@
       (.releaseAccessHandles sqlite repo)))
 
   (<transact-data [_this repo tx-data tx-meta]
-    (let [^js sqlite @*sqlite]
-      (p/let [_ (when sqlite (.transact sqlite repo (pr-str tx-data) (pr-str tx-meta)))]
-        nil)))
+    (let [^js sqlite @*sqlite
+          tx-data' (pr-str tx-data)
+          tx-meta' (pr-str tx-meta)]
+      (p/do!
+       (ipc/ipc :db-transact repo tx-data' tx-meta')
+       (when sqlite (.transact sqlite repo tx-data' tx-meta'))
+       nil)))
 
   (<fetch-initial-data [_this repo _opts]
     (when-let [^js sqlite @*sqlite]