From a9f62baf6df5967703a3ba8452ca4839b6a90c3f Mon Sep 17 00:00:00 2001
From: Tienson Qin <tiensonqin@gmail.com>
Date: Fri, 15 Dec 2023 09:39:56 +0800
Subject: [PATCH 1/5] wip: transact tx to disk to avoid data-loss

---
 deps/db/src/logseq/db/sqlite/db.cljs      | 29 ++++++++++++++++++++---
 src/electron/electron/core.cljs           |  2 ++
 src/electron/electron/db.cljs             | 14 +++++++++--
 src/electron/electron/handler.cljs        | 16 ++++++++++++-
 src/main/frontend/db_worker.cljs          | 17 ++++++-------
 src/main/frontend/handler.cljs            |  1 -
 src/main/frontend/persist_db.cljs         | 14 +----------
 src/main/frontend/persist_db/browser.cljs | 22 +++++++++++++----
 8 files changed, 82 insertions(+), 33 deletions(-)

diff --git a/deps/db/src/logseq/db/sqlite/db.cljs b/deps/db/src/logseq/db/sqlite/db.cljs
index 35bdc6bf89..14fac7cc3b 100644
--- a/deps/db/src/logseq/db/sqlite/db.cljs
+++ b/deps/db/src/logseq/db/sqlite/db.cljs
@@ -15,13 +15,29 @@
 ;; Reference same sqlite default class in cljs + nbb without needing .cljc
 (def sqlite (if (find-ns 'nbb.core) (aget sqlite3 "default") sqlite3))
 
+;; sqlite databases
+(defonce databases (atom nil))
+;; datascript conns
+(defonce conns (atom nil))
+
+(defn close!
+  []
+  (when @databases
+    (doseq [[_ database] @databases]
+      (.close database))
+    (reset! databases nil)))
+
 (defn sanitize-db-name
   [db-name]
   (-> db-name
       (string/replace sqlite-util/db-version-prefix "")
       (string/replace "/" "_")
       (string/replace "\\" "_")
-      (string/replace ":" "_"))) ;; windows
+      (string/replace ":" "_")))
+
+(defn get-conn
+  [repo]
+  (get @conns (sanitize-db-name repo)))
 
 (defn get-db-full-path
   [graphs-dir db-name]
@@ -72,9 +88,16 @@
   needed sqlite tables if not created and returns a datascript connection that's
   connected to the sqlite db"
   [graphs-dir db-name]
-  (let [[_db-sanitized-name db-full-path] (get-db-full-path graphs-dir db-name)
+  (let [[db-sanitized-name db-full-path] (get-db-full-path graphs-dir db-name)
         db (new sqlite db-full-path nil)]
     (sqlite-common-db/create-kvs-table! db)
+    (swap! databases assoc db-sanitized-name db)
     (let [storage (new-sqlite-storage db)
           conn (sqlite-common-db/get-storage-conn storage)]
-      conn)))
\ No newline at end of file
+      (swap! conns assoc db-sanitized-name conn)
+      conn)))
+
+(defn transact!
+  [repo tx-data tx-meta]
+  (when-let [conn (get-conn repo)]
+    (d/transact! conn tx-data tx-meta)))
diff --git a/src/electron/electron/core.cljs b/src/electron/electron/core.cljs
index c7116f41c7..09913b0df0 100644
--- a/src/electron/electron/core.cljs
+++ b/src/electron/electron/core.cljs
@@ -308,6 +308,7 @@
 (defn main []
   (if-not (.requestSingleInstanceLock app)
     (do
+      (db/close!)
       (search/close!)
       (.quit app))
     (let [privileges {:standard        true
@@ -337,6 +338,7 @@
                                      (logger/debug "window-all-closed" "Quitting...")
                                      (try
                                        (fs-watcher/close-watcher!)
+                                       (db/close!)
                                        (search/close!)
                                        (catch :default e
                                          (logger/error "window-all-closed" e)))
diff --git a/src/electron/electron/db.cljs b/src/electron/electron/db.cljs
index 1de65a5fbc..09b272616d 100644
--- a/src/electron/electron/db.cljs
+++ b/src/electron/electron/db.cljs
@@ -3,10 +3,12 @@
   (:require ["path" :as node-path]
             ["fs-extra" :as fs]
             ["electron" :refer [app]]
-            ;; [electron.logger :as logger]
+            [electron.logger :as logger]
             [logseq.db.sqlite.db :as sqlite-db]
             [electron.backup-file :as backup-file]))
 
+(def close! sqlite-db/close!)
+
 (defn get-graphs-dir
   []
   (let [path (.getPath ^object app "home")]
@@ -23,6 +25,14 @@
     (fs/ensureDirSync graph-dir)
     graph-dir))
 
+(defn open-db!
+  [db-name]
+  (let [graphs-dir (get-graphs-dir)]
+    (try (sqlite-db/open-db! graphs-dir db-name)
+         (catch :default e
+           (js/console.error e)
+           (logger/error (str e ": " db-name))))))
+
 (defn save-db!
   [db-name data]
   (let [graph-dir (ensure-graph-dir! db-name)
@@ -50,4 +60,4 @@
                     new-path)]
     (when (fs/existsSync path)
       (fs/ensureDirSync unlinked)
-      (fs/moveSync path new-path'))))
\ No newline at end of file
+      (fs/moveSync path new-path'))))
diff --git a/src/electron/electron/handler.cljs b/src/electron/electron/handler.cljs
index 756cbb28b8..13cd031a22 100644
--- a/src/electron/electron/handler.cljs
+++ b/src/electron/electron/handler.cljs
@@ -32,8 +32,10 @@
             [electron.window :as win]
             [electron.handler-interface :refer [handle]]
             [logseq.db.sqlite.util :as sqlite-util]
+            [logseq.db.sqlite.db :as sqlite-db]
             [logseq.common.graph :as common-graph]
-            [promesa.core :as p]))
+            [promesa.core :as p]
+            [clojure.edn :as edn]))
 
 (defmethod handle :mkdir [_window [_ dir]]
   (fs/mkdirSync dir))
@@ -366,6 +368,18 @@
   (db/ensure-graph-dir! repo)
   (db/save-db! repo data))
 
+(defmethod handle :db-open [_window [_ repo]]
+  (db/ensure-graph-dir! repo)
+  (db/open-db! repo)
+  nil)
+
+(defmethod handle :db-transact [_window [_ repo tx-data-str tx-meta-str]]
+  (when-let [conn (sqlite-db/get-conn repo)]
+    (let [tx-data (edn/read-string tx-data-str)
+          tx-meta (edn/read-string tx-meta-str)]
+      (sqlite-db/transact! repo tx-data tx-meta)
+      (:max-tx @conn))))
+
 ;; DB related IPCs End
 
 (defn clear-cache!
diff --git a/src/main/frontend/db_worker.cljs b/src/main/frontend/db_worker.cljs
index 35f32438e1..65eb4f6746 100644
--- a/src/main/frontend/db_worker.cljs
+++ b/src/main/frontend/db_worker.cljs
@@ -213,17 +213,18 @@
    (p/let [_ (close-other-dbs! repo)]
      (create-or-open-db! repo)))
 
+  (getMaxTx
+   [_this repo]
+   (when-let [conn (get-datascript-conn repo)]
+     (:max-tx @conn)))
+
   (transact
    [_this repo tx-data tx-meta]
    (when-let [conn (get-datascript-conn repo)]
-     (try
-       (let [tx-data (edn/read-string tx-data)
-             tx-meta (edn/read-string tx-meta)]
-         (d/transact! conn tx-data tx-meta)
-         nil)
-       (catch :default e
-         (prn :debug :error)
-         (js/console.error e)))))
+     (let [tx-data (edn/read-string tx-data)
+           tx-meta (edn/read-string tx-meta)]
+       (d/transact! conn tx-data tx-meta)
+       nil)))
 
   (getInitialData
    [_this repo]
diff --git a/src/main/frontend/handler.cljs b/src/main/frontend/handler.cljs
index 21c9ed1a5c..c89bfa4f79 100644
--- a/src/main/frontend/handler.cljs
+++ b/src/main/frontend/handler.cljs
@@ -247,7 +247,6 @@
                _ (mobile-util/hide-splash) ;; hide splash as early as ui is stable
                repo (or (state/get-current-repo) (:url (first repos)))
                _ (restore-and-setup! repo repos)]
-         (persist-db/run-export-periodically!)
          (when (mobile-util/native-platform?)
            (state/restore-mobile-theme!)))
        (p/catch (fn [e]
diff --git a/src/main/frontend/persist_db.cljs b/src/main/frontend/persist_db.cljs
index 98eb678585..137118fca7 100644
--- a/src/main/frontend/persist_db.cljs
+++ b/src/main/frontend/persist_db.cljs
@@ -43,16 +43,4 @@
 ;; @shuyu Do we still need this?
 (defn <new [repo]
   {:pre [(<= (count repo) 56)]}
-  (p/let [_ (protocol/<new (get-impl) repo)]
-    (<export-db repo {})))
-
-(defn run-export-periodically!
-  []
-  (js/setInterval
-   (fn []
-     (when-let [repo (state/get-current-repo)]
-       (when (and (util/electron?) (config/db-based-graph? repo))
-         (println :debug :save-db-to-disk repo)
-         (<export-db repo {}))))
-   ;; every 10 minutes
-   (* 10 60 1000)))
+  (protocol/<new (get-impl) repo))
diff --git a/src/main/frontend/persist_db/browser.cljs b/src/main/frontend/persist_db/browser.cljs
index 9e7adf24b5..1bce35990f 100644
--- a/src/main/frontend/persist_db/browser.cljs
+++ b/src/main/frontend/persist_db/browser.cljs
@@ -53,7 +53,8 @@
   protocol/PersistentDB
   (<new [_this repo]
     (when-let [^js sqlite @*sqlite]
-      (.createOrOpenDB sqlite repo)))
+      (.createOrOpenDB sqlite repo)
+      (ipc/ipc :db-open repo)))
 
   (<list-db [_this]
     (when-let [^js sqlite @*sqlite]
@@ -72,13 +73,24 @@
       (.unsafeUnlinkDB sqlite repo)))
 
   (<transact-data [_this repo tx-data tx-meta]
-    (let [^js sqlite @*sqlite]
-      (p/let [_ (when sqlite (.transact sqlite repo (pr-str tx-data) (pr-str tx-meta)))]
-        nil)))
+    (let [^js sqlite @*sqlite
+          tx-data' (pr-str tx-data)
+          tx-meta' (pr-str tx-meta)]
+      (p/do!
+       (p/let [start (util/time-ms)
+               _ (ipc/ipc :db-transact repo tx-data' tx-meta')]
+         (prn :debug :disk-db-transact-time-ms (- (util/time-ms) start)))
+       (p/let [start (util/time-ms)
+               _ (when sqlite (.transact sqlite repo tx-data' tx-meta'))]
+         (prn :debug :opfs-db-transact-time-ms (- (util/time-ms) start)
+              {:tx-data tx-data
+               :tx-meta tx-meta}))
+       nil)))
 
   (<fetch-initial-data [_this repo _opts]
     (when-let [^js sqlite @*sqlite]
-      (-> (p/let [_ (.createOrOpenDB sqlite repo)]
+      (-> (p/let [_ (.createOrOpenDB sqlite repo)
+                  _ (ipc/ipc :db-open repo)]
             (.getInitialData sqlite repo))
           (p/catch (fn [error]
                      (prn :debug :fetch-initial-data-error repo)

From d3b6d0f4ecb1414b8387c65082ce189a5ca9d0b9 Mon Sep 17 00:00:00 2001
From: Tienson Qin <tiensonqin@gmail.com>
Date: Tue, 19 Dec 2023 22:57:21 +0800
Subject: [PATCH 2/5] Remove debugging

---
 src/main/frontend/persist_db/browser.cljs | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/src/main/frontend/persist_db/browser.cljs b/src/main/frontend/persist_db/browser.cljs
index fb010e586f..7c1eae9f92 100644
--- a/src/main/frontend/persist_db/browser.cljs
+++ b/src/main/frontend/persist_db/browser.cljs
@@ -44,7 +44,7 @@
     (ipc/ipc :db-export repo data)
 
     ;; TODO: browser nfs-supported? auto backup
-    
+
     ;;
     :else
     nil))
@@ -82,14 +82,8 @@
           tx-data' (pr-str tx-data)
           tx-meta' (pr-str tx-meta)]
       (p/do!
-       (p/let [start (util/time-ms)
-               _ (ipc/ipc :db-transact repo tx-data' tx-meta')]
-         (prn :debug :disk-db-transact-time-ms (- (util/time-ms) start)))
-       (p/let [start (util/time-ms)
-               _ (when sqlite (.transact sqlite repo tx-data' tx-meta'))]
-         (prn :debug :opfs-db-transact-time-ms (- (util/time-ms) start)
-              {:tx-data tx-data
-               :tx-meta tx-meta}))
+       (ipc/ipc :db-transact repo tx-data' tx-meta')
+       (when sqlite (.transact sqlite repo tx-data' tx-meta'))
        nil)))
 
   (<fetch-initial-data [_this repo _opts]

From 666b4ab6d4fcb0a27006cf7ac34212dd137a718f Mon Sep 17 00:00:00 2001
From: Tienson Qin <tiensonqin@gmail.com>
Date: Tue, 19 Dec 2023 23:15:44 +0800
Subject: [PATCH 3/5] Move ipc calls outside db browser implementation

---
 src/main/frontend/db_worker.cljs          | 10 +++++-----
 src/main/frontend/persist_db.cljs         | 20 +++++++++++---------
 src/main/frontend/persist_db/browser.cljs |  6 ++----
 3 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/src/main/frontend/db_worker.cljs b/src/main/frontend/db_worker.cljs
index a8b6adb9bc..0f87fe92e9 100644
--- a/src/main/frontend/db_worker.cljs
+++ b/src/main/frontend/db_worker.cljs
@@ -212,11 +212,11 @@
                           (string/replace-first (.-name file) ".logseq-pool-" "")))
                       all-files)
                 distinct)]
-     (prn :debug :all-files (map #(.-name %) all-files))
-     (prn :debug :all-files-count (count (filter
-                                          #(= (.-kind %) "file")
-                                          all-files)))
-     (prn :dbs dbs)
+     ;; (prn :debug :all-files (map #(.-name %) all-files))
+     ;; (prn :debug :all-files-count (count (filter
+     ;;                                      #(= (.-kind %) "file")
+     ;;                                      all-files)))
+     ;; (prn :dbs dbs)
      (bean/->js dbs)))
 
   (createOrOpenDB
diff --git a/src/main/frontend/persist_db.cljs b/src/main/frontend/persist_db.cljs
index 0ee27e4e40..3d703d48a2 100644
--- a/src/main/frontend/persist_db.cljs
+++ b/src/main/frontend/persist_db.cljs
@@ -1,8 +1,9 @@
 (ns frontend.persist-db
-   "Backend of DB based graph"
-   (:require [frontend.persist-db.browser :as browser]
-             [frontend.persist-db.protocol :as protocol]
-             [promesa.core :as p]))
+  "Backend of DB based graph"
+  (:require [frontend.persist-db.browser :as browser]
+            [frontend.persist-db.protocol :as protocol]
+            [promesa.core :as p]
+            [electron.ipc :as ipc]))
 
 (defonce opfs-db (browser/->InBrowser))
 
@@ -32,16 +33,17 @@
   ([repo]
    (<fetch-init-data repo {}))
   ([repo opts]
-   (p/let [ret (protocol/<fetch-initial-data (get-impl) repo opts)]
-     (js/console.log "fetch-initial-data" ret)
-     ret)))
+   (p/do!
+    (ipc/ipc :db-open repo)
+    (protocol/<fetch-initial-data (get-impl) repo opts))))
 
 ;; FIXME: limit repo name's length
 ;; @shuyu Do we still need this?
 (defn <new [repo]
   {:pre [(<= (count repo) 56)]}
-  (p/let [_ (protocol/<new (get-impl) repo)]
-    (<export-db repo {})))
+  (p/let [_ (protocol/<new (get-impl) repo)
+          _ (<export-db repo {})]
+    (ipc/ipc :db-open repo)))
 
 (defn <release-access-handles
   [repo]
diff --git a/src/main/frontend/persist_db/browser.cljs b/src/main/frontend/persist_db/browser.cljs
index 7c1eae9f92..d70cf2276d 100644
--- a/src/main/frontend/persist_db/browser.cljs
+++ b/src/main/frontend/persist_db/browser.cljs
@@ -59,8 +59,7 @@
   protocol/PersistentDB
   (<new [_this repo]
     (when-let [^js sqlite @*sqlite]
-      (.createOrOpenDB sqlite repo)
-      (ipc/ipc :db-open repo)))
+      (.createOrOpenDB sqlite repo)))
 
   (<list-db [_this]
     (when-let [^js sqlite @*sqlite]
@@ -88,8 +87,7 @@
 
   (<fetch-initial-data [_this repo _opts]
     (when-let [^js sqlite @*sqlite]
-      (-> (p/let [_ (.createOrOpenDB sqlite repo)
-                  _ (ipc/ipc :db-open repo)]
+      (-> (p/let [_ (.createOrOpenDB sqlite repo)]
             (.getInitialData sqlite repo))
           (p/catch sqlite-error-handler))))
 

From 16aa7d3f123b9cf91db443a4c6cd01af39df05e9 Mon Sep 17 00:00:00 2001
From: Tienson Qin <tiensonqin@gmail.com>
Date: Wed, 20 Dec 2023 00:48:20 +0800
Subject: [PATCH 4/5] fix: can't create graphs name with spaces

Fixes LOG-2967
---
 src/main/frontend/db_worker.cljs | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/main/frontend/db_worker.cljs b/src/main/frontend/db_worker.cljs
index 0f87fe92e9..8f30e35660 100644
--- a/src/main/frontend/db_worker.cljs
+++ b/src/main/frontend/db_worker.cljs
@@ -18,6 +18,14 @@
 (defonce *datascript-conns (atom nil))
 (defonce *opfs-pools (atom nil))
 
+(defn sanitize-db-name
+  [db-name]
+  (-> db-name
+      (string/replace " " "_")
+      (string/replace "/" "_")
+      (string/replace "\\" "_")
+      (string/replace ":" "_")))
+
 (defn- get-sqlite-conn
   [repo]
   (get @*sqlite-conns repo))
@@ -33,7 +41,7 @@
 (defn- <get-opfs-pool
   [graph]
   (or (get-opfs-pool graph)
-      (p/let [^js pool (.installOpfsSAHPoolVfs @*sqlite #js {:name (str "logseq-pool-" graph)
+      (p/let [^js pool (.installOpfsSAHPoolVfs @*sqlite #js {:name (str "logseq-pool-" (sanitize-db-name graph))
                                                              :initialCapacity 10})]
         (swap! *opfs-pools assoc graph pool)
         pool)))
@@ -54,7 +62,7 @@
 
 (defn- get-repo-path
   [repo]
-  (str "/" repo ".sqlite"))
+  (str "/" (sanitize-db-name repo) ".sqlite"))
 
 (defn- <export-db-file
   [repo]

From 34a71eedaf29c282a8376ab4b35979ad0cb8c83d Mon Sep 17 00:00:00 2001
From: Gabriel Horner <gabriel@logseq.com>
Date: Tue, 19 Dec 2023 16:48:34 -0500
Subject: [PATCH 5/5] fix: only look to opfs for db graphs

For graphs with spaces, looking up electron db graphs was causing
duplicates to appear e.g. "name_space" and "name space". More
importantly, these electron db graphs don't actually exist in opfs
so they won't persist correctly. Part of LOG-2967
---
 src/electron/electron/handler.cljs | 21 +--------------------
 1 file changed, 1 insertion(+), 20 deletions(-)

diff --git a/src/electron/electron/handler.cljs b/src/electron/electron/handler.cljs
index 13cd031a22..4d86a56aa9 100644
--- a/src/electron/electron/handler.cljs
+++ b/src/electron/electron/handler.cljs
@@ -227,14 +227,6 @@
     (fs-extra/ensureDirSync dir)
     dir))
 
-(defn- get-db-based-graphs-dir
-  []
-  (let [dir (if utils/ci?
-              (.resolve node-path js/__dirname "../tmp/graphs")
-              (.join node-path (.homedir os) "logseq" "graphs"))]
-    (fs-extra/ensureDirSync dir)
-    dir))
-
 ;; TODO: move file based graphs to "~/logseq/graphs" too
 (defn- get-file-based-graphs
   "Returns all graph names in the cache directory (starting with `logseq_local_`)"
@@ -245,20 +237,9 @@
          (map #(node-path/basename % ".transit"))
          (map graph-name->path))))
 
-(defn- get-db-based-graphs
-  "Returns all graph names in the cache directory"
-  []
-  (let [dir (get-db-based-graphs-dir)]
-    (->> (common-graph/read-directories dir)
-         (remove (fn [s] (= s db/unlinked-graphs-dir)))
-         (map graph-name->path)
-         (map (fn [s] (str sqlite-util/db-version-prefix s))))))
-
 (defn- get-graphs
   []
-  (concat
-   (get-file-based-graphs)
-   (get-db-based-graphs)))
+  (get-file-based-graphs))
 
 ;; TODO support alias mechanism
 (defn get-graph-name