feat(dev): export client-ops sqlite from db worker

src/main/frontend/handler/common/developer.cljs

@@ -95,6 +95,31 @@
       (string/replace #"[\\/]+" "_")
       (str "_checksum_" (quot (util/time-ms) 1000))))
 
+(defn- client-ops-export-file-name
+  [repo]
+  (-> (or repo "graph")
+      (string/replace #"^/+" "")
+      (string/replace #"[\\/]+" "_")
+      (str "_client_ops_" (quot (util/time-ms) 1000))))
+
+(defn- ->uint8array
+  [data]
+  (cond
+    (instance? js/Uint8Array data)
+    data
+
+    (js/ArrayBuffer.isView data)
+    (js/Uint8Array. (.-buffer data) (.-byteOffset data) (.-byteLength data))
+
+    (instance? js/ArrayBuffer data)
+    (js/Uint8Array. data)
+
+    (array? data)
+    (js/Uint8Array. data)
+
+    :else
+    nil))
+
 (defn- <fetch-server-checksum-diagnostics
   [repo]
   (let [base (rtc-handler/http-base)
@@ -199,6 +224,29 @@
                    (notification/show! "Failed to compute graph checksum diagnostics." :error))))
     (notification/show! "No graph found" :warning)))
 
+(defn ^:export export-client-ops-sqlite
+  []
+  (if-let [repo (state/get-current-repo)]
+    (-> (state/<invoke-db-worker-direct-pass :thread-api/export-client-ops-db repo)
+        (p/then (fn [data]
+                  (if-let [payload (->uint8array data)]
+                    (let [filename (client-ops-export-file-name repo)
+                          blob (js/Blob. #js [payload] (clj->js {:type "application/octet-stream"}))]
+                      (utils/saveToFile blob filename "sqlite")
+                      (notification/show!
+                       (str "Client ops SQLite exported: " filename ".sqlite")
+                       :success
+                       false))
+                    (notification/show!
+                     (str "Client ops SQLite export failed: invalid payload type "
+                          (pr-str (type data))
+                          ".")
+                     :warning))))
+        (p/catch (fn [error]
+                   (js/console.error "export-client-ops-sqlite failed:" error)
+                   (notification/show! "Failed to export client ops SQLite." :error))))
+    (notification/show! "No graph found" :warning)))
+
 (defn import-chosen-graph
   [repo]
   (p/let [_ (persist-db/<unsafe-delete repo)]

src/main/frontend/modules/shortcut/config.cljs

@@ -511,6 +511,9 @@
    :dev/recompute-checksum {:binding []
                             :inactive (not (state/developer-mode?))
                             :fn :frontend.handler.common.developer/recompute-checksum-diagnostics}
+   :dev/export-client-ops-sqlite {:binding []
+                                  :inactive (not (state/developer-mode?))
+                                  :fn :frontend.handler.common.developer/export-client-ops-sqlite}
    :dev/rtc-stop {:binding []
                   :inactive (not (state/developer-mode?))
                   :fn :frontend.handler.common.developer/rtc-stop}
@@ -714,6 +717,7 @@
           :dev/replace-graph-with-db-file
           :dev/validate-db
           :dev/recompute-checksum
+          :dev/export-client-ops-sqlite
           :dev/gc-graph
           :dev/rtc-stop
           :dev/rtc-start
@@ -878,6 +882,7 @@
      :dev/replace-graph-with-db-file
      :dev/validate-db
      :dev/recompute-checksum
+     :dev/export-client-ops-sqlite
      :dev/gc-graph
      :dev/rtc-stop
      :dev/rtc-start

src/main/frontend/worker/db_worker.cljs

@@ -114,6 +114,44 @@
       nil)))
 
 (def repo-path "/db.sqlite")
+(def client-ops-repo-path (str "client-ops-" repo-path))
+
+(defn- ->uint8array
+  [data]
+  (cond
+    (instance? js/Uint8Array data)
+    data
+
+    (js/ArrayBuffer.isView data)
+    (js/Uint8Array. (.-buffer data) (.-byteOffset data) (.-byteLength data))
+
+    (instance? js/ArrayBuffer data)
+    (js/Uint8Array. data)
+
+    (array? data)
+    (js/Uint8Array. data)
+
+    :else
+    data))
+
+(defn- export-db-file-with-paths
+  [repo path-candidates]
+  (p/let [^js pool (<get-opfs-pool repo)]
+    (when pool
+      (loop [paths (->> path-candidates
+                        (filter string?)
+                        (remove string/blank?)
+                        distinct
+                        vec)]
+        (when-let [path (first paths)]
+          (let [result (try
+                         (.exportFile ^js pool path)
+                         (catch :default _e
+                           nil))
+                payload (->uint8array result)]
+            (if (instance? js/Uint8Array payload)
+              payload
+              (recur (subvec paths 1)))))))))
 
 (defn- <export-db-file
   ([repo]
@@ -251,7 +289,7 @@
                 (.unpauseVfs pool))
             db (new (.-OpfsSAHPoolDb pool) repo-path)
             search-db (new (.-OpfsSAHPoolDb pool) (str "search" repo-path))
-            client-ops-db (new (.-OpfsSAHPoolDb pool) (str "client-ops-" repo-path))]
+            client-ops-db (new (.-OpfsSAHPoolDb pool) client-ops-repo-path)]
       [db search-db client-ops-db])))
 
 (defn- gc-sqlite-dbs!
@@ -765,8 +803,24 @@
   [repo]
   (when-let [^js db (worker-state/get-sqlite-conn repo :db)]
     (.exec db "PRAGMA wal_checkpoint(2)"))
-  (p/let [data (<export-db-file repo)]
-    (Comlink/transfer data #js [(.-buffer data)])))
+  (p/let [data (<export-db-file repo)
+          payload (->uint8array data)]
+    (Comlink/transfer payload #js [(.-buffer payload)])))
+
+(def-thread-api :thread-api/export-client-ops-db
+  [repo]
+  (when-let [^js db (worker-state/get-sqlite-conn repo :client-ops)]
+    (.exec db "PRAGMA wal_checkpoint(2)"))
+  (let [^js client-ops-db (worker-state/get-sqlite-conn repo :client-ops)
+        db-filename (some-> client-ops-db (gobj/get "filename"))
+        export-paths [db-filename
+                      client-ops-repo-path
+                      (str "/" client-ops-repo-path)
+                      (str "client-ops" repo-path)
+                      (str "/client-ops" repo-path)]]
+    (p/let [payload (export-db-file-with-paths repo export-paths)]
+      (when (instance? js/Uint8Array payload)
+        (Comlink/transfer payload #js [(.-buffer payload)])))))
 
 (def-thread-api :thread-api/import-db
   [repo data]

src/resources/dicts/en.edn

@@ -581,6 +581,7 @@
   :dev/replace-graph-with-db-file "(Dev) Replace graph with its db.sqlite file"
   :dev/validate-db "(Dev) Validate current graph"
   :dev/recompute-checksum "(Dev) Recompute graph checksum"
+  :dev/export-client-ops-sqlite "(Dev) Export client ops sqlite"
   :dev/gc-graph "(Dev) Garbage collect graph (remove unused data in SQLite)"
   :dev/rtc-stop "(Dev) RTC Stop"
   :dev/rtc-start "(Dev) RTC Start"

src/test/frontend/worker/db_worker_test.cljs

@@ -352,3 +352,31 @@
          (finally
            (reset! db-sync/*repo->latest-remote-tx latest-tx-prev)
            (reset! db-sync/*repo->latest-remote-checksum latest-checksum-prev)))))))
+
+(deftest thread-api-export-client-ops-db-checkpoints-and-exports-client-ops-file-test
+  (async done
+    (restoring-worker-state
+     (fn []
+       (let [export-client-ops-db (@thread-api/*thread-apis :thread-api/export-client-ops-db)
+             sql-calls (atom [])
+             export-calls (atom [])
+             expected-data (js/Uint8Array. #js [1 2 3])
+             expected-buffer (.-buffer expected-data)
+             fake-pool #js {:exportFile (fn [path]
+                                          (swap! export-calls conj path)
+                                          expected-buffer)}]
+         (reset! worker-state/*opfs-pools {test-repo fake-pool})
+         (with-redefs [worker-state/get-sqlite-conn (fn [_repo which-db]
+                                                      (when (= :client-ops which-db)
+                                                        #js {:exec (fn [sql]
+                                                                     (swap! sql-calls conj sql))}))]
+           (-> (export-client-ops-db test-repo)
+               (p/then (fn [result]
+                         (is (= ["PRAGMA wal_checkpoint(2)"] @sql-calls))
+                         (is (= ["client-ops-/db.sqlite"] @export-calls))
+                         (is (instance? js/Uint8Array result))
+                         (is (= [1 2 3] (vec result)))
+                         (done)))
+               (p/catch (fn [error]
+                          (is false (str error))
+                          (done))))))))))