From 0a3600628812d50223359bab1bdd26d84eebf895 Mon Sep 17 00:00:00 2001 From: rcmerci Date: Fri, 11 Jul 2025 10:29:59 +0800 Subject: [PATCH] feat(rtc,wip): migration support in rtc --- src/main/frontend/worker/db/migrate.cljs | 31 +++++++++++++------ src/main/frontend/worker/db_worker.cljs | 2 +- src/main/frontend/worker/rtc/client_op.cljs | 8 +++++ src/main/frontend/worker/rtc/migrate.cljs | 21 +++++++++++++ .../frontend/worker/rtc/migrate_test.cljs | 14 +++++++++ 5 files changed, 65 insertions(+), 11 deletions(-) create mode 100644 src/test/frontend/worker/rtc/migrate_test.cljs diff --git a/src/main/frontend/worker/db/migrate.cljs b/src/main/frontend/worker/db/migrate.cljs index 14bbacf938..d55e5901a7 100644 --- a/src/main/frontend/worker/db/migrate.cljs +++ b/src/main/frontend/worker/db/migrate.cljs @@ -4,6 +4,7 @@ [clojure.walk :as walk] [datascript.core :as d] [datascript.impl.entity :as de] + [frontend.worker.rtc.client-op :as client-op] [frontend.worker.util :as worker-util] [logseq.common.config :as common-config] [logseq.common.util :as common-util] @@ -378,6 +379,7 @@ (js/console.warn (str "Current db schema-version is " db-schema/version ", max available schema-version is " max-schema-version)))))) (defn ensure-built-in-data-exists! + "Return tx-data" [conn] (let [*uuids (atom {}) data (->> (sqlite-create-graph/build-db-initial-data "") @@ -430,11 +432,13 @@ [:block/uuid (@*uuids (second f))] :else f)) - data)] - (d/transact! conn data' {:fix-db? true - :db-migrate? true}))) + data) + r (d/transact! conn data' {:fix-db? true + :db-migrate? true})] + (:tx-data r))) (defn- upgrade-version! + "Return tx-data" [conn db-based? version {:keys [properties classes fix]}] (let [version (db-schema/parse-schema-version version) db @conn @@ -462,14 +466,15 @@ tx-data (if db-based? (concat new-class-idents new-properties new-classes fixes) fixes) tx-data' (concat [(sqlite-util/kv :logseq.kv/schema-version version)] - tx-data)] - (ldb/transact! conn tx-data' {:db-migrate? true}) - (println "DB schema migrated to" version))) + tx-data) + r (ldb/transact! conn tx-data' {:db-migrate? true})] + (println "DB schema migrated to" version) + (:tx-data r))) (defn migrate "Migrate 'frontend' datascript schema and data. To add a new migration, add an entry to schema-version->updates and bump db-schema/version" - [conn] + [repo conn] (when (ldb/db-based-graph? @conn) (let [db @conn version-in-db (db-schema/parse-schema-version (or (:kv/value (d/entity db :logseq.kv/schema-version)) 0)) @@ -489,11 +494,17 @@ (when (and (neg? (db-schema/compare-schema-version version-in-db v*)) (not (pos? (db-schema/compare-schema-version v* db-schema/version)))) [v updates]))) - schema-version->updates)] + schema-version->updates) + *tx-data-coll (atom [])] (println "DB schema migrated from" version-in-db) (doseq [[v m] updates] - (upgrade-version! conn db-based? v m)) - (ensure-built-in-data-exists! conn)) + (let [tx-data (upgrade-version! conn db-based? v m)] + (swap! *tx-data-coll conj tx-data))) + (client-op/add-migration-datoms! repo version-in-db db-schema/version @*tx-data-coll) + (swap! *tx-data-coll conj (ensure-built-in-data-exists! conn)) + {:from-version version-in-db + :to-version db-schema/version + :tx-data-coll @*tx-data-coll}) (catch :default e (prn :error (str "DB migration failed to migrate to " db-schema/version " from " version-in-db ":")) (js/console.error e) diff --git a/src/main/frontend/worker/db_worker.cljs b/src/main/frontend/worker/db_worker.cljs index 6afe00efd6..aa2c4c8a19 100644 --- a/src/main/frontend/worker/db_worker.cljs +++ b/src/main/frontend/worker/db_worker.cljs @@ -297,7 +297,7 @@ (gc-sqlite-dbs! db client-ops-db conn {}) - (db-migrate/migrate conn) + (db-migrate/migrate repo conn) (db-listener/listen-db-changes! repo (get @*datascript-conns repo)))))) diff --git a/src/main/frontend/worker/rtc/client_op.cljs b/src/main/frontend/worker/rtc/client_op.cljs index 9a71b36fd5..4d20f6c530 100644 --- a/src/main/frontend/worker/rtc/client_op.cljs +++ b/src/main/frontend/worker/rtc/client_op.cljs @@ -82,6 +82,7 @@ :local-tx {:db/index true} :graph-uuid {:db/index true} :aes-key-jwk {:db/index true} + :migration-datoms {:db/index true} ;; device :device/uuid {:db/unique :db.unique/identity} @@ -469,3 +470,10 @@ (m/ap (let [_ (m/?> (c.m/throttle 100 db-updated-flow))] (datom-count-fn @conn)))))) + +(defn add-migration-datoms! + [repo from-version to-version datoms] + (when-let [conn (worker-state/get-client-ops-conn repo)] + (d/transact! conn [{:migration-datoms {:datoms datoms + :from from-version + :to to-version}}]))) diff --git a/src/main/frontend/worker/rtc/migrate.cljs b/src/main/frontend/worker/rtc/migrate.cljs index ee1380d650..9471c693b5 100644 --- a/src/main/frontend/worker/rtc/migrate.cljs +++ b/src/main/frontend/worker/rtc/migrate.cljs @@ -47,3 +47,24 @@ (migration-updates->client-ops db client-schema-version)))] (client-op/add-ops! repo ops) ops)) + +(defn local-datoms-tx-data=>remote-tx-data + [db datoms-tx-data] + (let [e->datoms (group-by :e datoms-tx-data) + e->datomvec-coll + (update-vals + e->datoms + (fn [datoms] + (let [e (:e (first datoms)) + need-block-uuid-datom? + (every? + (fn [{:keys [a added]}] + (or (not= :block/uuid a) + (and (= :block/uuid a) (false? added)))) + datoms) + block-uuid (when need-block-uuid-datom? (:block/uuid (d/entity db e))) + datoms* (cond->> datoms + (and need-block-uuid-datom? block-uuid) + (cons (d/datom e :block/uuid block-uuid)))] + (map (fn [{:keys [e a v tx added]}] [e a v tx added]) datoms*))))] + e->datomvec-coll)) diff --git a/src/test/frontend/worker/rtc/migrate_test.cljs b/src/test/frontend/worker/rtc/migrate_test.cljs new file mode 100644 index 0000000000..82aa76d7e5 --- /dev/null +++ b/src/test/frontend/worker/rtc/migrate_test.cljs @@ -0,0 +1,14 @@ +(ns frontend.worker.rtc.migrate-test + (:require ["fs" :as fs-node] + [cljs.test :refer [deftest is testing]] + [logseq.db :as ldb] + [frontend.worker.db.migrate :as db-migrate] + [datascript.core :as d] + [cljs.pprint :as pp])) + +(deftest ^:focus local-datoms-tx-data=>remote-tx-data-test + (let [db-transit (str (fs-node/readFileSync "src/test/migration/64.8.transit")) + db (ldb/read-transit-str db-transit) + conn (d/conn-from-db db) + tx-data-coll (db-migrate/migrate "rtc-migrate-test" conn)] + (pp/pprint tx-data-coll)))