From b54a73f298c0f0e378e46eac1dea91f5b6283783 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Tue, 24 Mar 2026 14:22:07 +0800 Subject: [PATCH] enhance(rtc): cleanup finished client ops every 3 hours --- src/main/frontend/worker/db_worker.cljs | 23 +++++++++++ src/main/frontend/worker/sync/client_op.cljs | 21 ++++++++++ src/main/frontend/worker/undo_redo.cljs | 12 ++++++ src/test/frontend/worker/db_worker_test.cljs | 40 +++++++++++++++++++ .../frontend/worker/sync/client_op_test.cljs | 39 +++++++++++++++++- 5 files changed, 134 insertions(+), 1 deletion(-) diff --git a/src/main/frontend/worker/db_worker.cljs b/src/main/frontend/worker/db_worker.cljs index 64d1cd6606..38e7384c60 100644 --- a/src/main/frontend/worker/db_worker.cljs +++ b/src/main/frontend/worker/db_worker.cljs @@ -84,6 +84,8 @@ (defonce *opfs-pools worker-state/*opfs-pools) (defonce *publishing? (atom false)) (defonce ^:private *search-index-build-ids (atom {})) +(defonce ^:private *client-ops-cleanup-timers (atom {})) +(def ^:private client-ops-cleanup-interval-ms (* 3 60 60 1000)) (defn- check-worker-scope! [] @@ -179,6 +181,9 @@ (defn- close-db-aux! [repo ^Object db ^Object search ^Object client-ops] + (when-let [timer (get @*client-ops-cleanup-timers repo)] + (js/clearInterval timer)) + (swap! *client-ops-cleanup-timers dissoc repo) (swap! *sqlite-conns dissoc repo) (swap! *datascript-conns dissoc repo) (swap! *client-ops-conns dissoc repo) @@ -258,6 +263,23 @@ :kv/value (common-util/time-ms)}] {:skip-validate-db? true})))) +(defn- run-client-ops-cleanup! + [repo] + (let [protected-tx-ids (worker-undo-redo/referenced-history-tx-ids repo)] + (client-op/cleanup-finished-history-ops! repo protected-tx-ids) + nil)) + +(defn- ensure-client-ops-cleanup-timer! + [repo] + (when (and (not @*publishing?) + repo + (nil? (get @*client-ops-cleanup-timers repo))) + (let [timer (js/setInterval (fn [] + (run-client-ops-cleanup! repo)) + client-ops-cleanup-interval-ms)] + (swap! *client-ops-cleanup-timers assoc repo timer)) + nil)) + (def ^:private recycle-gc-kv :logseq.kv/recycle-last-gc-at) (defn- maybe-run-recycle-gc! @@ -320,6 +342,7 @@ (swap! *client-ops-conns assoc repo client-ops-conn) (when (and (not @*publishing?) (not= client-op/schema-in-db (d/schema @client-ops-conn))) (d/reset-schema! client-ops-conn client-op/schema-in-db)) + (ensure-client-ops-cleanup-timer! repo) (let [initial-tx-report (when-not (or initial-data-exists? (seq datoms) sync-download-graph?) diff --git a/src/main/frontend/worker/sync/client_op.cljs b/src/main/frontend/worker/sync/client_op.cljs index 46dfd4a2e9..645646d103 100644 --- a/src/main/frontend/worker/sync/client_op.cljs +++ b/src/main/frontend/worker/sync/client_op.cljs @@ -205,3 +205,24 @@ (let [ent (d/entity @conn [:block/uuid asset-uuid])] (when-let [e (:db/id ent)] (ldb/transact! conn [[:db/retractEntity e]]))))) + +(defn cleanup-finished-history-ops! + [repo protected-tx-ids] + (if-let [conn (worker-state/get-client-ops-conn repo)] + (let [protected-tx-ids (set protected-tx-ids) + tx-ent-ids (->> (d/datoms @conn :avet :db-sync/tx-id) + (keep (fn [datom] + (let [tx-id (:v datom) + ent (d/entity @conn (:e datom))] + (when (and (uuid? tx-id) + (false? (:db-sync/pending? ent)) + (not (contains? protected-tx-ids tx-id))) + (:db/id ent))))) + vec)] + (when (seq tx-ent-ids) + (ldb/transact! conn + (mapv (fn [ent-id] + [:db/retractEntity ent-id]) + tx-ent-ids))) + (count tx-ent-ids)) + 0)) diff --git a/src/main/frontend/worker/undo_redo.cljs b/src/main/frontend/worker/undo_redo.cljs index bd57b59c09..a56d0ba84e 100644 --- a/src/main/frontend/worker/undo_redo.cljs +++ b/src/main/frontend/worker/undo_redo.cljs @@ -446,3 +446,15 @@ {:undo-ops (get @*undo-ops repo []) :redo-ops (get @*redo-ops repo []) :pending-editor-info (get @*pending-editor-info repo)}) + +(defn referenced-history-tx-ids + [repo] + (->> (concat (get @*undo-ops repo []) + (get @*redo-ops repo [])) + (mapcat identity) + (keep (fn [item] + (when (= ::db-transact (first item)) + (let [tx-id (:db-sync/tx-id (second item))] + (when (uuid? tx-id) + tx-id))))) + set)) diff --git a/src/test/frontend/worker/db_worker_test.cljs b/src/test/frontend/worker/db_worker_test.cljs index 1c05cd1253..eef435db54 100644 --- a/src/test/frontend/worker/db_worker_test.cljs +++ b/src/test/frontend/worker/db_worker_test.cljs @@ -81,6 +81,46 @@ (is (nil? (get @search/fuzzy-search-indices test-repo))) (is (nil? (get @worker-state/*sqlite-conns test-repo))))))) +(deftest client-ops-cleanup-timer-starts-once-and-clears-on-close-test + (restoring-worker-state + (fn [] + (let [scheduled (atom []) + cleared (atom []) + original-set-interval js/setInterval + original-clear-interval js/clearInterval + fake-db #js {:close (fn [] nil)} + timer-id #js {:id "timer-1"}] + (set! js/setInterval + (fn [f interval-ms] + (swap! scheduled conj {:fn f :interval-ms interval-ms}) + timer-id)) + (set! js/clearInterval + (fn [id] + (swap! cleared conj id))) + (try + (reset! worker-state/*sqlite-conns + {test-repo {:db fake-db + :search fake-db + :client-ops fake-db}}) + (reset! worker-state/*datascript-conns {test-repo :datascript}) + (reset! worker-state/*client-ops-conns {test-repo :client-ops}) + (reset! (deref #'db-worker/*client-ops-cleanup-timers) {}) + + (#'db-worker/ensure-client-ops-cleanup-timer! test-repo) + (#'db-worker/ensure-client-ops-cleanup-timer! test-repo) + + (is (= 1 (count @scheduled))) + (is (= (* 3 60 60 1000) (:interval-ms (first @scheduled)))) + (is (= timer-id (get @(deref #'db-worker/*client-ops-cleanup-timers) test-repo))) + + (db-worker/close-db! test-repo) + + (is (= [timer-id] @cleared)) + (is (nil? (get @(deref #'db-worker/*client-ops-cleanup-timers) test-repo))) + (finally + (set! js/setInterval original-set-interval) + (set! js/clearInterval original-clear-interval))))))) + (deftest complete-datoms-import-invalidates-existing-search-db-test (async done (restoring-worker-state diff --git a/src/test/frontend/worker/sync/client_op_test.cljs b/src/test/frontend/worker/sync/client_op_test.cljs index 3b882d159b..e63b7d3022 100644 --- a/src/test/frontend/worker/sync/client_op_test.cljs +++ b/src/test/frontend/worker/sync/client_op_test.cljs @@ -1,5 +1,5 @@ (ns frontend.worker.sync.client-op-test - (:require [cljs.test :refer [deftest is]] + (:require [cljs.test :refer [deftest is testing]] [datascript.core :as d] [frontend.worker.state :as worker-state] [frontend.worker.sync.client-op :as client-op])) @@ -17,3 +17,40 @@ (is (= #{"graph-2"} (set (map :v graph-uuid-datoms))))) (finally (reset! worker-state/*client-ops-conns prev-client-ops-conns))))) + +(deftest cleanup-finished-history-ops-removes-only-unreferenced-finished-txs-test + (let [repo "repo-cleanup" + conn (d/create-conn client-op/schema-in-db) + prev-client-ops-conns @worker-state/*client-ops-conns + keep-tx-id (random-uuid) + remove-tx-id (random-uuid) + pending-tx-id (random-uuid)] + (reset! worker-state/*client-ops-conns {repo conn}) + (try + (d/transact! conn + [{:db-sync/tx-id keep-tx-id + :db-sync/pending? false} + {:db-sync/tx-id remove-tx-id + :db-sync/pending? false} + {:db-sync/tx-id pending-tx-id + :db-sync/pending? true} + {:db-ident :metadata/local + :local-tx 99}]) + + (is (= 1 (client-op/cleanup-finished-history-ops! repo #{keep-tx-id}))) + (is (some? (d/entity @conn [:db-sync/tx-id keep-tx-id]))) + (is (nil? (d/entity @conn [:db-sync/tx-id remove-tx-id]))) + (is (some? (d/entity @conn [:db-sync/tx-id pending-tx-id]))) + (is (= 99 (:local-tx (d/entity @conn [:db-ident :metadata/local])))) + (finally + (reset! worker-state/*client-ops-conns prev-client-ops-conns))))) + +(deftest cleanup-finished-history-ops-no-conn-is-noop-test + (let [repo "repo-no-conn" + prev-client-ops-conns @worker-state/*client-ops-conns] + (reset! worker-state/*client-ops-conns {}) + (try + (testing "cleanup should be safe when client-ops conn is missing" + (is (= 0 (client-op/cleanup-finished-history-ops! repo #{})))) + (finally + (reset! worker-state/*client-ops-conns prev-client-ops-conns)))))