From 5a417c25ad995007287c0a1662b8f66d3d83179b Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Tue, 3 Mar 2026 16:57:27 +0800 Subject: [PATCH 1/3] fix: tests --- src/main/frontend/worker/sync.cljs | 27 +++++++++++----------- src/test/frontend/worker/db_sync_test.cljs | 10 ++++++++ 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/src/main/frontend/worker/sync.cljs b/src/main/frontend/worker/sync.cljs index 47e844e322..9488676be1 100644 --- a/src/main/frontend/worker/sync.cljs +++ b/src/main/frontend/worker/sync.cljs @@ -358,6 +358,18 @@ (remove (fn [[_op e]] (contains? rtc-const/ignore-entities-when-init-upload e))))) +(def ^:private non-retractable-block-attrs + #{:block/created-at :block/updated-at :block/title}) + +(defn- drop-non-retractable-attr-datoms + [tx-data] + (remove (fn [item] + (and (vector? item) + (>= (count item) 3) + (= :db/retract (first item)) + (contains? non-retractable-block-attrs (nth item 2)))) + tx-data)) + (defn- reverse-tx-data [tx-data] (->> tx-data @@ -663,18 +675,6 @@ (sequential? (first tx-data*)) (sequential? (first (first tx-data*))))) -(def ^:private non-retractable-block-attrs - #{:block/created-at :block/updated-at :block/title}) - -(defn- drop-non-retractable-attr-datoms - [tx-data] - (remove (fn [item] - (and (vector? item) - (>= (count item) 3) - (= :db/retract (first item)) - (contains? non-retractable-block-attrs (nth item 2)))) - tx-data)) - (defn- sanitize-tx-data [db tx-data local-deleted-ids] (let [sanitized-tx-data (->> tx-data @@ -1122,7 +1122,8 @@ [local-txs] (let [tx-data (->> local-txs reverse - (mapcat :reversed-tx)) + (mapcat :reversed-tx) + drop-non-retractable-attr-datoms) retract-block-ids (->> (keep (fn [[op e a _v _t]] (when (and (= op :db/retract) (= :block/uuid a)) e)) tx-data) diff --git a/src/test/frontend/worker/db_sync_test.cljs b/src/test/frontend/worker/db_sync_test.cljs index 54c7a4001f..acdac4a681 100644 --- a/src/test/frontend/worker/db_sync_test.cljs +++ b/src/test/frontend/worker/db_sync_test.cljs @@ -112,6 +112,16 @@ (is nil (str error)) (done)))))))) +(deftest get-reverse-tx-data-skips-non-retractable-block-attrs-test + (testing "pending reversed txs should not retract required block attrs" + (let [local-txs [{:reversed-tx [[:db/retract 1 :block/updated-at 100 10] + [:db/retract 1 :block/created-at 90 10] + [:db/retract 1 :block/title "Home" 10] + [:db/retract 1 :block/name "home" 10]]}] + reversed (#'db-sync/get-reverse-tx-data local-txs)] + (is (= [[:db/retract 1 :block/name "home" 10]] + reversed))))) + (deftest update-online-users-dedupes-identical-messages-test (let [client {:repo test-repo :online-users (atom []) From f89b72c5ff10c340b069bb0663bba5db869ee648 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Tue, 3 Mar 2026 17:46:08 +0800 Subject: [PATCH 2/3] add app startup measure logs --- src/main/frontend/worker/sync.cljs | 171 ++++++++++++++++++++++++++++- 1 file changed, 166 insertions(+), 5 deletions(-) diff --git a/src/main/frontend/worker/sync.cljs b/src/main/frontend/worker/sync.cljs index 9488676be1..57710fe4e3 100644 --- a/src/main/frontend/worker/sync.cljs +++ b/src/main/frontend/worker/sync.cljs @@ -93,6 +93,144 @@ (reset! *ws-state ws-state) (broadcast-rtc-state! client))) +(defn- timing-platform-fields + [] + {:mobile? (:mobile? (worker-state/get-context))}) + +(defn- mark-ws-open! + [client] + (when-let [*timing (:timing client)] + (let [now (common-util/time-ms) + ws-connect-start-ms (:ws-connect-start-ms @*timing) + ws-connect-duration-ms (when (number? ws-connect-start-ms) + (- now ws-connect-start-ms))] + (swap! *timing assoc + :ws-open-ms now + :first-remote-apply-start-ms nil + :first-remote-apply-end-ms nil) + (log/info :db-sync/ws-open + (merge (timing-platform-fields) + {:repo (:repo client) + :graph-id (:graph-id client) + :ws-connect-start-ms ws-connect-start-ms + :ws-open-ms now + :ws-connect-duration-ms ws-connect-duration-ms}))))) + +(defn- mark-ws-connect-start! + [client] + (when-let [*timing (:timing client)] + (swap! *timing assoc + :ws-connect-start-ms (common-util/time-ms) + :ws-open-ms nil + :hello-received-ms nil + :first-pull-sent-ms nil + :first-pull-ok-received-ms nil + :first-aes-key-start-ms nil + :first-aes-key-ready-ms nil + :first-remote-apply-start-ms nil + :first-remote-apply-end-ms nil))) + +(defn- mark-hello-received! + [client local-tx remote-tx] + (when-let [*timing (:timing client)] + (let [now (common-util/time-ms) + {:keys [ws-open-ms hello-received-ms]} @*timing] + (when (nil? hello-received-ms) + (swap! *timing assoc :hello-received-ms now) + (log/info :db-sync/hello-received + (merge (timing-platform-fields) + {:repo (:repo client) + :graph-id (:graph-id client) + :local-tx local-tx + :remote-tx remote-tx + :ws-open-ms ws-open-ms + :hello-received-ms now + :ws-open->hello-received-ms (when (number? ws-open-ms) + (- now ws-open-ms))})))))) + +(defn- mark-first-pull-sent! + [client reason since] + (when-let [*timing (:timing client)] + (let [now (common-util/time-ms) + {:keys [ws-open-ms hello-received-ms first-pull-sent-ms]} @*timing] + (when (nil? first-pull-sent-ms) + (swap! *timing assoc :first-pull-sent-ms now) + (log/info :db-sync/first-pull-sent + (merge (timing-platform-fields) + {:repo (:repo client) + :graph-id (:graph-id client) + :reason reason + :since since + :ws-open-ms ws-open-ms + :hello-received-ms hello-received-ms + :first-pull-sent-ms now + :ws-open->pull-sent-ms (when (number? ws-open-ms) + (- now ws-open-ms)) + :hello->pull-sent-ms (when (number? hello-received-ms) + (- now hello-received-ms))})))))) + +(defn- mark-first-pull-ok-received! + [client] + (when-let [*timing (:timing client)] + (let [now (common-util/time-ms) + {:keys [ws-open-ms hello-received-ms first-pull-sent-ms first-pull-ok-received-ms]} @*timing] + (when (nil? first-pull-ok-received-ms) + (swap! *timing assoc :first-pull-ok-received-ms now) + (log/info :db-sync/first-pull-ok-received + (merge (timing-platform-fields) + {:repo (:repo client) + :graph-id (:graph-id client) + :ws-open-ms ws-open-ms + :hello-received-ms hello-received-ms + :first-pull-sent-ms first-pull-sent-ms + :first-pull-ok-received-ms now + :ws-open->pull-ok-ms (when (number? ws-open-ms) + (- now ws-open-ms)) + :hello->pull-ok-ms (when (number? hello-received-ms) + (- now hello-received-ms)) + :pull-sent->pull-ok-ms (when (number? first-pull-sent-ms) + (- now first-pull-sent-ms))})))))) + +(defn- mark-first-aes-key-ready! + [client aes-key-start-ms aes-key-ready-ms graph-e2ee?] + (when-let [*timing (:timing client)] + (let [{:keys [first-aes-key-ready-ms first-pull-ok-received-ms ws-open-ms]} @*timing] + (when (nil? first-aes-key-ready-ms) + (swap! *timing assoc + :first-aes-key-start-ms aes-key-start-ms + :first-aes-key-ready-ms aes-key-ready-ms) + (log/info :db-sync/first-aes-key-ready + (merge (timing-platform-fields) + {:repo (:repo client) + :graph-id (:graph-id client) + :graph-e2ee? graph-e2ee? + :ws-open-ms ws-open-ms + :first-pull-ok-received-ms first-pull-ok-received-ms + :first-aes-key-start-ms aes-key-start-ms + :first-aes-key-ready-ms aes-key-ready-ms + :pull-ok->aes-key-ready-ms (when (number? first-pull-ok-received-ms) + (- aes-key-ready-ms first-pull-ok-received-ms)) + :aes-key-duration-ms (- aes-key-ready-ms aes-key-start-ms)})))))) + +(defn- mark-first-remote-apply! + [client apply-start-ms] + (when-let [*timing (:timing client)] + (let [{:keys [ws-open-ms first-remote-apply-end-ms]} @*timing] + (when (and (number? ws-open-ms) (nil? first-remote-apply-end-ms)) + (let [apply-end-ms (common-util/time-ms)] + (swap! *timing assoc + :first-remote-apply-start-ms apply-start-ms + :first-remote-apply-end-ms apply-end-ms) + (log/info :db-sync/first-remote-apply + (merge (timing-platform-fields) + {:repo (:repo client) + :graph-id (:graph-id client) + :ws-open-ms ws-open-ms + :apply-start-ms apply-start-ms + :apply-end-ms apply-end-ms + :ws-open->first-apply-ms (- apply-end-ms ws-open-ms) + :first-apply-duration-ms (- apply-end-ms apply-start-ms)}))))))) + (defn- update-online-users! [client users] (when-let [*online-users (:online-users client)] @@ -745,7 +883,16 @@ :stale-kill-timer (atom nil) :last-ws-message-ts (atom (common-util/time-ms)) :online-users (atom []) - :ws-state (atom :closed)}] + :ws-state (atom :closed) + :timing (atom {:ws-connect-start-ms nil + :ws-open-ms nil + :hello-received-ms nil + :first-pull-sent-ms nil + :first-pull-ok-received-ms nil + :first-aes-key-start-ms nil + :first-aes-key-ready-ms nil + :first-remote-apply-start-ms nil + :first-remote-apply-end-ms nil})}] (reset! worker-state/*db-sync-client client) client)) @@ -1296,8 +1443,10 @@ (case (:type message) "hello" (do (require-non-negative remote-tx {:repo repo :type "hello"}) + (mark-hello-received! client local-tx remote-tx) (broadcast-rtc-state! client) (when (> remote-tx local-tx) + (mark-first-pull-sent! client :hello local-tx) (send! (:ws client) {:type "pull" :since local-tx})) (enqueue-asset-sync! repo client) (flush-pending! repo client)) @@ -1326,16 +1475,23 @@ txs-data (mapv (fn [data] (parse-transit (:tx data) {:repo repo :type "pull/ok"})) txs)] + (mark-first-pull-ok-received! client) (when (seq txs-data) - (p/let [aes-key (sync-crypt/ Date: Tue, 3 Mar 2026 19:31:39 +0800 Subject: [PATCH 3/3] enhance: cache user keys to speedup first sync --- src/main/frontend/worker/sync.cljs | 2 +- src/main/frontend/worker/sync/crypt.cljs | 117 +++++++++++++++++++---- 2 files changed, 100 insertions(+), 19 deletions(-) diff --git a/src/main/frontend/worker/sync.cljs b/src/main/frontend/worker/sync.cljs index 57710fe4e3..383e2d03aa 100644 --- a/src/main/frontend/worker/sync.cljs +++ b/src/main/frontend/worker/sync.cljs @@ -95,7 +95,7 @@ (defn- timing-platform-fields [] - {:mobile? (:mobile? (worker-state/get-context))}) + {:mobile? (boolean (:mobile? (worker-state/get-context)))}) (defn- mark-ws-open! [client] diff --git a/src/main/frontend/worker/sync/crypt.cljs b/src/main/frontend/worker/sync/crypt.cljs index fc5b93cd6e..670cc8a07c 100644 --- a/src/main/frontend/worker/sync/crypt.cljs +++ b/src/main/frontend/worker/sync/crypt.cljs @@ -14,6 +14,7 @@ [promesa.core :as p])) (defonce ^:private *graph->aes-key (atom {})) +(defonce ^:private *user-rsa-key-pair-inflight (atom {})) (defonce ^:private e2ee-store (delay (idb-keyval/newStore "localforage" "keyvaluepairs" 2))) (defonce ^:private e2ee-password-file "e2ee-password") (defonce ^:private native-env? @@ -150,7 +151,7 @@ [k] (assert (and k @e2ee-store)) (p/let [r (idb-keyval/get k @e2ee-store)] - (js->clj r :keywordize-keys true))) + (some-> r (js->clj :keywordize-keys true)))) (defn- (p/let [cached (js body))} - {:response-schema :e2ee/user-keys}))) + (p/let [pair (fetch-json (str base "/e2ee/user-keys") + {:method "POST" + :headers {"content-type" "application/json"} + :body (js/JSON.stringify (clj->js body))} + {:response-schema :e2ee/user-keys}) + user-id (get-user-uuid) + _ ( (js body))} {:response-schema :e2ee/graph-aes-key}))) +(defn- ( (p/let [_ (