From 4fe1dde02ffabd81719d94b58d4ee2faa78d8e89 Mon Sep 17 00:00:00 2001 From: rcmerci Date: Sat, 7 Mar 2026 16:50:06 +0800 Subject: [PATCH] 052-logseq-cli-sync-upload-graph-uuid-alignment.md --- ...eq-cli-sync-upload-graph-uuid-alignment.md | 172 ++++++++++++++++++ docs/cli/logseq-cli.md | 2 + src/main/frontend/worker/sync.cljs | 40 ++-- src/test/frontend/worker/db_sync_test.cljs | 104 +++++++---- src/test/logseq/cli/integration_test.cljs | 55 ++++++ 5 files changed, 319 insertions(+), 54 deletions(-) create mode 100644 docs/agent-guide/052-logseq-cli-sync-upload-graph-uuid-alignment.md diff --git a/docs/agent-guide/052-logseq-cli-sync-upload-graph-uuid-alignment.md b/docs/agent-guide/052-logseq-cli-sync-upload-graph-uuid-alignment.md new file mode 100644 index 0000000000..288acc2032 --- /dev/null +++ b/docs/agent-guide/052-logseq-cli-sync-upload-graph-uuid-alignment.md @@ -0,0 +1,172 @@ +# CLI Sync Upload Graph UUID Alignment Implementation Plan + +Goal: Make `logseq sync upload` persist local graph UUID metadata so CLI and web app upload flows leave the graph in the same sync-ready state. + +Architecture: Keep `:thread-api/db-sync-upload-graph` as the single upload contract used by both CLI and web app. +Architecture: Align identity persistence inside `frontend.worker.sync` so every resolved graph id is written to both client-op storage and graph KV metadata. +Architecture: Use web app graph identity persistence semantics from `frontend.handler.db_based.sync/` succeeds, local graph metadata must contain a stable `:logseq.kv/graph-uuid` value in graph KV and the same graph id in client-op storage. + +The persistence rule must be enforced by worker sync code so both CLI and web app callers inherit identical behavior. + +When upload finds graph id from client-op fallback, worker must backfill missing graph KV UUID before returning success. + +Repeated uploads must be idempotent and must not create a new graph id or rewrite to a different value unexpectedly. + +## Architecture and integration points + +```text +CLI path + logseq.cli.command.sync/execute-sync-upload + -> logseq.cli.transport/invoke POST /v1/invoke + -> frontend.worker.db_worker_node /v1/invoke + -> :thread-api/db-sync-upload-graph in frontend.worker.db_core + -> frontend.worker.sync/ frontend.worker.sync/upload-graph! + +Web app path + frontend.handler.db_based.sync/ state/ frontend.worker.sync/ frontend.worker.sync/upload-graph! + +Web app source-of-truth identity persistence reference + frontend.handler.db_based.sync/ ldb/transact! with :logseq.kv/graph-uuid +``` + +## Testing Plan + +I will add worker unit tests that fail first when upload identity resolution does not persist `:logseq.kv/graph-uuid` into the graph database. + +I will add worker unit tests that fail first for the three identity branches, which are graph id from remote create, graph id from remote name match, and graph id from client-op fallback. + +I will add regression assertions that verify the persisted graph UUID is readable through `logseq.db/get-graph-rtc-uuid` and matches `client-op/get-graph-uuid`. + +I will add a CLI-facing regression check that validates upload success is followed by graph info data containing `logseq.kv/graph-uuid` in real or staged integration coverage. + +I will run targeted tests after each micro-change, then run broad lint and test commands before final review. + +NOTE: I will write *all* tests before I add any implementation behavior. + +## Implementation plan + +### Phase 1. Add failing worker tests for UUID persistence parity. + +1. Add a failing assertion in `/Users/rcmerci/gh-repos/logseq/src/test/frontend/worker/db_sync_test.cljs` for remote-create upload bootstrap to assert `ldb/get-graph-rtc-uuid` is set after ` --output json` and confirm `data.kv.logseq.kv/graph-uuid` is present. If it is missing, rerun `sync upload` for the same graph to trigger identity backfill. Sync download behavior: - `sync download` requires `--graph `. diff --git a/src/main/frontend/worker/sync.cljs b/src/main/frontend/worker/sync.cljs index 3f436daa78..29cac24c00 100644 --- a/src/main/frontend/worker/sync.cljs +++ b/src/main/frontend/worker/sync.cljs @@ -207,7 +207,6 @@ (def ^:private max-asset-size (* 100 1024 1024)) (def ^:private upload-kvs-batch-size 500) (def ^:private snapshot-content-type "application/transit+json") -(def ^:private snapshot-content-encoding "gzip") (def ^:private snapshot-text-encoder (js/TextEncoder.)) (def ^:private reconnect-base-delay-ms 1000) (def ^:private reconnect-max-delay-ms 30000) @@ -2140,32 +2139,37 @@ (.set out data 4) out)) -(defn- maybe-compress-stream [stream] - (if (exists? js/CompressionStream) - (.pipeThrough stream (js/CompressionStream. "gzip")) - stream)) - -(defn- uuid + [repo graph-id] + (when-not (seq graph-id) + (fail-fast :db-sync/missing-field {:repo repo :field :graph-id})) + (try + (uuid graph-id) + (catch :default e + (fail-fast :db-sync/invalid-field {:repo repo + :field :graph-id + :value graph-id + :error e})))) + (defn- set-graph-sync-metadata! - [repo graph-e2ee?] + [repo graph-id graph-e2ee?] (when-let [conn (worker-state/get-datascript-conn repo)] - (ldb/transact! conn [(ldb/kv :logseq.kv/graph-remote? true) + (ldb/transact! conn [(ldb/kv :logseq.kv/graph-uuid (graph-id->uuid repo graph-id)) + (ldb/kv :logseq.kv/graph-remote? true) (ldb/kv :logseq.kv/graph-rtc-e2ee? (true? graph-e2ee?))]))) (defn- persist-upload-graph-identity! [repo graph-id graph-e2ee?] - (let [graph-e2ee? (normalize-graph-e2ee? graph-e2ee?)] - (set-graph-sync-metadata! repo graph-e2ee?) + (let [graph-id (some-> graph-id str) + graph-e2ee? (normalize-graph-e2ee? graph-e2ee?)] + (when-not (seq graph-id) + (fail-fast :db-sync/missing-field {:repo repo :field :graph-id})) + (set-graph-sync-metadata! repo graph-id graph-e2ee?) (ensure-client-graph-uuid! repo graph-id) {:graph-id graph-id :graph-e2ee? graph-e2ee?})) @@ -2227,8 +2231,8 @@ (defn- repo common-config/strip-leading-db-version-prefix) local-graph-e2ee? (normalize-graph-e2ee? (sync-crypt/graph-e2ee? repo))] (if-not (seq target-graph-name) diff --git a/src/test/frontend/worker/db_sync_test.cljs b/src/test/frontend/worker/db_sync_test.cljs index 317428726b..1b8646bd69 100644 --- a/src/test/frontend/worker/db_sync_test.cljs +++ b/src/test/frontend/worker/db_sync_test.cljs @@ -297,35 +297,26 @@ :t 18 :txs [{:t 18 :tx tx-payload}]})) local-tx (atom 16) - orig-get-local-tx client-op/get-local-tx - orig-update-local-tx client-op/update-local-tx - orig-graph-e2ee? sync-crypt/graph-e2ee? - orig-ensure-graph-aes-key sync-crypt/ (p/let [_ (#'db-sync/handle-message! test-repo client raw-message) - error-state @(:last-sync-error client)] - (is (= 16 @local-tx)) - (is (= :missing-field (:code error-state))) - (is (= "missing-field" (:message error-state))) - (is (= :aes-key (get-in error-state [:data :field])))) + (-> (p/with-redefs [client-op/get-local-tx (fn [_repo] @local-tx) + client-op/update-local-tx (fn [_repo tx] (reset! local-tx tx)) + sync-crypt/graph-e2ee? (fn [_repo] true) + sync-crypt/latest-remote-tx {}) (with-datascript-conns conn client-ops-conn (fn [] - (-> (p/let [_ (#'db-sync/handle-message! test-repo client raw-new) - _ (#'db-sync/handle-message! test-repo client raw-stale) - parent' (d/entity @conn parent-id)] - (is (= "remote-new-title" (:block/title parent'))) - (is (= 2 (client-op/get-local-tx test-repo)))) + (-> (p/with-redefs [sync-crypt/graph-e2ee? (fn [_repo] false)] + (p/let [_ (client-op/update-local-tx test-repo 0) + _ (#'db-sync/handle-message! test-repo client raw-new) + _ (#'db-sync/handle-message! test-repo client raw-stale) + parent' (d/entity @conn parent-id)] + (is (= "remote-new-title" (:block/title parent'))) + (is (= 2 (client-op/get-local-tx test-repo))))) (p/finally (fn [] (reset! db-sync/*repo->latest-remote-tx latest-prev) (done)))))))))) @@ -1320,10 +1313,12 @@ (deftest ensure-upload-graph-identity-creates-remote-graph-when-local-graph-id-missing-test (testing "first upload bootstrap creates and persists a remote graph when local metadata is missing" (async done - (let [client-ops-conn (d/create-conn client-op/schema-in-db) + (let [conn (d/create-conn {}) + client-ops-conn (d/create-conn client-op/schema-in-db) self-prev (js* "globalThis.self") fetch-prev js/fetch config-prev @worker-state/*db-sync-config + created-graph-id "c52bf9d4-3a95-4f17-9c8c-7f86eef5f75d" fetch-calls (atom [])] (aset js/globalThis "self" #js {:postMessage (fn [_] nil)}) (reset! worker-state/*db-sync-config {:http-base "https://example.com" @@ -1339,19 +1334,21 @@ (and (= "https://example.com/graphs" url) (= "POST" method)) - (json-response 200 {:graph-id "created-graph-id" + (json-response 200 {:graph-id created-graph-id :graph-e2ee? false}) :else (js/Promise.reject (js/Error. (str "unexpected fetch url: " method " " url))))))) - (with-worker-conns {} {test-repo client-ops-conn} + (with-worker-conns {test-repo conn} {test-repo client-ops-conn} (fn [] (with-redefs [db-sync/coerce-http-request (fn [_schema-key body] body)] (-> (p/let [result (#'db-sync/ (ldb/get-graph-rtc-uuid @conn) str))) (is (= [{:url "https://example.com/graphs" :method "GET"} {:url "https://example.com/graphs" :method "POST"}] @fetch-calls))) @@ -1366,10 +1363,12 @@ (deftest ensure-upload-graph-identity-reuses-existing-remote-graph-when-local-graph-id-missing-test (testing "first upload bootstrap reuses a same-name remote graph instead of creating a duplicate" (async done - (let [client-ops-conn (d/create-conn client-op/schema-in-db) + (let [conn (d/create-conn {}) + client-ops-conn (d/create-conn client-op/schema-in-db) self-prev (js* "globalThis.self") fetch-prev js/fetch config-prev @worker-state/*db-sync-config + remote-graph-id "56cfcfc5-035a-4c6a-a5ba-70fe9cc27530" fetch-calls (atom [])] (aset js/globalThis "self" #js {:postMessage (fn [_] nil)}) (reset! worker-state/*db-sync-config {:http-base "https://example.com" @@ -1382,20 +1381,22 @@ (and (= "https://example.com/graphs" url) (= "GET" method)) (json-response 200 {:graphs [{:graph-name test-repo - :graph-id "remote-graph-id" + :graph-id remote-graph-id :graph-e2ee? false :created-at 1 :updated-at 1}]}) :else (js/Promise.reject (js/Error. (str "unexpected fetch url: " method " " url))))))) - (with-worker-conns {} {test-repo client-ops-conn} + (with-worker-conns {test-repo conn} {test-repo client-ops-conn} (fn [] (-> (p/let [result (#'db-sync/ (ldb/get-graph-rtc-uuid @conn) str))) (is (= [{:url "https://example.com/graphs" :method "GET"}] @fetch-calls))) (p/catch (fn [e] @@ -1490,6 +1491,37 @@ (reset! worker-state/*db-sync-config config-prev) (done))))))))))) +(deftest ensure-upload-graph-identity-backfills-graph-kv-from-client-op-fallback-test + (testing "upload identity fallback backfills graph kv and stays idempotent" + (async done + (let [conn (d/create-conn {}) + client-ops-conn (d/create-conn client-op/schema-in-db) + fallback-graph-id "6f3b0d47-4a62-4da7-8e56-7e5dbfbe3f47" + list-calls (atom 0)] + (with-worker-conns {test-repo conn} {test-repo client-ops-conn} + (fn [] + (with-redefs [sync-crypt/graph-e2ee? (fn [_repo] nil) + db-sync/get-graph-id (fn [_repo] fallback-graph-id) + db-sync/list-remote-graphs! (fn [] + (swap! list-calls inc) + (p/resolved []))] + (is (nil? (client-op/get-graph-uuid test-repo))) + (is (nil? (ldb/get-graph-rtc-uuid @conn))) + (let [first-result-p (#'db-sync/ (p/let [first-result first-result-p + second-result second-result-p] + (is (= fallback-graph-id (:graph-id first-result))) + (is (= fallback-graph-id (:graph-id second-result))) + (is (= (:graph-e2ee? first-result) + (:graph-e2ee? second-result))) + (is (= fallback-graph-id + (some-> (ldb/get-graph-rtc-uuid @conn) str))) + (is (= 0 @list-calls))) + (p/catch (fn [e] + (is false (str "unexpected error: " e)))) + (p/finally done)))))))))) + (deftest ^:long rehydrate-large-title-test (testing "rehydrate fills empty title from object storage" (async done diff --git a/src/test/logseq/cli/integration_test.cljs b/src/test/logseq/cli/integration_test.cljs index 22a1ccfa70..746e3b65ca 100644 --- a/src/test/logseq/cli/integration_test.cljs +++ b/src/test/logseq/cli/integration_test.cljs @@ -310,6 +310,61 @@ (set! cli-server/ensure-server! orig-ensure-server!) (set! transport/invoke orig-invoke))))))) +(deftest test-cli-sync-upload-followed-by-graph-info-shows-graph-uuid-test + (async done + (let [data-dir (node-helper/create-tmp-dir "db-worker-sync-upload-info-cli") + upload-repo "sync-upload-graph-info" + uploaded-graph-id "0f64b4a9-6f31-4f35-a83c-6b16f9ddf1ff" + orig-ensure-server! cli-server/ensure-server! + orig-invoke transport/invoke + invoke-calls (atom [])] + (-> (p/let [cfg-path (node-path/join (node-helper/create-tmp-dir "cli") "cli.edn") + _ (fs/writeFileSync cfg-path "{:output-format :json}") + create-result (run-cli ["graph" "create" "--graph" upload-repo] data-dir cfg-path) + create-payload (parse-json-output-safe create-result "graph create") + _ (is (= 0 (:exit-code create-result))) + _ (is (= "ok" (:status create-payload))) + _ (set! cli-server/ensure-server! + (fn [config _repo] + (p/resolved (assoc config :base-url "http://example")))) + _ (set! transport/invoke + (fn [_ method _direct-pass? args] + (swap! invoke-calls conj [method args]) + (case method + :thread-api/set-db-sync-config + (p/resolved nil) + + :thread-api/db-sync-upload-graph + (p/resolved {:graph-id uploaded-graph-id}) + + :thread-api/q + (p/resolved [[:logseq.kv/graph-uuid uploaded-graph-id] + [:logseq.kv/schema-version "65"]]) + + (p/resolved nil)))) + upload-result (run-cli ["--graph" upload-repo "sync" "upload"] data-dir cfg-path) + upload-payload (parse-json-output-safe upload-result "sync upload") + info-result (run-cli ["--graph" upload-repo "graph" "info"] data-dir cfg-path) + info-payload (parse-json-output-safe info-result "graph info after upload") + q-call (some (fn [[method args]] + (when (= :thread-api/q method) + args)) + @invoke-calls)] + (is (= 0 (:exit-code upload-result))) + (is (= "ok" (:status upload-payload))) + (is (= uploaded-graph-id (get-in upload-payload [:data :graph-id]))) + (is (= 0 (:exit-code info-result))) + (is (= "ok" (:status info-payload))) + (is (= uploaded-graph-id + (get-in info-payload [:data :kv :logseq.kv/graph-uuid]))) + (is (= "logseq_db_sync-upload-graph-info" (first q-call)))) + (p/catch (fn [e] + (is false (str "unexpected error: " e)))) + (p/finally (fn [] + (set! cli-server/ensure-server! orig-ensure-server!) + (set! transport/invoke orig-invoke) + (done))))))) + (deftest ^:long test-cli-graph-list (async done (let [data-dir (node-helper/create-tmp-dir "db-worker")]