From c8da78efd1c1635f72d235c1ac4af4d53c30199b Mon Sep 17 00:00:00 2001 From: rcmerci Date: Sat, 7 Mar 2026 10:32:29 +0800 Subject: [PATCH] 051-logseq-cli-sync-upload-fix.md --- .../051-logseq-cli-sync-upload-fix.md | 347 ++++++++++++++++++ docs/cli/logseq-cli.md | 10 + src/main/frontend/handler/db_based/sync.cljs | 5 +- src/main/frontend/worker/sync.cljs | 261 ++++++++----- src/main/frontend/worker_common/util.cljc | 2 +- src/main/logseq/cli/command/sync.cljs | 74 ++-- .../frontend/handler/db_based/sync_test.cljs | 34 ++ src/test/frontend/worker/db_sync_test.cljs | 270 ++++++++++++-- src/test/logseq/cli/command/sync_test.cljs | 32 ++ src/test/logseq/cli/integration_test.cljs | 46 +++ 10 files changed, 927 insertions(+), 154 deletions(-) create mode 100644 docs/agent-guide/051-logseq-cli-sync-upload-fix.md diff --git a/docs/agent-guide/051-logseq-cli-sync-upload-fix.md b/docs/agent-guide/051-logseq-cli-sync-upload-fix.md new file mode 100644 index 0000000000..e4c3c241ee --- /dev/null +++ b/docs/agent-guide/051-logseq-cli-sync-upload-fix.md @@ -0,0 +1,347 @@ +# Logseq CLI Sync Upload Fix Plan + +Goal: Fix `logseq sync upload` so a CLI-managed local graph can be uploaded successfully with the current `logseq-cli` + `db-worker-node` architecture, including the first upload of a graph that does not yet have a local `graph-id`. + +Architecture: Keep the CLI surface small and move the real fix into the worker-side sync layer so `:thread-api/db-sync-upload-graph` becomes self-sufficient: it should resolve or create the remote graph when needed, persist the resulting graph metadata locally, and then perform snapshot upload. + +Architecture: Remove the current error-swallowing behavior in worker upload so failures propagate back through `db-worker-node` and the CLI can report an actual error instead of a false success. + +Tech Stack: ClojureScript, `logseq-cli`, `db-worker-node`, `frontend.worker.sync`, db-sync HTTP endpoints, promesa, babashka test runner. + +Related: Builds on `docs/agent-guide/047-logseq-cli-sync-command.md`. + +Related: Relates to `docs/cli/logseq-cli.md` and `docs/developers/desktop-db-worker-node.md`. + +## Problem statement + +With the current implementation, `logseq sync upload --graph ` is wired only as: + +- CLI `sync upload` +- `logseq.cli.command.sync/execute` +- `:thread-api/db-sync-upload-graph` +- `frontend.worker.sync/upload-graph!` + +That path assumes the local graph already has a usable remote `graph-id`. + +However, `frontend.worker.sync/upload-graph!` currently fails when either `http-base` or `graph-id` is missing: + +- `http-base` comes from CLI sync config and is present in the provided `cli.edn`. +- `graph-id` is read from local graph metadata via `get-graph-id`. +- For a fresh local graph that has never been uploaded or downloaded before, `graph-id` is typically missing. + +The desktop/UI flow does not have this problem because `frontend.handler.db_based.sync/` should: + +1. start or reuse the graph's `db-worker-node` +2. apply sync config to the worker +3. detect whether the local graph already has a remote `graph-id` +4. if missing, resolve or create the remote graph +5. persist the resolved/created graph id and sync metadata locally +6. upload the graph snapshot +7. return a real success result only when upload actually succeeds +8. return a real error with context when any step fails + +This should work both for: + +- an already-linked graph that has a local `graph-id` +- a fresh local graph being uploaded for the first time + +## Recommended design + +### Option A: Fix inside worker upload orchestration + +Preferred approach: make `frontend.worker.sync/upload-graph!` capable of ensuring remote graph identity before snapshot upload. + +This keeps the CLI simple and makes `:thread-api/db-sync-upload-graph` a complete unit of behavior. + +Suggested worker-side flow: + +1. Read local `graph-id`. +2. If present, continue with existing upload logic. +3. If absent, list remote graphs visible to the current auth context. +4. Match by canonical graph name. +5. If a matching remote graph exists, bind local graph to that `graph-id`. +6. If no matching remote graph exists, create one with the current graph name and schema version. +7. Persist returned graph metadata locally. +8. Continue snapshot upload using the resolved `graph-id`. + +This mirrors the desktop behavior conceptually while avoiding duplicated orchestration in CLI code. + +### Option B: Add CLI-specific preflight before invoking upload + +Alternative approach: keep worker upload low-level and add a new CLI-side preflight that creates or resolves the remote graph before invoking `:thread-api/db-sync-upload-graph`. + +I do not recommend this because it duplicates sync orchestration across CLI and desktop-adjacent code and makes the worker thread API less useful as a stable contract. + +## Scope + +### In scope + +- Fix first-time `sync upload` for CLI-managed graphs. +- Make worker upload propagate failures. +- Reuse existing remote graph APIs (`GET /graphs`, `POST /graphs`) instead of inventing a new cloud protocol. +- Add tests for first-time upload, already-linked upload, and failure propagation. +- Update CLI docs so `sync upload` behavior is explicit. + +### Out of scope + +- Changing the cloud snapshot upload protocol. +- Adding a new top-level CLI command for graph creation. +- Reworking websocket sync start/stop behavior. +- Large refactors unrelated to upload bootstrap. + +## Implementation plan + +### Phase 1. Add failing tests for the broken behavior + +1. Add a worker sync test in `/Users/rcmerci/gh-repos/logseq/src/test/frontend/worker/db_sync_test.cljs` for first-time upload when local graph has no graph id. +2. Stub remote graph listing to return no match, stub graph creation to return a new id, and assert upload continues with that id. +3. Add a companion test where remote graph listing already contains the same graph name and assert upload reuses the existing graph id instead of creating a duplicate graph. +4. Add a failure test that simulates graph creation failure and asserts the promise rejects rather than resolving silently. +5. Add a failure test that simulates snapshot upload failure and asserts the CLI-visible result is an error, not success. +6. Add a CLI command execution/integration test in `/Users/rcmerci/gh-repos/logseq/src/test/logseq/cli/command/sync_test.cljs` or CLI integration tests that covers the real first-upload path, not just method wiring. + +### Phase 2. Move remote graph bootstrap into worker sync code + +7. Add a worker helper in `/Users/rcmerci/gh-repos/logseq/src/main/frontend/worker/sync.cljs` to ensure a usable remote graph id before snapshot upload. +8. That helper should read the local graph name from repo name using the same canonicalization rules already used by CLI and desktop sync. +9. If local graph id exists, return it immediately. +10. If local graph id is missing, call existing remote graph list API. +11. If a graph with the same canonical name exists, reuse its `graph-id` and persist it locally. +12. If no match exists, call the existing graph creation API and persist the new `graph-id` locally. +13. Keep local metadata writes inside worker code so later sync commands (`sync start`, asset upload, etc.) see a consistent graph identity. + +### Phase 3. Reuse or extract desktop graph-creation logic + +14. Avoid leaving two divergent graph-creation implementations. +15. Extract the parts of `frontend.handler.db_based.sync/`. +3. Confirm the command returns success only after remote graph bootstrap + snapshot upload finish. +4. Run `logseq sync status --graph ` and confirm the graph now has stable remote metadata. +5. Re-run `logseq sync upload --graph ` and confirm it reuses the existing graph id. +6. Force a failing auth token and confirm the CLI returns an error instead of false success. +7. Force a server-side snapshot upload failure and confirm the CLI returns an error with useful context. + +## Acceptance criteria + +The fix is complete when all of the following are true: + +- `logseq sync upload --graph ` works for a fresh local graph with no prior `graph-id`. +- The command reuses an existing same-name remote graph when appropriate. +- Worker upload failures propagate to CLI callers and no longer appear as success. +- Desktop and CLI upload bootstrap logic no longer diverge in a risky way. +- Test coverage includes first-time upload and failure propagation. +- CLI docs describe first-time upload behavior and required config without exposing secrets. + +## Risks and mitigations + +### Risk: duplicate remote graphs + +If upload always creates a graph when local `graph-id` is missing, we may create duplicates. + +Mitigation: list remote graphs first and only create when there is no exact name match. + +### Risk: inconsistent E2EE defaults + +Fresh graph creation from CLI may accidentally create an unencrypted graph while desktop defaults to encrypted. + +Mitigation: make fresh-upload `graph-e2ee?` behavior explicit and test it. + +### Risk: hidden regressions in desktop flow + +If desktop and worker share more code, UI expectations could shift. + +Mitigation: keep only transport/bootstrap logic shared; keep UI state refresh and notifications in frontend handler code. + +## Suggested file touch list + +Primary implementation files: + +- `/Users/rcmerci/gh-repos/logseq/src/main/frontend/worker/sync.cljs` +- `/Users/rcmerci/gh-repos/logseq/src/main/frontend/handler/db_based/sync.cljs` +- `/Users/rcmerci/gh-repos/logseq/src/main/logseq/cli/command/sync.cljs` + +Primary test files: + +- `/Users/rcmerci/gh-repos/logseq/src/test/frontend/worker/db_sync_test.cljs` +- `/Users/rcmerci/gh-repos/logseq/src/test/logseq/cli/command/sync_test.cljs` +- `/Users/rcmerci/gh-repos/logseq/src/test/logseq/cli/integration_test.cljs` + +Primary docs: + +- `/Users/rcmerci/gh-repos/logseq/docs/cli/logseq-cli.md` + +## Question + +No open question. The current implementation gap is clear enough to proceed with a worker-centered fix. diff --git a/docs/cli/logseq-cli.md b/docs/cli/logseq-cli.md index 16b4deb0c1..961e274c2e 100644 --- a/docs/cli/logseq-cli.md +++ b/docs/cli/logseq-cli.md @@ -103,6 +103,16 @@ Sync commands: - `sync config get [--graph ] ws-url|http-base|auth-token|e2ee-password` - get db-sync runtime config key - `sync config unset [--graph ] ws-url|http-base|auth-token|e2ee-password` - remove db-sync runtime config key +Sync upload behavior: +- `sync upload` requires `--graph `. +- The CLI starts or reuses that graph's `db-worker-node`, applies the current sync config, and uploads the local snapshot only after the worker has resolved a usable remote graph id. +- If the local graph already has sync metadata, upload reuses the stored remote `graph-id`. +- If the local graph does not have a stored remote `graph-id`, upload first lists visible remote graphs and reuses an exact same-name match when one exists. +- If no same-name remote graph exists, upload creates a new remote graph and persists the returned remote metadata locally before snapshot transfer. +- Fresh uploads default to encrypted remote graph creation unless local sync metadata explicitly marks the graph as non-e2ee. In headless CLI mode, set `e2ee-password` via `sync config set` (or in `--config`) before uploading encrypted graphs. +- `sync upload` returns a real error instead of false success when auth, remote graph bootstrap, or snapshot upload fails. +- Common upload failures include missing/invalid `auth-token`, missing `http-base`, remote graph creation failure, snapshot upload failure, and local DB/worker startup failure. + Sync download behavior: - `sync download` requires `--graph `. - If a local graph with the same name already exists, the CLI returns `graph-exists`. diff --git a/src/main/frontend/handler/db_based/sync.cljs b/src/main/frontend/handler/db_based/sync.cljs index 7d4e0544b3..b5718e3bfb 100644 --- a/src/main/frontend/handler/db_based/sync.cljs +++ b/src/main/frontend/handler/db_based/sync.cljs @@ -488,10 +488,9 @@ (defn js (with-auth-headers opts))) - text (.text resp) +(defn- parse-json-response-body + [resp url response-schema error-schema] + (p/let [text (.text resp) data (when (seq text) (js/JSON.parse text))] (if (.-ok resp) (let [body (js->clj data :keywordize-keys true) @@ -487,6 +499,11 @@ :url url :body body})))))) +(defn- fetch-json + [url opts {:keys [response-schema error-schema] :or {error-schema :error}}] + (p/let [resp (js/fetch url (clj->js (with-auth-headers opts)))] + (parse-json-response-body resp url response-schema error-schema))) + (defn- require-auth-token! [context] (when-not (seq (auth-token)) @@ -625,18 +642,20 @@ (-restore [_ addr] (restore-data-from-addr db addr)))) -(defn- create-temp-sqlite-db +(defn- clj result)))) +(defn- normalize-snapshot-row-value [value] + (cond + (nil? value) nil + (string? value) value + (number? value) value + (boolean? value) value + (= "bigint" (js* "typeof ~{}" value)) (str value) + (instance? js/Uint8Array value) (.decode text-decoder value) + (instance? js/ArrayBuffer value) (.decode text-decoder (js/Uint8Array. value)) + :else (str value))) + (defn- normalize-snapshot-rows [rows] - (mapv (fn [row] (vec row)) (array-seq rows))) + (mapv (fn [row] + (mapv normalize-snapshot-row-value (vec row))) + (array-seq rows))) (defn- encode-snapshot-rows [rows] (.encode snapshot-text-encoder (sqlite-util/write-transit-str rows))) @@ -2121,17 +2153,8 @@ (defn- repo common-config/strip-leading-db-version-prefix) + schema-version (some-> (worker-state/get-datascript-conn repo) + deref + ldb/get-graph-schema-version + :major + str) + graph-e2ee? (normalize-graph-e2ee? graph-e2ee?)] + (cond + (not (seq base)) + (fail-fast :db-sync/missing-field {:repo repo :field :http-base}) + + (not (seq graph-name)) + (fail-fast :db-sync/missing-field {:repo repo :field :graph-name}) + + :else + (do + (require-auth-token! {:repo repo :field :auth-token}) + (p/let [body (coerce-http-request :graphs/create + {:graph-name graph-name + :schema-version schema-version + :graph-e2ee? graph-e2ee?}) + _ (when (nil? body) + (fail-fast :db-sync/invalid-field {:repo repo + :field :create-graph-body})) + result (fetch-json (str base "/graphs") + {:method "POST" + :headers {"content-type" "application/json"} + :body (js/JSON.stringify (clj->js body))} + {:response-schema :graphs/create}) + graph-id (:graph-id result) + graph-e2ee? (normalize-graph-e2ee? (if (contains? result :graph-e2ee?) + (:graph-e2ee? result) + graph-e2ee?))] + (when-not (seq graph-id) + (fail-fast :db-sync/missing-field {:repo repo + :field :graph-id + :op :create-graph})) + (persist-upload-graph-identity! repo graph-id graph-e2ee?)))))) + +(defn- repo common-config/strip-leading-db-version-prefix) + local-graph-e2ee? (normalize-graph-e2ee? (sync-crypt/graph-e2ee? repo))] + (if-not (seq target-graph-name) + (fail-fast :db-sync/missing-field {:repo repo :field :graph-name}) + (p/let [remote-graphs (list-remote-graphs!) + matching-graphs (filterv (fn [{:keys [graph-name]}] + (= target-graph-name graph-name)) + remote-graphs)] + (cond + (> (count matching-graphs) 1) + (fail-fast :db-sync/ambiguous-graph-match {:repo repo + :graph-name target-graph-name + :match-count (count matching-graphs)}) + + (= 1 (count matching-graphs)) + (let [{:keys [graph-id graph-e2ee?]} (first matching-graphs)] + (persist-upload-graph-identity! repo graph-id (if (contains? (first matching-graphs) :graph-e2ee?) + graph-e2ee? + local-graph-e2ee?))) + + :else + ( - (let [base (http-base-url) - graph-id (get-graph-id repo) - update-progress (fn [payload] - (worker-util/post-message :rtc-log - (merge {:type :rtc.log/upload - :graph-uuid graph-id} - payload)))] - (if (and (seq base) (seq graph-id)) - (if-let [source-conn (worker-state/get-datascript-conn repo)] - (let [graph-e2ee? (true? (sync-crypt/graph-e2ee? repo))] - (p/let [aes-key (when graph-e2ee? - (sync-crypt/ - (p/loop [last-addr -1 - first-batch? true - loaded 0] - (let [rows (fetch-kvs-rows db last-addr upload-kvs-batch-size)] - (if (empty? rows) - (do - (client-op/remove-local-tx repo) - (client-op/update-local-tx repo 0) - (client-op/add-all-exists-asset-as-ops repo) - (update-progress {:sub-type :upload-completed - :message "Graph upload finished!"}) - {:graph-id graph-id}) - (let [max-addr (apply max (map first rows)) - rows (normalize-snapshot-rows rows) - upload-url (str base "/sync/" graph-id "/snapshot/upload?reset=" (if first-batch? "true" "false"))] - (p/let [{:keys [body encoding]} ( {"content-type" snapshot-content-type} - (string? encoding) (assoc "content-encoding" encoding)) - _ (fetch-json upload-url - {:method "POST" - :headers headers - :body body} - {:response-schema :sync/snapshot-upload})] - (let [loaded' (+ loaded (count rows))] - (update-progress {:sub-type :upload-progress - :message (str "Uploading " loaded' "/" total-rows)}) - (p/recur max-addr false loaded'))))))) - (p/finally - (fn [] - (cleanup-temp-sqlite! temp))))))) - (p/rejected (ex-info "db-sync missing datascript conn" - {:repo repo :graph-id graph-id}))) - (p/rejected (ex-info "db-sync missing upload info" - {:repo repo :base base :graph-id graph-id})))) - (p/catch (fn [error] - (js/console.error error))))) + (let [base (http-base-url)] + (if-not (seq base) + (p/rejected (ex-info "db-sync missing upload info" + {:repo repo :base base :graph-id nil})) + (if-let [source-conn (worker-state/get-datascript-conn repo)] + (p/let [{:keys [graph-id graph-e2ee?]} ( + (p/loop [last-addr -1 + first-batch? true + loaded 0] + (let [rows (fetch-kvs-rows db last-addr upload-kvs-batch-size)] + (if (empty? rows) + (do + (client-op/remove-local-tx repo) + (client-op/update-local-tx repo 0) + (client-op/add-all-exists-asset-as-ops repo) + (update-progress {:sub-type :upload-completed + :message "Graph upload finished!"}) + {:graph-id graph-id}) + (let [max-addr (apply max (map first rows)) + rows (normalize-snapshot-rows rows) + upload-url (str base "/sync/" graph-id "/snapshot/upload?reset=" (if first-batch? "true" "false"))] + (p/let [{:keys [body encoding]} ( {"content-type" snapshot-content-type} + (string? encoding) (assoc "content-encoding" encoding)) + resp (js/fetch upload-url + #js {:method "POST" + :headers (clj->js (merge (or (auth-headers) {}) headers)) + :body body}) + _ (parse-json-response-body resp upload-url :sync/snapshot-upload :error)] + (let [loaded' (+ loaded (count rows))] + (update-progress {:sub-type :upload-progress + :message (str "Uploading " loaded' "/" total-rows)}) + (p/recur max-addr false loaded'))))))) + (p/finally + (fn [] + (cleanup-temp-sqlite! temp))))))) + (p/rejected (ex-info "db-sync missing datascript conn" + {:repo repo})))))) diff --git a/src/main/frontend/worker_common/util.cljc b/src/main/frontend/worker_common/util.cljc index ec9dab4695..1a6767ec7e 100644 --- a/src/main/frontend/worker_common/util.cljc +++ b/src/main/frontend/worker_common/util.cljc @@ -28,7 +28,7 @@ (do (defn post-message [type data & {:keys [port]}] - (when-let [worker (or port js/self)] + (when-let [worker (or port (when (exists? js/self) js/self))] (.postMessage worker (ldb/write-transit-str [type data])))) (defn encode-graph-dir-name diff --git a/src/main/logseq/cli/command/sync.cljs b/src/main/logseq/cli/command/sync.cljs index 3f34ae92fc..2be65fa2ca 100644 --- a/src/main/logseq/cli/command/sync.cljs +++ b/src/main/logseq/cli/command/sync.cljs @@ -285,6 +285,46 @@ (poll!)))))] (poll!)))) +(defn- execute-sync-upload + [action config] + (-> (p/let [result (invoke-with-repo config (:repo action) + :thread-api/db-sync-upload-graph + [(:repo action)])] + {:status :ok + :data (if (map? result) + result + {:result result})}) + (p/catch (fn [error] + (exception->error error {:repo (:repo action)}))))) + +(defn- execute-sync-download + [action config] + (let [config' (download-config config)] + (-> (p/let [remote-graphs (invoke-global config' + :thread-api/db-sync-list-remote-graphs + []) + remote-graph (some (fn [graph] + (when (= (:graph action) (:graph-name graph)) + graph)) + remote-graphs)] + (if-not remote-graph + {:status :error + :error {:code :remote-graph-not-found + :message (str "remote graph not found: " (:graph action)) + :graph (:graph action)}} + (p/let [cfg (cli-server/ensure-server! config' (:repo action)) + _ (transport/invoke cfg :thread-api/set-db-sync-config false [(sync-config config')]) + _ (ensure-empty-download-db! cfg (:repo action)) + result (transport/invoke cfg :thread-api/db-sync-download-graph-by-id false + [(:repo action) (:graph-id remote-graph) (:graph-e2ee? remote-graph)])] + {:status :ok + :data (if (map? result) + result + {:result result})}))) + (p/catch (fn [error] + (exception->error error {:repo (:repo action) + :graph (:graph action)})))))) + (defn execute [action config] (case (:type action) @@ -312,40 +352,10 @@ :data {:result result}}) :sync-upload - (p/let [result (invoke-with-repo config (:repo action) - :thread-api/db-sync-upload-graph - [(:repo action)])] - {:status :ok - :data (if (map? result) - result - {:result result})}) + (execute-sync-upload action config) :sync-download - (let [config' (download-config config)] - (-> (p/let [remote-graphs (invoke-global config' - :thread-api/db-sync-list-remote-graphs - []) - remote-graph (some (fn [graph] - (when (= (:graph action) (:graph-name graph)) - graph)) - remote-graphs)] - (if-not remote-graph - {:status :error - :error {:code :remote-graph-not-found - :message (str "remote graph not found: " (:graph action)) - :graph (:graph action)}} - (p/let [cfg (cli-server/ensure-server! config' (:repo action)) - _ (transport/invoke cfg :thread-api/set-db-sync-config false [(sync-config config')]) - _ (ensure-empty-download-db! cfg (:repo action)) - result (transport/invoke cfg :thread-api/db-sync-download-graph-by-id false - [(:repo action) (:graph-id remote-graph) (:graph-e2ee? remote-graph)])] - {:status :ok - :data (if (map? result) - result - {:result result})}))) - (p/catch (fn [error] - (exception->error error {:repo (:repo action) - :graph (:graph action)}))))) + (execute-sync-download action config) :sync-remote-graphs (p/let [graphs (invoke-global config :thread-api/db-sync-list-remote-graphs [])] diff --git a/src/test/frontend/handler/db_based/sync_test.cljs b/src/test/frontend/handler/db_based/sync_test.cljs index 59b815747f..189856d5d4 100644 --- a/src/test/frontend/handler/db_based/sync_test.cljs +++ b/src/test/frontend/handler/db_based/sync_test.cljs @@ -153,6 +153,40 @@ (is false (str e)) (done))))))) +(deftest rtc-upload-graph-persists-local-e2ee-choice-and-defers-bootstrap-to-worker-test + (async done + (let [tx-called (atom nil) + worker-calls (atom []) + get-remote-graphs-calls (atom 0) + rtc-start-calls (atom 0)] + (-> (p/with-redefs [ldb/transact! (fn [repo tx-data] + (reset! tx-called {:repo repo :tx-data tx-data}) + nil) + state/js body)) + "")] + (js/Promise.resolve + #js {:ok (<= 200 status 299) + :status status + :text (fn [] (js/Promise.resolve payload))}))) + (deftest resolve-ws-token-refreshes-when-token-expired-test (async done (let [refresh-calls (atom 0) @@ -264,41 +291,41 @@ (deftest pull-ok-failure-records-last-sync-error-test (testing "pull/ok failures should surface structured runtime error state" (async done - (let [tx-payload (sqlite-util/write-transit-str [[:db/add 1 :block/title "remote"]]) - raw-message (js/JSON.stringify - (clj->js {:type "pull/ok" - :t 18 - :txs [{:t 18 :tx tx-payload}]})) - local-tx (atom 16) - orig-get-local-tx client-op/get-local-tx - orig-update-local-tx client-op/update-local-tx - orig-graph-e2ee? sync-crypt/graph-e2ee? - orig-ensure-graph-aes-key sync-crypt/ (p/let [_ (#'db-sync/handle-message! test-repo client raw-message) - error-state @(:last-sync-error client)] - (is (= 16 @local-tx)) - (is (= :missing-field (:code error-state))) - (is (= "missing-field" (:message error-state))) - (is (= :aes-key (get-in error-state [:data :field])))) - (p/catch (fn [e] - (is false (str "unexpected error: " e)))) - (p/finally (fn [] - (set! client-op/get-local-tx orig-get-local-tx) - (set! client-op/update-local-tx orig-update-local-tx) - (set! sync-crypt/graph-e2ee? orig-graph-e2ee?) - (set! sync-crypt/js {:type "pull/ok" + :t 18 + :txs [{:t 18 :tx tx-payload}]})) + local-tx (atom 16) + orig-get-local-tx client-op/get-local-tx + orig-update-local-tx client-op/update-local-tx + orig-graph-e2ee? sync-crypt/graph-e2ee? + orig-ensure-graph-aes-key sync-crypt/ (p/let [_ (#'db-sync/handle-message! test-repo client raw-message) + error-state @(:last-sync-error client)] + (is (= 16 @local-tx)) + (is (= :missing-field (:code error-state))) + (is (= "missing-field" (:message error-state))) + (is (= :aes-key (get-in error-state [:data :field])))) + (p/catch (fn [e] + (is false (str "unexpected error: " e)))) + (p/finally (fn [] + (set! client-op/get-local-tx orig-get-local-tx) + (set! client-op/update-local-tx orig-update-local-tx) + (set! sync-crypt/graph-e2ee? orig-graph-e2ee?) + (set! sync-crypt/ opts .-method) "GET")] + (swap! fetch-calls conj {:url url :method method}) + (cond + (and (= "https://example.com/graphs" url) + (= "GET" method)) + (json-response 200 {:graphs []}) + + (and (= "https://example.com/graphs" url) + (= "POST" method)) + (json-response 200 {:graph-id "created-graph-id" + :graph-e2ee? false}) + + :else + (js/Promise.reject (js/Error. (str "unexpected fetch url: " method " " url))))))) + (with-worker-conns {} {test-repo client-ops-conn} + (fn [] + (with-redefs [db-sync/coerce-http-request (fn [_schema-key body] body)] + (-> (p/let [result (#'db-sync/ opts .-method) "GET")] + (swap! fetch-calls conj {:url url :method method}) + (cond + (and (= "https://example.com/graphs" url) + (= "GET" method)) + (json-response 200 {:graphs [{:graph-name test-repo + :graph-id "remote-graph-id" + :graph-e2ee? false + :created-at 1 + :updated-at 1}]}) + + :else + (js/Promise.reject (js/Error. (str "unexpected fetch url: " method " " url))))))) + (with-worker-conns {} {test-repo client-ops-conn} + (fn [] + (-> (p/let [result (#'db-sync/ opts .-method) "GET")] + (cond + (and (= "https://example.com/graphs" url) + (= "GET" method)) + (json-response 200 {:graphs []}) + + (and (= "https://example.com/graphs" url) + (= "POST" method)) + (json-response 500 {:error "create failed"}) + + :else + (js/Promise.reject (js/Error. (str "unexpected fetch url: " method " " url))))))) + (with-worker-conns {} {test-repo client-ops-conn} + (fn [] + (with-redefs [db-sync/coerce-http-request (fn [_schema-key body] body)] + (-> (p/let [_ (#'db-sync/ opts .-method) "GET")] + (cond + (and (= "https://example.com/graphs" url) + (= "GET" method)) + (json-response 200 {:graphs []}) + + (and (= "https://example.com/graphs" url) + (= "POST" method)) + (do + (reset! create-body (-> opts .-body js/JSON.parse (js->clj :keywordize-keys true))) + (json-response 200 {:graph-id "encrypted-graph-id"})) + + :else + (js/Promise.reject (js/Error. (str "unexpected fetch url: " method " " url))))))) + (with-worker-conns {} {test-repo client-ops-conn} + (fn [] + (with-redefs [db-sync/coerce-http-request (fn [_schema-key body] body) + sync-crypt/graph-e2ee? (fn [_repo] nil)] + (-> (p/let [result (#'db-sync/ (p/let [result (sync-command/execute {:type :sync-upload + :repo "logseq_db_demo"} + {:data-dir "/tmp"})] + (is (= :error (:status result))) + (is (= :snapshot-upload-failed (get-in result [:error :code]))) + (is (= 500 (get-in result [:error :context :status]))) + (is (= "graph-1" (get-in result [:error :context :graph-id])))) + (p/catch (fn [e] + (is false (str "unexpected error: " e)))) + (p/finally (fn [] + (set! cli-server/ensure-server! orig-ensure-server!) + (set! transport/invoke orig-invoke) + (done))))))) + (deftest test-execute-sync-download (async done (let [orig-ensure-server! cli-server/ensure-server! diff --git a/src/test/logseq/cli/integration_test.cljs b/src/test/logseq/cli/integration_test.cljs index 10d3a32cfa..22a1ccfa70 100644 --- a/src/test/logseq/cli/integration_test.cljs +++ b/src/test/logseq/cli/integration_test.cljs @@ -264,6 +264,52 @@ (set! cli-server/ensure-server! orig-ensure-server!) (set! transport/invoke orig-invoke))))))) +(deftest test-cli-sync-upload-with-mocked-worker-bootstrap + (async done + (let [data-dir (node-helper/create-tmp-dir "db-worker-sync-upload-cli") + upload-repo "sync-upload-graph" + orig-ensure-server! cli-server/ensure-server! + orig-invoke transport/invoke + invoke-calls (atom [])] + (-> (p/let [cfg-path (node-path/join (node-helper/create-tmp-dir "cli") "cli.edn") + _ (fs/writeFileSync cfg-path "{:output-format :json}") + create-result (run-cli ["graph" "create" "--graph" upload-repo] data-dir cfg-path) + create-payload (parse-json-output-safe create-result "graph create") + _ (is (= 0 (:exit-code create-result))) + _ (is (= "ok" (:status create-payload))) + _ (set! cli-server/ensure-server! + (fn [config _repo] + (p/resolved (assoc config :base-url "http://example")))) + _ (set! transport/invoke + (fn [_ method _direct-pass? args] + (swap! invoke-calls conj [method args]) + (case method + :thread-api/set-db-sync-config + (p/resolved nil) + + :thread-api/db-sync-upload-graph + (p/resolved {:graph-id "created-graph-id"}) + + (p/resolved nil)))) + upload-result (run-cli ["--graph" upload-repo "sync" "upload"] data-dir cfg-path) + upload-payload (parse-json-output-safe upload-result "sync upload")] + (is (= 0 (:exit-code upload-result))) + (is (= "ok" (:status upload-payload))) + (is (= "created-graph-id" (get-in upload-payload [:data :graph-id]))) + (is (= [[:thread-api/set-db-sync-config [{:ws-url nil + :http-base nil + :auth-token nil + :e2ee-password nil}]] + [:thread-api/db-sync-upload-graph ["logseq_db_sync-upload-graph"]]] + @invoke-calls)) + (done)) + (p/catch (fn [e] + (is false (str "unexpected error: " e)) + (done))) + (p/finally (fn [] + (set! cli-server/ensure-server! orig-ensure-server!) + (set! transport/invoke orig-invoke))))))) + (deftest ^:long test-cli-graph-list (async done (let [data-dir (node-helper/create-tmp-dir "db-worker")]