diff --git a/docs/agent-guide/050-sync-download-create-empty-db.md b/docs/agent-guide/050-sync-download-create-empty-db.md new file mode 100644 index 0000000000..bcd6faf57f --- /dev/null +++ b/docs/agent-guide/050-sync-download-create-empty-db.md @@ -0,0 +1,188 @@ +# Sync Download Create Empty Db Implementation Plan + +Goal: Ensure `sync download` never writes `db-initial-data` before snapshot import by starting `db-worker-node` with an explicit empty-db mode. + +Architecture: Add a new db-worker startup flag `--create-empty-db` and plumb it through CLI server orchestration only for the `sync download` path. + +Architecture: When the flag is enabled, `db-worker-node` will call `:thread-api/create-or-open-db` with `{:datoms []}` during daemon startup so `frontend.worker.db-core/ /Users/rcmerci/gh-repos/logseq/src/main/logseq/cli/command/sync.cljs + -> /Users/rcmerci/gh-repos/logseq/src/main/logseq/cli/server.cljs + -> /Users/rcmerci/gh-repos/logseq/src/main/logseq/db_worker/daemon.cljs + -> /Users/rcmerci/gh-repos/logseq/src/main/frontend/worker/db_worker_node.cljs + -> /Users/rcmerci/gh-repos/logseq/src/main/frontend/worker/db_core.cljs + -> :thread-api/db-sync-download-graph-by-id import path +``` + +## Scope and non-goals + +In scope are CLI-to-daemon argument plumbing, db-worker startup behavior change behind a new flag, tests, and docs updates. + +Out of scope are sync protocol changes, snapshot payload format changes, and non-download command behavior changes. + +## Implementation plan + +### Phase 0. Baseline and failure capture. + +1. Run `bb dev:test -v logseq.db-worker.daemon-test` and record baseline. +2. Run `bb dev:test -v frontend.worker.db-worker-node-test` and record baseline. +3. Run `bb dev:test -v logseq.cli.server-test` and record baseline. +4. Run `bb dev:test -v logseq.cli.command.sync-test` and record baseline. + +### Phase 1. RED tests for daemon flag plumbing. + +5. Add a failing test in `/Users/rcmerci/gh-repos/logseq/src/test/logseq/db_worker/daemon_test.cljs` that `spawn-server!` appends `--create-empty-db` when `:create-empty-db? true` is passed. +6. Add a failing test in `/Users/rcmerci/gh-repos/logseq/src/test/logseq/db_worker/daemon_test.cljs` that `spawn-server!` keeps default args unchanged when `:create-empty-db?` is absent or false. +7. Add a failing test in `/Users/rcmerci/gh-repos/logseq/src/test/frontend/worker/db_worker_node_test.cljs` that `parse-args` recognizes `--create-empty-db` and maps it to `:create-empty-db? true`. +8. Add a failing test in `/Users/rcmerci/gh-repos/logseq/src/test/frontend/worker/db_worker_node_test.cljs` that help output documents `--create-empty-db` as a startup option. +9. Run `bb dev:test -v logseq.db-worker.daemon-test` and `bb dev:test -v frontend.worker.db-worker-node-test` and confirm RED. + +### Phase 2. GREEN implementation for daemon flag plumbing. + +10. Update `/Users/rcmerci/gh-repos/logseq/src/main/logseq/db_worker/daemon.cljs` so `spawn-server!` accepts `:create-empty-db?` and conditionally appends `--create-empty-db`. +11. Keep process scanning in `/Users/rcmerci/gh-repos/logseq/src/main/logseq/db_worker/daemon.cljs` compatible by ignoring unknown flags as today. +12. Update `/Users/rcmerci/gh-repos/logseq/src/main/frontend/worker/db_worker_node.cljs` `parse-args` to set `:create-empty-db? true` when `--create-empty-db` is present. +13. Update `/Users/rcmerci/gh-repos/logseq/src/main/frontend/worker/db_worker_node.cljs` `show-help!` to include one line for `--create-empty-db`. +14. Re-run `bb dev:test -v logseq.db-worker.daemon-test` and `bb dev:test -v frontend.worker.db-worker-node-test` and confirm green. + +### Phase 3. RED tests for sync download orchestration. + +15. Add a failing test in `/Users/rcmerci/gh-repos/logseq/src/test/logseq/cli/command/sync_test.cljs` that `:sync-download` calls `cli-server/ensure-server!` with `:create-empty-db? true`. +16. Add a failing test in `/Users/rcmerci/gh-repos/logseq/src/test/logseq/cli/server_test.cljs` that `spawn-server!` forwards `:create-empty-db?` from `ensure-server-started!` config when spawning a new daemon. +17. Add a failing test in `/Users/rcmerci/gh-repos/logseq/src/test/logseq/cli/commands_test.cljs` that `sync download` path keeps existing `graph-exists` guard unchanged while still enabling create-empty mode on success paths. +18. Add a failing test in `/Users/rcmerci/gh-repos/logseq/src/test/logseq/cli/command/sync_test.cljs` that `sync download` fails fast with a dedicated error when the graph DB is not empty. +19. Run `bb dev:test -v logseq.cli.command.sync-test` and `bb dev:test -v logseq.cli.server-test` and confirm RED. + +### Phase 4. GREEN implementation for sync download orchestration. + +20. Update `/Users/rcmerci/gh-repos/logseq/src/main/logseq/cli/command/sync.cljs` so the `:sync-download` execution path enriches config with `:create-empty-db? true` before `invoke-global` and `invoke-with-repo`. +21. Add a DB emptiness guard in `/Users/rcmerci/gh-repos/logseq/src/main/logseq/cli/command/sync.cljs` or worker thread-api path that explicitly validates target graph DB is empty before calling `:thread-api/db-sync-download-graph-by-id`. +22. Keep non-download sync actions in `/Users/rcmerci/gh-repos/logseq/src/main/logseq/cli/command/sync.cljs` unchanged so default startup remains backward compatible. +23. Update `/Users/rcmerci/gh-repos/logseq/src/main/logseq/cli/server.cljs` and its internal `spawn-server!` wrapper to pass through `:create-empty-db?` to daemon spawn. +24. Re-run `bb dev:test -v logseq.cli.command.sync-test` and `bb dev:test -v logseq.cli.server-test` and confirm green. + +### Phase 5. RED tests for startup create-or-open behavior. + +25. Add a failing test in `/Users/rcmerci/gh-repos/logseq/src/test/frontend/worker/db_worker_node_test.cljs` that startup invokes `thread-api/create-or-open-db` with `[repo {:datoms []}]` when `:create-empty-db? true`. +26. Add a failing test in `/Users/rcmerci/gh-repos/logseq/src/test/frontend/worker/db_worker_node_test.cljs` that default startup still invokes `[repo {}]` when flag is not set. +27. Add a failing behavior test in `/Users/rcmerci/gh-repos/logseq/src/test/frontend/worker/db_sync_test.cljs` or `/Users/rcmerci/gh-repos/logseq/src/test/frontend/worker/db_worker_node_test.cljs` validating download bootstrap path does not depend on locally generated initial data. +28. Add a failing behavior test validating non-empty graph DB state is rejected before import starts. +29. Run targeted tests and confirm RED failures are behavior-based. + +### Phase 6. GREEN implementation for empty-db startup. + +30. Update `/Users/rcmerci/gh-repos/logseq/src/main/frontend/worker/db_worker_node.cljs` `start-daemon!` to pass startup opts `{:datoms []}` to `:thread-api/create-or-open-db` only when `:create-empty-db? true`. +31. Keep lock lifecycle and health endpoints unchanged in `/Users/rcmerci/gh-repos/logseq/src/main/frontend/worker/db_worker_node.cljs`. +32. Add a tiny helper in `/Users/rcmerci/gh-repos/logseq/src/main/frontend/worker/db_worker_node.cljs` if needed to keep startup opts construction testable. +33. Implement or expose a reusable DB emptiness check in worker/CLI path and return a stable error code for non-empty DB. +34. Re-run `bb dev:test -v frontend.worker.db-worker-node-test` and the selected download behavior test namespace until green. + +### Phase 7. Docs and regression. + +35. Update `/Users/rcmerci/gh-repos/logseq/docs/cli/logseq-cli.md` to document that `sync download` starts db-worker in create-empty mode and rejects non-empty DB state. +36. Update `/Users/rcmerci/gh-repos/logseq/docs/developers/desktop-db-worker-node.md` with a note that create-empty mode is CLI download bootstrap behavior and does not change desktop default startup. +37. Run `bb dev:lint-and-test` from `/Users/rcmerci/gh-repos/logseq` and ensure full suite passes. +38. Run final review checklist in `/Users/rcmerci/gh-repos/logseq/prompts/review.md` before merge. + +## Edge cases to cover explicitly + +| Scenario | Expected behavior | +|---|---| +| `sync download` for a brand-new graph. | Daemon starts with `--create-empty-db` and startup `create-or-open-db` uses `{:datoms []}`. | +| `sync start`, `sync upload`, and graph CRUD commands. | No `--create-empty-db` flag is used, and existing startup behavior stays unchanged. | +| Existing local graph (`graph-exists`). | Command fails before daemon startup as it does today. | +| Existing ready daemon lock for the same repo. | Command must validate DB emptiness before download and fail fast when DB is not empty. | +| Missing remote graph. | `remote-graph-not-found` response remains unchanged, and no behavior regression in error shape occurs. | +| Flag typo or absent flag. | Startup remains backward compatible with default `create-or-open-db` opts `{}`. | + +## Verification command matrix + +| Command | Expected outcome | +|---|---| +| `bb dev:test -v logseq.db-worker.daemon-test` | Spawn arg and parser tests for `--create-empty-db` pass. | +| `bb dev:test -v frontend.worker.db-worker-node-test` | Startup invoke args and help text tests pass. | +| `bb dev:test -v logseq.cli.server-test` | CLI server pass-through tests for `:create-empty-db?` pass. | +| `bb dev:test -v logseq.cli.command.sync-test` | `sync download` ensures server with create-empty mode. | +| `bb dev:test -v logseq.cli.commands-test` | Existing `graph-exists` guard and command dispatch remain stable. | +| `bb dev:test -v logseq.cli.integration-test/test-cli-sync-download-and-start-readiness-with-mocked-sync` | Integration path remains green with download plus start readiness behavior. | +| `bb dev:lint-and-test` | Full lint and unit test suite exits with code `0`. | + +## Rollout notes + +This change is intentionally scoped to startup behavior for `sync download` and does not alter runtime sync protocol. + +If regressions appear, rollback is to remove `:create-empty-db?` wiring in sync command and keep daemon/parser support dormant until retried. + +## Testing Details + +Tests focus on externally visible behavior, namely which startup args are passed and whether startup create-or-open uses empty datoms in download bootstrap mode. + +Tests also verify `sync download` rejects non-empty graph DB state before any import side effect. + +Tests avoid asserting private implementation internals except where argument boundaries are the behavior contract. + +Integration coverage confirms existing sync download and sync start semantics remain stable after plumbing changes. + +## Implementation Details + +- Add `:create-empty-db?` option plumbing from sync command to CLI server spawn path. +- Add `--create-empty-db` process arg emission in daemon spawn helper. +- Parse `--create-empty-db` in db-worker-node entrypoint args. +- When create-empty is enabled, startup invoke payload uses `[repo {:datoms []}]`. +- Add a strict pre-download emptiness check and fail with a stable error code when DB is not empty. +- Keep default startup invoke payload `[repo {}]` for all other flows. +- Keep lock ownership and stale-lock cleanup behavior unchanged. +- Keep command-level `graph-exists` and `remote-graph-not-found` behaviors unchanged. +- Update CLI and developer docs with exact scope of the new flag. +- Run targeted tests first, then full `bb dev:lint-and-test`. +- Use `@test-driven-development` workflow and `@clojure-debug` when failures are non-obvious. + +## Question + +No open question. + +--- diff --git a/docs/cli/logseq-cli.md b/docs/cli/logseq-cli.md index 7f7c908c40..16b4deb0c1 100644 --- a/docs/cli/logseq-cli.md +++ b/docs/cli/logseq-cli.md @@ -107,6 +107,8 @@ Sync download behavior: - `sync download` requires `--graph `. - If a local graph with the same name already exists, the CLI returns `graph-exists`. - If no remote graph with that name exists, the CLI returns `remote-graph-not-found`. +- `sync download` starts `db-worker-node` in create-empty mode so local startup does not write `db-initial-data` before snapshot import. +- If the target graph DB is not empty at download time, the CLI returns `graph-db-not-empty` and aborts before import. - For e2ee remote graphs in headless CLI mode, set `e2ee-password` via `sync config set` (or in `--config`) before download. Sync config persistence: diff --git a/src/main/frontend/worker/db_core.cljs b/src/main/frontend/worker/db_core.cljs index 9575537022..d9eb15f09e 100644 --- a/src/main/frontend/worker/db_core.cljs +++ b/src/main/frontend/worker/db_core.cljs @@ -332,6 +332,7 @@ client-ops-conn (when-not @*publishing? (common-sqlite/get-storage-conn client-ops-storage client-op/schema-in-db)) + empty-startup? (and (some? datoms) (empty? datoms)) initial-data-exists? (when (nil? datoms) (and (d/entity @conn :logseq.class/Root) (= "db" (:kv/value (d/entity @conn :logseq.kv/db-type)))))] @@ -347,7 +348,8 @@ (gc-sqlite-dbs! db client-ops-db conn {}) - (db-migrate/migrate conn) + (when-not empty-startup? + (db-migrate/migrate conn)) (db-listener/listen-db-changes! repo (get @*datascript-conns repo)))))) diff --git a/src/main/frontend/worker/db_worker_node.cljs b/src/main/frontend/worker/db_worker_node.cljs index 58338f3b6d..47add71381 100644 --- a/src/main/frontend/worker/db_worker_node.cljs +++ b/src/main/frontend/worker/db_worker_node.cljs @@ -46,14 +46,15 @@ opts {}] (if (empty? args) opts - (let [[flag value & remaining] args] + (let [flag (first args)] (case flag - "--data-dir" (recur remaining (assoc opts :data-dir value)) - "--repo" (recur remaining (assoc opts :repo value)) - "--owner-source" (recur remaining (assoc opts :owner-source value)) - "--log-level" (recur remaining (assoc opts :log-level value)) - "--help" (recur remaining (assoc opts :help? true)) - (recur remaining opts)))))) + "--data-dir" (recur (subvec args 2) (assoc opts :data-dir (second args))) + "--repo" (recur (subvec args 2) (assoc opts :repo (second args))) + "--owner-source" (recur (subvec args 2) (assoc opts :owner-source (second args))) + "--log-level" (recur (subvec args 2) (assoc opts :log-level (second args))) + "--create-empty-db" (recur (subvec args 1) (assoc opts :create-empty-db? true)) + "--help" (recur (subvec args 1) (assoc opts :help? true)) + (recur (subvec args 1) opts)))))) (defn- normalize-owner-source [owner-source] @@ -283,9 +284,16 @@ (println (str (style/bold "db-worker-node") " " (style/bold "options") ":")) (println (str " " (style/bold "--data-dir") " (default ~/logseq/graphs)")) (println (str " " (style/bold "--repo") " (required)")) + (println (str " " (style/bold "--create-empty-db") " (start with empty initial datoms)")) (println (str " " (style/bold "--log-level") " (default info)")) (println " logs: //db-worker-node-YYYYMMDD.log (retains 7)")) +(defn- startup-db-opts + [{:keys [create-empty-db?]}] + (if create-empty-db? + {:datoms []} + {})) + (defn- pad2 [value] (if (< value 10) @@ -350,7 +358,7 @@ file-path)) (defn start-daemon! - [{:keys [data-dir repo log-level owner-source]}] + [{:keys [data-dir repo log-level owner-source] :as opts}] (let [host "127.0.0.1" port 0 owner-source (normalize-owner-source owner-source)] @@ -381,7 +389,7 @@ _ (reset! *lock-info {:path path :lock lock}) _ (let [method-kw :thread-api/create-or-open-db method-str (normalize-method-str method-kw)] - ( (p/let [{:keys [stop!] :as daemon} (start-daemon! {:data-dir data-dir :repo repo + :create-empty-db? (:create-empty-db? opts) :owner-source owner-source :log-level (:log-level opts)})] (log/info :db-worker-node-ready {:host (:host daemon) :port (:port daemon)}) diff --git a/src/main/logseq/cli/command/sync.cljs b/src/main/logseq/cli/command/sync.cljs index a265e20d01..3f34ae92fc 100644 --- a/src/main/logseq/cli/command/sync.cljs +++ b/src/main/logseq/cli/command/sync.cljs @@ -36,6 +36,16 @@ (def ^:private sync-start-skipped-states #{:inactive :stopped}) +(def ^:private sync-download-non-empty-query + '[:find (count ?e) . + :where + (or [?e :block/name] + [?e :block/page _] + [?e :block/parent _]) + (not [?e :logseq.property/built-in? true]) + (not [?e :db/ident]) + (not [?e :file/path])]) + (defn- missing-repo [label] {:ok? false @@ -169,30 +179,31 @@ :action {:type :sync-config-unset :config-key (:key key-result)}})) - {:ok? false + {:ok? false :error {:code :unknown-command :message (str "unknown sync command: " command)}})) +(defn- sync-config + [config] + {:ws-url (:ws-url config) + :http-base (:http-base config) + :auth-token (:auth-token config) + :e2ee-password (:e2ee-password config)}) + (defn- invoke-with-repo [config repo method args] - (let [sync-config {:ws-url (:ws-url config) - :http-base (:http-base config) - :auth-token (:auth-token config) - :e2ee-password (:e2ee-password config)}] + (let [sync-cfg (sync-config config)] (p/let [cfg (cli-server/ensure-server! config repo) - _ (transport/invoke cfg :thread-api/set-db-sync-config false [sync-config]) + _ (transport/invoke cfg :thread-api/set-db-sync-config false [sync-cfg]) result (transport/invoke cfg method false args)] result))) (defn- invoke-global [config method args] (let [base-url (:base-url config) - sync-config {:ws-url (:ws-url config) - :http-base (:http-base config) - :auth-token (:auth-token config) - :e2ee-password (:e2ee-password config)}] + sync-cfg (sync-config config)] (if (seq base-url) - (p/let [_ (transport/invoke config :thread-api/set-db-sync-config false [sync-config])] + (p/let [_ (transport/invoke config :thread-api/set-db-sync-config false [sync-cfg])] (transport/invoke config method false args)) (p/let [repo (or (core/resolve-repo (:graph config)) (p/let [graphs (cli-server/list-graphs config)] @@ -201,9 +212,24 @@ (cli-server/ensure-server! config repo) (p/rejected (ex-info "graph name is required" {:code :missing-graph}))) - _ (transport/invoke cfg :thread-api/set-db-sync-config false [sync-config])] + _ (transport/invoke cfg :thread-api/set-db-sync-config false [sync-cfg])] (transport/invoke cfg method false args))))) +(defn- download-config + [config] + (assoc config :create-empty-db? true)) + +(defn- ensure-empty-download-db! + [cfg repo] + (p/let [non-empty-entity-count (transport/invoke cfg :thread-api/q false [repo [sync-download-non-empty-query]])] + (when (and (number? non-empty-entity-count) + (pos? non-empty-entity-count)) + (throw (ex-info "graph db is not empty" + {:code :graph-db-not-empty + :repo repo + :non-empty-entity-count non-empty-entity-count}))) + nil)) + (defn- wait-sync-start-ready [config repo action] (let [timeout-ms (max 0 (or (:wait-timeout-ms action) sync-start-timeout-ms)) @@ -295,7 +321,8 @@ {:result result})}) :sync-download - (-> (p/let [remote-graphs (invoke-global config + (let [config' (download-config config)] + (-> (p/let [remote-graphs (invoke-global config' :thread-api/db-sync-list-remote-graphs []) remote-graph (some (fn [graph] @@ -307,8 +334,10 @@ :error {:code :remote-graph-not-found :message (str "remote graph not found: " (:graph action)) :graph (:graph action)}} - (p/let [result (invoke-with-repo config (:repo action) - :thread-api/db-sync-download-graph-by-id + (p/let [cfg (cli-server/ensure-server! config' (:repo action)) + _ (transport/invoke cfg :thread-api/set-db-sync-config false [(sync-config config')]) + _ (ensure-empty-download-db! cfg (:repo action)) + result (transport/invoke cfg :thread-api/db-sync-download-graph-by-id false [(:repo action) (:graph-id remote-graph) (:graph-e2ee? remote-graph)])] {:status :ok :data (if (map? result) @@ -316,7 +345,7 @@ {:result result})}))) (p/catch (fn [error] (exception->error error {:repo (:repo action) - :graph (:graph action)})))) + :graph (:graph action)}))))) :sync-remote-graphs (p/let [graphs (invoke-global config :thread-api/db-sync-list-remote-graphs [])] diff --git a/src/main/logseq/cli/server.cljs b/src/main/logseq/cli/server.cljs index 3b6f981e51..f5fb089003 100644 --- a/src/main/logseq/cli/server.cljs +++ b/src/main/logseq/cli/server.cljs @@ -130,11 +130,12 @@ (daemon/ready? lock)) (defn- spawn-server! - [{:keys [repo data-dir owner-source]}] + [{:keys [repo data-dir owner-source create-empty-db?]}] (daemon/spawn-server! {:script (db-worker-script-path) :repo repo :data-dir data-dir - :owner-source owner-source})) + :owner-source owner-source + :create-empty-db? create-empty-db?})) (defn- rewrite-lock-owner-source! [path lock owner-source] @@ -155,7 +156,8 @@ :data-dir data-dir}) (spawn-server! {:repo repo :data-dir data-dir - :owner-source requester-owner}) + :owner-source requester-owner + :create-empty-db? (:create-empty-db? config)}) (-> (wait-for-lock path) (p/catch (fn [e] (if (= :timeout (:code (ex-data e))) diff --git a/src/main/logseq/db_worker/daemon.cljs b/src/main/logseq/db_worker/daemon.cljs index 4885c2cce8..55f233dfbc 100644 --- a/src/main/logseq/db_worker/daemon.cljs +++ b/src/main/logseq/db_worker/daemon.cljs @@ -254,9 +254,10 @@ :interval-ms 250})) (defn spawn-server! - [{:keys [script repo data-dir owner-source]}] + [{:keys [script repo data-dir owner-source create-empty-db?]}] (let [owner-source (normalize-owner-source owner-source) - args #js [script "--repo" repo "--data-dir" data-dir "--owner-source" (name owner-source)] + args (clj->js (cond-> [script "--repo" repo "--data-dir" data-dir "--owner-source" (name owner-source)] + create-empty-db? (conj "--create-empty-db"))) env (js/Object.assign #js {} (.-env js/process) #js {:ELECTRON_RUN_AS_NODE "1"})] (if-not script (do diff --git a/src/test/frontend/worker/db_worker_node_test.cljs b/src/test/frontend/worker/db_worker_node_test.cljs index 225e680c53..dd4e30c4a2 100644 --- a/src/test/frontend/worker/db_worker_node_test.cljs +++ b/src/test/frontend/worker/db_worker_node_test.cljs @@ -5,11 +5,14 @@ [cljs.test :refer [async deftest is use-fixtures]] [clojure.string :as string] [frontend.test.node-helper :as node-helper] + [frontend.worker.db-core :as db-core] [frontend.worker.db-worker-node-lock :as db-lock] [frontend.worker.db-worker-node :as db-worker-node] + [frontend.worker.platform.node :as platform-node] [goog.object :as gobj] [logseq.cli.server :as cli-server] [logseq.cli.style :as style] + [logseq.common.config :as common-config] [logseq.db :as ldb] [promesa.core :as p])) @@ -249,6 +252,14 @@ (is (nil? (:rtc-ws-url result))) (is (= "logseq_db_parse_args" (:repo result))))) +(deftest db-worker-node-parse-args-recognizes-create-empty-db + (let [parse-args #'db-worker-node/parse-args + result (parse-args #js ["node" "dist/db-worker-node.js" + "--repo" "logseq_db_parse_args" + "--create-empty-db"])] + (is (= "logseq_db_parse_args" (:repo result))) + (is (= true (:create-empty-db? result))))) + (deftest db-worker-node-owner-source-cli-is-written-into-lock (async done (let [daemon (atom nil) @@ -323,9 +334,108 @@ (is (contains-bold? output "db-worker-node")) (is (contains-bold? output "--data-dir")) (is (contains-bold? output "--repo")) + (is (string/includes? plain-output "--create-empty-db")) + (is (contains-bold? output "--create-empty-db")) (is (not (contains-bold? output "--rtc-ws-url"))) (is (contains-bold? output "--log-level")))) +(deftest db-worker-node-start-daemon-uses-empty-datoms-when-create-empty-enabled + (async done + (let [data-dir (node-helper/create-tmp-dir "db-worker-create-empty-start") + repo (str "logseq_db_create_empty_start_" (subs (str (random-uuid)) 0 8)) + lock-file-path (lock-path data-dir repo) + invoke-calls (atom []) + original-platform platform-node/node-platform + original-init-core db-core/init-core! + original-ensure-lock db-lock/ensure-lock! + original-update-lock db-lock/update-lock!] + (set! platform-node/node-platform (fn [_opts] #js {})) + (set! db-core/init-core! + (fn [_platform] + #js {:remoteInvoke (fn [method direct-pass? args] + (swap! invoke-calls conj + [method + direct-pass? + (if direct-pass? + (vec (js->clj args)) + (ldb/read-transit-str args))]) + (p/resolved nil))})) + (set! db-lock/ensure-lock! + (fn [_] + (p/resolved {:path lock-file-path + :lock {:repo repo + :pid (.-pid js/process) + :host "127.0.0.1" + :port 0 + :lock-id "create-empty-lock"}}))) + (set! db-lock/update-lock! (fn [_path lock] lock)) + (-> (p/let [{:keys [stop!]} (db-worker-node/start-daemon! {:data-dir data-dir + :repo repo + :create-empty-db? true + :log-level "error"}) + _ (is (= ["thread-api/init" true []] + (first @invoke-calls))) + _ (is (= ["thread-api/create-or-open-db" false [repo {:datoms []}]] + (second @invoke-calls))) + _ (stop!)] + true) + (p/catch (fn [e] + (is false (str "unexpected error: " e)))) + (p/finally (fn [] + (set! platform-node/node-platform original-platform) + (set! db-core/init-core! original-init-core) + (set! db-lock/ensure-lock! original-ensure-lock) + (set! db-lock/update-lock! original-update-lock) + (done))))))) + +(deftest db-worker-node-start-daemon-uses-default-startup-opts-without-create-empty + (async done + (let [data-dir (node-helper/create-tmp-dir "db-worker-default-start") + repo (str "logseq_db_default_start_" (subs (str (random-uuid)) 0 8)) + lock-file-path (lock-path data-dir repo) + invoke-calls (atom []) + original-platform platform-node/node-platform + original-init-core db-core/init-core! + original-ensure-lock db-lock/ensure-lock! + original-update-lock db-lock/update-lock!] + (set! platform-node/node-platform (fn [_opts] #js {})) + (set! db-core/init-core! + (fn [_platform] + #js {:remoteInvoke (fn [method direct-pass? args] + (swap! invoke-calls conj + [method + direct-pass? + (if direct-pass? + (vec (js->clj args)) + (ldb/read-transit-str args))]) + (p/resolved nil))})) + (set! db-lock/ensure-lock! + (fn [_] + (p/resolved {:path lock-file-path + :lock {:repo repo + :pid (.-pid js/process) + :host "127.0.0.1" + :port 0 + :lock-id "default-lock"}}))) + (set! db-lock/update-lock! (fn [_path lock] lock)) + (-> (p/let [{:keys [stop!]} (db-worker-node/start-daemon! {:data-dir data-dir + :repo repo + :log-level "error"}) + _ (is (= ["thread-api/init" true []] + (first @invoke-calls))) + _ (is (= ["thread-api/create-or-open-db" false [repo {}]] + (second @invoke-calls))) + _ (stop!)] + true) + (p/catch (fn [e] + (is false (str "unexpected error: " e)))) + (p/finally (fn [] + (set! platform-node/node-platform original-platform) + (set! db-core/init-core! original-init-core) + (set! db-lock/ensure-lock! original-ensure-lock) + (set! db-lock/update-lock! original-update-lock) + (done))))))) + (deftest db-worker-node-repo-error-handles-keyword-methods (let [repo-error #'db-worker-node/repo-error bound-repo "logseq_db_bound"] @@ -370,6 +480,30 @@ (-> (stop!) (p/finally (fn [] (done)))) (done)))))))) +(deftest db-worker-node-create-empty-startup-skips-built-in-initial-data + (async done + (let [daemon (atom nil) + data-dir (node-helper/create-tmp-dir "db-worker-empty-initial-data") + repo (str "logseq_db_empty_initial_" (subs (str (random-uuid)) 0 8))] + (-> (p/let [{:keys [host port stop!]} + (start-daemon! {:data-dir data-dir + :repo repo + :create-empty-db? true}) + _ (reset! daemon {:stop! stop!}) + library-result (invoke host port "thread-api/q" + [repo + ['[:find ?e + :in $ ?title + :where [?e :block/title ?title]] + common-config/library-page-name]])] + (is (empty? library-result))) + (p/catch (fn [e] + (is false (str "unexpected error: " e)))) + (p/finally (fn [] + (if-let [stop! (:stop! @daemon)] + (-> (stop!) (p/finally (fn [] (done)))) + (done)))))))) + (deftest db-worker-node-sync-status-requires-repo-and-returns-structured-status (async done (let [daemon (atom nil) diff --git a/src/test/logseq/cli/command/sync_test.cljs b/src/test/logseq/cli/command/sync_test.cljs index ef4273564c..a8600182b8 100644 --- a/src/test/logseq/cli/command/sync_test.cljs +++ b/src/test/logseq/cli/command/sync_test.cljs @@ -261,6 +261,8 @@ (p/resolved [{:graph-id "remote-graph-id" :graph-name "demo" :graph-e2ee? true}]) + :thread-api/q + (p/resolved 0) :thread-api/db-sync-download-graph-by-id (p/resolved {:ok true}) (p/resolved nil)))) @@ -270,20 +272,28 @@ {:base-url "http://example" :data-dir "/tmp"})] (is (= [[{:base-url "http://example" + :create-empty-db? true :data-dir "/tmp"} "logseq_db_demo"]] @ensure-calls)) - (is (= [[:thread-api/set-db-sync-config false [{:ws-url nil - :http-base nil - :auth-token nil - :e2ee-password nil}]] - [:thread-api/db-sync-list-remote-graphs false []] - [:thread-api/set-db-sync-config false [{:ws-url nil - :http-base nil - :auth-token nil - :e2ee-password nil}]] - [:thread-api/db-sync-download-graph-by-id false ["logseq_db_demo" "remote-graph-id" true]]] - @invoke-calls))) + (is (= [:thread-api/set-db-sync-config false [{:ws-url nil + :http-base nil + :auth-token nil + :e2ee-password nil}]] + (nth @invoke-calls 0))) + (is (= [:thread-api/db-sync-list-remote-graphs false []] + (nth @invoke-calls 1))) + (is (= [:thread-api/set-db-sync-config false [{:ws-url nil + :http-base nil + :auth-token nil + :e2ee-password nil}]] + (nth @invoke-calls 2))) + (let [[method direct-pass? args] (nth @invoke-calls 3)] + (is (= :thread-api/q method)) + (is (= false direct-pass?)) + (is (= "logseq_db_demo" (first args)))) + (is (= [:thread-api/db-sync-download-graph-by-id false ["logseq_db_demo" "remote-graph-id" true]] + (nth @invoke-calls 4)))) (p/catch (fn [e] (is false (str "unexpected error: " e)))) (p/finally (fn [] @@ -307,6 +317,8 @@ (p/resolved [{:graph-id "remote-graph-id" :graph-name "demo" :graph-e2ee? false}]) + :thread-api/q + (p/resolved 0) :thread-api/db-sync-download-graph-by-id (p/resolved {:ok true}) (p/resolved nil)))) @@ -316,23 +328,32 @@ {:graph "demo" :data-dir "/tmp"})] (is (= [[{:graph "demo" + :create-empty-db? true :data-dir "/tmp"} "logseq_db_demo"] [{:graph "demo" + :create-empty-db? true :data-dir "/tmp"} "logseq_db_demo"]] @ensure-calls)) - (is (= [[:thread-api/set-db-sync-config false [{:ws-url nil - :http-base nil - :auth-token nil - :e2ee-password nil}]] - [:thread-api/db-sync-list-remote-graphs false []] - [:thread-api/set-db-sync-config false [{:ws-url nil - :http-base nil - :auth-token nil - :e2ee-password nil}]] - [:thread-api/db-sync-download-graph-by-id false ["logseq_db_demo" "remote-graph-id" false]]] - @invoke-calls))) + (is (= [:thread-api/set-db-sync-config false [{:ws-url nil + :http-base nil + :auth-token nil + :e2ee-password nil}]] + (nth @invoke-calls 0))) + (is (= [:thread-api/db-sync-list-remote-graphs false []] + (nth @invoke-calls 1))) + (is (= [:thread-api/set-db-sync-config false [{:ws-url nil + :http-base nil + :auth-token nil + :e2ee-password nil}]] + (nth @invoke-calls 2))) + (let [[method direct-pass? args] (nth @invoke-calls 3)] + (is (= :thread-api/q method)) + (is (= false direct-pass?)) + (is (= "logseq_db_demo" (first args)))) + (is (= [:thread-api/db-sync-download-graph-by-id false ["logseq_db_demo" "remote-graph-id" false]] + (nth @invoke-calls 4)))) (p/catch (fn [e] (is false (str "unexpected error: " e)))) (p/finally (fn [] @@ -388,6 +409,8 @@ (p/resolved [{:graph-id "remote-graph-id" :graph-name "demo" :graph-e2ee? true}]) + :thread-api/q + (p/resolved 0) :thread-api/db-sync-download-graph-by-id (p/rejected (ex-info "db-sync/incomplete-snapshot-frame" {:code :db-sync/incomplete-snapshot-frame @@ -407,6 +430,45 @@ (set! transport/invoke orig-invoke) (done))))))) +(deftest test-execute-sync-download-fails-fast-when-db-is-not-empty + (async done + (let [orig-ensure-server! cli-server/ensure-server! + orig-invoke transport/invoke + invoke-calls (atom [])] + (set! cli-server/ensure-server! (fn [config repo] + (p/resolved (assoc config + :repo repo + :base-url "http://example")))) + (set! transport/invoke (fn [_ method direct-pass? args] + (swap! invoke-calls conj [method direct-pass? args]) + (case method + :thread-api/db-sync-list-remote-graphs + (p/resolved [{:graph-id "remote-graph-id" + :graph-name "demo" + :graph-e2ee? true}]) + :thread-api/q + (p/resolved 2) + :thread-api/db-sync-download-graph-by-id + (p/resolved {:ok true}) + (p/resolved nil)))) + (-> (p/let [result (sync-command/execute {:type :sync-download + :repo "logseq_db_demo" + :graph "demo"} + {:data-dir "/tmp"})] + (is (= :error (:status result))) + (is (= :graph-db-not-empty (get-in result [:error :code]))) + (is (= "logseq_db_demo" (get-in result [:error :repo]))) + (is (= 2 (get-in result [:error :context :non-empty-entity-count]))) + (is (not-any? (fn [[method _ _]] + (= :thread-api/db-sync-download-graph-by-id method)) + @invoke-calls))) + (p/catch (fn [e] + (is false (str "unexpected error: " e)))) + (p/finally (fn [] + (set! cli-server/ensure-server! orig-ensure-server!) + (set! transport/invoke orig-invoke) + (done))))))) + (deftest test-execute-sync-remote-graphs (async done (let [orig-ensure-server! cli-server/ensure-server! diff --git a/src/test/logseq/cli/commands_test.cljs b/src/test/logseq/cli/commands_test.cljs index 44eacce467..65ec027e3a 100644 --- a/src/test/logseq/cli/commands_test.cljs +++ b/src/test/logseq/cli/commands_test.cljs @@ -3,6 +3,7 @@ [clojure.string :as string] [logseq.cli.command.add :as add-command] [logseq.cli.command.show :as show-command] + [logseq.cli.command.sync :as sync-command] [logseq.cli.commands :as commands] [logseq.cli.server :as cli-server] [logseq.cli.style :as style] @@ -2512,6 +2513,33 @@ (set! cli-server/ensure-server! orig-ensure-server!) (done))))))) +(deftest test-execute-sync-download-runs-command-when-graph-is-missing + (async done + (let [orig-list-graphs cli-server/list-graphs + orig-execute sync-command/execute + captured (atom nil)] + (set! cli-server/list-graphs (fn [_] [])) + (set! sync-command/execute + (fn [action config] + (reset! captured [action config]) + (p/resolved {:status :ok + :data {:repo (:repo action)}}))) + (-> (p/let [result (commands/execute {:type :sync-download + :repo "logseq_db_demo" + :graph "demo" + :allow-missing-graph true + :require-missing-graph true} + {:data-dir "/tmp"})] + (is (= :ok (:status result))) + (is (= "logseq_db_demo" (get-in result [:data :repo]))) + (is (= :sync-download (get-in @captured [0 :type])))) + (p/catch (fn [e] + (is false (str "unexpected error: " e)))) + (p/finally (fn [] + (set! cli-server/list-graphs orig-list-graphs) + (set! sync-command/execute orig-execute) + (done))))))) + (deftest test-execute-graph-export (async done (let [invoke-calls (atom []) diff --git a/src/test/logseq/cli/integration_test.cljs b/src/test/logseq/cli/integration_test.cljs index c97d69aea1..10d3a32cfa 100644 --- a/src/test/logseq/cli/integration_test.cljs +++ b/src/test/logseq/cli/integration_test.cljs @@ -237,7 +237,7 @@ :pending-server 0})) :thread-api/q - (p/resolved 3) + (p/resolved 0) (p/resolved nil)))) download-result (run-cli ["--graph" download-repo "sync" "download"] data-dir cfg-path) @@ -245,6 +245,7 @@ start-result (run-cli ["--graph" start-repo "sync" "start"] data-dir cfg-path) start-payload (parse-json-output-safe start-result "sync start")] (is (some #(= :thread-api/db-sync-download-graph-by-id (first %)) @invoke-calls)) + (is (some #(= :thread-api/q (first %)) @invoke-calls)) (is (some #(= :thread-api/db-sync-status (first %)) @invoke-calls)) (is (= 0 (:exit-code download-result))) (is (= "ok" (:status download-payload))) diff --git a/src/test/logseq/cli/server_test.cljs b/src/test/logseq/cli/server_test.cljs index 92918fb799..c659056625 100644 --- a/src/test/logseq/cli/server_test.cljs +++ b/src/test/logseq/cli/server_test.cljs @@ -259,3 +259,51 @@ (set! daemon/wait-for-lock original-wait-lock) (set! daemon/find-orphan-processes original-find-orphans) (done))))))) + +(deftest ensure-server-forwards-create-empty-db-flag-when-spawning-daemon + (async done + (let [data-dir (node-helper/create-tmp-dir "cli-server-create-empty") + repo (str "logseq_db_create_empty_" (subs (str (random-uuid)) 0 8)) + captured (atom nil) + lock {:repo repo + :pid (.-pid js/process) + :host "127.0.0.1" + :port 9301} + read-lock-calls (atom 0) + original-read-lock daemon/read-lock + original-cleanup-stale daemon/cleanup-stale-lock! + original-cleanup-orphans daemon/cleanup-orphan-processes! + original-spawn daemon/spawn-server! + original-wait-lock daemon/wait-for-lock + original-wait-ready daemon/wait-for-ready] + (set! daemon/read-lock + (fn [_] + (if (= 1 (swap! read-lock-calls inc)) + nil + lock))) + (set! daemon/cleanup-stale-lock! (fn [_ _] (p/resolved nil))) + (set! daemon/cleanup-orphan-processes! (fn [_] {:orphans [] :killed-pids []})) + (set! daemon/spawn-server! + (fn [opts] + (reset! captured opts) + nil)) + (set! daemon/wait-for-lock (fn [_] (p/resolved true))) + (set! daemon/wait-for-ready (fn [_] (p/resolved true))) + (-> (cli-server/ensure-server! {:data-dir data-dir + :create-empty-db? true} + repo) + (p/then (fn [_] + (is (= repo (:repo @captured))) + (is (= (cli-server/resolve-data-dir {:data-dir data-dir}) + (:data-dir @captured))) + (is (= true (:create-empty-db? @captured))))) + (p/catch (fn [e] + (is false (str "unexpected error: " e)))) + (p/finally (fn [] + (set! daemon/read-lock original-read-lock) + (set! daemon/cleanup-stale-lock! original-cleanup-stale) + (set! daemon/cleanup-orphan-processes! original-cleanup-orphans) + (set! daemon/spawn-server! original-spawn) + (set! daemon/wait-for-lock original-wait-lock) + (set! daemon/wait-for-ready original-wait-ready) + (done))))))) diff --git a/src/test/logseq/db_worker/daemon_test.cljs b/src/test/logseq/db_worker/daemon_test.cljs index 07242aa6e7..9093e88ac8 100644 --- a/src/test/logseq/db_worker/daemon_test.cljs +++ b/src/test/logseq/db_worker/daemon_test.cljs @@ -30,6 +30,45 @@ (finally (set! (.-spawn child-process) original-spawn))))) +(deftest spawn-server-appends-create-empty-db-flag-when-enabled + (let [captured (atom nil) + original-spawn (.-spawn child-process)] + (set! (.-spawn child-process) + (fn [cmd args opts] + (reset! captured {:cmd cmd + :args (vec (js->clj args)) + :opts (js->clj opts :keywordize-keys true)}) + (js-obj "unref" (fn [] nil)))) + (try + (daemon/spawn-server! {:script "/tmp/db-worker-node.js" + :repo "logseq_db_spawn_helper_test" + :data-dir "/tmp/logseq-db-worker" + :create-empty-db? true}) + (is (some #{"--create-empty-db"} (:args @captured))) + (finally + (set! (.-spawn child-process) original-spawn))))) + +(deftest spawn-server-omits-create-empty-db-flag-by-default + (let [captured (atom []) + original-spawn (.-spawn child-process) + capture! (fn [_cmd args _opts] + (swap! captured conj (vec (js->clj args))) + (js-obj "unref" (fn [] nil)))] + (set! (.-spawn child-process) capture!) + (try + (daemon/spawn-server! {:script "/tmp/db-worker-node.js" + :repo "logseq_db_spawn_helper_test" + :data-dir "/tmp/logseq-db-worker"}) + (daemon/spawn-server! {:script "/tmp/db-worker-node.js" + :repo "logseq_db_spawn_helper_test" + :data-dir "/tmp/logseq-db-worker" + :create-empty-db? false}) + (is (every? (fn [args] + (not-any? #{"--create-empty-db"} args)) + @captured)) + (finally + (set! (.-spawn child-process) original-spawn))))) + (deftest cleanup-stale-lock-removes-invalid-lock (async done (let [data-dir (node-helper/create-tmp-dir "db-worker-daemon-helper")