From 3f5b52cc070d434c8f98143f2e19bc7723447e12 Mon Sep 17 00:00:00 2001 From: rcmerci Date: Wed, 13 May 2026 16:26:39 +0800 Subject: [PATCH] feat(cli): unify backup impl for desktop and cli --- .../db-worker/005-unified-graph-backup.md | 477 ++++++++++++++++++ docs/cli/logseq-cli.md | 1 + src/electron/electron/db.cljs | 71 ++- src/main/frontend/config.cljs | 2 +- src/main/logseq/cli/command/graph.cljs | 128 +---- src/main/logseq/cli/format.cljs | 20 +- src/main/logseq/db_worker/graph_backup.cljs | 299 +++++++++++ src/test/electron/db_test.cljs | 175 +++++-- src/test/frontend/config_test.cljs | 5 + src/test/logseq/cli/command/graph_test.cljs | 64 ++- src/test/logseq/cli/format_test.cljs | 20 +- .../logseq/db_worker/graph_backup_test.cljs | 266 ++++++++++ 12 files changed, 1335 insertions(+), 193 deletions(-) create mode 100644 docs/agent-guide/db-worker/005-unified-graph-backup.md create mode 100644 src/main/logseq/db_worker/graph_backup.cljs create mode 100644 src/test/logseq/db_worker/graph_backup_test.cljs diff --git a/docs/agent-guide/db-worker/005-unified-graph-backup.md b/docs/agent-guide/db-worker/005-unified-graph-backup.md new file mode 100644 index 0000000000..a559073bb6 --- /dev/null +++ b/docs/agent-guide/db-worker/005-unified-graph-backup.md @@ -0,0 +1,477 @@ +# Unified Graph Backup Implementation Plan + +Goal: Unify Desktop app and `logseq-cli` graph backup behavior around one shared db-worker-node backup implementation. + +Architecture: Keep the live SQLite snapshot operation inside the existing db-worker-node runtime and continue to use `:thread-api/backup-db-sqlite`. +Architecture: Move backup naming, paths, target selection, metadata, listing, deletion, and retention into one shared Node-only namespace that both Electron main process code and CLI command code call. +Architecture: Do not add a new `thread-api` unless implementation proves that the existing `:thread-api/backup-db-sqlite` and `:thread-api/import-db-binary` APIs cannot preserve the required behavior. + +Tech Stack: ClojureScript, Node.js `fs` and `path`, Promesa, existing db-worker-node HTTP invoke bridge, existing `logseq.cli.server`, existing `electron.db-worker`, existing `logseq.db.sqlite.backup`. + +Related: Builds on `docs/agent-guide/db-worker/001-db-worker-node-restart-on-version-mismatch.md`. +Related: Relates to `docs/agent-guide/db-worker/002-desktop-db-worker-request-cap-switch-graph.md`. +Related: Relates to `docs/agent-guide/db-worker/003-server-list-write-lock.md`. +Related: Relates to `docs/agent-guide/074-db-worker-node-invoke-main-thread-refactor.md`. +Related: Relates to `docs/cli/logseq-cli.md`. + +## Problem statement + +Desktop and CLI already share the db-worker-node SQLite snapshot primitive, but they do not share graph backup orchestration. + +The db-worker-node runtime exposes `:thread-api/backup-db-sqlite` from `/Users/rcmerci/gh-repos/logseq/src/main/frontend/worker/db_core.cljs`. + +The db-worker-node HTTP server marks `:thread-api/backup-db-sqlite` as a write method in `/Users/rcmerci/gh-repos/logseq/src/main/frontend/worker/db_worker_node.cljs`, so it already validates the bound repo and the db-worker-node write lock before allowing the backup. + +The Node platform adapter in `/Users/rcmerci/gh-repos/logseq/src/main/frontend/worker/platform/node.cljs` implements the actual snapshot by calling the SQLite connection backup operation. + +Desktop graph backups currently live in `/Users/rcmerci/gh-repos/logseq/src/electron/electron/db.cljs`. + +Desktop uses `electron.db/backup-db-via-worker!` to ask db-worker-node to copy the live database to a temporary path, then reads that temporary file back into Electron and writes a timestamped `.sqlite` file through `/Users/rcmerci/gh-repos/logseq/src/electron/electron/backup_file.cljs`. + +Desktop stores graph backups under `//backups`. + +Desktop retains 12 versions and skips non-forced backups when the newest backup is less than one hour old. + +CLI graph backups currently live in `/Users/rcmerci/gh-repos/logseq/src/main/logseq/cli/command/graph.cljs`. + +CLI stores graph backups under `/graphs//backup//db.sqlite`. + +CLI can list, create, restore, and remove backups. + +CLI backup creation already invokes the existing `:thread-api/backup-db-sqlite` method directly against the final backup `db.sqlite` path. + +The two implementations now duplicate backup path and retention responsibilities in different places and use different filesystem layouts. + +This means Desktop automatic backups cannot be listed, restored, or removed by the CLI graph backup commands. + +This also means future backup behavior fixes must be made in multiple places. + +## Testing Plan + +I will use @Test-Driven Development (TDD) before changing implementation behavior. + +I will add a new unit test namespace at `/Users/rcmerci/gh-repos/logseq/src/test/logseq/db_worker/graph_backup_test.cljs`. + +I will test that the shared backup namespace derives backup roots as `//backup`. + +I will test that the shared backup namespace writes backup directories as `//db.sqlite`. + +I will test that backup names preserve the current CLI shape for graph backup create, including optional labels and UTC timestamps. + +I will test that list returns only backups that contain a `db.sqlite` file and ignores incomplete temporary directories. + +I will test that create calls a caller-provided snapshot function exactly once with a temporary SQLite destination and publishes the final backup only after the snapshot succeeds. + +I will test that a failed snapshot removes temporary files and leaves no listable backup. + +I will test that explicit CLI backups do not apply Desktop automatic backup throttling or retention. + +I will test that Desktop automatic backups skip creation when the latest explicit Desktop automatic backup is newer than one hour and `force-backup?` is false. + +I will test that Desktop automatic backup retention prunes only backups whose metadata explicitly identifies them as Desktop automatic backups. + +I will test that Desktop manual backups and CLI backups are not pruned by Desktop automatic retention. + +I will update `/Users/rcmerci/gh-repos/logseq/src/test/logseq/cli/command/graph_test.cljs` so CLI backup list, create, restore, and remove exercise the shared backup namespace behavior. + +I will update `/Users/rcmerci/gh-repos/logseq/src/test/electron/db_test.cljs` so Desktop graph backups use the same shared `backup` layout as CLI graph backups and still invoke db-worker-node through the existing `:thread-api/backup-db-sqlite`. + +I will update `/Users/rcmerci/gh-repos/logseq/src/test/frontend/persist_db_test.cljs` only if the renderer IPC contract changes. + +The preferred implementation keeps the renderer IPC contract stable, so existing renderer tests for `:db-export` should continue to pass without broad changes. + +I will update `/Users/rcmerci/gh-repos/logseq/src/test/frontend/handler/export_test.cljs` only if the visible SQLite export flow changes. + +The preferred implementation leaves `:db-export-as` as a direct SQLite export and not a graph backup. + +I will run focused RED tests before implementation and confirm they fail because the shared namespace does not exist or current Electron tests still expect the old `backups` layout. + +I will run focused tests after implementation with commands such as `bb dev:test -v logseq.db-worker.graph-backup-test`, `bb dev:test -v logseq.cli.command.graph-test`, and `bb dev:test -v electron.db-test`. + +I will run `bb dev:lint-and-test` after focused tests pass. + +I will run CLI E2E non-sync coverage after the unit suite passes with `bb -f cli-e2e/bb.edn build` followed by `bb -f cli-e2e/bb.edn test --skip-build`. + +NOTE: I will write *all* tests before I add any implementation behavior. + +## Current implementation snapshot + +| Concern | Current location | Current behavior | Unification issue | +|---------|------------------|------------------|-------------------| +| db-worker-node snapshot API | `/Users/rcmerci/gh-repos/logseq/src/main/frontend/worker/db_core.cljs` | `:thread-api/backup-db-sqlite` checkpoints the graph DB and calls the platform SQLite backup function. | This is already the right shared snapshot primitive. | +| db-worker-node HTTP boundary | `/Users/rcmerci/gh-repos/logseq/src/main/frontend/worker/db_worker_node.cljs` | `/v1/invoke` validates repo, write methods, and db-worker-node lock ownership. | No new HTTP endpoint or thread API should be needed. | +| Node SQLite backup | `/Users/rcmerci/gh-repos/logseq/src/main/frontend/worker/platform/node.cljs` | The Node platform calls `.backup` on the SQLite connection. | This should stay behind `:thread-api/backup-db-sqlite`. | +| Desktop backup orchestration | `/Users/rcmerci/gh-repos/logseq/src/electron/electron/db.cljs` | Copies through db-worker-node into a temp file, reads bytes into Electron, then writes through `electron.backup-file/backup-file`. | This duplicates retention and writes a second copy from Electron. | +| Desktop backup file helper | `/Users/rcmerci/gh-repos/logseq/src/electron/electron/backup_file.cljs` | Writes timestamped files under `backups` and prunes old versions. | This helper is still used by file write failure backup and should not own graph backup policy. | +| Desktop IPC | `/Users/rcmerci/gh-repos/logseq/src/electron/electron/handler.cljs` | `:db-export` calls `backup-db-via-worker!` and `:db-export-as` calls direct SQLite export. | `:db-export` should use shared graph backup orchestration, while `:db-export-as` remains an export. | +| Desktop open backup folder | `/Users/rcmerci/gh-repos/logseq/src/main/frontend/config.cljs` | `get-electron-backup-dir` points to `/backups`. | It should point to the unified `/backup` directory after graph backup unification. | +| CLI backup orchestration | `/Users/rcmerci/gh-repos/logseq/src/main/logseq/cli/command/graph.cljs` | Owns backup path helpers, list, target selection, create, restore, and remove. | These helpers should move to the shared backup namespace. | +| CLI docs | `/Users/rcmerci/gh-repos/logseq/docs/cli/logseq-cli.md` | Documents CLI backup snapshots under `/graphs//backup`. | The documented CLI layout should become the Desktop graph backup layout too. | + +## Target behavior + +One Node-only namespace owns graph backup filesystem behavior. + +Both Desktop and CLI use that namespace for graph backup path derivation, backup name creation, list, create, remove, metadata, and retention. + +The canonical graph backup root is `//backup`. + +Each listable backup is stored as `//db.sqlite`. + +The shared namespace must use `logseq.common.graph-dir` path encoding instead of ad hoc string replacement. + +The shared namespace must accept a caller-provided snapshot function so it does not depend on Electron windows, CLI config, or HTTP transport. + +Desktop snapshot creation supplies a snapshot function that ensures the db-worker-node runtime through `electron.db-worker/ensure-runtime!` and invokes `:thread-api/backup-db-sqlite`. + +CLI snapshot creation supplies a snapshot function that ensures the db-worker-node runtime through `logseq.cli.server/ensure-server!` and invokes `:thread-api/backup-db-sqlite`. + +Restore continues to use the existing CLI SQLite import flow and `:thread-api/import-db-binary`. + +Manual SQLite export through `:db-export-as` remains an export flow and does not become a graph backup. + +`electron.backup-file` remains available for non-graph file write failure backups, but graph backups no longer depend on it. + +No new `thread-api` should be added for this change. + +No browser renderer code should import the shared Node-only backup namespace. + +Desktop automatic backup retention must not remove CLI-created backups. + +Desktop automatic backup retention must not remove Desktop manual backups. + +Missing or invalid backup names at command boundaries should return actionable errors instead of silently picking defaults. + +## Proposed architecture + +The shared namespace should be `/Users/rcmerci/gh-repos/logseq/src/main/logseq/db_worker/graph_backup.cljs`. + +The namespace should be Node-only because it uses Node `fs` and `path`. + +The namespace should not require `electron.*` namespaces. + +The namespace should not require `logseq.cli.server`. + +The namespace should not call `transport/invoke`. + +The namespace should accept `graphs-dir`, `repo`, `backup-name`, `source`, and `snapshot!` from callers. + +The shared namespace should write optional metadata next to each backup. + +The metadata file should be used for retention filtering and debugging, not for the CLI output contract unless a later product decision asks for that. + +The metadata file should be named `metadata.edn`. + +The metadata should include `:schema-version`, `:name`, `:repo`, `:source`, `:created-at-ms`, and `:db-path`. + +The shared namespace should only prune backups that explicitly opt into the retention source being pruned. + +Missing metadata must not make a backup eligible for Desktop automatic retention. + +The implementation should keep the current CLI backup output fields unless a test proves an output update is required. + +```text +CLI command code + | + | ensures db-worker-node with logseq.cli.server + | passes snapshot! to shared graph backup namespace + v +logseq.db-worker.graph-backup + | + | owns backup path, create, list, remove, retention + | calls snapshot! with a temp sqlite path + v +db-worker-node /v1/invoke + | + | existing :thread-api/backup-db-sqlite + v +live SQLite connection backup + +Electron main process + | + | ensures db-worker-node with electron.db-worker + | passes snapshot! to shared graph backup namespace + v +same shared graph backup namespace +``` + +## Shared namespace responsibilities + +Add `/Users/rcmerci/gh-repos/logseq/src/main/logseq/db_worker/graph_backup.cljs`. + +Add `backup-root-path` to return `//backup`. + +Add `backup-dir-name` to encode a backup name with `logseq.common.graph-dir/graph-dir-key->encoded-dir-name`. + +Add `backup-dir-path` to return `/`. + +Add `backup-db-path` to return `/db.sqlite`. + +Add `backup-metadata-path` to return `/metadata.edn`. + +Add `build-backup-name` to preserve the current CLI naming shape. + +Add `next-backup-target` to find a non-existing target and append numeric suffixes when needed. + +Add `list-backups` to list only directories with a real `db.sqlite`. + +Add `/backup`. + +Do not add new visible UI text for the folder change unless product wants a migration note. + +If visible UI text is added or changed, load @logseq-i18n first and follow `/Users/rcmerci/gh-repos/logseq/.agents/skills/logseq-i18n/SKILL.md`. + +## CLI integration plan + +Update `/Users/rcmerci/gh-repos/logseq/src/main/logseq/cli/command/graph.cljs`. + +Before editing this file, read `/Users/rcmerci/gh-repos/logseq/src/main/logseq/cli/AGENTS.md`. + +Remove private backup path helpers from the CLI command namespace once the shared namespace owns them. + +Keep CLI parsing, validation, action construction, and presentation contracts in the CLI command namespaces. + +Keep the current `graph backup create` backup name shape by delegating name construction to the shared namespace. + +Change `execute-graph-backup-list` to call the shared `list-backups`. + +Change `execute-graph-backup-create` to call the shared `//backup//db.sqlite` as the canonical layout. +- Use `metadata.edn` only for retention filtering and diagnostics in the first implementation. +- Stop using `electron.backup-file/backup-file` for graph backups. +- Keep `electron.backup-file/backup-file` for non-graph file write failure backups. +- Keep `:db-export-as` as a direct export flow, not a graph backup flow. +- Preserve CLI machine output unless product explicitly wants metadata exposed. +- Run @logseq-review-workflow after implementation and before considering the work complete. + +## Question + +There are no open product questions after review. + +Do not migrate or read old Desktop backups under `/backups`. + +Do not add compatibility for the old Desktop graph backup layout. + +Keep CLI JSON and EDN output stable and do not expose backup metadata such as `:source`. + +Treat Desktop `force-backup? true` as a manual backup. + +Protect Desktop manual backups from automatic throttling and automatic retention. + +Prune only backups whose metadata explicitly marks them as `:electron-auto`. + +Keep the shared backup namespace Node-only. + +Do not add a new `thread-api` for this implementation. + +--- diff --git a/docs/cli/logseq-cli.md b/docs/cli/logseq-cli.md index d86b91cf6d..f109046672 100644 --- a/docs/cli/logseq-cli.md +++ b/docs/cli/logseq-cli.md @@ -135,6 +135,7 @@ For any command that requires `--graph`, if the target graph does not exist, the Backup scope note: - `graph backup create` copies only `db.sqlite`. - `search-db.sqlite` and `client-ops-db.sqlite` are intentionally excluded. +- Desktop graph backups use the same `/graphs//backup//db.sqlite` layout, so the CLI can list, restore, and remove Desktop-created graph backups when it uses the same root directory. Server commands: - `server list` - list running db-worker-node servers diff --git a/src/electron/electron/db.cljs b/src/electron/electron/db.cljs index 07c0ab5e71..6c6717c439 100644 --- a/src/electron/electron/db.cljs +++ b/src/electron/electron/db.cljs @@ -2,7 +2,6 @@ "Provides SQLite dbs for electron and manages files of those dbs" (:require ["fs-extra" :as fs] ["path" :as node-path] - [electron.backup-file :as backup-file] [electron.db-worker :as db-worker] [lambdaisland.glogi :as log] [logseq.cli.common.graph :as cli-common-graph] @@ -10,11 +9,14 @@ [logseq.common.graph-dir :as graph-dir] [logseq.db.common.sqlite :as common-sqlite] [logseq.db.sqlite.backup :as sqlite-backup] + [logseq.db-worker.graph-backup :as graph-backup] [promesa.core :as p])) (def ^:private backup-interval-ms (* 60 60 1000)) +(def ^:private automatic-backup-keep-versions 12) + (defonce *auto-backup (atom {:window->repo {} :interval-id nil})) @@ -38,36 +40,34 @@ (when (fs/existsSync db-path) (fs/readFileSync db-path)))) -(defn- temp-backup-path - [backups-path] - (node-path/join backups-path - (str ".tmp." - (.now js/Date) - "." - (rand-int 1000000) - ".sqlite"))) +(defn- backup-source + [{:keys [force-backup?]}] + (if (true? force-backup?) + :electron-manual + :electron-auto)) + +(defn- {:graphs-dir (cli-common-graph/get-db-graphs-dir) + :repo db-name + :backup-name (graph-backup/build-backup-name db-name nil) + :source source + :snapshot! snapshot!} + (= :electron-auto source) + (assoc :throttle-ms backup-interval-ms + :keep-versions automatic-backup-keep-versions))))) (defn backup-db-with-sqlite-backup! [db-name {:keys [force-backup? sqlite-backup!]}] - (let [_ (ensure-graph-dir! db-name) - [_db-name db-path] (common-sqlite/get-db-full-path (cli-common-graph/get-db-graphs-dir) db-name) - backups-path (common-sqlite/get-db-backups-path (cli-common-graph/get-db-graphs-dir) db-name)] - (when (fs/existsSync db-path) - (let [tmp-path (temp-backup-path backups-path)] - (-> (p/let [_ (fs/ensureDirSync backups-path) - _ (sqlite-backup! db-path tmp-path) - payload (fs/readFileSync tmp-path)] - (backup-file/backup-file db-name nil nil - ".sqlite" - payload - :backups-dir backups-path - :keep-versions 12 - :force-backup? force-backup?)) - (p/finally (fn [] - (try - (fs/removeSync tmp-path) - (catch :default _ - nil))))))))) + (let [[_db-name db-path] (common-sqlite/get-db-full-path (cli-common-graph/get-db-graphs-dir) db-name)] + ( value str string/trim not-empty)) @@ -211,10 +192,7 @@ :error {:code :missing-repo :message "repo is required for backup create"}} (let [graph (core/repo->graph repo) - name-part (trimmed-option name) - backup-name (if (seq name-part) - (str graph "-" name-part "-" (utc-timestamp)) - (str graph "-" (utc-timestamp)))] + backup-name (graph-backup/build-backup-name repo name)] {:ok? true :action {:type :graph-backup-create :command :graph-backup-create @@ -369,13 +347,6 @@ :legacy-undecodable (:legacy-dir item) nil)) -(defn- backup-root-path - [config repo] - (when-let [graph-dir-name (graph-dir/repo->encoded-graph-dir-name repo)] - (node-path/join (cli-server/graphs-dir config) - graph-dir-name - backup-root-dir-name))) - (defn- export-root-path [config repo] (when-let [graph-dir-name (graph-dir/repo->encoded-graph-dir-name repo)] @@ -393,61 +364,6 @@ (quot (.now js/Date) 1000) ".sqlite")))) -(defn- backup-dir-name - [backup-name] - (graph-dir/graph-dir-key->encoded-dir-name backup-name)) - -(defn- backup-dir-path - [config repo backup-name] - (some->> (backup-dir-name backup-name) - (node-path/join (backup-root-path config repo)))) - -(defn- backup-db-path - [config repo backup-name] - (some-> (backup-dir-path config repo backup-name) - (node-path/join backup-db-file-name))) - -(defn- backup-metadata - [^js dirent root-path] - (let [dir-name (.-name dirent) - backup-name (graph-dir/decode-graph-dir-name dir-name) - db-path (node-path/join root-path dir-name backup-db-file-name)] - (when (and (seq backup-name) - (fs/existsSync db-path)) - (let [stat (fs/statSync db-path)] - (when (.isFile stat) - {:name backup-name - :created-at (.-mtimeMs stat) - :size-bytes (.-size stat)}))))) - -(defn- list-backups - [config repo] - (if-let [root-path (backup-root-path config repo)] - (let [dirents (if (fs/existsSync root-path) - (fs/readdirSync root-path #js {:withFileTypes true}) - #js [])] - (->> dirents - (filter #(.isDirectory ^js %)) - (keep #(backup-metadata % root-path)) - (sort-by (juxt :name :created-at)) - vec)) - [])) - -(defn- next-backup-target - [config repo base-name] - (loop [suffix 0] - (let [backup-name (if (zero? suffix) - base-name - (str base-name "-" suffix)) - dir-path (backup-dir-path config repo backup-name)] - (if (and (seq dir-path) - (fs/existsSync dir-path)) - (recur (inc suffix)) - {:backup-name backup-name - :dir-path dir-path - :db-path (when (seq dir-path) - (node-path/join dir-path backup-db-file-name))})))) - (defn execute-graph-list [_action config] (let [graph-items (vec (cli-server/list-graph-items config)) @@ -466,30 +382,33 @@ (defn execute-graph-backup-list [action config] {:status :ok - :data {:backups (list-backups config (:repo action))}}) + :data {:backups (graph-backup/list-backups (cli-server/graphs-dir config) + (:repo action))}}) (defn execute-graph-backup-create [action config] (p/let [cfg (cli-server/ensure-server! config (:repo action)) - {:keys [backup-name dir-path db-path]} (next-backup-target config (:repo action) (:backup-name action)) - _ (when-not (seq dir-path) - (throw (ex-info "invalid backup target path" - {:code :invalid-backup-path - :backup-name backup-name}))) - _ (fs/mkdirSync dir-path #js {:recursive true}) - _ (transport/invoke cfg - :thread-api/backup-db-sqlite - [(:repo action) db-path])] + {:keys [backup-name path]} (graph-backup/encoded-graph-dir-name repo) + "encoded graph directory") + "graph directory") + backup-root-dir-name)) + +(defn backup-dir-name + [backup-name] + (required! backup-name "backup-name") + (required! (graph-dir/graph-dir-key->encoded-dir-name backup-name) + "encoded backup directory")) + +(defn backup-dir-path + [graphs-dir repo backup-name] + (child-path! (backup-root-path graphs-dir repo) + (backup-dir-name backup-name) + "backup directory")) + +(defn backup-db-path + [graphs-dir repo backup-name] + (node-path/join (backup-dir-path graphs-dir repo backup-name) + backup-db-file-name)) + +(defn backup-metadata-path + [graphs-dir repo backup-name] + (node-path/join (backup-dir-path graphs-dir repo backup-name) + backup-metadata-file-name)) + +(defn- pad2 + [value] + (if (< value 10) + (str "0" value) + (str value))) + +(defn- utc-timestamp + [] + (let [now (js/Date.)] + (str (.getUTCFullYear now) + (pad2 (inc (.getUTCMonth now))) + (pad2 (.getUTCDate now)) + "T" + (pad2 (.getUTCHours now)) + (pad2 (.getUTCMinutes now)) + (pad2 (.getUTCSeconds now)) + "Z"))) + +(defn- trimmed-option + [value] + (some-> value str string/trim not-empty)) + +(defn build-backup-name + ([repo label] + (build-backup-name repo label (utc-timestamp))) + ([repo label timestamp] + (let [graph-name (required! (graph-dir/repo->graph-dir-key repo) "graph name") + label-part (trimmed-option label)] + (if (seq label-part) + (str graph-name "-" label-part "-" timestamp) + (str graph-name "-" timestamp))))) + +(defn next-backup-target + [graphs-dir repo base-name] + (required! base-name "backup-name") + (loop [suffix 0] + (let [backup-name (if (zero? suffix) + base-name + (str base-name "-" suffix)) + dir-path (backup-dir-path graphs-dir repo backup-name) + db-path (node-path/join dir-path backup-db-file-name)] + (if (fs/existsSync dir-path) + (recur (inc suffix)) + {:backup-name backup-name + :dir-path dir-path + :db-path db-path})))) + +(defn- stat-file + [file-path] + (try + (let [stat (fs/statSync file-path)] + (when (.isFile stat) + stat)) + (catch :default e + (when-not (= "ENOENT" (.-code e)) + (throw e))))) + +(defn- read-metadata + [metadata-path] + (try + (reader/read-string (.toString (fs/readFileSync metadata-path) "utf8")) + (catch :default _ + nil))) + +(defn- backup-entry + [root-path ^js dirent] + (let [dir-name (.-name dirent) + backup-name (graph-dir/decode-graph-dir-name dir-name) + dir-path (node-path/join root-path dir-name) + db-path (node-path/join dir-path backup-db-file-name)] + (when (seq backup-name) + (when-let [stat (stat-file db-path)] + {:name backup-name + :dir-path dir-path + :path db-path + :created-at (.-mtimeMs stat) + :size-bytes (.-size stat) + :metadata (read-metadata (node-path/join dir-path backup-metadata-file-name))})))) + +(defn- backup-entries + [graphs-dir repo] + (let [root-path (backup-root-path graphs-dir repo)] + (if (fs/existsSync root-path) + (->> (fs/readdirSync root-path #js {:withFileTypes true}) + (filter #(.isDirectory ^js %)) + (keep #(backup-entry root-path %)) + (sort-by (juxt :name :created-at)) + vec) + []))) + +(defn list-backups + [graphs-dir repo] + (->> (backup-entries graphs-dir repo) + (mapv (fn [{:keys [metadata] :as entry}] + (cond-> (select-keys entry [:name :created-at :size-bytes]) + (some? (:source metadata)) + (assoc :source (:source metadata))))))) + +(defn latest-backup-info + [graphs-dir repo source] + (->> (backup-entries graphs-dir repo) + (filter #(= source (get-in % [:metadata :source]))) + (sort-by #(get-in % [:metadata :created-at-ms]) >) + first)) + +(defn prune-backups! + [{:keys [graphs-dir repo source keep-versions]}] + (required! graphs-dir "graphs-dir") + (required! repo "repo") + (required! source "source") + (when-not (nat-int? keep-versions) + (throw (js/Error. "keep-versions must be a non-negative integer"))) + (let [to-remove (->> (backup-entries graphs-dir repo) + (filter #(= source (get-in % [:metadata :source]))) + (sort-by #(get-in % [:metadata :created-at-ms]) >) + (drop keep-versions) + vec)] + (doseq [{:keys [dir-path]} to-remove] + (fs/rmSync dir-path #js {:recursive true :force true})) + (mapv #(select-keys % [:name :dir-path :path]) to-remove))) + +(defn remove-backup! + [graphs-dir repo backup-name] + (let [dir-path (backup-dir-path graphs-dir repo backup-name)] + (if (fs/existsSync dir-path) + (do + (fs/rmSync dir-path #js {:recursive true :force true}) + true) + false))) + +(defn- reserve-next-backup-target! + [graphs-dir repo base-name] + (fs/mkdirSync (backup-root-path graphs-dir repo) #js {:recursive true}) + (loop [suffix 0] + (let [backup-name (if (zero? suffix) + base-name + (str base-name "-" suffix)) + dir-path (backup-dir-path graphs-dir repo backup-name) + result (try + (fs/mkdirSync dir-path) + {:backup-name backup-name + :dir-path dir-path + :db-path (node-path/join dir-path backup-db-file-name)} + (catch :default e + (if (= "EEXIST" (.-code e)) + ::retry + (throw e))))] + (if (= ::retry result) + (recur (inc suffix)) + result)))) + +(defn- cleanup-reserved-target! + [dir-path tmp-db-path db-path] + (when (and (seq tmp-db-path) + (fs/existsSync tmp-db-path)) + (fs/rmSync tmp-db-path #js {:force true})) + (when-not (fs/existsSync db-path) + (fs/rmSync dir-path #js {:recursive true :force true}))) + +(defn- write-metadata! + [{:keys [graphs-dir repo backup-name source created-at-ms db-path]}] + (fs/writeFileSync + (backup-metadata-path graphs-dir repo backup-name) + (pr-str {:schema-version metadata-schema-version + :name backup-name + :repo repo + :source source + :created-at-ms created-at-ms + :db-path db-path}) + "utf8")) + +(defn- throttled? + [{:keys [graphs-dir repo source throttle-ms now-ms]}] + (when (and (pos-int? throttle-ms) + (some? source)) + (when-let [latest (latest-backup-info graphs-dir repo source)] + (let [created-at-ms (get-in latest [:metadata :created-at-ms])] + (and (number? created-at-ms) + (< (- now-ms created-at-ms) throttle-ms)))))) + +(defn (p/let [_ (snapshot! tmp-db-path) + _ (when-not (stat-file tmp-db-path) + (throw (ex-info "snapshot did not create sqlite backup" + {:code :missing-snapshot + :path tmp-db-path}))) + _ (fs/renameSync tmp-db-path db-path) + _ (write-metadata! {:graphs-dir graphs-dir + :repo repo + :backup-name backup-name + :source source + :created-at-ms created-at-ms + :db-path db-path}) + _ (when (some? keep-versions) + (prune-backups! {:graphs-dir graphs-dir + :repo repo + :source source + :keep-versions keep-versions}))] + {:backup-name backup-name + :path db-path + :created? true}) + (p/catch (fn [e] + (cleanup-reserved-target! dir-path tmp-db-path db-path) + (throw e))))))) + (catch :default e + (p/rejected e)))) diff --git a/src/test/electron/db_test.cljs b/src/test/electron/db_test.cljs index 880111492b..b3f700c753 100644 --- a/src/test/electron/db_test.cljs +++ b/src/test/electron/db_test.cljs @@ -1,5 +1,6 @@ (ns electron.db-test (:require ["node:sqlite" :as node-sqlite] + [cljs.reader :as reader] [cljs.test :refer [async deftest is]] [electron.backup-file :as backup-file] [electron.db :as electron-db] @@ -10,6 +11,7 @@ [logseq.cli.transport :as cli-transport] [logseq.db.common.sqlite :as common-sqlite] [logseq.db.sqlite.backup :as sqlite-backup] + [logseq.db-worker.graph-backup :as graph-backup] [promesa.core :as p] ["fs-extra" :as fs] ["path" :as node-path])) @@ -42,6 +44,26 @@ (is (= "db-data" (.toString (electron-db/get-db db-name))))))) +(defn- read-edn-file + [file-path] + (reader/read-string (fs/readFileSync file-path "utf8"))) + +(defn- write-backup! + [graphs-dir repo backup-name source created-at-ms] + (let [db-path (graph-backup/backup-db-path graphs-dir repo backup-name) + metadata-path (graph-backup/backup-metadata-path graphs-dir repo backup-name)] + (fs/ensureDirSync (node-path/dirname db-path)) + (fs/writeFileSync db-path (str "sqlite-" backup-name) "utf8") + (fs/writeFileSync metadata-path + (pr-str {:schema-version 1 + :name backup-name + :repo repo + :source source + :created-at-ms created-at-ms + :db-path db-path}) + "utf8") + db-path)) + (deftest backup-db-creates-sqlite-copy-from-existing-disk-db (async done (let [graphs-dir (node-helper/create-tmp-dir "electron-db-backup") @@ -49,38 +71,33 @@ DatabaseSync (resolve-database-sync-ctor) original-get-db-graphs-dir cli-common-graph/get-db-graphs-dir] (set! cli-common-graph/get-db-graphs-dir (fn [] graphs-dir)) - (let [[_ db-path] (common-sqlite/get-db-full-path graphs-dir db-name) - backups-path (common-sqlite/get-db-backups-path graphs-dir db-name)] + (let [[_ db-path] (common-sqlite/get-db-full-path graphs-dir db-name)] (fs/ensureDirSync (node-path/dirname db-path)) (let [source-db (new DatabaseSync db-path)] (.exec source-db "create table kvs (addr text primary key, content text);") (.exec source-db "insert into kvs (addr, content) values ('a', 'alpha')") (.close source-db)) - (-> (p/with-redefs [backup-file/backup-file (fn [_repo _dir _relative-path _ext content & {:keys [backups-dir]}] - (fs/ensureDirSync backups-dir) - (fs/writeFileSync (node-path/join backups-dir "copy.sqlite") content))] - (p/let [_ (electron-db/backup-db! db-name nil) - backup-files (vec (fs/readdirSync backups-path)) - backup-path (node-path/join backups-path (first backup-files)) + (-> (p/let [_ (electron-db/backup-db! db-name nil) + backups (graph-backup/list-backups graphs-dir db-name) + backup-path (graph-backup/backup-db-path graphs-dir db-name (:name (first backups))) backup-db (new DatabaseSync backup-path) stmt (.prepare ^js backup-db "select addr, content from kvs order by addr") rows (.all stmt) _ (.close backup-db)] - (is (= 1 (count backup-files))) + (is (= 1 (count backups))) (is (= "a" (gobj/get (first rows) "addr"))) - (is (= "alpha" (gobj/get (first rows) "content"))))) + (is (= "alpha" (gobj/get (first rows) "content")))) (p/catch (fn [e] (is false (str "unexpected error: " e)))) (p/finally (fn [] (set! cli-common-graph/get-db-graphs-dir original-get-db-graphs-dir) (done)))))))) -(deftest backup-db-uses-backup-file-rules +(deftest backup-db-uses-shared-backup-layout-and-metadata (async done (let [graphs-dir (node-helper/create-tmp-dir "electron-db-backup-rules") db-name "logseq_db_demo" [_ db-path] (common-sqlite/get-db-full-path graphs-dir db-name) - backups-path (common-sqlite/get-db-backups-path graphs-dir db-name) sqlite-calls (atom []) backup-calls (atom [])] (fs/ensureDirSync (node-path/dirname db-path)) @@ -101,17 +118,16 @@ (p/let [_ (electron-db/backup-db! db-name nil)] (is (= 1 (count @sqlite-calls))) (is (= db-path (first (first @sqlite-calls)))) - (is (= 1 (count @backup-calls))) - (let [{:keys [repo dir relative-path ext content opts]} (first @backup-calls)] - (is (= db-name repo)) - (is (= nil dir)) - (is (= nil relative-path)) - (is (= ".sqlite" ext)) - (is (= "copied" content)) - (is (= backups-path (:backups-dir opts))) - (is (= 12 (:keep-versions opts))) - (is (contains? opts :force-backup?)) - (is (nil? (:force-backup? opts)))))) + (is (empty? @backup-calls)) + (let [backups (graph-backup/list-backups graphs-dir db-name) + backup-name (:name (first backups)) + backup-db-path (graph-backup/backup-db-path graphs-dir db-name backup-name) + metadata (read-edn-file (graph-backup/backup-metadata-path graphs-dir db-name backup-name))] + (is (= 1 (count backups))) + (is (= "copied" (fs/readFileSync backup-db-path "utf8"))) + (is (= db-name (:repo metadata))) + (is (= :electron-auto (:source metadata))) + (is (= backup-name (:name metadata)))))) (p/catch (fn [e] (is false (str "unexpected error: " e)))) (p/finally done))))) @@ -121,7 +137,6 @@ (let [graphs-dir (node-helper/create-tmp-dir "electron-db-backup-custom-snapshot") db-name "logseq_db_demo" [_ db-path] (common-sqlite/get-db-full-path graphs-dir db-name) - backups-path (common-sqlite/get-db-backups-path graphs-dir db-name) custom-calls (atom []) fallback-calls (atom []) backup-calls (atom [])] @@ -147,13 +162,111 @@ (is (= [[db-path (second (first @custom-calls))]] @custom-calls)) (is (empty? @fallback-calls)) - (is (= 1 (count @backup-calls))) - (let [{:keys [repo ext content opts]} (first @backup-calls)] - (is (= db-name repo)) - (is (= ".sqlite" ext)) - (is (= "worker-copy" content)) - (is (= backups-path (:backups-dir opts))) - (is (= true (:force-backup? opts)))))) + (is (empty? @backup-calls)) + (let [backups (graph-backup/list-backups graphs-dir db-name) + backup-name (:name (first backups)) + backup-db-path (graph-backup/backup-db-path graphs-dir db-name backup-name) + metadata (read-edn-file (graph-backup/backup-metadata-path graphs-dir db-name backup-name))] + (is (= 1 (count backups))) + (is (= "worker-copy" (fs/readFileSync backup-db-path "utf8"))) + (is (= :electron-manual (:source metadata)))))) + (p/catch (fn [e] + (is false (str "unexpected error: " e)))) + (p/finally done))))) + +(deftest backup-db-via-worker-uses-shared-layout-and-worker-snapshot + (async done + (let [graphs-dir (node-helper/create-tmp-dir "electron-db-worker-backup") + db-name "logseq_db_demo" + runtime {:runtime true} + worker-calls (atom []) + backup-calls (atom [])] + (-> (p/with-redefs [cli-common-graph/get-db-graphs-dir (fn [] graphs-dir) + db-worker/ensure-runtime! (fn [repo window-id] + (swap! worker-calls conj [:ensure-runtime repo window-id]) + (p/resolved runtime)) + cli-transport/invoke (fn [runtime' method args] + (swap! worker-calls conj [:invoke runtime' method args]) + (fs/writeFileSync (second args) "worker-copy" "utf8") + (p/resolved {:path (second args)})) + backup-file/backup-file (fn [& args] + (swap! backup-calls conj args) + (p/resolved nil))] + (p/let [result (electron-db/backup-db-via-worker! db-name 7 {:force-backup? true}) + backups (graph-backup/list-backups graphs-dir db-name) + backup-name (:name (first backups)) + final-db-path (graph-backup/backup-db-path graphs-dir db-name backup-name) + metadata (read-edn-file (graph-backup/backup-metadata-path graphs-dir db-name backup-name)) + [_ runtime' method args] (second @worker-calls) + [repo snapshot-path] args] + (is (= true (:created? result))) + (is (= final-db-path (:path result))) + (is (= [[:ensure-runtime db-name 7] + [:invoke runtime :thread-api/backup-db-sqlite [db-name snapshot-path]]] + @worker-calls)) + (is (= runtime runtime')) + (is (= :thread-api/backup-db-sqlite method)) + (is (= db-name repo)) + (is (not= final-db-path snapshot-path)) + (is (= "worker-copy" (fs/readFileSync final-db-path "utf8"))) + (is (= :electron-manual (:source metadata))) + (is (empty? @backup-calls)))) + (p/catch (fn [e] + (is false (str "unexpected error: " e)))) + (p/finally done))))) + +(deftest automatic-worker-backup-throttles-recent-auto-backups + (async done + (let [graphs-dir (node-helper/create-tmp-dir "electron-db-worker-throttle") + db-name "logseq_db_demo" + runtime {:runtime true} + worker-calls (atom [])] + (-> (p/with-redefs [cli-common-graph/get-db-graphs-dir (fn [] graphs-dir) + db-worker/ensure-runtime! (fn [repo window-id] + (swap! worker-calls conj [:ensure-runtime repo window-id]) + (p/resolved runtime)) + cli-transport/invoke (fn [runtime' method args] + (swap! worker-calls conj [:invoke runtime' method args]) + (fs/writeFileSync (second args) "worker-copy" "utf8") + (p/resolved {:path (second args)}))] + (p/let [first-result (electron-db/backup-db-via-worker! db-name 7 {}) + second-result (electron-db/backup-db-via-worker! db-name 7 {})] + (is (= true (:created? first-result))) + (is (= {:backup-name nil + :path nil + :created? false + :reason :too-soon} + second-result)) + (is (= 1 (count (filter #(= :invoke (first %)) @worker-calls)))))) + (p/catch (fn [e] + (is false (str "unexpected error: " e)))) + (p/finally done))))) + +(deftest automatic-worker-backup-retains-only-twelve-auto-backups + (async done + (let [graphs-dir (node-helper/create-tmp-dir "electron-db-worker-retention") + db-name "logseq_db_demo" + runtime {:runtime true} + worker-calls (atom [])] + (doseq [idx (range 12)] + (write-backup! graphs-dir db-name (str "auto-" idx) :electron-auto idx)) + (write-backup! graphs-dir db-name "manual-old" :electron-manual 0) + (write-backup! graphs-dir db-name "cli-old" :cli 0) + (-> (p/with-redefs [cli-common-graph/get-db-graphs-dir (fn [] graphs-dir) + db-worker/ensure-runtime! (fn [repo window-id] + (swap! worker-calls conj [:ensure-runtime repo window-id]) + (p/resolved runtime)) + cli-transport/invoke (fn [runtime' method args] + (swap! worker-calls conj [:invoke runtime' method args]) + (fs/writeFileSync (second args) "worker-copy" "utf8") + (p/resolved {:path (second args)}))] + (p/let [result (electron-db/backup-db-via-worker! db-name 7 {}) + backup-names (set (map :name (graph-backup/list-backups graphs-dir db-name)))] + (is (= true (:created? result))) + (is (not (contains? backup-names "auto-0"))) + (is (contains? backup-names "manual-old")) + (is (contains? backup-names "cli-old")) + (is (= 14 (count backup-names))))) (p/catch (fn [e] (is false (str "unexpected error: " e)))) (p/finally done))))) diff --git a/src/test/frontend/config_test.cljs b/src/test/frontend/config_test.cljs index d1746f67f2..293ae3e951 100644 --- a/src/test/frontend/config_test.cljs +++ b/src/test/frontend/config_test.cljs @@ -9,6 +9,11 @@ (is (= "/tmp/home/logseq/graphs/foo~2Fbar" (config/get-local-dir (str common-config/db-version-prefix "foo/bar")))))) +(deftest get-electron-backup-dir-uses-unified-backup-directory + (with-redefs [state/state (atom {:system/info {:home-dir "/tmp/home"}})] + (is (= "/tmp/home/logseq/graphs/foo~2Fbar/backup" + (config/get-electron-backup-dir (str common-config/db-version-prefix "foo/bar")))))) + (deftest custom-url->ws-url-test (testing "https URL becomes wss" (is (= "wss://my-server.example.com/sync/%s" diff --git a/src/test/logseq/cli/command/graph_test.cljs b/src/test/logseq/cli/command/graph_test.cljs index 48d82b6e5f..6688a7b4aa 100644 --- a/src/test/logseq/cli/command/graph_test.cljs +++ b/src/test/logseq/cli/command/graph_test.cljs @@ -1,6 +1,7 @@ (ns logseq.cli.command.graph-test (:require ["fs" :as fs] ["path" :as node-path] + [cljs.reader :as reader] [cljs.test :refer [async deftest is]] [clojure.string :as string] [frontend.test.node-helper :as node-helper] @@ -10,6 +11,7 @@ [logseq.cli.server :as cli-server] [logseq.cli.transport :as transport] [logseq.common.graph-dir :as graph-dir] + [logseq.db-worker.graph-backup :as graph-backup] [promesa.core :as p])) (deftest test-graph-validate-result @@ -264,6 +266,7 @@ (p/resolved (assoc config :base-url "http://example"))) transport/invoke (fn [_ method args] (swap! invoke-calls conj [method args]) + (fs/writeFileSync (second args) "sqlite-copy" "utf8") (p/resolved {:path (second args)}))] (p/let [result (commands/execute {:type :graph-backup-create :repo "logseq_db_demo" @@ -273,34 +276,71 @@ (is (= :ok (:status result))) (is (= 1 (count @invoke-calls))) (let [[method [repo backup-db-path]] (first @invoke-calls) - expected-segment (node-path/join - (graph-dir/repo->encoded-graph-dir-name repo) - "backup" - (graph-dir/graph-dir-key->encoded-dir-name "demo-nightly-20260101T000000Z") - "db.sqlite")] + final-db-path (graph-backup/backup-db-path + (cli-server/graphs-dir {:root-dir root-dir}) + repo + "demo-nightly-20260101T000000Z") + metadata-path (graph-backup/backup-metadata-path + (cli-server/graphs-dir {:root-dir root-dir}) + repo + "demo-nightly-20260101T000000Z") + metadata (reader/read-string (fs/readFileSync metadata-path "utf8"))] (is (= :thread-api/backup-db-sqlite method)) (is (= "logseq_db_demo" repo)) (is (and (string? backup-db-path) - (string/includes? backup-db-path expected-segment)))))) + (string/starts-with? backup-db-path + (node-path/dirname final-db-path)))) + (is (not= final-db-path backup-db-path)) + (is (= final-db-path (get-in result [:data :path]))) + (is (= "sqlite-copy" (fs/readFileSync final-db-path "utf8"))) + (is (= :cli (:source metadata))) + (is (= "demo-nightly-20260101T000000Z" (:name metadata)))))) (p/catch (fn [e] (is false (str "unexpected error: " e)))) (p/finally done))))) +(deftest test-execute-graph-backup-create-cleans-reserved-target-after-worker-failure + (async done + (let [root-dir (node-helper/create-tmp-dir "cli-backup-create-cleanup") + graphs-dir (cli-server/graphs-dir {:root-dir root-dir}) + repo "logseq_db_demo" + backup-name "demo-failure" + backup-dir (graph-backup/backup-dir-path graphs-dir repo backup-name) + invoke-calls (atom [])] + (-> (p/with-redefs [cli-server/list-graphs (fn [_] ["demo"]) + cli-server/ensure-server! (fn [config _repo] + (p/resolved (assoc config :base-url "http://example"))) + transport/invoke (fn [_ method args] + (swap! invoke-calls conj [method args]) + (fs/writeFileSync (second args) "partial" "utf8") + (p/rejected (ex-info "snapshot failed" + {:code :snapshot-failed})))] + (graph-command/execute-graph-backup-create {:repo repo + :graph "demo" + :backup-name backup-name} + {:root-dir root-dir})) + (p/then (fn [_] + (is false "expected graph backup create to fail"))) + (p/catch (fn [e] + (is (= :snapshot-failed (:code (ex-data e)))) + (is (= 1 (count @invoke-calls))) + (is (not (fs/existsSync backup-dir))))) + (p/finally done))))) + (deftest test-execute-graph-backup-list-only-returns-current-graph-backups (async done (let [root-dir (node-helper/create-tmp-dir "cli-backup-list-scope") + graphs-dir (cli-server/graphs-dir {:root-dir root-dir}) demo-repo "logseq_db_demo" other-repo "logseq_db_other" demo-backup "demo-nightly" other-backup "other-nightly" - demo-db-path (node-path/join root-dir - "graphs" + demo-db-path (node-path/join graphs-dir (graph-dir/repo->encoded-graph-dir-name demo-repo) "backup" (graph-dir/graph-dir-key->encoded-dir-name demo-backup) "db.sqlite") - other-db-path (node-path/join root-dir - "graphs" + other-db-path (node-path/join graphs-dir (graph-dir/repo->encoded-graph-dir-name other-repo) "backup" (graph-dir/graph-dir-key->encoded-dir-name other-backup) @@ -366,9 +406,9 @@ stop-calls (atom []) restart-calls (atom []) root-dir (node-helper/create-tmp-dir "cli-backup-restore-flow") + graphs-dir (cli-server/graphs-dir {:root-dir root-dir}) sqlite-payload (js/Buffer.from "sqlite" "utf8") - backup-db-path (node-path/join root-dir - "graphs" + backup-db-path (node-path/join graphs-dir (graph-dir/repo->encoded-graph-dir-name "logseq_db_demo") "backup" (graph-dir/graph-dir-key->encoded-dir-name "demo-nightly") diff --git a/src/test/logseq/cli/format_test.cljs b/src/test/logseq/cli/format_test.cljs index d9fdd03313..0580af090f 100644 --- a/src/test/logseq/cli/format_test.cljs +++ b/src/test/logseq/cli/format_test.cljs @@ -996,18 +996,28 @@ (testing "graph backup list renders metadata table" (let [result (format/format-result {:status :ok :command :graph-backup-list - :data {:backups [{:name "demo-nightly" + :data {:backups [{:name "demo-auto" :created-at 90000 - :size-bytes 2048}]}} + :size-bytes 2048 + :source :electron-auto} + {:name "demo-cli" + :created-at 90000 + :size-bytes 4096 + :source :cli} + {:name "demo-legacy" + :created-at 90000 + :size-bytes 8192}]}} {:output-format nil :now-ms 100000})] (is (string/includes? result "NAME")) (is (string/includes? result "CREATED-AT")) (is (string/includes? result "SIZE-BYTES")) - (is (string/includes? result "demo-nightly")) + (is (string/includes? result "AUTO-SAVE")) + (is (some? (re-find #"demo-auto\s+10 seconds ago\s+2048\s+yes" result))) + (is (some? (re-find #"demo-cli\s+10 seconds ago\s+4096\s+no" result))) + (is (some? (re-find #"demo-legacy\s+10 seconds ago\s+8192\s+-" result))) (is (string/includes? result "10 seconds ago")) - (is (string/includes? result "2048")) - (is (string/includes? result "Count: 1")))) + (is (string/includes? result "Count: 3")))) (testing "graph backup create renders a succinct success line" (let [result (format/format-result {:status :ok diff --git a/src/test/logseq/db_worker/graph_backup_test.cljs b/src/test/logseq/db_worker/graph_backup_test.cljs new file mode 100644 index 0000000000..4dbe380f28 --- /dev/null +++ b/src/test/logseq/db_worker/graph_backup_test.cljs @@ -0,0 +1,266 @@ +(ns logseq.db-worker.graph-backup-test + (:require ["fs" :as fs] + ["path" :as node-path] + [cljs.reader :as reader] + [cljs.test :refer [async deftest is]] + [frontend.test.node-helper :as node-helper] + [logseq.common.graph-dir :as graph-dir] + [logseq.db-worker.graph-backup :as graph-backup] + [promesa.core :as p])) + +(defn- read-edn-file + [file-path] + (reader/read-string (.toString (fs/readFileSync file-path) "utf8"))) + +(defn- write-backup! + ([graphs-dir repo backup-name] + (write-backup! graphs-dir repo backup-name nil)) + ([graphs-dir repo backup-name metadata] + (let [db-path (graph-backup/backup-db-path graphs-dir repo backup-name) + metadata-path (graph-backup/backup-metadata-path graphs-dir repo backup-name)] + (fs/mkdirSync (node-path/dirname db-path) #js {:recursive true}) + (fs/writeFileSync db-path (str "sqlite-" backup-name) "utf8") + (when metadata + (fs/writeFileSync metadata-path (pr-str metadata) "utf8")) + db-path))) + +(deftest backup-paths-use-canonical-graph-backup-layout + (let [graphs-dir "/tmp/logseq-graphs" + repo "logseq_db_foo/bar" + backup-name "daily:name/with space" + encoded-graph (graph-dir/repo->encoded-graph-dir-name repo) + encoded-backup (graph-dir/graph-dir-key->encoded-dir-name backup-name) + backup-root (node-path/join graphs-dir encoded-graph "backup") + backup-dir (node-path/join backup-root encoded-backup)] + (is (= backup-root + (graph-backup/backup-root-path graphs-dir repo))) + (is (= encoded-backup + (graph-backup/backup-dir-name backup-name))) + (is (= backup-dir + (graph-backup/backup-dir-path graphs-dir repo backup-name))) + (is (= (node-path/join backup-dir "db.sqlite") + (graph-backup/backup-db-path graphs-dir repo backup-name))) + (is (= (node-path/join backup-dir "metadata.edn") + (graph-backup/backup-metadata-path graphs-dir repo backup-name))))) + +(deftest backup-paths-reject-directory-traversal-names + (is (thrown-with-msg? js/Error + #"invalid graph directory path" + (graph-backup/backup-root-path "/tmp/logseq-graphs" "logseq_db_.."))) + (is (thrown-with-msg? js/Error + #"invalid backup directory path" + (graph-backup/backup-dir-path "/tmp/logseq-graphs" "logseq_db_demo" ".."))) + (is (thrown-with-msg? js/Error + #"invalid backup directory path" + (graph-backup/backup-dir-path "/tmp/logseq-graphs" "logseq_db_demo" ".")))) + +(deftest build-backup-name-preserves-cli-shape + (is (= "demo-20260101T000000Z" + (graph-backup/build-backup-name "logseq_db_demo" nil "20260101T000000Z"))) + (is (= "demo-nightly-20260101T000000Z" + (graph-backup/build-backup-name "logseq_db_demo" " nightly " "20260101T000000Z")))) + +(deftest next-backup-target-appends-numeric-suffix + (let [graphs-dir (node-helper/create-tmp-dir "graph-backup-next-target") + repo "logseq_db_demo" + base-name "demo-nightly" + existing-dir (graph-backup/backup-dir-path graphs-dir repo base-name)] + (fs/mkdirSync existing-dir #js {:recursive true}) + (is (= {:backup-name "demo-nightly-1" + :dir-path (graph-backup/backup-dir-path graphs-dir repo "demo-nightly-1") + :db-path (graph-backup/backup-db-path graphs-dir repo "demo-nightly-1")} + (graph-backup/next-backup-target graphs-dir repo base-name))))) + +(deftest list-backups-only-returns-directories-with-sqlite-files + (let [graphs-dir (node-helper/create-tmp-dir "graph-backup-list") + repo "logseq_db_demo" + valid-name "demo-nightly" + incomplete-name "demo-incomplete" + root-path (graph-backup/backup-root-path graphs-dir repo) + incomplete-dir (graph-backup/backup-dir-path graphs-dir repo incomplete-name)] + (write-backup! graphs-dir repo valid-name) + (fs/mkdirSync incomplete-dir #js {:recursive true}) + (fs/writeFileSync (node-path/join root-path "not-a-directory") "ignored" "utf8") + (is (= [valid-name] + (mapv :name (graph-backup/list-backups graphs-dir repo)))))) + +(deftest list-backups-includes-source-when-metadata-exists + (let [graphs-dir (node-helper/create-tmp-dir "graph-backup-list-source") + repo "logseq_db_demo"] + (write-backup! graphs-dir repo "demo-auto" + {:schema-version 1 + :name "demo-auto" + :repo repo + :source :electron-auto + :created-at-ms 1770000000000 + :db-path (graph-backup/backup-db-path graphs-dir repo "demo-auto")}) + (write-backup! graphs-dir repo "demo-cli" + {:schema-version 1 + :name "demo-cli" + :repo repo + :source :cli + :created-at-ms 1770000001000 + :db-path (graph-backup/backup-db-path graphs-dir repo "demo-cli")}) + (write-backup! graphs-dir repo "demo-legacy") + (is (= [{:name "demo-auto" + :source :electron-auto} + {:name "demo-cli" + :source :cli} + {:name "demo-legacy"}] + (mapv #(select-keys % [:name :source]) + (graph-backup/list-backups graphs-dir repo)))))) + +(deftest create-backup-snapshots-to-temp-file-before-publishing + (async done + (let [graphs-dir (node-helper/create-tmp-dir "graph-backup-create") + repo "logseq_db_demo" + backup-name "demo-nightly" + snapshot-calls (atom []) + final-db-path (graph-backup/backup-db-path graphs-dir repo backup-name) + final-visible-during-snapshot? (atom nil)] + (-> (p/let [result (graph-backup/ (graph-backup/ (p/let [result (graph-backup/ (p/let [result (graph-backup/