From 06dbef87156e06fb970ff2d70b9ecd4caf977a71 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Tue, 5 May 2026 19:27:00 +0800 Subject: [PATCH] feat: add markdown mirror Add markdown mirror generation for DB graphs, including page and journal paths, regeneration, debounced write handling, and settings UI. Serialize page and block property values into mirrored Markdown by resolving property-value entities to their display content. Persist mirror files through the node worker platform and make browser worker mirror storage fail fast as unsupported. Fix Electron userAppCfgs writes to avoid returning unserializable Electron state over IPC. Add ADR and targeted tests for mirror generation, worker wiring, platform storage, and graph path handling. --- deps/cli/src/logseq/cli/common/file.cljs | 103 ++-- deps/common/src/logseq/common/graph.cljs | 3 +- .../common/test/logseq/common/graph_test.cljs | 4 +- docs/adr/0016-markdown-mirror.md | 283 +++++++++++ src/electron/electron/handler.cljs | 3 +- src/main/frontend/components/settings.cljs | 47 ++ src/main/frontend/persist_db/browser.cljs | 15 +- src/main/frontend/worker/db_core.cljs | 15 + src/main/frontend/worker/db_listener.cljs | 5 + src/main/frontend/worker/markdown_mirror.cljs | 357 ++++++++++++++ .../frontend/worker/platform/browser.cljs | 14 +- src/main/frontend/worker/platform/node.cljs | 22 + src/resources/dicts/en.edn | 5 + src/resources/dicts/zh-cn.edn | 5 + .../frontend/handler/editor_async_test.cljs | 2 +- src/test/frontend/worker/db_core_test.cljs | 1 + .../frontend/worker/db_listener_test.cljs | 16 +- src/test/frontend/worker/db_worker_test.cljs | 1 + .../frontend/worker/markdown_mirror_test.cljs | 447 ++++++++++++++++++ .../frontend/worker/platform_node_test.cljs | 20 + src/test/frontend/worker/platform_test.cljs | 20 + 21 files changed, 1351 insertions(+), 37 deletions(-) create mode 100644 docs/adr/0016-markdown-mirror.md create mode 100644 src/main/frontend/worker/markdown_mirror.cljs create mode 100644 src/test/frontend/worker/markdown_mirror_test.cljs diff --git a/deps/cli/src/logseq/cli/common/file.cljs b/deps/cli/src/logseq/cli/common/file.cljs index 0df8bff5fb..78e3d2c1f9 100644 --- a/deps/cli/src/logseq/cli/common/file.cljs +++ b/deps/cli/src/logseq/cli/common/file.cljs @@ -2,6 +2,7 @@ "Convert blocks to file content. Used for frontend exports and CLI" (:require [clojure.string :as string] [datascript.core :as d] + [datascript.impl.entity :as de] [logseq.common.util.date-time :as date-time-util] [logseq.db :as ldb] [logseq.db.frontend.content :as db-content] @@ -33,39 +34,58 @@ date-time-util/default-journal-title-formatter)))))) (defn- property-value->string - [property v context] - (cond - (and (map? v) (:db/id v)) - (str (db-property/property-value-content v)) + [db property v context] + (letfn [(entity-map [x] + (cond + (map? x) x + (de/entity? x) (into {} x))) + (entity-content [x] + (let [m (entity-map x)] + (or (:block/title m) + (:logseq.property/value m))))] + (cond + (some? (entity-content v)) + (str (entity-content v)) - (set? v) - (->> v - (sort-by (fn [item] - [(if (:block/order item) 0 1) - (str (or (:block/order item) - (property-value->string property item context)))])) - (map #(property-value->string property % context)) - (string/join ", ")) + (some? (:db/id (entity-map v))) + (let [entity (d/entity db (:db/id (entity-map v)))] + (str (or (entity-content entity) + (entity-content v) + ""))) - (sequential? v) - (->> v - (map #(property-value->string property % context)) - (string/join ", ")) + (set? v) + (->> v + (sort-by (fn [item] + [(if (:block/order item) 0 1) + (str (or (:block/order item) + (property-value->string db property item context)))])) + (map #(property-value->string db property % context)) + (string/join ", ")) - (keyword? v) - (name v) + (sequential? v) + (->> v + (map #(property-value->string db property % context)) + (string/join ", ")) - (and (= :datetime (:logseq.property/type property)) - (integer? v)) - (or (datetime-journal-title v context) - (str v)) + (keyword? v) + (name v) - (some? v) - (str v))) + (and (= :datetime (:logseq.property/type property)) + (integer? v)) + (or (datetime-journal-title v context) + (str v)) + + (some? v) + (str v)))) (defn- block-properties-content [db block spaces-tabs context] - (let [properties (->> (db-property/properties block) + (let [block (or (when-let [id (:db/id block)] + (d/entity db id)) + (when-let [uuid (:block/uuid block)] + (d/entity db [:block/uuid uuid])) + block) + properties (->> (db-property/properties block) (remove (fn [[k _]] (contains? db-property/db-attribute-properties k))) (remove (fn [[k _]] @@ -84,15 +104,34 @@ (:block/raw-title property) (name property-ident)) ":: " - (property-value->string property (get properties property-ident) context)))))) + (property-value->string db property (get properties property-ident) context)))))) (string/join "\n")))))) +(defn- property-value-block-content + [db b context] + (when-let [raw-block (d/entity db (:db/id b))] + (when-let [property (:logseq.property/created-from-property raw-block)] + (let [property-title (or (:block/title property) + (:block/raw-title property) + (some-> property :db/ident name)) + value (property-value->string db property + (or (:block/title raw-block) + (:logseq.property/value raw-block)) + context)] + (when property-title + (str property-title ":: " value)))))) + +(defn- block-title-content + [db b context] + (or (property-value-block-content db b context) + (db-content/recur-replace-uuid-in-block-title (d/entity db (:db/id b))))) + (defn- transform-content [db b level {:keys [heading-to-list? include-properties?] :or {include-properties? true}} context] (let [heading (:logseq.property/heading b) ;; replace [[uuid]] with block's content - title (db-content/recur-replace-uuid-in-block-title (d/entity db (:db/id b))) + title (block-title-content db b context) content (or title "") level (if (and heading-to-list? heading) (if (> heading 1) @@ -125,7 +164,15 @@ (if (nil? f) (->> block-contents persistent! flatten (remove nil?)) (let [page? (nil? (:block/page f)) - content (if (and page? (not link)) nil (transform-content db f level opts context)) + content (cond + (and page? (not link) (:include-page-properties? opts)) + (block-properties-content db f "" context) + + (and page? (not link)) + nil + + :else + (transform-content db f level opts context)) new-content (if-let [children (seq (:block/children f))] (cons content (tree->file-content-aux db children {:init-level (inc level)} context)) diff --git a/deps/common/src/logseq/common/graph.cljs b/deps/common/src/logseq/common/graph.cljs index 1cd38ae736..dbb28b938a 100644 --- a/deps/common/src/logseq/common/graph.cljs +++ b/deps/common/src/logseq/common/graph.cljs @@ -65,6 +65,7 @@ Rules: - Contents in '/logseq/.recycle/' are ignored - Contents in '/logseq/bak/' are ignored - Contents in with '/logseq/version-files/' are ignored +- Contents in '/markdown-mirror/' are ignored " [dir path] (let [dir (path/path-normalize dir) @@ -73,7 +74,7 @@ Rules: (when (string? path) (or (some #(string/starts-with? rpath %) - ["." "logseq/.recycle" "logseq/bak" "logseq/version-files"]) + ["." "logseq/.recycle" "logseq/bak" "logseq/version-files" "markdown-mirror"]) (contains? #{"logseq/graphs-txid.edn" "logseq/pages-metadata.edn"} rpath) (some #(string/includes? rpath (str "/" % "/")) ["node_modules"]) diff --git a/deps/common/test/logseq/common/graph_test.cljs b/deps/common/test/logseq/common/graph_test.cljs index 48cf766ae9..63b9207665 100644 --- a/deps/common/test/logseq/common/graph_test.cljs +++ b/deps/common/test/logseq/common/graph_test.cljs @@ -29,7 +29,9 @@ (fs/writeFileSync "tmp/test-graph/journals/2023_05_09.md" "") ;; Create files that are ignored (fs/mkdirSync (node-path/join "tmp/test-graph" "logseq" "bak")) + (fs/mkdirSync (node-path/join "tmp/test-graph" "markdown-mirror" "pages") #js {:recursive true}) (fs/writeFileSync "tmp/test-graph/logseq/bak/baz.md" "") (fs/writeFileSync "tmp/test-graph/logseq/.gitignore" "") + (fs/writeFileSync "tmp/test-graph/markdown-mirror/pages/foo.md" "") (is (= ["tmp/test-graph/journals/2023_05_09.md" "tmp/test-graph/pages/foo.md"] - (common-graph/get-files "tmp/test-graph")))) \ No newline at end of file + (common-graph/get-files "tmp/test-graph")))) diff --git a/docs/adr/0016-markdown-mirror.md b/docs/adr/0016-markdown-mirror.md new file mode 100644 index 0000000000..d04a19b236 --- /dev/null +++ b/docs/adr/0016-markdown-mirror.md @@ -0,0 +1,283 @@ +# ADR 0016: Electron Markdown Mirror + +Date: 2026-05-05 +Status: Proposed + +## Context +Logseq DB graphs do not expose one editable Markdown file per page in the graph +directory. Some desktop workflows still need a Markdown representation that can +be read by external tools, backed up, indexed, or inspected outside Logseq. + +The mirror must not become another graph source of truth. Editing should remain +fast, and saving a block must not wait for Markdown rendering or filesystem +writes on the renderer main thread. + +The first supported runtime is the Electron desktop app. Browser and mobile +builds do not have the same graph-directory filesystem guarantees. + +## Decision +1. Add an Electron-only Settings toggle for Markdown Mirror. +2. Persist the toggle in Electron user settings as + `:feature/markdown-mirror?`. +3. When the setting is enabled for the Electron app, Logseq writes derived + Markdown files under the current graph directory: + - journals: + `markdown-mirror/journals/.md` + - other pages: + `markdown-mirror/pages/.md` +4. For a graph at `~/logseq/graphs/graph-xxx`, mirror files are written under: + - `~/logseq/graphs/graph-xxx/markdown-mirror/journals/` + - `~/logseq/graphs/graph-xxx/markdown-mirror/pages/` +5. Markdown Mirror is derived output. The DB remains the source of truth. +6. Files under `markdown-mirror/**` must be ignored by graph import, file + watchers, and graph parsing so the mirror never feeds back into the graph. +7. The feature is not available in browser or mobile builds, even if a stale + setting value exists. +8. Settings exposes an explicit "Regenerate full mirror" action that asks the + DB worker to rewrite the complete mirror for the current graph. +9. Built-in pages and property pages, including user-created properties, are not + exported to the mirror. User Tag/Class pages are normal user content and are + exported. + +## Runtime Ownership +1. The renderer owns only: + - the Settings row + - reading and updating `:feature/markdown-mirror?` + - pushing the enabled state to the DB worker when it changes +2. The DB worker owns: + - detecting affected page ids from successful local transactions + - rendering page Markdown from the worker DB snapshot + - scheduling and coalescing mirror jobs + - running explicit full-mirror regeneration jobs + - invoking platform filesystem writes +3. The Electron main process must not render Markdown. It may provide filesystem + primitives if needed, but content generation stays with the worker. +4. Editor save paths enqueue mirror work and return immediately. They must not + wait for rendering, directory creation, stat, write, rename, or delete. + +## Reusable Core and CLI Path +1. Markdown Mirror path planning, filename normalization, page rendering, write + deduplication, atomic writes, rename cleanup, and delete cleanup live in a + worker/core namespace that does not depend on Electron UI state. +2. The Electron app only owns feature activation through Settings. +3. The CLI should be able to reuse the same core by passing an explicit graph, + DB snapshot, and node filesystem platform context. +4. Future CLI support should not introduce a second Markdown serializer or a + different filename normalization policy. +5. Future CLI support should reuse the same mirror-path allocation index so the + Electron app and CLI do not produce different file names for the same graph. + +## Output Layout and Naming +1. Journal pages are written below `markdown-mirror/journals/`. +2. Non-journal pages are written below `markdown-mirror/pages/`. +3. Journal file names use the existing Logseq journal file-name rules for the + graph configuration. +4. Non-journal page file names use the normalized page title: + `.md`. +5. Page file names must stay friendly to external Markdown tools such as Emacs, + VS Code, and Obsidian. Do not include page uuid in normal mirror file names. +6. Page title is not page identity. The page uuid is still the internal mirror + identity, but it is stored in the mirror index rather than exposed in the + file name. +7. Duplicate non-journal page titles are handled by stable title suffix + allocation: + - first allocated page: `pages/Foo.md` + - second allocated page: `pages/Foo (2).md` + - third allocated page: `pages/Foo (3).md` +8. Once a page uuid is assigned a mirror path, keep that path stable until the + page is renamed or deleted. Do not renumber existing duplicate-title mirror + paths when another duplicate is created or removed. +9. The implementation keeps a per-graph mirror index under + `markdown-mirror/.index.edn`. +10. The mirror index stores at least: + - page uuid -> relative mirror path + - relative mirror path -> page uuid + - page uuid -> last known normalized title stem +11. The mirror index is implementation metadata for path stability. It is not + graph content and must be ignored by graph import and watchers along with the + rest of `markdown-mirror/**`. +12. All mirror file names pass through a single cross-platform filename + normalizer before joining paths. +13. Duplicate journal-day entities indicate invalid graph state for the mirror. + The implementation must fail those journal mirror jobs and surface a + diagnostic instead of choosing a winner. +14. If two entities still map to the same mirror path, the implementation must + fail the mirror job for that path and surface a diagnostic instead of + overwriting an unrelated page. +15. Page rename moves the mirror by writing the new path, updating the mirror + index, and deleting the old path after the new file has been written. +16. Page deletion deletes the corresponding mirror file and removes the page uuid + from the mirror index. +17. The write guard must reject any computed path outside the graph's + `markdown-mirror/` directory. +18. Built-in pages and property pages are excluded from path allocation and + mirror writes. User Tag/Class pages are not excluded by this rule. If a + previously mirrored page becomes excluded, the old mirror file is removed. + +## Duplicate Page Title Allocation +1. For non-journal pages, compute the normalized title stem first. +2. If the page uuid already exists in the mirror index and the normalized title + stem did not change, reuse the indexed path. +3. If the page uuid is new for that title, allocate the first unused path in this + sequence: + - `pages/.md` + - `pages/ (2).md` + - `pages/ (3).md` +4. A path is considered unavailable when the mirror index maps it to a different + live page uuid. +5. Deleted page paths become available for future allocation only after the + deleted page uuid is removed from the index. +6. Rename is treated as a new allocation for the new title stem. Existing pages + with the old title keep their already allocated paths. +7. If the mirror index is missing or unreadable, rebuild it from the current DB + in deterministic page order before writing. Deterministic order should use a + stable key such as page title plus page uuid. +8. The rebuilt index is allowed to choose paths for pages that had no previous + allocation. It must not overwrite a live existing path that is already mapped + to another page uuid. + +## Rename and Delete +1. Page rename moves the mirror by writing the new path and deleting the old + path after the new file has been written. +2. Page deletion deletes the corresponding mirror file. +3. The implementation keeps a small per-graph mirror index keyed by page uuid so + rename and delete handling does not require scanning the mirror directory on + every transaction. + +## Filename Normalization +1. Mirror file names must be portable across macOS, Windows, and Linux. +2. Use one shared normalizer for journal and page mirror file names. +3. The normalizer must: + - reject or replace path separators (`/`, `\`) + - reject or replace Windows-invalid characters (`<`, `>`, `:`, `"`, `|`, + `?`, `*`) and ASCII control characters + - reject or rewrite reserved Windows device names such as `CON`, `PRN`, + `AUX`, `NUL`, `COM1` through `COM9`, and `LPT1` through `LPT9` + - trim trailing spaces and dots because Windows does not preserve them + - reject empty names after normalization + - bound each file-name component to a safe byte length before appending + `.md` +4. Normalize Unicode to one canonical form before sanitizing so the same page + title produces the same mirror path across filesystems with different Unicode + normalization behavior. +5. The normalizer must be deterministic and must not depend on the current + operating system. A graph mirrored on macOS should choose the same logical + file name as the same graph mirrored on Windows. +6. If normalization changes the display title segment, the mirror index and + duplicate-title suffix allocation still preserve identity for non-journal + pages. +7. If a journal title normalizes to an unsafe or colliding file name, fail the + journal mirror job and surface diagnostics instead of inventing a fallback + name. +8. Path construction must join only validated path components. It must never + concatenate unchecked page titles into filesystem paths. + +## Scheduling and Performance +1. Mirror rendering is incremental. A transaction schedules only pages affected + by that transaction. +2. Jobs are coalesced by page uuid. If a page is edited repeatedly before its + job runs, only the latest worker DB state is rendered. +3. Scheduling uses a short debounce window per graph to reduce write churn while + preserving near-real-time output. +4. Mirror writes are serialized per graph to avoid path races during rename and + delete. +5. Before writing, compare the generated Markdown with the current file content + or with the last written content hash. Skip the write when content is + unchanged. +6. Write files atomically: + - ensure the target directory exists + - write to a temporary file in the same directory + - rename the temporary file over the target +7. Heavy work is forbidden on the renderer main thread: + - no full-graph export + - no Markdown rendering + - no filesystem reads or writes + - no synchronous IPC waiting for mirror completion +8. Full regeneration is an explicit Settings action. The renderer only sends a + worker request; page selection, rendering, and filesystem writes stay in the + DB worker. +9. Enabling the setting starts incremental mirroring for subsequent page edits. + It does not implicitly run full regeneration. + +## Markdown Content +1. Reuse the existing page-to-Markdown export pipeline used by worker export + APIs instead of introducing a separate renderer-side serializer. +2. The mirror output should match normal Markdown export semantics for page + content. +3. Mirror files do not include Logseq-internal mirror metadata in the Markdown + body. +4. Mirror files include block and page property drawers, including user + properties, with rendered property values. +5. Assets are referenced as normal exported Markdown references. This ADR does + not copy assets into `markdown-mirror/`. +6. Page references remain in Logseq wiki-link form, for example `[[Foo]]`. +7. Ambiguous page references caused by duplicate page titles are an accepted + limitation of Markdown Mirror. Do not rewrite page references to uuid-based + links or relative Markdown links in this ADR. + +## Failure Handling +1. Filesystem and path errors fail the mirror job for the affected page. +2. Failures are logged with graph, page uuid, target path, and error details. +3. Repeated failures should be visible from Settings or diagnostics; do not show + a toast on every keystroke. +4. The feature must not silently fall back to browser storage, OPFS, or another + output directory. +5. If the graph directory is not available, the worker rejects mirror jobs for + that graph until the graph is reopened with a valid directory. + +## Non-goals +1. Markdown Mirror is not bidirectional sync. +2. Editing files in `markdown-mirror/` does not update the graph. +3. The mirror is not a backup format with guaranteed import fidelity. +4. The mirror does not replace existing graph export features. +5. The mirror does not support browser or mobile runtimes in this ADR. + +## Consequences + +### Positive +- Desktop users get a readable Markdown projection inside the graph directory. +- Editor latency is protected because rendering and disk I/O are worker-owned + and asynchronous. +- The output layout is predictable for tools that watch journals and pages + separately. +- Page file names remain readable and practical in external Markdown tools. +- Ignoring `markdown-mirror/**` prevents mirror-generated files from becoming + graph input. + +### Tradeoffs +- The mirror can lag slightly behind the latest edit because writes are + debounced and serialized. +- A per-graph mirror index is needed for reliable rename and delete cleanup. +- Duplicate page references such as `[[Foo]]` remain ambiguous in mirror output. +- The first version does not backfill every existing page automatically when the + setting is enabled; users run full regeneration explicitly. +- External edits to mirror files are overwritten by later Logseq edits. +- Property pages are intentionally absent from the mirror, so the output is not + a complete DB export even though page and block property drawers are included. + +## Verification +Implementation should add focused coverage for: + +```bash +bb dev:test -v frontend.worker.markdown-mirror-test/enabled-electron-edit-writes-page-mirror-test +bb dev:test -v frontend.worker.markdown-mirror-test/enabled-electron-edit-writes-journal-mirror-test +bb dev:test -v frontend.worker.markdown-mirror-test/disabled-setting-does-not-write-mirror-test +bb dev:test -v frontend.worker.markdown-mirror-test/repeated-edits-coalesce-to-latest-content-test +bb dev:test -v frontend.worker.markdown-mirror-test/rename-removes-old-mirror-path-test +bb dev:test -v frontend.worker.markdown-mirror-test/delete-removes-mirror-file-test +bb dev:test -v frontend.worker.markdown-mirror-test/same-title-pages-write-distinct-stable-friendly-paths-test +bb dev:test -v frontend.worker.markdown-mirror-test/page-references-remain-wiki-links-test +bb dev:test -v frontend.worker.markdown-mirror-test/page-mirror-exports-property-values-test +bb dev:test -v frontend.worker.markdown-mirror-test/page-mirror-exports-page-property-values-test +bb dev:test -v frontend.worker.markdown-mirror-test/full-regeneration-writes-existing-non-built-in-non-property-pages-test +bb dev:test -v frontend.worker.markdown-mirror-test/invalid-filename-characters-are-normalized-test +bb dev:test -v frontend.worker.markdown-mirror-test/windows-reserved-filename-fails-with-diagnostic-test +bb dev:test -v frontend.worker.markdown-mirror-test/mirror-path-collision-fails-without-overwrite-test +``` + +Additional checks: +- `markdown-mirror/**` is excluded from graph parsing and file watchers. +- Editor save does not await mirror completion. +- Browser and mobile builds do not expose the setting and do not schedule mirror + jobs. +- Atomic write failures do not leave partial target files. diff --git a/src/electron/electron/handler.cljs b/src/electron/electron/handler.cljs index beac372b75..c45baa5b6e 100644 --- a/src/electron/electron/handler.cljs +++ b/src/electron/electron/handler.cljs @@ -315,7 +315,8 @@ (do (cfgs/set-item! k v) (when (= k :spell-check) (spell-check/apply-window-spellcheck! window (spell-check/session-spellcheck-enabled? v))) - (state/set-state! [:config k] v)) + (state/set-state! [:config k] v) + nil) (cfgs/get-item k)) config))) diff --git a/src/main/frontend/components/settings.cljs b/src/main/frontend/components/settings.cljs index 798f565ad9..ffd6fc1db2 100644 --- a/src/main/frontend/components/settings.cljs +++ b/src/main/frontend/components/settings.cljs @@ -745,6 +745,51 @@ {:left-label (t :settings.features/enable-flashcards) :action (flashcards-enabled-switcher enable-flashcards?)})) +(rum/defcs markdown-mirror-row < rum/reactive + (rum/local false ::regenerating?) + [state t] + (let [enabled? (true? (state/sub [:electron/user-cfgs :feature/markdown-mirror?])) + *regenerating? (::regenerating? state) + regenerate! (fn [] + (let [repo (state/get-current-repo)] + (when (and repo @state/*db-worker (not @*regenerating?)) + (reset! *regenerating? true) + (-> (state/ (state/ (state/ (p/let [_ (state/ (p/let [_ (state/encoded-graph-dir-name repo) "/markdown-mirror")) + +(def ^:private invalid-file-name-chars-re + #"[<>:\"|?*\\/]") + +(def ^:private ascii-control-re + #"[\x00-\x1F]") + +(def ^:private trailing-space-or-dot-re + #"[ \.]+$") + +(def ^:private reserved-windows-device-names + (into #{"CON" "PRN" "AUX" "NUL"} + (concat (map #(str "COM" %) (range 1 10)) + (map #(str "LPT" %) (range 1 10))))) + +(def ^:private max-file-stem-length 160) + +(defonce ^:private *repo->enabled? (atom {})) +(defonce ^:private *repo->queued-page-jobs (atom {})) +(defonce ^:private *repo->flush-timeout (atom {})) + +(defn- normalize-unicode + [s] + (let [s (str s)] + (if (fn? (.-normalize s)) + (.normalize s "NFC") + s))) + +(defn- reserved-windows-device-name? + [s] + (contains? reserved-windows-device-names + (string/upper-case s))) + +(defn normalize-file-stem + [s] + (when (some? s) + (let [s' (-> (normalize-unicode s) + (string/replace invalid-file-name-chars-re "_") + (string/replace ascii-control-re "_") + (string/replace trailing-space-or-dot-re "")) + s' (if (> (count s') max-file-stem-length) + (subs s' 0 max-file-stem-length) + s')] + (when (and (not (string/blank? s')) + (not (reserved-windows-device-name? s'))) + s')))) + +(defn- journal-file-stem + [journal-day] + (when journal-day + (let [s (str journal-day)] + (when (= 8 (count s)) + (str (subs s 0 4) "_" (subs s 4 6) "_" (subs s 6 8)))))) + +(defn page-relative-path + ([db page] + (page-relative-path db page {})) + ([db page {:keys [journal-file-stem-fn] + :or {journal-file-stem-fn journal-file-stem}}] + (when page + (if (ldb/journal? page) + (when-let [stem (normalize-file-stem (journal-file-stem-fn (:block/journal-day page)))] + (str "journals/" stem ".md")) + (when-let [stem (normalize-file-stem (:block/title page))] + (let [duplicate-pages (->> (d/datoms db :avet :block/title (:block/title page)) + (map #(d/entity db (:e %))) + (filter #(and (ldb/page? %) + (not (ldb/journal? %)))) + (sort-by (comp str :block/uuid))) + index (inc (or (first (keep-indexed + (fn [idx p] + (when (= (:block/uuid page) (:block/uuid p)) + idx)) + duplicate-pages)) + 0)) + stem' (if (= 1 index) + stem + (str stem " (" index ")"))] + (str "pages/" stem' ".md"))))))) + +(defn- mirror-path + [repo relative-path] + (str (repo-mirror-dir repo) "/" relative-path)) + +(defn- page-id-for-entity + [db eid] + (when-let [entity (d/entity db eid)] + (cond + (ldb/page? entity) (:db/id entity) + (:block/page entity) (:db/id (:block/page entity)) + (and (:block/parent entity) (ldb/page? (:block/parent entity))) (:db/id (:block/parent entity)) + (some-> entity :block/parent :block/page) (:db/id (:block/page (:block/parent entity)))))) + +(defn affected-page-ids + [{:keys [db-before db-after tx-data]}] + (->> tx-data + (mapcat (fn [{:keys [e a v]}] + (cond-> [(page-id-for-entity db-before e) + (page-id-for-entity db-after e)] + (= a :block/page) + (conj v)))) + (remove nil?) + set)) + +(defn set-enabled! + [repo enabled?] + (if enabled? + (swap! *repo->enabled? assoc repo true) + (do + (when-let [timeout-id (get @*repo->flush-timeout repo)] + (js/clearTimeout timeout-id)) + (swap! *repo->enabled? dissoc repo) + (swap! *repo->queued-page-jobs dissoc repo) + (swap! *repo->flush-timeout dissoc repo))) + nil) + +(defn enabled? + [repo] + (true? (get @*repo->enabled? repo))) + +(defn- storage + [platform*] + (:storage platform*)) + +(defn- (f path) + (p/catch (constantly nil))) + (p/rejected (ex-info "platform storage/read-text! missing" {:path path})))) + +(defn- content + db + (:block/uuid page) + {:include-page-properties? true} + {:export-bullet-indentation (or (:export-bullet-indentation options) " ") + :date-formatter (:date-formatter options)})) + +(defn- mirrorable-page? + [page] + (and (ldb/page? page) + (not (ldb/built-in? page)) + (not (ldb/property? page)) + (not (ldb/hidden? page)) + (not (:logseq.property.user/email page)))) + +(defn- mirrorable-pages + [db] + (->> (d/datoms db :avet :block/name) + (map #(d/entity db (:e %))) + (filter mirrorable-page?) + (sort-by (fn [page] + [(if (ldb/journal? page) 0 1) + (str (:block/journal-day page)) + (string/lower-case (or (:block/title page) "")) + (str (:block/uuid page))])))) + +(defn- queued-page-jobs update-in [repo (:page-id job)] merge-job job)) + +(defn- drain-repo-jobs! + [repo] + (let [jobs (vals (get @*repo->queued-page-jobs repo))] + (swap! *repo->queued-page-jobs dissoc repo) + jobs)) + +(declare flush-timeout repo) + (let [timeout-id (js/setTimeout + (fn [] + (swap! *repo->flush-timeout dissoc repo) + (-> (flush-timeout assoc repo timeout-id)))) + +(defn- (.unlink (browser-pfs) (asset-path repo file-name)) - (p/catch (constantly nil)))) + (let [^js pfs (browser-pfs)] + (-> (.unlink pfs (asset-path repo file-name)) + (p/catch (constantly nil))))) + +(defn- unsupported-mirror-storage! + [& _args] + (throw (ex-info "Markdown mirror storage is not supported in browser workers" + {:platform :browser + :feature :markdown-mirror}))) (defn- websocket-connect [url] @@ -201,6 +208,9 @@ :remove-vfs! remove-vfs! :read-text! read-text! :write-text! write-text! + :write-text-atomic! unsupported-mirror-storage! + :delete-file! unsupported-mirror-storage! + :mirror-read-text! unsupported-mirror-storage! :asset-read-bytes! asset-read-bytes! :asset-write-bytes! asset-write-bytes! :asset-stat asset-stat diff --git a/src/main/frontend/worker/platform/node.cljs b/src/main/frontend/worker/platform/node.cljs index a92ba9adaf..61525b2cab 100644 --- a/src/main/frontend/worker/platform/node.cljs +++ b/src/main/frontend/worker/platform/node.cljs @@ -252,6 +252,26 @@ _ (ensure-dir! dir)] (fs/writeFile full-path text "utf8")))) +(defn- write-text-atomic! + [write-guard-fn data-dir path text] + (let [full-path (path-under-data-dir data-dir path) + dir (node-path/dirname full-path) + tmp-path (node-path/join dir (str "." (node-path/basename full-path) ".tmp-" (random-uuid)))] + (p/let [_ (when write-guard-fn + (write-guard-fn)) + _ (ensure-dir! dir) + _ (fs/writeFile tmp-path text "utf8") + _ (fs/rename tmp-path full-path)] + nil))) + +(defn- delete-file! + [write-guard-fn data-dir path] + (let [full-path (path-under-data-dir data-dir path)] + (p/let [_ (when write-guard-fn + (write-guard-fn))] + (-> (fs/rm full-path #js {:force true}) + (p/catch (constantly nil)))))) + (defn- asset-file-path [data-dir repo file-name] (node-path/join (repo-dir data-dir repo) @@ -433,6 +453,8 @@ :remove-vfs! (fn [pool] (remove-vfs! pool)) :read-text! (fn [path] (read-text! data-dir path)) :write-text! (fn [path text] (write-text! write-guard-fn data-dir path text)) + :write-text-atomic! (fn [path text] (write-text-atomic! write-guard-fn data-dir path text)) + :delete-file! (fn [path] (delete-file! write-guard-fn data-dir path)) :asset-read-bytes! (fn [repo file-name] (asset-read-bytes! data-dir repo file-name)) :asset-write-bytes! (fn [repo file-name payload] diff --git a/src/resources/dicts/en.edn b/src/resources/dicts/en.edn index 2f4541c532..ec54bf5916 100644 --- a/src/resources/dicts/en.edn +++ b/src/resources/dicts/en.edn @@ -1631,6 +1631,11 @@ :settings.features/home-default-page-update-success "Home default page updated successfully!" :settings.features/journals-enable-success "Journals enabled" :settings.features/login-prompt "To access new features before anyone else you must be an Open Collective Sponsor or Backer of Logseq and therefore log in first." + :settings.features/markdown-mirror "Markdown Mirror" + :settings.features/markdown-mirror-desc "Write a derived Markdown copy of edited pages to the graph's markdown-mirror folder. Desktop only." + :settings.features/markdown-mirror-regenerate "Regenerate full mirror" + :settings.features/markdown-mirror-regenerate-error "Failed to regenerate Markdown Mirror: {1}" + :settings.features/markdown-mirror-regenerate-success "Markdown Mirror regenerated" :settings.features/page-not-found "The page \"{1}\" doesn't exist yet. Please create that page first, and then try again." :settings.features/plugin-system "Plugins" diff --git a/src/resources/dicts/zh-cn.edn b/src/resources/dicts/zh-cn.edn index f6e7af6e90..79c92157ed 100644 --- a/src/resources/dicts/zh-cn.edn +++ b/src/resources/dicts/zh-cn.edn @@ -1621,6 +1621,11 @@ :settings.features/home-default-page-update-success "主页默认页面已更新成功！" :settings.features/journals-enable-success "已启用日志页" :settings.features/login-prompt "你必须是 Logseq 的 Open Collective Sponsor 或者 Backer 才能提前使用新功能（仍在测试中），因此需要登录。" + :settings.features/markdown-mirror "Markdown 镜像" + :settings.features/markdown-mirror-desc "将已编辑页面的派生 Markdown 副本写入图谱的 markdown-mirror 文件夹。仅桌面端可用。" + :settings.features/markdown-mirror-regenerate "重新生成完整镜像" + :settings.features/markdown-mirror-regenerate-error "重新生成 Markdown 镜像失败：{1}" + :settings.features/markdown-mirror-regenerate-success "Markdown 镜像已重新生成" :settings.features/page-not-found "页面“{1}”尚不存在。请先创建该页面，然后再试一次。" :settings.features/plugin-system "插件系统" diff --git a/src/test/frontend/handler/editor_async_test.cljs b/src/test/frontend/handler/editor_async_test.cljs index 03d790dd4a..b1c3e4cd04 100644 --- a/src/test/frontend/handler/editor_async_test.cljs +++ b/src/test/frontend/handler/editor_async_test.cljs @@ -1,5 +1,5 @@ (ns frontend.handler.editor-async-test - (:require [clojure.test :refer [is testing async use-fixtures]] + (:require [cljs.test :refer [is testing async use-fixtures]] [datascript.core :as d] [frontend.db :as db] [frontend.handler.editor :as editor] diff --git a/src/test/frontend/worker/db_core_test.cljs b/src/test/frontend/worker/db_core_test.cljs index 0dbb3220f9..960ef15cca 100644 --- a/src/test/frontend/worker/db_core_test.cljs +++ b/src/test/frontend/worker/db_core_test.cljs @@ -52,6 +52,7 @@ :thread-api/import-db-base64 :thread-api/search-blocks :thread-api/search-upsert-blocks :thread-api/search-delete-blocks :thread-api/search-truncate-tables :thread-api/search-build-blocks-indice :thread-api/search-build-blocks-indice-in-worker :thread-api/search-build-pages-indice :thread-api/apply-outliner-ops :thread-api/sync-app-state + :thread-api/markdown-mirror-set-enabled :thread-api/markdown-mirror-flush :thread-api/markdown-mirror-regenerate :thread-api/export-get-debug-datoms :thread-api/export-get-all-page->content :thread-api/validate-db :thread-api/recompute-checksum-diagnostics :thread-api/export-edn :thread-api/import-edn :thread-api/get-view-data :thread-api/get-class-objects :thread-api/get-property-values :thread-api/get-bidirectional-properties diff --git a/src/test/frontend/worker/db_listener_test.cljs b/src/test/frontend/worker/db_listener_test.cljs index 521f58f7d6..5c5523663a 100644 --- a/src/test/frontend/worker/db_listener_test.cljs +++ b/src/test/frontend/worker/db_listener_test.cljs @@ -1,6 +1,7 @@ (ns frontend.worker.db-listener-test (:require [cljs.test :refer [deftest is testing]] - [frontend.worker.db-listener :as db-listener])) + [frontend.worker.db-listener :as db-listener] + [frontend.worker.markdown-mirror :as markdown-mirror])) (deftest transit-safe-tx-meta-keeps-outliner-ops-test (testing "worker tx-meta sanitization should preserve semantic outliner ops" @@ -14,3 +15,16 @@ (is (= outliner-ops (:outliner-ops safe-tx-meta))) (is (= outliner-ops (:db-sync/inverse-outliner-ops safe-tx-meta))) (is (nil? (:error-handler safe-tx-meta)))))) + +(deftest markdown-mirror-listener-enqueues-worker-mirror-work-test + (let [calls (atom []) + tx-report {:tx-data [:tx]}] + (with-redefs [markdown-mirror/content :thread-api/validate-db :thread-api/recompute-checksum-diagnostics :thread-api/export-edn :thread-api/import-edn :thread-api/get-view-data :thread-api/get-class-objects :thread-api/get-property-values :thread-api/get-bidirectional-properties diff --git a/src/test/frontend/worker/markdown_mirror_test.cljs b/src/test/frontend/worker/markdown_mirror_test.cljs new file mode 100644 index 0000000000..c82f0e90d4 --- /dev/null +++ b/src/test/frontend/worker/markdown_mirror_test.cljs @@ -0,0 +1,447 @@ +(ns frontend.worker.markdown-mirror-test + (:require [cljs.test :refer [async deftest is testing]] + [datascript.core :as d] + [frontend.worker.markdown-mirror :as markdown-mirror] + [logseq.db.test.helper :as db-test] + [promesa.core :as p])) + +(def test-repo "logseq_db_graph-xxx") + +(defn- fake-platform + ([] (fake-platform {:runtime :node})) + ([env] + (let [files (atom {}) + writes (atom []) + deletes (atom [])] + {:platform {:env env + :storage {:read-text! (fn [path] + (p/resolved (get @files path))) + :write-text-atomic! (fn [path content] + (swap! writes conj [path content]) + (swap! files assoc path content) + (p/resolved nil)) + :delete-file! (fn [path] + (swap! deletes conj path) + (swap! files dissoc path) + (p/resolved nil))} + :broadcast {:post-message! (fn [& _] nil)}} + :files files + :writes writes + :deletes deletes}))) + +(defn- page-path [path] + (str (markdown-mirror/repo-mirror-dir test-repo) "/" path)) + +(defn- first-block [page] + (-> page :block/_page first)) + +(defn- F\"G|H")))) + + (testing "trailing spaces and dots are removed" + (is (= "title" + (markdown-mirror/normalize-file-stem "title. ")))) + + (testing "unicode is normalized before sanitizing" + (is (= (markdown-mirror/normalize-file-stem "e\u0301") + (markdown-mirror/normalize-file-stem "\u00e9")))) + + (testing "reserved Windows device names are rejected" + (is (nil? (markdown-mirror/normalize-file-stem "CON"))) + (is (nil? (markdown-mirror/normalize-file-stem "lpt9"))))) + +(deftest same-title-pages-write-distinct-stable-friendly-paths-test + (let [page-uuid-1 #uuid "11111111-1111-4111-8111-111111111111" + page-uuid-2 #uuid "22222222-2222-4222-8222-222222222222" + conn (db-test/create-conn-with-blocks + {:pages-and-blocks [{:page {:block/title "Same Name" + :block/uuid page-uuid-1} + :blocks [{:block/title "first"}]} + {:page {:block/title "Same Name" + :block/uuid page-uuid-2} + :blocks [{:block/title "second"}]}]}) + pages (->> (d/datoms @conn :avet :block/title "Same Name") + (map #(d/entity @conn (:e %))) + (filter #(nil? (:block/page %))) + (sort-by (comp str :block/uuid))) + paths (mapv #(markdown-mirror/page-relative-path @conn %) pages)] + (is (= ["pages/Same Name.md" + "pages/Same Name (2).md"] + paths)))) + +(deftest page-references-remain-wiki-links-test + (async done + (let [{:keys [platform files]} (fake-platform) + conn (db-test/create-conn-with-blocks + {:pages-and-blocks [{:page {:block/title "Source"} + :blocks [{:block/title "See [[Foo]]"}]} + {:page {:block/title "Foo"} + :blocks [{:block/title "target"}]} + {:page {:block/title "Foo"} + :blocks [{:block/title "duplicate"}]}]}) + page (db-test/find-page-by-title @conn "Source")] + (-> (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ ( (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (p/let [_ (markdown-mirror/ (p/let [_ (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (p/let [platform (platform-node/node-platform {:root-dir root-dir}) + storage (:storage platform) + path "graph-a/markdown-mirror/pages/page.md" + _ ((:write-text-atomic! storage) path "mirror") + content ((:read-text! storage) path) + _ ((:delete-file! storage) path) + deleted-content (-> ((:read-text! storage) path) + (p/catch (constantly nil)))] + (is (= "mirror" content)) + (is (nil? deleted-content)) + (is (empty? (filter #(string/includes? % ".tmp-") + (array-seq (fs/readdirSync + (node-path/join root-dir "graphs" "graph-a" "markdown-mirror" "pages"))))))) + (p/catch (fn [e] + (is false (str "unexpected error: " e)))) + (p/finally done))))) + (deftest node-platform-cli-owner-bypasses-keychain-in-cli-e2e-test (async done (let [root-dir (node-helper/create-tmp-dir "platform-node-cli-secrets") diff --git a/src/test/frontend/worker/platform_test.cljs b/src/test/frontend/worker/platform_test.cljs index 6d121fe86b..bafdebeefd 100644 --- a/src/test/frontend/worker/platform_test.cljs +++ b/src/test/frontend/worker/platform_test.cljs @@ -1,6 +1,7 @@ (ns frontend.worker.platform-test (:require [cljs.test :refer [async deftest is]] [frontend.worker.platform :as platform] + [frontend.worker.platform.browser :as platform-browser] [promesa.core :as p])) (deftest kv-get-normalizes-undefined-to-nil-test @@ -26,3 +27,22 @@ (p/catch (fn [e] (is false (str e)))) (p/finally done)))) + +(deftest browser-platform-mirror-storage-is-unsupported-test + (let [original-location (.-location js/globalThis)] + (try + (set! (.-location js/globalThis) #js {:href "http://localhost/?electron=true" + :search "?electron=true"}) + (let [storage (:storage (platform-browser/browser-platform))] + (doseq [[f args] [[(:mirror-read-text! storage) ["mirror.md"]] + [(:write-text-atomic! storage) ["mirror.md" "content"]] + [(:delete-file! storage) ["mirror.md"]]]] + (try + (apply f args) + (is false "Expected browser mirror storage to throw") + (catch :default e + (is (= {:platform :browser + :feature :markdown-mirror} + (ex-data e))))))) + (finally + (set! (.-location js/globalThis) original-location)))))