diff --git a/deps/cli/src/logseq/cli/common/file.cljs b/deps/cli/src/logseq/cli/common/file.cljs index 0df8bff5fb..78e3d2c1f9 100644 --- a/deps/cli/src/logseq/cli/common/file.cljs +++ b/deps/cli/src/logseq/cli/common/file.cljs @@ -2,6 +2,7 @@ "Convert blocks to file content. Used for frontend exports and CLI" (:require [clojure.string :as string] [datascript.core :as d] + [datascript.impl.entity :as de] [logseq.common.util.date-time :as date-time-util] [logseq.db :as ldb] [logseq.db.frontend.content :as db-content] @@ -33,39 +34,58 @@ date-time-util/default-journal-title-formatter)))))) (defn- property-value->string - [property v context] - (cond - (and (map? v) (:db/id v)) - (str (db-property/property-value-content v)) + [db property v context] + (letfn [(entity-map [x] + (cond + (map? x) x + (de/entity? x) (into {} x))) + (entity-content [x] + (let [m (entity-map x)] + (or (:block/title m) + (:logseq.property/value m))))] + (cond + (some? (entity-content v)) + (str (entity-content v)) - (set? v) - (->> v - (sort-by (fn [item] - [(if (:block/order item) 0 1) - (str (or (:block/order item) - (property-value->string property item context)))])) - (map #(property-value->string property % context)) - (string/join ", ")) + (some? (:db/id (entity-map v))) + (let [entity (d/entity db (:db/id (entity-map v)))] + (str (or (entity-content entity) + (entity-content v) + ""))) - (sequential? v) - (->> v - (map #(property-value->string property % context)) - (string/join ", ")) + (set? v) + (->> v + (sort-by (fn [item] + [(if (:block/order item) 0 1) + (str (or (:block/order item) + (property-value->string db property item context)))])) + (map #(property-value->string db property % context)) + (string/join ", ")) - (keyword? v) - (name v) + (sequential? v) + (->> v + (map #(property-value->string db property % context)) + (string/join ", ")) - (and (= :datetime (:logseq.property/type property)) - (integer? v)) - (or (datetime-journal-title v context) - (str v)) + (keyword? v) + (name v) - (some? v) - (str v))) + (and (= :datetime (:logseq.property/type property)) + (integer? v)) + (or (datetime-journal-title v context) + (str v)) + + (some? v) + (str v)))) (defn- block-properties-content [db block spaces-tabs context] - (let [properties (->> (db-property/properties block) + (let [block (or (when-let [id (:db/id block)] + (d/entity db id)) + (when-let [uuid (:block/uuid block)] + (d/entity db [:block/uuid uuid])) + block) + properties (->> (db-property/properties block) (remove (fn [[k _]] (contains? db-property/db-attribute-properties k))) (remove (fn [[k _]] @@ -84,15 +104,34 @@ (:block/raw-title property) (name property-ident)) ":: " - (property-value->string property (get properties property-ident) context)))))) + (property-value->string db property (get properties property-ident) context)))))) (string/join "\n")))))) +(defn- property-value-block-content + [db b context] + (when-let [raw-block (d/entity db (:db/id b))] + (when-let [property (:logseq.property/created-from-property raw-block)] + (let [property-title (or (:block/title property) + (:block/raw-title property) + (some-> property :db/ident name)) + value (property-value->string db property + (or (:block/title raw-block) + (:logseq.property/value raw-block)) + context)] + (when property-title + (str property-title ":: " value)))))) + +(defn- block-title-content + [db b context] + (or (property-value-block-content db b context) + (db-content/recur-replace-uuid-in-block-title (d/entity db (:db/id b))))) + (defn- transform-content [db b level {:keys [heading-to-list? include-properties?] :or {include-properties? true}} context] (let [heading (:logseq.property/heading b) ;; replace [[uuid]] with block's content - title (db-content/recur-replace-uuid-in-block-title (d/entity db (:db/id b))) + title (block-title-content db b context) content (or title "") level (if (and heading-to-list? heading) (if (> heading 1) @@ -125,7 +164,15 @@ (if (nil? f) (->> block-contents persistent! flatten (remove nil?)) (let [page? (nil? (:block/page f)) - content (if (and page? (not link)) nil (transform-content db f level opts context)) + content (cond + (and page? (not link) (:include-page-properties? opts)) + (block-properties-content db f "" context) + + (and page? (not link)) + nil + + :else + (transform-content db f level opts context)) new-content (if-let [children (seq (:block/children f))] (cons content (tree->file-content-aux db children {:init-level (inc level)} context)) diff --git a/deps/common/src/logseq/common/graph.cljs b/deps/common/src/logseq/common/graph.cljs index 1cd38ae736..dbb28b938a 100644 --- a/deps/common/src/logseq/common/graph.cljs +++ b/deps/common/src/logseq/common/graph.cljs @@ -65,6 +65,7 @@ Rules: - Contents in '/logseq/.recycle/' are ignored - Contents in '/logseq/bak/' are ignored - Contents in with '/logseq/version-files/' are ignored +- Contents in '/markdown-mirror/' are ignored " [dir path] (let [dir (path/path-normalize dir) @@ -73,7 +74,7 @@ Rules: (when (string? path) (or (some #(string/starts-with? rpath %) - ["." "logseq/.recycle" "logseq/bak" "logseq/version-files"]) + ["." "logseq/.recycle" "logseq/bak" "logseq/version-files" "markdown-mirror"]) (contains? #{"logseq/graphs-txid.edn" "logseq/pages-metadata.edn"} rpath) (some #(string/includes? rpath (str "/" % "/")) ["node_modules"]) diff --git a/deps/common/test/logseq/common/graph_test.cljs b/deps/common/test/logseq/common/graph_test.cljs index 48cf766ae9..63b9207665 100644 --- a/deps/common/test/logseq/common/graph_test.cljs +++ b/deps/common/test/logseq/common/graph_test.cljs @@ -29,7 +29,9 @@ (fs/writeFileSync "tmp/test-graph/journals/2023_05_09.md" "") ;; Create files that are ignored (fs/mkdirSync (node-path/join "tmp/test-graph" "logseq" "bak")) + (fs/mkdirSync (node-path/join "tmp/test-graph" "markdown-mirror" "pages") #js {:recursive true}) (fs/writeFileSync "tmp/test-graph/logseq/bak/baz.md" "") (fs/writeFileSync "tmp/test-graph/logseq/.gitignore" "") + (fs/writeFileSync "tmp/test-graph/markdown-mirror/pages/foo.md" "") (is (= ["tmp/test-graph/journals/2023_05_09.md" "tmp/test-graph/pages/foo.md"] - (common-graph/get-files "tmp/test-graph")))) \ No newline at end of file + (common-graph/get-files "tmp/test-graph")))) diff --git a/docs/adr/0016-markdown-mirror.md b/docs/adr/0016-markdown-mirror.md new file mode 100644 index 0000000000..d04a19b236 --- /dev/null +++ b/docs/adr/0016-markdown-mirror.md @@ -0,0 +1,283 @@ +# ADR 0016: Electron Markdown Mirror + +Date: 2026-05-05 +Status: Proposed + +## Context +Logseq DB graphs do not expose one editable Markdown file per page in the graph +directory. Some desktop workflows still need a Markdown representation that can +be read by external tools, backed up, indexed, or inspected outside Logseq. + +The mirror must not become another graph source of truth. Editing should remain +fast, and saving a block must not wait for Markdown rendering or filesystem +writes on the renderer main thread. + +The first supported runtime is the Electron desktop app. Browser and mobile +builds do not have the same graph-directory filesystem guarantees. + +## Decision +1. Add an Electron-only Settings toggle for Markdown Mirror. +2. Persist the toggle in Electron user settings as + `:feature/markdown-mirror?`. +3. When the setting is enabled for the Electron app, Logseq writes derived + Markdown files under the current graph directory: + - journals: + `markdown-mirror/journals/.md` + - other pages: + `markdown-mirror/pages/.md` +4. For a graph at `~/logseq/graphs/graph-xxx`, mirror files are written under: + - `~/logseq/graphs/graph-xxx/markdown-mirror/journals/` + - `~/logseq/graphs/graph-xxx/markdown-mirror/pages/` +5. Markdown Mirror is derived output. The DB remains the source of truth. +6. Files under `markdown-mirror/**` must be ignored by graph import, file + watchers, and graph parsing so the mirror never feeds back into the graph. +7. The feature is not available in browser or mobile builds, even if a stale + setting value exists. +8. Settings exposes an explicit "Regenerate full mirror" action that asks the + DB worker to rewrite the complete mirror for the current graph. +9. Built-in pages and property pages, including user-created properties, are not + exported to the mirror. User Tag/Class pages are normal user content and are + exported. + +## Runtime Ownership +1. The renderer owns only: + - the Settings row + - reading and updating `:feature/markdown-mirror?` + - pushing the enabled state to the DB worker when it changes +2. The DB worker owns: + - detecting affected page ids from successful local transactions + - rendering page Markdown from the worker DB snapshot + - scheduling and coalescing mirror jobs + - running explicit full-mirror regeneration jobs + - invoking platform filesystem writes +3. The Electron main process must not render Markdown. It may provide filesystem + primitives if needed, but content generation stays with the worker. +4. Editor save paths enqueue mirror work and return immediately. They must not + wait for rendering, directory creation, stat, write, rename, or delete. + +## Reusable Core and CLI Path +1. Markdown Mirror path planning, filename normalization, page rendering, write + deduplication, atomic writes, rename cleanup, and delete cleanup live in a + worker/core namespace that does not depend on Electron UI state. +2. The Electron app only owns feature activation through Settings. +3. The CLI should be able to reuse the same core by passing an explicit graph, + DB snapshot, and node filesystem platform context. +4. Future CLI support should not introduce a second Markdown serializer or a + different filename normalization policy. +5. Future CLI support should reuse the same mirror-path allocation index so the + Electron app and CLI do not produce different file names for the same graph. + +## Output Layout and Naming +1. Journal pages are written below `markdown-mirror/journals/`. +2. Non-journal pages are written below `markdown-mirror/pages/`. +3. Journal file names use the existing Logseq journal file-name rules for the + graph configuration. +4. Non-journal page file names use the normalized page title: + `.md`. +5. Page file names must stay friendly to external Markdown tools such as Emacs, + VS Code, and Obsidian. Do not include page uuid in normal mirror file names. +6. Page title is not page identity. The page uuid is still the internal mirror + identity, but it is stored in the mirror index rather than exposed in the + file name. +7. Duplicate non-journal page titles are handled by stable title suffix + allocation: + - first allocated page: `pages/Foo.md` + - second allocated page: `pages/Foo (2).md` + - third allocated page: `pages/Foo (3).md` +8. Once a page uuid is assigned a mirror path, keep that path stable until the + page is renamed or deleted. Do not renumber existing duplicate-title mirror + paths when another duplicate is created or removed. +9. The implementation keeps a per-graph mirror index under + `markdown-mirror/.index.edn`. +10. The mirror index stores at least: + - page uuid -> relative mirror path + - relative mirror path -> page uuid + - page uuid -> last known normalized title stem +11. The mirror index is implementation metadata for path stability. It is not + graph content and must be ignored by graph import and watchers along with the + rest of `markdown-mirror/**`. +12. All mirror file names pass through a single cross-platform filename + normalizer before joining paths. +13. Duplicate journal-day entities indicate invalid graph state for the mirror. + The implementation must fail those journal mirror jobs and surface a + diagnostic instead of choosing a winner. +14. If two entities still map to the same mirror path, the implementation must + fail the mirror job for that path and surface a diagnostic instead of + overwriting an unrelated page. +15. Page rename moves the mirror by writing the new path, updating the mirror + index, and deleting the old path after the new file has been written. +16. Page deletion deletes the corresponding mirror file and removes the page uuid + from the mirror index. +17. The write guard must reject any computed path outside the graph's + `markdown-mirror/` directory. +18. Built-in pages and property pages are excluded from path allocation and + mirror writes. User Tag/Class pages are not excluded by this rule. If a + previously mirrored page becomes excluded, the old mirror file is removed. + +## Duplicate Page Title Allocation +1. For non-journal pages, compute the normalized title stem first. +2. If the page uuid already exists in the mirror index and the normalized title + stem did not change, reuse the indexed path. +3. If the page uuid is new for that title, allocate the first unused path in this + sequence: + - `pages/.md` + - `pages/ (2).md` + - `pages/ (3).md` +4. A path is considered unavailable when the mirror index maps it to a different + live page uuid. +5. Deleted page paths become available for future allocation only after the + deleted page uuid is removed from the index. +6. Rename is treated as a new allocation for the new title stem. Existing pages + with the old title keep their already allocated paths. +7. If the mirror index is missing or unreadable, rebuild it from the current DB + in deterministic page order before writing. Deterministic order should use a + stable key such as page title plus page uuid. +8. The rebuilt index is allowed to choose paths for pages that had no previous + allocation. It must not overwrite a live existing path that is already mapped + to another page uuid. + +## Rename and Delete +1. Page rename moves the mirror by writing the new path and deleting the old + path after the new file has been written. +2. Page deletion deletes the corresponding mirror file. +3. The implementation keeps a small per-graph mirror index keyed by page uuid so + rename and delete handling does not require scanning the mirror directory on + every transaction. + +## Filename Normalization +1. Mirror file names must be portable across macOS, Windows, and Linux. +2. Use one shared normalizer for journal and page mirror file names. +3. The normalizer must: + - reject or replace path separators (`/`, `\`) + - reject or replace Windows-invalid characters (`<`, `>`, `:`, `"`, `|`, + `?`, `*`) and ASCII control characters + - reject or rewrite reserved Windows device names such as `CON`, `PRN`, + `AUX`, `NUL`, `COM1` through `COM9`, and `LPT1` through `LPT9` + - trim trailing spaces and dots because Windows does not preserve them + - reject empty names after normalization + - bound each file-name component to a safe byte length before appending + `.md` +4. Normalize Unicode to one canonical form before sanitizing so the same page + title produces the same mirror path across filesystems with different Unicode + normalization behavior. +5. The normalizer must be deterministic and must not depend on the current + operating system. A graph mirrored on macOS should choose the same logical + file name as the same graph mirrored on Windows. +6. If normalization changes the display title segment, the mirror index and + duplicate-title suffix allocation still preserve identity for non-journal + pages. +7. If a journal title normalizes to an unsafe or colliding file name, fail the + journal mirror job and surface diagnostics instead of inventing a fallback + name. +8. Path construction must join only validated path components. It must never + concatenate unchecked page titles into filesystem paths. + +## Scheduling and Performance +1. Mirror rendering is incremental. A transaction schedules only pages affected + by that transaction. +2. Jobs are coalesced by page uuid. If a page is edited repeatedly before its + job runs, only the latest worker DB state is rendered. +3. Scheduling uses a short debounce window per graph to reduce write churn while + preserving near-real-time output. +4. Mirror writes are serialized per graph to avoid path races during rename and + delete. +5. Before writing, compare the generated Markdown with the current file content + or with the last written content hash. Skip the write when content is + unchanged. +6. Write files atomically: + - ensure the target directory exists + - write to a temporary file in the same directory + - rename the temporary file over the target +7. Heavy work is forbidden on the renderer main thread: + - no full-graph export + - no Markdown rendering + - no filesystem reads or writes + - no synchronous IPC waiting for mirror completion +8. Full regeneration is an explicit Settings action. The renderer only sends a + worker request; page selection, rendering, and filesystem writes stay in the + DB worker. +9. Enabling the setting starts incremental mirroring for subsequent page edits. + It does not implicitly run full regeneration. + +## Markdown Content +1. Reuse the existing page-to-Markdown export pipeline used by worker export + APIs instead of introducing a separate renderer-side serializer. +2. The mirror output should match normal Markdown export semantics for page + content. +3. Mirror files do not include Logseq-internal mirror metadata in the Markdown + body. +4. Mirror files include block and page property drawers, including user + properties, with rendered property values. +5. Assets are referenced as normal exported Markdown references. This ADR does + not copy assets into `markdown-mirror/`. +6. Page references remain in Logseq wiki-link form, for example `[[Foo]]`. +7. Ambiguous page references caused by duplicate page titles are an accepted + limitation of Markdown Mirror. Do not rewrite page references to uuid-based + links or relative Markdown links in this ADR. + +## Failure Handling +1. Filesystem and path errors fail the mirror job for the affected page. +2. Failures are logged with graph, page uuid, target path, and error details. +3. Repeated failures should be visible from Settings or diagnostics; do not show + a toast on every keystroke. +4. The feature must not silently fall back to browser storage, OPFS, or another + output directory. +5. If the graph directory is not available, the worker rejects mirror jobs for + that graph until the graph is reopened with a valid directory. + +## Non-goals +1. Markdown Mirror is not bidirectional sync. +2. Editing files in `markdown-mirror/` does not update the graph. +3. The mirror is not a backup format with guaranteed import fidelity. +4. The mirror does not replace existing graph export features. +5. The mirror does not support browser or mobile runtimes in this ADR. + +## Consequences + +### Positive +- Desktop users get a readable Markdown projection inside the graph directory. +- Editor latency is protected because rendering and disk I/O are worker-owned + and asynchronous. +- The output layout is predictable for tools that watch journals and pages + separately. +- Page file names remain readable and practical in external Markdown tools. +- Ignoring `markdown-mirror/**` prevents mirror-generated files from becoming + graph input. + +### Tradeoffs +- The mirror can lag slightly behind the latest edit because writes are + debounced and serialized. +- A per-graph mirror index is needed for reliable rename and delete cleanup. +- Duplicate page references such as `[[Foo]]` remain ambiguous in mirror output. +- The first version does not backfill every existing page automatically when the + setting is enabled; users run full regeneration explicitly. +- External edits to mirror files are overwritten by later Logseq edits. +- Property pages are intentionally absent from the mirror, so the output is not + a complete DB export even though page and block property drawers are included. + +## Verification +Implementation should add focused coverage for: + +```bash +bb dev:test -v frontend.worker.markdown-mirror-test/enabled-electron-edit-writes-page-mirror-test +bb dev:test -v frontend.worker.markdown-mirror-test/enabled-electron-edit-writes-journal-mirror-test +bb dev:test -v frontend.worker.markdown-mirror-test/disabled-setting-does-not-write-mirror-test +bb dev:test -v frontend.worker.markdown-mirror-test/repeated-edits-coalesce-to-latest-content-test +bb dev:test -v frontend.worker.markdown-mirror-test/rename-removes-old-mirror-path-test +bb dev:test -v frontend.worker.markdown-mirror-test/delete-removes-mirror-file-test +bb dev:test -v frontend.worker.markdown-mirror-test/same-title-pages-write-distinct-stable-friendly-paths-test +bb dev:test -v frontend.worker.markdown-mirror-test/page-references-remain-wiki-links-test +bb dev:test -v frontend.worker.markdown-mirror-test/page-mirror-exports-property-values-test +bb dev:test -v frontend.worker.markdown-mirror-test/page-mirror-exports-page-property-values-test +bb dev:test -v frontend.worker.markdown-mirror-test/full-regeneration-writes-existing-non-built-in-non-property-pages-test +bb dev:test -v frontend.worker.markdown-mirror-test/invalid-filename-characters-are-normalized-test +bb dev:test -v frontend.worker.markdown-mirror-test/windows-reserved-filename-fails-with-diagnostic-test +bb dev:test -v frontend.worker.markdown-mirror-test/mirror-path-collision-fails-without-overwrite-test +``` + +Additional checks: +- `markdown-mirror/**` is excluded from graph parsing and file watchers. +- Editor save does not await mirror completion. +- Browser and mobile builds do not expose the setting and do not schedule mirror + jobs. +- Atomic write failures do not leave partial target files. diff --git a/src/electron/electron/handler.cljs b/src/electron/electron/handler.cljs index beac372b75..c45baa5b6e 100644 --- a/src/electron/electron/handler.cljs +++ b/src/electron/electron/handler.cljs @@ -315,7 +315,8 @@ (do (cfgs/set-item! k v) (when (= k :spell-check) (spell-check/apply-window-spellcheck! window (spell-check/session-spellcheck-enabled? v))) - (state/set-state! [:config k] v)) + (state/set-state! [:config k] v) + nil) (cfgs/get-item k)) config))) diff --git a/src/main/frontend/components/settings.cljs b/src/main/frontend/components/settings.cljs index 798f565ad9..ffd6fc1db2 100644 --- a/src/main/frontend/components/settings.cljs +++ b/src/main/frontend/components/settings.cljs @@ -745,6 +745,51 @@ {:left-label (t :settings.features/enable-flashcards) :action (flashcards-enabled-switcher enable-flashcards?)})) +(rum/defcs markdown-mirror-row < rum/reactive + (rum/local false ::regenerating?) + [state t] + (let [enabled? (true? (state/sub [:electron/user-cfgs :feature/markdown-mirror?])) + *regenerating? (::regenerating? state) + regenerate! (fn [] + (let [repo (state/get-current-repo)] + (when (and repo @state/*db-worker (not @*regenerating?)) + (reset! *regenerating? true) + (-> (state/ (state/ (state/ (p/let [_ (state/ (p/let [_ (state/encoded-graph-dir-name repo) "/markdown-mirror")) + +(def ^:private invalid-file-name-chars-re + #"[<>:\"|?*\\/]") + +(def ^:private ascii-control-re + #"[\x00-\x1F]") + +(def ^:private trailing-space-or-dot-re + #"[ \.]+$") + +(def ^:private reserved-windows-device-names + (into #{"CON" "PRN" "AUX" "NUL"} + (concat (map #(str "COM" %) (range 1 10)) + (map #(str "LPT" %) (range 1 10))))) + +(def ^:private max-file-stem-length 160) + +(defonce ^:private *repo->enabled? (atom {})) +(defonce ^:private *repo->queued-page-jobs (atom {})) +(defonce ^:private *repo->flush-timeout (atom {})) + +(defn- normalize-unicode + [s] + (let [s (str s)] + (if (fn? (.-normalize s)) + (.normalize s "NFC") + s))) + +(defn- reserved-windows-device-name? + [s] + (contains? reserved-windows-device-names + (string/upper-case s))) + +(defn normalize-file-stem + [s] + (when (some? s) + (let [s' (-> (normalize-unicode s) + (string/replace invalid-file-name-chars-re "_") + (string/replace ascii-control-re "_") + (string/replace trailing-space-or-dot-re "")) + s' (if (> (count s') max-file-stem-length) + (subs s' 0 max-file-stem-length) + s')] + (when (and (not (string/blank? s')) + (not (reserved-windows-device-name? s'))) + s')))) + +(defn- journal-file-stem + [journal-day] + (when journal-day + (let [s (str journal-day)] + (when (= 8 (count s)) + (str (subs s 0 4) "_" (subs s 4 6) "_" (subs s 6 8)))))) + +(defn page-relative-path + ([db page] + (page-relative-path db page {})) + ([db page {:keys [journal-file-stem-fn] + :or {journal-file-stem-fn journal-file-stem}}] + (when page + (if (ldb/journal? page) + (when-let [stem (normalize-file-stem (journal-file-stem-fn (:block/journal-day page)))] + (str "journals/" stem ".md")) + (when-let [stem (normalize-file-stem (:block/title page))] + (let [duplicate-pages (->> (d/datoms db :avet :block/title (:block/title page)) + (map #(d/entity db (:e %))) + (filter #(and (ldb/page? %) + (not (ldb/journal? %)))) + (sort-by (comp str :block/uuid))) + index (inc (or (first (keep-indexed + (fn [idx p] + (when (= (:block/uuid page) (:block/uuid p)) + idx)) + duplicate-pages)) + 0)) + stem' (if (= 1 index) + stem + (str stem " (" index ")"))] + (str "pages/" stem' ".md"))))))) + +(defn- mirror-path + [repo relative-path] + (str (repo-mirror-dir repo) "/" relative-path)) + +(defn- page-id-for-entity + [db eid] + (when-let [entity (d/entity db eid)] + (cond + (ldb/page? entity) (:db/id entity) + (:block/page entity) (:db/id (:block/page entity)) + (and (:block/parent entity) (ldb/page? (:block/parent entity))) (:db/id (:block/parent entity)) + (some-> entity :block/parent :block/page) (:db/id (:block/page (:block/parent entity)))))) + +(defn affected-page-ids + [{:keys [db-before db-after tx-data]}] + (->> tx-data + (mapcat (fn [{:keys [e a v]}] + (cond-> [(page-id-for-entity db-before e) + (page-id-for-entity db-after e)] + (= a :block/page) + (conj v)))) + (remove nil?) + set)) + +(defn set-enabled! + [repo enabled?] + (if enabled? + (swap! *repo->enabled? assoc repo true) + (do + (when-let [timeout-id (get @*repo->flush-timeout repo)] + (js/clearTimeout timeout-id)) + (swap! *repo->enabled? dissoc repo) + (swap! *repo->queued-page-jobs dissoc repo) + (swap! *repo->flush-timeout dissoc repo))) + nil) + +(defn enabled? + [repo] + (true? (get @*repo->enabled? repo))) + +(defn- storage + [platform*] + (:storage platform*)) + +(defn- (f path) + (p/catch (constantly nil))) + (p/rejected (ex-info "platform storage/read-text! missing" {:path path})))) + +(defn- content + db + (:block/uuid page) + {:include-page-properties? true} + {:export-bullet-indentation (or (:export-bullet-indentation options) " ") + :date-formatter (:date-formatter options)})) + +(defn- mirrorable-page? + [page] + (and (ldb/page? page) + (not (ldb/built-in? page)) + (not (ldb/property? page)) + (not (ldb/hidden? page)) + (not (:logseq.property.user/email page)))) + +(defn- mirrorable-pages + [db] + (->> (d/datoms db :avet :block/name) + (map #(d/entity db (:e %))) + (filter mirrorable-page?) + (sort-by (fn [page] + [(if (ldb/journal? page) 0 1) + (str (:block/journal-day page)) + (string/lower-case (or (:block/title page) "")) + (str (:block/uuid page))])))) + +(defn- queued-page-jobs update-in [repo (:page-id job)] merge-job job)) + +(defn- drain-repo-jobs! + [repo] + (let [jobs (vals (get @*repo->queued-page-jobs repo))] + (swap! *repo->queued-page-jobs dissoc repo) + jobs)) + +(declare flush-timeout repo) + (let [timeout-id (js/setTimeout + (fn [] + (swap! *repo->flush-timeout dissoc repo) + (-> (flush-timeout assoc repo timeout-id)))) + +(defn- (.unlink (browser-pfs) (asset-path repo file-name)) - (p/catch (constantly nil)))) + (let [^js pfs (browser-pfs)] + (-> (.unlink pfs (asset-path repo file-name)) + (p/catch (constantly nil))))) + +(defn- unsupported-mirror-storage! + [& _args] + (throw (ex-info "Markdown mirror storage is not supported in browser workers" + {:platform :browser + :feature :markdown-mirror}))) (defn- websocket-connect [url] @@ -201,6 +208,9 @@ :remove-vfs! remove-vfs! :read-text! read-text! :write-text! write-text! + :write-text-atomic! unsupported-mirror-storage! + :delete-file! unsupported-mirror-storage! + :mirror-read-text! unsupported-mirror-storage! :asset-read-bytes! asset-read-bytes! :asset-write-bytes! asset-write-bytes! :asset-stat asset-stat diff --git a/src/main/frontend/worker/platform/node.cljs b/src/main/frontend/worker/platform/node.cljs index a92ba9adaf..61525b2cab 100644 --- a/src/main/frontend/worker/platform/node.cljs +++ b/src/main/frontend/worker/platform/node.cljs @@ -252,6 +252,26 @@ _ (ensure-dir! dir)] (fs/writeFile full-path text "utf8")))) +(defn- write-text-atomic! + [write-guard-fn data-dir path text] + (let [full-path (path-under-data-dir data-dir path) + dir (node-path/dirname full-path) + tmp-path (node-path/join dir (str "." (node-path/basename full-path) ".tmp-" (random-uuid)))] + (p/let [_ (when write-guard-fn + (write-guard-fn)) + _ (ensure-dir! dir) + _ (fs/writeFile tmp-path text "utf8") + _ (fs/rename tmp-path full-path)] + nil))) + +(defn- delete-file! + [write-guard-fn data-dir path] + (let [full-path (path-under-data-dir data-dir path)] + (p/let [_ (when write-guard-fn + (write-guard-fn))] + (-> (fs/rm full-path #js {:force true}) + (p/catch (constantly nil)))))) + (defn- asset-file-path [data-dir repo file-name] (node-path/join (repo-dir data-dir repo) @@ -433,6 +453,8 @@ :remove-vfs! (fn [pool] (remove-vfs! pool)) :read-text! (fn [path] (read-text! data-dir path)) :write-text! (fn [path text] (write-text! write-guard-fn data-dir path text)) + :write-text-atomic! (fn [path text] (write-text-atomic! write-guard-fn data-dir path text)) + :delete-file! (fn [path] (delete-file! write-guard-fn data-dir path)) :asset-read-bytes! (fn [repo file-name] (asset-read-bytes! data-dir repo file-name)) :asset-write-bytes! (fn [repo file-name payload] diff --git a/src/resources/dicts/en.edn b/src/resources/dicts/en.edn index 2f4541c532..ec54bf5916 100644 --- a/src/resources/dicts/en.edn +++ b/src/resources/dicts/en.edn @@ -1631,6 +1631,11 @@ :settings.features/home-default-page-update-success "Home default page updated successfully!" :settings.features/journals-enable-success "Journals enabled" :settings.features/login-prompt "To access new features before anyone else you must be an Open Collective Sponsor or Backer of Logseq and therefore log in first." + :settings.features/markdown-mirror "Markdown Mirror" + :settings.features/markdown-mirror-desc "Write a derived Markdown copy of edited pages to the graph's markdown-mirror folder. Desktop only." + :settings.features/markdown-mirror-regenerate "Regenerate full mirror" + :settings.features/markdown-mirror-regenerate-error "Failed to regenerate Markdown Mirror: {1}" + :settings.features/markdown-mirror-regenerate-success "Markdown Mirror regenerated" :settings.features/page-not-found "The page \"{1}\" doesn't exist yet. Please create that page first, and then try again." :settings.features/plugin-system "Plugins" diff --git a/src/resources/dicts/zh-cn.edn b/src/resources/dicts/zh-cn.edn index f6e7af6e90..79c92157ed 100644 --- a/src/resources/dicts/zh-cn.edn +++ b/src/resources/dicts/zh-cn.edn @@ -1621,6 +1621,11 @@ :settings.features/home-default-page-update-success "主页默认页面已更新成功!" :settings.features/journals-enable-success "已启用日志页" :settings.features/login-prompt "你必须是 Logseq 的 Open Collective Sponsor 或者 Backer 才能提前使用新功能(仍在测试中),因此需要登录。" + :settings.features/markdown-mirror "Markdown 镜像" + :settings.features/markdown-mirror-desc "将已编辑页面的派生 Markdown 副本写入图谱的 markdown-mirror 文件夹。仅桌面端可用。" + :settings.features/markdown-mirror-regenerate "重新生成完整镜像" + :settings.features/markdown-mirror-regenerate-error "重新生成 Markdown 镜像失败:{1}" + :settings.features/markdown-mirror-regenerate-success "Markdown 镜像已重新生成" :settings.features/page-not-found "页面“{1}”尚不存在。请先创建该页面,然后再试一次。" :settings.features/plugin-system "插件系统" diff --git a/src/test/frontend/handler/editor_async_test.cljs b/src/test/frontend/handler/editor_async_test.cljs index 03d790dd4a..b1c3e4cd04 100644 --- a/src/test/frontend/handler/editor_async_test.cljs +++ b/src/test/frontend/handler/editor_async_test.cljs @@ -1,5 +1,5 @@ (ns frontend.handler.editor-async-test - (:require [clojure.test :refer [is testing async use-fixtures]] + (:require [cljs.test :refer [is testing async use-fixtures]] [datascript.core :as d] [frontend.db :as db] [frontend.handler.editor :as editor] diff --git a/src/test/frontend/worker/db_core_test.cljs b/src/test/frontend/worker/db_core_test.cljs index 0dbb3220f9..960ef15cca 100644 --- a/src/test/frontend/worker/db_core_test.cljs +++ b/src/test/frontend/worker/db_core_test.cljs @@ -52,6 +52,7 @@ :thread-api/import-db-base64 :thread-api/search-blocks :thread-api/search-upsert-blocks :thread-api/search-delete-blocks :thread-api/search-truncate-tables :thread-api/search-build-blocks-indice :thread-api/search-build-blocks-indice-in-worker :thread-api/search-build-pages-indice :thread-api/apply-outliner-ops :thread-api/sync-app-state + :thread-api/markdown-mirror-set-enabled :thread-api/markdown-mirror-flush :thread-api/markdown-mirror-regenerate :thread-api/export-get-debug-datoms :thread-api/export-get-all-page->content :thread-api/validate-db :thread-api/recompute-checksum-diagnostics :thread-api/export-edn :thread-api/import-edn :thread-api/get-view-data :thread-api/get-class-objects :thread-api/get-property-values :thread-api/get-bidirectional-properties diff --git a/src/test/frontend/worker/db_listener_test.cljs b/src/test/frontend/worker/db_listener_test.cljs index 521f58f7d6..5c5523663a 100644 --- a/src/test/frontend/worker/db_listener_test.cljs +++ b/src/test/frontend/worker/db_listener_test.cljs @@ -1,6 +1,7 @@ (ns frontend.worker.db-listener-test (:require [cljs.test :refer [deftest is testing]] - [frontend.worker.db-listener :as db-listener])) + [frontend.worker.db-listener :as db-listener] + [frontend.worker.markdown-mirror :as markdown-mirror])) (deftest transit-safe-tx-meta-keeps-outliner-ops-test (testing "worker tx-meta sanitization should preserve semantic outliner ops" @@ -14,3 +15,16 @@ (is (= outliner-ops (:outliner-ops safe-tx-meta))) (is (= outliner-ops (:db-sync/inverse-outliner-ops safe-tx-meta))) (is (nil? (:error-handler safe-tx-meta)))))) + +(deftest markdown-mirror-listener-enqueues-worker-mirror-work-test + (let [calls (atom []) + tx-report {:tx-data [:tx]}] + (with-redefs [markdown-mirror/content :thread-api/validate-db :thread-api/recompute-checksum-diagnostics :thread-api/export-edn :thread-api/import-edn :thread-api/get-view-data :thread-api/get-class-objects :thread-api/get-property-values :thread-api/get-bidirectional-properties diff --git a/src/test/frontend/worker/markdown_mirror_test.cljs b/src/test/frontend/worker/markdown_mirror_test.cljs new file mode 100644 index 0000000000..c82f0e90d4 --- /dev/null +++ b/src/test/frontend/worker/markdown_mirror_test.cljs @@ -0,0 +1,447 @@ +(ns frontend.worker.markdown-mirror-test + (:require [cljs.test :refer [async deftest is testing]] + [datascript.core :as d] + [frontend.worker.markdown-mirror :as markdown-mirror] + [logseq.db.test.helper :as db-test] + [promesa.core :as p])) + +(def test-repo "logseq_db_graph-xxx") + +(defn- fake-platform + ([] (fake-platform {:runtime :node})) + ([env] + (let [files (atom {}) + writes (atom []) + deletes (atom [])] + {:platform {:env env + :storage {:read-text! (fn [path] + (p/resolved (get @files path))) + :write-text-atomic! (fn [path content] + (swap! writes conj [path content]) + (swap! files assoc path content) + (p/resolved nil)) + :delete-file! (fn [path] + (swap! deletes conj path) + (swap! files dissoc path) + (p/resolved nil))} + :broadcast {:post-message! (fn [& _] nil)}} + :files files + :writes writes + :deletes deletes}))) + +(defn- page-path [path] + (str (markdown-mirror/repo-mirror-dir test-repo) "/" path)) + +(defn- first-block [page] + (-> page :block/_page first)) + +(defn- F\"G|H")))) + + (testing "trailing spaces and dots are removed" + (is (= "title" + (markdown-mirror/normalize-file-stem "title. ")))) + + (testing "unicode is normalized before sanitizing" + (is (= (markdown-mirror/normalize-file-stem "e\u0301") + (markdown-mirror/normalize-file-stem "\u00e9")))) + + (testing "reserved Windows device names are rejected" + (is (nil? (markdown-mirror/normalize-file-stem "CON"))) + (is (nil? (markdown-mirror/normalize-file-stem "lpt9"))))) + +(deftest same-title-pages-write-distinct-stable-friendly-paths-test + (let [page-uuid-1 #uuid "11111111-1111-4111-8111-111111111111" + page-uuid-2 #uuid "22222222-2222-4222-8222-222222222222" + conn (db-test/create-conn-with-blocks + {:pages-and-blocks [{:page {:block/title "Same Name" + :block/uuid page-uuid-1} + :blocks [{:block/title "first"}]} + {:page {:block/title "Same Name" + :block/uuid page-uuid-2} + :blocks [{:block/title "second"}]}]}) + pages (->> (d/datoms @conn :avet :block/title "Same Name") + (map #(d/entity @conn (:e %))) + (filter #(nil? (:block/page %))) + (sort-by (comp str :block/uuid))) + paths (mapv #(markdown-mirror/page-relative-path @conn %) pages)] + (is (= ["pages/Same Name.md" + "pages/Same Name (2).md"] + paths)))) + +(deftest page-references-remain-wiki-links-test + (async done + (let [{:keys [platform files]} (fake-platform) + conn (db-test/create-conn-with-blocks + {:pages-and-blocks [{:page {:block/title "Source"} + :blocks [{:block/title "See [[Foo]]"}]} + {:page {:block/title "Foo"} + :blocks [{:block/title "target"}]} + {:page {:block/title "Foo"} + :blocks [{:block/title "duplicate"}]}]}) + page (db-test/find-page-by-title @conn "Source")] + (-> (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ ( (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (p/let [_ (markdown-mirror/ (p/let [_ (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (p/let [platform (platform-node/node-platform {:root-dir root-dir}) + storage (:storage platform) + path "graph-a/markdown-mirror/pages/page.md" + _ ((:write-text-atomic! storage) path "mirror") + content ((:read-text! storage) path) + _ ((:delete-file! storage) path) + deleted-content (-> ((:read-text! storage) path) + (p/catch (constantly nil)))] + (is (= "mirror" content)) + (is (nil? deleted-content)) + (is (empty? (filter #(string/includes? % ".tmp-") + (array-seq (fs/readdirSync + (node-path/join root-dir "graphs" "graph-a" "markdown-mirror" "pages"))))))) + (p/catch (fn [e] + (is false (str "unexpected error: " e)))) + (p/finally done))))) + (deftest node-platform-cli-owner-bypasses-keychain-in-cli-e2e-test (async done (let [root-dir (node-helper/create-tmp-dir "platform-node-cli-secrets") diff --git a/src/test/frontend/worker/platform_test.cljs b/src/test/frontend/worker/platform_test.cljs index 6d121fe86b..bafdebeefd 100644 --- a/src/test/frontend/worker/platform_test.cljs +++ b/src/test/frontend/worker/platform_test.cljs @@ -1,6 +1,7 @@ (ns frontend.worker.platform-test (:require [cljs.test :refer [async deftest is]] [frontend.worker.platform :as platform] + [frontend.worker.platform.browser :as platform-browser] [promesa.core :as p])) (deftest kv-get-normalizes-undefined-to-nil-test @@ -26,3 +27,22 @@ (p/catch (fn [e] (is false (str e)))) (p/finally done)))) + +(deftest browser-platform-mirror-storage-is-unsupported-test + (let [original-location (.-location js/globalThis)] + (try + (set! (.-location js/globalThis) #js {:href "http://localhost/?electron=true" + :search "?electron=true"}) + (let [storage (:storage (platform-browser/browser-platform))] + (doseq [[f args] [[(:mirror-read-text! storage) ["mirror.md"]] + [(:write-text-atomic! storage) ["mirror.md" "content"]] + [(:delete-file! storage) ["mirror.md"]]]] + (try + (apply f args) + (is false "Expected browser mirror storage to throw") + (catch :default e + (is (= {:platform :browser + :feature :markdown-mirror} + (ex-data e))))))) + (finally + (set! (.-location js/globalThis) original-location)))))