feat: add markdown mirror

Add markdown mirror generation for DB graphs, including page and journal paths, regeneration, debounced write handling, and settings UI.

Serialize page and block property values into mirrored Markdown by resolving property-value entities to their display content.

Persist mirror files through the node worker platform and make browser worker mirror storage fail fast as unsupported.

Fix Electron userAppCfgs writes to avoid returning unserializable Electron state over IPC.

Add ADR and targeted tests for mirror generation, worker wiring, platform storage, and graph path handling.
This commit is contained in:
Tienson Qin
2026-05-05 19:27:00 +08:00
parent 7a731c6044
commit 06dbef8715
21 changed files with 1351 additions and 37 deletions

View File

@@ -2,6 +2,7 @@
"Convert blocks to file content. Used for frontend exports and CLI"
(:require [clojure.string :as string]
[datascript.core :as d]
[datascript.impl.entity :as de]
[logseq.common.util.date-time :as date-time-util]
[logseq.db :as ldb]
[logseq.db.frontend.content :as db-content]
@@ -33,39 +34,58 @@
date-time-util/default-journal-title-formatter))))))
(defn- property-value->string
[property v context]
(cond
(and (map? v) (:db/id v))
(str (db-property/property-value-content v))
[db property v context]
(letfn [(entity-map [x]
(cond
(map? x) x
(de/entity? x) (into {} x)))
(entity-content [x]
(let [m (entity-map x)]
(or (:block/title m)
(:logseq.property/value m))))]
(cond
(some? (entity-content v))
(str (entity-content v))
(set? v)
(->> v
(sort-by (fn [item]
[(if (:block/order item) 0 1)
(str (or (:block/order item)
(property-value->string property item context)))]))
(map #(property-value->string property % context))
(string/join ", "))
(some? (:db/id (entity-map v)))
(let [entity (d/entity db (:db/id (entity-map v)))]
(str (or (entity-content entity)
(entity-content v)
"")))
(sequential? v)
(->> v
(map #(property-value->string property % context))
(string/join ", "))
(set? v)
(->> v
(sort-by (fn [item]
[(if (:block/order item) 0 1)
(str (or (:block/order item)
(property-value->string db property item context)))]))
(map #(property-value->string db property % context))
(string/join ", "))
(keyword? v)
(name v)
(sequential? v)
(->> v
(map #(property-value->string db property % context))
(string/join ", "))
(and (= :datetime (:logseq.property/type property))
(integer? v))
(or (datetime-journal-title v context)
(str v))
(keyword? v)
(name v)
(some? v)
(str v)))
(and (= :datetime (:logseq.property/type property))
(integer? v))
(or (datetime-journal-title v context)
(str v))
(some? v)
(str v))))
(defn- block-properties-content
[db block spaces-tabs context]
(let [properties (->> (db-property/properties block)
(let [block (or (when-let [id (:db/id block)]
(d/entity db id))
(when-let [uuid (:block/uuid block)]
(d/entity db [:block/uuid uuid]))
block)
properties (->> (db-property/properties block)
(remove (fn [[k _]]
(contains? db-property/db-attribute-properties k)))
(remove (fn [[k _]]
@@ -84,15 +104,34 @@
(:block/raw-title property)
(name property-ident))
":: "
(property-value->string property (get properties property-ident) context))))))
(property-value->string db property (get properties property-ident) context))))))
(string/join "\n"))))))
(defn- property-value-block-content
[db b context]
(when-let [raw-block (d/entity db (:db/id b))]
(when-let [property (:logseq.property/created-from-property raw-block)]
(let [property-title (or (:block/title property)
(:block/raw-title property)
(some-> property :db/ident name))
value (property-value->string db property
(or (:block/title raw-block)
(:logseq.property/value raw-block))
context)]
(when property-title
(str property-title ":: " value))))))
(defn- block-title-content
[db b context]
(or (property-value-block-content db b context)
(db-content/recur-replace-uuid-in-block-title (d/entity db (:db/id b)))))
(defn- transform-content
[db b level {:keys [heading-to-list? include-properties?]
:or {include-properties? true}} context]
(let [heading (:logseq.property/heading b)
;; replace [[uuid]] with block's content
title (db-content/recur-replace-uuid-in-block-title (d/entity db (:db/id b)))
title (block-title-content db b context)
content (or title "")
level (if (and heading-to-list? heading)
(if (> heading 1)
@@ -125,7 +164,15 @@
(if (nil? f)
(->> block-contents persistent! flatten (remove nil?))
(let [page? (nil? (:block/page f))
content (if (and page? (not link)) nil (transform-content db f level opts context))
content (cond
(and page? (not link) (:include-page-properties? opts))
(block-properties-content db f "" context)
(and page? (not link))
nil
:else
(transform-content db f level opts context))
new-content
(if-let [children (seq (:block/children f))]
(cons content (tree->file-content-aux db children {:init-level (inc level)} context))

View File

@@ -65,6 +65,7 @@ Rules:
- Contents in '/logseq/.recycle/' are ignored
- Contents in '/logseq/bak/' are ignored
- Contents in with '/logseq/version-files/' are ignored
- Contents in '/markdown-mirror/' are ignored
"
[dir path]
(let [dir (path/path-normalize dir)
@@ -73,7 +74,7 @@ Rules:
(when (string? path)
(or
(some #(string/starts-with? rpath %)
["." "logseq/.recycle" "logseq/bak" "logseq/version-files"])
["." "logseq/.recycle" "logseq/bak" "logseq/version-files" "markdown-mirror"])
(contains? #{"logseq/graphs-txid.edn" "logseq/pages-metadata.edn"} rpath)
(some #(string/includes? rpath (str "/" % "/"))
["node_modules"])

View File

@@ -29,7 +29,9 @@
(fs/writeFileSync "tmp/test-graph/journals/2023_05_09.md" "")
;; Create files that are ignored
(fs/mkdirSync (node-path/join "tmp/test-graph" "logseq" "bak"))
(fs/mkdirSync (node-path/join "tmp/test-graph" "markdown-mirror" "pages") #js {:recursive true})
(fs/writeFileSync "tmp/test-graph/logseq/bak/baz.md" "")
(fs/writeFileSync "tmp/test-graph/logseq/.gitignore" "")
(fs/writeFileSync "tmp/test-graph/markdown-mirror/pages/foo.md" "")
(is (= ["tmp/test-graph/journals/2023_05_09.md" "tmp/test-graph/pages/foo.md"]
(common-graph/get-files "tmp/test-graph"))))
(common-graph/get-files "tmp/test-graph"))))

View File

@@ -0,0 +1,283 @@
# ADR 0016: Electron Markdown Mirror
Date: 2026-05-05
Status: Proposed
## Context
Logseq DB graphs do not expose one editable Markdown file per page in the graph
directory. Some desktop workflows still need a Markdown representation that can
be read by external tools, backed up, indexed, or inspected outside Logseq.
The mirror must not become another graph source of truth. Editing should remain
fast, and saving a block must not wait for Markdown rendering or filesystem
writes on the renderer main thread.
The first supported runtime is the Electron desktop app. Browser and mobile
builds do not have the same graph-directory filesystem guarantees.
## Decision
1. Add an Electron-only Settings toggle for Markdown Mirror.
2. Persist the toggle in Electron user settings as
`:feature/markdown-mirror?`.
3. When the setting is enabled for the Electron app, Logseq writes derived
Markdown files under the current graph directory:
- journals:
`markdown-mirror/journals/<journal-file-name>.md`
- other pages:
`markdown-mirror/pages/<page-file-name>.md`
4. For a graph at `~/logseq/graphs/graph-xxx`, mirror files are written under:
- `~/logseq/graphs/graph-xxx/markdown-mirror/journals/`
- `~/logseq/graphs/graph-xxx/markdown-mirror/pages/`
5. Markdown Mirror is derived output. The DB remains the source of truth.
6. Files under `markdown-mirror/**` must be ignored by graph import, file
watchers, and graph parsing so the mirror never feeds back into the graph.
7. The feature is not available in browser or mobile builds, even if a stale
setting value exists.
8. Settings exposes an explicit "Regenerate full mirror" action that asks the
DB worker to rewrite the complete mirror for the current graph.
9. Built-in pages and property pages, including user-created properties, are not
exported to the mirror. User Tag/Class pages are normal user content and are
exported.
## Runtime Ownership
1. The renderer owns only:
- the Settings row
- reading and updating `:feature/markdown-mirror?`
- pushing the enabled state to the DB worker when it changes
2. The DB worker owns:
- detecting affected page ids from successful local transactions
- rendering page Markdown from the worker DB snapshot
- scheduling and coalescing mirror jobs
- running explicit full-mirror regeneration jobs
- invoking platform filesystem writes
3. The Electron main process must not render Markdown. It may provide filesystem
primitives if needed, but content generation stays with the worker.
4. Editor save paths enqueue mirror work and return immediately. They must not
wait for rendering, directory creation, stat, write, rename, or delete.
## Reusable Core and CLI Path
1. Markdown Mirror path planning, filename normalization, page rendering, write
deduplication, atomic writes, rename cleanup, and delete cleanup live in a
worker/core namespace that does not depend on Electron UI state.
2. The Electron app only owns feature activation through Settings.
3. The CLI should be able to reuse the same core by passing an explicit graph,
DB snapshot, and node filesystem platform context.
4. Future CLI support should not introduce a second Markdown serializer or a
different filename normalization policy.
5. Future CLI support should reuse the same mirror-path allocation index so the
Electron app and CLI do not produce different file names for the same graph.
## Output Layout and Naming
1. Journal pages are written below `markdown-mirror/journals/`.
2. Non-journal pages are written below `markdown-mirror/pages/`.
3. Journal file names use the existing Logseq journal file-name rules for the
graph configuration.
4. Non-journal page file names use the normalized page title:
`<page-file-name>.md`.
5. Page file names must stay friendly to external Markdown tools such as Emacs,
VS Code, and Obsidian. Do not include page uuid in normal mirror file names.
6. Page title is not page identity. The page uuid is still the internal mirror
identity, but it is stored in the mirror index rather than exposed in the
file name.
7. Duplicate non-journal page titles are handled by stable title suffix
allocation:
- first allocated page: `pages/Foo.md`
- second allocated page: `pages/Foo (2).md`
- third allocated page: `pages/Foo (3).md`
8. Once a page uuid is assigned a mirror path, keep that path stable until the
page is renamed or deleted. Do not renumber existing duplicate-title mirror
paths when another duplicate is created or removed.
9. The implementation keeps a per-graph mirror index under
`markdown-mirror/.index.edn`.
10. The mirror index stores at least:
- page uuid -> relative mirror path
- relative mirror path -> page uuid
- page uuid -> last known normalized title stem
11. The mirror index is implementation metadata for path stability. It is not
graph content and must be ignored by graph import and watchers along with the
rest of `markdown-mirror/**`.
12. All mirror file names pass through a single cross-platform filename
normalizer before joining paths.
13. Duplicate journal-day entities indicate invalid graph state for the mirror.
The implementation must fail those journal mirror jobs and surface a
diagnostic instead of choosing a winner.
14. If two entities still map to the same mirror path, the implementation must
fail the mirror job for that path and surface a diagnostic instead of
overwriting an unrelated page.
15. Page rename moves the mirror by writing the new path, updating the mirror
index, and deleting the old path after the new file has been written.
16. Page deletion deletes the corresponding mirror file and removes the page uuid
from the mirror index.
17. The write guard must reject any computed path outside the graph's
`markdown-mirror/` directory.
18. Built-in pages and property pages are excluded from path allocation and
mirror writes. User Tag/Class pages are not excluded by this rule. If a
previously mirrored page becomes excluded, the old mirror file is removed.
## Duplicate Page Title Allocation
1. For non-journal pages, compute the normalized title stem first.
2. If the page uuid already exists in the mirror index and the normalized title
stem did not change, reuse the indexed path.
3. If the page uuid is new for that title, allocate the first unused path in this
sequence:
- `pages/<stem>.md`
- `pages/<stem> (2).md`
- `pages/<stem> (3).md`
4. A path is considered unavailable when the mirror index maps it to a different
live page uuid.
5. Deleted page paths become available for future allocation only after the
deleted page uuid is removed from the index.
6. Rename is treated as a new allocation for the new title stem. Existing pages
with the old title keep their already allocated paths.
7. If the mirror index is missing or unreadable, rebuild it from the current DB
in deterministic page order before writing. Deterministic order should use a
stable key such as page title plus page uuid.
8. The rebuilt index is allowed to choose paths for pages that had no previous
allocation. It must not overwrite a live existing path that is already mapped
to another page uuid.
## Rename and Delete
1. Page rename moves the mirror by writing the new path and deleting the old
path after the new file has been written.
2. Page deletion deletes the corresponding mirror file.
3. The implementation keeps a small per-graph mirror index keyed by page uuid so
rename and delete handling does not require scanning the mirror directory on
every transaction.
## Filename Normalization
1. Mirror file names must be portable across macOS, Windows, and Linux.
2. Use one shared normalizer for journal and page mirror file names.
3. The normalizer must:
- reject or replace path separators (`/`, `\`)
- reject or replace Windows-invalid characters (`<`, `>`, `:`, `"`, `|`,
`?`, `*`) and ASCII control characters
- reject or rewrite reserved Windows device names such as `CON`, `PRN`,
`AUX`, `NUL`, `COM1` through `COM9`, and `LPT1` through `LPT9`
- trim trailing spaces and dots because Windows does not preserve them
- reject empty names after normalization
- bound each file-name component to a safe byte length before appending
`.md`
4. Normalize Unicode to one canonical form before sanitizing so the same page
title produces the same mirror path across filesystems with different Unicode
normalization behavior.
5. The normalizer must be deterministic and must not depend on the current
operating system. A graph mirrored on macOS should choose the same logical
file name as the same graph mirrored on Windows.
6. If normalization changes the display title segment, the mirror index and
duplicate-title suffix allocation still preserve identity for non-journal
pages.
7. If a journal title normalizes to an unsafe or colliding file name, fail the
journal mirror job and surface diagnostics instead of inventing a fallback
name.
8. Path construction must join only validated path components. It must never
concatenate unchecked page titles into filesystem paths.
## Scheduling and Performance
1. Mirror rendering is incremental. A transaction schedules only pages affected
by that transaction.
2. Jobs are coalesced by page uuid. If a page is edited repeatedly before its
job runs, only the latest worker DB state is rendered.
3. Scheduling uses a short debounce window per graph to reduce write churn while
preserving near-real-time output.
4. Mirror writes are serialized per graph to avoid path races during rename and
delete.
5. Before writing, compare the generated Markdown with the current file content
or with the last written content hash. Skip the write when content is
unchanged.
6. Write files atomically:
- ensure the target directory exists
- write to a temporary file in the same directory
- rename the temporary file over the target
7. Heavy work is forbidden on the renderer main thread:
- no full-graph export
- no Markdown rendering
- no filesystem reads or writes
- no synchronous IPC waiting for mirror completion
8. Full regeneration is an explicit Settings action. The renderer only sends a
worker request; page selection, rendering, and filesystem writes stay in the
DB worker.
9. Enabling the setting starts incremental mirroring for subsequent page edits.
It does not implicitly run full regeneration.
## Markdown Content
1. Reuse the existing page-to-Markdown export pipeline used by worker export
APIs instead of introducing a separate renderer-side serializer.
2. The mirror output should match normal Markdown export semantics for page
content.
3. Mirror files do not include Logseq-internal mirror metadata in the Markdown
body.
4. Mirror files include block and page property drawers, including user
properties, with rendered property values.
5. Assets are referenced as normal exported Markdown references. This ADR does
not copy assets into `markdown-mirror/`.
6. Page references remain in Logseq wiki-link form, for example `[[Foo]]`.
7. Ambiguous page references caused by duplicate page titles are an accepted
limitation of Markdown Mirror. Do not rewrite page references to uuid-based
links or relative Markdown links in this ADR.
## Failure Handling
1. Filesystem and path errors fail the mirror job for the affected page.
2. Failures are logged with graph, page uuid, target path, and error details.
3. Repeated failures should be visible from Settings or diagnostics; do not show
a toast on every keystroke.
4. The feature must not silently fall back to browser storage, OPFS, or another
output directory.
5. If the graph directory is not available, the worker rejects mirror jobs for
that graph until the graph is reopened with a valid directory.
## Non-goals
1. Markdown Mirror is not bidirectional sync.
2. Editing files in `markdown-mirror/` does not update the graph.
3. The mirror is not a backup format with guaranteed import fidelity.
4. The mirror does not replace existing graph export features.
5. The mirror does not support browser or mobile runtimes in this ADR.
## Consequences
### Positive
- Desktop users get a readable Markdown projection inside the graph directory.
- Editor latency is protected because rendering and disk I/O are worker-owned
and asynchronous.
- The output layout is predictable for tools that watch journals and pages
separately.
- Page file names remain readable and practical in external Markdown tools.
- Ignoring `markdown-mirror/**` prevents mirror-generated files from becoming
graph input.
### Tradeoffs
- The mirror can lag slightly behind the latest edit because writes are
debounced and serialized.
- A per-graph mirror index is needed for reliable rename and delete cleanup.
- Duplicate page references such as `[[Foo]]` remain ambiguous in mirror output.
- The first version does not backfill every existing page automatically when the
setting is enabled; users run full regeneration explicitly.
- External edits to mirror files are overwritten by later Logseq edits.
- Property pages are intentionally absent from the mirror, so the output is not
a complete DB export even though page and block property drawers are included.
## Verification
Implementation should add focused coverage for:
```bash
bb dev:test -v frontend.worker.markdown-mirror-test/enabled-electron-edit-writes-page-mirror-test
bb dev:test -v frontend.worker.markdown-mirror-test/enabled-electron-edit-writes-journal-mirror-test
bb dev:test -v frontend.worker.markdown-mirror-test/disabled-setting-does-not-write-mirror-test
bb dev:test -v frontend.worker.markdown-mirror-test/repeated-edits-coalesce-to-latest-content-test
bb dev:test -v frontend.worker.markdown-mirror-test/rename-removes-old-mirror-path-test
bb dev:test -v frontend.worker.markdown-mirror-test/delete-removes-mirror-file-test
bb dev:test -v frontend.worker.markdown-mirror-test/same-title-pages-write-distinct-stable-friendly-paths-test
bb dev:test -v frontend.worker.markdown-mirror-test/page-references-remain-wiki-links-test
bb dev:test -v frontend.worker.markdown-mirror-test/page-mirror-exports-property-values-test
bb dev:test -v frontend.worker.markdown-mirror-test/page-mirror-exports-page-property-values-test
bb dev:test -v frontend.worker.markdown-mirror-test/full-regeneration-writes-existing-non-built-in-non-property-pages-test
bb dev:test -v frontend.worker.markdown-mirror-test/invalid-filename-characters-are-normalized-test
bb dev:test -v frontend.worker.markdown-mirror-test/windows-reserved-filename-fails-with-diagnostic-test
bb dev:test -v frontend.worker.markdown-mirror-test/mirror-path-collision-fails-without-overwrite-test
```
Additional checks:
- `markdown-mirror/**` is excluded from graph parsing and file watchers.
- Editor save does not await mirror completion.
- Browser and mobile builds do not expose the setting and do not schedule mirror
jobs.
- Atomic write failures do not leave partial target files.

View File

@@ -315,7 +315,8 @@
(do (cfgs/set-item! k v)
(when (= k :spell-check)
(spell-check/apply-window-spellcheck! window (spell-check/session-spellcheck-enabled? v)))
(state/set-state! [:config k] v))
(state/set-state! [:config k] v)
nil)
(cfgs/get-item k))
config)))

View File

@@ -745,6 +745,51 @@
{:left-label (t :settings.features/enable-flashcards)
:action (flashcards-enabled-switcher enable-flashcards?)}))
(rum/defcs markdown-mirror-row < rum/reactive
(rum/local false ::regenerating?)
[state t]
(let [enabled? (true? (state/sub [:electron/user-cfgs :feature/markdown-mirror?]))
*regenerating? (::regenerating? state)
regenerate! (fn []
(let [repo (state/get-current-repo)]
(when (and repo @state/*db-worker (not @*regenerating?))
(reset! *regenerating? true)
(-> (state/<invoke-db-worker :thread-api/markdown-mirror-regenerate repo)
(p/then (fn [_]
(notification/show!
(t :settings.features/markdown-mirror-regenerate-success)
:success)))
(p/catch (fn [error]
(log/error :markdown-mirror/regenerate-failed
{:repo repo
:error error})
(notification/show!
(t :settings.features/markdown-mirror-regenerate-error (str error))
:error)))
(p/finally #(reset! *regenerating? false))))))]
(toggle
"markdown-mirror"
(t :settings.features/markdown-mirror)
enabled?
#(let [next-enabled? (not enabled?)
repo (state/get-current-repo)]
(state/set-state! [:electron/user-cfgs :feature/markdown-mirror?] next-enabled?)
(ipc/ipc :userAppCfgs :feature/markdown-mirror? next-enabled?)
(when (and repo @state/*db-worker)
(-> (state/<invoke-db-worker :thread-api/markdown-mirror-set-enabled repo next-enabled?)
(p/catch (fn [error]
(log/error :markdown-mirror/settings-sync-failed
{:repo repo
:error error}))))))
[:div.flex.items-center.gap-2.flex-wrap
[:span.text-sm.opacity-50 (t :settings.features/markdown-mirror-desc)]
(ui/button
(t :settings.features/markdown-mirror-regenerate)
:icon "refresh"
:class "text-sm"
:disabled @*regenerating?
:on-click regenerate!)])))
(defn https-user-agent-row [agent-opts]
(row-with-button-action
{:left-label (t :settings.advanced/network-proxy)
@@ -1056,6 +1101,8 @@
(plugin-system-switcher-row))
(when (util/electron?)
(http-server-switcher-row))
(when (util/electron?)
(markdown-mirror-row t))
(flashcards-switcher-row enable-flashcards?)
(when-not web-platform?
[:div.mt-1.sm:mt-0.sm:col-span-2

View File

@@ -193,10 +193,20 @@
(log/error :sqlite-error error)
(notification/show! (t :storage/sqlitedb-error error) :error))))
(defn- <sync-markdown-mirror-setting!
[repo]
(if (and (util/electron?) repo)
(state/<invoke-db-worker :thread-api/markdown-mirror-set-enabled
repo
(true? (get-in @state/state [:electron/user-cfgs :feature/markdown-mirror?])))
(p/resolved nil)))
(defrecord InBrowser []
protocol/PersistentDB
(<new [_this repo opts]
(state/<invoke-db-worker :thread-api/create-or-open-db repo opts))
(p/let [result (state/<invoke-db-worker :thread-api/create-or-open-db repo opts)
_ (<sync-markdown-mirror-setting! repo)]
result))
(<list-db [_this]
(-> (state/<invoke-db-worker :thread-api/list-db)
@@ -209,7 +219,8 @@
(state/<invoke-db-worker :thread-api/release-access-handles repo))
(<fetch-initial-data [_this repo opts]
(-> (p/let [_ (state/<invoke-db-worker :thread-api/create-or-open-db repo opts)]
(-> (p/let [_ (state/<invoke-db-worker :thread-api/create-or-open-db repo opts)
_ (<sync-markdown-mirror-setting! repo)]
(state/<invoke-db-worker :thread-api/get-initial-data repo opts))
(p/catch sqlite-error-handler)))

View File

@@ -17,6 +17,7 @@
[frontend.worker.db.migrate :as db-migrate]
[frontend.worker.db.validate :as worker-db-validate]
[frontend.worker.export :as worker-export]
[frontend.worker.markdown-mirror :as markdown-mirror]
[frontend.worker.pipeline :as worker-pipeline]
[frontend.worker.platform :as platform]
[frontend.worker.publish]
@@ -1094,6 +1095,20 @@
(worker-state/set-new-state! new-state)
nil)
(def-thread-api :thread-api/markdown-mirror-set-enabled
[repo enabled?]
(markdown-mirror/set-enabled! repo enabled?)
nil)
(def-thread-api :thread-api/markdown-mirror-flush
[repo]
(markdown-mirror/<flush-repo! repo {}))
(def-thread-api :thread-api/markdown-mirror-regenerate
[repo]
(when-let [conn (worker-state/get-datascript-conn repo)]
(markdown-mirror/<mirror-repo! repo @conn {})))
(def-thread-api :thread-api/export-get-debug-datoms
[repo]
(when-let [conn (worker-state/get-datascript-conn repo)]

View File

@@ -3,6 +3,7 @@
(:require [datascript.core :as d]
[frontend.common.thread-api :as thread-api]
[frontend.worker.pipeline :as worker-pipeline]
[frontend.worker.markdown-mirror :as markdown-mirror]
[frontend.worker.search :as search]
[frontend.worker.shared-service :as shared-service]
[frontend.worker.state :as worker-state]
@@ -60,6 +61,10 @@
[_ {:keys [repo]} tx-report]
(db-sync/handle-local-tx! repo tx-report))
(defmethod listen-db-changes :markdown-mirror
[_ {:keys [repo]} tx-report]
(markdown-mirror/<handle-tx-report! repo nil tx-report {:defer? true}))
(defn listen-db-changes!
[repo conn & {:keys [handler-keys]}]
(let [handlers (if (seq handler-keys)

View File

@@ -0,0 +1,357 @@
(ns frontend.worker.markdown-mirror
"Markdown mirror derived-file support for DB graphs."
(:require [clojure.string :as string]
[datascript.core :as d]
[frontend.worker.graph-dir :as graph-dir]
[frontend.worker.platform :as platform]
[lambdaisland.glogi :as log]
[logseq.cli.common.file :as common-file]
[logseq.db :as ldb]
[promesa.core :as p]))
(defn repo-mirror-dir
[repo]
(str (graph-dir/repo->encoded-graph-dir-name repo) "/markdown-mirror"))
(def ^:private invalid-file-name-chars-re
#"[<>:\"|?*\\/]")
(def ^:private ascii-control-re
#"[\x00-\x1F]")
(def ^:private trailing-space-or-dot-re
#"[ \.]+$")
(def ^:private reserved-windows-device-names
(into #{"CON" "PRN" "AUX" "NUL"}
(concat (map #(str "COM" %) (range 1 10))
(map #(str "LPT" %) (range 1 10)))))
(def ^:private max-file-stem-length 160)
(defonce ^:private *repo->enabled? (atom {}))
(defonce ^:private *repo->queued-page-jobs (atom {}))
(defonce ^:private *repo->flush-timeout (atom {}))
(defn- normalize-unicode
[s]
(let [s (str s)]
(if (fn? (.-normalize s))
(.normalize s "NFC")
s)))
(defn- reserved-windows-device-name?
[s]
(contains? reserved-windows-device-names
(string/upper-case s)))
(defn normalize-file-stem
[s]
(when (some? s)
(let [s' (-> (normalize-unicode s)
(string/replace invalid-file-name-chars-re "_")
(string/replace ascii-control-re "_")
(string/replace trailing-space-or-dot-re ""))
s' (if (> (count s') max-file-stem-length)
(subs s' 0 max-file-stem-length)
s')]
(when (and (not (string/blank? s'))
(not (reserved-windows-device-name? s')))
s'))))
(defn- journal-file-stem
[journal-day]
(when journal-day
(let [s (str journal-day)]
(when (= 8 (count s))
(str (subs s 0 4) "_" (subs s 4 6) "_" (subs s 6 8))))))
(defn page-relative-path
([db page]
(page-relative-path db page {}))
([db page {:keys [journal-file-stem-fn]
:or {journal-file-stem-fn journal-file-stem}}]
(when page
(if (ldb/journal? page)
(when-let [stem (normalize-file-stem (journal-file-stem-fn (:block/journal-day page)))]
(str "journals/" stem ".md"))
(when-let [stem (normalize-file-stem (:block/title page))]
(let [duplicate-pages (->> (d/datoms db :avet :block/title (:block/title page))
(map #(d/entity db (:e %)))
(filter #(and (ldb/page? %)
(not (ldb/journal? %))))
(sort-by (comp str :block/uuid)))
index (inc (or (first (keep-indexed
(fn [idx p]
(when (= (:block/uuid page) (:block/uuid p))
idx))
duplicate-pages))
0))
stem' (if (= 1 index)
stem
(str stem " (" index ")"))]
(str "pages/" stem' ".md")))))))
(defn- mirror-path
[repo relative-path]
(str (repo-mirror-dir repo) "/" relative-path))
(defn- page-id-for-entity
[db eid]
(when-let [entity (d/entity db eid)]
(cond
(ldb/page? entity) (:db/id entity)
(:block/page entity) (:db/id (:block/page entity))
(and (:block/parent entity) (ldb/page? (:block/parent entity))) (:db/id (:block/parent entity))
(some-> entity :block/parent :block/page) (:db/id (:block/page (:block/parent entity))))))
(defn affected-page-ids
[{:keys [db-before db-after tx-data]}]
(->> tx-data
(mapcat (fn [{:keys [e a v]}]
(cond-> [(page-id-for-entity db-before e)
(page-id-for-entity db-after e)]
(= a :block/page)
(conj v))))
(remove nil?)
set))
(defn set-enabled!
[repo enabled?]
(if enabled?
(swap! *repo->enabled? assoc repo true)
(do
(when-let [timeout-id (get @*repo->flush-timeout repo)]
(js/clearTimeout timeout-id))
(swap! *repo->enabled? dissoc repo)
(swap! *repo->queued-page-jobs dissoc repo)
(swap! *repo->flush-timeout dissoc repo)))
nil)
(defn enabled?
[repo]
(true? (get @*repo->enabled? repo)))
(defn- storage
[platform*]
(:storage platform*))
(defn- <read-text
[platform* path]
(if-let [f (or (:mirror-read-text! (storage platform*))
(:read-text! (storage platform*)))]
(-> (f path)
(p/catch (constantly nil)))
(p/rejected (ex-info "platform storage/read-text! missing" {:path path}))))
(defn- <write-text-atomic!
[platform* path content]
(if-let [f (:write-text-atomic! (storage platform*))]
(f path content)
(p/rejected (ex-info "platform storage/write-text-atomic! missing" {:path path}))))
(defn- <delete-file!
[platform* path]
(if-let [f (:delete-file! (storage platform*))]
(f path)
(p/rejected (ex-info "platform storage/delete-file! missing" {:path path}))))
(defn- supported-runtime?
[platform*]
(or (= :node (get-in platform* [:env :runtime]))
(and (= :browser (get-in platform* [:env :runtime]))
(= :electron (get-in platform* [:env :owner-source])))))
(defn- duplicate-journal-day?
[db journal-day]
(when journal-day
(< 1 (count (d/datoms db :avet :block/journal-day journal-day)))))
(defn- render-page-content
[db page options]
(common-file/block->content
db
(:block/uuid page)
{:include-page-properties? true}
{:export-bullet-indentation (or (:export-bullet-indentation options) " ")
:date-formatter (:date-formatter options)}))
(defn- mirrorable-page?
[page]
(and (ldb/page? page)
(not (ldb/built-in? page))
(not (ldb/property? page))
(not (ldb/hidden? page))
(not (:logseq.property.user/email page))))
(defn- mirrorable-pages
[db]
(->> (d/datoms db :avet :block/name)
(map #(d/entity db (:e %)))
(filter mirrorable-page?)
(sort-by (fn [page]
[(if (ldb/journal? page) 0 1)
(str (:block/journal-day page))
(string/lower-case (or (:block/title page) ""))
(str (:block/uuid page))]))))
(defn- <write-if-changed!
[platform* path content]
(p/let [current (<read-text platform* path)]
(if (= current content)
{:status :skipped
:reason :unchanged
:path path}
(p/let [_ (<write-text-atomic! platform* path content)]
{:status :written
:path path}))))
(defn- invalid-file-name-result
[repo page]
(let [result {:status :error
:reason :invalid-file-name
:repo repo
:page-uuid (:block/uuid page)}]
(log/error :markdown-mirror/invalid-file-name result)
result))
(defn <mirror-page!
[repo db page-id {:keys [platform] :as opts}]
(let [platform* (or platform (platform/current))]
(if-not (supported-runtime? platform*)
(p/resolved {:status :skipped
:reason :unsupported-runtime})
(if-let [page (d/entity db page-id)]
(cond
(not (mirrorable-page? page))
(p/resolved {:status :skipped
:reason :excluded-page
:repo repo
:page-id page-id})
(and (ldb/journal? page)
(duplicate-journal-day? db (:block/journal-day page)))
(let [result {:status :error
:reason :duplicate-journal-day
:repo repo
:journal-day (:block/journal-day page)
:page-uuid (:block/uuid page)}]
(log/error :markdown-mirror/duplicate-journal-day result)
(p/resolved result))
:else
(if-let [relative-path (page-relative-path db page opts)]
(let [path (mirror-path repo relative-path)
content (render-page-content db page opts)]
(<write-if-changed! platform* path content))
(p/resolved (invalid-file-name-result repo page))))
(p/resolved {:status :skipped
:reason :missing-page
:repo repo
:page-id page-id})))))
(defn- deleted-page?
[page]
(or (nil? page)
(not (mirrorable-page? page))))
(defn- page-job
[repo {:keys [db-before db-after]} page-id opts]
(let [before-page (d/entity db-before page-id)
after-page (d/entity db-after page-id)
old-relative-path (when before-page (page-relative-path db-before before-page opts))
new-relative-path (when after-page (page-relative-path db-after after-page opts))]
{:repo repo
:page-id page-id
:db db-after
:old-path (when old-relative-path (mirror-path repo old-relative-path))
:new-path (when new-relative-path (mirror-path repo new-relative-path))
:delete? (deleted-page? after-page)}))
(defn- merge-job
[old-job new-job]
(assoc new-job :old-path (or (:old-path old-job)
(:old-path new-job))))
(defn- queue-job!
[repo job]
(swap! *repo->queued-page-jobs update-in [repo (:page-id job)] merge-job job))
(defn- drain-repo-jobs!
[repo]
(let [jobs (vals (get @*repo->queued-page-jobs repo))]
(swap! *repo->queued-page-jobs dissoc repo)
jobs))
(declare <flush-repo!)
(defn- schedule-flush!
[repo opts]
(when-not (get @*repo->flush-timeout repo)
(let [timeout-id (js/setTimeout
(fn []
(swap! *repo->flush-timeout dissoc repo)
(-> (<flush-repo! repo opts)
(p/catch (fn [error]
(log/error :markdown-mirror/flush-failed
{:repo repo
:error error})))))
(or (:debounce-ms opts) 250))]
(swap! *repo->flush-timeout assoc repo timeout-id))))
(defn- <run-job!
[platform* {:keys [repo db page-id old-path new-path delete?] :as _job} opts]
(cond
delete?
(if old-path
(p/let [_ (<delete-file! platform* old-path)]
{:status :deleted
:path old-path})
(p/resolved {:status :skipped
:reason :missing-old-path}))
:else
(p/let [result (<mirror-page! repo db page-id (assoc opts :platform platform*))
_ (when (and old-path
new-path
(not= old-path new-path)
(= :written (:status result)))
(<delete-file! platform* old-path))]
result)))
(defn <handle-tx-report!
[repo _conn tx-report {:keys [platform defer?] :as opts}]
(let [platform* (or platform (platform/current))]
(if (and (enabled? repo)
(supported-runtime? platform*)
(not (get-in tx-report [:tx-meta :from-disk?])))
(let [jobs (map #(page-job repo tx-report % opts)
(affected-page-ids tx-report))]
(if defer?
(do
(doseq [job jobs] (queue-job! repo job))
(schedule-flush! repo (assoc opts :platform platform*))
(p/resolved {:status :queued
:count (count jobs)}))
(p/all (map #(<run-job! platform* % opts) jobs))))
(p/resolved {:status :skipped
:reason :disabled-or-unsupported}))))
(defn <flush-repo!
[repo {:keys [platform] :as opts}]
(let [platform* (or platform (platform/current))
jobs (drain-repo-jobs! repo)]
(p/all (map #(<run-job! platform* % opts) jobs))))
(defn <mirror-repo!
[repo db {:keys [platform] :as opts}]
(let [platform* (or platform (platform/current))]
(if-not (supported-runtime? platform*)
(p/resolved {:status :skipped
:reason :unsupported-runtime})
(p/let [results (p/all
(map #(<mirror-page! repo db (:db/id %) (assoc opts :platform platform*))
(mirrorable-pages db)))]
{:status :completed
:count (count results)
:results results}))))

View File

@@ -155,8 +155,15 @@
(defn- asset-delete!
[repo file-name]
(-> (.unlink (browser-pfs) (asset-path repo file-name))
(p/catch (constantly nil))))
(let [^js pfs (browser-pfs)]
(-> (.unlink pfs (asset-path repo file-name))
(p/catch (constantly nil)))))
(defn- unsupported-mirror-storage!
[& _args]
(throw (ex-info "Markdown mirror storage is not supported in browser workers"
{:platform :browser
:feature :markdown-mirror})))
(defn- websocket-connect
[url]
@@ -201,6 +208,9 @@
:remove-vfs! remove-vfs!
:read-text! read-text!
:write-text! write-text!
:write-text-atomic! unsupported-mirror-storage!
:delete-file! unsupported-mirror-storage!
:mirror-read-text! unsupported-mirror-storage!
:asset-read-bytes! asset-read-bytes!
:asset-write-bytes! asset-write-bytes!
:asset-stat asset-stat

View File

@@ -252,6 +252,26 @@
_ (ensure-dir! dir)]
(fs/writeFile full-path text "utf8"))))
(defn- write-text-atomic!
[write-guard-fn data-dir path text]
(let [full-path (path-under-data-dir data-dir path)
dir (node-path/dirname full-path)
tmp-path (node-path/join dir (str "." (node-path/basename full-path) ".tmp-" (random-uuid)))]
(p/let [_ (when write-guard-fn
(write-guard-fn))
_ (ensure-dir! dir)
_ (fs/writeFile tmp-path text "utf8")
_ (fs/rename tmp-path full-path)]
nil)))
(defn- delete-file!
[write-guard-fn data-dir path]
(let [full-path (path-under-data-dir data-dir path)]
(p/let [_ (when write-guard-fn
(write-guard-fn))]
(-> (fs/rm full-path #js {:force true})
(p/catch (constantly nil))))))
(defn- asset-file-path
[data-dir repo file-name]
(node-path/join (repo-dir data-dir repo)
@@ -433,6 +453,8 @@
:remove-vfs! (fn [pool] (remove-vfs! pool))
:read-text! (fn [path] (read-text! data-dir path))
:write-text! (fn [path text] (write-text! write-guard-fn data-dir path text))
:write-text-atomic! (fn [path text] (write-text-atomic! write-guard-fn data-dir path text))
:delete-file! (fn [path] (delete-file! write-guard-fn data-dir path))
:asset-read-bytes! (fn [repo file-name]
(asset-read-bytes! data-dir repo file-name))
:asset-write-bytes! (fn [repo file-name payload]

View File

@@ -1631,6 +1631,11 @@
:settings.features/home-default-page-update-success "Home default page updated successfully!"
:settings.features/journals-enable-success "Journals enabled"
:settings.features/login-prompt "To access new features before anyone else you must be an Open Collective Sponsor or Backer of Logseq and therefore log in first."
:settings.features/markdown-mirror "Markdown Mirror"
:settings.features/markdown-mirror-desc "Write a derived Markdown copy of edited pages to the graph's markdown-mirror folder. Desktop only."
:settings.features/markdown-mirror-regenerate "Regenerate full mirror"
:settings.features/markdown-mirror-regenerate-error "Failed to regenerate Markdown Mirror: {1}"
:settings.features/markdown-mirror-regenerate-success "Markdown Mirror regenerated"
:settings.features/page-not-found "The page \"{1}\" doesn't exist yet. Please create that page first, and then try again."
:settings.features/plugin-system "Plugins"

View File

@@ -1621,6 +1621,11 @@
:settings.features/home-default-page-update-success "主页默认页面已更新成功!"
:settings.features/journals-enable-success "已启用日志页"
:settings.features/login-prompt "你必须是 Logseq 的 Open Collective Sponsor 或者 Backer 才能提前使用新功能(仍在测试中),因此需要登录。"
:settings.features/markdown-mirror "Markdown 镜像"
:settings.features/markdown-mirror-desc "将已编辑页面的派生 Markdown 副本写入图谱的 markdown-mirror 文件夹。仅桌面端可用。"
:settings.features/markdown-mirror-regenerate "重新生成完整镜像"
:settings.features/markdown-mirror-regenerate-error "重新生成 Markdown 镜像失败:{1}"
:settings.features/markdown-mirror-regenerate-success "Markdown 镜像已重新生成"
:settings.features/page-not-found "页面“{1}”尚不存在。请先创建该页面,然后再试一次。"
:settings.features/plugin-system "插件系统"

View File

@@ -1,5 +1,5 @@
(ns frontend.handler.editor-async-test
(:require [clojure.test :refer [is testing async use-fixtures]]
(:require [cljs.test :refer [is testing async use-fixtures]]
[datascript.core :as d]
[frontend.db :as db]
[frontend.handler.editor :as editor]

View File

@@ -52,6 +52,7 @@
:thread-api/import-db-base64 :thread-api/search-blocks :thread-api/search-upsert-blocks :thread-api/search-delete-blocks
:thread-api/search-truncate-tables :thread-api/search-build-blocks-indice :thread-api/search-build-blocks-indice-in-worker
:thread-api/search-build-pages-indice :thread-api/apply-outliner-ops :thread-api/sync-app-state
:thread-api/markdown-mirror-set-enabled :thread-api/markdown-mirror-flush :thread-api/markdown-mirror-regenerate
:thread-api/export-get-debug-datoms :thread-api/export-get-all-page->content :thread-api/validate-db
:thread-api/recompute-checksum-diagnostics :thread-api/export-edn :thread-api/import-edn :thread-api/get-view-data
:thread-api/get-class-objects :thread-api/get-property-values :thread-api/get-bidirectional-properties

View File

@@ -1,6 +1,7 @@
(ns frontend.worker.db-listener-test
(:require [cljs.test :refer [deftest is testing]]
[frontend.worker.db-listener :as db-listener]))
[frontend.worker.db-listener :as db-listener]
[frontend.worker.markdown-mirror :as markdown-mirror]))
(deftest transit-safe-tx-meta-keeps-outliner-ops-test
(testing "worker tx-meta sanitization should preserve semantic outliner ops"
@@ -14,3 +15,16 @@
(is (= outliner-ops (:outliner-ops safe-tx-meta)))
(is (= outliner-ops (:db-sync/inverse-outliner-ops safe-tx-meta)))
(is (nil? (:error-handler safe-tx-meta))))))
(deftest markdown-mirror-listener-enqueues-worker-mirror-work-test
(let [calls (atom [])
tx-report {:tx-data [:tx]}]
(with-redefs [markdown-mirror/<handle-tx-report!
(fn [repo conn tx-report opts]
(swap! calls conj [repo conn tx-report opts]))]
((get-method db-listener/listen-db-changes :markdown-mirror)
:markdown-mirror
{:repo "repo"}
tx-report))
(is (= [["repo" nil tx-report {:defer? true}]]
@calls))))

View File

@@ -49,6 +49,7 @@
:thread-api/import-db-base64 :thread-api/search-blocks :thread-api/search-upsert-blocks :thread-api/search-delete-blocks
:thread-api/search-truncate-tables :thread-api/search-build-blocks-indice :thread-api/search-build-blocks-indice-in-worker
:thread-api/search-build-pages-indice :thread-api/apply-outliner-ops :thread-api/sync-app-state
:thread-api/markdown-mirror-set-enabled :thread-api/markdown-mirror-flush :thread-api/markdown-mirror-regenerate
:thread-api/export-get-debug-datoms :thread-api/export-get-all-page->content :thread-api/validate-db
:thread-api/recompute-checksum-diagnostics :thread-api/export-edn :thread-api/import-edn :thread-api/get-view-data
:thread-api/get-class-objects :thread-api/get-property-values :thread-api/get-bidirectional-properties

View File

@@ -0,0 +1,447 @@
(ns frontend.worker.markdown-mirror-test
(:require [cljs.test :refer [async deftest is testing]]
[datascript.core :as d]
[frontend.worker.markdown-mirror :as markdown-mirror]
[logseq.db.test.helper :as db-test]
[promesa.core :as p]))
(def test-repo "logseq_db_graph-xxx")
(defn- fake-platform
([] (fake-platform {:runtime :node}))
([env]
(let [files (atom {})
writes (atom [])
deletes (atom [])]
{:platform {:env env
:storage {:read-text! (fn [path]
(p/resolved (get @files path)))
:write-text-atomic! (fn [path content]
(swap! writes conj [path content])
(swap! files assoc path content)
(p/resolved nil))
:delete-file! (fn [path]
(swap! deletes conj path)
(swap! files dissoc path)
(p/resolved nil))}
:broadcast {:post-message! (fn [& _] nil)}}
:files files
:writes writes
:deletes deletes})))
(defn- page-path [path]
(str (markdown-mirror/repo-mirror-dir test-repo) "/" path))
(defn- first-block [page]
(-> page :block/_page first))
(defn- <mirror-repo!
[& args]
(if-let [f (resolve 'frontend.worker.markdown-mirror/<mirror-repo!)]
(apply f args)
(p/resolved ::missing-mirror-repo-fn)))
(deftest normalize-file-name-is-cross-platform-and-deterministic-test
(testing "invalid filesystem characters and path separators are replaced"
(is (= "A_B_C_D_E_F_G_H"
(markdown-mirror/normalize-file-stem "A/B\\C:D<E>F\"G|H"))))
(testing "trailing spaces and dots are removed"
(is (= "title"
(markdown-mirror/normalize-file-stem "title. "))))
(testing "unicode is normalized before sanitizing"
(is (= (markdown-mirror/normalize-file-stem "e\u0301")
(markdown-mirror/normalize-file-stem "\u00e9"))))
(testing "reserved Windows device names are rejected"
(is (nil? (markdown-mirror/normalize-file-stem "CON")))
(is (nil? (markdown-mirror/normalize-file-stem "lpt9")))))
(deftest same-title-pages-write-distinct-stable-friendly-paths-test
(let [page-uuid-1 #uuid "11111111-1111-4111-8111-111111111111"
page-uuid-2 #uuid "22222222-2222-4222-8222-222222222222"
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "Same Name"
:block/uuid page-uuid-1}
:blocks [{:block/title "first"}]}
{:page {:block/title "Same Name"
:block/uuid page-uuid-2}
:blocks [{:block/title "second"}]}]})
pages (->> (d/datoms @conn :avet :block/title "Same Name")
(map #(d/entity @conn (:e %)))
(filter #(nil? (:block/page %)))
(sort-by (comp str :block/uuid)))
paths (mapv #(markdown-mirror/page-relative-path @conn %) pages)]
(is (= ["pages/Same Name.md"
"pages/Same Name (2).md"]
paths))))
(deftest page-references-remain-wiki-links-test
(async done
(let [{:keys [platform files]} (fake-platform)
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "Source"}
:blocks [{:block/title "See [[Foo]]"}]}
{:page {:block/title "Foo"}
:blocks [{:block/title "target"}]}
{:page {:block/title "Foo"}
:blocks [{:block/title "duplicate"}]}]})
page (db-test/find-page-by-title @conn "Source")]
(-> (markdown-mirror/<mirror-page! test-repo @conn (:db/id page) {:platform platform})
(p/then (fn [_]
(is (= "- See [[Foo]]"
(get @files (page-path "pages/Source.md"))))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest affected-page-ids-detects-edited-block-page-test
(let [conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "Page A"}
:blocks [{:block/title "before"}]}]})
page (db-test/find-page-by-title @conn "Page A")
block (first-block page)
tx-report (d/with @conn [{:db/id (:db/id block)
:block/title "after"}])]
(is (= #{(:db/id page)}
(markdown-mirror/affected-page-ids tx-report)))))
(deftest enabled-electron-edit-writes-page-mirror-test
(async done
(let [{:keys [platform files writes]} (fake-platform)
page-uuid #uuid "33333333-3333-4333-8333-333333333333"
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "Page A"
:block/uuid page-uuid}
:blocks [{:block/title "hello"}
{:block/title "world"}]}]})
page (db-test/find-page-by-title @conn "Page A")]
(-> (markdown-mirror/<mirror-page! test-repo @conn (:db/id page) {:platform platform})
(p/then (fn [_]
(let [path (page-path "pages/Page A.md")]
(is (= "- hello\n- world" (get @files path)))
(is (= [[path "- hello\n- world"]] @writes)))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest page-mirror-exports-property-values-test
(async done
(let [{:keys [platform files]} (fake-platform)
conn (db-test/create-conn-with-blocks
{:properties {:user.property/reproducible-steps {:logseq.property/type :default}
:user.property/rating {:logseq.property/type :number}}
:pages-and-blocks [{:page {:block/title "Issue"
:build/properties {:user.property/reproducible-steps "Open settings"
:logseq.property/heading 1}}
:blocks [{:block/title "TODO body"
:build/properties {:logseq.property/status :logseq.property/status.todo
:user.property/reproducible-steps "Click mirror"
:user.property/rating 5
:logseq.property/heading 2}}]}]})
page (db-test/find-page-by-title @conn "Issue")]
(-> (markdown-mirror/<mirror-page! test-repo @conn (:db/id page) {:platform platform})
(p/then (fn [_]
(let [content (get @files (page-path "pages/Issue.md"))]
(is (= "reproducible-steps:: Open settings\n- TODO body\n Status:: Todo\n reproducible-steps:: Click mirror\n rating:: 5"
content)))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest page-mirror-exports-page-property-values-test
(async done
(let [{:keys [platform files]} (fake-platform)
conn (db-test/create-conn-with-blocks
{:properties {:user.property/p1 {:logseq.property/type :default}
:user.property/p2 {:logseq.property/type :number}
:user.property/p3 {:logseq.property/type :default}}
:pages-and-blocks [{:page {:block/title "Page Props"
:build/properties {:user.property/p1 "hello"
:user.property/p2 1
:user.property/p3 "Author 1"}}
:blocks [{:block/title "body"}]}]})
page (db-test/find-page-by-title @conn "Page Props")]
(-> (markdown-mirror/<mirror-page! test-repo @conn (:db/id page) {:platform platform})
(p/then (fn [_]
(let [content (get @files (page-path "pages/Page Props.md"))]
(is (= "p1:: hello\np2:: 1\np3:: Author 1\n- body"
content)))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest journal-mirror-exports-page-and-block-property-values-test
(async done
(let [{:keys [platform files]} (fake-platform)
conn (db-test/create-conn-with-blocks
{:properties {:user.property/p1 {:logseq.property/type :default}
:user.property/p2 {:logseq.property/type :number}
:user.property/p3 {:logseq.property/type :default}}
:pages-and-blocks [{:page {:block/title "May 5th, 2026"
:block/name "may 5th, 2026"
:block/journal-day 20260505
:block/tags #{:logseq.class/Journal}
:build/properties {:user.property/p1 "hey"}}
:blocks [{:block/title "TODO hello great test"
:build/properties {:logseq.property/status :logseq.property/status.todo
:user.property/p1 "hello"
:user.property/p2 1
:user.property/p3 "Author 1"}}]}]})
journal (db-test/find-journal-by-journal-day @conn 20260505)]
(-> (markdown-mirror/<mirror-page! test-repo @conn (:db/id journal) {:platform platform})
(p/then (fn [_]
(let [content (get @files (page-path "journals/2026_05_05.md"))]
(is (= "p1:: hey\n- TODO hello great test\n Status:: Todo\n p1:: hello\n p2:: 1\n p3:: Author 1"
content)))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest full-regeneration-writes-existing-non-built-in-non-property-pages-test
(async done
(let [{:keys [platform files]} (fake-platform)
conn (db-test/create-conn-with-blocks
{:properties {:rating {:logseq.property/type :default}}
:pages-and-blocks [{:page {:block/title "Page A"}
:blocks [{:block/title "alpha"}]}
{:page {:block/title "Journal"
:block/journal-day 20240508
:block/tags #{:logseq.class/Journal}}
:blocks [{:block/title "journal"}]}
{:page {:block/title "Built In"
:build/properties {:logseq.property/built-in? true}}
:blocks [{:block/title "system"}]}
{:page {:block/title "Project"
:block/tags #{:logseq.class/Tag}
:db/ident :user.class/Project}
:blocks [{:block/title "class"}]}
{:page {:block/title "rating"
:block/tags #{:logseq.class/Property}
:db/ident :user.property/rating}
:blocks [{:block/title "property"}]}]})]
(-> (<mirror-repo! test-repo @conn {:platform platform})
(p/then (fn [result]
(is (not= ::missing-mirror-repo-fn result))
(is (= "- alpha"
(get @files (page-path "pages/Page A.md"))))
(is (= "- journal"
(get @files (page-path "journals/2024_05_08.md"))))
(is (= "- class"
(get @files (page-path "pages/Project.md"))))
(is (nil? (get @files (page-path "pages/Built In.md"))))
(is (nil? (get @files (page-path "pages/rating.md"))))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest electron-browser-worker-runtime-is-supported-test
(async done
(let [{:keys [platform files]} (fake-platform {:runtime :browser
:owner-source :electron})
page-uuid #uuid "88888888-8888-4888-8888-888888888888"
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "Page A"
:block/uuid page-uuid}
:blocks [{:block/title "desktop"}]}]})
page (db-test/find-page-by-title @conn "Page A")]
(-> (markdown-mirror/<mirror-page! test-repo @conn (:db/id page) {:platform platform})
(p/then (fn [_]
(is (= "- desktop"
(get @files (page-path "pages/Page A.md"))))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest non-electron-browser-runtime-is-skipped-test
(async done
(let [{:keys [platform writes]} (fake-platform {:runtime :browser
:owner-source :browser})
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "Page A"}
:blocks [{:block/title "web"}]}]})
page (db-test/find-page-by-title @conn "Page A")]
(-> (markdown-mirror/<mirror-page! test-repo @conn (:db/id page) {:platform platform})
(p/then (fn [result]
(is (= :skipped (:status result)))
(is (= :unsupported-runtime (:reason result)))
(is (empty? @writes))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest enabled-electron-edit-writes-journal-mirror-test
(async done
(let [{:keys [platform files]} (fake-platform)
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:build/journal 20240506}
:blocks [{:block/title "journal item"}]}]})
journal (db-test/find-journal-by-journal-day @conn 20240506)]
(-> (markdown-mirror/<mirror-page! test-repo @conn (:db/id journal) {:platform platform})
(p/then (fn [_]
(is (= "- journal item"
(get @files (page-path "journals/2024_05_06.md"))))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest disabled-setting-does-not-write-mirror-test
(async done
(let [{:keys [platform writes]} (fake-platform)
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "Page A"}
:blocks [{:block/title "before"}]}]})
page (db-test/find-page-by-title @conn "Page A")
block (first-block page)
tx-report (d/with @conn [{:db/id (:db/id block)
:block/title "after"}])]
(markdown-mirror/set-enabled! test-repo false)
(-> (markdown-mirror/<handle-tx-report! test-repo conn tx-report {:platform platform})
(p/then (fn [_]
(is (empty? @writes))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest disabling-setting-drops-queued-mirror-work-test
(async done
(let [{:keys [platform writes]} (fake-platform)
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "Page A"}
:blocks [{:block/title "before"}]}]})
page (db-test/find-page-by-title @conn "Page A")
block (first-block page)
tx-report (d/with @conn [{:db/id (:db/id block)
:block/title "after"}])]
(markdown-mirror/set-enabled! test-repo true)
(-> (p/let [_ (markdown-mirror/<handle-tx-report! test-repo conn tx-report {:platform platform
:defer? true})
_ (markdown-mirror/set-enabled! test-repo false)
_ (markdown-mirror/<flush-repo! test-repo {:platform platform})]
(is (empty? @writes)))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest repeated-edits-coalesce-to-latest-content-test
(async done
(let [{:keys [platform writes]} (fake-platform)
page-uuid #uuid "44444444-4444-4444-8444-444444444444"
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "Page A"
:block/uuid page-uuid}
:blocks [{:block/title "before"}]}]})
page (db-test/find-page-by-title @conn "Page A")
block (first-block page)
tx-report-1 (d/with @conn [{:db/id (:db/id block)
:block/title "middle"}])
_ (d/reset-conn! conn (:db-after tx-report-1))
tx-report-2 (d/with @conn [{:db/id (:db/id block)
:block/title "latest"}])]
(markdown-mirror/set-enabled! test-repo true)
(-> (p/let [_ (markdown-mirror/<handle-tx-report! test-repo conn tx-report-1 {:platform platform
:defer? true})
_ (markdown-mirror/<handle-tx-report! test-repo conn tx-report-2 {:platform platform
:defer? true})
_ (markdown-mirror/<flush-repo! test-repo {:platform platform})]
(is (= [[(page-path "pages/Page A.md") "- latest"]]
@writes)))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest rename-removes-old-mirror-path-test
(async done
(let [{:keys [platform files deletes]} (fake-platform)
page-uuid #uuid "55555555-5555-4555-8555-555555555555"
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "Old Name"
:block/uuid page-uuid}
:blocks [{:block/title "body"}]}]})
page (db-test/find-page-by-title @conn "Old Name")
old-path (page-path "pages/Old Name.md")
_ (swap! files assoc old-path "- body")
tx-report (d/with @conn [{:db/id (:db/id page)
:block/title "New Name"
:block/name "new name"}])
_ (d/reset-conn! conn (:db-after tx-report))]
(markdown-mirror/set-enabled! test-repo true)
(-> (markdown-mirror/<handle-tx-report! test-repo conn tx-report {:platform platform})
(p/then (fn [_]
(is (= [old-path] @deletes))
(is (= "- body"
(get @files (page-path "pages/New Name.md"))))
(is (nil? (get @files old-path)))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest delete-removes-mirror-file-test
(async done
(let [{:keys [platform files deletes writes]} (fake-platform)
page-uuid #uuid "66666666-6666-4666-8666-666666666666"
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "Delete Me"
:block/uuid page-uuid}
:blocks [{:block/title "body"}]}]})
page (db-test/find-page-by-title @conn "Delete Me")
old-path (page-path "pages/Delete Me.md")
_ (swap! files assoc old-path "- body")
tx-report (d/with @conn [[:db/retractEntity (:db/id page)]])
_ (d/reset-conn! conn (:db-after tx-report))]
(markdown-mirror/set-enabled! test-repo true)
(-> (markdown-mirror/<handle-tx-report! test-repo conn tx-report {:platform platform})
(p/then (fn [_]
(is (= [old-path] @deletes))
(is (empty? @writes))
(is (nil? (get @files old-path)))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest unchanged-content-skips-write-test
(async done
(let [{:keys [platform files writes]} (fake-platform)
page-uuid #uuid "77777777-7777-4777-8777-777777777777"
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "Page A"
:block/uuid page-uuid}
:blocks [{:block/title "same"}]}]})
page (db-test/find-page-by-title @conn "Page A")
path (page-path "pages/Page A.md")
_ (swap! files assoc path "- same")]
(-> (markdown-mirror/<mirror-page! test-repo @conn (:db/id page) {:platform platform})
(p/then (fn [_]
(is (empty? @writes))
(is (= "- same" (get @files path)))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest windows-reserved-journal-filename-fails-with-diagnostic-test
(async done
(let [{:keys [platform writes]} (fake-platform)
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "CON"
:block/name "con"
:block/journal-day 20240507
:block/tags #{:logseq.class/Journal}}
:blocks [{:block/title "journal"}]}]})
journal (db-test/find-journal-by-journal-day @conn 20240507)]
(-> (markdown-mirror/<mirror-page! test-repo @conn (:db/id journal) {:platform platform
:journal-file-stem-fn (constantly "CON")})
(p/then (fn [result]
(is (= :error (:status result)))
(is (= :invalid-file-name (:reason result)))
(is (empty? @writes))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest duplicate-journal-day-fails-without-overwrite-test
(async done
(let [{:keys [platform writes]} (fake-platform)
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "May 7th, 2024"
:block/name "may 7th, 2024"
:block/journal-day 20240507
:block/tags #{:logseq.class/Journal}}
:blocks [{:block/title "first"}]}
{:page {:block/title "May 07, 2024"
:block/name "may 07, 2024"
:block/journal-day 20240507
:block/tags #{:logseq.class/Journal}}
:blocks [{:block/title "second"}]}]})
journal (db-test/find-journal-by-journal-day @conn 20240507)]
(-> (markdown-mirror/<mirror-page! test-repo @conn (:db/id journal) {:platform platform})
(p/then (fn [result]
(is (= :error (:status result)))
(is (= :duplicate-journal-day (:reason result)))
(is (empty? @writes))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))

View File

@@ -77,6 +77,26 @@
(is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest node-platform-writes-text-atomically-and-deletes-files
(async done
(let [root-dir (node-helper/create-tmp-dir "platform-node-text-files")]
(-> (p/let [platform (platform-node/node-platform {:root-dir root-dir})
storage (:storage platform)
path "graph-a/markdown-mirror/pages/page.md"
_ ((:write-text-atomic! storage) path "mirror")
content ((:read-text! storage) path)
_ ((:delete-file! storage) path)
deleted-content (-> ((:read-text! storage) path)
(p/catch (constantly nil)))]
(is (= "mirror" content))
(is (nil? deleted-content))
(is (empty? (filter #(string/includes? % ".tmp-")
(array-seq (fs/readdirSync
(node-path/join root-dir "graphs" "graph-a" "markdown-mirror" "pages")))))))
(p/catch (fn [e]
(is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest node-platform-cli-owner-bypasses-keychain-in-cli-e2e-test
(async done
(let [root-dir (node-helper/create-tmp-dir "platform-node-cli-secrets")

View File

@@ -1,6 +1,7 @@
(ns frontend.worker.platform-test
(:require [cljs.test :refer [async deftest is]]
[frontend.worker.platform :as platform]
[frontend.worker.platform.browser :as platform-browser]
[promesa.core :as p]))
(deftest kv-get-normalizes-undefined-to-nil-test
@@ -26,3 +27,22 @@
(p/catch (fn [e]
(is false (str e))))
(p/finally done))))
(deftest browser-platform-mirror-storage-is-unsupported-test
(let [original-location (.-location js/globalThis)]
(try
(set! (.-location js/globalThis) #js {:href "http://localhost/?electron=true"
:search "?electron=true"})
(let [storage (:storage (platform-browser/browser-platform))]
(doseq [[f args] [[(:mirror-read-text! storage) ["mirror.md"]]
[(:write-text-atomic! storage) ["mirror.md" "content"]]
[(:delete-file! storage) ["mirror.md"]]]]
(try
(apply f args)
(is false "Expected browser mirror storage to throw")
(catch :default e
(is (= {:platform :browser
:feature :markdown-mirror}
(ex-data e)))))))
(finally
(set! (.-location js/globalThis) original-location)))))