clone/logseq
1
0
Fork 0
mirror of https://github.com/logseq/logseq.git synced 2026-05-24 12:44:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add markdown mirror

Browse Source 
Add markdown mirror generation for DB graphs, including page and journal paths, regeneration, debounced write handling, and settings UI.

Serialize page and block property values into mirrored Markdown by resolving property-value entities to their display content.

Persist mirror files through the node worker platform and make browser worker mirror storage fail fast as unsupported.

Fix Electron userAppCfgs writes to avoid returning unserializable Electron state over IPC.

Add ADR and targeted tests for mirror generation, worker wiring, platform storage, and graph path handling.
This commit is contained in:
Tienson Qin
2026-05-05 19:27:00 +08:00
parent 7a731c6044
commit 06dbef8715
21 changed files with 1351 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/electron/electron/handler.cljs
View File

@@ -315,7 +315,8 @@
(do (cfgs/set-item! k v)
(when (= k :spell-check)
(spell-check/apply-window-spellcheck! window (spell-check/session-spellcheck-enabled? v)))
(state/set-state! [:config k] v))
(state/set-state! [:config k] v)
nil)
(cfgs/get-item k))
config)))

47
src/main/frontend/components/settings.cljs
View File

@@ -745,6 +745,51 @@
{:left-label (t :settings.features/enable-flashcards)
:action (flashcards-enabled-switcher enable-flashcards?)}))
(rum/defcs markdown-mirror-row < rum/reactive
(rum/local false ::regenerating?)
[state t]
(let [enabled? (true? (state/sub [:electron/user-cfgs :feature/markdown-mirror?]))
*regenerating? (::regenerating? state)
regenerate! (fn []
(let [repo (state/get-current-repo)]
(when (and repo @state/*db-worker (not @*regenerating?))
(reset! *regenerating? true)
(-> (state/<invoke-db-worker :thread-api/markdown-mirror-regenerate repo)
(p/then (fn [_]
(notification/show!
(t :settings.features/markdown-mirror-regenerate-success)
:success)))
(p/catch (fn [error]
(log/error :markdown-mirror/regenerate-failed
{:repo repo
:error error})
(notification/show!
(t :settings.features/markdown-mirror-regenerate-error (str error))
:error)))
(p/finally #(reset! *regenerating? false))))))]
(toggle
"markdown-mirror"
(t :settings.features/markdown-mirror)
enabled?
#(let [next-enabled? (not enabled?)
repo (state/get-current-repo)]
(state/set-state! [:electron/user-cfgs :feature/markdown-mirror?] next-enabled?)
(ipc/ipc :userAppCfgs :feature/markdown-mirror? next-enabled?)
(when (and repo @state/*db-worker)
(-> (state/<invoke-db-worker :thread-api/markdown-mirror-set-enabled repo next-enabled?)
(p/catch (fn [error]
(log/error :markdown-mirror/settings-sync-failed
{:repo repo
:error error}))))))
[:div.flex.items-center.gap-2.flex-wrap
[:span.text-sm.opacity-50 (t :settings.features/markdown-mirror-desc)]
(ui/button
(t :settings.features/markdown-mirror-regenerate)
:icon "refresh"
:class "text-sm"
:disabled @*regenerating?
:on-click regenerate!)])))
(defn https-user-agent-row [agent-opts]
(row-with-button-action
{:left-label (t :settings.advanced/network-proxy)
@@ -1056,6 +1101,8 @@
(plugin-system-switcher-row))
(when (util/electron?)
(http-server-switcher-row))
(when (util/electron?)
(markdown-mirror-row t))
(flashcards-switcher-row enable-flashcards?)
(when-not web-platform?
[:div.mt-1.sm:mt-0.sm:col-span-2

15
src/main/frontend/persist_db/browser.cljs
View File

@@ -193,10 +193,20 @@
(log/error :sqlite-error error)
(notification/show! (t :storage/sqlitedb-error error) :error))))
(defn- <sync-markdown-mirror-setting!
[repo]
(if (and (util/electron?) repo)
(state/<invoke-db-worker :thread-api/markdown-mirror-set-enabled
repo
(true? (get-in @state/state [:electron/user-cfgs :feature/markdown-mirror?])))
(p/resolved nil)))
(defrecord InBrowser []
protocol/PersistentDB
(<new [_this repo opts]
(state/<invoke-db-worker :thread-api/create-or-open-db repo opts))
(p/let [result (state/<invoke-db-worker :thread-api/create-or-open-db repo opts)
_ (<sync-markdown-mirror-setting! repo)]
result))
(<list-db [_this]
(-> (state/<invoke-db-worker :thread-api/list-db)
@@ -209,7 +219,8 @@
(state/<invoke-db-worker :thread-api/release-access-handles repo))
(<fetch-initial-data [_this repo opts]
(-> (p/let [_ (state/<invoke-db-worker :thread-api/create-or-open-db repo opts)]
(-> (p/let [_ (state/<invoke-db-worker :thread-api/create-or-open-db repo opts)
_ (<sync-markdown-mirror-setting! repo)]
(state/<invoke-db-worker :thread-api/get-initial-data repo opts))
(p/catch sqlite-error-handler)))

15
src/main/frontend/worker/db_core.cljs
View File

@@ -17,6 +17,7 @@
[frontend.worker.db.migrate :as db-migrate]
[frontend.worker.db.validate :as worker-db-validate]
[frontend.worker.export :as worker-export]
[frontend.worker.markdown-mirror :as markdown-mirror]
[frontend.worker.pipeline :as worker-pipeline]
[frontend.worker.platform :as platform]
[frontend.worker.publish]
@@ -1094,6 +1095,20 @@
(worker-state/set-new-state! new-state)
nil)
(def-thread-api :thread-api/markdown-mirror-set-enabled
[repo enabled?]
(markdown-mirror/set-enabled! repo enabled?)
nil)
(def-thread-api :thread-api/markdown-mirror-flush
[repo]
(markdown-mirror/<flush-repo! repo {}))
(def-thread-api :thread-api/markdown-mirror-regenerate
[repo]
(when-let [conn (worker-state/get-datascript-conn repo)]
(markdown-mirror/<mirror-repo! repo @conn {})))
(def-thread-api :thread-api/export-get-debug-datoms
[repo]
(when-let [conn (worker-state/get-datascript-conn repo)]

5
src/main/frontend/worker/db_listener.cljs
View File

@@ -3,6 +3,7 @@
(:require [datascript.core :as d]
[frontend.common.thread-api :as thread-api]
[frontend.worker.pipeline :as worker-pipeline]
[frontend.worker.markdown-mirror :as markdown-mirror]
[frontend.worker.search :as search]
[frontend.worker.shared-service :as shared-service]
[frontend.worker.state :as worker-state]
@@ -60,6 +61,10 @@
[_ {:keys [repo]} tx-report]
(db-sync/handle-local-tx! repo tx-report))
(defmethod listen-db-changes :markdown-mirror
[_ {:keys [repo]} tx-report]
(markdown-mirror/<handle-tx-report! repo nil tx-report {:defer? true}))
(defn listen-db-changes!
[repo conn & {:keys [handler-keys]}]
(let [handlers (if (seq handler-keys)

357
src/main/frontend/worker/markdown_mirror.cljs Normal file
View File

@@ -0,0 +1,357 @@
(ns frontend.worker.markdown-mirror
"Markdown mirror derived-file support for DB graphs."
(:require [clojure.string :as string]
[datascript.core :as d]
[frontend.worker.graph-dir :as graph-dir]
[frontend.worker.platform :as platform]
[lambdaisland.glogi :as log]
[logseq.cli.common.file :as common-file]
[logseq.db :as ldb]
[promesa.core :as p]))
(defn repo-mirror-dir
[repo]
(str (graph-dir/repo->encoded-graph-dir-name repo) "/markdown-mirror"))
(def ^:private invalid-file-name-chars-re
#"[<>:\"|?*\\/]")
(def ^:private ascii-control-re
#"[\x00-\x1F]")
(def ^:private trailing-space-or-dot-re
#"[ \.]+$")
(def ^:private reserved-windows-device-names
(into #{"CON" "PRN" "AUX" "NUL"}
(concat (map #(str "COM" %) (range 1 10))
(map #(str "LPT" %) (range 1 10)))))
(def ^:private max-file-stem-length 160)
(defonce ^:private *repo->enabled? (atom {}))
(defonce ^:private *repo->queued-page-jobs (atom {}))
(defonce ^:private *repo->flush-timeout (atom {}))
(defn- normalize-unicode
[s]
(let [s (str s)]
(if (fn? (.-normalize s))
(.normalize s "NFC")
s)))
(defn- reserved-windows-device-name?
[s]
(contains? reserved-windows-device-names
(string/upper-case s)))
(defn normalize-file-stem
[s]
(when (some? s)
(let [s' (-> (normalize-unicode s)
(string/replace invalid-file-name-chars-re "_")
(string/replace ascii-control-re "_")
(string/replace trailing-space-or-dot-re ""))
s' (if (> (count s') max-file-stem-length)
(subs s' 0 max-file-stem-length)
s')]
(when (and (not (string/blank? s'))
(not (reserved-windows-device-name? s')))
s'))))
(defn- journal-file-stem
[journal-day]
(when journal-day
(let [s (str journal-day)]
(when (= 8 (count s))
(str (subs s 0 4) "_" (subs s 4 6) "_" (subs s 6 8))))))
(defn page-relative-path
([db page]
(page-relative-path db page {}))
([db page {:keys [journal-file-stem-fn]
:or {journal-file-stem-fn journal-file-stem}}]
(when page
(if (ldb/journal? page)
(when-let [stem (normalize-file-stem (journal-file-stem-fn (:block/journal-day page)))]
(str "journals/" stem ".md"))
(when-let [stem (normalize-file-stem (:block/title page))]
(let [duplicate-pages (->> (d/datoms db :avet :block/title (:block/title page))
(map #(d/entity db (:e %)))
(filter #(and (ldb/page? %)
(not (ldb/journal? %))))
(sort-by (comp str :block/uuid)))
index (inc (or (first (keep-indexed
(fn [idx p]
(when (= (:block/uuid page) (:block/uuid p))
idx))
duplicate-pages))
0))
stem' (if (= 1 index)
stem
(str stem " (" index ")"))]
(str "pages/" stem' ".md")))))))
(defn- mirror-path
[repo relative-path]
(str (repo-mirror-dir repo) "/" relative-path))
(defn- page-id-for-entity
[db eid]
(when-let [entity (d/entity db eid)]
(cond
(ldb/page? entity) (:db/id entity)
(:block/page entity) (:db/id (:block/page entity))
(and (:block/parent entity) (ldb/page? (:block/parent entity))) (:db/id (:block/parent entity))
(some-> entity :block/parent :block/page) (:db/id (:block/page (:block/parent entity))))))
(defn affected-page-ids
[{:keys [db-before db-after tx-data]}]
(->> tx-data
(mapcat (fn [{:keys [e a v]}]
(cond-> [(page-id-for-entity db-before e)
(page-id-for-entity db-after e)]
(= a :block/page)
(conj v))))
(remove nil?)
set))
(defn set-enabled!
[repo enabled?]
(if enabled?
(swap! *repo->enabled? assoc repo true)
(do
(when-let [timeout-id (get @*repo->flush-timeout repo)]
(js/clearTimeout timeout-id))
(swap! *repo->enabled? dissoc repo)
(swap! *repo->queued-page-jobs dissoc repo)
(swap! *repo->flush-timeout dissoc repo)))
nil)
(defn enabled?
[repo]
(true? (get @*repo->enabled? repo)))
(defn- storage
[platform*]
(:storage platform*))
(defn- <read-text
[platform* path]
(if-let [f (or (:mirror-read-text! (storage platform*))
(:read-text! (storage platform*)))]
(-> (f path)
(p/catch (constantly nil)))
(p/rejected (ex-info "platform storage/read-text! missing" {:path path}))))
(defn- <write-text-atomic!
[platform* path content]
(if-let [f (:write-text-atomic! (storage platform*))]
(f path content)
(p/rejected (ex-info "platform storage/write-text-atomic! missing" {:path path}))))
(defn- <delete-file!
[platform* path]
(if-let [f (:delete-file! (storage platform*))]
(f path)
(p/rejected (ex-info "platform storage/delete-file! missing" {:path path}))))
(defn- supported-runtime?
[platform*]
(or (= :node (get-in platform* [:env :runtime]))
(and (= :browser (get-in platform* [:env :runtime]))
(= :electron (get-in platform* [:env :owner-source])))))
(defn- duplicate-journal-day?
[db journal-day]
(when journal-day
(< 1 (count (d/datoms db :avet :block/journal-day journal-day)))))
(defn- render-page-content
[db page options]
(common-file/block->content
db
(:block/uuid page)
{:include-page-properties? true}
{:export-bullet-indentation (or (:export-bullet-indentation options) " ")
:date-formatter (:date-formatter options)}))
(defn- mirrorable-page?
[page]
(and (ldb/page? page)
(not (ldb/built-in? page))
(not (ldb/property? page))
(not (ldb/hidden? page))
(not (:logseq.property.user/email page))))
(defn- mirrorable-pages
[db]
(->> (d/datoms db :avet :block/name)
(map #(d/entity db (:e %)))
(filter mirrorable-page?)
(sort-by (fn [page]
[(if (ldb/journal? page) 0 1)
(str (:block/journal-day page))
(string/lower-case (or (:block/title page) ""))
(str (:block/uuid page))]))))
(defn- <write-if-changed!
[platform* path content]
(p/let [current (<read-text platform* path)]
(if (= current content)
{:status :skipped
:reason :unchanged
:path path}
(p/let [_ (<write-text-atomic! platform* path content)]
{:status :written
:path path}))))
(defn- invalid-file-name-result
[repo page]
(let [result {:status :error
:reason :invalid-file-name
:repo repo
:page-uuid (:block/uuid page)}]
(log/error :markdown-mirror/invalid-file-name result)
result))
(defn <mirror-page!
[repo db page-id {:keys [platform] :as opts}]
(let [platform* (or platform (platform/current))]
(if-not (supported-runtime? platform*)
(p/resolved {:status :skipped
:reason :unsupported-runtime})
(if-let [page (d/entity db page-id)]
(cond
(not (mirrorable-page? page))
(p/resolved {:status :skipped
:reason :excluded-page
:repo repo
:page-id page-id})
(and (ldb/journal? page)
(duplicate-journal-day? db (:block/journal-day page)))
(let [result {:status :error
:reason :duplicate-journal-day
:repo repo
:journal-day (:block/journal-day page)
:page-uuid (:block/uuid page)}]
(log/error :markdown-mirror/duplicate-journal-day result)
(p/resolved result))
:else
(if-let [relative-path (page-relative-path db page opts)]
(let [path (mirror-path repo relative-path)
content (render-page-content db page opts)]
(<write-if-changed! platform* path content))
(p/resolved (invalid-file-name-result repo page))))
(p/resolved {:status :skipped
:reason :missing-page
:repo repo
:page-id page-id})))))
(defn- deleted-page?
[page]
(or (nil? page)
(not (mirrorable-page? page))))
(defn- page-job
[repo {:keys [db-before db-after]} page-id opts]
(let [before-page (d/entity db-before page-id)
after-page (d/entity db-after page-id)
old-relative-path (when before-page (page-relative-path db-before before-page opts))
new-relative-path (when after-page (page-relative-path db-after after-page opts))]
{:repo repo
:page-id page-id
:db db-after
:old-path (when old-relative-path (mirror-path repo old-relative-path))
:new-path (when new-relative-path (mirror-path repo new-relative-path))
:delete? (deleted-page? after-page)}))
(defn- merge-job
[old-job new-job]
(assoc new-job :old-path (or (:old-path old-job)
(:old-path new-job))))
(defn- queue-job!
[repo job]
(swap! *repo->queued-page-jobs update-in [repo (:page-id job)] merge-job job))
(defn- drain-repo-jobs!
[repo]
(let [jobs (vals (get @*repo->queued-page-jobs repo))]
(swap! *repo->queued-page-jobs dissoc repo)
jobs))
(declare <flush-repo!)
(defn- schedule-flush!
[repo opts]
(when-not (get @*repo->flush-timeout repo)
(let [timeout-id (js/setTimeout
(fn []
(swap! *repo->flush-timeout dissoc repo)
(-> (<flush-repo! repo opts)
(p/catch (fn [error]
(log/error :markdown-mirror/flush-failed
{:repo repo
:error error})))))
(or (:debounce-ms opts) 250))]
(swap! *repo->flush-timeout assoc repo timeout-id))))
(defn- <run-job!
[platform* {:keys [repo db page-id old-path new-path delete?] :as _job} opts]
(cond
delete?
(if old-path
(p/let [_ (<delete-file! platform* old-path)]
{:status :deleted
:path old-path})
(p/resolved {:status :skipped
:reason :missing-old-path}))
:else
(p/let [result (<mirror-page! repo db page-id (assoc opts :platform platform*))
_ (when (and old-path
new-path
(not= old-path new-path)
(= :written (:status result)))
(<delete-file! platform* old-path))]
result)))
(defn <handle-tx-report!
[repo _conn tx-report {:keys [platform defer?] :as opts}]
(let [platform* (or platform (platform/current))]
(if (and (enabled? repo)
(supported-runtime? platform*)
(not (get-in tx-report [:tx-meta :from-disk?])))
(let [jobs (map #(page-job repo tx-report % opts)
(affected-page-ids tx-report))]
(if defer?
(do
(doseq [job jobs] (queue-job! repo job))
(schedule-flush! repo (assoc opts :platform platform*))
(p/resolved {:status :queued
:count (count jobs)}))
(p/all (map #(<run-job! platform* % opts) jobs))))
(p/resolved {:status :skipped
:reason :disabled-or-unsupported}))))
(defn <flush-repo!
[repo {:keys [platform] :as opts}]
(let [platform* (or platform (platform/current))
jobs (drain-repo-jobs! repo)]
(p/all (map #(<run-job! platform* % opts) jobs))))
(defn <mirror-repo!
[repo db {:keys [platform] :as opts}]
(let [platform* (or platform (platform/current))]
(if-not (supported-runtime? platform*)
(p/resolved {:status :skipped
:reason :unsupported-runtime})
(p/let [results (p/all
(map #(<mirror-page! repo db (:db/id %) (assoc opts :platform platform*))
(mirrorable-pages db)))]
{:status :completed
:count (count results)
:results results}))))

14
src/main/frontend/worker/platform/browser.cljs
View File

@@ -155,8 +155,15 @@
(defn- asset-delete!
[repo file-name]
(-> (.unlink (browser-pfs) (asset-path repo file-name))
(p/catch (constantly nil))))
(let [^js pfs (browser-pfs)]
(-> (.unlink pfs (asset-path repo file-name))
(p/catch (constantly nil)))))
(defn- unsupported-mirror-storage!
[& _args]
(throw (ex-info "Markdown mirror storage is not supported in browser workers"
{:platform :browser
:feature :markdown-mirror})))
(defn- websocket-connect
[url]
@@ -201,6 +208,9 @@
:remove-vfs! remove-vfs!
:read-text! read-text!
:write-text! write-text!
:write-text-atomic! unsupported-mirror-storage!
:delete-file! unsupported-mirror-storage!
:mirror-read-text! unsupported-mirror-storage!
:asset-read-bytes! asset-read-bytes!
:asset-write-bytes! asset-write-bytes!
:asset-stat asset-stat

22
src/main/frontend/worker/platform/node.cljs
View File

@@ -252,6 +252,26 @@
_ (ensure-dir! dir)]
(fs/writeFile full-path text "utf8"))))
(defn- write-text-atomic!
[write-guard-fn data-dir path text]
(let [full-path (path-under-data-dir data-dir path)
dir (node-path/dirname full-path)
tmp-path (node-path/join dir (str "." (node-path/basename full-path) ".tmp-" (random-uuid)))]
(p/let [_ (when write-guard-fn
(write-guard-fn))
_ (ensure-dir! dir)
_ (fs/writeFile tmp-path text "utf8")
_ (fs/rename tmp-path full-path)]
nil)))
(defn- delete-file!
[write-guard-fn data-dir path]
(let [full-path (path-under-data-dir data-dir path)]
(p/let [_ (when write-guard-fn
(write-guard-fn))]
(-> (fs/rm full-path #js {:force true})
(p/catch (constantly nil))))))
(defn- asset-file-path
[data-dir repo file-name]
(node-path/join (repo-dir data-dir repo)
@@ -433,6 +453,8 @@
:remove-vfs! (fn [pool] (remove-vfs! pool))
:read-text! (fn [path] (read-text! data-dir path))
:write-text! (fn [path text] (write-text! write-guard-fn data-dir path text))
:write-text-atomic! (fn [path text] (write-text-atomic! write-guard-fn data-dir path text))
:delete-file! (fn [path] (delete-file! write-guard-fn data-dir path))
:asset-read-bytes! (fn [repo file-name]
(asset-read-bytes! data-dir repo file-name))
:asset-write-bytes! (fn [repo file-name payload]

5
src/resources/dicts/en.edn
View File

@@ -1631,6 +1631,11 @@
:settings.features/home-default-page-update-success "Home default page updated successfully!"
:settings.features/journals-enable-success "Journals enabled"
:settings.features/login-prompt "To access new features before anyone else you must be an Open Collective Sponsor or Backer of Logseq and therefore log in first."
:settings.features/markdown-mirror "Markdown Mirror"
:settings.features/markdown-mirror-desc "Write a derived Markdown copy of edited pages to the graph's markdown-mirror folder. Desktop only."
:settings.features/markdown-mirror-regenerate "Regenerate full mirror"
:settings.features/markdown-mirror-regenerate-error "Failed to regenerate Markdown Mirror: {1}"
:settings.features/markdown-mirror-regenerate-success "Markdown Mirror regenerated"
:settings.features/page-not-found "The page \"{1}\" doesn't exist yet. Please create that page first, and then try again."
:settings.features/plugin-system "Plugins"

5
src/resources/dicts/zh-cn.edn
View File

@@ -1621,6 +1621,11 @@
:settings.features/home-default-page-update-success "主页默认页面已更新成功！"
:settings.features/journals-enable-success "已启用日志页"
:settings.features/login-prompt "你必须是 Logseq 的 Open Collective Sponsor 或者 Backer 才能提前使用新功能（仍在测试中），因此需要登录。"
:settings.features/markdown-mirror "Markdown 镜像"
:settings.features/markdown-mirror-desc "将已编辑页面的派生 Markdown 副本写入图谱的 markdown-mirror 文件夹。仅桌面端可用。"
:settings.features/markdown-mirror-regenerate "重新生成完整镜像"
:settings.features/markdown-mirror-regenerate-error "重新生成 Markdown 镜像失败：{1}"
:settings.features/markdown-mirror-regenerate-success "Markdown 镜像已重新生成"
:settings.features/page-not-found "页面“{1}”尚不存在。请先创建该页面，然后再试一次。"
:settings.features/plugin-system "插件系统"

2
src/test/frontend/handler/editor_async_test.cljs
View File

@@ -1,5 +1,5 @@
(ns frontend.handler.editor-async-test
(:require [clojure.test :refer [is testing async use-fixtures]]
(:require [cljs.test :refer [is testing async use-fixtures]]
[datascript.core :as d]
[frontend.db :as db]
[frontend.handler.editor :as editor]

1
src/test/frontend/worker/db_core_test.cljs
View File

@@ -52,6 +52,7 @@
:thread-api/import-db-base64 :thread-api/search-blocks :thread-api/search-upsert-blocks :thread-api/search-delete-blocks
:thread-api/search-truncate-tables :thread-api/search-build-blocks-indice :thread-api/search-build-blocks-indice-in-worker
:thread-api/search-build-pages-indice :thread-api/apply-outliner-ops :thread-api/sync-app-state
:thread-api/markdown-mirror-set-enabled :thread-api/markdown-mirror-flush :thread-api/markdown-mirror-regenerate
:thread-api/export-get-debug-datoms :thread-api/export-get-all-page->content :thread-api/validate-db
:thread-api/recompute-checksum-diagnostics :thread-api/export-edn :thread-api/import-edn :thread-api/get-view-data
:thread-api/get-class-objects :thread-api/get-property-values :thread-api/get-bidirectional-properties

16
src/test/frontend/worker/db_listener_test.cljs
View File

@@ -1,6 +1,7 @@
(ns frontend.worker.db-listener-test
(:require [cljs.test :refer [deftest is testing]]
[frontend.worker.db-listener :as db-listener]))
[frontend.worker.db-listener :as db-listener]
[frontend.worker.markdown-mirror :as markdown-mirror]))
(deftest transit-safe-tx-meta-keeps-outliner-ops-test
(testing "worker tx-meta sanitization should preserve semantic outliner ops"
@@ -14,3 +15,16 @@
(is (= outliner-ops (:outliner-ops safe-tx-meta)))
(is (= outliner-ops (:db-sync/inverse-outliner-ops safe-tx-meta)))
(is (nil? (:error-handler safe-tx-meta))))))
(deftest markdown-mirror-listener-enqueues-worker-mirror-work-test
(let [calls (atom [])
tx-report {:tx-data [:tx]}]
(with-redefs [markdown-mirror/<handle-tx-report!
(fn [repo conn tx-report opts]
(swap! calls conj [repo conn tx-report opts]))]
((get-method db-listener/listen-db-changes :markdown-mirror)
:markdown-mirror
{:repo "repo"}
tx-report))
(is (= [["repo" nil tx-report {:defer? true}]]
@calls))))

1
src/test/frontend/worker/db_worker_test.cljs
View File

@@ -49,6 +49,7 @@
:thread-api/import-db-base64 :thread-api/search-blocks :thread-api/search-upsert-blocks :thread-api/search-delete-blocks
:thread-api/search-truncate-tables :thread-api/search-build-blocks-indice :thread-api/search-build-blocks-indice-in-worker
:thread-api/search-build-pages-indice :thread-api/apply-outliner-ops :thread-api/sync-app-state
:thread-api/markdown-mirror-set-enabled :thread-api/markdown-mirror-flush :thread-api/markdown-mirror-regenerate
:thread-api/export-get-debug-datoms :thread-api/export-get-all-page->content :thread-api/validate-db
:thread-api/recompute-checksum-diagnostics :thread-api/export-edn :thread-api/import-edn :thread-api/get-view-data
:thread-api/get-class-objects :thread-api/get-property-values :thread-api/get-bidirectional-properties

447
src/test/frontend/worker/markdown_mirror_test.cljs Normal file
View File

@@ -0,0 +1,447 @@
(ns frontend.worker.markdown-mirror-test
(:require [cljs.test :refer [async deftest is testing]]
[datascript.core :as d]
[frontend.worker.markdown-mirror :as markdown-mirror]
[logseq.db.test.helper :as db-test]
[promesa.core :as p]))
(def test-repo "logseq_db_graph-xxx")
(defn- fake-platform
([] (fake-platform {:runtime :node}))
([env]
(let [files (atom {})
writes (atom [])
deletes (atom [])]
{:platform {:env env
:storage {:read-text! (fn [path]
(p/resolved (get @files path)))
:write-text-atomic! (fn [path content]
(swap! writes conj [path content])
(swap! files assoc path content)
(p/resolved nil))
:delete-file! (fn [path]
(swap! deletes conj path)
(swap! files dissoc path)
(p/resolved nil))}
:broadcast {:post-message! (fn [& _] nil)}}
:files files
:writes writes
:deletes deletes})))
(defn- page-path [path]
(str (markdown-mirror/repo-mirror-dir test-repo) "/" path))
(defn- first-block [page]
(-> page :block/_page first))
(defn- <mirror-repo!
[& args]
(if-let [f (resolve 'frontend.worker.markdown-mirror/<mirror-repo!)]
(apply f args)
(p/resolved ::missing-mirror-repo-fn)))
(deftest normalize-file-name-is-cross-platform-and-deterministic-test
(testing "invalid filesystem characters and path separators are replaced"
(is (= "A_B_C_D_E_F_G_H"
(markdown-mirror/normalize-file-stem "A/B\\C:D<E>F\"G|H"))))
(testing "trailing spaces and dots are removed"
(is (= "title"
(markdown-mirror/normalize-file-stem "title. "))))
(testing "unicode is normalized before sanitizing"
(is (= (markdown-mirror/normalize-file-stem "e\u0301")
(markdown-mirror/normalize-file-stem "\u00e9"))))
(testing "reserved Windows device names are rejected"
(is (nil? (markdown-mirror/normalize-file-stem "CON")))
(is (nil? (markdown-mirror/normalize-file-stem "lpt9")))))
(deftest same-title-pages-write-distinct-stable-friendly-paths-test
(let [page-uuid-1 #uuid "11111111-1111-4111-8111-111111111111"
page-uuid-2 #uuid "22222222-2222-4222-8222-222222222222"
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "Same Name"
:block/uuid page-uuid-1}
:blocks [{:block/title "first"}]}
{:page {:block/title "Same Name"
:block/uuid page-uuid-2}
:blocks [{:block/title "second"}]}]})
pages (->> (d/datoms @conn :avet :block/title "Same Name")
(map #(d/entity @conn (:e %)))
(filter #(nil? (:block/page %)))
(sort-by (comp str :block/uuid)))
paths (mapv #(markdown-mirror/page-relative-path @conn %) pages)]
(is (= ["pages/Same Name.md"
"pages/Same Name (2).md"]
paths))))
(deftest page-references-remain-wiki-links-test
(async done
(let [{:keys [platform files]} (fake-platform)
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "Source"}
:blocks [{:block/title "See [[Foo]]"}]}
{:page {:block/title "Foo"}
:blocks [{:block/title "target"}]}
{:page {:block/title "Foo"}
:blocks [{:block/title "duplicate"}]}]})
page (db-test/find-page-by-title @conn "Source")]
(-> (markdown-mirror/<mirror-page! test-repo @conn (:db/id page) {:platform platform})
(p/then (fn [_]
(is (= "- See [[Foo]]"
(get @files (page-path "pages/Source.md"))))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest affected-page-ids-detects-edited-block-page-test
(let [conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "Page A"}
:blocks [{:block/title "before"}]}]})
page (db-test/find-page-by-title @conn "Page A")
block (first-block page)
tx-report (d/with @conn [{:db/id (:db/id block)
:block/title "after"}])]
(is (= #{(:db/id page)}
(markdown-mirror/affected-page-ids tx-report)))))
(deftest enabled-electron-edit-writes-page-mirror-test
(async done
(let [{:keys [platform files writes]} (fake-platform)
page-uuid #uuid "33333333-3333-4333-8333-333333333333"
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "Page A"
:block/uuid page-uuid}
:blocks [{:block/title "hello"}
{:block/title "world"}]}]})
page (db-test/find-page-by-title @conn "Page A")]
(-> (markdown-mirror/<mirror-page! test-repo @conn (:db/id page) {:platform platform})
(p/then (fn [_]
(let [path (page-path "pages/Page A.md")]
(is (= "- hello\n- world" (get @files path)))
(is (= [[path "- hello\n- world"]] @writes)))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest page-mirror-exports-property-values-test
(async done
(let [{:keys [platform files]} (fake-platform)
conn (db-test/create-conn-with-blocks
{:properties {:user.property/reproducible-steps {:logseq.property/type :default}
:user.property/rating {:logseq.property/type :number}}
:pages-and-blocks [{:page {:block/title "Issue"
:build/properties {:user.property/reproducible-steps "Open settings"
:logseq.property/heading 1}}
:blocks [{:block/title "TODO body"
:build/properties {:logseq.property/status :logseq.property/status.todo
:user.property/reproducible-steps "Click mirror"
:user.property/rating 5
:logseq.property/heading 2}}]}]})
page (db-test/find-page-by-title @conn "Issue")]
(-> (markdown-mirror/<mirror-page! test-repo @conn (:db/id page) {:platform platform})
(p/then (fn [_]
(let [content (get @files (page-path "pages/Issue.md"))]
(is (= "reproducible-steps:: Open settings\n- TODO body\n Status:: Todo\n reproducible-steps:: Click mirror\n rating:: 5"
content)))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest page-mirror-exports-page-property-values-test
(async done
(let [{:keys [platform files]} (fake-platform)
conn (db-test/create-conn-with-blocks
{:properties {:user.property/p1 {:logseq.property/type :default}
:user.property/p2 {:logseq.property/type :number}
:user.property/p3 {:logseq.property/type :default}}
:pages-and-blocks [{:page {:block/title "Page Props"
:build/properties {:user.property/p1 "hello"
:user.property/p2 1
:user.property/p3 "Author 1"}}
:blocks [{:block/title "body"}]}]})
page (db-test/find-page-by-title @conn "Page Props")]
(-> (markdown-mirror/<mirror-page! test-repo @conn (:db/id page) {:platform platform})
(p/then (fn [_]
(let [content (get @files (page-path "pages/Page Props.md"))]
(is (= "p1:: hello\np2:: 1\np3:: Author 1\n- body"
content)))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest journal-mirror-exports-page-and-block-property-values-test
(async done
(let [{:keys [platform files]} (fake-platform)
conn (db-test/create-conn-with-blocks
{:properties {:user.property/p1 {:logseq.property/type :default}
:user.property/p2 {:logseq.property/type :number}
:user.property/p3 {:logseq.property/type :default}}
:pages-and-blocks [{:page {:block/title "May 5th, 2026"
:block/name "may 5th, 2026"
:block/journal-day 20260505
:block/tags #{:logseq.class/Journal}
:build/properties {:user.property/p1 "hey"}}
:blocks [{:block/title "TODO hello great test"
:build/properties {:logseq.property/status :logseq.property/status.todo
:user.property/p1 "hello"
:user.property/p2 1
:user.property/p3 "Author 1"}}]}]})
journal (db-test/find-journal-by-journal-day @conn 20260505)]
(-> (markdown-mirror/<mirror-page! test-repo @conn (:db/id journal) {:platform platform})
(p/then (fn [_]
(let [content (get @files (page-path "journals/2026_05_05.md"))]
(is (= "p1:: hey\n- TODO hello great test\n Status:: Todo\n p1:: hello\n p2:: 1\n p3:: Author 1"
content)))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest full-regeneration-writes-existing-non-built-in-non-property-pages-test
(async done
(let [{:keys [platform files]} (fake-platform)
conn (db-test/create-conn-with-blocks
{:properties {:rating {:logseq.property/type :default}}
:pages-and-blocks [{:page {:block/title "Page A"}
:blocks [{:block/title "alpha"}]}
{:page {:block/title "Journal"
:block/journal-day 20240508
:block/tags #{:logseq.class/Journal}}
:blocks [{:block/title "journal"}]}
{:page {:block/title "Built In"
:build/properties {:logseq.property/built-in? true}}
:blocks [{:block/title "system"}]}
{:page {:block/title "Project"
:block/tags #{:logseq.class/Tag}
:db/ident :user.class/Project}
:blocks [{:block/title "class"}]}
{:page {:block/title "rating"
:block/tags #{:logseq.class/Property}
:db/ident :user.property/rating}
:blocks [{:block/title "property"}]}]})]
(-> (<mirror-repo! test-repo @conn {:platform platform})
(p/then (fn [result]
(is (not= ::missing-mirror-repo-fn result))
(is (= "- alpha"
(get @files (page-path "pages/Page A.md"))))
(is (= "- journal"
(get @files (page-path "journals/2024_05_08.md"))))
(is (= "- class"
(get @files (page-path "pages/Project.md"))))
(is (nil? (get @files (page-path "pages/Built In.md"))))
(is (nil? (get @files (page-path "pages/rating.md"))))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest electron-browser-worker-runtime-is-supported-test
(async done
(let [{:keys [platform files]} (fake-platform {:runtime :browser
:owner-source :electron})
page-uuid #uuid "88888888-8888-4888-8888-888888888888"
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "Page A"
:block/uuid page-uuid}
:blocks [{:block/title "desktop"}]}]})
page (db-test/find-page-by-title @conn "Page A")]
(-> (markdown-mirror/<mirror-page! test-repo @conn (:db/id page) {:platform platform})
(p/then (fn [_]
(is (= "- desktop"
(get @files (page-path "pages/Page A.md"))))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest non-electron-browser-runtime-is-skipped-test
(async done
(let [{:keys [platform writes]} (fake-platform {:runtime :browser
:owner-source :browser})
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "Page A"}
:blocks [{:block/title "web"}]}]})
page (db-test/find-page-by-title @conn "Page A")]
(-> (markdown-mirror/<mirror-page! test-repo @conn (:db/id page) {:platform platform})
(p/then (fn [result]
(is (= :skipped (:status result)))
(is (= :unsupported-runtime (:reason result)))
(is (empty? @writes))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest enabled-electron-edit-writes-journal-mirror-test
(async done
(let [{:keys [platform files]} (fake-platform)
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:build/journal 20240506}
:blocks [{:block/title "journal item"}]}]})
journal (db-test/find-journal-by-journal-day @conn 20240506)]
(-> (markdown-mirror/<mirror-page! test-repo @conn (:db/id journal) {:platform platform})
(p/then (fn [_]
(is (= "- journal item"
(get @files (page-path "journals/2024_05_06.md"))))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest disabled-setting-does-not-write-mirror-test
(async done
(let [{:keys [platform writes]} (fake-platform)
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "Page A"}
:blocks [{:block/title "before"}]}]})
page (db-test/find-page-by-title @conn "Page A")
block (first-block page)
tx-report (d/with @conn [{:db/id (:db/id block)
:block/title "after"}])]
(markdown-mirror/set-enabled! test-repo false)
(-> (markdown-mirror/<handle-tx-report! test-repo conn tx-report {:platform platform})
(p/then (fn [_]
(is (empty? @writes))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest disabling-setting-drops-queued-mirror-work-test
(async done
(let [{:keys [platform writes]} (fake-platform)
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "Page A"}
:blocks [{:block/title "before"}]}]})
page (db-test/find-page-by-title @conn "Page A")
block (first-block page)
tx-report (d/with @conn [{:db/id (:db/id block)
:block/title "after"}])]
(markdown-mirror/set-enabled! test-repo true)
(-> (p/let [_ (markdown-mirror/<handle-tx-report! test-repo conn tx-report {:platform platform
:defer? true})
_ (markdown-mirror/set-enabled! test-repo false)
_ (markdown-mirror/<flush-repo! test-repo {:platform platform})]
(is (empty? @writes)))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest repeated-edits-coalesce-to-latest-content-test
(async done
(let [{:keys [platform writes]} (fake-platform)
page-uuid #uuid "44444444-4444-4444-8444-444444444444"
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "Page A"
:block/uuid page-uuid}
:blocks [{:block/title "before"}]}]})
page (db-test/find-page-by-title @conn "Page A")
block (first-block page)
tx-report-1 (d/with @conn [{:db/id (:db/id block)
:block/title "middle"}])
_ (d/reset-conn! conn (:db-after tx-report-1))
tx-report-2 (d/with @conn [{:db/id (:db/id block)
:block/title "latest"}])]
(markdown-mirror/set-enabled! test-repo true)
(-> (p/let [_ (markdown-mirror/<handle-tx-report! test-repo conn tx-report-1 {:platform platform
:defer? true})
_ (markdown-mirror/<handle-tx-report! test-repo conn tx-report-2 {:platform platform
:defer? true})
_ (markdown-mirror/<flush-repo! test-repo {:platform platform})]
(is (= [[(page-path "pages/Page A.md") "- latest"]]
@writes)))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest rename-removes-old-mirror-path-test
(async done
(let [{:keys [platform files deletes]} (fake-platform)
page-uuid #uuid "55555555-5555-4555-8555-555555555555"
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "Old Name"
:block/uuid page-uuid}
:blocks [{:block/title "body"}]}]})
page (db-test/find-page-by-title @conn "Old Name")
old-path (page-path "pages/Old Name.md")
_ (swap! files assoc old-path "- body")
tx-report (d/with @conn [{:db/id (:db/id page)
:block/title "New Name"
:block/name "new name"}])
_ (d/reset-conn! conn (:db-after tx-report))]
(markdown-mirror/set-enabled! test-repo true)
(-> (markdown-mirror/<handle-tx-report! test-repo conn tx-report {:platform platform})
(p/then (fn [_]
(is (= [old-path] @deletes))
(is (= "- body"
(get @files (page-path "pages/New Name.md"))))
(is (nil? (get @files old-path)))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest delete-removes-mirror-file-test
(async done
(let [{:keys [platform files deletes writes]} (fake-platform)
page-uuid #uuid "66666666-6666-4666-8666-666666666666"
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "Delete Me"
:block/uuid page-uuid}
:blocks [{:block/title "body"}]}]})
page (db-test/find-page-by-title @conn "Delete Me")
old-path (page-path "pages/Delete Me.md")
_ (swap! files assoc old-path "- body")
tx-report (d/with @conn [[:db/retractEntity (:db/id page)]])
_ (d/reset-conn! conn (:db-after tx-report))]
(markdown-mirror/set-enabled! test-repo true)
(-> (markdown-mirror/<handle-tx-report! test-repo conn tx-report {:platform platform})
(p/then (fn [_]
(is (= [old-path] @deletes))
(is (empty? @writes))
(is (nil? (get @files old-path)))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest unchanged-content-skips-write-test
(async done
(let [{:keys [platform files writes]} (fake-platform)
page-uuid #uuid "77777777-7777-4777-8777-777777777777"
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "Page A"
:block/uuid page-uuid}
:blocks [{:block/title "same"}]}]})
page (db-test/find-page-by-title @conn "Page A")
path (page-path "pages/Page A.md")
_ (swap! files assoc path "- same")]
(-> (markdown-mirror/<mirror-page! test-repo @conn (:db/id page) {:platform platform})
(p/then (fn [_]
(is (empty? @writes))
(is (= "- same" (get @files path)))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest windows-reserved-journal-filename-fails-with-diagnostic-test
(async done
(let [{:keys [platform writes]} (fake-platform)
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "CON"
:block/name "con"
:block/journal-day 20240507
:block/tags #{:logseq.class/Journal}}
:blocks [{:block/title "journal"}]}]})
journal (db-test/find-journal-by-journal-day @conn 20240507)]
(-> (markdown-mirror/<mirror-page! test-repo @conn (:db/id journal) {:platform platform
:journal-file-stem-fn (constantly "CON")})
(p/then (fn [result]
(is (= :error (:status result)))
(is (= :invalid-file-name (:reason result)))
(is (empty? @writes))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest duplicate-journal-day-fails-without-overwrite-test
(async done
(let [{:keys [platform writes]} (fake-platform)
conn (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "May 7th, 2024"
:block/name "may 7th, 2024"
:block/journal-day 20240507
:block/tags #{:logseq.class/Journal}}
:blocks [{:block/title "first"}]}
{:page {:block/title "May 07, 2024"
:block/name "may 07, 2024"
:block/journal-day 20240507
:block/tags #{:logseq.class/Journal}}
:blocks [{:block/title "second"}]}]})
journal (db-test/find-journal-by-journal-day @conn 20240507)]
(-> (markdown-mirror/<mirror-page! test-repo @conn (:db/id journal) {:platform platform})
(p/then (fn [result]
(is (= :error (:status result)))
(is (= :duplicate-journal-day (:reason result)))
(is (empty? @writes))))
(p/catch (fn [e] (is false (str "unexpected error: " e))))
(p/finally done)))))

20
src/test/frontend/worker/platform_node_test.cljs
View File

@@ -77,6 +77,26 @@
(is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest node-platform-writes-text-atomically-and-deletes-files
(async done
(let [root-dir (node-helper/create-tmp-dir "platform-node-text-files")]
(-> (p/let [platform (platform-node/node-platform {:root-dir root-dir})
storage (:storage platform)
path "graph-a/markdown-mirror/pages/page.md"
_ ((:write-text-atomic! storage) path "mirror")
content ((:read-text! storage) path)
_ ((:delete-file! storage) path)
deleted-content (-> ((:read-text! storage) path)
(p/catch (constantly nil)))]
(is (= "mirror" content))
(is (nil? deleted-content))
(is (empty? (filter #(string/includes? % ".tmp-")
(array-seq (fs/readdirSync
(node-path/join root-dir "graphs" "graph-a" "markdown-mirror" "pages")))))))
(p/catch (fn [e]
(is false (str "unexpected error: " e))))
(p/finally done)))))
(deftest node-platform-cli-owner-bypasses-keychain-in-cli-e2e-test
(async done
(let [root-dir (node-helper/create-tmp-dir "platform-node-cli-secrets")

20
src/test/frontend/worker/platform_test.cljs
View File

@@ -1,6 +1,7 @@
(ns frontend.worker.platform-test
(:require [cljs.test :refer [async deftest is]]
[frontend.worker.platform :as platform]
[frontend.worker.platform.browser :as platform-browser]
[promesa.core :as p]))
(deftest kv-get-normalizes-undefined-to-nil-test
@@ -26,3 +27,22 @@
(p/catch (fn [e]
(is false (str e))))
(p/finally done))))
(deftest browser-platform-mirror-storage-is-unsupported-test
(let [original-location (.-location js/globalThis)]
(try
(set! (.-location js/globalThis) #js {:href "http://localhost/?electron=true"
:search "?electron=true"})
(let [storage (:storage (platform-browser/browser-platform))]
(doseq [[f args] [[(:mirror-read-text! storage) ["mirror.md"]]
[(:write-text-atomic! storage) ["mirror.md" "content"]]
[(:delete-file! storage) ["mirror.md"]]]]
(try
(apply f args)
(is false "Expected browser mirror storage to throw")
(catch :default e
(is (= {:platform :browser
:feature :markdown-mirror}
(ex-data e)))))))
(finally
(set! (.-location js/globalThis) original-location)))))