diff --git a/clj-e2e/test/logseq/e2e/outliner_basic_test.clj b/clj-e2e/test/logseq/e2e/outliner_basic_test.clj index b018ed281c..a221182fab 100644 --- a/clj-e2e/test/logseq/e2e/outliner_basic_test.clj +++ b/clj-e2e/test/logseq/e2e/outliner_basic_test.clj @@ -54,6 +54,7 @@ (b/new-blocks ["b3" ""]) (util/input-command "Node embed") (util/press-seq "Page embed" {:delay 60}) + (w/wait-for "#ac-0.menu-link:has-text('Page embed')") (k/press "Enter" {:delay 60}) (util/exit-edit) (b/new-blocks ["b4"]) diff --git a/deps/cli/src/logseq/cli/common/export/common.cljs b/deps/cli/src/logseq/cli/common/export/common.cljs index d45e32ce18..16fdea0828 100644 --- a/deps/cli/src/logseq/cli/common/export/common.cljs +++ b/deps/cli/src/logseq/cli/common/export/common.cljs @@ -2,7 +2,6 @@ "common fns for exporting. exclude some fns which produce lazy-seq, which can cause strange behaviors when use together with dynamic var." - (:refer-clojure :exclude [map filter]) (:require [cljs.core.match :refer [match]] [clojure.string :as string] [datascript.core :as d] diff --git a/deps/cli/src/logseq/cli/common/export/text.cljs b/deps/cli/src/logseq/cli/common/export/text.cljs index 9edff27868..4b841b3579 100644 --- a/deps/cli/src/logseq/cli/common/export/text.cljs +++ b/deps/cli/src/logseq/cli/common/export/text.cljs @@ -21,8 +21,8 @@ (declare inline-ast->simple-ast block-ast->simple-ast) -(defn- block-heading - [{:keys [title _tags marker level _numbering priority _anchor _meta _unordered size]}] +(defn- block-heading-prefix + [{:keys [marker level priority size]}] (let [indent-style (get-in *state* [:export-options :indent-style]) priority* (and priority (raw-text (cli-export-common/priority->string priority))) heading* (if (= indent-style "dashes") @@ -31,23 +31,80 @@ size* (and size [space (raw-text (reduce str (repeat size "#")))]) marker* (and marker (raw-text marker))] (set! *state* (assoc *state* :current-level level)) - (let [simple-asts - (removev nil? (concatv - (when (and (get-in *state* [:export-options :newline-after-block]) - (not (get-in *state* [:newline-after-block :current-block-is-first-heading-block?]))) - [(newline* 2)]) - heading* size* - [space marker* space priority* space] - (mapcatv inline-ast->simple-ast title) - [(newline* 1)]))] + (removev nil? (concatv heading* size* + [space marker* space priority* space])))) + +(defn- heading-continuation-indent + [{:keys [level]}] + (case (get-in *state* [:export-options :indent-style]) + "dashes" (indent (dec level) 2) + ("spaces" "no-indent") (indent (dec level) 0) + (assert false (print-str "unknown indent-style:" (get-in *state* [:export-options :indent-style]))))) + +(defn- block-heading + [{:keys [title] :as heading}] + (let [simple-asts + (removev nil? (concatv + (when (and (get-in *state* [:export-options :newline-after-block]) + (not (get-in *state* [:newline-after-block :current-block-is-first-heading-block?]))) + [(newline* 2)]) + (block-heading-prefix heading) + (mapcatv inline-ast->simple-ast title) + [(newline* 1)]))] (set! *state* (assoc-in *state* [:newline-after-block :current-block-is-first-heading-block?] false)) - simple-asts))) + simple-asts)) (declare block-list) + +(defn- list-continuation-indent + [current-level] + (indent-with-2-spaces (dec current-level))) + +(defn- src-in-list-item + [{:keys [lines language]} continuation-indent] + (concatv [(raw-text "```")] + (when language [(raw-text language)]) + [(newline* 1)] + (mapv raw-text lines) + [continuation-indent (raw-text "```") (newline* 1)])) + +(defn- quote-line + [line] + (let [line (string/trimr line)] + (if (string/blank? line) + ">" + (str "> " line)))) + +(defn- quote-in-list-item + [block-coll continuation-indent] + (let [lines (->> block-coll + (mapcatv block-ast->simple-ast) + simple-asts->string + string/split-lines) + lines (if (seq lines) lines [""])] + (mapcatv (fn [idx line] + (concatv (when (pos? idx) [continuation-indent]) + [(raw-text (quote-line line)) + (newline* 1)])) + (range) + lines))) + +(defn- block-level-content-in-list-item + [content continuation-indent] + (when (= 1 (count content)) + (let [[ast-type ast-content] (first content)] + (case ast-type + "Src" + (src-in-list-item ast-content continuation-indent) + + "Quote" + (quote-in-list-item ast-content continuation-indent) + + nil)))) + (defn- block-list-item [{:keys [content items number _name checkbox]}] - (let [content* (mapcatv block-ast->simple-ast content) - number* (raw-text + (let [number* (raw-text (if number (str number ". ") "* ")) @@ -59,6 +116,9 @@ current-level (get *state* :current-level 1) indent' (when (> current-level 1) (indent (dec current-level) 0)) + continuation-indent (list-continuation-indent current-level) + content* (or (block-level-content-in-list-item content continuation-indent) + (mapcatv block-ast->simple-ast content)) items* (block-list items :in-list? true)] (concatv [indent' number* checkbox* space] content* @@ -114,29 +174,59 @@ (mapv (fn [line] (string/replace-first line pattern "")) lines))) (defn- block-src - [{:keys [lines language]}] + [{:keys [lines language]} {:keys [heading-prefix]}] (let [level (dec (get *state* :current-level 1)) lines* (if (= "no-indent" (get-in *state* [:export-options :indent-style])) (remove-max-prefix-spaces lines) lines)] - (concatv - [(indent-with-2-spaces level) (raw-text "```")] - (when language [(raw-text language)]) - [(newline* 1)] - (mapv raw-text lines*) - [(indent-with-2-spaces level) (raw-text "```") (newline* 1)]))) + (if heading-prefix + (concatv + (block-heading-prefix heading-prefix) + [(raw-text "```")] + (when language [(raw-text language)]) + [(newline* 1)] + (mapv raw-text lines*) + [(heading-continuation-indent heading-prefix) (raw-text "```") (newline* 1)]) + (concatv + [(indent-with-2-spaces level) (raw-text "```")] + (when language [(raw-text language)]) + [(newline* 1)] + (mapv raw-text lines*) + [(indent-with-2-spaces level) (raw-text "```") (newline* 1)])))) + +(defn- quote-block-lines + [block-coll] + (let [lines (->> block-coll + (mapcatv block-ast->simple-ast) + simple-asts->string + string/split-lines)] + (if (seq lines) lines [""]))) + +(defn- quote-lines-with-prefix + [lines prefix continuation-indent] + (mapcatv (fn [idx line] + (concatv (if (zero? idx) prefix [continuation-indent]) + [(raw-text (quote-line line)) + (newline* 1)])) + (range) + lines)) (defn- block-quote - [block-coll] + [block-coll {:keys [heading-prefix]}] (let [level (dec (get *state* :current-level 1))] - (binding [*state* (assoc *state* :indent-after-break-line? true)] - (concatv (mapcatv (fn [block] - (let [block-simple-ast (block-ast->simple-ast block)] - (when (seq block-simple-ast) - (concatv [(indent-with-2-spaces level) (raw-text ">") space] - block-simple-ast)))) - block-coll) - [(newline* 2)])))) + (if heading-prefix + (binding [*state* (assoc *state* :indent-after-break-line? true)] + (quote-lines-with-prefix (quote-block-lines block-coll) + (block-heading-prefix heading-prefix) + (heading-continuation-indent heading-prefix))) + (binding [*state* (assoc *state* :indent-after-break-line? true)] + (concatv (mapcatv (fn [block] + (let [block-simple-ast (block-ast->simple-ast block)] + (when (seq block-simple-ast) + (concatv [(indent-with-2-spaces level) (raw-text ">") space] + block-simple-ast)))) + block-coll) + [(newline* 2)]))))) (declare inline-latex-fragment) (defn- block-latex-fragment @@ -369,9 +459,9 @@ "Example" (block-example ast-content) "Src" - (block-src ast-content) + (block-src ast-content (meta block)) "Quote" - (block-quote ast-content) + (block-quote ast-content (meta block)) "Latex_Fragment" (block-latex-fragment ast-content) "Latex_Environment" @@ -489,5 +579,19 @@ ast*** (if-not (empty? config-for-walk-block-ast) (mapv (partial cli-export-common/walk-block-ast config-for-walk-block-ast) ast**) ast**) - simple-asts (mapcatv block-ast->simple-ast ast***)] - (simple-asts->string simple-asts))))) \ No newline at end of file + ast**** (loop [remaining ast*** + result []] + (if-let [block (first remaining)] + (let [[ast-type ast-content] block + next-block (second remaining) + [next-ast-type] next-block] + (if (and (= "Heading" ast-type) + (empty? (:title ast-content)) + (contains? #{"Quote" "Src"} next-ast-type)) + (recur (nnext remaining) + (conj result (with-meta next-block (assoc (meta next-block) :heading-prefix ast-content)))) + (recur (rest remaining) + (conj result block)))) + result)) + simple-asts (mapcatv block-ast->simple-ast ast****)] + (simple-asts->string simple-asts))))) diff --git a/deps/cli/src/logseq/cli/common/file.cljs b/deps/cli/src/logseq/cli/common/file.cljs index 0df8bff5fb..6517708316 100644 --- a/deps/cli/src/logseq/cli/common/file.cljs +++ b/deps/cli/src/logseq/cli/common/file.cljs @@ -2,6 +2,7 @@ "Convert blocks to file content. Used for frontend exports and CLI" (:require [clojure.string :as string] [datascript.core :as d] + [datascript.impl.entity :as de] [logseq.common.util.date-time :as date-time-util] [logseq.db :as ldb] [logseq.db.frontend.content :as db-content] @@ -33,39 +34,58 @@ date-time-util/default-journal-title-formatter)))))) (defn- property-value->string - [property v context] - (cond - (and (map? v) (:db/id v)) - (str (db-property/property-value-content v)) + [db property v context] + (letfn [(entity-map [x] + (cond + (map? x) x + (de/entity? x) (into {} x))) + (entity-content [x] + (let [m (entity-map x)] + (or (:block/title m) + (:logseq.property/value m))))] + (cond + (some? (entity-content v)) + (str (entity-content v)) - (set? v) - (->> v - (sort-by (fn [item] - [(if (:block/order item) 0 1) - (str (or (:block/order item) - (property-value->string property item context)))])) - (map #(property-value->string property % context)) - (string/join ", ")) + (some? (:db/id (entity-map v))) + (let [entity (d/entity db (:db/id (entity-map v)))] + (str (or (entity-content entity) + (entity-content v) + ""))) - (sequential? v) - (->> v - (map #(property-value->string property % context)) - (string/join ", ")) + (set? v) + (->> v + (sort-by (fn [item] + [(if (:block/order item) 0 1) + (str (or (:block/order item) + (property-value->string db property item context)))])) + (map #(property-value->string db property % context)) + (string/join ", ")) - (keyword? v) - (name v) + (sequential? v) + (->> v + (map #(property-value->string db property % context)) + (string/join ", ")) - (and (= :datetime (:logseq.property/type property)) - (integer? v)) - (or (datetime-journal-title v context) - (str v)) + (keyword? v) + (name v) - (some? v) - (str v))) + (and (= :datetime (:logseq.property/type property)) + (integer? v)) + (or (datetime-journal-title v context) + (str v)) + + (some? v) + (str v)))) (defn- block-properties-content [db block spaces-tabs context] - (let [properties (->> (db-property/properties block) + (let [block (or (when-let [id (:db/id block)] + (d/entity db id)) + (when-let [block-uuid (:block/uuid block)] + (d/entity db [:block/uuid block-uuid])) + block) + properties (->> (db-property/properties block) (remove (fn [[k _]] (contains? db-property/db-attribute-properties k))) (remove (fn [[k _]] @@ -84,15 +104,92 @@ (:block/raw-title property) (name property-ident)) ":: " - (property-value->string property (get properties property-ident) context)))))) + (property-value->string db property (get properties property-ident) context)))))) (string/join "\n")))))) +(defn- property-value-block-content + [db b context] + (when-let [raw-block (d/entity db (:db/id b))] + (when-let [property (:logseq.property/created-from-property raw-block)] + (let [property-title (or (:block/title property) + (:block/raw-title property) + (some-> property :db/ident name)) + value (property-value->string db property + (or (:block/title raw-block) + (:logseq.property/value raw-block)) + context)] + (when property-title + (str property-title ":: " value)))))) + +(defn- block-title-content + [db b context] + (or (property-value-block-content db b context) + (db-content/recur-replace-uuid-in-block-title (d/entity db (:db/id b))))) + +(defn- bounded-heading-level + [heading level] + (cond + (integer? heading) + (-> heading (max 1) (min 6)) + + (true? heading) + (min (inc level) 6) + + :else + nil)) + +(defn- strip-heading-prefix + [content] + (-> (string/replace content #"^\s?#+\s+" "") + (string/replace #"^\s?#+\s?$" ""))) + +(defn- quote-content + [content] + (->> (or (seq (string/split-lines content)) [""]) + (map (fn [line] + (if (string/blank? line) + ">" + (str "> " line)))) + (string/join "\n"))) + +(defn- code-fence + [content] + (apply str (repeat (max 3 (inc (apply max 0 (map count (re-seq #"`+" content))))) "`"))) + +(defn- fenced-code-content + [content lang] + (let [fence (code-fence content)] + (str fence (when-not (string/blank? lang) lang) + "\n" content "\n" fence))) + +(defn- displayed-math-content + [content] + (str "$$\n" content "\n$$")) + +(defn- format-markdown-block-content + [b content level heading-to-list?] + (let [content (or content "")] + (case (:logseq.property.node/display-type b) + :quote + (quote-content content) + + :code + (fenced-code-content content (:logseq.property.code/lang b)) + + :math + (displayed-math-content content) + + (if-let [heading-level (and (not heading-to-list?) + (bounded-heading-level (:logseq.property/heading b) level))] + (str (apply str (repeat heading-level "#")) " " (strip-heading-prefix content)) + content)))) + (defn- transform-content [db b level {:keys [heading-to-list? include-properties?] :or {include-properties? true}} context] (let [heading (:logseq.property/heading b) ;; replace [[uuid]] with block's content - title (db-content/recur-replace-uuid-in-block-title (d/entity db (:db/id b))) + title (block-title-content db b context) content (or title "") level (if (and heading-to-list? heading) (if (> heading 1) @@ -105,9 +202,8 @@ prefix (str spaces-tabs "-") property-spaces-tabs (str spaces-tabs " ") content (if heading-to-list? - (-> (string/replace content #"^\s?#+\s+" "") - (string/replace #"^\s?#+\s?$" "")) - content) + (strip-heading-prefix content) + (format-markdown-block-content b content level heading-to-list?)) new-content (indented-block-content (string/trim content) property-spaces-tabs) sep (if (string/blank? new-content) "" @@ -125,7 +221,15 @@ (if (nil? f) (->> block-contents persistent! flatten (remove nil?)) (let [page? (nil? (:block/page f)) - content (if (and page? (not link)) nil (transform-content db f level opts context)) + content (cond + (and page? (not link) (:include-page-properties? opts)) + (block-properties-content db f "" context) + + (and page? (not link)) + nil + + :else + (transform-content db f level opts context)) new-content (if-let [children (seq (:block/children f))] (cons content (tree->file-content-aux db children {:init-level (inc level)} context)) diff --git a/deps/common/src/logseq/common/graph.cljs b/deps/common/src/logseq/common/graph.cljs index 1cd38ae736..49e47a9a19 100644 --- a/deps/common/src/logseq/common/graph.cljs +++ b/deps/common/src/logseq/common/graph.cljs @@ -65,6 +65,7 @@ Rules: - Contents in '/logseq/.recycle/' are ignored - Contents in '/logseq/bak/' are ignored - Contents in with '/logseq/version-files/' are ignored +- Contents in '/mirror/markdown/' are ignored " [dir path] (let [dir (path/path-normalize dir) @@ -73,7 +74,7 @@ Rules: (when (string? path) (or (some #(string/starts-with? rpath %) - ["." "logseq/.recycle" "logseq/bak" "logseq/version-files"]) + ["." "logseq/.recycle" "logseq/bak" "logseq/version-files" "mirror/markdown"]) (contains? #{"logseq/graphs-txid.edn" "logseq/pages-metadata.edn"} rpath) (some #(string/includes? rpath (str "/" % "/")) ["node_modules"]) diff --git a/deps/common/test/logseq/common/graph_test.cljs b/deps/common/test/logseq/common/graph_test.cljs index 48cf766ae9..32d2d10563 100644 --- a/deps/common/test/logseq/common/graph_test.cljs +++ b/deps/common/test/logseq/common/graph_test.cljs @@ -29,7 +29,9 @@ (fs/writeFileSync "tmp/test-graph/journals/2023_05_09.md" "") ;; Create files that are ignored (fs/mkdirSync (node-path/join "tmp/test-graph" "logseq" "bak")) + (fs/mkdirSync (node-path/join "tmp/test-graph" "mirror" "markdown" "pages") #js {:recursive true}) (fs/writeFileSync "tmp/test-graph/logseq/bak/baz.md" "") (fs/writeFileSync "tmp/test-graph/logseq/.gitignore" "") + (fs/writeFileSync "tmp/test-graph/mirror/markdown/pages/foo.md" "") (is (= ["tmp/test-graph/journals/2023_05_09.md" "tmp/test-graph/pages/foo.md"] - (common-graph/get-files "tmp/test-graph")))) \ No newline at end of file + (common-graph/get-files "tmp/test-graph")))) diff --git a/docs/adr/0016-markdown-mirror.md b/docs/adr/0016-markdown-mirror.md new file mode 100644 index 0000000000..608d2b9793 --- /dev/null +++ b/docs/adr/0016-markdown-mirror.md @@ -0,0 +1,283 @@ +# ADR 0016: Electron Markdown Mirror + +Date: 2026-05-05 +Status: Accepted + +## Context +Logseq DB graphs do not expose one editable Markdown file per page in the graph +directory. Some desktop workflows still need a Markdown representation that can +be read by external tools, backed up, indexed, or inspected outside Logseq. + +The mirror must not become another graph source of truth. Editing should remain +fast, and saving a block must not wait for Markdown rendering or filesystem +writes on the renderer main thread. + +The first supported runtime is the Electron desktop app. Browser and mobile +builds do not have the same graph-directory filesystem guarantees. + +## Decision +1. Add an Electron-only Settings toggle for Markdown Mirror. +2. Persist the toggle in Electron user settings as + `:feature/markdown-mirror?`. +3. When the setting is enabled for the Electron app, Logseq writes derived + Markdown files under the current graph directory: + - journals: + `mirror/markdown/journals/.md` + - other pages: + `mirror/markdown/pages/.md` +4. For a graph at `~/logseq/graphs/graph-xxx`, mirror files are written under: + - `~/logseq/graphs/graph-xxx/mirror/markdown/journals/` + - `~/logseq/graphs/graph-xxx/mirror/markdown/pages/` +5. Markdown Mirror is derived output. The DB remains the source of truth. +6. Files under `mirror/markdown/**` must be ignored by graph import, file + watchers, and graph parsing so the mirror never feeds back into the graph. +7. The feature is not available in browser or mobile builds, even if a stale + setting value exists. +8. Settings exposes an explicit "Regenerate full mirror" action that asks the + DB worker to rewrite the complete mirror for the current graph. +9. Built-in pages and property pages, including user-created properties, are not + exported to the mirror. User Tag/Class pages are normal user content and are + exported. + +## Runtime Ownership +1. The renderer owns only: + - the Settings row + - reading and updating `:feature/markdown-mirror?` + - pushing the enabled state to the DB worker when it changes +2. The DB worker owns: + - detecting affected page ids from successful local transactions + - rendering page Markdown from the worker DB snapshot + - scheduling and coalescing mirror jobs + - running explicit full-mirror regeneration jobs + - invoking platform filesystem writes +3. The Electron main process must not render Markdown. It may provide filesystem + primitives if needed, but content generation stays with the worker. +4. Editor save paths enqueue mirror work and return immediately. They must not + wait for rendering, directory creation, stat, write, rename, or delete. + +## Reusable Core and CLI Path +1. Markdown Mirror path planning, filename normalization, page rendering, write + deduplication, atomic writes, rename cleanup, and delete cleanup live in a + worker/core namespace that does not depend on Electron UI state. +2. The Electron app only owns feature activation through Settings. +3. The CLI should be able to reuse the same core by passing an explicit graph, + DB snapshot, and node filesystem platform context. +4. Future CLI support should not introduce a second Markdown serializer or a + different filename normalization policy. +5. Future CLI support should reuse the same mirror-path allocation index so the + Electron app and CLI do not produce different file names for the same graph. + +## Output Layout and Naming +1. Journal pages are written below `mirror/markdown/journals/`. +2. Non-journal pages are written below `mirror/markdown/pages/`. +3. Journal file names use the existing Logseq journal file-name rules for the + graph configuration. +4. Non-journal page file names use the normalized page title: + `.md`. +5. Page file names must stay friendly to external Markdown tools such as Emacs, + VS Code, and Obsidian. Do not include page uuid in normal mirror file names. +6. Page title is not page identity. The page uuid is still the internal mirror + identity, but it is stored in the mirror index rather than exposed in the + file name. +7. Duplicate non-journal page titles are handled by stable title suffix + allocation: + - first allocated page: `pages/Foo.md` + - second allocated page: `pages/Foo (2).md` + - third allocated page: `pages/Foo (3).md` +8. Once a page uuid is assigned a mirror path, keep that path stable until the + page is renamed or deleted. Do not renumber existing duplicate-title mirror + paths when another duplicate is created or removed. +9. The implementation keeps a per-graph mirror index under + `mirror/markdown/.index.edn`. +10. The mirror index stores at least: + - page uuid -> relative mirror path + - relative mirror path -> page uuid + - page uuid -> last known normalized title stem +11. The mirror index is implementation metadata for path stability. It is not + graph content and must be ignored by graph import and watchers along with the + rest of `mirror/markdown/**`. +12. All mirror file names pass through a single cross-platform filename + normalizer before joining paths. +13. Duplicate journal-day entities indicate invalid graph state for the mirror. + The implementation must fail those journal mirror jobs and surface a + diagnostic instead of choosing a winner. +14. If two entities still map to the same mirror path, the implementation must + fail the mirror job for that path and surface a diagnostic instead of + overwriting an unrelated page. +15. Page rename moves the mirror by writing the new path, updating the mirror + index, and deleting the old path after the new file has been written. +16. Page deletion deletes the corresponding mirror file and removes the page uuid + from the mirror index. +17. The write guard must reject any computed path outside the graph's + `mirror/markdown/` directory. +18. Built-in pages and property pages are excluded from path allocation and + mirror writes. User Tag/Class pages are not excluded by this rule. If a + previously mirrored page becomes excluded, the old mirror file is removed. + +## Duplicate Page Title Allocation +1. For non-journal pages, compute the normalized title stem first. +2. If the page uuid already exists in the mirror index and the normalized title + stem did not change, reuse the indexed path. +3. If the page uuid is new for that title, allocate the first unused path in this + sequence: + - `pages/.md` + - `pages/ (2).md` + - `pages/ (3).md` +4. A path is considered unavailable when the mirror index maps it to a different + live page uuid. +5. Deleted page paths become available for future allocation only after the + deleted page uuid is removed from the index. +6. Rename is treated as a new allocation for the new title stem. Existing pages + with the old title keep their already allocated paths. +7. If the mirror index is missing or unreadable, rebuild it from the current DB + in deterministic page order before writing. Deterministic order should use a + stable key such as page title plus page uuid. +8. The rebuilt index is allowed to choose paths for pages that had no previous + allocation. It must not overwrite a live existing path that is already mapped + to another page uuid. + +## Rename and Delete +1. Page rename moves the mirror by writing the new path and deleting the old + path after the new file has been written. +2. Page deletion deletes the corresponding mirror file. +3. The implementation keeps a small per-graph mirror index keyed by page uuid so + rename and delete handling does not require scanning the mirror directory on + every transaction. + +## Filename Normalization +1. Mirror file names must be portable across macOS, Windows, and Linux. +2. Use one shared normalizer for journal and page mirror file names. +3. The normalizer must: + - reject or replace path separators (`/`, `\`) + - reject or replace Windows-invalid characters (`<`, `>`, `:`, `"`, `|`, + `?`, `*`) and ASCII control characters + - reject or rewrite reserved Windows device names such as `CON`, `PRN`, + `AUX`, `NUL`, `COM1` through `COM9`, and `LPT1` through `LPT9` + - trim trailing spaces and dots because Windows does not preserve them + - reject empty names after normalization + - bound each file-name component to a safe byte length before appending + `.md` +4. Normalize Unicode to one canonical form before sanitizing so the same page + title produces the same mirror path across filesystems with different Unicode + normalization behavior. +5. The normalizer must be deterministic and must not depend on the current + operating system. A graph mirrored on macOS should choose the same logical + file name as the same graph mirrored on Windows. +6. If normalization changes the display title segment, the mirror index and + duplicate-title suffix allocation still preserve identity for non-journal + pages. +7. If a journal title normalizes to an unsafe or colliding file name, fail the + journal mirror job and surface diagnostics instead of inventing a fallback + name. +8. Path construction must join only validated path components. It must never + concatenate unchecked page titles into filesystem paths. + +## Scheduling and Performance +1. Mirror rendering is incremental. A transaction schedules only pages affected + by that transaction. +2. Jobs are coalesced by page uuid. If a page is edited repeatedly before its + job runs, only the latest worker DB state is rendered. +3. Scheduling uses a short debounce window per graph to reduce write churn while + preserving near-real-time output. +4. Mirror writes are serialized per graph to avoid path races during rename and + delete. +5. Before writing, compare the generated Markdown with the current file content + or with the last written content hash. Skip the write when content is + unchanged. +6. Write files atomically: + - ensure the target directory exists + - write to a temporary file in the same directory + - rename the temporary file over the target +7. Heavy work is forbidden on the renderer main thread: + - no full-graph export + - no Markdown rendering + - no filesystem reads or writes + - no synchronous IPC waiting for mirror completion +8. Full regeneration is an explicit Settings action. The renderer only sends a + worker request; page selection, rendering, and filesystem writes stay in the + DB worker. +9. Enabling the setting starts incremental mirroring for subsequent page edits. + It does not implicitly run full regeneration. + +## Markdown Content +1. Reuse the existing page-to-Markdown export pipeline used by worker export + APIs instead of introducing a separate renderer-side serializer. +2. The mirror output should match normal Markdown export semantics for page + content. +3. Mirror files do not include Logseq-internal mirror metadata in the Markdown + body. +4. Mirror files include block and page property drawers, including user + properties, with rendered property values. +5. Assets are referenced as normal exported Markdown references. This ADR does + not copy assets into `mirror/markdown/`. +6. Page references remain in Logseq wiki-link form, for example `[[Foo]]`. +7. Ambiguous page references caused by duplicate page titles are an accepted + limitation of Markdown Mirror. Do not rewrite page references to uuid-based + links or relative Markdown links in this ADR. + +## Failure Handling +1. Filesystem and path errors fail the mirror job for the affected page. +2. Failures are logged with graph, page uuid, target path, and error details. +3. Repeated failures should be visible from Settings or diagnostics; do not show + a toast on every keystroke. +4. The feature must not silently fall back to browser storage, OPFS, or another + output directory. +5. If the graph directory is not available, the worker rejects mirror jobs for + that graph until the graph is reopened with a valid directory. + +## Non-goals +1. Markdown Mirror is not bidirectional sync. +2. Editing files in `mirror/markdown/` does not update the graph. +3. The mirror is not a backup format with guaranteed import fidelity. +4. The mirror does not replace existing graph export features. +5. The mirror does not support browser or mobile runtimes in this ADR. + +## Consequences + +### Positive +- Desktop users get a readable Markdown projection inside the graph directory. +- Editor latency is protected because rendering and disk I/O are worker-owned + and asynchronous. +- The output layout is predictable for tools that watch journals and pages + separately. +- Page file names remain readable and practical in external Markdown tools. +- Ignoring `mirror/markdown/**` prevents mirror-generated files from becoming + graph input. + +### Tradeoffs +- The mirror can lag slightly behind the latest edit because writes are + debounced and serialized. +- A per-graph mirror index is needed for reliable rename and delete cleanup. +- Duplicate page references such as `[[Foo]]` remain ambiguous in mirror output. +- The first version does not backfill every existing page automatically when the + setting is enabled; users run full regeneration explicitly. +- External edits to mirror files are overwritten by later Logseq edits. +- Property pages are intentionally absent from the mirror, so the output is not + a complete DB export even though page and block property drawers are included. + +## Verification +Implementation should add focused coverage for: + +```bash +bb dev:test -v frontend.worker.markdown-mirror-test/enabled-electron-edit-writes-page-mirror-test +bb dev:test -v frontend.worker.markdown-mirror-test/enabled-electron-edit-writes-journal-mirror-test +bb dev:test -v frontend.worker.markdown-mirror-test/disabled-setting-does-not-write-mirror-test +bb dev:test -v frontend.worker.markdown-mirror-test/repeated-edits-coalesce-to-latest-content-test +bb dev:test -v frontend.worker.markdown-mirror-test/rename-removes-old-mirror-path-test +bb dev:test -v frontend.worker.markdown-mirror-test/delete-removes-mirror-file-test +bb dev:test -v frontend.worker.markdown-mirror-test/same-title-pages-write-distinct-stable-friendly-paths-test +bb dev:test -v frontend.worker.markdown-mirror-test/page-references-remain-wiki-links-test +bb dev:test -v frontend.worker.markdown-mirror-test/page-mirror-exports-property-values-test +bb dev:test -v frontend.worker.markdown-mirror-test/page-mirror-exports-page-property-values-test +bb dev:test -v frontend.worker.markdown-mirror-test/full-regeneration-writes-existing-non-built-in-non-property-pages-test +bb dev:test -v frontend.worker.markdown-mirror-test/invalid-filename-characters-are-normalized-test +bb dev:test -v frontend.worker.markdown-mirror-test/windows-reserved-filename-fails-with-diagnostic-test +bb dev:test -v frontend.worker.markdown-mirror-test/mirror-path-collision-fails-without-overwrite-test +``` + +Additional checks: +- `mirror/markdown/**` is excluded from graph parsing and file watchers. +- Editor save does not await mirror completion. +- Browser and mobile builds do not expose the setting and do not schedule mirror + jobs. +- Atomic write failures do not leave partial target files. diff --git a/src/electron/electron/handler.cljs b/src/electron/electron/handler.cljs index d24dc48dcd..ce638e7d2c 100644 --- a/src/electron/electron/handler.cljs +++ b/src/electron/electron/handler.cljs @@ -315,7 +315,8 @@ (do (cfgs/set-item! k v) (when (= k :spell-check) (spell-check/apply-window-spellcheck! window (spell-check/session-spellcheck-enabled? v))) - (state/set-state! [:config k] v)) + (state/set-state! [:config k] v) + nil) (cfgs/get-item k)) config))) diff --git a/src/main/frontend/components/settings.cljs b/src/main/frontend/components/settings.cljs index 798f565ad9..235a1b0398 100644 --- a/src/main/frontend/components/settings.cljs +++ b/src/main/frontend/components/settings.cljs @@ -745,6 +745,50 @@ {:left-label (t :settings.features/enable-flashcards) :action (flashcards-enabled-switcher enable-flashcards?)})) +(rum/defcs markdown-mirror-row < rum/reactive + (rum/local false ::regenerating?) + [state t] + (let [repo (state/get-current-repo) + enabled? (true? (:feature/markdown-mirror? (when repo (state/sub [:config repo])))) + *regenerating? (::regenerating? state) + regenerate! (fn [] + (when (and repo @state/*db-worker (not @*regenerating?)) + (reset! *regenerating? true) + (-> (state/ (state/ (state/ (state/ (p/let [_ (state/ (p/let [_ (state/encoded-graph-dir-name repo) "/mirror/markdown")) + +(def ^:private invalid-file-name-chars-re + #"[<>:\"|?*\\/]") + +(def ^:private ascii-control-re + #"[\x00-\x1F]") + +(def ^:private trailing-space-or-dot-re + #"[ \.]+$") + +(def ^:private reserved-windows-device-names + (into #{"CON" "PRN" "AUX" "NUL"} + (concat (map #(str "COM" %) (range 1 10)) + (map #(str "LPT" %) (range 1 10))))) + +(def ^:private max-file-stem-length 160) + +(defonce ^:private *repo->enabled? (atom {})) +(defonce ^:private *repo->queued-page-jobs (atom {})) +(defonce ^:private *repo->flush-timeout (atom {})) + +(defn- normalize-unicode + [s] + (let [s (str s)] + (if (fn? (.-normalize s)) + (.normalize s "NFC") + s))) + +(defn- reserved-windows-device-name? + [s] + (contains? reserved-windows-device-names + (string/upper-case s))) + +(defn normalize-file-stem + [s] + (when (some? s) + (let [s' (-> (normalize-unicode s) + (string/replace invalid-file-name-chars-re "_") + (string/replace ascii-control-re "_") + (string/replace trailing-space-or-dot-re "")) + s' (if (> (count s') max-file-stem-length) + (subs s' 0 max-file-stem-length) + s')] + (when (and (not (string/blank? s')) + (not (reserved-windows-device-name? s'))) + s')))) + +(defn- journal-file-stem + [journal-day] + (when journal-day + (let [s (str journal-day)] + (when (= 8 (count s)) + (str (subs s 0 4) "_" (subs s 4 6) "_" (subs s 6 8)))))) + +(defn page-relative-path + ([db page] + (page-relative-path db page {})) + ([db page {:keys [journal-file-stem-fn] + :or {journal-file-stem-fn journal-file-stem}}] + (when page + (if (ldb/journal? page) + (when-let [stem (normalize-file-stem (journal-file-stem-fn (:block/journal-day page)))] + (str "journals/" stem ".md")) + (when-let [stem (normalize-file-stem (:block/title page))] + (let [duplicate-pages (->> (d/datoms db :avet :block/title (:block/title page)) + (map #(d/entity db (:e %))) + (filter #(and (ldb/page? %) + (not (ldb/journal? %)))) + (sort-by (comp str :block/uuid))) + index (inc (or (first (keep-indexed + (fn [idx p] + (when (= (:block/uuid page) (:block/uuid p)) + idx)) + duplicate-pages)) + 0)) + stem' (if (= 1 index) + stem + (str stem " (" index ")"))] + (str "pages/" stem' ".md"))))))) + +(defn- mirror-path + [repo relative-path] + (str (repo-mirror-dir repo) "/" relative-path)) + +(defn- page-id-for-entity + [db eid] + (when-let [entity (d/entity db eid)] + (cond + (ldb/page? entity) (:db/id entity) + (:block/page entity) (:db/id (:block/page entity)) + (and (:block/parent entity) (ldb/page? (:block/parent entity))) (:db/id (:block/parent entity)) + (some-> entity :block/parent :block/page) (:db/id (:block/page (:block/parent entity)))))) + +(defn affected-page-ids + [{:keys [db-before db-after tx-data]}] + (->> tx-data + (mapcat (fn [{:keys [e a v]}] + (cond-> [(page-id-for-entity db-before e) + (page-id-for-entity db-after e)] + (= a :block/page) + (conj v)))) + (remove nil?) + set)) + +(defn set-enabled! + [repo enabled?] + (if enabled? + (swap! *repo->enabled? assoc repo true) + (do + (when-let [timeout-id (get @*repo->flush-timeout repo)] + (js/clearTimeout timeout-id)) + (swap! *repo->enabled? dissoc repo) + (swap! *repo->queued-page-jobs dissoc repo) + (swap! *repo->flush-timeout dissoc repo))) + nil) + +(defn enabled? + [repo] + (true? (get @*repo->enabled? repo))) + +(defn- storage + [platform*] + (:storage platform*)) + +(defn- (f path) + (p/catch (constantly nil))) + (p/rejected (ex-info "platform storage/read-text! missing" {:path path})))) + +(defn- content + db + (:block/uuid page) + {:include-page-properties? true} + {:export-bullet-indentation (or (:export-bullet-indentation options) " ") + :date-formatter (:date-formatter options)})) + +(defn- mirrorable-page? + [page] + (and (ldb/page? page) + (not (ldb/built-in? page)) + (not (ldb/property? page)) + (not (ldb/hidden? page)) + (not (:logseq.property.user/email page)))) + +(defn- mirrorable-pages + [db] + (->> (d/datoms db :avet :block/name) + (map #(d/entity db (:e %))) + (filter mirrorable-page?) + (sort-by (fn [page] + [(if (ldb/journal? page) 0 1) + (str (:block/journal-day page)) + (string/lower-case (or (:block/title page) "")) + (str (:block/uuid page))])))) + +(defn- queued-page-jobs update-in [repo (:page-id job)] merge-job job)) + +(defn- drain-repo-jobs! + [repo] + (let [jobs (vals (get @*repo->queued-page-jobs repo))] + (swap! *repo->queued-page-jobs dissoc repo) + jobs)) + +(declare flush-timeout repo) + (let [timeout-id (js/setTimeout + (fn [] + (swap! *repo->flush-timeout dissoc repo) + (-> (flush-timeout assoc repo timeout-id)))) + +(defn- (.unlink ^js (browser-pfs) (asset-path repo file-name)) - (p/catch (constantly nil)))) + (let [^js pfs (browser-pfs)] + (-> (.unlink pfs (asset-path repo file-name)) + (p/catch (constantly nil))))) + +(defn- unsupported-mirror-storage! + [& _args] + (throw (ex-info "Markdown mirror storage is not supported in browser workers" + {:platform :browser + :feature :markdown-mirror}))) (defn- websocket-connect [url] @@ -203,6 +210,9 @@ :remove-vfs! remove-vfs! :read-text! read-text! :write-text! write-text! + :write-text-atomic! unsupported-mirror-storage! + :delete-file! unsupported-mirror-storage! + :mirror-read-text! unsupported-mirror-storage! :asset-read-bytes! asset-read-bytes! :asset-write-bytes! asset-write-bytes! :asset-stat asset-stat diff --git a/src/main/frontend/worker/platform/node.cljs b/src/main/frontend/worker/platform/node.cljs index a92ba9adaf..61525b2cab 100644 --- a/src/main/frontend/worker/platform/node.cljs +++ b/src/main/frontend/worker/platform/node.cljs @@ -252,6 +252,26 @@ _ (ensure-dir! dir)] (fs/writeFile full-path text "utf8")))) +(defn- write-text-atomic! + [write-guard-fn data-dir path text] + (let [full-path (path-under-data-dir data-dir path) + dir (node-path/dirname full-path) + tmp-path (node-path/join dir (str "." (node-path/basename full-path) ".tmp-" (random-uuid)))] + (p/let [_ (when write-guard-fn + (write-guard-fn)) + _ (ensure-dir! dir) + _ (fs/writeFile tmp-path text "utf8") + _ (fs/rename tmp-path full-path)] + nil))) + +(defn- delete-file! + [write-guard-fn data-dir path] + (let [full-path (path-under-data-dir data-dir path)] + (p/let [_ (when write-guard-fn + (write-guard-fn))] + (-> (fs/rm full-path #js {:force true}) + (p/catch (constantly nil)))))) + (defn- asset-file-path [data-dir repo file-name] (node-path/join (repo-dir data-dir repo) @@ -433,6 +453,8 @@ :remove-vfs! (fn [pool] (remove-vfs! pool)) :read-text! (fn [path] (read-text! data-dir path)) :write-text! (fn [path text] (write-text! write-guard-fn data-dir path text)) + :write-text-atomic! (fn [path text] (write-text-atomic! write-guard-fn data-dir path text)) + :delete-file! (fn [path] (delete-file! write-guard-fn data-dir path)) :asset-read-bytes! (fn [repo file-name] (asset-read-bytes! data-dir repo file-name)) :asset-write-bytes! (fn [repo file-name payload] diff --git a/src/resources/dicts/en.edn b/src/resources/dicts/en.edn index 57b7695a49..4fe5350aa8 100644 --- a/src/resources/dicts/en.edn +++ b/src/resources/dicts/en.edn @@ -1630,6 +1630,11 @@ :settings.features/home-default-page-update-success "Home default page updated successfully!" :settings.features/journals-enable-success "Journals enabled" :settings.features/login-prompt "To access new features before anyone else you must be an Open Collective Sponsor or Backer of Logseq and therefore log in first." + :settings.features/markdown-mirror "Markdown Mirror" + :settings.features/markdown-mirror-desc "Write a derived Markdown copy of edited pages to the graph's mirror/markdown folder. Desktop only." + :settings.features/markdown-mirror-regenerate "Regenerate full mirror" + :settings.features/markdown-mirror-regenerate-error "Failed to regenerate Markdown Mirror: {1}" + :settings.features/markdown-mirror-regenerate-success "Markdown Mirror regenerated" :settings.features/page-not-found "The page \"{1}\" doesn't exist yet. Please create that page first, and then try again." :settings.features/plugin-system "Plugins" diff --git a/src/resources/dicts/zh-cn.edn b/src/resources/dicts/zh-cn.edn index fbb6886fd4..248911ba12 100644 --- a/src/resources/dicts/zh-cn.edn +++ b/src/resources/dicts/zh-cn.edn @@ -1620,6 +1620,11 @@ :settings.features/home-default-page-update-success "主页默认页面已更新成功!" :settings.features/journals-enable-success "已启用日志页" :settings.features/login-prompt "你必须是 Logseq 的 Open Collective Sponsor 或者 Backer 才能提前使用新功能(仍在测试中),因此需要登录。" + :settings.features/markdown-mirror "Markdown 镜像" + :settings.features/markdown-mirror-desc "将已编辑页面的派生 Markdown 副本写入图谱的 mirror/markdown 文件夹。仅桌面端可用。" + :settings.features/markdown-mirror-regenerate "重新生成完整镜像" + :settings.features/markdown-mirror-regenerate-error "重新生成 Markdown 镜像失败:{1}" + :settings.features/markdown-mirror-regenerate-success "Markdown 镜像已重新生成" :settings.features/page-not-found "页面“{1}”尚不存在。请先创建该页面,然后再试一次。" :settings.features/plugin-system "插件系统" diff --git a/src/test/frontend/handler/editor_async_test.cljs b/src/test/frontend/handler/editor_async_test.cljs index c727f7dfa9..af69075666 100644 --- a/src/test/frontend/handler/editor_async_test.cljs +++ b/src/test/frontend/handler/editor_async_test.cljs @@ -1,5 +1,5 @@ (ns frontend.handler.editor-async-test - (:require [clojure.test :refer [is testing async use-fixtures]] + (:require [cljs.test :refer [is testing async use-fixtures]] [datascript.core :as d] [frontend.db :as db] [frontend.handler.block :as block-handler] diff --git a/src/test/frontend/handler/export_test.cljs b/src/test/frontend/handler/export_test.cljs index df510b7208..807b8022c2 100644 --- a/src/test/frontend/handler/export_test.cljs +++ b/src/test/frontend/handler/export_test.cljs @@ -15,7 +15,11 @@ uuid-5 #uuid "708f7836-c1e2-4212-bd26-b53c7e9f1449" uuid-6 #uuid "de7724d5-b045-453d-a643-31b81d310071" uuid-p3 #uuid "de13830f-9691-4074-a0d6-cc8ab9cf9074" - uuid-7 #uuid "f81f4f64-578a-42ff-8741-19adac45f42a"] + uuid-7 #uuid "f81f4f64-578a-42ff-8741-19adac45f42a" + uuid-p5 #uuid "9dfeae55-c426-4957-8de9-40ff71c622f0" + uuid-8 #uuid "c370c72d-97b8-45f1-8a87-184e1a77792c" + uuid-9 #uuid "253c84fb-bf6f-4936-8370-4662930c8e6d" + uuid-10 #uuid "e6741341-2426-4c46-b09f-6aec73a4357b"] [{:page {:block/title "page1"} :blocks [{:block/title "1" @@ -57,7 +61,25 @@ [{:block/title "issue" :build/keep-uuid? true :block/uuid uuid-7 - :build/properties {:user.property/reproducible-steps "Switch to a password protected graph"}}]}])) + :build/properties {:user.property/reproducible-steps "Switch to a password protected graph"}}]} + {:page {:block/title "page5" + :block/uuid uuid-p5} + :blocks + [{:block/title "Heading block" + :build/keep-uuid? true + :block/uuid uuid-8 + :build/properties {:logseq.property/heading 2}} + {:block/title "quote line 1\nquote line 2" + :build/keep-uuid? true + :block/uuid uuid-9 + :build/tags [:logseq.class/Quote-block] + :build/properties {:logseq.property.node/display-type :quote}} + {:block/title "(println \"hi\")\n(+ 1 2)" + :build/keep-uuid? true + :block/uuid uuid-10 + :build/tags [:logseq.class/Code-block] + :build/properties {:logseq.property.node/display-type :code + :logseq.property.code/lang "clojure"}}]}])) (use-fixtures :once {:before (fn [] @@ -108,6 +130,21 @@ [(uuid "f81f4f64-578a-42ff-8741-19adac45f42a")] {:remove-options #{:property}}))))) +(deftest export-page-as-markdown-preserves-semantic-block-formatting + (is (= (string/trim " +- ## Heading block +- > quote line 1 + > quote line 2 +- ```clojure + (println \"hi\") + (+ 1 2) + ```") + (string/trim + (export-text/export-blocks-as-markdown + (state/get-current-repo) + [(uuid "9dfeae55-c426-4957-8de9-40ff71c622f0")] + {:remove-options #{:property}}))))) + (deftest export-blocks-as-markdown-levelFakeRemote repo + (fn [qkw & args] + (swap! worker-calls conj [qkw args]) + (p/resolved nil))))) + (set! remote/stop! (fn [_] (p/resolved true))) + (-> (p/let [_ (ensure-remote! "logseq_db_graph_a")] + (is (= [:thread-api/markdown-mirror-set-enabled + ["logseq_db_graph_a" true]] + (first (filter #(= :thread-api/markdown-mirror-set-enabled (first %)) + @worker-calls))))) + (p/catch (fn [e] + (is false (str "unexpected error: " e)))) + (p/finally (fn [] + (reset! state/state original-state) + (set! ipc/ipc original-ipc) + (set! remote/start! original-start!) + (set! remote/stop! original-stop!) + (done))))))) + +(deftest electron-ensure-remote-pushes-graph-markdown-mirror-setting-on-start-test + (async done + (let [worker-calls (atom []) + ensure-remote! #'persist-db/ original-state + (assoc :electron/user-cfgs {:feature/markdown-mirror? true}) + (assoc-in [:config ::state/global-config] {:feature/markdown-mirror? true}) + (assoc-in [:config "logseq_db_graph_a"] {}) + (assoc-in [:config "logseq_db_graph_b"] {:feature/markdown-mirror? true}))) + (set! ipc/ipc (fn [channel repo] + (is (= "db-worker-runtime" channel)) + (p/resolved {:base-url "http://127.0.0.1:9101" + :auth-token nil + :repo repo}))) + (set! remote/start! (fn [{:keys [repo]}] + (->FakeRemote repo + (fn [qkw & args] + (swap! worker-calls conj [qkw args]) + (p/resolved nil))))) + (set! remote/stop! (fn [_] (p/resolved true))) + (-> (p/let [_ (ensure-remote! "logseq_db_graph_a") + _ (ensure-remote! "logseq_db_graph_b")] + (is (= [[:thread-api/markdown-mirror-set-enabled + ["logseq_db_graph_a" false]] + [:thread-api/markdown-mirror-set-enabled + ["logseq_db_graph_b" true]]] + (filterv #(= :thread-api/markdown-mirror-set-enabled (first %)) + @worker-calls)))) + (p/catch (fn [e] + (is false (str "unexpected error: " e)))) + (p/finally (fn [] + (reset! state/state original-state) + (set! ipc/ipc original-ipc) + (set! remote/start! original-start!) + (set! remote/stop! original-stop!) + (done))))))) + +(deftest electron-ensure-remote-uses-graph-markdown-mirror-setting-before-sync-test + (async done + (let [ipc-calls (atom []) + worker-calls (atom []) + ensure-remote! #'persist-db/ state + (assoc :electron/user-cfgs nil) + (assoc-in [:config "logseq_db_graph_a" :feature/markdown-mirror?] true)))) + (set! util/electron? (constantly true)) + (set! ipc/ipc (fn [channel & args] + (swap! ipc-calls conj (into [channel] args)) + (case channel + "db-worker-runtime" + (p/resolved {:base-url "http://127.0.0.1:9101" + :auth-token nil + :repo (first args)}) + + (p/resolved nil)))) + (set! remote/start! (fn [{:keys [repo]}] + (->FakeRemote repo + (fn [qkw & args] + (swap! worker-calls conj [qkw args]) + (p/resolved nil))))) + (set! remote/stop! (fn [_] (p/resolved true))) + (-> (p/let [_ (ensure-remote! "logseq_db_graph_a")] + (is (= [["db-worker-runtime" "logseq_db_graph_a"]] + @ipc-calls)) + (is (= [:thread-api/markdown-mirror-set-enabled + ["logseq_db_graph_a" true]] + (first (filter #(= :thread-api/markdown-mirror-set-enabled (first %)) + @worker-calls))))) + (p/catch (fn [e] + (is false (str "unexpected error: " e)))) + (p/finally (fn [] + (reset! state/state original-state) + (set! util/electron? original-electron?) + (set! ipc/ipc original-ipc) + (set! remote/start! original-start!) + (set! remote/stop! original-stop!) + (done))))))) + +(deftest browser-fetch-initial-data-pushes-graph-markdown-mirror-setting-test + (async done + (let [worker-calls (atom []) + original-state @state/state + original-electron? util/electron? + original-invoke state/ original-state + (assoc :electron/user-cfgs {:feature/markdown-mirror? true}) + (assoc-in [:config ::state/global-config] {:feature/markdown-mirror? true}) + (assoc-in [:config "logseq_db_graph_a"] {}))) + (set! util/electron? (constantly true)) + (set! state/ (protocol/InBrowser) "logseq_db_graph_a" {}) + (p/then (fn [_] + (is (= [:thread-api/markdown-mirror-set-enabled + ["logseq_db_graph_a" false]] + (first (filter #(= :thread-api/markdown-mirror-set-enabled (first %)) + @worker-calls)))))) + (p/catch (fn [e] + (is false (str "unexpected error: " e)))) + (p/finally (fn [] + (reset! state/state original-state) + (set! util/electron? original-electron?) + (set! state/ (p/delay 0) + (p/then (fn [_] + (is (= [[:thread-api/markdown-mirror-set-enabled + [repo true]]] + @worker-calls)))) + (p/catch (fn [e] + (is false (str "unexpected error: " e)))) + (p/finally (fn [] + (remove-watch state/state :sync-markdown-mirror-setting) + (reset! state/state original-state) + (set! state/content :thread-api/validate-db :thread-api/recompute-checksum-diagnostics :thread-api/export-edn :thread-api/import-edn :thread-api/get-view-data :thread-api/get-class-objects :thread-api/get-property-values :thread-api/get-bidirectional-properties diff --git a/src/test/frontend/worker/db_listener_test.cljs b/src/test/frontend/worker/db_listener_test.cljs index 521f58f7d6..5c5523663a 100644 --- a/src/test/frontend/worker/db_listener_test.cljs +++ b/src/test/frontend/worker/db_listener_test.cljs @@ -1,6 +1,7 @@ (ns frontend.worker.db-listener-test (:require [cljs.test :refer [deftest is testing]] - [frontend.worker.db-listener :as db-listener])) + [frontend.worker.db-listener :as db-listener] + [frontend.worker.markdown-mirror :as markdown-mirror])) (deftest transit-safe-tx-meta-keeps-outliner-ops-test (testing "worker tx-meta sanitization should preserve semantic outliner ops" @@ -14,3 +15,16 @@ (is (= outliner-ops (:outliner-ops safe-tx-meta))) (is (= outliner-ops (:db-sync/inverse-outliner-ops safe-tx-meta))) (is (nil? (:error-handler safe-tx-meta)))))) + +(deftest markdown-mirror-listener-enqueues-worker-mirror-work-test + (let [calls (atom []) + tx-report {:tx-data [:tx]}] + (with-redefs [markdown-mirror/content :thread-api/validate-db :thread-api/recompute-checksum-diagnostics :thread-api/export-edn :thread-api/import-edn :thread-api/get-view-data :thread-api/get-class-objects :thread-api/get-property-values :thread-api/get-bidirectional-properties diff --git a/src/test/frontend/worker/markdown_mirror_test.cljs b/src/test/frontend/worker/markdown_mirror_test.cljs new file mode 100644 index 0000000000..49d8e7799f --- /dev/null +++ b/src/test/frontend/worker/markdown_mirror_test.cljs @@ -0,0 +1,528 @@ +(ns frontend.worker.markdown-mirror-test + (:require [cljs.test :refer [async deftest is testing]] + [datascript.core :as d] + [frontend.worker.db-listener :as db-listener] + [frontend.worker.markdown-mirror :as markdown-mirror] + [frontend.worker.platform :as worker-platform] + [logseq.db :as ldb] + [logseq.db.test.helper :as db-test] + [promesa.core :as p])) + +(def test-repo "logseq_db_graph-xxx") + +(defn- fake-platform + ([] (fake-platform {:runtime :node})) + ([env] + (let [files (atom {}) + writes (atom []) + deletes (atom [])] + {:platform {:env env + :storage {:read-text! (fn [path] + (p/resolved (get @files path))) + :write-text-atomic! (fn [path content] + (swap! writes conj [path content]) + (swap! files assoc path content) + (p/resolved nil)) + :delete-file! (fn [path] + (swap! deletes conj path) + (swap! files dissoc path) + (p/resolved nil))} + :broadcast {:post-message! (fn [& _] nil)}} + :files files + :writes writes + :deletes deletes}))) + +(defn- page-path [path] + (str (markdown-mirror/repo-mirror-dir test-repo) "/" path)) + +(defn- first-block [page] + (-> page :block/_page first)) + +(defn- F\"G|H")))) + + (testing "trailing spaces and dots are removed" + (is (= "title" + (markdown-mirror/normalize-file-stem "title. ")))) + + (testing "unicode is normalized before sanitizing" + (is (= (markdown-mirror/normalize-file-stem "e\u0301") + (markdown-mirror/normalize-file-stem "\u00e9")))) + + (testing "reserved Windows device names are rejected" + (is (nil? (markdown-mirror/normalize-file-stem "CON"))) + (is (nil? (markdown-mirror/normalize-file-stem "lpt9"))))) + +(deftest same-title-pages-write-distinct-stable-friendly-paths-test + (let [page-uuid-1 #uuid "11111111-1111-4111-8111-111111111111" + page-uuid-2 #uuid "22222222-2222-4222-8222-222222222222" + conn (db-test/create-conn-with-blocks + {:pages-and-blocks [{:page {:block/title "Same Name" + :block/uuid page-uuid-1} + :blocks [{:block/title "first"}]} + {:page {:block/title "Same Name" + :block/uuid page-uuid-2} + :blocks [{:block/title "second"}]}]}) + pages (->> (d/datoms @conn :avet :block/title "Same Name") + (map #(d/entity @conn (:e %))) + (filter #(nil? (:block/page %))) + (sort-by (comp str :block/uuid))) + paths (mapv #(markdown-mirror/page-relative-path @conn %) pages)] + (is (= ["pages/Same Name.md" + "pages/Same Name (2).md"] + paths)))) + +(deftest page-references-remain-wiki-links-test + (async done + (let [{:keys [platform files]} (fake-platform) + conn (db-test/create-conn-with-blocks + {:pages-and-blocks [{:page {:block/title "Source"} + :blocks [{:block/title "See [[Foo]]"}]} + {:page {:block/title "Foo"} + :blocks [{:block/title "target"}]} + {:page {:block/title "Foo"} + :blocks [{:block/title "duplicate"}]}]}) + page (db-test/find-page-by-title @conn "Source")] + (-> (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ ( (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (p/let [_ (markdown-mirror/ (p/let [_ (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (markdown-mirror/ (p/let [platform (platform-node/node-platform {:root-dir root-dir}) + storage (:storage platform) + path "graph-a/mirror/markdown/pages/page.md" + _ ((:write-text-atomic! storage) path "mirror") + content ((:read-text! storage) path) + _ ((:delete-file! storage) path) + deleted-content (-> ((:read-text! storage) path) + (p/catch (constantly nil)))] + (is (= "mirror" content)) + (is (nil? deleted-content)) + (is (empty? (filter #(string/includes? % ".tmp-") + (array-seq (fs/readdirSync + (node-path/join root-dir "graphs" "graph-a" "mirror" "markdown" "pages"))))))) + (p/catch (fn [e] + (is false (str "unexpected error: " e)))) + (p/finally done))))) + (deftest node-platform-cli-owner-bypasses-keychain-in-cli-e2e-test (async done (let [root-dir (node-helper/create-tmp-dir "platform-node-cli-secrets") diff --git a/src/test/frontend/worker/platform_test.cljs b/src/test/frontend/worker/platform_test.cljs index 6d121fe86b..bafdebeefd 100644 --- a/src/test/frontend/worker/platform_test.cljs +++ b/src/test/frontend/worker/platform_test.cljs @@ -1,6 +1,7 @@ (ns frontend.worker.platform-test (:require [cljs.test :refer [async deftest is]] [frontend.worker.platform :as platform] + [frontend.worker.platform.browser :as platform-browser] [promesa.core :as p])) (deftest kv-get-normalizes-undefined-to-nil-test @@ -26,3 +27,22 @@ (p/catch (fn [e] (is false (str e)))) (p/finally done)))) + +(deftest browser-platform-mirror-storage-is-unsupported-test + (let [original-location (.-location js/globalThis)] + (try + (set! (.-location js/globalThis) #js {:href "http://localhost/?electron=true" + :search "?electron=true"}) + (let [storage (:storage (platform-browser/browser-platform))] + (doseq [[f args] [[(:mirror-read-text! storage) ["mirror.md"]] + [(:write-text-atomic! storage) ["mirror.md" "content"]] + [(:delete-file! storage) ["mirror.md"]]]] + (try + (apply f args) + (is false "Expected browser mirror storage to throw") + (catch :default e + (is (= {:platform :browser + :feature :markdown-mirror} + (ex-data e))))))) + (finally + (set! (.-location js/globalThis) original-location)))))