diff --git a/deps/graph-parser/src/logseq/graph_parser.cljs b/deps/graph-parser/src/logseq/graph_parser.cljs index ec6a1d0525..e861707541 100644 --- a/deps/graph-parser/src/logseq/graph_parser.cljs +++ b/deps/graph-parser/src/logseq/graph_parser.cljs @@ -74,7 +74,8 @@ Options available: * :new? - Boolean which indicates if this file already exists. Default is true. * :delete-blocks-fn - Optional fn which is called with the new page, file and existing block uuids - which may be referenced elsewhere. + which may be referenced elsewhere. Used to delete the existing blocks before saving the new ones. + Implemented in file-common-handler/validate-and-get-blocks-to-delete for IoC * :skip-db-transact? - Boolean which skips transacting in order to batch transactions. Default is false * :extract-options - Options map to pass to extract/extract" ([conn file content] (parse-file conn file content {})) diff --git a/deps/graph-parser/src/logseq/graph_parser/block.cljs b/deps/graph-parser/src/logseq/graph_parser/block.cljs index efa776c330..530fd86575 100644 --- a/deps/graph-parser/src/logseq/graph_parser/block.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/block.cljs @@ -371,7 +371,7 @@ refs (distinct (concat (:refs block) ref-blocks))] (assoc block :refs refs))) -(defn- block-keywordize +(defn block-keywordize [block] (update-keys block @@ -381,6 +381,7 @@ (keyword "block" k))))) (defn- sanity-blocks-data + "Clean up blocks data and add `block` ns to all keys" [blocks] (map (fn [block] (if (map? block) @@ -396,7 +397,7 @@ [:block/name (gp-util/page-name-sanity-lc tag)])) tags)) block)) -(defn- get-block-content +(defn get-block-content [utf8-content block format meta block-pattern] (let [content (if-let [end-pos (:end_pos meta)] (utf8/substring utf8-content @@ -608,25 +609,38 @@ [block] (println "Logseq will assign a new id for this block: " block) (-> block - (assoc :uuid (d/squuid)) - (update :properties dissoc :id) - (update :properties-text-values dissoc :id) - (update :properties-order #(vec (remove #{:id} %))) - (update :content (fn [c] + (assoc :block/uuid (d/squuid)) + (update :block/properties dissoc :id) + (update :block/properties-text-values dissoc :id) + (update :block/properties-order #(vec (remove #{:id} %))) + (update :block/content (fn [c] (let [replace-str (re-pattern (str "\n*\\s*" - (if (= :markdown (:format block)) - (str "id" gp-property/colons " " (:uuid block)) - (str (gp-property/colons-org "id") " " (:uuid block)))))] + (if (= :markdown (:block/format block)) + (str "id" gp-property/colons " " (:block/uuid block)) + (str (gp-property/colons-org "id") " " (:block/uuid block)))))] (string/replace-first c replace-str "")))))) -(defn block-exists-in-another-page? +(defn block-exists-in-another-page? + "For sanity check only. + For renaming file externally, the file is actually deleted and transacted before-hand." [db block-uuid current-page-name] (when (and db current-page-name) (when-let [block-page-name (:block/name (:block/page (d/entity db [:block/uuid block-uuid])))] (not= current-page-name block-page-name)))) +(defn fix-block-id-if-duplicated! + "If the block exists in another page, we need to fix it + If the block exists in the current extraction process, we also need to fix it" + [db page-name *block-exists-in-extraction block] + (let [block (if (or (@*block-exists-in-extraction (:block/uuid block)) + (block-exists-in-another-page? db (:block/uuid block) page-name)) + (fix-duplicate-id block) + block)] + (swap! *block-exists-in-extraction conj (:block/uuid block)) + block)) + (defn extract-blocks "Extract headings from mldoc ast. Args: @@ -635,12 +649,10 @@ `with-id?`: If `with-id?` equals to true, all the referenced pages will have new db ids. `format`: content's format, it could be either :markdown or :org-mode. `options`: Options supported are :user-config, :block-pattern :supported-formats, - :extract-macros, :extracted-block-ids, :date-formatter, :page-name and :db" - [blocks content with-id? format {:keys [user-config db page-name extracted-block-ids] :as options}] + :extract-macros, :date-formatter, :page-name and :db" + [blocks content with-id? format {:keys [user-config] :as options}] {:pre [(seq blocks) (string? content) (boolean? with-id?) (contains? #{:markdown :org} format)]} (let [encoded-content (utf8/encode content) - *block-ids (or extracted-block-ids (atom #{})) - ;; TODO: nbb doesn't support `Atom` [blocks body pre-block-properties] (loop [headings [] blocks (reverse blocks) @@ -666,14 +678,8 @@ (heading-block? block) (let [block' (construct-block block properties timestamps body encoded-content format pos-meta with-id? options) - block'' (assoc block' :macros (extract-macros-from-ast (cons block body))) - block-uuid (:uuid block'') - fixed-block (if (or (@*block-ids block-uuid) - (block-exists-in-another-page? db block-uuid page-name)) - (fix-duplicate-id block'') - block'')] - (swap! *block-ids conj (:uuid fixed-block)) - (recur (conj headings fixed-block) (rest blocks) {} {} [])) + block'' (assoc block' :macros (extract-macros-from-ast (cons block body)))] + (recur (conj headings block'') (rest blocks) {} {} [])) :else (recur headings (rest blocks) timestamps properties (conj body block)))) diff --git a/deps/graph-parser/src/logseq/graph_parser/extract.cljc b/deps/graph-parser/src/logseq/graph_parser/extract.cljc index 3c9b850894..fa0998629b 100644 --- a/deps/graph-parser/src/logseq/graph_parser/extract.cljc +++ b/deps/graph-parser/src/logseq/graph_parser/extract.cljc @@ -127,14 +127,44 @@ (seq invalid-properties) (assoc :block/invalid-properties invalid-properties)))) +(defn- attach-block-ids-if-match + "If block-ids are provided and match the number of blocks, attach them to blocks + If block-ids are provided but don't match the number of blocks, WARN and ignore + If block-ids are not provided (nil), just ignore" + [block-ids blocks] + (or (when block-ids + (if (= (count block-ids) (count blocks)) + (mapv (fn [block-id block] + (if (some? block-id) + (assoc block :block/uuid (uuid block-id)) + block)) + block-ids blocks) + (log/error :gp-extract/attach-block-ids-not-match "attach-block-ids-if-match: block-ids provided, but doesn't match the number of blocks, ignoring"))) + blocks)) + ;; TODO: performance improvement (defn- extract-pages-and-blocks - [format ast properties file content {:keys [date-formatter db filename-format] :as options}] + "uri-encoded? - if is true, apply URL decode on the file path + options - + :extracted-block-ids - An atom that contains all block ids that have been extracted in the current page (not yet saved to db) + :resolve-uuid-fn - Optional fn which is called to resolve uuids of each block. Enables diff-merge + (2 ways diff) based uuid resolution upon external editing. + returns a list of the uuids, given the receiving ast, or nil if not able to resolve. + Implemented in file-common-handler/diff-merge-uuids for IoC + Called in gp-extract/extract as AST is being parsed and properties are extracted there" + [format ast properties file content {:keys [date-formatter db filename-format extracted-block-ids resolve-uuid-fn] + :or {extracted-block-ids (atom #{}) + resolve-uuid-fn (constantly nil)} + :as options}] (try (let [page (get-page-name file ast false filename-format) [page page-name _journal-day] (gp-block/convert-page-if-journal page date-formatter) options' (assoc options :page-name page-name) + ;; In case of diff-merge (2way) triggered, use the uuids to override the ones extracted from the AST + override-uuids (resolve-uuid-fn format ast content options') blocks (->> (gp-block/extract-blocks ast content false format options') + (attach-block-ids-if-match override-uuids) + (mapv #(gp-block/fix-block-id-if-duplicated! db page-name extracted-block-ids %)) (gp-block/with-parent-and-left {:block/name page-name}) (vec)) ref-pages (atom #{}) diff --git a/deps/graph-parser/test/logseq/graph_parser/block_test.cljs b/deps/graph-parser/test/logseq/graph_parser/block_test.cljs index 72c36349ba..aee85bb38a 100644 --- a/deps/graph-parser/test/logseq/graph_parser/block_test.cljs +++ b/deps/graph-parser/test/logseq/graph_parser/block_test.cljs @@ -19,11 +19,11 @@ (deftest test-fix-duplicate-id (are [x y] - (let [result (gp-block/fix-duplicate-id x)] - (and (:uuid result) - (not= (:uuid x) (:uuid result)) + (let [result (gp-block/fix-duplicate-id (gp-block/block-keywordize x))] + (and (:block/uuid result) + (not= (:uuid x) (:block/uuid result)) (= (select-keys result - [:properties :content :properties-text-values :properties-order]) y))) + [:block/properties :block/content :block/properties-text-values :block/properties-order]) (gp-block/block-keywordize y)))) {:properties {:id "63f199bc-c737-459f-983d-84acfcda14fe"}, :tags [], :format :markdown, :meta {:start_pos 51, :end_pos 101}, :macros [], :unordered true, :content "bar\nid:: 63f199bc-c737-459f-983d-84acfcda14fe", :properties-text-values {:id "63f199bc-c737-459f-983d-84acfcda14fe"}, :level 1, :uuid #uuid "63f199bc-c737-459f-983d-84acfcda14fe", :properties-order [:id]} {:properties {}, :content "bar", diff --git a/package.json b/package.json index a3f7bfa4f9..4e333ceccd 100644 --- a/package.json +++ b/package.json @@ -91,6 +91,7 @@ "@excalidraw/excalidraw": "0.12.0", "@hugotomazi/capacitor-navigation-bar": "^2.0.0", "@logseq/capacitor-file-sync": "0.0.22", + "@logseq/diff-merge": "^0.0.1", "@logseq/react-tweet-embed": "1.3.1-1", "@sentry/react": "^6.18.2", "@sentry/tracing": "^6.18.2", diff --git a/src/main/frontend/db/model.cljs b/src/main/frontend/db/model.cljs index fac9f46709..b29ded6fe0 100644 --- a/src/main/frontend/db/model.cljs +++ b/src/main/frontend/db/model.cljs @@ -225,6 +225,7 @@ db-utils/seq-flatten))) (defn set-file-last-modified-at! + "Refresh file timestamps to DB" [repo path last-modified-at] (when (and repo path last-modified-at) (when-let [conn (conn/get-db repo false)] @@ -459,6 +460,34 @@ independent of format as format specific heading characters are stripped" blocks-map (zipmap (map :db/id blocks) blocks)] (keep blocks-map sorted-ids))) +;; Diverged of get-sorted-page-block-ids +(defn get-sorted-page-block-ids-and-levels + "page-name: the page name, original name + return: a list with elements in: + :id - a list of block ids, sorted by :block/left + :level - the level of the block, 1 for root, 2 for children of root, etc." + [page-name] + {:pre [(string? page-name)]} + (let [sanitized-page (gp-util/page-name-sanity-lc page-name) + page-id (:db/id (db-utils/entity [:block/name sanitized-page])) + root (db-utils/entity page-id)] + (loop [result [] + children (sort-by-left (:block/_parent root) root) + ;; BFS log of walking depth + levels (repeat (count children) 1)] + (if (seq children) + (let [child (first children) + cur-level (first levels) + next-children (sort-by-left (:block/_parent child) child)] + (recur (conj result {:id (:db/id child) :level cur-level}) + (concat + next-children + (rest children)) + (concat + (repeat (count next-children) (inc cur-level)) + (rest levels)))) + result)))) + (defn has-children? ([block-id] (has-children? (conn/get-db) block-id)) diff --git a/src/main/frontend/fs/diff_merge.cljc b/src/main/frontend/fs/diff_merge.cljc new file mode 100644 index 0000000000..bb62f1f8d6 --- /dev/null +++ b/src/main/frontend/fs/diff_merge.cljc @@ -0,0 +1,93 @@ +(ns frontend.fs.diff-merge + ;; Disable clj linters since we don't support clj + #?(:clj {:clj-kondo/config {:linters {:unresolved-namespace {:level :off} + :unresolved-symbol {:level :off}}}}) + (:require #?(:org.babashka/nbb ["@logseq/diff-merge$default" :refer [Merger Differ visualizeAsHTML attach_uuids]] + :default ["@logseq/diff-merge" :refer [Differ Merger visualizeAsHTML attach_uuids]]) + [logseq.graph-parser.block :as gp-block] + [logseq.graph-parser.property :as gp-property] + [logseq.graph-parser.utf8 :as utf8] + [cljs-bean.core :as bean] + [frontend.db.utils :as db-utils] + [frontend.db.model :as db-model])) + +;; (defn diff-merge +;; "N-ways diff & merge +;; Accept: blocks +;; https://github.com/logseq/diff-merge/blob/44546f2427f20bd417b898c8ba7b7d10a9254774/lib/mldoc.ts#L17-L22 +;; https://github.com/logseq/diff-merge/blob/85ca7e9bf7740d3880ed97d535a4f782a963395d/lib/merge.ts#L40" +;; [base & branches] +;; () +;; (let [merger (Merger.)] +;; (.mergeBlocks merger (bean/->js base) (bean/->js branches)))) + +(defn diff + "2-ways diff + Accept: blocks + https://github.com/logseq/diff-merge/blob/44546f2427f20bd417b898c8ba7b7d10a9254774/lib/mldoc.ts#L17-L22" + [base income] + (let [differ (Differ.)] + (.diff_logseqMode differ (bean/->js base) (bean/->js income)))) + +;; (defonce getHTML visualizeAsHTML) + +(defonce attachUUID attach_uuids) + +(defn db->diff-blocks + "db: datascript db + page-name: string" + [page-name] + {:pre (string? page-name)} + (let [walked (db-model/get-sorted-page-block-ids-and-levels page-name) + blocks (db-utils/pull-many [:block/uuid :block/content :block/level] (map :id walked)) + levels (map :level walked) + blocks (map (fn [block level] + {:uuid (str (:block/uuid block)) ;; Force to be string + :body (:block/content block) + :level level}) + blocks levels)] + blocks)) + +;; TODO Junyi: merge back to gp-block/extract-blocks +;; From back to first to ensure end_pos is correct +(defn ast->diff-blocks + "Prepare the blocks for diff-merge + blocks: ast of blocks + content: corresponding raw content" + [blocks content format {:keys [user-config block-pattern]}] + {:pre [(string? content) (contains? #{:markdown :org} format)]} + (let [encoded-content (utf8/encode content)] + (loop [headings [] + blocks (reverse blocks) + properties {} + end-pos (.-length encoded-content)] + (if (seq blocks) + (let [[block pos-meta] (first blocks) + ;; fix start_pos + pos-meta (assoc pos-meta :end_pos end-pos)] + (cond + (gp-block/heading-block? block) + (let [content (gp-block/get-block-content encoded-content block format pos-meta block-pattern)] + (recur (conj headings {:body content + :level (:level (second block)) + :uuid (:id properties)}) + (rest blocks) {} (:start_pos pos-meta))) ;; The current block's start pos is the next block's end pos + + (gp-property/properties-ast? block) + (let [new-props (:properties (gp-block/extract-properties (second block) (assoc user-config :format format)))] + ;; sending the current end pos to next, as it's not finished yet + ;; supports multiple properties sub-block possible in future + (recur headings (rest blocks) (merge properties new-props) (:end_pos pos-meta))) + + :else + (recur headings (rest blocks) properties (:end_pos pos-meta)))) + (if (empty? properties) + (reverse headings) + (let [[block _] (first blocks) + pos-meta {:start_pos 0 :end_pos end-pos} + content (gp-block/get-block-content encoded-content block format pos-meta block-pattern) + uuid (:id properties)] + (cons {:body content + :level 1 + :uuid uuid} + (reverse headings)))))))) diff --git a/src/main/frontend/fs/watcher_handler.cljs b/src/main/frontend/fs/watcher_handler.cljs index f821ad53bf..a66c5a7083 100644 --- a/src/main/frontend/fs/watcher_handler.cljs +++ b/src/main/frontend/fs/watcher_handler.cljs @@ -23,6 +23,7 @@ ;; all IPC paths must be normalized! (via gp-util/path-normalize) (defn- set-missing-block-ids! + "For every referred block in the content, fix their block ids in files if missing." [content] (when (string? content) (doseq [block-id (block-ref/get-all-block-ref-ids content)] @@ -43,7 +44,8 @@ (p/catch #(js/console.error "❌ Bak Error: " path %)))) _ (file-handler/alter-file repo path content {:re-render-root? true - :from-disk? true})] + :from-disk? true + :fs/event :fs/local-file-change})] (set-missing-block-ids! content) (db/set-file-last-modified-at! repo path mtime))) diff --git a/src/main/frontend/handler/common/file.cljs b/src/main/frontend/handler/common/file.cljs index 53249eebc4..702d7444ee 100644 --- a/src/main/frontend/handler/common/file.cljs +++ b/src/main/frontend/handler/common/file.cljs @@ -6,10 +6,12 @@ [logseq.graph-parser :as graph-parser] [logseq.graph-parser.util :as gp-util] [logseq.graph-parser.config :as gp-config] + [frontend.fs.diff-merge :as diff-merge] [frontend.fs :as fs] [frontend.context.i18n :refer [t]] + [promesa.core :as p] [clojure.string :as string] - [promesa.core :as p])) + [cljs-bean.core :as bean])) (defn- page-exists-in-another-file "Conflict of files towards same page" @@ -20,12 +22,18 @@ current-file)))) (defn- validate-existing-file + "Handle the case when the file is already exists in db + Likely caused by renaming between caps and non-caps, then cause file system + bugs on some OS + e.g. on macOS, it doesn't fire the file change event when renaming between + caps and non-caps" [repo-url file-page file-path] (when-let [current-file (page-exists-in-another-file repo-url file-page file-path)] (when (not= file-path current-file) (cond - (= (string/lower-case current-file) - (string/lower-case file-path)) + ;; TODO: handle case sensitive file system + (= (gp-util/path-normalize (string/lower-case current-file)) + (gp-util/path-normalize (string/lower-case file-path))) ;; case renamed (when-let [file (db/pull [:file/path current-file])] (p/let [disk-content (fs/read-file "" current-file)] @@ -41,17 +49,53 @@ :clear? false}])))))) (defn- validate-and-get-blocks-to-delete + "An implementation for the delete-blocks-fn in graph-parser/parse-file" [repo-url db file-page file-path retain-uuid-blocks] (validate-existing-file repo-url file-page file-path) (graph-parser/get-blocks-to-delete db file-page file-path retain-uuid-blocks)) +(defn- diff-merge-uuids + "Infer new uuids from existing DB data and diff with the new AST + Return a list of uuids for the new blocks" + [format ast content {:keys [page-name] :as options}] + (let [base-diffblocks (diff-merge/db->diff-blocks page-name) + income-diffblocks (diff-merge/ast->diff-blocks ast content format options) + diff-ops (diff-merge/diff base-diffblocks income-diffblocks) + new-uuids (diff-merge/attachUUID diff-ops (map :uuid base-diffblocks))] + (bean/->clj new-uuids))) + +(defn- reset-file!-impl + "Parse file considering diff-merge with local or remote file + Decide how to treat the parsed file based on the file's triggering event + options - + :fs/reset-event - the event that triggered the file update + :fs/local-file-change - file changed on local disk + :fs/remote-file-change - file changed on remote" + [repo-url file content {:fs/keys [event] :as options}] + (let [db-conn (db/get-db repo-url false)] + (case event + ;; the file is already in db, so we can use the existing file's blocks + ;; to do the diff-merge + :fs/local-file-change + (graph-parser/parse-file db-conn file content (assoc-in options [:extract-options :resolve-uuid-fn] diff-merge-uuids)) + + ;; TODO Junyi: 3 ways to handle remote file change + ;; The file is on remote, so we should have + ;; 1. a "common ancestor" file locally + ;; the worst case is that the file is not in db, so we should use the + ;; empty file as the common ancestor + ;; 2. a "remote version" just fetched from remote + + ;; default to parse the file + (graph-parser/parse-file db-conn file content options)))) + (defn reset-file! "Main fn for updating a db with the results of a parsed file" ([repo-url file-path content] (reset-file! repo-url file-path content {})) - ([repo-url file-path content {:keys [verbose] :as options}] + ([repo-url file-path content {:keys [verbose extracted-block-ids] :as options}] (let [new? (nil? (db/entity [:file/path file-path])) - options (merge (dissoc options :verbose) + options (merge (dissoc options :verbose :extracted-block-ids) {:new? new? :delete-blocks-fn (partial validate-and-get-blocks-to-delete repo-url) ;; Options here should also be present in gp-cli/parse-graph @@ -60,7 +104,8 @@ :date-formatter (state/get-date-formatter) :block-pattern (config/get-block-pattern (gp-util/get-format file-path)) :supported-formats (gp-config/supported-formats) - :filename-format (state/get-filename-format repo-url) - :extracted-block-ids (:extracted-block-ids options)} + :filename-format (state/get-filename-format repo-url)} + ;; To avoid skipping the `:or` bounds for keyword destructuring + (when (some? extracted-block-ids) {:extracted-block-ids extracted-block-ids}) (when (some? verbose) {:verbose verbose}))})] - (:tx (graph-parser/parse-file (db/get-db repo-url false) file-path content options))))) + (:tx (reset-file!-impl repo-url file-path content options))))) diff --git a/src/main/frontend/handler/file.cljs b/src/main/frontend/handler/file.cljs index f476ebc0c4..1dbe0a27f7 100644 --- a/src/main/frontend/handler/file.cljs +++ b/src/main/frontend/handler/file.cljs @@ -143,6 +143,7 @@ "Write any in-DB file, e.g. repo config, page, whiteboard, etc." [repo path content {:keys [reset? re-render-root? from-disk? skip-compare? new-graph? verbose skip-db-transact? extracted-block-ids] + :fs/keys [event] :or {reset? true re-render-root? false from-disk? false @@ -156,7 +157,7 @@ (let [opts {:new-graph? new-graph? :from-disk? from-disk? :skip-db-transact? skip-db-transact? - :extracted-block-ids extracted-block-ids} + :fs/event event} result (if reset? (do (when-not skip-db-transact? @@ -167,6 +168,8 @@ opts))) (file-common-handler/reset-file! repo path content (merge opts + ;; To avoid skipping the `:or` bounds for keyword destructuring + (when (some? extracted-block-ids) {:extracted-block-ids extracted-block-ids}) (when (some? verbose) {:verbose verbose})))) (db/set-file-content! repo path content opts))] (-> (p/let [_ (when-not from-disk? diff --git a/src/main/frontend/handler/repo.cljs b/src/main/frontend/handler/repo.cljs index f69f19b583..b8b2b2f975 100644 --- a/src/main/frontend/handler/repo.cljs +++ b/src/main/frontend/handler/repo.cljs @@ -141,8 +141,9 @@ (merge {:new-graph? new-graph? :re-render-root? false :from-disk? true - :skip-db-transact? skip-db-transact? - :extracted-block-ids extracted-block-ids} + :skip-db-transact? skip-db-transact?} + ;; To avoid skipping the `:or` bounds for keyword destructuring + (when (some? extracted-block-ids) {:extracted-block-ids extracted-block-ids}) (when (some? verbose) {:verbose verbose})))) (state/set-parsing-state! (fn [m] (update m :finished inc))) diff --git a/src/test/frontend/fs/diff_merge_test.cljs b/src/test/frontend/fs/diff_merge_test.cljs new file mode 100644 index 0000000000..5af79bdbe0 --- /dev/null +++ b/src/test/frontend/fs/diff_merge_test.cljs @@ -0,0 +1,339 @@ +(ns frontend.fs.diff-merge-test + (:require [datascript.core :as d] + [cljs.test :refer [deftest are is]] + [logseq.db :as ldb] + [logseq.graph-parser :as graph-parser] + [frontend.fs.diff-merge :as fs-diff] + [frontend.handler.common.file :as file-common-handler] + [frontend.db.conn :as conn] + [logseq.graph-parser.mldoc :as gp-mldoc] + [cljs-bean.core :as bean])) + +(defn test-db->diff-blocks + "A hijacked version of db->diff-blocks for testing. + It overwrites the internal db getter with the test db connection." + [conn & args] + (with-redefs [conn/get-db (constantly @conn)] + (apply fs-diff/db->diff-blocks args))) + +(defn org-text->diffblocks + [text] + (-> (gp-mldoc/->edn text (gp-mldoc/default-config :org)) + (fs-diff/ast->diff-blocks text :org {:block-pattern "-"}))) + +(deftest org->ast->diff-blocks-test + (are [text diff-blocks] + (= (org-text->diffblocks text) + diff-blocks) + ":PROPERTIES: +:ID: 72289d9a-eb2f-427b-ad97-b605a4b8c59b +:END: +#+tItLe: Well parsed!" +[{:body ":PROPERTIES:\n:ID: 72289d9a-eb2f-427b-ad97-b605a4b8c59b\n:END:\n#+tItLe: Well parsed!" + :uuid "72289d9a-eb2f-427b-ad97-b605a4b8c59b" + :level 1}] + + "#+title: Howdy" + [{:body "#+title: Howdy" :uuid nil :level 1}] + + ":PROPERTIES: +:fiction: [[aldsjfklsda]] +:END:\n#+title: Howdy" + [{:body ":PROPERTIES:\n:fiction: [[aldsjfklsda]]\n:END:\n#+title: Howdy" + :uuid nil + :level 1}])) + +(deftest db<->ast-diff-blocks-test + (let [conn (ldb/start-conn) + text ":PROPERTIES: +:ID: 72289d9a-eb2f-427b-ad97-b605a4b8c59b +:END: +#+tItLe: Well parsed!"] + (graph-parser/parse-file conn "foo.org" text {}) + (is (= (test-db->diff-blocks conn "Well parsed!") + (org-text->diffblocks text))))) + +(defn text->diffblocks + [text] + (-> (gp-mldoc/->edn text (gp-mldoc/default-config :markdown)) + (fs-diff/ast->diff-blocks text :markdown {:block-pattern "-"}))) + +(deftest md->ast->diff-blocks-test + (are [text diff-blocks] + (= (text->diffblocks text) + diff-blocks) + "- a +\t- b +\t\t- c" + [{:body "a" :uuid nil :level 1} + {:body "b" :uuid nil :level 2} + {:body "c" :uuid nil :level 3}] + + "## hello +\t- world +\t\t- nice +\t\t\t- nice +\t\t\t- bingo +\t\t\t- world" + [{:body "## hello" :uuid nil :level 2} + {:body "world" :uuid nil :level 2} + {:body "nice" :uuid nil :level 3} + {:body "nice" :uuid nil :level 4} + {:body "bingo" :uuid nil :level 4} + {:body "world" :uuid nil :level 4}] + + "# a +## b +### c +#### d +### e +- f +\t- g +\t\t- h +\t- i +- j" + [{:body "# a" :uuid nil :level 1} + {:body "## b" :uuid nil :level 2} + {:body "### c" :uuid nil :level 3} + {:body "#### d" :uuid nil :level 4} + {:body "### e" :uuid nil :level 3} + {:body "f" :uuid nil :level 1} + {:body "g" :uuid nil :level 2} + {:body "h" :uuid nil :level 3} + {:body "i" :uuid nil :level 2} + {:body "j" :uuid nil :level 1}] + + "- a\n id:: 63e25526-3612-4fb1-8cf9-f66db1254a58 +\t- b +\t\t- c" +[{:body "a\n id:: 63e25526-3612-4fb1-8cf9-f66db1254a58" + :uuid "63e25526-3612-4fb1-8cf9-f66db1254a58" :level 1} + {:body "b" :uuid nil :level 2} + {:body "c" :uuid nil :level 3}])) + +(deftest diff-test + (are [text1 text2 diffs] + (= (bean/->clj (fs-diff/diff (text->diffblocks text1) + (text->diffblocks text2))) + diffs) + "## hello +\t- world +\t\t- nice +\t\t\t- nice +\t\t\t- bingo +\t\t\t- world" + "## Halooooo +\t- world +\t\t- nice +\t\t\t- nice +\t\t\t- bingo +\t\t\t- world" + [[[-1 {:body "## hello" + :level 2 + :uuid nil}] + [1 {:body "## Halooooo" + :level 2 + :uuid nil}]] + [[0 {:body "world" + :level 2 + :uuid nil}]] + [[0 {:body "nice" + :level 3 + :uuid nil}]] + [[0 {:body "nice" + :level 4 + :uuid nil}]] + [[0 {:body "bingo" + :level 4 + :uuid nil}]] + [[0 {:body "world" + :level 4 + :uuid nil}]]] + + "## hello +\t- world +\t id:: 63e25526-3612-4fb1-8cf9-abcd12354abc +\t\t- nice +\t\t\t- nice +\t\t\t- bingo +\t\t\t- world" +"## Halooooo +\t- world +\t\t- nice +\t\t\t- nice +\t\t\t- bingo +\t\t\t- world" +[[[-1 {:body "## hello" + :level 2 + :uuid nil}] + [1 {:body "## Halooooo" + :level 2 + :uuid nil}] + [1 {:body "world" + :level 2 + :uuid nil}]] + [[-1 {:body "world\n id:: 63e25526-3612-4fb1-8cf9-abcd12354abc" + :level 2 + :uuid "63e25526-3612-4fb1-8cf9-abcd12354abc"}]] + [[0 {:body "nice" + :level 3 + :uuid nil}]] + [[0 {:body "nice" + :level 4 + :uuid nil}]] + [[0 {:body "bingo" + :level 4 + :uuid nil}]] + [[0 {:body "world" + :level 4 + :uuid nil}]]] + +"" +"- abc def" +[[[1 {:body "abc def" + :level 1 + :uuid nil}]]])) + +(deftest db->diffblocks + (let [conn (ldb/start-conn)] + (graph-parser/parse-file conn + "foo.md" + (str "- abc + id:: 11451400-0000-0000-0000-000000000000\n" + "- def + id:: 63246324-6324-6324-6324-632463246324\n") + {}) + (graph-parser/parse-file conn + "bar.md" + (str "- ghi + id:: 11451411-1111-1111-1111-111111111111\n" + "\t- jkl +\t id:: 63241234-1234-1234-1234-123412341234\n") + {}) + (are [page-name diff-blocks] (= (test-db->diff-blocks conn page-name) + diff-blocks) + "foo" + [{:body "abc\nid:: 11451400-0000-0000-0000-000000000000" :uuid "11451400-0000-0000-0000-000000000000" :level 1} + {:body "def\nid:: 63246324-6324-6324-6324-632463246324" :uuid "63246324-6324-6324-6324-632463246324" :level 1}] + + "bar" + [{:body "ghi\nid:: 11451411-1111-1111-1111-111111111111" :uuid "11451411-1111-1111-1111-111111111111" :level 1} + {:body "jkl\nid:: 63241234-1234-1234-1234-123412341234" :uuid "63241234-1234-1234-1234-123412341234" :level 2}]) + + (are [page-name text new-uuids] (= (let [old-blks (test-db->diff-blocks conn page-name) + new-blks (text->diffblocks text) + diff-ops (fs-diff/diff old-blks new-blks)] + (bean/->clj (fs-diff/attachUUID diff-ops (bean/->js (map :uuid old-blks)) "NEW_ID"))) + new-uuids) + "foo" + "- abc +- def" + ["11451400-0000-0000-0000-000000000000" + "NEW_ID"] + + "bar" + "- ghi +\t- jkl" + ["11451411-1111-1111-1111-111111111111" + "NEW_ID"] + + "non exist page" + "- k\n\t- l" + ["NEW_ID" "NEW_ID"] + + "another non exist page" + ":PROPERTIES: +:ID: 72289d9a-eb2f-427b-ad97-b605a4b8c59b +:END: +#+tItLe: Well parsed!" + ["72289d9a-eb2f-427b-ad97-b605a4b8c59b"]))) + +(deftest ast->diff-blocks-test + (are [ast text diff-blocks] + (= (fs-diff/ast->diff-blocks ast text :org {:block-pattern "-"}) + diff-blocks) + [[["Properties" [["TiTlE" "Howdy" []]]] nil]] + "#+title: Howdy" + [{:body "#+title: Howdy", :level 1, :uuid nil}]) + + (are [ast text diff-blocks] + (= (fs-diff/ast->diff-blocks ast text :org {:block-pattern "-" :user-config {:property-pages/enabled? true}}) + diff-blocks) + [[["Property_Drawer" [["foo" "#bar" [["Tag" [["Plain" "bar"]]]]] ["baz" "#bing" [["Tag" [["Plain" "bing"]]]]]]] {:start_pos 0, :end_pos 22}]] + "foo:: #bar\nbaz:: #bing" + [{:body "foo:: #bar\nbaz:: #bing", :level 1, :uuid nil}])) + +(deftest ast-empty-diff-test + (are [ast text diff-ops] + (= (bean/->clj (->> (fs-diff/ast->diff-blocks ast text :org {:block-pattern "-" :user-config {:property-pages/enabled? true}}) + (fs-diff/diff []))) + diff-ops) + [[["Property_Drawer" [["foo" "#bar" [["Tag" [["Plain" "bar"]]]]] ["baz" "#bing" [["Tag" [["Plain" "bing"]]]]]]] {:start_pos 0, :end_pos 22}]] + "foo:: #bar\nbaz:: #bing" + [[[1 {:body "foo:: #bar\nbaz:: #bing", :level 1, :uuid nil}]]])) + +;; Ensure diff-merge-uuids follows the id:: in the content +(deftest diff-merge-uuid-extract-test + (let [conn (ldb/start-conn) + foo-content (str "- abc + id:: 11451400-0000-0000-0000-000000000000\n" + "- def + id:: 63246324-6324-6324-6324-632463246324\n") + bar-content (str "- ghi + id:: 11451411-1111-1111-1111-111111111111\n" + "\t- jkl +\t id:: 63241234-1234-1234-1234-123412341234\n") ] + (graph-parser/parse-file conn "foo.md" foo-content {}) + (graph-parser/parse-file conn "bar.md" bar-content {}) + (are [ast content page-name uuids] + (= (with-redefs [conn/get-db (constantly @conn)] + (#'file-common-handler/diff-merge-uuids :markdown ast content {:page-name page-name + :block-pattern "-"})) + uuids) + + (gp-mldoc/->edn (str foo-content "- newline\n") (gp-mldoc/default-config :markdown)) + (str foo-content "- newline\n") + "foo" + ["11451400-0000-0000-0000-000000000000" + "63246324-6324-6324-6324-632463246324" + nil] + + (gp-mldoc/->edn (str bar-content "- newline\n") (gp-mldoc/default-config :markdown)) + (str bar-content "- newline\n") + "bar" + ["11451411-1111-1111-1111-111111111111" + "63241234-1234-1234-1234-123412341234" + nil]))) + +;; Ensure diff-merge-uuids keeps the block uuids unchanged at best effort +(deftest diff-merge-uuid-persist-test + (let [conn (ldb/start-conn) + foo-content (str "- abc\n" + "- def\n") + bar-content (str "- ghi\n" + "\t- jkl\n")] + (graph-parser/parse-file conn "foo.md" foo-content {}) + (graph-parser/parse-file conn "bar.md" bar-content {}) + (are [ast content page-name uuids] + (= (with-redefs [conn/get-db (constantly @conn)] + (#'file-common-handler/diff-merge-uuids :markdown ast content {:page-name page-name + :block-pattern "-"})) + ;; Get all uuids under the page + (conj (->> page-name + (test-db->diff-blocks conn) + (map :uuid) + (vec)) nil)) + + (gp-mldoc/->edn (str foo-content "- newline\n") (gp-mldoc/default-config :markdown)) + (str foo-content "- newline\n") + "foo" + ["11451400-0000-0000-0000-000000000000" + "63246324-6324-6324-6324-632463246324" + nil] + + (gp-mldoc/->edn (str bar-content "- newline\n") (gp-mldoc/default-config :markdown)) + (str bar-content "- newline\n") + "bar" + ["11451411-1111-1111-1111-111111111111" + "63241234-1234-1234-1234-123412341234" + nil]))) diff --git a/yarn.lock b/yarn.lock index 32dc89511b..e7d739de46 100644 --- a/yarn.lock +++ b/yarn.lock @@ -492,6 +492,11 @@ resolved "https://registry.yarnpkg.com/@logseq/capacitor-file-sync/-/capacitor-file-sync-0.0.22.tgz#3fa94d40e5c44c70a12537ce17cf3089ff72f93b" integrity sha512-lb0+43YAaWy0umBCP2mPKyAPlIr2YHrLBfqGkCJUGAbrhTCAj37KZzb3snwSqeLA8dUSks9PcAN3jSgS74VMMw== +"@logseq/diff-merge@^0.0.1": + version "0.0.1" + resolved "https://registry.yarnpkg.com/@logseq/diff-merge/-/diff-merge-0.0.1.tgz#75d826a7e6fae96cd624faeea1310438de179ac7" + integrity sha512-g69EQOdWDD+zxxCVSTIzWmxCLAoPFZLNxiqPu1TMVsDNol4iJONcToNp2yPI9hgbrXXZ8ajivZJvlY5H7qrKZw== + "@logseq/react-tweet-embed@1.3.1-1": version "1.3.1-1" resolved "https://registry.yarnpkg.com/@logseq/react-tweet-embed/-/react-tweet-embed-1.3.1-1.tgz#119d22be8234de006fc35c3fa2a36f85634c5be6"