diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index 998a57dfca..153885f6e8 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -196,7 +196,6 @@ logseq.db.sqlite.export sqlite-export logseq.db.sqlite.util sqlite-util logseq.db.test.helper db-test - logseq.graph-parser graph-parser logseq.graph-parser.block gp-block logseq.graph-parser.mldoc gp-mldoc logseq.graph-parser.property gp-property diff --git a/deps/cli/src/logseq/cli/common/file.cljs b/deps/cli/src/logseq/cli/common/file.cljs index 055ad01566..6887d313dd 100644 --- a/deps/cli/src/logseq/cli/common/file.cljs +++ b/deps/cli/src/logseq/cli/common/file.cljs @@ -4,11 +4,8 @@ (:require [clojure.string :as string] [datascript.core :as d] [logseq.db :as ldb] - [logseq.db.common.entity-plus :as entity-plus] [logseq.db.frontend.content :as db-content] [logseq.db.sqlite.create-graph :as sqlite-create-graph] - [logseq.db.sqlite.util :as sqlite-util] - [logseq.graph-parser.property :as gp-property] [logseq.outliner.tree :as otree])) (defn- indented-block-content @@ -16,31 +13,11 @@ (let [lines (string/split-lines content)] (string/join (str "\n" spaces-tabs) lines))) -(defn- content-with-collapsed-state - "Only accept nake content (without any indentation)" - [repo format content collapsed?] - (cond - collapsed? - (gp-property/insert-property repo format content :collapsed true) - - ;; Don't check properties. Collapsed is an internal state log as property in file, but not counted into properties - (false? collapsed?) - (gp-property/remove-property format :collapsed content) - - :else - content)) - (defn- ^:large-vars/cleanup-todo transform-content - [repo db {:block/keys [collapsed? format pre-block? properties] :as b} level {:keys [heading-to-list?]} context {:keys [db-based?]}] - (let [title (or (:block/raw-title b) (:block/title b)) - block-ref-not-saved? (and (not db-based?) - (first (:block/_refs (d/entity db (:db/id b)))) - (not (string/includes? title (str (:block/uuid b))))) - heading (:heading properties) - title (if db-based? + [db {:block/keys [format pre-block? properties] :as b} level {:keys [heading-to-list?]} context] + (let [heading (:heading properties) ;; replace [[uuid]] with block's content - (db-content/recur-replace-uuid-in-block-title (d/entity db (:db/id b))) - title) + title (db-content/recur-replace-uuid-in-block-title (d/entity db (:db/id b))) content (or title "") content (cond pre-block? @@ -69,25 +46,21 @@ (-> (string/replace content #"^\s?#+\s+" "") (string/replace #"^\s?#+\s?$" "")) content) - content (if db-based? content (content-with-collapsed-state repo format content collapsed?)) new-content (indented-block-content (string/trim content) spaces-tabs) sep (if (string/blank? new-content) "" " ")] (str prefix sep new-content)))] - (if block-ref-not-saved? - (gp-property/insert-property repo format content :id (str (:block/uuid b))) - content))) + content)) (defn- tree->file-content-aux [repo db tree {:keys [init-level link] :as opts} context] - (let [db-based? (sqlite-util/db-based-graph? repo) - block-contents (transient [])] + (let [block-contents (transient [])] (loop [[f & r] tree level init-level] (if (nil? f) (->> block-contents persistent! flatten (remove nil?)) (let [page? (nil? (:block/page f)) - content (if (and page? (not link)) nil (transform-content repo db f level opts context {:db-based? db-based?})) + content (if (and page? (not link)) nil (transform-content db f level opts context)) new-content (if-let [children (seq (:block/children f))] (cons content (tree->file-content-aux repo db children {:init-level (inc level)} context)) @@ -104,9 +77,7 @@ (defn- update-block-content [db item eid] ;; This may not be needed if this becomes a file-graph only context - (if (entity-plus/db-based-graph? db) - (db-content/update-block-content db item eid) - item)) + (db-content/update-block-content db item eid)) (defn block->content "Converts a block including its children (recursively) to plain-text." @@ -126,11 +97,9 @@ (defn get-all-page->content "Exports a graph's pages as tuples of page name and page content" [repo db options] - (let [filter-fn (if (ldb/db-based-graph? db) - (fn [ent] - (or (not (:logseq.property/built-in? ent)) - (contains? sqlite-create-graph/built-in-pages-names (:block/title ent)))) - (constantly true))] + (let [filter-fn (fn [ent] + (or (not (:logseq.property/built-in? ent)) + (contains? sqlite-create-graph/built-in-pages-names (:block/title ent))))] (->> (d/datoms db :avet :block/name) (map #(d/entity db (:e %))) (filter filter-fn) diff --git a/deps/graph-parser/.carve/config.edn b/deps/graph-parser/.carve/config.edn index c5e6321ae9..a7525e329d 100644 --- a/deps/graph-parser/.carve/config.edn +++ b/deps/graph-parser/.carve/config.edn @@ -1,7 +1,4 @@ {:paths ["src"] - :api-namespaces [logseq.graph-parser.property - logseq.graph-parser.exporter - ;; Used in tests - logseq.graph-parser.test.docs-graph-helper + :api-namespaces [logseq.graph-parser.exporter logseq.graph-parser.schema.mldoc] :report {:format :ignore}} diff --git a/deps/graph-parser/.carve/ignore b/deps/graph-parser/.carve/ignore index 245628d4c5..84b5c73e18 100644 --- a/deps/graph-parser/.carve/ignore +++ b/deps/graph-parser/.carve/ignore @@ -1,9 +1,9 @@ ;; For CLI logseq.graph-parser.mldoc/ast-export-markdown ;; API -logseq.graph-parser.mldoc/link? +logseq.graph-parser.mldoc/block-with-title? ;; API -logseq.graph-parser/get-blocks-to-delete +logseq.graph-parser.mldoc/link? ;; API logseq.graph-parser.text/get-file-basename ;; API @@ -11,17 +11,9 @@ logseq.graph-parser.mldoc/mldoc-link? ;; public var logseq.graph-parser.schema.mldoc/block-ast-coll-schema ;; API -logseq.graph-parser/import-file-to-db-graph -;; API -logseq.graph-parser.block/extract-plain -;; API -logseq.graph-parser.block/extract-refs-from-text -;; API logseq.graph-parser.text/get-page-name logseq.graph-parser.text/get-namespace-last-part ;; API logseq.graph-parser.whiteboard/shape->block ;; API -logseq.graph-parser/parse-file -;; API -logseq.graph-parser/filter-files \ No newline at end of file +logseq.graph-parser.test.docs-graph-helper/clone-docs-repo-if-not-exists \ No newline at end of file diff --git a/deps/graph-parser/.clj-kondo/config.edn b/deps/graph-parser/.clj-kondo/config.edn index a48b63cfe2..480b5c2f6f 100644 --- a/deps/graph-parser/.clj-kondo/config.edn +++ b/deps/graph-parser/.clj-kondo/config.edn @@ -9,7 +9,6 @@ :consistent-alias {:aliases {clojure.string string datascript.core d - logseq.graph-parser graph-parser logseq.graph-parser.text text logseq.graph-parser.block gp-block logseq.graph-parser.mldoc gp-mldoc diff --git a/deps/graph-parser/README.md b/deps/graph-parser/README.md index f9f88b9e34..42e37cdbb2 100644 --- a/deps/graph-parser/README.md +++ b/deps/graph-parser/README.md @@ -1,18 +1,16 @@ ## Description -This library parses a logseq graph directory and returns it as a datascript -database connection. This library powers the Logseq app and also runs from the -commandline, _independent_ of the app. This is powerful as this can run anywhere -that a Node.js script has access to a Logseq graph e.g. on CI processes like -Github Actions. This library is compatible with ClojureScript and with +This library parses a file graph directory and returns it as a datascript +database connection. This library mainly exists to convert file graphs to DB graphs. +This library is compatible with ClojureScript and with [nbb-logseq](https://github.com/logseq/nbb-logseq) to respectively provide frontend and commandline functionality. ## API This library is under the parent namespace `logseq.graph-parser`. This library -provides two main namespaces for parsing, `logseq.graph-parser`. -`logseq.graph-parser/parse-file` is the main fn for the frontend. +provides two main namespaces, `logseq.graph-parser.exporter` and +`logseq.graph-parser.extract`. ## Usage diff --git a/deps/graph-parser/src/logseq/graph_parser.cljs b/deps/graph-parser/src/logseq/graph_parser.cljs deleted file mode 100644 index 4a92699eac..0000000000 --- a/deps/graph-parser/src/logseq/graph_parser.cljs +++ /dev/null @@ -1,131 +0,0 @@ -(ns logseq.graph-parser - "For file graphs, provides main ns to parse graph from source files. - Used by logseq app to parse graph and then save to the given database connection" - (:require [clojure.set :as set] - [clojure.string :as string] - [datascript.core :as d] - [logseq.common.config :as common-config] - [logseq.common.util :as common-util] - [logseq.db :as ldb] - [logseq.db.file-based.schema :as file-schema] - [logseq.graph-parser.extract :as extract])) - -(defn- retract-blocks-tx - [blocks retain-uuids] - (mapcat (fn [{uuid' :block/uuid eid :db/id}] - (if (and uuid' (contains? retain-uuids uuid')) - (map (fn [attr] [:db.fn/retractAttribute eid attr]) file-schema/retract-attributes) - (when eid [[:db.fn/retractEntity eid]]))) - blocks)) - -(defn- get-file-page - "Copy of file-model/get-file-page. Too basic to couple to main app" - [db file-path] - (ffirst - (d/q - '[:find ?page - :in $ ?path - :where - [?file :file/path ?path] - [?page :block/file ?file]] - db - file-path))) - -(defn get-blocks-to-delete - "Returns the transactional operations to retract blocks belonging to the - given page name and file path. This function is required when a file is being - parsed from disk; before saving the parsed, blocks from the previous version - of that file need to be retracted. - - The 'Page' parsed from the new file version is passed separately from the - file-path, as the page name can be set via properties in the file, and thus - can change between versions. If it has changed, existing blocks for both the - old and new page name will be retracted. - - Blocks are by default fully cleared via retractEntity. However, a collection - of block UUIDs to retain can be passed, and any blocks with matching uuids - will instead have their attributes cleared individually via - 'retractAttribute'. This will preserve block references to the retained - UUIDs." - [db file-page file-path retain-uuid-blocks] - (let [existing-file-page (get-file-page db file-path) - pages-to-clear (distinct (filter some? [existing-file-page (:db/id file-page)])) - blocks (mapcat (fn [page-id] - (:block/_page (d/entity db page-id))) - pages-to-clear) - retain-uuids (set (keep :block/uuid retain-uuid-blocks))] - (retract-blocks-tx (distinct blocks) retain-uuids))) - -(defn parse-file - "Parse file and save parsed data to the given db. Main parse fn used by logseq app. -Options available: - - * :delete-blocks-fn - Optional fn which is called with the new page, file and existing block uuids - which may be referenced elsewhere. Used to delete the existing blocks before saving the new ones. - Implemented in file-common-handler/validate-and-get-blocks-to-delete for IoC - * :extract-options - Options map to pass to extract/extract" - ([conn file-path content] (parse-file conn file-path content {})) - ([conn file-path content {:keys [delete-blocks-fn extract-options ctime mtime] - :or {delete-blocks-fn (constantly [])} - :as options}] - (let [format (common-util/get-format file-path) - file-content [{:file/path file-path}] - {:keys [tx ast]} - (let [extract-options' (merge {:block-pattern (common-config/get-block-pattern format) - :date-formatter "MMM do, yyyy" - :uri-encoded? false - :filename-format :legacy} - extract-options - {:db @conn}) - {:keys [pages blocks ast refs] - :or {pages [] - blocks [] - ast []}} - (cond (contains? common-config/mldoc-support-formats format) - (extract/extract file-path content extract-options') - - (common-config/whiteboard? file-path) - (extract/extract-whiteboard-edn file-path content extract-options') - - :else nil) - block-ids (map (fn [block] {:block/uuid (:block/uuid block)}) blocks) - delete-blocks (delete-blocks-fn (first pages) file-path block-ids) - block-refs-ids (->> (mapcat :block/refs blocks) - (filter (fn [ref] (and (vector? ref) - (= :block/uuid (first ref))))) - (map (fn [ref] {:block/uuid (second ref)})) - (seq)) - ;; To prevent "unique constraint" on datascript - block-ids (set/union (set block-ids) (set block-refs-ids)) - pages (extract/with-ref-pages pages blocks) - pages-index (map #(select-keys % [:block/name]) pages)] - ;; does order matter? - {:tx (concat file-content refs pages-index delete-blocks pages block-ids blocks) - :ast ast}) - file-entity (d/entity @conn [:file/path file-path]) - tx (concat tx [(cond-> {:file/path file-path - :file/content content} - (or ctime (nil? file-entity)) - (assoc :file/created-at (or ctime (js/Date.))) - mtime - (assoc :file/last-modified-at mtime))])] - (ldb/transact! conn tx (select-keys options [:new-graph? :from-disk?])) - {:tx tx - :ast ast}))) - -(defn filter-files - "Filters files in preparation for parsing. Only includes files that are - supported by parser" - [files] - (let [support-files (filter - (fn [file] - (let [format (common-util/get-format (:file/path file))] - (contains? (set/union #{:edn :css} common-config/mldoc-support-formats) format))) - files) - support-files (sort-by :file/path support-files) - {journals true non-journals false} (group-by (fn [file] (string/includes? (:file/path file) "journals/")) support-files) - {built-in true others false} (group-by (fn [file] - (or (string/includes? (:file/path file) "contents.") - (string/includes? (:file/path file) ".edn") - (string/includes? (:file/path file) "custom.css"))) non-journals)] - (concat (reverse journals) built-in others))) diff --git a/deps/graph-parser/src/logseq/graph_parser/block.cljs b/deps/graph-parser/src/logseq/graph_parser/block.cljs index 9ff74cdf87..1eb3f4d08e 100644 --- a/deps/graph-parser/src/logseq/graph_parser/block.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/block.cljs @@ -5,7 +5,6 @@ [clojure.string :as string] [clojure.walk :as walk] [datascript.core :as d] - [datascript.impl.entity :as de] [logseq.common.config :as common-config] [logseq.common.date :as common-date] [logseq.common.util :as common-util] @@ -866,61 +865,4 @@ [others parents' result'])))] (recur blocks parents result)))) result' (map (fn [block] (assoc block :block/order (db-order/gen-key))) result)] - (concat result' other-blocks))) - -(defn extract-plain - "Extract plain elements including page refs" - [repo content] - (let [ast (gp-mldoc/->edn repo content :markdown) - *result (atom [])] - (walk/prewalk - (fn [f] - (cond - ;; tag - (and (vector? f) - (= "Tag" (first f))) - nil - - ;; nested page ref - (and (vector? f) - (= "Nested_link" (first f))) - (swap! *result conj (:content (second f))) - - ;; page ref - (and (vector? f) - (= "Link" (first f)) - (map? (second f)) - (vector? (:url (second f))) - (= "Page_ref" (first (:url (second f))))) - (swap! *result conj - (:full_text (second f))) - - ;; plain - (and (vector? f) - (= "Plain" (first f))) - (swap! *result conj (second f)) - - :else - f)) - ast) - (-> (string/trim (apply str @*result)) - text/page-ref-un-brackets!))) - -(defn extract-refs-from-text - [repo db text date-formatter] - (when (string? text) - (let [ast-refs (gp-mldoc/get-references text (gp-mldoc/get-default-config repo :markdown)) - page-refs (map #(get-page-reference % :markdown) ast-refs) - block-refs (map get-block-reference ast-refs) - refs' (->> (concat page-refs block-refs) - (remove string/blank?) - distinct)] - (-> (map #(cond - (de/entity? %) - {:block/uuid (:block/uuid %)} - (common-util/uuid-string? %) - {:block/uuid (uuid %)} - :else - (page-name->map % db true date-formatter)) - refs') - set)))) + (concat result' other-blocks))) \ No newline at end of file diff --git a/deps/graph-parser/src/logseq/graph_parser/mldoc.cljc b/deps/graph-parser/src/logseq/graph_parser/mldoc.cljc index 5172d9af26..ee7c4cca5f 100644 --- a/deps/graph-parser/src/logseq/graph_parser/mldoc.cljc +++ b/deps/graph-parser/src/logseq/graph_parser/mldoc.cljc @@ -212,27 +212,9 @@ (or (contains? #{"Nested_link"} (first result')) (contains? #{"Page_ref" "Block_ref" "Complex"} (first (:url (second result'))))))))) -(defn properties? - [ast] - (contains? #{"Properties" "Property_Drawer"} (ffirst ast))) - (defn block-with-title? [type] (contains? #{"Paragraph" "Raw_Html" "Hiccup" - "Heading"} type)) - -(defn- has-title? - [repo content format] - (let [ast (->edn repo content format)] - (block-with-title? (ffirst (map first ast))))) - -(defn get-title&body - "parses content and returns [title body] - returns nil if no title" - [repo content format] - (let [lines (string/split-lines content)] - (if (has-title? repo content format) - [(first lines) (string/join "\n" (rest lines))] - [nil (string/join "\n" lines)]))) + "Heading"} type)) \ No newline at end of file diff --git a/deps/graph-parser/src/logseq/graph_parser/property.cljs b/deps/graph-parser/src/logseq/graph_parser/property.cljs index 629241b812..c43d997591 100644 --- a/deps/graph-parser/src/logseq/graph_parser/property.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/property.cljs @@ -4,9 +4,7 @@ [clojure.string :as string] [goog.string :as gstring] [goog.string.format] - [logseq.common.util :as common-util] - [logseq.common.util.page-ref :as page-ref] - [logseq.graph-parser.mldoc :as gp-mldoc])) + [logseq.common.util :as common-util])) (def colons "Property delimiter for markdown mode" "::") (defn colons-org @@ -14,13 +12,6 @@ [property] (str ":" property ":")) -(defn ->block-content - "Creates a block content string from properties map" - [properties] - (->> properties - (map #(str (name (key %)) (str colons " ") (val %))) - (string/join "\n"))) - (defn properties-ast? [block] (and @@ -40,11 +31,6 @@ ;; these properties are hidden from the user but a few like the editable ones ;; are visible for the user to edit. -(def built-in-extended-properties (atom #{})) -(defn register-built-in-properties - [props] - (reset! built-in-extended-properties (set/union @built-in-extended-properties props))) - (def editable-linkable-built-in-properties "Properties used by logseq that user can edit and that can have linkable property values" #{:alias :aliases :tags}) @@ -61,16 +47,14 @@ (defn hidden-built-in-properties "Properties used by logseq that user can't edit or see" [] - (set/union - #{:custom-id :background_color :created_at :last_modified_at ; backward compatibility only - :id :background-color :heading :collapsed - :created-at :updated-at :last-modified-at - :query-table :query-properties :query-sort-by :query-sort-desc :ls-type - :hl-type :hl-page :hl-stamp :hl-color :hl-value :logseq.macro-name :logseq.macro-arguments - :logseq.order-list-type :logseq.tldraw.page :logseq.tldraw.shape + #{:custom-id :background_color :created_at :last_modified_at ; backward compatibility only + :id :background-color :heading :collapsed + :created-at :updated-at :last-modified-at + :query-table :query-properties :query-sort-by :query-sort-desc :ls-type + :hl-type :hl-page :hl-stamp :hl-color :hl-value :logseq.macro-name :logseq.macro-arguments + :logseq.order-list-type :logseq.tldraw.page :logseq.tldraw.shape ; task markers - :todo :doing :now :later :done} - @built-in-extended-properties)) + :todo :doing :now :later :done}) (def built-in-property-types "Types for built-in properties. Built-in properties whose values are to be @@ -106,9 +90,6 @@ [] (set/difference (set/union (hidden-built-in-properties) (editable-built-in-properties)) - ;; Most of these need to be auto-parsed as integers so exclude - ;; them until we have ones that must be unparsed - @built-in-extended-properties ;; Refs need to be parsed editable-linkable-built-in-properties ;; All these should be parsed by gp-text/parse-non-string-property-value @@ -119,7 +100,7 @@ (defonce properties-end-pattern (re-pattern (gstring/format "%s[\t\r ]*\n|(%s\\s*$)" properties-end properties-end))) -(defn contains-properties? +(defn- contains-properties? [content] (when content (and (string/includes? content properties-start) @@ -150,143 +131,12 @@ content)) content)) -(defn- build-properties-str - [format properties] - (when (seq properties) - (let [org? (= format :org) - kv-format (if org? ":%s: %s" (str "%s" colons " %s")) - full-format (if org? ":PROPERTIES:\n%s\n:END:" "%s\n") - properties-content (->> (map (fn [[k v]] (gstring/format kv-format (name k) v)) properties) - (string/join "\n"))] - (gstring/format full-format properties-content)))) - (defn simplified-property? [line] (boolean (and (string? line) (re-find (re-pattern (str "^\\s?[^ ]+" colons)) line)))) -(defn- front-matter-property? - [line] - (boolean - (and (string? line) - (common-util/safe-re-find #"^\s*[^ ]+:" line)))) - -(defn- insert-property-not-org - [key* value lines {:keys [front-matter? has-properties? title?]}] - (let [exists? (atom false) - sym (if front-matter? ": " (str colons " ")) - new-property-s (str key* sym value) - property-f (if front-matter? front-matter-property? simplified-property?) - groups (partition-by property-f lines) - compose-lines (fn [] - (mapcat (fn [lines] - (if (property-f (first lines)) - (let [lines (doall - (mapv (fn [text] - (let [[k v] (common-util/split-first sym text)] - (if (and k v) - (let [key-exists? (= k key*) - _ (when key-exists? (reset! exists? true)) - v (if key-exists? value v)] - (str k sym (string/trim v))) - text))) - lines)) - lines (if @exists? lines (conj lines new-property-s))] - lines) - lines)) - groups)) - lines (cond - has-properties? - (compose-lines) - - title? - (cons (first lines) (cons new-property-s (rest lines))) - - :else - (cons new-property-s lines))] - (string/join "\n" lines))) - -(defn insert-property - "Only accept nake content (without any indentation)" - ([repo format content key value] - (insert-property repo format content key value false)) - ([repo format content key value front-matter?] - (when (string? content) - (let [ast (gp-mldoc/->edn repo content format) - title? (gp-mldoc/block-with-title? (ffirst (map first ast))) - has-properties? (or (and title? - (or (gp-mldoc/properties? (second ast)) - (gp-mldoc/properties? (second - (remove - (fn [[x _]] - (contains? #{"Hiccup" "Raw_Html"} (first x))) - ast))))) - (gp-mldoc/properties? (first ast))) - lines (string/split-lines content) - [title body] (gp-mldoc/get-title&body repo content format) - scheduled (filter #(string/starts-with? % "SCHEDULED") lines) - deadline (filter #(string/starts-with? % "DEADLINE") lines) - body-without-timestamps (filter - #(not (or (string/starts-with? % "SCHEDULED") - (string/starts-with? % "DEADLINE"))) - (string/split-lines body)) - org? (= :org format) - key (string/lower-case (name key)) - value (string/trim (str value)) - start-idx (.indexOf lines properties-start) - end-idx (.indexOf lines properties-end) - result (cond - (and org? (not has-properties?)) - (let [properties (build-properties-str format {key value})] - (if title - (string/join "\n" (concat [title] scheduled deadline [properties] body-without-timestamps)) - (str properties "\n" content))) - - (and has-properties? (>= start-idx 0) (> end-idx 0) (> end-idx start-idx)) - (let [exists? (atom false) - before (subvec lines 0 start-idx) - middle (doall - (->> (subvec lines (inc start-idx) end-idx) - (mapv (fn [text] - (let [[k v] (common-util/split-first ":" (subs text 1))] - (if (and k v) - (let [key-exists? (= k key) - _ (when key-exists? (reset! exists? true)) - v (if key-exists? value v)] - (str ":" k ": " (string/trim v))) - text)))))) - middle (if @exists? middle (conj middle (str ":" key ": " value))) - after (subvec lines (inc end-idx)) - lines (concat before [properties-start] middle [properties-end] after)] - (string/join "\n" lines)) - - (not org?) - (insert-property-not-org key value lines {:has-properties? has-properties? - :title? title? - :front-matter? front-matter?}) - - :else - content)] - (string/trimr result))))) - -(defn remove-property - ([format key content] - (remove-property format key content true)) - ([format key content first?] - (when (not (string/blank? (name key))) - (let [format (or format :markdown) - key (string/lower-case (name key)) - remove-f (if first? common-util/remove-first remove)] - (if (and (= format :org) (not (contains-properties? content))) - content - (let [lines (->> (string/split-lines content) - (remove-f (fn [line] - (let [s (string/triml (string/lower-case line))] - (or (string/starts-with? s (str ":" key ":")) - (string/starts-with? s (str key colons " ")))))))] - (string/join "\n" lines))))))) - (defn remove-properties [format content] (cond @@ -325,22 +175,4 @@ (string/join "\n" lines)) :else - content)) - -(defn insert-properties - [repo format content kvs] - (reduce - (fn [content [k v]] - (let [k (if (string? k) - (keyword (-> (string/lower-case k) - (string/replace " " "-"))) - k) - v (if (coll? v) - (some->> - (seq v) - (distinct) - (map (fn [item] (page-ref/->page-ref (page-ref/page-ref-un-brackets! item)))) - (string/join ", ")) - (if (keyword? v) (name v) v))] - (insert-property repo format content k v))) - content kvs)) + content)) \ No newline at end of file diff --git a/deps/graph-parser/src/logseq/graph_parser/test/docs_graph_helper.cljs b/deps/graph-parser/src/logseq/graph_parser/test/docs_graph_helper.cljs index 78af850cce..bd56aa84a5 100644 --- a/deps/graph-parser/src/logseq/graph_parser/test/docs_graph_helper.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/test/docs_graph_helper.cljs @@ -1,11 +1,7 @@ (ns logseq.graph-parser.test.docs-graph-helper "Helper fns for setting up and running tests against docs graph" (:require ["child_process" :as child-process] - ["fs" :as fs] - [cljs.test :refer [is testing]] - [clojure.string :as string] - [datascript.core :as d] - [logseq.common.config :as common-config])) + ["fs" :as fs])) ;; Helper fns for test setup ;; ========================= @@ -21,150 +17,4 @@ [dir branch] (when-not (.existsSync fs dir) (sh ["git" "clone" "--depth" "1" "-b" branch "-c" "advice.detachedHead=false" - "https://github.com/logseq/docs" dir] {}))) - -;; Fns for common test assertions -;; ============================== -(defn get-top-block-properties - [db] - (->> (d/q '[:find (pull ?b [*]) - :where - [?b :block/properties] - [(missing? $ ?b :block/name)]] - db) - (map first) - (map (fn [m] (zipmap (keys (:block/properties m)) (repeat 1)))) - (apply merge-with +) - (filter #(>= (val %) 5)) - (into {}))) - -(defn get-all-page-properties - [db] - (->> (d/q '[:find (pull ?b [*]) - :where - [?b :block/properties] - [?b :block/name]] - db) - (map first) - (map (fn [m] (zipmap (keys (:block/properties m)) (repeat 1)))) - (apply merge-with +) - (into {}))) - -(defn get-block-format-counts - [db] - (->> (d/q '[:find (pull ?b [*]) :where [?b :block/format]] db) - (map first) - (group-by :block/format) - (map (fn [[k v]] [k (count v)])) - (into {}))) - -(defn- get-journal-page-count [db] - (->> (d/q '[:find (count ?b) - :where - [?b :block/journal-day] - [?b :block/name] - [?b :block/file]] - db) - ffirst)) - -(defn- get-counts-for-common-attributes [db] - (->> [:block/scheduled :block/priority :block/deadline :block/collapsed? - :block/repeated?] - (map (fn [attr] - [attr - (ffirst (d/q [:find (list 'count '?b) :where ['?b attr]] - db))])) - (into {}))) - -(defn- query-assertions - [db graph-dir files] - (testing "Query based stats" - (is (= (->> files - ;; logseq files aren't saved under :block/file - (remove #(string/includes? % (str graph-dir "/" common-config/app-name "/"))) - ;; edn files being listed in docs by parse-graph aren't graph files - (remove #(and (not (common-config/whiteboard? %)) (string/ends-with? % ".edn"))) - set) - (->> (d/q '[:find (pull ?b [* {:block/file [:file/path]}]) - :where [?b :block/name] [?b :block/file]] - db) - (map (comp #(get-in % [:block/file :file/path]) first)) - set)) - "Files on disk should equal ones in db") - - (is (= (count (filter #(re-find #"journals/" %) files)) - (get-journal-page-count db)) - "Journal page count on disk equals count in db") - - (is (= {"CANCELED" 2 "DONE" 6 "LATER" 4 "NOW" 5 "WAIT" 1 "IN-PROGRESS" 1 "CANCELLED" 1 "TODO" 19} - (->> (d/q '[:find (pull ?b [*]) :where [?b :block/marker]] - db) - (map first) - (group-by :block/marker) - (map (fn [[k v]] [k (count v)])) - (into {}))) - "Task marker counts") - - (is (= {:markdown 7372 :org 500} (get-block-format-counts db)) - "Block format counts") - - (is (= {:rangeincludes 13, :description 137, :updated-at 46, :tags 5, :logseq.order-list-type 16, :query-table 8, - :logseq.macro-arguments 105, :parent 14, :logseq.tldraw.shape 79, :card-last-score 5, :card-repeats 5, - :name 16, :card-next-schedule 5, :ls-type 79, :card-last-interval 5, :type - 166, :template 5, :domainincludes 7, :title 114, :alias 62, :supports 6, :id - 146, :url 30, :card-ease-factor 5, :logseq.macro-name 105, :created-at 46, - :card-last-reviewed 5, :platforms 79, :initial-version 16, :heading 332} - (get-top-block-properties db)) - "Counts for top block properties") - - (is (= {:rangeincludes 13, :description 117, :tags 5, :unique 2, :meta 2, :parent 14, - :ls-type 1, :type 147, :source 1, :domainincludes 7, :sameas 4, :title 113, :author 1, - :alias 62, :logseq.tldraw.page 1, :supports 6, :url 30, :platforms 78, - :initial-version 15, :full-title 1} - (get-all-page-properties db)) - "Counts for all page properties") - - (is (= {:block/scheduled 2 - :block/priority 4 - :block/deadline 1 - :block/collapsed? 90 - :block/repeated? 1} - (get-counts-for-common-attributes db)) - "Counts for blocks with common block attributes") - - (let [no-name (->> (d/q '[:find (pull ?n [*]) :where [?b :block/namespace ?n]] db) - (filter (fn [x] - (when-not (:block/title (first x)) - x)))) - all-namespaces (->> (d/q '[:find (pull ?n [*]) :where [?b :block/namespace ?n]] db) - (map (comp :block/title first)) - set)] - (is (= #{"term" "setting" "book" "templates" "page" "Community" "Tweet" - "Whiteboard" "Whiteboard/Tool" "Whiteboard/Tool/Shape" "Whiteboard/Object" "Whiteboard/Action Bar"} - all-namespaces) - (str "Has correct namespaces: " no-name))) - - (is (empty? (->> (d/q '[:find ?n :where [?b :block/name ?n]] db) - (map first) - (filter #(string/includes? % "___")))) - "Block names don't have the slash/triple-lowbar delimiter"))) - -(defn docs-graph-assertions - "These are common assertions that should pass in both graph-parser and main - logseq app. It is important to run these in both contexts to ensure that the - functionality in frontend.handler.repo and logseq.graph-parser remain the - same" - [db graph-dir files] - ;; Counts assertions help check for no major regressions. These counts should - ;; only increase over time as the docs graph rarely has deletions - (testing "Counts" - (is (= 339 (count files)) "Correct file count") - (is (= 33 - (ffirst - (d/q '[:find (count ?b) - :where [?b :block/title ?content] - [(clojure.string/includes? ?content "+BEGIN_QUERY")]] - db))) - "Advanced query count")) - - (query-assertions db graph-dir files)) + "https://github.com/logseq/docs" dir] {}))) \ No newline at end of file diff --git a/deps/graph-parser/test/logseq/graph_parser/property_test.cljs b/deps/graph-parser/test/logseq/graph_parser/property_test.cljs index afdc7dbdae..9bd9bca0e3 100644 --- a/deps/graph-parser/test/logseq/graph_parser/property_test.cljs +++ b/deps/graph-parser/test/logseq/graph_parser/property_test.cljs @@ -27,86 +27,6 @@ "hello\n:PROPERTIES:\n:foo: bar\n:nice\n:END:\nnice" "hello\nfoo:: bar\n:nice\nnice")) -(deftest test-insert-property - (are [x y] (= x y) - (gp-property/insert-property test-db :org "hello" "a" "b") - "hello\n:PROPERTIES:\n:a: b\n:END:" - - (gp-property/insert-property test-db :org "hello" "a" false) - "hello\n:PROPERTIES:\n:a: false\n:END:" - - (gp-property/insert-property test-db :org "hello\n:PROPERTIES:\n:a: b\n:END:\n" "c" "d") - "hello\n:PROPERTIES:\n:a: b\n:c: d\n:END:" - - (gp-property/insert-property test-db :org "hello\n:PROPERTIES:\n:a: b\n:END:\nworld\n" "c" "d") - "hello\n:PROPERTIES:\n:a: b\n:c: d\n:END:\nworld" - - (gp-property/insert-property test-db :org "#+BEGIN_QUOTE - hello world - #+END_QUOTE" "c" "d") - ":PROPERTIES:\n:c: d\n:END:\n#+BEGIN_QUOTE\n hello world\n #+END_QUOTE" - - (gp-property/insert-property test-db :org "hello -DEADLINE: <2021-10-25 Mon> -SCHEDULED: <2021-10-25 Mon>" "a" "b") - "hello\nSCHEDULED: <2021-10-25 Mon>\nDEADLINE: <2021-10-25 Mon>\n:PROPERTIES:\n:a: b\n:END:" - - (gp-property/insert-property test-db :org "hello -DEADLINE: <2021-10-25 Mon> -SCHEDULED: <2021-10-25 Mon>\n:PROPERTIES:\n:a: b\n:END:\n" "c" "d") - "hello\nDEADLINE: <2021-10-25 Mon>\nSCHEDULED: <2021-10-25 Mon>\n:PROPERTIES:\n:a: b\n:c: d\n:END:" - - (gp-property/insert-property test-db :org "hello -DEADLINE: <2021-10-25 Mon> -SCHEDULED: <2021-10-25 Mon>\n:PROPERTIES:\n:a: b\n:END:\nworld\n" "c" "d") - "hello\nDEADLINE: <2021-10-25 Mon>\nSCHEDULED: <2021-10-25 Mon>\n:PROPERTIES:\n:a: b\n:c: d\n:END:\nworld" - - (gp-property/insert-property test-db :markdown "hello\na:: b\nworld\n" "c" "d") - "hello\na:: b\nc:: d\nworld" - - (gp-property/insert-property test-db :markdown "> quote" "c" "d") - "c:: d\n> quote" - - (gp-property/insert-property test-db :markdown "#+BEGIN_QUOTE - hello world - #+END_QUOTE" "c" "d") - "c:: d\n#+BEGIN_QUOTE\n hello world\n #+END_QUOTE")) - -(deftest test-insert-properties - (are [x y] (= x y) - (gp-property/insert-properties test-db :markdown "" {:foo "bar"}) - "foo:: bar" - - (gp-property/insert-properties test-db :markdown "" {"foo" "bar"}) - "foo:: bar" - - (gp-property/insert-properties test-db :markdown "" {"foo space" "bar"}) - "foo-space:: bar" - - (gp-property/insert-properties test-db :markdown "" {:foo #{"bar" "baz"}}) - "foo:: [[bar]], [[baz]]" - - (gp-property/insert-properties test-db :markdown "" {:foo ["bar" "bar" "baz"]}) - "foo:: [[bar]], [[baz]]" - - (gp-property/insert-properties test-db :markdown "a\nb\n" {:foo ["bar" "bar" "baz"]}) - "a\nfoo:: [[bar]], [[baz]]\nb" - - (gp-property/insert-properties test-db :markdown "" {:foo "\"bar, baz\""}) - "foo:: \"bar, baz\"" - - (gp-property/insert-properties test-db :markdown "abcd\nempty::" {:id "123" :foo "bar"}) - "abcd\nempty::\nid:: 123\nfoo:: bar" - - (gp-property/insert-properties test-db :markdown "abcd\nempty:: " {:id "123" :foo "bar"}) - "abcd\nempty:: \nid:: 123\nfoo:: bar" - - (gp-property/insert-properties test-db :markdown "abcd\nempty::" {:id "123"}) - "abcd\nempty::\nid:: 123" - - (gp-property/insert-properties test-db :markdown "abcd\nempty::\nanother-empty::" {:id "123"}) - "abcd\nempty::\nanother-empty::\nid:: 123")) - (deftest test-remove-properties (testing "properties with non-blank lines" (are [x y] (= x y) diff --git a/deps/outliner/src/logseq/outliner/core.cljs b/deps/outliner/src/logseq/outliner/core.cljs index 5992ff0da6..f4d6557022 100644 --- a/deps/outliner/src/logseq/outliner/core.cljs +++ b/deps/outliner/src/logseq/outliner/core.cljs @@ -5,7 +5,6 @@ [clojure.walk :as walk] [datascript.core :as d] [datascript.impl.entity :as de :refer [Entity]] - [logseq.common.config :as common-config] [logseq.common.util :as common-util] [logseq.common.util.page-ref :as page-ref] [logseq.common.uuid :as common-uuid] @@ -14,8 +13,6 @@ [logseq.db.frontend.class :as db-class] [logseq.db.frontend.schema :as db-schema] [logseq.db.sqlite.create-graph :as sqlite-create-graph] - [logseq.db.sqlite.util :as sqlite-util] - [logseq.graph-parser.block :as gp-block] [logseq.outliner.batch-tx :include-macros true :as batch-tx] [logseq.outliner.datascript :as ds] [logseq.outliner.pipeline :as outliner-pipeline] @@ -130,43 +127,9 @@ (declare move-blocks) -(defn- file-rebuild-block-refs - [repo db date-formatter {:block/keys [properties] :as block}] - (let [property-key-refs (->> (keys properties) - (keep (fn [property-id] - (:block/uuid (ldb/get-page db (name property-id)))))) - property-value-refs (->> (vals properties) - (mapcat (fn [v] - (cond - (and (coll? v) (uuid? (first v))) - v - - (uuid? v) - (when-let [_entity (d/entity db [:block/uuid v])] - [v]) - - (and (coll? v) (string? (first v))) - (mapcat #(gp-block/extract-refs-from-text repo db % date-formatter) v) - - (string? v) - (gp-block/extract-refs-from-text repo db v date-formatter) - - :else - nil)))) - property-refs (->> (concat property-key-refs property-value-refs) - (map (fn [id-or-map] (if (uuid? id-or-map) {:block/uuid id-or-map} id-or-map))) - (remove (fn [b] (nil? (d/entity db [:block/uuid (:block/uuid b)]))))) - content-refs (when-let [content (:block/title block)] - (let [format (or (:block/format block) :markdown) - content' (str (common-config/get-block-pattern format) " " content)] - (gp-block/extract-refs-from-text repo db content' date-formatter)))] - (concat property-refs content-refs))) - (defn ^:api rebuild-block-refs - [repo db date-formatter block] - (if (sqlite-util/db-based-graph? repo) - (outliner-pipeline/db-rebuild-block-refs db block) - (file-rebuild-block-refs repo db date-formatter block))) + [db block] + (outliner-pipeline/db-rebuild-block-refs db block)) (defn- fix-tag-ids "Fix or remove tags related when entered via `Escape`" diff --git a/scripts/src/logseq/tasks/dev/db_and_file_graphs.clj b/scripts/src/logseq/tasks/dev/db_and_file_graphs.clj index 96b534b21a..6fcc0d3ebf 100644 --- a/scripts/src/logseq/tasks/dev/db_and_file_graphs.clj +++ b/scripts/src/logseq/tasks/dev/db_and_file_graphs.clj @@ -78,7 +78,6 @@ ["deps/db/src/logseq/db/file_based" "deps/graph-parser/src/logseq/graph_parser/extract.cljc" "deps/graph-parser/src/logseq/graph_parser/property.cljs" - "deps/graph-parser/src/logseq/graph_parser.cljs" "src/main/frontend/fs" "src/main/frontend/util/fs.cljs"]) diff --git a/src/main/frontend/worker/db_worker.cljs b/src/main/frontend/worker/db_worker.cljs index 84c5560dea..37c8889977 100644 --- a/src/main/frontend/worker/db_worker.cljs +++ b/src/main/frontend/worker/db_worker.cljs @@ -246,9 +246,7 @@ (when-not @*publishing? (common-sqlite/create-kvs-table! client-ops-db)) (rtc-debug-log/create-tables! debug-log-db) (search/create-tables-and-triggers! search-db) - (ldb/register-transact-pipeline-fn! - (fn [tx-report] - (worker-pipeline/transact-pipeline repo tx-report))) + (ldb/register-transact-pipeline-fn! worker-pipeline/transact-pipeline) (let [schema (ldb/get-schema repo) conn (common-sqlite/get-storage-conn storage schema) _ (db-fix/check-and-fix-schema! repo conn) diff --git a/src/main/frontend/worker/pipeline.cljs b/src/main/frontend/worker/pipeline.cljs index f1d722802d..5b5790ba53 100644 --- a/src/main/frontend/worker/pipeline.cljs +++ b/src/main/frontend/worker/pipeline.cljs @@ -35,14 +35,13 @@ (:undo? tx-meta) (:redo? tx-meta))))) (defn- rebuild-block-refs - [repo {:keys [tx-meta db-after db-before]} blocks] + [{:keys [tx-meta db-after db-before]} blocks] (when (or (and (:outliner-op tx-meta) (refs-need-recalculated? tx-meta)) (:rtc-tx? tx-meta) (:rtc-op? tx-meta)) (mapcat (fn [block] (when (d/entity db-after (:db/id block)) - (let [date-formatter (worker-state/get-date-formatter repo) - refs (->> (outliner-core/rebuild-block-refs repo db-after date-formatter block) set) + (let [refs (->> (outliner-core/rebuild-block-refs db-after block) set) old-refs (->> (:block/refs (d/entity db-before (:db/id block))) (map :db/id) set) @@ -464,7 +463,7 @@ (defn transact-pipeline "Compute extra tx-data and block/refs, should ensure it's a pure function and doesn't call `d/transact!` or `ldb/transact!`." - [repo {:keys [db-after tx-meta] :as tx-report}] + [{:keys [db-after tx-meta] :as tx-report}] (let [extra-tx-data (compute-extra-tx-data tx-report) tx-report* (if (seq extra-tx-data) (let [result (d/with db-after extra-tx-data)] @@ -477,7 +476,7 @@ deleted-block-ids (set (map :db/id deleted-blocks)) blocks' (remove (fn [b] (deleted-block-ids (:db/id b))) blocks) block-refs (when (seq blocks') - (rebuild-block-refs repo tx-report* blocks')) + (rebuild-block-refs tx-report* blocks')) tx-id-data (let [db-after (:db-after tx-report*) updated-blocks (remove (fn [b] (contains? deleted-block-ids (:db/id b))) (concat pages blocks)) diff --git a/src/test/frontend/format/mldoc_test.cljs b/src/test/frontend/format/mldoc_test.cljs deleted file mode 100644 index 46d41dc631..0000000000 --- a/src/test/frontend/format/mldoc_test.cljs +++ /dev/null @@ -1,37 +0,0 @@ -(ns frontend.format.mldoc-test - (:require [logseq.graph-parser.block :as gp-block] - [cljs.test :refer [deftest testing are]] - [frontend.test.helper :as test-helper])) - -(deftest test-extract-plain - (testing "normalize date values" - (are [x y] (= (gp-block/extract-plain test-helper/test-db x) y) - "foo #book #[[nice test]]" - "foo" - - "foo #book #[[nice test]]" - "foo" - - "**foo** #book #[[nice test]]" - "foo" - - "foo [[bar]] #book #[[nice test]]" - "foo [[bar]]" - - "foo [[bar]] #book #[[nice test]]" - "foo [[bar]]" - - "[[foo bar]]" - "foo bar" - - "[[Foo Bar]]" - "Foo Bar" - - "[[Foo [[Bar]]]]" - "Foo [[Bar]]" - - "foo [[Foo [[Bar]]]]" - "foo [[Foo [[Bar]]]]" - - "foo [[Foo [[Bar]]]] #tag" - "foo [[Foo [[Bar]]]]"))) diff --git a/src/test/frontend/test/helper.cljs b/src/test/frontend/test/helper.cljs index 3511238f34..cdc396823b 100644 --- a/src/test/frontend/test/helper.cljs +++ b/src/test/frontend/test/helper.cljs @@ -35,9 +35,7 @@ [& {:keys [build-init-data?] :or {build-init-data? true} :as opts}] (state/set-current-repo! test-db) (conn/start! test-db opts) - (ldb/register-transact-pipeline-fn! - (fn [tx-report] - (worker-pipeline/transact-pipeline test-db tx-report))) + (ldb/register-transact-pipeline-fn! worker-pipeline/transact-pipeline) (let [conn (conn/get-db test-db false)] (when build-init-data? (d/transact! conn (sqlite-create-graph/build-db-initial-data config/config-default-content))) (d/listen! conn ::listen-db-changes! diff --git a/src/test/frontend/worker/pipeline_test.cljs b/src/test/frontend/worker/pipeline_test.cljs index 715e9df4fc..72c00959e4 100644 --- a/src/test/frontend/worker/pipeline_test.cljs +++ b/src/test/frontend/worker/pipeline_test.cljs @@ -1,7 +1,6 @@ (ns frontend.worker.pipeline-test (:require [cljs.test :refer [deftest is testing]] [datascript.core :as d] - [frontend.test.helper :as test-helper] [frontend.worker.pipeline :as worker-pipeline] [logseq.db :as ldb] [logseq.db.test.helper :as db-test])) @@ -42,8 +41,7 @@ (set (#'worker-pipeline/remove-conflict-datoms datoms))))))) (deftest test-built-in-page-updates-that-should-be-reverted - (let [graph test-helper/test-db - conn (db-test/create-conn-with-blocks + (let [conn (db-test/create-conn-with-blocks [{:page {:block/title "page1"} :blocks [{:block/title "b1"} {:block/title "b2" :build/tags [:tag1]}]}]) @@ -51,7 +49,7 @@ (ldb/register-transact-pipeline-fn! (fn [tx-report] - (worker-pipeline/transact-pipeline graph tx-report))) + (worker-pipeline/transact-pipeline tx-report))) (testing "Using built-in pages as tags" (let [page-1 (ldb/get-page @conn "page1") @@ -95,8 +93,7 @@ (ldb/register-transact-pipeline-fn! identity))) (deftest ensure-query-property-on-tag-additions-test - (let [graph test-helper/test-db - conn (db-test/create-conn-with-blocks + (let [conn (db-test/create-conn-with-blocks {:pages-and-blocks [{:page {:block/title "page1"} :blocks [{:block/title "b1"} {:block/title "b2"}]}] @@ -106,9 +103,7 @@ b1 (some #(when (= "b1" (:block/title %)) %) blocks) b2 (some #(when (= "b2" (:block/title %)) %) blocks) query-child (ldb/get-page @conn "QueryChild")] - (ldb/register-transact-pipeline-fn! - (fn [tx-report] - (worker-pipeline/transact-pipeline graph tx-report))) + (ldb/register-transact-pipeline-fn! worker-pipeline/transact-pipeline) (testing "tagging with #Query adds query property" (ldb/transact! conn [[:db/add (:db/id b1) :block/tags :logseq.class/Query]])