diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index 8a68ba32e4..428f0fa862 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -72,7 +72,6 @@ frontend.commands commands frontend.common.file-based.db common-file-db frontend.common.date common-date - frontend.common.file.core common-file frontend.common.file.util wfu frontend.common.missionary-util c.m frontend.common.schema-register sr @@ -167,6 +166,9 @@ lambdaisland.glogi log logseq.cli.common.graph cli-common-graph logseq.cli.text-util cli-text-util + logseq.cli.common.export.common cli-export-common + logseq.cli.common.export.text cli-export-text + logseq.cli.common.file common-file logseq.common.config common-config logseq.common.date-time-util date-time-util logseq.common.graph common-graph diff --git a/deps/cli/.clj-kondo/config.edn b/deps/cli/.clj-kondo/config.edn index 473ba03391..e89549e62b 100644 --- a/deps/cli/.clj-kondo/config.edn +++ b/deps/cli/.clj-kondo/config.edn @@ -14,7 +14,8 @@ logseq.cli.commands.graph cli-graph logseq.cli.common.graph cli-common-graph logseq.cli.common.export.text cli-export-text - ;; logseq.cli.common.export.common cli-export-common + logseq.cli.common.export.common cli-export-common + logseq.cli.common.file common-file logseq.cli.util cli-util logseq.cli.text-util cli-text-util logseq.common.config common-config diff --git a/deps/cli/bb.edn b/deps/cli/bb.edn index 746fe4ca02..a374894e42 100644 --- a/deps/cli/bb.edn +++ b/deps/cli/bb.edn @@ -40,5 +40,5 @@ :tasks/config {:large-vars - {:max-lines-count 30 + {:max-lines-count 45 :metadata-exceptions #{:large-vars/cleanup-todo}}}} diff --git a/deps/cli/src/logseq/cli/commands/export.cljs b/deps/cli/src/logseq/cli/commands/export.cljs index 7b09044133..eb8857fbb9 100644 --- a/deps/cli/src/logseq/cli/commands/export.cljs +++ b/deps/cli/src/logseq/cli/commands/export.cljs @@ -4,39 +4,38 @@ [cljs.pprint] [clojure.string :as string] [datascript.core :as d] - [logseq.cli.common.export.text :as cli-export-text] [logseq.cli.common.export.common :as cli-export-common] - [logseq.cli.common.file :as cli-common-file] - [logseq.cli.common.zip :as cli-common-zip] + [logseq.cli.common.export.text :as cli-export-text] + [logseq.cli.common.file :as common-file] + [logseq.cli.common.util :as cli-common-util] [logseq.cli.util :as cli-util] [logseq.common.config :as common-config] [logseq.common.util :as common-util] - [logseq.db.common.entity-plus :as entity-plus] - [logseq.db.common.sqlite-cli :as sqlite-cli] - [logseq.db.sqlite.create-graph :as sqlite-create-graph])) + [logseq.db.common.sqlite-cli :as sqlite-cli])) -(defn get-all-page->content - [repo db options] - ;; TODO: entity-plus or sqlite-util load faster? - (let [filter-fn (if (entity-plus/db-based-graph? db) - (fn [ent] - (or (not (:logseq.property/built-in? ent)) - (contains? sqlite-create-graph/built-in-pages-names (:block/title ent)))) - (constantly true))] - (->> (d/datoms db :avet :block/name) - (map #(d/entity db (:e %))) - (filter filter-fn) - (map (fn [e] - [(:block/title e) - (cli-common-file/block->content repo db (:block/uuid e) {} options)]))))) +(defn- get-content-config [db] + (let [repo-config (-> (d/q '[:find ?content + :where [?b :file/path "logseq/config.edn"] [?b :file/content ?content]] + db) + ffirst + common-util/safe-read-map-string) + indent + ;; Copy of state/get-export-bullet-indentation + (case (get repo-config :export/bullet-indentation :tab) + :eight-spaces + " " + :four-spaces + " " + :two-spaces + " " + :tab + "\t")] + {:export-bullet-indentation indent})) -(defn content (get-all-page->content repo - db - ;; TODO: Indentation - {:export-bullet-indentation "\t"})] +(defn- get-file-contents + "Modified version of export.common/content (common-file/get-all-page->content repo db content-config)] (map (fn [[page-title content]] {:path (str page-title "." suffix) :content content @@ -44,29 +43,30 @@ :format :markdown}) page->content))) -(defn export-files-as-markdown - "options see also `export-blocks-as-markdown`" +(defn- export-files-as-markdown + "Modified version of handler.export.text/export-files-as-markdown for the CLI" [repo files options] (mapv (fn [{:keys [path title content]}] [(or path title) (cli-export-text/export-helper repo content :markdown options)]) files)) -(defn export-repo-as-markdown! +(defn- export-repo-as-markdown! + "Modified version of handler.export.text/export-repo-as-markdown for the CLI" [repo db] - (let [files* ( (.generateNodeStream zip #js {:streamFiles true :type "nodebuffer"}) - (.pipe (fs/createWriteStream (str zip-file-name ".zip")))) - (println "Exported graph to" (str zip-file-name ".zip"))))))) + zip-file-name (str repo' "_markdown_" (quot (common-util/time-ms) 1000)) + zip (cli-common-util/make-export-zip zip-file-name files)] + (-> (.generateNodeStream zip #js {:streamFiles true :type "nodebuffer"}) + (.pipe (fs/createWriteStream (str zip-file-name ".zip")))) + (println "Exported graph to" (str zip-file-name ".zip")))))) (defn export [{{:keys [graph]} :opts}] (if (fs/existsSync (cli-util/get-graph-dir graph)) diff --git a/deps/cli/src/logseq/cli/common/export/common.cljs b/deps/cli/src/logseq/cli/common/export/common.cljs index b66430c369..b29bb9a565 100644 --- a/deps/cli/src/logseq/cli/common/export/common.cljs +++ b/deps/cli/src/logseq/cli/common/export/common.cljs @@ -6,7 +6,8 @@ (:require [cljs.core.match :refer [match]] [clojure.string :as string] [datascript.core :as d] - [logseq.cli.common.file :as cli-common-file] + [logseq.cli.common.file :as common-file] + [logseq.cli.common.util :as cli-common-util :refer-macros [removev concatv mapcatv]] [logseq.common.util :as common-util] [logseq.db :as ldb] [logseq.graph-parser.mldoc :as gp-mldoc] @@ -51,40 +52,15 @@ :keep-only-level<=N :all :newline-after-block false}}) -;; Fn workarounds -;; ============== - +;; Global vars that are not explicitly passed in all fns +;; These vars must be bound in order to use most fns in this namespace (def ^:dynamic *current-db* nil) (def ^:dynamic *current-repo* nil) +;; Config used by logseq.cli.common.file fns (def ^:dynamic *content-config* nil) -(defn get-block-by-uuid - [id] - (d/entity *current-db* [:block/uuid (if (uuid? id) id (uuid id))])) - -(defn zero-pad - [n] - (if (< n 10) - (str "0" n) - (str n))) - -(defmacro concatv - "Vector version of concat. non-lazy" - [& args] - `(vec (concat ~@args))) - -(defmacro mapcatv - "Vector version of mapcat. non-lazy" - [f coll & colls] - `(vec (mapcat ~f ~coll ~@colls))) - -(defmacro removev - "Vector version of remove. non-lazy" - [pred coll] - `(vec (remove ~pred ~coll))) - ;;; internal utils -(defn- get-blocks-contents +(defn ^:api get-blocks-contents [repo root-block-uuid & {:keys [init-level] :or {init-level 1}}] (let [block (d/entity *current-db* [:block/uuid root-block-uuid]) @@ -93,22 +69,16 @@ root-id (:block/uuid block') blocks (ldb/get-block-and-children *current-db* root-id) tree (otree/blocks->vec-tree repo *current-db* blocks root-id {:link link})] - (cli-common-file/tree->file-content *current-repo* *current-db* tree - {:init-level init-level :link link} - *content-config*))) - -;; (defn root-block-uuids->content -;; [repo root-block-uuids] -;; (let [contents (mapv (fn [id] -;; (get-blocks-contents repo id)) root-block-uuids)] -;; (string/join "\n" (mapv string/trim-newline contents)))) + (common-file/tree->file-content *current-repo* *current-db* tree + {:init-level init-level :link link} + *content-config*))) (declare remove-block-ast-pos Properties-block-ast?) (defn- block-uuid->ast [block-uuid] - (let [block (into {} (get-block-by-uuid block-uuid)) - content (cli-common-file/tree->file-content *current-repo* *current-db* [block] {:init-level 1} *content-config*) + (let [block (into {} (d/entity *current-db* [:block/uuid block-uuid])) + content (common-file/tree->file-content *current-repo* *current-db* [block] {:init-level 1} *content-config*) format :markdown] (when content (removev Properties-block-ast? @@ -124,13 +94,9 @@ (mapv remove-block-ast-pos (gp-mldoc/->edn *current-repo* content format)))))) -(defn get-page-content +(defn ^:api get-page-content [page-uuid] - (let [repo *current-repo* - db *current-db*] - (cli-common-file/block->content repo db page-uuid - nil - *content-config*))) + (common-file/block->content *current-repo* *current-db* page-uuid nil *content-config*)) (defn- page-name->ast [page-name] @@ -160,9 +126,9 @@ [inline-coll meta] (with-meta ["Paragraph" inline-coll] meta)) -;; ;;; internal utils (ends) +;;; internal utils (ends) -;; ;;; utils +;;; utils (defn priority->string [priority] @@ -184,8 +150,8 @@ repetition (if repetition (str " " (repetition-to-string repetition)) "") - hour (when hour (zero-pad hour)) - min (when min (zero-pad min)) + hour (when hour (cli-common-util/zero-pad hour)) + min (when min (cli-common-util/zero-pad min)) time (cond (and hour min) (common-util/format " %s:%s" hour min) @@ -196,12 +162,13 @@ (common-util/format "%s%s-%s-%s %s%s%s%s" open (str year) - (zero-pad month) - (zero-pad day) + (cli-common-util/zero-pad month) + (cli-common-util/zero-pad day) wday time repetition close))) + (defn hashtag-value->string [inline-coll] (reduce str @@ -217,32 +184,9 @@ ast-content))) inline-coll))) -;; (defn content -;; [repo options] -;; (state/content repo options)) - -;; (defn content (content repo -;; {:export-bullet-indentation (state/get-export-bullet-indentation)})] -;; (clojure.core/map (fn [[page-title content]] -;; {:path (str page-title "." suffix) -;; :content content -;; :title page-title -;; :format :markdown}) -;; page->content))) - -;; ;;; utils (ends) - -;; ;;; replace block-ref, block-embed, page-embed +;;; replace block-ref, block-embed, page-embed (defn- replace-block-reference-in-heading [{:keys [title] :as ast-content}] @@ -534,7 +478,7 @@ (if (get-in *state* [:replace-ref-embed :block&page-embed-replaced?]) (do (set! *state* (assoc-in *state* [:replace-ref-embed :block&page-embed-replaced?] false)) (recur block-ast-coll result-block-ast-tcoll - (vec (concat block-ast-coll-to-replace-references block-ast-coll-replaced)) + (concatv block-ast-coll-to-replace-references block-ast-coll-replaced) (vec other-block-asts-to-replace-embed))) (recur block-ast-coll (reduce conj! result-block-ast-tcoll block-ast-coll-replaced) (vec block-ast-coll-to-replace-references) (vec other-block-asts-to-replace-embed)))) @@ -547,7 +491,7 @@ (conj block-ast-coll-to-replace-references block-ast) (vec block-ast-coll-to-replace-embeds))))))) -;; ;;; replace block-ref, block-embed, page-embed (ends) +;;; replace block-ref, block-embed, page-embed (ends) (def remove-block-ast-pos "[[ast-type ast-content] _pos] -> [ast-type ast-content]" @@ -595,7 +539,7 @@ :result-ast-tcoll persistent!)) -;; ;;; inline transformers +;;; inline transformers (defn remove-emphasis ":mapcat-fns-on-inline-ast" @@ -654,9 +598,9 @@ {:r [] :after-break-line? true} inline-coll))) -;; ;;; inline transformers (ends) +;;; inline transformers (ends) -;; ;;; walk on block-ast, apply inline transformers +;;; walk on block-ast, apply inline transformers (defn- walk-block-ast-helper [inline-coll map-fns-on-inline-ast mapcat-fns-on-inline-ast fns-on-inline-coll] @@ -726,9 +670,9 @@ ;; else block-ast))) -;; ;;; walk on block-ast, apply inline transformers (ends) +;;; walk on block-ast, apply inline transformers (ends) -;; ;;; simple ast +;;; simple ast (def simple-ast-malli-schema (mu/closed-schema [:or @@ -763,7 +707,7 @@ :indent (reduce str (concatv (repeat (:level simple-ast) "\t") (repeat (:extra-space-count simple-ast) " "))))) -(defn- merge-adjacent-spaces&newlines +(defn- ^:large-vars/cleanup-todo merge-adjacent-spaces&newlines [simple-ast-coll] (loop [r (transient []) last-ast nil diff --git a/deps/cli/src/logseq/cli/common/export/text.cljs b/deps/cli/src/logseq/cli/common/export/text.cljs index f044017c33..1e094ccbbd 100644 --- a/deps/cli/src/logseq/cli/common/export/text.cljs +++ b/deps/cli/src/logseq/cli/common/export/text.cljs @@ -1,25 +1,15 @@ (ns logseq.cli.common.export.text + "Common fns between frontend and CLI for exporting as markdown" (:require [clojure.string :as string] - [logseq.cli.common.export.common :as common :refer + [logseq.cli.common.export.common :as cli-export-common :refer [*state* newline* indent raw-text space simple-asts->string]] - [logseq.graph-parser.mldoc :as gp-mldoc])) + [logseq.cli.common.util :refer-macros [removev concatv mapcatv]] + [logseq.graph-parser.mldoc :as gp-mldoc] + [logseq.graph-parser.schema.mldoc :as mldoc-schema])) -(defmacro concatv - "Vector version of concat. non-lazy" - [& args] - `(vec (concat ~@args))) +;;; block-ast, inline-ast -> simple-ast -(defmacro mapcatv - "Vector version of mapcat. non-lazy" - [f coll & colls] - `(vec (mapcat ~f ~coll ~@colls))) - -(defmacro removev - "Vector version of remove. non-lazy" - [pred coll] - `(vec (remove ~pred ~coll))) - -(defn indent-with-2-spaces +(defn ^:api indent-with-2-spaces "also consider (get-in *state* [:export-options :indent-style])" [level] (let [indent-style (get-in *state* [:export-options :indent-style])] @@ -34,7 +24,7 @@ (defn- block-heading [{:keys [title _tags marker level _numbering priority _anchor _meta _unordered size]}] (let [indent-style (get-in *state* [:export-options :indent-style]) - priority* (and priority (raw-text (common/priority->string priority))) + priority* (and priority (raw-text (cli-export-common/priority->string priority))) heading* (if (= indent-style "dashes") [(indent (dec level) 0) (raw-text "-")] [(indent (dec level) 0)]) @@ -293,13 +283,13 @@ [ast-content] (let [[type timestamp-content] ast-content] (-> (case type - "Scheduled" ["SCHEDULED: " (common/timestamp-to-string timestamp-content)] - "Deadline" ["DEADLINE: " (common/timestamp-to-string timestamp-content)] - "Date" [(common/timestamp-to-string timestamp-content)] - "Closed" ["CLOSED: " (common/timestamp-to-string timestamp-content)] - "Clock" ["CLOCK: " (common/timestamp-to-string (second timestamp-content))] + "Scheduled" ["SCHEDULED: " (cli-export-common/timestamp-to-string timestamp-content)] + "Deadline" ["DEADLINE: " (cli-export-common/timestamp-to-string timestamp-content)] + "Date" [(cli-export-common/timestamp-to-string timestamp-content)] + "Closed" ["CLOSED: " (cli-export-common/timestamp-to-string timestamp-content)] + "Clock" ["CLOCK: " (cli-export-common/timestamp-to-string (second timestamp-content))] "Range" (let [{:keys [start stop]} timestamp-content] - [(str (common/timestamp-to-string start) "--" (common/timestamp-to-string stop))])) + [(str (cli-export-common/timestamp-to-string start) "--" (cli-export-common/timestamp-to-string stop))])) string/join raw-text vector))) @@ -344,9 +334,9 @@ (when (> current-level 1) (indent-with-2-spaces (dec current-level)))))]) -;; {:malli/schema ...} only works on public vars, so use m/=> here -;; (m/=> block-ast->simple-ast [:=> [:cat mldoc-schema/block-ast-schema] [:sequential simple-ast-malli-schema]]) -(defn- block-ast->simple-ast +;; {:malli/schema ...} only works on public vars so make this public +(defn ^:large-vars/cleanup-todo ^:api block-ast->simple-ast + {:malli/schema [:=> [:cat mldoc-schema/block-ast-schema] [:sequential cli-export-common/simple-ast-malli-schema]]} [block] (let [newline-after-block? (get-in *state* [:export-options :newline-after-block])] (removev @@ -406,7 +396,7 @@ (block-hiccup ast-content) (assert false (print-str :block-ast->simple-ast ast-type "not implemented yet"))))))) -(defn- inline-ast->simple-ast +(defn- ^:large-vars/cleanup-todo inline-ast->simple-ast [inline] (let [[ast-type ast-content] inline] (case ast-type @@ -419,7 +409,7 @@ "Code" [(raw-text "`" ast-content "`")] "Tag" - [(raw-text (str "#" (common/hashtag-value->string ast-content)))] + [(raw-text (str "#" (cli-export-common/hashtag-value->string ast-content)))] "Spaces" ; what's this ast-type for ? nil "Plain" @@ -458,8 +448,9 @@ nil (assert false (print-str :inline-ast->simple-ast ast-type "not implemented yet"))))) +;;; block-ast, inline-ast -> simple-ast (ends) -(defn export-helper +(defn ^:large-vars/cleanup-todo export-helper [repo content format options] (let [remove-options (set (:remove-options options)) other-options (:other-options options)] @@ -473,30 +464,30 @@ :keep-only-level<=N (:keep-only-level<=N other-options) :newline-after-block (:newline-after-block other-options)}})] (let [ast (gp-mldoc/->edn repo content format) - ast (mapv common/remove-block-ast-pos ast) - ast (vec (remove common/Properties-block-ast? ast)) - ast* (common/replace-block&page-reference&embed ast) + ast (mapv cli-export-common/remove-block-ast-pos ast) + ast (removev cli-export-common/Properties-block-ast? ast) + ast* (cli-export-common/replace-block&page-reference&embed ast) keep-level<=n (get-in *state* [:export-options :keep-only-level<=N]) ast* (if (pos? keep-level<=n) - (common/keep-only-level<=n ast* keep-level<=n) + (cli-export-common/keep-only-level<=n ast* keep-level<=n) ast*) ast** (if (= "no-indent" (get-in *state* [:export-options :indent-style])) - (mapv common/replace-Heading-with-Paragraph ast*) + (mapv cli-export-common/replace-Heading-with-Paragraph ast*) ast*) config-for-walk-block-ast (cond-> {} (get-in *state* [:export-options :remove-emphasis?]) - (update :mapcat-fns-on-inline-ast conj common/remove-emphasis) + (update :mapcat-fns-on-inline-ast conj cli-export-common/remove-emphasis) (get-in *state* [:export-options :remove-page-ref-brackets?]) - (update :map-fns-on-inline-ast conj common/remove-page-ref-brackets) + (update :map-fns-on-inline-ast conj cli-export-common/remove-page-ref-brackets) (get-in *state* [:export-options :remove-tags?]) - (update :mapcat-fns-on-inline-ast conj common/remove-tags) + (update :mapcat-fns-on-inline-ast conj cli-export-common/remove-tags) (= "no-indent" (get-in *state* [:export-options :indent-style])) - (update :fns-on-inline-coll conj common/remove-prefix-spaces-in-Plain)) + (update :fns-on-inline-coll conj cli-export-common/remove-prefix-spaces-in-Plain)) ast*** (if-not (empty? config-for-walk-block-ast) - (mapv (partial common/walk-block-ast config-for-walk-block-ast) ast**) + (mapv (partial cli-export-common/walk-block-ast config-for-walk-block-ast) ast**) ast**) - simple-asts (vec (mapcat block-ast->simple-ast ast***))] + simple-asts (mapcatv block-ast->simple-ast ast***)] (simple-asts->string simple-asts))))) \ No newline at end of file diff --git a/deps/cli/src/logseq/cli/common/file.cljs b/deps/cli/src/logseq/cli/common/file.cljs index a380103fc2..055ad01566 100644 --- a/deps/cli/src/logseq/cli/common/file.cljs +++ b/deps/cli/src/logseq/cli/common/file.cljs @@ -1,11 +1,12 @@ (ns logseq.cli.common.file - "Convert blocks to file content. Used for exports and saving file to disk. Shared - by CLI, worker and frontend namespaces" + "Convert blocks to file content for file and DB graphs. Used for exports and + saving file to disk. Shared by CLI, worker and frontend namespaces" (:require [clojure.string :as string] [datascript.core :as d] [logseq.db :as ldb] [logseq.db.common.entity-plus :as entity-plus] [logseq.db.frontend.content :as db-content] + [logseq.db.sqlite.create-graph :as sqlite-create-graph] [logseq.db.sqlite.util :as sqlite-util] [logseq.graph-parser.property :as gp-property] [logseq.outliner.tree :as otree])) @@ -29,7 +30,7 @@ :else content)) -(defn- transform-content +(defn- ^:large-vars/cleanup-todo transform-content [repo db {:block/keys [collapsed? format pre-block? properties] :as b} level {:keys [heading-to-list?]} context {:keys [db-based?]}] (let [title (or (:block/raw-title b) (:block/title b)) block-ref-not-saved? (and (not db-based?) @@ -121,3 +122,18 @@ (tree->file-content repo db tree (assoc tree->file-opts :init-level init-level) context))) + +(defn get-all-page->content + "Exports a graph's pages as tuples of page name and page content" + [repo db options] + (let [filter-fn (if (ldb/db-based-graph? db) + (fn [ent] + (or (not (:logseq.property/built-in? ent)) + (contains? sqlite-create-graph/built-in-pages-names (:block/title ent)))) + (constantly true))] + (->> (d/datoms db :avet :block/name) + (map #(d/entity db (:e %))) + (filter filter-fn) + (map (fn [e] + [(:block/title e) + (block->content repo db (:block/uuid e) {} options)]))))) \ No newline at end of file diff --git a/deps/cli/src/logseq/cli/common/graph.cljs b/deps/cli/src/logseq/cli/common/graph.cljs index 28521131d2..43cc8b0792 100644 --- a/deps/cli/src/logseq/cli/common/graph.cljs +++ b/deps/cli/src/logseq/cli/common/graph.cljs @@ -7,7 +7,7 @@ [logseq.common.config :as common-config] [logseq.common.graph :as common-graph])) -(defn- graph-name->path +(defn ^:api graph-name->path [graph-name] (when graph-name (-> graph-name diff --git a/deps/cli/src/logseq/cli/common/util.cljc b/deps/cli/src/logseq/cli/common/util.cljc new file mode 100644 index 0000000000..756737f968 --- /dev/null +++ b/deps/cli/src/logseq/cli/common/util.cljc @@ -0,0 +1,42 @@ +(ns logseq.cli.common.util + "Common util fns between CLI and frontend" + (:require #?(:org.babashka/nbb ["jszip$default" :as JSZip] + :cljs ["jszip" :as JSZip]) + #_:clj-kondo/ignore + [clojure.string :as string])) + +#?(:cljs + (defn make-export-zip + "Makes a zipfile for an exported version of graph. Removes files with blank content" + [zip-filename file-name-content] + (let [zip (JSZip.) + folder (.folder zip zip-filename)] + (doseq [[file-name content] file-name-content] + (when-not (string/blank? content) + (.file folder (-> file-name + (string/replace #"^/+" "")) + content))) + zip))) + +(defn zero-pad + [n] + (if (< n 10) + (str "0" n) + (str n))) + +;; Macros are defined at top-level for frontend and nbb + +(defmacro concatv + "Vector version of concat. non-lazy" + [& args] + `(vec (concat ~@args))) + +(defmacro mapcatv + "Vector version of mapcat. non-lazy" + [f coll & colls] + `(vec (mapcat ~f ~coll ~@colls))) + +(defmacro removev + "Vector version of remove. non-lazy" + [pred coll] + `(vec (remove ~pred ~coll))) diff --git a/deps/cli/src/logseq/cli/common/zip.cljs b/deps/cli/src/logseq/cli/common/zip.cljs deleted file mode 100644 index a9278e2842..0000000000 --- a/deps/cli/src/logseq/cli/common/zip.cljs +++ /dev/null @@ -1,22 +0,0 @@ -(ns logseq.cli.common.zip - ;; TODO: nbb - (:require ["jszip$default" :as JSZip] - [clojure.string :as string])) - -(defn make-file [content file-name args] - (let [blob-content (clj->js [content]) - last-modified (or (aget content "lastModified") (js/Date.)) - args (clj->js args)] - (aset args "lastModified" last-modified) - (js/File. blob-content file-name args))) - -;; TODO: reuse -(defn make-zip [zip-filename file-name-content _repo] - (let [zip (JSZip.) - folder (.folder zip zip-filename)] - (doseq [[file-name content] file-name-content] - (when-not (string/blank? content) - (.file folder (-> file-name - (string/replace #"^/+" "")) - content))) - zip)) diff --git a/src/main/frontend/common/file/core.cljs b/src/main/frontend/common/file/core.cljs deleted file mode 100644 index b4120b19f0..0000000000 --- a/src/main/frontend/common/file/core.cljs +++ /dev/null @@ -1,122 +0,0 @@ -(ns frontend.common.file.core - "Convert blocks to file content. Used for exports and saving file to disk. Shared - by worker and frontend namespaces" - (:require [clojure.string :as string] - [datascript.core :as d] - [logseq.db :as ldb] - [logseq.db.common.entity-plus :as entity-plus] - [logseq.db.frontend.content :as db-content] - [logseq.db.sqlite.util :as sqlite-util] - [logseq.graph-parser.property :as gp-property] - [logseq.outliner.tree :as otree])) - -(defn- indented-block-content - [content spaces-tabs] - (let [lines (string/split-lines content)] - (string/join (str "\n" spaces-tabs) lines))) - -(defn- content-with-collapsed-state - "Only accept nake content (without any indentation)" - [repo format content collapsed?] - (cond - collapsed? - (gp-property/insert-property repo format content :collapsed true) - - ;; Don't check properties. Collapsed is an internal state log as property in file, but not counted into properties - (false? collapsed?) - (gp-property/remove-property format :collapsed content) - - :else - content)) - -(defn- transform-content - [repo db {:block/keys [collapsed? format pre-block? properties] :as b} level {:keys [heading-to-list?]} context {:keys [db-based?]}] - (let [title (or (:block/raw-title b) (:block/title b)) - block-ref-not-saved? (and (not db-based?) - (first (:block/_refs (d/entity db (:db/id b)))) - (not (string/includes? title (str (:block/uuid b))))) - heading (:heading properties) - title (if db-based? - ;; replace [[uuid]] with block's content - (db-content/recur-replace-uuid-in-block-title (d/entity db (:db/id b))) - title) - content (or title "") - content (cond - pre-block? - (let [content (string/trim content)] - (str content "\n")) - - :else - (let [[prefix spaces-tabs] - (cond - (= format :org) - [(->> - (repeat level "*") - (apply str)) ""] - - :else - (let [level (if (and heading-to-list? heading) - (if (> heading 1) - (dec heading) - heading) - level) - spaces-tabs (->> - (repeat (dec level) (:export-bullet-indentation context)) - (apply str))] - [(str spaces-tabs "-") (str spaces-tabs " ")])) - content (if heading-to-list? - (-> (string/replace content #"^\s?#+\s+" "") - (string/replace #"^\s?#+\s?$" "")) - content) - content (if db-based? content (content-with-collapsed-state repo format content collapsed?)) - new-content (indented-block-content (string/trim content) spaces-tabs) - sep (if (string/blank? new-content) - "" - " ")] - (str prefix sep new-content)))] - (if block-ref-not-saved? - (gp-property/insert-property repo format content :id (str (:block/uuid b))) - content))) - -(defn- tree->file-content-aux - [repo db tree {:keys [init-level link] :as opts} context] - (let [db-based? (sqlite-util/db-based-graph? repo) - block-contents (transient [])] - (loop [[f & r] tree level init-level] - (if (nil? f) - (->> block-contents persistent! flatten (remove nil?)) - (let [page? (nil? (:block/page f)) - content (if (and page? (not link)) nil (transform-content repo db f level opts context {:db-based? db-based?})) - new-content - (if-let [children (seq (:block/children f))] - (cons content (tree->file-content-aux repo db children {:init-level (inc level)} context)) - [content])] - (conj! block-contents new-content) - (recur r level)))))) - -(defn tree->file-content - "Used by both file and DB graphs for export and for file-graph specific features" - [repo db tree opts context] - (->> (tree->file-content-aux repo db tree opts context) (string/join "\n"))) - -(defn- update-block-content - [db item eid] - ;; This may not be needed if this becomes a file-graph only context - (if (entity-plus/db-based-graph? db) - (db-content/update-block-content db item eid) - item)) - -(defn block->content - "Converts a block including its children (recursively) to plain-text." - [repo db root-block-uuid tree->file-opts context] - (assert (uuid? root-block-uuid)) - (let [init-level (or (:init-level tree->file-opts) - (if (ldb/page? (d/entity db [:block/uuid root-block-uuid])) - 0 - 1)) - blocks (->> (d/pull-many db '[*] (keep :db/id (ldb/get-block-and-children db root-block-uuid))) - (map #(update-block-content db % (:db/id %)))) - tree (otree/blocks->vec-tree repo db blocks (str root-block-uuid))] - (tree->file-content repo db tree - (assoc tree->file-opts :init-level init-level) - context))) diff --git a/src/main/frontend/extensions/zip.cljs b/src/main/frontend/extensions/zip.cljs index ff0c7dcbb2..f87df5302c 100644 --- a/src/main/frontend/extensions/zip.cljs +++ b/src/main/frontend/extensions/zip.cljs @@ -1,6 +1,5 @@ (ns frontend.extensions.zip - (:require [clojure.string :as string] - ["jszip" :as JSZip] + (:require [logseq.cli.common.util :as cli-common-util] [promesa.core :as p])) (defn make-file [content file-name args] @@ -11,12 +10,6 @@ (js/File. blob-content file-name args))) (defn make-zip [zip-filename file-name-content _repo] - (let [zip (JSZip.) - folder (.folder zip zip-filename)] - (doseq [[file-name content] file-name-content] - (when-not (string/blank? content) - (.file folder (-> file-name - (string/replace #"^/+" "")) - content))) + (let [zip (cli-common-util/make-export-zip zip-filename file-name-content)] (p/let [zip-blob (.generateAsync zip #js {:type "blob"})] (make-file zip-blob (str zip-filename ".zip") {:type "application/zip"})))) diff --git a/src/main/frontend/handler/export/common.cljs b/src/main/frontend/handler/export/common.cljs index 2ead1c4947..4946ddaf63 100644 --- a/src/main/frontend/handler/export/common.cljs +++ b/src/main/frontend/handler/export/common.cljs @@ -3,189 +3,34 @@ exclude some fns which produce lazy-seq, which can cause strange behaviors when use together with dynamic var." (:refer-clojure :exclude [map filter mapcat concat remove]) - (:require [cljs.core.match :refer [match]] - [clojure.string :as string] - [frontend.common.file.core :as common-file] - [frontend.db :as db] - [frontend.format.mldoc :as mldoc] - [frontend.modules.file.core :as outliner-file] - [frontend.modules.outliner.tree :as outliner-tree] + (:require [clojure.string :as string] + [frontend.db.conn :as conn] [frontend.state :as state] - [frontend.util :as util :refer [concatv mapcatv removev]] - [malli.core :as m] - [malli.util :as mu] + [logseq.cli.common.export.common :as cli-export-common] [promesa.core :as p])) -;;; TODO: split frontend.handler.export.text related states -(def ^:dynamic *state* - "dynamic var, state used for exporting" - {;; current level of Heading, start from 1(same as mldoc), use when `block-ast->simple-ast` - :current-level 1 - ;; emphasis symbol (use when `block-ast->simple-ast`) - :outside-em-symbol nil - ;; (use when `block-ast->simple-ast`) - :indent-after-break-line? false - ;; TODO: :last-empty-heading? false - ;; current: | want: - ;; - | - xxx - ;; xxx | yyy - ;; yyy | - - ;; this submap is used when replace block-reference, block-embed, page-embed - :replace-ref-embed - {;; start from 1 - :current-level 1 - :block-ref-replaced? false - :block&page-embed-replaced? false} - - ;; submap for :newline-after-block internal state - :newline-after-block - {:current-block-is-first-heading-block? true} - - ;; export-options submap - :export-options - {;; dashes, spaces, no-indent - :indent-style "dashes" - :remove-page-ref-brackets? false - :remove-emphasis? false - :remove-tags? false - :remove-properties? true - :keep-only-level<=N :all - :newline-after-block false}}) - -;;; internal utils -(defn- get-blocks-contents - [repo root-block-uuid & {:keys [init-level] - :or {init-level 1}}] - (let [block (db/entity [:block/uuid root-block-uuid]) - link (:block/link block) - block' (or link block) - root-id (:block/uuid block') - blocks (db/get-block-and-children repo root-id)] - (-> (outliner-tree/blocks->vec-tree repo blocks root-id {:link link}) - (outliner-file/tree->file-content {:init-level init-level - :link link})))) +(defn get-content-config [] + {:export-bullet-indentation (state/get-export-bullet-indentation)}) (defn root-block-uuids->content + "Converts given block uuids to content for given repo" [repo root-block-uuids] - (let [contents (mapv (fn [id] - (get-blocks-contents repo id)) root-block-uuids)] - (string/join "\n" (mapv string/trim-newline contents)))) - -(declare remove-block-ast-pos Properties-block-ast?) - -(defn- block-uuid->ast - [block-uuid] - (let [block (into {} (db/get-block-by-uuid block-uuid)) - content (outliner-file/tree->file-content [block] {:init-level 1}) - format :markdown] - (when content - (removev Properties-block-ast? - (mapv remove-block-ast-pos - (mldoc/->edn content format)))))) - -(defn- block-uuid->ast-with-children - [block-uuid] - (let [content (get-blocks-contents (state/get-current-repo) block-uuid) - format :markdown] - (when content - (removev Properties-block-ast? - (mapv remove-block-ast-pos - (mldoc/->edn content format)))))) + (binding [cli-export-common/*current-repo* repo + cli-export-common/*current-db* (conn/get-db repo) + cli-export-common/*content-config* (get-content-config)] + (let [contents (mapv (fn [id] + (cli-export-common/get-blocks-contents repo id)) root-block-uuids)] + (string/join "\n" (mapv string/trim-newline contents))))) (defn get-page-content + "Gets page content for current repo, db and state" [page-uuid] - (let [repo (state/get-current-repo) - db (db/get-db repo)] - (common-file/block->content repo db page-uuid - nil - {:export-bullet-indentation (state/get-export-bullet-indentation)}))) - -(defn- page-name->ast - [page-name] - (let [page (db/get-page page-name)] - (when-let [content (get-page-content (:block/uuid page))] - (when content - (let [format :markdown] - (removev Properties-block-ast? - (mapv remove-block-ast-pos - (mldoc/->edn content format)))))))) - -(defn- update-level-in-block-ast-coll - [block-ast-coll origin-level] - (mapv - (fn [block-ast] - (let [[ast-type ast-content] block-ast] - (if (= ast-type "Heading") - [ast-type (update ast-content :level #(+ (dec %) origin-level))] - block-ast))) - block-ast-coll)) - -(defn- plain-indent-inline-ast - [level & {:keys [spaces] :or {spaces " "}}] - ["Plain" (str (reduce str (repeat (dec level) "\t")) spaces)]) - -(defn- mk-paragraph-ast - [inline-coll meta] - (with-meta ["Paragraph" inline-coll] meta)) - -;;; internal utils (ends) - -;;; utils - -(defn priority->string - [priority] - (str "[#" priority "]")) - -(defn- repetition-to-string - [[[kind] [duration] n]] - (let [kind (case kind - "Dotted" "." - "Plus" "+" - "DoublePlus" "++")] - (str kind n (string/lower-case (str (first duration)))))) - -(defn timestamp-to-string - [{:keys [date time repetition wday active]}] - (let [{:keys [year month day]} date - {:keys [hour min]} time - [open close] (if active ["<" ">"] ["[" "]"]) - repetition (if repetition - (str " " (repetition-to-string repetition)) - "") - hour (when hour (util/zero-pad hour)) - min (when min (util/zero-pad min)) - time (cond - (and hour min) - (util/format " %s:%s" hour min) - hour - (util/format " %s" hour) - :else - "")] - (util/format "%s%s-%s-%s %s%s%s%s" - open - (str year) - (util/zero-pad month) - (util/zero-pad day) - wday - time - repetition - close))) -(defn hashtag-value->string - [inline-coll] - (reduce str - (mapv - (fn [inline] - (let [[ast-type ast-content] inline] - (case ast-type - "Nested_link" - (:content ast-content) - "Link" - (:full_text ast-content) - "Plain" - ast-content))) - inline-coll))) + (binding [cli-export-common/*current-repo* (state/get-current-repo) + cli-export-common/*current-db* (conn/get-db (state/get-current-repo)) + cli-export-common/*content-config* (get-content-config)] + (cli-export-common/get-page-content page-uuid))) +;; Utils (defn content))) -;;; utils (ends) +;; Aliased fns requiring cli-export-common dynamic bindings e.g. cli-export-common/*current-repo* +(def replace-block&page-reference&embed cli-export-common/replace-block&page-reference&embed) +(def replace-Heading-with-Paragraph cli-export-common/replace-Heading-with-Paragraph) -;;; replace block-ref, block-embed, page-embed - -(defn- replace-block-reference-in-heading - [{:keys [title] :as ast-content}] - (let [inline-coll title - inline-coll* - (mapcatv - #(match [%] - [["Link" {:url ["Block_ref" block-uuid]}]] - (let [[[_ {title-inline-coll :title}]] - (block-uuid->ast (uuid block-uuid))] - (set! *state* (assoc-in *state* [:replace-ref-embed :block-ref-replaced?] true)) - title-inline-coll) - - :else [%]) - inline-coll)] - (assoc ast-content :title inline-coll*))) - -(defn- replace-block-reference-in-paragraph - [inline-coll] - (mapcatv - #(match [%] - [["Link" {:url ["Block_ref" block-uuid]}]] - (let [[[_ {title-inline-coll :title}]] - (block-uuid->ast (uuid block-uuid))] - (set! *state* (assoc-in *state* [:replace-ref-embed :block-ref-replaced?] true)) - title-inline-coll) - :else [%]) - inline-coll)) - -(declare replace-block-references) - -(defn- replace-block-reference-in-list - [list-items] - (mapv - (fn [{block-ast-coll :content sub-items :items :as item}] - (assoc item - :content (mapv replace-block-references block-ast-coll) - :items (replace-block-reference-in-list sub-items))) - list-items)) - -(defn- replace-block-reference-in-quote - [block-ast-coll] - (mapv replace-block-references block-ast-coll)) - -(defn- replace-block-reference-in-table - [{:keys [header groups] :as table}] - (let [header* - (mapv - (fn [col] - (mapcatv - #(match [%] - [["Link" {:url ["Block_ref" block-uuid]}]] - (let [[[_ {title-inline-coll :title}]] - (block-uuid->ast (uuid block-uuid))] - (set! *state* (assoc-in *state* [:replace-ref-embed :block-ref-replaced?] true)) - title-inline-coll) - :else [%]) - col)) - header) - groups* - (mapv - (fn [group] - (mapv - (fn [row] - (mapv - (fn [col] - (mapcatv - #(match [%] - [["Link" {:url ["Block_ref" block-uuid]}]] - (let [[[_ {title-inline-coll :title}]] - (block-uuid->ast (uuid block-uuid))] - (set! *state* (assoc-in *state* [:replace-ref-embed :block-ref-replaced?] true)) - title-inline-coll) - :else [%]) - col)) - row)) - group)) - groups)] - (assoc table :header header* :groups groups*))) - -(defn- replace-block-references - [block-ast] - (let [[ast-type ast-content] block-ast] - (case ast-type - "Heading" - [ast-type (replace-block-reference-in-heading ast-content)] - - "Paragraph" - (mk-paragraph-ast (replace-block-reference-in-paragraph ast-content) (meta block-ast)) - - "List" - [ast-type (replace-block-reference-in-list ast-content)] - - "Quote" - [ast-type (replace-block-reference-in-quote ast-content)] - - "Table" - [ast-type (replace-block-reference-in-table ast-content)] - ;; else - block-ast))) - -(defn- replace-block-references-until-stable - [block-ast] - (binding [*state* *state*] - (loop [block-ast block-ast] - (let [block-ast* (replace-block-references block-ast)] - (if (get-in *state* [:replace-ref-embed :block-ref-replaced?]) - (do (set! *state* (assoc-in *state* [:replace-ref-embed :block-ref-replaced?] false)) - (recur block-ast*)) - block-ast*))))) - -(defn- replace-block-embeds-helper - [current-paragraph-inlines block-uuid blocks-tcoll level] - (let [block-uuid* (subs block-uuid 2 (- (count block-uuid) 2)) - ast-coll (update-level-in-block-ast-coll - (block-uuid->ast-with-children (uuid block-uuid*)) - level)] - (cond-> blocks-tcoll - (seq current-paragraph-inlines) - (conj! ["Paragraph" current-paragraph-inlines]) - true - (#(reduce conj! % ast-coll))))) - -(defn- replace-page-embeds-helper - [current-paragraph-inlines page-name blocks-tcoll level] - (let [page-name* (subs page-name 2 (- (count page-name) 2)) - ast-coll (update-level-in-block-ast-coll - (page-name->ast page-name*) - level)] - (cond-> blocks-tcoll - (seq current-paragraph-inlines) - (conj! ["Paragraph" current-paragraph-inlines]) - true - (#(reduce conj! % ast-coll))))) - -(defn- replace-block&page-embeds-in-heading - [{inline-coll :title origin-level :level :as ast-content}] - (set! *state* (assoc-in *state* [:replace-ref-embed :current-level] origin-level)) - (if (empty? inline-coll) - ;; it's just a empty Heading, return itself - [["Heading" ast-content]] - (loop [[inline & other-inlines] inline-coll - heading-exist? false - current-paragraph-inlines [] - r (transient [])] - (if-not inline - (persistent! - (if (seq current-paragraph-inlines) - (conj! r (if heading-exist? - ["Paragraph" current-paragraph-inlines] - ["Heading" (assoc ast-content :title current-paragraph-inlines)])) - r)) - (match [inline] - [["Macro" {:name "embed" :arguments [block-uuid-or-page-name]}]] - (cond - (and (string/starts-with? block-uuid-or-page-name "((") - (string/ends-with? block-uuid-or-page-name "))")) - (do (set! *state* (assoc-in *state* [:replace-ref-embed :block&page-embed-replaced?] true)) - (recur other-inlines true [] - (replace-block-embeds-helper - current-paragraph-inlines block-uuid-or-page-name r origin-level))) - (and (string/starts-with? block-uuid-or-page-name "[[") - (string/ends-with? block-uuid-or-page-name "]]")) - (do (set! *state* (assoc-in *state* [:replace-ref-embed :block&page-embed-replaced?] true)) - (recur other-inlines true [] - (replace-page-embeds-helper - current-paragraph-inlines block-uuid-or-page-name r origin-level))) - :else ;; not ((block-uuid)) or [[page-name]], just drop the original ast - (recur other-inlines heading-exist? current-paragraph-inlines r)) - - :else - (let [current-paragraph-inlines* - (if (and (empty? current-paragraph-inlines) - heading-exist?) - (conj current-paragraph-inlines (plain-indent-inline-ast origin-level)) - current-paragraph-inlines)] - (recur other-inlines heading-exist? (conj current-paragraph-inlines* inline) r))))))) - -(defn- replace-block&page-embeds-in-paragraph - [inline-coll meta] - (let [current-level (get-in *state* [:replace-ref-embed :current-level])] - (loop [[inline & other-inlines] inline-coll - current-paragraph-inlines [] - just-after-embed? false - blocks (transient [])] - (if-not inline - (let [[first-block & other-blocks] (persistent! - (if (seq current-paragraph-inlines) - (conj! blocks ["Paragraph" current-paragraph-inlines]) - blocks))] - (if first-block - (apply vector (with-meta first-block meta) other-blocks) - [])) - (match [inline] - [["Macro" {:name "embed" :arguments [block-uuid-or-page-name]}]] - (cond - (and (string/starts-with? block-uuid-or-page-name "((") - (string/ends-with? block-uuid-or-page-name "))")) - (do (set! *state* (assoc-in *state* [:replace-ref-embed :block&page-embed-replaced?] true)) - (recur other-inlines [] true - (replace-block-embeds-helper - current-paragraph-inlines block-uuid-or-page-name blocks current-level))) - (and (string/starts-with? block-uuid-or-page-name "[[") - (string/ends-with? block-uuid-or-page-name "]]")) - (do (set! *state* (assoc-in *state* [:replace-ref-embed :block&page-embed-replaced?] true)) - (recur other-inlines [] true - (replace-page-embeds-helper - current-paragraph-inlines block-uuid-or-page-name blocks current-level))) - :else ;; not ((block-uuid)) or [[page-name]], just drop the original ast - (recur other-inlines current-paragraph-inlines false blocks)) - - :else - (let [current-paragraph-inlines* - (if just-after-embed? - (conj current-paragraph-inlines (plain-indent-inline-ast current-level)) - current-paragraph-inlines)] - (recur other-inlines (conj current-paragraph-inlines* inline) false blocks))))))) - -(declare replace-block&page-embeds) - -(defn- replace-block&page-embeds-in-list-helper - [list-items] - (binding [*state* (update-in *state* [:replace-ref-embed :current-level] inc)] - (mapv - (fn [{block-ast-coll :content sub-items :items :as item}] - (assoc item - :content (mapcatv replace-block&page-embeds block-ast-coll) - :items (replace-block&page-embeds-in-list-helper sub-items))) - list-items))) - -(defn- replace-block&page-embeds-in-list - [list-items] - [["List" (replace-block&page-embeds-in-list-helper list-items)]]) - -(defn- replace-block&page-embeds-in-quote - [block-ast-coll] - (->> block-ast-coll - (mapcatv replace-block&page-embeds) - (vector "Quote") - vector)) - -(defn- replace-block&page-embeds - [block-ast] - (let [[ast-type ast-content] block-ast] - (case ast-type - "Heading" - (replace-block&page-embeds-in-heading ast-content) - "Paragraph" - (replace-block&page-embeds-in-paragraph ast-content (meta block-ast)) - "List" - (replace-block&page-embeds-in-list ast-content) - "Quote" - (replace-block&page-embeds-in-quote ast-content) - "Table" - ;; TODO: block&page embeds in table are not replaced yet - [block-ast] - ;; else - [block-ast]))) - -(defn replace-block&page-reference&embed - "add meta :embed-depth to the embed replaced block-ast, - to avoid too deep block-ref&embed (or maybe it's a cycle)" - [block-ast-coll] - (loop [block-ast-coll block-ast-coll - result-block-ast-tcoll (transient []) - block-ast-coll-to-replace-references [] - block-ast-coll-to-replace-embeds []] - (cond - (seq block-ast-coll-to-replace-references) - (let [[block-ast-to-replace-ref & other-block-asts-to-replace-ref] - block-ast-coll-to-replace-references - embed-depth (:embed-depth (meta block-ast-to-replace-ref) 0) - block-ast-replaced (-> (replace-block-references-until-stable block-ast-to-replace-ref) - (with-meta {:embed-depth embed-depth}))] - (if (>= embed-depth 5) - ;; if :embed-depth >= 5, dont replace embed for this block anymore - ;; there is too deep, or maybe it just a ref/embed cycle - (recur block-ast-coll (conj! result-block-ast-tcoll block-ast-replaced) - (vec other-block-asts-to-replace-ref) block-ast-coll-to-replace-embeds) - (recur block-ast-coll result-block-ast-tcoll (vec other-block-asts-to-replace-ref) - (conj block-ast-coll-to-replace-embeds block-ast-replaced)))) - - (seq block-ast-coll-to-replace-embeds) - (let [[block-ast-to-replace-embed & other-block-asts-to-replace-embed] - block-ast-coll-to-replace-embeds - embed-depth (:embed-depth (meta block-ast-to-replace-embed) 0) - block-ast-coll-replaced (->> (replace-block&page-embeds block-ast-to-replace-embed) - (mapv #(with-meta % {:embed-depth (inc embed-depth)})))] - (if (get-in *state* [:replace-ref-embed :block&page-embed-replaced?]) - (do (set! *state* (assoc-in *state* [:replace-ref-embed :block&page-embed-replaced?] false)) - (recur block-ast-coll result-block-ast-tcoll - (concatv block-ast-coll-to-replace-references block-ast-coll-replaced) - (vec other-block-asts-to-replace-embed))) - (recur block-ast-coll (reduce conj! result-block-ast-tcoll block-ast-coll-replaced) - (vec block-ast-coll-to-replace-references) (vec other-block-asts-to-replace-embed)))) - - :else - (let [[block-ast & other-block-ast] block-ast-coll] - (if-not block-ast - (persistent! result-block-ast-tcoll) - (recur other-block-ast result-block-ast-tcoll - (conj block-ast-coll-to-replace-references block-ast) - (vec block-ast-coll-to-replace-embeds))))))) - -;;; replace block-ref, block-embed, page-embed (ends) - -(def remove-block-ast-pos - "[[ast-type ast-content] _pos] -> [ast-type ast-content]" - first) - -(defn Properties-block-ast? - [[tp _]] - (= tp "Properties")) - -(defn replace-Heading-with-Paragraph - "works on block-ast - replace all heading with paragraph when indent-style is no-indent" - [heading-ast] - (let [[heading-type {:keys [title marker priority size]}] heading-ast] - (if (= heading-type "Heading") - (let [inline-coll - (cond->> title - priority (cons ["Plain" (str (priority->string priority) " ")]) - marker (cons ["Plain" (str marker " ")]) - size (cons ["Plain" (str (reduce str (repeat size "#")) " ")]) - true vec)] - (mk-paragraph-ast inline-coll {:origin-ast heading-ast})) - heading-ast))) - -(defn keep-only-level<=n - [block-ast-coll n] - (-> (reduce - (fn [{:keys [result-ast-tcoll accepted-heading] :as r} ast] - (let [[heading-type {level :level}] ast - is-heading? (= heading-type "Heading")] - (cond - (and (not is-heading?) accepted-heading) - {:result-ast-tcoll (conj! result-ast-tcoll ast) :accepted-heading accepted-heading} - - (and (not is-heading?) (not accepted-heading)) - r - - (and is-heading? (<= level n)) - {:result-ast-tcoll (conj! result-ast-tcoll ast) :accepted-heading true} - - (and is-heading? (> level n)) - {:result-ast-tcoll result-ast-tcoll :accepted-heading false}))) - {:result-ast-tcoll (transient []) :accepted-heading false} - block-ast-coll) - :result-ast-tcoll - persistent!)) - -;;; inline transformers - -(defn remove-emphasis - ":mapcat-fns-on-inline-ast" - [inline-ast] - (let [[ast-type ast-content] inline-ast] - (case ast-type - "Emphasis" - (let [[_ inline-coll] ast-content] - inline-coll) - ;; else - [inline-ast]))) - -(defn remove-page-ref-brackets - ":map-fns-on-inline-ast" - [inline-ast] - (let [[ast-type ast-content] inline-ast] - (case ast-type - "Link" - (let [{:keys [url label]} ast-content] - (if (and (= "Page_ref" (first url)) - (or (empty? label) - (= label [["Plain" ""]]))) - ["Plain" (second url)] - inline-ast)) - ;; else - inline-ast))) - -(defn remove-tags - ":mapcat-fns-on-inline-ast" - [inline-ast] - (let [[ast-type _ast-content] inline-ast] - (case ast-type - "Tag" - [] - ;; else - [inline-ast]))) - -(defn remove-prefix-spaces-in-Plain - [inline-coll] - (:r - (reduce - (fn [{:keys [r after-break-line?]} ast] - (let [[ast-type ast-content] ast] - (case ast-type - "Plain" - (let [trimmed-content (string/triml ast-content)] - (if after-break-line? - (if (empty? trimmed-content) - {:r r :after-break-line? false} - {:r (conj r ["Plain" trimmed-content]) :after-break-line? false}) - {:r (conj r ast) :after-break-line? false})) - ("Break_Line" "Hard_Break_Line") - {:r (conj r ast) :after-break-line? true} - ;; else - {:r (conj r ast) :after-break-line? false}))) - {:r [] :after-break-line? true} - inline-coll))) - -;;; inline transformers (ends) - -;;; walk on block-ast, apply inline transformers - -(defn- walk-block-ast-helper - [inline-coll map-fns-on-inline-ast mapcat-fns-on-inline-ast fns-on-inline-coll] - (->> - (reduce (fn [inline-coll f] (f inline-coll)) inline-coll fns-on-inline-coll) - (mapv #(reduce (fn [inline-ast f] (f inline-ast)) % map-fns-on-inline-ast)) - (mapcatv #(reduce - (fn [inline-ast-coll f] (mapcatv f inline-ast-coll)) [%] mapcat-fns-on-inline-ast)))) - -(declare walk-block-ast) - -(defn- walk-block-ast-for-list - [list-items map-fns-on-inline-ast mapcat-fns-on-inline-ast] - (mapv - (fn [{block-ast-coll :content sub-items :items :as item}] - (assoc item - :content - (mapv - (partial walk-block-ast - {:map-fns-on-inline-ast map-fns-on-inline-ast - :mapcat-fns-on-inline-ast mapcat-fns-on-inline-ast}) - block-ast-coll) - :items - (walk-block-ast-for-list sub-items map-fns-on-inline-ast mapcat-fns-on-inline-ast))) - list-items)) - -(defn walk-block-ast - [{:keys [map-fns-on-inline-ast mapcat-fns-on-inline-ast fns-on-inline-coll] :as fns} - block-ast] - (let [[ast-type ast-content] block-ast] - (case ast-type - "Paragraph" - (mk-paragraph-ast - (walk-block-ast-helper ast-content map-fns-on-inline-ast mapcat-fns-on-inline-ast fns-on-inline-coll) - (meta block-ast)) - "Heading" - (let [{:keys [title]} ast-content] - ["Heading" - (assoc ast-content - :title - (walk-block-ast-helper title map-fns-on-inline-ast mapcat-fns-on-inline-ast fns-on-inline-coll))]) - "List" - ["List" (walk-block-ast-for-list ast-content map-fns-on-inline-ast mapcat-fns-on-inline-ast)] - "Quote" - ["Quote" (mapv (partial walk-block-ast fns) ast-content)] - "Footnote_Definition" - (let [[name contents] (rest block-ast)] - ["Footnote_Definition" - name (walk-block-ast-helper contents map-fns-on-inline-ast mapcat-fns-on-inline-ast fns-on-inline-coll)]) - "Table" - (let [{:keys [header groups]} ast-content - header* (mapv - #(walk-block-ast-helper % map-fns-on-inline-ast mapcat-fns-on-inline-ast fns-on-inline-coll) - header) - groups* (mapv - (fn [group] - (mapv - (fn [row] - (mapv - (fn [col] - (walk-block-ast-helper col map-fns-on-inline-ast mapcat-fns-on-inline-ast fns-on-inline-coll)) - row)) - group)) - groups)] - ["Table" (assoc ast-content :header header* :groups groups*)]) - - ;; else - block-ast))) - -;;; walk on block-ast, apply inline transformers (ends) - -;;; simple ast -(def simple-ast-malli-schema - (mu/closed-schema - [:or - [:map - [:type [:= :raw-text]] - [:content :string]] - [:map - [:type [:= :space]]] - [:map - [:type [:= :newline]] - [:line-count :int]] - [:map - [:type [:= :indent]] - [:level :int] - [:extra-space-count :int]]])) - -(defn raw-text [& contents] - {:type :raw-text :content (reduce str contents)}) -(def space {:type :space}) -(defn newline* [line-count] - {:type :newline :line-count line-count}) -(defn indent [level extra-space-count] - {:type :indent :level level :extra-space-count extra-space-count}) - -(defn- simple-ast->string - [simple-ast] - {:pre [(m/validate simple-ast-malli-schema simple-ast)]} - (case (:type simple-ast) - :raw-text (:content simple-ast) - :space " " - :newline (reduce str (repeat (:line-count simple-ast) "\n")) - :indent (reduce str (concatv (repeat (:level simple-ast) "\t") - (repeat (:extra-space-count simple-ast) " "))))) - -(defn- merge-adjacent-spaces&newlines - [simple-ast-coll] - (loop [r (transient []) - last-ast nil - last-raw-text-space-suffix? false - last-raw-text-newline-suffix? false - [simple-ast & other-ast-coll] simple-ast-coll] - (if (nil? simple-ast) - (persistent! (if last-ast (conj! r last-ast) r)) - (let [tp (:type simple-ast) - last-ast-type (:type last-ast)] - (case tp - :space - (if (or (contains? #{:space :newline :indent} last-ast-type) - last-raw-text-space-suffix? - last-raw-text-newline-suffix?) - ;; drop this :space - (recur r last-ast last-raw-text-space-suffix? last-raw-text-newline-suffix? other-ast-coll) - (recur (if last-ast (conj! r last-ast) r) simple-ast false false other-ast-coll)) - - :newline - (case last-ast-type - (:space :indent) ;; drop last-ast - (recur r simple-ast false false other-ast-coll) - :newline - (let [last-newline-count (:line-count last-ast) - current-newline-count (:line-count simple-ast) - kept-ast (if (> last-newline-count current-newline-count) last-ast simple-ast)] - (recur r kept-ast false false other-ast-coll)) - :raw-text - (if last-raw-text-newline-suffix? - (recur r last-ast last-raw-text-space-suffix? last-raw-text-newline-suffix? other-ast-coll) - (recur (if last-ast (conj! r last-ast) r) simple-ast false false other-ast-coll)) - ;; no-last-ast - (recur r simple-ast false false other-ast-coll)) - - :indent - (case last-ast-type - (:space :indent) ; drop last-ast - (recur r simple-ast false false other-ast-coll) - :newline - (recur (if last-ast (conj! r last-ast) r) simple-ast false false other-ast-coll) - :raw-text - (if last-raw-text-space-suffix? - ;; drop this :indent - (recur r last-ast last-raw-text-space-suffix? last-raw-text-newline-suffix? other-ast-coll) - (recur (if last-ast (conj! r last-ast) r) simple-ast false false other-ast-coll)) - ;; no-last-ast - (recur r simple-ast false false other-ast-coll)) - - :raw-text - (let [content (:content simple-ast) - empty-content? (empty? content) - first-ch (first content) - last-ch (let [num (count content)] - (when (pos? num) - (nth content (dec num)))) - newline-prefix? (some-> first-ch #{"\r" "\n"} boolean) - newline-suffix? (some-> last-ch #{"\n"} boolean) - space-prefix? (some-> first-ch #{" "} boolean) - space-suffix? (some-> last-ch #{" "} boolean)] - (cond - empty-content? ;drop this raw-text - (recur r last-ast last-raw-text-space-suffix? last-raw-text-newline-suffix? other-ast-coll) - newline-prefix? - (case last-ast-type - (:space :indent :newline) ;drop last-ast - (recur r simple-ast space-suffix? newline-suffix? other-ast-coll) - :raw-text - (recur (if last-ast (conj! r last-ast) r) simple-ast space-suffix? newline-suffix? other-ast-coll) - ;; no-last-ast - (recur r simple-ast space-suffix? newline-suffix? other-ast-coll)) - space-prefix? - (case last-ast-type - (:space :indent) ;drop last-ast - (recur r simple-ast space-suffix? newline-suffix? other-ast-coll) - (:newline :raw-text) - (recur (if last-ast (conj! r last-ast) r) simple-ast space-suffix? newline-suffix? other-ast-coll) - ;; no-last-ast - (recur r simple-ast space-suffix? newline-suffix? other-ast-coll)) - :else - (recur (if last-ast (conj! r last-ast) r) simple-ast space-suffix? newline-suffix? other-ast-coll)))))))) - -(defn simple-asts->string - [simple-ast-coll] - (->> simple-ast-coll - merge-adjacent-spaces&newlines - merge-adjacent-spaces&newlines - (mapv simple-ast->string) - string/join)) - -;;; simple ast (ends) - -;;; TODO: walk the hiccup tree, -;;; and call escape-html on all its contents -;;; - -;;; walk the hiccup tree, -;;; and call escape-html on all its contents (ends) +;; Aliased fns +(def priority->string cli-export-common/priority->string) +(def timestamp-to-string cli-export-common/timestamp-to-string) +(def hashtag-value->string cli-export-common/hashtag-value->string) +(def remove-block-ast-pos cli-export-common/remove-block-ast-pos) +(def Properties-block-ast? cli-export-common/Properties-block-ast?) +(def keep-only-level<=n cli-export-common/keep-only-level<=n) +(def remove-emphasis cli-export-common/remove-emphasis) +(def remove-page-ref-brackets cli-export-common/remove-page-ref-brackets) +(def remove-tags cli-export-common/remove-tags) +(def remove-prefix-spaces-in-Plain cli-export-common/remove-prefix-spaces-in-Plain) +(def walk-block-ast cli-export-common/walk-block-ast) \ No newline at end of file diff --git a/src/main/frontend/handler/export/html.cljs b/src/main/frontend/handler/export/html.cljs index e4b2756352..f140f061cf 100644 --- a/src/main/frontend/handler/export/html.cljs +++ b/src/main/frontend/handler/export/html.cljs @@ -6,12 +6,15 @@ [clojure.zip :as z] [frontend.db :as db] [frontend.format.mldoc :as mldoc] - [frontend.handler.export.common :as common :refer [*state*]] + [frontend.handler.export.common :as common] [frontend.handler.export.zip-helper :refer [get-level goto-last goto-level]] - [frontend.util :as util :refer [concatv mapcatv removev]] + [frontend.util :as util] [hiccups.runtime :as h] - [malli.core :as m])) + [logseq.cli.common.export.common :as cli-export-common :refer [*state*]] + [logseq.cli.common.util :refer-macros [concatv mapcatv removev]] + [malli.core :as m] + [frontend.db.conn :as conn])) (def ^:private hiccup-malli-schema [:cat :keyword [:* :any]]) @@ -426,6 +429,9 @@ first-block (and (coll? root-block-uuids-or-page-uuid) (db/entity [:block/uuid (first root-block-uuids-or-page-uuid)])) format (get first-block :block/format :markdown)] - (export-helper content format options))) + (binding [cli-export-common/*current-repo* repo + cli-export-common/*current-db* (conn/get-db repo) + cli-export-common/*content-config* (common/get-content-config)] + (export-helper content format options)))) ;;; export fns (ends) diff --git a/src/main/frontend/handler/export/opml.cljs b/src/main/frontend/handler/export/opml.cljs index a2dac12f11..3d210edcb9 100644 --- a/src/main/frontend/handler/export/opml.cljs +++ b/src/main/frontend/handler/export/opml.cljs @@ -8,15 +8,18 @@ [frontend.db :as db] [frontend.extensions.zip :as zip] [frontend.format.mldoc :as mldoc] - [frontend.handler.export.common :as common :refer - [*state* raw-text simple-asts->string space]] + [frontend.handler.export.common :as common] [frontend.handler.export.zip-helper :refer [get-level goto-last goto-level]] - [frontend.util :as util :refer [concatv mapcatv removev]] + [frontend.util :as util] [goog.dom :as gdom] [hiccups.runtime :as h] + [logseq.cli.common.export.common :as cli-export-common :refer + [*state* raw-text simple-asts->string space]] + [logseq.cli.common.util :refer-macros [concatv mapcatv removev]] [logseq.common.path :as path] - [promesa.core :as p])) + [promesa.core :as p] + [frontend.db.conn :as conn])) ;;; *opml-state* (def ^:private ^:dynamic @@ -435,29 +438,35 @@ {:pre [(or (coll? root-block-uuids-or-page-uuid) (uuid? root-block-uuids-or-page-uuid))]} (util/profile - :export-blocks-as-opml - (let [content - (if (uuid? root-block-uuids-or-page-uuid) + :export-blocks-as-opml + (let [content + (if (uuid? root-block-uuids-or-page-uuid) ;; page - (common/get-page-content root-block-uuids-or-page-uuid) - (common/root-block-uuids->content repo root-block-uuids-or-page-uuid)) - title (if (uuid? root-block-uuids-or-page-uuid) - (:block/title (db/entity [:block/uuid root-block-uuids-or-page-uuid])) - "untitled") - first-block (and (coll? root-block-uuids-or-page-uuid) - (db/entity [:block/uuid (first root-block-uuids-or-page-uuid)])) - format (get first-block :block/format :markdown)] - (export-helper content format options :title title)))) + (common/get-page-content root-block-uuids-or-page-uuid) + (common/root-block-uuids->content repo root-block-uuids-or-page-uuid)) + title (if (uuid? root-block-uuids-or-page-uuid) + (:block/title (db/entity [:block/uuid root-block-uuids-or-page-uuid])) + "untitled") + first-block (and (coll? root-block-uuids-or-page-uuid) + (db/entity [:block/uuid (first root-block-uuids-or-page-uuid)])) + format (get first-block :block/format :markdown)] + (binding [cli-export-common/*current-repo* repo + cli-export-common/*current-db* (conn/get-db repo) + cli-export-common/*content-config* (common/get-content-config)] + (export-helper content format options :title title))))) (defn- export-files-as-opml "options see also `export-blocks-as-opml`" - [files options] - (mapv - (fn [{:keys [path content title format]}] - (when (and title (not (string/blank? content))) - (util/profile (print-str :export-files-as-opml path) - [path (export-helper content format options :title title)]))) - files)) + [repo files options] + (binding [cli-export-common/*current-repo* repo + cli-export-common/*current-db* (conn/get-db repo) + cli-export-common/*content-config* (common/get-content-config)] + (mapv + (fn [{:keys [path content title format]}] + (when (and title (not (string/blank? content))) + (util/profile (print-str :export-files-as-opml path) + [path (export-helper content format options :title title)]))) + files))) (defn export-repo-as-opml! [repo] @@ -466,7 +475,7 @@ (let [repo' (if (config/db-based-graph? repo) (string/replace repo config/db-version-prefix "") (path/basename repo)) - files (->> (export-files-as-opml files nil) + files (->> (export-files-as-opml repo files nil) (clojure.core/remove nil?)) zip-file-name (str repo' "_opml_" (quot (util/time-ms) 1000))] (p/let [zipfile (zip/make-zip zip-file-name files repo')] diff --git a/src/main/frontend/handler/export/text.cljs b/src/main/frontend/handler/export/text.cljs index e5cea3593a..e0f1c70d12 100644 --- a/src/main/frontend/handler/export/text.cljs +++ b/src/main/frontend/handler/export/text.cljs @@ -3,506 +3,20 @@ (:require [clojure.string :as string] [frontend.config :as config] [frontend.db :as db] + [frontend.db.conn :as conn] [frontend.extensions.zip :as zip] - [frontend.format.mldoc :as mldoc] - [frontend.handler.export.common :as common :refer - [*state* indent newline* raw-text simple-ast-malli-schema - simple-asts->string space]] - [frontend.util :as util :refer [concatv mapcatv removev]] + [frontend.handler.export.common :as common] + [frontend.state :as state] + [frontend.util :as util] [goog.dom :as gdom] [logseq.common.path :as path] + [logseq.cli.common.export.common :as cli-export-common] + [logseq.cli.common.export.text :as cli-export-text] [logseq.db :as ldb] - [logseq.graph-parser.schema.mldoc :as mldoc-schema] - [malli.core :as m] [promesa.core :as p])) -;;; block-ast, inline-ast -> simple-ast - -(defn indent-with-2-spaces - "also consider (get-in *state* [:export-options :indent-style])" - [level] - (let [indent-style (get-in *state* [:export-options :indent-style])] - (case indent-style - "dashes" (indent level 2) - ("spaces" "no-indent") (indent level 0) - (assert false (print-str "unknown indent-style:" indent-style))))) - -(declare inline-ast->simple-ast - block-ast->simple-ast) - -(defn- block-heading - [{:keys [title _tags marker level _numbering priority _anchor _meta _unordered size]}] - (let [indent-style (get-in *state* [:export-options :indent-style]) - priority* (and priority (raw-text (common/priority->string priority))) - heading* (if (= indent-style "dashes") - [(indent (dec level) 0) (raw-text "-")] - [(indent (dec level) 0)]) - size* (and size [space (raw-text (reduce str (repeat size "#")))]) - marker* (and marker (raw-text marker))] - (set! *state* (assoc *state* :current-level level)) - (let [simple-asts - (removev nil? (concatv - (when (and (get-in *state* [:export-options :newline-after-block]) - (not (get-in *state* [:newline-after-block :current-block-is-first-heading-block?]))) - [(newline* 2)]) - heading* size* - [space marker* space priority* space] - (mapcatv inline-ast->simple-ast title) - [(newline* 1)]))] - (set! *state* (assoc-in *state* [:newline-after-block :current-block-is-first-heading-block?] false)) - simple-asts))) - -(declare block-list) -(defn- block-list-item - [{:keys [content items number _name checkbox]}] - (let [content* (mapcatv block-ast->simple-ast content) - number* (raw-text - (if number - (str number ". ") - "* ")) - checkbox* (raw-text - (if (some? checkbox) - (if (boolean checkbox) - "[X]" "[ ]") - "")) - current-level (get *state* :current-level 1) - indent' (when (> current-level 1) - (indent (dec current-level) 0)) - items* (block-list items :in-list? true)] - (concatv [indent' number* checkbox* space] - content* - [(newline* 1)] - items* - [(newline* 1)]))) - -(defn- block-list - [l & {:keys [in-list?]}] - (binding [*state* (update *state* :current-level inc)] - (concatv (mapcatv block-list-item l) - (when (and (pos? (count l)) - (not in-list?)) - [(newline* 2)])))) - -(defn- block-property-drawer - [properties] - (when-not (get-in *state* [:export-options :remove-properties?]) - (let [level (dec (get *state* :current-level 1)) - indent' (indent-with-2-spaces level)] - (reduce - (fn [r [k v]] - (conj r indent' (raw-text k "::") space (raw-text v) (newline* 1))) - [] properties)))) - -(defn- block-example - [l] - (let [level (dec (get *state* :current-level 1))] - (mapcatv - (fn [line] - [(indent-with-2-spaces level) - (raw-text " ") - (raw-text line) - (newline* 1)]) - l))) - -(defn- remove-max-prefix-spaces - [lines] - (let [common-prefix-spaces - (reduce - (fn [r line] - (if (string/blank? line) - r - (let [leading-spaces (re-find #"^\s+" line)] - (if (nil? r) - leading-spaces - (if (string/starts-with? r leading-spaces) - leading-spaces - r))))) - nil - lines) - pattern (re-pattern (str "^" common-prefix-spaces))] - (mapv (fn [line] (string/replace-first line pattern "")) lines))) - -(defn- block-src - [{:keys [lines language]}] - (let [level (dec (get *state* :current-level 1)) - lines* (if (= "no-indent" (get-in *state* [:export-options :indent-style])) - (remove-max-prefix-spaces lines) - lines)] - (concatv - [(indent-with-2-spaces level) (raw-text "```")] - (when language [(raw-text language)]) - [(newline* 1)] - (mapv raw-text lines*) - [(indent-with-2-spaces level) (raw-text "```") (newline* 1)]))) - -(defn- block-quote - [block-coll] - (let [level (dec (get *state* :current-level 1))] - (binding [*state* (assoc *state* :indent-after-break-line? true)] - (concatv (mapcatv (fn [block] - (let [block-simple-ast (block-ast->simple-ast block)] - (when (seq block-simple-ast) - (concatv [(indent-with-2-spaces level) (raw-text ">") space] - block-simple-ast)))) - block-coll) - [(newline* 2)])))) - -(declare inline-latex-fragment) -(defn- block-latex-fragment - [ast-content] - (inline-latex-fragment ast-content)) - -(defn- block-latex-env - [[name options content]] - (let [level (dec (get *state* :current-level 1))] - [(indent-with-2-spaces level) (raw-text "\\begin{" name "}" options) - (newline* 1) - (indent-with-2-spaces level) (raw-text content) - (newline* 1) - (indent-with-2-spaces level) (raw-text "\\end{" name "}") - (newline* 1)])) - -(defn- block-displayed-math - [ast-content] - [space (raw-text "$$" ast-content "$$") space]) - -(defn- block-drawer - [[name lines]] - (let [level (dec (get *state* :current-level))] - (concatv - [(raw-text ":" name ":") - (newline* 1)] - (mapcatv (fn [line] [(indent-with-2-spaces level) (raw-text line)]) lines) - [(newline* 1) (raw-text ":END:") (newline* 1)]))) - -(defn- block-footnote-definition - [[name content]] - (concatv - [(raw-text "[^" name "]:") space] - (mapcatv inline-ast->simple-ast content) - [(newline* 1)])) - -(def ^:private block-horizontal-rule [(newline* 1) (raw-text "---") (newline* 1)]) - -(defn- block-table - [{:keys [header groups]}] - (let [level (dec (get *state* :current-level 1)) - sep-line (raw-text "|" (string/join "|" (repeat (count header) "---")) "|") - header-line - (concatv (mapcatv - (fn [h] (concatv [space (raw-text "|") space] (mapcatv inline-ast->simple-ast h))) - header) - [space (raw-text "|")]) - group-lines - (mapcatv - (fn [group] - (mapcatv - (fn [row] - (concatv [(indent-with-2-spaces level)] - (mapcatv - (fn [col] - (concatv [(raw-text "|") space] - (mapcatv inline-ast->simple-ast col) - [space])) - row) - [(raw-text "|") (newline* 1)])) - group)) - groups)] - (concatv [(newline* 1) (indent-with-2-spaces level)] - (when (seq header) header-line) - (when (seq header) [(newline* 1) (indent-with-2-spaces level) sep-line (newline* 1)]) - group-lines))) - -(defn- block-comment - [s] - (let [level (dec (get *state* :current-level 1))] - [(indent-with-2-spaces level) (raw-text "") (newline* 1)])) - -(defn- block-raw-html - [s] - (let [level (dec (get *state* :current-level 1))] - [(indent-with-2-spaces level) (raw-text s) (newline* 1)])) - -(defn- block-hiccup - [s] - (let [level (dec (get *state* :current-level 1))] - [(indent-with-2-spaces level) (raw-text s) space])) - -(defn- inline-link - [{full-text :full_text}] - [(raw-text full-text)]) - -(defn- inline-nested-link - [{content :content}] - [(raw-text content)]) - -(defn- inline-subscript - [inline-coll] - (concatv [(raw-text "_{")] - (mapcatv (fn [inline] (cons space (inline-ast->simple-ast inline))) inline-coll) - [(raw-text "}")])) - -(defn- inline-superscript - [inline-coll] - (concatv [(raw-text "^{")] - (mapcatv (fn [inline] (cons space (inline-ast->simple-ast inline))) inline-coll) - [(raw-text "}")])) - -(defn- inline-footnote-reference - [{name :name}] - [(raw-text "[" name "]")]) - -(defn- inline-cookie - [ast-content] - [(raw-text - (case (first ast-content) - "Absolute" - (let [[_ current total] ast-content] - (str "[" current "/" total "]")) - "Percent" - (str "[" (second ast-content) "%]")))]) - -(defn- inline-latex-fragment - [ast-content] - (let [[type content] ast-content - wrapper (case type - "Inline" "$" - "Displayed" "$$")] - [space (raw-text (str wrapper content wrapper)) space])) - -(defn- inline-macro - [{:keys [name arguments]}] - (-> - (if (= name "cloze") - (string/join "," arguments) - (let [l (cond-> ["{{" name] - (pos? (count arguments)) (conj "(" (string/join "," arguments) ")") - true (conj "}}"))] - (string/join l))) - raw-text - vector)) - -(defn- inline-entity - [{unicode :unicode}] - [(raw-text unicode)]) - -(defn- inline-timestamp - [ast-content] - (let [[type timestamp-content] ast-content] - (-> (case type - "Scheduled" ["SCHEDULED: " (common/timestamp-to-string timestamp-content)] - "Deadline" ["DEADLINE: " (common/timestamp-to-string timestamp-content)] - "Date" [(common/timestamp-to-string timestamp-content)] - "Closed" ["CLOSED: " (common/timestamp-to-string timestamp-content)] - "Clock" ["CLOCK: " (common/timestamp-to-string (second timestamp-content))] - "Range" (let [{:keys [start stop]} timestamp-content] - [(str (common/timestamp-to-string start) "--" (common/timestamp-to-string stop))])) - string/join - raw-text - vector))) - -(defn- inline-email - [{:keys [local_part domain]}] - [(raw-text (str "<" local_part "@" domain ">"))]) - -(defn- emphasis-wrap-with - [inline-coll em-symbol] - (binding [*state* (assoc *state* :outside-em-symbol (first em-symbol))] - (concatv [(raw-text em-symbol)] - (mapcatv inline-ast->simple-ast inline-coll) - [(raw-text em-symbol)]))) - -(defn- inline-emphasis - [emphasis] - (let [[[type] inline-coll] emphasis - outside-em-symbol (:outside-em-symbol *state*)] - (case type - "Bold" - (emphasis-wrap-with inline-coll (if (= outside-em-symbol "*") "__" "**")) - "Italic" - (emphasis-wrap-with inline-coll (if (= outside-em-symbol "*") "_" "*")) - "Underline" - (binding [*state* (assoc *state* :outside-em-symbol outside-em-symbol)] - (mapcatv (fn [inline] (cons space (inline-ast->simple-ast inline))) inline-coll)) - "Strike_through" - (emphasis-wrap-with inline-coll "~~") - "Highlight" - (emphasis-wrap-with inline-coll "^^") - ;; else - (assert false (print-str :inline-emphasis emphasis "is invalid"))))) - -(defn- inline-break-line - [] - [(if (= "no-indent" (get-in *state* [:export-options :indent-style])) - (raw-text "\n") - (raw-text " \n")) - (when (:indent-after-break-line? *state*) - (let [current-level (get *state* :current-level 1)] - (when (> current-level 1) - (indent-with-2-spaces (dec current-level)))))]) - -;; {:malli/schema ...} only works on public vars, so use m/=> here -(m/=> block-ast->simple-ast [:=> [:cat mldoc-schema/block-ast-schema] [:sequential simple-ast-malli-schema]]) -(defn- block-ast->simple-ast - [block] - (let [newline-after-block? (get-in *state* [:export-options :newline-after-block])] - (removev - nil? - (let [[ast-type ast-content] block] - (case ast-type - "Paragraph" - (let [{:keys [origin-ast]} (meta block) - current-block-is-first-heading-block? (get-in *state* [:newline-after-block :current-block-is-first-heading-block?])] - (set! *state* (assoc-in *state* [:newline-after-block :current-block-is-first-heading-block?] false)) - (concatv - (when (and origin-ast newline-after-block? (not current-block-is-first-heading-block?)) - [(newline* 2)]) - (mapcatv inline-ast->simple-ast ast-content) - (let [last-element (last ast-content) - [last-element-type] last-element] - (when (and newline-after-block? (= "Break_Line" last-element-type)) - (inline-break-line))) - [(newline* 1)])) - "Paragraph_line" - (assert false "Paragraph_line is mldoc internal ast") - "Paragraph_Sep" - [(newline* ast-content)] - "Heading" - (block-heading ast-content) - "List" - (block-list ast-content) - ("Directive" "Results" "Export" "CommentBlock" "Custom") - nil - "Example" - (block-example ast-content) - "Src" - (block-src ast-content) - "Quote" - (block-quote ast-content) - "Latex_Fragment" - (block-latex-fragment ast-content) - "Latex_Environment" - (block-latex-env (rest block)) - "Displayed_Math" - (block-displayed-math ast-content) - "Drawer" - (block-drawer (rest block)) - "Property_Drawer" - (block-property-drawer ast-content) - "Footnote_Definition" - (block-footnote-definition (rest block)) - "Horizontal_Rule" - block-horizontal-rule - "Table" - (block-table ast-content) - "Comment" - (block-comment ast-content) - "Raw_Html" - (block-raw-html ast-content) - "Hiccup" - (block-hiccup ast-content) - (assert false (print-str :block-ast->simple-ast ast-type "not implemented yet"))))))) - -(defn- inline-ast->simple-ast - [inline] - (let [[ast-type ast-content] inline] - (case ast-type - "Emphasis" - (inline-emphasis ast-content) - ("Break_Line" "Hard_Break_Line") - (inline-break-line) - "Verbatim" - [(raw-text ast-content)] - "Code" - [(raw-text "`" ast-content "`")] - "Tag" - [(raw-text (str "#" (common/hashtag-value->string ast-content)))] - "Spaces" ; what's this ast-type for ? - nil - "Plain" - [(raw-text ast-content)] - "Link" - (inline-link ast-content) - "Nested_link" - (inline-nested-link ast-content) - "Target" - [(raw-text (str "<<" ast-content ">>"))] - "Subscript" - (inline-subscript ast-content) - "Superscript" - (inline-superscript ast-content) - "Footnote_Reference" - (inline-footnote-reference ast-content) - "Cookie" - (inline-cookie ast-content) - "Latex_Fragment" - (inline-latex-fragment ast-content) - "Macro" - (inline-macro ast-content) - "Entity" - (inline-entity ast-content) - "Timestamp" - (inline-timestamp ast-content) - "Radio_Target" - [(raw-text (str "<<<" ast-content ">>>"))] - "Email" - (inline-email ast-content) - "Inline_Hiccup" - [(raw-text ast-content)] - "Inline_Html" - [(raw-text ast-content)] - ("Export_Snippet" "Inline_Source_Block") - nil - (assert false (print-str :inline-ast->simple-ast ast-type "not implemented yet"))))) - -;;; block-ast, inline-ast -> simple-ast (ends) - ;;; export fns -(defn- export-helper - [content format options] - (let [remove-options (set (:remove-options options)) - other-options (:other-options options)] - (binding [*state* (merge *state* - {:export-options - {:indent-style (or (:indent-style options) "dashes") - :remove-emphasis? (contains? remove-options :emphasis) - :remove-page-ref-brackets? (contains? remove-options :page-ref) - :remove-tags? (contains? remove-options :tag) - :remove-properties? (contains? remove-options :property) - :keep-only-level<=N (:keep-only-level<=N other-options) - :newline-after-block (:newline-after-block other-options)}})] - (let [ast (mldoc/->edn content format) - ast (mapv common/remove-block-ast-pos ast) - ast (removev common/Properties-block-ast? ast) - ast* (common/replace-block&page-reference&embed ast) - keep-level<=n (get-in *state* [:export-options :keep-only-level<=N]) - ast* (if (pos? keep-level<=n) - (common/keep-only-level<=n ast* keep-level<=n) - ast*) - ast** (if (= "no-indent" (get-in *state* [:export-options :indent-style])) - (mapv common/replace-Heading-with-Paragraph ast*) - ast*) - config-for-walk-block-ast (cond-> {} - (get-in *state* [:export-options :remove-emphasis?]) - (update :mapcat-fns-on-inline-ast conj common/remove-emphasis) - - (get-in *state* [:export-options :remove-page-ref-brackets?]) - (update :map-fns-on-inline-ast conj common/remove-page-ref-brackets) - - (get-in *state* [:export-options :remove-tags?]) - (update :mapcat-fns-on-inline-ast conj common/remove-tags) - - (= "no-indent" (get-in *state* [:export-options :indent-style])) - (update :fns-on-inline-coll conj common/remove-prefix-spaces-in-Plain)) - ast*** (if-not (empty? config-for-walk-block-ast) - (mapv (partial common/walk-block-ast config-for-walk-block-ast) ast**) - ast**) - simple-asts (mapcatv block-ast->simple-ast ast***)] - (simple-asts->string simple-asts))))) - (defn export-blocks-as-markdown "options: :indent-style \"dashes\" | \"spaces\" | \"no-indent\" @@ -512,34 +26,44 @@ {:pre [(or (coll? root-block-uuids-or-page-uuid) (uuid? root-block-uuids-or-page-uuid))]} (util/profile - :export-blocks-as-markdown - (try - (let [content - (cond + :export-blocks-as-markdown + (try + (let [content + (cond ;; page - (and (= 1 (count root-block-uuids-or-page-uuid)) - (ldb/page? (db/entity [:block/uuid (first root-block-uuids-or-page-uuid)]))) - (common/get-page-content (first root-block-uuids-or-page-uuid)) - (and (coll? root-block-uuids-or-page-uuid) (every? #(ldb/page? (db/entity [:block/uuid %])) root-block-uuids-or-page-uuid)) - (->> (mapv (fn [id] (:block/title (db/entity [:block/uuid id]))) root-block-uuids-or-page-uuid) - (string/join "\n")) - :else - (common/root-block-uuids->content repo root-block-uuids-or-page-uuid)) - first-block (and (coll? root-block-uuids-or-page-uuid) - (db/entity [:block/uuid (first root-block-uuids-or-page-uuid)])) - format (get first-block :block/format :markdown)] - (export-helper content format options)) - (catch :default e - (js/console.error e))))) + (and (= 1 (count root-block-uuids-or-page-uuid)) + (ldb/page? (db/entity [:block/uuid (first root-block-uuids-or-page-uuid)]))) + (common/get-page-content (first root-block-uuids-or-page-uuid)) + (and (coll? root-block-uuids-or-page-uuid) (every? #(ldb/page? (db/entity [:block/uuid %])) root-block-uuids-or-page-uuid)) + (->> (mapv (fn [id] (:block/title (db/entity [:block/uuid id]))) root-block-uuids-or-page-uuid) + (string/join "\n")) + :else + (common/root-block-uuids->content repo root-block-uuids-or-page-uuid)) + first-block (and (coll? root-block-uuids-or-page-uuid) + (db/entity [:block/uuid (first root-block-uuids-or-page-uuid)])) + format (get first-block :block/format :markdown)] + (binding [cli-export-common/*current-db* (conn/get-db repo) + cli-export-common/*current-repo* repo + cli-export-common/*content-config* (common/get-content-config)] + (cli-export-text/export-helper repo content format options))) + (catch :default e + (js/console.error e))))) (defn export-files-as-markdown "options see also `export-blocks-as-markdown`" [files options] - (mapv - (fn [{:keys [path title content]}] - (util/profile (print-str :export-files-as-markdown title) - [(or path title) (export-helper content :markdown options)])) - files)) + (let [repo (state/get-current-repo) + db (conn/get-db repo) + content-config (common/get-content-config)] + (mapv + (fn [{:keys [path title content]}] + (util/profile (print-str :export-files-as-markdown title) + [(or path title) + (binding [cli-export-common/*current-db* db + cli-export-common/*current-repo* repo + cli-export-common/*content-config* content-config] + (cli-export-text/export-helper repo content :markdown options))])) + files))) (defn export-repo-as-markdown! "TODO: indent-style and remove-options" diff --git a/src/main/frontend/modules/file/core.cljs b/src/main/frontend/modules/file/core.cljs deleted file mode 100644 index 78e99fa472..0000000000 --- a/src/main/frontend/modules/file/core.cljs +++ /dev/null @@ -1,14 +0,0 @@ -(ns frontend.modules.file.core - "Convert block trees to content" - (:require [frontend.common.file.core :as common-file] - [frontend.state :as state] - [frontend.db :as db])) - -;; TODO: remove this file and move export related code to worker - -(defn tree->file-content - [tree opts] - (when-let [repo (state/get-current-repo)] - (let [db (db/get-db repo) - context {:export-bullet-indentation (state/get-export-bullet-indentation)}] - (common-file/tree->file-content repo db tree opts context)))) diff --git a/src/main/frontend/util.cljc b/src/main/frontend/util.cljc index 37a1409675..17af5879d2 100644 --- a/src/main/frontend/util.cljc +++ b/src/main/frontend/util.cljc @@ -1448,21 +1448,6 @@ Arg *stop: atom, reset to true to stop the loop" (async/content - [repo db options] - (let [filter-fn (if (ldb/db-based-graph? db) - (fn [ent] - (or (not (:logseq.property/built-in? ent)) - (contains? sqlite-create-graph/built-in-pages-names (:block/title ent)))) - (constantly true))] - (->> (d/datoms db :avet :block/name) - (map #(d/entity db (:e %))) - (filter filter-fn) - (map (fn [e] - [(:block/title e) - (common-file/block->content repo db (:block/uuid e) {} options)]))))) +(def get-all-page->content common-file/get-all-page->content) (defn get-debug-datoms [conn] diff --git a/src/main/frontend/worker/file.cljs b/src/main/frontend/worker/file.cljs index 4ed8d9d738..badc0e9cb1 100644 --- a/src/main/frontend/worker/file.cljs +++ b/src/main/frontend/worker/file.cljs @@ -7,7 +7,7 @@ [clojure.string :as string] [datascript.core :as d] [frontend.common.async-util :as async-util] - [frontend.common.file.core :as common-file] + [logseq.cli.common.file :as common-file] [frontend.common.file.util :as wfu] [frontend.worker-common.util :as worker-util] [frontend.worker.state :as worker-state]