clone/logseq
1
0
Fork 0
mirror of https://github.com/logseq/logseq.git synced 2026-04-27 15:45:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feat/ai-lab

Browse Source
This commit is contained in:
Tienson Qin
2023-05-18 15:45:52 +08:00
parent d2cf025860 c917f2e087
commit bcf35b76d4
114 changed files with 3139 additions and 803 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
deps/graph-parser/.carve/ignore vendored
View File

@@ -46,3 +46,7 @@ logseq.graph-parser.text/get-file-basename
logseq.graph-parser.mldoc/mldoc-link?
;; public var
logseq.graph-parser.schema.mldoc/block-ast-coll-schema
;; API
logseq.graph-parser.config/img-formats
;; API
logseq.graph-parser.config/text-formats

1
deps/graph-parser/src/logseq/graph_parser.cljs vendored
View File

@@ -89,7 +89,6 @@ Options available:
{:keys [tx ast]}
(let [extract-options' (merge {:block-pattern (gp-config/get-block-pattern format)
:date-formatter "MMM do, yyyy"
:supported-formats (gp-config/supported-formats)
:uri-encoded? false
:filename-format :legacy}
extract-options

44
deps/graph-parser/src/logseq/graph_parser/block.cljs vendored
View File

@@ -35,40 +35,32 @@
(string/join))))
(defn- get-page-reference
[block supported-formats]
[block format]
(let [page (cond
(and (vector? block) (= "Link" (first block)))
(let [typ (first (:url (second block)))
(let [url-type (first (:url (second block)))
value (second (:url (second block)))]
;; {:url ["File" "file:../pages/hello_world.org"], :label [["Plain" "hello world"]], :title nil}
(or
(and
(= typ "Page_ref")
(= url-type "Page_ref")
(and (string? value)
(not (or (gp-config/local-asset? value)
(gp-config/draw? value))))
value)
(and
(= typ "Search")
(= url-type "Search")
(page-ref/page-ref? value)
(text/page-ref-un-brackets! value))
(and
(= typ "Search")
(not (contains? #{\# \* \/ \[} (first value)))
;; FIXME: use `gp-util/get-format` instead
(let [ext (some-> (gp-util/get-file-ext value) keyword)]
(when (and (not (string/starts-with? value "http:"))
(not (string/starts-with? value "https:"))
(not (string/starts-with? value "file:"))
(not (gp-config/local-asset? value))
(or (#{:excalidraw :tldr} ext)
(not (contains? supported-formats ext))))
value)))
(and (= url-type "Search")
(= format :org)
(not (gp-config/local-asset? value))
value)
(and
(= typ "File")
(= url-type "File")
(second (first (:label (second block)))))))
(and (vector? block) (= "Nested_link" (first block)))
@@ -329,7 +321,7 @@
nil))
(defn- with-page-refs
[{:keys [title body tags refs marker priority] :as block} with-id? supported-formats db date-formatter]
[{:keys [title body tags refs marker priority] :as block} with-id? db date-formatter]
(let [refs (->> (concat tags refs [marker priority])
(remove string/blank?)
(distinct))
@@ -340,7 +332,7 @@
(when-not (and (vector? form)
(= (first form) "Custom")
(= (second form) "query"))
(when-let [page (get-page-reference form supported-formats)]
(when-let [page (get-page-reference form (:format block))]
(swap! *refs conj page))
(when-let [tag (get-tag form)]
(let [tag (text/page-ref-un-brackets! tag)]
@@ -431,9 +423,9 @@
(map (fn [page] (page-name->map page true db true date-formatter)) page-refs)))
(defn- with-page-block-refs
[block with-id? supported-formats db date-formatter]
[block with-id? db date-formatter]
(some-> block
(with-page-refs with-id? supported-formats db date-formatter)
(with-page-refs with-id? db date-formatter)
with-block-refs
block-tags->pages
(update :refs (fn [col] (remove nil? col)))))
@@ -507,7 +499,7 @@
(mapv macro->block @*result)))
(defn with-pre-block-if-exists
[blocks body pre-block-properties encoded-content {:keys [supported-formats db date-formatter user-config]}]
[blocks body pre-block-properties encoded-content {:keys [db date-formatter user-config]}]
(let [first-block (first blocks)
first-block-start-pos (get-in first-block [:block/meta :start_pos])
@@ -539,7 +531,7 @@
:block/macros (extract-macros-from-ast body)
:block/body body}
{:keys [tags refs]}
(with-page-block-refs {:body body :refs property-refs} false supported-formats db date-formatter)]
(with-page-block-refs {:body body :refs property-refs} false db date-formatter)]
(cond-> block
tags
(assoc :block/tags tags)
@@ -557,7 +549,7 @@
properties))
(defn- construct-block
[block properties timestamps body encoded-content format pos-meta with-id? {:keys [block-pattern supported-formats db date-formatter]}]
[block properties timestamps body encoded-content format pos-meta with-id? {:keys [block-pattern db date-formatter]}]
(let [id (get-custom-id-or-new-id properties)
ref-pages-in-properties (->> (:page-refs properties)
(remove string/blank?))
@@ -594,7 +586,7 @@
(merge block (timestamps->scheduled-and-deadline timestamps))
block)
block (assoc block :body body)
block (with-page-block-refs block with-id? supported-formats db date-formatter)
block (with-page-block-refs block with-id? db date-formatter)
block (update block :refs concat (:block-refs properties))
{:keys [created-at updated-at]} (:properties properties)
block (cond-> block
@@ -648,7 +640,7 @@
`content`: markdown or org-mode text.
`with-id?`: If `with-id?` equals to true, all the referenced pages will have new db ids.
`format`: content's format, it could be either :markdown or :org-mode.
`options`: Options supported are :user-config, :block-pattern :supported-formats,
`options`: Options supported are :user-config, :block-pattern,
:extract-macros, :date-formatter, :page-name and :db"
[blocks content with-id? format {:keys [user-config] :as options}]
{:pre [(seq blocks) (string? content) (boolean? with-id?) (contains? #{:markdown :org} format)]}

1
deps/graph-parser/src/logseq/graph_parser/cli.cljs vendored
View File

@@ -48,7 +48,6 @@ TODO: Fail fast when process exits 1"
[conn files {:keys [config] :as options}]
(let [extract-options (merge {:date-formatter (gp-config/get-date-formatter config)
:user-config config
:supported-formats (gp-config/supported-formats)
:filename-format (or (:file/name-format config) :legacy)
:extracted-block-ids (atom #{})}
(select-keys options [:verbose]))]

8
deps/graph-parser/src/logseq/graph_parser/config.cljs vendored
View File

@@ -1,7 +1,6 @@
(ns logseq.graph-parser.config
"App config that is shared between graph-parser and rest of app"
(:require [clojure.set :as set]
[clojure.string :as string]
(:require [clojure.string :as string]
[goog.object :as gobj]))
(def app-name
@@ -75,11 +74,6 @@
[]
#{:gif :svg :jpeg :ico :png :jpg :bmp :webp})
(defn supported-formats
[]
(set/union (text-formats)
(img-formats)))
(defn get-date-formatter
[config]
(or

7
deps/graph-parser/src/logseq/graph_parser/property.cljs vendored
View File

@@ -51,7 +51,12 @@
"Properties used by logseq that user can edit"
[]
(into #{:title :icon :template :template-including-parent :public :filters :exclude-from-graph-view
:logseq.query/nlp-date
:logseq.query/nlp-date
;; view props
:logseq.color
;; table props
:logseq.table.version :logseq.table.compact :logseq.table.headers :logseq.table.hover
:logseq.table.borders :logseq.table.stripes :logseq.table.max-width
;; org-mode only
:macro :filetags}
editable-linkable-built-in-properties))

54
deps/graph-parser/test/logseq/graph_parser_test.cljs vendored
View File

@@ -117,7 +117,7 @@
:where
[?b :block/name ?name]
[?b :block/type "whiteboard"]]
@conn)]
@conn)]
(is (= pages #{["foo"] ["bar"]}))))))
(defn- test-property-order [num-properties]
@@ -153,11 +153,11 @@
"- desc:: \"#foo is not a ref\""
{:extract-options {:user-config {}}})
block (->> (d/q '[:find (pull ?b [* {:block/refs [*]}])
:in $
:where [?b :block/properties]]
@conn)
(map first)
first)]
:in $
:where [?b :block/properties]]
@conn)
(map first)
first)]
(is (= {:desc "\"#foo is not a ref\""}
(:block/properties block))
"Quoted value is unparsed")
@@ -274,9 +274,9 @@
set)
(set refs))
; pre-block/page has expected refs
db-properties (first (map :block/refs blocks))
db-properties (first (map :block/refs blocks))
;; block has expected refs
block-db-properties (second (map :block/refs blocks))))))
block-db-properties (second (map :block/refs blocks))))))
(deftest property-relationships
(let [properties {:single-link "[[bar]]"
@@ -324,7 +324,7 @@
(map #(select-keys % [:block/properties :block/invalid-properties]))))
"Has correct (in)valid page properties")))
(deftest correct-page-names-created
(deftest correct-page-names-created-from-title
(testing "from title"
(let [conn (ldb/start-conn)
built-in-pages (set (map string/lower-case default-db/built-in-pages-names))]
@@ -358,17 +358,41 @@
@conn)
(map (comp :block/original-name first))
(remove built-in-pages)
set)))))
set))))))
(testing "for file and web uris"
(deftest correct-page-names-created-from-page-refs
(testing "for file, mailto, web and other uris in markdown"
(let [conn (ldb/start-conn)
built-in-pages (set (map string/lower-case default-db/built-in-pages-names))]
(graph-parser/parse-file conn
"foo.md"
(str "- [Filename.txt](file:///E:/test/Filename.txt)\n"
"- [example](https://example.com)")
{})
(is (= #{"foo"}
(str "- [title]([[bar]])\n"
;; all of the uris below do not create pages
"- ![image.png](../assets/image_1630480711363_0.png)\n"
"- [Filename.txt](file:///E:/test/Filename.txt)\n"
"- [mail](mailto:test@test.com?subject=TestSubject)\n"
"- [onenote link](onenote:https://d.docs.live.net/b2127346582e6386a/blablabla/blablabla/blablabla%20blablabla.one#Etat%202019&section-id={133DDF16-9A1F-4815-9A05-44303784442E6F94}&page-id={3AAB677F0B-328F-41D0-AFF5-66408819C085}&end)\n"
"- [lock file](deps/graph-parser/yarn.lock)"
"- [example](https://example.com)"))
(is (= #{"foo" "bar"}
(->> (d/q '[:find (pull ?b [*])
:in $
:where [?b :block/name]]
@conn)
(map (comp :block/name first))
(remove built-in-pages)
set)))))
(testing "for web and page uris in org"
(let [conn (ldb/start-conn)
built-in-pages (set (map string/lower-case default-db/built-in-pages-names))]
(graph-parser/parse-file conn
"foo.org"
(str "* [[bar][title]]\n"
;; all of the uris below do not create pages
"* [[https://example.com][example]]\n"
"* [[../assets/conga_parrot.gif][conga]]"))
(is (= #{"foo" "bar"}
(->> (d/q '[:find (pull ?b [*])
:in $
:where [?b :block/name]]