clone/logseq
1
0
Fork 0
mirror of https://github.com/logseq/logseq.git synced 2026-04-24 22:25:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into refactor/page-parent

Browse Source
This commit is contained in:
Tienson Qin
2025-06-17 19:44:34 +08:00
parent 3759046885 3b718f70c8
commit 7b46076170
33 changed files with 496 additions and 221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
deps/db/.carve/config.edn vendored
View File

@@ -9,6 +9,7 @@
logseq.db.common.order
logseq.db.sqlite.create-graph
logseq.db.frontend.malli-schema
logseq.db.frontend.asset
;; Some fns are used by frontend but not worth moving over yet
logseq.db.frontend.schema
logseq.db.frontend.validate

2
deps/db/src/logseq/db/common/entity_plus.cljc vendored
View File

@@ -22,7 +22,7 @@
:block/pre-block? :block/scheduled :block/deadline :block/type :block/name :block/marker
:block.temp/ast-title
:block.temp/fully-loaded? :block.temp/has-children? :block.temp/ast-body
:block.temp/load-status :block.temp/has-children? :block.temp/ast-body
:db/valueType :db/cardinality :db/ident :db/index

150
deps/db/src/logseq/db/common/initial_data.cljs vendored
View File

@@ -1,7 +1,6 @@
(ns logseq.db.common.initial-data
"Provides db helper fns for graph initialization and lazy loading entities"
(:require [clojure.set :as set]
[clojure.string :as string]
(:require [clojure.string :as string]
[datascript.core :as d]
[datascript.impl.entity :as de]
[logseq.common.config :as common-config]
@@ -87,10 +86,6 @@
(update block :block/link (fn [link] (d/pull db '[*] (:db/id link))))
block)))
(defn- mark-block-fully-loaded
[b]
(assoc b :block.temp/fully-loaded? true))
(comment
(defn- property-without-db-attrs
[property]
@@ -133,27 +128,30 @@
(defn get-block-children-ids
"Returns children UUIDs"
[db block-uuid]
[db block-uuid & {:keys [include-collapsed-children?]
:or {include-collapsed-children? true}}]
(when-let [eid (:db/id (d/entity db [:block/uuid block-uuid]))]
(let [seen (volatile! [])]
(loop [steps 100 ;check result every 100 steps
eids-to-expand [eid]]
(let [seen (volatile! #{})]
(loop [eids-to-expand [eid]]
(when (seq eids-to-expand)
(let [eids-to-expand*
(mapcat (fn [eid] (map first (d/datoms db :avet :block/parent eid))) eids-to-expand)
uuids-to-add (remove nil? (map #(:block/uuid (d/entity db %)) eids-to-expand*))]
(when (and (zero? steps)
(seq (set/intersection (set @seen) (set uuids-to-add))))
(throw (ex-info "bad outliner data, need to re-index to fix"
{:seen @seen :eids-to-expand eids-to-expand})))
(let [children
(mapcat (fn [eid]
(let [e (d/entity db eid)]
(when (or include-collapsed-children?
(not (:block/collapsed? e))
(common-entity-util/page? e))
(:block/_parent e)))) eids-to-expand)
uuids-to-add (keep :block/uuid children)]
(vswap! seen (partial apply conj) uuids-to-add)
(recur (if (zero? steps) 100 (dec steps)) eids-to-expand*))))
(recur (keep :db/id children)))))
@seen)))
(defn get-block-children
"Including nested children."
[db block-uuid]
(let [ids (get-block-children-ids db block-uuid)]
{:arglists '([db block-uuid & {:keys [include-collapsed-children?]}])}
[db block-uuid & {:as opts}]
(let [ids (get-block-children-ids db block-uuid opts)]
(when (seq ids)
(map (fn [id] (d/entity db [:block/uuid id])) ids))))
@@ -164,10 +162,36 @@
m))
(defn- entity->map
[entity]
(-> (into {} entity)
(with-raw-title entity)
(assoc :db/id (:db/id entity))))
[entity & {:keys [level]
:or {level 0}}]
(let [opts {:level (inc level)}
f (if (> level 0)
identity
(fn [e]
(keep (fn [[k v]]
(when-not (contains? #{:block/path-refs} k)
(let [v' (cond
(= k :block/parent)
(:db/id v)
(= k :block/tags)
(set (map :db/id v))
(= k :logseq.property/created-by-ref)
(:db/id v)
(= k :block/refs)
(map #(select-keys % [:db/id :block/uuid :block/title]) v)
(de/entity? v)
(entity->map v opts)
(and (coll? v) (every? de/entity? v))
(map #(entity->map % opts) v)
:else
v)]
[k v'])))
e)))
m (->> (f entity)
(into {}))]
(-> m
(with-raw-title entity)
(assoc :db/id (:db/id entity)))))
(defn hidden-ref?
"Whether ref-block (for block with the `id`) should be hidden."
@@ -204,7 +228,8 @@
0))
(defn ^:large-vars/cleanup-todo get-block-and-children
[db id-or-page-name {:keys [children? children-only? nested-children? properties children-props]}]
[db id-or-page-name {:keys [children? properties include-collapsed-children?]
:or {include-collapsed-children? false}}]
(let [block (let [eid (cond (uuid? id-or-page-name)
[:block/uuid id-or-page-name]
(integer? id-or-page-name)
@@ -217,64 +242,29 @@
(d/entity db (get-first-page-by-name db (name id-or-page-name)))
:else
nil))
block-refs-count? (some #{:block.temp/refs-count} properties)
whiteboard? (common-entity-util/whiteboard? block)]
block-refs-count? (some #{:block.temp/refs-count} properties)]
(when block
(let [children (when (or children? children-only?)
(let [page? (common-entity-util/page? block)
children (->>
(cond
(and nested-children? (not page?))
(get-block-children db (:block/uuid block))
nested-children?
(:block/_page block)
:else
(let [short-page? (when page?
(<= (count (:block/_page block)) 100))]
(if short-page?
(:block/_page block)
(:block/_parent block))))
(remove (fn [e] (or (:logseq.property/created-from-property e)
(:block/closed-value-property e)))))
children-props (if whiteboard?
'[*]
(or children-props
[:db/id :block/uuid :block/parent :block/order :block/collapsed? :block/title
;; pre-loading feature-related properties to avoid UI refreshing
:logseq.property/status :logseq.property.node/display-type]))]
(let [children (when children?
(let [children (let [children (get-block-children db (:block/uuid block) {:include-collapsed-children? include-collapsed-children?})
children' (if (>= (count children) 100)
(:block/_parent block)
children)]
(->> children'
(remove (fn [e] (:block/closed-value-property e)))))]
(map
(fn [block]
(if (= children-props '[*])
(entity->map block)
(-> (select-keys block children-props)
(with-raw-title block)
(assoc :block.temp/has-children? (some? (:block/_parent block))))))
children)))]
(if children-only?
{:children children}
(let [block' (if (seq properties)
(-> (select-keys block properties)
(with-raw-title block)
(assoc :db/id (:db/id block)))
(entity->map block))
block' (cond->
(mark-block-fully-loaded block')
true
(update-vals (fn [v]
(cond
(de/entity? v)
(entity->map v)
(and (coll? v) (every? de/entity? v))
(map entity->map v)
:else
v)))
block-refs-count?
(assoc :block.temp/refs-count (get-block-refs-count db (:db/id block))))]
(cond->
{:block block'}
children?
(assoc :children children))))))))
(-> block
(assoc :block.temp/has-children? (some? (:block/_parent block)))
(dissoc :block/tx-id :block/created-at :block/updated-at)
entity->map))
children)))
block' (cond-> (entity->map block)
block-refs-count?
(assoc :block.temp/refs-count (get-block-refs-count db (:db/id block))))]
(cond->
{:block block'}
children?
(assoc :children children))))))
(defn get-latest-journals
[db]

27
deps/db/src/logseq/db/frontend/asset.cljs vendored Normal file
View File

@@ -0,0 +1,27 @@
(ns logseq.db.frontend.asset
"Asset fns used in node and browser contexts"
(:require ["path" :as node-path]
[clojure.string :as string]))
(defn- decode-digest
[^js/Uint8Array digest]
(.. (js/Array.from digest)
(map (fn [s] (.. s (toString 16) (padStart 2 "0"))))
(join "")))
(defn <get-file-array-buffer-checksum
"Given a file's ArrayBuffer, returns its checksum in a promise"
[file-array-buffer]
(-> (js/crypto.subtle.digest "SHA-256" file-array-buffer)
(.then (fn [dig] (js/Uint8Array. dig)))
(.then decode-digest)))
(defn asset-path->type
"Create asset type given asset path"
[path]
(string/lower-case (.substr (node-path/extname path) 1)))
(defn asset-name->title
"Create asset title given asset path's basename"
[path-basename]
(.-name (node-path/parse path-basename)))

36
deps/db/src/logseq/db/frontend/malli_schema.cljs vendored
View File

@@ -40,6 +40,11 @@
{:error/message "should be a valid user property namespace"}
user-property?]])
(def plugin-property-ident
[:and :qualified-keyword [:fn
{:error/message "should be a valid plugin property namespace"}
db-property/plugin-property?]])
(def logseq-ident-namespaces
"Set of all namespaces Logseq uses for :db/ident except for
db-attribute-ident. It's important to grow this list purposefully and have it
@@ -315,19 +320,40 @@
(vec
(concat
[:map
;; class-ident allows for a class to be used as a property
[:db/ident [:or user-property-ident class-ident]]
[:db/ident user-property-ident]
[:logseq.property/type (apply vector :enum db-property-type/user-built-in-property-types)]]
property-common-schema-attrs
property-attrs
page-attrs
page-or-block-attrs)))
(def plugin-property
(vec
(concat
[:map
[:db/ident plugin-property-ident]
[:logseq.property/type (apply vector :enum (conj db-property-type/user-built-in-property-types :string))]]
property-common-schema-attrs
property-attrs
page-attrs
page-or-block-attrs)))
(def property-page
[:multi {:dispatch (fn [m]
(or (some->> (:db/ident m) db-property/logseq-property?)
(contains? db-property/db-attribute-properties (:db/ident m))))}
[true internal-property]
(let [ident (:db/ident m)]
(cond
(or (some->> ident db-property/logseq-property?)
(contains? db-property/db-attribute-properties (:db/ident m)))
:internal
(some->> ident db-property/plugin-property?)
:plugin
:else
:user)))}
[:internal internal-property]
[:plugin plugin-property]
[:malli.core/default user-property]])
(def hidden-page

5
deps/db/src/logseq/db/frontend/property.cljs vendored
View File

@@ -623,6 +623,11 @@
[s]
(string/includes? s ".property"))
(defn plugin-property?
"Determines if keyword is a plugin property"
[kw]
(string/starts-with? (namespace kw) "plugin.property."))
(defn internal-property?
"Determines if ident kw is an internal property. This includes db-attribute properties
unlike logseq-property? and doesn't include non-property idents unlike internal-ident?"

2
deps/db/src/logseq/db/frontend/validate.cljs vendored
View File

@@ -97,7 +97,7 @@
ent-maps* (db-malli-schema/datoms->entities datoms)
ent-maps (mapv
;; Remove some UI interactions adding this e.g. import
#(dissoc % :block.temp/fully-loaded? :block.temp/has-children?)
#(dissoc % :block.temp/load-status :block.temp/has-children?)
(db-malli-schema/update-properties-in-ents db ent-maps*))
errors (binding [db-malli-schema/*db-for-validate-fns* db]
(-> (map (fn [e]

33
deps/db/test/logseq/db_test.cljs vendored
View File

@@ -2,31 +2,24 @@
(:require [cljs.test :refer [deftest is]]
[datascript.core :as d]
[logseq.db :as ldb]
[logseq.db.file-based.schema :as file-schema]
[logseq.db.test.helper :as db-test]))
;;; datoms
;;; - 1 <----+
;;; - 2 |
;;; - 3 -+
(def broken-outliner-data-with-cycle
[{:db/id 1
:block/uuid #uuid"e538d319-48d4-4a6d-ae70-c03bb55b6fe4"
:block/parent 3}
{:db/id 2
:block/uuid #uuid"c46664c0-ea45-4998-adf0-4c36486bb2e5"
:block/parent 1}
{:db/id 3
:block/uuid #uuid"2b736ac4-fd49-4e04-b00f-48997d2c61a2"
:block/parent 2}])
(deftest get-block-children-ids-on-bad-outliner-data
(let [db (d/db-with (d/empty-db file-schema/schema)
broken-outliner-data-with-cycle)]
(is (= "bad outliner data, need to re-index to fix"
(try (ldb/get-block-children-ids db #uuid "e538d319-48d4-4a6d-ae70-c03bb55b6fe4")
(catch :default e
(ex-message e)))))))
(comment
;; TODO: throw error or fix broken data when cycle detected
(def broken-outliner-data-with-cycle
[{:db/id 1
:block/uuid #uuid"e538d319-48d4-4a6d-ae70-c03bb55b6fe4"
:block/parent 3}
{:db/id 2
:block/uuid #uuid"c46664c0-ea45-4998-adf0-4c36486bb2e5"
:block/parent 1}
{:db/id 3
:block/uuid #uuid"2b736ac4-fd49-4e04-b00f-48997d2c61a2"
:block/parent 2}]))
(def class-parents-data
[{:block/tags :logseq.class/Tag
@@ -82,4 +75,4 @@
"Class pages correctly found for given class")
(is (= nil
(ldb/page-exists? @conn "movie" #{:logseq.class/Property}))
"Class pages correctly not found for given class")))
"Class pages correctly not found for given class")))

31
deps/graph-parser/script/db_import.cljs vendored
View File

@@ -10,8 +10,10 @@
[clojure.set :as set]
[clojure.string :as string]
[datascript.core :as d]
[logseq.common.config :as common-config]
[logseq.common.graph :as common-graph]
[logseq.db.common.sqlite-cli :as sqlite-cli]
[logseq.db.frontend.asset :as db-asset]
[logseq.graph-parser.exporter :as gp-exporter]
[logseq.outliner.cli :as outliner-cli]
[logseq.outliner.pipeline :as outliner-pipeline]
@@ -47,11 +49,25 @@
(p/let [s (fsp/readFile (:path file))]
(str s)))
(defn- <copy-asset-file [file db-graph-dir file-graph-dir]
(p/let [parent-dir (node-path/dirname
(node-path/join db-graph-dir (node-path/relative file-graph-dir (:path file))))
(defn- <read-asset-file [file assets]
(p/let [buffer (fs/readFileSync (:path file))
checksum (db-asset/<get-file-array-buffer-checksum buffer)]
(swap! assets assoc
(gp-exporter/asset-path->name (:path file))
{:size (.-length buffer)
:checksum checksum
:type (db-asset/asset-path->type (:path file))
:path (:path file)})))
(defn- <copy-asset-file [asset-m db-graph-dir]
(p/let [parent-dir (node-path/join db-graph-dir common-config/local-assets-dir)
_ (fsp/mkdir parent-dir #js {:recursive true})]
(fsp/copyFile (:path file) (node-path/join parent-dir (node-path/basename (:path file))))))
(if (:block/uuid asset-m)
(fsp/copyFile (:path asset-m) (node-path/join parent-dir (str (:block/uuid asset-m) "." (:type asset-m))))
(do
(println "[INFO]" "Copied asset" (pr-str (node-path/basename (:path asset-m)))
"by its name since it was unused.")
(fsp/copyFile (:path asset-m) (node-path/join parent-dir (node-path/basename (:path asset-m))))))))
(defn- notify-user [{:keys [continue debug]} m]
(println (:msg m))
@@ -103,7 +119,8 @@
(default-export-options options)
;; asset file options
{:<copy-asset (fn copy-asset [file]
(<copy-asset-file file db-graph-dir file-graph-dir))})]
(<copy-asset-file file db-graph-dir))
:<read-asset <read-asset-file})]
(p/with-redefs [d/transact! dev-transact!]
(gp-exporter/export-file-graph conn conn config-file *files options))))
@@ -184,10 +201,12 @@
(when-let [ignored-props (seq @(:ignored-properties import-state))]
(println "Ignored properties:" (pr-str ignored-props)))
(when-let [ignored-assets (seq @(:ignored-assets import-state))]
(println "Ignored assets:" (pr-str ignored-assets)))
(when-let [ignored-files (seq @(:ignored-files import-state))]
(println (count ignored-files) "ignored file(s):" (pr-str (vec ignored-files))))
(when (:verbose options') (println "Transacted" (count (d/datoms @conn :eavt)) "datoms"))
(println "Created graph" (str db-name "!")))))
(when (= nbb/*file* (nbb/invoked-file))
(-main *command-line-args*))
(-main *command-line-args*))

51
deps/graph-parser/src/logseq/graph_parser/block.cljs vendored
View File

@@ -132,7 +132,7 @@
(when (some-> block-id parse-uuid)
block-id)))
(defn- paragraph-block?
(defn paragraph-block?
[block]
(and
(vector? block)
@@ -711,36 +711,36 @@
(defn extract-blocks
"Extract headings from mldoc ast. Args:
*`blocks`: mldoc ast.
* `content`: markdown or org-mode text.
* `format`: content's format, it could be either :markdown or :org-mode.
* `options`: Options are :user-config, :block-pattern, :parse-block, :date-formatter, :db and
* `ast`: mldoc ast.
* `content`: markdown or org-mode text.
* `format`: content's format, it could be either :markdown or :org-mode.
* `options`: Options are :user-config, :block-pattern, :parse-block, :date-formatter, :db and
* :db-graph-mode? : Set when a db graph in the frontend
* :export-to-db-graph? : Set when exporting to a db graph"
[blocks content format {:keys [user-config db-graph-mode? export-to-db-graph?] :as options}]
{:pre [(seq blocks) (string? content) (contains? #{:markdown :org} format)]}
[ast content format {:keys [user-config db-graph-mode? export-to-db-graph?] :as options}]
{:pre [(seq ast) (string? content) (contains? #{:markdown :org} format)]}
(let [encoded-content (utf8/encode content)
all-blocks (vec (reverse blocks))
all-blocks (vec (reverse ast))
[blocks body pre-block-properties]
(loop [headings []
blocks (reverse blocks)
ast-blocks (reverse ast)
block-idx 0
timestamps {}
properties {}
body []]
(if (seq blocks)
(let [[block pos-meta] (first blocks)]
(if (seq ast-blocks)
(let [[ast-block pos-meta] (first ast-blocks)]
(cond
(paragraph-timestamp-block? block)
(let [timestamps (extract-timestamps block)
(paragraph-timestamp-block? ast-block)
(let [timestamps (extract-timestamps ast-block)
timestamps' (merge timestamps timestamps)]
(recur headings (rest blocks) (inc block-idx) timestamps' properties body))
(recur headings (rest ast-blocks) (inc block-idx) timestamps' properties body))
(gp-property/properties-ast? block)
(let [properties (extract-properties (second block) (assoc user-config :format format))]
(recur headings (rest blocks) (inc block-idx) timestamps properties body))
(gp-property/properties-ast? ast-block)
(let [properties (extract-properties (second ast-block) (assoc user-config :format format))]
(recur headings (rest ast-blocks) (inc block-idx) timestamps properties body))
(heading-block? block)
(heading-block? ast-block)
;; for db-graphs cut multi-line when there is property, deadline/scheduled or logbook text in :block/title
(let [cut-multiline? (and export-to-db-graph?
(when-let [prev-block (first (get all-blocks (dec block-idx)))]
@@ -762,14 +762,19 @@
(and export-to-db-graph?
(and (gp-property/properties-ast? (first (get all-blocks (dec block-idx))))
(= "Custom" (ffirst (get all-blocks (- block-idx 2)))))))
block' (construct-block block properties timestamps body encoded-content format pos-meta' options')
block'' (if (or db-graph-mode? export-to-db-graph?)
block' (construct-block ast-block properties timestamps body encoded-content format pos-meta' options')
block'' (cond
db-graph-mode?
block'
(assoc block' :macros (extract-macros-from-ast (cons block body))))]
(recur (conj headings block'') (rest blocks) (inc block-idx) {} {} []))
export-to-db-graph?
(assoc block' :block.temp/ast-blocks (cons ast-block body))
:else
(assoc block' :macros (extract-macros-from-ast (cons ast-block body))))]
(recur (conj headings block'') (rest ast-blocks) (inc block-idx) {} {} []))
:else
(recur headings (rest blocks) (inc block-idx) timestamps properties (conj body block))))
(recur headings (rest ast-blocks) (inc block-idx) timestamps properties (conj body ast-block))))
[(-> (reverse headings)
sanity-blocks-data)
body

179
deps/graph-parser/src/logseq/graph_parser/exporter.cljs vendored
View File

@@ -1,12 +1,14 @@
(ns logseq.graph-parser.exporter
"Exports a file graph to DB graph. Used by the File to DB graph importer and
by nbb-logseq CLIs"
(:require [borkdude.rewrite-edn :as rewrite]
(:require ["path" :as node-path]
[borkdude.rewrite-edn :as rewrite]
[cljs-time.coerce :as tc]
[cljs.pprint]
[clojure.edn :as edn]
[clojure.set :as set]
[clojure.string :as string]
[clojure.walk :as walk]
[datascript.core :as d]
[logseq.common.config :as common-config]
[logseq.common.path :as path]
@@ -19,6 +21,7 @@
[logseq.db :as ldb]
[logseq.db.common.order :as db-order]
[logseq.db.common.property-util :as db-property-util]
[logseq.db.frontend.asset :as db-asset]
[logseq.db.frontend.class :as db-class]
[logseq.db.frontend.content :as db-content]
[logseq.db.frontend.db-ident :as db-ident]
@@ -889,6 +892,89 @@
(:block/name (:block/parent block))
(assoc :block/parent {:block/uuid (get-page-uuid page-names-to-uuids (:block/name (:block/parent block)) {:block block :block/parent (:block/parent block)})})))
(defn asset-path->name
"Given an asset's relative or full path, create a unique name for identifying an asset.
Must handle to paths as ../assets/*, assets/* and with subdirectories"
[path]
(re-find #"assets/.*$" path))
(defn- find-all-asset-links
"Walks each ast block in order to its full depth as Link asts can be in different
locations e.g. a Heading vs a Paragraph ast block"
[ast-blocks]
(let [results (atom [])]
(walk/prewalk
(fn [x]
(when (and (vector? x)
(= "Link" (first x))
(common-config/local-asset? (second (:url (second x)))))
(swap! results conj x))
x)
ast-blocks)
@results))
(defn- update-asset-links-in-block-title [block-title asset-name-to-uuids ignored-assets]
(reduce (fn [acc [asset-name asset-uuid]]
(let [new-title (string/replace acc
(re-pattern (str "!?\\[[^\\]]*?\\]\\([^\\)]*?"
asset-name
"\\)(\\{[^}]*\\})?"))
(page-ref/->page-ref asset-uuid))]
(when (string/includes? new-title asset-name)
(swap! ignored-assets conj
{:reason "Some asset links were not updated to block references"
:path asset-name
:location {:block new-title}}))
new-title))
block-title
asset-name-to-uuids))
(defn- handle-assets-in-block
[block* {:keys [assets ignored-assets]}]
(let [block (dissoc block* :block.temp/ast-blocks)
asset-links (find-all-asset-links (:block.temp/ast-blocks block*))]
(if (seq asset-links)
(let [asset-maps
(keep
(fn [asset-link]
(let [asset-name (-> asset-link second :url second asset-path->name)]
(if-let [asset-data (and asset-name (get @assets asset-name))]
(if (:block/uuid asset-data)
{:asset-name-uuid [asset-name (:block/uuid asset-data)]}
(let [new-block (sqlite-util/block-with-timestamps
{:block/uuid (d/squuid)
:block/order (db-order/gen-key)
:block/page :logseq.class/Asset
:block/parent :logseq.class/Asset})
new-asset (merge new-block
{:block/tags [:logseq.class/Asset]
:logseq.property.asset/type (:type asset-data)
:logseq.property.asset/checksum (:checksum asset-data)
:logseq.property.asset/size (:size asset-data)
:block/title (db-asset/asset-name->title (node-path/basename asset-name))}
(when-let [metadata (not-empty (common-util/safe-read-map-string (:metadata (second asset-link))))]
{:logseq.property.asset/resize-metadata metadata}))]
;; (prn :asset-added! (node-path/basename asset-name) #_(get @assets asset-name))
;; (cljs.pprint/pprint asset-link)
(swap! assets assoc-in [asset-name :block/uuid] (:block/uuid new-block))
{:asset-name-uuid [asset-name (:block/uuid new-asset)]
:asset new-asset}))
(do
(swap! ignored-assets conj
{:reason "No asset data found for this asset path"
:path (-> asset-link second :url second)
:location {:block (:block/title block)}})
nil))))
asset-links)
asset-blocks (keep :asset asset-maps)
asset-names-to-uuids
(into {} (map :asset-name-uuid asset-maps))]
(cond-> {:block
(update block :block/title update-asset-links-in-block-title asset-names-to-uuids ignored-assets)}
(seq asset-blocks)
(assoc :asset-blocks-tx asset-blocks)))
{:block block})))
(defn- build-block-tx
[db block* pre-blocks {:keys [page-names-to-uuids] :as per-file-state} {:keys [import-state journal-created-ats] :as options}]
;; (prn ::block-in block*)
@@ -896,9 +982,11 @@
{:keys [block properties-tx]}
(handle-block-properties block* db page-names-to-uuids (:block/refs block*) options)
{block-after-built-in-props :block deadline-properties-tx :properties-tx} (update-block-deadline block page-names-to-uuids options)
{block-after-assets :block :keys [asset-blocks-tx]}
(handle-assets-in-block block-after-built-in-props (select-keys import-state [:assets :ignored-assets]))
;; :block/page should be [:block/page NAME]
journal-page-created-at (some-> (:block/page block*) second journal-created-ats)
prepared-block (cond-> block-after-built-in-props
prepared-block (cond-> block-after-assets
journal-page-created-at
(assoc :block/created-at journal-page-created-at))
block' (-> prepared-block
@@ -913,8 +1001,8 @@
(dissoc :block/left :block/format)
;; ((fn [x] (prn :block-out x) x))
)]
;; Order matters as properties are referenced in block
(concat properties-tx deadline-properties-tx [block'])))
;; Order matters as previous txs are referenced in block
(concat properties-tx deadline-properties-tx asset-blocks-tx [block'])))
(defn- update-page-alias
[m page-names-to-uuids]
@@ -1147,6 +1235,8 @@
:ignored-properties (atom [])
;; Vec of maps with keys :path and :reason
:ignored-files (atom [])
;; Vec of maps with keys :path, :reason and :location (optional).
:ignored-assets (atom [])
;; Map of property names (keyword) and their current schemas (map of qualified properties).
;; Used for adding schemas to properties and detecting changes across a property's usage
:property-schemas (atom {})
@@ -1158,7 +1248,9 @@
:classes-from-property-parents (atom #{})
;; Map of block uuids to their :block/properties-text-values value.
;; Used if a property value changes to :default
:block-properties-text-values (atom {})})
:block-properties-text-values (atom {})
;; Track asset data for use across asset and doc import steps
:assets (atom {})})
(defn- build-tx-options [{:keys [user-options] :as options}]
(merge
@@ -1555,32 +1647,58 @@
class-to-prop-uuids)]
(ldb/transact! repo-or-conn tx)))
(defn- export-asset-files
"Exports files under assets/"
[*asset-files <copy-asset-file {:keys [notify-user set-ui-state]
(defn- <safe-async-loop
"Calls async-fn with each element in args-to-loop. Catches an unexpected error in loop and notifies user"
[async-fn args-to-loop notify-user]
(-> (p/loop [_ (async-fn (get args-to-loop 0))
i 0]
(when-not (>= i (dec (count args-to-loop)))
(p/recur (async-fn (get args-to-loop (inc i)))
(inc i))))
(p/catch (fn [e]
(notify-user {:msg (str "Import has an unexpected error:\n" (.-message e))
:level :error
:ex-data {:error e}})))))
(defn- read-asset-files
"Reads files under assets/"
[*asset-files <read-asset-file {:keys [notify-user set-ui-state assets]
:or {set-ui-state (constantly nil)}}]
(assert <read-asset-file "read-asset-file fn required")
(let [asset-files (mapv #(assoc %1 :idx %2)
;; Sort files to ensure reproducible import behavior
(sort-by :path *asset-files)
(range 0 (count *asset-files)))
copy-asset (fn copy-asset [{:keys [path] :as file}]
read-asset (fn read-asset [{:keys [path] :as file}]
(-> (<read-asset-file file assets)
(p/catch
(fn [error]
(notify-user {:msg (str "Import failed to read " (pr-str path) " with error:\n" (.-message error))
:level :error
:ex-data {:path path :error error}})))))]
(when (seq asset-files)
(set-ui-state [:graph/importing-state :current-page] "Read asset files")
(<safe-async-loop read-asset asset-files notify-user))))
(defn- copy-asset-files
"Copy files under assets/"
[asset-maps* <copy-asset-file {:keys [notify-user set-ui-state]
:or {set-ui-state (constantly nil)}}]
(assert <copy-asset-file "copy-asset-file fn required")
(let [asset-maps (mapv #(assoc %1 :idx %2)
;; Sort files to ensure reproducible import behavior
(sort-by :path asset-maps*)
(range 0 (count asset-maps*)))
copy-asset (fn copy-asset [{:keys [path] :as asset-m}]
(p/catch
(<copy-asset-file file)
(<copy-asset-file asset-m)
(fn [error]
(notify-user {:msg (str "Import failed on " (pr-str path) " with error:\n" (.-message error))
(notify-user {:msg (str "Import failed to copy " (pr-str path) " with error:\n" (.-message error))
:level :error
:ex-data {:path path :error error}}))))]
(when (seq asset-files)
(set-ui-state [:graph/importing-state :current-page] "Asset files")
(-> (p/loop [_ (copy-asset (get asset-files 0))
i 0]
(when-not (>= i (dec (count asset-files)))
(p/recur (copy-asset (get asset-files (inc i)))
(inc i))))
(p/catch (fn [e]
(notify-user {:msg (str "Import has an unexpected error:\n" (.-message e))
:level :error
:ex-data {:error e}})))))))
(when (seq asset-maps)
(set-ui-state [:graph/importing-state :current-page] "Copy asset files")
(<safe-async-loop copy-asset asset-maps notify-user))))
(defn- insert-favorites
"Inserts favorited pages as uuids into a new favorite page"
@@ -1612,7 +1730,7 @@
(log-fn :no-favorites-found {:favorites favorites})))))
(defn build-doc-options
"Builds options for use with export-doc-files"
"Builds options for use with export-doc-files and assets"
[config options]
(-> {:extract-options {:date-formatter (common-config/get-date-formatter config)
;; Remove config keys that break importing
@@ -1661,9 +1779,10 @@
* :<save-config-file - fn which saves a config file
* :<save-logseq-file - fn which saves a logseq file
* :<copy-asset - fn which copies asset file
* :<read-asset - fn which reads asset file
Note: See export-doc-files for additional options that are only for it"
[repo-or-conn conn config-file *files {:keys [<read-file <copy-asset rpath-key log-fn]
[repo-or-conn conn config-file *files {:keys [<read-file <copy-asset <read-asset rpath-key log-fn]
:or {rpath-key :path log-fn println}
:as options}]
(reset! gp-block/*export-to-db-graph? true)
@@ -1685,12 +1804,20 @@
(export-logseq-files repo-or-conn (filter logseq-file? files) <read-file
(-> (select-keys options [:notify-user :<save-logseq-file])
(set/rename-keys {:<save-logseq-file :<save-file})))
(export-asset-files asset-files <copy-asset (select-keys options [:notify-user :set-ui-state]))
;; Assets are read first as doc-files need data from them to make Asset blocks.
;; Assets are copied after after doc-files as they need block/uuid's from them to name assets
(read-asset-files asset-files <read-asset (merge (select-keys options [:notify-user :set-ui-state])
{:assets (get-in doc-options [:import-state :assets])}))
(export-doc-files conn doc-files <read-file doc-options)
(copy-asset-files (vals @(get-in doc-options [:import-state :assets]))
<copy-asset
(select-keys options [:notify-user :set-ui-state]))
(export-favorites-from-config-edn conn repo-or-conn config {})
(export-class-properties conn repo-or-conn)
(move-top-parent-pages-to-library conn repo-or-conn)
{:import-state (:import-state doc-options)
{:import-state (-> (:import-state doc-options)
;; don't leak full asset content (which could be large) out of this ns
(dissoc :assets))
:files files})))
(p/finally (fn [_]
(reset! gp-block/*export-to-db-graph? false)))

60
deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs vendored
View File

@@ -1,7 +1,7 @@
(ns ^:node-only logseq.graph-parser.exporter-test
(:require ["fs" :as fs]
["path" :as node-path]
[cljs.test :refer [testing is]]
[cljs.test :refer [testing is are deftest]]
[clojure.set :as set]
[clojure.string :as string]
[datascript.core :as d]
@@ -10,6 +10,7 @@
[logseq.common.util.date-time :as date-time-util]
[logseq.db :as ldb]
[logseq.db.common.entity-plus :as entity-plus]
[logseq.db.frontend.asset :as db-asset]
[logseq.db.frontend.content :as db-content]
[logseq.db.frontend.malli-schema :as db-malli-schema]
[logseq.db.frontend.rules :as rules]
@@ -93,6 +94,17 @@
;; TODO: Add actual default
:default-config {}})
;; Copied from db-import
(defn- <read-asset-file [file assets]
(p/let [buffer (fs/readFileSync (:path file))
checksum (db-asset/<get-file-array-buffer-checksum buffer)]
(swap! assets assoc
(gp-exporter/asset-path->name (:path file))
{:size (.-length buffer)
:checksum checksum
:type (db-asset/asset-path->type (:path file))
:path (:path file)})))
;; Copied from db-import script and tweaked for an in-memory import
(defn- import-file-graph-to-db
"Import a file graph dir just like UI does. However, unlike the UI the
@@ -105,7 +117,12 @@
options' (merge default-export-options
{:user-options (merge {:convert-all-tags? false} (dissoc options :assets :verbose))
;; asset file options
:<copy-asset #(swap! assets conj %)}
:<read-asset <read-asset-file
:<copy-asset (fn copy-asset [m]
(when-not (:block/uuid m)
(println "[INFO]" "Asset" (pr-str (node-path/basename (:path m)))
"does not have a :block/uuid"))
(swap! assets conj m))}
(select-keys options [:verbose]))]
(gp-exporter/export-file-graph conn conn config-file *files options')))
@@ -128,6 +145,24 @@
;; Tests
;; =====
(deftest update-asset-links-in-block-title
(are [x y]
(= y (@#'gp-exporter/update-asset-links-in-block-title (first x) {(second x) "UUID"} (atom {})))
;; Standard image link with metadata
["![greg-popovich-thumbs-up.png](../assets/greg-popovich-thumbs-up_1704749687791_0.png){:height 288, :width 100} says pop"
"assets/greg-popovich-thumbs-up_1704749687791_0.png"]
"[[UUID]] says pop"
;; Image link with no metadata
["![some-title](../assets/CleanShot_2022-10-12_at_15.53.20@2x_1665561216083_0.png)"
"assets/CleanShot_2022-10-12_at_15.53.20@2x_1665561216083_0.png"]
"[[UUID]]"
;; 2nd link
["[[FIRST UUID]] and ![dino!](assets/subdir/partydino.gif)"
"assets/subdir/partydino.gif"]
"[[FIRST UUID]] and [[UUID]]"))
(deftest-async ^:integration export-docs-graph-with-convert-all-tags
(p/let [file-graph-dir "test/resources/docs-0.10.12"
start-time (cljs.core/system-time)
@@ -146,6 +181,7 @@
(is (empty? (map :entity (:errors (db-validate/validate-db! @conn))))
"Created graph has no validation errors")
(is (= 0 (count @(:ignored-properties import-state))) "No ignored properties")
(is (= 0 (count @(:ignored-assets import-state))) "No ignored assets")
(is (= []
(->> (d/q '[:find (pull ?b [:block/title {:block/tags [:db/ident]}])
:where [?b :block/tags :logseq.class/Tag]]
@@ -170,8 +206,9 @@
;; Counts
;; Includes journals as property values e.g. :logseq.property/deadline
(is (= 25 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Journal]] @conn))))
(is (= 26 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Journal]] @conn))))
(is (= 3 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Asset]] @conn))))
(is (= 4 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Task]] @conn))))
(is (= 4 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Query]] @conn))))
(is (= 2 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Card]] @conn))))
@@ -195,8 +232,9 @@
"Correct number of user classes")
(is (= 4 (count (d/datoms @conn :avet :block/tags :logseq.class/Whiteboard))))
(is (= 0 (count @(:ignored-properties import-state))) "No ignored properties")
(is (= 0 (count @(:ignored-assets import-state))) "No ignored assets")
(is (= 1 (count @(:ignored-files import-state))) "Ignore .edn for now")
(is (= 1 (count @assets))))
(is (= 3 (count @assets))))
(testing "logseq files"
(is (= ".foo {}\n"
@@ -365,7 +403,19 @@
;; Cards
(is (= {:block/tags [:logseq.class/Card]}
(db-test/readable-properties (db-test/find-block-by-content @conn "card 1")))
"None of the card properties are imported since they are deprecated"))
"None of the card properties are imported since they are deprecated")
;; Assets
(is (= {:block/tags [:logseq.class/Asset]
:logseq.property.asset/type "png"
:logseq.property.asset/checksum "3d5e620cac62159d8196c118574bfea7a16e86fa86efd1c3fa15a00a0a08792d"
:logseq.property.asset/size 753471
:logseq.property.asset/resize-metadata {:height 288, :width 252}}
(db-test/readable-properties (db-test/find-block-by-content @conn "greg-popovich-thumbs-up_1704749687791_0")))
"Asset has correct properties")
(is (= (d/entity @conn :logseq.class/Asset)
(:block/page (db-test/find-block-by-content @conn "greg-popovich-thumbs-up_1704749687791_0")))
"Imported into Asset page"))
(testing "tags convert to classes"
(is (= :user.class/Quotes___life

BIN
deps/graph-parser/test/resources/exporter-test-graph/assets/HEART_Teams.png vendored Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 84 KiB

BIN
deps/graph-parser/test/resources/exporter-test-graph/assets/subdir/partydino.gif vendored Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 32 KiB

2
deps/graph-parser/test/resources/exporter-test-graph/journals/2024_01_08.md vendored
View File

@@ -1,3 +1,3 @@
- [[some page]] tests page ref being parsed before page
- [[prop-num]] tests page existing before being used as and converted into a property
- ![greg-popovich-thumbs-up.png](../assets/greg-popovich-thumbs-up_1704749687791_0.png){:height 288, :width 252}
- ![greg-popovich-thumbs-up.png](../assets/greg-popovich-thumbs-up_1704749687791_0.png){:height 288, :width 252}

5
deps/graph-parser/test/resources/exporter-test-graph/journals/2025_06_12.md vendored Normal file
View File

@@ -0,0 +1,5 @@
- Test paragraph before an asset
![dino!](assets/subdir/partydino.gif){:width 105} tests an asset with a manual link, custom title and in a subdirectory
- ![greg-popovich-thumbs-up.png](../assets/greg-popovich-thumbs-up_1704749687791_0.png){:height 288, :width 252} and ![HEART Teams](../assets/HEART_Teams.png){:width 335.99774169921875}

2
deps/outliner/src/logseq/outliner/core.cljs vendored
View File

@@ -240,7 +240,7 @@
m* (cond->
(-> data'
(dissoc :block/children :block/meta :block/unordered
:block.temp/ast-title :block.temp/ast-body :block/level :block.temp/fully-loaded?
:block.temp/ast-title :block.temp/ast-body :block/level :block.temp/load-status
:block.temp/has-children?)
common-util/remove-nils

6
src/main/frontend/common_keywords.cljs
View File

@@ -9,4 +9,8 @@
:block/raw-title {:doc "like `:block/title`,
but when eval `(:block/raw-title block-entity)`, return raw title of this block"}
:kv/value {:doc "Used to store key-value, the value could be anything,
e.g. {:db/ident :logseq.kv/xxx :kv/value value}"})
e.g. {:db/ident :logseq.kv/xxx :kv/value value}"}
;; :block.temp/xxx keywords
:block.temp/load-status {:doc "`:full` means the block and its children have been loaded,
`:self` means the block itself has been loaded."})

3
src/main/frontend/components/block.cljs
View File

@@ -2656,8 +2656,7 @@
(editor-handler/clear-selection!)
(editor-handler/unhighlight-blocks!)
(let [f #(p/do!
(when-not (:block.temp/fully-loaded? (db/entity (:db/id block)))
(db-async/<get-block (state/get-current-repo) (:db/id block) {:children? false}))
(db-async/<get-block (state/get-current-repo) (:db/id block) {:children? false})
(let [cursor-range (some-> (gdom/getElement block-id)
(dom/by-class "block-content-inner")
first

39
src/main/frontend/components/imports.cljs
View File

@@ -1,6 +1,7 @@
(ns frontend.components.imports
"Import data into Logseq."
(:require [cljs-time.core :as t]
(:require ["path" :as node-path]
[cljs-time.core :as t]
[cljs.pprint :as pprint]
[clojure.string :as string]
[frontend.components.onboarding.setups :as setups]
@@ -26,7 +27,9 @@
[goog.functions :refer [debounce]]
[goog.object :as gobj]
[lambdaisland.glogi :as log]
[logseq.common.config :as common-config]
[logseq.common.path :as path]
[logseq.db.frontend.asset :as db-asset]
[logseq.db.frontend.validate :as db-validate]
[logseq.graph-parser.exporter :as gp-exporter]
[logseq.shui.dialog.core :as shui-dialog]
@@ -297,6 +300,13 @@
:info false)
(log/error :import-ignored-files {:msg (str "Import ignored " (count ignored-files) " file(s)")})
(pprint/pprint ignored-files))
(when-let [ignored-assets (seq @(:ignored-assets import-state))]
(notification/show! (str "Import ignored " (count ignored-assets) " "
(if (= 1 (count ignored-assets)) "asset" "assets")
". See the javascript console for more details.")
:info false)
(log/error :import-ignored-assets {:msg (str "Import ignored " (count ignored-assets) " asset(s)")})
(pprint/pprint ignored-assets))
(when-let [ignored-props (seq @(:ignored-properties import-state))]
(notification/show!
[:.mb-2
@@ -338,14 +348,32 @@
(log/error :import-error ex-data)))
(notification/show! msg :warning false)))
(defn- copy-asset [repo repo-dir file]
(defn- read-asset [file assets]
(-> (.arrayBuffer (:file-object file))
(p/then (fn [buffer]
(p/let [checksum (db-asset/<get-file-array-buffer-checksum buffer)]
(swap! assets assoc
(gp-exporter/asset-path->name (:path file))
{:size (.-size (:file-object file))
:checksum checksum
:type (db-asset/asset-path->type (:path file))
:path (:path file)
;; Save buffer to avoid reading asset twice
::array-buffer buffer}))))))
(defn- copy-asset [repo repo-dir asset-m]
(-> (::array-buffer asset-m)
(p/then (fn [buffer]
(let [content (js/Uint8Array. buffer)
parent-dir (path/path-join repo-dir (path/dirname (:path file)))]
assets-dir (path/path-join repo-dir common-config/local-assets-dir)]
(p/do!
(fs/mkdir-if-not-exists parent-dir)
(fs/write-plain-text-file! repo repo-dir (:path file) content {:skip-transact? true})))))))
(fs/mkdir-if-not-exists assets-dir)
(if (:block/uuid asset-m)
(fs/write-plain-text-file! repo assets-dir (str (:block/uuid asset-m) "." (:type asset-m)) content {:skip-transact? true})
(do
(println "Copied asset" (pr-str (node-path/basename (:path asset-m)))
"by its name since it was unused.")
(fs/write-plain-text-file! repo assets-dir (node-path/basename (:path asset-m)) content {:skip-transact? true})))))))))
(defn- import-file-graph
[*files
@@ -375,6 +403,7 @@
:<save-logseq-file (fn save-logseq-file [_ path content]
(db-editor-handler/save-file! path content))
;; asset file options
:<read-asset read-asset
:<copy-asset #(copy-asset repo (config/get-repo-dir repo) %)
;; doc file options
;; Write to frontend first as writing to worker first is poor ux with slow streaming changes

2
src/main/frontend/components/page.cljs
View File

@@ -697,7 +697,7 @@
*loading? (atom true)
page (db/get-page page-id-uuid-or-name)
*page (atom page)]
(when (:block.temp/fully-loaded? page) (reset! *loading? false))
(when (:block.temp/load-status page) (reset! *loading? false))
(p/let [page-block (db-async/<get-block (state/get-current-repo) page-id-uuid-or-name)]
(reset! *loading? false)
(reset! *page (db/entity (:db/id page-block)))

2
src/main/frontend/components/views.cljs
View File

@@ -909,7 +909,7 @@
(rum/defc table-row < rum/reactive db-mixins/query
[table row props option]
(let [block (db/sub-block (:db/id row))
row' (if (:block.temp/fully-loaded? block)
row' (if (:block.temp/load-status block)
(assoc block :block.temp/refs-count (:block.temp/refs-count row))
row)]
(table-row-inner table row' props option)))

26
src/main/frontend/db/async.cljs
View File

@@ -80,11 +80,11 @@
(assoc opts :property-ident property-id))))
(defn <get-block
[graph id-uuid-or-name & {:keys [children? nested-children? skip-transact? skip-refresh? children-only? properties]
[graph id-uuid-or-name & {:keys [children? skip-transact? skip-refresh? children-only? properties]
:or {children? true}
:as opts}]
;; (prn :debug :<get-block id-uuid-or-name)
;; (prn :debug :<get-block id-uuid-or-name :children? children?)
;; (js/console.trace)
(let [name' (str id-uuid-or-name)
opts (assoc opts :children? children?)
@@ -97,12 +97,11 @@
(db/get-page name'))
id (or (and (:block/uuid e) (str (:block/uuid e)))
(and (util/uuid-string? name') name')
id-uuid-or-name)]
id-uuid-or-name)
load-status (:block.temp/load-status e)]
(cond
(and (:block.temp/fully-loaded? e) ; children may not be fully loaded
(not children-only?)
(not children?)
(not nested-children?)
(and (or (= load-status :full)
(and (= load-status :self) (not children?) (not children-only?)))
(not (some #{:block.temp/refs-count} properties)))
(p/promise e)
@@ -113,11 +112,16 @@
{:keys [block children]} (first result)]
(when-not skip-transact?
(let [conn (db/get-db graph false)
load-status' (if (or children? children-only?) :full :self)
block-load-status-tx (when block
[{:db/id (:db/id block)
:block.temp/load-status load-status'}])
block-and-children (if block (cons block children) children)
affected-keys [[:frontend.worker.react/block (:db/id block)]]
tx-data (->> (remove (fn [b] (:block.temp/fully-loaded? (db/entity (:db/id b)))) block-and-children)
tx-data (->> (remove (fn [b] (:block.temp/load-status (db/entity (:db/id b)))) block-and-children)
(common-util/fast-remove-nils)
(remove empty?))]
(remove empty?)
(concat block-load-status-tx))]
(when (seq tx-data) (d/transact! conn tx-data))
(when-not skip-refresh?
(react/refresh-affected-queries! graph affected-keys {:skip-kv-custom-keys? true}))))
@@ -129,7 +133,7 @@
(defn <get-blocks
[graph ids* & {:as opts}]
(let [ids (remove (fn [id] (:block.temp/fully-loaded? (db/entity id))) ids*)]
(let [ids (remove (fn [id] (:block.temp/load-status (db/entity id))) ids*)]
(when (seq ids)
(p/let [result (state/<invoke-db-worker :thread-api/get-blocks graph
(map (fn [id]
@@ -138,7 +142,7 @@
(let [conn (db/get-db graph false)
result' (map :block result)]
(when (seq result')
(let [result'' (map (fn [b] (assoc b :block.temp/fully-loaded? true)) result')]
(let [result'' (map (fn [b] (assoc b :block.temp/load-status :self)) result')]
(d/transact! conn result'')))
result')))))

3
src/main/frontend/extensions/pdf/assets.cljs
View File

@@ -187,8 +187,7 @@
[hls-page]
(p/let [result (db-async/<get-block (state/get-current-repo)
(:block/uuid hls-page)
{:children? true
:nested-children? false})]
{:children? true})]
{:highlights (keep :logseq.property.pdf/hl-value result)}))
(defn file-based-load-hls-data$

11
src/main/frontend/handler/assets.cljs
View File

@@ -11,6 +11,7 @@
[logseq.common.config :as common-config]
[logseq.common.path :as path]
[logseq.common.util :as common-util]
[logseq.db.frontend.asset :as db-asset]
[medley.core :as medley]
[missionary.core :as m]
[promesa.core :as p])
@@ -177,18 +178,10 @@
blob (js/Blob. (array binary) (clj->js {:type "image"}))]
(when blob (js/URL.createObjectURL blob)))))))
(defn- decode-digest
[^js/Uint8Array digest]
(.. (js/Array.from digest)
(map (fn [s] (.. s (toString 16) (padStart 2 "0"))))
(join "")))
(defn get-file-checksum
[^js/Blob file]
(-> (.arrayBuffer file)
(.then (fn [buf] (js/crypto.subtle.digest "SHA-256" buf)))
(.then (fn [dig] (js/Uint8Array. dig)))
(.then decode-digest)))
(.then db-asset/<get-file-array-buffer-checksum)))
(defn <get-all-assets
[]

3
src/main/frontend/handler/block.cljs
View File

@@ -178,8 +178,7 @@
(util/mobile-keep-keyboard-open)
(let [repo (state/get-current-repo)]
(p/do!
(when-not (:block.temp/fully-loaded? (db/entity (:db/id block)))
(db-async/<get-block repo (:db/id block) {:children? false}))
(db-async/<get-block repo (:db/id block) {:children? false})
(when save-code-editor? (state/pub-event! [:editor/save-code-editor]))
(when (not= (:block/uuid block) (:block/uuid (state/get-edit-block)))
(state/clear-edit! {:clear-editing-block? false}))

19
src/main/frontend/handler/editor.cljs
View File

@@ -1,5 +1,6 @@
(ns ^:no-doc frontend.handler.editor
(:require [clojure.set :as set]
(:require ["path" :as node-path]
[clojure.set :as set]
[clojure.string :as string]
[clojure.walk :as w]
[dommy.core :as dom]
@@ -58,6 +59,7 @@
[logseq.db :as ldb]
[logseq.db.common.entity-plus :as entity-plus]
[logseq.db.file-based.schema :as file-schema]
[logseq.db.frontend.asset :as db-asset]
[logseq.db.frontend.property :as db-property]
[logseq.graph-parser.block :as gp-block]
[logseq.graph-parser.mldoc :as gp-mldoc]
@@ -1448,15 +1450,14 @@
(p/all
(for [[_index ^js file] (map-indexed vector files)]
;; WARN file name maybe fully qualified path when paste file
(p/let [file-name (util/node-path.basename (.-name file))
file-name-without-ext (.-name (util/node-path.parse file-name))
(p/let [file-name (node-path/basename (.-name file))
file-name-without-ext (db-asset/asset-name->title file-name)
checksum (assets-handler/get-file-checksum file)
existing-asset (db-async/<get-asset-with-checksum repo checksum)]
(if existing-asset
existing-asset
(p/let [block-id (ldb/new-block-id)
ext (when file-name
(string/lower-case (.substr (util/node-path.extname file-name) 1)))
ext (when file-name (db-asset/asset-path->type file-name))
_ (when (string/blank? ext)
(throw (ex-info "File doesn't have a valid ext."
{:file-name file-name})))
@@ -2116,8 +2117,7 @@
(db-async/<get-template-by-name (name db-id)))
block (when (:block/uuid block)
(db-async/<get-block repo (:block/uuid block)
{:children? true
:nested-children? true}))]
{:children? true}))]
(when (:db/id block)
(let [journal? (ldb/journal? target)
target (or target (state/get-edit-block))
@@ -3511,7 +3511,7 @@
repo (state/get-current-repo)
result (db-async/<get-block repo (or block-id page-id)
{:children-only? true
:nested-children? true})
:include-collapsed-children? true})
blocks (if page-id
result
(cons (db/entity [:block/uuid block-id]) result))
@@ -3587,7 +3587,8 @@
(defn expand-block! [block-id & {:keys [skip-db-collpsing?]}]
(let [repo (state/get-current-repo)]
(p/do!
(db-async/<get-block repo block-id {:children-only? true})
(db-async/<get-block repo block-id {:children-only? true
:include-collapsed-children? true})
(when-not (or skip-db-collpsing? (skip-collapsing-in-db?))
(set-blocks-collapsed! [block-id] false))
(state/set-collapsed-block! block-id false))))

3
src/main/frontend/handler/events.cljs
View File

@@ -422,8 +422,7 @@
;; load all nested children here for copy/export
(p/all (map (fn [id]
(db-async/<get-block (state/get-current-repo) id
{:nested-children? true
:skip-refresh? false})) ids))))
{:skip-refresh? false})) ids))))
(defn run!
[]

2
src/main/frontend/modules/outliner/pipeline.cljs
View File

@@ -63,7 +63,7 @@
(if (contains? #{:create-property-text-block :insert-blocks} (:outliner-op tx-meta))
(let [update-blocks-fully-loaded (keep (fn [datom] (when (= :block/uuid (:a datom))
{:db/id (:e datom)
:block.temp/fully-loaded? true})) tx-data)]
:block.temp/load-status :self})) tx-data)]
(concat update-blocks-fully-loaded tx-data))
tx-data))]
(d/transact! conn tx-data' tx-meta))

2
src/main/frontend/util.cljc
View File

@@ -817,7 +817,7 @@
(pr-str
{:graph graph
:embed-block? embed-block?
:blocks (mapv #(dissoc % :block.temp/fully-loaded? %) blocks)}))}))]
:blocks (mapv #(dissoc % :block.temp/load-status %) blocks)}))}))]
(if owner-window
(utils/writeClipboard data owner-window)
(utils/writeClipboard data)))))

4
src/main/frontend/worker/db/migrate.cljs
View File

@@ -232,7 +232,9 @@
[:db/retract id :logseq.property/default-value]
[:db/retract id :logseq.property/hide-empty-value]
[:db/retract id :logseq.property/enable-history?]]
datoms (d/datoms db :avet ident)]
datoms (if (:db/index class)
(d/datoms db :avet ident)
(filter (fn [d] (= ident (:a d))) (d/datoms db :eavt)))]
(concat [new-property]
retract-property-attrs
(mapcat

4
src/main/logseq/api.cljs
View File

@@ -717,9 +717,7 @@
(defn- <ensure-page-loaded
[block-uuid-or-page-name]
(p/let [repo (state/get-current-repo)
block (db-async/<get-block repo (str block-uuid-or-page-name)
{:children-props '[*]
:nested-children? true})
block (db-async/<get-block repo (str block-uuid-or-page-name) {})
_ (when-let [page-id (:db/id (:block/page block))]
(when-let [page-uuid (:block/uuid (db/entity page-id))]
(db-async/<get-block repo page-uuid)))]

2
src/main/logseq/api/block.cljs
View File

@@ -149,7 +149,7 @@
;; nested children results
(let [blocks (->> (db-model/get-block-and-children repo uuid)
(map (fn [b]
(dissoc (db-utils/pull (:db/id b)) :block.temp/fully-loaded?))))]
(dissoc (db-utils/pull (:db/id b)) :block.temp/load-status))))]
(first (outliner-tree/blocks->vec-tree blocks uuid)))
;; attached shallow children
(assoc block :block/children