mirror of
https://github.com/logseq/logseq.git
synced 2026-05-29 23:19:38 +00:00
enhance(view): speed up large-graph get-view-data
This commit is contained in:
Tienson Qin
308
deps/db/src/logseq/db/common/view.cljs
vendored
308
deps/db/src/logseq/db/common/view.cljs
vendored
@@ -80,32 +80,36 @@
|
||||
[db {:keys [id asc?] :as sorting} entities partition?]
|
||||
(let [property (or (d/entity db id) {:db/ident id})
|
||||
get-value-fn (memoize (get-value-for-sort property))
|
||||
entities' (if (vector? entities) entities (vec entities))
|
||||
datom-sort-supported? (contains? #{:block/updated-at :block/created-at :block/title}
|
||||
(:db/ident property))
|
||||
use-datom-sort? (and datom-sort-supported?
|
||||
(not= :db.type/ref (:db/valueType property))
|
||||
(> (count entities') 10000))
|
||||
sorted-entities (->>
|
||||
(cond
|
||||
(= id :block.temp/refs-count)
|
||||
(cond-> (sort-by :block.temp/refs-count entities)
|
||||
(cond-> (sort-by :block.temp/refs-count entities')
|
||||
(not asc?)
|
||||
reverse)
|
||||
|
||||
use-datom-sort?
|
||||
(let [datoms (cond->
|
||||
(->> (d/datoms db :avet id)
|
||||
(common-util/distinct-by :e)
|
||||
vec)
|
||||
(not asc?)
|
||||
rseq)
|
||||
row-ids (set (map :db/id entities'))
|
||||
id->row (zipmap (map :db/id entities') entities')]
|
||||
(keep
|
||||
(fn [d]
|
||||
(when (row-ids (:e d))
|
||||
(id->row (:e d))))
|
||||
datoms))
|
||||
|
||||
:else
|
||||
(let [ref-type? (= :db.type/ref (:db/valueType property))]
|
||||
(if (or ref-type? (not (contains?
|
||||
#{:block/updated-at :block/created-at :block/title}
|
||||
(:db/ident property))))
|
||||
(sort-ref-entities-by-single-property entities sorting get-value-fn)
|
||||
(let [datoms (cond->
|
||||
(->> (d/datoms db :avet id)
|
||||
(common-util/distinct-by :e)
|
||||
vec)
|
||||
(not asc?)
|
||||
rseq)
|
||||
row-ids (set (map :db/id entities))
|
||||
id->row (zipmap (map :db/id entities) entities)]
|
||||
(keep
|
||||
(fn [d]
|
||||
(when (row-ids (:e d))
|
||||
(id->row (:e d))))
|
||||
datoms)))))
|
||||
(sort-ref-entities-by-single-property entities' sorting get-value-fn))
|
||||
|
||||
distinct)]
|
||||
(if partition?
|
||||
@@ -289,6 +293,7 @@
|
||||
(transient #{})
|
||||
(concat
|
||||
(d/datoms db :avet :logseq.property/hide? true)
|
||||
(d/datoms db :avet :logseq.property/deleted-at)
|
||||
(d/datoms db :avet :logseq.property/built-in? true)
|
||||
(d/datoms db :avet :block/tags property-tag-id))))))
|
||||
|
||||
@@ -310,6 +315,44 @@
|
||||
(transient [])
|
||||
(d/datoms db :avet property-ident)))))
|
||||
|
||||
(def ^:private fast-all-pages-sort-ids
|
||||
"Sort ids supported by a datom-order fast path"
|
||||
#{:block/updated-at :block/created-at :block/title :block/name})
|
||||
|
||||
(defn- get-all-page-ids-fast
|
||||
"Fast path for all-pages where only sorted page ids are needed. Avoids
|
||||
hydrating every page entity by deriving ordering from indexed datoms."
|
||||
[db sorting]
|
||||
(let [major-sorting (or (first sorting)
|
||||
{:id :block/updated-at :asc? false})
|
||||
minor-sorting (seq (rest sorting))]
|
||||
(when (and (empty? minor-sorting)
|
||||
(contains? fast-all-pages-sort-ids (:id major-sorting)))
|
||||
(let [exclude-ids (get-exclude-page-ids db)
|
||||
page-ids (persistent!
|
||||
(reduce (fn [result datom]
|
||||
(let [eid (:e datom)]
|
||||
(if (contains? exclude-ids eid)
|
||||
result
|
||||
(conj! result eid))))
|
||||
(transient [])
|
||||
(d/datoms db :avet :block/name)))
|
||||
sort-id (:id major-sorting)
|
||||
asc? (:asc? major-sorting)
|
||||
get-sort-value (memoize
|
||||
(fn [eid]
|
||||
(get (entity-plus/unsafe->Entity db eid) sort-id)))
|
||||
cmp (fn [eid-a eid-b]
|
||||
(let [va (get-sort-value eid-a)
|
||||
vb (get-sort-value eid-b)
|
||||
c (cond
|
||||
(and (nil? va) (nil? vb)) 0
|
||||
(nil? va) 1
|
||||
(nil? vb) -1
|
||||
:else (compare va vb))]
|
||||
(if asc? c (- c))))]
|
||||
(sort cmp page-ids)))))
|
||||
|
||||
(defn- get-entities
|
||||
[db view feat-type property-ident view-for-id* sorting]
|
||||
(let [view-for (:logseq.property/view-for view)
|
||||
@@ -440,111 +483,122 @@
|
||||
[db view-id {:keys [journals? _view-for-id view-feature-type group-by-property-ident input query-entity-ids query filters sorting]
|
||||
:as opts}]
|
||||
;; TODO: create a view for journals maybe?
|
||||
(time
|
||||
(cond
|
||||
journals?
|
||||
(let [ids (->> (ldb/get-latest-journals db)
|
||||
(mapv :db/id))]
|
||||
{:count (count ids)
|
||||
:data ids})
|
||||
:else
|
||||
(let [view (d/entity db view-id)
|
||||
group-by-property (:logseq.property.view/group-by-property view)
|
||||
list-view? (= :logseq.property.view/type.list (:db/ident (:logseq.property.view/type view)))
|
||||
group-by-property-ident (or (:db/ident group-by-property) group-by-property-ident)
|
||||
group-by-closed-values? (some? (:property/closed-values group-by-property))
|
||||
ref-property? (= (:db/valueType group-by-property) :db.type/ref)
|
||||
filters (or (:logseq.property.table/filters view) filters)
|
||||
feat-type (or view-feature-type (:logseq.property.view/feature-type view))
|
||||
query? (= feat-type :query-result)
|
||||
query-entity-ids (when (seq query-entity-ids) (set query-entity-ids))
|
||||
entities-result (if query?
|
||||
(keep (fn [id]
|
||||
(let [e (d/entity db id)]
|
||||
(when-not (= :logseq.property/query (:db/ident (:logseq.property/created-from-property e)))
|
||||
e)))
|
||||
query-entity-ids)
|
||||
(get-view-entities db view-id opts))
|
||||
entities (if (= feat-type :linked-references)
|
||||
(:ref-blocks entities-result)
|
||||
entities-result)
|
||||
sorting (let [sorting* (:logseq.property.table/sorting view)]
|
||||
(if (or (= sorting* :logseq.property/empty-placeholder) (empty? sorting*))
|
||||
(or sorting [{:id :block/updated-at, :asc? false}])
|
||||
sorting*))
|
||||
filtered-entities (if (or (seq filters) (not (string/blank? input)))
|
||||
(filter (fn [row] (row-matched? db row filters input)) entities)
|
||||
entities)
|
||||
group-by-page? (= group-by-property-ident :block/page)
|
||||
readable-property-value-or-ent
|
||||
(fn readable-property-value-or-ent [ent]
|
||||
(let [pvalue (get ent group-by-property-ident)]
|
||||
(if (de/entity? pvalue)
|
||||
(if (match-property-value-as-entity? pvalue group-by-property)
|
||||
pvalue
|
||||
(db-property/property-value-content pvalue))
|
||||
pvalue)))
|
||||
result (if group-by-property-ident
|
||||
(let [groups-sort-by-property-ident (or (:db/ident (:logseq.property.view/sort-groups-by-property view))
|
||||
:block/journal-day)
|
||||
desc? (:logseq.property.view/sort-groups-desc? view)
|
||||
result (->> filtered-entities
|
||||
(group-by readable-property-value-or-ent)
|
||||
(seq))
|
||||
keyfn (fn [groups-sort-by-property-ident]
|
||||
(fn [[by-value _]]
|
||||
(cond
|
||||
group-by-page?
|
||||
(let [v (get by-value groups-sort-by-property-ident)]
|
||||
(if (and (= groups-sort-by-property-ident :block/journal-day) (not desc?)
|
||||
(nil? (:block/journal-day by-value)))
|
||||
;; Use MAX_SAFE_INTEGER so non-journal pages (without :block/journal-day) are sorted
|
||||
;; after all journal pages when sorting by journal date.
|
||||
js/Number.MAX_SAFE_INTEGER
|
||||
v))
|
||||
group-by-closed-values?
|
||||
(:block/order by-value)
|
||||
ref-property?
|
||||
(db-property/property-value-content by-value)
|
||||
:else
|
||||
by-value)))]
|
||||
(sort (common-util/by-sorting
|
||||
(cond->
|
||||
[{:get-value (keyfn groups-sort-by-property-ident)
|
||||
:asc? (not desc?)}]
|
||||
(not= groups-sort-by-property-ident :block/title)
|
||||
(conj {:get-value (keyfn :block/title)
|
||||
:asc? (not desc?)})))
|
||||
result))
|
||||
(sort-entities db sorting filtered-entities))
|
||||
data' (if group-by-property-ident
|
||||
(map
|
||||
(fn [[by-value entities]]
|
||||
(let [by-value' (if (de/entity? by-value)
|
||||
(select-keys by-value [:db/id :db/ident :block/uuid :block/title :block/name :logseq.property/value :logseq.property/icon :block/tags])
|
||||
by-value)
|
||||
pages? (not (some :block/page entities))
|
||||
group (if (and list-view? (not pages?))
|
||||
(let [parent-groups (->> entities
|
||||
(group-by :block/parent)
|
||||
(sort-by (fn [[parent _]] (:block/order parent))))]
|
||||
(map
|
||||
(fn [[_parent blocks]]
|
||||
[(:block/uuid (first blocks))
|
||||
(map (fn [b]
|
||||
{:db/id (:db/id b)
|
||||
:block/parent (:block/uuid (:block/parent b))})
|
||||
(ldb/sort-by-order blocks))])
|
||||
parent-groups))
|
||||
(->> (sort-entities db sorting entities)
|
||||
(map :db/id)))]
|
||||
[by-value' group]))
|
||||
result)
|
||||
(map :db/id result))]
|
||||
(cond->
|
||||
{:count (count filtered-entities)
|
||||
:data (distinct data')}
|
||||
(= feat-type :linked-references)
|
||||
(merge (select-keys entities-result [:ref-pages-count :ref-matched-children-ids]))
|
||||
query?
|
||||
(assoc :properties (get-query-properties query entities-result)))))))
|
||||
(cond
|
||||
journals?
|
||||
(let [ids (->> (ldb/get-latest-journals db)
|
||||
(mapv :db/id))]
|
||||
{:count (count ids)
|
||||
:data ids})
|
||||
:else
|
||||
(let [view (d/entity db view-id)
|
||||
group-by-property (:logseq.property.view/group-by-property view)
|
||||
list-view? (= :logseq.property.view/type.list (:db/ident (:logseq.property.view/type view)))
|
||||
group-by-property-ident (or (:db/ident group-by-property) group-by-property-ident)
|
||||
group-by-closed-values? (some? (:property/closed-values group-by-property))
|
||||
ref-property? (= (:db/valueType group-by-property) :db.type/ref)
|
||||
filters (or (:logseq.property.table/filters view) filters)
|
||||
feat-type (or view-feature-type (:logseq.property.view/feature-type view))
|
||||
query? (= feat-type :query-result)
|
||||
query-entity-ids (when (seq query-entity-ids) (set query-entity-ids))
|
||||
sorting (let [sorting* (:logseq.property.table/sorting view)]
|
||||
(if (or (= sorting* :logseq.property/empty-placeholder) (empty? sorting*))
|
||||
(or sorting [{:id :block/updated-at :asc? false}])
|
||||
sorting*))
|
||||
fast-all-pages-ids (when (and (= feat-type :all-pages)
|
||||
(not query?)
|
||||
(nil? group-by-property-ident)
|
||||
(empty? filters)
|
||||
(string/blank? input))
|
||||
(get-all-page-ids-fast db sorting))]
|
||||
(if fast-all-pages-ids
|
||||
{:count (count fast-all-pages-ids)
|
||||
:data fast-all-pages-ids}
|
||||
(let [entities-result (if query?
|
||||
(keep (fn [id]
|
||||
(let [e (d/entity db id)]
|
||||
(when-not (= :logseq.property/query (:db/ident (:logseq.property/created-from-property e)))
|
||||
e)))
|
||||
query-entity-ids)
|
||||
(get-view-entities db view-id opts))
|
||||
entities (if (= feat-type :linked-references)
|
||||
(:ref-blocks entities-result)
|
||||
entities-result)
|
||||
filtered-entities (if (or (seq filters) (not (string/blank? input)))
|
||||
(into [] (filter (fn [row] (row-matched? db row filters input))) entities)
|
||||
entities)
|
||||
group-by-page? (= group-by-property-ident :block/page)
|
||||
readable-property-value-or-ent
|
||||
(fn readable-property-value-or-ent [ent]
|
||||
(let [pvalue (get ent group-by-property-ident)]
|
||||
(if (de/entity? pvalue)
|
||||
(if (match-property-value-as-entity? pvalue group-by-property)
|
||||
pvalue
|
||||
(db-property/property-value-content pvalue))
|
||||
pvalue)))
|
||||
result (if group-by-property-ident
|
||||
(let [groups-sort-by-property-ident (or (:db/ident (:logseq.property.view/sort-groups-by-property view))
|
||||
:block/journal-day)
|
||||
desc? (:logseq.property.view/sort-groups-desc? view)
|
||||
result (->> filtered-entities
|
||||
(group-by readable-property-value-or-ent)
|
||||
(seq))
|
||||
keyfn (fn [groups-sort-by-property-ident]
|
||||
(fn [[by-value _]]
|
||||
(cond
|
||||
group-by-page?
|
||||
(let [v (get by-value groups-sort-by-property-ident)]
|
||||
(if (and (= groups-sort-by-property-ident :block/journal-day) (not desc?)
|
||||
(nil? (:block/journal-day by-value)))
|
||||
;; Use MAX_SAFE_INTEGER so non-journal pages (without :block/journal-day) are sorted
|
||||
;; after all journal pages when sorting by journal date.
|
||||
js/Number.MAX_SAFE_INTEGER
|
||||
v))
|
||||
group-by-closed-values?
|
||||
(:block/order by-value)
|
||||
ref-property?
|
||||
(db-property/property-value-content by-value)
|
||||
:else
|
||||
by-value)))]
|
||||
(sort (common-util/by-sorting
|
||||
(cond->
|
||||
[{:get-value (keyfn groups-sort-by-property-ident)
|
||||
:asc? (not desc?)}]
|
||||
(not= groups-sort-by-property-ident :block/title)
|
||||
(conj {:get-value (keyfn :block/title)
|
||||
:asc? (not desc?)})))
|
||||
result))
|
||||
(sort-entities db sorting filtered-entities))
|
||||
data' (if group-by-property-ident
|
||||
(map
|
||||
(fn [[by-value entities]]
|
||||
(let [by-value' (if (de/entity? by-value)
|
||||
(select-keys by-value [:db/id :db/ident :block/uuid :block/title :block/name :logseq.property/value :logseq.property/icon :block/tags])
|
||||
by-value)
|
||||
pages? (not (some :block/page entities))
|
||||
group (if (and list-view? (not pages?))
|
||||
(let [parent-groups (->> entities
|
||||
(group-by :block/parent)
|
||||
(sort-by (fn [[parent _]] (:block/order parent))))]
|
||||
(map
|
||||
(fn [[_parent blocks]]
|
||||
[(:block/uuid (first blocks))
|
||||
(map (fn [b]
|
||||
{:db/id (:db/id b)
|
||||
:block/parent (:block/uuid (:block/parent b))})
|
||||
(ldb/sort-by-order blocks))])
|
||||
parent-groups))
|
||||
(->> (sort-entities db sorting entities)
|
||||
(map :db/id)))]
|
||||
[by-value' group]))
|
||||
result)
|
||||
(map :db/id result))
|
||||
dedupe-data? (or (= feat-type :property-objects) query?)]
|
||||
(cond->
|
||||
{:count (count filtered-entities)
|
||||
:data (if dedupe-data?
|
||||
(distinct data')
|
||||
data')}
|
||||
(= feat-type :linked-references)
|
||||
(merge (select-keys entities-result [:ref-pages-count :ref-matched-children-ids]))
|
||||
query?
|
||||
(assoc :properties (get-query-properties query entities-result))))))))
|
||||
|
||||
67
deps/db/test/logseq/db/common/view_test.cljs
vendored
Normal file
67
deps/db/test/logseq/db/common/view_test.cljs
vendored
Normal file
@@ -0,0 +1,67 @@
|
||||
(ns logseq.db.common.view-test
|
||||
(:require [cljs.test :refer [deftest is]]
|
||||
[datascript.core :as d]
|
||||
[logseq.db.common.view :as db-view]
|
||||
[logseq.db.test.helper :as db-test]))
|
||||
|
||||
(defn- all-pages-view-id [conn]
|
||||
(let [tx (d/transact! conn [{:db/id -100
|
||||
:block/title "All pages test view"
|
||||
:block/uuid #uuid "00000000-0000-0000-0000-000000000100"
|
||||
:logseq.property.view/feature-type :all-pages
|
||||
:logseq.property.view/type :logseq.property.view/type.table}])]
|
||||
(get-in tx [:tempids -100])))
|
||||
|
||||
(deftest get-view-data-all-pages-sorts-and-filters-hidden-test
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks
|
||||
[{:page {:block/title "Alpha" :block/updated-at 10}}
|
||||
{:page {:block/title "Beta" :block/updated-at 20}}
|
||||
{:page {:block/title "Hidden" :block/updated-at 30 :logseq.property/hide? true}}
|
||||
{:page {:block/title "Deleted" :block/updated-at 40 :logseq.property/deleted-at 1}}]})
|
||||
view-id (all-pages-view-id conn)
|
||||
result (db-view/get-view-data @conn view-id {:view-feature-type :all-pages
|
||||
:sorting [{:id :block/updated-at :asc? false}]})
|
||||
ids (:data result)
|
||||
titles (map (fn [id] (:block/title (d/entity @conn id))) ids)]
|
||||
(is (= 2 (:count result)))
|
||||
(is (= ["Beta" "Alpha"] titles))))
|
||||
|
||||
(deftest get-view-data-all-pages-title-sort-test
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks
|
||||
[{:page {:block/title "gamma" :block/updated-at 1}}
|
||||
{:page {:block/title "alpha" :block/updated-at 2}}
|
||||
{:page {:block/title "beta" :block/updated-at 3}}]})
|
||||
view-id (all-pages-view-id conn)
|
||||
result (db-view/get-view-data @conn view-id {:view-feature-type :all-pages
|
||||
:sorting [{:id :block/title :asc? true}]})
|
||||
ids (:data result)
|
||||
titles (map (fn [id] (:block/title (d/entity @conn id))) ids)]
|
||||
(is (= ["alpha" "beta" "gamma"] titles))))
|
||||
|
||||
(deftest get-view-data-class-objects-sort-keeps-rows-with-missing-sort-value-test
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:classes {:Topic {:block/title "Topic"}}
|
||||
:pages-and-blocks
|
||||
[{:page {:block/title "With timestamp"
|
||||
:block/updated-at 20
|
||||
:build/tags [:Topic]}}
|
||||
{:page {:block/title "Without timestamp"
|
||||
:block/updated-at 10
|
||||
:build/tags [:Topic]}}]})
|
||||
class-id (:db/id (d/entity @conn :user.class/Topic))
|
||||
without-ts-id (d/q '[:find ?e .
|
||||
:in $ ?title
|
||||
:where [?e :block/title ?title]]
|
||||
@conn
|
||||
"Without timestamp")
|
||||
without-ts-value (:block/updated-at (d/entity @conn without-ts-id))
|
||||
_ (d/transact! conn [[:db/retract without-ts-id :block/updated-at without-ts-value]])
|
||||
view-id (all-pages-view-id conn)
|
||||
result (db-view/get-view-data @conn view-id {:view-feature-type :class-objects
|
||||
:view-for-id class-id
|
||||
:sorting [{:id :block/updated-at :asc? false}]})
|
||||
titles (map (fn [id] (:block/title (d/entity @conn id))) (:data result))]
|
||||
(is (= 2 (:count result)))
|
||||
(is (= #{"With timestamp" "Without timestamp"} (set titles)))))
|
||||
Reference in New Issue
Block a user