From 995d0bf4a9ce85722c2e488a0bf4bcd223b5786b Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Tue, 14 Apr 2026 04:52:10 +0800 Subject: [PATCH] enhance(view): speed up large-graph get-view-data --- deps/db/src/logseq/db/common/view.cljs | 308 +++++++++++-------- deps/db/test/logseq/db/common/view_test.cljs | 67 ++++ 2 files changed, 248 insertions(+), 127 deletions(-) create mode 100644 deps/db/test/logseq/db/common/view_test.cljs diff --git a/deps/db/src/logseq/db/common/view.cljs b/deps/db/src/logseq/db/common/view.cljs index bb615be4e8..b3aa71d1df 100644 --- a/deps/db/src/logseq/db/common/view.cljs +++ b/deps/db/src/logseq/db/common/view.cljs @@ -80,32 +80,36 @@ [db {:keys [id asc?] :as sorting} entities partition?] (let [property (or (d/entity db id) {:db/ident id}) get-value-fn (memoize (get-value-for-sort property)) + entities' (if (vector? entities) entities (vec entities)) + datom-sort-supported? (contains? #{:block/updated-at :block/created-at :block/title} + (:db/ident property)) + use-datom-sort? (and datom-sort-supported? + (not= :db.type/ref (:db/valueType property)) + (> (count entities') 10000)) sorted-entities (->> (cond (= id :block.temp/refs-count) - (cond-> (sort-by :block.temp/refs-count entities) + (cond-> (sort-by :block.temp/refs-count entities') (not asc?) reverse) + use-datom-sort? + (let [datoms (cond-> + (->> (d/datoms db :avet id) + (common-util/distinct-by :e) + vec) + (not asc?) + rseq) + row-ids (set (map :db/id entities')) + id->row (zipmap (map :db/id entities') entities')] + (keep + (fn [d] + (when (row-ids (:e d)) + (id->row (:e d)))) + datoms)) + :else - (let [ref-type? (= :db.type/ref (:db/valueType property))] - (if (or ref-type? (not (contains? - #{:block/updated-at :block/created-at :block/title} - (:db/ident property)))) - (sort-ref-entities-by-single-property entities sorting get-value-fn) - (let [datoms (cond-> - (->> (d/datoms db :avet id) - (common-util/distinct-by :e) - vec) - (not asc?) - rseq) - row-ids (set (map :db/id entities)) - id->row (zipmap (map :db/id entities) entities)] - (keep - (fn [d] - (when (row-ids (:e d)) - (id->row (:e d)))) - datoms))))) + (sort-ref-entities-by-single-property entities' sorting get-value-fn)) distinct)] (if partition? @@ -289,6 +293,7 @@ (transient #{}) (concat (d/datoms db :avet :logseq.property/hide? true) + (d/datoms db :avet :logseq.property/deleted-at) (d/datoms db :avet :logseq.property/built-in? true) (d/datoms db :avet :block/tags property-tag-id)))))) @@ -310,6 +315,44 @@ (transient []) (d/datoms db :avet property-ident))))) +(def ^:private fast-all-pages-sort-ids + "Sort ids supported by a datom-order fast path" + #{:block/updated-at :block/created-at :block/title :block/name}) + +(defn- get-all-page-ids-fast + "Fast path for all-pages where only sorted page ids are needed. Avoids + hydrating every page entity by deriving ordering from indexed datoms." + [db sorting] + (let [major-sorting (or (first sorting) + {:id :block/updated-at :asc? false}) + minor-sorting (seq (rest sorting))] + (when (and (empty? minor-sorting) + (contains? fast-all-pages-sort-ids (:id major-sorting))) + (let [exclude-ids (get-exclude-page-ids db) + page-ids (persistent! + (reduce (fn [result datom] + (let [eid (:e datom)] + (if (contains? exclude-ids eid) + result + (conj! result eid)))) + (transient []) + (d/datoms db :avet :block/name))) + sort-id (:id major-sorting) + asc? (:asc? major-sorting) + get-sort-value (memoize + (fn [eid] + (get (entity-plus/unsafe->Entity db eid) sort-id))) + cmp (fn [eid-a eid-b] + (let [va (get-sort-value eid-a) + vb (get-sort-value eid-b) + c (cond + (and (nil? va) (nil? vb)) 0 + (nil? va) 1 + (nil? vb) -1 + :else (compare va vb))] + (if asc? c (- c))))] + (sort cmp page-ids))))) + (defn- get-entities [db view feat-type property-ident view-for-id* sorting] (let [view-for (:logseq.property/view-for view) @@ -440,111 +483,122 @@ [db view-id {:keys [journals? _view-for-id view-feature-type group-by-property-ident input query-entity-ids query filters sorting] :as opts}] ;; TODO: create a view for journals maybe? - (time - (cond - journals? - (let [ids (->> (ldb/get-latest-journals db) - (mapv :db/id))] - {:count (count ids) - :data ids}) - :else - (let [view (d/entity db view-id) - group-by-property (:logseq.property.view/group-by-property view) - list-view? (= :logseq.property.view/type.list (:db/ident (:logseq.property.view/type view))) - group-by-property-ident (or (:db/ident group-by-property) group-by-property-ident) - group-by-closed-values? (some? (:property/closed-values group-by-property)) - ref-property? (= (:db/valueType group-by-property) :db.type/ref) - filters (or (:logseq.property.table/filters view) filters) - feat-type (or view-feature-type (:logseq.property.view/feature-type view)) - query? (= feat-type :query-result) - query-entity-ids (when (seq query-entity-ids) (set query-entity-ids)) - entities-result (if query? - (keep (fn [id] - (let [e (d/entity db id)] - (when-not (= :logseq.property/query (:db/ident (:logseq.property/created-from-property e))) - e))) - query-entity-ids) - (get-view-entities db view-id opts)) - entities (if (= feat-type :linked-references) - (:ref-blocks entities-result) - entities-result) - sorting (let [sorting* (:logseq.property.table/sorting view)] - (if (or (= sorting* :logseq.property/empty-placeholder) (empty? sorting*)) - (or sorting [{:id :block/updated-at, :asc? false}]) - sorting*)) - filtered-entities (if (or (seq filters) (not (string/blank? input))) - (filter (fn [row] (row-matched? db row filters input)) entities) - entities) - group-by-page? (= group-by-property-ident :block/page) - readable-property-value-or-ent - (fn readable-property-value-or-ent [ent] - (let [pvalue (get ent group-by-property-ident)] - (if (de/entity? pvalue) - (if (match-property-value-as-entity? pvalue group-by-property) - pvalue - (db-property/property-value-content pvalue)) - pvalue))) - result (if group-by-property-ident - (let [groups-sort-by-property-ident (or (:db/ident (:logseq.property.view/sort-groups-by-property view)) - :block/journal-day) - desc? (:logseq.property.view/sort-groups-desc? view) - result (->> filtered-entities - (group-by readable-property-value-or-ent) - (seq)) - keyfn (fn [groups-sort-by-property-ident] - (fn [[by-value _]] - (cond - group-by-page? - (let [v (get by-value groups-sort-by-property-ident)] - (if (and (= groups-sort-by-property-ident :block/journal-day) (not desc?) - (nil? (:block/journal-day by-value))) - ;; Use MAX_SAFE_INTEGER so non-journal pages (without :block/journal-day) are sorted - ;; after all journal pages when sorting by journal date. - js/Number.MAX_SAFE_INTEGER - v)) - group-by-closed-values? - (:block/order by-value) - ref-property? - (db-property/property-value-content by-value) - :else - by-value)))] - (sort (common-util/by-sorting - (cond-> - [{:get-value (keyfn groups-sort-by-property-ident) - :asc? (not desc?)}] - (not= groups-sort-by-property-ident :block/title) - (conj {:get-value (keyfn :block/title) - :asc? (not desc?)}))) - result)) - (sort-entities db sorting filtered-entities)) - data' (if group-by-property-ident - (map - (fn [[by-value entities]] - (let [by-value' (if (de/entity? by-value) - (select-keys by-value [:db/id :db/ident :block/uuid :block/title :block/name :logseq.property/value :logseq.property/icon :block/tags]) - by-value) - pages? (not (some :block/page entities)) - group (if (and list-view? (not pages?)) - (let [parent-groups (->> entities - (group-by :block/parent) - (sort-by (fn [[parent _]] (:block/order parent))))] - (map - (fn [[_parent blocks]] - [(:block/uuid (first blocks)) - (map (fn [b] - {:db/id (:db/id b) - :block/parent (:block/uuid (:block/parent b))}) - (ldb/sort-by-order blocks))]) - parent-groups)) - (->> (sort-entities db sorting entities) - (map :db/id)))] - [by-value' group])) - result) - (map :db/id result))] - (cond-> - {:count (count filtered-entities) - :data (distinct data')} - (= feat-type :linked-references) - (merge (select-keys entities-result [:ref-pages-count :ref-matched-children-ids])) - query? - (assoc :properties (get-query-properties query entities-result))))))) + (cond + journals? + (let [ids (->> (ldb/get-latest-journals db) + (mapv :db/id))] + {:count (count ids) + :data ids}) + :else + (let [view (d/entity db view-id) + group-by-property (:logseq.property.view/group-by-property view) + list-view? (= :logseq.property.view/type.list (:db/ident (:logseq.property.view/type view))) + group-by-property-ident (or (:db/ident group-by-property) group-by-property-ident) + group-by-closed-values? (some? (:property/closed-values group-by-property)) + ref-property? (= (:db/valueType group-by-property) :db.type/ref) + filters (or (:logseq.property.table/filters view) filters) + feat-type (or view-feature-type (:logseq.property.view/feature-type view)) + query? (= feat-type :query-result) + query-entity-ids (when (seq query-entity-ids) (set query-entity-ids)) + sorting (let [sorting* (:logseq.property.table/sorting view)] + (if (or (= sorting* :logseq.property/empty-placeholder) (empty? sorting*)) + (or sorting [{:id :block/updated-at :asc? false}]) + sorting*)) + fast-all-pages-ids (when (and (= feat-type :all-pages) + (not query?) + (nil? group-by-property-ident) + (empty? filters) + (string/blank? input)) + (get-all-page-ids-fast db sorting))] + (if fast-all-pages-ids + {:count (count fast-all-pages-ids) + :data fast-all-pages-ids} + (let [entities-result (if query? + (keep (fn [id] + (let [e (d/entity db id)] + (when-not (= :logseq.property/query (:db/ident (:logseq.property/created-from-property e))) + e))) + query-entity-ids) + (get-view-entities db view-id opts)) + entities (if (= feat-type :linked-references) + (:ref-blocks entities-result) + entities-result) + filtered-entities (if (or (seq filters) (not (string/blank? input))) + (into [] (filter (fn [row] (row-matched? db row filters input))) entities) + entities) + group-by-page? (= group-by-property-ident :block/page) + readable-property-value-or-ent + (fn readable-property-value-or-ent [ent] + (let [pvalue (get ent group-by-property-ident)] + (if (de/entity? pvalue) + (if (match-property-value-as-entity? pvalue group-by-property) + pvalue + (db-property/property-value-content pvalue)) + pvalue))) + result (if group-by-property-ident + (let [groups-sort-by-property-ident (or (:db/ident (:logseq.property.view/sort-groups-by-property view)) + :block/journal-day) + desc? (:logseq.property.view/sort-groups-desc? view) + result (->> filtered-entities + (group-by readable-property-value-or-ent) + (seq)) + keyfn (fn [groups-sort-by-property-ident] + (fn [[by-value _]] + (cond + group-by-page? + (let [v (get by-value groups-sort-by-property-ident)] + (if (and (= groups-sort-by-property-ident :block/journal-day) (not desc?) + (nil? (:block/journal-day by-value))) + ;; Use MAX_SAFE_INTEGER so non-journal pages (without :block/journal-day) are sorted + ;; after all journal pages when sorting by journal date. + js/Number.MAX_SAFE_INTEGER + v)) + group-by-closed-values? + (:block/order by-value) + ref-property? + (db-property/property-value-content by-value) + :else + by-value)))] + (sort (common-util/by-sorting + (cond-> + [{:get-value (keyfn groups-sort-by-property-ident) + :asc? (not desc?)}] + (not= groups-sort-by-property-ident :block/title) + (conj {:get-value (keyfn :block/title) + :asc? (not desc?)}))) + result)) + (sort-entities db sorting filtered-entities)) + data' (if group-by-property-ident + (map + (fn [[by-value entities]] + (let [by-value' (if (de/entity? by-value) + (select-keys by-value [:db/id :db/ident :block/uuid :block/title :block/name :logseq.property/value :logseq.property/icon :block/tags]) + by-value) + pages? (not (some :block/page entities)) + group (if (and list-view? (not pages?)) + (let [parent-groups (->> entities + (group-by :block/parent) + (sort-by (fn [[parent _]] (:block/order parent))))] + (map + (fn [[_parent blocks]] + [(:block/uuid (first blocks)) + (map (fn [b] + {:db/id (:db/id b) + :block/parent (:block/uuid (:block/parent b))}) + (ldb/sort-by-order blocks))]) + parent-groups)) + (->> (sort-entities db sorting entities) + (map :db/id)))] + [by-value' group])) + result) + (map :db/id result)) + dedupe-data? (or (= feat-type :property-objects) query?)] + (cond-> + {:count (count filtered-entities) + :data (if dedupe-data? + (distinct data') + data')} + (= feat-type :linked-references) + (merge (select-keys entities-result [:ref-pages-count :ref-matched-children-ids])) + query? + (assoc :properties (get-query-properties query entities-result)))))))) diff --git a/deps/db/test/logseq/db/common/view_test.cljs b/deps/db/test/logseq/db/common/view_test.cljs new file mode 100644 index 0000000000..69954b235c --- /dev/null +++ b/deps/db/test/logseq/db/common/view_test.cljs @@ -0,0 +1,67 @@ +(ns logseq.db.common.view-test + (:require [cljs.test :refer [deftest is]] + [datascript.core :as d] + [logseq.db.common.view :as db-view] + [logseq.db.test.helper :as db-test])) + +(defn- all-pages-view-id [conn] + (let [tx (d/transact! conn [{:db/id -100 + :block/title "All pages test view" + :block/uuid #uuid "00000000-0000-0000-0000-000000000100" + :logseq.property.view/feature-type :all-pages + :logseq.property.view/type :logseq.property.view/type.table}])] + (get-in tx [:tempids -100]))) + +(deftest get-view-data-all-pages-sorts-and-filters-hidden-test + (let [conn (db-test/create-conn-with-blocks + {:pages-and-blocks + [{:page {:block/title "Alpha" :block/updated-at 10}} + {:page {:block/title "Beta" :block/updated-at 20}} + {:page {:block/title "Hidden" :block/updated-at 30 :logseq.property/hide? true}} + {:page {:block/title "Deleted" :block/updated-at 40 :logseq.property/deleted-at 1}}]}) + view-id (all-pages-view-id conn) + result (db-view/get-view-data @conn view-id {:view-feature-type :all-pages + :sorting [{:id :block/updated-at :asc? false}]}) + ids (:data result) + titles (map (fn [id] (:block/title (d/entity @conn id))) ids)] + (is (= 2 (:count result))) + (is (= ["Beta" "Alpha"] titles)))) + +(deftest get-view-data-all-pages-title-sort-test + (let [conn (db-test/create-conn-with-blocks + {:pages-and-blocks + [{:page {:block/title "gamma" :block/updated-at 1}} + {:page {:block/title "alpha" :block/updated-at 2}} + {:page {:block/title "beta" :block/updated-at 3}}]}) + view-id (all-pages-view-id conn) + result (db-view/get-view-data @conn view-id {:view-feature-type :all-pages + :sorting [{:id :block/title :asc? true}]}) + ids (:data result) + titles (map (fn [id] (:block/title (d/entity @conn id))) ids)] + (is (= ["alpha" "beta" "gamma"] titles)))) + +(deftest get-view-data-class-objects-sort-keeps-rows-with-missing-sort-value-test + (let [conn (db-test/create-conn-with-blocks + {:classes {:Topic {:block/title "Topic"}} + :pages-and-blocks + [{:page {:block/title "With timestamp" + :block/updated-at 20 + :build/tags [:Topic]}} + {:page {:block/title "Without timestamp" + :block/updated-at 10 + :build/tags [:Topic]}}]}) + class-id (:db/id (d/entity @conn :user.class/Topic)) + without-ts-id (d/q '[:find ?e . + :in $ ?title + :where [?e :block/title ?title]] + @conn + "Without timestamp") + without-ts-value (:block/updated-at (d/entity @conn without-ts-id)) + _ (d/transact! conn [[:db/retract without-ts-id :block/updated-at without-ts-value]]) + view-id (all-pages-view-id conn) + result (db-view/get-view-data @conn view-id {:view-feature-type :class-objects + :view-for-id class-id + :sorting [{:id :block/updated-at :asc? false}]}) + titles (map (fn [id] (:block/title (d/entity @conn id))) (:data result))] + (is (= 2 (:count result))) + (is (= #{"With timestamp" "Without timestamp"} (set titles)))))