Simple query performance enhancements (#12262)

* perf: separate scalar from ref property query

* fix: query bindings

* perf: separate default value query from others

* fix: import properties first and then other datoms
This commit is contained in:
Tienson Qin
2025-12-19 19:37:51 +08:00
committed by GitHub
parent 045cd5f537
commit aea9f09fd1
5 changed files with 271 additions and 156 deletions

View File

@@ -96,19 +96,11 @@
[(>= ?d ?start)]
[(<= ?d ?end)]]
:existing-property-value
'[;; non-ref value
[(existing-property-value ?b ?prop ?val)
[?prop-e :db/ident ?prop]
[(missing? $ ?prop-e :db/valueType)]
[?b ?prop ?val]]
;; ref value
[(existing-property-value ?b ?prop ?val)
[?prop-e :db/ident ?prop]
[?prop-e :db/valueType :db.type/ref]
[?b ?prop ?pv]
(or [?pv :block/title ?val]
[?pv :logseq.property/value ?val])]]
:ref->val
'[[(ref->val ?pv ?val)
[?pv :block/title ?val]]
[(ref->val ?pv ?val)
[?pv :logseq.property/value ?val]]]
:property-missing-value
'[(property-missing-value ?b ?prop-e ?default-p ?default-v)
@@ -121,31 +113,30 @@
[(= ?prop-v "N/A")]
[?prop-e ?default-p ?default-v]]
:property-scalar-default-value
'[(property-scalar-default-value ?b ?prop-e ?default-p ?val)
(property-missing-value ?b ?prop-e ?default-p ?default-v)
[(missing? $ ?prop-e :db/valueType)]
[?prop-e ?default-p ?val]]
:property-default-value
'[(property-default-value ?b ?prop-e ?default-p ?val)
(property-missing-value ?b ?prop-e ?default-p ?default-v)
(or
[?default-v :block/title ?val]
[?default-v :logseq.property/value ?val])]
:property-value
'[[(property-value ?b ?prop-e ?val)
:scalar-property-value
'[[(scalar-property-value ?b ?prop-e ?val)
[?prop-e :db/ident ?prop]
(existing-property-value ?b ?prop ?val)]
[(property-value ?b ?prop-e ?val)
(or
(and
[(missing? $ ?prop-e :db/valueType)]
(property-scalar-default-value ?b ?prop-e :logseq.property/scalar-default-value ?val))
(and
[?prop-e :db/valueType :db.type/ref]
(property-default-value ?b ?prop-e :logseq.property/default-value ?val)))]]
[?b ?prop ?val]]]
:scalar-property-value-with-default
'[[(scalar-property-value-with-default ?b ?prop-e ?val)
(scalar-property-value ?b ?prop-e ?val)]
[(scalar-property-value-with-default ?b ?prop-e ?val)
(property-missing-value ?b ?prop-e :logseq.property/scalar-default-value ?val)]]
:ref-property-value
'[[(ref-property-value ?b ?prop-e ?val)
[?prop-e :db/ident ?prop]
[?b ?prop ?pv]
(ref->val ?pv ?val)]]
:ref-property-value-with-default
'[[(ref-property-value-with-default ?b ?prop-e ?val)
(ref-property-value ?b ?prop-e ?val)]
[(ref-property-value-with-default ?b ?prop-e ?val)
(property-missing-value ?b ?prop-e :logseq.property/default-value ?pv)
(ref->val ?pv ?val)]]
:object-has-class-property
'[(object-has-class-property? ?b ?prop)
@@ -190,7 +181,65 @@
[(missing? $ ?prop-e :logseq.property/public?)]
[?prop-e :logseq.property/public? true])]
;; Checks if a property has a value for any features that are not simple queries
;; Checks if a property has a value for simple queries. Supports default values
:scalar-property
'[(scalar-property ?b ?prop ?val)
[?prop-e :db/ident ?prop]
(scalar-property-value ?b ?prop-e ?val)
(or
[(missing? $ ?prop-e :logseq.property/public?)]
[?prop-e :logseq.property/public? true])]
:scalar-property-with-default
'[(scalar-property-with-default ?b ?prop ?val)
[?prop-e :db/ident ?prop]
(scalar-property-value-with-default ?b ?prop-e ?val)
(or
[(missing? $ ?prop-e :logseq.property/public?)]
[?prop-e :logseq.property/public? true])]
:ref-property
'[(ref-property ?b ?prop ?val)
[?prop-e :db/ident ?prop]
(ref-property-value ?b ?prop-e ?val)
(or
[(missing? $ ?prop-e :logseq.property/public?)]
[?prop-e :logseq.property/public? true])]
:ref-property-with-default
'[(ref-property-with-default ?b ?prop ?val)
[?prop-e :db/ident ?prop]
(ref-property-value-with-default ?b ?prop-e ?val)
(or
[(missing? $ ?prop-e :logseq.property/public?)]
[?prop-e :logseq.property/public? true])]
;; Same as ref-property/scalar-property except it returns public and private properties like :block/title
:private-scalar-property
'[(private-scalar-property ?b ?prop ?val)
[?prop-e :db/ident ?prop]
[?prop-e :block/tags :logseq.class/Property]
(scalar-property-value ?b ?prop-e ?val)]
:private-scalar-property-with-default
'[(private-scalar-property-with-default ?b ?prop ?val)
[?prop-e :db/ident ?prop]
[?prop-e :block/tags :logseq.class/Property]
(scalar-property-value-with-default ?b ?prop-e ?val)]
:private-ref-property
'[(private-ref-property ?b ?prop ?val)
[?prop-e :db/ident ?prop]
[?prop-e :block/tags :logseq.class/Property]
(ref-property-value ?b ?prop-e ?val)]
:private-ref-property-with-default
'[(private-ref-property-with-default ?b ?prop ?val)
[?prop-e :db/ident ?prop]
[?prop-e :block/tags :logseq.class/Property]
(ref-property-value-with-default ?b ?prop-e ?val)]
;; `property` is slow, don't use it for user-facing queries
:property
'[(property ?b ?prop ?val)
[?prop-e :db/ident ?prop]
@@ -210,23 +259,6 @@
(or [?pv :block/title ?val]
[?pv :logseq.property/value ?val])))]
;; Checks if a property has a value for simple queries. Supports default values
:simple-query-property
'[(simple-query-property ?b ?prop ?val)
[?prop-e :db/ident ?prop]
[?prop-e :block/tags :logseq.class/Property]
(or
[(missing? $ ?prop-e :logseq.property/public?)]
[?prop-e :logseq.property/public? true])
(property-value ?b ?prop-e ?val)]
;; Same as property except it returns public and private properties like :block/title
:private-simple-query-property
'[(private-simple-query-property ?b ?prop ?val)
[?prop-e :db/ident ?prop]
[?prop-e :block/tags :logseq.class/Property]
(property-value ?b ?prop-e ?val)]
:tags
'[(tags ?b ?tags)
[?b :block/tags ?tag]
@@ -235,15 +267,13 @@
:task
'[(task ?b ?statuses)
;; and needed to avoid binding error
(and (simple-query-property ?b :logseq.property/status ?val)
[(contains? ?statuses ?val)])]
(ref-property-with-default ?b :logseq.property/status ?val)
[(contains? ?statuses ?val)]]
:priority
'[(priority ?b ?priorities)
;; and needed to avoid binding error
(and (simple-query-property ?b :logseq.property/priority ?priority)
[(contains? ?priorities ?priority)])]}))
(ref-property-with-default ?b :logseq.property/priority ?priority)
[(contains? ?priorities ?priority)]]}))
(def rules-dependencies
"For db graphs, a map of rule names and the rules they depend on. If this map
@@ -251,18 +281,26 @@
like find-rules-in-where"
{:has-ref #{:parent}
:page-ref #{:has-ref}
:task #{:simple-query-property}
:priority #{:simple-query-property}
:property-missing-value #{:object-has-class-property}
;; simple query helpers
:task #{:ref-property-with-default}
:priority #{:ref-property-with-default}
:has-property-or-object-property #{:object-has-class-property}
:object-has-class-property #{:class-extends}
:has-simple-query-property #{:has-property-or-object-property}
:has-private-simple-query-property #{:has-property-or-object-property}
:property-default-value #{:existing-property-value :property-missing-value}
:property-scalar-default-value #{:existing-property-value :property-missing-value}
:property-value #{:property-default-value :property-scalar-default-value}
:simple-query-property #{:property-value}
:private-simple-query-property #{:property-value}})
:property-missing-value #{:object-has-class-property}
:ref-property-value #{:ref->val}
:scalar-property #{:scalar-property-value}
:scalar-property-with-default #{:scalar-property-value-with-default}
:scalar-property-value-with-default #{:scalar-property-value :property-missing-value}
:ref-property #{:ref-property-value}
:ref-property-value-with-default #{:ref-property-value :property-missing-value}
:ref-property-with-default #{:ref-property-value-with-default}
:private-scalar-property #{:scalar-property-value}
:private-scalar-property-with-default #{:scalar-property-value-with-default}
:private-ref-property #{:ref-property-value}
:private-ref-property-with-default #{:ref-property-value-with-default}})
(defn- get-full-deps
[deps rules-deps]

View File

@@ -11,17 +11,14 @@
(rules/extract-rules rules/db-query-dsl-rules)))
(deftest get-full-deps
(let [default-value-deps #{:property-default-value :property-missing-value :existing-property-value
:object-has-class-property :class-extends}
property-value-deps (conj default-value-deps :property-value :property-scalar-default-value)
property-deps (conj property-value-deps :simple-query-property)
(let [property-value-deps #{:ref->val :class-extends :object-has-class-property :property-missing-value :ref-property-value :ref-property-value-with-default}
property-deps (conj property-value-deps :ref-property-with-default)
task-deps (conj property-deps :task)
priority-deps (conj property-deps :priority)
task-priority-deps (into priority-deps task-deps)]
(are [x y] (= y (#'rules/get-full-deps x rules/rules-dependencies))
[:property-default-value] default-value-deps
[:property-value] property-value-deps
[:simple-query-property] property-deps
[:ref-property-value-with-default] property-value-deps
[:ref-property-with-default] property-deps
[:task] task-deps
[:priority] priority-deps
[:task :priority] task-priority-deps)))
@@ -50,7 +47,7 @@
@conn))
"has-property can bind to property arg")))
(deftest property-rule
(deftest ref-property-rule
(let [conn (db-test/create-conn-with-blocks
{:properties {:foo {:logseq.property/type :default}
:foo2 {:logseq.property/type :default}
@@ -65,36 +62,36 @@
:build/properties {:foo "bar A"}}}]})]
(testing "cardinality :one property"
(is (= ["Page1"]
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (property ?b :user.property/foo "bar")]
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (ref-property ?b :user.property/foo "bar")]
@conn)
(map (comp :block/title first))))
"property returns result when page has property")
(is (= []
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (property ?b :user.property/foo "baz")]
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (ref-property ?b :user.property/foo "baz")]
@conn)
(map (comp :block/title first))))
"property returns no result when page doesn't have property value")
(is (= #{:user.property/foo}
(->> (q-with-rules '[:find [?p ...]
:where (property ?b ?p "bar") [?b :block/title "Page1"]]
:where (ref-property ?b ?p "bar") [?b :block/title "Page1"]]
@conn)
set))
"property can bind to property arg with bound property value"))
(testing "cardinality :many property"
(is (= ["Page1"]
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (property ?b :user.property/number-many 5)]
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (ref-property ?b :user.property/number-many 5)]
@conn)
(map (comp :block/title first))))
"property returns result when page has property")
(is (= []
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (property ?b :user.property/number-many 20)]
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (ref-property ?b :user.property/number-many 20)]
@conn)
(map (comp :block/title first))))
"property returns no result when page doesn't have property value")
(is (= #{:user.property/number-many}
(->> (q-with-rules '[:find [?p ...]
:where (property ?b ?p 5) [?b :block/title "Page1"]]
:where (ref-property ?b ?p 5) [?b :block/title "Page1"]]
@conn)
set))
"property can bind to property arg with bound property value"))
@@ -103,7 +100,7 @@
(testing ":ref property"
(is (= ["Page1"]
(->> (q-with-rules '[:find (pull ?b [:block/title])
:where (property ?b :user.property/page-many "Page A")]
:where (ref-property ?b :user.property/page-many "Page A")]
@conn)
(map (comp :block/title first))))
"property returns result when page has property")

View File

@@ -178,32 +178,51 @@
(defonce remove-nil? (partial remove nil?))
(defn- not-clause? [c]
(and (seq? c) (= 'not (first c))))
(defn- distinct-preserve-order [xs]
(let [seen (volatile! #{})]
(reduce (fn [acc x]
(if (contains? @seen x)
acc
(do (vswap! seen conj x)
(conj acc x))))
[] xs)))
(defn- build-and-or-not
[e {:keys [current-filter vars] :as env} level fe]
(let [raw-clauses (map (fn [form]
(build-query form (assoc env :current-filter fe) (inc level)))
(rest e))
;; preserve order (no hash-order surprises)
clauses (->> raw-clauses
(map :query)
remove-nil?
(distinct))
(distinct-preserve-order))
;; for (and ...), ensure any (not ...) comes AFTER positive binders
clauses (if (= fe 'and)
(let [[nots others] (reduce (fn [[ns os] c]
(if (not-clause? c)
[(conj ns c) os]
[ns (conj os c)]))
[[] []]
clauses)]
(concat others nots))
clauses)
nested-and? (and (= fe 'and) (= current-filter 'and))]
(when (seq clauses)
(let [result (build-and-or-not-result
fe clauses current-filter nested-and?)
vars' (set/union (set @vars) (collect-vars result))
query (cond
nested-and?
result
(and (zero? level) (contains? #{'and 'or} fe))
result
(and (= 'not fe) (some? current-filter))
result
:else
[result])]
(let [result (build-and-or-not-result fe clauses current-filter nested-and?)
vars' (set/union (set @vars) (collect-vars result))
query (cond
nested-and? result
(and (zero? level) (contains? #{'and 'or} fe)) result
(and (= 'not fe) (some? current-filter)) result
:else [result])]
(reset! vars vars')
{:query query
:rules (distinct (mapcat :rules raw-clauses))}))))
@@ -343,13 +362,38 @@
[e {:keys [db-graph? private-property?]}]
(let [k (if db-graph? (->db-keyword-property (nth e 1)) (->file-keyword-property (nth e 1)))
v (nth e 2)
v' (if db-graph? (->db-property-value k v) (->file-property-value v))]
v' (if db-graph? (->db-property-value k v) (->file-property-value v))
property (when (qualified-keyword? k)
(db-utils/entity k))
ref-type? (= :db.type/ref (:db/valueType property))]
(if db-graph?
(if private-property?
{:query (list 'private-simple-query-property '?b k v')
:rules [:private-simple-query-property]}
{:query (list 'simple-query-property '?b k v')
:rules [:simple-query-property]})
(let [default-value (if ref-type?
(when-let [value (:logseq.property/default-value property)]
(or (:block/title value)
(:logseq.property/value value)))
(:logseq.property/scalar-default-value property))
default-value? (and (some? v') (= default-value v'))
rule (if private-property?
(cond
(and ref-type? default-value?)
:private-ref-property-with-default
ref-type?
:private-ref-property
default-value?
:private-scalar-property-with-default
:else
:private-scalar-property)
(cond
(and ref-type? default-value?)
:ref-property-with-default
ref-type?
:ref-property
default-value?
:scalar-property-with-default
:else
:scalar-property))]
{:query (list (symbol (name rule)) '?b k v')
:rules [rule]})
{:query (list 'property '?b k v')
:rules [:property]})))
@@ -662,35 +706,69 @@ Some bindings in this fn:
(string/replace #"^#" "#tag ")
(string/replace tag-placeholder "#")))))
(defn- lvar? [x]
(and (symbol? x) (= \? (first (name x)))))
(defn- collect-vars-by-polarity
"Returns {:pos #{?vars} :neg #{?vars}}.
Vars inside (not ...) are counted as negative."
[form]
(let [pos (volatile! #{})
neg (volatile! #{})]
(letfn [(walk* [x positive?]
(cond
(lvar? x)
(vswap! (if positive? pos neg) conj x)
(and (seq? x) (= 'not (first x)))
(doseq [c (rest x)] (walk* c false))
(sequential? x)
(doseq [c x] (walk* c positive?))
(map? x)
(do (doseq [k (keys x)] (walk* k positive?))
(doseq [v (vals x)] (walk* v positive?)))
:else nil))]
(walk* form true)
{:pos @pos :neg @neg})))
(defn- add-bindings!
[q {:keys [db-graph?]}]
(let [forms (set (flatten q))
syms ['?b '?p 'not]
[b? p? not?] (-> (set/intersection (set syms) forms)
(map syms))]
(cond
not?
(cond
(and b? p?)
(concat [['?b :block/uuid] ['?p :block/name] ['?b :block/page '?p]] q)
(let [{:keys [pos neg]} (collect-vars-by-polarity q)
b?
(if db-graph?
;; This keeps built-in properties from showing up in not results.
;; May need to be revisited as more class and property filters are explored
(concat [['?b :block/uuid] '[(missing? $ ?b :logseq.property/built-in?)]] q)
(concat [['?b :block/uuid]] q))
appears? (fn [v] (or (contains? pos v) (contains? neg v)))
needs-domain? (fn [v] (and (appears? v) (not (contains? pos v))))
p?
(concat [['?p :block/name]] q)
b-need? (needs-domain? '?b)
p-need? (needs-domain? '?p)
:else
q)
;; CASE 1: both needed → link them, do NOT enumerate all blocks
bindings
(cond
(and b-need? p-need?)
[['?b :block/page '?p]]
(and b? p?)
(concat [['?b :block/page '?p]] q)
;; CASE 2: only ?b needed → last-resort domain (true global negation)
b-need?
(if db-graph?
[['?b :block/uuid]
'[(missing? $ ?b :logseq.property/built-in?)]]
[['?b :block/uuid]])
:else
;; CASE 3: only ?p needed
p-need?
[['?p :block/name]]
;; CASE 4: both already positive → optional link (cheap + useful)
(and (contains? pos '?b) (contains? pos '?p))
[['?b :block/page '?p]]
:else
nil)]
(if (seq bindings)
(concat bindings q) ;; IMPORTANT: bindings FIRST
q)))
(defn simplify-query

View File

@@ -2,7 +2,6 @@
"Custom queries."
(:require [clojure.string :as string]
[clojure.walk :as walk]
[frontend.config :as config]
[frontend.date :as date]
[frontend.db.conn :as conn]
[frontend.db.model :as model]
@@ -94,27 +93,16 @@
(defn react-query
[repo {:keys [query inputs rules] :as query'} query-opts]
(let [pprint (if config/dev? #(when (state/developer-mode?) (apply prn %&)) (fn [_] nil))
start-time (.now js/performance)]
(when config/dev? (js/console.groupCollapsed "react-query logs:"))
(pprint "================")
(pprint "Use the following to debug your datalog queries:")
(pprint query')
(let [query (resolve-query query)
repo (or repo (state/get-current-repo))
db (conn/get-db repo)
resolve-with (select-keys query-opts [:current-page-fn :current-block-uuid])
resolved-inputs (mapv #(resolve-input db % resolve-with) inputs)
inputs (cond-> resolved-inputs
rules
(conj rules))
k [:custom
(or (:query-string query') (dissoc query' :title))
(:today-query? query-opts)
inputs]]
(pprint "inputs (post-resolution):" resolved-inputs)
(pprint "query-opts:" query-opts)
(pprint (str "time elapsed: " (.toFixed (- (.now js/performance) start-time) 2) "ms"))
(when config/dev? (js/console.groupEnd))
[k (apply react/q repo k query-opts query inputs)])))
(let [query (resolve-query query)
repo (or repo (state/get-current-repo))
db (conn/get-db repo)
resolve-with (select-keys query-opts [:current-page-fn :current-block-uuid])
resolved-inputs (mapv #(resolve-input db % resolve-with) inputs)
inputs (cond-> resolved-inputs
rules
(conj rules))
k [:custom
(or (:query-string query') (dissoc query' :title))
(:today-query? query-opts)
inputs]]
[k (apply react/q repo k query-opts query inputs)]))

View File

@@ -49,6 +49,7 @@
[logseq.db.common.sqlite :as common-sqlite]
[logseq.db.common.view :as db-view]
[logseq.db.frontend.class :as db-class]
[logseq.db.frontend.property :as db-property]
[logseq.db.sqlite.create-graph :as sqlite-create-graph]
[logseq.db.sqlite.export :as sqlite-export]
[logseq.db.sqlite.gc :as sqlite-gc]
@@ -244,8 +245,19 @@
conn (common-sqlite/get-storage-conn storage schema)
_ (db-fix/check-and-fix-schema! repo conn)
_ (when datoms
(let [data (map (fn [datom]
[:db/add (:e datom) (:a datom) (:v datom)]) datoms)]
(let [eid->datoms (group-by :e datoms)
{properties true non-properties false} (group-by
(fn [[_eid datoms]]
(boolean
(some (fn [datom] (and (= (:a datom) :db/ident)
(db-property/property? (:v datom))))
datoms)))
eid->datoms)
datoms (concat (mapcat second properties)
(mapcat second non-properties))
data (map (fn [datom]
[:db/add (:e datom) (:a datom) (:v datom)])
datoms)]
(d/transact! conn data {:initial-db? true})))
client-ops-conn (when-not @*publishing? (common-sqlite/get-storage-conn
client-ops-storage
@@ -387,7 +399,9 @@
(def-thread-api :thread-api/q
[repo inputs]
(when-let [conn (worker-state/get-datascript-conn repo)]
(apply d/q (first inputs) @conn (rest inputs))))
(worker-util/profile
(str "Datalog query: " inputs)
(apply d/q (first inputs) @conn (rest inputs)))))
(def-thread-api :thread-api/datoms
[repo & args]