diff --git a/deps/db/src/logseq/db/frontend/rules.cljc b/deps/db/src/logseq/db/frontend/rules.cljc index 60df8e372e..38ef2616b6 100644 --- a/deps/db/src/logseq/db/frontend/rules.cljc +++ b/deps/db/src/logseq/db/frontend/rules.cljc @@ -96,19 +96,11 @@ [(>= ?d ?start)] [(<= ?d ?end)]] - :existing-property-value - '[;; non-ref value - [(existing-property-value ?b ?prop ?val) - [?prop-e :db/ident ?prop] - [(missing? $ ?prop-e :db/valueType)] - [?b ?prop ?val]] - ;; ref value - [(existing-property-value ?b ?prop ?val) - [?prop-e :db/ident ?prop] - [?prop-e :db/valueType :db.type/ref] - [?b ?prop ?pv] - (or [?pv :block/title ?val] - [?pv :logseq.property/value ?val])]] + :ref->val + '[[(ref->val ?pv ?val) + [?pv :block/title ?val]] + [(ref->val ?pv ?val) + [?pv :logseq.property/value ?val]]] :property-missing-value '[(property-missing-value ?b ?prop-e ?default-p ?default-v) @@ -121,31 +113,30 @@ [(= ?prop-v "N/A")] [?prop-e ?default-p ?default-v]] - :property-scalar-default-value - '[(property-scalar-default-value ?b ?prop-e ?default-p ?val) - (property-missing-value ?b ?prop-e ?default-p ?default-v) - [(missing? $ ?prop-e :db/valueType)] - [?prop-e ?default-p ?val]] - - :property-default-value - '[(property-default-value ?b ?prop-e ?default-p ?val) - (property-missing-value ?b ?prop-e ?default-p ?default-v) - (or - [?default-v :block/title ?val] - [?default-v :logseq.property/value ?val])] - - :property-value - '[[(property-value ?b ?prop-e ?val) + :scalar-property-value + '[[(scalar-property-value ?b ?prop-e ?val) [?prop-e :db/ident ?prop] - (existing-property-value ?b ?prop ?val)] - [(property-value ?b ?prop-e ?val) - (or - (and - [(missing? $ ?prop-e :db/valueType)] - (property-scalar-default-value ?b ?prop-e :logseq.property/scalar-default-value ?val)) - (and - [?prop-e :db/valueType :db.type/ref] - (property-default-value ?b ?prop-e :logseq.property/default-value ?val)))]] + [?b ?prop ?val]]] + + :scalar-property-value-with-default + '[[(scalar-property-value-with-default ?b ?prop-e ?val) + (scalar-property-value ?b ?prop-e ?val)] + + [(scalar-property-value-with-default ?b ?prop-e ?val) + (property-missing-value ?b ?prop-e :logseq.property/scalar-default-value ?val)]] + + :ref-property-value + '[[(ref-property-value ?b ?prop-e ?val) + [?prop-e :db/ident ?prop] + [?b ?prop ?pv] + (ref->val ?pv ?val)]] + + :ref-property-value-with-default + '[[(ref-property-value-with-default ?b ?prop-e ?val) + (ref-property-value ?b ?prop-e ?val)] + [(ref-property-value-with-default ?b ?prop-e ?val) + (property-missing-value ?b ?prop-e :logseq.property/default-value ?pv) + (ref->val ?pv ?val)]] :object-has-class-property '[(object-has-class-property? ?b ?prop) @@ -190,7 +181,65 @@ [(missing? $ ?prop-e :logseq.property/public?)] [?prop-e :logseq.property/public? true])] - ;; Checks if a property has a value for any features that are not simple queries + ;; Checks if a property has a value for simple queries. Supports default values + :scalar-property + '[(scalar-property ?b ?prop ?val) + [?prop-e :db/ident ?prop] + (scalar-property-value ?b ?prop-e ?val) + (or + [(missing? $ ?prop-e :logseq.property/public?)] + [?prop-e :logseq.property/public? true])] + + :scalar-property-with-default + '[(scalar-property-with-default ?b ?prop ?val) + [?prop-e :db/ident ?prop] + (scalar-property-value-with-default ?b ?prop-e ?val) + (or + [(missing? $ ?prop-e :logseq.property/public?)] + [?prop-e :logseq.property/public? true])] + + :ref-property + '[(ref-property ?b ?prop ?val) + [?prop-e :db/ident ?prop] + (ref-property-value ?b ?prop-e ?val) + (or + [(missing? $ ?prop-e :logseq.property/public?)] + [?prop-e :logseq.property/public? true])] + + :ref-property-with-default + '[(ref-property-with-default ?b ?prop ?val) + [?prop-e :db/ident ?prop] + (ref-property-value-with-default ?b ?prop-e ?val) + (or + [(missing? $ ?prop-e :logseq.property/public?)] + [?prop-e :logseq.property/public? true])] + + ;; Same as ref-property/scalar-property except it returns public and private properties like :block/title + :private-scalar-property + '[(private-scalar-property ?b ?prop ?val) + [?prop-e :db/ident ?prop] + [?prop-e :block/tags :logseq.class/Property] + (scalar-property-value ?b ?prop-e ?val)] + + :private-scalar-property-with-default + '[(private-scalar-property-with-default ?b ?prop ?val) + [?prop-e :db/ident ?prop] + [?prop-e :block/tags :logseq.class/Property] + (scalar-property-value-with-default ?b ?prop-e ?val)] + + :private-ref-property + '[(private-ref-property ?b ?prop ?val) + [?prop-e :db/ident ?prop] + [?prop-e :block/tags :logseq.class/Property] + (ref-property-value ?b ?prop-e ?val)] + + :private-ref-property-with-default + '[(private-ref-property-with-default ?b ?prop ?val) + [?prop-e :db/ident ?prop] + [?prop-e :block/tags :logseq.class/Property] + (ref-property-value-with-default ?b ?prop-e ?val)] + + ;; `property` is slow, don't use it for user-facing queries :property '[(property ?b ?prop ?val) [?prop-e :db/ident ?prop] @@ -210,23 +259,6 @@ (or [?pv :block/title ?val] [?pv :logseq.property/value ?val])))] - ;; Checks if a property has a value for simple queries. Supports default values - :simple-query-property - '[(simple-query-property ?b ?prop ?val) - [?prop-e :db/ident ?prop] - [?prop-e :block/tags :logseq.class/Property] - (or - [(missing? $ ?prop-e :logseq.property/public?)] - [?prop-e :logseq.property/public? true]) - (property-value ?b ?prop-e ?val)] - - ;; Same as property except it returns public and private properties like :block/title - :private-simple-query-property - '[(private-simple-query-property ?b ?prop ?val) - [?prop-e :db/ident ?prop] - [?prop-e :block/tags :logseq.class/Property] - (property-value ?b ?prop-e ?val)] - :tags '[(tags ?b ?tags) [?b :block/tags ?tag] @@ -235,15 +267,13 @@ :task '[(task ?b ?statuses) - ;; and needed to avoid binding error - (and (simple-query-property ?b :logseq.property/status ?val) - [(contains? ?statuses ?val)])] + (ref-property-with-default ?b :logseq.property/status ?val) + [(contains? ?statuses ?val)]] :priority '[(priority ?b ?priorities) - ;; and needed to avoid binding error - (and (simple-query-property ?b :logseq.property/priority ?priority) - [(contains? ?priorities ?priority)])]})) + (ref-property-with-default ?b :logseq.property/priority ?priority) + [(contains? ?priorities ?priority)]]})) (def rules-dependencies "For db graphs, a map of rule names and the rules they depend on. If this map @@ -251,18 +281,26 @@ like find-rules-in-where" {:has-ref #{:parent} :page-ref #{:has-ref} - :task #{:simple-query-property} - :priority #{:simple-query-property} - :property-missing-value #{:object-has-class-property} + + ;; simple query helpers + :task #{:ref-property-with-default} + :priority #{:ref-property-with-default} :has-property-or-object-property #{:object-has-class-property} :object-has-class-property #{:class-extends} :has-simple-query-property #{:has-property-or-object-property} :has-private-simple-query-property #{:has-property-or-object-property} - :property-default-value #{:existing-property-value :property-missing-value} - :property-scalar-default-value #{:existing-property-value :property-missing-value} - :property-value #{:property-default-value :property-scalar-default-value} - :simple-query-property #{:property-value} - :private-simple-query-property #{:property-value}}) + :property-missing-value #{:object-has-class-property} + :ref-property-value #{:ref->val} + :scalar-property #{:scalar-property-value} + :scalar-property-with-default #{:scalar-property-value-with-default} + :scalar-property-value-with-default #{:scalar-property-value :property-missing-value} + :ref-property #{:ref-property-value} + :ref-property-value-with-default #{:ref-property-value :property-missing-value} + :ref-property-with-default #{:ref-property-value-with-default} + :private-scalar-property #{:scalar-property-value} + :private-scalar-property-with-default #{:scalar-property-value-with-default} + :private-ref-property #{:ref-property-value} + :private-ref-property-with-default #{:ref-property-value-with-default}}) (defn- get-full-deps [deps rules-deps] diff --git a/deps/db/test/logseq/db/frontend/rules_test.cljs b/deps/db/test/logseq/db/frontend/rules_test.cljs index a6b760001d..5b2f43179a 100644 --- a/deps/db/test/logseq/db/frontend/rules_test.cljs +++ b/deps/db/test/logseq/db/frontend/rules_test.cljs @@ -11,17 +11,14 @@ (rules/extract-rules rules/db-query-dsl-rules))) (deftest get-full-deps - (let [default-value-deps #{:property-default-value :property-missing-value :existing-property-value - :object-has-class-property :class-extends} - property-value-deps (conj default-value-deps :property-value :property-scalar-default-value) - property-deps (conj property-value-deps :simple-query-property) + (let [property-value-deps #{:ref->val :class-extends :object-has-class-property :property-missing-value :ref-property-value :ref-property-value-with-default} + property-deps (conj property-value-deps :ref-property-with-default) task-deps (conj property-deps :task) priority-deps (conj property-deps :priority) task-priority-deps (into priority-deps task-deps)] (are [x y] (= y (#'rules/get-full-deps x rules/rules-dependencies)) - [:property-default-value] default-value-deps - [:property-value] property-value-deps - [:simple-query-property] property-deps + [:ref-property-value-with-default] property-value-deps + [:ref-property-with-default] property-deps [:task] task-deps [:priority] priority-deps [:task :priority] task-priority-deps))) @@ -50,7 +47,7 @@ @conn)) "has-property can bind to property arg"))) -(deftest property-rule +(deftest ref-property-rule (let [conn (db-test/create-conn-with-blocks {:properties {:foo {:logseq.property/type :default} :foo2 {:logseq.property/type :default} @@ -65,36 +62,36 @@ :build/properties {:foo "bar A"}}}]})] (testing "cardinality :one property" (is (= ["Page1"] - (->> (q-with-rules '[:find (pull ?b [:block/title]) :where (property ?b :user.property/foo "bar")] + (->> (q-with-rules '[:find (pull ?b [:block/title]) :where (ref-property ?b :user.property/foo "bar")] @conn) (map (comp :block/title first)))) "property returns result when page has property") (is (= [] - (->> (q-with-rules '[:find (pull ?b [:block/title]) :where (property ?b :user.property/foo "baz")] + (->> (q-with-rules '[:find (pull ?b [:block/title]) :where (ref-property ?b :user.property/foo "baz")] @conn) (map (comp :block/title first)))) "property returns no result when page doesn't have property value") (is (= #{:user.property/foo} (->> (q-with-rules '[:find [?p ...] - :where (property ?b ?p "bar") [?b :block/title "Page1"]] + :where (ref-property ?b ?p "bar") [?b :block/title "Page1"]] @conn) set)) "property can bind to property arg with bound property value")) (testing "cardinality :many property" (is (= ["Page1"] - (->> (q-with-rules '[:find (pull ?b [:block/title]) :where (property ?b :user.property/number-many 5)] + (->> (q-with-rules '[:find (pull ?b [:block/title]) :where (ref-property ?b :user.property/number-many 5)] @conn) (map (comp :block/title first)))) "property returns result when page has property") (is (= [] - (->> (q-with-rules '[:find (pull ?b [:block/title]) :where (property ?b :user.property/number-many 20)] + (->> (q-with-rules '[:find (pull ?b [:block/title]) :where (ref-property ?b :user.property/number-many 20)] @conn) (map (comp :block/title first)))) "property returns no result when page doesn't have property value") (is (= #{:user.property/number-many} (->> (q-with-rules '[:find [?p ...] - :where (property ?b ?p 5) [?b :block/title "Page1"]] + :where (ref-property ?b ?p 5) [?b :block/title "Page1"]] @conn) set)) "property can bind to property arg with bound property value")) @@ -103,7 +100,7 @@ (testing ":ref property" (is (= ["Page1"] (->> (q-with-rules '[:find (pull ?b [:block/title]) - :where (property ?b :user.property/page-many "Page A")] + :where (ref-property ?b :user.property/page-many "Page A")] @conn) (map (comp :block/title first)))) "property returns result when page has property") diff --git a/src/main/frontend/db/query_dsl.cljs b/src/main/frontend/db/query_dsl.cljs index 82c2df03e3..dad28ad445 100644 --- a/src/main/frontend/db/query_dsl.cljs +++ b/src/main/frontend/db/query_dsl.cljs @@ -178,32 +178,51 @@ (defonce remove-nil? (partial remove nil?)) +(defn- not-clause? [c] + (and (seq? c) (= 'not (first c)))) + +(defn- distinct-preserve-order [xs] + (let [seen (volatile! #{})] + (reduce (fn [acc x] + (if (contains? @seen x) + acc + (do (vswap! seen conj x) + (conj acc x)))) + [] xs))) + (defn- build-and-or-not [e {:keys [current-filter vars] :as env} level fe] (let [raw-clauses (map (fn [form] (build-query form (assoc env :current-filter fe) (inc level))) (rest e)) + + ;; preserve order (no hash-order surprises) clauses (->> raw-clauses (map :query) remove-nil? - (distinct)) + (distinct-preserve-order)) + + ;; for (and ...), ensure any (not ...) comes AFTER positive binders + clauses (if (= fe 'and) + (let [[nots others] (reduce (fn [[ns os] c] + (if (not-clause? c) + [(conj ns c) os] + [ns (conj os c)])) + [[] []] + clauses)] + (concat others nots)) + clauses) + nested-and? (and (= fe 'and) (= current-filter 'and))] + (when (seq clauses) - (let [result (build-and-or-not-result - fe clauses current-filter nested-and?) - vars' (set/union (set @vars) (collect-vars result)) - query (cond - nested-and? - result - - (and (zero? level) (contains? #{'and 'or} fe)) - result - - (and (= 'not fe) (some? current-filter)) - result - - :else - [result])] + (let [result (build-and-or-not-result fe clauses current-filter nested-and?) + vars' (set/union (set @vars) (collect-vars result)) + query (cond + nested-and? result + (and (zero? level) (contains? #{'and 'or} fe)) result + (and (= 'not fe) (some? current-filter)) result + :else [result])] (reset! vars vars') {:query query :rules (distinct (mapcat :rules raw-clauses))})))) @@ -343,13 +362,38 @@ [e {:keys [db-graph? private-property?]}] (let [k (if db-graph? (->db-keyword-property (nth e 1)) (->file-keyword-property (nth e 1))) v (nth e 2) - v' (if db-graph? (->db-property-value k v) (->file-property-value v))] + v' (if db-graph? (->db-property-value k v) (->file-property-value v)) + property (when (qualified-keyword? k) + (db-utils/entity k)) + ref-type? (= :db.type/ref (:db/valueType property))] (if db-graph? - (if private-property? - {:query (list 'private-simple-query-property '?b k v') - :rules [:private-simple-query-property]} - {:query (list 'simple-query-property '?b k v') - :rules [:simple-query-property]}) + (let [default-value (if ref-type? + (when-let [value (:logseq.property/default-value property)] + (or (:block/title value) + (:logseq.property/value value))) + (:logseq.property/scalar-default-value property)) + default-value? (and (some? v') (= default-value v')) + rule (if private-property? + (cond + (and ref-type? default-value?) + :private-ref-property-with-default + ref-type? + :private-ref-property + default-value? + :private-scalar-property-with-default + :else + :private-scalar-property) + (cond + (and ref-type? default-value?) + :ref-property-with-default + ref-type? + :ref-property + default-value? + :scalar-property-with-default + :else + :scalar-property))] + {:query (list (symbol (name rule)) '?b k v') + :rules [rule]}) {:query (list 'property '?b k v') :rules [:property]}))) @@ -662,35 +706,69 @@ Some bindings in this fn: (string/replace #"^#" "#tag ") (string/replace tag-placeholder "#"))))) +(defn- lvar? [x] + (and (symbol? x) (= \? (first (name x))))) + +(defn- collect-vars-by-polarity + "Returns {:pos #{?vars} :neg #{?vars}}. + Vars inside (not ...) are counted as negative." + [form] + (let [pos (volatile! #{}) + neg (volatile! #{})] + (letfn [(walk* [x positive?] + (cond + (lvar? x) + (vswap! (if positive? pos neg) conj x) + + (and (seq? x) (= 'not (first x))) + (doseq [c (rest x)] (walk* c false)) + + (sequential? x) + (doseq [c x] (walk* c positive?)) + + (map? x) + (do (doseq [k (keys x)] (walk* k positive?)) + (doseq [v (vals x)] (walk* v positive?))) + + :else nil))] + (walk* form true) + {:pos @pos :neg @neg}))) + (defn- add-bindings! [q {:keys [db-graph?]}] - (let [forms (set (flatten q)) - syms ['?b '?p 'not] - [b? p? not?] (-> (set/intersection (set syms) forms) - (map syms))] - (cond - not? - (cond - (and b? p?) - (concat [['?b :block/uuid] ['?p :block/name] ['?b :block/page '?p]] q) + (let [{:keys [pos neg]} (collect-vars-by-polarity q) - b? - (if db-graph? - ;; This keeps built-in properties from showing up in not results. - ;; May need to be revisited as more class and property filters are explored - (concat [['?b :block/uuid] '[(missing? $ ?b :logseq.property/built-in?)]] q) - (concat [['?b :block/uuid]] q)) + appears? (fn [v] (or (contains? pos v) (contains? neg v))) + needs-domain? (fn [v] (and (appears? v) (not (contains? pos v)))) - p? - (concat [['?p :block/name]] q) + b-need? (needs-domain? '?b) + p-need? (needs-domain? '?p) - :else - q) + ;; CASE 1: both needed → link them, do NOT enumerate all blocks + bindings + (cond + (and b-need? p-need?) + [['?b :block/page '?p]] - (and b? p?) - (concat [['?b :block/page '?p]] q) + ;; CASE 2: only ?b needed → last-resort domain (true global negation) + b-need? + (if db-graph? + [['?b :block/uuid] + '[(missing? $ ?b :logseq.property/built-in?)]] + [['?b :block/uuid]]) - :else + ;; CASE 3: only ?p needed + p-need? + [['?p :block/name]] + + ;; CASE 4: both already positive → optional link (cheap + useful) + (and (contains? pos '?b) (contains? pos '?p)) + [['?b :block/page '?p]] + + :else + nil)] + (if (seq bindings) + (concat bindings q) ;; IMPORTANT: bindings FIRST q))) (defn simplify-query diff --git a/src/main/frontend/db/query_react.cljs b/src/main/frontend/db/query_react.cljs index cd1518f260..bc41873261 100644 --- a/src/main/frontend/db/query_react.cljs +++ b/src/main/frontend/db/query_react.cljs @@ -2,7 +2,6 @@ "Custom queries." (:require [clojure.string :as string] [clojure.walk :as walk] - [frontend.config :as config] [frontend.date :as date] [frontend.db.conn :as conn] [frontend.db.model :as model] @@ -94,27 +93,16 @@ (defn react-query [repo {:keys [query inputs rules] :as query'} query-opts] - (let [pprint (if config/dev? #(when (state/developer-mode?) (apply prn %&)) (fn [_] nil)) - start-time (.now js/performance)] - (when config/dev? (js/console.groupCollapsed "react-query logs:")) - (pprint "================") - (pprint "Use the following to debug your datalog queries:") - (pprint query') - - (let [query (resolve-query query) - repo (or repo (state/get-current-repo)) - db (conn/get-db repo) - resolve-with (select-keys query-opts [:current-page-fn :current-block-uuid]) - resolved-inputs (mapv #(resolve-input db % resolve-with) inputs) - inputs (cond-> resolved-inputs - rules - (conj rules)) - k [:custom - (or (:query-string query') (dissoc query' :title)) - (:today-query? query-opts) - inputs]] - (pprint "inputs (post-resolution):" resolved-inputs) - (pprint "query-opts:" query-opts) - (pprint (str "time elapsed: " (.toFixed (- (.now js/performance) start-time) 2) "ms")) - (when config/dev? (js/console.groupEnd)) - [k (apply react/q repo k query-opts query inputs)]))) + (let [query (resolve-query query) + repo (or repo (state/get-current-repo)) + db (conn/get-db repo) + resolve-with (select-keys query-opts [:current-page-fn :current-block-uuid]) + resolved-inputs (mapv #(resolve-input db % resolve-with) inputs) + inputs (cond-> resolved-inputs + rules + (conj rules)) + k [:custom + (or (:query-string query') (dissoc query' :title)) + (:today-query? query-opts) + inputs]] + [k (apply react/q repo k query-opts query inputs)])) diff --git a/src/main/frontend/worker/db_worker.cljs b/src/main/frontend/worker/db_worker.cljs index 962ba43b09..311131f125 100644 --- a/src/main/frontend/worker/db_worker.cljs +++ b/src/main/frontend/worker/db_worker.cljs @@ -49,6 +49,7 @@ [logseq.db.common.sqlite :as common-sqlite] [logseq.db.common.view :as db-view] [logseq.db.frontend.class :as db-class] + [logseq.db.frontend.property :as db-property] [logseq.db.sqlite.create-graph :as sqlite-create-graph] [logseq.db.sqlite.export :as sqlite-export] [logseq.db.sqlite.gc :as sqlite-gc] @@ -244,8 +245,19 @@ conn (common-sqlite/get-storage-conn storage schema) _ (db-fix/check-and-fix-schema! repo conn) _ (when datoms - (let [data (map (fn [datom] - [:db/add (:e datom) (:a datom) (:v datom)]) datoms)] + (let [eid->datoms (group-by :e datoms) + {properties true non-properties false} (group-by + (fn [[_eid datoms]] + (boolean + (some (fn [datom] (and (= (:a datom) :db/ident) + (db-property/property? (:v datom)))) + datoms))) + eid->datoms) + datoms (concat (mapcat second properties) + (mapcat second non-properties)) + data (map (fn [datom] + [:db/add (:e datom) (:a datom) (:v datom)]) + datoms)] (d/transact! conn data {:initial-db? true}))) client-ops-conn (when-not @*publishing? (common-sqlite/get-storage-conn client-ops-storage @@ -387,7 +399,9 @@ (def-thread-api :thread-api/q [repo inputs] (when-let [conn (worker-state/get-datascript-conn repo)] - (apply d/q (first inputs) @conn (rest inputs)))) + (worker-util/profile + (str "Datalog query: " inputs) + (apply d/q (first inputs) @conn (rest inputs))))) (def-thread-api :thread-api/datoms [repo & args]