diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index d275251765..998a57dfca 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -176,7 +176,6 @@ logseq.db.common.property-util db-property-util logseq.db.common.sqlite common-sqlite logseq.db.common.view db-view - logseq.db.file-based.rules file-rules logseq.db.file-based.schema file-schema logseq.db.file-based.entity-util file-entity-util logseq.db.frontend.class db-class diff --git a/deps/common/.carve/config.edn b/deps/common/.carve/config.edn index 2206bb8cd0..954cbd2144 100644 --- a/deps/common/.carve/config.edn +++ b/deps/common/.carve/config.edn @@ -8,7 +8,6 @@ logseq.common.util.date-time logseq.common.date logseq.common.util.macro - logseq.common.marker logseq.common.config logseq.common.defkeywords] :report {:format :ignore}} diff --git a/deps/common/src/logseq/common/marker.cljs b/deps/common/src/logseq/common/marker.cljs deleted file mode 100644 index d47b773212..0000000000 --- a/deps/common/src/logseq/common/marker.cljs +++ /dev/null @@ -1,13 +0,0 @@ -(ns logseq.common.marker - "Marker patterns. File graph only" - (:require [clojure.string :as string])) - -(defn marker-pattern [format] - (re-pattern - (str "^" (if (= format :markdown) "(#+\\s+)?" "(\\*+\\s+)?") - "(NOW|LATER|TODO|DOING|DONE|WAITING|WAIT|CANCELED|CANCELLED|IN-PROGRESS)?\\s?"))) - - -(defn clean-marker - [content format] - (string/replace-first content (marker-pattern format) "")) diff --git a/deps/db/.clj-kondo/config.edn b/deps/db/.clj-kondo/config.edn index b15523fa08..f71fa58565 100644 --- a/deps/db/.clj-kondo/config.edn +++ b/deps/db/.clj-kondo/config.edn @@ -23,7 +23,6 @@ logseq.db.frontend.property db-property logseq.db.frontend.property.build db-property-build logseq.db.frontend.property.type db-property-type - logseq.db.file-based.rules file-rules logseq.db.file-based.schema file-schema logseq.db.file-based.entity-util file-entity-util logseq.db.frontend.rules rules diff --git a/deps/db/bb.edn b/deps/db/bb.edn index 1a01b08931..f4e2662e37 100644 --- a/deps/db/bb.edn +++ b/deps/db/bb.edn @@ -24,14 +24,11 @@ lint:rules {:requires ([logseq.bb-tasks.lint.datalog :as datalog] - [logseq.db.file-based.rules :as file-rules] [logseq.db.frontend.rules :as rules]) :doc "Lint datalog rules for parsability and unbound variables" :task (datalog/lint-rules (set - (concat (mapcat val (merge file-rules/rules (dissoc rules/rules :self-ref))) - ;; TODO: Update linter to handle false positive on ?str-val for :property - (rules/extract-rules (dissoc file-rules/query-dsl-rules :property)) + (concat (mapcat val (dissoc rules/rules :self-ref)) ;; TODO: Update linter to handle false positive on :task, :priority, :*property* rules (rules/extract-rules (dissoc rules/db-query-dsl-rules :task :priority diff --git a/deps/db/src/logseq/db/file_based/rules.cljc b/deps/db/src/logseq/db/file_based/rules.cljc deleted file mode 100644 index bc0aa6079d..0000000000 --- a/deps/db/src/logseq/db/file_based/rules.cljc +++ /dev/null @@ -1,92 +0,0 @@ -(ns ^:bb-compatible logseq.db.file-based.rules - "Datalog rules for file graphs") - -(def rules - "File graph rules used in db.model queries" - {:namespace - '[[(namespace ?p ?c) - [?c :block/namespace ?p]] - [(namespace ?p ?c) - [?t :block/namespace ?p] - (namespace ?t ?c)]]}) - -(def ^:large-vars/data-var query-dsl-rules - "Rules used by frontend.db.query-dsl for file graphs. The symbols ?b and ?p - respectively refer to block and page. Do not alter them as they are - programmatically built by the query-dsl ns" - {:page-property - '[(page-property ?p ?key ?val) - [?p :block/name] - [?p :block/properties ?prop] - [(get ?prop ?key) ?v] - (or [(= ?v ?val)] [(contains? ?v ?val)])] - - :has-page-property - '[(has-page-property ?p ?key) - [?p :block/name] - [?p :block/properties ?prop] - [(get ?prop ?key)]] - - :task - '[(task ?b ?markers) - [?b :block/marker ?marker] - [(contains? ?markers ?marker)]] - - :priority - '[(priority ?b ?priorities) - [?b :block/priority ?priority] - [(contains? ?priorities ?priority)]] - - :page-tags - '[(page-tags ?p ?tags) - [?p :block/tags ?t] - [?t :block/name ?tag] - [(contains? ?tags ?tag)]] - - :all-page-tags - '[(all-page-tags ?p) - [_ :block/tags ?p]] - - :between - '[(between ?b ?start ?end) - [?b :block/page ?p] - [?p :block/type "journal"] - [?p :block/journal-day ?d] - [(>= ?d ?start)] - [(<= ?d ?end)]] - - :has-property - '[(has-property ?b ?prop) - [?b :block/properties ?bp] - [(missing? $ ?b :block/name)] - [(get ?bp ?prop)]] - - :block-content - '[(block-content ?b ?query) - [?b :block/title ?content] - [(clojure.string/includes? ?content ?query)]] - - :page - '[(page ?b ?page-name) - [?b :block/page ?bp] - [?bp :block/name ?page-name]] - - :namespace - '[(namespace ?p ?namespace) - [?p :block/namespace ?parent] - [?parent :block/name ?namespace]] - - :property - '[(property ?b ?key ?val) - [?b :block/properties ?prop] - [(missing? $ ?b :block/name)] - [(get ?prop ?key) ?v] - [(str ?val) ?str-val] - (or [(= ?v ?val)] - [(contains? ?v ?val)] - ;; For integer pages that aren't strings - [(contains? ?v ?str-val)])] - - :page-ref - '[(page-ref ?b ?ref) - (has-ref ?b ?ref)]}) diff --git a/deps/db/src/logseq/db/frontend/rules.cljc b/deps/db/src/logseq/db/frontend/rules.cljc index d9cb8d69f6..36f8031f5f 100644 --- a/deps/db/src/logseq/db/frontend/rules.cljc +++ b/deps/db/src/logseq/db/frontend/rules.cljc @@ -1,7 +1,5 @@ (ns ^:bb-compatible logseq.db.frontend.rules - "Datalog rules mostly for DB graphs. `rules` - is the only var also used by file graphs" - (:require [logseq.db.file-based.rules :as file-rules])) + "Datalog rules for DB graphs") (def ^:large-vars/data-var rules "Rules used mainly in frontend.db.model for both DB and file graphs" @@ -83,12 +81,23 @@ (def ^:large-vars/data-var db-query-dsl-rules "Rules used by frontend.query.dsl for DB graphs" (merge - (dissoc file-rules/query-dsl-rules :namespace - :page-property :has-page-property - :page-tags :all-page-tags) rules - {:between + {:page-ref + '[(page-ref ?b ?ref) + (has-ref ?b ?ref)] + + :block-content + '[(block-content ?b ?query) + [?b :block/title ?content] + [(clojure.string/includes? ?content ?query)]] + + :page + '[(page ?b ?page-name) + [?b :block/page ?bp] + [?bp :block/name ?page-name]] + + :between '[(between ?b ?start ?end) [?b :block/page ?p] [?p :block/tags :logseq.class/Journal] diff --git a/deps/db/src/logseq/db/sqlite/build.cljs b/deps/db/src/logseq/db/sqlite/build.cljs index e1f9bc06ea..70f5f13582 100644 --- a/deps/db/src/logseq/db/sqlite/build.cljs +++ b/deps/db/src/logseq/db/sqlite/build.cljs @@ -550,8 +550,8 @@ (remove existing-pages)))) distinct (map #(hash-map :page {:block/title %})))] - (when (seq new-pages-from-refs) - (println "Building additional pages from content refs:" (pr-str (mapv #(get-in % [:page :block/title]) new-pages-from-refs)))) + ;; (when (seq new-pages-from-refs) + ;; (prn :debug "Building additional pages from content refs:" (pr-str (mapv #(get-in % [:page :block/title]) new-pages-from-refs)))) (concat new-pages-from-refs pages-and-blocks))) (defn- add-new-pages-from-properties @@ -565,9 +565,9 @@ distinct (remove existing-pages) (map #(hash-map :page %)))] - (when (seq new-pages) - (println "Building additional pages from property values:" - (pr-str (mapv #(or (get-in % [:page :block/title]) (get-in % [:page :build/journal])) new-pages)))) + ;; (when (seq new-pages) + ;; (prn :debug "Building additional pages from property values:" + ;; (pr-str (mapv #(or (get-in % [:page :block/title]) (get-in % [:page :build/journal])) new-pages)))) ;; new-pages must come first because they are referenced by pages-and-blocks (concat new-pages pages-and-blocks))) diff --git a/scripts/src/logseq/tasks/dev/db_and_file_graphs.clj b/scripts/src/logseq/tasks/dev/db_and_file_graphs.clj index bf6423e16f..96b534b21a 100644 --- a/scripts/src/logseq/tasks/dev/db_and_file_graphs.clj +++ b/scripts/src/logseq/tasks/dev/db_and_file_graphs.clj @@ -141,8 +141,7 @@ allowed-exceptions #{":block/pre-block? :block/scheduled :block/deadline :block/type :block/name :block/marker" "(dissoc :block/format))]" "{:block/name page-title})" - ;; TODO: Mv these 2 file-based ns out of db files - "(:require [logseq.db.file-based.rules :as file-rules]))" + ;; TODO: Mv this file-based ns out of db file "[logseq.db.file-based.schema :as file-schema]))" ;; :block/name ones from src/main/mobile "(if-let [journal (db/get-page page-name)]" diff --git a/src/main/frontend/commands.cljs b/src/main/frontend/commands.cljs index a1e0899f1d..05440c8d83 100644 --- a/src/main/frontend/commands.cljs +++ b/src/main/frontend/commands.cljs @@ -34,11 +34,11 @@ [:li.mb-1 [:code "{{query #tag}}"]] [:li.mb-1 [:code "{{query [[page]]}}"]] [:li.mb-1 [:code "{{query \"full-text search\"}}"]] - [:li.mb-1 [:code "{{query (and [[project]] (task NOW LATER))}}"]] + [:li.mb-1 [:code "{{query (and [[project]] (task Todo Doing))}}"]] [:li.mb-1 [:code "{{query (or [[page 1]] [[page 2]])}}"]] - [:li.mb-1 [:code "{{query (and (between -7d +7d) (task DONE))}}"]] + [:li.mb-1 [:code "{{query (and (between -7d +7d) (task Done))}}"]] [:li.mb-1 [:code "{{query (property key value)}}"]] - [:li.mb-1 [:code "{{query (page-tags #tag)}}"]]] + [:li.mb-1 [:code "{{query (tags #tag)}}"]]] [:p "Check more examples at " [:a {:href "https://docs.logseq.com/#/page/queries" diff --git a/src/main/frontend/components/query/builder.cljs b/src/main/frontend/components/query/builder.cljs index 1364e1226f..417573b14c 100644 --- a/src/main/frontend/components/query/builder.cljs +++ b/src/main/frontend/components/query/builder.cljs @@ -302,9 +302,6 @@ (map name filters-and-ops) (fn [{:keys [value]}] (cond - (= value "all page tags") - (append-tree! *tree opts loc [:all-page-tags]) - (operator? value) (append-tree! *tree opts loc [(keyword value)]) @@ -348,7 +345,7 @@ (= (keyword f) :page-ref) (ref/->page-ref (uuid->page-title (second clause))) - (contains? #{:tags :page-tags} (keyword f)) + (contains? #{:tags} (keyword f)) (cond (string? (second clause)) (str "#" (uuid->page-title (second clause))) @@ -357,7 +354,7 @@ :else (str "#" (uuid->page-title (second (second clause))))) - (contains? #{:property :private-property :page-property} (keyword f)) + (contains? #{:property :private-property} (keyword f)) (str (if (qualified-keyword? (second clause)) (:block/title (db/entity (second clause))) (some-> (second clause) name)) @@ -412,7 +409,7 @@ (str (name f) ": " (string/join " | " (rest clause))) - (contains? #{:page :task :namespace} (keyword f)) + (contains? #{:page :task} (keyword f)) (str (name f) ": " (if (vector? (second clause)) (second (second clause)) (second clause))) @@ -568,7 +565,7 @@ (assoc state ::tree *tree))) :will-mount (fn [state] (let [q-str (get-q (first (:rum/args state))) - blocks-query? (:blocks? (query-dsl/parse-query q-str)) + blocks-query? (:blocks? (query-dsl/parse-query q-str (db/get-db))) find-mode (cond blocks-query? :block diff --git a/src/main/frontend/db/query_dsl.cljs b/src/main/frontend/db/query_dsl.cljs index e6041b3164..c05c5f1a12 100644 --- a/src/main/frontend/db/query_dsl.cljs +++ b/src/main/frontend/db/query_dsl.cljs @@ -6,24 +6,18 @@ [clojure.set :as set] [clojure.string :as string] [clojure.walk :as walk] - [frontend.config :as config] + [datascript.core :as d] [frontend.date :as date] [frontend.db.conn :as db-conn] [frontend.db.query-react :as query-react] - [frontend.db.utils :as db-utils] - [frontend.state :as state] [frontend.template :as template] - [frontend.util :as util] - [frontend.util.text :as text-util] [logseq.common.util :as common-util] [logseq.common.util.date-time :as date-time-util] [logseq.common.util.page-ref :as page-ref] [logseq.db :as ldb] - [logseq.db.file-based.rules :as file-rules] [logseq.db.frontend.class :as db-class] [logseq.db.frontend.property :as db-property] - [logseq.db.frontend.rules :as rules] - [logseq.graph-parser.text :as text])) + [logseq.db.frontend.rules :as rules])) ;; Query fields: @@ -45,14 +39,6 @@ ;; sample ;; full-text-search "" -;; namespace -;; page-property (page) -;; page-tags (page) -;; all-page-tags - -;; Sort by (field, asc/desc): -;; (sort-by created-at asc) - ;; Time helpers ;; ============ (defn- ->journal-day-int [input] @@ -88,7 +74,7 @@ (let [input (string/lower-case (name input))] (cond (= "now" input) - (util/time-ms) + (common-util/time-ms) (= "today" input) (tc/to-long (t/today)) @@ -228,6 +214,11 @@ ;; build-query fns ;; =============== + +;; Current DB when query is run +(def ^:dynamic *current-db* + nil) + (defn- resolve-timestamp-property [e] (let [k (second e)] @@ -237,7 +228,7 @@ (string/lower-case) (string/replace "_" "-") keyword)] - (if (and (config/db-based-graph?) (db-property/property? k')) + (if (db-property/property? k') k' (case k' :created-at @@ -249,11 +240,8 @@ (defn get-timestamp-property [e] (when-let [k (resolve-timestamp-property e)] - (if (config/db-based-graph?) - (when (keyword? k) - k) - (when (contains? #{:block/created-at :block/updated-at} k) - k)))) + (when (keyword? k) + k))) (defn- build-journal-between-two-arg [e] @@ -263,19 +251,6 @@ {:query (list 'between '?b start end) :rules [:between]})) -(defn- file-based-build-between-three-arg - [e] - (when-let [k (get-timestamp-property e)] - (let [start (->timestamp (nth e 2)) - end (->timestamp (nth e 3))] - (when (and start end) - (let [[start end] (sort [start end]) - sym '?v] - {:query [['?b :block/properties '?prop] - [(list 'get '?prop k) sym] - [(list '>= sym start)] - [(list '< sym end)]]}))))) - (defn- db-based-build-between-three-arg [e] (when-let [k (get-timestamp-property e)] @@ -293,34 +268,17 @@ (db-based-build-between-three-arg (concat e ['now]))) (defn- build-between - [e {:keys [db-graph?]}] + [e] (cond (= 3 (count e)) (let [k (get-timestamp-property e)] - (if (and db-graph? k) + (if k (db-based-build-between-two-arg e) (build-journal-between-two-arg e))) ;; (between created_at -1d today) (= 4 (count e)) - (if db-graph? - (db-based-build-between-three-arg e) - (file-based-build-between-three-arg e)))) - -(defn ->file-property-value - "Parses property values for file graphs and handles non-string values or any page-ref like values" - [v*] - (if (some? v*) - (let [v (str v*) - result (if-some [res (text/parse-non-string-property-value v)] - res - (if (string/starts-with? v "#") - (subs v 1) - (or (page-ref/get-page-name v) v)))] - (if (string? result) - (or (parse-double result) (string/trim result)) - result)) - v*)) + (db-based-build-between-three-arg e))) (defn ->db-property-value "Parses property values for DB graphs" @@ -331,82 +289,70 @@ (subs v' 1) (or (page-ref/get-page-name v') v')) ;; Convert number pages to string - (and (double? v) (= :node (:logseq.property/type (db-utils/entity k)))) + (and (double? v) (= :node (:logseq.property/type (d/entity *current-db* k)))) (str v) :else v'))) -(defn- ->file-keyword-property - "Case-insensitive property names for file graphs. Users manually type queries to enter them as they appear" - [property-name] - (-> (string/replace (name property-name) "_" "-") - string/lower-case - keyword)) - (defn- ->db-keyword-property "Returns property db-ident given case sensitive property names for db graphs" [property-name] (if (qualified-keyword? property-name) property-name (or (some->> (name property-name) - (db-utils/q '[:find [(pull ?b [:db/ident]) ...] - :in $ ?title - :where [?b :block/tags :logseq.class/Property] [?b :block/title ?title]]) + (d/q '[:find [(pull ?b [:db/ident]) ...] + :in $ ?title + :where [?b :block/tags :logseq.class/Property] [?b :block/title ?title]] + *current-db*) first :db/ident) ;; Don't return nil as that incorrectly matches all properties ::no-property-found))) (defn- build-property-two-arg - [e {:keys [db-graph? private-property?]}] - (let [k (if db-graph? (->db-keyword-property (nth e 1)) (->file-keyword-property (nth e 1))) + [e {:keys [private-property?]}] + (let [k (->db-keyword-property (nth e 1)) v (nth e 2) - v' (if db-graph? (->db-property-value k v) (->file-property-value v)) + v' (->db-property-value k v) property (when (qualified-keyword? k) - (db-utils/entity k)) - ref-type? (= :db.type/ref (:db/valueType property))] - (if db-graph? - (let [default-value (if ref-type? - (when-let [value (:logseq.property/default-value property)] - (or (:block/title value) - (:logseq.property/value value))) - (:logseq.property/scalar-default-value property)) - default-value? (and (some? v') (= default-value v')) - rule (if private-property? - (cond - (and ref-type? default-value?) - :private-ref-property-with-default - ref-type? - :private-ref-property - default-value? - :private-scalar-property-with-default - :else - :private-scalar-property) - (cond - (and ref-type? default-value?) - :ref-property-with-default - ref-type? - :ref-property - default-value? - :scalar-property-with-default - :else - :scalar-property))] - {:query (list (symbol (name rule)) '?b k v') - :rules [rule]}) - {:query (list 'property '?b k v') - :rules [:property]}))) + (d/entity *current-db* k)) + ref-type? (= :db.type/ref (:db/valueType property)) + default-value (if ref-type? + (when-let [value (:logseq.property/default-value property)] + (or (:block/title value) + (:logseq.property/value value))) + (:logseq.property/scalar-default-value property)) + default-value? (and (some? v') (= default-value v')) + rule (if private-property? + (cond + (and ref-type? default-value?) + :private-ref-property-with-default + ref-type? + :private-ref-property + default-value? + :private-scalar-property-with-default + :else + :private-scalar-property) + (cond + (and ref-type? default-value?) + :ref-property-with-default + ref-type? + :ref-property + default-value? + :scalar-property-with-default + :else + :scalar-property))] + {:query (list (symbol (name rule)) '?b k v') + :rules [rule]})) (defn- build-property-one-arg - [e {:keys [db-graph? private-property?]}] - (let [k (if db-graph? (->db-keyword-property (nth e 1)) (->file-keyword-property (nth e 1)))] - (if db-graph? - (if private-property? - {:query (list 'has-private-simple-query-property '?b k) - :rules [:has-private-simple-query-property]} - {:query (list 'has-simple-query-property '?b k) - :rules [:has-simple-query-property]}) - {:query (list 'has-property '?b k) - :rules [:has-property]}))) + [e {:keys [private-property?]}] + (let [k (->db-keyword-property (nth e 1))] + (if private-property? + {:query (list 'has-private-simple-query-property '?b k) + :rules [:has-private-simple-query-property]} + {:query (list 'has-simple-query-property '?b k) + :rules [:has-simple-query-property]}))) (defn- build-property [e env] (cond @@ -417,85 +363,46 @@ (build-property-one-arg e env))) (defn- build-task - [e {:keys [db-graph?]}] + [e] (let [markers (if (coll? (first (rest e))) (first (rest e)) (rest e))] (when (seq markers) - (if db-graph? - (let [markers' (set (map (comp common-util/capitalize-all name) markers))] - {:query (list 'task '?b (set markers')) - :rules [:task]}) - (let [markers (set (map (comp string/upper-case name) markers))] - {:query (list 'task '?b markers) - :rules [:task]}))))) + (let [markers' (set (map (comp common-util/capitalize-all name) markers))] + {:query (list 'task '?b (set markers')) + :rules [:task]})))) (defn- build-priority - [e {:keys [db-graph?]}] + [e] (let [priorities (if (coll? (first (rest e))) (first (rest e)) (rest e))] (when (seq priorities) - (if db-graph? - (let [priorities (set (map (comp string/capitalize name) priorities))] - {:query (list 'priority '?b priorities) - :rules [:priority]}) - (let [priorities (set (map (comp string/upper-case name) priorities))] - {:query (list 'priority '?b priorities) - :rules [:priority]}))))) - -(defn- build-page-property - [e] - (let [[k v] (rest e) - k' (->file-keyword-property k)] - (if (some? v) - (let [v' (->file-property-value v)] - {:query (list 'page-property '?p k' v') - :rules [:page-property]}) - {:query (list 'has-page-property '?p k') - :rules [:has-page-property]}))) + (let [priorities (set (map (comp string/capitalize name) priorities))] + {:query (list 'priority '?b priorities) + :rules [:priority]})))) (defn- build-tags - [e {:keys [db-graph?]}] + [e] (let [tags (if (coll? (first (rest e))) (first (rest e)) (rest e)) tags (map name tags)] (when (seq tags) - (let [tags (set (map (comp page-ref/get-page-name!) tags)) - lower-cased-tags (set (map (comp page-ref/get-page-name! string/lower-case) tags))] + (let [tags (set (map (comp page-ref/get-page-name!) tags))] {:query (list 'tags - (if db-graph? '?b '?p) - (if db-graph? - (let [db (db-conn/get-db)] - (->> tags - (mapcat (fn [tag-name] - (when-let [tag-id (if (common-util/uuid-string? tag-name) - [:block/uuid (uuid tag-name)] - (first (ldb/page-exists? db tag-name #{:logseq.class/Tag})))] - (when-let [tag (db-utils/entity tag-id)] - (->> (db-class/get-structured-children db (:db/id tag)) - (cons (:db/id tag))))))) - set)) - lower-cased-tags)) + '?b + (->> tags + (mapcat (fn [tag-name] + (when-let [tag-id (if (common-util/uuid-string? tag-name) + [:block/uuid (uuid tag-name)] + (first (ldb/page-exists? *current-db* tag-name #{:logseq.class/Tag})))] + (when-let [tag (d/entity *current-db* tag-id)] + (->> (db-class/get-structured-children *current-db* (:db/id tag)) + (cons (:db/id tag))))))) + set)) :rules [:tags]})))) -(defn- build-page-tags - [e] - (let [tags (if (coll? (first (rest e))) - (first (rest e)) - (rest e)) - tags (map (comp string/lower-case name) tags)] - (when (seq tags) - (let [tags (set (map (comp page-ref/get-page-name! string/lower-case name) tags))] - {:query (list 'page-tags '?p tags) - :rules [:page-tags]})))) - -(defn- build-all-page-tags - [] - {:query (list 'all-page-tags '?p) - :rules [:all-page-tags]}) - (defn- build-sample [e sample] (when-let [num (second e)] @@ -504,46 +411,18 @@ ;; blank b/c this post-process filter doesn't effect query {}))) -(defn- build-sort-by - [e sort-by_] - (let [[k order*] (map keyword (rest e)) - order (if (contains? #{:asc :desc} order*) - order* - :desc) - comp (if (= order :desc) - ;; Handle nil so that is always less than a value e.g. treated as a "" for a string. - ;; Otherwise sort bugs occur that prevent non-nil values from being sorted - #(if (nil? %2) true (>= %1 %2)) - #(if (nil? %1) true (<= %1 %2)))] - (reset! sort-by_ - (fn sort-results [result property-val-fn] - ;; first because there is one binding result in query-wrapper - (sort-by #(-> % first (property-val-fn k)) - comp - result))) - ;; blank b/c this post-process filter doesn't effect query - {})) - (defn- build-page [e] (let [page-name (page-ref/get-page-name! (str (first (rest e)))) - page-name (util/page-name-sanity-lc page-name)] + page-name (common-util/page-name-sanity-lc page-name)] {:query (list 'page '?b page-name) :rules [:page]})) -(defn- build-namespace - [e] - (let [page-name (page-ref/get-page-name! (str (first (rest e)))) - page (util/page-name-sanity-lc page-name)] - (when-not (string/blank? page) - {:query (list 'namespace '?p page) - :rules [:namespace]}))) - (defn- build-page-ref [e] (let [page-name (-> (page-ref/get-page-name! e) - (util/page-name-sanity-lc)) - page (ldb/get-page (db-conn/get-db) page-name)] + (common-util/page-name-sanity-lc)) + page (ldb/get-page *current-db* page-name)] (when page {:query (list 'page-ref '?b (:db/id page)) :rules [:page-ref]}))) @@ -551,8 +430,8 @@ (defn- build-self-ref [e] (let [page-name (-> (page-ref/get-page-name! e) - (util/page-name-sanity-lc)) - page (ldb/get-page (db-conn/get-db) page-name)] + (common-util/page-name-sanity-lc)) + page (ldb/get-page *current-db* page-name)] (when page {:query (list 'self-ref '?b (:db/id page)) :rules [:self-ref]}))) @@ -574,24 +453,6 @@ ;; function (list? (first e))))))) -(defn- build-file-query - [e fe {:keys [sort-by]}] - (cond - (= 'namespace fe) - (build-namespace e) - - (= 'page-property fe) - (build-page-property e) - - (= 'page-tags fe) - (build-page-tags e) - - (= 'all-page-tags fe) - (build-all-page-tags) - - (= 'sort-by fe) - (build-sort-by e sort-by))) - (defn build-query "This fn converts a form/list in a query e.g. `(operator arg1 arg2)` to its datalog equivalent. This fn is called recursively on sublists for boolean operators @@ -616,9 +477,6 @@ Some bindings in this fn: (string/lower-case (str fe)))) page-ref? (page-ref/page-ref? e)] (when (or - (:db-graph? env) - (and page-ref? - (not (contains? #{'page-property 'page-tags} current-filter))) (contains? #{'between 'property 'private-property 'todo 'task 'priority 'page} fe) (and (not page-ref?) (string? e))) (reset! blocks? true)) @@ -626,7 +484,7 @@ Some bindings in this fn: (nil? e) nil - (and (:db-graph? env) (datalog-clause? e)) + (datalog-clause? e) {:query [e] :rules []} @@ -645,7 +503,7 @@ Some bindings in this fn: (build-and-or-not e env level fe) (= 'between fe) - (build-between e env) + (build-between e) (= 'property fe) (build-property e env) @@ -655,10 +513,10 @@ Some bindings in this fn: ;; task is the new name and todo is the old one (or (= 'todo fe) (= 'task fe)) - (build-task e env) + (build-task e) (= 'priority fe) - (build-priority e env) + (build-priority e) (= 'page fe) (build-page e) @@ -667,10 +525,7 @@ Some bindings in this fn: (build-sample e sample) (= 'tags fe) - (build-tags e env) - - (not (:db-graph? env)) - (build-file-query e fe env) + (build-tags e) :else nil)))) @@ -688,18 +543,18 @@ Some bindings in this fn: (str "\"" page-ref/left-brackets match' page-ref/right-brackets "\"")))] (some-> s (string/replace #"\"?\[\[(.*?)\]\]\"?" quoted-page-ref) - (string/replace text-util/between-re + (string/replace #"\(between ([^\)]+)\)" (fn [[_ x]] (->> (string/split x #" ") (remove string/blank?) (map (fn [x] (if (or (contains? #{"+" "-"} (first x)) - (and (util/safe-re-find #"\d" (first x)) + (and (common-util/safe-re-find #"\d" (first x)) (some #(string/ends-with? x %) ["y" "m" "d" "h" "min"]))) (keyword (name x)) x))) (string/join " ") - (util/format "(between %s)")))) + (common-util/format "(between %s)")))) (string/replace #"\"[^\"]+\"" (fn [s] (string/replace s "#" tag-placeholder))) (string/replace " #" " #tag ") (string/replace #"^#" "#tag ") @@ -734,7 +589,7 @@ Some bindings in this fn: {:pos @pos :neg @neg}))) (defn- add-bindings! - [q {:keys [db-graph?]}] + [q] (let [{:keys [pos neg]} (collect-vars-by-polarity q) appears? (fn [v] (or (contains? pos v) (contains? neg v))) @@ -751,10 +606,8 @@ Some bindings in this fn: ;; CASE 2: only ?b needed → last-resort domain (true global negation) b-need? - (if db-graph? - [['?b :block/uuid] - '[(missing? $ ?b :logseq.property/built-in?)]] - [['?b :block/uuid]]) + [['?b :block/uuid] + '[(missing? $ ?b :logseq.property/built-in?)]] ;; CASE 3: only ?p needed p-need? @@ -786,48 +639,42 @@ Some bindings in this fn: (def custom-readers {:readers {'tag (fn [x] (page-ref/->page-ref x))}}) (defn parse - [s {:keys [db-graph? cards?] :as opts}] + [s db {:keys [cards?]}] (when (and (string? s) (not (string/blank? s))) - (let [s (if (= \# (first s)) (page-ref/->page-ref (subs s 1)) s) - form (some->> s - (pre-transform) - (reader/read-string custom-readers)) - sort-by (atom nil) - blocks? (atom nil) - sample (atom nil) - form (simplify-query form) - {result :query rules :rules} - (when form (build-query form {:form form - :sort-by sort-by - :blocks? blocks? - :db-graph? db-graph? - :sample sample - :cards? cards?})) - result' (when (seq result) - (let [key (if (coll? (first result)) + (binding [*current-db* db] + (let [s (if (= \# (first s)) (page-ref/->page-ref (subs s 1)) s) + form (some->> s + (pre-transform) + (reader/read-string custom-readers)) + sort-by (atom nil) + blocks? (atom nil) + sample (atom nil) + form (simplify-query form) + {result :query rules :rules} + (when form (build-query form {:form form + :sort-by sort-by + :blocks? blocks? + :sample sample + :cards? cards?})) + result' (when (seq result) + (let [key (if (coll? (first result)) ;; Only queries for this branch are not's like: ;; [(not (page-ref ?b "page 2"))] - (keyword (ffirst result)) - (keyword (first result)))] - (add-bindings! (if (= key :and) (rest result) result) opts))) - extract-rules (fn [rules] - (rules/extract-rules rules/db-query-dsl-rules rules {:deps rules/rules-dependencies})) - rules' (if db-graph? - (let [rules' (if (contains? (set rules) :page-ref) - (conj (set rules) :self-ref) - rules)] - (extract-rules rules')) - (->> (concat (map file-rules/query-dsl-rules (remove #{:page-ref} rules)) - (when (some #{:page-ref :self-ref} rules) - (extract-rules [:self-ref :page-ref]))) - (remove nil?) - vec))] - {:query result' - :rules rules' - :sort-by @sort-by - :blocks? (boolean @blocks?) - :sample sample}))) + (keyword (ffirst result)) + (keyword (first result)))] + (add-bindings! (if (= key :and) (rest result) result)))) + extract-rules (fn [rules] + (rules/extract-rules rules/db-query-dsl-rules rules {:deps rules/rules-dependencies})) + rules' (let [rules' (if (contains? (set rules) :page-ref) + (conj (set rules) :self-ref) + rules)] + (extract-rules rules'))] + {:query result' + :rules rules' + :sort-by @sort-by + :blocks? (boolean @blocks?) + :sample sample})))) ;; Main fns ;; ======== @@ -847,10 +694,10 @@ Some bindings in this fn: (conj q where)))) (defn parse-query - ([q] (parse-query q {})) - ([q options] + ([q db] (parse-query q db {})) + ([q db options] (let [q' (template/resolve-dynamic-template! q)] - (parse q' (merge {:db-graph? (config/db-based-graph? (state/get-current-repo))} options))))) + (parse q' db options)))) (defn pre-transform-query [q] @@ -868,24 +715,22 @@ Some bindings in this fn: (query repo query-string {})) ([repo query-string query-opts] (when (and (string? query-string) (not= "\"\"" query-string)) - (let [db-graph? (config/db-based-graph? repo) - {query* :query :keys [rules sort-by blocks? sample]} (parse-query query-string {:cards? (:cards? query-opts)}) + (let [db (db-conn/get-db) + {query* :query :keys [rules sample]} (parse-query query-string db {:cards? (:cards? query-opts)}) query* (if (:cards? query-opts) - (let [card-id (:db/id (db-utils/entity :logseq.class/Card))] - (util/concat-without-nil + (let [card-id (:db/id (d/entity db :logseq.class/Card))] + (common-util/concat-without-nil [['?b :block/tags card-id]] (if (coll? (first query*)) query* [query*]))) query*) - blocks? (if db-graph? true blocks?)] + blocks? true] (when-let [query' (some-> query* (query-wrapper {:blocks? blocks? - :block-attrs (when db-graph? db-block-attrs)}))] + :block-attrs db-block-attrs}))] (let [random-samples (if (and sample @sample) (fn [col] (take @sample (shuffle col))) identity) - sort-by' (if (and sort-by (not (config/db-based-graph? repo))) - #(sort-by % (fn [m prop] (get-in m [:block/properties prop]))) - identity) + sort-by' identity transform-fn (comp sort-by' random-samples)] (last (query-react/react-query repo {:query query' @@ -899,11 +744,11 @@ Some bindings in this fn: "Runs a dsl query with query as a seq. Primary use is from advanced query" [repo query-m query-opts] (when (seq (:query query-m)) - (let [query-string (template/resolve-dynamic-template! (pr-str (:query query-m))) - db-graph? (config/db-based-graph? repo) - {query* :query :keys [sort-by blocks? rules]} (parse query-string {:db-graph? db-graph?})] + (let [db (db-conn/get-db) + query-string (template/resolve-dynamic-template! (pr-str (:query query-m))) + {query* :query :keys [sort-by blocks? rules]} (parse query-string db {})] (when-let [query' (some-> query* (query-wrapper {:blocks? blocks? - :block-attrs (when db-graph? db-block-attrs)}))] + :block-attrs db-block-attrs}))] (last (query-react/react-query repo (merge query-m @@ -912,10 +757,7 @@ Some bindings in this fn: (merge query-opts (when sort-by - {:transform-fn - (if db-graph? - identity - #(sort-by % (fn [m prop] (get-in m [:block/properties prop]))))})))))))) + {:transform-fn identity})))))))) (comment (query "(and [[foo]] [[bar]])") @@ -937,8 +779,4 @@ Some bindings in this fn: (query "(and [[some page]] (priority A))") ;; nested query - (query "(and [[baz]] (or [[foo]] [[bar]]))") - - (query "(and [[some page]] (sort-by created-at))") - - (query "(and (page-property foo bar) [[hello]])")) + (query "(and [[baz]] (or [[foo]] [[bar]]))")) diff --git a/src/main/frontend/db/query_react.cljs b/src/main/frontend/db/query_react.cljs index 10d6f79b32..e5b07ff5e6 100644 --- a/src/main/frontend/db/query_react.cljs +++ b/src/main/frontend/db/query_react.cljs @@ -9,7 +9,6 @@ [frontend.db.utils :as db-utils] [frontend.extensions.sci :as sci] [frontend.state :as state] - [frontend.util :as util] [lambdaisland.glogi :as log] [logseq.common.util.page-ref :as page-ref] [logseq.db :as ldb] @@ -84,11 +83,6 @@ page-ref (string/lower-case page-ref)] (list 'contains? sym (page-ref/get-page-name page-ref))) - (and (vector? f) - (= (first f) 'page-property) - (keyword? (util/nth-safe f 2))) - (update f 2 (fn [k] (keyword (string/replace (name k) "_" "-")))) - :else f)) query))) diff --git a/src/main/frontend/extensions/fsrs.cljs b/src/main/frontend/extensions/fsrs.cljs index 45998f3e09..ae90ea9e23 100644 --- a/src/main/frontend/extensions/fsrs.cljs +++ b/src/main/frontend/extensions/fsrs.cljs @@ -90,7 +90,7 @@ (when-let [query (:logseq.property/query cards)] (when-not (string/blank? (:block/title query)) (:block/title query)))) - result (query-dsl/parse query {:db-graph? true}) + result (query-dsl/parse query (db/get-db) {}) card-tag-id (:db/id (db/entity :logseq.class/Card)) card-tag-children-ids (db-model/get-structured-children repo card-tag-id) card-ids (cons card-tag-id card-tag-children-ids) diff --git a/src/main/frontend/handler/query/builder.cljs b/src/main/frontend/handler/query/builder.cljs index 7af01cb881..7f9d58c0bd 100644 --- a/src/main/frontend/handler/query/builder.cljs +++ b/src/main/frontend/handler/query/builder.cljs @@ -144,20 +144,17 @@ (and (vector? f) (= :tags (keyword (first f)))) [(symbol :tags) (->page-ref (second f))] - (and (vector? f) (= :page-tags (keyword (first f)))) - [(symbol :page-tags) (->page-ref (second f))] - (and (vector? f) (= :between (keyword (first f)))) (into [(symbol :between)] (map ->page-ref (rest f))) ;; property key value - (and (vector? f) (= 3 (count f)) (contains? #{:page-property :property :private-property} (keyword (first f)))) + (and (vector? f) (= 3 (count f)) (contains? #{:property :private-property} (keyword (first f)))) (let [l (if (page-ref/page-ref? (str (last f))) (symbol (last f)) (last f))] (into [(symbol (first f))] [(second f) l])) - (and (vector? f) (contains? #{:page :tags :namespace} (keyword (first f)))) + (and (vector? f) (contains? #{:page :tags} (keyword (first f)))) (into [(symbol (first f))] (map ->page-ref (rest f))) :else f)) diff --git a/src/main/frontend/util/text.cljs b/src/main/frontend/util/text.cljs index 3b2e9c312e..c9a9135059 100644 --- a/src/main/frontend/util/text.cljs +++ b/src/main/frontend/util/text.cljs @@ -7,8 +7,6 @@ [goog.string :as gstring] [logseq.cli.text-util :as cli-text-util])) -(defonce between-re #"\(between ([^\)]+)\)") - (def bilibili-regex #"^((?:https?:)?//)?((?:www).)?((?:bilibili.com))(/(?:video/)?)([\w-]+)(\?p=(\d+))?(\S+)?$") (def loom-regex #"^((?:https?:)?//)?((?:www).)?((?:loom.com))(/(?:share/|embed/))([\w-]+)(\S+)?$") (def vimeo-regex #"^((?:https?:)?//)?((?:www).)?((?:player.vimeo.com|vimeo.com))(/(?:video/)?)([\w-]+)(\S+)?$") @@ -32,24 +30,6 @@ [media-formats s] (some (fn [fmt] (util/safe-re-find (re-pattern (str "(?i)\\." fmt "(?:\\?([^#]*))?(?:#(.*))?$")) s)) media-formats)) -(defn add-timestamp - [content key value] - (let [new-line (str (string/upper-case key) ": " value) - lines (string/split-lines content) - new-lines (map (fn [line] - (string/trim - (if (string/starts-with? (string/lower-case line) key) - new-line - line))) - lines) - new-lines (if (not= (map string/trim lines) new-lines) - new-lines - (cons (first new-lines) ;; title - (cons - new-line - (rest new-lines))))] - (string/join "\n" new-lines))) - (defn get-current-line-by-pos [s pos] (let [lines (string/split-lines s) diff --git a/src/test/frontend/db/query_dsl_test.cljs b/src/test/frontend/db/query_dsl_test.cljs index 3548870e95..08d7408d52 100644 --- a/src/test/frontend/db/query_dsl_test.cljs +++ b/src/test/frontend/db/query_dsl_test.cljs @@ -463,8 +463,8 @@ :blocks [{:build.test/title "DONE b1 [[page 1]] [[page 3]]"} {:build.test/title "DONE b2Z [[page 1]]"}]} {:page {:block/title "page2", :build/properties {:foo "bar"}} - :blocks [{:build.test/title "NOW b3 [[page 1]]"} - {:build.test/title "LATER b4Z [[page 2]]"}]}]) + :blocks [{:build.test/title "DOING b3 [[page 1]]"} + {:build.test/title "TODO b4Z [[page 2]]"}]}]) (let [task-filter "(task doing todo)"] (is (= [] @@ -635,7 +635,7 @@ (load-test-files [{:page {:build/journal 20201226} :blocks [{:build.test/title "DONE 26-b1", :block/created-at 1608968448113} - {:build.test/title "LATER 26-b2-modified-later", :block/created-at 1608968448114} + {:build.test/title "TODO 26-b2-modified-later", :block/created-at 1608968448114} {:build.test/title "DONE 26-b3", :block/created-at 1608968448115} {:block/title "26-b4", :block/created-at 1608968448116}]}]) @@ -654,9 +654,9 @@ (deftest custom-query-test (load-test-files [{:page {:block/title "page1", :build/properties {:foo "bar"}} - :blocks [{:build.test/title "NOW b1"} + :blocks [{:build.test/title "DOING b1"} {:build.test/title "TODO b2"} - {:build.test/title "LATER b3"} + {:build.test/title "TODO b3"} {:block/title "b3"}]}]) (let [task-query '(task doing)] diff --git a/src/test/frontend/handler/query/builder_test.cljs b/src/test/frontend/handler/query/builder_test.cljs index c3ae548ce2..888276f7ca 100644 --- a/src/test/frontend/handler/query/builder_test.cljs +++ b/src/test/frontend/handler/query/builder_test.cljs @@ -34,5 +34,5 @@ [:and [:page-ref "foo"] [:page-ref "bar"]])) (is (= (query-builder/from-dsl '(and [[foo]] (or [[bar]] (:property :key :value)))) [:and [:page-ref "foo"] [:or [:page-ref "bar"] [:property :key :value]]])) - (is (= (query-builder/from-dsl '(and (priority A) (task NOW))) - [:and ['priority 'A] ['task 'NOW]]))) + (is (= (query-builder/from-dsl '(and (priority A) (task Doing))) + [:and ['priority 'A] ['task 'Doing]]))) diff --git a/src/test/frontend/test/helper.cljs b/src/test/frontend/test/helper.cljs index 9db4b1d225..c5d2b89e9c 100644 --- a/src/test/frontend/test/helper.cljs +++ b/src/test/frontend/test/helper.cljs @@ -20,7 +20,7 @@ [logseq.db.test.helper :as db-test])) (def bare-marker-pattern - #"(NOW|LATER|TODO|DOING|DONE|WAITING|WAIT|CANCELED|CANCELLED|IN-PROGRESS){1}\s+") + #"(TODO|DOING|DONE|WAIT|CANCELED|CANCELLED){1}\s+") (def node? (exists? js/process)) @@ -51,13 +51,8 @@ (def file-to-db-statuses {"TODO" :logseq.property/status.todo - "LATER" :logseq.property/status.todo - "IN-PROGRESS" :logseq.property/status.doing - "NOW" :logseq.property/status.doing "DOING" :logseq.property/status.doing "DONE" :logseq.property/status.done - "WAIT" :logseq.property/status.backlog - "WAITING" :logseq.property/status.backlog "CANCELED" :logseq.property/status.canceled "CANCELLED" :logseq.property/status.canceled}) diff --git a/src/test/frontend/util/datalog_test.cljs b/src/test/frontend/util/datalog_test.cljs index be59c069a2..34ee36c904 100644 --- a/src/test/frontend/util/datalog_test.cljs +++ b/src/test/frontend/util/datalog_test.cljs @@ -1,7 +1,7 @@ (ns frontend.util.datalog-test (:require [cljs.test :refer [deftest is]] [frontend.util.datalog :as datalog-util] - [logseq.db.file-based.rules :as file-rules])) + [logseq.db.frontend.rules :as rules])) (deftest add-to-end-of-query-in (is (= '[:find ?b @@ -31,8 +31,7 @@ "Add to :in at end of query")) (deftest find-rules-in-where - (is (= [:page-property] + (is (= [:task] (datalog-util/find-rules-in-where - ['(page-property ?b :foo "bar") - '(page-property ?b :bar "baz")] - (-> file-rules/query-dsl-rules keys set))))) + ['(task ?b #{"TODO"})] + (-> rules/db-query-dsl-rules keys set))))) diff --git a/src/test/frontend/util/text_test.cljs b/src/test/frontend/util/text_test.cljs index 89b7ce9527..e86d254a6d 100644 --- a/src/test/frontend/util/text_test.cljs +++ b/src/test/frontend/util/text_test.cljs @@ -2,23 +2,6 @@ (:require [cljs.test :refer [are deftest]] [frontend.util.text :as text-util])) -(deftest test-add-timestamp - (are [x y] (= x y) - (text-util/add-timestamp "LATER hello world\nhello" - "scheduled" - "<2021-08-25 Wed>") - "LATER hello world\nSCHEDULED: <2021-08-25 Wed>\nhello" - - (text-util/add-timestamp "LATER hello world " - "scheduled" - "<2021-08-25 Wed>") - "LATER hello world\nSCHEDULED: <2021-08-25 Wed>" - - (text-util/add-timestamp "LATER hello world\nfoo:: bar\ntest" - "scheduled" - "<2021-08-25 Wed>") - "LATER hello world\nSCHEDULED: <2021-08-25 Wed>\nfoo:: bar\ntest")) - (deftest get-string-all-indexes (are [x y] (= x y) (text-util/get-string-all-indexes "[[hello]] [[world]]" "[[" {})