diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index b30e20ac38..e899f43cfe 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -166,6 +166,8 @@ logseq.common.util.namespace ns-util logseq.common.util.page-ref page-ref logseq.db ldb + logseq.db.file-based.rules file-rules + logseq.db.file-based.schema file-schema logseq.db.frontend.class db-class logseq.db.frontend.content db-content logseq.db.frontend.db-ident db-ident diff --git a/deps/db/.carve/ignore b/deps/db/.carve/ignore index d8634be328..69f3fbfc50 100644 --- a/deps/db/.carve/ignore +++ b/deps/db/.carve/ignore @@ -1,5 +1,5 @@ ;; API -logseq.db.frontend.rules/query-dsl-rules +logseq.db.file-based.schema/retract-attributes ;; API logseq.db.frontend.rules/db-query-dsl-rules ;; API diff --git a/deps/db/.clj-kondo/config.edn b/deps/db/.clj-kondo/config.edn index 20b9501964..7337b63615 100644 --- a/deps/db/.clj-kondo/config.edn +++ b/deps/db/.clj-kondo/config.edn @@ -18,6 +18,8 @@ logseq.db.frontend.property.build db-property-build logseq.db.frontend.property.type db-property-type logseq.db.common.property-util db-property-util + logseq.db.file-based.rules file-rules + logseq.db.file-based.schema file-schema logseq.db.frontend.entity-plus entity-plus logseq.db.frontend.rules rules logseq.db.frontend.schema db-schema diff --git a/deps/db/bb.edn b/deps/db/bb.edn index 000c6db732..b9972456ca 100644 --- a/deps/db/bb.edn +++ b/deps/db/bb.edn @@ -24,13 +24,14 @@ lint:rules {:requires ([logseq.bb-tasks.lint.datalog :as datalog] + [logseq.db.file-based.rules :as file-rules] [logseq.db.frontend.rules :as rules]) :doc "Lint datalog rules for parsability and unbound variables" :task (datalog/lint-rules (set - (concat (mapcat val rules/rules) + (concat (mapcat val (merge file-rules/rules rules/rules)) ;; TODO: Update linter to handle false positive on ?str-val for :property - (rules/extract-rules (dissoc rules/query-dsl-rules :property)) + (rules/extract-rules (dissoc file-rules/query-dsl-rules :property)) ;; TODO: Update linter to handle false positive on :task, :priority, :*property* rules (rules/extract-rules (dissoc rules/db-query-dsl-rules :task :priority diff --git a/deps/db/src/logseq/db.cljs b/deps/db/src/logseq/db.cljs index a724148b30..68b17f61d6 100644 --- a/deps/db/src/logseq/db.cljs +++ b/deps/db/src/logseq/db.cljs @@ -18,7 +18,8 @@ [logseq.db.frontend.property :as db-property] [logseq.db.frontend.rules :as rules] [logseq.db.sqlite.common-db :as sqlite-common-db] - [logseq.db.sqlite.util :as sqlite-util]) + [logseq.db.sqlite.util :as sqlite-util] + [logseq.db.file-based.rules :as file-rules]) (:refer-clojure :exclude [object?])) (defonce *transact-fn (atom nil)) @@ -548,7 +549,7 @@ ['?p :block/name '?namespace] (list 'namespace '?p '?c)] db - (:namespace rules/rules) + (:namespace file-rules/rules) namespace''))) (defn get-pages-by-name-partition diff --git a/deps/db/src/logseq/db/file_based/rules.cljc b/deps/db/src/logseq/db/file_based/rules.cljc new file mode 100644 index 0000000000..c00df09993 --- /dev/null +++ b/deps/db/src/logseq/db/file_based/rules.cljc @@ -0,0 +1,93 @@ +(ns ^:bb-compatible logseq.db.file-based.rules + "Datalog rules for file graphs") + +(def rules + "File graph rules used in db.model queries" + {:namespace + '[[(namespace ?p ?c) + [?c :block/namespace ?p]] + [(namespace ?p ?c) + [?t :block/namespace ?p] + (namespace ?t ?c)]]}) + +(def ^:large-vars/data-var query-dsl-rules + "Rules used by frontend.db.query-dsl for file graphs. The symbols ?b and ?p + respectively refer to block and page. Do not alter them as they are + programmatically built by the query-dsl ns" + {:page-property + '[(page-property ?p ?key ?val) + [?p :block/name] + [?p :block/properties ?prop] + [(get ?prop ?key) ?v] + (or [(= ?v ?val)] [(contains? ?v ?val)])] + + :has-page-property + '[(has-page-property ?p ?key) + [?p :block/name] + [?p :block/properties ?prop] + [(get ?prop ?key)]] + + :task + '[(task ?b ?markers) + [?b :block/marker ?marker] + [(contains? ?markers ?marker)]] + + :priority + '[(priority ?b ?priorities) + [?b :block/priority ?priority] + [(contains? ?priorities ?priority)]] + + :page-tags + '[(page-tags ?p ?tags) + [?p :block/tags ?t] + [?t :block/name ?tag] + [(contains? ?tags ?tag)]] + + :all-page-tags + '[(all-page-tags ?p) + [_ :block/tags ?p]] + + :between + '[(between ?b ?start ?end) + [?b :block/page ?p] + [?p :block/type "journal"] + [?p :block/journal-day ?d] + [(>= ?d ?start)] + [(<= ?d ?end)]] + + :has-property + '[(has-property ?b ?prop) + [?b :block/properties ?bp] + [(missing? $ ?b :block/name)] + [(get ?bp ?prop)]] + + :block-content + '[(block-content ?b ?query) + [?b :block/title ?content] + [(clojure.string/includes? ?content ?query)]] + + :page + '[(page ?b ?page-name) + [?b :block/page ?bp] + [?bp :block/name ?page-name]] + + :namespace + '[(namespace ?p ?namespace) + [?p :block/namespace ?parent] + [?parent :block/name ?namespace]] + + :property + '[(property ?b ?key ?val) + [?b :block/properties ?prop] + [(missing? $ ?b :block/name)] + [(get ?prop ?key) ?v] + [(str ?val) ?str-val] + (or [(= ?v ?val)] + [(contains? ?v ?val)] + ;; For integer pages that aren't strings + [(contains? ?v ?str-val)])] + + :page-ref + '[(page-ref ?b ?page-name) + [?b :block/path-refs ?br] + [?br :block/name ?page-name]]}) \ No newline at end of file diff --git a/deps/db/src/logseq/db/file_based/schema.cljs b/deps/db/src/logseq/db/file_based/schema.cljs new file mode 100644 index 0000000000..b111d2565e --- /dev/null +++ b/deps/db/src/logseq/db/file_based/schema.cljs @@ -0,0 +1,125 @@ +(ns logseq.db.file-based.schema + "Schema related vars for file graphs") + +;; A page is a special block, a page can corresponds to multiple files with the same ":block/name". +(def ^:large-vars/data-var schema + "Schema for file graphs" + {:db/ident {:db/unique :db.unique/identity} + :kv/value {} + :recent/pages {} + + ;; :block/type is a string type of the current block + ;; "whiteboard" for whiteboards + ;; "property" for property blocks + ;; "class" for structured page + :block/type {:db/index true} + :block/uuid {:db/unique :db.unique/identity} + :block/parent {:db/valueType :db.type/ref + :db/index true} + :block/order {:db/index true} + :block/collapsed? {} + + ;; :markdown, :org + :block/format {} + + ;; belongs to which page + :block/page {:db/valueType :db.type/ref + :db/index true} + ;; reference blocks + :block/refs {:db/valueType :db.type/ref + :db/cardinality :db.cardinality/many} + ;; referenced pages inherited from the parents + :block/path-refs {:db/valueType :db.type/ref + :db/cardinality :db.cardinality/many} + + ;; tags are structured classes + :block/tags {:db/valueType :db.type/ref + :db/cardinality :db.cardinality/many} + + ;; which block this block links to, used for tag, embeds + :block/link {:db/valueType :db.type/ref + :db/index true} + + ;; page's namespace + :block/namespace {:db/valueType :db.type/ref} + + ;; for pages + :block/alias {:db/valueType :db.type/ref + :db/cardinality :db.cardinality/many} + + ;; todo keywords, e.g. "TODO", "DOING", "DONE" + :block/marker {} + + ;; "A", "B", "C" + :block/priority {} + + ;; map, key -> set of refs in property value or full text if none are found + :block/properties {} + ;; vector + :block/properties-order {} + ;; map, key -> original property value's content + :block/properties-text-values {} + + ;; first block that's not a heading or unordered list + :block/pre-block? {} + + ;; scheduled day + :block/scheduled {} + + ;; deadline day + :block/deadline {} + + ;; whether blocks is a repeated block (usually a task) + :block/repeated? {} + + :block/created-at {:db/index true} + :block/updated-at {:db/index true} + + ;; page additional attributes + ;; page's name, lowercase + :block/name {:db/unique :db.unique/identity} + + ;; page's original name + :block/title {:db/index true} + + ;; page's journal day + :block/journal-day {:db/index true} + + ;; macros in block + :block/macros {:db/valueType :db.type/ref + :db/cardinality :db.cardinality/many} + + ;; block's file + :block/file {:db/valueType :db.type/ref} + + ;; latest tx that affected the block + :block/tx-id {} + + ;; file + :file/path {:db/unique :db.unique/identity} + :file/content {} + :file/created-at {} + :file/last-modified-at {} + :file/size {}}) + +(def file-only-attributes + [:block/namespace :block/properties-text-values :block/pre-block? :recent/pages :block/file + :block/properties :block/properties-order :block/repeated? :block/deadline :block/scheduled :block/priority + :block/marker :block/macros :block/type :block/format]) + +(def retract-attributes + #{:block/refs + :block/tags + :block/alias + :block/marker + :block/priority + :block/scheduled + :block/deadline + :block/repeated? + :block/pre-block? + :block/properties + :block/properties-order + :block/properties-text-values + :block/macros + :block/invalid-properties + :block/warning}) diff --git a/deps/db/src/logseq/db/frontend/malli_schema.cljs b/deps/db/src/logseq/db/frontend/malli_schema.cljs index 509b89bf2b..e34b37384c 100644 --- a/deps/db/src/logseq/db/frontend/malli_schema.cljs +++ b/deps/db/src/logseq/db/frontend/malli_schema.cljs @@ -515,7 +515,7 @@ :property-value-placeholder property-value-placeholder})) (def DB - "Malli schema for entities from schema/schema-for-db-based-graph. In order to + "Malli schema for entities from db-schema/schema. In order to thoroughly validate properties, the entities and this schema should be prepared with update-properties-in-ents and update-properties-in-schema respectively" diff --git a/deps/db/src/logseq/db/frontend/rules.cljc b/deps/db/src/logseq/db/frontend/rules.cljc index 13b118d72f..121f66c262 100644 --- a/deps/db/src/logseq/db/frontend/rules.cljc +++ b/deps/db/src/logseq/db/frontend/rules.cljc @@ -1,17 +1,12 @@ (ns ^:bb-compatible logseq.db.frontend.rules - "Datalog rules for use with logseq.db.frontend.schema") + "Datalog rules mostly for DB graphs. `rules` + is the only var also used by file graphs" + (:require [logseq.db.file-based.rules :as file-rules])) (def ^:large-vars/data-var rules - "Rules used mainly in frontend.db.model" + "Rules used mainly in frontend.db.model for both DB and file graphs" ;; rule "parent" is optimized for parent node -> child node nesting queries - {:namespace - '[[(namespace ?p ?c) - [?c :block/namespace ?p]] - [(namespace ?p ?c) - [?t :block/namespace ?p] - (namespace ?t ?c)]] - - :parent + {:parent '[[(parent ?p ?c) [?c :logseq.property/parent ?p]] [(parent ?p ?c) @@ -67,96 +62,13 @@ ;; (not-join [?e ?v] ;; [?e ?a ?v]))] -(def ^:large-vars/data-var query-dsl-rules - "Rules used by frontend.db.query-dsl for file graphs. The symbols ?b and ?p - respectively refer to block and page. Do not alter them as they are - programmatically built by the query-dsl ns" - {:page-property - '[(page-property ?p ?key ?val) - [?p :block/name] - [?p :block/properties ?prop] - [(get ?prop ?key) ?v] - (or [(= ?v ?val)] [(contains? ?v ?val)])] - - :has-page-property - '[(has-page-property ?p ?key) - [?p :block/name] - [?p :block/properties ?prop] - [(get ?prop ?key)]] - - :task - '[(task ?b ?markers) - [?b :block/marker ?marker] - [(contains? ?markers ?marker)]] - - :priority - '[(priority ?b ?priorities) - [?b :block/priority ?priority] - [(contains? ?priorities ?priority)]] - - :page-tags - '[(page-tags ?p ?tags) - [?p :block/tags ?t] - [?t :block/name ?tag] - [(contains? ?tags ?tag)]] - - :all-page-tags - '[(all-page-tags ?p) - [_ :block/tags ?p]] - - :between - '[(between ?b ?start ?end) - [?b :block/page ?p] - [?p :block/type "journal"] - [?p :block/journal-day ?d] - [(>= ?d ?start)] - [(<= ?d ?end)]] - - :has-property - '[(has-property ?b ?prop) - [?b :block/properties ?bp] - [(missing? $ ?b :block/name)] - [(get ?bp ?prop)]] - - :block-content - '[(block-content ?b ?query) - [?b :block/title ?content] - [(clojure.string/includes? ?content ?query)]] - - :page - '[(page ?b ?page-name) - [?b :block/page ?bp] - [?bp :block/name ?page-name]] - - :namespace - '[(namespace ?p ?namespace) - [?p :block/namespace ?parent] - [?parent :block/name ?namespace]] - - :property - '[(property ?b ?key ?val) - [?b :block/properties ?prop] - [(missing? $ ?b :block/name)] - [(get ?prop ?key) ?v] - [(str ?val) ?str-val] - (or [(= ?v ?val)] - [(contains? ?v ?val)] - ;; For integer pages that aren't strings - [(contains? ?v ?str-val)])] - - :page-ref - '[(page-ref ?b ?page-name) - [?b :block/path-refs ?br] - [?br :block/name ?page-name]]}) - (def ^:large-vars/data-var db-query-dsl-rules - "Rules used by frontend.query.dsl for db graphs" + "Rules used by frontend.query.dsl for DB graphs" (merge - (dissoc query-dsl-rules :namespace + (dissoc file-rules/query-dsl-rules :namespace :page-property :has-page-property :page-tags :all-page-tags) - - (dissoc rules :namespace) + rules {:between '[(between ?b ?start ?end) diff --git a/deps/db/src/logseq/db/frontend/schema.cljs b/deps/db/src/logseq/db/frontend/schema.cljs index bc03a0c943..7c7b1ad6c0 100644 --- a/deps/db/src/logseq/db/frontend/schema.cljs +++ b/deps/db/src/logseq/db/frontend/schema.cljs @@ -1,7 +1,8 @@ (ns logseq.db.frontend.schema - "Main datascript schemas for the Logseq app" + "Schema related fns for DB and file graphs" (:require [clojure.set :as set] - [clojure.string :as string])) + [clojure.string :as string] + [logseq.db.file-based.schema :as file-schema])) (def schema-version? (every-pred map? :major)) @@ -59,137 +60,18 @@ (str (:major schema-version))) :else (throw (ex-info "Not a schema-version" {:data schema-version})))) -;; A page is a special block, a page can corresponds to multiple files with the same ":block/name". -(def ^:large-vars/data-var schema - {:db/ident {:db/unique :db.unique/identity} - :kv/value {} - :recent/pages {} - - ;; :block/type is a string type of the current block - ;; "whiteboard" for whiteboards - ;; "property" for property blocks - ;; "class" for structured page - :block/type {:db/index true} - :block/uuid {:db/unique :db.unique/identity} - :block/parent {:db/valueType :db.type/ref - :db/index true} - :block/order {:db/index true} - :block/collapsed? {} - - ;; :markdown, :org - :block/format {} - - ;; belongs to which page - :block/page {:db/valueType :db.type/ref - :db/index true} - ;; reference blocks - :block/refs {:db/valueType :db.type/ref - :db/cardinality :db.cardinality/many} - ;; referenced pages inherited from the parents - :block/path-refs {:db/valueType :db.type/ref - :db/cardinality :db.cardinality/many} - - ;; tags are structured classes - :block/tags {:db/valueType :db.type/ref - :db/cardinality :db.cardinality/many} - - ;; which block this block links to, used for tag, embeds - :block/link {:db/valueType :db.type/ref - :db/index true} - - ;; page's namespace - :block/namespace {:db/valueType :db.type/ref} - - ;; for pages - :block/alias {:db/valueType :db.type/ref - :db/cardinality :db.cardinality/many} - - ;; todo keywords, e.g. "TODO", "DOING", "DONE" - :block/marker {} - - ;; "A", "B", "C" - :block/priority {} - - ;; map, key -> set of refs in property value or full text if none are found - :block/properties {} - ;; vector - :block/properties-order {} - ;; map, key -> original property value's content - :block/properties-text-values {} - - ;; first block that's not a heading or unordered list - :block/pre-block? {} - - ;; scheduled day - :block/scheduled {} - - ;; deadline day - :block/deadline {} - - ;; whether blocks is a repeated block (usually a task) - :block/repeated? {} - - :block/created-at {:db/index true} - :block/updated-at {:db/index true} - - ;; page additional attributes - ;; page's name, lowercase - :block/name {:db/unique :db.unique/identity} - - ;; page's original name - :block/title {:db/index true} - - ;; page's journal day - :block/journal-day {:db/index true} - - ;; macros in block - :block/macros {:db/valueType :db.type/ref - :db/cardinality :db.cardinality/many} - - ;; block's file - :block/file {:db/valueType :db.type/ref} - - ;; latest tx that affected the block - :block/tx-id {} - - ;; file - :file/path {:db/unique :db.unique/identity} - :file/content {} - :file/created-at {} - :file/last-modified-at {} - :file/size {}}) - -(def schema-for-db-based-graph +(def schema + "Schema for DB graphs" (merge - (dissoc schema - :block/namespace :block/properties-text-values :block/pre-block? :recent/pages :block/file - :block/properties :block/properties-order :block/repeated? :block/deadline :block/scheduled :block/priority - :block/marker :block/macros - :block/type :block/format) + (apply dissoc file-schema/schema file-schema/file-only-attributes) {:block/name {:db/index true} ; remove db/unique for :block/name ;; closed value :block/closed-value-property {:db/valueType :db.type/ref :db/cardinality :db.cardinality/many}})) -(def retract-attributes - #{:block/refs - :block/tags - :block/alias - :block/marker - :block/priority - :block/scheduled - :block/deadline - :block/repeated? - :block/pre-block? - :block/properties - :block/properties-order - :block/properties-text-values - :block/macros - :block/invalid-properties - :block/warning}) - ;; If only block/title changes -(def db-version-retract-attributes +(def retract-attributes + "Retract attributes for DB graphs" #{:block/refs :block/warning}) @@ -200,14 +82,14 @@ (keep (fn [[attr-name attr-body-map]] (when (= :db.type/ref (:db/valueType attr-body-map)) attr-name))) - schema-for-db-based-graph)) + schema)) (def card-many-attributes (into #{} (keep (fn [[attr-name attr-body-map]] (when (= :db.cardinality/many (:db/cardinality attr-body-map)) attr-name))) - schema-for-db-based-graph)) + schema)) (def card-many-ref-type-attributes (set/intersection card-many-attributes ref-type-attributes)) @@ -216,7 +98,7 @@ (set/difference ref-type-attributes card-many-attributes)) (def db-non-ref-attributes - (->> schema-for-db-based-graph + (->> schema (keep (fn [[k v]] (when (not (:db/valueType v)) k))) diff --git a/deps/db/src/logseq/db/sqlite/cli.cljs b/deps/db/src/logseq/db/sqlite/cli.cljs index 2a25b7acc0..7b0f2788da 100644 --- a/deps/db/src/logseq/db/sqlite/cli.cljs +++ b/deps/db/src/logseq/db/sqlite/cli.cljs @@ -1,5 +1,5 @@ (ns ^:node-only logseq.db.sqlite.cli - "Primary ns to interact with DB graphs with node.js based CLIs" + "Primary ns to interact with DB files for DB and file graphs with node.js based CLIs" (:require ["better-sqlite3" :as sqlite3] [logseq.db.sqlite.common-db :as sqlite-common-db] ;; FIXME: datascript.core has to come before datascript.storage or else nbb fails @@ -7,6 +7,7 @@ [datascript.core :as d] [datascript.storage :refer [IStorage]] [logseq.db.frontend.schema :as db-schema] + [logseq.db.file-based.schema :as file-schema] [logseq.db.sqlite.util :as sqlite-util] [cljs-bean.core :as bean] ["fs" :as fs] @@ -91,8 +92,8 @@ db (new sqlite db-full-path nil) ;; For both desktop and CLI, only file graphs have db-name that indicate their db type schema (if (sqlite-util/local-file-based-graph? db-name) - db-schema/schema - db-schema/schema-for-db-based-graph)] + file-schema/schema + db-schema/schema)] (sqlite-common-db/create-kvs-table! db) (let [storage (new-sqlite-storage db) conn (sqlite-common-db/get-storage-conn storage schema)] diff --git a/deps/db/src/logseq/db/sqlite/util.cljs b/deps/db/src/logseq/db/sqlite/util.cljs index dc888556d8..d9d17a5a76 100644 --- a/deps/db/src/logseq/db/sqlite/util.cljs +++ b/deps/db/src/logseq/db/sqlite/util.cljs @@ -11,7 +11,8 @@ [logseq.db.common.order :as db-order] [logseq.db.frontend.property :as db-property] [logseq.db.frontend.property.type :as db-property-type] - [logseq.db.frontend.schema :as db-schema])) + [logseq.db.frontend.schema :as db-schema] + [logseq.db.file-based.schema :as file-schema])) (defonce db-version-prefix "logseq_db_") (defonce file-version-prefix "logseq_local_") @@ -65,8 +66,8 @@ "Returns schema for given repo" [repo] (if (db-based-graph? repo) - db-schema/schema-for-db-based-graph - db-schema/schema)) + db-schema/schema + file-schema/schema)) (def block-with-timestamps common-util/block-with-timestamps) diff --git a/deps/db/src/logseq/db/test/helper.cljs b/deps/db/src/logseq/db/test/helper.cljs index b607d03bfb..76d3c88ea3 100644 --- a/deps/db/src/logseq/db/test/helper.cljs +++ b/deps/db/src/logseq/db/test/helper.cljs @@ -76,7 +76,7 @@ (defn create-conn "Create a conn for a DB graph seeded with initial data" [] - (let [conn (d/create-conn db-schema/schema-for-db-based-graph) + (let [conn (d/create-conn db-schema/schema) _ (d/transact! conn (sqlite-create-graph/build-db-initial-data "{}"))] (entity-plus/reset-immutable-entities-cache!) conn)) diff --git a/deps/db/test/logseq/db/frontend/inputs_test.cljs b/deps/db/test/logseq/db/frontend/inputs_test.cljs index a5024086b9..c836faba61 100644 --- a/deps/db/test/logseq/db/frontend/inputs_test.cljs +++ b/deps/db/test/logseq/db/frontend/inputs_test.cljs @@ -17,7 +17,7 @@ (map first)))) (deftest resolve-input-for-page-and-block-inputs - (let [conn (d/create-conn db-schema/schema-for-db-based-graph) + (let [conn (d/create-conn db-schema/schema) _ (d/transact! conn [{:db/ident :logseq.class/Page}]) _ (sqlite-build/create-blocks conn diff --git a/deps/db/test/logseq/db/sqlite/create_graph_test.cljs b/deps/db/test/logseq/db/sqlite/create_graph_test.cljs index a4e2f4b511..2ff3d5365f 100644 --- a/deps/db/test/logseq/db/sqlite/create_graph_test.cljs +++ b/deps/db/test/logseq/db/sqlite/create_graph_test.cljs @@ -134,7 +134,7 @@ "New graph has no validation errors"))) (deftest property-types - (let [conn (d/create-conn db-schema/schema-for-db-based-graph) + (let [conn (d/create-conn db-schema/schema) _ (d/transact! conn (sqlite-create-graph/build-db-initial-data (pr-str {:macros {"docs-base-url" "https://docs.logseq.com/#/page/$1"}})))] diff --git a/deps/db/test/logseq/db_test.cljs b/deps/db/test/logseq/db_test.cljs index b8bf19ae60..7a5911746a 100644 --- a/deps/db/test/logseq/db_test.cljs +++ b/deps/db/test/logseq/db_test.cljs @@ -2,8 +2,8 @@ (:require [cljs.test :refer [deftest is]] [datascript.core :as d] [logseq.db :as ldb] - [logseq.db.frontend.schema :as db-schema] - [logseq.db.test.helper :as db-test])) + [logseq.db.test.helper :as db-test] + [logseq.db.file-based.schema :as file-schema])) ;;; datoms ;;; - 1 <----+ @@ -21,7 +21,7 @@ :block/parent 2}]) (deftest get-block-children-ids-on-bad-outliner-data - (let [db (d/db-with (d/empty-db db-schema/schema) + (let [db (d/db-with (d/empty-db file-schema/schema) broken-outliner-data-with-cycle)] (is (= "bad outliner data, need to re-index to fix" (try (ldb/get-block-children-ids db #uuid "e538d319-48d4-4a6d-ae70-c03bb55b6fe4") diff --git a/deps/graph-parser/src/logseq/graph_parser.cljs b/deps/graph-parser/src/logseq/graph_parser.cljs index bb9bb37963..99f6fee6f8 100644 --- a/deps/graph-parser/src/logseq/graph_parser.cljs +++ b/deps/graph-parser/src/logseq/graph_parser.cljs @@ -4,17 +4,17 @@ (:require [clojure.set :as set] [clojure.string :as string] [datascript.core :as d] - [logseq.db.frontend.schema :as db-schema] [logseq.graph-parser.extract :as extract] [logseq.common.util :as common-util] [logseq.common.config :as common-config] - [logseq.db :as ldb])) + [logseq.db :as ldb] + [logseq.db.file-based.schema :as file-schema])) (defn- retract-blocks-tx [blocks retain-uuids] (mapcat (fn [{uuid' :block/uuid eid :db/id}] (if (and uuid' (contains? retain-uuids uuid')) - (map (fn [attr] [:db.fn/retractAttribute eid attr]) db-schema/retract-attributes) + (map (fn [attr] [:db.fn/retractAttribute eid attr]) file-schema/retract-attributes) (when eid [[:db.fn/retractEntity eid]]))) blocks)) diff --git a/deps/graph-parser/src/logseq/graph_parser/db.cljs b/deps/graph-parser/src/logseq/graph_parser/db.cljs index 0445f25ac4..7eebb2f04a 100644 --- a/deps/graph-parser/src/logseq/graph_parser/db.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/db.cljs @@ -4,8 +4,8 @@ [clojure.set :as set] [clojure.string :as string] [logseq.common.util :as common-util] - [logseq.db.frontend.schema :as db-schema] - [logseq.db :as ldb])) + [logseq.db :as ldb] + [logseq.db.file-based.schema :as file-schema])) (defonce built-in-markers ["NOW" "LATER" "DOING" "DONE" "CANCELED" "CANCELLED" "IN-PROGRESS" "TODO" "WAIT" "WAITING"]) @@ -50,7 +50,7 @@ (defn start-conn "Create datascript conn with schema and default data" [] - (let [db-conn (d/create-conn db-schema/schema)] + (let [db-conn (d/create-conn file-schema/schema)] (create-default-pages! db-conn) db-conn)) diff --git a/deps/graph-parser/test/logseq/graph_parser/extract_test.cljs b/deps/graph-parser/test/logseq/graph_parser/extract_test.cljs index 0eb2f2e3a9..e20566d203 100644 --- a/deps/graph-parser/test/logseq/graph_parser/extract_test.cljs +++ b/deps/graph-parser/test/logseq/graph_parser/extract_test.cljs @@ -2,7 +2,7 @@ (:require [cljs.test :refer [deftest is are]] [logseq.graph-parser.extract :as extract] [datascript.core :as d] - [logseq.db.frontend.schema :as db-schema])) + [logseq.db.file-based.schema :as file-schema])) ;; This is a copy of frontend.util.fs/multiplatform-reserved-chars for reserved chars testing (def multiplatform-reserved-chars ":\\*\\?\"<>|\\#\\\\") @@ -45,7 +45,7 @@ (defn- extract [file content & [options]] (extract/extract file content - (merge {:block-pattern "-" :db (d/empty-db db-schema/schema) + (merge {:block-pattern "-" :db (d/empty-db file-schema/schema) :verbose false} options))) diff --git a/deps/outliner/src/logseq/outliner/core.cljs b/deps/outliner/src/logseq/outliner/core.cljs index c83aeb1817..f09786c6ca 100644 --- a/deps/outliner/src/logseq/outliner/core.cljs +++ b/deps/outliner/src/logseq/outliner/core.cljs @@ -8,6 +8,7 @@ [logseq.db :as ldb] [logseq.db.common.order :as db-order] [logseq.db.frontend.schema :as db-schema] + [logseq.db.file-based.schema :as file-schema] [logseq.db.sqlite.create-graph :as sqlite-create-graph] [logseq.db.sqlite.util :as sqlite-util] [logseq.graph-parser.block :as gp-block] @@ -283,8 +284,8 @@ (seq retract-attributes)) (let [retract-attributes (concat (if db-based? - db-schema/db-version-retract-attributes - db-schema/retract-attributes) + db-schema/retract-attributes + file-schema/retract-attributes) retract-attributes)] (swap! txs-state (fn [txs] (vec diff --git a/deps/outliner/src/logseq/outliner/property.cljs b/deps/outliner/src/logseq/outliner/property.cljs index b6759021b8..03eaf9be31 100644 --- a/deps/outliner/src/logseq/outliner/property.cljs +++ b/deps/outliner/src/logseq/outliner/property.cljs @@ -289,7 +289,7 @@ (let [block-eid (->eid block-eid) _ (assert (qualified-keyword? property-id) "property-id should be a keyword") block (d/entity @conn block-eid) - db-attribute? (some? (db-schema/schema-for-db-based-graph property-id))] + db-attribute? (some? (db-schema/schema property-id))] (when (= property-id :block/tags) (outliner-validate/validate-tags-property @conn [block-eid] v)) (when (= property-id :logseq.property/parent) diff --git a/deps/outliner/test/logseq/outliner/pipeline_test.cljs b/deps/outliner/test/logseq/outliner/pipeline_test.cljs index f3ca5fb8a0..6d9d345282 100644 --- a/deps/outliner/test/logseq/outliner/pipeline_test.cljs +++ b/deps/outliner/test/logseq/outliner/pipeline_test.cljs @@ -23,7 +23,7 @@ (deftest compute-block-path-refs-tx (testing "when a block's :refs change, descendants of block have correct :block/path-refs" - (let [conn (d/create-conn db-schema/schema-for-db-based-graph) + (let [conn (d/create-conn db-schema/schema) ;; needed in order for path-refs to be setup correctly with init data _ (db-pipeline/add-listener conn) _ (d/transact! conn (sqlite-create-graph/build-db-initial-data "{}")) diff --git a/scripts/src/logseq/tasks/dev/db_and_file_graphs.clj b/scripts/src/logseq/tasks/dev/db_and_file_graphs.clj index 3225e5f20a..b4542e19cc 100644 --- a/scripts/src/logseq/tasks/dev/db_and_file_graphs.clj +++ b/scripts/src/logseq/tasks/dev/db_and_file_graphs.clj @@ -12,7 +12,7 @@ (def db-graph-ns "Namespaces or parent namespaces _only_ for DB graphs. Use a '.' at end of a namespace for parent namespaces" (mapv escape-shell-regex - ["logseq.db.sqlite." "logseq.db.frontend.property" "logseq.db.frontend.malli-schema" + ["logseq.db.sqlite." "logseq.db.frontend." "electron.db" "frontend.handler.db-based." "frontend.worker.handler.page.db-based" @@ -114,7 +114,7 @@ "git grep -E" (str "(" (string/join "|" file-concepts) ")") db-graph-paths) invalid-lines (when (= 0 (:exit res)) - (remove #(->> (string/split % #":\s+") second string/trim (contains? allowed-exceptions)) + (remove #(some->> (string/split % #":\s+") second string/trim (contains? allowed-exceptions)) (string/split-lines (:out res))))] (when (> (:exit res) 1) (System/exit 1)) (when (and (= 0 (:exit res)) (seq invalid-lines)) diff --git a/src/main/frontend/db/conn.cljs b/src/main/frontend/db/conn.cljs index 5e6615b3d3..0cd01ce3af 100644 --- a/src/main/frontend/db/conn.cljs +++ b/src/main/frontend/db/conn.cljs @@ -81,7 +81,7 @@ ([repo {:keys [listen-handler]}] (let [db-name (db-conn-state/get-repo-path repo) db-conn (if (config/db-based-graph? repo) - (d/create-conn db-schema/schema-for-db-based-graph) + (d/create-conn db-schema/schema) (gp-db/start-conn))] (swap! conns assoc db-name db-conn) (when listen-handler diff --git a/src/main/frontend/db/query_custom.cljs b/src/main/frontend/db/query_custom.cljs index 031ca64d61..0d40e957f6 100644 --- a/src/main/frontend/db/query_custom.cljs +++ b/src/main/frontend/db/query_custom.cljs @@ -7,7 +7,8 @@ [logseq.db.frontend.rules :as rules] [frontend.util.datalog :as datalog-util] [clojure.walk :as walk] - [frontend.config :as config])) + [frontend.config :as config] + [logseq.db.file-based.rules :as file-rules])) ;; FIXME: what if users want to query other attributes than block-attrs? (defn- replace-star-with-block-attrs! @@ -27,7 +28,7 @@ "Searches query's :where for rules and adds them to query if used" [{:keys [query] :as query-m} {:keys [db-graph?]}] (let [{:keys [where in]} (datalog-util/query-vec->map query) - query-dsl-rules (if db-graph? rules/db-query-dsl-rules rules/query-dsl-rules) + query-dsl-rules (if db-graph? rules/db-query-dsl-rules file-rules/query-dsl-rules) rules-found (datalog-util/find-rules-in-where where (-> query-dsl-rules keys set))] (if (seq rules-found) (if (and (= '% (last in)) (vector? (last (:inputs query-m)))) diff --git a/src/main/frontend/db/query_dsl.cljs b/src/main/frontend/db/query_dsl.cljs index 63d262d2a9..ca28540a4d 100644 --- a/src/main/frontend/db/query_dsl.cljs +++ b/src/main/frontend/db/query_dsl.cljs @@ -19,7 +19,8 @@ [logseq.common.util.date-time :as date-time-util] [logseq.common.util.page-ref :as page-ref] [logseq.db.frontend.rules :as rules] - [logseq.graph-parser.text :as text])) + [logseq.graph-parser.text :as text] + [logseq.db.file-based.rules :as file-rules])) ;; Query fields: @@ -696,7 +697,7 @@ Some bindings in this fn: {:query result' :rules (if db-graph? (rules/extract-rules rules/db-query-dsl-rules rules {:deps rules/rules-dependencies}) - (mapv rules/query-dsl-rules rules)) + (mapv file-rules/query-dsl-rules rules)) :sort-by @sort-by :blocks? (boolean @blocks?) :sample sample}))) diff --git a/src/main/frontend/handler/editor.cljs b/src/main/frontend/handler/editor.cljs index 0db5d35cdb..6c17d02147 100644 --- a/src/main/frontend/handler/editor.cljs +++ b/src/main/frontend/handler/editor.cljs @@ -55,9 +55,9 @@ [logseq.common.util.block-ref :as block-ref] [logseq.common.util.page-ref :as page-ref] [logseq.db :as ldb] + [logseq.db.file-based.schema :as file-schema] [logseq.db.frontend.entity-plus :as entity-plus] [logseq.db.frontend.property :as db-property] - [logseq.db.frontend.schema :as db-schema] [logseq.graph-parser.block :as gp-block] [logseq.graph-parser.mldoc :as gp-mldoc] [logseq.graph-parser.property :as gp-property] @@ -385,7 +385,7 @@ selection-end (util/get-selection-end input) [fst-block-text snd-block-text] (compute-fst-snd-block-text value selection-start selection-end) current-block (assoc block :block/title fst-block-text) - current-block (apply dissoc current-block db-schema/retract-attributes) + current-block (apply dissoc current-block file-schema/retract-attributes) new-m {:block/uuid (db/new-block-id) :block/title snd-block-text} next-block (-> (merge (select-keys block [:block/parent :block/format :block/page]) diff --git a/src/main/frontend/handler/file_based/editor.cljs b/src/main/frontend/handler/file_based/editor.cljs index a4e2dbe3ea..ec099560e8 100644 --- a/src/main/frontend/handler/file_based/editor.cljs +++ b/src/main/frontend/handler/file_based/editor.cljs @@ -23,7 +23,7 @@ [logseq.common.util :as common-util] [logseq.common.util.block-ref :as block-ref] [logseq.db :as ldb] - [logseq.db.frontend.schema :as db-schema])) + [logseq.db.file-based.schema :as file-schema])) (defn- remove-non-existed-refs! [refs] @@ -119,7 +119,7 @@ block (assoc block :block/title content' :block/format format) - block (apply dissoc block (remove #{:block/pre-block?} db-schema/retract-attributes)) + block (apply dissoc block (remove #{:block/pre-block?} file-schema/retract-attributes)) block (block/parse-block block) block (if (and first-block? (:block/pre-block? block)) block diff --git a/src/main/frontend/worker/db_worker.cljs b/src/main/frontend/worker/db_worker.cljs index 0e743b325e..ee81b0ccec 100644 --- a/src/main/frontend/worker/db_worker.cljs +++ b/src/main/frontend/worker/db_worker.cljs @@ -114,7 +114,7 @@ "Persistent-sorted-set has been broken, used addresses can't be found" [datascript-conn sqlite-db import-type] (let [datoms (get-all-datoms-from-sqlite-db sqlite-db) - db (d/init-db [] db-schema/schema-for-db-based-graph + db (d/init-db [] db-schema/schema {:storage (storage/storage @datascript-conn)}) db (d/db-with db (map (fn [d] diff --git a/src/main/frontend/worker/rtc/full_upload_download_graph.cljs b/src/main/frontend/worker/rtc/full_upload_download_graph.cljs index de2075492a..b51876e851 100644 --- a/src/main/frontend/worker/rtc/full_upload_download_graph.cljs +++ b/src/main/frontend/worker/rtc/full_upload_download_graph.cljs @@ -266,7 +266,7 @@ (defn- create-graph-for-rtc-test "it's complex to setup db-worker related stuff, when I only want to test rtc related logic" [repo init-tx-data other-tx-data] - (let [conn (d/create-conn db-schema/schema-for-db-based-graph) + (let [conn (d/create-conn db-schema/schema) db-initial-data (sqlite-create-graph/build-db-initial-data "")] (swap! worker-state/*datascript-conns assoc repo conn) (d/transact! conn db-initial-data {:initial-db? true diff --git a/src/test/frontend/test/fixtures.cljs b/src/test/frontend/test/fixtures.cljs index 8f0564c42f..23620e0efb 100644 --- a/src/test/frontend/test/fixtures.cljs +++ b/src/test/frontend/test/fixtures.cljs @@ -1,10 +1,10 @@ (ns frontend.test.fixtures (:require [datascript.core :as d] - [logseq.db.frontend.schema :as db-schema] [frontend.db.conn :as conn] [frontend.db.react :as react] [frontend.state :as state] - [frontend.test.helper :as test-helper])) + [frontend.test.helper :as test-helper] + [logseq.db.file-based.schema :as file-schema])) (defn react-components [f] @@ -16,7 +16,7 @@ (defn- reset-datascript [repo] (let [db-name (conn/get-repo-path repo) - db-conn (d/create-conn db-schema/schema)] + db-conn (d/create-conn file-schema/schema)] (state/set-current-repo! repo) (swap! conn/conns assoc db-name db-conn))) diff --git a/src/test/frontend/util/datalog_test.cljs b/src/test/frontend/util/datalog_test.cljs index a57ba75aa9..be59c069a2 100644 --- a/src/test/frontend/util/datalog_test.cljs +++ b/src/test/frontend/util/datalog_test.cljs @@ -1,7 +1,7 @@ (ns frontend.util.datalog-test (:require [cljs.test :refer [deftest is]] [frontend.util.datalog :as datalog-util] - [logseq.db.frontend.rules :as rules])) + [logseq.db.file-based.rules :as file-rules])) (deftest add-to-end-of-query-in (is (= '[:find ?b @@ -35,4 +35,4 @@ (datalog-util/find-rules-in-where ['(page-property ?b :foo "bar") '(page-property ?b :bar "baz")] - (-> rules/query-dsl-rules keys set))))) + (-> file-rules/query-dsl-rules keys set))))) diff --git a/src/test/frontend/worker/rtc/client_test.cljs b/src/test/frontend/worker/rtc/client_test.cljs index fb779b40df..088bd73e81 100644 --- a/src/test/frontend/worker/rtc/client_test.cljs +++ b/src/test/frontend/worker/rtc/client_test.cljs @@ -6,7 +6,7 @@ [logseq.db :as ldb] [logseq.db.frontend.schema :as db-schema])) -(def empty-db (d/empty-db db-schema/schema-for-db-based-graph)) +(def empty-db (d/empty-db db-schema/schema)) (deftest local-block-ops->remote-ops-test (testing "user.class/yyy creation" diff --git a/src/test/frontend/worker/rtc/remote_update_test.cljs b/src/test/frontend/worker/rtc/remote_update_test.cljs index cca2a10e21..bc65b108b8 100644 --- a/src/test/frontend/worker/rtc/remote_update_test.cljs +++ b/src/test/frontend/worker/rtc/remote_update_test.cljs @@ -8,7 +8,7 @@ (deftest remote-op-value->tx-data-test (let [[block-uuid ref-uuid1 ref-uuid2] (repeatedly random-uuid) - db (d/db-with (d/empty-db db-schema/schema-for-db-based-graph) + db (d/db-with (d/empty-db db-schema/schema) (sqlite-create-graph/build-db-initial-data "{}" {}))] (testing ":block/title" (let [db (d/db-with db [{:block/uuid block-uuid