From 9db946c6050f930293c4a0124c81ea5ee504e82e Mon Sep 17 00:00:00 2001 From: rcmerci Date: Thu, 9 Jan 2025 17:35:19 +0800 Subject: [PATCH] enhance: apply defkeywords to built-in classes, add some rtc related keywords --- .clj-kondo/hooks/defkeywords.clj | 8 +- .../common/src/logseq/common/defkeywords.cljc | 9 +- deps/db/src/logseq/db/frontend/class.cljs | 114 +++++++++--------- deps/db/src/logseq/db/frontend/kv_entity.cljs | 28 +++++ deps/db/src/logseq/db/frontend/property.cljs | 44 +++++-- src/main/frontend/core.cljs | 1 + src/main/frontend/worker/rtc/const.cljs | 36 +++++- .../rtc/full_upload_download_graph.cljs | 66 +++++----- 8 files changed, 206 insertions(+), 100 deletions(-) create mode 100644 deps/db/src/logseq/db/frontend/kv_entity.cljs diff --git a/.clj-kondo/hooks/defkeywords.clj b/.clj-kondo/hooks/defkeywords.clj index abcc331976..25bd8603c7 100644 --- a/.clj-kondo/hooks/defkeywords.clj +++ b/.clj-kondo/hooks/defkeywords.clj @@ -19,11 +19,9 @@ :type :defkeywords/invalid-arg)) :else (let [new-node (api/list-node - (map (fn [k] + (map (fn [[kw v]] (api/list-node - [(api/token-node 'logseq.common.defkeywords/defkeyword) - k - (api/token-node "")])) - kws))] + [(api/token-node 'logseq.common.defkeywords/defkeyword) kw v])) + kw->v))] {:node (with-meta new-node (meta node))})))) diff --git a/deps/common/src/logseq/common/defkeywords.cljc b/deps/common/src/logseq/common/defkeywords.cljc index ad4b8e9918..63be7bda3b 100644 --- a/deps/common/src/logseq/common/defkeywords.cljc +++ b/deps/common/src/logseq/common/defkeywords.cljc @@ -1,8 +1,9 @@ (ns logseq.common.defkeywords - "Macro 'defkeywords' to def keyword with docstring" + "Macro 'defkeywords' to def keyword with config" #?(:cljs (:require-macros [logseq.common.defkeywords]))) (def ^:private *defined-kws (volatile! {})) +(def *defined-kw->config (volatile! {})) #_:clj-kondo/ignore (defmacro defkeyword @@ -24,4 +25,8 @@ (vswap! *defined-kws assoc kw current-meta) (throw (ex-info "keyword already defined somewhere else" {:kw kw :info info})))) (vswap! *defined-kws assoc kw current-meta))) - `(vector ~@keyvals)) + (let [kw->v (partition 2 keyvals)] + `(do + (doseq [[kw# config#] '~kw->v] + (vswap! *defined-kw->config assoc kw# config#)) + (vector ~@keyvals)))) diff --git a/deps/db/src/logseq/db/frontend/class.cljs b/deps/db/src/logseq/db/frontend/class.cljs index 4e9fdcca72..7e0f0acb11 100644 --- a/deps/db/src/logseq/db/frontend/class.cljs +++ b/deps/db/src/logseq/db/frontend/class.cljs @@ -2,6 +2,7 @@ "Class related fns for DB graphs and frontend/datascript usage" (:require [clojure.set :as set] [flatland.ordered.map :refer [ordered-map]] + [logseq.common.defkeywords :refer [defkeywords]] [logseq.db.frontend.db-ident :as db-ident] [logseq.db.sqlite.util :as sqlite-util])) @@ -10,76 +11,77 @@ (def ^:large-vars/data-var built-in-classes "Map of built-in classes for db graphs with their :db/ident as keys" - (ordered-map - :logseq.class/Root {:title "Root Tag"} + (apply + ordered-map + (defkeywords + :logseq.class/Root {:title "Root Tag"} - :logseq.class/Tag {:title "Tag"} + :logseq.class/Tag {:title "Tag"} - :logseq.class/Property {:title "Property"} + :logseq.class/Property {:title "Property"} - :logseq.class/Page {:title "Page"} + :logseq.class/Page {:title "Page"} - :logseq.class/Journal - {:title "Journal" - :properties {:logseq.property/parent :logseq.class/Page - :logseq.property.journal/title-format "MMM do, yyyy"}} + :logseq.class/Journal + {:title "Journal" + :properties {:logseq.property/parent :logseq.class/Page + :logseq.property.journal/title-format "MMM do, yyyy"}} - :logseq.class/Whiteboard - {:title "Whiteboard" - :properties {:logseq.property/parent :logseq.class/Page}} + :logseq.class/Whiteboard + {:title "Whiteboard" + :properties {:logseq.property/parent :logseq.class/Page}} - :logseq.class/Task - {:title "Task" - :schema {:properties [:logseq.task/status :logseq.task/priority :logseq.task/deadline :logseq.task/scheduled]}} + :logseq.class/Task + {:title "Task" + :schema {:properties [:logseq.task/status :logseq.task/priority :logseq.task/deadline :logseq.task/scheduled]}} - :logseq.class/Query - {:title "Query" - :properties {:logseq.property/icon {:type :tabler-icon :id "search"}} - :schema {:properties [:logseq.property/query]}} + :logseq.class/Query + {:title "Query" + :properties {:logseq.property/icon {:type :tabler-icon :id "search"}} + :schema {:properties [:logseq.property/query]}} - :logseq.class/Card - {:title "Card" - :schema {:properties [:logseq.property.fsrs/state :logseq.property.fsrs/due]}} + :logseq.class/Card + {:title "Card" + :schema {:properties [:logseq.property.fsrs/state :logseq.property.fsrs/due]}} - :logseq.class/Cards - {:title "Cards" - :properties {:logseq.property/icon {:type :tabler-icon :id "search"} - :logseq.property/parent :logseq.class/Query}} + :logseq.class/Cards + {:title "Cards" + :properties {:logseq.property/icon {:type :tabler-icon :id "search"} + :logseq.property/parent :logseq.class/Query}} - :logseq.class/Asset - {:title "Asset" - :properties {;; :logseq.property/icon {:type :tabler-icon :id "file"} - :logseq.property.class/hide-from-node true - :logseq.property.view/type :logseq.property.view/type.gallery} - :schema {:properties [:logseq.property.asset/type :logseq.property.asset/size :logseq.property.asset/checksum] - :required-properties [:logseq.property.asset/type :logseq.property.asset/size :logseq.property.asset/checksum]}} + :logseq.class/Asset + {:title "Asset" + :properties {;; :logseq.property/icon {:type :tabler-icon :id "file"} + :logseq.property.class/hide-from-node true + :logseq.property.view/type :logseq.property.view/type.gallery} + :schema {:properties [:logseq.property.asset/type :logseq.property.asset/size :logseq.property.asset/checksum] + :required-properties [:logseq.property.asset/type :logseq.property.asset/size :logseq.property.asset/checksum]}} - :logseq.class/Code-block - {:title "Code" - :properties {:logseq.property.class/hide-from-node true} - :schema {:properties [:logseq.property.node/display-type :logseq.property.code/lang]}} + :logseq.class/Code-block + {:title "Code" + :properties {:logseq.property.class/hide-from-node true} + :schema {:properties [:logseq.property.node/display-type :logseq.property.code/lang]}} - :logseq.class/Quote-block - {:title "Quote" - :properties {:logseq.property.class/hide-from-node true} - :schema {:properties [:logseq.property.node/display-type]}} + :logseq.class/Quote-block + {:title "Quote" + :properties {:logseq.property.class/hide-from-node true} + :schema {:properties [:logseq.property.node/display-type]}} - :logseq.class/Math-block - {:title "Math" - :properties {:logseq.property.class/hide-from-node true} - :schema {:properties [:logseq.property.node/display-type]}} + :logseq.class/Math-block + {:title "Math" + :properties {:logseq.property.class/hide-from-node true} + :schema {:properties [:logseq.property.node/display-type]}} - :logseq.class/Pdf-annotation - {:title "PDF Annotation" - :properties {:logseq.property.class/hide-from-node true} - :schema {:properties [:logseq.property/ls-type :logseq.property.pdf/hl-color :logseq.property/asset - :logseq.property.pdf/hl-page :logseq.property.pdf/hl-value - :logseq.property.pdf/hl-type :logseq.property.pdf/hl-image] - :required-properties [:logseq.property/ls-type :logseq.property.pdf/hl-color :logseq.property/asset - :logseq.property.pdf/hl-page :logseq.property.pdf/hl-value]}} - -;; TODO: Add more classes such as :book, :paper, :movie, :music, :project) - )) + :logseq.class/Pdf-annotation + {:title "PDF Annotation" + :properties {:logseq.property.class/hide-from-node true} + :schema {:properties [:logseq.property/ls-type :logseq.property.pdf/hl-color :logseq.property/asset + :logseq.property.pdf/hl-page :logseq.property.pdf/hl-value + :logseq.property.pdf/hl-type :logseq.property.pdf/hl-image] + :required-properties [:logseq.property/ls-type :logseq.property.pdf/hl-color :logseq.property/asset + :logseq.property.pdf/hl-page :logseq.property.pdf/hl-value]}} + ;; TODO: Add more classes such as :book, :paper, :movie, :music, :project) + ))) (def page-children-classes "Children of :logseq.class/Page" diff --git a/deps/db/src/logseq/db/frontend/kv_entity.cljs b/deps/db/src/logseq/db/frontend/kv_entity.cljs new file mode 100644 index 0000000000..44030c74a8 --- /dev/null +++ b/deps/db/src/logseq/db/frontend/kv_entity.cljs @@ -0,0 +1,28 @@ +(ns logseq.db.frontend.kv-entity + "Define kv entities used by logseq db" + (:require [logseq.common.defkeywords :refer [defkeywords]])) + +(defkeywords + :logseq.kv/db-type {:doc ":kv/value = \"db\" if it's a db-graph"} + :logseq.kv/graph-uuid {:doc "store graph-uuid if it's a rtc enabled graph" + :rtc {:rtc/ignore-entity-when-init-upload true + :rtc/ignore-entity-when-init-download true}} + :logseq.kv/import-type {:doc ":sqlite-db if import from sqlite. + FIXME: any other values?" + :rtc {:rtc/ignore-entity-when-init-upload true + :rtc/ignore-entity-when-init-download true}} + :logseq.kv/imported-at {:doc "graph-import time" + :rtc {:rtc/ignore-entity-when-init-upload true + :rtc/ignore-entity-when-init-download true}} + :logseq.kv/graph-local-tx {:doc "local rtc tx-id" + :rtc {:rtc/ignore-entity-when-init-upload true + :rtc/ignore-entity-when-init-download true}} + :logseq.kv/schema-version {:doc "schema version"} + :logseq.kv/graph-created-at {:doc "graph create time"} + :logseq.kv/latest-code-lang {:doc "latest lang used by code-block" + :rtc {:rtc/ignore-entity-when-init-upload true + :rtc/ignore-entity-when-init-download true}} + :logseq.kv/graph-backup-folder {:doc "graph backup-folder" + :rtc {:rtc/ignore-entity-when-init-upload true + :rtc/ignore-entity-when-init-download true}} + :logseq.kv/graph-initial-schema-version {:doc "schema-version when graph created"}) diff --git a/deps/db/src/logseq/db/frontend/property.cljs b/deps/db/src/logseq/db/frontend/property.cljs index 613417f09f..51f81d646b 100644 --- a/deps/db/src/logseq/db/frontend/property.cljs +++ b/deps/db/src/logseq/db/frontend/property.cljs @@ -52,7 +52,9 @@ * :attribute - Property keyword that is saved to a datascript attribute outside of :block/properties * :queryable? - Boolean for whether property can be queried in the query builder * :closed-values - Vec of closed-value maps for properties with choices. Map - has keys :value, :db-ident, :uuid and :icon" + has keys :value, :db-ident, :uuid and :icon + * :rtc - submap for RTC configs. view docs by jumping to keyword definitions. + " (apply ordered-map (defkeywords @@ -435,38 +437,56 @@ [:logseq.property.view/type.list "List View"] [:logseq.property.view/type.gallery "Gallery View"]]) :properties {:logseq.property/default-value :logseq.property.view/type.table} - :queryable? true} + :queryable? true + :rtc {:rtc/ignore-attr-when-init-upload true + :rtc/ignore-attr-when-init-download true + :rtc/ignore-attr-when-syncing true}} :logseq.property.table/sorting {:title "View sorting" :schema {:type :coll :hide? true - :public? false}} + :public? false} + :rtc {:rtc/ignore-attr-when-init-upload true + :rtc/ignore-attr-when-init-download true + :rtc/ignore-attr-when-syncing true}} :logseq.property.table/filters {:title "View filters" :schema {:type :coll :hide? true - :public? false}} + :public? false} + :rtc {:rtc/ignore-attr-when-init-upload true + :rtc/ignore-attr-when-init-download true + :rtc/ignore-attr-when-syncing true}} :logseq.property.table/hidden-columns {:title "View hidden columns" :schema {:type :keyword :cardinality :many :hide? true - :public? false}} + :public? false} + :rtc {:rtc/ignore-attr-when-init-upload true + :rtc/ignore-attr-when-init-download true + :rtc/ignore-attr-when-syncing true}} :logseq.property.table/ordered-columns {:title "View ordered columns" :schema {:type :coll :hide? true - :public? false}} + :public? false} + :rtc {:rtc/ignore-attr-when-init-upload true + :rtc/ignore-attr-when-init-download true + :rtc/ignore-attr-when-syncing true}} :logseq.property.table/sized-columns {:title "View columns settings" :schema {:type :map :hide? true - :public? false}} + :public? false} + :rtc {:rtc/ignore-attr-when-init-upload true + :rtc/ignore-attr-when-init-download true + :rtc/ignore-attr-when-syncing true}} :logseq.property/view-for {:title "This view belongs to" :schema @@ -490,12 +510,18 @@ :logseq.property.asset/last-visit-page {:title "Last visit page" :schema {:type :raw-number :hide? true - :public? false}} + :public? false} + :rtc {:rtc/ignore-attr-when-init-upload true + :rtc/ignore-attr-when-init-download true + :rtc/ignore-attr-when-syncing true}} :logseq.property.asset/remote-metadata {:title "File remote metadata" :schema {:type :map :hide? true - :public? false}} + :public? false} + :rtc {:rtc/ignore-attr-when-init-upload true + :rtc/ignore-attr-when-init-download true + :rtc/ignore-attr-when-syncing true}} :logseq.property.asset/resize-metadata {:title "Asset resize metadata" :schema {:type :map :hide? true diff --git a/src/main/frontend/core.cljs b/src/main/frontend/core.cljs index 614074a480..4f9a62e70b 100644 --- a/src/main/frontend/core.cljs +++ b/src/main/frontend/core.cljs @@ -14,6 +14,7 @@ [frontend.routes :as routes] [frontend.spec] [logseq.api] + [logseq.db.frontend.kv-entity] [malli.dev.cljs :as md] [reitit.frontend :as rf] [reitit.frontend.easy :as rfe] diff --git a/src/main/frontend/worker/rtc/const.cljs b/src/main/frontend/worker/rtc/const.cljs index efb9a5e39e..81235fcd30 100644 --- a/src/main/frontend/worker/rtc/const.cljs +++ b/src/main/frontend/worker/rtc/const.cljs @@ -1,5 +1,39 @@ (ns frontend.worker.rtc.const - "RTC constants") + "RTC constants" + (:require [logseq.common.defkeywords :as common-def :refer [defkeywords]])) (goog-define RTC-E2E-TEST* false) (def RTC-E2E-TEST RTC-E2E-TEST*) + +(defkeywords + :rtc/ignore-attr-when-init-upload + {:doc "keyword option for RTC. ignore this *attr* when initial uploading graph. Default false"} + :rtc/ignore-attr-when-init-download + {:doc "keyword option for RTC. ignore this *attr* when initial downloading graph. Default false"} + :rtc/ignore-attr-when-syncing + {:doc "keyword option for RTC. ignore this *attr* when syncing graph. Default false"} + :rtc/ignore-entity-when-init-upload + {:doc "keyword option for RTC. ignore this *entity* when initial uploading graph. Default false"} + :rtc/ignore-entity-when-init-download + {:doc "keyword option for RTC. ignore this *entity* when initial downloading graph. Default false"} + + ;; only blocks(:block/uuid) will be synced, this option is meaningless for now + ;; :rtc/ignore-entity-when-syncing + ;; {:doc "keyword option for RTC. ignore this *entity* when syncing graph. Default false"} + ) + + +(def *ignore-attrs-when-init-upload + (delay (into #{} + (keep (fn [[kw config]] (when (get-in config [:rtc :rtc/ignore-attr-when-init-upload]) kw))) + @common-def/*defined-kw->config))) + +(def *ignore-attrs-when-init-download + (delay (into #{} + (keep (fn [[kw config]] (when (get-in config [:rtc :rtc/ignore-attr-when-init-download]) kw))) + @common-def/*defined-kw->config))) + +(def *ignore-attrs-when-syncing + (delay (into #{} + (keep (fn [[kw config]] (when (get-in config [:rtc :rtc/ignore-attr-when-syncing]) kw))) + @common-def/*defined-kw->config))) diff --git a/src/main/frontend/worker/rtc/full_upload_download_graph.cljs b/src/main/frontend/worker/rtc/full_upload_download_graph.cljs index e239be6cf6..c4379a9a96 100644 --- a/src/main/frontend/worker/rtc/full_upload_download_graph.cljs +++ b/src/main/frontend/worker/rtc/full_upload_download_graph.cljs @@ -78,7 +78,7 @@ db-schema))) (defn- export-as-blocks - [db] + [db & {:keys [ignore-attr-set]}] (let [datoms (d/datoms db :eavt) db-schema (d/schema db) card-many-attrs (schema->card-many-attrs db-schema) @@ -92,18 +92,19 @@ (when (and (contains? #{:block/parent} (:a datom)) (not (pos-int? (:v datom)))) (throw (ex-info "invalid block data" {:datom datom}))) - (let [a (:a datom) - card-many? (contains? card-many-attrs a) - ref? (contains? ref-type-attrs a)] - (case [ref? card-many?] - [true true] - (update r a conj (str (:v datom))) - [true false] - (assoc r a (str (:v datom))) - [false true] - (update r a conj (ldb/write-transit-str (:v datom))) - [false false] - (assoc r a (ldb/write-transit-str (:v datom)))))) + (let [a (:a datom)] + (when-not (contains? ignore-attr-set a) + (let [card-many? (contains? card-many-attrs a) + ref? (contains? ref-type-attrs a)] + (case [ref? card-many?] + [true true] + (update r a conj (str (:v datom))) + [true false] + (assoc r a (str (:v datom))) + [false true] + (update r a conj (ldb/write-transit-str (:v datom))) + [false false] + (assoc r a (ldb/write-transit-str (:v datom)))))))) {:db/id (str (:e (first datoms)))} datoms)))) (map (fn [block] @@ -122,7 +123,8 @@ vector (ws-util/send&recv get-ws-create-task {:action "presign-put-temp-s3-obj"}) (m/sp - (let [all-blocks (export-as-blocks @conn)] + (let [all-blocks (export-as-blocks + @conn :ignore-attr-set @rtc-const/*ignore-attrs-when-init-upload)] (ldb/write-transit-str all-blocks)))))] (rtc-log-and-state/rtc-log :rtc.log/upload {:sub-type :upload-data :message "uploading data"}) @@ -269,22 +271,24 @@ (assoc :db/ident ident)))) ids) id-ref-exists? (fn [v] (and (string? v) (or (get id->ident v) (get id->uuid v)))) blocks-tx-data (map (fn [block] - (->> (map (fn [[k v]] - (let [v (cond - (id-ref-exists? v) - (or (get id->ident v) [:block/uuid (get id->uuid v)]) + (->> (map + (fn [[k v]] + (let [v (cond + (id-ref-exists? v) + (or (get id->ident v) [:block/uuid (get id->uuid v)]) - (and (sequential? v) (every? id-ref-exists? v)) - (map (fn [id] (or (get id->ident id) [:block/uuid (get id->uuid id)])) v) + (and (sequential? v) (every? id-ref-exists? v)) + (map (fn [id] (or (get id->ident id) [:block/uuid (get id->uuid id)])) v) - :else - v)] - [k v])) (dissoc block :db/id)) + :else + v)] + [k v])) + (dissoc block :db/id)) (into {}))) blocks)] (concat id-tx-data blocks-tx-data))) -(defn- new-task--transact-remote-all-blocks - [all-blocks repo graph-uuid] +(defn- remote-all-blocks=>client-blocks+t + [all-blocks ignore-attr-set] (let [{:keys [t blocks]} all-blocks card-one-attrs (blocks->card-one-attrs blocks) blocks1 (worker-util/profile :convert-card-one-value-from-value-coll @@ -294,7 +298,15 @@ ;;TODO: remove this, client/schema already converted to :db/cardinality, :db/valueType by remote, ;; and :client/schema should be removed by remote too blocks (map #(dissoc % :client/schema) blocks2) - blocks (fill-block-fields blocks) + blocks (if (seq ignore-attr-set) + (map (fn [block] (into {} (remove (comp (partial contains? ignore-attr-set) first)) block)) blocks) + blocks) + blocks (fill-block-fields blocks)] + {:blocks blocks :t t})) + +(defn- new-task--transact-remote-all-blocks + [all-blocks repo graph-uuid] + (let [{:keys [t blocks]} (remote-all-blocks=>client-blocks+t all-blocks @rtc-const/*ignore-attrs-when-init-download) [schema-blocks normal-blocks] (blocks->schema-blocks+normal-blocks blocks) tx-data (concat (blocks-resolve-temp-id normal-blocks) @@ -314,7 +326,7 @@ (.exportDB worker-obj repo) (.transact worker-obj repo init-tx-data {:rtc-download-graph? true :gen-undo-ops? false - ;; only transact db schema, skip validation to avoid warning + ;; only transact db schema, skip validation to avoid warning :frontend.worker.pipeline/skip-validate-db? true :persist-op? false} (worker-state/get-context)) (.transact worker-obj repo tx-data {:rtc-download-graph? true