From 12c6928ba05a547444863216d835bc77a2e08fb9 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 15 May 2025 17:59:22 +0800 Subject: [PATCH 01/17] fix: can't switch to another graph if failed to open current one --- src/main/frontend/worker/shared_service.cljs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/frontend/worker/shared_service.cljs b/src/main/frontend/worker/shared_service.cljs index b330a28c0d..bd7b40e70e 100644 --- a/src/main/frontend/worker/shared_service.cljs +++ b/src/main/frontend/worker/shared_service.cljs @@ -294,10 +294,13 @@ (.postMessage common-channel #js {:type "master-changed" :master-client-id master-client-id :serviceName service-name}) - (p/do! - (on-become-master-handler service-name) - ( + (p/do! + (on-become-master-handler service-name) + ( Date: Thu, 15 May 2025 18:40:39 +0800 Subject: [PATCH 02/17] enhance: fail fast when there're missing addresses --- src/main/frontend/worker/db_worker.cljs | 155 ++++++++++-------------- 1 file changed, 67 insertions(+), 88 deletions(-) diff --git a/src/main/frontend/worker/db_worker.cljs b/src/main/frontend/worker/db_worker.cljs index 5d7402115e..d974becb3f 100644 --- a/src/main/frontend/worker/db_worker.cljs +++ b/src/main/frontend/worker/db_worker.cljs @@ -32,7 +32,6 @@ [frontend.worker.util :as worker-util] [goog.object :as gobj] [lambdaisland.glogi.console :as glogi-console] - [logseq.common.log :as log] [logseq.common.util :as common-util] [logseq.db :as ldb] [logseq.db.common.entity-plus :as entity-plus] @@ -101,38 +100,39 @@ [^js pool data] (.importDb ^js pool repo-path data)) -(defn- get-all-datoms-from-sqlite-db - [db] - (some->> (.exec db #js {:sql "select * from kvs" - :rowMode "array"}) - bean/->clj - (mapcat - (fn [[_addr content _addresses]] - (let [content' (sqlite-util/transit-read content) - datoms (when (map? content') - (:keys content'))] - datoms))) - distinct - (map (fn [[e a v t]] - (d/datom e a v t))))) +(comment + (defn- get-all-datoms-from-sqlite-db + [db] + (some->> (.exec db #js {:sql "select * from kvs" + :rowMode "array"}) + bean/->clj + (mapcat + (fn [[_addr content _addresses]] + (let [content' (sqlite-util/transit-read content) + datoms (when (map? content') + (:keys content'))] + datoms))) + distinct + (map (fn [[e a v t]] + (d/datom e a v t))))) -(defn- rebuild-db-from-datoms! - "Persistent-sorted-set has been broken, used addresses can't be found" - [datascript-conn sqlite-db import-type] - (let [datoms (get-all-datoms-from-sqlite-db sqlite-db) - db (d/init-db [] db-schema/schema - {:storage (storage/storage @datascript-conn)}) - db (d/db-with db - (map (fn [d] - [:db/add (:e d) (:a d) (:v d) (:t d)]) datoms))] - (prn :debug :rebuild-db-from-datoms :datoms-count (count datoms)) + (defn- rebuild-db-from-datoms! + "Persistent-sorted-set has been broken, used addresses can't be found" + [datascript-conn sqlite-db import-type] + (let [datoms (get-all-datoms-from-sqlite-db sqlite-db) + db (d/init-db [] db-schema/schema + {:storage (storage/storage @datascript-conn)}) + db (d/db-with db + (map (fn [d] + [:db/add (:e d) (:a d) (:v d) (:t d)]) datoms))] + (prn :debug :rebuild-db-from-datoms :datoms-count (count datoms)) ;; export db first - (when-not import-type - (worker-util/post-message :notification ["The SQLite db will be exported to avoid any data-loss." :warning false]) - (worker-util/post-message :export-current-db [])) - (.exec sqlite-db #js {:sql "delete from kvs"}) - (d/reset-conn! datascript-conn db) - (db-migrate/fix-db! datascript-conn))) + (when-not import-type + (worker-util/post-message :notification ["The SQLite db will be exported to avoid any data-loss." :warning false]) + (worker-util/post-message :export-current-db [])) + (.exec sqlite-db #js {:sql "delete from kvs"}) + (d/reset-conn! datascript-conn db) + (db-migrate/fix-db! datascript-conn)))) (comment (defn- gc-kvs-table! @@ -158,57 +158,46 @@ :bind #js [addr]})))))))) (defn- find-missing-addresses - [^Object db & {:keys [delete-addrs upsert-addr-content? open-db?]}] - (worker-util/profile - "find-missing-addresses" - (let [schema (some->> (.exec db #js {:sql "select content from kvs where addr = 0" - :rowMode "array"}) - bean/->clj - ffirst - sqlite-util/transit-read) - result (->> (.exec db #js {:sql "select addr, addresses from kvs" - :rowMode "array"}) - bean/->clj - (keep (fn [[addr addresses]] - (when-not (and delete-addrs (delete-addrs addr)) - [addr (bean/->clj (js/JSON.parse addresses))])))) - used-addresses (-> (set (concat (mapcat second result) - [0 1 (:eavt schema) (:avet schema) (:aevt schema)])) - (clojure.set/difference delete-addrs)) - missing-addresses (clojure.set/difference used-addresses (set (map first result)))] - (when (seq missing-addresses) - (prn :error :missing-addresses missing-addresses) - (if worker-util/dev? - (throw (ex-info "Found missing addresses that shouldn't happen" {:missing-addresses missing-addresses})) - (worker-util/post-message :capture-error - {:error "v2-db-missing-addresses" - :payload {:missing-addresses missing-addresses - :upsert-addr-content upsert-addr-content? - :open-db open-db?}}))) - missing-addresses))) + [conn ^Object db & {:keys [delete-addrs]}] + (let [schema (some->> (.exec db #js {:sql "select content from kvs where addr = 0" + :rowMode "array"}) + bean/->clj + ffirst + sqlite-util/transit-read) + result (->> (.exec db #js {:sql "select addr, addresses from kvs" + :rowMode "array"}) + bean/->clj + (keep (fn [[addr addresses]] + (when-not (and delete-addrs (delete-addrs addr)) + [addr (bean/->clj (js/JSON.parse addresses))])))) + used-addresses (-> (set (concat (mapcat second result) + [0 1 (:eavt schema) (:avet schema) (:aevt schema)])) + (clojure.set/difference delete-addrs)) + missing-addresses (clojure.set/difference used-addresses (set (map first result)))] + (when (seq missing-addresses) + (let [version-in-db (when conn (db-schema/parse-schema-version (or (:kv/value (d/entity @conn :logseq.kv/schema-version)) 0))) + compare-result (when version-in-db (db-schema/compare-schema-version version-in-db "64.8"))] + (when (and compare-result (not (neg? compare-result))) ; >= 64.8 + (worker-util/post-message :capture-error + {:error "db-missing-addresses-v2" + :payload {:missing-addresses missing-addresses}})))) + missing-addresses)) (defn upsert-addr-content! "Upsert addr+data-seq. Update sqlite-cli/upsert-addr-content! when making changes" [repo data delete-addrs* & {:keys [client-ops-db?] :or {client-ops-db? false}}] (let [^Object db (worker-state/get-sqlite-conn repo (if client-ops-db? :client-ops :db)) - delete-addrs (set delete-addrs*)] + delete-addrs (clojure.set/difference (set delete-addrs*) #{0 1})] (assert (some? db) "sqlite db not exists") (.transaction db (fn [tx] (doseq [item data] (.exec tx #js {:sql "INSERT INTO kvs (addr, content, addresses) values ($addr, $content, $addresses) on conflict(addr) do update set content = $content, addresses = $addresses" :bind item})))) (when (seq delete-addrs) - (let [missing-addrs (when worker-util/dev? - (seq (find-missing-addresses db {:delete-addrs delete-addrs - :upsert-addr-content? true}))) - delete-addrs' (if missing-addrs - (remove (set missing-addrs) delete-addrs) - delete-addrs)] - (when (seq delete-addrs') - (.transaction db (fn [tx] - (doseq [addr delete-addrs'] - (.exec tx #js {:sql "Delete from kvs WHERE addr = ? AND NOT EXISTS (SELECT 1 FROM json_each(addresses) WHERE value = ?);" - :bind #js [addr]}))))))))) + (.transaction db (fn [tx] + (doseq [addr delete-addrs] + (.exec tx #js {:sql "Delete from kvs WHERE addr = ? AND NOT EXISTS (SELECT 1 FROM json_each(addresses) WHERE value = ?);" + :bind #js [addr]}))))))) (defn restore-data-from-addr "Update sqlite-cli/restore-data-from-addr when making changes" @@ -374,24 +363,14 @@ (when import-type {:import-type import-type}))] (d/transact! conn initial-data {:initial-db? true}))) - (try + ;; TODO: remove this once we can ensure there's no bug for missing addresses + ;; because it's slow for large graphs + (when-not import-type + (when-let [missing-addresses (seq (find-missing-addresses conn db))] + (worker-util/post-message :notification ["It seems that the DB has been broken, please export a backup and contact Logseq team for help." :error false]) + (throw (ex-info "DB missing addresses" {:missing-addresses missing-addresses})))) - ;; TODO: remove this once we can ensure there's no bug for missing addresses - ;; because it's slow for large graphs - (when-not import-type - (when-let [missing-addresses (seq (find-missing-addresses db {:open-db? true}))] - (throw (ex-info "DB missing addresses" {:missing-addresses missing-addresses})))) - - (db-migrate/migrate conn search-db) - - ;; TODO: Remove this once we can ensure there's no bug for missing addresses - (catch :default e - (log/error (str "DB migrate failed for " repo ", error:") e) - (if (= (.-message e) "DB missing addresses") - (do - (rebuild-db-from-datoms! conn db import-type) - (db-migrate/migrate conn search-db)) - (throw e)))) + (db-migrate/migrate conn search-db) (db-listener/listen-db-changes! repo (get @*datascript-conns repo)))))) From b5bf3ff27a545c4c7b22d35a39e79e41d09c73ff Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 15 May 2025 20:41:51 +0800 Subject: [PATCH 03/17] chore: remove new-sqlite-client-ops-storage --- src/main/frontend/worker/db_worker.cljs | 80 ++++++++++--------------- 1 file changed, 31 insertions(+), 49 deletions(-) diff --git a/src/main/frontend/worker/db_worker.cljs b/src/main/frontend/worker/db_worker.cljs index d974becb3f..884ebed626 100644 --- a/src/main/frontend/worker/db_worker.cljs +++ b/src/main/frontend/worker/db_worker.cljs @@ -185,9 +185,8 @@ (defn upsert-addr-content! "Upsert addr+data-seq. Update sqlite-cli/upsert-addr-content! when making changes" - [repo data delete-addrs* & {:keys [client-ops-db?] :or {client-ops-db? false}}] - (let [^Object db (worker-state/get-sqlite-conn repo (if client-ops-db? :client-ops :db)) - delete-addrs (clojure.set/difference (set delete-addrs*) #{0 1})] + [db data delete-addrs*] + (let [delete-addrs (clojure.set/difference (set delete-addrs*) #{0 1})] (assert (some? db) "sqlite db not exists") (.transaction db (fn [tx] (doseq [item data] @@ -197,28 +196,36 @@ (.transaction db (fn [tx] (doseq [addr delete-addrs] (.exec tx #js {:sql "Delete from kvs WHERE addr = ? AND NOT EXISTS (SELECT 1 FROM json_each(addresses) WHERE value = ?);" - :bind #js [addr]}))))))) + :bind #js [addr]})))) + (let [missing-addrs (when worker-util/dev? + (seq (find-missing-addresses nil db {:delete-addrs delete-addrs})))] + (if (seq missing-addrs) + (worker-util/post-message :notification [(str "Bug!! Missing addresses: " missing-addrs) :error false]) + (when (seq delete-addrs) + (.transaction db (fn [tx] + (doseq [addr delete-addrs] + (.exec tx #js {:sql "Delete from kvs WHERE addr = ? AND NOT EXISTS (SELECT 1 FROM json_each(addresses) WHERE value = ?);" + :bind #js [addr]})))))))))) (defn restore-data-from-addr "Update sqlite-cli/restore-data-from-addr when making changes" - [repo addr & {:keys [client-ops-db?] :or {client-ops-db? false}}] - (let [^Object db (worker-state/get-sqlite-conn repo (if client-ops-db? :client-ops :db))] - (assert (some? db) "sqlite db not exists") - (when-let [result (-> (.exec db #js {:sql "select content, addresses from kvs where addr = ?" - :bind #js [addr] - :rowMode "array"}) - first)] - (let [[content addresses] (bean/->clj result) - addresses (when addresses - (js/JSON.parse addresses)) - data (sqlite-util/transit-read content)] - (if (and addresses (map? data)) - (assoc data :addresses addresses) - data))))) + [db addr] + (assert (some? db) "sqlite db not exists") + (when-let [result (-> (.exec db #js {:sql "select content, addresses from kvs where addr = ?" + :bind #js [addr] + :rowMode "array"}) + first)] + (let [[content addresses] (bean/->clj result) + addresses (when addresses + (js/JSON.parse addresses)) + data (sqlite-util/transit-read content)] + (if (and addresses (map? data)) + (assoc data :addresses addresses) + data)))) (defn new-sqlite-storage "Update sqlite-cli/new-sqlite-storage when making changes" - [repo _opts] + [^Object db] (reify IStorage (-store [_ addr+data-seq delete-addrs] (let [used-addrs (set (mapcat @@ -238,36 +245,10 @@ :$content (sqlite-util/transit-write data') :$addresses addresses})) addr+data-seq)] - (upsert-addr-content! repo data delete-addrs))) + (upsert-addr-content! db data delete-addrs))) (-restore [_ addr] - (restore-data-from-addr repo addr)))) - -(defn new-sqlite-client-ops-storage - [repo] - (reify IStorage - (-store [_ addr+data-seq delete-addrs] - (let [used-addrs (set (mapcat - (fn [[addr data]] - (cons addr - (when (map? data) - (:addresses data)))) - addr+data-seq)) - delete-addrs (remove used-addrs delete-addrs) - data (map - (fn [[addr data]] - (let [data' (if (map? data) (dissoc data :addresses) data) - addresses (when (map? data) - (when-let [addresses (:addresses data)] - (js/JSON.stringify (bean/->js addresses))))] - #js {:$addr addr - :$content (sqlite-util/transit-write data') - :$addresses addresses})) - addr+data-seq)] - (upsert-addr-content! repo data delete-addrs :client-ops-db? true))) - - (-restore [_ addr] - (restore-data-from-addr repo addr :client-ops-db? true)))) + (restore-data-from-addr db addr)))) (defn- close-db-aux! [repo ^Object db ^Object search ^Object client-ops] @@ -327,8 +308,9 @@ [repo {:keys [config import-type datoms]}] (when-not (worker-state/get-sqlite-conn repo) (p/let [[db search-db client-ops-db :as dbs] (get-dbs repo) - storage (new-sqlite-storage repo {}) - client-ops-storage (when-not @*publishing? (new-sqlite-client-ops-storage repo)) + storage (new-sqlite-storage db) + client-ops-storage (when-not @*publishing? + (new-sqlite-storage client-ops-db)) db-based? (sqlite-util/db-based-graph? repo)] (swap! *sqlite-conns assoc repo {:db db :search search-db From 38267ece78e797a9d0d6c39a19f6d8c6dc57f856 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 15 May 2025 21:30:43 +0800 Subject: [PATCH 04/17] chore: remove debug --- src/main/frontend/state.cljs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/frontend/state.cljs b/src/main/frontend/state.cljs index de80543ec0..e8ce5cdbe5 100644 --- a/src/main/frontend/state.cljs +++ b/src/main/frontend/state.cljs @@ -2376,7 +2376,6 @@ Similar to re-frame subscriptions" (defn set-highlight-recent-days! [days] - (prn :debug :set :days days) (reset! (:ui/highlight-recent-days @state) days) (storage/set :ui/highlight-recent-days days)) From 42fb68ff6e22548f05c0598b9d0805600e67fc57 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 15 May 2025 21:51:00 +0800 Subject: [PATCH 05/17] fix: invalidate blocks with table filters but not title --- src/main/frontend/worker/db/migrate.cljs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/frontend/worker/db/migrate.cljs b/src/main/frontend/worker/db/migrate.cljs index becefecf72..49931c1e74 100644 --- a/src/main/frontend/worker/db/migrate.cljs +++ b/src/main/frontend/worker/db/migrate.cljs @@ -1200,7 +1200,8 @@ (= #{:block/tx-id} (set (keys entity))) [[:db/retractEntity (:db/id entity)]] - (and (seq (:block/refs entity)) + (and (or (seq (:block/refs entity)) + (:logseq.property.table/filters entity)) (not (or (:block/title entity) (:block/content entity) (:property.value/content entity)))) [[:db/retractEntity (:db/id entity)]] From f5efa2d63829af6c31dc1de97bd17a3b0dd53370 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 15 May 2025 22:36:00 +0800 Subject: [PATCH 06/17] fix: remove invalid blocks that have no dispatch key --- deps/db/src/logseq/db/frontend/malli_schema.cljs | 8 ++------ src/main/frontend/worker/db/migrate.cljs | 8 ++++++-- src/main/frontend/worker/pipeline.cljs | 8 +++++--- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/deps/db/src/logseq/db/frontend/malli_schema.cljs b/deps/db/src/logseq/db/frontend/malli_schema.cljs index fa0be75713..413a42f59b 100644 --- a/deps/db/src/logseq/db/frontend/malli_schema.cljs +++ b/deps/db/src/logseq/db/frontend/malli_schema.cljs @@ -3,9 +3,9 @@ (:require [clojure.set :as set] [clojure.string :as string] [datascript.core :as d] + [logseq.db.common.entity-plus :as entity-plus] [logseq.db.common.order :as db-order] [logseq.db.frontend.class :as db-class] - [logseq.db.common.entity-plus :as entity-plus] [logseq.db.frontend.entity-util :as entity-util] [logseq.db.frontend.property :as db-property] [logseq.db.frontend.property.type :as db-property-type] @@ -482,14 +482,10 @@ :file-block (:logseq.property.history/block d) :property-history-block - (:block/closed-value-property d) :closed-value-block - - (and (:logseq.property/created-from-property d) - (:logseq.property/value d)) + (and (:logseq.property/created-from-property d) (:logseq.property/value d)) :property-value-block - (:block/uuid d) :block (= (:db/ident d) :logseq.property/empty-placeholder) diff --git a/src/main/frontend/worker/db/migrate.cljs b/src/main/frontend/worker/db/migrate.cljs index 49931c1e74..d81a866a5c 100644 --- a/src/main/frontend/worker/db/migrate.cljs +++ b/src/main/frontend/worker/db/migrate.cljs @@ -16,6 +16,7 @@ [logseq.db.common.order :as db-order] [logseq.db.frontend.class :as db-class] [logseq.db.frontend.content :as db-content] + [logseq.db.frontend.malli-schema :as db-malli-schema] [logseq.db.frontend.property :as db-property] [logseq.db.frontend.property.build :as db-property-build] [logseq.db.frontend.schema :as db-schema] @@ -1172,8 +1173,11 @@ (js/console.error e) (throw e))))))) -(defn- build-invalid-tx [entity eid] +(defn- build-invalid-tx [db entity eid] (cond + (nil? (db-malli-schema/entity-dispatch-key db entity)) + [[:db/retractEntity eid]] + (:block/schema entity) [[:db/retract eid :block/schema]] @@ -1286,7 +1290,7 @@ [:db/retract (:db/id entity) k])))))) (into {} entity)) eid (:db/id entity) - fix (build-invalid-tx entity eid)] + fix (build-invalid-tx db entity eid)] (into fix wrong-choice))) invalid-entity-ids) distinct)] diff --git a/src/main/frontend/worker/pipeline.cljs b/src/main/frontend/worker/pipeline.cljs index 08c1bb788e..c3374e57da 100644 --- a/src/main/frontend/worker/pipeline.cljs +++ b/src/main/frontend/worker/pipeline.cljs @@ -107,9 +107,11 @@ (let [valid? (if (get-in tx-report [:tx-meta :reset-conn!]) true (db-validate/validate-tx-report! tx-report (:validate-db-options context)))] - (when (and (get-in context [:validate-db-options :fail-invalid?]) (not valid?)) - (shared-service/broadcast-to-clients! :notification - [["Invalid DB!"] :error])))) + (when-not valid? + (when (or (get-in context [:validate-db-options :fail-invalid?]) worker-util/dev?) + (shared-service/broadcast-to-clients! :notification + [["Invalid DB!"] :error])) + (throw (ex-info "Invalid data" {:graph repo}))))) ;; Ensure :block/order is unique for any block that has :block/parent (when (or (:dev? context) (exists? js/process)) From a02e842ea31ad503e5b3fbaf5b8b36e1550322c0 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 15 May 2025 22:46:19 +0800 Subject: [PATCH 07/17] enhance: move :block/title to page-or-block-attrs Because both page and block require `:block/title`. --- deps/db/src/logseq/db/frontend/malli_schema.cljs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/deps/db/src/logseq/db/frontend/malli_schema.cljs b/deps/db/src/logseq/db/frontend/malli_schema.cljs index 413a42f59b..4cc6b60992 100644 --- a/deps/db/src/logseq/db/frontend/malli_schema.cljs +++ b/deps/db/src/logseq/db/frontend/malli_schema.cljs @@ -249,6 +249,7 @@ (def page-or-block-attrs "Common attributes for page and normal blocks" [[:block/uuid :uuid] + [:block/title :string] [:block/created-at :int] [:block/updated-at :int] ;; Injected by update-properties-in-ents @@ -261,7 +262,6 @@ (def page-attrs "Common attributes for pages" [[:block/name :string] - [:block/title :string] [:block/path-refs {:optional true} [:set :int]]]) (def property-attrs @@ -339,8 +339,7 @@ (def block-attrs "Common attributes for normal blocks" - [[:block/title :string] - [:block/parent :int] + [[:block/parent :int] [:block/order block-order] ;; refs [:block/page :int] @@ -353,8 +352,7 @@ (vec (concat [:map] - [[:block/title :string] - [:block/parent :int] + [[:block/parent :int] ;; These blocks only associate with pages of type "whiteboard" [:block/page :int] [:block/path-refs {:optional true} [:set :int]]] @@ -367,7 +365,7 @@ [:map] [[:logseq.property/value [:or :string :double :boolean]] [:logseq.property/created-from-property :int]] - (remove #(#{:block/title :logseq.property/created-from-property} (first %)) block-attrs) + (remove #(#{:logseq.property/created-from-property} (first %)) block-attrs) page-or-block-attrs))) (def property-history-block* @@ -400,8 +398,8 @@ [:logseq.property/value {:optional true} [:or :string :double]] [:logseq.property/created-from-property :int] [:block/closed-value-property {:optional true} [:set :int]]] - (remove #(#{:block/title :logseq.property/created-from-property} (first %)) block-attrs) - page-or-block-attrs))) + (remove #(#{:logseq.property/created-from-property} (first %)) block-attrs) + (remove #(#{:block/title} (first %)) page-or-block-attrs)))) (def closed-value-block "A closed value for a property with closed/allowed values" From 693b2c166da30edab6c3876e2c67ebb0a240983d Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 15 May 2025 23:13:04 +0800 Subject: [PATCH 08/17] enhance: add db schema versions to error capture --- src/main/frontend/handler/events.cljs | 6 +++++- src/main/frontend/modules/instrumentation/sentry.cljs | 8 ++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/frontend/handler/events.cljs b/src/main/frontend/handler/events.cljs index 2c8f530334..d674f90e81 100644 --- a/src/main/frontend/handler/events.cljs +++ b/src/main/frontend/handler/events.cljs @@ -50,6 +50,7 @@ [frontend.util.persist-var :as persist-var] [goog.dom :as gdom] [lambdaisland.glogi :as log] + [logseq.db.frontend.schema :as db-schema] [promesa.core :as p])) ;; TODO: should we move all events here? @@ -199,7 +200,10 @@ :user-id user-uuid :graph-id graph-uuid :tx-id tx-id - :db-based (config/db-based-graph? (state/get-current-repo)))] + :db-based (config/db-based-graph? (state/get-current-repo)) + :schema-version db-schema/version + :db-schema-version (when-let [db (frontend.db/get-db)] + (:kv/value (frontend.db/entity db :logseq.kv/schema-version))))] (Sentry/captureException error (bean/->js {:tags payload})))) diff --git a/src/main/frontend/modules/instrumentation/sentry.cljs b/src/main/frontend/modules/instrumentation/sentry.cljs index b75130f8be..3d3558da73 100644 --- a/src/main/frontend/modules/instrumentation/sentry.cljs +++ b/src/main/frontend/modules/instrumentation/sentry.cljs @@ -16,14 +16,14 @@ version) :environment (if config/dev? "development" "production") :initialScope {:tags - (merge - (when (not-empty config/revision) - {:revision config/revision}) + (cond-> {:platform (cond (util/electron?) "electron" (mobile-util/native-platform?) "mobile" :else "web") - :publishing config/publishing?})} + :publishing config/publishing?} + (not-empty config/revision) + (assoc :revision config/revision))} ;; :integrations [(new posthog/SentryIntegration posthog "logseq" 5311485) ;; (new BrowserTracing)] :debug config/dev? From 09b333a5c3a56b75b3d2ed592dce39c1ab64939b Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 15 May 2025 23:47:56 +0800 Subject: [PATCH 09/17] fix: pages displayed as invalid ref in table view --- src/main/frontend/components/block.cljs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index ba31bff564..44329b9f09 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -62,7 +62,6 @@ [frontend.mobile.util :as mobile-util] [frontend.modules.outliner.tree :as tree] [frontend.modules.shortcut.utils :as shortcut-utils] - [frontend.util.ref :as ref] [frontend.security :as security] [frontend.state :as state] [frontend.template :as template] @@ -70,6 +69,7 @@ [frontend.util :as util] [frontend.util.file-based.clock :as clock] [frontend.util.file-based.drawer :as drawer] + [frontend.util.ref :as ref] [frontend.util.text :as text-util] [goog.dom :as gdom] [goog.functions :refer [debounce]] @@ -939,13 +939,13 @@ "Component for a page. `page` argument contains :block/name which can be (un)sanitized page name. Keys for `config`: - `:preview?`: Is this component under preview mode? (If true, `page-preview-trigger` won't be registered to this `page-cp`)" - [state {:keys [label children preview? disable-preview? show-non-exists-page? table-view? tag? _skip-async-load?] :as config} page] + [state {:keys [label children preview? disable-preview? show-non-exists-page? tag? _skip-async-load?] :as config} page] (when-let [entity' (rum/react (:*entity state))] (let [entity (or (db/sub-block (:db/id entity')) entity') config (assoc config :block entity)] (cond entity - (if (ldb/page? entity) + (if (or (ldb/page? entity) (not (:block/page entity))) (let [page-name (some-> (:block/title entity) util/page-name-sanity-lc) whiteboard-page? (model/whiteboard-page? entity) inner (page-inner (assoc config :whiteboard-page? whiteboard-page?) entity children label) @@ -962,10 +962,7 @@ (gp-mldoc/inline->edn label (mldoc/get-default-config :markdown)) label))) - (and (:block/name page) (util/uuid-string? (:block/name page))) - (invalid-node-ref (:block/name page)) - - (and (:block/name page) (or show-non-exists-page? table-view?)) + (and (:block/name page) show-non-exists-page?) (page-inner config (merge {:block/title (:block/name page) :block/name (:block/name page)} From 765568bd4eb5028d17dba998d0fdc14806a27ef7 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Fri, 16 May 2025 00:07:43 +0800 Subject: [PATCH 10/17] enhance: filter recently updated when input Also, don't show new page for built-in files --- src/main/frontend/components/cmdk/core.cljs | 24 ++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/main/frontend/components/cmdk/core.cljs b/src/main/frontend/components/cmdk/core.cljs index b92d6900a0..ada9d2cc06 100644 --- a/src/main/frontend/components/cmdk/core.cljs +++ b/src/main/frontend/components/cmdk/core.cljs @@ -81,7 +81,9 @@ (string/replace input #"^#+" "")) (defn create-items [q] - (when (and (not (string/blank? q)) (not config/publishing?)) + (when (and (not (string/blank? q)) + (not (#{"config.edn" "custom.js" "custom.css"} q)) + (not config/publishing?)) (let [class? (string/starts-with? q "#")] (->> [{:text (if class? "Create tag" "Create page") :icon "new-page" :icon-theme :gray @@ -236,6 +238,25 @@ (hash-map :status :success :items) (swap! !results update group merge))))) +(defmethod load-results :recently-updated-pages [group state] + (let [!input (::input state) + !results (::results state)] + (swap! !results assoc-in [group :status] :loading) + (let [recent-pages (ldb/get-recent-updated-pages (db/get-db)) + search-results (if (string/blank? @!input) + recent-pages + (search/fuzzy-search recent-pages @!input {:extract-fn :block/title}))] + (->> search-results + (map (fn [block] + (let [text (block-handler/block-unique-title block) + icon (get-page-icon block)] + {:icon icon + :icon-theme :gray + :text text + :source-block block}))) + (hash-map :status :success :items) + (swap! !results update group merge))))) + (defn highlight-content-query "Return hiccup of highlighted content FTS result" [content q] @@ -396,6 +417,7 @@ (load-results :nodes state) (load-results :filters state) (load-results :files state) + (load-results :recently-updated-pages state) ;; (load-results :recents state) ))))) From a83285f7051f11f9f718baae91b1b9d890bb4784 Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Thu, 15 May 2025 12:08:32 -0400 Subject: [PATCH 11/17] Revert "enhance: move :block/title to page-or-block-attrs" This commit causing graph invalid errors and failing tests. This reverts commit a02e842ea31ad503e5b3fbaf5b8b36e1550322c0. --- deps/db/src/logseq/db/frontend/malli_schema.cljs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/deps/db/src/logseq/db/frontend/malli_schema.cljs b/deps/db/src/logseq/db/frontend/malli_schema.cljs index 4cc6b60992..413a42f59b 100644 --- a/deps/db/src/logseq/db/frontend/malli_schema.cljs +++ b/deps/db/src/logseq/db/frontend/malli_schema.cljs @@ -249,7 +249,6 @@ (def page-or-block-attrs "Common attributes for page and normal blocks" [[:block/uuid :uuid] - [:block/title :string] [:block/created-at :int] [:block/updated-at :int] ;; Injected by update-properties-in-ents @@ -262,6 +261,7 @@ (def page-attrs "Common attributes for pages" [[:block/name :string] + [:block/title :string] [:block/path-refs {:optional true} [:set :int]]]) (def property-attrs @@ -339,7 +339,8 @@ (def block-attrs "Common attributes for normal blocks" - [[:block/parent :int] + [[:block/title :string] + [:block/parent :int] [:block/order block-order] ;; refs [:block/page :int] @@ -352,7 +353,8 @@ (vec (concat [:map] - [[:block/parent :int] + [[:block/title :string] + [:block/parent :int] ;; These blocks only associate with pages of type "whiteboard" [:block/page :int] [:block/path-refs {:optional true} [:set :int]]] @@ -365,7 +367,7 @@ [:map] [[:logseq.property/value [:or :string :double :boolean]] [:logseq.property/created-from-property :int]] - (remove #(#{:logseq.property/created-from-property} (first %)) block-attrs) + (remove #(#{:block/title :logseq.property/created-from-property} (first %)) block-attrs) page-or-block-attrs))) (def property-history-block* @@ -398,8 +400,8 @@ [:logseq.property/value {:optional true} [:or :string :double]] [:logseq.property/created-from-property :int] [:block/closed-value-property {:optional true} [:set :int]]] - (remove #(#{:logseq.property/created-from-property} (first %)) block-attrs) - (remove #(#{:block/title} (first %)) page-or-block-attrs)))) + (remove #(#{:block/title :logseq.property/created-from-property} (first %)) block-attrs) + page-or-block-attrs))) (def closed-value-block "A closed value for a property with closed/allowed values" From dda13d61b93fbbd493a23eef8705738477721220 Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Thu, 15 May 2025 12:12:29 -0400 Subject: [PATCH 12/17] enhance: improved config.edn for new db graphs New db config.edn don't include any of the file graph keys or comments. Previously all comments were confusingly left lying around. Also moved conversion of imported config.edn so that CLI can use it. Also temporarily moved one dep to https://github.com/nextjournal/nbb-test-runner/pull/2 until the PR is merged --- deps/common/nbb.edn | 5 +- deps/common/resources/templates/config.edn | 195 +++++++++--------- deps/common/src/logseq/common/config.cljs | 42 +++- .../test/logseq/common/config_test.cljc | 45 ++++ .../test/logseq/common/config_test.cljs | 21 -- .../src/logseq/graph_parser/exporter.cljs | 7 +- .../logseq/graph_parser/exporter_test.cljs | 2 +- src/main/frontend/components/imports.cljs | 17 +- .../frontend/handler/common/config_edn.cljs | 39 +--- src/main/frontend/handler/repo.cljs | 16 +- 10 files changed, 216 insertions(+), 173 deletions(-) create mode 100644 deps/common/test/logseq/common/config_test.cljc delete mode 100644 deps/common/test/logseq/common/config_test.cljs diff --git a/deps/common/nbb.edn b/deps/common/nbb.edn index cb82c3e8e3..0c73190ba5 100644 --- a/deps/common/nbb.edn +++ b/deps/common/nbb.edn @@ -1,4 +1,5 @@ {:paths ["src" "resources"] :deps - {io.github.nextjournal/nbb-test-runner - {:git/sha "60ed57aa04bca8d604f5ba6b28848bd887109347"}}} + ;; TODO: Remove fork when https://github.com/nextjournal/nbb-test-runner/pull/2 is merged + {io.github.colin-p-hill/nbb-test-runner + {:git/sha "def2cbdb5b3a0e1612b28bf64f5d869c27c733d3"}}} diff --git a/deps/common/resources/templates/config.edn b/deps/common/resources/templates/config.edn index 6611f68839..9e46af8c11 100644 --- a/deps/common/resources/templates/config.edn +++ b/deps/common/resources/templates/config.edn @@ -1,5 +1,7 @@ {:meta/version 1 + ;; == FILE GRAPH CONFIG == + ;; ;; Set the preferred format. ;; This is _only_ for file graphs. ;; Available options: @@ -42,6 +44,102 @@ ;; Default value: "yyyy_MM_dd" ;; :journal/file-name-format "yyyy_MM_dd" + ;; Set the default location for storing notes. + ;; This is _only_ for file graphs. + ;; Default value: "pages" + ;; :pages-directory "pages" + + ;; Set the default location for storing journals. + ;; This is _only_ for file graphs. + ;; Default value: "journals" + ;; :journals-directory "journals" + + ;; Set the default location for storing whiteboards. + ;; This is _only_ for file graphs. + ;; Default value: "whiteboards" + ;; :whiteboards-directory "whiteboards" + + ;; Enabling this option converts + ;; [[Grant Ideas]] to [[file:./grant_ideas.org][Grant Ideas]] for org-mode. + ;; For more information, visit https://github.com/logseq/logseq/issues/672 + ;; This is _only_ for file graphs. + ;; :org-mode/insert-file-link? false + +;; Favorites to list on the left sidebar + ;; This is _only_ for file graphs. + :favorites [] + + ;; Set flashcards interval. + ;; This is _only_ for file graphs. + ;; Expected value: + ;; - Float between 0 and 1 + ;; higher values result in faster changes to the next review interval. + ;; Default value: 0.5 + ;; :srs/learning-fraction 0.5 + + ;; Set the initial interval after the first successful review of a card. + ;; This is _only_ for file graphs. + ;; Default value: 4 + ;; :srs/initial-interval 4 + + ;; Hide specific block properties. + ;; This is _only_ for file graphs. + ;; Example usage: + ;; :block-hidden-properties #{:public :icon} + + ;; Create a page for all properties. + ;; This is _only_ for file graphs. + ;; Default value: true + :property-pages/enabled? true + + ;; Properties to exclude from having property pages + ;; This is _only_ for file graphs. + ;; Example usage: + ;; :property-pages/excludelist #{:duration :author} + + ;; By default, property value separated by commas will not be treated as + ;; page references. You can add properties to enable it. + ;; This is _only_ for file graphs. + ;; Example usage: + ;; :property/separated-by-commas #{:alias :tags} + + ;; Properties that are ignored when parsing property values for references + ;; This is _only_ for file graphs. + ;; Example usage: + ;; :ignored-page-references-keywords #{:author :website} + + ;; logbook configuration. + ;; This is _only_ for file graphs. + ;; :logbook/settings + ;; {:with-second-support? false ;limit logbook to minutes, seconds will be eliminated + ;; :enabled-in-all-blocks true ;display logbook in all blocks after timetracking + ;; :enabled-in-timestamped-blocks false ;don't display logbook at all + ;; } + + ;; File sync options + ;; Ignore these files when syncing, regexp is supported. + ;; This is _only_ for file graphs. + ;; :file-sync/ignore-files [] + + ;; Configure the escaping method for special characters in page titles. + ;; This is _only_ for file graphs. + ;; Warning: + ;; This is a dangerous operation. To modify the setting, + ;; you'll need to manually rename all affected files and + ;; re-index them on all clients after synchronization. + ;; Incorrect handling may result in messy page titles. + ;; Available options: + ;; - :triple-lowbar (default) + ;; ;use triple underscore `___` for slash `/` in page title + ;; ;use Percent-encoding for other invalid characters + :file/name-format :triple-lowbar + ;; == END OF FILE GRAPH CONFIG == + + ;; Hide empty block properties + ;; This is _only_ for DB graphs. + ;; Default value: false + ;; :ui/hide-empty-properties? false + ;; Enable tooltip preview on hover. ;; Default value: true :ui/enable-tooltip? true @@ -58,11 +156,6 @@ ;; Default value: true :ui/auto-expand-block-refs? true - ;; Hide empty block properties - ;; This is _only_ for DB graphs. - ;; Default value: false - ;; :ui/hide-empty-properties? false - ;; Disable accent marks when searching. ;; After changing this setting, rebuild the search index by pressing (^C ^S). ;; Default value: true @@ -137,27 +230,6 @@ ;; 3. Set "home" as the home page and display multiple pages in the right sidebar: ;; :default-home {:page "home", :sidebar ["Page A" "Page B"]} - ;; Set the default location for storing notes. - ;; This is _only_ for file graphs. - ;; Default value: "pages" - ;; :pages-directory "pages" - - ;; Set the default location for storing journals. - ;; This is _only_ for file graphs. - ;; Default value: "journals" - ;; :journals-directory "journals" - - ;; Set the default location for storing whiteboards. - ;; This is _only_ for file graphs. - ;; Default value: "whiteboards" - ;; :whiteboards-directory "whiteboards" - - ;; Enabling this option converts - ;; [[Grant Ideas]] to [[file:./grant_ideas.org][Grant Ideas]] for org-mode. - ;; For more information, visit https://github.com/logseq/logseq/issues/672 - ;; This is _only_ for file graphs. - ;; :org-mode/insert-file-link? false - ;; Configure custom shortcuts. ;; Syntax: ;; 1. + indicates simultaneous key presses, e.g., `Ctrl+Shift+a`. @@ -274,58 +346,6 @@ ;; :charge-strength -600 ; Default value: -600 ;; :charge-range 600} ; Default value: 600 - - ;; Favorites to list on the left sidebar - ;; This is _only_ for file graphs. - :favorites [] - - ;; Set flashcards interval. - ;; This is _only_ for file graphs. - ;; Expected value: - ;; - Float between 0 and 1 - ;; higher values result in faster changes to the next review interval. - ;; Default value: 0.5 - ;; :srs/learning-fraction 0.5 - - ;; Set the initial interval after the first successful review of a card. - ;; This is _only_ for file graphs. - ;; Default value: 4 - ;; :srs/initial-interval 4 - - ;; Hide specific block properties. - ;; This is _only_ for file graphs. - ;; Example usage: - ;; :block-hidden-properties #{:public :icon} - - ;; Create a page for all properties. - ;; This is _only_ for file graphs. - ;; Default value: true - :property-pages/enabled? true - - ;; Properties to exclude from having property pages - ;; This is _only_ for file graphs. - ;; Example usage: - ;; :property-pages/excludelist #{:duration :author} - - ;; By default, property value separated by commas will not be treated as - ;; page references. You can add properties to enable it. - ;; This is _only_ for file graphs. - ;; Example usage: - ;; :property/separated-by-commas #{:alias :tags} - - ;; Properties that are ignored when parsing property values for references - ;; This is _only_ for file graphs. - ;; Example usage: - ;; :ignored-page-references-keywords #{:author :website} - - ;; logbook configuration. - ;; This is _only_ for file graphs. - ;; :logbook/settings - ;; {:with-second-support? false ;limit logbook to minutes, seconds will be eliminated - ;; :enabled-in-all-blocks true ;display logbook in all blocks after timetracking - ;; :enabled-in-timestamped-blocks false ;don't display logbook at all - ;; } - ;; Mobile photo upload configuration. ;; :mobile/photo ;; {:allow-editing? true @@ -375,11 +395,6 @@ ;; :redirect-page? false ;; Default value: false ;; :default-page "quick capture"} ;; Default page: "quick capture" - ;; File sync options - ;; Ignore these files when syncing, regexp is supported. - ;; This is _only_ for file graphs. - ;; :file-sync/ignore-files [] - ;; Configure the Enter key behavior for ;; context-aware editing with DWIM (Do What I Mean). ;; context-aware Enter key behavior implies that pressing Enter will @@ -393,16 +408,4 @@ ;; :page-ref? true ;; Default value: true ;; :properties? true ;; Default value: true ;; :list? false} ;; Default value: false - - ;; Configure the escaping method for special characters in page titles. - ;; This is _only_ for file graphs. - ;; Warning: - ;; This is a dangerous operation. To modify the setting, - ;; you'll need to manually rename all affected files and - ;; re-index them on all clients after synchronization. - ;; Incorrect handling may result in messy page titles. - ;; Available options: - ;; - :triple-lowbar (default) - ;; ;use triple underscore `___` for slash `/` in page title - ;; ;use Percent-encoding for other invalid characters - :file/name-format :triple-lowbar} + } \ No newline at end of file diff --git a/deps/common/src/logseq/common/config.cljs b/deps/common/src/logseq/common/config.cljs index 45df37c299..eb8f385b5b 100644 --- a/deps/common/src/logseq/common/config.cljs +++ b/deps/common/src/logseq/common/config.cljs @@ -1,5 +1,5 @@ (ns logseq.common.config - "Common config and constants that are shared between deps and app" + "Common config constants and fns that are shared between deps and app" (:require [clojure.string :as string] [goog.object :as gobj])) @@ -115,3 +115,43 @@ "*" "-"))) + +(defn create-config-for-db-graph + "Given a new config.edn file string, creates a config.edn for use with only DB graphs" + [config] + (string/replace config #"(?m)[\s]*;; == FILE GRAPH CONFIG ==(?:.|\n)*?;; == END OF FILE GRAPH CONFIG ==\n?" "")) + +(def file-only-config + "File only config keys that are deprecated in DB graphs along with + descriptions for their deprecation." + (merge + (zipmap + [:file/name-format + :file-sync/ignore-files + :hidden + :ignored-page-references-keywords + :journal/file-name-format + :journal/page-title-format + :journals-directory + :logbook/settings + :org-mode/insert-file-link? + :pages-directory + :preferred-workflow + :property/separated-by-commas + :property-pages/excludelist + :srs/learning-fraction + :srs/initial-interval + :whiteboards-directory] + (repeat "is not used in DB graphs")) + {:preferred-format + "is not used in DB graphs as there is only markdown mode." + :property-pages/enabled? + "is not used in DB graphs as all properties have pages" + :block-hidden-properties + "is not used in DB graphs as hiding a property is done in its configuration" + :feature/enable-block-timestamps? + "is not used in DB graphs as it is always enabled" + :favorites + "is not stored in config for DB graphs" + :default-templates + "is replaced by #Template and the `Apply template to tags` property"})) \ No newline at end of file diff --git a/deps/common/test/logseq/common/config_test.cljc b/deps/common/test/logseq/common/config_test.cljc new file mode 100644 index 0000000000..c66be429cc --- /dev/null +++ b/deps/common/test/logseq/common/config_test.cljc @@ -0,0 +1,45 @@ +(ns logseq.common.config-test + (:require [cljs.test :refer [deftest is]] + [clojure.string :as string] + [logseq.common.config :as common-config] + #?(:org.babashka/nbb [nbb.classpath :as cp]) + ["fs" :as fs] + ["path" :as node-path])) + +(deftest remove-hidden-files + (let [files ["pages/foo.md" "pages/bar.md" + "script/README.md" "script/config.edn" + "dev/README.md" "dev/config.edn"]] + (is (= ["pages/foo.md" "pages/bar.md"] + #_:clj-kondo/ignore ;; buggy unresolved var + (common-config/remove-hidden-files + files + {:hidden ["script" "/dev"]} + identity)) + "Removes hidden relative files") + + (is (= ["/pages/foo.md" "/pages/bar.md"] + (common-config/remove-hidden-files + (map #(str "/" %) files) + {:hidden ["script" "/dev"]} + identity)) + "Removes hidden files if they start with '/'"))) + +(defn find-on-classpath [classpath rel-path] + (some (fn [dir] + (let [f (node-path/join dir rel-path)] + (when (fs/existsSync f) f))) + (string/split classpath #":"))) + +#?(:org.babashka/nbb + (deftest create-config-for-db-graph + (let [original-config (some-> (find-on-classpath (cp/get-classpath) "templates/config.edn") fs/readFileSync str) + _ (assert original-config "config.edn must not be blank") + migrated-config (common-config/create-config-for-db-graph original-config) + forbidden-kws-regex (re-pattern (str (string/join "|" (keys common-config/file-only-config))))] + ;; (println migrated-config) + (is (not (string/includes? migrated-config "== FILE ONLY CONFIG")) + "No longer includes file config header") + (assert (re-find forbidden-kws-regex original-config) "File config keys present in original config") + (is (not (re-find forbidden-kws-regex migrated-config)) + "File config keys no longer present in migrated config")))) \ No newline at end of file diff --git a/deps/common/test/logseq/common/config_test.cljs b/deps/common/test/logseq/common/config_test.cljs deleted file mode 100644 index c37d576ffa..0000000000 --- a/deps/common/test/logseq/common/config_test.cljs +++ /dev/null @@ -1,21 +0,0 @@ -(ns logseq.common.config-test - (:require [logseq.common.config :as common-config] - [cljs.test :refer [deftest is]])) - -(deftest remove-hidden-files - (let [files ["pages/foo.md" "pages/bar.md" - "script/README.md" "script/config.edn" - "dev/README.md" "dev/config.edn"]] - (is (= ["pages/foo.md" "pages/bar.md"] - (common-config/remove-hidden-files - files - {:hidden ["script" "/dev"]} - identity)) - "Removes hidden relative files") - - (is (= ["/pages/foo.md" "/pages/bar.md"] - (common-config/remove-hidden-files - (map #(str "/" %) files) - {:hidden ["script" "/dev"]} - identity)) - "Removes hidden files if they start with '/'"))) \ No newline at end of file diff --git a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs index 6b526e9549..eb62a4d078 100644 --- a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs @@ -764,6 +764,7 @@ {:block block'' :properties-tx properties-tx})) (defn- pretty-print-dissoc + "Remove list of keys from a given map string while preserving whitespace" [s dissoc-keys] (-> (reduce rewrite/dissoc (rewrite/parse-string s) @@ -1496,7 +1497,11 @@ ( (reduce rewrite/dissoc + (rewrite/parse-string (str content)) + (keys common-config/file-only-config)) + str)) + (defn- import-file-graph [*files {:keys [graph-name tag-classes property-classes property-parent-classes] :as user-options} @@ -370,7 +381,7 @@ ;; config file options :default-config config/config-default-content : (:url %) string/lower-case) full-graph-name) (state/get-repos)))) -(defn migrate-db-config - [content] - (-> (reduce rewrite/dissoc - (rewrite/parse-string (str content)) - (keys config-edn-common-handler/file-only-config)) - str)) - (defn- create-db [full-graph-name {:keys [file-graph-import?]}] (-> - (p/let [config (migrate-db-config config/config-default-content) + (p/let [config (common-config/create-config-for-db-graph config/config-default-content) _ (persist-db/ {:config config} file-graph-import? (assoc :import-type :file-graph))) From 8f23966a7285623b87ad0b0b3b12cdbb6205230b Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Thu, 15 May 2025 12:18:56 -0400 Subject: [PATCH 13/17] fix: remove duplicate conversion of config.edn from UI Conversion of config.edn was moved to exporter in previous commit --- src/main/frontend/components/imports.cljs | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/src/main/frontend/components/imports.cljs b/src/main/frontend/components/imports.cljs index 1160019fe7..e700a1c69a 100644 --- a/src/main/frontend/components/imports.cljs +++ b/src/main/frontend/components/imports.cljs @@ -1,7 +1,6 @@ (ns frontend.components.imports "Import data into Logseq." - (:require [borkdude.rewrite-edn :as rewrite] - [cljs-time.core :as t] + (:require [cljs-time.core :as t] [cljs.pprint :as pprint] [clojure.string :as string] [frontend.components.onboarding.setups :as setups] @@ -35,8 +34,7 @@ [logseq.shui.hooks :as hooks] [logseq.shui.ui :as shui] [promesa.core :as p] - [rum.core :as rum] - [logseq.common.config :as common-config])) + [rum.core :as rum])) ;; Can't name this component as `frontend.components.import` since shadow-cljs ;; will complain about it. @@ -349,15 +347,6 @@ (fs/mkdir-if-not-exists parent-dir) (fs/write-plain-text-file! repo repo-dir (:path file) content {:skip-transact? true}))))))) -(defn- convert-file-config-for-db-graph - "Converts a file graph config.edn for use with DB graphs. Unlike common-config/create-config-for-db-graph, - have to manually dissoc deprecated keys for config to be valid" - [content] - (-> (reduce rewrite/dissoc - (rewrite/parse-string (str content)) - (keys common-config/file-only-config)) - str)) - (defn- import-file-graph [*files {:keys [graph-name tag-classes property-classes property-parent-classes] :as user-options} @@ -381,8 +370,7 @@ ;; config file options :default-config config/config-default-content : Date: Thu, 15 May 2025 14:05:37 -0400 Subject: [PATCH 14/17] enhance(dev): CLI built db graphs have same config.edn as UI ones. Also improve over hack to add additional config now that rewrite-clj is available --- deps/db/deps.edn | 1 + deps/db/nbb.edn | 2 ++ deps/outliner/src/logseq/outliner/cli.cljs | 22 ++++++++++++++++------ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/deps/db/deps.edn b/deps/db/deps.edn index e9e5bfaace..b749423da2 100644 --- a/deps/db/deps.edn +++ b/deps/db/deps.edn @@ -12,6 +12,7 @@ logseq/common {:local/root "../common"} logseq/clj-fractional-indexing {:git/url "https://github.com/logseq/clj-fractional-indexing" :sha "7182b7878410f78536dc2b6df35ed32ef9cd6b61"} + borkdude/rewrite-edn {:mvn/version "0.4.9"} metosin/malli {:mvn/version "0.16.1"} medley/medley {:mvn/version "1.4.0"}} diff --git a/deps/db/nbb.edn b/deps/db/nbb.edn index 128592ccad..8da6ac9129 100644 --- a/deps/db/nbb.edn +++ b/deps/db/nbb.edn @@ -4,6 +4,8 @@ {:local/root "../common"} medley/medley {:mvn/version "1.4.0"} metosin/malli {:mvn/version "0.16.1"} + ;; Used by db scripts with outliner.cli + borkdude/rewrite-edn {:mvn/version "0.4.9"} logseq/clj-fractional-indexing {:git/url "https://github.com/logseq/clj-fractional-indexing" :sha "7182b7878410f78536dc2b6df35ed32ef9cd6b61"} io.github.nextjournal/nbb-test-runner diff --git a/deps/outliner/src/logseq/outliner/cli.cljs b/deps/outliner/src/logseq/outliner/cli.cljs index dcd05e274a..fdb3142fd0 100644 --- a/deps/outliner/src/logseq/outliner/cli.cljs +++ b/deps/outliner/src/logseq/outliner/cli.cljs @@ -1,13 +1,15 @@ (ns ^:node-only logseq.outliner.cli "Primary ns for outliner CLI fns" - (:require [clojure.string :as string] + (:require [borkdude.rewrite-edn :as rewrite] + [clojure.string :as string] [datascript.core :as d] [logseq.db.sqlite.create-graph :as sqlite-create-graph] [logseq.db.sqlite.build :as sqlite-build] [logseq.db.common.sqlite-cli :as sqlite-cli] [logseq.outliner.db-pipeline :as db-pipeline] ["fs" :as fs] - ["path" :as node-path])) + ["path" :as node-path] + [logseq.common.config :as common-config])) (defn- find-on-classpath [classpath rel-path] (some (fn [dir] @@ -15,6 +17,15 @@ (when (fs/existsSync f) f))) (string/split classpath #":"))) +(defn- pretty-print-merge + "Merge map into string while preversing whitespace" + [s m] + (-> (reduce (fn [acc [k v]] + (rewrite/assoc acc k v)) + (rewrite/parse-string s) + m) + str)) + (defn- setup-init-data "Setup initial data same as frontend.handler.repo/create-db" [conn {:keys [additional-config classpath import-type] @@ -23,11 +34,10 @@ (cond-> (or (some-> (find-on-classpath classpath "templates/config.edn") fs/readFileSync str) (do (println "Setting graph's config to empty since no templates/config.edn was found.") "{}")) + true + (common-config/create-config-for-db-graph) additional-config - ;; TODO: Replace with rewrite-clj when it's available - (string/replace-first #"(:file/name-format :triple-lowbar)" - (str "$1 " - (string/replace-first (str additional-config) #"^\{(.*)\}$" "$1"))))] + (pretty-print-merge additional-config))] (d/transact! conn (sqlite-create-graph/build-db-initial-data config-content {:import-type import-type})))) (defn init-conn From bd115eef0fa0615e4efbb2dac6339587b6b5a83d Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Thu, 15 May 2025 17:23:42 -0400 Subject: [PATCH 15/17] enhance: add import option to allow existing pages to keep properties --- deps/db/src/logseq/db/sqlite/export.cljs | 37 +++++----- .../db/test/logseq/db/sqlite/export_test.cljs | 67 +++++++++++-------- 2 files changed, 60 insertions(+), 44 deletions(-) diff --git a/deps/db/src/logseq/db/sqlite/export.cljs b/deps/db/src/logseq/db/sqlite/export.cljs index b6b80447bc..375d2d5fb7 100644 --- a/deps/db/src/logseq/db/sqlite/export.cljs +++ b/deps/db/src/logseq/db/sqlite/export.cljs @@ -847,21 +847,24 @@ ;; Import fns ;; ========== (defn- add-uuid-to-page-if-exists - [db import-to-existing-page-uuids m] - (if-let [ent (some->> (:build/journal m) - (d/datoms db :avet :block/journal-day) - first - :e - (d/entity db))] + [db import-to-existing-page-uuids {:keys [existing-pages-keep-properties?]} m] + (if-let [ent (if (:build/journal m) + (some->> (:build/journal m) + (d/datoms db :avet :block/journal-day) + first + :e + (d/entity db)) + ;; TODO: For now only check page uniqueness by title. Could handle more uniqueness checks later + (some->> (:block/title m) (ldb/get-case-page db)))] (do (swap! import-to-existing-page-uuids assoc (:block/uuid m) (:block/uuid ent)) - (assoc m :block/uuid (:block/uuid ent))) - ;; TODO: For now only check page uniqueness by title. Could handle more uniqueness checks later - (if-let [ent (some->> (:block/title m) (ldb/get-case-page db))] - (do - (swap! import-to-existing-page-uuids assoc (:block/uuid m) (:block/uuid ent)) - (assoc m :block/uuid (:block/uuid ent))) - m))) + (cond-> (assoc m :block/uuid (:block/uuid ent)) + (and (:build/properties m) existing-pages-keep-properties?) + (update :build/properties (fn [props] + (->> props + (remove (fn [[k _v]] (get ent k))) + (into {})))))) + m)) (defn- update-existing-properties "Updates existing properties by ident. Also check imported and existing properties have @@ -884,7 +887,7 @@ (defn- check-for-existing-entities "Checks export map for existing entities and adds :block/uuid to them if they exist in graph to import. Also checks for property conflicts between existing properties and properties to be imported" - [db {:keys [pages-and-blocks classes properties] ::keys [export-type] :as export-map} property-conflicts] + [db {:keys [pages-and-blocks classes properties] ::keys [export-type import-options] :as export-map} property-conflicts] (let [import-to-existing-page-uuids (atom {}) export-map (cond-> {:build-existing-tx? true @@ -892,7 +895,7 @@ (seq pages-and-blocks) (assoc :pages-and-blocks (mapv (fn [m] - (update m :page (partial add-uuid-to-page-if-exists db import-to-existing-page-uuids))) + (update m :page (partial add-uuid-to-page-if-exists db import-to-existing-page-uuids import-options))) pages-and-blocks)) (seq classes) (assoc :classes @@ -915,7 +918,7 @@ (walk/postwalk (fn [f] (if (and (vector? f) (= :build/page (first f))) [:build/page - (add-uuid-to-page-if-exists db import-to-existing-page-uuids (second f))] + (add-uuid-to-page-if-exists db import-to-existing-page-uuids import-options (second f))] f)) export-map)) ;; Update uuid references of all pages that had their uuids updated to reference an existing page @@ -948,6 +951,8 @@ * ::kv-values - Vec of :kv/value maps for a :graph export * ::auto-include-namespaces - A set of parent namespaces to include from properties and classes for a :graph export. See :exclude-namespaces in build-graph-export for a similar option + * ::import-options - A map of options that alters importing behavior. Has the following keys: + * :existing-pages-keep-properties? - Boolean which disables upsert of :build/properties on This fn then returns a map of txs to transact with the following keys: * :init-tx - Txs that must be transacted first, usually because they define new properties diff --git a/deps/db/test/logseq/db/sqlite/export_test.cljs b/deps/db/test/logseq/db/sqlite/export_test.cljs index 3c1c9da457..d9603d0ca5 100644 --- a/deps/db/test/logseq/db/sqlite/export_test.cljs +++ b/deps/db/test/logseq/db/sqlite/export_test.cljs @@ -826,54 +826,65 @@ (-> (:classes imported-graph) (medley/dissoc-in [:user.property/p1 :build/properties])))))) -(deftest build-import-can-import-existing-page-with-different-uuid +(defn- test-import-existing-page [import-options expected-page-properties] (let [original-data {:properties {:user.property/node {:logseq.property/type :node :db/cardinality :db.cardinality/many}} :pages-and-blocks [{:page {:block/title "page1" - :build/properties {:user.property/node #{[:build/page {:block/title "node1"}]}}}}]} + :build/properties {:user.property/node + #{[:build/page {:block/title "existing page" + :build/properties {:logseq.property/description "first description"}}]}}}}]} conn (db-test/create-conn-with-blocks original-data) - page-uuid (:block/uuid (db-test/find-page-by-title @conn "node1")) + page-uuid (:block/uuid (db-test/find-page-by-title @conn "existing page")) _ (validate-db @conn) ;; This is just a temp uuid used to link to the page during import temp-uuid (random-uuid) - existing-data + import-data {:properties {:user.property/node {:logseq.property/type :node :db/cardinality :db.cardinality/many}} :pages-and-blocks - [{:page {:block/title "node1" + [{:page {:block/title "existing page" :block/uuid temp-uuid - :build/keep-uuid? true}} + :build/keep-uuid? true + :build/properties {:logseq.property/description "second description" + :logseq.property/exclude-from-graph-view true}}} {:page {:block/title "page2" - :build/properties {:user.property/node #{[:block/uuid temp-uuid]}}}}]} + :build/properties {:user.property/node #{[:block/uuid temp-uuid]}}}}] + ::sqlite-export/import-options import-options} {:keys [init-tx block-props-tx] :as _txs} - (sqlite-export/build-import existing-data @conn {}) + (sqlite-export/build-import import-data @conn {}) ;; _ (cljs.pprint/pprint _txs) _ (d/transact! conn init-tx) _ (d/transact! conn block-props-tx) _ (validate-db @conn) expected-pages-and-blocks - [{:page - {:block/uuid page-uuid - :build/keep-uuid? true, - :block/title "node1"}, - :blocks []} - {:page - {:build/properties - {:user.property/node - #{[:block/uuid page-uuid]}}, - :block/title "page1"}, - :blocks []} - {:page - {:build/properties - {:user.property/node - #{[:block/uuid page-uuid]}}, - :block/title "page2"}, - :blocks []}], + [{:block/uuid page-uuid + :build/keep-uuid? true, + :block/title "existing page" + :build/properties + expected-page-properties} + {:build/properties + {:user.property/node + #{[:block/uuid page-uuid]}}, + :block/title "page1"} + {:build/properties + {:user.property/node + #{[:block/uuid page-uuid]}}, + :block/title "page2"}] exported-graph (sqlite-export/build-export @conn {:export-type :graph :graph-options {:exclude-built-in-pages? true}})] (is (= expected-pages-and-blocks - (:pages-and-blocks exported-graph)) - "page uuid ('node1') is preserved across imports even when its assigned a temporary - uuid to relate it to other nodes"))) \ No newline at end of file + (map :page (:pages-and-blocks exported-graph))) + "page uuid of 'existing page' is preserved across imports even when its assigned a temporary + uuid to relate it to other nodes"))) + +(deftest build-import-can-import-existing-page-with-different-uuid + (testing "By default any properties passed to an existing page are upserted" + (test-import-existing-page {} + {:logseq.property/description "second description" + :logseq.property/exclude-from-graph-view true})) + (testing "With ::existing-pages-keep-properties?, existing properties on existing pages are not overwritten by imported data" + (test-import-existing-page {:existing-pages-keep-properties? true} + {:logseq.property/description "first description" + :logseq.property/exclude-from-graph-view true}))) \ No newline at end of file From c308e04b934842051a07c54bba11f06c14fc4133 Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Thu, 15 May 2025 17:35:33 -0400 Subject: [PATCH 16/17] refactor: Ensure all ::sqlite-export keys aren't passed to sqlite.build Enumerating each namespaced key was error prone and too tightly coupled --- deps/db/src/logseq/db/sqlite/export.cljs | 38 ++++++++++++++---------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/deps/db/src/logseq/db/sqlite/export.cljs b/deps/db/src/logseq/db/sqlite/export.cljs index 375d2d5fb7..330eb1f797 100644 --- a/deps/db/src/logseq/db/sqlite/export.cljs +++ b/deps/db/src/logseq/db/sqlite/export.cljs @@ -803,21 +803,29 @@ set)] (set/difference ref-uuids known-uuids))) +(defn- remove-namespaced-keys + "Removes keys from this ns for maps passed sqlite.build fns as they don't need to validate or use them" + [m] + (->> m + (remove (fn [[k _v]] (= "logseq.db.sqlite.export" (namespace k)))) + (into {}))) + (defn- ensure-export-is-valid "Checks that export map is usable by sqlite.build including checking that all referenced properties and classes are defined. Checks related to properties and classes are disabled when :exclude-namespaces is set because those checks can't be done" - [export-map {:keys [graph-options]}] - (when-not (seq (:exclude-namespaces graph-options)) (sqlite-build/validate-options export-map)) - (let [undefined-uuids (find-undefined-uuids export-map) - undefined (cond-> {} - (empty? (:exclude-namespaces graph-options)) - (merge (find-undefined-classes-and-properties export-map)) - (seq undefined-uuids) - (assoc :uuids undefined-uuids))] - (when (seq undefined) - (throw (ex-info (str "The following classes, uuids and properties are not defined: " (pr-str undefined)) - undefined))))) + [export-map* {:keys [graph-options]}] + (let [export-map (remove-namespaced-keys export-map*)] + (when-not (seq (:exclude-namespaces graph-options)) (sqlite-build/validate-options export-map)) + (let [undefined-uuids (find-undefined-uuids export-map) + undefined (cond-> {} + (empty? (:exclude-namespaces graph-options)) + (merge (find-undefined-classes-and-properties export-map)) + (seq undefined-uuids) + (assoc :uuids undefined-uuids))] + (when (seq undefined) + (throw (ex-info (str "The following classes, uuids and properties are not defined: " (pr-str undefined)) + undefined)))))) (defn build-export "Handles exporting db by given export-type" @@ -838,10 +846,10 @@ (build-graph-export db (:graph-options options)))] (if (get-in options [:graph-options :catch-validation-errors?]) (try - (ensure-export-is-valid (dissoc export-map ::block ::graph-files ::kv-values ::schema-version) options) + (ensure-export-is-valid export-map options) (catch ExceptionInfo e (println "Caught error:" e))) - (ensure-export-is-valid (dissoc export-map ::block ::graph-files ::kv-values ::schema-version) options)) + (ensure-export-is-valid export-map options)) (assoc export-map ::export-type export-type))) ;; Import fns @@ -974,7 +982,7 @@ {:error (str "The following imported properties conflict with the current graph: " (pr-str (mapv :property-id @property-conflicts)))}) (if (= :graph (::export-type export-map'')) - (-> (sqlite-build/build-blocks-tx (dissoc export-map'' ::graph-files ::kv-values ::export-type ::schema-version)) + (-> (sqlite-build/build-blocks-tx (remove-namespaced-keys export-map'')) (assoc :misc-tx (vec (concat (::graph-files export-map'') (::kv-values export-map''))))) - (sqlite-build/build-blocks-tx export-map''))))) + (sqlite-build/build-blocks-tx (remove-namespaced-keys export-map'')))))) From 120730a8dc77def8a68511ef817b889b20b2fc7a Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Thu, 15 May 2025 17:53:55 -0400 Subject: [PATCH 17/17] enhance: enable counts for linked references This was previous behavior and is helpful for getting overview of linked refs and seeing that filters are roughly working --- src/main/frontend/components/reference.cljs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/frontend/components/reference.cljs b/src/main/frontend/components/reference.cljs index 3cbd3a494f..6a85a2d1e0 100644 --- a/src/main/frontend/components/reference.cljs +++ b/src/main/frontend/components/reference.cljs @@ -43,6 +43,7 @@ (views/view {:view-parent page-entity :view-feature-type :linked-references + :show-items-count? true :additional-actions [reference-filter] :columns (views/build-columns config [] {}) :config config})))