From 4025b8d83b976a32b8bbc5b7b79119dddcd7b576 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Wed, 16 Apr 2025 01:39:23 +0800 Subject: [PATCH 01/15] fix: default property value are not displayed in table view --- .../frontend/components/property/value.cljs | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/main/frontend/components/property/value.cljs b/src/main/frontend/components/property/value.cljs index 4447cdd5a0..2b687b7bc6 100644 --- a/src/main/frontend/components/property/value.cljs +++ b/src/main/frontend/components/property/value.cljs @@ -1270,15 +1270,17 @@ type (:logseq.property/type property) multiple-values? (db-property/many? property) v (let [v (get block (:db/ident property))] - (cond - (and multiple-values? (or (set? v) (and (coll? v) (empty? v)) (nil? v))) - v - multiple-values? - #{v} - (set? v) - (first v) - :else - v)) + (or + (cond + (and multiple-values? (or (set? v) (and (coll? v) (empty? v)) (nil? v))) + v + multiple-values? + #{v} + (set? v) + (first v) + :else + v) + (:logseq.property/default-value property))) self-value-or-embedded? (fn [v] (or (= (:db/id v) (:db/id block)) ;; property value self embedded From 04a93f66135ebc85cc990da8479433a813f8c2ed Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Tue, 15 Apr 2025 22:43:28 -0400 Subject: [PATCH 02/15] chore: mv db or file graph specific ldb fns to db or file graph namespaces. Follow up to https://github.com/logseq/logseq/pull/11774#discussion_r2035354875 to help avoid bugs that are hard to spot b/c of no organization --- .clj-kondo/config.edn | 1 + deps/db/.clj-kondo/config.edn | 1 + deps/db/src/logseq/db.cljs | 130 +++--------------- deps/db/src/logseq/db/frontend/db.cljs | 113 +++++++++++++++ .../src/logseq/graph_parser/db.cljs | 7 + src/main/frontend/common/graph_view.cljs | 6 +- src/main/frontend/db/model.cljs | 2 +- 7 files changed, 143 insertions(+), 117 deletions(-) create mode 100644 deps/db/src/logseq/db/frontend/db.cljs diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index 79bc386151..7b7f19d53c 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -182,6 +182,7 @@ logseq.db.file-based.entity-util file-entity-util logseq.db.frontend.class db-class logseq.db.frontend.content db-content + logseq.db.frontend.db db-db logseq.db.frontend.db-ident db-ident logseq.db.frontend.entity-plus entity-plus logseq.db.frontend.entity-util entity-util diff --git a/deps/db/.clj-kondo/config.edn b/deps/db/.clj-kondo/config.edn index c05ef9884c..53cd1bd430 100644 --- a/deps/db/.clj-kondo/config.edn +++ b/deps/db/.clj-kondo/config.edn @@ -16,6 +16,7 @@ logseq.db.common.view db-view logseq.db.frontend.content db-content logseq.db.frontend.class db-class + logseq.db.frontend.db db-db logseq.db.frontend.db-ident db-ident logseq.db.frontend.inputs db-inputs logseq.db.frontend.property db-property diff --git a/deps/db/src/logseq/db.cljs b/deps/db/src/logseq/db.cljs index 75448218e6..80508c3514 100644 --- a/deps/db/src/logseq/db.cljs +++ b/deps/db/src/logseq/db.cljs @@ -1,19 +1,19 @@ (ns logseq.db - "Main namespace for public db fns. For DB and file graphs. - For shared file graph only fns, use logseq.graph-parser.db" + "Main namespace for db fns that handles DB and file graphs. For db graph only + fns, use logseq.db.frontend.db and for file graph only fns, use + logseq.graph-parser.db" (:require [clojure.set :as set] [clojure.string :as string] [clojure.walk :as walk] [datascript.core :as d] [datascript.impl.entity :as de] [logseq.common.util :as common-util] - [logseq.common.util.namespace :as ns-util] - [logseq.common.util.page-ref :as page-ref] [logseq.common.uuid :as common-uuid] [logseq.db.common.delete-blocks :as delete-blocks] ;; Load entity extensions [logseq.db.common.entity-util :as common-entity-util] [logseq.db.common.sqlite :as sqlite-common-db] [logseq.db.frontend.class :as db-class] + [logseq.db.frontend.db :as db-db] [logseq.db.frontend.entity-plus :as entity-plus] [logseq.db.frontend.entity-util :as entity-util] [logseq.db.frontend.property :as db-property] @@ -412,11 +412,6 @@ [] (common-uuid/gen-uuid)) -(defn get-classes-with-property - "Get classes which have given property as a class property" - [db property-id] - (:logseq.property.class/_properties (d/entity db property-id))) - (defn get-alias-source-page "return the source page (page-name) of an alias" [db alias-id] @@ -492,37 +487,13 @@ e)))))) (def built-in? entity-util/built-in?) - -(defn built-in-class-property? - "Whether property a built-in property for the specific class" - [class-entity property-entity] - (and (built-in? class-entity) - (class? class-entity) - (built-in? property-entity) - (contains? (set (get-in (db-class/built-in-classes (:db/ident class-entity)) [:schema :properties])) - (:db/ident property-entity)))) - -(defn private-built-in-page? - "Private built-in pages should not be navigable or searchable by users. Later it - could be useful to use this for the All Pages view" - [page] - (cond (property? page) - (not (public-built-in-property? page)) - (or (class? page) (internal-page? page)) - false - ;; Default to true for closed value and future internal types. - ;; Other types like whiteboard are not considered because they aren't built-in - :else - true)) +(def get-classes-with-property db-db/get-classes-with-property) +(def built-in-class-property? db-db/built-in-class-property?) +(def private-built-in-page? db-db/private-built-in-page?) (def write-transit-str sqlite-util/write-transit-str) (def read-transit-str sqlite-util/read-transit-str) - -(defn build-favorite-tx - "Builds tx for a favorite block in favorite page" - [favorite-uuid] - {:block/link [:block/uuid favorite-uuid] - :block/title ""}) +(def build-favorite-tx db-db/build-favorite-tx) (defn get-key-value [db key-ident] @@ -544,87 +515,20 @@ [db] (when db (get-key-value db :logseq.kv/remote-schema-version))) -(defn get-all-properties - [db] - (->> (d/datoms db :avet :block/tags :logseq.class/Property) - (map (fn [d] - (d/entity db (:e d)))))) - -(defn get-page-parents - [node & {:keys [node-class?]}] - (when-let [parent (:logseq.property/parent node)] - (loop [current-parent parent - parents' []] - (if (and - current-parent - (if node-class? (class? current-parent) true) - (not (contains? parents' current-parent))) - (recur (:logseq.property/parent current-parent) - (conj parents' current-parent)) - (vec (reverse parents')))))) - -(defn get-title-with-parents - [entity] - (if (or (entity-util/class? entity) (entity-util/internal-page? entity)) - (let [parents' (->> (get-page-parents entity) - (remove (fn [e] (= :logseq.class/Root (:db/ident e)))) - vec)] - (string/join - ns-util/parent-char - (map :block/title (conj (vec parents') entity)))) - (:block/title entity))) - -(defn get-classes-parents - [tags] - (let [tags' (filter class? tags) - result (mapcat #(get-page-parents % {:node-class? true}) tags')] - (set result))) - -(defn class-instance? - "Whether `object` is an instance of `class`" - [class object] - (let [tags (:block/tags object) - tags-ids (set (map :db/id tags))] - (or - (contains? tags-ids (:db/id class)) - (let [class-parent-ids (set (map :db/id (get-classes-parents tags)))] - (contains? (set/union class-parent-ids tags-ids) (:db/id class)))))) - -(defn inline-tag? - [block-raw-title tag] - (assert (string? block-raw-title) "block-raw-title should be a string") - (string/includes? block-raw-title (str "#" (page-ref/->page-ref (:block/uuid tag))))) - -(defonce node-display-type-classes - #{:logseq.class/Code-block :logseq.class/Math-block :logseq.class/Quote-block}) - -(defn get-class-ident-by-display-type - [display-type] - (case display-type - :code :logseq.class/Code-block - :math :logseq.class/Math-block - :quote :logseq.class/Quote-block - nil)) - -(defn get-display-type-by-class-ident - [class-ident] - (case class-ident - :logseq.class/Code-block :code - :logseq.class/Math-block :math - :logseq.class/Quote-block :quote - nil)) +(def get-all-properties db-db/get-all-properties) +(def get-page-parents db-db/get-page-parents) +(def get-classes-parents db-db/get-classes-parents) +(def get-title-with-parents db-db/get-title-with-parents) +(def class-instance? db-db/class-instance?) +(def inline-tag? db-db/inline-tag?) +(def node-display-type-classes db-db/node-display-type-classes) +(def get-class-ident-by-display-type db-db/get-class-ident-by-display-type) +(def get-display-type-by-class-ident db-db/get-display-type-by-class-ident) (def get-recent-updated-pages sqlite-common-db/get-recent-updated-pages) (def get-latest-journals sqlite-common-db/get-latest-journals) -(defn get-all-namespace-relation - [db] - (d/q '[:find ?page ?parent - :where - [?page :block/namespace ?parent]] - db)) - (defn get-pages-relation [db with-journal?] (if (entity-plus/db-based-graph? db) diff --git a/deps/db/src/logseq/db/frontend/db.cljs b/deps/db/src/logseq/db/frontend/db.cljs new file mode 100644 index 0000000000..5d83ec31f0 --- /dev/null +++ b/deps/db/src/logseq/db/frontend/db.cljs @@ -0,0 +1,113 @@ +(ns logseq.db.frontend.db + "DB graph fns commonly used outside db dep" + (:require [clojure.set :as set] + [clojure.string :as string] + [datascript.core :as d] + [logseq.common.util.namespace :as ns-util] + [logseq.common.util.page-ref :as page-ref] + [logseq.db.frontend.class :as db-class] + [logseq.db.frontend.entity-util :as entity-util] + [logseq.db.frontend.property :as db-property])) + +(defn get-classes-with-property + "Get classes which have given property as a class property" + [db property-id] + (:logseq.property.class/_properties (d/entity db property-id))) + +(defn built-in-class-property? + "Whether property a built-in property for the specific class" + [class-entity property-entity] + (and (entity-util/built-in? class-entity) + (entity-util/class? class-entity) + (entity-util/built-in? property-entity) + (contains? (set (get-in (db-class/built-in-classes (:db/ident class-entity)) [:schema :properties])) + (:db/ident property-entity)))) + +(defn private-built-in-page? + "Private built-in pages should not be navigable or searchable by users. Later it + could be useful to use this for the All Pages view" + [page] + (cond (entity-util/property? page) + (not (db-property/public-built-in-property? page)) + (or (entity-util/class? page) (entity-util/internal-page? page)) + false + ;; Default to true for closed value and future internal types. + ;; Other types like whiteboard are not considered because they aren't built-in + :else + true)) + +(defn build-favorite-tx + "Builds tx for a favorite block in favorite page" + [favorite-uuid] + {:block/link [:block/uuid favorite-uuid] + :block/title ""}) + +(defn get-all-properties + [db] + (->> (d/datoms db :avet :block/tags :logseq.class/Property) + (map (fn [d] + (d/entity db (:e d)))))) + +(defn get-page-parents + [node & {:keys [node-class?]}] + (when-let [parent (:logseq.property/parent node)] + (loop [current-parent parent + parents' []] + (if (and + current-parent + (if node-class? (entity-util/class? current-parent) true) + (not (contains? parents' current-parent))) + (recur (:logseq.property/parent current-parent) + (conj parents' current-parent)) + (vec (reverse parents')))))) + +(defn get-title-with-parents + [entity] + (if (or (entity-util/class? entity) (entity-util/internal-page? entity)) + (let [parents' (->> (get-page-parents entity) + (remove (fn [e] (= :logseq.class/Root (:db/ident e)))) + vec)] + (string/join + ns-util/parent-char + (map :block/title (conj (vec parents') entity)))) + (:block/title entity))) + +(defn get-classes-parents + [tags] + (let [tags' (filter entity-util/class? tags) + result (mapcat #(get-page-parents % {:node-class? true}) tags')] + (set result))) + +(defn class-instance? + "Whether `object` is an instance of `class`" + [class object] + (let [tags (:block/tags object) + tags-ids (set (map :db/id tags))] + (or + (contains? tags-ids (:db/id class)) + (let [class-parent-ids (set (map :db/id (get-classes-parents tags)))] + (contains? (set/union class-parent-ids tags-ids) (:db/id class)))))) + +(defn inline-tag? + [block-raw-title tag] + (assert (string? block-raw-title) "block-raw-title should be a string") + (string/includes? block-raw-title (str "#" (page-ref/->page-ref (:block/uuid tag))))) + +(defonce node-display-type-classes + #{:logseq.class/Code-block :logseq.class/Math-block :logseq.class/Quote-block}) + +(defn get-class-ident-by-display-type + [display-type] + (case display-type + :code :logseq.class/Code-block + :math :logseq.class/Math-block + :quote :logseq.class/Quote-block + nil)) + +(defn get-display-type-by-class-ident + [class-ident] + (case class-ident + :logseq.class/Code-block :code + :logseq.class/Math-block :math + :logseq.class/Quote-block :quote + nil)) diff --git a/deps/graph-parser/src/logseq/graph_parser/db.cljs b/deps/graph-parser/src/logseq/graph_parser/db.cljs index 5dd1b7740b..52782b90b5 100644 --- a/deps/graph-parser/src/logseq/graph_parser/db.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/db.cljs @@ -59,3 +59,10 @@ [db page-name] (some-> (ldb/get-page db page-name) :block/file)) + +(defn get-all-namespace-relation + [db] + (d/q '[:find ?page ?parent + :where + [?page :block/namespace ?parent]] + db)) diff --git a/src/main/frontend/common/graph_view.cljs b/src/main/frontend/common/graph_view.cljs index 4a6cecd43c..d677f4aa52 100644 --- a/src/main/frontend/common/graph_view.cljs +++ b/src/main/frontend/common/graph_view.cljs @@ -79,7 +79,7 @@ (let [dark? (= "dark" theme) relation (ldb/get-pages-relation db journal?) tagged-pages (ldb/get-all-tagged-pages db) - namespaces (ldb/get-all-namespace-relation db) + namespaces (gp-db/get-all-namespace-relation db) tags (set (map second tagged-pages)) full-pages (ldb/get-all-pages db) db-based? (entity-plus/db-based-graph? db) @@ -178,7 +178,7 @@ tags (set (remove #(= page-id %) tags)) ref-pages (get-page-referenced-pages db page-id) mentioned-pages (get-pages-that-mentioned-page db page-id show-journal) - namespaces (ldb/get-all-namespace-relation db) + namespaces (gp-db/get-all-namespace-relation db) links (concat namespaces (map (fn [ref-page] @@ -216,7 +216,7 @@ (if (ldb/page? b) b (:block/page b)))) (remove (fn [node] (= (:db/id block) (:db/id node)))) (common-util/distinct-by :db/id)) - namespaces (ldb/get-all-namespace-relation db) + namespaces (gp-db/get-all-namespace-relation db) links (->> (concat namespaces (map (fn [p] [(:db/id block) (:db/id p)]) ref-blocks)) diff --git a/src/main/frontend/db/model.cljs b/src/main/frontend/db/model.cljs index 683d58202f..1bccc4325c 100644 --- a/src/main/frontend/db/model.cljs +++ b/src/main/frontend/db/model.cljs @@ -793,7 +793,7 @@ independent of format as format specific heading characters are stripped" (defn get-all-namespace-relation [repo] - (ldb/get-all-namespace-relation (conn/get-db repo))) + (gp-db/get-all-namespace-relation (conn/get-db repo))) (defn get-all-namespace-parents [repo] From 34b33c16826eba81938824a509ff187a62513ebc Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Wed, 16 Apr 2025 14:01:45 +0800 Subject: [PATCH 03/15] fix: debug transit export --- deps/db/src/logseq/db/sqlite/util.cljs | 3 ++- src/main/frontend/handler/export.cljs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/deps/db/src/logseq/db/sqlite/util.cljs b/deps/db/src/logseq/db/sqlite/util.cljs index 3171fafd8b..52fd9cec91 100644 --- a/deps/db/src/logseq/db/sqlite/util.cljs +++ b/deps/db/src/logseq/db/sqlite/util.cljs @@ -54,7 +54,8 @@ (fn read-transit-str* [s] ;; TODO: delete the following pred later ;; https://github.com/logseq/logseq/pull/11790#discussion_r2014120469 - (if (and (string? s) (identical? "[" (first s))) + (if (and (string? s) (or (identical? "[" (first s)) + (identical? "(" (first s)))) (transit/read reader s) (do (prn :invalid-transit-string s) s))))) diff --git a/src/main/frontend/handler/export.cljs b/src/main/frontend/handler/export.cljs index ca62651eb2..12d487f867 100644 --- a/src/main/frontend/handler/export.cljs +++ b/src/main/frontend/handler/export.cljs @@ -195,7 +195,7 @@ (p/let [result (export-common-handler/ Date: Wed, 16 Apr 2025 14:17:02 +0800 Subject: [PATCH 04/15] chore: remove outdated code --- deps/db/src/logseq/db/sqlite/util.cljs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/deps/db/src/logseq/db/sqlite/util.cljs b/deps/db/src/logseq/db/sqlite/util.cljs index 52fd9cec91..437d66e15f 100644 --- a/deps/db/src/logseq/db/sqlite/util.cljs +++ b/deps/db/src/logseq/db/sqlite/util.cljs @@ -52,13 +52,7 @@ (merge read-handlers)) reader (transit/reader :json {:handlers read-handlers*})] (fn read-transit-str* [s] - ;; TODO: delete the following pred later - ;; https://github.com/logseq/logseq/pull/11790#discussion_r2014120469 - (if (and (string? s) (or (identical? "[" (first s)) - (identical? "(" (first s)))) - (transit/read reader s) - (do (prn :invalid-transit-string s) - s))))) + (transit/read reader s)))) (defn db-based-graph? [graph-name] From 62514b74d8617d6255e4e59759147561cecef9d8 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Wed, 16 Apr 2025 15:18:20 +0800 Subject: [PATCH 05/15] perf: page filter and search --- deps/db/src/logseq/db/common/view.cljs | 4 +- src/main/frontend/components/block.css | 6 + .../components/reference_filters.cljs | 118 +++++++++++------- src/main/frontend/components/views.cljs | 4 +- src/main/frontend/worker/db_worker.cljs | 8 +- 5 files changed, 92 insertions(+), 48 deletions(-) diff --git a/deps/db/src/logseq/db/common/view.cljs b/deps/db/src/logseq/db/common/view.cljs index 238f464708..0812b60879 100644 --- a/deps/db/src/logseq/db/common/view.cljs +++ b/deps/db/src/logseq/db/common/view.cljs @@ -311,6 +311,7 @@ (= (:db/id block) id) (= id (:db/id (:block/page block))) (ldb/hidden? (:block/page block)) + (ldb/hidden? block) (contains? (set (map :db/id (:block/tags block))) (:db/id entity)) (some? (get block (:db/ident entity)))) (or @@ -331,7 +332,8 @@ distinct)) full-ref-blocks) (remove nil?) - (frequencies))] + (frequencies) + (sort-by second #(> %1 %2)))] {:ref-pages-count ref-pages-count :ref-blocks ref-blocks})) diff --git a/src/main/frontend/components/block.css b/src/main/frontend/components/block.css index 65ceea96fe..65ae30c1bc 100644 --- a/src/main/frontend/components/block.css +++ b/src/main/frontend/components/block.css @@ -1082,3 +1082,9 @@ html.is-mac { aspect-ratio: 16 / 9; height: auto; } + +.ls-filters { + div[data-testid='virtuoso-item-list'] button { + @apply mb-2; + } +} diff --git a/src/main/frontend/components/reference_filters.cljs b/src/main/frontend/components/reference_filters.cljs index bd0a0ad7a7..8625be138e 100644 --- a/src/main/frontend/components/reference_filters.cljs +++ b/src/main/frontend/components/reference_filters.cljs @@ -5,12 +5,14 @@ [frontend.config :as config] [frontend.context.i18n :refer [t]] [frontend.db :as db] + [frontend.db-mixins :as db-mixins] [frontend.handler.page :as page-handler] [frontend.search :as search] [frontend.state :as state] [frontend.ui :as ui] [frontend.util :as util] [logseq.db.common.view :as db-view] + [logseq.shui.hooks :as hooks] [promesa.core :as p] [rum.core :as rum])) @@ -18,50 +20,71 @@ [references] (sort-by second #(> %1 %2) references)) -(defn filtered-refs - [page filters filtered-references*] - [:div.flex.gap-2.flex-wrap.items-center - (let [filtered-references (if (de/entity? (first filtered-references*)) - (map (fn [e] [(:block/title e)]) filtered-references*) - filtered-references*)] < - (for [[ref-name ref-count] filtered-references] - (when ref-name - (let [lc-reference (string/lower-case ref-name)] - (ui/button - [:span - ref-name - (when ref-count [:sup " " ref-count])] - :on-click (fn [e] - (let [db-based? (config/db-based-graph? (state/get-current-repo)) - includes (set (map :block/name (:included filters))) - excludes (set (map :block/name (:excluded filters))) - included? (includes lc-reference) - not-in-filters? (and (not included?) (not (excludes lc-reference))) - shift? (.-shiftKey e)] - (if db-based? - (page-handler/db-based-save-filter! page (:db/id (db/get-page lc-reference)) - {:add? not-in-filters? - :include? (if not-in-filters? (not shift?) included?)}) - (let [filters-m (->> (concat (map #(vector % true) includes) (map #(vector % false) excludes)) - (into {})) - filters' (if not-in-filters? - (assoc filters-m lc-reference (not shift?)) - (dissoc filters-m lc-reference))] - (page-handler/file-based-save-filter! page filters'))))) - :small? true - :variant :outline - :key ref-name)))))]) +(rum/defc ref-button + [page filters ref-name ref-count] + (let [lc-reference (string/lower-case ref-name)] + (ui/button + [:span + ref-name + (when ref-count [:sup " " ref-count])] + :on-click (fn [e] + (let [db-based? (config/db-based-graph? (state/get-current-repo)) + includes (set (map :block/name (:included filters))) + excludes (set (map :block/name (:excluded filters))) + included? (includes lc-reference) + not-in-filters? (and (not included?) (not (excludes lc-reference))) + shift? (.-shiftKey e)] + (if db-based? + (page-handler/db-based-save-filter! page (:db/id (db/get-page lc-reference)) + {:add? not-in-filters? + :include? (if not-in-filters? (not shift?) included?)}) + (let [filters-m (->> (concat (map #(vector % true) includes) (map #(vector % false) excludes)) + (into {})) + filters' (if not-in-filters? + (assoc filters-m lc-reference (not shift?)) + (dissoc filters-m lc-reference))] + (page-handler/file-based-save-filter! page filters'))))) + :small? true + :variant :outline))) -(rum/defcs filter-dialog < (rum/local "" ::filterSearch) rum/reactive - [state page references] - (let [page-entity (db/sub-block (:db/id page)) - filter-search (get state ::filterSearch) +(defn filtered-refs + [page filters filtered-references* virtual?] + (let [filtered-references (if (de/entity? (first filtered-references*)) + (map (fn [e] [(:block/title e)]) filtered-references*) + filtered-references*)] + (if (and (> (count filtered-references) 100) + (not (false? virtual?))) + (ui/virtualized-list + {:style {:height 500 + :width 500 + :max-width 500} + :total-count (count filtered-references) + :compute-item-key (fn [idx] + (str "ref-button-" idx)) + :item-content (fn [idx] + (let [[ref-name ref-count] (util/nth-safe filtered-references idx)] + (ref-button page filters ref-name ref-count)))}) + [:div.flex.gap-2.flex-wrap.items-center + {:style {:width 500 + :max-width 500}} + (for [[ref-name ref-count] filtered-references] + (rum/with-key (ref-button page filters ref-name ref-count) + (str "ref-" ref-name)))]))) + +(rum/defc filter-dialog-aux + [page-entity references] + (let [[filter-search set-filter-search!] (hooks/use-state "") + [filtered-references set-filtered-references!] (hooks/use-state references) filters (db-view/get-filters (db/get-db) page-entity) - filtered-references (frequencies-sort - (if (= @filter-search "") - references - (search/fuzzy-search references @filter-search :limit 500 :extract-fn first))) {:keys [included excluded]} filters] + (hooks/use-effect! + (fn [] + (let [references (if (= filter-search "") + references + (->> (search/fuzzy-search references filter-search :limit 100 :extract-fn first) + frequencies-sort))] + (set-filtered-references! references))) + [(hooks/use-debounced-value filter-search 200)]) [:div.ls-filters.filters [:div.sm:flex.sm:items-start [:div.mx-auto.flex-shrink-0.flex.items-center.justify-center.h-12.w-12.rounded-full.bg-gray-200.text-gray-500.sm:mx-0.sm:h-10.sm:w-10 @@ -75,12 +98,12 @@ (when (seq included) [:div.flex.flex-row.flex-wrap.center-items [:div.mr-1.font-medium.py-1 (t :linked-references/filter-includes)] - (filtered-refs page-entity filters included)]) + (filtered-refs page-entity filters included false)]) (when (seq excluded) [:div.flex.flex-row.flex-wrap [:div.mr-1.font-medium.py-1 (t :linked-references/filter-excludes)] - (filtered-refs page-entity filters excluded)])]) + (filtered-refs page-entity filters excluded false)])]) [:div.cp__filters-input-panel.flex.focus-within:bg-gray-03 (ui/icon "search") [:input.cp__filters-input.w-full.bg-transparent @@ -89,7 +112,7 @@ :ref (fn [^js el] (when el (-> (p/delay 32) (p/then #(.focus el))))) :on-change (fn [e] - (reset! filter-search (util/evalue e)))}]] + (set-filter-search! (util/evalue e)))}]] (let [all-filters (set (concat (map :block/name included) (map :block/name excluded))) @@ -97,4 +120,9 @@ filtered-references)] (when (seq refs) [:div.mt-4 - (filtered-refs page-entity filters refs)]))])) + (filtered-refs page-entity filters refs true)]))])) + +(rum/defc filter-dialog < rum/reactive db-mixins/query + [page references] + (let [page-entity (db/sub-block (:db/id page))] + (filter-dialog-aux page-entity references))) diff --git a/src/main/frontend/components/views.cljs b/src/main/frontend/components/views.cljs index 4dec279a8b..1b3e1e6e2a 100644 --- a/src/main/frontend/components/views.cljs +++ b/src/main/frontend/components/views.cljs @@ -1339,6 +1339,8 @@ {:ref #(reset! *scroller-ref %) :total-count (count blocks) :custom-scroll-parent (gdom/getElement "main-content-container") + :compute-item-key (fn [idx] + (str (:db/id view-entity) "-card-" idx)) :item-content (fn [idx] (lazy-item (:data table) idx {} (fn [block] @@ -1698,7 +1700,7 @@ :custom-scroll-parent (gdom/getElement "main-content-container") :increase-viewport-by {:top 300 :bottom 300} :compute-item-key (fn [idx] - (str "table-group" idx)) + (str "table-group-" idx)) :skipAnimationFrameInResizeObserver true :total-count (count (:rows table)) :item-content (fn [idx] diff --git a/src/main/frontend/worker/db_worker.cljs b/src/main/frontend/worker/db_worker.cljs index 79194dde5e..7d94543766 100644 --- a/src/main/frontend/worker/db_worker.cljs +++ b/src/main/frontend/worker/db_worker.cljs @@ -532,7 +532,13 @@ (boolean (some ;; check if there's any entity reference this `block` except the view-entity - (fn [ref] (not= id (:db/id (:logseq.property/view-for ref)))) + (fn [ref] + (not + (or (= id (:db/id (:logseq.property/view-for ref))) + (ldb/hidden? (:block/page ref)) + (ldb/hidden? ref) + (contains? (set (map :db/id (:block/tags ref))) id) + (some? (get ref (:db/ident block)))))) (:block/_refs block))))))) (def-thread-api :thread-api/get-block-parents From d378c7e754b1f11072f7be50a889bbddcd0c20ec Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Wed, 16 Apr 2025 17:57:29 +0800 Subject: [PATCH 06/15] enhance: don't update :block/updated-at when collapsing or expanding --- deps/outliner/src/logseq/outliner/core.cljs | 21 +++++++++++++-------- src/main/frontend/handler/editor.cljs | 6 +++--- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/deps/outliner/src/logseq/outliner/core.cljs b/deps/outliner/src/logseq/outliner/core.cljs index 895752af55..5917442da7 100644 --- a/deps/outliner/src/logseq/outliner/core.cljs +++ b/deps/outliner/src/logseq/outliner/core.cljs @@ -221,7 +221,7 @@ (extend-type Entity otree/INode - (-save [this *txs-state db repo _date-formatter {:keys [retract-attributes? retract-attributes] + (-save [this *txs-state db repo _date-formatter {:keys [retract-attributes? retract-attributes outliner-op] :or {retract-attributes? true}}] (assert (ds/outliner-txs-state? *txs-state) "db should be satisfied outliner-tx-state?") @@ -233,12 +233,16 @@ data) db-based? (dissoc :block/properties)) - m* (-> data' - (dissoc :block/children :block/meta :block/unordered - :block.temp/ast-title :block.temp/ast-body :block/level :block.temp/fully-loaded?) - common-util/remove-nils - block-with-updated-at - (fix-tag-ids db {:db-graph? db-based?})) + collapse-or-expand? (= outliner-op :collapse-expand-blocks) + m* (cond-> + (-> data' + (dissoc :block/children :block/meta :block/unordered + :block.temp/ast-title :block.temp/ast-body :block/level :block.temp/fully-loaded?) + common-util/remove-nils + + (fix-tag-ids db {:db-graph? db-based?})) + (not collapse-or-expand?) + block-with-updated-at) db-id (:db/id this) block-uuid (:block/uuid this) eid (or db-id (when block-uuid [:block/uuid block-uuid])) @@ -292,7 +296,8 @@ retract-attributes))))))) ;; Update block's page attributes - (update-page-when-save-block *txs-state block-entity m) + (when-not collapse-or-expand? + (update-page-when-save-block *txs-state block-entity m)) ;; Remove orphaned refs from block (when (and (:block/title m) (not= (:block/title m) (:block/title block-entity))) (remove-orphaned-refs-when-save db *txs-state block-entity m {:db-graph? db-based?}))) diff --git a/src/main/frontend/handler/editor.cljs b/src/main/frontend/handler/editor.cljs index b8d78a3936..6a39c91edb 100644 --- a/src/main/frontend/handler/editor.cljs +++ b/src/main/frontend/handler/editor.cljs @@ -77,8 +77,8 @@ (def edit-block! block-handler/edit-block!) (defn- outliner-save-block! - [block] - (outliner-op/save-block! block)) + [block & {:as opts}] + (outliner-op/save-block! block opts)) (defn get-block-own-order-list-type [block] @@ -3642,7 +3642,7 @@ (when-not (= current-value value) (let [block {:block/uuid block-id :block/collapsed? value}] - (outliner-save-block! block))))))) + (outliner-save-block! block {:outliner-op :collapse-expand-blocks}))))))) (doseq [block-id block-ids] (state/set-collapsed-block! block-id value))))) From a1504e4d9e1dcd321c1b8fa169f78271dee8e860 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Wed, 16 Apr 2025 18:06:41 +0800 Subject: [PATCH 07/15] fix: sort order for references --- deps/db/src/logseq/db/common/view.cljs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/deps/db/src/logseq/db/common/view.cljs b/deps/db/src/logseq/db/common/view.cljs index 0812b60879..adb52cf36f 100644 --- a/deps/db/src/logseq/db/common/view.cljs +++ b/deps/db/src/logseq/db/common/view.cljs @@ -566,9 +566,11 @@ [(:block/uuid (first blocks)) (map (fn [b] {:db/id (:db/id b) - :block/parent (:block/uuid (:block/parent b))}) blocks)]) + :block/parent (:block/uuid (:block/parent b))}) + (ldb/sort-by-order blocks))]) parent-groups)) - (map :db/id entities))] + (->> (sort-entities db sorting entities) + (map :db/id)))] [by-value' group])) result) (map :db/id result))] From d73e78598aa77778fcd26b87249d5c77b94bd605 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Wed, 16 Apr 2025 18:15:59 +0800 Subject: [PATCH 08/15] fix: list view re-render when collapsing and expanding a block --- src/main/frontend/components/views.cljs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/frontend/components/views.cljs b/src/main/frontend/components/views.cljs index 1b3e1e6e2a..0707aa6c4b 100644 --- a/src/main/frontend/components/views.cljs +++ b/src/main/frontend/components/views.cljs @@ -1814,7 +1814,7 @@ [(:db/id view-entity) (hooks/use-debounced-value input 300) sorting-filters - (:logseq.property.view/group-by-property view-entity) + (:db/id (:logseq.property.view/group-by-property view-entity)) ;; page filters (:logseq.property.linked-references/includes view-parent) (:logseq.property.linked-references/excludes view-parent) From fd40bbeec533debbd09c0f6ee4b526470f5164e0 Mon Sep 17 00:00:00 2001 From: rcmerci Date: Wed, 16 Apr 2025 21:56:46 +0800 Subject: [PATCH 09/15] enhance: c.m/backoff add :reset-flow option We can use worker-flows/online-event-flow to trigger a retry immediately upon receiving an 'online' event, instead of waiting for the next retry delay. --- deps.edn | 2 +- src/main/frontend/common/missionary.cljs | 65 +++++++++++++++-------- src/main/frontend/handler.cljs | 6 ++- src/main/frontend/worker/db_worker.cljs | 1 + src/main/frontend/worker/flows.cljs | 13 +++++ src/main/frontend/worker/rtc/client.cljs | 6 ++- src/main/frontend/worker/rtc/ws.cljs | 4 +- src/main/frontend/worker/state.cljs | 7 ++- src/main/frontend/worker/thread_atom.cljs | 12 +++++ src/rtc_e2e_test/client_steps.cljs | 8 +-- src/rtc_e2e_test/helper.cljs | 14 ++--- 11 files changed, 100 insertions(+), 38 deletions(-) create mode 100644 src/main/frontend/worker/flows.cljs create mode 100644 src/main/frontend/worker/thread_atom.cljs diff --git a/deps.edn b/deps.edn index d37bbd76b6..718dfb7ba9 100644 --- a/deps.edn +++ b/deps.edn @@ -40,7 +40,7 @@ logseq/shui {:local/root "deps/shui"} metosin/malli {:mvn/version "0.16.1"} com.cognitect/transit-cljs {:mvn/version "0.8.280"} - missionary/missionary {:mvn/version "b.39"} + missionary/missionary {:mvn/version "b.44"} meander/epsilon {:mvn/version "0.0.650"} io.github.open-spaced-repetition/cljc-fsrs {:git/sha "0e70e96a73cf63c85dcc2df4d022edf12806b239" diff --git a/src/main/frontend/common/missionary.cljs b/src/main/frontend/common/missionary.cljs index 033b912613..4ed36b194d 100644 --- a/src/main/frontend/common/missionary.cljs +++ b/src/main/frontend/common/missionary.cljs @@ -21,32 +21,55 @@ (m/reductions {} init-value) (m/latest identity)))) -(def delays (reductions * 1000 (repeat 2))) - -(def ^:private retry-sentinel (js-obj)) -(defn backoff - "Retry task when it throw exception `(get ex-data :missionary/retry)`" - [delays-seq task] - (m/sp - (loop [[delay & rest-delays] (seq delays-seq)] - (let [r (try - (m/? task) - (catch :default e - (if (and (some-> e ex-data :missionary/retry) - (pos-int? delay)) - (do (m/? (m/sleep delay)) - (println :missionary/retry "after" delay "ms (" (ex-message e) ")") - retry-sentinel) - (throw e))))] - (if (identical? r retry-sentinel) - (recur rest-delays) - r))))) - (defn mix "Return a flow which is mixed by `flows`" [& flows] (m/ap (m/?> (m/?> (count flows) (m/seed flows))))) +(def never-flow (m/ap (m/? m/never))) + +(def delays (reductions * 1000 (repeat 2))) + +(def ^:private retry-sentinel (js-obj)) +(defn backoff + "Retry task when it throw exception `(get ex-data :missionary/retry)` + :delay-seq - retry delay-msecs + :reset-flow - retry immediately when getting value from flow and reset delays to init state" + [{:keys [delay-seq reset-flow] + :or {delay-seq (take 4 delays) + reset-flow never-flow}} + task] + (let [reset-flow* (mix reset-flow never-flow)] + (m/sp + (loop [[delay & rest-delays] (seq delay-seq)] + (let [r (try + (m/? task) + (catch :default e + (if (and (some-> e ex-data :missionary/retry) + (pos-int? delay)) + (let [delay-or-reset + (m/? (m/race (m/sleep delay :delay) + (m/reduce (fn [_ r] (when r (reduced :reset))) nil + (->> (continue-flow reset-flow*) + (m/eduction (drop 1) (take 1)))))) + rest-delays* + (case delay-or-reset + :delay + (do (println :missionary/retry "after" delay "ms (" (ex-message e) ")") + rest-delays) + :reset + (do (println :missionary/retry "retry now (" (ex-message e) ")") + delay-seq))] + [retry-sentinel rest-delays*]) + (throw e))))] + (if (and (vector? r) + (first r) ;; if delete this `(first r)`, + ;; the code continues to the next line even if r=0... + ;; I suspect it's a bug in missionary. + (identical? retry-sentinel (first r))) + (recur (second r)) + r)))))) + (defn clock "Return a flow that emits `value` every `interval-ms`." ([interval-ms] diff --git a/src/main/frontend/handler.cljs b/src/main/frontend/handler.cljs index ba24d76b32..3fb2b748f3 100644 --- a/src/main/frontend/handler.cljs +++ b/src/main/frontend/handler.cljs @@ -104,7 +104,9 @@ (defn- handle-connection-change [e] (let [online? (= (gobj/get e "type") "online")] - (state/set-online! online?))) + (state/set-online! online?) + (state/ (util/indexeddb-check?) (p/catch (fn [_e] @@ -177,6 +178,7 @@ _ (if (empty? repos) (repo-handler/new-db! config/demo-repo) (restore-and-setup! repo))] + (set-network-watcher!) (when (util/electron?) (persist-db/run-export-periodically!)) (when (mobile-util/native-platform?) diff --git a/src/main/frontend/worker/db_worker.cljs b/src/main/frontend/worker/db_worker.cljs index 7d94543766..2051e6259f 100644 --- a/src/main/frontend/worker/db_worker.cljs +++ b/src/main/frontend/worker/db_worker.cljs @@ -25,6 +25,7 @@ [frontend.worker.rtc.db-listener] [frontend.worker.search :as search] [frontend.worker.state :as worker-state] + [frontend.worker.thread-atom] [frontend.worker.undo-redo :as undo-redo] [frontend.worker.util :as worker-util] [goog.object :as gobj] diff --git a/src/main/frontend/worker/flows.cljs b/src/main/frontend/worker/flows.cljs new file mode 100644 index 0000000000..4f5c058f7a --- /dev/null +++ b/src/main/frontend/worker/flows.cljs @@ -0,0 +1,13 @@ +(ns frontend.worker.flows + "common flows in worker thread" + (:require [frontend.worker.state :as worker-state] + [missionary.core :as m])) + +(def online-event-flow + (->> (m/watch (get @worker-state/*state :thread-atom/online-event)) + (m/eduction + (drop-while nil?) + (filter true?)))) + +(comment + ((m/reduce (fn [_ x] (prn :xxx x)) online-event-flow) prn prn)) diff --git a/src/main/frontend/worker/rtc/client.cljs b/src/main/frontend/worker/rtc/client.cljs index 63452934f5..f908663f31 100644 --- a/src/main/frontend/worker/rtc/client.cljs +++ b/src/main/frontend/worker/rtc/client.cljs @@ -3,6 +3,7 @@ (:require [clojure.string :as string] [datascript.core :as d] [frontend.common.missionary :as c.m] + [frontend.worker.flows :as worker-flows] [frontend.worker.rtc.branch-graph :as r.branch-graph] [frontend.worker.rtc.client-op :as client-op] [frontend.worker.rtc.exception :as r.ex] @@ -64,7 +65,10 @@ (let [{:keys [max-remote-schema-version]} (m/? (c.m/backoff - (take 5 (drop 2 c.m/delays)) ;retry 5 times if remote-graph is creating (4000 8000 16000 32000 64000) + {:delay-seq + ;retry 5 times if remote-graph is creating (4000 8000 16000 32000 64000) + (take 5 (drop 2 c.m/delays)) + :reset-flow worker-flows/online-event-flow} (new-task--register-graph-updates get-ws-create-task graph-uuid major-schema-version repo)))] (when max-remote-schema-version (add-log-fn :rtc.log/higher-remote-schema-version-exists diff --git a/src/main/frontend/worker/rtc/ws.cljs b/src/main/frontend/worker/rtc/ws.cljs index dc3ec0a7da..5aa3086ec0 100644 --- a/src/main/frontend/worker/rtc/ws.cljs +++ b/src/main/frontend/worker/rtc/ws.cljs @@ -4,6 +4,7 @@ https://github.com/ReilySiegel/missionary-websocket/blob/master/src/com/reilysiegel/missionary/websocket.cljs" (:require [cljs-http-missionary.client :as http] [frontend.common.missionary :as c.m] + [frontend.worker.flows :as worker-flows] [frontend.worker.rtc.exception :as r.ex] [frontend.worker.rtc.malli-schema :as rtc-schema] [missionary.core :as m])) @@ -88,7 +89,8 @@ (pos-int? open-ws-timeout)) [retry-count open-ws-timeout]) (c.m/backoff - (take retry-count c.m/delays) + {:delay-seq (take retry-count c.m/delays) + :reset-flow worker-flows/online-event-flow} (m/sp (try (if-let [ws (m/? (m/timeout (create-mws* url) open-ws-timeout))] diff --git a/src/main/frontend/worker/state.cljs b/src/main/frontend/worker/state.cljs index ca88a6dcf2..2cec459016 100644 --- a/src/main/frontend/worker/state.cljs +++ b/src/main/frontend/worker/state.cljs @@ -48,7 +48,12 @@ ;; new implementation :undo/repo->ops (atom {}) - :redo/repo->ops (atom {})})) + :redo/repo->ops (atom {}) + + + ;; thread atoms, these atoms' value are syncing from ui-thread + :thread-atom/online-event (atom nil) + })) (defonce *rtc-ws-url (atom nil)) diff --git a/src/main/frontend/worker/thread_atom.cljs b/src/main/frontend/worker/thread_atom.cljs new file mode 100644 index 0000000000..26f09a44a4 --- /dev/null +++ b/src/main/frontend/worker/thread_atom.cljs @@ -0,0 +1,12 @@ +(ns frontend.worker.thread-atom + "atoms from ui-thread" + (:require [frontend.common.thread-api :as thread-api :refer [def-thread-api]] + [frontend.worker.state :as worker-state])) + +(def-thread-api :thread-api/update-thread-atom + [atom-key new-value] + (assert (and (keyword? atom-key) + (identical? "thread-atom" (namespace atom-key)))) + (when-let [a (get @worker-state/*state atom-key)] + (reset! a new-value) + nil)) diff --git a/src/rtc_e2e_test/client_steps.cljs b/src/rtc_e2e_test/client_steps.cljs index b903a6b7ab..9127758e9e 100644 --- a/src/rtc_e2e_test/client_steps.cljs +++ b/src/rtc_e2e_test/client_steps.cljs @@ -44,7 +44,7 @@ (is (nil? r))) (m/? (c.m/backoff - (take 4 c.m/delays) + {} (m/sp (let [conn (helper/get-downloaded-test-conn) page1 (d/pull @conn '[*] [:block/uuid const/page1-uuid]) @@ -72,7 +72,7 @@ (m/? (helper/new-task--wait-all-client-ops-sent)))) :client2 (c.m/backoff - (take 4 c.m/delays) + {} (m/sp (let [conn (helper/get-downloaded-test-conn) page (d/pull @conn '[*] [:block/uuid const/page2-uuid])] @@ -107,7 +107,7 @@ (m/? (helper/new-task--wait-all-client-ops-sent)))) :client2 (c.m/backoff - (take 4 c.m/delays) + {} (m/sp (let [conn (helper/get-downloaded-test-conn) block1 (d/pull @conn @@ -231,7 +231,7 @@ client2: (m/? (helper/new-task--client2-sync-barrier-1->2 "step6")) (m/? (c.m/backoff - (take 4 c.m/delays) + {} (m/sp (let [page (d/pull @conn '[*] [:block/uuid const/step6-page-uuid]) page-blocks (when-let [page-id (:db/id page)] diff --git a/src/rtc_e2e_test/helper.cljs b/src/rtc_e2e_test/helper.cljs index afec11063c..03192cd2a6 100644 --- a/src/rtc_e2e_test/helper.cljs +++ b/src/rtc_e2e_test/helper.cljs @@ -25,7 +25,7 @@ (defn new-task--wait-creating-graph [graph-uuid] (c.m/backoff - (take 4 c.m/delays) + {} (m/sp (let [graphs (m/? (rtc.core/new-task--get-graphs const/test-token)) graph (some (fn [graph] (when (= graph-uuid (:graph-uuid graph)) graph)) graphs)] @@ -47,7 +47,7 @@ (def new-task--get-remote-example-graph-uuid (c.m/backoff - (take 5 c.m/delays) + {} (m/sp (let [graphs (m/? (rtc.core/new-task--get-graphs const/test-token)) graph @@ -94,11 +94,11 @@ #_:clj-kondo/ignore (me/find client-op - [?op-type _ {:block-uuid ?block-uuid :av-coll [[!a !v _ !add] ...]}] - [?op-type ?block-uuid (map vector !a !v !add)] + [?op-type _ {:block-uuid ?block-uuid :av-coll [[!a !v _ !add] ...]}] + [?op-type ?block-uuid (map vector !a !v !add)] - [?op-type _ {:block-uuid ?block-uuid}] - [?op-type ?block-uuid])) + [?op-type _ {:block-uuid ?block-uuid}] + [?op-type ?block-uuid])) (defn new-task--wait-all-client-ops-sent [& {:keys [timeout] :or {timeout 10000}}] @@ -144,7 +144,7 @@ "Return a task that return message from other client" [block-title-pred-fn & {:keys [retry-message retry-count] :or {retry-count 4}}] (c.m/backoff - (take retry-count c.m/delays) + {:delay-seq (take retry-count c.m/delays)} (m/sp (let [conn (get-downloaded-test-conn) message-page-id (:db/id (ldb/get-page @conn const/message-page-uuid)) From b78edc1b59b2fb7a2f945cdd4bd797f6c19c1769 Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Wed, 16 Apr 2025 15:04:59 -0400 Subject: [PATCH 10/15] fix: :build/property-classes for built-in classes --- deps/db/src/logseq/db/sqlite/build.cljs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/deps/db/src/logseq/db/sqlite/build.cljs b/deps/db/src/logseq/db/sqlite/build.cljs index 22a071d2be..6fbbd883f2 100644 --- a/deps/db/src/logseq/db/sqlite/build.cljs +++ b/deps/db/src/logseq/db/sqlite/build.cljs @@ -659,8 +659,10 @@ (if (:logseq.property/classes m) (update m :logseq.property/classes (fn [cs] - (mapv #(or (some->> (:db/ident %) class-ident->id (hash-map :db/id)) - (throw (ex-info (str "No :db/id found for :db/ident " (pr-str (:db/ident %))) {}))) + (mapv #(if (db-class/logseq-class? (:db/ident %)) + % + (or (some->> (:db/ident %) class-ident->id (hash-map :db/id)) + (throw (ex-info (str "No :db/id found for :db/ident " (pr-str %)) {})))) cs))) m)) properties-tx) From 2f4b40aa228c456722d312949afa4e94473beaab Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Wed, 16 Apr 2025 15:11:04 -0400 Subject: [PATCH 11/15] fix: property value blocks failing export+import Property value blocks that were referenced elsewhere failed to export. Property value blocks with ref-able properties are not supported yet and failed import. Fixes https://test.logseq.com/#/page/67fe80a2-e7b3-44ab-aaad-1f365c5b2541 --- deps/db/src/logseq/db/sqlite/export.cljs | 55 ++++++++++++++++++------ 1 file changed, 43 insertions(+), 12 deletions(-) diff --git a/deps/db/src/logseq/db/sqlite/export.cljs b/deps/db/src/logseq/db/sqlite/export.cljs index 6592365c0a..0e398d8647 100644 --- a/deps/db/src/logseq/db/sqlite/export.cljs +++ b/deps/db/src/logseq/db/sqlite/export.cljs @@ -9,11 +9,13 @@ [logseq.db :as ldb] [logseq.db.frontend.class :as db-class] [logseq.db.frontend.content :as db-content] + [logseq.db.frontend.db :as db-db] [logseq.db.frontend.entity-plus :as entity-plus] [logseq.db.frontend.entity-util :as entity-util] [logseq.db.frontend.property :as db-property] [logseq.db.sqlite.build :as sqlite-build] - [medley.core :as medley])) + [medley.core :as medley] + [logseq.db.frontend.property.type :as db-property-type])) ;; Export fns ;; ========== @@ -40,7 +42,7 @@ (defn- build-pvalue-entity-for-build-page [pvalue] - (cond (ldb/internal-page? pvalue) + (cond (entity-util/internal-page? pvalue) ;; Should page properties be pulled here? [:build/page (cond-> (shallow-copy-page pvalue) (seq (:block/tags pvalue)) @@ -48,7 +50,10 @@ (entity-util/journal? pvalue) [:build/page {:build/journal (:block/journal-day pvalue)}])) -(defn- build-pvalue-entity-default [ent-properties pvalue options] +(defn- build-pvalue-entity-default [db ent-properties pvalue + {:keys [include-uuid-fn] + :or {include-uuid-fn (constantly false)} + :as options}] (if (or (seq ent-properties) (seq (:block/tags pvalue))) (cond-> {:build/property-value :block :block/title (or (block-title pvalue) @@ -57,7 +62,17 @@ (assoc :build/tags (->build-tags (:block/tags pvalue))) (seq ent-properties) - (assoc :build/properties ent-properties) + (assoc :build/properties + ;; TODO: Add support for ref properties here and in sqlite.build + (->> ent-properties + (keep (fn [[k v]] + (let [prop-type (:logseq.property/type (d/entity db k))] + (when-not (contains? db-property-type/all-ref-property-types prop-type) + [k v])))) + (into {}))) + + (include-uuid-fn (:block/uuid pvalue)) + (assoc :block/uuid (:block/uuid pvalue) :build/keep-uuid? true) (:include-timestamps? options) (merge (select-keys pvalue [:block/created-at :block/updated-at]))) @@ -87,7 +102,7 @@ (medley/filter-keys db-property/internal-property?)) ent-properties (when (and (not (:block/closed-value-property pvalue)) (seq ent-properties*)) (buildable-properties db' ent-properties* properties-config' options'))] - (build-pvalue-entity-default ent-properties pvalue options'))))))] + (build-pvalue-entity-default db ent-properties pvalue options'))))))] (->> ent-properties (map (fn [[k v]] [k @@ -259,7 +274,8 @@ (mapcat (fn [val-or-vals] (keep #(when (and (vector? %) (= :block/uuid (first %)) - (::existing-property-value? (meta %))) (second %)) + (::existing-property-value? (meta %))) + (second %)) (if (set? val-or-vals) val-or-vals [val-or-vals])))) set)) @@ -293,7 +309,7 @@ (when-let [prop-ids (seq (map :db/ident (filter entity-util/property? ents)))] (build-export-properties db prop-ids export-opts)) classes - (when-let [class-ents (seq (filter ldb/class? ents))] + (when-let [class-ents (seq (filter entity-util/class? ents))] (->> class-ents (map #(vector (:db/ident %) (build-export-class % export-opts))) (into {})))] @@ -328,7 +344,7 @@ (let [class-parent-ents (->> classes-config (filter #(:build/class-parent (val %))) (map #(d/entity db (key %))) - ldb/get-classes-parents) + db-db/get-classes-parents) classes (->> class-parent-ents (remove #(db-class/logseq-class? (:db/ident %))) @@ -670,8 +686,16 @@ (update :pages-and-blocks (fn [pages-and-blocks] (mapv (fn [{:keys [page blocks]}] - {:page (remove-uuid-if-not-ref page) - :blocks (sqlite-build/update-each-block blocks remove-uuid-if-not-ref)}) + (let [page-map {:page (remove-uuid-if-not-ref page) + :blocks (sqlite-build/update-each-block blocks remove-uuid-if-not-ref)} + ;; TODO: Walk data structure via :build/properties instead of slower walk + page-map' + (walk/postwalk (fn [f] + (if (and (map? f) (:build/property-value f)) + (remove-uuid-if-not-ref f) + f)) + page-map)] + page-map')) pages-and-blocks)))))) (defn- add-ontology-for-include-namespaces @@ -750,12 +774,19 @@ undefined)) (defn- find-undefined-uuids [{:keys [classes properties pages-and-blocks]}] - (let [known-uuids + (let [pvalue-known-uuids (atom #{}) + _ (walk/postwalk (fn [f] + (if (and (map? f) (:build/property-value f) (:block/uuid f)) + (swap! pvalue-known-uuids conj (:block/uuid f)) + f)) + pages-and-blocks) + known-uuids (->> (concat (keep :block/uuid (vals classes)) (keep :block/uuid (vals properties)) (keep #(get-in % [:page :block/uuid]) pages-and-blocks) (mapcat #(sqlite-build/extract-from-blocks (:blocks %) (fn [m] (some-> m :block/uuid vector))) - pages-and-blocks)) + pages-and-blocks) + @pvalue-known-uuids) set) ;; Only looks one-level deep in properties e.g. not inside :build/page ;; Doesn't find :block/link refs From 04d1a2f47ee29af0754900c2a37b6cee4f59337a Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 17 Apr 2025 10:29:05 +0800 Subject: [PATCH 12/15] fix: keep title for built-in blocks when exporting debug transit --- src/main/frontend/worker/export.cljs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/frontend/worker/export.cljs b/src/main/frontend/worker/export.cljs index f13c8e189d..39018c5f96 100644 --- a/src/main/frontend/worker/export.cljs +++ b/src/main/frontend/worker/export.cljs @@ -65,6 +65,7 @@ (if (and (contains? #{:block/title :block/name} a) (let [entity (d/entity @conn e)] (and (not (:db/ident entity)) - (not (ldb/journal? entity))))) + (not (ldb/journal? entity)) + (not (:logseq.property/built-in? entity))))) (d/datom e a (str "debug " e) t) (d/datom e a v t)))))) From c50bb0a73a3de07bdc8c06592e30536b79ad174a Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 17 Apr 2025 11:27:28 +0800 Subject: [PATCH 13/15] fix: whitespaces are hidden from table name cells --- src/main/frontend/components/views.cljs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/frontend/components/views.cljs b/src/main/frontend/components/views.cljs index 0707aa6c4b..392a3b6bc3 100644 --- a/src/main/frontend/components/views.cljs +++ b/src/main/frontend/components/views.cljs @@ -215,7 +215,7 @@ {:align :start}) (editor-handler/edit-block! block :max {:container-id :unknown-container}))))))} (if block - (inline-title (:block/title block)) + [:div (inline-title (:block/title block))] [:div])])) (defn build-columns From e5544885e65d3bde6066bd889bfe8ea5e1981236 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 17 Apr 2025 11:32:50 +0800 Subject: [PATCH 14/15] enhance(ux): show only the first line of block title for name cells --- src/main/frontend/components/views.cljs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/frontend/components/views.cljs b/src/main/frontend/components/views.cljs index 392a3b6bc3..8b37eba3f3 100644 --- a/src/main/frontend/components/views.cljs +++ b/src/main/frontend/components/views.cljs @@ -215,7 +215,11 @@ {:align :start}) (editor-handler/edit-block! block :max {:container-id :unknown-container}))))))} (if block - [:div (inline-title (:block/title block))] + [:div (inline-title + (some->> (:block/title block) + string/trim + string/split-lines + first))] [:div])])) (defn build-columns From b5b95e7b8c4bfbfbe5b749e46d3c5739b1a4ccd4 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 17 Apr 2025 12:39:56 +0800 Subject: [PATCH 15/15] fix: broken text and unstable scrolling for list views --- src/main/frontend/components/views.cljs | 91 ++++++++++++------------- 1 file changed, 43 insertions(+), 48 deletions(-) diff --git a/src/main/frontend/components/views.cljs b/src/main/frontend/components/views.cljs index 8b37eba3f3..4e6d2db644 100644 --- a/src/main/frontend/components/views.cljs +++ b/src/main/frontend/components/views.cljs @@ -1224,7 +1224,8 @@ db-id (cond (map? item) (:db/id item) (number? item) item :else nil) - [item set-item!] (hooks/use-state nil) + block (some-> db-id db/entity) + [item set-item!] (hooks/use-state (when (:block.temp/fully-loaded? block) block)) opts (if list-view? {:skip-refresh? true :children? false} @@ -1255,7 +1256,7 @@ (gdom/getElement "main-content-container")) :compute-item-key (fn [idx] (let [block-id (util/nth-safe rows idx)] - (str "table-row-" (:group-idx option) "-" block-id))) + (str "table-row-" block-id))) :skipAnimationFrameInResizeObserver true :total-count (count rows) :context {:scrolling scrolling?} @@ -1302,8 +1303,8 @@ :compute-item-key (fn [idx] (let [block-id (util/nth-safe rows idx)] (str "list-row-" block-id))) - ;; :skipAnimationFrameInResizeObserver true :total-count (count rows) + :skipAnimationFrameInResizeObserver true :item-content (fn [idx] (lazy-item-render rows idx))}))) breadcrumb (state/get-component :block/breadcrumb) all-numbers? (every? number? rows)] @@ -1343,6 +1344,7 @@ {:ref #(reset! *scroller-ref %) :total-count (count blocks) :custom-scroll-parent (gdom/getElement "main-content-container") + :skipAnimationFrameInResizeObserver true :compute-item-key (fn [idx] (str (:db/id view-entity) "-card-" idx)) :item-content (fn [idx] @@ -1698,52 +1700,45 @@ :add-new-object! add-new-object!}] (if (and group-by-property-ident (not (number? (first (:rows table))))) (when (seq (:rows table)) - [:div.flex.flex-col.border-t.pt-2 - (ui/virtualized-list - {:class (when list-view? "group-list-view") - :custom-scroll-parent (gdom/getElement "main-content-container") - :increase-viewport-by {:top 300 :bottom 300} - :compute-item-key (fn [idx] - (str "table-group-" idx)) - :skipAnimationFrameInResizeObserver true - :total-count (count (:rows table)) - :item-content (fn [idx] - (let [[value group] (nth (:rows table) idx) - add-new-object! (when (fn? add-new-object!) - (fn [_] - (add-new-object! view-entity table - {:properties {(:db/ident group-by-property) (or (and (map? value) (:db/id value)) value)}}))) - table' (shui/table-option (-> table-map - (assoc-in [:data-fns :add-new-object!] add-new-object!) - (assoc :data group ; data for this group - ))) - readable-property-value #(if (and (map? %) (or (:block/title %) (:logseq.property/value %))) - (db-property/property-value-content %) - (str %)) - group-by-page? (or (= :block/page group-by-property-ident) - (and (not db-based?) (contains? #{:linked-references :unlinked-references} display-type)))] - (rum/with-key - (ui/foldable - [:div - (cond - group-by-page? - (if value - (let [c (state/get-component :block/page-cp)] - (c {:disable-preview? true} value)) - [:div.text-muted-foreground.text-sm - "Pages"]) + [:div.flex.flex-col.border-t.pt-2.gap-2 + (map-indexed + (fn [idx [value group]] + (let [add-new-object! (when (fn? add-new-object!) + (fn [_] + (add-new-object! view-entity table + {:properties {(:db/ident group-by-property) (or (and (map? value) (:db/id value)) value)}}))) + table' (shui/table-option (-> table-map + (assoc-in [:data-fns :add-new-object!] add-new-object!) + (assoc :data group ; data for this group + ))) + readable-property-value #(if (and (map? %) (or (:block/title %) (:logseq.property/value %))) + (db-property/property-value-content %) + (str %)) + group-by-page? (or (= :block/page group-by-property-ident) + (and (not db-based?) (contains? #{:linked-references :unlinked-references} display-type)))] + (rum/with-key + (ui/foldable + [:div + (cond + group-by-page? + (if value + (let [c (state/get-component :block/page-cp)] + (c {:disable-preview? true} value)) + [:div.text-muted-foreground.text-sm + "Pages"]) - (some? value) - (let [icon (pu/get-block-property-value value :logseq.property/icon)] - [:div.flex.flex-row.gap-1.items-center - (when icon (icon-component/icon icon {:color? true})) - (readable-property-value value)]) - :else - (str "No " (:block/title group-by-property)))] - (let [render (view-cp view-entity (assoc table' :rows group :group-idx idx) option view-opts)] - (if list-view? [:div.-ml-2 render] render)) - {:title-trigger? false}) - (str "group-" idx))))})]) + (some? value) + (let [icon (pu/get-block-property-value value :logseq.property/icon)] + [:div.flex.flex-row.gap-1.items-center + (when icon (icon-component/icon icon {:color? true})) + (readable-property-value value)]) + :else + (str "No " (:block/title group-by-property)))] + (let [render (view-cp view-entity (assoc table' :rows group) option view-opts)] + (if list-view? [:div.-ml-2 render] render)) + {:title-trigger? false}) + (str (:db/id view-entity) "-group-idx-" idx)))) + (:rows table))]) (view-cp view-entity table option view-opts)))] (merge {:title-trigger? false} foldable-options))]))