From 3addb5905b5cd2797f7e22ab9a95cdb9f49cee90 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Mon, 29 Jan 2024 13:17:37 +0800 Subject: [PATCH 01/60] perf: load partial data when loading a graph Instead of all the datoms --- .../src/logseq/common/util/date_time.cljs} | 8 ++- deps/db/src/logseq/db/sqlite/common_db.cljs | 54 ++++++++++++++++--- .../src/logseq/graph_parser/block.cljs | 2 +- .../src/logseq/graph_parser/util/db.cljs | 8 +-- src/main/frontend/date.cljs | 2 +- src/main/frontend/db/restore.cljs | 12 ++--- src/main/frontend/db_worker.cljs | 12 +++-- src/main/frontend/handler/import.cljs | 2 +- 8 files changed, 73 insertions(+), 27 deletions(-) rename deps/{graph-parser/src/logseq/graph_parser/date_time_util.cljs => common/src/logseq/common/util/date_time.cljs} (92%) diff --git a/deps/graph-parser/src/logseq/graph_parser/date_time_util.cljs b/deps/common/src/logseq/common/util/date_time.cljs similarity index 92% rename from deps/graph-parser/src/logseq/graph_parser/date_time_util.cljs rename to deps/common/src/logseq/common/util/date_time.cljs index e1af4acc30..4cde403de4 100644 --- a/deps/graph-parser/src/logseq/graph_parser/date_time_util.cljs +++ b/deps/common/src/logseq/common/util/date_time.cljs @@ -1,4 +1,4 @@ -(ns logseq.graph-parser.date-time-util +(ns logseq.common.util.date-time "cljs-time util fns for graph-parser" (:require [cljs-time.format :as tf] [clojure.string :as string] @@ -73,3 +73,9 @@ ([date sep] (let [{:keys [year month day]} (year-month-day-padded (get-date date))] (str year sep month sep day)))) + +(defn date->int + "Given a date object, returns its journal page integer" + [date] + (parse-long + (string/replace (ymd date) "/" ""))) diff --git a/deps/db/src/logseq/db/sqlite/common_db.cljs b/deps/db/src/logseq/db/sqlite/common_db.cljs index 5b5b80ff44..e952978a69 100644 --- a/deps/db/src/logseq/db/sqlite/common_db.cljs +++ b/deps/db/src/logseq/db/sqlite/common_db.cljs @@ -3,18 +3,58 @@ (:require [datascript.core :as d] ["path" :as node-path] [clojure.string :as string] - [logseq.db.sqlite.util :as sqlite-util])) + [logseq.db.sqlite.util :as sqlite-util] + [logseq.common.util.date-time :as date-time-util])) -(defn get-initial-data - "Returns initial data as vec of datoms" +(defn- get-built-in-files [db] - (->> (d/datoms db :eavt) - vec)) + (let [files ["logseq/config.edn" + "logseq/custom.css" + "logseq/custom.js"]] + (map #(d/pull db '[*] [:file/path %]) files))) + +(defn get-all-pages + [db] + (->> (d/datoms db :avet :block/name) + (map (fn [e] + (d/pull db '[*] (:e e)))))) + +(defn get-latest-journals + [db n] + (let [date (js/Date.) + _ (.setDate date (- (.getDate date) (dec n))) + today (date-time-util/date->int (js/Date.))] + (->> + (d/q '[:find [(pull ?page [*]) ...] + :in $ ?today + :where + [?page :block/name ?page-name] + [?page :block/journal? true] + [?page :block/journal-day ?journal-day] + [(<= ?journal-day ?today)]] + db + today) + (sort-by :block/journal-day) + (reverse) + (take n)))) + +;; built-in files + latest journals + favorites +(defn get-initial-data + "Returns initial data" + [db] + (let [files (get-built-in-files db) + journals (get-latest-journals db 3) + journal-blocks (mapcat (fn [journal] + (let [blocks (:block/_page (d/entity db (:db/id journal)))] + (map (fn [b] (d/pull db '[*] (:db/id b))) blocks))) journals)] + (concat files journals journal-blocks))) (defn restore-initial-data "Given initial sqlite data and schema, returns a datascript connection" - [datoms schema] - (d/conn-from-datoms datoms schema)) + [data schema] + (let [conn (d/create-conn schema)] + (d/transact! conn data) + conn)) (defn create-kvs-table! "Creates a sqlite table for use with datascript.storage if one doesn't exist" diff --git a/deps/graph-parser/src/logseq/graph_parser/block.cljs b/deps/graph-parser/src/logseq/graph_parser/block.cljs index 778dd10edc..e333242bd1 100644 --- a/deps/graph-parser/src/logseq/graph_parser/block.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/block.cljs @@ -5,7 +5,7 @@ [clojure.walk :as walk] [datascript.core :as d] [logseq.common.config :as common-config] - [logseq.graph-parser.date-time-util :as date-time-util] + [logseq.common.util.date-time :as date-time-util] [logseq.graph-parser.mldoc :as gp-mldoc] [logseq.graph-parser.property :as gp-property] [logseq.graph-parser.text :as text] diff --git a/deps/graph-parser/src/logseq/graph_parser/util/db.cljs b/deps/graph-parser/src/logseq/graph_parser/util/db.cljs index 0028259b1b..4b214b3a29 100644 --- a/deps/graph-parser/src/logseq/graph_parser/util/db.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/util/db.cljs @@ -2,7 +2,7 @@ "Db util fns that are useful for the frontend and nbb-logseq. This may be used by the graph-parser soon but if not, it should be in its own library" (:require [cljs-time.core :as t] - [logseq.graph-parser.date-time-util :as date-time-util] + [logseq.common.util.date-time :as date-time-util] [logseq.common.util :as common-util] [logseq.common.util.page-ref :as page-ref] [datascript.core :as d] @@ -17,11 +17,7 @@ it will return 1622433600000, which is equivalent to Mon May 31 2021 00 :00:00." ([date hours mins secs millisecs] (.setHours (js/Date. date) hours mins secs millisecs))) -(defn date->int - "Given a date object, returns its journal page integer" - [date] - (parse-long - (string/replace (date-time-util/ymd date) "/" ""))) +(def date->int date-time-util/date->int) (defn old->new-relative-date-format [input] (let [count (re-find #"^\d+" (name input)) diff --git a/src/main/frontend/date.cljs b/src/main/frontend/date.cljs index 8294eb8ad2..2db831361e 100644 --- a/src/main/frontend/date.cljs +++ b/src/main/frontend/date.cljs @@ -7,7 +7,7 @@ [cljs-time.format :as tf] [cljs-time.local :as tl] [frontend.state :as state] - [logseq.graph-parser.date-time-util :as date-time-util] + [logseq.common.util.date-time :as date-time-util] [goog.object :as gobj] [lambdaisland.glogi :as log] [frontend.worker.date :as worker-date])) diff --git a/src/main/frontend/db/restore.cljs b/src/main/frontend/db/restore.cljs index 580fef9376..c517de06ce 100644 --- a/src/main/frontend/db/restore.cljs +++ b/src/main/frontend/db/restore.cljs @@ -6,8 +6,8 @@ [frontend.persist-db :as persist-db] [promesa.core :as p] [cljs-time.core :as t] - [datascript.transit :as dt] - [logseq.db.sqlite.common-db :as sqlite-common-db])) + [logseq.db.sqlite.common-db :as sqlite-common-db] + [clojure.edn :as edn])) (comment (defn- old-schema? @@ -35,16 +35,14 @@ (p/let [start-time (t/now) data (persist-db/> (sqlite-common-db/get-initial-data @conn) - dt/write-transit-str))) + (let [data (->> (sqlite-common-db/get-initial-data @conn) + pr-str)] + (async/go + (let [all-pages (sqlite-common-db/get-all-pages @conn)] + (worker-util/post-message :sync-db-changes (pr-str + {:repo repo + :tx-data all-pages + :tx-meta nil})))) + data))) (closeDB [_this repo] diff --git a/src/main/frontend/handler/import.cljs b/src/main/frontend/handler/import.cljs index 8c51ba3722..46ef191d29 100644 --- a/src/main/frontend/handler/import.cljs +++ b/src/main/frontend/handler/import.cljs @@ -15,7 +15,7 @@ [logseq.graph-parser.mldoc :as gp-mldoc] [logseq.common.util :as common-util] [logseq.graph-parser.whiteboard :as gp-whiteboard] - [logseq.graph-parser.date-time-util :as date-time-util] + [logseq.common.util.date-time :as date-time-util] [frontend.handler.page :as page-handler] [frontend.handler.editor :as editor] [frontend.handler.notification :as notification] From 752f88de19e6288e5f23df4e8e597f067e41e888 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Mon, 29 Jan 2024 14:48:41 +0800 Subject: [PATCH 02/60] Async get block and children --- deps/db/src/logseq/db.cljs | 1 + deps/db/src/logseq/db/sqlite/common_db.cljs | 30 ++++++++++++++++++++- src/main/frontend/components/page.cljs | 25 ++++++++++------- src/main/frontend/db/async.cljs | 19 ++++++++++++- src/main/frontend/db_worker.cljs | 6 +++++ 5 files changed, 70 insertions(+), 11 deletions(-) diff --git a/deps/db/src/logseq/db.cljs b/deps/db/src/logseq/db.cljs index 712e7b5f72..4ca02a5de1 100644 --- a/deps/db/src/logseq/db.cljs +++ b/deps/db/src/logseq/db.cljs @@ -342,6 +342,7 @@ parents)))) (defn get-block-children-ids + "Returns children UUIDs" [db block-uuid] (when-let [eid (:db/id (d/entity db [:block/uuid block-uuid]))] (let [seen (volatile! [])] diff --git a/deps/db/src/logseq/db/sqlite/common_db.cljs b/deps/db/src/logseq/db/sqlite/common_db.cljs index e952978a69..ad0db09043 100644 --- a/deps/db/src/logseq/db/sqlite/common_db.cljs +++ b/deps/db/src/logseq/db/sqlite/common_db.cljs @@ -4,7 +4,9 @@ ["path" :as node-path] [clojure.string :as string] [logseq.db.sqlite.util :as sqlite-util] - [logseq.common.util.date-time :as date-time-util])) + [logseq.db :as ldb] + [logseq.common.util.date-time :as date-time-util] + [logseq.common.util :as common-util])) (defn- get-built-in-files [db] @@ -19,6 +21,32 @@ (map (fn [e] (d/pull db '[*] (:e e)))))) +(defn get-block-and-children + [db name children?] + (let [get-children (fn [col] + (map (fn [e] + (select-keys e [:db/id :block/uuid :block/page :block/left :block/parent])) + col))] + (if (common-util/uuid-string? name) ; block + (let [id (uuid name) + block (d/entity db [:block/uuid id])] + (when block + (let [result {:block (d/pull db '[*] (:db/id block))}] + (if children? + (let [children (->> (ldb/get-block-children-ids db id) + (map #(d/entity db [:block/uuid %])))] + (assoc result :children (get-children children))) + result)) + (cond-> + {:block (d/pull db '[*] (:db/id block))} + children? + (assoc :children (get-children (:block/_parent block)))))) + (when-let [block (d/entity db [:block/name name])] + (cond-> + {:block (d/pull db '[*] (:db/id block))} + children? + (assoc :children (get-children (:block/_page block)))))))) + (defn get-latest-journals [db n] (let [date (js/Date.) diff --git a/src/main/frontend/components/page.cljs b/src/main/frontend/components/page.cljs index 5f89f8f067..6d1cfeee94 100644 --- a/src/main/frontend/components/page.cljs +++ b/src/main/frontend/components/page.cljs @@ -48,7 +48,8 @@ [medley.core :as medley] [promesa.core :as p] [reitit.frontend.easy :as rfe] - [rum.core :as rum])) + [rum.core :as rum] + [frontend.db.async :as db-async])) (defn- get-page-name [state] @@ -445,15 +446,16 @@ (state/get-current-page))) (defn- get-page-entity - [repo path-page-name page-name] + [page-name] (if-let [block-id (parse-uuid page-name)] (let [entity (db/entity [:block/uuid block-id])] entity) - (do - (when-not (db/entity repo [:block/name page-name]) - (let [m (block/page-name->map path-page-name true)] - (db/transact! repo [m]))) - (db/entity [:block/name page-name])))) + (db/entity [:block/name page-name]))) + +(defn- get-sanity-page-name + [state page-name] + (when-let [path-page-name (get-path-page-name state page-name)] + (util/page-name-sanity-lc path-page-name))) ;; A page is just a logical block (rum/defcs ^:large-vars/cleanup-todo page-inner < rum/reactive db-mixins/query @@ -461,12 +463,17 @@ (rum/local false ::control-show?) (rum/local nil ::current-page) (rum/local false ::hover-title?) + {:init (fn [state] + (let [page-name (:page-name (first (:rum/args state))) + page-name' (get-sanity-page-name state page-name)] + (db-async/js result))))) + (get-block-and-children + [_this repo name children?] + (assert (string? name)) + (when-let [conn (worker-state/get-datascript-conn repo)] + (pr-str (sqlite-common-db/get-block-and-children @conn name children?)))) + (transact [_this repo tx-data tx-meta context] (when repo (worker-state/set-db-latest-tx-time! repo)) From eba0fe9545b88f1815b710977cbb626793263c86 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Mon, 29 Jan 2024 15:25:42 +0800 Subject: [PATCH 03/60] Lazy load block container --- deps/db/src/logseq/db/sqlite/common_db.cljs | 7 +- src/main/frontend/components/block.cljs | 35 +-- src/main/frontend/components/page.cljs | 214 +++++++++--------- src/main/frontend/db/async.cljs | 5 +- src/main/frontend/db/restore.cljs | 2 - src/main/frontend/handler/whiteboard.cljs | 4 - .../frontend/modules/outliner/pipeline.cljs | 8 - src/main/frontend/state.cljs | 7 - 8 files changed, 129 insertions(+), 153 deletions(-) diff --git a/deps/db/src/logseq/db/sqlite/common_db.cljs b/deps/db/src/logseq/db/sqlite/common_db.cljs index ad0db09043..84f5277849 100644 --- a/deps/db/src/logseq/db/sqlite/common_db.cljs +++ b/deps/db/src/logseq/db/sqlite/common_db.cljs @@ -71,11 +71,8 @@ "Returns initial data" [db] (let [files (get-built-in-files db) - journals (get-latest-journals db 3) - journal-blocks (mapcat (fn [journal] - (let [blocks (:block/_page (d/entity db (:db/id journal)))] - (map (fn [b] (d/pull db '[*] (:db/id b))) blocks))) journals)] - (concat files journals journal-blocks))) + journals (get-latest-journals db 3)] + (concat files journals))) (defn restore-initial-data "Given initial sqlite data and schema, returns a datascript connection" diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index 866737ff83..7dc75ed0ab 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -84,7 +84,8 @@ [rum.core :as rum] [shadow.loader :as loader] [logseq.common.path :as path] - [electron.ipc :as ipc])) + [electron.ipc :as ipc] + [frontend.db.async :as db-async])) ;; local state (defonce *dragging? @@ -3090,7 +3091,11 @@ (rum/local false ::show-block-right-menu?) {:init (fn [state] (let [[config block] (:rum/args state) - block-id (:block/uuid block)] + block-id (:block/uuid block) + *loading? (atom true)] + (if (:block/content (db/entity [:block/uuid block-id])) + (reset! *loading? false) + (db-async/int (date/today)))) (state/pub-event! [:journal/insert-template page-name]))) state)} - [state repo page-e {:keys [sidebar? whiteboard?] :as config}] + [state _repo page-e {:keys [sidebar? whiteboard?] :as config}] (when page-e (let [page-name (or (:block/name page-e) (str (:block/uuid page-e))) block-id (parse-uuid page-name) block? (boolean block-id) block (get-block page-name) - block-unloaded? (state/sub-block-unloaded? repo (:block/uuid block)) children (:block/_parent block)] (cond - block-unloaded? - (ui/loading "Loading...") - (and (not block?) (empty? children)) @@ -465,127 +460,126 @@ (rum/local false ::hover-title?) {:init (fn [state] (let [page-name (:page-name (first (:rum/args state))) - page-name' (get-sanity-page-name state page-name)] - (db-async/int page-name)) - whiteboard? (:whiteboard? option) ;; in a whiteboard portal shape? - whiteboard-page? (model/whiteboard-page? page-name) ;; is this page a whiteboard? - route-page-name path-page-name - page-name (:block/name page) - page-original-name (:block/original-name page) - title (or page-original-name page-name) - today? (and - journal? - (= page-name (util/page-name-sanity-lc (date/journal-name)))) - *control-show? (::control-show? state) - *all-collapsed? (::all-collapsed? state) - *current-block-page (::current-page state) - block-or-whiteboard? (or block? whiteboard?) - home? (= :home (state/get-current-route))] - (when (or page-name block-or-whiteboard?) - [:div.flex-1.page.relative - (merge (if (seq (:block/tags page)) - (let [page-names (model/get-page-names-by-ids (map :db/id (:block/tags page)))] - {:data-page-tags (text-util/build-data-value page-names)}) - {}) + (when-not (rum/react (::loading? state)) + (when-let [path-page-name (get-path-page-name state page-name)] + (let [current-repo (state/sub :git/current-repo) + repo (or repo current-repo) + page-name (util/page-name-sanity-lc path-page-name) + page (get-page-entity page-name) + block-id (:block/uuid page) + block? (some? (:block/page page)) + journal? (db/journal-page? page-name) + db-based? (config/db-based-graph? repo) + fmt-journal? (boolean (date/journal-title->int page-name)) + whiteboard? (:whiteboard? option) ;; in a whiteboard portal shape? + whiteboard-page? (model/whiteboard-page? page-name) ;; is this page a whiteboard? + route-page-name path-page-name + page-name (:block/name page) + page-original-name (:block/original-name page) + title (or page-original-name page-name) + today? (and + journal? + (= page-name (util/page-name-sanity-lc (date/journal-name)))) + *control-show? (::control-show? state) + *all-collapsed? (::all-collapsed? state) + *current-block-page (::current-page state) + block-or-whiteboard? (or block? whiteboard?) + home? (= :home (state/get-current-route))] + (when (or page-name block-or-whiteboard?) + [:div.flex-1.page.relative + (merge (if (seq (:block/tags page)) + (let [page-names (model/get-page-names-by-ids (map :db/id (:block/tags page)))] + {:data-page-tags (text-util/build-data-value page-names)}) + {}) - {:key path-page-name - :class (util/classnames [{:is-journals (or journal? fmt-journal?)}])}) + {:key path-page-name + :class (util/classnames [{:is-journals (or journal? fmt-journal?)}])}) - (if (and whiteboard-page? (not sidebar?)) - [:div ((state/get-component :whiteboard/tldraw-preview) page-name)] ;; FIXME: this is not reactive - [:div.relative - (when (and (not sidebar?) (not block?)) - [:div.flex.flex-row.space-between - (when (or (mobile-util/native-platform?) (util/mobile?)) - [:div.flex.flex-row.pr-2 - {:style {:margin-left -15} - :on-mouse-over (fn [e] - (page-mouse-over e *control-show? *all-collapsed?)) - :on-mouse-leave (fn [e] - (page-mouse-leave e *control-show?))} - (page-blocks-collapse-control title *control-show? *all-collapsed?)]) - (let [original-name (:block/original-name (db/entity [:block/name (util/page-name-sanity-lc page-name)]))] - (when (and (not whiteboard?) original-name) - (page-title page-name {:journal? journal? - :fmt-journal? fmt-journal? - :preview? preview? - :*hover? (::hover-title? state)}))) - (when (not config/publishing?) - (when config/lsp-enabled? - [:div.flex.flex-row - (plugins/hook-ui-slot :page-head-actions-slotted nil) - (plugins/hook-ui-items :pagebar)]))]) + (if (and whiteboard-page? (not sidebar?)) + [:div ((state/get-component :whiteboard/tldraw-preview) page-name)] ;; FIXME: this is not reactive + [:div.relative + (when (and (not sidebar?) (not block?)) + [:div.flex.flex-row.space-between + (when (or (mobile-util/native-platform?) (util/mobile?)) + [:div.flex.flex-row.pr-2 + {:style {:margin-left -15} + :on-mouse-over (fn [e] + (page-mouse-over e *control-show? *all-collapsed?)) + :on-mouse-leave (fn [e] + (page-mouse-leave e *control-show?))} + (page-blocks-collapse-control title *control-show? *all-collapsed?)]) + (let [original-name (:block/original-name (db/entity [:block/name (util/page-name-sanity-lc page-name)]))] + (when (and (not whiteboard?) original-name) + (page-title page-name {:journal? journal? + :fmt-journal? fmt-journal? + :preview? preview? + :*hover? (::hover-title? state)}))) + (when (not config/publishing?) + (when config/lsp-enabled? + [:div.flex.flex-row + (plugins/hook-ui-slot :page-head-actions-slotted nil) + (plugins/hook-ui-items :pagebar)]))]) - (when (and db-based? (not block?)) - [:div.pb-4 - (db-page/page-info page (::hover-title? state))]) + (when (and db-based? (not block?)) + [:div.pb-4 + (db-page/page-info page (::hover-title? state))]) - [:div - (when (and block? (not sidebar?) (not whiteboard?)) - (let [config {:id "block-parent" - :block? true}] - [:div.mb-4 - (component-block/breadcrumb config repo block-id {:level-limit 3})])) + [:div + (when (and block? (not sidebar?) (not whiteboard?)) + (let [config {:id "block-parent" + :block? true}] + [:div.mb-4 + (component-block/breadcrumb config repo block-id {:level-limit 3})])) - (when (and db-based? (not block?) (not preview?)) - (db-page/page-properties-react page {:configure? false})) + (when (and db-based? (not block?) (not preview?)) + (db-page/page-properties-react page {:configure? false})) ;; blocks - (let [_ (and block? page (reset! *current-block-page (:block/name (:block/page page)))) - _ (when (and block? (not page)) - (route-handler/redirect-to-page! @*current-block-page))] - (page-blocks-cp repo page {:sidebar? sidebar? :whiteboard? whiteboard?}))]]) + (let [_ (and block? page (reset! *current-block-page (:block/name (:block/page page)))) + _ (when (and block? (not page)) + (route-handler/redirect-to-page! @*current-block-page))] + (page-blocks-cp repo page {:sidebar? sidebar? :whiteboard? whiteboard?}))]]) - (when today? - (today-queries repo today? sidebar?)) + (when today? + (today-queries repo today? sidebar?)) - (when today? - (scheduled/scheduled-and-deadlines page-name)) + (when today? + (scheduled/scheduled-and-deadlines page-name)) - (when-not block? - (tagged-pages repo page-name page-original-name)) + (when-not block? + (tagged-pages repo page-name page-original-name)) ;; referenced blocks - (when-not block-or-whiteboard? - [:div {:key "page-references"} - (rum/with-key - (reference/references route-page-name) - (str route-page-name "-refs"))]) + (when-not block-or-whiteboard? + [:div {:key "page-references"} + (rum/with-key + (reference/references route-page-name) + (str route-page-name "-refs"))]) - (when-not block-or-whiteboard? - (when (not journal?) - (hierarchy/structures route-page-name))) + (when-not block-or-whiteboard? + (when (not journal?) + (hierarchy/structures route-page-name))) - (when-not (or block-or-whiteboard? sidebar? home?) - [:div {:key "page-unlinked-references"} - (reference/unlinked-references route-page-name)])])))) + (when-not (or block-or-whiteboard? sidebar? home?) + [:div {:key "page-unlinked-references"} + (reference/unlinked-references route-page-name)])]))))) -(rum/defcs page < rum/reactive +(rum/defcs page < rum/static [state option] - (let [path-page-name (get-path-page-name state (:page-name option)) - page-name (util/page-name-sanity-lc path-page-name) - repo (state/get-current-repo) - page (get-page-entity page-name) - block? (some? (:block/page page)) - page-unloaded? (or (state/sub-page-unloaded? repo page-name) (nil? page))] - (if (and page-unloaded? (not block?)) - (state/update-state! [repo :unloaded-pages] (fn [pages] (conj (set pages) page-name))) - (rum/with-key - (page-inner option) - (or (:page-name option) - (get-page-name state)))))) + (rum/with-key + (page-inner option) + (or (:page-name option) + (get-page-name state)))) (defonce layout (atom [js/window.innerWidth js/window.innerHeight])) diff --git a/src/main/frontend/db/async.cljs b/src/main/frontend/db/async.cljs index 6d7ed19286..c154c8b56f 100644 --- a/src/main/frontend/db/async.cljs +++ b/src/main/frontend/db/async.cljs @@ -104,14 +104,15 @@ (file-async/ Date: Mon, 29 Jan 2024 18:23:55 +0800 Subject: [PATCH 04/60] Lazy load page/block references and embeds --- src/main/frontend/components/block.cljs | 261 +++++++++--------- src/main/frontend/components/page.cljs | 14 +- .../frontend/components/property/value.cljs | 21 +- src/main/frontend/db/async.cljs | 40 ++- src/main/frontend/db/react.cljs | 2 +- src/main/frontend/handler/events.cljs | 4 + src/main/frontend/rum.cljs | 9 + src/main/frontend/state.cljs | 18 +- 8 files changed, 208 insertions(+), 161 deletions(-) diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index 7dc75ed0ab..bdb2c8bfe4 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -82,6 +82,7 @@ [promesa.core :as p] [reitit.frontend.easy :as rfe] [rum.core :as rum] + [frontend.rum :as r] [shadow.loader :as loader] [logseq.common.path :as path] [electron.ipc :as ipc] @@ -810,26 +811,34 @@ (declare block-container) (rum/defc block-embed < rum/reactive + {:init (fn [state] + (let [block-id (second (:rum/args state))] + (db-async/elem - :span.block-ref - (map-inline config label)) - title)] - [:div.block-ref-wrap.inline - {:data-type (name (or block-type :default)) - :data-hl-type hl-type - :on-mouse-down - (fn [^js/MouseEvent e] - (if (util/right-click? e) - (state/set-state! :block-ref/context {:block (:block config) - :block-ref block-id}) - (when (and - (or (gobj/get e "shiftKey") - (not (.. e -target (closest ".blank")))) - (not (util/right-click? e))) - (util/stop e) + (if (state/sub-block-unloaded? (str block-id)) + [:span "Loading..."] + (let [block (db/entity [:block/uuid block-id]) + db-id (:db/id block) + block (when db-id (db/sub-block db-id)) + properties (:block/properties block) + block-type (keyword (pu/lookup properties :ls-type)) + hl-type (pu/lookup properties :hl-type) + repo (state/get-current-repo) + stop-inner-events? (= block-type :whiteboard-shape)] + (if (and block (:block/content block)) + (let [title [:span.block-ref + (block-content (assoc config :block-ref? true :stop-events? stop-inner-events?) + block nil (:block/uuid block) + (:slide? config) + false + (atom nil))] + inner (if label + (->elem + :span.block-ref + (map-inline config label)) + title)] + [:div.block-ref-wrap.inline + {:data-type (name (or block-type :default)) + :data-hl-type hl-type + :on-mouse-down + (fn [^js/MouseEvent e] + (if (util/right-click? e) + (state/set-state! :block-ref/context {:block (:block config) + :block-ref block-id}) + (when (and + (or (gobj/get e "shiftKey") + (not (.. e -target (closest ".blank")))) + (not (util/right-click? e))) + (util/stop e) - (cond - (gobj/get e "shiftKey") - (state/sidebar-add-block! - (state/get-current-repo) - (:db/id block) - :block-ref) + (cond + (gobj/get e "shiftKey") + (state/sidebar-add-block! + (state/get-current-repo) + (:db/id block) + :block-ref) - (and (util/meta-key? e) (whiteboard-handler/inside-portal? (.-target e))) - (whiteboard-handler/add-new-block-portal-shape! - (:block/uuid block) - (whiteboard-handler/closest-shape (.-target e))) + (and (util/meta-key? e) (whiteboard-handler/inside-portal? (.-target e))) + (whiteboard-handler/add-new-block-portal-shape! + (:block/uuid block) + (whiteboard-handler/closest-shape (.-target e))) - :else - (match [block-type (util/electron?)] + :else + (match [block-type (util/electron?)] ;; pdf annotation - [:annotation true] (pdf-assets/open-block-ref! block) + [:annotation true] (pdf-assets/open-block-ref! block) - [:whiteboard-shape true] (route-handler/redirect-to-whiteboard! - (get-in block [:block/page :block/name]) {:block-id block-id}) + [:whiteboard-shape true] (route-handler/redirect-to-whiteboard! + (get-in block [:block/page :block/name]) {:block-id block-id}) ;; default open block page - :else (route-handler/redirect-to-page! id))))))} + :else (route-handler/redirect-to-page! id))))))} - (if (and (not (util/mobile?)) - (not (:preview? config)) - (not (:modal/show? @state/state)) - (nil? block-type)) - (ui/tippy {:html (fn [] - [:div.tippy-wrapper.overflow-y-auto.p-4 - {:style {:width 735 - :text-align "left" - :max-height 600}} - [(breadcrumb config repo block-id {:indent? true}) - (blocks-container - (db/get-block-and-children repo block-id) - (assoc config :id (str id) :preview? true))]]) - :interactive true - :in-editor? true - :delay [1000, 100]} inner) - inner)]) - [:span.warning.mr-1 {:title "Block ref invalid"} - (block-ref/->block-ref id)])))) + (if (and (not (util/mobile?)) + (not (:preview? config)) + (not (:modal/show? @state/state)) + (nil? block-type)) + (ui/tippy {:html (fn [] + [:div.tippy-wrapper.overflow-y-auto.p-4 + {:style {:width 735 + :text-align "left" + :max-height 600}} + [(breadcrumb config repo block-id {:indent? true}) + (blocks-container + (db/get-block-and-children repo block-id) + (assoc config :id (str id) :preview? true))]]) + :interactive true + :in-editor? true + :delay [1000, 100]} inner) + inner)]) + [:span.warning.mr-1 {:title "Block ref invalid"} + (block-ref/->block-ref id)]))) [:span.warning.mr-1 {:title "Block ref invalid"} (block-ref/->block-ref id)])) @@ -2884,21 +2898,21 @@ (defn- get-hidden-atom [sub-id *ref {:keys [initial-value]}] (let [*latest-value (atom nil) - *hidden? (rum/derived-atom [(:ui/main-container-scroll-top @state/state)] [::lazy-display sub-id] - (fn [_top] - (if (false? @*latest-value) - @*latest-value - (let [value (cond - (some? initial-value) - initial-value + *hidden? (r/cached-derived-atom (:ui/main-container-scroll-top @state/state) [::lazy-display sub-id] + (fn [_top] + (if (false? @*latest-value) + @*latest-value + (let [value (cond + (some? initial-value) + initial-value - @*ref - (hide-block? @*ref) + @*ref + (hide-block? @*ref) - :else - true)] - (reset! *latest-value value) - value))))] + :else + true)] + (reset! *latest-value value) + value))))] *hidden?)) (rum/defcs ^:large-vars/cleanup-todo block-container-inner < rum/reactive db-mixins/query @@ -3091,11 +3105,8 @@ (rum/local false ::show-block-right-menu?) {:init (fn [state] (let [[config block] (:rum/args state) - block-id (:block/uuid block) - *loading? (atom true)] - (if (:block/content (db/entity [:block/uuid block-id])) - (reset! *loading? false) - (db-async/atom {}) + (let [^js sqlite @db-browser/*worker] (p/let [writes-finished? (when sqlite (.file-writes-finished? sqlite (state/get-current-repo))) request-finished? (ldb/request-finished?)] diff --git a/src/main/frontend/rum.cljs b/src/main/frontend/rum.cljs index b4afe69c5c..12ac35ee01 100644 --- a/src/main/frontend/rum.cljs +++ b/src/main/frontend/rum.cljs @@ -155,3 +155,12 @@ #(js/document.removeEventListener event listener capture?))) [ref]) set-ref)) + +(defonce *key->atom (atom {})) +(defn cached-derived-atom + "Make sure to return the same atom if `key` is the same." + [ref key f] + (or (get @*key->atom key) + (let [a (rum/derived-atom [ref] key f)] + (swap! *key->atom assoc key a) + a))) diff --git a/src/main/frontend/state.cljs b/src/main/frontend/state.cljs index 10d7f6f0c2..f0483ac39d 100644 --- a/src/main/frontend/state.cljs +++ b/src/main/frontend/state.cljs @@ -18,7 +18,8 @@ [logseq.common.config :as common-config] [medley.core :as medley] [promesa.core :as p] - [rum.core :as rum])) + [rum.core :as rum] + [frontend.rum :as r])) (defonce *profile-state (atom {})) @@ -307,7 +308,8 @@ :history/tx->editor-cursor (atom {}) :system/info {} ;; Whether block is selected - :ui/select-query-cache (atom {})}))) + :ui/select-query-cache (atom {}) + :restore/unloaded-blocks (atom #{})}))) ;; Block ast state ;; =============== @@ -688,8 +690,8 @@ Similar to re-frame subscriptions" (let [*cache (:ui/select-query-cache @state) keys [::select-block block-uuid] atom (or (get @*cache keys) - (let [result (rum/derived-atom - [(:selection/blocks @state)] + (let [result (r/cached-derived-atom + (:selection/blocks @state) keys (fn [s] (contains? (set (get-selected-block-ids s)) block-uuid)))] @@ -2302,11 +2304,11 @@ Similar to re-frame subscriptions" (storage/remove :user-groups)) (defn sub-block-unloaded? - [repo block-uuid] + [block-uuid] (rum/react - (rum/derived-atom [(rum/cursor-in state [repo :restore/unloaded-blocks])] [::block-unloaded repo block-uuid] - (fn [s] - (contains? s (str block-uuid)))))) + (r/cached-derived-atom (:restore/unloaded-blocks @state) [(get-current-repo) ::block-unloaded (str block-uuid)] + (fn [s] + (contains? s (str block-uuid)))))) (defn get-color-accent [] (get @state :ui/radix-color)) From 0a5ca160f97d16880f855be03e44edcab22fb2aa Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Mon, 29 Jan 2024 18:40:24 +0800 Subject: [PATCH 05/60] fix: lazy load contents --- src/main/frontend/components/page.cljs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/frontend/components/page.cljs b/src/main/frontend/components/page.cljs index 0d9c4554a8..292ad55840 100644 --- a/src/main/frontend/components/page.cljs +++ b/src/main/frontend/components/page.cljs @@ -202,11 +202,6 @@ {:page page-name})] (add-button args)))]))))) -(defn contents-page - [page] - (when-let [repo (state/get-current-repo)] - (page-blocks-cp repo page {:sidebar? true}))) - (rum/defc today-queries < rum/reactive [repo today? sidebar?] (when (and today? (not sidebar?)) @@ -576,6 +571,15 @@ (or (:page-name option) (get-page-name state)))) +(rum/defc contents-page < rum/reactive + {:init (fn [state] + (db-async/ Date: Mon, 29 Jan 2024 18:54:52 +0800 Subject: [PATCH 06/60] fix: lazy load whiteboards --- deps/db/src/logseq/db/sqlite/common_db.cljs | 33 ++++++++---------- src/main/frontend/components/whiteboard.cljs | 7 ++-- src/main/frontend/extensions/tldraw.cljs | 36 ++++++++++++-------- 3 files changed, 41 insertions(+), 35 deletions(-) diff --git a/deps/db/src/logseq/db/sqlite/common_db.cljs b/deps/db/src/logseq/db/sqlite/common_db.cljs index 84f5277849..8be78e204d 100644 --- a/deps/db/src/logseq/db/sqlite/common_db.cljs +++ b/deps/db/src/logseq/db/sqlite/common_db.cljs @@ -4,7 +4,6 @@ ["path" :as node-path] [clojure.string :as string] [logseq.db.sqlite.util :as sqlite-util] - [logseq.db :as ldb] [logseq.common.util.date-time :as date-time-util] [logseq.common.util :as common-util])) @@ -26,26 +25,24 @@ (let [get-children (fn [col] (map (fn [e] (select-keys e [:db/id :block/uuid :block/page :block/left :block/parent])) - col))] - (if (common-util/uuid-string? name) ; block - (let [id (uuid name) - block (d/entity db [:block/uuid id])] - (when block - (let [result {:block (d/pull db '[*] (:db/id block))}] - (if children? - (let [children (->> (ldb/get-block-children-ids db id) - (map #(d/entity db [:block/uuid %])))] - (assoc result :children (get-children children))) - result)) - (cond-> - {:block (d/pull db '[*] (:db/id block))} - children? - (assoc :children (get-children (:block/_parent block)))))) - (when-let [block (d/entity db [:block/name name])] + col)) + uuid? (common-util/uuid-string? name) + block (when uuid? + (let [id (uuid name)] + (d/entity db [:block/uuid id])))] + (if (and block (not (:block/name block))) ; not a page + (cond-> + {:block (d/pull db '[*] (:db/id block))} + children? + (assoc :children (get-children (:block/_parent block)))) + (when-let [block (or block (d/entity db [:block/name name]))] (cond-> {:block (d/pull db '[*] (:db/id block))} children? - (assoc :children (get-children (:block/_page block)))))))) + (assoc :children + (if (contains? (:block/type block) "whiteboard") + (d/pull-many db '[*] (map :db/id (:block/_page block))) + (get-children (:block/_page block))))))))) (defn get-latest-journals [db n] diff --git a/src/main/frontend/components/whiteboard.cljs b/src/main/frontend/components/whiteboard.cljs index 6f6b0f5275..f0fb2723e1 100644 --- a/src/main/frontend/components/whiteboard.cljs +++ b/src/main/frontend/components/whiteboard.cljs @@ -20,7 +20,8 @@ [promesa.core :as p] [rum.core :as rum] [shadow.loader :as loader] - [frontend.config :as config])) + [frontend.config :as config] + [frontend.db.async :as db-async])) (defonce tldraw-loaded? (atom false)) (rum/defc tldraw-app < rum/reactive @@ -39,13 +40,15 @@ {:init (fn [state] (p/let [_ (loader/load :tldraw)] (reset! tldraw-loaded? true)) + (let [page-name (first (:rum/args state))] + (db-async/tldr! page-name) generate-preview (when loaded? (resolve 'frontend.extensions.tldraw/generate-preview))] - (when generate-preview + (when (and generate-preview (not (state/sub-block-unloaded? page-name))) (generate-preview tldr)))) ;; TODO: use frontend.ui instead of making a new one diff --git a/src/main/frontend/extensions/tldraw.cljs b/src/main/frontend/extensions/tldraw.cljs index 70260ec238..9087951dc9 100644 --- a/src/main/frontend/extensions/tldraw.cljs +++ b/src/main/frontend/extensions/tldraw.cljs @@ -22,7 +22,8 @@ [promesa.core :as p] [rum.core :as rum] [frontend.ui :as ui] - [frontend.components.whiteboard :as whiteboard])) + [frontend.components.whiteboard :as whiteboard] + [frontend.db.async :as db-async])) (def tldraw (r/adapt-class (gobj/get TldrawLogseq "App"))) @@ -201,21 +202,26 @@ :model data})]) (rum/defc tldraw-app-inner < rum/reactive + {:init (fn [state] + (let [page-name (first (:rum/args state))] + (db-async/tldr! page-name)] - (when data - (tldraw-inner page-name data populate-onboarding? loaded-app on-mount)))) + (when-not (state/sub-block-unloaded? page-name) + (let [populate-onboarding? (whiteboard-handler/should-populate-onboarding-whiteboard? page-name) + on-mount (fn [^js tln] + (when tln + (set! (.-appUndo tln) undo) + (set! (.-appRedo tln) redo) + (when-let [^js api (gobj/get tln "api")] + (p/then (when populate-onboarding? + (whiteboard-handler/populate-onboarding-whiteboard api)) + #(do (whiteboard-handler/cleanup! (.-currentPage tln)) + (state/focus-whiteboard-shape tln block-id) + (set-loaded-app tln)))))) + data (whiteboard-handler/page-name->tldr! page-name)] + (when data + (tldraw-inner page-name data populate-onboarding? loaded-app on-mount))))) (rum/defc tldraw-app [page-name block-id] From 059e8a89485b87111b284b7170e222c08ba7c939 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Mon, 29 Jan 2024 18:56:30 +0800 Subject: [PATCH 07/60] Remove block loading placeholder --- src/main/frontend/components/block.cljs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index bdb2c8bfe4..864ebfab04 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -3132,17 +3132,7 @@ *navigating-block (get state ::navigating-block) navigating-block (rum/react *navigating-block) navigated? (and (not= (:block/uuid block) navigating-block) navigating-block)] - (cond - (state/sub-block-unloaded? (:block/uuid block)) - [:div.ls-block.flex-1.flex-col.rounded-sm {:style {:width "100%"}} - [:div.flex.flex-row - [:div.flex.flex-row.items-center.mr-2.ml-1 {:style {:height 24}} - [:span.bullet-container.cursor - [:span.bullet]]] - [:div.flex.flex-1 - ""]]] - - :else + (when-not (state/sub-block-unloaded? (:block/uuid block)) (let [[original-block block] (build-block config block {:navigating-block navigating-block :navigated? navigated?}) config' (if original-block (assoc config :original-block original-block) From 26d46485a0dcfad7cdac361bab9a4bfdf592bfd1 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Mon, 29 Jan 2024 21:56:54 +0800 Subject: [PATCH 08/60] wip: async custom query --- src/main/frontend/db/async/util.cljs | 11 +- src/main/frontend/db/model.cljs | 21 +--- src/main/frontend/db/react.cljs | 124 ++++++++++------------ src/main/frontend/db_worker.cljs | 2 +- src/main/frontend/persist_db/browser.cljs | 2 +- src/main/frontend/state.cljs | 2 + 6 files changed, 67 insertions(+), 95 deletions(-) diff --git a/src/main/frontend/db/async/util.cljs b/src/main/frontend/db/async/util.cljs index 35a2cb1c91..19ae608857 100644 --- a/src/main/frontend/db/async/util.cljs +++ b/src/main/frontend/db/async/util.cljs @@ -1,12 +1,13 @@ (ns frontend.db.async.util "Async util helper" - (:require [frontend.persist-db.browser :as db-browser] - [cljs-bean.core :as bean] - [promesa.core :as p])) + (:require [frontend.state :as state] + [promesa.core :as p] + [clojure.edn :as edn])) (defn clj result)))) + (when result + (edn/read-string result))))) diff --git a/src/main/frontend/db/model.cljs b/src/main/frontend/db/model.cljs index 84ecb40446..ac5915df12 100644 --- a/src/main/frontend/db/model.cljs +++ b/src/main/frontend/db/model.cljs @@ -336,25 +336,8 @@ independent of format as format specific heading characters are stripped" (-> (react/q repo [:frontend.worker.react/block id] {:query-fn (fn [_] - (let [e (db-utils/entity id) - children (map :db/id (sort-by-left (:block/_parent e) e))] - [e {:name (:block/name e) - :original-name (:block/original-name e) - :link (:block/link e) - :namespace (:block/namespace e) - :types (:block/type e) - :schema (:block/schema e) - :content (:block/content e) - :marker (:block/marker e) - :priority (:block/priority e) - :properties (:block/properties e) - :properties-values (:block/properties-text-values e) - :alias (:block/alias e) - :tags (:block/tags e) - :children children - :collapsed? (:block/collapsed? e) - :collapsed-properties (:block/collapsed-properties e) - :refs-count (count (:block/_refs e))}]))} + (let [e (db-utils/entity id)] + [e (:block/tx-id e)]))} nil) react first))) diff --git a/src/main/frontend/db/react.cljs b/src/main/frontend/db/react.cljs index a4aeb5690d..fd67b7bc1d 100644 --- a/src/main/frontend/db/react.cljs +++ b/src/main/frontend/db/react.cljs @@ -4,13 +4,14 @@ It'll be great if we can find an automatically resolving and performant solution. " - (:require [datascript.core :as d] - [frontend.date :as date] + (:require [frontend.date :as date] [frontend.db.conn :as conn] [frontend.db.utils :as db-utils] [frontend.state :as state] [frontend.util :as util :refer [react]] - [clojure.core.async :as async])) + [clojure.core.async :as async] + [frontend.db.async.util :as db-async-util] + [promesa.core :as p])) ;; Query atom of map of Key ([repo q inputs]) -> atom ;; TODO: replace with LRUCache, only keep the latest 20 or 50 items? @@ -43,15 +44,13 @@ (reset! query-state {})) (defn add-q! - [k query time inputs result-atom transform-fn query-fn inputs-fn] - (let [time' (int (util/safe-parse-float time))] ;; for robustness. `time` should already be float - (swap! query-state assoc k {:query query - :query-time time' - :inputs inputs - :result result-atom - :transform-fn transform-fn - :query-fn query-fn - :inputs-fn inputs-fn})) + [k query inputs result-atom transform-fn query-fn inputs-fn] + (swap! query-state assoc k {:query query + :inputs inputs + :result result-atom + :transform-fn transform-fn + :query-fn query-fn + :inputs-fn inputs-fn}) result-atom) (defn remove-q! @@ -85,16 +84,36 @@ (when-let [result (get @query-state k)] (when (satisfies? IWithMeta @(:result result)) (set! (.-state (:result result)) - (with-meta @(:result result) {:query-time (:query-time result)}))) + @(:result result))) (:result result))) +(defn- (cond - query-fn - (query-fn db nil) - - inputs-fn - (let [inputs (inputs-fn)] - (apply d/q query db inputs)) - - kv? - (db-utils/entity db (last k)) - - (seq inputs) - (apply d/q query db inputs) - - :else - (d/q query db)) - transform-fn)) - result-atom (or result-atom (atom nil))] - ;; Don't notify watches now - (set! (.-state result-atom) result) - (if disable-reactive? - result-atom - (add-q! k query time inputs result-atom transform-fn query-fn inputs-fn)))))))) + (let [result-atom (or result-atom (atom nil)) + p-or-value ( - (cond - query-fn - (let [result (query-fn db result)] - (if (coll? result) - (doall result) - result)) - - inputs-fn - (let [inputs (inputs-fn)] - (apply d/q query db inputs)) - - (keyword? query) - (db-utils/get-key-value graph query) - - (seq inputs) - (apply d/q query db inputs) - - :else - (d/q query db)) - transform-fn)] - (when-not (= new-result result) - (set-new-result! k new-result))))) + [graph db k {:keys [query inputs transform-fn query-fn inputs-fn result] + :or {transform-fn identity}}] + (p/let [p-or-value (js result))))) + (pr-str result))))) (get-block-and-children [_this repo name children?] diff --git a/src/main/frontend/persist_db/browser.cljs b/src/main/frontend/persist_db/browser.cljs index c0841ea3f6..eb02ba7dc4 100644 --- a/src/main/frontend/persist_db/browser.cljs +++ b/src/main/frontend/persist_db/browser.cljs @@ -15,7 +15,7 @@ [logseq.db :as ldb] [frontend.date :as date])) -(defonce *worker (atom nil)) +(defonce *worker state/*db-worker) (defn- ask-persist-permission! [] diff --git a/src/main/frontend/state.cljs b/src/main/frontend/state.cljs index f0483ac39d..0e3e56ebe5 100644 --- a/src/main/frontend/state.cljs +++ b/src/main/frontend/state.cljs @@ -26,6 +26,8 @@ (defonce *editor-editing-ref (atom nil)) +(defonce *db-worker (atom nil)) + ;; Stores main application state (defonce ^:large-vars/data-var state (let [document-mode? (or (storage/get :document/mode?) false) From 3fd30736b5aef70a28548209255ebf431577c95b Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Tue, 30 Jan 2024 15:29:21 +0800 Subject: [PATCH 09/60] fix: can't open block on cmdk --- src/main/frontend/components/cmdk.cljs | 33 +++++++++++---------- src/main/frontend/db/model.cljs | 2 ++ src/main/frontend/external/roam_export.cljs | 1 + src/main/frontend/worker/export.cljs | 1 + 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/main/frontend/components/cmdk.cljs b/src/main/frontend/components/cmdk.cljs index 92ff44cd49..a9d75a43e7 100644 --- a/src/main/frontend/components/cmdk.cljs +++ b/src/main/frontend/components/cmdk.cljs @@ -29,7 +29,8 @@ [logseq.common.path :as path] [electron.ipc :as ipc] [frontend.util.text :as text-util] - [goog.userAgent])) + [goog.userAgent] + [frontend.db.async :as db-async])) (defn translate [t {:keys [id desc]}] (when id @@ -414,20 +415,22 @@ (state/close-modal!))) (defmethod handle-action :open-block [_ state _event] - (let [block-id (some-> state state->highlighted-item :source-block :block/uuid) - get-block-page (partial model/get-block-page (state/get-current-repo)) - block (db/entity [:block/uuid block-id])] - (when block - (when-let [page (some-> block-id get-block-page)] - (let [page-name (:block/name page)] - (cond - (= (:block/type page) "whiteboard") - (route-handler/redirect-to-whiteboard! page-name {:block-id block-id}) - (model/parents-collapsed? (state/get-current-repo) block-id) - (route-handler/redirect-to-page! (:block/uuid block)) - :else - (route-handler/redirect-to-page! page-name {:anchor (str "ls-block-" block-id)}))) - (state/close-modal!))))) + (when-let [block-id (some-> state state->highlighted-item :source-block :block/uuid)] + (p/let [repo (state/get-current-repo) + _ (db-async/ block-id get-block-page)] + (let [page-name (:block/name page)] + (cond + (= (:block/type page) "whiteboard") + (route-handler/redirect-to-whiteboard! page-name {:block-id block-id}) + (model/parents-collapsed? (state/get-current-repo) block-id) + (route-handler/redirect-to-page! block-id) + :else + (route-handler/redirect-to-page! page-name {:anchor (str "ls-block-" block-id)}))) + (state/close-modal!))))))) (defmethod handle-action :open-page-right [_ state _event] (when-let [page-name (get-highlighted-page-name state)] diff --git a/src/main/frontend/db/model.cljs b/src/main/frontend/db/model.cljs index ac5915df12..7502d8c587 100644 --- a/src/main/frontend/db/model.cljs +++ b/src/main/frontend/db/model.cljs @@ -946,6 +946,7 @@ independent of format as format specific heading characters are stripped" (sort-by :block/name) (first)))) +;; TODO: async query (defn get-all-referenced-blocks-uuid "Get all uuids of blocks with any back link exists." [] @@ -982,6 +983,7 @@ independent of format as format specific heading characters are stripped" :file/content content}] (db-utils/transact! repo [tx-data] (merge opts {:skip-refresh? true})))))) +;; TODO: async query (defn get-pre-block [repo page-id] (-> (d/q '[:find (pull ?b [*]) diff --git a/src/main/frontend/external/roam_export.cljs b/src/main/frontend/external/roam_export.cljs index 425bf4037d..cc44116321 100644 --- a/src/main/frontend/external/roam_export.cljs +++ b/src/main/frontend/external/roam_export.cljs @@ -20,6 +20,7 @@ (->> (repeatedly 9 nano-id-char) (str/join))) +;; TODO: async (defn uuid->uid-map [] (let [db (db/get-db (state/get-current-repo))] (->> diff --git a/src/main/frontend/worker/export.cljs b/src/main/frontend/worker/export.cljs index 6f9e1179d2..04ab1fa5a6 100644 --- a/src/main/frontend/worker/export.cljs +++ b/src/main/frontend/worker/export.cljs @@ -31,6 +31,7 @@ (gp-property/valid-property-name? (str k))) properties) (into {})))))) +;; TODO: async query (defn get-all-pages "Get all pages and their children blocks." [repo db] From 1e6daf6563f957d7321a62ec7e32fd0cf5ed45fc Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Tue, 30 Jan 2024 17:10:38 +0800 Subject: [PATCH 10/60] fix: page/block refs and refs count --- deps/db/src/logseq/db.cljs | 36 +++++ src/main/frontend/components/block.cljs | 26 ++-- src/main/frontend/components/page.cljs | 18 +-- .../frontend/components/property/value.cljs | 6 +- src/main/frontend/components/reference.cljs | 145 ++++++++++-------- src/main/frontend/components/whiteboard.cljs | 2 +- src/main/frontend/db/async.cljs | 22 ++- src/main/frontend/db/model.cljs | 3 +- src/main/frontend/db_worker.cljs | 10 ++ src/main/frontend/extensions/tldraw.cljs | 2 +- src/main/frontend/handler/events.cljs | 2 +- src/main/frontend/state.cljs | 12 +- 12 files changed, 184 insertions(+), 100 deletions(-) diff --git a/deps/db/src/logseq/db.cljs b/deps/db/src/logseq/db.cljs index 4ca02a5de1..4847571966 100644 --- a/deps/db/src/logseq/db.cljs +++ b/deps/db/src/logseq/db.cljs @@ -528,6 +528,42 @@ '[:db/id :block/name :block/original-name] ids))))) +(defn get-page-alias + [db page-id] + (->> + (d/q + '[:find [?e ...] + :in $ ?page % + :where + (alias ?page ?e)] + db + page-id + (:alias rules/rules)) + distinct)) + +(defn get-page-refs + [db id] + (let [alias (->> (get-page-alias db id) + (cons id) + distinct) + refs (->> (mapcat (fn [id] (:block/_path-refs (d/entity db id))) alias) + distinct)] + (d/pull-many db '[*] (map :db/id refs)))) + +(defn get-block-refs + [db id] + (let [block (d/entity db id)] + (if (:block/name block) + (get-page-refs db id) + (let [refs (:block/_refs (d/entity db id))] + (d/pull-many db '[*] (map :db/id refs)))))) + +(defn get-block-refs-count + [db id] + (some-> (d/entity db id) + :block/_refs + count)) + (comment (defn db-based-graph? "Whether the current graph is db-only" diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index 864ebfab04..5367c13a6b 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -816,7 +816,7 @@ (db-async/ refs-count 0)) - (let [refs-cp (state/get-component :block/linked-references)] + (when-let [refs-cp (state/get-component :block/linked-references)] (refs-cp uuid)))]))])) (rum/defc single-block-cp @@ -3132,7 +3140,7 @@ *navigating-block (get state ::navigating-block) navigating-block (rum/react *navigating-block) navigated? (and (not= (:block/uuid block) navigating-block) navigating-block)] - (when-not (state/sub-block-unloaded? (:block/uuid block)) + (when-not (state/sub-async-query-loading (:block/uuid block)) (let [[original-block block] (build-block config block {:navigating-block navigating-block :navigated? navigated?}) config' (if original-block (assoc config :original-block original-block) diff --git a/src/main/frontend/components/page.cljs b/src/main/frontend/components/page.cljs index 292ad55840..88f0c62a87 100644 --- a/src/main/frontend/components/page.cljs +++ b/src/main/frontend/components/page.cljs @@ -457,10 +457,9 @@ (let [page-name (:page-name (first (:rum/args state))) page-name' (get-sanity-page-name state page-name)] (db-async/ (db/get-page-referenced-blocks (:block/name e)) - db-utils/group-by-page) - (db/get-block-referenced-blocks block-id)) - ref-hiccup (block/->hiccup ref-blocks - {:id (str block-id) - :ref? true - :breadcrumb-show? true - :group-by-page? true - :editor-box editor/box} - {})] - [:div.references-blocks - (content/content block-id - {:hiccup ref-hiccup})])) + (when-let [e (db/entity [:block/uuid block-id])] + (when-not (state/sub-async-query-loading (str (:db/id e) "-refs")) + (let [page? (some? (:block/name e)) + ref-blocks (if page? + (-> (db/get-page-referenced-blocks (:block/name e)) + db-utils/group-by-page) + (db/get-block-referenced-blocks block-id))] + (when (> (count ref-blocks) 0) + (let [ref-hiccup (block/->hiccup ref-blocks + {:id (str block-id) + :ref? true + :breadcrumb-show? true + :group-by-page? true + :editor-box editor/box} + {})] + [:div.references-blocks + (content/content block-id + {:hiccup ref-hiccup})])))))) (rum/defc references-inner [page-name filters filtered-ref-blocks] @@ -191,59 +198,65 @@ (rum/defcs references* < rum/reactive db-mixins/query (rum/local nil ::ref-pages) {:init (fn [state] - (let [page-name (first (:rum/args state)) + (let [page-name (->> (first (:rum/args state)) + util/page-name-sanity-lc) + page (db/entity [:block/name page-name]) filters (when page-name (atom nil))] + (when page (db-async/ (group-by second filter-state) - (update-vals #(map first %)))) - filtered-ref-blocks (->> (block-handler/filter-blocks ref-blocks filters) - (block-handler/get-filtered-ref-blocks-with-parents ref-blocks)) - total (count top-level-blocks) - filtered-top-blocks (filter (fn [b] (top-level-blocks-ids (:db/id b))) filtered-ref-blocks) - filter-n (count filtered-top-blocks) - parent->blocks (group-by (fn [x] (:db/id (x :block/parent))) filtered-ref-blocks) - result (->> (group-by :block/page filtered-top-blocks) - (map (fn [[page blocks]] - (let [blocks (sort-by (fn [b] (not= (:db/id page) (:db/id (:block/parent b)))) blocks) - result (map (fn [block] - (let [filtered-children (get-filtered-children block parent->blocks) - refs (when-not (contains? top-level-blocks-ids (:db/id (:block/parent block))) - (block-handler/get-blocks-refed-pages aliases (cons block filtered-children))) - block' (assoc (tree/block-entity->map block) :block/children filtered-children)] - [block' refs])) blocks) - blocks' (map first result) - page' (if (contains? aliases-exclude-self (:db/id page)) - {:db/id (:db/id page) - :block/alias? true - :block/journal-day (:block/journal-day page)} - page)] - [[page' blocks'] (mapcat second result)])))) - filtered-ref-blocks' (map first result) - ref-pages (->> - (mapcat second result) - (map :block/original-name) - frequencies)] - (reset! *ref-pages ref-pages) - (when (or (seq filter-state) (> filter-n 0)) - [:div.references.page-linked.flex-1.flex-row - (sub-page-properties-changed page-name page-props-v filters-atom) - [:div.content.pt-6 - (references-cp page-name filters filters-atom filter-state total filter-n filtered-ref-blocks' *ref-pages)]])))) + (let [repo (state/get-current-repo) + page-name (util/page-name-sanity-lc page-name) + page-entity (db/entity repo [:block/name page-name])] + (when page-entity + (when-not (state/sub-async-query-loading (str (:db/id page-entity) "-refs")) + (let [page-props-v (state/sub-page-properties-changed page-name) + *ref-pages (::ref-pages state) + filters-atom (get state ::filters) + filter-state (rum/react filters-atom) + ref-blocks (db/get-page-referenced-blocks page-name) + page-id (:db/id page-entity) + aliases (db/page-alias-set repo page-name) + aliases-exclude-self (set (remove #{page-id} aliases)) + top-level-blocks (filter (fn [b] (some aliases (set (map :db/id (:block/refs b))))) ref-blocks) + top-level-blocks-ids (set (map :db/id top-level-blocks)) + filters (when (seq filter-state) + (-> (group-by second filter-state) + (update-vals #(map first %)))) + filtered-ref-blocks (->> (block-handler/filter-blocks ref-blocks filters) + (block-handler/get-filtered-ref-blocks-with-parents ref-blocks)) + total (count top-level-blocks) + filtered-top-blocks (filter (fn [b] (top-level-blocks-ids (:db/id b))) filtered-ref-blocks) + filter-n (count filtered-top-blocks) + parent->blocks (group-by (fn [x] (:db/id (x :block/parent))) filtered-ref-blocks) + result (->> (group-by :block/page filtered-top-blocks) + (map (fn [[page blocks]] + (let [blocks (sort-by (fn [b] (not= (:db/id page) (:db/id (:block/parent b)))) blocks) + result (map (fn [block] + (let [filtered-children (get-filtered-children block parent->blocks) + refs (when-not (contains? top-level-blocks-ids (:db/id (:block/parent block))) + (block-handler/get-blocks-refed-pages aliases (cons block filtered-children))) + block' (assoc (tree/block-entity->map block) :block/children filtered-children)] + [block' refs])) blocks) + blocks' (map first result) + page' (if (contains? aliases-exclude-self (:db/id page)) + {:db/id (:db/id page) + :block/alias? true + :block/journal-day (:block/journal-day page)} + page)] + [[page' blocks'] (mapcat second result)])))) + filtered-ref-blocks' (map first result) + ref-pages (->> + (mapcat second result) + (map :block/original-name) + frequencies)] + (reset! *ref-pages ref-pages) + (when (or (seq filter-state) (> filter-n 0)) + [:div.references.page-linked.flex-1.flex-row + (sub-page-properties-changed page-name page-props-v filters-atom) + [:div.content.pt-6 + (references-cp page-name filters filters-atom filter-state total filter-n filtered-ref-blocks' *ref-pages)]]))))))) (rum/defc references [page-name] diff --git a/src/main/frontend/components/whiteboard.cljs b/src/main/frontend/components/whiteboard.cljs index f0fb2723e1..0d6d8abd34 100644 --- a/src/main/frontend/components/whiteboard.cljs +++ b/src/main/frontend/components/whiteboard.cljs @@ -48,7 +48,7 @@ tldr (whiteboard-handler/page-name->tldr! page-name) generate-preview (when loaded? (resolve 'frontend.extensions.tldraw/generate-preview))] - (when (and generate-preview (not (state/sub-block-unloaded? page-name))) + (when (and generate-preview (not (state/sub-async-query-loading page-name))) (generate-preview tldr)))) ;; TODO: use frontend.ui instead of making a new one diff --git a/src/main/frontend/db/async.cljs b/src/main/frontend/db/async.cljs index 285ed7f992..2ecf8663ac 100644 --- a/src/main/frontend/db/async.cljs +++ b/src/main/frontend/db/async.cljs @@ -120,13 +120,31 @@ :else (when-let [^Object sqlite @db-browser/*worker] - (state/update-state! :restore/unloaded-blocks (fn [s] (conj s name'))) + (state/update-state! :db/async-queries (fn [s] (conj s name'))) (p/let [result (.get-block-and-children sqlite graph name' children?) {:keys [block children] :as result'} (edn/read-string result) conn (db/get-db graph false) _ (d/transact! conn (cons block children))] - (state/update-state! :restore/unloaded-blocks (fn [s] (disj s name'))) + (state/update-state! :db/async-queries (fn [s] (disj s name'))) (react/refresh-affected-queries! graph [[:frontend.worker.react/block (:db/id block)]]) (if children? block result')))))) + +(defn > (react/q repo [:frontend.worker.react/refs page-id] - {:use-cache? false - :query-fn (fn [] + {:query-fn (fn [] (let [entities (mapcat (fn [id] (:block/_path-refs (db-utils/entity id))) pages) blocks (map (fn [e] diff --git a/src/main/frontend/db_worker.cljs b/src/main/frontend/db_worker.cljs index e49556fd1b..48a9953812 100644 --- a/src/main/frontend/db_worker.cljs +++ b/src/main/frontend/db_worker.cljs @@ -291,6 +291,16 @@ (when-let [conn (worker-state/get-datascript-conn repo)] (pr-str (sqlite-common-db/get-block-and-children @conn name children?)))) + (get-block-refs + [_this repo id] + (when-let [conn (worker-state/get-datascript-conn repo)] + (pr-str (ldb/get-block-refs @conn id)))) + + (get-block-refs-count + [_this repo id] + (when-let [conn (worker-state/get-datascript-conn repo)] + (ldb/get-block-refs-count @conn id))) + (transact [_this repo tx-data tx-meta context] (when repo (worker-state/set-db-latest-tx-time! repo)) diff --git a/src/main/frontend/extensions/tldraw.cljs b/src/main/frontend/extensions/tldraw.cljs index 9087951dc9..bbdd32c9ba 100644 --- a/src/main/frontend/extensions/tldraw.cljs +++ b/src/main/frontend/extensions/tldraw.cljs @@ -207,7 +207,7 @@ (db-async/atom {}) (let [^js sqlite @db-browser/*worker] diff --git a/src/main/frontend/state.cljs b/src/main/frontend/state.cljs index 0e3e56ebe5..db52fc82a2 100644 --- a/src/main/frontend/state.cljs +++ b/src/main/frontend/state.cljs @@ -311,7 +311,7 @@ :system/info {} ;; Whether block is selected :ui/select-query-cache (atom {}) - :restore/unloaded-blocks (atom #{})}))) + :db/async-queries (atom #{})}))) ;; Block ast state ;; =============== @@ -2305,12 +2305,12 @@ Similar to re-frame subscriptions" [] (storage/remove :user-groups)) -(defn sub-block-unloaded? - [block-uuid] +(defn sub-async-query-loading + [k] + (assert (some? k)) (rum/react - (r/cached-derived-atom (:restore/unloaded-blocks @state) [(get-current-repo) ::block-unloaded (str block-uuid)] - (fn [s] - (contains? s (str block-uuid)))))) + (r/cached-derived-atom (:db/async-queries @state) [(get-current-repo) ::async-query (str k)] + (fn [s] (contains? s (str k)))))) (defn get-color-accent [] (get @state :ui/radix-color)) From 694e9d046b117d96c841fb80773f808ed3c48f3e Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Tue, 30 Jan 2024 17:26:41 +0800 Subject: [PATCH 11/60] fix: page not full loaded --- src/main/frontend/db/async.cljs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/frontend/db/async.cljs b/src/main/frontend/db/async.cljs index 2ecf8663ac..f7d0366f97 100644 --- a/src/main/frontend/db/async.cljs +++ b/src/main/frontend/db/async.cljs @@ -115,9 +115,6 @@ (and e (not children?) (:block/tx-id e)) e - (and e children? (:block/tx-id e) (seq (:block/_parent e))) - e - :else (when-let [^Object sqlite @db-browser/*worker] (state/update-state! :db/async-queries (fn [s] (conj s name'))) From 7b754c672f454bdf05813c1b68ffaa8c566f6888 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Tue, 30 Jan 2024 18:31:06 +0800 Subject: [PATCH 12/60] fix: page unlinked refs Use search indice to search first and then filter the result by the db worker. This avoids using `d/datoms` which loads all the blocks to the memory. --- deps/db/src/logseq/db.cljs | 22 ++++++++++- src/main/frontend/components/block.cljs | 21 +++++----- src/main/frontend/components/reference.cljs | 43 +++++++++++---------- src/main/frontend/db.cljs | 2 +- src/main/frontend/db/async.cljs | 6 +++ src/main/frontend/db/async/util.cljs | 10 ++++- src/main/frontend/db/model.cljs | 41 +------------------- src/main/frontend/db_worker.cljs | 21 ++++++++-- src/main/frontend/search.cljs | 25 +++++++++++- 9 files changed, 112 insertions(+), 79 deletions(-) diff --git a/deps/db/src/logseq/db.cljs b/deps/db/src/logseq/db.cljs index 4847571966..accdb18f38 100644 --- a/deps/db/src/logseq/db.cljs +++ b/deps/db/src/logseq/db.cljs @@ -548,7 +548,8 @@ distinct) refs (->> (mapcat (fn [id] (:block/_path-refs (d/entity db id))) alias) distinct)] - (d/pull-many db '[*] (map :db/id refs)))) + (when (seq refs) + (d/pull-many db '[*] (map :db/id refs))))) (defn get-block-refs [db id] @@ -556,7 +557,8 @@ (if (:block/name block) (get-page-refs db id) (let [refs (:block/_refs (d/entity db id))] - (d/pull-many db '[*] (map :db/id refs)))))) + (when (seq refs) + (d/pull-many db '[*] (map :db/id refs))))))) (defn get-block-refs-count [db id] @@ -564,6 +566,22 @@ :block/_refs count)) +(defn get-page-unlinked-refs + "Get unlinked refs from search result" + [db page-id search-result-eids] + (let [alias (->> (get-page-alias db page-id) + (cons page-id) + set) + eids (remove + (fn [eid] + (when-let [e (d/entity db eid)] + (or (some alias (map :db/id (:block/refs e))) + (:block/link e) + (nil? (:block/content e))))) + search-result-eids)] + (when (seq eids) + (d/pull-many db '[*] eids)))) + (comment (defn db-based-graph? "Whether the current graph is db-only" diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index 5367c13a6b..90861b9865 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -3140,16 +3140,17 @@ *navigating-block (get state ::navigating-block) navigating-block (rum/react *navigating-block) navigated? (and (not= (:block/uuid block) navigating-block) navigating-block)] - (when-not (state/sub-async-query-loading (:block/uuid block)) - (let [[original-block block] (build-block config block {:navigating-block navigating-block :navigated? navigated?}) - config' (if original-block - (assoc config :original-block original-block) - config) - opts {}] - (rum/with-key - (block-container-inner state repo config' block - (merge opts {:navigating-block navigating-block :navigated? navigated?})) - (str "block-inner" (:block/uuid block))))))) + (when (:block/uuid block) + (when-not (state/sub-async-query-loading (:block/uuid block)) + (let [[original-block block] (build-block config block {:navigating-block navigating-block :navigated? navigated?}) + config' (if original-block + (assoc config :original-block original-block) + config) + opts {}] + (rum/with-key + (block-container-inner state repo config' block + (merge opts {:navigating-block navigating-block :navigated? navigated?})) + (str "block-inner" (:block/uuid block)))))))) (defn divide-lists [[f & l]] diff --git a/src/main/frontend/components/reference.cljs b/src/main/frontend/components/reference.cljs index 4c236f463f..8ed853b9fb 100644 --- a/src/main/frontend/components/reference.cljs +++ b/src/main/frontend/components/reference.cljs @@ -17,7 +17,9 @@ [frontend.util :as util] [rum.core :as rum] [frontend.modules.outliner.tree :as tree] - [frontend.db.async :as db-async])) + [frontend.db.async :as db-async] + [promesa.core :as p] + [frontend.search :as search])) (defn- frequencies-sort [references] @@ -271,27 +273,26 @@ (rum/defcs unlinked-references-aux < rum/reactive db-mixins/query - {:wrap-render - (fn [render-fn] - (fn [state] - (reset! (second (:rum/args state)) - (apply + - (for [[_ rfs] - (db/get-page-unlinked-references - (first (:rum/args state)))] - (count rfs)))) - (render-fn state)))} + {:init + (fn [state] + (let [*result (atom nil) + [page-name *n-ref] (:rum/args state)] + (p/let [result (search/get-page-unlinked-refs page-name)] + (reset! *n-ref (count result)) + (reset! *result result)) + (assoc state ::result *result)))} [state page-name _n-ref] - (let [ref-blocks (db/get-page-unlinked-references page-name)] - [:div.references-blocks - (let [ref-hiccup (block/->hiccup ref-blocks - {:id (str page-name "-unlinked-") - :ref? true - :group-by-page? true - :editor-box editor/box} - {})] - (content/content page-name - {:hiccup ref-hiccup}))])) + (let [ref-blocks (rum/react (::result state))] + (when (seq ref-blocks) + [:div.references-blocks + (let [ref-hiccup (block/->hiccup ref-blocks + {:id (str page-name "-unlinked-") + :ref? true + :group-by-page? true + :editor-box editor/box} + {})] + (content/content page-name + {:hiccup ref-hiccup}))]))) (rum/defcs unlinked-references < rum/reactive (rum/local nil ::n-ref) diff --git a/src/main/frontend/db.cljs b/src/main/frontend/db.cljs index 7ffbabb2d8..fcf616e9ce 100644 --- a/src/main/frontend/db.cljs +++ b/src/main/frontend/db.cljs @@ -39,7 +39,7 @@ get-files get-files-blocks get-files-full get-journals-length get-pages-with-file get-latest-journals get-page get-page-alias get-page-alias-names get-page-blocks-count get-page-blocks-no-cache get-page-file get-page-format get-page-properties - get-page-referenced-blocks get-page-referenced-blocks-full get-page-referenced-pages get-page-unlinked-references + get-page-referenced-blocks get-page-referenced-blocks-full get-page-referenced-pages get-all-pages get-pages-relation get-pages-that-mentioned-page get-tag-pages journal-page? page-alias-set sub-block set-file-last-modified-at! page-empty? page-exists? page-empty-or-dummy? get-alias-source-page diff --git a/src/main/frontend/db/async.cljs b/src/main/frontend/db/async.cljs index f7d0366f97..f24586081b 100644 --- a/src/main/frontend/db/async.cljs +++ b/src/main/frontend/db/async.cljs @@ -16,6 +16,7 @@ [frontend.db.react :as react])) (def > - (d/q '[:find ?e - :in $ ?page-name % - :where - [?page :block/name ?page-name] - (alias ?page ?e)] - (conn/get-db repo-url) - (util/safe-page-name-sanity-lc page) - (:alias rules/rules)) - db-utils/seq-flatten + (ldb/get-page-alias (conn/get-db repo-url) page-id) (set) (set/union #{page-id})))) @@ -863,37 +855,6 @@ independent of format as format specific heading characters are stripped" (sort-by-left-recursive) db-utils/group-by-page)))))) -(defn- pattern [name] - (re-pattern (str "(?i)(^|[^\\[#0-9a-zA-Z]|((^|[^\\[])\\[))" - (util/regex-escape name) - "($|[^0-9a-zA-Z])"))) - -(defn get-page-unlinked-references - [page] - (when-let [repo (state/get-current-repo)] - (let [page (util/safe-page-name-sanity-lc page) - page-id (:db/id (db-utils/entity [:block/name page])) - alias-names (get-page-alias-names repo page) - patterns (->> (conj alias-names page) - (map pattern)) - filter-fn (fn [datom] - (some (fn [p] - (re-find p (->> (:v datom) - (drawer/remove-logbook)))) - patterns))] - (->> (react/q repo [:frontend.worker.react/page-unlinked-refs page-id] - {:query-fn (fn [db _result] - (let [ids - (->> (d/datoms db :aevt :block/content) - (filter filter-fn) - (map :e)) - result (db-utils/pull-many repo block-attrs ids)] - (remove (fn [block] (= page-id (:db/id (:block/page block)))) result)))} - nil) - react - (sort-by-left-recursive) - db-utils/group-by-page)))) - (defn get-block-referenced-blocks ([block-uuid] (get-block-referenced-blocks block-uuid {})) diff --git a/src/main/frontend/db_worker.cljs b/src/main/frontend/db_worker.cljs index 48a9953812..a2955d4f18 100644 --- a/src/main/frontend/db_worker.cljs +++ b/src/main/frontend/db_worker.cljs @@ -281,9 +281,18 @@ (q [_this repo inputs-str] "Datascript q" (when-let [conn (worker-state/get-datascript-conn repo)] - (let [inputs (edn/read-string inputs-str)] - (let [result (apply d/q (first inputs) @conn (rest inputs))] - (pr-str result))))) + (let [inputs (edn/read-string inputs-str) + result (apply d/q (first inputs) @conn (rest inputs))] + (pr-str result)))) + + (pull-many + [_this repo selector-str ids-str] + (when-let [conn (worker-state/get-datascript-conn repo)] + (let [selector (edn/read-string selector-str) + ids (edn/read-string ids-str) + result (d/pull-many @conn selector ids)] + (let [] + (pr-str result))))) (get-block-and-children [_this repo name children?] @@ -301,6 +310,12 @@ (when-let [conn (worker-state/get-datascript-conn repo)] (ldb/get-block-refs-count @conn id))) + (get-page-unlinked-refs + [_this repo page-id search-result-eids-str] + (when-let [conn (worker-state/get-datascript-conn repo)] + (let [search-result-eids (edn/read-string search-result-eids-str)] + (pr-str (ldb/get-page-unlinked-refs @conn page-id search-result-eids))))) + (transact [_this repo tx-data tx-meta context] (when repo (worker-state/set-db-latest-tx-time! repo)) diff --git a/src/main/frontend/search.cljs b/src/main/frontend/search.cljs index f57be94b01..2d2b6183a4 100644 --- a/src/main/frontend/search.cljs +++ b/src/main/frontend/search.cljs @@ -14,7 +14,11 @@ [frontend.config :as config] [logseq.db.frontend.property :as db-property] [frontend.handler.file-based.property.util :as property-util] - [cljs-bean.core :as bean])) + [cljs-bean.core :as bean] + [frontend.db :as db] + [frontend.db.model :as db-model] + [frontend.db.utils :as db-utils] + [clojure.edn :as edn])) (def fuzzy-search fuzzy/fuzzy-search) @@ -128,3 +132,22 @@ [repo data] (when-let [engine (get-engine repo)] (protocol/transact-blocks! engine data))) + +(defn get-page-unlinked-refs + "Get matched result from search first, and then filter by worker db" + [page] + (when-let [repo (state/get-current-repo)] + (p/let [page-name (util/safe-page-name-sanity-lc page) + page (db/entity [:block/name page-name]) + alias-names (conj (set (map util/safe-page-name-sanity-lc + (db/get-page-alias-names repo page-name))) page-name) + q (string/join " " alias-names) + result (block-search repo q {:limit 100}) + eids (map (fn [b] [:block/uuid (:block/uuid b)]) result) + result (when (seq eids) + (.get-page-unlinked-refs ^Object @state/*db-worker repo (:db/id page) (pr-str eids))) + result' (when result (edn/read-string result))] + (when result' (db/transact! repo result')) + (some->> result' + db-model/sort-by-left-recursive + db-utils/group-by-page)))) From b8c9206bc4d0230564769bcca71d31e9d7c0dcf8 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Tue, 30 Jan 2024 18:33:07 +0800 Subject: [PATCH 13/60] chore: code cleanup --- src/main/frontend/db/async.cljs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/frontend/db/async.cljs b/src/main/frontend/db/async.cljs index f24586081b..43ef0d8ecc 100644 --- a/src/main/frontend/db/async.cljs +++ b/src/main/frontend/db/async.cljs @@ -16,7 +16,8 @@ [frontend.db.react :as react])) (def Date: Tue, 30 Jan 2024 18:55:18 +0800 Subject: [PATCH 14/60] chore: code cleanup --- src/main/frontend/db.cljs | 2 +- src/main/frontend/db/utils.cljs | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/main/frontend/db.cljs b/src/main/frontend/db.cljs index fcf616e9ce..7654a84f0f 100644 --- a/src/main/frontend/db.cljs +++ b/src/main/frontend/db.cljs @@ -22,7 +22,7 @@ remove-conn!] [frontend.db.utils - db->json db->edn-str db->string get-max-tx-id get-tx-id + db->edn-str db->string get-max-tx-id get-tx-id group-by-page seq-flatten string->db diff --git a/src/main/frontend/db/utils.cljs b/src/main/frontend/db/utils.cljs index 58c8c04469..bcd0406cb7 100644 --- a/src/main/frontend/db/utils.cljs +++ b/src/main/frontend/db/utils.cljs @@ -12,12 +12,6 @@ (defn db->string [db] (dt/write-transit-str db)) -(defn db->json [db] - (js/JSON.stringify - (into-array - (for [d (d/datoms db :eavt)] - #js [(:e d) (name (:a d)) (:v d)])))) - (defn db->edn-str [db] (pr-str db)) From 99b6d543f05b785d2ca2a24cf7d77a66e93b95a5 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Tue, 30 Jan 2024 20:03:06 +0800 Subject: [PATCH 15/60] fix: more async queries and export to roam --- .../graph-parser/src/logseq/graph_parser.cljs | 2 +- deps/outliner/src/logseq/outliner/core.cljs | 14 ++++---- .../src/logseq/outliner/datascript.cljs | 2 +- src/main/frontend/db.cljs | 2 +- src/main/frontend/db/async.cljs | 9 ++++++ src/main/frontend/db/model.cljs | 6 ++-- src/main/frontend/external/roam_export.cljs | 32 +++++++++---------- src/main/frontend/handler/import.cljs | 10 +++--- src/main/frontend/worker/export.cljs | 1 - 9 files changed, 45 insertions(+), 33 deletions(-) diff --git a/deps/graph-parser/src/logseq/graph_parser.cljs b/deps/graph-parser/src/logseq/graph_parser.cljs index b05700d857..678346136d 100644 --- a/deps/graph-parser/src/logseq/graph_parser.cljs +++ b/deps/graph-parser/src/logseq/graph_parser.cljs @@ -15,7 +15,7 @@ (mapcat (fn [{uuid :block/uuid eid :db/id}] (if (and uuid (contains? retain-uuids uuid)) (map (fn [attr] [:db.fn/retractAttribute eid attr]) db-schema/retract-attributes) - [[:db.fn/retractEntity eid]])) + (when eid [[:db.fn/retractEntity eid]]))) blocks)) (defn- get-file-page diff --git a/deps/outliner/src/logseq/outliner/core.cljs b/deps/outliner/src/logseq/outliner/core.cljs index 8802d23125..d72349eb20 100644 --- a/deps/outliner/src/logseq/outliner/core.cljs +++ b/deps/outliner/src/logseq/outliner/core.cljs @@ -153,7 +153,7 @@ ;; Only delete if last reference (keep #(when (<= (count (:block/_macros (d/entity db (:db/id %)))) 1) - (vector :db.fn/retractEntity (:db/id %))) + (when (:db/id %) (vector :db.fn/retractEntity (:db/id %)))) (:block/macros block-entity))))))) (comment @@ -413,11 +413,13 @@ (assert (ds/outliner-txs-state? txs-state) "db should be satisfied outliner-tx-state?") (let [block-id (otree/-get-id this conn) - ids (set (if children? - (let [children (ldb/get-block-children @conn block-id) - children-ids (map :block/uuid children)] - (conj children-ids block-id)) - [block-id])) + ids (->> + (if children? + (let [children (ldb/get-block-children @conn block-id) + children-ids (map :block/uuid children)] + (conj children-ids block-id)) + [block-id]) + (remove nil?)) txs (map (fn [id] [:db.fn/retractEntity [:block/uuid id]]) ids) txs (if-not children? (let [immediate-children (ldb/get-block-immediate-children @conn block-id)] diff --git a/deps/outliner/src/logseq/outliner/datascript.cljs b/deps/outliner/src/logseq/outliner/datascript.cljs index 7ce74ced29..e231050279 100644 --- a/deps/outliner/src/logseq/outliner/datascript.cljs +++ b/deps/outliner/src/logseq/outliner/datascript.cljs @@ -65,7 +65,7 @@ ;; Only delete if last reference (keep #(when (<= (count (:block/_macros (d/entity db (:db/id %)))) 1) - (vector :db.fn/retractEntity (:db/id %))) + (when (:db/id %) (vector :db.fn/retractEntity (:db/id %)))) (:block/macros b))) retracted-blocks)] (when (and (seq retracted-tx') (fn? set-state-fn)) diff --git a/src/main/frontend/db.cljs b/src/main/frontend/db.cljs index 7654a84f0f..1ac137df49 100644 --- a/src/main/frontend/db.cljs +++ b/src/main/frontend/db.cljs @@ -32,7 +32,7 @@ delete-blocks get-pre-block delete-files delete-pages-by-files get-all-tagged-pages get-block-and-children get-block-by-uuid get-block-children sort-by-left - get-block-parent get-block-parents parents-collapsed? get-block-referenced-blocks get-all-referenced-blocks-uuid + get-block-parent get-block-parents parents-collapsed? get-block-referenced-blocks get-block-immediate-children get-block-page get-custom-css get-date-scheduled-or-deadlines get-file-last-modified-at get-file get-file-page get-file-page-id file-exists? diff --git a/src/main/frontend/db/async.cljs b/src/main/frontend/db/async.cljs index 43ef0d8ecc..1cbed81186 100644 --- a/src/main/frontend/db/async.cljs +++ b/src/main/frontend/db/async.cljs @@ -147,3 +147,12 @@ (assert (integer? eid)) (when-let [^Object worker @db-browser/*worker] (.get-block-refs-count worker graph eid))) + +(defn > (get-files-blocks repo-url files) + (remove nil?))] (mapv (fn [eid] [:db.fn/retractEntity eid]) blocks)))) (defn delete-files @@ -943,7 +943,7 @@ independent of format as format specific heading characters are stripped" :file/content content}] (db-utils/transact! repo [tx-data] (merge opts {:skip-refresh? true})))))) -;; TODO: async query +;; TODO: check whether this works when adding pdf back on Web (defn get-pre-block [repo page-id] (-> (d/q '[:find (pull ?b [*]) diff --git a/src/main/frontend/external/roam_export.cljs b/src/main/frontend/external/roam_export.cljs index cc44116321..1f90497294 100644 --- a/src/main/frontend/external/roam_export.cljs +++ b/src/main/frontend/external/roam_export.cljs @@ -2,9 +2,9 @@ (:require [clojure.set :as s] [clojure.string :as str] [clojure.walk :as walk] - [datascript.core :as d] - [frontend.db :as db] - [frontend.state :as state])) + [frontend.db.async :as db-async] + [frontend.state :as state] + [promesa.core :as p])) (def todo-marker-regex #"^(NOW|LATER|TODO|DOING|WAITING|WAIT|CANCELED|CANCELLED|STARTED|IN-PROGRESS)") @@ -20,18 +20,18 @@ (->> (repeatedly 9 nano-id-char) (str/join))) -;; TODO: async -(defn uuid->uid-map [] - (let [db (db/get-db (state/get-current-repo))] - (->> - (d/q '[:find (pull ?r [:block/uuid]) - :in $ - :where - [?b :block/refs ?r]] db) - (map (comp :block/uuid first)) - (distinct) - (map (fn [uuid] [uuid (nano-id)])) - (into {})))) +(defn uid-map [] + (let [repo (state/get-current-repo)] + (p/let [result (db-async/> result + (map (comp :block/uuid first)) + (distinct) + (map (fn [uuid] [uuid (nano-id)])) + (into {}))))) (defn update-content [content uuid->uid-map] (when content ; page block doesn't have content @@ -66,7 +66,7 @@ (defn traverse [keyseq vec-tree] - (let [uuid->uid-map (uuid->uid-map)] + (p/let [uuid->uid-map (uid-map)] (walk/postwalk (fn [x] (cond diff --git a/src/main/frontend/handler/import.cljs b/src/main/frontend/handler/import.cljs index 46ef191d29..a93e36a4ef 100644 --- a/src/main/frontend/handler/import.cljs +++ b/src/main/frontend/handler/import.cljs @@ -21,9 +21,11 @@ [frontend.handler.notification :as notification] [frontend.util :as util] [clojure.core.async :as async] + [cljs.core.async.interop :refer [p->c]] [medley.core :as medley] [frontend.persist-db :as persist-db] - [promesa.core :as p])) + [promesa.core :as p] + [frontend.db.async :as db-async])) (defn index-files! "Create file structure, then parse into DB (client only)" @@ -194,9 +196,9 @@ (async/c (db-async/ Date: Tue, 30 Jan 2024 20:42:53 +0800 Subject: [PATCH 16/60] fix: wrong kv? check --- src/main/frontend/db/react.cljs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/frontend/db/react.cljs b/src/main/frontend/db/react.cljs index fd67b7bc1d..e4f18490b7 100644 --- a/src/main/frontend/db/react.cljs +++ b/src/main/frontend/db/react.cljs @@ -89,7 +89,7 @@ (defn- Date: Tue, 30 Jan 2024 21:17:18 +0800 Subject: [PATCH 17/60] fix: async > (d/datoms db :avet :file/path) + (map (fn [e] + {:db/id (:e e) + :file/path (:v e)})))) + (defn get-block-and-children [db name children?] (let [get-children (fn [col] diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index 90861b9865..1f64e5ba1b 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -813,7 +813,7 @@ (rum/defc block-embed < rum/reactive {:init (fn [state] (let [block-id (second (:rum/args state))] - (db-async/ state state->highlighted-item :source-block :block/uuid)] (p/let [repo (state/get-current-repo) - _ (db-async/> (d/q - '[:find ?path ?modified-at - :where - [?file :file/path ?path] - [(get-else $ ?file :file/last-modified-at 0) ?modified-at]] - db) - (seq) - (reverse)))) - (defn get-files-blocks [repo-url paths] (let [paths (set paths) diff --git a/src/main/frontend/db_worker.cljs b/src/main/frontend/db_worker.cljs index a2955d4f18..cd615eae24 100644 --- a/src/main/frontend/db_worker.cljs +++ b/src/main/frontend/db_worker.cljs @@ -285,6 +285,14 @@ result (apply d/q (first inputs) @conn (rest inputs))] (pr-str result)))) + (pull + [_this repo selector-str id-str] + (when-let [conn (worker-state/get-datascript-conn repo)] + (let [selector (edn/read-string selector-str) + id (edn/read-string id-str) + result (d/pull @conn selector id)] + (pr-str result)))) + (pull-many [_this repo selector-str ids-str] (when-let [conn (worker-state/get-datascript-conn repo)] @@ -366,10 +374,11 @@ (let [data (->> (sqlite-common-db/get-initial-data @conn) pr-str)] (async/go - (let [all-pages (sqlite-common-db/get-all-pages @conn)] + (let [all-pages (sqlite-common-db/get-all-pages @conn) + all-files (sqlite-common-db/get-all-files @conn)] (worker-util/post-message :sync-db-changes (pr-str {:repo repo - :tx-data all-pages + :tx-data (concat all-files all-pages) :tx-meta nil})))) data))) diff --git a/src/main/frontend/extensions/tldraw.cljs b/src/main/frontend/extensions/tldraw.cljs index bbdd32c9ba..32a07b40b2 100644 --- a/src/main/frontend/extensions/tldraw.cljs +++ b/src/main/frontend/extensions/tldraw.cljs @@ -204,7 +204,7 @@ (rum/defc tldraw-app-inner < rum/reactive {:init (fn [state] (let [page-name (first (:rum/args state))] - (db-async/> (db/get-files graph) - (map first)) exclude-files (set (or exclude-files []))] ;; read all files in the repo dir, notify if readdir error - (p/let [[files deleted-files] + (p/let [db-files' (db-async/ (fs/readdir repo-dir :path-only? true) (p/chain (fn [files] (->> files diff --git a/src/main/frontend/handler.cljs b/src/main/frontend/handler.cljs index 92773840f3..3caafc96e4 100644 --- a/src/main/frontend/handler.cljs +++ b/src/main/frontend/handler.cljs @@ -11,7 +11,6 @@ [frontend.components.whiteboard :as whiteboard] [frontend.config :as config] [frontend.context.i18n :as i18n] - [frontend.db :as db] [frontend.db.restore :as db-restore] [frontend.db.conn :as conn] [frontend.db.react :as react] @@ -41,7 +40,8 @@ [frontend.mobile.core :as mobile] [cljs-bean.core :as bean] [frontend.handler.test :as test] - [frontend.persist-db.browser :as db-browser])) + [frontend.persist-db.browser :as db-browser] + [frontend.db.async :as db-async])) (defn- set-global-error-notification! [] @@ -87,17 +87,18 @@ ;; install after config is restored (shortcut/refresh!) - (cond - (and (not (seq (db/get-files config/demo-repo))) - ;; Not native local directory - (not (some config/local-file-based-graph? (map :url repos))) - (not (mobile-util/native-platform?)) - (not (config/db-based-graph? repo))) - ;; will execute `(state/set-db-restoring! false)` inside - (repo-handler/setup-demo-repo-if-not-exists!) + (p/let [files (db-async/ Date: Wed, 31 Jan 2024 03:05:45 +0800 Subject: [PATCH 18/60] partition load initial pages data Also, add a script to build a test graph --- src/dev-cljs/shadow/build_large_graph.cljs | 59 ++++++++++++++ src/main/frontend/db_worker.cljs | 33 ++++---- .../frontend/modules/outliner/pipeline.cljs | 80 ++++++++++--------- 3 files changed, 117 insertions(+), 55 deletions(-) create mode 100644 src/dev-cljs/shadow/build_large_graph.cljs diff --git a/src/dev-cljs/shadow/build_large_graph.cljs b/src/dev-cljs/shadow/build_large_graph.cljs new file mode 100644 index 0000000000..9b8b026660 --- /dev/null +++ b/src/dev-cljs/shadow/build_large_graph.cljs @@ -0,0 +1,59 @@ +(ns shadow.build-large-graph) + +(comment + + (in-ns 'frontend.db-worker) + (def repo "logseq_db_large-db-demo") + (def conn (worker-state/get-datascript-conn repo)) + + (defonce *ids (atom (set (map :v (d/datoms @conn :avet :block/uuid))))) + (defn get-next-id + [] + (let [id (random-uuid)] + (if (@*ids id) + (get-next-id) + (do + (swap! *ids conj id) + id)))) + + (defn pages + [start-idx n] + (let [ids (repeatedly n get-next-id)] + (map-indexed + (fn [idx id] + {:block/uuid id + :block/original-name (str "page-" (+ start-idx idx)) + :block/name (str "page-" (+ start-idx idx)) + :block/format :markdown}) + ids))) + + (defn blocks + [page-id size] + (let [page-id [:block/uuid page-id] + blocks (vec (repeatedly size (fn [] + (let [id (get-next-id)] + {:block/uuid id + :block/content (str id) + :block/format :markdown + :block/page page-id + :block/parent page-id}))))] + (map-indexed + (fn [i b] + (if (zero? i) + (assoc b :block/left page-id) + (let [left (nth blocks (dec i))] + (assoc b :block/left [:block/uuid (:block/uuid left)])))) + blocks))) + + (defn create-graph! + [conn page-size blocks-size start-idx] + (let [pages (pages start-idx page-size) + page-blocks (map (fn [p] + (cons p + (blocks (:block/uuid p) blocks-size))) pages)] + (doseq [data (partition-all 1000 page-blocks)] + (let [tx-data (apply concat data)] + (prn :debug :progressing (:block/name (first tx-data))) + (d/transact! conn tx-data {:new-graph? true}))))) + + (create-graph! conn 10000 20 0)) diff --git a/src/main/frontend/db_worker.cljs b/src/main/frontend/db_worker.cljs index cd615eae24..a29b30de91 100644 --- a/src/main/frontend/db_worker.cljs +++ b/src/main/frontend/db_worker.cljs @@ -30,7 +30,6 @@ (defonce *datascript-conns worker-state/*datascript-conns) (defonce *opfs-pools worker-state/*opfs-pools) (defonce *publishing? (atom false)) -(defonce *store-jobs (atom #{})) (defn- get-pool-name [graph-name] @@ -108,11 +107,8 @@ (fn [[addr data]] #js {:$addr addr :$content (pr-str data)}) - addr+data-seq) - p (p/do! (upsert-addr-content! repo data delete-addrs))] - (swap! *store-jobs conj p) - (p/then p (fn [] (swap! *store-jobs disj p))) - p)) + addr+data-seq)] + (upsert-addr-content! repo data delete-addrs))) (-restore [_ addr] (restore-data-from-addr repo addr)))) @@ -263,13 +259,7 @@ [_this repo & {:keys [close-other-db?] :or {close-other-db? true}}] (p/do! - ;; Store the current db if store jobs not finished yet - (when (seq @*store-jobs) - (-> (p/all @*store-jobs) - (p/then (fn [_] - (reset! *store-jobs #{}) - (println "DB store job finished"))))) - (when close-other-db? + (when close-other-db? (close-other-dbs! repo)) (create-or-open-db! repo))) @@ -374,12 +364,19 @@ (let [data (->> (sqlite-common-db/get-initial-data @conn) pr-str)] (async/go + ;; TODO: after UI db transacted + (async/ Date: Wed, 31 Jan 2024 03:26:13 +0800 Subject: [PATCH 19/60] Raise it to 30k pages --- src/dev-cljs/shadow/build_large_graph.cljs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dev-cljs/shadow/build_large_graph.cljs b/src/dev-cljs/shadow/build_large_graph.cljs index 9b8b026660..9dcf295e62 100644 --- a/src/dev-cljs/shadow/build_large_graph.cljs +++ b/src/dev-cljs/shadow/build_large_graph.cljs @@ -56,4 +56,4 @@ (prn :debug :progressing (:block/name (first tx-data))) (d/transact! conn tx-data {:new-graph? true}))))) - (create-graph! conn 10000 20 0)) + (create-graph! conn 30000 20 0)) From 44dca4f506c39f023372a10f61eb17d0e14231cb Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Wed, 31 Jan 2024 21:51:52 +0800 Subject: [PATCH 20/60] wip: plugin api calls Also avoid calls to worker if a block has been fully loaded --- deps/outliner/src/logseq/outliner/core.cljs | 2 +- src/main/frontend/components/page.cljs | 20 +-- src/main/frontend/components/page_menu.cljs | 2 +- src/main/frontend/db/async.cljs | 20 +-- src/main/frontend/db/model.cljs | 1 - src/main/frontend/db_worker.cljs | 10 +- .../frontend/extensions/zotero/handler.cljs | 2 +- src/main/frontend/fs/watcher_handler.cljs | 2 +- src/main/frontend/handler/common/page.cljs | 15 ++- src/main/frontend/handler/page.cljs | 2 +- src/main/frontend/worker/rtc/core.cljs | 2 +- src/main/logseq/api.cljs | 124 +++++++++++------- 12 files changed, 118 insertions(+), 84 deletions(-) diff --git a/deps/outliner/src/logseq/outliner/core.cljs b/deps/outliner/src/logseq/outliner/core.cljs index d72349eb20..a47ddb09fb 100644 --- a/deps/outliner/src/logseq/outliner/core.cljs +++ b/deps/outliner/src/logseq/outliner/core.cljs @@ -341,7 +341,7 @@ data) m (-> data' (dissoc :block/children :block/meta :block.temp/top? :block.temp/bottom? - :block/title :block/body :block/level) + :block/title :block/body :block/level :block.temp/fully-loaded?) common-util/remove-nils block-with-updated-at fix-tag-ids) diff --git a/src/main/frontend/components/page.cljs b/src/main/frontend/components/page.cljs index 1caf2844d2..8a71a60a47 100644 --- a/src/main/frontend/components/page.cljs +++ b/src/main/frontend/components/page.cljs @@ -960,16 +960,16 @@ :on-click (fn [] (close-fn) (let [failed-pages (atom [])] - (doseq [page-name (map :block/name pages)] - (page-handler/delete! page-name #() - {:error-handler - (fn [msg] - (js/console.error msg) - (swap! failed-pages conj page-name))})) - (if (seq @failed-pages) - (notification/show! (t :all-pages/failed-to-delete-pages (string/join ", " (map pr-str @failed-pages))) - :warning false) - (notification/show! (t :tips/all-done) :success))) + (p/let [_ (p/all (map (fn [page-name] + (page-handler/js {:result result})))) + (page-delete + [this repo page-name] + (when-let [conn (worker-state/get-datascript-conn repo)] + (let [config (worker-state/get-config repo) + result (worker-page/delete! repo conn page-name nil {})] + (bean/->js {:result result})))) + (file-writes-finished? [this repo] (let [conn (worker-state/get-datascript-conn repo) diff --git a/src/main/frontend/extensions/zotero/handler.cljs b/src/main/frontend/extensions/zotero/handler.cljs index 8fb7ef834c..af1b2ea280 100644 --- a/src/main/frontend/extensions/zotero/handler.cljs +++ b/src/main/frontend/extensions/zotero/handler.cljs @@ -79,7 +79,7 @@ (when-not (str/blank? page-name) (if (db/page-exists? (str/lower-case page-name)) (if (setting/setting :overwrite-mode?) - (page-handler/delete! + (page-handler/ (p/let [repo (state/get-current-repo) + _ (.page-delete worker repo page-name)] + (when ok-handler (ok-handler))) + (p/catch (fn [error] + (when error-handler (error-handler error)))))))) ;; other fns ;; ========= diff --git a/src/main/frontend/handler/page.cljs b/src/main/frontend/handler/page.cljs index c04970175b..5dfc3b91c0 100644 --- a/src/main/frontend/handler/page.cljs +++ b/src/main/frontend/handler/page.cljs @@ -45,7 +45,7 @@ (def create! page-common-handler/create!) (def templates - (update-vals db/pull) (sdk-utils/normalize-keyword-for-json) (bean/->js)))))) @@ -538,16 +547,21 @@ (def ^:export get_current_page (fn [] (when-let [page (state/get-current-page)] - (when-let [page (db-model/get-page page)] + (p/let [page (js (sdk-utils/normalize-keyword-for-json (db-utils/pull (:db/id page)))))))) (def ^:export get_page (fn [id-or-page-name] - (when-let [page (cond - (number? id-or-page-name) (db-utils/pull id-or-page-name) - (string? id-or-page-name) (db-model/get-page id-or-page-name))] - (when-not (contains? page :block/left) - (bean/->js (sdk-utils/normalize-keyword-for-json (db-utils/pull (:db/id page)))))))) + (p/let [page (db-async/js (sdk-utils/normalize-keyword-for-json page)))))) (def ^:export get_all_pages (fn [repo] @@ -558,7 +572,7 @@ (fn [name ^js properties ^js opts] (let [properties (bean/->clj properties) {:keys [redirect createFirstBlock format journal]} (bean/->clj opts)] - (p/let [page (db-model/get-page name) + (p/let [page (clj opts) - [page-name block-uuid] (if (util/uuid-string? block-uuid-or-page-name) - [nil (uuid block-uuid-or-page-name)] - [block-uuid-or-page-name nil]) - page-name (when page-name (util/page-name-sanity-lc page-name)) - _ (when (and page-name (not (db/entity [:block/name page-name]))) - (page-handler/js (sdk-utils/normalize-keyword-for-json new-block))))) + (p/let [repo (state/get-current-repo) + block? (util/uuid-string? (str block-uuid-or-page-name)) + ;; Ensure page or block's page has been loaded + result (db-async/clj opts) + [page-name block-uuid] (if (util/uuid-string? block-uuid-or-page-name) + [nil (uuid block-uuid-or-page-name)] + [block-uuid-or-page-name nil]) + page-name (when page-name (util/page-name-sanity-lc page-name)) + _ (when (and page-name (not (db/entity [:block/name page-name]))) + (page-handler/js (sdk-utils/normalize-keyword-for-json new-block))))))) (def ^:export insert_batch_block (fn [block-uuid ^js batch-blocks ^js opts] From 85f6b3dce79d384b3afe5fbdaa3cbb223aa6ed44 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 1 Feb 2024 00:33:43 +0800 Subject: [PATCH 21/60] fix: plugin api calls --- src/main/frontend/db/async.cljs | 8 + src/main/frontend/db/model.cljs | 15 -- src/main/frontend/db/react.cljs | 10 +- src/main/frontend/handler/editor.cljs | 123 ++++++------ src/main/logseq/api.cljs | 268 ++++++++++++++------------ 5 files changed, 221 insertions(+), 203 deletions(-) diff --git a/src/main/frontend/db/async.cljs b/src/main/frontend/db/async.cljs index cd649f9fa4..c1d31ff1cc 100644 --- a/src/main/frontend/db/async.cljs +++ b/src/main/frontend/db/async.cljs @@ -39,6 +39,12 @@ [(get ?p :template) ?t]])] (into {} result))) +(defn > (d/q - '[:find [(pull ?b [*]) ...] - :in $ ?name - :where - [?b :block/properties ?p] - [(get ?p :template) ?t] - [(= ?t ?name)]] - (conn/get-db) - name) - (sort-by :block/name) - (first)))) - (defn get-all-referenced-blocks-uuid "Get all uuids of blocks with any back link exists." [] diff --git a/src/main/frontend/db/react.cljs b/src/main/frontend/db/react.cljs index e4f18490b7..9214566580 100644 --- a/src/main/frontend/db/react.cljs +++ b/src/main/frontend/db/react.cljs @@ -109,7 +109,7 @@ (apply db-async-util/ (first sorted-blocks) - (update :block/properties-text-values dissoc :template) - (update :block/properties-order (fn [keys] - (vec (remove #{:template} keys))))) - (rest sorted-blocks)) - blocks (if template-including-parent? - sorted-blocks - (drop 1 sorted-blocks))] - (when element-id - (insert-command! element-id "" format {:end-pattern commands/command-trigger})) - (let [exclude-properties [:id :template :template-including-parent] - content-update-fn (fn [content] - (->> content - (property-file/remove-property-when-file-based repo format "template") - (property-file/remove-property-when-file-based repo format "template-including-parent") - template/resolve-dynamic-template!)) - page (if (:block/name block) block - (when target (:block/page (db/entity (:db/id target))))) - blocks' (map (fn [block] - (paste-block-cleanup repo block page exclude-properties format content-update-fn false)) - blocks) - sibling? (:sibling? opts) - sibling?' (cond - (some? sibling?) - sibling? + (let [block (if (integer? db-id) + (db-async/ (first sorted-blocks) + (update :block/properties-text-values dissoc :template) + (update :block/properties-order (fn [keys] + (vec (remove #{:template} keys))))) + (rest sorted-blocks)) + blocks (if template-including-parent? + sorted-blocks + (drop 1 sorted-blocks))] + (when element-id + (insert-command! element-id "" format {:end-pattern commands/command-trigger})) + (let [exclude-properties [:id :template :template-including-parent] + content-update-fn (fn [content] + (->> content + (property-file/remove-property-when-file-based repo format "template") + (property-file/remove-property-when-file-based repo format "template-including-parent") + template/resolve-dynamic-template!)) + page (if (:block/name block) block + (when target (:block/page (db/entity (:db/id target))))) + blocks' (map (fn [block] + (paste-block-cleanup repo block page exclude-properties format content-update-fn false)) + blocks) + sibling? (:sibling? opts) + sibling?' (cond + (some? sibling?) + sibling? - (db/has-children? (:block/uuid target)) - false + (db/has-children? (:block/uuid target)) + false - :else - true)] - (try - (let [*result (atom nil)] - (p/do! - (ui-outliner-tx/transact! - {:outliner-op :insert-blocks - :created-from-journal-template? journal?} - (when-not (string/blank? (state/get-edit-content)) - (save-current-block!)) - (let [result (outliner-core/insert-blocks! repo (db/get-db false) blocks' - target - (assoc opts :sibling? sibling?'))] - (reset! *result result))) - (some-> @*result edit-last-block-after-inserted!))) + :else + true)] + (try + (let [*result (atom nil)] + (p/do! + (ui-outliner-tx/transact! + {:outliner-op :insert-blocks + :created-from-journal-template? journal?} + (when-not (string/blank? (state/get-edit-content)) + (save-current-block!)) + (let [result (outliner-core/insert-blocks! repo (db/get-db false) blocks' + target + (assoc opts :sibling? sibling?'))] + (reset! *result result))) + (some-> @*result edit-last-block-after-inserted!))) - (catch :default ^js/Error e - (notification/show! - [:p.content - (util/format "Template insert error: %s" (.-message e))] - :error)))))))))) + (catch :default ^js/Error e + (notification/show! + [:p.content + (util/format "Template insert error: %s" (.-message e))] + :error))))))))))) (defn template-on-chosen-handler [element-id] diff --git a/src/main/logseq/api.cljs b/src/main/logseq/api.cljs index b08dd2fd9c..27965ec782 100644 --- a/src/main/logseq/api.cljs +++ b/src/main/logseq/api.cljs @@ -616,18 +616,22 @@ (let [{:keys [pos] :or {pos :max}} (bean/->clj opts)] (editor-handler/edit-block! block pos block-uuid)))))) +(defn- clj opts) @@ -674,52 +678,60 @@ (def ^:export insert_batch_block (fn [block-uuid ^js batch-blocks ^js opts] - (when-let [block (db-model/query-block-by-uuid (sdk-utils/uuid-or-throw-error block-uuid))] - (when-let [bb (bean/->clj batch-blocks)] - (let [bb (if-not (vector? bb) (vector bb) bb) - {:keys [sibling keepUUID before]} (bean/->clj opts) - keep-uuid? (or keepUUID false) - _ (when keep-uuid? (doseq - [block (outliner-core/tree-vec-flatten bb :children)] - (let [uuid (:id (:properties block))] - (when (and uuid (db-model/query-block-by-uuid (sdk-utils/uuid-or-throw-error uuid))) - (throw (js/Error. - (util/format "Custom block UUID already exists (%s)." uuid))))))) - block (if (and before sibling) - (db/pull (:db/id (:block/left block))) block) - _ (editor-handler/insert-block-tree-after-target - (:db/id block) sibling bb (:block/format block) keep-uuid?)] - nil))))) + (p/let [block (clj batch-blocks)] + (let [bb (if-not (vector? bb) (vector bb) bb) + {:keys [sibling keepUUID before]} (bean/->clj opts) + keep-uuid? (or keepUUID false) + _ (when keep-uuid? (doseq + [block (outliner-core/tree-vec-flatten bb :children)] + (let [uuid (:id (:properties block))] + (when (and uuid (db-model/query-block-by-uuid (sdk-utils/uuid-or-throw-error uuid))) + (throw (js/Error. + (util/format "Custom block UUID already exists (%s)." uuid))))))) + block (if (and before sibling) + (db/pull (:db/id (:block/left block))) block) + _ (editor-handler/insert-block-tree-after-target + (:db/id block) sibling bb (:block/format block) keep-uuid?)] + nil)))))) (def ^:export remove_block (fn [block-uuid ^js _opts] - (let [repo (state/get-current-repo)] + (p/let [repo (state/get-current-repo) + _ (clj opts))))) + (sdk-utils/uuid-or-throw-error block-uuid) content (bean/->clj opts))))) (def ^:export move_block (fn [src-block-uuid target-block-uuid ^js opts] - (let [{:keys [before children]} (bean/->clj opts) - move-to (cond - (boolean before) - :top + (p/let [_ (clj opts) + move-to (cond + (boolean before) + :top - (boolean children) - :nested + (boolean children) + :nested - :else - nil) - src-block (db-model/query-block-by-uuid (sdk-utils/uuid-or-throw-error src-block-uuid)) - target-block (db-model/query-block-by-uuid (sdk-utils/uuid-or-throw-error target-block-uuid))] - (editor-dnd-handler/move-blocks nil [src-block] target-block nil move-to)))) + :else + nil) + src-block (db-model/query-block-by-uuid (sdk-utils/uuid-or-throw-error src-block-uuid)) + target-block (db-model/query-block-by-uuid (sdk-utils/uuid-or-throw-error target-block-uuid))] + (editor-dnd-handler/move-blocks nil [src-block] target-block nil move-to))))) -(def ^:export get_block api-block/get_block) +(def ^:export get_block + (fn [id ^js opts] + (p/let [_ (db-async/js (sdk-utils/normalize-keyword-for-json block))))))) + (p/let [id (sdk-utils/uuid-or-throw-error block-uuid) + _ (js (sdk-utils/normalize-keyword-for-json block)))))))) (def ^:export get_next_sibling_block (fn [block-uuid] - (when-let [block (db-model/query-block-by-uuid (sdk-utils/uuid-or-throw-error block-uuid))] - (when-let [right-sibling (outliner-core/get-right-sibling (db/get-db) (:db/id block))] - (let [block (db/pull (:db/id right-sibling))] - (bean/->js (sdk-utils/normalize-keyword-for-json block))))))) + (p/let [id (sdk-utils/uuid-or-throw-error block-uuid) + _ (js (sdk-utils/normalize-keyword-for-json block)))))))) (def ^:export set_block_collapsed (fn [block-uuid ^js opts] - (let [block-uuid (sdk-utils/uuid-or-throw-error block-uuid)] + (p/let [block-uuid (sdk-utils/uuid-or-throw-error block-uuid) + _ (db-async/clj opts) opts (if (or (string? opts) (boolean? opts)) {:flag opts} opts) @@ -756,34 +773,42 @@ (not (util/collapsed? block)) (boolean flag))] (if flag (editor-handler/collapse-block! block-uuid) - (editor-handler/expand-block! block-uuid)) + (editor-handler/expand-block! block-uuid)) nil))))) (def ^:export upsert_block_property (fn [block-uuid key value] - (property-handler/set-block-property! - (state/get-current-repo) - (sdk-utils/uuid-or-throw-error block-uuid) key value))) + (p/let [block-uuid (sdk-utils/uuid-or-throw-error block-uuid) + _ (db-async/js (sdk-utils/normalize-keyword-for-json properties)))))) + (p/let [block-uuid (sdk-utils/uuid-or-throw-error block-uuid) + _ (db-async/js (sdk-utils/normalize-keyword-for-json properties))))))) (def ^:export get_current_page_blocks_tree (fn [] @@ -796,21 +821,25 @@ (def ^:export get_page_blocks_tree (fn [id-or-page-name] - (when-let [page-name (:block/name (db-model/get-page id-or-page-name))] - (let [blocks (db-model/get-page-blocks-no-cache page-name) - blocks (outliner-tree/blocks->vec-tree blocks page-name) - blocks (sdk-utils/normalize-keyword-for-json blocks)] - (bean/->js blocks))))) + (p/let [_ (vec-tree blocks page-name) + blocks (sdk-utils/normalize-keyword-for-json blocks)] + (bean/->js blocks)))))) (defn ^:export get_page_linked_references [page-name-or-uuid] - (when-let [page (and page-name-or-uuid (db-model/get-page page-name-or-uuid))] - (let [page-name (:block/name page) + (p/let [repo (state/get-current-repo) + block (db-async/js (sdk-utils/normalize-keyword-for-json ref-blocks))))) + (bean/->js (sdk-utils/normalize-keyword-for-json ref-blocks)))) (defn ^:export get_pages_from_namespace [ns] @@ -824,16 +853,6 @@ (when-let [pages (db-model/get-namespace-hierarchy repo ns)] (bean/->js (sdk-utils/normalize-keyword-for-json pages))))) -(defn first-child-of-block - [block] - (when-let [children (:block/_parent block)] - (first (db-model/sort-by-left children block)))) - -(defn second-child-of-block - [block] - (when-let [children (:block/_parent block)] - (second (db-model/sort-by-left children block)))) - (defn last-child-of-block [block] (when-let [children (:block/_parent block)] @@ -841,36 +860,32 @@ (defn ^:export prepend_block_in_page [uuid-or-page-name content ^js opts] - (p/let [page? (not (util/uuid-string? uuid-or-page-name)) + (p/let [_ (clj opts) - opts (merge opts {:sibling sibling? :before sibling?}) - src (if sibling? (str (:block/uuid block')) uuid-or-page-name)] - (insert_block src content (bean/->js opts)))))) + (let [opts (bean/->clj opts) + target (str (:block/uuid block))] + (insert_block target content (bean/->js opts)))))) (defn ^:export append_block_in_page [uuid-or-page-name content ^js opts] - (p/let [page? (not (util/uuid-string? uuid-or-page-name)) + (p/let [_ (clj opts) - opts (merge opts {:sibling sibling?} - (when sibling? {:before false})) - src (if sibling? (str (:block/uuid block')) uuid-or-page-name)] - (insert_block src content (bean/->js opts)))))) + target (str (:block/uuid block))] + (insert_block target content (bean/->js opts)))))) ;; plugins (defn ^:export validate_external_plugins [urls] @@ -887,34 +902,36 @@ (defn ^:export q [query-string] (when-let [repo (state/get-current-repo)] - (when-let [result (query-dsl/query repo query-string - {:disable-reactive? true})] + (p/let [result (query-dsl/query repo query-string + {:disable-reactive? true + :return-promise? true})] (bean/->js (sdk-utils/normalize-keyword-for-json (flatten @result)))))) (defn ^:export datascript_query [query & inputs] (when-let [repo (state/get-current-repo)] (when-let [db (db/get-db repo)] - (let [query (cljs.reader/read-string query) - resolved-inputs (map #(cond - (string? %) - (some->> % (cljs.reader/read-string) (query-react/resolve-input db)) + (p/let [query (cljs.reader/read-string query) + resolved-inputs (map #(cond + (string? %) + (some->> % (cljs.reader/read-string) (query-react/resolve-input db)) - (fn? %) - (fn [& args] - (.apply % nil (clj->js (mapv bean/->js args)))) + (fn? %) + (fn [& args] + (.apply % nil (clj->js (mapv bean/->js args)))) - :else %) - inputs) - result (apply d/q query db resolved-inputs)] + :else %) + inputs) + result (db-async/js (sdk-utils/normalize-keyword-for-json result false)))))) (defn ^:export custom_query [query-string] - (let [result (let [query (cljs.reader/read-string query-string)] - (db/custom-query {:query query - :disable-reactive? true}))] - (bean/->js (sdk-utils/normalize-keyword-for-json (flatten @result))))) + (p/let [result (let [query (cljs.reader/read-string query-string)] + (db/custom-query {:query query + :disable-reactive? true + :return-promise? true}))] + (bean/->js (sdk-utils/normalize-keyword-for-json (flatten result))))) (defn ^:export download_graph_db [] @@ -987,10 +1004,10 @@ ;; templates (defn ^:export get_template [name] - (some-> name - (db-model/get-template-by-name) - (sdk-utils/normalize-keyword-for-json) - (bean/->js))) + (p/let [block (when name (db-async/ block + (sdk-utils/normalize-keyword-for-json) + (bean/->js)))) (defn ^:export insert_template [target-uuid template-name] @@ -1007,20 +1024,21 @@ [target-uuid template-name ^js opts] (when (and template-name (db-model/get-block-by-uuid target-uuid)) (p/let [{:keys [overwrite]} (bean/->clj opts) - exist? (page-handler/ Date: Thu, 1 Feb 2024 01:22:41 +0800 Subject: [PATCH 22/60] fix: async query for scheduled and deadlines --- .../components/scheduled_deadlines.cljs | 18 ++++++--- src/main/frontend/db.cljs | 2 +- src/main/frontend/db/async.cljs | 40 ++++++++++++++++++- src/main/frontend/db/model.cljs | 37 ----------------- src/main/logseq/api.cljs | 2 +- 5 files changed, 53 insertions(+), 46 deletions(-) diff --git a/src/main/frontend/components/scheduled_deadlines.cljs b/src/main/frontend/components/scheduled_deadlines.cljs index a1d68e4e87..2a413f612e 100644 --- a/src/main/frontend/components/scheduled_deadlines.cljs +++ b/src/main/frontend/components/scheduled_deadlines.cljs @@ -7,8 +7,8 @@ [clojure.string :as string] [frontend.components.editor :as editor] [rum.core :as rum] - [frontend.db :as db] - [frontend.db-mixins :as db-mixins])) + [frontend.db.async :as db-async] + [promesa.core :as p])) (defn- scheduled-or-deadlines? [page-name] @@ -16,10 +16,16 @@ (not (true? (state/scheduled-deadlines-disabled?))) (= (string/lower-case page-name) (string/lower-case (date/journal-name))))) -(rum/defc scheduled-and-deadlines-inner < rum/reactive db-mixins/query - [page-name] - (let [scheduled-or-deadlines (when (scheduled-or-deadlines? page-name) - (db/get-date-scheduled-or-deadlines (string/capitalize page-name)))] +(rum/defcs scheduled-and-deadlines-inner < rum/reactive + {:init (fn [state] + (let [*result (atom nil) + page-name (first (:rum/args state))] + (p/let [result (when (scheduled-or-deadlines? page-name) + (db-async/int journal-title)] + (let [future-days (state/get-scheduled-future-days) + date-format (tf/formatter "yyyyMMdd") + current-day (tf/parse date-format (str date)) + future-day (some->> (t/plus current-day (t/days future-days)) + (tf/unparse date-format) + (parse-long))] + (when future-day + (when-let [repo (state/get-current-repo)] + (p/let [result (= ?d ?day)])] + date + future-day + db-model/block-attrs)] + (->> result + db-model/sort-by-left-recursive + db-utils/group-by-page))))))) diff --git a/src/main/frontend/db/model.cljs b/src/main/frontend/db/model.cljs index 60cdbf2e75..4faa132307 100644 --- a/src/main/frontend/db/model.cljs +++ b/src/main/frontend/db/model.cljs @@ -17,8 +17,6 @@ [logseq.graph-parser.text :as text] [logseq.graph-parser.util.db :as db-util] [logseq.common.util :as common-util] - [cljs-time.core :as t] - [cljs-time.format :as tf] [frontend.config :as config] [logseq.db :as ldb])) @@ -807,41 +805,6 @@ independent of format as format specific heading characters are stripped" :entities (remove (fn [block] (= page-id (:db/id (:block/page block))))))))))) -(defn get-date-scheduled-or-deadlines - [journal-title] - (when-let [date (date/journal-title->int journal-title)] - (let [future-days (state/get-scheduled-future-days) - date-format (tf/formatter "yyyyMMdd") - current-day (tf/parse date-format (str date)) - future-day (some->> (t/plus current-day (t/days future-days)) - (tf/unparse date-format) - (parse-long))] - (when future-day - (when-let [repo (state/get-current-repo)] - (->> (react/q repo [:custom :scheduled-deadline journal-title] - {:use-cache? false} - '[:find [(pull ?block ?block-attrs) ...] - :in $ ?day ?future ?block-attrs - :where - (or - [?block :block/scheduled ?d] - [?block :block/deadline ?d]) - [(get-else $ ?block :block/repeated? false) ?repeated] - [(get-else $ ?block :block/marker "NIL") ?marker] - [(not= ?marker "DONE")] - [(not= ?marker "CANCELED")] - [(not= ?marker "CANCELLED")] - [(<= ?d ?future)] - (or-join [?repeated ?d ?day] - [(true? ?repeated)] - [(>= ?d ?day)])] - date - future-day - block-attrs) - react - (sort-by-left-recursive) - db-utils/group-by-page)))))) - (defn get-block-referenced-blocks ([block-uuid] (get-block-referenced-blocks block-uuid {})) diff --git a/src/main/logseq/api.cljs b/src/main/logseq/api.cljs index 27965ec782..eb37a18cea 100644 --- a/src/main/logseq/api.cljs +++ b/src/main/logseq/api.cljs @@ -922,7 +922,7 @@ :else %) inputs) - result (db-async/js (sdk-utils/normalize-keyword-for-json result false)))))) (defn ^:export custom_query From 9355520a8a1bc4df487dd0e17b3c854111faee57 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 1 Feb 2024 01:24:20 +0800 Subject: [PATCH 23/60] fix: lint warnings --- src/main/frontend/components/reference.cljs | 3 +-- src/main/logseq/api.cljs | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/frontend/components/reference.cljs b/src/main/frontend/components/reference.cljs index 8ed853b9fb..ce00fb8ed8 100644 --- a/src/main/frontend/components/reference.cljs +++ b/src/main/frontend/components/reference.cljs @@ -18,8 +18,7 @@ [rum.core :as rum] [frontend.modules.outliner.tree :as tree] [frontend.db.async :as db-async] - [promesa.core :as p] - [frontend.search :as search])) + [promesa.core :as p])) (defn- frequencies-sort [references] diff --git a/src/main/logseq/api.cljs b/src/main/logseq/api.cljs index eb37a18cea..7ce8543377 100644 --- a/src/main/logseq/api.cljs +++ b/src/main/logseq/api.cljs @@ -7,7 +7,6 @@ [logseq.sdk.ui :as sdk-ui] [logseq.sdk.assets :as sdk-assets] [clojure.string :as string] - [datascript.core :as d] [electron.ipc :as ipc] [frontend.commands :as commands] [frontend.components.plugins :as plugins] @@ -616,6 +615,8 @@ (let [{:keys [pos] :or {pos :max}} (bean/->clj opts)] (editor-handler/edit-block! block pos block-uuid)))))) +;; TODO: perf improvement, some operations such as delete-block doesn't need to load the full page +;; instead, the db worker should provide those calls (defn- Date: Thu, 1 Feb 2024 02:03:16 +0800 Subject: [PATCH 24/60] fix: unit tests --- src/main/frontend/db/model.cljs | 31 ++++++++++++++-------------- src/main/frontend/db/react.cljs | 14 ++++++++----- src/test/frontend/db/model_test.cljs | 10 ++++----- 3 files changed, 30 insertions(+), 25 deletions(-) diff --git a/src/main/frontend/db/model.cljs b/src/main/frontend/db/model.cljs index 4faa132307..a86c6af98a 100644 --- a/src/main/frontend/db/model.cljs +++ b/src/main/frontend/db/model.cljs @@ -785,25 +785,26 @@ independent of format as format specific heading characters are stripped" (when repo (when (conn/get-db repo) (let [page-id (:db/id (db-utils/entity [:block/name (util/safe-page-name-sanity-lc page)])) - pages (page-alias-set repo page) - aliases (set/difference pages #{page-id})] + pages (page-alias-set repo page)] (->> (react/q repo - [:frontend.worker.react/refs page-id] - {:query-fn (fn [] - (let [entities (mapcat (fn [id] - (:block/_path-refs (db-utils/entity id))) pages) - blocks (map (fn [e] - {:block/parent (:block/parent e) - :block/left (:block/left e) - :block/page (:block/page e) - :block/collapsed? (:block/collapsed? e)}) entities)] - {:entities entities - :blocks blocks}))} - nil) + [:frontend.worker.react/refs page-id] + {:query-fn (fn [] + (let [entities (mapcat (fn [id] + (:block/_path-refs (db-utils/entity id))) pages) + blocks (map (fn [e] + {:block/parent (:block/parent e) + :block/left (:block/left e) + :block/page (:block/page e) + :block/collapsed? (:block/collapsed? e)}) entities)] + {:entities entities + :blocks blocks}))} + nil) react :entities - (remove (fn [block] (= page-id (:db/id (:block/page block))))))))))) + (remove (fn [block] + (= page-id (:db/id (:block/page block))))) + (util/distinct-by :db/id))))))) (defn get-block-referenced-blocks ([block-uuid] diff --git a/src/main/frontend/db/react.cljs b/src/main/frontend/db/react.cljs index 9214566580..765116d853 100644 --- a/src/main/frontend/db/react.cljs +++ b/src/main/frontend/db/react.cljs @@ -11,7 +11,8 @@ [frontend.util :as util :refer [react]] [clojure.core.async :as async] [frontend.db.async.util :as db-async-util] - [promesa.core :as p])) + [promesa.core :as p] + [datascript.core :as d])) ;; Query atom of map of Key ([repo q inputs]) -> atom ;; TODO: replace with LRUCache, only keep the latest 20 or 50 items? @@ -89,7 +90,10 @@ (defn- Date: Thu, 1 Feb 2024 02:57:28 +0800 Subject: [PATCH 25/60] fix: can't save file --- deps/db/src/logseq/db/sqlite/common_db.cljs | 3 +- src/main/frontend/components/page.cljs | 8 ++- src/main/frontend/fs/watcher_handler.cljs | 60 ++------------------- src/main/frontend/handler/events.cljs | 6 +-- src/main/frontend/worker/file/core.cljs | 3 +- 5 files changed, 15 insertions(+), 65 deletions(-) diff --git a/deps/db/src/logseq/db/sqlite/common_db.cljs b/deps/db/src/logseq/db/sqlite/common_db.cljs index b2fd85f1a2..f2cf0795df 100644 --- a/deps/db/src/logseq/db/sqlite/common_db.cljs +++ b/deps/db/src/logseq/db/sqlite/common_db.cljs @@ -25,7 +25,8 @@ (->> (d/datoms db :avet :file/path) (map (fn [e] {:db/id (:e e) - :file/path (:v e)})))) + :file/path (:v e) + :file/content (:file/content (d/entity db (:e e)))})))) (defn get-block-and-children [db name children?] diff --git a/src/main/frontend/components/page.cljs b/src/main/frontend/components/page.cljs index 8a71a60a47..7b1cc0addc 100644 --- a/src/main/frontend/components/page.cljs +++ b/src/main/frontend/components/page.cljs @@ -519,9 +519,13 @@ (plugins/hook-ui-slot :page-head-actions-slotted nil) (plugins/hook-ui-items :pagebar)]))]) - (when (and db-based? (not block?)) + (cond + (and db-based? (not block?)) [:div.pb-4 - (db-page/page-info page (::hover-title? state))]) + (db-page/page-info page (::hover-title? state))] + + (and (not db-based?) (not block?)) + [:div.pb-4]) [:div (when (and block? (not sidebar?) (not whiteboard?)) diff --git a/src/main/frontend/fs/watcher_handler.cljs b/src/main/frontend/fs/watcher_handler.cljs index d41ee5c00b..371e90395c 100644 --- a/src/main/frontend/fs/watcher_handler.cljs +++ b/src/main/frontend/fs/watcher_handler.cljs @@ -137,64 +137,11 @@ ;; return nil, otherwise the entire db will be transferred by ipc nil))) -(defn preload-graph-homepage-files! - "Preload the homepage file for the current graph. Return loaded file paths. - - Prerequisites: - - current graph is set - - config is loaded" - [] - (when-let [repo (state/get-current-repo)] - (when (and (not (state/loading-files? repo)) - (config/local-file-based-graph? repo)) - (let [repo-dir (config/get-repo-dir repo) - page-name (if (state/enable-journals? repo) - (date/today) - (or (:page (state/get-default-home)) "Contents")) - page-name (util/page-name-sanity-lc page-name) - file-rpath (or (:file/path (db/get-page-file page-name)) - (let [format (state/get-preferred-format repo) - ext (config/get-file-extension format) - file-name (if (state/enable-journals? repo) - (date/journal-title->default (date/today)) - (or (:page (state/get-default-home)) "contents")) - parent-dir (if (state/enable-journals? repo) - (config/get-journals-directory) - (config/get-pages-directory))] - (str parent-dir "/" file-name "." ext)))] - (prn ::preload-homepage file-rpath) - (p/let [file-exists? (fs/file-exists? repo-dir file-rpath) - _ (when file-exists? - ;; BUG: avoid active-editing block content overwrites incoming fs changes - (editor-handler/escape-editing false)) - file-content (when file-exists? - (fs/read-file repo-dir file-rpath)) - file-mtime (when file-exists? - (:mtime (fs/stat repo-dir file-rpath))) - db-empty? (db/page-empty? repo page-name) - db-content (if-not db-empty? - (db-async/> files - (remove #(contains? exclude-files %))) + [files deleted-files]))) (p/catch (fn [error] (when-not (config/demo-graph? graph) diff --git a/src/main/frontend/handler/events.cljs b/src/main/frontend/handler/events.cljs index 752937abe8..e5fe0451f0 100644 --- a/src/main/frontend/handler/events.cljs +++ b/src/main/frontend/handler/events.cljs @@ -396,14 +396,12 @@ (when (and (not dir-exists?) (not util/nfs?)) (state/pub-event! [:graph/dir-gone dir])))) - (p/let [loaded-homepage-files (when-not (config/db-based-graph? repo) - (fs-watcher/preload-graph-homepage-files!)) - ;; re-render-root is async and delegated to rum, so we need to wait for main ui to refresh + (p/let [;; re-render-root is async and delegated to rum, so we need to wait for main ui to refresh _ (js/setTimeout #(mobile/mobile-postinit) 1000) ;; FIXME: an ugly implementation for redirecting to page on new window is restored _ (repo-handler/graph-ready! repo) _ (when-not (config/db-based-graph? repo) - (fs-watcher/load-graph-files! repo loaded-homepage-files))])) + (fs-watcher/load-graph-files! repo))])) (defmethod handle :notification/show [[_ {:keys [content status clear?]}]] (notification/show! content status clear?)) diff --git a/src/main/frontend/worker/file/core.cljs b/src/main/frontend/worker/file/core.cljs index 9093c49f25..7a1d49b23e 100644 --- a/src/main/frontend/worker/file/core.cljs +++ b/src/main/frontend/worker/file/core.cljs @@ -8,7 +8,8 @@ [logseq.db :as ldb] [frontend.worker.date :as worker-date] [frontend.worker.util :as worker-util] - [logseq.db.sqlite.util :as sqlite-util])) + [logseq.db.sqlite.util :as sqlite-util] + [promesa.core :as p])) (defonce *writes (atom {})) (defonce *request-id (atom 0)) From fd306bf127e1d79adc4f1fdf0bb70892113dd923 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 1 Feb 2024 03:46:26 +0800 Subject: [PATCH 26/60] refactor: reindex graph equals delete and re-add the same graph --- src/main/frontend/handler/repo.cljs | 43 +++++++++++++------------- src/main/frontend/handler/web/nfs.cljs | 17 ++++------ 2 files changed, 27 insertions(+), 33 deletions(-) diff --git a/src/main/frontend/handler/repo.cljs b/src/main/frontend/handler/repo.cljs index 448db3bd57..76a8eba0aa 100644 --- a/src/main/frontend/handler/repo.cljs +++ b/src/main/frontend/handler/repo.cljs @@ -341,28 +341,27 @@ (load-contents add-or-modify-files options))))))) (defn remove-repo! - [{:keys [url] :as repo}] - (let [db-based? (config/db-based-graph? url) - delete-db-f (fn [] - (let [current-repo (state/get-current-repo)] - (db/remove-conn! url) - (db-persist/delete-graph! url) - (search/remove-db! url) - (state/delete-repo! repo) - (if (= current-repo url) - (when-let [graph (:url (first (state/get-repos)))] - (notification/show! (str "Removed graph " - (pr-str (text-util/get-graph-name-from-path url)) - ". Redirecting to graph " - (pr-str (text-util/get-graph-name-from-path graph))) - :success) - (state/pub-event! [:graph/switch graph {:persist? false}])) - (notification/show! (str "Removed graph " (pr-str (text-util/get-graph-name-from-path url))) :success))))] - (when (or (config/local-file-based-graph? url) - db-based? - (config/demo-graph? url)) - (-> (idb/clear-local-db! url) ; clear file handles - (p/finally delete-db-f))))) + [{:keys [url] :as repo} & {:keys [switch-graph?] + :or {switch-graph? true}}] + (let [current-repo (state/get-current-repo) + db-based? (config/db-based-graph? url)] + (when (or (config/local-file-based-graph? url) db-based?) + (p/do! + (idb/clear-local-db! url) ; clear file handles + (db/remove-conn! url) + (db-persist/delete-graph! url) + (search/remove-db! url) + (state/delete-repo! repo) + (when switch-graph? + (if (= current-repo url) + (when-let [graph (:url (first (state/get-repos)))] + (notification/show! (str "Removed graph " + (pr-str (text-util/get-graph-name-from-path url)) + ". Redirecting to graph " + (pr-str (text-util/get-graph-name-from-path graph))) + :success) + (state/pub-event! [:graph/switch graph {:persist? false}])) + (notification/show! (str "Removed graph " (pr-str (text-util/get-graph-name-from-path url))) :success))))))) (defn start-repo-db-if-not-exists! [repo & {:as opts}] diff --git a/src/main/frontend/handler/web/nfs.cljs b/src/main/frontend/handler/web/nfs.cljs index 5185aea7ae..5a9155f3b7 100644 --- a/src/main/frontend/handler/web/nfs.cljs +++ b/src/main/frontend/handler/web/nfs.cljs @@ -288,17 +288,12 @@ (defn rebuild-index! [repo ok-handler] - (let [ok-handler (fn [] - (ok-handler) - (state/set-nfs-refreshing! false))] - (when repo - (state/set-nfs-refreshing! true) - (search/reset-indice! repo) - (db/remove-conn! repo) - (db/clear-query-state!) - (db/start-db-conn! repo {:db-graph? (config/db-based-graph? repo)}) - (reload-dir! repo {:re-index? true - :ok-handler ok-handler})))) + (let [graph-dir (config/get-repo-dir repo)] + (when (and repo (not (config/db-based-graph? repo))) + (p/do! + (repo-handler/remove-repo! {:url repo} :switch-graph? false) + (ls-dir-files-with-path! graph-dir) + (when (fn? ok-handler) (ok-handler)))))) ;; TODO: move to frontend.handler.repo (defn refresh! From c9f52e8a9b4940afa9720d6ba66e2197735fd5de Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 1 Feb 2024 04:29:32 +0800 Subject: [PATCH 27/60] fix: ref not shows when starting the app --- deps/db/src/logseq/db/sqlite/common_db.cljs | 24 ++++++++++++++++----- src/main/frontend/components/block.cljs | 22 +++++++++---------- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/deps/db/src/logseq/db/sqlite/common_db.cljs b/deps/db/src/logseq/db/sqlite/common_db.cljs index f2cf0795df..103dc2d107 100644 --- a/deps/db/src/logseq/db/sqlite/common_db.cljs +++ b/deps/db/src/logseq/db/sqlite/common_db.cljs @@ -28,21 +28,35 @@ :file/path (:v e) :file/content (:file/content (d/entity db (:e e)))})))) +(defn- get-block-with-refs + [db block] + (update block :block/refs (fn [refs] + (map (fn [ref] + (let [e (d/entity db (:db/id ref))] + (if (and e (:block/name e)) + (assoc ref + :block/uuid (:block/uuid e) + :block/original-name (:block/name e) + :block/name (:block/name e)) + ref))) refs)))) + (defn get-block-and-children [db name children?] (let [get-children (fn [col] (map (fn [e] - (select-keys e [:db/id :block/uuid :block/page :block/left :block/parent])) + (select-keys e [:db/id :block/uuid :block/page :block/left :block/parent :block/collapsed?])) col)) uuid? (common-util/uuid-string? name) block (when uuid? (let [id (uuid name)] (d/entity db [:block/uuid id])))] (if (and block (not (:block/name block))) ; not a page - (cond-> - {:block (d/pull db '[*] (:db/id block))} - children? - (assoc :children (get-children (:block/_parent block)))) + (let [block' (d/pull db '[*] (:db/id block)) + block-with-refs (get-block-with-refs db block')] + (cond-> + {:block block-with-refs} + children? + (assoc :children (get-children (:block/_parent block))))) (when-let [block (or block (d/entity db [:block/name name]))] (cond-> {:block (d/pull db '[*] (:db/id block))} diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index 1f64e5ba1b..d4b2082de2 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -3139,18 +3139,18 @@ (let [repo (state/get-current-repo) *navigating-block (get state ::navigating-block) navigating-block (rum/react *navigating-block) - navigated? (and (not= (:block/uuid block) navigating-block) navigating-block)] + navigated? (and (not= (:block/uuid block) navigating-block) navigating-block) + _ (state/sub-async-query-loading (:block/uuid block))] (when (:block/uuid block) - (when-not (state/sub-async-query-loading (:block/uuid block)) - (let [[original-block block] (build-block config block {:navigating-block navigating-block :navigated? navigated?}) - config' (if original-block - (assoc config :original-block original-block) - config) - opts {}] - (rum/with-key - (block-container-inner state repo config' block - (merge opts {:navigating-block navigating-block :navigated? navigated?})) - (str "block-inner" (:block/uuid block)))))))) + (let [[original-block block] (build-block config block {:navigating-block navigating-block :navigated? navigated?}) + config' (if original-block + (assoc config :original-block original-block) + config) + opts {}] + (rum/with-key + (block-container-inner state repo config' block + (merge opts {:navigating-block navigating-block :navigated? navigated?})) + (str "block-inner" (:block/uuid block))))))) (defn divide-lists [[f & l]] From d110dc41b4cc7fb25d0603ea0225fbde22b61997 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 1 Feb 2024 05:20:19 +0800 Subject: [PATCH 28/60] fix: don't create page if exists --- src/main/frontend/components/block.cljs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index d4b2082de2..b5f24693f6 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -511,14 +511,16 @@ [e config page-name redirect-page-name page-name-in-block contents-page? whiteboard-page?] (util/stop e) (when (not (util/right-click? e)) - (let [redirect-page-name (or redirect-page-name - (model/get-redirect-page-name page-name (:block/alias? config)))] + (p/let [redirect-page-name (or redirect-page-name + (model/get-redirect-page-name page-name (:block/alias? config))) + page (when redirect-page-name + (db-async/ Date: Thu, 1 Feb 2024 17:04:44 +0800 Subject: [PATCH 29/60] fix: load all pages and files when app starts --- deps/db/src/logseq/db/sqlite/common_db.cljs | 6 ++-- src/main/frontend/db_worker.cljs | 31 +++++++++++---------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/deps/db/src/logseq/db/sqlite/common_db.cljs b/deps/db/src/logseq/db/sqlite/common_db.cljs index 103dc2d107..7a3381c0bb 100644 --- a/deps/db/src/logseq/db/sqlite/common_db.cljs +++ b/deps/db/src/logseq/db/sqlite/common_db.cljs @@ -89,9 +89,9 @@ (defn get-initial-data "Returns initial data" [db] - (let [files (get-built-in-files db) - journals (get-latest-journals db 3)] - (concat files journals))) + (let [all-pages (get-all-pages db) + all-files (get-all-files db)] + (concat all-pages all-files))) (defn restore-initial-data "Given initial sqlite data and schema, returns a datascript connection" diff --git a/src/main/frontend/db_worker.cljs b/src/main/frontend/db_worker.cljs index 4f9ffc4eee..3b00e1d968 100644 --- a/src/main/frontend/db_worker.cljs +++ b/src/main/frontend/db_worker.cljs @@ -260,7 +260,7 @@ [_this repo & {:keys [close-other-db?] :or {close-other-db? true}}] (p/do! - (when close-other-db? + (when close-other-db? (close-other-dbs! repo)) (create-or-open-db! repo))) @@ -364,20 +364,21 @@ (when-let [conn (worker-state/get-datascript-conn repo)] (let [data (->> (sqlite-common-db/get-initial-data @conn) pr-str)] - (async/go - ;; TODO: after UI db transacted - (async/ Date: Thu, 1 Feb 2024 17:18:23 +0800 Subject: [PATCH 30/60] fix: load closed values --- deps/db/src/logseq/db/sqlite/common_db.cljs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/deps/db/src/logseq/db/sqlite/common_db.cljs b/deps/db/src/logseq/db/sqlite/common_db.cljs index 7a3381c0bb..3bd2f08dae 100644 --- a/deps/db/src/logseq/db/sqlite/common_db.cljs +++ b/deps/db/src/logseq/db/sqlite/common_db.cljs @@ -85,13 +85,21 @@ (reverse) (take n)))) +(defn get-closed-values + [db] + (->> (d/datoms db :avet :block/type) + (keep (fn [e] + (when (= (:v e) "closed value") + (d/pull db '[*] (:e e))))))) + ;; built-in files + latest journals + favorites (defn get-initial-data "Returns initial data" [db] (let [all-pages (get-all-pages db) - all-files (get-all-files db)] - (concat all-pages all-files))) + all-files (get-all-files db) + closed-values (get-closed-values db)] + (concat all-pages all-files closed-values))) (defn restore-initial-data "Given initial sqlite data and schema, returns a datascript connection" From 95cd7d51fca1fcbe281fafbb38571a87547a208b Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Thu, 1 Feb 2024 20:07:39 +0800 Subject: [PATCH 31/60] fix: enable promise usage in outliner-ui/transact! --- deps/outliner/src/logseq/outliner/core.cljs | 16 +-- .../src/logseq/outliner/transaction.cljc | 48 ++++---- src/main/frontend/db/async.cljs | 12 ++ src/main/frontend/handler/dnd.cljs | 3 +- src/main/frontend/handler/editor.cljs | 107 +++++++++--------- .../frontend/handler/file_based/editor.cljs | 3 +- .../handler/file_based/page_property.cljs | 3 +- src/main/frontend/modules/outliner/ui.cljc | 3 +- src/main/logseq/api.cljs | 34 ++---- 9 files changed, 115 insertions(+), 114 deletions(-) diff --git a/deps/outliner/src/logseq/outliner/core.cljs b/deps/outliner/src/logseq/outliner/core.cljs index a47ddb09fb..203e52c2d8 100644 --- a/deps/outliner/src/logseq/outliner/core.cljs +++ b/deps/outliner/src/logseq/outliner/core.cljs @@ -1131,27 +1131,21 @@ ;;; ### write-operations have side-effects (do transactions) ;;;;;;;;;;;;;;;; -(def ^:private ^:dynamic *transaction-data* +(def *transaction-data "Stores transaction-data that are generated by one or more write-operations, see also `logseq.outliner.transaction/transact!`" - nil) + (atom nil)) -(def ^:private ^:dynamic #_:clj-kondo/ignore *transaction-opts* +(def #_:clj-kondo/ignore *transaction-opts "Stores transaction opts that are generated by one or more write-operations, see also `logseq.outliner.transaction/transact!`" - nil) - -(def ^:private ^:dynamic #_:clj-kondo/ignore *transaction-args* - "Stores transaction args which can be fetched in all op-transact functions." - nil) + (atom nil)) (defn- op-transact! [fn-var & args] {:pre [(var? fn-var)]} - (when (nil? *transaction-data*) - (throw (js/Error. (str (:name (meta fn-var)) " is not used in (transact! ...)")))) (let [result (apply @fn-var args)] - (conj! *transaction-data* (select-keys result [:tx-data :tx-meta])) + (swap! *transaction-data conj (select-keys result [:tx-data :tx-meta])) result)) (defn save-block! diff --git a/deps/outliner/src/logseq/outliner/transaction.cljc b/deps/outliner/src/logseq/outliner/transaction.cljc index cd221d2aac..79384b616b 100644 --- a/deps/outliner/src/logseq/outliner/transaction.cljc +++ b/deps/outliner/src/logseq/outliner/transaction.cljc @@ -2,7 +2,8 @@ "Provides a wrapper around logseq.outliner.datascript/transact! using transient state from logseq.outliner.core" #?(:cljs (:require-macros [logseq.outliner.transaction])) - #?(:cljs (:require [malli.core :as m]))) + #?(:cljs (:require [malli.core :as m] + [promesa.core :as p]))) #_:clj-kondo/ignore (def ^:private transact-opts [:or :symbol :map]) @@ -28,35 +29,38 @@ (move-blocks! ...) (delete-blocks! ...))" [opts & body] - `(let [transact-data# logseq.outliner.core/*transaction-data* - transaction-opts# logseq.outliner.core/*transaction-opts* + `(let [transact-data# logseq.outliner.core/*transaction-data + transaction-opts# logseq.outliner.core/*transaction-opts opts*# ~opts _# (assert (or (map? opts*#) (symbol? opts*#)) (str "opts is not a map or symbol, type: " (type opts*#))) - opts# (if transact-data# + opts# (if @transact-data# (assoc opts*# :nested-transaction? true) opts*#)] - (if transact-data# + (if @transact-data# (do - (when transaction-opts# - (conj! transaction-opts# opts#)) + (when @transaction-opts# + (swap! transaction-opts# conj opts#)) ~@body) (let [transaction-args# (cond-> {} (get opts*# :persist-op? true) (assoc :persist-op? true))] - (binding [logseq.outliner.core/*transaction-data* (transient []) - logseq.outliner.core/*transaction-opts* (transient []) - logseq.outliner.core/*transaction-args* transaction-args#] - (conj! logseq.outliner.core/*transaction-opts* opts#) - ~@body - (let [r# (persistent! logseq.outliner.core/*transaction-data*) - tx# (mapcat :tx-data r#) + (reset! logseq.outliner.core/*transaction-data []) + (reset! logseq.outliner.core/*transaction-opts [opts#]) + (p/do! + ~@body + (let [r# @logseq.outliner.core/*transaction-data + tx# (mapcat :tx-data r#) ;; FIXME: should we merge all the tx-meta? - tx-meta# (first (map :tx-meta r#)) - all-tx# (concat tx# (:additional-tx opts#)) - o# (persistent! logseq.outliner.core/*transaction-opts*) - full-opts# (apply merge (reverse o#)) - opts## (merge (dissoc full-opts# :additional-tx :current-block :nested-transaction?) tx-meta#)] + tx-meta# (first (map :tx-meta r#)) + all-tx# (concat tx# (:additional-tx opts#)) + o# @logseq.outliner.core/*transaction-opts + full-opts# (apply merge (reverse o#)) + opts## (merge (dissoc full-opts# :additional-tx :current-block :nested-transaction?) tx-meta#)] - (when (seq all-tx#) ;; If it's empty, do nothing - (when-not (:nested-transaction? opts#) ; transact only for the whole transaction - (logseq.outliner.datascript/transact! all-tx# (dissoc opts## :transact-opts) (:transact-opts opts##)))))))))) + (when (seq all-tx#) ;; If it's empty, do nothing + (when-not (:nested-transaction? opts#) ; transact only for the whole transaction + (prn :debug :data @transact-data#) + (logseq.outliner.datascript/transact! all-tx# (dissoc opts## :transact-opts) (:transact-opts opts##)))) + + (reset! logseq.outliner.core/*transaction-data nil) + (reset! logseq.outliner.core/*transaction-opts nil))))))) diff --git a/src/main/frontend/db/async.cljs b/src/main/frontend/db/async.cljs index bbb3172d4e..825e3c779e 100644 --- a/src/main/frontend/db/async.cljs +++ b/src/main/frontend/db/async.cljs @@ -212,3 +212,15 @@ (->> result db-model/sort-by-left-recursive db-utils/group-by-page))))))) + +;; TODO: perf improvement, some operations such as delete-block doesn't need to load the full page +;; instead, the db worker should provide those calls +(defn right needs to point its `left` to block->left - (when (and block-right (not= (:db/id (:block/parent prev-block)) - (:db/id (:block/parent block)))) - (outliner-save-block! {:db/id (:db/id block-right) - :block/left (:db/id (:block/left block))})) +;; block->right needs to point its `left` to block->left + (when (and block-right (not= (:db/id (:block/parent prev-block)) + (:db/id (:block/parent block)))) + (outliner-save-block! {:db/id (:db/id block-right) + :block/left (:db/id (:block/left block))})) ;; update prev-block's children to point to the refed block - (when (or (:block/collapsed? prev-block) - (= (:db/id prev-block) (:db/id (:block/parent block)))) - (let [children (:block/_parent prev-block)] - (doseq [child children] - (when-not (= (:db/id child) (:db/id block)) - (outliner-save-block! {:db/id (:db/id child) - :block/parent (:db/id block) - :block/left (:db/id block)}))))) + (when (or (:block/collapsed? prev-block) + (= (:db/id prev-block) (:db/id (:block/parent block)))) + (let [children (:block/_parent prev-block)] + (doseq [child children] + (when-not (= (:db/id child) (:db/id block)) + (outliner-save-block! {:db/id (:db/id child) + :block/parent (:db/id block) + :block/left (:db/id block)}))))) ;; parent will be removed - (when (= (:db/id prev-block) (:db/id (:block/parent block))) - (when-let [parent-right (outliner-core/get-right-sibling (db/get-db) (:db/id prev-block))] - (outliner-save-block! {:db/id (:db/id parent-right) - :block/left (:db/id block)}))) + (when (= (:db/id prev-block) (:db/id (:block/parent block))) + (when-let [parent-right (outliner-core/get-right-sibling (db/get-db) (:db/id prev-block))] + (outliner-save-block! {:db/id (:db/id parent-right) + :block/left (:db/id block)}))) - (when db-based? - (outliner-save-block! {:db/id (:db/id block) - :block/properties new-properties}))) + (when db-based? + (outliner-save-block! {:db/id (:db/id block) + :block/properties new-properties})))) - (do - (delete-block-fn block) - (save-block! repo prev-block new-content {}) - (when db-based? - (outliner-save-block! {:db/id (:db/id prev-block) - :block/properties new-properties}))))) + (p/do! + (delete-block-fn block) + (save-block! repo prev-block new-content {}) + (when db-based? + (outliner-save-block! {:db/id (:db/id prev-block) + :block/properties new-properties}))))) :else (delete-block-fn block))))))))))))) @@ -2864,19 +2864,18 @@ (let [editor (state/get-input) pos (some-> editor cursor/pos) {:keys [block]} (get-state)] - (p/do! - (when block - (state/set-editor-last-pos! pos) - (ui-outliner-tx/transact! - {:outliner-op :move-blocks - :real-outliner-op :indent-outdent} - (save-current-block!) - (outliner-core/indent-outdent-blocks! (state/get-current-repo) - (db/get-db false) - (block-handler/get-top-level-blocks [block]) - indent? - {:get-first-block-original block-handler/get-first-block-original - :logical-outdenting? (state/logical-outdenting?)})))))) + (when block + (state/set-editor-last-pos! pos) + (ui-outliner-tx/transact! + {:outliner-op :move-blocks + :real-outliner-op :indent-outdent} + (save-current-block!) + (outliner-core/indent-outdent-blocks! (state/get-current-repo) + (db/get-db false) + (block-handler/get-top-level-blocks [block]) + indent? + {:get-first-block-original block-handler/get-first-block-original + :logical-outdenting? (state/logical-outdenting?)}))))) (defn keydown-tab-handler [direction] diff --git a/src/main/frontend/handler/file_based/editor.cljs b/src/main/frontend/handler/file_based/editor.cljs index c3e82e7c16..e5fde6f28d 100644 --- a/src/main/frontend/handler/file_based/editor.cljs +++ b/src/main/frontend/handler/file_based/editor.cljs @@ -17,7 +17,8 @@ [frontend.handler.file-based.property :as file-property-handler] [frontend.handler.file-based.property.util :as property-util] [logseq.db.frontend.schema :as db-schema] - [logseq.common.util.block-ref :as block-ref])) + [logseq.common.util.block-ref :as block-ref] + [promesa.core :as p])) (defn- remove-non-existed-refs! [refs] diff --git a/src/main/frontend/handler/file_based/page_property.cljs b/src/main/frontend/handler/file_based/page_property.cljs index 6d42544d82..75f5142cc5 100644 --- a/src/main/frontend/handler/file_based/page_property.cljs +++ b/src/main/frontend/handler/file_based/page_property.cljs @@ -5,7 +5,8 @@ [logseq.outliner.core :as outliner-core] [frontend.modules.outliner.ui :as ui-outliner-tx] [frontend.state :as state] - [frontend.util :as util])) + [frontend.util :as util] + [promesa.core :as p])) (defn insert-property [format content key value] diff --git a/src/main/frontend/modules/outliner/ui.cljc b/src/main/frontend/modules/outliner/ui.cljc index 5227d715ff..983796ab0b 100644 --- a/src/main/frontend/modules/outliner/ui.cljc +++ b/src/main/frontend/modules/outliner/ui.cljc @@ -3,7 +3,8 @@ #?(:cljs (:require-macros [frontend.modules.outliner.ui])) #?(:cljs (:require [frontend.state :as state] [frontend.config :as config] - [frontend.db :as db]))) + [frontend.db :as db] + [promesa.core :as p]))) #?(:cljs (do diff --git a/src/main/logseq/api.cljs b/src/main/logseq/api.cljs index 7ce8543377..5c94606d75 100644 --- a/src/main/logseq/api.cljs +++ b/src/main/logseq/api.cljs @@ -615,24 +615,12 @@ (let [{:keys [pos] :or {pos :max}} (bean/->clj opts)] (editor-handler/edit-block! block pos block-uuid)))))) -;; TODO: perf improvement, some operations such as delete-block doesn't need to load the full page -;; instead, the db worker should provide those calls -(defn- clj opts) @@ -679,7 +667,7 @@ (def ^:export insert_batch_block (fn [block-uuid ^js batch-blocks ^js opts] - (p/let [block (clj batch-blocks)] (let [bb (if-not (vector? bb) (vector bb) bb) @@ -700,21 +688,21 @@ (def ^:export remove_block (fn [block-uuid ^js _opts] (p/let [repo (state/get-current-repo) - _ (clj opts))))) (def ^:export move_block (fn [src-block-uuid target-block-uuid ^js opts] - (p/let [_ (clj opts) move-to (cond (boolean before) @@ -747,7 +735,7 @@ (def ^:export get_previous_sibling_block (fn [block-uuid] (p/let [id (sdk-utils/uuid-or-throw-error block-uuid) - _ (vec-tree blocks page-name) @@ -861,7 +849,7 @@ (defn ^:export prepend_block_in_page [uuid-or-page-name content ^js opts] - (p/let [_ ( Date: Fri, 2 Feb 2024 00:16:49 +0800 Subject: [PATCH 32/60] refactor indent-and-outdent --- deps/outliner/src/logseq/outliner/core.cljs | 55 ++++++++++--------- .../src/logseq/outliner/transaction.cljc | 1 - src/main/frontend/db_worker.cljs | 42 +++++++++++++- src/main/frontend/handler/block.cljs | 29 +++++----- src/main/frontend/handler/editor.cljs | 50 ++++++----------- src/main/frontend/worker/state.cljs | 7 ++- 6 files changed, 108 insertions(+), 76 deletions(-) diff --git a/deps/outliner/src/logseq/outliner/core.cljs b/deps/outliner/src/logseq/outliner/core.cljs index 203e52c2d8..1e66f1a59d 100644 --- a/deps/outliner/src/logseq/outliner/core.cljs +++ b/deps/outliner/src/logseq/outliner/core.cljs @@ -1067,7 +1067,7 @@ (defn- ^:large-vars/cleanup-todo indent-outdent-blocks "Indent or outdent `blocks`." - [repo conn blocks indent? & {:keys [get-first-block-original logical-outdenting?]}] + [repo conn blocks indent? & {:keys [parent-original logical-outdenting?]}] {:pre [(seq blocks) (boolean? indent?)]} (let [db @conn top-level-blocks (map (fn [b] (d/entity db (:db/id b))) blocks) @@ -1100,34 +1100,33 @@ (concat-tx-fn result collapsed-tx)) (move-blocks repo conn blocks' left (merge opts {:sibling? false :indent? true})))))) - (let [parent-original (when get-first-block-original (get-first-block-original))] - (if parent-original + (if parent-original + (let [blocks' (take-while (fn [b] + (not= (:db/id (:block/parent b)) + (:db/id (:block/parent parent)))) + top-level-blocks)] + (move-blocks repo conn blocks' parent-original (merge opts {:outliner-op :indent-outdent-blocks + :sibling? true + :indent? false}))) + + (when (and parent (not (page-block? (d/entity db (:db/id parent))))) (let [blocks' (take-while (fn [b] (not= (:db/id (:block/parent b)) (:db/id (:block/parent parent)))) - top-level-blocks)] - (move-blocks repo conn blocks' parent-original (merge opts {:outliner-op :indent-outdent-blocks - :sibling? true - :indent? false}))) - - (when (and parent (not (page-block? (d/entity db (:db/id parent))))) - (let [blocks' (take-while (fn [b] - (not= (:db/id (:block/parent b)) - (:db/id (:block/parent parent)))) - top-level-blocks) - result (move-blocks repo conn blocks' parent (merge opts {:sibling? true}))] - (if logical-outdenting? - result + top-level-blocks) + result (move-blocks repo conn blocks' parent (merge opts {:sibling? true}))] + (if logical-outdenting? + result ;; direct outdenting (default behavior) - (let [last-top-block (d/entity db (:db/id (last blocks'))) - right-siblings (->> (get-right-siblings conn (block db last-top-block)) - (map :data))] - (if (seq right-siblings) - (let [result2 (if-let [last-direct-child-id (ldb/get-block-last-direct-child-id db (:db/id last-top-block))] - (move-blocks repo conn right-siblings (d/entity db last-direct-child-id) (merge opts {:sibling? true})) - (move-blocks repo conn right-siblings last-top-block (merge opts {:sibling? false})))] - (concat-tx-fn result result2)) - result)))))))))))) + (let [last-top-block (d/entity db (:db/id (last blocks'))) + right-siblings (->> (get-right-siblings conn (block db last-top-block)) + (map :data))] + (if (seq right-siblings) + (let [result2 (if-let [last-direct-child-id (ldb/get-block-last-direct-child-id db (:db/id last-top-block))] + (move-blocks repo conn right-siblings (d/entity db last-direct-child-id) (merge opts {:sibling? true})) + (move-blocks repo conn right-siblings last-top-block (merge opts {:sibling? false})))] + (concat-tx-fn result result2)) + result))))))))))) ;;; ### write-operations have side-effects (do transactions) ;;;;;;;;;;;;;;;; @@ -1141,6 +1140,12 @@ see also `logseq.outliner.transaction/transact!`" (atom nil)) +(defn clear-and-get-tx-data! + [] + (let [result @*transaction-data] + (reset! *transaction-data nil) + result)) + (defn- op-transact! [fn-var & args] {:pre [(var? fn-var)]} diff --git a/deps/outliner/src/logseq/outliner/transaction.cljc b/deps/outliner/src/logseq/outliner/transaction.cljc index 79384b616b..c91b1c08b9 100644 --- a/deps/outliner/src/logseq/outliner/transaction.cljc +++ b/deps/outliner/src/logseq/outliner/transaction.cljc @@ -59,7 +59,6 @@ (when (seq all-tx#) ;; If it's empty, do nothing (when-not (:nested-transaction? opts#) ; transact only for the whole transaction - (prn :debug :data @transact-data#) (logseq.outliner.datascript/transact! all-tx# (dissoc opts## :transact-opts) (:transact-opts opts##)))) (reset! logseq.outliner.core/*transaction-data nil) diff --git a/src/main/frontend/db_worker.cljs b/src/main/frontend/db_worker.cljs index 3b00e1d968..aa2b160948 100644 --- a/src/main/frontend/db_worker.cljs +++ b/src/main/frontend/db_worker.cljs @@ -24,7 +24,9 @@ [frontend.worker.async-util :include-macros true :refer [js {:result result})))) + ;; block ops + + (delete-blocks + [this repo block-ids opts-str] + (when-let [conn (worker-state/get-datascript-conn repo)] + (let [block-ids (remove nil? block-ids)] + (when (seq block-ids) + (let [opts (if opts-str + (edn/read-string opts-str) + {}) + blocks (map #(d/entity @conn %) block-ids)] + (outliner-tx/transact! + {:local-tx? true ; keep this transaction in undo/redo history + :outliner-op :delete-blocks + :transact-opts {:repo repo + :conn conn}} + (outliner-core/delete-blocks! repo conn + (worker-state/get-date-formatter repo) + blocks + opts))))) + nil)) + + (move-blocks-up-down + [this repo block-ids up?] + (when-let [conn (worker-state/get-datascript-conn repo)] + (let [block-ids (remove nil? block-ids)] + (when (seq block-ids) + (let [blocks (map #(d/entity @conn %) block-ids) + _ (outliner-core/move-blocks-up-down! repo conn blocks up?)] + (pr-str (outliner-core/clear-and-get-tx-data!))))))) + (file-writes-finished? [this repo] (let [conn (worker-state/get-datascript-conn repo) diff --git a/src/main/frontend/handler/block.cljs b/src/main/frontend/handler/block.cljs index 5f9994b2bd..4bf7369c9e 100644 --- a/src/main/frontend/handler/block.cljs +++ b/src/main/frontend/handler/block.cljs @@ -326,17 +326,20 @@ last)] (get-original-block-by-dom last-block-node))))) -(defn indent-outdent-block! - [block direction] - (ui-outliner-tx/transact! - {:outliner-op :move-blocks - :real-outliner-op :indent-outdent} - (outliner-core/indent-outdent-blocks! (state/get-current-repo) - (db/get-db false) - (get-top-level-blocks [block]) - (= direction :right) - {:get-first-block-original get-first-block-original - :logical-outdenting? (state/logical-outdenting?)}))) +(defn indent-outdent-blocks! + [blocks indent? save-current-block] + (when (seq blocks) + (let [blocks (get-top-level-blocks blocks)] + (ui-outliner-tx/transact! + {:outliner-op :move-blocks + :real-outliner-op :indent-outdent} + (when save-current-block (save-current-block)) + (outliner-core/indent-outdent-blocks! (state/get-current-repo) + (db/get-db false) + (get-top-level-blocks blocks) + indent? + {:parent-original (get-first-block-original) + :logical-outdenting? (state/logical-outdenting?)}))))) (def *swipe (atom nil)) @@ -461,13 +464,13 @@ (and left-menu (>= (.-clientWidth left-menu) 40)) (when (indentable? block) (haptics/with-haptics-impact - (indent-outdent-block! block :right) + (indent-outdent-blocks! [block] true nil) :light)) (and right-menu (<= 40 (.-clientWidth right-menu) 79)) (when (outdentable? block) (haptics/with-haptics-impact - (indent-outdent-block! block :left) + (indent-outdent-blocks! [block] false nil) :light)) (and right-menu (>= (.-clientWidth right-menu) 80)) diff --git a/src/main/frontend/handler/editor.cljs b/src/main/frontend/handler/editor.cljs index ded8af6a5d..6a61224a42 100644 --- a/src/main/frontend/handler/editor.cljs +++ b/src/main/frontend/handler/editor.cljs @@ -686,14 +686,15 @@ (when block (let [blocks (block-handler/get-top-level-blocks [block])] (state/set-state! :ui/deleting-block uuid) - (ui-outliner-tx/transact! - {:outliner-op :delete-blocks} - (outliner-core/delete-blocks! repo (db/get-db false) - (state/get-date-formatter) - blocks - (merge - delete-opts - {:children? children?}))))))) + (p/do! + (ui-outliner-tx/transact! + {:outliner-op :delete-blocks} + (outliner-core/delete-blocks! repo (db/get-db false) + (state/get-date-formatter) + blocks + (merge + delete-opts + {:children? children?})))))))) (defn- move-to-prev-block [repo sibling-block format _id value] @@ -845,9 +846,8 @@ (p/do! (ui-outliner-tx/transact! {:outliner-op :delete-blocks} - (outliner-core/delete-blocks! repo (db/get-db false) - (state/get-date-formatter) - blocks' {})) + (when-let [^Object worker @state/*db-worker] + (.delete-blocks worker repo (clj->js (map :db/id blocks')) nil))) (when sibling-block (move-to-prev-block repo sibling-block (:block/format block) @@ -1740,12 +1740,14 @@ [up?] (fn [event] (util/stop event) - (let [edit-block-id (:block/uuid (state/get-edit-block)) + (let [repo (state/get-current-repo) + edit-block-id (:block/uuid (state/get-edit-block)) move-nodes (fn [blocks] (let [blocks' (block-handler/get-top-level-blocks blocks) result (ui-outliner-tx/transact! {:outliner-op :move-blocks} - (outliner-core/move-blocks-up-down! (state/get-current-repo) (db/get-db false) blocks' up?))] + (when-let [^Object worker @state/*db-worker] + (.move-blocks-up-down worker repo (clj->js (map :db/id blocks')) up?)))] (when-let [block-node (util/get-first-block-by-id (:block/uuid (first blocks)))] (.scrollIntoView block-node #js {:behavior "smooth" :block "nearest"})) result))] @@ -1782,16 +1784,7 @@ "`direction` = :left | :right." [direction] (let [blocks (get-selected-ordered-blocks)] - (when (seq blocks) - (ui-outliner-tx/transact! - {:outliner-op :move-blocks - :real-outliner-op :indent-outdent} - (outliner-core/indent-outdent-blocks! (state/get-current-repo) - (db/get-db false) - (block-handler/get-top-level-blocks blocks) - (= direction :right) - {:get-first-block-original block-handler/get-first-block-original - :logical-outdenting? (state/logical-outdenting?)}))))) + (block-handler/indent-outdent-blocks! blocks (= direction :right) nil))) (defn- get-link [format link label] (let [link (or link "") @@ -2866,16 +2859,7 @@ {:keys [block]} (get-state)] (when block (state/set-editor-last-pos! pos) - (ui-outliner-tx/transact! - {:outliner-op :move-blocks - :real-outliner-op :indent-outdent} - (save-current-block!) - (outliner-core/indent-outdent-blocks! (state/get-current-repo) - (db/get-db false) - (block-handler/get-top-level-blocks [block]) - indent? - {:get-first-block-original block-handler/get-first-block-original - :logical-outdenting? (state/logical-outdenting?)}))))) + (block-handler/indent-outdent-blocks! [block] indent? save-current-block!)))) (defn keydown-tab-handler [direction] diff --git a/src/main/frontend/worker/state.cljs b/src/main/frontend/worker/state.cljs index 5ed6ef9b1d..3750cf3777 100644 --- a/src/main/frontend/worker/state.cljs +++ b/src/main/frontend/worker/state.cljs @@ -1,6 +1,7 @@ (ns frontend.worker.state "State hub for worker" - (:require [logseq.common.util :as common-util])) + (:require [logseq.common.util :as common-util] + [logseq.common.config :as common-config])) (defonce *state (atom {:worker/object nil @@ -79,3 +80,7 @@ (defn get-worker-object [] (:worker/object @*state)) + +(defn get-date-formatter + [repo] + (common-config/get-date-formatter (get-config repo))) From 7a4251856a6187750145a55546358ce63c940eb7 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Fri, 2 Feb 2024 00:20:41 +0800 Subject: [PATCH 33/60] Revert "fix: enable promise usage in outliner-ui/transact!" This reverts commit 95cd7d51fca1fcbe281fafbb38571a87547a208b. --- deps/outliner/src/logseq/outliner/core.cljs | 18 ++-- .../src/logseq/outliner/transaction.cljc | 46 +++++----- src/main/frontend/db/async.cljs | 12 --- src/main/frontend/db_worker.cljs | 2 +- src/main/frontend/handler/dnd.cljs | 3 +- src/main/frontend/handler/editor.cljs | 83 ++++++++++--------- .../frontend/handler/file_based/editor.cljs | 3 +- .../handler/file_based/page_property.cljs | 3 +- src/main/frontend/modules/outliner/ui.cljc | 3 +- src/main/logseq/api.cljs | 34 +++++--- 10 files changed, 98 insertions(+), 109 deletions(-) diff --git a/deps/outliner/src/logseq/outliner/core.cljs b/deps/outliner/src/logseq/outliner/core.cljs index 1e66f1a59d..26a38af315 100644 --- a/deps/outliner/src/logseq/outliner/core.cljs +++ b/deps/outliner/src/logseq/outliner/core.cljs @@ -1130,27 +1130,23 @@ ;;; ### write-operations have side-effects (do transactions) ;;;;;;;;;;;;;;;; -(def *transaction-data +(def ^:private ^:dynamic *transaction-data* "Stores transaction-data that are generated by one or more write-operations, see also `logseq.outliner.transaction/transact!`" - (atom nil)) + nil) -(def #_:clj-kondo/ignore *transaction-opts +(def ^:private ^:dynamic #_:clj-kondo/ignore *transaction-opts* "Stores transaction opts that are generated by one or more write-operations, see also `logseq.outliner.transaction/transact!`" - (atom nil)) - -(defn clear-and-get-tx-data! - [] - (let [result @*transaction-data] - (reset! *transaction-data nil) - result)) + nil) (defn- op-transact! [fn-var & args] {:pre [(var? fn-var)]} + (when (nil? *transaction-data*) + (throw (js/Error. (str (:name (meta fn-var)) " is not used in (transact! ...)")))) (let [result (apply @fn-var args)] - (swap! *transaction-data conj (select-keys result [:tx-data :tx-meta])) + (conj! *transaction-data* (select-keys result [:tx-data :tx-meta])) result)) (defn save-block! diff --git a/deps/outliner/src/logseq/outliner/transaction.cljc b/deps/outliner/src/logseq/outliner/transaction.cljc index c91b1c08b9..87be8019f3 100644 --- a/deps/outliner/src/logseq/outliner/transaction.cljc +++ b/deps/outliner/src/logseq/outliner/transaction.cljc @@ -2,8 +2,7 @@ "Provides a wrapper around logseq.outliner.datascript/transact! using transient state from logseq.outliner.core" #?(:cljs (:require-macros [logseq.outliner.transaction])) - #?(:cljs (:require [malli.core :as m] - [promesa.core :as p]))) + #?(:cljs (:require [malli.core :as m]))) #_:clj-kondo/ignore (def ^:private transact-opts [:or :symbol :map]) @@ -29,37 +28,34 @@ (move-blocks! ...) (delete-blocks! ...))" [opts & body] - `(let [transact-data# logseq.outliner.core/*transaction-data - transaction-opts# logseq.outliner.core/*transaction-opts + `(let [transact-data# logseq.outliner.core/*transaction-data* + transaction-opts# logseq.outliner.core/*transaction-opts* opts*# ~opts _# (assert (or (map? opts*#) (symbol? opts*#)) (str "opts is not a map or symbol, type: " (type opts*#))) - opts# (if @transact-data# + opts# (if transact-data# (assoc opts*# :nested-transaction? true) opts*#)] - (if @transact-data# + (if transact-data# (do - (when @transaction-opts# - (swap! transaction-opts# conj opts#)) + (when transaction-opts# + (conj! transaction-opts# opts#)) ~@body) (let [transaction-args# (cond-> {} (get opts*# :persist-op? true) (assoc :persist-op? true))] - (reset! logseq.outliner.core/*transaction-data []) - (reset! logseq.outliner.core/*transaction-opts [opts#]) - (p/do! - ~@body - (let [r# @logseq.outliner.core/*transaction-data - tx# (mapcat :tx-data r#) + (binding [logseq.outliner.core/*transaction-data* (transient []) + logseq.outliner.core/*transaction-opts* (transient [])] + (conj! logseq.outliner.core/*transaction-opts* opts#) + ~@body + (let [r# (persistent! logseq.outliner.core/*transaction-data*) + tx# (mapcat :tx-data r#) ;; FIXME: should we merge all the tx-meta? - tx-meta# (first (map :tx-meta r#)) - all-tx# (concat tx# (:additional-tx opts#)) - o# @logseq.outliner.core/*transaction-opts - full-opts# (apply merge (reverse o#)) - opts## (merge (dissoc full-opts# :additional-tx :current-block :nested-transaction?) tx-meta#)] + tx-meta# (first (map :tx-meta r#)) + all-tx# (concat tx# (:additional-tx opts#)) + o# (persistent! logseq.outliner.core/*transaction-opts*) + full-opts# (apply merge (reverse o#)) + opts## (merge (dissoc full-opts# :additional-tx :current-block :nested-transaction?) tx-meta#)] - (when (seq all-tx#) ;; If it's empty, do nothing - (when-not (:nested-transaction? opts#) ; transact only for the whole transaction - (logseq.outliner.datascript/transact! all-tx# (dissoc opts## :transact-opts) (:transact-opts opts##)))) - - (reset! logseq.outliner.core/*transaction-data nil) - (reset! logseq.outliner.core/*transaction-opts nil))))))) + (when (seq all-tx#) ;; If it's empty, do nothing + (when-not (:nested-transaction? opts#) ; transact only for the whole transaction + (logseq.outliner.datascript/transact! all-tx# (dissoc opts## :transact-opts) (:transact-opts opts##)))))))))) diff --git a/src/main/frontend/db/async.cljs b/src/main/frontend/db/async.cljs index 825e3c779e..bbb3172d4e 100644 --- a/src/main/frontend/db/async.cljs +++ b/src/main/frontend/db/async.cljs @@ -212,15 +212,3 @@ (->> result db-model/sort-by-left-recursive db-utils/group-by-page))))))) - -;; TODO: perf improvement, some operations such as delete-block doesn't need to load the full page -;; instead, the db worker should provide those calls -(defn right needs to point its `left` to block->left - (when (and block-right (not= (:db/id (:block/parent prev-block)) - (:db/id (:block/parent block)))) - (outliner-save-block! {:db/id (:db/id block-right) - :block/left (:db/id (:block/left block))})) + ;; block->right needs to point its `left` to block->left + (when (and block-right (not= (:db/id (:block/parent prev-block)) + (:db/id (:block/parent block)))) + (outliner-save-block! {:db/id (:db/id block-right) + :block/left (:db/id (:block/left block))})) ;; update prev-block's children to point to the refed block - (when (or (:block/collapsed? prev-block) - (= (:db/id prev-block) (:db/id (:block/parent block)))) - (let [children (:block/_parent prev-block)] - (doseq [child children] - (when-not (= (:db/id child) (:db/id block)) - (outliner-save-block! {:db/id (:db/id child) - :block/parent (:db/id block) - :block/left (:db/id block)}))))) + (when (or (:block/collapsed? prev-block) + (= (:db/id prev-block) (:db/id (:block/parent block)))) + (let [children (:block/_parent prev-block)] + (doseq [child children] + (when-not (= (:db/id child) (:db/id block)) + (outliner-save-block! {:db/id (:db/id child) + :block/parent (:db/id block) + :block/left (:db/id block)}))))) ;; parent will be removed - (when (= (:db/id prev-block) (:db/id (:block/parent block))) - (when-let [parent-right (outliner-core/get-right-sibling (db/get-db) (:db/id prev-block))] - (outliner-save-block! {:db/id (:db/id parent-right) - :block/left (:db/id block)}))) + (when (= (:db/id prev-block) (:db/id (:block/parent block))) + (when-let [parent-right (outliner-core/get-right-sibling (db/get-db) (:db/id prev-block))] + (outliner-save-block! {:db/id (:db/id parent-right) + :block/left (:db/id block)}))) - (when db-based? - (outliner-save-block! {:db/id (:db/id block) - :block/properties new-properties})))) + (when db-based? + (outliner-save-block! {:db/id (:db/id block) + :block/properties new-properties}))) - (p/do! - (delete-block-fn block) - (save-block! repo prev-block new-content {}) - (when db-based? - (outliner-save-block! {:db/id (:db/id prev-block) - :block/properties new-properties}))))) + (do + (delete-block-fn block) + (save-block! repo prev-block new-content {}) + (when db-based? + (outliner-save-block! {:db/id (:db/id prev-block) + :block/properties new-properties}))))) :else (delete-block-fn block))))))))))))) @@ -2861,6 +2861,7 @@ (state/set-editor-last-pos! pos) (block-handler/indent-outdent-blocks! [block] indent? save-current-block!)))) + (defn keydown-tab-handler [direction] (fn [e] diff --git a/src/main/frontend/handler/file_based/editor.cljs b/src/main/frontend/handler/file_based/editor.cljs index e5fde6f28d..c3e82e7c16 100644 --- a/src/main/frontend/handler/file_based/editor.cljs +++ b/src/main/frontend/handler/file_based/editor.cljs @@ -17,8 +17,7 @@ [frontend.handler.file-based.property :as file-property-handler] [frontend.handler.file-based.property.util :as property-util] [logseq.db.frontend.schema :as db-schema] - [logseq.common.util.block-ref :as block-ref] - [promesa.core :as p])) + [logseq.common.util.block-ref :as block-ref])) (defn- remove-non-existed-refs! [refs] diff --git a/src/main/frontend/handler/file_based/page_property.cljs b/src/main/frontend/handler/file_based/page_property.cljs index 75f5142cc5..6d42544d82 100644 --- a/src/main/frontend/handler/file_based/page_property.cljs +++ b/src/main/frontend/handler/file_based/page_property.cljs @@ -5,8 +5,7 @@ [logseq.outliner.core :as outliner-core] [frontend.modules.outliner.ui :as ui-outliner-tx] [frontend.state :as state] - [frontend.util :as util] - [promesa.core :as p])) + [frontend.util :as util])) (defn insert-property [format content key value] diff --git a/src/main/frontend/modules/outliner/ui.cljc b/src/main/frontend/modules/outliner/ui.cljc index 983796ab0b..5227d715ff 100644 --- a/src/main/frontend/modules/outliner/ui.cljc +++ b/src/main/frontend/modules/outliner/ui.cljc @@ -3,8 +3,7 @@ #?(:cljs (:require-macros [frontend.modules.outliner.ui])) #?(:cljs (:require [frontend.state :as state] [frontend.config :as config] - [frontend.db :as db] - [promesa.core :as p]))) + [frontend.db :as db]))) #?(:cljs (do diff --git a/src/main/logseq/api.cljs b/src/main/logseq/api.cljs index 5c94606d75..7ce8543377 100644 --- a/src/main/logseq/api.cljs +++ b/src/main/logseq/api.cljs @@ -615,12 +615,24 @@ (let [{:keys [pos] :or {pos :max}} (bean/->clj opts)] (editor-handler/edit-block! block pos block-uuid)))))) +;; TODO: perf improvement, some operations such as delete-block doesn't need to load the full page +;; instead, the db worker should provide those calls +(defn- clj opts) @@ -667,7 +679,7 @@ (def ^:export insert_batch_block (fn [block-uuid ^js batch-blocks ^js opts] - (p/let [block (db-async/clj batch-blocks)] (let [bb (if-not (vector? bb) (vector bb) bb) @@ -688,21 +700,21 @@ (def ^:export remove_block (fn [block-uuid ^js _opts] (p/let [repo (state/get-current-repo) - _ (db-async/clj opts))))) (def ^:export move_block (fn [src-block-uuid target-block-uuid ^js opts] - (p/let [_ (db-async/clj opts) move-to (cond (boolean before) @@ -735,7 +747,7 @@ (def ^:export get_previous_sibling_block (fn [block-uuid] (p/let [id (sdk-utils/uuid-or-throw-error block-uuid) - _ (db-async/vec-tree blocks page-name) @@ -849,7 +861,7 @@ (defn ^:export prepend_block_in_page [uuid-or-page-name content ^js opts] - (p/let [_ (db-async/ Date: Fri, 2 Feb 2024 02:16:33 +0800 Subject: [PATCH 34/60] refactor: build tx-data in db-worker instead of the UI thread So the tx-data will be built on top of the full db instead of the partial db from the UI thread, it also reduces the risk to use outdated data since RTC can transact data from the worker. --- src/main/frontend/db_worker.cljs | 37 +----- src/main/frontend/handler/block.cljs | 11 +- src/main/frontend/handler/common/page.cljs | 24 ++-- .../frontend/handler/db_based/editor.cljs | 6 +- src/main/frontend/handler/dnd.cljs | 12 +- src/main/frontend/handler/editor.cljs | 113 ++++++++---------- .../frontend/handler/file_based/editor.cljs | 6 +- .../handler/file_based/page_property.cljs | 4 +- .../frontend/handler/file_based/property.cljs | 6 +- src/main/frontend/handler/page.cljs | 18 ++- src/main/frontend/modules/outliner/op.cljs | 69 +++++++++++ src/main/frontend/modules/outliner/ui.cljc | 31 ++--- src/main/frontend/worker/outliner_op.cljs | 53 ++++++++ 13 files changed, 227 insertions(+), 163 deletions(-) create mode 100644 src/main/frontend/modules/outliner/op.cljs create mode 100644 src/main/frontend/worker/outliner_op.cljs diff --git a/src/main/frontend/db_worker.cljs b/src/main/frontend/db_worker.cljs index e9afbf7e32..92b196b762 100644 --- a/src/main/frontend/db_worker.cljs +++ b/src/main/frontend/db_worker.cljs @@ -25,8 +25,7 @@ [frontend.worker.util :as worker-util] [frontend.worker.handler.page.rename :as worker-page-rename] [frontend.worker.handler.page :as worker-page] - [logseq.outliner.core :as outliner-core] - [logseq.outliner.transaction :as outliner-tx])) + [frontend.worker.outliner-op :as worker-outliner-op])) (defonce *sqlite worker-state/*sqlite) (defonce *sqlite-conns worker-state/*sqlite-conns) @@ -470,36 +469,12 @@ (let [result (worker-page/delete! repo conn page-name nil {})] (bean/->js {:result result})))) - ;; block ops - - (delete-blocks - [this repo block-ids opts-str] + (apply-outliner-ops + [this repo ops-str opts-str] (when-let [conn (worker-state/get-datascript-conn repo)] - (let [block-ids (remove nil? block-ids)] - (when (seq block-ids) - (let [opts (if opts-str - (edn/read-string opts-str) - {}) - blocks (map #(d/entity @conn %) block-ids)] - (outliner-tx/transact! - {:local-tx? true ; keep this transaction in undo/redo history - :outliner-op :delete-blocks - :transact-opts {:repo repo - :conn conn}} - (outliner-core/delete-blocks! repo conn - (worker-state/get-date-formatter repo) - blocks - opts))))) - nil)) - - (move-blocks-up-down - [this repo block-ids up?] - (when-let [conn (worker-state/get-datascript-conn repo)] - (let [block-ids (remove nil? block-ids)] - (when (seq block-ids) - (let [blocks (map #(d/entity @conn %) block-ids) - _ (outliner-core/move-blocks-up-down! repo conn blocks up?)] - ))))) + (let [ops (edn/read-string ops-str) + opts (edn/read-string opts-str)] + (worker-outliner-op/apply-ops! repo conn ops opts)))) (file-writes-finished? [this repo] diff --git a/src/main/frontend/handler/block.cljs b/src/main/frontend/handler/block.cljs index 4bf7369c9e..d4e0f770a0 100644 --- a/src/main/frontend/handler/block.cljs +++ b/src/main/frontend/handler/block.cljs @@ -8,6 +8,7 @@ [frontend.mobile.haptics :as haptics] [logseq.outliner.core :as outliner-core] [frontend.modules.outliner.ui :as ui-outliner-tx] + [frontend.modules.outliner.op :as outliner-op] [frontend.state :as state] [frontend.util :as util] [frontend.util.drawer :as drawer] @@ -334,12 +335,10 @@ {:outliner-op :move-blocks :real-outliner-op :indent-outdent} (when save-current-block (save-current-block)) - (outliner-core/indent-outdent-blocks! (state/get-current-repo) - (db/get-db false) - (get-top-level-blocks blocks) - indent? - {:parent-original (get-first-block-original) - :logical-outdenting? (state/logical-outdenting?)}))))) + (outliner-op/indent-outdent-blocks! (get-top-level-blocks blocks) + indent? + {:parent-original (get-first-block-original) + :logical-outdenting? (state/logical-outdenting?)}))))) (def *swipe (atom nil)) diff --git a/src/main/frontend/handler/common/page.cljs b/src/main/frontend/handler/common/page.cljs index 4ad2aa181a..2c1be6d455 100644 --- a/src/main/frontend/handler/common/page.cljs +++ b/src/main/frontend/handler/common/page.cljs @@ -21,7 +21,7 @@ [frontend.db.conn :as conn] [datascript.core :as d] [frontend.modules.outliner.ui :as ui-outliner-tx] - [logseq.outliner.core :as outliner-core])) + [frontend.modules.outliner.op :as outliner-op])) (defn build-hidden-page-tx-data [page-name] @@ -126,28 +126,26 @@ (defn js (map :db/id blocks')) nil))) + (outliner-op/delete-blocks! blocks' nil)) (when sibling-block (move-to-prev-block repo sibling-block (:block/format block) @@ -1740,14 +1734,12 @@ [up?] (fn [event] (util/stop event) - (let [repo (state/get-current-repo) - edit-block-id (:block/uuid (state/get-edit-block)) + (let [edit-block-id (:block/uuid (state/get-edit-block)) move-nodes (fn [blocks] (let [blocks' (block-handler/get-top-level-blocks blocks) result (ui-outliner-tx/transact! {:outliner-op :move-blocks} - (when-let [^Object worker @state/*db-worker] - (.move-blocks-up-down worker repo (clj->js (map :db/id blocks')) up?)))] + (outliner-op/move-blocks-up-down! blocks' up?))] (when-let [block-node (util/get-first-block-by-id (:block/uuid (first blocks)))] (.scrollIntoView block-node #js {:behavior "smooth" :block "nearest"})) result))] @@ -2046,7 +2038,7 @@ page (if (:block/name block) block (when target-block (:block/page (db/entity (:db/id target-block))))) empty-target? (if (true? skip-empty-target?) false - (string/blank? (:block/content target-block))) + (string/blank? (:block/content target-block))) paste-nested-blocks? (nested-blocks blocks) target-block-has-children? (db/has-children? (:block/uuid target-block)) replace-empty-target? (and empty-target? @@ -2074,23 +2066,21 @@ {:outliner-op :save-block} (outliner-save-block! editing-block))) - (p/let [*insert-result (atom nil) - _ (ui-outliner-tx/transact! - {:outliner-op :insert-blocks - :additional-tx revert-cut-txs} - (when target-block' - (let [format (or (:block/format target-block') (state/get-preferred-format)) - repo (state/get-current-repo) - blocks' (map (fn [block] - (paste-block-cleanup repo block page exclude-properties format content-update-fn keep-uuid?)) - blocks) - result (outliner-core/insert-blocks! repo (db/get-db false) blocks' target-block' {:sibling? sibling? - :outliner-op :paste - :replace-empty-target? replace-empty-target? - :keep-uuid? keep-uuid?})] - (reset! *insert-result result))))] + (p/let [result (ui-outliner-tx/transact! + {:outliner-op :insert-blocks + :additional-tx revert-cut-txs} + (when target-block' + (let [format (or (:block/format target-block') (state/get-preferred-format)) + repo (state/get-current-repo) + blocks' (map (fn [block] + (paste-block-cleanup repo block page exclude-properties format content-update-fn keep-uuid?)) + blocks)] + (outliner-op/insert-blocks! blocks' target-block' {:sibling? sibling? + :outliner-op :paste + :replace-empty-target? replace-empty-target? + :keep-uuid? keep-uuid?}))))] (state/set-block-op-type! nil) - (when-let [result @*insert-result] (edit-last-block-after-inserted! result))))) + (when result (edit-last-block-after-inserted! (edn/read-string result)))))) (defn- block-tree->blocks "keep-uuid? - maintain the existing :uuid in tree vec" @@ -2190,18 +2180,14 @@ :else true)] (try - (let [*result (atom nil)] - (p/do! - (ui-outliner-tx/transact! - {:outliner-op :insert-blocks - :created-from-journal-template? journal?} - (when-not (string/blank? (state/get-edit-content)) - (save-current-block!)) - (let [result (outliner-core/insert-blocks! repo (db/get-db false) blocks' - target - (assoc opts :sibling? sibling?'))] - (reset! *result result))) - (some-> @*result edit-last-block-after-inserted!))) + (p/let [result (ui-outliner-tx/transact! + {:outliner-op :insert-blocks + :created-from-journal-template? journal?} + (when-not (string/blank? (state/get-edit-content)) + (save-current-block!)) + (outliner-op/insert-blocks! blocks' target + (assoc opts :sibling? sibling?')))] + (when result (edit-last-block-after-inserted! (edn/read-string result)))) (catch :default ^js/Error e (notification/show! @@ -2266,9 +2252,8 @@ :real-outliner-op :indent-outdent} (save-current-block!) (when target - (outliner-core/move-blocks! (state/get-current-repo) (db/get-db false) - (block-handler/get-top-level-blocks [block]) - target true))) + (outliner-op/move-blocks! (block-handler/get-top-level-blocks [block]) + target true))) (when original-block (util/schedule #(edit-block! block pos nil)))))) diff --git a/src/main/frontend/handler/file_based/editor.cljs b/src/main/frontend/handler/file_based/editor.cljs index c3e82e7c16..5a942df3be 100644 --- a/src/main/frontend/handler/file_based/editor.cljs +++ b/src/main/frontend/handler/file_based/editor.cljs @@ -6,8 +6,8 @@ [frontend.format.block :as block] [frontend.db :as db] [frontend.format.mldoc :as mldoc] - [logseq.outliner.core :as outliner-core] [frontend.state :as state] + [frontend.modules.outliner.op :as outliner-op] [frontend.modules.outliner.ui :as ui-outliner-tx] [frontend.util :as util] [frontend.util.clock :as clock] @@ -195,9 +195,7 @@ {:outliner-op :save-block} (doseq [block-id block-ids] (when-let [block (set-heading-aux! block-id heading)] - (outliner-core/save-block! (state/get-current-repo) (db/get-db false) - (state/get-date-formatter) - block))))) + (outliner-op/save-block! block))))) (defn set-blocks-id! "Persist block uuid to file if the uuid is valid, and it's not persisted in file. diff --git a/src/main/frontend/handler/file_based/page_property.cljs b/src/main/frontend/handler/file_based/page_property.cljs index 6d42544d82..3493e3fb46 100644 --- a/src/main/frontend/handler/file_based/page_property.cljs +++ b/src/main/frontend/handler/file_based/page_property.cljs @@ -2,7 +2,7 @@ "Page property fns for file graphs" (:require [clojure.string :as string] [frontend.db :as db] - [logseq.outliner.core :as outliner-core] + [frontend.modules.outliner.op :as outliner-op] [frontend.modules.outliner.ui :as ui-outliner-tx] [frontend.state :as state] [frontend.util :as util])) @@ -85,4 +85,4 @@ (ui-outliner-tx/transact! {:outliner-op :insert-blocks :additional-tx page-properties-tx} - (outliner-core/insert-blocks! repo (db/get-db false) block page {:sibling? false})))))))) + (outliner-op/insert-blocks! block page {:sibling? false})))))))) diff --git a/src/main/frontend/handler/file_based/property.cljs b/src/main/frontend/handler/file_based/property.cljs index fb28821a56..f61bfd30ed 100644 --- a/src/main/frontend/handler/file_based/property.cljs +++ b/src/main/frontend/handler/file_based/property.cljs @@ -3,7 +3,7 @@ (:require [frontend.db :as db] [frontend.handler.block :as block-handler] [frontend.handler.file-based.property.util :as property-util] - [logseq.outliner.core :as outliner-core] + [frontend.modules.outliner.op :as outliner-op] [frontend.modules.outliner.ui :as ui-outliner-tx] [frontend.state :as state] [logseq.common.util :as common-util] @@ -55,9 +55,7 @@ :block/properties-order property-ks :block/properties-text-values properties-text-values :block/content content}] - (outliner-core/save-block! (state/get-current-repo) (db/get-db false) - (state/get-date-formatter) - block)))))) + (outliner-op/save-block! block)))))) (let [block-id (ffirst col) block-id (if (string? block-id) (uuid block-id) block-id) input-pos (or (state/get-edit-pos) :max)] diff --git a/src/main/frontend/handler/page.cljs b/src/main/frontend/handler/page.cljs index 6983bf723f..29b8e15b9a 100644 --- a/src/main/frontend/handler/page.cljs +++ b/src/main/frontend/handler/page.cljs @@ -41,7 +41,7 @@ [frontend.db.conn :as conn] [logseq.db :as ldb] [frontend.modules.outliner.ui :as ui-outliner-tx] - [logseq.outliner.core :as outliner-core])) + [frontend.modules.outliner.op :as outliner-op])) (def create! page-common-handler/create!) (def Date: Fri, 2 Feb 2024 03:46:12 +0800 Subject: [PATCH 35/60] fix: add db requests for outliner core tx --- deps/db/src/logseq/db.cljs | 12 ++++++++++-- deps/outliner/src/logseq/outliner/core.cljs | 11 ++++++----- src/main/frontend/modules/outliner/op.cljs | 8 ++++++-- src/main/frontend/modules/outliner/pipeline.cljs | 6 +++--- src/main/frontend/modules/outliner/ui.cljc | 7 ++++++- src/main/frontend/state.cljs | 4 ++++ 6 files changed, 35 insertions(+), 13 deletions(-) diff --git a/deps/db/src/logseq/db.cljs b/deps/db/src/logseq/db.cljs index accdb18f38..294f0c21f0 100644 --- a/deps/db/src/logseq/db.cljs +++ b/deps/db/src/logseq/db.cljs @@ -70,6 +70,14 @@ (when-let [callback (:callback (get new request-id))] (callback))))) +(defn get-next-request-id + [] + (swap! *request-id inc)) + +(defn add-request! + [request-id data] + (swap! *request-id->response assoc request-id (if (map? data) data {:response data}))) + (defn transact! "`repo-or-conn`: repo for UI thread and conn for worker/node" ([repo-or-conn tx-data] @@ -87,7 +95,7 @@ (let [f (or @*transact-fn d/transact!) sync? (= f d/transact!) - request-id (when-not sync? (swap! *request-id inc)) + request-id (when-not sync? (get-next-request-id)) tx-meta' (cond-> tx-meta (not sync?) (assoc :request-id request-id))] @@ -100,7 +108,7 @@ {:response resp} {:response resp :callback #(f repo-or-conn tx-data tx-meta')})] - (swap! *request-id->response assoc request-id value)) + (add-request! request-id value)) resp))))))) (defn build-default-pages-tx diff --git a/deps/outliner/src/logseq/outliner/core.cljs b/deps/outliner/src/logseq/outliner/core.cljs index 26a38af315..590d65e3b2 100644 --- a/deps/outliner/src/logseq/outliner/core.cljs +++ b/deps/outliner/src/logseq/outliner/core.cljs @@ -370,11 +370,12 @@ m (if db-based? (dissoc m :block/tags) m)] ;; Ensure block UUID never changes - (when (and db-id block-uuid) - (let [uuid-not-changed? (= block-uuid (:block/uuid (d/entity db db-id)))] - (when-not uuid-not-changed? - (js/console.error "Block UUID shouldn't be changed once created")) - (assert uuid-not-changed? "Block UUID changed"))) + (let [e (d/entity db db-id)] + (when (and e block-uuid) + (let [uuid-not-changed? (= block-uuid (:block/uuid e))] + (when-not uuid-not-changed? + (js/console.error "Block UUID shouldn't be changed once created")) + (assert uuid-not-changed? "Block UUID changed")))) (when eid ;; Retract attributes to prepare for tx which rewrites block attributes diff --git a/src/main/frontend/modules/outliner/op.cljs b/src/main/frontend/modules/outliner/op.cljs index 8e43c46811..8818177ee8 100644 --- a/src/main/frontend/modules/outliner/op.cljs +++ b/src/main/frontend/modules/outliner/op.cljs @@ -1,5 +1,6 @@ (ns frontend.modules.outliner.op - "Build outliner ops") + "Build outliner ops" + (:require [datascript.impl.entity :as de])) (def ^:private ^:dynamic *outliner-ops* "Stores outliner ops that are generated by the following calls" @@ -16,7 +17,10 @@ (defn save-block [block] - [:save-block [block]]) + (when-let [block' (if (de/entity? block) + (assoc (.-kv ^js block) :db/id (:db/id block)) + block)] + [:save-block [block']])) (defn insert-blocks [blocks target-block opts] diff --git a/src/main/frontend/modules/outliner/pipeline.cljs b/src/main/frontend/modules/outliner/pipeline.cljs index 292839e9a2..1caf0e3bfd 100644 --- a/src/main/frontend/modules/outliner/pipeline.cljs +++ b/src/main/frontend/modules/outliner/pipeline.cljs @@ -105,6 +105,6 @@ (when request-id (when-let [deferred (ldb/get-deferred-response request-id)] - (p/resolve! deferred {:tx-meta tx-meta - :tx-data tx-data}) - (swap! ldb/*request-id->response dissoc request-id))))) + (when (p/promise? deferred) + (p/resolve! deferred {:tx-meta tx-meta :tx-data tx-data}))) + (swap! ldb/*request-id->response dissoc request-id)))) diff --git a/src/main/frontend/modules/outliner/ui.cljc b/src/main/frontend/modules/outliner/ui.cljc index b775264a58..71764114c8 100644 --- a/src/main/frontend/modules/outliner/ui.cljc +++ b/src/main/frontend/modules/outliner/ui.cljc @@ -16,4 +16,9 @@ (let [r# (persistent! frontend.modules.outliner.op/*outliner-ops*) worker# @state/*db-worker] (when (and worker# (seq r#)) - (.apply-outliner-ops ^Object worker# (state/get-current-repo) (pr-str r#) (pr-str ~opts))))))))) + (let [request-id# (state/get-worker-next-request-id) + response# (.apply-outliner-ops ^Object worker# (state/get-current-repo) + (pr-str r#) + (pr-str (assoc ~opts :request-id request-id#)))] + (state/add-worker-request! request-id# :outliner-tx) + response#)))))))) diff --git a/src/main/frontend/state.cljs b/src/main/frontend/state.cljs index ecb69ed1f8..3141da589a 100644 --- a/src/main/frontend/state.cljs +++ b/src/main/frontend/state.cljs @@ -16,6 +16,7 @@ [goog.dom :as gdom] [goog.object :as gobj] [logseq.common.config :as common-config] + [logseq.db :as ldb] [medley.core :as medley] [promesa.core :as p] [rum.core :as rum] @@ -2371,3 +2372,6 @@ Similar to re-frame subscriptions" (defn update-favorites-updated! [] (update-state! :favorites/updated? inc)) + +(def get-worker-next-request-id ldb/get-next-request-id) +(def add-worker-request! ldb/add-request!) From 8c4d8644e66b121138276b6f1cd577a757b68e2e Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Fri, 2 Feb 2024 16:09:53 +0800 Subject: [PATCH 36/60] enhance: add malli schema for outliner ops --- src/main/frontend/worker/outliner_op.cljs | 40 ++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/src/main/frontend/worker/outliner_op.cljs b/src/main/frontend/worker/outliner_op.cljs index 363eef00fc..aa13b1177a 100644 --- a/src/main/frontend/worker/outliner_op.cljs +++ b/src/main/frontend/worker/outliner_op.cljs @@ -4,10 +4,48 @@ [logseq.outliner.core :as outliner-core] [frontend.worker.state :as worker-state] [datascript.core :as d] - [promesa.core :as p])) + [promesa.core :as p] + [malli.core :as m])) + +(def op-schema + [:multi {:dispatch first} + [:save-block + [:catn + [:op :keyword] + [:args [:tuple ::block]]]] + [:insert-blocks + [:catn + [:op :keyword] + [:args [:tuple ::blocks ::id ::option]]]] + [:delete-blocks + [:catn + [:op :keyword] + [:args [:tuple ::ids ::option]]]] + [:move-blocks + [:catn + [:op :keyword] + [:args [:tuple ::ids ::id :boolean]]]] + [:move-blocks-up-down + [:catn + [:op :keyword] + [:args [:tuple ::ids :boolean]]]] + [:indent-outdent-blocks + [:catn + [:op :keyword] + [:args [:tuple ::ids :boolean ::option]]]]]) + +(def ops-schema [:schema {:registry {::id int? + ::block map? + ::option [:maybe map?] + ::blocks [:sequential ::block] + ::ids [:sequential ::id]}} + [:sequential op-schema]]) + +(def ops-validator (m/validator ops-schema)) (defn apply-ops! [repo conn ops opts] + (assert (ops-validator ops) ops) (let [opts' (assoc opts :transact-opts {:repo repo :conn conn} :local-tx? true) From 8f845a714563046d69eccbaca566e089a356439e Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Sat, 3 Feb 2024 07:09:52 +0800 Subject: [PATCH 37/60] attach block left and parent for > (d/pull db '[*] (:db/id block)) + (with-parent-and-left db))] (cond-> - {:block block-with-refs} + {:block block'} children? (assoc :children (get-children (:block/_parent block))))) (when-let [block (or block (d/entity db [:block/name name]))] diff --git a/src/main/frontend/db/async.cljs b/src/main/frontend/db/async.cljs index bbb3172d4e..5d83a5e079 100644 --- a/src/main/frontend/db/async.cljs +++ b/src/main/frontend/db/async.cljs @@ -145,6 +145,16 @@ block result')))))) +(defn > (d/pull @conn selector id) + (sqlite-common-db/with-parent-and-left @conn))] (pr-str result)))) (pull-many @@ -291,11 +292,13 @@ (let [selector (edn/read-string selector-str) ids (edn/read-string ids-str) result (d/pull-many @conn selector ids)] - (let [] - (pr-str result))))) + (pr-str result)))) - (get-block-neighbors - [_this repo id]) + (get-right-sibling + [_this repo db-id] + (when-let [conn (worker-state/get-datascript-conn repo)] + (let [result (ldb/get-right-sibling @conn db-id)] + (pr-str result)))) (get-block-and-children [_this repo name children?] diff --git a/src/main/frontend/handler/editor.cljs b/src/main/frontend/handler/editor.cljs index 706c389a95..44d77c7fd9 100644 --- a/src/main/frontend/handler/editor.cljs +++ b/src/main/frontend/handler/editor.cljs @@ -764,16 +764,18 @@ (delete-block-aux! block delete-children? {:children-check? false}))] (when-not (and has-children? left-has-children?) (when block-parent-id - (let [block-parent (gdom/getElement block-parent-id) - sibling-block (if (:embed? config) - (util/get-prev-block-non-collapsed - block-parent - {:container (util/rec-get-blocks-container block-parent)}) - (util/get-prev-block-non-collapsed-non-embed block-parent)) - {:keys [prev-block new-content]} (move-to-prev-block repo sibling-block format id value) - concat-prev-block? (boolean (and prev-block new-content)) - transact-opts {:outliner-op :delete-blocks} - db-based? (config/db-based-graph? repo)] + (p/let [block-parent (gdom/getElement block-parent-id) + sibling-block (if (:embed? config) + (util/get-prev-block-non-collapsed + block-parent + {:container (util/rec-get-blocks-container block-parent)}) + (util/get-prev-block-non-collapsed-non-embed block-parent)) + {:keys [prev-block new-content]} (move-to-prev-block repo sibling-block format id value) + concat-prev-block? (boolean (and prev-block new-content)) + transact-opts {:outliner-op :delete-blocks} + db-based? (config/db-based-graph? repo) + block-right (db-async/clj opts) @@ -700,21 +701,21 @@ (def ^:export remove_block (fn [block-uuid ^js _opts] (p/let [repo (state/get-current-repo) - _ (clj opts))))) (def ^:export move_block (fn [src-block-uuid target-block-uuid ^js opts] - (p/let [_ (clj opts) move-to (cond (boolean before) @@ -747,20 +748,21 @@ (def ^:export get_previous_sibling_block (fn [block-uuid] (p/let [id (sdk-utils/uuid-or-throw-error block-uuid) - _ (js (sdk-utils/normalize-keyword-for-json block)))))))) + block (js (sdk-utils/normalize-keyword-for-json block)))))))) (def ^:export get_next_sibling_block (fn [block-uuid] (p/let [id (sdk-utils/uuid-or-throw-error block-uuid) - _ (js (sdk-utils/normalize-keyword-for-json block)))))))) + block (js (sdk-utils/normalize-keyword-for-json sibling)))))))) (def ^:export set_block_collapsed (fn [block-uuid ^js opts] @@ -861,7 +863,7 @@ (defn ^:export prepend_block_in_page [uuid-or-page-name content ^js opts] - (p/let [_ ( Date: Sat, 3 Feb 2024 08:11:01 +0800 Subject: [PATCH 38/60] fix: block breadcrumb --- deps/shui/src/logseq/shui/list_item/v1.cljs | 2 +- src/main/frontend/components/block.cljs | 17 +++++++++++++---- src/main/frontend/db/async.cljs | 13 +++++++++++++ src/main/frontend/db_worker.cljs | 8 ++++++++ 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/deps/shui/src/logseq/shui/list_item/v1.cljs b/deps/shui/src/logseq/shui/list_item/v1.cljs index fc8b506c2c..e75735313a 100644 --- a/deps/shui/src/logseq/shui/list_item/v1.cljs +++ b/deps/shui/src/logseq/shui/list_item/v1.cljs @@ -58,7 +58,7 @@ (recur more)))))) (defn highlight-query* [app-config query text] - (if (vector? text) ; hiccup + (if (or (vector? text) (object? text)) ; hiccup text (let [text-string (to-string text)] (if-not (seq text-string) diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index b5f24693f6..34778b3684 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -2644,21 +2644,30 @@ (ui/icon "chevron-right" {:style {:font-size 20} :class "opacity-50 mx-1"})) -(defn breadcrumb - "block-id - uuid of the target block of breadcrumb. page uuid is also acceptable" +;; "block-id - uuid of the target block of breadcrumb. page uuid is also acceptable" +(rum/defc breadcrumb < rum/reactive + {:init (fn [state] + (let [args (:rum/args state) + block-id (nth args 2) + depth (:level-limit (last args))] + (p/let [id (:db/id (db/entity [:block/uuid block-id]))] + (when id (db-async/> (ldb/get-block-parents @conn block-id {:depth (or depth 3)}) + (map (fn [b] (d/pull @conn '[*] (:db/id b)))))] + (pr-str parents)))) + (get-page-unlinked-refs [_this repo page-id search-result-eids-str] (when-let [conn (worker-state/get-datascript-conn repo)] From b8b01071233ccb36aa2564c453d20121a76856d0 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Sat, 3 Feb 2024 08:51:23 +0800 Subject: [PATCH 39/60] fix: query result display --- src/main/frontend/components/block.cljs | 2 +- src/main/frontend/db/async/util.cljs | 20 +++++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index 34778b3684..45b9e41ce7 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -3151,7 +3151,7 @@ *navigating-block (get state ::navigating-block) navigating-block (rum/react *navigating-block) navigated? (and (not= (:block/uuid block) navigating-block) navigating-block) - _ (state/sub-async-query-loading (:block/uuid block))] + _ (when (:block/uuid block) (state/sub-async-query-loading (:block/uuid block)))] (when (:block/uuid block) (let [[original-block block] (build-block config block {:navigating-block navigating-block :navigated? navigated?}) config' (if original-block diff --git a/src/main/frontend/db/async/util.cljs b/src/main/frontend/db/async/util.cljs index 0880f265e0..7283cab530 100644 --- a/src/main/frontend/db/async/util.cljs +++ b/src/main/frontend/db/async/util.cljs @@ -2,7 +2,9 @@ "Async util helper" (:require [frontend.state :as state] [promesa.core :as p] - [clojure.edn :as edn])) + [clojure.edn :as edn] + [frontend.db.conn :as db-conn] + [datascript.core :as d])) (defn Date: Sat, 3 Feb 2024 10:34:21 +0800 Subject: [PATCH 40/60] avoid async queries if blocks are not rendered yet --- .../src/logseq/outliner/datascript.cljs | 10 ++--- src/main/frontend/components/block.cljs | 39 ++++++++++++------- src/main/frontend/db/async.cljs | 3 +- src/main/frontend/db/async/util.cljs | 5 ++- 4 files changed, 33 insertions(+), 24 deletions(-) diff --git a/deps/outliner/src/logseq/outliner/datascript.cljs b/deps/outliner/src/logseq/outliner/datascript.cljs index e231050279..6b67eef453 100644 --- a/deps/outliner/src/logseq/outliner/datascript.cljs +++ b/deps/outliner/src/logseq/outliner/datascript.cljs @@ -76,9 +76,8 @@ txs)) (defn transact! - [txs tx-meta {:keys [repo conn unlinked-graph? set-state-fn]}] - (let [db-based? (and repo (sqlite-util/db-based-graph? repo)) - txs (map (fn [m] + [txs tx-meta {:keys [repo conn set-state-fn]}] + (let [txs (map (fn [m] (if (map? m) (dissoc m :block/children :block/meta :block/top? :block/bottom? :block/anchor :block/title :block/body :block/level :block/container :db/other-tx @@ -92,10 +91,7 @@ true (distinct))] - (when (and (seq txs) - (or db-based? - (and (fn? unlinked-graph?) (not (unlinked-graph?))) - (some? js/process))) + (when (seq txs) ;; (prn :debug "DB transact") ;; (cljs.pprint/pprint txs) diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index 45b9e41ce7..875b0a8809 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -2945,7 +2945,18 @@ *hidden? (get-hidden-atom id *ref {:initial-value (when (or disable-lazy? editing?) false) :id (:db/id current-block) - :content (:block/content current-block)})] + :content (:block/content current-block)}) + Date: Sat, 3 Feb 2024 10:54:27 +0800 Subject: [PATCH 41/60] fix: indent on file graphs --- deps/db/src/logseq/db/sqlite/common_db.cljs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/deps/db/src/logseq/db/sqlite/common_db.cljs b/deps/db/src/logseq/db/sqlite/common_db.cljs index 0f8cefc2df..5588fdb83a 100644 --- a/deps/db/src/logseq/db/sqlite/common_db.cljs +++ b/deps/db/src/logseq/db/sqlite/common_db.cljs @@ -30,11 +30,15 @@ (defn with-parent-and-left [db block] - (if (:block/name block) + (cond + (:block/name block) block + (:block/page block) (assoc block :block/left (select-keys (d/entity db (:db/id (:block/left block))) [:db/id :block/uuid]) - :block/parent (select-keys (d/entity db (:db/id (:block/parent block))) [:db/id :block/uuid])))) + :block/parent (select-keys (d/entity db (:db/id (:block/parent block))) [:db/id :block/uuid])) + :else + block)) (defn get-block-and-children [db name children?] From 5363aa872ed6338fc9e27e7e0083bc8dd402a826 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Sat, 3 Feb 2024 11:01:49 +0800 Subject: [PATCH 42/60] fix: deleting block not works when pressing backspace too fast --- src/main/frontend/handler/editor.cljs | 34 +++++++++++++-------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/main/frontend/handler/editor.cljs b/src/main/frontend/handler/editor.cljs index 44d77c7fd9..e1d8fdb020 100644 --- a/src/main/frontend/handler/editor.cljs +++ b/src/main/frontend/handler/editor.cljs @@ -764,18 +764,17 @@ (delete-block-aux! block delete-children? {:children-check? false}))] (when-not (and has-children? left-has-children?) (when block-parent-id - (p/let [block-parent (gdom/getElement block-parent-id) - sibling-block (if (:embed? config) - (util/get-prev-block-non-collapsed - block-parent - {:container (util/rec-get-blocks-container block-parent)}) - (util/get-prev-block-non-collapsed-non-embed block-parent)) - {:keys [prev-block new-content]} (move-to-prev-block repo sibling-block format id value) - concat-prev-block? (boolean (and prev-block new-content)) - transact-opts {:outliner-op :delete-blocks} - db-based? (config/db-based-graph? repo) - block-right (db-async/right needs to point its `left` to block->left - (when (and block-right (not= (:db/id (:block/parent prev-block)) - (:db/id (:block/parent block)))) - (outliner-save-block! {:db/id (:db/id block-right) - :block/left (:db/id (:block/left block))})) + (let [block-right (outliner-core/get-right-sibling db (:db/id block))] + (when (and block-right (not= (:db/id (:block/parent prev-block)) + (:db/id (:block/parent block)))) + (outliner-save-block! {:db/id (:db/id block-right) + :block/left (:db/id (:block/left block))}))) ;; update prev-block's children to point to the refed block (when (or (:block/collapsed? prev-block) @@ -814,7 +814,7 @@ ;; parent will be removed (when (= (:db/id prev-block) (:db/id (:block/parent block))) - (when parent-right + (when-let [parent-right (when prev-block (outliner-core/get-right-sibling db (:db/id prev-block)))] (outliner-save-block! {:db/id (:db/id parent-right) :block/left (:db/id block)}))) From 873fdc6339ccd6c941aa91e9ce06c2844298a7d2 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Sat, 3 Feb 2024 12:42:45 +0800 Subject: [PATCH 43/60] enhance: lazy load all pages --- deps/db/src/logseq/db/sqlite/common_db.cljs | 37 ++-- src/main/frontend/components/cmdk.cljs | 8 +- src/main/frontend/components/page.cljs | 186 ++++++++++---------- src/main/frontend/db/async.cljs | 5 + src/main/frontend/db/react.cljs | 3 +- src/main/frontend/db/restore.cljs | 14 +- src/main/frontend/db_worker.cljs | 34 ++-- 7 files changed, 159 insertions(+), 128 deletions(-) diff --git a/deps/db/src/logseq/db/sqlite/common_db.cljs b/deps/db/src/logseq/db/sqlite/common_db.cljs index 5588fdb83a..ab8395565a 100644 --- a/deps/db/src/logseq/db/sqlite/common_db.cljs +++ b/deps/db/src/logseq/db/sqlite/common_db.cljs @@ -7,12 +7,13 @@ [logseq.common.util.date-time :as date-time-util] [logseq.common.util :as common-util])) -(defn- get-built-in-files - [db] - (let [files ["logseq/config.edn" - "logseq/custom.css" - "logseq/custom.js"]] - (map #(d/pull db '[*] [:file/path %]) files))) +(comment + (defn- get-built-in-files + [db] + (let [files ["logseq/config.edn" + "logseq/custom.css" + "logseq/custom.js"]] + (map #(d/pull db '[*] [:file/path %]) files)))) (defn get-all-pages [db] @@ -28,15 +29,29 @@ :file/path (:v e) :file/content (:file/content (d/entity db (:e e)))})))) +(defn- with-block-refs + [db block] + (update block :block/refs (fn [refs] + (map (fn [ref] + (let [e (d/entity db (:db/id ref))] + (if (and e (:block/name e)) + (assoc ref + :block/uuid (:block/uuid e) + :block/original-name (:block/name e) + :block/name (:block/name e)) + ref))) refs)))) + (defn with-parent-and-left [db block] (cond (:block/name block) block (:block/page block) - (assoc block - :block/left (select-keys (d/entity db (:db/id (:block/left block))) [:db/id :block/uuid]) - :block/parent (select-keys (d/entity db (:db/id (:block/parent block))) [:db/id :block/uuid])) + (->> + (assoc block + :block/left (select-keys (d/entity db (:db/id (:block/left block))) [:db/id :block/uuid]) + :block/parent (select-keys (d/entity db (:db/id (:block/parent block))) [:db/id :block/uuid])) + (with-block-refs db)) :else block)) @@ -96,10 +111,10 @@ (defn get-initial-data "Returns initial data" [db] - (let [all-pages (get-all-pages db) + (let [latest-journals (get-latest-journals db 3) all-files (get-all-files db) closed-values (get-closed-values db)] - (concat all-pages all-files closed-values))) + (concat latest-journals all-files closed-values))) (defn restore-initial-data "Given initial sqlite data and schema, returns a datascript connection" diff --git a/src/main/frontend/components/cmdk.cljs b/src/main/frontend/components/cmdk.cljs index 1aacf3a9be..76a0a4ffd3 100644 --- a/src/main/frontend/components/cmdk.cljs +++ b/src/main/frontend/components/cmdk.cljs @@ -417,7 +417,7 @@ (defmethod handle-action :open-block [_ state _event] (when-let [block-id (some-> state state->highlighted-item :source-block :block/uuid)] (p/let [repo (state/get-current-repo) - _ (db-async/ state state->highlighted-item :source-block :block/uuid)] - (editor-handler/open-block-in-sidebar! block-uuid) - (state/close-modal!))) + (p/let [repo (state/get-current-repo) + _ (db-async/int page-name)) - whiteboard? (:whiteboard? option) ;; in a whiteboard portal shape? - whiteboard-page? (model/whiteboard-page? page-name) ;; is this page a whiteboard? - route-page-name path-page-name - page-name (:block/name page) - page-original-name (:block/original-name page) - title (or page-original-name page-name) - today? (and - journal? - (= page-name (util/page-name-sanity-lc (date/journal-name)))) - *control-show? (::control-show? state) - *all-collapsed? (::all-collapsed? state) - *current-block-page (::current-page state) - block-or-whiteboard? (or block? whiteboard?) - home? (= :home (state/get-current-route))] - (when (or page-name block-or-whiteboard?) - [:div.flex-1.page.relative - (merge (if (seq (:block/tags page)) - (let [page-names (model/get-page-names-by-ids (map :db/id (:block/tags page)))] - {:data-page-tags (text-util/build-data-value page-names)}) - {}) + page (get-page-entity page-name)] + (when-not (and loading? (nil? page)) + (let [block-id (:block/uuid page) + block? (some? (:block/page page)) + journal? (db/journal-page? page-name) + db-based? (config/db-based-graph? repo) + fmt-journal? (boolean (date/journal-title->int page-name)) + whiteboard? (:whiteboard? option) ;; in a whiteboard portal shape? + whiteboard-page? (model/whiteboard-page? page-name) ;; is this page a whiteboard? + route-page-name path-page-name + page-name (:block/name page) + page-original-name (:block/original-name page) + title (or page-original-name page-name) + today? (and + journal? + (= page-name (util/page-name-sanity-lc (date/journal-name)))) + *control-show? (::control-show? state) + *all-collapsed? (::all-collapsed? state) + *current-block-page (::current-page state) + block-or-whiteboard? (or block? whiteboard?) + home? (= :home (state/get-current-route))] + (when (or page-name block-or-whiteboard?) + [:div.flex-1.page.relative + (merge (if (seq (:block/tags page)) + (let [page-names (model/get-page-names-by-ids (map :db/id (:block/tags page)))] + {:data-page-tags (text-util/build-data-value page-names)}) + {}) - {:key path-page-name - :class (util/classnames [{:is-journals (or journal? fmt-journal?)}])}) + {:key path-page-name + :class (util/classnames [{:is-journals (or journal? fmt-journal?)}])}) - (if (and whiteboard-page? (not sidebar?)) - [:div ((state/get-component :whiteboard/tldraw-preview) page-name)] ;; FIXME: this is not reactive - [:div.relative - (when (and (not sidebar?) (not block?)) - [:div.flex.flex-row.space-between - (when (or (mobile-util/native-platform?) (util/mobile?)) - [:div.flex.flex-row.pr-2 - {:style {:margin-left -15} - :on-mouse-over (fn [e] - (page-mouse-over e *control-show? *all-collapsed?)) - :on-mouse-leave (fn [e] - (page-mouse-leave e *control-show?))} - (page-blocks-collapse-control title *control-show? *all-collapsed?)]) - (let [original-name (:block/original-name (db/entity [:block/name (util/page-name-sanity-lc page-name)]))] - (when (and (not whiteboard?) original-name) - (page-title page-name {:journal? journal? - :fmt-journal? fmt-journal? - :preview? preview? - :*hover? (::hover-title? state)}))) - (when (not config/publishing?) - (when config/lsp-enabled? - [:div.flex.flex-row - (plugins/hook-ui-slot :page-head-actions-slotted nil) - (plugins/hook-ui-items :pagebar)]))]) + (if (and whiteboard-page? (not sidebar?)) + [:div ((state/get-component :whiteboard/tldraw-preview) page-name)] ;; FIXME: this is not reactive + [:div.relative + (when (and (not sidebar?) (not block?)) + [:div.flex.flex-row.space-between + (when (or (mobile-util/native-platform?) (util/mobile?)) + [:div.flex.flex-row.pr-2 + {:style {:margin-left -15} + :on-mouse-over (fn [e] + (page-mouse-over e *control-show? *all-collapsed?)) + :on-mouse-leave (fn [e] + (page-mouse-leave e *control-show?))} + (page-blocks-collapse-control title *control-show? *all-collapsed?)]) + (let [original-name (:block/original-name (db/entity [:block/name (util/page-name-sanity-lc page-name)]))] + (when (and (not whiteboard?) original-name) + (page-title page-name {:journal? journal? + :fmt-journal? fmt-journal? + :preview? preview? + :*hover? (::hover-title? state)}))) + (when (not config/publishing?) + (when config/lsp-enabled? + [:div.flex.flex-row + (plugins/hook-ui-slot :page-head-actions-slotted nil) + (plugins/hook-ui-items :pagebar)]))]) - (cond - (and db-based? (not block?)) - [:div.pb-4 - (db-page/page-info page (::hover-title? state))] + (cond + (and db-based? (not block?)) + [:div.pb-4 + (db-page/page-info page (::hover-title? state))] - (and (not db-based?) (not block?)) - [:div.pb-4]) + (and (not db-based?) (not block?)) + [:div.pb-4]) - [:div - (when (and block? (not sidebar?) (not whiteboard?)) - (let [config {:id "block-parent" - :block? true}] - [:div.mb-4 - (component-block/breadcrumb config repo block-id {:level-limit 3})])) + [:div + (when (and block? (not sidebar?) (not whiteboard?)) + (let [config {:id "block-parent" + :block? true}] + [:div.mb-4 + (component-block/breadcrumb config repo block-id {:level-limit 3})])) - (when (and db-based? (not block?) (not preview?)) - (db-page/page-properties-react page {:configure? false})) + (when (and db-based? (not block?) (not preview?)) + (db-page/page-properties-react page {:configure? false})) - ;; blocks - (let [_ (and block? page (reset! *current-block-page (:block/name (:block/page page)))) - _ (when (and block? (not page)) - (route-handler/redirect-to-page! @*current-block-page))] - (page-blocks-cp repo page {:sidebar? sidebar? :whiteboard? whiteboard?}))]]) + ;; blocks + (if loading? + [:div.space-y-2 + (shui-ui/skeleton {:class "h-6 w-full"}) + (shui-ui/skeleton {:class "h-6 w-full"})] + (let [_ (and block? page (reset! *current-block-page (:block/name (:block/page page)))) + _ (when (and block? (not page)) + (route-handler/redirect-to-page! @*current-block-page))] + (page-blocks-cp repo page {:sidebar? sidebar? :whiteboard? whiteboard?})))]]) - (when today? - (today-queries repo today? sidebar?)) + (when today? + (today-queries repo today? sidebar?)) - (when today? - (scheduled/scheduled-and-deadlines page-name)) + (when today? + (scheduled/scheduled-and-deadlines page-name)) - (when-not block? - (tagged-pages repo page-name page-original-name)) + (when-not block? + (tagged-pages repo page-name page-original-name)) ;; referenced blocks - (when-not block-or-whiteboard? - (when page - [:div {:key "page-references"} - (rum/with-key - (reference/references route-page-name) - (str route-page-name "-refs"))])) + (when-not block-or-whiteboard? + (when page + [:div {:key "page-references"} + (rum/with-key + (reference/references route-page-name) + (str route-page-name "-refs"))])) - (when-not block-or-whiteboard? - (when (not journal?) - (hierarchy/structures route-page-name))) + (when-not block-or-whiteboard? + (when (not journal?) + (hierarchy/structures route-page-name))) - (when-not (or block-or-whiteboard? sidebar? home?) - [:div {:key "page-unlinked-references"} - (reference/unlinked-references route-page-name)])]))))) + (when-not (or block-or-whiteboard? sidebar? home?) + [:div {:key "page-unlinked-references"} + (reference/unlinked-references route-page-name)])]))))))) (rum/defcs page < rum/static [state option] diff --git a/src/main/frontend/db/async.cljs b/src/main/frontend/db/async.cljs index a4dcb1db7a..ac4f84a99b 100644 --- a/src/main/frontend/db/async.cljs +++ b/src/main/frontend/db/async.cljs @@ -236,3 +236,8 @@ (->> result db-model/sort-by-left-recursive db-utils/group-by-page))))))) + +(defn > (sqlite-common-db/get-initial-data @conn) - pr-str)] - ;; (comment - ;; (async/go - ;; ;; TODO: after UI db transacted - ;; (async/ Date: Sat, 3 Feb 2024 19:02:25 +0800 Subject: [PATCH 44/60] Load whole data for shorter pages --- deps/db/src/logseq/db/sqlite/common_db.cljs | 39 ++++++++++++++------- src/main/frontend/components/container.cljs | 7 +--- src/main/frontend/db/async.cljs | 2 +- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/deps/db/src/logseq/db/sqlite/common_db.cljs b/deps/db/src/logseq/db/sqlite/common_db.cljs index ab8395565a..f4868640f7 100644 --- a/deps/db/src/logseq/db/sqlite/common_db.cljs +++ b/deps/db/src/logseq/db/sqlite/common_db.cljs @@ -55,30 +55,43 @@ :else block)) +(defn- mark-block-fully-loaded + [b] + (assoc b :block.temp/fully-loaded? true)) + (defn get-block-and-children [db name children?] - (let [get-children (fn [col] - (map (fn [e] - (select-keys e [:db/id :block/uuid :block/page :block/left :block/parent :block/collapsed?])) - col)) - uuid? (common-util/uuid-string? name) + (let [uuid? (common-util/uuid-string? name) block (when uuid? (let [id (uuid name)] - (d/entity db [:block/uuid id])))] + (d/entity db [:block/uuid id]))) + get-children (fn [children] + (let [long-page? (> (count children) 500)] + (if long-page? + (map (fn [e] + (select-keys e [:db/id :block/uuid :block/page :block/left :block/parent :block/collapsed?])) + children) + (->> (d/pull-many db '[*] (map :db/id children)) + (map #(with-block-refs db %)) + (map mark-block-fully-loaded)))))] (if (and block (not (:block/name block))) ; not a page (let [block' (->> (d/pull db '[*] (:db/id block)) - (with-parent-and-left db))] + (with-parent-and-left db) + mark-block-fully-loaded)] (cond-> {:block block'} children? (assoc :children (get-children (:block/_parent block))))) (when-let [block (or block (d/entity db [:block/name name]))] (cond-> - {:block (d/pull db '[*] (:db/id block))} + {:block (-> (d/pull db '[*] (:db/id block)) + mark-block-fully-loaded)} children? (assoc :children (if (contains? (:block/type block) "whiteboard") - (d/pull-many db '[*] (map :db/id (:block/_page block))) + (->> (d/pull-many db '[*] (map :db/id (:block/_page block))) + (map #(with-block-refs db %)) + (map mark-block-fully-loaded)) (get-children (:block/_page block))))))))) (defn get-latest-journals @@ -100,11 +113,11 @@ (reverse) (take n)))) -(defn get-closed-values +(defn get-structured-blocks [db] (->> (d/datoms db :avet :block/type) (keep (fn [e] - (when (= (:v e) "closed value") + (when (contains? #{"closed value" "property" "class"} (:v e)) (d/pull db '[*] (:e e))))))) ;; built-in files + latest journals + favorites @@ -113,8 +126,8 @@ [db] (let [latest-journals (get-latest-journals db 3) all-files (get-all-files db) - closed-values (get-closed-values db)] - (concat latest-journals all-files closed-values))) + structured-blocks (get-structured-blocks db)] + (concat latest-journals all-files structured-blocks))) (defn restore-initial-data "Given initial sqlite data and schema, returns a datascript connection" diff --git a/src/main/frontend/components/container.cljs b/src/main/frontend/components/container.cljs index 0b2ffd052a..fe4ac6981e 100644 --- a/src/main/frontend/components/container.cljs +++ b/src/main/frontend/components/container.cljs @@ -576,14 +576,9 @@ (widgets/demo-graph-alert)) (cond - (not indexeddb-support?) + (or db-restoring? (not indexeddb-support?)) nil - db-restoring? - [:div.mt-20 - [:div.ls-center - (ui/loading)]] - :else [:div {:class (if (or onboarding-and-home? margin-less-pages?) "" (util/hiccup->class "mx-auto.pb-24")) diff --git a/src/main/frontend/db/async.cljs b/src/main/frontend/db/async.cljs index ac4f84a99b..254422c33b 100644 --- a/src/main/frontend/db/async.cljs +++ b/src/main/frontend/db/async.cljs @@ -136,7 +136,7 @@ (p/let [result (.get-block-and-children sqlite graph name' children?) {:keys [block children] :as result'} (edn/read-string result) conn (db/get-db graph false) - block-and-children (cons (assoc block :block.temp/fully-loaded? true) children) + block-and-children (cons block children) _ (d/transact! conn block-and-children)] (state/update-state! :db/async-queries (fn [s] (disj s name'))) (react/refresh-affected-queries! From 6a73ffa5f7b8000fe6c0a2fbfc0d2011a2157945 Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Mon, 5 Feb 2024 17:03:44 -0500 Subject: [PATCH 45/60] fix: lints and a docstring --- .carve/ignore | 4 ---- .clj-kondo/config.edn | 5 +++-- deps/common/.carve/config.edn | 1 + deps/common/src/logseq/common/util/date_time.cljs | 2 +- deps/outliner/.carve/ignore | 2 -- deps/outliner/src/logseq/outliner/datascript.cljs | 1 - src/main/frontend/fs/watcher_handler.cljs | 3 --- src/main/frontend/worker/file/core.cljs | 3 +-- 8 files changed, 6 insertions(+), 15 deletions(-) diff --git a/.carve/ignore b/.carve/ignore index 07cf364b1c..bbaaea7ca7 100644 --- a/.carve/ignore +++ b/.carve/ignore @@ -46,10 +46,6 @@ frontend.image/get-orientation frontend.mixins/perf-measure-mixin ;; Previously useful fn frontend.mobile.util/get-idevice-statusbar-height -;; Used in macro -frontend.modules.outliner.datascript/transact! -frontend.modules.outliner.core/*transaction-opts* -frontend.modules.outliner.core/*transaction-args* ;; Referenced in comment frontend.page/route-view ;; placeholder fn diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index 4d86fcd519..0861c0988f 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -102,7 +102,8 @@ frontend.idb idb frontend.loader loader frontend.mixins mixins - frontend.modules.outliner.core outliner-core + frontend.modules.outliner.op outliner-op + frontend.modules.outliner.ui ui-outliner-tx frontend.mobile.util mobile-util frontend.page page frontend.persist-db persist-db @@ -148,6 +149,7 @@ logseq.common.util.block-ref block-ref logseq.graph-parser.util.db db-util logseq.graph-parser.date-time-util date-time-util + logseq.outliner.core outliner-core logseq.outliner.pipeline outliner-pipeline logseq.outliner.datascript-report ds-report medley.core medley @@ -170,7 +172,6 @@ clojure.test.check.clojure-test/defspec clojure.core/def clojure.test.check.properties/for-all clojure.core/for ;; src/main - frontend.modules.outliner.datascript/auto-transact! clojure.core/let frontend.namespaces/import-vars potemkin/import-vars ;; src/test frontend.test.helper/deftest-async clojure.test/deftest diff --git a/deps/common/.carve/config.edn b/deps/common/.carve/config.edn index 4f832a3375..f835abed5f 100644 --- a/deps/common/.carve/config.edn +++ b/deps/common/.carve/config.edn @@ -3,6 +3,7 @@ logseq.common.util.page-ref logseq.common.util.block-ref logseq.common.util + logseq.common.util.date-time logseq.common.marker logseq.common.config] :report {:format :ignore}} diff --git a/deps/common/src/logseq/common/util/date_time.cljs b/deps/common/src/logseq/common/util/date_time.cljs index 4cde403de4..2338a29ee6 100644 --- a/deps/common/src/logseq/common/util/date_time.cljs +++ b/deps/common/src/logseq/common/util/date_time.cljs @@ -1,5 +1,5 @@ (ns logseq.common.util.date-time - "cljs-time util fns for graph-parser" + "cljs-time util fns for deps" (:require [cljs-time.format :as tf] [clojure.string :as string] [logseq.common.util :as common-util])) diff --git a/deps/outliner/.carve/ignore b/deps/outliner/.carve/ignore index a3da980267..d22c72d50d 100644 --- a/deps/outliner/.carve/ignore +++ b/deps/outliner/.carve/ignore @@ -2,7 +2,5 @@ logseq.outliner.cli.pipeline/add-listener ;; private logseq.outliner.core/*transaction-opts* -;; private -logseq.outliner.core/*transaction-args* ;; API fn logseq.outliner.datascript/transact! diff --git a/deps/outliner/src/logseq/outliner/datascript.cljs b/deps/outliner/src/logseq/outliner/datascript.cljs index 6b67eef453..5aed2b023a 100644 --- a/deps/outliner/src/logseq/outliner/datascript.cljs +++ b/deps/outliner/src/logseq/outliner/datascript.cljs @@ -2,7 +2,6 @@ "Provides fns related to wrapping datascript's transact!" (:require [logseq.common.util :as common-util] [logseq.common.util.block-ref :as block-ref] - [logseq.db.sqlite.util :as sqlite-util] [logseq.graph-parser.property :as gp-property] [datascript.core :as d] [clojure.string :as string] diff --git a/src/main/frontend/fs/watcher_handler.cljs b/src/main/frontend/fs/watcher_handler.cljs index 371e90395c..89a4c8db3a 100644 --- a/src/main/frontend/fs/watcher_handler.cljs +++ b/src/main/frontend/fs/watcher_handler.cljs @@ -3,12 +3,10 @@ (:require [clojure.set :as set] [clojure.string :as string] [frontend.config :as config] - [frontend.date :as date] [frontend.db :as db] [frontend.db.model :as model] [frontend.fs :as fs] [logseq.common.path :as path] - [frontend.handler.editor :as editor-handler] [frontend.handler.file :as file-handler] [frontend.handler.file-based.property :as file-property-handler] [frontend.handler.global-config :as global-config-handler] @@ -16,7 +14,6 @@ [frontend.handler.page :as page-handler] [frontend.handler.ui :as ui-handler] [frontend.state :as state] - [frontend.util :as util] [frontend.util.fs :as fs-util] [lambdaisland.glogi :as log] [logseq.common.config :as common-config] diff --git a/src/main/frontend/worker/file/core.cljs b/src/main/frontend/worker/file/core.cljs index 7a1d49b23e..9093c49f25 100644 --- a/src/main/frontend/worker/file/core.cljs +++ b/src/main/frontend/worker/file/core.cljs @@ -8,8 +8,7 @@ [logseq.db :as ldb] [frontend.worker.date :as worker-date] [frontend.worker.util :as worker-util] - [logseq.db.sqlite.util :as sqlite-util] - [promesa.core :as p])) + [logseq.db.sqlite.util :as sqlite-util])) (defonce *writes (atom {})) (defonce *request-id (atom 0)) From 757104b737cbe234591cf98d763ac1b8f837edbc Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Tue, 6 Feb 2024 11:19:46 -0500 Subject: [PATCH 46/60] enhance: add logs to provide more info on reload and failure also remove migration comments for deleted migration namespace --- src/main/frontend/db/async/util.cljs | 3 ++- src/main/frontend/db/restore.cljs | 24 +----------------------- 2 files changed, 3 insertions(+), 24 deletions(-) diff --git a/src/main/frontend/db/async/util.cljs b/src/main/frontend/db/async/util.cljs index 69d6b1482d..199e72f830 100644 --- a/src/main/frontend/db/async/util.cljs +++ b/src/main/frontend/db/async/util.cljs @@ -22,7 +22,8 @@ result')] (try (d/transact! conn tx-data) - (catch :default _e + (catch :default e + (js/console.error " Date: Tue, 6 Feb 2024 12:51:02 -0500 Subject: [PATCH 47/60] enhance: add script to build large graphs with configurable sizes --- scripts/README.md | 19 +++++ .../create_graph_with_large_sizes.cljs | 84 +++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs diff --git a/scripts/README.md b/scripts/README.md index 47788e4d17..b7221a58d5 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -31,6 +31,25 @@ properties. Read the docs in [logseq.tasks.db-graph.create-graph](src/logseq/tasks/db_graph/create_graph.cljs) for specifics on the EDN map. +To create large graphs with varying size: + +``` +$ yarn -s nbb-logseq src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs large +Building tx ... +Built 21000 tx, 1000 pages and 20000 blocks ... +Transacting chunk 1 of 21 starting with block: #:block{:name "page-0"} +... +Created graph large with 187810 datoms! + +# To see options available +$ yarn -s nbb-logseq src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs -h +Usage: $0 GRAPH-NAME [OPTIONS] +Options: + -h, --help Print help + -p, --pages 1000 Number of pages to create + -b, --blocks 20 Number of blocks to create +``` + Another example is the `create_graph_with_schema_org.cljs` script which creates a graph with the https://schema.org/ ontology with as many of the classes and properties as possible: diff --git a/scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs b/scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs new file mode 100644 index 0000000000..fc425c2fc3 --- /dev/null +++ b/scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs @@ -0,0 +1,84 @@ +(ns logseq.tasks.db-graph.create-graph-with-large-sizes + "Script that generates graphs at large sizes" + (:require [logseq.tasks.db-graph.create-graph :as create-graph] + [clojure.string :as string] + [datascript.core :as d] + [babashka.cli :as cli] + ["path" :as node-path] + ["os" :as os] + [nbb.core :as nbb])) + +(def *ids (atom #{})) +(defn get-next-id + [] + (let [id (random-uuid)] + (if (@*ids id) + (get-next-id) + (do + (swap! *ids conj id) + id)))) + +(defn build-pages + [start-idx n] + (let [ids (repeatedly n get-next-id)] + (map-indexed + (fn [idx id] + {:block/uuid id + :block/name (str "page-" (+ start-idx idx))}) + ids))) + +(defn build-blocks + [size] + (vec (repeatedly size + (fn [] + (let [id (get-next-id)] + {:block/uuid id + :block/content (str id)}))))) + +(defn- create-init-data + [options] + (let [pages (build-pages 0 (:pages options))] + {:pages-and-blocks + (mapv #(hash-map :page % :blocks (build-blocks (:blocks options))) + pages)})) + +(def spec + "Options spec" + {:help {:alias :h + :desc "Print help"} + :pages {:alias :p + :default 1000 + :desc "Number of pages to create"} + :blocks {:alias :b + :default 20 + :desc "Number of blocks to create"}}) + +(defn -main [args] + (let [graph-dir (first args) + options (cli/parse-opts args {:spec spec}) + _ (when (or (nil? graph-dir) (:help options)) + (println (str "Usage: $0 GRAPH-NAME [OPTIONS]\nOptions:\n" + (cli/format-opts {:spec spec}))) + (js/process.exit 1)) + [dir db-name] (if (string/includes? graph-dir "/") + ((juxt node-path/dirname node-path/basename) graph-dir) + [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]) + conn (create-graph/init-conn dir db-name) + _ (println "Building tx ...") + blocks-tx (create-graph/create-blocks-tx (create-init-data options))] + (println "Built" (count blocks-tx) "tx," (count (filter :block/name blocks-tx)) "pages and" + (count (filter :block/content blocks-tx)) "blocks ...") + ;; Vary the chunking with page size for now + (let [tx-chunks (partition-all (:pages options) blocks-tx)] + (loop [chunks tx-chunks + chunk-num 1] + (when-let [chunk (first chunks)] + (println "Transacting chunk" chunk-num "of" (count tx-chunks) + "starting with block:" (pr-str (select-keys (first chunk) [:block/content :block/name]))) + (d/transact! conn chunk) + (recur (rest chunks) (inc chunk-num))))) + #_(d/transact! conn blocks-tx) + (println "Created graph" (str db-name " with " (count (d/datoms @conn :eavt)) " datoms!")))) + +(when (= nbb/*file* (:file (meta #'-main))) + (-main *command-line-args*)) \ No newline at end of file From 140287e3ea78bfaaf0feeb376d58ca9db0f9ddca Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Wed, 7 Feb 2024 03:43:45 +0800 Subject: [PATCH 48/60] perf: don't load properties and classes for initial data This commit also fixed the issue that some pages such as "Status" will be shown as lower cased. --- deps/db/src/logseq/db/sqlite/common_db.cljs | 29 +++++++++------------ deps/outliner/src/logseq/outliner/core.cljs | 5 +++- src/main/frontend/components/container.cljs | 8 +++++- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/deps/db/src/logseq/db/sqlite/common_db.cljs b/deps/db/src/logseq/db/sqlite/common_db.cljs index f4868640f7..ce53019a62 100644 --- a/deps/db/src/logseq/db/sqlite/common_db.cljs +++ b/deps/db/src/logseq/db/sqlite/common_db.cljs @@ -31,15 +31,7 @@ (defn- with-block-refs [db block] - (update block :block/refs (fn [refs] - (map (fn [ref] - (let [e (d/entity db (:db/id ref))] - (if (and e (:block/name e)) - (assoc ref - :block/uuid (:block/uuid e) - :block/original-name (:block/name e) - :block/name (:block/name e)) - ref))) refs)))) + (update block :block/refs (fn [refs] (map (fn [ref] (d/pull db '[*] (:db/id ref))) refs)))) (defn with-parent-and-left [db block] @@ -47,11 +39,16 @@ (:block/name block) block (:block/page block) - (->> - (assoc block - :block/left (select-keys (d/entity db (:db/id (:block/left block))) [:db/id :block/uuid]) - :block/parent (select-keys (d/entity db (:db/id (:block/parent block))) [:db/id :block/uuid])) - (with-block-refs db)) + (let [left (when-let [e (d/entity db (:db/id (:block/left block)))] + (select-keys e [:db/id :block/uuid])) + parent (when-let [e (d/entity db (:db/id (:block/parent block)))] + (select-keys e [:db/id :block/uuid]))] + (->> + (assoc block + :block/left left + :block/parent parent) + (common-util/remove-nils-non-nested) + (with-block-refs db))) :else block)) @@ -77,6 +74,7 @@ (if (and block (not (:block/name block))) ; not a page (let [block' (->> (d/pull db '[*] (:db/id block)) (with-parent-and-left db) + (with-block-refs db) mark-block-fully-loaded)] (cond-> {:block block'} @@ -117,10 +115,9 @@ [db] (->> (d/datoms db :avet :block/type) (keep (fn [e] - (when (contains? #{"closed value" "property" "class"} (:v e)) + (when (contains? #{"closed value"} (:v e)) (d/pull db '[*] (:e e))))))) -;; built-in files + latest journals + favorites (defn get-initial-data "Returns initial data" [db] diff --git a/deps/outliner/src/logseq/outliner/core.cljs b/deps/outliner/src/logseq/outliner/core.cljs index 6d1d8f80e8..4fdaabaa08 100644 --- a/deps/outliner/src/logseq/outliner/core.cljs +++ b/deps/outliner/src/logseq/outliner/core.cljs @@ -234,7 +234,10 @@ (let [refs (->> (rebuild-block-refs repo conn date-formatter block (:block/properties block) :skip-content-parsing? true) (concat (:block/refs m)) - (concat (:block/tags m)))] + (concat (if (seq (:block/tags m)) + (:block/tags m) + (map :db/id (:block/tags (d/entity @conn [:block/uuid (:block/uuid block)]))))) + (remove nil?))] (swap! txs-state (fn [txs] (concat txs [{:db/id (:db/id block) :block/refs refs}])))))) diff --git a/src/main/frontend/components/container.cljs b/src/main/frontend/components/container.cljs index fe4ac6981e..d2da5e27f8 100644 --- a/src/main/frontend/components/container.cljs +++ b/src/main/frontend/components/container.cljs @@ -576,9 +576,15 @@ (widgets/demo-graph-alert)) (cond - (or db-restoring? (not indexeddb-support?)) + (not indexeddb-support?) nil + db-restoring? + [:div.space-y-2 + (shui/skeleton {:class "h-8 w-1/3 mb-8"}) + (shui/skeleton {:class "h-6 w-full"}) + (shui/skeleton {:class "h-6 w-full"})] + :else [:div {:class (if (or onboarding-and-home? margin-less-pages?) "" (util/hiccup->class "mx-auto.pb-24")) From b1bcc1b2dd9bb7f902a3ea761e7fda95031cf39c Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Tue, 6 Feb 2024 14:52:48 -0500 Subject: [PATCH 49/60] fix: unit test, lint + script --- .clj-kondo/config.edn | 12 ++++++------ .../db/sqlite/{db_test.cljs => common_db_test.cljs} | 5 ++--- .../db_graph/create_graph_with_large_sizes.cljs | 4 ++-- 3 files changed, 10 insertions(+), 11 deletions(-) rename deps/db/test/logseq/db/sqlite/{db_test.cljs => common_db_test.cljs} (95%) diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index 0861c0988f..bff142efde 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -128,9 +128,13 @@ frontend.worker.handler.page.rename worker-page-rename frontend.worker.handler.file.util wfu lambdaisland.glogi log - logseq.common.path path - logseq.common.graph common-graph logseq.common.config common-config + logseq.common.graph common-graph + logseq.common.date-time-util date-time-util + logseq.common.path path + logseq.common.util common-util + logseq.common.util.page-ref page-ref + logseq.common.util.block-ref block-ref logseq.db ldb logseq.db.frontend.property db-property logseq.db.frontend.property.type db-property-type @@ -143,12 +147,8 @@ logseq.graph-parser.text text logseq.graph-parser.block gp-block logseq.graph-parser.mldoc gp-mldoc - logseq.common.util common-util logseq.graph-parser.property gp-property - logseq.common.util.page-ref page-ref - logseq.common.util.block-ref block-ref logseq.graph-parser.util.db db-util - logseq.graph-parser.date-time-util date-time-util logseq.outliner.core outliner-core logseq.outliner.pipeline outliner-pipeline logseq.outliner.datascript-report ds-report diff --git a/deps/db/test/logseq/db/sqlite/db_test.cljs b/deps/db/test/logseq/db/sqlite/common_db_test.cljs similarity index 95% rename from deps/db/test/logseq/db/sqlite/db_test.cljs rename to deps/db/test/logseq/db/sqlite/common_db_test.cljs index 8cd8e78588..f1b43805e6 100644 --- a/deps/db/test/logseq/db/sqlite/db_test.cljs +++ b/deps/db/test/logseq/db/sqlite/common_db_test.cljs @@ -1,4 +1,4 @@ -(ns logseq.db.sqlite.db-test +(ns logseq.db.sqlite.common-db-test (:require [cljs.test :refer [deftest async use-fixtures is testing]] ["fs" :as fs] ["path" :as node-path] @@ -27,8 +27,7 @@ (create-graph-dir "tmp/graphs" "test-db") (let [conn* (sqlite-db/open-db! "tmp/graphs" "test-db") - blocks [{:block/uuid (random-uuid) - :file/path "logseq/config.edn" + blocks [{:file/path "logseq/config.edn" :file/content "{:foo :bar}"}] _ (d/transact! conn* blocks) ;; Simulate getting data from sqlite and restoring it for frontend diff --git a/scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs b/scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs index fc425c2fc3..cf72034e94 100644 --- a/scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs +++ b/scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs @@ -68,8 +68,8 @@ blocks-tx (create-graph/create-blocks-tx (create-init-data options))] (println "Built" (count blocks-tx) "tx," (count (filter :block/name blocks-tx)) "pages and" (count (filter :block/content blocks-tx)) "blocks ...") - ;; Vary the chunking with page size for now - (let [tx-chunks (partition-all (:pages options) blocks-tx)] + ;; Vary the chunking with page size up to a max to avoid OOM + (let [tx-chunks (partition-all (min (:pages options) 30000) blocks-tx)] (loop [chunks tx-chunks chunk-num 1] (when-let [chunk (first chunks)] From 6c6cc8ec34d4f675966bc54c3aa6910dc8d3c253 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Wed, 7 Feb 2024 04:47:57 +0800 Subject: [PATCH 50/60] Load classes for initial data since task commands require them --- deps/db/src/logseq/db/sqlite/common_db.cljs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/db/src/logseq/db/sqlite/common_db.cljs b/deps/db/src/logseq/db/sqlite/common_db.cljs index ce53019a62..50f236602b 100644 --- a/deps/db/src/logseq/db/sqlite/common_db.cljs +++ b/deps/db/src/logseq/db/sqlite/common_db.cljs @@ -115,7 +115,7 @@ [db] (->> (d/datoms db :avet :block/type) (keep (fn [e] - (when (contains? #{"closed value"} (:v e)) + (when (contains? #{"closed value" "class"} (:v e)) (d/pull db '[*] (:e e))))))) (defn get-initial-data From bd9ebaf3fdbd991e88f7dc8ebefb2b485f409896 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Wed, 7 Feb 2024 04:54:54 +0800 Subject: [PATCH 51/60] perf: skip loading classes for the initial data re-init commands after pages has been loaded --- deps/db/src/logseq/db/sqlite/common_db.cljs | 2 +- src/main/frontend/modules/outliner/pipeline.cljs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/deps/db/src/logseq/db/sqlite/common_db.cljs b/deps/db/src/logseq/db/sqlite/common_db.cljs index 50f236602b..ce53019a62 100644 --- a/deps/db/src/logseq/db/sqlite/common_db.cljs +++ b/deps/db/src/logseq/db/sqlite/common_db.cljs @@ -115,7 +115,7 @@ [db] (->> (d/datoms db :avet :block/type) (keep (fn [e] - (when (contains? #{"closed value" "class"} (:v e)) + (when (contains? #{"closed value"} (:v e)) (d/pull db '[*] (:e e))))))) (defn get-initial-data diff --git a/src/main/frontend/modules/outliner/pipeline.cljs b/src/main/frontend/modules/outliner/pipeline.cljs index 1caf0e3bfd..70a4d6889a 100644 --- a/src/main/frontend/modules/outliner/pipeline.cljs +++ b/src/main/frontend/modules/outliner/pipeline.cljs @@ -51,6 +51,7 @@ (do (util/profile "transact initial-pages" (d/transact! conn tx-data tx-meta)) (when end? + (state/pub-event! [:init/commands]) (ui-handler/re-render-root!))) (do (let [tx-report (d/transact! conn tx-data tx-meta)] From 74649841f3d4db4c870ef25b32afe01633ceb2a1 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Wed, 7 Feb 2024 05:20:53 +0800 Subject: [PATCH 52/60] fix: no edits are able to save after tab and then enter --- src/main/frontend/worker/outliner_op.cljs | 80 +++++++++++++---------- 1 file changed, 44 insertions(+), 36 deletions(-) diff --git a/src/main/frontend/worker/outliner_op.cljs b/src/main/frontend/worker/outliner_op.cljs index aa13b1177a..ec40042ee5 100644 --- a/src/main/frontend/worker/outliner_op.cljs +++ b/src/main/frontend/worker/outliner_op.cljs @@ -4,8 +4,8 @@ [logseq.outliner.core :as outliner-core] [frontend.worker.state :as worker-state] [datascript.core :as d] - [promesa.core :as p] - [malli.core :as m])) + [malli.core :as m] + [frontend.worker.util :as worker-util])) (def op-schema [:multi {:dispatch first} @@ -50,42 +50,50 @@ :transact-opts {:repo repo :conn conn} :local-tx? true) date-formatter (worker-state/get-date-formatter repo) - *insert-result (atom nil)] - (p/do! - (outliner-tx/transact! - opts' - (doseq [[op args] ops] - (case op - :save-block - (apply outliner-core/save-block! repo conn date-formatter args) + *insert-result (atom nil) + max-tx (:max-tx @conn) + _ (outliner-tx/transact! + opts' + (doseq [[op args] ops] + (case op + :save-block + (apply outliner-core/save-block! repo conn date-formatter args) - :insert-blocks - (let [[blocks target-block-id opts] args] - (when-let [target-block (d/entity @conn target-block-id)] - (let [result (outliner-core/insert-blocks! repo conn blocks target-block opts)] - (reset! *insert-result result)))) + :insert-blocks + (let [[blocks target-block-id opts] args] + (when-let [target-block (d/entity @conn target-block-id)] + (let [result (outliner-core/insert-blocks! repo conn blocks target-block opts)] + (reset! *insert-result result)))) - :delete-blocks - (let [[block-ids opts] args - blocks (keep #(d/entity @conn %) block-ids)] - (outliner-core/delete-blocks! repo conn date-formatter blocks opts)) + :delete-blocks + (let [[block-ids opts] args + blocks (keep #(d/entity @conn %) block-ids)] + (outliner-core/delete-blocks! repo conn date-formatter blocks opts)) - :move-blocks - (let [[block-ids target-block-id sibling?] args - blocks (keep #(d/entity @conn %) block-ids) - target-block (d/entity @conn target-block-id)] - (when (and target-block (seq blocks)) - (outliner-core/move-blocks! repo conn blocks target-block sibling?))) + :move-blocks + (let [[block-ids target-block-id sibling?] args + blocks (keep #(d/entity @conn %) block-ids) + target-block (d/entity @conn target-block-id)] + (when (and target-block (seq blocks)) + (outliner-core/move-blocks! repo conn blocks target-block sibling?))) - :move-blocks-up-down - (let [[block-ids up?] args - blocks (keep #(d/entity @conn %) block-ids)] - (when (seq blocks) - (outliner-core/move-blocks-up-down! repo conn blocks up?))) + :move-blocks-up-down + (let [[block-ids up?] args + blocks (keep #(d/entity @conn %) block-ids)] + (when (seq blocks) + (outliner-core/move-blocks-up-down! repo conn blocks up?))) - :indent-outdent-blocks - (let [[block-ids indent? opts] args - blocks (keep #(d/entity @conn %) block-ids)] - (when (seq blocks) - (outliner-core/indent-outdent-blocks! repo conn blocks indent? opts)))))) - (pr-str @*insert-result)))) + :indent-outdent-blocks + (let [[block-ids indent? opts] args + blocks (keep #(d/entity @conn %) block-ids)] + (when (seq blocks) + (outliner-core/indent-outdent-blocks! repo conn blocks indent? opts)))))) + after-max-tx (:max-tx @conn)] + (if (= after-max-tx max-tx) + ;; remove task from ldb/*request-id->response + (worker-util/post-message :sync-db-changes (pr-str + {:request-id (:request-id opts) + :repo repo + :tx-data [] + :tx-meta nil})) + (pr-str @*insert-result)))) From ad67e90a533092ac8f37aca28936393ca52416d8 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Wed, 7 Feb 2024 05:36:50 +0800 Subject: [PATCH 53/60] fix: lint warnings --- src/main/frontend/components/page.cljs | 2 +- src/main/frontend/handler.cljs | 12 ------------ .../frontend/handler/file_based/events.cljs | 16 +++++++++------- src/main/frontend/handler/file_based/nfs.cljs | 2 +- src/main/frontend/idb.cljs | 17 ----------------- src/main/frontend/storage.cljs | 19 ++++++++++--------- 6 files changed, 21 insertions(+), 47 deletions(-) diff --git a/src/main/frontend/components/page.cljs b/src/main/frontend/components/page.cljs index 040a736601..ef99f8e2d8 100644 --- a/src/main/frontend/components/page.cljs +++ b/src/main/frontend/components/page.cljs @@ -168,7 +168,7 @@ (date/journal-title->int (date/today)))) (state/pub-event! [:journal/insert-template page-name]))) state)} - [state repo page-e {:keys [sidebar? whiteboard?] :as config}] + [state _repo page-e {:keys [sidebar? whiteboard?] :as config}] (when page-e (let [page-name (or (:block/name page-e) (str (:block/uuid page-e))) diff --git a/src/main/frontend/handler.cljs b/src/main/frontend/handler.cljs index 0d35fef536..3d20fec79b 100644 --- a/src/main/frontend/handler.cljs +++ b/src/main/frontend/handler.cljs @@ -140,18 +140,6 @@ nil))))))) -(defn clear-cache! - [] - (notification/show! "Clearing..." :warning false) - (p/let [_ (when (util/electron?) - (ipc/ipc "clearCache")) - _ (idb/clear-local-storage-and-idb!)] - (js/setTimeout - (fn [] (if (util/electron?) - (ipc/ipc :reloadWindowPage) - (js/window.location.reload))) - 2000))) - (defn- register-components-fns! [] (state/set-page-blocks-cp! page/page) diff --git a/src/main/frontend/handler/file_based/events.cljs b/src/main/frontend/handler/file_based/events.cljs index 3d77736afa..140bc83a30 100644 --- a/src/main/frontend/handler/file_based/events.cljs +++ b/src/main/frontend/handler/file_based/events.cljs @@ -9,7 +9,8 @@ [frontend.fs.sync :as sync] [frontend.state :as state] [frontend.ui :as ui] - [frontend.util :as util])) + [frontend.util :as util] + [frontend.config :as config])) (defmethod events/handle :graph/ask-for-re-index [[_ *multiple-windows? ui]] ;; *multiple-windows? - if the graph is opened in multiple windows, boolean atom @@ -35,9 +36,10 @@ (defmethod events/handle :graph/re-index [[_]] ;; Ensure the graph only has ONE window instance - (async/go - (async/ - (p/let [_ (idb-keyval/clear @store) - dbs (js/window.indexedDB.databases)] - (doseq [db dbs] - (js/window.indexedDB.deleteDatabase (gobj/get db "name")))) - (p/catch (fn [_e]))))) - -(defn clear-local-storage-and-idb! - [] - (storage/clear) - (clear-idb!)) - (defn remove-item! [key] (when (and key @store) diff --git a/src/main/frontend/storage.cljs b/src/main/frontend/storage.cljs index 52faf9fea9..73fb4165a7 100644 --- a/src/main/frontend/storage.cljs +++ b/src/main/frontend/storage.cljs @@ -3,7 +3,6 @@ localStorage values are edn" (:refer-clojure :exclude [get set remove]) (:require [cljs.reader :as reader] - [datascript.transit :as dt] [frontend.spec.storage :as storage-spec] [cljs.spec.alpha :as s] [frontend.util :as util])) @@ -23,17 +22,19 @@ (when-not util/node-test? (.setItem ^js js/localStorage (name key) (pr-str value)))) -(defn get-transit - [key] - (when-not util/node-test? - (dt/read-transit-str ^js (.getItem js/localStorage (name key))))) +(comment + (defn get-transit + [key] + (when-not util/node-test? + (dt/read-transit-str ^js (.getItem js/localStorage (name key)))))) (defn remove [key] (when-not util/node-test? (.removeItem ^js js/localStorage (name key)))) -(defn clear - [] - (when-not util/node-test? - (.clear ^js js/localStorage))) +(comment + (defn clear + [] + (when-not util/node-test? + (.clear ^js js/localStorage)))) From 0a028ba2350a0ef636391dce6cc0f99e9e3a60ca Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Wed, 7 Feb 2024 06:38:52 +0800 Subject: [PATCH 54/60] fix: editor unit tests --- deps/outliner/src/logseq/outliner/op.cljs | 87 +++++++++++++++++++ src/main/frontend/db_worker.cljs | 16 +++- src/main/frontend/modules/outliner/ui.cljc | 44 ++++++---- src/main/frontend/worker/outliner_op.cljs | 99 ---------------------- 4 files changed, 126 insertions(+), 120 deletions(-) create mode 100644 deps/outliner/src/logseq/outliner/op.cljs delete mode 100644 src/main/frontend/worker/outliner_op.cljs diff --git a/deps/outliner/src/logseq/outliner/op.cljs b/deps/outliner/src/logseq/outliner/op.cljs new file mode 100644 index 0000000000..305160cab5 --- /dev/null +++ b/deps/outliner/src/logseq/outliner/op.cljs @@ -0,0 +1,87 @@ +(ns logseq.outliner.op + "Transact outliner ops" + (:require [logseq.outliner.transaction :as outliner-tx] + [logseq.outliner.core :as outliner-core] + [datascript.core :as d] + [malli.core :as m])) + +(def op-schema + [:multi {:dispatch first} + [:save-block + [:catn + [:op :keyword] + [:args [:tuple ::block]]]] + [:insert-blocks + [:catn + [:op :keyword] + [:args [:tuple ::blocks ::id ::option]]]] + [:delete-blocks + [:catn + [:op :keyword] + [:args [:tuple ::ids ::option]]]] + [:move-blocks + [:catn + [:op :keyword] + [:args [:tuple ::ids ::id :boolean]]]] + [:move-blocks-up-down + [:catn + [:op :keyword] + [:args [:tuple ::ids :boolean]]]] + [:indent-outdent-blocks + [:catn + [:op :keyword] + [:args [:tuple ::ids :boolean ::option]]]]]) + +(def ops-schema [:schema {:registry {::id int? + ::block map? + ::option [:maybe map?] + ::blocks [:sequential ::block] + ::ids [:sequential ::id]}} + [:sequential op-schema]]) + +(def ops-validator (m/validator ops-schema)) + +(defn apply-ops! + [repo conn ops date-formatter opts] + (assert (ops-validator ops) ops) + (let [opts' (assoc opts + :transact-opts {:repo repo :conn conn} + :local-tx? true) + *insert-result (atom nil)] + (outliner-tx/transact! + opts' + (doseq [[op args] ops] + (case op + :save-block + (apply outliner-core/save-block! repo conn date-formatter args) + + :insert-blocks + (let [[blocks target-block-id opts] args] + (when-let [target-block (d/entity @conn target-block-id)] + (let [result (outliner-core/insert-blocks! repo conn blocks target-block opts)] + (reset! *insert-result result)))) + + :delete-blocks + (let [[block-ids opts] args + blocks (keep #(d/entity @conn %) block-ids)] + (outliner-core/delete-blocks! repo conn date-formatter blocks opts)) + + :move-blocks + (let [[block-ids target-block-id sibling?] args + blocks (keep #(d/entity @conn %) block-ids) + target-block (d/entity @conn target-block-id)] + (when (and target-block (seq blocks)) + (outliner-core/move-blocks! repo conn blocks target-block sibling?))) + + :move-blocks-up-down + (let [[block-ids up?] args + blocks (keep #(d/entity @conn %) block-ids)] + (when (seq blocks) + (outliner-core/move-blocks-up-down! repo conn blocks up?))) + + :indent-outdent-blocks + (let [[block-ids indent? opts] args + blocks (keep #(d/entity @conn %) block-ids)] + (when (seq blocks) + (outliner-core/indent-outdent-blocks! repo conn blocks indent? opts)))))) + @*insert-result)) diff --git a/src/main/frontend/db_worker.cljs b/src/main/frontend/db_worker.cljs index dc67c46bba..41c1f467ed 100644 --- a/src/main/frontend/db_worker.cljs +++ b/src/main/frontend/db_worker.cljs @@ -25,7 +25,7 @@ [frontend.worker.util :as worker-util] [frontend.worker.handler.page.rename :as worker-page-rename] [frontend.worker.handler.page :as worker-page] - [frontend.worker.outliner-op :as worker-outliner-op])) + [logseq.outliner.op :as outliner-op])) (defonce *sqlite worker-state/*sqlite) (defonce *sqlite-conns worker-state/*sqlite-conns) @@ -482,8 +482,18 @@ [this repo ops-str opts-str] (when-let [conn (worker-state/get-datascript-conn repo)] (let [ops (edn/read-string ops-str) - opts (edn/read-string opts-str)] - (worker-outliner-op/apply-ops! repo conn ops opts)))) + opts (edn/read-string opts-str) + start-tx (:max-tx @conn) + result (outliner-op/apply-ops! repo conn ops (worker-state/get-date-formatter repo) opts) + end-tx (:max-tx @conn)] + (when (= start-tx end-tx) ; nothing changes + ;; remove task from ldb/*request-id->response + (worker-util/post-message :sync-db-changes (pr-str + {:request-id (:request-id opts) + :repo repo + :tx-data [] + :tx-meta nil}))) + (pr-str result)))) (file-writes-finished? [this repo] diff --git a/src/main/frontend/modules/outliner/ui.cljc b/src/main/frontend/modules/outliner/ui.cljc index 71764114c8..80581294b8 100644 --- a/src/main/frontend/modules/outliner/ui.cljc +++ b/src/main/frontend/modules/outliner/ui.cljc @@ -1,24 +1,32 @@ (ns frontend.modules.outliner.ui #?(:cljs (:require-macros [frontend.modules.outliner.ui])) #?(:cljs (:require [frontend.state :as state] - [frontend.db :as db]))) + [frontend.db :as db] + [logseq.outliner.op]))) (defmacro transact! [opts & body] - `(when (db/request-finished?) - (when (nil? @(:history/tx-before-editor-cursor @state/state)) - (state/set-state! :history/tx-before-editor-cursor (state/get-current-edit-block-and-position))) - (let [ops# frontend.modules.outliner.op/*outliner-ops*] - (if ops# - (do ~@body) ; nested transact! - (binding [frontend.modules.outliner.op/*outliner-ops* (transient [])] - ~@body - (let [r# (persistent! frontend.modules.outliner.op/*outliner-ops*) - worker# @state/*db-worker] - (when (and worker# (seq r#)) - (let [request-id# (state/get-worker-next-request-id) - response# (.apply-outliner-ops ^Object worker# (state/get-current-repo) - (pr-str r#) - (pr-str (assoc ~opts :request-id request-id#)))] - (state/add-worker-request! request-id# :outliner-tx) - response#)))))))) + `(let [test?# frontend.util/node-test?] + (when (or test?# (db/request-finished?)) + (when (nil? @(:history/tx-before-editor-cursor @state/state)) + (state/set-state! :history/tx-before-editor-cursor (state/get-current-edit-block-and-position))) + (let [ops# frontend.modules.outliner.op/*outliner-ops*] + (if ops# + (do ~@body) ; nested transact! + (binding [frontend.modules.outliner.op/*outliner-ops* (transient [])] + ~@body + (let [r# (persistent! frontend.modules.outliner.op/*outliner-ops*) + worker# @state/*db-worker] + (if (and test?# (seq r#)) + (logseq.outliner.op/apply-ops! (state/get-current-repo) + (db/get-db false) + r# + (state/get-date-formatter) + ~opts) + (when (and worker# (seq r#)) + (let [request-id# (state/get-worker-next-request-id) + response# (.apply-outliner-ops ^Object worker# (state/get-current-repo) + (pr-str r#) + (pr-str (assoc ~opts :request-id request-id#)))] + (state/add-worker-request! request-id# :outliner-tx) + response#)))))))))) diff --git a/src/main/frontend/worker/outliner_op.cljs b/src/main/frontend/worker/outliner_op.cljs deleted file mode 100644 index ec40042ee5..0000000000 --- a/src/main/frontend/worker/outliner_op.cljs +++ /dev/null @@ -1,99 +0,0 @@ -(ns frontend.worker.outliner-op - "Transact outliner ops from UI" - (:require [logseq.outliner.transaction :as outliner-tx] - [logseq.outliner.core :as outliner-core] - [frontend.worker.state :as worker-state] - [datascript.core :as d] - [malli.core :as m] - [frontend.worker.util :as worker-util])) - -(def op-schema - [:multi {:dispatch first} - [:save-block - [:catn - [:op :keyword] - [:args [:tuple ::block]]]] - [:insert-blocks - [:catn - [:op :keyword] - [:args [:tuple ::blocks ::id ::option]]]] - [:delete-blocks - [:catn - [:op :keyword] - [:args [:tuple ::ids ::option]]]] - [:move-blocks - [:catn - [:op :keyword] - [:args [:tuple ::ids ::id :boolean]]]] - [:move-blocks-up-down - [:catn - [:op :keyword] - [:args [:tuple ::ids :boolean]]]] - [:indent-outdent-blocks - [:catn - [:op :keyword] - [:args [:tuple ::ids :boolean ::option]]]]]) - -(def ops-schema [:schema {:registry {::id int? - ::block map? - ::option [:maybe map?] - ::blocks [:sequential ::block] - ::ids [:sequential ::id]}} - [:sequential op-schema]]) - -(def ops-validator (m/validator ops-schema)) - -(defn apply-ops! - [repo conn ops opts] - (assert (ops-validator ops) ops) - (let [opts' (assoc opts - :transact-opts {:repo repo :conn conn} - :local-tx? true) - date-formatter (worker-state/get-date-formatter repo) - *insert-result (atom nil) - max-tx (:max-tx @conn) - _ (outliner-tx/transact! - opts' - (doseq [[op args] ops] - (case op - :save-block - (apply outliner-core/save-block! repo conn date-formatter args) - - :insert-blocks - (let [[blocks target-block-id opts] args] - (when-let [target-block (d/entity @conn target-block-id)] - (let [result (outliner-core/insert-blocks! repo conn blocks target-block opts)] - (reset! *insert-result result)))) - - :delete-blocks - (let [[block-ids opts] args - blocks (keep #(d/entity @conn %) block-ids)] - (outliner-core/delete-blocks! repo conn date-formatter blocks opts)) - - :move-blocks - (let [[block-ids target-block-id sibling?] args - blocks (keep #(d/entity @conn %) block-ids) - target-block (d/entity @conn target-block-id)] - (when (and target-block (seq blocks)) - (outliner-core/move-blocks! repo conn blocks target-block sibling?))) - - :move-blocks-up-down - (let [[block-ids up?] args - blocks (keep #(d/entity @conn %) block-ids)] - (when (seq blocks) - (outliner-core/move-blocks-up-down! repo conn blocks up?))) - - :indent-outdent-blocks - (let [[block-ids indent? opts] args - blocks (keep #(d/entity @conn %) block-ids)] - (when (seq blocks) - (outliner-core/indent-outdent-blocks! repo conn blocks indent? opts)))))) - after-max-tx (:max-tx @conn)] - (if (= after-max-tx max-tx) - ;; remove task from ldb/*request-id->response - (worker-util/post-message :sync-db-changes (pr-str - {:request-id (:request-id opts) - :repo repo - :tx-data [] - :tx-meta nil})) - (pr-str @*insert-result)))) From bee1cbfeb52f87d6ec2f0f77f0f9681eb08f7eea Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Wed, 7 Feb 2024 07:22:40 +0800 Subject: [PATCH 55/60] fix: load page tags --- deps/db/src/logseq/db/sqlite/common_db.cljs | 19 +++++++++++++------ src/main/frontend/components/block.cljs | 2 +- src/main/frontend/components/page.cljs | 2 +- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/deps/db/src/logseq/db/sqlite/common_db.cljs b/deps/db/src/logseq/db/sqlite/common_db.cljs index ce53019a62..d165bdf0c5 100644 --- a/deps/db/src/logseq/db/sqlite/common_db.cljs +++ b/deps/db/src/logseq/db/sqlite/common_db.cljs @@ -52,6 +52,10 @@ :else block)) +(defn- with-tags + [db block] + (update block :block/tags (fn [tags] (d/pull-many db '[*] (map :db/id tags))))) + (defn- mark-block-fully-loaded [b] (assoc b :block.temp/fully-loaded? true)) @@ -82,8 +86,9 @@ (assoc :children (get-children (:block/_parent block))))) (when-let [block (or block (d/entity db [:block/name name]))] (cond-> - {:block (-> (d/pull db '[*] (:db/id block)) - mark-block-fully-loaded)} + {:block (->> (d/pull db '[*] (:db/id block)) + (with-tags db) + mark-block-fully-loaded)} children? (assoc :children (if (contains? (:block/type block) "whiteboard") @@ -113,10 +118,12 @@ (defn get-structured-blocks [db] - (->> (d/datoms db :avet :block/type) - (keep (fn [e] - (when (contains? #{"closed value"} (:v e)) - (d/pull db '[*] (:e e))))))) + (let [special-pages (map #(d/pull db '[*] [:block/name %]) #{"tags"}) + closed-values (->> (d/datoms db :avet :block/type) + (keep (fn [e] + (when (contains? #{"closed value"} (:v e)) + (d/pull db '[*] (:e e))))))] + (concat special-pages closed-values))) (defn get-initial-data "Returns initial data" diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index 55f26dc209..50189a5844 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -852,7 +852,7 @@ [:section.flex.items-center.p-1.embed-header [:div.mr-3 svg/page] (page-cp config {:block/name page-name})] - (if (state/sub-async-query-loading page-name) + (if (and page-name (state/sub-async-query-loading page-name)) [:span "Loading..."] (when (and (not= (util/page-name-sanity-lc (or current-page "")) diff --git a/src/main/frontend/components/page.cljs b/src/main/frontend/components/page.cljs index ef99f8e2d8..2c611dc46d 100644 --- a/src/main/frontend/components/page.cljs +++ b/src/main/frontend/components/page.cljs @@ -461,7 +461,7 @@ (db-async/ Date: Wed, 7 Feb 2024 12:22:50 -0500 Subject: [PATCH 56/60] fix: test and lints also remove unused binding not caught by linter b/c it's used --- .clj-kondo/config.edn | 2 +- deps/db/src/logseq/db/sqlite/common_db.cljs | 4 +--- .../test/logseq/db/sqlite/common_db_test.cljs | 18 ++++++++++++------ deps/outliner/.carve/ignore | 2 ++ 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index bff142efde..d58b5c160e 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -102,7 +102,6 @@ frontend.idb idb frontend.loader loader frontend.mixins mixins - frontend.modules.outliner.op outliner-op frontend.modules.outliner.ui ui-outliner-tx frontend.mobile.util mobile-util frontend.page page @@ -150,6 +149,7 @@ logseq.graph-parser.property gp-property logseq.graph-parser.util.db db-util logseq.outliner.core outliner-core + logseq.outliner.op outliner-op logseq.outliner.pipeline outliner-pipeline logseq.outliner.datascript-report ds-report medley.core medley diff --git a/deps/db/src/logseq/db/sqlite/common_db.cljs b/deps/db/src/logseq/db/sqlite/common_db.cljs index d165bdf0c5..3a5317b761 100644 --- a/deps/db/src/logseq/db/sqlite/common_db.cljs +++ b/deps/db/src/logseq/db/sqlite/common_db.cljs @@ -99,9 +99,7 @@ (defn get-latest-journals [db n] - (let [date (js/Date.) - _ (.setDate date (- (.getDate date) (dec n))) - today (date-time-util/date->int (js/Date.))] + (let [today (date-time-util/date->int (js/Date.))] (->> (d/q '[:find [(pull ?page [*]) ...] :in $ ?today diff --git a/deps/db/test/logseq/db/sqlite/common_db_test.cljs b/deps/db/test/logseq/db/sqlite/common_db_test.cljs index f1b43805e6..67325e9f82 100644 --- a/deps/db/test/logseq/db/sqlite/common_db_test.cljs +++ b/deps/db/test/logseq/db/sqlite/common_db_test.cljs @@ -5,7 +5,9 @@ [datascript.core :as d] [logseq.db.sqlite.common-db :as sqlite-common-db] [logseq.db.frontend.schema :as db-schema] - [logseq.db.sqlite.db :as sqlite-db])) + [logseq.common.util.date-time :as date-time-util] + [logseq.db.sqlite.db :as sqlite-db] + [clojure.string :as string])) (use-fixtures :each @@ -40,16 +42,20 @@ "Correct file with content is found")))) (deftest restore-initial-data - (testing "Restore a journal page with its block" + (testing "Restore a journal page" (create-graph-dir "tmp/graphs" "test-db") (let [conn* (sqlite-db/open-db! "tmp/graphs" "test-db") page-uuid (random-uuid) block-uuid (random-uuid) created-at (js/Date.now) + date-int (date-time-util/date->int (js/Date.)) + date-title (date-time-util/int->journal-title date-int "MMM do, yyyy") blocks [{:db/id 100001 :block/uuid page-uuid - :block/journal-day 20230629 - :block/name "jun 29th, 2023" + :block/journal? true + :block/journal-day date-int + :block/name (string/lower-case date-title) + :block/original-name date-title :block/created-at created-at :block/updated-at created-at} {:db/id 100002 @@ -62,9 +68,9 @@ ;; Simulate getting data from sqlite and restoring it for frontend conn (-> (sqlite-common-db/get-initial-data @conn*) (sqlite-common-db/restore-initial-data db-schema/schema-for-db-based-graph))] - (is (= blocks + (is (= (take 1 blocks) (->> (d/q '[:find (pull ?b [*]) :where [?b :block/created-at]] @conn) (map first))) - "Datascript db matches data inserted into sqlite")))) + "Journal page is included in initial restore while its block is not")))) diff --git a/deps/outliner/.carve/ignore b/deps/outliner/.carve/ignore index d22c72d50d..b84a32a883 100644 --- a/deps/outliner/.carve/ignore +++ b/deps/outliner/.carve/ignore @@ -4,3 +4,5 @@ logseq.outliner.cli.pipeline/add-listener logseq.outliner.core/*transaction-opts* ;; API fn logseq.outliner.datascript/transact! +;; API fn +logseq.outliner.op/apply-ops! From 121bc212e5ce262acc90cb69213fbf6f209fdeda Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Wed, 7 Feb 2024 12:42:00 -0500 Subject: [PATCH 57/60] chore: simplify create script with date helpers now available in dep --- .../tasks/db_graph/create_graph_with_properties.cljs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/scripts/src/logseq/tasks/db_graph/create_graph_with_properties.cljs b/scripts/src/logseq/tasks/db_graph/create_graph_with_properties.cljs index f210ae0264..259742711c 100644 --- a/scripts/src/logseq/tasks/db_graph/create_graph_with_properties.cljs +++ b/scripts/src/logseq/tasks/db_graph/create_graph_with_properties.cljs @@ -3,7 +3,7 @@ Also creates a page of queries that exercises most properties NOTE: This script is also used in CI to confirm graph creation works" (:require [logseq.tasks.db-graph.create-graph :as create-graph] - [logseq.common.util :as common-util] + [logseq.common.util.date-time :as date-time-util] [logseq.db.frontend.property.type :as db-property-type] [clojure.string :as string] [datascript.core :as d] @@ -12,15 +12,9 @@ [nbb.core :as nbb])) (defn- date-journal-title [date] - (let [title (.toLocaleString date "en-US" #js {:month "short" :day "numeric" :year "numeric"}) - suffixes {1 "st" 21 "st" 31 "st" 2 "nd" 22 "nd" 3 "rd" 23 "rd" 33 "rd"}] - (common-util/page-name-sanity-lc - (string/replace-first title #"(\d+)" (str "$1" (suffixes (.getDate date) "th")))))) + (string/lower-case (date-time-util/int->journal-title (date-time-util/date->int date) "MMM do, yyyy"))) -(defn- date-journal-day [date] - (js/parseInt (str (.toLocaleString date "en-US" #js {:year "numeric"}) - (.toLocaleString date "en-US" #js {:month "2-digit"}) - (.toLocaleString date "en-US" #js {:day "2-digit"})))) +(def date-journal-day date-time-util/date->int) (defn- subtract-days [date days] From 5487b90feb9b262ff2b5e112ad933bd7ecf21d5d Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Wed, 7 Feb 2024 12:44:30 -0500 Subject: [PATCH 58/60] fix: enter on db import submits --- src/main/frontend/components/imports.cljs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/frontend/components/imports.cljs b/src/main/frontend/components/imports.cljs index cc12450bd3..eba2a4e09e 100644 --- a/src/main/frontend/components/imports.cljs +++ b/src/main/frontend/components/imports.cljs @@ -157,9 +157,9 @@ {:auto-focus true :on-change (fn [e] (reset! *input (util/evalue e))) - :on-key-press (fn [e] - (when (= "Enter" (util/ekey e)) - (on-submit)))}] + :on-key-down (fn [e] + (when (= "Enter" (util/ekey e)) + (on-submit)))}] [:div.mt-5.sm:mt-4.flex (ui/button "Submit" From f9063e7a7c3cf833fdaf121e4b135a994c6e244f Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Wed, 7 Feb 2024 12:48:33 -0500 Subject: [PATCH 59/60] fix: remove unused electron handler Removed in 0d6c65eec48fa3fd68099c9859fd73399c8c3f4b --- src/electron/electron/handler.cljs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/electron/electron/handler.cljs b/src/electron/electron/handler.cljs index 171add5765..471f3c7663 100644 --- a/src/electron/electron/handler.cljs +++ b/src/electron/electron/handler.cljs @@ -580,11 +580,6 @@ (f window graph-name) (state/set-state! :window/once-graph-ready nil))) -(defmethod handle :reloadWindowPage [^js win] - (logger/warn ::reload-window-page) - (when-let [web-content (.-webContents win)] - (.reload web-content))) - (defmethod handle :window-minimize [^js win] (.minimize win)) From 59456464f3a7fbd82e37573192d63e4ea980f43b Mon Sep 17 00:00:00 2001 From: Gabriel Horner Date: Wed, 7 Feb 2024 15:20:20 -0500 Subject: [PATCH 60/60] enhance: make loading indicator noticeable for publishing publishing apps can take awhile to load so we want the loading to be obvious, not muted --- src/main/frontend/components/container.cljs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/main/frontend/components/container.cljs b/src/main/frontend/components/container.cljs index 695affd233..7c45298956 100644 --- a/src/main/frontend/components/container.cljs +++ b/src/main/frontend/components/container.cljs @@ -581,10 +581,15 @@ nil db-restoring? - [:div.space-y-2 - (shui/skeleton {:class "h-8 w-1/3 mb-8"}) - (shui/skeleton {:class "h-6 w-full"}) - (shui/skeleton {:class "h-6 w-full"})] + (if config/publishing? + [:div.space-y-2 + (shui/skeleton {:class "h-8 w-1/3 mb-8 bg-gray-400"}) + (shui/skeleton {:class "h-6 w-full bg-gray-400"}) + (shui/skeleton {:class "h-6 w-full bg-gray-400"})] + [:div.space-y-2 + (shui/skeleton {:class "h-8 w-1/3 mb-8"}) + (shui/skeleton {:class "h-6 w-full"}) + (shui/skeleton {:class "h-6 w-full"})]) :else [:div