diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index e9d16b7b8c..81d0b2ce99 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -104,6 +104,7 @@ frontend.modules.outliner.core outliner-core frontend.mobile.util mobile-util frontend.page page + frontend.persist-db persist-db frontend.schema.handler.common-config common-config-schema frontend.search search frontend.state state diff --git a/deps.edn b/deps.edn index 74ca9e36db..907aa72a22 100644 --- a/deps.edn +++ b/deps.edn @@ -3,8 +3,8 @@ {org.clojure/clojure {:mvn/version "1.11.1"} rum/rum {:mvn/version "0.12.9"} - datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork - :sha "ec937bc2bc932f0e1fdbbc1f67413d55c18ddc4a"} + datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork + :sha "15234c650df20844ec089c1738d28c00eccacd6e"} datascript-transit/datascript-transit {:mvn/version "0.3.0"} borkdude/rewrite-edn {:mvn/version "0.4.7"} diff --git a/deps/common/package.json b/deps/common/package.json index 8588a27f85..a4ffbd386a 100644 --- a/deps/common/package.json +++ b/deps/common/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "private": true, "devDependencies": { - "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v1" + "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v2" }, "scripts": { "test": "yarn nbb-logseq -cp test -m nextjournal.test-runner" diff --git a/deps/common/yarn.lock b/deps/common/yarn.lock index 6bd40cbfe8..3c18c2f3d3 100644 --- a/deps/common/yarn.lock +++ b/deps/common/yarn.lock @@ -2,9 +2,9 @@ # yarn lockfile v1 -"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v1": - version "1.2.173-feat-db-v1" - resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/e4910dfb12043404c97962d8faab3a946ab89f81" +"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v2": + version "1.2.173-feat-db-v2" + resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/9e891fd8a8a33ec78dd96a72bbdded191e1f0c1b" dependencies: import-meta-resolve "^2.1.0" diff --git a/deps/db/deps.edn b/deps/db/deps.edn index 1ad88d0e9a..6474a4f740 100644 --- a/deps/db/deps.edn +++ b/deps/db/deps.edn @@ -1,7 +1,7 @@ {:deps ;; External deps should be kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn {datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork - :sha "3b578bb4b39efda730cdae5f8e679060101b5110"} + :sha "15234c650df20844ec089c1738d28c00eccacd6e"} com.cognitect/transit-cljs {:mvn/version "0.8.280"} cljs-bean/cljs-bean {:mvn/version "1.5.0"}} :aliases diff --git a/deps/db/package.json b/deps/db/package.json index 409cdef9cb..015c0fce5b 100644 --- a/deps/db/package.json +++ b/deps/db/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "private": true, "devDependencies": { - "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v1" + "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v2" }, "dependencies": { "better-sqlite3": "8.0.1" diff --git a/deps/db/src/logseq/db/sqlite/db.cljs b/deps/db/src/logseq/db/sqlite/db.cljs index 14fac7cc3b..2df1bf942b 100644 --- a/deps/db/src/logseq/db/sqlite/db.cljs +++ b/deps/db/src/logseq/db/sqlite/db.cljs @@ -52,12 +52,16 @@ (defn upsert-addr-content! "Upsert addr+data-seq" - [db data] + [db data delete-addrs] (let [insert (.prepare db "INSERT INTO kvs (addr, content) values (@addr, @content) on conflict(addr) do update set content = @content") + delete (.prepare db "DELETE from kvs where addr = ?") insert-many (.transaction ^object db (fn [data] (doseq [item data] - (.run ^object insert item))))] + (.run ^object insert item)) + (doseq [addr delete-addrs] + (when addr + (.run ^object delete addr)))))] (insert-many data))) (defn restore-data-from-addr @@ -70,7 +74,7 @@ "Creates a datascript storage for sqlite. Should be functionally equivalent to db-worker/new-sqlite-storage" [db] (reify IStorage - (-store [_ addr+data-seq] + (-store [_ addr+data-seq delete-addrs] (let [data (->> (map (fn [[addr data]] @@ -78,7 +82,7 @@ :content (pr-str data)}) addr+data-seq) (to-array))] - (upsert-addr-content! db data))) + (upsert-addr-content! db data delete-addrs))) (-restore [_ addr] (let [content (restore-data-from-addr db addr)] (edn/read-string content))))) diff --git a/deps/db/yarn.lock b/deps/db/yarn.lock index fec3493db2..f9cf0acf7f 100644 --- a/deps/db/yarn.lock +++ b/deps/db/yarn.lock @@ -2,9 +2,9 @@ # yarn lockfile v1 -"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v1": - version "1.2.173-feat-db-v1" - resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/e4910dfb12043404c97962d8faab3a946ab89f81" +"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v2": + version "1.2.173-feat-db-v2" + resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/9e891fd8a8a33ec78dd96a72bbdded191e1f0c1b" dependencies: import-meta-resolve "^2.1.0" diff --git a/deps/graph-parser/package.json b/deps/graph-parser/package.json index 0b462b7250..56e9374be0 100644 --- a/deps/graph-parser/package.json +++ b/deps/graph-parser/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "private": true, "devDependencies": { - "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v1", + "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v2", "better-sqlite3": "8.0.1" }, "dependencies": { diff --git a/deps/graph-parser/yarn.lock b/deps/graph-parser/yarn.lock index 13b86d9c99..0b1bcdba0e 100644 --- a/deps/graph-parser/yarn.lock +++ b/deps/graph-parser/yarn.lock @@ -2,9 +2,9 @@ # yarn lockfile v1 -"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v1": - version "1.2.173-feat-db-v1" - resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/e4910dfb12043404c97962d8faab3a946ab89f81" +"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v2": + version "1.2.173-feat-db-v2" + resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/9e891fd8a8a33ec78dd96a72bbdded191e1f0c1b" dependencies: import-meta-resolve "^2.1.0" diff --git a/deps/outliner/deps.edn b/deps/outliner/deps.edn index b8ac37cffd..c4f21f759e 100644 --- a/deps/outliner/deps.edn +++ b/deps/outliner/deps.edn @@ -1,7 +1,7 @@ {:deps ;; External deps should be kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn {datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork - :sha "3b578bb4b39efda730cdae5f8e679060101b5110"} + :sha "15234c650df20844ec089c1738d28c00eccacd6e"} logseq/db {:local/root "../db"} com.cognitect/transit-cljs {:mvn/version "0.8.280"}} :aliases diff --git a/deps/outliner/package.json b/deps/outliner/package.json index 77f34a6e12..590e1626f6 100644 --- a/deps/outliner/package.json +++ b/deps/outliner/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "private": true, "devDependencies": { - "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v1" + "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v2" }, "dependencies": { "better-sqlite3": "8.0.1" diff --git a/deps/outliner/yarn.lock b/deps/outliner/yarn.lock index 9209f2e203..00f7551efd 100644 --- a/deps/outliner/yarn.lock +++ b/deps/outliner/yarn.lock @@ -2,9 +2,9 @@ # yarn lockfile v1 -"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v1": - version "1.2.173-feat-db-v1" - resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/e4910dfb12043404c97962d8faab3a946ab89f81" +"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v2": + version "1.2.173-feat-db-v2" + resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/9e891fd8a8a33ec78dd96a72bbdded191e1f0c1b" dependencies: import-meta-resolve "^2.1.0" diff --git a/deps/publishing/package.json b/deps/publishing/package.json index 78208d05d3..d70534df3c 100644 --- a/deps/publishing/package.json +++ b/deps/publishing/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "private": true, "devDependencies": { - "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v1", + "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v2", "mldoc": "^1.5.1" }, "dependencies": { diff --git a/deps/publishing/yarn.lock b/deps/publishing/yarn.lock index 92d389e145..9595ffc1a3 100644 --- a/deps/publishing/yarn.lock +++ b/deps/publishing/yarn.lock @@ -2,9 +2,9 @@ # yarn lockfile v1 -"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v1": - version "1.2.173-feat-db-v1" - resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/e4910dfb12043404c97962d8faab3a946ab89f81" +"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v2": + version "1.2.173-feat-db-v2" + resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/9e891fd8a8a33ec78dd96a72bbdded191e1f0c1b" dependencies: import-meta-resolve "^2.1.0" diff --git a/package.json b/package.json index 8da8bf8628..b6475df162 100644 --- a/package.json +++ b/package.json @@ -101,7 +101,7 @@ "@logseq/capacitor-file-sync": "5.0.1", "@logseq/diff-merge": "0.2.2", "@logseq/react-tweet-embed": "1.3.1-1", - "@logseq/sqlite-wasm": "=0.0.7", + "@logseq/sqlite-wasm": "=0.1.0", "@radix-ui/colors": "^0.1.8", "@sentry/react": "^6.18.2", "@sentry/tracing": "^6.18.2", diff --git a/scripts/package.json b/scripts/package.json index 952e9906d7..4a7581f330 100644 --- a/scripts/package.json +++ b/scripts/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "private": true, "devDependencies": { - "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v1" + "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v2" }, "dependencies": { "better-sqlite3": "8.0.1", diff --git a/scripts/src/logseq/tasks/dev/db_and_file_graphs.clj b/scripts/src/logseq/tasks/dev/db_and_file_graphs.clj index 0bac0addda..617b778620 100644 --- a/scripts/src/logseq/tasks/dev/db_and_file_graphs.clj +++ b/scripts/src/logseq/tasks/dev/db_and_file_graphs.clj @@ -15,7 +15,8 @@ ["logseq.db.sqlite." "logseq.db.frontend.property" "logseq.db.frontend.malli-schema" "electron.db" "frontend.handler.db-based." - "frontend.components.property" "frontend.components.class" "frontend.components.db-based"])) + "frontend.components.property" "frontend.components.class" "frontend.components.db-based" + "frontend.persist-db"])) (def file-graph-ns "Namespaces or parent namespaces _only_ for file graphs" @@ -34,6 +35,9 @@ "src/main/frontend/components/property.cljs" "src/main/frontend/components/property" "src/main/frontend/components/db_based" + ;; TODO: Enable this when run-export-periodically is deleted or moved out of the ns + #_"src/main/frontend/persist_db.cljs" + "src/main/frontend/persist_db" "src/electron/electron/db.cljs"]) (def file-graph-paths diff --git a/scripts/yarn.lock b/scripts/yarn.lock index 4bbaf67256..c038afd803 100644 --- a/scripts/yarn.lock +++ b/scripts/yarn.lock @@ -2,9 +2,9 @@ # yarn lockfile v1 -"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v1": - version "1.2.173-feat-db-v1" - resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/e4910dfb12043404c97962d8faab3a946ab89f81" +"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v2": + version "1.2.173-feat-db-v2" + resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/9e891fd8a8a33ec78dd96a72bbdded191e1f0c1b" dependencies: import-meta-resolve "^2.1.0" diff --git a/src/main/electron/listener.cljs b/src/main/electron/listener.cljs index 33ab63ab97..2f04d77968 100644 --- a/src/main/electron/listener.cljs +++ b/src/main/electron/listener.cljs @@ -22,7 +22,7 @@ [logseq.graph-parser.util :as gp-util] [promesa.core :as p] [frontend.handler.property.util :as pu] - [frontend.persist-db :as persistent-db])) + [frontend.persist-db :as persist-db])) (defn- safe-api-call "Force the callback result to be nil, otherwise, ipc calls could lead to @@ -39,7 +39,7 @@ (notification/show! error :error))] (if (config/db-based-graph? repo) (-> - (p/let [_ (persistent-db/c (persist-db/c (persist-db/c (persist-db/> (keep (fn [child-id] (let [e (d/entity (:db-before tx-report) [:block/uuid child-id])] (when (:created-from-property (:block/metadata (:block/parent e))) @@ -60,23 +60,21 @@ parent-now))))) deleted-block-uuids) distinct)] (when (seq empty-property-parents) - (let [tx-data (->> - (mapcat (fn [b] - (let [{:keys [created-from-block created-from-property]} (:block/metadata b) - created-block (db/entity [:block/uuid created-from-block]) - properties (assoc (:block/properties created-block) created-from-property "")] - (when (and created-block created-from-property) - [[:db/retractEntity (:db/id b)] - [:db/add (:db/id created-block) :block/properties properties]]))) - empty-property-parents) - (remove nil?))] - (db/transact! repo tx-data {:outliner/transact? true - :replace? true}))))) + (->> + (mapcat (fn [b] + (let [{:keys [created-from-block created-from-property]} (:block/metadata b) + created-block (db/entity [:block/uuid created-from-block]) + properties (assoc (:block/properties created-block) created-from-property "")] + (when (and created-block created-from-property) + [[:db/retractEntity (:db/id b)] + [:db/add (:db/id created-block) :block/properties properties]]))) + empty-property-parents) + (remove nil?))))) (defn invoke-hooks [tx-report] (let [tx-meta (:tx-meta tx-report) - {:keys [compute-path-refs? from-disk? new-graph? replace?]} tx-meta] + {:keys [from-disk? new-graph? pipeline-replace?]} tx-meta] (when (and (not from-disk?) (not new-graph?)) (try @@ -87,51 +85,46 @@ (let [{:keys [pages blocks]} (ds-report/get-blocks-and-pages tx-report) repo (state/get-current-repo) - tx (when-not compute-path-refs? - (util/profile - "Compute path refs: " - (set (compute-block-path-refs-tx tx-report blocks)))) - tx-report' (if (seq tx) - (let [refs-tx-data' (:tx-data (db/transact! repo tx {:outliner/transact? true - :replace? true - :compute-path-refs? true}))] - ;; merge - (assoc tx-report :tx-data (concat (:tx-data tx-report) refs-tx-data'))) - tx-report) importing? (:graph/importing @state/state) - deleted-block-uuids (set (outliner-pipeline/filter-deleted-blocks (:tx-data tx-report)))] + deleted-block-uuids (set (outliner-pipeline/filter-deleted-blocks (:tx-data tx-report))) + replace-full-tx (when-not pipeline-replace? + (concat + ;; block path refs + (util/profile + "Compute path refs: " + (set (compute-block-path-refs-tx tx-report blocks))) - (when (and (seq deleted-block-uuids) (not replace?) - (not compute-path-refs?)) - (delete-property-parent-block-if-empty! repo tx-report deleted-block-uuids)) + ;; delete empty property parent block + (when (seq deleted-block-uuids) + (delete-property-parent-block-if-empty tx-report deleted-block-uuids)) - (let [updated-blocks (remove (fn [b] (contains? (set deleted-block-uuids) (:block/uuid b))) blocks) - tx-id (get-in tx-report' [:tempids :db/current-tx]) - update-tx-ids (->> - (map (fn [b] - (when-let [db-id (:db/id b)] - {:db/id db-id - :block/tx-id tx-id})) updated-blocks) - (remove nil?))] - (when (and (seq update-tx-ids) - (not (:update-tx-ids? tx-meta))) - (db/transact! repo update-tx-ids {:replace? true - :update-tx-ids? true})) - (when (and (config/db-based-graph? repo) (not config/publishing?)) - (persist-db/> + (map (fn [b] + (when-let [db-id (:db/id b)] + {:db/id db-id + :block/tx-id tx-id})) updated-blocks) + (remove nil?)))))] - (when-not importing? - (react/refresh! repo tx-report')) + (when (and (not config/publishing?) (not pipeline-replace?)) + (let [tx-report' (db/transact! repo replace-full-tx {:replace? true + :pipeline-replace? true}) + full-tx-data (concat (:tx-data tx-report) (:tx-data tx-report'))] + (persist-db/InBrowser)) @@ -43,4 +40,21 @@ ;; @shuyu Do we still need this? (defn (.listDB sqlite) (p/then (fn [result] (bean/->clj result))) - (p/catch (fn [error] - (prn :debug :list-db-error (js/Date.)) - (if (= "NoModificationAllowedError" (.-name error)) - (state/pub-event! [:db/multiple-tabs-opfs-failed]) - (notification/show! [:div (str "SQLiteDB error: " error)] :error)) - []))))) + (p/catch sqlite-error-handler)))) ( (p/let [_ (.createOrOpenDB sqlite repo) _ (ipc/ipc :db-open repo)] (.getInitialData sqlite repo)) - (p/catch (fn [error] - (prn :debug :fetch-initial-data-error repo) - (js/console.error error) - (notification/show! [:div (str "SQLiteDB fetch error: " error)] :error) {}))))) + (p/catch sqlite-error-handler)))) (