diff --git a/deps.edn b/deps.edn index 359ea31163..877f35d09d 100644 --- a/deps.edn +++ b/deps.edn @@ -3,8 +3,9 @@ {org.clojure/clojure {:mvn/version "1.11.1"} rum/rum {:mvn/version "0.12.9"} - datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork - :sha "8c7168b0c29ae2dc3c3efb5d28188916e991bde1"} + datascript/datascript {:local/root "../../Datascript"} + ;; datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork + ;; :sha "8c7168b0c29ae2dc3c3efb5d28188916e991bde1"} datascript-transit/datascript-transit {:mvn/version "0.3.0"} borkdude/rewrite-edn {:mvn/version "0.4.7"} diff --git a/deps/db/deps.edn b/deps/db/deps.edn index 65ebf2f501..2a874a0c69 100644 --- a/deps/db/deps.edn +++ b/deps/db/deps.edn @@ -1,7 +1,8 @@ {:deps ;; These deps are kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn - {datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork - :sha "8c7168b0c29ae2dc3c3efb5d28188916e991bde1"} + {datascript/datascript {:local/root "../../../../Datascript"} + ;; datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork + ;; :sha "8c7168b0c29ae2dc3c3efb5d28188916e991bde1"} datascript-transit/datascript-transit {:mvn/version "0.3.0" :exclusions [datascript/datascript]} cljs-bean/cljs-bean {:mvn/version "1.5.0"} diff --git a/deps/db/src/logseq/db/sqlite/common_db.cljs b/deps/db/src/logseq/db/sqlite/common_db.cljs index 4a0d5eb854..7a7a57bbed 100644 --- a/deps/db/src/logseq/db/sqlite/common_db.cljs +++ b/deps/db/src/logseq/db/sqlite/common_db.cljs @@ -282,7 +282,7 @@ (defn create-kvs-table! "Creates a sqlite table for use with datascript.storage if one doesn't exist" [sqlite-db] - (.exec sqlite-db "create table if not exists kvs (addr INTEGER primary key, content TEXT)")) + (.exec sqlite-db "create table if not exists kvs (addr INTEGER primary key, content TEXT, addresses JSON)")) (defn get-storage-conn "Given a datascript storage, returns a datascript connection for it" diff --git a/deps/outliner/deps.edn b/deps/outliner/deps.edn index e6957354b3..fff036dcf6 100644 --- a/deps/outliner/deps.edn +++ b/deps/outliner/deps.edn @@ -1,7 +1,8 @@ {:deps ;; External deps should be kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn - {datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork - :sha "8c7168b0c29ae2dc3c3efb5d28188916e991bde1"} + {datascript/datascript {:local/root "../../../../Datascript"} + ;; datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork + ;; :sha "8c7168b0c29ae2dc3c3efb5d28188916e991bde1"} logseq/db {:local/root "../db"} logseq/graph-parser {:local/root "../db"} com.cognitect/transit-cljs {:mvn/version "0.8.280"} diff --git a/src/main/frontend/worker/db/migrate.cljs b/src/main/frontend/worker/db/migrate.cljs index 5f53916265..8fffe8a051 100644 --- a/src/main/frontend/worker/db/migrate.cljs +++ b/src/main/frontend/worker/db/migrate.cljs @@ -117,8 +117,9 @@ :else (first types)) types)] - [[:db/retract id :block/type] - [:db/add id :block/type type]])))) + (when type + [[:db/retract id :block/type] + [:db/add id :block/type type]]))))) schema (:schema db)] (ldb/transact! conn new-type-tx {:db-migrate? true}) (d/reset-schema! conn (update schema :block/type #(assoc % :db/cardinality :db.cardinality/one))) diff --git a/src/main/frontend/worker/db_worker.cljs b/src/main/frontend/worker/db_worker.cljs index cf3f0b45e8..1059a74c92 100644 --- a/src/main/frontend/worker/db_worker.cljs +++ b/src/main/frontend/worker/db_worker.cljs @@ -5,6 +5,7 @@ [cljs-bean.core :as bean] [clojure.edn :as edn] [clojure.string :as string] + [clojure.set :as set] [datascript.core :as d] [datascript.storage :refer [IStorage]] [frontend.common.file.core :as common-file] @@ -88,6 +89,29 @@ [^js pool data] (.importDb ^js pool repo-path data)) +(comment + (defn- gc-kvs-table! + [^Object db] + (let [schema (some->> (.exec db #js {:sql "select content from kvs where addr = 0" + :rowMode "array"}) + bean/->clj + ffirst + sqlite-util/transit-read) + result (->> (.exec db #js {:sql "select addr, addresses from kvs" + :rowMode "array"}) + bean/->clj + (map (fn [[addr addresses]] + [addr (bean/->clj (js/JSON.parse addresses))]))) + used-addresses (set (concat (mapcat second result) + [0 1 (:eavt schema) (:avet schema) (:aevt schema)])) + unused-addresses (set/difference (set (map first result)) used-addresses)] + (when unused-addresses + (prn :debug :db-gc :unused-addresses unused-addresses) + (.transaction db (fn [tx] + (doseq [addr unused-addresses] + (.exec tx #js {:sql "Delete from kvs where addr = ?" + :bind #js [addr]})))))))) + (defn upsert-addr-content! "Upsert addr+data-seq" [repo data delete-addrs & {:keys [client-ops-db?] :or {client-ops-db? false}}] @@ -95,37 +119,51 @@ (assert (some? db) "sqlite db not exists") (.transaction db (fn [tx] (doseq [item data] - (.exec tx #js {:sql "INSERT INTO kvs (addr, content) values ($addr, $content) on conflict(addr) do update set content = $content" - :bind item})) - - (doseq [addr delete-addrs] - (.exec db #js {:sql "Delete from kvs where addr = ?" - :bind #js [addr]})))))) + (.exec tx #js {:sql "INSERT INTO kvs (addr, content, addresses) values ($addr, $content, $addresses) on conflict(addr) do update set content = $content, addresses = $addresses" + :bind item})))) + (when (seq delete-addrs) + (.transaction db (fn [tx] + (prn :debug :delete-addrs delete-addrs) + (doseq [addr delete-addrs] + (.exec tx #js {:sql "Delete from kvs WHERE addr = ? AND NOT EXISTS (SELECT 1 FROM json_each(addresses) WHERE value = ?);" + :bind #js [addr]}))))))) (defn restore-data-from-addr [repo addr & {:keys [client-ops-db?] :or {client-ops-db? false}}] (let [^Object db (worker-state/get-sqlite-conn repo (if client-ops-db? :client-ops :db))] (assert (some? db) "sqlite db not exists") - (when-let [content (-> (.exec db #js {:sql "select content from kvs where addr = ?" - :bind #js [addr] - :rowMode "array"}) - ffirst)] - (try - (let [data (sqlite-util/transit-read content)] - (if-let [addresses (:addresses data)] - (assoc data :addresses (bean/->js addresses)) - data)) - (catch :default _e ; TODO: remove this once db goes to test - (edn/read-string content)))))) + (when-let [result (-> (.exec db #js {:sql "select content, addresses from kvs where addr = ?" + :bind #js [addr] + :rowMode "array"}) + first)] + (let [[content addresses] (bean/->clj result) + addresses (when addresses + (js/JSON.parse addresses)) + data (sqlite-util/transit-read content)] + (if (and addresses (map? data)) + (assoc data :addresses addresses) + data))))) (defn new-sqlite-storage [repo _opts] (reify IStorage (-store [_ addr+data-seq delete-addrs] - (let [data (map + (let [used-addrs (set (mapcat + (fn [[addr data]] + (cons addr + (when (map? data) + (:addresses data)))) + addr+data-seq)) + delete-addrs (remove used-addrs delete-addrs) + data (map (fn [[addr data]] - #js {:$addr addr - :$content (sqlite-util/transit-write data)}) + (let [data' (if (map? data) (dissoc data :addresses) data) + addresses (when (map? data) + (when-let [addresses (:addresses data)] + (js/JSON.stringify (bean/->js addresses))))] + #js {:$addr addr + :$content (sqlite-util/transit-write data') + :$addresses addresses})) addr+data-seq)] (upsert-addr-content! repo data delete-addrs))) @@ -136,10 +174,22 @@ [repo] (reify IStorage (-store [_ addr+data-seq delete-addrs] - (let [data (map + (let [used-addrs (set (mapcat + (fn [[addr data]] + (cons addr + (when (map? data) + (:addresses data)))) + addr+data-seq)) + delete-addrs (remove used-addrs delete-addrs) + data (map (fn [[addr data]] - #js {:$addr addr - :$content (sqlite-util/transit-write data)}) + (let [data' (if (map? data) (dissoc data :addresses) data) + addresses (when (map? data) + (when-let [addresses (:addresses data)] + (js/JSON.stringify (bean/->js addresses))))] + #js {:$addr addr + :$content (sqlite-util/transit-write data') + :$addresses addresses})) addr+data-seq)] (upsert-addr-content! repo data delete-addrs :client-ops-db? true))) @@ -201,7 +251,8 @@ (let [schema (sqlite-util/get-schema repo) conn (sqlite-common-db/get-storage-conn storage schema) client-ops-conn (sqlite-common-db/get-storage-conn client-ops-storage client-op/schema-in-db) - initial-data-exists? (d/entity @conn :logseq.class/Root)] + initial-data-exists? (and (d/entity @conn :logseq.class/Root) + (= "db" (:kv/value (d/entity @conn :logseq.kv/db-type))))] (swap! *datascript-conns assoc repo conn) (swap! *client-ops-conns assoc repo client-ops-conn) (when (and (sqlite-util/db-based-graph? repo) (not initial-data-exists?)) @@ -209,9 +260,12 @@ initial-data (sqlite-create-graph/build-db-initial-data config)] (d/transact! conn initial-data {:initial-db? true}))) - (when-not (ldb/page-exists? @conn common-config/views-page-name "hidden") - (ldb/create-views-page! conn)) + (try + (when-not (ldb/page-exists? @conn common-config/views-page-name "hidden") + (ldb/create-views-page! conn)) + (catch :default _e)) + ;; (gc-kvs-table! db) (db-migrate/migrate conn search-db) (db-listener/listen-db-changes! repo conn)))))