clone/logseq
1
0
Fork 0
mirror of https://github.com/logseq/logseq.git synced 2026-04-24 14:14:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: don't delete rows from kvs if they're still used by other rows

Browse Source
This commit is contained in:
Tienson Qin
2024-08-22 21:28:44 +08:00
parent 15a78d4f1b
commit 674dbfb890
6 changed files with 93 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
deps.edn
View File

@@ -3,8 +3,9 @@
{org.clojure/clojure {:mvn/version "1.11.1"}
rum/rum {:mvn/version "0.12.9"}
datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
:sha "8c7168b0c29ae2dc3c3efb5d28188916e991bde1"}
datascript/datascript {:local/root "../../Datascript"}
;; datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
;; :sha "8c7168b0c29ae2dc3c3efb5d28188916e991bde1"}
datascript-transit/datascript-transit {:mvn/version "0.3.0"}
borkdude/rewrite-edn {:mvn/version "0.4.7"}

5
deps/db/deps.edn vendored
View File

@@ -1,7 +1,8 @@
{:deps
;; These deps are kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn
{datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
:sha "8c7168b0c29ae2dc3c3efb5d28188916e991bde1"}
{datascript/datascript {:local/root "../../../../Datascript"}
;; datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
;; :sha "8c7168b0c29ae2dc3c3efb5d28188916e991bde1"}
datascript-transit/datascript-transit {:mvn/version "0.3.0"
:exclusions [datascript/datascript]}
cljs-bean/cljs-bean {:mvn/version "1.5.0"}

2
deps/db/src/logseq/db/sqlite/common_db.cljs vendored
View File

@@ -282,7 +282,7 @@
(defn create-kvs-table!
"Creates a sqlite table for use with datascript.storage if one doesn't exist"
[sqlite-db]
(.exec sqlite-db "create table if not exists kvs (addr INTEGER primary key, content TEXT)"))
(.exec sqlite-db "create table if not exists kvs (addr INTEGER primary key, content TEXT, addresses JSON)"))
(defn get-storage-conn
"Given a datascript storage, returns a datascript connection for it"

5
deps/outliner/deps.edn vendored
View File

@@ -1,7 +1,8 @@
{:deps
;; External deps should be kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn
{datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
:sha "8c7168b0c29ae2dc3c3efb5d28188916e991bde1"}
{datascript/datascript {:local/root "../../../../Datascript"}
;; datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
;; :sha "8c7168b0c29ae2dc3c3efb5d28188916e991bde1"}
logseq/db {:local/root "../db"}
logseq/graph-parser {:local/root "../db"}
com.cognitect/transit-cljs {:mvn/version "0.8.280"}

5
src/main/frontend/worker/db/migrate.cljs
View File

@@ -117,8 +117,9 @@
:else
(first types))
types)]
[[:db/retract id :block/type]
[:db/add id :block/type type]]))))
(when type
[[:db/retract id :block/type]
[:db/add id :block/type type]])))))
schema (:schema db)]
(ldb/transact! conn new-type-tx {:db-migrate? true})
(d/reset-schema! conn (update schema :block/type #(assoc % :db/cardinality :db.cardinality/one)))

106
src/main/frontend/worker/db_worker.cljs
View File

@@ -5,6 +5,7 @@
[cljs-bean.core :as bean]
[clojure.edn :as edn]
[clojure.string :as string]
[clojure.set :as set]
[datascript.core :as d]
[datascript.storage :refer [IStorage]]
[frontend.common.file.core :as common-file]
@@ -88,6 +89,29 @@
[^js pool data]
(.importDb ^js pool repo-path data))
(comment
(defn- gc-kvs-table!
[^Object db]
(let [schema (some->> (.exec db #js {:sql "select content from kvs where addr = 0"
:rowMode "array"})
bean/->clj
ffirst
sqlite-util/transit-read)
result (->> (.exec db #js {:sql "select addr, addresses from kvs"
:rowMode "array"})
bean/->clj
(map (fn [[addr addresses]]
[addr (bean/->clj (js/JSON.parse addresses))])))
used-addresses (set (concat (mapcat second result)
[0 1 (:eavt schema) (:avet schema) (:aevt schema)]))
unused-addresses (set/difference (set (map first result)) used-addresses)]
(when unused-addresses
(prn :debug :db-gc :unused-addresses unused-addresses)
(.transaction db (fn [tx]
(doseq [addr unused-addresses]
(.exec tx #js {:sql "Delete from kvs where addr = ?"
:bind #js [addr]}))))))))
(defn upsert-addr-content!
"Upsert addr+data-seq"
[repo data delete-addrs & {:keys [client-ops-db?] :or {client-ops-db? false}}]
@@ -95,37 +119,51 @@
(assert (some? db) "sqlite db not exists")
(.transaction db (fn [tx]
(doseq [item data]
(.exec tx #js {:sql "INSERT INTO kvs (addr, content) values ($addr, $content) on conflict(addr) do update set content = $content"
:bind item}))
(doseq [addr delete-addrs]
(.exec db #js {:sql "Delete from kvs where addr = ?"
:bind #js [addr]}))))))
(.exec tx #js {:sql "INSERT INTO kvs (addr, content, addresses) values ($addr, $content, $addresses) on conflict(addr) do update set content = $content, addresses = $addresses"
:bind item}))))
(when (seq delete-addrs)
(.transaction db (fn [tx]
(prn :debug :delete-addrs delete-addrs)
(doseq [addr delete-addrs]
(.exec tx #js {:sql "Delete from kvs WHERE addr = ? AND NOT EXISTS (SELECT 1 FROM json_each(addresses) WHERE value = ?);"
:bind #js [addr]})))))))
(defn restore-data-from-addr
[repo addr & {:keys [client-ops-db?] :or {client-ops-db? false}}]
(let [^Object db (worker-state/get-sqlite-conn repo (if client-ops-db? :client-ops :db))]
(assert (some? db) "sqlite db not exists")
(when-let [content (-> (.exec db #js {:sql "select content from kvs where addr = ?"
:bind #js [addr]
:rowMode "array"})
ffirst)]
(try
(let [data (sqlite-util/transit-read content)]
(if-let [addresses (:addresses data)]
(assoc data :addresses (bean/->js addresses))
data))
(catch :default _e ; TODO: remove this once db goes to test
(edn/read-string content))))))
(when-let [result (-> (.exec db #js {:sql "select content, addresses from kvs where addr = ?"
:bind #js [addr]
:rowMode "array"})
first)]
(let [[content addresses] (bean/->clj result)
addresses (when addresses
(js/JSON.parse addresses))
data (sqlite-util/transit-read content)]
(if (and addresses (map? data))
(assoc data :addresses addresses)
data)))))
(defn new-sqlite-storage
[repo _opts]
(reify IStorage
(-store [_ addr+data-seq delete-addrs]
(let [data (map
(let [used-addrs (set (mapcat
(fn [[addr data]]
(cons addr
(when (map? data)
(:addresses data))))
addr+data-seq))
delete-addrs (remove used-addrs delete-addrs)
data (map
(fn [[addr data]]
#js {:$addr addr
:$content (sqlite-util/transit-write data)})
(let [data' (if (map? data) (dissoc data :addresses) data)
addresses (when (map? data)
(when-let [addresses (:addresses data)]
(js/JSON.stringify (bean/->js addresses))))]
#js {:$addr addr
:$content (sqlite-util/transit-write data')
:$addresses addresses}))
addr+data-seq)]
(upsert-addr-content! repo data delete-addrs)))
@@ -136,10 +174,22 @@
[repo]
(reify IStorage
(-store [_ addr+data-seq delete-addrs]
(let [data (map
(let [used-addrs (set (mapcat
(fn [[addr data]]
(cons addr
(when (map? data)
(:addresses data))))
addr+data-seq))
delete-addrs (remove used-addrs delete-addrs)
data (map
(fn [[addr data]]
#js {:$addr addr
:$content (sqlite-util/transit-write data)})
(let [data' (if (map? data) (dissoc data :addresses) data)
addresses (when (map? data)
(when-let [addresses (:addresses data)]
(js/JSON.stringify (bean/->js addresses))))]
#js {:$addr addr
:$content (sqlite-util/transit-write data')
:$addresses addresses}))
addr+data-seq)]
(upsert-addr-content! repo data delete-addrs :client-ops-db? true)))
@@ -201,7 +251,8 @@
(let [schema (sqlite-util/get-schema repo)
conn (sqlite-common-db/get-storage-conn storage schema)
client-ops-conn (sqlite-common-db/get-storage-conn client-ops-storage client-op/schema-in-db)
initial-data-exists? (d/entity @conn :logseq.class/Root)]
initial-data-exists? (and (d/entity @conn :logseq.class/Root)
(= "db" (:kv/value (d/entity @conn :logseq.kv/db-type))))]
(swap! *datascript-conns assoc repo conn)
(swap! *client-ops-conns assoc repo client-ops-conn)
(when (and (sqlite-util/db-based-graph? repo) (not initial-data-exists?))
@@ -209,9 +260,12 @@
initial-data (sqlite-create-graph/build-db-initial-data config)]
(d/transact! conn initial-data {:initial-db? true})))
(when-not (ldb/page-exists? @conn common-config/views-page-name "hidden")
(ldb/create-views-page! conn))
(try
(when-not (ldb/page-exists? @conn common-config/views-page-name "hidden")
(ldb/create-views-page! conn))
(catch :default _e))
;; (gc-kvs-table! db)
(db-migrate/migrate conn search-db)
(db-listener/listen-db-changes! repo conn)))))