diff --git a/deps/db/src/logseq/db/sqlite/common_db.cljs b/deps/db/src/logseq/db/sqlite/common_db.cljs index 8a5eb04f9f..4ad928b7b2 100644 --- a/deps/db/src/logseq/db/sqlite/common_db.cljs +++ b/deps/db/src/logseq/db/sqlite/common_db.cljs @@ -1,7 +1,6 @@ (ns logseq.db.sqlite.common-db "Common sqlite db fns for browser and node" (:require [datascript.core :as d] - [logseq.db.frontend.schema :as db-schema] ["path" :as node-path] [clojure.string :as string] [logseq.db.sqlite.util :as sqlite-util])) @@ -14,8 +13,8 @@ (defn restore-initial-data "Given initial sqlite data, returns a datascript connection" - [datoms] - (d/conn-from-datoms datoms db-schema/schema-for-db-based-graph)) + [datoms schema] + (d/conn-from-datoms datoms schema)) (defn create-kvs-table! "Creates a sqlite table for use with datascript.storage if one doesn't exist" @@ -24,9 +23,9 @@ (defn get-storage-conn "Given a datascript storage, returns a datascript connection for it" - [storage] + [storage schema] (or (d/restore-conn storage) - (d/create-conn db-schema/schema-for-db-based-graph {:storage storage}))) + (d/create-conn schema {:storage storage}))) (defn sanitize-db-name [db-name] diff --git a/deps/db/src/logseq/db/sqlite/db.cljs b/deps/db/src/logseq/db/sqlite/db.cljs index 4ede8d94bb..ee284f01d5 100644 --- a/deps/db/src/logseq/db/sqlite/db.cljs +++ b/deps/db/src/logseq/db/sqlite/db.cljs @@ -7,7 +7,9 @@ [datascript.core :as d] [datascript.storage :refer [IStorage]] [goog.object :as gobj] - [clojure.edn :as edn])) + [clojure.edn :as edn] + [logseq.db.frontend.schema :as db-schema] + [logseq.db.sqlite.util :as sqlite-util])) ;; sqlite databases (defonce databases (atom nil)) @@ -79,11 +81,14 @@ (defn open-db! [graphs-dir db-name] (let [[db-sanitized-name db-full-path] (get-db-full-path graphs-dir db-name) - db (new sqlite db-full-path nil)] + db (new sqlite db-full-path nil) + schema (if (sqlite-util/db-based-graph? db-name) + db-schema/db-version-retract-attributes + db-schema/schema)] (sqlite-common-db/create-kvs-table! db) (swap! databases assoc db-sanitized-name db) (let [storage (new-sqlite-storage db) - conn (sqlite-common-db/get-storage-conn storage)] + conn (sqlite-common-db/get-storage-conn storage schema)] (swap! conns assoc db-sanitized-name conn))) nil) diff --git a/deps/db/src/logseq/db/sqlite/db.cljs.~f3d032d83dc8b313e17a5ac21b43f9152d7336b5~ b/deps/db/src/logseq/db/sqlite/db.cljs.~f3d032d83dc8b313e17a5ac21b43f9152d7336b5~ new file mode 100644 index 0000000000..4ede8d94bb --- /dev/null +++ b/deps/db/src/logseq/db/sqlite/db.cljs.~f3d032d83dc8b313e17a5ac21b43f9152d7336b5~ @@ -0,0 +1,98 @@ +(ns ^:node-only logseq.db.sqlite.db + "Sqlite fns for db graphs" + (:require ["path" :as node-path] + ["better-sqlite3" :as sqlite3] + [logseq.db.sqlite.common-db :as sqlite-common-db] + ;; FIXME: datascript.core has to come before datascript.storage or else nbb fails + [datascript.core :as d] + [datascript.storage :refer [IStorage]] + [goog.object :as gobj] + [clojure.edn :as edn])) + +;; sqlite databases +(defonce databases (atom nil)) +;; datascript conns +(defonce conns (atom nil)) + +;; Reference same sqlite default class in cljs + nbb without needing .cljc +(def sqlite (if (find-ns 'nbb.core) (aget sqlite3 "default") sqlite3)) + +(defn close! + [] + (when @databases + (doseq [[_ database] @databases] + (.close database)) + (reset! databases nil))) + +(def sanitize-db-name sqlite-common-db/sanitize-db-name) + +(def get-db-full-path sqlite-common-db/get-db-full-path) + +(defn get-conn + [repo] + (get @conns (sanitize-db-name repo))) + +(defn get-db-full-path + [graphs-dir db-name] + (let [db-name' (sanitize-db-name db-name) + graph-dir (node-path/join graphs-dir db-name')] + [db-name' (node-path/join graph-dir "db.sqlite")])) + +(defn query + [db sql] + (let [stmt (.prepare db sql)] + (.all ^object stmt))) + +(defn upsert-addr-content! + "Upsert addr+data-seq" + [db data] + (let [insert (.prepare db "INSERT INTO kvs (addr, content) values (@addr, @content) on conflict(addr) do update set content = @content") + insert-many (.transaction ^object db + (fn [data] + (doseq [item data] + (.run ^object insert item))))] + (insert-many data))) + +(defn restore-data-from-addr + [db addr] + (-> (query db (str "select content from kvs where addr = " addr)) + first + (gobj/get "content"))) + +(defn new-sqlite-storage + "Creates a datascript storage for sqlite. Should be functionally equivalent to db-worker/new-sqlite-storage" + [db] + (reify IStorage + (-store [_ addr+data-seq] + (let [data (->> + (map + (fn [[addr data]] + #js {:addr addr + :content (pr-str data)}) + addr+data-seq) + (to-array))] + (upsert-addr-content! db data))) + (-restore [_ addr] + (let [content (restore-data-from-addr db addr)] + (edn/read-string content))))) + +(defn open-db! + [graphs-dir db-name] + (let [[db-sanitized-name db-full-path] (get-db-full-path graphs-dir db-name) + db (new sqlite db-full-path nil)] + (sqlite-common-db/create-kvs-table! db) + (swap! databases assoc db-sanitized-name db) + (let [storage (new-sqlite-storage db) + conn (sqlite-common-db/get-storage-conn storage)] + (swap! conns assoc db-sanitized-name conn))) + nil) + +;; TODO: Remove as it looks unused +(defn transact! + [repo tx-data tx-meta] + (when-let [conn (get-conn repo)] + (try + (d/transact! conn tx-data tx-meta) + (catch :default e + (prn :debug :error) + (js/console.error e))))) diff --git a/deps/db/src/logseq/db/sqlite/util.cljs b/deps/db/src/logseq/db/sqlite/util.cljs index 23e93e053f..de22c844ab 100644 --- a/deps/db/src/logseq/db/sqlite/util.cljs +++ b/deps/db/src/logseq/db/sqlite/util.cljs @@ -2,10 +2,22 @@ "Utils fns for backend sqlite db" (:require [cljs-time.coerce :as tc] [cljs-time.core :as t] - [clojure.string :as string])) + [clojure.string :as string] + [logseq.db.frontend.schema :as db-schema])) (defonce db-version-prefix "logseq_db_") +(defn db-based-graph? + [graph-name] + (string/starts-with? graph-name db-version-prefix)) + +(defn get-schema + "Returns schema for given repo" + [repo] + (if (db-based-graph? repo) + db-schema/schema-for-db-based-graph + db-schema/schema)) + (defn time-ms "Copy of util/time-ms. Too basic to couple this to main app" [] @@ -35,4 +47,4 @@ (merge {:block/type "property" :block/journal? false :block/format :markdown} - block))) \ No newline at end of file + block))) diff --git a/scripts/src/logseq/tasks/dev/db_and_file_graphs.clj b/scripts/src/logseq/tasks/dev/db_and_file_graphs.clj index 0bac0addda..51d483e0ba 100644 --- a/scripts/src/logseq/tasks/dev/db_and_file_graphs.clj +++ b/scripts/src/logseq/tasks/dev/db_and_file_graphs.clj @@ -39,7 +39,6 @@ (def file-graph-paths "Paths _only_ for file graphs" ["src/main/frontend/handler/file_based" "src/main/frontend/handler/conversion.cljs" "src/main/frontend/handler/file_sync.cljs" - "src/main/frontend/db/file_based" "src/main/frontend/fs" "src/main/frontend/components/conversion.cljs" "src/main/frontend/components/file_sync.cljs" "src/main/frontend/util/fs.cljs" @@ -101,4 +100,4 @@ (validate-file-ns-not-in-db) (validate-file-concepts-not-in-db) (validate-multi-graph-fns-not-in-file-or-db) - (println "✅ All checks passed!")) \ No newline at end of file + (println "✅ All checks passed!")) diff --git a/src/electron/electron/handler.cljs b/src/electron/electron/handler.cljs index a7d7398a69..81bd794d46 100644 --- a/src/electron/electron/handler.cljs +++ b/src/electron/electron/handler.cljs @@ -202,13 +202,6 @@ (bean/->js {:path path :files files}))) -(defn- sanitize-graph-name - [graph-name] - (when graph-name - (-> graph-name - (string/replace "/" "++") - (string/replace ":" "+3A+")))) - (defn- graph-name->path [graph-name] (when graph-name @@ -297,36 +290,9 @@ (defmethod handle :readGraphTxIdInfo [_win [_ root]] (read-txid-info! root)) -(defn- get-graph-path - [graph-name] +(defmethod handle :deleteGraph [_window [_ graph graph-name _db-based?]] (when graph-name - (let [graph-name (sanitize-graph-name graph-name) - dir (get-graphs-dir)] - (.join node-path dir (str graph-name ".transit"))))) - -(defn- get-serialized-graph - [graph-name] - (when graph-name - (when-let [file-path (get-graph-path graph-name)] - (when (fs/existsSync file-path) - (utils/read-file file-path))))) - -(defmethod handle :getSerializedGraph [_window [_ graph-name]] - (get-serialized-graph graph-name)) - -(defmethod handle :saveGraph [_window [_ graph-name value-str]] - ;; NOTE: graph-name is a plain "local" for demo graph. - (when (and graph-name value-str (not (= "local" graph-name))) - (when-let [file-path (get-graph-path graph-name)] - (fs/writeFileSync file-path value-str)))) - -(defmethod handle :deleteGraph [_window [_ graph graph-name db-based?]] - (when graph-name - (if (and db-based? graph) - (db/unlink-graph! graph) - (when-let [file-path (get-graph-path graph-name)] - (when (fs/existsSync file-path) - (fs-extra/removeSync file-path)))))) + (db/unlink-graph! graph))) (defmethod handle :persistent-dbs-saved [_window _] (async/put! state/persistent-dbs-chan true) diff --git a/src/main/frontend/config.cljs b/src/main/frontend/config.cljs index a428182fa3..76a792b029 100644 --- a/src/main/frontend/config.cljs +++ b/src/main/frontend/config.cljs @@ -380,7 +380,7 @@ [s] (boolean (and (string? s) - (string/starts-with? s db-version-prefix)))) + (string/starts-with? s db-version-prefix)))) (defn get-local-asset-absolute-path [s] diff --git a/src/main/frontend/db/conn.cljs b/src/main/frontend/db/conn.cljs index 6182df85eb..ebbd61b2a6 100644 --- a/src/main/frontend/db/conn.cljs +++ b/src/main/frontend/db/conn.cljs @@ -10,7 +10,8 @@ [logseq.db :as ldb] [logseq.db.frontend.schema :as db-schema] [logseq.graph-parser.util :as gp-util] - [datascript.core :as d])) + [datascript.core :as d] + [logseq.db.sqlite.util :as sqlite-util])) (defonce conns (atom {})) @@ -54,12 +55,7 @@ (str (if (util/electron?) "" config/idb-db-prefix) path)))) -(defn get-schema - "Returns schema for given repo" - [repo] - (if (config/db-based-graph? repo) - db-schema/schema-for-db-based-graph - db-schema/schema)) +(def get-schema sqlite-util/get-schema) (defn get-db ([] @@ -75,9 +71,6 @@ @conn conn))))) -(defn reset-conn! [conn db] - (reset! conn db)) - (defn remove-conn! [repo] (swap! conns dissoc (datascript-db repo))) diff --git a/src/main/frontend/db/file_based/migrate.cljs b/src/main/frontend/db/file_based/migrate.cljs deleted file mode 100644 index 37cc57a327..0000000000 --- a/src/main/frontend/db/file_based/migrate.cljs +++ /dev/null @@ -1,87 +0,0 @@ -(ns frontend.db.file-based.migrate - "Do DB migration for file graphs, in a version-by-version style. - - `:schema/version` is not touched" - (:require [clojure.string :as string] - [datascript.core :as d] - [frontend.config :as config] - [frontend.state :as state] - [logseq.common.path :as path] - [logseq.db.frontend.schema :as db-schema])) - - -(defn get-schema-version - "Get schema version from db, the current version is defined in db-schema/version" - [db] - (d/q - '[:find (max ?v) . - :where - [_ :schema/version ?v]] - db)) - -(defn get-collapsed-blocks - [db] - (d/q - '[:find [?b ...] - :where - [?b :block/properties ?properties] - [(get ?properties :collapsed) ?collapsed] - [(= true ?collapsed)]] - db)) - -(defn migrate-collapsed-blocks [db] - (when db - (let [collapsed-blocks (get-collapsed-blocks db)] - (if (seq collapsed-blocks) - (let [tx-data (map (fn [id] {:db/id id - :block/collapsed? true}) collapsed-blocks)] - (prn :migrate-collapsed-blocks {:count (count collapsed-blocks)}) - (d/db-with db tx-data)) - db)))) - -(defn migrate-absolute-file-path-to-relative [db] - (when db - (let [all-files (d/q - '[:find [(pull ?b [:db/id :file/path]) ...] - :where - [?b :file/path]] - db) - repo-dir (config/get-repo-dir (state/get-current-repo))] - (if (seq all-files) - (let [tx-data (->> all-files - (filter (fn [db-file] - (and (path/absolute? (:file/path db-file)) - (string/starts-with? (:file/path db-file) repo-dir)))) - (mapv (fn [db-file] {:db/id (:db/id db-file) - :file/path (path/trim-dir-prefix repo-dir (:file/path db-file))})))] - (when tx-data - (state/pub-event! [:notification/show - {:content [:div "Migrated from an old version of DB, please re-index the graph from the graph list dropdown."] - :status :warning - :clear? false}])) - (prn :migrate-absolute-file-path-to-relative {:count (count tx-data)}) - (d/db-with db tx-data)) - db)))) - - -(defmulti do-migration get-schema-version) - -(defmethod do-migration 0 - [db] - (-> db - migrate-collapsed-blocks - migrate-absolute-file-path-to-relative)) - -(defmethod do-migration 1 - [db] - (-> db - migrate-absolute-file-path-to-relative)) - -(defmethod do-migration :default - [db] - db) - -(defn migrate - [db] - (prn ::migrate {:from (get-schema-version db) :to db-schema/version}) - (do-migration db)) diff --git a/src/main/frontend/db/persist.cljs b/src/main/frontend/db/persist.cljs index fa69099a3e..97f3056a26 100644 --- a/src/main/frontend/db/persist.cljs +++ b/src/main/frontend/db/persist.cljs @@ -2,7 +2,6 @@ "Handles operations to persisting db to disk or indexedDB" (:require [frontend.util :as util] [frontend.idb :as idb] - [frontend.config :as config] [electron.ipc :as ipc] [frontend.db.conn :as db-conn] [promesa.core :as p] @@ -16,24 +15,12 @@ electron-disk-graphs (when (util/electron?) (ipc/ipc "getGraphs"))] (distinct (concat repos db-repos (some-> electron-disk-graphs bean/->clj))))) -(defn get-serialized-graph - [graph-name] - (if (util/electron?) - (p/let [result (ipc/ipc "getSerializedGraph" graph-name) - result (if result result - (let [graph-name (str config/idb-db-prefix graph-name)] - (idb/get-item graph-name)))] - result) - (idb/get-item graph-name))) - (defn delete-graph! [graph] - (let [key (db-conn/datascript-db graph) - db-based? (config/db-based-graph? graph)] + (let [key (db-conn/datascript-db graph)] (p/let [_ (persist-db/db stored) - (catch :default _e - (js/console.warn "Invalid graph cache") - (d/empty-db (db-conn/get-schema repo)))) - db (if (old-schema? stored-db) - (db-migrate/migrate stored-db) - stored-db)] - (db-conn/reset-conn! db-conn db)))] - (d/transact! db-conn [{:schema/version db-schema/version}]))) + :else + true)))) (defn- update-built-in-properties! [conn] @@ -89,8 +68,8 @@ (when (seq txs) (d/transact! conn txs)))) -(defn- restore-graph-from-sqlite! - "Load initial data from SQLite" +(defn restore-graph! + "Restore db from SQLite" [repo] (state/set-state! :graph/loading? true) (p/let [start-time (t/now) @@ -98,13 +77,15 @@ _ (assert (some? data) "No data found when reloading db") datoms (dt/read-transit-str data) datoms-count (count datoms) - conn (sqlite-common-db/restore-initial-data datoms) + db-schema (db-conn/get-schema repo) + conn (sqlite-common-db/restore-initial-data datoms db-schema) db-name (db-conn/datascript-db repo) _ (swap! db-conn/conns assoc db-name conn) - end-time (t/now)] + end-time (t/now) + db-based? (config/db-based-graph? repo)] ;; FIXME: why not do this when creating the db? - (update-built-in-properties! conn) + (when db-based? (update-built-in-properties! conn)) (println :restore-graph-from-sqlite!-prepare (t/in-millis (t/interval start-time end-time)) "ms" " Datoms in total: " datoms-count) @@ -118,12 +99,3 @@ (state/set-state! :graph/loading? false) (react/clear-query-state!) (state/pub-event! [:ui/re-render-root])))) - -(defn restore-graph! - "Restore db from serialized db cache" - [repo] - (if (config/db-based-graph? repo) - (restore-graph-from-sqlite! repo) - (p/let [db-name (db-conn/datascript-db repo) - stored (db-persist/get-serialized-graph db-name)] - (restore-graph-from-text! repo stored)))) diff --git a/src/main/frontend/db_worker.cljs b/src/main/frontend/db_worker.cljs index ed6891f8b1..eb2ef3962a 100644 --- a/src/main/frontend/db_worker.cljs +++ b/src/main/frontend/db_worker.cljs @@ -12,7 +12,8 @@ ["comlink" :as Comlink] [clojure.string :as string] [cljs-bean.core :as bean] - [frontend.worker.search :as search])) + [frontend.worker.search :as search] + [logseq.db.sqlite.util :as sqlite-util])) (defonce *sqlite (atom nil)) ;; repo -> {:db conn :search conn} @@ -142,7 +143,9 @@ (.exec db "PRAGMA locking_mode=exclusive") (sqlite-common-db/create-kvs-table! db) (search/create-tables-and-triggers! search-db) - (let [conn (sqlite-common-db/get-storage-conn storage)] + (prn :debug :repo repo) + (let [schema (sqlite-util/get-schema repo) + conn (sqlite-common-db/get-storage-conn storage schema)] (swap! *datascript-conns assoc repo conn) nil)))) diff --git a/src/main/frontend/persist_db.cljs b/src/main/frontend/persist_db.cljs index e504b41020..842cb4b78f 100644 --- a/src/main/frontend/persist_db.cljs +++ b/src/main/frontend/persist_db.cljs @@ -39,10 +39,11 @@ (js/console.log "fetch-initial-data" ret) ret))) -;; FIXME: limit repo name's length +;; FIXME: limit repo name's length and sanity +;; original size is 56 ;; @shuyu Do we still need this? (defn