diff --git a/src/electron/electron/db.cljs b/src/electron/electron/db.cljs index 709dd44054..a916e791ff 100644 --- a/src/electron/electron/db.cljs +++ b/src/electron/electron/db.cljs @@ -99,7 +99,6 @@ (defn upsert-blocks! [repo blocks] - (def xx blocks) (if-let [db (get-db repo)] (let [insert (prepare db "INSERT INTO blocks (uuid, type, page_uuid, page_journal_day, name, content,datoms, created_at, updated_at) VALUES (@uuid, @type, @page_uuid, @page_journal_day, @name, @content, @datoms, @created_at, @updated_at) ON CONFLICT (uuid) DO UPDATE SET (type, page_uuid, page_journal_day, name, content, datoms, created_at, updated_at) = (@type, @page_uuid, @page_journal_day, @name, @content, @datoms, @created_at, @updated_at)" repo) @@ -123,7 +122,7 @@ ;; Initial data: ;; All pages and block ids ;; latest 3 journals -;; core data such as config, custom css/js +;; other data such as config.edn, custom css/js ;; current page, sidebar blocks (defn- query diff --git a/src/electron/electron/handler.cljs b/src/electron/electron/handler.cljs index 362b54bd33..968697178a 100644 --- a/src/electron/electron/handler.cljs +++ b/src/electron/electron/handler.cljs @@ -371,6 +371,7 @@ :created_at (or (:block/created-at b) (utils/time-ms)) :updated_at (or (:block/updated-at b) (utils/time-ms))}) blocks)] + (db/upsert-blocks! repo (bean/->js blocks')))))) (defmethod handle :get-initial-data [window [_ repo _opts]] diff --git a/src/main/frontend/db.cljs b/src/main/frontend/db.cljs index fc2d42bd85..e96b861b0a 100644 --- a/src/main/frontend/db.cljs +++ b/src/main/frontend/db.cljs @@ -73,24 +73,6 @@ [logseq.db.default built-in-pages-names built-in-pages]) -(defn- old-schema? - "Requires migration if the schema version is older than db-schema/version" - [db] - (let [v (db-migrate/get-schema-version db) - ;; backward compatibility - v (if (integer? v) v 0)] - (cond - (= db-schema/version v) - false - - (< db-schema/version v) - (do - (js/console.error "DB schema version is newer than the app, please update the app. " ":db-version" v) - false) - - :else - true))) - ;; persisting DBs between page reloads (defn persist! [repo] (let [key (datascript-db repo) @@ -159,149 +141,6 @@ (assoc option :listen-handler listen-and-persist!)))) -(defn restore-graph-from-text! - "Swap db string into the current db status - stored: the text to restore from" - [repo stored] - (p/let [db-name (datascript-db repo) - db-conn (d/create-conn db-schema/schema) - _ (swap! conns assoc db-name db-conn) - _ (when stored - (let [stored-db (try (string->db stored) - (catch :default _e - (js/console.warn "Invalid graph cache") - (d/empty-db db-schema/schema))) - attached-db (d/db-with stored-db - default-db/built-in-pages) ;; TODO bug overriding uuids? - db (if (old-schema? attached-db) - (db-migrate/migrate attached-db) - attached-db)] - (conn/reset-conn! db-conn db)))] - (d/transact! db-conn [{:schema/version db-schema/version}]))) - -(defn- uuid-str->uuid-in-av-vec - [[a v]] - (cond - (and (= :block/uuid a) (string? v)) - [a (uuid v)] - - (and (coll? v) (= 2 (count v)) - (= :block/uuid (first v)) - (string? (second v))) - [a [:block/uuid (uuid (second v))]] - - :else - [a v])) - -(defn- add-tempid-to-av-colls - [start-tempid av-colls] - (map-indexed (fn [idx av-coll] - (map (partial cons (dec (- start-tempid idx))) av-coll)) - av-colls)) - -(defn restore-other-data-from-sqlite! - [repo data] - (let [per-length 2000] - (p/loop [data data] - (cond - (not= repo (state/get-current-repo)) ; switched to another graph - nil - - (empty? data) - nil - - (not (state/input-idle? repo)) ; wait until input is idle - (js/setTimeout #(restore-other-data-from-sqlite! repo data) 5000) - - :else - (let [part (->> (take per-length data) - (map (fn [block] - (map uuid-str->uuid-in-av-vec - (edn/read-string (gobj/get block "datoms"))))) - (map-indexed (fn [idx av-coll] - (->> av-coll - (map (partial cons (dec (- idx)))) - (sort-by #(if (= :block/uuid (second %)) 0 1))))) - (apply concat) - (map (fn [eav] (cons :db/add eav))))] - (transact! repo part {:skip-persist? true}) - (p/let [_ (p/delay 200)] - (p/recur (drop per-length data))) - ))))) - -(defn restore-graph-from-sqlite! - "Load initial data from SQLite" - [repo] - (p/let [db-name (datascript-db repo) - db-conn (d/create-conn db-schema/schema) - _ (swap! conns assoc db-name db-conn) - data (ipc/ipc :get-initial-data repo) - {:keys [all-pages all-blocks journal-blocks]} (bean/->clj data) - pages (map (fn [page] - (->> page - :datoms - edn/read-string - (map uuid-str->uuid-in-av-vec))) - all-pages) - all-blocks' (map (fn [b] - [[:block/uuid (uuid (:uuid b))] - [:block/page [:block/uuid (uuid (:page_uuid b))]]]) - all-blocks) - journal-blocks' (map (fn [b] - (->> b - :datoms - edn/read-string - (map uuid-str->uuid-in-av-vec))) - journal-blocks) - pages-eav-colls (add-tempid-to-av-colls 0 pages) - pages-eav-coll (->> pages-eav-colls - (apply concat) - (sort-by (fn [eav] (if (= :block/uuid (second eav)) 0 1)))) - blocks-eav-colls (->> (concat all-blocks' journal-blocks') - (add-tempid-to-av-colls (- (count pages-eav-colls))) - (apply concat)) - tx-data (map (partial cons :db/add) (concat pages-eav-coll blocks-eav-colls))] - (def xx [all-pages all-blocks journal-blocks tx-data]) - (d/transact! db-conn tx-data) - - ;; TODO: Store schema in sqlite - ;; (db-migrate/migrate attached-db) - - (d/transact! db-conn [(react/kv :db/type "db") - {:schema/version db-schema/version}] - {:skip-persist? true}) - (println :restore-graph-from-sqlite! :done) - - (js/setTimeout - (fn [] - (p/let [other-data (ipc/ipc :get-other-data repo (map :uuid journal-blocks))] - (restore-other-data-from-sqlite! repo other-data))) - 1000) - )) - -(defn restore-graph! - "Restore db from serialized db cache" - [repo] - (if (string/starts-with? repo config/db-version-prefix) - (restore-graph-from-sqlite! repo) - (p/let [db-name (datascript-db repo) - stored (db-persist/get-serialized-graph db-name)] - (restore-graph-from-text! repo stored)))) - -(defn restore! - [repo] - (p/let [_ (restore-graph! repo)] - (listen-and-persist! repo))) - -(defn run-batch-txs! - [] - (let [chan (state/get-db-batch-txs-chan)] - (async/go-loop [] - (let [f (async/db stored) + (catch :default _e + (js/console.warn "Invalid graph cache") + (d/empty-db db-schema/schema))) + attached-db (d/db-with stored-db + default-db/built-in-pages) ;; TODO bug overriding uuids? + db (if (old-schema? attached-db) + (db-migrate/migrate attached-db) + attached-db)] + (db-conn/reset-conn! db-conn db)))] + (d/transact! db-conn [{:schema/version db-schema/version}]))) + + +(defn- uuid-str->uuid-in-av-vec + [[a v]] + (cond + (and (= :block/uuid a) (string? v)) + [a (uuid v)] + + (and (coll? v) (= 2 (count v)) + (= :block/uuid (first v)) + (string? (second v))) + [a [:block/uuid (uuid (second v))]] + + :else + [a v])) + +(defn- add-tempid-to-av-colls + [start-tempid av-colls] + (map-indexed (fn [idx av-coll] + (map (partial cons (dec (- start-tempid idx))) av-coll)) + av-colls)) + +(defn- restore-other-data-from-sqlite! + [repo data] + (let [per-length 2000 + conn (db-conn/get-db repo false)] + (p/loop [data data] + (cond + (not= repo (state/get-current-repo)) ; switched to another graph + nil + + (empty? data) + nil + + (not (state/input-idle? repo)) ; wait until input is idle + (p/do! (p/delay 5000) + (p/recur (data))) + + :else + (let [part (->> (take per-length data) + (map (fn [block] + (map uuid-str->uuid-in-av-vec + (edn/read-string (gobj/get block "datoms"))))) + (map-indexed (fn [idx av-coll] + (->> av-coll + (map (partial cons (dec (- idx)))) + (sort-by #(if (= :block/uuid (second %)) 0 1))))) + (apply concat) + (map (fn [eav] (cons :db/add eav))))] + (d/transact! conn part {:skip-persist? true}) + (p/let [_ (p/delay 200)] + (p/recur (drop per-length data)))))))) + +(defn- restore-graph-from-sqlite! + "Load initial data from SQLite" + [repo] + (p/let [db-name (db-conn/datascript-db repo) + db-conn (d/create-conn db-schema/schema) + _ (swap! db-conn/conns assoc db-name db-conn) + data (ipc/ipc :get-initial-data repo) + {:keys [all-pages all-blocks journal-blocks]} (bean/->clj data) + pages (map (fn [page] + (->> page + :datoms + edn/read-string + (map uuid-str->uuid-in-av-vec))) + all-pages) + all-blocks' (map (fn [b] + [[:block/uuid (uuid (:uuid b))] + [:block/page [:block/uuid (uuid (:page_uuid b))]]]) + all-blocks) + journal-blocks' (map (fn [b] + (->> b + :datoms + edn/read-string + (map uuid-str->uuid-in-av-vec))) + journal-blocks) + pages-eav-colls (add-tempid-to-av-colls 0 pages) + pages-eav-coll (->> pages-eav-colls + (apply concat) + (sort-by (fn [eav] (if (= :block/uuid (second eav)) 0 1)))) + blocks-eav-colls (->> (concat all-blocks' journal-blocks') + (add-tempid-to-av-colls (- (count pages-eav-colls))) + (apply concat)) + tx-data (map (partial cons :db/add) (concat pages-eav-coll blocks-eav-colls))] + (d/transact! db-conn tx-data) + + ;; TODO: Store schema in sqlite + ;; (db-migrate/migrate attached-db) + + (d/transact! db-conn [(react/kv :db/type "db") + {:schema/version db-schema/version}] + {:skip-persist? true}) + (println :restore-graph-from-sqlite! :done) + + (js/setTimeout + (fn [] + (p/let [other-data (ipc/ipc :get-other-data repo (map :uuid journal-blocks))] + (restore-other-data-from-sqlite! repo other-data))) + 1000))) + +(defn restore-graph! + "Restore db from serialized db cache" + [repo] + (if (string/starts-with? repo config/db-version-prefix) + (restore-graph-from-sqlite! repo) + (p/let [db-name (db-conn/datascript-db repo) + stored (db-persist/get-serialized-graph db-name)] + (restore-graph-from-text! repo stored)))) diff --git a/src/main/frontend/handler.cljs b/src/main/frontend/handler.cljs index a821704695..a70c0d9d4d 100644 --- a/src/main/frontend/handler.cljs +++ b/src/main/frontend/handler.cljs @@ -13,6 +13,7 @@ [frontend.config :as config] [frontend.context.i18n :as i18n :refer [t]] [frontend.db :as db] + [frontend.db.restore :as db-restore] [frontend.db.conn :as conn] [frontend.db.persist :as db-persist] [frontend.db.react :as react] @@ -82,9 +83,10 @@ (defn restore-and-setup! [repos] (when-let [repo (or (state/get-current-repo) (:url (first repos)))] - (-> (db/restore! repo) + (-> (db-restore/restore-graph! repo) (p/then (fn [] + (db/listen-and-persist! repo) ;; try to load custom css only for current repo (ui-handler/add-style-if-exists!) @@ -234,7 +236,6 @@ (p/finally (fn [] (state/set-db-restoring! false)))) - (db/run-batch-txs!) (file/