download remote graph and transact into sqlite

src/main/frontend/db/rtc/core.cljs

@@ -34,6 +34,7 @@
 
 (def state-schema
   "
+  | :user-uuid             | string                                                       |
   | :data-from-ws-chan     | channel for receive messages from server websocket           |
   | :data-from-ws-pub      | pub of :data-from-ws-chan, dispatch by :req-id               |
   | :client-op-update-chan | channel to notify that there're some new operations          |
@@ -42,6 +43,7 @@
   | :ws                    | websocket                                                    |
 "
   [:map
+   [:user-uuid :string]
    [:data-from-ws-chan :any]
    [:data-from-ws-pub :any]
    [:client-op-update-chan :any]
@@ -177,23 +179,30 @@
         nil))))
 
 (defn init-state
-  [ws data-from-ws-chan]
+  [ws data-from-ws-chan user-uuid]
   (m/parse state-schema
-           {:data-from-ws-chan data-from-ws-chan
+           {:user-uuid user-uuid
+            :data-from-ws-chan data-from-ws-chan
             :data-from-ws-pub (async/pub data-from-ws-chan :req-id)
             :client-op-update-chan (chan)
             :upload-graph-chan (chan)
             :download-graph-chan (chan)
             :ws ws}))
 
+(defn ensure-ws-connected
+  [state]
+
+  )
+
 (defn <init
   []
   (go
     (let [data-from-ws-chan (chan (async/sliding-buffer 100))
           ws-opened-ch (chan)
-          ws (ws-listen! "f92bb5b3-0f72-4a74-9ad8-1793e655c309" data-from-ws-chan ws-opened-ch)]
+          user-uuid "f92bb5b3-0f72-4a74-9ad8-1793e655c309"
+          ws (ws-listen! user-uuid data-from-ws-chan ws-opened-ch)]
       (<! ws-opened-ch)
-      (init-state ws data-from-ws-chan))))
+      (init-state ws data-from-ws-chan user-uuid))))
 
 (comment
   (go

src/main/frontend/db/rtc/full_upload_download_graph.cljs

@@ -5,8 +5,12 @@
             [frontend.db.rtc.ws :refer [send]]
             [frontend.state :as state]
             [cljs.core.async :as async :refer [chan go <!]]
+            [cljs.core.async.interop :refer [p->c]]
             [cljs-http.client :as http]
-            [cognitect.transit :as transit]))
+            [cognitect.transit :as transit]
+            [logseq.db.schema :as db-schema]
+            [logseq.db.sqlite.util :as sqlite-util]
+            [electron.ipc :as ipc]))
 
 
 (defn- export-as-blocks
@@ -40,7 +44,70 @@
 (defn- <download-graph
   [state graph-uuid]
   (go
-    (let [r (with-sub-data-from-ws state
-              (send (:ws state) {:req-id (get-req-id) :action "full-download-graph" :graph-uuid graph-uuid})
-              (<! (get-result-ch)))]
-      (prn r))))
+    (let [{:keys [url]}
+          (with-sub-data-from-ws state
+            (send (:ws state) {:req-id (get-req-id) :action "full-download-graph" :graph-uuid graph-uuid})
+            (<! (get-result-ch)))
+          {:keys [status body] :as r} (<! (http/get url))]
+      (if (not= 200 status)
+        (ex-info "<download-graph failed" r)
+        (let [reader (transit/reader :json)
+              all-blocks (transit/read reader body)]
+          all-blocks)))))
+
+
+(defn- replace-db-id-with-temp-id
+  [blocks]
+  (mapv
+   (fn [block]
+     (let [db-id (:db/id block)
+           block-parent (:db/id (:block/parent block))
+           block-left (:db/id (:block/left block))]
+       (cond-> (assoc block :db/id (str db-id))
+         block-parent (assoc :block/parent (str block-parent))
+         block-left (assoc :block/left (str block-left)))))
+   blocks))
+
+(def page-of-block
+  (memoize
+   (fn [id->block-map block]
+     (when-let [parent-id (:block/parent block)]
+       (when-let [parent (id->block-map parent-id)]
+         (if (:block/name parent)
+           parent
+           (page-of-block id->block-map parent)))))))
+
+(defn- fill-block-fields
+  [blocks]
+  (let [groups (group-by #(boolean (:block/name %)) blocks)
+        ;; _page-blocks (get groups true)
+        other-blocks (set (get groups false))
+        id->block (into {} (map (juxt :db/id identity) blocks))
+        block-id->page-id (into {} (map (fn [b] [(:db/id b) (:db/id (page-of-block id->block b))]) other-blocks))]
+    (mapv (fn [b]
+            (let [b (assoc b :block/format :markdown)]
+              (if-let [page-id (block-id->page-id (:db/id b))]
+                (assoc b :block/page page-id)
+                b)))
+          blocks)))
+
+
+(defn- <transact-remote-all-blocks-to-sqlite
+  [all-blocks graph-uuid]
+  (go
+    (let [{:keys [t blocks]} all-blocks
+          conn (d/create-conn db-schema/schema-for-db-based-graph)
+          blocks* (replace-db-id-with-temp-id blocks)
+          blocks-with-page-id (fill-block-fields blocks*)]
+      (d/transact! conn blocks-with-page-id)
+      (let [db (d/db conn)
+            blocks*
+            (d/pull-many db '[*] (keep (fn [b] (when-let [uuid (:block/uuid b)] [:block/uuid uuid])) blocks))
+            blocks**
+            (mapv (fn [b]
+                    (cond-> (assoc b :datoms (sqlite-util/block-map->datoms-str blocks* b))
+                      (:block/parent b) (assoc :page_uuid (str (:block/uuid (d/entity db (:db/id (:block/page b))))))))
+                  blocks*)
+            repo (str "rtc-" graph-uuid)]
+        (<! (p->c (ipc/ipc :db-new repo)))
+        (<! (p->c (ipc/ipc :db-transact-data repo (pr-str {:blocks blocks**}))))))))