add debug-ui for rtc

src/main/frontend/components/right_sidebar.cljs

@@ -21,7 +21,8 @@
             [medley.core :as medley]
             [reitit.frontend.easy :as rfe]
             [rum.core :as rum]
-            [frontend.handler.common :as common-handler]))
+            [frontend.handler.common :as common-handler]
+            [frontend.db.rtc.debug-ui :as rtc-debug-ui]))
 
 (rum/defc toggle
   []
@@ -133,8 +134,8 @@
     #_:clj-kondo/ignore
     (let [lookup (if (integer? db-id) db-id [:block/uuid db-id])]
       (when-let [block (db/entity repo lookup)]
-       [(t :right-side-bar/block-ref)
-        (block-with-breadcrumb repo block idx [repo db-id block-type] true)]))
+        [(t :right-side-bar/block-ref)
+         (block-with-breadcrumb repo block idx [repo db-id block-type] true)]))
 
     :block
     #_:clj-kondo/ignore
@@ -164,6 +165,10 @@
     [[:.flex.items-center (ui/icon "command" {:class "text-md mr-2"}) (t :help/shortcuts)]
      (shortcut-settings)]
 
+    :rtc
+    [[:.flex.items-center (ui/icon "cloud" {:class "text-md mr-2"}) "(Dev) RTC"]
+     (rtc-debug-ui/rtc-debug-ui)]
+
     ["" [:span]]))
 
 (defonce *drag-to
@@ -417,7 +422,14 @@
           [:div.text-sm
            [:button.button.cp__right-sidebar-settings-btn {:on-click (fn [_e]
                                                                        (state/sidebar-add-block! repo "history" :history))}
-            (t :right-side-bar/history)]])]]
+            (t :right-side-bar/history)]])
+
+        (when (and config/dev? (state/sub [:ui/developer-mode?]))
+          [:div.text-sm
+           [:button.button.cp__right-sidebar-settings-btn {:on-click (fn [_e]
+                                                                       (state/sidebar-add-block! repo "rtc" :rtc))}
+            "(Dev) RTC"]])
+        ]]
 
       [:.sidebar-item-list.flex-1.scrollbar-spacing.ml-2.pr-3
        (if @*anim-finished?

src/main/frontend/db/rtc/core.cljs

@@ -8,7 +8,6 @@
             [cljs.core.async :as async :refer [<! >! chan go go-loop offer!
                                                poll! timeout]]
             [cljs.core.async.interop :refer [p->c]]
-            [electron.ipc :as ipc]
             [malli.core :as m]
             [frontend.modules.outliner.transaction :as outliner-tx]
             [frontend.modules.outliner.core :as outliner-core]
@@ -20,6 +19,10 @@
             [frontend.db.rtc.full-upload-download-graph :as full-upload-download-graph]
             [frontend.handler.page :as page-handler]))
 
+;;; TODO:
+;;; 1. two clients, each one has a page with same name but different block-uuid.
+;;;    happens when generating journal page everyday automatically.
+
 
 
 
@@ -33,6 +36,7 @@
   | :client-op-update-chan | channel to notify that there're some new operations |
   | :*stop-rtc-loop-chan   | atom of chan to stop <loop-for-rtc                  |
   | :*ws                   | atom of websocket                                   |
+  | :*rtc-state            | atom of state of current rtc progress               |
 "
   [:map
    [:user-uuid :string]
@@ -42,9 +46,14 @@
    [:data-from-ws-pub :any]
    [:client-op-update-chan :any]
    [:*stop-rtc-loop-chan :any]
-   [:*ws :any]])
+   [:*ws :any]
+   [:*rtc-state :any]])
 (def state-validator (m/validator state-schema))
 
+(def rtc-state-schema
+  [:enum :open :closed])
+(def rtc-state-validator (m/validator rtc-state-schema))
+
 (def data-from-ws-schema
   [:map
    [:req-id :string]
@@ -342,9 +351,10 @@
                                  (when-let [b (db/entity repo [:block/uuid (uuid block-uuid)])]
                                    (let [left-uuid (some-> b :block/left :block/uuid str)
                                          parent-uuid (some-> b :block/parent :block/uuid str)]
-                                     ["update" {:block-uuid block-uuid
-                                                :target-uuid left-uuid :sibling? (not= left-uuid parent-uuid)
-                                                :content (:block/content b)}])))))
+                                     (when (and left-uuid parent-uuid)
+                                       ["update" {:block-uuid block-uuid
+                                                  :target-uuid left-uuid :sibling? (not= left-uuid parent-uuid)
+                                                  :content (:block/content b "")}]))))))
         update-page-ops* (->> update-page-uuids
                               (keep (fn [block-uuid]
                                       (when-let [page-name (:block/name (db/entity repo [:block/uuid (uuid block-uuid)]))]
@@ -388,7 +398,7 @@
     (let [{:keys [data-from-ws-pub client-op-update-chan]} state
           push-data-from-ws-ch (chan (async/sliding-buffer 100))
           stop-rtc-loop-chan (chan)]
-      (reset! (:*stop-rtc-loop-chan state) (chan))
+      (reset! (:*stop-rtc-loop-chan state) stop-rtc-loop-chan)
       (with-sub-data-from-ws state
         (<! (ws/<send! state {:action "register-graph-updates" :req-id (get-req-id) :graph-uuid graph-uuid}))
         (<! (get-result-ch)))
@@ -407,7 +417,9 @@
                 client-op-update
                 (do (<! (<client-op-update-handler state))
                     (recur))
-                stop (prn :stop-loop-for-rtc graph-uuid)
+                stop
+                (do (ws/stop @(:*ws state))
+                    (reset! (:*rtc-state state) :closed))
                 :else
                 nil))))
       (async/unsub data-from-ws-pub "push-updates" push-data-from-ws-ch))))
@@ -415,7 +427,8 @@
 (defn init-state
   [ws data-from-ws-chan user-uuid]
   (m/parse state-schema
-           {:user-uuid user-uuid
+           {:*rtc-state (atom :open :validator rtc-state-validator)
+            :user-uuid user-uuid
             :*graph-uuid (atom nil)
             :*repo (atom nil)
             :data-from-ws-chan data-from-ws-chan
@@ -455,11 +468,11 @@
     (let [state (<! (<init))]
       (<! (full-upload-download-graph/<download-graph state repo graph-uuid)))))
 
-(defn ^:export upload-graph
-  []
-  (go
-    (let [state (<! (<init))]
-      (<! (full-upload-download-graph/<upload-graph state)))))
+;; (defn ^:export upload-graph
+;;   []
+;;   (go
+;;     (let [state (<! (<init))]
+;;       (<! (full-upload-download-graph/<upload-graph state)))))
 
 (defn ^:export debug-client-push-updates
   []

src/main/frontend/db/rtc/debug_ui.cljs

@@ -0,0 +1,105 @@
+(ns frontend.db.rtc.debug-ui
+  "Debug UI for rtc module"
+  (:require [frontend.ui :as ui]
+            [rum.core :as rum]
+            [frontend.db.rtc.core :as rtc-core]
+            [cljs.core.async :as async :refer [go <! chan go-loop]]
+            [cljs.core.async.interop :refer [p->c]]
+            [frontend.db.rtc.op :as op]
+            [frontend.state :as state]
+            [frontend.db.rtc.ws :as ws]
+            [fipp.edn :as fipp]
+            [frontend.db.rtc.full-upload-download-graph :as full-upload-download-graph]
+            [frontend.util :as util]
+            [frontend.handler.notification :as notification]))
+
+(def ^:private debug-state (atom nil))
+(def debug-graph-uuid "6478874f-20a7-4335-9379-4cfb1cfa1b25")
+
+(defn- <start
+  []
+  (go
+    (let [repo (state/get-current-repo)
+          state (<! (rtc-core/<init))]
+      (if-let [graph-uuid (<! (p->c (op/<get-graph-uuid repo)))]
+        (do (reset! debug-state state)
+            (<! (rtc-core/<loop-for-rtc state graph-uuid (state/get-current-repo)))
+            state)
+        (do (notification/show! "not a rtc-graph" :error false)
+            nil)))))
+
+(defn- <stop
+  []
+  (async/close! @(:*stop-rtc-loop-chan @debug-state)))
+
+(defn- push-pending-ops
+  []
+  (async/put! (:client-op-update-chan @debug-state) true))
+
+(defn- <download-graph
+  [repo graph-uuid]
+  (go
+    (let [state (<! (rtc-core/<init))]
+      (<! (full-upload-download-graph/<download-graph state repo graph-uuid)))))
+
+(defn- <upload-graph
+  []
+  (go
+    (let [state (<! (rtc-core/<init))
+          repo (state/get-current-repo)]
+      (<! (full-upload-download-graph/<upload-graph state repo)))))
+
+(rum/defcs rtc-debug-ui <
+  rum/reactive
+  (rum/local nil ::graph-uuid)
+  (rum/local nil ::local-tx)
+  (rum/local nil ::ops)
+  (rum/local nil ::ws-state)
+  (rum/local nil ::download-graph-to-repo)
+  [state]
+  (let [s (rum/react debug-state)
+        rtc-state (and s (rum/react (:*rtc-state s)))]
+    [:div
+     [:a
+      {:on-mouse-down (fn [_] (go
+                                (let [repo (state/get-current-repo)
+                                      {:keys [local-tx ops]}
+                                      (<! (p->c (op/<get-ops&local-tx repo)))
+                                      graph-uuid (<! (p->c (op/<get-graph-uuid repo)))]
+                                  (reset! (::local-tx state) local-tx)
+                                  (reset! (::ops state) (count ops))
+                                  (reset! (::graph-uuid state) graph-uuid)
+                                  (reset! (::ws-state state) (and s (ws/get-state @(:*ws s)))))))}
+      (ui/icon "refresh" {:style {:font-size 20}})]
+
+     [:pre (-> {:graph @(::graph-uuid state)
+                :rtc-state rtc-state
+                :ws (and s (ws/get-state @(:*ws s)))
+                :local-tx @(::local-tx state)
+                :pending-ops @(::ops state)}
+               (fipp/pprint {:width 20})
+               with-out-str)]
+     (if (or (nil? s)
+             (= :closed rtc-state))
+       (ui/button "start" {:class "my-2"
+                           :on-click (fn [] (<start))})
+
+       [:div.my-2
+        [:div.my-2 (ui/button (str "send pending ops")
+                         {:on-click (fn [] (push-pending-ops))})]
+        [:div (ui/button "stop" {:on-click (fn [] (<stop))})]])
+     [:hr]
+     [:div.flex
+      (ui/button (str "download graph to")
+                 {:class "mr-2"
+                  :on-click (fn []
+                              (go
+                                (when-let [repo @(::download-graph-to-repo state)]
+                                  (<! (<download-graph repo debug-graph-uuid))
+                                  (notification/show! "download graph successfully"))))})
+      (ui/ls-textarea {:on-change (fn [e] (reset! (::download-graph-to-repo state) (util/evalue e)))})]
+     [:div.flex.my-2
+      (ui/button (str "upload graph") {:on-click (fn []
+                                                   (go
+                                                     (<! (<upload-graph))
+                                                     (notification/show! "upload graph successfully")))})]]))

src/main/frontend/db/rtc/full_upload_download_graph.cljs

@@ -12,7 +12,8 @@
             [cognitect.transit :as transit]
             [logseq.db.schema :as db-schema]
             [logseq.db.sqlite.util :as sqlite-util]
-            [frontend.persist-db :as persist-db]))
+            [frontend.persist-db :as persist-db]
+            [frontend.db.rtc.op :as op]))
 
 
 (defn- export-as-blocks
@@ -30,12 +31,12 @@
 
 (defn <upload-graph
   "Upload current repo to remote, return remote {:req-id xxx :graph-uuid <new-remote-graph-uuid>}"
-  [state]
+  [state repo]
   (go
     (let [{:keys [url key all-blocks-str]}
           (with-sub-data-from-ws state
             (<! (<send! state {:req-id (get-req-id) :action "presign-put-temp-s3-obj" :graph-uuid "not-yet"}))
-            (let [all-blocks (export-as-blocks (state/get-current-repo))
+            (let [all-blocks (export-as-blocks repo)
                   all-blocks-str (transit/write (transit/writer :json) all-blocks)]
               (merge (<! (get-result-ch)) {:all-blocks-str all-blocks-str})))]
       (<! (http/put url {:body all-blocks-str}))
@@ -44,7 +45,8 @@
         (let [r (<! (get-result-ch))]
           (if-not (:graph-uuid r)
             (ex-info "upload graph failed" r)
-            r))))))
+            (do (<! (p->c (op/<update-graph-uuid! repo (:graph-uuid r))))
+                r)))))))
 
 
 (defn- replace-db-id-with-temp-id
@@ -98,10 +100,10 @@
             (mapv (fn [b]
                     (cond-> (assoc b :datoms (sqlite-util/block-map->datoms-str blocks* b))
                       (:block/parent b) (assoc :page_uuid (str (:block/uuid (d/entity db (:db/id (:block/page b))))))))
-                  blocks*)
-            repo (str "logseq_db_rtc-" repo)]
+                  blocks*)]
         (<! (p->c (persist-db/<new repo)))
-        (<! (persist-db/<transact-data repo blocks** nil))))))
+        (<! (persist-db/<transact-data repo blocks** nil))
+        (<! (p->c (op/<update-local-tx! repo t)))))))
 
 
 (defn <download-graph
@@ -111,9 +113,11 @@
           (with-sub-data-from-ws state
             (<send! state {:req-id (get-req-id) :action "full-download-graph" :graph-uuid graph-uuid})
             (<! (get-result-ch)))
-          {:keys [status body] :as r} (<! (http/get url))]
+          {:keys [status body] :as r} (<! (http/get url))
+          repo (str "logseq_db_rtc-" repo)]
       (if (not= 200 status)
         (ex-info "<download-graph failed" r)
         (let [reader (transit/reader :json)
               all-blocks (transit/read reader body)]
-          (<! (<transact-remote-all-blocks-to-sqlite all-blocks repo)))))))
+          (<! (<transact-remote-all-blocks-to-sqlite all-blocks repo))
+          (<! (p->c (op/<update-graph-uuid! repo graph-uuid))))))))

src/main/frontend/db/rtc/macro.clj

@@ -2,7 +2,8 @@
 
 
 (defmacro with-sub-data-from-ws
-  "- sub :data-from-ws-pub
+  "TODO: handle exceptions for ws
+  - sub :data-from-ws-pub
   - run body, use `get-req-id` to get req-id, and `get-result-ch` to get result-ch
   - unsub :data-from-ws-pub"
   [state & body]

src/main/frontend/db/rtc/op.cljs

@@ -52,3 +52,7 @@
   [repo graph-uuid]
   {:pre [(some? graph-uuid)]}
   (op-store/<update-graph-uuid! repo graph-uuid))
+
+(defn <get-graph-uuid
+  [repo]
+  (op-store/<get-graph-uuid repo))

src/main/frontend/db/rtc/ops_idb_store.cljs

@@ -61,3 +61,8 @@
           keys (idb-keyval/keys store)]
     (-> (p/all (mapv (fn [k] (p/chain (idb-keyval/get k store) (partial vector k))) keys))
         (p/then (fn [items] (mapv #(js->clj % :keywordize-keys true) items))))))
+
+(defn <get-graph-uuid
+  [repo]
+  (p/let [store (ensure-store repo)]
+    (idb-keyval/get "graph-uuid" store)))

src/main/frontend/db/rtc/ws.cljs

@@ -55,3 +55,11 @@
   (set! (.-onmessage ws) nil)
   (set! (.-onerror ws) nil)
   (.close ws))
+
+(defn get-state
+  [ws]
+  (case (.-readyState ws)
+    0 :connecting
+    1 :open
+    2 :closing
+    3 :closed))