diff --git a/deps/db-sync/src/logseq/db_sync/index.cljs b/deps/db-sync/src/logseq/db_sync/index.cljs index 5bf38f19f0..aeb45cb845 100644 --- a/deps/db-sync/src/logseq/db_sync/index.cljs +++ b/deps/db-sync/src/logseq/db_sync/index.cljs @@ -20,6 +20,14 @@ "email_verified INTEGER," "username TEXT" ");")) + (common/clj result :keywordize-keys true) + body (coerce-http-request :e2ee/user-keys body) + user-id (aget claims "sub")] + (cond + (not (string? user-id)) + (unauthorized) + + (nil? body) + (bad-request "invalid body") + + :else + (let [{:keys [public-key encrypted-private-key]} body] + (p/let [_ (index/ {} + (some? public-key) + (assoc :public-key public-key))))) + + (and (= method "GET") + (= 4 (count parts)) + (= "e2ee" (first parts)) + (= "graphs" (nth parts 1)) + (= "aes-key" (nth parts 3))) + (let [graph-id (nth parts 2) + user-id (aget claims "sub")] + (cond + (not (string? user-id)) + (unauthorized) + + :else + (p/let [access? (index/ {} + (some? encrypted-aes-key) + (assoc :encrypted-aes-key encrypted-aes-key)))))))) + + (and (= method "POST") + (= 4 (count parts)) + (= "e2ee" (first parts)) + (= "graphs" (nth parts 1)) + (= "aes-key" (nth parts 3))) + (let [graph-id (nth parts 2) + user-id (aget claims "sub")] + (cond + (not (string? user-id)) + (unauthorized) + + :else + (.then (common/read-json request) + (fn [result] + (if (nil? result) + (bad-request "missing body") + (let [body (js->clj result :keywordize-keys true) + body (coerce-http-request :e2ee/graph-aes-key body)] + (if (nil? body) + (bad-request "invalid body") + (p/let [access? (index/clj result :keywordize-keys true) + body (coerce-http-request :e2ee/grant-access body)] + (if (nil? body) + (bad-request "invalid body") + (p/let [manager? (index/ {:ok true} + (seq @missing) + (assoc :missing-users @missing)))))))))))))) + (and (= method "DELETE") (= 2 (count parts)) (= "graphs" (first parts))) diff --git a/deps/db-sync/test/logseq/db_sync/index_test.cljs b/deps/db-sync/test/logseq/db_sync/index_test.cljs index 2541fb9c1e..3c3a6d5faf 100644 --- a/deps/db-sync/test/logseq/db_sync/index_test.cljs +++ b/deps/db-sync/test/logseq/db_sync/index_test.cljs @@ -25,6 +25,22 @@ (defn- run-sql! [state sql args] (record-exec! state sql) (cond + (string/includes? sql "insert into user_rsa_keys") + (let [[user-id public-key encrypted-private-key created-at updated-at] args] + (swap! state update :user-keys assoc user-id {:user-id user-id + :public-key public-key + :encrypted-private-key encrypted-private-key + :created-at created-at + :updated-at updated-at})) + + (string/includes? sql "insert into graph_aes_keys") + (let [[graph-id user-id encrypted-aes-key created-at updated-at] args] + (swap! state update :graph-keys assoc [graph-id user-id] {:graph-id graph-id + :user-id user-id + :encrypted-aes-key encrypted-aes-key + :created-at created-at + :updated-at updated-at})) + (string/includes? sql "insert into users") (let [[user-id email email-verified username] args] (swap! state update :users assoc user-id {:id user-id @@ -85,6 +101,30 @@ (defn- all-sql [state sql args] (record-exec! state sql) (cond + (string/includes? sql "from user_rsa_keys") + (if (string/includes? sql "left join users") + (let [[email] args + user-id (some (fn [[_ row]] + (when (= email (:email row)) + (:id row))) + (:users @state)) + row (when user-id (get-in @state [:user-keys user-id]))] + (if row + (js-rows [row]) + (js-rows []))) + (let [[user-id] args + row (get-in @state [:user-keys user-id])] + (if row + (js-rows [row]) + (js-rows [])))) + + (string/includes? sql "from graph_aes_keys") + (let [[graph-id user-id] args + row (get-in @state [:graph-keys [graph-id user-id]])] + (if row + (js-rows [row]) + (js-rows []))) + (string/includes? sql "from graph_members where graph_id") (let [graph-id (first args) members (->> (:graph-members @state) @@ -154,6 +194,8 @@ (async done (let [state (atom {:executed [] :users {} + :user-keys {} + :graph-keys {} :graph-members {} :graphs {}}) db (make-d1 state)] @@ -161,7 +203,9 @@ (p/then (fn [_] (let [sqls (:executed @state)] (is (some #(string/includes? % "create table if not exists users") sqls)) - (is (some #(string/includes? % "create table if not exists graph_members") sqls))) + (is (some #(string/includes? % "create table if not exists graph_members") sqls)) + (is (some #(string/includes? % "create table if not exists user_rsa_keys") sqls)) + (is (some #(string/includes? % "create table if not exists graph_aes_keys") sqls))) (done))) (p/catch (fn [e] (is false (str e)) @@ -171,6 +215,8 @@ (async done (let [state (atom {:executed [] :users {} + :user-keys {} + :graph-keys {} :graph-members {} :graphs {}}) db (make-d1 state) @@ -193,6 +239,8 @@ (async done (let [state (atom {:executed [] :users {} + :user-keys {} + :graph-keys {} :graph-members {} :graphs {}}) db (make-d1 state)] @@ -215,6 +263,8 @@ (async done (let [state (atom {:executed [] :users {} + :user-keys {} + :graph-keys {} :graph-members {} :graphs {}}) db (make-d1 state)] @@ -240,6 +290,8 @@ (async done (let [state (atom {:executed [] :users {} + :user-keys {} + :graph-keys {} :graph-members {} :graphs {}}) db (make-d1 state)] @@ -252,3 +304,68 @@ (p/catch (fn [e] (is false (str e)) (done))))))) + +(deftest e2ee-user-rsa-key-pair-upsert-test + (async done + (let [state (atom {:executed [] + :users {} + :user-keys {} + :graph-keys {} + :graph-members {} + :graphs {}}) + db (make-d1 state)] + (-> (p/do! + (index/ (p/do! + (index/ (p/do! + (index/ (.text resp) + (.then (fn [text] + (is (= "ok" text)) + (is (some? @called)) + (done))) + (.catch (fn [e] + (is false (str e)) + (done))))))))) diff --git a/docs/agent-guide/db-sync/protocol.md b/docs/agent-guide/db-sync/protocol.md index 4e92d707a7..69b56fc7e9 100644 --- a/docs/agent-guide/db-sync/protocol.md +++ b/docs/agent-guide/db-sync/protocol.md @@ -62,6 +62,24 @@ - `DELETE /graphs/:graph-id` - Delete graph and reset data. Response: `{"graph-id":"...","deleted":true}` or `400` (missing graph id). +### E2EE (index DO) +- `GET /e2ee/user-keys` + - Fetch current user's RSA key pair. Response: `{"public-key":"","encrypted-private-key":""}` or `{}` when missing. +- `POST /e2ee/user-keys` + - Upsert current user's RSA key pair. Body: `{"public-key":"","encrypted-private-key":"","reset-private-key":false?}`. + - Response mirrors the stored keys: `{"public-key":"","encrypted-private-key":""}`. +- `GET /e2ee/user-public-key?email=` + - Fetch a user's RSA public key by email. Response: `{"public-key":""}` or `{}` when missing. +- `GET /e2ee/graphs/:graph-id/aes-key` + - Fetch current user's encrypted graph AES key. Response: `{"encrypted-aes-key":""}` or `{}` when missing. +- `POST /e2ee/graphs/:graph-id/aes-key` + - Upsert current user's encrypted graph AES key. Body: `{"encrypted-aes-key":""}`. + - Response: `{"encrypted-aes-key":""}`. +- `POST /e2ee/graphs/:graph-id/grant-access` + - Manager-only. Upsert encrypted graph AES keys for members. + - Body: `{"target-user-email+encrypted-aes-key-coll":[{"user/email":"","encrypted-aes-key":""}...]}`. + - Response: `{"ok":true,"missing-users":["", ...]?}`. + ### Sync (per-graph DO, via `/sync/:graph-id/...`) - `GET /sync/:graph-id/health` - Health check. Response: `{"ok":true}`. diff --git a/src/main/frontend/components/repo.cljs b/src/main/frontend/components/repo.cljs index 089df77786..9ecc90d7e1 100644 --- a/src/main/frontend/components/repo.cljs +++ b/src/main/frontend/components/repo.cljs @@ -469,17 +469,12 @@ db-name (util/trim-safe (.-value (rum/deref input-ref)))] (when (and cloud? refresh-token token user-uuid (not e2ee-rsa-key-ensured?)) - (p/do! - (state/ (p/let [rsa-key-pair (state/ (p/let [rsa-key-pair (state/uint8 bytes)))) +(defn- snapshot-rows-e2ee? + [rows] + (boolean + (some (fn [[_ content _]] + (try + (let [data (sqlite-util/read-transit-str content)] + (and (map? data) + (= :logseq.kv/graph-rtc-e2ee? (:db/ident data)))) + (catch :default _ + false))) + rows))) + (defn- frame-len [^js data offset] (let [view (js/DataView. (.-buffer data) offset 4)] (.getUint32 view 0 false))) @@ -267,10 +279,11 @@ total' (+ total (count rows)) total-rows' (into total-rows rows)] (when (seq total-rows') - (p/do! - (state/js body))} - {:response-schema :graph-members/create})] + {:response-schema :graph-members/create}) + repo (state/get-current-repo) + e2ee? (ldb/get-graph-rtc-e2ee? (db/get-db)) + _ (when (and repo e2ee?) + (state/latest-remote-tx (atom {})) +(defonce ^:private *repo->aes-key (atom {})) +(defonce ^:private e2ee-store (delay (idb-keyval/newStore "localforage" "keyvaluepairs" 2))) (defn- current-client [repo] @@ -33,20 +39,21 @@ (defn- sync-counts [repo] - (let [pending-local (when-let [conn (client-ops-conn repo)] - (count (d/datoms @conn :avet :db-sync/created-at))) - pending-asset (client-op/get-unpushed-asset-ops-count repo) - local-tx (client-op/get-local-tx repo) - remote-tx (get @*repo->latest-remote-tx repo) - pending-server (when (and (number? local-tx) (number? remote-tx)) - (max 0 (- remote-tx local-tx))) - graph-uuid (client-op/get-graph-uuid repo)] - {:pending-local pending-local - :pending-asset pending-asset - :pending-server pending-server - :local-tx local-tx - :remote-tx remote-tx - :graph-uuid graph-uuid})) + (when (worker-state/get-datascript-conn repo) + (let [pending-local (when-let [conn (client-ops-conn repo)] + (count (d/datoms @conn :avet :db-sync/created-at))) + pending-asset (client-op/get-unpushed-asset-ops-count repo) + local-tx (client-op/get-local-tx repo) + remote-tx (get @*repo->latest-remote-tx repo) + pending-server (when (and (number? local-tx) (number? remote-tx)) + (max 0 (- remote-tx local-tx))) + graph-uuid (client-op/get-graph-uuid repo)] + {:pending-local pending-local + :pending-asset pending-asset + :pending-server pending-server + :local-tx local-tx + :remote-tx remote-tx + :graph-uuid graph-uuid}))) (defn- normalize-online-users [users] @@ -309,6 +316,333 @@ :url url :body body})))))) +(def ^:private invalid-transit ::invalid-transit) + +(declare encrypt-snapshot-rows decrypt-snapshot-rows) + +(defn- try-read-transit [value] + (try + (ldb/read-transit-str value) + (catch :default _ + invalid-transit))) + +(defn- graph-e2ee? + [repo] + (when-let [conn (worker-state/get-datascript-conn repo)] + (true? (ldb/get-graph-rtc-e2ee? @conn)))) + +(defn- user-uuid [] + (some-> (worker-state/get-id-token) worker-util/parse-jwt :sub)) + +(defn- graph-encrypted-aes-key-idb-key + [graph-id] + (str "rtc-encrypted-aes-key###" graph-id)) + +(defn- clj r :keywordize-keys true))) + +(defn- js body))} + {:response-schema :e2ee/user-keys}))) + +(defn- (js body))} + {:response-schema :e2ee/graph-aes-key}))) + +(defn- aes-key repo)] + (p/resolved cached) + (let [base (e2ee-base) + user-id (user-uuid)] + (when-not (and (string? base) (string? user-id)) + (fail-fast :db-sync/missing-field {:base base :user-id user-id :graph-id graph-id})) + (p/let [{:keys [public-key encrypted-private-key]} (aes-key assoc repo aes-key) + aes-key))))) + +(defn- aes-key assoc repo aes-key) + aes-key))))) + +(defn js body))} + {:response-schema :e2ee/grant-access})] + nil)))))) + +(defn grant-graph-access! + [repo graph-id target-email] + ( {"content-type" snapshot-content-type} (string? encoding) (assoc "content-encoding" encoding)) _ (fetch-json upload-url diff --git a/src/main/frontend/worker/db_worker.cljs b/src/main/frontend/worker/db_worker.cljs index ceebbcd01c..8143e625ee 100644 --- a/src/main/frontend/worker/db_worker.cljs +++ b/src/main/frontend/worker/db_worker.cljs @@ -429,6 +429,14 @@ [editing-block-uuid] (db-sync/update-presence! editing-block-uuid)) +(def-thread-api :thread-api/db-sync-grant-graph-access + [repo graph-id target-email] + (db-sync/grant-graph-access! repo graph-id target-email)) + +(def-thread-api :thread-api/db-sync-ensure-user-rsa-keys + [] + (db-sync/ensure-user-rsa-keys!)) + (def-thread-api :thread-api/db-sync-upload-graph [repo] (db-sync/upload-graph! repo)) @@ -607,12 +615,15 @@ nil) (def-thread-api :thread-api/db-sync-import-kvs-rows - [repo rows reset?] + [repo rows reset? graph-id e2ee?] (p/let [_ (when reset? (close-db! repo)) + rows* (if (true? e2ee?) + (db-sync/sqlite-binds rows))) + (when (seq rows*) + (upsert-addr-content! db (rows->sqlite-binds rows*))) nil)) (def-thread-api :thread-api/db-sync-finalize-kvs-import diff --git a/src/test/frontend/worker/db_sync_test.cljs b/src/test/frontend/worker/db_sync_test.cljs index 2286d2adaa..005402da80 100644 --- a/src/test/frontend/worker/db_sync_test.cljs +++ b/src/test/frontend/worker/db_sync_test.cljs @@ -1,11 +1,14 @@ (ns frontend.worker.db-sync-test - (:require [cljs.test :refer [deftest is testing run-test]] + (:require [cljs.test :refer [deftest is testing run-test async]] [datascript.core :as d] + [frontend.common.crypt :as crypt] [frontend.worker.db-sync :as db-sync] [frontend.worker.rtc.client-op :as client-op] [frontend.worker.state :as worker-state] + [logseq.db.sqlite.util :as sqlite-util] [logseq.db.test.helper :as db-test] - [logseq.outliner.core :as outliner-core])) + [logseq.outliner.core :as outliner-core] + [promesa.core :as p])) (def ^:private test-repo "test-db-sync-repo") @@ -63,6 +66,64 @@ (is (= (:db/id child1') (:db/id (:block/parent parent')))) (is (= (:db/id page') (:db/id (:block/parent child1')))))))))) +(deftest encrypt-decrypt-tx-data-test + (async done + (-> (p/let [aes-key (crypt/ (p/let [aes-key (crypt/ (p/let [resp (db-sync/ensure-user-rsa-keys!)] + (is (map? resp)) + (is (= 2 (count @upload-called))) + (done)) + (p/catch (fn [e] + (is false (str e)) + (done)))))))) + (deftest two-children-cycle-test (testing "cycle from remote sync overwrite client (2 children)" (let [{:keys [conn client-ops-conn child1 child2]} (setup-parent-child)]