clone/logseq
1
0
Fork 0
mirror of https://github.com/logseq/logseq.git synced 2026-04-24 22:25:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

another try of client fix

Browse Source
This commit is contained in:
Tienson Qin
2026-01-13 18:08:36 +08:00
parent 8c8ca9ce45
commit 2afc7fe48d
6 changed files with 228 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
deps/db-sync/src/logseq/db_sync/worker_core.cljs → deps/db-sync/src/logseq/db_sync/order.cljs vendored
View File

@@ -1,4 +1,4 @@
(ns logseq.db-sync.worker-core
(ns logseq.db-sync.order
(:require [datascript.core :as d]
[logseq.db.common.order :as db-order]))

4
deps/db-sync/src/logseq/db_sync/worker.cljs vendored
View File

@@ -9,10 +9,10 @@
[logseq.db :as ldb]
[logseq.db-sync.common :as common :refer [cors-headers]]
[logseq.db-sync.malli-schema :as db-sync-schema]
[logseq.db-sync.order :as sync-order]
[logseq.db-sync.parent-missing :as db-sync-parent-missing]
[logseq.db-sync.protocol :as protocol]
[logseq.db-sync.storage :as storage]
[logseq.db-sync.worker-core :as worker-core]
[logseq.db.common.normalize :as db-normalize]
[promesa.core :as p]
[shadow.cljs.modern :refer (defclass)]))
@@ -347,7 +347,7 @@
;; TODO: fix cycle
(db-sync-parent-missing/fix-parent-missing! temp-conn tx-report)
(prn :debug :fix-duplicate-orders)
(worker-core/fix-duplicate-orders! temp-conn @*batch-tx-data))))
(sync-order/fix-duplicate-orders! temp-conn @*batch-tx-data))))
(prn :debug :finished-db-transact)
(let [new-t (storage/get-t sql)]
;; FIXME: no need to broadcast if client tx is less than remote tx

9
deps/db-sync/test/logseq/db_sync/worker_test.cljs vendored
View File

@@ -1,7 +1,7 @@
(ns logseq.db-sync.worker-test
(:require [cljs.test :refer [deftest is]]
[datascript.core :as d]
[logseq.db-sync.worker-core :as worker-core]
[logseq.db-sync.order :as sync-order]
[logseq.db.common.order :as db-order]
[logseq.db.frontend.schema :as db-schema]))
@@ -23,9 +23,8 @@
:block/parent [:block/uuid parent]
:block/order order-b}])
(let [tx [[:db/add [:block/uuid block-b] :block/order order-a]]
fixed (worker-core/fix-duplicate-orders! @conn tx)
db' (d/db-with @conn fixed)
order-a' (:block/order (d/entity db' [:block/uuid block-a]))
order-b' (:block/order (d/entity db' [:block/uuid block-b]))]
_ (sync-order/fix-duplicate-orders! @conn tx)
order-a' (:block/order (d/entity @conn [:block/uuid block-a]))
order-b' (:block/order (d/entity @conn [:block/uuid block-b]))]
(is (= order-a order-a'))
(is (not= order-a' order-b')))))

44
rebase.md Normal file
View File

@@ -0,0 +1,44 @@
# Remote Rebase (Client)
This doc describes how the client rebases and applies remote tx data in
`rebase-apply-remote-tx!` to keep the server thin and to tolerate offline
conflicts.
## Goals
- Apply remote tx data on top of the local DB even when the client made offline
changes that would otherwise invalidate the tx.
- Keep server-side logic minimal; the client is responsible for sanitizing and
repairing remote txs before transact.
## Flow Summary
1. Collect pending local txs and derive a reversed tx set.
2. Build a rebase DB by applying reversed local txs to the current DB. This
represents the server base state.
3. Sanitize the remote tx data against the current DB (with offline changes)
so it can be safely transacted.
4. Compute a tx-report on the rebase DB to identify deletes and handle them
first (pages before blocks).
5. Transact the sanitized remote tx data and fix duplicate orders.
## Sanitation Rules
Remote tx data is transformed before transact to avoid invalid operations:
- Convert :block/uuid retracts into :db.fn/retractEntity.
- Keep only the last :block/parent update per entity.
- Drop datoms that reference missing entities or missing ref targets.
- Repair parent cycles by reparenting to the page root.
- Drop class extends updates that would introduce a cycle.
## Delete Semantics
Deletes are detected using the rebase DB. When a parent is deleted on the
server, the client deletes that parent and all of its children, even if the
client moved some of those children offline.
## Testing
See `src/test/frontend/worker/db_sync_test.cljs` for rebase-related tests, such
as cycle handling and invalid parent updates.

182
src/main/frontend/worker/db_sync.cljs
View File

@@ -2,15 +2,21 @@
"Simple db-sync client based on promesa + WebSocket."
(:require [clojure.string :as string]
[datascript.core :as d]
[frontend.worker.handler.page :as worker-page]
[frontend.worker.rtc.client-op :as client-op]
[frontend.worker.state :as worker-state]
[lambdaisland.glogi :as log]
[logseq.common.path :as path]
[logseq.common.util :as common-util]
[logseq.db :as ldb]
[logseq.db-sync.cycle :as db-sync-cycle]
[logseq.db-sync.malli-schema :as db-sync-schema]
[logseq.db-sync.order :as sync-order]
[logseq.db.common.normalize :as db-normalize]
[logseq.db.sqlite.util :as sqlite-util]
[logseq.outliner.core :as outliner-core]
[logseq.outliner.pipeline :as outliner-pipeline]
[logseq.outliner.transaction :as outliner-tx]
[promesa.core :as p]))
(defonce *repo->latest-remote-tx (atom {}))
@@ -278,6 +284,116 @@
[(second item) (nth item 2)]
item)))))
(defn- normalize-entity-ref
[ref]
(cond
(uuid? ref) [:block/uuid ref]
(keyword? ref) [:db/ident ref]
:else ref))
(defn- entity-ref->entid
[db ref]
(cond
(nil? ref) nil
(number? ref) (when (pos? ref) ref)
(uuid? ref) (d/entid db [:block/uuid ref])
(vector? ref) (d/entid db ref)
(keyword? ref) (d/entid db [:db/ident ref])
:else nil))
(defn- entity-ref-matches?
[db target ref]
(let [target-entid (entity-ref->entid db target)
ref-entid (entity-ref->entid db ref)]
(cond
(and target-entid ref-entid) (= target-entid ref-entid)
:else (= (normalize-entity-ref target) (normalize-entity-ref ref)))))
(defn- valid-entity-ref?
[db ref]
(cond
(nil? ref) false
(number? ref) (or (neg? ref) (some? (d/entity db ref)))
:else (some? (entity-ref->entid db ref))))
(defn- ref-attr?
[db attr]
(when attr
(= :db.type/ref (:db/valueType (d/entity db attr)))))
(defn- drop-invalid-refs
[db tx-data]
(->> tx-data
(keep (fn [tx]
(cond
(and (vector? tx) (= :db.fn/retractEntity (first tx)))
(when (valid-entity-ref? db (second tx))
tx)
(and (vector? tx) (= 4 (count tx)))
(let [[_op e a v] tx]
(when (valid-entity-ref? db e)
(if (and (ref-attr? db a) (some? v))
(when (valid-entity-ref? db v)
tx)
tx)))
:else
tx)))))
(defn- remove-attr-updates
[db tx-data attr entity-ref]
(remove (fn [tx]
(and (vector? tx)
(= attr (nth tx 2 nil))
(entity-ref-matches? db entity-ref (nth tx 1 nil))))
tx-data))
(defn- tx-entity-ref
[db entity-ref]
(or (entity-ref->entid db entity-ref)
(normalize-entity-ref entity-ref)))
(defn- replace-parent-with-page-root
[db tx-data entity-ref]
(let [entity-entid (entity-ref->entid db entity-ref)
page-uuid (when entity-entid
(some-> (d/entity db entity-entid) :block/page :block/uuid))
page-ref (when page-uuid [:block/uuid page-uuid])]
(cond-> (remove-attr-updates db tx-data :block/parent entity-ref)
page-ref
(conj [:db/add (tx-entity-ref db entity-ref) :block/parent page-ref]))))
(defn- fix-cycle-updates
[db tx-data]
(loop [tx-data tx-data
attempt 0]
(if (>= attempt 4)
tx-data
(if-let [{:keys [attr entity]} (db-sync-cycle/detect-cycle db tx-data)]
(let [tx-data' (case attr
:block/parent (replace-parent-with-page-root db tx-data entity)
:logseq.property.class/extends (remove-attr-updates db tx-data attr entity)
(remove-attr-updates db tx-data attr entity))]
(recur tx-data' (inc attempt)))
tx-data))))
(defn- sanitize-remote-tx-data
[db tx-data]
(let [tx-data (vec tx-data)
original-count (count tx-data)
tx-data (->> tx-data
db-normalize/replace-attr-retract-with-retract-entity-v2
keep-last-parent-update
(drop-invalid-refs db)
(fix-cycle-updates db))
dropped (- original-count (count tx-data))]
(when (pos? dropped)
(log/info :db-sync/remote-tx-sanitized
{:dropped dropped
:original original-count}))
tx-data))
(defn- flush-pending!
[repo client]
(let [inflight @(:inflight client)
@@ -534,29 +650,49 @@
(mapcat :reversed-tx)
reverse
db-normalize/replace-attr-retract-with-retract-entity-v2)
*tx-report (atom nil)
_ (ldb/transact-with-temp-conn!
conn
{:rtc-tx? true}
(fn [conn]
(prn :debug :reversed-tx-data reversed-tx-data)
(when (seq reversed-tx-data)
(ldb/transact! conn reversed-tx-data))
(let [;; 2. apply remote tx
tx-data' (db-normalize/replace-attr-retract-with-retract-entity tx-data)
_ (prn :debug :apply-remote-tx-data tx-data')
_ (prn :debug :original-tx-data tx-data')
tx-report (ldb/transact! conn tx-data')]
(reset! *tx-report tx-report)
;; TODO: 3. compute and compare checksum
;; Notice: no need to restore local changes, we'll send pending-txs to server
;; and let the server to fix the tx-data if there's invalidation
)))
tx-report @*tx-report
db-after (:db-after tx-report)
asset-uuids (asset-uuids-from-tx db-after (:tx-data tx-report))]
(when (seq asset-uuids)
(enqueue-asset-downloads! repo client asset-uuids)))
tx-report (ldb/transact-with-temp-conn!
conn
{:rtc-tx? true}
(fn [temp-conn _*batch-tx-data]
(let [db @temp-conn
;; 1. rebase
rebase (:db-after (d/with db reversed-tx-data))
tx-report (d/with rebase tx-data)]
;; TODO: 2. ensure checksum matches between client & server
;; 3. fix data
(let [deleted-blocks (outliner-pipeline/filter-deleted-blocks (:tx-data tx-report))
tx-meta {:persist-op? false
:gen-undo-ops? false}]
(when (seq deleted-blocks)
(let [nodes (map #(d/entity @temp-conn (:db/id %)) deleted-blocks)
pages (filter ldb/page? nodes)
blocks (remove ldb/page? nodes)]
;; deleting pages first
(doseq [page pages]
(worker-page/delete! temp-conn (:block/uuid page) tx-meta))
(when (seq blocks)
(outliner-tx/transact!
(assoc tx-meta
:outliner-op :delete-blocks
:transact-opts {:conn temp-conn})
(outliner-core/delete-blocks! temp-conn blocks {})))))
;; 4. apply remote tx-data
(when (seq tx-data)
(let [rtc-tx-data (sanitize-remote-tx-data @temp-conn tx-data)
tx-report (ldb/transact! temp-conn rtc-tx-data)]
(prn :debug :tx-data rtc-tx-data)
(sync-order/fix-duplicate-orders! temp-conn (:tx-data tx-report))))))))]
(when tx-report
(let [db-after (:db-after tx-report)
asset-uuids (asset-uuids-from-tx db-after (:tx-data tx-report))]
(when (seq asset-uuids)
(enqueue-asset-downloads! repo client asset-uuids))))
;; TODO: Remove all pending txs, insert the above one
)
(catch :default e
(log/error :db-sync/apply-remote-tx-failed {:error e})
(throw e)))

18
src/test/frontend/worker/db_sync_test.cljs
View File

@@ -61,3 +61,21 @@
(is (some? page'))
(is (= (:db/id page') (:db/id (:block/parent parent'))))
(is (= (:db/id parent') (:db/id (:block/parent child'))))))))))
(deftest drop-missing-parent-update-test
(testing "drop invalid parent updates during remote rebase"
(let [{:keys [conn child]} (setup-parent-child)
child-uuid (:block/uuid child)
original-parent-uuid (:block/uuid (:block/parent (d/entity @conn (:db/id child))))
missing-parent-uuid (random-uuid)]
(prn :debug :missing-parent-uuid missing-parent-uuid)
(with-datascript-conn conn
(fn []
(#'db-sync/rebase-apply-remote-tx!
test-repo
nil
[[:db/add [:block/uuid child-uuid]
:block/parent [:block/uuid missing-parent-uuid]]])
(let [child' (d/entity @conn (:db/id child))]
(is (= original-parent-uuid
(:block/uuid (:block/parent child'))))))))))