fix: extends cycle

This commit is contained in:
Tienson Qin
2026-01-26 18:30:53 +08:00
parent 0cf01ba780
commit ebf768e340
7 changed files with 247 additions and 144 deletions

View File

@@ -195,20 +195,37 @@
(let [edges (cycle-edges cycle)
cycle-nodes (set (distinct (butlast cycle)))
remote-parent-fn (:remote-parent-fn attr-opts)
remote-candidates (when (and (= :block/parent attr) remote-parent-fn)
(keep (fn [[from to]]
(let [remote-parent (remote-parent-fn db from to)
remote-parent (entid db remote-parent)
to-id (entid db to)]
(when (and remote-parent
(not= remote-parent to-id))
{:victim from
:bad-v to-id
:safe remote-parent
:outside? (not (contains? cycle-nodes remote-parent))})))
edges))
remote-choice (or (some #(when (:outside? %) %) remote-candidates)
(first remote-candidates))
remote-ref-set-fn (:remote-ref-set-fn attr-opts)
remote-parent-candidates (when (and (= :block/parent attr) remote-parent-fn)
(keep (fn [[from to]]
(let [remote-parent (remote-parent-fn db from to)
remote-parent (entid db remote-parent)
to-id (entid db to)]
(when (and remote-parent
(not= remote-parent to-id))
{:victim from
:bad-v to-id
:safe remote-parent
:outside? (not (contains? cycle-nodes remote-parent))})))
edges))
remote-ref-candidates (when (and (= :logseq.property.class/extends attr) remote-ref-set-fn)
(keep (fn [[from to]]
(let [to-id (entid db to)
remote-set (remote-ref-set-fn db from)
remote-set (set (keep #(entid db %) remote-set))]
(when (and (seq remote-set)
(not (contains? remote-set to-id)))
{:victim from
:bad-v to-id})))
edges))
remote-choice (cond
(seq remote-parent-candidates)
(or (some #(when (:outside? %) %) remote-parent-candidates)
(first remote-parent-candidates))
(seq remote-ref-candidates)
(first remote-ref-candidates)
:else
nil)
victim (or (:victim remote-choice) (pick-victim cycle touched))
[_from bad-v] (or (when (and remote-choice (:bad-v remote-choice))
[victim (:bad-v remote-choice)])
@@ -356,7 +373,12 @@
(fn [db e attr bad-v]
(let [remote-parent (some-> (d/entity remote-db e) :block/parent :db/id)
remote-parent (when (and remote-parent (not= remote-parent bad-v)) remote-parent)]
(or remote-parent (safe-target-for-block-parent db e attr bad-v))))))
(or remote-parent (safe-target-for-block-parent db e attr bad-v)))))
remote-db
(assoc-in [:logseq.property.class/extends :remote-ref-set-fn]
(fn [_db e]
(some->> (d/entity remote-db e)
:logseq.property.class/extends))))
remote-touched-by-attr (touched-eids-many (:tx-data remote-tx-report))
local-touched-by-attr (touched-eids-many (:tx-data rebase-tx-report))
candidates-by-attr (union-candidates remote-touched-by-attr local-touched-by-attr)

View File

@@ -206,3 +206,25 @@
(is (some #(= :fix-cycle (:outliner-op %)) tx-metas))
(is (every? #(false? (:gen-undo-ops? %)) tx-metas))
(is (every? #(false? (:persist-op? %)) tx-metas))))))
(deftest class-extends-cycle-prefers-remote-set-test
(let [conn (new-conn)]
(d/transact! conn [{:db/ident :logseq.class/Root}
{:db/ident :user.class/B}
{:db/ident :user.class/A :logseq.property.class/extends :logseq.class/Root}])
(testing "prefers remote extends set when breaking cycles"
(let [remote-report (d/transact! conn [{:db/ident :user.class/B
:logseq.property.class/extends :user.class/A}])
rebase-report (d/transact! conn [{:db/ident :user.class/A
:logseq.property.class/extends :user.class/B}])
tx-metas (fix-cycle! conn remote-report rebase-report)]
(let [a (d/entity @conn :user.class/A)
b (d/entity @conn :user.class/B)
extends-a (set (map :db/ident (:logseq.property.class/extends a)))
extends-b (set (map :db/ident (:logseq.property.class/extends b)))]
(is (not (contains? extends-a :user.class/B)))
(is (contains? extends-a :logseq.class/Root))
(is (contains? extends-b :user.class/A)))
(is (some #(= :fix-cycle (:outliner-op %)) tx-metas))
(is (every? #(false? (:gen-undo-ops? %)) tx-metas))
(is (every? #(false? (:persist-op? %)) tx-metas))))))

View File

@@ -88,24 +88,26 @@
(fn [d]
(if (= (count d) 5)
(let [[e a v t added] d
retract? (not added)
e' (if retract?
(eid->lookup db-before e)
(or (eid->lookup db-before e)
(eid->tempid db-after e)))
v' (if (and (integer? v)
(pos? v)
(or (= :db.type/ref (:db/valueType (d/entity db-after a)))
(= :db.type/ref (:db/valueType (d/entity db-before a)))))
(if retract?
(eid->lookup db-before v)
(or (eid->lookup db-before v)
(eid->tempid db-after v)))
v)]
(when (and (some? e') (some? v'))
(if added
[:db/add e' a v' t]
[:db/retract e' a v' t])))
retract? (not added)]
(when-not (and retract?
(contains? #{:block/created-at :block/updated-at :block/title} a))
(let [e' (if retract?
(eid->lookup db-before e)
(or (eid->lookup db-before e)
(eid->tempid db-after e)))
v' (if (and (integer? v)
(pos? v)
(or (= :db.type/ref (:db/valueType (d/entity db-after a)))
(= :db.type/ref (:db/valueType (d/entity db-before a)))))
(if retract?
(eid->lookup db-before v)
(or (eid->lookup db-before v)
(eid->tempid db-after v)))
v)]
(when (and (some? e') (some? v'))
(if added
[:db/add e' a v' t]
[:db/retract e' a v' t])))))
d)))
(remove-retract-entity-ref db-after)
distinct))

View File

@@ -795,6 +795,9 @@
(db-normalize/replace-attr-retract-with-retract-entity-v2 db)
(remove (fn [item]
(or (= :db/retractEntity (first item))
(and (= :db/retract (first item))
(contains? #{:block/created-at :block/updated-at :block/title}
(nth item 2)))
(contains? local-deleted-ids (get-lookup-id (last item))))))
keep-last-update)]
;; (when (not= tx-data sanitized-tx-data)
@@ -1133,45 +1136,46 @@
:persist-op? false}
tx-report
(if has-local-changes?
(ldb/transact-with-temp-conn!
conn
{:rtc-tx? true}
(fn [temp-conn _*batch-tx-data]
(let [tx-meta temp-tx-meta
reversed-tx-report (ldb/transact! temp-conn reversed-tx-data (assoc tx-meta :op :reverse))
_ (reset! *reversed-tx-report reversed-tx-report)
(let [batch-tx-meta {:rtc-tx? true}]
(ldb/transact-with-temp-conn!
conn
batch-tx-meta
(fn [temp-conn _*batch-tx-data]
(let [tx-meta temp-tx-meta
reversed-tx-report (ldb/transact! temp-conn reversed-tx-data (assoc tx-meta :op :reverse))
_ (reset! *reversed-tx-report reversed-tx-report)
;; 2. transact remote tx-data
remote-tx-report (let [tx-meta (assoc tx-meta :op :transact-remote-tx-data)]
(ldb/transact! temp-conn safe-remote-tx-data tx-meta))
_ (reset! *remote-tx-report remote-tx-report)
local-deleted-blocks (get-local-deleted-blocks reversed-tx-report reversed-tx-data)
_ (when (seq remote-deleted-blocks)
(reset! *remote-deleted-ids (set (map :block/uuid remote-deleted-blocks))))
remote-tx-report (let [tx-meta (assoc tx-meta :op :transact-remote-tx-data)]
(ldb/transact! temp-conn safe-remote-tx-data tx-meta))
_ (reset! *remote-tx-report remote-tx-report)
local-deleted-blocks (get-local-deleted-blocks reversed-tx-report reversed-tx-data)
_ (when (seq remote-deleted-blocks)
(reset! *remote-deleted-ids (set (map :block/uuid remote-deleted-blocks))))
;; _ (prn :debug
;; :local-deleted-blocks (map (fn [b] (select-keys b [:db/id :block/title])) local-deleted-blocks)
;; :remote-deleted-blocks remote-deleted-blocks)
deleted-nodes (concat local-deleted-blocks remote-deleted-blocks)
deleted-ids (set (keep :block/uuid deleted-nodes))
deleted-nodes (concat local-deleted-blocks remote-deleted-blocks)
deleted-ids (set (keep :block/uuid deleted-nodes))
;; 3. rebase pending local txs
rebase-tx-report (when (seq local-txs)
(let [pending-tx-data (mapcat :tx local-txs)
rebased-tx-data (sanitize-tx-data
(or (:db-after remote-tx-report)
(:db-after reversed-tx-report))
pending-tx-data
(set (map :block/uuid local-deleted-blocks)))]
rebase-tx-report (when (seq local-txs)
(let [pending-tx-data (mapcat :tx local-txs)
rebased-tx-data (sanitize-tx-data
(or (:db-after remote-tx-report)
(:db-after reversed-tx-report))
pending-tx-data
(set (map :block/uuid local-deleted-blocks)))]
;; (prn :debug :pending-tx-data pending-tx-data)
;; (prn :debug :rebased-tx-data rebased-tx-data)
(when (seq rebased-tx-data)
(ldb/transact! temp-conn rebased-tx-data (assoc tx-meta :op :rebase)))))
(when (seq rebased-tx-data)
(ldb/transact! temp-conn rebased-tx-data (assoc tx-meta :op :rebase)))))
;; 4. delete nodes and fix tx data
db @temp-conn
deleted-nodes (keep (fn [id] (d/entity db [:block/uuid id])) deleted-ids)]
(delete-nodes! temp-conn deleted-nodes (assoc tx-meta :op :delete-blocks))
(fix-tx! temp-conn remote-tx-report rebase-tx-report (assoc tx-meta :op :fix))))
{:listen-db (fn [{:keys [tx-meta tx-data]}]
(when-not (contains? #{:reverse :transact-remote-tx-data} (:op tx-meta))
(swap! *rebase-tx-data into tx-data)))})
db @temp-conn
deleted-nodes (keep (fn [id] (d/entity db [:block/uuid id])) deleted-ids)]
(delete-nodes! temp-conn deleted-nodes (assoc tx-meta :op :delete-blocks))
(fix-tx! temp-conn remote-tx-report rebase-tx-report (assoc tx-meta :op :fix))))
{:listen-db (fn [{:keys [tx-meta tx-data]}]
(when-not (contains? #{:reverse :transact-remote-tx-data} (:op tx-meta))
(swap! *rebase-tx-data into tx-data)))}))
(ldb/transact-with-temp-conn!
conn
{:rtc-tx? true}

View File

@@ -6,70 +6,70 @@
(deftest download-graph-e2ee-detection-test
(async done
(with-redefs [db-sync/fetch-json (fn [_ _ _]
(p/resolved {:encrypted-aes-key "k"}))]
(-> (p/let [enabled? (#'db-sync/fetch-graph-e2ee? "http://base" "graph-1")]
(-> (p/with-redefs [db-sync/fetch-json (fn [_ _ _]
(p/resolved {:encrypted-aes-key "k"}))]
(p/let [enabled? (#'db-sync/fetch-graph-e2ee? "http://base" "graph-1")]
(is (true? enabled?))
(done))
(p/catch (fn [e]
(is false (str e))
(done)))))))
(done)))
(p/catch (fn [e]
(is false (str e))
(done))))))
(deftest download-graph-e2ee-missing-key-test
(async done
(with-redefs [db-sync/fetch-json (fn [_ _ _]
(p/resolved {}))]
(-> (p/let [enabled? (#'db-sync/fetch-graph-e2ee? "http://base" "graph-1")]
(-> (p/with-redefs [db-sync/fetch-json (fn [_ _ _]
(p/resolved {}))]
(p/let [enabled? (#'db-sync/fetch-graph-e2ee? "http://base" "graph-1")]
(is (false? enabled?))
(done))
(p/catch (fn [e]
(is false (str e))
(done)))))))
(done)))
(p/catch (fn [e]
(is false (str e))
(done))))))
(deftest remove-member-request-test
(async done
(let [called (atom nil)]
(with-redefs [db-sync/http-base (fn [] "http://base")
db-sync/fetch-json (fn [url opts _]
(reset! called {:url url :opts opts})
(p/resolved {:ok true}))
user-handler/task--ensure-id&access-token (fn [resolve _reject]
(resolve true))]
(-> (p/let [_ (db-sync/<rtc-remove-member! "graph-1" "user-2")
(-> (p/with-redefs [db-sync/http-base (fn [] "http://base")
db-sync/fetch-json (fn [url opts _]
(reset! called {:url url :opts opts})
(p/resolved {:ok true}))
user-handler/task--ensure-id&access-token (fn [resolve _reject]
(resolve true))]
(p/let [_ (db-sync/<rtc-remove-member! "graph-1" "user-2")
{:keys [url opts]} @called]
(is (= "http://base/graphs/graph-1/members/user-2" url))
(is (= "DELETE" (:method opts)))
(done))
(p/catch (fn [e]
(is false (str e))
(done))))))))
(done)))
(p/catch (fn [e]
(is false (str e))
(done)))))))
(deftest leave-graph-uses-current-user-test
(async done
(let [called (atom nil)]
(with-redefs [db-sync/http-base (fn [] "http://base")
db-sync/fetch-json (fn [url opts _]
(reset! called {:url url :opts opts})
(p/resolved {:ok true}))
user-handler/task--ensure-id&access-token (fn [resolve _reject]
(resolve true))
user-handler/user-uuid (fn [] "user-1")]
(-> (p/let [_ (db-sync/<rtc-leave-graph! "graph-1")
(-> (p/with-redefs [db-sync/http-base (fn [] "http://base")
db-sync/fetch-json (fn [url opts _]
(reset! called {:url url :opts opts})
(p/resolved {:ok true}))
user-handler/task--ensure-id&access-token (fn [resolve _reject]
(resolve true))
user-handler/user-uuid (fn [] "user-1")]
(p/let [_ (db-sync/<rtc-leave-graph! "graph-1")
{:keys [url opts]} @called]
(is (= "http://base/graphs/graph-1/members/user-1" url))
(is (= "DELETE" (:method opts)))
(done))
(p/catch (fn [e]
(is false (str e))
(done))))))))
(done)))
(p/catch (fn [e]
(is false (str e))
(done)))))))
(deftest leave-graph-missing-user-test
(async done
(with-redefs [user-handler/user-uuid (fn [] nil)]
(-> (db-sync/<rtc-leave-graph! "graph-1")
(p/then (fn [_]
(is false "expected rejection")
(done)))
(p/catch (fn [e]
(is (= :db-sync/invalid-member (:type (ex-data e))))
(done)))))))
(-> (p/with-redefs [user-handler/user-uuid (fn [] nil)]
(db-sync/<rtc-leave-graph! "graph-1"))
(p/then (fn [_]
(is false "expected rejection")
(done)))
(p/catch (fn [e]
(is (= :db-sync/invalid-member (:type (ex-data e))))
(done))))))

View File

@@ -190,8 +190,7 @@ This can be called in synchronous contexts as no async fns should be invoked"
{:block/uuid page-uuid
:block/name "test"
:block/title "Test"
;; :block/tags #{:logseq.class/Page}
}
:block/tags #{:logseq.class/Page}}
;; first block
{:block/uuid first-block-uuid
:block/page page-id

View File

@@ -5,6 +5,7 @@
[frontend.worker.db-sync :as db-sync]
[frontend.worker.rtc.client-op :as client-op]
[frontend.worker.state :as worker-state]
[logseq.db :as ldb]
[logseq.db.sqlite.util :as sqlite-util]
[logseq.db.test.helper :as db-test]
[logseq.outliner.core :as outliner-core]
@@ -75,8 +76,8 @@
encrypted (#'db-sync/<encrypt-tx-data aes-key tx-data)]
(is (not= tx-data encrypted))
(is (string? (nth (first encrypted) 3)))
(is (= (nth (second encrypted) 3)
"page"))
(is (string? (nth (second encrypted) 3)))
(is (not= (nth (second encrypted) 3) "page"))
(p/let [decrypted (#'db-sync/<decrypt-tx-data aes-key encrypted)]
(is (= tx-data decrypted))
(done)))
@@ -96,8 +97,15 @@
(let [[_ content* _] (first encrypted)]
(is (string? content*))
(is (not= content content*)))
(p/let [decrypted (#'db-sync/<decrypt-snapshot-rows aes-key encrypted)]
(is (= rows decrypted))
(p/let [decrypted (#'db-sync/<decrypt-snapshot-rows aes-key encrypted)
normalize (fn [content']
(let [data (ldb/read-transit-str content')]
(if (map? data)
(update data :keys (fnil vec []))
data)))
decoded-original (normalize content)
decoded-decrypted (normalize (nth (first decrypted) 1))]
(is (= decoded-original decoded-decrypted))
(done)))
(p/catch (fn [e]
(is false (str e))
@@ -116,21 +124,21 @@
(-> (p/let [aes-key (crypt/<generate-aes-key)
tx-data (#'db-sync/<encrypt-datoms aes-key datoms)
title-tx (first (filter (fn [item]
(and (= (:e title-datom) (nth item 1))
(= :block/title (nth item 2))))
(and (= (:e title-datom) (:e item))
(= :block/title (:a item))))
tx-data))
name-tx (first (filter (fn [item]
(and (= (:e name-datom) (nth item 1))
(= :block/name (nth item 2))))
(and (= (:e name-datom) (:e item))
(= :block/name (:a item))))
tx-data))
uuid-tx (first (filter (fn [item]
(and (= (:e uuid-datom) (nth item 1))
(= :block/uuid (nth item 2))))
(and (= (:e uuid-datom) (:e item))
(= :block/uuid (:a item))))
tx-data))]
(is (string? (nth title-tx 3)))
(is (string? (nth name-tx 3)))
(is (not= (:v title-datom) (nth title-tx 3)))
(is (= (:v uuid-datom) (nth uuid-tx 3)))
(is (string? (:v title-tx)))
(is (string? (:v name-tx)))
(is (not= (:v title-datom) (:v title-tx)))
(is (= (:v uuid-datom) (:v uuid-tx)))
(done))
(p/catch (fn [e]
(is false (str e))
@@ -139,23 +147,23 @@
(deftest ensure-user-rsa-keys-test
(async done
(let [upload-called (atom nil)]
(with-redefs [db-sync/e2ee-base (fn [] "http://base")
db-sync/<fetch-user-rsa-key-pair-raw (fn [_] (p/resolved {}))
db-sync/<upload-user-rsa-key-pair! (fn [_ public-key encrypted-private-key]
(reset! upload-called [public-key encrypted-private-key])
(p/resolved {:public-key public-key
:encrypted-private-key encrypted-private-key}))
crypt/<generate-rsa-key-pair (fn [] (p/resolved #js {:publicKey :pub :privateKey :priv}))
crypt/<export-public-key (fn [_] (p/resolved :pub-export))
crypt/<encrypt-private-key (fn [_ _] (p/resolved :priv-encrypted))
worker-state/<invoke-main-thread (fn [_] (p/resolved {:password "pw"}))]
(-> (p/let [resp (db-sync/ensure-user-rsa-keys!)]
(-> (p/with-redefs [db-sync/e2ee-base (fn [] "http://base")
db-sync/<fetch-user-rsa-key-pair-raw (fn [_] (p/resolved {}))
db-sync/<upload-user-rsa-key-pair! (fn [_ public-key encrypted-private-key]
(reset! upload-called [public-key encrypted-private-key])
(p/resolved {:public-key public-key
:encrypted-private-key encrypted-private-key}))
crypt/<generate-rsa-key-pair (fn [] (p/resolved #js {:publicKey :pub :privateKey :priv}))
crypt/<export-public-key (fn [_] (p/resolved :pub-export))
crypt/<encrypt-private-key (fn [_ _] (p/resolved :priv-encrypted))
worker-state/<invoke-main-thread (fn [_] (p/resolved {:password "pw"}))]
(p/let [resp (db-sync/ensure-user-rsa-keys!)]
(is (map? resp))
(is (= 2 (count @upload-called)))
(done))
(p/catch (fn [e]
(is false (str e))
(done))))))))
(done)))
(p/catch (fn [e]
(is false (str e))
(done)))))))
(deftest two-children-cycle-test
(testing "cycle from remote sync overwrite client (2 children)"
@@ -169,7 +177,7 @@
[[:db/add (:db/id child2) :block/parent (:db/id child1)]])
(let [child1' (d/entity @conn (:db/id child1))
child2' (d/entity @conn (:db/id child2))]
(is (= "page 1" (:block/title (:block/parent child1'))))
(is (= "parent" (:block/title (:block/parent child1'))))
(is (= "child 1" (:block/title (:block/parent child2'))))))))))
(deftest three-children-cycle-test
@@ -188,8 +196,8 @@
child2' (d/entity @conn (:db/id child2))
child3' (d/entity @conn (:db/id child3))]
(is (= "child 2" (:block/title (:block/parent child'))))
(is (= "page 1" (:block/title (:block/parent child2'))))
(is (= "child 2" (:block/title (:block/parent child3'))))))))))
(is (= "child 3" (:block/title (:block/parent child2'))))
(is (= "parent" (:block/title (:block/parent child3'))))))))))
(deftest ignore-missing-parent-update-after-local-delete-test
(testing "remote parent retracted while local adds another child"
@@ -239,6 +247,52 @@
(is (some? (:block/order child1')))
(is (not= (:block/order child1') (:block/order child2')))))))))
(deftest two-clients-extends-cycle-test
(testing "remote extends wins when two clients create a cycle"
(let [conn (db-test/create-conn)
client-ops-conn (d/create-conn client-op/schema-in-db)
root-id (d/entid @conn :logseq.class/Root)
tag-id (d/entid @conn :logseq.class/Tag)
now 1710000000000
a-uuid (random-uuid)
b-uuid (random-uuid)]
(d/transact! conn [{:db/ident :user.class/A
:block/uuid a-uuid
:block/name "a"
:block/title "A"
:block/created-at now
:block/updated-at now
:block/tags #{tag-id}
:logseq.property.class/extends #{root-id}}
{:db/ident :user.class/B
:block/uuid b-uuid
:block/name "b"
:block/title "B"
:block/created-at now
:block/updated-at now
:block/tags #{tag-id}
:logseq.property.class/extends #{root-id}}])
(with-datascript-conns conn client-ops-conn
(fn []
(let [a-id (d/entid @conn :user.class/A)
b-id (d/entid @conn :user.class/B)]
(d/transact! conn [[:db/add a-id
:logseq.property.class/extends
b-id]])
(#'db-sync/apply-remote-tx!
test-repo
nil
[[:db/add b-id
:logseq.property.class/extends
a-id]])
(let [a (d/entity @conn :user.class/A)
b (d/entity @conn :user.class/B)
extends-a (set (map :db/ident (:logseq.property.class/extends a)))
extends-b (set (map :db/ident (:logseq.property.class/extends b)))]
(is (not (contains? extends-a :user.class/B)))
(is (contains? extends-a :logseq.class/Root))
(is (contains? extends-b :user.class/A)))))))))
(deftest fix-duplicate-orders-with-local-and-remote-new-blocks-test
(testing "local and remote new sibling blocks at the same location get unique orders"
(let [{:keys [conn client-ops-conn parent]} (setup-parent-child)
@@ -322,7 +376,7 @@
test-repo
nil
[[:db/add (:db/id child1) :block/title "same"]])
(is (= 1 (count (#'db-sync/pending-txs test-repo))))))))))
(is (= 0 (count (#'db-sync/pending-txs test-repo))))))))))
(deftest normalize-online-users-include-editing-block-test
(testing "online user normalization preserves editing block info"