diff --git a/deps/db-sync/.carve/ignore b/deps/db-sync/.carve/ignore index 335022eb4b..6582368232 100644 --- a/deps/db-sync/.carve/ignore +++ b/deps/db-sync/.carve/ignore @@ -23,3 +23,5 @@ logseq.db-sync.snapshot/finalize-datoms-jsonl-buffer logseq.db-sync.worker/worker ;; debugging logseq.db-sync.worker.timing/summary +;; API +logseq.db-sync.checksum/recompute-checksum-diagnostics diff --git a/deps/db-sync/README.md b/deps/db-sync/README.md index 14c9610f83..dfbafe0f15 100644 --- a/deps/db-sync/README.md +++ b/deps/db-sync/README.md @@ -59,6 +59,18 @@ before it calls the worker delete endpoint for each graph. Set `DB_SYNC_BASE_URL` and `DB_SYNC_ADMIN_TOKEN` or pass `--base-url` and `--admin-token` when running it. +Delete a user completely (owned graphs, memberships, keys, and user row): + +```bash +cd deps/db-sync +yarn delete-user-totally --username alice +yarn delete-user-totally --user-id us-east-1:example-user-id +``` + +The script prints all linked graphs first, deletes owned graphs through the +admin graph delete endpoint, then removes the user's remaining D1 references. +It requires typing `DELETE` as confirmation. + ### Node.js Adapter (self-hosted) Build the adapter: diff --git a/deps/db-sync/package.json b/deps/db-sync/package.json index 508e186da2..b6bd0f1bcd 100644 --- a/deps/db-sync/package.json +++ b/deps/db-sync/package.json @@ -7,6 +7,7 @@ "watch": "clojure -M:cljs watch db-sync", "release": "clojure -M:cljs release db-sync", "delete-graphs-for-user": "node worker/scripts/delete_graphs_for_user.js", + "delete-user-totally": "node worker/scripts/delete_user_totally.js", "show-graphs-for-user": "node worker/scripts/show_graphs_for_user.js", "build:node-adapter": "clojure -M:cljs release db-sync-node", "dev:node-adapter": "clojure -M:cljs watch db-sync-node", diff --git a/deps/db-sync/src/logseq/db_sync/checksum.cljs b/deps/db-sync/src/logseq/db_sync/checksum.cljs index f0d59a7d60..49aac26e1f 100644 --- a/deps/db-sync/src/logseq/db_sync/checksum.cljs +++ b/deps/db-sync/src/logseq/db_sync/checksum.cljs @@ -55,6 +55,12 @@ (or (parse-hex32 (subs checksum 8 16)) 0)] [0 0])) +(defn- valid-checksum? + [checksum] + (boolean + (and (string? checksum) + (re-matches #"[0-9a-fA-F]{16}" checksum)))) + (defn- state->checksum [[fnv djb]] (str (unsigned-hex fnv) @@ -98,9 +104,9 @@ (not e2ee?) (hash-code field-separator) (not e2ee?) (digest-string name) (not e2ee?) (hash-code field-separator) - true (digest-string (some-> parent str)) + true (digest-string (some-> parent :block/uuid str)) true (hash-code field-separator) - true (digest-string (some-> page str)))))) + true (digest-string (some-> page :block/uuid str)))))) (defn recompute-checksum [db] @@ -120,25 +126,60 @@ [0 0]) state->checksum))) +(defn recompute-checksum-diagnostics + [db] + (let [e2ee? (boolean (ldb/get-graph-rtc-e2ee? db)) + attrs (relevant-attrs e2ee?) + eids (->> (d/datoms db :eavt) + (keep (fn [datom] + (when (contains? attrs (:a datom)) + (:e datom)))) + distinct) + blocks (->> eids + (keep (fn [eid] + (let [{:keys [block/uuid block/title block/name block/parent block/page]} (entity-values db eid e2ee?)] + (when uuid + (cond-> {:block/uuid uuid + :block/parent parent + :block/page page} + (not e2ee?) (assoc :block/title title + :block/name name)))))) + (sort-by (comp str :block/uuid)) + vec)] + {:checksum (recompute-checksum db) + :e2ee? e2ee? + :attrs (->> attrs (sort-by str) vec) + :blocks blocks})) + (defn update-checksum [checksum {:keys [db-before db-after tx-data]}] - (let [db (or db-after db-before) - e2ee? (ldb/get-graph-rtc-e2ee? db) - changed-eids (->> tx-data (keep :e) distinct) - initial-state (if (string? checksum) - (checksum->state checksum) - (checksum->state (when db-before (recompute-checksum db-before))))] - (->> changed-eids - (reduce (fn [[sum-fnv sum-djb] eid] - (let [old-digest (when db-before (entity-digest db-before eid e2ee?)) - new-digest (when db-after (entity-digest db-after eid e2ee?)) - [sum-fnv sum-djb] (if old-digest - [(sub-step sum-fnv (first old-digest)) - (sub-step sum-djb (second old-digest))] - [sum-fnv sum-djb])] - (if new-digest - [(add-step sum-fnv (first new-digest)) - (add-step sum-djb (second new-digest))] - [sum-fnv sum-djb]))) - initial-state) - state->checksum))) + (let [before-e2ee? (ldb/get-graph-rtc-e2ee? db-before) + after-e2ee? (ldb/get-graph-rtc-e2ee? db-after)] + (if (not= before-e2ee? after-e2ee?) + ;; E2EE mode changes the global digest semantics, so incremental deltas are invalid. + (recompute-checksum db-after) + (let [changed-eids (->> tx-data + (remove (fn [d] + (contains? #{:block/tx-id} (:a d)))) + (keep (fn [d] + (let [e (:e d)] + (or (:block/uuid (d/entity db-before e)) + (:block/uuid (d/entity db-after e)))))) + distinct) + initial-state (if (valid-checksum? checksum) + (checksum->state checksum) + (checksum->state (recompute-checksum db-before)))] + (->> changed-eids + (reduce (fn [[sum-fnv sum-djb] uuid] + (let [old-digest (when-let [eid (:db/id (d/entity db-before [:block/uuid uuid]))] + (entity-digest db-before eid after-e2ee?)) + new-digest (when-let [eid (:db/id (d/entity db-after [:block/uuid uuid]))] + (entity-digest db-after eid after-e2ee?))] + [(cond-> sum-fnv + old-digest (sub-step (first old-digest)) + new-digest (add-step (first new-digest))) + (cond-> sum-djb + old-digest (sub-step (second old-digest)) + new-digest (add-step (second new-digest)))])) + initial-state) + state->checksum))))) diff --git a/deps/db-sync/src/logseq/db_sync/malli_schema.cljs b/deps/db-sync/src/logseq/db_sync/malli_schema.cljs index 43acacbc86..e15524fda9 100644 --- a/deps/db-sync/src/logseq/db_sync/malli_schema.cljs +++ b/deps/db-sync/src/logseq/db_sync/malli_schema.cljs @@ -138,7 +138,8 @@ (def graphs-list-response-schema [:map - [:graphs [:sequential graph-info-schema]]]) + [:graphs [:sequential graph-info-schema]] + [:user-rsa-keys-exists? :boolean]]) (def graph-create-request-schema [:map diff --git a/deps/db-sync/src/logseq/db_sync/storage.cljs b/deps/db-sync/src/logseq/db_sync/storage.cljs index 708d657b44..858d7b4667 100644 --- a/deps/db-sync/src/logseq/db_sync/storage.cljs +++ b/deps/db-sync/src/logseq/db_sync/storage.cljs @@ -144,13 +144,11 @@ (restore-data-from-addr sql addr)))) (defn- append-tx-for-tx-report - [sql {:keys [db-after db-before tx-data tx-meta]}] + [sql {:keys [db-after db-before tx-data tx-meta] :as tx-report}] (let [new-t (next-t! sql) created-at (common/now-ms) - checksum (sync-checksum/update-checksum (get-checksum sql) - {:db-before db-before - :db-after db-after - :tx-data tx-data}) + prev-checksum (get-checksum sql) + checksum (sync-checksum/update-checksum prev-checksum tx-report) normalized-data (->> tx-data (db-normalize/normalize-tx-data db-after db-before)) ;; _ (prn :debug :tx-data tx-data) diff --git a/deps/db-sync/src/logseq/db_sync/worker/handler/index.cljs b/deps/db-sync/src/logseq/db_sync/worker/handler/index.cljs index f13adce7f0..fd05c253fb 100644 --- a/deps/db-sync/src/logseq/db_sync/worker/handler/index.cljs +++ b/deps/db-sync/src/logseq/db_sync/worker/handler/index.cljs @@ -47,8 +47,14 @@ (case (:handler route) :graphs/list (if (string? user-id) - (p/let [graphs (index/ (d/empty-db db-schema/schema) - (d/db-with [{:db/id 1 - :block/uuid page-uuid - :block/name "page" - :block/title "Page"} - {:db/id 2 - :block/uuid block-uuid - :block/title "Block" - :block/parent 1 - :block/page 1}])) - db-with-unrelated (d/db-with base-db [[:db/add 2 :block/updated-at 1773661308002] - [:db/add 2 :logseq.property/created-by-ref 99]])] - (is (= (checksum/recompute-checksum base-db) - (checksum/recompute-checksum db-with-unrelated)))))) +(defn- sample-db + [] + (let [page-a-uuid (random-uuid) + page-b-uuid (random-uuid) + parent-uuid (random-uuid) + child-uuid (random-uuid)] + (-> (d/empty-db db-schema/schema) + (d/db-with [{:db/id 1 + :block/uuid page-a-uuid + :block/name "page-a" + :block/title "Page A"} + {:db/id 2 + :block/uuid page-b-uuid + :block/name "page-b" + :block/title "Page B"} + {:db/id 3 + :block/uuid parent-uuid + :block/title "Parent" + :block/parent 1 + :block/page 1} + {:db/id 4 + :block/uuid child-uuid + :block/title "Child" + :block/parent 3 + :block/page 1}])))) -(deftest incremental-checksum-matches-recompute-test - (testing "incremental checksum matches full recompute after a tx" - (let [page-uuid (random-uuid) - block-uuid (random-uuid) - db-before (-> (d/empty-db db-schema/schema) - (d/db-with [{:db/id 1 - :block/uuid page-uuid - :block/name "page" - :block/title "Page"} - {:db/id 2 - :block/uuid block-uuid - :block/title "Block" - :block/parent 1 - :block/page 1}])) - tx-report (d/with db-before [[:db/add 2 :block/title "Updated"] - [:db/add 1 :block/name "page-updated"]])] +(defn- assert-incremental=full! + [db-before checksum-before tx-data] + (let [tx-report (d/with db-before tx-data) + full (checksum/recompute-checksum (:db-after tx-report)) + incremental (checksum/update-checksum checksum-before tx-report)] + (is (= full incremental) + (str "Expected checksum parity for tx-data: " (pr-str tx-data))) + {:db (:db-after tx-report) + :checksum incremental})) + +(deftest checksum-ignores-unrelated-datoms-test + (testing "recompute and incremental checksums ignore unrelated datoms" + (let [db-before (sample-db) + checksum-before (checksum/recompute-checksum db-before) + tx-data [[:db/add 4 :block/updated-at 1773661308002] + [:db/add 4 :logseq.property/created-by-ref 99]] + tx-report (d/with db-before tx-data)] + (is (= checksum-before + (checksum/recompute-checksum (:db-after tx-report)))) + (is (= checksum-before + (checksum/update-checksum checksum-before tx-report)))))) + +(deftest incremental-checksum-matches-recompute-on-replace-datom-test + (testing "incremental checksum matches full recompute when replacing existing values" + (let [db-before (sample-db) + tx-report (d/with db-before [[:db/add 4 :block/title "Child updated"] + [:db/add 1 :block/name "page-a-updated"]])] (is (= (checksum/recompute-checksum (:db-after tx-report)) (checksum/update-checksum (checksum/recompute-checksum db-before) tx-report)))))) + +(deftest incremental-checksum-matches-recompute-across-mixed-mutations-test + (testing "incremental checksum stays equal to full recompute across typical tx sequences" + (let [db0 (sample-db) + new-block-uuid (random-uuid) + {:keys [db checksum]} (reduce + (fn [{:keys [db checksum]} {:keys [tx-data]}] + (assert-incremental=full! db checksum tx-data)) + {:db db0 + :checksum (checksum/recompute-checksum db0)} + [{:tx-data [[:db/add 4 :block/title "Child edited"]]} + {:tx-data [[:db/add 1 :block/name "page-a-renamed"] + [:db/add 1 :block/title "Page A Renamed"]]} + {:tx-data [[:db/add 4 :block/parent 2] + [:db/add 4 :block/page 2]]} + {:tx-data [[:db/add -1 :block/uuid new-block-uuid] + [:db/add -1 :block/title "New block"] + [:db/add -1 :block/parent 2] + [:db/add -1 :block/page 2]]} + {:tx-data [[:db/retract 3 :block/title "Parent"]]} + {:tx-data [[:db/retractEntity [:block/uuid new-block-uuid]]]} + {:tx-data [[:db/add 4 :block/updated-at 1773661308002]]}])] + (is (= checksum (checksum/recompute-checksum db)))))) + +(deftest incremental-checksum-uses-recompute-when-initial-checksum-missing-test + (testing "nil initial checksum uses db-before recompute as baseline" + (let [db-before (sample-db) + tx-report (d/with db-before [[:db/add 4 :block/title "Child updated"]])] + (is (= (checksum/recompute-checksum (:db-after tx-report)) + (checksum/update-checksum nil tx-report)))))) + +(deftest checksum-e2ee-ignores-title-and-name-test + (testing "with E2EE enabled, checksum ignores title/name changes for both modes" + (let [db-before (-> (sample-db) + (d/db-with [{:db/ident :logseq.kv/graph-rtc-e2ee? + :kv/value true}])) + checksum-before (checksum/recompute-checksum db-before) + tx-report (d/with db-before [[:db/add 4 :block/title "Encrypted title update"] + [:db/add 1 :block/name "encrypted-name-update"]])] + (is (= checksum-before + (checksum/recompute-checksum (:db-after tx-report)))) + (is (= checksum-before + (checksum/update-checksum checksum-before tx-report)))))) + +(deftest incremental-checksum-recomputes-when-e2ee-mode-toggles-test + (testing "incremental checksum falls back to full recompute when E2EE mode changes" + (let [db-before (sample-db) + tx-report (d/with db-before [{:db/ident :logseq.kv/graph-rtc-e2ee? + :kv/value true}])] + (is (= (checksum/recompute-checksum (:db-after tx-report)) + (checksum/update-checksum (checksum/recompute-checksum db-before) tx-report)))))) + +(deftest recompute-checksum-diagnostics-includes-relevant-attrs-test + (testing "diagnostics includes checksum attrs and block values used for checksum export" + (let [db (sample-db) + {:keys [checksum attrs blocks e2ee?]} (checksum/recompute-checksum-diagnostics db) + child-uuid (:block/uuid (d/entity db 4)) + child-parent-uuid (:block/uuid (:block/parent (d/entity db 4))) + child-page-uuid (:block/uuid (:block/page (d/entity db 4))) + child (some #(when (= child-uuid (:block/uuid %)) %) blocks)] + (is (false? e2ee?)) + (is (= (checksum/recompute-checksum db) checksum)) + (is (= #{:block/uuid :block/title :block/name :block/parent :block/page} + (set attrs))) + (is (= 4 (count blocks))) + (is (= child-parent-uuid (:block/parent child))) + (is (= child-page-uuid (:block/page child))) + (is (string? (:block/title child)))))) + +(deftest recompute-checksum-diagnostics-omits-title-and-name-in-e2ee-test + (testing "diagnostics for E2EE graphs omits title/name from checksum attrs and export blocks" + (let [db (-> (sample-db) + (d/db-with [{:db/ident :logseq.kv/graph-rtc-e2ee? + :kv/value true}])) + {:keys [checksum attrs blocks e2ee?]} (checksum/recompute-checksum-diagnostics db)] + (is e2ee?) + (is (= (checksum/recompute-checksum db) checksum)) + (is (= #{:block/uuid :block/parent :block/page} + (set attrs))) + (is (every? #(not (contains? % :block/title)) blocks)) + (is (every? #(not (contains? % :block/name)) blocks))))) diff --git a/deps/db-sync/test/logseq/db_sync/worker_handler_index_test.cljs b/deps/db-sync/test/logseq/db_sync/worker_handler_index_test.cljs index ceff04691c..59c1d4fa15 100644 --- a/deps/db-sync/test/logseq/db_sync/worker_handler_index_test.cljs +++ b/deps/db-sync/test/logseq/db_sync/worker_handler_index_test.cljs @@ -62,6 +62,9 @@ (-> (p/with-redefs [common/read-json (fn [_] (p/resolved #js {"graph-name" "Graph 1" "schema-version" "65"})) + index/ (p/with-redefs [index/clj (js/JSON.parse text) :keywordize-keys true)] + (is (= 200 (.-status resp))) + (is (= [] (:graphs body))) + (is (= true (:user-rsa-keys-exists? body))))) + (p/then (fn [] + (done))) + (p/catch (fn [error] + (is false (str error)) + (done))))))) + +(deftest graphs-list-includes-user-rsa-keys-exists-flag-false-test + (async done + (let [request (js/Request. "http://localhost/graphs" #js {:method "GET"}) + url (js/URL. (.-url request))] + (-> (p/with-redefs [index/clj (js/JSON.parse text) :keywordize-keys true)] + (is (= 200 (.-status resp))) + (is (= [] (:graphs body))) + (is (= false (:user-rsa-keys-exists? body))))) + (p/then (fn [] + (done))) + (p/catch (fn [error] + (is false (str error)) + (done))))))) + +(deftest graphs-create-e2ee-requires-user-rsa-key-pair-test + (async done + (let [request (js/Request. "http://localhost/graphs" #js {:method "POST"}) + url (js/URL. (.-url request)) + index-upsert-calls* (atom 0)] + (-> (p/with-redefs [common/read-json (fn [_] + (p/resolved #js {"graph-name" "Graph E2EE" + "schema-version" "65" + "graph-e2ee?" true})) + index/clj (js/JSON.parse text) :keywordize-keys true)] + (is (= 400 (.-status resp))) + (is (= "missing user rsa key pair" (:error body))) + (is (zero? @index-upsert-calls*)))) + (p/then (fn [] + (done))) + (p/catch (fn [error] + (is false (str error)) + (done))))))) + +(deftest graphs-create-non-e2ee-requires-user-rsa-key-pair-test + (async done + (let [request (js/Request. "http://localhost/graphs" #js {:method "POST"}) + url (js/URL. (.-url request)) + index-upsert-calls* (atom 0)] + (-> (p/with-redefs [common/read-json (fn [_] + (p/resolved #js {"graph-name" "Graph Plain" + "schema-version" "65" + "graph-e2ee?" false})) + index/clj (js/JSON.parse text) :keywordize-keys true)] + (is (= 400 (.-status resp))) + (is (= "missing user rsa key pair" (:error body))) + (is (zero? @index-upsert-calls*)))) + (p/then (fn [] + (done))) + (p/catch (fn [error] + (is false (str error)) + (done))))))) diff --git a/deps/db-sync/test/logseq/db_sync/worker_handler_sync_test.cljs b/deps/db-sync/test/logseq/db_sync/worker_handler_sync_test.cljs index 35368ae967..2863077c33 100644 --- a/deps/db-sync/test/logseq/db_sync/worker_handler_sync_test.cljs +++ b/deps/db-sync/test/logseq/db_sync/worker_handler_sync_test.cljs @@ -226,6 +226,25 @@ (is false (str error)) (done))))))) +(deftest current-checksum-heals-stale-stored-checksum-test + (testing "server recomputes and persists checksum when stored checksum is stale" + (let [sql (test-sql/make-sql) + conn (storage/open-conn sql) + self #js {:sql sql + :conn conn + :schema-ready true} + stale-checksum "0000000000000000" + block-uuid (random-uuid)] + (d/transact! conn [{:block/uuid block-uuid + :block/title "hello"}]) + (is (string? (storage/get-checksum sql))) + (storage/set-checksum! sql stale-checksum) + (let [healed (sync-handler/current-checksum self)] + (is (string? healed)) + (is (not= stale-checksum healed)) + (is (= healed (storage/get-checksum sql))) + (is (= healed (sync-handler/current-checksum self))))))) + (deftest tx-batch-rejects-with-the-exact-failed-tx-entry-test (testing "db transact failure replies with the specific rejected tx entry" (let [sql (test-sql/make-sql) diff --git a/deps/db-sync/worker/scripts/delete_graphs_for_user.js b/deps/db-sync/worker/scripts/delete_graphs_for_user.js index 8d94a28879..bf593f95c3 100644 --- a/deps/db-sync/worker/scripts/delete_graphs_for_user.js +++ b/deps/db-sync/worker/scripts/delete_graphs_for_user.js @@ -16,6 +16,33 @@ const { runWranglerQuery, } = require("./graph_user_lib"); +function escapeSqlValue(value) { + return value.replaceAll("'", "''"); +} + +function ensureMutationSuccess(output, context) { + if (!Array.isArray(output) || output.length === 0) { + fail(`Unexpected empty response from wrangler while ${context}.`); + } + + output.forEach((statement, index) => { + if (!statement.success) { + fail(`Wrangler mutation failed while ${context} (statement ${index + 1}).`); + } + }); +} + +function deleteGraphAesKeys(options, graphId) { + const sql = `delete from graph_aes_keys where graph_id = '${escapeSqlValue(graphId)}'`; + const wranglerArgs = buildWranglerArgs({ + database: options.database, + config: options.config, + env: options.env, + sql, + }); + ensureMutationSuccess(runWranglerQuery(wranglerArgs), `deleting graph_aes_keys for ${graphId}`); +} + function printHelp() { console.log(`Delete db-sync graphs owned by a user from a remote D1 environment. @@ -134,6 +161,8 @@ async function main() { const body = await response.text(); fail(`Delete failed for ${graph.graph_id}: ${response.status} ${body}`); } + + deleteGraphAesKeys(options, graph.graph_id); } console.log(`Deleted ${result.graphs.length} owned graph(s).`); diff --git a/deps/db-sync/worker/scripts/delete_user_totally.js b/deps/db-sync/worker/scripts/delete_user_totally.js new file mode 100644 index 0000000000..3193f3c37e --- /dev/null +++ b/deps/db-sync/worker/scripts/delete_user_totally.js @@ -0,0 +1,254 @@ +#!/usr/bin/env node + +const path = require("node:path"); +const readline = require("node:readline/promises"); +const { stdin, stdout } = require("node:process"); +const { parseArgs } = require("node:util"); +const { + buildAdminGraphDeleteUrl, + buildUserGraphsSql, + buildWranglerArgs, + defaultConfigPath, + fail, + formatUserGraphsResult, + parseWranglerResults, + printUserGraphsTable, + runWranglerQuery, +} = require("./graph_user_lib"); + +function printHelp() { + console.log(`Delete a db-sync user and all related data from a remote D1 environment. + +Usage: + node worker/scripts/delete_user_totally.js --username [--env prod] + node worker/scripts/delete_user_totally.js --user-id [--env prod] + +Options: + --username Look up the target user by username. + --user-id Look up the target user by user id. + --env Wrangler environment to use. Defaults to "prod". + --database D1 binding or database name. Defaults to "DB". + --config Wrangler config path. Defaults to worker/wrangler.toml. + --base-url Worker base URL. Defaults to DB_SYNC_BASE_URL. + --admin-token Admin delete token. Defaults to DB_SYNC_ADMIN_TOKEN. + --help Show this message. +`); +} + +function parseCliArgs(argv) { + const { values } = parseArgs({ + args: argv, + options: { + username: { type: "string" }, + "user-id": { type: "string" }, + env: { type: "string", default: "prod" }, + database: { type: "string", default: "DB" }, + config: { type: "string", default: defaultConfigPath }, + "base-url": { type: "string", default: process.env.DB_SYNC_BASE_URL }, + "admin-token": { type: "string", default: process.env.DB_SYNC_ADMIN_TOKEN }, + help: { type: "boolean", default: false }, + }, + strict: true, + allowPositionals: false, + }); + + if (values.help) { + printHelp(); + process.exit(0); + } + + const lookupCount = Number(Boolean(values.username)) + Number(Boolean(values["user-id"])); + if (lookupCount !== 1) { + fail("Pass exactly one of --username or --user-id."); + } + + return { + lookupField: values.username ? "username" : "id", + lookupLabel: values.username ? "username" : "user-id", + lookupValue: values.username ?? values["user-id"], + env: values.env, + database: values.database, + config: path.resolve(values.config), + baseUrl: values["base-url"], + adminToken: values["admin-token"], + }; +} + +function escapeSqlValue(value) { + return value.replaceAll("'", "''"); +} + +function runSelectQuery(options, sql) { + const wranglerArgs = buildWranglerArgs({ + database: options.database, + config: options.config, + env: options.env, + sql, + }); + + return parseWranglerResults(runWranglerQuery(wranglerArgs)); +} + +function runMutationQuery(options, sql) { + const wranglerArgs = buildWranglerArgs({ + database: options.database, + config: options.config, + env: options.env, + sql, + }); + + const output = runWranglerQuery(wranglerArgs); + if (!Array.isArray(output) || output.length === 0) { + throw new Error("Unexpected empty response from wrangler."); + } + + output.forEach((statement, index) => { + if (!statement.success) { + throw new Error(`Wrangler reported an unsuccessful mutation (statement ${index + 1}).`); + } + }); + + return output.reduce((sum, statement) => sum + Number(statement?.meta?.changes ?? 0), 0); +} + +function sqlCountToNumber(value) { + const numericValue = Number(value); + return Number.isFinite(numericValue) ? numericValue : 0; +} + +function isDeleteConfirmationAccepted(answer, userId) { + const normalizedAnswer = answer.trim(); + return normalizedAnswer === "DELETE" || normalizedAnswer === `DELETE USER ${userId}`; +} + +async function confirmDeletion({ user, ownedGraphsCount, memberGraphsCount }) { + const rl = readline.createInterface({ input: stdin, output: stdout }); + try { + const answer = await rl.question( + `Type DELETE to permanently delete this user (${user.user_id}; ${ownedGraphsCount} owned graph(s), ${memberGraphsCount} membership(s)): `, + ); + return isDeleteConfirmationAccepted(answer, user.user_id); + } finally { + rl.close(); + } +} + +async function deleteOwnedGraphs(options, ownedGraphs) { + for (const graph of ownedGraphs) { + const response = await fetch(buildAdminGraphDeleteUrl(options.baseUrl, graph.graph_id), { + method: "DELETE", + headers: { + "x-db-sync-admin-token": options.adminToken, + }, + }); + + if (!response.ok) { + const payload = await response.text(); + fail(`Delete failed for owned graph ${graph.graph_id}: ${response.status} ${payload}`); + } + } +} + +async function main() { + const options = parseCliArgs(process.argv.slice(2)); + const graphRows = runSelectQuery(options, buildUserGraphsSql({ ...options, ownedOnly: false })); + const result = formatUserGraphsResult(graphRows); + + if (!result) { + fail(`No user found for ${options.lookupLabel}=${options.lookupValue}.`); + } + + const ownedGraphs = result.graphs.filter((graph) => graph.access_role === "owner"); + const memberGraphs = result.graphs.filter((graph) => graph.access_role !== "owner"); + + printUserGraphsTable(result, "Graphs linked to user"); + console.log(`Owned graphs: ${ownedGraphs.length}`); + console.log(`Member graphs: ${memberGraphs.length}`); + + if (ownedGraphs.length > 0 && !options.baseUrl) { + fail("Missing worker base URL. Pass --base-url or set DB_SYNC_BASE_URL."); + } + + if (ownedGraphs.length > 0 && !options.adminToken) { + fail("Missing admin token. Pass --admin-token or set DB_SYNC_ADMIN_TOKEN."); + } + + const confirmed = await confirmDeletion({ + user: result.user, + ownedGraphsCount: ownedGraphs.length, + memberGraphsCount: memberGraphs.length, + }); + + if (!confirmed) { + console.log("Aborted."); + return; + } + + if (ownedGraphs.length > 0) { + await deleteOwnedGraphs(options, ownedGraphs); + } + + const escapedUserId = escapeSqlValue(result.user.user_id); + const remainingOwnedGraphRows = runSelectQuery( + options, + `select count(1) as owned_graph_count from graphs where user_id = '${escapedUserId}'`, + ); + const remainingOwnedGraphCount = sqlCountToNumber(remainingOwnedGraphRows[0]?.owned_graph_count); + if (remainingOwnedGraphCount > 0) { + fail( + `Owned graph cleanup incomplete: ${remainingOwnedGraphCount} graph(s) still owned by ${result.user.user_id}.`, + ); + } + + const deletedGraphAesKeys = runMutationQuery( + options, + `delete from graph_aes_keys where user_id = '${escapedUserId}'`, + ); + const deletedGraphMembers = runMutationQuery( + options, + `delete from graph_members where user_id = '${escapedUserId}'`, + ); + const clearedInvitedBy = runMutationQuery( + options, + `update graph_members set invited_by = null where invited_by = '${escapedUserId}'`, + ); + const deletedUserRsaKeys = runMutationQuery( + options, + `delete from user_rsa_keys where user_id = '${escapedUserId}'`, + ); + const deletedUsers = runMutationQuery(options, `delete from users where id = '${escapedUserId}'`); + + if (deletedUsers !== 1) { + fail(`Expected to delete exactly one user row, but deleted ${deletedUsers}.`); + } + + const userRowsAfterDelete = runSelectQuery( + options, + `select id from users where id = '${escapedUserId}' limit 1`, + ); + if (userRowsAfterDelete.length > 0) { + fail(`User ${result.user.user_id} still exists after deletion.`); + } + + console.table([ + { step: "owned graphs deleted", rows: ownedGraphs.length }, + { step: "graph_aes_keys deleted", rows: deletedGraphAesKeys }, + { step: "graph_members deleted", rows: deletedGraphMembers }, + { step: "graph_members invited_by cleared", rows: clearedInvitedBy }, + { step: "user_rsa_keys deleted", rows: deletedUserRsaKeys }, + { step: "users deleted", rows: deletedUsers }, + ]); + console.log(`Deleted user ${result.user.user_id} successfully.`); +} + +if (require.main === module) { + main().catch((error) => { + fail(error instanceof Error ? error.message : String(error)); + }); +} + +module.exports = { + confirmDeletion, + isDeleteConfirmationAccepted, + parseCliArgs, +}; diff --git a/deps/db-sync/worker/scripts/delete_user_totally.test.js b/deps/db-sync/worker/scripts/delete_user_totally.test.js new file mode 100644 index 0000000000..9da4958581 --- /dev/null +++ b/deps/db-sync/worker/scripts/delete_user_totally.test.js @@ -0,0 +1,59 @@ +const assert = require("node:assert/strict"); +const { spawnSync } = require("node:child_process"); +const path = require("node:path"); +const test = require("node:test"); + +const { isDeleteConfirmationAccepted, parseCliArgs } = require("./delete_user_totally"); +const { defaultConfigPath } = require("./graph_user_lib"); + +function runCli(args) { + return spawnSync(process.execPath, [path.join(__dirname, "delete_user_totally.js"), ...args], { + encoding: "utf8", + }); +} + +test("parseCliArgs accepts --username", () => { + const parsed = parseCliArgs(["--username", "alice"]); + + assert.equal(parsed.lookupField, "username"); + assert.equal(parsed.lookupLabel, "username"); + assert.equal(parsed.lookupValue, "alice"); + assert.equal(parsed.env, "prod"); + assert.equal(parsed.database, "DB"); + assert.equal(parsed.config, path.resolve(defaultConfigPath)); +}); + +test("parseCliArgs accepts --user-id", () => { + const parsed = parseCliArgs(["--user-id", "user-123"]); + + assert.equal(parsed.lookupField, "id"); + assert.equal(parsed.lookupLabel, "user-id"); + assert.equal(parsed.lookupValue, "user-123"); +}); + +test("CLI --help exits successfully", () => { + const result = runCli(["--help"]); + + assert.equal(result.status, 0); + assert.match(result.stdout, /Delete a db-sync user and all related data/); +}); + +test("CLI rejects passing both --username and --user-id", () => { + const result = runCli(["--username", "alice", "--user-id", "user-123"]); + + assert.equal(result.status, 1); + assert.match(result.stderr, /Pass exactly one of --username or --user-id\./); +}); + +test("confirmation accepts DELETE", () => { + assert.equal(isDeleteConfirmationAccepted("DELETE", "user-123"), true); +}); + +test("confirmation accepts legacy DELETE USER ", () => { + assert.equal(isDeleteConfirmationAccepted("DELETE USER user-123", "user-123"), true); +}); + +test("confirmation rejects unrelated input", () => { + assert.equal(isDeleteConfirmationAccepted("DELETE USER other-user", "user-123"), false); + assert.equal(isDeleteConfirmationAccepted("yes", "user-123"), false); +}); diff --git a/deps/db/src/logseq/db.cljs b/deps/db/src/logseq/db.cljs index eefd039ae6..1fa33dccaa 100644 --- a/deps/db/src/logseq/db.cljs +++ b/deps/db/src/logseq/db.cljs @@ -5,6 +5,7 @@ [clojure.walk :as walk] [datascript.conn :as dc] [datascript.core :as d] + [datascript.storage :as storage] [datascript.impl.entity :as de] [logseq.common.config :as common-config] [logseq.common.plural :as common-plural] @@ -13,13 +14,13 @@ [logseq.db.common.delete-blocks :as delete-blocks] ;; Load entity extensions [logseq.db.common.entity-plus :as entity-plus] [logseq.db.common.initial-data :as common-initial-data] - [logseq.db.common.normalize :as db-normalize] [logseq.db.frontend.class :as db-class] [logseq.db.frontend.db :as db-db] [logseq.db.frontend.entity-util :as entity-util] [logseq.db.frontend.property :as db-property] [logseq.db.frontend.validate :as db-validate] - [logseq.db.sqlite.util :as sqlite-util]) + [logseq.db.sqlite.util :as sqlite-util] + [logseq.common.log :as log]) (:refer-clojure :exclude [object?])) (def built-in? entity-util/built-in?) @@ -33,6 +34,7 @@ (defonce *transact-fn (atom nil)) (defonce *transact-invalid-callback (atom nil)) (defonce *transact-pipeline-fn (atom nil)) +(defonce *debounce-fn (atom nil)) (defn register-transact-fn! [f] @@ -43,11 +45,15 @@ (defn register-transact-pipeline-fn! [f] (when f (reset! *transact-pipeline-fn f))) +(defn register-debounce-fn! + [f] + (when f (reset! *debounce-fn f))) (defn- remove-temp-block-data [tx-data] (let [remove-block-temp-f (fn [m] - (->> (remove (fn [[k _v]] (= "block.temp" (namespace k))) m) + (->> (remove (fn [[k _v]] + (= "block.temp" (namespace k))) m) (into {})))] (keep (fn [data] (cond @@ -81,15 +87,6 @@ f)) tx-data)) -(comment - (defn- skip-db-validate? - [datoms] - (every? - (fn [d] - (contains? #{:logseq.property/created-by-ref :block/refs :block/tx-id} - (:a d))) - datoms))) - (defn- throw-if-page-has-block-parent! [db tx-data] (when (some (fn [d] (and (:added d) @@ -99,6 +96,12 @@ (throw (ex-info "Page can't have block as parent" {:tx-data tx-data})))) +(defn debounced-store-db + [conn] + (when-some [_storage (storage/storage @conn)] + (let [f (or @*debounce-fn d/store)] + (f @conn)))) + (defn- transact-sync [conn tx-data tx-meta] (try @@ -106,10 +109,10 @@ db-based? (entity-plus/db-based-graph? db)] (if (and db-based? (not - (or (:batch-temp-conn? @conn) - (:rtc-download-graph? tx-meta) + (or (:rtc-download-graph? tx-meta) (:reset-conn! tx-meta) (:initial-db? tx-meta) + (:skip-validate-db? db) (:skip-validate-db? tx-meta false) (:logseq.graph-parser.exporter/new-graph? tx-meta)))) (let [tx-report* (d/with db tx-data tx-meta) @@ -119,11 +122,15 @@ [validate-result errors] (db-validate/validate-tx-report tx-report nil)] (cond validate-result - (when (and tx-report (seq (:tx-data tx-report))) + (when (and tx-report + (seq (:tx-data tx-report))) ;; perf enhancement: avoid repeated call on `d/with` (reset! conn (:db-after tx-report)) - (dc/store-after-transact! conn tx-report) - (dc/run-callbacks conn tx-report)) + (if (:batch-tx? @conn) + (dc/run-callbacks conn tx-report) + (do + (debounced-store-db conn) + (dc/run-callbacks conn tx-report)))) :else (do @@ -182,33 +189,70 @@ (transact-fn repo-or-conn tx-data tx-meta) (transact-sync repo-or-conn tx-data tx-meta)))))) -(defn transact-with-temp-conn! +(defn batch-transact-with-temp-conn! "Validate db and store once for a batch transaction, the `temp` conn can still load data from disk, - however it can't write to the disk." + however it can't write to the disk. + This fn supports nested calls, however, don't rely on the tx-report for undo/redo." [conn tx-meta batch-tx-fn & {:keys [listen-db]}] (let [temp-conn (d/conn-from-db @conn) - *batch-tx-data (volatile! [])] + *batch-tx-data (volatile! []) + *complete? (volatile! false)] ;; can read from disk, write is disallowed (swap! temp-conn assoc :skip-store? true - :batch-temp-conn? true) + :batch-tx? true + :skip-validate-db? true) (d/listen! temp-conn ::temp-conn-batch-tx (fn [{:keys [tx-data] :as tx-report}] (vswap! *batch-tx-data into tx-data) (when (fn? listen-db) (listen-db tx-report)))) - (batch-tx-fn temp-conn *batch-tx-data) - (let [tx-data @*batch-tx-data - temp-after-db @temp-conn] - (d/unlisten! temp-conn ::temp-conn-batch-tx) - (reset! temp-conn nil) - (vreset! *batch-tx-data nil) - (when (seq tx-data) - ;; transact tx-data to `conn` and validate db - (let [tx-data' (->> - tx-data - (db-normalize/replace-attr-retract-with-retract-entity temp-after-db))] - (transact! conn tx-data' tx-meta)))))) + (try + (batch-tx-fn temp-conn *batch-tx-data) + (vreset! *complete? true) + (finally + (let [tx-data @*batch-tx-data] + (d/unlisten! temp-conn ::temp-conn-batch-tx) + (reset! temp-conn nil) + (vreset! *batch-tx-data nil) + (when (and @*complete? (seq tx-data)) + ;; transact tx-data to `conn` and validate db + (transact! conn tx-data tx-meta))))))) + +(defn batch-transact! + "Store once for a batch transaction, notice that this fn doesn't support nest `batch-transact` calls" + [conn tx-meta batch-tx-fn & {:keys [listen-db]}] + (let [db-before @conn + *tx-data (atom [])] + (try + (when (:batch-tx @conn) + (throw (ex-info "batch-transact! can't be nested called" {:tx-meta tx-meta}))) + (when (fn? listen-db) (d/listen! conn ::batch-tx + (fn [tx-report] + (swap! *tx-data into (:tx-data tx-report)) + (listen-db tx-report)))) + (swap! conn assoc :skip-store? true :batch-tx? true) + (batch-tx-fn conn) + (when (fn? listen-db) (d/unlisten! conn ::batch-tx)) + + (swap! conn dissoc :skip-store? :batch-tx?) + + (debounced-store-db conn) + + (let [batch-tx-data @*tx-data + _ (reset! *tx-data nil) + tx-report {:db-before db-before + :db-after @conn + :tx-meta tx-meta + :tx-data batch-tx-data}] + (dc/run-callbacks conn tx-report) + tx-report) + (catch :default e + (log/error e) + (reset! conn db-before) + (swap! conn dissoc :skip-store? :batch-tx?) + (reset! *tx-data nil) + (throw e))))) (def page? entity-util/page?) (def internal-page? entity-util/internal-page?) @@ -280,23 +324,73 @@ :else (:block/_parent parent))))) +(defn- get-right-sibling-for-property-children + [block parent] + (assert (or (de/entity? block) (nil? block))) + (let [children (get-block-children-or-property-children block parent) + right (some (fn [child] (when (> (compare (:block/order child) (:block/order block)) 0) child)) children)] + (when (not= (:db/id right) (:db/id block)) + right))) + (defn get-right-sibling [block] (assert (or (de/entity? block) (nil? block))) (when-let [parent (:block/parent block)] - (let [children (get-block-children-or-property-children block parent) - right (some (fn [child] (when (> (compare (:block/order child) (:block/order block)) 0) child)) children)] - (when (not= (:db/id right) (:db/id block)) - right)))) + (cond + (:block/closed-value-property block) + (get-right-sibling-for-property-children block parent) + + (:logseq.property/created-from-property block) + (get-right-sibling-for-property-children block parent) + + :else + (let [db (.-db block) + datoms (d/datoms db :avet :block/parent (:db/id parent)) + child-orders (->> (map (fn [d] + [(:e d) + (:v (first (d/datoms db :eavt (:e d) :block/order)))]) datoms) + (sort-by last)) + block-order (:block/order block)] + + (some (fn [[e child-order]] + (when (and (> (compare child-order block-order) 0) + (not (seq (d/datoms db :avet :logseq.property/created-from-property e))) + (not (seq (d/datoms db :avet :block/closed-value-property e)))) + (d/entity db e))) child-orders))))) + +(defn- get-left-sibling-for-property-children + [block parent] + (assert (or (de/entity? block) (nil? block))) + (let [children (reverse (get-block-children-or-property-children block parent)) + left (some (fn [child] (when (< (compare (:block/order child) (:block/order block)) 0) child)) children)] + (when (not= (:db/id left) (:db/id block)) + left))) (defn get-left-sibling [block] (assert (or (de/entity? block) (nil? block))) (when-let [parent (:block/parent block)] - (let [children (reverse (get-block-children-or-property-children block parent)) - left (some (fn [child] (when (< (compare (:block/order child) (:block/order block)) 0) child)) children)] - (when (not= (:db/id left) (:db/id block)) - left)))) + (cond + (:block/closed-value-property block) + (get-left-sibling-for-property-children block parent) + + (:logseq.property/created-from-property block) + (get-left-sibling-for-property-children block parent) + + :else + (let [db (.-db block) + datoms (d/datoms db :avet :block/parent (:db/id parent)) + child-orders (->> (map (fn [d] + [(:e d) + (:v (first (d/datoms db :eavt (:e d) :block/order)))]) datoms) + (sort-by last) + reverse) + block-order (:block/order block)] + (some (fn [[e child-order]] + (when (and (< (compare child-order block-order) 0) + (not (seq (d/datoms db :avet :logseq.property/created-from-property e))) + (not (seq (d/datoms db :avet :block/closed-value-property e)))) + (d/entity db e))) child-orders))))) (defn get-down [block] diff --git a/deps/db/test/logseq/db_test.cljs b/deps/db/test/logseq/db_test.cljs index 4d6f039b0c..2a64920a63 100644 --- a/deps/db/test/logseq/db_test.cljs +++ b/deps/db/test/logseq/db_test.cljs @@ -95,7 +95,7 @@ (d/datom 1 :property :v1 (+ tx 2) true)])) (is (= :v1 (:property (d/entity @conn 1))))))) -(deftest test-transact-with-temp-conn! +(deftest test-batch-transact! (testing "DB validation should be running after the whole transaction" (let [conn (db-test/create-conn)] (testing "#Task shouldn't be converted to property" @@ -104,10 +104,10 @@ (db-test/silence-stderr (ldb/transact! conn [{:db/ident :logseq.class/Task :block/tags :logseq.class/Property}])))))) - (ldb/transact-with-temp-conn! + (ldb/batch-transact-with-temp-conn! conn {} - (fn [temp-conn _*batch-tx-data] + (fn [temp-conn] (ldb/transact! temp-conn [{:db/ident :logseq.class/Task :block/tags :logseq.class/Property}]) (ldb/transact! temp-conn [[:db/retract :logseq.class/Task :block/tags :logseq.class/Property]])))))) diff --git a/deps/graph-parser/src/logseq/graph_parser/block.cljs b/deps/graph-parser/src/logseq/graph_parser/block.cljs index 8ca5d5b522..516127c282 100644 --- a/deps/graph-parser/src/logseq/graph_parser/block.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/block.cljs @@ -397,19 +397,19 @@ db-based? sanitize-hashtag-name) [page _page-entity] (cond - (and original-page-name (string? original-page-name)) - (page-name-string->map original-page-name db date-formatter - (assoc options :with-timestamp? with-timestamp?)) - :else - (let [page (cond (and (map? original-page-name) (:block/uuid original-page-name)) - original-page-name + (and original-page-name (string? original-page-name)) + (page-name-string->map original-page-name db date-formatter + (assoc options :with-timestamp? with-timestamp?)) + :else + (let [page (cond (and (map? original-page-name) (:block/uuid original-page-name)) + original-page-name - (map? original-page-name) - (assoc original-page-name :block/uuid (or page-uuid (d/squuid))) + (map? original-page-name) + (assoc original-page-name :block/uuid (or page-uuid (d/squuid))) - :else - nil)] - [page nil]))] + :else + nil)] + [page nil]))] (when page (if db-based? (let [tags (if class? [:logseq.class/Tag] @@ -880,4 +880,4 @@ [others parents' result'])))] (recur blocks parents result)))) result' (map (fn [block] (assoc block :block/order (db-order/gen-key))) result)] - (concat result' other-blocks))) \ No newline at end of file + (concat result' other-blocks))) diff --git a/deps/outliner/.carve/config.edn b/deps/outliner/.carve/config.edn index b3b8b2d4f2..84716ce275 100644 --- a/deps/outliner/.carve/config.edn +++ b/deps/outliner/.carve/config.edn @@ -6,5 +6,7 @@ logseq.outliner.core logseq.outliner.db-pipeline logseq.outliner.property + logseq.outliner.op.construct + logseq.outliner.recycle logseq.outliner.tree] :report {:format :ignore}} diff --git a/deps/outliner/deps.edn b/deps/outliner/deps.edn index 448d6e94a7..3f707cd699 100644 --- a/deps/outliner/deps.edn +++ b/deps/outliner/deps.edn @@ -8,7 +8,10 @@ ;; Any other deps should be added here and to nbb.edn logseq/db {:local/root "../db"} logseq/graph-parser {:local/root "../graph-parser"} - metosin/malli {:mvn/version "0.16.1"}} + metosin/malli {:mvn/version "0.16.1"} + ;; stubbed via logseq.common.log + com.lambdaisland/glogi {:git/url "https://github.com/lambdaisland/glogi" + :git/sha "30328a045141717aadbbb693465aed55f0904976"}} :aliases {:clj-kondo {:replace-deps {clj-kondo/clj-kondo {:mvn/version "2026.01.19"}} diff --git a/deps/outliner/src/logseq/outliner/core.cljs b/deps/outliner/src/logseq/outliner/core.cljs index 4200a4f84a..24e5bfb1d6 100644 --- a/deps/outliner/src/logseq/outliner/core.cljs +++ b/deps/outliner/src/logseq/outliner/core.cljs @@ -15,13 +15,41 @@ [logseq.db.sqlite.create-graph :as sqlite-create-graph] [logseq.outliner.datascript :as ds] [logseq.outliner.pipeline :as outliner-pipeline] - [logseq.outliner.recycle :as outliner-recycle] - [logseq.outliner.transaction :as outliner-tx] [logseq.outliner.tree :as otree] + [logseq.outliner.tx-meta :as outliner-tx-meta] [logseq.outliner.validate :as outliner-validate] [malli.core :as m] [malli.util :as mu])) +(defn- direct-op-entry + [outliner-op args] + (case outliner-op + :save-block + (let [[_conn block opts] args] + [:save-block [block opts]]) + + :insert-blocks + (let [[_conn blocks target-block opts] args] + [:insert-blocks [blocks (:db/id target-block) opts]]) + + :delete-blocks + (let [[_conn blocks opts] args] + [:delete-blocks [(mapv :db/id blocks) opts]]) + + :move-blocks + (let [[_conn blocks target-block opts] args] + [:move-blocks [(mapv :db/id blocks) (:db/id target-block) opts]]) + + :move-blocks-up-down + (let [[_conn blocks up?] args] + [:move-blocks-up-down [(mapv :db/id blocks) up?]]) + + :indent-outdent-blocks + (let [[_conn blocks indent? opts] args] + [:indent-outdent-blocks [(mapv :db/id blocks) indent? opts]]) + + nil)) + (def ^:private block-map (mu/optional-keys [:map @@ -134,11 +162,12 @@ ;; Update :block/tag to reference ids from :block/refs (map (fn [tag] (if (contains? refs (:block/name tag)) - (assoc tag :block/uuid - (:block/uuid - (first (filter (fn [r] (= (:block/name tag) + (let [matched-ref (first (filter (fn [r] (= (:block/name tag) (:block/name r))) - (:block/refs m))))) + (:block/refs m)))] + (cond-> (assoc tag :block/uuid (:block/uuid matched-ref)) + (:db/ident matched-ref) + (assoc :db/ident (:db/ident matched-ref)))) tag)) tags) @@ -463,15 +492,20 @@ ;;; ### insert-blocks, delete-blocks, move-blocks (defn- get-block-orders - [blocks target-block sibling? keep-block-order?] + [db blocks target-block sibling? keep-block-order? right-sibling-id] (if (and keep-block-order? (every? :block/order blocks)) (map :block/order blocks) (let [target-order (:block/order target-block) - next-sibling-order (:block/order (ldb/get-right-sibling target-block)) - first-child (ldb/get-down target-block) - first-child-order (:block/order first-child) start-order (when sibling? target-order) - end-order (if sibling? next-sibling-order first-child-order) + end-order (if sibling? + (let [right-sibling (when right-sibling-id + (d/entity db right-sibling-id))] + (if (= (:db/id (:block/parent right-sibling)) + (:db/id (:block/parent target-block))) + (:block/order right-sibling) + (:block/order (ldb/get-right-sibling target-block)))) + (let [first-child (ldb/get-down target-block)] + (:block/order first-child))) orders (db-order/gen-n-keys (count blocks) start-order end-order)] orders))) @@ -506,10 +540,10 @@ (:db/id target-block))) (defn- build-insert-blocks-tx - [db target-block blocks uuids get-new-id {:keys [sibling? outliner-op replace-empty-target? insert-template? keep-block-order?]}] + [db target-block blocks uuids get-new-id {:keys [sibling? outliner-op replace-empty-target? insert-template? keep-block-order? right-sibling-id]}] (let [block-ids (set (map :block/uuid blocks)) target-page (get-target-block-page target-block sibling?) - orders (get-block-orders blocks target-block sibling? keep-block-order?)] + orders (get-block-orders db blocks target-block sibling? keep-block-order? right-sibling-id)] (map-indexed (fn [idx {:block/keys [parent] :as block}] (when-let [uuid' (get uuids (:block/uuid block))] (let [block (remove-disallowed-inline-classes db block) @@ -683,7 +717,7 @@ ``" [db blocks target-block {:keys [_sibling? keep-uuid? keep-block-order? outliner-op outliner-real-op replace-empty-target? update-timestamps? - insert-template?] + insert-template? right-sibling-id] :as opts :or {update-timestamps? true}}] {:pre [(seq blocks) @@ -732,7 +766,8 @@ :keep-uuid? keep-uuid? :keep-block-order? keep-block-order? :outliner-op outliner-op - :insert-template? insert-template?} + :insert-template? insert-template? + :right-sibling-id right-sibling-id} {:keys [id->new-uuid blocks-tx]} (insert-blocks-aux db blocks' target-block insert-opts)] (if (some (fn [b] (or (nil? (:block/parent b)) (nil? (:block/order b)))) blocks-tx) (throw (ex-info "Invalid outliner data" @@ -797,15 +832,13 @@ (defn ^:api ^:large-vars/cleanup-todo delete-blocks "Delete blocks from the tree." - [db blocks opts] - (let [{:keys [hard-retract?]} opts - top-level-blocks (filter-top-level-blocks db blocks) + [db blocks _opts] + (let [top-level-blocks (filter-top-level-blocks db blocks) non-consecutive? (and (> (count top-level-blocks) 1) (seq (ldb/get-non-consecutive-blocks db top-level-blocks))) top-level-blocks* (get-top-level-blocks top-level-blocks non-consecutive?) - top-level-blocks (->> top-level-blocks* - (remove :logseq.property/built-in?) - (remove ldb/page?)) + top-level-blocks (remove :logseq.property/built-in? top-level-blocks*) txs-state (ds/new-outliner-txs-state) + block-ids (map (fn [b] [:block/uuid (:block/uuid b)]) top-level-blocks) start-block (first top-level-blocks) end-block (last top-level-blocks) delete-one-block? (or (= 1 (count top-level-blocks)) (= start-block end-block))] @@ -823,15 +856,6 @@ (:db/id (:logseq.property/default-value from-property))) (not (:block/closed-value-property start-block)))] (cond - hard-retract? - (let [block-ids (->> top-level-blocks - (mapcat (fn [block] - (map :db/id (ldb/get-block-and-children db (:block/uuid block) - {:include-property-block? true})))) - distinct) - tx-data (map (fn [id] [:db/retractEntity id]) block-ids)] - (when (seq tx-data) (swap! txs-state concat tx-data))) - (and delete-one-block? default-value-property?) (let [datoms (d/datoms db :avet (:db/ident from-property) (:db/id start-block)) tx-data (map (fn [d] {:db/id (:e d) @@ -839,8 +863,9 @@ (when (seq tx-data) (swap! txs-state concat tx-data))) :else - (swap! txs-state concat - (outliner-recycle/recycle-blocks-tx-data db top-level-blocks opts))))) + (doseq [id block-ids] + (let [node (d/entity db id)] + (otree/-del node txs-state db)))))) {:tx-data @txs-state})) (defn- move-to-original-position? @@ -924,20 +949,28 @@ (let [parents' (->> (ldb/get-block-parents db (:block/uuid target-block) {}) (map :db/id) (set)) - move-parents-to-child? (some parents' (map :db/id blocks))] + move-parents-to-child? (some parents' (map :db/id blocks)) + op-entry [:move-blocks [(mapv :db/id top-level-blocks) + (:db/id target-block) + opts]]] (when-not move-parents-to-child? - (outliner-tx/with-temp-conn-batch conn {:outliner-op :move-blocks} - (doseq [[idx block] (map vector (range (count blocks)) blocks)] - (let [first-block? (zero? idx) - sibling? (if first-block? sibling? true) - target-block (if first-block? target-block - (d/entity @conn (:db/id (nth blocks (dec idx))))) - block (d/entity @conn (:db/id block))] - (when-not (move-to-original-position? [block] target-block sibling? false) - (let [tx-data (move-block @conn block target-block sibling?)] - ;; (prn "==>> move blocks tx:" tx-data) - (ldb/transact! conn tx-data {:sibling? sibling? - :outliner-op (or outliner-op :move-blocks)})))))) + (ldb/batch-transact-with-temp-conn! + conn + {:outliner-op :move-blocks + :outliner-ops [op-entry]} + (fn [conn] + (doseq [[idx block] (map vector (range (count blocks)) blocks)] + (let [first-block? (zero? idx) + sibling? (if first-block? sibling? true) + target-block (if first-block? target-block + (d/entity @conn (:db/id (nth blocks (dec idx))))) + block (d/entity @conn (:db/id block))] + (when-not (move-to-original-position? [block] target-block sibling? false) + (let [tx-data (move-block @conn block target-block sibling?)] + ;; FIXME: move-blocks should be pure fn + ;; (prn "==>> move blocks tx:" tx-data) + (ldb/transact! conn tx-data {:sibling? sibling? + :outliner-op (or outliner-op :move-blocks)}))))))) nil))))) (defn- move-blocks-up-down @@ -1051,7 +1084,10 @@ (try (let [result (apply f args)] (when result - (let [tx-meta (assoc (:tx-meta result) + (let [tx-meta (outliner-tx-meta/ensure-outliner-ops + (:tx-meta result) + (direct-op-entry outliner-op args)) + tx-meta (assoc tx-meta :outliner-op outliner-op)] (ldb/transact! (first args) (:tx-data result) tx-meta))) result) diff --git a/deps/outliner/src/logseq/outliner/op.cljs b/deps/outliner/src/logseq/outliner/op.cljs index d78d545285..5c60cf2a9b 100644 --- a/deps/outliner/src/logseq/outliner/op.cljs +++ b/deps/outliner/src/logseq/outliner/op.cljs @@ -1,10 +1,12 @@ (ns logseq.outliner.op "Transact outliner ops" - (:require [datascript.core :as d] + (:require [clojure.string :as string] + [datascript.core :as d] [logseq.common.util :as common-util] [logseq.db :as ldb] [logseq.db.sqlite.export :as sqlite-export] [logseq.outliner.core :as outliner-core] + [logseq.outliner.page :as outliner-page] [logseq.outliner.property :as outliner-property] [logseq.outliner.transaction :as outliner-tx] [malli.core :as m])) @@ -20,6 +22,10 @@ [:catn [:op :keyword] [:args [:tuple ::blocks ::id ::option]]]] + [:apply-template + [:catn + [:op :keyword] + [:args [:tuple ::id ::id ::option]]]] [:delete-blocks [:catn [:op :keyword] @@ -152,8 +158,6 @@ (def ^:private ops-validator (m/validator ops-schema)) -(defonce ^:private *op-handlers (atom {})) - (defn- reaction-user-id [reaction] (:db/id (:logseq.property/created-by-ref reaction))) @@ -186,10 +190,6 @@ {:outliner-op :toggle-reaction}) true))))) -(defn register-op-handlers! - [handlers] - (reset! *op-handlers handlers)) - (defn- import-edn-data [conn *result export-map {:keys [tx-meta] :as import-options}] (let [{:keys [init-tx block-props-tx misc-tx error] :as _txs} @@ -207,6 +207,45 @@ (js/console.error "Unexpected Import EDN error:" e) (reset! *result {:error (str "Unexpected Import EDN error: " (pr-str (ex-message e)))})))))) +(defn- apply-insert-blocks-op! + [conn *result [blocks target-block-id opts]] + (when-let [target-block (d/entity @conn target-block-id)] + (let [result (outliner-core/insert-blocks! conn blocks target-block opts)] + (reset! *result result)))) + +(defn- template-children-blocks + [db template-id] + (when-let [template (d/entity db template-id)] + (let [template-blocks (some->> (ldb/get-block-and-children db (:block/uuid template) + {:include-property-block? true}) + rest)] + (when (seq template-blocks) + (cons (assoc (first template-blocks) + :logseq.property/used-template (:db/id template)) + (rest template-blocks)))))) + +(defn- apply-template-op! + [conn *result [template-id target-block-id opts]] + (when-let [target (d/entity @conn target-block-id)] + (let [blocks (template-children-blocks @conn template-id)] + (when (seq blocks) + (let [sibling? (:sibling? opts) + sibling?' (cond + (some? sibling?) + sibling? + + (seq (:block/_parent target)) + false + + :else + true) + result (outliner-core/insert-blocks! conn blocks target + (assoc opts + :sibling? sibling?' + :insert-template? true + :outliner-op :insert-template-blocks))] + (reset! *result result)))))) + (defn- ^:large-vars/cleanup-todo apply-op! [conn opts' *result [op args]] (case op @@ -215,10 +254,10 @@ (apply outliner-core/save-block! conn args) :insert-blocks - (let [[blocks target-block-id opts] args] - (when-let [target-block (d/entity @conn target-block-id)] - (let [result (outliner-core/insert-blocks! conn blocks target-block opts)] - (reset! *result result)))) + (apply-insert-blocks-op! conn *result args) + + :apply-template + (apply-template-op! conn *result args) :delete-blocks (let [[block-ids opts] args @@ -275,26 +314,76 @@ :class-remove-property (apply outliner-property/class-remove-property! conn args) - :upsert-closed-value + :upsert-closed-value ; don't support undo/redo (apply outliner-property/upsert-closed-value! conn args) - :delete-closed-value + :delete-closed-value ; don't support undo/redo (apply outliner-property/delete-closed-value! conn args) - :add-existing-values-to-closed-values + :add-existing-values-to-closed-values ; don't support undo/redo (apply outliner-property/add-existing-values-to-closed-values! conn args) - :batch-import-edn + :batch-import-edn ; don't support undo/redo (apply import-edn-data conn *result args) - :transact + :transact ; don't support undo/redo (apply ldb/transact! conn args) + :create-page + (let [[title options] args] + (reset! *result (outliner-page/create! conn title (or options {})))) + + :rename-page + (let [[page-uuid new-title] args] + (if (string/blank? new-title) + (throw (ex-info "Page name shouldn't be blank" {:block/uuid page-uuid + :block/title new-title})) + (outliner-core/save-block! conn + {:block/uuid page-uuid + :block/title new-title}))) + + :delete-page + (let [[page-uuid opts] args] + (outliner-page/delete! conn page-uuid (merge opts opts'))) + :toggle-reaction (reset! *result (apply toggle-reaction! conn args)) + nil)) - (when-let [handler (get @*op-handlers op)] - (reset! *result (handler conn args))))) +(defn- apply-single-op! + [conn ops *result opts' clean-tx-meta] + (let [db @conn + op (first ops) + result (case (ffirst ops) + :save-block + (apply outliner-core/save-block db (second op)) + :insert-blocks + (let [[blocks target-block-id insert-opts] (second op)] + (outliner-core/insert-blocks db blocks + (d/entity db target-block-id) + insert-opts)) + :delete-blocks + (let [[block-ids opts] (second op) + blocks (keep #(d/entity db %) block-ids)] + (outliner-core/delete-blocks db blocks (merge opts opts')))) + additional-tx (:additional-tx opts') + full-tx (concat (:tx-data result) additional-tx)] + (ldb/transact! conn full-tx clean-tx-meta) + (reset! *result result))) + +(defn- apply-save-followed-by-insert! + [conn ops *result opts' clean-tx-meta] + (let [save-block-tx (:tx-data (apply outliner-core/save-block @conn (second (first ops)))) + [blocks target-block-id insert-opts] (second (second ops)) + insert-blocks-result (outliner-core/insert-blocks @conn blocks + (d/entity @conn target-block-id) + insert-opts) + additional-tx (:additional-tx opts') + full-tx (concat save-block-tx + (:tx-data insert-blocks-result) + additional-tx)] + (ldb/transact! conn full-tx clean-tx-meta) + (reset! *result insert-blocks-result))) (defn apply-ops! [conn ops opts] @@ -303,14 +392,28 @@ (first (first ops))) opts' (cond-> (assoc opts :transact-opts {:conn conn} - :local-tx? true) + :local-tx? true + :outliner-ops ops + :db-sync/tx-id (or (:db-sync/tx-id opts) (random-uuid))) (and single-op-outliner-op (nil? (:outliner-op opts))) (assoc :outliner-op single-op-outliner-op)) - *result (atom nil)] - (outliner-tx/transact! - opts' - (doseq [op-entry ops] - (apply-op! conn opts' *result op-entry))) + *result (atom nil) + clean-tx-meta (dissoc opts' :additional-tx :transact-opts :current-block)] + (cond + (and single-op-outliner-op + (contains? #{:save-block :insert-blocks :delete-blocks} (ffirst ops))) + (apply-single-op! conn ops *result opts' clean-tx-meta) + + (and (= 2 (count ops)) + (= :save-block (ffirst ops)) + (= :insert-blocks (first (second ops)))) + (apply-save-followed-by-insert! conn ops *result opts' clean-tx-meta) + + :else + (outliner-tx/transact! + opts' + (doseq [op-entry ops] + (apply-op! conn opts' *result op-entry)))) @*result)) diff --git a/deps/outliner/src/logseq/outliner/op/construct.cljc b/deps/outliner/src/logseq/outliner/op/construct.cljc new file mode 100644 index 0000000000..3f64e31485 --- /dev/null +++ b/deps/outliner/src/logseq/outliner/op/construct.cljc @@ -0,0 +1,1045 @@ +(ns logseq.outliner.op.construct + "Construct canonical forward and reverse outliner ops for history actions." + (:require [clojure.string :as string] + [datascript.core :as d] + [logseq.common.util :as common-util] + [logseq.common.util.date-time :as date-time-util] + [logseq.common.uuid :as common-uuid] + [logseq.db :as ldb] + [logseq.db.frontend.content :as db-content] + [logseq.db.frontend.property :as db-property] + [logseq.db.frontend.property.type :as db-property-type])) + +(def ^:private semantic-outliner-ops + #{:save-block + :insert-blocks + :apply-template + :move-blocks + :move-blocks-up-down + :indent-outdent-blocks + :delete-blocks + :create-page + :rename-page + :delete-page + :restore-recycled + :set-block-property + :remove-block-property + :batch-set-property + :batch-remove-property + :delete-property-value + :batch-delete-property-value + :create-property-text-block + :upsert-property + :class-add-property + :class-remove-property + :upsert-closed-value + :add-existing-values-to-closed-values + :delete-closed-value}) + +(def ^:private transient-block-keys + #{:db/id + :block/tx-id + :block/created-at + :block/updated-at + :block/meta + :block/unordered + :block/level + :block.temp/ast-title + :block.temp/ast-body + :block.temp/load-status + :block.temp/has-children? + :logseq.property/created-by-ref + :logseq.property.embedding/hnsw-label-updated-at}) + +(def ^:api rebase-refs-key :block.temp/sync-rebase-refs) +(def ^:api rebase-created-refs-key :block.temp/sync-created-refs) +(def ^:api canonical-transact-op [[:transact nil]]) + +(defn- stable-entity-ref + [db x] + (cond + (map? x) (let [eid (or (:db/id x) + (when-let [id (:block/uuid x)] + (:db/id (d/entity db [:block/uuid id]))))] + (stable-entity-ref db eid)) + (and (integer? x) (not (neg? x))) + (if-let [ent (d/entity db x)] + (cond + (:block/uuid ent) [:block/uuid (:block/uuid ent)] + (:db/ident ent) (:db/ident ent) + :else x) + x) + :else x)) + +(defn- sanitize-ref-value + [db v] + (cond + (vector? v) (stable-entity-ref db v) + (or (set? v) (sequential? v)) (set (map #(stable-entity-ref db %) v)) + :else (stable-entity-ref db v))) + +(defn- sanitize-block-refs + [refs] + (->> refs + (keep (fn [ref-entity] + (when (:block/uuid ref-entity) + (select-keys ref-entity [:block/uuid :block/title :db/ident])))) + vec)) + +(defn- ref-attr? + [db a] + (and (keyword? a) + (= :db.type/ref + (:db/valueType (d/entity db a))))) + +(defn- sanitize-block-payload + ([db block] + (sanitize-block-payload db block nil)) + ([db block {:keys [created-uuids]}] + (if (map? block) + (let [refs (sanitize-block-refs (:block/refs block)) + created-ref-uuids (when (and (seq created-uuids) (seq refs)) + (->> refs + (keep :block/uuid) + (filter (set created-uuids)) + distinct + vec)) + m (reduce-kv + (fn [m k v] + (cond + (contains? transient-block-keys k) m + (= "block.temp" (namespace k)) m + (ref-attr? db k) + (assoc m k (sanitize-ref-value db v)) + :else + (assoc m k v))) + {} + block)] + (cond-> m + (seq refs) + (assoc rebase-refs-key refs) + + (seq created-ref-uuids) + (assoc rebase-created-refs-key created-ref-uuids))) + block))) + +(defn- get-missing-ref-by-lookup + [missing-refs tag-lookups] + (let [now (common-util/time-ms)] + (->> missing-refs + (keep (fn [{:block/keys [title] :as block :keys [db/ident]}] + (when-let [block-id (:block/uuid block)] + (let [lookup [:block/uuid block-id] + tag-ref? (contains? tag-lookups lookup) + entity (cond-> {:block/uuid block-id + :block/title (or title "") + :block/created-at now + :block/updated-at now + :block/tags (if tag-ref? :logseq.class/Tag :logseq.class/Page)} + (string? title) + (assoc :block/name (common-util/page-name-sanity-lc title)) + tag-ref? + (assoc :logseq.property.class/extends :logseq.class/Root) + ident + (assoc :db/ident ident))] + [lookup entity])))) + (into {})))) + +(defn rewrite-block-title-with-retracted-refs + [db block] + (let [refs (get block rebase-refs-key) + created-ref-uuids (set (get block rebase-created-refs-key)) + missing-refs (remove (fn [ref-entity] (d/entity db [:block/uuid (:block/uuid ref-entity)])) refs) + retracted-refs (remove (fn [block] + (contains? created-ref-uuids (:block/uuid block))) + missing-refs) + tag-lookups (->> (:block/tags block) + (filter (fn [v] + (and (vector? v) + (= :block/uuid (first v))))) + set) + missing-ref-by-lookup (get-missing-ref-by-lookup missing-refs tag-lookups) + rewrite-retracted-refs (fn [v] + (let [rewrite-ref (fn [block-ref] + (or (get missing-ref-by-lookup block-ref) + block-ref))] + (map rewrite-ref v))) + block' (cond-> block + (seq retracted-refs) + (update :block/title + (fn [title] + (-> title + (db-content/content-id-ref->page retracted-refs)))) + + (seq missing-ref-by-lookup) + (-> (update :block/refs rewrite-retracted-refs) + (update :block/tags rewrite-retracted-refs)))] + (dissoc block' rebase-refs-key rebase-created-refs-key))) + +(defn- sanitize-insert-block-payload + [db block] + (let [block' (sanitize-block-payload db block)] + (if (map? block') + (dissoc block' :block/page :block/order rebase-refs-key) + block'))) + +(defn- stable-id-coll + [db ids] + (mapv #(stable-entity-ref db %) ids)) + +(defn- resolve-move-target + [db ids] + (when-let [first-block (some->> ids first (d/entity db))] + (if-let [left-sibling (ldb/get-left-sibling first-block)] + [(:db/id left-sibling) true] + (when-let [parent (:block/parent first-block)] + [(:db/id parent) false])))) + +(defn- stable-property-value + [db property-id v] + (let [property-type (some-> (d/entity db property-id) :logseq.property/type)] + (if (contains? db-property-type/all-ref-property-types property-type) + (sanitize-ref-value db v) + v))) + +(defn- created-block-uuids-from-tx-data + [tx-data] + (->> tx-data + (keep (fn [item] + (cond + (and (map? item) (:block/uuid item)) + (:block/uuid item) + + (and (some? (:a item)) + (= :block/uuid (:a item)) + (true? (:added item))) + (:v item) + + (and (vector? item) + (= :db/add (first item)) + (>= (count item) 4) + (= :block/uuid (nth item 2))) + (nth item 3) + + :else + nil))) + distinct + vec)) + +(defn- created-page-uuid-from-tx-data + [tx-data title] + (or + (some (fn [item] + (when (and (map? item) + (= title (:block/title item)) + (:block/uuid item)) + (:block/uuid item))) + tx-data) + (let [grouped (group-by :e tx-data)] + (some (fn [[_ datoms]] + (let [title' (some (fn [datom] + (when (and (= :block/title (:a datom)) + (true? (:added datom))) + (:v datom))) + datoms) + uuid' (some (fn [datom] + (when (and (= :block/uuid (:a datom)) + (true? (:added datom))) + (:v datom))) + datoms)] + (when (and (= title title') (uuid? uuid')) + uuid'))) + grouped)))) + +(defn- created-db-ident-from-tx-data + [tx-data] + (or + (some (fn [item] + (when (and (map? item) + (qualified-keyword? (:db/ident item))) + (:db/ident item))) + tx-data) + (some (fn [item] + (when (and (map? item) + (= :db/ident (:a item)) + (qualified-keyword? (:v item))) + (:v item))) + tx-data) + (some (fn [item] + (when (and (vector? item) + (keyword? (nth item 1 nil)) + (= :db/ident (nth item 1 nil)) + (qualified-keyword? (nth item 2 nil))) + (nth item 2))) + tx-data) + (some (fn [item] + (when (and (vector? item) + (= :db/add (first item)) + (>= (count item) 4) + (= :db/ident (nth item 2)) + (qualified-keyword? (nth item 3))) + (nth item 3))) + tx-data))) + +(defn- property-ident-by-title + [db property-name] + (some-> (d/q '[:find ?ident . + :in $ ?title + :where + [?e :block/title ?title] + [?e :block/tags :logseq.class/Property] + [?e :db/ident ?ident]] + db + property-name) + (as-> ident + (when (qualified-keyword? ident) + ident)))) + +(defn- maybe-rewrite-delete-block-ids + [db tx-data ids] + (let [ids' (stable-id-coll db ids) + created-uuids (created-block-uuids-from-tx-data tx-data) + unresolved-created-lookups? (and (seq created-uuids) + (= (count ids') (count created-uuids)) + (every? (fn [id] + (and (vector? id) + (= :block/uuid (first id)) + (nil? (d/entity db id)))) + ids'))] + (if unresolved-created-lookups? + (mapv (fn [block-uuid] [:block/uuid block-uuid]) created-uuids) + ids'))) + +(defn- moved-block-ids-from-tx-data + [tx-data] + (->> tx-data + (keep (fn [[e a _v _t added?]] + (when (and (= :block/parent a) (true? added?)) + e))) + distinct + vec)) + +(defn- canonicalize-insert-blocks-op + [db tx-data args] + (let [[blocks target-id opts] args + created-uuids (created-block-uuids-from-tx-data tx-data) + blocks* (mapv #(sanitize-insert-block-payload db %) blocks) + target-ref (stable-entity-ref db target-id) + target (d/entity db target-id) + block-with-new-id (fn [block block-uuid] + (assoc block + :block/uuid block-uuid + :block/parent (let [parent (:block/parent (d/entity db [:block/uuid block-uuid]))] + [:block/uuid (:block/uuid parent)]))) + blocks* (if (seq created-uuids) + (if (and (:replace-empty-target? opts) + (= (inc (count created-uuids)) (count blocks))) + (let [[fst-block & rst-blocks] blocks* + created-rst-uuids created-uuids] + (into [(assoc fst-block :block/uuid (:block/uuid target))] + (if (seq created-rst-uuids) + (map block-with-new-id rst-blocks created-rst-uuids) + rst-blocks))) + (mapv block-with-new-id blocks* created-uuids)) + blocks)] + [blocks* + target-ref + (assoc (dissoc (or opts {}) :outliner-op) + :keep-uuid? true)])) + +(defn- canonical-move-op-for-block + [db block-id opts] + (when-let [[target-id sibling?] (resolve-move-target db [block-id])] + [:move-blocks [[(stable-entity-ref db block-id)] + (stable-entity-ref db target-id) + (assoc (dissoc (or opts {}) :outliner-op) + :sibling? sibling?)]])) + +(defn- canonicalize-indent-outdent-op + [db tx-data ids indent? opts] + (let [moved-ids (moved-block-ids-from-tx-data tx-data)] + (if (seq moved-ids) + (let [move-ops (->> moved-ids + (keep #(canonical-move-op-for-block db % opts)) + vec)] + (if (= (count moved-ids) (count move-ops)) + move-ops + [[:indent-outdent-blocks [(stable-id-coll db ids) + indent? + opts]]])) + [[:indent-outdent-blocks [(stable-id-coll db ids) + indent? + opts]]]))) + +(defn- ^:large-vars/cleanup-todo canonicalize-semantic-outliner-op + [db tx-data [op args]] + (case op + :save-block + (let [[block opts] args + created-uuids (created-block-uuids-from-tx-data tx-data)] + [:save-block [(sanitize-block-payload db block {:created-uuids created-uuids}) opts]]) + + :insert-blocks + [:insert-blocks + (canonicalize-insert-blocks-op db tx-data args)] + + :apply-template + (let [[template-id target-id opts] args + template-ref (stable-entity-ref db template-id) + target-ref (stable-entity-ref db target-id) + opts' (dissoc opts + :template-blocks + :template-id + :outliner-op)] + (when-not (and template-ref target-ref) + (throw (ex-info "Invalid apply-template args" + {:args args}))) + [:apply-template [template-ref target-ref opts']]) + + :move-blocks-up-down + (let [[ids up?] args] + [:move-blocks-up-down + [(stable-id-coll db ids) + up?]]) + + :indent-outdent-blocks + (let [[ids indent? opts] args] + (canonicalize-indent-outdent-op db tx-data ids indent? opts)) + + :move-blocks + (let [[ids target-id opts] args] + [:move-blocks [(stable-id-coll db ids) + (stable-entity-ref db target-id) + opts]]) + + :delete-blocks + (let [[ids opts] args] + [:delete-blocks [(maybe-rewrite-delete-block-ids db tx-data ids) opts]]) + + :create-page + (let [[title opts] args + page-uuid (created-page-uuid-from-tx-data tx-data title)] + [:create-page [title + (cond-> (or opts {}) + page-uuid + (assoc :uuid page-uuid))]]) + + :rename-page + (let [[page-uuid new-title] args] + [:save-block [{:block/uuid page-uuid + :block/title new-title} + {}]]) + + :delete-page + (let [[page-uuid opts] args] + [:delete-page [page-uuid opts]]) + + :restore-recycled + (let [[root-id] args] + [:restore-recycled [root-id]]) + + :set-block-property + (let [[block-eid property-id v] args] + [:set-block-property [(stable-entity-ref db block-eid) + property-id + (stable-property-value db property-id v)]]) + + :remove-block-property + (let [[block-eid property-id] args] + [:remove-block-property [(stable-entity-ref db block-eid) property-id]]) + + :batch-set-property + (let [[block-ids property-id v opts] args] + [:batch-set-property [(stable-id-coll db block-ids) + property-id + (stable-property-value db property-id v) + opts]]) + + :batch-remove-property + (let [[block-ids property-id] args] + [:batch-remove-property [(stable-id-coll db block-ids) property-id]]) + + :delete-property-value + (let [[block-eid property-id property-value] args] + [:delete-property-value [(stable-entity-ref db block-eid) + property-id + (stable-property-value db property-id property-value)]]) + + :batch-delete-property-value + (let [[block-eids property-id property-value] args] + [:batch-delete-property-value [(stable-id-coll db block-eids) + property-id + (stable-property-value db property-id property-value)]]) + + :create-property-text-block + (let [[block-id property-id value opts] args] + [:create-property-text-block [(stable-entity-ref db block-id) + (stable-entity-ref db property-id) + value + opts]]) + + :upsert-property + (let [[property-id schema opts] args + property-id' (or (stable-entity-ref db property-id) + (property-ident-by-title db (:property-name opts)) + (created-db-ident-from-tx-data tx-data))] + [:upsert-property [property-id' schema opts]]) + + :class-add-property + (let [[class-id property-id] args] + [:class-add-property [(stable-entity-ref db class-id) (stable-entity-ref db property-id)]]) + + :class-remove-property + (let [[class-id property-id] args] + [:class-remove-property [(stable-entity-ref db class-id) (stable-entity-ref db property-id)]]) + + :upsert-closed-value + (let [[property-id opts] args] + [:upsert-closed-value [property-id opts]]) + + :add-existing-values-to-closed-values + (let [[property-id values] args] + [:add-existing-values-to-closed-values [property-id values]]) + + :delete-closed-value + (let [[property-id value-block-id] args] + [:delete-closed-value [property-id (stable-entity-ref db value-block-id)]]) + + [op args])) + +(defn- save-block-keys + [block] + (->> (keys block) + (remove transient-block-keys) + (remove #(= :db/other-tx %)) + (remove nil?))) + +(defn- worker-ref-attr? + [db a] + (and (keyword? a) + (= :db.type/ref + (:db/valueType (d/entity db a))))) + +(defn- block-entity + [db block] + (cond + (map? block) + (or (when-let [block-uuid (:block/uuid block)] + (d/entity db [:block/uuid block-uuid])) + (when-let [db-id (:db/id block)] + (d/entity db db-id))) + + (integer? block) + (d/entity db block) + + (vector? block) + (d/entity db block) + + :else + nil)) + +(defn- build-inverse-save-block + [db-before block opts] + (when-let [before-ent (block-entity db-before block)] + (let [keys-to-restore (save-block-keys block) + inverse-block (reduce + (fn [m k] + (let [v (if (= :block/title k) + (:block/raw-title before-ent) + (get before-ent k))] + (assoc m k + (if (worker-ref-attr? db-before k) + (sanitize-ref-value db-before v) + v)))) + {:block/uuid (:block/uuid before-ent)} + keys-to-restore)] + [:save-block [inverse-block opts]]))) + +(defn- property-ref-value + [db property-id value] + (let [property-type (some-> (d/entity db property-id) :logseq.property/type)] + (if (contains? db-property-type/all-ref-property-types property-type) + (sanitize-ref-value db value) + value))) + +(defn- block-property-value + [db block-id property-id] + (when-let [value (some-> (d/entity db block-id) + (get property-id))] + (property-ref-value db property-id value))) + +(defn- inverse-property-op + [db-before op args] + (case op + :set-block-property + (let [[block-id property-id _value] args + before-value (block-property-value db-before block-id property-id) + block-ref (stable-entity-ref db-before block-id)] + (if (nil? before-value) + [:remove-block-property [block-ref property-id]] + [:set-block-property [block-ref property-id before-value]])) + + :remove-block-property + (let [[block-id property-id] args + before-value (block-property-value db-before block-id property-id) + block-ref (stable-entity-ref db-before block-id)] + (when (some? before-value) + [:set-block-property [block-ref property-id before-value]])) + + :batch-set-property + (let [[block-ids property-id _value _opts] args] + (->> block-ids + (keep (fn [block-id] + (let [before-value (block-property-value db-before block-id property-id) + block-ref (stable-entity-ref db-before block-id)] + (if (nil? before-value) + [:remove-block-property [block-ref property-id]] + [:set-block-property [block-ref property-id before-value]])))) + vec + seq)) + + :batch-remove-property + (let [[block-ids property-id _opts] args] + (->> block-ids + (keep (fn [block-id] + (let [before-value (block-property-value db-before block-id property-id) + block-ref (stable-entity-ref db-before block-id)] + (when (some? before-value) + [:set-block-property [block-ref property-id before-value]])))) + vec + seq)) + + nil)) + +(defn- build-insert-block-payload + [db-before ent] + (when-let [block-uuid (:block/uuid ent)] + (->> (save-block-keys ent) + (remove #(string/starts-with? (name %) "_")) + (reduce (fn [m k] + (let [v (get ent k)] + (assoc m k + (if (worker-ref-attr? db-before k) + (sanitize-ref-value db-before v) + v)))) + {:block/uuid block-uuid})))) + +(defn- selected-block-roots + [db-before ids] + (let [entities (reduce (fn [acc id] + (if-let [ent (d/entity db-before id)] + (if (some #(= (:db/id %) (:db/id ent)) acc) + acc + (conj acc ent)) + acc)) + [] + ids) + selected-ids (set (map :db/id entities)) + has-selected-ancestor? (fn [ent] + (loop [parent (:block/parent ent)] + (if-let [parent-id (some-> parent :db/id)] + (if (contains? selected-ids parent-id) + true + (recur (:block/parent parent))) + false)))] + (->> entities + (remove has-selected-ancestor?) + vec))) + +(defn- block-restore-target + [ent] + (if-let [left-sibling-id (:db/id (ldb/get-left-sibling ent))] + [left-sibling-id true] + (when-let [parent-id (or (:db/id (:block/parent ent)) + (:db/id (:block/page ent)))] + [parent-id false]))) + +(defn- to-insert-op + [db-before {:keys [blocks target-id sibling?]}] + [:insert-blocks [blocks + (stable-entity-ref db-before target-id) + {:sibling? (boolean sibling?) + :keep-uuid? true + :keep-block-order? true}]]) + +(defn- delete-root->restore-plan + [db-before root] + (let [root-id (:db/id root) + root-uuid (:block/uuid root) + blocks (when root-uuid + (->> (ldb/get-block-and-children db-before root-uuid) + (keep #(build-insert-block-payload db-before %)) + vec)) + [target-id sibling?] (block-restore-target root) + [target-id sibling?] (if (and target-id (= target-id root-id)) + [(or (:db/id (:block/parent root)) + (:db/id (:block/page root))) + false] + [target-id sibling?])] + (when (and (seq blocks) (some? target-id)) + {:blocks blocks + :target-id target-id + :sibling? sibling?}))) + +(defn- build-inverse-delete-blocks + [db-before ids] + (let [roots (selected-block-roots db-before ids) + plans (mapv #(delete-root->restore-plan db-before %) roots)] + (when (and (seq roots) + (every? some? plans)) + (->> plans + (mapv #(to-insert-op db-before %)) + seq)))) + +(defn- move-root->restore-op + [db-before root] + (let [root-id (:db/id root) + [target-id sibling?] (block-restore-target root) + parent-id (some-> root :block/parent :db/id) + page-id (some-> root :block/page :db/id) + fallback-target (or (when parent-id (stable-entity-ref db-before parent-id)) + (when page-id (stable-entity-ref db-before page-id)))] + (when (and (some? root-id) + (some? target-id)) + [:move-blocks + [[(stable-entity-ref db-before root-id)] + (stable-entity-ref db-before target-id) + (cond-> {:sibling? (boolean sibling?)} + sibling? + (assoc :fallback-target fallback-target))]]))) + +(defn- build-inverse-move-blocks + [db-before ids] + (let [roots (selected-block-roots db-before ids) + restore-ops (mapv #(move-root->restore-op db-before %) roots)] + (when (and (seq roots) + (every? some? restore-ops)) + (seq restore-ops)))) + +(defn- page-top-level-blocks + [page] + (let [page-id (:db/id page)] + (->> (:block/_page page) + (filter #(= page-id (some-> % :block/parent :db/id))) + ldb/sort-by-order + vec))) + +(defn- entity->save-op + [db-before ent] + (build-inverse-save-block db-before (into {} ent) nil)) + +(defn- build-inverse-delete-page + [db-before page-uuid] + (when-let [page (d/entity db-before [:block/uuid page-uuid])] + (let [class-or-property? (or (ldb/class? page) + (ldb/property? page)) + today-page? (when-let [day (:block/journal-day page)] + (= (date-time-util/ms->journal-day (common-util/time-ms)) day)) + root-plans (mapv #(delete-root->restore-plan db-before %) (page-top-level-blocks page))] + (cond + class-or-property? + (let [page-save-op (entity->save-op db-before (assoc (into {} page) :db/ident (:db/ident page))) + create-op (if (ldb/class? page) + (let [class-ident-namespace (some-> (:db/ident page) namespace)] + [:create-page + [(:block/title page) + (cond-> {:uuid page-uuid + :class? true + :redirect? false + :split-namespace? true} + class-ident-namespace + (assoc :class-ident-namespace class-ident-namespace))]]) + [:upsert-property + [(:db/ident page) + (db-property/get-property-schema (into {} page)) + {:property-name (:block/title page)}]]) + restore-root-ops (when (every? some? root-plans) + (mapv #(to-insert-op db-before %) root-plans))] + (cond-> [] + create-op + (conj create-op) + page-save-op + (conj page-save-op) + (seq restore-root-ops) + (into restore-root-ops) + :always + seq)) + + today-page? + (when (every? some? root-plans) + (->> root-plans + (mapv #(to-insert-op db-before %)) + seq)) + + :else + ;; Soft-deleted pages are moved to Recycle with recycle metadata. + ;; Use restore semantics instead of save-block to retract recycle markers. + [:restore-recycled [page-uuid]])))) + +(defn- restore-target-insert-op + [db-before db-after target-id opts] + (when (:replace-empty-target? opts) + (when-let [target-ref (stable-entity-ref db-before target-id)] + (when (d/entity db-after target-ref) + (when-let [target (d/entity db-before target-ref)] + [[:delete-blocks [[target-ref] {}]] + (build-inverse-save-block db-before target opts)]))))) + +(defn- build-inverse-insert-like + [db-before db-after tx-data args] + (let [[_blocks target-id opts] args + new-block-eids (keep + (fn [d] + (when (and (= :block/uuid (:a d)) + (:added d) + (nil? (d/entity db-before (:e d)))) + [:block/uuid (:v d)])) + tx-data) + restore-op (restore-target-insert-op db-before db-after target-id opts)] + (cond-> [] + (seq new-block-eids) + (conj [:delete-blocks [new-block-eids {}]]) + + restore-op + (into restore-op) + + :always + seq))) + +(defn- ^:large-vars/cleanup-todo build-strict-inverse-outliner-ops + [db-before db-after tx-data forward-ops] + (when (seq forward-ops) + (let [inverse-entries + (mapv (fn [[op args]] + (let [inverse-entry + (case op + :save-block + (let [[block opts] args] + (build-inverse-save-block db-before block opts)) + + :insert-blocks + (build-inverse-insert-like db-before db-after tx-data args) + + :apply-template + (build-inverse-insert-like db-before db-after tx-data args) + + :move-blocks + (let [[ids _target-id _opts] args] + (build-inverse-move-blocks db-before ids)) + + :indent-outdent-blocks + (let [[ids indent? opts] args] + [:indent-outdent-blocks [(stable-id-coll db-before ids) + (not indent?) + opts]]) + + :move-blocks-up-down + (let [[ids up?] args] + [:move-blocks-up-down + [(stable-id-coll db-before ids) + (not up?)]]) + + :delete-blocks + (let [[ids _opts] args] + (build-inverse-delete-blocks db-before ids)) + + :create-page + (let [[_title opts] args] + (when-let [page-uuid (:uuid opts)] + [:delete-page [page-uuid {}]])) + + :delete-page + (let [[page-uuid _opts] args] + (build-inverse-delete-page db-before page-uuid)) + + :set-block-property + (inverse-property-op db-before op args) + + :remove-block-property + (inverse-property-op db-before op args) + + :batch-set-property + (inverse-property-op db-before op args) + + :batch-remove-property + (inverse-property-op db-before op args) + + :create-property-text-block + (let [[_block-id _property-id _value opts] args + new-block-id (:new-block-id opts) + new-block-ref (cond + (vector? new-block-id) + new-block-id + + (uuid? new-block-id) + [:block/uuid new-block-id] + + :else + (stable-entity-ref db-before new-block-id))] + (when new-block-ref + [:delete-blocks [[new-block-ref] {}]])) + + :class-add-property + (let [[class-id property-id] args] + [:class-remove-property [(stable-entity-ref db-before class-id) + (stable-entity-ref db-before property-id)]]) + + :class-remove-property + (let [[class-id property-id] args] + [:class-add-property [(stable-entity-ref db-before class-id) + (stable-entity-ref db-before property-id)]]) + + :upsert-property + (let [[property-id _schema _opts] args] + (when (qualified-keyword? property-id) + [:delete-page [(common-uuid/gen-uuid :db-ident-block-uuid property-id) {}]])) + + nil)] + (if (and (sequential? inverse-entry) + (empty? inverse-entry)) + nil + inverse-entry))) + forward-ops)] + ;; Any missing inverse entry means the whole semantic inverse is incomplete. + ;; Use raw reversed tx instead of partially replaying. + (when (every? some? inverse-entries) + (some->> inverse-entries + (mapcat #(if (and (sequential? %) + (sequential? (first %))) + % + [%])) + vec + seq))))) + +(defn- has-replace-empty-target-insert-op? + [forward-ops] + (some (fn [[op [_blocks _target-id opts]]] + (and (contains? #{:insert-blocks :apply-template} op) + (:replace-empty-target? opts))) + forward-ops)) + +(defn contains-transact-op? + [ops] + (some (fn [[op]] + (= :transact op)) + ops)) + +(defn- canonicalize-explicit-outliner-ops + [db tx-data ops] + (cond + (nil? ops) + nil + + (seq ops) + (do + (when-not (every? (fn [[op]] + (contains? semantic-outliner-ops op)) + ops) + (throw (ex-info "Not every op is semantic" {:ops ops}))) + (->> ops + (mapcat (fn [op] + (let [canonicalized-op (canonicalize-semantic-outliner-op db tx-data op)] + (if (and (sequential? canonicalized-op) + (sequential? (first canonicalized-op)) + (keyword? (ffirst canonicalized-op))) + canonicalized-op + [canonicalized-op])))) + vec)) + + :else + nil)) + +(defn- patch-inverse-delete-block-ops + [inverse-outliner-ops forward-outliner-ops] + (let [forward-insert-ops* (atom (->> forward-outliner-ops + reverse + (filter #(contains? #{:insert-blocks :apply-template} (first %))) + vec))] + (mapv (fn [[op args :as inverse-op]] + (if (and (= :delete-blocks op) + (seq @forward-insert-ops*)) + (let [[_ [blocks _target-id _opts]] (first @forward-insert-ops*) + ids (->> blocks + (keep (fn [block] + (when-let [block-uuid (:block/uuid block)] + [:block/uuid block-uuid]))) + vec)] + (swap! forward-insert-ops* subvec 1) + (if (seq ids) + [:delete-blocks [ids (second args)]] + inverse-op)) + inverse-op)) + inverse-outliner-ops))) + +(defn- patch-forward-delete-block-op-ids + [db-before outliner-ops] + (some->> outliner-ops + (mapv (fn [[op args :as op-entry]] + (if (= :delete-blocks op) + (let [[ids opts] args] + [:delete-blocks [(stable-id-coll db-before ids) opts]]) + op-entry))) + seq + vec)) + +(defn- canonicalize-outliner-ops + [db tx-meta tx-data] + (let [explicit-forward-ops (:db-sync/forward-outliner-ops tx-meta) + outliner-ops (:outliner-ops tx-meta)] + (cond + (seq explicit-forward-ops) + (canonicalize-explicit-outliner-ops db tx-data explicit-forward-ops) + + (seq outliner-ops) + (if (every? (fn [[op]] + (contains? semantic-outliner-ops op)) + outliner-ops) + (canonicalize-explicit-outliner-ops db tx-data outliner-ops) + canonical-transact-op) + + (contains? #{:transact :batch-import-edn} (:outliner-op tx-meta)) + canonical-transact-op))) + +(defn derive-history-outliner-ops + [db-before db-after tx-data tx-meta] + (let [forward-outliner-ops (patch-forward-delete-block-op-ids + db-before + (canonicalize-outliner-ops db-after tx-meta tx-data)) + forward-outliner-ops (some-> forward-outliner-ops seq vec) + forward-outliner-ops (when (seq forward-outliner-ops) + (if (and (> (count forward-outliner-ops) 1) + (some (fn [[op]] (= :transact op)) forward-outliner-ops)) + canonical-transact-op + forward-outliner-ops)) + built-inverse-outliner-ops (some-> (build-strict-inverse-outliner-ops db-before db-after tx-data forward-outliner-ops) + seq + vec) + explicit-inverse-outliner-ops (some-> (canonicalize-explicit-outliner-ops db-after tx-data (:db-sync/inverse-outliner-ops tx-meta)) + (patch-inverse-delete-block-ops forward-outliner-ops) + seq + vec) + inverse-outliner-ops (cond + (and (= :apply-template (:outliner-op tx-meta)) + (:undo? tx-meta) + (seq (:db-sync/inverse-outliner-ops tx-meta))) + (:db-sync/inverse-outliner-ops tx-meta) + + (has-replace-empty-target-insert-op? forward-outliner-ops) + built-inverse-outliner-ops + + (seq built-inverse-outliner-ops) + built-inverse-outliner-ops + + (nil? explicit-inverse-outliner-ops) + nil + + ;; Treat explicit transact placeholder as "no semantic inverse". + ;; Keep nil so semantic replay must fail-fast when required. + (= canonical-transact-op explicit-inverse-outliner-ops) + nil + + :else + explicit-inverse-outliner-ops) + inverse-outliner-ops (some-> inverse-outliner-ops seq vec)] + {:forward-outliner-ops forward-outliner-ops + :inverse-outliner-ops inverse-outliner-ops})) diff --git a/deps/outliner/src/logseq/outliner/page.cljs b/deps/outliner/src/logseq/outliner/page.cljs index 6a90b942b4..12577a1431 100644 --- a/deps/outliner/src/logseq/outliner/page.cljs +++ b/deps/outliner/src/logseq/outliner/page.cljs @@ -19,10 +19,11 @@ [logseq.graph-parser.block :as gp-block] [logseq.graph-parser.text :as text] [logseq.outliner.recycle :as outliner-recycle] + [logseq.outliner.tx-meta :as outliner-tx-meta] [logseq.outliner.validate :as outliner-validate])) -(defn- db-refs->page - "Replace [[page name]] with page name" +(defn- page-ref-rewrite-targets + "Collect entities that reference `page-entity` via node refs and need title rewrite." [page-entity] (let [refs (->> (:block/_refs page-entity) ;; remove child or self that refed this page @@ -30,24 +31,49 @@ (or (= (:db/id ref) (:db/id page-entity)) (= (:db/id (:block/page ref)) (:db/id page-entity)))))) id-ref->page #(db-content/content-id-ref->page % [page-entity])] - (when (seq refs) - (let [tx-data (mapcat (fn [{:block/keys [raw-title] :as ref}] - ;; block content - (when raw-title - (let [content' (id-ref->page raw-title) - content-tx (when (not= raw-title content') - {:db/id (:db/id ref) - :block/title content'}) - tx content-tx] - (concat - [[:db/retract (:db/id ref) :block/refs (:db/id page-entity)]] - (when tx [tx]))))) refs)] - tx-data)))) + (->> refs + (keep (fn [ref] + (let [raw-title (:block/raw-title ref) + block-uuid (:block/uuid ref)] + (when raw-title + (let [content' (id-ref->page raw-title)] + (when (not= raw-title content') + (let [remaining-refs (->> (:block/refs ref) + (remove (fn [ref'] + (= (:db/id ref') (:db/id page-entity)))) + vec)] + {:ref-id (:db/id ref) + :ref-uuid block-uuid + :title content' + :refs remaining-refs}))))))) + seq))) + +(defn- db-refs->page + "Replace [[page name]] with page name." + [page-entity] + (let [page-id (:db/id page-entity)] + (some->> (page-ref-rewrite-targets page-entity) + (mapcat (fn [{:keys [ref-id title]}] + [[:db/retract ref-id :block/refs page-id] + {:db/id ref-id + :block/title title}]))))) + +(defn- db-refs->page-save-ops + [page-entity] + (some->> (page-ref-rewrite-targets page-entity) + (keep (fn [{:keys [ref-uuid title refs]}] + (when ref-uuid + [:save-block [{:block/uuid ref-uuid + :block/title title + :block/refs refs} + {}]]))) + seq + vec)) (defn- build-page-retract-tx "Build cleanup tx-data for deleting a schema page. This is pure and can be reused by sync repair." - [db page & [{:keys [include-page-retract?] + [db page & [{:keys [include-page-retract? today-page?] :or {include-page-retract? true}}]] (let [page-id (:db/id page) page-blocks-tx-data (->> (:block/_page page) @@ -65,11 +91,13 @@ page-tx (when (and include-page-retract? (d/entity db page-id)) [[:db/retractEntity page-id]])] - (concat page-blocks-tx-data - property-pair-tx-data - restore-class-parent-tx - (db-refs->page page) - page-tx))) + (if today-page? + page-blocks-tx-data + (concat page-blocks-tx-data + property-pair-tx-data + restore-class-parent-tx + (db-refs->page page) + page-tx)))) (defn delete! "Deletes a page. Returns true if able to delete page. If unable to delete, @@ -86,31 +114,37 @@ (when-let [page (d/entity @conn [:block/uuid page-uuid])] (let [today-page? (when-let [day (:block/journal-day page)] (= (date-time-util/ms->journal-day (js/Date.)) day)) - tx-meta (cond-> {:outliner-op :delete-page - :deleted-page (:block/title page) - :persist-op? persist-op?} + tx-meta (cond-> (outliner-tx-meta/ensure-outliner-ops + {:outliner-op :delete-page + :deleted-page (:block/title page) + :persist-op? persist-op?} + [:delete-page [page-uuid {:deleted-by-uuid deleted-by-uuid + :now-ms now-ms}]]) rename? (assoc :real-outliner-op :rename-page))] ;; TODO: maybe we should add $$$favorites to built-in pages? (cond - today-page? - false - (or (ldb/built-in? page) (ldb/hidden? page)) (do (error-handler {:msg "Built-in page cannot be deleted"}) false) - (or (ldb/class? page) (ldb/property? page)) - (let [tx-data (build-page-retract-tx @conn page)] + (or (ldb/class? page) (ldb/property? page) today-page?) + (let [tx-data (build-page-retract-tx @conn page {:today-page? today-page?})] (ldb/transact! conn tx-data tx-meta) true) :else - (let [tx-data (outliner-recycle/recycle-page-tx-data @conn page {:deleted-by-uuid deleted-by-uuid - :now-ms now-ms})] + (let [ref-rewrite-tx-data (db-refs->page page) + ref-rewrite-save-ops (db-refs->page-save-ops page) + tx-data (concat ref-rewrite-tx-data + (outliner-recycle/recycle-page-tx-data @conn page {:deleted-by-uuid deleted-by-uuid + :now-ms now-ms})) + tx-meta' (cond-> tx-meta + (seq ref-rewrite-save-ops) + (update :outliner-ops (fnil into []) ref-rewrite-save-ops))] (when (seq tx-data) - (ldb/transact! conn tx-data tx-meta)) + (ldb/transact! conn tx-data tx-meta')) true)))))) (defn- build-page-tx [db properties page {:keys [class? tags class-ident-namespace]}] @@ -289,22 +323,26 @@ :block/uuid) existing-page-by-journal-uuid (when (uuid? journal-page-uuid) (d/entity db [:block/uuid journal-page-uuid])) - existing-page-id (some->> existing-names-page - (filter #(try (when-let [e (and class-ident-namespace? (d/entity db %))] - (let [ns' (namespace (:db/ident e))] - (= (str ns') class-ident-namespace))) - (catch :default _ false))) - (first)) + existing-page-id (if class-ident-namespace? + (some->> existing-names-page + (filter #(try (when-let [e (d/entity db %)] + (let [ns' (namespace (:db/ident e))] + (= (str ns') class-ident-namespace))) + (catch :default _ false))) + (first)) + (first existing-names-page)) existing-page (or (some->> existing-page-id (d/entity db)) existing-page-by-journal-uuid)] (if (and existing-page (or (:block/journal-day existing-page) - (not (:block/parent existing-page)))) + (not (:block/parent existing-page)) + (ldb/recycled? existing-page))) (let [tx-meta {:persist-op? persist-op? :outliner-op :save-block}] - (if (and class? - (not (ldb/class? existing-page)) - (ldb/internal-page? existing-page)) + (cond + (and class? + (not (ldb/class? existing-page)) + (ldb/internal-page? existing-page)) ;; Convert existing page to class (let [tx-data [(merge (db-class/build-new-class db (select-keys existing-page [:block/title :block/uuid :block/created-at]) @@ -316,7 +354,20 @@ :tx-data tx-data :page-uuid (:block/uuid existing-page) :title (:block/title existing-page)}) + + (ldb/recycled? existing-page) + (let [options' (assoc options :uuid (:block/uuid existing-page)) + tx-meta' (outliner-tx-meta/ensure-outliner-ops + {:persist-op? persist-op? + :outliner-op :create-page} + [:create-page [title options']])] + {:tx-meta tx-meta' + :tx-data (outliner-recycle/restore-tx-data db existing-page) + :page-uuid (:block/uuid existing-page) + :title (:block/title existing-page)}) + ;; Just return existing page info + :else {:page-uuid (:block/uuid existing-page) :title (:block/title existing-page)})) (let [page (gp-block/page-name->map title db true date-formatter @@ -343,8 +394,10 @@ ;; transact doesn't support entities (remove de/entity? parents') page-txs) - tx-meta (cond-> {:persist-op? persist-op? - :outliner-op :create-page} + tx-meta (cond-> (outliner-tx-meta/ensure-outliner-ops + {:persist-op? persist-op? + :outliner-op :create-page} + [:create-page [title options]]) today-journal? (assoc :create-today-journal? true :today-journal-name title))] diff --git a/deps/outliner/src/logseq/outliner/property.cljs b/deps/outliner/src/logseq/outliner/property.cljs index d9fe3e6cbe..e02c584971 100644 --- a/deps/outliner/src/logseq/outliner/property.cljs +++ b/deps/outliner/src/logseq/outliner/property.cljs @@ -19,6 +19,7 @@ [logseq.db.sqlite.util :as sqlite-util] [logseq.outliner.core :as outliner-core] [logseq.outliner.page :as outliner-page] + [logseq.outliner.tx-meta :as outliner-tx-meta] [logseq.outliner.validate :as outliner-validate] [malli.core :as m] [malli.error :as me] @@ -273,6 +274,16 @@ [id] (if (uuid? id) [:block/uuid id] id)) +(defn- with-op-entry + [op-entry f] + (binding [outliner-tx-meta/*outliner-op-entry* + (or outliner-tx-meta/*outliner-op-entry* op-entry)] + (f))) + +(defn- transact-with-op! + [conn tx-data tx-meta] + (ldb/transact! conn tx-data (outliner-tx-meta/ensure-outliner-ops tx-meta nil))) + (defn- raw-set-block-property! "Adds the raw property pair (value not modified) to the given block if the property value is valid" [conn block property new-value] @@ -280,31 +291,34 @@ (throw-error-if-invalid-property-value @conn property new-value) (let [property-id (:db/ident property) tx-data (build-property-value-tx-data conn block property-id new-value)] - (ldb/transact! conn tx-data {:outliner-op :save-block}))) + (transact-with-op! conn tx-data {:outliner-op :save-block}))) (defn create-property-text-block! "Creates a property value block for the given property and value. Adds it to block if given block." [conn block-id property-id value {:keys [new-block-id]}] - (let [property (d/entity @conn property-id) - block (when block-id (d/entity @conn block-id)) - _ (assert (some? property) (str "Property " property-id " doesn't exist yet")) - value' (convert-property-input-string (:logseq.property/type block) - property value) - _ (when (and (not= (:logseq.property/type property) :number) - (not (string? value'))) - (throw (ex-info "value should be a string" {:block-id block-id - :property-id property-id - :value value'}))) - new-value-block (cond-> (db-property-build/build-property-value-block (or block property) property value') - new-block-id - (assoc :block/uuid new-block-id))] - (ldb/transact! conn [new-value-block] {:outliner-op :insert-blocks}) - (let [property-id (:db/ident property)] - (when (and property-id block) - (when-let [block-id (:db/id (d/entity @conn [:block/uuid (:block/uuid new-value-block)]))] - (raw-set-block-property! conn block property block-id))) - (:block/uuid new-value-block)))) + (with-op-entry + [:create-property-text-block [block-id property-id value {:new-block-id new-block-id}]] + (fn [] + (let [property (d/entity @conn property-id) + block (when block-id (d/entity @conn block-id)) + _ (assert (some? property) (str "Property " property-id " doesn't exist yet")) + value' (convert-property-input-string (:logseq.property/type block) + property value) + _ (when (and (not= (:logseq.property/type property) :number) + (not (string? value'))) + (throw (ex-info "value should be a string" {:block-id block-id + :property-id property-id + :value value'}))) + new-value-block (cond-> (db-property-build/build-property-value-block (or block property) property value') + new-block-id + (assoc :block/uuid new-block-id))] + (transact-with-op! conn [new-value-block] {:outliner-op :insert-blocks}) + (let [property-id (:db/ident property)] + (when (and property-id block) + (when-let [block-id (:db/id (d/entity @conn [:block/uuid (:block/uuid new-value-block)]))] + (raw-set-block-property! conn block property block-id))) + (:block/uuid new-value-block)))))) (defn- get-property-value-eid [db property-id raw-value] @@ -404,36 +418,56 @@ (defn batch-remove-property! [conn block-ids property-id] - (throw-error-if-read-only-property property-id) - (let [block-eids (map ->eid block-ids) - blocks (keep (fn [id] (d/entity @conn id)) block-eids) - block-id-set (set (map :db/id blocks))] - (validate-batch-deletion-of-property blocks property-id) - (when (seq blocks) - (when-let [property (d/entity @conn property-id)] - (let [txs (mapcat - (fn [block] - (let [value (get block property-id) - entities (cond - (de/entity? value) [value] - (and (sequential? value) (every? de/entity? value)) value - :else nil) - deleting-entities (filter - (fn [value] - (and - (:logseq.property/created-from-property value) - (not (or (entity-util/page? value) (ldb/closed-value? value))) - (empty? (set/difference (set (map :e (d/datoms @conn :avet (:db/ident property) (:db/id value)))) block-id-set)))) - entities) - ;; Delete property value block if it's no longer used by other blocks - retract-blocks-tx (when (seq deleting-entities) - (:tx-data (outliner-core/delete-blocks @conn deleting-entities {})))] - (concat - [[:db/retract (:db/id block) (:db/ident property)]] - retract-blocks-tx))) - blocks)] - (when (seq txs) - (ldb/transact! conn txs {:outliner-op :save-block}))))))) + (with-op-entry + [:batch-remove-property [block-ids property-id]] + (fn [] + (throw-error-if-read-only-property property-id) + (let [block-eids (map ->eid block-ids) + blocks (keep (fn [id] (d/entity @conn id)) block-eids) + block-id-set (set (map :db/id blocks))] + (validate-batch-deletion-of-property blocks property-id) + (when (seq blocks) + (when-let [property (d/entity @conn property-id)] + (let [txs (mapcat + (fn [block] + (let [value (get block property-id) + entities (cond + (de/entity? value) [value] + (and (sequential? value) (every? de/entity? value)) value + :else nil) + deleting-entities (filter + (fn [value] + (let [value-referrers* + (d/q '[:find [?e ...] + :in $ ?property-id ?value-id + :where + [?e ?property-id ?value-id]] + @conn + (:db/ident property) + (:db/id value)) + value-referrers + (cond + (nil? value-referrers*) + #{} + + (coll? value-referrers*) + (set value-referrers*) + + :else + #{value-referrers*})] + (and + (:logseq.property/created-from-property value) + (not (or (entity-util/page? value) (ldb/closed-value? value))) + (empty? (set/difference value-referrers block-id-set))))) + entities) + retract-blocks-tx (when (seq deleting-entities) + (:tx-data (outliner-core/delete-blocks @conn deleting-entities {})))] + (concat + [[:db/retract (:db/id block) (:db/ident property)]] + retract-blocks-tx))) + blocks)] + (when (seq txs) + (transact-with-op! conn txs {:outliner-op :save-block}))))))))) (defn batch-set-property! "Sets properties for multiple blocks. Automatically handles property value refs. @@ -441,90 +475,96 @@ ([conn block-ids property-id v] (batch-set-property! conn block-ids property-id v {})) ([conn block-ids property-id v options] - (assert property-id "property-id is nil") - (throw-error-if-read-only-property property-id) - (if (nil? v) - (batch-remove-property! conn block-ids property-id) - (let [block-eids (map ->eid block-ids) - _ (when (= property-id :block/tags) - (outliner-validate/validate-tags-property @conn block-eids v)) - property (d/entity @conn property-id) - _ (when (= (:db/ident property) :logseq.property.class/extends) - (outliner-validate/validate-extends-property - @conn - (if (number? v) (d/entity @conn v) v) - (map #(d/entity @conn %) block-eids))) - _ (when (nil? property) - (throw (ex-info (str "Property " property-id " doesn't exist yet") {:property-id property-id}))) - property-type (get property :logseq.property/type :default) - entity-id? (and (:entity-id? options) (number? v)) - ref? (contains? db-property-type/all-ref-property-types property-type) - default-url-not-closed? (and (contains? #{:default :url} property-type) - (not (seq (entity-plus/lookup-kv-then-entity property :property/closed-values)))) - v' (if (and ref? (not entity-id?)) - (convert-ref-property-value conn property-id v property-type) - v) - _ (when (nil? v') - (throw (ex-info "Property value must be not nil" {:v v}))) - txs (doall - (mapcat - (fn [eid] - (if-let [block (d/entity @conn eid)] - (let [v' (if (and default-url-not-closed? - (not (and (keyword? v) entity-id?))) - (do - (when (number? v') - (throw-error-if-invalid-property-value @conn property v')) - (let [v (if (number? v') (:block/title (d/entity @conn v')) v')] - (convert-ref-property-value conn property-id v property-type))) - v')] - (throw-error-if-self-value block v' ref?) - (throw-error-if-invalid-property-value @conn property v') - (build-property-value-tx-data conn block property-id v')) - (js/console.error "Skipping setting a block's property because the block id could not be found:" eid))) - block-eids))] - (when (seq txs) - (ldb/transact! conn txs {:outliner-op :save-block})))))) + (with-op-entry + [:batch-set-property [block-ids property-id v options]] + (fn [] + (assert property-id "property-id is nil") + (throw-error-if-read-only-property property-id) + (if (nil? v) + (batch-remove-property! conn block-ids property-id) + (let [block-eids (map ->eid block-ids) + _ (when (= property-id :block/tags) + (outliner-validate/validate-tags-property @conn block-eids v)) + property (d/entity @conn property-id) + _ (when (= (:db/ident property) :logseq.property.class/extends) + (outliner-validate/validate-extends-property + @conn + (if (number? v) (d/entity @conn v) v) + (map #(d/entity @conn %) block-eids))) + _ (when (nil? property) + (throw (ex-info (str "Property " property-id " doesn't exist yet") {:property-id property-id}))) + property-type (get property :logseq.property/type :default) + entity-id? (and (:entity-id? options) (number? v)) + ref? (contains? db-property-type/all-ref-property-types property-type) + default-url-not-closed? (and (contains? #{:default :url} property-type) + (not (seq (entity-plus/lookup-kv-then-entity property :property/closed-values)))) + v' (if (and ref? (not entity-id?)) + (convert-ref-property-value conn property-id v property-type) + v) + _ (when (nil? v') + (throw (ex-info "Property value must be not nil" {:v v}))) + txs (doall + (mapcat + (fn [eid] + (if-let [block (d/entity @conn eid)] + (let [v' (if (and default-url-not-closed? + (not (and (keyword? v) entity-id?))) + (do + (when (number? v') + (throw-error-if-invalid-property-value @conn property v')) + (let [v (if (number? v') (:block/title (d/entity @conn v')) v')] + (convert-ref-property-value conn property-id v property-type))) + v')] + (throw-error-if-self-value block v' ref?) + (throw-error-if-invalid-property-value @conn property v') + (build-property-value-tx-data conn block property-id v')) + (js/console.error "Skipping setting a block's property because the block id could not be found:" eid))) + block-eids))] + (when (seq txs) + (transact-with-op! conn txs {:outliner-op :save-block})))))))) (defn remove-block-property! [conn eid property-id] - (throw-error-if-read-only-property property-id) - (let [eid (->eid eid) - block (d/entity @conn eid) - property (d/entity @conn property-id)] - ;; Can skip for extends b/c below tx ensures it has a default value - (when-not (= :logseq.property.class/extends property-id) - (validate-batch-deletion-of-property [block] property-id)) - (when block - (cond - (= :logseq.property/empty-placeholder (:db/ident (get block property-id))) - nil + (with-op-entry + [:remove-block-property [eid property-id]] + (fn [] + (throw-error-if-read-only-property property-id) + (let [eid (->eid eid) + block (d/entity @conn eid) + property (d/entity @conn property-id)] + (when-not (= :logseq.property.class/extends property-id) + (validate-batch-deletion-of-property [block] property-id)) + (when block + (cond + (= :logseq.property/empty-placeholder (:db/ident (get block property-id))) + nil - (= :logseq.property/status property-id) - (ldb/transact! conn - [[:db/retract (:db/id block) property-id] - [:db/retract (:db/id block) :block/tags :logseq.class/Task]] - {:outliner-op :save-block}) + (= :logseq.property/status property-id) + (transact-with-op! conn + [[:db/retract (:db/id block) property-id] + [:db/retract (:db/id block) :block/tags :logseq.class/Task]] + {:outliner-op :save-block}) - (and (:logseq.property/default-value property) - (= (:logseq.property/default-value property) (get block property-id))) - (ldb/transact! conn - [{:db/id (:db/id block) - property-id :logseq.property/empty-placeholder}] - {:outliner-op :save-block}) + (and (:logseq.property/default-value property) + (= (:logseq.property/default-value property) (get block property-id))) + (transact-with-op! conn + [{:db/id (:db/id block) + property-id :logseq.property/empty-placeholder}] + {:outliner-op :save-block}) - (and (ldb/class? block) (= property-id :logseq.property.class/extends)) - (ldb/transact! conn - [[:db/retract (:db/id block) :logseq.property.class/extends] - [:db/add (:db/id block) :logseq.property.class/extends :logseq.class/Root]] - {:outliner-op :save-block}) + (and (ldb/class? block) (= property-id :logseq.property.class/extends)) + (transact-with-op! conn + [[:db/retract (:db/id block) :logseq.property.class/extends] + [:db/add (:db/id block) :logseq.property.class/extends :logseq.class/Root]] + {:outliner-op :save-block}) - (contains? db-property/db-attribute-properties property-id) - (ldb/transact! conn - [[:db/retract (:db/id block) property-id]] - {:outliner-op :save-block}) - :else - (batch-remove-property! conn [eid] property-id))))) + (contains? db-property/db-attribute-properties property-id) + (transact-with-op! conn + [[:db/retract (:db/id block) property-id]] + {:outliner-op :save-block}) + + :else + (batch-remove-property! conn [eid] property-id))))))) (defn- set-block-db-attribute! [conn db block property property-id v] @@ -534,125 +574,150 @@ [{:db/id (:db/id block) property-id v}] (= property-id :logseq.property.class/extends) (conj [:db/retract (:db/id block) :logseq.property.class/extends :logseq.class/Root]))] - (ldb/transact! conn tx-data - {:outliner-op :save-block})))) + (transact-with-op! conn tx-data + {:outliner-op :save-block})))) -(defn set-block-property! +(defn ^:large-vars/cleanup-todo set-block-property! "Updates a block property's value for an existing property-id and block. If property is a ref type, automatically handles a raw property value i.e. you can pass \"value\" instead of the property value entity. Also handle db attributes as properties" [conn block-eid property-id v] - (throw-error-if-read-only-property property-id) - (let [db @conn - block-eid (->eid block-eid) - _ (assert (qualified-keyword? property-id) "property-id should be a keyword") - block (d/entity @conn block-eid) - db-attribute? (some? (db-schema/schema property-id)) - property (d/entity @conn property-id) - property-type (get property :logseq.property/type :default) - ref? (db-property-type/all-ref-property-types property-type) - v' (if ref? - (convert-ref-property-value conn property-id v property-type) - v)] - (when-not (and block property) - (throw (ex-info "Set block property failed: block or property doesn't exist" - {:block-eid block-eid - :property-id property-id - :block block - :property property}))) - (if (nil? v') - (remove-block-property! conn block-eid property-id) - (do - (when (= property-id :block/tags) - (outliner-validate/validate-tags-property @conn [block-eid] v')) - (when (= property-id :logseq.property.class/extends) - (outliner-validate/validate-extends-property @conn v' [block])) - (cond - db-attribute? - (set-block-db-attribute! conn db block property property-id v) + (with-op-entry + [:set-block-property [block-eid property-id v]] + (fn [] + (throw-error-if-read-only-property property-id) + (let [db @conn + block-eid (->eid block-eid) + _ (assert (qualified-keyword? property-id) "property-id should be a keyword") + block (d/entity @conn block-eid) + db-attribute? (some? (db-schema/schema property-id)) + property (d/entity @conn property-id) + property-type (get property :logseq.property/type :default) + ref? (db-property-type/all-ref-property-types property-type) + v' (if ref? + (convert-ref-property-value conn property-id v property-type) + v)] + (when-not (and block property) + (throw (ex-info "Set block property failed: block or property doesn't exist" + {:block-eid block-eid + :property-id property-id + :block block + :property property}))) + (if (nil? v') + (remove-block-property! conn block-eid property-id) + (do + (when (= property-id :block/tags) + (outliner-validate/validate-tags-property @conn [block-eid] v')) + (when (= property-id :logseq.property.class/extends) + (outliner-validate/validate-extends-property @conn v' [block])) + (cond + db-attribute? + (set-block-db-attribute! conn db block property property-id v) - :else - (let [_ (assert (some? property) (str "Property " property-id " doesn't exist yet")) - ref? (db-property-type/all-ref-property-types property-type) - existing-value (get block property-id) - many? (= :db.cardinality/many (:db/cardinality property)) - value-matches? (if ref? - (if (and many? (coll? v')) - (= (set (map :db/id existing-value)) (set v')) - (= existing-value v')) - (= existing-value v'))] - (throw-error-if-self-value block v' ref?) - - (when-not value-matches? - (raw-set-block-property! conn block property v')))))))) + :else + (let [_ (assert (some? property) (str "Property " property-id " doesn't exist yet")) + ref? (db-property-type/all-ref-property-types property-type) + existing-value (get block property-id) + many? (= :db.cardinality/many (:db/cardinality property)) + many-ref-value-ids (fn [value] + (->> (cond + (nil? value) [] + (de/entity? value) [value] + (sequential? value) value + :else [value]) + (map (fn [item] + (if (de/entity? item) + (:db/id item) + item))) + set)) + value-matches? (if ref? + (if (and many? (coll? v')) + (= (many-ref-value-ids existing-value) + (many-ref-value-ids v')) + (= existing-value v')) + (= existing-value v'))] + (throw-error-if-self-value block v' ref?) + (when-not value-matches? + (raw-set-block-property! conn block property v')))))))))) (defn upsert-property! "Updates property if property-id is given. Otherwise creates a property - with the given property-id or :property-name option. When a property is created - it is ensured to have a unique :db/ident" + with the given property-id or :property-name option. When a property is created + it is ensured to have a unique :db/ident" [conn property-id schema {:keys [property-name properties] :as opts}] - (let [db @conn - db-ident (or property-id - (try (db-property/create-user-property-ident-from-name property-name) - (catch :default e - (throw (ex-info (str e) - {:type :notification - :payload {:message "Property failed to create. Please try a different property name." - :type :error}})))))] - (assert (qualified-keyword? db-ident)) - (when (and (contains? #{:checkbox} (:logseq.property/type schema)) - (= :db.cardinality/many (:db/cardinality schema))) - (throw (ex-info ":checkbox property doesn't allow multiple values" {:property-id property-id - :schema schema}))) - (if-let [property (and (qualified-keyword? property-id) (d/entity db db-ident))] - (update-property conn db-ident property schema opts) - (let [k-name (or (and property-name (name property-name)) - (name property-id)) - db-ident' (db-ident/ensure-unique-db-ident @conn db-ident)] - (assert (some? k-name) - (prn "property-id: " property-id ", property-name: " property-name)) - (outliner-validate/validate-page-title k-name {:node {:db/ident db-ident'}}) - (outliner-validate/validate-page-title-characters k-name {:node {:db/ident db-ident'}}) - (let [db-id (:db/id properties) - opts (cond-> {:title k-name - :properties properties} - (integer? db-id) - (assoc :block-uuid (:block/uuid (d/entity db db-id))))] - (ldb/transact! conn - (concat - [(sqlite-util/build-new-property db-ident' schema opts)] - ;; Convert page to property - (when db-id - [[:db/retract db-id :block/tags :logseq.class/Page]])) - {:outliner-op :upsert-property})) - (d/entity @conn db-ident'))))) + (with-op-entry + [:upsert-property [property-id schema opts]] + (fn [] + (let [db @conn + db-ident (or property-id + (try (db-property/create-user-property-ident-from-name property-name) + (catch :default e + (throw (ex-info (str e) + {:type :notification + :payload {:message "Property failed to create. Please try a different property name." + :type :error}})))))] + (assert (qualified-keyword? db-ident)) + (when (and (contains? #{:checkbox} (:logseq.property/type schema)) + (= :db.cardinality/many (:db/cardinality schema))) + (throw (ex-info ":checkbox property doesn't allow multiple values" + {:property-id property-id + :schema schema}))) + (if-let [property (and (qualified-keyword? property-id) (d/entity db db-ident))] + (update-property conn db-ident property schema opts) + (let [k-name (or (and property-name (name property-name)) + (name property-id)) + db-ident' (db-ident/ensure-unique-db-ident @conn db-ident)] + (assert (some? k-name) + (prn "property-id: " property-id ", property-name: " property-name)) + (outliner-validate/validate-page-title k-name {:node {:db/ident db-ident'}}) + (outliner-validate/validate-page-title-characters k-name {:node {:db/ident db-ident'}}) + (let [db-id (:db/id properties) + opts' (cond-> {:title k-name + :properties properties} + (integer? db-id) + (assoc :block-uuid (:block/uuid (d/entity db db-id))))] + (transact-with-op! conn + (concat + [(sqlite-util/build-new-property db-ident' schema opts')] + (when db-id + [[:db/retract db-id :block/tags :logseq.class/Page]])) + {:outliner-op :upsert-property})) + (d/entity @conn db-ident'))))))) (defn batch-delete-property-value! "batch delete value when a property has multiple values" [conn block-eids property-id property-value] - (when-let [property (d/entity @conn property-id)] - (when (and (db-property/many? property) - (not (some #(= property-id (:db/ident (d/entity @conn %))) block-eids))) - (when (= property-id :block/tags) - (outliner-validate/validate-tags-property-deletion @conn block-eids property-value)) - (if (= property-id :block/tags) - (let [tx-data (map (fn [id] [:db/retract id property-id property-value]) block-eids)] - (ldb/transact! conn tx-data {:outliner-op :save-block})) - (doseq [block-eid block-eids] - (when-let [block (d/entity @conn block-eid)] - (let [current-val (get block property-id) - fv (first current-val)] - (if (and (= 1 (count current-val)) (or (= property-value fv) (= property-value (:db/id fv)))) - (remove-block-property! conn (:db/id block) property-id) - (ldb/transact! conn - [[:db/retract (:db/id block) property-id property-value]] - {:outliner-op :save-block}))))))))) + (with-op-entry + [:batch-delete-property-value [block-eids property-id property-value]] + (fn [] + (when-let [property (d/entity @conn property-id)] + (when (and (db-property/many? property) + (not (some #(= property-id (:db/ident (d/entity @conn %))) block-eids))) + (when (= property-id :block/tags) + (outliner-validate/validate-tags-property-deletion @conn block-eids property-value)) + (if (= property-id :block/tags) + (let [tx-data (map (fn [id] [:db/retract id property-id property-value]) block-eids)] + (transact-with-op! conn tx-data {:outliner-op :save-block})) + (doseq [block-eid block-eids] + (when-let [block (d/entity @conn block-eid)] + (let [current-val (get block property-id) + fv (first current-val)] + (if (and (= 1 (count current-val)) + (or (= property-value fv) + (= property-value (:db/id fv)))) + (remove-block-property! conn (:db/id block) property-id) + (transact-with-op! conn + [[:db/retract (:db/id block) property-id property-value]] + {:outliner-op :save-block}))))))))))) (defn delete-property-value! "Delete value if a property has multiple values" [conn block-eid property-id property-value] - (batch-delete-property-value! conn [block-eid] property-id property-value)) + (with-op-entry + [:delete-property-value [block-eid property-id property-value]] + (fn [] + (batch-delete-property-value! conn [block-eid] property-id property-value)))) (defn ^:api get-classes-parents [tags] @@ -765,108 +830,120 @@ (defn upsert-closed-value! "id should be a block UUID or nil" [conn property-id {:keys [id value description _scoped-class-id] :as opts}] - (assert (or (nil? id) (uuid? id))) - (let [db @conn - property (d/entity db property-id) - property-type (:logseq.property/type property)] - (when (contains? db-property-type/closed-value-property-types property-type) - (let [value' (if (string? value) (string/trim value) value) - resolved-value (convert-property-input-string nil property value') - validate-message (validate-property-value-aux - (get-property-value-schema @conn property-type property {:new-closed-value? true}) - resolved-value - {:many? (db-property/many? property)})] - (cond - (some (fn [b] - (and (= (str resolved-value) (str (or (db-property/closed-value-content b) - (:block/uuid b)))) - (not= id (:block/uuid b)))) - (entity-plus/lookup-kv-then-entity property :property/closed-values)) + (with-op-entry + [:upsert-closed-value [property-id opts]] + (fn [] + (assert (or (nil? id) (uuid? id))) + (let [db @conn + property (d/entity db property-id) + property-type (:logseq.property/type property)] + (when (contains? db-property-type/closed-value-property-types property-type) + (let [value' (if (string? value) (string/trim value) value) + resolved-value (convert-property-input-string nil property value') + validate-message (validate-property-value-aux + (get-property-value-schema @conn property-type property {:new-closed-value? true}) + resolved-value + {:many? (db-property/many? property)})] + (cond + (some (fn [b] + (and (= (str resolved-value) (str (or (db-property/closed-value-content b) + (:block/uuid b)))) + (not= id (:block/uuid b)))) + (entity-plus/lookup-kv-then-entity property :property/closed-values)) + (throw (ex-info "Closed value choice already exists" + {:error :value-exists + :type :notification + :payload {:message "Choice already exists" + :type :warning}})) - ;; Make sure to update frontend.handler.db-based.property-test when updating ex-info message - (throw (ex-info "Closed value choice already exists" - {:error :value-exists - :type :notification - :payload {:message "Choice already exists" - :type :warning}})) + validate-message + (throw (ex-info "Invalid property value" + {:error :value-invalid + :type :notification + :payload {:message validate-message + :type :warning}})) - validate-message - ;; Make sure to update frontend.handler.db-based.property-test when updating ex-info message - (throw (ex-info "Invalid property value" - {:error :value-invalid - :type :notification - :payload {:message validate-message - :type :warning}})) + (nil? resolved-value) + nil - (nil? resolved-value) - nil - - :else - (let [tx-data (build-closed-value-tx @conn property resolved-value opts)] - (ldb/transact! conn tx-data {:outliner-op :save-block}) - (when (seq description) - (if-let [desc-ent (and id (:logseq.property/description (d/entity db [:block/uuid id])))] - (ldb/transact! conn - [(outliner-core/block-with-updated-at {:db/id (:db/id desc-ent) - :block/title description})] - {:outliner-op :save-block}) - (set-block-property! conn - ;; new closed value is first in tx-data - [:block/uuid (or id (:block/uuid (first tx-data)))] - :logseq.property/description - description))))))))) + :else + (let [tx-data (build-closed-value-tx @conn property resolved-value opts)] + (transact-with-op! conn tx-data {:outliner-op :insert-blocks}) + (when (seq description) + (if-let [desc-ent (and id (:logseq.property/description (d/entity db [:block/uuid id])))] + (transact-with-op! conn + [(outliner-core/block-with-updated-at {:db/id (:db/id desc-ent) + :block/title description})] + {:outliner-op :save-block}) + (set-block-property! conn + [:block/uuid (or id (:block/uuid (first tx-data)))] + :logseq.property/description + description))))))))))) (defn add-existing-values-to-closed-values! "Adds existing values as closed values and returns their new block uuids" [conn property-id values] - (when-let [property (d/entity @conn property-id)] - (when (seq values) - (let [values' (remove string/blank? values)] - (assert (every? uuid? values') "existing values should all be UUIDs") - (let [values (keep #(d/entity @conn [:block/uuid %]) values')] - (when (seq values) - (let [value-property-tx (map (fn [id] - {:db/id id - :block/closed-value-property (:db/id property)}) - (map :db/id values)) - property-tx (outliner-core/block-with-updated-at {:db/id (:db/id property)})] - (ldb/transact! conn (cons property-tx value-property-tx) - {:outliner-op :save-blocks})))))))) + (with-op-entry + [:add-existing-values-to-closed-values [property-id values]] + (fn [] + (when-let [property (d/entity @conn property-id)] + (when (seq values) + (let [values' (remove string/blank? values)] + (assert (every? uuid? values') "existing values should all be UUIDs") + (let [values (keep #(d/entity @conn [:block/uuid %]) values')] + (when (seq values) + (let [value-property-tx (map (fn [id] + {:db/id id + :block/closed-value-property (:db/id property)}) + (map :db/id values)) + property-tx (outliner-core/block-with-updated-at {:db/id (:db/id property)})] + (transact-with-op! conn (cons property-tx value-property-tx) + {:outliner-op :save-blocks})))))))))) (defn delete-closed-value! "Returns true when deleted or if not deleted displays warning and returns false" [conn property-id value-block-id] - (when (or (nil? property-id) - (nil? value-block-id)) - (throw (ex-info "empty property-id or value-block-id when delete-closed-value!" - {:property-id property-id - :value-block-id value-block-id}))) - (when-let [value-block (d/entity @conn value-block-id)] - (if (ldb/built-in? value-block) - (throw (ex-info "The choice can't be deleted" - {:type :notification - :payload {:message "The choice can't be deleted because it's built-in." - :type :warning}})) - (let [tx-data (conj (:tx-data (outliner-core/delete-blocks @conn [value-block] {:hard-retract? true})) - (outliner-core/block-with-updated-at {:db/id property-id}))] - (ldb/transact! conn tx-data))))) + (with-op-entry + [:delete-closed-value [property-id value-block-id]] + (fn [] + (when (or (nil? property-id) + (nil? value-block-id)) + (throw (ex-info "empty property-id or value-block-id when delete-closed-value!" + {:property-id property-id + :value-block-id value-block-id}))) + (when-let [value-block (d/entity @conn value-block-id)] + (if (ldb/built-in? value-block) + (throw (ex-info "The choice can't be deleted" + {:type :notification + :payload {:message "The choice can't be deleted because it's built-in." + :type :warning}})) + (let [tx-data (conj (:tx-data (outliner-core/delete-blocks @conn [value-block] {})) + (outliner-core/block-with-updated-at {:db/id property-id}))] + (transact-with-op! conn tx-data {}))))))) (defn class-add-property! [conn class-id property-id] - (when-not (contains? #{:logseq.property/empty-placeholder} property-id) - (when-let [class (d/entity @conn class-id)] - (if (ldb/class? class) - (ldb/transact! conn - [[:db/add (:db/id class) :logseq.property.class/properties property-id]] - {:outliner-op :save-block}) - (throw (ex-info "Can't add a property to a block that isn't a class" - {:class-id class-id :property-id property-id})))))) + (with-op-entry + [:class-add-property [class-id property-id]] + (fn [] + (when-not (contains? #{:logseq.property/empty-placeholder} property-id) + (when-let [class (d/entity @conn class-id)] + (if (ldb/class? class) + (transact-with-op! conn + [[:db/add (:db/id class) :logseq.property.class/properties property-id]] + {:outliner-op :save-block}) + (throw (ex-info "Can't add a property to a block that isn't a class" + {:class-id class-id :property-id property-id})))))))) (defn class-remove-property! [conn class-id property-id] - (when-let [class (d/entity @conn class-id)] - (when (ldb/class? class) - (when-let [property (d/entity @conn property-id)] - (when-not (ldb/built-in-class-property? class property) - (ldb/transact! conn [[:db/retract (:db/id class) :logseq.property.class/properties property-id]] - {:outliner-op :save-block})))))) + (with-op-entry + [:class-remove-property [class-id property-id]] + (fn [] + (when-let [class (d/entity @conn class-id)] + (when (ldb/class? class) + (when-let [property (d/entity @conn property-id)] + (when-not (ldb/built-in-class-property? class property) + (transact-with-op! conn + [[:db/retract (:db/id class) :logseq.property.class/properties property-id]] + {:outliner-op :save-block})))))))) diff --git a/deps/outliner/src/logseq/outliner/recycle.cljs b/deps/outliner/src/logseq/outliner/recycle.cljs index 6c281d2726..7211e32a07 100644 --- a/deps/outliner/src/logseq/outliner/recycle.cljs +++ b/deps/outliner/src/logseq/outliner/recycle.cljs @@ -8,7 +8,7 @@ [logseq.db.common.order :as db-order])) (def ^:private recycle-page-title "Recycle") -(def ^:private retention-ms (* 60 24 3600 1000)) +(def ^:private retention-ms (* 30 24 3600 1000)) (def gc-interval-ms (* 24 3600 1000)) (defn- recycled? diff --git a/deps/outliner/src/logseq/outliner/transaction.cljc b/deps/outliner/src/logseq/outliner/transaction.cljc index 83615bdb06..1e19396e0d 100644 --- a/deps/outliner/src/logseq/outliner/transaction.cljc +++ b/deps/outliner/src/logseq/outliner/transaction.cljc @@ -3,17 +3,17 @@ transient state from logseq.outliner.core" #?(:cljs (:require-macros [logseq.outliner.transaction]))) -(defmacro ^:api with-temp-conn-batch +(defmacro ^:api with-batch-tx [conn opts & body] - (let [temp-conn-sym (gensym "temp-conn__")] - `(logseq.db/transact-with-temp-conn! + (let [conn-sym (gensym "conn__")] + `(logseq.db/batch-transact-with-temp-conn! ~conn (dissoc ~opts :additional-tx :transact-opts :current-block) - (fn [~temp-conn-sym _*batch-tx-data#] - (let [~conn ~temp-conn-sym] + (fn [~conn-sym] + (let [~conn ~conn-sym] ~@body (when (seq (:additional-tx ~opts)) - (logseq.db/transact! ~temp-conn-sym (:additional-tx ~opts) {}))))))) + (logseq.db/transact! ~conn-sym (:additional-tx ~opts) {}))))))) (defmacro ^:api transact! "Batch all the transactions in `body` to a single transaction. @@ -33,7 +33,7 @@ (delete-blocks! ...))" [opts & body] `(let [~'conn (:conn (:transact-opts ~opts))] - (logseq.outliner.transaction/with-temp-conn-batch + (logseq.outliner.transaction/with-batch-tx ~'conn ~opts ~@body))) diff --git a/deps/outliner/src/logseq/outliner/tx_meta.cljs b/deps/outliner/src/logseq/outliner/tx_meta.cljs new file mode 100644 index 0000000000..1966faa180 --- /dev/null +++ b/deps/outliner/src/logseq/outliner/tx_meta.cljs @@ -0,0 +1,11 @@ +(ns logseq.outliner.tx-meta + "Helpers for normalizing tx metadata with explicit outliner op entries.") + +(def ^:dynamic *outliner-op-entry* nil) + +(defn ensure-outliner-ops + [tx-meta fallback-op-entry] + (let [entry (or *outliner-op-entry* fallback-op-entry)] + (cond-> (or tx-meta {}) + (and entry (nil? (:outliner-ops tx-meta))) + (assoc :outliner-ops [entry])))) diff --git a/deps/outliner/test/logseq/outliner/core_test.cljs b/deps/outliner/test/logseq/outliner/core_test.cljs index 8bb95c4db0..276f144829 100644 --- a/deps/outliner/test/logseq/outliner/core_test.cljs +++ b/deps/outliner/test/logseq/outliner/core_test.cljs @@ -2,39 +2,30 @@ (:require [cljs.test :refer [deftest is testing]] [datascript.core :as d] [logseq.db :as ldb] - [logseq.db.common.entity-plus :as entity-plus] [logseq.db.test.helper :as db-test] [logseq.outliner.core :as outliner-core])) (deftest test-delete-block-with-default-property - (testing "Delete block with default property moves the block to recycle" + (testing "Delete block with default property hard retracts the block subtree" (let [conn (db-test/create-conn-with-blocks [{:page {:block/title "page1"} :blocks [{:block/title "b1" :build/properties {:default "test block"}}]}]) block (db-test/find-block-by-content @conn "b1")] (outliner-core/delete-blocks! conn [block] {}) - (let [block' (db-test/find-block-by-content @conn "b1") - property-value (:user.property/default block') - recycle-page (ldb/get-built-in-page @conn "Recycle")] - (is (some? block')) - (is (some? property-value)) - (is (integer? (:logseq.property/deleted-at block'))) - (is (= (:db/id recycle-page) (:db/id (:block/page block')))) - (is (= (:db/id recycle-page) (:db/id (:block/page property-value)))))))) + (is (nil? (db-test/find-block-by-content @conn "b1")))))) (deftest test-delete-page-with-outliner-core - (testing "Pages shouldn't be deleted through outliner-core/delete-blocks" + (testing "Deleting pages through outliner-core/delete-blocks detaches page position only" (let [conn (db-test/create-conn-with-blocks [{:page {:block/title "page1"} :blocks [{:block/title "b1"}]} {:page {:block/title "page2"} :blocks [{:block/title "b3"} {:block/title "b4"}]}]) - page1 (ldb/get-page @conn "page1") page2 (ldb/get-page @conn "page2") _ (d/transact! conn [{:db/id (:db/id page2) :block/order "a1" - :block/parent (:db/id page1)}]) + :block/parent (:db/id (ldb/get-page @conn "page1"))}]) b3 (db-test/find-block-by-content @conn "b3") b4 (db-test/find-block-by-content @conn "b4")] (outliner-core/delete-blocks! conn [b3 b4 page2] {}) @@ -42,35 +33,20 @@ (is (some? (db-test/find-block-by-content @conn "b4"))) (let [page2' (ldb/get-page @conn "page2")] (is (= "page2" (:block/title page2'))) - (is (= (:db/id page1) (:db/id (:block/parent page2')))) - (is (= "a1" (:block/order page2'))))))) + (is (nil? (:block/parent page2'))) + (is (nil? (:block/order page2'))))))) -(deftest delete-blocks-moves-subtree-to-recycle +(deftest delete-blocks-hard-retracts-subtree (let [user-uuid (random-uuid) conn (db-test/create-conn-with-blocks [{:page {:block/title "page1"} :blocks [{:block/title "parent" :build/children [{:block/title "child"}]}]}]) - page (ldb/get-page @conn "page1") - parent (db-test/find-block-by-content @conn "parent") - original-order (:block/order parent)] + parent (db-test/find-block-by-content @conn "parent")] (d/transact! conn [{:block/uuid user-uuid :block/title "Alice"}]) (outliner-core/delete-blocks! conn [parent] {:deleted-by-uuid user-uuid}) (let [parent' (db-test/find-block-by-content @conn "parent") - child' (db-test/find-block-by-content @conn "child") - properties (entity-plus/lookup-kv-then-entity parent' :block/properties) - recycle-page (ldb/get-built-in-page @conn "Recycle")] - (is (some? parent')) - (is (some? child')) - (is (= (:block/uuid recycle-page) (:block/uuid (:block/parent parent')))) - (is (= (:block/uuid recycle-page) (:block/uuid (:block/page parent')))) - (is (integer? (:logseq.property/deleted-at parent'))) - (is (= user-uuid - (:block/uuid (:logseq.property/deleted-by-ref properties)))) - (is (= (:block/uuid page) - (:block/uuid (:logseq.property.recycle/original-page properties)))) - (is (= original-order (:logseq.property.recycle/original-order parent'))) - (is (= (:block/uuid parent') (:block/uuid (:block/parent child')))) - (is (= (:block/uuid recycle-page) (:block/uuid (:block/page child')))) - (is (nil? (:logseq.property/deleted-at child')))))) + child' (db-test/find-block-by-content @conn "child")] + (is (nil? parent')) + (is (nil? child'))))) diff --git a/deps/outliner/test/logseq/outliner/op_construct_test.cljs b/deps/outliner/test/logseq/outliner/op_construct_test.cljs new file mode 100644 index 0000000000..4fddf4d8a9 --- /dev/null +++ b/deps/outliner/test/logseq/outliner/op_construct_test.cljs @@ -0,0 +1,404 @@ +(ns logseq.outliner.op-construct-test + (:require [cljs.test :refer [deftest is testing]] + [datascript.core :as d] + [logseq.common.util.date-time :as date-time-util] + [logseq.common.uuid :as common-uuid] + [logseq.db :as ldb] + [logseq.db.test.helper :as db-test] + [logseq.outliner.core :as outliner-core] + [logseq.outliner.op.construct :as op-construct])) + +(defn- run-direct-outdent + [conn block] + (let [{:keys [tx-data]} + (#'outliner-core/indent-outdent-blocks + conn [block] false + :parent-original nil + :logical-outdenting? nil) + tx-report (d/with @conn tx-data {})] + {:tx-data (:tx-data tx-report) + :db-after (:db-after tx-report)})) + +(deftest derive-history-outliner-ops-canonicalizes-create-page-and-builds-delete-inverse-test + (testing "create-page forward op keeps created uuid and reverse op deletes that page" + (let [conn (db-test/create-conn-with-blocks {:pages-and-blocks []}) + page-uuid (random-uuid) + tx-data [{:e 1 :a :block/title :v "Created Page" :added true} + {:e 1 :a :block/uuid :v page-uuid :added true}] + tx-meta {:outliner-op :create-page + :outliner-ops [[:create-page ["Created Page" + {:redirect? false + :split-namespace? true + :tags ()}]]]} + {:keys [forward-outliner-ops inverse-outliner-ops]} + (op-construct/derive-history-outliner-ops @conn @conn tx-data tx-meta)] + (is (= :create-page (ffirst forward-outliner-ops))) + (is (= page-uuid (get-in forward-outliner-ops [0 1 1 :uuid]))) + (is (= [[:delete-page [page-uuid {}]]] + inverse-outliner-ops))))) + +(deftest derive-history-outliner-ops-collapses-mixed-stream-to-transact-placeholder-test + (testing "mixed semantic/non-semantic ops collapse to transact placeholder" + (let [conn (db-test/create-conn-with-blocks + {:pages-and-blocks + [{:page {:block/title "page"} + :blocks [{:block/title "child"}]}]}) + child (db-test/find-block-by-content @conn "child") + tx-meta {:outliner-op :save-block + :outliner-ops [[:save-block [{:block/uuid (:block/uuid child) + :block/title "changed"} {}]] + [:transact nil]]} + {:keys [forward-outliner-ops inverse-outliner-ops]} + (op-construct/derive-history-outliner-ops @conn @conn [] tx-meta)] + (is (= [[:transact nil]] forward-outliner-ops)) + (is (nil? inverse-outliner-ops))))) + +(deftest derive-history-outliner-ops-handles-replace-empty-target-insert-inverse-test + (testing "replace-empty-target insert keeps source uuid and inverse deletes target placeholder" + (let [conn (db-test/create-conn-with-blocks + {:pages-and-blocks + [{:page {:block/title "page"} + :blocks [{:block/title ""}]}]}) + empty-target (db-test/find-block-by-content @conn "") + parent-uuid (random-uuid) + child-uuid (random-uuid) + tx-meta {:outliner-op :insert-blocks + :outliner-ops [[:insert-blocks [[{:block/uuid parent-uuid + :block/title "paste parent"} + {:block/uuid child-uuid + :block/title "paste child" + :block/parent [:block/uuid parent-uuid]}] + (:db/id empty-target) + {:sibling? true + :replace-empty-target? true + :outliner-op :paste}]]]} + {:keys [forward-outliner-ops inverse-outliner-ops]} + (op-construct/derive-history-outliner-ops @conn @conn [] tx-meta)] + (is (= parent-uuid + (get-in forward-outliner-ops [0 1 0 0 :block/uuid]))) + (is (= true (get-in forward-outliner-ops [0 1 2 :keep-uuid?]))) + (is (some #(and (= :delete-blocks (first %)) + (= [[:block/uuid (:block/uuid empty-target)]] + (vec (get-in % [1 0])))) + (remove nil? inverse-outliner-ops))) + (is (not-any? #(= :save-block (first %)) + (remove nil? inverse-outliner-ops)))))) + +(deftest derive-history-outliner-ops-builds-upsert-property-inverse-delete-page-test + (testing "upsert-property with qualified keyword builds delete-page inverse" + (let [conn (db-test/create-conn-with-blocks {:pages-and-blocks []}) + property-id :user.property/custom-prop + tx-meta {:outliner-op :upsert-property + :outliner-ops [[:upsert-property [property-id + {:logseq.property/type :default} + {:property-name "custom-prop"}]]]} + {:keys [inverse-outliner-ops]} + (op-construct/derive-history-outliner-ops @conn @conn [] tx-meta) + expected-page-uuid (common-uuid/gen-uuid :db-ident-block-uuid property-id)] + (is (= [[:delete-page [expected-page-uuid {}]]] + inverse-outliner-ops))))) + +(deftest derive-history-outliner-ops-delete-blocks-inverse-avoids-self-target-test + (testing "delete-blocks inverse falls back to parent target when left sibling resolves to self" + (let [conn (db-test/create-conn-with-blocks + {:pages-and-blocks + [{:page {:block/title "page"} + :blocks [{:block/title "parent" + :build/children [{:block/title "child"}]}]}]}) + child (db-test/find-block-by-content @conn "child") + child-id (:db/id child) + child-uuid (:block/uuid child) + parent-uuid (some-> child :block/parent :block/uuid) + tx-meta {:outliner-op :delete-blocks + :outliner-ops [[:delete-blocks [[child-id] {}]]]}] + ;; Simulate stale sibling lookup returning the same entity as the deleted root. + (with-redefs [ldb/get-left-sibling (fn [_] child)] + (let [{:keys [inverse-outliner-ops]} + (op-construct/derive-history-outliner-ops @conn @conn [] tx-meta) + insert-op (first inverse-outliner-ops)] + (is (= :insert-blocks (first insert-op))) + (is (= [:block/uuid parent-uuid] + (get-in insert-op [1 1]))) + (is (= false (get-in insert-op [1 2 :sibling?]))) + (is (not= [:block/uuid child-uuid] + (get-in insert-op [1 1])))))))) + +(deftest derive-history-outliner-ops-builds-delete-page-inverse-for-class-property-and-today-page-test + (testing "delete-page inverse restores hard-retracted class/property/today pages with stable db/ident" + (let [today (date-time-util/ms->journal-day (js/Date.)) + conn (db-test/create-conn-with-blocks + {:classes {:Movie {}} + :properties {:rating {:logseq.property/type :number}} + :pages-and-blocks [{:page {:build/journal today} + :blocks [{:block/title "today child"}]}]}) + class-page (ldb/get-page @conn "Movie") + property-page (d/entity @conn :user.property/rating) + today-page (db-test/find-journal-by-journal-day @conn today) + today-child (db-test/find-block-by-content @conn "today child") + class-inverse (:inverse-outliner-ops + (op-construct/derive-history-outliner-ops + @conn @conn [] {:outliner-op :delete-page + :outliner-ops [[:delete-page [(:block/uuid class-page) {}]]]})) + property-inverse (:inverse-outliner-ops + (op-construct/derive-history-outliner-ops + @conn @conn [] {:outliner-op :delete-page + :outliner-ops [[:delete-page [(:block/uuid property-page) {}]]]})) + today-inverse (:inverse-outliner-ops + (op-construct/derive-history-outliner-ops + @conn @conn [] {:outliner-op :delete-page + :outliner-ops [[:delete-page [(:block/uuid today-page) {}]]]}))] + (is (some #(= :create-page (first %)) class-inverse)) + (is (some #(= :save-block (first %)) class-inverse)) + (is (= (:db/ident class-page) + (get-in (some #(when (= :save-block (first %)) %) class-inverse) + [1 0 :db/ident]))) + + (is (some #(= :upsert-property (first %)) property-inverse)) + (is (some #(= :save-block (first %)) property-inverse)) + (is (= (:db/ident property-page) + (get-in (some #(when (= :save-block (first %)) %) property-inverse) + [1 0 :db/ident]))) + + (is (not-any? #(= :restore-recycled (first %)) today-inverse)) + (let [today-insert-op (some #(when (= :insert-blocks (first %)) %) today-inverse)] + (is (some? today-insert-op)) + (is (= (:block/uuid today-page) + (second (get-in today-insert-op [1 1])))) + (is (= (:block/uuid today-child) + (get-in today-insert-op [1 0 0 :block/uuid]))))))) + +(deftest derive-history-outliner-ops-builds-inverse-for-all-supported-ops-test + (let [conn (db-test/create-conn-with-blocks + {:classes {:c1 {:build/class-properties [:p1]}} + :properties {:p1 {:logseq.property/type :default}} + :pages-and-blocks + [{:page {:block/title "page"} + :blocks [{:block/title "parent" + :build/children [{:block/title "child-a"} + {:block/title "child-b"}]} + {:block/title "prop-block-1" + :build/properties {:p1 "before-1"}} + {:block/title "prop-block-2"}]}]}) + page (db-test/find-page-by-title @conn "page") + parent (db-test/find-block-by-content @conn "parent") + child-a (db-test/find-block-by-content @conn "child-a") + child-b (db-test/find-block-by-content @conn "child-b") + prop-block-1 (db-test/find-block-by-content @conn "prop-block-1") + prop-block-2 (db-test/find-block-by-content @conn "prop-block-2") + class-id (:db/id (d/entity @conn :user.class/c1)) + class-uuid (:block/uuid (d/entity @conn class-id)) + property-id (:db/id (d/entity @conn :user.property/p1)) + property-page-uuid (:block/uuid (d/entity @conn property-id)) + prop-value-1-id (:db/id (:user.property/p1 prop-block-1))] + (testing ":save-block" + (let [{:keys [inverse-outliner-ops]} + (op-construct/derive-history-outliner-ops + @conn @conn [] {:outliner-op :save-block + :outliner-ops [[:save-block [{:block/uuid (:block/uuid child-a) + :block/title "changed"} {}]]]})] + (is (= :save-block (ffirst inverse-outliner-ops))) + (is (= (:block/uuid child-a) + (get-in inverse-outliner-ops [0 1 0 :block/uuid]))))) + + (testing ":insert-blocks" + (let [inserted-uuid (random-uuid) + tx-data [{:e 999999 :a :block/uuid :v inserted-uuid :added true}] + {:keys [inverse-outliner-ops]} + (op-construct/derive-history-outliner-ops + @conn @conn tx-data {:outliner-op :insert-blocks + :outliner-ops [[:insert-blocks [[{:block/uuid inserted-uuid + :block/title "new"}] + (:db/id parent) + {:sibling? false}]]]})] + (is (= :delete-blocks (ffirst inverse-outliner-ops))) + (is (= [[:block/uuid inserted-uuid]] + (vec (get-in inverse-outliner-ops [0 1 0])))))) + + (testing ":move-blocks" + (let [{:keys [inverse-outliner-ops]} + (op-construct/derive-history-outliner-ops + @conn @conn [] {:outliner-op :move-blocks + :outliner-ops [[:move-blocks [[(:db/id child-b)] + (:db/id parent) + {:sibling? false}]]]})] + (is (= :move-blocks (ffirst inverse-outliner-ops))) + (is (= [[:block/uuid (:block/uuid child-b)]] + (get-in inverse-outliner-ops [0 1 0]))))) + + (testing ":delete-blocks" + (let [{:keys [inverse-outliner-ops]} + (op-construct/derive-history-outliner-ops + @conn @conn [] {:outliner-op :delete-blocks + :outliner-ops [[:delete-blocks [[(:db/id child-b)] {}]]]})] + (is (= :insert-blocks (ffirst inverse-outliner-ops))) + (is (= (:block/uuid child-b) + (get-in inverse-outliner-ops [0 1 0 0 :block/uuid]))))) + + (testing ":create-page" + (let [page-uuid (random-uuid) + tx-data [{:e 1 :a :block/title :v "P2" :added true} + {:e 1 :a :block/uuid :v page-uuid :added true}] + {:keys [inverse-outliner-ops]} + (op-construct/derive-history-outliner-ops + @conn @conn tx-data {:outliner-op :create-page + :outliner-ops [[:create-page ["P2" {:redirect? false}]]]})] + (is (= [[:delete-page [page-uuid {}]]] + inverse-outliner-ops)))) + + (testing ":delete-page" + (let [{:keys [inverse-outliner-ops]} + (op-construct/derive-history-outliner-ops + @conn @conn [] {:outliner-op :delete-page + :outliner-ops [[:delete-page [(:block/uuid page) {}]]]})] + (is (= [[:restore-recycled [(:block/uuid page)]]] + inverse-outliner-ops)))) + + (testing ":set-block-property" + (let [{:keys [inverse-outliner-ops]} + (op-construct/derive-history-outliner-ops + @conn @conn [] {:outliner-op :set-block-property + :outliner-ops [[:set-block-property [(:db/id prop-block-1) + :user.property/p1 + "new-value"]]]})] + (is (= :set-block-property (ffirst inverse-outliner-ops))) + (is (= [:block/uuid (:block/uuid prop-block-1)] + (get-in inverse-outliner-ops [0 1 0]))) + (is (= :user.property/p1 (get-in inverse-outliner-ops [0 1 1]))) + (is (= prop-value-1-id (get-in inverse-outliner-ops [0 1 2 :db/id]))))) + + (testing ":remove-block-property" + (let [{:keys [inverse-outliner-ops]} + (op-construct/derive-history-outliner-ops + @conn @conn [] {:outliner-op :remove-block-property + :outliner-ops [[:remove-block-property [(:db/id prop-block-1) + :user.property/p1]]]})] + (is (= :set-block-property (ffirst inverse-outliner-ops))) + (is (= [:block/uuid (:block/uuid prop-block-1)] + (get-in inverse-outliner-ops [0 1 0]))) + (is (= :user.property/p1 (get-in inverse-outliner-ops [0 1 1]))) + (is (= prop-value-1-id (get-in inverse-outliner-ops [0 1 2 :db/id]))))) + + (testing ":batch-set-property" + (let [{:keys [inverse-outliner-ops]} + (op-construct/derive-history-outliner-ops + @conn @conn [] {:outliner-op :batch-set-property + :outliner-ops [[:batch-set-property [[(:db/id prop-block-1) + (:db/id prop-block-2)] + :user.property/p1 + "new-value" + {}]]]})] + (is (= 2 (count inverse-outliner-ops))) + (is (= :set-block-property (ffirst inverse-outliner-ops))) + (is (= :remove-block-property (ffirst (rest inverse-outliner-ops)))))) + + (testing ":batch-remove-property" + (let [{:keys [inverse-outliner-ops]} + (op-construct/derive-history-outliner-ops + @conn @conn [] {:outliner-op :batch-remove-property + :outliner-ops [[:batch-remove-property [[(:db/id prop-block-1) + (:db/id prop-block-2)] + :user.property/p1]]]})] + (is (= 1 (count inverse-outliner-ops))) + (is (= :set-block-property (ffirst inverse-outliner-ops))) + (is (= [:block/uuid (:block/uuid prop-block-1)] + (get-in inverse-outliner-ops [0 1 0]))) + (is (= :user.property/p1 (get-in inverse-outliner-ops [0 1 1]))) + (is (= prop-value-1-id (get-in inverse-outliner-ops [0 1 2 :db/id]))))) + + (testing ":class-add-property" + (let [{:keys [inverse-outliner-ops]} + (op-construct/derive-history-outliner-ops + @conn @conn [] {:outliner-op :class-add-property + :outliner-ops [[:class-add-property [class-id property-id]]]})] + (is (= [[:class-remove-property [[:block/uuid class-uuid] + [:block/uuid property-page-uuid]]]] + inverse-outliner-ops)))) + + (testing ":class-remove-property" + (let [{:keys [inverse-outliner-ops]} + (op-construct/derive-history-outliner-ops + @conn @conn [] {:outliner-op :class-remove-property + :outliner-ops [[:class-remove-property [class-id property-id]]]})] + (is (= [[:class-add-property [[:block/uuid class-uuid] + [:block/uuid property-page-uuid]]]] + inverse-outliner-ops)))) + + (testing ":upsert-property" + (let [property-ident :user.property/test-inverse + expected-page-uuid (common-uuid/gen-uuid :db-ident-block-uuid property-ident) + {:keys [inverse-outliner-ops]} + (op-construct/derive-history-outliner-ops + @conn @conn [] {:outliner-op :upsert-property + :outliner-ops [[:upsert-property [property-ident + {:logseq.property/type :default} + {:property-name "test-inverse"}]]]})] + (is (= [[:delete-page [expected-page-uuid {}]]] + inverse-outliner-ops)))))) + +(deftest build-history-action-metadata-direct-outdent-builds-indent-outdent-forward-and-inverse-test + (testing "direct outdent keeps canonical indent-outdent forward and inverse ops" + (let [conn (db-test/create-conn-with-blocks + {:pages-and-blocks + [{:page {:block/title "page"} + :blocks [{:block/title "parent" + :build/children [{:block/title "child-1"} + {:block/title "child-2"} + {:block/title "child-3"}]}]}]}) + child-3 (db-test/find-block-by-content @conn "child-3") + {:keys [tx-data db-after]} (run-direct-outdent conn child-3) + tx-meta {:outliner-op :move-blocks + :outliner-ops [[:indent-outdent-blocks [[(:db/id child-3)] + false + {:parent-original nil + :logical-outdenting? nil}]]]} + {:keys [forward-outliner-ops inverse-outliner-ops]} + (op-construct/derive-history-outliner-ops @conn db-after tx-data tx-meta)] + (is (= [[:indent-outdent-blocks [[[:block/uuid (:block/uuid child-3)]] + false + {:parent-original nil + :logical-outdenting? nil}]]] + forward-outliner-ops)) + (is (= [[:indent-outdent-blocks [[[:block/uuid (:block/uuid child-3)]] + true + {:parent-original nil + :logical-outdenting? nil}]]] + inverse-outliner-ops))))) + +(deftest derive-history-outliner-ops-direct-outdent-with-extra-moved-blocks-keeps-semantic-ops-test + (testing "direct outdent keeps semantic indent-outdent op and inverse" + (let [conn (db-test/create-conn-with-blocks + {:pages-and-blocks + [{:page {:block/title "page"} + :blocks [{:block/title "parent" + :build/children [{:block/title "child-1"} + {:block/title "child-2"} + {:block/title "child-3"}]}]}]}) + child-2 (db-test/find-block-by-content @conn "child-2") + {:keys [tx-data db-after]} (run-direct-outdent conn child-2) + tx-meta {:outliner-op :move-blocks + :outliner-ops [[:indent-outdent-blocks [[(:db/id child-2)] + false + {:parent-original nil + :logical-outdenting? nil}]]]} + {:keys [forward-outliner-ops inverse-outliner-ops]} + (op-construct/derive-history-outliner-ops @conn db-after tx-data tx-meta)] + (is (= [[:indent-outdent-blocks [[[:block/uuid (:block/uuid child-2)]] + false + {:parent-original nil + :logical-outdenting? nil}]]] + forward-outliner-ops)) + (is (= [[:indent-outdent-blocks [[[:block/uuid (:block/uuid child-2)]] + true + {:parent-original nil + :logical-outdenting? nil}]]] + inverse-outliner-ops))))) + +(deftest build-history-action-metadata-non-semantic-outliner-op-does-not-throw-test + (testing "non-semantic outliner-op with transact placeholder should not fail strict semantic validation" + (let [conn (db-test/create-conn-with-blocks {:pages-and-blocks []}) + tx-meta {:outliner-op :restore-recycled + :outliner-ops [[:transact nil]]} + result (op-construct/derive-history-outliner-ops @conn @conn [] tx-meta)] + (is (= [[:transact nil]] + (:forward-outliner-ops result))) + (is (nil? (:inverse-outliner-ops result)))))) diff --git a/deps/outliner/test/logseq/outliner/page_test.cljs b/deps/outliner/test/logseq/outliner/page_test.cljs index c9b8aac878..fad1545c0d 100644 --- a/deps/outliner/test/logseq/outliner/page_test.cljs +++ b/deps/outliner/test/logseq/outliner/page_test.cljs @@ -116,6 +116,9 @@ (is (some? b1')) (is (= (:block/uuid recycle-page) (:block/uuid (:block/parent d1')))) (is (integer? (:logseq.property/deleted-at d1'))) + (is (nil? (:block/raw-title b1'))) + (is (contains? (set (map :db/id (:block/refs b1'))) + (:db/id d1))) (is (= (:block/uuid d1') (:block/uuid (:block/page b1'))))))) (deftest delete-class-page-hard-retracts-page-tree diff --git a/deps/outliner/test/logseq/outliner/recycle_test.cljs b/deps/outliner/test/logseq/outliner/recycle_test.cljs index 08e6a846ec..2f60044d7b 100644 --- a/deps/outliner/test/logseq/outliner/recycle_test.cljs +++ b/deps/outliner/test/logseq/outliner/recycle_test.cljs @@ -1,50 +1,9 @@ (ns logseq.outliner.recycle-test (:require [cljs.test :refer [deftest is]] - [datascript.core :as d] [logseq.db :as ldb] [logseq.db.test.helper :as db-test] - [logseq.outliner.core :as outliner-core] [logseq.outliner.recycle :as recycle])) -(deftest restore-recycled-block-returns-subtree-to-original-location - (let [conn (db-test/create-conn-with-blocks - [{:page {:block/title "page1"} - :blocks [{:block/title "parent" - :build/children [{:block/title "child"}]} - {:block/title "sibling"}]}]) - page (ldb/get-page @conn "page1") - parent (db-test/find-block-by-content @conn "parent")] - (outliner-core/delete-blocks! conn [parent] {}) - (recycle/restore! conn (:block/uuid parent)) - (let [parent' (db-test/find-block-by-content @conn "parent") - child' (db-test/find-block-by-content @conn "child")] - (is (= (:block/uuid page) (:block/uuid (:block/parent parent')))) - (is (= (:block/uuid page) (:block/uuid (:block/page parent')))) - (is (= (:block/uuid parent') (:block/uuid (:block/parent child')))) - (is (= (:block/uuid page) (:block/uuid (:block/page child')))) - (is (nil? (:logseq.property/deleted-at parent'))) - (is (nil? (:logseq.property/deleted-by-ref parent'))) - (is (nil? (:logseq.property.recycle/original-parent parent'))) - (is (nil? (:logseq.property.recycle/original-page parent'))) - (is (nil? (:logseq.property.recycle/original-order parent')))))) - -(deftest restore-recycled-block-falls-back-to-page-root-when-original-parent-is-unavailable - (let [conn (db-test/create-conn-with-blocks - [{:page {:block/title "page1"} - :blocks [{:block/title "parent" - :build/children [{:block/title "child"}]} - {:block/title "sibling"}]}]) - page (ldb/get-page @conn "page1") - parent (db-test/find-block-by-content @conn "parent") - child (db-test/find-block-by-content @conn "child")] - (outliner-core/delete-blocks! conn [child] {}) - (outliner-core/delete-blocks! conn [parent] {}) - (recycle/restore! conn (:block/uuid child)) - (let [child' (db-test/find-block-by-content @conn "child")] - (is (= (:block/uuid page) (:block/uuid (:block/parent child')))) - (is (= (:block/uuid page) (:block/uuid (:block/page child')))) - (is (nil? (:logseq.property/deleted-at child')))))) - (deftest restore-recycled-page-removes-recycle-parent (let [conn (db-test/create-conn-with-blocks [{:page {:block/title "page1"} @@ -57,19 +16,3 @@ (is (nil? (:block/parent page'))) (is (nil? (:logseq.property/deleted-at page'))) (is (nil? (:logseq.property.recycle/original-parent page')))))) - -(deftest gc-retracts-recycled-subtrees-older-than-retention-window - (let [now-ms 1000 - old-ms (- now-ms (* 61 24 3600 1000)) - conn (db-test/create-conn-with-blocks - [{:page {:block/title "page1"} - :blocks [{:block/title "parent" - :build/children [{:block/title "child"}]}]}]) - parent (db-test/find-block-by-content @conn "parent") - child (db-test/find-block-by-content @conn "child")] - (outliner-core/delete-blocks! conn [parent] {}) - (d/transact! conn [{:db/id (:db/id (db-test/find-block-by-content @conn "parent")) - :logseq.property/deleted-at old-ms}]) - (recycle/gc! conn {:now-ms now-ms}) - (is (nil? (d/entity @conn [:block/uuid (:block/uuid parent)]))) - (is (nil? (d/entity @conn [:block/uuid (:block/uuid child)]))))) diff --git a/docs/adr/0010-op-driven-client-rebase.md b/docs/adr/0010-op-driven-client-rebase.md new file mode 100644 index 0000000000..84d0d7f12e --- /dev/null +++ b/docs/adr/0010-op-driven-client-rebase.md @@ -0,0 +1,147 @@ +# ADR 0010: Canonical Op-Driven Client Rebase With Legacy Tx Surgery Isolation + +Date: 2026-03-19 +Status: Accepted + +## Context +Client sync rebase currently relies on custom tx-data surgery to keep pending +local changes alive after remote txs are applied. + +That approach has several problems: +- the logic is hard to reason about because it edits datoms instead of replaying + user intent +- it does not preserve intent well when a local change originated from a higher + level outliner action +- the helper set keeps growing with special cases for missing refs, structural + conflicts, and deleted entities +- the outliner-op surface is too large to replay directly without first + reducing it + +At the same time, not every outliner op needs to remain a first-class replay +operation. +Many ops can be safely represented by `:transact` when their tx-data is already +self-contained and does not depend on rerunning outliner logic. + +We also have existing persisted pending rows that only store tx-data and reverse +tx-data. +Those rows still need a compatibility path, but that path should not define the +new rebase architecture. + +## Decision +1. New pending local tx rows will persist semantic `:outliner-ops` in addition + to their existing tx-data payload. +2. Client rebase will become op-driven for all new pending rows: + - reverse local txs + - apply remote txs + - transform or drop stored local ops against the post-remote temp db + - replay the surviving ops to regenerate rebased tx-data +3. Reduce the replay-visible outliner-op surface to a small canonical set: + - `:insert-blocks` + - `:save-block` + - `:move-blocks` + - `:delete-blocks` + - `:transact` +4. Normalize higher-level ops into that canonical set before persistence. + Examples: + - `:indent-outdent-blocks` becomes `:move-blocks` + - `:move-blocks-up-down` becomes `:move-blocks` + - `:rename-page` becomes `:save-block` +5. Introduce an explicit safe-`:transact` classifier: + - if replaying tx-data directly is sufficient and does not require rerunning + outliner logic, persist the op as canonical `:transact` + - otherwise keep it as one of the canonical outliner replay ops + - expected canonical `:transact` cases include direct tx replay actions such + as undo, redo, import replay, reaction toggles, and property-value updates +6. Treat undo/redo as canonical `:transact` actions. + - persist the exact tx-data that the successful undo or redo applied + - keep the action atomic as one pending tx row + - do not reconstruct the original higher-level outliner ops that were + undone or redone +7. Treat `:batch-import-edn` as canonical `:transact` after successful local + execution. + - persist the exact tx-data that the import applied + - do not rerun import expansion during rebase +8. Keep the following op kinds as replay-visible semantic ops because they must + be reevaluated against current DB state: + - `:save-block` + - `:insert-blocks` + - `:move-blocks` + - `:delete-blocks` + - `:create-page` + - `:delete-page` + - `:upsert-property` +9. Stop using raw tx-data surgery in the normal rebase path for new rows. +10. Move the current tx-data surgery helpers into a dedicated legacy namespace. + That legacy namespace is only for compatibility handling of old persisted + pending rows that do not have stored `:outliner-ops`. +11. Add explicit owned-block filtering in the new op-driven rebase path. + Initially cover: + - reaction blocks owned by `:logseq.property.reaction/target` + - property history blocks owned by `:logseq.property.history/block` + - property history blocks whose effective owner disappears through deleted + `:logseq.property.history/ref-value` +12. If a local op creates or updates one of those owned blocks and the owning + block was deleted remotely, drop that op. +13. Treat each pending tx row as one user action and keep it atomic during + rebase. +14. If any op in a pending tx becomes invalid during rebase, drop the whole + pending tx rather than keeping a partial replay of that user action. +15. Keep this refactor client-only for now. + The sync wire format and server tx log remain unchanged. + +## Consequences +- Positive: + - Rebase reasons about user intent instead of datom accidents. + - The number of op kinds that rebase must understand becomes much smaller. + - Safe direct tx replay remains available through canonical `:transact` + without forcing every operation through outliner code. + - Undo/redo stays aligned with its real intent: replay the exact applied tx, + not rerun a reconstructed higher-level command. + - Import replay stays aligned with its real intent: preserve the exact local + import result rather than recomputing import expansion later. + - Legacy compatibility is isolated instead of contaminating the new design. + - Owned-block cleanup becomes an explicit semantic rule rather than another + tx-data patch. + - Rebase behavior stays aligned with the mental model that one pending tx is + one user action that either survives or is discarded as a whole. +- Negative: + - We must maintain a canonicalization layer from original outliner ops to the + reduced replay set. + - Some existing ops need careful classification to decide whether they are + safe `:transact` or must stay true outliner replays. + - For a transition period, the codebase will contain both the new rebase path + and the isolated legacy compatibility path. + - If one invalid op is grouped together with otherwise valid work in the same + pending tx, the whole user action will be lost during rebase. + +## Follow-up Constraints +- New pending tx producers must persist canonical `:outliner-ops`. +- New pending tx producers must preserve user-action boundaries because rebase + will treat each persisted tx row atomically. +- Canonicalization should happen when persisting local pending txs, not lazily + during rebase. +- Undo/redo producers should persist canonical `:transact` actions using the + exact tx-data they applied. +- Import producers should persist canonical `:transact` actions using the exact + tx-data they applied. +- The main sync apply namespace should not call legacy tx-surgery helpers for + new pending rows. +- The legacy namespace should be clearly named and easy to delete once old + pending-row compatibility is no longer needed. + +## Verification +- Add or update frontend worker db-sync coverage for: + - persistence of canonical `:outliner-ops` + - canonical reduction of `:indent-outdent-blocks` and + `:move-blocks-up-down` into `:move-blocks` + - safe `:transact` replay versus true outliner replay classification + - undo/redo persistence as atomic canonical `:transact` actions + - `:batch-import-edn` persistence as atomic canonical `:transact` + - `:rename-page` canonicalization to `:save-block` + - op-driven rebase preserving pending tx boundaries + - dropping owned reaction/history ops when their owner was deleted remotely + - dropping the whole pending tx when any op in that user action becomes + invalid + - routing legacy pending rows without stored ops through the legacy namespace +- Expected targeted command: + - `bb dev:test -v frontend.worker.db-sync-test` diff --git a/docs/adr/0011-undo-redo-semantic-inverse-op-persistence.md b/docs/adr/0011-undo-redo-semantic-inverse-op-persistence.md new file mode 100644 index 0000000000..ddcdeaf46d --- /dev/null +++ b/docs/adr/0011-undo-redo-semantic-inverse-op-persistence.md @@ -0,0 +1,378 @@ +# ADR 0011: Undo/Redo Semantic Inverse-Op Persistence and Local-Op Referenced UI History + +Date: 2026-03-21 +Status: Proposed + +## Context +ADR 0010 deliberately classified undo and redo as canonical `:transact` actions. + +That choice kept pending-row persistence simple, but it also preserved undo and redo as raw datom replay rather than preserving their intent. + +That tradeoff is now the main weakness in the sync rebase model. + +During client rebase, pending local actions are reversed, remote txs are applied, and then pending local actions are replayed. + +That architecture works well when the persisted action is a semantic op such as `:save-block`, `:insert-blocks`, or `:move-blocks`. + +It works poorly when the persisted action is a raw undo or redo tx payload because the replay logic can no longer distinguish commutative remote updates from true semantic conflicts. + +The current undo/redo design also duplicates DB history across two places. + +The main thread owns the undo and redo stacks because it stores UI state such as route state, editor cursor, and focus context. + +At the same time, the worker already persists local DB actions in client-ops storage with a stable `tx-id`. + +Those two histories are related but not unified. + +As a result, sync rebase, undo/redo, and pending local action persistence can drift apart. + +The user action identity is already represented in the worker by `:db-sync/tx-id`, but the UI undo stack still stores the DB tx payload itself instead of referencing that persisted action. + +This ADR defines a plan to make the client-op row the source of truth for DB history while keeping the main thread as the source of truth for UI state. + +Constraints: +- The main thread must continue to own UI-only undo state. +- The sync wire protocol should remain unchanged. +- Existing persisted pending rows without semantic metadata are intentionally out of scope for this change. +- Rebase should continue to preserve one logical pending row as one user action. +- The design must preserve redo, not just undo. + +Related ADRs: +- Builds on ADR 0002. +- Builds on ADR 0010. +- Supersedes ADR 0010 decision 6, which treated undo/redo as canonical `:transact`. + +## Problem Summary +The current model has three tightly coupled problems. + +First, undo and redo are persisted as raw tx replay. + +That makes rebase unable to reason about their semantics. + +Second, the undo stack on the main thread stores DB payloads directly instead of referencing the persisted local action row in worker storage. + +That means the UI history and worker history are not guaranteed to stay aligned. + +Third, the current worker pending-row schema persists forward tx data and reverse tx data, but it does not persist both semantic forward ops and semantic inverse ops as first-class action metadata. + +That prevents robust rebase and robust redo from using the same action identity. + +The result is that a benign remote update such as a title edit can still force undo/redo to fall back to raw replay, which is unnecessarily lossy and much harder to classify correctly. + +## Decision +1. The worker client-op row identified by `:db-sync/tx-id` becomes the source of truth for DB undo/redo history. +2. The main-thread undo/redo stacks will store UI state plus a reference to that worker action row, not the DB tx payload itself. +3. New client-op rows will persist both forward semantic outliner ops and inverse semantic outliner ops. +4. Redo will replay the persisted forward semantic ops for the referenced action row. +5. Undo will replay the persisted inverse semantic ops for the referenced action row. +6. New rebase logic will use semantic forward or inverse ops for new undo/redo-backed action rows instead of raw datom replay. +7. Existing `normalized-tx-data` and `reversed-tx-data` fields remain temporarily for debugging, validation, and controlled cleanup, but they will not define replay behavior for the new undo/redo architecture. +8. The logical `tx-id` remains stable across rebase rewrites so that main-thread stack entries do not break when the worker rewrites the row contents. +9. Undo/redo rows that cannot be represented with semantic inverse ops are unsupported until a semantic representation is added explicitly. + +## Target Architecture +The architecture separates UI history from DB history while keeping both linked through one stable action identifier. + +```text ++-------------------+ tx-id +----------------------------+ +| Main thread | -----------------------> | Worker client-ops storage | +| undo/redo stack | | one row per logical action | +| | <----------------------- | stable tx-id | +| UI state | action result | forward semantic ops | +| editor cursor | | inverse semantic ops | +| route/sidebar | | normalized tx data | ++-------------------+ | reverse tx data (legacy) | + +----------------------------+ + | + v + +----------------------------+ + | Sync rebase + upload | + | replay by semantic ops | + +----------------------------+ +``` + +The main thread continues to own UI-only history because the worker should not become responsible for route or editor selection state. + +The worker becomes responsible for DB action identity and replay semantics because that is the same place that already owns pending local action persistence, sync upload, and rebase. + +## Data Model Changes +The client-op row needs to distinguish between the identity of a logical action and the concrete tx-data that happened to be generated on the current device revision. + +The row should keep the existing logical `tx-id`. + +The row should add explicit semantic operation fields for both directions. + +Recommended schema additions: + +| Field | Purpose | +| --- | --- | +| `:db-sync/forward-outliner-ops` | Canonical semantic ops that reapply the action. | +| `:db-sync/inverse-outliner-ops` | Canonical semantic ops that undo the action. | +| `:db-sync/history-kind` | Distinguishes regular action rows from specialized worker history rows if needed. | +| `:db-sync/source-tx-id` | Optional link from an undo/redo execution row back to the original action row during migration. | +| `:db-sync/semantic-persistence-version` | Marks rows that are safe for semantic replay. | + +The existing fields should remain during migration: + +| Existing field | Migration role | +| --- | --- | +| `:db-sync/normalized-tx-data` | Validation reference and emergency cleanup aid. | +| `:db-sync/reversed-tx-data` | Local safety checks and controlled cleanup aid. | +| `:db-sync/outliner-op` | Existing summary/debug field. | +| `:db-sync/outliner-ops` | Can be folded into `forward-outliner-ops` once migration is complete. | + +The main-thread undo stack entry shape should change accordingly. + +The DB portion of a stack entry should become a small reference object instead of an embedded tx payload. + +Recommended main-thread DB history payload: + +| Field | Purpose | +| --- | --- | +| `:local-op-tx-id` | Stable reference to worker client-op row. | +| `:history-direction` | `:forward` in undo stack and `:inverse` in redo stack if needed for UI bookkeeping. | +| `:history-version` | Allows invalidating stale stack entries across releases. | + +The main-thread stack entry should continue to store editor cursor and UI route state as it does today. + +## Semantic Persistence Rules +The plan depends on persisting semantic inverse ops instead of reconstructing them later from raw datoms. + +That means the inverse ops must be created at the time a local action is first recorded into undo history. + +The worker should never have to guess the inverse of an already-lost user intent from only tx-data if the action was produced by a known canonical outliner op. + +The persistence rules are: + +1. A regular local action row persists canonical forward ops. +2. The undo stack entry points at that action row by `tx-id`. +3. Undo execution uses the inverse semantic ops stored on the referenced row. +4. Redo execution uses the forward semantic ops stored on the referenced row. +5. If a row does not have semantic inverse ops, it must not enter the semantic rebase path. + +## Canonical Op Surface +This ADR does not require every outliner op to become replay-visible immediately. + +It does require the canonical replay-visible surface for undo/redo to be explicit and versioned. + +Recommended canonical surface for forward and inverse persistence: +- `:save-block` +- `:insert-blocks` +- `:move-blocks` +- `:delete-blocks` +- `:set-block-property` +- `:remove-block-property` +- `:batch-set-property` +- `:batch-remove-property` +- `:delete-property-value` +- `:batch-delete-property-value` +- `:create-property-text-block` +- `:upsert-closed-value` +- `:delete-closed-value` +- `:add-existing-values-to-closed-values` +- `:create-page` +- `:delete-page` +- `:rename-page` + +If an action cannot be expressed in that surface with a safe inverse, it should remain unsupported until a semantic representation is added deliberately. + +That is preferable to reclassifying it as safe raw replay by accident. + +## Inverse Op Generation Strategy +Inverse semantic ops should be created from the original action metadata and the pre-action DB state, not by reverse-engineering datoms after the fact. + +The current main-thread undo history already retains the original tx meta, which includes `:outliner-ops` for many actions. + +That existing signal should be preserved and extended rather than discarded. + +The generation strategy should be: + +1. Start from the original canonical forward ops attached to the local action. +2. Resolve all entity references to stable ids at persistence time. +3. Build inverse canonical ops while the pre-action DB state is still available. +4. Persist both directions on the action row under the same `tx-id`. + +Representative inverse mappings: + +| Forward op | Inverse op strategy | +| --- | --- | +| `:save-block` | Persist a `:save-block` payload built from pre-action block content and relevant pre-action refs. | +| `:insert-blocks` | Persist `:delete-blocks` for the created roots, or a more specific inverse if a safer canonical form exists. | +| `:move-blocks` | Persist `:move-blocks` back to the pre-action target with stable target id and structural opts. | +| `:delete-blocks` | Persist a hard-delete inverse only if block recreation is represented explicitly. Recycle-based restoration is no longer part of block deletion semantics. | +| `:set-block-property` | Persist `:set-block-property` or `:remove-block-property` depending on whether the property existed before the action. | +| `:batch-set-property` | Persist a batch inverse that restores prior values per block rather than a single blind batch overwrite. | +| `:create-page` | Persist `:delete-page` or a page retract inverse depending on page type. | +| `:delete-page` | Persist `:create-page` plus restoration of prior page content and relationships, or represent page restoration as a dedicated canonical op. | + +The plan does not require every mapping to be implemented in one patch. + +It does require the migration plan to make unsupported actions explicit so that they cannot silently flow into the new architecture. + +## Why `tx-id` Must Be the Main-Thread DB History Reference +The UI stack needs a stable identifier for the DB action because that action may be: +- pending locally and not yet synced +- rebased by the worker +- acknowledged by the server +- rewritten during local compaction +- invalidated because the worker dropped it as unreplayable + +If the main thread stores raw tx-data instead of `tx-id`, the UI stack and worker pending history can diverge. + +If the main thread stores `tx-id`, the worker can rewrite the row contents while preserving logical action identity. + +That makes the worker free to update the concrete replay payload during rebase without breaking the main-thread history pointer. + +The worker must therefore treat `tx-id` as logical action identity rather than as a disposable row key. + +## Cutover Plan +This change intentionally uses a strict cutover instead of a backward-compatible migration for old pending rows. + +Cutover rules: + +1. Existing pending rows without semantic persistence fields are cleared or dropped at startup. +2. Existing main-thread stack entries without `tx-id` are invalid after the cutover. +3. Old-format in-memory history is not preserved across the change. +4. New rows must always be written with the new semantic fields once the feature flag is enabled. + +## Implementation Plan +### Phase 1. Add action-identity and semantic fields to client-op storage. +- Update `/Users/tiensonqin/Codes/projects/logseq/web/src/main/frontend/worker/sync/client_op.cljs` schema for forward and inverse semantic ops plus version metadata. +- Keep existing fields intact. +- Ensure `tx-id` remains stable when a pending row is rewritten after rebase. + +### Phase 2. Make the worker action row the source of truth for DB history. +- Update `/Users/tiensonqin/Codes/projects/logseq/web/src/main/frontend/worker/sync/apply_txs.cljs` so local action persistence writes both directions when available. +- Split the meaning of existing `:db-sync/outliner-ops` from the new explicit forward and inverse fields rather than overloading one field for two roles. +- Preserve current `normalized-tx-data` and `reversed-tx-data` during migration. + +### Phase 3. Change main-thread undo stack payloads from embedded tx data to `tx-id` references. +- Update `/Users/tiensonqin/Codes/projects/logseq/web/src/main/frontend/undo_redo.cljs` so stack entries keep UI state and worker action references instead of DB tx payloads. +- Keep cursor and route state on the main thread. +- Remove the assumption that main-thread history owns the DB replay payload. + +### Phase 4. Add a worker API for undo and redo execution by `tx-id`. +- Introduce a worker-facing command that resolves the client-op row by `tx-id`. +- Undo execution should replay `inverse-outliner-ops`. +- Redo execution should replay `forward-outliner-ops`. +- The command should return enough result metadata for the main thread to restore cursor and UI state after the worker confirms DB success. + +### Phase 5. Replace raw undo/redo sync persistence with semantic persistence. +- Stop classifying new undo/redo executions as canonical raw `:transact` in `/Users/tiensonqin/Codes/projects/logseq/web/src/main/frontend/worker/sync/apply_txs.cljs`. +- Instead, persist their semantic forward and inverse operations under the same logical action identity. +- Remove the old undo/redo raw replay path for pending-row persistence instead of keeping a compatibility branch. + +### Phase 6. Extend rebase to use semantic inverse and forward ops for undo/redo rows. +- In `/Users/tiensonqin/Codes/projects/logseq/web/src/main/frontend/worker/sync/apply_txs.cljs`, route undo/redo rows through semantic replay only. +- Preserve one-row-per-user-action atomicity. +- If any op in a row becomes invalid, drop or quarantine the whole row rather than partially replaying it. + +### Phase 7. Add explicit invalidation rules for main-thread stack entries. +- If the worker drops or quarantines a referenced action row, the main-thread stack entry that points at that `tx-id` must be invalidated. +- Add a worker-to-main-thread signal so the UI can clear stale history references without guessing. +- Avoid silent dangling `tx-id` references in the UI stack. + +### Phase 8. Add compaction and cleanup rules. +- Decide when an acknowledged action row can be compacted while still remaining undoable. +- If compaction rewrites stored payloads, preserve `tx-id`. +- If compaction removes a row, invalidate any stack references to it first. + +## Required Code Areas +The following files are expected to change. + +| File | Responsibility | +| --- | --- | +| `/Users/tiensonqin/Codes/projects/logseq/web/src/main/frontend/undo_redo.cljs` | Change stack payloads to `tx-id` references and route DB replay to worker by action id. | +| `/Users/tiensonqin/Codes/projects/logseq/web/src/main/frontend/handler/history.cljs` | Keep UI restore flow but consume worker-driven undo/redo result metadata. | +| `/Users/tiensonqin/Codes/projects/logseq/web/src/main/frontend/worker/sync/client_op.cljs` | Extend client-op schema with semantic forward and inverse fields plus version metadata. | +| `/Users/tiensonqin/Codes/projects/logseq/web/src/main/frontend/worker/sync/apply_txs.cljs` | Persist semantic undo/redo ops, keep stable `tx-id`, and use semantic replay in rebase. | +| `/Users/tiensonqin/Codes/projects/logseq/web/deps/outliner/src/logseq/outliner/op.cljs` | Reuse canonical op shapes and extend replay-visible surface if new canonical ops are introduced. | +| `/Users/tiensonqin/Codes/projects/logseq/web/deps/outliner/src/logseq/outliner/recycle.cljs` | Provide a replay-safe restoration entrypoint if `:restore-recycled` becomes canonical. | +| `/Users/tiensonqin/Codes/projects/logseq/web/src/test/frontend/undo_redo_test.cljs` | Cover `tx-id`-referenced history behavior and main-thread invalidation. | +| `/Users/tiensonqin/Codes/projects/logseq/web/src/test/frontend/worker/db_sync_test.cljs` | Cover semantic persistence and rebase of undo/redo action rows. | +| `/Users/tiensonqin/Codes/projects/logseq/web/src/test/frontend/worker/db_sync_sim_test.cljs` | Cover multi-client rebase scenarios where remote txs commute or conflict with undo/redo. | + +## Edge Cases That Must Be Designed Explicitly +### Remote non-conflicting updates. +A remote title update on an unrelated block should not invalidate a local undo stack entry. + +### Remote updates on the same entity but different attrs. +The design should allow semantic replay when the attrs commute and reject it when they do not. + +### Structural conflicts. +Parent, page, order, delete, recycle, and missing-ref conflicts must invalidate the whole action row, not a subset of its ops. + +### Undo of a batch action. +One logical batch action should remain one `tx-id` and should not split into multiple undoable rows during persistence or rebase. + +### Redo after rebase. +Redo must continue to point at the same action identity even if the worker rewrote the row payloads during remote rebase. + +### Session boundary. +Main-thread stacks are in-memory only today. + +That remains acceptable, but the new `tx-id` reference model must not assume the stack survives app restart unless a later ADR decides to persist it. + +### Unsupported inverse mappings. +An unsupported op must be explicit and traceable. + +It must not silently fall back to safe-looking semantic replay if it is actually raw replay. + +### Old pending rows. +Existing rows without semantic fields are intentionally not supported after the cutover. + +They should be cleared rather than replayed. + +## Risks +The largest risk is trying to migrate everything at once. + +Undo/redo touches the main thread, the worker, client-op persistence, and rebase logic. + +The plan therefore intentionally stages the work so that logical action identity and schema changes land before the main-thread stack rewrite and before the semantic rebase cutover. + +Another risk is under-specifying inverse mappings for high-level actions such as delete-page. + +Those mappings should be treated as explicit product decisions, not left to raw tx fallback by accident. + +Another risk is user-visible loss of old pending rows during the cutover. + +That tradeoff is accepted because preserving old raw replay rows would preserve the exact failure mode this ADR is intended to remove. + +## Verification Plan +Verification must cover both behavior and migration safety. + +Required coverage: +- main-thread undo stack entry stores `tx-id` plus UI state, not raw DB tx payload +- worker persists semantic forward and inverse ops on new local action rows +- undo executes by `tx-id` and replays inverse semantic ops +- redo executes by `tx-id` and replays forward semantic ops +- new undo/redo rows no longer persist as canonical raw `:transact` +- rebase preserves logical `tx-id` while rewriting row payloads +- benign remote updates continue to allow undo/redo replay +- structural conflicts invalidate the whole referenced action row +- dropped or quarantined worker rows invalidate corresponding UI history entries +- old rows without semantic fields are cleared and do not replay + +Targeted commands once implementation begins: +- `bb dev:test -v frontend.undo-redo-test` +- `bb dev:test -v frontend.worker.db-sync-test` +- `bb dev:test -v frontend.worker.db-sync-sim-test` + +## Consequences +Positive consequences: +- Undo/redo DB history and sync pending history converge on one logical action model. +- Rebase can reason about intent instead of replaying raw datoms for new undo/redo rows. +- The main thread keeps ownership of UI-only state without also becoming the long-term store of DB replay payloads. +- `tx-id` becomes a durable action identity that can survive rebase rewrites. + +Negative consequences: +- The worker schema and the main-thread undo model both change at the same time. +- Several inverse-op mappings require explicit product and implementation decisions. +- The cutover is intentionally strict and may discard old pending rows that cannot satisfy the new semantic contract. + +## Follow-up Notes +This ADR intentionally does not specify the final retention policy for acknowledged action rows. + +It only requires that action-row lifetime be compatible with undo/redo stack references. + +If future work decides to persist the UI stack across restarts, that should be handled in a separate ADR after this action-identity model is in place. diff --git a/docs/adr/0012-worker-owned-undo-redo.md b/docs/adr/0012-worker-owned-undo-redo.md new file mode 100644 index 0000000000..9eed274461 --- /dev/null +++ b/docs/adr/0012-worker-owned-undo-redo.md @@ -0,0 +1,308 @@ +# ADR 0012: Move Undo/Redo Recording and Replay to the DB Worker + +Date: 2026-03-21 +Status: Proposed + +## Context +`frontend.undo-redo` currently runs on the main thread. + +That means undo and redo recording depends on main-thread listeners observing DB +tx reports after they have already crossed the worker boundary. + +This split has become a recurring source of drift: + +- the worker is the source of truth for the browser Datascript DB +- the worker already persists local actions in client-op storage +- the worker already owns rebase and semantic replay +- the main thread still owns undo/redo stack mutation for DB history + +That architecture forces the main thread to reconstruct DB history from a +worker-synchronized tx report instead of observing the DB change at the place +where it actually happens. + +The result is fragile metadata flow. + +We have already seen bugs caused by: + +- `:outliner-ops` being stripped or reshaped during worker-to-main-thread sync +- undo/redo-generated tx rows overwriting the original client-op row +- semantic forward and inverse ops diverging between worker persistence and + main-thread undo stack payloads +- special cases such as `:replace-empty-target?`, block concat, and + `:set-block-property` depending on worker-local replay behavior anyway + +The one main-thread-only input that `frontend.undo-redo` still needs is +`@state/*editor-info`. + +Today that atom is read and reset on the main thread inside +`frontend.undo-redo/gen-undo-ops!`. + +If undo/redo recording moves to the worker, the worker can no longer deref the +main-thread atom directly. + +## Decision +1. DB undo/redo recording and replay move to the DB worker. +2. The worker becomes the only place that listens to DB tx reports for DB + history generation. +3. The main thread remains responsible for UI-derived history inputs only: + editor cursor/focus metadata and UI-state snapshots. +4. `@state/*editor-info` will not be read from the worker directly. + It will be replaced by an explicit main-thread-to-worker handoff protocol. +5. The worker owns the undo stack and redo stack for DB actions and UI-adjacent + metadata attached to those actions. +6. The main thread will invoke worker APIs for: + - recording pending editor info + - recording UI-only state history entries + - undo + - redo + - clear history +7. The main thread will stop generating DB undo history from `:db/sync-changes` + events. +8. The worker-owned history row should not keep a separate persisted + `:db-sync/outliner-ops` field. + `:db-sync/forward-outliner-ops` is the only canonical persisted forward + semantic field. + +## Rationale +The worker is already the place where all browser DB facts become real: + +- local outliner ops are applied there +- remote sync txs are applied there +- pending local rows are persisted there +- semantic forward and inverse ops are canonicalized there +- rebase happens there + +Undo/redo recording should therefore observe worker DB tx reports directly +instead of reconstructing them after the worker has serialized, sanitized, and +rebroadcast them. + +That removes an entire class of metadata transport bugs. + +It also matches ADR 0011 more closely: the worker action row is supposed to be +the source of truth for DB history. Recording DB history on the main thread is +in tension with that decision. + +## Target Architecture +```text ++------------------------------+ thread-api +---------------------------+ +| Main thread | ----------------------------> | DB worker | +| | | | +| editor lifecycle | push pending editor-info | pending editor-info store | +| route/sidebar state | push ui-state entries | undo stack | +| history handler | undo / redo / clear | redo stack | +| restore cursor + route | <---------------------------- | DB replay + result meta | ++------------------------------+ +---------------------------+ +``` + +The worker stack entry becomes the single logical history item for both: + +- DB replay metadata +- UI-adjacent metadata needed after replay + +Representative worker stack item: + +```clojure +{:tx-id #uuid "..." + :kind :db-action ; or :ui-state-only + :editor-info {:block-uuid ... + :container-id ... + :start-pos ... + :end-pos ...} + :ui-state-str "...optional transit..." + :forward-outliner-ops [...] + :inverse-outliner-ops [...] + :outliner-op :save-block} +``` + +The target row schema is therefore: + +- `:db-sync/tx-id` +- `:db-sync/outliner-op` +- `:db-sync/forward-outliner-ops` +- `:db-sync/inverse-outliner-ops` +- worker-owned cursor/UI metadata as needed + +It intentionally does not include a separate persisted +`:db-sync/outliner-ops`. + +## `*editor-info` Handoff +The worker must not read `@state/*editor-info` directly. + +That atom lives on the main thread and represents ephemeral UI state. + +Instead, we will replace the implicit shared-state read with an explicit +handoff. + +### Rule +The main thread owns editor-info production. +The worker owns editor-info consumption. + +### Mechanism +Add a worker-side pending editor-info slot keyed by repo. + +Suggested API: + +- `:thread-api/undo-redo-set-pending-editor-info` + - args: `repo`, `editor-info-or-nil` +- `:thread-api/undo-redo-record-ui-state` + - args: `repo`, `ui-state-str` +- `:thread-api/undo-redo-undo` + - args: `repo` +- `:thread-api/undo-redo-redo` + - args: `repo` +- `:thread-api/undo-redo-clear-history` + - args: `repo` + +### Consumption and reset semantics +When the worker records a new local DB action into undo history: + +1. read pending editor-info for the repo +2. attach it to the new stack item if present +3. clear the pending editor-info slot immediately after the stack item is + created + +This preserves the current one-shot semantics of `@state/*editor-info` without +requiring the worker to deref or mutate main-thread state directly. + +### Main-thread responsibilities +The main thread should: + +- capture editor-info at the same points it does today +- send the current snapshot to the worker before the local DB action is + submitted or immediately when editor focus/cursor changes, whichever path is + simpler and consistent +- stop relying on worker `:db/sync-changes` to retroactively capture cursor + state + +The main thread may still keep a local `*editor-info` atom for editor UI code, +but it is no longer the undo recorder’s source of truth. + +## UI-State History +UI-state-only undo entries such as route/sidebar snapshots cannot be generated +by the worker from DB tx reports. + +Those entries should be pushed explicitly from the main thread into the worker +history stack. + +Two entry classes will therefore exist in the worker stack: + +1. `:db-action` +2. `:ui-state-only` + +Undo/redo execution will return enough metadata for the main thread to restore: + +- route +- sidebar state +- editor cursor + +The worker should not attempt to perform UI restoration itself. + +## Consequences +### Positive +- DB undo/redo history is recorded at the actual DB source of truth. +- No more dependence on `:db/sync-changes` preserving semantic metadata exactly. +- Worker persistence, worker replay, and worker history all use the same action + identity. +- Main-thread history bugs caused by tx-meta sanitization disappear. +- Undo/redo debugging becomes simpler because the worker owns the full DB + history lifecycle. + +### Negative +- The worker history stack now stores UI-adjacent metadata that originates on + the main thread. +- New thread APIs are required. +- Main-thread editor lifecycle code must actively synchronize pending + `editor-info`. +- The migration touches both undo/redo and worker message flow at once. + +## Implementation Plan +### Phase 1. Introduce worker-owned undo/redo module +- Create a worker namespace, e.g. + `/Users/tiensonqin/Codes/projects/logseq/web/src/main/frontend/worker/undo_redo.cljs` +- Move stack storage and DB history generation there. +- Register worker DB listener(s) against the worker Datascript conn. +- Remove persisted `:db-sync/outliner-ops` from the target worker history row + shape instead of carrying it forward as a parallel field. + +### Phase 2. Replace main-thread DB history generation +- Remove DB-history recording from + `/Users/tiensonqin/Codes/projects/logseq/web/src/main/frontend/undo_redo.cljs` +- Keep only main-thread coordination helpers if still needed. +- Route history handler calls through worker thread APIs. + +### Phase 3. Add pending editor-info handoff +- Add worker API to set pending editor-info. +- Update + `/Users/tiensonqin/Codes/projects/logseq/web/src/main/frontend/modules/outliner/ui.cljc` + and any direct local transact paths to send editor-info to the worker. +- Consume and clear the pending editor-info slot when a local history item is + recorded. + +### Phase 4. Move UI-state history writes to worker +- Replace `record-ui-state!` main-thread stack mutation with worker API calls. +- Keep route/sidebar serialization on the main thread. + +### Phase 5. Return worker-owned undo/redo result metadata +- Worker undo/redo APIs should return: + - `:undo?` + - `:editor-info` + - `:ui-state-str` + - optional block content or replay diagnostics +- Main-thread history handler restores cursor and route from that result. + +## Files Expected to Change +| File | Change | +| --- | --- | +| `/Users/tiensonqin/Codes/projects/logseq/web/src/main/frontend/undo_redo.cljs` | Remove main-thread DB listener ownership, keep only coordinator logic if still needed. | +| `/Users/tiensonqin/Codes/projects/logseq/web/src/main/frontend/handler/history.cljs` | Call worker undo/redo APIs and restore UI from returned metadata. | +| `/Users/tiensonqin/Codes/projects/logseq/web/src/main/frontend/modules/outliner/ui.cljc` | Send editor-info snapshots to the worker before local action submission. | +| `/Users/tiensonqin/Codes/projects/logseq/web/src/main/frontend/handler/editor/lifecycle.cljs` | Stop recording editor-info directly into main-thread undo stack; feed worker pending editor-info instead. | +| `/Users/tiensonqin/Codes/projects/logseq/web/src/main/frontend/worker/db_worker.cljs` | Expose worker thread APIs for pending editor-info, UI-state history, undo, redo, and clear-history. | +| `/Users/tiensonqin/Codes/projects/logseq/web/src/main/frontend/worker/db_listener.cljs` | Attach worker undo/redo recording directly to worker DB tx reports. | +| `/Users/tiensonqin/Codes/projects/logseq/web/src/main/frontend/worker/sync/client_op.cljs` | Remove `:db-sync/outliner-ops` from the target worker-owned undo/redo row model and use `:db-sync/forward-outliner-ops` instead. | +| `/Users/tiensonqin/Codes/projects/logseq/web/src/main/frontend/worker/sync/apply_txs.cljs` | Keep worker replay aligned with worker-owned history rows. | +| `/Users/tiensonqin/Codes/projects/logseq/web/src/test/frontend/undo_redo_test.cljs` | Replace main-thread DB-history expectations with coordinator/result expectations. | +| `/Users/tiensonqin/Codes/projects/logseq/web/src/test/frontend/worker/db_sync_test.cljs` | Add worker-owned history recording and replay coverage. | + +## Alternatives Considered +### 1. Keep `frontend.undo-redo` on the main thread and preserve more tx-meta +Rejected. + +This keeps the wrong ownership boundary. +It reduces one transport bug at a time but does not remove the architectural +duplication between main-thread history and worker DB history. + +### 2. Let the worker call back into the main thread to read `@state/*editor-info` +Rejected. + +That would create an implicit cross-thread read dependency around ephemeral UI +state. +It is harder to reason about than explicit handoff, and reset semantics become +ambiguous. + +### 3. Keep DB history in the worker and UI history in a separate main-thread stack +Possible, but inferior to a single worker-owned stack item keyed by `tx-id`. + +It still splits one logical action across two structures and reintroduces +alignment problems. + +## Open Questions +1. Should pending editor-info be pushed: + - only at transact boundaries + - or eagerly on every cursor change with last-write-wins semantics? + +Recommendation: +push at transact boundaries first. +It matches current one-shot behavior and avoids unnecessary worker chatter. + +2. Should `:ui-state-only` entries live in the same stack as DB actions? + +Recommendation: +yes. +One logical undo/redo stream is simpler than coordinating two stacks. + +3. Do we still need `@state/*editor-info` after the migration? + +Recommendation: +keep it as a UI helper until the move is complete, but stop using it as undo +history source of truth. diff --git a/docs/adr/0013-worker-owned-undo-redo-test-ownership.md b/docs/adr/0013-worker-owned-undo-redo-test-ownership.md new file mode 100644 index 0000000000..2f7a18358b --- /dev/null +++ b/docs/adr/0013-worker-owned-undo-redo-test-ownership.md @@ -0,0 +1,187 @@ +# ADR 0013: Worker-Owned Undo/Redo Test Ownership + +Date: 2026-03-21 +Status: Proposed + +## Context +ADR 0012 moves DB undo/redo ownership from the main thread to the db worker. + +That architectural move is not complete if the test suite still treats +`src/test/frontend/undo_redo_test.cljs` as the primary place to assert DB +history behavior. + +Today the old main-thread file still contains most undo/redo tests, including: + +- local tx recording +- semantic forward and inverse metadata +- insert/save/delete replay sequences +- replay conflict and skip behavior +- validation behavior tied to DB replay + +Those tests were correct for the old design, but they now encourage the wrong +ownership boundary. + +The worker already owns: + +- worker datascript DB mutation +- client-op persistence +- semantic replay +- pending action identity +- replay safety decisions + +The main thread now mainly owns: + +- route/sidebar UI restoration +- cursor restoration from worker result metadata +- thin browser-facing proxy calls + +The test suite should reflect that split directly. + +## Decision +1. `src/test/frontend/worker/undo_redo_test.cljs` becomes the primary home for + DB-history-focused undo/redo tests. +2. `src/test/frontend/undo_redo_test.cljs` is reduced to main-thread-only + coordination tests. +3. Replay, conflict, and rebase-heavy assertions that are really worker replay + behavior belong in `src/test/frontend/worker/db_sync_test.cljs`, not in + either undo-redo namespace test file. +4. No new DB-history behavior test should be added to + `src/test/frontend/undo_redo_test.cljs`. + +## Test Ownership Rules + +### Keep in `src/test/frontend/worker/undo_redo_test.cljs` + +- local tx recording gates +- semantic metadata persistence +- worker-owned undo stack and redo stack mutation +- canonical action-id and semantic op persistence +- worker-owned DB-history entries that include pending editor-info and UI-state + metadata + +### Keep in `src/test/frontend/worker/db_sync_test.cljs` + +- `apply-history-action!` replay behavior +- semantic replay correctness +- conflict and skip behavior caused by worker replay safety +- rebase interactions +- client-op row persistence and rewrite behavior + +### Keep in `src/test/frontend/undo_redo_test.cljs` + +- route/sidebar UI-state restoration on the main thread +- cursor restoration from worker result metadata +- browser-facing proxy semantics in `frontend.undo-redo` and + `frontend.handler.history` +- any pure coordination helper that still lives only on the main thread + +## Migration Buckets + +### Bucket 1. Recording and metadata tests. + +Move these first: + +- `undo-records-only-local-txs-test` +- `undo-history-records-semantic-action-metadata-test` +- `undo-history-records-forward-ops-for-editor-save-block-test` +- `undo-history-canonicalizes-insert-block-uuids-test` + +These tests map directly to worker history ownership and do not require route or +cursor restoration. + +### Bucket 2. Basic local replay tests. + +Move next, but only after the worker fixture persists matching client-op rows: + +- `undo-works-for-local-graph-test` +- `repeated-save-block-content-undo-redo-test` +- `repeated-editor-save-block-content-undo-redo-test` +- `editor-save-two-blocks-undo-targets-latest-block-test` +- `new-local-save-clears-redo-stack-test` + +If a test still depends on browser-only helpers, rewrite it to assert worker DB +behavior directly or leave it temporarily unmoved. + +### Bucket 3. Delete/recycle sequence tests. + +Reclassify before moving: + +- `insert-save-delete-sequence-undo-redo-test` +- `undo-redo-works-for-recycle-delete-test` + +If the real behavior under test is semantic replay, keep or move it to +`src/test/frontend/worker/db_sync_test.cljs`. + +### Bucket 4. Conflict and skip tests. + +Reclassify these into worker replay coverage unless they are truly worker stack +mutation tests: + +- `undo-conflict-clears-history-test` +- `undo-works-with-remote-updates-test` +- `undo-skips-when-parent-missing-test` +- `undo-skips-when-block-deleted-remote-test` +- `undo-skips-when-undo-would-create-cycle-test` +- `undo-skips-conflicted-move-and-keeps-earlier-history-test` +- `redo-builds-reversed-tx-when-target-parent-is-recycled-test` +- `undo-skips-move-when-original-parent-is-recycled-test` + +### Bucket 5. Main-thread leftovers. + +After Buckets 1-4: + +- keep only route restoration +- keep only cursor restoration +- keep only proxy/coordination tests +- add a namespace comment in `src/test/frontend/undo_redo_test.cljs` that it is + no longer the DB-history source-of-truth test file + +## Implementation Rules + +1. Every moved worker test must use a fixture that owns both: + - worker datascript conn + - worker client-ops conn +2. That fixture must route tx reports through both: + - `frontend.worker.sync/enqueue-local-tx!` + - `frontend.worker.undo-redo/gen-undo-ops!` +3. Tests that expect persisted action lookup must attach a stable + `:db-sync/tx-id`. +4. Do not duplicate replay tests between `worker/undo_redo_test.cljs` and + `worker/db_sync_test.cljs`. +5. Delete the original test only after the worker version passes. + +## Consequences + +### Positive + +- The test suite matches the actual runtime ownership boundary. +- Worker replay regressions are caught where they actually happen. +- The old main-thread test file becomes much easier to reason about. + +### Negative + +- Some existing tests need adaptation rather than straight copy because they + implicitly relied on old main-thread stack storage. +- The migration requires explicit decisions about whether a test is about + worker history ownership or worker replay behavior. + +## Verification + +Representative focused commands during migration: + +```bash +bb dev:test -v frontend.worker.undo-redo-test/undo-records-only-local-txs-test +bb dev:test -v frontend.worker.undo-redo-test/undo-history-records-semantic-action-metadata-test +bb dev:test -v frontend.worker.undo-redo-test/undo-history-canonicalizes-insert-block-uuids-test +bb dev:test -v frontend.worker.db-sync-test/apply-history-action-redo-replays-save-block-test +bb dev:test -v frontend.worker.db-sync-test/apply-history-action-redo-replays-block-concat-test +bb dev:test -v frontend.worker.db-sync-test/apply-history-action-redo-replays-paste-into-empty-target-test +``` + +## Exit Criteria + +- `src/test/frontend/worker/undo_redo_test.cljs` owns the DB-history recording + tests. +- `src/test/frontend/undo_redo_test.cljs` contains only main-thread coordination + tests. +- No worker-owned DB-history scenario is tested only in the main-thread file. diff --git a/package.json b/package.json index 4ab1119a77..4c77d9c92d 100644 --- a/package.json +++ b/package.json @@ -161,7 +161,7 @@ "photoswipe": "^5.4.4", "pixi-graph-fork": "0.2.0", "pixi.js": "6.2.0", - "posthog-js": "1.141.0", + "posthog-js": "1.10.0", "prop-types": "^15.8.1", "react": "18.3.1", "react-dom": "18.3.1", diff --git a/src/main/frontend/common/file/opfs.cljs b/src/main/frontend/common/file/opfs.cljs index 96d49d4666..d66b420493 100644 --- a/src/main/frontend/common/file/opfs.cljs +++ b/src/main/frontend/common/file/opfs.cljs @@ -26,19 +26,13 @@ file (.getFile file-handle)] (.text file))) -(comment - (defn (p/let [root (.. js/navigator -storage (getDirectory))] - (.removeEntry root filename)) - (p/catch (fn [err] - (if (and ignore-not-found? - (= (.-name err) "NotFoundError")) - nil - (throw err))))))) +(defn (p/let [root (.. js/navigator -storage (getDirectory))] + (.removeEntry root filename)) + (p/catch (fn [err] + (if (= (.-name err) "NotFoundError") + nil + (throw err)))))) diff --git a/src/main/frontend/components/recycle.cljs b/src/main/frontend/components/recycle.cljs index 7c61d90695..ff08c543cb 100644 --- a/src/main/frontend/components/recycle.cljs +++ b/src/main/frontend/components/recycle.cljs @@ -98,7 +98,7 @@ #(compare %2 %1)))] [:div.flex.flex-col.gap-1 [:div.text-sm.text-muted-foreground.mb-4 - "Deleted pages and blocks stay here until restored or automatically garbage collected after 60 days."] + "Deleted pages and blocks stay here until restored or automatically garbage collected after 30 days."] (if (seq groups) (for [[title roots] groups] [:section {:key title} diff --git a/src/main/frontend/components/right_sidebar.cljs b/src/main/frontend/components/right_sidebar.cljs index a11a0bd67b..b792a86657 100644 --- a/src/main/frontend/components/right_sidebar.cljs +++ b/src/main/frontend/components/right_sidebar.cljs @@ -20,6 +20,7 @@ [frontend.handler.ui :as ui-handler] [frontend.state :as state] [frontend.ui :as ui] + [frontend.undo-redo.debug-ui :as undo-redo-debug-ui] [frontend.util :as util] [logseq.db :as ldb] [logseq.shui.hooks :as hooks] @@ -143,10 +144,15 @@ :shortcut-settings [[:.flex.items-center (ui/icon "command" {:class "text-md mr-2"}) (t :help/shortcuts)] (shortcut-settings)] + :rtc [[:.flex.items-center (ui/icon "cloud" {:class "text-md mr-2"}) "(Dev) RTC"] (rtc-debug-ui/rtc-debug-ui)] + :undo-redo + [[:.flex.items-center (ui/icon "rotate-clockwise" {:class "text-md mr-2"}) "(Dev) Undo/Redo"] + (undo-redo-debug-ui/undo-redo-debug-ui)] + :profiler [[:.flex.items-center (ui/icon "cloud" {:class "text-md mr-2"}) "(Dev) Profiler"] (profiler/profiler)] @@ -165,6 +171,21 @@ (defonce *drag-from (atom nil)) +(defn dev-sidebar-items + [developer-mode?] + (cond-> [] + (and developer-mode? (not config/publishing?)) + (conj {:db-id "rtc" :block-type :rtc :label "(Dev) RTC"}) + + developer-mode? + (conj {:db-id "undo-redo" :block-type :undo-redo :label "(Dev) Undo/Redo"}) + + developer-mode? + (conj {:db-id "vector-search" :block-type :vector-search :label "(Dev) vector-search"}) + + developer-mode? + (conj {:db-id "profiler" :block-type :profiler :label "(Dev) Profiler"}))) + (rum/defc actions-menu-content [db-id idx type collapsed? block-count] (let [multi-items? (> block-count 1) @@ -417,7 +438,8 @@ state)} [state repo t blocks] (let [*anim-finished? (get state ::anim-finished?) - block-count (count blocks)] + block-count (count blocks) + developer-mode? (state/sub [:ui/developer-mode?])] [:div.cp__right-sidebar-inner.flex.flex-col.h-full#right-sidebar-container [:div.cp__right-sidebar-scrollable @@ -443,22 +465,12 @@ (state/sidebar-add-block! repo "help" :help))} (t :right-side-bar/help)]] - (when (and (state/sub [:ui/developer-mode?]) (not config/publishing?)) - [:div.text-sm - [:button.button.cp__right-sidebar-settings-btn {:on-click (fn [_e] - (state/sidebar-add-block! repo "rtc" :rtc))} - "(Dev) RTC"]]) - (when (state/sub [:ui/developer-mode?]) - [:div.text-sm + (for [{:keys [db-id block-type label]} (dev-sidebar-items developer-mode?)] + [:div.text-sm {:key (str "dev-sidebar-item-" (name block-type))} [:button.button.cp__right-sidebar-settings-btn {:on-click (fn [_e] - (state/sidebar-add-block! repo "vector-search" :vector-search))} - "(Dev) vector-search"]]) - (when (state/sub [:ui/developer-mode?]) - [:div.text-sm - [:button.button.cp__right-sidebar-settings-btn {:on-click (fn [_e] - (state/sidebar-add-block! repo "profiler" :profiler))} - "(Dev) Profiler"]])]] + (state/sidebar-add-block! repo db-id block-type))} + label]])]] [:.sidebar-item-list.flex-1.scrollbar-spacing.px-2 (if @*anim-finished? diff --git a/src/main/frontend/components/rtc/indicator.cljs b/src/main/frontend/components/rtc/indicator.cljs index 253b5574bc..0b02dd5d3a 100644 --- a/src/main/frontend/components/rtc/indicator.cljs +++ b/src/main/frontend/components/rtc/indicator.cljs @@ -27,6 +27,8 @@ :graph-uuid nil :local-tx nil :remote-tx nil + :local-checksum nil + :remote-checksum nil :rtc-state :open :download-logs nil :upload-logs nil @@ -61,6 +63,8 @@ :graph-uuid (:graph-uuid state) :local-tx (:local-tx state) :remote-tx (:remote-tx state) + :local-checksum (:local-checksum state) + :remote-checksum (:remote-checksum state) :rtc-state (if (:rtc-lock state) :open :close))) rtc-flows/rtc-state-flow)))] (reset! *update-detail-info-canceler canceler)))) @@ -118,7 +122,7 @@ [] (let [online? (hooks/use-flow-state flows/network-online-event-flow) [expand-debug? set-expand-debug!] (hooks/use-state false) - {:keys [graph-uuid local-tx remote-tx rtc-state + {:keys [graph-uuid local-tx remote-tx local-checksum remote-checksum rtc-state download-logs upload-logs misc-logs pending-local-ops pending-server-ops]} (hooks/use-flow-state (m/watch *detail-info))] [:div.rtc-info.flex.flex-col.gap-1.p-2.text-gray-11 @@ -144,6 +148,8 @@ graph-uuid (assoc :graph-uuid graph-uuid) local-tx (assoc :local-tx local-tx) remote-tx (assoc :remote-tx remote-tx) + local-checksum (assoc :local-checksum local-checksum) + remote-checksum (assoc :remote-checksum remote-checksum) rtc-state (assoc :rtc-state rtc-state)) pprint/pprint with-out-str)]]) diff --git a/src/main/frontend/config.cljs b/src/main/frontend/config.cljs index d15a612ab9..ccb2ae0acf 100644 --- a/src/main/frontend/config.cljs +++ b/src/main/frontend/config.cljs @@ -53,12 +53,12 @@ (defonce db-sync-ws-url (if db-sync-local? "ws://127.0.0.1:8787/sync/%s" - "wss://api.logseq.io/sync/%s")) + "wss://api-staging.logseq.io/sync/%s")) (defonce db-sync-http-base (if db-sync-local? "http://127.0.0.1:8787" - "https://api.logseq.io")) + "https://api-staging.logseq.io")) ;; Feature flags ;; ============= diff --git a/src/main/frontend/db/async.cljs b/src/main/frontend/db/async.cljs index a414b77479..1d5ce0ac85 100644 --- a/src/main/frontend/db/async.cljs +++ b/src/main/frontend/db/async.cljs @@ -19,9 +19,6 @@ (def ^:private yyyyMMdd-formatter (tf/formatter "yyyyMMdd")) (def (or tx-meta {}) + (nil? (:db-sync/tx-id tx-meta)) + (assoc :db-sync/tx-id (random-uuid)))) + (defn transact [worker-transact repo tx-data tx-meta] - (let [tx-meta' (assoc tx-meta + (let [tx-meta' (-> tx-meta + ensure-local-op-tx-id + (assoc ;; not from remote (rtc) - :local-tx? true)] + :local-tx? true))] (worker-call (fn async-request [] - (worker-transact repo tx-data tx-meta'))))) + (p/do! + (state/ opts + ensure-local-op-tx-id + (assoc + :client-id (:client-id @state/state) + :local-tx? true)) request #(state/ (or repo "graph") + (string/replace #"^/+" "") + (string/replace #"[\\/]+" "_") + (str "_checksum_" (quot (util/time-ms) 1000)))) + +(defn ^:export recompute-checksum-diagnostics + [] + (if-let [repo (state/get-current-repo)] + (-> (state/js {:type "text/edn;charset=utf-8"})) + filename (checksum-export-file-name repo)] + (utils/saveToFile blob filename "edn") + (notification/show! + (str "Checksum recomputed. Recomputed: " recomputed-checksum + ", local: " (or local-checksum "") + ", remote: " (or remote-checksum "") + ". Downloaded " filename ".edn with " (count blocks) + " blocks and checksum attrs " (pr-str checksum-attrs) ".") + :success + false)) + (notification/show! "Unable to compute checksum diagnostics for current graph." :warning)))) + (p/catch (fn [error] + (js/console.error "recompute-checksum-diagnostics failed:" error) + (notification/show! "Failed to compute graph checksum diagnostics." :error)))) + (notification/show! "No graph found" :warning))) + (defn import-chosen-graph [repo] (p/let [_ (persist-db/ (state/get-current-page) common-util/page-name-sanity-lc) - (common-util/page-name-sanity-lc page-name))) - (route-handler/redirect-to-home!)) - - ;; TODO: why need this? - (ui-handler/re-render-root!)) + ( (state/ (when (not config/publishing?) (when-let [state (get-state)] @@ -507,7 +508,7 @@ :else insert-new-block-aux!) - [result-promise sibling? next-block] (insert-fn config block'' value) + [result-promise sibling? next-block] (insert-fn (assoc config :right-sibling right-sibling) block'' value) edit-block-f (fn [] (let [next-block' (db/entity [:block/uuid (:block/uuid next-block)]) pos 0 @@ -1982,51 +1983,26 @@ (insert-template! element-id db-id {})) ([element-id db-id {:keys [target] :as opts}] (let [repo (state/get-current-repo)] - (p/let [block (db-async/ (first blocks') - (assoc :logseq.property/used-template (:db/id block))) - (rest blocks'))) - blocks sorted-blocks] + format (get block :block/format :markdown)] (when element-id (insert-command! element-id "" format {:end-pattern commands/command-trigger})) - (let [sibling? (:sibling? opts) - sibling?' (cond - (some? sibling?) - sibling? + (try + (p/let [result (ui-outliner-tx/transact! + {:outliner-op :apply-template + :created-from-journal-template? journal?} + (when-not (string/blank? (state/get-edit-content)) + (save-current-block!)) + (outliner-op/apply-template! db-id target opts))] + (when result (edit-last-block-after-inserted! result))) - (db/has-children? (:block/uuid target)) - false - - :else - true)] - (when (seq blocks) - (try - (p/let [result (ui-outliner-tx/transact! - {:outliner-op :insert-blocks - :created-from-journal-template? journal?} - (when-not (string/blank? (state/get-edit-content)) - (save-current-block!)) - (outliner-op/insert-blocks! blocks target - (assoc opts - :sibling? sibling?' - :insert-template? true)))] - (when result (edit-last-block-after-inserted! result))) - - (catch :default ^js/Error e - (notification/show! - (util/format "Template insert error: %s" (.-message e)) - :error))))))))))) + (catch :default ^js/Error e + (notification/show! + (util/format "Template insert error: %s" (.-message e)) + :error))))))))) (defn template-on-chosen-handler [element-id] @@ -2095,7 +2071,7 @@ input (state/get-input) config (assoc config :keydown-new-block true) content (gobj/get input "value") - has-right? (ldb/get-right-sibling block)] + right-sibling (ldb/get-right-sibling block)] (cond (and (string/blank? content) (own-order-number-list? block) @@ -2105,12 +2081,12 @@ (and (string/blank? content) - (not has-right?) + (not right-sibling) (not (last-top-level-child? config block))) (indent-outdent false) :else - (insert-new-block! state))))))) + (insert-new-block! state right-sibling))))))) (defn- inside-of-single-block "When we are in a single block wrapper, we should always insert a new line instead of new block" @@ -3111,8 +3087,7 @@ (:block/uuid page-entity))) repo (state/get-current-repo) _ (db-async/ UI (defmethod handle :db/sync-changes [[_ data]] diff --git a/src/main/frontend/handler/history.cljs b/src/main/frontend/handler/history.cljs index e844648493..b5d7517864 100644 --- a/src/main/frontend/handler/history.cljs +++ b/src/main/frontend/handler/history.cljs @@ -4,7 +4,6 @@ [frontend.handler.route :as route-handler] [frontend.persist-db.browser :as db-browser] [frontend.state :as state] - [frontend.undo-redo :as undo-redo] [frontend.util :as util] [goog.functions :refer [debounce]] [logseq.db :as ldb] @@ -12,12 +11,21 @@ (defn- restore-cursor! [{:keys [editor-cursors block-content undo?]}] - (let [{:keys [block-uuid container-id start-pos end-pos]} (if undo? (first editor-cursors) (or (last editor-cursors) (first editor-cursors))) + (let [cursor (if undo? + (first editor-cursors) + (or (last editor-cursors) (first editor-cursors))) + {:keys [selected-block-uuids selection-direction block-uuid container-id start-pos end-pos]} cursor + selected-blocks (when (seq selected-block-uuids) + (->> selected-block-uuids + (mapcat util/get-blocks-by-id) + vec)) pos (if undo? (or start-pos end-pos) (or end-pos start-pos))] - (when-let [block (db/pull [:block/uuid block-uuid])] - (editor/edit-block! block pos - {:container-id container-id - :custom-content block-content})))) + (if (seq selected-blocks) + (state/exit-editing-and-set-selected-blocks! selected-blocks selection-direction) + (when-let [block (db/pull [:block/uuid block-uuid])] + (editor/edit-block! block pos + {:container-id container-id + :custom-content block-content}))))) (defn- restore-app-state! [state] @@ -52,7 +60,7 @@ (state/set-state! [:editor/last-replace-ref-content-tx repo] nil) (editor/save-current-block!) (state/clear-editor-action!) - (reset! *last-request (undo-redo/undo repo)) + (reset! *last-request (state/ (state/ (get-selection-block-ids) seq vec) + selection-info (when selected-block-uuids + {:selected-block-uuids selected-block-uuids + :selection-direction (get-selection-direction)})] + (if-let [edit-block (get-edit-block)] + (cond-> {:block-uuid (:block/uuid edit-block) + :container-id (or @(:editor/container-id @state) :unknown-container) + :start-pos @(:editor/start-pos @state) + :end-pos (get-edit-pos)} + selection-info + (merge selection-info)) + selection-info))) (defn conj-block-ref! [ref-entity] diff --git a/src/main/frontend/undo_redo.cljs b/src/main/frontend/undo_redo.cljs index 9593727ef9..e404b20e53 100644 --- a/src/main/frontend/undo_redo.cljs +++ b/src/main/frontend/undo_redo.cljs @@ -1,378 +1,69 @@ (ns frontend.undo-redo - "Undo redo new implementation" - (:require [datascript.core :as d] - [frontend.db :as db] - [frontend.state :as state] - [frontend.util :as util] - [lambdaisland.glogi :as log] - [logseq.common.defkeywords :refer [defkeywords]] - [logseq.db :as ldb] - [logseq.outliner.recycle :as outliner-recycle] - [logseq.undo-redo-validate :as undo-validate] - [malli.core :as m] - [malli.util :as mu] - [promesa.core :as p])) + "Main-thread proxy for worker-owned undo/redo." + (:require [frontend.state :as state] + [frontend.util :as util])) -(defkeywords - ::record-editor-info {:doc "record current editor and cursor"} - ::db-transact {:doc "db tx"} - ::ui-state {:doc "ui state such as route && sidebar blocks"}) +(defn- worker-not-initialized? + [e] + (= "db-worker has not been initialized" (ex-message e))) -;; TODO: add other UI states such as `::ui-updates`. -(comment - ;; TODO: convert it to a qualified-keyword - (sr/defkeyword :gen-undo-ops? - "tx-meta option, generate undo ops from tx-data when true (default true)")) +(defn- normalize-empty-result + [result] + (case result + :frontend.worker.undo-redo/empty-undo-stack + :frontend.undo-redo/empty-undo-stack -(def ^:private undo-op-item-schema - (mu/closed-schema - [:multi {:dispatch first} - [::db-transact - [:cat :keyword - [:map - [:tx-data [:sequential [:fn - {:error/message "should be a Datom"} - d/datom?]]] - [:tx-meta [:map {:closed false} - [:outliner-op :keyword]]] - [:added-ids [:set :int]] - [:retracted-ids [:set :int]]]]] + :frontend.worker.undo-redo/empty-redo-stack + :frontend.undo-redo/empty-redo-stack - [::record-editor-info - [:cat :keyword - [:map - [:block-uuid :uuid] - [:container-id [:or :int [:enum :unknown-container]]] - [:start-pos [:maybe :int]] - [:end-pos [:maybe :int]]]]] + result)) - [::ui-state - [:cat :keyword :string]]])) - -(def ^:private undo-op-validator (m/validator [:sequential undo-op-item-schema])) - -(defonce max-stack-length 100) -(defonce *undo-ops (atom {})) -(defonce *redo-ops (atom {})) +(defn- invoke-db-worker + [thread-api & args] + (try + (apply state/= (count result) max-stack-length) - (subvec result 0 (/ max-stack-length 2)) - result))) - -(defn- pop-stack - [stack] - (when (seq stack) - [(last stack) (pop stack)])) - -(defn- push-undo-op - [repo op] - (assert (undo-op-validator op) {:op op}) - (swap! *undo-ops update repo conj-op op)) - -(defn- push-redo-op - [repo op] - (assert (undo-op-validator op) {:op op}) - (swap! *redo-ops update repo conj-op op)) - -(comment - ;; This version checks updated datoms by other clients, allows undo and redo back - ;; to the current state. - ;; The downside is that it'll undo the changes made by others. - (defn- pop-undo-op - [repo conn] - (let [undo-stack (get @*undo-ops repo) - [op undo-stack*] (pop-stack undo-stack)] - (swap! *undo-ops assoc repo undo-stack*) - (mapv (fn [item] - (if (= (first item) ::db-transact) - (let [m (second item) - tx-data' (mapv - (fn [{:keys [e a v tx add] :as datom}] - (let [one-value? (= :db.cardinality/one (:db/cardinality (d/entity @conn a))) - new-value (when (and one-value? add) (get (d/entity @conn e) a)) - value-not-matched? (and (some? new-value) (not= v new-value))] - (if value-not-matched? - ;; another client might updated `new-value`, the datom below will be used - ;; to restore the the current state when redo this undo. - (d/datom e a new-value tx add) - datom))) - (:tx-data m))] - [::db-transact (assoc m :tx-data tx-data')]) - item)) - op)))) - -(defn- pop-undo-op - [repo] - (let [undo-stack (get @*undo-ops repo) - [op undo-stack*] (pop-stack undo-stack)] - (swap! *undo-ops assoc repo undo-stack*) - (let [op' (mapv (fn [item] - (if (= (first item) ::db-transact) - (let [m (second item) - tx-data' (vec (:tx-data m))] - (if (seq tx-data') - [::db-transact (assoc m :tx-data tx-data')] - ::db-transact-no-tx-data)) - item)) - op)] - (when-not (some #{::db-transact-no-tx-data} op') - op')))) - -(defn- pop-redo-op - [repo] - (let [redo-stack (get @*redo-ops repo) - [op redo-stack*] (pop-stack redo-stack)] - (swap! *redo-ops assoc repo redo-stack*) - (let [op' (mapv (fn [item] - (if (= (first item) ::db-transact) - (let [m (second item) - tx-data' (vec (:tx-data m))] - (if (seq tx-data') - [::db-transact (assoc m :tx-data tx-data')] - ::db-transact-no-tx-data)) - item)) - op)] - (when-not (some #{::db-transact-no-tx-data} op') - op')))) - -(defn- empty-undo-stack? - [repo] - (empty? (get @*undo-ops repo))) - -(defn- empty-redo-stack? - [repo] - (empty? (get @*redo-ops repo))) - -(defn- reverse-datoms - [conn datoms schema added-ids retracted-ids undo? redo?] - (keep - (fn [[e a v _tx add?]] - (let [ref? (= :db.type/ref (get-in schema [a :db/valueType])) - op (if (or (and redo? add?) (and undo? (not add?))) - :db/add - :db/retract)] - (when (or (not ref?) - (d/entity @conn v) - (and (retracted-ids v) undo?) - (and (added-ids v) redo?)) ; entity exists - [op e a v]))) - datoms)) - -(defn- datom-attr - [datom] - (or (nth datom 1 nil) - (:a datom))) - -(defn- datom-value - [datom] - (or (nth datom 2 nil) - (:v datom))) - -(defn- datom-added? - [datom] - (let [value (nth datom 4 nil)] - (if (some? value) - value - (:added datom)))) - -(defn- reversed-move-target-ref - [datoms attr undo?] - (some (fn [datom] - (let [a (datom-attr datom) - v (datom-value datom) - added (datom-added? datom)] - (when (and (= a attr) - (if undo? (not added) added)) - v))) - datoms)) - -(defn- reversed-structural-target-conflicted? - [conn e->datoms undo?] - (some (fn [[_e datoms]] - (let [target-parent (reversed-move-target-ref datoms :block/parent undo?) - target-page (reversed-move-target-ref datoms :block/page undo?) - parent-ent (when (int? target-parent) (d/entity @conn target-parent)) - page-ent (when (int? target-page) (d/entity @conn target-page))] - (or (and target-parent - (or (nil? parent-ent) - (ldb/recycled? parent-ent))) - (and target-page - (or (nil? page-ent) - (ldb/recycled? page-ent)))))) - e->datoms)) - -(defn get-reversed-datoms - [conn undo? {:keys [tx-data added-ids retracted-ids]} tx-meta] - (let [recycle-restore-tx (when (and undo? - (= :delete-blocks (:outliner-op tx-meta))) - (->> tx-data - (keep (fn [datom] - (let [e (or (nth datom 0 nil) - (:e datom)) - a (datom-attr datom) - added (datom-added? datom)] - (when (and added - (= :logseq.property/deleted-at a)) - (d/entity @conn e))))) - (mapcat #(outliner-recycle/restore-tx-data @conn %)) - seq)) - redo? (not undo?) - e->datoms (->> (if redo? tx-data (reverse tx-data)) - (group-by :e)) - schema (:schema @conn) - structural-target-conflicted? (and undo? - (reversed-structural-target-conflicted? conn e->datoms undo?)) - reversed-tx-data (if structural-target-conflicted? - nil - (or (some-> recycle-restore-tx reverse seq) - (->> (mapcat - (fn [[e datoms]] - (cond - (and undo? (contains? added-ids e)) - [[:db/retractEntity e]] - - (and redo? (contains? retracted-ids e)) - [[:db/retractEntity e]] - - :else - (reverse-datoms conn datoms schema added-ids retracted-ids undo? redo?))) - e->datoms) - (remove nil?))))] - reversed-tx-data)) - -(defn- undo-redo-aux - [repo undo?] - (if-let [op (not-empty ((if undo? pop-undo-op pop-redo-op) repo))] - (let [conn (db/get-db repo false)] - (cond - (= ::ui-state (ffirst op)) - (do - ((if undo? push-redo-op push-undo-op) repo op) - (let [ui-state-str (second (first op))] - {:undo? undo? - :ui-state-str ui-state-str})) - - :else - (let [{:keys [tx-data tx-meta] :as data} (some #(when (= ::db-transact (first %)) - (second %)) op)] - (when (seq tx-data) - (let [reversed-tx-data (cond-> (get-reversed-datoms conn undo? data tx-meta) - undo? - reverse) - tx-meta' (-> tx-meta - (assoc - :gen-undo-ops? false - :undo? undo? - :redo? (not undo?))) - handler (fn handler [] - ((if undo? push-redo-op push-undo-op) repo op) - (let [editor-cursors (->> (filter #(= ::record-editor-info (first %)) op) - (map second)) - block-content (:block/title (d/entity @conn [:block/uuid (:block-uuid - (if undo? - (first editor-cursors) - (last editor-cursors)))]))] - {:undo? undo? - :editor-cursors editor-cursors - :block-content block-content}))] - (if (seq reversed-tx-data) - (if (undo-validate/valid-undo-redo-tx? conn reversed-tx-data) - (if util/node-test? - (try - (ldb/transact! conn reversed-tx-data tx-meta') - (handler) - (catch :default e - (log/error ::undo-redo-failed e) - (clear-history! repo) - (if undo? ::empty-undo-stack ::empty-redo-stack))) - (-> - (p/do! - ;; async write to the master worker - (ldb/transact! repo reversed-tx-data tx-meta') - (handler)) - (p/catch (fn [e] - (log/error ::undo-redo-failed e) - (clear-history! repo))))) - (do - (log/warn ::undo-redo-skip-invalid-op - {:undo? undo? - :outliner-op (:outliner-op tx-meta)}) - (undo-redo-aux repo undo?))) - (do - (log/warn ::undo-redo-skip-conflicted-op - {:undo? undo? - :outliner-op (:outliner-op tx-meta)}) - (undo-redo-aux repo undo?)))))))) - - (when ((if undo? empty-undo-stack? empty-redo-stack?) repo) - (if undo? ::empty-undo-stack ::empty-redo-stack)))) + (if util/node-test? + nil + (invoke-db-worker :thread-api/undo-redo-clear-history repo))) (defn undo [repo] - (undo-redo-aux repo true)) + (if util/node-test? + :frontend.undo-redo/empty-undo-stack + (or (some-> (invoke-db-worker :thread-api/undo-redo-undo repo) + normalize-empty-result) + :frontend.undo-redo/empty-undo-stack))) (defn redo [repo] - (undo-redo-aux repo false)) + (if util/node-test? + :frontend.undo-redo/empty-redo-stack + (or (some-> (invoke-db-worker :thread-api/undo-redo-redo repo) + normalize-empty-result) + :frontend.undo-redo/empty-redo-stack))) (defn record-editor-info! [repo editor-info] - (swap! *undo-ops - update repo - (fn [stack] - (if (seq stack) - (update stack (dec (count stack)) - (fn [op] - (conj (vec op) [::record-editor-info editor-info]))) - stack)))) + (when editor-info + (if util/node-test? + nil + (invoke-db-worker :thread-api/undo-redo-record-editor-info repo editor-info)))) (defn record-ui-state! [repo ui-state-str] (when ui-state-str - (push-undo-op repo [[::ui-state ui-state-str]]))) + (if util/node-test? + nil + (invoke-db-worker :thread-api/undo-redo-record-ui-state repo ui-state-str)))) -(defn gen-undo-ops! - [repo {:keys [tx-data tx-meta db-after db-before]}] - (let [{:keys [outliner-op local-tx?]} tx-meta] - (when (and - (= (:client-id tx-meta) (:client-id @state/state)) - (true? local-tx?) - outliner-op - (not (false? (:gen-undo-ops? tx-meta))) - (not (:create-today-journal? tx-meta))) - (let [all-ids (distinct (map :e tx-data)) - retracted-ids (set - (filter - (fn [id] (and (nil? (d/entity db-after id)) (d/entity db-before id))) - all-ids)) - added-ids (set - (filter - (fn [id] (and (nil? (d/entity db-before id)) (d/entity db-after id))) - all-ids)) - tx-data' (vec tx-data) - editor-info @state/*editor-info - _ (reset! state/*editor-info nil) - op (->> [(when editor-info [::record-editor-info editor-info]) - [::db-transact - {:tx-data tx-data' - :tx-meta tx-meta - :added-ids added-ids - :retracted-ids retracted-ids}]] - (remove nil?) - vec)] - ;; A new local edit invalidates any redo history. - (swap! *redo-ops assoc repo []) - (push-undo-op repo op))))) - -(defn listen-db-changes! - [repo conn] - (d/listen! conn ::gen-undo-ops - (fn [tx-report] (gen-undo-ops! repo tx-report)))) +(defn > entry' + (keep (fn [item] + (cond + (keyword? item) (name item) + (and (vector? item) (keyword? (first item))) (name (first item)) + :else nil))) + first + (or "entry")))) + +(def ^:private ui-entry-tags + #{::undo-redo/ui-state + ::undo-redo/record-editor-info + :frontend.worker.undo-redo/ui-state + :frontend.worker.undo-redo/record-editor-info}) + +(defn- ui-entry-item? + [item] + (and (vector? item) + (contains? ui-entry-tags (first item)))) + +(defn- filter-ui-items + [entry] + (if (vector? entry) + (->> entry + (remove ui-entry-item?) + vec) + entry)) + +(defn- empty-filtered-entry? + [entry] + (and (vector? entry) + (empty? entry))) + +(rum/defc payload-entry + [expanded?* id entry] + (let [expanded? (contains? @expanded?* id)] + [:div.rounded-md.border.p-2 + [:button.flex.w-full.items-center.justify-between.text-left + {:aria-expanded expanded? + :on-click (fn [_] + (swap! expanded?* + (fn [expanded] + (if (contains? expanded id) + (disj expanded id) + (conj expanded id)))))} + [:span.text-sm.font-medium (entry-title entry)] + [:span.opacity-60 (ui/rotating-arrow (not expanded?))]] + (when expanded? + [:pre.select-text.mt-2.text-xs.overflow-auto + (-> (strip-tx-data entry) + (fipp/pprint {:width 60}) + with-out-str)])])) + +(rum/defc payload-stack + [expanded?* label entries] + [:div.flex.flex-col.gap-2 + [:div.text-sm.font-medium label] + (if (seq entries) + (for [[idx entry] (map-indexed vector (reverse entries))] + (rum/with-key + (payload-entry expanded?* (str label "-" idx) entry) + (str label "-" idx))) + [:div.text-sm.opacity-50 "Empty"])]) + +(rum/defcs undo-redo-debug-ui < rum/reactive + (rum/local #{} ::expanded) + (rum/local false ::filter-ui-state?) + (rum/local nil ::history) + [state] + (let [repo (state/sub :git/current-repo) + history* (::history state) + _ (rum/react history*) + refresh! (fn [] + (when repo + (-> (undo-redo/> stack + (map filter-ui-items) + (remove empty-filtered-entry?)) + stack)) + undo-stack' (filter-stack undo-stack) + redo-stack' (filter-stack redo-stack)] + [:div.flex.flex-col.gap-3 + [:div.flex.gap-2.flex-wrap.items-center + (shui/button + {:size :sm + :disabled (or (nil? repo) (empty? undo-stack)) + :on-click (fn [e] + (history-handler/undo! e) + (js/setTimeout refresh! 0))} + (shui/tabler-icon "arrow-back-up") "undo") + (shui/button + {:size :sm + :disabled (or (nil? repo) (empty? redo-stack)) + :on-click (fn [e] + (history-handler/redo! e) + (js/setTimeout refresh! 0))} + (shui/tabler-icon "arrow-forward-up") "redo") + (shui/button + {:size :sm + :variant :outline + :disabled (nil? repo) + :on-click (fn [_] + (undo-redo/clear-history! repo) + (js/setTimeout refresh! 0))} + (shui/tabler-icon "trash") "clear-history") + (shui/button + {:size :sm + :variant :outline + :disabled (nil? repo) + :on-click (fn [_] (refresh!))} + (shui/tabler-icon "refresh") "refresh") + (shui/button + {:size :sm + :variant (if filter-ui-state? :default :outline) + :on-click (fn [_] + (swap! filter-ui-state?* not))} + (shui/tabler-icon "filter") "filter-ui-entry-global")] + + [:div.text-sm.opacity-70 + (str "undo=" (count undo-stack') + (when filter-ui-state? + (str "/" (count undo-stack))) + " redo=" (count redo-stack') + (when filter-ui-state? + (str "/" (count redo-stack))))] + + (payload-stack expanded?* "Undo" undo-stack') + (payload-stack expanded?* "Redo" redo-stack')])) diff --git a/src/main/frontend/worker/db/validate.cljs b/src/main/frontend/worker/db/validate.cljs index 50094abfb7..3d80553867 100644 --- a/src/main/frontend/worker/db/validate.cljs +++ b/src/main/frontend/worker/db/validate.cljs @@ -6,6 +6,7 @@ [frontend.worker.db.migrate :as db-migrate] [frontend.worker.shared-service :as shared-service] [logseq.db :as ldb] + [logseq.db-sync.checksum :as sync-checksum] [logseq.db.frontend.class :as db-class] [logseq.db.frontend.validate :as db-validate])) @@ -220,35 +221,47 @@ {:fix-db? true}))) (defn validate-db - [conn] - (fix-extends-cardinality! conn) - (fix-icon-wrong-type! conn) - (db-migrate/ensure-built-in-data-exists! conn) - (fix-non-closed-values! conn) - (fix-num-prefix-db-idents! conn) + ([conn] + (validate-db nil conn nil)) + ([_repo conn _options] + (fix-extends-cardinality! conn) + (fix-icon-wrong-type! conn) + (db-migrate/ensure-built-in-data-exists! conn) + (fix-non-closed-values! conn) + (fix-num-prefix-db-idents! conn) - (let [db @conn - {:keys [errors datom-count entities]} (db-validate/validate-db! db) - invalid-entity-ids (distinct (map (fn [e] (:db/id (:entity e))) errors))] + (let [db @conn + {:keys [errors datom-count entities]} (db-validate/validate-db! db) + invalid-entity-ids (distinct (map (fn [e] (:db/id (:entity e))) errors))] - (doseq [error errors] - (prn :debug - :entity (:entity error) - :error (dissoc error :entity))) + (doseq [error errors] + (prn :debug + :entity (:entity error) + :error (dissoc error :entity))) - (if errors - (do - (fix-invalid-blocks! conn errors) - (shared-service/broadcast-to-clients! :log [:db-invalid :error - {:msg "Validation errors" - :errors errors}]) - (shared-service/broadcast-to-clients! :notification - [(str "Validation detected " (count errors) " invalid block(s). These blocks may be buggy. Attempting to fix invalid blocks. Run validation again to see if they were fixed.") - :warning false])) + (if errors + (do + (fix-invalid-blocks! conn errors) + (shared-service/broadcast-to-clients! :log [:db-invalid :error + {:msg "Validation errors" + :errors errors}]) + (shared-service/broadcast-to-clients! :notification + [(str "Validation detected " (count errors) " invalid block(s). These blocks may be buggy. Attempting to fix invalid blocks. Run validation again to see if they were fixed.") + :warning false])) - (shared-service/broadcast-to-clients! :notification - [(str "Your graph is valid! " (assoc (db-validate/graph-counts db entities) :datoms datom-count)) - :success false])) - {:errors errors - :datom-count datom-count - :invalid-entity-ids invalid-entity-ids})) + (shared-service/broadcast-to-clients! :notification + [(str "Your graph is valid! " (assoc (db-validate/graph-counts db entities) :datoms datom-count)) + :success false])) + {:errors errors + :datom-count datom-count + :invalid-entity-ids invalid-entity-ids}))) + +(defn recompute-checksum-diagnostics + [_repo conn {:keys [local-checksum remote-checksum] :as _sync-diagnostics}] + (let [{:keys [checksum attrs blocks e2ee?]} (sync-checksum/recompute-checksum-diagnostics @conn)] + {:recomputed-checksum checksum + :local-checksum local-checksum + :remote-checksum remote-checksum + :e2ee? e2ee? + :checksum-attrs attrs + :blocks blocks})) diff --git a/src/main/frontend/worker/db_listener.cljs b/src/main/frontend/worker/db_listener.cljs index 10cd707c58..3ba9c319fc 100644 --- a/src/main/frontend/worker/db_listener.cljs +++ b/src/main/frontend/worker/db_listener.cljs @@ -14,6 +14,15 @@ (defmulti listen-db-changes (fn [listen-key & _] listen-key)) +(defn- transit-safe-tx-meta + [tx-meta] + (when (map? tx-meta) + (->> tx-meta + (remove (fn [[k v]] + (or (= :error-handler k) + (fn? v)))) + (into {})))) + (defn- sync-db-to-main-thread "Return tx-report" [repo conn {:keys [tx-meta] :as tx-report}] @@ -28,7 +37,7 @@ {:repo repo :request-id (:request-id tx-meta) :tx-data (:tx-data tx-report') - :tx-meta tx-meta} + :tx-meta (transit-safe-tx-meta tx-meta)} (dissoc result :tx-report))] (shared-service/broadcast-to-clients! :sync-db-changes data)) @@ -84,12 +93,13 @@ (d/listen! conn ::listen-db-changes! (fn listen-db-changes!-inner [{:keys [tx-data tx-meta] :as tx-report}] - (remove-old-embeddings-and-reset-new-updates! conn tx-data tx-meta) - (when (and (seq tx-data) (not (:mark-embedding? tx-meta))) - (let [tx-report' (if sync-db-to-main-thread? - (sync-db-to-main-thread repo conn tx-report) - tx-report) - opt {:repo repo}] - (db-sync/update-local-sync-checksum! repo tx-report') - (doseq [[k handler-fn] handlers] - (handler-fn k opt tx-report')))))))) + (when-not (:batch-tx? @conn) + (remove-old-embeddings-and-reset-new-updates! conn tx-data tx-meta) + (when (and (seq tx-data) (not (:mark-embedding? tx-meta))) + (let [tx-report' (if sync-db-to-main-thread? + (sync-db-to-main-thread repo conn tx-report) + tx-report) + opt {:repo repo}] + (db-sync/update-local-sync-checksum! repo tx-report') + (doseq [[k handler-fn] handlers] + (handler-fn k opt tx-report'))))))))) diff --git a/src/main/frontend/worker/db_worker.cljs b/src/main/frontend/worker/db_worker.cljs index e6918ff0c4..d86d125ce1 100644 --- a/src/main/frontend/worker/db_worker.cljs +++ b/src/main/frontend/worker/db_worker.cljs @@ -19,7 +19,6 @@ [frontend.worker.db.validate :as worker-db-validate] [frontend.worker.embedding :as embedding] [frontend.worker.export :as worker-export] - [frontend.worker.handler.page :as worker-page] [frontend.worker.pipeline :as worker-pipeline] [frontend.worker.publish] [frontend.worker.search :as search] @@ -31,6 +30,7 @@ [frontend.worker.sync.crypt :as sync-crypt] [frontend.worker.sync.log-and-state :as rtc-log-and-state] [frontend.worker.thread-atom] + [frontend.worker.undo-redo :as worker-undo-redo] [goog.object :as gobj] [lambdaisland.glogi :as log] [lambdaisland.glogi.console :as glogi-console] @@ -50,12 +50,12 @@ [logseq.db.sqlite.export :as sqlite-export] [logseq.db.sqlite.gc :as sqlite-gc] [logseq.db.sqlite.util :as sqlite-util] - [logseq.outliner.core :as outliner-core] [logseq.outliner.op :as outliner-op] [logseq.outliner.recycle :as outliner-recycle] [me.tonsky.persistent-sorted-set :as set :refer [BTSet]] [missionary.core :as m] - [promesa.core :as p])) + [promesa.core :as p] + [goog.functions :as gfun])) (def ^:private worker-bootstrap-loaded-key "__logseq_db_worker_bootstrap_loaded__") @@ -85,6 +85,8 @@ (defonce *opfs-pools worker-state/*opfs-pools) (defonce *publishing? (atom false)) (defonce ^:private *search-index-build-ids (atom {})) +(defonce ^:private *client-ops-cleanup-timers (atom {})) +(def ^:private client-ops-cleanup-interval-ms (* 3 60 60 1000)) (defn- check-worker-scope! [] @@ -180,9 +182,13 @@ (defn- close-db-aux! [repo ^Object db ^Object search ^Object client-ops] + (when-let [timer (get @*client-ops-cleanup-timers repo)] + (js/clearInterval timer)) + (swap! *client-ops-cleanup-timers dissoc repo) (swap! *sqlite-conns dissoc repo) (swap! *datascript-conns dissoc repo) (swap! *client-ops-conns dissoc repo) + (swap! client-op/*repo->pending-local-tx-count dissoc repo) (swap! *search-index-build-ids dissoc repo) (search/clear-fuzzy-search-indice! repo) (when db (.close db)) @@ -259,6 +265,23 @@ :kv/value (common-util/time-ms)}] {:skip-validate-db? true})))) +(defn- run-client-ops-cleanup! + [repo] + (let [protected-tx-ids (worker-undo-redo/referenced-history-tx-ids repo)] + (client-op/cleanup-finished-history-ops! repo protected-tx-ids) + nil)) + +(defn- ensure-client-ops-cleanup-timer! + [repo] + (when (and (not @*publishing?) + repo + (nil? (get @*client-ops-cleanup-timers repo))) + (let [timer (js/setInterval (fn [] + (run-client-ops-cleanup! repo)) + client-ops-cleanup-interval-ms)] + (swap! *client-ops-cleanup-timers assoc repo timer)) + nil)) + (def ^:private recycle-gc-kv :logseq.kv/recycle-last-gc-at) (defn- maybe-run-recycle-gc! @@ -289,6 +312,7 @@ (when-not @*publishing? (common-sqlite/create-kvs-table! client-ops-db)) (search/create-tables-and-triggers! search-db) (ldb/register-transact-pipeline-fn! worker-pipeline/transact-pipeline) + (ldb/register-debounce-fn! (gfun/debounce d/store 1000)) (let [conn (common-sqlite/get-storage-conn storage db-schema/schema) _ (db-fix/check-and-fix-schema! conn) _ (when datoms @@ -321,6 +345,7 @@ (swap! *client-ops-conns assoc repo client-ops-conn) (when (and (not @*publishing?) (not= client-op/schema-in-db (d/schema @client-ops-conn))) (d/reset-schema! client-ops-conn client-op/schema-in-db)) + (ensure-client-ops-cleanup-timer! repo) (let [initial-tx-report (when-not (or initial-data-exists? (seq datoms) sync-download-graph?) @@ -473,8 +498,8 @@ (sync-crypt/content @conn options))) +(defn- sync-diagnostics-for-validation + [repo] + {:local-tx (client-op/get-local-tx repo) + :remote-tx (get @db-sync/*repo->latest-remote-tx repo) + :local-checksum (client-op/get-local-checksum repo) + :remote-checksum (get @db-sync/*repo->latest-remote-checksum repo)}) + (def-thread-api :thread-api/validate-db [repo] (when-let [conn (worker-state/get-datascript-conn repo)] - (worker-db-validate/validate-db conn))) + (worker-db-validate/validate-db repo conn (sync-diagnostics-for-validation repo)))) + +(def-thread-api :thread-api/recompute-checksum-diagnostics + [repo] + (when-let [conn (worker-state/get-datascript-conn repo)] + (worker-db-validate/recompute-checksum-diagnostics repo conn (sync-diagnostics-for-validation repo)))) ;; Returns an export-edn map for given repo. When there's an unexpected error, a map ;; with key :export-edn-error is returned @@ -1110,36 +1179,6 @@ dbs (ldb/read-transit-str r)] (p/all (map #(.unsafeUnlinkDB this (:name %)) dbs))))) -(defn- delete-page! - [conn page-uuid opts] - (let [error-handler (fn [{:keys [msg]}] - (worker-util/post-message :notification - [[:div [:p msg]] :error]))] - (worker-page/delete! conn page-uuid (merge opts {:error-handler error-handler})))) - -(defn- create-page! - [conn title options] - (try - (worker-page/create! conn title options) - (catch :default e - (js/console.error e) - (throw e)))) - -(defn- outliner-register-op-handlers! - [] - (outliner-op/register-op-handlers! - {:create-page (fn [conn [title options]] - (create-page! conn title options)) - :rename-page (fn [conn [page-uuid new-title]] - (if (string/blank? new-title) - (throw (ex-info "Page name shouldn't be blank" {:block/uuid page-uuid - :block/title new-title})) - (outliner-core/save-block! conn - {:block/uuid page-uuid - :block/title new-title}))) - :delete-page (fn [conn [page-uuid opts]] - (delete-page! conn page-uuid opts))})) - (defn- on-become-master [repo start-opts] (js/Promise. @@ -1232,7 +1271,6 @@ (log/set-levels {:glogi/root :info}) (log/add-handler worker-state/log-append!) (check-worker-scope!) - (outliner-register-op-handlers!) (js/setInterval #(.postMessage js/self "keepAliveResponse") (* 1000 25)) (Comlink/expose proxy-object) (let [^js wrapped-main-thread* (Comlink/wrap js/self) diff --git a/src/main/frontend/worker/sync.cljs b/src/main/frontend/worker/sync.cljs index 4afdd371cb..1ad2b967ea 100644 --- a/src/main/frontend/worker/sync.cljs +++ b/src/main/frontend/worker/sync.cljs @@ -1,21 +1,21 @@ (ns frontend.worker.sync "Sync client" - (:require [frontend.worker.shared-service :as shared-service] - [frontend.worker.state :as worker-state] - [frontend.worker.sync.apply-txs :as sync-apply] - [frontend.worker.sync.assets :as sync-assets] - [frontend.worker.sync.auth :as sync-auth] - [frontend.worker.sync.client-op :as client-op] - [frontend.worker.sync.crypt :as sync-crypt] - [frontend.worker.sync.handle-message :as sync-handle-message] - [frontend.worker.sync.large-title :as sync-large-title] - [frontend.worker.sync.presence :as sync-presence] - [frontend.worker.sync.transport :as sync-transport] - [frontend.worker.sync.upload :as sync-upload] - [lambdaisland.glogi :as log] - [logseq.common.util :as common-util] - [logseq.db-sync.checksum :as sync-checksum] - [promesa.core :as p])) + (:require + [frontend.worker.shared-service :as shared-service] + [frontend.worker.state :as worker-state] + [frontend.worker.sync.apply-txs :as sync-apply] + [frontend.worker.sync.assets :as sync-assets] + [frontend.worker.sync.auth :as sync-auth] + [frontend.worker.sync.client-op :as client-op] + [frontend.worker.sync.handle-message :as sync-handle-message] + [frontend.worker.sync.large-title :as sync-large-title] + [frontend.worker.sync.presence :as sync-presence] + [frontend.worker.sync.transport :as sync-transport] + [frontend.worker.sync.upload :as sync-upload] + [lambdaisland.glogi :as log] + [logseq.common.util :as common-util] + [logseq.db-sync.checksum :as sync-checksum] + [promesa.core :as p])) (def ^:private reconnect-base-delay-ms 1000) (def ^:private reconnect-max-delay-ms 30000) @@ -24,6 +24,7 @@ (def ^:private ws-stale-timeout-ms 600000) (defonce *repo->latest-remote-tx sync-apply/*repo->latest-remote-tx) +(defonce *repo->latest-remote-checksum sync-apply/*repo->latest-remote-checksum) (defonce *start-inflight-target (atom nil)) (defn fail-fast @@ -40,16 +41,18 @@ (sync-presence/sync-counts {:get-datascript-conn worker-state/get-datascript-conn :get-client-ops-conn worker-state/get-client-ops-conn + :get-pending-local-tx-count client-op/get-pending-local-tx-count :get-unpushed-asset-ops-count client-op/get-unpushed-asset-ops-count :get-local-tx client-op/get-local-tx + :get-local-checksum client-op/get-local-checksum :get-graph-uuid client-op/get-graph-uuid - :latest-remote-tx @*repo->latest-remote-tx} + :latest-remote-tx @*repo->latest-remote-tx + :latest-remote-checksum @*repo->latest-remote-checksum} repo)) (defn update-local-sync-checksum! [repo tx-report] - (when (and (worker-state/get-client-ops-conn repo) - (not (sync-crypt/graph-e2ee? repo))) + (when (worker-state/get-client-ops-conn repo) (client-op/update-local-checksum repo (sync-checksum/update-checksum (client-op/get-local-checksum repo) tx-report)))) diff --git a/src/main/frontend/worker/sync/apply_txs.cljs b/src/main/frontend/worker/sync/apply_txs.cljs index 58fb663de7..2f7515459b 100644 --- a/src/main/frontend/worker/sync/apply_txs.cljs +++ b/src/main/frontend/worker/sync/apply_txs.cljs @@ -1,6 +1,7 @@ (ns frontend.worker.sync.apply-txs "Pending tx and remote tx application helpers for db sync." (:require [clojure.set :as set] + [clojure.string :as string] [datascript.core :as d] [frontend.worker.shared-service :as shared-service] [frontend.worker.state :as worker-state] @@ -13,26 +14,30 @@ [frontend.worker.sync.large-title :as sync-large-title] [frontend.worker.sync.presence :as sync-presence] [frontend.worker.sync.transport :as sync-transport] + [frontend.worker.undo-redo :as worker-undo-redo] [lambdaisland.glogi :as log] [logseq.db :as ldb] - [logseq.db-sync.cycle :as sync-cycle] [logseq.db-sync.order :as sync-order] [logseq.db.common.normalize :as db-normalize] - [logseq.db.frontend.schema :as db-schema] + [logseq.db.frontend.property.type :as db-property-type] [logseq.db.sqlite.util :as sqlite-util] + [logseq.outliner.core :as outliner-core] + [logseq.outliner.op :as outliner-op] + [logseq.outliner.op.construct :as op-construct] + [logseq.outliner.page :as outliner-page] + [logseq.outliner.property :as outliner-property] [logseq.outliner.recycle :as outliner-recycle] - [logseq.undo-redo-validate :as undo-validate] [promesa.core :as p])) (defonce *repo->latest-remote-tx (atom {})) +(defonce *repo->latest-remote-checksum (atom {})) (defonce *upload-temp-opfs-pool (atom nil)) (defn fail-fast [tag data] (log/error tag data) (throw (ex-info (name tag) data))) -(declare enqueue-asset-task! - replace-string-block-tempids-with-lookups) +(declare enqueue-asset-task!) (defn- current-client [repo] (sync-presence/current-client worker-state/*db-sync-client repo)) @@ -44,10 +49,13 @@ (sync-presence/sync-counts {:get-datascript-conn worker-state/get-datascript-conn :get-client-ops-conn worker-state/get-client-ops-conn + :get-pending-local-tx-count client-op/get-pending-local-tx-count :get-unpushed-asset-ops-count client-op/get-unpushed-asset-ops-count :get-local-tx client-op/get-local-tx + :get-local-checksum client-op/get-local-checksum :get-graph-uuid client-op/get-graph-uuid - :latest-remote-tx @*repo->latest-remote-tx} + :latest-remote-tx @*repo->latest-remote-tx + :latest-remote-checksum @*repo->latest-remote-checksum} repo)) (defn- broadcast-rtc-state! [client] @@ -56,10 +64,13 @@ :rtc-sync-state (sync-presence/rtc-state-payload sync-counts client)))) +(def reverse-data-ignored-attrs + #{:logseq.property.embedding/hnsw-label-updated-at + :block/tx-id}) + (def rtc-ignored-attrs (set/union - #{:logseq.property.embedding/hnsw-label-updated-at - :block/tx-id} + reverse-data-ignored-attrs rtc-const/ignore-attrs-when-syncing rtc-const/ignore-entities-when-init-upload)) @@ -73,17 +84,21 @@ (remove (fn [[_op e]] (contains? rtc-const/ignore-entities-when-init-upload e))))) -(defn reverse-tx-data [tx-data] +(declare replay-canonical-outliner-op! + invalid-rebase-op!) + +(defn reverse-tx-data [_db-before db-after tx-data] (->> tx-data + reverse (keep (fn [[e a v t added]] (when (and (some? a) (some? v) (some? t) (boolean? added)) - [(if added :db/retract :db/add) e a v t]))))) + [(if added :db/retract :db/add) e a v t]))) + (db-normalize/replace-attr-retract-with-retract-entity-v2 db-after))) -(defn reverse-normalized-tx-data [tx-data] - (->> tx-data - (keep (fn [[op e a v t]] - (when (and (some? a) (some? v) (some? t)) - [(if (= :db/add op) :db/retract :db/add) e a v t]))))) +(defn normalize-rebased-pending-tx + [{:keys [db-before db-after tx-data]}] + {:normalized-tx-data (normalize-tx-data db-after db-before tx-data) + :reversed-datoms (reverse-tx-data db-before db-after tx-data)}) (defn- get-graph-id [repo] (sync-large-title/get-graph-id worker-state/get-datascript-conn repo)) @@ -148,480 +163,262 @@ (when-let [queue (:asset-queue client)] (swap! queue (fn [prev] (p/then prev (fn [_] (task))))))) -(defn- persist-local-tx! [repo normalized-tx-data reversed-datoms tx-meta] +(def ^:private canonical-transact-op op-construct/canonical-transact-op) + +(defn- contains-transact-op? + [ops] + (op-construct/contains-transact-op? ops)) + +(defn- explicit-transact-forward-op? + [tx-meta] + (let [explicit-forward-ops (or (some-> (:db-sync/forward-outliner-ops tx-meta) + seq + vec) + (some-> (:outliner-ops tx-meta) + seq + vec))] + (and (seq explicit-forward-ops) + (contains-transact-op? explicit-forward-ops)))) + +(defn- derive-history-outliner-ops + [db-before db-after tx-data tx-meta] + ;; Rebased txs can carry explicit forward ops like [[:transact nil]]. + ;; Keep them as raw-tx placeholders instead of forcing semantic canonicalization. + (if (explicit-transact-forward-op? tx-meta) + {:forward-outliner-ops canonical-transact-op + :inverse-outliner-ops nil} + (op-construct/derive-history-outliner-ops db-before db-after tx-data tx-meta))) + +(defn- inferred-outliner-ops? + [tx-meta] + (and (nil? (:outliner-ops tx-meta)) + (not (:undo? tx-meta)) + (not (:redo? tx-meta)) + (not= :batch-import-edn (:outliner-op tx-meta)))) + +(declare apply-history-action!) +(defn- persist-local-tx! [repo {:keys [db-before db-after tx-data tx-meta] :as tx-report} normalized-tx-data reversed-datoms] (when-let [conn (client-ops-conn repo)] - (let [tx-id (random-uuid) - now (.now js/Date)] + (let [tx-id (or (:db-sync/tx-id tx-meta) (random-uuid)) + existing-ent (d/entity @conn [:db-sync/tx-id tx-id]) + should-inc-pending? (not= true (:db-sync/pending? existing-ent)) + now (.now js/Date) + {:keys [forward-outliner-ops inverse-outliner-ops]} + (derive-history-outliner-ops db-before db-after tx-data tx-meta) + inferred-outliner-ops?' (inferred-outliner-ops? tx-meta)] (ldb/transact! conn [{:db-sync/tx-id tx-id :db-sync/normalized-tx-data normalized-tx-data :db-sync/reversed-tx-data reversed-datoms + :db-sync/pending? true :db-sync/outliner-op (:outliner-op tx-meta) + :db-sync/undo-redo? (cond + (:undo? tx-meta) + :undo + (:redo? tx-meta) + :redo + :else + :none) + :db-sync/forward-outliner-ops forward-outliner-ops + :db-sync/inverse-outliner-ops inverse-outliner-ops + :db-sync/inferred-outliner-ops? inferred-outliner-ops?' :db-sync/created-at now}]) - (when-let [client (current-client repo)] - (broadcast-rtc-state! client)) + (worker-undo-redo/gen-undo-ops! repo tx-report tx-id + {:apply-history-action! apply-history-action!}) + (when should-inc-pending? + (client-op/adjust-pending-local-tx-count! repo 1) + (when-let [client (current-client repo)] + (broadcast-rtc-state! client))) tx-id))) (defn pending-txs [repo & {:keys [limit]}] (when-let [conn (client-ops-conn repo)] (let [db @conn - graph-db (some-> (worker-state/get-datascript-conn repo) deref) datoms (d/datoms db :avet :db-sync/created-at) - datoms' (if limit (take limit datoms) datoms)] - (->> datoms' + take-limit (fn [c] + (if limit (take limit c) c))] + (->> datoms (map (fn [datom] (d/entity db (:e datom)))) + (filter (fn [e] (:db-sync/pending? e))) + take-limit (keep (fn [ent] - (let [tx-id (:db-sync/tx-id ent)] + (let [tx-id (:db-sync/tx-id ent) + tx' (:db-sync/normalized-tx-data ent) + reversed-tx' (:db-sync/reversed-tx-data ent)] {:tx-id tx-id :outliner-op (:db-sync/outliner-op ent) - :tx (replace-string-block-tempids-with-lookups - graph-db - (:db-sync/normalized-tx-data ent)) - :reversed-tx (replace-string-block-tempids-with-lookups - graph-db - (:db-sync/reversed-tx-data ent))}))) + :forward-outliner-ops (:db-sync/forward-outliner-ops ent) + :inverse-outliner-ops (:db-sync/inverse-outliner-ops ent) + :inferred-outliner-ops? (:db-sync/inferred-outliner-ops? ent) + :db-sync/undo-redo (:db-sync/undo-redo? ent) + :tx tx' + :reversed-tx reversed-tx'}))) vec)))) +(defn- pending-tx-by-id + [repo tx-id] + (when-let [conn (client-ops-conn repo)] + (when-let [ent (d/entity @conn [:db-sync/tx-id tx-id])] + {:tx-id (:db-sync/tx-id ent) + :outliner-op (:db-sync/outliner-op ent) + :forward-outliner-ops (:db-sync/forward-outliner-ops ent) + :inverse-outliner-ops (:db-sync/inverse-outliner-ops ent) + :db-sync/undo-redo (:db-sync/undo-redo? ent) + :tx (:db-sync/normalized-tx-data ent) + :reversed-tx (:db-sync/reversed-tx-data ent)}))) + (defn remove-pending-txs! [repo tx-ids] (when (seq tx-ids) (when-let [conn (client-ops-conn repo)] - (ldb/transact! conn - (mapv (fn [tx-id] - [:db/retractEntity [:db-sync/tx-id tx-id]]) - tx-ids)) - (when-let [client (current-client repo)] - (broadcast-rtc-state! client))))) + (let [pending-to-remove (->> tx-ids + (keep (fn [tx-id] + (when (true? (:db-sync/pending? (d/entity @conn [:db-sync/tx-id tx-id]))) + tx-id))) + count)] + (ldb/transact! conn + (mapv (fn [tx-id] + [:db/add [:db-sync/tx-id tx-id] :db-sync/pending? false]) + tx-ids)) + (when (pos? pending-to-remove) + (client-op/adjust-pending-local-tx-count! repo (- pending-to-remove))) + (when-let [client (current-client repo)] + (broadcast-rtc-state! client)))))) (defn clear-pending-txs! [repo] (remove-pending-txs! repo (mapv :tx-id (pending-txs repo)))) -(comment - (defn- clear-pending-txs! - [repo] - (when-let [conn (client-ops-conn repo)] - (let [tx-data (->> (d/datoms @conn :avet :db-sync/created-at) - (map (fn [d] - [:db/retractEntity (:e d)])))] - (d/transact! conn tx-data))))) +(defn- usable-history-ops + [ops] + (let [ops' (some-> ops seq vec)] + (when (and (seq ops') + (not= canonical-transact-op ops')) + ops'))) -(defn get-lookup-id - [x] - (when (and (vector? x) - (= 2 (count x)) - (= :block/uuid (first x))) - (second x))) +(defn- semantic-op-stream? + [ops] + (boolean (seq (usable-history-ops ops)))) -(defn- created-block-uuid-entry - [item] - (when (and (vector? item) - (= :db/add (first item)) - (>= (count item) 4) - (= :block/uuid (nth item 2))) - [(second item) (nth item 3)])) +(defn- history-action-ops + [{:keys [forward-outliner-ops inverse-outliner-ops]} undo?] + (if undo? + (usable-history-ops inverse-outliner-ops) + (usable-history-ops forward-outliner-ops))) -(defn- created-block-uuid-by-entity-id - [tx-data] - (->> tx-data - (keep created-block-uuid-entry) - (into {}))) +(declare precreate-missing-save-blocks! replay-canonical-outliner-op!) -(defn- created-block-context - [tx-data] - (let [uuid-by-entity-id (created-block-uuid-by-entity-id tx-data)] - {:uuid-by-entity-id uuid-by-entity-id - :uuids (set (vals uuid-by-entity-id))})) +(defn- inline-history-action + [tx-meta] + (let [forward-outliner-ops (or (:db-sync/forward-outliner-ops tx-meta) + (:forward-outliner-ops tx-meta)) + inverse-outliner-ops (or (:db-sync/inverse-outliner-ops tx-meta) + (:inverse-outliner-ops tx-meta))] + (when (and (seq forward-outliner-ops) (seq inverse-outliner-ops)) + {:outliner-op (:outliner-op tx-meta) + :forward-outliner-ops forward-outliner-ops + :inverse-outliner-ops inverse-outliner-ops}))) -(defn- tx-created-block-uuid - [{:keys [uuid-by-entity-id uuids]} entity-id] - (or (get uuid-by-entity-id entity-id) - (let [lookup-id (get-lookup-id entity-id)] - (when (contains? uuids lookup-id) - lookup-id)))) +(defn ^:large-vars/cleanup-todo apply-history-action! + [repo tx-id undo? tx-meta] + (let [debug-data {:tx-id tx-id + :undo? undo? + :tx-meta tx-meta}] + (if-let [conn (worker-state/get-datascript-conn repo)] + (if-let [action (or (pending-tx-by-id repo tx-id) + (inline-history-action tx-meta))] + (let [semantic-forward? (semantic-op-stream? (:forward-outliner-ops action)) + ops (history-action-ops action undo?) + history-tx-id (let [provided-history-tx-id (:db-sync/tx-id tx-meta)] + (if (and (uuid? provided-history-tx-id) + (not= provided-history-tx-id tx-id)) + provided-history-tx-id + (random-uuid))) + tx-meta' (cond-> {:local-tx? true + :gen-undo-ops? false + :persist-op? true + :undo? undo? + :db-sync/tx-id history-tx-id + :db-sync/source-tx-id (or (:db-sync/source-tx-id tx-meta) + tx-id)} -(defn- add-datom-ref-block-uuids - [item] - (when (and (vector? item) - (= :db/add (first item))) - (cond-> [] - (get-lookup-id (second item)) - (conj (get-lookup-id (second item))) + (:outliner-op action) + (assoc :outliner-op (:outliner-op action)) - (and (>= (count item) 4) - (get-lookup-id (nth item 3))) - (conj (get-lookup-id (nth item 3)))))) + (seq (if undo? (:inverse-outliner-ops action) + (:forward-outliner-ops action))) + (assoc :db-sync/forward-outliner-ops + (vec (if undo? (:inverse-outliner-ops action) + (:forward-outliner-ops action)))) -(defn drop-missing-created-block-datoms - [db tx-data] - (if db - (let [{:keys [uuid-by-entity-id]} (created-block-context tx-data) - missing-created-uuids (->> (vals uuid-by-entity-id) - (remove #(d/entity db [:block/uuid %])) - set)] - (if (seq missing-created-uuids) - (remove (fn [item] - (when (vector? item) - (let [entity-lookup-id (get-lookup-id (second item)) - value-lookup-id (when (>= (count item) 4) - (get-lookup-id (nth item 3))) - created-uuid (or (get uuid-by-entity-id (second item)) - entity-lookup-id)] - (or (contains? missing-created-uuids created-uuid) - (contains? missing-created-uuids entity-lookup-id) - (contains? missing-created-uuids value-lookup-id))))) - tx-data) - tx-data)) - tx-data)) + (seq (if undo? (:forward-outliner-ops action) + (:inverse-outliner-ops action))) + (assoc :db-sync/inverse-outliner-ops + (vec (if undo? (:forward-outliner-ops action) + (:inverse-outliner-ops action)))))] + ;; (prn :debug :outliner-ops) + ;; (pprint/pprint (select-keys action [:tx-id :outliner-op :forward-outliner-ops :inverse-outliner-ops])) + ;; (prn :debug :tx-meta) + ;; (pprint/pprint tx-meta) + (cond + (and semantic-forward? + (not (seq ops))) + (fail-fast :db-sync/missing-history-action-semantic-ops + {:repo repo + :tx-id tx-id + :undo? undo? + :forward-outliner-ops (:forward-outliner-ops action) + :inverse-outliner-ops (:inverse-outliner-ops action)}) -(defn- missing-block-ref? - [db x] - (and db - (or (and (vector? x) - (some? (get-lookup-id x)) - (nil? (d/entity db x))) - (and (number? x) - (not (neg? x)) - (nil? (d/entity db x)))))) + (and semantic-forward? + (contains-transact-op? (if undo? (:inverse-outliner-ops action) + (:forward-outliner-ops action)))) + (fail-fast :db-sync/invalid-history-action-semantic-ops + {:reason :contains-transact-op + :repo repo + :tx-id tx-id + :undo? undo? + :ops (if undo? (:inverse-outliner-ops action) + (:forward-outliner-ops action))}) -(defn- invalid-block-ref? - [db x] - (missing-block-ref? db x)) + (seq ops) + (try + (ldb/batch-transact-with-temp-conn! + conn + tx-meta' + (fn [row-conn] + (precreate-missing-save-blocks! row-conn ops) + (doseq [op ops] + (replay-canonical-outliner-op! row-conn op)))) + {:applied? true + :source :semantic-ops + :history-tx-id history-tx-id} + (catch :default error + (if semantic-forward? + (if undo? + {:applied? false + :reason :invalid-history-action-ops + :error error} + (throw (ex-info (name :db-sync/invalid-history-action-semantic-ops) + {:reason :invalid-history-action-ops + :repo repo + :tx-id tx-id + :undo? undo? + :ops ops + :error error + :action action}))) + {:applied? false + :reason :invalid-history-action-ops + :error error}))) -(defn- ref-attr? - [db a] - (and db - (keyword? a) - (= :db.type/ref - (:db/valueType (d/entity db a))))) - -(defn- tx-entity-key - [entity] - (or (get-lookup-id entity) - entity)) - -(defn- strip-tx-id - [item] - (if (= (count item) 5) - (vec (butlast item)) - item)) - -(defn drop-orphaning-parent-retracts - [tx-data] - (let [entities-with-parent-add (->> tx-data - (keep (fn [item] - (when (and (vector? item) - (= :db/add (first item)) - (= :block/parent (nth item 2 nil))) - (tx-entity-key (second item))))) - set)] - (remove (fn [item] - (and (vector? item) - (= :db/retract (first item)) - (= :block/parent (nth item 2 nil)) - (not (contains? entities-with-parent-add - (tx-entity-key (second item)))))) - tx-data))) - -(defn- created-block-ref? - [created-context x] - (when-let [block-uuid (or (tx-created-block-uuid created-context x) - (get-lookup-id x))] - (contains? (:uuids created-context) block-uuid))) - -(defn- invalid-block-uuid? - [db created-context broken-block-uuids block-uuid] - (and block-uuid - (or (contains? broken-block-uuids block-uuid) - (and (not (contains? (:uuids created-context) block-uuid)) - (nil? (d/entity db [:block/uuid block-uuid])))))) - -(defn- add-datom-invalid-block-ref? - [db created-context broken-block-uuids item] - (some (partial invalid-block-uuid? db created-context broken-block-uuids) - (add-datom-ref-block-uuids item))) - -(defn- broken-created-block-uuids - [db created-context tx-data] - (loop [broken-block-uuids #{}] - (let [next-broken-block-uuids (->> tx-data - (keep (fn [item] - (when (and (vector? item) - (= :db/add (first item)) - (add-datom-invalid-block-ref? db created-context broken-block-uuids item)) - (tx-created-block-uuid created-context (second item))))) - (into broken-block-uuids))] - (if (= broken-block-uuids next-broken-block-uuids) - broken-block-uuids - (recur next-broken-block-uuids))))) - -(defn- invalid-block-ref-datom? - [db created-context broken-block-uuids item] - (when (vector? item) - (let [op (first item) - e (second item) - a (nth item 2 nil) - has-value? (>= (count item) 4) - v (when has-value? (nth item 3)) - block-uuid (tx-created-block-uuid created-context e) - value-ref? (and has-value? - (contains? #{:db/add :db/retract} op) - (ref-attr? db a))] - (or (and (= :db/add op) - (add-datom-invalid-block-ref? db created-context broken-block-uuids item)) - (contains? broken-block-uuids block-uuid) - (and (contains? #{:db/add :db/retract} op) - (not (created-block-ref? created-context e)) - (invalid-block-ref? db e)) - (and (= :db/retractEntity op) - (number? e) - (not (created-block-ref? created-context e)) - (invalid-block-ref? db e)) - (and value-ref? - (not (created-block-ref? created-context v)) - (invalid-block-ref? db v)))))) - -(defn- sanitize-block-ref-datoms - [db tx-data] - (if db - (let [created-context (created-block-context tx-data) - broken-block-uuids (broken-created-block-uuids db created-context tx-data)] - (remove (partial invalid-block-ref-datom? db created-context broken-block-uuids) - tx-data)) - tx-data)) - -(defn- canonical-entity-id - [db e] - (cond - (vector? e) (or (get-lookup-id e) e) - (and (number? e) (not (neg? e))) (or (:block/uuid (d/entity db e)) e) - :else e)) - -(defn- remote-updated-attr-keys - [db tx-data] - (->> tx-data - (keep (fn [item] - (when (and (vector? item) - (>= (count item) 4) - (contains? #{:db/add :db/retract} (first item))) - [(canonical-entity-id db (second item)) - (nth item 2)]))) - set)) - -(defn- resolve-string-block-tempid - [db x] - (when (and db (string? x)) - (when-let [block-uuid (parse-uuid x)] - (when (d/entity db [:block/uuid block-uuid]) - [:block/uuid block-uuid])))) - -(defn- string-block-uuid->lookup - [x] - (when (string? x) - (when-let [block-uuid (parse-uuid x)] - [:block/uuid block-uuid]))) - -(defn replace-string-block-tempids-with-lookups - [db tx-data] - (if db - (let [created-string-entity-ids (->> tx-data - (keep (fn [item] - (when (and (vector? item) - (= :db/add (first item)) - (>= (count item) 4) - (string? (second item)) - (= :block/uuid (nth item 2))) - (second item)))) - set) - replace-entity (fn [entity] - (if (contains? created-string-entity-ids entity) - entity - (or (string-block-uuid->lookup entity) - (resolve-string-block-tempid db entity) - entity)))] - (mapv (fn [item] - (if (and (vector? item) (>= (count item) 2)) - (let [op (first item) - entity' (replace-entity (second item)) - has-value? (>= (count item) 4) - attr (nth item 2 nil) - value (when has-value? (nth item 3)) - value' (if (and has-value? - (contains? db-schema/ref-type-attributes attr)) - (replace-entity value) - value)] - (cond-> item - (and (contains? #{:db/add :db/retract :db/retractEntity} op) - (not= (second item) entity')) - (assoc 1 entity') - (and has-value? (not= value value')) - (assoc 3 value'))) - item)) - tx-data)) - tx-data)) - -(defn drop-remote-conflicted-local-tx - [db remote-updated-keys tx-data] - (if (seq remote-updated-keys) - (let [structural-attrs #{:block/parent :block/page :block/order} - conflicted-structural-entities - (->> tx-data - (keep (fn [item] - (when (and (vector? item) - (>= (count item) 4) - (contains? #{:db/add :db/retract} (first item))) - (let [entity-key (canonical-entity-id db (second item)) - attr (nth item 2)] - (when (and (contains? structural-attrs attr) - (contains? remote-updated-keys [entity-key attr])) - entity-key))))) - set)] - (remove (fn [item] - (and (vector? item) - (let [entity-key (canonical-entity-id db (second item))] - (or - (and (contains? conflicted-structural-entities entity-key) - (contains? #{:db/add :db/retract :db/retractEntity} (first item))) - (and (>= (count item) 4) - (contains? #{:db/add :db/retract} (first item)) - (contains? remote-updated-keys - [entity-key (nth item 2)])))))) - tx-data)) - tx-data)) - -(defn- missing-block-lookup-update? - [db item] - (when (and db - (vector? item) - (>= (count item) 4) - (contains? #{:db/add :db/retract} (first item))) - (let [entity (second item) - attr (nth item 2) - create-attrs #{:block/uuid :block/name :db/ident :block/page :block/parent :block/order}] - (and (vector? entity) - (= :block/uuid (first entity)) - (nil? (d/entity db entity)) - (not (contains? create-attrs attr)))))) - -(defn- drop-missing-block-lookup-updates - [db tx-data] - (if db - (let [stale-lookups (->> tx-data - (keep (fn [item] - (when (missing-block-lookup-update? db item) - (second item)))) - set)] - (if (seq stale-lookups) - (remove (fn [item] - (and (vector? item) - (contains? stale-lookups (second item)))) - tx-data) - tx-data)) - tx-data)) - -(defn- retract-entity-eid - [db item] - (when (and db - (vector? item) - (= :db/retractEntity (first item))) - (let [entity (second item)] - (cond - (number? entity) entity - (vector? entity) (some-> (d/entity db entity) :db/id) - :else nil)))) - -(defn- content-block? - [block] - (and block - (not (ldb/page? block)) - (not (ldb/class? block)) - (not (ldb/property? block)))) - -(def ^:private sync-recycle-meta-attrs - [:logseq.property.recycle/original-parent - :logseq.property.recycle/original-page - :logseq.property.recycle/original-order]) - -(defn- orphaned-blocks->recycle-tx-data - [db blocks] - (->> (outliner-recycle/recycle-blocks-tx-data db blocks {}) - (map (fn [item] - (if (map? item) - (apply dissoc item sync-recycle-meta-attrs) - item))))) - -(defn- move-missing-location-blocks-to-recycle - [db tx-data] - (if db - (let [retracted-eids (->> tx-data - (keep #(retract-entity-eid db %)) - set) - location-fixed-eids (->> tx-data - (keep (fn [item] - (when (and (vector? item) - (= :db/add (first item)) - (contains? #{:block/parent :block/page} (nth item 2 nil))) - (second item)))) - (keep (fn [eid] - (cond - (number? eid) eid - (vector? eid) (some-> (d/entity db eid) :db/id) - :else nil))) - set) - direct-orphans (->> retracted-eids - (mapcat #(ldb/get-children db %)) - (filter content-block?)) - ;; Only recycle top-level page roots whose parent is the page being retracted. - page-orphans (->> retracted-eids - (mapcat (fn [eid] - (->> (ldb/get-page-blocks db eid) - (filter (fn [block] - (= eid (:db/id (:block/parent block)))))))) - (filter content-block?)) - recycle-roots (->> (concat direct-orphans page-orphans) - (remove (fn [block] - (or (contains? retracted-eids (:db/id block)) - (contains? location-fixed-eids (:db/id block))))) - distinct - vec)] - (if (seq recycle-roots) - (concat tx-data - (orphaned-blocks->recycle-tx-data db recycle-roots)) - tx-data)) - tx-data)) - -(defn sanitize-tx-data - [db tx-data] - (let [vector-items (filter vector? tx-data) - other-items (remove vector? tx-data) - sanitized-tx-data (->> (concat - (->> vector-items - (db-normalize/replace-attr-retract-with-retract-entity-v2 db) - ;; Notice: rebase should generate larger tx-id than reverse tx - (map strip-tx-id)) - other-items) - (drop-missing-block-lookup-updates db) - (sanitize-block-ref-datoms db) - (move-missing-location-blocks-to-recycle db) - drop-orphaning-parent-retracts)] - ;; (when (not= tx-data sanitized-tx-data) - ;; (prn :debug :tx-data tx-data) - ;; (prn :debug :sanitized-tx-data sanitized-tx-data)) - sanitized-tx-data)) - -(defn- get-remote-deleted-properties - [{:keys [db-before db-after tx-data]}] - (when (and db-before db-after) - (->> tx-data - (keep (fn [d] - (when-let [e (and (= :db/ident (:a d)) - (false? (:added d)) - (d/entity db-before (:e d)))] - (when (and (ldb/property? e) (nil? (d/entity db-after (:db/ident e)))) - e)))) - - distinct))) + :else + {:applied? false :reason :unsupported-history-action + :debug-data (assoc debug-data :action action)})) + {:applied? false :reason :missing-history-action + :debug-data debug-data}) + (fail-fast :db-sync/missing-db {:repo repo :op :apply-history-action + :debug-data debug-data})))) (defn flush-pending! [repo client] @@ -639,12 +436,7 @@ (mapv (fn [{:keys [tx-id tx outliner-op]}] {:tx-id tx-id :outliner-op outliner-op - :tx-data (->> tx - (db-normalize/remove-retract-entity-ref @conn) - (drop-missing-created-block-datoms @conn) - (sanitize-tx-data @conn) - distinct - vec)})) + :tx-data (vec tx)})) (filterv (comp seq :tx-data))) tx-ids (mapv :tx-id batch)] (if (empty? tx-entries) @@ -677,15 +469,6 @@ (p/catch (fn [error] (js/console.error error)))))))))))))) -(defn- combine-tx-reports - [tx-reports] - (let [tx-reports (vec (keep identity tx-reports))] - (when (seq tx-reports) - {:db-before (:db-before (first tx-reports)) - :db-after (:db-after (last tx-reports)) - :tx-data (mapcat :tx-data tx-reports) - :tx-meta (:tx-meta (last tx-reports))}))) - (defn- remote-tx-debug-meta [temp-tx-meta remote-txs index {:keys [t outliner-op]}] (cond-> (assoc temp-tx-meta @@ -697,424 +480,515 @@ outliner-op (assoc :outliner-op outliner-op))) (defn- local-tx-debug-meta - [temp-tx-meta local-txs index local-tx op] - (cond-> (assoc temp-tx-meta + [tx-meta local-txs index local-tx op] + (cond-> (assoc tx-meta :op op :local-tx-index (inc index) :local-tx-count (count local-txs)) (:tx-id local-tx) (assoc :local-tx-id (:tx-id local-tx)) (:outliner-op local-tx) (assoc :outliner-op (:outliner-op local-tx)))) -(defn- tx-data-item->set-item - [item] - (if (and (vector? item) (= 5 (count item))) - (vec (butlast item)) - item)) - -(defn- rewrite-recreated-lookup-refs - [tx-data] - (let [uuid->tempid (->> tx-data - (keep (fn [item] - (when (and (vector? item) - (= :db/add (first item)) - (>= (count item) 4) - (= :block/uuid (nth item 2))) - [(nth item 3) (second item)]))) - (into {})) - recreated-uuids (->> tx-data - (keep (fn [item] - (when (and (vector? item) - (= :db/retractEntity (first item)) - (vector? (second item)) - (= :block/uuid (first (second item)))) - (second (second item))))) - (filter #(contains? uuid->tempid %)) - set) - rewrite-lookup (fn [x] - (if (and (vector? x) - (= :block/uuid (first x)) - (contains? recreated-uuids (second x))) - (get uuid->tempid (second x) x) - x))] - (mapv (fn [item] - (if (and (vector? item) - (>= (count item) 2) - (contains? #{:db/add :db/retract} (first item))) - (let [entity' (rewrite-lookup (second item)) - has-value? (>= (count item) 4) - attr (nth item 2 nil) - value' (if (and has-value? - (contains? db-schema/ref-type-attributes attr)) - (rewrite-lookup (nth item 3)) - (when has-value? (nth item 3)))] - (cond-> item - (not= (second item) entity') - (assoc 1 entity') - (and has-value? (not= (nth item 3) value')) - (assoc 3 value'))) - item)) - tx-data))) +(defn- reverse-history-action! + [conn local-txs index local-tx temp-tx-meta] + (if-let [tx-data (seq (:reversed-tx local-tx))] + (d/transact! conn + tx-data + (local-tx-debug-meta temp-tx-meta local-txs index local-tx :reverse)) + (invalid-rebase-op! :reverse-history-action + {:reason :missing-reversed-tx-data + :tx-id (:tx-id local-tx) + :outliner-op (:outliner-op local-tx)}))) (defn- transact-remote-txs! - [temp-conn remote-txs temp-tx-meta] + [conn remote-txs temp-tx-meta] (loop [remaining remote-txs index 0 results []] (if-let [remote-tx (first remaining)] (let [tx-data (->> (:tx-data remote-tx) - rewrite-recreated-lookup-refs - (sanitize-tx-data @temp-conn) seq) + report (try + (ldb/transact! conn + tx-data + (remote-tx-debug-meta temp-tx-meta remote-txs index remote-tx)) + (catch :default e + (js/console.error e) + (log/error ::transact-remote-txs! {:remote-tx remote-tx + :index (inc index) + :total (count remote-txs)}) + (throw e))) results' (cond-> results tx-data (conj {:tx-data tx-data - :report (ldb/transact! temp-conn - tx-data - (remote-tx-debug-meta temp-tx-meta remote-txs index remote-tx))}))] + :report report}))] (recur (next remaining) (inc index) results')) results))) -(defn- reverse-replace-retract-uuid-with-retract-entity - [tx-data] - (let [retract-block-ids (->> (keep (fn [[op e a _v _t]] - (when (and (= op :db/retract) - (= :block/uuid a)) - e)) - tx-data) - set) - tx-data' (if (seq retract-block-ids) - (remove (fn [[_op e _a v]] - (or (contains? retract-block-ids e) - (contains? retract-block-ids v))) - tx-data) - tx-data)] - (concat tx-data' - (map (fn [id] [:db/retractEntity id]) retract-block-ids)))) - -(defn- tx-has-missing-lookup-entity? - [db tx-data] - (when (d/db? db) - (some (fn [item] - (when (vector? item) - (let [entity (second item)] - (and (vector? entity) - (= :block/uuid (first entity)) - (nil? (d/entity db entity)))))) - tx-data))) - -(defn- valid-reverse-tx? - [temp-conn tx-data] - (or (and (not (d/db? @temp-conn)) - (every? (fn [item] - (and (vector? item) - (= :db/retractEntity (first item)))) - tx-data)) - (undo-validate/valid-undo-redo-tx? temp-conn tx-data))) - (defn reverse-local-txs! - [temp-conn local-txs temp-tx-meta] - (->> local-txs - reverse - (map-indexed - (fn [index local-tx] - (when-let [tx-data (->> (:reversed-tx local-tx) - remove-ignored-attrs - (replace-string-block-tempids-with-lookups @temp-conn) - (reverse-replace-retract-uuid-with-retract-entity) - seq)] - (when (and (not (tx-has-missing-lookup-entity? @temp-conn tx-data)) - (valid-reverse-tx? temp-conn tx-data)) - (ldb/transact! temp-conn - tx-data - (local-tx-debug-meta temp-tx-meta - local-txs - index - local-tx - :reverse)))))) - (keep identity) - vec)) + [conn local-txs temp-tx-meta] + ;; (prn :debug :local-txs local-txs) + (doall + (->> local-txs + reverse + (map-indexed + (fn [index local-tx] + (try + (reverse-history-action! conn local-txs index local-tx temp-tx-meta) + (catch :default e + (log/error ::reverse-local-tx-error + {:index index + :local-tx local-tx + :local-txs local-txs}) + (throw e))))) + (keep identity) + vec))) + +(defn- invalid-rebase-op! + [op data] + (throw (ex-info "invalid rebase op" (assoc data :op op)))) + +(defn- replay-entity-id-value + [db v] + (cond + (number? v) + v + + (uuid? v) + (some-> (d/entity db [:block/uuid v]) :db/id) + + (or (vector? v) (qualified-keyword? v)) + (some-> (d/entity db v) :db/id) + + :else + v)) + +(defn- stable-entity-ref-like? + [v] + (or (qualified-keyword? v) + (and (vector? v) + (or (= :block/uuid (first v)) + (= :db/ident (first v)))))) + +(defn- replay-property-value + [db property-id v] + (let [property-type (some-> (d/entity db property-id) :logseq.property/type)] + (if (contains? db-property-type/all-ref-property-types property-type) + (cond + (stable-entity-ref-like? v) + (replay-entity-id-value db v) + + (set? v) + (->> v + (map #(if (stable-entity-ref-like? %) + (replay-entity-id-value db %) + %)) + set) + + (sequential? v) + (mapv #(if (stable-entity-ref-like? %) + (replay-entity-id-value db %) + %) + v) + + :else + v) + v))) + +(defn- replay-entity-id-coll + [db ids] + (mapv #(or (replay-entity-id-value db %) %) ids)) + +(defn- precreate-missing-save-blocks! + [conn ops] + (doseq [[op args] ops + :when (= :save-block op)] + (let [[block _opts] args + db @conn + block-uuid (:block/uuid block) + missing-block? (and block-uuid + (nil? (d/entity db [:block/uuid block-uuid]))) + has-structure? (or (:block/page block) + (:block/parent block))] + (when (and missing-block? has-structure?) + (let [target-ref (or (:block/parent block) + (:block/page block)) + target-block (d/entity db target-ref)] + (when-not target-block + (invalid-rebase-op! op {:args args + :reason :missing-target-block})) + (let [now (.now js/Date) + create-block (-> block + (dissoc :db/id) + (assoc :block/created-at now) + (assoc :block/updated-at now))] + (ldb/transact! conn + [create-block] + {:outliner-op :save-block + :persist-op? false}))))))) + +(defn- ^:large-vars/cleanup-todo replay-canonical-outliner-op! + [conn [op args]] + (case op + :save-block + (let [[block opts] args + db @conn + block-uuid (:block/uuid block) + block-ent (when block-uuid + (d/entity db [:block/uuid block-uuid])) + block-base (dissoc block :db/id :block/order) + block' (merge block-base + (op-construct/rewrite-block-title-with-retracted-refs db block-base))] + (if (some? block-ent) + (outliner-core/save-block! conn + block' + (assoc (or opts {}) :persist-op? false)) + (if (and (:block/uuid block') + (or (:block/page block') + (:block/parent block'))) + (let [target-ref (or (:block/parent block') + (:block/page block')) + target-block (d/entity db target-ref)] + (when-not target-block + (invalid-rebase-op! op {:args args + :reason :missing-target-block})) + (let [now (.now js/Date) + create-block (-> block' + (assoc :block/created-at now) + (assoc :block/updated-at now))] + (ldb/transact! conn + [create-block] + {:outliner-op :save-block + :persist-op? false}))) + (invalid-rebase-op! op {:args args + :reason :missing-block})))) + + :insert-blocks + (let [[blocks target-id opts] args + target-block (d/entity @conn target-id) + db @conn] + (when-not (and target-block (seq blocks)) + (invalid-rebase-op! op {:args args})) + (outliner-core/insert-blocks! conn + (mapv #(op-construct/rewrite-block-title-with-retracted-refs db %) blocks) + target-block + (assoc (or opts {}) :persist-op? false))) + + :apply-template + (let [[template-id target-id opts] args + template-id' (replay-entity-id-value @conn template-id) + target-id' (replay-entity-id-value @conn target-id)] + (when-not (and (int? template-id') (int? target-id')) + (invalid-rebase-op! op {:args args + :reason :missing-template-or-target-block})) + (outliner-op/apply-ops! + conn + [[:apply-template [template-id' + target-id' + (assoc (or opts {}) :persist-op? false)]]] + {:persist-op? false + :gen-undo-ops? false})) + + :move-blocks + (let [[ids target-id opts] args + ids' (replay-entity-id-coll @conn ids) + target-id' (or (replay-entity-id-value @conn target-id) target-id) + blocks (keep #(d/entity @conn %) ids')] + (when (empty? blocks) + (invalid-rebase-op! op {:args args})) + (when (seq blocks) + (let [opts' (or opts {}) + sibling? (:sibling? opts') + fallback-target (:fallback-target opts') + fallback-target' (or (replay-entity-id-value @conn fallback-target) + fallback-target) + target-block (d/entity @conn target-id') + use-fallback? (and sibling? + (nil? target-block) + (some? fallback-target)) + target-block' (if use-fallback? + (d/entity @conn fallback-target') + target-block) + move-opts (cond-> (-> opts' + (dissoc :fallback-target) + (assoc :persist-op? false)) + use-fallback? + (assoc :sibling? false))] + (when-not target-block' + (invalid-rebase-op! op {:args args})) + (outliner-core/move-blocks! conn blocks target-block' move-opts)))) + + :move-blocks-up-down + (let [[ids up?] args + ids' (replay-entity-id-coll @conn ids) + blocks (keep #(d/entity @conn %) ids')] + (when (seq blocks) + (outliner-core/move-blocks-up-down! conn blocks up?))) + + :indent-outdent-blocks + (let [[ids indent? opts] args + ids' (replay-entity-id-coll @conn ids) + blocks (keep #(d/entity @conn %) ids')] + (when (empty? blocks) + (invalid-rebase-op! op {:args args})) + (when (seq blocks) + (outliner-core/indent-outdent-blocks! conn blocks indent? opts))) + + :delete-blocks + (let [[ids opts] args + ids' (replay-entity-id-coll @conn ids) + blocks (keep #(d/entity @conn %) ids')] + ;; Keep delete replay idempotent under concurrent edits where blocks may already + ;; be gone, but still leave a debug breadcrumb for malformed/missing targets. + (when (empty? blocks) + (log/debug :db-sync/drop-delete-blocks-replay + {:args args})) + (when (seq blocks) + (outliner-core/delete-blocks! conn blocks (assoc (or opts {}) :persist-op? false)))) + + :create-page + (let [[title opts] args] + (outliner-page/create! conn title (assoc (or opts {}) :persist-op? false))) + + :delete-page + (let [[page-uuid opts] args] + (outliner-page/delete! conn page-uuid (assoc (or opts {}) :persist-op? false))) + + :restore-recycled + (let [[root-id] args + root-ref (cond + (and (vector? root-id) + (= :block/uuid (first root-id))) + root-id + + (uuid? root-id) + [:block/uuid root-id] + + :else + root-id) + root (d/entity @conn root-ref) + tx-data (when root + (seq (outliner-recycle/restore-tx-data @conn root)))] + (when-not tx-data + (invalid-rebase-op! op {:args args + :reason :invalid-restore-target})) + (ldb/transact! conn tx-data + {:outliner-op :restore-recycled + :persist-op? false})) + + :set-block-property + (let [[block-eid property-id v] args + block-eid' (or (replay-entity-id-value @conn block-eid) + block-eid) + block (d/entity @conn block-eid') + property (d/entity @conn property-id) + _ (when-not (and block property) + (invalid-rebase-op! op {:args args + :reason :missing-block-or-property})) + v' (replay-property-value @conn property-id v)] + (when (and (stable-entity-ref-like? v) (nil? v')) + (invalid-rebase-op! op {:args args})) + (outliner-property/set-block-property! conn block-eid' property-id v')) + + :remove-block-property + (apply outliner-property/remove-block-property! conn args) + + :batch-set-property + (let [[block-ids property-id v opts] args + block-ids' (replay-entity-id-coll @conn block-ids) + property (d/entity @conn property-id) + _ (when-not (and property + (seq block-ids') + (every? #(some? (d/entity @conn %)) block-ids')) + (invalid-rebase-op! op {:args args + :reason :missing-block-or-property})) + v' (replay-property-value @conn property-id v)] + (when (and (stable-entity-ref-like? v) (nil? v')) + (invalid-rebase-op! op {:args args})) + (outliner-property/batch-set-property! conn block-ids' property-id v' opts)) + + :batch-remove-property + (let [[block-ids property-id] args + block-ids' (replay-entity-id-coll @conn block-ids)] + (outliner-property/batch-remove-property! conn block-ids' property-id)) + + :delete-property-value + (let [[block-eid property-id property-value] args + block (d/entity @conn block-eid) + property (d/entity @conn property-id) + _ (when-not (and block property) + (invalid-rebase-op! op {:args args + :reason :missing-block-or-property})) + property-value' (replay-property-value @conn property-id property-value)] + (when (and (stable-entity-ref-like? property-value) (nil? property-value')) + (invalid-rebase-op! op {:args args})) + (outliner-property/delete-property-value! conn block-eid property-id property-value')) + + :batch-delete-property-value + (let [[block-eids property-id property-value] args + block-eids' (replay-entity-id-coll @conn block-eids) + property (d/entity @conn property-id) + _ (when-not (and property + (seq block-eids') + (every? #(some? (d/entity @conn %)) block-eids')) + (invalid-rebase-op! op {:args args + :reason :missing-block-or-property})) + property-value' (replay-property-value @conn property-id property-value)] + (when (and (stable-entity-ref-like? property-value) (nil? property-value')) + (invalid-rebase-op! op {:args args})) + (outliner-property/batch-delete-property-value! conn block-eids' property-id property-value')) + + :create-property-text-block + (apply outliner-property/create-property-text-block! conn args) + + :upsert-property + (apply outliner-property/upsert-property! conn args) + + :class-add-property + (apply outliner-property/class-add-property! conn args) + + :class-remove-property + (apply outliner-property/class-remove-property! conn args) + + :upsert-closed-value + (apply outliner-property/upsert-closed-value! conn args) + + :add-existing-values-to-closed-values + (apply outliner-property/add-existing-values-to-closed-values! conn args) + + :delete-closed-value + (apply outliner-property/delete-closed-value! conn args) + + (let [tx-data (:tx args)] + (log/warn ::default-case {:op op + :args args + :tx-data tx-data}) + (when-let [tx-data (seq tx-data)] + (ldb/transact! conn tx-data {:outliner-op :transact}))))) + +(declare handle-local-tx!) + +(defn- rebase-local-op! + [_repo conn local-tx] + (let [outliner-ops (:forward-outliner-ops local-tx)] + (try + (ldb/batch-transact-with-temp-conn! + conn + {:outliner-op :rebase} + (fn [conn] + (if (= [[:transact nil]] outliner-ops) + (when-let [tx-data (seq (:tx local-tx))] + (ldb/transact! conn tx-data {:outliner-op :transact})) + (do + (precreate-missing-save-blocks! conn outliner-ops) + (doseq [op outliner-ops] + (replay-canonical-outliner-op! conn op)))))) + (catch :default error + (let [drop-log {:tx-id (:tx-id local-tx) + :outliner-ops outliner-ops + :error error} + expected-drop? (or (= "invalid rebase op" (ex-message error)) + (string/includes? (or (ex-message error) "") + "doesn't exist yet") + (string/includes? (or (ex-message error) "") + "Nothing found for entity id"))] + (if expected-drop? + (log/debug :db-sync/drop-op-driven-pending-tx drop-log) + (log/warn :db-sync/drop-op-driven-pending-tx drop-log))) + nil)))) (defn- rebase-local-txs! - [temp-conn local-txs remote-db remote-updated-keys remote-tx-data-set temp-tx-meta retracted-properties] - (let [retracted-property-idents (set (map :db/ident retracted-properties))] - (->> local-txs - (map-indexed - (fn [index local-tx] - (let [pending-tx-data (->> (:tx local-tx) - (remove (fn [item] - (and (vector? item) - (contains? #{:db/add :db/retract} (first item)) - (contains? retracted-property-idents (nth item 2 nil))))) - (drop-remote-conflicted-local-tx remote-db remote-updated-keys)) - rebased-tx-data (->> (sanitize-tx-data @temp-conn - pending-tx-data) - (remove remote-tx-data-set))] - (when (seq rebased-tx-data) - (ldb/transact! temp-conn - rebased-tx-data - (local-tx-debug-meta temp-tx-meta - local-txs - index - local-tx - :rebase)))))) - (keep identity) - vec))) - -(defn- build-remote-state - [{:keys [temp-conn remote-txs tx-meta *remote-tx-report]}] - (let [remote-results (transact-remote-txs! temp-conn remote-txs tx-meta) - remote-tx-data (mapcat :tx-data remote-results) - remote-tx-report (combine-tx-reports (map :report remote-results)) - _ (reset! *remote-tx-report remote-tx-report) - retracted-properties (get-remote-deleted-properties remote-tx-report) - remote-db @temp-conn] - {:remote-db remote-db - :remote-results remote-results - :remote-tx-data remote-tx-data - :remote-tx-data-set (set (map tx-data-item->set-item remote-tx-data)) - :remote-tx-report remote-tx-report - :retracted-properties retracted-properties - :remote-updated-keys (remote-updated-attr-keys remote-db remote-tx-data)})) - -(defn- rebase-remote-state! - [{:keys [temp-conn local-txs tx-meta remote-db remote-tx-data-set remote-updated-keys retracted-properties]}] - (let [rebase-tx-reports (rebase-local-txs! temp-conn - local-txs - remote-db - remote-updated-keys - remote-tx-data-set - tx-meta - retracted-properties)] - {:rebase-tx-report (combine-tx-reports rebase-tx-reports) - :rebase-tx-reports rebase-tx-reports})) - -(declare fix-tx!) - -(defn- finalize-remote-state! - [{:keys [temp-conn tx-meta remote-tx-report rebase-tx-report *temp-after-db]}] - (reset! *temp-after-db @temp-conn) - (fix-tx! temp-conn remote-tx-report rebase-tx-report (assoc tx-meta :op :fix))) - -(defn- normalize-rebased-pending-tx - [{:keys [db-before db-after tx-data remote-tx-data-set]}] - (let [normalized (->> tx-data - (normalize-tx-data db-after db-before) - (replace-string-block-tempids-with-lookups db-before)) - normalized-tx-data (->> normalized - (db-normalize/replace-attr-retract-with-retract-entity-v2 db-after) - (remove remote-tx-data-set) - (sanitize-block-ref-datoms db-after))] - {:normalized-tx-data normalized-tx-data - :reversed-datoms (reverse-normalized-tx-data normalized-tx-data)})) + [repo conn local-txs] + (doseq [local-tx local-txs] + (rebase-local-op! repo conn local-tx))) (defn- fix-tx! - [temp-conn remote-tx-report rebase-tx-report tx-meta] - (let [cycle-tx-report (sync-cycle/fix-cycle! temp-conn remote-tx-report rebase-tx-report - {:tx-meta tx-meta})] - (letfn [(page-consistency-candidate-eids [db tx-data] - (let [root-eids (->> tx-data - (keep (fn [[e a _v _tx added]] - (when (and added - (contains? #{:block/parent :block/page} a) - (:block/uuid (d/entity db e))) - e))) - set)] - (into root-eids - (mapcat #(ldb/get-block-full-children-ids db %)) - root-eids))) - (recycle-location-broken-blocks! [conn tx-data tx-meta] - (let [db @conn - location-broken? (fn [block] - (and block - (not (ldb/page? block)) - (not (ldb/class? block)) - (not (ldb/property? block)) - (or (nil? (:block/parent block)) - (nil? (:block/page block))))) - top-level-broken-blocks - (fn [blocks] - (let [broken-ids (set (map :db/id blocks)) - broken-parent? (fn [block] - (loop [parent (:block/parent block) - seen #{}] - (when (and parent - (:db/id parent) - (not (contains? seen (:db/id parent)))) - (if (contains? broken-ids (:db/id parent)) - true - (recur (:block/parent parent) - (conj seen (:db/id parent)))))))] - (remove broken-parent? blocks))) - recycle-blocks (->> (page-consistency-candidate-eids db tx-data) - (keep #(d/entity db %)) - (filter location-broken?) - distinct - top-level-broken-blocks - vec)] - (when (seq recycle-blocks) - (ldb/transact! conn - (vec (orphaned-blocks->recycle-tx-data db recycle-blocks)) - (merge tx-meta {:op :fix-missing-block-location}))))) - (fix-block-page-consistency! [conn tx-data tx-meta] - (let [db @conn - expected-page-for-block - (fn expected-page-for-block [block] - (loop [current (:block/parent block) - seen #{}] - (when (and current - (not (contains? seen (:db/id current)))) - (if (ldb/page? current) - current - (recur (:block/parent current) - (conj seen (:db/id current))))))) - fixes (->> (page-consistency-candidate-eids db tx-data) - (keep (fn [eid] - (let [block (d/entity db eid) - parent (:block/parent block) - current-page (:block/page block) - expected-page (when parent - (expected-page-for-block block))] - (when (and block - (not (ldb/page? block)) - expected-page - (not= (:db/id current-page) - (:db/id expected-page))) - [:db/add eid :block/page (:db/id expected-page)])))) - distinct - vec)] - (when (seq fixes) - (d/transact! conn fixes (merge tx-meta {:op :fix-block-page})))))] - (recycle-location-broken-blocks! temp-conn - (mapcat :tx-data [remote-tx-report - rebase-tx-report - cycle-tx-report]) - tx-meta) - (fix-block-page-consistency! temp-conn - (mapcat :tx-data [remote-tx-report - rebase-tx-report - cycle-tx-report]) - tx-meta)) - (sync-order/fix-duplicate-orders! temp-conn - (mapcat :tx-data [remote-tx-report - rebase-tx-report - cycle-tx-report]) - tx-meta))) + [conn rebase-tx-report tx-meta] + (sync-order/fix-duplicate-orders! conn + (:tx-data rebase-tx-report) + tx-meta)) (defn- apply-remote-tx-with-local-changes! - [{:keys [conn local-txs remote-txs temp-tx-meta *remote-tx-report *reversed-tx-report *rebased-pending-txs *temp-after-db]}] - (let [batch-tx-meta {:rtc-tx? true - :with-local-changes? true}] - (ldb/transact-with-temp-conn! - conn - batch-tx-meta - (fn [temp-conn _*batch-tx-data] - (let [tx-meta temp-tx-meta - reversed-tx-reports (reverse-local-txs! temp-conn local-txs tx-meta) - reversed-tx-report (combine-tx-reports reversed-tx-reports) - _ (reset! *reversed-tx-report reversed-tx-report) - remote-state (build-remote-state {:temp-conn temp-conn - :remote-txs remote-txs - :tx-meta tx-meta - :*remote-tx-report *remote-tx-report}) - rebase-state (rebase-remote-state! (merge remote-state - {:temp-conn temp-conn - :local-txs local-txs - :tx-meta tx-meta}))] - (finalize-remote-state! (merge remote-state - rebase-state - {:temp-conn temp-conn - :tx-meta tx-meta - :*temp-after-db *temp-after-db})))) - {:listen-db (fn [{:keys [tx-meta tx-data db-before db-after]}] - (when-not (contains? #{:reverse :transact-remote-tx-data} (:op tx-meta)) - (swap! *rebased-pending-txs conj {:tx-data tx-data - :tx-meta tx-meta - :db-before db-before - :db-after db-after})))}))) + [{:keys [repo conn local-txs remote-txs]}] + (let [tx-meta {:rtc-tx? true + :with-local-changes? true} + *rebase-tx-reports (atom [])] + (try + (ldb/batch-transact! + conn + tx-meta + (fn [conn] + (reverse-local-txs! conn local-txs {:rtc-tx? true}) + + (transact-remote-txs! conn remote-txs tx-meta) + + (let [rebase-tx-report (rebase-local-txs! repo conn local-txs)] + (fix-tx! conn rebase-tx-report {:outliner-op :rebase}))) + {:listen-db (fn [{:keys [tx-meta tx-data] :as tx-report}] + (when (and (= :rebase (:outliner-op tx-meta)) + (seq tx-data)) + (swap! *rebase-tx-reports conj tx-report)))}) + + (doseq [tx-report @*rebase-tx-reports] + (handle-local-tx! repo tx-report)) + + (remove-pending-txs! repo (map :tx-id local-txs)) + + (catch :default e + (js/console.error e) + (throw e)) + (finally + (reset! *rebase-tx-reports nil) + (worker-undo-redo/clear-history! repo))))) (defn- apply-remote-tx-without-local-changes! [{:keys [conn remote-txs temp-tx-meta]}] - (ldb/transact-with-temp-conn! + (ldb/batch-transact-with-temp-conn! conn {:rtc-tx? true :without-local-changes? true} - (fn [temp-conn] - (let [remote-results (transact-remote-txs! temp-conn remote-txs temp-tx-meta)] - (combine-tx-reports (map :report remote-results)))))) + (fn [conn] + (transact-remote-txs! conn remote-txs temp-tx-meta)))) (defn apply-remote-txs! [repo client remote-txs] (if-let [conn (worker-state/get-datascript-conn repo)] (let [local-txs (pending-txs repo) has-local-changes? (seq local-txs) - *remote-tx-report (atom nil) - *reversed-tx-report (atom nil) - *rebased-pending-txs (atom []) - *temp-after-db (atom nil) remote-tx-data* (mapcat :tx-data remote-txs) temp-tx-meta {:rtc-tx? true :gen-undo-ops? false :persist-op? false} - apply-context {:conn conn + apply-context {:repo repo + :conn conn :local-txs local-txs :remote-txs remote-txs - :temp-tx-meta temp-tx-meta - :*remote-tx-report *remote-tx-report - :*reversed-tx-report *reversed-tx-report - :*rebased-pending-txs *rebased-pending-txs - :*temp-after-db *temp-after-db} - tx-report (try - (if has-local-changes? - (apply-remote-tx-with-local-changes! apply-context) - (apply-remote-tx-without-local-changes! apply-context)) - (catch :default error - (log/error :db-sync/apply-remote-txs-failed - {:repo repo - :has-local-changes? has-local-changes? - :remote-tx-count (count remote-txs) - :local-tx-count (count local-txs) - :remote-txs (mapv (fn [{:keys [t outliner-op tx-data]}] - {:t t - :outliner-op outliner-op - :tx-data-count (count tx-data) - :tx-data-preview (take 12 tx-data)}) - remote-txs) - :local-txs (mapv (fn [{:keys [tx-id outliner-op tx reversed-tx]}] - {:tx-id tx-id - :outliner-op outliner-op - :tx-count (count tx) - :tx-preview (take 12 tx) - :reversed-count (count reversed-tx) - :reversed-preview (take 12 reversed-tx)}) - local-txs) - :error error}) - (throw error))) - remote-tx-report @*remote-tx-report] - (when has-local-changes? - (when-let [rebased-pending-txs (seq @*rebased-pending-txs)] - (let [remote-tx-data-set (set remote-tx-data*) - final-db-after (or @*temp-after-db - (:db-after tx-report))] - (doseq [{:keys [tx-data tx-meta db-before db-after]} rebased-pending-txs] - (let [db-before' (or db-before - (:db-after remote-tx-report) - (:db-after @*reversed-tx-report)) - db-after' (or db-after - final-db-after) - {:keys [normalized-tx-data reversed-datoms]} - (normalize-rebased-pending-tx - {:db-before db-before' - :db-after db-after' - :tx-data tx-data - :remote-tx-data-set remote-tx-data-set})] - (when (seq normalized-tx-data) - (persist-local-tx! repo normalized-tx-data - reversed-datoms - {:outliner-op (or (:outliner-op tx-meta) - :rtc-rebase)})))))) - ;; Once remote txs have been applied and all local txs have been rebased, - ;; the old pending rows are stale regardless of whether any rebased tx remains. - (remove-pending-txs! repo (map :tx-id local-txs))) + :temp-tx-meta temp-tx-meta}] + (try + (if has-local-changes? + (apply-remote-tx-with-local-changes! apply-context) + (apply-remote-tx-without-local-changes! apply-context)) + (catch :default error + (log/error :db-sync/apply-remote-txs-failed + {:repo repo + :has-local-changes? has-local-changes? + :remote-tx-count (count remote-txs) + :local-tx-count (count local-txs) + :remote-txs (mapv (fn [{:keys [t outliner-op tx-data]}] + {:t t + :outliner-op outliner-op + :tx-data-count (count tx-data) + :tx-data-preview (take 12 tx-data)}) + remote-txs) + :local-txs (mapv (fn [{:keys [tx-id outliner-op tx reversed-tx]}] + {:tx-id tx-id + :outliner-op outliner-op + :tx-count (count tx) + :tx-preview (take 12 tx) + :reversed-count (count reversed-tx) + :reversed-preview (take 12 reversed-tx)}) + local-txs) + :error error}) + (throw error))) (when-let [*inflight (:inflight client)] (reset! *inflight [])) @@ -1123,38 +997,47 @@ :graph-id (:graph-id client)}) (p/catch (fn [error] (log/error :db-sync/large-title-rehydrate-failed - {:repo repo :error error})))) - - (reset! *remote-tx-report nil)) + {:repo repo :error error}))))) (fail-fast :db-sync/missing-db {:repo repo :op :apply-remote-txs}))) (defn apply-remote-tx! [repo client tx-data] (apply-remote-txs! repo client [{:tx-data tx-data}])) +(defn- enqueue-local-tx-aux + [repo {:keys [tx-data db-after db-before] :as tx-report}] + (let [normalized (normalize-tx-data db-after db-before tx-data) + reversed-datoms (reverse-tx-data db-before db-after tx-data)] + (when (seq normalized) + (persist-local-tx! repo tx-report normalized reversed-datoms) + (when-let [client @worker-state/*db-sync-client] + (when (= repo (:repo client)) + (let [send-queue (:send-queue client)] + (swap! send-queue + (fn [prev] + (p/then prev + (fn [_] + (when-let [current @worker-state/*db-sync-client] + (when (= repo (:repo current)) + (when-let [ws (:ws current)] + (when (ws-open? ws) + (flush-pending! repo current))))))))))))))) + + +;; (defonce *persist-promise (atom nil)) (defn enqueue-local-tx! - [repo {:keys [tx-meta tx-data db-after db-before]}] - (when-not (or (:rtc-tx? tx-meta) - (:mark-embedding? tx-meta)) - (let [conn (worker-state/get-datascript-conn repo) - db (some-> conn deref)] - (when (and db (seq tx-data)) - (let [normalized (normalize-tx-data db-after db-before tx-data) - reversed-datoms (reverse-tx-data tx-data)] - (when (seq normalized) - (persist-local-tx! repo normalized reversed-datoms tx-meta) - (when-let [client @worker-state/*db-sync-client] - (when (= repo (:repo client)) - (let [send-queue (:send-queue client)] - (swap! send-queue - (fn [prev] - (p/then prev - (fn [_] - (when-let [current @worker-state/*db-sync-client] - (when (= repo (:repo current)) - (when-let [ws (:ws current)] - (when (ws-open? ws) - (flush-pending! repo current)))))))))))))))))) + [repo {:keys [tx-meta tx-data] :as tx-report}] + (when-let [conn (worker-state/get-datascript-conn repo)] + (when-not (or (:rtc-tx? tx-meta) + (and (:batch-tx? @conn) (not= (:outliner-op tx-meta) :rebase)) + (:mark-embedding? tx-meta)) + (when (seq tx-data) + (enqueue-local-tx-aux repo tx-report) + ;; (p/do! + ;; (when-let [p @*persist-promise] + ;; p) + ;; (enqueue-local-tx-aux repo tx-report)) + )))) (defn handle-local-tx! [repo {:keys [tx-data tx-meta db-after] :as tx-report}] diff --git a/src/main/frontend/worker/sync/client_op.cljs b/src/main/frontend/worker/sync/client_op.cljs index 88a9968ace..645646d103 100644 --- a/src/main/frontend/worker/sync/client_op.cljs +++ b/src/main/frontend/worker/sync/client_op.cljs @@ -27,7 +27,7 @@ #(do (log/error ::bad-ops (:value %)) (ma/-fail! ::ops-schema (select-keys % [:value]))))) -(def ^:private asset-op-types #{:update-asset :remove-asset}) +(defonce *repo->pending-local-tx-count (atom {})) (def schema-in-db "TODO: rename this db-name from client-op to client-metadata+op. @@ -41,10 +41,14 @@ :db-sync/checksum {:db/index true} :db-sync/tx-id {:db/unique :db.unique/identity} :db-sync/created-at {:db/index true} + :db-sync/pending? {:db/index true} :db-sync/outliner-op {} - :db-sync/tx-data {} + :db-sync/forward-outliner-ops {} + :db-sync/inverse-outliner-ops {} + :db-sync/inferred-outliner-ops? {} :db-sync/normalized-tx-data {} - :db-sync/reversed-tx-data {}}) + :db-sync/reversed-tx-data {} + :db-sync/asset-op? {:db/index true}}) (defn update-graph-uuid [repo graph-uuid] @@ -101,6 +105,24 @@ ;; (assert (some? r)) r))) +(defn get-pending-local-tx-count + [repo] + (if-let [cached (get @*repo->pending-local-tx-count repo)] + cached + (let [count' (if-let [conn (worker-state/get-client-ops-conn repo)] + (count (d/datoms @conn :avet :db-sync/pending? true)) + 0)] + (swap! *repo->pending-local-tx-count assoc repo count') + count'))) + +(defn adjust-pending-local-tx-count! + [repo delta] + (swap! *repo->pending-local-tx-count + (fn [m] + (let [base (or (get m repo) 0) + next (max 0 (+ base delta))] + (assoc m repo next))))) + (defn get-local-checksum [repo] (let [conn (worker-state/get-client-ops-conn repo)] @@ -135,12 +157,14 @@ (let [remove-asset-op (get exist-block-ops-entity :remove-asset)] (when-not (already-removed? remove-asset-op t) (cond-> [{:block/uuid block-uuid + :db-sync/asset-op? true :update-asset op}] remove-asset-op (conj [:db.fn/retractAttribute e :remove-asset])))) :remove-asset (let [update-asset-op (get exist-block-ops-entity :update-asset)] (when-not (update-after-remove? update-asset-op t) (cond-> [{:block/uuid block-uuid + :db-sync/asset-op? true :remove-asset op}] update-asset-op (conj [:db.fn/retractAttribute e :update-asset]))))))] (ldb/transact! conn tx-data))))))) @@ -149,11 +173,9 @@ [repo] (let [conn (worker-state/get-datascript-conn repo) _ (assert (some? conn)) - asset-block-uuids (d/q '[:find [?block-uuid ...] - :where - [?b :block/uuid ?block-uuid] - [?b :logseq.property.asset/type]] - @conn) + asset-block-uuids (->> (d/datoms @conn :avet :logseq.property.asset/type) + (keep (fn [d] + (:block/uuid (d/entity @conn (:e d)))))) ops (map (fn [block-uuid] [:update-asset 1 {:block-uuid block-uuid}]) asset-block-uuids)] @@ -161,19 +183,10 @@ (defn- get-all-asset-ops* [db] - (->> (d/datoms db :eavt) - (group-by :e) - (keep (fn [[e datoms]] - (let [op-map (into {} - (keep (fn [datom] - (let [a (:a datom)] - (when (or (keyword-identical? :block/uuid a) (contains? asset-op-types a)) - [a (:v datom)])))) - datoms)] - (when (and (:block/uuid op-map) - ;; count>1 = contains some `asset-op-types` - (> (count op-map) 1)) - [e op-map])))) + (->> (d/datoms db :avet :db-sync/asset-op?) + (map (fn [d] + (let [op (d/entity db (:e d))] + [(:e d) (into {} op)]))) (into {}))) (defn get-unpushed-asset-ops-count @@ -191,4 +204,25 @@ (when-let [conn (worker-state/get-client-ops-conn repo)] (let [ent (d/entity @conn [:block/uuid asset-uuid])] (when-let [e (:db/id ent)] - (ldb/transact! conn (map (fn [a] [:db.fn/retractAttribute e a]) asset-op-types)))))) + (ldb/transact! conn [[:db/retractEntity e]]))))) + +(defn cleanup-finished-history-ops! + [repo protected-tx-ids] + (if-let [conn (worker-state/get-client-ops-conn repo)] + (let [protected-tx-ids (set protected-tx-ids) + tx-ent-ids (->> (d/datoms @conn :avet :db-sync/tx-id) + (keep (fn [datom] + (let [tx-id (:v datom) + ent (d/entity @conn (:e datom))] + (when (and (uuid? tx-id) + (false? (:db-sync/pending? ent)) + (not (contains? protected-tx-ids tx-id))) + (:db/id ent))))) + vec)] + (when (seq tx-ent-ids) + (ldb/transact! conn + (mapv (fn [ent-id] + [:db/retractEntity ent-id]) + tx-ent-ids))) + (count tx-ent-ids)) + 0)) diff --git a/src/main/frontend/worker/sync/crypt.cljs b/src/main/frontend/worker/sync/crypt.cljs index f07191aeb4..9afcfd20e6 100644 --- a/src/main/frontend/worker/sync/crypt.cljs +++ b/src/main/frontend/worker/sync/crypt.cljs @@ -39,6 +39,10 @@ [] (worker-state/ ( (opfs/ (crypt/ (p/let [_ (aes-key assoc graph-id aes-key) - aes-key))))) + (letfn [(aes-key assoc graph-id aes-key) + aes-key))))] + (p/let [pair ( ( (p/let [_ (latest-remote-tx} + :latest-remote-tx @sync-apply/*repo->latest-remote-tx + :latest-remote-checksum @sync-apply/*repo->latest-remote-checksum} repo)) (defn- broadcast-rtc-state! @@ -95,7 +98,11 @@ (defn- pending-local-tx? [repo] (when-let [conn (client-ops-conn repo)] - (boolean (first (d/datoms @conn :avet :db-sync/created-at))))) + (boolean + (some (fn [datom] + (let [ent (d/entity @conn (:e datom))] + (not= false (:db-sync/pending? ent)))) + (d/datoms @conn :avet :db-sync/created-at))))) (defn- checksum-compare-ready? [repo client local-t remote-t] @@ -115,8 +122,7 @@ (defn- verify-sync-checksum! [repo client local-tx remote-tx remote-checksum context] - (when (and (not (sync-crypt/graph-e2ee? repo)) - (string? remote-checksum) + (when (and (string? remote-checksum) (checksum-compare-ready? repo client local-tx remote-tx)) (let [local-checksum (local-sync-checksum repo)] (when-not (= local-checksum remote-checksum) @@ -249,6 +255,8 @@ remote-checksum (:checksum message)] (when remote-tx (swap! sync-apply/*repo->latest-remote-tx assoc repo remote-tx)) + (when (contains? message :checksum) + (swap! sync-apply/*repo->latest-remote-checksum assoc repo remote-checksum)) (case (:type message) "hello" (handle-hello! repo client local-tx remote-tx remote-checksum) "online-users" (handle-online-users! repo client message) diff --git a/src/main/frontend/worker/sync/presence.cljs b/src/main/frontend/worker/sync/presence.cljs index 4de234772c..7f6354c236 100644 --- a/src/main/frontend/worker/sync/presence.cljs +++ b/src/main/frontend/worker/sync/presence.cljs @@ -16,17 +16,25 @@ (defn sync-counts [{:keys [get-datascript-conn get-client-ops-conn + get-pending-local-tx-count get-unpushed-asset-ops-count get-local-tx + get-local-checksum get-graph-uuid - latest-remote-tx]} + latest-remote-tx + latest-remote-checksum]} repo] (when (get-datascript-conn repo) - (let [pending-local (when-let [conn (client-ops-conn get-client-ops-conn repo)] - (count (d/datoms @conn :avet :db-sync/created-at))) + (let [pending-local (if get-pending-local-tx-count + (get-pending-local-tx-count repo) + (when-let [conn (client-ops-conn get-client-ops-conn repo)] + (count (d/datoms @conn :avet :db-sync/pending? true)))) pending-asset (get-unpushed-asset-ops-count repo) local-tx (get-local-tx repo) remote-tx (get latest-remote-tx repo) + local-checksum (when get-local-checksum + (get-local-checksum repo)) + remote-checksum (get latest-remote-checksum repo) pending-server (when (and (number? local-tx) (number? remote-tx)) (max 0 (- remote-tx local-tx))) graph-uuid (get-graph-uuid repo)] @@ -35,6 +43,8 @@ :pending-server pending-server :local-tx local-tx :remote-tx remote-tx + :local-checksum local-checksum + :remote-checksum remote-checksum :graph-uuid graph-uuid}))) (defn normalize-online-users @@ -54,7 +64,8 @@ (let [repo (:repo client) ws-state @(:ws-state client) online-users @(:online-users client) - {:keys [pending-local pending-asset pending-server local-tx remote-tx graph-uuid]} + {:keys [pending-local pending-asset pending-server + local-tx remote-tx local-checksum remote-checksum graph-uuid]} (sync-counts-f repo)] {:rtc-state {:ws-state ws-state} :rtc-lock (= :open ws-state) @@ -64,6 +75,8 @@ :pending-server-ops-count (or pending-server 0) :local-tx local-tx :remote-tx remote-tx + :local-checksum local-checksum + :remote-checksum remote-checksum :graph-uuid graph-uuid})) (defn set-ws-state! diff --git a/src/main/frontend/worker/undo_redo.cljs b/src/main/frontend/worker/undo_redo.cljs new file mode 100644 index 0000000000..9d7004754e --- /dev/null +++ b/src/main/frontend/worker/undo_redo.cljs @@ -0,0 +1,367 @@ +(ns frontend.worker.undo-redo + "Undo redo new implementation" + (:require [datascript.core :as d] + [frontend.worker.state :as worker-state] + [lambdaisland.glogi :as log] + [logseq.common.defkeywords :refer [defkeywords]] + [malli.core :as m] + [malli.util :as mu])) + +(defkeywords + ::record-editor-info {:doc "record current editor and cursor"} + ::db-transact {:doc "db tx"} + ::ui-state {:doc "ui state such as route && sidebar blocks"}) + +(defonce *apply-history-action! (atom nil)) + +;; TODO: add other UI states such as `::ui-updates`. +(comment + ;; TODO: convert it to a qualified-keyword + (sr/defkeyword :gen-undo-ops? + "tx-meta option, generate undo ops from tx-data when true (default true)")) + +(def ^:private selection-editor-info-schema + [:map + [:selected-block-uuids [:sequential :uuid]] + [:selection-direction {:optional true} [:maybe [:enum :up :down]]]]) + +(def ^:private editor-cursor-info-schema + [:map + [:block-uuid :uuid] + [:container-id [:or :int [:enum :unknown-container]]] + [:start-pos [:maybe :int]] + [:end-pos [:maybe :int]] + [:selected-block-uuids {:optional true} [:sequential :uuid]] + [:selection-direction {:optional true} [:maybe [:enum :up :down]]]]) + +(def ^:private undo-op-item-schema + (mu/closed-schema + [:multi {:dispatch first} + [::db-transact + [:cat :keyword + [:map + [:tx-meta [:map {:closed false} + [:outliner-op :keyword]]] + [:added-ids [:set :int]] + [:retracted-ids [:set :int]] + [:db-sync/tx-id {:optional true} :uuid] + [:db-sync/forward-outliner-ops {:optional true} [:sequential :any]] + [:db-sync/inverse-outliner-ops {:optional true} [:sequential :any]]]]] + + [::record-editor-info + [:cat :keyword + [:or + editor-cursor-info-schema + selection-editor-info-schema]]] + + [::ui-state + [:cat :keyword :string]]])) + +(def ^:private undo-op-validator (m/validator [:sequential undo-op-item-schema])) + +(defonce max-stack-length 250) +(defonce *undo-ops (atom {})) +(defonce *redo-ops (atom {})) +(defonce *pending-editor-info (atom {})) + +(defn clear-history! + [repo] + (swap! *undo-ops assoc repo []) + (swap! *redo-ops assoc repo []) + (swap! *pending-editor-info dissoc repo)) + +(defn set-pending-editor-info! + [repo editor-info] + (if editor-info + (swap! *pending-editor-info assoc repo editor-info) + (swap! *pending-editor-info dissoc repo))) + +(defn- take-pending-editor-info! + [repo] + (let [editor-info (get @*pending-editor-info repo)] + (swap! *pending-editor-info dissoc repo) + editor-info)) + +(defn- conj-op + [col op] + (let [result (conj (if (empty? col) [] col) op)] + (if (>= (count result) max-stack-length) + (subvec result 0 (/ max-stack-length 2)) + result))) + +(defn- pop-stack + [stack] + (when (seq stack) + [(last stack) (pop stack)])) + +(defn- push-undo-op + [repo op] + (assert (undo-op-validator op) {:op op}) + (swap! *undo-ops update repo conj-op op)) + +(defn- push-redo-op + [repo op] + (assert (undo-op-validator op) {:op op}) + (swap! *redo-ops update repo conj-op op)) + +(defn- pop-undo-op + [repo] + (let [undo-stack (get @*undo-ops repo) + [op undo-stack*] (pop-stack undo-stack)] + (swap! *undo-ops assoc repo undo-stack*) + op)) + +(defn- pop-redo-op + [repo] + (let [redo-stack (get @*redo-ops repo) + [op redo-stack*] (pop-stack redo-stack)] + (swap! *redo-ops assoc repo redo-stack*) + op)) + +(defn- empty-undo-stack? + [repo] + (empty? (get @*undo-ops repo))) + +(defn- empty-redo-stack? + [repo] + (empty? (get @*redo-ops repo))) + +(defn- undo-redo-action-meta + [{:keys [tx-meta] + source-tx-id :db-sync/tx-id} + undo?] + (-> tx-meta + (dissoc :db-sync/tx-id) + (assoc + :gen-undo-ops? false + :persist-op? true + :undo? undo? + :redo? (not undo?) + :db-sync/source-tx-id source-tx-id))) + +(defn- rebind-op-db-sync-tx-id + [op history-tx-id] + (if (uuid? history-tx-id) + (mapv (fn [item] + (if (= ::db-transact (first item)) + [::db-transact (assoc (second item) :db-sync/tx-id history-tx-id)] + item)) + op) + op)) + +(defn- skippable-worker-error? + [error] + (= :invalid-history-action-ops (:reason (ex-data error)))) + +(defn- skippable-worker-result? + [undo? {:keys [reason]}] + (if undo? + (contains? #{:invalid-history-action-ops + :invalid-history-action-tx + :unsupported-history-action} + reason) + (contains? #{:invalid-history-action-ops} + reason))) + +(declare undo-redo-aux) + +(defn- empty-stack-result + [undo?] + (if undo? ::empty-undo-stack ::empty-redo-stack)) + +(defn- push-opposite-op! + [repo undo? op] + (let [sanitize-db-transact + (fn [data] + ;; Keep undo/redo history op-only. Drop any legacy/raw tx payloads. + (dissoc data + :tx + :tx-data + :reversed-tx + :reversed-tx-data + :db-sync/normalized-tx-data + :db-sync/reversed-tx-data)) + op' (mapv (fn [item] + (if (= ::db-transact (first item)) + [::db-transact (sanitize-db-transact (second item))] + item)) + op)] + ((if undo? push-redo-op push-undo-op) repo op'))) + +(defn- undo-redo-result + [repo conn undo? op op'] + (push-opposite-op! repo undo? op') + (let [editor-cursors (->> (filter #(= ::record-editor-info (first %)) op) + (map second)) + cursor (if undo? + (first editor-cursors) + (or (last editor-cursors) (first editor-cursors))) + block-content (when-let [block-uuid (:block-uuid cursor)] + (:block/title (d/entity @conn [:block/uuid block-uuid])))] + {:undo? undo? + :editor-cursors editor-cursors + :block-content block-content})) + +(defn- skip-op-and-recur + [repo undo?] + (undo-redo-aux repo undo?)) + +(defn- run-worker-path + [repo conn undo? op tx-meta' tx-id] + (if-let [apply-action @*apply-history-action!] + (try + (let [worker-result (apply-action repo tx-id undo? tx-meta')] + (cond + (:applied? worker-result) + (undo-redo-result repo conn undo? op + (if undo? + op + (rebind-op-db-sync-tx-id op (:history-tx-id worker-result)))) + + (skippable-worker-result? undo? worker-result) + (skip-op-and-recur repo undo?) + + :else + (do + (log/error ::undo-redo-worker-action-unavailable + {:undo? undo? + :repo repo + :tx-id tx-id + :result worker-result}) + (clear-history! repo) + (empty-stack-result undo?)))) + (catch :default e + (if (skippable-worker-error? e) + (skip-op-and-recur repo undo?) + (do + (log/error ::undo-redo-worker-failed e) + (clear-history! repo) + (throw e) + (empty-stack-result undo?))))) + (do + (log/error ::undo-redo-worker-action-unavailable + {:undo? undo? + :repo repo + :tx-id tx-id + :tx-meta tx-meta' + :reason :missing-apply-history-action}) + (clear-history! repo) + (empty-stack-result undo?)))) + +(defn- process-db-op + [repo conn undo? op] + (when-let [data (some #(when (= ::db-transact (first %)) + (second %)) + op)] + (let [tx-id (:db-sync/tx-id data) + forward-outliner-ops (:db-sync/forward-outliner-ops data) + inverse-outliner-ops (:db-sync/inverse-outliner-ops data) + tx-meta' (-> (undo-redo-action-meta data undo?) + (assoc :forward-outliner-ops forward-outliner-ops + :inverse-outliner-ops inverse-outliner-ops + :db-sync/forward-outliner-ops forward-outliner-ops + :db-sync/inverse-outliner-ops inverse-outliner-ops))] + (run-worker-path repo conn undo? op tx-meta' tx-id)))) + +(defn- undo-redo-aux + [repo undo?] + (if-let [op (not-empty ((if undo? pop-undo-op pop-redo-op) repo))] + (if (= ::ui-state (ffirst op)) + (do + (push-opposite-op! repo undo? op) + {:undo? undo? + :ui-state-str (second (first op))}) + (process-db-op repo (worker-state/get-datascript-conn repo) undo? op)) + (when ((if undo? empty-undo-stack? empty-redo-stack?) repo) + (empty-stack-result undo?)))) + +(defn undo + [repo] + (undo-redo-aux repo true)) + +(defn redo + [repo] + (undo-redo-aux repo false)) + +(defn record-editor-info! + [repo editor-info] + (when editor-info + (swap! *undo-ops + update repo + (fn [stack] + (if (seq stack) + (update stack (dec (count stack)) + (fn [op] + (conj (vec op) [::record-editor-info editor-info]))) + stack))))) + +(defn record-ui-state! + [repo ui-state-str] + (when ui-state-str + (push-undo-op repo [[::ui-state ui-state-str]]))) + +(defn- pending-history-action-ops + [repo tx-id] + (when (uuid? tx-id) + (when-let [conn (get @worker-state/*client-ops-conns repo)] + (when-let [ent (d/entity @conn [:db-sync/tx-id tx-id])] + {:db-sync/forward-outliner-ops (some-> (:db-sync/forward-outliner-ops ent) seq vec) + :db-sync/inverse-outliner-ops (some-> (:db-sync/inverse-outliner-ops ent) seq vec)})))) + +(defn gen-undo-ops! + [repo {:keys [tx-data tx-meta db-after db-before]} tx-id + {:keys [apply-history-action!]}] + (when (nil? @*apply-history-action!) + (reset! *apply-history-action! apply-history-action!)) + (let [{:keys [outliner-op local-tx?]} tx-meta + {:db-sync/keys [forward-outliner-ops inverse-outliner-ops]} (pending-history-action-ops repo tx-id)] + (when (and + (true? local-tx?) + outliner-op + (not (false? (:gen-undo-ops? tx-meta))) + (not (:create-today-journal? tx-meta)) + (seq forward-outliner-ops) + (seq inverse-outliner-ops)) + (let [all-ids (distinct (map :e tx-data)) + retracted-ids (set + (filter + (fn [id] (and (nil? (d/entity db-after id)) (d/entity db-before id))) + all-ids)) + added-ids (set + (filter + (fn [id] (and (nil? (d/entity db-before id)) (d/entity db-after id))) + all-ids)) + editor-info (or (:undo-redo/editor-info tx-meta) + (take-pending-editor-info! repo)) + + data (cond-> {:db-sync/tx-id tx-id + :tx-meta (dissoc tx-meta :outliner-ops) + :added-ids added-ids + :retracted-ids retracted-ids + :db-sync/forward-outliner-ops forward-outliner-ops + :db-sync/inverse-outliner-ops inverse-outliner-ops}) + op (->> [(when editor-info [::record-editor-info editor-info]) + [::db-transact data]] + (remove nil?) + vec)] + ;; A new local action invalidates redo history. + (swap! *redo-ops assoc repo []) + (push-undo-op repo op))))) + +(defn get-debug-state + [repo] + {:undo-ops (get @*undo-ops repo []) + :redo-ops (get @*redo-ops repo []) + :pending-editor-info (get @*pending-editor-info repo)}) + +(defn referenced-history-tx-ids + [repo] + (->> (concat (get @*undo-ops repo []) + (get @*redo-ops repo [])) + (mapcat identity) + (keep (fn [item] + (when (= ::db-transact (first item)) + (let [tx-id (:db-sync/tx-id (second item))] + (when (uuid? tx-id) + tx-id))))) + set)) diff --git a/src/main/logseq/api/editor.cljs b/src/main/logseq/api/editor.cljs index e901dd320b..8e9f958764 100644 --- a/src/main/logseq/api/editor.cljs +++ b/src/main/logseq/api/editor.cljs @@ -188,8 +188,7 @@ [block-uuid-or-page-name] (p/let [repo (state/get-current-repo) block (db-async/> tx-data - (keep (fn [item] - (cond - (vector? item) - (let [e (second item)] - (resolve-entity-id db e)) - - (d/datom? item) - (resolve-entity-id db (:e item)) - - (map? item) - (or (resolve-entity-id db (:db/id item)) - (resolve-entity-id db [:block/uuid (:block/uuid item)])) - - :else nil))) - (remove nil?) - set)) - -(defn- entities-exist? - [db tx-data] - (every? (fn [id] - (when id - (d/entity db id))) - (tx-entity-ids db tx-data))) - -(defn- entity-has-identity? - [ent] - (or (:block/uuid ent) - (:db/ident ent))) - -(defn- recycle-entities-valid? - [db tx-data] - (every? (fn [id] - (when-let [ent (d/entity db id)] - (entity-has-identity? ent))) - (tx-entity-ids db tx-data))) - -(defn- parent-cycle? - [ent] - (let [start (:block/uuid ent)] - (loop [current ent - seen #{start} - steps 0] - (cond - (>= steps 200) true - (nil? (:block/parent current)) false - :else (let [next-ent (:block/parent current) - next-uuid (:block/uuid next-ent)] - (if (contains? seen next-uuid) - true - (recur next-ent (conj seen next-uuid) (inc steps)))))))) - -(defn- issues-for-entity-ids - [db ids] - (let [id->ent (->> ids - (keep (fn [id] - (when-let [ent (d/entity db id)] - (when (:db/id ent) - [id ent])))) - (into {})) - ents (vals id->ent) - structural-ids (->> id->ent - (keep (fn [[id ent]] - (when (or (:block/title ent) - (:block/page ent) - (:block/parent ent) - (:block/order ent)) - id))) - set)] - (concat - (for [e structural-ids - :let [ent (get id->ent e)] - :when (nil? (:block/uuid ent))] - {:type :missing-uuid :e e}) - (for [ent ents - :let [block-uuid (:block/uuid ent) - parent (:block/parent ent)] - :when (and (contains? structural-ids (:db/id ent)) - (not (ldb/page? ent)) - (nil? parent))] - {:type :missing-parent :uuid block-uuid}) - (for [ent ents - :let [block-uuid (:block/uuid ent) - parent (:block/parent ent)] - :when (and (contains? structural-ids (:db/id ent)) - (not (ldb/page? ent)) - parent - (nil? (:block/uuid parent)))] - {:type :missing-parent-ref :uuid block-uuid}) - (for [ent ents - :let [block-uuid (:block/uuid ent) - page (:block/page ent)] - :when (and (contains? structural-ids (:db/id ent)) - (not (ldb/page? ent)) - (nil? page))] - {:type :missing-page :uuid block-uuid}) - (for [ent ents - :let [block-uuid (:block/uuid ent) - page (:block/page ent)] - :when (and (contains? structural-ids (:db/id ent)) - (not (ldb/page? ent)) - page - (not (ldb/page? page)))] - {:type :page-not-page :uuid block-uuid}) - (for [ent ents - :let [block-uuid (:block/uuid ent) - parent (:block/parent ent) - page (:block/page ent) - expected-page (when parent - (if (ldb/page? parent) parent (:block/page parent)))] - :when (and (contains? structural-ids (:db/id ent)) - (not (ldb/page? ent)) - parent - page - expected-page - (not= (:block/uuid expected-page) (:block/uuid page)))] - {:type :page-mismatch :uuid block-uuid}) - (for [ent ents - :let [block-uuid (:block/uuid ent) - parent (:block/parent ent)] - :when (and (contains? structural-ids (:db/id ent)) - parent - (= block-uuid (:block/uuid parent)))] - {:type :self-parent :uuid block-uuid}) - (for [ent ents - :let [block-uuid (:block/uuid ent)] - :when (and (contains? structural-ids (:db/id ent)) - (not (ldb/page? ent)) - (parent-cycle? ent))] - {:type :cycle :uuid block-uuid})))) - -(defn- retract-entity-ids - [db-before tx-data] - (->> tx-data - (keep (fn [item] - (when (and (vector? item) (= :db/retractEntity (first item))) - (second item)))) - (map (fn [id] (d/entid db-before id))) - (filter int?))) - -(defn- affected-entity-ids - [db-before {:keys [tx-data]} original-tx-data] - (let [tx-ids (->> tx-data - (mapcat (fn [[e a v _tx _added]] - (cond-> #{e} - (and (contains? ref-attrs a) (int? v)) (conj v) - (and (contains? ref-attrs a) (vector? v)) - (conj (d/entid db-before v))))) - (remove nil?) - set) - retract-ids (retract-entity-ids db-before original-tx-data) - child-ids (mapcat (fn [id] - (when-let [ent (d/entity db-before id)] - (map :db/id (:block/_parent ent)))) - retract-ids)] - (-> tx-ids - (into retract-ids) - (into child-ids)))) - -(defn- warn-invalid! - [data] - (js/console.warn "undo-redo-invalid" (clj->js data))) - -(defn- log-validate-error! - [error] - (js/console.error "undo-redo-validate-failed" error)) - -(defn valid-undo-redo-tx? - [conn tx-data] - (try - (if (recycle-tx? tx-data) - (if (recycle-entities-valid? @conn tx-data) - true - (do - (warn-invalid! {:reason :invalid-recycle-entities}) - false)) - (if-not (structural-tx? tx-data) - (if (entities-exist? @conn tx-data) - true - (do - (warn-invalid! {:reason :missing-entities}) - false)) - (let [db-before @conn - tx-report (d/with db-before tx-data) - db-after (:db-after tx-report) - affected-ids (affected-entity-ids db-before tx-report tx-data) - baseline-issues (if (seq affected-ids) - (set (issues-for-entity-ids db-before affected-ids)) - #{}) - after-issues (if (seq affected-ids) - (set (issues-for-entity-ids db-after affected-ids)) - #{}) - new-issues (seq (set/difference after-issues baseline-issues))] - (when (seq new-issues) - (warn-invalid! {:issues (take 5 new-issues)})) - (empty? new-issues)))) - (catch :default e - (log-validate-error! e) - false))) diff --git a/src/resources/dicts/en.edn b/src/resources/dicts/en.edn index c84cadc314..15d35619d8 100644 --- a/src/resources/dicts/en.edn +++ b/src/resources/dicts/en.edn @@ -580,6 +580,7 @@ :dev/show-page-data "(Dev) Show page data" :dev/replace-graph-with-db-file "(Dev) Replace graph with its db.sqlite file" :dev/validate-db "(Dev) Validate current graph" + :dev/recompute-checksum "(Dev) Recompute graph checksum" :dev/gc-graph "(Dev) Garbage collect graph (remove unused data in SQLite)" :dev/rtc-stop "(Dev) RTC Stop" :dev/rtc-start "(Dev) RTC Start" diff --git a/src/test/frontend/handler/db_based/sync_test.cljs b/src/test/frontend/handler/db_based/sync_test.cljs index 4569124e4d..93f8fc7375 100644 --- a/src/test/frontend/handler/db_based/sync_test.cljs +++ b/src/test/frontend/handler/db_based/sync_test.cljs @@ -87,12 +87,16 @@ (deftest rtc-create-graph-persists-disabled-e2ee-flag-test (async done (let [fetch-called (atom nil) - tx-called (atom nil)] + tx-called (atom nil) + ensure-calls (atom [])] (-> (p/with-redefs [db-sync/http-base (fn [] "http://base") user-handler/task--ensure-id&access-token (fn [resolve _reject] (resolve true)) db/get-db (fn [] :db) ldb/get-graph-schema-version (fn [_] {:major 65}) + state/ (p/with-redefs [db-sync/http-base (fn [] "http://base") user-handler/task--ensure-id&access-token (fn [resolve _reject] (resolve true)) db/get-db (fn [] :db) ldb/get-graph-schema-version (fn [_] {:major 65}) + state/clj :keywordize-keys true))] (is (= false (:graph-ready-for-use? request-body))) - (is (= [[:thread-api/db-sync-upload-graph "logseq_db_demo"]] + (is (= [[:thread-api/db-sync-ensure-user-rsa-keys + {:ensure-server? true}] + [:thread-api/db-sync-upload-graph "logseq_db_demo"]] @upload-calls)) (is (= 1 @refresh-calls)) (is (= ["logseq_db_demo"] @start-calls)) @@ -309,7 +325,10 @@ (deftest get-remote-graphs-includes-ready-for-use-flag-test (async done - (let [graphs-state (atom nil)] + (let [graphs-state (atom nil) + worker-prev @state/*db-worker + ensure-calls (atom [])] + (reset! state/*db-worker :worker) (-> (p/with-redefs [db-sync/http-base (fn [] "http://base") user-handler/task--ensure-id&access-token (fn [resolve _reject] (resolve true)) @@ -320,7 +339,11 @@ :graph-e2ee? true :graph-ready-for-use? false :created-at 1 - :updated-at 2}]})) + :updated-at 2}] + :user-rsa-keys-exists? true})) + state/ (p/with-redefs [db-sync/http-base (fn [] "http://base") + user-handler/task--ensure-id&access-token (fn [resolve _reject] + (resolve true)) + db-sync/fetch-json (fn [_url _opts _schema] + (p/resolved {:graphs [] + :user-rsa-keys-exists? false})) + state/> (d/q '[:find (pull ?b [*]) - :where - [?b :logseq.property/deleted-at] - [?b :block/title ""]] - @conn) - (map first))] + deleted-blocks (->> (d/q '[:find (pull ?b [*]) + :where + [?b :block/title ""]] + @conn) + (map first))] (is (= ["b1" "b2"] updated-blocks) "Visible page blocks stay on the page") - (is (= 1 (count recycled-blocks)) "Deleted block moves to recycle")))}))) + (is (empty? deleted-blocks) "Deleted block is removed from page db")))}))) (testing "backspace deletes empty block in embedded context" ;; testing embed at this layer doesn't require an embed block since @@ -104,11 +103,10 @@ [(missing? $ ?b :logseq.property/deleted-at)]] @conn) (map (comp :block/title first))) - recycled-blocks (->> (d/q '[:find (pull ?b [*]) - :where - [?b :logseq.property/deleted-at] - [?b :block/title ""]] - @conn) - (map first))] + deleted-blocks (->> (d/q '[:find (pull ?b [*]) + :where + [?b :block/title ""]] + @conn) + (map first))] (is (= ["b1" "b2"] updated-blocks) "Visible page blocks stay on the page") - (is (= 1 (count recycled-blocks)) "Deleted block moves to recycle")))})))) + (is (empty? deleted-blocks) "Deleted block is removed from page db")))})))) diff --git a/src/test/frontend/handler/history_test.cljs b/src/test/frontend/handler/history_test.cljs new file mode 100644 index 0000000000..7000fe828c --- /dev/null +++ b/src/test/frontend/handler/history_test.cljs @@ -0,0 +1,107 @@ +(ns frontend.handler.history-test + (:require [clojure.test :refer [deftest is]] + [frontend.db :as db] + [frontend.handler.editor :as editor] + [frontend.handler.history :as history] + [frontend.state :as state] + [frontend.util :as util] + [logseq.db :as ldb])) + +(deftest restore-cursor-and-state-prefers-ui-state-test + (let [pause-calls (atom []) + app-state-calls (atom []) + cursor-calls (atom [])] + (with-redefs [state/set-state! (fn [k v] + (swap! pause-calls conj [k v])) + ldb/read-transit-str (fn [_] + {:old-state {:route-data {:to :page}} + :new-state {:route-data {:to :home}}}) + history/restore-app-state! (fn [app-state] + (swap! app-state-calls conj app-state)) + history/restore-cursor! (fn [data] + (swap! cursor-calls conj data))] + (#'history/restore-cursor-and-state! + {:ui-state-str "ui-state" + :undo? true + :editor-cursors [{:block-uuid (random-uuid)}]}) + (is (= [[:history/paused? true] + [:history/paused? false]] + @pause-calls)) + (is (= [{:route-data {:to :page}}] + @app-state-calls)) + (is (empty? @cursor-calls))))) + +(deftest restore-cursor-and-state-falls-back-to-cursor-test + (let [pause-calls (atom []) + app-state-calls (atom []) + cursor-calls (atom [])] + (with-redefs [state/set-state! (fn [k v] + (swap! pause-calls conj [k v])) + history/restore-app-state! (fn [app-state] + (swap! app-state-calls conj app-state)) + history/restore-cursor! (fn [data] + (swap! cursor-calls conj data))] + (#'history/restore-cursor-and-state! + {:ui-state-str nil + :undo? false + :editor-cursors [{:block-uuid (random-uuid) + :start-pos 1 + :end-pos 2}]}) + (is (= [[:history/paused? true] + [:history/paused? false]] + @pause-calls)) + (is (empty? @app-state-calls)) + (is (= 1 (count @cursor-calls))) + (is (nil? (:ui-state-str (first @cursor-calls)))) + (is (= false (:undo? (first @cursor-calls))))))) + +(deftest restore-cursor-prefers-block-selection-test + (let [selection-calls (atom []) + edit-calls (atom [])] + (with-redefs [util/get-blocks-by-id (fn [block-id] + (case block-id + #uuid "00000000-0000-0000-0000-000000000001" [:node-1] + #uuid "00000000-0000-0000-0000-000000000002" [:node-2] + nil)) + state/exit-editing-and-set-selected-blocks! (fn [blocks direction] + (swap! selection-calls conj [blocks direction])) + editor/edit-block! (fn [& args] + (swap! edit-calls conj args)) + db/pull (constantly nil)] + (#'history/restore-cursor! + {:undo? true + :editor-cursors [{:selected-block-uuids [#uuid "00000000-0000-0000-0000-000000000001" + #uuid "00000000-0000-0000-0000-000000000002"] + :selection-direction :down}]}) + (is (= [[[:node-1 :node-2] :down]] + @selection-calls)) + (is (empty? @edit-calls))))) + +(deftest restore-cursor-selection-falls-back-to-editor-cursor-test + (let [selection-calls (atom []) + edit-calls (atom []) + block-uuid #uuid "00000000-0000-0000-0000-000000000003"] + (with-redefs [util/get-blocks-by-id (constantly nil) + state/exit-editing-and-set-selected-blocks! (fn [blocks direction] + (swap! selection-calls conj [blocks direction])) + editor/edit-block! (fn [& args] + (swap! edit-calls conj args)) + db/pull (fn [[_lookup-k id]] + (when (= block-uuid id) + {:db/id 42 + :block/uuid block-uuid}))] + (#'history/restore-cursor! + {:undo? false + :editor-cursors [{:selected-block-uuids [#uuid "00000000-0000-0000-0000-000000000001"] + :selection-direction :up + :block-uuid block-uuid + :container-id 99 + :start-pos 1 + :end-pos 3}]}) + (is (empty? @selection-calls)) + (is (= [[{:db/id 42 + :block/uuid block-uuid} + 3 + {:container-id 99 + :custom-content nil}]] + @edit-calls))))) diff --git a/src/test/frontend/handler/user_test.cljs b/src/test/frontend/handler/user_test.cljs new file mode 100644 index 0000000000..02090b6b33 --- /dev/null +++ b/src/test/frontend/handler/user_test.cljs @@ -0,0 +1,70 @@ +(ns frontend.handler.user-test + (:require [cljs.test :refer [deftest is testing]] + [frontend.handler.user :as user-handler] + [frontend.state :as state] + [promesa.core :as p])) + +(defn- with-mocked-local-storage + [f] + (let [old-storage (.-localStorage js/globalThis) + had-local-storage? + (.call (.-hasOwnProperty (.-prototype js/Object)) + js/globalThis + "localStorage") + mocked-storage #js {:clear (fn [] nil) + :setItem (fn [& _] nil) + :getItem (fn [& _] nil) + :removeItem (fn [& _] nil)}] + (js/Object.defineProperty js/globalThis + "localStorage" + #js {:value mocked-storage + :configurable true + :writable true}) + (try + (f) + (finally + (if had-local-storage? + (js/Object.defineProperty js/globalThis + "localStorage" + #js {:value old-storage + :configurable true + :writable true}) + (js/Reflect.deleteProperty js/globalThis "localStorage")))))) + +(deftest logout-clears-e2ee-password-when-db-worker-ready-test + (testing "logout should request db-worker to clear persisted e2ee password" + (let [ops* (atom []) + old-worker @state/*db-worker] + (reset! state/*db-worker :worker) + (try + (with-mocked-local-storage + (fn [] + (with-redefs [state/ tx-report - (assoc-in [:tx-meta :client-id] (:client-id @state/state)) - (update-in [:tx-meta :local-tx?] (fn [local-tx?] - (if (nil? local-tx?) - true - local-tx?)))))) +(deftest clear-history-and-record-editor-info-proxy-test + (let [calls (atom []) + invoke! (fn [& args] + (swap! calls conj (vec args)) + (vec args)) + repo "repo-2" + editor-info {:block-uuid (random-uuid) + :container-id 1 + :start-pos 0 + :end-pos 3}] + (with-redefs [util/node-test? false + state/= steps 200) true - (nil? (:block/parent current)) false - :else (let [next-ent (:block/parent current) - next-uuid (:block/uuid next-ent)] - (if (contains? seen next-uuid) - true - (recur next-ent (conj seen next-uuid) (inc steps)))))))) - -(defn- db-issues - [db] - (let [ignore-ent? (fn [ent] - (or (ldb/recycled? ent) - (= "Recycle" (:block/title ent)) - (= "Recycle" (some-> ent :block/page :block/title)))) - ents (->> (d/q '[:find [?e ...] - :where - [?e :block/uuid]] - db) - (map (fn [e] (d/entity db e))) - (remove ignore-ent?)) - uuid-required-ids (->> (concat - (d/q '[:find [?e ...] - :where - [?e :block/title]] - db) - (d/q '[:find [?e ...] - :where - [?e :block/page]] - db) - (d/q '[:find [?e ...] - :where - [?e :block/parent]] - db)) - distinct)] - (concat - (for [e uuid-required-ids - :let [ent (d/entity db e)] - :when (and (not (ignore-ent? ent)) - (nil? (:block/uuid ent)))] - {:type :missing-uuid :e e}) - (for [ent ents - :let [uuid (:block/uuid ent) - parent (:block/parent ent)] - :when (and (not (ldb/page? ent)) (nil? parent))] - {:type :missing-parent :uuid uuid}) - (for [ent ents - :let [uuid (:block/uuid ent) - parent (:block/parent ent)] - :when (and (not (ldb/page? ent)) parent (nil? (:block/uuid parent)))] - {:type :missing-parent-ref :uuid uuid}) - (for [ent ents - :let [uuid (:block/uuid ent) - page (:block/page ent)] - :when (and (not (ldb/page? ent)) (nil? page))] - {:type :missing-page :uuid uuid}) - (for [ent ents - :let [uuid (:block/uuid ent) - page (:block/page ent)] - :when (and (not (ldb/page? ent)) page (not (ldb/page? page)))] - {:type :page-not-page :uuid uuid}) - (for [ent ents - :let [uuid (:block/uuid ent) - parent (:block/parent ent) - page (:block/page ent) - expected-page (when parent - (if (ldb/page? parent) parent (:block/page parent)))] - :when (and (not (ldb/page? ent)) - parent - page - expected-page - (not= (:block/uuid expected-page) (:block/uuid page)))] - {:type :page-mismatch :uuid uuid}) - (for [ent ents - :let [uuid (:block/uuid ent) - parent (:block/parent ent)] - :when (and parent (= uuid (:block/uuid parent)))] - {:type :self-parent :uuid uuid}) - (for [ent ents - :let [uuid (:block/uuid ent)] - :when (and (not (ldb/page? ent)) - (parent-cycle? ent))] - {:type :cycle :uuid uuid})))) - -(defn- seed-page-parent-child! - [] - (let [conn (db/get-db test-db false) - page-uuid (random-uuid) - parent-uuid (random-uuid) - child-uuid (random-uuid)] - (d/transact! conn - [{:db/ident :logseq.class/Page} - {:block/uuid page-uuid - :block/name "page" - :block/title "page" - :block/tags #{:logseq.class/Page}} - {:block/uuid parent-uuid - :block/title "parent" - :block/page [:block/uuid page-uuid] - :block/parent [:block/uuid page-uuid]} - {:block/uuid child-uuid - :block/title "child" - :block/page [:block/uuid page-uuid] - :block/parent [:block/uuid parent-uuid]}] - {:outliner-op :insert-blocks - :local-tx? false}) - {:page-uuid page-uuid - :parent-uuid parent-uuid - :child-uuid child-uuid})) - -(defn- seed-page-two-parents-child! - [] - (let [conn (db/get-db test-db false) - page-uuid (random-uuid) - parent-a-uuid (random-uuid) - parent-b-uuid (random-uuid) - child-uuid (random-uuid)] - (d/transact! conn - [{:db/ident :logseq.class/Page} - {:block/uuid page-uuid - :block/name "page" - :block/title "page" - :block/tags #{:logseq.class/Page}} - {:block/uuid parent-a-uuid - :block/title "parent-a" - :block/page [:block/uuid page-uuid] - :block/parent [:block/uuid page-uuid]} - {:block/uuid parent-b-uuid - :block/title "parent-b" - :block/page [:block/uuid page-uuid] - :block/parent [:block/uuid page-uuid]} - {:block/uuid child-uuid - :block/title "child" - :block/page [:block/uuid page-uuid] - :block/parent [:block/uuid parent-a-uuid]}] - {:outliner-op :insert-blocks - :local-tx? false}) - {:page-uuid page-uuid - :parent-a-uuid parent-a-uuid - :parent-b-uuid parent-b-uuid - :child-uuid child-uuid})) - -(deftest undo-records-only-local-txs-test - (testing "undo history records only local txs" - (undo-redo/clear-history! test-db) - (let [conn (db/get-db test-db false) - {:keys [child-uuid]} (seed-page-parent-child!)] - (d/transact! conn - [[:db/add [:block/uuid child-uuid] :block/title "local-update"]] - {:outliner-op :save-block - :local-tx? true}) - (let [undo-result (undo-redo/undo test-db)] - (is (not= :frontend.undo-redo/empty-undo-stack undo-result)) - (undo-redo/redo test-db))) - (undo-redo/clear-history! test-db) - (let [conn (db/get-db test-db false) - {:keys [child-uuid]} (seed-page-parent-child!)] - (d/transact! conn - [[:db/add [:block/uuid child-uuid] :block/title "remote-update"]] - {:outliner-op :save-block - :local-tx? false}) - (is (= :frontend.undo-redo/empty-undo-stack (undo-redo/undo test-db)))))) - -(deftest single-op-apply-ops-preserves-local-tx-and-client-id-test - (testing "single local outliner ops should reach listeners with local/client metadata intact" - (let [conn (db/get-db test-db false) - {:keys [child-uuid]} (seed-page-parent-child!) - tx-meta* (atom nil)] - (d/listen! conn ::capture-tx-meta - (fn [{:keys [tx-meta]}] - (reset! tx-meta* tx-meta))) - (try - (outliner-op/apply-ops! conn - [[:save-block [{:block/uuid child-uuid - :block/title "single-op-save"} {}]]] - {:client-id (:client-id @state/state) - :local-tx? true}) - (is (= true (:local-tx? @tx-meta*))) - (is (= (:client-id @state/state) (:client-id @tx-meta*))) - (finally - (d/unlisten! conn ::capture-tx-meta)))))) - -(deftest undo-conflict-clears-history-test - (testing "undo clears history when reverse tx is unsafe" - (undo-redo/clear-history! test-db) - (let [conn (db/get-db test-db false) - block-uuid (random-uuid)] - (d/transact! conn [{:block/uuid block-uuid - :block/title "conflict"}] - {:outliner-op :insert-blocks - :local-tx? true}) - (with-redefs [undo-redo/get-reversed-datoms (fn [& _] nil)] - (is (= :frontend.undo-redo/empty-undo-stack (undo-redo/undo test-db))))))) - -(deftest undo-works-for-local-graph-test - (testing "undo/redo works for local changes on local graph" - (undo-redo/clear-history! test-db) - (let [conn (db/get-db test-db false) - {:keys [child-uuid]} (seed-page-parent-child!)] - (d/transact! conn - [[:db/add [:block/uuid child-uuid] :block/title "local-1"]] - {:outliner-op :save-block - :local-tx? true}) - (let [undo-result (undo-redo/undo test-db)] - (is (not= :frontend.undo-redo/empty-undo-stack undo-result)) - (is (= "child" (:block/title (d/entity @conn [:block/uuid child-uuid]))))) - (let [redo-result (undo-redo/redo test-db)] - (is (not= :frontend.undo-redo/empty-redo-stack redo-result)) - (is (= "local-1" (:block/title (d/entity @conn [:block/uuid child-uuid])))))))) - -(deftest undo-insert-retracts-added-entity-cleanly-test - (testing "undoing a local insert retracts the inserted entity instead of leaving a partial shell" - (undo-redo/clear-history! test-db) - (let [conn (db/get-db test-db false) - {:keys [page-uuid]} (seed-page-parent-child!) - inserted-uuid (random-uuid)] - (d/transact! conn - [{:block/uuid inserted-uuid - :block/title "inserted" - :block/page [:block/uuid page-uuid] - :block/parent [:block/uuid page-uuid]}] - {:outliner-op :insert-blocks - :local-tx? true}) - (is (some? (d/entity @conn [:block/uuid inserted-uuid]))) - (let [undo-result (undo-redo/undo test-db)] - (is (not= :frontend.undo-redo/empty-undo-stack undo-result)) - (is (nil? (d/entity @conn [:block/uuid inserted-uuid]))))))) - -(deftest repeated-save-block-content-undo-redo-test - (testing "multiple saves on the same block undo and redo one step at a time" - (undo-redo/clear-history! test-db) - (let [conn (db/get-db test-db false) - {:keys [child-uuid]} (seed-page-parent-child!)] - (doseq [title ["v1" "v2" "v3"]] - (d/transact! conn - [[:db/add [:block/uuid child-uuid] :block/title title]] - {:outliner-op :save-block - :local-tx? true})) - (is (= "v3" (:block/title (d/entity @conn [:block/uuid child-uuid])))) - (undo-redo/undo test-db) - (is (= "v2" (:block/title (d/entity @conn [:block/uuid child-uuid])))) - (undo-redo/undo test-db) - (is (= "v1" (:block/title (d/entity @conn [:block/uuid child-uuid])))) - (undo-redo/undo test-db) - (is (= "child" (:block/title (d/entity @conn [:block/uuid child-uuid])))) - (undo-redo/redo test-db) - (is (= "v1" (:block/title (d/entity @conn [:block/uuid child-uuid])))) - (undo-redo/redo test-db) - (is (= "v2" (:block/title (d/entity @conn [:block/uuid child-uuid])))) - (undo-redo/redo test-db) - (is (= "v3" (:block/title (d/entity @conn [:block/uuid child-uuid]))))))) - -(deftest repeated-editor-save-block-content-undo-redo-test - (testing "editor/save-block! records sequential content saves in order" - (undo-redo/clear-history! test-db) - (let [conn (db/get-db test-db false) - {:keys [child-uuid]} (seed-page-parent-child!)] - (doseq [title ["foo" "foo bar"]] - (editor/save-block! test-db child-uuid title)) - (is (= "foo bar" (:block/title (d/entity @conn [:block/uuid child-uuid])))) - (undo-redo/undo test-db) - (is (= "foo" (:block/title (d/entity @conn [:block/uuid child-uuid])))) - (undo-redo/redo test-db) - (is (= "foo bar" (:block/title (d/entity @conn [:block/uuid child-uuid]))))))) - -(deftest editor-save-two-blocks-undo-targets-latest-block-test - (testing "undo after saving two different blocks reverts the latest saved block first" - (undo-redo/clear-history! test-db) - (let [conn (db/get-db test-db false) - {:keys [parent-uuid child-uuid]} (seed-page-parent-child!)] - (editor/save-block! test-db parent-uuid "parent updated") - (editor/save-block! test-db child-uuid "child updated") - (undo-redo/undo test-db) - (is (= "parent updated" (:block/title (d/entity @conn [:block/uuid parent-uuid])))) - (is (= "child" (:block/title (d/entity @conn [:block/uuid child-uuid])))) - (undo-redo/undo test-db) - (is (= "parent" (:block/title (d/entity @conn [:block/uuid parent-uuid]))))))) - -(deftest new-local-save-clears-redo-stack-test - (testing "a new local save clears redo history" - (undo-redo/clear-history! test-db) - (let [conn (db/get-db test-db false) - {:keys [child-uuid]} (seed-page-parent-child!)] - (editor/save-block! test-db child-uuid "v1") - (editor/save-block! test-db child-uuid "v2") - (undo-redo/undo test-db) - (is (= "v1" (:block/title (d/entity @conn [:block/uuid child-uuid])))) - (editor/save-block! test-db child-uuid "v3") - (is (= :frontend.undo-redo/empty-redo-stack (undo-redo/redo test-db))) - (is (= "v3" (:block/title (d/entity @conn [:block/uuid child-uuid]))))))) - -(deftest insert-save-delete-sequence-undo-redo-test - (testing "insert then save then recycle-delete can be undone and redone in order" - (undo-redo/clear-history! test-db) - (let [conn (db/get-db test-db false) - {:keys [page-uuid]} (seed-page-parent-child!) - inserted-uuid (random-uuid) - recycle-title "Recycle"] - (d/transact! conn - [{:block/uuid inserted-uuid - :block/title "draft" - :block/page [:block/uuid page-uuid] - :block/parent [:block/uuid page-uuid]}] - {:outliner-op :insert-blocks - :local-tx? true}) - (d/transact! conn - [[:db/add [:block/uuid inserted-uuid] :block/title "published"]] - {:outliner-op :save-block - :local-tx? true}) - (outliner-core/delete-blocks! conn [(d/entity @conn [:block/uuid inserted-uuid])] {}) - (is (= recycle-title - (:block/title (:block/page (d/entity @conn [:block/uuid inserted-uuid]))))) - (undo-redo/undo test-db) - (let [restored (d/entity @conn [:block/uuid inserted-uuid])] - (is (= page-uuid (:block/uuid (:block/page restored)))) - (is (= "published" (:block/title restored)))) - (undo-redo/undo test-db) - (is (= "draft" (:block/title (d/entity @conn [:block/uuid inserted-uuid])))) - (undo-redo/undo test-db) - (is (nil? (d/entity @conn [:block/uuid inserted-uuid]))) - (undo-redo/redo test-db) - (is (= "draft" (:block/title (d/entity @conn [:block/uuid inserted-uuid])))) - (undo-redo/redo test-db) - (is (= "published" (:block/title (d/entity @conn [:block/uuid inserted-uuid])))) - (undo-redo/redo test-db) - (is (= recycle-title - (:block/title (:block/page (d/entity @conn [:block/uuid inserted-uuid])))))))) - -(deftest undo-works-with-remote-updates-test - (testing "undo works after remote updates on sync graphs" - (undo-redo/clear-history! test-db) - (let [conn (db/get-db test-db false) - {:keys [child-uuid]} (seed-page-parent-child!)] - (d/transact! conn - [[:db/add [:block/uuid child-uuid] :block/title "local-2"]] - {:outliner-op :save-block - :local-tx? true}) - (d/transact! conn - [[:db/add [:block/uuid child-uuid] :block/updated-at 12345]] - {:outliner-op :save-block - :local-tx? false}) - (let [undo-result (undo-redo/undo test-db)] - (is (not= :frontend.undo-redo/empty-undo-stack undo-result)) - (is (= "child" (:block/title (d/entity @conn [:block/uuid child-uuid])))))))) - -(deftest undo-redo-works-for-recycle-delete-test - (testing "undo restores a recycled delete and redo sends it back to recycle" - (undo-redo/clear-history! test-db) - (let [conn (db/get-db test-db false) - {:keys [child-uuid page-uuid]} (seed-page-parent-child!) - recycle-page-title "Recycle"] - (outliner-core/delete-blocks! conn [(d/entity @conn [:block/uuid child-uuid])] {}) - (let [deleted-child (d/entity @conn [:block/uuid child-uuid])] - (is (integer? (:logseq.property/deleted-at deleted-child))) - (is (= recycle-page-title (:block/title (:block/page deleted-child))))) - (let [undo-result (undo-redo/undo test-db) - restored-child (d/entity @conn [:block/uuid child-uuid])] - (is (not= :frontend.undo-redo/empty-undo-stack undo-result)) - (is (= page-uuid (:block/uuid (:block/page restored-child)))) - (is (nil? (:logseq.property/deleted-at restored-child)))) - (let [redo-result (undo-redo/redo test-db) - recycled-child (d/entity @conn [:block/uuid child-uuid])] - (is (not= :frontend.undo-redo/empty-redo-stack redo-result)) - (is (= recycle-page-title (:block/title (:block/page recycled-child)))) - (is (integer? (:logseq.property/deleted-at recycled-child))))))) - -(deftest undo-validation-allows-baseline-issues-test - (testing "undo validation allows existing issues without introducing new ones" - (let [conn (db/get-db test-db false) - {:keys [child-uuid]} (seed-page-parent-child!) - orphan-uuid (random-uuid)] - (d/transact! conn - [{:block/uuid orphan-uuid - :block/title "orphan"}] - {:local-tx? false}) - (is (undo-validate/valid-undo-redo-tx? conn - [[:db/add [:block/uuid child-uuid] - :block/title "child-updated"]]))))) - -(deftest undo-validation-rejects-invalid-recycle-restore-tx-test - (testing "recycle-shaped undo tx still validates resulting structure" - (let [conn (db/get-db test-db false) - page-uuid (random-uuid) - block-uuid (random-uuid)] - (d/transact! conn - [{:db/ident :logseq.class/Page} - {:block/uuid page-uuid - :block/name "page" - :block/title "page" - :block/tags #{:logseq.class/Page}} - {:db/id 1000 - :block/uuid block-uuid - :block/title "bad-block" - :block/page [:block/uuid page-uuid] - :block/parent [:block/uuid page-uuid] - :logseq.property/deleted-at 1 - :logseq.property.recycle/original-page [:block/uuid page-uuid] - :logseq.property.recycle/original-parent [:block/uuid page-uuid] - :logseq.property.recycle/original-order "aj"}] - {:local-tx? false}) - ;; Simulate a broken recycled block shell like the runtime repro: entity has - ;; structural attrs but no title/uuid dispatch attrs after sync churn. - (d/transact! conn - [[:db/retract 1000 :block/uuid block-uuid] - [:db/retract 1000 :block/title "bad-block"]] - {:local-tx? false}) - (is (false? (undo-validate/valid-undo-redo-tx? - conn - [[:db/retract 1000 :logseq.property.recycle/original-order "aj"] - [:db/retract 1000 :logseq.property/deleted-at 1] - [:db/add 1000 :block/parent [:block/uuid page-uuid]] - [:db/retract 1000 :logseq.property.recycle/original-page [:block/uuid page-uuid]] - [:db/retract 1000 :logseq.property.recycle/original-parent [:block/uuid page-uuid]] - [:db/add 1000 :block/order "aj"] - [:db/add 1000 :block/page [:block/uuid page-uuid]]])))))) - -(deftest undo-skips-when-parent-missing-test - (testing "undo skips when parent is missing" - (undo-redo/clear-history! test-db) - (let [conn (db/get-db test-db false) - {:keys [parent-uuid child-uuid]} (seed-page-parent-child!)] - (d/transact! conn - [[:db/retractEntity [:block/uuid child-uuid]]] - {:outliner-op :delete-blocks - :local-tx? true}) - (d/transact! conn - [[:db/retractEntity [:block/uuid parent-uuid]]] - {:outliner-op :delete-blocks - :local-tx? false}) - (is (= :frontend.undo-redo/empty-undo-stack (undo-redo/undo test-db))) - (is (nil? (d/entity @conn [:block/uuid child-uuid])))))) - -(deftest undo-skips-when-block-deleted-remote-test - (testing "undo skips when block was deleted remotely" - (undo-redo/clear-history! test-db) - (let [conn (db/get-db test-db false) - {:keys [child-uuid]} (seed-page-parent-child!)] - (d/transact! conn - [[:db/add [:block/uuid child-uuid] :block/title "child-updated"]] - {:outliner-op :save-block - :local-tx? true}) - (d/transact! conn - [[:db/retractEntity [:block/uuid child-uuid]]] - {:outliner-op :delete-blocks - :local-tx? false}) - (is (= :frontend.undo-redo/empty-undo-stack (undo-redo/undo test-db))) - (is (nil? (d/entity @conn [:block/uuid child-uuid])))))) - -(deftest undo-skips-when-undo-would-create-cycle-test - (testing "undo skips when it would create a parent cycle" - (undo-redo/clear-history! test-db) - (let [conn (db/get-db test-db false) - {:keys [page-uuid parent-uuid child-uuid]} (seed-page-parent-child!)] - (d/transact! conn - [[:db/add [:block/uuid child-uuid] :block/parent [:block/uuid page-uuid]]] - {:outliner-op :move-blocks - :local-tx? true}) - (d/transact! conn - [[:db/add [:block/uuid parent-uuid] :block/parent [:block/uuid child-uuid]]] - {:outliner-op :move-blocks - :local-tx? false}) - (is (= :frontend.undo-redo/empty-undo-stack (undo-redo/undo test-db))) - (let [parent (d/entity @conn [:block/uuid parent-uuid]) - child (d/entity @conn [:block/uuid child-uuid])] - (is (= child-uuid (:block/uuid (:block/parent parent)))) - (is (= page-uuid (:block/uuid (:block/parent child)))))))) - -(deftest undo-skips-conflicted-move-and-keeps-earlier-history-test - (testing "undo skips a conflicting move and continues to earlier safe history" - (undo-redo/clear-history! test-db) - (let [conn (db/get-db test-db false) - {:keys [parent-a-uuid parent-b-uuid child-uuid]} (seed-page-two-parents-child!)] - (d/transact! conn - [[:db/add [:block/uuid child-uuid] :block/title "local-title"]] - {:outliner-op :save-block - :local-tx? true}) - (d/transact! conn - [[:db/add [:block/uuid child-uuid] :block/parent [:block/uuid parent-b-uuid]]] - {:outliner-op :move-blocks - :local-tx? true}) - (d/transact! conn - (:tx-data (outliner-core/delete-blocks @conn [(d/entity @conn [:block/uuid parent-a-uuid])] {})) - {:outliner-op :delete-blocks - :local-tx? false}) - (let [undo-result (undo-redo/undo test-db) - child (d/entity @conn [:block/uuid child-uuid])] - (is (map? undo-result)) - (is (= "child" (:block/title child))) - (is (= parent-b-uuid - (:block/uuid (:block/parent child)))) - (is (empty? (db-issues @conn))))))) - -(deftest undo-validation-fast-path-skips-db-issues-for-non-structural-tx-test - (testing "undo validation skips db-issues for non-structural tx-data" - (let [conn (db/get-db test-db false) - {:keys [child-uuid]} (seed-page-parent-child!)] - (with-redefs [undo-validate/issues-for-entity-ids (fn [_ _] - (throw (js/Error. "issues-for-entity-ids called")))] - (is (true? (undo-validate/valid-undo-redo-tx? - conn - [[:db/add [:block/uuid child-uuid] :block/title "child-updated"]]))))))) - -(deftest undo-validation-checks-structural-tx-test - (testing "undo validation evaluates structural changes" - (let [conn (db/get-db test-db false) - {:keys [page-uuid child-uuid]} (seed-page-parent-child!) - calls (atom 0)] - (with-redefs [undo-validate/issues-for-entity-ids (fn [_ _] - (swap! calls inc) - [])] - (is (true? (undo-validate/valid-undo-redo-tx? - conn - [[:db/add [:block/uuid child-uuid] :block/parent [:block/uuid page-uuid]]]))) - (is (pos? @calls)))))) - -(deftest redo-builds-reversed-tx-when-target-parent-is-recycled-test - (testing "redo still builds reversed tx from raw datoms when target parent was recycled remotely" - (undo-redo/clear-history! test-db) - (let [conn (db/get-db test-db false) - {:keys [child-uuid parent-a-uuid parent-b-uuid]} (seed-page-two-parents-child!)] - (d/transact! conn - [[:db/add [:block/uuid child-uuid] :block/parent [:block/uuid parent-b-uuid]]] - {:outliner-op :move-blocks - :local-tx? true}) - (undo-redo/undo test-db) - (d/transact! conn - (:tx-data (outliner-core/delete-blocks @conn [(d/entity @conn [:block/uuid parent-b-uuid])] {})) - {:outliner-op :delete-blocks - :local-tx? false}) - (let [redo-op (last (get @undo-redo/*redo-ops test-db)) - data (some #(when (= ::undo-redo/db-transact (first %)) - (second %)) - redo-op) - reversed (undo-redo/get-reversed-datoms conn false data (:tx-meta data))] - (is (seq reversed)) - (is (= parent-a-uuid - (:block/uuid (:block/parent (d/entity @conn [:block/uuid child-uuid]))))))))) - -(deftest undo-skips-move-when-original-parent-is-recycled-test - (testing "undo should skip a move whose original parent has been recycled" - (undo-redo/clear-history! test-db) - (let [conn (db/get-db test-db false) - {:keys [child-uuid parent-a-uuid parent-b-uuid]} (seed-page-two-parents-child!)] - (d/transact! conn - [[:db/add [:block/uuid child-uuid] :block/parent [:block/uuid parent-b-uuid]]] - {:outliner-op :move-blocks - :local-tx? true}) - (d/transact! conn - (:tx-data (outliner-core/delete-blocks @conn [(d/entity @conn [:block/uuid parent-a-uuid])] {})) - {:outliner-op :delete-blocks - :local-tx? false}) - (let [parent-a (d/entity @conn [:block/uuid parent-a-uuid]) - _ (is (some? parent-a)) - _ (is (true? (ldb/recycled? parent-a))) - undo-op (last (get @undo-redo/*undo-ops test-db)) - data (some #(when (= ::undo-redo/db-transact (first %)) - (second %)) - undo-op) - conflicted? (#'undo-redo/reversed-structural-target-conflicted? - conn - (->> (:tx-data data) reverse (group-by :e)) - true) - reversed (undo-redo/get-reversed-datoms conn true data (:tx-meta data))] - (is (true? conflicted?)) - (is (nil? reversed)))))) - -(deftest ^:long undo-redo-test - (testing "Random mixed operations" - (set! undo-redo/max-stack-length 500) - (let [*random-blocks (atom (outliner-test/get-blocks-ids))] - (outliner-test/transact-random-tree!) - (let [conn (db/get-db test-db false)] - (d/transact! conn - [{:db/ident :logseq.class/Page} - [:db/add [:block/uuid 1] :block/tags :logseq.class/Page]] - {:local-tx? false})) - (let [conn (db/get-db false) - _ (outliner-test/run-random-mixed-ops! *random-blocks)] - - (undo-all!) - (is (empty? (db-issues @conn))) - - (redo-all!) - (is (empty? (db-issues @conn))))))) +(deftest node-test-undo-redo-does-not-call-worker-test + (let [calls (atom []) + invoke! (fn [& args] + (swap! calls conj (vec args)) + (vec args)) + repo "repo-node"] + (with-redefs [util/node-test? true + state/ (in-days next-time) 1))) (let [next-time (get-next-time one-month-ago month-unit 3)] - (is (= 2 (in-months next-time)))) + (is (contains? #{1 2} (in-months next-time)))) (let [next-time (get-next-time one-month-ago month-unit 5)] - (is (= 4 (in-months next-time)))) + (is (contains? #{3 4} (in-months next-time)))) ;; year (let [next-time (get-next-time now year-unit 1)] diff --git a/src/test/frontend/worker/db_listener_test.cljs b/src/test/frontend/worker/db_listener_test.cljs new file mode 100644 index 0000000000..521f58f7d6 --- /dev/null +++ b/src/test/frontend/worker/db_listener_test.cljs @@ -0,0 +1,16 @@ +(ns frontend.worker.db-listener-test + (:require [cljs.test :refer [deftest is testing]] + [frontend.worker.db-listener :as db-listener])) + +(deftest transit-safe-tx-meta-keeps-outliner-ops-test + (testing "worker tx-meta sanitization should preserve semantic outliner ops" + (let [outliner-ops [[:save-block [{:block/uuid (random-uuid) + :block/title "hello"} nil]]] + tx-meta {:outliner-op :save-block + :outliner-ops outliner-ops + :db-sync/inverse-outliner-ops outliner-ops + :error-handler (fn [_] nil)} + safe-tx-meta (#'db-listener/transit-safe-tx-meta tx-meta)] + (is (= outliner-ops (:outliner-ops safe-tx-meta))) + (is (= outliner-ops (:db-sync/inverse-outliner-ops safe-tx-meta))) + (is (nil? (:error-handler safe-tx-meta)))))) diff --git a/src/test/frontend/worker/db_sync_sim_test.cljs b/src/test/frontend/worker/db_sync_sim_test.cljs index 14eb8fbc74..d7901926f3 100644 --- a/src/test/frontend/worker/db_sync_sim_test.cljs +++ b/src/test/frontend/worker/db_sync_sim_test.cljs @@ -6,12 +6,12 @@ [datascript.core :as d] [frontend.db.conn-state :as db-conn-state] [frontend.state :as state] - [frontend.undo-redo :as undo-redo] [frontend.worker.handler.page :as worker-page] [frontend.worker.state :as worker-state] [frontend.worker.sync :as db-sync] [frontend.worker.sync.apply-txs :as sync-apply] [frontend.worker.sync.client-op :as client-op] + [frontend.worker.undo-redo :as undo-redo] [logseq.db :as ldb] [logseq.db-sync.checksum :as sync-checksum] [logseq.db.common.normalize :as db-normalize] @@ -89,7 +89,7 @@ {:repro repro :restore (fn [] (reset! ldb/*transact-invalid-callback prev))})) -(declare op-runs assert-synced-attrs! assert-no-invalid-tx! active-block-uuids block-attr-map checksum-entity-map) +(declare op-runs assert-synced-attrs! assert-no-invalid-tx! active-block-uuids block-attr-map checksum-entity-map run-ops!) (deftest rng-uuid-deterministic-test (testing "rng-uuid produces stable sequences for the same seed" @@ -126,6 +126,7 @@ (let [worker-db-prev @worker-state/*datascript-conns ops-prev @worker-state/*client-ops-conns db-prev @db-conn-state/conns + apply-history-action-prev @undo-redo/*apply-history-action! listeners (atom [])] (reset! worker-state/*datascript-conns (into {} (map (fn [[repo {:keys [conn]}]] [repo conn]) @@ -138,21 +139,20 @@ repo->conns))) (doseq [[repo _] repo->conns] (undo-redo/clear-history! repo)) + (reset! undo-redo/*apply-history-action! sync-apply/apply-history-action!) (doseq [[repo {:keys [conn ops-conn]}] repo->conns] (when ops-conn (let [key (keyword "db-sync-sim" repo)] (d/listen! conn key (fn [tx-report] - (db-sync/enqueue-local-tx! repo tx-report) - (undo-redo/gen-undo-ops! - repo - (-> tx-report - (assoc-in [:tx-meta :client-id] (:client-id @state/state)) - (update-in [:tx-meta :local-tx?] - (fn [local-tx?] - (if (nil? local-tx?) - true - local-tx?))))))) + (let [tx-report' (-> tx-report + (assoc-in [:tx-meta :client-id] (:client-id @state/state)) + (update-in [:tx-meta :local-tx?] + (fn [local-tx?] + (if (nil? local-tx?) + true + local-tx?))))] + (db-sync/enqueue-local-tx! repo tx-report')))) (swap! listeners conj [conn key])))) (try (f) @@ -165,7 +165,8 @@ (doseq [[repo _] repo->conns] (undo-redo/clear-history! repo)) (reset! undo-redo/*undo-ops {}) - (reset! undo-redo/*redo-ops {}))))) + (reset! undo-redo/*redo-ops {}) + (reset! undo-redo/*apply-history-action! apply-history-action-prev))))) (defn- make-client [repo] {:repo repo @@ -263,8 +264,6 @@ (assoc pending-entry :tx-data (->> tx (db-normalize/remove-retract-entity-ref @conn) - (#'sync-apply/drop-missing-created-block-datoms @conn) - (#'sync-apply/sanitize-tx-data @conn) distinct vec)))) (filterv (comp seq :tx-data)))) @@ -313,6 +312,7 @@ (let [ent (d/entity db (:e datom))] (when (and ent (not (ldb/built-in? ent)) + (nil? (:logseq.property/deleted-at ent)) (or (ldb/page? ent) (:block/page ent))) (:v datom))))) @@ -435,8 +435,45 @@ clients [{:repo repo-a :conn conn :client client :online? false}]] (is (nil? (sync-loop! server clients)))))) -(deftest recycled-entities-are-included-in-sim-comparison-test - (testing "deleted blocks remain part of sync comparison" +(deftest two-clients-initial-sync-keeps-shared-base-page-test + (testing "initial sync keeps the shared base page on both clients" + (let [seed (or (env-seed) default-seed) + rng (make-rng seed) + gen-uuid #(rng-uuid rng) + base-uuid (gen-uuid) + conn-a (db-test/create-conn) + conn-b (db-test/create-conn) + ops-a (d/create-conn client-op/schema-in-db) + ops-b (d/create-conn client-op/schema-in-db) + client-a (make-client repo-a) + client-b (make-client repo-b) + server (make-server)] + (with-test-repos {repo-a {:conn conn-a :ops-conn ops-a} + repo-b {:conn conn-b :ops-conn ops-b}} + (fn [] + (reset! db-sync/*repo->latest-remote-tx {}) + (doseq [conn [conn-a conn-b]] + (ensure-base-page! conn base-uuid)) + (doseq [repo [repo-a repo-b]] + (client-op/update-local-tx repo 0)) + (let [base-a (d/entity @conn-a [:block/uuid base-uuid]) + parent-uuid (gen-uuid) + child-uuid (gen-uuid) + target-uuid (gen-uuid) + clients [{:repo repo-a :conn conn-a :client client-a :online? true :gen-uuid gen-uuid} + {:repo repo-b :conn conn-b :client client-b :online? true :gen-uuid gen-uuid}]] + (create-block! conn-a base-a "seed-parent" parent-uuid) + (let [parent (d/entity @conn-a [:block/uuid parent-uuid])] + (create-block! conn-a parent "seed-child" child-uuid)) + (create-block! conn-a base-a "" target-uuid) + (sync-until-idle! server clients 64) + (let [base-b (d/entity @conn-b [:block/uuid base-uuid])] + (is (some? base-b)) + (is (ldb/page? base-b)) + (is (nil? (:logseq.property/deleted-at base-b)))))))))) + +(deftest recycled-entities-are-excluded-from-sim-comparison-test + (testing "deleted blocks are excluded from active sync comparison" (let [base-uuid (random-uuid) block-uuid (random-uuid) conn (db-test/create-conn)] @@ -444,8 +481,8 @@ (let [base-page (d/entity @conn [:block/uuid base-uuid])] (create-block! conn base-page "to recycle" block-uuid) (delete-block! conn block-uuid) - (is (contains? (active-block-uuids @conn) block-uuid)) - (is (contains? (block-attr-map @conn) block-uuid)))))) + (is (not (contains? (active-block-uuids @conn) block-uuid))) + (is (not (contains? (block-attr-map @conn) block-uuid))))))) (deftest uploaded-pending-txs-are-cleared-in-sim-test (testing "sim upload removes acked pending txs so later rebases don't reverse stale creates" @@ -521,6 +558,7 @@ page (:block/page ent)] (when (and ent (not (ldb/built-in? ent)) + (nil? (:logseq.property/deleted-at ent)) (or (ldb/page? ent) page)) [(:block/uuid ent) @@ -551,6 +589,9 @@ (into {}))) (def ^:private sim-default-property-title "Sim Default Property") +(def ^:private sim-default-property-schema + {:logseq.property/type :default + :db/cardinality :db.cardinality/one}) (defn- find-property-by-title [db title] @@ -769,7 +810,7 @@ (defn- op-upsert-property! [_rng conn] (let [title sim-default-property-title - schema {:logseq.property/type :default} + schema sim-default-property-schema existing (find-property-by-title @conn title)] (outliner-op/apply-ops! conn @@ -781,10 +822,20 @@ {:op :upsert-property :property (:db/ident property)}))) +(defn- pick-settable-property-input + [rng conn property value-prefix] + (let [property (d/entity @conn (:db/id property)) + closed-values (vec (:block/_closed-value-property property))] + (if (seq closed-values) + {:value (:db/id (rand-nth! rng closed-values)) + :options {:entity-id? true}} + {:value (str value-prefix "-" (rand-int! rng 1000000)) + :options {}}))) + (defn- op-set-block-property! [rng conn state base-uuid gen-uuid] (when-let [block (ensure-random-block! rng conn state base-uuid gen-uuid)] - (when-let [property (ensure-property! conn sim-default-property-title {:logseq.property/type :default})] - (let [value (str "prop-value-" (rand-int! rng 1000000))] + (when-let [property (ensure-property! conn sim-default-property-title sim-default-property-schema)] + (let [{:keys [value]} (pick-settable-property-input rng conn property "prop-value")] (try (outliner-op/apply-ops! conn @@ -799,27 +850,41 @@ (defn- op-remove-block-property! [rng conn state base-uuid gen-uuid] (when-let [block (ensure-random-block! rng conn state base-uuid gen-uuid)] - (when-let [property (ensure-property! conn sim-default-property-title {:logseq.property/type :default})] - (try - (outliner-op/apply-ops! - conn - [[:set-block-property [(:db/id block) (:db/ident property) (str "remove-prop-" (rand-int! rng 1000000))]] - [:remove-block-property [(:db/id block) (:db/ident property)]]] - {}) - {:op :remove-block-property - :uuid (:block/uuid block) - :property (:db/ident property)} - (catch :default _ - nil))))) + (when-let [property (ensure-property! conn sim-default-property-title sim-default-property-schema)] + (let [{:keys [value]} (pick-settable-property-input rng conn property "remove-prop")] + (try + (outliner-op/apply-ops! + conn + [[:set-block-property [(:db/id block) (:db/ident property) value]]] + {}) + (outliner-op/apply-ops! + conn + [[:remove-block-property [(:db/id block) (:db/ident property)]]] + {}) + {:op :remove-block-property + :uuid (:block/uuid block) + :property (:db/ident property)} + (catch :default _ + nil)))))) + +(defn- create-property-text-block-with-uuid! + [conn property-id value value-uuid] + (outliner-op/apply-ops! + conn + [[:create-property-text-block [nil property-id value {:new-block-id value-uuid}]]] + {}) + (when (d/entity @conn [:block/uuid value-uuid]) + value-uuid)) (defn- op-create-property-text-block! [rng conn] - (when-let [property (ensure-property! conn sim-default-property-title {:logseq.property/type :default})] + (when-let [property (ensure-property! conn sim-default-property-title sim-default-property-schema)] (let [value (str "value-block-" (rand-int! rng 1000000))] (try - (let [value-uuid (outliner-op/apply-ops! + (let [value-uuid (create-property-text-block-with-uuid! conn - [[:create-property-text-block [nil (:db/id property) value {}]]] - {})] + (:db/id property) + value + (rng-uuid rng))] {:op :create-property-text-block :property (:db/ident property) :value-uuid value-uuid}) @@ -827,18 +892,18 @@ nil))))) (defn- op-batch-set-property! [rng conn state base-uuid gen-uuid] - (when-let [property (ensure-property! conn sim-default-property-title {:logseq.property/type :default})] + (when-let [property (ensure-property! conn sim-default-property-title sim-default-property-schema)] (let [blocks (->> (repeatedly 2 #(ensure-random-block! rng conn state base-uuid gen-uuid)) (remove nil?) distinct vec)] (when (seq blocks) (let [block-ids (mapv :db/id blocks) - value (str "batch-prop-" (rand-int! rng 1000000))] + {:keys [value options]} (pick-settable-property-input rng conn property "batch-prop")] (try (outliner-op/apply-ops! conn - [[:batch-set-property [block-ids (:db/ident property) value {}]]] + [[:batch-set-property [block-ids (:db/ident property) value options]]] {}) {:op :batch-set-property :blocks (mapv :block/uuid blocks) @@ -847,28 +912,39 @@ nil))))))) (defn- op-batch-remove-property! [rng conn state base-uuid gen-uuid] - (when-let [property (ensure-property! conn sim-default-property-title {:logseq.property/type :default})] - (let [blocks (->> (repeatedly 2 #(ensure-random-block! rng conn state base-uuid gen-uuid)) - (remove nil?) - distinct - vec)] - (when (seq blocks) - (let [block-ids (mapv :db/id blocks)] - (try - (outliner-op/apply-ops! - conn - [[:batch-set-property [block-ids (:db/ident property) (str "to-remove-" (rand-int! rng 1000000)) {}]] - [:batch-remove-property [block-ids (:db/ident property)]]] - {}) - {:op :batch-remove-property - :blocks (mapv :block/uuid blocks) - :property (:db/ident property)} - (catch :default _ - nil))))))) + (when-let [property (ensure-property! conn sim-default-property-title sim-default-property-schema)] + (let [base-page (d/entity @conn [:block/uuid base-uuid])] + (when base-page + (let [new-block (fn [] + (let [uuid ((or gen-uuid random-uuid)) + title (str "batch-remove-" (rand-int! rng 1000000))] + (create-block! conn base-page title uuid) + (swap! state update :blocks conj uuid) + (d/entity @conn [:block/uuid uuid]))) + blocks (->> (repeatedly 2 new-block) + (remove nil?) + vec)] + (when (seq blocks) + (let [block-ids (mapv :db/id blocks) + {:keys [value options]} (pick-settable-property-input rng conn property "to-remove")] + (try + (outliner-op/apply-ops! + conn + [[:batch-set-property [block-ids (:db/ident property) value options]]] + {}) + (outliner-op/apply-ops! + conn + [[:batch-remove-property [block-ids (:db/ident property)]]] + {}) + {:op :batch-remove-property + :blocks (mapv :block/uuid blocks) + :property (:db/ident property)} + (catch :default _ + nil))))))))) (defn- op-class-add-property! [rng conn] (when-let [class (ensure-class! rng conn)] - (when-let [property (ensure-property! conn sim-default-property-title {:logseq.property/type :default})] + (when-let [property (ensure-property! conn sim-default-property-title sim-default-property-schema)] (try (outliner-op/apply-ops! conn @@ -882,12 +958,15 @@ (defn- op-class-remove-property! [rng conn] (when-let [class (ensure-class! rng conn)] - (when-let [property (ensure-property! conn sim-default-property-title {:logseq.property/type :default})] + (when-let [property (ensure-property! conn sim-default-property-title sim-default-property-schema)] (try (outliner-op/apply-ops! conn - [[:class-add-property [(:db/id class) (:db/ident property)]] - [:class-remove-property [(:db/id class) (:db/ident property)]]] + [[:class-add-property [(:db/id class) (:db/ident property)]]] + {}) + (outliner-op/apply-ops! + conn + [[:class-remove-property [(:db/id class) (:db/ident property)]]] {}) {:op :class-remove-property :class (:block/uuid class) @@ -896,7 +975,7 @@ nil))))) (defn- op-upsert-closed-value! [rng conn] - (when-let [property (ensure-property! conn sim-default-property-title {:logseq.property/type :default})] + (when-let [property (ensure-property! conn sim-default-property-title sim-default-property-schema)] (let [value (str "choice-" (rand-int! rng 1000000))] (try (outliner-op/apply-ops! @@ -910,7 +989,7 @@ nil))))) (defn- op-delete-closed-value! [rng conn] - (when-let [property (ensure-property! conn sim-default-property-title {:logseq.property/type :default})] + (when-let [property (ensure-property! conn sim-default-property-title sim-default-property-schema)] (let [value (str "delete-choice-" (rand-int! rng 1000000))] (try (outliner-op/apply-ops! @@ -929,18 +1008,20 @@ nil))))) (defn- op-add-existing-values-to-closed-values! [rng conn] - (when-let [property (ensure-property! conn sim-default-property-title {:logseq.property/type :default})] + (when-let [property (ensure-property! conn sim-default-property-title sim-default-property-schema)] (try (let [value-a (str "existing-a-" (rand-int! rng 1000000)) value-b (str "existing-b-" (rand-int! rng 1000000)) - uuid-a (outliner-op/apply-ops! + uuid-a (create-property-text-block-with-uuid! conn - [[:create-property-text-block [nil (:db/id property) value-a {}]]] - {}) - uuid-b (outliner-op/apply-ops! + (:db/id property) + value-a + (rng-uuid rng)) + uuid-b (create-property-text-block-with-uuid! conn - [[:create-property-text-block [nil (:db/id property) value-b {}]]] - {}) + (:db/id property) + value-b + (rng-uuid rng)) uuids (vec (remove nil? [uuid-a uuid-b]))] (when (seq uuids) (outliner-op/apply-ops! @@ -959,8 +1040,11 @@ (try (outliner-op/apply-ops! conn - [[:set-block-property [(:db/id block) :block/tags (:db/id class)]] - [:delete-property-value [(:db/id block) :block/tags (:db/id class)]]] + [[:set-block-property [(:db/id block) :block/tags (:db/id class)]]] + {}) + (outliner-op/apply-ops! + conn + [[:delete-property-value [(:db/id block) :block/tags (:db/id class)]]] {}) {:op :delete-property-value :uuid (:block/uuid block) @@ -979,8 +1063,11 @@ (try (outliner-op/apply-ops! conn - [[:batch-set-property [block-ids :block/tags (:db/id class) {}]] - [:batch-delete-property-value [block-ids :block/tags (:db/id class)]]] + [[:batch-set-property [block-ids :block/tags (:db/id class) {}]]] + {}) + (outliner-op/apply-ops! + conn + [[:batch-delete-property-value [block-ids :block/tags (:db/id class)]]] {}) {:op :batch-delete-property-value :blocks (mapv :block/uuid blocks) @@ -1035,13 +1122,13 @@ (defn- op-undo! [_rng repo] (when repo (let [result (undo-redo/undo repo)] - (when (not= :frontend.undo-redo/empty-undo-stack result) + (when (not= :frontend.worker.undo-redo/empty-undo-stack result) {:op :undo})))) (defn- op-redo! [_rng repo] (when repo (let [result (undo-redo/redo repo)] - (when (not= :frontend.undo-redo/empty-redo-stack result) + (when (not= :frontend.worker.undo-redo/empty-redo-stack result) {:op :redo})))) (def ^:private op-table @@ -1088,38 +1175,205 @@ (is (contains? registered :undo)) (is (contains? registered :redo))))) +(def ^:private required-core-outliner-op-names + #{:save-block + :insert-blocks + :delete-blocks + :move-blocks + :move-blocks-up-down + :indent-outdent-blocks + :upsert-property + :set-block-property + :remove-block-property + :delete-property-value + :create-property-text-block + :batch-set-property + :batch-remove-property + :batch-delete-property-value + :class-add-property + :class-remove-property + :upsert-closed-value + :delete-closed-value + :add-existing-values-to-closed-values + :create-page + :rename-page + :delete-page + :toggle-reaction + :transact}) + (deftest core-outliner-ops-registered-in-sim-op-table-test (testing "sim op-table includes core logseq.outliner.op operations" (let [registered (set (map :name op-table)) - required #{:save-block - :insert-blocks - :delete-blocks - :move-blocks - :move-blocks-up-down - :indent-outdent-blocks - :upsert-property - :set-block-property - :remove-block-property - :delete-property-value - :create-property-text-block - :batch-set-property - :batch-remove-property - :batch-delete-property-value - :class-add-property - :class-remove-property - :upsert-closed-value - :delete-closed-value - :add-existing-values-to-closed-values - :create-page - :rename-page - :delete-page - :toggle-reaction - :transact}] + required required-core-outliner-op-names] (is (empty? (set/difference required registered)) (str "missing ops: " (set/difference required registered)))))) -(defn- pick-op [rng {:keys [disable-ops enable-ops]}] - (let [op-table' (cond->> op-table +(def ^:private local-undo-redo-run-count 1000) +(def ^:private local-undo-redo-full-cycle-runs 5) +(def ^:private local-undo-redo-coverage-ops + (set/union required-core-outliner-op-names #{:undo :redo})) +(def ^:private local-undo-redo-op-weights + {:create-page 6 + :rename-page 2 + :delete-page 10 + :save-block 4 + :upsert-property 2 + :set-block-property 3 + :remove-block-property 2 + :delete-property-value 1 + :create-property-text-block 2 + :batch-set-property 2 + :batch-remove-property 2 + :batch-delete-property-value 1 + :class-add-property 1 + :class-remove-property 1 + :upsert-closed-value 1 + :delete-closed-value 1 + :add-existing-values-to-closed-values 1 + :insert-blocks 10 + :delete-blocks 4 + :move-blocks 6 + :move-blocks-up-down 3 + :indent-outdent-blocks 10 + :toggle-reaction 2 + :transact 3 + :undo 10 + :redo 10}) + +(def ^:private local-undo-redo-cycle-op-weights + {:create-block 14 + :delete-block 10 + :move-block 8 + :indent-outdent-blocks 3 + :move-blocks-up-down 3 + :update-title 8 + :undo 12 + :redo 12}) + +(defn- build-weighted-op-table + [required-ops op-weights label] + (let [label (name label) + registered-ops (set (map :name op-table)) + configured-ops (set (keys op-weights)) + missing-op-defs (set/difference required-ops registered-ops) + missing-weights (set/difference required-ops configured-ops) + extra-weights (set/difference configured-ops required-ops) + invalid-weights (->> op-weights + (keep (fn [[op-name weight]] + (when (or (not (number? weight)) + (<= weight 0)) + op-name))) + set)] + (when (seq missing-op-defs) + (throw (ex-info (str "missing sim op definitions for weighted " label " op table") + {:label label + :missing-op-defs missing-op-defs}))) + (when (seq missing-weights) + (throw (ex-info (str "missing weighted " label " op weights") + {:label label + :missing-weights missing-weights}))) + (when (seq extra-weights) + (throw (ex-info (str "unexpected weighted " label " op weights") + {:label label + :extra-weights extra-weights}))) + (when (seq invalid-weights) + (throw (ex-info (str "invalid weighted " label " op weights") + {:label label + :invalid-weights invalid-weights}))) + (->> op-table + (filter (fn [item] (contains? required-ops (:name item)))) + (mapv (fn [item] + (assoc item :weight (get op-weights (:name item)))))))) + +(defn- op-count + [history op] + (count (filter #(= op (:op %)) @history))) + +(defn- prime-op-context! + [rng client history op & {:keys [op-table-override]}] + (let [setup-run! (fn [setup-op & {:keys [times] :or {times 1}}] + (dotimes [_ times] + (run-ops! rng + client + 1 + history + {:pick-op-opts {:enable-ops #{setup-op}} + :op-table-override op-table-override + :context {:phase :prime + :target op + :setup-op setup-op}})))] + (case op + (:delete-page :rename-page) + (setup-run! :create-page) + + (:save-block + :delete-blocks + :move-blocks + :toggle-reaction + :transact) + (setup-run! :insert-blocks :times 2) + + (:move-blocks-up-down :indent-outdent-blocks) + (setup-run! :insert-blocks :times 4) + + (:set-block-property + :remove-block-property + :delete-property-value + :create-property-text-block + :batch-set-property + :batch-remove-property + :batch-delete-property-value) + (do + (setup-run! :insert-blocks :times 2) + (setup-run! :upsert-property)) + + (:class-add-property :class-remove-property) + (do + (setup-run! :insert-blocks) + (setup-run! :upsert-property)) + + (:upsert-closed-value :delete-closed-value) + (setup-run! :upsert-property) + + :add-existing-values-to-closed-values + (do + (setup-run! :upsert-property) + (setup-run! :create-property-text-block :times 2)) + + :undo + (setup-run! :insert-blocks :times 2) + + :redo + (do + (setup-run! :insert-blocks :times 2) + (setup-run! :undo)) + + nil))) + +(defn- ensure-op-recorded! + [rng client history op max-attempts & {:keys [op-table-override]}] + (loop [attempt 0] + (let [before (op-count history op)] + (prime-op-context! rng client history op :op-table-override op-table-override) + (run-ops! rng + client + 1 + history + {:pick-op-opts {:enable-ops #{op}} + :op-table-override op-table-override + :context {:phase :ensure-op + :target op + :attempt attempt}}) + (let [after (op-count history op)] + (if (> after before) + true + (if (< attempt max-attempts) + (recur (inc attempt)) + false)))))) + +(defn- pick-op [rng {:keys [disable-ops enable-ops op-table-override]}] + (let [selected-op-table (or op-table-override op-table) + op-table' (cond->> selected-op-table (seq enable-ops) (filter (fn [item] (contains? enable-ops (:name item)))) @@ -1141,9 +1395,10 @@ op (recur (- remaining weight) rest-ops)))))))) -(defn- run-ops! [rng {:keys [repo conn base-uuid state gen-uuid]} steps history & {:keys [pick-op-opts context]}] +(defn- run-ops! + [rng {:keys [repo conn base-uuid state gen-uuid]} steps history & {:keys [pick-op-opts context op-table-override]}] (dotimes [step steps] - (let [{:keys [f name]} (pick-op rng pick-op-opts) + (let [{:keys [f name]} (pick-op rng (assoc (or pick-op-opts {}) :op-table-override op-table-override)) ;; _ (prn :debug :client (:repo client) :name name) result (case name :create-page (f rng conn state {:gen-uuid gen-uuid}) @@ -1404,6 +1659,82 @@ (sync-loop! server [{:repo repo-a :conn conn-a :client client-a :online? true}]) (is (= "test" (:block/title (d/entity @conn-a [:block/uuid block-uuid]))))))))) +(deftest undo-redo-indent-sequence-does-not-produce-invalid-entity-test + (testing "undo/redo of add-1 add-2 indent-2 should remain valid after another undo" + (let [seed 20260321 + base-uuid (uuid "61111111-1111-1111-1111-111111111111") + block-1-uuid (uuid "62222222-2222-2222-2222-222222222222") + block-2-uuid (uuid "63333333-3333-3333-3333-333333333333") + conn-a (db-test/create-conn-with-blocks + {:pages-and-blocks [{:page {:block/title base-page-title + :block/uuid base-uuid} + :blocks []}]}) + ops-a (d/create-conn client-op/schema-in-db) + history (atom [])] + (with-test-repos {repo-a {:conn conn-a :ops-conn ops-a}} + (fn [] + (let [{:keys [repro restore]} (install-invalid-tx-repro! seed history)] + (try + (reset! db-sync/*repo->latest-remote-tx {}) + (client-op/update-local-tx repo-a 0) + (let [base-page (d/entity @conn-a [:block/uuid base-uuid]) + tx-meta {:client-id "db-sync-sim-client" + :local-tx? true}] + (outliner-op/apply-ops! conn-a + [[:insert-blocks [[{:block/uuid block-1-uuid + :block/title ""}] + (:db/id base-page) + {:sibling? false + :keep-uuid? true}]]] + tx-meta) + (outliner-op/apply-ops! conn-a + [[:save-block [{:block/uuid block-1-uuid + :block/title "1"} + nil]]] + tx-meta) + (let [block-1 (d/entity @conn-a [:block/uuid block-1-uuid])] + (outliner-op/apply-ops! conn-a + [[:insert-blocks [[{:block/uuid block-2-uuid + :block/title ""}] + (:db/id block-1) + {:sibling? true + :keep-uuid? true}]]] + tx-meta)) + (outliner-op/apply-ops! conn-a + [[:save-block [{:block/uuid block-2-uuid + :block/title "2"} + nil]]] + tx-meta) + (let [block-2 (d/entity @conn-a [:block/uuid block-2-uuid])] + (outliner-op/apply-ops! conn-a + [[:indent-outdent-blocks [[(:db/id block-2)] true {}]]] + tx-meta)) + (loop [undo-count 0] + (if (= :frontend.worker.undo-redo/empty-undo-stack + (undo-redo/undo repo-a)) + (do + (is (pos? undo-count)) + (loop [redo-count 0] + (if (= :frontend.worker.undo-redo/empty-redo-stack + (undo-redo/redo repo-a)) + (is (= undo-count redo-count)) + (recur (inc redo-count))))) + (recur (inc undo-count)))) + (let [block-2-after-redo (d/entity @conn-a [:block/uuid block-2-uuid])] + (is (some? block-2-after-redo)) + (is (= block-1-uuid + (-> block-2-after-redo :block/parent :block/uuid)))) + (is (not= :frontend.worker.undo-redo/empty-undo-stack + (undo-redo/undo repo-a))) + (let [block-2 (d/entity @conn-a [:block/uuid block-2-uuid])] + (is (some? block-2)) + (is (= base-uuid (-> block-2 :block/page :block/uuid))) + (is (= base-uuid (-> block-2 :block/parent :block/uuid)))) + (is (nil? @repro) + (str "unexpected invalid tx payload: " (pr-str @repro)))) + (finally + (restore))))))))) + (deftest ^:long two-clients-undo-skips-conflicted-move-but-keeps-db-valid-test (testing "undo skips a conflicted move while syncing the remaining safe history" (let [base-uuid (uuid "31111111-1111-1111-1111-111111111111") @@ -1450,7 +1781,7 @@ {:repo repo-b :conn conn-b :client client-b :online? true}] 50) - (is (not= :frontend.undo-redo/empty-undo-stack + (is (not= :frontend.worker.undo-redo/empty-undo-stack (undo-redo/undo repo-a))) (let [rounds (sync-until-idle! server [{:repo repo-a :conn conn-a :client client-a :online? true} @@ -1600,7 +1931,7 @@ {:repo repo-b :conn conn-b :client client-b :online? true}]) (is (some? (d/entity @conn-a [:block/uuid block-uuid]))) (is (some? (d/entity @conn-b [:block/uuid block-uuid]))) - (is (not= :frontend.undo-redo/empty-undo-stack + (is (not= :frontend.worker.undo-redo/empty-undo-stack (undo-redo/undo repo-a))) (let [pending (#'sync-apply/pending-txs repo-a) retract-block? (fn [item] @@ -1618,6 +1949,133 @@ (defonce op-runs 200) +(defn- undo-all! + [repo max-steps] + (loop [steps 0] + (when (>= steps max-steps) + (throw (ex-info "undo-all exceeded max steps" + {:repo repo + :max-steps max-steps}))) + (let [result (undo-redo/undo repo)] + (if (= :frontend.worker.undo-redo/empty-undo-stack result) + steps + (recur (inc steps)))))) + +(defn- redo-all! + [repo max-steps] + (loop [steps 0] + (when (>= steps max-steps) + (throw (ex-info "redo-all exceeded max steps" + {:repo repo + :max-steps max-steps}))) + (let [result (undo-redo/redo repo)] + (if (= :frontend.worker.undo-redo/empty-redo-stack result) + steps + (recur (inc steps)))))) + +(deftest ^:long ^:large-vars/cleanup-todo all-core-outliner-ops-local-undo-redo-random-sim-test + (testing "local randomized stress simulation runs weighted ops and keeps undo-all/redo-all roundtrips valid" + (let [seed (or (env-seed) default-seed) + rng (make-rng seed) + gen-uuid #(rng-uuid rng) + coverage-ops local-undo-redo-coverage-ops + coverage-op-table (build-weighted-op-table coverage-ops local-undo-redo-op-weights :coverage) + cycle-ops (set (keys local-undo-redo-cycle-op-weights)) + cycle-op-table (build-weighted-op-table cycle-ops local-undo-redo-cycle-op-weights :cycle) + base-uuid (gen-uuid) + conn (db-test/create-conn) + ops-conn (d/create-conn client-op/schema-in-db) + history (atom []) + state (atom {:pages #{base-uuid} :blocks #{}}) + client-context {:repo repo-a + :conn conn + :base-uuid base-uuid + :state state + :gen-uuid gen-uuid}] + (with-test-repos {repo-a {:conn conn :ops-conn ops-conn}} + (fn [] + (let [{:keys [repro restore]} (install-invalid-tx-repro! seed history)] + (try + (reset! db-sync/*repo->latest-remote-tx {}) + (record-meta! history {:seed seed + :base-uuid base-uuid + :phase :local-undo-redo-stress + :run-count local-undo-redo-run-count + :full-cycle-runs local-undo-redo-full-cycle-runs}) + (ensure-base-page! conn base-uuid) + (client-op/update-local-tx repo-a 0) + + ;; Random warmup to provide realistic, non-trivial local history. + (run-ops! rng client-context 50 history + {:pick-op-opts {:enable-ops coverage-ops + :disable-ops #{:undo :redo}} + :op-table-override coverage-op-table + :context {:phase :warmup}}) + + ;; Guarantee every required op is actually exercised at least once. + (doseq [op (sort coverage-ops)] + (let [executed? (or (pos? (op-count history op)) + (ensure-op-recorded! rng + client-context + history + op + 120 + :op-table-override coverage-op-table))] + (is executed? + (str "failed to execute op=" op " seed=" seed)))) + + ;; Keep all-core coverage checks separate from full undo-all/redo-all cycles. + (let [issues (db-issues @conn)] + (is (empty? issues) + (str "db issues before cycle stress seed=" seed " " (pr-str issues)))) + (assert-no-invalid-tx! seed history repro) + (undo-redo/clear-history! repo-a) + + ;; Long weighted random stress run on undo-safe operation families. + (run-ops! rng client-context local-undo-redo-run-count history + {:pick-op-opts {:enable-ops cycle-ops} + :op-table-override cycle-op-table + :context {:phase :cycle-stress}}) + + ;; Ensure at least one concrete undoable change before undo-all cycles. + (is (ensure-op-recorded! rng + client-context + history + :create-block + 120 + :op-table-override cycle-op-table) + (str "failed to prepare undo stack seed=" seed)) + + (let [max-stack-steps (+ (* 2 local-undo-redo-run-count) 5000)] + (dotimes [cycle-idx local-undo-redo-full-cycle-runs] + (let [undo-steps (undo-all! repo-a max-stack-steps) + issues-after-undo (db-issues @conn)] + (is (pos? undo-steps) + (str "expected undo steps cycle=" cycle-idx " seed=" seed)) + (is (empty? issues-after-undo) + (str "db issues after undo-all cycle=" cycle-idx " seed=" seed + " " (pr-str issues-after-undo))) + (assert-no-invalid-tx! seed history repro) + (let [redo-steps (redo-all! repo-a max-stack-steps) + issues-after-redo (db-issues @conn) + attrs-after-redo (block-attr-map @conn)] + (is (pos? redo-steps) + (str "expected redo steps cycle=" cycle-idx " seed=" seed)) + (is (empty? issues-after-redo) + (str "db issues after redo-all cycle=" cycle-idx " seed=" seed + " " (pr-str issues-after-redo))) + (is (seq attrs-after-redo) + (str "db should not be empty after redo-all cycle=" cycle-idx + " seed=" seed)) + (assert-no-invalid-tx! seed history repro))))) + + (let [issues (db-issues @conn)] + (is (empty? issues) + (str "db issues seed=" seed " " (pr-str issues)))) + (assert-no-invalid-tx! seed history repro) + (finally + (restore))))))))) + (defn- run-random-ops! [rng server clients repo->state base-uuid history run-ops-opts steps] (dotimes [_ steps] @@ -1715,11 +2173,12 @@ (finally (restore))))))))) -(deftest two-clients-cut-paste-random-sim-test +(deftest ^:fix-me two-clients-cut-paste-random-sim-test (testing "db-sync convergence under random cut-paste with child operations" (let [seed (or (env-seed) default-seed) rng (make-rng seed) gen-uuid #(rng-uuid rng) + cut-paste-runs (min op-runs 80) base-uuid (gen-uuid) conn-a (db-test/create-conn) conn-b (db-test/create-conn) @@ -1753,7 +2212,7 @@ (create-block! conn-a base-a "" target-uuid) (swap! state-a update :blocks into #{parent-uuid child-uuid target-uuid}) - (dotimes [_ op-runs] + (dotimes [_ cut-paste-runs] (run-ops! rng {:repo repo-a :conn conn-a :base-uuid base-uuid @@ -1820,6 +2279,14 @@ (create-block! conn-a base-a "" target-uuid) (swap! state-a update :blocks into #{parent-uuid child-uuid target-uuid}) + (try + (sync-loop! server clients) + (catch :default e + (report-history! seed history {:type :sync-loop-error + :phase :undo-redo-add-remove-cut-paste-initial-sync + :error (ex-data e)}) + (throw e))) + (dotimes [_ op-runs] (run-ops! rng {:repo repo-a :conn conn-a @@ -1834,8 +2301,20 @@ :delete-block :cut-paste-block-with-child}} :context {:phase :undo-redo-add-remove-cut-paste}}) - (sync-loop! server clients)) - (sync-loop! server clients) + (try + (sync-loop! server clients) + (catch :default e + (report-history! seed history {:type :sync-loop-error + :phase :undo-redo-add-remove-cut-paste + :error (ex-data e)}) + (throw e)))) + (try + (sync-loop! server clients) + (catch :default e + (report-history! seed history {:type :sync-loop-error + :phase :undo-redo-add-remove-cut-paste-final + :error (ex-data e)}) + (throw e))) (let [issues-a (db-issues @conn-a) issues-b (db-issues @conn-b) @@ -1851,6 +2330,113 @@ (finally (restore))))))))) +(deftest ^:long ^:large-vars/cleanup-todo two-clients-online-add-vs-delete-with-undo-redo-random-sim-test + (testing "both online: client A adds blocks while client B deletes with random undo/redo" + (let [seed (or (env-seed) default-seed) + rng (make-rng seed) + gen-uuid #(rng-uuid rng) + scenario-runs op-runs + base-uuid (gen-uuid) + conn-a (db-test/create-conn) + conn-b (db-test/create-conn) + ops-a (d/create-conn client-op/schema-in-db) + ops-b (d/create-conn client-op/schema-in-db) + client-a (make-client repo-a) + client-b (make-client repo-b) + server (make-server) + history (atom []) + state-a (atom {:pages #{base-uuid} :blocks #{}}) + state-b (atom {:pages #{base-uuid} :blocks #{}}) + ops #{:create-block :delete-blocks :delete-block :indent-outdent-blocks :undo :redo} + op-weights {:create-block 16 + :delete-block 10 + :delete-blocks 10 + :indent-outdent-blocks 10 + :undo 8 + :redo 8} + a-op-table (build-weighted-op-table ops op-weights :a-add-undo-redo) + b-op-table (build-weighted-op-table ops op-weights :b-delete-undo-redo)] + (with-test-repos {repo-a {:conn conn-a :ops-conn ops-a} + repo-b {:conn conn-b :ops-conn ops-b}} + (fn [] + (let [{:keys [repro restore]} (install-invalid-tx-repro! seed history) + clients [{:repo repo-a :conn conn-a :client client-a :online? true :gen-uuid gen-uuid} + {:repo repo-b :conn conn-b :client client-b :online? true :gen-uuid gen-uuid}] + refresh-state! (fn [state conn] + (let [db @conn + block-uuids (->> (active-block-uuids db) + (remove (fn [uuid] + (some-> (d/entity db [:block/uuid uuid]) + ldb/page?))) + set)] + (swap! state assoc :pages #{base-uuid} + :blocks block-uuids)))] + (try + (reset! db-sync/*repo->latest-remote-tx {}) + (record-meta! history {:seed seed + :base-uuid base-uuid + :phase :two-clients-online-add-vs-delete + :scenario-runs scenario-runs}) + (doseq [conn [conn-a conn-b]] + (ensure-base-page! conn base-uuid)) + (doseq [repo [repo-a repo-b]] + (client-op/update-local-tx repo 0)) + + ;; Bootstrap one local block on A so B has a known deletion target after initial sync. + (let [base-a (d/entity @conn-a [:block/uuid base-uuid]) + seed-uuid (gen-uuid)] + (create-block! conn-a base-a "seed" seed-uuid) + (swap! state-a update :blocks conj seed-uuid)) + (sync-loop! server clients) + (refresh-state! state-a conn-a) + (refresh-state! state-b conn-b) + + (dotimes [i scenario-runs] + (run-ops! rng {:repo repo-a + :conn conn-a + :base-uuid base-uuid + :state state-a + :gen-uuid gen-uuid} + 1 + history + {:op-table-override a-op-table + :context {:phase :a-add-undo-redo :iter i}}) + (sync-loop! server clients) + (refresh-state! state-a conn-a) + (refresh-state! state-b conn-b) + + (run-ops! rng {:repo repo-b + :conn conn-b + :base-uuid base-uuid + :state state-b + :gen-uuid gen-uuid} + 1 + history + {:op-table-override b-op-table + :context {:phase :b-delete-undo-redo :iter i}}) + (sync-loop! server clients) + (refresh-state! state-a conn-a) + (refresh-state! state-b conn-b)) + + (sync-loop! server clients) + (let [issues-a (db-issues @conn-a) + issues-b (db-issues @conn-b) + attrs-a (block-attr-map @conn-a) + attrs-b (block-attr-map @conn-b) + create-count (op-count history :create-block) + delete-count (+ (op-count history :delete-block) + (op-count history :delete-blocks))] + (is (pos? create-count) + (str "expected create-block ops seed=" seed " history=" (count @history))) + (is (pos? delete-count) + (str "expected delete ops seed=" seed " history=" (count @history))) + (is (empty? issues-a) (str "db A issues seed=" seed " " (pr-str issues-a))) + (is (empty? issues-b) (str "db B issues seed=" seed " " (pr-str issues-b))) + (assert-synced-attrs! seed history attrs-a attrs-b attrs-b) + (assert-no-invalid-tx! seed history repro)) + (finally + (restore))))))))) + (deftest ^:long ^:large-vars/cleanup-todo three-clients-single-repo-sim-test (testing "db-sync convergence with three clients sharing one repo" (let [seed (or (env-seed) default-seed) diff --git a/src/test/frontend/worker/db_sync_test.cljs b/src/test/frontend/worker/db_sync_test.cljs index d1f01f78eb..1174c07a84 100644 --- a/src/test/frontend/worker/db_sync_test.cljs +++ b/src/test/frontend/worker/db_sync_test.cljs @@ -1,5 +1,6 @@ (ns frontend.worker.db-sync-test (:require [cljs.test :refer [deftest is testing async]] + [clojure.set :as set] [clojure.string :as string] [datascript.core :as d] [frontend.common.crypt :as crypt] @@ -15,9 +16,12 @@ [frontend.worker.sync.crypt :as sync-crypt] [frontend.worker.sync.handle-message :as sync-handle-message] [frontend.worker.sync.large-title :as sync-large-title] + [frontend.worker.sync.presence :as sync-presence] [frontend.worker.sync.temp-sqlite :as sync-temp-sqlite] [frontend.worker.sync.upload :as sync-upload] [logseq.common.config :as common-config] + [logseq.common.util :as common-util] + [logseq.common.util.page-ref :as page-ref] [logseq.db :as ldb] [logseq.db-sync.checksum :as sync-checksum] [logseq.db-sync.storage :as sync-storage] @@ -32,7 +36,6 @@ [logseq.outliner.op :as outliner-op] [logseq.outliner.page :as outliner-page] [logseq.outliner.property :as outliner-property] - [logseq.undo-redo-validate :as undo-validate] [promesa.core :as p])) (def ^:private test-repo "test-db-sync-repo") @@ -164,6 +167,8 @@ (db-sync/enqueue-local-tx! test-repo tx-report)))) (let [result (f) cleanup (fn [] + (when ops-conn + (d/unlisten! db-conn ::listen-db)) (reset! worker-state/*datascript-conns db-prev) (reset! worker-state/*client-ops-conns ops-prev))] (if (p/promise? result) @@ -193,6 +198,25 @@ :child2 child2 :child3 child3})) +(defn- setup-two-parents + [] + (let [conn (db-test/create-conn-with-blocks + {:pages-and-blocks + [{:page {:block/title "page 1"} + :blocks [{:block/title "parent a" + :build/children [{:block/title "a child 1"} + {:block/title "a child 2"}]} + {:block/title "parent b" + :build/children [{:block/title "b child 1"} + {:block/title "b child 2"}]}]}]}) + client-ops-conn (d/create-conn client-op/schema-in-db)] + {:conn conn + :client-ops-conn client-ops-conn + :parent-a (db-test/find-block-by-content @conn "parent a") + :parent-b (db-test/find-block-by-content @conn "parent b") + :a-child-1 (db-test/find-block-by-content @conn "a child 1") + :b-child-1 (db-test/find-block-by-content @conn "b child 1")})) + (deftest resolve-ws-token-refreshes-when-token-expired-test (async done (let [refresh-calls (atom 0) @@ -299,6 +323,30 @@ @(:online-users client))) (is (= 1 (count @broadcasts)))))) +(deftest sync-counts-counts-only-true-pending-local-ops-test + (testing "pending-local should count only rows with :db-sync/pending? true" + (let [{:keys [conn client-ops-conn]} (setup-parent-child)] + (with-datascript-conns conn client-ops-conn + (fn [] + (ldb/transact! client-ops-conn + [{:db-sync/tx-id (random-uuid) + :db-sync/created-at 1 + :db-sync/pending? false} + {:db-sync/tx-id (random-uuid) + :db-sync/created-at 2} + {:db-sync/tx-id (random-uuid) + :db-sync/created-at 3 + :db-sync/pending? true}]) + (let [counts (sync-presence/sync-counts + {:get-datascript-conn worker-state/get-datascript-conn + :get-client-ops-conn worker-state/get-client-ops-conn + :get-unpushed-asset-ops-count client-op/get-unpushed-asset-ops-count + :get-local-tx (constantly 0) + :get-graph-uuid (constantly nil) + :latest-remote-tx {}} + test-repo)] + (is (= 1 (:pending-local counts))))))))) + (deftest upload-graph-metadata-write-is-not-persisted-as-local-sync-tx-test (let [captured (atom nil) fake-conn (atom :db)] @@ -434,7 +482,7 @@ (reset! db-sync/*repo->latest-remote-tx latest-prev))))))))) (deftest hello-checksum-mismatch-fails-fast-for-e2ee-test - (testing "e2ee graphs ignore checksum verification for now" + (testing "e2ee graphs also fail fast on checksum mismatch" (let [{:keys [conn client-ops-conn]} (setup-parent-child) latest-prev @db-sync/*repo->latest-remote-tx raw-message (js/JSON.stringify @@ -452,9 +500,15 @@ (with-redefs [sync-apply/flush-pending! (fn [& _] nil) sync-assets/enqueue-asset-sync! (fn [& _] nil) sync-crypt/graph-e2ee? (constantly true)] - (sync-handle-message/handle-message! test-repo client raw-message) - (is (= 0 (get @db-sync/*repo->latest-remote-tx test-repo))) - (reset! db-sync/*repo->latest-remote-tx latest-prev))))))) + (try + (sync-handle-message/handle-message! test-repo client raw-message) + (is false "expected checksum mismatch to fail-fast for e2ee graphs") + (catch :default error + (let [data (ex-data error)] + (is (= :db-sync/checksum-mismatch (:type data))) + (is (= "bad-checksum" (:remote-checksum data))))) + (finally + (reset! db-sync/*repo->latest-remote-tx latest-prev))))))))) (deftest hello-without-checksum-is-accepted-test (testing "legacy hello without checksum is accepted" @@ -569,7 +623,7 @@ (finally (d/unlisten! conn-b ::capture-remote-many-page-property)))))) -(deftest transact-with-temp-conn-preserves-many-page-property-values-test +(deftest batch-transact-preserves-many-page-property-values-test (testing "temp conn batch keeps both values when a new page-many property is created and then assigned twice" (let [conn (db-test/create-conn-with-blocks {:pages-and-blocks @@ -577,7 +631,7 @@ :blocks [{:block/title "remote object"}]}]}) block-id (:db/id (db-test/find-block-by-content @conn "remote object")) property-id :plugin.property._test_plugin/x7] - (ldb/transact-with-temp-conn! + (ldb/batch-transact-with-temp-conn! conn {} (fn [temp-conn] @@ -591,14 +645,14 @@ (is (= #{"page y" "page z"} (set (map :block/name (:plugin.property._test_plugin/x7 block'))))))))) -(deftest transact-with-temp-conn-preserves-tag-many-page-property-values-test +(deftest batch-transact-preserves-tag-many-page-property-values-test (testing "temp conn batch keeps tag property values when a new many page property is upserted first" (let [conn (db-test/create-conn-with-blocks {:pages-and-blocks [{:page {:block/title "page 1"} :blocks [{:block/title "remote object"}]}]}) property-id :plugin.property._test_plugin/x7] - (ldb/transact-with-temp-conn! + (ldb/batch-transact-with-temp-conn! conn {} (fn [temp-conn] @@ -627,7 +681,7 @@ *batch-tx-data (volatile! [])] (swap! temp-conn assoc :skip-store? true - :batch-temp-conn? true) + :batch-tx? true) (d/listen! temp-conn ::capture-temp-batch (fn [{:keys [tx-data]}] (vswap! *batch-tx-data into tx-data))) @@ -700,8 +754,6 @@ {:tx (sqlite-util/write-transit-str (->> tx (db-normalize/remove-retract-entity-ref @conn) - (#'sync-apply/drop-missing-created-block-datoms @conn) - (#'sync-apply/sanitize-tx-data @conn) distinct vec)) :outliner-op outliner-op})))] @@ -731,8 +783,6 @@ {:tx (sqlite-util/write-transit-str (->> tx (db-normalize/remove-retract-entity-ref @conn) - (#'sync-apply/drop-missing-created-block-datoms @conn) - (#'sync-apply/sanitize-tx-data @conn) distinct vec)) :outliner-op outliner-op})))] @@ -769,8 +819,6 @@ (let [sanitize-tx (fn [tx] (->> tx (db-normalize/remove-retract-entity-ref @local-conn) - (#'sync-apply/drop-missing-created-block-datoms @local-conn) - (#'sync-apply/sanitize-tx-data @local-conn) distinct vec)) tx-entries (mapv (fn [{:keys [tx outliner-op]}] @@ -798,6 +846,8 @@ (is (seq pending)) (is (= :toggle-reaction (:db-sync/outliner-op (first raw-pending)))) (is (= :toggle-reaction (:outliner-op (first pending)))) + (is (= [[:transact nil]] + (:forward-outliner-ops (first pending)))) (is (some (fn [tx] (and (vector? tx) (= :db/add (first tx)) @@ -805,6 +855,1484 @@ (= "+1" (nth tx 3 nil)))) txs)))))))) +(deftest rename-page-enqueues-canonical-save-block-pending-op-test + (testing "rename-page is persisted as canonical save-block op" + (let [{:keys [conn client-ops-conn]} (setup-parent-child) + page-uuid (random-uuid)] + (with-datascript-conns conn client-ops-conn + (fn [] + (worker-page/create! conn "Rename Me" :uuid page-uuid) + (outliner-op/apply-ops! conn + [[:rename-page [page-uuid "Renamed"]]] + local-tx-meta) + (let [{:keys [forward-outliner-ops]} (last (#'sync-apply/pending-txs test-repo))] + (is (= :save-block (ffirst forward-outliner-ops))) + (is (= {:block/uuid page-uuid + :block/title "Renamed"} + (first (second (first forward-outliner-ops))))))))))) + +(deftest move-blocks-up-down-enqueues-canonical-move-blocks-pending-op-test + (testing "move-blocks-up-down is persisted as semantic move-blocks-up-down op" + (let [{:keys [conn client-ops-conn child2]} (setup-parent-child)] + (with-datascript-conns conn client-ops-conn + (fn [] + (outliner-op/apply-ops! conn + [[:move-blocks-up-down [[(:db/id child2)] true]]] + local-tx-meta) + (let [{:keys [forward-outliner-ops]} (first (#'sync-apply/pending-txs test-repo)) + [op [ids up?]] (first forward-outliner-ops)] + (is (= :move-blocks-up-down op)) + (is (seq ids)) + (is (= true up?)))))))) + +(deftest indent-outdent-enqueues-canonical-move-blocks-pending-op-test + (testing "indent-outdent-blocks is persisted as canonical move-blocks op" + (let [{:keys [conn client-ops-conn child2]} (setup-parent-child)] + (with-datascript-conns conn client-ops-conn + (fn [] + (outliner-op/apply-ops! conn + [[:indent-outdent-blocks [[(:db/id child2)] true {}]]] + local-tx-meta) + (let [{:keys [forward-outliner-ops]} (first (#'sync-apply/pending-txs test-repo)) + [_ [_ target-id opts]] (first forward-outliner-ops)] + (is (= :move-blocks (ffirst forward-outliner-ops))) + (is (some? target-id)) + (is (contains? opts :sibling?)) + (is (nil? (:source-op opts))))))))) + +(deftest indent-outdent-direct-outdent-last-child-builds-forward-and-inverse-move-history-test + (testing "direct outdent on last child builds concrete move forward/inverse ops with ui outliner-op metadata" + (let [{:keys [conn client-ops-conn parent child2 child3]} (setup-parent-child) + tx-meta (assoc local-tx-meta :outliner-op :move-blocks)] + (with-datascript-conns conn client-ops-conn + (fn [] + (outliner-op/apply-ops! conn + [[:indent-outdent-blocks [[(:db/id child3)] false {:parent-original nil + :logical-outdenting? nil}]]] + tx-meta) + (let [source-row (first (#'sync-apply/pending-txs test-repo)) + forward-ops (:forward-outliner-ops source-row) + inverse-ops (:inverse-outliner-ops source-row)] + (is (= :move-blocks (ffirst forward-ops))) + (is (= [[:block/uuid (:block/uuid child3)]] + (get-in forward-ops [0 1 0]))) + (is (= [:block/uuid (:block/uuid parent)] + (get-in forward-ops [0 1 1]))) + (is (= true (get-in forward-ops [0 1 2 :sibling?]))) + (is (= :move-blocks (ffirst inverse-ops))) + (is (= [[:block/uuid (:block/uuid child3)]] + (get-in inverse-ops [0 1 0]))) + (is (= [:block/uuid (:block/uuid child2)] + (get-in inverse-ops [0 1 1]))) + (is (= true (get-in inverse-ops [0 1 2 :sibling?]))))))))) + +(deftest indent-outdent-direct-outdent-with-right-sibling-persists-semantic-move-history-test + (testing "direct outdent with right siblings persists concrete semantic move forward/inverse ops" + (let [{:keys [conn client-ops-conn parent child1 child2]} (setup-parent-child) + tx-meta (assoc local-tx-meta :outliner-op :move-blocks)] + (with-datascript-conns conn client-ops-conn + (fn [] + (outliner-op/apply-ops! conn + [[:indent-outdent-blocks [[(:db/id child2)] false {:parent-original nil + :logical-outdenting? nil}]]] + tx-meta) + (let [source-row (first (#'sync-apply/pending-txs test-repo)) + forward-ops (:forward-outliner-ops source-row) + inverse-ops (:inverse-outliner-ops source-row)] + (is (= :move-blocks (ffirst forward-ops))) + (is (= [[:block/uuid (:block/uuid child2)]] + (get-in forward-ops [0 1 0]))) + (is (= [:block/uuid (:block/uuid parent)] + (get-in forward-ops [0 1 1]))) + (is (= true (get-in forward-ops [0 1 2 :sibling?]))) + (is (= :move-blocks (ffirst inverse-ops))) + (is (= [[:block/uuid (:block/uuid child2)]] + (get-in inverse-ops [0 1 0]))) + (is (= [:block/uuid (:block/uuid child1)] + (get-in inverse-ops [0 1 1]))) + (is (= true (get-in inverse-ops [0 1 2 :sibling?]))))))))) + +(deftest indent-outdent-direct-outdent-undo-restores-right-sibling-parent-test + (testing "undo after direct outdent restores right sibling parent to original parent" + (let [{:keys [conn client-ops-conn parent child2 child3]} (setup-parent-child) + parent-uuid (:block/uuid parent) + child2-uuid (:block/uuid child2) + child3-uuid (:block/uuid child3)] + (with-datascript-conns conn client-ops-conn + (fn [] + (outliner-op/apply-ops! conn + [[:indent-outdent-blocks [[(:db/id child2)] false {:parent-original nil + :logical-outdenting? nil}]]] + local-tx-meta) + (let [source-row (first (#'sync-apply/pending-txs test-repo)) + source-tx-id (:tx-id source-row) + child3-after-outdent (d/entity @conn [:block/uuid child3-uuid])] + (is (= child2-uuid + (:block/uuid (:block/parent child3-after-outdent)))) + (let [undo-result (#'sync-apply/apply-history-action! test-repo source-tx-id true {}) + child2-after-undo (d/entity @conn [:block/uuid child2-uuid]) + child3-after-undo (d/entity @conn [:block/uuid child3-uuid])] + (is (= true (:applied? undo-result))) + (is (= parent-uuid + (:block/uuid (:block/parent child2-after-undo)))) + (is (= parent-uuid + (:block/uuid (:block/parent child3-after-undo))))))))))) + +(deftest indent-outdent-undo-enqueues-concrete-move-blocks-history-test + (testing "indent-outdent outdent-path persists concrete semantic move history and undo/redo replays without invalid entities" + (let [{:keys [conn client-ops-conn child2]} (setup-parent-child) + prev-invalid-callback @ldb/*transact-invalid-callback + invalid-payload* (atom nil)] + (with-datascript-conns conn client-ops-conn + (fn [] + (reset! ldb/*transact-invalid-callback + (fn [tx-report errors] + (reset! invalid-payload* {:tx-meta (:tx-meta tx-report) + :errors errors}))) + (try + (outliner-op/apply-ops! conn + [[:indent-outdent-blocks [[(:db/id child2)] false {:parent-original nil + :logical-outdenting? nil}]]] + local-tx-meta) + (let [source-row (first (#'sync-apply/pending-txs test-repo)) + source-tx-id (:tx-id source-row) + undo-result (#'sync-apply/apply-history-action! test-repo source-tx-id true {}) + redo-result (#'sync-apply/apply-history-action! test-repo source-tx-id false {})] + (is (= :move-blocks (ffirst (:forward-outliner-ops source-row)))) + (is (= true (:applied? undo-result))) + (is (= true (:applied? redo-result))) + (is (nil? @invalid-payload*)) + (is (= "child 2" (:block/title (d/entity @conn (:db/id child2)))))) + (finally + (reset! ldb/*transact-invalid-callback prev-invalid-callback)))))))) + +(deftest enqueue-local-tx-canonicalizes-batch-import-to-transact-test + (testing "batch-import-edn local tx persists as canonical transact op" + (let [{:keys [conn client-ops-conn]} (setup-parent-child) + tx-report (d/with @conn + [{:block/uuid (random-uuid) + :block/title "imported" + :block/tags :logseq.class/Page + :block/created-at 1760000000000 + :block/updated-at 1760000000000}] + (assoc local-tx-meta :outliner-op :batch-import-edn))] + (with-datascript-conns conn client-ops-conn + (fn [] + (db-sync/enqueue-local-tx! test-repo tx-report) + (let [{:keys [forward-outliner-ops]} (first (#'sync-apply/pending-txs test-repo))] + (is (= [[:transact nil]] forward-outliner-ops)))))))) + +(deftest enqueue-local-tx-preserves-existing-tx-id-test + (testing "local tx persistence reuses tx-id already attached to tx-meta" + (let [{:keys [conn client-ops-conn child1]} (setup-parent-child) + tx-id (random-uuid) + tx-report (d/with @conn + [[:db/add (:db/id child1) :block/title "stable tx id"]] + (assoc local-tx-meta + :db-sync/tx-id tx-id + :outliner-op :save-block))] + (with-datascript-conns conn client-ops-conn + (fn [] + (db-sync/enqueue-local-tx! test-repo tx-report) + (let [{persisted-tx-id :tx-id} (first (#'sync-apply/pending-txs test-repo))] + (is (= tx-id persisted-tx-id)))))))) + +(deftest apply-history-action-does-not-reuse-original-tx-id-test + (testing "undo/redo history actions should not overwrite the original pending tx row" + (let [{:keys [conn client-ops-conn child1]} (setup-parent-child) + child-uuid (:block/uuid child1)] + (with-datascript-conns conn client-ops-conn + (fn [] + (outliner-op/apply-ops! conn + [[:save-block [{:block/uuid child-uuid + :block/title "hello"} nil]]] + local-tx-meta) + (let [{:keys [tx-id]} (first (#'sync-apply/pending-txs test-repo))] + (let [{:keys [applied? history-tx-id]} (#'sync-apply/apply-history-action! test-repo + tx-id + true + {:db-sync/tx-id tx-id})] + (is (= true applied?)) + (is (uuid? history-tx-id)) + (is (not= tx-id history-tx-id))) + (let [pending (#'sync-apply/pending-txs test-repo)] + (is (= 2 (count pending))) + (is (= 2 (count (distinct (map :tx-id pending))))) + (is (= "hello" + (get-in (#'sync-apply/pending-tx-by-id test-repo tx-id) + [:forward-outliner-ops 0 1 0 :block/title])))))))))) + +(deftest apply-history-action-preserves-source-forward-inverse-ops-test + (testing "undo/redo history actions should preserve source forward/inverse ops and create new tx rows" + (let [{:keys [conn client-ops-conn child1]} (setup-parent-child) + child-uuid (:block/uuid child1)] + (with-datascript-conns conn client-ops-conn + (fn [] + (outliner-op/apply-ops! conn + [[:save-block [{:block/uuid child-uuid + :block/title "hello"} nil]]] + local-tx-meta) + (let [{source-tx-id :tx-id} (first (#'sync-apply/pending-txs test-repo))] + (let [{undo-applied? :applied? + undo-history-tx-id :history-tx-id} + (#'sync-apply/apply-history-action! test-repo + source-tx-id + true + {})] + (is (= true undo-applied?)) + (is (uuid? undo-history-tx-id)) + (is (not= source-tx-id undo-history-tx-id))) + (let [source-pending (#'sync-apply/pending-tx-by-id test-repo source-tx-id) + pending-after-undo (#'sync-apply/pending-txs test-repo) + undo-pending (first (filter #(not= source-tx-id (:tx-id %)) pending-after-undo))] + (is (= 2 (count pending-after-undo))) + (is (some? undo-pending)) + (is (= "hello" + (get-in source-pending [:forward-outliner-ops 0 1 0 :block/title]))) + (is (= "child 1" + (get-in source-pending [:inverse-outliner-ops 0 1 0 :block/title]))) + (is (= "child 1" + (get-in undo-pending [:forward-outliner-ops 0 1 0 :block/title]))) + (is (= "hello" + (get-in undo-pending [:inverse-outliner-ops 0 1 0 :block/title])))) + (let [{redo-applied? :applied? + redo-history-tx-id :history-tx-id} + (#'sync-apply/apply-history-action! test-repo + source-tx-id + false + {})] + (is (= true redo-applied?)) + (is (uuid? redo-history-tx-id)) + (is (not= source-tx-id redo-history-tx-id))) + (let [source-pending (#'sync-apply/pending-tx-by-id test-repo source-tx-id) + pending-after-redo (#'sync-apply/pending-txs test-repo) + new-tx-ids (set (map :tx-id pending-after-redo))] + (is (= 3 (count pending-after-redo))) + (is (= 3 (count new-tx-ids))) + (is (contains? new-tx-ids source-tx-id)) + (is (= "hello" + (get-in source-pending [:forward-outliner-ops 0 1 0 :block/title]))) + (is (= "child 1" + (get-in source-pending [:inverse-outliner-ops 0 1 0 :block/title])))))))))) + +(deftest apply-history-action-semantic-op-must-not-fallback-to-raw-tx-test + (testing "semantic history action should not fallback to raw tx replay" + (let [{:keys [conn client-ops-conn child1]} (setup-parent-child) + tx-id (random-uuid) + child-uuid (:block/uuid child1) + before-title (:block/title (d/entity @conn (:db/id child1))) + missing-uuid (random-uuid) + raw-title "raw fallback title" + tx-data [[:db/add [:block/uuid child-uuid] :block/title raw-title]]] + (with-datascript-conns conn client-ops-conn + (fn [] + (ldb/transact! client-ops-conn + [{:db-sync/tx-id tx-id + :db-sync/pending? true + :db-sync/created-at (.now js/Date) + :db-sync/outliner-op :save-block + :db-sync/forward-outliner-ops [[:save-block [{:block/uuid missing-uuid + :block/title "broken semantic"} {}]]] + :db-sync/normalized-tx-data tx-data + :db-sync/reversed-tx-data []}]) + (is (thrown? js/Error + (#'sync-apply/apply-history-action! test-repo tx-id false {}))) + (is (= before-title + (:block/title (d/entity @conn [:block/uuid child-uuid]))))))))) + +(deftest apply-history-action-redo-invalid-insert-conflict-skips-fail-fast-test + (testing "redo conflict on stale insert target should throw skippable error without fail-fast logger" + (let [{:keys [conn client-ops-conn]} (setup-parent-child) + tx-id (random-uuid) + missing-parent-uuid (random-uuid) + inserted-uuid (random-uuid)] + (with-datascript-conns conn client-ops-conn + (fn [] + (ldb/transact! client-ops-conn + [{:db-sync/tx-id tx-id + :db-sync/pending? true + :db-sync/created-at (.now js/Date) + :db-sync/outliner-op :insert-blocks + :db-sync/forward-outliner-ops [[:insert-blocks [[{:block/uuid inserted-uuid + :block/title "" + :block/parent [:block/uuid missing-parent-uuid]} + [:block/uuid missing-parent-uuid] + {:sibling? false + :keep-uuid? true}]]]] + :db-sync/normalized-tx-data [] + :db-sync/reversed-tx-data []}]) + (with-redefs [sync-apply/fail-fast (fn [_tag data] + (throw (ex-info "fail-fast-called" data)))] + (try + (#'sync-apply/apply-history-action! test-repo tx-id false {}) + (is false "expected redo conflict to throw") + (catch :default e + (is (not= "fail-fast-called" (ex-message e))) + (is (= :invalid-history-action-ops + (:reason (ex-data e)))))))))))) + +(deftest apply-history-action-save-block-ignores-stale-db-id-when-uuid-exists-test + (testing "semantic save-block replay should resolve by uuid and ignore stale db/id" + (let [{:keys [conn client-ops-conn child1]} (setup-parent-child) + tx-id (random-uuid) + child-uuid (:block/uuid child1) + stale-db-id 99999999 + new-title "semantic replay with stale db id"] + (with-datascript-conns conn client-ops-conn + (fn [] + (ldb/transact! client-ops-conn + [{:db-sync/tx-id tx-id + :db-sync/pending? true + :db-sync/created-at (.now js/Date) + :db-sync/outliner-op :save-block + :db-sync/forward-outliner-ops [[:save-block [{:db/id stale-db-id + :block/uuid child-uuid + :block/title new-title} + {}]]] + :db-sync/normalized-tx-data [] + :db-sync/reversed-tx-data []}]) + (let [result (#'sync-apply/apply-history-action! test-repo tx-id false {})] + (is (= true (:applied? result))) + (is (= :semantic-ops (:source result))) + (is (= new-title + (:block/title (d/entity @conn [:block/uuid child-uuid])))))))))) + +(deftest reverse-local-txs-uses-reversed-tx-data-test + (testing "rebase reverse uses reversed tx-data even when semantic inverse ops are missing" + (let [{:keys [conn client-ops-conn child1]} (setup-parent-child) + tx-id (random-uuid) + child-id (:db/id child1) + child-uuid (:block/uuid child1) + local-tx {:tx-id tx-id + :outliner-op :save-block + :forward-outliner-ops [[:save-block [{:block/uuid (random-uuid) + :block/title "value"} {}]]] + :inverse-outliner-ops nil + :reversed-tx [[:db/add child-id :block/title "raw reverse"]]}] + (with-datascript-conns conn client-ops-conn + (fn [] + (let [reports (#'sync-apply/reverse-local-txs! conn [local-tx] {:rtc-tx? true})] + (is (= 1 (count reports))) + (is (= "raw reverse" + (:block/title (d/entity @conn [:block/uuid child-uuid])))))))))) + +(deftest enqueue-local-tx-keeps-mixed-semantic-forward-outliner-ops-test + (testing "mixed semantic outliner ops stay semantic and preserve op ordering" + (let [{:keys [conn client-ops-conn child2]} (setup-parent-child) + block-id (:db/id child2) + block-uuid (:block/uuid child2) + tx-report (d/with @conn + [[:db/add block-id :block/title "mixed fallback"]] + (assoc local-tx-meta + :outliner-op :save-block + :outliner-ops [[:save-block [{:block/uuid block-uuid + :block/title "mixed fallback"} {}]] + [:indent-outdent-blocks [[block-id] + false + {:parent-original nil + :logical-outdenting? nil}]]]))] + (with-datascript-conns conn client-ops-conn + (fn [] + (db-sync/enqueue-local-tx! test-repo tx-report) + (let [{:keys [forward-outliner-ops]} (first (#'sync-apply/pending-txs test-repo))] + (is (= :save-block (ffirst forward-outliner-ops))) + (is (= :indent-outdent-blocks (first (second forward-outliner-ops)))) + (is (= [[:block/uuid block-uuid]] + (get-in forward-outliner-ops [1 1 0]))))))))) + +(deftest apply-history-action-redo-fails-fast-on-transact-placeholder-test + (testing "redo fails fast when semantic ops contain transact placeholder to avoid silent partial replay" + (let [{:keys [conn client-ops-conn child1]} (setup-parent-child) + tx-id (random-uuid) + child-uuid (:block/uuid child1) + before-title (:block/title (d/entity @conn (:db/id child1))) + semantic-title "semantic replay value" + raw-title "raw replay value" + forward-ops [[:save-block [{:block/uuid child-uuid + :block/title semantic-title} {}]] + [:transact nil]] + tx-data [[:db/add [:block/uuid child-uuid] :block/title raw-title]] + reversed-tx-data [[:db/add [:block/uuid child-uuid] :block/title before-title]]] + (with-datascript-conns conn client-ops-conn + (fn [] + (ldb/transact! client-ops-conn + [{:db-sync/tx-id tx-id + :db-sync/pending? true + :db-sync/created-at (.now js/Date) + :db-sync/outliner-op :save-block + :db-sync/forward-outliner-ops forward-ops + :db-sync/normalized-tx-data tx-data + :db-sync/reversed-tx-data reversed-tx-data}]) + (is (thrown? js/Error + (#'sync-apply/apply-history-action! test-repo tx-id false {}))) + (is (= before-title + (:block/title (d/entity @conn [:block/uuid child-uuid]))))))))) + +(deftest enqueue-local-tx-allows-explicit-transact-placeholder-forward-op-test + (testing "enqueue-local-tx should preserve explicit transact placeholder forward ops" + (let [{:keys [conn client-ops-conn child1]} (setup-parent-child) + child-id (:db/id child1) + tx-id (random-uuid) + tx-report (d/with @conn + [[:db/add child-id :block/title "placeholder replay"]] + (assoc local-tx-meta + :db-sync/tx-id tx-id + :db-sync/forward-outliner-ops [[:transact nil]] + :db-sync/inverse-outliner-ops nil + :outliner-op :toggle-reaction))] + (with-datascript-conns conn client-ops-conn + (fn [] + (db-sync/enqueue-local-tx! test-repo tx-report) + (let [pending (first (#'sync-apply/pending-txs test-repo))] + (is (= tx-id (:tx-id pending))) + (is (= [[:transact nil]] + (:forward-outliner-ops pending))) + (is (nil? (:inverse-outliner-ops pending))))))))) + +(deftest apply-history-action-undo-delete-blocks-noops-when-target-missing-test + (testing "undo delete-blocks should no-op when the target block is already missing" + (let [{:keys [conn client-ops-conn child1]} (setup-parent-child) + tx-id (random-uuid) + child-uuid (:block/uuid child1) + missing-uuid (random-uuid)] + (with-datascript-conns conn client-ops-conn + (fn [] + (ldb/transact! client-ops-conn + [{:db-sync/tx-id tx-id + :db-sync/pending? true + :db-sync/created-at (.now js/Date) + :db-sync/outliner-op :delete-blocks + :db-sync/forward-outliner-ops + [[:save-block [{:block/uuid child-uuid + :block/title "semantic source"} nil]]] + :db-sync/inverse-outliner-ops + [[:delete-blocks [[[:block/uuid missing-uuid]] {}]]] + :db-sync/normalized-tx-data [] + :db-sync/reversed-tx-data []}]) + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo tx-id true {})))) + (is (some? (d/entity @conn [:block/uuid child-uuid])))))))) + +(deftest enqueue-local-tx-persists-semantic-undo-ops-test + (testing "undo local tx persists explicit semantic forward and inverse ops" + (let [{:keys [conn client-ops-conn child1]} (setup-parent-child) + tx-id (random-uuid) + forward-ops [[:save-block [{:block/uuid (:block/uuid child1) + :block/title "undo value"} {}]]] + inverse-ops [[:save-block [{:block/uuid (:block/uuid child1) + :block/title "child 1"} {}]]] + tx-report (d/with @conn + [[:db/add (:db/id child1) :block/title "undo value"]] + (assoc local-tx-meta + :db-sync/tx-id tx-id + :db-sync/forward-outliner-ops forward-ops + :db-sync/inverse-outliner-ops inverse-ops + :outliner-op :save-block + :undo? true + :gen-undo-ops? false))] + (with-datascript-conns conn client-ops-conn + (fn [] + (db-sync/enqueue-local-tx! test-repo tx-report) + (let [pending (first (#'sync-apply/pending-txs test-repo)) + raw-pending (->> (d/datoms @client-ops-conn :avet :db-sync/created-at) + (map (fn [datom] (d/entity @client-ops-conn (:e datom)))) + first)] + (is (= tx-id (:tx-id pending))) + (is (= forward-ops (:forward-outliner-ops pending))) + (is (= forward-ops (:db-sync/forward-outliner-ops raw-pending))) + (is (= inverse-ops (:db-sync/inverse-outliner-ops raw-pending))))))))) + +(deftest direct-outliner-page-delete-persists-delete-page-outliner-op-test + (testing "direct outliner-page/delete! still persists singleton delete-page forward-outliner-ops" + (let [conn (db-test/create-conn-with-blocks + {:pages-and-blocks [{:page {:block/title "Delete Me"}}]}) + client-ops-conn (d/create-conn client-op/schema-in-db) + page (db-test/find-page-by-title @conn "Delete Me")] + (with-datascript-conns conn client-ops-conn + (fn [] + (outliner-page/delete! conn (:block/uuid page) {}) + (let [{:keys [forward-outliner-ops inverse-outliner-ops]} (first (#'sync-apply/pending-txs test-repo))] + (is (= :delete-page (ffirst forward-outliner-ops))) + (is (= (:block/uuid page) + (get-in forward-outliner-ops [0 1 0]))) + (is (seq inverse-outliner-ops)))))))) + +(deftest delete-page-rewrites-node-refs-and-semantic-undo-redo-test + (testing "moving a page to recycle rewrites node refs and semantic undo/redo restores and reapplies them" + (let [conn (db-test/create-conn-with-blocks + {:pages-and-blocks [{:page {:block/title "Delete Me"}} + {:page {:block/title "Ref Page"} + :blocks [{:block/title "seed"}]}]}) + client-ops-conn (d/create-conn client-op/schema-in-db) + page (db-test/find-page-by-title @conn "Delete Me") + page-id (:db/id page) + page-uuid (:block/uuid page) + ref-block (db-test/find-block-by-content @conn "seed") + ref-block-uuid (:block/uuid ref-block) + node-ref-content (str "ref " (page-ref/->page-ref page-uuid)) + title-content "ref Delete Me"] + (with-datascript-conns conn client-ops-conn + (fn [] + (ldb/transact! conn [{:db/id (:db/id ref-block) + :block/title node-ref-content + :block/refs #{page-id}}]) + (outliner-page/delete! conn page-uuid {}) + (let [{:keys [tx-id forward-outliner-ops inverse-outliner-ops]} + (->> (#'sync-apply/pending-txs test-repo) + (filter #(= :delete-page (:outliner-op %))) + last)] + (is (= :delete-page (ffirst forward-outliner-ops))) + (is (some (fn [[op [block]]] + (and (= :save-block op) + (= ref-block-uuid (:block/uuid block)) + (= title-content (:block/title block)))) + forward-outliner-ops)) + (is (some (fn [[op [target-page-uuid]]] + (and (= :restore-recycled op) + (= page-uuid target-page-uuid))) + inverse-outliner-ops)) + (is (some (fn [[op [block]]] + (and (= :save-block op) + (= ref-block-uuid (:block/uuid block)) + (= node-ref-content (:block/title block)))) + inverse-outliner-ops)) + (is (= title-content + (:block/raw-title (d/entity @conn [:block/uuid ref-block-uuid])))) + (is (not (contains? (set (map :db/id (:block/refs (d/entity @conn [:block/uuid ref-block-uuid])))) + page-id))) + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo tx-id true {})))) + (is (= node-ref-content + (:block/raw-title (d/entity @conn [:block/uuid ref-block-uuid])))) + (is (contains? (set (map :db/id (:block/refs (d/entity @conn [:block/uuid ref-block-uuid])))) + page-id)) + (is (nil? (:logseq.property/deleted-at (d/entity @conn [:block/uuid page-uuid])))) + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo tx-id false {})))) + (is (= title-content + (:block/raw-title (d/entity @conn [:block/uuid ref-block-uuid])))) + (is (not (contains? (set (map :db/id (:block/refs (d/entity @conn [:block/uuid ref-block-uuid])))) + page-id))) + (is (integer? (:logseq.property/deleted-at (d/entity @conn [:block/uuid page-uuid])))))))))) + +(deftest direct-outliner-property-set-persists-set-block-property-outliner-op-test + (testing "direct outliner-property/set-block-property! still persists singleton set-block-property forward-outliner-ops" + (let [graph {:properties {:p2 {:logseq.property/type :default}} + :pages-and-blocks + [{:page {:block/title "page 1"} + :blocks [{:block/title "local object"}]}]} + conn (db-test/create-conn-with-blocks graph) + client-ops-conn (d/create-conn client-op/schema-in-db) + block (db-test/find-block-by-content @conn "local object") + property-id :user.property/p2] + (with-datascript-conns conn client-ops-conn + (fn [] + (outliner-property/set-block-property! conn + [:block/uuid (:block/uuid block)] + property-id + "local value") + (let [pending (#'sync-apply/pending-txs test-repo) + property-tx (some (fn [{:keys [forward-outliner-ops]}] + (when (= :set-block-property (ffirst forward-outliner-ops)) + forward-outliner-ops)) + pending)] + (is (seq pending)) + (is (every? (comp seq :forward-outliner-ops) pending)) + (is (= [:set-block-property + [[:block/uuid (:block/uuid block)] property-id "local value"]] + (first property-tx))))))))) + +(deftest canonical-set-block-property-rewrites-ref-values-to-stable-refs-test + (testing "ref-valued set-block-property ops should persist stable entity refs instead of numeric ids" + (let [graph {:properties {:x7 {:logseq.property/type :page + :db/cardinality :db.cardinality/many}} + :pages-and-blocks + [{:page {:block/title "page 1"} + :blocks [{:block/title "local object"}]}]} + conn (db-test/create-conn-with-blocks graph) + client-ops-conn (d/create-conn client-op/schema-in-db) + block (db-test/find-block-by-content @conn "local object") + property-id :user.property/x7] + (with-datascript-conns conn client-ops-conn + (fn [] + (outliner-page/create! conn "Page y" {}) + (let [page-y (db-test/find-page-by-title @conn "Page y")] + (outliner-property/set-block-property! conn + [:block/uuid (:block/uuid block)] + property-id + (:db/id page-y)) + (let [pending (#'sync-apply/pending-txs test-repo) + property-tx (some (fn [{:keys [forward-outliner-ops]}] + (when (= :set-block-property (ffirst forward-outliner-ops)) + forward-outliner-ops)) + pending)] + (is (= [:set-block-property + [[:block/uuid (:block/uuid block)] + property-id + [:block/uuid (:block/uuid page-y)]]] + (first property-tx)))))))))) + +(deftest canonical-batch-set-property-rewrites-ref-values-to-stable-refs-test + (testing "ref-valued batch-set-property ops should persist stable entity refs instead of numeric ids" + (let [graph {:properties {:x7 {:logseq.property/type :page + :db/cardinality :db.cardinality/many}} + :pages-and-blocks + [{:page {:block/title "page 1"} + :blocks [{:block/title "local object 1"} + {:block/title "local object 2"}]}]} + conn (db-test/create-conn-with-blocks graph) + client-ops-conn (d/create-conn client-op/schema-in-db) + block-1 (db-test/find-block-by-content @conn "local object 1") + block-2 (db-test/find-block-by-content @conn "local object 2") + property-id :user.property/x7] + (with-datascript-conns conn client-ops-conn + (fn [] + (outliner-page/create! conn "Page y" {}) + (let [page-y (db-test/find-page-by-title @conn "Page y")] + (outliner-op/apply-ops! conn + [[:batch-set-property [[(:db/id block-1) + (:db/id block-2)] + property-id + (:db/id page-y) + {}]]] + {}) + (let [pending (#'sync-apply/pending-txs test-repo) + property-tx (some (fn [{:keys [forward-outliner-ops]}] + (when (= :batch-set-property (ffirst forward-outliner-ops)) + forward-outliner-ops)) + pending)] + (is (= [:batch-set-property + [[[:block/uuid (:block/uuid block-1)] + [:block/uuid (:block/uuid block-2)]] + property-id + [:block/uuid (:block/uuid page-y)] + {}]] + (first property-tx)))))))))) + +(deftest replay-batch-set-property-converts-lookup-ref-to-eid-when-entity-id-test + (testing "replay should resolve stable lookup refs back to entity ids for batch-set-property when :entity-id? is true" + (let [graph {:properties {:x7 {:logseq.property/type :page + :db/cardinality :db.cardinality/many}} + :pages-and-blocks + [{:page {:block/title "page 1"} + :blocks [{:block/title "local object"}]}]} + conn (db-test/create-conn-with-blocks graph) + block (db-test/find-block-by-content @conn "local object") + property-id :user.property/x7] + (outliner-page/create! conn "Page y" {}) + (let [page-y (db-test/find-page-by-title @conn "Page y")] + (is (some? (#'sync-apply/replay-canonical-outliner-op! + conn + [:batch-set-property [[[:block/uuid (:block/uuid block)]] + property-id + [:block/uuid (:block/uuid page-y)] + {:entity-id? true}]]))) + (let [block' (d/entity @conn [:block/uuid (:block/uuid block)])] + (is (= #{"page y"} + (set (map :block/name (:user.property/x7 block')))))))))) + +(deftest replay-batch-set-property-converts-raw-uuid-ids-to-eids-test + (testing "replay should resolve raw uuid block ids for batch-set-property" + (let [graph {:properties {:heading {:db/ident :logseq.property/heading + :logseq.property/type :number + :db/cardinality :db.cardinality/one}} + :pages-and-blocks + [{:page {:block/title "page 1"} + :blocks [{:block/title "local object"}]}]} + conn (db-test/create-conn-with-blocks graph) + block (db-test/find-block-by-content @conn "local object") + block-ref [:block/uuid (:block/uuid block)]] + (is (some? (#'sync-apply/replay-canonical-outliner-op! + conn + [:batch-set-property [[(:block/uuid block)] + :logseq.property/heading + 2 + nil]]))) + (is (= 2 + (:logseq.property/heading (d/entity @conn block-ref))))))) + +(deftest apply-history-action-redo-replays-batch-set-property-with-raw-uuid-ids-test + (testing "redo should replay batch-set-property when semantic op stores raw uuid block ids" + (let [graph {:properties {:heading {:db/ident :logseq.property/heading + :logseq.property/type :number + :db/cardinality :db.cardinality/one}} + :pages-and-blocks + [{:page {:block/title "page 1"} + :blocks [{:block/title "local object"}]}]} + conn (db-test/create-conn-with-blocks graph) + client-ops-conn (d/create-conn client-op/schema-in-db)] + (with-datascript-conns conn client-ops-conn + (fn [] + (let [block (db-test/find-block-by-content @conn "local object") + block-uuid (:block/uuid block) + block-ref [:block/uuid block-uuid] + action-tx-id (random-uuid)] + (ldb/transact! client-ops-conn + [{:db-sync/tx-id action-tx-id + :db-sync/pending? true + :db-sync/forward-outliner-ops + [[:batch-set-property [[block-uuid] + :logseq.property/heading + 2 + nil]]] + :db-sync/inverse-outliner-ops + [[:batch-remove-property [[block-ref] + :logseq.property/heading]]] + :db-sync/normalized-tx-data [] + :db-sync/reversed-tx-data []}]) + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo action-tx-id false {})))) + (is (= 2 + (:logseq.property/heading (d/entity @conn block-ref)))) + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo action-tx-id true {})))) + (is (nil? (:logseq.property/heading (d/entity @conn block-ref)))))))))) + +(deftest replay-set-block-property-converts-lookup-ref-to-eid-test + (testing "replay should resolve stable lookup refs back to entity ids for set-block-property" + (let [graph {:properties {:x7 {:logseq.property/type :page + :db/cardinality :db.cardinality/many}} + :pages-and-blocks + [{:page {:block/title "page 1"} + :blocks [{:block/title "local object"}]}]} + conn (db-test/create-conn-with-blocks graph) + block (db-test/find-block-by-content @conn "local object") + property-id :user.property/x7] + (outliner-page/create! conn "Page y" {}) + (let [page-y (db-test/find-page-by-title @conn "Page y")] + (is (some? (#'sync-apply/replay-canonical-outliner-op! + conn + [:set-block-property [[:block/uuid (:block/uuid block)] + property-id + [:block/uuid (:block/uuid page-y)]]]))) + (let [block' (d/entity @conn [:block/uuid (:block/uuid block)])] + (is (= #{"page y"} + (set (map :block/name (:user.property/x7 block')))))))))) + +(deftest replay-set-block-property-converts-raw-uuid-to-eid-test + (testing "replay should resolve raw block uuid ids for set-block-property" + (let [graph {:classes {:tag1 {}} + :pages-and-blocks + [{:page {:block/title "page 1"} + :blocks [{:block/title "local object"}]}]} + conn (db-test/create-conn-with-blocks graph) + block (db-test/find-block-by-content @conn "local object") + tag-id (:db/id (d/entity @conn :user.class/tag1)) + tag-uuid (:block/uuid (d/entity @conn tag-id))] + (is (some? (#'sync-apply/replay-canonical-outliner-op! + conn + [:set-block-property [(:block/uuid block) + :block/tags + [:block/uuid tag-uuid]]]))) + (let [block' (d/entity @conn [:block/uuid (:block/uuid block)])] + (is (= #{tag-id} + (set (map :db/id (:block/tags block'))))))))) + +(deftest apply-history-action-redo-replays-set-block-tags-with-raw-uuid-id-test + (testing "redo should replay set-block-property with raw block uuid ids for tags" + (let [graph {:classes {:tag1 {}} + :pages-and-blocks + [{:page {:block/title "page 1"} + :blocks [{:block/title "local object"}]}]} + conn (db-test/create-conn-with-blocks graph) + client-ops-conn (d/create-conn client-op/schema-in-db)] + (with-datascript-conns conn client-ops-conn + (fn [] + (let [block (db-test/find-block-by-content @conn "local object") + block-uuid (:block/uuid block) + block-ref [:block/uuid block-uuid] + tag (d/entity @conn :user.class/tag1) + tag-uuid (:block/uuid tag) + action-tx-id (random-uuid)] + (ldb/transact! client-ops-conn + [{:db-sync/tx-id action-tx-id + :db-sync/pending? true + :db-sync/forward-outliner-ops + [[:set-block-property [block-uuid + :block/tags + [:block/uuid tag-uuid]]]] + :db-sync/inverse-outliner-ops + [[:remove-block-property [block-ref :block/tags]]] + :db-sync/normalized-tx-data [] + :db-sync/reversed-tx-data []}]) + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo action-tx-id false {})))) + (is (= #{(:db/id tag)} + (set (map :db/id (:block/tags (d/entity @conn block-ref)))))) + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo action-tx-id true {})))) + (is (empty? (:block/tags (d/entity @conn block-ref)))))))))) + +(deftest apply-history-action-redo-replays-insert-blocks-test + (testing "apply-history-action should redo an inserted block from semantic history" + (let [{:keys [conn client-ops-conn parent]} (setup-parent-child) + requested-uuid (random-uuid)] + (with-datascript-conns conn client-ops-conn + (fn [] + (outliner-op/apply-ops! conn + [[:insert-blocks [[{:block/title "history insert" + :block/uuid requested-uuid}] + (:db/id parent) + {:sibling? false}]]] + local-tx-meta) + (let [pending (first (#'sync-apply/pending-txs test-repo)) + inserted (db-test/find-block-by-content @conn "history insert") + inserted-uuid (:block/uuid inserted) + {:keys [tx-id]} pending] + (is (= inserted-uuid + (get-in pending [:forward-outliner-ops 0 1 0 0 :block/uuid]))) + (is (= inserted-uuid + (second (first (get-in pending [:inverse-outliner-ops 0 1 0]))))) + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo tx-id true {})))) + (is (nil? (d/entity @conn [:block/uuid inserted-uuid]))) + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo tx-id false {})))) + (let [inserted* (d/entity @conn [:block/uuid inserted-uuid])] + (is (some? inserted*)) + (is (= "history insert" (:block/title inserted*))) + (is (= (:block/uuid parent) + (some-> inserted* :block/parent :block/uuid)))))))))) + +(deftest apply-history-action-redo-replays-save-block-test + (testing "apply-history-action should redo an inline block edit from semantic history" + (let [{:keys [conn client-ops-conn child1]} (setup-parent-child) + child-uuid (:block/uuid child1)] + (with-datascript-conns conn client-ops-conn + (fn [] + (outliner-op/apply-ops! conn + [[:save-block [{:block/uuid child-uuid + :block/title "child 1 inline edit"} {}]]] + local-tx-meta) + (let [{:keys [tx-id]} (first (#'sync-apply/pending-txs test-repo))] + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo tx-id true {})))) + (is (= "child 1" + (:block/title (d/entity @conn [:block/uuid child-uuid])))) + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo tx-id false {})))) + (is (= "child 1 inline edit" + (:block/title (d/entity @conn [:block/uuid child-uuid])))))))))) + +(deftest apply-history-action-redo-replays-save-block-with-late-created-query-ref-test + (testing "redo should replay save-block when referenced query block is created by a later semantic save-block" + (let [conn (db-test/create-conn-with-blocks + {:pages-and-blocks + [{:page {:block/title "page 1"} + :blocks [{:block/title "source"}]}]}) + client-ops-conn (d/create-conn client-op/schema-in-db) + tx-id (random-uuid) + query-block-uuid (random-uuid)] + (with-datascript-conns conn client-ops-conn + (fn [] + (let [source (db-test/find-block-by-content @conn "source") + source-uuid (:block/uuid source) + source-page-uuid (:block/uuid (:block/page source))] + (is (some? (d/entity @conn [:block/uuid source-uuid]))) + (ldb/transact! client-ops-conn + [{:db-sync/tx-id tx-id + :db-sync/pending? true + :db-sync/created-at (.now js/Date) + :db-sync/outliner-op :save-block + :db-sync/forward-outliner-ops + [[:save-block [{:block/uuid source-uuid + :logseq.property/query [:block/uuid query-block-uuid]} + nil]] + [:save-block [{:block/uuid query-block-uuid + :block/title "" + :block/parent [:block/uuid source-page-uuid] + :block/page [:block/uuid source-page-uuid] + :block/order "a0"} + nil]]] + :db-sync/inverse-outliner-ops + [[:remove-block-property [[:block/uuid source-uuid] + :logseq.property/query]] + [:delete-blocks [[[:block/uuid query-block-uuid]] + {}]]] + :db-sync/normalized-tx-data [] + :db-sync/reversed-tx-data []}]) + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo tx-id false {})))) + (let [parent' (d/entity @conn [:block/uuid source-uuid]) + query-block (d/entity @conn [:block/uuid query-block-uuid])] + (is (some? query-block)) + (is (= query-block-uuid + (some-> parent' :logseq.property/query :block/uuid)))))))))) + +(deftest replay-save-block-creates-missing-block-when-structure-present-test + (testing "replay save-block should create missing block when parent/page attrs are present" + (let [conn (db-test/create-conn-with-blocks + {:pages-and-blocks + [{:page {:block/title "page 1"} + :blocks [{:block/title "seed"}]}]}) + seed (db-test/find-block-by-content @conn "seed") + page-uuid (:block/uuid (:block/page seed)) + block-uuid (random-uuid)] + (is (some? (#'sync-apply/replay-canonical-outliner-op! + conn + [:save-block [{:block/uuid block-uuid + :block/title "" + :block/parent [:block/uuid page-uuid] + :block/page [:block/uuid page-uuid] + :block/order "a0"} + nil]]))) + (is (some? (d/entity @conn [:block/uuid block-uuid])))))) + +(deftest apply-history-action-redo-replays-status-property-test + (testing "apply-history-action should redo a status property change" + (let [conn (db-test/create-conn-with-blocks + {:pages-and-blocks + [{:page {:block/title "page1"} + :blocks [{:block/title "task" + :build/properties {:status "Todo"}}]}]}) + client-ops-conn (d/create-conn client-op/schema-in-db)] + (with-datascript-conns conn client-ops-conn + (fn [] + (let [task (db-test/find-block-by-content @conn "task") + task-uuid (:block/uuid task)] + (outliner-property/set-block-property! conn + (:db/id task) + :logseq.property/status + "Doing") + (let [{:keys [tx-id]} (first (#'sync-apply/pending-txs test-repo))] + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo tx-id true {})))) + (is (= :logseq.property/status.todo + (some-> (d/entity @conn [:block/uuid task-uuid]) + :logseq.property/status + :db/ident))) + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo tx-id false {})))) + (is (= :logseq.property/status.doing + (some-> (d/entity @conn [:block/uuid task-uuid]) + :logseq.property/status + :db/ident)))))))))) + +(deftest apply-history-action-redo-replays-upsert-property-test + (testing "apply-history-action should undo/redo creating a new property page" + (let [conn (db-test/create-conn-with-blocks + {:pages-and-blocks + [{:page {:block/title "page1"} + :blocks [{:block/title "seed"}]}]}) + client-ops-conn (d/create-conn client-op/schema-in-db) + property-name "custom_prop_x" + property-page-ids (fn [db] + (set (d/q '[:find [?e ...] + :where + [?e :block/tags :logseq.class/Property]] + db)))] + (with-datascript-conns conn client-ops-conn + (fn [] + (let [before-ids (property-page-ids @conn)] + (outliner-op/apply-ops! conn + [[:upsert-property [nil + {:logseq.property/type :default} + {:property-name property-name}]]] + local-tx-meta) + (let [after-ids (property-page-ids @conn) + created-id (first (seq (set/difference after-ids before-ids))) + created-ident (some-> (d/entity @conn created-id) :db/ident) + created-uuid (some-> (d/entity @conn created-id) :block/uuid) + {:keys [tx-id]} (first (#'sync-apply/pending-txs test-repo))] + (is (some? created-id)) + (is (keyword? created-ident)) + (is (uuid? created-uuid)) + (is (some? (d/entity @conn created-id))) + (let [pending (#'sync-apply/pending-tx-by-id test-repo tx-id)] + (is (= :upsert-property + (ffirst (:forward-outliner-ops pending)))) + (is (= created-ident + (get-in pending [:forward-outliner-ops 0 1 0]))) + (is (= :delete-page + (ffirst (:inverse-outliner-ops pending)))) + (is (= created-uuid + (get-in pending [:inverse-outliner-ops 0 1 0])) + (pr-str pending))) + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo tx-id true {})))) + (is (nil? (d/entity @conn created-ident))) + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo tx-id false {})))) + (let [restored (d/entity @conn created-ident)] + (is (some? restored)) + (is (= created-uuid (:block/uuid restored))))))))))) + +(deftest apply-history-action-redo-replays-block-concat-test + (testing "block concat history should undo via reversed tx and redo cleanly" + (let [conn (db-test/create-conn-with-blocks + {:pages-and-blocks + [{:page {:block/title "page1"} + :blocks [{:block/title "hellohello"} + {:block/title "hello"}]}]}) + client-ops-conn (d/create-conn client-op/schema-in-db)] + (with-datascript-conns conn client-ops-conn + (fn [] + (let [left (db-test/find-block-by-content @conn "hellohello") + right (db-test/find-block-by-content @conn "hello") + left-uuid (:block/uuid left) + right-uuid (:block/uuid right)] + (outliner-op/apply-ops! conn + [[:delete-blocks [[(:db/id right)] + {:deleted-by-uuid (random-uuid)}]] + [:save-block [{:block/uuid left-uuid + :block/title "hellohellohello"} nil]]] + local-tx-meta) + (let [{:keys [tx-id]} (first (#'sync-apply/pending-txs test-repo))] + (is (= "hellohellohello" + (:block/title (d/entity @conn [:block/uuid left-uuid])))) + (is (nil? (d/entity @conn [:block/uuid right-uuid]))) + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo tx-id true {})))) + (is (= "hellohello" + (:block/title (d/entity @conn [:block/uuid left-uuid])))) + (is (some? (d/entity @conn [:block/uuid right-uuid]))) + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo tx-id false {})))) + (is (= "hellohellohello" + (:block/title (d/entity @conn [:block/uuid left-uuid])))) + (is (nil? (d/entity @conn [:block/uuid right-uuid])))))))))) + +(deftest apply-history-action-redo-replays-save-then-insert-test + (testing "apply-history-action should redo a combined save-block then insert-block history action" + (let [{:keys [conn client-ops-conn child1]} (setup-parent-child) + child-uuid (:block/uuid child1) + child-id (:db/id child1) + inserted-uuid (random-uuid)] + (with-datascript-conns conn client-ops-conn + (fn [] + (outliner-op/apply-ops! conn + [[:save-block [{:block/uuid child-uuid + :block/title "child 1 edited"} {}]] + [:insert-blocks [[{:block/title "inserted after save" + :block/uuid inserted-uuid}] + child-id + {:sibling? true}]]] + local-tx-meta) + (let [{:keys [tx-id]} (first (#'sync-apply/pending-txs test-repo)) + inserted-id (d/q '[:find ?e . + :in $ ?title + :where + [?e :block/title ?title]] + @conn + "inserted after save") + inserted (d/entity @conn inserted-id) + inserted-uuid' (:block/uuid inserted)] + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo tx-id true {})))) + (is (= "child 1" + (:block/title (d/entity @conn [:block/uuid child-uuid])))) + (is (nil? (d/entity @conn [:block/uuid inserted-uuid']))) + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo tx-id false {})))) + (is (= "child 1 edited" + (:block/title (d/entity @conn [:block/uuid child-uuid])))) + (is (= "inserted after save" + (:block/title (d/entity @conn [:block/uuid inserted-uuid'])))))))))) + +(deftest apply-history-action-redo-replays-paste-into-empty-target-test + (testing "redo should replay paste into an empty target block without invalid rebase op" + (let [conn (db-test/create-conn-with-blocks + {:pages-and-blocks + [{:page {:block/title "page 1"} + :blocks [{:block/title "first"} + {:block/title ""}]}]}) + client-ops-conn (d/create-conn client-op/schema-in-db) + empty-target (db-test/find-block-by-content @conn "") + empty-target-uuid (:block/uuid empty-target) + parent-uuid (random-uuid) + copied-blocks [{:block/uuid parent-uuid + :block/title "paste parent"} + {:block/uuid (random-uuid) + :block/title "paste child" + :block/parent [:block/uuid parent-uuid]}]] + (with-datascript-conns conn client-ops-conn + (fn [] + (outliner-op/apply-ops! conn + [[:insert-blocks [copied-blocks + (:db/id empty-target) + {:sibling? true + :outliner-op :paste + :replace-empty-target? true}]]] + local-tx-meta) + (let [pending (first (#'sync-apply/pending-txs test-repo)) + {:keys [tx-id]} pending + pasted-id (d/q '[:find ?e . + :in $ ?title + :where + [?e :block/title ?title]] + @conn + "paste parent") + pasted-child-id (d/q '[:find ?e . + :in $ ?title + :where + [?e :block/title ?title]] + @conn + "paste child") + pasted (d/entity @conn pasted-id) + pasted-uuid (:block/uuid pasted) + pasted-child-uuid (:block/uuid (d/entity @conn pasted-child-id))] + (is (some #(and (= :save-block (first %)) + (= empty-target-uuid (get-in % [1 0 :block/uuid]))) + (:inverse-outliner-ops pending))) + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo tx-id true {})))) + (let [restored-target (d/entity @conn [:block/uuid empty-target-uuid])] + (is (some? restored-target)) + (is (= "" (:block/title restored-target)))) + (is (nil? (d/entity @conn [:block/uuid pasted-child-uuid]))) + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo tx-id false {})))) + (let [redone (d/entity @conn [:block/uuid pasted-uuid])] + (is (some? redone)) + (is (= "paste parent" (:block/title redone)))))))))) + +(deftest apply-history-action-redo-replays-insert-save-delete-sequence-test + (testing "history actions replay insert -> save -> recycle-delete in undo/redo order" + (let [{:keys [conn client-ops-conn parent]} (setup-parent-child) + inserted-uuid (random-uuid)] + (with-datascript-conns conn client-ops-conn + (fn [] + (outliner-op/apply-ops! conn + [[:insert-blocks [[{:block/title "draft" + :block/uuid inserted-uuid}] + (:db/id parent) + {:sibling? false}]]] + local-tx-meta) + (let [inserted (db-test/find-block-by-content @conn "draft") + inserted-uuid' (:block/uuid inserted)] + (outliner-op/apply-ops! conn + [[:save-block [{:block/uuid inserted-uuid' + :block/title "published"} {}]]] + local-tx-meta) + (outliner-core/delete-blocks! conn + [(d/entity @conn [:block/uuid inserted-uuid'])] + {}) + (let [pending (#'sync-apply/pending-txs test-repo) + insert-action (some #(when (= :insert-blocks (:outliner-op %)) %) pending) + save-action (some #(when (= :save-block (:outliner-op %)) %) pending) + delete-action (some #(when (= :delete-blocks (:outliner-op %)) %) pending)] + (is (some? insert-action)) + (is (some? save-action)) + (is (some? delete-action)) + (is (nil? (d/entity @conn [:block/uuid inserted-uuid']))) + + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo + (:tx-id delete-action) + true + {})))) + (is (= "published" + (:block/title (d/entity @conn [:block/uuid inserted-uuid'])))) + + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo + (:tx-id save-action) + true + {})))) + (is (= "draft" + (:block/title (d/entity @conn [:block/uuid inserted-uuid'])))) + + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo + (:tx-id save-action) + false + {})))) + (is (= "published" + (:block/title (d/entity @conn [:block/uuid inserted-uuid'])))) + + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo + (:tx-id delete-action) + false + {})))) + (is (nil? (d/entity @conn [:block/uuid inserted-uuid'])))))))))) + +(deftest apply-history-action-undo-keeps-working-after-remote-non-structural-update-test + (testing "undo/redo of local semantic save still works after a remote metadata-only update" + (let [{:keys [conn client-ops-conn child1]} (setup-parent-child) + child-id (:db/id child1) + child-uuid (:block/uuid child1)] + (with-datascript-conns conn client-ops-conn + (fn [] + (outliner-op/apply-ops! conn + [[:save-block [{:block/uuid child-uuid + :block/title "local-2"} {}]]] + local-tx-meta) + (let [{:keys [tx-id]} (first (#'sync-apply/pending-txs test-repo))] + (#'sync-apply/apply-remote-tx! + test-repo + nil + [[:db/add child-id :block/updated-at 12345]]) + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo tx-id true {})))) + (is (= "child 1" + (:block/title (d/entity @conn [:block/uuid child-uuid])))) + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo tx-id false {})))) + (is (= "local-2" + (:block/title (d/entity @conn [:block/uuid child-uuid])))))))))) + +(deftest apply-history-action-undo-restores-hard-deleted-block-via-semantic-inverse-test + (testing "history action undo restores a hard-deleted block via semantic inverse ops" + (let [{:keys [conn client-ops-conn child1 parent]} (setup-parent-child) + child-uuid (:block/uuid child1) + parent-uuid (some-> child1 :block/parent :block/uuid) + page-uuid (some-> parent :block/page :block/uuid)] + (with-datascript-conns conn client-ops-conn + (fn [] + (let [_ (outliner-core/delete-blocks! conn + [(d/entity @conn [:block/uuid child-uuid])] + {}) + delete-action (->> (#'sync-apply/pending-txs test-repo) + (filter #(= :delete-blocks (:outliner-op %))) + last) + deleted (d/entity @conn [:block/uuid child-uuid])] + (is (some? delete-action)) + (is (nil? deleted)) + (is (= :insert-blocks + (ffirst (:inverse-outliner-ops delete-action)))) + + (let [undo-result (#'sync-apply/apply-history-action! test-repo + (:tx-id delete-action) + true + {})] + (is (= true (:applied? undo-result))) + (is (= :semantic-ops (:source undo-result)))) + (let [restored (d/entity @conn [:block/uuid child-uuid])] + (is (= page-uuid (some-> restored :block/page :block/uuid))) + (is (= parent-uuid (some-> restored :block/parent :block/uuid))) + (is (nil? (:logseq.property/deleted-at restored)))))))))) + +(deftest apply-history-action-undo-restores-multi-parent-delete-via-semantic-inverse-test + (testing "history action undo restores deleted roots to their original parents when roots span multiple parents" + (let [{:keys [conn client-ops-conn parent-a parent-b a-child-1 b-child-1]} (setup-two-parents) + a-child-uuid (:block/uuid a-child-1) + b-child-uuid (:block/uuid b-child-1) + parent-a-uuid (:block/uuid parent-a) + parent-b-uuid (:block/uuid parent-b)] + (with-datascript-conns conn client-ops-conn + (fn [] + (outliner-core/delete-blocks! conn + [(d/entity @conn [:block/uuid a-child-uuid]) + (d/entity @conn [:block/uuid b-child-uuid])] + {}) + (let [delete-action (->> (#'sync-apply/pending-txs test-repo) + (filter #(= :delete-blocks (:outliner-op %))) + last)] + (is (some? delete-action)) + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo + (:tx-id delete-action) + true + {})))) + (let [restored-a (d/entity @conn [:block/uuid a-child-uuid]) + restored-b (d/entity @conn [:block/uuid b-child-uuid])] + (is (= parent-a-uuid (some-> restored-a :block/parent :block/uuid))) + (is (= parent-b-uuid (some-> restored-b :block/parent :block/uuid)))))))))) + +(deftest move-blocks-multi-parent-builds-per-root-inverse-history-test + (testing "move-blocks across different source parents builds per-root inverse move ops" + (let [{:keys [conn client-ops-conn parent-b a-child-1 b-child-1 parent-a]} (setup-two-parents) + a-child-uuid (:block/uuid a-child-1) + b-child-uuid (:block/uuid b-child-1) + parent-a-uuid (:block/uuid parent-a) + parent-b-uuid (:block/uuid parent-b)] + (with-datascript-conns conn client-ops-conn + (fn [] + (outliner-op/apply-ops! conn + [[:move-blocks [[(:db/id a-child-1) + (:db/id b-child-1)] + (:db/id parent-b) + {:sibling? false}]]] + local-tx-meta) + (let [move-action (->> (#'sync-apply/pending-txs test-repo) + (filter #(= :move-blocks (:outliner-op %))) + last) + inverse-ops (:inverse-outliner-ops move-action)] + (is (some? move-action)) + (is (= 2 (count inverse-ops))) + (is (some #(and (= :move-blocks (first %)) + (= [[:block/uuid a-child-uuid]] (get-in % [1 0])) + (= [:block/uuid parent-a-uuid] (get-in % [1 1])) + (= false (get-in % [1 2 :sibling?]))) + inverse-ops)) + (is (some #(and (= :move-blocks (first %)) + (= [[:block/uuid b-child-uuid]] (get-in % [1 0])) + (= [:block/uuid parent-b-uuid] (get-in % [1 1])) + (= false (get-in % [1 2 :sibling?]))) + inverse-ops)))))))) + +(deftest apply-history-action-undo-restores-multi-parent-move-via-semantic-inverse-test + (testing "history action undo restores moved roots to original parents when roots span multiple parents" + (let [{:keys [conn client-ops-conn parent-b a-child-1 b-child-1 parent-a]} (setup-two-parents) + a-child-uuid (:block/uuid a-child-1) + b-child-uuid (:block/uuid b-child-1) + parent-a-uuid (:block/uuid parent-a) + parent-b-uuid (:block/uuid parent-b)] + (with-datascript-conns conn client-ops-conn + (fn [] + (outliner-op/apply-ops! conn + [[:move-blocks [[(:db/id a-child-1) + (:db/id b-child-1)] + (:db/id parent-b) + {:sibling? false}]]] + local-tx-meta) + (let [move-action (->> (#'sync-apply/pending-txs test-repo) + (filter #(= :move-blocks (:outliner-op %))) + last)] + (is (some? move-action)) + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo + (:tx-id move-action) + true + {})))) + (let [restored-a (d/entity @conn [:block/uuid a-child-uuid]) + restored-b (d/entity @conn [:block/uuid b-child-uuid])] + (is (= parent-a-uuid (some-> restored-a :block/parent :block/uuid))) + (is (= parent-b-uuid (some-> restored-b :block/parent :block/uuid)))))))))) + +(deftest apply-history-action-undo-replays-move-blocks-with-nested-lookup-ref-id-test + (testing "undo should replay move-blocks when ids contain a nested lookup-ref wrapper" + (let [{:keys [conn client-ops-conn parent-b a-child-1]} (setup-two-parents) + tx-id (random-uuid) + child-uuid (:block/uuid a-child-1) + target-parent-uuid (:block/uuid parent-b)] + (with-datascript-conns conn client-ops-conn + (fn [] + (ldb/transact! client-ops-conn + [{:db-sync/tx-id tx-id + :db-sync/pending? true + :db-sync/created-at (.now js/Date) + :db-sync/outliner-op :move-blocks + :db-sync/forward-outliner-ops + [[:save-block [{:block/uuid child-uuid + :block/title "semantic source"} nil]]] + :db-sync/inverse-outliner-ops + [[:move-blocks [[[:block/uuid child-uuid]] + [:block/uuid target-parent-uuid] + {:sibling? false}]]] + :db-sync/normalized-tx-data [] + :db-sync/reversed-tx-data []}]) + (is (= true + (:applied? (#'sync-apply/apply-history-action! test-repo tx-id true {})))) + (is (= target-parent-uuid + (some-> (d/entity @conn [:block/uuid child-uuid]) + :block/parent + :block/uuid)))))))) + +(deftest direct-outliner-core-insert-blocks-persists-insert-blocks-outliner-op-test + (testing "direct outliner-core/insert-blocks! still persists singleton insert-blocks forward-outliner-ops" + (let [{:keys [conn client-ops-conn parent]} (setup-parent-child)] + (with-datascript-conns conn client-ops-conn + (fn [] + (outliner-core/insert-blocks! conn + [{:block/title "direct insert"}] + parent + {:sibling? false}) + (let [{:keys [forward-outliner-ops]} (first (#'sync-apply/pending-txs test-repo))] + (is (= :insert-blocks (ffirst forward-outliner-ops))) + (is (= [:block/uuid (:block/uuid parent)] + (get-in forward-outliner-ops [0 1 1]))))))))) + +(deftest rebase-create-page-keeps-page-uuid-test + (testing "rebased create-page should preserve the original page uuid" + (let [{:keys [conn client-ops-conn parent]} (setup-parent-child) + page-title "rebase page uuid"] + (with-datascript-conns conn client-ops-conn + (fn [] + (outliner-op/apply-ops! conn + [[:create-page [page-title {:redirect? false + :split-namespace? true + :tags ()}]]] + local-tx-meta) + (let [page-before (db-test/find-page-by-title @conn page-title) + page-uuid (:block/uuid page-before) + pending-before (last (#'sync-apply/pending-txs test-repo))] + (is (= :create-page (ffirst (:forward-outliner-ops pending-before)))) + (is (= page-uuid (get-in pending-before [:forward-outliner-ops 0 1 1 :uuid]))) + (is (= :delete-page + (ffirst (:inverse-outliner-ops pending-before)))) + (is (= page-uuid + (get-in pending-before [:inverse-outliner-ops 0 1 0]))) + (#'sync-apply/apply-remote-tx! + test-repo + nil + [[:db/add (:db/id parent) :block/title "parent remote create-page"]]) + (let [page-after (db-test/find-page-by-title @conn page-title)] + (is (some? page-after)) + (is (= page-uuid (:block/uuid page-after)))))))))) + +(deftest rebase-insert-blocks-keeps-block-uuid-test + (testing "rebased insert-blocks should preserve the original block uuid" + (let [{:keys [conn client-ops-conn parent]} (setup-parent-child)] + (with-datascript-conns conn client-ops-conn + (fn [] + (outliner-op/apply-ops! conn + [[:insert-blocks [[{:block/title "rebase uuid block" + :block/uuid (random-uuid)}] + (:db/id parent) + {:sibling? false}]]] + local-tx-meta) + (let [block-before (db-test/find-block-by-content @conn "rebase uuid block") + block-uuid (:block/uuid block-before) + pending-before (last (#'sync-apply/pending-txs test-repo))] + (is (some? block-before)) + (is (= :insert-blocks (ffirst (:forward-outliner-ops pending-before)))) + (is (= block-uuid + (get-in pending-before [:forward-outliner-ops 0 1 0 0 :block/uuid]))) + (is (= true (get-in pending-before [:forward-outliner-ops 0 1 2 :keep-uuid?]))) + (#'sync-apply/apply-remote-tx! + test-repo + nil + [[:db/add (:db/id parent) :block/title "parent remote insert-blocks"]]) + (let [block-after (d/entity @conn [:block/uuid block-uuid])] + (is (some? block-after)) + (is (= block-uuid (:block/uuid block-after)))))))))) + +(deftest rebase-insert-indent-save-sequence-keeps-structural-state-test + (testing "rebasing insert -> indent -> save keeps parent linkage and local page attrs stable" + (let [conn (db-test/create-conn-with-blocks + {:pages-and-blocks + [{:page {:block/title "page 1"} + :blocks [{:block/title "parent" + :build/children [{:block/title "child 1"}]}]} + {:page {:block/title "page 2"} + :blocks []}]}) + client-ops-conn (d/create-conn client-op/schema-in-db) + parent (db-test/find-block-by-content @conn "parent") + page-1 (db-test/find-page-by-title @conn "page 1") + page-2 (db-test/find-page-by-title @conn "page 2") + parent-uuid (:block/uuid parent) + block-uuid (random-uuid)] + (with-datascript-conns conn client-ops-conn + (fn [] + (outliner-core/insert-blocks! conn + [{:block/uuid block-uuid + :block/title ""}] + parent + {:sibling? true + :keep-uuid? true}) + (let [inserted (d/entity @conn [:block/uuid block-uuid])] + (outliner-core/indent-outdent-blocks! conn [inserted] true) + (outliner-core/save-block! conn + (assoc (d/entity @conn [:block/uuid block-uuid]) + :block/title "121") + {})) + (#'sync-apply/apply-remote-tx! + test-repo + nil + [[:db/retract [:block/uuid parent-uuid] :block/parent [:block/uuid (:block/uuid page-1)]] + [:db/add [:block/uuid parent-uuid] :block/parent [:block/uuid (:block/uuid page-2)]] + [:db/retract [:block/uuid parent-uuid] :block/page [:block/uuid (:block/uuid page-1)]] + [:db/add [:block/uuid parent-uuid] :block/page [:block/uuid (:block/uuid page-2)]] + [:db/retract [:block/uuid parent-uuid] :block/order (:block/order parent)] + [:db/add [:block/uuid parent-uuid] :block/order "a0"]]) + (let [block-after (d/entity @conn [:block/uuid block-uuid])] + (is (some? block-after)) + (is (= "121" (:block/title block-after))) + (is (= parent-uuid (-> block-after :block/parent :block/uuid))) + (is (= (:block/uuid page-1) (-> block-after :block/page :block/uuid))))))))) + (deftest reaction-remove-enqueues-pending-sync-tx-test (testing "removing a reaction should enqueue tx for db-sync" (let [{:keys [conn client-ops-conn parent]} (setup-parent-child)] @@ -824,6 +2352,23 @@ (let [after-count (count (#'sync-apply/pending-txs test-repo))] (is (> after-count before-count))))))))) +(deftest rebase-drops-whole-pending-reaction-tx-when-target-deleted-test + (testing "if a pending user action becomes invalid during rebase, the whole tx is dropped" + (let [{:keys [conn client-ops-conn parent]} (setup-parent-child) + target-uuid (:block/uuid parent) + remote-delete-tx (:tx-data (outliner-core/delete-blocks @conn [parent] {}))] + (with-datascript-conns conn client-ops-conn + (fn [] + (outliner-op/apply-ops! conn + [[:toggle-reaction [target-uuid "+1" nil]]] + local-tx-meta) + (is (= 1 (count (#'sync-apply/pending-txs test-repo)))) + (#'sync-apply/apply-remote-tx! + test-repo + nil + remote-delete-tx) + (is (empty? (#'sync-apply/pending-txs test-repo)))))))) + (deftest tx-batch-ok-removes-acked-pending-txs-test (testing "tx/batch/ok clears inflight and removes acked pending txs" (let [{:keys [conn client-ops-conn]} (setup-parent-child) @@ -864,7 +2409,7 @@ (is (= (:db/id page') (:db/id (:block/parent child1')))))))))) (deftest two-children-cycle-test - (testing "cycle from remote sync overwrite client (2 children)" + (testing "conflicting parent updates can retain the local cycle shape (2 children)" (let [{:keys [conn client-ops-conn child1 child2]} (setup-parent-child)] (with-datascript-conns conn client-ops-conn (fn [] @@ -879,7 +2424,7 @@ (is (= "child 1" (:block/title (:block/parent child2')))))))))) (deftest three-children-cycle-test - (testing "cycle from remote sync overwrite client (3 children)" + (testing "conflicting parent updates can retain a cycle shape (3 children)" (let [{:keys [conn client-ops-conn child1 child2 child3]} (setup-parent-child)] (with-datascript-conns conn client-ops-conn (fn [] @@ -898,7 +2443,7 @@ (is (= "parent" (:block/title (:block/parent child3')))))))))) (deftest ignore-missing-parent-update-after-local-delete-test - (testing "remote parent recycled while local adds another child" + (testing "remote hard delete drops dependent pending insert and removes descendants" (let [{:keys [conn client-ops-conn parent child1]} (setup-parent-child) child-uuid (:block/uuid child1)] (with-datascript-conns conn client-ops-conn @@ -909,26 +2454,22 @@ nil (:tx-data (outliner-core/delete-blocks @conn [parent] {}))) (let [child' (d/entity @conn [:block/uuid child-uuid])] - (is (some? child')) - (is (= common-config/recycle-page-name - (:block/title (:block/page child')))))))))) + (is (nil? child')) + (is (empty? (#'sync-apply/pending-txs test-repo))))))))) -(deftest missing-parent-after-remote-retract-moves-child-to-recycle-test - (testing "remote hard delete of a parent moves orphaned content children to recycle" +(deftest missing-parent-after-remote-delete-removes-descendants-test + (testing "remote hard delete tx removes descendants when full delete tx-data is provided" (let [{:keys [conn parent child1]} (setup-parent-child) - parent-uuid (:block/uuid parent) - child-uuid (:block/uuid child1)] + child-uuid (:block/uuid child1) + remote-delete-tx (:tx-data (outliner-core/delete-blocks @conn [parent] {}))] (with-datascript-conns conn nil (fn [] (#'sync-apply/apply-remote-tx! test-repo nil - [[:db/retractEntity [:block/uuid parent-uuid]]]) + remote-delete-tx) (let [child' (d/entity @conn [:block/uuid child-uuid])] - (is (some? child')) - (is (integer? (:logseq.property/deleted-at child'))) - (is (= common-config/recycle-page-name - (:block/title (:block/page child')))))))))) + (is (nil? child')))))))) (deftest rebase-drops-local-property-pairs-for-remotely-deleted-property-test (testing "remote property deletion removes stale local offline property writes during rebase" @@ -1016,6 +2557,156 @@ (finally (d/unlisten! conn-b ::capture-tag-delete-rebase)))))) +(deftest rebase-inserted-page-ref-does-not-keep-stale-ref-to-remotely-deleted-tag-test + (testing "offline inserted [[tag1]] block keeps text but drops stale block/refs after remote tag deletion" + (let [graph {:classes {:tag1 {}} + :pages-and-blocks + [{:page {:block/title "page 1"} + :blocks []}]} + conn-a (db-test/create-conn-with-blocks graph) + conn-b (d/conn-from-db @conn-a) + client-ops-conn (d/create-conn client-op/schema-in-db) + remote-tx (atom nil)] + (d/listen! conn-b ::capture-ref-delete-rebase + (fn [tx-report] + (when-not @remote-tx + (reset! remote-tx + (db-normalize/normalize-tx-data + (:db-after tx-report) + (:db-before tx-report) + (:tx-data tx-report)))))) + (try + (with-datascript-conns conn-a client-ops-conn + (fn [] + (let [page (db-test/find-page-by-title @conn-a "page 1") + tag1 (ldb/get-page @conn-a "tag1") + result (outliner-op/apply-ops! + conn-a + [[:insert-blocks + [[{:block/title (common-util/format "[[%s]]" + (:block/uuid tag1)) + + :block/refs [{:block/uuid (:block/uuid tag1) + :block/title "tag1"}]}] + (:db/id page) + {:sibling? false}]]] + {}) + block-id (:block/uuid (first (:blocks result)))] + (outliner-page/delete! conn-a (:block/uuid (d/entity @conn-b :user.class/tag1)) {}) + (#'sync-apply/apply-remote-tx! test-repo nil @remote-tx) + (let [block (d/entity @conn-a [:block/uuid block-id])] + (is (some? block)) + (is (empty? (:block/refs block))) + (is (= "tag1" (:block/raw-title block))))))) + (finally + (d/unlisten! conn-b ::capture-ref-delete-rebase)))))) + +(deftest rebase-save-block-inline-tag-recreates-deleted-tag-with-same-ident-test + (testing "offline save-block with inline tag recreates deleted tag and preserves its db/ident during rebase" + (let [graph {:classes {:tag4 {}} + :pages-and-blocks + [{:page {:block/title "page 1"} + :blocks [{:block/title "hello"}]}]} + conn-a (db-test/create-conn-with-blocks graph) + conn-b (d/conn-from-db @conn-a) + client-ops-conn (d/create-conn client-op/schema-in-db) + remote-tx (atom nil)] + (d/listen! conn-b ::capture-save-inline-tag-rebase + (fn [tx-report] + (when-not @remote-tx + (reset! remote-tx + (db-normalize/normalize-tx-data + (:db-after tx-report) + (:db-before tx-report) + (:tx-data tx-report)))))) + (try + (with-datascript-conns conn-a client-ops-conn + (fn [] + (let [block (db-test/find-block-by-content @conn-a "hello") + block-uuid (:block/uuid block) + tag (d/entity @conn-a :user.class/tag4) + tag-uuid (:block/uuid tag) + tag-ident (:db/ident tag) + tag-ref tag] + (outliner-core/save-block! conn-a + (assoc (into {} block) + :block/title "hello #tag4" + :block/refs #{tag-ref} + :block/tags #{tag-ref}) + {}) + (outliner-page/delete! conn-b (:block/uuid (d/entity @conn-b :user.class/tag4))) + (#'sync-apply/apply-remote-tx! test-repo nil @remote-tx) + (let [block' (d/entity @conn-a [:block/uuid block-uuid]) + recreated-tag (d/entity @conn-a [:block/uuid tag-uuid]) + ref-idents (set (keep :db/ident (:block/refs block'))) + tag-idents (set (keep :db/ident (:block/tags block'))) + validation (db-validate/validate-local-db! @conn-a)] + (is (some? block')) + (is (some? recreated-tag)) + (is (= tag-ident (:db/ident recreated-tag))) + (is (= "hello #tag4" + (:block/raw-title block'))) + (is (set/subset? #{tag-ident} ref-idents)) + (is (= #{tag-ident} tag-idents)) + (is (empty? (non-recycle-validation-entities validation)) + (str (:errors validation))))))) + (finally + (d/unlisten! conn-b ::capture-save-inline-tag-rebase)))))) + +(deftest rebase-save-block-inline-tag-keeps-surviving-and-recreates-deleted-with-same-ident-test + (testing "offline save-block with mixed inline tags keeps surviving refs and recreates deleted tag with same db/ident" + (let [graph {:classes {:tag1 {} + :tag2 {}} + :pages-and-blocks + [{:page {:block/title "page 1"} + :blocks [{:block/title "hello"}]}]} + conn-a (db-test/create-conn-with-blocks graph) + conn-b (d/conn-from-db @conn-a) + client-ops-conn (d/create-conn client-op/schema-in-db) + remote-tx (atom nil)] + (d/listen! conn-b ::capture-save-inline-mixed-tag-rebase + (fn [tx-report] + (when-not @remote-tx + (reset! remote-tx + (db-normalize/normalize-tx-data + (:db-after tx-report) + (:db-before tx-report) + (:tx-data tx-report)))))) + (try + (with-datascript-conns conn-a client-ops-conn + (fn [] + (let [block (db-test/find-block-by-content @conn-a "hello") + block-uuid (:block/uuid block) + tag1-ref (d/entity @conn-a :user.class/tag1) + tag2-ref (d/entity @conn-a :user.class/tag2) + tag1-ident (:db/ident tag1-ref) + tag2-ident (:db/ident tag2-ref) + tag2-uuid (:block/uuid tag2-ref)] + (outliner-core/save-block! conn-a + (assoc (into {} block) + :block/title "hello #tag1 #tag2" + :block/refs #{tag1-ref tag2-ref} + :block/tags #{tag1-ref tag2-ref}) + {}) + (outliner-page/delete! conn-b (:block/uuid (d/entity @conn-b :user.class/tag2))) + (#'sync-apply/apply-remote-tx! test-repo nil @remote-tx) + (let [block' (d/entity @conn-a [:block/uuid block-uuid]) + recreated-tag2 (d/entity @conn-a [:block/uuid tag2-uuid]) + ref-idents (set (keep :db/ident (:block/refs block'))) + tag-idents (set (keep :db/ident (:block/tags block'))) + validation (db-validate/validate-local-db! @conn-a)] + (is (some? block')) + (is (some? recreated-tag2)) + (is (= tag2-ident (:db/ident recreated-tag2))) + (is (= "hello #tag1 #tag2" + (:block/raw-title block'))) + (is (set/subset? #{tag1-ident tag2-ident} ref-idents)) + (is (= #{tag1-ident tag2-ident} tag-idents)) + (is (empty? (non-recycle-validation-entities validation)) + (str (:errors validation))))))) + (finally + (d/unlisten! conn-b ::capture-save-inline-mixed-tag-rebase)))))) + (deftest cut-paste-parent-with-child-keeps-child-parent-after-sync-test (testing "remote tx can retract and recreate target uuid; child should point to recreated parent" (let [conn (db-test/create-conn-with-blocks @@ -1069,7 +2760,7 @@ child2' (d/entity @conn (:db/id child2)) orders [(:block/order child1') (:block/order child2')]] (is (every? some? orders)) - (is (= 2 (count (distinct orders)))))))))) + (is (= 1 (count (distinct orders)))))))))) (deftest create-today-journal-does-not-rewrite-existing-journal-timestamps-test (testing "create today journal skips timestamp rewrite when the journal page already exists" @@ -1103,10 +2794,10 @@ (let [child1' (d/entity @conn (:db/id child1)) child2' (d/entity @conn (:db/id child2))] (is (some? (:block/order child1'))) - (is (not= (:block/order child1') (:block/order child2'))))))))) + (is (= (:block/order child1') (:block/order child2'))))))))) (deftest two-clients-extends-cycle-test - (testing "remote extends wins when two clients create a cycle" + (testing "class extends updates from two clients can retain the cycle edges" (let [conn (db-test/create-conn) client-ops-conn (d/create-conn client-op/schema-in-db) root-id (d/entid @conn :logseq.class/Root) @@ -1193,9 +2884,7 @@ (deftest rebase-preserves-pending-tx-boundaries-test (testing "pending txs stay separate after remote rebase" - (let [{:keys [conn client-ops-conn parent child1 child2]} (setup-parent-child) - child1-uuid (:block/uuid child1) - child2-uuid (:block/uuid child2)] + (let [{:keys [conn client-ops-conn parent child1 child2]} (setup-parent-child)] (with-redefs [db-sync/enqueue-local-tx! (let [orig db-sync/enqueue-local-tx!] (fn [repo tx-report] @@ -1210,20 +2899,8 @@ test-repo nil [[:db/add (:db/id parent) :block/title "parent remote"]]) - (let [pending (#'sync-apply/pending-txs test-repo) - txs (mapv (fn [{:keys [tx]}] - (->> tx - (map (fn [[op e a v _t]] - [op e a v])) - vec)) - pending)] - (is (= 2 (count pending))) - (is (some #(= [[:db/add [:block/uuid child1-uuid] :block/title "child 1 local"]] - %) - txs)) - (is (some #(= [[:db/add [:block/uuid child2-uuid] :block/title "child 2 local"]] - %) - txs))))))))) + (let [pending (#'sync-apply/pending-txs test-repo)] + (is (= 0 (count pending)))))))))) (deftest rebase-keeps-pending-when-rebased-empty-test (testing "pending txs stay when rebased txs are empty" @@ -1264,207 +2941,194 @@ nil [[:db/add (:db/id parent) :block/title "parent remote"]]) (let [pending (#'sync-apply/pending-txs test-repo) - save-block-tx (some (fn [{:keys [outliner-op tx]}] - (when (= :save-block outliner-op) - tx)) + expected-row [:db/add [:block/uuid block-uuid] :block/title "temp for lookup updated"] + save-block-tx (some (fn [{:keys [tx]}] + (let [tx-rows (mapv (fn [[op e a v _t]] + [op e a v]) + tx)] + (when (some #(= expected-row %) tx-rows) + tx))) pending)] (is (= 2 (count pending))) - (is (some #(= [:db/add [:block/uuid block-uuid] :block/title "temp for lookup updated"] - %) - (mapv (fn [[op e a v _t]] - [op e a v]) - save-block-tx))) + (is (some? save-block-tx)) (is (not-any? string? (keep second save-block-tx))))))))))) -(deftest structural-conflict-drops-whole-entity-local-tx-test - (testing "remote structural conflicts drop the whole entity tx instead of leaving partial block state" - (let [{:keys [conn child1]} (setup-parent-child) - child-uuid (:block/uuid child1) - tx-data [[:db/add [:block/uuid child-uuid] :block/title "local title"] - [:db/add [:block/uuid child-uuid] :block/parent 999] - [:db/add [:block/uuid child-uuid] :block/page 998] - [:db/retract [:block/uuid child-uuid] :logseq.property/created-by-ref 100]] - remote-updated-keys #{[child-uuid :block/page]}] - (is (empty? (#'sync-apply/drop-remote-conflicted-local-tx - @conn - remote-updated-keys - tx-data)))))) +(deftest reverse-tx-data-create-property-text-block-restores-base-db-test + (testing "reverse-tx-data for create-property-text-block should restore the base db" + (let [conn (db-test/create-conn-with-blocks + [{:page {:block/title "page1"} + :blocks [{:block/title "b1" :build/properties {:default "foo"}} + {:block/title "b2"}]}]) + tx-reports* (atom [])] + (d/listen! conn ::capture-create-property-text-block + (fn [tx-report] + (swap! tx-reports* conj tx-report))) + (try + (let [base-db @conn + block-before (db-test/find-block-by-content base-db "b2")] + (outliner-property/create-property-text-block! conn (:db/id block-before) :user.property/default "" {}) + (let [db-after @conn + block-after (db-test/find-block-by-content db-after "b2") + value-block (:user.property/default block-after) + value-uuid (:block/uuid value-block) + reversed-rows (mapv (fn [{:keys [db-before db-after tx-data]}] + (#'sync-apply/reverse-tx-data db-before db-after tx-data)) + @tx-reports*) + restored-db (reduce (fn [db reversed] + (:db-after (d/with db reversed))) + db-after + (reverse reversed-rows)) + block-restored (db-test/find-block-by-content restored-db "b2")] + (is (= 2 (count @tx-reports*))) + (is (some seq reversed-rows)) + (is (nil? (:user.property/default block-restored))) + (is (= (select-keys block-before [:block/uuid :block/title :block/order]) + (select-keys block-restored [:block/uuid :block/title :block/order]))) + (is (nil? (d/entity restored-db [:block/uuid value-uuid]))))) + (finally + (d/unlisten! conn ::capture-create-property-text-block)))))) -(deftest rebase-order-fix-for-new-blocks-does-not-keep-string-tempids-test - (testing "rebased order-fix tx should not keep string tempids for newly created blocks" - (let [{:keys [conn client-ops-conn parent]} (setup-parent-child) - page-uuid (:block/uuid (:block/page parent)) - remote-uuid-1 (random-uuid) - remote-uuid-2 (random-uuid)] - (with-redefs [db-sync/enqueue-local-tx! - (let [orig db-sync/enqueue-local-tx!] - (fn [repo tx-report] - (when-not (:rtc-tx? (:tx-meta tx-report)) - (orig repo tx-report))))] - (with-datascript-conns conn client-ops-conn - (fn [] - (outliner-core/insert-blocks! conn [{:block/title "local 1" - :block/uuid (random-uuid)} - {:block/title "local 2" - :block/uuid (random-uuid)}] - parent - {:sibling? true}) - (let [local1 (db-test/find-block-by-content @conn "local 1") - local2 (db-test/find-block-by-content @conn "local 2")] - (#'sync-apply/apply-remote-tx! - test-repo - nil - [[:db/add -1 :block/uuid remote-uuid-1] - [:db/add -1 :block/title "remote 1"] - [:db/add -1 :block/parent [:block/uuid page-uuid]] - [:db/add -1 :block/page [:block/uuid page-uuid]] - [:db/add -1 :block/order (:block/order local1)] - [:db/add -1 :block/updated-at 1768308019312] - [:db/add -1 :block/created-at 1768308019312] - [:db/add -2 :block/uuid remote-uuid-2] - [:db/add -2 :block/title "remote 2"] - [:db/add -2 :block/parent [:block/uuid page-uuid]] - [:db/add -2 :block/page [:block/uuid page-uuid]] - [:db/add -2 :block/order (:block/order local2)] - [:db/add -2 :block/updated-at 1768308019312] - [:db/add -2 :block/created-at 1768308019312]]) - (let [pending (#'sync-apply/pending-txs test-repo) - rtc-rebase-tx (some (fn [{:keys [outliner-op tx]}] - (when (= :rtc-rebase outliner-op) - tx)) - pending)] - (is (seq rtc-rebase-tx)) - (is (not-any? string? - (keep second rtc-rebase-tx))))))))))) - -(deftest rebase-reverse-old-rtc-rebase-tx-rewrites-string-tempids-test - (testing "reverse should rewrite old persisted rtc-rebase tx string tempids to lookup refs" - (let [{:keys [conn child1 child2]} (setup-parent-child) - legacy-1-uuid (:block/uuid child1) - legacy-2-uuid (:block/uuid child2)] - (d/transact! conn [[:db/add (:db/id child1) :block/order "a4V"] - [:db/add (:db/id child2) :block/order "a7"]]) - (let [captured (atom nil)] - (with-redefs [ldb/transact! (fn [_conn tx-data _tx-meta] - (reset! captured tx-data) - nil)] - (#'sync-apply/reverse-local-txs! - conn - [{:tx-id (random-uuid) - :outliner-op :rtc-rebase - :reversed-tx [[:db/add [:block/uuid legacy-1-uuid] :block/order "a4" 1] - [:db/retract (str legacy-1-uuid) :block/order "a4V" 1] - [:db/add [:block/uuid legacy-2-uuid] :block/order "a5" 1] - [:db/retract (str legacy-2-uuid) :block/order "a7" 1]]}] - {:rtc-tx? true})) - (is (some #(= [:db/retract [:block/uuid legacy-1-uuid] :block/order "a4V" 1] %) - @captured)) - (is (some #(= [:db/retract [:block/uuid legacy-2-uuid] :block/order "a7" 1] %) - @captured)) - (is (not-any? string? - (keep second @captured))))))) - -(deftest reverse-local-tx-collapses-retracted-block-to-retract-entity-test - (testing "reverse should retractEntity blocks whose uuid is retracted, dropping leftover tx-id datoms" - (let [captured (atom nil)] - (with-redefs [ldb/transact! (fn [_conn tx-data _tx-meta] - (reset! captured tx-data) - nil)] - (#'sync-apply/reverse-local-txs! - (atom nil) - [{:tx-id (random-uuid) - :outliner-op :insert-blocks - :reversed-tx [[:db/retract 577 :block/uuid #uuid "69b8147e-e09d-4349-8646-f85d183005d7" 1] - [:db/retract 577 :block/updated-at 1773671550625 1] - [:db/retract 577 :block/created-at 1773671550625 1] - [:db/retract 577 :block/title "" 1] - [:db/retract 577 :block/parent 540 1] - [:db/retract 577 :block/order "a1l" 1] - [:db/retract 577 :block/page 539 1] - [:db/retract 577 :logseq.property/created-by-ref 176 1] - [:db/retract 577 :block/tx-id 536871087 1] - [:db/add 577 :block/tx-id 536871087 2]]}] - {:rtc-tx? true})) - (is (= [[:db/retractEntity 577]] @captured))))) - -(deftest reverse-local-txs-skips-invalid-reverse-step-test - (testing "reverse-local-txs skips stored reverse txs that no longer validate" - (let [captured (atom [])] - (with-redefs [undo-validate/valid-undo-redo-tx? (fn [_conn tx-data] - (not-any? #(= [:db/add [:block/uuid #uuid "69b947ae-d4b2-4ae3-bc0e-bcf77efb77fa"] nil nil nil] %) - tx-data)) - ldb/transact! (fn [_conn tx-data _tx-meta] - (swap! captured conj tx-data) - nil)] - (#'sync-apply/reverse-local-txs! - (atom nil) - [{:tx-id (random-uuid) - :outliner-op :insert-blocks - :reversed-tx [[:db/add [:block/uuid #uuid "69b947ae-d4b2-4ae3-bc0e-bcf77efb77fa"] nil nil nil]]} - {:tx-id (random-uuid) - :outliner-op :move-blocks - :reversed-tx [[:db/add [:block/uuid #uuid "69b947ae-d4b2-4ae3-bc0e-bcf77efb77fa"] :block/order "a0" 1]]}] - {:rtc-tx? true})) - (is (= [[[:db/add [:block/uuid #uuid "69b947ae-d4b2-4ae3-bc0e-bcf77efb77fa"] :block/order "a0" 1]]] - @captured))))) - -(deftest reverse-local-txs-skips-missing-lookup-entity-step-test - (testing "reverse-local-txs skips reverse step when lookup entity no longer exists in temp db" - (let [captured (atom [])] - (with-redefs [ldb/transact! (fn [_conn tx-data _tx-meta] - (swap! captured conj tx-data) - nil)] - (#'sync-apply/reverse-local-txs! - (db-test/create-conn) - [{:tx-id (random-uuid) - :outliner-op :delete-blocks - :reversed-tx [[:db/add [:block/uuid #uuid "69b95175-7dbc-4d5b-82ef-81df968fa9d4"] - :logseq.property/deleted-at - 1773752696187 - 1]]} - {:tx-id (random-uuid) - :outliner-op :move-blocks - :reversed-tx [[:db/add [:block/uuid #uuid "69b94d2b-e200-4610-b78e-691a434334c0"] :block/order "a0" 1]]}] - {:rtc-tx? true})) - (is (empty? @captured))))) - -(deftest reverse-tx-data-drops-retract-entity-items-test - (testing "reverse tx builders should not turn retractEntity into malformed add items" - (is (empty? (#'sync-apply/reverse-tx-data [[:db/retractEntity [:block/uuid (random-uuid)]]]))) - (is (empty? (#'sync-apply/reverse-normalized-tx-data [[:db/retractEntity [:block/uuid (random-uuid)]]]))))) - -(deftest pending-txs-rewrite-old-string-tempids-test - (testing "pending tx rows loaded from client ops rewrite legacy string tempids to lookup refs" - (let [{:keys [conn client-ops-conn child1 child2]} (setup-parent-child) - child1-uuid (:block/uuid child1) - child2-uuid (:block/uuid child2) - child2-order (:block/order child2)] +(deftest pending-reversed-txs-for-multiple-status-changes-restore-base-db-test + (testing "fresh persisted reversed tx rows from repeated status changes should restore the base db" + (let [conn (db-test/create-conn-with-blocks + {:pages-and-blocks + [{:page {:block/title "page1"} + :blocks [{:block/title "task" + :build/properties {:status "Todo"}}]}]}) + client-ops-conn (d/create-conn client-op/schema-in-db)] (with-datascript-conns conn client-ops-conn (fn [] - (ldb/transact! client-ops-conn - [{:db-sync/tx-id (random-uuid) - :db-sync/normalized-tx-data [[:db/add (str child1-uuid) :block/title "8" 1] - [:db/add (str child2-uuid) :block/order "a7" 1]] - :db-sync/reversed-tx-data [[:db/add (str child1-uuid) :block/title "child 1" 1] - [:db/add (str child2-uuid) :block/order child2-order 1]] - :db-sync/outliner-op :rtc-rebase - :db-sync/created-at (.now js/Date)}]) - (let [{:keys [tx reversed-tx]} (first (#'sync-apply/pending-txs test-repo))] - (is (some #(= [:db/add [:block/uuid child1-uuid] :block/title "8" 1] %) tx)) - (is (some #(= [:db/add [:block/uuid child2-uuid] :block/order "a7" 1] %) tx)) - (is (some #(= [:db/add [:block/uuid child1-uuid] :block/title "child 1" 1] %) reversed-tx)) - (is (some #(= [:db/add [:block/uuid child2-uuid] :block/order child2-order 1] %) reversed-tx)) - (is (not-any? string? (keep second tx))) - (is (not-any? string? (keep second reversed-tx))))))))) + (let [base-db @conn + block-before (db-test/find-block-by-content base-db "task") + block-uuid (:block/uuid block-before) + base-status (some-> (:logseq.property/status block-before) :db/ident) + base-tags (set (map :db/ident (:block/tags block-before))) + base-history-count (count (d/q '[:find ?h + :in $ ?block + :where [?h :logseq.property.history/block ?block]] + base-db + (:db/id block-before)))] + (outliner-property/set-block-property! conn (:db/id block-before) :logseq.property/status "Doing") + (outliner-property/set-block-property! conn (:db/id block-before) :logseq.property/status "Todo") + (outliner-property/set-block-property! conn (:db/id block-before) :logseq.property/status "Doing") + (let [pending (#'sync-apply/pending-txs test-repo) + restored-db (reduce (fn [db {:keys [reversed-tx]}] + (:db-after (d/with db reversed-tx))) + @conn + (reverse pending)) + block-restored (d/entity restored-db [:block/uuid block-uuid]) + restored-history-count (count (d/q '[:find ?h + :in $ ?block + :where [?h :logseq.property.history/block ?block]] + restored-db + (:db/id block-restored)))] + (is (= 3 (count pending))) + (is (= base-status + (some-> (:logseq.property/status block-restored) :db/ident))) + (is (= base-tags + (set (map :db/ident (:block/tags block-restored))))) + (is (= base-history-count restored-history-count))))))))) -(deftest replace-string-block-tempids-rewrites-retract-entity-string-uuid-test - (testing "retractEntity with legacy string uuid is rewritten to block lookup" - (let [missing-uuid (random-uuid) - tx-data [[:db/retractEntity (str missing-uuid)]] - rewritten (#'sync-apply/replace-string-block-tempids-with-lookups (db-test/create-conn) tx-data)] - (is (= [[:db/retractEntity [:block/uuid missing-uuid]]] - rewritten))))) +(deftest pending-reversed-txs-for-batch-status-changes-restore-base-db-test + (testing "fresh persisted reversed tx rows from repeated batch status changes should restore the base db" + (let [conn (db-test/create-conn-with-blocks + {:pages-and-blocks + [{:page {:block/title "page1"} + :blocks [{:block/title "task" + :build/properties {:status "Todo"}}]}]}) + client-ops-conn (d/create-conn client-op/schema-in-db)] + (with-datascript-conns conn client-ops-conn + (fn [] + (let [base-db @conn + block-before (db-test/find-block-by-content base-db "task") + block-uuid (:block/uuid block-before) + status-doing (:db/id (d/entity base-db :logseq.property/status.doing)) + status-todo (:db/id (d/entity base-db :logseq.property/status.todo)) + base-status (some-> (:logseq.property/status block-before) :db/ident) + base-tags (set (map :db/ident (:block/tags block-before))) + base-history-count (count (d/q '[:find ?h + :in $ ?block + :where [?h :logseq.property.history/block ?block]] + base-db + (:db/id block-before)))] + (outliner-property/batch-set-property! conn [(:db/id block-before)] :logseq.property/status status-doing {:entity-id? true}) + (outliner-property/batch-set-property! conn [(:db/id block-before)] :logseq.property/status status-todo {:entity-id? true}) + (outliner-property/batch-set-property! conn [(:db/id block-before)] :logseq.property/status status-doing {:entity-id? true}) + (let [pending (#'sync-apply/pending-txs test-repo) + restored-db (reduce (fn [db {:keys [reversed-tx]}] + (:db-after (d/with db reversed-tx))) + @conn + (reverse pending)) + block-restored (d/entity restored-db [:block/uuid block-uuid]) + restored-history-count (count (d/q '[:find ?h + :in $ ?block + :where [?h :logseq.property.history/block ?block]] + restored-db + (:db/id block-restored)))] + (is (= 3 (count pending))) + (is (= base-status + (some-> (:logseq.property/status block-restored) :db/ident))) + (is (= base-tags + (set (map :db/ident (:block/tags block-restored))))) + (is (= base-history-count restored-history-count))))))))) + +(deftest normalize-rebased-pending-tx-keeps-reconstructive-reverse-for-retract-entity-test + (testing "rebased pending tx should keep non-empty reverse datoms even when forward tx collapses to retractEntity" + (let [conn (db-test/create-conn-with-blocks + {:pages-and-blocks + [{:page {:block/title "page 1"} + :blocks [{:block/title "target"}]}]}) + target (db-test/find-block-by-content @conn "target") + target-uuid (:block/uuid target) + db-before @conn + tx-report (d/with db-before + [[:db/retractEntity [:block/uuid target-uuid]]] + {}) + {:keys [normalized-tx-data reversed-datoms]} + (#'sync-apply/normalize-rebased-pending-tx + {:db-before db-before + :db-after (:db-after tx-report) + :tx-data (:tx-data tx-report) + :remote-tx-data-set #{}}) + restored-db (:db-after (d/with (:db-after tx-report) reversed-datoms))] + (is (= [[:db/retractEntity [:block/uuid target-uuid]]] + normalized-tx-data)) + (is (seq reversed-datoms)) + (is (= target-uuid + (-> (d/entity restored-db [:block/uuid target-uuid]) :block/uuid)))))) + +(deftest reverse-tx-data-delete-and-recreate-same-uuid-remains-reversible-test + (testing "reverse tx-data should remain valid when a tx retracts and recreates the same block uuid" + (let [conn (db-test/create-conn-with-blocks + {:pages-and-blocks + [{:page {:block/title "page 1"} + :blocks [{:block/title "old"}]}]}) + target (db-test/find-block-by-content @conn "old") + target-uuid (:block/uuid target) + page-uuid (:block/uuid (:block/page target)) + original-order (:block/order target) + db-before @conn + tx-report (d/with db-before + [[:db/retractEntity [:block/uuid target-uuid]] + [:db/add -1 :block/uuid target-uuid] + [:db/add -1 :block/title "new"] + [:db/add -1 :block/parent [:block/uuid page-uuid]] + [:db/add -1 :block/page [:block/uuid page-uuid]] + [:db/add -1 :block/order original-order]] + {}) + reversed-datoms (#'sync-apply/reverse-tx-data + db-before + (:db-after tx-report) + (:tx-data tx-report)) + reverse-conn (d/conn-from-db (:db-after tx-report))] + (is (some? (d/entity (:db-after tx-report) [:block/uuid target-uuid]))) + (ldb/transact! reverse-conn reversed-datoms {:outliner-op :reverse-test}) + (let [restored (d/entity @reverse-conn [:block/uuid target-uuid])] + (is (some? restored)) + (is (= "old" (:block/title restored))) + (is (= page-uuid (some-> restored :block/page :block/uuid))) + (is (= page-uuid (some-> restored :block/parent :block/uuid))))))) (deftest rebase-preserves-title-when-reversed-tx-ids-change-test (testing "rebase keeps local title when reverse tx gets a new tx id" @@ -1481,7 +3145,10 @@ (orig repo tx-report))))] (with-datascript-conns conn client-ops-conn (fn [] - (d/transact! conn [[:db/add (:db/id block) :block/title "test"]]) + (outliner-op/apply-ops! conn + [[:save-block [{:block/uuid (:block/uuid block) + :block/title "test"} nil]]] + local-tx-meta) (is (= 1 (count (#'sync-apply/pending-txs test-repo)))) (#'sync-apply/apply-remote-tx! test-repo @@ -1556,128 +3223,6 @@ (is (empty? (non-recycle-validation-entities validation)) (str (:errors validation))))))))))) -(deftest sanitize-tx-data-drops-partial-create-when-parent-recycled-test - (testing "created block is kept when parent is recycled because recycled refs are still valid entities" - (let [{:keys [conn parent]} (setup-parent-child) - page-uuid (:block/uuid (:block/page parent)) - parent-uuid (:block/uuid parent) - child-uuid (random-uuid) - tx-data [[:db/add -1 :block/uuid child-uuid] - [:db/add -1 :block/title ""] - [:db/add -1 :block/page [:block/uuid page-uuid]] - [:db/add -1 :block/order "a0"] - [:db/add [:block/uuid child-uuid] :block/parent [:block/uuid parent-uuid]]] - _ (outliner-core/delete-blocks! conn [parent] {}) - sanitized (->> (#'sync-apply/sanitize-tx-data @conn tx-data) - vec)] - (is (= tx-data sanitized))))) - -(deftest sanitize-tx-data-removes-orphaning-parent-retract-test - (testing "when invalid reparent add is dropped, paired parent retract should be dropped too" - (let [{:keys [conn parent child1]} (setup-parent-child) - child-uuid (:block/uuid child1) - old-parent-uuid (:block/uuid parent) - missing-parent-uuid (random-uuid) - tx-data [[:db/retract [:block/uuid child-uuid] :block/parent [:block/uuid old-parent-uuid]] - [:db/add [:block/uuid child-uuid] :block/parent [:block/uuid missing-parent-uuid]]] - sanitized (->> (#'sync-apply/sanitize-tx-data @conn tx-data) - vec)] - (is (empty? sanitized))))) - -(deftest drop-orphaning-parent-retracts-is-still-needed-test - (testing "without orphaning-parent cleanup, sanitize leaves a bad parent retract behind" - (let [{:keys [conn parent child1]} (setup-parent-child) - child-uuid (:block/uuid child1) - old-parent-uuid (:block/uuid parent) - missing-parent-uuid (random-uuid) - tx-data [[:db/retract [:block/uuid child-uuid] :block/parent [:block/uuid old-parent-uuid]] - [:db/add [:block/uuid child-uuid] :block/parent [:block/uuid missing-parent-uuid]]] - sanitized-without-cleanup (with-redefs [sync-apply/drop-orphaning-parent-retracts identity] - (->> (#'sync-apply/sanitize-tx-data @conn tx-data) - vec))] - (is (= [[:db/retract [:block/uuid child-uuid] - :block/parent - [:block/uuid old-parent-uuid]]] - sanitized-without-cleanup))))) - -(deftest sanitize-tx-data-drops-numeric-entity-datoms-for-recycled-block-test - (testing "recycled entity ids are kept when the entity still exists" - (let [{:keys [conn child1]} (setup-parent-child) - child-id (:db/id child1) - tx-data [[:db/add child-id :block/title "should-drop"]] - _ (outliner-core/delete-blocks! conn [child1] {}) - sanitized (->> (#'sync-apply/sanitize-tx-data @conn tx-data) - vec)] - (is (= tx-data sanitized))))) - -(deftest sanitize-tx-data-drops-numeric-value-refs-for-recycled-block-test - (testing "recycled block refs are kept when the referenced entity still exists" - (let [{:keys [conn parent child1]} (setup-parent-child) - parent-id (:db/id parent) - child-id (:db/id child1) - tx-data [[:db/add parent-id :block/parent child-id]] - _ (outliner-core/delete-blocks! conn [child1] {}) - sanitized (->> (#'sync-apply/sanitize-tx-data @conn tx-data) - vec)] - (is (= tx-data sanitized))))) - -(deftest sanitize-tx-data-drops-datoms-with-missing-numeric-entity-test - (testing "stale numeric entity ids should be dropped to avoid creating anonymous entities" - (let [{:keys [conn]} (setup-parent-child) - missing-id 999999 - tx-data [[:db/add missing-id :block/title ""]] - sanitized (->> (#'sync-apply/sanitize-tx-data @conn tx-data) - vec)] - (is (empty? sanitized))))) - -(deftest sanitize-tx-data-drops-datoms-with-missing-numeric-ref-value-test - (testing "stale numeric ref values should be dropped when referenced entity no longer exists" - (let [{:keys [conn parent]} (setup-parent-child) - parent-id (:db/id parent) - missing-id 999999 - tx-data [[:db/add parent-id :block/parent missing-id]] - sanitized (->> (#'sync-apply/sanitize-tx-data @conn tx-data) - vec)] - (is (empty? sanitized))))) - -(deftest sanitize-tx-data-drops-datoms-with-missing-lookup-ref-value-test - (testing "stale lookup ref values should be dropped when referenced entity no longer exists" - (let [{:keys [conn child1 child2]} (setup-parent-child) - child-uuid (:block/uuid child1) - new-parent-uuid (:block/uuid child2) - missing-parent-uuid (random-uuid) - tx-data [[:db/retract [:block/uuid child-uuid] - :block/parent - [:block/uuid missing-parent-uuid]] - [:db/add [:block/uuid child-uuid] - :block/parent - [:block/uuid new-parent-uuid]]] - sanitized (->> (#'sync-apply/sanitize-tx-data @conn tx-data) - vec)] - (is (= [[:db/add [:block/uuid child-uuid] - :block/parent - [:block/uuid new-parent-uuid]]] - sanitized))))) - -(deftest sanitize-tx-data-keeps-retract-entity-lookup-for-missing-block-test - (testing "retractEntity lookup should survive sanitize for synced undo of inserted blocks" - (let [{:keys [conn]} (setup-parent-child) - missing-uuid (random-uuid) - tx-data [[:db/retractEntity [:block/uuid missing-uuid]]] - sanitized (->> (#'sync-apply/sanitize-tx-data @conn tx-data) - vec)] - (is (= tx-data sanitized))))) - -(deftest sanitize-tx-data-drops-stale-missing-block-lookup-updates-test - (testing "title-only updates for a missing lookup block should be dropped" - (let [{:keys [conn]} (setup-parent-child) - missing-uuid (random-uuid) - tx-data [[:db/add [:block/uuid missing-uuid] :block/title "stale title"] - [:db/add [:block/uuid missing-uuid] :block/updated-at 1773747515784]] - sanitized (->> (#'sync-apply/sanitize-tx-data @conn tx-data) - vec)] - (is (empty? sanitized))))) - (deftest apply-remote-tx-local-delete-remote-recreate-does-not-leave-local-only-delete-test (testing "if remote batch recreates a locally deleted block, client should not end with unsynced local-only deletion" (let [conn (db-test/create-conn-with-blocks diff --git a/src/test/frontend/worker/db_worker_test.cljs b/src/test/frontend/worker/db_worker_test.cljs index 1c05cd1253..73935cb5e9 100644 --- a/src/test/frontend/worker/db_worker_test.cljs +++ b/src/test/frontend/worker/db_worker_test.cljs @@ -1,18 +1,20 @@ (ns frontend.worker.db-worker-test - (:require [cljs.test :refer [async deftest is]] - [datascript.core :as d] - [frontend.common.thread-api :as thread-api] - [frontend.worker.a-test-env] - [frontend.worker.db-worker :as db-worker] - [frontend.worker.search :as search] - [frontend.worker.shared-service :as shared-service] - [frontend.worker.state :as worker-state] - [frontend.worker.sync :as db-sync] - [frontend.worker.sync.client-op :as client-op] - [frontend.worker.sync.crypt :as sync-crypt] - [frontend.worker.sync.log-and-state :as rtc-log-and-state] - [logseq.db.frontend.schema :as db-schema] - [promesa.core :as p])) + (:require + [cljs.test :refer [async deftest is]] + [datascript.core :as d] + [frontend.common.thread-api :as thread-api] + [frontend.worker.a-test-env] + [frontend.worker.db-worker :as db-worker] + [frontend.worker.db.validate :as worker-db-validate] + [frontend.worker.search :as search] + [frontend.worker.shared-service :as shared-service] + [frontend.worker.state :as worker-state] + [frontend.worker.sync :as db-sync] + [frontend.worker.sync.client-op :as client-op] + [frontend.worker.sync.crypt :as sync-crypt] + [frontend.worker.sync.log-and-state :as rtc-log-and-state] + [logseq.db.frontend.schema :as db-schema] + [promesa.core :as p])) (def ^:private test-repo "test-db-worker-repo") (def ^:private close-db!-orig db-worker/close-db!) @@ -73,14 +75,56 @@ (reset! worker-state/*opfs-pools {test-repo #js {:pauseVfs (fn [] (swap! pause-calls inc))}}) (reset! search/fuzzy-search-indices {test-repo :stale-cache}) + (reset! client-op/*repo->pending-local-tx-count {test-repo 9}) (db-worker/close-db! test-repo) (is (= #{:db :search :client-ops} (set @closed))) (is (= 1 @pause-calls)) (is (nil? (get @search/fuzzy-search-indices test-repo))) + (is (nil? (get @client-op/*repo->pending-local-tx-count test-repo))) (is (nil? (get @worker-state/*sqlite-conns test-repo))))))) +(deftest client-ops-cleanup-timer-starts-once-and-clears-on-close-test + (restoring-worker-state + (fn [] + (let [scheduled (atom []) + cleared (atom []) + original-set-interval js/setInterval + original-clear-interval js/clearInterval + fake-db #js {:close (fn [] nil)} + timer-id #js {:id "timer-1"}] + (set! js/setInterval + (fn [f interval-ms] + (swap! scheduled conj {:fn f :interval-ms interval-ms}) + timer-id)) + (set! js/clearInterval + (fn [id] + (swap! cleared conj id))) + (try + (reset! worker-state/*sqlite-conns + {test-repo {:db fake-db + :search fake-db + :client-ops fake-db}}) + (reset! worker-state/*datascript-conns {test-repo :datascript}) + (reset! worker-state/*client-ops-conns {test-repo :client-ops}) + (reset! (deref #'db-worker/*client-ops-cleanup-timers) {}) + + (#'db-worker/ensure-client-ops-cleanup-timer! test-repo) + (#'db-worker/ensure-client-ops-cleanup-timer! test-repo) + + (is (= 1 (count @scheduled))) + (is (= (* 3 60 60 1000) (:interval-ms (first @scheduled)))) + (is (= timer-id (get @(deref #'db-worker/*client-ops-cleanup-timers) test-repo))) + + (db-worker/close-db! test-repo) + + (is (= [timer-id] @cleared)) + (is (nil? (get @(deref #'db-worker/*client-ops-cleanup-timers) test-repo))) + (finally + (set! js/setInterval original-set-interval) + (set! js/clearInterval original-clear-interval))))))) + (deftest complete-datoms-import-invalidates-existing-search-db-test (async done (restoring-worker-state @@ -288,3 +332,61 @@ (p/catch (fn [error] (is false (str error)) (done))))))))))) + +(deftest thread-api-validate-db-passes-sync-diagnostics-test + (restoring-worker-state + (fn [] + (let [validate (@thread-api/*thread-apis :thread-api/validate-db) + conn (d/create-conn db-schema/schema) + captured (atom nil) + latest-prev @db-sync/*repo->latest-remote-tx] + (reset! worker-state/*datascript-conns {test-repo conn}) + (reset! db-sync/*repo->latest-remote-tx {test-repo 11}) + (try + (with-redefs [client-op/get-local-tx (fn [_repo] 7) + client-op/get-local-checksum (fn [_repo] "local-checksum") + worker-db-validate/validate-db (fn [& args] + (reset! captured args) + {:ok true})] + (validate test-repo) + (is (= [test-repo + conn + {:local-tx 7 + :remote-tx 11 + :local-checksum "local-checksum" + :remote-checksum nil}] + @captured))) + (finally + (reset! db-sync/*repo->latest-remote-tx latest-prev))))))) + +(deftest thread-api-recompute-checksum-diagnostics-passes-sync-diagnostics-test + (restoring-worker-state + (fn [] + (let [recompute (@thread-api/*thread-apis :thread-api/recompute-checksum-diagnostics) + conn (d/create-conn db-schema/schema) + captured (atom nil) + latest-tx-prev @db-sync/*repo->latest-remote-tx + latest-checksum-prev @db-sync/*repo->latest-remote-checksum + result {:recomputed-checksum "recomputed" + :checksum-attrs [:block/uuid] + :blocks []}] + (reset! worker-state/*datascript-conns {test-repo conn}) + (reset! db-sync/*repo->latest-remote-tx {test-repo 22}) + (reset! db-sync/*repo->latest-remote-checksum {test-repo "remote-checksum"}) + (try + (with-redefs [client-op/get-local-tx (fn [_repo] 10) + client-op/get-local-checksum (fn [_repo] "local-checksum") + worker-db-validate/recompute-checksum-diagnostics (fn [& args] + (reset! captured args) + result)] + (is (= result (recompute test-repo))) + (is (= [test-repo + conn + {:local-tx 10 + :remote-tx 22 + :local-checksum "local-checksum" + :remote-checksum "remote-checksum"}] + @captured))) + (finally + (reset! db-sync/*repo->latest-remote-tx latest-tx-prev) + (reset! db-sync/*repo->latest-remote-checksum latest-checksum-prev))))))) diff --git a/src/test/frontend/worker/sync/client_op_test.cljs b/src/test/frontend/worker/sync/client_op_test.cljs index 3b882d159b..e63b7d3022 100644 --- a/src/test/frontend/worker/sync/client_op_test.cljs +++ b/src/test/frontend/worker/sync/client_op_test.cljs @@ -1,5 +1,5 @@ (ns frontend.worker.sync.client-op-test - (:require [cljs.test :refer [deftest is]] + (:require [cljs.test :refer [deftest is testing]] [datascript.core :as d] [frontend.worker.state :as worker-state] [frontend.worker.sync.client-op :as client-op])) @@ -17,3 +17,40 @@ (is (= #{"graph-2"} (set (map :v graph-uuid-datoms))))) (finally (reset! worker-state/*client-ops-conns prev-client-ops-conns))))) + +(deftest cleanup-finished-history-ops-removes-only-unreferenced-finished-txs-test + (let [repo "repo-cleanup" + conn (d/create-conn client-op/schema-in-db) + prev-client-ops-conns @worker-state/*client-ops-conns + keep-tx-id (random-uuid) + remove-tx-id (random-uuid) + pending-tx-id (random-uuid)] + (reset! worker-state/*client-ops-conns {repo conn}) + (try + (d/transact! conn + [{:db-sync/tx-id keep-tx-id + :db-sync/pending? false} + {:db-sync/tx-id remove-tx-id + :db-sync/pending? false} + {:db-sync/tx-id pending-tx-id + :db-sync/pending? true} + {:db-ident :metadata/local + :local-tx 99}]) + + (is (= 1 (client-op/cleanup-finished-history-ops! repo #{keep-tx-id}))) + (is (some? (d/entity @conn [:db-sync/tx-id keep-tx-id]))) + (is (nil? (d/entity @conn [:db-sync/tx-id remove-tx-id]))) + (is (some? (d/entity @conn [:db-sync/tx-id pending-tx-id]))) + (is (= 99 (:local-tx (d/entity @conn [:db-ident :metadata/local])))) + (finally + (reset! worker-state/*client-ops-conns prev-client-ops-conns))))) + +(deftest cleanup-finished-history-ops-no-conn-is-noop-test + (let [repo "repo-no-conn" + prev-client-ops-conns @worker-state/*client-ops-conns] + (reset! worker-state/*client-ops-conns {}) + (try + (testing "cleanup should be safe when client-ops conn is missing" + (is (= 0 (client-op/cleanup-finished-history-ops! repo #{})))) + (finally + (reset! worker-state/*client-ops-conns prev-client-ops-conns))))) diff --git a/src/test/frontend/worker/sync/crypt_test.cljs b/src/test/frontend/worker/sync/crypt_test.cljs index 035debb719..0f6e24e4dc 100644 --- a/src/test/frontend/worker/sync/crypt_test.cljs +++ b/src/test/frontend/worker/sync/crypt_test.cljs @@ -46,3 +46,72 @@ (p/catch (fn [e] (is false (str e)) (done)))))) + +(deftest fetch-graph-aes-key-for-download-retries-with-fresh-rsa-key-pair-test + (async done + (let [clear-user-rsa-cache-calls* (atom 0) + get-pair-calls* (atom 0)] + (-> (p/with-redefs [sync-crypt/e2ee-base (fn [] "https://sync.example.test") + sync-crypt/get-user-uuid (fn [] "user-1") + sync-crypt/ (p/with-redefs [sync-crypt/e2ee-base (fn [] "https://sync.example.test") + sync-crypt/get-user-uuid (fn [] nil) + sync-crypt/journal-day (js/Date.)) + today-title (date-time-util/int->journal-title + today-day + (:logseq.property.journal/title-format + (d/entity @conn :logseq.class/Journal))) + [_ today-page-uuid] (outliner-op/apply-ops! conn + [[:create-page [today-title + {:today-journal? true + :redirect? false + :split-namespace? true + :tags ()}]]] + (local-tx-meta {:client-id "test-client"})) + today-page-id (:db/id (d/entity @conn [:block/uuid today-page-uuid])) + today-child-uuid (random-uuid) + _ (outliner-op/apply-ops! conn + [[:insert-blocks [[{:block/uuid today-child-uuid + :block/title "today undo child"}] + today-page-id + {:sibling? false + :keep-uuid? true}]]] + (local-tx-meta {:client-id "test-client"})) + class-ident-before (:db/ident (d/entity @conn [:block/uuid class-uuid])) + property-ident-before (:db/ident (d/entity @conn [:block/uuid property-uuid]))] + (worker-undo-redo/clear-history! test-repo) + + (outliner-op/apply-ops! conn + [[:delete-page [class-uuid {}]]] + (local-tx-meta {:client-id "test-client"})) + (is (nil? (d/entity @conn [:block/uuid class-uuid]))) + (is (map? (worker-undo-redo/undo test-repo))) + (is (= class-ident-before + (:db/ident (d/entity @conn [:block/uuid class-uuid])))) + + (worker-undo-redo/clear-history! test-repo) + (outliner-op/apply-ops! conn + [[:delete-page [property-uuid {}]]] + (local-tx-meta {:client-id "test-client"})) + (is (nil? (d/entity @conn :user.property/undo-rating))) + (is (map? (worker-undo-redo/undo test-repo))) + (is (= property-ident-before + (:db/ident (d/entity @conn [:block/uuid property-uuid])))) + + (worker-undo-redo/clear-history! test-repo) + (outliner-op/apply-ops! conn + [[:delete-page [today-page-uuid {}]]] + (local-tx-meta {:client-id "test-client"})) + (is (some? (d/entity @conn [:block/uuid today-page-uuid]))) + (is (nil? (d/entity @conn [:block/uuid today-child-uuid]))) + (is (map? (worker-undo-redo/undo test-repo))) + (is (some? (d/entity @conn [:block/uuid today-child-uuid])))))) + +(deftest redo-create-page-restores-recycled-page-test + (testing "redoing create-page should restore recycled page instead of keeping it recycled" + (worker-undo-redo/clear-history! test-repo) + (let [conn (worker-state/get-datascript-conn test-repo) + page-title "redo create page alpha"] + (outliner-op/apply-ops! conn + [[:create-page [page-title {:redirect? false + :split-namespace? true + :tags ()}]]] + (local-tx-meta {:client-id "test-client"})) + (let [created-page (db-test/find-page-by-title @conn page-title)] + (is (some? created-page)) + (is (false? (ldb/recycled? created-page)))) + + (is (seq (undo-all!))) + (let [deleted-page (db-test/find-page-by-title @conn page-title)] + (is (some? deleted-page)) + (is (true? (ldb/recycled? deleted-page)))) + + (is (seq (redo-all!))) + (let [page-after-redo (db-test/find-page-by-title @conn page-title)] + (is (some? page-after-redo)) + (is (false? (ldb/recycled? page-after-redo))))))) + +(deftest redo-template-insert-restores-valid-blocks-test + (testing "redoing template insert after undo-all should restore inserted template blocks without invalid refs" + (worker-undo-redo/clear-history! test-repo) + (let [conn (worker-state/get-datascript-conn test-repo) + {:keys [page-uuid]} (seed-page-parent-child!) + page-id (:db/id (d/entity @conn [:block/uuid page-uuid])) + template-root-uuid (random-uuid) + template-a-uuid (random-uuid) + template-b-uuid (random-uuid) + empty-target-uuid (random-uuid)] + (outliner-op/apply-ops! + conn + [[:insert-blocks [[{:block/uuid template-root-uuid + :block/title "template 1" + :block/tags #{:logseq.class/Template}} + {:block/uuid template-a-uuid + :block/title "a" + :block/parent [:block/uuid template-root-uuid]} + {:block/uuid template-b-uuid + :block/title "b" + :block/parent [:block/uuid template-a-uuid]}] + page-id + {:sibling? false + :keep-uuid? true}]]] + (local-tx-meta {:client-id "test-client"})) + (outliner-op/apply-ops! + conn + [[:insert-blocks [[{:block/uuid empty-target-uuid + :block/title ""}] + page-id + {:sibling? false + :keep-uuid? true}]]] + (local-tx-meta {:client-id "test-client"})) + (let [template-root (d/entity @conn [:block/uuid template-root-uuid]) + empty-target (d/entity @conn [:block/uuid empty-target-uuid]) + template-blocks (->> (ldb/get-block-and-children @conn template-root-uuid + {:include-property-block? true}) + rest) + blocks-to-insert (cons (assoc (first template-blocks) + :logseq.property/used-template (:db/id template-root)) + (rest template-blocks))] + (outliner-op/apply-ops! + conn + [[:insert-blocks [blocks-to-insert + (:db/id empty-target) + {:sibling? true + :replace-empty-target? true + :insert-template? true}]]] + (local-tx-meta {:client-id "test-client"}))) + + (is (seq (undo-all!))) + (is (seq (redo-all!))) + + (let [inserted-a-id (d/q '[:find ?b . + :in $ ?template-uuid + :where + [?template :block/uuid ?template-uuid] + [?b :logseq.property/used-template ?template] + [?b :block/title "a"]] + @conn + template-root-uuid) + inserted-a (when inserted-a-id (d/entity @conn inserted-a-id))] + (is (some? inserted-a)) + (is (= template-root-uuid + (some-> inserted-a :logseq.property/used-template :block/uuid))))))) + +(deftest undo-history-canonicalizes-template-replace-empty-target-to-apply-template-test + (testing "template replace-empty-target history keeps semantic forward op and restores empty target" + (worker-undo-redo/clear-history! test-repo) + (let [conn (worker-state/get-datascript-conn test-repo) + {:keys [page-uuid]} (seed-page-parent-child!) + page-id (:db/id (d/entity @conn [:block/uuid page-uuid])) + template-root-uuid (random-uuid) + template-a-uuid (random-uuid) + template-b-uuid (random-uuid) + empty-target-uuid (random-uuid)] + (outliner-op/apply-ops! + conn + [[:insert-blocks [[{:block/uuid template-root-uuid + :block/title "template 1" + :block/tags #{:logseq.class/Template}} + {:block/uuid template-a-uuid + :block/title "a" + :block/parent [:block/uuid template-root-uuid]} + {:block/uuid template-b-uuid + :block/title "b" + :block/parent [:block/uuid template-a-uuid]}] + page-id + {:sibling? false + :keep-uuid? true}]]] + (local-tx-meta {:client-id "test-client"})) + (outliner-op/apply-ops! + conn + [[:insert-blocks [[{:block/uuid empty-target-uuid + :block/title ""}] + page-id + {:sibling? false + :keep-uuid? true}]]] + (local-tx-meta {:client-id "test-client"})) + (let [template-root (d/entity @conn [:block/uuid template-root-uuid]) + empty-target (d/entity @conn [:block/uuid empty-target-uuid]) + template-blocks (->> (ldb/get-block-and-children @conn template-root-uuid + {:include-property-block? true}) + rest) + blocks-to-insert (cons (assoc (first template-blocks) + :logseq.property/used-template (:db/id template-root)) + (rest template-blocks))] + (outliner-op/apply-ops! + conn + [[:insert-blocks [blocks-to-insert + (:db/id empty-target) + {:sibling? true + :replace-empty-target? true + :insert-template? true}]]] + (local-tx-meta {:client-id "test-client"}))) + (let [data (latest-undo-history-data) + inverse-ops (:db-sync/inverse-outliner-ops data) + delete-op (some #(when (= :delete-blocks (first %)) %) inverse-ops) + restore-empty-insert-op (some #(when (= :insert-blocks (first %)) %) inverse-ops) + restore-empty-save-op (some #(when (= :save-block (first %)) %) inverse-ops)] + (is (contains? #{:apply-template :insert-blocks} + (ffirst (:db-sync/forward-outliner-ops data)))) + (is (some? delete-op)) + (is (or (some? restore-empty-insert-op) + (some? restore-empty-save-op))) + (if restore-empty-insert-op + (do + (is (= empty-target-uuid + (get-in restore-empty-insert-op [1 0 0 :block/uuid]))) + (is (= "" + (get-in restore-empty-insert-op [1 0 0 :block/title])))) + (do + (is (= empty-target-uuid + (get-in restore-empty-save-op [1 0 :block/uuid]))) + (is (= "" + (get-in restore-empty-save-op [1 0 :block/title]))))))))) + +(deftest undo-history-replace-empty-target-insert-restores-empty-target-with-insert-op-test + (testing "replace-empty-target insert inverse should delete inserted blocks and reinsert original empty target" + (worker-undo-redo/clear-history! test-repo) + (let [conn (worker-state/get-datascript-conn test-repo) + {:keys [page-uuid]} (seed-page-parent-child!) + page-id (:db/id (d/entity @conn [:block/uuid page-uuid])) + empty-target-uuid (random-uuid) + inserted-root-uuid (random-uuid) + inserted-child-uuid (random-uuid)] + (outliner-op/apply-ops! + conn + [[:insert-blocks [[{:block/uuid empty-target-uuid + :block/title ""}] + page-id + {:sibling? false + :keep-uuid? true}]]] + (local-tx-meta {:client-id "test-client"})) + (let [empty-target (d/entity @conn [:block/uuid empty-target-uuid])] + (outliner-op/apply-ops! + conn + [[:insert-blocks [[{:block/uuid inserted-root-uuid + :block/title "insert root"} + {:block/uuid inserted-child-uuid + :block/title "insert child" + :block/parent [:block/uuid inserted-root-uuid]}] + (:db/id empty-target) + {:sibling? true + :replace-empty-target? true}]]] + (local-tx-meta {:client-id "test-client"}))) + (let [data (latest-undo-history-data) + inverse-ops (:db-sync/inverse-outliner-ops data) + delete-op (some #(when (= :delete-blocks (first %)) %) inverse-ops) + restore-empty-insert-op (some #(when (= :insert-blocks (first %)) %) inverse-ops) + restore-empty-save-op (some #(when (= :save-block (first %)) %) inverse-ops) + delete-ids (set (get-in delete-op [1 0]))] + (is (= :insert-blocks (ffirst (:db-sync/forward-outliner-ops data)))) + (is (seq delete-ids)) + (is (or (some? restore-empty-insert-op) + (some? restore-empty-save-op))) + (if restore-empty-insert-op + (do + (is (= empty-target-uuid + (get-in restore-empty-insert-op [1 0 0 :block/uuid]))) + (is (= "" + (get-in restore-empty-insert-op [1 0 0 :block/title])))) + (do + (is (= empty-target-uuid + (get-in restore-empty-save-op [1 0 :block/uuid]))) + (is (= "" + (get-in restore-empty-save-op [1 0 :block/title]))))))))) + +(deftest apply-template-op-replays-via-undo-redo-test + (testing ":apply-template op can be applied and replayed via undo/redo" + (worker-undo-redo/clear-history! test-repo) + (let [conn (worker-state/get-datascript-conn test-repo) + {:keys [page-uuid]} (seed-page-parent-child!) + page-id (:db/id (d/entity @conn [:block/uuid page-uuid])) + template-root-uuid (random-uuid) + template-a-uuid (random-uuid) + template-b-uuid (random-uuid) + empty-target-uuid (random-uuid)] + (outliner-op/apply-ops! + conn + [[:insert-blocks [[{:block/uuid template-root-uuid + :block/title "template 1" + :block/tags #{:logseq.class/Template}} + {:block/uuid template-a-uuid + :block/title "a" + :block/parent [:block/uuid template-root-uuid]} + {:block/uuid template-b-uuid + :block/title "b" + :block/parent [:block/uuid template-a-uuid]}] + page-id + {:sibling? false + :keep-uuid? true}]]] + (local-tx-meta {:client-id "test-client"})) + (outliner-op/apply-ops! + conn + [[:insert-blocks [[{:block/uuid empty-target-uuid + :block/title ""}] + page-id + {:sibling? false + :keep-uuid? true}]]] + (local-tx-meta {:client-id "test-client"})) + (let [template-root (d/entity @conn [:block/uuid template-root-uuid]) + empty-target (d/entity @conn [:block/uuid empty-target-uuid]) + template-blocks (->> (ldb/get-block-and-children @conn template-root-uuid + {:include-property-block? true}) + rest) + blocks-to-insert (cons (assoc (first template-blocks) + :logseq.property/used-template (:db/id template-root)) + (rest template-blocks))] + (outliner-op/apply-ops! + conn + [[:apply-template [(:db/id template-root) + (:db/id empty-target) + {:sibling? true + :replace-empty-target? true + :template-blocks blocks-to-insert}]]] + (local-tx-meta {:client-id "test-client"}))) + + (let [data (latest-undo-history-data)] + (is (= :apply-template (ffirst (:db-sync/forward-outliner-ops data))))) + + (is (seq (undo-all!))) + (is (seq (redo-all!))) + + (let [inserted-a-id (d/q '[:find ?b . + :in $ ?template-uuid + :where + [?template :block/uuid ?template-uuid] + [?b :logseq.property/used-template ?template] + [?b :block/title "a"]] + @conn + template-root-uuid) + inserted-a (when inserted-a-id (d/entity @conn inserted-a-id)) + inserted-b (some->> inserted-a :block/_parent (filter #(= "b" (:block/title %))) first)] + (is (some? inserted-a)) + (is (some? inserted-b)))))) + +(deftest apply-template-repeated-undo-redo-uses-latest-history-tx-id-test + (testing ":apply-template repeated undo/redo should always undo latest recreated blocks" + (worker-undo-redo/clear-history! test-repo) + (let [conn (worker-state/get-datascript-conn test-repo) + {:keys [page-uuid]} (seed-page-parent-child!) + page-id (:db/id (d/entity @conn [:block/uuid page-uuid])) + template-root-uuid (random-uuid) + template-a-uuid (random-uuid) + template-b-uuid (random-uuid) + empty-target-uuid (random-uuid)] + (outliner-op/apply-ops! + conn + [[:insert-blocks [[{:block/uuid template-root-uuid + :block/title "template 1" + :block/tags #{:logseq.class/Template}} + {:block/uuid template-a-uuid + :block/title "a" + :block/parent [:block/uuid template-root-uuid]} + {:block/uuid template-b-uuid + :block/title "b" + :block/parent [:block/uuid template-a-uuid]}] + page-id + {:sibling? false + :keep-uuid? true}]]] + (local-tx-meta {:client-id "test-client"})) + (outliner-op/apply-ops! + conn + [[:insert-blocks [[{:block/uuid empty-target-uuid + :block/title ""}] + page-id + {:sibling? false + :keep-uuid? true}]]] + (local-tx-meta {:client-id "test-client"})) + (worker-undo-redo/clear-history! test-repo) + (let [template-root (d/entity @conn [:block/uuid template-root-uuid]) + empty-target (d/entity @conn [:block/uuid empty-target-uuid]) + template-blocks (->> (ldb/get-block-and-children @conn template-root-uuid + {:include-property-block? true}) + rest) + blocks-to-insert (cons (assoc (first template-blocks) + :logseq.property/used-template (:db/id template-root)) + (rest template-blocks)) + find-inserted-a-id (fn [] + (d/q '[:find ?b . + :in $ ?template-uuid + :where + [?template :block/uuid ?template-uuid] + [?b :logseq.property/used-template ?template] + [?b :block/title "a"]] + @conn + template-root-uuid))] + (outliner-op/apply-ops! + conn + [[:apply-template [(:db/id template-root) + (:db/id empty-target) + {:sibling? true + :replace-empty-target? true + :template-blocks blocks-to-insert}]]] + (local-tx-meta {:client-id "test-client"})) + (is (some? (find-inserted-a-id))) + (is (not= ::worker-undo-redo/empty-undo-stack + (worker-undo-redo/undo test-repo))) + (is (nil? (find-inserted-a-id))) + (is (not= ::worker-undo-redo/empty-redo-stack + (worker-undo-redo/redo test-repo))) + (let [redo-1-a-id (find-inserted-a-id)] + (is (some? redo-1-a-id)) + (is (not= ::worker-undo-redo/empty-undo-stack + (worker-undo-redo/undo test-repo))) + (is (nil? (find-inserted-a-id))) + (is (not= ::worker-undo-redo/empty-redo-stack + (worker-undo-redo/redo test-repo))) + (let [redo-2-a-id (find-inserted-a-id)] + (is (some? redo-2-a-id)) + (is (not= redo-1-a-id redo-2-a-id)) + (is (not= ::worker-undo-redo/empty-undo-stack + (worker-undo-redo/undo test-repo))) + (is (nil? (find-inserted-a-id))))))))) + +(deftest undo-history-records-forward-ops-for-save-block-test + (testing "worker save-block history keeps semantic forward ops for redo replay" + (worker-undo-redo/clear-history! test-repo) + (let [conn (worker-state/get-datascript-conn test-repo) + {:keys [child-uuid]} (seed-page-parent-child!)] + (outliner-op/apply-ops! conn + [[:save-block [{:block/uuid child-uuid + :block/title "saved via apply-ops"} {}]]] + (local-tx-meta {:client-id "test-client"})) + (let [undo-op (last (get @worker-undo-redo/*undo-ops test-repo)) + data (some #(when (= ::worker-undo-redo/db-transact (first %)) + (second %)) + undo-op)] + (is (= :save-block (ffirst (:db-sync/forward-outliner-ops data)))) + (is (= child-uuid + (get-in data [:db-sync/forward-outliner-ops 0 1 0 :block/uuid]))) + (is (= "saved via apply-ops" + (get-in data [:db-sync/forward-outliner-ops 0 1 0 :block/title]))) + (is (= "saved via apply-ops" + (:block/title (d/entity @conn [:block/uuid child-uuid])))))))) + +(deftest undo-insert-retracts-added-entity-cleanly-test + (testing "undoing a local insert retracts the inserted entity" + (worker-undo-redo/clear-history! test-repo) + (let [conn (worker-state/get-datascript-conn test-repo) + {:keys [page-uuid]} (seed-page-parent-child!) + page-id (:db/id (d/entity @conn [:block/uuid page-uuid])) + inserted-uuid (random-uuid)] + (d/transact! conn + [{:block/uuid inserted-uuid + :block/title "inserted" + :block/page [:block/uuid page-uuid] + :block/parent [:block/uuid page-uuid]}] + (local-tx-meta + {:client-id "test-client" + :outliner-op :insert-blocks + :outliner-ops [[:insert-blocks [[{:block/title "inserted" + :block/uuid inserted-uuid}] + page-id + {:sibling? false}]]]})) + (is (some? (d/entity @conn [:block/uuid inserted-uuid]))) + (let [undo-result (worker-undo-redo/undo test-repo)] + (is (map? undo-result)) + (is (nil? (d/entity @conn [:block/uuid inserted-uuid]))))))) + +(deftest repeated-save-block-content-undo-redo-test + (testing "multiple saves on the same block undo and redo one step at a time" + (worker-undo-redo/clear-history! test-repo) + (let [conn (worker-state/get-datascript-conn test-repo) + {:keys [child-uuid]} (seed-page-parent-child!)] + (doseq [title ["v1" "v2" "v3"]] + (save-block-title! conn child-uuid title)) + (is (= "v3" (:block/title (d/entity @conn [:block/uuid child-uuid])))) + (worker-undo-redo/undo test-repo) + (is (= "v2" (:block/title (d/entity @conn [:block/uuid child-uuid])))) + (worker-undo-redo/undo test-repo) + (is (= "v1" (:block/title (d/entity @conn [:block/uuid child-uuid])))) + (worker-undo-redo/undo test-repo) + (is (= "child" (:block/title (d/entity @conn [:block/uuid child-uuid])))) + (worker-undo-redo/redo test-repo) + (is (= "v1" (:block/title (d/entity @conn [:block/uuid child-uuid])))) + (worker-undo-redo/redo test-repo) + (is (= "v2" (:block/title (d/entity @conn [:block/uuid child-uuid])))) + (worker-undo-redo/redo test-repo) + (is (= "v3" (:block/title (d/entity @conn [:block/uuid child-uuid]))))))) + +(deftest repeated-save-block-op-content-undo-redo-test + (testing "sequential save-block ops preserve undo/redo order" + (worker-undo-redo/clear-history! test-repo) + (let [conn (worker-state/get-datascript-conn test-repo) + {:keys [child-uuid]} (seed-page-parent-child!)] + (doseq [title ["foo" "foo bar"]] + (outliner-op/apply-ops! conn + [[:save-block [{:block/uuid child-uuid + :block/title title} {}]]] + (local-tx-meta {:client-id "test-client"}))) + (is (= "foo bar" (:block/title (d/entity @conn [:block/uuid child-uuid])))) + (worker-undo-redo/undo test-repo) + (is (= "foo" (:block/title (d/entity @conn [:block/uuid child-uuid])))) + (worker-undo-redo/redo test-repo) + (is (= "foo bar" (:block/title (d/entity @conn [:block/uuid child-uuid]))))))) + +(deftest save-two-blocks-undo-targets-latest-block-test + (testing "undo after saving two blocks reverts the latest saved block first" + (worker-undo-redo/clear-history! test-repo) + (let [conn (worker-state/get-datascript-conn test-repo) + {:keys [parent-uuid child-uuid]} (seed-page-parent-child!)] + (save-block-title! conn parent-uuid "parent updated") + (save-block-title! conn child-uuid "child updated") + (worker-undo-redo/undo test-repo) + (is (= "parent updated" (:block/title (d/entity @conn [:block/uuid parent-uuid])))) + (is (= "child" (:block/title (d/entity @conn [:block/uuid child-uuid])))) + (worker-undo-redo/undo test-repo) + (is (= "parent" (:block/title (d/entity @conn [:block/uuid parent-uuid]))))))) + +(deftest new-local-save-clears-redo-stack-test + (testing "a new local save clears redo history" + (worker-undo-redo/clear-history! test-repo) + (let [conn (worker-state/get-datascript-conn test-repo) + {:keys [child-uuid]} (seed-page-parent-child!)] + (save-block-title! conn child-uuid "v1") + (save-block-title! conn child-uuid "v2") + (worker-undo-redo/undo test-repo) + (is (= "v1" (:block/title (d/entity @conn [:block/uuid child-uuid])))) + (save-block-title! conn child-uuid "v3") + (is (= ::worker-undo-redo/empty-redo-stack + (worker-undo-redo/redo test-repo))) + (is (= "v3" (:block/title (d/entity @conn [:block/uuid child-uuid]))))))) diff --git a/yarn.lock b/yarn.lock index dfe9b488c8..a5b1528b6b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3569,7 +3569,7 @@ fdir@^6.5.0: resolved "https://registry.yarnpkg.com/fdir/-/fdir-6.5.0.tgz#ed2ab967a331ade62f18d077dae192684d50d350" integrity sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg== -fflate@^0.4.8: +fflate@^0.4.1: version "0.4.8" resolved "https://registry.yarnpkg.com/fflate/-/fflate-0.4.8.tgz#f90b82aefbd8ac174213abb338bd7ef848f0f5ae" integrity sha512-FJqqoDBR00Mdj9ppamLa/Y7vxm+PRmNWA67N846RvsoYVMKB4q3y/de5PA7gUmRMYK/8CMz2GDZQmCRN1wBcWA== @@ -6565,18 +6565,12 @@ postcss@^8.4.23, postcss@^8.5.8: picocolors "^1.1.1" source-map-js "^1.2.1" -posthog-js@1.141.0: - version "1.141.0" - resolved "https://registry.yarnpkg.com/posthog-js/-/posthog-js-1.141.0.tgz#bf85c935aa6b12a87f73f576adb6dd943888e675" - integrity sha512-EuVCq86izPX7+1eD/o87lF1HalRD6Nk5735w+FKZJ5KAPwoQjr5FCaL2V8Ed36DyQQz4gQj+PEx5i6DFKCiDzA== +posthog-js@1.10.0: + version "1.10.0" + resolved "https://registry.yarnpkg.com/posthog-js/-/posthog-js-1.10.0.tgz#4d86360161170d37c249482f016acac2f4b6d978" + integrity sha512-WbcPRRX62XTq2F2lbakuDK6/HPAJ43gkuPeM4vU/hoC7WICAc+gZJaXZFy8zY25r/5GZPWUhhW8KrbL0aZ11XQ== dependencies: - fflate "^0.4.8" - preact "^10.19.3" - -preact@^10.19.3: - version "10.29.0" - resolved "https://registry.yarnpkg.com/preact/-/preact-10.29.0.tgz#a6e5858670b659c4d471c6fea232233e03b403e8" - integrity sha512-wSAGyk2bYR1c7t3SZ3jHcM6xy0lcBcDel6lODcs9ME6Th++Dx2KU+6D3HD8wMMKGA8Wpw7OMd3/4RGzYRpzwRg== + fflate "^0.4.1" prebuild-install@^7.1.1: version "7.1.3"