mirror of
https://github.com/logseq/logseq.git
synced 2026-05-29 23:19:38 +00:00
fix: validate and repair invalid db blocks
This commit is contained in:
Tienson Qin
7
bb.edn
7
bb.edn
@@ -114,6 +114,13 @@
|
||||
"pnpm exec nbb-logseq script/dump_datoms.cljs"
|
||||
*command-line-args*)}
|
||||
|
||||
dev:validate-and-fix
|
||||
{:doc "Validate a DB graph and run shared invalid data repairs"
|
||||
:requires ([babashka.fs :as fs])
|
||||
:task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
|
||||
"pnpm exec nbb-logseq -cp src:../../src/main:../cli/src:../graph-parser/src:../outliner/src:../db-sync/src script/validate_and_fix.cljs"
|
||||
*command-line-args*)}
|
||||
|
||||
dev:diff-datoms
|
||||
logseq.tasks.dev/diff-datoms
|
||||
|
||||
|
||||
84
deps/db/script/validate_and_fix.cljs
vendored
Normal file
84
deps/db/script/validate_and_fix.cljs
vendored
Normal file
@@ -0,0 +1,84 @@
|
||||
(ns validate-and-fix
|
||||
"Validate a DB graph and run frontend.worker.db.validate-fix repairs."
|
||||
(:require [babashka.cli :as cli]
|
||||
[clojure.pprint :as pprint]
|
||||
[frontend.worker.db.validate-fix :as validate-fix]
|
||||
[logseq.db.common.sqlite-cli :as sqlite-cli]
|
||||
[nbb.core :as nbb]))
|
||||
|
||||
(def spec
|
||||
{:help {:alias :h
|
||||
:desc "Print help"}
|
||||
:no-fix {:desc "Only validate; do not run invalid data repairs"}
|
||||
:verbose {:alias :v
|
||||
:desc "Print invalid entity ids and final summary"}})
|
||||
|
||||
(defn- usage []
|
||||
(str "Usage: bb dev:validate-and-fix GRAPH [OPTIONS]\n\n"
|
||||
"GRAPH can be a DB graph name or a sqlite file path, e.g. ~/Downloads/db.sqlite.\n\n"
|
||||
"Options:\n"
|
||||
(cli/format-opts {:spec spec})))
|
||||
|
||||
(defn- expand-home
|
||||
[path]
|
||||
(if (.startsWith path "~/")
|
||||
(str js/process.env.HOME (.slice path 1))
|
||||
path))
|
||||
|
||||
(defn- print-result!
|
||||
[graph {:keys [errors datom-count invalid-entity-ids]} options]
|
||||
(if (seq errors)
|
||||
(do
|
||||
(println "Graph" (pr-str graph) "is still invalid.")
|
||||
(println "Found" (count errors)
|
||||
(if (= 1 (count errors)) "entity" "entities")
|
||||
"with errors.")
|
||||
(when (:verbose options)
|
||||
(println "Invalid entity ids:")
|
||||
(pprint/pprint invalid-entity-ids)
|
||||
(println "Errors:")
|
||||
(pprint/pprint errors))
|
||||
(js/process.exit 1))
|
||||
(do
|
||||
(println "Valid!")
|
||||
(when (:verbose options)
|
||||
(println "Datoms:" datom-count)))))
|
||||
|
||||
(defn- open-graph!
|
||||
[graph]
|
||||
(apply sqlite-cli/open-sqlite-datascript! (sqlite-cli/->open-db-args graph)))
|
||||
|
||||
(defn- close-sqlite!
|
||||
[sqlite]
|
||||
(when sqlite
|
||||
(.close sqlite)))
|
||||
|
||||
(defn- validate-open-graph!
|
||||
[graph options]
|
||||
(let [{:keys [sqlite conn]} (open-graph! graph)]
|
||||
(try
|
||||
(if (:no-fix options)
|
||||
(let [{:keys [errors] :as result} (validate-fix/validate-db-result @conn)]
|
||||
(validate-fix/log-validation-errors! errors)
|
||||
result)
|
||||
(validate-fix/validate-and-fix-invalid-blocks! conn))
|
||||
(finally
|
||||
(close-sqlite! sqlite)))))
|
||||
|
||||
(defn validate-and-fix-graph!
|
||||
[graph options]
|
||||
(print-result! graph
|
||||
(validate-open-graph! (expand-home graph) options)
|
||||
options))
|
||||
|
||||
(defn -main
|
||||
[args]
|
||||
(let [{options :opts args' :args} (cli/parse-args args {:spec spec})
|
||||
graph (first args')]
|
||||
(when (or (:help options) (nil? graph))
|
||||
(println (usage))
|
||||
(js/process.exit (if (:help options) 0 1)))
|
||||
(validate-and-fix-graph! graph options)))
|
||||
|
||||
(when (= nbb/*file* (nbb/invoked-file))
|
||||
(-main *command-line-args*))
|
||||
@@ -183,8 +183,14 @@
|
||||
(update acc e update a (fnil conj #{}) v)
|
||||
;; If there's already a val, don't clobber it and automatically start collecting it as a :many
|
||||
(if-let [existing-val (get-in acc [e a])]
|
||||
(if (set? existing-val)
|
||||
(cond
|
||||
(= existing-val v)
|
||||
acc
|
||||
|
||||
(set? existing-val)
|
||||
(update acc e assoc a (conj existing-val v))
|
||||
|
||||
:else
|
||||
(update acc e assoc a #{existing-val v}))
|
||||
(update acc e assoc a v))))
|
||||
{}
|
||||
|
||||
67
deps/outliner/src/logseq/outliner/recycle.cljs
vendored
67
deps/outliner/src/logseq/outliner/recycle.cljs
vendored
@@ -12,9 +12,17 @@
|
||||
(def ^:private retention-ms (* 30 24 3600 1000))
|
||||
(def gc-interval-ms (* 24 3600 1000))
|
||||
|
||||
(defn- has-attr?
|
||||
[db entity attr]
|
||||
(boolean
|
||||
(some #(= attr (:a %))
|
||||
(d/datoms db :eavt (:db/id entity)))))
|
||||
|
||||
(defn- recycled?
|
||||
[entity]
|
||||
(some? (:logseq.property/deleted-at entity)))
|
||||
[db entity]
|
||||
(or (ldb/recycled? entity)
|
||||
(and (= recycle-page-title (:block/title (:block/page entity)))
|
||||
(has-attr? db entity :logseq.property.recycle/original-page))))
|
||||
|
||||
(defn- build-recycle-page-tx
|
||||
[db-id]
|
||||
@@ -110,6 +118,22 @@
|
||||
[db tx-data]
|
||||
(distinct (concat tx-data (delete-blocks/update-refs-history db tx-data {}))))
|
||||
|
||||
(defn- retract-entity-tx-data
|
||||
[db db-id]
|
||||
(map (fn [datom]
|
||||
[:db/retract (:e datom) (:a datom) (:v datom)])
|
||||
(d/datoms db :eavt db-id)))
|
||||
|
||||
(defn- permanently-delete-entity-tx-data
|
||||
[db root]
|
||||
(->> (if (ldb/page? root)
|
||||
(page-tree-ids db root)
|
||||
(map :db/id (block-subtree db root)))
|
||||
(mapcat #(retract-entity-tx-data db %))
|
||||
seq
|
||||
(with-delete-cleanup-tx db)
|
||||
seq))
|
||||
|
||||
(defn recycle-blocks-tx-data
|
||||
[db blocks {:keys [deleted-by-uuid now-ms]}]
|
||||
(let [{:keys [page page-id tx-data]} (ensure-recycle-page db)
|
||||
@@ -174,7 +198,7 @@
|
||||
(let [original-parent (resolve-entity db (:logseq.property.recycle/original-parent root))
|
||||
original-page (resolve-entity db (:logseq.property.recycle/original-page root))
|
||||
parent-valid? (and original-parent
|
||||
(not (recycled? original-parent))
|
||||
(not (recycled? db original-parent))
|
||||
(d/entity db (:db/id original-parent)))]
|
||||
(cond
|
||||
(ldb/page? root)
|
||||
@@ -191,7 +215,7 @@
|
||||
|
||||
(and original-page
|
||||
(d/entity db (:db/id original-page))
|
||||
(not (recycled? original-page)))
|
||||
(not (recycled? db original-page)))
|
||||
{:parent original-page
|
||||
:page original-page
|
||||
:order (restore-order original-page)}
|
||||
@@ -236,25 +260,28 @@
|
||||
|
||||
(defn ^:api permanently-delete-tx-data
|
||||
[db root]
|
||||
(when (and root (recycled? root))
|
||||
(some->> (if (ldb/page? root)
|
||||
(keep (fn [id]
|
||||
(some-> (d/entity db id) :block/uuid))
|
||||
(page-tree-ids db root))
|
||||
(keep :block/uuid (block-subtree db root)))
|
||||
(map (fn [block-uuid]
|
||||
[:db/retractEntity [:block/uuid block-uuid]]))
|
||||
distinct
|
||||
seq
|
||||
(with-delete-cleanup-tx db)
|
||||
seq)))
|
||||
(when (and root (recycled? db root))
|
||||
(permanently-delete-entity-tx-data db root)))
|
||||
|
||||
(defn ^:api permanently-delete!
|
||||
[conn root-uuid]
|
||||
(when-let [root (d/entity @conn [:block/uuid root-uuid])]
|
||||
(when-let [tx-data (permanently-delete-tx-data @conn root)]
|
||||
(ldb/transact! conn tx-data {:outliner-op :recycle-delete-permanently})
|
||||
true)))
|
||||
(let [root-id (:db/id root)]
|
||||
(loop [deleted? false]
|
||||
(if-let [root' (d/entity @conn root-id)]
|
||||
(let [datom-count-before (count (d/datoms @conn :eavt root-id))]
|
||||
(if (zero? datom-count-before)
|
||||
deleted?
|
||||
(if-let [tx-data (permanently-delete-entity-tx-data @conn root')]
|
||||
(do
|
||||
(ldb/transact! conn tx-data {:outliner-op :recycle-delete-permanently
|
||||
:skip-validate-db? true})
|
||||
(when (>= (count (d/datoms @conn :eavt root-id)) datom-count-before)
|
||||
(throw (ex-info "Failed to make progress while permanently deleting recycled block"
|
||||
{:root-id root-id})))
|
||||
(recur true))
|
||||
deleted?)))
|
||||
deleted?)))))
|
||||
|
||||
(defn- gc-tx-data
|
||||
[db {:keys [now-ms] :or {now-ms (common-util/time-ms)}}]
|
||||
@@ -267,7 +294,7 @@
|
||||
[(<= ?deleted-at ?cutoff)]]
|
||||
db cutoff)
|
||||
(map #(d/entity db %))
|
||||
(filter recycled?)
|
||||
(filter #(recycled? db %))
|
||||
(mapcat (fn [entity]
|
||||
(if (ldb/page? entity)
|
||||
(map (fn [id] [:db/retractEntity id]) (page-tree-ids db entity))
|
||||
|
||||
@@ -4,188 +4,12 @@
|
||||
[datascript.core :as d]
|
||||
[datascript.impl.entity :as de]
|
||||
[frontend.worker.db.migrate :as db-migrate]
|
||||
[frontend.worker.db.validate-fix :as validate-fix]
|
||||
[frontend.worker.shared-service :as shared-service]
|
||||
[logseq.db :as ldb]
|
||||
[logseq.db-sync.checksum :as sync-checksum]
|
||||
[logseq.db.frontend.class :as db-class]
|
||||
[logseq.db.frontend.validate :as db-validate]))
|
||||
|
||||
(defn- get-property-by-title
|
||||
[db title]
|
||||
(when title
|
||||
(some->> (first (ldb/page-exists? db title [:logseq.class/Property]))
|
||||
(d/entity db))))
|
||||
|
||||
(defn- block-missing-uuid?
|
||||
[entity]
|
||||
(and (nil? (:block/uuid entity))
|
||||
(string? (:block/title entity))
|
||||
(:block/page entity)
|
||||
(:block/parent entity)
|
||||
(string? (:block/order entity))
|
||||
(int? (:block/created-at entity))
|
||||
(int? (:block/updated-at entity))))
|
||||
|
||||
(defn- normal-page-missing-updated-at?
|
||||
[entity dispatch-key]
|
||||
(and (= dispatch-key :normal-page)
|
||||
(nil? (:block/updated-at entity))
|
||||
(int? (:block/created-at entity))))
|
||||
|
||||
(defn- ^:large-vars/cleanup-todo fix-invalid-blocks!
|
||||
[conn errors]
|
||||
(let [db @conn
|
||||
fix-tx-data (mapcat
|
||||
(fn [{:keys [entity dispatch-key]}]
|
||||
(let [entity (d/entity db (:db/id entity))]
|
||||
(cond
|
||||
(some? (:logseq.property/parent entity))
|
||||
[[:db/retract (:db/id entity) :logseq.property/parent]]
|
||||
(some? (:hide? entity))
|
||||
[[:db/retract (:db/id entity) :hide?]]
|
||||
(some? (:public? entity))
|
||||
[[:db/retract (:db/id entity) :public?]]
|
||||
(some? (:block/pre-block? entity))
|
||||
[[:db/retract (:db/id entity) :block/pre-block?]]
|
||||
(some? (:logseq.property.embedding/hnsw-label entity))
|
||||
[[:db/retract (:db/id entity) :logseq.property.embedding/hnsw-label]]
|
||||
(some? (:logseq.property.embedding/hnsw-label-updated-at entity))
|
||||
[[:db/retract (:db/id entity) :logseq.property.embedding/hnsw-label-updated-at]]
|
||||
(normal-page-missing-updated-at? entity dispatch-key)
|
||||
[[:db/add (:db/id entity) :block/updated-at (:block/created-at entity)]]
|
||||
(and (= "External URL" (:block/title entity))
|
||||
(nil? (:block/tags entity)))
|
||||
[[:db/retractEntity (:db/id entity)]]
|
||||
(and (ldb/property? entity)
|
||||
(some #(= (:db/ident %) :logseq.class/Tag) (:block/tags entity)))
|
||||
[[:db/retract (:db/id entity) :block/tags :logseq.class/Tag]]
|
||||
(and (:db/ident entity)
|
||||
(db-class/user-class-namespace? (str (:db/ident entity)))
|
||||
(not (:logseq.property/built-in? entity))
|
||||
(not (ldb/class? entity)))
|
||||
[[:db/add (:db/id entity) :block/tags :logseq.class/Tag]
|
||||
[:db/retract (:db/id entity) :block/tags :logseq.class/Page]]
|
||||
(and (ldb/class? entity) (:kv/value entity))
|
||||
[[:db/retract (:db/id entity) :kv/value]]
|
||||
(and (ldb/property? entity)
|
||||
(:logseq.property.class/extends entity))
|
||||
(mapv (fn [class]
|
||||
[:db/retract (:db/id entity) :logseq.property.class/extends (:db/id class)])
|
||||
(:logseq.property.class/extends entity))
|
||||
(:block/level entity)
|
||||
[[:db/retract (:db/id entity) :block/level]]
|
||||
;; missing :db/ident
|
||||
(and (ldb/class? entity) (nil? (:db/ident entity)) (:block/title entity))
|
||||
[[:db/add (:db/id entity) :db/ident (db-class/create-user-class-ident-from-name db (:block/title entity))]]
|
||||
(and
|
||||
(= (:block/title (:logseq.property/created-from-property entity)) "description")
|
||||
(nil? (:block/page entity)))
|
||||
(let [property-id (:db/id (:logseq.property/created-from-property entity))]
|
||||
[[:db/add (:db/id entity) :block/page property-id]
|
||||
[:db/add (:db/id entity) :block/parent property-id]])
|
||||
(and (:db/ident entity)
|
||||
(:logseq.property/built-in? entity)
|
||||
(:block/parent entity))
|
||||
[[:db/retract (:db/id entity) :block/parent]]
|
||||
(:block/format entity)
|
||||
[[:db/retract (:db/id entity) :block/format]]
|
||||
(= :whiteboard-shape (:logseq.property/ls-type entity))
|
||||
[[:db/retractEntity (:db/id entity)]]
|
||||
(and (:block/page entity) (not (:block/parent entity)))
|
||||
[[:db/add (:db/id entity) :block/parent (:db/id (:block/page entity))]]
|
||||
(and (:logseq.property/created-by-ref entity)
|
||||
(not (de/entity? (:logseq.property/created-by-ref entity))))
|
||||
[[:db/retractEntity (:db/id entity)]]
|
||||
(block-missing-uuid? entity)
|
||||
[[:db/add (:db/id entity) :block/uuid (random-uuid)]]
|
||||
(vector? (:logseq.property/value entity))
|
||||
[[:db/retractEntity (:db/id entity)]]
|
||||
(and (:block/tx-id entity) (nil? (:block/title entity)))
|
||||
[[:db/retractEntity (:db/id entity)]]
|
||||
(and (:block/title entity) (nil? (:block/page entity)) (nil? (:block/parent entity)) (nil? (:block/name entity)))
|
||||
[[:db/retractEntity (:db/id entity)]]
|
||||
(= :block/path-refs (:db/ident entity))
|
||||
(try
|
||||
(db-migrate/remove-block-path-refs db)
|
||||
(catch :default _e
|
||||
nil))
|
||||
(not-every? (fn [e] (ldb/class? e)) (:block/tags entity))
|
||||
(let [non-tags (remove ldb/class? (:block/tags entity))]
|
||||
(map (fn [tag]
|
||||
[:db/retract (:db/id entity) :block/tags (:db/id tag)]) non-tags))
|
||||
(and (= dispatch-key :normal-page) (:block/page entity))
|
||||
[[:db/retract (:db/id entity) :block/page]]
|
||||
(and (= dispatch-key :block) (nil? (:block/title entity)))
|
||||
[[:db/retractEntity (:db/id entity)]]
|
||||
(and (= dispatch-key :block) (nil? (:block/page entity)))
|
||||
(let [latest-journal-id (:db/id (first (ldb/get-latest-journals db)))
|
||||
page-id (:db/id (:block/page (:block/parent entity)))]
|
||||
(cond
|
||||
page-id
|
||||
[[:db/add (:db/id entity) :block/page page-id]]
|
||||
latest-journal-id
|
||||
[[:db/add (:db/id entity) :block/page latest-journal-id]
|
||||
[:db/add (:db/id entity) :block/parent latest-journal-id]]
|
||||
:else
|
||||
(js/console.error (str "Don't know where to put the block " (:db/id entity)))))
|
||||
|
||||
(and (= dispatch-key :block)
|
||||
(some (fn [k] (= "user.class" (namespace k))) (keys (:logseq.property.table/sized-columns entity))))
|
||||
(let [new-value (->> (keep (fn [[k v]]
|
||||
(if (= "user.class" (namespace k))
|
||||
(when-let [property (get-property-by-title db (:block/title (d/entity db k)))]
|
||||
[(:db/ident property) v])
|
||||
[k v]))
|
||||
(:logseq.property.table/sized-columns entity))
|
||||
(into {}))]
|
||||
[[:db/add (:db/id entity) :logseq.property.table/sized-columns new-value]])
|
||||
|
||||
(some (fn [k] (= "block.temp" (namespace k))) (keys entity))
|
||||
(let [ks (filter (fn [k] (= "block.temp" (namespace k))) (keys entity))]
|
||||
(mapv (fn [k] [:db/retract (:db/id entity) k]) ks))
|
||||
(and (not (:block/page entity)) (not (:block/parent entity)) (not (:block/name entity)))
|
||||
[[:db/retractEntity (:db/id entity)]]
|
||||
(and (= dispatch-key :property-value-block) (:block/title entity))
|
||||
[[:db/retract (:db/id entity) :block/title]]
|
||||
(and (ldb/class? entity) (not (:logseq.property.class/extends entity))
|
||||
(not= (:db/ident entity) :logseq.class/Root))
|
||||
[[:db/add (:db/id entity) :logseq.property.class/extends :logseq.class/Root]]
|
||||
(and (or (ldb/class? entity) (ldb/property? entity)) (ldb/internal-page? entity))
|
||||
[[:db/retract (:db/id entity) :block/tags :logseq.class/Page]]
|
||||
|
||||
(and (:logseq.property.asset/remote-metadata entity) (nil? (:logseq.property.asset/type entity)))
|
||||
[[:db/retractEntity (:db/id entity)]]
|
||||
|
||||
:else
|
||||
nil)))
|
||||
errors)
|
||||
class-as-properties (concat
|
||||
(mapcat
|
||||
(fn [ident]
|
||||
(->> (d/datoms db :avet ident)
|
||||
(mapcat (fn [d]
|
||||
(let [entity (d/entity db (:v d))]
|
||||
(when (ldb/class? entity)
|
||||
(if-let [property (get-property-by-title db (:block/title entity))]
|
||||
[[:db/retract (:e d) (:a d) (:v d)]
|
||||
[:db/add (:e d) (:a d) (:db/id property)]]
|
||||
[[:db/retract (:e d) (:a d) (:v d)]])))))))
|
||||
[:logseq.property.view/group-by-property :logseq.property.table/pinned-columns])
|
||||
(->> (d/datoms db :eavt)
|
||||
(filter (fn [d] (= (namespace (:a d)) "user.class")))
|
||||
(mapcat (fn [d]
|
||||
(let [class-title (:block/title (d/entity db (:a d)))
|
||||
property (get-property-by-title db class-title)]
|
||||
(if property
|
||||
[[:db/retract (:e d) (:a d) (:v d)]
|
||||
[:db/add (:e d) (:db/ident property) (:v d)]]
|
||||
[[:db/retract (:e d) (:a d) (:v d)]]))))))
|
||||
tx-data (concat fix-tx-data
|
||||
class-as-properties)]
|
||||
(when (seq tx-data)
|
||||
(let [tx-report (d/transact! conn tx-data {:fix-db? true})]
|
||||
(seq (:tx-data tx-report))))))
|
||||
|
||||
(defn- fix-num-prefix-db-idents!
|
||||
"Fix invalid db/ident keywords for both classes and properties"
|
||||
[conn]
|
||||
@@ -261,30 +85,6 @@
|
||||
:db/index true}]
|
||||
{:fix-db? true})))
|
||||
|
||||
(defn- validate-db-result
|
||||
[db]
|
||||
(let [{:keys [errors datom-count entities]} (db-validate/validate-db db)
|
||||
invalid-entity-ids (distinct (map (fn [e] (:db/id (:entity e))) errors))]
|
||||
{:errors errors
|
||||
:datom-count datom-count
|
||||
:entities entities
|
||||
:invalid-entity-ids invalid-entity-ids}))
|
||||
|
||||
(defn- log-validation-errors!
|
||||
[errors]
|
||||
(doseq [error errors]
|
||||
(prn :debug
|
||||
:entity (:entity error)
|
||||
:error (dissoc error :entity))))
|
||||
|
||||
(defn- validate-and-fix-invalid-blocks!
|
||||
[conn]
|
||||
(loop [{:keys [errors] :as result} (validate-db-result @conn)]
|
||||
(log-validation-errors! errors)
|
||||
(if (and (seq errors) (fix-invalid-blocks! conn errors))
|
||||
(recur (validate-db-result @conn))
|
||||
result)))
|
||||
|
||||
(defn validate-db
|
||||
[conn & {:keys [fix] :or {fix true}}]
|
||||
(when fix
|
||||
@@ -294,11 +94,14 @@
|
||||
(fix-non-closed-values! conn)
|
||||
(fix-num-prefix-db-idents! conn))
|
||||
|
||||
(let [{:keys [errors datom-count entities invalid-entity-ids]}
|
||||
(let [{:keys [errors datom-count entities invalid-entity-ids]
|
||||
:as result}
|
||||
(if fix
|
||||
(validate-and-fix-invalid-blocks! conn)
|
||||
(let [{:keys [errors] :as result} (validate-db-result @conn)]
|
||||
(log-validation-errors! errors)
|
||||
(validate-fix/validate-and-fix-invalid-blocks!
|
||||
conn
|
||||
{:remove-block-path-refs-fn db-migrate/remove-block-path-refs})
|
||||
(let [{:keys [errors] :as result} (validate-fix/validate-db-result @conn)]
|
||||
(validate-fix/log-validation-errors! errors)
|
||||
result))
|
||||
db @conn]
|
||||
|
||||
@@ -316,9 +119,10 @@
|
||||
(shared-service/broadcast-to-clients! :notification
|
||||
[(str "Your graph is valid! " (assoc (db-validate/graph-counts db entities) :datoms datom-count))
|
||||
:success false]))
|
||||
{:errors errors
|
||||
:datom-count datom-count
|
||||
:invalid-entity-ids invalid-entity-ids}))
|
||||
(assoc result
|
||||
:errors errors
|
||||
:datom-count datom-count
|
||||
:invalid-entity-ids invalid-entity-ids)))
|
||||
|
||||
(defn recompute-checksum-diagnostics
|
||||
[_repo conn {:keys [local-checksum remote-checksum] :as _sync-diagnostics}]
|
||||
|
||||
330
src/main/frontend/worker/db/validate_fix.cljs
Normal file
330
src/main/frontend/worker/db/validate_fix.cljs
Normal file
@@ -0,0 +1,330 @@
|
||||
(ns frontend.worker.db.validate-fix
|
||||
"Script-safe DB validation repairs shared by worker validation and dev tasks."
|
||||
(:require [clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[datascript.impl.entity :as de]
|
||||
[logseq.db :as ldb]
|
||||
[logseq.db.frontend.class :as db-class]
|
||||
[logseq.db.frontend.validate :as db-validate]))
|
||||
|
||||
(defn- get-property-by-title
|
||||
[db title]
|
||||
(when title
|
||||
(some->> (first (ldb/page-exists? db title [:logseq.class/Property]))
|
||||
(d/entity db))))
|
||||
|
||||
(defn- block-missing-uuid?
|
||||
[entity]
|
||||
(and (nil? (:block/uuid entity))
|
||||
(string? (:block/title entity))
|
||||
(:block/page entity)
|
||||
(:block/parent entity)
|
||||
(string? (:block/order entity))
|
||||
(int? (:block/created-at entity))
|
||||
(int? (:block/updated-at entity))))
|
||||
|
||||
(defn- normal-page-missing-updated-at?
|
||||
[entity dispatch-key]
|
||||
(and (= dispatch-key :normal-page)
|
||||
(nil? (:block/updated-at entity))
|
||||
(int? (:block/created-at entity))))
|
||||
|
||||
(defn- on-recycle-page?
|
||||
[entity]
|
||||
(= "recycle" (:block/name (:block/page entity))))
|
||||
|
||||
(defn- has-attr?
|
||||
[db entity attr]
|
||||
(boolean
|
||||
(some #(= attr (:a %))
|
||||
(d/datoms db :eavt (:db/id entity)))))
|
||||
|
||||
(defn- invalid-recycled-block?
|
||||
[db entity dispatch-key]
|
||||
(and (= dispatch-key :block)
|
||||
(or (ldb/recycled? entity)
|
||||
(and (on-recycle-page? entity)
|
||||
(has-attr? db entity :logseq.property.recycle/original-page)))))
|
||||
|
||||
(defn- resolve-entity-id
|
||||
[db entity]
|
||||
(letfn [(valid-entity-id [id]
|
||||
(when (and (int? id)
|
||||
(<= id 2147483647))
|
||||
id))]
|
||||
(or (valid-entity-id (:db/id entity))
|
||||
(when (de/entity? entity)
|
||||
(valid-entity-id (.-eid ^js entity)))
|
||||
(some (fn [[property value]]
|
||||
(when-let [attr (:db/ident property)]
|
||||
(:e (first (d/datoms db :avet attr value)))))
|
||||
(:block/properties entity))
|
||||
(some->> (re-find #":db/id (\d+)" (pr-str entity))
|
||||
second
|
||||
parse-long
|
||||
valid-entity-id))))
|
||||
|
||||
(defn- invalid-normal-page-alias-datoms
|
||||
[db entity dispatch-key]
|
||||
(let [eid (resolve-entity-id db entity)]
|
||||
(when (and (= dispatch-key :normal-page)
|
||||
eid)
|
||||
(seq
|
||||
(remove
|
||||
(fn [datom]
|
||||
(ldb/page? (d/entity db (:v datom))))
|
||||
(d/datoms db :eavt eid :block/alias))))))
|
||||
|
||||
(defn- invalid-recycle-fragment?
|
||||
[entity dispatch-key]
|
||||
(and (nil? dispatch-key)
|
||||
(or (:logseq.property.recycle/original-order entity)
|
||||
(:logseq.property.recycle/original-page entity)
|
||||
(:logseq.property.recycle/original-parent entity)
|
||||
(:logseq.property/deleted-at entity)
|
||||
(string/includes? (pr-str entity) ":logseq.property.recycle/"))))
|
||||
|
||||
(defn- invalid-orphan-fragment?
|
||||
[entity dispatch-key]
|
||||
(and (nil? dispatch-key)
|
||||
(not (block-missing-uuid? entity))
|
||||
(or (:logseq.property.embedding/hnsw-label-updated-at entity)
|
||||
(:logseq.property/deleted-at entity)
|
||||
(:block/uuid entity)
|
||||
(string/includes? (pr-str entity) ":logseq.property.embedding/hnsw-label-updated-at")
|
||||
(string/includes? (pr-str entity) ":logseq.property/deleted-at"))))
|
||||
|
||||
(defn- invalid-fragment-error?
|
||||
[{:keys [entity dispatch-key]}]
|
||||
(or (invalid-recycle-fragment? entity dispatch-key)
|
||||
(invalid-orphan-fragment? entity dispatch-key)))
|
||||
|
||||
(defn invalid-fragment-ids
|
||||
[db errors]
|
||||
(distinct
|
||||
(keep (fn [{error-entity-id :entity-id :as error}]
|
||||
(when (invalid-fragment-error? error)
|
||||
(or error-entity-id (resolve-entity-id db (:entity error)))))
|
||||
errors)))
|
||||
|
||||
(defn- remove-block-path-refs
|
||||
[db]
|
||||
(if (d/entity db :block/path-refs)
|
||||
(let [remove-datoms (->> (d/datoms db :avet :block/path-refs)
|
||||
(map :e)
|
||||
(distinct)
|
||||
(mapv (fn [id]
|
||||
[:db/retract id :block/path-refs])))]
|
||||
(conj remove-datoms [:db/retractEntity :block/path-refs]))
|
||||
(map (fn [eid]
|
||||
[:db/retract eid :block/path-refs])
|
||||
(d/q '[:find [?e ...]
|
||||
:where
|
||||
[?e :block/path-refs ?v]]
|
||||
db))))
|
||||
|
||||
(defn fix-invalid-blocks!
|
||||
([conn errors]
|
||||
(fix-invalid-blocks! conn errors nil))
|
||||
([conn errors {:keys [remove-block-path-refs-fn]}]
|
||||
(let [db @conn
|
||||
fix-tx-data (mapcat
|
||||
(fn [{error-entity-id :entity-id
|
||||
:keys [entity dispatch-key]}]
|
||||
(when-let [entity-id (or error-entity-id (resolve-entity-id db entity))]
|
||||
(let [error-entity entity
|
||||
entity (d/entity db entity-id)
|
||||
invalid-alias-datoms (invalid-normal-page-alias-datoms db entity dispatch-key)]
|
||||
(cond
|
||||
(invalid-recycle-fragment? error-entity dispatch-key)
|
||||
[[:db/retractEntity entity-id]]
|
||||
(invalid-orphan-fragment? error-entity dispatch-key)
|
||||
[[:db/retractEntity entity-id]]
|
||||
(invalid-recycled-block? db entity dispatch-key)
|
||||
[[:db/retractEntity entity-id]]
|
||||
|
||||
(some? (:logseq.property/parent entity))
|
||||
[[:db/retract entity-id :logseq.property/parent]]
|
||||
(some? (:hide? entity))
|
||||
[[:db/retract entity-id :hide?]]
|
||||
(some? (:public? entity))
|
||||
[[:db/retract entity-id :public?]]
|
||||
(some? (:block/pre-block? entity))
|
||||
[[:db/retract entity-id :block/pre-block?]]
|
||||
(some? (:logseq.property.embedding/hnsw-label entity))
|
||||
[[:db/retract entity-id :logseq.property.embedding/hnsw-label]]
|
||||
(some? (:logseq.property.embedding/hnsw-label-updated-at entity))
|
||||
[[:db/retract entity-id :logseq.property.embedding/hnsw-label-updated-at]]
|
||||
(normal-page-missing-updated-at? entity dispatch-key)
|
||||
[[:db/add entity-id :block/updated-at (:block/created-at entity)]]
|
||||
(and (= "External URL" (:block/title entity))
|
||||
(nil? (:block/tags entity)))
|
||||
[[:db/retractEntity entity-id]]
|
||||
(and (ldb/property? entity)
|
||||
(some #(= (:db/ident %) :logseq.class/Tag) (:block/tags entity)))
|
||||
[[:db/retract entity-id :block/tags :logseq.class/Tag]]
|
||||
(and (:db/ident entity)
|
||||
(db-class/user-class-namespace? (str (:db/ident entity)))
|
||||
(not (:logseq.property/built-in? entity))
|
||||
(not (ldb/class? entity)))
|
||||
[[:db/add entity-id :block/tags :logseq.class/Tag]
|
||||
[:db/retract entity-id :block/tags :logseq.class/Page]]
|
||||
(and (ldb/class? entity) (:kv/value entity))
|
||||
[[:db/retract entity-id :kv/value]]
|
||||
(and (ldb/property? entity)
|
||||
(:logseq.property.class/extends entity))
|
||||
(mapv (fn [class]
|
||||
[:db/retract entity-id :logseq.property.class/extends (:db/id class)])
|
||||
(:logseq.property.class/extends entity))
|
||||
invalid-alias-datoms
|
||||
(mapv (fn [datom]
|
||||
[:db/retract (:e datom) (:a datom) (:v datom)])
|
||||
invalid-alias-datoms)
|
||||
(:block/level entity)
|
||||
[[:db/retract entity-id :block/level]]
|
||||
(and (ldb/class? entity) (nil? (:db/ident entity)) (:block/title entity))
|
||||
[[:db/add entity-id :db/ident (db-class/create-user-class-ident-from-name db (:block/title entity))]]
|
||||
(and
|
||||
(= (:block/title (:logseq.property/created-from-property entity)) "description")
|
||||
(nil? (:block/page entity)))
|
||||
(let [property-id (:db/id (:logseq.property/created-from-property entity))]
|
||||
[[:db/add entity-id :block/page property-id]
|
||||
[:db/add entity-id :block/parent property-id]])
|
||||
(and (:db/ident entity)
|
||||
(:logseq.property/built-in? entity)
|
||||
(:block/parent entity))
|
||||
[[:db/retract entity-id :block/parent]]
|
||||
(:block/format entity)
|
||||
[[:db/retract entity-id :block/format]]
|
||||
(= :whiteboard-shape (:logseq.property/ls-type entity))
|
||||
[[:db/retractEntity entity-id]]
|
||||
(and (:block/page entity) (not (:block/parent entity)))
|
||||
[[:db/add entity-id :block/parent (:db/id (:block/page entity))]]
|
||||
(and (:logseq.property/created-by-ref entity)
|
||||
(not (de/entity? (:logseq.property/created-by-ref entity))))
|
||||
[[:db/retractEntity entity-id]]
|
||||
(block-missing-uuid? entity)
|
||||
[[:db/add entity-id :block/uuid (random-uuid)]]
|
||||
(vector? (:logseq.property/value entity))
|
||||
[[:db/retractEntity entity-id]]
|
||||
(and (:block/tx-id entity) (nil? (:block/title entity)))
|
||||
[[:db/retractEntity entity-id]]
|
||||
(and (:block/title entity) (nil? (:block/page entity)) (nil? (:block/parent entity)) (nil? (:block/name entity)))
|
||||
[[:db/retractEntity entity-id]]
|
||||
(= :block/path-refs (:db/ident entity))
|
||||
(try
|
||||
((or remove-block-path-refs-fn remove-block-path-refs) db)
|
||||
(catch :default _e
|
||||
nil))
|
||||
(not-every? (fn [e] (ldb/class? e)) (:block/tags entity))
|
||||
(let [non-tags (remove ldb/class? (:block/tags entity))]
|
||||
(map (fn [tag]
|
||||
[:db/retract entity-id :block/tags (:db/id tag)]) non-tags))
|
||||
(and (= dispatch-key :normal-page) (:block/page entity))
|
||||
[[:db/retract entity-id :block/page]]
|
||||
(and (= dispatch-key :block) (nil? (:block/title entity)))
|
||||
[[:db/retractEntity entity-id]]
|
||||
(and (= dispatch-key :block) (nil? (:block/page entity)))
|
||||
(let [latest-journal-id (:db/id (first (ldb/get-latest-journals db)))
|
||||
page-id (:db/id (:block/page (:block/parent entity)))]
|
||||
(cond
|
||||
page-id
|
||||
[[:db/add entity-id :block/page page-id]]
|
||||
latest-journal-id
|
||||
[[:db/add entity-id :block/page latest-journal-id]
|
||||
[:db/add entity-id :block/parent latest-journal-id]]
|
||||
:else
|
||||
(js/console.error (str "Don't know where to put the block " entity-id))))
|
||||
|
||||
(and (= dispatch-key :block)
|
||||
(some (fn [k] (= "user.class" (namespace k))) (keys (:logseq.property.table/sized-columns entity))))
|
||||
(let [new-value (->> (keep (fn [[k v]]
|
||||
(if (= "user.class" (namespace k))
|
||||
(when-let [property (get-property-by-title db (:block/title (d/entity db k)))]
|
||||
[(:db/ident property) v])
|
||||
[k v]))
|
||||
(:logseq.property.table/sized-columns entity))
|
||||
(into {}))]
|
||||
[[:db/add entity-id :logseq.property.table/sized-columns new-value]])
|
||||
|
||||
(some (fn [k] (= "block.temp" (namespace k))) (keys entity))
|
||||
(let [ks (filter (fn [k] (= "block.temp" (namespace k))) (keys entity))]
|
||||
(mapv (fn [k] [:db/retract entity-id k]) ks))
|
||||
(and (not (:block/page entity)) (not (:block/parent entity)) (not (:block/name entity)))
|
||||
[[:db/retractEntity entity-id]]
|
||||
(and (= dispatch-key :property-value-block) (:block/title entity))
|
||||
[[:db/retract entity-id :block/title]]
|
||||
(and (ldb/class? entity) (not (:logseq.property.class/extends entity))
|
||||
(not= (:db/ident entity) :logseq.class/Root))
|
||||
[[:db/add entity-id :logseq.property.class/extends :logseq.class/Root]]
|
||||
(and (or (ldb/class? entity) (ldb/property? entity)) (ldb/internal-page? entity))
|
||||
[[:db/retract entity-id :block/tags :logseq.class/Page]]
|
||||
|
||||
(and (:logseq.property.asset/remote-metadata entity) (nil? (:logseq.property.asset/type entity)))
|
||||
[[:db/retractEntity entity-id]]
|
||||
|
||||
:else
|
||||
nil))))
|
||||
errors)
|
||||
class-as-properties (concat
|
||||
(mapcat
|
||||
(fn [ident]
|
||||
(->> (d/datoms db :eavt)
|
||||
(filter (fn [d] (= ident (:a d))))
|
||||
(mapcat (fn [d]
|
||||
(let [entity (d/entity db (:v d))]
|
||||
(when (ldb/class? entity)
|
||||
(if-let [property (get-property-by-title db (:block/title entity))]
|
||||
[[:db/retract (:e d) (:a d) (:v d)]
|
||||
[:db/add (:e d) (:a d) (:db/id property)]]
|
||||
[[:db/retract (:e d) (:a d) (:v d)]])))))))
|
||||
[:logseq.property.view/group-by-property :logseq.property.table/pinned-columns])
|
||||
(->> (d/datoms db :eavt)
|
||||
(filter (fn [d] (= (namespace (:a d)) "user.class")))
|
||||
(mapcat (fn [d]
|
||||
(let [class-title (:block/title (d/entity db (:a d)))
|
||||
property (get-property-by-title db class-title)]
|
||||
(if-let [property-ident (:db/ident property)]
|
||||
[[:db/retract (:e d) (:a d) (:v d)]
|
||||
[:db/add (:e d) property-ident (:v d)]]
|
||||
[[:db/retract (:e d) (:a d) (:v d)]]))))))
|
||||
tx-data (concat fix-tx-data
|
||||
class-as-properties)]
|
||||
(when (seq tx-data)
|
||||
(let [tx-report (d/transact! conn tx-data {:fix-db? true})]
|
||||
(seq (:tx-data tx-report)))))))
|
||||
|
||||
(defn validate-db-result
|
||||
[db]
|
||||
(let [{:keys [errors datom-count entities]} (db-validate/validate-db db)
|
||||
errors' (map (fn [error]
|
||||
(assoc error :entity-id (resolve-entity-id db (:entity error))))
|
||||
errors)
|
||||
invalid-entity-ids (distinct (map :entity-id errors'))]
|
||||
{:errors errors'
|
||||
:datom-count datom-count
|
||||
:entities entities
|
||||
:invalid-entity-ids invalid-entity-ids}))
|
||||
|
||||
(defn log-validation-errors!
|
||||
[errors]
|
||||
(doseq [error errors]
|
||||
(prn :debug
|
||||
:entity (:entity error)
|
||||
:error (dissoc error :entity))))
|
||||
|
||||
(defn validate-and-fix-invalid-blocks!
|
||||
([conn]
|
||||
(validate-and-fix-invalid-blocks! conn nil))
|
||||
([conn opts]
|
||||
(loop [{:keys [errors] :as result} (validate-db-result @conn)
|
||||
seen-validation-states #{}]
|
||||
(log-validation-errors! errors)
|
||||
(let [validation-state (hash (pr-str errors))
|
||||
fix-progress? (and (seq errors)
|
||||
(not (contains? seen-validation-states validation-state))
|
||||
(fix-invalid-blocks! conn errors opts))]
|
||||
(if fix-progress?
|
||||
(recur (validate-db-result @conn)
|
||||
(conj seen-validation-states validation-state))
|
||||
result)))))
|
||||
@@ -2,10 +2,12 @@
|
||||
(:require [cljs.test :refer [deftest is]]
|
||||
[datascript.core :as d]
|
||||
[frontend.worker.db.validate :as worker-db-validate]
|
||||
[frontend.worker.db.validate-fix :as validate-fix]
|
||||
[frontend.worker.shared-service :as shared-service]
|
||||
[logseq.db.frontend.schema :as db-schema]
|
||||
[logseq.db.frontend.validate :as db-validate]
|
||||
[logseq.db.sqlite.create-graph :as sqlite-create-graph]))
|
||||
[logseq.db.sqlite.create-graph :as sqlite-create-graph]
|
||||
[logseq.outliner.recycle :as recycle]))
|
||||
|
||||
(defn- create-db-graph-conn
|
||||
[]
|
||||
@@ -80,3 +82,143 @@
|
||||
(is (nil? (:logseq.property.class/extends property)))
|
||||
(is (nil? (:kv/value class)))
|
||||
(is (empty? (:errors (worker-db-validate/validate-db conn))))))))
|
||||
|
||||
(deftest validate-db-deletes-invalid-recycled-blocks
|
||||
(let [conn (create-db-graph-conn)
|
||||
page-id (get (:tempids
|
||||
(d/transact! conn [{:db/id "page"
|
||||
:block/uuid (random-uuid)
|
||||
:block/created-at 1
|
||||
:block/updated-at 1
|
||||
:block/name "test page"
|
||||
:block/title "Test Page"
|
||||
:block/tags :logseq.class/Page}]))
|
||||
"page")
|
||||
block-id (get (:tempids
|
||||
(d/transact! conn [{:db/id "block"
|
||||
:block/uuid (random-uuid)
|
||||
:block/created-at 1
|
||||
:block/updated-at 2
|
||||
:block/page page-id
|
||||
:block/parent page-id
|
||||
:block/order "a0"
|
||||
:block/title "Deleted block"}]))
|
||||
"block")
|
||||
block (d/entity @conn block-id)]
|
||||
(d/transact! conn (recycle/recycle-blocks-tx-data @conn [block] {}) {:outliner-op :delete-blocks})
|
||||
(d/transact! conn [[:db/add block-id :block/pre-block? true]])
|
||||
(is (seq (:errors (db-validate/validate-db @conn))))
|
||||
(with-redefs [shared-service/broadcast-to-clients! (fn [& _args] nil)]
|
||||
(is (empty? (:errors (worker-db-validate/validate-db conn))))
|
||||
(is (nil? (d/entity @conn block-id))))))
|
||||
|
||||
(deftest validate-db-deletes-invalid-recycled-fragments
|
||||
(let [conn (create-db-graph-conn)
|
||||
tempids (:tempids
|
||||
(d/transact! conn [{:db/id "recycle-fragment"
|
||||
:logseq.property.embedding/hnsw-label-updated-at 0
|
||||
:logseq.property.recycle/original-order "a2b"}
|
||||
{:db/id "orphan-fragment"
|
||||
:block/uuid (random-uuid)}]))
|
||||
recycle-fragment-id (get tempids "recycle-fragment")
|
||||
orphan-fragment-id (get tempids "orphan-fragment")]
|
||||
(let [errors (:errors (db-validate/validate-db @conn))]
|
||||
(is (seq errors))
|
||||
(is (= #{recycle-fragment-id}
|
||||
(set (validate-fix/invalid-fragment-ids @conn errors)))))
|
||||
(let [result (validate-fix/validate-and-fix-invalid-blocks! conn)]
|
||||
(is (empty? (:errors result)))
|
||||
(is (nil? (d/entity @conn recycle-fragment-id)))
|
||||
(is (nil? (d/entity @conn orphan-fragment-id)))
|
||||
(is (empty? (:errors (validate-fix/validate-and-fix-invalid-blocks! conn)))))))
|
||||
|
||||
(deftest validate-db-repairs-normal-page-alias-to-non-page
|
||||
(let [conn (create-db-graph-conn)
|
||||
page-id (get (:tempids
|
||||
(d/transact! conn [{:db/id "page"
|
||||
:block/uuid (random-uuid)
|
||||
:block/created-at 1
|
||||
:block/updated-at 1
|
||||
:block/name "object oriented programming"
|
||||
:block/title "Object Oriented Programming"
|
||||
:block/tags :logseq.class/Page}]))
|
||||
"page")
|
||||
block-id (get (:tempids
|
||||
(d/transact! conn [{:db/id "block"
|
||||
:block/uuid (random-uuid)
|
||||
:block/created-at 1
|
||||
:block/updated-at 2
|
||||
:block/page page-id
|
||||
:block/parent page-id
|
||||
:block/order "a0"
|
||||
:block/title "Not a Page"}]))
|
||||
"block")]
|
||||
(d/transact! conn [[:db/add page-id :block/alias block-id]])
|
||||
(is (seq (:errors (db-validate/validate-db @conn))))
|
||||
(with-redefs [shared-service/broadcast-to-clients! (fn [& _args] nil)]
|
||||
(let [result (worker-db-validate/validate-db conn)
|
||||
page (d/entity @conn page-id)]
|
||||
(is (empty? (:errors result)))
|
||||
(is (empty? (:block/alias page)))
|
||||
(is (empty? (:errors (worker-db-validate/validate-db conn))))))))
|
||||
|
||||
(deftest validate-db-repairs-invalid-raw-block-tag
|
||||
(let [conn (create-db-graph-conn)
|
||||
page-id (get (:tempids
|
||||
(d/transact! conn [{:db/id "page"
|
||||
:block/uuid (random-uuid)
|
||||
:block/created-at 1
|
||||
:block/updated-at 1
|
||||
:block/name "test page"
|
||||
:block/title "Test Page"
|
||||
:block/tags :logseq.class/Page}]))
|
||||
"page")
|
||||
block-id (get (:tempids
|
||||
(d/transact! conn [{:db/id "block"
|
||||
:block/uuid (random-uuid)
|
||||
:block/created-at 1
|
||||
:block/updated-at 2
|
||||
:block/page page-id
|
||||
:block/parent page-id
|
||||
:block/order "a0"
|
||||
:block/title "Tagged block"}]))
|
||||
"block")
|
||||
corrupt-db (d/init-db (conj (vec (d/datoms @conn :eavt))
|
||||
(d/datom block-id :block/tags :missing.ident/tag 1 true))
|
||||
(:schema @conn))
|
||||
corrupt-conn (d/conn-from-db corrupt-db)]
|
||||
(is (seq (:errors (db-validate/validate-db @corrupt-conn))))
|
||||
(with-redefs [shared-service/broadcast-to-clients! (fn [& _args] nil)]
|
||||
(validate-fix/fix-invalid-blocks! corrupt-conn (:errors (db-validate/validate-db @corrupt-conn)))
|
||||
(is (empty? (remove #(= :logseq.class/Page (:db/ident %))
|
||||
(:block/tags (d/entity @corrupt-conn block-id))))))))
|
||||
|
||||
(deftest validate-db-does-not-add-class-property-with-missing-ident
|
||||
(let [conn (create-db-graph-conn)
|
||||
block-id (get (:tempids
|
||||
(d/transact! conn [{:db/id "class"
|
||||
:block/uuid (random-uuid)
|
||||
:block/created-at 1
|
||||
:block/updated-at 1
|
||||
:block/name "topic"
|
||||
:block/title "Topic"
|
||||
:block/tags :logseq.class/Tag
|
||||
:db/ident :user.class/topic}
|
||||
{:db/id "property"
|
||||
:block/uuid (random-uuid)
|
||||
:block/created-at 1
|
||||
:block/updated-at 1
|
||||
:block/name "topic"
|
||||
:block/title "Topic"
|
||||
:block/tags :logseq.class/Property}
|
||||
{:db/id "block"
|
||||
:block/uuid (random-uuid)
|
||||
:block/created-at 1
|
||||
:block/updated-at 2
|
||||
:block/title "Block"
|
||||
:user.class/topic "value"}]))
|
||||
"block")]
|
||||
(validate-fix/fix-invalid-blocks! conn [])
|
||||
(let [block (d/entity @conn block-id)]
|
||||
(is (nil? (:user.class/topic block)))
|
||||
(is (nil? (get block nil))))))
|
||||
|
||||
17
src/test/logseq/db/frontend/malli_schema_test.cljs
Normal file
17
src/test/logseq/db/frontend/malli_schema_test.cljs
Normal file
@@ -0,0 +1,17 @@
|
||||
(ns logseq.db.frontend.malli-schema-test
|
||||
(:require [cljs.test :refer [deftest is]]
|
||||
[logseq.db.frontend.malli-schema :as db-malli-schema]))
|
||||
|
||||
(deftest datoms->entity-maps-ignores-duplicate-identical-cardinality-one-datoms
|
||||
(is (= {1 {:block/title "Title"
|
||||
:block/created-at 1}}
|
||||
(db-malli-schema/datoms->entity-maps
|
||||
[{:e 1 :a :block/title :v "Title"}
|
||||
{:e 1 :a :block/title :v "Title"}
|
||||
{:e 1 :a :block/created-at :v 1}]))))
|
||||
|
||||
(deftest datoms->entity-maps-keeps-conflicting-cardinality-one-values
|
||||
(is (= {1 {:block/title #{"Old" "New"}}}
|
||||
(db-malli-schema/datoms->entity-maps
|
||||
[{:e 1 :a :block/title :v "Old"}
|
||||
{:e 1 :a :block/title :v "New"}]))))
|
||||
60
src/test/logseq/outliner/recycle_test.cljs
Normal file
60
src/test/logseq/outliner/recycle_test.cljs
Normal file
@@ -0,0 +1,60 @@
|
||||
(ns logseq.outliner.recycle-test
|
||||
(:require [cljs.test :refer [deftest is]]
|
||||
[datascript.core :as d]
|
||||
[logseq.db :as ldb]
|
||||
[logseq.db.test.helper :as db-test]
|
||||
[logseq.outliner.recycle :as recycle]))
|
||||
|
||||
(deftest permanently-delete-recycled-block-removes-uuid-missing-descendants
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
[{:page {:block/title "page1"}
|
||||
:blocks [{:block/title "parent"
|
||||
:build/children [{:block/title "child"}]}]}])
|
||||
parent (db-test/find-block-by-content @conn "parent")
|
||||
child (db-test/find-block-by-content @conn "child")
|
||||
parent-uuid (:block/uuid parent)
|
||||
child-id (:db/id child)]
|
||||
(ldb/transact! conn (recycle/recycle-blocks-tx-data @conn [parent] {}) {:outliner-op :delete-blocks})
|
||||
(d/transact! conn [[:db/retract child-id :block/uuid (:block/uuid child)]])
|
||||
(is (true? (recycle/permanently-delete! conn parent-uuid)))
|
||||
(is (nil? (d/entity @conn child-id)))))
|
||||
|
||||
(deftest permanently-delete-recycled-descendant
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
[{:page {:block/title "page1"}
|
||||
:blocks [{:block/title "parent"
|
||||
:build/children [{:block/title "child"}]}]}])
|
||||
parent (db-test/find-block-by-content @conn "parent")
|
||||
child (db-test/find-block-by-content @conn "child")
|
||||
child-uuid (:block/uuid child)]
|
||||
(ldb/transact! conn (recycle/recycle-blocks-tx-data @conn [parent] {}) {:outliner-op :delete-blocks})
|
||||
(is (true? (ldb/recycled? (d/entity @conn [:block/uuid child-uuid]))))
|
||||
(is (true? (recycle/permanently-delete! conn child-uuid)))
|
||||
(is (nil? (d/entity @conn [:block/uuid child-uuid])))))
|
||||
|
||||
(deftest permanently-delete-block-stuck-on-recycle-page
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
[{:page {:block/title "page1"}
|
||||
:blocks [{:block/title "parent"
|
||||
:build/children [{:block/title "child"}]}]}])
|
||||
page (ldb/get-page @conn "page1")
|
||||
parent (db-test/find-block-by-content @conn "parent")
|
||||
child (db-test/find-block-by-content @conn "child")
|
||||
child-uuid (:block/uuid child)]
|
||||
(ldb/transact! conn (recycle/recycle-blocks-tx-data @conn [parent] {}) {:outliner-op :delete-blocks})
|
||||
(d/transact! conn [[:db/retract (:db/id parent) :logseq.property/deleted-at]
|
||||
[:db/add (:db/id child) :logseq.property.recycle/original-page (:db/id page)]])
|
||||
(is (false? (ldb/recycled? (d/entity @conn [:block/uuid child-uuid]))))
|
||||
(is (true? (recycle/permanently-delete! conn child-uuid)))
|
||||
(is (nil? (d/entity @conn [:block/uuid child-uuid])))))
|
||||
|
||||
(deftest permanently-delete-invalid-recycled-block
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
[{:page {:block/title "page1"}
|
||||
:blocks [{:block/title "parent"}]}])
|
||||
parent (db-test/find-block-by-content @conn "parent")
|
||||
parent-uuid (:block/uuid parent)]
|
||||
(ldb/transact! conn (recycle/recycle-blocks-tx-data @conn [parent] {}) {:outliner-op :delete-blocks})
|
||||
(d/transact! conn [[:db/add (:db/id parent) :block/pre-block? true]])
|
||||
(is (true? (recycle/permanently-delete! conn parent-uuid)))
|
||||
(is (nil? (d/entity @conn [:block/uuid parent-uuid])))))
|
||||
Reference in New Issue
Block a user