mirror of
https://github.com/logseq/logseq.git
synced 2026-06-01 19:01:22 +00:00
fix: address validate db ci failures
This commit is contained in:
Tienson Qin
@@ -123,173 +123,185 @@
|
||||
[?e :block/path-refs ?v]]
|
||||
db))))
|
||||
|
||||
(defn- invalid-fragment-tx-data
|
||||
[db entity error-entity dispatch-key entity-id]
|
||||
(let [invalid-alias-datoms (invalid-normal-page-alias-datoms db entity dispatch-key)]
|
||||
(cond
|
||||
(invalid-recycle-fragment? error-entity dispatch-key)
|
||||
[[:db/retractEntity entity-id]]
|
||||
(invalid-orphan-fragment? error-entity dispatch-key)
|
||||
[[:db/retractEntity entity-id]]
|
||||
(invalid-recycled-block? db entity dispatch-key)
|
||||
[[:db/retractEntity entity-id]]
|
||||
|
||||
(some? (:logseq.property/parent entity))
|
||||
[[:db/retract entity-id :logseq.property/parent]]
|
||||
(some? (:hide? entity))
|
||||
[[:db/retract entity-id :hide?]]
|
||||
(some? (:public? entity))
|
||||
[[:db/retract entity-id :public?]]
|
||||
(some? (:block/pre-block? entity))
|
||||
[[:db/retract entity-id :block/pre-block?]]
|
||||
(some? (:logseq.property.embedding/hnsw-label entity))
|
||||
[[:db/retract entity-id :logseq.property.embedding/hnsw-label]]
|
||||
(some? (:logseq.property.embedding/hnsw-label-updated-at entity))
|
||||
[[:db/retract entity-id :logseq.property.embedding/hnsw-label-updated-at]]
|
||||
(normal-page-missing-updated-at? entity dispatch-key)
|
||||
[[:db/add entity-id :block/updated-at (:block/created-at entity)]]
|
||||
(and (= "External URL" (:block/title entity))
|
||||
(nil? (:block/tags entity)))
|
||||
[[:db/retractEntity entity-id]]
|
||||
(and (ldb/property? entity)
|
||||
(some #(= (:db/ident %) :logseq.class/Tag) (:block/tags entity)))
|
||||
[[:db/retract entity-id :block/tags :logseq.class/Tag]]
|
||||
(and (:db/ident entity)
|
||||
(db-class/user-class-namespace? (str (:db/ident entity)))
|
||||
(not (:logseq.property/built-in? entity))
|
||||
(not (ldb/class? entity)))
|
||||
[[:db/add entity-id :block/tags :logseq.class/Tag]
|
||||
[:db/retract entity-id :block/tags :logseq.class/Page]]
|
||||
(and (ldb/class? entity) (:kv/value entity))
|
||||
[[:db/retract entity-id :kv/value]]
|
||||
(and (ldb/property? entity)
|
||||
(:logseq.property.class/extends entity))
|
||||
(mapv (fn [class]
|
||||
[:db/retract entity-id :logseq.property.class/extends (:db/id class)])
|
||||
(:logseq.property.class/extends entity))
|
||||
invalid-alias-datoms
|
||||
(mapv (fn [datom]
|
||||
[:db/retract (:e datom) (:a datom) (:v datom)])
|
||||
invalid-alias-datoms))))
|
||||
|
||||
(defn- invalid-location-tx-data
|
||||
[db entity dispatch-key entity-id remove-block-path-refs-fn]
|
||||
(cond
|
||||
(:block/level entity)
|
||||
[[:db/retract entity-id :block/level]]
|
||||
(and (ldb/class? entity) (nil? (:db/ident entity)) (:block/title entity))
|
||||
[[:db/add entity-id :db/ident (db-class/create-user-class-ident-from-name db (:block/title entity))]]
|
||||
(and
|
||||
(= (:block/title (:logseq.property/created-from-property entity)) "description")
|
||||
(nil? (:block/page entity)))
|
||||
(let [property-id (:db/id (:logseq.property/created-from-property entity))]
|
||||
[[:db/add entity-id :block/page property-id]
|
||||
[:db/add entity-id :block/parent property-id]])
|
||||
(and (:db/ident entity)
|
||||
(:logseq.property/built-in? entity)
|
||||
(:block/parent entity))
|
||||
[[:db/retract entity-id :block/parent]]
|
||||
(:block/format entity)
|
||||
[[:db/retract entity-id :block/format]]
|
||||
(= :whiteboard-shape (:logseq.property/ls-type entity))
|
||||
[[:db/retractEntity entity-id]]
|
||||
(and (:block/page entity) (not (:block/parent entity)))
|
||||
[[:db/add entity-id :block/parent (:db/id (:block/page entity))]]
|
||||
(and (:logseq.property/created-by-ref entity)
|
||||
(not (de/entity? (:logseq.property/created-by-ref entity))))
|
||||
[[:db/retractEntity entity-id]]
|
||||
(block-missing-uuid? entity)
|
||||
[[:db/add entity-id :block/uuid (random-uuid)]]
|
||||
(vector? (:logseq.property/value entity))
|
||||
[[:db/retractEntity entity-id]]
|
||||
(and (:block/tx-id entity) (nil? (:block/title entity)))
|
||||
[[:db/retractEntity entity-id]]
|
||||
(and (:block/title entity) (nil? (:block/page entity)) (nil? (:block/parent entity)) (nil? (:block/name entity)))
|
||||
[[:db/retractEntity entity-id]]
|
||||
(= :block/path-refs (:db/ident entity))
|
||||
(try
|
||||
((or remove-block-path-refs-fn remove-block-path-refs) db)
|
||||
(catch :default _e
|
||||
nil))))
|
||||
|
||||
(defn- invalid-block-tx-data
|
||||
[db entity dispatch-key entity-id]
|
||||
(cond
|
||||
(not-every? (fn [e] (ldb/class? e)) (:block/tags entity))
|
||||
(let [non-tags (remove ldb/class? (:block/tags entity))]
|
||||
(map (fn [tag]
|
||||
[:db/retract entity-id :block/tags (:db/id tag)]) non-tags))
|
||||
(and (= dispatch-key :normal-page) (:block/page entity))
|
||||
[[:db/retract entity-id :block/page]]
|
||||
(and (= dispatch-key :block) (nil? (:block/title entity)))
|
||||
[[:db/retractEntity entity-id]]
|
||||
(and (= dispatch-key :block) (nil? (:block/page entity)))
|
||||
(let [latest-journal-id (:db/id (first (ldb/get-latest-journals db)))
|
||||
page-id (:db/id (:block/page (:block/parent entity)))]
|
||||
(cond
|
||||
page-id
|
||||
[[:db/add entity-id :block/page page-id]]
|
||||
latest-journal-id
|
||||
[[:db/add entity-id :block/page latest-journal-id]
|
||||
[:db/add entity-id :block/parent latest-journal-id]]
|
||||
:else
|
||||
(js/console.error (str "Don't know where to put the block " entity-id))))
|
||||
|
||||
(and (= dispatch-key :block)
|
||||
(some (fn [k] (= "user.class" (namespace k))) (keys (:logseq.property.table/sized-columns entity))))
|
||||
(let [new-value (->> (keep (fn [[k v]]
|
||||
(if (= "user.class" (namespace k))
|
||||
(when-let [property (get-property-by-title db (:block/title (d/entity db k)))]
|
||||
[(:db/ident property) v])
|
||||
[k v]))
|
||||
(:logseq.property.table/sized-columns entity))
|
||||
(into {}))]
|
||||
[[:db/add entity-id :logseq.property.table/sized-columns new-value]])
|
||||
|
||||
(some (fn [k] (= "block.temp" (namespace k))) (keys entity))
|
||||
(let [ks (filter (fn [k] (= "block.temp" (namespace k))) (keys entity))]
|
||||
(mapv (fn [k] [:db/retract entity-id k]) ks))
|
||||
(and (not (:block/page entity)) (not (:block/parent entity)) (not (:block/name entity)))
|
||||
[[:db/retractEntity entity-id]]
|
||||
(and (= dispatch-key :property-value-block) (:block/title entity))
|
||||
[[:db/retract entity-id :block/title]]
|
||||
(and (ldb/class? entity) (not (:logseq.property.class/extends entity))
|
||||
(not= (:db/ident entity) :logseq.class/Root))
|
||||
[[:db/add entity-id :logseq.property.class/extends :logseq.class/Root]]
|
||||
(and (or (ldb/class? entity) (ldb/property? entity)) (ldb/internal-page? entity))
|
||||
[[:db/retract entity-id :block/tags :logseq.class/Page]]
|
||||
|
||||
(and (:logseq.property.asset/remote-metadata entity) (nil? (:logseq.property.asset/type entity)))
|
||||
[[:db/retractEntity entity-id]]))
|
||||
|
||||
(defn- invalid-entity-tx-data
|
||||
[db {:keys [entity dispatch-key] error-entity-id :entity-id} opts]
|
||||
(when-let [entity-id (or error-entity-id (resolve-entity-id db entity))]
|
||||
(let [current-entity (d/entity db entity-id)]
|
||||
(or (invalid-fragment-tx-data db current-entity entity dispatch-key entity-id)
|
||||
(invalid-location-tx-data db current-entity dispatch-key entity-id (:remove-block-path-refs-fn opts))
|
||||
(invalid-block-tx-data db current-entity dispatch-key entity-id)))))
|
||||
|
||||
(defn- class-as-properties-tx-data
|
||||
[db]
|
||||
(concat
|
||||
(mapcat
|
||||
(fn [ident]
|
||||
(->> (d/datoms db :eavt)
|
||||
(filter (fn [d] (= ident (:a d))))
|
||||
(mapcat (fn [d]
|
||||
(let [entity (d/entity db (:v d))]
|
||||
(when (ldb/class? entity)
|
||||
(if-let [property (get-property-by-title db (:block/title entity))]
|
||||
[[:db/retract (:e d) (:a d) (:v d)]
|
||||
[:db/add (:e d) (:a d) (:db/id property)]]
|
||||
[[:db/retract (:e d) (:a d) (:v d)]])))))))
|
||||
[:logseq.property.view/group-by-property :logseq.property.table/pinned-columns])
|
||||
(->> (d/datoms db :eavt)
|
||||
(filter (fn [d] (= (namespace (:a d)) "user.class")))
|
||||
(mapcat (fn [d]
|
||||
(let [class-title (:block/title (d/entity db (:a d)))
|
||||
property (get-property-by-title db class-title)]
|
||||
(if-let [property-ident (:db/ident property)]
|
||||
[[:db/retract (:e d) (:a d) (:v d)]
|
||||
[:db/add (:e d) property-ident (:v d)]]
|
||||
[[:db/retract (:e d) (:a d) (:v d)]])))))))
|
||||
|
||||
(defn fix-invalid-blocks!
|
||||
([conn errors]
|
||||
(fix-invalid-blocks! conn errors nil))
|
||||
([conn errors {:keys [remove-block-path-refs-fn]}]
|
||||
([conn errors opts]
|
||||
(let [db @conn
|
||||
fix-tx-data (mapcat
|
||||
(fn [{error-entity-id :entity-id
|
||||
:keys [entity dispatch-key]}]
|
||||
(when-let [entity-id (or error-entity-id (resolve-entity-id db entity))]
|
||||
(let [error-entity entity
|
||||
entity (d/entity db entity-id)
|
||||
invalid-alias-datoms (invalid-normal-page-alias-datoms db entity dispatch-key)]
|
||||
(cond
|
||||
(invalid-recycle-fragment? error-entity dispatch-key)
|
||||
[[:db/retractEntity entity-id]]
|
||||
(invalid-orphan-fragment? error-entity dispatch-key)
|
||||
[[:db/retractEntity entity-id]]
|
||||
(invalid-recycled-block? db entity dispatch-key)
|
||||
[[:db/retractEntity entity-id]]
|
||||
|
||||
(some? (:logseq.property/parent entity))
|
||||
[[:db/retract entity-id :logseq.property/parent]]
|
||||
(some? (:hide? entity))
|
||||
[[:db/retract entity-id :hide?]]
|
||||
(some? (:public? entity))
|
||||
[[:db/retract entity-id :public?]]
|
||||
(some? (:block/pre-block? entity))
|
||||
[[:db/retract entity-id :block/pre-block?]]
|
||||
(some? (:logseq.property.embedding/hnsw-label entity))
|
||||
[[:db/retract entity-id :logseq.property.embedding/hnsw-label]]
|
||||
(some? (:logseq.property.embedding/hnsw-label-updated-at entity))
|
||||
[[:db/retract entity-id :logseq.property.embedding/hnsw-label-updated-at]]
|
||||
(normal-page-missing-updated-at? entity dispatch-key)
|
||||
[[:db/add entity-id :block/updated-at (:block/created-at entity)]]
|
||||
(and (= "External URL" (:block/title entity))
|
||||
(nil? (:block/tags entity)))
|
||||
[[:db/retractEntity entity-id]]
|
||||
(and (ldb/property? entity)
|
||||
(some #(= (:db/ident %) :logseq.class/Tag) (:block/tags entity)))
|
||||
[[:db/retract entity-id :block/tags :logseq.class/Tag]]
|
||||
(and (:db/ident entity)
|
||||
(db-class/user-class-namespace? (str (:db/ident entity)))
|
||||
(not (:logseq.property/built-in? entity))
|
||||
(not (ldb/class? entity)))
|
||||
[[:db/add entity-id :block/tags :logseq.class/Tag]
|
||||
[:db/retract entity-id :block/tags :logseq.class/Page]]
|
||||
(and (ldb/class? entity) (:kv/value entity))
|
||||
[[:db/retract entity-id :kv/value]]
|
||||
(and (ldb/property? entity)
|
||||
(:logseq.property.class/extends entity))
|
||||
(mapv (fn [class]
|
||||
[:db/retract entity-id :logseq.property.class/extends (:db/id class)])
|
||||
(:logseq.property.class/extends entity))
|
||||
invalid-alias-datoms
|
||||
(mapv (fn [datom]
|
||||
[:db/retract (:e datom) (:a datom) (:v datom)])
|
||||
invalid-alias-datoms)
|
||||
(:block/level entity)
|
||||
[[:db/retract entity-id :block/level]]
|
||||
(and (ldb/class? entity) (nil? (:db/ident entity)) (:block/title entity))
|
||||
[[:db/add entity-id :db/ident (db-class/create-user-class-ident-from-name db (:block/title entity))]]
|
||||
(and
|
||||
(= (:block/title (:logseq.property/created-from-property entity)) "description")
|
||||
(nil? (:block/page entity)))
|
||||
(let [property-id (:db/id (:logseq.property/created-from-property entity))]
|
||||
[[:db/add entity-id :block/page property-id]
|
||||
[:db/add entity-id :block/parent property-id]])
|
||||
(and (:db/ident entity)
|
||||
(:logseq.property/built-in? entity)
|
||||
(:block/parent entity))
|
||||
[[:db/retract entity-id :block/parent]]
|
||||
(:block/format entity)
|
||||
[[:db/retract entity-id :block/format]]
|
||||
(= :whiteboard-shape (:logseq.property/ls-type entity))
|
||||
[[:db/retractEntity entity-id]]
|
||||
(and (:block/page entity) (not (:block/parent entity)))
|
||||
[[:db/add entity-id :block/parent (:db/id (:block/page entity))]]
|
||||
(and (:logseq.property/created-by-ref entity)
|
||||
(not (de/entity? (:logseq.property/created-by-ref entity))))
|
||||
[[:db/retractEntity entity-id]]
|
||||
(block-missing-uuid? entity)
|
||||
[[:db/add entity-id :block/uuid (random-uuid)]]
|
||||
(vector? (:logseq.property/value entity))
|
||||
[[:db/retractEntity entity-id]]
|
||||
(and (:block/tx-id entity) (nil? (:block/title entity)))
|
||||
[[:db/retractEntity entity-id]]
|
||||
(and (:block/title entity) (nil? (:block/page entity)) (nil? (:block/parent entity)) (nil? (:block/name entity)))
|
||||
[[:db/retractEntity entity-id]]
|
||||
(= :block/path-refs (:db/ident entity))
|
||||
(try
|
||||
((or remove-block-path-refs-fn remove-block-path-refs) db)
|
||||
(catch :default _e
|
||||
nil))
|
||||
(not-every? (fn [e] (ldb/class? e)) (:block/tags entity))
|
||||
(let [non-tags (remove ldb/class? (:block/tags entity))]
|
||||
(map (fn [tag]
|
||||
[:db/retract entity-id :block/tags (:db/id tag)]) non-tags))
|
||||
(and (= dispatch-key :normal-page) (:block/page entity))
|
||||
[[:db/retract entity-id :block/page]]
|
||||
(and (= dispatch-key :block) (nil? (:block/title entity)))
|
||||
[[:db/retractEntity entity-id]]
|
||||
(and (= dispatch-key :block) (nil? (:block/page entity)))
|
||||
(let [latest-journal-id (:db/id (first (ldb/get-latest-journals db)))
|
||||
page-id (:db/id (:block/page (:block/parent entity)))]
|
||||
(cond
|
||||
page-id
|
||||
[[:db/add entity-id :block/page page-id]]
|
||||
latest-journal-id
|
||||
[[:db/add entity-id :block/page latest-journal-id]
|
||||
[:db/add entity-id :block/parent latest-journal-id]]
|
||||
:else
|
||||
(js/console.error (str "Don't know where to put the block " entity-id))))
|
||||
|
||||
(and (= dispatch-key :block)
|
||||
(some (fn [k] (= "user.class" (namespace k))) (keys (:logseq.property.table/sized-columns entity))))
|
||||
(let [new-value (->> (keep (fn [[k v]]
|
||||
(if (= "user.class" (namespace k))
|
||||
(when-let [property (get-property-by-title db (:block/title (d/entity db k)))]
|
||||
[(:db/ident property) v])
|
||||
[k v]))
|
||||
(:logseq.property.table/sized-columns entity))
|
||||
(into {}))]
|
||||
[[:db/add entity-id :logseq.property.table/sized-columns new-value]])
|
||||
|
||||
(some (fn [k] (= "block.temp" (namespace k))) (keys entity))
|
||||
(let [ks (filter (fn [k] (= "block.temp" (namespace k))) (keys entity))]
|
||||
(mapv (fn [k] [:db/retract entity-id k]) ks))
|
||||
(and (not (:block/page entity)) (not (:block/parent entity)) (not (:block/name entity)))
|
||||
[[:db/retractEntity entity-id]]
|
||||
(and (= dispatch-key :property-value-block) (:block/title entity))
|
||||
[[:db/retract entity-id :block/title]]
|
||||
(and (ldb/class? entity) (not (:logseq.property.class/extends entity))
|
||||
(not= (:db/ident entity) :logseq.class/Root))
|
||||
[[:db/add entity-id :logseq.property.class/extends :logseq.class/Root]]
|
||||
(and (or (ldb/class? entity) (ldb/property? entity)) (ldb/internal-page? entity))
|
||||
[[:db/retract entity-id :block/tags :logseq.class/Page]]
|
||||
|
||||
(and (:logseq.property.asset/remote-metadata entity) (nil? (:logseq.property.asset/type entity)))
|
||||
[[:db/retractEntity entity-id]]
|
||||
|
||||
:else
|
||||
nil))))
|
||||
errors)
|
||||
class-as-properties (concat
|
||||
(mapcat
|
||||
(fn [ident]
|
||||
(->> (d/datoms db :eavt)
|
||||
(filter (fn [d] (= ident (:a d))))
|
||||
(mapcat (fn [d]
|
||||
(let [entity (d/entity db (:v d))]
|
||||
(when (ldb/class? entity)
|
||||
(if-let [property (get-property-by-title db (:block/title entity))]
|
||||
[[:db/retract (:e d) (:a d) (:v d)]
|
||||
[:db/add (:e d) (:a d) (:db/id property)]]
|
||||
[[:db/retract (:e d) (:a d) (:v d)]])))))))
|
||||
[:logseq.property.view/group-by-property :logseq.property.table/pinned-columns])
|
||||
(->> (d/datoms db :eavt)
|
||||
(filter (fn [d] (= (namespace (:a d)) "user.class")))
|
||||
(mapcat (fn [d]
|
||||
(let [class-title (:block/title (d/entity db (:a d)))
|
||||
property (get-property-by-title db class-title)]
|
||||
(if-let [property-ident (:db/ident property)]
|
||||
[[:db/retract (:e d) (:a d) (:v d)]
|
||||
[:db/add (:e d) property-ident (:v d)]]
|
||||
[[:db/retract (:e d) (:a d) (:v d)]]))))))
|
||||
tx-data (concat fix-tx-data
|
||||
class-as-properties)]
|
||||
tx-data (concat (mapcat #(invalid-entity-tx-data db % opts) errors)
|
||||
(class-as-properties-tx-data db))]
|
||||
(when (seq tx-data)
|
||||
(let [tx-report (d/transact! conn tx-data {:fix-db? true})]
|
||||
(seq (:tx-data tx-report)))))))
|
||||
@@ -301,7 +313,7 @@
|
||||
(assoc error :entity-id (resolve-entity-id db (:entity error))))
|
||||
errors)
|
||||
invalid-entity-ids (distinct (map :entity-id errors'))]
|
||||
{:errors errors'
|
||||
{:errors (seq errors')
|
||||
:datom-count datom-count
|
||||
:entities entities
|
||||
:invalid-entity-ids invalid-entity-ids}))
|
||||
|
||||
@@ -15,6 +15,10 @@
|
||||
(d/transact! conn (sqlite-create-graph/build-db-initial-data ""))
|
||||
conn))
|
||||
|
||||
(deftest validate-db-result-returns-nil-errors-for-valid-db
|
||||
(let [conn (create-db-graph-conn)]
|
||||
(is (nil? (:errors (validate-fix/validate-db-result @conn))))))
|
||||
|
||||
(deftest validate-db-repairs-block-missing-uuid
|
||||
(let [conn (create-db-graph-conn)
|
||||
page-uuid (random-uuid)
|
||||
|
||||
Reference in New Issue
Block a user