diff --git a/deps/outliner/src/logseq/outliner/recycle.cljs b/deps/outliner/src/logseq/outliner/recycle.cljs index 5345874f0d..d9d57432fc 100644 --- a/deps/outliner/src/logseq/outliner/recycle.cljs +++ b/deps/outliner/src/logseq/outliner/recycle.cljs @@ -124,15 +124,19 @@ [:db/retract (:e datom) (:a datom) (:v datom)]) (d/datoms db :eavt db-id))) +(defn- permanently-delete-entity-ids + [db root] + (if (ldb/page? root) + (page-tree-ids db root) + (map :db/id (block-subtree db root)))) + (defn- permanently-delete-entity-tx-data [db root] - (->> (if (ldb/page? root) - (page-tree-ids db root) - (map :db/id (block-subtree db root))) - (mapcat #(retract-entity-tx-data db %)) - seq - (with-delete-cleanup-tx db) - seq)) + (let [entity-ids (permanently-delete-entity-ids db root) + retract-entities-tx (map (fn [id] [:db/retractEntity id]) entity-ids) + retract-datoms-tx (mapcat #(retract-entity-tx-data db %) entity-ids)] + (seq (distinct (concat retract-datoms-tx + (delete-blocks/update-refs-history db retract-entities-tx {})))))) (defn recycle-blocks-tx-data [db blocks {:keys [deleted-by-uuid now-ms]}] diff --git a/src/main/frontend/worker/db/validate_fix.cljs b/src/main/frontend/worker/db/validate_fix.cljs index 88efa36e57..d58744a7fe 100644 --- a/src/main/frontend/worker/db/validate_fix.cljs +++ b/src/main/frontend/worker/db/validate_fix.cljs @@ -123,173 +123,185 @@ [?e :block/path-refs ?v]] db)))) +(defn- invalid-fragment-tx-data + [db entity error-entity dispatch-key entity-id] + (let [invalid-alias-datoms (invalid-normal-page-alias-datoms db entity dispatch-key)] + (cond + (invalid-recycle-fragment? error-entity dispatch-key) + [[:db/retractEntity entity-id]] + (invalid-orphan-fragment? error-entity dispatch-key) + [[:db/retractEntity entity-id]] + (invalid-recycled-block? db entity dispatch-key) + [[:db/retractEntity entity-id]] + + (some? (:logseq.property/parent entity)) + [[:db/retract entity-id :logseq.property/parent]] + (some? (:hide? entity)) + [[:db/retract entity-id :hide?]] + (some? (:public? entity)) + [[:db/retract entity-id :public?]] + (some? (:block/pre-block? entity)) + [[:db/retract entity-id :block/pre-block?]] + (some? (:logseq.property.embedding/hnsw-label entity)) + [[:db/retract entity-id :logseq.property.embedding/hnsw-label]] + (some? (:logseq.property.embedding/hnsw-label-updated-at entity)) + [[:db/retract entity-id :logseq.property.embedding/hnsw-label-updated-at]] + (normal-page-missing-updated-at? entity dispatch-key) + [[:db/add entity-id :block/updated-at (:block/created-at entity)]] + (and (= "External URL" (:block/title entity)) + (nil? (:block/tags entity))) + [[:db/retractEntity entity-id]] + (and (ldb/property? entity) + (some #(= (:db/ident %) :logseq.class/Tag) (:block/tags entity))) + [[:db/retract entity-id :block/tags :logseq.class/Tag]] + (and (:db/ident entity) + (db-class/user-class-namespace? (str (:db/ident entity))) + (not (:logseq.property/built-in? entity)) + (not (ldb/class? entity))) + [[:db/add entity-id :block/tags :logseq.class/Tag] + [:db/retract entity-id :block/tags :logseq.class/Page]] + (and (ldb/class? entity) (:kv/value entity)) + [[:db/retract entity-id :kv/value]] + (and (ldb/property? entity) + (:logseq.property.class/extends entity)) + (mapv (fn [class] + [:db/retract entity-id :logseq.property.class/extends (:db/id class)]) + (:logseq.property.class/extends entity)) + invalid-alias-datoms + (mapv (fn [datom] + [:db/retract (:e datom) (:a datom) (:v datom)]) + invalid-alias-datoms)))) + +(defn- invalid-location-tx-data + [db entity dispatch-key entity-id remove-block-path-refs-fn] + (cond + (:block/level entity) + [[:db/retract entity-id :block/level]] + (and (ldb/class? entity) (nil? (:db/ident entity)) (:block/title entity)) + [[:db/add entity-id :db/ident (db-class/create-user-class-ident-from-name db (:block/title entity))]] + (and + (= (:block/title (:logseq.property/created-from-property entity)) "description") + (nil? (:block/page entity))) + (let [property-id (:db/id (:logseq.property/created-from-property entity))] + [[:db/add entity-id :block/page property-id] + [:db/add entity-id :block/parent property-id]]) + (and (:db/ident entity) + (:logseq.property/built-in? entity) + (:block/parent entity)) + [[:db/retract entity-id :block/parent]] + (:block/format entity) + [[:db/retract entity-id :block/format]] + (= :whiteboard-shape (:logseq.property/ls-type entity)) + [[:db/retractEntity entity-id]] + (and (:block/page entity) (not (:block/parent entity))) + [[:db/add entity-id :block/parent (:db/id (:block/page entity))]] + (and (:logseq.property/created-by-ref entity) + (not (de/entity? (:logseq.property/created-by-ref entity)))) + [[:db/retractEntity entity-id]] + (block-missing-uuid? entity) + [[:db/add entity-id :block/uuid (random-uuid)]] + (vector? (:logseq.property/value entity)) + [[:db/retractEntity entity-id]] + (and (:block/tx-id entity) (nil? (:block/title entity))) + [[:db/retractEntity entity-id]] + (and (:block/title entity) (nil? (:block/page entity)) (nil? (:block/parent entity)) (nil? (:block/name entity))) + [[:db/retractEntity entity-id]] + (= :block/path-refs (:db/ident entity)) + (try + ((or remove-block-path-refs-fn remove-block-path-refs) db) + (catch :default _e + nil)))) + +(defn- invalid-block-tx-data + [db entity dispatch-key entity-id] + (cond + (not-every? (fn [e] (ldb/class? e)) (:block/tags entity)) + (let [non-tags (remove ldb/class? (:block/tags entity))] + (map (fn [tag] + [:db/retract entity-id :block/tags (:db/id tag)]) non-tags)) + (and (= dispatch-key :normal-page) (:block/page entity)) + [[:db/retract entity-id :block/page]] + (and (= dispatch-key :block) (nil? (:block/title entity))) + [[:db/retractEntity entity-id]] + (and (= dispatch-key :block) (nil? (:block/page entity))) + (let [latest-journal-id (:db/id (first (ldb/get-latest-journals db))) + page-id (:db/id (:block/page (:block/parent entity)))] + (cond + page-id + [[:db/add entity-id :block/page page-id]] + latest-journal-id + [[:db/add entity-id :block/page latest-journal-id] + [:db/add entity-id :block/parent latest-journal-id]] + :else + (js/console.error (str "Don't know where to put the block " entity-id)))) + + (and (= dispatch-key :block) + (some (fn [k] (= "user.class" (namespace k))) (keys (:logseq.property.table/sized-columns entity)))) + (let [new-value (->> (keep (fn [[k v]] + (if (= "user.class" (namespace k)) + (when-let [property (get-property-by-title db (:block/title (d/entity db k)))] + [(:db/ident property) v]) + [k v])) + (:logseq.property.table/sized-columns entity)) + (into {}))] + [[:db/add entity-id :logseq.property.table/sized-columns new-value]]) + + (some (fn [k] (= "block.temp" (namespace k))) (keys entity)) + (let [ks (filter (fn [k] (= "block.temp" (namespace k))) (keys entity))] + (mapv (fn [k] [:db/retract entity-id k]) ks)) + (and (not (:block/page entity)) (not (:block/parent entity)) (not (:block/name entity))) + [[:db/retractEntity entity-id]] + (and (= dispatch-key :property-value-block) (:block/title entity)) + [[:db/retract entity-id :block/title]] + (and (ldb/class? entity) (not (:logseq.property.class/extends entity)) + (not= (:db/ident entity) :logseq.class/Root)) + [[:db/add entity-id :logseq.property.class/extends :logseq.class/Root]] + (and (or (ldb/class? entity) (ldb/property? entity)) (ldb/internal-page? entity)) + [[:db/retract entity-id :block/tags :logseq.class/Page]] + + (and (:logseq.property.asset/remote-metadata entity) (nil? (:logseq.property.asset/type entity))) + [[:db/retractEntity entity-id]])) + +(defn- invalid-entity-tx-data + [db {:keys [entity dispatch-key] error-entity-id :entity-id} opts] + (when-let [entity-id (or error-entity-id (resolve-entity-id db entity))] + (let [current-entity (d/entity db entity-id)] + (or (invalid-fragment-tx-data db current-entity entity dispatch-key entity-id) + (invalid-location-tx-data db current-entity dispatch-key entity-id (:remove-block-path-refs-fn opts)) + (invalid-block-tx-data db current-entity dispatch-key entity-id))))) + +(defn- class-as-properties-tx-data + [db] + (concat + (mapcat + (fn [ident] + (->> (d/datoms db :eavt) + (filter (fn [d] (= ident (:a d)))) + (mapcat (fn [d] + (let [entity (d/entity db (:v d))] + (when (ldb/class? entity) + (if-let [property (get-property-by-title db (:block/title entity))] + [[:db/retract (:e d) (:a d) (:v d)] + [:db/add (:e d) (:a d) (:db/id property)]] + [[:db/retract (:e d) (:a d) (:v d)]]))))))) + [:logseq.property.view/group-by-property :logseq.property.table/pinned-columns]) + (->> (d/datoms db :eavt) + (filter (fn [d] (= (namespace (:a d)) "user.class"))) + (mapcat (fn [d] + (let [class-title (:block/title (d/entity db (:a d))) + property (get-property-by-title db class-title)] + (if-let [property-ident (:db/ident property)] + [[:db/retract (:e d) (:a d) (:v d)] + [:db/add (:e d) property-ident (:v d)]] + [[:db/retract (:e d) (:a d) (:v d)]]))))))) + (defn fix-invalid-blocks! ([conn errors] (fix-invalid-blocks! conn errors nil)) - ([conn errors {:keys [remove-block-path-refs-fn]}] + ([conn errors opts] (let [db @conn - fix-tx-data (mapcat - (fn [{error-entity-id :entity-id - :keys [entity dispatch-key]}] - (when-let [entity-id (or error-entity-id (resolve-entity-id db entity))] - (let [error-entity entity - entity (d/entity db entity-id) - invalid-alias-datoms (invalid-normal-page-alias-datoms db entity dispatch-key)] - (cond - (invalid-recycle-fragment? error-entity dispatch-key) - [[:db/retractEntity entity-id]] - (invalid-orphan-fragment? error-entity dispatch-key) - [[:db/retractEntity entity-id]] - (invalid-recycled-block? db entity dispatch-key) - [[:db/retractEntity entity-id]] - - (some? (:logseq.property/parent entity)) - [[:db/retract entity-id :logseq.property/parent]] - (some? (:hide? entity)) - [[:db/retract entity-id :hide?]] - (some? (:public? entity)) - [[:db/retract entity-id :public?]] - (some? (:block/pre-block? entity)) - [[:db/retract entity-id :block/pre-block?]] - (some? (:logseq.property.embedding/hnsw-label entity)) - [[:db/retract entity-id :logseq.property.embedding/hnsw-label]] - (some? (:logseq.property.embedding/hnsw-label-updated-at entity)) - [[:db/retract entity-id :logseq.property.embedding/hnsw-label-updated-at]] - (normal-page-missing-updated-at? entity dispatch-key) - [[:db/add entity-id :block/updated-at (:block/created-at entity)]] - (and (= "External URL" (:block/title entity)) - (nil? (:block/tags entity))) - [[:db/retractEntity entity-id]] - (and (ldb/property? entity) - (some #(= (:db/ident %) :logseq.class/Tag) (:block/tags entity))) - [[:db/retract entity-id :block/tags :logseq.class/Tag]] - (and (:db/ident entity) - (db-class/user-class-namespace? (str (:db/ident entity))) - (not (:logseq.property/built-in? entity)) - (not (ldb/class? entity))) - [[:db/add entity-id :block/tags :logseq.class/Tag] - [:db/retract entity-id :block/tags :logseq.class/Page]] - (and (ldb/class? entity) (:kv/value entity)) - [[:db/retract entity-id :kv/value]] - (and (ldb/property? entity) - (:logseq.property.class/extends entity)) - (mapv (fn [class] - [:db/retract entity-id :logseq.property.class/extends (:db/id class)]) - (:logseq.property.class/extends entity)) - invalid-alias-datoms - (mapv (fn [datom] - [:db/retract (:e datom) (:a datom) (:v datom)]) - invalid-alias-datoms) - (:block/level entity) - [[:db/retract entity-id :block/level]] - (and (ldb/class? entity) (nil? (:db/ident entity)) (:block/title entity)) - [[:db/add entity-id :db/ident (db-class/create-user-class-ident-from-name db (:block/title entity))]] - (and - (= (:block/title (:logseq.property/created-from-property entity)) "description") - (nil? (:block/page entity))) - (let [property-id (:db/id (:logseq.property/created-from-property entity))] - [[:db/add entity-id :block/page property-id] - [:db/add entity-id :block/parent property-id]]) - (and (:db/ident entity) - (:logseq.property/built-in? entity) - (:block/parent entity)) - [[:db/retract entity-id :block/parent]] - (:block/format entity) - [[:db/retract entity-id :block/format]] - (= :whiteboard-shape (:logseq.property/ls-type entity)) - [[:db/retractEntity entity-id]] - (and (:block/page entity) (not (:block/parent entity))) - [[:db/add entity-id :block/parent (:db/id (:block/page entity))]] - (and (:logseq.property/created-by-ref entity) - (not (de/entity? (:logseq.property/created-by-ref entity)))) - [[:db/retractEntity entity-id]] - (block-missing-uuid? entity) - [[:db/add entity-id :block/uuid (random-uuid)]] - (vector? (:logseq.property/value entity)) - [[:db/retractEntity entity-id]] - (and (:block/tx-id entity) (nil? (:block/title entity))) - [[:db/retractEntity entity-id]] - (and (:block/title entity) (nil? (:block/page entity)) (nil? (:block/parent entity)) (nil? (:block/name entity))) - [[:db/retractEntity entity-id]] - (= :block/path-refs (:db/ident entity)) - (try - ((or remove-block-path-refs-fn remove-block-path-refs) db) - (catch :default _e - nil)) - (not-every? (fn [e] (ldb/class? e)) (:block/tags entity)) - (let [non-tags (remove ldb/class? (:block/tags entity))] - (map (fn [tag] - [:db/retract entity-id :block/tags (:db/id tag)]) non-tags)) - (and (= dispatch-key :normal-page) (:block/page entity)) - [[:db/retract entity-id :block/page]] - (and (= dispatch-key :block) (nil? (:block/title entity))) - [[:db/retractEntity entity-id]] - (and (= dispatch-key :block) (nil? (:block/page entity))) - (let [latest-journal-id (:db/id (first (ldb/get-latest-journals db))) - page-id (:db/id (:block/page (:block/parent entity)))] - (cond - page-id - [[:db/add entity-id :block/page page-id]] - latest-journal-id - [[:db/add entity-id :block/page latest-journal-id] - [:db/add entity-id :block/parent latest-journal-id]] - :else - (js/console.error (str "Don't know where to put the block " entity-id)))) - - (and (= dispatch-key :block) - (some (fn [k] (= "user.class" (namespace k))) (keys (:logseq.property.table/sized-columns entity)))) - (let [new-value (->> (keep (fn [[k v]] - (if (= "user.class" (namespace k)) - (when-let [property (get-property-by-title db (:block/title (d/entity db k)))] - [(:db/ident property) v]) - [k v])) - (:logseq.property.table/sized-columns entity)) - (into {}))] - [[:db/add entity-id :logseq.property.table/sized-columns new-value]]) - - (some (fn [k] (= "block.temp" (namespace k))) (keys entity)) - (let [ks (filter (fn [k] (= "block.temp" (namespace k))) (keys entity))] - (mapv (fn [k] [:db/retract entity-id k]) ks)) - (and (not (:block/page entity)) (not (:block/parent entity)) (not (:block/name entity))) - [[:db/retractEntity entity-id]] - (and (= dispatch-key :property-value-block) (:block/title entity)) - [[:db/retract entity-id :block/title]] - (and (ldb/class? entity) (not (:logseq.property.class/extends entity)) - (not= (:db/ident entity) :logseq.class/Root)) - [[:db/add entity-id :logseq.property.class/extends :logseq.class/Root]] - (and (or (ldb/class? entity) (ldb/property? entity)) (ldb/internal-page? entity)) - [[:db/retract entity-id :block/tags :logseq.class/Page]] - - (and (:logseq.property.asset/remote-metadata entity) (nil? (:logseq.property.asset/type entity))) - [[:db/retractEntity entity-id]] - - :else - nil)))) - errors) - class-as-properties (concat - (mapcat - (fn [ident] - (->> (d/datoms db :eavt) - (filter (fn [d] (= ident (:a d)))) - (mapcat (fn [d] - (let [entity (d/entity db (:v d))] - (when (ldb/class? entity) - (if-let [property (get-property-by-title db (:block/title entity))] - [[:db/retract (:e d) (:a d) (:v d)] - [:db/add (:e d) (:a d) (:db/id property)]] - [[:db/retract (:e d) (:a d) (:v d)]]))))))) - [:logseq.property.view/group-by-property :logseq.property.table/pinned-columns]) - (->> (d/datoms db :eavt) - (filter (fn [d] (= (namespace (:a d)) "user.class"))) - (mapcat (fn [d] - (let [class-title (:block/title (d/entity db (:a d))) - property (get-property-by-title db class-title)] - (if-let [property-ident (:db/ident property)] - [[:db/retract (:e d) (:a d) (:v d)] - [:db/add (:e d) property-ident (:v d)]] - [[:db/retract (:e d) (:a d) (:v d)]])))))) - tx-data (concat fix-tx-data - class-as-properties)] + tx-data (concat (mapcat #(invalid-entity-tx-data db % opts) errors) + (class-as-properties-tx-data db))] (when (seq tx-data) (let [tx-report (d/transact! conn tx-data {:fix-db? true})] (seq (:tx-data tx-report))))))) @@ -301,7 +313,7 @@ (assoc error :entity-id (resolve-entity-id db (:entity error)))) errors) invalid-entity-ids (distinct (map :entity-id errors'))] - {:errors errors' + {:errors (seq errors') :datom-count datom-count :entities entities :invalid-entity-ids invalid-entity-ids})) diff --git a/src/test/frontend/worker/db_validate_test.cljs b/src/test/frontend/worker/db_validate_test.cljs index 6000523725..1d777133f0 100644 --- a/src/test/frontend/worker/db_validate_test.cljs +++ b/src/test/frontend/worker/db_validate_test.cljs @@ -15,6 +15,10 @@ (d/transact! conn (sqlite-create-graph/build-db-initial-data "")) conn)) +(deftest validate-db-result-returns-nil-errors-for-valid-db + (let [conn (create-db-graph-conn)] + (is (nil? (:errors (validate-fix/validate-db-result @conn)))))) + (deftest validate-db-repairs-block-missing-uuid (let [conn (create-db-graph-conn) page-uuid (random-uuid)