fix: rm the last file-based namespaces in db dep

Also remove the following related file schema
attributes and their dependent code:
:block/marker, :block/priority, :block/scheduled,
:block/deadline,:block/repeated?, :block/pre-block?,
:block/properties-order, :block/properties-text-values, :block/macros,
:block/invalid-properties
This commit is contained in:
Gabriel Horner
2026-01-11 10:13:23 -05:00
parent ab137a962a
commit 4bb7430682
31 changed files with 166 additions and 371 deletions

View File

@@ -1,8 +1,4 @@
;; API
logseq.db.file-based.rules/rules
;; API
logseq.db.file-based.schema/retract-attributes
;; API
logseq.db.frontend.rules/db-query-dsl-rules
;; API
logseq.db.frontend.rules/extract-rules
@@ -51,6 +47,6 @@ logseq.db.sqlite.gc/gc-kvs-table-node-version!
;; API
logseq.db.sqlite.gc/ensure-no-garbage
;; API
logseq.db.common.entity-util/entity->map
logseq.db.frontend.entity-util/entity->map
;; documenting keywords
logseq.db.frontend.kv-entity/kv-entities

View File

@@ -10,7 +10,6 @@
{:aliases {clojure.string string
logseq.db ldb
logseq.db.common.entity-plus entity-plus
logseq.db.common.entity-util common-entity-util
logseq.db.common.order db-order
logseq.db.common.property-util db-property-util
logseq.db.common.sqlite common-sqlite
@@ -23,8 +22,6 @@
logseq.db.frontend.property db-property
logseq.db.frontend.property.build db-property-build
logseq.db.frontend.property.type db-property-type
logseq.db.file-based.schema file-schema
logseq.db.file-based.entity-util file-entity-util
logseq.db.frontend.rules rules
logseq.db.frontend.schema db-schema
logseq.db.frontend.validate db-validate

View File

@@ -13,14 +13,11 @@
[logseq.common.uuid :as common-uuid]
[logseq.db.common.delete-blocks :as delete-blocks] ;; Load entity extensions
[logseq.db.common.entity-plus :as entity-plus]
[logseq.db.common.entity-util :as common-entity-util]
[logseq.db.common.initial-data :as common-initial-data]
[logseq.db.file-based.schema :as file-schema]
[logseq.db.frontend.class :as db-class]
[logseq.db.frontend.db :as db-db]
[logseq.db.frontend.entity-util :as entity-util]
[logseq.db.frontend.property :as db-property]
[logseq.db.frontend.schema :as db-schema]
[logseq.db.frontend.validate :as db-validate]
[logseq.db.sqlite.util :as sqlite-util])
(:refer-clojure :exclude [object?]))
@@ -206,13 +203,13 @@
;; transact tx-data to `conn` and validate db
(transact! conn tx-data tx-meta)))))
(def page? common-entity-util/page?)
(def page? entity-util/page?)
(def internal-page? entity-util/internal-page?)
(def class? entity-util/class?)
(def property? entity-util/property?)
(def closed-value? entity-util/closed-value?)
(def whiteboard? common-entity-util/whiteboard?)
(def journal? common-entity-util/journal?)
(def whiteboard? entity-util/whiteboard?)
(def journal? entity-util/journal?)
(def hidden? entity-util/hidden?)
(def object? entity-util/object?)
(def asset? entity-util/asset?)
@@ -655,13 +652,6 @@
[?page :block/tags ?tag]]
db))
(defn get-schema
"Returns schema for given repo"
[repo]
(if (sqlite-util/db-based-graph? repo)
db-schema/schema
file-schema/schema))
(defn page-in-library?
"Check whether a `page` exists on the Library page"
[db page]

View File

@@ -5,8 +5,6 @@
[logseq.common.util :as common-util]
[logseq.common.util.block-ref :as block-ref]
[logseq.common.util.page-ref :as page-ref]
[logseq.db.common.entity-plus :as entity-plus]
[logseq.db.common.entity-util :as common-entity-util]
[logseq.db.frontend.entity-util :as entity-util]))
(defn- replace-ref-with-deleted-block-title
@@ -56,19 +54,11 @@
(contains? #{:db.fn/retractEntity :db/retractEntity} (first tx)))
(second tx))) txs)
(filter (fn [id]
(not (common-entity-util/page? (d/entity db id))))))]
(not (entity-util/page? (d/entity db id))))))]
(when (seq retracted-block-ids)
(let [retracted-blocks (map #(d/entity db %) retracted-block-ids)
retracted-tx (build-retracted-tx retracted-blocks)
retract-history-tx (mapcat (fn [e]
(map (fn [history] [:db/retractEntity (:db/id history)])
(:logseq.property.history/_block e))) retracted-blocks)
macros-tx (when-not (entity-plus/db-based-graph? db)
(mapcat (fn [b]
;; Only delete if last reference
(keep #(when (<= (count (:block/_macros (d/entity db (:db/id %))))
1)
(when (:db/id %) (vector :db.fn/retractEntity (:db/id %))))
(:block/macros b)))
retracted-blocks))]
(concat txs retracted-tx retract-history-tx macros-tx)))))
(:logseq.property.history/_block e))) retracted-blocks)]
(concat txs retracted-tx retract-history-tx)))))

View File

@@ -17,12 +17,9 @@
(def nil-db-ident-entities
"No such entities with these :db/ident, but `(d/entity <db> <ident>)` has been called somewhere."
#{:block/tx-id :block/uuid :block/journal-day :block/_refs :block/level :block/heading-level :block/warning
;; File graph only attributes. Can these be removed if this is only called in db graphs?
:block/pre-block? :block/scheduled :block/deadline :block/type :block/name :block/marker
:block.temp/ast-title
:block.temp/load-status :block.temp/has-children? :block.temp/ast-body
#{:block/tx-id :block/uuid :block/journal-day :block/_refs :block/level :block/heading-level
:block/warning :block/name
:block.temp/ast-title :block.temp/load-status :block.temp/has-children? :block.temp/ast-body
:db/valueType :db/cardinality :db/ident :db/index

View File

@@ -1,26 +0,0 @@
(ns logseq.db.common.entity-util
"Lower level entity util fns for DB and file graphs"
(:require [datascript.impl.entity :as de]
[logseq.db.file-based.entity-util :as file-entity-util]
[logseq.db.frontend.entity-util :as entity-util]))
(defn whiteboard?
[entity]
(or (entity-util/whiteboard? entity)
(file-entity-util/whiteboard? entity)))
(defn journal?
[entity]
(or (entity-util/journal? entity)
(file-entity-util/journal? entity)))
(defn page?
[entity]
(or (entity-util/page? entity)
(file-entity-util/page? entity)))
(defn entity->map
"Convert a db Entity to a map"
[e]
(assert (de/entity? e))
(assoc (into {} e) :db/id (:db/id e)))

View File

@@ -7,7 +7,6 @@
[logseq.common.util :as common-util]
[logseq.common.util.date-time :as date-time-util]
[logseq.db.common.entity-plus :as entity-plus]
[logseq.db.common.entity-util :as common-entity-util]
[logseq.db.common.order :as db-order]
[logseq.db.frontend.class :as db-class]
[logseq.db.frontend.db :as db-db]
@@ -29,7 +28,7 @@
(->> (d/datoms db :avet :block/title page-name)
(filter (fn [d]
(let [e (d/entity db (:e d))]
(common-entity-util/page? e))))
(entity-util/page? e))))
(map :e)
sort
first))
@@ -98,7 +97,7 @@
(let [e (d/entity db eid)]
(when (or include-collapsed-children?
(not (:block/collapsed? e))
(common-entity-util/page? e))
(entity-util/page? e))
(:block/_parent e)))) eids-to-expand)
ids-to-add (keep :db/id children)]
(vswap! seen (partial apply conj) ids-to-add)
@@ -263,7 +262,7 @@
(keep (fn [d]
(when (<= (:v d) today)
(let [e (d/entity db (:e d))]
(when (and (common-entity-util/journal? e) (:db/id e))
(when (and (entity-util/journal? e) (:db/id e))
e))))))))
(defn- get-structured-datoms
@@ -311,7 +310,7 @@
(string/blank? title)))
(let [e (d/entity db (:e datom))]
(when (and
(common-entity-util/page? e)
(entity-util/page? e)
(not (entity-util/hidden? e)))
e))))))
(take 15))))

View File

@@ -8,7 +8,6 @@
[clojure.string :as string]
[datascript.storage :refer [IStorage]]
[logseq.db.common.sqlite :as common-sqlite]
[logseq.db.file-based.schema :as file-schema]
[logseq.db.frontend.schema :as db-schema]
[logseq.db.sqlite.util :as sqlite-util]))
@@ -76,18 +75,14 @@
([db-full-path]
(open-sqlite-datascript! nil db-full-path))
([graphs-dir db-name]
(let [[base-name db-full-path]
(let [db-full-path
(if (nil? graphs-dir)
[(node-path/basename db-name) db-name]
[db-name (second (common-sqlite/get-db-full-path graphs-dir db-name))])
db (new sqlite db-full-path nil)
;; For both desktop and CLI, only file graphs have db-name that indicate their db type
schema (if (common-sqlite/local-file-based-graph? base-name)
file-schema/schema
db-schema/schema)]
db-name
(second (common-sqlite/get-db-full-path graphs-dir db-name)))
db (new sqlite db-full-path nil)]
(common-sqlite/create-kvs-table! db)
(let [storage (new-sqlite-storage db)
conn (common-sqlite/get-storage-conn storage schema)]
conn (common-sqlite/get-storage-conn storage db-schema/schema)]
{:sqlite db
:conn conn}))))

View File

@@ -1,14 +0,0 @@
(ns logseq.db.file-based.entity-util
"Lower level entity util fns for file graphs")
(defn whiteboard?
[entity]
(identical? "whiteboard" (:block/type entity)))
(defn journal?
[entity]
(identical? "journal" (:block/type entity)))
(defn page?
[entity]
(contains? #{"page" "journal" "whiteboard"} (:block/type entity)))

View File

@@ -1,120 +0,0 @@
(ns logseq.db.file-based.schema
"Schema related vars for file graphs")
;; A page is a special block, a page can corresponds to multiple files with the same ":block/name".
(def ^:large-vars/data-var schema
"Schema for file graphs"
{:db/ident {:db/unique :db.unique/identity}
:kv/value {}
:recent/pages {}
;; :block/type is a string type of the current block
;; "whiteboard" for whiteboards
;; "property" for property blocks
;; "class" for structured page
:block/type {:db/index true}
:block/uuid {:db/unique :db.unique/identity}
:block/parent {:db/valueType :db.type/ref
:db/index true}
:block/order {:db/index true}
:block/collapsed? {}
;; :markdown, :org
:block/format {}
;; belongs to which page
:block/page {:db/valueType :db.type/ref
:db/index true}
;; reference blocks
:block/refs {:db/valueType :db.type/ref
:db/cardinality :db.cardinality/many}
:block/tags {:db/valueType :db.type/ref
:db/cardinality :db.cardinality/many}
;; which block this block links to, used for tag, embeds
:block/link {:db/valueType :db.type/ref
:db/index true}
;; page's namespace
:block/namespace {:db/valueType :db.type/ref}
;; for pages
:block/alias {:db/valueType :db.type/ref
:db/cardinality :db.cardinality/many
:db/index true}
;; todo keywords, e.g. "TODO", "DOING", "DONE"
:block/marker {}
;; "A", "B", "C"
:block/priority {}
;; map, key -> set of refs in property value or full text if none are found
:block/properties {}
;; vector
:block/properties-order {}
;; map, key -> original property value's content
:block/properties-text-values {}
;; first block that's not a heading or unordered list
:block/pre-block? {}
;; scheduled day
:block/scheduled {}
;; deadline day
:block/deadline {}
;; whether blocks is a repeated block (usually a task)
:block/repeated? {}
:block/created-at {:db/index true}
:block/updated-at {:db/index true}
;; page additional attributes
;; page's name, lowercase
:block/name {:db/unique :db.unique/identity}
;; page's original name
:block/title {:db/index true}
;; page's journal day
:block/journal-day {:db/index true}
;; macros in block
:block/macros {:db/valueType :db.type/ref
:db/cardinality :db.cardinality/many}
;; block's file
:block/file {:db/valueType :db.type/ref}
;; latest tx that affected the block
:block/tx-id {}
;; file
:file/path {:db/unique :db.unique/identity}
:file/content {}
:file/created-at {}
:file/last-modified-at {}
:file/size {}})
(def file-only-attributes
[:block/namespace :block/properties-text-values :block/pre-block? :recent/pages :block/file
:block/properties :block/properties-order :block/repeated? :block/deadline :block/scheduled :block/priority
:block/marker :block/macros :block/type :block/format])
(def retract-attributes
#{:block/tags
:block/alias
:block/marker
:block/priority
:block/scheduled
:block/deadline
:block/repeated?
:block/pre-block?
:block/properties
:block/properties-order
:block/properties-text-values
:block/macros
:block/invalid-properties
:block/warning})

View File

@@ -4,7 +4,6 @@
[datascript.core :as d]
[logseq.common.util :as common-util]
[logseq.common.util.page-ref :as page-ref]
[logseq.db.common.entity-util :as common-entity-util]
[logseq.db.frontend.entity-util :as entity-util]))
;; [[uuid]]
@@ -49,7 +48,7 @@
;; The caller need to handle situations including
;; mutual references and circle references.
refs*
(cond->> (filter common-entity-util/page? refs*)
(cond->> (filter entity-util/page? refs*)
(and db (false? replace-pages-with-same-name?))
(remove (fn [e]
(> (count (entity-util/get-pages-by-name db (:block/title e))) 1)))))

View File

@@ -92,3 +92,9 @@
(defn get-pages-by-name
[db page-name]
(d/datoms db :avet :block/name (common-util/page-name-sanity-lc page-name)))
(defn entity->map
"Convert a db Entity to a map"
[e]
(assert (de/entity? e))
(assoc (into {} e) :db/id (:db/id e)))

View File

@@ -1,8 +1,7 @@
(ns logseq.db.frontend.schema
"Schema related fns for DB and file graphs"
(:require [clojure.set :as set]
[clojure.string :as string]
[logseq.db.file-based.schema :as file-schema]))
[clojure.string :as string]))
(def schema-version? (every-pred map? :major))
@@ -60,14 +59,59 @@
(str (:major schema-version)))
:else (throw (ex-info "Not a schema-version" {:data schema-version}))))
(def schema
"Schema for DB graphs. :block/tags are classes in this schema"
(merge
(apply dissoc file-schema/schema file-schema/file-only-attributes)
{:block/name {:db/index true} ; remove db/unique for :block/name
;; closed value
:block/closed-value-property {:db/valueType :db.type/ref
:db/cardinality :db.cardinality/many}}))
(def ^:large-vars/data-var schema
"Schema for file graphs"
{:db/ident {:db/unique :db.unique/identity}
:kv/value {}
:block/uuid {:db/unique :db.unique/identity}
:block/parent {:db/valueType :db.type/ref
:db/index true}
:block/order {:db/index true}
:block/collapsed? {}
;; belongs to which page
:block/page {:db/valueType :db.type/ref
:db/index true}
;; reference blocks
:block/refs {:db/valueType :db.type/ref
:db/cardinality :db.cardinality/many}
:block/tags {:db/valueType :db.type/ref
:db/cardinality :db.cardinality/many}
;; which block this block links to, used for tag, embeds
:block/link {:db/valueType :db.type/ref
:db/index true}
;; for pages
:block/alias {:db/valueType :db.type/ref
:db/cardinality :db.cardinality/many
:db/index true}
:block/created-at {:db/index true}
:block/updated-at {:db/index true}
;; page additional attributes
;; page's name, lowercase
:block/name {:db/index true} ; remove db/unique for :block/name
;; page's original name
:block/title {:db/index true}
;; page's journal day
:block/journal-day {:db/index true}
;; latest tx that affected the block
:block/tx-id {}
:block/closed-value-property {:db/valueType :db.type/ref
:db/cardinality :db.cardinality/many}
;; file
:file/path {:db/unique :db.unique/identity}
:file/content {}
:file/created-at {}
:file/last-modified-at {}
:file/size {}})
;; If only block/title changes
(def retract-attributes

View File

@@ -15,6 +15,7 @@
[logseq.db :as ldb]
[logseq.db.common.order :as db-order]
[logseq.db.frontend.class :as db-class]
[logseq.db.frontend.entity-util :as entity-util]
[logseq.graph-parser.mldoc :as gp-mldoc]
[logseq.graph-parser.property :as gp-property]
[logseq.graph-parser.text :as text]
@@ -365,6 +366,12 @@
[s]
(string/replace s "#" "HashTag-"))
(defn- page-entity?
"Support DB or file graphs because of exporter"
[entity]
(or (entity-util/page? entity)
(contains? #{"page" "journal" "whiteboard"} (:block/type entity))))
;; TODO: refactor
(defn page-name->map
"Create a page's map structure given a original page name (string).
@@ -376,7 +383,7 @@
[original-page-name db with-timestamp? date-formatter
& {:keys [page-uuid class?] :as options}]
(when-not (and db (common-util/uuid-string? original-page-name)
(not (ldb/page? (d/entity db [:block/uuid (uuid original-page-name)]))))
(not (page-entity? (d/entity db [:block/uuid (uuid original-page-name)]))))
(let [db-based? (ldb/db-based-graph? db)
original-page-name (cond-> (string/trim original-page-name)
db-based?

View File

@@ -27,6 +27,7 @@
[logseq.db.frontend.class :as db-class]
[logseq.db.frontend.content :as db-content]
[logseq.db.frontend.db-ident :as db-ident]
[logseq.db.frontend.entity-util :as entity-util]
[logseq.db.frontend.malli-schema :as db-malli-schema]
[logseq.db.frontend.property :as db-property]
[logseq.db.frontend.property.build :as db-property-build]
@@ -328,6 +329,21 @@
(string/includes? path (str "whiteboards" "/"))
(string/ends-with? path ".edn")))
(defn- whiteboard-entity?
[entity]
(or (entity-util/whiteboard? entity)
(identical? "whiteboard" (:block/type entity))))
(defn- journal-entity?
[entity]
(or (entity-util/journal? entity)
(identical? "journal" (:block/type entity))))
(defn- page-entity?
[entity]
(or (entity-util/page? entity)
(contains? #{"page" "journal" "whiteboard"} (:block/type entity))))
(defn- find-or-create-deadline-scheduled-value
"Given a :block/scheduled or :block/deadline value, creates the datetime property value
and any optional journal tx associated with that value"
@@ -413,7 +429,7 @@
(let [prop-type (cond (and (coll? prop-val)
(seq prop-val)
(set/subset? prop-val
(set (keep #(when (ldb/journal? %)
(set (keep #(when (journal-entity? %)
(:block/title %)) refs))))
:date
(and (coll? prop-val) (seq prop-val) (text-with-refs? prop-val prop-val-text))
@@ -1863,7 +1879,7 @@
{:keys [pages-tx page-properties-tx per-file-state existing-pages]} (build-pages-tx conn pages blocks tx-options)
whiteboard-pages (->> pages-tx
;; support old and new whiteboards
(filter ldb/whiteboard?)
(filter whiteboard-entity?)
(map (fn [page-block]
(-> page-block
(assoc :logseq.property/ls-type :whiteboard-page)))))
@@ -2136,7 +2152,7 @@
(keep (fn [d]
(let [child (d/entity db (:e d))
parent (d/entity db (:v d))]
(when (and (nil? (:block/parent parent)) (ldb/page? child) (ldb/page? parent))
(when (and (nil? (:block/parent parent)) (page-entity? child) (page-entity? parent))
parent))))
(common-util/distinct-by :block/uuid))
tx-data (map

View File

@@ -1,7 +1,6 @@
(ns logseq.graph-parser.extract-test
(:require [cljs.test :refer [deftest is are]]
[datascript.core :as d]
[logseq.db.file-based.schema :as file-schema]
[logseq.graph-parser.extract :as extract]))
;; This is a copy of frontend.components.repo/multiplatform-reserved-chars for reserved chars testing
@@ -42,10 +41,15 @@
(is (= "asldk lakls" (#'extract/path->file-body "file://data/app/asldk lakls.as")))
(is (= "中文asldk lakls" (#'extract/path->file-body "file://中文data/app/中文asldk lakls.as"))))
;; Bare minimum schema to test extract
(def file-schema
{:block/uuid {:db/unique :db.unique/identity}
:block/name {:db/unique :db.unique/identity}})
(defn- extract [file content & [options]]
(extract/extract file
content
(merge {:block-pattern "-" :db (d/empty-db file-schema/schema)
(merge {:block-pattern "-" :db (d/empty-db file-schema)
:verbose false}
options)))

View File

@@ -83,28 +83,13 @@
(swap! txs-state (fn [state] (vec (concat state tx)))))))))
(defn- update-page-when-save-block
[txs-state block-entity m]
[txs-state block-entity]
(when-let [e (:block/page block-entity)]
(let [m' (cond-> {:db/id (:db/id e)
:block/updated-at (common-util/time-ms)}
(not (:block/created-at e))
(assoc :block/created-at (common-util/time-ms)))
txs (if (or (:block/pre-block? block-entity)
(:block/pre-block? m))
(let [properties (:block/properties m)
alias (set (:alias properties))
tags (set (:tags properties))
alias (map (fn [p] {:block/name (common-util/page-name-sanity-lc p)}) alias)
tags (map (fn [p] {:block/name (common-util/page-name-sanity-lc p)}) tags)
deleteable-page-attributes {:block/alias alias
:block/tags tags
:block/properties properties
:block/properties-text-values (:block/properties-text-values m)}
;; Retract page attributes to allow for deletion of page attributes
page-retractions
(mapv #(vector :db/retract (:db/id e) %) (keys deleteable-page-attributes))]
(conj page-retractions (merge m' deleteable-page-attributes)))
[m'])]
txs [m']]
(swap! txs-state into txs))))
(defn- remove-orphaned-refs-when-save
@@ -295,7 +280,7 @@
(outliner-validate/validate-block-title db (:block/title m*) block-entity))
m (cond-> m*
true
(dissoc :block/format :block/pre-block? :block/priority :block/marker :block/properties-order))]
(dissoc :block/format))]
;; Ensure block UUID never changes
(let [e (d/entity db db-id)]
(when (and e block-uuid)
@@ -320,7 +305,7 @@
;; Update block's page attributes
(when-not collapse-or-expand?
(update-page-when-save-block *txs-state block-entity m))
(update-page-when-save-block *txs-state block-entity))
;; Remove orphaned refs from block
(when (and (:block/title m) (not= (:block/title m) (:block/title block-entity)))
(remove-orphaned-refs-when-save db *txs-state block-entity m)))
@@ -355,16 +340,8 @@
[:db/retract (:db/id block) :block/order]
[:db/retract (:db/id block) :block/page]])
(let [ids (cons (:db/id this) (ldb/get-block-full-children-ids db (:db/id block)))
txs (map (fn [id] [:db.fn/retractEntity id]) ids)
page-tx (let [block (d/entity db [:block/uuid block-id])]
(when (:block/pre-block? block)
(when-let [id (:db/id (:block/page block))]
[[:db/retract id :block/properties]
[:db/retract id :block/properties-order]
[:db/retract id :block/properties-text-values]
[:db/retract id :block/alias]
[:db/retract id :block/tags]])))]
(swap! *txs-state concat txs page-tx)
txs (map (fn [id] [:db.fn/retractEntity id]) ids)]
(swap! *txs-state concat txs)
block-id)))))
(defn- assoc-level-aux