clone/logseq
1
0
Fork 0
mirror of https://github.com/logseq/logseq.git synced 2026-02-01 22:47:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: split logseq.db.common.sqlite into 2 namespaces

Browse Source 
The majority of this ns is focused on graph initilization and
entity lazy loading so split this out to new ns,
logseq.db.common.initial-data. Originally
initialization was tied to sqlite util fns but it has not made
sense for awhile to think of them as sqlite util fns
This commit is contained in:
Gabriel Horner
2025-04-28 12:50:09 -04:00
parent 9067635054
commit d3b7289b20
16 changed files with 380 additions and 382 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.clj-kondo/config.edn
View File

@@ -173,10 +173,12 @@
logseq.common.util.namespace ns-util
logseq.common.util.page-ref page-ref
logseq.db ldb
logseq.db.common.entity-plus entity-plus
logseq.db.common.entity-util common-entity-util
logseq.db.common.initial-data common-initial-data
logseq.db.common.order db-order
logseq.db.common.property-util db-property-util
logseq.db.common.sqlite sqlite-common-db
logseq.db.common.sqlite common-sqlite
logseq.db.common.view db-view
logseq.db.file-based.rules file-rules
logseq.db.file-based.schema file-schema
@@ -185,7 +187,6 @@
logseq.db.frontend.content db-content
logseq.db.frontend.db db-db
logseq.db.frontend.db-ident db-ident
logseq.db.common.entity-plus entity-plus
logseq.db.frontend.entity-util entity-util
logseq.db.frontend.inputs db-inputs
logseq.db.frontend.property db-property

6
deps/db/.carve/ignore vendored
View File

@@ -28,3 +28,9 @@ logseq.db.sqlite.export/build-import
logseq.db.common.view/get-property-values
;; API
logseq.db.common.view/get-view-data
;; API
logseq.db.common.initial-data/with-parent
;; API
logseq.db.common.initial-data/get-block-and-children
;; API
logseq.db.common.initial-data/get-initial-data

5
deps/db/.clj-kondo/config.edn vendored
View File

@@ -9,10 +9,11 @@
:consistent-alias
{:aliases {clojure.string string
logseq.db ldb
logseq.db.common.entity-plus entity-plus
logseq.db.common.entity-util common-entity-util
logseq.db.common.order db-order
logseq.db.common.property-util db-property-util
logseq.db.common.sqlite sqlite-common-db
logseq.db.common.sqlite common-sqlite
logseq.db.common.view db-view
logseq.db.frontend.content db-content
logseq.db.frontend.class db-class
@@ -25,11 +26,11 @@
logseq.db.file-based.rules file-rules
logseq.db.file-based.schema file-schema
logseq.db.file-based.entity-util file-entity-util
logseq.db.common.entity-plus entity-plus
logseq.db.frontend.rules rules
logseq.db.frontend.schema db-schema
logseq.db.frontend.validate db-validate
logseq.db.sqlite.build sqlite-build
logseq.db.common.initial-data common-initial-data
logseq.db.common.sqlite-cli sqlite-cli
logseq.db.sqlite.create-graph sqlite-create-graph
logseq.db.sqlite.export sqlite-export

14
deps/db/src/logseq/db.cljs vendored
View File

@@ -12,7 +12,7 @@
[logseq.db.common.delete-blocks :as delete-blocks] ;; Load entity extensions
[logseq.db.common.entity-plus :as entity-plus]
[logseq.db.common.entity-util :as common-entity-util]
[logseq.db.common.sqlite :as sqlite-common-db]
[logseq.db.common.initial-data :as common-initial-data]
[logseq.db.file-based.schema :as file-schema]
[logseq.db.frontend.class :as db-class]
[logseq.db.frontend.db :as db-db]
@@ -192,7 +192,7 @@
(map first)
(remove hidden?)))
(def get-first-page-by-name sqlite-common-db/get-first-page-by-name)
(def get-first-page-by-name common-initial-data/get-first-page-by-name)
(def db-based-graph? entity-plus/db-based-graph?)
@@ -250,7 +250,7 @@
(if-let [id (if (uuid? page-name-or-uuid) page-name-or-uuid
(parse-uuid page-name-or-uuid))]
(d/entity db [:block/uuid id])
(d/entity db (sqlite-common-db/get-first-page-by-title db page-name-or-uuid)))))
(d/entity db (common-initial-data/get-first-page-by-title db page-name-or-uuid)))))
(defn page-empty?
"Whether a page is empty. Does it has a non-page block?
@@ -349,8 +349,8 @@
(recur (:block/uuid parent) (conj parents' parent) (inc d))
parents'))))
(def get-block-children-ids sqlite-common-db/get-block-children-ids)
(def get-block-children sqlite-common-db/get-block-children)
(def get-block-children-ids common-initial-data/get-block-children-ids)
(def get-block-children common-initial-data/get-block-children)
(defn- get-sorted-page-block-ids
[db page-id]
@@ -543,9 +543,9 @@
(def get-class-ident-by-display-type db-db/get-class-ident-by-display-type)
(def get-display-type-by-class-ident db-db/get-display-type-by-class-ident)
(def get-recent-updated-pages sqlite-common-db/get-recent-updated-pages)
(def get-recent-updated-pages common-initial-data/get-recent-updated-pages)
(def get-latest-journals sqlite-common-db/get-latest-journals)
(def get-latest-journals common-initial-data/get-latest-journals)
(defn get-pages-relation
[db with-journal?]

321
deps/db/src/logseq/db/common/initial_data.cljs vendored Normal file
View File

@@ -0,0 +1,321 @@
(ns logseq.db.common.initial-data
"Provides db helper fns for graph initialization and lazy loading entities"
(:require [clojure.set :as set]
[datascript.core :as d]
[datascript.impl.entity :as de]
[logseq.common.config :as common-config]
[logseq.common.util :as common-util]
[logseq.common.util.date-time :as date-time-util]
[logseq.db.common.entity-util :as common-entity-util]
[logseq.db.common.order :as db-order]
[logseq.db.common.entity-plus :as entity-plus]
[logseq.db.frontend.entity-util :as entity-util]))
(defn- get-pages-by-name
[db page-name]
(d/datoms db :avet :block/name (common-util/page-name-sanity-lc page-name)))
(defn get-first-page-by-name
"Return the oldest page's db id for :block/name"
[db page-name]
(when (and db (string? page-name))
(first (sort (map :e (get-pages-by-name db page-name))))))
(defn get-first-page-by-title
"Return the oldest page's db id for :block/title"
[db page-name]
{:pre [(string? page-name)]}
(->> (d/datoms db :avet :block/title page-name)
(filter (fn [d]
(let [e (d/entity db (:e d))]
(common-entity-util/page? e))))
(map :e)
sort
first))
(comment
(defn- get-built-in-files
[db]
(let [files ["logseq/config.edn"
"logseq/custom.css"
"logseq/custom.js"]]
(map #(d/pull db '[*] [:file/path %]) files))))
(defn- get-all-files
[db]
(->> (d/datoms db :avet :file/path)
(mapcat (fn [e] (d/datoms db :eavt (:e e))))))
(defn- with-block-refs
[db block]
(update block :block/refs (fn [refs] (map (fn [ref] (d/pull db '[*] (:db/id ref))) refs))))
(defn with-parent
[db block]
(cond
(:block/page block)
(let [parent (when-let [e (d/entity db (:db/id (:block/parent block)))]
(select-keys e [:db/id :block/uuid]))]
(->>
(assoc block :block/parent parent)
(common-util/remove-nils-non-nested)
(with-block-refs db)))
:else
block))
(comment
(defn- with-block-link
[db block]
(if (:block/link block)
(update block :block/link (fn [link] (d/pull db '[*] (:db/id link))))
block)))
(defn- mark-block-fully-loaded
[b]
(assoc b :block.temp/fully-loaded? true))
(comment
(defn- property-without-db-attrs
[property]
(dissoc property :db/index :db/valueType :db/cardinality))
(defn- property-with-values
[db block properties]
(when (entity-plus/db-based-graph? db)
(let [block (d/entity db (:db/id block))
property-vals (if properties
(map block properties)
(vals (:block/properties block)))]
(->> property-vals
(mapcat
(fn [property-values]
(let [values (->>
(if (and (coll? property-values)
(map? (first property-values)))
property-values
#{property-values}))
value-ids (when (every? map? values)
(->> (map :db/id values)
(filter (fn [id] (or (int? id) (keyword? id))))))
value-blocks (->>
(when (seq value-ids)
(map
(fn [id] (d/pull db '[:db/id :block/uuid
:block/name :block/title
:logseq.property/value
:block/tags :block/page
:logseq.property/created-from-property] id))
value-ids))
;; FIXME: why d/pull returns {:db/id db-ident} instead of {:db/id number-eid}?
(keep (fn [block]
(let [from-property-id (get-in block [:logseq.property/created-from-property :db/id])]
(if (keyword? from-property-id)
(assoc-in block [:logseq.property/created-from-property :db/id] (:db/id (d/entity db from-property-id)))
block)))))]
value-blocks))))))))
(defn get-block-children-ids
"Returns children UUIDs"
[db block-uuid]
(when-let [eid (:db/id (d/entity db [:block/uuid block-uuid]))]
(let [seen (volatile! [])]
(loop [steps 100 ;check result every 100 steps
eids-to-expand [eid]]
(when (seq eids-to-expand)
(let [eids-to-expand*
(mapcat (fn [eid] (map first (d/datoms db :avet :block/parent eid))) eids-to-expand)
uuids-to-add (remove nil? (map #(:block/uuid (d/entity db %)) eids-to-expand*))]
(when (and (zero? steps)
(seq (set/intersection (set @seen) (set uuids-to-add))))
(throw (ex-info "bad outliner data, need to re-index to fix"
{:seen @seen :eids-to-expand eids-to-expand})))
(vswap! seen (partial apply conj) uuids-to-add)
(recur (if (zero? steps) 100 (dec steps)) eids-to-expand*))))
@seen)))
(defn get-block-children
"Including nested children."
[db block-uuid]
(let [ids (get-block-children-ids db block-uuid)]
(when (seq ids)
(let [ids' (map (fn [id] [:block/uuid id]) ids)]
(d/pull-many db '[*] ids')))))
(defn- with-raw-title
[m entity]
(if-let [raw-title (:block/raw-title entity)]
(assoc m :block/title raw-title)
m))
(defn- entity->map
[entity]
(-> (into {} entity)
(with-raw-title entity)
(assoc :db/id (:db/id entity))))
(defn ^:large-vars/cleanup-todo get-block-and-children
[db id {:keys [children? children-only? nested-children? properties children-props]}]
(let [block (d/entity db (if (uuid? id)
[:block/uuid id]
id))
block-refs-count? (some #{:block.temp/refs-count} properties)
whiteboard? (common-entity-util/whiteboard? block)]
(when block
(let [children (when (or children? children-only?)
(let [page? (common-entity-util/page? block)
children (->>
(cond
(and nested-children? (not page?))
(get-block-children db (:block/uuid block))
nested-children?
(:block/_page block)
:else
(let [short-page? (when page?
(<= (count (:block/_page block)) 100))]
(if short-page?
(:block/_page block)
(:block/_parent block))))
(remove (fn [e] (or (:logseq.property/created-from-property e)
(:block/closed-value-property e)))))
children-props (if whiteboard?
'[*]
(or children-props
[:db/id :block/uuid :block/parent :block/order :block/collapsed? :block/title
;; pre-loading feature-related properties to avoid UI refreshing
:logseq.task/status :logseq.property.node/display-type]))]
(map
(fn [block]
(if (= children-props '[*])
(entity->map block)
(-> (select-keys block children-props)
(with-raw-title block))))
children)))]
(if children-only?
{:children children}
(let [block' (if (seq properties)
(-> (select-keys block properties)
(with-raw-title block)
(assoc :db/id (:db/id block)))
(entity->map block))
block' (cond->
(mark-block-fully-loaded block')
true
(update-vals (fn [v]
(cond
(de/entity? v)
(entity->map v)
(and (coll? v) (every? de/entity? v))
(map entity->map v)
:else
v)))
block-refs-count?
(assoc :block.temp/refs-count (count (:block/_refs block))))]
(cond->
{:block block'}
children?
(assoc :children children))))))))
(defn get-latest-journals
[db]
(let [today (date-time-util/date->int (js/Date.))]
(->> (d/datoms db :avet :block/journal-day)
vec
rseq
(keep (fn [d]
(and (<= (:v d) today)
(let [e (d/entity db (:e d))]
(when (and (common-entity-util/journal? e) (:db/id e))
e))))))))
(defn- get-structured-datoms
[db]
(let [class-property-id (:db/id (d/entity db :logseq.class/Property))]
(->> (concat
(d/datoms db :avet :block/tags :logseq.class/Tag)
(d/datoms db :avet :block/tags :logseq.class/Property)
(d/datoms db :avet :block/closed-value-property))
(mapcat (fn [d]
(let [block-datoms (d/datoms db :eavt (:e d))
property-desc-datoms (when (= (:v d) class-property-id)
(when-let [desc (:logseq.property/description (d/entity db (:e d)))]
(d/datoms db :eavt (:db/id desc))))]
(if property-desc-datoms
(concat block-datoms property-desc-datoms)
block-datoms)))))))
(defn- get-favorites
"Favorites page and its blocks"
[db]
(let [page-id (get-first-page-by-name db common-config/favorites-page-name)
block (d/entity db page-id)
children (:block/_page block)]
(when block
(concat (d/datoms db :eavt (:db/id block))
(->> (keep :block/link children)
(mapcat (fn [l]
(d/datoms db :eavt (:db/id l)))))
(mapcat (fn [child]
(d/datoms db :eavt (:db/id child)))
children)))))
(defn- get-views-data
[db]
(let [page-id (get-first-page-by-name db common-config/views-page-name)
children (when page-id (:block/_parent (d/entity db page-id)))]
(when (seq children)
(into
(mapcat (fn [b] (d/datoms db :eavt (:db/id b)))
children)
(d/datoms db :eavt page-id)))))
(defn get-recent-updated-pages
[db]
(some->>
(d/datoms db :avet :block/updated-at)
rseq
(keep (fn [datom]
(let [e (d/entity db (:e datom))]
(when (and (common-entity-util/page? e) (not (entity-util/hidden? e)))
e))))
(take 30)))
(defn get-initial-data
"Returns current database schema and initial data.
NOTE: This fn is called by DB and file graphs"
[db]
(let [db-graph? (entity-plus/db-based-graph? db)
_ (when db-graph?
(reset! db-order/*max-key (db-order/get-max-order db)))
schema (:schema db)
idents (mapcat (fn [id]
(when-let [e (d/entity db id)]
(d/datoms db :eavt (:db/id e))))
[:logseq.kv/db-type
:logseq.kv/schema-version
:logseq.kv/graph-uuid
:logseq.kv/latest-code-lang
:logseq.kv/graph-backup-folder
:logseq.property/empty-placeholder])
favorites (when db-graph? (get-favorites db))
views (when db-graph? (get-views-data db))
all-files (get-all-files db)
structured-datoms (when db-graph?
(get-structured-datoms db))
recent-updated-pages (let [pages (get-recent-updated-pages db)]
(mapcat (fn [p] (d/datoms db :eavt (:db/id p))) pages))
pages-datoms (let [contents-id (get-first-page-by-title db "Contents")
views-id (get-first-page-by-title db common-config/views-page-name)]
(mapcat #(d/datoms db :eavt %)
(remove nil? [contents-id views-id])))
data (distinct
(concat idents
structured-datoms
favorites
recent-updated-pages
views
all-files
pages-datoms))]
{:schema schema
:initial-data data}))

334
deps/db/src/logseq/db/common/sqlite.cljs vendored
View File

@@ -1,343 +1,11 @@
(ns logseq.db.common.sqlite
"Provides common sqlite db fns for file and DB graphs. These fns work on
"Provides common sqlite util fns for file and DB graphs. These fns work on
browser and node"
(:require ["path" :as node-path]
[clojure.set :as set]
[clojure.string :as string]
[datascript.core :as d]
[datascript.impl.entity :as de]
[logseq.common.config :as common-config]
[logseq.common.util :as common-util]
[logseq.common.util.date-time :as date-time-util]
[logseq.db.common.entity-util :as common-entity-util]
[logseq.db.common.order :as db-order]
[logseq.db.common.entity-plus :as entity-plus]
[logseq.db.frontend.entity-util :as entity-util]
[logseq.db.sqlite.util :as sqlite-util]))
(defn- get-pages-by-name
[db page-name]
(d/datoms db :avet :block/name (common-util/page-name-sanity-lc page-name)))
(defn get-first-page-by-name
"Return the oldest page's db id for :block/name"
[db page-name]
(when (and db (string? page-name))
(first (sort (map :e (get-pages-by-name db page-name))))))
(defn get-first-page-by-title
"Return the oldest page's db id for :block/title"
[db page-name]
{:pre [(string? page-name)]}
(->> (d/datoms db :avet :block/title page-name)
(filter (fn [d]
(let [e (d/entity db (:e d))]
(common-entity-util/page? e))))
(map :e)
sort
first))
(comment
(defn- get-built-in-files
[db]
(let [files ["logseq/config.edn"
"logseq/custom.css"
"logseq/custom.js"]]
(map #(d/pull db '[*] [:file/path %]) files))))
(defn get-all-files
[db]
(->> (d/datoms db :avet :file/path)
(mapcat (fn [e] (d/datoms db :eavt (:e e))))))
(defn- with-block-refs
[db block]
(update block :block/refs (fn [refs] (map (fn [ref] (d/pull db '[*] (:db/id ref))) refs))))
(defn with-parent
[db block]
(cond
(:block/page block)
(let [parent (when-let [e (d/entity db (:db/id (:block/parent block)))]
(select-keys e [:db/id :block/uuid]))]
(->>
(assoc block :block/parent parent)
(common-util/remove-nils-non-nested)
(with-block-refs db)))
:else
block))
(comment
(defn- with-block-link
[db block]
(if (:block/link block)
(update block :block/link (fn [link] (d/pull db '[*] (:db/id link))))
block)))
(defn- mark-block-fully-loaded
[b]
(assoc b :block.temp/fully-loaded? true))
(comment
(defn- property-without-db-attrs
[property]
(dissoc property :db/index :db/valueType :db/cardinality))
(defn- property-with-values
[db block properties]
(when (entity-plus/db-based-graph? db)
(let [block (d/entity db (:db/id block))
property-vals (if properties
(map block properties)
(vals (:block/properties block)))]
(->> property-vals
(mapcat
(fn [property-values]
(let [values (->>
(if (and (coll? property-values)
(map? (first property-values)))
property-values
#{property-values}))
value-ids (when (every? map? values)
(->> (map :db/id values)
(filter (fn [id] (or (int? id) (keyword? id))))))
value-blocks (->>
(when (seq value-ids)
(map
(fn [id] (d/pull db '[:db/id :block/uuid
:block/name :block/title
:logseq.property/value
:block/tags :block/page
:logseq.property/created-from-property] id))
value-ids))
;; FIXME: why d/pull returns {:db/id db-ident} instead of {:db/id number-eid}?
(keep (fn [block]
(let [from-property-id (get-in block [:logseq.property/created-from-property :db/id])]
(if (keyword? from-property-id)
(assoc-in block [:logseq.property/created-from-property :db/id] (:db/id (d/entity db from-property-id)))
block)))))]
value-blocks))))))))
(defn get-block-children-ids
"Returns children UUIDs"
[db block-uuid]
(when-let [eid (:db/id (d/entity db [:block/uuid block-uuid]))]
(let [seen (volatile! [])]
(loop [steps 100 ;check result every 100 steps
eids-to-expand [eid]]
(when (seq eids-to-expand)
(let [eids-to-expand*
(mapcat (fn [eid] (map first (d/datoms db :avet :block/parent eid))) eids-to-expand)
uuids-to-add (remove nil? (map #(:block/uuid (d/entity db %)) eids-to-expand*))]
(when (and (zero? steps)
(seq (set/intersection (set @seen) (set uuids-to-add))))
(throw (ex-info "bad outliner data, need to re-index to fix"
{:seen @seen :eids-to-expand eids-to-expand})))
(vswap! seen (partial apply conj) uuids-to-add)
(recur (if (zero? steps) 100 (dec steps)) eids-to-expand*))))
@seen)))
(defn get-block-children
"Including nested children."
[db block-uuid]
(let [ids (get-block-children-ids db block-uuid)]
(when (seq ids)
(let [ids' (map (fn [id] [:block/uuid id]) ids)]
(d/pull-many db '[*] ids')))))
(defn- with-raw-title
[m entity]
(if-let [raw-title (:block/raw-title entity)]
(assoc m :block/title raw-title)
m))
(defn- entity->map
[entity]
(-> (into {} entity)
(with-raw-title entity)
(assoc :db/id (:db/id entity))))
(defn ^:large-vars/cleanup-todo get-block-and-children
[db id {:keys [children? children-only? nested-children? properties children-props]}]
(let [block (d/entity db (if (uuid? id)
[:block/uuid id]
id))
block-refs-count? (some #{:block.temp/refs-count} properties)
whiteboard? (common-entity-util/whiteboard? block)]
(when block
(let [children (when (or children? children-only?)
(let [page? (common-entity-util/page? block)
children (->>
(cond
(and nested-children? (not page?))
(get-block-children db (:block/uuid block))
nested-children?
(:block/_page block)
:else
(let [short-page? (when page?
(<= (count (:block/_page block)) 100))]
(if short-page?
(:block/_page block)
(:block/_parent block))))
(remove (fn [e] (or (:logseq.property/created-from-property e)
(:block/closed-value-property e)))))
children-props (if whiteboard?
'[*]
(or children-props
[:db/id :block/uuid :block/parent :block/order :block/collapsed? :block/title
;; pre-loading feature-related properties to avoid UI refreshing
:logseq.task/status :logseq.property.node/display-type]))]
(map
(fn [block]
(if (= children-props '[*])
(entity->map block)
(-> (select-keys block children-props)
(with-raw-title block))))
children)))]
(if children-only?
{:children children}
(let [block' (if (seq properties)
(-> (select-keys block properties)
(with-raw-title block)
(assoc :db/id (:db/id block)))
(entity->map block))
block' (cond->
(mark-block-fully-loaded block')
true
(update-vals (fn [v]
(cond
(de/entity? v)
(entity->map v)
(and (coll? v) (every? de/entity? v))
(map entity->map v)
:else
v)))
block-refs-count?
(assoc :block.temp/refs-count (count (:block/_refs block))))]
(cond->
{:block block'}
children?
(assoc :children children))))))))
(defn get-latest-journals
[db]
(let [today (date-time-util/date->int (js/Date.))]
(->> (d/datoms db :avet :block/journal-day)
vec
rseq
(keep (fn [d]
(and (<= (:v d) today)
(let [e (d/entity db (:e d))]
(when (and (common-entity-util/journal? e) (:db/id e))
e))))))))
(defn get-page->refs-count
[db]
(let [datoms (d/datoms db :avet :block/name)]
(->>
(map (fn [d]
[(:e d)
(count (:block/_refs (d/entity db (:e d))))]) datoms)
(into {}))))
(defn get-structured-datoms
[db]
(let [class-property-id (:db/id (d/entity db :logseq.class/Property))]
(->> (concat
(d/datoms db :avet :block/tags :logseq.class/Tag)
(d/datoms db :avet :block/tags :logseq.class/Property)
(d/datoms db :avet :block/closed-value-property))
(mapcat (fn [d]
(let [block-datoms (d/datoms db :eavt (:e d))
property-desc-datoms (when (= (:v d) class-property-id)
(when-let [desc (:logseq.property/description (d/entity db (:e d)))]
(d/datoms db :eavt (:db/id desc))))]
(if property-desc-datoms
(concat block-datoms property-desc-datoms)
block-datoms)))))))
(defn get-favorites
"Favorites page and its blocks"
[db]
(let [page-id (get-first-page-by-name db common-config/favorites-page-name)
block (d/entity db page-id)
children (:block/_page block)]
(when block
(concat (d/datoms db :eavt (:db/id block))
(->> (keep :block/link children)
(mapcat (fn [l]
(d/datoms db :eavt (:db/id l)))))
(mapcat (fn [child]
(d/datoms db :eavt (:db/id child)))
children)))))
(defn get-views-data
[db]
(let [page-id (get-first-page-by-name db common-config/views-page-name)
children (when page-id (:block/_parent (d/entity db page-id)))]
(when (seq children)
(into
(mapcat (fn [b] (d/datoms db :eavt (:db/id b)))
children)
(d/datoms db :eavt page-id)))))
(defn get-recent-updated-pages
[db]
(some->>
(d/datoms db :avet :block/updated-at)
rseq
(keep (fn [datom]
(let [e (d/entity db (:e datom))]
(when (and (common-entity-util/page? e) (not (entity-util/hidden? e)))
e))))
(take 30)))
(defn get-initial-data
"Returns current database schema and initial data.
NOTE: This fn is called by DB and file graphs"
[db]
(let [db-graph? (entity-plus/db-based-graph? db)
_ (when db-graph?
(reset! db-order/*max-key (db-order/get-max-order db)))
schema (:schema db)
idents (mapcat (fn [id]
(when-let [e (d/entity db id)]
(d/datoms db :eavt (:db/id e))))
[:logseq.kv/db-type
:logseq.kv/schema-version
:logseq.kv/graph-uuid
:logseq.kv/latest-code-lang
:logseq.kv/graph-backup-folder
:logseq.property/empty-placeholder])
favorites (when db-graph? (get-favorites db))
views (when db-graph? (get-views-data db))
all-files (get-all-files db)
structured-datoms (when db-graph?
(get-structured-datoms db))
recent-updated-pages (let [pages (get-recent-updated-pages db)]
(mapcat (fn [p] (d/datoms db :eavt (:db/id p))) pages))
pages-datoms (let [contents-id (get-first-page-by-title db "Contents")
views-id (get-first-page-by-title db common-config/views-page-name)]
(mapcat #(d/datoms db :eavt %)
(remove nil? [contents-id views-id])))
data (distinct
(concat idents
structured-datoms
favorites
recent-updated-pages
views
all-files
pages-datoms))]
{:schema schema
:initial-data data}))
(defn restore-initial-data
"Given initial Datascript datoms and schema, returns a datascript connection"
[data schema]
(d/conn-from-datoms data schema))
(defn create-kvs-table!
"Creates a sqlite table for use with datascript.storage if one doesn't exist"
[sqlite-db]

10
deps/db/src/logseq/db/common/sqlite_cli.cljs vendored
View File

@@ -7,7 +7,7 @@
;; FIXME: datascript.core has to come before datascript.storage or else nbb fails
[datascript.core]
[datascript.storage :refer [IStorage]]
[logseq.db.common.sqlite :as sqlite-common-db]
[logseq.db.common.sqlite :as common-sqlite]
[logseq.db.file-based.schema :as file-schema]
[logseq.db.frontend.schema :as db-schema]
[logseq.db.sqlite.util :as sqlite-util]))
@@ -87,13 +87,13 @@
needed sqlite tables if not created and returns a datascript connection that's
connected to the sqlite db"
[graphs-dir db-name]
(let [[_db-sanitized-name db-full-path] (sqlite-common-db/get-db-full-path graphs-dir db-name)
(let [[_db-sanitized-name db-full-path] (common-sqlite/get-db-full-path graphs-dir db-name)
db (new sqlite db-full-path nil)
;; For both desktop and CLI, only file graphs have db-name that indicate their db type
schema (if (sqlite-common-db/local-file-based-graph? db-name)
schema (if (common-sqlite/local-file-based-graph? db-name)
file-schema/schema
db-schema/schema)]
(sqlite-common-db/create-kvs-table! db)
(common-sqlite/create-kvs-table! db)
(let [storage (new-sqlite-storage db)
conn (sqlite-common-db/get-storage-conn storage schema)]
conn (common-sqlite/get-storage-conn storage schema)]
conn)))

12
deps/db/test/logseq/db/common/sqlite_test.cljs → deps/db/test/logseq/db/common/initial_data_test.cljs vendored
View File

@@ -1,11 +1,11 @@
(ns logseq.db.common.sqlite-test
(ns logseq.db.common.initial-data-test
"This ns is the only one to test against file based datascript connections.
These are useful integration tests"
(:require ["fs" :as fs]
["path" :as node-path]
[cljs.test :refer [deftest async use-fixtures is testing]]
[datascript.core :as d]
[logseq.db.common.sqlite :as sqlite-common-db]
[logseq.db.common.initial-data :as common-initial-data]
[logseq.db.sqlite.build :as sqlite-build]
[logseq.db.common.sqlite-cli :as sqlite-cli]
[logseq.db.sqlite.create-graph :as sqlite-create-graph]
@@ -35,8 +35,8 @@
:file/content "{:foo :bar}"}]
_ (d/transact! conn* blocks)
;; Simulate getting data from sqlite and restoring it for frontend
{:keys [schema initial-data]} (sqlite-common-db/get-initial-data @conn*)
conn (sqlite-common-db/restore-initial-data initial-data schema)]
{:keys [schema initial-data]} (common-initial-data/get-initial-data @conn*)
conn (d/conn-from-datoms initial-data schema)]
(is (= blocks
(->> @conn
(d/q '[:find (pull ?b [:block/uuid :file/path :file/content]) :where [?b :file/content]])
@@ -54,7 +54,7 @@
:blocks [{:block/title "b1"}]}]})
_ (d/transact! conn* init-tx)
;; Simulate getting data from sqlite and restoring it for frontend
{:keys [schema initial-data]} (sqlite-common-db/get-initial-data @conn*)
conn (sqlite-common-db/restore-initial-data initial-data schema)]
{:keys [schema initial-data]} (common-initial-data/get-initial-data @conn*)
conn (d/conn-from-datoms initial-data schema)]
(is (some? (db-test/find-page-by-title @conn "page1"))
"Restores recently updated page")))

10
src/electron/electron/db.cljs
View File

@@ -3,7 +3,7 @@
(:require ["electron" :refer [app]]
["fs-extra" :as fs]
["path" :as node-path]
[logseq.db.common.sqlite :as sqlite-common-db]))
[logseq.db.common.sqlite :as common-sqlite]))
(defn get-graphs-dir
[]
@@ -17,19 +17,19 @@
(defn ensure-graph-dir!
[db-name]
(ensure-graphs-dir!)
(let [graph-dir (node-path/join (get-graphs-dir) (sqlite-common-db/sanitize-db-name db-name))]
(let [graph-dir (node-path/join (get-graphs-dir) (common-sqlite/sanitize-db-name db-name))]
(fs/ensureDirSync graph-dir)
graph-dir))
(defn save-db!
[db-name data]
(let [[_db-name db-path] (sqlite-common-db/get-db-full-path (get-graphs-dir) db-name)]
(let [[_db-name db-path] (common-sqlite/get-db-full-path (get-graphs-dir) db-name)]
(fs/writeFileSync db-path data)))
(defn get-db
[db-name]
(let [_ (ensure-graph-dir! db-name)
[_db-name db-path] (sqlite-common-db/get-db-full-path (get-graphs-dir) db-name)]
[_db-name db-path] (common-sqlite/get-db-full-path (get-graphs-dir) db-name)]
(when (fs/existsSync db-path)
(fs/readFileSync db-path))))
@@ -37,7 +37,7 @@
(defn unlink-graph!
[repo]
(let [db-name (sqlite-common-db/sanitize-db-name repo)
(let [db-name (common-sqlite/sanitize-db-name repo)
path (node-path/join (get-graphs-dir) db-name)
unlinked (node-path/join (get-graphs-dir) unlinked-graphs-dir)
new-path (node-path/join unlinked db-name)

6
src/electron/electron/handler.cljs
View File

@@ -31,7 +31,7 @@
[electron.window :as win]
[goog.functions :refer [debounce]]
[logseq.common.graph :as common-graph]
[logseq.db.common.sqlite :as sqlite-common-db]
[logseq.db.common.sqlite :as common-sqlite]
[logseq.db.sqlite.util :as sqlite-util]
[promesa.core :as p]))
@@ -240,7 +240,7 @@
(remove (fn [s] (= s db/unlinked-graphs-dir)))
(map graph-name->path)
(map (fn [s]
(if (string/starts-with? s sqlite-common-db/file-version-prefix)
(if (string/starts-with? s common-sqlite/file-version-prefix)
s
(str sqlite-util/db-version-prefix s)))))))
@@ -294,7 +294,7 @@
(defmethod handle :deleteGraph [_window [_ graph graph-name _db-based?]]
(when graph-name
(db/unlink-graph! graph)
(let [old-transit-path (node-path/join (get-graphs-dir) (str (sqlite-common-db/sanitize-db-name graph) ".transit"))]
(let [old-transit-path (node-path/join (get-graphs-dir) (str (common-sqlite/sanitize-db-name graph) ".transit"))]
(when (fs/existsSync old-transit-path)
(fs/unlinkSync old-transit-path)))))

4
src/main/frontend/db/restore.cljs
View File

@@ -1,12 +1,12 @@
(ns frontend.db.restore
"Fns for DB restore(from text or sqlite)"
(:require [cljs-time.core :as t]
[datascript.core :as d]
[frontend.db.conn :as db-conn]
[frontend.persist-db :as persist-db]
[frontend.state :as state]
[frontend.undo-redo :as undo-redo]
[logseq.db :as ldb]
[logseq.db.common.sqlite :as sqlite-common-db]
[promesa.core :as p]))
(defn restore-graph!
@@ -20,7 +20,7 @@
_ (when (nil? schema)
(throw (ex-info "No valid schema found when reloading db" {:repo repo})))
conn (try
(sqlite-common-db/restore-initial-data initial-data schema)
(d/conn-from-datoms initial-data schema)
(catch :default e
(prn :error :restore-initial-data-failed
(ldb/write-transit-str {:schema schema

4
src/main/frontend/handler/db_based/rtc.cljs
View File

@@ -10,7 +10,7 @@
[frontend.util :as util]
[lambdaisland.glogi :as log]
[logseq.db :as ldb]
[logseq.db.common.sqlite :as sqlite-common-db]
[logseq.db.common.sqlite :as common-sqlite]
[logseq.shui.ui :as shui]
[promesa.core :as p]))
@@ -19,7 +19,7 @@
(p/do!
(js/Promise. user-handler/task--ensure-id&access-token)
(let [token (state/get-auth-id-token)
repo-name (sqlite-common-db/sanitize-db-name repo)]
repo-name (common-sqlite/sanitize-db-name repo)]
(state/<invoke-db-worker :thread-api/rtc-async-upload-graph repo token repo-name))))
(defn <rtc-delete-graph!

6
src/main/frontend/handler/export.cljs
View File

@@ -22,7 +22,7 @@
[goog.dom :as gdom]
[lambdaisland.glogi :as log]
[logseq.db :as ldb]
[logseq.db.common.sqlite :as sqlite-common-db]
[logseq.db.common.sqlite :as common-sqlite]
[logseq.publishing.html :as publish-html]
[promesa.core :as p])
(:import
@@ -58,7 +58,7 @@
[repo]
(p/let [db-data (persist-db/<export-db repo {:return-data? true})
filename "db.sqlite"
repo-name (sqlite-common-db/sanitize-db-name repo)
repo-name (common-sqlite/sanitize-db-name repo)
assets (assets-handler/<get-all-assets)
files (cons [filename db-data] assets)
zipfile (zip/make-zip repo-name files repo)]
@@ -255,7 +255,7 @@
(when (and repo (= repo (state/get-current-repo)))
(when-let [backup-folder (ldb/get-key-value (db/get-db repo) :logseq.kv/graph-backup-folder)]
(p/let [handle (idb/get-item (str "handle/" (js/btoa repo) "/" backup-folder))
repo-name (sqlite-common-db/sanitize-db-name repo)]
repo-name (common-sqlite/sanitize-db-name repo)]
(if handle
(->
(p/let [graph-dir-handle (.getDirectoryHandle handle repo-name #js {:create true})

17
src/main/frontend/worker/db_worker.cljs
View File

@@ -35,8 +35,9 @@
[logseq.common.util :as common-util]
[logseq.db :as ldb]
[logseq.db.common.entity-plus :as entity-plus]
[logseq.db.common.initial-data :as common-initial-data]
[logseq.db.common.order :as db-order]
[logseq.db.common.sqlite :as sqlite-common-db]
[logseq.db.common.sqlite :as common-sqlite]
[logseq.db.common.view :as db-view]
[logseq.db.frontend.schema :as db-schema]
[logseq.db.sqlite.create-graph :as sqlite-create-graph]
@@ -329,19 +330,19 @@
:client-ops client-ops-db})
(doseq [db' dbs]
(enable-sqlite-wal-mode! db'))
(sqlite-common-db/create-kvs-table! db)
(when-not @*publishing? (sqlite-common-db/create-kvs-table! client-ops-db))
(common-sqlite/create-kvs-table! db)
(when-not @*publishing? (common-sqlite/create-kvs-table! client-ops-db))
(db-migrate/migrate-sqlite-db db)
(when-not @*publishing? (db-migrate/migrate-sqlite-db client-ops-db))
(search/create-tables-and-triggers! search-db)
(let [schema (ldb/get-schema repo)
conn (sqlite-common-db/get-storage-conn storage schema)
conn (common-sqlite/get-storage-conn storage schema)
_ (db-fix/check-and-fix-schema! repo conn)
_ (when datoms
(let [data (map (fn [datom]
[:db/add (:e datom) (:a datom) (:v datom)]) datoms)]
(d/transact! conn data {:initial-db? true})))
client-ops-conn (when-not @*publishing? (sqlite-common-db/get-storage-conn
client-ops-conn (when-not @*publishing? (common-sqlite/get-storage-conn
client-ops-storage
client-op/schema-in-db))
initial-data-exists? (when (nil? datoms)
@@ -492,7 +493,7 @@
id)]
(some->> eid
(d/pull @conn selector)
(sqlite-common-db/with-parent @conn)))))
(common-initial-data/with-parent @conn)))))
(def ^:private *get-blocks-cache (volatile! (cache/lru-cache-factory {} :threshold 1000)))
(def ^:private get-blocks-with-cache
@@ -506,7 +507,7 @@
(when db
(mapv (fn [{:keys [id opts]}]
(let [id' (if (and (string? id) (common-util/uuid-string? id)) (uuid id) id)]
(-> (sqlite-common-db/get-block-and-children db id' opts)
(-> (common-initial-data/get-block-and-children db id' opts)
(assoc :id id)))) requests)))))
(def-thread-api :thread-api/get-blocks
@@ -604,7 +605,7 @@
(def-thread-api :thread-api/get-initial-data
[repo]
(when-let [conn (worker-state/get-datascript-conn repo)]
(sqlite-common-db/get-initial-data @conn)))
(common-initial-data/get-initial-data @conn)))
(def-thread-api :thread-api/reset-db
[repo db-transit]

4
src/main/frontend/worker/pipeline.cljs
View File

@@ -12,7 +12,7 @@
[logseq.common.util :as common-util]
[logseq.common.uuid :as common-uuid]
[logseq.db :as ldb]
[logseq.db.common.sqlite :as sqlite-common-db]
[logseq.db.common.sqlite :as common-sqlite]
[logseq.db.frontend.validate :as db-validate]
[logseq.db.sqlite.export :as sqlite-export]
[logseq.db.sqlite.util :as sqlite-util]
@@ -233,7 +233,7 @@
:db-after (:db-after result)))
tx-report)
{:keys [pages blocks]} (ds-report/get-blocks-and-pages tx-report*)
_ (when (sqlite-common-db/local-file-based-graph? repo)
_ (when (common-sqlite/local-file-based-graph? repo)
(let [page-ids (distinct (map :db/id pages))]
(doseq [page-id page-ids]
(when (d/entity @conn page-id)

4
src/main/frontend/worker/util.cljc
View File

@@ -6,7 +6,7 @@
[goog.crypt :as crypt]
[goog.crypt.Hmac]
[goog.crypt.Sha256]
[logseq.db.common.sqlite :as sqlite-common-db]
[logseq.db.common.sqlite :as common-sqlite]
[frontend.common.file.util :as wfu])))
;; Copied from https://github.com/tonsky/datascript-todo
@@ -27,7 +27,7 @@
(defn get-pool-name
[graph-name]
(str "logseq-pool-" (sqlite-common-db/sanitize-db-name graph-name)))
(str "logseq-pool-" (common-sqlite/sanitize-db-name graph-name)))
(defn- decode-username
[username]