clone/logseq
1
0
Fork 0
mirror of https://github.com/logseq/logseq.git synced 2026-04-24 22:25:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feat/db' into feat/capacitor-new

Browse Source
This commit is contained in:
charlie
2025-05-16 15:40:05 +08:00
parent bf040e07db 120730a8dc
commit 5cbd2a4ae5
26 changed files with 459 additions and 401 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
deps/common/nbb.edn vendored
View File

@@ -1,4 +1,5 @@
{:paths ["src" "resources"] {:paths ["src" "resources"]
:deps :deps
{io.github.nextjournal/nbb-test-runner ;; TODO: Remove fork when https://github.com/nextjournal/nbb-test-runner/pull/2 is merged
{:git/sha "60ed57aa04bca8d604f5ba6b28848bd887109347"}}} {io.github.colin-p-hill/nbb-test-runner
{:git/sha "def2cbdb5b3a0e1612b28bf64f5d869c27c733d3"}}}

195
deps/common/resources/templates/config.edn vendored
View File

@@ -1,5 +1,7 @@
{:meta/version 1 {:meta/version 1
;; == FILE GRAPH CONFIG ==
;;
;; Set the preferred format. ;; Set the preferred format.
;; This is _only_ for file graphs. ;; This is _only_ for file graphs.
;; Available options: ;; Available options:
@@ -42,6 +44,102 @@
;; Default value: "yyyy_MM_dd" ;; Default value: "yyyy_MM_dd"
;; :journal/file-name-format "yyyy_MM_dd" ;; :journal/file-name-format "yyyy_MM_dd"
;; Set the default location for storing notes.
;; This is _only_ for file graphs.
;; Default value: "pages"
;; :pages-directory "pages"
;; Set the default location for storing journals.
;; This is _only_ for file graphs.
;; Default value: "journals"
;; :journals-directory "journals"
;; Set the default location for storing whiteboards.
;; This is _only_ for file graphs.
;; Default value: "whiteboards"
;; :whiteboards-directory "whiteboards"
;; Enabling this option converts
;; [[Grant Ideas]] to [[file:./grant_ideas.org][Grant Ideas]] for org-mode.
;; For more information, visit https://github.com/logseq/logseq/issues/672
;; This is _only_ for file graphs.
;; :org-mode/insert-file-link? false
;; Favorites to list on the left sidebar
;; This is _only_ for file graphs.
:favorites []
;; Set flashcards interval.
;; This is _only_ for file graphs.
;; Expected value:
;; - Float between 0 and 1
;; higher values result in faster changes to the next review interval.
;; Default value: 0.5
;; :srs/learning-fraction 0.5
;; Set the initial interval after the first successful review of a card.
;; This is _only_ for file graphs.
;; Default value: 4
;; :srs/initial-interval 4
;; Hide specific block properties.
;; This is _only_ for file graphs.
;; Example usage:
;; :block-hidden-properties #{:public :icon}
;; Create a page for all properties.
;; This is _only_ for file graphs.
;; Default value: true
:property-pages/enabled? true
;; Properties to exclude from having property pages
;; This is _only_ for file graphs.
;; Example usage:
;; :property-pages/excludelist #{:duration :author}
;; By default, property value separated by commas will not be treated as
;; page references. You can add properties to enable it.
;; This is _only_ for file graphs.
;; Example usage:
;; :property/separated-by-commas #{:alias :tags}
;; Properties that are ignored when parsing property values for references
;; This is _only_ for file graphs.
;; Example usage:
;; :ignored-page-references-keywords #{:author :website}
;; logbook configuration.
;; This is _only_ for file graphs.
;; :logbook/settings
;; {:with-second-support? false ;limit logbook to minutes, seconds will be eliminated
;; :enabled-in-all-blocks true ;display logbook in all blocks after timetracking
;; :enabled-in-timestamped-blocks false ;don't display logbook at all
;; }
;; File sync options
;; Ignore these files when syncing, regexp is supported.
;; This is _only_ for file graphs.
;; :file-sync/ignore-files []
;; Configure the escaping method for special characters in page titles.
;; This is _only_ for file graphs.
;; Warning:
;; This is a dangerous operation. To modify the setting,
;; you'll need to manually rename all affected files and
;; re-index them on all clients after synchronization.
;; Incorrect handling may result in messy page titles.
;; Available options:
;; - :triple-lowbar (default)
;; ;use triple underscore `___` for slash `/` in page title
;; ;use Percent-encoding for other invalid characters
:file/name-format :triple-lowbar
;; == END OF FILE GRAPH CONFIG ==
;; Hide empty block properties
;; This is _only_ for DB graphs.
;; Default value: false
;; :ui/hide-empty-properties? false
;; Enable tooltip preview on hover. ;; Enable tooltip preview on hover.
;; Default value: true ;; Default value: true
:ui/enable-tooltip? true :ui/enable-tooltip? true
@@ -58,11 +156,6 @@
;; Default value: true ;; Default value: true
:ui/auto-expand-block-refs? true :ui/auto-expand-block-refs? true
;; Hide empty block properties
;; This is _only_ for DB graphs.
;; Default value: false
;; :ui/hide-empty-properties? false
;; Disable accent marks when searching. ;; Disable accent marks when searching.
;; After changing this setting, rebuild the search index by pressing (^C ^S). ;; After changing this setting, rebuild the search index by pressing (^C ^S).
;; Default value: true ;; Default value: true
@@ -137,27 +230,6 @@
;; 3. Set "home" as the home page and display multiple pages in the right sidebar: ;; 3. Set "home" as the home page and display multiple pages in the right sidebar:
;; :default-home {:page "home", :sidebar ["Page A" "Page B"]} ;; :default-home {:page "home", :sidebar ["Page A" "Page B"]}
;; Set the default location for storing notes.
;; This is _only_ for file graphs.
;; Default value: "pages"
;; :pages-directory "pages"
;; Set the default location for storing journals.
;; This is _only_ for file graphs.
;; Default value: "journals"
;; :journals-directory "journals"
;; Set the default location for storing whiteboards.
;; This is _only_ for file graphs.
;; Default value: "whiteboards"
;; :whiteboards-directory "whiteboards"
;; Enabling this option converts
;; [[Grant Ideas]] to [[file:./grant_ideas.org][Grant Ideas]] for org-mode.
;; For more information, visit https://github.com/logseq/logseq/issues/672
;; This is _only_ for file graphs.
;; :org-mode/insert-file-link? false
;; Configure custom shortcuts. ;; Configure custom shortcuts.
;; Syntax: ;; Syntax:
;; 1. + indicates simultaneous key presses, e.g., `Ctrl+Shift+a`. ;; 1. + indicates simultaneous key presses, e.g., `Ctrl+Shift+a`.
@@ -274,58 +346,6 @@
;; :charge-strength -600 ; Default value: -600 ;; :charge-strength -600 ; Default value: -600
;; :charge-range 600} ; Default value: 600 ;; :charge-range 600} ; Default value: 600
;; Favorites to list on the left sidebar
;; This is _only_ for file graphs.
:favorites []
;; Set flashcards interval.
;; This is _only_ for file graphs.
;; Expected value:
;; - Float between 0 and 1
;; higher values result in faster changes to the next review interval.
;; Default value: 0.5
;; :srs/learning-fraction 0.5
;; Set the initial interval after the first successful review of a card.
;; This is _only_ for file graphs.
;; Default value: 4
;; :srs/initial-interval 4
;; Hide specific block properties.
;; This is _only_ for file graphs.
;; Example usage:
;; :block-hidden-properties #{:public :icon}
;; Create a page for all properties.
;; This is _only_ for file graphs.
;; Default value: true
:property-pages/enabled? true
;; Properties to exclude from having property pages
;; This is _only_ for file graphs.
;; Example usage:
;; :property-pages/excludelist #{:duration :author}
;; By default, property value separated by commas will not be treated as
;; page references. You can add properties to enable it.
;; This is _only_ for file graphs.
;; Example usage:
;; :property/separated-by-commas #{:alias :tags}
;; Properties that are ignored when parsing property values for references
;; This is _only_ for file graphs.
;; Example usage:
;; :ignored-page-references-keywords #{:author :website}
;; logbook configuration.
;; This is _only_ for file graphs.
;; :logbook/settings
;; {:with-second-support? false ;limit logbook to minutes, seconds will be eliminated
;; :enabled-in-all-blocks true ;display logbook in all blocks after timetracking
;; :enabled-in-timestamped-blocks false ;don't display logbook at all
;; }
;; Mobile photo upload configuration. ;; Mobile photo upload configuration.
;; :mobile/photo ;; :mobile/photo
;; {:allow-editing? true ;; {:allow-editing? true
@@ -375,11 +395,6 @@
;; :redirect-page? false ;; Default value: false ;; :redirect-page? false ;; Default value: false
;; :default-page "quick capture"} ;; Default page: "quick capture" ;; :default-page "quick capture"} ;; Default page: "quick capture"
;; File sync options
;; Ignore these files when syncing, regexp is supported.
;; This is _only_ for file graphs.
;; :file-sync/ignore-files []
;; Configure the Enter key behavior for ;; Configure the Enter key behavior for
;; context-aware editing with DWIM (Do What I Mean). ;; context-aware editing with DWIM (Do What I Mean).
;; context-aware Enter key behavior implies that pressing Enter will ;; context-aware Enter key behavior implies that pressing Enter will
@@ -393,16 +408,4 @@
;; :page-ref? true ;; Default value: true ;; :page-ref? true ;; Default value: true
;; :properties? true ;; Default value: true ;; :properties? true ;; Default value: true
;; :list? false} ;; Default value: false ;; :list? false} ;; Default value: false
}
;; Configure the escaping method for special characters in page titles.
;; This is _only_ for file graphs.
;; Warning:
;; This is a dangerous operation. To modify the setting,
;; you'll need to manually rename all affected files and
;; re-index them on all clients after synchronization.
;; Incorrect handling may result in messy page titles.
;; Available options:
;; - :triple-lowbar (default)
;; ;use triple underscore `___` for slash `/` in page title
;; ;use Percent-encoding for other invalid characters
:file/name-format :triple-lowbar}

42
deps/common/src/logseq/common/config.cljs vendored
View File

@@ -1,5 +1,5 @@
(ns logseq.common.config (ns logseq.common.config
"Common config and constants that are shared between deps and app" "Common config constants and fns that are shared between deps and app"
(:require [clojure.string :as string] (:require [clojure.string :as string]
[goog.object :as gobj])) [goog.object :as gobj]))
@@ -115,3 +115,43 @@
"*" "*"
"-"))) "-")))
(defn create-config-for-db-graph
"Given a new config.edn file string, creates a config.edn for use with only DB graphs"
[config]
(string/replace config #"(?m)[\s]*;; == FILE GRAPH CONFIG ==(?:.|\n)*?;; == END OF FILE GRAPH CONFIG ==\n?" ""))
(def file-only-config
"File only config keys that are deprecated in DB graphs along with
descriptions for their deprecation."
(merge
(zipmap
[:file/name-format
:file-sync/ignore-files
:hidden
:ignored-page-references-keywords
:journal/file-name-format
:journal/page-title-format
:journals-directory
:logbook/settings
:org-mode/insert-file-link?
:pages-directory
:preferred-workflow
:property/separated-by-commas
:property-pages/excludelist
:srs/learning-fraction
:srs/initial-interval
:whiteboards-directory]
(repeat "is not used in DB graphs"))
{:preferred-format
"is not used in DB graphs as there is only markdown mode."
:property-pages/enabled?
"is not used in DB graphs as all properties have pages"
:block-hidden-properties
"is not used in DB graphs as hiding a property is done in its configuration"
:feature/enable-block-timestamps?
"is not used in DB graphs as it is always enabled"
:favorites
"is not stored in config for DB graphs"
:default-templates
"is replaced by #Template and the `Apply template to tags` property"}))

45
deps/common/test/logseq/common/config_test.cljc vendored Normal file
View File

@@ -0,0 +1,45 @@
(ns logseq.common.config-test
(:require [cljs.test :refer [deftest is]]
[clojure.string :as string]
[logseq.common.config :as common-config]
#?(:org.babashka/nbb [nbb.classpath :as cp])
["fs" :as fs]
["path" :as node-path]))
(deftest remove-hidden-files
(let [files ["pages/foo.md" "pages/bar.md"
"script/README.md" "script/config.edn"
"dev/README.md" "dev/config.edn"]]
(is (= ["pages/foo.md" "pages/bar.md"]
#_:clj-kondo/ignore ;; buggy unresolved var
(common-config/remove-hidden-files
files
{:hidden ["script" "/dev"]}
identity))
"Removes hidden relative files")
(is (= ["/pages/foo.md" "/pages/bar.md"]
(common-config/remove-hidden-files
(map #(str "/" %) files)
{:hidden ["script" "/dev"]}
identity))
"Removes hidden files if they start with '/'")))
(defn find-on-classpath [classpath rel-path]
(some (fn [dir]
(let [f (node-path/join dir rel-path)]
(when (fs/existsSync f) f)))
(string/split classpath #":")))
#?(:org.babashka/nbb
(deftest create-config-for-db-graph
(let [original-config (some-> (find-on-classpath (cp/get-classpath) "templates/config.edn") fs/readFileSync str)
_ (assert original-config "config.edn must not be blank")
migrated-config (common-config/create-config-for-db-graph original-config)
forbidden-kws-regex (re-pattern (str (string/join "|" (keys common-config/file-only-config))))]
;; (println migrated-config)
(is (not (string/includes? migrated-config "== FILE ONLY CONFIG"))
"No longer includes file config header")
(assert (re-find forbidden-kws-regex original-config) "File config keys present in original config")
(is (not (re-find forbidden-kws-regex migrated-config))
"File config keys no longer present in migrated config"))))

21
deps/common/test/logseq/common/config_test.cljs vendored
View File

@@ -1,21 +0,0 @@
(ns logseq.common.config-test
(:require [logseq.common.config :as common-config]
[cljs.test :refer [deftest is]]))
(deftest remove-hidden-files
(let [files ["pages/foo.md" "pages/bar.md"
"script/README.md" "script/config.edn"
"dev/README.md" "dev/config.edn"]]
(is (= ["pages/foo.md" "pages/bar.md"]
(common-config/remove-hidden-files
files
{:hidden ["script" "/dev"]}
identity))
"Removes hidden relative files")
(is (= ["/pages/foo.md" "/pages/bar.md"]
(common-config/remove-hidden-files
(map #(str "/" %) files)
{:hidden ["script" "/dev"]}
identity))
"Removes hidden files if they start with '/'")))

1
deps/db/deps.edn vendored
View File

@@ -12,6 +12,7 @@
logseq/common {:local/root "../common"} logseq/common {:local/root "../common"}
logseq/clj-fractional-indexing {:git/url "https://github.com/logseq/clj-fractional-indexing" logseq/clj-fractional-indexing {:git/url "https://github.com/logseq/clj-fractional-indexing"
:sha "7182b7878410f78536dc2b6df35ed32ef9cd6b61"} :sha "7182b7878410f78536dc2b6df35ed32ef9cd6b61"}
borkdude/rewrite-edn {:mvn/version "0.4.9"}
metosin/malli {:mvn/version "0.16.1"} metosin/malli {:mvn/version "0.16.1"}
medley/medley {:mvn/version "1.4.0"}} medley/medley {:mvn/version "1.4.0"}}

2
deps/db/nbb.edn vendored
View File

@@ -4,6 +4,8 @@
{:local/root "../common"} {:local/root "../common"}
medley/medley {:mvn/version "1.4.0"} medley/medley {:mvn/version "1.4.0"}
metosin/malli {:mvn/version "0.16.1"} metosin/malli {:mvn/version "0.16.1"}
;; Used by db scripts with outliner.cli
borkdude/rewrite-edn {:mvn/version "0.4.9"}
logseq/clj-fractional-indexing {:git/url "https://github.com/logseq/clj-fractional-indexing" logseq/clj-fractional-indexing {:git/url "https://github.com/logseq/clj-fractional-indexing"
:sha "7182b7878410f78536dc2b6df35ed32ef9cd6b61"} :sha "7182b7878410f78536dc2b6df35ed32ef9cd6b61"}
io.github.nextjournal/nbb-test-runner io.github.nextjournal/nbb-test-runner

8
deps/db/src/logseq/db/frontend/malli_schema.cljs vendored
View File

@@ -3,9 +3,9 @@
(:require [clojure.set :as set] (:require [clojure.set :as set]
[clojure.string :as string] [clojure.string :as string]
[datascript.core :as d] [datascript.core :as d]
[logseq.db.common.entity-plus :as entity-plus]
[logseq.db.common.order :as db-order] [logseq.db.common.order :as db-order]
[logseq.db.frontend.class :as db-class] [logseq.db.frontend.class :as db-class]
[logseq.db.common.entity-plus :as entity-plus]
[logseq.db.frontend.entity-util :as entity-util] [logseq.db.frontend.entity-util :as entity-util]
[logseq.db.frontend.property :as db-property] [logseq.db.frontend.property :as db-property]
[logseq.db.frontend.property.type :as db-property-type] [logseq.db.frontend.property.type :as db-property-type]
@@ -482,14 +482,10 @@
:file-block :file-block
(:logseq.property.history/block d) (:logseq.property.history/block d)
:property-history-block :property-history-block
(:block/closed-value-property d) (:block/closed-value-property d)
:closed-value-block :closed-value-block
(and (:logseq.property/created-from-property d) (:logseq.property/value d))
(and (:logseq.property/created-from-property d)
(:logseq.property/value d))
:property-value-block :property-value-block
(:block/uuid d) (:block/uuid d)
:block :block
(= (:db/ident d) :logseq.property/empty-placeholder) (= (:db/ident d) :logseq.property/empty-placeholder)

75
deps/db/src/logseq/db/sqlite/export.cljs vendored
View File

@@ -803,21 +803,29 @@
set)] set)]
(set/difference ref-uuids known-uuids))) (set/difference ref-uuids known-uuids)))
(defn- remove-namespaced-keys
"Removes keys from this ns for maps passed sqlite.build fns as they don't need to validate or use them"
[m]
(->> m
(remove (fn [[k _v]] (= "logseq.db.sqlite.export" (namespace k))))
(into {})))
(defn- ensure-export-is-valid (defn- ensure-export-is-valid
"Checks that export map is usable by sqlite.build including checking that "Checks that export map is usable by sqlite.build including checking that
all referenced properties and classes are defined. Checks related to properties and all referenced properties and classes are defined. Checks related to properties and
classes are disabled when :exclude-namespaces is set because those checks can't be done" classes are disabled when :exclude-namespaces is set because those checks can't be done"
[export-map {:keys [graph-options]}] [export-map* {:keys [graph-options]}]
(when-not (seq (:exclude-namespaces graph-options)) (sqlite-build/validate-options export-map)) (let [export-map (remove-namespaced-keys export-map*)]
(let [undefined-uuids (find-undefined-uuids export-map) (when-not (seq (:exclude-namespaces graph-options)) (sqlite-build/validate-options export-map))
undefined (cond-> {} (let [undefined-uuids (find-undefined-uuids export-map)
(empty? (:exclude-namespaces graph-options)) undefined (cond-> {}
(merge (find-undefined-classes-and-properties export-map)) (empty? (:exclude-namespaces graph-options))
(seq undefined-uuids) (merge (find-undefined-classes-and-properties export-map))
(assoc :uuids undefined-uuids))] (seq undefined-uuids)
(when (seq undefined) (assoc :uuids undefined-uuids))]
(throw (ex-info (str "The following classes, uuids and properties are not defined: " (pr-str undefined)) (when (seq undefined)
undefined))))) (throw (ex-info (str "The following classes, uuids and properties are not defined: " (pr-str undefined))
undefined))))))
(defn build-export (defn build-export
"Handles exporting db by given export-type" "Handles exporting db by given export-type"
@@ -838,30 +846,33 @@
(build-graph-export db (:graph-options options)))] (build-graph-export db (:graph-options options)))]
(if (get-in options [:graph-options :catch-validation-errors?]) (if (get-in options [:graph-options :catch-validation-errors?])
(try (try
(ensure-export-is-valid (dissoc export-map ::block ::graph-files ::kv-values ::schema-version) options) (ensure-export-is-valid export-map options)
(catch ExceptionInfo e (catch ExceptionInfo e
(println "Caught error:" e))) (println "Caught error:" e)))
(ensure-export-is-valid (dissoc export-map ::block ::graph-files ::kv-values ::schema-version) options)) (ensure-export-is-valid export-map options))
(assoc export-map ::export-type export-type))) (assoc export-map ::export-type export-type)))
;; Import fns ;; Import fns
;; ========== ;; ==========
(defn- add-uuid-to-page-if-exists (defn- add-uuid-to-page-if-exists
[db import-to-existing-page-uuids m] [db import-to-existing-page-uuids {:keys [existing-pages-keep-properties?]} m]
(if-let [ent (some->> (:build/journal m) (if-let [ent (if (:build/journal m)
(d/datoms db :avet :block/journal-day) (some->> (:build/journal m)
first (d/datoms db :avet :block/journal-day)
:e first
(d/entity db))] :e
(d/entity db))
;; TODO: For now only check page uniqueness by title. Could handle more uniqueness checks later
(some->> (:block/title m) (ldb/get-case-page db)))]
(do (do
(swap! import-to-existing-page-uuids assoc (:block/uuid m) (:block/uuid ent)) (swap! import-to-existing-page-uuids assoc (:block/uuid m) (:block/uuid ent))
(assoc m :block/uuid (:block/uuid ent))) (cond-> (assoc m :block/uuid (:block/uuid ent))
;; TODO: For now only check page uniqueness by title. Could handle more uniqueness checks later (and (:build/properties m) existing-pages-keep-properties?)
(if-let [ent (some->> (:block/title m) (ldb/get-case-page db))] (update :build/properties (fn [props]
(do (->> props
(swap! import-to-existing-page-uuids assoc (:block/uuid m) (:block/uuid ent)) (remove (fn [[k _v]] (get ent k)))
(assoc m :block/uuid (:block/uuid ent))) (into {}))))))
m))) m))
(defn- update-existing-properties (defn- update-existing-properties
"Updates existing properties by ident. Also check imported and existing properties have "Updates existing properties by ident. Also check imported and existing properties have
@@ -884,7 +895,7 @@
(defn- check-for-existing-entities (defn- check-for-existing-entities
"Checks export map for existing entities and adds :block/uuid to them if they exist in graph to import. "Checks export map for existing entities and adds :block/uuid to them if they exist in graph to import.
Also checks for property conflicts between existing properties and properties to be imported" Also checks for property conflicts between existing properties and properties to be imported"
[db {:keys [pages-and-blocks classes properties] ::keys [export-type] :as export-map} property-conflicts] [db {:keys [pages-and-blocks classes properties] ::keys [export-type import-options] :as export-map} property-conflicts]
(let [import-to-existing-page-uuids (atom {}) (let [import-to-existing-page-uuids (atom {})
export-map export-map
(cond-> {:build-existing-tx? true (cond-> {:build-existing-tx? true
@@ -892,7 +903,7 @@
(seq pages-and-blocks) (seq pages-and-blocks)
(assoc :pages-and-blocks (assoc :pages-and-blocks
(mapv (fn [m] (mapv (fn [m]
(update m :page (partial add-uuid-to-page-if-exists db import-to-existing-page-uuids))) (update m :page (partial add-uuid-to-page-if-exists db import-to-existing-page-uuids import-options)))
pages-and-blocks)) pages-and-blocks))
(seq classes) (seq classes)
(assoc :classes (assoc :classes
@@ -915,7 +926,7 @@
(walk/postwalk (fn [f] (walk/postwalk (fn [f]
(if (and (vector? f) (= :build/page (first f))) (if (and (vector? f) (= :build/page (first f)))
[:build/page [:build/page
(add-uuid-to-page-if-exists db import-to-existing-page-uuids (second f))] (add-uuid-to-page-if-exists db import-to-existing-page-uuids import-options (second f))]
f)) f))
export-map)) export-map))
;; Update uuid references of all pages that had their uuids updated to reference an existing page ;; Update uuid references of all pages that had their uuids updated to reference an existing page
@@ -948,6 +959,8 @@
* ::kv-values - Vec of :kv/value maps for a :graph export * ::kv-values - Vec of :kv/value maps for a :graph export
* ::auto-include-namespaces - A set of parent namespaces to include from properties and classes * ::auto-include-namespaces - A set of parent namespaces to include from properties and classes
for a :graph export. See :exclude-namespaces in build-graph-export for a similar option for a :graph export. See :exclude-namespaces in build-graph-export for a similar option
* ::import-options - A map of options that alters importing behavior. Has the following keys:
* :existing-pages-keep-properties? - Boolean which disables upsert of :build/properties on
This fn then returns a map of txs to transact with the following keys: This fn then returns a map of txs to transact with the following keys:
* :init-tx - Txs that must be transacted first, usually because they define new properties * :init-tx - Txs that must be transacted first, usually because they define new properties
@@ -969,7 +982,7 @@
{:error (str "The following imported properties conflict with the current graph: " {:error (str "The following imported properties conflict with the current graph: "
(pr-str (mapv :property-id @property-conflicts)))}) (pr-str (mapv :property-id @property-conflicts)))})
(if (= :graph (::export-type export-map'')) (if (= :graph (::export-type export-map''))
(-> (sqlite-build/build-blocks-tx (dissoc export-map'' ::graph-files ::kv-values ::export-type ::schema-version)) (-> (sqlite-build/build-blocks-tx (remove-namespaced-keys export-map''))
(assoc :misc-tx (vec (concat (::graph-files export-map'') (assoc :misc-tx (vec (concat (::graph-files export-map'')
(::kv-values export-map''))))) (::kv-values export-map'')))))
(sqlite-build/build-blocks-tx export-map''))))) (sqlite-build/build-blocks-tx (remove-namespaced-keys export-map''))))))

67
deps/db/test/logseq/db/sqlite/export_test.cljs vendored
View File

@@ -826,54 +826,65 @@
(-> (:classes imported-graph) (-> (:classes imported-graph)
(medley/dissoc-in [:user.property/p1 :build/properties])))))) (medley/dissoc-in [:user.property/p1 :build/properties]))))))
(deftest build-import-can-import-existing-page-with-different-uuid (defn- test-import-existing-page [import-options expected-page-properties]
(let [original-data (let [original-data
{:properties {:user.property/node {:logseq.property/type :node {:properties {:user.property/node {:logseq.property/type :node
:db/cardinality :db.cardinality/many}} :db/cardinality :db.cardinality/many}}
:pages-and-blocks :pages-and-blocks
[{:page {:block/title "page1" [{:page {:block/title "page1"
:build/properties {:user.property/node #{[:build/page {:block/title "node1"}]}}}}]} :build/properties {:user.property/node
#{[:build/page {:block/title "existing page"
:build/properties {:logseq.property/description "first description"}}]}}}}]}
conn (db-test/create-conn-with-blocks original-data) conn (db-test/create-conn-with-blocks original-data)
page-uuid (:block/uuid (db-test/find-page-by-title @conn "node1")) page-uuid (:block/uuid (db-test/find-page-by-title @conn "existing page"))
_ (validate-db @conn) _ (validate-db @conn)
;; This is just a temp uuid used to link to the page during import ;; This is just a temp uuid used to link to the page during import
temp-uuid (random-uuid) temp-uuid (random-uuid)
existing-data import-data
{:properties {:user.property/node {:logseq.property/type :node {:properties {:user.property/node {:logseq.property/type :node
:db/cardinality :db.cardinality/many}} :db/cardinality :db.cardinality/many}}
:pages-and-blocks :pages-and-blocks
[{:page {:block/title "node1" [{:page {:block/title "existing page"
:block/uuid temp-uuid :block/uuid temp-uuid
:build/keep-uuid? true}} :build/keep-uuid? true
:build/properties {:logseq.property/description "second description"
:logseq.property/exclude-from-graph-view true}}}
{:page {:block/title "page2" {:page {:block/title "page2"
:build/properties {:user.property/node #{[:block/uuid temp-uuid]}}}}]} :build/properties {:user.property/node #{[:block/uuid temp-uuid]}}}}]
::sqlite-export/import-options import-options}
{:keys [init-tx block-props-tx] :as _txs} {:keys [init-tx block-props-tx] :as _txs}
(sqlite-export/build-import existing-data @conn {}) (sqlite-export/build-import import-data @conn {})
;; _ (cljs.pprint/pprint _txs) ;; _ (cljs.pprint/pprint _txs)
_ (d/transact! conn init-tx) _ (d/transact! conn init-tx)
_ (d/transact! conn block-props-tx) _ (d/transact! conn block-props-tx)
_ (validate-db @conn) _ (validate-db @conn)
expected-pages-and-blocks expected-pages-and-blocks
[{:page [{:block/uuid page-uuid
{:block/uuid page-uuid :build/keep-uuid? true,
:build/keep-uuid? true, :block/title "existing page"
:block/title "node1"}, :build/properties
:blocks []} expected-page-properties}
{:page {:build/properties
{:build/properties {:user.property/node
{:user.property/node #{[:block/uuid page-uuid]}},
#{[:block/uuid page-uuid]}}, :block/title "page1"}
:block/title "page1"}, {:build/properties
:blocks []} {:user.property/node
{:page #{[:block/uuid page-uuid]}},
{:build/properties :block/title "page2"}]
{:user.property/node
#{[:block/uuid page-uuid]}},
:block/title "page2"},
:blocks []}],
exported-graph (sqlite-export/build-export @conn {:export-type :graph exported-graph (sqlite-export/build-export @conn {:export-type :graph
:graph-options {:exclude-built-in-pages? true}})] :graph-options {:exclude-built-in-pages? true}})]
(is (= expected-pages-and-blocks (is (= expected-pages-and-blocks
(:pages-and-blocks exported-graph)) (map :page (:pages-and-blocks exported-graph)))
"page uuid ('node1') is preserved across imports even when its assigned a temporary "page uuid of 'existing page' is preserved across imports even when its assigned a temporary
uuid to relate it to other nodes"))) uuid to relate it to other nodes")))
(deftest build-import-can-import-existing-page-with-different-uuid
(testing "By default any properties passed to an existing page are upserted"
(test-import-existing-page {}
{:logseq.property/description "second description"
:logseq.property/exclude-from-graph-view true}))
(testing "With ::existing-pages-keep-properties?, existing properties on existing pages are not overwritten by imported data"
(test-import-existing-page {:existing-pages-keep-properties? true}
{:logseq.property/description "first description"
:logseq.property/exclude-from-graph-view true})))

7
deps/graph-parser/src/logseq/graph_parser/exporter.cljs vendored
View File

@@ -764,6 +764,7 @@
{:block block'' :properties-tx properties-tx})) {:block block'' :properties-tx properties-tx}))
(defn- pretty-print-dissoc (defn- pretty-print-dissoc
"Remove list of keys from a given map string while preserving whitespace"
[s dissoc-keys] [s dissoc-keys]
(-> (reduce rewrite/dissoc (-> (reduce rewrite/dissoc
(rewrite/parse-string s) (rewrite/parse-string s)
@@ -1496,7 +1497,11 @@
<save-file default-save-file}}] <save-file default-save-file}}]
(-> (<read-file config-file) (-> (<read-file config-file)
(p/then #(p/do! (p/then #(p/do!
(<save-file repo-or-conn "logseq/config.edn" %) (<save-file repo-or-conn
"logseq/config.edn"
;; Converts a file graph config.edn for use with DB graphs. Unlike common-config/create-config-for-db-graph,
;; manually dissoc deprecated keys for config to be valid
(pretty-print-dissoc % (keys common-config/file-only-config)))
(let [config (edn/read-string %)] (let [config (edn/read-string %)]
(when-let [title-format (or (:journal/page-title-format config) (:date-formatter config))] (when-let [title-format (or (:journal/page-title-format config) (:date-formatter config))]
(ldb/transact! repo-or-conn [{:db/ident :logseq.class/Journal (ldb/transact! repo-or-conn [{:db/ident :logseq.class/Journal

2
deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs vendored
View File

@@ -693,7 +693,7 @@
(deftest-async export-config-file-sets-title-format (deftest-async export-config-file-sets-title-format
(p/let [conn (db-test/create-conn) (p/let [conn (db-test/create-conn)
read-file #(p/do! (pr-str {:journal/page-title-format "yyyy-MM-dd"})) read-file #(p/do! "{:journal/page-title-format \"yyyy-MM-dd\"}")
_ (gp-exporter/export-config-file conn "logseq/config.edn" read-file {})] _ (gp-exporter/export-config-file conn "logseq/config.edn" read-file {})]
(is (= "yyyy-MM-dd" (is (= "yyyy-MM-dd"
(:logseq.property.journal/title-format (d/entity @conn :logseq.class/Journal))) (:logseq.property.journal/title-format (d/entity @conn :logseq.class/Journal)))

22
deps/outliner/src/logseq/outliner/cli.cljs vendored
View File

@@ -1,13 +1,15 @@
(ns ^:node-only logseq.outliner.cli (ns ^:node-only logseq.outliner.cli
"Primary ns for outliner CLI fns" "Primary ns for outliner CLI fns"
(:require [clojure.string :as string] (:require [borkdude.rewrite-edn :as rewrite]
[clojure.string :as string]
[datascript.core :as d] [datascript.core :as d]
[logseq.db.sqlite.create-graph :as sqlite-create-graph] [logseq.db.sqlite.create-graph :as sqlite-create-graph]
[logseq.db.sqlite.build :as sqlite-build] [logseq.db.sqlite.build :as sqlite-build]
[logseq.db.common.sqlite-cli :as sqlite-cli] [logseq.db.common.sqlite-cli :as sqlite-cli]
[logseq.outliner.db-pipeline :as db-pipeline] [logseq.outliner.db-pipeline :as db-pipeline]
["fs" :as fs] ["fs" :as fs]
["path" :as node-path])) ["path" :as node-path]
[logseq.common.config :as common-config]))
(defn- find-on-classpath [classpath rel-path] (defn- find-on-classpath [classpath rel-path]
(some (fn [dir] (some (fn [dir]
@@ -15,6 +17,15 @@
(when (fs/existsSync f) f))) (when (fs/existsSync f) f)))
(string/split classpath #":"))) (string/split classpath #":")))
(defn- pretty-print-merge
"Merge map into string while preversing whitespace"
[s m]
(-> (reduce (fn [acc [k v]]
(rewrite/assoc acc k v))
(rewrite/parse-string s)
m)
str))
(defn- setup-init-data (defn- setup-init-data
"Setup initial data same as frontend.handler.repo/create-db" "Setup initial data same as frontend.handler.repo/create-db"
[conn {:keys [additional-config classpath import-type] [conn {:keys [additional-config classpath import-type]
@@ -23,11 +34,10 @@
(cond-> (or (some-> (find-on-classpath classpath "templates/config.edn") fs/readFileSync str) (cond-> (or (some-> (find-on-classpath classpath "templates/config.edn") fs/readFileSync str)
(do (println "Setting graph's config to empty since no templates/config.edn was found.") (do (println "Setting graph's config to empty since no templates/config.edn was found.")
"{}")) "{}"))
true
(common-config/create-config-for-db-graph)
additional-config additional-config
;; TODO: Replace with rewrite-clj when it's available (pretty-print-merge additional-config))]
(string/replace-first #"(:file/name-format :triple-lowbar)"
(str "$1 "
(string/replace-first (str additional-config) #"^\{(.*)\}$" "$1"))))]
(d/transact! conn (sqlite-create-graph/build-db-initial-data config-content {:import-type import-type})))) (d/transact! conn (sqlite-create-graph/build-db-initial-data config-content {:import-type import-type}))))
(defn init-conn (defn init-conn

11
src/main/frontend/components/block.cljs
View File

@@ -62,7 +62,6 @@
[frontend.mobile.util :as mobile-util] [frontend.mobile.util :as mobile-util]
[frontend.modules.outliner.tree :as tree] [frontend.modules.outliner.tree :as tree]
[frontend.modules.shortcut.utils :as shortcut-utils] [frontend.modules.shortcut.utils :as shortcut-utils]
[frontend.util.ref :as ref]
[frontend.security :as security] [frontend.security :as security]
[frontend.state :as state] [frontend.state :as state]
[frontend.template :as template] [frontend.template :as template]
@@ -70,6 +69,7 @@
[frontend.util :as util] [frontend.util :as util]
[frontend.util.file-based.clock :as clock] [frontend.util.file-based.clock :as clock]
[frontend.util.file-based.drawer :as drawer] [frontend.util.file-based.drawer :as drawer]
[frontend.util.ref :as ref]
[frontend.util.text :as text-util] [frontend.util.text :as text-util]
[goog.dom :as gdom] [goog.dom :as gdom]
[goog.functions :refer [debounce]] [goog.functions :refer [debounce]]
@@ -939,13 +939,13 @@
"Component for a page. `page` argument contains :block/name which can be (un)sanitized page name. "Component for a page. `page` argument contains :block/name which can be (un)sanitized page name.
Keys for `config`: Keys for `config`:
- `:preview?`: Is this component under preview mode? (If true, `page-preview-trigger` won't be registered to this `page-cp`)" - `:preview?`: Is this component under preview mode? (If true, `page-preview-trigger` won't be registered to this `page-cp`)"
[state {:keys [label children preview? disable-preview? show-non-exists-page? table-view? tag? _skip-async-load?] :as config} page] [state {:keys [label children preview? disable-preview? show-non-exists-page? tag? _skip-async-load?] :as config} page]
(when-let [entity' (rum/react (:*entity state))] (when-let [entity' (rum/react (:*entity state))]
(let [entity (or (db/sub-block (:db/id entity')) entity') (let [entity (or (db/sub-block (:db/id entity')) entity')
config (assoc config :block entity)] config (assoc config :block entity)]
(cond (cond
entity entity
(if (ldb/page? entity) (if (or (ldb/page? entity) (not (:block/page entity)))
(let [page-name (some-> (:block/title entity) util/page-name-sanity-lc) (let [page-name (some-> (:block/title entity) util/page-name-sanity-lc)
whiteboard-page? (model/whiteboard-page? entity) whiteboard-page? (model/whiteboard-page? entity)
inner (page-inner (assoc config :whiteboard-page? whiteboard-page?) entity children label) inner (page-inner (assoc config :whiteboard-page? whiteboard-page?) entity children label)
@@ -962,10 +962,7 @@
(gp-mldoc/inline->edn label (mldoc/get-default-config :markdown)) (gp-mldoc/inline->edn label (mldoc/get-default-config :markdown))
label))) label)))
(and (:block/name page) (util/uuid-string? (:block/name page))) (and (:block/name page) show-non-exists-page?)
(invalid-node-ref (:block/name page))
(and (:block/name page) (or show-non-exists-page? table-view?))
(page-inner config (merge (page-inner config (merge
{:block/title (:block/name page) {:block/title (:block/name page)
:block/name (:block/name page)} :block/name (:block/name page)}

24
src/main/frontend/components/cmdk/core.cljs
View File

@@ -81,7 +81,9 @@
(string/replace input #"^#+" "")) (string/replace input #"^#+" ""))
(defn create-items [q] (defn create-items [q]
(when (and (not (string/blank? q)) (not config/publishing?)) (when (and (not (string/blank? q))
(not (#{"config.edn" "custom.js" "custom.css"} q))
(not config/publishing?))
(let [class? (string/starts-with? q "#")] (let [class? (string/starts-with? q "#")]
(->> [{:text (if class? "Create tag" "Create page") :icon "new-page" (->> [{:text (if class? "Create tag" "Create page") :icon "new-page"
:icon-theme :gray :icon-theme :gray
@@ -236,6 +238,25 @@
(hash-map :status :success :items) (hash-map :status :success :items)
(swap! !results update group merge))))) (swap! !results update group merge)))))
(defmethod load-results :recently-updated-pages [group state]
(let [!input (::input state)
!results (::results state)]
(swap! !results assoc-in [group :status] :loading)
(let [recent-pages (ldb/get-recent-updated-pages (db/get-db))
search-results (if (string/blank? @!input)
recent-pages
(search/fuzzy-search recent-pages @!input {:extract-fn :block/title}))]
(->> search-results
(map (fn [block]
(let [text (block-handler/block-unique-title block)
icon (get-page-icon block)]
{:icon icon
:icon-theme :gray
:text text
:source-block block})))
(hash-map :status :success :items)
(swap! !results update group merge)))))
(defn highlight-content-query (defn highlight-content-query
"Return hiccup of highlighted content FTS result" "Return hiccup of highlighted content FTS result"
[content q] [content q]
@@ -396,6 +417,7 @@
(load-results :nodes state) (load-results :nodes state)
(load-results :filters state) (load-results :filters state)
(load-results :files state) (load-results :files state)
(load-results :recently-updated-pages state)
;; (load-results :recents state) ;; (load-results :recents state)
))))) )))))

3
src/main/frontend/components/imports.cljs
View File

@@ -370,8 +370,7 @@
;; config file options ;; config file options
:default-config config/config-default-content :default-config config/config-default-content
:<save-config-file (fn save-config-file [_ path content] :<save-config-file (fn save-config-file [_ path content]
(let [migrated-content (repo-handler/migrate-db-config content)] (db-editor-handler/save-file! path content))
(db-editor-handler/save-file! path migrated-content)))
;; logseq file options ;; logseq file options
:<save-logseq-file (fn save-logseq-file [_ path content] :<save-logseq-file (fn save-logseq-file [_ path content]
(db-editor-handler/save-file! path content)) (db-editor-handler/save-file! path content))

1
src/main/frontend/components/reference.cljs
View File

@@ -43,6 +43,7 @@
(views/view (views/view
{:view-parent page-entity {:view-parent page-entity
:view-feature-type :linked-references :view-feature-type :linked-references
:show-items-count? true
:additional-actions [reference-filter] :additional-actions [reference-filter]
:columns (views/build-columns config [] {}) :columns (views/build-columns config [] {})
:config config}))) :config config})))

39
src/main/frontend/handler/common/config_edn.cljs
View File

@@ -7,7 +7,8 @@
[lambdaisland.glogi :as log] [lambdaisland.glogi :as log]
[frontend.handler.notification :as notification] [frontend.handler.notification :as notification]
[goog.string :as gstring] [goog.string :as gstring]
[reitit.frontend.easy :as rfe])) [reitit.frontend.easy :as rfe]
[logseq.common.config :as common-config]))
(defn- humanize-more (defn- humanize-more
"Make error maps from me/humanize more readable for users. Doesn't try to handle "Make error maps from me/humanize more readable for users. Doesn't try to handle
@@ -86,40 +87,6 @@ nested keys or positional errors e.g. tuples"
:else :else
(validate-config-map parsed-body schema path)))) (validate-config-map parsed-body schema path))))
(def file-only-config
"File only config that is deprecated in DB graphs"
(merge
(zipmap
[:file/name-format
:file-sync/ignore-files
:hidden
:ignored-page-references-keywords
:journal/file-name-format
:journal/page-title-format
:journals-directory
:logbook/settings
:org-mode/insert-file-link?
:pages-directory
:preferred-workflow
:property/separated-by-commas
:property-pages/excludelist
:srs/learning-fraction
:srs/initial-interval
:whiteboards-directory]
(repeat "is not used in DB graphs"))
{:preferred-format
"is not used in DB graphs as there is only markdown mode."
:property-pages/enabled?
"is not used in DB graphs as all properties have pages"
:block-hidden-properties
"is not used in DB graphs as hiding a property is done in its configuration"
:feature/enable-block-timestamps?
"is not used in DB graphs as it is always enabled"
:favorites
"is not stored in config for DB graphs"
:default-templates
"is replaced by #Template and the `Apply template to tags` property"}))
(defn detect-deprecations (defn detect-deprecations
"Detects config keys that will or have been deprecated" "Detects config keys that will or have been deprecated"
[path content {:keys [db-graph?]}] [path content {:keys [db-graph?]}]
@@ -132,7 +99,7 @@ nested keys or positional errors e.g. tuples"
"is no longer supported."} "is no longer supported."}
db-graph? db-graph?
(merge (merge
file-only-config))] common-config/file-only-config))]
(cond (cond
(= body ::failed-to-detect) (= body ::failed-to-detect)
(log/info :msg "Skip deprecation check since config is not valid edn") (log/info :msg "Skip deprecation check since config is not valid edn")

6
src/main/frontend/handler/events.cljs
View File

@@ -50,6 +50,7 @@
[frontend.util.persist-var :as persist-var] [frontend.util.persist-var :as persist-var]
[goog.dom :as gdom] [goog.dom :as gdom]
[lambdaisland.glogi :as log] [lambdaisland.glogi :as log]
[logseq.db.frontend.schema :as db-schema]
[promesa.core :as p])) [promesa.core :as p]))
;; TODO: should we move all events here? ;; TODO: should we move all events here?
@@ -199,7 +200,10 @@
:user-id user-uuid :user-id user-uuid
:graph-id graph-uuid :graph-id graph-uuid
:tx-id tx-id :tx-id tx-id
:db-based (config/db-based-graph? (state/get-current-repo)))] :db-based (config/db-based-graph? (state/get-current-repo))
:schema-version db-schema/version
:db-schema-version (when-let [db (frontend.db/get-db)]
(:kv/value (frontend.db/entity db :logseq.kv/schema-version))))]
(Sentry/captureException error (Sentry/captureException error
(bean/->js {:tags payload})))) (bean/->js {:tags payload}))))

16
src/main/frontend/handler/repo.cljs
View File

@@ -1,8 +1,7 @@
(ns frontend.handler.repo (ns frontend.handler.repo
"System-component-like ns that manages user's repos/graphs" "System-component-like ns that manages user's repos/graphs"
(:refer-clojure :exclude [clone]) (:refer-clojure :exclude [clone])
(:require [borkdude.rewrite-edn :as rewrite] (:require [cljs-bean.core :as bean]
[cljs-bean.core :as bean]
[clojure.string :as string] [clojure.string :as string]
[electron.ipc :as ipc] [electron.ipc :as ipc]
[frontend.config :as config] [frontend.config :as config]
@@ -11,7 +10,6 @@
[frontend.db.persist :as db-persist] [frontend.db.persist :as db-persist]
[frontend.db.react :as react] [frontend.db.react :as react]
[frontend.db.restore :as db-restore] [frontend.db.restore :as db-restore]
[frontend.handler.common.config-edn :as config-edn-common-handler]
[frontend.handler.global-config :as global-config-handler] [frontend.handler.global-config :as global-config-handler]
[frontend.handler.graph :as graph-handler] [frontend.handler.graph :as graph-handler]
[frontend.handler.notification :as notification] [frontend.handler.notification :as notification]
@@ -27,7 +25,8 @@
[frontend.util :as util] [frontend.util :as util]
[frontend.util.fs :as util-fs] [frontend.util.fs :as util-fs]
[frontend.util.text :as text-util] [frontend.util.text :as text-util]
[promesa.core :as p])) [promesa.core :as p]
[logseq.common.config :as common-config]))
;; Project settings should be checked in two situations: ;; Project settings should be checked in two situations:
;; 1. User changes the config.edn directly in logseq.com (fn: alter-file) ;; 1. User changes the config.edn directly in logseq.com (fn: alter-file)
@@ -177,16 +176,9 @@
(let [full-graph-name (string/lower-case (str config/db-version-prefix graph-name))] (let [full-graph-name (string/lower-case (str config/db-version-prefix graph-name))]
(some #(= (some-> (:url %) string/lower-case) full-graph-name) (state/get-repos)))) (some #(= (some-> (:url %) string/lower-case) full-graph-name) (state/get-repos))))
(defn migrate-db-config
[content]
(-> (reduce rewrite/dissoc
(rewrite/parse-string (str content))
(keys config-edn-common-handler/file-only-config))
str))
(defn- create-db [full-graph-name {:keys [file-graph-import?]}] (defn- create-db [full-graph-name {:keys [file-graph-import?]}]
(-> (->
(p/let [config (migrate-db-config config/config-default-content) (p/let [config (common-config/create-config-for-db-graph config/config-default-content)
_ (persist-db/<new full-graph-name _ (persist-db/<new full-graph-name
(cond-> {:config config} (cond-> {:config config}
file-graph-import? (assoc :import-type :file-graph))) file-graph-import? (assoc :import-type :file-graph)))

8
src/main/frontend/modules/instrumentation/sentry.cljs
View File

@@ -16,14 +16,14 @@
version) version)
:environment (if config/dev? "development" "production") :environment (if config/dev? "development" "production")
:initialScope {:tags :initialScope {:tags
(merge (cond->
(when (not-empty config/revision)
{:revision config/revision})
{:platform (cond {:platform (cond
(util/electron?) "electron" (util/electron?) "electron"
(mobile-util/native-platform?) "mobile" (mobile-util/native-platform?) "mobile"
:else "web") :else "web")
:publishing config/publishing?})} :publishing config/publishing?}
(not-empty config/revision)
(assoc :revision config/revision))}
;; :integrations [(new posthog/SentryIntegration posthog "logseq" 5311485) ;; :integrations [(new posthog/SentryIntegration posthog "logseq" 5311485)
;; (new BrowserTracing)] ;; (new BrowserTracing)]
:debug config/dev? :debug config/dev?

1
src/main/frontend/state.cljs
View File

@@ -2376,7 +2376,6 @@ Similar to re-frame subscriptions"
(defn set-highlight-recent-days! (defn set-highlight-recent-days!
[days] [days]
(prn :debug :set :days days)
(reset! (:ui/highlight-recent-days @state) days) (reset! (:ui/highlight-recent-days @state) days)
(storage/set :ui/highlight-recent-days days)) (storage/set :ui/highlight-recent-days days))

11
src/main/frontend/worker/db/migrate.cljs
View File

@@ -16,6 +16,7 @@
[logseq.db.common.order :as db-order] [logseq.db.common.order :as db-order]
[logseq.db.frontend.class :as db-class] [logseq.db.frontend.class :as db-class]
[logseq.db.frontend.content :as db-content] [logseq.db.frontend.content :as db-content]
[logseq.db.frontend.malli-schema :as db-malli-schema]
[logseq.db.frontend.property :as db-property] [logseq.db.frontend.property :as db-property]
[logseq.db.frontend.property.build :as db-property-build] [logseq.db.frontend.property.build :as db-property-build]
[logseq.db.frontend.schema :as db-schema] [logseq.db.frontend.schema :as db-schema]
@@ -1172,8 +1173,11 @@
(js/console.error e) (js/console.error e)
(throw e))))))) (throw e)))))))
(defn- build-invalid-tx [entity eid] (defn- build-invalid-tx [db entity eid]
(cond (cond
(nil? (db-malli-schema/entity-dispatch-key db entity))
[[:db/retractEntity eid]]
(:block/schema entity) (:block/schema entity)
[[:db/retract eid :block/schema]] [[:db/retract eid :block/schema]]
@@ -1200,7 +1204,8 @@
(= #{:block/tx-id} (set (keys entity))) (= #{:block/tx-id} (set (keys entity)))
[[:db/retractEntity (:db/id entity)]] [[:db/retractEntity (:db/id entity)]]
(and (seq (:block/refs entity)) (and (or (seq (:block/refs entity))
(:logseq.property.table/filters entity))
(not (or (:block/title entity) (:block/content entity) (:property.value/content entity)))) (not (or (:block/title entity) (:block/content entity) (:property.value/content entity))))
[[:db/retractEntity (:db/id entity)]] [[:db/retractEntity (:db/id entity)]]
@@ -1285,7 +1290,7 @@
[:db/retract (:db/id entity) k])))))) [:db/retract (:db/id entity) k]))))))
(into {} entity)) (into {} entity))
eid (:db/id entity) eid (:db/id entity)
fix (build-invalid-tx entity eid)] fix (build-invalid-tx db entity eid)]
(into fix wrong-choice))) (into fix wrong-choice)))
invalid-entity-ids) invalid-entity-ids)
distinct)] distinct)]

229
src/main/frontend/worker/db_worker.cljs
View File

@@ -32,7 +32,6 @@
[frontend.worker.util :as worker-util] [frontend.worker.util :as worker-util]
[goog.object :as gobj] [goog.object :as gobj]
[lambdaisland.glogi.console :as glogi-console] [lambdaisland.glogi.console :as glogi-console]
[logseq.common.log :as log]
[logseq.common.util :as common-util] [logseq.common.util :as common-util]
[logseq.db :as ldb] [logseq.db :as ldb]
[logseq.db.common.entity-plus :as entity-plus] [logseq.db.common.entity-plus :as entity-plus]
@@ -101,38 +100,39 @@
[^js pool data] [^js pool data]
(.importDb ^js pool repo-path data)) (.importDb ^js pool repo-path data))
(defn- get-all-datoms-from-sqlite-db (comment
[db] (defn- get-all-datoms-from-sqlite-db
(some->> (.exec db #js {:sql "select * from kvs" [db]
:rowMode "array"}) (some->> (.exec db #js {:sql "select * from kvs"
bean/->clj :rowMode "array"})
(mapcat bean/->clj
(fn [[_addr content _addresses]] (mapcat
(let [content' (sqlite-util/transit-read content) (fn [[_addr content _addresses]]
datoms (when (map? content') (let [content' (sqlite-util/transit-read content)
(:keys content'))] datoms (when (map? content')
datoms))) (:keys content'))]
distinct datoms)))
(map (fn [[e a v t]] distinct
(d/datom e a v t))))) (map (fn [[e a v t]]
(d/datom e a v t)))))
(defn- rebuild-db-from-datoms! (defn- rebuild-db-from-datoms!
"Persistent-sorted-set has been broken, used addresses can't be found" "Persistent-sorted-set has been broken, used addresses can't be found"
[datascript-conn sqlite-db import-type] [datascript-conn sqlite-db import-type]
(let [datoms (get-all-datoms-from-sqlite-db sqlite-db) (let [datoms (get-all-datoms-from-sqlite-db sqlite-db)
db (d/init-db [] db-schema/schema db (d/init-db [] db-schema/schema
{:storage (storage/storage @datascript-conn)}) {:storage (storage/storage @datascript-conn)})
db (d/db-with db db (d/db-with db
(map (fn [d] (map (fn [d]
[:db/add (:e d) (:a d) (:v d) (:t d)]) datoms))] [:db/add (:e d) (:a d) (:v d) (:t d)]) datoms))]
(prn :debug :rebuild-db-from-datoms :datoms-count (count datoms)) (prn :debug :rebuild-db-from-datoms :datoms-count (count datoms))
;; export db first ;; export db first
(when-not import-type (when-not import-type
(worker-util/post-message :notification ["The SQLite db will be exported to avoid any data-loss." :warning false]) (worker-util/post-message :notification ["The SQLite db will be exported to avoid any data-loss." :warning false])
(worker-util/post-message :export-current-db [])) (worker-util/post-message :export-current-db []))
(.exec sqlite-db #js {:sql "delete from kvs"}) (.exec sqlite-db #js {:sql "delete from kvs"})
(d/reset-conn! datascript-conn db) (d/reset-conn! datascript-conn db)
(db-migrate/fix-db! datascript-conn))) (db-migrate/fix-db! datascript-conn))))
(comment (comment
(defn- gc-kvs-table! (defn- gc-kvs-table!
@@ -158,78 +158,74 @@
:bind #js [addr]})))))))) :bind #js [addr]}))))))))
(defn- find-missing-addresses (defn- find-missing-addresses
[^Object db & {:keys [delete-addrs upsert-addr-content? open-db?]}] [conn ^Object db & {:keys [delete-addrs]}]
(worker-util/profile (let [schema (some->> (.exec db #js {:sql "select content from kvs where addr = 0"
"find-missing-addresses" :rowMode "array"})
(let [schema (some->> (.exec db #js {:sql "select content from kvs where addr = 0" bean/->clj
:rowMode "array"}) ffirst
bean/->clj sqlite-util/transit-read)
ffirst result (->> (.exec db #js {:sql "select addr, addresses from kvs"
sqlite-util/transit-read) :rowMode "array"})
result (->> (.exec db #js {:sql "select addr, addresses from kvs" bean/->clj
:rowMode "array"}) (keep (fn [[addr addresses]]
bean/->clj (when-not (and delete-addrs (delete-addrs addr))
(keep (fn [[addr addresses]] [addr (bean/->clj (js/JSON.parse addresses))]))))
(when-not (and delete-addrs (delete-addrs addr)) used-addresses (-> (set (concat (mapcat second result)
[addr (bean/->clj (js/JSON.parse addresses))])))) [0 1 (:eavt schema) (:avet schema) (:aevt schema)]))
used-addresses (-> (set (concat (mapcat second result) (clojure.set/difference delete-addrs))
[0 1 (:eavt schema) (:avet schema) (:aevt schema)])) missing-addresses (clojure.set/difference used-addresses (set (map first result)))]
(clojure.set/difference delete-addrs)) (when (seq missing-addresses)
missing-addresses (clojure.set/difference used-addresses (set (map first result)))] (let [version-in-db (when conn (db-schema/parse-schema-version (or (:kv/value (d/entity @conn :logseq.kv/schema-version)) 0)))
(when (seq missing-addresses) compare-result (when version-in-db (db-schema/compare-schema-version version-in-db "64.8"))]
(prn :error :missing-addresses missing-addresses) (when (and compare-result (not (neg? compare-result))) ; >= 64.8
(if worker-util/dev? (worker-util/post-message :capture-error
(throw (ex-info "Found missing addresses that shouldn't happen" {:missing-addresses missing-addresses})) {:error "db-missing-addresses-v2"
(worker-util/post-message :capture-error :payload {:missing-addresses missing-addresses}}))))
{:error "v2-db-missing-addresses" missing-addresses))
:payload {:missing-addresses missing-addresses
:upsert-addr-content upsert-addr-content?
:open-db open-db?}})))
missing-addresses)))
(defn upsert-addr-content! (defn upsert-addr-content!
"Upsert addr+data-seq. Update sqlite-cli/upsert-addr-content! when making changes" "Upsert addr+data-seq. Update sqlite-cli/upsert-addr-content! when making changes"
[repo data delete-addrs* & {:keys [client-ops-db?] :or {client-ops-db? false}}] [db data delete-addrs*]
(let [^Object db (worker-state/get-sqlite-conn repo (if client-ops-db? :client-ops :db)) (let [delete-addrs (clojure.set/difference (set delete-addrs*) #{0 1})]
delete-addrs (set delete-addrs*)]
(assert (some? db) "sqlite db not exists") (assert (some? db) "sqlite db not exists")
(.transaction db (fn [tx] (.transaction db (fn [tx]
(doseq [item data] (doseq [item data]
(.exec tx #js {:sql "INSERT INTO kvs (addr, content, addresses) values ($addr, $content, $addresses) on conflict(addr) do update set content = $content, addresses = $addresses" (.exec tx #js {:sql "INSERT INTO kvs (addr, content, addresses) values ($addr, $content, $addresses) on conflict(addr) do update set content = $content, addresses = $addresses"
:bind item})))) :bind item}))))
(when (seq delete-addrs) (when (seq delete-addrs)
(.transaction db (fn [tx]
(doseq [addr delete-addrs]
(.exec tx #js {:sql "Delete from kvs WHERE addr = ? AND NOT EXISTS (SELECT 1 FROM json_each(addresses) WHERE value = ?);"
:bind #js [addr]}))))
(let [missing-addrs (when worker-util/dev? (let [missing-addrs (when worker-util/dev?
(seq (find-missing-addresses db {:delete-addrs delete-addrs (seq (find-missing-addresses nil db {:delete-addrs delete-addrs})))]
:upsert-addr-content? true}))) (if (seq missing-addrs)
delete-addrs' (if missing-addrs (worker-util/post-message :notification [(str "Bug!! Missing addresses: " missing-addrs) :error false])
(remove (set missing-addrs) delete-addrs) (when (seq delete-addrs)
delete-addrs)] (.transaction db (fn [tx]
(when (seq delete-addrs') (doseq [addr delete-addrs]
(.transaction db (fn [tx] (.exec tx #js {:sql "Delete from kvs WHERE addr = ? AND NOT EXISTS (SELECT 1 FROM json_each(addresses) WHERE value = ?);"
(doseq [addr delete-addrs'] :bind #js [addr]}))))))))))
(.exec tx #js {:sql "Delete from kvs WHERE addr = ? AND NOT EXISTS (SELECT 1 FROM json_each(addresses) WHERE value = ?);"
:bind #js [addr]})))))))))
(defn restore-data-from-addr (defn restore-data-from-addr
"Update sqlite-cli/restore-data-from-addr when making changes" "Update sqlite-cli/restore-data-from-addr when making changes"
[repo addr & {:keys [client-ops-db?] :or {client-ops-db? false}}] [db addr]
(let [^Object db (worker-state/get-sqlite-conn repo (if client-ops-db? :client-ops :db))] (assert (some? db) "sqlite db not exists")
(assert (some? db) "sqlite db not exists") (when-let [result (-> (.exec db #js {:sql "select content, addresses from kvs where addr = ?"
(when-let [result (-> (.exec db #js {:sql "select content, addresses from kvs where addr = ?" :bind #js [addr]
:bind #js [addr] :rowMode "array"})
:rowMode "array"}) first)]
first)] (let [[content addresses] (bean/->clj result)
(let [[content addresses] (bean/->clj result) addresses (when addresses
addresses (when addresses (js/JSON.parse addresses))
(js/JSON.parse addresses)) data (sqlite-util/transit-read content)]
data (sqlite-util/transit-read content)] (if (and addresses (map? data))
(if (and addresses (map? data)) (assoc data :addresses addresses)
(assoc data :addresses addresses) data))))
data)))))
(defn new-sqlite-storage (defn new-sqlite-storage
"Update sqlite-cli/new-sqlite-storage when making changes" "Update sqlite-cli/new-sqlite-storage when making changes"
[repo _opts] [^Object db]
(reify IStorage (reify IStorage
(-store [_ addr+data-seq delete-addrs] (-store [_ addr+data-seq delete-addrs]
(let [used-addrs (set (mapcat (let [used-addrs (set (mapcat
@@ -249,36 +245,10 @@
:$content (sqlite-util/transit-write data') :$content (sqlite-util/transit-write data')
:$addresses addresses})) :$addresses addresses}))
addr+data-seq)] addr+data-seq)]
(upsert-addr-content! repo data delete-addrs))) (upsert-addr-content! db data delete-addrs)))
(-restore [_ addr] (-restore [_ addr]
(restore-data-from-addr repo addr)))) (restore-data-from-addr db addr))))
(defn new-sqlite-client-ops-storage
[repo]
(reify IStorage
(-store [_ addr+data-seq delete-addrs]
(let [used-addrs (set (mapcat
(fn [[addr data]]
(cons addr
(when (map? data)
(:addresses data))))
addr+data-seq))
delete-addrs (remove used-addrs delete-addrs)
data (map
(fn [[addr data]]
(let [data' (if (map? data) (dissoc data :addresses) data)
addresses (when (map? data)
(when-let [addresses (:addresses data)]
(js/JSON.stringify (bean/->js addresses))))]
#js {:$addr addr
:$content (sqlite-util/transit-write data')
:$addresses addresses}))
addr+data-seq)]
(upsert-addr-content! repo data delete-addrs :client-ops-db? true)))
(-restore [_ addr]
(restore-data-from-addr repo addr :client-ops-db? true))))
(defn- close-db-aux! (defn- close-db-aux!
[repo ^Object db ^Object search ^Object client-ops] [repo ^Object db ^Object search ^Object client-ops]
@@ -338,8 +308,9 @@
[repo {:keys [config import-type datoms]}] [repo {:keys [config import-type datoms]}]
(when-not (worker-state/get-sqlite-conn repo) (when-not (worker-state/get-sqlite-conn repo)
(p/let [[db search-db client-ops-db :as dbs] (get-dbs repo) (p/let [[db search-db client-ops-db :as dbs] (get-dbs repo)
storage (new-sqlite-storage repo {}) storage (new-sqlite-storage db)
client-ops-storage (when-not @*publishing? (new-sqlite-client-ops-storage repo)) client-ops-storage (when-not @*publishing?
(new-sqlite-storage client-ops-db))
db-based? (sqlite-util/db-based-graph? repo)] db-based? (sqlite-util/db-based-graph? repo)]
(swap! *sqlite-conns assoc repo {:db db (swap! *sqlite-conns assoc repo {:db db
:search search-db :search search-db
@@ -374,24 +345,14 @@
(when import-type {:import-type import-type}))] (when import-type {:import-type import-type}))]
(d/transact! conn initial-data {:initial-db? true}))) (d/transact! conn initial-data {:initial-db? true})))
(try ;; TODO: remove this once we can ensure there's no bug for missing addresses
;; because it's slow for large graphs
(when-not import-type
(when-let [missing-addresses (seq (find-missing-addresses conn db))]
(worker-util/post-message :notification ["It seems that the DB has been broken, please export a backup and contact Logseq team for help." :error false])
(throw (ex-info "DB missing addresses" {:missing-addresses missing-addresses}))))
;; TODO: remove this once we can ensure there's no bug for missing addresses (db-migrate/migrate conn search-db)
;; because it's slow for large graphs
(when-not import-type
(when-let [missing-addresses (seq (find-missing-addresses db {:open-db? true}))]
(throw (ex-info "DB missing addresses" {:missing-addresses missing-addresses}))))
(db-migrate/migrate conn search-db)
;; TODO: Remove this once we can ensure there's no bug for missing addresses
(catch :default e
(log/error (str "DB migrate failed for " repo ", error:") e)
(if (= (.-message e) "DB missing addresses")
(do
(rebuild-db-from-datoms! conn db import-type)
(db-migrate/migrate conn search-db))
(throw e))))
(db-listener/listen-db-changes! repo (get @*datascript-conns repo)))))) (db-listener/listen-db-changes! repo (get @*datascript-conns repo))))))

8
src/main/frontend/worker/pipeline.cljs
View File

@@ -107,9 +107,11 @@
(let [valid? (if (get-in tx-report [:tx-meta :reset-conn!]) (let [valid? (if (get-in tx-report [:tx-meta :reset-conn!])
true true
(db-validate/validate-tx-report! tx-report (:validate-db-options context)))] (db-validate/validate-tx-report! tx-report (:validate-db-options context)))]
(when (and (get-in context [:validate-db-options :fail-invalid?]) (not valid?)) (when-not valid?
(shared-service/broadcast-to-clients! :notification (when (or (get-in context [:validate-db-options :fail-invalid?]) worker-util/dev?)
[["Invalid DB!"] :error])))) (shared-service/broadcast-to-clients! :notification
[["Invalid DB!"] :error]))
(throw (ex-info "Invalid data" {:graph repo})))))
;; Ensure :block/order is unique for any block that has :block/parent ;; Ensure :block/order is unique for any block that has :block/parent
(when (or (:dev? context) (exists? js/process)) (when (or (:dev? context) (exists? js/process))

11
src/main/frontend/worker/shared_service.cljs
View File

@@ -294,10 +294,13 @@
(.postMessage common-channel #js {:type "master-changed" (.postMessage common-channel #js {:type "master-changed"
:master-client-id master-client-id :master-client-id master-client-id
:serviceName service-name}) :serviceName service-name})
(p/do! (->
(on-become-master-handler service-name) (p/do!
(<re-requests-in-flight-on-master! target) (on-become-master-handler service-name)
(p/resolve! status-ready-deferred-p))) (<re-requests-in-flight-on-master! target))
(p/finally
(fn []
(p/resolve! status-ready-deferred-p)))))
(defn <create-service (defn <create-service
"broadcast-data-types - For data matching these types, "broadcast-data-types - For data matching these types,