diff --git a/deps/common/nbb.edn b/deps/common/nbb.edn index cb82c3e8e3..0c73190ba5 100644 --- a/deps/common/nbb.edn +++ b/deps/common/nbb.edn @@ -1,4 +1,5 @@ {:paths ["src" "resources"] :deps - {io.github.nextjournal/nbb-test-runner - {:git/sha "60ed57aa04bca8d604f5ba6b28848bd887109347"}}} + ;; TODO: Remove fork when https://github.com/nextjournal/nbb-test-runner/pull/2 is merged + {io.github.colin-p-hill/nbb-test-runner + {:git/sha "def2cbdb5b3a0e1612b28bf64f5d869c27c733d3"}}} diff --git a/deps/common/resources/templates/config.edn b/deps/common/resources/templates/config.edn index 6611f68839..9e46af8c11 100644 --- a/deps/common/resources/templates/config.edn +++ b/deps/common/resources/templates/config.edn @@ -1,5 +1,7 @@ {:meta/version 1 + ;; == FILE GRAPH CONFIG == + ;; ;; Set the preferred format. ;; This is _only_ for file graphs. ;; Available options: @@ -42,6 +44,102 @@ ;; Default value: "yyyy_MM_dd" ;; :journal/file-name-format "yyyy_MM_dd" + ;; Set the default location for storing notes. + ;; This is _only_ for file graphs. + ;; Default value: "pages" + ;; :pages-directory "pages" + + ;; Set the default location for storing journals. + ;; This is _only_ for file graphs. + ;; Default value: "journals" + ;; :journals-directory "journals" + + ;; Set the default location for storing whiteboards. + ;; This is _only_ for file graphs. + ;; Default value: "whiteboards" + ;; :whiteboards-directory "whiteboards" + + ;; Enabling this option converts + ;; [[Grant Ideas]] to [[file:./grant_ideas.org][Grant Ideas]] for org-mode. + ;; For more information, visit https://github.com/logseq/logseq/issues/672 + ;; This is _only_ for file graphs. + ;; :org-mode/insert-file-link? false + +;; Favorites to list on the left sidebar + ;; This is _only_ for file graphs. + :favorites [] + + ;; Set flashcards interval. + ;; This is _only_ for file graphs. + ;; Expected value: + ;; - Float between 0 and 1 + ;; higher values result in faster changes to the next review interval. + ;; Default value: 0.5 + ;; :srs/learning-fraction 0.5 + + ;; Set the initial interval after the first successful review of a card. + ;; This is _only_ for file graphs. + ;; Default value: 4 + ;; :srs/initial-interval 4 + + ;; Hide specific block properties. + ;; This is _only_ for file graphs. + ;; Example usage: + ;; :block-hidden-properties #{:public :icon} + + ;; Create a page for all properties. + ;; This is _only_ for file graphs. + ;; Default value: true + :property-pages/enabled? true + + ;; Properties to exclude from having property pages + ;; This is _only_ for file graphs. + ;; Example usage: + ;; :property-pages/excludelist #{:duration :author} + + ;; By default, property value separated by commas will not be treated as + ;; page references. You can add properties to enable it. + ;; This is _only_ for file graphs. + ;; Example usage: + ;; :property/separated-by-commas #{:alias :tags} + + ;; Properties that are ignored when parsing property values for references + ;; This is _only_ for file graphs. + ;; Example usage: + ;; :ignored-page-references-keywords #{:author :website} + + ;; logbook configuration. + ;; This is _only_ for file graphs. + ;; :logbook/settings + ;; {:with-second-support? false ;limit logbook to minutes, seconds will be eliminated + ;; :enabled-in-all-blocks true ;display logbook in all blocks after timetracking + ;; :enabled-in-timestamped-blocks false ;don't display logbook at all + ;; } + + ;; File sync options + ;; Ignore these files when syncing, regexp is supported. + ;; This is _only_ for file graphs. + ;; :file-sync/ignore-files [] + + ;; Configure the escaping method for special characters in page titles. + ;; This is _only_ for file graphs. + ;; Warning: + ;; This is a dangerous operation. To modify the setting, + ;; you'll need to manually rename all affected files and + ;; re-index them on all clients after synchronization. + ;; Incorrect handling may result in messy page titles. + ;; Available options: + ;; - :triple-lowbar (default) + ;; ;use triple underscore `___` for slash `/` in page title + ;; ;use Percent-encoding for other invalid characters + :file/name-format :triple-lowbar + ;; == END OF FILE GRAPH CONFIG == + + ;; Hide empty block properties + ;; This is _only_ for DB graphs. + ;; Default value: false + ;; :ui/hide-empty-properties? false + ;; Enable tooltip preview on hover. ;; Default value: true :ui/enable-tooltip? true @@ -58,11 +156,6 @@ ;; Default value: true :ui/auto-expand-block-refs? true - ;; Hide empty block properties - ;; This is _only_ for DB graphs. - ;; Default value: false - ;; :ui/hide-empty-properties? false - ;; Disable accent marks when searching. ;; After changing this setting, rebuild the search index by pressing (^C ^S). ;; Default value: true @@ -137,27 +230,6 @@ ;; 3. Set "home" as the home page and display multiple pages in the right sidebar: ;; :default-home {:page "home", :sidebar ["Page A" "Page B"]} - ;; Set the default location for storing notes. - ;; This is _only_ for file graphs. - ;; Default value: "pages" - ;; :pages-directory "pages" - - ;; Set the default location for storing journals. - ;; This is _only_ for file graphs. - ;; Default value: "journals" - ;; :journals-directory "journals" - - ;; Set the default location for storing whiteboards. - ;; This is _only_ for file graphs. - ;; Default value: "whiteboards" - ;; :whiteboards-directory "whiteboards" - - ;; Enabling this option converts - ;; [[Grant Ideas]] to [[file:./grant_ideas.org][Grant Ideas]] for org-mode. - ;; For more information, visit https://github.com/logseq/logseq/issues/672 - ;; This is _only_ for file graphs. - ;; :org-mode/insert-file-link? false - ;; Configure custom shortcuts. ;; Syntax: ;; 1. + indicates simultaneous key presses, e.g., `Ctrl+Shift+a`. @@ -274,58 +346,6 @@ ;; :charge-strength -600 ; Default value: -600 ;; :charge-range 600} ; Default value: 600 - - ;; Favorites to list on the left sidebar - ;; This is _only_ for file graphs. - :favorites [] - - ;; Set flashcards interval. - ;; This is _only_ for file graphs. - ;; Expected value: - ;; - Float between 0 and 1 - ;; higher values result in faster changes to the next review interval. - ;; Default value: 0.5 - ;; :srs/learning-fraction 0.5 - - ;; Set the initial interval after the first successful review of a card. - ;; This is _only_ for file graphs. - ;; Default value: 4 - ;; :srs/initial-interval 4 - - ;; Hide specific block properties. - ;; This is _only_ for file graphs. - ;; Example usage: - ;; :block-hidden-properties #{:public :icon} - - ;; Create a page for all properties. - ;; This is _only_ for file graphs. - ;; Default value: true - :property-pages/enabled? true - - ;; Properties to exclude from having property pages - ;; This is _only_ for file graphs. - ;; Example usage: - ;; :property-pages/excludelist #{:duration :author} - - ;; By default, property value separated by commas will not be treated as - ;; page references. You can add properties to enable it. - ;; This is _only_ for file graphs. - ;; Example usage: - ;; :property/separated-by-commas #{:alias :tags} - - ;; Properties that are ignored when parsing property values for references - ;; This is _only_ for file graphs. - ;; Example usage: - ;; :ignored-page-references-keywords #{:author :website} - - ;; logbook configuration. - ;; This is _only_ for file graphs. - ;; :logbook/settings - ;; {:with-second-support? false ;limit logbook to minutes, seconds will be eliminated - ;; :enabled-in-all-blocks true ;display logbook in all blocks after timetracking - ;; :enabled-in-timestamped-blocks false ;don't display logbook at all - ;; } - ;; Mobile photo upload configuration. ;; :mobile/photo ;; {:allow-editing? true @@ -375,11 +395,6 @@ ;; :redirect-page? false ;; Default value: false ;; :default-page "quick capture"} ;; Default page: "quick capture" - ;; File sync options - ;; Ignore these files when syncing, regexp is supported. - ;; This is _only_ for file graphs. - ;; :file-sync/ignore-files [] - ;; Configure the Enter key behavior for ;; context-aware editing with DWIM (Do What I Mean). ;; context-aware Enter key behavior implies that pressing Enter will @@ -393,16 +408,4 @@ ;; :page-ref? true ;; Default value: true ;; :properties? true ;; Default value: true ;; :list? false} ;; Default value: false - - ;; Configure the escaping method for special characters in page titles. - ;; This is _only_ for file graphs. - ;; Warning: - ;; This is a dangerous operation. To modify the setting, - ;; you'll need to manually rename all affected files and - ;; re-index them on all clients after synchronization. - ;; Incorrect handling may result in messy page titles. - ;; Available options: - ;; - :triple-lowbar (default) - ;; ;use triple underscore `___` for slash `/` in page title - ;; ;use Percent-encoding for other invalid characters - :file/name-format :triple-lowbar} + } \ No newline at end of file diff --git a/deps/common/src/logseq/common/config.cljs b/deps/common/src/logseq/common/config.cljs index 45df37c299..eb8f385b5b 100644 --- a/deps/common/src/logseq/common/config.cljs +++ b/deps/common/src/logseq/common/config.cljs @@ -1,5 +1,5 @@ (ns logseq.common.config - "Common config and constants that are shared between deps and app" + "Common config constants and fns that are shared between deps and app" (:require [clojure.string :as string] [goog.object :as gobj])) @@ -115,3 +115,43 @@ "*" "-"))) + +(defn create-config-for-db-graph + "Given a new config.edn file string, creates a config.edn for use with only DB graphs" + [config] + (string/replace config #"(?m)[\s]*;; == FILE GRAPH CONFIG ==(?:.|\n)*?;; == END OF FILE GRAPH CONFIG ==\n?" "")) + +(def file-only-config + "File only config keys that are deprecated in DB graphs along with + descriptions for their deprecation." + (merge + (zipmap + [:file/name-format + :file-sync/ignore-files + :hidden + :ignored-page-references-keywords + :journal/file-name-format + :journal/page-title-format + :journals-directory + :logbook/settings + :org-mode/insert-file-link? + :pages-directory + :preferred-workflow + :property/separated-by-commas + :property-pages/excludelist + :srs/learning-fraction + :srs/initial-interval + :whiteboards-directory] + (repeat "is not used in DB graphs")) + {:preferred-format + "is not used in DB graphs as there is only markdown mode." + :property-pages/enabled? + "is not used in DB graphs as all properties have pages" + :block-hidden-properties + "is not used in DB graphs as hiding a property is done in its configuration" + :feature/enable-block-timestamps? + "is not used in DB graphs as it is always enabled" + :favorites + "is not stored in config for DB graphs" + :default-templates + "is replaced by #Template and the `Apply template to tags` property"})) \ No newline at end of file diff --git a/deps/common/test/logseq/common/config_test.cljc b/deps/common/test/logseq/common/config_test.cljc new file mode 100644 index 0000000000..c66be429cc --- /dev/null +++ b/deps/common/test/logseq/common/config_test.cljc @@ -0,0 +1,45 @@ +(ns logseq.common.config-test + (:require [cljs.test :refer [deftest is]] + [clojure.string :as string] + [logseq.common.config :as common-config] + #?(:org.babashka/nbb [nbb.classpath :as cp]) + ["fs" :as fs] + ["path" :as node-path])) + +(deftest remove-hidden-files + (let [files ["pages/foo.md" "pages/bar.md" + "script/README.md" "script/config.edn" + "dev/README.md" "dev/config.edn"]] + (is (= ["pages/foo.md" "pages/bar.md"] + #_:clj-kondo/ignore ;; buggy unresolved var + (common-config/remove-hidden-files + files + {:hidden ["script" "/dev"]} + identity)) + "Removes hidden relative files") + + (is (= ["/pages/foo.md" "/pages/bar.md"] + (common-config/remove-hidden-files + (map #(str "/" %) files) + {:hidden ["script" "/dev"]} + identity)) + "Removes hidden files if they start with '/'"))) + +(defn find-on-classpath [classpath rel-path] + (some (fn [dir] + (let [f (node-path/join dir rel-path)] + (when (fs/existsSync f) f))) + (string/split classpath #":"))) + +#?(:org.babashka/nbb + (deftest create-config-for-db-graph + (let [original-config (some-> (find-on-classpath (cp/get-classpath) "templates/config.edn") fs/readFileSync str) + _ (assert original-config "config.edn must not be blank") + migrated-config (common-config/create-config-for-db-graph original-config) + forbidden-kws-regex (re-pattern (str (string/join "|" (keys common-config/file-only-config))))] + ;; (println migrated-config) + (is (not (string/includes? migrated-config "== FILE ONLY CONFIG")) + "No longer includes file config header") + (assert (re-find forbidden-kws-regex original-config) "File config keys present in original config") + (is (not (re-find forbidden-kws-regex migrated-config)) + "File config keys no longer present in migrated config")))) \ No newline at end of file diff --git a/deps/common/test/logseq/common/config_test.cljs b/deps/common/test/logseq/common/config_test.cljs deleted file mode 100644 index c37d576ffa..0000000000 --- a/deps/common/test/logseq/common/config_test.cljs +++ /dev/null @@ -1,21 +0,0 @@ -(ns logseq.common.config-test - (:require [logseq.common.config :as common-config] - [cljs.test :refer [deftest is]])) - -(deftest remove-hidden-files - (let [files ["pages/foo.md" "pages/bar.md" - "script/README.md" "script/config.edn" - "dev/README.md" "dev/config.edn"]] - (is (= ["pages/foo.md" "pages/bar.md"] - (common-config/remove-hidden-files - files - {:hidden ["script" "/dev"]} - identity)) - "Removes hidden relative files") - - (is (= ["/pages/foo.md" "/pages/bar.md"] - (common-config/remove-hidden-files - (map #(str "/" %) files) - {:hidden ["script" "/dev"]} - identity)) - "Removes hidden files if they start with '/'"))) \ No newline at end of file diff --git a/deps/db/deps.edn b/deps/db/deps.edn index e9e5bfaace..b749423da2 100644 --- a/deps/db/deps.edn +++ b/deps/db/deps.edn @@ -12,6 +12,7 @@ logseq/common {:local/root "../common"} logseq/clj-fractional-indexing {:git/url "https://github.com/logseq/clj-fractional-indexing" :sha "7182b7878410f78536dc2b6df35ed32ef9cd6b61"} + borkdude/rewrite-edn {:mvn/version "0.4.9"} metosin/malli {:mvn/version "0.16.1"} medley/medley {:mvn/version "1.4.0"}} diff --git a/deps/db/nbb.edn b/deps/db/nbb.edn index 128592ccad..8da6ac9129 100644 --- a/deps/db/nbb.edn +++ b/deps/db/nbb.edn @@ -4,6 +4,8 @@ {:local/root "../common"} medley/medley {:mvn/version "1.4.0"} metosin/malli {:mvn/version "0.16.1"} + ;; Used by db scripts with outliner.cli + borkdude/rewrite-edn {:mvn/version "0.4.9"} logseq/clj-fractional-indexing {:git/url "https://github.com/logseq/clj-fractional-indexing" :sha "7182b7878410f78536dc2b6df35ed32ef9cd6b61"} io.github.nextjournal/nbb-test-runner diff --git a/deps/db/src/logseq/db/frontend/malli_schema.cljs b/deps/db/src/logseq/db/frontend/malli_schema.cljs index fa0be75713..413a42f59b 100644 --- a/deps/db/src/logseq/db/frontend/malli_schema.cljs +++ b/deps/db/src/logseq/db/frontend/malli_schema.cljs @@ -3,9 +3,9 @@ (:require [clojure.set :as set] [clojure.string :as string] [datascript.core :as d] + [logseq.db.common.entity-plus :as entity-plus] [logseq.db.common.order :as db-order] [logseq.db.frontend.class :as db-class] - [logseq.db.common.entity-plus :as entity-plus] [logseq.db.frontend.entity-util :as entity-util] [logseq.db.frontend.property :as db-property] [logseq.db.frontend.property.type :as db-property-type] @@ -482,14 +482,10 @@ :file-block (:logseq.property.history/block d) :property-history-block - (:block/closed-value-property d) :closed-value-block - - (and (:logseq.property/created-from-property d) - (:logseq.property/value d)) + (and (:logseq.property/created-from-property d) (:logseq.property/value d)) :property-value-block - (:block/uuid d) :block (= (:db/ident d) :logseq.property/empty-placeholder) diff --git a/deps/db/src/logseq/db/sqlite/export.cljs b/deps/db/src/logseq/db/sqlite/export.cljs index b6b80447bc..330eb1f797 100644 --- a/deps/db/src/logseq/db/sqlite/export.cljs +++ b/deps/db/src/logseq/db/sqlite/export.cljs @@ -803,21 +803,29 @@ set)] (set/difference ref-uuids known-uuids))) +(defn- remove-namespaced-keys + "Removes keys from this ns for maps passed sqlite.build fns as they don't need to validate or use them" + [m] + (->> m + (remove (fn [[k _v]] (= "logseq.db.sqlite.export" (namespace k)))) + (into {}))) + (defn- ensure-export-is-valid "Checks that export map is usable by sqlite.build including checking that all referenced properties and classes are defined. Checks related to properties and classes are disabled when :exclude-namespaces is set because those checks can't be done" - [export-map {:keys [graph-options]}] - (when-not (seq (:exclude-namespaces graph-options)) (sqlite-build/validate-options export-map)) - (let [undefined-uuids (find-undefined-uuids export-map) - undefined (cond-> {} - (empty? (:exclude-namespaces graph-options)) - (merge (find-undefined-classes-and-properties export-map)) - (seq undefined-uuids) - (assoc :uuids undefined-uuids))] - (when (seq undefined) - (throw (ex-info (str "The following classes, uuids and properties are not defined: " (pr-str undefined)) - undefined))))) + [export-map* {:keys [graph-options]}] + (let [export-map (remove-namespaced-keys export-map*)] + (when-not (seq (:exclude-namespaces graph-options)) (sqlite-build/validate-options export-map)) + (let [undefined-uuids (find-undefined-uuids export-map) + undefined (cond-> {} + (empty? (:exclude-namespaces graph-options)) + (merge (find-undefined-classes-and-properties export-map)) + (seq undefined-uuids) + (assoc :uuids undefined-uuids))] + (when (seq undefined) + (throw (ex-info (str "The following classes, uuids and properties are not defined: " (pr-str undefined)) + undefined)))))) (defn build-export "Handles exporting db by given export-type" @@ -838,30 +846,33 @@ (build-graph-export db (:graph-options options)))] (if (get-in options [:graph-options :catch-validation-errors?]) (try - (ensure-export-is-valid (dissoc export-map ::block ::graph-files ::kv-values ::schema-version) options) + (ensure-export-is-valid export-map options) (catch ExceptionInfo e (println "Caught error:" e))) - (ensure-export-is-valid (dissoc export-map ::block ::graph-files ::kv-values ::schema-version) options)) + (ensure-export-is-valid export-map options)) (assoc export-map ::export-type export-type))) ;; Import fns ;; ========== (defn- add-uuid-to-page-if-exists - [db import-to-existing-page-uuids m] - (if-let [ent (some->> (:build/journal m) - (d/datoms db :avet :block/journal-day) - first - :e - (d/entity db))] + [db import-to-existing-page-uuids {:keys [existing-pages-keep-properties?]} m] + (if-let [ent (if (:build/journal m) + (some->> (:build/journal m) + (d/datoms db :avet :block/journal-day) + first + :e + (d/entity db)) + ;; TODO: For now only check page uniqueness by title. Could handle more uniqueness checks later + (some->> (:block/title m) (ldb/get-case-page db)))] (do (swap! import-to-existing-page-uuids assoc (:block/uuid m) (:block/uuid ent)) - (assoc m :block/uuid (:block/uuid ent))) - ;; TODO: For now only check page uniqueness by title. Could handle more uniqueness checks later - (if-let [ent (some->> (:block/title m) (ldb/get-case-page db))] - (do - (swap! import-to-existing-page-uuids assoc (:block/uuid m) (:block/uuid ent)) - (assoc m :block/uuid (:block/uuid ent))) - m))) + (cond-> (assoc m :block/uuid (:block/uuid ent)) + (and (:build/properties m) existing-pages-keep-properties?) + (update :build/properties (fn [props] + (->> props + (remove (fn [[k _v]] (get ent k))) + (into {})))))) + m)) (defn- update-existing-properties "Updates existing properties by ident. Also check imported and existing properties have @@ -884,7 +895,7 @@ (defn- check-for-existing-entities "Checks export map for existing entities and adds :block/uuid to them if they exist in graph to import. Also checks for property conflicts between existing properties and properties to be imported" - [db {:keys [pages-and-blocks classes properties] ::keys [export-type] :as export-map} property-conflicts] + [db {:keys [pages-and-blocks classes properties] ::keys [export-type import-options] :as export-map} property-conflicts] (let [import-to-existing-page-uuids (atom {}) export-map (cond-> {:build-existing-tx? true @@ -892,7 +903,7 @@ (seq pages-and-blocks) (assoc :pages-and-blocks (mapv (fn [m] - (update m :page (partial add-uuid-to-page-if-exists db import-to-existing-page-uuids))) + (update m :page (partial add-uuid-to-page-if-exists db import-to-existing-page-uuids import-options))) pages-and-blocks)) (seq classes) (assoc :classes @@ -915,7 +926,7 @@ (walk/postwalk (fn [f] (if (and (vector? f) (= :build/page (first f))) [:build/page - (add-uuid-to-page-if-exists db import-to-existing-page-uuids (second f))] + (add-uuid-to-page-if-exists db import-to-existing-page-uuids import-options (second f))] f)) export-map)) ;; Update uuid references of all pages that had their uuids updated to reference an existing page @@ -948,6 +959,8 @@ * ::kv-values - Vec of :kv/value maps for a :graph export * ::auto-include-namespaces - A set of parent namespaces to include from properties and classes for a :graph export. See :exclude-namespaces in build-graph-export for a similar option + * ::import-options - A map of options that alters importing behavior. Has the following keys: + * :existing-pages-keep-properties? - Boolean which disables upsert of :build/properties on This fn then returns a map of txs to transact with the following keys: * :init-tx - Txs that must be transacted first, usually because they define new properties @@ -969,7 +982,7 @@ {:error (str "The following imported properties conflict with the current graph: " (pr-str (mapv :property-id @property-conflicts)))}) (if (= :graph (::export-type export-map'')) - (-> (sqlite-build/build-blocks-tx (dissoc export-map'' ::graph-files ::kv-values ::export-type ::schema-version)) + (-> (sqlite-build/build-blocks-tx (remove-namespaced-keys export-map'')) (assoc :misc-tx (vec (concat (::graph-files export-map'') (::kv-values export-map''))))) - (sqlite-build/build-blocks-tx export-map''))))) + (sqlite-build/build-blocks-tx (remove-namespaced-keys export-map'')))))) diff --git a/deps/db/test/logseq/db/sqlite/export_test.cljs b/deps/db/test/logseq/db/sqlite/export_test.cljs index 3c1c9da457..d9603d0ca5 100644 --- a/deps/db/test/logseq/db/sqlite/export_test.cljs +++ b/deps/db/test/logseq/db/sqlite/export_test.cljs @@ -826,54 +826,65 @@ (-> (:classes imported-graph) (medley/dissoc-in [:user.property/p1 :build/properties])))))) -(deftest build-import-can-import-existing-page-with-different-uuid +(defn- test-import-existing-page [import-options expected-page-properties] (let [original-data {:properties {:user.property/node {:logseq.property/type :node :db/cardinality :db.cardinality/many}} :pages-and-blocks [{:page {:block/title "page1" - :build/properties {:user.property/node #{[:build/page {:block/title "node1"}]}}}}]} + :build/properties {:user.property/node + #{[:build/page {:block/title "existing page" + :build/properties {:logseq.property/description "first description"}}]}}}}]} conn (db-test/create-conn-with-blocks original-data) - page-uuid (:block/uuid (db-test/find-page-by-title @conn "node1")) + page-uuid (:block/uuid (db-test/find-page-by-title @conn "existing page")) _ (validate-db @conn) ;; This is just a temp uuid used to link to the page during import temp-uuid (random-uuid) - existing-data + import-data {:properties {:user.property/node {:logseq.property/type :node :db/cardinality :db.cardinality/many}} :pages-and-blocks - [{:page {:block/title "node1" + [{:page {:block/title "existing page" :block/uuid temp-uuid - :build/keep-uuid? true}} + :build/keep-uuid? true + :build/properties {:logseq.property/description "second description" + :logseq.property/exclude-from-graph-view true}}} {:page {:block/title "page2" - :build/properties {:user.property/node #{[:block/uuid temp-uuid]}}}}]} + :build/properties {:user.property/node #{[:block/uuid temp-uuid]}}}}] + ::sqlite-export/import-options import-options} {:keys [init-tx block-props-tx] :as _txs} - (sqlite-export/build-import existing-data @conn {}) + (sqlite-export/build-import import-data @conn {}) ;; _ (cljs.pprint/pprint _txs) _ (d/transact! conn init-tx) _ (d/transact! conn block-props-tx) _ (validate-db @conn) expected-pages-and-blocks - [{:page - {:block/uuid page-uuid - :build/keep-uuid? true, - :block/title "node1"}, - :blocks []} - {:page - {:build/properties - {:user.property/node - #{[:block/uuid page-uuid]}}, - :block/title "page1"}, - :blocks []} - {:page - {:build/properties - {:user.property/node - #{[:block/uuid page-uuid]}}, - :block/title "page2"}, - :blocks []}], + [{:block/uuid page-uuid + :build/keep-uuid? true, + :block/title "existing page" + :build/properties + expected-page-properties} + {:build/properties + {:user.property/node + #{[:block/uuid page-uuid]}}, + :block/title "page1"} + {:build/properties + {:user.property/node + #{[:block/uuid page-uuid]}}, + :block/title "page2"}] exported-graph (sqlite-export/build-export @conn {:export-type :graph :graph-options {:exclude-built-in-pages? true}})] (is (= expected-pages-and-blocks - (:pages-and-blocks exported-graph)) - "page uuid ('node1') is preserved across imports even when its assigned a temporary - uuid to relate it to other nodes"))) \ No newline at end of file + (map :page (:pages-and-blocks exported-graph))) + "page uuid of 'existing page' is preserved across imports even when its assigned a temporary + uuid to relate it to other nodes"))) + +(deftest build-import-can-import-existing-page-with-different-uuid + (testing "By default any properties passed to an existing page are upserted" + (test-import-existing-page {} + {:logseq.property/description "second description" + :logseq.property/exclude-from-graph-view true})) + (testing "With ::existing-pages-keep-properties?, existing properties on existing pages are not overwritten by imported data" + (test-import-existing-page {:existing-pages-keep-properties? true} + {:logseq.property/description "first description" + :logseq.property/exclude-from-graph-view true}))) \ No newline at end of file diff --git a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs index 6b526e9549..eb62a4d078 100644 --- a/deps/graph-parser/src/logseq/graph_parser/exporter.cljs +++ b/deps/graph-parser/src/logseq/graph_parser/exporter.cljs @@ -764,6 +764,7 @@ {:block block'' :properties-tx properties-tx})) (defn- pretty-print-dissoc + "Remove list of keys from a given map string while preserving whitespace" [s dissoc-keys] (-> (reduce rewrite/dissoc (rewrite/parse-string s) @@ -1496,7 +1497,11 @@ ( (reduce (fn [acc [k v]] + (rewrite/assoc acc k v)) + (rewrite/parse-string s) + m) + str)) + (defn- setup-init-data "Setup initial data same as frontend.handler.repo/create-db" [conn {:keys [additional-config classpath import-type] @@ -23,11 +34,10 @@ (cond-> (or (some-> (find-on-classpath classpath "templates/config.edn") fs/readFileSync str) (do (println "Setting graph's config to empty since no templates/config.edn was found.") "{}")) + true + (common-config/create-config-for-db-graph) additional-config - ;; TODO: Replace with rewrite-clj when it's available - (string/replace-first #"(:file/name-format :triple-lowbar)" - (str "$1 " - (string/replace-first (str additional-config) #"^\{(.*)\}$" "$1"))))] + (pretty-print-merge additional-config))] (d/transact! conn (sqlite-create-graph/build-db-initial-data config-content {:import-type import-type})))) (defn init-conn diff --git a/src/main/frontend/components/block.cljs b/src/main/frontend/components/block.cljs index ba31bff564..44329b9f09 100644 --- a/src/main/frontend/components/block.cljs +++ b/src/main/frontend/components/block.cljs @@ -62,7 +62,6 @@ [frontend.mobile.util :as mobile-util] [frontend.modules.outliner.tree :as tree] [frontend.modules.shortcut.utils :as shortcut-utils] - [frontend.util.ref :as ref] [frontend.security :as security] [frontend.state :as state] [frontend.template :as template] @@ -70,6 +69,7 @@ [frontend.util :as util] [frontend.util.file-based.clock :as clock] [frontend.util.file-based.drawer :as drawer] + [frontend.util.ref :as ref] [frontend.util.text :as text-util] [goog.dom :as gdom] [goog.functions :refer [debounce]] @@ -939,13 +939,13 @@ "Component for a page. `page` argument contains :block/name which can be (un)sanitized page name. Keys for `config`: - `:preview?`: Is this component under preview mode? (If true, `page-preview-trigger` won't be registered to this `page-cp`)" - [state {:keys [label children preview? disable-preview? show-non-exists-page? table-view? tag? _skip-async-load?] :as config} page] + [state {:keys [label children preview? disable-preview? show-non-exists-page? tag? _skip-async-load?] :as config} page] (when-let [entity' (rum/react (:*entity state))] (let [entity (or (db/sub-block (:db/id entity')) entity') config (assoc config :block entity)] (cond entity - (if (ldb/page? entity) + (if (or (ldb/page? entity) (not (:block/page entity))) (let [page-name (some-> (:block/title entity) util/page-name-sanity-lc) whiteboard-page? (model/whiteboard-page? entity) inner (page-inner (assoc config :whiteboard-page? whiteboard-page?) entity children label) @@ -962,10 +962,7 @@ (gp-mldoc/inline->edn label (mldoc/get-default-config :markdown)) label))) - (and (:block/name page) (util/uuid-string? (:block/name page))) - (invalid-node-ref (:block/name page)) - - (and (:block/name page) (or show-non-exists-page? table-view?)) + (and (:block/name page) show-non-exists-page?) (page-inner config (merge {:block/title (:block/name page) :block/name (:block/name page)} diff --git a/src/main/frontend/components/cmdk/core.cljs b/src/main/frontend/components/cmdk/core.cljs index b92d6900a0..ada9d2cc06 100644 --- a/src/main/frontend/components/cmdk/core.cljs +++ b/src/main/frontend/components/cmdk/core.cljs @@ -81,7 +81,9 @@ (string/replace input #"^#+" "")) (defn create-items [q] - (when (and (not (string/blank? q)) (not config/publishing?)) + (when (and (not (string/blank? q)) + (not (#{"config.edn" "custom.js" "custom.css"} q)) + (not config/publishing?)) (let [class? (string/starts-with? q "#")] (->> [{:text (if class? "Create tag" "Create page") :icon "new-page" :icon-theme :gray @@ -236,6 +238,25 @@ (hash-map :status :success :items) (swap! !results update group merge))))) +(defmethod load-results :recently-updated-pages [group state] + (let [!input (::input state) + !results (::results state)] + (swap! !results assoc-in [group :status] :loading) + (let [recent-pages (ldb/get-recent-updated-pages (db/get-db)) + search-results (if (string/blank? @!input) + recent-pages + (search/fuzzy-search recent-pages @!input {:extract-fn :block/title}))] + (->> search-results + (map (fn [block] + (let [text (block-handler/block-unique-title block) + icon (get-page-icon block)] + {:icon icon + :icon-theme :gray + :text text + :source-block block}))) + (hash-map :status :success :items) + (swap! !results update group merge))))) + (defn highlight-content-query "Return hiccup of highlighted content FTS result" [content q] @@ -396,6 +417,7 @@ (load-results :nodes state) (load-results :filters state) (load-results :files state) + (load-results :recently-updated-pages state) ;; (load-results :recents state) ))))) diff --git a/src/main/frontend/components/imports.cljs b/src/main/frontend/components/imports.cljs index 7c316c45be..e700a1c69a 100644 --- a/src/main/frontend/components/imports.cljs +++ b/src/main/frontend/components/imports.cljs @@ -370,8 +370,7 @@ ;; config file options :default-config config/config-default-content :js {:tags payload})))) diff --git a/src/main/frontend/handler/repo.cljs b/src/main/frontend/handler/repo.cljs index 47b150d9b6..dfd7992cbb 100644 --- a/src/main/frontend/handler/repo.cljs +++ b/src/main/frontend/handler/repo.cljs @@ -1,8 +1,7 @@ (ns frontend.handler.repo "System-component-like ns that manages user's repos/graphs" (:refer-clojure :exclude [clone]) - (:require [borkdude.rewrite-edn :as rewrite] - [cljs-bean.core :as bean] + (:require [cljs-bean.core :as bean] [clojure.string :as string] [electron.ipc :as ipc] [frontend.config :as config] @@ -11,7 +10,6 @@ [frontend.db.persist :as db-persist] [frontend.db.react :as react] [frontend.db.restore :as db-restore] - [frontend.handler.common.config-edn :as config-edn-common-handler] [frontend.handler.global-config :as global-config-handler] [frontend.handler.graph :as graph-handler] [frontend.handler.notification :as notification] @@ -27,7 +25,8 @@ [frontend.util :as util] [frontend.util.fs :as util-fs] [frontend.util.text :as text-util] - [promesa.core :as p])) + [promesa.core :as p] + [logseq.common.config :as common-config])) ;; Project settings should be checked in two situations: ;; 1. User changes the config.edn directly in logseq.com (fn: alter-file) @@ -177,16 +176,9 @@ (let [full-graph-name (string/lower-case (str config/db-version-prefix graph-name))] (some #(= (some-> (:url %) string/lower-case) full-graph-name) (state/get-repos)))) -(defn migrate-db-config - [content] - (-> (reduce rewrite/dissoc - (rewrite/parse-string (str content)) - (keys config-edn-common-handler/file-only-config)) - str)) - (defn- create-db [full-graph-name {:keys [file-graph-import?]}] (-> - (p/let [config (migrate-db-config config/config-default-content) + (p/let [config (common-config/create-config-for-db-graph config/config-default-content) _ (persist-db/ {:config config} file-graph-import? (assoc :import-type :file-graph))) diff --git a/src/main/frontend/modules/instrumentation/sentry.cljs b/src/main/frontend/modules/instrumentation/sentry.cljs index b75130f8be..3d3558da73 100644 --- a/src/main/frontend/modules/instrumentation/sentry.cljs +++ b/src/main/frontend/modules/instrumentation/sentry.cljs @@ -16,14 +16,14 @@ version) :environment (if config/dev? "development" "production") :initialScope {:tags - (merge - (when (not-empty config/revision) - {:revision config/revision}) + (cond-> {:platform (cond (util/electron?) "electron" (mobile-util/native-platform?) "mobile" :else "web") - :publishing config/publishing?})} + :publishing config/publishing?} + (not-empty config/revision) + (assoc :revision config/revision))} ;; :integrations [(new posthog/SentryIntegration posthog "logseq" 5311485) ;; (new BrowserTracing)] :debug config/dev? diff --git a/src/main/frontend/state.cljs b/src/main/frontend/state.cljs index de80543ec0..e8ce5cdbe5 100644 --- a/src/main/frontend/state.cljs +++ b/src/main/frontend/state.cljs @@ -2376,7 +2376,6 @@ Similar to re-frame subscriptions" (defn set-highlight-recent-days! [days] - (prn :debug :set :days days) (reset! (:ui/highlight-recent-days @state) days) (storage/set :ui/highlight-recent-days days)) diff --git a/src/main/frontend/worker/db/migrate.cljs b/src/main/frontend/worker/db/migrate.cljs index becefecf72..d81a866a5c 100644 --- a/src/main/frontend/worker/db/migrate.cljs +++ b/src/main/frontend/worker/db/migrate.cljs @@ -16,6 +16,7 @@ [logseq.db.common.order :as db-order] [logseq.db.frontend.class :as db-class] [logseq.db.frontend.content :as db-content] + [logseq.db.frontend.malli-schema :as db-malli-schema] [logseq.db.frontend.property :as db-property] [logseq.db.frontend.property.build :as db-property-build] [logseq.db.frontend.schema :as db-schema] @@ -1172,8 +1173,11 @@ (js/console.error e) (throw e))))))) -(defn- build-invalid-tx [entity eid] +(defn- build-invalid-tx [db entity eid] (cond + (nil? (db-malli-schema/entity-dispatch-key db entity)) + [[:db/retractEntity eid]] + (:block/schema entity) [[:db/retract eid :block/schema]] @@ -1200,7 +1204,8 @@ (= #{:block/tx-id} (set (keys entity))) [[:db/retractEntity (:db/id entity)]] - (and (seq (:block/refs entity)) + (and (or (seq (:block/refs entity)) + (:logseq.property.table/filters entity)) (not (or (:block/title entity) (:block/content entity) (:property.value/content entity)))) [[:db/retractEntity (:db/id entity)]] @@ -1285,7 +1290,7 @@ [:db/retract (:db/id entity) k])))))) (into {} entity)) eid (:db/id entity) - fix (build-invalid-tx entity eid)] + fix (build-invalid-tx db entity eid)] (into fix wrong-choice))) invalid-entity-ids) distinct)] diff --git a/src/main/frontend/worker/db_worker.cljs b/src/main/frontend/worker/db_worker.cljs index 5d7402115e..884ebed626 100644 --- a/src/main/frontend/worker/db_worker.cljs +++ b/src/main/frontend/worker/db_worker.cljs @@ -32,7 +32,6 @@ [frontend.worker.util :as worker-util] [goog.object :as gobj] [lambdaisland.glogi.console :as glogi-console] - [logseq.common.log :as log] [logseq.common.util :as common-util] [logseq.db :as ldb] [logseq.db.common.entity-plus :as entity-plus] @@ -101,38 +100,39 @@ [^js pool data] (.importDb ^js pool repo-path data)) -(defn- get-all-datoms-from-sqlite-db - [db] - (some->> (.exec db #js {:sql "select * from kvs" - :rowMode "array"}) - bean/->clj - (mapcat - (fn [[_addr content _addresses]] - (let [content' (sqlite-util/transit-read content) - datoms (when (map? content') - (:keys content'))] - datoms))) - distinct - (map (fn [[e a v t]] - (d/datom e a v t))))) +(comment + (defn- get-all-datoms-from-sqlite-db + [db] + (some->> (.exec db #js {:sql "select * from kvs" + :rowMode "array"}) + bean/->clj + (mapcat + (fn [[_addr content _addresses]] + (let [content' (sqlite-util/transit-read content) + datoms (when (map? content') + (:keys content'))] + datoms))) + distinct + (map (fn [[e a v t]] + (d/datom e a v t))))) -(defn- rebuild-db-from-datoms! - "Persistent-sorted-set has been broken, used addresses can't be found" - [datascript-conn sqlite-db import-type] - (let [datoms (get-all-datoms-from-sqlite-db sqlite-db) - db (d/init-db [] db-schema/schema - {:storage (storage/storage @datascript-conn)}) - db (d/db-with db - (map (fn [d] - [:db/add (:e d) (:a d) (:v d) (:t d)]) datoms))] - (prn :debug :rebuild-db-from-datoms :datoms-count (count datoms)) + (defn- rebuild-db-from-datoms! + "Persistent-sorted-set has been broken, used addresses can't be found" + [datascript-conn sqlite-db import-type] + (let [datoms (get-all-datoms-from-sqlite-db sqlite-db) + db (d/init-db [] db-schema/schema + {:storage (storage/storage @datascript-conn)}) + db (d/db-with db + (map (fn [d] + [:db/add (:e d) (:a d) (:v d) (:t d)]) datoms))] + (prn :debug :rebuild-db-from-datoms :datoms-count (count datoms)) ;; export db first - (when-not import-type - (worker-util/post-message :notification ["The SQLite db will be exported to avoid any data-loss." :warning false]) - (worker-util/post-message :export-current-db [])) - (.exec sqlite-db #js {:sql "delete from kvs"}) - (d/reset-conn! datascript-conn db) - (db-migrate/fix-db! datascript-conn))) + (when-not import-type + (worker-util/post-message :notification ["The SQLite db will be exported to avoid any data-loss." :warning false]) + (worker-util/post-message :export-current-db [])) + (.exec sqlite-db #js {:sql "delete from kvs"}) + (d/reset-conn! datascript-conn db) + (db-migrate/fix-db! datascript-conn)))) (comment (defn- gc-kvs-table! @@ -158,78 +158,74 @@ :bind #js [addr]})))))))) (defn- find-missing-addresses - [^Object db & {:keys [delete-addrs upsert-addr-content? open-db?]}] - (worker-util/profile - "find-missing-addresses" - (let [schema (some->> (.exec db #js {:sql "select content from kvs where addr = 0" - :rowMode "array"}) - bean/->clj - ffirst - sqlite-util/transit-read) - result (->> (.exec db #js {:sql "select addr, addresses from kvs" - :rowMode "array"}) - bean/->clj - (keep (fn [[addr addresses]] - (when-not (and delete-addrs (delete-addrs addr)) - [addr (bean/->clj (js/JSON.parse addresses))])))) - used-addresses (-> (set (concat (mapcat second result) - [0 1 (:eavt schema) (:avet schema) (:aevt schema)])) - (clojure.set/difference delete-addrs)) - missing-addresses (clojure.set/difference used-addresses (set (map first result)))] - (when (seq missing-addresses) - (prn :error :missing-addresses missing-addresses) - (if worker-util/dev? - (throw (ex-info "Found missing addresses that shouldn't happen" {:missing-addresses missing-addresses})) - (worker-util/post-message :capture-error - {:error "v2-db-missing-addresses" - :payload {:missing-addresses missing-addresses - :upsert-addr-content upsert-addr-content? - :open-db open-db?}}))) - missing-addresses))) + [conn ^Object db & {:keys [delete-addrs]}] + (let [schema (some->> (.exec db #js {:sql "select content from kvs where addr = 0" + :rowMode "array"}) + bean/->clj + ffirst + sqlite-util/transit-read) + result (->> (.exec db #js {:sql "select addr, addresses from kvs" + :rowMode "array"}) + bean/->clj + (keep (fn [[addr addresses]] + (when-not (and delete-addrs (delete-addrs addr)) + [addr (bean/->clj (js/JSON.parse addresses))])))) + used-addresses (-> (set (concat (mapcat second result) + [0 1 (:eavt schema) (:avet schema) (:aevt schema)])) + (clojure.set/difference delete-addrs)) + missing-addresses (clojure.set/difference used-addresses (set (map first result)))] + (when (seq missing-addresses) + (let [version-in-db (when conn (db-schema/parse-schema-version (or (:kv/value (d/entity @conn :logseq.kv/schema-version)) 0))) + compare-result (when version-in-db (db-schema/compare-schema-version version-in-db "64.8"))] + (when (and compare-result (not (neg? compare-result))) ; >= 64.8 + (worker-util/post-message :capture-error + {:error "db-missing-addresses-v2" + :payload {:missing-addresses missing-addresses}})))) + missing-addresses)) (defn upsert-addr-content! "Upsert addr+data-seq. Update sqlite-cli/upsert-addr-content! when making changes" - [repo data delete-addrs* & {:keys [client-ops-db?] :or {client-ops-db? false}}] - (let [^Object db (worker-state/get-sqlite-conn repo (if client-ops-db? :client-ops :db)) - delete-addrs (set delete-addrs*)] + [db data delete-addrs*] + (let [delete-addrs (clojure.set/difference (set delete-addrs*) #{0 1})] (assert (some? db) "sqlite db not exists") (.transaction db (fn [tx] (doseq [item data] (.exec tx #js {:sql "INSERT INTO kvs (addr, content, addresses) values ($addr, $content, $addresses) on conflict(addr) do update set content = $content, addresses = $addresses" :bind item})))) (when (seq delete-addrs) + (.transaction db (fn [tx] + (doseq [addr delete-addrs] + (.exec tx #js {:sql "Delete from kvs WHERE addr = ? AND NOT EXISTS (SELECT 1 FROM json_each(addresses) WHERE value = ?);" + :bind #js [addr]})))) (let [missing-addrs (when worker-util/dev? - (seq (find-missing-addresses db {:delete-addrs delete-addrs - :upsert-addr-content? true}))) - delete-addrs' (if missing-addrs - (remove (set missing-addrs) delete-addrs) - delete-addrs)] - (when (seq delete-addrs') - (.transaction db (fn [tx] - (doseq [addr delete-addrs'] - (.exec tx #js {:sql "Delete from kvs WHERE addr = ? AND NOT EXISTS (SELECT 1 FROM json_each(addresses) WHERE value = ?);" - :bind #js [addr]}))))))))) + (seq (find-missing-addresses nil db {:delete-addrs delete-addrs})))] + (if (seq missing-addrs) + (worker-util/post-message :notification [(str "Bug!! Missing addresses: " missing-addrs) :error false]) + (when (seq delete-addrs) + (.transaction db (fn [tx] + (doseq [addr delete-addrs] + (.exec tx #js {:sql "Delete from kvs WHERE addr = ? AND NOT EXISTS (SELECT 1 FROM json_each(addresses) WHERE value = ?);" + :bind #js [addr]})))))))))) (defn restore-data-from-addr "Update sqlite-cli/restore-data-from-addr when making changes" - [repo addr & {:keys [client-ops-db?] :or {client-ops-db? false}}] - (let [^Object db (worker-state/get-sqlite-conn repo (if client-ops-db? :client-ops :db))] - (assert (some? db) "sqlite db not exists") - (when-let [result (-> (.exec db #js {:sql "select content, addresses from kvs where addr = ?" - :bind #js [addr] - :rowMode "array"}) - first)] - (let [[content addresses] (bean/->clj result) - addresses (when addresses - (js/JSON.parse addresses)) - data (sqlite-util/transit-read content)] - (if (and addresses (map? data)) - (assoc data :addresses addresses) - data))))) + [db addr] + (assert (some? db) "sqlite db not exists") + (when-let [result (-> (.exec db #js {:sql "select content, addresses from kvs where addr = ?" + :bind #js [addr] + :rowMode "array"}) + first)] + (let [[content addresses] (bean/->clj result) + addresses (when addresses + (js/JSON.parse addresses)) + data (sqlite-util/transit-read content)] + (if (and addresses (map? data)) + (assoc data :addresses addresses) + data)))) (defn new-sqlite-storage "Update sqlite-cli/new-sqlite-storage when making changes" - [repo _opts] + [^Object db] (reify IStorage (-store [_ addr+data-seq delete-addrs] (let [used-addrs (set (mapcat @@ -249,36 +245,10 @@ :$content (sqlite-util/transit-write data') :$addresses addresses})) addr+data-seq)] - (upsert-addr-content! repo data delete-addrs))) + (upsert-addr-content! db data delete-addrs))) (-restore [_ addr] - (restore-data-from-addr repo addr)))) - -(defn new-sqlite-client-ops-storage - [repo] - (reify IStorage - (-store [_ addr+data-seq delete-addrs] - (let [used-addrs (set (mapcat - (fn [[addr data]] - (cons addr - (when (map? data) - (:addresses data)))) - addr+data-seq)) - delete-addrs (remove used-addrs delete-addrs) - data (map - (fn [[addr data]] - (let [data' (if (map? data) (dissoc data :addresses) data) - addresses (when (map? data) - (when-let [addresses (:addresses data)] - (js/JSON.stringify (bean/->js addresses))))] - #js {:$addr addr - :$content (sqlite-util/transit-write data') - :$addresses addresses})) - addr+data-seq)] - (upsert-addr-content! repo data delete-addrs :client-ops-db? true))) - - (-restore [_ addr] - (restore-data-from-addr repo addr :client-ops-db? true)))) + (restore-data-from-addr db addr)))) (defn- close-db-aux! [repo ^Object db ^Object search ^Object client-ops] @@ -338,8 +308,9 @@ [repo {:keys [config import-type datoms]}] (when-not (worker-state/get-sqlite-conn repo) (p/let [[db search-db client-ops-db :as dbs] (get-dbs repo) - storage (new-sqlite-storage repo {}) - client-ops-storage (when-not @*publishing? (new-sqlite-client-ops-storage repo)) + storage (new-sqlite-storage db) + client-ops-storage (when-not @*publishing? + (new-sqlite-storage client-ops-db)) db-based? (sqlite-util/db-based-graph? repo)] (swap! *sqlite-conns assoc repo {:db db :search search-db @@ -374,24 +345,14 @@ (when import-type {:import-type import-type}))] (d/transact! conn initial-data {:initial-db? true}))) - (try + ;; TODO: remove this once we can ensure there's no bug for missing addresses + ;; because it's slow for large graphs + (when-not import-type + (when-let [missing-addresses (seq (find-missing-addresses conn db))] + (worker-util/post-message :notification ["It seems that the DB has been broken, please export a backup and contact Logseq team for help." :error false]) + (throw (ex-info "DB missing addresses" {:missing-addresses missing-addresses})))) - ;; TODO: remove this once we can ensure there's no bug for missing addresses - ;; because it's slow for large graphs - (when-not import-type - (when-let [missing-addresses (seq (find-missing-addresses db {:open-db? true}))] - (throw (ex-info "DB missing addresses" {:missing-addresses missing-addresses})))) - - (db-migrate/migrate conn search-db) - - ;; TODO: Remove this once we can ensure there's no bug for missing addresses - (catch :default e - (log/error (str "DB migrate failed for " repo ", error:") e) - (if (= (.-message e) "DB missing addresses") - (do - (rebuild-db-from-datoms! conn db import-type) - (db-migrate/migrate conn search-db)) - (throw e)))) + (db-migrate/migrate conn search-db) (db-listener/listen-db-changes! repo (get @*datascript-conns repo)))))) diff --git a/src/main/frontend/worker/pipeline.cljs b/src/main/frontend/worker/pipeline.cljs index 08c1bb788e..c3374e57da 100644 --- a/src/main/frontend/worker/pipeline.cljs +++ b/src/main/frontend/worker/pipeline.cljs @@ -107,9 +107,11 @@ (let [valid? (if (get-in tx-report [:tx-meta :reset-conn!]) true (db-validate/validate-tx-report! tx-report (:validate-db-options context)))] - (when (and (get-in context [:validate-db-options :fail-invalid?]) (not valid?)) - (shared-service/broadcast-to-clients! :notification - [["Invalid DB!"] :error])))) + (when-not valid? + (when (or (get-in context [:validate-db-options :fail-invalid?]) worker-util/dev?) + (shared-service/broadcast-to-clients! :notification + [["Invalid DB!"] :error])) + (throw (ex-info "Invalid data" {:graph repo}))))) ;; Ensure :block/order is unique for any block that has :block/parent (when (or (:dev? context) (exists? js/process)) diff --git a/src/main/frontend/worker/shared_service.cljs b/src/main/frontend/worker/shared_service.cljs index b330a28c0d..bd7b40e70e 100644 --- a/src/main/frontend/worker/shared_service.cljs +++ b/src/main/frontend/worker/shared_service.cljs @@ -294,10 +294,13 @@ (.postMessage common-channel #js {:type "master-changed" :master-client-id master-client-id :serviceName service-name}) - (p/do! - (on-become-master-handler service-name) - ( + (p/do! + (on-become-master-handler service-name) + (