Merge branch 'feat/db' into feat/text-template

This commit is contained in:
Tienson Qin
2025-03-08 07:03:02 +08:00
committed by GitHub
22 changed files with 175 additions and 140 deletions

View File

@@ -175,7 +175,7 @@ jobs:
run: cd deps/db && yarn install --frozen-lockfile
- name: Validate created DB graphs
run: cd deps/db && yarn nbb-logseq script/validate_client_db.cljs ../../scripts/db-graph-with-props ../../scripts/db-graph-with-schema --closed-maps --group-errors
run: cd deps/db && yarn nbb-logseq script/validate_db.cljs ../../scripts/db-graph-with-props ../../scripts/db-graph-with-schema --closed-maps --group-errors
e2e-test:
# TODO: Re-enable when ready to enable tests for file graphs

4
bb.edn
View File

@@ -64,7 +64,7 @@
{:doc "Validate a DB graph's datascript schema"
:requires ([babashka.fs :as fs])
:task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
"yarn -s nbb-logseq script/validate_client_db.cljs"
"yarn -s nbb-logseq script/validate_db.cljs"
*command-line-args*)}
dev:db-query
@@ -81,7 +81,7 @@
{:doc "Create a DB graph given a sqlite.build EDN file"
:requires ([babashka.fs :as fs])
:task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
"yarn -s nbb-logseq -cp src:../outliner/src script/create_graph.cljs" *command-line-args*)}
"yarn -s nbb-logseq -cp src:../outliner/src:script script/create_graph.cljs" *command-line-args*)}
dev:db-import
{:doc "Import a file graph to db graph"

View File

@@ -1,12 +1,15 @@
(ns create-graph
"An example script that creates a DB graph given a sqlite.build EDN file"
(:require [logseq.outliner.cli :as outliner-cli]
[clojure.string :as string]
[clojure.edn :as edn]
[datascript.core :as d]
["path" :as node-path]
"A script that creates a DB graph given a sqlite.build EDN file"
(:require ["fs" :as fs]
["os" :as os]
["fs" :as fs]
["path" :as node-path]
[babashka.cli :as cli]
[clojure.edn :as edn]
[clojure.string :as string]
[datascript.core :as d]
#_:clj-kondo/ignore
[logseq.outliner.cli :as outliner-cli]
[validate-db]
[nbb.classpath :as cp]
[nbb.core :as nbb]))
@@ -17,11 +20,20 @@
path
(node-path/join (or js/process.env.ORIGINAL_PWD ".") path)))
(def spec
"Options spec"
{:help {:alias :h
:desc "Print help"}
:validate {:alias :v
:desc "Validate db after creation"}})
(defn -main [args]
(when (not= 2 (count args))
(println "Usage: $0 GRAPH-DIR EDN-PATH")
(js/process.exit 1))
(let [[graph-dir edn-path] args
(let [{options :opts args' :args} (cli/parse-args args {:spec spec})
[graph-dir edn-path] args'
_ (when (or (nil? graph-dir) (nil? edn-path) (:help options))
(println (str "Usage: $0 GRAPH-NAME EDN-PATH [OPTIONS]\nOptions:\n"
(cli/format-opts {:spec spec})))
(js/process.exit 1))
[dir db-name] (if (string/includes? graph-dir "/")
((juxt node-path/dirname node-path/basename) graph-dir)
[(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
@@ -34,7 +46,9 @@
;; (cljs.pprint/pprint _txs)
(d/transact! conn init-tx)
(d/transact! conn block-props-tx)
(println "Created graph" (str db-name "!"))))
(println "Created graph" (str db-name "!"))
(when (:validate options)
(validate-db/validate-db @conn db-name {:group-errors true :closed-maps true :humanize true}))))
(when (= nbb/*file* (:file (meta #'-main)))
(when (= nbb/*file* (nbb/invoked-file))
(-main *command-line-args*))

View File

@@ -1,5 +1,5 @@
(ns dump-datoms
"An example script that dumps all eavt datoms to a specified edn file
"A script that dumps all eavt datoms to a specified edn file
$ yarn -s nbb-logseq script/dump_datoms.cljs db-name datoms.edn"
(:require [datascript.core :as d]
@@ -28,5 +28,5 @@
(println "Writing" (count datoms) "datoms to" file)
(fs/writeFileSync file (with-out-str (pprint/pprint datoms)))))
(when (= nbb/*file* (:file (meta #'-main)))
(when (= nbb/*file* (nbb/invoked-file))
(-main *command-line-args*))

View File

@@ -1,5 +1,5 @@
(ns query
"An example script that queries any db graph from the commandline e.g.
"A script that queries any db graph from the commandline e.g.
$ yarn -s nbb-logseq script/query.cljs db-name '[:find (pull ?b [:block/name :block/title]) :where [?b :block/created-at]]'"
(:require [datascript.core :as d]
@@ -44,8 +44,8 @@
:desc "Lookup entities instead of query"}})
(defn -main [args]
(let [[graph-dir & args'] args
options (cli/parse-opts args' {:spec spec})
(let [{options :opts args' :args} (cli/parse-args args {:spec spec})
[graph-dir & args''] args'
_ (when (or (nil? graph-dir) (:help options))
(println (str "Usage: $0 GRAPH-NAME [& ARGS] [OPTIONS]\nOptions:\n"
(cli/format-opts {:spec spec})))
@@ -60,7 +60,7 @@
(update :block/properties (fn [props] (map (fn [m] (into {} m)) props)))))
(:entity options))
;; assumes no :in are in queries
(let [query (into (edn/read-string (first args')) [:in '$ '%])
(let [query (into (edn/read-string (first args'')) [:in '$ '%])
res (d/q query @conn (rules/extract-rules rules/db-query-dsl-rules))]
;; Remove nesting for most queries which just have one :find binding
(if (= 1 (count (first res))) (mapv first res) res)))]
@@ -71,5 +71,5 @@
(sh ["puget"] {:input (pr-str results) :stdio ["pipe" "inherit" "inherit"]})
(pprint/pprint results)))))
(when (= nbb/*file* (:file (meta #'-main)))
(when (= nbb/*file* (nbb/invoked-file))
(-main *command-line-args*))

View File

@@ -1,4 +1,4 @@
(ns validate-client-db
(ns validate-db
"Script that validates the datascript db of a DB graph
NOTE: This script is also used in CI to confirm our db's schema is up to date"
(:require ["os" :as os]
@@ -14,7 +14,7 @@
[malli.error :as me]
[nbb.core :as nbb]))
(defn validate-client-db
(defn validate-db*
"Validate datascript db as a vec of entity maps"
[db ent-maps* {:keys [verbose group-errors humanize closed-maps]}]
(let [ent-maps (db-malli-schema/update-properties-in-ents db ent-maps*)
@@ -66,6 +66,14 @@
:default true
:desc "Groups errors by their entity id"}})
(defn validate-db [db db-name options]
(let [datoms (d/datoms db :eavt)
ent-maps (db-malli-schema/datoms->entities datoms)]
(println "Read graph" (str db-name " with counts: "
(pr-str (assoc (db-validate/graph-counts db ent-maps)
:datoms (count datoms)))))
(validate-db* db ent-maps options)))
(defn- validate-graph [graph-dir options]
(let [[dir db-name] (if (string/includes? graph-dir "/")
(let [graph-dir'
@@ -75,13 +83,8 @@
conn (try (sqlite-cli/open-db! dir db-name)
(catch :default e
(println "Error: For graph" (str (pr-str graph-dir) ":") (str e))
(js/process.exit 1)))
datoms (d/datoms @conn :eavt)
ent-maps (db-malli-schema/datoms->entities datoms)]
(println "Read graph" (str db-name " with counts: "
(pr-str (assoc (db-validate/graph-counts @conn ent-maps)
:datoms (count datoms)))))
(validate-client-db @conn ent-maps options)))
(js/process.exit 1)))]
(validate-db @conn db-name options)))
(defn -main [argv]
(let [{:keys [args opts]} (cli/parse-args argv {:spec spec})
@@ -92,5 +95,5 @@
(doseq [graph-dir args]
(validate-graph graph-dir opts))))
(when (= nbb/*file* (:file (meta #'-main)))
(when (= nbb/*file* (nbb/invoked-file))
(-main *command-line-args*))

View File

@@ -153,7 +153,9 @@
:schema {:type :node
:public? true
:view-context :page}
:queryable? true}
:queryable? true
:properties
{:logseq.property/description "Provides parent-child relationships between nodes. For tags this enables inheritance and for pages this enables namespaces."}}
:logseq.property/default-value {:title "Default value"
:schema {:type :entity
:public? false
@@ -172,7 +174,9 @@
:logseq.property/hide-empty-value {:title "Hide empty value"
:schema {:type :checkbox
:public? true
:view-context :property}}
:view-context :property}
:properties
{:logseq.property/description "Hides a property's value on any node when empty e.g. when a property appears on a node through a tag."}}
:logseq.property.class/hide-from-node {:title "Hide from Node"
:schema {:type :checkbox
:public? true
@@ -186,7 +190,9 @@
:schema {:type :page
:public? true
:view-context :page
:cardinality :many}}
:cardinality :many}
:properties
{:logseq.property/description "Provides a way for a page to associate to another page i.e. backward compatible tagging."}}
:logseq.property/background-color {:title "Background color"
:schema {:type :default :hide? true}}
:logseq.property/background-image {:title "Background image"
@@ -548,7 +554,9 @@
:logseq.property/enable-history? {:title "Enable property history"
:schema {:type :checkbox
:public? true
:view-context :property}}
:view-context :property}
:properties
{:logseq.property/description "Records history anytime a property's value changes on a node."}}
:logseq.property.history/block {:title "History block"
:schema {:type :entity
:hide? true}}

View File

@@ -183,14 +183,14 @@
{:property-attributes
(merge {:db/id (or (property-db-ids prop-name)
(throw (ex-info "No :db/id for property" {:property prop-name})))}
(select-keys prop-m [:build/properties-ref-types]))}))
(select-keys prop-m [:build/properties-ref-types :block/created-at :block/updated-at]))}))
[(merge (sqlite-util/build-new-property (get-ident all-idents prop-name)
(db-property/get-property-schema prop-m)
{:block-uuid (:block/uuid prop-m)
:title (:block/title prop-m)})
{:db/id (or (property-db-ids prop-name)
(throw (ex-info "No :db/id for property" {:property prop-name})))}
(select-keys prop-m [:build/properties-ref-types]))])
(select-keys prop-m [:build/properties-ref-types :block/created-at :block/updated-at]))])
pvalue-tx-m
(->property-value-tx-m new-block (:build/properties prop-m) properties all-idents)]
(cond-> []
@@ -415,8 +415,8 @@
(vec
(mapcat
(fn [{:keys [page blocks]}]
(let [ignore-page-tx? (and build-existing-tx? (not (::new-page? (meta page))) (not (:build/keep-uuid? page)))
page' (if ignore-page-tx?
(let [build-existing-tx?' (and build-existing-tx? (not (::new-page? (meta page))) (not (:build/keep-uuid? page)))
page' (if build-existing-tx?'
page
(merge
;; TODO: Use sqlite-util/build-new-page
@@ -430,8 +430,8 @@
page-id-fn)]
(into
;; page tx
(if ignore-page-tx?
[]
(if build-existing-tx?'
[(select-keys page [:block/uuid :block/created-at :block/updated-at])]
(build-page-tx page' all-idents page-uuids properties))
;; blocks tx
(reduce (fn [acc m]
@@ -707,8 +707,8 @@
See auto-create-ontology for more details
* :build-existing-tx? - When set to true, blocks, pages, properties and classes with :block/uuid are treated as
existing in DB and are skipped for creation. This is useful for building tx on existing DBs e.g. for importing.
Blocks are updated with any attributes passed to it while all other node types are ignored for update unless
:build/keep-uuid? is set.
Blocks and pages are updated with any attributes passed to it while all other node types are ignored for update
unless :build/keep-uuid? is set.
* :page-id-fn - custom fn that returns ent lookup id for page refs e.g. `[:block/uuid X]`
Default is :db/id

View File

@@ -105,7 +105,9 @@
as built-in?. Returns their tx data as well as data needed for subsequent build steps"
[]
;; bootstrap-idents must either be necessary to define a property or be used on every property
(let [bootstrap-idents #{:logseq.property/type :logseq.property/hide? :logseq.property/built-in?}
(let [bootstrap-idents #{:logseq.property/type :logseq.property/hide? :logseq.property/built-in?
;; Required to define :properties on a property
:logseq.property/created-from-property}
bootstrap-properties (map build-bootstrap-property bootstrap-idents)
;; First tx bootstrap properties so they can take affect. Then tx the bootstrap properties on themselves
bootstrap-properties-tx (into (mapv #(apply dissoc % bootstrap-idents) bootstrap-properties)

View File

@@ -492,11 +492,12 @@
(mapcat #(sqlite-build/extract-from-blocks (:blocks %) (fn [m] (some-> m :block/uuid vector)))
pages-and-blocks))
set)
;; only looks one-level deep in properties e.g. not inside :build/page
ref-uuids
(->> (concat (mapcat (comp get-pvalue-uuids :build/properties) (vals classes))
(mapcat (comp get-pvalue-uuids :build/properties) (vals properties))
(mapcat (comp get-pvalue-uuids :build/properties :page) pages-and-blocks)
(mapcat #(sqlite-build/extract-from-blocks (:blocks %) (comp get-pvalue-uuids :build/properties)) pages-and-blocks))
(->> (concat (mapcat get-pvalue-uuids (vals classes))
(mapcat get-pvalue-uuids (vals properties))
(mapcat (comp get-pvalue-uuids :page) pages-and-blocks)
(mapcat #(sqlite-build/extract-from-blocks (:blocks %) get-pvalue-uuids) pages-and-blocks))
set)]
(set/difference ref-uuids known-uuids)))

View File

@@ -189,5 +189,5 @@
(when (:verbose options') (println "Transacted" (count (d/datoms @conn :eavt)) "datoms"))
(println "Created graph" (str db-name "!")))))
(when (= nbb/*file* (:file (meta #'-main)))
(when (= nbb/*file* (nbb/invoked-file))
(-main *command-line-args*))

View File

@@ -37,5 +37,5 @@
(d/transact! conn update-tx)
(println "Updated" (count update-tx) "block(s) for graph" (str db-name "!"))))))
(when (= nbb/*file* (:file (meta #'-main)))
(when (= nbb/*file* (nbb/invoked-file))
(-main *command-line-args*))

View File

@@ -60,5 +60,5 @@
(publish-db-graph static-dir graph-dir output-path options)
(publish-file-graph static-dir graph-dir output-path options))))
(when (= nbb/*file* (:file (meta #'-main)))
(when (= nbb/*file* (nbb/invoked-file))
(-main *command-line-args*))

View File

@@ -122,14 +122,13 @@
([] (when-let [id (some-> (get-popups) (last) :id)] (hide! id 0)))
([id] (hide! id 0 {}))
([id delay] (hide! id delay {}))
([id delay {:keys [all?]}]
([id delay {:keys [_all?]}]
(when-let [popup (get-popup id)]
(let [config (last popup)
target (:target config)
f #(if all?
(reset! *popups [])
(do (detach-popup! id)
(some-> (:on-after-hide config) (apply []))))]
f (fn []
(detach-popup! id)
(some-> (:on-after-hide config) (apply [])))]
(some-> (:on-before-hide config) (apply []))
(some-> target (d/remove-attr! "data-popup-active"))
(if (and (number? delay) (> delay 0))
@@ -146,20 +145,6 @@
auto-side? _auto-focus? _target root-props content-props
_on-before-hide _on-after-hide]
:as _props}]
;; disableOutsidePointerEvents
;(rum/use-effect!
; (fn []
; (when-not as-dropdown?
; (let [^js style js/document.body.style
; set-pointer-event! #(set! (. style -pointerEvents) %)
; try-unset! #(when (nil? (seq @*popups))
; (set-pointer-event! nil))]
; (if open?
; (set-pointer-event! "none")
; (try-unset!))
; #(try-unset!))))
; [open?])
(when-let [[x y _ height] position]
(let [popup-root (if (not force-popover?) dropdown-menu popover)
popup-trigger (if (not force-popover?) dropdown-menu-trigger popover-trigger)
@@ -174,7 +159,12 @@
auto-side? (if (boolean? auto-side?) auto-side? true)
content-props (cond-> content-props
auto-side? (assoc :side (auto-side-fn)))
hide (fn [] (hide! id 1))]
handle-key-escape! (fn [^js e]
(when-not (false? (some-> content-props (:onEscapeKeyDown) (apply [e])))
(hide! id 1)))
handle-pointer-outside! (fn [^js e]
(when-not (false? (some-> content-props (:onPointerDownOutside) (apply [e])))
(hide! id 1)))]
(popup-root
(merge root-props {:open open?})
(popup-trigger
@@ -186,10 +176,9 @@
:width 1
:top y
:left x}} ""))
(let [content-props (cond-> (merge {:onEscapeKeyDown hide
:disableOutsideScroll false
:onPointerDownOutside hide}
content-props)
(let [content-props (cond-> (merge content-props {:onEscapeKeyDown handle-key-escape!
:disableOutsideScroll false
:onPointerDownOutside handle-pointer-outside!})
(and (not force-popover?)
(not as-dropdown?))
(assoc :on-key-down (fn [^js e]

View File

@@ -284,11 +284,11 @@ div[data-radix-popper-content-wrapper] {
@apply overflow-y-auto;
&[data-side=top] {
max-height: calc(var(--radix-dropdown-menu-content-available-height) - 40px);
max-height: calc(var(--radix-dropdown-menu-content-available-height) - 20px);
}
&[data-side=bottom] {
max-height: calc(var(--radix-dropdown-menu-content-available-height) - 20px);
max-height: calc(var(--radix-dropdown-menu-content-available-height) - 10px);
}
&.text-popover-foreground {

View File

@@ -83,5 +83,5 @@
#_(d/transact! conn blocks-tx)
(println "Created graph" (str db-name " with " (count (d/datoms @conn :eavt)) " datoms!"))))
(when (= nbb/*file* (:file (meta #'-main)))
(when (= nbb/*file* (nbb/invoked-file))
(-main *command-line-args*))

View File

@@ -222,5 +222,5 @@
(d/transact! conn block-props-tx)
(println "Created graph" (str db-name " with " (count (d/datoms @conn :eavt)) " datoms!"))))
(when (= nbb/*file* (:file (meta #'-main)))
(when (= nbb/*file* (nbb/invoked-file))
(-main *command-line-args*))

View File

@@ -423,5 +423,5 @@
(when (:debug options) (write-debug-file @conn))
(println "Created graph" (str db-name "!"))))
(when (= nbb/*file* (:file (meta #'-main)))
(when (= nbb/*file* (nbb/invoked-file))
(-main *command-line-args*))

View File

@@ -245,7 +245,11 @@
@apply w-full inline;
> .ui__checkbox {
@apply relative top-[2px];
@apply relative top-0.5;
&.checked {
@apply top-1;
}
> span {
@apply h-full;

View File

@@ -661,61 +661,63 @@
(rum/defc shui-editor-popups
[id format action _data]
(hooks/use-effect!
(fn []
(let [pid (case action
:commands
(open-editor-popup! :commands
(commands id format)
{:content-props {:withoutAnimation false}})
(fn []
(let [pid (case action
:commands
(open-editor-popup! :commands
(commands id format)
{:content-props {:withoutAnimation false}})
(:block-search :page-search :page-search-hashtag)
(open-editor-popup! action
(if (= :block-search action)
(block-search id format)
(page-search id format))
{:root-props {:onOpenChange
#(when-not %
(when (contains?
#{:block-search :page-search :page-search-hashtag}
(state/get-editor-action))
(state/clear-editor-action!)))}})
(:block-search :page-search :page-search-hashtag)
(open-editor-popup! action
(if (= :block-search action)
(block-search id format)
(page-search id format))
{:root-props {:onOpenChange
#(when-not %
(when (contains?
#{:block-search :page-search :page-search-hashtag}
(state/get-editor-action))
(state/clear-editor-action!)))}})
:datepicker
(open-editor-popup! :datepicker
(datetime-comp/date-picker id format nil) {})
:datepicker
(open-editor-popup! :datepicker
(datetime-comp/date-picker id format nil) {})
:input
(open-editor-popup! :input
(editor-input id
(fn [command m]
(editor-handler/handle-command-input command id format m))
(fn []
(editor-handler/handle-command-input-close id)))
{:content-props {:onOpenAutoFocus #()}})
:input
(open-editor-popup! :input
(editor-input id
;; on-submit
(fn [command m]
(editor-handler/handle-command-input command id format m))
;; on-cancel
(fn []
(editor-handler/handle-command-input-close id)))
{:content-props {:onOpenAutoFocus #()}})
:select-code-block-mode
(open-editor-popup! :code-block-mode-picker
(code-block-mode-picker id format) {})
:select-code-block-mode
(open-editor-popup! :code-block-mode-picker
(code-block-mode-picker id format) {})
:template-search
(open-editor-popup! :template-search
(template-search id format) {})
:template-search
(open-editor-popup! :template-search
(template-search id format) {})
(:property-search :property-value-search)
(open-editor-popup! action
(if (= :property-search action)
(property-search id) (property-value-search id))
{})
(:property-search :property-value-search)
(open-editor-popup! action
(if (= :property-search action)
(property-search id) (property-value-search id))
{})
:zotero
(open-editor-popup! :zotero
(zotero/zotero-search id) {})
:zotero
(open-editor-popup! :zotero
(zotero/zotero-search id) {})
;; TODO: try remove local model state
false)]
#(when pid
(shui/popup-hide! pid))))
[action])
false)]
#(when pid
(shui/popup-hide! pid))))
[action])
[:<>])
(rum/defc command-popups <

View File

@@ -79,7 +79,9 @@
(db-property-handler/remove-block-property!
(:db/id block)
:logseq.property/icon))
(clear-overlay!))]
(clear-overlay!)
(when editing?
(editor-handler/restore-last-saved-cursor!)))]
(hooks/use-effect!
(fn []
@@ -92,13 +94,16 @@
(fn []
(when-let [^js target (some-> (.querySelector container (str "#ls-block-" (str (:block/uuid block))))
(.querySelector ".block-main-container"))]
(state/set-editor-action! :property-icon-picker)
(shui/popup-show! target
#(icon-component/icon-search
{:on-chosen on-chosen!
:icon-value icon
:del-btn? (some? icon)})
{:id :ls-icon-picker
:align :start})))))))
#(icon-component/icon-search
{:on-chosen on-chosen!
:icon-value icon
:del-btn? (some? icon)})
{:id :ls-icon-picker
:on-after-hide #(state/set-editor-action! nil)
:content-props {:onEscapeKeyDown #(when editing? (editor-handler/restore-last-saved-cursor!))}
:align :start})))))))
[editing?])
[:div.col-span-3.flex.flex-row.items-center.gap-2
@@ -1271,10 +1276,11 @@
(or (= (:db/id v) (:db/id block))
;; property value self embedded
(= (:db/id (:block/link v)) (:db/id block))))]
(if (or (and (de/entity? v) (self-value-or-embedded? v))
(and (coll? v) (every? de/entity? v)
(some self-value-or-embedded? v))
(and (= p-block (:db/id block)) (= p-property (:db/id property))))
(if (and (or (and (de/entity? v) (self-value-or-embedded? v))
(and (coll? v) (every? de/entity? v)
(some self-value-or-embedded? v))
(and (= p-block (:db/id block)) (= p-property (:db/id property))))
(not= :logseq.class/Tag (:db/ident block)))
[:div.flex.flex-row.items-center.gap-1
[:div.warning "Self reference"]
(shui/button {:variant :outline

View File

@@ -1895,6 +1895,12 @@
(handle-command-input-close id))
(defn restore-last-saved-cursor!
([] (restore-last-saved-cursor! (state/get-input)))
([input]
(when-let [saved-cursor (and input (state/get-editor-last-pos))]
(cursor/move-cursor-to input saved-cursor true))))
(defn- close-autocomplete-if-outside
[input]
(when (and input