enhance: batch update nodes tool

This improves and replaces all previous tools. Features include:
* Add pages, blocks to pages, tags and properties
* Tags can set parents and tag properties
* Properties can have type, cardinality and classes set for :node
* Add tags to blocks
* Edit blocks
* api and local tools work
* Thorough tool validation
* When doing these operations, most operations can reference new or existing
  entities if they are referenced by uuid

deps/cli/bb.edn

@@ -40,5 +40,5 @@
 
  :tasks/config
  {:large-vars
-  {:max-lines-count 45
+  {:max-lines-count 50
    :metadata-exceptions #{:large-vars/cleanup-todo}}}}

deps/cli/src/logseq/cli/commands/mcp_server.cljs

@@ -24,12 +24,21 @@
 (defn- local-list-tags [conn _args]
   (cli-common-mcp-server/mcp-success-response (cli-common-mcp-tools/list-tags @conn)))
 
+(defn- local-upsert-nodes [conn args]
+  (cli-common-mcp-server/mcp-success-response
+   (cli-common-mcp-tools/upsert-nodes
+    conn
+    ;; string is used by a -t invocation
+    (-> (if (string? (.-operations args)) (js/JSON.parse (.-operations args)) (.-operations args))
+        (js->clj :keywordize-keys true)))))
+
 (def ^:private local-tools
   "MCP Tools when running with a local graph"
   (let [tools {:getPage {:fn local-get-page}
                :listPages {:fn local-list-pages}
                :listProperties {:fn local-list-properties}
-               :listTags {:fn local-list-tags}}]
+               :listTags {:fn local-list-tags}
+               :upsertNodes {:fn local-upsert-nodes}}]
     (merge-with
      merge
      (select-keys cli-common-mcp-server/api-tools (keys tools))

deps/cli/src/logseq/cli/common/mcp/server.cljs

@@ -16,7 +16,7 @@
 ;; for how to respond to different MCP requests
 (defn handle-post-request [mcp-server {:keys [port host]} req res]
   (let [session-id (aget (.-headers req) "mcp-session-id")]
-    (js/console.log "POST /mcp request" session-id (.-body req))
+    (js/console.log "POST /mcp request" session-id (pr-str (.-body req)))
     (cond
       (and session-id (@transports session-id))
       (let [^js transport (@transports session-id)]
@@ -131,7 +131,11 @@
   [call-api-fn args]
   (call-api-fn "logseq.app.search" [(aget args "searchTerm") #js {:enable-snippet? false}]))
 
-(def api-tools
+(defn- api-upsert-nodes
+  [call-api-fn args]
+  (call-api-fn "logseq.cli.upsertNodes" [(aget args "operations")]))
+
+(def ^:large-vars/data-var api-tools
   "MCP Tools when calling API server"
   {:listPages
    {:fn api-list-pages
@@ -140,7 +144,7 @@
    :getPage
    {:fn api-get-page
     :config #js {:title "Get Page"
-                 :description "Get a page's content including its blocks"
+                 :description "Get a page's content including its blocks. A property and a tag are pages."
                  :inputSchema #js {:pageName (-> (z/string) (.describe "The page's name or uuid"))}}}
    :addToPage
    {:fn api-add-to-page
@@ -157,6 +161,67 @@
                  :description "Update block with new content"
                  :inputSchema #js {:blockUUID (z/string)
                                    :content (-> (z/string) (.describe "Block content"))}}}
+   :upsertNodes
+   {:fn api-upsert-nodes
+    :config
+    #js {:title "Upsert Nodes"
+         :description
+         "Takes an object with field :operations, which is an array of operation objects.
+          Each operation creates or edits a page, block, tag or property. Each operation is a object
+          that must have :operation, :entityType and :data fields. More about fields in an operation object:
+            * :operation  - Either :add or :edit
+            * :entityType - What type of node, e.g. :block, :page, :tag or :property
+            * :id - For :edit, this _must_ be a string uuid. For :add, use a temporary unique string if the new page is referenced by later operations e.g. add blocks
+            * :data - A map of fields to set or update. This map can have the following keys:
+              * :title - A page/tag/property's name or a block's content
+              * :page-id - A page string uuid of a block. Required when entityType is :block.
+              * :tags - A list of tags as string uuids
+              * :property-type - A property's type
+              * :property-cardinality - A property's cardinality. Must be :one or :many
+              * :property-classes - A property's list of allowed tags, each being a uuid string or a tag's name
+              * :class-extends - List of parent tags, each being a uuid string or a tag's name
+              * :class-properties - A tag's list of properties, each eing a uuid string or a property's name
+
+         Example inputs with their prompt, description and data as clojure EDN:
+
+         Description: This input adds a new block to page with id '119268a6-704f-4e9e-8c34-36dfc6133729' and update the title of a page with uuid '119268a6-704f-4e9e-8c34-36dfc6133729':
+
+         {:operations
+          [{:operation :add
+            :entityType :block
+            :id nil
+            :data {:page-id \"119268a6-704f-4e9e-8c34-36dfc6133729\"
+                   :title \"New block text\"}}
+           {:operation :edit
+            :entity :page
+            :id \"119268a6-704f-4e9e-8c34-36dfc6133729\"
+            :data {:title \"Revised page title\"}}]}
+
+        Prompt: Add task 't1' to new page 'Inbox'
+        Description: This input creates a page 'Inbox' and adds a 't1' block with tag \"00000002-1282-1814-5700-000000000000\" (task) to it:
+
+        {:operations
+          [{:operation :add
+            :entityType :page
+            :id \"temp-Inbox\"
+            :data {:title \"Inbox\"}}
+           {:operation :add
+            :entityType :block
+            :data {:page-id \"temp-Inbox\"
+                   :title \"t1\"
+                   :tags [\"00000002-1282-1814-5700-000000000000\"]}}]}
+
+         Additional advice for building operations:
+         * When building a block update operation, use the 'page' key of the searchBlocks tool to fill in the value of :page-id under :data
+         * Before creating any page, tag or property, check that it exists with getPage"
+         :inputSchema
+         #js {:operations
+              (z/array
+               (z/object
+                #js {:operation   (z/enum #js ["add" "edit"])
+                     :entityType  (z/enum #js ["block" "page" "tag" "property"])
+                     :id          (.optional (z/union #js [(z/string) (z/number) (z/null)]))
+                     :data        (-> (z/object #js {}) (.passthrough))}))}}}
    :searchBlocks
    {:fn api-search-blocks
     :config #js {:title "Search Blocks"

deps/cli/src/logseq/cli/common/mcp/tools.cljs

@@ -1,10 +1,19 @@
 (ns logseq.cli.common.mcp.tools
   "MCP tool related fns shared between CLI and frontend"
-  (:require [datascript.core :as d]
+  (:require [clojure.string :as string]
+            [datascript.core :as d]
+            [logseq.common.util :as common-util]
+            [logseq.common.util.date-time :as date-time-util]
             [logseq.db :as ldb]
+            [logseq.db.frontend.class :as db-class]
             [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.frontend.property :as db-property]
-            [logseq.outliner.tree :as otree]))
+            [logseq.db.frontend.property.type :as db-property-type]
+            [logseq.db.sqlite.export :as sqlite-export]
+            [logseq.outliner.tree :as otree]
+            [logseq.outliner.validate :as outliner-validate]
+            [malli.core :as m]
+            [malli.error :as me]))
 
 (defn list-properties
   "Main fn for ListProperties tool"
@@ -83,4 +92,264 @@
                  ;; Until there are options to limit pages, return minimal info to avoid
                  ;; exceeding max payload size
                  (select-keys [:block/uuid :block/title :block/created-at :block/updated-at])
-                 (update :block/uuid str)))))
+                 (update :block/uuid str)))))
+
+;; upsert-nodes tool
+;; =================
+(defn- import-edn-data
+  [conn export-map]
+  (let [{:keys [init-tx block-props-tx misc-tx error] :as _txs}
+        (sqlite-export/build-import export-map @conn {})]
+    ;; (cljs.pprint/pprint _txs)
+    (when error
+      (throw (ex-info (str "Error while building import data: " error) {})))
+    (let [tx-meta {::sqlite-export/imported-data? true
+                   :import-db? true}]
+      (ldb/transact! conn (vec (concat init-tx block-props-tx misc-tx)) tx-meta))))
+
+(defn- get-ident [idents title]
+  (or (get idents title)
+      (throw (ex-info (str "No ident found for " (pr-str title)) {}))))
+
+(defn- ops->pages-and-blocks
+  [operations {:keys [class-idents]}]
+  (let [blocks-by-page
+        (group-by #(get-in % [:data :page-id])
+                  (filter #(= "block" (:entityType %)) operations))
+        new-pages (filter #(and (= "page" (:entityType %)) (= "add" (:operation %))) operations)
+        pages-and-blocks
+        (into (mapv (fn [op]
+                      (cond-> {:page (if-let [journal-day (date-time-util/journal-title->int
+                                                           (get-in op [:data :title])
+                                                           ;; consider user's date-formatter as needed
+                                                           (date-time-util/safe-journal-title-formatters nil))]
+                                       {:build/journal journal-day}
+                                       {:block/title (get-in op [:data :title])})}
+                        (some-> (:id op) (get blocks-by-page))
+                        (assoc :blocks
+                               (mapv #(hash-map :block/title (get-in % [:data :title]))
+                                     (get blocks-by-page (:id op))))))
+                    new-pages)
+              ;; existing pages
+              (map (fn [[page-id ops]]
+                     {:page {:block/uuid (uuid page-id)}
+                      :blocks (mapv (fn [op]
+                                      (if (= "add" (:operation op))
+                                        (cond-> {:block/title (get-in op [:data :title])}
+                                          (get-in op [:data :tags])
+                                          (assoc :build/tags (mapv #(get-ident class-idents %) (get-in op [:data :tags]))))
+                                        ;; edit
+                                        (cond-> {:block/uuid (uuid (:id op))}
+                                          (get-in op [:data :title])
+                                          (assoc :block/title (get-in op [:data :title])))))
+                                    ops)})
+                   (apply dissoc blocks-by-page (map :id new-pages))))]
+    pages-and-blocks))
+
+(defn- ops->classes
+  [operations {:keys [property-idents class-idents existing-idents]}]
+  (let [new-classes (filter #(and (= "tag" (:entityType %)) (= "add" (:operation %))) operations)
+        classes (merge
+                 (into {} (keep (fn [[k v]]
+                                  ;; Removing existing until edits are supported
+                                  (when-not (existing-idents v) [v {:block/title k}]))
+                                class-idents))
+                 (->> new-classes
+                      (map (fn [{:keys [data] :as op}]
+                             (let [title (get-in op [:data :title])
+                                   class-m (cond-> {:block/title title}
+                                             (:class-extends data)
+                                             (assoc :build/class-extends (mapv #(get-ident class-idents %) (:class-extends data)))
+                                             (:class-properties data)
+                                             (assoc :build/class-properties (mapv #(get-ident property-idents %) (:class-properties data))))]
+                               [(get-ident class-idents title) class-m])))
+                      (into {})))]
+    classes))
+
+(defn- ops->properties
+  [operations {:keys [property-idents class-idents existing-idents]}]
+  (let [new-properties (filter #(and (= "property" (:entityType %)) (= "add" (:operation %))) operations)
+        properties
+        (merge
+         (into {} (map (fn [[k v]]
+                         ;; Removing existing until edits are supported
+                         (when-not (existing-idents v) [v {:block/title k}]))
+                       property-idents))
+         (->> new-properties
+              (map (fn [{:keys [data] :as op}]
+                     (let [title (get-in op [:data :title])
+                           prop-m (cond-> {:block/title title}
+                                    (some->> (:property-type data) keyword (contains? (set db-property-type/user-built-in-property-types)))
+                                    (assoc :logseq.property/type (keyword (:property-type data)))
+                                    (= "many" (:property-cardinality data))
+                                    (assoc :db/cardinality :db.cardinality/many)
+                                    (:property-classes data)
+                                    (assoc :build/property-classes
+                                           (mapv #(get-ident class-idents %) (:property-classes data))
+                                           :logseq.property/type :node))]
+                       [(get-ident property-idents title) prop-m])))
+              (into {})))]
+    properties))
+
+(defn- operations->idents
+  "Creates property and class idents from all uses of them in operations"
+  [db operations]
+  (let [existing-idents (atom #{})
+        property-idents
+        (->> (filter #(and (= "property" (:entityType %)) (= "add" (:operation %)))
+                     operations)
+             (map #(get-in % [:data :title]))
+             (into (mapcat #(get-in % [:data :class-properties])
+                           (filter #(and (= "tag" (:entityType %)) (= "add" (:operation %)))
+                                   operations)))
+             distinct
+             (map #(vector % (if (common-util/uuid-string? %)
+                               (let [ent (d/entity db [:block/uuid (uuid %)])
+                                     ident (:db/ident ent)]
+                                 (when-not (entity-util/property? ent)
+                                   (throw (ex-info (str (pr-str (:block/title ent))
+                                                        " is not a property and can't be used as one")
+                                                   {})))
+                                 (swap! existing-idents conj ident)
+                                 ident)
+                               (db-property/create-user-property-ident-from-name %))))
+             (into {}))
+        class-idents
+        (->> (filter #(and (= "tag" (:entityType %)) (= "add" (:operation %))) operations)
+             (mapcat (fn [op]
+                       (into [(get-in op [:data :title])] (get-in op [:data :class-extends]))))
+             (into (mapcat #(get-in % [:data :property-classes])
+                           (filter #(and (= "property" (:entityType %)) (= "add" (:operation %)))
+                                   operations)))
+             (into (mapcat #(get-in % [:data :tags])
+                           (filter #(and (= "block" (:entityType %)) (= "add" (:operation %)))
+                                   operations)))
+             distinct
+             (map #(vector % (if (common-util/uuid-string? %)
+                               (let [ent (d/entity db [:block/uuid (uuid %)])
+                                     ident (:db/ident ent)]
+                                 (when-not (entity-util/class? ent)
+                                   (throw (ex-info (str (pr-str (:block/title ent))
+                                                        " is not a tag and can't be used as one")
+                                                   {})))
+                                 (swap! existing-idents conj ident)
+                                 ident)
+                               (db-class/create-user-class-ident-from-name db %))))
+             (into {}))]
+    {:property-idents property-idents
+     :class-idents class-idents
+     :existing-idents @existing-idents}))
+
+(def ^:private add-non-block-schema
+  [:map
+   [:data [:map
+           [:title :string]]]])
+
+(def ^:private uuid-string
+  [:and :string [:fn {:error/message "Must be a uuid string"} common-util/uuid-string?]])
+
+(def ^:private upsert-nodes-operation-schema
+  [:and
+   ;; Base schema. Has some overlap with inputSchema
+   [:map
+    {:closed true}
+    [:operation [:enum "add" "edit"]]
+    [:entityType [:enum "block" "page" "tag" "property"]]
+    [:id {:optional true} [:or :string :nil]]
+    [:data [:map
+            [:title {:optional true} :string]
+            [:page-id {:optional true} :string]
+            [:tags {:optional true} [:sequential uuid-string]]
+            [:property-type {:optional true} :string]
+            [:property-cardinality {:optional true} [:enum "many" "one"]]
+            [:property-classes {:optional true} [:sequential :string]]
+            [:class-extends {:optional true} [:sequential :string]]
+            [:class-properties {:optional true} [:sequential :string]]]]]
+   ;; Validate special cases of operation and entityType e.g. required keys and uuid strings
+   [:multi {:dispatch (juxt :operation :entityType)}
+    [["add" "block"] [:map
+                      [:data [:map
+                              [:title :string]
+                              [:page-id :string]]]]]
+    [["add" "page"] add-non-block-schema]
+    [["add" "tag"] add-non-block-schema]
+    [["add" "property"] add-non-block-schema]
+    [["edit" "block"] [:map
+                       [:id uuid-string]
+                       ;; :tags not supported yet
+                       [:data [:map {:closed true}
+                               [:page-id uuid-string]
+                               [:title :string]]]]]
+    ;; other edit's
+    [::m/default [:map [:id uuid-string]]]]])
+
+(def ^:private Upsert-nodes-operations-schema
+  [:sequential upsert-nodes-operation-schema])
+
+(defn- validate-import-edn
+  "Validates everything as coming from add operations, failing fast on first invalid
+  node. Will need to adjust add operation assumption when supporting editing pages"
+  [{:keys [pages-and-blocks properties classes]}]
+  (try
+    (doseq [{:block/keys [title] :as m} (vals properties)]
+      (outliner-validate/validate-property-title title {:entity-type :property :title title :entity-map m})
+      (outliner-validate/validate-page-title-characters title {:entity-type :property :title title :entity-map m})
+      (outliner-validate/validate-page-title title {:entity-type :property :title title :entity-map m}))
+    (doseq [{:block/keys [title] :as m} (vals classes)]
+      (outliner-validate/validate-page-title-characters title {:entity-type :tag :title title :entity-map m})
+      (outliner-validate/validate-page-title title {:entity-type :tag :title title :entity-map m}))
+    (doseq [{:block/keys [title] :as m} (map :page pages-and-blocks)]
+      ;; title is only present for new pages
+      (when title
+        (outliner-validate/validate-page-title-characters title {:entity-type :page :title title :entity-map m})
+        (outliner-validate/validate-page-title title {:entity-type :page :title title :entity-map m})))
+    (catch :default e
+      (js/console.error e)
+      (throw (ex-info (str (string/capitalize (name (get (ex-data e) :entity-type :page)))
+                           " " (pr-str (:title (ex-data e))) " is invalid: " (ex-message e))
+                      (ex-data e))))))
+
+(defn- summarize-upsert-operations [operations]
+  (let [counts (reduce (fn [acc op]
+                         (let [entity-type (keyword (:entityType op))
+                               operation-type (keyword (:operation op))]
+                           (update-in acc [operation-type entity-type] (fnil inc 0))))
+                       {}
+                       operations)]
+    (str (when (counts :add)
+           (str "Added: " (pr-str (counts :add)) "."))
+         (when (counts :edit)
+           (str " Edited: " (pr-str (counts :edit)) ".")))))
+
+(defn upsert-nodes
+  [conn operations*]
+  ;; Only support these operations with appropriate outliner validations
+  (when (seq (filter #(and (#{"page" "tag" "property"} (:entityType %)) (= "edit" (:operation %))) operations*))
+    (throw (ex-info "Editing a page, tag or property isn't supported yet" {})))
+  (let [operations
+        (->> operations*
+             ;; normalize classes as they sometimes have titles in :name
+             (map #(if (and (= "tag" (:entityType %)) (= "add" (:operation %)))
+                     (assoc-in % [:data :title]
+                               (or (get-in % [:data :name]) (get-in % [:data :title])))
+                     %)))
+        _ (prn :ops operations)
+        _ (when-let [errors (m/explain Upsert-nodes-operations-schema operations)]
+            (throw (ex-info (str "Tool arguments are invalid:\n" (me/humanize errors))
+                            {:errors errors})))
+        idents (operations->idents @conn operations)
+        pages-and-blocks (ops->pages-and-blocks operations idents)
+        classes (ops->classes operations idents)
+        properties (ops->properties operations idents)
+        import-edn
+        (cond-> {}
+          (seq pages-and-blocks)
+          (assoc :pages-and-blocks pages-and-blocks)
+          (seq classes)
+          (assoc :classes classes)
+          (seq properties)
+          (assoc :properties properties))]
+    (prn :import-edn import-edn)
+    (validate-import-edn import-edn)
+    (import-edn-data conn import-edn)
+    (summarize-upsert-operations operations*)))

deps/outliner/src/logseq/outliner/validate.cljs

@@ -144,12 +144,14 @@
 
 (defn validate-property-title
   "Validates a property's title when it has changed"
-  [new-title]
-  (when-not (db-property/valid-property-name? new-title)
-    (throw (ex-info "Property name is invalid"
-                    {:type :notification
-                     :payload {:message "This is an invalid property name. A property name cannot start with page reference characters '#' or '[['."
-                               :type :error}}))))
+  ([new-title] (validate-property-title new-title {}))
+  ([new-title meta-m]
+   (when-not (db-property/valid-property-name? new-title)
+     (throw (ex-info "Property name is invalid"
+                     (merge meta-m
+                            {:type :notification
+                             :payload {:message "This is an invalid property name. A property name cannot start with page reference characters '#' or '[['."
+                                       :type :error}}))))))
 
 (defn- validate-extends-property-have-correct-type
   "Validates whether given parent and children are classes"

src/main/logseq/api.cljs

@@ -33,6 +33,7 @@
             [frontend.handler.route :as route-handler]
             [frontend.handler.search :as search-handler]
             [frontend.handler.shell :as shell]
+            [frontend.handler.ui :as ui-handler]
             [frontend.idb :as idb]
             [frontend.loader :as loader]
             [frontend.modules.layout.core]
@@ -60,7 +61,8 @@
             [logseq.sdk.ui :as sdk-ui]
             [logseq.sdk.utils :as sdk-utils]
             [promesa.core :as p]
-            [reitit.frontend.easy :as rfe]))
+            [reitit.frontend.easy :as rfe]
+            [logseq.cli.common.mcp.tools :as cli-common-mcp-tools]))
 
 ;; Alert: this namespace shouldn't invoke any reactive queries
 
@@ -1198,6 +1200,13 @@
       (clj->js resp)
       #js {:error (str "Page " (pr-str page-title) " not found")})))
 
+(defn ^:export upsert_nodes
+  "Given a list of MCP operations, batch upserts resulting EDN data"
+  [operations]
+  (p/let [resp (cli-common-mcp-tools/upsert-nodes (conn/get-db false) (js->clj operations :keywordize-keys true))]
+    (ui-handler/re-render-root!)
+    resp))
+
 ;; ui
 (def ^:export show_msg sdk-ui/-show_msg)
 (def ^:export query_element_rect sdk-ui/query_element_rect)