diff --git a/cli-e2e/spec/non_sync_cases.edn b/cli-e2e/spec/non_sync_cases.edn index c282c469c6..fe430b7c5b 100644 --- a/cli-e2e/spec/non_sync_cases.edn +++ b/cli-e2e/spec/non_sync_cases.edn @@ -610,6 +610,25 @@ :search ["--content"]}}, :tags [:upsert :search], :extends :non-sync/graph-json-env} + {:id "qmd-init-and-qsearch-json", + :setup + ["python3 - <<'PY'\nimport base64, pathlib\np = pathlib.Path('{{tmp-dir}}/fake-bin/qmd')\np.parent.mkdir(parents=True, exist_ok=True)\np.write_bytes(base64.b64decode('IyEvdXNyL2Jpbi9lbnYgcHl0aG9uMwppbXBvcnQganNvbgppbXBvcnQgb3MKaW1wb3J0IHBhdGhsaWIKaW1wb3J0IHJlCmltcG9ydCBzeXMKCmFyZ3MgPSBzeXMuYXJndlsxOl0Kc3RhdGUgPSBwYXRobGliLlBhdGgob3MuZW52aXJvblsnUU1EX0ZBS0VfU1RBVEUnXSkKCmlmIGFyZ3MgPT0gWyctLWhlbHAnXToKICAgIHByaW50KCdxbWQgaGVscCcpCiAgICBzeXMuZXhpdCgwKQoKaWYgYXJnc1s6Ml0gPT0gWydjb2xsZWN0aW9uJywgJ3Nob3cnXToKICAgIGlmIG5vdCBzdGF0ZS5leGlzdHMoKToKICAgICAgICBwcmludCgnQ29sbGVjdGlvbiBub3QgZm91bmQnLCBmaWxlPXN5cy5zdGRlcnIpCiAgICAgICAgc3lzLmV4aXQoMSkKICAgIHByaW50KCdDb2xsZWN0aW9uOiAnICsgYXJnc1syXSkKICAgIHByaW50KCcgIFBhdGg6ICAgICAnICsgc3RhdGUucmVhZF90ZXh0KCkuc3RyaXAoKSkKICAgIHByaW50KCcgIFBhdHRlcm46ICAqKi8qLm1kJykKICAgIHN5cy5leGl0KDApCgppZiBhcmdzWzoyXSA9PSBbJ2NvbGxlY3Rpb24nLCAnYWRkJ106CiAgICBzdGF0ZS53cml0ZV90ZXh0KGFyZ3NbMl0pCiAgICBwcmludCgnY3JlYXRlZCcpCiAgICBzeXMuZXhpdCgwKQoKaWYgYXJnc1s6MV0gPT0gWyd1cGRhdGUnXToKICAgIHByaW50KCd1cGRhdGVkJykKICAgIHN5cy5leGl0KDApCgppZiBhcmdzWzoxXSA9PSBbJ3F1ZXJ5J106CiAgICByb290ID0gcGF0aGxpYi5QYXRoKHN0YXRlLnJlYWRfdGV4dCgpLnN0cmlwKCkpCiAgICByb3dzID0gW10KICAgIGZvciBwYXRoIGluIHJvb3Qucmdsb2IoJyoubWQnKToKICAgICAgICB0ZXh0ID0gcGF0aC5yZWFkX3RleHQoZW5jb2Rpbmc9J3V0ZjgnKQogICAgICAgIG1hdGNoID0gcmUuc2VhcmNoKHInPCEtLSBpZDogKFxkKykgLS0+JywgdGV4dCkKICAgICAgICBpZiBtYXRjaDoKICAgICAgICAgICAgcm93cy5hcHBlbmQoewogICAgICAgICAgICAgICAgJ3Njb3JlJzogMSwKICAgICAgICAgICAgICAgICdmaWxlJzogJ3FtZDovL2N1c3RvbS8nICsgcGF0aC5uYW1lLAogICAgICAgICAgICAgICAgJ3NuaXBwZXQnOiAnLSBxbWQgdGFyZ2V0IDwhLS0gaWQ6ICcgKyBtYXRjaC5ncm91cCgxKSArICcgLS0+JywKICAgICAgICAgICAgfSkKICAgICAgICAgICAgYnJlYWsKICAgIHByaW50KGpzb24uZHVtcHMocm93cykpCiAgICBzeXMuZXhpdCgwKQoKcHJpbnQoJ3VuZXhwZWN0ZWQgYXJnczogJyArIHJlcHIoYXJncyksIGZpbGU9c3lzLnN0ZGVycikKc3lzLmV4aXQoMikK'))\np.chmod(0o755)\nPY" + "{{cli}} --root-dir {{root-dir-arg}} --config {{config-path-arg}} --output json upsert block --graph {{graph-arg}} --target-page QmdSearchPage --content \"qmd target\" >/dev/null"], + :cmds + ["QMD_FAKE_STATE={{tmp-dir-arg}}/qmd-state PATH={{tmp-dir-arg}}/fake-bin:$PATH {{cli}} --root-dir {{root-dir-arg}} --config {{config-path-arg}} --output json qmd init --graph {{graph-arg}} --collection custom" + "QMD_FAKE_STATE={{tmp-dir-arg}}/qmd-state PATH={{tmp-dir-arg}}/fake-bin:$PATH {{cli}} --root-dir {{root-dir-arg}} --config {{config-path-arg}} --output json qsearch qmd target --graph {{graph-arg}} --collection custom --no-rerank"], + :expect + {:exit 0, + :stdout-json-paths {[:status] "ok"}, + :stdout-contains ["qmd target"]}, + :covers + {:commands ["qmd init" "qsearch"], + :options + {:global ["--config" "--graph" "--root-dir" "--output"], + :qmd ["--collection"] + :qsearch ["--collection" "--no-rerank"]}}, + :tags [:qmd :search], + :extends :non-sync/graph-json-env} {:id "search-page-json", :setup ["{{cli}} --root-dir {{root-dir-arg}} --config {{config-path-arg}} --output json upsert page --graph {{graph-arg}} --page SearchPageTarget >/dev/null"], diff --git a/docs/agent-guide/logseq-cli/011-qmd-search.md b/docs/agent-guide/logseq-cli/011-qmd-search.md new file mode 100644 index 0000000000..891501d384 --- /dev/null +++ b/docs/agent-guide/logseq-cli/011-qmd-search.md @@ -0,0 +1,70 @@ +# QMD Search CLI Implementation Plan + +## Goal + +Add `logseq qmd init` and `logseq qsearch` so CLI users can search DB graph Markdown Mirror files through QMD and map search hits back to Logseq block entities. + +## Architecture + +- Keep the QMD integration in the CLI process. +- Call `qmd` as an external executable through Node child process APIs with argument vectors. +- Reuse db-worker-node startup and transport helpers. +- Reuse the existing `:thread-api/markdown-mirror-regenerate` API for mirror generation. +- Reuse the existing `:thread-api/pull` API for entity lookup. +- Do not add a new thread API for this feature. + +## Public Interface + +`logseq qmd init --graph [--index ] [--collection ]` + +- Verify that `qmd` is executable. +- Regenerate Markdown Mirror for the selected graph. +- Resolve the mirror directory as `/graphs//mirror/markdown`. +- Initialize the QMD collection with `qmd collection add --name --mask "**/*.md"`. +- If the collection already points at the same mirror directory, run `qmd update`. +- If the collection name points at a different path, fail fast. + +`logseq qsearch --graph [-n ] [--index ] [--collection ] [--no-rerank]` + +- Run `qmd query --json -c -n `. +- Parse noisy QMD stdout defensively. +- Extract block ids from Markdown Mirror comments matching ``. +- Pull entities through `:thread-api/pull`. +- Deduplicate ids in QMD rank order. +- Return Logseq list-style output plus `missing-ids` for stale QMD results. + +Default collection names use `logseq--` to avoid collisions between graphs with similar display names. + +## Data Flow + +```mermaid +flowchart LR + A["logseq qmd init"] --> B["db-worker-node"] + B --> C["markdown-mirror-regenerate"] + C --> D["Markdown Mirror directory"] + D --> E["qmd collection add/update"] + + F["logseq qsearch"] --> G["qmd query --json"] + G --> H["Extract block id comments"] + H --> I["thread-api/pull"] + I --> J["List-style CLI output"] +``` + +## Implementation Tasks + +- Add `src/main/logseq/cli/command/qmd.cljs` for command entries, action builders, QMD process execution, collection initialization, QMD JSON parsing, id extraction, and qsearch normalization. +- Wire `qmd init` and `qsearch` into `src/main/logseq/cli/commands.cljs`. +- Show `qmd` in utility help and `qsearch` in graph inspect/search help. +- Format `:qmd-init` and `:qsearch` in `src/main/logseq/cli/format.cljs` for human, JSON, and EDN output. +- Add unit tests for command entries, graph requirements, default collection names, mirror path derivation, QMD executable checks, collection create/update/mismatch handling, noisy JSON parsing, block id extraction, dedupe order, missing ids, and entity pull calls. +- Add parser tests for `qmd init`, `qsearch`, help output, positional query handling, and invalid option behavior. +- Add format tests for human table output and JSON/EDN payload stability. +- Add a CLI E2E case with a fake `qmd` executable on `PATH` so the test does not depend on local QMD models, embeddings, or network access. + +## Verification + +- Run focused unit tests for `logseq.cli.command.qmd-test`, `logseq.cli.commands-test/test-qmd-and-qsearch-parse`, and qsearch/qmd format tests. +- Run `bb dev:lint-and-test`. +- Run `bb -f cli-e2e/bb.edn build`. +- Run `bb -f cli-e2e/bb.edn test --skip-build`. +- Review the finished diff with `logseq-review-workflow`, applying common, Clojure/CLJS, promesa/Node process, babashka CLI, shadow-cljs Node, logseq-cli, and search-indexing rules. diff --git a/src/main/logseq/cli/command/core.cljs b/src/main/logseq/cli/command/core.cljs index 20255b0d5a..f3a0b95a5e 100644 --- a/src/main/logseq/cli/command/core.cljs +++ b/src/main/logseq/cli/command/core.cljs @@ -116,16 +116,17 @@ (defn top-level-summary [table] (let [groups [{:title "Graph Inspect and Edit" - :commands #{"list" "upsert" "remove" "query" "search" "show"}} + :commands #{"list" "upsert" "remove" "query" "qsearch" "search" "show"}} {:title "Graph Management" :commands #{"graph" "server" "doctor" "sync"}} {:title "Authentication" :commands #{"login" "logout"}} {:title "Utilities" - :commands #{"completion" "debug" "example" "skill"} + :commands #{"completion" "debug" "example" "qmd" "skill"} :top-level-only? true :desc-overrides {"debug" "Pull raw entity data for debugging" "example" "Show command examples" + "qmd" "Initialize and manage QMD search" "skill" "Show/install built-in logseq-cli skill"}}] to-top-level-entries (fn [entries commands desc-overrides] (->> commands diff --git a/src/main/logseq/cli/command/qmd.cljs b/src/main/logseq/cli/command/qmd.cljs new file mode 100644 index 0000000000..2324981d2d --- /dev/null +++ b/src/main/logseq/cli/command/qmd.cljs @@ -0,0 +1,336 @@ +(ns logseq.cli.command.qmd + "QMD-backed CLI search commands." + (:require ["child_process" :as child-process] + ["crypto" :as crypto] + ["fs" :as fs] + ["path" :as node-path] + [clojure.string :as string] + [logseq.cli.command.core :as core] + [logseq.cli.root-dir :as root-dir] + [logseq.cli.server :as cli-server] + [logseq.cli.transport :as transport] + [logseq.common.graph-dir :as graph-dir] + [promesa.core :as p])) + +(def ^:private markdown-glob "**/*.md") +(def ^:private block-id-comment-re #"") + +(def ^:private qmd-common-spec + {:index {:desc "QMD index name"} + :collection {:desc "QMD collection name"}}) + +(def ^:private qmd-init-spec + qmd-common-spec) + +(def ^:private qsearch-spec + (merge qmd-common-spec + {:limit {:desc "Limit results" + :alias :n + :coerce :long} + :no-rerank {:desc "Skip QMD reranking" + :coerce :boolean}})) + +(def entries + [(core/command-entry ["qmd" "init"] :qmd-init + "Initialize QMD for the graph Markdown Mirror" + qmd-init-spec + {:examples ["logseq qmd init --graph my-graph"]}) + (core/command-entry ["qsearch"] :qsearch + "Search graph Markdown Mirror with QMD" + qsearch-spec + {:examples ["logseq qsearch \"markdown mirror\" --graph my-graph"]})]) + +(defn- sha1-prefix + [value length] + (subs (.digest (.update (.createHash crypto "sha1") (str value)) "hex") + 0 + length)) + +(defn- slug + [value] + (let [value (-> (str value) + string/lower-case + (string/replace #"[^a-z0-9]+" "-") + (string/replace #"^-+" "") + (string/replace #"-+$" ""))] + (if (seq value) value "graph"))) + +(defn default-collection-name + [repo] + (str "logseq-" + (slug (core/repo->graph repo)) + "-" + (sha1-prefix repo 8))) + +(defn mirror-dir + [config repo] + (node-path/join (root-dir/graphs-dir (:root-dir config)) + (graph-dir/repo->encoded-graph-dir-name repo) + "mirror" + "markdown")) + +(defn- qmd-args + [index args] + (cond-> [] + (seq index) (into ["--index" index]) + true (into args))) + +(defn js args) + #js {:stdio #js ["ignore" "pipe" "pipe"]})] + (some-> (.-stdout child) + (.on "data" (fn [chunk] + (swap! stdout str (.toString chunk))))) + (some-> (.-stderr child) + (.on "data" (fn [chunk] + (swap! stderr str (.toString chunk))))) + (.on child "error" + (fn [error] + (when-not @settled? + (reset! settled? true) + (resolve {:exit 127 + :out @stdout + :err (or (.-message error) (str error)) + :error error + :args args})))) + (.on child "close" + (fn [code] + (when-not @settled? + (reset! settled? true) + (resolve {:exit (or code 0) + :out @stdout + :err @stderr + :args args})))))))) + +(defn- qmd-error + [code message result] + {:status :error + :error (cond-> {:code code + :message message} + (:err result) (assoc :stderr (:err result)) + (:out result) (assoc :stdout (:out result)))}) + +(defn- (re-find #"(?m)^\s*Path:\s+([^\r\n]+)\s*$" (or output "")) + second + string/trim)) + +(defn- normalize-path + [path] + (some-> path + node-path/resolve + (as-> resolved + (try + (.realpathSync fs resolved) + (catch :default _ resolved))))) + +(defn- same-path? + [left right] + (= (normalize-path left) + (normalize-path right))) + +(defn- qmd-command-failed + [message result] + (qmd-error :qmd-command-failed message result)) + +(defn build-init-action + [options repo] + (if-not (seq repo) + {:ok? false + :error {:code :missing-repo + :message "repo is required for qmd init"}} + {:ok? true + :action {:type :qmd-init + :repo repo + :graph (core/repo->graph repo) + :collection (or (:collection options) + (default-collection-name repo)) + :index (:index options)}})) + +(defn build-search-action + [options args repo] + (let [query (->> args + (map str) + (string/join " ") + string/trim)] + (cond + (not (seq repo)) + {:ok? false + :error {:code :missing-repo + :message "repo is required for qsearch"}} + + (not (seq query)) + {:ok? false + :error {:code :missing-query-text + :message "query text is required"}} + + :else + {:ok? true + :action {:type :qsearch + :repo repo + :graph (core/repo->graph repo) + :query query + :limit (:limit options) + :collection (or (:collection options) + (default-collection-name repo)) + :index (:index options) + :no-rerank (true? (:no-rerank options))}}))) + +(defn execute-qmd-init + [action config] + (p/let [qmd-check (clj parsed :keywordize-keys true) + (recur (string/index-of output "[" (inc start))))))))) + +(defn extract-block-ids + [results] + (->> (or results []) + (mapcat (fn [result] + (map (fn [[_ id]] + (js/parseInt id 10)) + (re-seq block-id-comment-re (or (:snippet result) ""))))) + (reduce (fn [acc id] + (if (some #{id} acc) + acc + (conj acc id))) + []))) + +(def ^:private qsearch-pull-selector + [:db/id :block/title :block/uuid + {:block/page [:db/id :block/title :block/name :block/uuid]}]) + +(defn- qmd-result-by-id + [results] + (reduce-kv (fn [acc idx result] + (reduce (fn [acc' id] + (if (contains? acc' id) + acc' + (assoc acc' id (assoc result :qmd/rank (inc idx))))) + acc + (extract-block-ids [result]))) + {} + (vec (or results [])))) + +(defn- normalize-qsearch-item + [entity qmd-result] + (let [page (:block/page entity)] + (cond-> {:db/id (:db/id entity) + :block/title (:block/title entity) + :qmd/rank (:qmd/rank qmd-result)} + (:block/uuid entity) (assoc :block/uuid (:block/uuid entity)) + (:score qmd-result) (assoc :qmd/score (:score qmd-result)) + (:file qmd-result) (assoc :qmd/file (:file qmd-result)) + (:db/id page) (assoc :block/page-id (:db/id page)) + (or (:block/title page) (:block/name page)) + (assoc :block/page-title (or (:block/title page) (:block/name page)))))) + +(defn- qsearch-args + [{:keys [query collection limit no-rerank index]}] + (cond-> (qmd-args index ["query" query "--json" "-c" collection]) + limit (into ["-n" (str limit)]) + no-rerank (conj "--no-rerank"))) + +(defn execute-qsearch + [action config] + (p/let [cfg (cli-server/ensure-server! config (:repo action)) + qmd-result (` and retry"}} + (let [result-by-id (qmd-result-by-id results)] + (p/let [entities (p/all + (map (fn [id] + (transport/invoke cfg :thread-api/pull + [(:repo action) qsearch-pull-selector id])) + ids)) + pairs (mapv vector ids entities) + items (->> pairs + (keep (fn [[id entity]] + (when entity + (normalize-qsearch-item entity + (get result-by-id id))))) + vec) + missing-ids (->> pairs + (keep (fn [[id entity]] + (when-not entity id))) + vec)] + {:status :ok + :data {:items items + :missing-ids missing-ids + :qmd {:collection (:collection action) + :index (:index action) + :result-count (count results)}}}))))))) diff --git a/src/main/logseq/cli/commands.cljs b/src/main/logseq/cli/commands.cljs index aaf9fe502b..b90e180d35 100644 --- a/src/main/logseq/cli/commands.cljs +++ b/src/main/logseq/cli/commands.cljs @@ -10,6 +10,7 @@ [logseq.cli.command.example :as example-command] [logseq.cli.command.graph :as graph-command] [logseq.cli.command.list :as list-command] + [logseq.cli.command.qmd :as qmd-command] [logseq.cli.command.query :as query-command] [logseq.cli.command.remove :as remove-command] [logseq.cli.command.search :as search-command] @@ -97,6 +98,7 @@ upsert-command/entries remove-command/entries query-command/entries + qmd-command/entries search-command/entries show-command/entries doctor-command/entries @@ -134,6 +136,45 @@ {:args args :id-from-stdin? false}) {:args args :id-from-stdin? false})) +(def ^:private qsearch-value-options + #{"--graph" "-g" "--root-dir" "--config" "--timeout-ms" "--output" "-o" + "--index" "--collection" "--limit" "-n"}) + +(def ^:private qsearch-flag-options + #{"--no-rerank" "--verbose" "-v" "--profile" "--help" "-h"}) + +(defn- normalize-qsearch-args + [args] + (if (= "qsearch" (first args)) + (loop [remaining (vec (rest args)) + option-tokens [] + query-tokens []] + (if-let [token (first remaining)] + (cond + (contains? qsearch-value-options token) + (let [value (second remaining)] + (recur (subvec remaining (if value 2 1)) + (cond-> (conj option-tokens token) + value (conj value)) + query-tokens)) + + (contains? qsearch-flag-options token) + (recur (subvec remaining 1) + (conj option-tokens token) + query-tokens) + + (string/starts-with? token "-") + (recur (subvec remaining 1) + (conj option-tokens token) + query-tokens) + + :else + (recur (subvec remaining 1) + option-tokens + (conj query-tokens token))) + (vec (concat ["qsearch"] option-tokens query-tokens)))) + args)) + (defn- unknown-command-message [{:keys [dispatch wrong-input]}] (string/join " " (cond-> (vec dispatch) @@ -187,9 +228,6 @@ (def ^:private upsert-validation-commands #{:upsert-block :upsert-page :upsert-task :upsert-tag :upsert-property :upsert-asset}) -(def ^:private search-validation-commands - #{:search-block :search-page :search-property :search-tag}) - (def ^:private list-validation-commands #{:list-page :list-tag :list-property :list-task :list-node :list-asset}) @@ -197,7 +235,7 @@ #{:remove-block :remove-page :remove-tag :remove-property}) (def ^:private server-graph-required-commands - #{:server-start :server-stop :server-restart}) + #{:server-start :server-stop :server-restart :qmd-init :qsearch}) (def ^:private supported-completion-shells #{"zsh" "bash"}) @@ -290,14 +328,18 @@ (not (seq (some-> (:name opts) string/trim)))) (missing-query-result summary) - (and (search-validation-commands command) + (and (contains? #{:search-block :search-page :search-property :search-tag} command) (seq args)) (command-core/invalid-options-result summary (legacy-search-query-guidance cmds)) - (and (search-validation-commands command) + (and (contains? #{:search-block :search-page :search-property :search-tag} command) (not (seq (some-> (:content opts) str string/trim)))) (assoc (missing-query-text-result summary) :command command) + (and (= :qsearch command) + (not (seq args))) + (assoc (missing-query-text-result summary) :command command) + :else nil)) @@ -495,7 +537,7 @@ [raw-args] (let [summary (command-core/top-level-summary table) {:keys [opts args]} (command-core/parse-leading-global-opts raw-args) - {:keys [args id-from-stdin?]} (inject-stdin-id-arg (vec args)) + {:keys [args id-from-stdin?]} (inject-stdin-id-arg (normalize-qsearch-args (vec args))) group-path (group-help-path args)] (cond (:version opts) @@ -618,6 +660,12 @@ (:search-block :search-page :search-property :search-tag) (search-command/build-action command options repo) + :qmd-init + (qmd-command/build-init-action options repo) + + :qsearch + (qmd-command/build-search-action options args repo) + :upsert-block (upsert-command/build-block-action options args repo) @@ -717,6 +765,8 @@ :search-page (search-command/execute-search-page action config) :search-property (search-command/execute-search-property action config) :search-tag (search-command/execute-search-tag action config) + :qmd-init (qmd-command/execute-qmd-init action config) + :qsearch (qmd-command/execute-qsearch action config) :upsert-block (upsert-command/execute-upsert-block action config) :upsert-page (upsert-command/execute-upsert-page action config) :upsert-task (upsert-command/execute-upsert-task action config) diff --git a/src/main/logseq/cli/format.cljs b/src/main/logseq/cli/format.cljs index 2aa031922e..573830fe13 100644 --- a/src/main/logseq/cli/format.cljs +++ b/src/main/logseq/cli/format.cljs @@ -191,6 +191,7 @@ :search-page "Use: logseq search page --content " :search-property "Use: logseq search property --content " :search-tag "Use: logseq search tag --content " + :qsearch "Use: logseq qsearch --graph " "Use: logseq search --content ")) (defn- error-hint @@ -202,6 +203,7 @@ :missing-tag-name "Use --name " :missing-query "Use --query " :missing-query-text (missing-search-query-hint command) + :qmd-no-block-ids "Run `logseq qmd init --graph ` and retry" :unknown-query "Use `logseq query list` to see available queries" :ambiguous-tag-name "Retry with --id " :ambiguous-property-name "Retry with --id " @@ -418,6 +420,39 @@ (format-list-dynamic items now-ms list-node-columns {:title-max-display-width title-max-display-width :truncate-cell-max-lines list-human-cell-max-lines}))) +(def ^:private qsearch-columns + [["RANK" (fn [item _] (:qmd/rank item)) [:qmd/rank] true] + ["ID" (fn [item _] (or (:db/id item) (:id item))) [:db/id :id] true] + ["TITLE" (fn [item _] (or (:title item) (:block/title item) (:name item))) [:title :block/title :name]] + ["PAGE-ID" (fn [item _] (:block/page-id item)) [:block/page-id]] + ["PAGE-TITLE" (fn [item _] (:block/page-title item)) [:block/page-title]] + ["SCORE" (fn [item _] (:qmd/score item)) [:qmd/score]] + ["FILE" (fn [item _] (:qmd/file item)) [:qmd/file]]]) + +(defn- format-qsearch + [data now-ms title-max-display-width] + (let [table (format-list-dynamic (:items data) + now-ms + qsearch-columns + {:title-max-display-width title-max-display-width + :truncate-cell-max-lines list-human-cell-max-lines}) + missing-ids (vec (or (:missing-ids data) []))] + (cond-> table + (seq missing-ids) + (str "\nMissing ids: " (string/join ", " missing-ids))))) + +(defn- format-qmd-init + [{:keys [collection mirror-dir action]}] + (string/join "\n" + [(str "QMD collection " + (case action + :updated "updated" + :created "created" + "ready") + ": " + (or collection "-")) + (str "Mirror: " (or mirror-dir "-"))])) + (defn- normalize-asset-type [value] (cond @@ -1028,6 +1063,8 @@ :list-asset (format-list-asset (:items data) now-ms list-title-max-display-width) (:search-block :search-page :search-property :search-tag) (format-list-page (:items data) now-ms) + :qmd-init (format-qmd-init data) + :qsearch (format-qsearch data now-ms list-title-max-display-width) :upsert-block (format-upsert-block context (:result data)) :upsert-page (format-upsert-page context (:result data)) :upsert-task (format-upsert-task context (:result data)) diff --git a/src/test/logseq/cli/command/qmd_test.cljs b/src/test/logseq/cli/command/qmd_test.cljs new file mode 100644 index 0000000000..7ba32caeb4 --- /dev/null +++ b/src/test/logseq/cli/command/qmd_test.cljs @@ -0,0 +1,309 @@ +(ns logseq.cli.command.qmd-test + (:require ["fs" :as fs] + ["os" :as os] + ["path" :as node-path] + [cljs.test :refer [async deftest is testing]] + [clojure.string :as string] + [logseq.cli.command.qmd :as qmd-command] + [logseq.cli.server :as cli-server] + [logseq.cli.transport :as transport] + [promesa.core :as p])) + +(deftest test-qmd-command-entries + (let [entries qmd-command/entries + by-command (into {} (map (juxt :command identity) entries))] + (is (= #{:qmd-init :qsearch} + (set (keys by-command)))) + (is (= ["qmd" "init"] (:cmds (:qmd-init by-command)))) + (is (= ["qsearch"] (:cmds (:qsearch by-command)))) + (is (contains? (get-in by-command [:qmd-init :spec]) :collection)) + (is (contains? (get-in by-command [:qmd-init :spec]) :index)) + (is (contains? (get-in by-command [:qsearch :spec]) :collection)) + (is (contains? (get-in by-command [:qsearch :spec]) :index)) + (is (contains? (get-in by-command [:qsearch :spec]) :limit)) + (is (= :n (get-in by-command [:qsearch :spec :limit :alias]))) + (is (contains? (get-in by-command [:qsearch :spec]) :no-rerank)))) + +(deftest test-build-actions + (testing "qmd init requires repo" + (let [result (qmd-command/build-init-action {:collection "notes"} nil)] + (is (false? (:ok? result))) + (is (= :missing-repo (get-in result [:error :code]))))) + + (testing "qmd init builds action with deterministic default collection" + (let [result (qmd-command/build-init-action {} "logseq_db_My Graph")] + (is (true? (:ok? result))) + (is (= :qmd-init (get-in result [:action :type]))) + (is (= "logseq_db_My Graph" (get-in result [:action :repo]))) + (is (re-matches #"logseq-my-graph-[0-9a-f]{8}" + (get-in result [:action :collection]))))) + + (testing "qsearch requires repo" + (let [result (qmd-command/build-search-action {} ["alpha"] nil)] + (is (false? (:ok? result))) + (is (= :missing-repo (get-in result [:error :code]))))) + + (testing "qsearch requires query text" + (let [result (qmd-command/build-search-action {} [] "logseq_db_demo")] + (is (false? (:ok? result))) + (is (= :missing-query-text (get-in result [:error :code]))))) + + (testing "qsearch joins positional query text" + (let [result (qmd-command/build-search-action {:limit 10 :no-rerank true} + ["markdown" "mirror"] + "logseq_db_demo")] + (is (true? (:ok? result))) + (is (= {:type :qsearch + :repo "logseq_db_demo" + :graph "demo" + :query "markdown mirror" + :limit 10 + :collection "logseq-demo-9d477851" + :index nil + :no-rerank true} + (:action result)))))) + +(deftest test-default-collection-name-is-deterministic-and-collision-resistant + (let [a (qmd-command/default-collection-name "logseq_db_My Graph") + b (qmd-command/default-collection-name "logseq_db_My Graph") + c (qmd-command/default-collection-name "logseq_db_My Graph 2")] + (is (= a b)) + (is (not= a c)) + (is (re-matches #"logseq-my-graph-[0-9a-f]{8}" a)))) + +(deftest test-mirror-dir + (is (= "/tmp/logseq/graphs/foo~2Fbar/mirror/markdown" + (qmd-command/mirror-dir {:root-dir "/tmp/logseq"} "logseq_db_foo/bar")))) + +(deftest test-parse-qmd-json-output-tolerates-noisy-output + (let [payload "Warning: embeddings pending\nSearching...\n[{\"file\":\"qmd://demo/pages/A.md\",\"snippet\":\"- hello \"}]\n"] + (is (= [{:file "qmd://demo/pages/A.md" + :snippet "- hello "}] + (qmd-command/parse-qmd-json-output payload)))) + (let [payload "[WARN] embeddings pending\n[{\"file\":\"qmd://demo/pages/A.md\",\"snippet\":\"- hello \"}]\n"] + (is (= [{:file "qmd://demo/pages/A.md" + :snippet "- hello "}] + (qmd-command/parse-qmd-json-output payload))))) + +(deftest test-extract-block-ids-preserves-rank-and-dedupes + (let [results [{:snippet "- one \n- two "} + {:snippet "- duplicate "} + {:snippet "- missing id"} + {:snippet "- three "}]] + (is (= [7 8 9] + (qmd-command/extract-block-ids results))))) + +(deftest test-execute-qmd-init-creates-missing-collection + (async done + (let [calls (atom [])] + (-> (p/with-redefs [qmd-command/ (p/with-redefs [qmd-command/ (p/with-redefs [qmd-command/ (p/with-redefs [qmd-command/ (p/with-redefs [qmd-command/\\n- stale \"}," + "{\"score\":0.5,\"file\":\"qmd://custom/pages/B.md\"," + "\"snippet\":\"- duplicate \"}]") + :err ""})) + cli-server/ensure-server! (fn [config repo] + (assoc config :ensured-repo repo)) + transport/invoke (fn [_ method args] + (swap! calls conj {:invoke [method args]}) + (case method + :thread-api/pull + (let [[_repo _selector id] args] + (p/resolved + (case id + 3 {:db/id 3 + :block/title "alpha" + :block/page {:db/id 1 + :block/title "Home"}} + 5 nil)))))] + (qmd-command/execute-qsearch + {:type :qsearch + :repo "logseq_db_demo" + :query "alpha" + :limit 10 + :collection "custom" + :no-rerank true} + {})) + (p/then (fn [result] + (is (= :ok (:status result))) + (is (= [{:db/id 3 + :block/title "alpha" + :block/page-id 1 + :block/page-title "Home" + :qmd/rank 1 + :qmd/score 1 + :qmd/file "qmd://custom/pages/A.md"}] + (get-in result [:data :items]))) + (is (= [5] (get-in result [:data :missing-ids]))) + (is (= [{:qmd ["query" "alpha" "--json" "-c" "custom" "-n" "10" "--no-rerank"]} + {:invoke [:thread-api/pull + ["logseq_db_demo" + [:db/id :block/title :block/uuid + {:block/page [:db/id :block/title :block/name :block/uuid]}] + 3]]} + {:invoke [:thread-api/pull + ["logseq_db_demo" + [:db/id :block/title :block/uuid + {:block/page [:db/id :block/title :block/name :block/uuid]}] + 5]]}] + @calls)))) + (p/catch (fn [e] (is false (str "unexpected error: " e)))) + (p/finally done))))) + +(deftest test-execute-qsearch-errors-when-qmd-results-have-no-block-ids + (async done + (-> (p/with-redefs [qmd-command/ [options]")))) + + (testing "qsearch accepts positional query text" + (let [result (commands/parse-args ["qsearch" "markdown" "mirror" "--graph" "demo" "-n" "10" "--no-rerank"])] + (is (true? (:ok? result))) + (is (= :qsearch (:command result))) + (is (= ["markdown" "mirror"] (:args result))) + (is (= "demo" (get-in result [:options :graph]))) + (is (= 10 (get-in result [:options :limit]))) + (is (true? (get-in result [:options :no-rerank]))))) + + (testing "qsearch requires query text" + (let [result (commands/parse-args ["qsearch" "--graph" "demo"])] + (is (false? (:ok? result))) + (is (= :missing-query-text (get-in result [:error :code]))))) + + (testing "qsearch requires graph" + (let [result (commands/parse-args ["qsearch" "markdown" "mirror"])] + (is (false? (:ok? result))) + (is (= :missing-graph (get-in result [:error :code]))))) + + (testing "qsearch rejects unknown options after positional query" + (let [result (commands/parse-args ["qsearch" "markdown" "--unknown" "--graph" "demo"])] + (is (false? (:ok? result))) + (is (= :invalid-options (get-in result [:error :code]))))) + + (testing "qmd init requires graph" + (let [result (commands/parse-args ["qmd" "init"])] + (is (false? (:ok? result))) + (is (= :missing-graph (get-in result [:error :code])))))) + (deftest test-parse-args-help-groups-primary (testing "graph/list/upsert/server groups show subcommands" (doseq [[group plain-entries bold-entries] diff --git a/src/test/logseq/cli/format_test.cljs b/src/test/logseq/cli/format_test.cljs index 61fe1450ab..f25a954718 100644 --- a/src/test/logseq/cli/format_test.cljs +++ b/src/test/logseq/cli/format_test.cljs @@ -602,6 +602,60 @@ (is (= "logseq.class/Tag" (get-in parsed-json [:data :items 0 :db/ident]))) (is (= :logseq.class/Tag (get-in parsed-edn [:data :items 0 :db/ident]))))) +(deftest test-human-output-qmd-init + (let [created (format/format-result {:status :ok + :command :qmd-init + :data {:repo "logseq_db_demo" + :collection "custom" + :mirror-dir "/tmp/root/graphs/demo/mirror/markdown" + :action :created}} + {:output-format nil}) + updated (format/format-result {:status :ok + :command :qmd-init + :data {:repo "logseq_db_demo" + :collection "custom" + :mirror-dir "/tmp/root/graphs/demo/mirror/markdown" + :action :updated}} + {:output-format nil})] + (is (string/includes? created "QMD collection created: custom")) + (is (string/includes? created "/tmp/root/graphs/demo/mirror/markdown")) + (is (string/includes? updated "QMD collection updated: custom")))) + +(deftest test-human-output-qsearch + (let [result (format/format-result {:status :ok + :command :qsearch + :data {:items [{:db/id 3 + :block/title "alpha" + :block/page-id 1 + :block/page-title "Home" + :qmd/rank 1 + :qmd/score 0.75 + :qmd/file "qmd://custom/pages/Home.md"}] + :missing-ids [5] + :qmd {:collection "custom" + :result-count 2}}} + {:output-format nil})] + (is (string/includes? result "ID")) + (is (string/includes? result "TITLE")) + (is (string/includes? result "PAGE-TITLE")) + (is (string/includes? result "alpha")) + (is (string/includes? result "Home")) + (is (string/includes? result "Missing ids: 5")) + (is (string/includes? result "Count: 1")))) + +(deftest test-structured-output-qsearch + (let [payload {:status :ok + :command :qsearch + :data {:items [{:db/id 3 + :block/title "alpha" + :qmd/rank 1}] + :missing-ids [5] + :qmd {:collection "custom"}}} + json-result (format/format-result payload {:output-format :json}) + edn-result (format/format-result payload {:output-format :edn})] + (is (string/includes? json-result "\"qmd/rank\"")) + (is (string/includes? edn-result ":qmd/rank")))) + (deftest test-list-property-json-edn-cardinality-shape (testing "list property json keeps namespaced db/cardinality while edn stays unchanged" (let [base-result {:status :ok