diff --git a/.github/workflows/clj-e2e.yml b/.github/workflows/clj-e2e.yml index 2314a7aac3..d4af9983e0 100644 --- a/.github/workflows/clj-e2e.yml +++ b/.github/workflows/clj-e2e.yml @@ -78,7 +78,7 @@ jobs: # NOTE: require the app to be build with DEV-RELEASE flag - name: Prepare E2E test build run: | - yarn gulp:build && clojure -M:cljs release app db-worker inference-worker --config-merge "{:closure-defines {frontend.config/DEV-RELEASE true}}" --debug && yarn webpack-app-build + yarn gulp:build && clojure -M:cljs release app db-worker --config-merge "{:closure-defines {frontend.config/DEV-RELEASE true}}" --debug && yarn webpack-app-build - name: Run e2e tests run: cd clj-e2e && timeout 30m bb dev diff --git a/.github/workflows/clj-rtc-e2e.yml b/.github/workflows/clj-rtc-e2e.yml index e42171ebd3..ee3c4bedf9 100644 --- a/.github/workflows/clj-rtc-e2e.yml +++ b/.github/workflows/clj-rtc-e2e.yml @@ -79,7 +79,7 @@ jobs: # NOTE: require the app to be build with DEV-RELEASE flag - name: Prepare E2E test build run: | - yarn gulp:build && clojure -M:cljs release app db-worker inference-worker --config-merge "{:closure-defines {frontend.config/DEV-RELEASE true}}" --debug && yarn webpack-app-build + yarn gulp:build && clojure -M:cljs release app db-worker --config-merge "{:closure-defines {frontend.config/DEV-RELEASE true}}" --debug && yarn webpack-app-build - name: Upload build artifact uses: actions/upload-artifact@v4 diff --git a/.github/workflows/deploy-and-branch.yml b/.github/workflows/deploy-and-branch.yml index 2897a90835..8e910c8eeb 100644 --- a/.github/workflows/deploy-and-branch.yml +++ b/.github/workflows/deploy-and-branch.yml @@ -51,7 +51,7 @@ jobs: - name: Build web + workers run: | yarn gulp:build - clojure -M:cljs release app db-worker inference-worker \ + clojure -M:cljs release app db-worker \ --config-merge '{:compiler-options {:source-map true :source-map-include-sources-content true :source-map-detail-level :symbols}}' yarn webpack-app-build diff --git a/.github/workflows/deploy-db-test-pages.yml b/.github/workflows/deploy-db-test-pages.yml index e8f83858b0..96d8073b27 100644 --- a/.github/workflows/deploy-db-test-pages.yml +++ b/.github/workflows/deploy-db-test-pages.yml @@ -56,7 +56,7 @@ jobs: - name: Build Released-Web run: | - yarn gulp:build && clojure -M:cljs release app db-worker inference-worker --config-merge '{:compiler-options {:source-map true :source-map-include-sources-content true :source-map-detail-level :symbols}}' && yarn webpack-app-build + yarn gulp:build && clojure -M:cljs release app db-worker --config-merge '{:compiler-options {:source-map true :source-map-include-sources-content true :source-map-detail-level :symbols}}' && yarn webpack-app-build rsync -avz --exclude node_modules --exclude android --exclude ios --exclude mobile ./static/ ./public/ ls -lR ./public && mkdir r2 && mv ./public/js/main.js.map ./r2/db-test.main.js.map sed -i 's/=main.js.map/=https:\/\/assets.logseq.io\/db-test.main.js.map/g' ./public/js/main.js diff --git a/.github/workflows/deploy-sync-test.yml b/.github/workflows/deploy-sync-test.yml index d300330100..4c63178a1e 100644 --- a/.github/workflows/deploy-sync-test.yml +++ b/.github/workflows/deploy-sync-test.yml @@ -51,7 +51,7 @@ jobs: - name: Build web + workers run: | yarn gulp:build - clojure -M:cljs release app db-worker inference-worker \ + clojure -M:cljs release app db-worker \ --config-merge '{:compiler-options {:source-map true :source-map-include-sources-content true :source-map-detail-level :symbols}}' yarn webpack-app-build diff --git a/AGENTS.md b/AGENTS.md index 32dfe19279..3150b4caa9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,7 +5,7 @@ - `src/main/` contains core application logic. - `src/main/mobile/` is the mobile app code. - `src/main/frontend/components/` houses UI components. - - `src/main/frontend/inference_worker/` and `src/main/frontend/worker/` hold webworker code, including RTC in `src/main/frontend/worker/rtc/`. + - `src/main/frontend/worker/` holds webworker code, including RTC in `src/main/frontend/worker/rtc/`. - `src/electron/` is Electron-specific code. - `src/test/` contains unit tests. - `deps/` contains internal dependencies/modules. diff --git a/deps/cli/src/logseq/cli/common/mcp/tools.cljs b/deps/cli/src/logseq/cli/common/mcp/tools.cljs index f9f8ef838e..eba43a1da0 100644 --- a/deps/cli/src/logseq/cli/common/mcp/tools.cljs +++ b/deps/cli/src/logseq/cli/common/mcp/tools.cljs @@ -27,7 +27,7 @@ (cond-> (into {} e) true (dissoc e :block/tags :block/order :block/refs :block/name :db/index - :logseq.property.embedding/hnsw-label-updated-at :logseq.property/default-value) + :logseq.property/default-value) true (update :block/uuid str) (:logseq.property/classes e) @@ -46,8 +46,7 @@ (if expand (cond-> (into {} e) true - (dissoc e :block/tags :block/order :block/refs :block/name - :logseq.property.embedding/hnsw-label-updated-at) + (dissoc e :block/tags :block/order :block/refs :block/name) true (update :block/uuid str) (:logseq.property.class/extends e) @@ -75,7 +74,7 @@ [m] (->> (remove (fn [[k _v]] (or (= "block.temp" (namespace k)) - (contains? #{:logseq.property.embedding/hnsw-label-updated-at :block/tx-id} k))) m) + (contains? #{:block/tx-id} k))) m) (into {}))) (defn get-page-data @@ -395,4 +394,4 @@ [conn operations* {:keys [dry-run] :as opts}] (let [import-edn (build-upsert-nodes-edn @conn operations*)] (when-not dry-run (import-edn-data conn import-edn)) - (summarize-upsert-operations operations* opts))) \ No newline at end of file + (summarize-upsert-operations operations* opts))) diff --git a/deps/db/src/logseq/db/common/initial_data.cljs b/deps/db/src/logseq/db/common/initial_data.cljs index fa1b8db2ee..f8bece5cbe 100644 --- a/deps/db/src/logseq/db/common/initial_data.cljs +++ b/deps/db/src/logseq/db/common/initial_data.cljs @@ -333,7 +333,6 @@ :logseq.kv/graph-remote? :logseq.kv/latest-code-lang :logseq.kv/graph-backup-folder - :logseq.kv/graph-text-embedding-model-name :logseq.property/empty-placeholder]) favorites (get-favorites db) recent-updated-pages (let [pages (get-recent-updated-pages db)] diff --git a/deps/db/src/logseq/db/frontend/kv_entity.cljs b/deps/db/src/logseq/db/frontend/kv_entity.cljs index 1cd3adedb4..21d38e8320 100644 --- a/deps/db/src/logseq/db/frontend/kv_entity.cljs +++ b/deps/db/src/logseq/db/frontend/kv_entity.cljs @@ -29,8 +29,5 @@ RTC won't start when major-schema-versions don't match" :logseq.kv/graph-last-gc-at {:doc "Last time graph gc at" :rtc {:rtc/ignore-entity-when-init-upload true}} - - :logseq.kv/graph-text-embedding-model-name {:doc "Graph's text-embedding model name" - :rtc {:rtc/ignore-entity-when-init-upload true}} :logseq.kv/graph-rtc-e2ee? {:doc "true if it's a rtc graph with E2EE enabled"} :logseq.kv/graph-remote? {:doc "true if it's a remote graph"}))) diff --git a/deps/db/src/logseq/db/frontend/property.cljs b/deps/db/src/logseq/db/frontend/property.cljs index 3d3e8a8d73..9d02308b1c 100644 --- a/deps/db/src/logseq/db/frontend/property.cljs +++ b/deps/db/src/logseq/db/frontend/property.cljs @@ -660,13 +660,7 @@ :logseq.property.sync/large-title-object {:title "Reference to large block title stored in remote object storage" :schema {:type :map :public? false - :hide? true}} - :logseq.property.embedding/hnsw-label-updated-at {:title "HNSW label updated-at" - :schema {:type :datetime - :public? false - :hide? true} - :queryable? false - :rtc property-ignore-rtc}))) + :hide? true}}))) (def db-attribute-properties "Internal properties that are also db schema attributes" @@ -718,7 +712,7 @@ "logseq.property.linked-references" "logseq.property.asset" "logseq.property.table" "logseq.property.node" "logseq.property.code" "logseq.property.repeat" "logseq.property.journal" "logseq.property.class" "logseq.property.view" - "logseq.property.user" "logseq.property.history" "logseq.property.embedding" + "logseq.property.user" "logseq.property.history" "logseq.property.reaction" "logseq.property.sync" "logseq.property.publish" "logseq.property.recycle"}) diff --git a/deps/db/src/logseq/db/sqlite/export.cljs b/deps/db/src/logseq/db/sqlite/export.cljs index df7af776e5..63bea95746 100644 --- a/deps/db/src/logseq/db/sqlite/export.cljs +++ b/deps/db/src/logseq/db/sqlite/export.cljs @@ -85,7 +85,7 @@ (merge (select-keys pvalue [:block/created-at :block/updated-at]))) property-value-content'))) -(defonce ignored-properties [:logseq.property/created-by-ref :logseq.property.embedding/hnsw-label-updated-at]) +(defonce ignored-properties [:logseq.property/created-by-ref]) ;; buildable-properties and build-blocks-export depend on each other (declare build-blocks-export) diff --git a/deps/db/src/logseq/db/sqlite/util.cljs b/deps/db/src/logseq/db/sqlite/util.cljs index a39747c277..2e8d540e3f 100644 --- a/deps/db/src/logseq/db/sqlite/util.cljs +++ b/deps/db/src/logseq/db/sqlite/util.cljs @@ -142,5 +142,4 @@ :logseq.kv/graph-local-tx ;rtc related :logseq.kv/remote-schema-version ;rtc related :logseq.kv/graph-rtc-e2ee? ;rtc related - :logseq.kv/graph-text-embedding-model-name ;embedding ]))) diff --git a/deps/outliner/src/logseq/outliner/op/construct.cljc b/deps/outliner/src/logseq/outliner/op/construct.cljc index e43cfb79c0..575a0c297c 100644 --- a/deps/outliner/src/logseq/outliner/op/construct.cljc +++ b/deps/outliner/src/logseq/outliner/op/construct.cljc @@ -48,8 +48,7 @@ :block.temp/ast-body :block.temp/load-status :block.temp/has-children? - :logseq.property/created-by-ref - :logseq.property.embedding/hnsw-label-updated-at}) + :logseq.property/created-by-ref}) (def ^:api rebase-refs-key :block.temp/sync-rebase-refs) (def ^:api rebase-created-refs-key :block.temp/sync-created-refs) diff --git a/deps/outliner/src/logseq/outliner/pipeline.cljs b/deps/outliner/src/logseq/outliner/pipeline.cljs index 27cc6e9f39..341ff2d008 100644 --- a/deps/outliner/src/logseq/outliner/pipeline.cljs +++ b/deps/outliner/src/logseq/outliner/pipeline.cljs @@ -65,7 +65,7 @@ (let [property-ent (d/entity db property) allowed-datetime? (and (= :datetime (:logseq.property/type property-ent)) ;; Only allow a few built-in properties as some built-in properties - ;; like :logseq.property.embedding/hnsw-label-updated-at create undesirable refs + ;; can create undesirable refs (if (db-property/internal-property? (:db/ident property-ent)) (contains? #{:logseq.property/scheduled :logseq.property/deadline} (:db/ident property-ent)) ;; All user properties are allowed to create refs but not plugin properties diff --git a/package.json b/package.json index 4c77d9c92d..53b46bf985 100644 --- a/package.json +++ b/package.json @@ -71,28 +71,28 @@ "gulp:buildMobile": "cross-env NODE_ENV=production gulp buildMobile", "css:build": "postcss tailwind.all.css -o static/css/style.css --verbose --env production", "css:watch": "cross-env TAILWIND_MODE=watch postcss tailwind.all.css -o static/css/style.css --verbose --watch", - "cljs:watch": "clojure -M:cljs watch app db-worker inference-worker electron", + "cljs:watch": "clojure -M:cljs watch app db-worker electron", "gulp:mobile-watch": "gulp watchMobile", "css:mobile-build": "postcss tailwind.mobile.css -o static/mobile/css/style.css --verbose --env production", "css:mobile-watch": "cross-env TAILWIND_MODE=watch postcss tailwind.mobile.css -o static/mobile/css/style.css --verbose --watch", "cljs:mobile-watch": "clojure -M:cljs watch mobile db-worker --config-merge \"{:output-dir \\\"./static/mobile/js\\\" :asset-path \\\"/static/mobile/js\\\" :release {:asset-path \\\"http://localhost\\\"}}\"", "cljs:release-mobile": "clojure -M:cljs release mobile db-worker --config-merge \"{:output-dir \\\"./static/mobile/js\\\" :asset-path \\\"/static/mobile/js\\\" :release {:asset-path \\\"http://localhost\\\"}}\"", - "cljs:dev-watch": "clojure -M:cljs watch app db-worker inference-worker electron mobile", - "cljs:app-watch": "clojure -M:cljs watch app db-worker inference-worker", - "cljs:electron-watch": "clojure -M:cljs watch app db-worker inference-worker electron test --config-merge \"{:asset-path \\\"./js\\\"}\"", - "cljs:release": "clojure -M:cljs release app db-worker inference-worker publishing electron", - "cljs:release-electron": "clojure -M:cljs release app db-worker inference-worker electron --debug && clojure -M:cljs release publishing", - "cljs:release-app": "clojure -M:cljs release app db-worker inference-worker", + "cljs:dev-watch": "clojure -M:cljs watch app db-worker electron mobile", + "cljs:app-watch": "clojure -M:cljs watch app db-worker", + "cljs:electron-watch": "clojure -M:cljs watch app db-worker electron test --config-merge \"{:asset-path \\\"./js\\\"}\"", + "cljs:release": "clojure -M:cljs release app db-worker publishing electron", + "cljs:release-electron": "clojure -M:cljs release app db-worker electron --debug && clojure -M:cljs release publishing", + "cljs:release-app": "clojure -M:cljs release app db-worker", "cljs:release-publishing": "clojure -M:cljs release app publishing", "cljs:test": "clojure -M:test compile test", "cljs:run-test": "node static/tests.js -r \"^(?!logseq.db-sync.).*\" -e fix-me", "cljs:test-no-worker": "clojure -M:test compile test-no-worker", "cljs:run-test-no-worker": "node static/tests-no-worker.js", - "cljs:dev-release-app": "clojure -M:cljs release app db-worker inference-worker --config-merge \"{:closure-defines {frontend.config/DEV-RELEASE true}}\"", - "cljs:dev-release-electron": "clojure -M:cljs release app db-worker inference-worker electron --debug --config-merge \"{:closure-defines {frontend.config/DEV-RELEASE true}}\" && clojure -M:cljs release publishing", - "cljs:debug": "clojure -M:cljs release app db-worker inference-worker --debug", - "cljs:report": "clojure -M:cljs run shadow.cljs.build-report app db-worker inference-worker report.html", - "cljs:build-electron": "clojure -A:cljs compile app db-worker inference-worker electron", + "cljs:dev-release-app": "clojure -M:cljs release app db-worker --config-merge \"{:closure-defines {frontend.config/DEV-RELEASE true}}\"", + "cljs:dev-release-electron": "clojure -M:cljs release app db-worker electron --debug --config-merge \"{:closure-defines {frontend.config/DEV-RELEASE true}}\" && clojure -M:cljs release publishing", + "cljs:debug": "clojure -M:cljs release app db-worker --debug", + "cljs:report": "clojure -M:cljs run shadow.cljs.build-report app db-worker report.html", + "cljs:build-electron": "clojure -A:cljs compile app db-worker electron", "cljs:lint": "clojure -M:clj-kondo --parallel --lint src --cache false", "ios:dev": "cross-env PLATFORM=ios gulp cap", "android:dev": "cross-env PLATFORM=android gulp cap", @@ -125,7 +125,6 @@ "@emoji-mart/react": "^1.1.1", "@glidejs/glide": "^3.7.1", "@highlightjs/cdn-assets": "11.11.1", - "@huggingface/transformers": "^3.8.1", "@isomorphic-git/lightning-fs": "^4.6.2", "@js-joda/core": "3.2.0", "@js-joda/locale_en-us": "3.1.1", @@ -147,7 +146,6 @@ "fs-extra": "^11.3.4", "fuse.js": "7.1.0", "graphology": "0.20.0", - "hnswlib-wasm": "^0.8.2", "html2canvas": "^1.4.1", "inter-ui": "^4.1.1", "interactjs": "^1.10.27", diff --git a/prompts/review.md b/prompts/review.md index 99619bccc2..459db24d2f 100644 --- a/prompts/review.md +++ b/prompts/review.md @@ -20,7 +20,7 @@ You're Clojure(script) expert, you're responsible to check those common errors: - NOTE: `log/` function takes key-value pairs as arguments - After adding a new property in `logseq.db.frontend.property/built-in-properties`, you need to add a corresponding migration in `frontend.worker.db.migrate/schema-version->updates`. - - e.g. `["65.9" {:properties [:logseq.property.embedding/hnsw-label-updated-at]}]` + - e.g. `["65.10" {:properties [:block/journal-day]}]` - If common keywords are added or modified, make corresponding changes in their definitions. - common keywords are defined by `logseq.common.defkeywords/defkeywords` diff --git a/scripts/src/logseq/tasks/dev.clj b/scripts/src/logseq/tasks/dev.clj index 0e3f70a913..c9cf9633e5 100644 --- a/scripts/src/logseq/tasks/dev.clj +++ b/scripts/src/logseq/tasks/dev.clj @@ -81,7 +81,7 @@ (fs/glob "." "{src/main,deps/graph-parser/src}/**")))))] (do (println "Building publishing js asset...") - (shell "clojure -M:cljs release publishing db-worker inference-worker")) + (shell "clojure -M:cljs release publishing db-worker")) (println "Publishing js asset is up to date"))) (defn publishing-backend diff --git a/scripts/src/logseq/tasks/dev/lint.clj b/scripts/src/logseq/tasks/dev/lint.clj index 3a6e0da472..e199091de2 100644 --- a/scripts/src/logseq/tasks/dev/lint.clj +++ b/scripts/src/logseq/tasks/dev/lint.clj @@ -46,10 +46,10 @@ [] (let [res (shell {:out :string} "git grep -h" "\\[frontend.*:as" - "src/main/frontend/worker" "src/main/frontend/worker_common" "src/main/frontend/inference_worker") + "src/main/frontend/worker" "src/main/frontend/worker_common") req-lines (->> (:out res) string/split-lines - (remove #(re-find #"frontend\.worker|frontend\.common|frontend\.inference-worker" %)))] + (remove #(re-find #"frontend\.worker|frontend\.common" %)))] (if (seq req-lines) (do @@ -65,8 +65,7 @@ "\\[frontend.worker.*:" "--" "src/main/frontend") ;; allow reset-file b/c it's only affects tests allowed-exceptions #{"src/main/frontend/handler/file_based/file.cljs: [frontend.worker.file.reset :as file-reset]"} - excluded-path-prefixes ["src/main/frontend/worker/" - "src/main/frontend/inference_worker/"] + excluded-path-prefixes ["src/main/frontend/worker/"] invalid-lines (when (= 0 (:exit res)) (->> (:out res) string/split-lines diff --git a/shadow-cljs.edn b/shadow-cljs.edn index 65a7453f24..4ea5b33ff4 100644 --- a/shadow-cljs.edn +++ b/shadow-cljs.edn @@ -87,32 +87,6 @@ shadow.remote.runtime.cljs.browser] :loader-mode :eval}} - :inference-worker {:target :browser - :module-loader true - :js-options {:js-provider :external - :external-index "target/inference-worker.js" - :external-index-format :esm - :entry-keys ["module" "browser" "main"] - :export-conditions ["module" "import", "browser" "require" "default"]} - :modules {:inference-worker - {:init-fn frontend.inference-worker.inference-worker/init - :web-worker true - :prepend-js "importScripts('inference-worker-bundle.js');\n"}} - - :output-dir "./static/js" - :asset-path "/static/js" - :release {:asset-path "https://asset.logseq.com/static/js"} - :compiler-options {:infer-externs :auto - :output-feature-set :es-next-in - :source-map true - :externs ["datascript/externs.js" - "externs.js"] - :warnings {:fn-deprecated false - :redef false}} - :closure-defines {goog.debug.LOGGING_ENABLED true} - :devtools {:watch-path "/static" - :preloads [devtools.preload - shadow.remote.runtime.cljs.browser]}} :mobile {:target :browser :module-loader true :js-options {:ignore-asset-requires true diff --git a/src/main/frontend/common/thread_api.cljc b/src/main/frontend/common/thread_api.cljc index 31f84b1877..1fb109a206 100644 --- a/src/main/frontend/common/thread_api.cljc +++ b/src/main/frontend/common/thread_api.cljc @@ -39,10 +39,8 @@ (let [qkw (keyword qualified-kw-str)] (vswap! *profile update qkw inc) (if-let [f (@*thread-apis qkw)] - (let [result (if (= qkw :thread-api/set-infer-worker-proxy) - (f args-transit-str-or-args-array) - (apply f (cond-> args-transit-str-or-args-array - (not direct-pass?) ldb/read-transit-str))) + (let [result (apply f (cond-> args-transit-str-or-args-array + (not direct-pass?) ldb/read-transit-str)) result-promise (if (fn? result) ;; missionary task is a fn (js/Promise. result) diff --git a/src/main/frontend/components/header.cljs b/src/main/frontend/components/header.cljs index 6a3ae4afbe..bb6d9e26bf 100644 --- a/src/main/frontend/components/header.cljs +++ b/src/main/frontend/components/header.cljs @@ -19,7 +19,6 @@ [frontend.db :as db] [frontend.handler :as handler] [frontend.handler.db-based.rtc-flows :as rtc-flows] - [frontend.handler.db-based.vector-search-flows :as vector-search-flows] [frontend.handler.page :as page-handler] [frontend.handler.plugin :as plugin-handler] [frontend.handler.route :as route-handler] @@ -353,30 +352,6 @@ (:block/uuid page) {:header? true})]]))) -(rum/defc semantic-search-progressing - [repo] - (let [[vec-search-state set-vec-search-state] (hooks/use-state nil) - {:keys [indexing?]} (get-in vec-search-state [:repo->index-info repo])] - (hooks/use-effect! - (fn [] - (c.m/run-task - ::update-vec-search-state - (m/reduce - (fn [_ v] - (set-vec-search-state v)) - (m/ap - (m/?> vector-search-flows/infer-worker-ready-flow) - (c.m/ vector-search-flows/vector-search-state-flow))) - :succ (constantly nil))) - []) - (when indexing? - (shui/button - {:class "opacity-50" - :variant :ghost - :size :sm} - "Embedding...")))) - (rum/defc ^:large-vars/cleanup-todo header-aux < rum/reactive [{:keys [current-repo default-home new-block-mode]}] (let [electron-mac? (and util/mac? (util/electron?)) @@ -440,8 +415,6 @@ (when (user-handler/logged-in?) (rtc-indicator/uploading-detail)) - (semantic-search-progressing current-repo) - (when (and (not= (state/get-current-route) :home) (not custom-home-page?)) (home-button)) diff --git a/src/main/frontend/components/right_sidebar.cljs b/src/main/frontend/components/right_sidebar.cljs index b792a86657..5b7fb742cc 100644 --- a/src/main/frontend/components/right_sidebar.cljs +++ b/src/main/frontend/components/right_sidebar.cljs @@ -8,7 +8,6 @@ [frontend.components.page :as page] [frontend.components.profiler :as profiler] [frontend.components.shortcut-help :as shortcut-help] - [frontend.components.vector-search.sidebar :as vector-search] [frontend.config :as config] [frontend.context.i18n :refer [t]] [frontend.date :as date] @@ -157,10 +156,6 @@ [[:.flex.items-center (ui/icon "cloud" {:class "text-md mr-2"}) "(Dev) Profiler"] (profiler/profiler)] - :vector-search - [[:.flex.items-center (ui/icon "file-search" {:class "text-md mr-2"}) "(Dev) VectorSearch"] - (vector-search/vector-search-sidebar)] - ["" [:span]]))) (p/catch (fn [error] (js/console.error error))))) @@ -180,9 +175,6 @@ developer-mode? (conj {:db-id "undo-redo" :block-type :undo-redo :label "(Dev) Undo/Redo"}) - developer-mode? - (conj {:db-id "vector-search" :block-type :vector-search :label "(Dev) vector-search"}) - developer-mode? (conj {:db-id "profiler" :block-type :profiler :label "(Dev) Profiler"}))) diff --git a/src/main/frontend/components/settings.cljs b/src/main/frontend/components/settings.cljs index f6c90b4eda..3b993b146a 100644 --- a/src/main/frontend/components/settings.cljs +++ b/src/main/frontend/components/settings.cljs @@ -1,6 +1,5 @@ (ns frontend.components.settings (:require [clojure.string :as string] - [clojure.walk :as walk] [electron.ipc :as ipc] [frontend.colors :as colors] [frontend.common.missionary :as c.m] @@ -14,7 +13,6 @@ [frontend.dicts :as dicts] [frontend.handler.config :as config-handler] [frontend.handler.db-based.sync :as rtc-handler] - [frontend.handler.db-based.vector-search-flows :as vector-search-flows] [frontend.handler.global-config :as global-config-handler] [frontend.handler.notification :as notification] [frontend.handler.plugin :as plugin-handler] @@ -24,7 +22,6 @@ [frontend.mobile.util :as mobile-util] [frontend.modules.instrumentation.core :as instrument] [frontend.modules.shortcut.data-helper :as shortcut-helper] - [frontend.persist-db.browser :as db-browser] [frontend.spec.storage :as storage-spec] [frontend.state :as state] [frontend.storage :as storage] @@ -1164,94 +1161,9 @@ (rum/defc settings-ai [] - (let [[model-info set-model-info] (hooks/use-state nil) - [load-model-progress set-load-model-progress] (hooks/use-state nil) - {:keys [status]} load-model-progress - repo (state/get-current-repo) - current-model (:graph-text-embedding-model-name model-info) - [webgpu? set-webgpu?] (hooks/use-state nil)] - (hooks/use-effect! - (fn [] - (p/let [webgpu? (db-browser/ vector-search-flows/infer-worker-ready-flow) - (let [model-info (c.m/ - {:on-value-change (fn [model-name] - (c.m/run-task - ::load-model - (m/sp - (set-model-info (assoc model-info :graph-text-embedding-model-name model-name)) - (c.m/ vector-search-flows/infer-worker-ready-flow) - (c.m/ vector-search-flows/vector-search-state-flow))) - :succ (constantly nil))) - []) - (hooks/use-effect! - (fn [] - (c.m/run-task - ::update-load-model-progress - (m/reduce - (fn [_ v] (set-load-model-progress v)) - vector-search-flows/load-model-progress-flow) - :succ (constantly nil))) - []) - (hooks/use-effect! - (fn [] - (c.m/run-task - ::fetch-model-info - (m/reduce - (constantly nil) - (m/ap - (m/?> vector-search-flows/infer-worker-ready-flow) - (let [model-info (c.m/ (c.m/index-info repo]) - :load-model-progress load-model-progress)] - [:pre.select-text - (with-out-str - (fipp/pprint state-map {:width 10}))]) - [:hr] - [:b "Actions"] - [:div - (shui/button - {:size :sm - :class "mx-2" - :on-click (fn [_] - (state/index-info repo :indexing?]) - (shui/button - {:size :sm - :class "mx-2" - :on-click (fn [_] - (state/> properties - (remove #{:logseq.property.embedding/hnsw-label-updated-at}) (map db/entity) (ldb/sort-by-order) ((fn [cs] (build-columns config cs {:add-tags-column? false diff --git a/src/main/frontend/core.cljs b/src/main/frontend/core.cljs index 4a75cda7ea..cff73af1fe 100644 --- a/src/main/frontend/core.cljs +++ b/src/main/frontend/core.cljs @@ -8,7 +8,6 @@ [frontend.config :as config] [frontend.handler :as handler] [frontend.handler.db-based.rtc-background-tasks] - [frontend.handler.db-based.vector-search-background-tasks] [frontend.handler.plugin :as plugin-handler] [frontend.handler.route :as route-handler] [frontend.log] diff --git a/src/main/frontend/handler.cljs b/src/main/frontend/handler.cljs index 2d7d8a08fc..4f9890a139 100644 --- a/src/main/frontend/handler.cljs +++ b/src/main/frontend/handler.cljs @@ -16,7 +16,6 @@ [frontend.db.restore :as db-restore] [frontend.error :as error] [frontend.handler.command-palette :as command-palette] - [frontend.handler.db-based.vector-search-flows :as vector-search-flows] [frontend.handler.e2ee] [frontend.handler.events :as events] [frontend.handler.events.export] @@ -171,15 +170,7 @@ (p/finally (fn [] (state/set-db-restoring! false) (p/resolve! state/app-ready-promise true) - (log/info ::app-init-spent-time (- (util/time-ms) t1)) - (when-not (util/mobile?) - (p/let [webgpu-available? (db-browser/js (keys infer-worker.text-embedding/available-embedding-models))) - - (text-embedding - [_this text-coll] - (p/chain - (infer-worker.text-embedding/js)) - - (text-embedding+store! - ;; return labels(js array) - [_this repo text-array labels replace-deleted?] - (p/chain - (js/Promise. (infer-worker.text-embedding/task--text-embedding&store! repo text-array labels replace-deleted?)) - clj->js)) - - (delete-labels - [_this repo labels] - (infer-worker.text-embedding/delete-items repo labels)) - - (force-reset-index! - [_this repo] - (js/Promise. (infer-worker.text-embedding/task--force-reset-index! repo))) - - (write-index! - [_this repo] - (js/Promise. (infer-worker.text-embedding/task--write-index! repo))) - - (search - [_this repo query-string nums-neighbors] - (infer-worker.text-embedding/js (infer-worker.text-embedding/index-info repo)))) - -(defn init - []) - -(.addEventListener js/self "connect" - (fn [^js e] - (glogi-console/install!) - (let [port (first (.-ports e)) - ^js obj (InferenceWorker.)] - (reset! infer-worker.text-embedding/*port port) - (.start port) - (Comlink/expose obj port)))) diff --git a/src/main/frontend/inference_worker/state.cljs b/src/main/frontend/inference_worker/state.cljs deleted file mode 100644 index 8b34e76e8c..0000000000 --- a/src/main/frontend/inference_worker/state.cljs +++ /dev/null @@ -1,10 +0,0 @@ -(ns frontend.inference-worker.state - "State hub for inference-worker") - -(defonce *hnswlib (atom nil)) - -;;repo -> index -(defonce *hnsw-index (atom {})) - -(defonce *extractor (atom nil)) -(defonce *model-name+config (atom nil)) diff --git a/src/main/frontend/inference_worker/text_embedding.cljs b/src/main/frontend/inference_worker/text_embedding.cljs deleted file mode 100644 index d7d34614c4..0000000000 --- a/src/main/frontend/inference_worker/text_embedding.cljs +++ /dev/null @@ -1,238 +0,0 @@ -(ns frontend.inference-worker.text-embedding - "text embedding fns" - (:require ["@huggingface/transformers" :refer [pipeline]] - ["hnswlib-wasm" :refer [loadHnswlib]] - [clojure.data :as data] - [frontend.common.missionary :as c.m] - [frontend.inference-worker.state :as infer-worker.state] - [frontend.worker-common.util :as worker-util] - [lambdaisland.glogi :as log] - [logseq.common.config :as common-config] - [missionary.core :as m] - [promesa.core :as p])) - -(add-watch infer-worker.state/*hnsw-index :delete-obj-when-dissoc - (fn [_ _ o n] - (let [[old-only] (data/diff o n)] - (doseq [[repo ^js hnsw-index] old-only] - (when hnsw-index - (log/info :delete-hnsw-index repo) - (.delete hnsw-index)))))) - -(defonce *port (atom nil)) - -(def ^:private embedding-opts #js{"pooling" "mean" "normalize" true}) - -(def ^:private init-max-elems 100) - -(defn- split-into-chunks - [js-array chunk-size] - (let [length (alength js-array) - result (array)] - (loop [i 0] - (when (< i length) - (.push result (.slice js-array i (+ i chunk-size))) - (recur (+ i chunk-size)))) - result)) - -(defn- init-index! - [^js hnsw] - (.initIndex hnsw init-max-elems 16 200 100) - (.setEfSearch hnsw 32)) - -(defn- ^js get-hnsw-index - [repo] - (when repo - (or (@infer-worker.state/*hnsw-index repo) - (let [hnsw-ctor (.-HierarchicalNSW ^js @infer-worker.state/*hnswlib) - hnsw (new hnsw-ctor "cosine" (or (:dims (:hnsw-config (second @infer-worker.state/*model-name+config))) 384) "") - file-exists? (.checkFileExists (.-EmscriptenFileSystemManager ^js @infer-worker.state/*hnswlib) repo)] - (when file-exists? - (.readIndex hnsw repo init-max-elems) - (swap! infer-worker.state/*hnsw-index assoc repo hnsw) - hnsw))))) - -(defn- ^js new-hnsw-index! - [repo] - (when (get-hnsw-index repo) - (swap! infer-worker.state/*hnsw-index dissoc repo)) - (let [hnsw-ctor (.-HierarchicalNSW ^js @infer-worker.state/*hnswlib) - hnsw (new hnsw-ctor "cosine" (or (:dims (:hnsw-config (second @infer-worker.state/*model-name+config))) 384) "")] - (init-index! hnsw) - (swap! infer-worker.state/*hnsw-index assoc repo hnsw) - hnsw)) - -(defn- model-loaded? - [] - (and @infer-worker.state/*extractor - @infer-worker.state/*model-name+config)) - -(defn = (+ add-count current-count) max-elems) - (let [new-size (+ current-count (max (* 2 add-count) current-count))] - (log/info :hnsw-resize {:from current-count :to new-size}) - (.resizeIndex hnsw new-size))) - ;; (.addItems hnsw data-coll labels replace-deleted?) - (dorun - (mapcat - (fn [embedding label] - (assert (and embedding label) {:embedding embedding - :label label}) - (.addPoint hnsw embedding label replace-deleted?)) - data-coll - labels)))) - -(defn delete-items - [repo labels] - (when-let [hnsw (get-hnsw-index repo)] - (.markDeleteItems hnsw (into-array labels)))) - -(defn task--text-embedding&store! - "return labels(js-array)" - [repo text-array labels replace-deleted?] - (m/sp - (when (model-loaded?) - (let [hnsw (or (get-hnsw-index repo) (new-hnsw-index! repo)) - {:keys [data _type dims _size]} (worker-util/profile :js - (-> (:tf-config config) - (assoc "device" "webgpu") - (assoc "progress_callback" #(reset! *load-model-progress %)))))] - (reset! infer-worker.state/*extractor extractor) - (reset! infer-worker.state/*model-name+config [model-name config]) - true)))) - -(defn > (d/datoms @conn :avet :logseq.property.embedding/hnsw-label-updated-at) - (map (fn [d] - [:db/retract (:e d) :logseq.property.embedding/hnsw-label-updated-at])))) - ;; Mark vector embedding - mark-embedding-tx-data (->> (keep (fn [datom] (when (and (= :block/title (:a datom)) - (:added datom) - (not (string/blank? (:block/title (d/entity @conn (:e datom)))))) - (:e datom))) tx-data) - ;; Mark block embedding to be computed - (map (fn [id] [:db/add id :logseq.property.embedding/hnsw-label-updated-at 0]))) - tx-data' (concat remove-old-hnsw-tx-data mark-embedding-tx-data)] - (when (seq tx-data) - (ldb/transact! conn tx-data' {:skip-validate-db? true - :mark-embedding? true})))) - (defn listen-db-changes! [repo conn & {:keys [handler-keys]}] (let [handlers (if (seq handler-keys) @@ -92,10 +70,9 @@ (d/unlisten! conn ::listen-db-changes!) (d/listen! conn ::listen-db-changes! (fn listen-db-changes!-inner - [{:keys [tx-data tx-meta] :as tx-report}] + [{:keys [tx-data] :as tx-report}] (when-not (:batch-tx? @conn) - (remove-old-embeddings-and-reset-new-updates! conn tx-data tx-meta) - (when (and (seq tx-data) (not (:mark-embedding? tx-meta))) + (when (seq tx-data) (let [tx-report' (if sync-db-to-main-thread? (sync-db-to-main-thread repo conn tx-report) tx-report) diff --git a/src/main/frontend/worker/db_worker.cljs b/src/main/frontend/worker/db_worker.cljs index 7e3017bf70..cc53d25928 100644 --- a/src/main/frontend/worker/db_worker.cljs +++ b/src/main/frontend/worker/db_worker.cljs @@ -17,7 +17,6 @@ [frontend.worker.db.fix :as db-fix] [frontend.worker.db.migrate :as db-migrate] [frontend.worker.db.validate :as worker-db-validate] - [frontend.worker.embedding :as embedding] [frontend.worker.export :as worker-export] [frontend.worker.pipeline :as worker-pipeline] [frontend.worker.publish] @@ -509,11 +508,6 @@ [repo graph-id graph-e2ee?] (sync-download/download-graph-by-id! repo graph-id graph-e2ee?)) -(def-thread-api :thread-api/set-infer-worker-proxy - [infer-worker-proxy] - (reset! worker-state/*infer-worker infer-worker-proxy) - nil) - ;; [graph service] (defonce *service (atom [])) @@ -996,34 +990,6 @@ (gc-sqlite-dbs! db client-ops conn {:full-gc? true}) nil))) -(def-thread-api :thread-api/vec-search-embedding-model-info - [repo] - (embedding/task--embedding-model-info repo)) - -(def-thread-api :thread-api/vec-search-init-embedding-model - [repo] - (js/Promise. (embedding/task--init-embedding-model repo))) - -(def-thread-api :thread-api/vec-search-load-model - [repo model-name] - (js/Promise. (embedding/task--load-model repo model-name))) - -(def-thread-api :thread-api/vec-search-embedding-graph - [repo opts] - (embedding/embedding-graph! repo opts)) - -(def-thread-api :thread-api/vec-search-search - [repo query-string nums-neighbors] - (embedding/task--search repo query-string nums-neighbors)) - -(def-thread-api :thread-api/vec-search-cancel-indexing - [repo] - (embedding/cancel-indexing repo)) - -(def-thread-api :thread-api/vec-search-update-index-info - [repo] - (js/Promise. (embedding/task--update-index-info! repo))) - (def-thread-api :thread-api/mobile-logs [] @worker-state/*log) @@ -1141,9 +1107,8 @@ ;; wait for service ready (js-invoke (:proxy service) k args))) - (or - (contains? #{:thread-api/set-infer-worker-proxy :thread-api/sync-app-state} method-k) - (nil? service)) + (or (= :thread-api/sync-app-state method-k) + (nil? service)) ;; only proceed down this branch before shared-service is initialized (apply f args) diff --git a/src/main/frontend/worker/embedding.cljs b/src/main/frontend/worker/embedding.cljs deleted file mode 100644 index 63b07af839..0000000000 --- a/src/main/frontend/worker/embedding.cljs +++ /dev/null @@ -1,274 +0,0 @@ -(ns frontend.worker.embedding - "Fns about text-embedding, add/delete/search items in hnsw" - (:require [clojure.string :as string] - [datascript.core :as d] - [frontend.common.missionary :as c.m] - [frontend.worker-common.util :as worker-util] - [frontend.worker.state :as worker-state] - [lambdaisland.glogi :as log] - [logseq.common.config :as common-config] - [logseq.db :as ldb] - [logseq.db.frontend.content :as db-content] - [missionary.core :as m])) - -;;; TODOs: -;;; - [x] add :logseq.property/description into text-to-embedding -;;; - [x] add tags to text-to-embedding -;;; - [x] check webgpu available, transformers.js is slow without webgpu(the difference is ~70 times) -;;; - [x] expose index-state to ui - -(def ^:private empty-vector-search-state - {:repo->index-info {} ;; repo->index-info - :repo->canceler {} ;; repo->canceler - }) - -(def ^:private vector-search-state-keys (set (keys empty-vector-search-state))) - -(def ^:private *vector-search-state (atom empty-vector-search-state - :validator - (fn [v] (= vector-search-state-keys (set (keys v)))))) - -(defn- reset-*vector-search-state! - [repo & {:keys [index-info canceler]}] - (reset! *vector-search-state - (cond-> @*vector-search-state - index-info (assoc :repo->index-info {repo index-info}) - canceler (assoc-in [:repo->canceler repo] canceler))) - nil) - -(defn cancel-indexing - [repo] - (when-let [canceler (get-in @*vector-search-state [:repo->canceler repo])] - (canceler) - (swap! *vector-search-state assoc-in [:repo->canceler repo] nil) - (swap! *vector-search-state assoc-in [:repo->index-info repo :indexing?] false) - nil)) - -(defn- indexing? - [repo] - (get-in @*vector-search-state [:repo->index-info repo :indexing?])) - -(defn- stale-block-filter-preds - "When `reset?`, ignore :logseq.property.embedding/hnsw-label-updated-at in block" - [reset?] - (let [preds (cond->> (list (fn [b] - (let [title (:block/title b)] - (and (not (string/blank? title)) - (not (ldb/hidden? b)) - (nil? (:logseq.property/view-for b)) - (not (keyword-identical? - :logseq.property/description - (:db/ident (:logseq.property/created-from-property b)))))))) - - (not reset?) - (cons (fn [b] - (let [block-updated-at (:block/updated-at b) - hnsw-label-updated-at (:logseq.property.embedding/hnsw-label-updated-at b)] - (or (nil? hnsw-label-updated-at) - (> block-updated-at hnsw-label-updated-at))))))] - (apply every-pred preds))) - -(defn- stale-block-lazy-seq - [db reset?] - (let [datoms (if reset? - (rseq (d/index-range db :block/updated-at nil nil)) - (d/datoms db :avet :logseq.property.embedding/hnsw-label-updated-at 0))] - (->> datoms - (sequence - (comp (map #(d/entity db (:e %))) - (filter (stale-block-filter-preds reset?)) - (map (fn [b] - (assoc b :block.temp/text-to-embedding - (db-content/recur-replace-uuid-in-block-title b) - ;; FIXME: tags and properties can affect sorting - ;; (str (db-content/recur-replace-uuid-in-block-title b) - ;; (let [tags (->> (:block/tags b) - ;; (map :block/title))] - ;; (when (seq tags) - ;; (str " " (string/join ", " (map (fn [t] (str "#" t)) tags))))) - ;; (when-let [desc (:block/title (:logseq.property/description b))] - ;; (str "\nDescription: " desc))) - )))))))) -(defn- partition-by-text-size - [text-size] - (let [*current-size (volatile! 0) - *partition-index (volatile! 0)] - (partition-by - (fn [block] - (let [block-text-size (count (:block.temp/text-to-embedding block))] - (vswap! *current-size + block-text-size) - (if (>= text-size @*current-size) - @*partition-index - (do (vreset! *current-size block-text-size) - (vswap! *partition-index inc)))))))) - -(defn- labels-update-tx-data - [db e+updated-at-coll] - (let [es (map first e+updated-at-coll) - exist-es (set (keep - (fn [b] (when (:block/uuid b) (:db/id b))) - (d/pull-many db [:block/uuid :db/id] es)))] - (keep - (fn [[e updated-at]] - (when (contains? exist-es e) - [:db/add e :logseq.property.embedding/hnsw-label-updated-at updated-at])) - e+updated-at-coll))) - -(defn- task--update-index-info!* - ([repo ^js infer-worker] - (m/sp - (reset-*vector-search-state! repo :index-info - (merge (:index-info @*vector-search-state) - (js->clj (c.m/clj (c.m/ :logseq.property.embedding/hnsw-label-updated-at" - [repo reset-embedding?] - (m/sp - (when-let [^js infer-worker @worker-state/*infer-worker] - (when-let [conn (worker-state/get-datascript-conn repo)] - (let [stale-blocks (stale-block-lazy-seq @conn false)] - (when (seq stale-blocks) - (m/? (task--update-index-info!* repo infer-worker true)) - (when reset-embedding? - (c.m/ id 2147483647)) (map :db/id stale-block-chunk)) - (throw (ex-info "Wrong db/id" {:data (filter (fn [item] (> (:db/id item) 2147483647)) stale-block-chunk)}))) - _ (c.m/> - (d/datoms @conn :avet :block/title) - (map (fn [d] - [:db/add (:e d) :logseq.property.embedding/hnsw-label-updated-at 0])))] - (ldb/transact! conn mark-embedding-tx-data {:skip-refresh? true}))) - - (embedding-stale-blocks! repo reset-embedding?))))) - -(defn task--embedding-model-info - [repo] - (m/sp - (when-let [^js infer-worker @worker-state/*infer-worker] - (let [available-model-names (c.m/clj (c.m/> (map vector distances neighbors) - (keep (fn [[distance label]] - ;; (prn :debug :semantic-search-result - ;; :block (:block/title (d/entity @conn label)) - ;; :page? (ldb/page? (d/entity @conn label)) - ;; :distance distance - ;; :label label) - (when-not (or (js/isNaN distance) (>= distance 0.6) - (> label 2147483647)) - (when-let [block (d/entity @conn label)] - (when (:block/title block) - {:block block - :distance distance})))))))))))) - -(def ^:private vector-search-state-flow - (m/eduction - (map (fn [m] (dissoc m :repo->canceler))) - (c.m/throttle 300 (m/watch *vector-search-state)))) - -(when-not common-config/PUBLISHING ; NOTE: we may support vector-search in publishing mode later - (c.m/run-background-task - ::subscribe-state - (m/reduce - (fn [_ m] (worker-util/post-message :vector-search-sync-state m)) - vector-search-state-flow))) - -(comment - (def repo (frontend.worker.state/get-current-repo)) - (def conn (frontend.worker.state/get-datascript-conn (frontend.worker.state/get-current-repo))) - (.force-reset-index! @worker-state/*infer-worker repo) - ((task--embedding-stale-blocks! repo) prn js/console.log) - ((task--re-embedding-graph-data! repo) prn js/console.log) - - ((task--search repo "perf performance datomic stat" 10) prn js/console.log)) diff --git a/src/main/frontend/worker/publish.cljs b/src/main/frontend/worker/publish.cljs index 01e9d7cc31..9951acba46 100644 --- a/src/main/frontend/worker/publish.cljs +++ b/src/main/frontend/worker/publish.cljs @@ -208,7 +208,7 @@ (d/datoms db :eavt eid))) eids) (remove (fn [[_e a _v _tx _added]] - (contains? #{:block/tx-id :logseq.property.user/email :logseq.property.embedding/hnsw-label-updated-at} a)))) + (contains? #{:block/tx-id :logseq.property.user/email} a)))) datoms (if (entity-util/page? entity) raw-datoms (normalize-block-publish-datoms raw-datoms (set (map :db/id blocks)) (:db/id entity)))] diff --git a/src/main/frontend/worker/search.cljs b/src/main/frontend/worker/search.cljs index 6f00f13047..220835c471 100644 --- a/src/main/frontend/worker/search.cljs +++ b/src/main/frontend/worker/search.cljs @@ -6,7 +6,6 @@ [clojure.string :as string] [datascript.core :as d] [frontend.common.search-fuzzy :as fuzzy] - [frontend.worker.embedding :as embedding] [goog.object :as gobj] [logseq.common.config :as common-config] [logseq.common.util :as common-util] @@ -29,25 +28,6 @@ [repo] (swap! fuzzy-search-indices dissoc repo)) -;; Configuration for re-ranking -(def config - {:keyword-weight 0.9 - :semantic-weight 0.1}) - -(defn- log-score - [score] - (if (> score 2) - (js/Math.log score) - score)) - -;; Normalize scores to [0, 1] range using min-max normalization -(defn normalize-score [score min-score max-score] - (if (= min-score max-score) - 0.0 - (let [normalized (/ (log-score (- score min-score)) - (log-score (- max-score min-score)))] - (max 0.0 (min 1.0 normalized))))) - (defn- add-blocks-fts-triggers! "Table bindings of blocks tables and the blocks FTS virtual tables" [db] @@ -500,30 +480,18 @@ DROP TRIGGER IF EXISTS blocks_au; (filter (fn [{:keys [title]}] (exact-matched? q title))))))))) -;; Combine and re-rank results +;; Combine and re-rank keyword results (defn combine-results - [db keyword-results semantic-results] - (let [;; Extract score ranges for normalization - keyword-scores (map :keyword-score keyword-results) - k-min (if (seq keyword-scores) (apply min keyword-scores) 0.0) - k-max (if (seq keyword-scores) (apply max keyword-scores) 1.0) - all-ids (set/union (set (map :id keyword-results)) - (set (map :id semantic-results))) + [db keyword-results] + (let [all-ids (set (map :id keyword-results)) merged (keep (fn [id] (let [block (when id (d/entity db [:block/uuid (uuid id)]))] (when-not (ldb/hidden? block) - (let [k-result (first (filter #(= (:id %) id) keyword-results)) - s-result (first (filter #(= (:id %) id) semantic-results)) - result (merge s-result k-result) - page? (ldb/page? block) - keyword-score (if page? (+ (:keyword-score k-result) 2) (:keyword-score k-result)) - k-score (or keyword-score 0.0) - s-score (or (:semantic-score s-result) 0.0) - norm-k-score (normalize-score k-score k-min k-max) - ;; Weighted combination - combined-score (+ (* (:keyword-weight config) - norm-k-score) - (* (:semantic-weight config) s-score) + (let [result (first (filter #(= (:id %) id) keyword-results)) + keyword-score (if (ldb/page? block) + (+ (or (:keyword-score result) 0.0) 2) + (or (:keyword-score result) 0.0)) + combined-score (+ keyword-score (cond (ldb/page? block) 0.02 @@ -531,10 +499,9 @@ DROP TRIGGER IF EXISTS blocks_au; 0.01 :else 0))] - (merge result - {:combined-score combined-score - :keyword-score k-score - :semantic-score s-score}))))) + (assoc result + :combined-score combined-score + :keyword-score keyword-score))))) all-ids) sorted-result (sort-by :combined-score #(compare %2 %1) merged)] sorted-result)) @@ -632,22 +599,10 @@ DROP TRIGGER IF EXISTS blocks_au; (->> (fuzzy-search repo @conn q option) (map (fn [result] (assoc result :keyword-score (fuzzy/score q (:title result))))))) - semantic-search-result* (m/? (embedding/task--search repo q 10)) - semantic-search-result (->> semantic-search-result* - (map (fn [{:keys [block distance]}] - (let [page-id (when-let [id (:block/uuid (:block/page block))] (str id))] - (cond-> - {:id (str (:block/uuid block)) - :title (:block/title block) - :semantic-score (/ 1.0 (+ 1.0 distance))} - page-id - (assoc :page page-id)))))) ;; _ (prn :debug "Search results before combine:" enable-snippet? (map :snippet matched-result)) ;; _ (doseq [item (concat fuzzy-result matched-result)] ;; (prn :debug :keyword-search-result item)) - ;; _ (doseq [item semantic-search-result] - ;; (prn :debug :semantic-search-item item)) - combined-result (combine-results @conn (concat fuzzy-result matched-result non-match-result) semantic-search-result) + combined-result (combine-results @conn (concat fuzzy-result matched-result non-match-result)) code-class (when code-only? (d/entity @conn :logseq.class/Code-block)) result (->> combined-result diff --git a/src/main/frontend/worker/state.cljs b/src/main/frontend/worker/state.cljs index 2ecf49d7f7..2872fb536d 100644 --- a/src/main/frontend/worker/state.cljs +++ b/src/main/frontend/worker/state.cljs @@ -3,7 +3,6 @@ (:require [logseq.common.util :as common-util])) (defonce *main-thread (atom nil)) -(defonce *infer-worker (atom nil)) (defonce *deleted-block-uuid->db-id (atom {})) (defn- =2.4.0" -"@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": - version "1.1.2" - resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf" - integrity sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ== - -"@protobufjs/base64@^1.1.2": - version "1.1.2" - resolved "https://registry.yarnpkg.com/@protobufjs/base64/-/base64-1.1.2.tgz#4c85730e59b9a1f1f349047dbf24296034bb2735" - integrity sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg== - -"@protobufjs/codegen@^2.0.4": - version "2.0.4" - resolved "https://registry.yarnpkg.com/@protobufjs/codegen/-/codegen-2.0.4.tgz#7ef37f0d010fb028ad1ad59722e506d9262815cb" - integrity sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg== - -"@protobufjs/eventemitter@^1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz#355cbc98bafad5978f9ed095f397621f1d066b70" - integrity sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q== - -"@protobufjs/fetch@^1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@protobufjs/fetch/-/fetch-1.1.0.tgz#ba99fb598614af65700c1619ff06d454b0d84c45" - integrity sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ== - dependencies: - "@protobufjs/aspromise" "^1.1.1" - "@protobufjs/inquire" "^1.1.0" - -"@protobufjs/float@^1.0.2": - version "1.0.2" - resolved "https://registry.yarnpkg.com/@protobufjs/float/-/float-1.0.2.tgz#5e9e1abdcb73fc0a7cb8b291df78c8cbd97b87d1" - integrity sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ== - -"@protobufjs/inquire@^1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@protobufjs/inquire/-/inquire-1.1.0.tgz#ff200e3e7cf2429e2dcafc1140828e8cc638f089" - integrity sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q== - -"@protobufjs/path@^1.1.2": - version "1.1.2" - resolved "https://registry.yarnpkg.com/@protobufjs/path/-/path-1.1.2.tgz#6cc2b20c5c9ad6ad0dccfd21ca7673d8d7fbf68d" - integrity sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA== - -"@protobufjs/pool@^1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@protobufjs/pool/-/pool-1.1.0.tgz#09fd15f2d6d3abfa9b65bc366506d6ad7846ff54" - integrity sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw== - -"@protobufjs/utf8@^1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" - integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== - "@radix-ui/colors@^3.0.0": version "3.0.0" resolved "https://registry.yarnpkg.com/@radix-ui/colors/-/colors-3.0.0.tgz#e8a591a303c44e503bd1212cacf40a09511165e0" @@ -1387,7 +1182,7 @@ resolved "https://registry.yarnpkg.com/@types/minimist/-/minimist-1.2.5.tgz#ec10755e871497bcd83efe927e43ec46e8c0747e" integrity sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag== -"@types/node@*", "@types/node@>=13.7.0": +"@types/node@*": version "24.2.0" resolved "https://registry.yarnpkg.com/@types/node/-/node-24.2.0.tgz#cde712f88c5190006d6b069232582ecd1f94a760" integrity sha512-3xyG3pMCq3oYCNg7/ZP+E1ooTaGB4cG8JWRsqqOYQdbWNY4zbaV0Ennrd7stjiJEFZCaybcIgpTjJWHRfBSIDw== @@ -1990,11 +1785,6 @@ boolbase@^1.0.0: resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e" integrity sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww== -boolean@^3.0.1: - version "3.2.0" - resolved "https://registry.yarnpkg.com/boolean/-/boolean-3.2.0.tgz#9e5294af4e98314494cbb17979fa54ca159f116b" - integrity sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw== - bplist-creator@0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/bplist-creator/-/bplist-creator-0.1.0.tgz#018a2d1b587f769e379ef5519103730f8963ba1e" @@ -3076,16 +2866,11 @@ detect-file@^1.0.0: resolved "https://registry.yarnpkg.com/detect-file/-/detect-file-1.0.0.tgz#f0d66d03672a825cb1b73bdb3fe62310c8e552b7" integrity sha512-DtCOLG98P007x7wiiOmfI0fi3eIKyWiLTGJ2MDnVi/E04lWGbf+JzrRHMm0rgIIZJGtHpKpbVgLWHrv8xXpc3Q== -detect-libc@^2.0.0, detect-libc@^2.0.2, detect-libc@^2.0.4: +detect-libc@^2.0.0, detect-libc@^2.0.2: version "2.0.4" resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.0.4.tgz#f04715b8ba815e53b4d8109655b6508a6865a7e8" integrity sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA== -detect-node@^2.0.4: - version "2.1.0" - resolved "https://registry.yarnpkg.com/detect-node/-/detect-node-2.1.0.tgz#c9c70775a49c3d03bc2c06d9a73be550f978f8b1" - integrity sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g== - dezalgo@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/dezalgo/-/dezalgo-1.0.4.tgz#751235260469084c132157dfa857f386d4c33d81" @@ -3381,11 +3166,6 @@ es-object-atoms@^1.0.0, es-object-atoms@^1.1.1: dependencies: es-errors "^1.3.0" -es6-error@^4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/es6-error/-/es6-error-4.1.1.tgz#9e3af407459deed47e9a91f9b885a84eb05c561d" - integrity sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg== - escalade@^3.1.1, escalade@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.2.0.tgz#011a3f69856ba189dffa7dc8fcce99d2a87903e5" @@ -3396,11 +3176,6 @@ escape-string-regexp@^1.0.3, escape-string-regexp@^1.0.5: resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" integrity sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg== -escape-string-regexp@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz#14ba83a5d373e3d311e5afca29cf5bfad965bf34" - integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA== - eslint-scope@5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-5.1.1.tgz#e786e59a66cb92b3f6c1fb0d508aab174848f48c" @@ -3655,11 +3430,6 @@ flat@^5.0.2: resolved "https://registry.yarnpkg.com/flat/-/flat-5.0.2.tgz#8ca6fe332069ffa9d324c327198c598259ceb241" integrity sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ== -flatbuffers@^25.1.24: - version "25.2.10" - resolved "https://registry.yarnpkg.com/flatbuffers/-/flatbuffers-25.2.10.tgz#308b750545f62db670ca4c9d7dbc66161420a95e" - integrity sha512-7JlN9ZvLDG1McO3kbX0k4v+SUAg48L1rIwEvN6ZQl/eCtgJz9UylTMzE9wrmYrcorgxm3CX/3T/w5VAub99UUw== - flatted@^3.2.9: version "3.3.3" resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.3.3.tgz#67c8fad95454a7c7abebf74bb78ee74a44023358" @@ -3986,18 +3756,6 @@ glob@^9.2.0: minipass "^4.2.4" path-scurry "^1.6.1" -global-agent@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/global-agent/-/global-agent-3.0.0.tgz#ae7cd31bd3583b93c5a16437a1afe27cc33a1ab6" - integrity sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q== - dependencies: - boolean "^3.0.1" - es6-error "^4.1.1" - matcher "^3.0.0" - roarr "^2.15.3" - semver "^7.3.2" - serialize-error "^7.0.1" - global-modules@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-1.0.0.tgz#6d770f0eb523ac78164d72b5e71a8877265cc3ea" @@ -4034,14 +3792,6 @@ global-prefix@^3.0.0: kind-of "^6.0.2" which "^1.3.1" -globalthis@^1.0.1: - version "1.0.4" - resolved "https://registry.yarnpkg.com/globalthis/-/globalthis-1.0.4.tgz#7430ed3a975d97bfb59bcce41f5cabbafa651236" - integrity sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ== - dependencies: - define-properties "^1.2.1" - gopd "^1.0.1" - globby@^11.0.1, globby@^11.0.3: version "11.1.0" resolved "https://registry.yarnpkg.com/globby/-/globby-11.1.0.tgz#bd4be98bb042f83d796f7e3811991fbe82a0d34b" @@ -4098,11 +3848,6 @@ graphology@0.20.0: events "^3.3.0" obliterator "^1.6.1" -guid-typescript@^1.0.9: - version "1.0.9" - resolved "https://registry.yarnpkg.com/guid-typescript/-/guid-typescript-1.0.9.tgz#e35f77003535b0297ea08548f5ace6adb1480ddc" - integrity sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ== - gulp-cli@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/gulp-cli/-/gulp-cli-3.1.0.tgz#92590e9b209142b176c95ad5c7066d2592017268" @@ -4263,11 +4008,6 @@ hmac-drbg@^1.0.1: minimalistic-assert "^1.0.0" minimalistic-crypto-utils "^1.0.1" -hnswlib-wasm@^0.8.2: - version "0.8.2" - resolved "https://registry.yarnpkg.com/hnswlib-wasm/-/hnswlib-wasm-0.8.2.tgz#8b6a9534d99f23d30b1fd29ac7c45410ee5941c1" - integrity sha512-qEgKETj4rMOYRA1esP0bxVosw9Wrz5S/HvjI2FBWOXG5rf5/Es4OoEWGVvztFihDNU5if61l6QGhW5ILtt+PqA== - homedir-polyfill@^1.0.1: version "1.0.3" resolved "https://registry.yarnpkg.com/homedir-polyfill/-/homedir-polyfill-1.0.3.tgz#743298cef4e5af3e194161fbadcc2151d3a058e8" @@ -4993,11 +4733,6 @@ log-symbols@^4.1.0: chalk "^4.1.0" is-unicode-supported "^0.1.0" -long@^5.0.0, long@^5.2.3: - version "5.3.2" - resolved "https://registry.yarnpkg.com/long/-/long-5.3.2.tgz#1d84463095999262d7d7b7f8bfd4a8cc55167f83" - integrity sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA== - longest-streak@^2.0.0: version "2.0.4" resolved "https://registry.yarnpkg.com/longest-streak/-/longest-streak-2.0.4.tgz#b8599957da5b5dab64dee3fe316fa774597d90e4" @@ -5073,13 +4808,6 @@ marked@^17.0.5: resolved "https://registry.yarnpkg.com/marked/-/marked-17.0.5.tgz#8fc6878a439463a007b05d346d2ad50a87ec3f0e" integrity sha512-6hLvc0/JEbRjRgzI6wnT2P1XuM1/RrrDEX0kPt0N7jGm1133g6X7DlxFasUIx+72aKAr904GTxhSLDrd5DIlZg== -matcher@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/matcher/-/matcher-3.0.0.tgz#bd9060f4c5b70aa8041ccc6f80368760994f30ca" - integrity sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng== - dependencies: - escape-string-regexp "^4.0.0" - math-intrinsics@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/math-intrinsics/-/math-intrinsics-1.1.0.tgz#a0dd74be81e2aa5c2f27e65ce283605ee4e2b7f9" @@ -5355,13 +5083,6 @@ minizlib@^2.1.1: minipass "^3.0.0" yallist "^4.0.0" -minizlib@^3.0.1: - version "3.0.2" - resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-3.0.2.tgz#f33d638eb279f664439aa38dc5f91607468cb574" - integrity sha512-oG62iEk+CYt5Xj2YqI5Xi9xWUeZhDI8jjQmC5oThVH5JGCTgIjr7ciJDzC7MBzYd//WvR1OTmP5Q38Q8ShQtVA== - dependencies: - minipass "^7.1.2" - minizlib@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-3.1.0.tgz#6ad76c3a8f10227c9b51d1c9ac8e30b27f5a251c" @@ -5379,11 +5100,6 @@ mkdirp@^1.0.3: resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== -mkdirp@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-3.0.1.tgz#e44e4c5607fb279c168241713cc6e0fea9adcb50" - integrity sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg== - mldoc@^1.5.9: version "1.5.9" resolved "https://registry.yarnpkg.com/mldoc/-/mldoc-1.5.9.tgz#43d740351c64285f0f4988ac9497922d54ae66fc" @@ -5703,37 +5419,6 @@ once@^1.3.0, once@^1.3.1, once@^1.4.0: dependencies: wrappy "1" -onnxruntime-common@1.21.0: - version "1.21.0" - resolved "https://registry.yarnpkg.com/onnxruntime-common/-/onnxruntime-common-1.21.0.tgz#a81d4191d418acbbff2546a954cc2cc23eeb09f8" - integrity sha512-Q632iLLrtCAVOTO65dh2+mNbQir/QNTVBG3h/QdZBpns7mZ0RYbLRBgGABPbpU9351AgYy7SJf1WaeVwMrBFPQ== - -onnxruntime-common@1.22.0-dev.20250409-89f8206ba4: - version "1.22.0-dev.20250409-89f8206ba4" - resolved "https://registry.yarnpkg.com/onnxruntime-common/-/onnxruntime-common-1.22.0-dev.20250409-89f8206ba4.tgz#3d4a39563b93db3d0428b5527cba58a3c8f826c2" - integrity sha512-vDJMkfCfb0b1A836rgHj+ORuZf4B4+cc2bASQtpeoJLueuFc5DuYwjIZUBrSvx/fO5IrLjLz+oTrB3pcGlhovQ== - -onnxruntime-node@1.21.0: - version "1.21.0" - resolved "https://registry.yarnpkg.com/onnxruntime-node/-/onnxruntime-node-1.21.0.tgz#7f4f59455baf851181e207fc8401288ac2eb10d1" - integrity sha512-NeaCX6WW2L8cRCSqy3bInlo5ojjQqu2fD3D+9W5qb5irwxhEyWKXeH2vZ8W9r6VxaMPUan+4/7NDwZMtouZxEw== - dependencies: - global-agent "^3.0.0" - onnxruntime-common "1.21.0" - tar "^7.0.1" - -onnxruntime-web@1.22.0-dev.20250409-89f8206ba4: - version "1.22.0-dev.20250409-89f8206ba4" - resolved "https://registry.yarnpkg.com/onnxruntime-web/-/onnxruntime-web-1.22.0-dev.20250409-89f8206ba4.tgz#d1e3a04e03dfee392b41d420ef547b6a0351b06b" - integrity sha512-0uS76OPgH0hWCPrFKlL8kYVV7ckM7t/36HfbgoFw6Nd0CZVVbQC4PkrR8mBX8LtNUFZO25IQBqV2Hx2ho3FlbQ== - dependencies: - flatbuffers "^25.1.24" - guid-typescript "^1.0.9" - long "^5.2.3" - onnxruntime-common "1.22.0-dev.20250409-89f8206ba4" - platform "^1.3.6" - protobufjs "^7.2.4" - open@^8.4.0: version "8.4.2" resolved "https://registry.yarnpkg.com/open/-/open-8.4.2.tgz#5b5ffe2a8f793dcd2aad73e550cb87b59cb084f9" @@ -6140,11 +5825,6 @@ pkg-dir@^4.2.0: dependencies: find-up "^4.0.0" -platform@^1.3.6: - version "1.3.6" - resolved "https://registry.yarnpkg.com/platform/-/platform-1.3.6.tgz#48b4ce983164b209c2d45a107adb31f473a6e7a7" - integrity sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg== - playwright-core@1.58.2: version "1.58.2" resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.58.2.tgz#ac5f5b4b10d29bcf934415f0b8d133b34b0dcb13" @@ -6637,24 +6317,6 @@ prop-types@^15.6.2, prop-types@^15.7.2, prop-types@^15.8.1: object-assign "^4.1.1" react-is "^16.13.1" -protobufjs@^7.2.4: - version "7.5.3" - resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-7.5.3.tgz#13f95a9e3c84669995ec3652db2ac2fb00b89363" - integrity sha512-sildjKwVqOI2kmFDiXQ6aEB0fjYTafpEvIBs8tOR8qI4spuL9OPROLVu2qZqi/xgCfsHIwVqlaF8JBjWFHnKbw== - dependencies: - "@protobufjs/aspromise" "^1.1.2" - "@protobufjs/base64" "^1.1.2" - "@protobufjs/codegen" "^2.0.4" - "@protobufjs/eventemitter" "^1.1.0" - "@protobufjs/fetch" "^1.1.0" - "@protobufjs/float" "^1.0.2" - "@protobufjs/inquire" "^1.1.0" - "@protobufjs/path" "^1.1.2" - "@protobufjs/pool" "^1.1.0" - "@protobufjs/utf8" "^1.1.0" - "@types/node" ">=13.7.0" - long "^5.0.0" - public-encrypt@^4.0.3: version "4.0.3" resolved "https://registry.yarnpkg.com/public-encrypt/-/public-encrypt-4.0.3.tgz#4fcc9d77a07e48ba7527e7cbe0de33d0701331e0" @@ -7056,18 +6718,6 @@ ripemd160@^2.0.0, ripemd160@^2.0.1: hash-base "^3.0.0" inherits "^2.0.1" -roarr@^2.15.3: - version "2.15.4" - resolved "https://registry.yarnpkg.com/roarr/-/roarr-2.15.4.tgz#f5fe795b7b838ccfe35dc608e0282b9eba2e7afd" - integrity sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A== - dependencies: - boolean "^3.0.1" - detect-node "^2.0.4" - globalthis "^1.0.1" - json-stringify-safe "^5.0.1" - semver-compare "^1.0.0" - sprintf-js "^1.1.2" - run-parallel@^1.1.9: version "1.2.0" resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.2.0.tgz#66d1368da7bdf921eb9d95bd1a9229e7f21a43ee" @@ -7133,11 +6783,6 @@ schema-utils@^4.3.0, schema-utils@^4.3.3: ajv-formats "^2.1.1" ajv-keywords "^5.1.0" -semver-compare@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/semver-compare/-/semver-compare-1.0.0.tgz#0dee216a1c941ab37e9efb1788f6afc5ff5537fc" - integrity sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow== - semver-greatest-satisfied-range@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/semver-greatest-satisfied-range/-/semver-greatest-satisfied-range-2.0.0.tgz#4b62942a7a1ccbdb252e5329677c003bac546fe7" @@ -7150,7 +6795,7 @@ semver-greatest-satisfied-range@^2.0.0: resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== -semver@7.7.4, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.5.4, semver@^7.6.3, semver@^7.7.2: +semver@7.7.4, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.5.4, semver@^7.6.3: version "7.7.4" resolved "https://registry.yarnpkg.com/semver/-/semver-7.7.4.tgz#28464e36060e991fa7a11d0279d2d3f3b57a7e8a" integrity sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA== @@ -7167,13 +6812,6 @@ send-intent@^7.0.0: dependencies: "@capacitor/cli" "^7.0.0" -serialize-error@^7.0.1: - version "7.0.1" - resolved "https://registry.yarnpkg.com/serialize-error/-/serialize-error-7.0.1.tgz#f1360b0447f61ffb483ec4157c737fab7d778e18" - integrity sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw== - dependencies: - type-fest "^0.13.1" - set-blocking@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" @@ -7243,38 +6881,6 @@ sharp@0.32.6: tar-fs "^3.0.4" tunnel-agent "^0.6.0" -sharp@^0.34.1: - version "0.34.3" - resolved "https://registry.yarnpkg.com/sharp/-/sharp-0.34.3.tgz#10a03bcd15fb72f16355461af0b9245ccb8a5da3" - integrity sha512-eX2IQ6nFohW4DbvHIOLRB3MHFpYqaqvXd3Tp5e/T/dSH83fxaNJQRvDMhASmkNTsNTVF2/OOopzRCt7xokgPfg== - dependencies: - color "^4.2.3" - detect-libc "^2.0.4" - semver "^7.7.2" - optionalDependencies: - "@img/sharp-darwin-arm64" "0.34.3" - "@img/sharp-darwin-x64" "0.34.3" - "@img/sharp-libvips-darwin-arm64" "1.2.0" - "@img/sharp-libvips-darwin-x64" "1.2.0" - "@img/sharp-libvips-linux-arm" "1.2.0" - "@img/sharp-libvips-linux-arm64" "1.2.0" - "@img/sharp-libvips-linux-ppc64" "1.2.0" - "@img/sharp-libvips-linux-s390x" "1.2.0" - "@img/sharp-libvips-linux-x64" "1.2.0" - "@img/sharp-libvips-linuxmusl-arm64" "1.2.0" - "@img/sharp-libvips-linuxmusl-x64" "1.2.0" - "@img/sharp-linux-arm" "0.34.3" - "@img/sharp-linux-arm64" "0.34.3" - "@img/sharp-linux-ppc64" "0.34.3" - "@img/sharp-linux-s390x" "0.34.3" - "@img/sharp-linux-x64" "0.34.3" - "@img/sharp-linuxmusl-arm64" "0.34.3" - "@img/sharp-linuxmusl-x64" "0.34.3" - "@img/sharp-wasm32" "0.34.3" - "@img/sharp-win32-arm64" "0.34.3" - "@img/sharp-win32-ia32" "0.34.3" - "@img/sharp-win32-x64" "0.34.3" - shebang-command@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-1.2.0.tgz#44aac65b695b03398968c39f363fee5deafdf1ea" @@ -7510,11 +7116,6 @@ split@^1.0.0: dependencies: through "2" -sprintf-js@^1.1.2: - version "1.1.3" - resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.1.3.tgz#4914b903a2f8b685d17fdf78a70e917e872e444a" - integrity sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA== - stream-browserify@^2.0.1: version "2.0.2" resolved "https://registry.yarnpkg.com/stream-browserify/-/stream-browserify-2.0.2.tgz#87521d38a44aa7ee91ce1cd2a47df0cb49dd660b" @@ -7937,18 +7538,6 @@ tar@^6.1.11: mkdirp "^1.0.3" yallist "^4.0.0" -tar@^7.0.1: - version "7.4.3" - resolved "https://registry.yarnpkg.com/tar/-/tar-7.4.3.tgz#88bbe9286a3fcd900e94592cda7a22b192e80571" - integrity sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw== - dependencies: - "@isaacs/fs-minipass" "^4.0.0" - chownr "^3.0.0" - minipass "^7.1.2" - minizlib "^3.0.1" - mkdirp "^3.0.1" - yallist "^5.0.0" - tar@^7.5.3: version "7.5.11" resolved "https://registry.yarnpkg.com/tar/-/tar-7.5.11.tgz#1250fae45d98806b36d703b30973fa8e0a6d8868" @@ -8192,11 +7781,6 @@ tunnel-agent@^0.6.0: dependencies: safe-buffer "^5.0.1" -type-fest@^0.13.1: - version "0.13.1" - resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.13.1.tgz#0172cb5bce80b0bd542ea348db50c7e21834d934" - integrity sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg== - type-fest@^0.16.0: version "0.16.0" resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.16.0.tgz#3240b891a78b0deae910dbeb86553e552a148860"