enhance(text-embedding): connect db-worker and infer-worker

This commit is contained in:
rcmerci
2025-03-14 16:45:14 +08:00
parent 53cc7485d8
commit eda3928937
7 changed files with 30 additions and 5 deletions

View File

@@ -168,6 +168,7 @@
(mobile/mobile-preinit))
(-> (p/let [_ (db-browser/start-db-worker!)
_ (db-browser/start-inference-worker!)
_ (db-browser/<connect-db-worker-and-infer-worker!)
repos (repo-handler/get-repos)
_ (state/set-repos! repos)
_ (mobile-util/hide-splash) ;; hide splash as early as ui is stable

View File

@@ -1,13 +1,15 @@
(ns frontend.inference-worker.inference-worker
"Worker used for text embedding and vector-db"
(:require ["comlink" :as Comlink]
[frontend.inference-worker.state :as infer-worker.state]
[frontend.inference-worker.text-embedding :as infer-worker.text-embedding]
[lambdaisland.glogi :as log]
[lambdaisland.glogi.console :as glogi-console]
[logseq.db :as ldb]
[promesa.core :as p]
[shadow.cljs.modern :refer [defclass]]))
#_:clj-kondo/ignore
#_{:clj-kondo/ignore [:unresolved-symbol]}
(defclass InferenceWorker
(extends js/Object)
(constructor
@@ -19,6 +21,11 @@
[_this]
(infer-worker.text-embedding/<init))
(set-db-worker-proxy
[_this proxy]
(reset! infer-worker.state/*db-worker proxy)
(log/info :set-db-worker-proxy :done))
(text-embedding
[_this text-coll]
(p/let [output (infer-worker.text-embedding/<text-embedding text-coll)]

View File

@@ -1,6 +1,8 @@
(ns frontend.inference-worker.state
"State hub for inference-worker")
(defonce *db-worker (atom nil))
(defonce *hnswlib (atom nil))
;;repo -> index

View File

@@ -163,6 +163,13 @@
(.init wrapped-worker)
(log/info "init infer-worker spent:" (str (- (util/time-ms) t1) "ms"))))))
(defn <connect-db-worker-and-infer-worker!
[]
(assert (and @state/*infer-worker @state/*db-worker))
(p/do!
(.set-db-worker-proxy ^js @state/*infer-worker (Comlink/proxy @state/*db-worker))
(.set-infer-worker-proxy ^js @state/*db-worker (Comlink/proxy @state/*infer-worker))))
(defn <export-db!
[repo data]
(cond

View File

@@ -437,7 +437,8 @@
(let [result (when-not (= result @worker-state/*state) result)]
(ldb/write-transit-str result)))))
#_:clj-kondo/ignore
#_{:clj-kondo/ignore [:unresolved-symbol]}
(defclass DBWorker
(extends js/Object)
@@ -457,6 +458,11 @@
(reset! worker-state/*rtc-ws-url rtc-ws-url)
(init-sqlite-module!))
(set-infer-worker-proxy
[_this proxy]
(reset! worker-state/*infer-worker proxy)
(log/info :set-infer-worker-proxy :done))
(storeMetadata
[_this repo metadata-str]
(worker-db-metadata/<store repo metadata-str))
@@ -634,7 +640,7 @@
[_this repo]
(p/let [pool (<get-opfs-pool repo)
_ (close-db! repo)
result (remove-vfs! pool)]
_result (remove-vfs! pool)]
nil))
(releaseAccessHandles
@@ -965,7 +971,8 @@
[]
(glogi-console/install!)
(check-worker-scope!)
(let [^js obj (DBWorker.)]
(let [^js obj #_{:clj-kondo/ignore [:unresolved-symbol]}
(DBWorker.)]
(outliner-register-op-handlers!)
(worker-state/set-worker-object! obj)
(<ratelimit-file-writes!)

View File

@@ -30,7 +30,7 @@
(swap! *writes assoc request-id page-id)
request-id))
(defn- dissoc-request!
(defn dissoc-request!
[request-id]
(when-let [page-id (get @*writes request-id)]
(let [old-page-request-ids (keep (fn [[r p]]

View File

@@ -9,6 +9,7 @@
:undo/repo->page-block-uuid->redo-ops {:doc "{repo {<page-block-uuid> [op1 op2 ...]}}"})
(defonce *main-thread (atom nil))
(defonce *infer-worker (atom nil))
(defonce *state (atom {:worker/object nil