enhance: index embedding for updated blocks every 30s

src/main/frontend/handler/db_based/vector_search_background_tasks.cljs

@@ -3,9 +3,9 @@
   (:require [frontend.common.missionary :as c.m]
             [frontend.config :as config]
             [frontend.flows :as flows]
+            [frontend.handler.db-based.vector-search-flows :as vector-search-flows]
             [frontend.state :as state]
-            [missionary.core :as m]
-            [promesa.core :as p]))
+            [missionary.core :as m]))
 
 (defn- run-background-task-when-not-publishing
   [key' task]
@@ -15,7 +15,14 @@
 (run-background-task-when-not-publishing
  ::init-load-model-when-switch-graph
  (m/reduce
-  (fn [_ repo]
-    (when repo
-      (state/<invoke-db-worker :thread-api/vec-search-init-embedding-model repo)))
-  flows/current-repo-flow))
+  (constantly nil)
+  (m/ap
+    (m/?> vector-search-flows/infer-worker-ready-flow)
+    (when-let [repo (m/?< flows/current-repo-flow)]
+      (c.m/<? (state/<invoke-db-worker :thread-api/vec-search-init-embedding-model repo))
+      (m/?
+       (m/reduce
+        (fn []
+          (state/<invoke-db-worker :thread-api/vec-search-embedding-graph repo))
+        ;; index new updates every 30s
+        (c.m/clock (* 30 1000))))))))

src/main/frontend/inference_worker/text_embedding.cljs

@@ -93,7 +93,6 @@
 (defn task--text-embedding&store!
   "return labels(js-array)"
   [repo text-array delete-labels replace-deleted?]
-  (prn :debug :text-array text-array)
   (m/sp
     (when (model-loaded?)
       (let [hnsw (or (get-hnsw-index repo) (new-hnsw-index! repo))

src/main/frontend/worker/db_worker.cljs

@@ -739,6 +739,10 @@
   [repo]
   (embedding/re-embedding-graph-data! repo))
 
+(def-thread-api :thread-api/vec-search-embedding-graph
+  [repo]
+  (embedding/embedding-graph! repo))
+
 (def-thread-api :thread-api/vec-search-search
   [repo query-string nums-neighbors]
   (embedding/task--search repo query-string nums-neighbors))

src/main/frontend/worker/embedding.cljs

@@ -192,6 +192,14 @@
                      :succ (constantly nil))]
       (reset-*vector-search-state! repo :canceler canceler))))
 
+(defn embedding-graph!
+  [repo]
+  (when-not (indexing? repo)
+    (when-let [conn (worker-state/get-datascript-conn repo)]
+      (if (first (d/datoms @conn :avet :logseq.property.embedding/hnsw-label-updated-at)) ; embedding exists
+        (embedding-stale-blocks! repo)
+        (re-embedding-graph-data! repo)))))
+
 (defn task--embedding-model-info
   [repo]
   (m/sp