diff --git a/src/main/frontend/components/header.cljs b/src/main/frontend/components/header.cljs index bb6d9e26bf..1e1957faa9 100644 --- a/src/main/frontend/components/header.cljs +++ b/src/main/frontend/components/header.cljs @@ -347,11 +347,25 @@ (ldb/page? page) (:block/parent page)) [:div.ls-block-breadcrumb [:div.text-sm - (component-block/breadcrumb {} + (component-block/breadcrumb {} (state/get-current-repo) (:block/uuid page) {:header? true})]]))) +(rum/defc search-index-progress < rum/reactive + [] + (let [current-repo (state/get-current-repo) + {:keys [running? repo progress]} (or (state/sub :search/index-build) {}) + progress' (-> (or progress 0) + (max 0) + (min 100))] + (when (and running? (= repo current-repo)) + [:div.search-index-progress + [ui/loading ""] + [:span.search-index-progress__text (str "Indexing " progress' "%")] + [:div.search-index-progress__bar + [:div.search-index-progress__bar-fill {:style {:width (str progress' "%")}}]]]))) + (rum/defc ^:large-vars/cleanup-todo header-aux < rum/reactive [{:keys [current-repo default-home new-block-mode]}] (let [electron-mac? (and util/mac? (util/electron?)) @@ -414,6 +428,7 @@ (rtc-indicator/downloading-detail)) (when (user-handler/logged-in?) (rtc-indicator/uploading-detail)) + (search-index-progress) (when (and (not= (state/get-current-route) :home) (not custom-home-page?)) diff --git a/src/main/frontend/components/header.css b/src/main/frontend/components/header.css index 7999fe0874..bab347a43e 100644 --- a/src/main/frontend/components/header.css +++ b/src/main/frontend/components/header.css @@ -341,4 +341,24 @@ html.is-zoomed-native-ios { max-width: 34ch; } } + + .search-index-progress { + @apply flex items-center gap-2 rounded px-2 py-1 text-xs opacity-90; + -webkit-app-region: no-drag; + background-color: var(--ls-tertiary-background-color); + } + + .search-index-progress__text { + @apply whitespace-nowrap; + } + + .search-index-progress__bar { + @apply h-1 w-16 overflow-hidden rounded; + background-color: var(--ls-quaternary-background-color); + } + + .search-index-progress__bar-fill { + @apply h-full transition-all duration-200; + background-color: var(--ls-link-text-color); + } } diff --git a/src/main/frontend/handler/events.cljs b/src/main/frontend/handler/events.cljs index 959770ab6c..93590304f4 100644 --- a/src/main/frontend/handler/events.cljs +++ b/src/main/frontend/handler/events.cljs @@ -62,6 +62,12 @@ [error] (string/includes? (or (ex-message error) (str error)) "decrypt-aes-key")) +(defn- (state/ (state/ (worker-state/> batch - (keep #(d/entity db (:e %))) - (remove search/hidden-entity?) - (keep search/block->index))] - (prn :debug :build-search-indice :remaining (count remaining)) - (when (seq indexed) - (search/upsert-blocks! search-db (bean/->js indexed))) - (p/let [_ (js/Promise. (fn [resolve] (js/setTimeout resolve 0)))] - (p/recur remaining'))) - (do - (ensure-active-search-index-build! repo build-id) - (.exec search-db (str "PRAGMA user_version = " search-db-version)))))))) + (let [db @conn + datoms (d/datoms db :avet :block/uuid) + total (count datoms)] + (p/do! + (report-search-index-progress! repo {:build-id build-id + :status :running + :progress 0 + :processed 0 + :total total}) + (> batch + (keep #(d/entity db (:e %))) + (remove search/hidden-entity?) + (keep search/block->index)) + progress (if (zero? total) + 100 + (min 100 (int (* 100 (/ processed' total))))) + should-report? (> progress last-progress)] + (when (seq indexed) + (search/upsert-blocks! search-db (bean/->js indexed))) + (when should-report? + (report-search-index-progress! repo {:build-id build-id + :status :running + :progress progress + :processed processed' + :total total})) + (p/let [_ (js/Promise. (fn [resolve] (js/setTimeout resolve 0)))] + (p/recur remaining' processed' (if should-report? progress last-progress)))) + (do + (ensure-active-search-index-build! repo build-id) + (.exec search-db (str "PRAGMA user_version = " search-db-version)) + (report-search-index-progress! repo {:build-id build-id + :status :completed + :progress 100 + :processed total + :total total}))))))) (def-thread-api :thread-api/search-build-blocks-indice-in-worker [repo & [force?]] @@ -929,6 +959,9 @@ (when-not (= :search/stale-index-build (:type (ex-data error))) (throw error)))) (p/finally (fn [] + (when (= build-id (get @*search-index-build-ids repo)) + (report-search-index-progress! repo {:build-id build-id + :status :idle})) (clear-search-index-build! repo build-id))))))))))) (def-thread-api :thread-api/search-build-pages-indice diff --git a/src/main/frontend/worker/search.cljs b/src/main/frontend/worker/search.cljs index 1244918cea..1ce60b7d73 100644 --- a/src/main/frontend/worker/search.cljs +++ b/src/main/frontend/worker/search.cljs @@ -102,21 +102,43 @@ DROP TRIGGER IF EXISTS blocks_au; (str "(" (->> (map (fn [id] (str "'" id "'")) ids) (string/join ", ")) ")")) +(def ^:private upsert-blocks-batch-size 2000) + +(def ^:private upsert-blocks-sql + (memoize + (fn [row-count] + (str "INSERT INTO blocks (id, title, page) VALUES " + (string/join ", " (repeat row-count "(?, ?, ?)")) + " ON CONFLICT (id) DO UPDATE SET (title, page) = (excluded.title, excluded.page)")))) + +(defn- valid-upsert-block? + [item] + (and (common-util/uuid-string? (.-id item)) + (common-util/uuid-string? (.-page item)))) + +(defn- throw-upsert-blocks-error! + [item] + (js/console.error "Upsert blocks wrong data: ") + (js/console.dir item) + (throw (ex-info "Search upsert-blocks wrong data: " + (bean/->clj item)))) + +(defn- upsert-bind-params + [batch] + (into-array + (mapcat (fn [item] + [(.-id item) (.-title item) (.-page item)]) + batch))) + (defn upsert-blocks! [^Object db blocks] (.transaction db (fn [tx] - (doseq [item blocks] - (if (and (common-util/uuid-string? (.-id item)) - (common-util/uuid-string? (.-page item))) - (.exec tx #js {:sql "INSERT INTO blocks (id, title, page) VALUES ($id, $title, $page) ON CONFLICT (id) DO UPDATE SET (title, page) = ($title, $page)" - :bind #js {:$id (.-id item) - :$title (.-title item) - :$page (.-page item)}}) - (do - (js/console.error "Upsert blocks wrong data: ") - (js/console.dir item) - (throw (ex-info "Search upsert-blocks wrong data: " - (bean/->clj item))))))))) + (doseq [batch (partition-all upsert-blocks-batch-size blocks)] + (doseq [item blocks] + (when-not (valid-upsert-block? item) + (throw-upsert-blocks-error! item))) + (.exec tx #js {:sql (upsert-blocks-sql (count batch)) + :bind (upsert-bind-params batch)}))))) (defn delete-blocks! [db ids] @@ -448,20 +470,18 @@ DROP TRIGGER IF EXISTS blocks_au; "Build a block title indice from scratch. Incremental page title indice is implemented in frontend.search.sync-search-indice!" [repo db] - (prn :debug :build-fuzzy-search-indice :graph repo) - (time - (let [blocks (->> (get-all-fuzzy-supported-blocks db) - (map block->index) - (bean/->js)) - indice (fuse. blocks - (clj->js {:keys ["title"] - :shouldSort true - :tokenize true - :distance 1024 - :threshold 0.5 ;; search for 50% match from the start - :minMatchCharLength 1}))] - (swap! fuzzy-search-indices assoc repo indice) - indice))) + (let [blocks (->> (get-all-fuzzy-supported-blocks db) + (map block->index) + (bean/->js)) + indice (fuse. blocks + (clj->js {:keys ["title"] + :shouldSort true + :tokenize true + :distance 1024 + :threshold 0.5 ;; search for 50% match from the start + :minMatchCharLength 1}))] + (swap! fuzzy-search-indices assoc repo indice) + indice)) (defn fuzzy-search "Return a list of blocks (pages && tagged blocks) that match the query. Takes the following diff --git a/src/test/frontend/worker/search_test.cljs b/src/test/frontend/worker/search_test.cljs index ddb70b3fa2..bd7789a468 100644 --- a/src/test/frontend/worker/search_test.cljs +++ b/src/test/frontend/worker/search_test.cljs @@ -224,3 +224,38 @@ (is (some? ctor)) (is (= 1 (count result))) (is (= "alpha beta" (get-in result [0 :item :title]))))))) + +(deftest upsert-blocks-batches-rows-into-single-sql-statement + (let [calls (atom []) + tx #js {:exec (fn [opts] + (swap! calls conj {:sql (aget opts "sql") + :bind (js->clj (aget opts "bind"))}))} + db #js {:transaction (fn [f] (f tx))} + blocks (clj->js [{:id "67e55044-10b1-426f-9247-bb680e5fe0c8" + :title "alpha" + :page "67e55044-10b1-426f-9247-bb680e5fe0c8"} + {:id "8f14e45f-ea6e-4be8-b53f-bf0f2ca8a5db" + :title "beta" + :page "8f14e45f-ea6e-4be8-b53f-bf0f2ca8a5db"} + {:id "9d5ed678-fe57-4bcf-bf4d-6f2fd5f8995d" + :title "gamma" + :page "9d5ed678-fe57-4bcf-bf4d-6f2fd5f8995d"}])] + (search/upsert-blocks! db blocks) + (is (= 1 (count @calls))) + (is (= "INSERT INTO blocks (id, title, page) VALUES (?, ?, ?), (?, ?, ?), (?, ?, ?) ON CONFLICT (id) DO UPDATE SET (title, page) = (excluded.title, excluded.page)" + (:sql (first @calls)))) + (is (= ["67e55044-10b1-426f-9247-bb680e5fe0c8" "alpha" "67e55044-10b1-426f-9247-bb680e5fe0c8" + "8f14e45f-ea6e-4be8-b53f-bf0f2ca8a5db" "beta" "8f14e45f-ea6e-4be8-b53f-bf0f2ca8a5db" + "9d5ed678-fe57-4bcf-bf4d-6f2fd5f8995d" "gamma" "9d5ed678-fe57-4bcf-bf4d-6f2fd5f8995d"] + (:bind (first @calls)))))) + +(deftest upsert-blocks-throws-on-invalid-input + (let [tx #js {:exec (fn [_opts] nil)} + db #js {:transaction (fn [f] (f tx))} + error (try + (search/upsert-blocks! db (clj->js [{:id "not-uuid" :title "alpha" :page "not-uuid"}])) + nil + (catch :default e e))] + (is (some? error)) + (is (re-find #"Search upsert-blocks wrong data" + (or (ex-message error) (str error))))))