From 0bdf2c65e56fdfa6ac4ba4f4cd6b51da185160c2 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Sun, 13 Dec 2020 22:40:06 +0800 Subject: [PATCH 1/5] perf(search): use fuzzysort for both pages and blocks searching --- package.json | 6 +- src/main/frontend/components/search.cljs | 6 +- src/main/frontend/db.cljs | 31 ++++- src/main/frontend/extensions/code.cljs | 4 +- src/main/frontend/handler.cljs | 16 ++- src/main/frontend/handler/editor.cljs | 18 +-- src/main/frontend/handler/search.cljs | 13 +- src/main/frontend/keyboards.cljs | 8 +- src/main/frontend/search.cljs | 168 ++++++++++++++++++++--- src/main/frontend/search/db.cljs | 8 ++ src/main/frontend/sync/dropbox.cljs | 58 -------- src/main/frontend/sync/protocol.cljs | 11 -- yarn.lock | 136 ++---------------- 13 files changed, 236 insertions(+), 247 deletions(-) create mode 100644 src/main/frontend/search/db.cljs delete mode 100644 src/main/frontend/sync/dropbox.cljs delete mode 100644 src/main/frontend/sync/protocol.cljs diff --git a/package.json b/package.json index f41db00c27..88248d6603 100644 --- a/package.json +++ b/package.json @@ -44,17 +44,15 @@ "dependencies": { "codemirror": "^5.58.1", "diff": "^4.0.2", - "dropbox": "^5.2.0", + "fuzzysort": "^1.1.4", "ignore": "^5.1.8", "jszip": "^3.5.0", - "localforage": "^1.7.3", + "localforage": "^1.9.0", "mousetrap": "^1.6.5", - "parinfer-codemirror": "^1.4.2", "react": "^16.12.0", "react-dom": "^16.12.0", "react-textarea-autosize": "^8.0.1", "react-transition-group": "^4.3.0", - "webdav": "^3.3.0", "yargs-parser": "^20.2.4" } } diff --git a/src/main/frontend/components/search.cljs b/src/main/frontend/components/search.cljs index 13983cfa51..4f8366a986 100644 --- a/src/main/frontend/components/search.cljs +++ b/src/main/frontend/components/search.cljs @@ -85,7 +85,8 @@ :path-params {:path data}}) :block - (let [page (:page/name (:block/page data)) + (let [block-uuid (uuid (:block/uuid data)) + page (:page/name (:block/page (db/entity [:block/uuid block-uuid]))) path (str "/page/" (util/encode-str page) "#ls-block-" (:block/uuid data))] (route/redirect-with-fragment! path)) nil)) @@ -100,7 +101,8 @@ {:page page})) :block - (let [block (db/entity [:block/uuid (:block/uuid data)])] + (let [block-uuid (uuid (:block/uuid data)) + block (db/entity [:block/uuid block-uuid])] (state/sidebar-add-block! (state/get-current-repo) (:db/id block) diff --git a/src/main/frontend/db.cljs b/src/main/frontend/db.cljs index 028c5e6369..3eeaa4cb3f 100644 --- a/src/main/frontend/db.cljs +++ b/src/main/frontend/db.cljs @@ -1,6 +1,7 @@ (ns frontend.db (:require [datascript.core :as d] [frontend.date :as date] + [frontend.search.db :as search-db] [medley.core :as medley] [datascript.transit :as dt] [frontend.format :as format] @@ -21,7 +22,8 @@ [frontend.db-schema :as db-schema] [clojure.core.async :as async] [lambdaisland.glogi :as log] - [frontend.idb :as idb])) + [frontend.idb :as idb] + [cljs-bean.core :as bean])) ;; Query atom of map of Key ([repo q inputs]) -> atom ;; TODO: replace with LRUCache, only keep the latest 20 or 50 items? @@ -1930,6 +1932,9 @@ (swap! persistent-jobs assoc repo job))) ;; only save when user's idle + +;; TODO: pass as a parameter +(defonce *sync-search-indice-f (atom nil)) (defn- repo-listen-to-tx! [repo conn files-db?] (d/listen! conn :persistence @@ -1937,7 +1942,17 @@ (let [tx-id (get-tx-id tx-report)] (state/set-last-transact-time! repo (util/time-ms)) ;; (state/persist-transaction! repo files-db? tx-id (:tx-data tx-report)) - (persist-if-idle! repo))))) + (persist-if-idle! repo)) + + ;; rebuild search indices + (when-not files-db? + (let [data (:tx-data tx-report) + datoms (filter + (fn [datom] + (contains? #{:page/name :block/content} (:a datom))) + data)] + (when-let [f @*sync-search-indice-f] + (f datoms))))))) (defn- listen-and-persist! [repo] @@ -2415,6 +2430,18 @@ (reset! blocks-count-cache n) n)))) +;; block/uuid and block/content +(defn get-all-block-contents + [] + (->> (d/datoms (get-conn) :avet :block/uuid) + (map :v) + (map (fn [id] + (let [e (entity [:block/uuid id])] + {:db/id (:db/id e) + :block/uuid id + :block/content (:block/content e) + :block/format (:block/format e)}))))) + (defn get-all-templates [] (let [pred (fn [db properties] diff --git a/src/main/frontend/extensions/code.cljs b/src/main/frontend/extensions/code.cljs index 322b006555..e7c8d459bc 100644 --- a/src/main/frontend/extensions/code.cljs +++ b/src/main/frontend/extensions/code.cljs @@ -36,9 +36,7 @@ ["codemirror/mode/smalltalk/smalltalk"] ["codemirror/mode/sql/sql"] ["codemirror/mode/swift/swift"] - ["codemirror/mode/xml/xml"] - ;; ["parinfer-codemirror" :as par-cm] -)) + ["codemirror/mode/xml/xml"])) ;; codemirror diff --git a/src/main/frontend/handler.cljs b/src/main/frontend/handler.cljs index 1a7ed6264e..0f1992fb14 100644 --- a/src/main/frontend/handler.cljs +++ b/src/main/frontend/handler.cljs @@ -9,6 +9,8 @@ [promesa.core :as p] [cljs-bean.core :as bean] [frontend.date :as date] + [frontend.search :as search] + [frontend.search.db :as search-db] [frontend.handler.notification :as notification] [frontend.handler.page :as page-handler] [frontend.handler.repo :as repo-handler] @@ -25,9 +27,16 @@ (defn- watch-for-date! [] - (js/setInterval (fn [] - (when-not (state/nfs-refreshing?) - (repo-handler/create-today-journal!))) 1000)) + (let [f (fn [] + (when-not (state/nfs-refreshing?) + (repo-handler/create-today-journal!)) + (when-let [repo (state/get-current-repo)] + (when (and (search-db/empty? repo) + (not (state/file-in-writing!)) + (state/input-idle? repo)) + (search/rebuild-indices!))))] + (f) + (js/setInterval f 5000))) (defn store-schema! [] @@ -157,6 +166,7 @@ :example? true}])] (state/set-repos! repos) (restore-and-setup! me repos logged?))) + (reset! db/*sync-search-indice-f search/sync-search-indice!) (db/run-batch-txs!) (file-handler/run-writes-chan!) (editor-handler/periodically-save!))) diff --git a/src/main/frontend/handler/editor.cljs b/src/main/frontend/handler/editor.cljs index a2b0e5a809..a05a302832 100644 --- a/src/main/frontend/handler/editor.cljs +++ b/src/main/frontend/handler/editor.cljs @@ -1547,17 +1547,11 @@ editing-page (and block (when-let [page-id (:db/id (:block/page block))] (:page/name (db/entity page-id))))] - (let [pages (db/get-pages (state/get-current-repo)) - pages (if editing-page - ;; To prevent self references - (remove (fn [p] (= (string/lower-case p) editing-page)) pages) - pages)] - (filter - (fn [page] - (string/index-of - (string/lower-case page) - (string/lower-case q))) - pages)))) + (let [pages (search/page-search q 20)] + (if editing-page + ;; To prevent self references + (remove (fn [p] (= (string/lower-case p) editing-page)) pages) + pages)))) (defn get-matched-blocks [q] @@ -1567,7 +1561,7 @@ (fn [h] (= (:block/uuid current-block) (:block/uuid h))) - (search/search q 21)))) + (search/search q 10)))) (defn get-matched-templates [q] diff --git a/src/main/frontend/handler/search.cljs b/src/main/frontend/handler/search.cljs index 6bed5e7747..e7bd0b31f8 100644 --- a/src/main/frontend/handler/search.cljs +++ b/src/main/frontend/handler/search.cljs @@ -2,14 +2,15 @@ (:require [goog.object :as gobj] [frontend.state :as state] [goog.dom :as gdom] - [frontend.search :as search])) + [frontend.search :as search] + [frontend.handler.notification :as notification-handler])) (defn search [q] (swap! state/state assoc :search/result {:pages (search/page-search q) :files (search/file-search q) - :blocks (search/search q)})) + :blocks (search/search q 10)})) (defn clear-search! [] @@ -18,3 +19,11 @@ :search/q "") (when-let [input (gdom/getElement "search_field")] (gobj/set input "value" ""))) + +(defn rebuild-indices! + [] + (println "Starting to rebuild search indices!") + (search/rebuild-indices!) + (notification-handler/show! + "Search indices rebuilt successfully!" + :success)) diff --git a/src/main/frontend/keyboards.cljs b/src/main/frontend/keyboards.cljs index 080d128a1e..00f81dba16 100644 --- a/src/main/frontend/keyboards.cljs +++ b/src/main/frontend/keyboards.cljs @@ -3,7 +3,9 @@ [frontend.handler.history :as history-handler] [frontend.handler.ui :as ui-handler] [frontend.handler.route :as route-handler] + [frontend.handler.search :as search-handler] [frontend.state :as state] + [frontend.search :as search] [frontend.util :as util] [medley.core :as medley] ["mousetrap" :as mousetrap] @@ -70,7 +72,8 @@ "ctrl+h" editor-handler/highlight-format! "ctrl+shift+a" editor-handler/select-all-blocks! "alt+shift+up" (fn [state e] (editor-handler/move-up-down e true)) - "alt+shift+down" (fn [state e] (editor-handler/move-up-down e false))} + "alt+shift+down" (fn [state e] (editor-handler/move-up-down e false)) + "ctrl+c ctrl+s" (fn [state e] (search-handler/rebuild-indices!))} (medley/map-keys util/->system-modifier))) (defonce chords @@ -79,7 +82,8 @@ "t t" state/toggle-theme! "t r" ui-handler/toggle-right-sidebar! "t e" state/toggle-new-block-shortcut! - "s" route-handler/toggle-between-page-and-file!}) + "s" route-handler/toggle-between-page-and-file! + }) (defonce bind! (gobj/get mousetrap "bind")) diff --git a/src/main/frontend/search.cljs b/src/main/frontend/search.cljs index b1f98dec14..207be55532 100644 --- a/src/main/frontend/search.cljs +++ b/src/main/frontend/search.cljs @@ -1,12 +1,68 @@ (ns frontend.search (:require [frontend.db :as db] + [frontend.search.db :as search-db :refer [indices]] [frontend.config :as config] [frontend.state :as state] [frontend.util :as util] [cljs-bean.core :as bean] [clojure.string :as string] + [clojure.set :as set] [frontend.regex :as regex] - [frontend.text :as text])) + [frontend.text :as text] + [cljs-bean.core :as bean] + [goog.object :as gobj] + ["fuzzysort" :as fuzzy])) + +(def fuzzy-go (gobj/get fuzzy "go")) +(defonce prepare (gobj/get fuzzy "prepare")) +(defonce highlight (gobj/get fuzzy "highlight")) + +(defn go + [q indice opts] + (fuzzy-go q indice opts)) + +(defn block->index + [{:block/keys [uuid content format] :as block}] + (when (<= (count content) 1000) ; performance + (when-let [result (->> (text/remove-level-spaces content format) + (text/remove-properties!) + (prepare))] + (gobj/set result "id" (:db/id block)) + (gobj/set result "uuid" (str uuid)) + result))) + +(defn make-blocks-indice! + [] + (when-let [repo (state/get-current-repo)] + (let [blocks (->> (db/get-all-block-contents) + (map block->index) + (remove nil?) + (bean/->js))] + (swap! indices assoc-in [repo :blocks] blocks) + blocks))) + +(defn make-pages-indice! + [] + (when-let [repo (state/get-current-repo)] + (let [pages (->> (db/get-pages (state/get-current-repo)) + (remove string/blank?) + (map (fn [p] {:name p})) + (bean/->js))] + (when (seq pages) + (swap! indices assoc-in [repo :pages] pages) + pages)))) + +;; TODO: persist indices to indexeddb, it'll be better if the future db +;; can has the direct fuzzy search support. +(defn rebuild-indices! + ([] + (rebuild-indices! (state/get-current-repo))) + ([repo] + (when repo + (let [result {:pages (make-pages-indice!) + :blocks (make-blocks-indice!)}] + (swap! indices assoc repo result) + result)))) ;; Copied from https://gist.github.com/vaughnd/5099299 (defn str-len-distance @@ -78,32 +134,48 @@ (defn search "Block search" ([q] - (search q 20)) + (search q 10)) ([q limit] - (when-not (string/blank? q) - (let [q (escape-str q) - q-pattern (re-pattern (str "(?i)" q))] - (when-not (string/blank? q) - (let [blocks (db/get-matched-blocks - (fn [content] - (re-find q-pattern content)) - ;; (fn [content] - ;; (> (score q (.toLowerCase content)) 0)) - limit)] - (map (fn [{:block/keys [content format _properties] :as block}] - (assoc block :block/content - (->> (text/remove-level-spaces content format) - (text/remove-properties!)))) blocks))))))) + (when-let [repo (state/get-current-repo)] + (when-not (string/blank? q) + (let [q (string/lower-case q) + q (escape-str q)] + (when-not (string/blank? q) + (let [indice (or (get-in @indices [repo :blocks]) + (make-blocks-indice!)) + result (-> + (go q indice (clj->js {:limit limit + :allowTypo false + :threshold -10000})) + (bean/->clj))] + (->> + (map + (fn [{:keys [target uuid]}] + {:block/uuid uuid + :block/content target}) + result) + (remove nil?))))))))) (defn page-search ([q] (page-search q 3)) ([q limit] - (let [q (clean-str q)] - (when-not (string/blank? q) - (let [pages (db/get-pages (state/get-current-repo))] - (when (seq pages) - (fuzzy-search pages q :limit limit))))))) + (when-let [repo (state/get-current-repo)] + (let [q (string/lower-case q) + q (clean-str q)] + (when-not (string/blank? q) + (let [indice (or (get-in @indices [repo :pages]) + (make-pages-indice!)) + result (->> (go q indice (clj->js {:limit limit + :key "name" + :allowTypo false + :threshold -10000})) + (bean/->clj))] + (->> (map + (fn [{:keys [obj]}] + (:name obj)) + result) + (remove nil?)))))))) (defn file-search ([q] @@ -128,3 +200,57 @@ (when (seq templates) (let [result (fuzzy-search (keys templates) q :limit limit)] (vec (select-keys templates result)))))))) + +(defn sync-search-indice! + [datoms] + (when (seq datoms) + (when-let [repo (state/get-current-repo)] + (let [datoms (group-by :a datoms) + pages (:page/name datoms) + blocks (:block/content datoms)] + (when (seq pages) + (let [pages-result (db/pull-many '[:db/id :page/original-name] (set (map :e pages))) + pages-to-add-set (->> (filter :added pages) + (map :e) + (set)) + pages-to-add (->> (filter (fn [page] + (contains? pages-to-add-set (:db/id page))) pages-result) + (map (fn [p] {:name (:page/original-name p)})) + (set)) + pages-to-remove-set (->> (remove :added pages) + (map :v) + (set))] + (swap! search-db/indices update-in [repo :pages] + (fn [pages] + (let [pages (set (bean/->clj pages)) + pages (if (seq pages-to-remove-set) + (remove (fn [page] + (contains? pages-to-remove-set + (string/lower-case (:name page)))) + pages) + pages) + pages (set/union (set pages) pages-to-add)] + (bean/->js pages)))))) + (when (seq blocks) + (let [blocks-result (db/pull-many '[:db/id :block/uuid :block/format :block/content] (set (map :e blocks))) + blocks-to-add-set (->> (filter :added blocks) + (map :e) + (set)) + blocks-to-add (->> (filter (fn [block] + (contains? blocks-to-add-set (:db/id block))) + blocks-result) + (map block->index) + (set)) + blocks-to-remove-set (->> (remove :added blocks) + (map :e) + (set))] + (swap! search-db/indices update-in [repo :blocks] + (fn [blocks] + (let [blocks (set (bean/->clj blocks)) + blocks (if (seq blocks-to-remove-set) + (remove (fn [block] + (contains? blocks-to-remove-set (:id block))) + blocks) + blocks) + blocks (set/union (set blocks) blocks-to-add)] + (bean/->js blocks)))))))))) diff --git a/src/main/frontend/search/db.cljs b/src/main/frontend/search/db.cljs new file mode 100644 index 0000000000..362264043b --- /dev/null +++ b/src/main/frontend/search/db.cljs @@ -0,0 +1,8 @@ +(ns frontend.search.db + (:refer-clojure :exclude [empty?])) + +(defonce indices (atom nil)) + +(defn empty? + [repo] + (nil? (get @indices repo))) diff --git a/src/main/frontend/sync/dropbox.cljs b/src/main/frontend/sync/dropbox.cljs deleted file mode 100644 index 6a67c8c852..0000000000 --- a/src/main/frontend/sync/dropbox.cljs +++ /dev/null @@ -1,58 +0,0 @@ -(ns frontend.sync.dropbox) - -;; (ns frontend.sync.dropbox -;; (:require ["dropbox" :as dropbox] -;; [goog.object :as gobj] -;; [frontend.sync.protocol :refer [Sync] :as sync] -;; [promesa.core :as p] -;; [cljs-bean.core :as bean])) - -;; ;; Note: there's also a `DropboxTeam` -;; (defonce DropboxModule (gobj/get dropbox "Dropbox")) - -;; (defonce *dropbox-client (atom nil)) - -;; (defn upload-file -;; [client path contents] -;; (.filesUpload ^Object client -;; (bean/->js {:path path -;; :contents contents -;; :mode {".tag" "overwrite"} -;; :autorename true}))) - -;; (defrecord Dropbox [token] -;; Sync -;; (get-client [this] -;; (if-let [client @*dropbox-client] -;; client -;; (let [client (DropboxModule. #js {:accessToken token})] -;; (reset! *dropbox-client client) -;; client))) -;; (signed? [this] -;; true) -;; (get-dir [this path] -;; (p/let [resp (.filesListFolder ^Object (sync/get-client this) (bean/->js {:path path}))] -;; (bean/->clj resp))) -;; (get-more-dir [this cursor] -;; (p/let [resp (.filesListFolderContinue ^Object (sync/get-client this) (bean/->js {:cursor cursor}))] -;; (bean/->clj resp))) -;; (create-file [this path contents] -;; (upload-file ^Object (sync/get-client this) path contents)) -;; (update-file [this path contents] -;; (upload-file ^Object (sync/get-client this) path contents)) -;; (get-file-contents-and-metadata [this path] -;; (-> -;; (p/let [resp (.filesDownload ^Object (sync/get-client this) (bean/->js {:path path})) -;; file-binary (gobj/get resp "fileBinary") -;; contents (.toString file-binary) -;; last-modified-at (gobj/get resp "server_modified")] -;; {:contents contents -;; :last-modified-at last-modified-at}) -;; (p/catch -;; (fn [error] -;; ;; TODO: -;; (println "Dropbox get file " path " failed:") -;; (js/console.dir error)) -;; ))) -;; (delete-file [this path] -;; (.filesDelete ^Object (sync/get-client this) (bean/->js {:path path})))) diff --git a/src/main/frontend/sync/protocol.cljs b/src/main/frontend/sync/protocol.cljs deleted file mode 100644 index 26dbe2a29c..0000000000 --- a/src/main/frontend/sync/protocol.cljs +++ /dev/null @@ -1,11 +0,0 @@ -(ns frontend.sync.protocol) - -(defprotocol Sync - (get-client [this]) - (signed? [this]) - (get-dir [this path]) - (get-more-dir [this more-state]) - (create-file [this path contents]) - (update-file [this path contents]) - (get-file-contents-and-metadata [this path]) - (delete-file [this path])) diff --git a/yarn.lock b/yarn.lock index 75ac54d66e..5dcfc07776 100644 --- a/yarn.lock +++ b/yarn.lock @@ -359,13 +359,6 @@ autoprefixer@^9.4.5: postcss "^7.0.26" postcss-value-parser "^4.0.2" -axios@^0.19.2: - version "0.19.2" - resolved "https://registry.yarnpkg.com/axios/-/axios-0.19.2.tgz#3ea36c5d8818d0d5f8a8a97a6d36b86cdc00cb27" - integrity sha512-fjgm5MvRHLhx+osE2xoekY70AhARk3a6hkN+3Io1jc00jtquGvxYlKlsFUhmUET0V5te6CcZI7lcv2Ym61mjHA== - dependencies: - follow-redirects "1.5.10" - bach@^1.0.0: version "1.2.0" resolved "https://registry.yarnpkg.com/bach/-/bach-1.2.0.tgz#4b3ce96bf27134f79a1b414a51c14e34c3bd9880" @@ -386,11 +379,6 @@ balanced-match@^1.0.0: resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c= -base-64@^0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/base-64/-/base-64-0.1.0.tgz#780a99c84e7d600260361511c4877613bf24f6bb" - integrity sha1-eAqZyE59YAJgNhURxId2E78k9rs= - base64-js@^1.0.2: version "1.3.1" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.3.1.tgz#58ece8cb75dd07e71ed08c736abc5fac4dbf8df1" @@ -569,14 +557,6 @@ buffer@^4.3.0: ieee754 "^1.1.4" isarray "^1.0.0" -buffer@^5.6.0: - version "5.6.0" - resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.6.0.tgz#a31749dc7d81d84db08abf937b6b8c4033f62786" - integrity sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw== - dependencies: - base64-js "^1.0.2" - ieee754 "^1.1.4" - builtin-status-codes@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz#85982878e21b98e1c66425e03d0174788f569ee8" @@ -1154,13 +1134,6 @@ d@1, d@^1.0.1: es5-ext "^0.10.50" type "^1.0.1" -debug@=3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" - integrity sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g== - dependencies: - ms "2.0.0" - debug@^2.2.0, debug@^2.3.3: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" @@ -1339,14 +1312,6 @@ dot-prop@^5.2.0: dependencies: is-obj "^2.0.0" -dropbox@^5.2.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/dropbox/-/dropbox-5.2.0.tgz#dac7ce165ef8a0190f9c218932c90cf2e7e5cc14" - integrity sha512-9IdBQYQF/gY7fZ4fteTjkyID/2nz7XBpCfYvRqhd7RPKfi7EggaeXtZ0Rwiqy+iUN1QJoK03ufYILIWftSl/DQ== - dependencies: - buffer "^5.6.0" - moment "^2.25.3" - duplexify@^3.6.0: version "3.7.1" resolved "https://registry.yarnpkg.com/duplexify/-/duplexify-3.7.1.tgz#2a4df5317f6ccfd91f86d6fd25d8d8a103b88309" @@ -1628,11 +1593,6 @@ fast-levenshtein@^1.0.0: resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-1.1.4.tgz#e6a754cc8f15e58987aa9cbd27af66fd6f4e5af9" integrity sha1-5qdUzI8V5YmHqpy9J69m/W9OWvk= -fast-xml-parser@^3.16.0: - version "3.17.4" - resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-3.17.4.tgz#d668495fb3e4bbcf7970f3c24ac0019d82e76477" - integrity sha512-qudnQuyYBgnvzf5Lj/yxMcf4L9NcVWihXJg7CiU1L+oUCq8MUnFEfH2/nXR/W5uq+yvUN1h7z6s7vs2v1WkL1A== - fastq@^1.6.0: version "1.9.0" resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.9.0.tgz#e16a72f338eaca48e91b5c23593bcc2ef66b7947" @@ -1721,13 +1681,6 @@ flush-write-stream@^1.0.2: inherits "^2.0.3" readable-stream "^2.3.6" -follow-redirects@1.5.10: - version "1.5.10" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.5.10.tgz#7b7a9f9aea2fdff36786a94ff643ed07f4ff5e2a" - integrity sha512-0V5l4Cizzvqt5D44aTXbFZz+FtyXV1vrDN6qrelxtfYQKW0KO0W2T/hkE8xvGa/540LkZlkaUjO4ailYTFtHVQ== - dependencies: - debug "=3.1.0" - for-in@^1.0.1, for-in@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80" @@ -1791,6 +1744,11 @@ function-bind@^1.1.1: resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== +fuzzysort@^1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/fuzzysort/-/fuzzysort-1.1.4.tgz#a0510206ed44532cbb52cf797bf5a3cb12acd4ba" + integrity sha512-JzK/lHjVZ6joAg3OnCjylwYXYVjRiwTY6Yb25LvfpJHK8bjisfnZJ5bY8aVWwTwCXgxPNgLAtmHL+Hs5q1ddLQ== + get-caller-file@^1.0.1: version "1.0.3" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.3.tgz#f978fa4c90d1dfe7ff2d6beda2a515e713bdcf4a" @@ -2076,11 +2034,6 @@ hash.js@^1.0.0, hash.js@^1.0.3: inherits "^2.0.3" minimalistic-assert "^1.0.1" -he@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f" - integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw== - hex-color-regex@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/hex-color-regex/-/hex-color-regex-1.1.0.tgz#4c06fccb4602fe2602b3c93df82d7e7dbf1a8a8e" @@ -2112,11 +2065,6 @@ hosted-git-info@^2.1.4: resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.8.tgz#7539bd4bc1e0e0a895815a2e0262420b12858488" integrity sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg== -hot-patcher@^0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/hot-patcher/-/hot-patcher-0.5.0.tgz#9d401424585aaf3a91646b816ceff40eb6a916b9" - integrity sha512-2Uu2W0s8+dnqXzdlg0MRsRzPoDCs1wVjOGSyMRRaMzLDX4bgHw6xDYKccsWafXPPxQpkQfEjgW6+17pwcg60bw== - hsl-regex@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/hsl-regex/-/hsl-regex-1.0.0.tgz#d49330c789ed819e276a4c0d272dffa30b18fe6e" @@ -2687,10 +2635,10 @@ load-json-file@^4.0.0: pify "^3.0.0" strip-bom "^3.0.0" -localforage@^1.7.3: - version "1.7.3" - resolved "https://registry.yarnpkg.com/localforage/-/localforage-1.7.3.tgz#0082b3ca9734679e1bd534995bdd3b24cf10f204" - integrity sha512-1TulyYfc4udS7ECSBT2vwJksWbkwwTX8BzeUIiq8Y07Riy7bDAAnxDaPU/tWyOVmQAcWJIEIFP9lPfBGqVoPgQ== +localforage@^1.9.0: + version "1.9.0" + resolved "https://registry.yarnpkg.com/localforage/-/localforage-1.9.0.tgz#f3e4d32a8300b362b4634cc4e066d9d00d2f09d1" + integrity sha512-rR1oyNrKulpe+VM9cYmcFn6tsHuokyVHFaCM3+osEmxaHTbEk8oQu6eGDfS6DQLWi/N67XRmB8ECG37OES368g== dependencies: lie "3.1.1" @@ -2909,11 +2857,6 @@ mkdirp@~0.5.1: dependencies: minimist "^1.2.5" -moment@^2.25.3: - version "2.27.0" - resolved "https://registry.yarnpkg.com/moment/-/moment-2.27.0.tgz#8bff4e3e26a236220dfe3e36de756b6ebaa0105d" - integrity sha512-al0MUK7cpIcglMv3YF13qSgdAIqxHTO7brRtaz3DlSULbqfazqkc5kEjNrLDOM7fsjshoFIihnU8snrP7zUvhQ== - mousetrap@^1.6.5: version "1.6.5" resolved "https://registry.yarnpkg.com/mousetrap/-/mousetrap-1.6.5.tgz#8a766d8c272b08393d5f56074e0b5ec183485bf9" @@ -2951,11 +2894,6 @@ nanomatch@^1.2.9: snapdragon "^0.8.1" to-regex "^3.0.1" -nested-property@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/nested-property/-/nested-property-1.0.4.tgz#4068c4289d1f3ac9f367e784502ff342d277ee2f" - integrity sha512-6fNIumJJUyP3rkB4FyVYCYpdW+PKUCaxRWRGLLf0kv/RKoG4mbTvInedA9x3zOyuOmOkGudKuAtPSI+dnhwj2g== - next-tick@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.0.0.tgz#ca86d1fe8828169b0120208e3dc8424b9db8342c" @@ -3285,18 +3223,6 @@ pako@~1.0.5: resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.10.tgz#4328badb5086a426aa90f541977d4955da5c9732" integrity sha512-0DTvPVU3ed8+HNXOu5Bs+o//Mbdj9VNQMUOe9oKCwh8l0GNwpTDMKCWbRjgtD291AWnkAgkqA/LOnQS8AmS1tw== -parinfer-codemirror@^1.4.2: - version "1.4.2" - resolved "https://registry.yarnpkg.com/parinfer-codemirror/-/parinfer-codemirror-1.4.2.tgz#2b9b7f27af65bf14282034a17731d4a82367d046" - integrity sha1-K5t/J69lvxQoIDShdzHUqCNn0EY= - dependencies: - parinfer "^3.11.0" - -parinfer@^3.11.0: - version "3.12.0" - resolved "https://registry.yarnpkg.com/parinfer/-/parinfer-3.12.0.tgz#6ea16236319717d579275aa9a94ed1197c3cbd62" - integrity sha512-iViQ8vtJ6CEa9x0SxcQCtsOYSCVVu7ILnuUJfKHPEGjxM0Z6R1EdAJX3kWyqZDrszvQyYWl/XpD9pN2IFgSF/g== - parse-asn1@^5.0.0: version "5.1.5" resolved "https://registry.yarnpkg.com/parse-asn1/-/parse-asn1-5.1.5.tgz#003271343da58dc94cace494faef3d2147ecea0e" @@ -3390,11 +3316,6 @@ path-parse@^1.0.6: resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.6.tgz#d62dbb5679405d72c4737ec58600e9ddcf06d24c" integrity sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw== -path-posix@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/path-posix/-/path-posix-1.0.0.tgz#06b26113f56beab042545a23bfa88003ccac260f" - integrity sha1-BrJhE/Vr6rBCVFojv6iAA8ysJg8= - path-root-regex@^0.1.0: version "0.1.2" resolved "https://registry.yarnpkg.com/path-root-regex/-/path-root-regex-0.1.2.tgz#bfccdc8df5b12dc52c8b43ec38d18d72c04ba96d" @@ -3969,11 +3890,6 @@ querystring@0.2.0: resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA= -querystringify@^2.1.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/querystringify/-/querystringify-2.1.1.tgz#60e5a5fd64a7f8bfa4d2ab2ed6fdf4c85bad154e" - integrity sha512-w7fLxIRCRT7U8Qu53jQnJyPkYZIaR4n5151KMfcJlO/A9397Wxb1amJvROTK6TOnp7PfoAmg/qXiNHI+08jRfA== - randombytes@^2.0.0, randombytes@^2.0.1, randombytes@^2.0.5: version "2.1.0" resolved "https://registry.yarnpkg.com/randombytes/-/randombytes-2.1.0.tgz#df6f84372f0270dc65cdf6291349ab7a473d4f2a" @@ -4203,11 +4119,6 @@ require-main-filename@^1.0.1: resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-1.0.1.tgz#97f717b69d48784f5f526a6c5aa8ffdda055a4d1" integrity sha1-l/cXtp1IeE9fUmpsWqj/3aBVpNE= -requires-port@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff" - integrity sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8= - resolve-dir@^1.0.0, resolve-dir@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/resolve-dir/-/resolve-dir-1.0.1.tgz#79a40644c362be82f26effe739c9bb5382046f43" @@ -4967,19 +4878,6 @@ urix@^0.1.0: resolved "https://registry.yarnpkg.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72" integrity sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI= -url-join@^4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/url-join/-/url-join-4.0.1.tgz#b642e21a2646808ffa178c4c5fda39844e12cde7" - integrity sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA== - -url-parse@^1.4.7: - version "1.4.7" - resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.4.7.tgz#a8a83535e8c00a316e403a5db4ac1b9b853ae278" - integrity sha512-d3uaVyzDB9tQoSXFvuSUNFibTd9zxd2bkVrDRvF5TmvWWQwqE4lgYJ5m+x1DbecWkw+LK4RNl2CU1hHuOKPVlg== - dependencies: - querystringify "^2.1.1" - requires-port "^1.0.0" - url@^0.11.0: version "0.11.0" resolved "https://registry.yarnpkg.com/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1" @@ -5119,22 +5017,6 @@ vm-browserify@^1.0.1: resolved "https://registry.yarnpkg.com/vm-browserify/-/vm-browserify-1.1.2.tgz#78641c488b8e6ca91a75f511e7a3b32a86e5dda0" integrity sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ== -webdav@^3.3.0: - version "3.3.0" - resolved "https://registry.yarnpkg.com/webdav/-/webdav-3.3.0.tgz#5741e1db1540338809849a082d2a80e943e8d12c" - integrity sha512-wTfLNbeK1++T1ooL/ZJaUTJGb5NUuO4zAwuTShNPbzN0mRMRIaoZYG7sI5TtyH1uqOPIOW5ZGTtZiBypLG86KQ== - dependencies: - axios "^0.19.2" - base-64 "^0.1.0" - fast-xml-parser "^3.16.0" - he "^1.2.0" - hot-patcher "^0.5.0" - minimatch "^3.0.4" - nested-property "^1.0.4" - path-posix "^1.0.0" - url-join "^4.0.1" - url-parse "^1.4.7" - which-module@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/which-module/-/which-module-1.0.0.tgz#bba63ca861948994ff307736089e3b96026c2a4f" From 1acf7ffe05faa7fbf0003c9d951fa3ac5e17b41d Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Sun, 13 Dec 2020 23:10:07 +0800 Subject: [PATCH 2/5] chore: remove localforage --- package.json | 1 - yarn.lock | 14 -------------- 2 files changed, 15 deletions(-) diff --git a/package.json b/package.json index 5d79aff509..1772a5b388 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,6 @@ "fuzzysort": "^1.1.4", "ignore": "^5.1.8", "jszip": "^3.5.0", - "localforage": "^1.9.0", "mousetrap": "^1.6.5", "react": "^16.12.0", "react-dom": "^16.12.0", diff --git a/yarn.lock b/yarn.lock index dba8b864a4..42d488fa76 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3012,13 +3012,6 @@ lead@^1.0.0: dependencies: flush-write-stream "^1.0.2" -lie@3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/lie/-/lie-3.1.1.tgz#9a436b2cc7746ca59de7a41fa469b3efb76bd87e" - integrity sha1-mkNrLMd0bKWd56QfpGmz77dr2H4= - dependencies: - immediate "~3.0.5" - lie@~3.3.0: version "3.3.0" resolved "https://registry.yarnpkg.com/lie/-/lie-3.3.0.tgz#dcf82dee545f46074daf200c7c1c5a08e0f40f6a" @@ -3066,13 +3059,6 @@ load-json-file@^4.0.0: pify "^3.0.0" strip-bom "^3.0.0" -localforage@^1.9.0: - version "1.9.0" - resolved "https://registry.yarnpkg.com/localforage/-/localforage-1.9.0.tgz#f3e4d32a8300b362b4634cc4e066d9d00d2f09d1" - integrity sha512-rR1oyNrKulpe+VM9cYmcFn6tsHuokyVHFaCM3+osEmxaHTbEk8oQu6eGDfS6DQLWi/N67XRmB8ECG37OES368g== - dependencies: - lie "3.1.1" - locate-path@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-5.0.0.tgz#1afba396afd676a6d42504d0a67a3a7eb9f62aa0" From 45e9390b6ce277c8087e0604982d24858347986e Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Sun, 13 Dec 2020 23:31:54 +0800 Subject: [PATCH 3/5] fix: both ->clj and ->js are too expensive --- src/main/frontend/db.cljs | 3 ++- src/main/frontend/search.cljs | 29 ++++++++++------------------- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/src/main/frontend/db.cljs b/src/main/frontend/db.cljs index 3eeaa4cb3f..c5cde35dc4 100644 --- a/src/main/frontend/db.cljs +++ b/src/main/frontend/db.cljs @@ -1952,7 +1952,8 @@ (contains? #{:page/name :block/content} (:a datom))) data)] (when-let [f @*sync-search-indice-f] - (f datoms))))))) + (profile "rebuild indice" + (f datoms)))))))) (defn- listen-and-persist! [repo] diff --git a/src/main/frontend/search.cljs b/src/main/frontend/search.cljs index 207be55532..96a7608bd3 100644 --- a/src/main/frontend/search.cljs +++ b/src/main/frontend/search.cljs @@ -3,7 +3,7 @@ [frontend.search.db :as search-db :refer [indices]] [frontend.config :as config] [frontend.state :as state] - [frontend.util :as util] + [frontend.util :as util :refer-macros [profile]] [cljs-bean.core :as bean] [clojure.string :as string] [clojure.set :as set] @@ -222,17 +222,13 @@ (set))] (swap! search-db/indices update-in [repo :pages] (fn [pages] - (let [pages (set (bean/->clj pages)) - pages (if (seq pages-to-remove-set) - (remove (fn [page] - (contains? pages-to-remove-set - (string/lower-case (:name page)))) - pages) - pages) - pages (set/union (set pages) pages-to-add)] - (bean/->js pages)))))) + (let [pages (.filter pages (fn [page] + (not (contains? pages-to-remove-set + (string/lower-case (gobj/get page "name"))))))] + (.concat pages (bean/->js pages-to-add))))))) (when (seq blocks) - (let [blocks-result (db/pull-many '[:db/id :block/uuid :block/format :block/content] (set (map :e blocks))) + (let [blocks-result (profile "pull many" + (db/pull-many '[:db/id :block/uuid :block/format :block/content] (set (map :e blocks)))) blocks-to-add-set (->> (filter :added blocks) (map :e) (set)) @@ -246,11 +242,6 @@ (set))] (swap! search-db/indices update-in [repo :blocks] (fn [blocks] - (let [blocks (set (bean/->clj blocks)) - blocks (if (seq blocks-to-remove-set) - (remove (fn [block] - (contains? blocks-to-remove-set (:id block))) - blocks) - blocks) - blocks (set/union (set blocks) blocks-to-add)] - (bean/->js blocks)))))))))) + (let [blocks (.filter blocks (fn [block] + (not (contains? blocks-to-remove-set (gobj/get block "id")))))] + (.concat blocks (bean/->js blocks-to-add))))))))))) From 4662281c9af43bde7edf44a02f3e9b40bea6cb47 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Sun, 13 Dec 2020 23:32:51 +0800 Subject: [PATCH 4/5] chore: remove profile --- src/main/frontend/db.cljs | 3 +-- src/main/frontend/search.cljs | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/frontend/db.cljs b/src/main/frontend/db.cljs index c5cde35dc4..3eeaa4cb3f 100644 --- a/src/main/frontend/db.cljs +++ b/src/main/frontend/db.cljs @@ -1952,8 +1952,7 @@ (contains? #{:page/name :block/content} (:a datom))) data)] (when-let [f @*sync-search-indice-f] - (profile "rebuild indice" - (f datoms)))))))) + (f datoms))))))) (defn- listen-and-persist! [repo] diff --git a/src/main/frontend/search.cljs b/src/main/frontend/search.cljs index 96a7608bd3..79cc6cb685 100644 --- a/src/main/frontend/search.cljs +++ b/src/main/frontend/search.cljs @@ -227,8 +227,7 @@ (string/lower-case (gobj/get page "name"))))))] (.concat pages (bean/->js pages-to-add))))))) (when (seq blocks) - (let [blocks-result (profile "pull many" - (db/pull-many '[:db/id :block/uuid :block/format :block/content] (set (map :e blocks)))) + (let [blocks-result (db/pull-many '[:db/id :block/uuid :block/format :block/content] (set (map :e blocks))) blocks-to-add-set (->> (filter :added blocks) (map :e) (set)) From 10314b1117df92ac8bd8b290f7f0a59ce2e555a0 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Sun, 13 Dec 2020 23:39:27 +0800 Subject: [PATCH 5/5] fix: both pages and blocks indices default to an empty array --- externs.js | 3 +++ src/main/frontend/search.cljs | 11 ++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/externs.js b/externs.js index c7138f931c..5a5eabb6ef 100644 --- a/externs.js +++ b/externs.js @@ -47,3 +47,6 @@ dummy.createWritable = function() {}; dummy.write = function() {}; dummy.close = function() {}; dummy.values = function() {}; +// Do we really need those? +dummy.filter = function() {}; +dummy.concat = function() {}; diff --git a/src/main/frontend/search.cljs b/src/main/frontend/search.cljs index 79cc6cb685..aca58acbbe 100644 --- a/src/main/frontend/search.cljs +++ b/src/main/frontend/search.cljs @@ -48,9 +48,8 @@ (remove string/blank?) (map (fn [p] {:name p})) (bean/->js))] - (when (seq pages) - (swap! indices assoc-in [repo :pages] pages) - pages)))) + (swap! indices assoc-in [repo :pages] pages) + pages))) ;; TODO: persist indices to indexeddb, it'll be better if the future db ;; can has the direct fuzzy search support. @@ -222,7 +221,8 @@ (set))] (swap! search-db/indices update-in [repo :pages] (fn [pages] - (let [pages (.filter pages (fn [page] + (let [pages (or pages (array)) + pages (.filter pages (fn [page] (not (contains? pages-to-remove-set (string/lower-case (gobj/get page "name"))))))] (.concat pages (bean/->js pages-to-add))))))) @@ -241,6 +241,7 @@ (set))] (swap! search-db/indices update-in [repo :blocks] (fn [blocks] - (let [blocks (.filter blocks (fn [block] + (let [blocks (or blocks (array)) + blocks (.filter blocks (fn [block] (not (contains? blocks-to-remove-set (gobj/get block "id")))))] (.concat blocks (bean/->js blocks-to-add)))))))))))