Use web workers to speed up parsing (#2655)

* wip: use web workers to speed up parsing

* chore: uncomment forget.config.js

* fix: parser pool initialization

* fix: extract parser-worker

* fix: can't run the parser worker in the release mode

* fix: extract async tests

* fix: dsl query async test

* fix: img path in dev mode
This commit is contained in:
Tienson Qin
2021-08-18 17:02:40 +08:00
committed by GitHub
parent 0b3c2bf3f3
commit e87f83c395
23 changed files with 5289 additions and 228 deletions

View File

@@ -141,21 +141,20 @@
(db/set-file-content! repo-url file content)
(let [format (format/get-format file)
utf8-content (utf8/encode content)
file-content [{:file/path file}]
tx (if (contains? config/mldoc-support-formats format)
(let [delete-blocks (db/delete-file-blocks! repo-url file)
[pages block-ids blocks] (extract-handler/extract-blocks-pages repo-url file content utf8-content)
blocks (remove-non-exists-refs! blocks)
_ (util/pprint blocks)
pages (extract-handler/with-ref-pages pages blocks)]
(concat file-content delete-blocks pages block-ids blocks))
file-content)
tx (concat tx [(let [t (tc/to-long (t/now))]
(cond->
{:file/path file}
new?
(assoc :file/created-at t)))])]
(db/transact! repo-url tx))))
file-content [{:file/path file}]]
(p/let [tx (if (contains? config/mldoc-support-formats format)
(p/let [delete-blocks (db/delete-file-blocks! repo-url file)
[pages block-ids blocks] (extract-handler/extract-blocks-pages repo-url file content utf8-content)
blocks (remove-non-exists-refs! blocks)
pages (extract-handler/with-ref-pages pages blocks)]
(concat file-content delete-blocks pages block-ids blocks))
file-content)]
(let [tx (concat tx [(let [t (tc/to-long (t/now))]
(cond->
{:file/path file}
new?
(assoc :file/created-at t)))])]
(db/transact! repo-url tx))))))
;; TODO: Remove this function in favor of `alter-files`
(defn alter-file