clone/logseq
1
0
Fork 0
mirror of https://github.com/logseq/logseq.git synced 2026-04-24 14:14:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use web workers to speed up parsing (#2655)

Browse Source 
* wip: use web workers to speed up parsing

* chore: uncomment forget.config.js

* fix: parser pool initialization

* fix: extract parser-worker

* fix: can't run the parser worker in the release mode

* fix: extract async tests

* fix: dsl query async test

* fix: img path in dev mode
This commit is contained in:
Tienson Qin
2021-08-18 17:02:40 +08:00
committed by GitHub
parent 0b3c2bf3f3
commit e87f83c395
23 changed files with 5289 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
deps.edn
View File

@@ -29,7 +29,7 @@
hiccups/hiccups {:mvn/version "0.3.0"}
tongue/tongue {:mvn/version "0.2.9"}
org.clojure/core.async {:mvn/version "1.3.610"}
thheller/shadow-cljs {:mvn/version "2.12.5"}
thheller/shadow-cljs {:mvn/version "2.15.3"}
expound/expound {:mvn/version "0.8.6"}
com.lambdaisland/glogi {:mvn/version "1.0.116"}
binaryage/devtools {:mvn/version "1.0.2"}
@@ -39,14 +39,14 @@
frankiesardo/linked {:mvn/version "1.3.0"}}
:aliases {:cljs {:extra-paths ["src/dev-cljs/" "src/test/" "src/electron/"]
:extra-deps {org.clojure/clojurescript {:mvn/version "1.10.844"}
:extra-deps {org.clojure/clojurescript {:mvn/version "1.10.879"}
org.clojure/tools.namespace {:mvn/version "0.2.11"}
cider/cider-nrepl {:mvn/version "0.26.0"}
org.clojars.knubie/cljs-run-test {:mvn/version "1.0.1"}}
:main-opts ["-m" "shadow.cljs.devtools.cli"]}
:test
{:extra-paths ["src/test/"]
:extra-deps {org.clojure/clojurescript {:mvn/version "1.10.844"}
:extra-deps {org.clojure/clojurescript {:mvn/version "1.10.879"}
org.clojure/test.check {:mvn/version "RELEASE"}
org.clojars.knubie/cljs-run-test {:mvn/version "1.0.1"}}
:main-opts ["-m" "shadow.cljs.devtools.cli"]}

22
package.json
View File

@@ -19,7 +19,7 @@
"postcss-import-ext-glob": "^2.0.1",
"postcss-nested": "5.0.5",
"purgecss": "4.0.2",
"shadow-cljs": "2.12.5",
"shadow-cljs": "2.15.3",
"stylelint": "^13.8.0",
"stylelint-config-standard": "^20.0.0",
"tailwindcss": "2.2.4"
@@ -44,19 +44,18 @@
"gulp:build": "cross-env NODE_ENV=production gulp build",
"css:build": "postcss tailwind.all.css -o static/css/style.css --verbose --env production",
"css:watch": "TAILWIND_MODE=watch postcss tailwind.all.css -o static/css/style.css --verbose --watch",
"cljs:watch": "clojure -M:cljs watch app electron",
"cljs:electron-watch": "clojure -M:cljs watch app electron",
"cljs:release": "clojure -M:cljs release app publishing electron",
"cljs:electron-release": "clojure -M:cljs release app publishing electron --config-merge '{:asset-path \"./js\"}'",
"cljs:watch": "clojure -M:cljs watch parser-worker app electron",
"cljs:electron-watch": "clojure -M:cljs watch parser-worker app electron",
"cljs:release": "clojure -M:cljs release parser-worker app publishing electron",
"cljs:test": "clojure -M:test compile test",
"cljs:run-test": "node static/tests.js",
"cljs:watch-app": "clojure -M:cljs watch app cards",
"cljs:release-app": "clojure -M:cljs release app",
"cljs:release-publishing": "clojure -M:cljs release publishing",
"cljs:watch-app": "clojure -M:cljs watch parser-worker app cards",
"cljs:release-app": "clojure -M:cljs release parser-worker app",
"cljs:release-publishing": "clojure -M:cljs release parser-worker publishing",
"cljs:dev-release-app": "clojure -M:cljs release app --config-merge '{:closure-defines {frontend.config/DEV-RELEASE true}}'",
"cljs:debug": "clojure -M:cljs release app --debug",
"cljs:report": "clojure -M:cljs run shadow.cljs.build-report app report.html",
"cljs:build-electron": "clojure -A:cljs compile app electron"
"cljs:debug": "clojure -M:cljs release parser-worker app --debug",
"cljs:report": "clojure -M:cljs run shadow.cljs.build-report parser-worker app report.html",
"cljs:build-electron": "clojure -A:cljs compile parser-worker app electron"
},
"dependencies": {
"@excalidraw/excalidraw": "^0.4.2",
@@ -96,6 +95,7 @@
"react-transition-group": "^4.3.0",
"react-tweet-embed": "^1.2.2",
"reakit": "^0.11.1",
"threads": "^1.6.5",
"yargs-parser": "^20.2.4"
}
}

25
public/index.html
View File

@@ -13,25 +13,24 @@
<meta content="black-translucent" name="apple-mobile-web-app-status-bar-style">
<meta content="yes" name="mobile-web-app-capable">
<meta content="summary" name="twitter:card">
<meta content="A local-first knowledge base which can be synced using Git." name="twitter:description">
<meta content="A privacy-first, open-source platform for knowledge management and collaboration." name="twitter:description">
<meta content="@logseq" name="twitter:site">
<meta content="A local-first knowledge base." name="twitter:title">
<meta content="A privacy-first, open-source platform for knowledge management and collaboration." name="twitter:title">
<meta content="https://asset.logseq.com/static/img/logo.png" name="twitter:image:src">
<meta content="A local-first knowledge base which can be synced using Git." name="twitter:image:alt">
<meta content="A local-first knowledge base." property="og:title">
<meta content="A privacy-first, open-source platform for knowledge management and collaboration." name="twitter:image:alt">
<meta content="A privacy-first, open-source platform for knowledge management and collaboration." property="og:title">
<meta content="site" property="og:type">
<meta content="https://logseq.com" property="og:url">
<meta content="https://asset.logseq.com/static/img/logo.png" property="og:image">
<meta content="A local-first knowledge base which can be synced using Git." property="og:description">
<title>Logseq: A local-first knowledge base</title>
<meta content="A privacy-first, open-source platform for knowledge management and collaboration." property="og:description">
<title>Logseq: "A privacy-first platform for knowledge management and collaboration."</title>
<meta content="logseq" property="og:site_name">
<meta description="A local-first knowledge base which can be synced using Git.">
<meta content="A privacy-first, open-source platform for knowledge management and collaboration." name="description">
</head>
<body>
<div id="root"></div>
<script>window.user = null</script>
<script>window.__LSP__HOST__ = true</script>
<script defer src="/static/js/lsplugin.core.js"></script>
<script src="/static/js/magic_portal.js"></script>
<script>let worker = new Worker('/static/js/worker.js')
const portal = new MagicPortal(worker);
@@ -48,8 +47,12 @@ const portal = new MagicPortal(worker);
window.workerThread = workerThread
})()
</script>
<script src="/static/js/main.js"></script>
<script src="/static/js/highlight.min.js"></script>
<script src="/static/js/interact.min.js"></script>
<script defer src="/static/js/highlight.min.js"></script>
<script defer src="/static/js/interact.min.js"></script>
<script defer src="/static/js/lsplugin.core.js"></script>
<script defer src="/static/js/main.js"></script>
<script defer src="/static/js/code-editor.js"></script>
<script defer src="/static/js/age-encryption.js"></script>
<script defer src="/static/js/excalidraw.js"></script>
</body>
</html>

62
resources/electron-dev.html
View File

@@ -1,62 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta content="minimum-scale=1, initial-scale=1, width=device-width, shrink-to-fit=no" name="viewport">
<link href="./css/style.css" rel="stylesheet" type="text/css">
<link href="./img/logo.png" rel="shortcut icon" type="image/png">
<link href="./img/logo.png" rel="shortcut icon" sizes="192x192">
<link href="./img/logo.png" rel="apple-touch-icon">
<meta content="Logseq" name="apple-mobile-web-app-title">
<meta content="yes" name="apple-mobile-web-app-capable">
<meta content="yes" name="apple-touch-fullscreen">
<meta content="black-translucent" name="apple-mobile-web-app-status-bar-style">
<meta content="yes" name="mobile-web-app-capable">
<meta content="summary" name="twitter:card">
<meta content="A privacy-first, open-source platform for knowledge management and collaboration." name="twitter:description">
<meta content="@logseq" name="twitter:site">
<meta content="A privacy-first, open-source platform for knowledge management and collaboration." name="twitter:title">
<meta content="https://asset.logseq.com/static/img/logo.png" name="twitter:image:src">
<meta content="A privacy-first, open-source platform for knowledge management and collaboration." name="twitter:image:alt">
<meta content="A privacy-first, open-source platform for knowledge management and collaboration." property="og:title">
<meta content="site" property="og:type">
<meta content="https://logseq.com" property="og:url">
<meta content="https://asset.logseq.com/static/img/logo.png" property="og:image">
<meta content="A privacy-first, open-source platform for knowledge management and collaboration." property="og:description">
<title>Logseq: A privacy-first platform for knowledge management and collaboration</title>
<meta content="logseq" property="og:site_name">
<meta content="A privacy-first, open-source platform for knowledge management and collaboration." name="description">
<script>window.localStorage.getItem('http-entry-port') && (location.href = 'http://localhost:' + window.localStorage.getItem('http-entry-port'))</script>
</head>
<body>
<div id="root"></div>
<script>window.__LSP__HOST__ = true</script>
<script>window.user = null</script>
<script>
window.EXCALIDRAW_ASSET_PATH = "./js/";
</script>
<script src="./js/magic_portal.js"></script>
<script>let worker = new Worker('./js/worker.js')
const portal = new MagicPortal(worker);
(async () => {
const git = await portal.get('git')
window.git = git
const fs = await portal.get('fs')
window.fs = fs
const pfs = await portal.get('pfs')
window.pfs = pfs
const gitHttp = await portal.get('gitHttp')
window.gitHttp = gitHttp
const workerThread = await portal.get('workerThread')
window.workerThread = workerThread
})()
</script>
<script defer src="./js/highlight.min.js"></script>
<script defer src="./js/interact.min.js"></script>
<script defer src="./js/lsplugin.core.js"></script>
<script defer src="./js/main.js"></script>
<script defer src="./js/code-editor.js"></script>
<script defer src="./js/age-encryption.js"></script>
<script defer src="./js/excalidraw.js"></script>
</body>
</html>

4
resources/package.json
View File

@@ -18,7 +18,7 @@
"forge": "./forge.config.js"
},
"dependencies": {
"better-sqlite3": "7.4.1",
"better-sqlite3": "7.4.3",
"chokidar": "^3.5.1",
"electron-log": "^4.3.1",
"electron-squirrel-startup": "^1.0.0",
@@ -36,7 +36,7 @@
"@electron-forge/maker-rpm": "^6.0.0-beta.57",
"@electron-forge/maker-squirrel": "^6.0.0-beta.57",
"@electron-forge/maker-zip": "^6.0.0-beta.57",
"electron": "^13.0.0",
"electron": "^13.1.9",
"electron-builder": "^22.11.7",
"electron-forge-maker-appimage": "trusktr/electron-forge-maker-appimage#patch-1",
"electron-rebuild": "^2.3.5"

4946
resources/yarn.lock Normal file
View File

File diff suppressed because it is too large Load Diff

10
shadow-cljs.edn
View File

@@ -29,7 +29,6 @@
:output-feature-set :es-next-in
:source-map true
:source-map-include-sources-content true
:source-map-detail-level :all
:externs ["datascript/externs.js"
"externs.js"]
:warnings {:fn-deprecated false}}
@@ -62,6 +61,13 @@
"externs.js"]
:warnings {:fn-deprecated false}}}
:parser-worker
{:target :browser
:output-dir "./static/js"
:modules {:parser-worker {:entries [frontend.worker.parser]
:web-worker true}}
:release {:compiler-options {:infer-externs :auto}}}
:test
{:target :node-test
:output-to "static/tests.js"
@@ -114,7 +120,7 @@
:after-load nubank.workspaces.core/after-load
:loader-mode :eval
:http-root "public/workspaces"
:http-port 9670
:http-port 3002
:watch-path "static"
:preloads [devtools.preload
shadow.remote.runtime.cljs.browser]}

5
src/electron/electron/core.cljs
View File

@@ -13,8 +13,9 @@
[clojure.core.async :as async]
[electron.state :as state]))
(def ROOT_PATH (path/join js/__dirname ".."))
(def MAIN_WINDOW_ENTRY (str "file://" (path/join js/__dirname (if dev? "electron-dev.html" "electron.html"))))
(def MAIN_WINDOW_ENTRY (if dev?
"http://localhost:3001"
(str "file://" (path/join js/__dirname "electron.html"))))
(defonce *setup-fn (volatile! nil))
(defonce *teardown-fn (volatile! nil))

3
src/electron/electron/handler.cljs
View File

@@ -195,6 +195,9 @@
(cfgs/set-item! (keyword k) v)
(cfgs/get-item (keyword k)))))
(defmethod handle :getDirname [_]
js/__dirname)
(defmethod handle :default [args]
(println "Error: no ipc handler for: " (bean/->js args)))

2
src/main/frontend/components/block.cljs
View File

@@ -706,7 +706,7 @@
[:div
[:img.w-full.h-full.absolute
{:src (if (util/electron?)
"img/tutorial-thumb.jpg"
(str (config/get-static-path) "img/tutorial-thumb.jpg")
"https://img.youtube.com/vi/Afmqowr0qEQ/maxresdefault.jpg")}]
[:button
{:class "absolute bg-red-300 w-16 h-16 -m-8 top-1/2 left-1/2 rounded-full"

7
src/main/frontend/components/onboarding.cljs
View File

@@ -7,7 +7,8 @@
[frontend.handler.route :as route-handler]
[frontend.ui :as ui]
[frontend.util :as util]
[rum.core :as rum]))
[rum.core :as rum]
[frontend.config :as config]))
(rum/defc intro
[]
@@ -47,7 +48,7 @@
[:img.shadow-2xl
{:src
(if (util/electron?)
"img/screenshot.png"
(str (config/get-static-path) "img/screenshot.png")
"https://cdn.logseq.com/%2F8b9a461d-437e-4ca5-a2da-18b51077b5142020_07_25_Screenshot%202020-07-25%2013-29-49%20%2B0800.png?Expires=4749255017&Signature=Qbx6jkgAytqm6nLxVXQQW1igfcf~umV1OcG6jXUt09TOVhgXyA2Z5jHJ3AGJASNcphs31pZf4CjFQ5mRCyVKw6N8wb8Nn-MxuTJl0iI8o-jLIAIs9q1v-2cusCvuFfXH7bq6ir8Lpf0KYAprzuZ00FENin3dn6RBW35ENQwUioEr5Ghl7YOCr8bKew3jPV~OyL67MttT3wJig1j3IC8lxDDT8Ov5IMG2GWcHERSy00F3mp3tJtzGE17-OUILdeuTFz6d-NDFAmzB8BebiurYz0Bxa4tkcdLUpD5ToFHU08jKzZExoEUY8tvaZ1-t7djmo3d~BAXDtlEhC2L1YC2aVQ__&Key-Pair-Id=APKAJE5CCD6X7MP6PTEA")
:alt "screenshot"}]
@@ -171,7 +172,7 @@
[:img {:src
(if (util/electron?)
"img/credits.png"
(str (config/get-static-path) "img/credits.png")
"https://asset.logseq.com/static/img/credits.png")
:style {:margin "12px 0 0 0"}}]]]]))

6
src/main/frontend/config.cljs
View File

@@ -389,3 +389,9 @@
(defn get-block-hidden-properties
[]
(get-in @state/state [:config (state/get-current-repo) :block-hidden-properties]))
(defn get-static-path
[]
(if (and (util/electron?) dev?)
"static/"
""))

22
src/main/frontend/format/mldoc.cljs
View File

@@ -10,7 +10,9 @@
[lambdaisland.glogi :as log]
[medley.core :as medley]
["mldoc" :as mldoc :refer [Mldoc]]
[linked.core :as linked]))
[linked.core :as linked]
[promesa.core :as p]
[frontend.util.pool :as pool]))
(defonce parseJson (gobj/get Mldoc "parseJson"))
(defonce parseInlineJson (gobj/get Mldoc "parseInlineJson"))
@@ -230,7 +232,7 @@
[content config]
(try
(if (string/blank? content)
{}
[]
(-> content
(parse-json config)
(util/json->clj)
@@ -240,6 +242,22 @@
(log/error :edn/convert-failed e)
[])))
(defn ->edn-async
[content config]
(if util/node-test?
(p/resolved (->edn content config))
(try
(if (string/blank? content)
(p/resolved [])
(p/let [v (pool/add-parse-job! content config)]
(-> v
(util/json->clj)
(update-src-full-content content)
(collect-page-properties))))
(catch js/Error e
(log/error :edn/convert-failed e)
(p/resolved [])))))
(defn opml->edn
[content]
(try

4
src/main/frontend/handler.cljs
View File

@@ -27,7 +27,8 @@
[lambdaisland.glogi :as log]
[promesa.core :as p]
[frontend.ui :as ui]
[frontend.error :as error]))
[frontend.error :as error]
[frontend.util.pool :as pool]))
(defn set-global-error-notification!
[]
@@ -200,6 +201,7 @@
(reset! db/*sync-search-indice-f search/sync-search-indice!)
(db/run-batch-txs!)
(file-handler/run-writes-chan!)
(pool/init-parser-pool!)
(when (util/electron?)
(el/listen!))))

108
src/main/frontend/handler/extract.cljs
View File

@@ -17,7 +17,8 @@
[medley.core :as medley]
[clojure.walk :as walk]
[frontend.state :as state]
[frontend.config :as config]))
[frontend.config :as config]
[promesa.core :as p]))
(defn- extract-page-list
[content]
@@ -149,30 +150,31 @@
(extract-blocks-pages repo-url file content (utf8/encode content)))
([repo-url file content utf8-content]
(if (string/blank? content)
[]
(let [journal? (config/journal? file)
format (format/get-format file)
ast (mldoc/->edn content
(mldoc/default-config format))
first-block (ffirst ast)
properties (let [properties (and (property/properties-ast? first-block)
(->> (last first-block)
(map (fn [[x y]]
[x (if (string? y)
(property/parse-property x y)
y)]))
(into {})
(walk/keywordize-keys)))]
(when (and properties (seq properties))
(if (:filters properties)
(update properties :filters
(fn [v]
(string/replace (or v "") "\\" "")))
properties)))]
(extract-pages-and-blocks
repo-url
format ast properties
file content utf8-content journal?)))))
(p/resolved [])
(p/let [format (format/get-format file)
_ (println "Parsing start : " file)
ast (mldoc/->edn-async content (mldoc/default-config format))]
_ (println "Parsing finished : " file)
(let [journal? (config/journal? file)
first-block (ffirst ast)
properties (let [properties (and (property/properties-ast? first-block)
(->> (last first-block)
(map (fn [[x y]]
[x (if (string? y)
(property/parse-property x y)
y)]))
(into {})
(walk/keywordize-keys)))]
(when (and properties (seq properties))
(if (:filters properties)
(update properties :filters
(fn [v]
(string/replace (or v "") "\\" "")))
properties)))]
(extract-pages-and-blocks
repo-url
format ast properties
file content utf8-content journal?))))))
(defn with-block-uuid
[pages]
@@ -207,31 +209,31 @@
(defn extract-all-blocks-pages
[repo-url files metadata refresh?]
(when (seq files)
(let [result (->> files
(map
(fn [{:file/keys [path content]} contents]
(println "Parsing : " path)
(when content
;; TODO: remove `text/scheduled-deadline-dash->star` once migration is done
(let [org? (= "org" (string/lower-case (util/get-file-ext path)))]
(let [content (if org?
content
(text/scheduled-deadline-dash->star content))
utf8-content (utf8/encode content)]
(extract-blocks-pages repo-url path content utf8-content))))))
(remove empty?))]
(when (seq result)
(let [[pages block-ids blocks] (apply map concat result)
block-ids (remove (fn [b] (or (nil? b)
(nil? (:block/uuid b)))) block-ids)
pages (with-ref-pages pages blocks)
blocks (map (fn [block]
(let [id (:block/uuid block)
properties (get-in metadata [:block/properties id])]
(update block :block/properties merge properties)))
blocks)
;; To prevent "unique constraint" on datascript
pages-index (map #(select-keys % [:block/name]) pages)
block-ids-set (set (map (fn [{:block/keys [uuid]}] [:block/uuid uuid]) block-ids))
blocks (map #(remove-illegal-refs % block-ids-set refresh?) blocks)]
(apply concat [pages-index pages block-ids blocks]))))))
(-> (p/all (map
(fn [{:file/keys [path content]} contents]
(when content
;; TODO: remove `text/scheduled-deadline-dash->star` once migration is done
(let [org? (= "org" (string/lower-case (util/get-file-ext path)))]
(let [content (if org?
content
(text/scheduled-deadline-dash->star content))
utf8-content (utf8/encode content)]
(extract-blocks-pages repo-url path content utf8-content)))))
files))
(p/then (fn [result]
(let [result (remove empty? result)]
(when (seq result)
(let [[pages block-ids blocks] (apply map concat result)
block-ids (remove (fn [b] (or (nil? b)
(nil? (:block/uuid b)))) block-ids)
pages (with-ref-pages pages blocks)
blocks (map (fn [block]
(let [id (:block/uuid block)
properties (get-in metadata [:block/properties id])]
(update block :block/properties merge properties)))
blocks)
;; To prevent "unique constraint" on datascript
pages-index (map #(select-keys % [:block/name]) pages)
block-ids-set (set (map (fn [{:block/keys [uuid]}] [:block/uuid uuid]) block-ids))
blocks (map #(remove-illegal-refs % block-ids-set refresh?) blocks)]
(apply concat [pages-index pages block-ids blocks])))))))))

29
src/main/frontend/handler/file.cljs
View File

@@ -141,21 +141,20 @@
(db/set-file-content! repo-url file content)
(let [format (format/get-format file)
utf8-content (utf8/encode content)
file-content [{:file/path file}]
tx (if (contains? config/mldoc-support-formats format)
(let [delete-blocks (db/delete-file-blocks! repo-url file)
[pages block-ids blocks] (extract-handler/extract-blocks-pages repo-url file content utf8-content)
blocks (remove-non-exists-refs! blocks)
_ (util/pprint blocks)
pages (extract-handler/with-ref-pages pages blocks)]
(concat file-content delete-blocks pages block-ids blocks))
file-content)
tx (concat tx [(let [t (tc/to-long (t/now))]
(cond->
{:file/path file}
new?
(assoc :file/created-at t)))])]
(db/transact! repo-url tx))))
file-content [{:file/path file}]]
(p/let [tx (if (contains? config/mldoc-support-formats format)
(p/let [delete-blocks (db/delete-file-blocks! repo-url file)
[pages block-ids blocks] (extract-handler/extract-blocks-pages repo-url file content utf8-content)
blocks (remove-non-exists-refs! blocks)
pages (extract-handler/with-ref-pages pages blocks)]
(concat file-content delete-blocks pages block-ids blocks))
file-content)]
(let [tx (concat tx [(let [t (tc/to-long (t/now))]
(cond->
{:file/path file}
new?
(assoc :file/created-at t)))])]
(db/transact! repo-url tx))))))
;; TODO: Remove this function in favor of `alter-files`
(defn alter-file

35
src/main/frontend/handler/repo.cljs
View File

@@ -121,14 +121,14 @@
_ (fs/mkdir-if-not-exists (str repo-dir "/" (config/get-journals-directory)))
file-exists? (fs/file-exists? repo-dir file-path)]
(when-not file-exists?
(file-handler/reset-file! repo-url path content)
(if write-file?
(p/let [_ (fs/create-if-not-exists repo-url repo-dir file-path content)]
(p/let [_ (file-handler/reset-file! repo-url path content)]
(if write-file?
(p/let [_ (fs/create-if-not-exists repo-url repo-dir file-path content)]
(when-not (state/editing?)
(ui-handler/re-render-root!))
(git-handler/git-add repo-url path))
(when-not (state/editing?)
(ui-handler/re-render-root!))
(git-handler/git-add repo-url path))
(when-not (state/editing?)
(ui-handler/re-render-root!))))))))))
(ui-handler/re-render-root!)))))))))))
(defn create-default-files!
([repo-url]
@@ -199,15 +199,15 @@
(defn- parse-files-and-create-default-files-inner!
[repo-url files delete-files delete-blocks file-paths first-clone? db-encrypted? re-render? re-render-opts metadata opts]
(let [refresh? (:refresh? opts)
parsed-files (filter
(fn [file]
(let [format (format/get-format (:file/path file))]
(contains? config/mldoc-support-formats format)))
files)
blocks-pages (if (seq parsed-files)
(extract-handler/extract-all-blocks-pages repo-url parsed-files metadata refresh?)
[])]
(p/let [refresh? (:refresh? opts)
parsed-files (filter
(fn [file]
(let [format (format/get-format (:file/path file))]
(contains? config/mldoc-support-formats format)))
files)
blocks-pages (if (seq parsed-files)
(extract-handler/extract-all-blocks-pages repo-url parsed-files metadata refresh?)
[])]
(let [config-file (config/get-config-path)]
(when (contains? (set file-paths) config-file)
(when-let [content (some #(when (= (:file/path %) config-file)
@@ -549,7 +549,8 @@
(create-config-file-if-not-exists repo)
(create-contents-file repo)
(create-custom-theme repo)
(state/set-db-restoring! false)))
(state/set-db-restoring! false)
(ui-handler/re-render-root!)))
(js/setTimeout setup-local-repo-if-not-exists! 100)))
(defn periodically-pull-current-repo

19
src/main/frontend/handler/web/nfs.cljs
View File

@@ -157,16 +157,15 @@
(p/then (fn [result]
(let [files (map #(dissoc % :file/file) result)]
(repo-handler/start-repo-db-if-not-exists! repo {:db-type :local-native-fs})
(repo-handler/load-repo-to-db! repo
{:first-clone? true
:nfs-files files})
(state/add-repo! {:url repo :nfs? true})
(state/set-loading-files! false)
(and ok-handler (ok-handler))
(when (util/electron?)
(fs/watch-dir! dir-name))
(state/pub-event! [:graph/added repo]))))
(p/let [_ (repo-handler/load-repo-to-db! repo
{:first-clone? true
:nfs-files files})]
(state/add-repo! {:url repo :nfs? true})
(state/set-loading-files! false)
(and ok-handler (ok-handler))
(when (util/electron?)
(fs/watch-dir! dir-name))
(state/pub-event! [:graph/added repo])))))
(p/catch (fn [error]
(log/error :nfs/load-files-error repo)
(log/error :exception error)))))

67
src/main/frontend/util/pool.cljs Normal file
View File

@@ -0,0 +1,67 @@
(ns frontend.util.pool
(:require ["threads" :refer [spawn Pool Worker]]
[promesa.core :as p]
[frontend.util :as util]
["path" :as path]
[electron.ipc :as ipc]
[frontend.config :as config]))
(defonce parser-pool (atom nil))
(defn create-parser-pool!
([]
(create-parser-pool! 8))
([num]
(p/let [static-path (if (and (util/electron?) (not config/dev?))
(ipc/ipc :getDirname)
"/static")]
(Pool.
(fn []
(spawn (Worker. (str static-path "/js/parser-worker.js")) num))))))
;; (defn finish-pool!
;; [{:keys [pool tasks]} ok-handler]
;; (-> (p/all @tasks)
;; (p/then (fn [result]
;; (ok-handler result)
;; (.completed pool)
;; (.terminate pool)
;; (reset! tasks nil)))))
(defn terminate-pool!
[^js pool]
(p/let [_ (.completed pool)]
(.terminate pool)))
(defn terminate-parser-pool!
[]
(when-let [pool @parser-pool]
(terminate-pool! pool)))
(defn add-parse-job!
[content config]
(when-let [pool @parser-pool]
(.queue ^js pool
(fn [parser]
(try
(parser.parse content config)
(catch js/Error e
(js/console.error e)
nil)))))
;; (let [task (.queue ^js pool
;; (fn [parser]
;; (parser.parse content config)))]
;; (swap! (:tasks m) conj task)
;; task)
)
(defn init-parser-pool!
[]
(p/let [pool (create-parser-pool!)]
(reset! parser-pool pool)))
(comment
(add-parse-job! "- hello" (frontend.format.mldoc/default-config :markdown))
(add-parse-job! "*world*" (frontend.format.mldoc/default-config :markdown))
)

14
src/main/frontend/worker/parser.cljs Normal file
View File

@@ -0,0 +1,14 @@
(ns frontend.worker.parser
(:require ["mldoc" :refer [Mldoc]]
["threads/worker" :refer [expose]]))
(def parse-json (.-parseJson Mldoc))
(expose (clj->js {:parse parse-json}))
(defn init
[]
(println "Parser worker initialized!")
(js/self.addEventListener "message"
(fn [^js e]
(js/postMessage (.. e -data)))))

8
src/test/frontend/db/query_dsl_test.cljs
View File

@@ -7,7 +7,7 @@
[frontend.db-schema :as schema]
[frontend.handler.repo :as repo-handler]
[promesa.core :as p]
[cljs.test :refer [deftest is are testing use-fixtures run-tests]]))
[cljs.test :refer [deftest is are testing use-fixtures run-tests async]]))
;; TODO: quickcheck
;; 1. generate query filters
@@ -457,8 +457,10 @@ last-modified-at:: 1609084800002"}]]
(use-fixtures :once
{:before (fn []
(config/start-test-db!)
(import-test-data!))
(async done
(config/start-test-db!)
(p/let [_ (import-test-data!)]
(done))))
:after config/destroy-test-db!})
#_(run-tests)

60
src/test/frontend/handler/extract_test.cljs
View File

@@ -1,36 +1,49 @@
(ns frontend.handler.extract-test
(:require [cljs.test :refer [deftest is are testing use-fixtures run-tests]]
(:require [cljs.test :refer [deftest is are testing use-fixtures run-tests async]]
[cljs-run-test :refer [run-test]]
[frontend.util :as util]
[frontend.handler.extract :as extract]))
[frontend.handler.extract :as extract]
[promesa.core :as p]))
(defn- extract
[text]
(let [result (last (extract/extract-blocks-pages "repo" "a.md" text))
lefts (map (juxt :block/parent :block/left) result)]
(p/let [result (extract/extract-blocks-pages "repo" "a.md" text)
result (last result)
lefts (map (juxt :block/parent :block/left) result)]
(if (not= (count lefts) (count (distinct lefts)))
(do
(util/pprint (map (fn [x] (select-keys x [:block/uuid :block/level :block/content :block/left])) result))
(throw (js/Error. ":block/parent && :block/left conflicts")))
(mapv (juxt :block/level :block/content) result))))
(defn- async-test
[x y]
(async done
(p/then
(extract x)
(fn [v]
(is (= y))
(done)))))
(deftest test-extract-blocks-pages
[]
(are [x y] (= (extract x) y)
"- a
(async-test
"- a
- b
- c"
[[1 "a"] [2 "b"] [3 "c"]]
[[1 "a"] [2 "b"] [3 "c"]])
"## hello
(async-test
"## hello
- world
- nice
- nice
- bingo
- world"
[[1 "## hello"] [2 "world"] [3 "nice"] [4 "nice"] [3 "bingo"] [3 "world"]]
[[1 "## hello"] [2 "world"] [3 "nice"] [4 "nice"] [3 "bingo"] [3 "world"]])
"# a
(async-test
"# a
## b
### c
#### d
@@ -40,24 +53,25 @@
- h
- i
- j"
[[1 "# a"]
[1 "## b"]
[1 "### c"]
[1 "#### d"]
[1 "### e"]
[1 "f"]
[2 "g"]
[3 "h"]
[2 "i"]
[1 "j"]]))
[[1 "# a"]
[1 "## b"]
[1 "### c"]
[1 "#### d"]
[1 "### e"]
[1 "f"]
[2 "g"]
[3 "h"]
[2 "i"]
[1 "j"]]))
(deftest test-regression-1902
[]
(are [x y] (= (extract x) y)
"- line1
(async-test
"- line1
- line2
- line3
- line4"
[[1 "line1"] [2 "line2"] [3 "line3"] [3 "line4"]]))
[[1 "line1"] [2 "line2"] [3 "line3"] [3 "line4"]]))
#_(run-tests)

53
yarn.lock
View File

@@ -2399,7 +2399,7 @@ callsites@^2.0.0:
resolved "https://registry.yarnpkg.com/callsites/-/callsites-2.0.0.tgz"
integrity sha1-BuuE8A7qQT2oav/vrL/7Ngk7PFA=
callsites@^3.0.0:
callsites@^3.0.0, callsites@^3.1.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz"
integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==
@@ -3339,7 +3339,7 @@ dayjs@^1.10.4:
resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.10.5.tgz#5600df4548fc2453b3f163ebb2abbe965ccfb986"
integrity sha512-BUFis41ikLz+65iH6LHQCDm4YPMj5r1YFLdupPIyM4SGcXMmtiLQ7U37i+hGS8urIuqe7I/ou3IS1jVc4nbN4g==
debug@4, debug@4.3.2:
debug@4, debug@4.3.2, debug@^4.2.0:
version "4.3.2"
resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.2.tgz#f0a49c18ac8779e31d4a0c6029dfb76873c7428b"
integrity sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==
@@ -3830,6 +3830,11 @@ escape-string-regexp@^4.0.0:
resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz"
integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==
esm@^3.2.25:
version "3.2.25"
resolved "https://registry.yarnpkg.com/esm/-/esm-3.2.25.tgz#342c18c29d56157688ba5ce31f8431fbb795cc10"
integrity sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==
esprima@^4.0.0:
version "4.0.1"
resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz"
@@ -5303,6 +5308,11 @@ is-obj@^2.0.0:
resolved "https://registry.yarnpkg.com/is-obj/-/is-obj-2.0.0.tgz"
integrity sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==
is-observable@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/is-observable/-/is-observable-2.1.0.tgz#5c8d733a0b201c80dff7bb7c0df58c6a255c7c69"
integrity sha512-DailKdLb0WU+xX8K5w7VsJhapwHLZ9jjmazqCJq4X12CTgqq73TKnbRcnSLuXYPOoLQgV5IrD7ePiX/h1vnkBw==
is-path-cwd@^2.2.0:
version "2.2.0"
resolved "https://registry.yarnpkg.com/is-path-cwd/-/is-path-cwd-2.2.0.tgz#67d43b82664a7b5191fd9119127eb300048a9fdb"
@@ -6532,6 +6542,11 @@ obliterator@^1.6.1:
resolved "https://registry.yarnpkg.com/obliterator/-/obliterator-1.6.1.tgz#dea03e8ab821f6c4d96a299e17aef6a3af994ef3"
integrity sha512-9WXswnqINnnhOG/5SLimUlzuU1hFJUc8zkwyD59Sd+dPOMf05PmnYG/d6Q7HZ+KmgkZJa1PxRso6QdM3sTNHig==
observable-fns@^0.6.1:
version "0.6.1"
resolved "https://registry.yarnpkg.com/observable-fns/-/observable-fns-0.6.1.tgz#636eae4fdd1132e88c0faf38d33658cc79d87e37"
integrity sha512-9gRK4+sRWzeN6AOewNBTLXir7Zl/i3GB6Yl26gK4flxz8BXVpD3kt8amREmWNb0mxYOGDotvE5a4N+PtGGKdkg==
once@^1.3.0, once@^1.3.1, once@^1.3.2, once@^1.4.0:
version "1.4.0"
resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz"
@@ -8246,17 +8261,17 @@ shadow-cljs@2.10.18:
which "^1.3.1"
ws "^3.0.0"
shadow-cljs@2.12.5:
version "2.12.5"
resolved "https://registry.yarnpkg.com/shadow-cljs/-/shadow-cljs-2.12.5.tgz#d3cf29fc1f1e02dd875939549419979e0feadbf4"
integrity sha512-o3xo3coRgnlkI/iI55ccHjj6AU3F1+ovk3hhK86e3P2JGGOpNTAwsGNxUpMC5JAwS9Nz0v6sSk73hWjEOnm6fQ==
shadow-cljs@2.15.3:
version "2.15.3"
resolved "https://registry.yarnpkg.com/shadow-cljs/-/shadow-cljs-2.15.3.tgz#11a0a04f1c31d9277838a5f649457edbb06baafb"
integrity sha512-KK7G9kSc0dwrOkN74o5yrxhrpsJ3BJCL11nGHBakzHLodc7pdiCLDeq2ChXyKl2yu9aJIzfSz8Hum38wpLQ1DA==
dependencies:
node-libs-browser "^2.2.1"
readline-sync "^1.4.7"
shadow-cljs-jar "1.3.2"
source-map-support "^0.4.15"
which "^1.3.1"
ws "^3.0.0"
ws "^7.4.6"
shallowequal@^1.1.0:
version "1.1.0"
@@ -8930,6 +8945,18 @@ tailwindcss@2.2.4:
resolve "^1.20.0"
tmp "^0.2.1"
threads@^1.6.5:
version "1.6.5"
resolved "https://registry.yarnpkg.com/threads/-/threads-1.6.5.tgz#5cee7f139e3e147c5a64f0134844ee92469932a5"
integrity sha512-yL1NN4qZ25crW8wDoGn7TqbENJ69w3zCEjIGXpbqmQ4I+QHrG8+DLaZVKoX74OQUXWCI2lbbrUxDxAbr1xjDGQ==
dependencies:
callsites "^3.1.0"
debug "^4.2.0"
is-observable "^2.1.0"
observable-fns "^0.6.1"
optionalDependencies:
tiny-worker ">= 2"
throttleit@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/throttleit/-/throttleit-1.0.0.tgz#9e785836daf46743145a5984b6268d828528ac6c"
@@ -8992,6 +9019,13 @@ tiny-typed-emitter@^2.0.3:
resolved "https://registry.yarnpkg.com/tiny-typed-emitter/-/tiny-typed-emitter-2.0.3.tgz#4335e3a75127ae7faba91b02e91615d97dc8db7d"
integrity sha512-MaCqhHlp6EAWN25yqBlajgd4scxxI2eJr7+EgoUAOV9UkMU3us/yp2bEnc2yOvyeDF8TUWuaz3zZCPGTKFJIpA==
"tiny-worker@>= 2":
version "2.3.0"
resolved "https://registry.yarnpkg.com/tiny-worker/-/tiny-worker-2.3.0.tgz#715ae34304c757a9af573ae9a8e3967177e6011e"
integrity sha512-pJ70wq5EAqTAEl9IkGzA+fN0836rycEuz2Cn6yeZ6FRzlVS5IDOkFHpIoEsksPRQV34GDqXm65+OlnZqUSyK2g==
dependencies:
esm "^3.2.25"
tippy.js@^6.3.1:
version "6.3.1"
resolved "https://registry.yarnpkg.com/tippy.js/-/tippy.js-6.3.1.tgz#3788a007be7015eee0fd589a66b98fb3f8f10181"
@@ -9653,6 +9687,11 @@ ws@^3.0.0:
safe-buffer "~5.1.0"
ultron "~1.1.0"
ws@^7.4.6:
version "7.5.3"
resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.3.tgz#160835b63c7d97bfab418fc1b8a9fced2ac01a74"
integrity sha512-kQ/dHIzuLrS6Je9+uv81ueZomEwH0qVYstcAQ4/Z93K8zeko9gtAbttJWzoC5ukqXY1PpoouV3+VSOqEAFt5wg==
xtend@^4.0.0, xtend@^4.0.1, xtend@^4.0.2, xtend@~4.0.0, xtend@~4.0.1:
version "4.0.2"
resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz"