Improve long page editing performance (#3855)

* Remove expensive parsing when saving files

* Add limit to page blocks query

* Don't collapse block's body to make it compatible with other tools

* Alert if there're unsaved changes when switching graphs

* DB schema migration for :block/collapsed? from it's property

Co-authored-by: Andelf <andelf@gmail.com>
This commit is contained in:
Tienson Qin
2022-01-18 10:37:31 +08:00
committed by GitHub
parent 6a151937d7
commit 6aba8c3241
28 changed files with 429 additions and 348 deletions

View File

@@ -139,6 +139,7 @@ jobs:
- name: Run Playwright test
run: xvfb-run -- yarn e2e-test
env:
CI: true
DEBUG: "pw:test"
- name: Save test artifacts

View File

@@ -85,7 +85,7 @@ test('create page and blocks', async ({ page }) => {
await page.waitForTimeout(500)
expect(await page.$$('.ls-block')).toHaveLength(5)
await page.waitForTimeout(500)
await page.waitForTimeout(1000)
const contentOnDisk = await fs.readFile(
path.join(graphDir, `pages/${pageTitle}.md`),

View File

@@ -10,7 +10,7 @@ test('create page and blocks (diacritics)', async ({ page }) => {
hotkeyOpenLink = 'Meta+o'
hotkeyBack = 'Meta+['
}
const rand = randomString(20)
// diacritic opening test
@@ -22,14 +22,9 @@ test('create page and blocks (diacritics)', async ({ page }) => {
// build target Page with diacritics
await activateNewPage(page)
await page.type(':nth-match(textarea, 1)', 'Diacritic title test content')
await page.keyboard.press(hotkeyBack)
// visit target Page with diacritics (looks same but not same in Unicode)
await newBlock(page)
await page.type(':nth-match(textarea, 1)', '[[Einführung in die Allgemeine Sprachwissenschaft' + rand + ']] diacritic-block-2')
await page.keyboard.press(hotkeyOpenLink)
await lastInnerBlock(page)
expect(await page.inputValue(':nth-match(textarea, 1)')).toBe('Diacritic title test content')
await page.keyboard.press('Enter')
await page.fill(':nth-match(textarea, 1)', '[[Einführung in die Allgemeine Sprachwissenschaft' + rand + ']] diacritic-block-2')
await page.keyboard.press(hotkeyBack)
// check if diacritics are indexed
@@ -41,4 +36,4 @@ test('create page and blocks (diacritics)', async ({ page }) => {
const results = await page.$$('#ui__ac-inner .block')
expect(results.length).toEqual(3) // 2 blocks + 1 page
await page.keyboard.press("Escape")
})
})

View File

@@ -57,7 +57,7 @@ test('switch code editing mode', async ({ page }) => {
await page.waitForSelector('.CodeMirror pre', { state: 'hidden' })
expect(await page.inputValue('.block-editor textarea')).toBe('```clojure\n;; comment\n\n \n(+ 1 1)\n```')
await page.waitForTimeout(500)
await page.waitForTimeout(1000)
})
@@ -70,13 +70,13 @@ test('convert from block content to code', async ({ page }) => {
await page.press('.block-editor textarea', 'Escape')
await page.waitForSelector('.CodeMirror pre', { state: 'visible' })
await page.waitForTimeout(500)
await page.waitForTimeout(1000)
await page.click('.CodeMirror pre')
await page.waitForTimeout(500)
await page.waitForTimeout(1000)
expect(await page.locator('.CodeMirror-gutter-wrapper .CodeMirror-linenumber >> nth=-1').innerText()).toBe('1')
await page.press('.CodeMirror textarea', 'Escape')
await page.waitForTimeout(500)
await page.waitForTimeout(1000)
expect(await page.inputValue('.block-editor textarea')).toBe('```\n```')
@@ -113,13 +113,13 @@ test('code block mixed input source', async ({ page }) => {
await createRandomPage(page)
await page.fill('.block-editor textarea', '```\n ABC\n```')
await page.waitForTimeout(100) // wait for fill
await page.waitForTimeout(500) // wait for fill
await escapeToCodeEditor(page)
await page.type('.CodeMirror textarea', ' DEF\nGHI')
await page.waitForTimeout(500)
await page.waitForTimeout(1000)
await page.press('.CodeMirror textarea', 'Escape')
await page.waitForTimeout(500)
await page.waitForTimeout(1000)
// NOTE: auto-indent is on
expect(await page.inputValue('.block-editor textarea')).toBe('```\n ABC DEF\n GHI\n```')
})
@@ -128,13 +128,13 @@ test('code block with text around', async ({ page }) => {
await createRandomPage(page)
await page.fill('.block-editor textarea', 'Heading\n```\n```\nFooter')
await page.waitForTimeout(100)
await page.waitForTimeout(200)
await escapeToCodeEditor(page)
await page.type('.CodeMirror textarea', 'first\n second')
await page.waitForTimeout(500)
await page.waitForTimeout(1000)
await page.press('.CodeMirror textarea', 'Escape')
await page.waitForTimeout(500)
await page.waitForTimeout(1000)
expect(await page.inputValue('.block-editor textarea')).toBe('Heading\n```\nfirst\n second\n```\nFooter')
})
@@ -149,15 +149,15 @@ test('multiple code block', async ({ page }) => {
await page.waitForSelector('.CodeMirror pre', { state: 'visible' })
// first
await page.waitForTimeout(500)
await page.waitForTimeout(1000)
await page.click('.CodeMirror pre >> nth=0')
await page.waitForTimeout(500)
await page.waitForTimeout(1000)
await page.type('.CodeMirror textarea >> nth=0', ':key-test\n', { strict: true })
await page.waitForTimeout(500)
await page.waitForTimeout(1000)
await page.press('.CodeMirror textarea >> nth=0', 'Escape')
await page.waitForTimeout(500)
await page.waitForTimeout(1000)
expect(await page.inputValue('.block-editor textarea'))
.toBe('中文 Heading\n```clojure\n:key-test\n\n```\nMiddle 🚀\n```clojure\n```\nFooter')
@@ -165,15 +165,15 @@ test('multiple code block', async ({ page }) => {
await page.press('.block-editor textarea', 'Escape')
await page.waitForSelector('.CodeMirror pre', { state: 'visible' })
await page.waitForTimeout(500)
await page.waitForTimeout(1000)
await page.click('.CodeMirror pre >> nth=1')
await page.waitForTimeout(500)
await page.waitForTimeout(1000)
await page.type('.CodeMirror textarea >> nth=1', '\n :key-test 日本語\n', { strict: true })
await page.waitForTimeout(500)
await page.waitForTimeout(1000)
await page.press('.CodeMirror textarea >> nth=1', 'Escape')
await page.waitForTimeout(500)
await page.waitForTimeout(1000)
expect(await page.inputValue('.block-editor textarea'))
.toBe('中文 Heading\n```clojure\n:key-test\n\n```\nMiddle 🚀\n```clojure\n\n :key-test 日本語\n\n```\nFooter')
})
@@ -182,18 +182,18 @@ test('click outside to exit', async ({ page }) => {
await createRandomPage(page)
await page.fill('.block-editor textarea', 'Header ``Click``\n```\n ABC\n```')
await page.waitForTimeout(100) // wait for fill
await page.waitForTimeout(200) // wait for fill
await escapeToCodeEditor(page)
await page.type('.CodeMirror textarea', ' DEF\nGHI')
await page.waitForTimeout(500)
await page.waitForTimeout(1000)
await page.click('text=Click')
await page.waitForTimeout(500)
await page.waitForTimeout(1000)
// NOTE: auto-indent is on
expect(await page.inputValue('.block-editor textarea')).toBe('Header ``Click``\n```\n ABC DEF\n GHI\n```')
})
test('click lanuage label to exit #3463', async ({ page }) => {
test('click language label to exit #3463', async ({ page }) => {
await createRandomPage(page)
await page.press('.block-editor textarea', 'Enter')
@@ -204,9 +204,9 @@ test('click lanuage label to exit #3463', async ({ page }) => {
await escapeToCodeEditor(page)
await page.type('.CodeMirror textarea', '#include<iostream>')
await page.waitForTimeout(500)
await page.waitForTimeout(1000)
await page.click('text=cpp') // the language label
await page.waitForTimeout(500)
await page.waitForTimeout(1000)
expect(await page.inputValue('.block-editor textarea')).toBe('```cpp\n#include<iostream>\n```')
})
@@ -227,12 +227,12 @@ test('multi properties with code', async ({ page }) => {
// first character of code
await page.click('.CodeMirror pre', { position: { x: 1, y: 5 } })
await page.waitForTimeout(500)
await page.waitForTimeout(1000)
await page.type('.CodeMirror textarea', '// Returns nil\n')
await page.waitForTimeout(500)
await page.waitForTimeout(1000)
await page.press('.CodeMirror textarea', 'Escape')
await page.waitForTimeout(500)
await page.waitForTimeout(1000)
expect(await page.inputValue('.block-editor textarea')).toBe(
'type:: code\n' +
'类型:: 代码\n' +

View File

@@ -8,9 +8,14 @@ let electronApp: ElectronApplication
let context: BrowserContext
let page: Page
// NOTE: Will test against a newly opened graph
const repoName = 'Test' + randomString(6)
export const graphDir = path.resolve(__dirname, '../tmp/e2e-graph', repoName)
let repoName = randomString(10)
let testTmpDir = path.resolve(__dirname, '../tmp')
if (fs.existsSync(testTmpDir)) {
fs.rmdirSync(testTmpDir, { recursive: true })
}
export let graphDir = path.resolve(testTmpDir, "e2e-test", repoName)
// NOTE: This is a console log watcher for error logs.
const consoleLogWatcher = (msg: ConsoleMessage) => {

View File

@@ -37,7 +37,7 @@
"dev-electron-app": "gulp electron",
"release-electron": "run-s gulp:build && gulp electronMaker",
"debug-electron": "cd static/ && yarn electron:debug",
"e2e-test": "npx playwright test --reporter github",
"e2e-test": "cross-env CI=true npx playwright test --reporter github",
"run-android-release": "yarn clean && yarn release-app && rm -rf ./public/static && rm -rf ./static/js/*.map && mv static ./public && npx cap sync android && npx cap run android",
"run-ios-release": "yarn clean && yarn release-app && rm -rf ./public/static && rm -rf ./static/js/*.map && mv static ./public && npx cap sync ios && npx cap run ios",
"clean": "gulp clean",

View File

@@ -167,16 +167,17 @@
(defn- get-graphs-dir
[]
(let [dir (.join path (.homedir os) ".logseq" "graphs")]
(let [dir (if utils/ci?
(.resolve path js/__dirname "../tmp/graphs")
(.join path (.homedir os) ".logseq" "graphs"))]
(fs-extra/ensureDirSync dir)
dir))
(defn- get-graphs
[]
(let [dir (get-graphs-dir)
graphs-path (.join path (.homedir os) ".logseq" "graphs")]
(let [dir (get-graphs-dir)]
(->> (readdir dir)
(remove #{graphs-path})
(remove #{dir})
(map #(path/basename % ".transit"))
(map graph-name->path))))

View File

@@ -11,6 +11,11 @@
(defonce linux? (= (.-platform js/process) "linux"))
(defonce prod? (= js/process.env.NODE_ENV "production"))
(defonce ci? (let [v js/process.env.CI]
(or (true? v)
(= v "true"))))
(defonce dev? (not prod?))
(defonce logger (js/require "electron-log"))

View File

@@ -1422,18 +1422,13 @@
(every? #(= % ["Horizontal_Rule"]) body)))
(rum/defcs block-control < rum/reactive
[state config block uuid block-id body children collapsed? *control-show? edit?]
[state config block uuid block-id children collapsed? *control-show? edit?]
(let [doc-mode? (state/sub :document/mode?)
has-children-blocks? (and (coll? children) (seq children))
has-child? (and
(not (:pre-block? block))
(or has-children-blocks?
(and (seq (:block/title block)) (seq body))))
control-show? (and
(or (and (seq (:block/title block))
(seq body))
has-children-blocks?)
(util/react *control-show?))
has-children-blocks?)
control-show? (util/react *control-show?)
ref? (:ref? config)
block? (:block? config)
empty-content? (block-content-empty? block)]
@@ -1864,7 +1859,7 @@
[config {:block/keys [uuid content children properties scheduled deadline format pre-block?] :as block} edit-input-id block-id slide?]
(let [{:block/keys [title body] :as block} (if (:block/title block) block
(merge block (block/parse-title-and-body uuid format pre-block? content)))
collapsed? (get properties :collapsed)
collapsed? (util/collapsed? block)
block-ref? (:block-ref? config)
block-ref-with-title? (and block-ref? (seq title))
block-type (or (:ls-type properties) :default)
@@ -2121,9 +2116,12 @@
(editor-handler/unhighlight-blocks!))
(defn- block-mouse-over
[e *control-show? block-id doc-mode?]
[uuid e *control-show? block-id doc-mode?]
(util/stop e)
(reset! *control-show? true)
(when (or
(model/block-collapsed? uuid)
(editor-handler/collapsable? uuid))
(reset! *control-show? true))
(when-let [parent (gdom/getElement block-id)]
(let [node (.querySelector parent ".bullet-container")]
(when doc-mode?
@@ -2178,26 +2176,27 @@
(rum/defcs block-container < rum/reactive
{:init (fn [state]
(let [[config block] (:rum/args state)]
(when-not (some? (state/sub-collapsed (:block/uuid block)))
(when (and (not (some? (state/sub-collapsed (:block/uuid block))))
(or (:ref? config) (:block? config)))
(state/set-collapsed-block! (:block/uuid block)
(editor-handler/block-default-collapsed? block config)))
(assoc state
::init-collapsed? (get-in block [:block/properties :collapsed])
::control-show? (atom false))))
(assoc state ::control-show? (atom false))))
:should-update (fn [old-state new-state]
(let [compare-keys [:block/uuid :block/properties
:block/parent :block/left
:block/children :block/content
(let [compare-keys [:block/uuid :block/content :block/parent :block/collapsed? :block/children
:block/properties
:block/_refs]
config-compare-keys [:show-cloze?]]
(or
(not= (select-keys (second (:rum/args old-state)) compare-keys)
(select-keys (second (:rum/args new-state)) compare-keys))
(not= (select-keys (first (:rum/args old-state)) config-compare-keys)
(select-keys (first (:rum/args new-state)) config-compare-keys)))))}
[state config {:block/keys [uuid repo children pre-block? top? properties refs heading-level level type format content] :as block}]
(let [block (merge block (block/parse-title-and-body uuid format pre-block? content))
body (:block/body block)
config-compare-keys [:show-cloze?]
b1 (second (:rum/args old-state))
b2 (second (:rum/args new-state))
result (or
(not= (select-keys b1 compare-keys)
(select-keys b2 compare-keys))
(not= (select-keys (first (:rum/args old-state)) config-compare-keys)
(select-keys (first (:rum/args new-state)) config-compare-keys)))]
(boolean result)))}
[state config {:block/keys [uuid children pre-block? top? refs heading-level level type format content] :as block}]
(let [repo (state/get-current-repo)
block (merge block (block/parse-title-and-body uuid format pre-block? content))
blocks-container-id (:blocks-container-id config)
config (update config :block merge block)
;; Each block might have multiple queries, but we store only the first query's result
@@ -2210,7 +2209,7 @@
block? (boolean (:block? config))
collapsed? (if (or ref? block?)
(state/sub-collapsed uuid)
(:collapsed properties))
(util/collapsed? block))
breadcrumb-show? (:breadcrumb-show? config)
slide? (boolean (:slide? config))
custom-query? (boolean (:custom-query? config))
@@ -2221,8 +2220,8 @@
has-child? (boolean
(and
(not pre-block?)
(or (and (coll? children) (seq children))
(seq body))))
(coll? children)
(seq children)))
attrs (on-drag-and-mouse-attrs block uuid top? block-id *move-to)
children-refs (get-children-refs children)
data-refs (build-refs-data-value children-refs)
@@ -2269,11 +2268,11 @@
[:div.flex.flex-row.pr-2
{:class (if heading? "items-baseline" "")
:on-mouse-over (fn [e]
(block-mouse-over e *control-show? block-id doc-mode?))
(block-mouse-over uuid e *control-show? block-id doc-mode?))
:on-mouse-leave (fn [e]
(block-mouse-leave e *control-show? block-id doc-mode?))}
(when (not slide?)
(block-control config block uuid block-id body children collapsed? *control-show? edit?))
(block-control config block uuid block-id children collapsed? *control-show? edit?))
(block-content-or-editor config block edit-input-id block-id heading-level edit?)]
@@ -2866,48 +2865,47 @@
(def initial-blocks-length 200)
(def step-loading-blocks 50)
(defn- flat-blocks-tree
[vec-tree]
(->> (mapcat (fn [x] (tree-seq map? :block/children x)) vec-tree)
(map #(dissoc % :block/children))))
(defn- get-segment
[_config flat-blocks idx blocks->vec-tree]
(let [new-idx (if-not (zero? idx)
(+ idx step-loading-blocks)
initial-blocks-length)
[flat-blocks idx blocks->vec-tree]
(let [new-idx (if (< idx initial-blocks-length)
initial-blocks-length
(+ idx step-loading-blocks))
max-idx (count flat-blocks)
idx (min max-idx new-idx)
blocks (util/safe-subvec flat-blocks 0 idx)]
[(blocks->vec-tree blocks)
idx]))
(rum/defcs lazy-blocks <
(rum/defcs lazy-blocks < rum/reactive
{:did-remount (fn [_old-state new-state]
;; Loading more when pressing Enter or paste
;; FIXME: what if users paste too many blocks?
;; or, a template with a lot of blocks?
(swap! (::last-idx new-state) + 100)
(let [*last-idx (::last-idx new-state)
new-idx (if (zero? *last-idx)
1
(inc @*last-idx))]
(reset! *last-idx new-idx))
new-state)}
(rum/local 0 ::last-idx)
[state config flat-blocks blocks->vec-tree]
(let [*last-idx (::last-idx state)
[segment idx] (get-segment config
flat-blocks
[segment idx] (get-segment flat-blocks
@*last-idx
blocks->vec-tree)
bottom-reached (fn []
(reset! *last-idx idx)
(reset! ignore-scroll? false))
has-more? (>= (count flat-blocks) (inc idx))]
has-more? (and (>= (count flat-blocks) (inc idx))
(not (and (:block? config)
(state/sub-collapsed (uuid (:id config))))))]
[:div#lazy-blocks
(ui/infinite-list
"main-content-container"
(block-list config segment)
{:on-load bottom-reached
:threshold 1000
:has-more has-more?
:more (if (:preview? config) "More" (ui/loading "Loading"))})]))
:more (if (or (:preview? config) (:sidebar? config))
"More"
(ui/loading "Loading"))})]))
(rum/defcs blocks-container <
{:init (fn [state]
@@ -2923,12 +2921,7 @@
doc-mode? (:document/mode? config)]
(when (seq blocks)
(let [blocks->vec-tree #(if (custom-query-or-ref? config) % (tree/blocks->vec-tree % (:id config)))
blocks-tree (blocks->vec-tree blocks)
blocks-tree (if (seq blocks-tree) blocks-tree blocks)
flat-blocks (if (custom-query-or-ref? config)
blocks-tree
(flat-blocks-tree blocks-tree))
flat-blocks (vec flat-blocks)]
flat-blocks (vec blocks)]
[:div.blocks-container.flex-1
{:class (when doc-mode? "document-mode")}
(lazy-blocks config flat-blocks blocks->vec-tree)]))))

View File

@@ -62,9 +62,13 @@
state)
(rum/defc page-blocks-inner <
{:did-mount open-first-block!
{:init (fn [state]
(when-let [block-id (last (:rum/args state))]
(state/set-collapsed-block! block-id false))
state)
:did-mount open-first-block!
:did-update open-first-block!}
[page-name _page-blocks hiccup sidebar? _preview? _block-uuid]
[page-name _blocks hiccup sidebar? _block-uuid]
[:div.page-blocks-inner {:style {:margin-left (if sidebar? 0 -20)}}
(rum/with-key
(content/content page-name
@@ -111,7 +115,7 @@
(rum/defc page-blocks-cp < rum/reactive
db-mixins/query
[repo page-e {:keys [sidebar? preview?] :as config}]
[repo page-e {:keys [sidebar?] :as config}]
(when page-e
(let [page-name (or (:block/name page-e)
(str (:block/uuid page-e)))
@@ -131,7 +135,7 @@
hiccup-config (common-handler/config-with-document-mode hiccup-config)
hiccup (block/->hiccup page-blocks hiccup-config {})]
[:div
(page-blocks-inner page-name page-blocks hiccup sidebar? preview? block-id)
(page-blocks-inner page-name page-blocks hiccup sidebar? block-id)
(when-not config/publishing?
(let [args (if block-id
{:block-uuid block-id}

View File

@@ -15,6 +15,7 @@
[frontend.handler.route :as route-handler]
[frontend.handler.ui :as ui-handler]
[frontend.handler.web.nfs :as nfs-handler]
[frontend.handler.notification :as notification]
[frontend.modules.shortcut.core :as shortcut]
[frontend.state :as state]
[frontend.ui :as ui]
@@ -22,6 +23,7 @@
[frontend.fs :as fs]
[frontend.version :as version]
[reitit.frontend.easy :as rfe]
[frontend.modules.outliner.file :as outliner-file]
[rum.core :as rum]
[frontend.mobile.util :as mobile-util]
[frontend.text :as text]
@@ -39,6 +41,14 @@
(when-let [dir-name (config/get-repo-dir url)]
(fs/watch-dir! dir-name)))
(defn- switch-repo-if-writes-finished?
[url]
(if (outliner-file/writes-finished?)
(open-repo-url url)
(notification/show!
"Please wait seconds until all changes are saved for the current graph."
:warning)))
(rum/defc add-repo
[args]
(if-let [graph-types (get-in args [:query-params :graph-types])]
@@ -82,7 +92,7 @@
(let [local-dir (config/get-local-dir url)
graph-name (text/get-graph-name-from-path local-dir)]
[:a {:title local-dir
:on-click #(open-repo-url url)}
:on-click #(switch-repo-if-writes-finished? url)}
graph-name])
[:a {:target "_blank"
:href url}
@@ -226,7 +236,7 @@
{:title short-repo-name
:hover-detail repo-path ;; show full path on hover
:options {:class "ml-1"
:on-click #(open-repo-url url)}}))
:on-click #(switch-repo-if-writes-finished? url)}}))
switch-repos)
links (->>
(concat repo-links

View File

@@ -10,6 +10,7 @@
[frontend.db.react]
[frontend.db.utils]
[frontend.db.persist :as db-persist]
[frontend.db.migrate :as db-migrate]
[frontend.namespaces :refer [import-vars]]
[frontend.state :as state]
[frontend.util :as util]
@@ -29,7 +30,7 @@
[frontend.db.utils
date->int db->json db->edn-str db->string get-max-tx-id get-tx-id
group-by-page seq-flatten sort-by-pos
string->db with-repo
string->db
entity pull pull-many transact! get-key-value]
@@ -48,7 +49,7 @@
get-page-blocks-count get-page-blocks-no-cache get-page-file get-page-format get-page-properties
get-page-referenced-blocks get-page-referenced-pages get-page-unlinked-references get-page-referenced-blocks-no-cache
get-all-pages get-pages get-pages-relation get-pages-that-mentioned-page get-public-pages get-tag-pages
journal-page? local-native-fs? mark-repo-as-cloned! page-alias-set page-blocks-transform pull-block
journal-page? mark-repo-as-cloned! page-alias-set pull-block
set-file-last-modified-at! transact-files-db! page-empty? page-empty-or-dummy? get-alias-source-page
set-file-content! has-children? get-namespace-pages get-all-namespace-relation get-pages-by-name-partition]
@@ -66,6 +67,21 @@
[frontend.db.default built-in-pages-names built-in-pages])
(defn get-schema-version [db]
(d/q
'[:find ?v .
:where
[_ :schema/version ?v]]
db))
(defn old-schema?
[db]
(let [v (get-schema-version db)]
(if (integer? v)
(> db-schema/version v)
;; backward compatibility
true)))
;; persisting DBs between page reloads
(defn persist! [repo]
(let [key (datascript-db repo)
@@ -137,6 +153,8 @@
(assoc option
:listen-handler listen-and-persist!))))
;; TODO: only restore the current graph instead of all the graphs to speedup and
;; reduce memory usage.
(defn restore!
[{:keys [repos] :as me} _old-db-schema restore-config-handler]
(let [logged? (:name me)]
@@ -145,17 +163,20 @@
(let [repo url]
(p/let [db-name (datascript-db repo)
db-conn (d/create-conn db-schema/schema)
_ (d/transact! db-conn [{:schema/version db-schema/version}])
_ (swap! conns assoc db-name db-conn)
stored (db-persist/get-serialized-graph db-name)
_ (if stored
(let [stored-db (string->db stored)
attached-db (d/db-with stored-db (concat
[(me-tx stored-db me)]
default-db/built-in-pages))]
(conn/reset-conn! db-conn attached-db))
default-db/built-in-pages))
db (if (old-schema? attached-db)
(db-migrate/migrate attached-db)
attached-db)]
(conn/reset-conn! db-conn db))
(when logged?
(d/transact! db-conn [(me-tx (d/db db-conn) me)])))]
(d/transact! db-conn [{:schema/version db-schema/version}])
(restore-config-handler repo)
(listen-and-persist! repo)))))))

View File

@@ -1,28 +1,21 @@
(ns frontend.db.migrate
(:require [datascript.core :as d]
[frontend.db-schema :as db-schema]
[frontend.state :as state]))
(:require [datascript.core :as d]))
(defn- migrate-attribute
[f]
(if (and (keyword? f) (= "page" (namespace f)))
(let [k (keyword "block" (name f))]
(case k
:block/ref-pages
:block/refs
k))
f))
(defn with-schema [db new-schema]
(let [datoms (->> (d/datoms db :eavt)
(map (fn [d]
(let [a (migrate-attribute (:a d))]
(d/datom (:e d) a (:v d) (:tx d) (:added d))))))]
(-> (d/empty-db new-schema)
(with-meta (meta db))
(d/db-with datoms))))
(defn get-collapsed-blocks
[db]
(d/q
'[:find [?b ...]
:where
[?b :block/properties ?properties]
[(get ?properties :collapsed) ?collapsed]
[(= true ?collapsed)]]
db))
(defn migrate
[repo db]
(state/pub-event! [:graph/migrated repo])
(with-schema db db-schema/schema))
[db]
(when db
(let [collapsed-blocks (get-collapsed-blocks db)]
(when (seq collapsed-blocks)
(let [tx-data (map (fn [id] {:db/id id
:block/collapsed? true}) collapsed-blocks)]
(d/db-with db tx-data))))))

View File

@@ -26,8 +26,9 @@
(def block-attrs
'[:db/id
:block/uuid
:block/type
:block/parent
:block/left
:block/collapsed?
:block/format
:block/refs
:block/_refs
@@ -44,7 +45,6 @@
:block/created-at
:block/updated-at
:block/file
:block/parent
:block/heading-level
{:block/page [:db/id :block/name :block/original-name :block/journal-day]}
{:block/_parent ...}])
@@ -374,10 +374,6 @@
(map (fn [m]
(or (:block/original-name m) (:block/name m)))))))))
(defn page-blocks-transform
[repo-url result]
(db-utils/with-repo repo-url result))
(defn with-pages
[blocks]
(let [pages-ids (->> (map (comp :db/id :block/page) blocks)
@@ -400,8 +396,8 @@
;; FIXME: alert
(defn sort-by-left
([blocks parent]
(sort-by-left blocks parent true))
([blocks parent check?]
(sort-by-left blocks parent {:check? true}))
([blocks parent {:keys [check?]}]
(when check?
(when (not= (count blocks) (count (set (map :block/left blocks))))
(let [duplicates (->> (map (comp :db/id :block/left) blocks)
@@ -437,24 +433,45 @@
f))
form))
(defn flatten-blocks-sort-by-left
[blocks parent]
(let [ids->blocks (zipmap (map (fn [b] [(:db/id (:block/parent b))
(:db/id (:block/left b))]) blocks) blocks)]
;; TODO: both zipmap and map lookup are slow in cljs
;; zipmap 20k blocks takes 30ms on my M1 Air.
(defn sort-blocks
[blocks parent limit]
(let [ids->blocks (zipmap (map
(fn [b]
[(:db/id (:block/parent b))
(:db/id (:block/left b))])
blocks)
blocks)]
(loop [node parent
next-siblings '()
result []]
(let [id (:db/id node)
child-block (get ids->blocks [id id])
next-sibling (get ids->blocks [(:db/id (:block/parent node)) id])
next-siblings (if (and next-sibling child-block)
(cons next-sibling next-siblings)
next-siblings)]
(if-let [node (or child-block next-sibling)]
(recur node next-siblings (conj result node))
(if-let [sibling (first next-siblings)]
(recur sibling (rest next-siblings) (conj result sibling))
result))))))
(if (or (nil? node) (and limit (= (count result) limit)))
result
(let [id (:db/id node)
child-block (get ids->blocks [id id])
next-sibling (get ids->blocks [(:db/id (:block/parent node)) id])
next-siblings (if (and next-sibling child-block)
(cons next-sibling next-siblings)
next-siblings)]
(if-let [node (and
(not (:block/collapsed? node))
(or child-block next-sibling))]
(recur node next-siblings (conj result node))
(if-let [sibling (first next-siblings)]
(recur sibling (rest next-siblings) (conj result sibling))
result)))))))
(comment
(let [page "Scripture (NASB 1995)"
page-entity (db-utils/pull [:block/name (string/lower-case page)])
blocks (->> (get-page-blocks (state/get-current-repo) (string/lower-case page) {:use-cache? false})
(map (fn [b] (assoc b :block/content (:block/content (db-utils/entity (:db/id b)))))))]
(def page-entity page-entity)
(def blocks blocks)
(time (prn (count (sort-blocks blocks page-entity 1)))))
)
(defn get-block-refs-count
[block-id]
@@ -467,32 +484,65 @@
nil)
react))))
;; FIXME: merge get-page-blocks and get-block-and-children to simplify the logic
;; TODO: native sort and limit support in DB
(defn- get-limited-blocks
[db page block-eids limit]
(let [lefts (d/datoms db :avet :block/left)
lefts (zipmap (map :e lefts) lefts)
collapsed (d/datoms db :avet :block/collapsed?)
collapsed (zipmap (map :e collapsed) collapsed)
parents (d/datoms db :avet :block/parent)
parents (zipmap (map :e parents) parents)
blocks (map (fn [id]
(let [collapsed? (:v (get collapsed id))]
(cond->
{:db/id id
:block/left {:db/id (:v (get lefts id))}
:block/parent {:db/id (:v (get parents id))}}
collapsed?
(assoc :block/collapsed? true))))
block-eids)
blocks (sort-blocks blocks page limit)]
(map :db/id blocks)))
;; Use datoms index and provide limit support
(defn get-page-blocks
([page]
(get-page-blocks (state/get-current-repo) page nil))
([repo-url page]
(get-page-blocks repo-url page nil))
([repo-url page {:keys [use-cache? pull-keys]
([repo-url page {:keys [use-cache? pull-keys limit]
:or {use-cache? true
pull-keys '[*]}}]
(when page
(let [page (util/page-name-sanity-lc (string/trim page))
page-entity (or (db-utils/entity repo-url [:block/name page])
(db-utils/entity repo-url [:block/original-name page]))
page-id (:db/id page-entity)]
(let [page-entity (if (integer? page)
(db-utils/entity repo-url page)
(let [page (util/page-name-sanity-lc (string/trim page))]
(db-utils/entity repo-url [:block/name page])))
page-id (:db/id page-entity)
db (conn/get-conn repo-url)
bare-page-map {:db/id page-id
:block/name (:block/name page-entity)
:block/original-name (:block/original-name page-entity)
:block/journal-day (:block/journal-day page-entity)}]
(when page-id
(some->
(react/q repo-url [:page/blocks page-id]
{:use-cache? use-cache?
:transform-fn #(page-blocks-transform repo-url %)
:query-fn (fn [db]
(let [datoms (d/datoms db :avet :block/page page-id)
block-eids (mapv :e datoms)]
(db-utils/pull-many repo-url pull-keys block-eids)))}
block-eids (mapv :e datoms)
;; TODO: needs benchmark
long-page? (> (count datoms) 1000)
block-eids (if long-page?
(get-limited-blocks db page-entity block-eids limit)
block-eids)
blocks (db-utils/pull-many repo-url pull-keys block-eids)
blocks (if long-page? blocks
(sort-blocks blocks page-entity nil))]
(map (fn [b] (assoc b :block/page bare-page-map)) blocks)))}
nil)
react
(flatten-blocks-sort-by-left page-entity)))))))
react))))))
(defn get-page-blocks-no-cache
([page]
@@ -503,14 +553,12 @@
:or {pull-keys '[*]}}]
(when page
(let [page (util/page-name-sanity-lc page)
page-id (or (:db/id (db-utils/entity repo-url [:block/name page]))
(:db/id (db-utils/entity repo-url [:block/original-name page])))
page-id (:db/id (db-utils/entity repo-url [:block/name page]))
db (conn/get-conn repo-url)]
(when page-id
(let [datoms (d/datoms db :avet :block/page page-id)
block-eids (mapv :e datoms)]
(some->> (db-utils/pull-many repo-url pull-keys block-eids)
(page-blocks-transform repo-url))))))))
(db-utils/pull-many repo-url pull-keys block-eids)))))))
(defn get-page-blocks-count
[repo page-id]
@@ -572,15 +620,14 @@
[repo block-id]
(when-let [block (:block/parent (get-block-parents-v2 repo block-id))]
(->> (tree-seq map? (fn [x] [(:block/parent x)]) block)
(map (comp :collapsed :block/properties))
(some true?))))
(some util/collapsed?))))
(defn block-collapsed?
([block-id]
(block-collapsed? (state/get-current-repo) block-id))
([repo block-id]
(when-let [block (db-utils/entity repo [:block/uuid block-id])]
(get-in block [:block/properties :collapsed]))))
(util/collapsed? block))))
(defn get-block-page
[repo block-id]
@@ -603,10 +650,8 @@
ids))))))
(defn block-and-children-transform
[result repo-url _block-uuid]
(some->> result
db-utils/seq-flatten
(db-utils/with-repo repo-url)))
[result _repo-url _block-uuid]
(db-utils/seq-flatten result))
(defn get-block-children-ids
[repo block-uuid]
@@ -638,8 +683,8 @@
(sort-by-left (db-utils/entity [:block/uuid block-uuid])))))
(defn get-blocks-by-page
[id-or-lookup-ref]
(when-let [conn (conn/get-conn)]
[repo id-or-lookup-ref]
(when-let [conn (conn/get-conn repo)]
(->
(d/q
'[:find (pull ?block [*])
@@ -1182,9 +1227,10 @@
[repo]
(db-utils/get-key-value repo :db/type))
(defn local-native-fs?
[repo]
(= :local-native-fs (get-db-type repo)))
(defn db-graph?
"Is current graph a database graph instead of a graph on plain-text files?"
[]
(= :database (get-db-type (state/get-current-repo))))
(defn get-public-pages
[db]
@@ -1358,12 +1404,12 @@
(defn delete-page-blocks
[repo-url page]
(when page
(let [db (conn/get-conn repo-url)
page (db-utils/pull [:block/name (util/page-name-sanity-lc page)])]
(when page
(let [datoms (d/datoms db :avet :block/page (:db/id page))
block-eids (mapv :e datoms)]
(mapv (fn [eid] [:db.fn/retractEntity eid]) block-eids))))))
(when-let [db (conn/get-conn repo-url)]
(let [page (db-utils/pull [:block/name (util/page-name-sanity-lc page)])]
(when page
(let [datoms (d/datoms db :avet :block/page (:db/id page))
block-eids (mapv :e datoms)]
(mapv (fn [eid] [:db.fn/retractEntity eid]) block-eids)))))))
(defn delete-file-pages!
[repo-url path]

View File

@@ -59,8 +59,7 @@
(defn custom-query-result-transform
[query-result remove-blocks q]
(try
(let [repo (state/get-current-repo)
result (db-utils/seq-flatten query-result)
(let [result (db-utils/seq-flatten query-result)
block? (:block/uuid (first result))
result (if block?
(let [result (if (seq remove-blocks)
@@ -72,7 +71,6 @@
(some->> result
remove-nested-children-blocks
(model/sort-by-left-recursive)
(db-utils/with-repo repo)
(model/with-pages)))
result)]
(if-let [result-transform (:result-transform q)]

View File

@@ -54,12 +54,6 @@
(util/parse-int
(string/replace (date/ymd date) "/" "")))
(defn with-repo
[repo blocks]
(map (fn [block]
(assoc block :block/repo repo))
blocks))
(defn entity
([id-or-lookup-ref]
(entity (state/get-current-repo) id-or-lookup-ref))

View File

@@ -1,7 +1,7 @@
(ns frontend.db-schema)
(defonce version "0.0.2")
(defonce ast-version "0.0.1")
(defonce version 1)
(defonce ast-version 1)
;; A page is a special block, a page can corresponds to multiple files with the same ":block/name".
(def schema
{:schema/version {}
@@ -24,8 +24,10 @@
:block/type {}
:block/uuid {:db/unique :db.unique/identity}
:block/parent {:db/valueType :db.type/ref}
:block/parent {:db/valueType :db.type/ref
:db/index true}
:block/left {:db/valueType :db.type/ref}
:block/collapsed? {:db/index true}
;; :markdown, :org
:block/format {}

View File

@@ -20,7 +20,8 @@
[cljs-time.coerce :as tc]
[clojure.string :as string]
[rum.core :as rum]
[frontend.modules.shortcut.core :as shortcut]))
[frontend.modules.shortcut.core :as shortcut]
[medley.core :as medley]))
;;; ================================================================
;;; Commentary
@@ -200,7 +201,10 @@
(defn- clear-collapsed-property
"Clear block's collapsed property if exists"
[blocks]
(let [result (map (fn [block] (assoc-in block [:block/properties :collapsed] false)) blocks)]
(let [result (map (fn [block]
(-> block
(dissoc :block/collapsed?)
(medley/dissoc-in [:block/properties :collapsed]))) blocks)]
result))
;;; ================================================================

View File

@@ -603,6 +603,9 @@
(seq (:properties-order properties))
(assoc :properties-order (:properties-order properties)))
block (if (get-in block [:properties :collapsed])
(assoc block :collapsed? true)
block)
block (-> block
(assoc-in [:meta :start-pos] start_pos)
(assoc-in [:meta :end-pos] last-pos)

View File

@@ -130,10 +130,7 @@
(js/console.error "Failed to request GitHub app tokens."))))
(watch-for-date!)
(file-handler/watch-for-current-graph-dir!)
;; (when-not (state/logged?)
;; (state/pub-event! [:after-db-restore repos]))
))
(file-handler/watch-for-current-graph-dir!)))
(p/catch (fn [error]
(log/error :db/restore-failed error))))))
interval-id (js/setInterval inner-fn 50)]

View File

@@ -481,7 +481,7 @@
(boolean? sibling?)
sibling?
(:collapsed (:block/properties current-block))
(util/collapsed? current-block)
true
:else
@@ -1189,7 +1189,7 @@
[repo block-ids]
(let [blocks (db-utils/pull-many repo '[*] (mapv (fn [id] [:block/uuid id]) block-ids))
blocks* (flatten
(mapv (fn [b] (if (:collapsed (:block/properties b))
(mapv (fn [b] (if (util/collapsed? b)
(vec (tree/sort-blocks (db/get-block-children repo (:block/uuid b)) b))
[b])) blocks))
block-ids* (mapv :block/uuid blocks*)
@@ -1280,7 +1280,7 @@
;; filter out blocks not belong to page with 'page-id'
(remove (fn [block] (some-> (:db/id (:block/page block)) (not= page-id))))
;; expand collapsed blocks
(mapv (fn [b] (if (:collapsed (:block/properties b))
(mapv (fn [b] (if (util/collapsed? b)
(vec (tree/sort-blocks (db/get-block-children repo (:block/uuid b)) b))
[b])))
(flatten))
@@ -2065,8 +2065,7 @@
(defn edit-box-on-change!
[e block id]
(let [value (util/evalue e)
repo (or (:block/repo block)
(state/get-current-repo))]
repo (state/get-current-repo)]
(state/set-edit-content! id value false)
(when @*auto-save-timeout
(js/clearTimeout @*auto-save-timeout))
@@ -2157,7 +2156,7 @@
block-self? (block-self-alone-when-insert? config block-id)
has-children? (db/has-children? (state/get-current-repo)
(:block/uuid editing-block))
collapsed? (:collapsed (:block/properties editing-block))]
collapsed? (util/collapsed? editing-block)]
(conj (match (mapv boolean [(seq fst-block-text) (seq snd-block-text)
block-self? has-children? (= parent left) collapsed?])
;; when zoom at editing-block
@@ -2779,7 +2778,7 @@
repo (state/get-current-repo)
right (outliner-core/get-right-node (outliner-core/block current-block))
current-block-has-children? (db/has-children? repo (:block/uuid current-block))
collapsed? (:collapsed (:block/properties current-block))
collapsed? (util/collapsed? current-block)
first-child (:data (tree/-get-down (outliner-core/block current-block)))
next-block (if (or collapsed? (not current-block-has-children?))
(:data right)
@@ -3451,11 +3450,9 @@
(defn collapsable? [block-id]
(when block-id
(if-let [block (db-model/query-block-by-uuid block-id)]
(let [block (block/parse-title-and-body block)]
(and
(nil? (-> block :block/properties :collapsed))
(or (not-empty (:block/body block))
(db-model/has-children? block-id))))
(and
(not (util/collapsed? block))
(db-model/has-children? block-id))
false)))
(defn all-blocks-with-level
@@ -3495,7 +3492,7 @@
collapse?
(w/postwalk
(fn [b]
(if (and (map? b) (-> b :block/properties :collapsed))
(if (and (map? b) (util/collapsed? b))
(assoc b :block/children []) b)))
true
@@ -3513,15 +3510,44 @@
(let [config (:config (state/get-editor-args))]
(or (:ref? config) (:block? config))))
(defn- set-blocks-collapsed!
[block-ids value]
(let [block-ids (map (fn [block-id] (if (string? block-id) (uuid block-id) block-id)) block-ids)
repo (state/get-current-repo)
value (boolean value)]
(when repo
(ds/auto-transact!
[txs-state (ds/new-outliner-txs-state)]
{:outliner-op :collapse-expand-blocks
:skip-transact? false}
(doseq [block-id block-ids]
(when-let [block (db/entity [:block/uuid block-id])]
(let [current-value (boolean (util/collapsed? block))]
(when-not (= current-value value)
(let [block (outliner-core/block {:block/uuid block-id
:block/collapsed? value})]
(outliner-core/save-node block {:txs-state txs-state})))))))
(let [block-id (first block-ids)
input-pos (or (state/get-edit-pos) :max)]
(db/refresh! (state/get-current-repo)
{:key :block/change
:data [(db/pull [:block/uuid block-id])]})
;; update editing input content
(when-let [editing-block (state/get-edit-block)]
(when (= (:block/uuid editing-block) block-id)
(edit-block! editing-block
input-pos
(state/get-edit-input-id))))))))
(defn collapse-block! [block-id]
(when (collapsable? block-id)
(when-not (skip-collapsing-in-db?)
(set-block-property! block-id :collapsed true)))
(set-blocks-collapsed! [block-id] true)))
(state/set-collapsed-block! block-id true))
(defn expand-block! [block-id]
(when-not (skip-collapsing-in-db?)
(remove-block-property! block-id :collapsed))
(set-blocks-collapsed! [block-id] false))
(state/set-collapsed-block! block-id false))
(defn expand!
@@ -3552,8 +3578,7 @@
nil
(let [blocks-to-expand (->> blocks-with-level
(filter (fn [b] (= (:block/level b) level)))
(filter (fn [{:block/keys [properties]}]
(contains? properties :collapsed))))]
(filter util/collapsed?))]
(if (empty? blocks-to-expand)
(recur (inc level))
(doseq [{:block/keys [uuid]} blocks-to-expand]
@@ -3599,17 +3624,15 @@
([]
(collapse-all! nil))
([block-id]
(let [blocks-to-collapse (all-blocks-with-level {:expanded? true :root-block block-id})]
(doseq [{:block/keys [uuid]} blocks-to-collapse]
(collapse-block! uuid)))))
(let [blocks (all-blocks-with-level {:expanded? true :root-block block-id})]
(set-blocks-collapsed! (map :block/uuid blocks) true))))
(defn expand-all!
([]
(expand-all! nil))
([block-id]
(->> (all-blocks-with-level {:root-block block-id})
(map (comp expand-block! :block/uuid))
dorun)))
(let [blocks (all-blocks-with-level {:root-block block-id})]
(set-blocks-collapsed! (map :block/uuid blocks) false))))
(defn- get-block-with-its-children
[block-uuid]
@@ -3621,7 +3644,7 @@
(defn expand-all?
[block-uuid]
(let [blocks (get-block-with-its-children block-uuid)]
(some #(get-in % [:block/properties :collapsed]) blocks)))
(some util/collapsed? blocks)))
(defn collapse-all?
[block-uuid]
@@ -3727,7 +3750,7 @@
false
(and (:block? config)
(get-in block [:block/properties :collapsed]))
(util/collapsed? block))
true
:else
@@ -3740,4 +3763,4 @@
(state/get-ref-open-blocks-level)))))))]
(if (or (:ref? config) (:block? config))
collapsed?
(get-in block [:block/properties :collapsed]))))
(util/collapsed? block))))

View File

@@ -7,37 +7,21 @@
[frontend.db.utils :as db-utils]
[frontend.state :as state]
[frontend.util :as util]
[frontend.debug :as debug]
[frontend.format.block :as block]))
[frontend.debug :as debug]))
(defn- indented-block-content
[content spaces-tabs]
(let [lines (string/split-lines content)]
(string/join (str "\n" spaces-tabs) lines)))
(defn- allowed-block-as-title?
"Allowed to be in the first line of a block (a.k.a block title)"
[title body properties]
(and (not (seq title))
(or
(seq properties)
(contains?
#{"Quote" "Table" "Drawer" "Property_Drawer" "Footnote_Definition" "Custom" "Export" "Src" "Example" "Horizontal_Rule"}
(ffirst body)))))
(defn transform-content
[{:block/keys [uuid format properties pre-block? unordered content heading-level left page parent]}
level
{:keys [heading-to-list?]}]
(let [{:block/keys [title body]} (block/parse-title-and-body uuid format pre-block? content)
content (or content "")
heading-with-title? (seq title)
allowed-block-as-title? (allowed-block-as-title? title body properties)
[{:block/keys [format pre-block? unordered content heading-level left page parent]} level {:keys [heading-to-list?]}]
(let [content (or content "")
first-block? (= left page)
pre-block? (and first-block? pre-block?)
markdown? (= format :markdown)
content (cond
(and first-block? pre-block?)
pre-block?
(let [content (string/trim content)]
(str content "\n"))
@@ -71,18 +55,10 @@
(string/replace #"^\s?#+\s?$" ""))
content)
new-content (indented-block-content (string/trim content) spaces-tabs)
sep (cond
markdown-top-heading?
sep (if (or markdown-top-heading?
(string/blank? new-content))
""
(or heading-with-title? allowed-block-as-title?)
" "
(string/blank? new-content)
""
:else
(str "\n" spaces-tabs))]
" ")]
(str prefix sep new-content)))]
content))
@@ -92,16 +68,19 @@
(loop [block-contents []
[f & r] tree
level init-level]
(if (nil? f)
(string/join "\n" block-contents)
(let [page? (nil? (:block/page f))
content (if page? nil (transform-content f level opts))
new-content
(->> (if-let [children (seq (:block/children f))]
[content (tree->file-content children {:init-level (inc level)})]
[content])
(remove nil?))]
(recur (into block-contents new-content) r level)))))
(let [f (if (:block/collapsed? f)
(assoc-in f [:block/properties :collapsed] true)
f)]
(if (nil? f)
(string/join "\n" block-contents)
(let [page? (nil? (:block/page f))
content (if page? nil (transform-content f level opts))
new-content
(->> (if-let [children (seq (:block/children f))]
[content (tree->file-content children {:init-level (inc level)})]
[content])
(remove nil?))]
(recur (into block-contents new-content) r level))))))
(def init-level 1)

View File

@@ -229,13 +229,8 @@
sorted-children)))))))))
(defn set-block-collapsed! [txs-state id collapsed?]
(let [e (db/entity id)
properties (:block/properties e)
properties (if collapsed?
(assoc properties :collapsed true)
(dissoc properties :collapsed))]
(swap! txs-state concat [{:db/id id
:block/properties properties}])))
(swap! txs-state concat [{:db/id id
:block/collapsed? collapsed?}]))
(defn save-node
([node]

View File

@@ -9,39 +9,47 @@
[frontend.modules.outliner.tree :as tree]
[frontend.util :as util]
[goog.object :as gobj]
[lambdaisland.glogi :as log]))
[lambdaisland.glogi :as log]
[frontend.state :as state]))
(def write-chan (async/chan))
(defonce write-chan (async/chan 100))
(defonce write-chan-batch-buf (atom []))
(def batch-write-interval 1000)
;; FIXME name conflicts between multiple graphs
(defn writes-finished?
[]
(empty? @write-chan-batch-buf))
(defn do-write-file!
[page-db-id]
(let [page-block (db/pull page-db-id)
[repo page-db-id]
(let [page-block (db/pull repo '[*] page-db-id)
page-db-id (:db/id page-block)
blocks (model/get-blocks-by-page page-db-id)]
blocks (model/get-blocks-by-page repo page-db-id)]
(when-not (and (= 1 (count blocks))
(string/blank? (:block/content (first blocks)))
(nil? (:block/file page-block)))
(let [tree (tree/blocks->vec-tree blocks (:block/name page-block))]
(let [tree (tree/blocks->vec-tree repo blocks (:block/name page-block))]
(if page-block
(file/save-tree page-block tree)
(js/console.error (str "can't find page id: " page-db-id)))))))
(defn write-files!
[page-db-ids]
(when (seq page-db-ids)
[pages]
(when (seq pages)
(when-not config/publishing?
(doseq [page-db-id (set page-db-ids)]
(try (do-write-file! page-db-id)
(catch js/Error e
(notification/show!
[:div
[:p "Write file failed, please copy the changes to other editors in case of losing data."]
"Error: " (str (gobj/get e "stack"))]
:error)
(log/error :file/write-file-error {:error e})))))))
(if (state/input-idle? (state/get-current-repo))
(doseq [[repo page-id] (set pages)]
(try (do-write-file! repo page-id)
(catch js/Error e
(notification/show!
[:div
[:p "Write file failed, please copy the changes to other editors in case of losing data."]
"Error: " (str (gobj/get e "stack"))]
:error)
(log/error :file/write-file-error {:error e}))))
(doseq [page pages]
(async/put! write-chan page))))))
(defn sync-to-file
[{page-db-id :db/id}]
@@ -49,8 +57,10 @@
(notification/show!
"Write file failed, can't find the current page!"
:error)
(async/put! write-chan page-db-id)))
(when-let [repo (state/get-current-repo)]
(async/put! write-chan [repo page-db-id]))))
(util/batch write-chan
batch-write-interval
write-files!)
write-files!
write-chan-batch-buf)

View File

@@ -1,6 +1,8 @@
(ns frontend.modules.outliner.tree
(:require [frontend.db :as db]
[frontend.util :as util]))
[frontend.util :as util]
[clojure.string :as string]
[frontend.state :as state]))
(defprotocol INode
(-get-id [this])
@@ -41,25 +43,27 @@
(block-children root 1)))
(defn- get-root-and-page
[page-name-or-block-id]
(if (string? page-name-or-block-id)
(if (util/uuid-string? page-name-or-block-id)
[false (db/entity [:block/uuid (uuid page-name-or-block-id)])]
[true (db/entity [:block/name (util/page-name-sanity-lc page-name-or-block-id)])])
[false page-name-or-block-id]))
[repo root-id]
(if (string? root-id)
(if (util/uuid-string? root-id)
[false (db/entity repo [:block/uuid (uuid root-id)])]
[true (db/entity repo [:block/name (string/lower-case root-id)])])
[false root-id]))
(defn blocks->vec-tree
[blocks page-name-or-block-id]
(let [[page? root] (get-root-and-page (str page-name-or-block-id))]
(if-not root ; custom query
blocks
(let [result (blocks->vec-tree-aux blocks root)]
(if page?
result
;; include root block
(let [root-block (some #(when (= (:db/id %) (:db/id root)) %) blocks)
root-block (assoc root-block :block/children result)]
[root-block]))))))
([blocks root-id]
(blocks->vec-tree (state/get-current-repo) blocks root-id))
([repo blocks root-id]
(let [[page? root] (get-root-and-page repo (str root-id))]
(if-not root ; custom query
blocks
(let [result (blocks->vec-tree-aux blocks root)]
(if page?
result
;; include root block
(let [root-block (some #(when (= (:db/id %) (:db/id root)) %) blocks)
root-block (assoc root-block :block/children result)]
[root-block])))))))
(defn- sort-blocks-aux
[parents parent-groups]

View File

@@ -15,8 +15,7 @@
[lambdaisland.glogi :as log]
[promesa.core :as p]
[rum.core :as rum]
[frontend.mobile.util :as mobile-util]
[cljs.cache :as cache]))
[frontend.mobile.util :as mobile-util]))
(defonce state
(let [document-mode? (or (storage/get :document/mode?) false)
@@ -207,33 +206,24 @@
:srs/mode? false
:srs/cards-due-count nil})))
:srs/cards-due-count nil
})))
;; block uuid -> {content(String) -> ast}
(def blocks-ast-cache (atom (cache/lru-cache-factory {} :threshold 5000)))
(def blocks-ast-cache (atom {}))
(defn add-block-ast-cache!
[block-uuid content ast]
(when (and block-uuid content ast)
(let [k block-uuid
add-cache! (fn []
(reset! blocks-ast-cache (cache/evict @blocks-ast-cache block-uuid))
(reset! blocks-ast-cache (cache/miss @blocks-ast-cache k {content ast})))]
(if (cache/has? @blocks-ast-cache k)
(let [m (cache/lookup @blocks-ast-cache k)]
(if (and (map? m) (get m content))
(reset! blocks-ast-cache (cache/hit @blocks-ast-cache k))
(add-cache!)))
(add-cache!)))))
(let [new-value (assoc-in @blocks-ast-cache [block-uuid content] ast)
new-value (if (> (count new-value) 10000)
(into {} (take 5000 new-value))
new-value)]
(reset! blocks-ast-cache new-value))))
(defn get-block-ast
[block-uuid content]
(when (and block-uuid content)
(let [k block-uuid]
(when (cache/has? @blocks-ast-cache k)
(let [m (cache/lookup @blocks-ast-cache k)]
(when-let [result (and (map? m) (get m content))]
(reset! blocks-ast-cache (cache/hit @blocks-ast-cache k))
result))))))
(get-in @blocks-ast-cache [block-uuid content])))
(defn sub
[ks]

View File

@@ -1383,17 +1383,19 @@
(defn keyname [key] (str (namespace key) "/" (name key)))
(defn batch [in max-time handler]
(async/go-loop [buf [] t (async/timeout max-time)]
(defn batch [in max-time handler buf-atom]
(async/go-loop [buf buf-atom t (async/timeout max-time)]
(let [[v p] (async/alts! [in t])]
(cond
(or (= p t) (nil? v))
(let [timeout (async/timeout max-time)]
(handler buf)
(recur [] timeout))
(handler @buf)
(reset! buf [])
(recur buf timeout))
:else
(recur (conj buf v) t)))))
(do (swap! buf conj v)
(recur buf t))))))
#?(:cljs
(defn trace!
@@ -1638,3 +1640,9 @@
(if (and (not route?) (electron?))
(js/window.apis.openExternal url)
(set! (.-href js/window.location) url)))))
(defn collapsed?
[block]
(or (:block/collapsed? block)
;; for backward compatiblity
(get-in block [:properties :collapsed])))

View File

@@ -533,7 +533,7 @@
(when-let [block (db-model/get-block-by-uuid uuid)]
(let [{:keys [flag]} (bean/->clj opts)
flag (if (= "toggle" flag)
(not (-> block :block/properties :collapsed))
(not (util/collapsed? block))
(boolean flag))]
(if flag (editor-handler/collapse-block! uuid)
(editor-handler/expand-block! uuid))))))