refactor: remove block/path-refs (#12081)

1. refactor: use :block/refs and has-ref rule instead of path-refs
2. remove block/path-refs
3. removes :block-parent rule since there's already :parent
4. enhance: skip pipeline calculation for rtc initial download tx
5. refactor(rtc): remove memoize

---------

Co-authored-by: rcmerci <rcmerci@gmail.com>
This commit is contained in:
Tienson Qin
2025-09-03 20:33:33 +08:00
committed by GitHub
parent ebfcf3c5f4
commit 248d8c33f2
33 changed files with 222 additions and 399 deletions

View File

@@ -39,8 +39,7 @@
tx (cond->
(mapcat
(fn [block]
[[:db/retract (:db/id ref) :block/refs (:db/id block)]
[:db/retract (:db/id ref) :block/path-refs (:db/id block)]]) retracted-blocks)
[[:db/retract (:db/id ref) :block/refs (:db/id block)]]) retracted-blocks)
replaced-title
(conj [:db/add id :block/title replaced-title]))]
tx))

View File

@@ -33,7 +33,7 @@
it means `(db/entity :block/title)` always return same result"
#{:block/link :block/updated-at :block/refs :block/closed-value-property
:block/created-at :block/collapsed? :block/tags :block/title
:block/path-refs :block/parent :block/order :block/page
:block/parent :block/order :block/page
:logseq.property/created-from-property
:logseq.property/icon

View File

@@ -141,8 +141,7 @@
identity
(fn [e]
(keep (fn [[k v]]
(when (and (not (contains? #{:block/path-refs} k))
(or (empty? properties) (properties k)))
(when (or (empty? properties) (properties k))
(let [v' (cond
(= k :block/parent)
(:db/id v)

View File

@@ -8,7 +8,8 @@
[logseq.db :as ldb]
[logseq.db.common.entity-plus :as entity-plus]
[logseq.db.common.initial-data :as common-initial-data]
[logseq.db.frontend.class :as db-class]))
[logseq.db.frontend.class :as db-class]
[logseq.db.frontend.rules :as rules]))
(defn get-filters
[db page]
@@ -36,32 +37,39 @@
(log/error :syntax/filters e)))))))
(defn- build-include-exclude-query
[variable includes excludes]
[includes excludes]
(concat
(for [include includes]
[variable :block/path-refs include])
(list 'has-ref '?b include))
(for [exclude excludes]
(list 'not [variable :block/path-refs exclude]))))
(list 'not (list 'has-ref '?b exclude)))))
(defn- filter-refs-query
[attribute includes excludes class-ids]
[includes excludes class-ids]
(let [clauses (concat
(build-include-exclude-query '?b includes excludes)
(build-include-exclude-query includes excludes)
(for [class-id class-ids]
(list 'not ['?b :block/tags class-id])))]
(into [:find '[?b ...]
:in '$ '[?id ...]
:in '$ '% '[?id ...]
:where
['?b attribute '?id]]
(list 'has-ref '?b '?id)]
clauses)))
(defn- get-path-refs
[db entity]
(let [refs (mapcat :block/refs (ldb/get-block-parents db (:block/uuid entity)))
block-page (:block/page entity)]
(->> (cond->> refs (some? block-page) (cons block-page))
distinct)))
(defn- get-ref-pages-count
[db id ref-blocks children-ids]
(when (seq ref-blocks)
(let [children (->> children-ids
(map (fn [id] (d/entity db id))))]
(->> (concat (mapcat :block/path-refs ref-blocks)
(mapcat :block/refs children))
(->> (concat (mapcat #(get-path-refs db %) ref-blocks)
(mapcat :block/refs (concat ref-blocks children)))
frequencies
(keep (fn [[ref size]]
(when (and (ldb/page? ref)
@@ -99,7 +107,12 @@
(set (conj class-children id))))
full-ref-block-ids (->> (mapcat (fn [id] (map :db/id (:block/_refs (d/entity db id)))) ids)
set)
matched-ref-block-ids (set (d/q (filter-refs-query :block/path-refs includes excludes class-ids) db ids))
matched-ref-block-ids (set (d/q (filter-refs-query includes excludes class-ids)
db
(rules/extract-rules rules/db-query-dsl-rules
[:has-ref]
{:deps rules/rules-dependencies})
ids))
matched-refs-with-children-ids (let [*result (atom #{})]
(doseq [ref-id matched-ref-block-ids]
(get-block-parents-until-top-ref db id ref-id full-ref-block-ids *result))

View File

@@ -14,13 +14,7 @@
"Rules used by frontend.db.query-dsl for file graphs. The symbols ?b and ?p
respectively refer to block and page. Do not alter them as they are
programmatically built by the query-dsl ns"
{:block-parent
'[[(block-parent ?p ?c)
[?c :block/parent ?p]]
[(block-parent ?p ?c)
[?t :block/parent ?p]
(block-parent ?t ?c)]]
:page-property
{:page-property
'[(page-property ?p ?key ?val)
[?p :block/name]
[?p :block/properties ?prop]
@@ -95,5 +89,5 @@
:page-ref
'[(page-ref ?b ?page-name)
[?b :block/path-refs ?br]
[?br :block/name ?page-name]]})
[?br :block/name ?page-name]
(has-ref ?b ?br)]})

View File

@@ -28,10 +28,6 @@
;; reference blocks
:block/refs {:db/valueType :db.type/ref
:db/cardinality :db.cardinality/many}
;; referenced pages inherited from the parents
:block/path-refs {:db/valueType :db.type/ref
:db/cardinality :db.cardinality/many}
:block/tags {:db/valueType :db.type/ref
:db/cardinality :db.cardinality/many}

View File

@@ -273,8 +273,7 @@
(def page-attrs
"Common attributes for pages"
[[:block/name :string]
[:block/title :string]
[:block/path-refs {:optional true} [:set :int]]])
[:block/title :string]])
(def property-attrs
"Common attributes for properties"
@@ -387,7 +386,6 @@
[:block/order block-order]
;; refs
[:block/page :int]
[:block/path-refs {:optional true} [:set :int]]
[:block/link {:optional true} :int]
[:logseq.property/created-from-property {:optional true} :int]])
@@ -399,8 +397,7 @@
[[:block/title :string]
[:block/parent :int]
;; These blocks only associate with pages of type "whiteboard"
[:block/page :int]
[:block/path-refs {:optional true} [:set :int]]]
[:block/page :int]]
page-or-block-attrs)))
(def property-value-block

View File

@@ -106,12 +106,6 @@
:cardinality :many
:public? false
:hide? true}}
:block/path-refs {:title "Node path references"
:attribute :block/path-refs
:schema {:type :entity
:cardinality :many
:public? false
:hide? true}}
:block/link {:title "Node links to"
:attribute :block/link
:schema {:type :entity
@@ -584,7 +578,7 @@
"Internal properties that are also db schema attributes"
#{:block/alias :block/tags :block/parent
:block/order :block/collapsed? :block/page
:block/refs :block/path-refs :block/link
:block/refs :block/link
:block/title :block/closed-value-property :block/journal-day
:block/created-at :block/updated-at})

View File

@@ -30,7 +30,14 @@
[?e2 :block/alias ?e3]]
[(alias ?e3 ?e1)
[?e1 :block/alias ?e2]
[?e2 :block/alias ?e3]]]})
[?e2 :block/alias ?e3]]]
:has-ref
'[[(has-ref ?b ?r)
[?b :block/refs ?r]]
[(has-ref ?b ?r)
(parent ?p ?b)
[?p :block/refs ?r]]]})
;; Rules writing advice
;; ====================
@@ -239,7 +246,9 @@
"For db graphs, a map of rule names and the rules they depend on. If this map
becomes long or brittle, we could do scan rules for their deps with something
like find-rules-in-where"
{:task #{:simple-query-property}
{:has-ref #{:parent}
:page-ref #{:has-ref}
:task #{:simple-query-property}
:priority #{:simple-query-property}
:property-missing-value #{:object-has-class-property}
:has-property-or-object-property #{:object-has-class-property}

View File

@@ -37,7 +37,7 @@
(map (juxt :major :minor)
[(parse-schema-version x) (parse-schema-version y)])))
(def version (parse-schema-version "65.10"))
(def version (parse-schema-version "65.11"))
(defn major-version
"Return a number.

View File

@@ -47,12 +47,9 @@
(concat portal-refs shape-link-refs)))
(defn- with-whiteboard-block-refs
[shape page-id]
[shape]
(let [refs (or (get-shape-refs shape) [])]
(merge {:block/refs (if (seq refs) refs [])
:block/path-refs (if (seq refs)
(conj refs page-id)
[])})))
{:block/refs (if (seq refs) refs [])}))
(defn- with-whiteboard-content
"Main purpose of this function is to populate contents when shapes are used as references in outliner."
@@ -72,7 +69,7 @@
(merge (when shape?
(merge
{:block/uuid (uuid (:id shape))}
(with-whiteboard-block-refs shape page-id)
(with-whiteboard-block-refs shape)
(with-whiteboard-content shape)))
(when (nil? (:block/parent block)) {:block/parent page-id})
(when (nil? (:block/format block)) {:block/format :markdown}) ;; TODO: read from config

View File

@@ -196,7 +196,7 @@
;; This graph will contain basic examples of different features to import
(p/let [file-graph-dir "test/resources/exporter-test-graph"
conn (db-test/create-conn)
;; Calculate refs and path-refs like frontend
;; Calculate refs like frontend
_ (db-pipeline/add-listener conn)
assets (atom [])
{:keys [import-state]} (import-file-graph-to-db file-graph-dir conn {:assets assets :convert-all-tags? true})]
@@ -587,16 +587,12 @@
(is (= "multiline block\na 2nd\nand a 3rd" (:block/title (db-test/find-block-by-content @conn #"multiline block"))))
(is (= "logbook block" (:block/title (db-test/find-block-by-content @conn #"logbook block")))))
(testing ":block/refs and :block/path-refs"
(testing ":block/refs"
(let [page (db-test/find-page-by-title @conn "chat-gpt")]
(is (set/subset?
#{"type" "LargeLanguageModel"}
(->> page :block/refs (map #(:block/title (d/entity @conn (:db/id %)))) set))
"Page has correct property and property value :block/refs")
(is (set/subset?
#{"type" "LargeLanguageModel"}
(->> page :block/path-refs (map #(:block/title (d/entity @conn (:db/id %)))) set))
"Page has correct property and property value :block/path-refs"))
"Page has correct property and property value :block/refs"))
(let [block (db-test/find-block-by-content @conn "old todo block")]
(is (set/subset?
@@ -605,14 +601,7 @@
:block/refs
(map #(:db/ident (d/entity @conn (:db/id %))))
set))
"Block has correct task tag and property :block/refs")
(is (set/subset?
#{:logseq.property/status :logseq.class/Task}
(->> block
:block/path-refs
(map #(:db/ident (d/entity @conn (:db/id %))))
set))
"Block has correct task tag and property :block/path-refs")))
"Block has correct task tag and property :block/refs")))
(testing "whiteboards"
(let [block-with-props (db-test/find-block-by-content @conn #"block with props")]

View File

@@ -285,7 +285,7 @@
collapse-or-expand? (= outliner-op :collapse-expand-blocks)
m* (cond->
(-> data'
(dissoc :block/children :block/meta :block/unordered :block/path-refs
(dissoc :block/children :block/meta :block/unordered
:block.temp/ast-title :block.temp/ast-body :block/level :block.temp/load-status
:block.temp/has-children?)
common-util/remove-nils
@@ -761,7 +761,7 @@
:block/title (or (:block/raw-title e) (:block/title e))}
b)
b)
dissoc-keys (concat [:block/tx-id :block/path-refs]
dissoc-keys (concat [:block/tx-id]
(when (contains? #{:insert-template-blocks :paste} outliner-op)
[:block/refs]))]
(apply dissoc b dissoc-keys))

View File

@@ -1,7 +1,6 @@
(ns logseq.outliner.pipeline
"Core fns for use with frontend worker and node"
(:require [clojure.set :as set]
[datascript.core :as d]
(:require [datascript.core :as d]
[datascript.impl.entity :as de]
[logseq.common.util.date-time :as date-time-util]
[logseq.db :as ldb]
@@ -19,103 +18,6 @@
:block/uuid (:v d)}))
datoms))
(defn- calculate-children-refs
[db-after children new-refs]
(let [;; Builds map of children ids to their parent id and :block/refs ids
children-maps (into {}
(keep (fn [id]
(when-let [entity (d/entity db-after [:block/uuid id])]
(let [from-property (:logseq.property/created-from-property entity)
default? (= :default (:logseq.property/type from-property))
page? (ldb/page? entity)]
(when-not (or page? (and from-property (not default?)))
[(:db/id entity)
{:parent-id (get-in entity [:block/parent :db/id])
:block-ref-ids (map :db/id (:block/refs entity))}]))))
children))
children-refs (map (fn [[id {:keys [block-ref-ids] :as child-map}]]
{:db/id id
;; Recalculate :block/path-refs as db contains stale data for this attribute
:block/path-refs
(set/union
;; Refs from top-level parent
new-refs
;; Refs from current block
block-ref-ids
;; Refs from parents in between top-level
;; parent and current block
(loop [parent-refs #{}
parent-id (:parent-id child-map)]
(if-let [parent (children-maps parent-id)]
(recur (into parent-refs (:block-ref-ids parent))
(:parent-id parent))
;; exits when top-level parent is reached
(remove nil? parent-refs))))})
children-maps)]
children-refs))
;; TODO: it'll be great if we can calculate the :block/path-refs before any
;; outliner transaction, this way we can group together the real outliner tx
;; and the new path-refs changes, which makes both undo/redo and
;; react-query/refresh! easier.
;; TODO: also need to consider whiteboard transactions
;; Steps:
;; 1. For each changed block, new-refs = its page + :block/refs + parents :block/refs
;; 2. Its children' block/path-refs might need to be updated too.
(defn- compute-block-path-refs
[{:keys [db-before db-after]} blocks*]
(let [*computed-ids (atom #{})
blocks (remove (fn [block]
(let [from-property (:logseq.property/created-from-property block)
default? (= :default (:logseq.property/type from-property))]
(and from-property (not default?))))
blocks*)]
(->>
(mapcat (fn [block]
(when-not (@*computed-ids (:block/uuid block))
(let [page? (ldb/page? block)
from-property (:logseq.property/created-from-property block)
parents' (when-not page?
(ldb/get-block-parents db-after (:block/uuid block) {}))
parents-refs (->> (cond->>
(mapcat :block/path-refs parents')
from-property
(remove (fn [parent] (and (ldb/property? parent) (not= (:db/id parent) (:db/id from-property))))))
(map :db/id))
old-refs (if db-before
(set (map :db/id (:block/path-refs (d/entity db-before (:db/id block)))))
#{})
new-refs (->>
(concat
(some-> (:db/id (:block/page block)) vector)
(map :db/id (:block/refs block))
parents-refs)
(remove nil?)
set)
refs-changed? (not= old-refs new-refs)
children (when refs-changed?
(when-not page?
(ldb/get-block-children-ids db-after (:block/uuid block))))
children-refs (when children
(calculate-children-refs db-after children new-refs))]
(swap! *computed-ids set/union (set (cons (:block/uuid block) children)))
(concat
(when (and (seq new-refs) refs-changed? (d/entity db-after (:db/id block)))
[{:db/id (:db/id block)
:block/path-refs new-refs}])
children-refs))))
blocks)
distinct)))
(defn ^:api compute-block-path-refs-tx
"Main fn for computing path-refs"
[tx-report blocks]
(let [refs-tx (compute-block-path-refs tx-report blocks)
truncate-refs-tx (map (fn [m] [:db/retract (:db/id m) :block/path-refs]) refs-tx)]
(concat truncate-refs-tx refs-tx)))
(defn- ref->eid
"ref: entity, map, int, eid"
[ref]
@@ -234,17 +136,10 @@
blocks))
(defn transact-new-db-graph-refs
"Transacts :block/refs and :block/path-refs for a new or imported DB graph"
"Transacts :block/refs for a new or imported DB graph"
[conn tx-report]
(let [{:keys [blocks]} (ds-report/get-blocks-and-pages tx-report)
refs-tx-report (when-let [refs-tx (and (seq blocks) (rebuild-block-refs-tx tx-report blocks))]
(ldb/transact! conn refs-tx {:pipeline-replace? true
::original-tx-meta (:tx-meta tx-report)}))
blocks' (if refs-tx-report
(keep (fn [b] (d/entity (:db-after refs-tx-report) (:db/id b))) blocks)
blocks)
block-path-refs-tx (distinct (compute-block-path-refs-tx tx-report blocks'))
path-refs-tx-report (when (seq block-path-refs-tx)
(ldb/transact! conn block-path-refs-tx {:pipeline-replace? true}))]
{:refs-tx-report refs-tx-report
:path-refs-tx-export path-refs-tx-report}))
::original-tx-meta (:tx-meta tx-report)}))]
refs-tx-report))

View File

@@ -1,65 +1,9 @@
(ns logseq.outliner.pipeline-test
(:require [cljs.test :refer [deftest is testing]]
[clojure.set :as set]
[clojure.string :as string]
[datascript.core :as d]
(:require [cljs.test :refer [deftest is]]
[logseq.common.util.page-ref :as page-ref]
[logseq.db.frontend.schema :as db-schema]
[logseq.db.sqlite.build :as sqlite-build]
[logseq.db.sqlite.create-graph :as sqlite-create-graph]
[logseq.db.test.helper :as db-test]
[logseq.outliner.db-pipeline :as db-pipeline]
[logseq.outliner.pipeline :as outliner-pipeline]))
(defn- get-blocks [db]
(->> (d/q '[:find (pull ?b [* {:block/path-refs [:block/name :db/id]}])
:in $
:where
[?b :block/page]
[?b :block/title]
[(missing? $ ?b :logseq.property/built-in?)]]
db)
(map first)))
(deftest compute-block-path-refs-tx
(testing "when a block's :refs change, descendants of block have correct :block/path-refs"
(let [conn (d/create-conn db-schema/schema)
;; needed in order for path-refs to be setup correctly with init data
_ (db-pipeline/add-listener conn)
_ (d/transact! conn (sqlite-create-graph/build-db-initial-data "{}"))
_ (sqlite-build/create-blocks
conn
[{:page {:block/title "bar"}}
{:page {:block/title "page1"}
:blocks [{:block/title "parent [[foo]]"
:build/children
[{:block/title "child [[baz]]"
:build/children
[{:block/title "grandchild [[bing]]"}]}]}]}])
blocks (get-blocks @conn)
;; Update parent block to replace 'foo' with 'bar' ref
new-tag-id (ffirst (d/q '[:find ?b :where [?b :block/title "bar"]] @conn))
modified-blocks (map #(if (string/starts-with? (:block/title %) "parent")
(assoc %
:block/refs [{:db/id new-tag-id}]
:block/path-refs [{:db/id new-tag-id}])
%)
blocks)
refs-tx (outliner-pipeline/compute-block-path-refs-tx {:db-after @conn} modified-blocks)
_ (d/transact! conn refs-tx {:pipeline-replace? true})
updated-blocks (->> (get-blocks @conn)
;; Only keep enough of content to uniquely identify block
(map #(hash-map :block/title (re-find #"\w+" (:block/title %))
:path-ref-names (set (map :block/name (:block/path-refs %))))))
page-tag-refs #{"page" "tags"}]
(is (= [{:block/title "parent"
:path-ref-names (set/union page-tag-refs #{"page1" "bar"})}
{:block/title "child"
:path-ref-names (set/union page-tag-refs #{"page1" "bar" "baz"})}
{:block/title "grandchild"
:path-ref-names (set/union page-tag-refs #{"page1" "bar" "baz" "bing"})}]
updated-blocks)))))
(deftest block-content-refs
(let [conn (db-test/create-conn-with-blocks
[{:page {:block/title "page1"} :blocks [{:block/title "b1"}]}])

View File

@@ -353,7 +353,7 @@ These tasks are specific to database graphs. For these tasks there is a one time
```sh
$ bb dev:db-query woot '[:find (pull ?b [*]) :where (block-content ?b "Dogma")]'
DB contains 833 datoms
[{:block/tx-id 536870923, :block/link #:db{:id 100065}, :block/uuid #uuid "65565c26-f972-4400-bce4-a15df488784d", :block/updated-at 1700158508564, :block/order "a0", :block/refs [#:db{:id 100064}], :block/created-at 1700158502056, :block/format :markdown, :block/tags [#:db{:id 100064}], :block/title "Dogma #[[65565c2a-b1c5-4dc8-a0f0-81b786bc5c6d]]", :db/id 100090, :block/path-refs [#:db{:id 100051} #:db{:id 100064}], :block/parent #:db{:id 100051}, :block/page #:db{:id 100051}}]
[{:block/tx-id 536870923, :block/link #:db{:id 100065}, :block/uuid #uuid "65565c26-f972-4400-bce4-a15df488784d", :block/updated-at 1700158508564, :block/order "a0", :block/refs [#:db{:id 100064}], :block/created-at 1700158502056, :block/format :markdown, :block/tags [#:db{:id 100064}], :block/title "Dogma #[[65565c2a-b1c5-4dc8-a0f0-81b786bc5c6d]]", :db/id 100090, :block/parent #:db{:id 100051}, :block/page #:db{:id 100051}}]
```
* `dev:db-transact` - Run a `d/transact!` against the queried results of a DB graph
@@ -424,9 +424,6 @@ These tasks are specific to database graphs. For these tasks there is a one time
[162 :block/format :markdown 536871037 true]
[162 :block/page 149 536871037 true]
[162 :block/parent 149 536871037 true]
[162 :block/path-refs 108 536871044 true]
[162 :block/path-refs 149 536871044 true]
[162 :block/path-refs 160 536871044 true]
[162
:block/properties
{#uuid "21be4275-bba9-48b8-9351-c9ca27883159"
@@ -462,9 +459,6 @@ These tasks are specific to database graphs. For these tasks there is a one time
[162 :block/order "a0" 536871037 true]
[162 :block/page 149 536871037 true]
[162 :block/parent 149 536871037 true]
[162 :block/path-refs 108 536871044 true]
[162 :block/path-refs 149 536871044 true]
[162 :block/path-refs 160 536871044 true]
[162
:block/properties
{#uuid "21be4275-bba9-48b8-9351-c9ca27883159"

View File

@@ -1,7 +1,7 @@
(ns ^:no-doc frontend.db.debug
(:require [frontend.db.utils :as db-utils]
(:require [datascript.core :as d]
[frontend.db :as db]
[datascript.core :as d]))
[frontend.db.utils :as db-utils]))
;; shortcut for query a block with string ref
(defn qb
@@ -16,8 +16,7 @@
(:block/page block)]
(:block/tags block)
(:block/alias block)
(:block/refs block)
(:block/path-refs block))
(:block/refs block))
(remove nil?)
(some (fn [x]
(and

View File

@@ -20,7 +20,6 @@
:block/format
:block/refs
:block/_refs
:block/path-refs
:block/tags
:block/link
:block/title

View File

@@ -381,10 +381,11 @@ independent of format as format specific heading characters are stripped"
(->>
(d/q
'[:find [(pull ?block ?block-attrs) ...]
:in $ [?ref-page ...] ?block-attrs
:in $ % [?ref-page ...] ?block-attrs
:where
[?block :block/path-refs ?ref-page]]
(has-ref ?block ?ref-page)]
db
(rules/extract-rules rules/db-query-dsl-rules [:parent :has-ref])
pages
(butlast file-model/file-graph-block-attrs))
(remove (fn [block] (= page-id (:db/id (:block/page block)))))

View File

@@ -699,11 +699,16 @@ Some bindings in this fn:
;; [(not (page-ref ?b "page 2"))]
(keyword (ffirst result))
(keyword (first result)))]
(add-bindings! (if (= key :and) (rest result) result) opts)))]
(add-bindings! (if (= key :and) (rest result) result) opts)))
extract-rules (fn [rules]
(rules/extract-rules rules/db-query-dsl-rules rules {:deps rules/rules-dependencies}))]
{:query result'
:rules (if db-graph?
(rules/extract-rules rules/db-query-dsl-rules rules {:deps rules/rules-dependencies})
(mapv file-rules/query-dsl-rules rules))
(extract-rules rules)
(->> (concat (map file-rules/query-dsl-rules (remove #{:page-ref} rules))
(when (some #{:page-ref} rules)
(extract-rules [:page-ref])))
vec))
:sort-by @sort-by
:blocks? (boolean @blocks?)
:sample sample})))

View File

@@ -340,8 +340,7 @@
(filter
(fn [id] (and (nil? (d/entity db-before id)) (d/entity db-after id)))
all-ids))
tx-data' (->> (remove (fn [d] (contains? #{:block/path-refs} (:a d))) tx-data)
vec)
tx-data' (vec tx-data)
editor-info @state/*editor-info
_ (reset! state/*editor-info nil)
op (->> [(when editor-info [::record-editor-info editor-info])

View File

@@ -293,8 +293,7 @@
[[:db/add id :db/ident (db-class/create-user-class-ident-from-name db title)]
[:db/add id :logseq.property.class/extends :logseq.class/Root]
[:db/retract id :block/tags :logseq.class/Page]
[:db/retract id :block/refs :logseq.class/Page]
[:db/retract id :block/path-refs :logseq.class/Page]]))
[:db/retract id :block/refs :logseq.class/Page]]))
class-ids)))
(defn fix-using-properties-as-tags
@@ -354,6 +353,14 @@
:block/name (common-util/page-name-sanity-lc (:block/title page))})))
pages)))
(defn- remove-block-path-refs-datoms
[db]
(->> (d/datoms db :avet :block/path-refs)
(map :e)
(distinct)
(map (fn [id]
[:db/retract id :block/path-refs]))))
(def schema-version->updates
"A vec of tuples defining datascript migrations. Each tuple consists of the
schema version integer and a migration map. A migration map can have keys of :properties, :classes
@@ -368,7 +375,8 @@
["65.7" {:fix add-quick-add-page}]
["65.8" {:fix add-missing-page-name}]
["65.9" {:properties [:logseq.property.embedding/hnsw-label-updated-at]}]
["65.10" {:properties [:block/journal-day :logseq.property.view/sort-groups-by-property :logseq.property.view/sort-groups-desc?]}]])
["65.10" {:properties [:block/journal-day :logseq.property.view/sort-groups-by-property :logseq.property.view/sort-groups-desc?]}]
["65.11" {:fix remove-block-path-refs-datoms}]])
(let [[major minor] (last (sort (map (comp (juxt :major :minor) db-schema/parse-schema-version first)
schema-version->updates)))]

View File

@@ -19,27 +19,28 @@
"Return tx-report"
[repo conn {:keys [tx-meta] :as tx-report}]
(when repo (worker-state/set-db-latest-tx-time! repo))
(let [{:keys [from-disk?]} tx-meta
result (worker-pipeline/invoke-hooks repo conn tx-report (worker-state/get-context))
tx-report' (:tx-report result)]
(when (and result (not (:rtc-download-graph? tx-meta)))
(let [data (merge
{:request-id (:request-id tx-meta)
:repo repo
:tx-data (:tx-data tx-report')
:tx-meta tx-meta}
(dissoc result :tx-report))]
(shared-service/broadcast-to-clients! :sync-db-changes data))
(when-not (:rtc-download-graph? tx-meta)
(let [{:keys [from-disk?]} tx-meta
result (worker-pipeline/invoke-hooks repo conn tx-report (worker-state/get-context))
tx-report' (:tx-report result)]
(when result
(let [data (merge
{:request-id (:request-id tx-meta)
:repo repo
:tx-data (:tx-data tx-report')
:tx-meta tx-meta}
(dissoc result :tx-report))]
(shared-service/broadcast-to-clients! :sync-db-changes data))
(when-not from-disk?
(p/do!
(when-not from-disk?
(p/do!
;; Sync SQLite search
(let [{:keys [blocks-to-remove-set blocks-to-add]} (search/sync-search-indice repo tx-report')]
(when (seq blocks-to-remove-set)
((@thread-api/*thread-apis :thread-api/search-delete-blocks) repo blocks-to-remove-set))
(when (seq blocks-to-add)
((@thread-api/*thread-apis :thread-api/search-upsert-blocks) repo blocks-to-add))))))
tx-report'))
(let [{:keys [blocks-to-remove-set blocks-to-add]} (search/sync-search-indice repo tx-report')]
(when (seq blocks-to-remove-set)
((@thread-api/*thread-apis :thread-api/search-delete-blocks) repo blocks-to-remove-set))
(when (seq blocks-to-add)
((@thread-api/*thread-apis :thread-api/search-upsert-blocks) repo blocks-to-add))))))
tx-report')))
(comment
(defmethod listen-db-changes :debug-listen-db-changes

View File

@@ -172,7 +172,7 @@
(when (and from-page to-page (not= from-page-name to-page-name))
(let [datoms (d/datoms @conn :avet :block/page from-id)
block-eids (mapv :e datoms)
blocks (d/pull-many db '[:db/id :block/page :block/refs :block/path-refs :block/order :block/parent] block-eids)
blocks (d/pull-many db '[:db/id :block/page :block/refs :block/order :block/parent] block-eids)
blocks-tx-data (map (fn [block]
(let [id (:db/id block)]
(cond->

View File

@@ -36,14 +36,6 @@
(contains? #{:collapse-expand-blocks :delete-blocks} outliner-op)
(:undo? tx-meta) (:redo? tx-meta)))))
(defn- compute-block-path-refs-tx
[{:keys [tx-meta] :as tx-report} blocks]
(when (or (:rtc-tx? tx-meta)
(and (:outliner-op tx-meta) (refs-need-recalculated? tx-meta))
(:from-disk? tx-meta)
(:new-graph? tx-meta))
(outliner-pipeline/compute-block-path-refs-tx tx-report blocks)))
(defn- rebuild-block-refs
[repo {:keys [tx-meta db-after]} blocks]
(when (or (and (:outliner-op tx-meta) (refs-need-recalculated? tx-meta))
@@ -302,19 +294,16 @@
(:added d))
(when-let [display-type (ldb/get-display-type-by-class-ident (:db/ident (d/entity db (:v d))))]
[(cond->
{:db/id (:e d)
:logseq.property.node/display-type display-type}
{:db/id (:e d)
:logseq.property.node/display-type display-type}
(and (= display-type :code) (d/entity db :logseq.kv/latest-code-lang))
(assoc :logseq.property.code/lang (:kv/value (d/entity db :logseq.kv/latest-code-lang))))])))
datoms)))
(defn- invoke-hooks-for-imported-graph [conn {:keys [tx-meta] :as tx-report}]
(let [{:keys [refs-tx-report path-refs-tx-report]}
(outliner-pipeline/transact-new-db-graph-refs conn tx-report)
full-tx-data (concat (:tx-data tx-report)
(:tx-data refs-tx-report)
(:tx-data path-refs-tx-report))
final-tx-report (-> (or path-refs-tx-report refs-tx-report tx-report)
(let [refs-tx-report (outliner-pipeline/transact-new-db-graph-refs conn tx-report)
full-tx-data (concat (:tx-data tx-report) (:tx-data refs-tx-report))
final-tx-report (-> (or refs-tx-report tx-report)
(assoc :tx-data full-tx-data
:tx-meta tx-meta
:db-before (:db-before tx-report)))]
@@ -447,11 +436,6 @@
:skip-store? true}))
replace-tx (let [db-after (or (:db-after refs-tx-report) (:db-after tx-report*))]
(concat
;; block path refs
(when (seq blocks')
(let [blocks' (keep (fn [b] (d/entity db-after (:db/id b))) blocks')]
(compute-block-path-refs-tx tx-report* blocks')))
;; update block/tx-id
(let [updated-blocks (remove (fn [b] (contains? deleted-block-ids (:db/id b)))
(concat pages blocks))
@@ -492,17 +476,7 @@
(let [{:keys [from-disk? new-graph?]} tx-meta]
(cond
(or from-disk? new-graph?)
(let [{:keys [blocks]} (ds-report/get-blocks-and-pages tx-report)
path-refs (distinct (compute-block-path-refs-tx tx-report blocks))
tx-report' (if (seq path-refs)
(ldb/transact! conn path-refs {:pipeline-replace? true})
tx-report)
full-tx-data (concat (:tx-data tx-report) (:tx-data tx-report'))
final-tx-report (assoc tx-report'
:tx-meta (:tx-meta tx-report)
:tx-data full-tx-data
:db-before (:db-before tx-report))]
{:tx-report final-tx-report})
{:tx-report tx-report}
(or (::gp-exporter/new-graph? tx-meta) (::sqlite-export/imported-data? tx-meta))
(invoke-hooks-for-imported-graph conn tx-report)

View File

@@ -36,7 +36,7 @@
(map :v)
(distinct))
refs (->> (filter (fn [datom]
(when (contains? #{:block/refs :block/path-refs} (:a datom))
(when (contains? #{:block/refs} (:a datom))
(not= (:v datom)
(:db/id (:block/page (d/entity db-after (:e datom))))))) tx-data)
(map :v)
@@ -67,13 +67,13 @@
blocks [(when-let [parent-id (:db/id (:block/parent block))]
[::block parent-id])
[::block (:db/id block)]]
path-refs (:block/path-refs block)
path-refs' (->> (keep (fn [ref]
(when-not (= (:db/id ref) page-id)
[[::refs (:db/id ref)]
[::block (:db/id ref)]])) path-refs)
(apply concat))]
(concat blocks path-refs')))
block-refs (:block/refs block)
refs (->> (keep (fn [ref]
(when-not (= (:db/id ref) page-id)
[[::refs (:db/id ref)]
[::block (:db/id ref)]])) block-refs)
(apply concat))]
(concat blocks refs)))
block-entities)
(mapcat

View File

@@ -15,7 +15,8 @@
[frontend.worker.rtc.ws-util :as ws-util]
[logseq.db :as ldb]
[logseq.db.frontend.schema :as db-schema]
[missionary.core :as m]))
[missionary.core :as m]
[tick.core :as tick]))
(defn- new-task--register-graph-updates
[get-ws-create-task graph-uuid major-schema-version repo]
@@ -34,61 +35,78 @@
(throw (ex-info "remote graph is still creating" {:missionary/retry true} e))
(throw e))))))
(defn- ensure-register-graph-updates*
(def ^:private *register-graph-updates-sent
"ws -> [bool, added-inst, [graph-uuid,major-schema-version,repo]]"
(atom {}))
(defn- clean-old-keys-in-sent!
[]
(let [hours-ago (tick/<< (tick/instant) (tick/new-duration 3 :hours))
old-ks
(keep (fn [[k [_ added-inst]]]
(when (tick/< added-inst hours-ago)
k))
@*register-graph-updates-sent)]
(doseq [k old-ks]
(swap! *register-graph-updates-sent dissoc k))))
(defn ensure-register-graph-updates--memoized
"Return a task: get or create a mws(missionary wrapped websocket).
see also `ws/get-mws-create`.
But ensure `register-graph-updates` and `calibrate-graph-skeleton` has been sent"
[get-ws-create-task graph-uuid major-schema-version repo conn
*last-calibrate-t *online-users *server-schema-version add-log-fn]
(assert (some? graph-uuid))
(let [*sent (atom {}) ;; ws->bool
]
(m/sp
(let [ws (m/? get-ws-create-task)]
(when-not (contains? @*sent ws)
(swap! *sent assoc ws false))
(when (not (@*sent ws))
(let [recv-flow (ws/recv-flow (m/? get-ws-create-task))]
(c.m/run-task :update-online-user-when-register-graph-updates
(m/sp
(when-let [online-users (:online-users
(m/?
(m/timeout
(m/reduce
(fn [_ v]
(when (= "online-users-updated" (:req-id v))
(reduced v)))
recv-flow)
10000)))]
(reset! *online-users online-users)))
:succ (constantly nil)))
(let [{:keys [max-remote-schema-version]}
(m/sp
(let [ws (m/? get-ws-create-task)
sent-3rd-value [graph-uuid major-schema-version repo]
origin-v (@*register-graph-updates-sent ws)]
(when (or (nil? origin-v)
(not= (last origin-v) sent-3rd-value))
(swap! *register-graph-updates-sent assoc ws [false (tick/instant) sent-3rd-value])
(clean-old-keys-in-sent!))
(when (not (first (@*register-graph-updates-sent ws)))
(swap! *register-graph-updates-sent assoc-in [ws 0] true)
(let [recv-flow (ws/recv-flow (m/? get-ws-create-task))]
(c.m/run-task :update-online-user-when-register-graph-updates
(m/sp
(when-let [online-users (:online-users
(m/?
(m/timeout
(m/reduce
(fn [_ v]
(when (= "online-users-updated" (:req-id v))
(reduced v)))
recv-flow)
10000)))]
(reset! *online-users online-users)))
:succ (constantly nil)))
(let [{:keys [max-remote-schema-version]}
(try
(m/?
(c.m/backoff
{:delay-seq
;retry 5 times if remote-graph is creating (4000 8000 16000 32000 64000)
{:delay-seq ;retry 5 times if remote-graph is creating (4000 8000 16000 32000 64000)
(take 5 (drop 2 c.m/delays))
:reset-flow worker-flows/online-event-flow}
(new-task--register-graph-updates get-ws-create-task graph-uuid major-schema-version repo)))]
(when max-remote-schema-version
(add-log-fn :rtc.log/higher-remote-schema-version-exists
{:sub-type (r.branch-graph/compare-schemas
max-remote-schema-version db-schema/version major-schema-version)
:repo repo
:graph-uuid graph-uuid
:remote-schema-version max-remote-schema-version})))
(let [t (client-op/get-local-tx repo)]
(when (or (nil? @*last-calibrate-t)
(< 500 (- t @*last-calibrate-t)))
(let [{:keys [server-schema-version _server-builtin-db-idents]}
(m/? (r.skeleton/new-task--calibrate-graph-skeleton
get-ws-create-task graph-uuid major-schema-version @conn))]
(reset! *server-schema-version server-schema-version))
(reset! *last-calibrate-t t)))
(swap! *sent assoc ws true))
ws))))
(def ensure-register-graph-updates (memoize ensure-register-graph-updates*))
(new-task--register-graph-updates get-ws-create-task graph-uuid major-schema-version repo)))
(catch :default e
(swap! *register-graph-updates-sent assoc-in [ws 0] false)
(throw e)))]
(when max-remote-schema-version
(add-log-fn :rtc.log/higher-remote-schema-version-exists
{:sub-type (r.branch-graph/compare-schemas
max-remote-schema-version db-schema/version major-schema-version)
:repo repo
:graph-uuid graph-uuid
:remote-schema-version max-remote-schema-version})))
(let [t (client-op/get-local-tx repo)]
(when (or (nil? @*last-calibrate-t)
(< 500 (- t @*last-calibrate-t)))
(let [{:keys [server-schema-version _server-builtin-db-idents]}
(m/? (r.skeleton/new-task--calibrate-graph-skeleton
get-ws-create-task graph-uuid major-schema-version @conn))]
(reset! *server-schema-version server-schema-version))
(reset! *last-calibrate-t t))))
ws)))
(defn- ->pos
[parent-uuid order]

View File

@@ -230,7 +230,7 @@
(rtc-log-and-state/rtc-log type (assoc message :graph-uuid graph-uuid)))
{:keys [*current-ws get-ws-create-task]}
(gen-get-ws-create-map--memoized ws-url)
get-ws-create-task (r.client/ensure-register-graph-updates
get-ws-create-task (r.client/ensure-register-graph-updates--memoized
get-ws-create-task graph-uuid major-schema-version
repo conn *last-calibrate-t *online-users *server-schema-version add-log-fn)
{:keys [assets-sync-loop-task]}

File diff suppressed because one or more lines are too long

View File

@@ -651,11 +651,13 @@ prop-d:: [[nada]]"}])
- [[Child page]]
- p2 [[Parent page]]
- Non linked content"}]))
(is (= ["Non linked content"
"p2"
"p1"]
(map testable-content
(dsl-query "(and [[Parent page]] (not [[Child page]]))")))))
(is (= (set
["Non linked content"
"p2"
"p1"])
(set
(map testable-content
(dsl-query "(and [[Parent page]] (not [[Child page]]))"))))))
(deftest between-queries
(load-test-files [{:file/path "journals/2020_12_26.md"

View File

@@ -76,7 +76,8 @@
(testing "Linked references without filters"
(let [{:keys [ref-pages-count ref-blocks ref-matched-children-ids]} (db-reference/get-linked-references db (:db/id foo))]
(is (= [["baz" 4] ["Journal" 2] ["Jun 11th, 2025" 2] ["bar" 2]] (vec ref-pages-count))
(is (= (set [["baz" 4] ["Journal" 2] ["Jun 11th, 2025" 2] ["bar" 2]])
(set ref-pages-count))
"ref-pages-count check failed")
(is (empty? ref-matched-children-ids)
"ref-matched-children-ids check failed")
@@ -88,7 +89,8 @@
[{:db/id (:db/id foo)
:logseq.property.linked-references/includes (:db/id bar)}])
(let [{:keys [ref-pages-count ref-blocks ref-matched-children-ids]} (db-reference/get-linked-references @conn (:db/id foo))]
(is (= [["baz" 3] ["Journal" 2] ["Jun 11th, 2025" 2] ["bar" 2]] (vec ref-pages-count))
(is (= (set [["baz" 3] ["Journal" 2] ["Jun 11th, 2025" 2] ["bar" 2]])
(set ref-pages-count))
"ref-pages-count check failed")
(is (= 7 (count ref-matched-children-ids))
"ref-matched-children-ids check failed")
@@ -100,7 +102,8 @@
[{:db/id (:db/id foo)
:logseq.property.linked-references/includes (:db/id baz)}])
(let [{:keys [ref-pages-count ref-blocks ref-matched-children-ids]} (db-reference/get-linked-references @conn (:db/id foo))]
(is (= [["baz" 3] ["Journal" 2] ["Jun 11th, 2025" 2] ["bar" 2]] (vec ref-pages-count))
(is (= (set [["baz" 3] ["Journal" 2] ["Jun 11th, 2025" 2] ["bar" 2]])
(set ref-pages-count))
"ref-pages-count check failed")
(is (= 7 (count ref-matched-children-ids))
"ref-matched-children-ids check failed")
@@ -113,7 +116,7 @@
[{:db/id (:db/id foo)
:logseq.property.linked-references/excludes (:db/id bar)}])
(let [{:keys [ref-pages-count ref-blocks ref-matched-children-ids]} (db-reference/get-linked-references @conn (:db/id foo))]
(is (= [["Journal" 2] ["Jun 11th, 2025" 2] ["baz" 2]] (vec ref-pages-count))
(is (= (set [["Journal" 2] ["Jun 11th, 2025" 2] ["baz" 2]]) (set ref-pages-count))
"ref-pages-count check failed")
(is (= 2 (count ref-matched-children-ids))
"ref-matched-children-ids check failed")
@@ -126,7 +129,7 @@
[{:db/id (:db/id foo)
:logseq.property.linked-references/excludes (:db/id baz)}])
(let [{:keys [ref-pages-count ref-blocks ref-matched-children-ids]} (db-reference/get-linked-references @conn (:db/id foo))]
(is (= [["Journal" 2] ["Jun 11th, 2025" 2] ["bar" 1]] (vec ref-pages-count))
(is (= (set [["Journal" 2] ["Jun 11th, 2025" 2] ["bar" 1]]) (set ref-pages-count))
"ref-pages-count check failed")
(is (= 3 (count ref-matched-children-ids))
"ref-matched-children-ids check failed")
@@ -139,7 +142,7 @@
[{:db/id (:db/id foo)
:logseq.property.linked-references/excludes #{(:db/id baz) (:db/id bar)}}])
(let [{:keys [ref-pages-count ref-blocks ref-matched-children-ids]} (db-reference/get-linked-references @conn (:db/id foo))]
(is (= [["Journal" 2] ["Jun 11th, 2025" 2]] (vec ref-pages-count))
(is (= (set [["Journal" 2] ["Jun 11th, 2025" 2]]) (set ref-pages-count))
"ref-pages-count check failed")
(is (zero? (count ref-matched-children-ids))
"ref-matched-children-ids check failed")
@@ -153,7 +156,7 @@
:logseq.property.linked-references/includes (:db/id bar)
:logseq.property.linked-references/excludes (:db/id baz)}])
(let [{:keys [ref-pages-count ref-blocks ref-matched-children-ids]} (db-reference/get-linked-references @conn (:db/id foo))]
(is (= [["Journal" 1] ["Jun 11th, 2025" 1] ["bar" 1]] (vec ref-pages-count))
(is (= (set [["Journal" 1] ["Jun 11th, 2025" 1] ["bar" 1]]) (set ref-pages-count))
"ref-pages-count check failed")
(is (= 3 (count ref-matched-children-ids))
"ref-matched-children-ids check failed")
@@ -167,7 +170,7 @@
:logseq.property.linked-references/includes (:db/id baz)
:logseq.property.linked-references/excludes (:db/id bar)}])
(let [{:keys [ref-pages-count ref-blocks ref-matched-children-ids]} (db-reference/get-linked-references @conn (:db/id foo))]
(is (= [["Journal" 2] ["Jun 11th, 2025" 2] ["baz" 2]] (vec ref-pages-count))
(is (= (set [["Journal" 2] ["Jun 11th, 2025" 2] ["baz" 2]]) (set ref-pages-count))
"ref-pages-count check failed")
(is (= 2 (count ref-matched-children-ids))
"ref-matched-children-ids check failed")

View File

@@ -1,11 +1,11 @@
(ns frontend.handler.editor-test
(:require [frontend.handler.editor :as editor]
[frontend.db :as db]
[clojure.test :refer [deftest is testing are use-fixtures]]
(:require [clojure.test :refer [deftest is testing are use-fixtures]]
[datascript.core :as d]
[frontend.test.helper :as test-helper :refer [load-test-files]]
[frontend.db :as db]
[frontend.db.model :as model]
[frontend.handler.editor :as editor]
[frontend.state :as state]
[frontend.test.helper :as test-helper :refer [load-test-files]]
[frontend.util.cursor :as cursor]))
(use-fixtures :each test-helper/start-and-destroy-db)
@@ -239,22 +239,16 @@
(load-test-files [{:file/path "pages/page1.md"
:file/content "\n
- b1 #foo"}])
(testing "updating block's content changes content and preserves path-refs"
(let [conn (db/get-db test-helper/test-db false)
block (->> (d/q '[:find (pull ?b [* {:block/path-refs [:block/name]}])
:where [?b :block/title "b1 #foo"]]
@conn)
ffirst)
prev-path-refs (set (map :block/name (:block/path-refs block)))
_ (assert (= #{"page1" "foo"} prev-path-refs)
"block has expected :block/path-refs")
(testing "updating block's content changes content"
(let [conn (db/get-db test-helper/test-db false)
block (->> (d/q '[:find (pull ?b [*])
:where [?b :block/title "b1 #foo"]]
@conn)
ffirst)
;; Use same options as edit-box-on-change!
_ (editor/save-block-aux! block "b12 #foo" {:skip-properties? true})
updated-block (d/pull @conn '[* {:block/path-refs [:block/name]}] [:block/uuid (:block/uuid block)])]
(is (= "b12 #foo" (:block/title updated-block)) "Content updated correctly")
(is (= prev-path-refs
(set (map :block/name (:block/path-refs updated-block))))
"Path-refs remain the same"))))
_ (editor/save-block-aux! block "b12 #foo" {:skip-properties? true})
updated-block (d/pull @conn '[*] [:block/uuid (:block/uuid block)])]
(is (= "b12 #foo" (:block/title updated-block)) "Content updated correctly"))))
(deftest save-block!
(testing "Saving blocks with and without properties"

View File

@@ -26,12 +26,12 @@
(docs-graph-helper/docs-graph-assertions db graph-dir (map :file/path files))
(testing "Additional Counts"
(is (= 77370 (count (d/datoms db :eavt))) "Correct datoms count")
(is (= 58149 (count (d/datoms db :eavt))) "Correct datoms count")
(is (= 7095
(is (= 2065
(ffirst
(d/q '[:find (count ?b)
:where [?b :block/path-refs ?bp] [?bp :block/name]] db)))
:where [?b :block/refs ?bp] [?bp :block/name]] db)))
"Correct referenced blocks count"))))
(deftest parse-files-and-load-to-db-with-block-refs-on-reload