clone/logseq
1
0
Fork 0
mirror of https://github.com/logseq/logseq.git synced 2026-05-24 12:44:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feat/agents

Browse Source
This commit is contained in:
Tienson Qin
2026-02-26 22:59:19 +08:00
parent 171da1bc5b 7d268fbd42
commit 477fca1d9f
35 changed files with 1338 additions and 416 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
clj-e2e/test/logseq/e2e/editor_basic_test.clj
View File

@@ -17,6 +17,35 @@
fixtures/new-logseq-page
fixtures/validate-graph)
(defn- drag-and-drop-file!
[file-name file-type]
(w/eval-js
(format "(() => {
const container = document.querySelector('#main-content-container');
if (!container) {
throw new Error('main-content-container not found');
}
const dataTransfer = new DataTransfer();
dataTransfer.items.add(new File(['logseq-e2e-drag-drop'], %s, { type: %s }));
container.dispatchEvent(new DragEvent('dragover', { dataTransfer, bubbles: true, cancelable: true }));
container.dispatchEvent(new DragEvent('drop', { dataTransfer, bubbles: true, cancelable: true }));
})();"
(pr-str file-name)
(pr-str file-type))))
(deftest drag-and-drop-asset-does-not-create-blank-asset
(testing "dragging and dropping a file should keep non-empty asset title"
(let [asset-title "drag-drop-regression"
file-name (str asset-title ".png")]
(b/new-block "")
(drag-and-drop-file! file-name "image/png")
(w/wait-for ".ls-page-blocks .ls-block .asset-container img")
;; Exit edit mode to trigger a save; this used to overwrite the new asset with blank content.
(util/exit-edit)
(assert/assert-have-count ".ls-page-blocks .ls-block .asset-container img" 1)
(assert/assert-is-visible
(format ".ls-page-blocks .ls-block .block-title-wrap:text('%s')" asset-title)))))
(deftest toggle-between-page-and-block
(testing "Convert block to page and back"
(b/new-block "b1")

6
deps/db-sync/shadow-cljs.edn vendored
View File

@@ -13,7 +13,8 @@
AgentSessionDO logseq.db-sync.worker/AgentSessionDO
Sandbox logseq.db-sync.worker/Sandbox}}}
:js-options {:js-provider :import}
:closure-defines {shadow.cljs.devtools.client.env/enabled false}
:closure-defines {shadow.cljs.devtools.client.env/enabled false
goog.debug.LOGGING_ENABLED true}
:devtools {:enabled false}}
:db-sync-node {:target :node-script
:output-to "worker/dist/node-adapter.js"
@@ -21,7 +22,8 @@
:compiler-options {:source-map true
:warnings {:fn-deprecated false
:redef false}}
:devtools {:enabled false}}
:devtools {:enabled false}
:closure-defines {goog.debug.LOGGING_ENABLED true}}
:db-sync-test {:target :node-test
:output-to "worker/dist/worker-test.js"
:devtools {:enabled false}

3
deps/db-sync/src/logseq/db_sync/worker.cljs vendored
View File

@@ -72,8 +72,7 @@
(ws/send! ws {:type "error" :message "server error"}))))
(webSocketClose [this ws _code _reason]
(presence/remove-presence! this ws)
(presence/broadcast-online-users! this)
(log/info :db-sync/ws-closed true))
(presence/broadcast-online-users! this))
(webSocketError [this ws error]
(presence/remove-presence! this ws)
(presence/broadcast-online-users! this)

20
deps/db-sync/src/logseq/db_sync/worker/auth.cljs vendored
View File

@@ -40,12 +40,22 @@
(let [message (or (ex-message error) (some-> error .-message))]
(contains? recoverable-auth-errors message))))
(defn- expired-token?
[token]
(when-let [claims (unsafe-jwt-claims token)]
(let [exp (aget claims "exp")
now-s (js/Math.floor (/ (.now js/Date) 1000))]
(and (number? exp)
(<= exp now-s)))))
(defn auth-claims [request env]
(let [token (token-from-request request)]
(if (string? token)
(-> (authorization/verify-jwt token env)
(p/catch (fn [error]
(if (recoverable-auth-error? error)
nil
(p/rejected error)))))
(if (expired-token? token)
(p/resolved nil)
(-> (authorization/verify-jwt token env)
(p/catch (fn [error]
(if (recoverable-auth-error? error)
nil
(p/rejected error))))))
(p/resolved nil))))

121
deps/db-sync/src/logseq/db_sync/worker/handler/index.cljs vendored
View File

@@ -321,12 +321,117 @@
(log/error :db-sync/index-error error)
(http/error-response (str "server error: " error) 500)))))
(defn graph-access-response [request env graph-id]
(def ^:private graph-access-cache-ttl-ms 5000)
(def ^:private graph-access-cache-capacity 256)
(defonce ^:private *graph-access-cache (atom {}))
(defn- now-ms []
(.now js/Date))
(defn- unauthorized-timing [jwt-verify-ms]
{:access-ok? false
:cache-hit? false
:jwt-verify-ms jwt-verify-ms
:access-query-ms 0
:access-check-ms jwt-verify-ms})
(defn- fresh-cache?
[cached-at current-ms]
(and (number? cached-at)
(< (- current-ms cached-at) graph-access-cache-ttl-ms)))
(defn- lookup-graph-access-cache
[graph-id token current-ms]
(let [cache-key [graph-id token]]
(when-let [{:keys [allowed? cached-at]} (get @*graph-access-cache cache-key)]
(if (fresh-cache? cached-at current-ms)
{:allowed? allowed?}
(do
(swap! *graph-access-cache dissoc cache-key)
nil)))))
(defn- prune-graph-access-cache
[cache current-ms]
(let [fresh (into {}
(filter (fn [[_ {:keys [cached-at]}]]
(fresh-cache? cached-at current-ms)))
cache)]
(if (<= (count fresh) graph-access-cache-capacity)
fresh
(let [drop-count (- (count fresh) graph-access-cache-capacity)]
(->> fresh
(sort-by (comp :cached-at val))
(drop drop-count)
(into {}))))))
(defn- cache-graph-access!
[graph-id token allowed? current-ms]
(let [cache-key [graph-id token]]
(swap! *graph-access-cache
(fn [cache]
(-> cache
(assoc cache-key {:allowed? allowed? :cached-at current-ms})
(prune-graph-access-cache current-ms))))))
(defn graph-access-response-with-timing
[request env graph-id]
(let [token (auth/token-from-request request)
url (js/URL. (.-url request))
access-url (str (.-origin url) "/graphs/" graph-id "/access")
headers (js/Headers. (.-headers request))
index-self #js {:env env :d1 (aget env "DB")}]
(when (string? token)
(.set headers "authorization" (str "Bearer " token)))
(handle-fetch index-self (js/Request. access-url #js {:method "GET" :headers headers}))))
db (aget env "DB")]
(cond
(or (not (string? token))
(not (seq token)))
(p/resolved {:response (http/unauthorized)
:timing (unauthorized-timing 0)})
(nil? db)
(p/resolved {:response (http/error-response "server error" 500)
:timing {:access-ok? false
:cache-hit? false}})
:else
(let [current-ms (now-ms)]
(if-let [{:keys [allowed?]} (lookup-graph-access-cache graph-id token current-ms)]
(p/resolved {:response (if allowed?
(http/json-response :graphs/access {:ok true})
(http/forbidden))
:timing {:access-ok? allowed?
:cache-hit? true
:jwt-verify-ms 0
:access-query-ms 0
:access-check-ms 0}})
(let [jwt-start-ms (now-ms)]
(->
(p/let [claims (auth/auth-claims request env)
jwt-end-ms (now-ms)
jwt-verify-ms (- jwt-end-ms jwt-start-ms)]
(if (nil? claims)
{:response (http/unauthorized)
:timing (unauthorized-timing jwt-verify-ms)}
(let [user-id (aget claims "sub")]
(if-not (string? user-id)
{:response (http/unauthorized)
:timing (unauthorized-timing jwt-verify-ms)}
(p/let [query-start-ms (now-ms)
access? (index/<user-has-access-to-graph? db graph-id user-id)
query-end-ms (now-ms)
access-query-ms (- query-end-ms query-start-ms)
access-check-ms (+ jwt-verify-ms access-query-ms)
_ (cache-graph-access! graph-id token (true? access?) query-end-ms)
response (if access?
(http/json-response :graphs/access {:ok true})
(http/forbidden))]
{:response response
:timing {:access-ok? (true? access?)
:cache-hit? false
:jwt-verify-ms jwt-verify-ms
:access-query-ms access-query-ms
:access-check-ms access-check-ms}})))))
(p/catch (fn [error]
(log/error :db-sync/index-error error)
(p/resolved {:response (http/error-response (str "server error: " error) 500)
:timing {:access-ok? false
:cache-hit? false}}))))))))))
(defn graph-access-response [request env graph-id]
(p/let [{:keys [response]} (graph-access-response-with-timing request env graph-id)]
response))

45
deps/db-sync/test/logseq/db_sync/normalize_test.cljs vendored Normal file
View File

@@ -0,0 +1,45 @@
(ns logseq.db-sync.normalize-test
(:require [cljs.test :refer [deftest is testing]]
[datascript.core :as d]
[logseq.db.common.normalize :as db-normalize]
[logseq.db.test.helper :as db-test]))
(defn- new-conn []
(db-test/create-conn))
(defn- create-page!
[conn title]
(let [page-uuid (random-uuid)]
(d/transact! conn [{:block/uuid page-uuid
:block/name title
:block/title title}])
page-uuid))
(defn- op-e-a-v
[datom]
(subvec (vec datom) 0 4))
(deftest normalize-tx-data-keeps-title-retract-without-replacement-test
(let [conn (new-conn)
page-uuid (create-page! conn "Page")
tx-report (d/transact! conn [[:db/retract [:block/uuid page-uuid] :block/title "Page"]])
normalized (db-normalize/normalize-tx-data (:db-after tx-report)
(:db-before tx-report)
(:tx-data tx-report))
tx-data (mapv op-e-a-v normalized)]
(testing "keeps :block/title retract when no replacement title exists in same tx"
(is (= [[:db/retract [:block/uuid page-uuid] :block/title "Page"]]
tx-data)))))
(deftest normalize-tx-data-drops-title-retract-when-replaced-test
(let [conn (new-conn)
page-uuid (create-page! conn "Page")
tx-report (d/transact! conn [{:block/uuid page-uuid
:block/title "Page 2"}])
normalized (db-normalize/normalize-tx-data (:db-after tx-report)
(:db-before tx-report)
(:tx-data tx-report))
tx-data (mapv op-e-a-v normalized)]
(testing "drops old :block/title retract and keeps new add during title update"
(is (some #(= [:db/add [:block/uuid page-uuid] :block/title "Page 2"] %) tx-data))
(is (not-any? #(= [:db/retract [:block/uuid page-uuid] :block/title "Page"] %) tx-data)))))

2
deps/db-sync/test/logseq/db_sync/test_runner.cljs vendored
View File

@@ -9,9 +9,11 @@
[logseq.db-sync.node-adapter-test]
[logseq.db-sync.node-config-test]
[logseq.db-sync.node-server-test]
[logseq.db-sync.normalize-test]
[logseq.db-sync.platform-test]
[logseq.db-sync.worker-auth-test]
[logseq.db-sync.worker-handler-assets-test]
[logseq.db-sync.worker-handler-index-test]
[logseq.db-sync.worker-handler-sync-test]
[logseq.db-sync.worker-handler-ws-test]
[logseq.db-sync.worker-routes-test]

18
deps/db-sync/test/logseq/db_sync/worker_auth_test.cljs vendored
View File

@@ -56,3 +56,21 @@
(p/catch (fn [error]
(is (= "jwks" (ex-message error)))
(done)))))))
(deftest auth-claims-expired-jwt-short-circuits-verification-test
(async done
(let [expired-token "eyJhbGciOiJSUzI1NiJ9.eyJleHAiOjEsInN1YiI6InUxIn0.signature"
request (js/Request. "http://localhost/graphs"
#js {:headers #js {"authorization" (str "Bearer " expired-token)}})
verify-called? (atom false)]
(-> (p/with-redefs [authorization/verify-jwt
(fn [_token _env]
(reset! verify-called? true)
(p/rejected (ex-info "should-not-be-called" {})))]
(p/let [claims (auth/auth-claims request #js {})]
(is (nil? claims))
(is (false? @verify-called?))))
(p/then (fn [] (done)))
(p/catch (fn [error]
(is false (str error))
(done)))))))

53
deps/db-sync/test/logseq/db_sync/worker_handler_index_test.cljs vendored Normal file
View File

@@ -0,0 +1,53 @@
(ns logseq.db-sync.worker-handler-index-test
(:require [cljs.test :refer [async deftest is]]
[logseq.db-sync.index :as index]
[logseq.db-sync.worker.auth :as auth]
[logseq.db-sync.worker.handler.index :as index-handler]
[promesa.core :as p]))
(deftest graph-access-response-with-timing-caches-result-test
(async done
(let [request (js/Request. "http://localhost/sync/graph-1"
#js {:headers #js {"authorization" "Bearer token-cache-hit"}})
env #js {"DB" #js {}}
auth-count (atom 0)
query-count (atom 0)]
(-> (p/with-redefs [auth/auth-claims (fn [_request _env]
(swap! auth-count inc)
(p/resolved #js {"sub" "user-1"}))
index/<user-has-access-to-graph? (fn [_db _graph-id _user-id]
(swap! query-count inc)
(p/resolved true))]
(p/let [first-result (index-handler/graph-access-response-with-timing request env "graph-1")
second-result (index-handler/graph-access-response-with-timing request env "graph-1")]
(is (= 200 (.-status (:response first-result))))
(is (= 200 (.-status (:response second-result))))
(is (= 1 @auth-count))
(is (= 1 @query-count))
(is (false? (get-in first-result [:timing :cache-hit?])))
(is (true? (get-in second-result [:timing :cache-hit?])))))
(p/then (fn []
(done)))
(p/catch (fn [error]
(is false (str error))
(done)))))))
(deftest graph-access-response-with-timing-does-not-upsert-user-test
(async done
(let [request (js/Request. "http://localhost/sync/graph-2"
#js {:headers #js {"authorization" "Bearer token-no-upsert"}})
env #js {"DB" #js {}}]
(-> (p/with-redefs [auth/auth-claims (fn [_request _env]
(p/resolved #js {"sub" "user-2"}))
index/<user-upsert! (fn [& _]
(throw (ex-info "should-not-upsert" {})))
index/<user-has-access-to-graph? (fn [_db _graph-id _user-id]
(p/resolved true))]
(p/let [result (index-handler/graph-access-response-with-timing request env "graph-2")]
(is (= 200 (.-status (:response result))))
(is (= true (get-in result [:timing :access-ok?])))))
(p/then (fn []
(done)))
(p/catch (fn [error]
(is false (str error))
(done)))))))

3
deps/db-sync/worker/wrangler.toml vendored
View File

@@ -18,6 +18,9 @@ binding = "CF_VERSION_METADATA"
[observability]
enabled = true
[observability.logs]
invocation_logs = false
[[durable_objects.bindings]]
name = "LOGSEQ_SYNC_DO"
class_name = "SyncDO"

19
deps/db/src/logseq/db.cljs vendored
View File

@@ -670,9 +670,11 @@
(d/q '[:find ?e ?a
:in $ ?v
:where
[?e ?a ?v]
[?c :logseq.property.class/enable-bidirectional? ?c-enable?]
[(true? ?c-enable?)]
[?ea :logseq.property/classes ?c]
[?ea :db/ident ?a]
[?ea :logseq.property/classes]]
[?e ?a ?v]]
db
v))
@@ -687,10 +689,19 @@
(fn [acc class-id entity]
(if class-id
(update acc class-id (fnil conj #{}) entity)
acc))]
acc))
*attr->bidirectional? (volatile! {})
bidirectional-property-attr-cached?
(fn [attr]
(let [cache @*attr->bidirectional?]
(if (contains? cache attr)
(get cache attr)
(let [result (bidirectional-property-attr? db attr)]
(vswap! *attr->bidirectional? assoc attr result)
result))))]
(->> (get-ea-by-v db target-id)
(keep (fn [[e a]]
(when (bidirectional-property-attr? db a)
(when (bidirectional-property-attr-cached? a)
(when-let [entity (d/entity db e)]
(when (and (not= (:db/id entity) target-id)
(not (entity-util/class? entity))

74
deps/db/src/logseq/db/common/normalize.cljs vendored
View File

@@ -92,34 +92,46 @@
(defn normalize-tx-data
[db-after db-before tx-data]
(->> tx-data
remove-conflict-datoms
(replace-attr-retract-with-retract-entity db-after)
sort-datoms
(keep
(fn [d]
(if (= (count d) 5)
(let [[e a v t added] d
retract? (not added)]
(when-not (and retract?
(contains? #{:block/created-at :block/updated-at :block/title} a))
(let [e' (if retract?
(eid->lookup db-before e)
(or (eid->lookup db-before e)
(eid->tempid db-after e)))
v' (if (and (integer? v)
(pos? v)
(or (= :db.type/ref (:db/valueType (d/entity db-after a)))
(= :db.type/ref (:db/valueType (d/entity db-before a)))))
(if retract?
(eid->lookup db-before v)
(or (eid->lookup db-before v)
(eid->tempid db-after v)))
v)]
(when (and (some? e') (some? v'))
(if added
[:db/add e' a v' t]
[:db/retract e' a v' t])))))
d)))
(remove-retract-entity-ref db-after)
distinct))
(let [title-updated-entities
(->> tx-data
(keep (fn [d]
(when (= (count d) 5)
(let [[e a _v _t added] d]
(when (and added (= :block/title a))
e)))))
set)]
(->> tx-data
remove-conflict-datoms
(replace-attr-retract-with-retract-entity db-after)
sort-datoms
(keep
(fn [d]
(if (= (count d) 5)
(let [[e a v t added] d
retract? (not added)
drop-retract?
(and retract?
(or (contains? #{:block/created-at :block/updated-at} a)
(and (= :block/title a)
(contains? title-updated-entities e))))]
(when-not drop-retract?
(let [e' (if retract?
(eid->lookup db-before e)
(or (eid->lookup db-before e)
(eid->tempid db-after e)))
v' (if (and (integer? v)
(pos? v)
(or (= :db.type/ref (:db/valueType (d/entity db-after a)))
(= :db.type/ref (:db/valueType (d/entity db-before a)))))
(if retract?
(eid->lookup db-before v)
(or (eid->lookup db-before v)
(eid->tempid db-after v)))
v)]
(when (and (some? e') (some? v'))
(if added
[:db/add e' a v' t]
[:db/retract e' a v' t])))))
d)))
(remove-retract-entity-ref db-after)
distinct)))

15
deps/db/src/logseq/db/sqlite/build.cljs vendored
View File

@@ -411,7 +411,7 @@
[:block/title :string]
[:build/children {:optional true} [:vector [:ref ::block]]]
[:build/properties {:optional true} User-properties]
[:build/tags {:optional true} [:vector Class]]
[:build/tags {:optional true} [:or [:set Class] [:vector Class]]]
[:build/keep-uuid? {:optional true} :boolean]]}}
[:page [:and
[:map
@@ -419,7 +419,7 @@
[:block/title {:optional true} :string]
[:build/journal {:optional true} :int]
[:build/properties {:optional true} User-properties]
[:build/tags {:optional true} [:vector Class]]
[:build/tags {:optional true} [:or [:set Class] [:vector Class]]]
[:build/keep-uuid? {:optional true} :boolean]]
[:fn {:error/message ":block/title, :block/uuid or :build/journal required"
:error/path [:block/title]}
@@ -434,13 +434,14 @@
[:build/properties {:optional true} User-properties]
[:build/properties-ref-types {:optional true}
[:map-of :keyword :keyword]]
;; TODO: Make this respect :block/order or allow :set
[:build/closed-values
{:optional true}
[:vector [:map
[:value [:or :string :double]]
[:uuid {:optional true} :uuid]
[:icon {:optional true} :map]]]]
[:build/property-classes {:optional true} [:vector Class]]
[:build/property-classes {:optional true} [:or [:set Class] [:vector Class]]]
[:build/keep-uuid? {:optional true} :boolean]]])
(def Classes
@@ -448,13 +449,19 @@
Class
[:map
[:build/properties {:optional true} User-properties]
[:build/class-extends {:optional true} [:vector Class]]
[:build/class-extends {:optional true} [:or [:set Class] [:vector Class]]]
[:build/class-properties {:optional true} [:vector Property]]
[:build/keep-uuid? {:optional true} :boolean]]])
(def Options
"Main malli schema that validates a sqlite.build EDN map. If an inner schema
uses :vector e.g. :blocks, it's to preserve :block/order-ing for that node's
attribute. If an inner schema uses :vector or :set e.g. :build/class-extends,
it's to indicate it is order-less and also allow users to write the more
familiar vector syntax"
[:map
{:closed true}
;; TODO: Make this respect :block/order or allow :set
[:pages-and-blocks {:optional true} [:vector Page-blocks]]
[:properties {:optional true} Properties]
[:classes {:optional true} Classes]

65
deps/db/src/logseq/db/sqlite/export.cljs vendored
View File

@@ -27,7 +27,7 @@
;; These classes are redundant as :build/journal is enough for Journal and Page
;; is implied by being in :pages-and-blocks
(remove #{:logseq.class/Page :logseq.class/Journal})
vec))
set))
(defn- block-title
"Get an entity's original title"
@@ -147,7 +147,7 @@
(and (not shallow-copy?) include-alias? (:block/alias property))
(assoc :block/alias (set (map #(vector :block/uuid (:block/uuid %)) (:block/alias property))))
(and (not shallow-copy?) (:logseq.property/classes property))
(assoc :build/property-classes (mapv :db/ident (:logseq.property/classes property)))
(assoc :build/property-classes (set (map :db/ident (:logseq.property/classes property))))
(seq closed-values)
(assoc :build/closed-values
(mapv #(cond-> {:value (db-property/property-value-content %)
@@ -194,7 +194,7 @@
(:logseq.property.class/extends class-ent)
(not= [:logseq.class/Root] (mapv :db/ident (:logseq.property.class/extends class-ent))))
(assoc :build/class-extends
(mapv :db/ident (:logseq.property.class/extends class-ent)))))
(set (map :db/ident (:logseq.property.class/extends class-ent))))))
(defn block-property-value? [%]
(and (map? %) (:build/property-value %)))
@@ -766,6 +766,33 @@
(remove #(= :logseq.kv/schema-version (:db/ident %)))
vec))
(defn- build-property-history
"Builds property history. Always include timestamps regardless of :include-timestamps? because
timestamps are a necessary part of history"
[db]
(->> (d/q '[:find [(pull ?b [:block/uuid
:block/created-at
{:logseq.property.history/block [:block/uuid]}
{:logseq.property.history/property [:db/ident]}
{:logseq.property.history/ref-value [:db/ident :block/uuid]}
:logseq.property.history/scalar-value]) ...]
:where [?b :logseq.property.history/block]] db)
(map (fn [history]
(cond-> (-> history
(update :logseq.property.history/block
(fn [m] [:block/uuid (:block/uuid m)]))
(update :logseq.property.history/property :db/ident)
(update :logseq.property.history/ref-value
(fn [m]
(if (:db/ident m)
(:db/ident m)
[:block/uuid (:block/uuid m)]))))
(nil? (:logseq.property.history/ref-value history))
(dissoc :logseq.property.history/ref-value)
(not (contains? history :logseq.property.history/scalar-value))
(dissoc :logseq.property.history/scalar-value))))
set))
(defn remove-uuids-if-not-ref [export-map all-ref-uuids]
(let [remove-uuid-if-not-ref (partial remove-uuid-if-not-ref-given-uuids all-ref-uuids)]
(-> export-map
@@ -825,7 +852,16 @@
graph-export (if (seq (:exclude-namespaces options))
(assoc graph-export* ::auto-include-namespaces (:exclude-namespaces options))
graph-export*)
all-ref-uuids (set/union content-ref-uuids ontology-pvalue-uuids (:pvalue-uuids pages-export))
property-history (build-property-history db)
property-history-ref-uuids
(->> property-history
(mapcat (fn [history]
(keep #(when (vector? %) (second %))
[(:logseq.property.history/block history)
(:logseq.property.history/ref-value history)])))
set)
all-ref-uuids (set/union content-ref-uuids ontology-pvalue-uuids (:pvalue-uuids pages-export)
property-history-ref-uuids)
files (when-not exclude-files? (build-graph-files db options))
kv-values (build-kv-values db)
;; Remove all non-ref uuids after all nodes are built.
@@ -837,7 +873,9 @@
(not exclude-files?)
(assoc ::graph-files files)
true
(assoc ::kv-values kv-values))))
(assoc ::kv-values kv-values)
true
(assoc ::property-history property-history))))
(defn- find-undefined-classes-and-properties [{:keys [classes properties pages-and-blocks]}]
(let [referenced-classes
@@ -1074,6 +1112,7 @@
* ::block - Block map for a :block export
* ::graph-files - Vec of files for a :graph export
* ::kv-values - Vec of :kv/value maps for a :graph export
* ::property-history - Set of property history blocks for a :graph export
* ::auto-include-namespaces - A set of parent namespaces to include from properties and classes
for a :graph export. See :exclude-namespaces in build-graph-export for a similar option
* ::import-options - A map of options that alters importing behavior. Has the following keys:
@@ -1101,7 +1140,8 @@
(if (= :graph (::export-type export-map''))
(-> (sqlite-build/build-blocks-tx (remove-namespaced-keys export-map''))
(assoc :misc-tx (vec (concat (::graph-files export-map'')
(::kv-values export-map'')))))
(::kv-values export-map'')
(::property-history export-map'')))))
(sqlite-build/build-blocks-tx (remove-namespaced-keys export-map''))))))
(defn create-conn
@@ -1133,22 +1173,13 @@
{:error (str "The exported EDN is unexpectedly invalid: " (pr-str (ex-message e)))})))
(defn- prepare-export-to-diff
"This prepares a graph's exported edn to be diffed with another"
"Prepare a graph's exported edn to be diffed with another"
[m]
(-> m
;; TODO: Fix order of these :build/* keys
(update :classes update-vals (fn [m]
(cond-> m
(:build/class-extends m)
(update :build/class-extends sort))))
(update :properties update-vals (fn [m]
(cond-> m
(:build/property-classes m)
(update :build/property-classes sort))))
(update ::kv-values
(fn [kvs]
(->> kvs
;; Ignore extra metadata that a copied graph can add
;; This varies per copied graph so ignore it
(remove #(#{:logseq.kv/import-type :logseq.kv/imported-at :logseq.kv/local-graph-uuid}
(:db/ident %)))
(sort-by :db/ident)

120
deps/db/test/logseq/db/sqlite/export_test.cljs vendored
View File

@@ -61,7 +61,8 @@
imported-page (export-page-and-import-to-another-graph conn conn2 page-title)
updated-page (db-test/find-page-by-title @conn2 page-title)
expected-page-and-blocks
(update-in (:pages-and-blocks original-data) [0 :blocks] transform-expected-blocks)
(-> (:pages-and-blocks original-data)
(update-in [0 :blocks] transform-expected-blocks))
filter-imported-page (if build-journal
#(= build-journal (get-in % [:page :build/journal]))
#(= (get-in % [:page :block/title]) page-title))]
@@ -91,8 +92,7 @@
imported-graph))
(defn- expand-properties
"Add default values to properties of an input export map to test against a
db-based export map"
"Modify given properties so that they match properties exported from the imported graph"
[properties]
(->> properties
(map (fn [[k m]]
@@ -100,20 +100,23 @@
(cond->
(merge {:db/cardinality :db.cardinality/one}
m)
(:build/property-classes m)
(update :build/property-classes set)
(not (:block/title m))
(assoc :block/title (name k)))]))
(into {})))
(defn- expand-classes
"Add default values to classes of an input export map to test against a
db-based export map"
"Modify given classes so that they match classes exported from the imported graph"
[classes]
(->> classes
(map (fn [[k m]]
[k
(cond-> m
(not (:block/title m))
(assoc :block/title (name k)))]))
(assoc :block/title (name k))
(:build/class-extends m)
(update :build/class-extends set))]))
(into {})))
(def sort-pages-and-blocks sqlite-export/sort-pages-and-blocks)
@@ -159,7 +162,7 @@
[{:page {:block/title "page1"}
:blocks [{:block/title "export"
:build/properties {:user.property/default-many #{"foo" "bar" "baz"}}
:build/tags [:user.class/MyClass]}
:build/tags #{:user.class/MyClass}}
{:block/title "import"}]}]}
conn (db-test/create-conn-with-blocks original-data)
imported-block (export-block-and-import-to-another-block conn conn "export" "import")]
@@ -184,7 +187,7 @@
[{:page {:block/title "page1"}
:blocks [{:block/title "export"
:build/properties {:user.property/num-many #{3 6 9}}
:build/tags [:user.class/MyClass]}]}]}
:build/tags #{:user.class/MyClass}}]}]}
conn (db-test/create-conn-with-blocks original-data)
conn2 (db-test/create-conn-with-blocks
{:pages-and-blocks [{:page {:block/title "page2"}
@@ -249,10 +252,10 @@
{:block/title "b1ab"}]}
{:block/title "b1b"}]}
{:block/title "b2"
:build/tags [:user.class/MyClass]}
:build/tags #{:user.class/MyClass}}
{:block/title "some task"
:build/properties {:logseq.property/status :logseq.property/status.doing}
:build/tags [:logseq.class/Task]}]}]}
:build/tags #{:logseq.class/Task}}]}]}
conn (db-test/create-conn-with-blocks original-data)
conn2 (db-test/create-conn)
imported-page (export-page-and-import-to-another-graph conn conn2 "page1")]
@@ -388,9 +391,9 @@
:pages-and-blocks
[{:page {:block/title "page1"
:build/properties {:user.property/p1 "woot"}
:build/tags [:user.class/ChildClass]}
:build/tags #{:user.class/ChildClass}}
:blocks [{:block/title "child object"
:build/tags [:user.class/ChildClass2]}]}]}
:build/tags #{:user.class/ChildClass2}}]}]}
conn (db-test/create-conn-with-blocks original-data)
conn2 (db-test/create-conn)
imported-page (export-page-and-import-to-another-graph conn conn2 "page1")]
@@ -471,14 +474,14 @@
:build/properties {:user.property/date [:build/page {:build/journal 20250203}]}}
{:block/title "node block"
:build/properties {:user.property/node #{[:build/page {:block/title "page object"
:build/tags [:user.class/MyClass]}]
:build/tags #{:user.class/MyClass}}]
[:block/uuid block-object-uuid]
:logseq.class/Task}}}
{:block/title "map block"
:build/properties {:user.property/map {:foo :bar :num 2}}}]}
{:page {:block/title "Blocks"}
:blocks [{:block/title "myclass object"
:build/tags [:user.class/MyClass]
:build/tags #{:user.class/MyClass}
:block/uuid block-object-uuid
:build/keep-uuid? true}]}]}
conn (db-test/create-conn-with-blocks original-data)
@@ -570,7 +573,7 @@
:build/properties {:user.property/p1 "ok"}
:build/children [{:block/title "b2"}]}
{:block/title "b3"
:build/tags [:user.class/class1]
:build/tags #{:user.class/class1}
:build/children [{:block/title "b4"}]}]}
{:page {:block/title "page2"}
:blocks [{:block/title "dont export"}]}]}
@@ -655,7 +658,7 @@
{:block/title "b2" :build/properties {:user.property/node #{[:block/uuid page-object-uuid]}}}
{:block/title "b3" :build/properties {:user.property/node #{[:block/uuid page-object-uuid]}}}
{:block/title "Example advanced query",
:build/tags [:logseq.class/Query],
:build/tags #{:logseq.class/Query},
:build/properties
{:logseq.property/query
{:build/property-value :block
@@ -668,24 +671,24 @@
{:user.property/url
{:build/property-value :block
:block/title "https://example.com"
:build/tags [:user.class/MyClass]}}}]}
:build/tags #{:user.class/MyClass}}}}]}
{:page {:block/title "page object"
:block/uuid page-object-uuid
:build/keep-uuid? true}
:blocks []}
{:page {:block/title "page2" :build/tags [:user.class/MyClass2]}
{:page {:block/title "page2" :build/tags #{:user.class/MyClass2}}
:blocks [{:block/title "hola" :block/uuid internal-block-uuid :build/keep-uuid? true}
{:block/title "myclass object 1"
:build/tags [:user.class/MyClass]
:build/tags #{:user.class/MyClass}
:block/uuid block-pvalue-uuid
:build/keep-uuid? true}
(cond-> {:block/title "myclass object 2"
:build/tags [:user.class/MyClass]}
:build/tags #{:user.class/MyClass}}
(not exclude-namespaces?)
(merge {:block/uuid property-pvalue-uuid
:build/keep-uuid? true}))
{:block/title "myclass object 3"
:build/tags [:user.class/MyClass]
:build/tags #{:user.class/MyClass}
:block/uuid page-pvalue-uuid
:build/keep-uuid? true}
{:block/title "ref blocks"
@@ -836,6 +839,69 @@
(sqlite-export/diff-exports export-map export-map2))
"No diff between original export and export after importing into a new graph")))
(deftest ^:long import-graph-preserves-property-history
(let [now (common-util/time-ms)
original-data
{:properties {:user.property/num {:logseq.property/type :number}
:user.property/node {:logseq.property/type :node
:db/cardinality :db.cardinality/many}}
:pages-and-blocks [{:page {:block/title "page1"}
:blocks [{:block/title "num block"
:build/properties {:user.property/num 44}}
{:block/title "status block"
:build/properties {:logseq.property/status :logseq.property/status.doing}}
{:block/title "node block"}
{:block/title "object 1"}
{:block/title "object 2"}]}]}
conn (db-test/create-conn-with-import-map original-data)
num-block (db-test/find-block-by-content @conn "num block")
status-block (db-test/find-block-by-content @conn "status block")
node-block (db-test/find-block-by-content @conn "node block")
original-property-history
[{:block/uuid (random-uuid)
:block/created-at now
:logseq.property.history/block [:block/uuid (:block/uuid num-block)]
:logseq.property.history/property :user.property/num
:logseq.property.history/scalar-value 42}
{:block/uuid (random-uuid)
:block/created-at (+ now 1000)
:logseq.property.history/block [:block/uuid (:block/uuid num-block)]
:logseq.property.history/property :user.property/num
:logseq.property.history/scalar-value 44}
{:block/uuid (random-uuid)
:block/created-at now
:logseq.property.history/block [:block/uuid (:block/uuid node-block)]
:logseq.property.history/property :user.property/node
:logseq.property.history/ref-value [:block/uuid (:block/uuid (db-test/find-block-by-content @conn "object 1"))]}
{:block/uuid (random-uuid)
:block/created-at (+ now 1000)
:logseq.property.history/block [:block/uuid (:block/uuid node-block)]
:logseq.property.history/property :user.property/node
:logseq.property.history/ref-value [:block/uuid (:block/uuid (db-test/find-block-by-content @conn "object 2"))]}
{:block/uuid (random-uuid)
:block/created-at now
:logseq.property.history/block [:block/uuid (:block/uuid status-block)]
:logseq.property.history/property :logseq.property/status
:logseq.property.history/ref-value :logseq.property/status.todo}
{:block/uuid (random-uuid)
:block/created-at (+ now 1000)
:logseq.property.history/block [:block/uuid (:block/uuid status-block)]
:logseq.property.history/property :logseq.property/status
:logseq.property.history/ref-value :logseq.property/status.doing}]
_ (d/transact! conn original-property-history)
export-map (sqlite-export/build-export @conn {:export-type :graph})
valid-result (sqlite-export/validate-export export-map)
_ (assert (not (:error valid-result)) "No error when importing export-map into new graph")
_ (validate-db (:db valid-result))
export-map2 (sqlite-export/build-export (:db valid-result) {:export-type :graph})
property-history (::sqlite-export/property-history export-map2)]
(is (= nil
(sqlite-export/diff-exports export-map export-map2))
"No diff between original export and export after importing into a new graph")
;; (cljs.pprint/pprint (clojure.data/diff original-property-history property-history))
(is (= (set original-property-history) property-history)
"Original property history equals exported property history")))
(deftest import-graph-with-different-property-value-cases
(let [pvalue-uuid1 (random-uuid)
original-data
@@ -849,7 +915,7 @@
:blocks [{:block/title "block with pvalue that has :build/tags"
:build/properties {:user.property/default {:build/property-value :block
:block/title "tags pvalue"
:build/tags [:user.class/C1]}}}
:build/tags #{:user.class/C1}}}}
{:block/title "block with pvalue that has a view"
:build/properties {:user.property/default {:build/property-value :block
:block/title "view pvalue"
@@ -861,7 +927,7 @@
#{"yep"
{:build/property-value :block
:block/title ":many pvalue"
:build/tags [:user.class/C1]}}}}]}
:build/tags #{:user.class/C1}}}}}]}
{:page {:block/title "$$$views2"}
:blocks [{:block/title "Unlinked references",
:build/properties
@@ -967,25 +1033,25 @@
:blocks [{:block/title "asset block"
:block/uuid asset-uuid
:build/keep-uuid? true
:build/tags [:logseq.class/Asset]
:build/tags #{:logseq.class/Asset}
:build/properties {:logseq.property.asset/type "pdf"
:logseq.property.asset/checksum "abc"
:logseq.property.asset/size 42}}
{:block/title "annotation block"
:build/tags [:logseq.class/Pdf-annotation]
:build/tags #{:logseq.class/Pdf-annotation}
:build/properties {:logseq.property/asset [:block/uuid asset-uuid]}}]}
{:page {:block/title "page2"}
:blocks [{:block/title "asset image block"
:block/uuid asset2-uuid
:build/keep-uuid? true
:build/tags [:logseq.class/Asset]
:build/tags #{:logseq.class/Asset}
:build/properties {:logseq.property.asset/type "png"
:logseq.property.asset/checksum "img-checksum"
:logseq.property.asset/width 100
:logseq.property.asset/height 200
:logseq.property.asset/size 300}}
{:block/title "annotation with image"
:build/tags [:logseq.class/Pdf-annotation]
:build/tags #{:logseq.class/Pdf-annotation}
:build/properties {:logseq.property.pdf/hl-image [:block/uuid asset2-uuid]}}]}]}
conn (db-test/create-conn-with-blocks original-data)
conn2 (db-test/create-conn)

55
deps/db/test/logseq/db_test.cljs vendored
View File

@@ -148,3 +148,58 @@
(is (= "People" (:title (first results))))
(is (= ["Alice"]
(map :block/title (:entities (first results))))))))
(defn- bidirectional-perf-conn
[n property-titles]
(let [target-page {:page {:block/title "Target"}}
properties (into {}
(map (fn [property-title]
[property-title {:logseq.property/type :node
:build/property-classes [:Person]}]))
property-titles)
person-properties (into {}
(map (fn [property-title]
[property-title [:build/page {:block/title "Target"}]]))
property-titles)
pages (vec (concat [target-page]
(map (fn [i]
{:page {:block/title (str "Person " i)
:build/tags [:Person]
:build/properties person-properties}})
(range n))))]
(db-test/create-conn-with-blocks
{:properties properties
:classes {:Person {:build/properties {:logseq.property.class/enable-bidirectional? true}}}
:pages-and-blocks pages})))
(deftest ^:long get-bidirectional-properties-performance-single-property
(testing "attribute lookups scale with unique properties, not entities"
(let [conn (bidirectional-perf-conn 400 [:friend])
target-id (:db/id (db-test/find-page-by-title @conn "Target"))
original-entity d/entity
attr-lookups (atom 0)
results (with-redefs [d/entity (fn [db eid]
(when (keyword? eid)
(swap! attr-lookups inc))
(original-entity db eid))]
(ldb/get-bidirectional-properties @conn target-id))]
(is (= 1 (count results)))
(is (= 400 (count (:entities (first results)))))
(is (<= @attr-lookups 8)
(str "expected bounded attr lookups, got " @attr-lookups)))))
(deftest ^:long get-bidirectional-properties-performance-multi-property
(testing "attribute lookups stay bounded with multiple matching properties"
(let [conn (bidirectional-perf-conn 300 [:friend :colleague])
target-id (:db/id (db-test/find-page-by-title @conn "Target"))
original-entity d/entity
attr-lookups (atom 0)
results (with-redefs [d/entity (fn [db eid]
(when (keyword? eid)
(swap! attr-lookups inc))
(original-entity db eid))]
(ldb/get-bidirectional-properties @conn target-id))]
(is (= 1 (count results)))
(is (= 300 (count (:entities (first results)))))
(is (<= @attr-lookups 12)
(str "expected bounded attr lookups, got " @attr-lookups)))))

18
scripts/src/logseq/tasks/db_graph/create_graph_with_schema_org.cljs
View File

@@ -334,23 +334,7 @@
:desc "Verbose mode"}})
(defn- write-export-file [db]
(let [export-map* (sqlite-export/build-export db {:export-type :graph-ontology})
;; Modify export to provide stable diff like prepare-export-to-diff
;; TODO: Remove this when prepare-export-to-diff TODO is done i.e.
;; when export has stable sort order for these keys
export-map (-> export-map*
(update :classes update-vals
(fn [m]
(cond-> m
(:build/class-extends m)
(update :build/class-extends (comp vec sort))
(:build/class-properties m)
(update :build/class-properties (comp vec sort)))))
(update :properties update-vals
(fn [m]
(cond-> m
(:build/property-classes m)
(update :build/property-classes (comp vec sort))))))]
(let [export-map (sqlite-export/build-export db {:export-type :graph-ontology})]
(fs/writeFileSync "schema.edn"
(with-out-str (pprint/pprint export-map)))))

10
src/main/frontend/components/dnd.cljs
View File

@@ -51,12 +51,12 @@
sensors (useSensors (useSensor MouseSensor (bean/->js {:activationConstraint {:distance 8}})))
dnd-opts {:sensors sensors
:collisionDetection closestCenter
:onDragStart (fn [event]
:onDragStart (fn [^js event]
(when-not (state/editing?)
(set-active-id (.-id (.-active event)))))
:onDragEnd (fn [event]
(let [active-id (.-id (.-active event))
over-id (.-id (.-over event))]
(set-active-id (.-id ^js (.-active event)))))
:onDragEnd (fn [^js event]
(let [active-id (.-id ^js (.-active event))
over-id (.-id ^js (.-over event))]
(when active-id
(when-not (= active-id over-id)
(let [old-index (.indexOf ids active-id)

20
src/main/frontend/components/query.cljs
View File

@@ -20,6 +20,19 @@
(when (seq queries)
(boolean (some #(= % title) (map :title queries))))))
(defn- grouped-by-page-result?
[result group-by-page?]
(let [first-group (first result)
first-page (first first-group)
first-block (first (second first-group))]
(boolean
(and group-by-page?
(seq result)
(coll? first-group)
(or (:block/name first-page)
(:db/id first-page))
(:block/uuid first-block)))))
(rum/defcs custom-query-inner < rum/static
[state {:keys [dsl-query?] :as config} {:keys [query breadcrumb-show?]}
{:keys [query-error-atom
@@ -30,12 +43,7 @@
(let [{:keys [->hiccup]} config
*query-error query-error-atom
only-blocks? (:block/uuid (first result))
blocks-grouped-by-page? (and group-by-page?
(seq result)
(coll? (first result))
(:block/name (ffirst result))
(:block/uuid (first (second (first result))))
true)]
blocks-grouped-by-page? (grouped-by-page-result? result group-by-page?)]
(if @*query-error
(do
(log/error :exception @*query-error)

13
src/main/frontend/db/model.cljs
View File

@@ -98,11 +98,18 @@ independent of format as format specific heading characters are stripped"
(remove nil?))
pages (when (seq pages-ids)
(db-utils/pull-many '[:db/id :block/name :block/title :block/journal-day] pages-ids))
pages-map (reduce (fn [acc p] (assoc acc (:db/id p) p)) {} pages)
pages-map (reduce (fn [acc p]
(if (map? p)
(assoc acc (:db/id p) p)
acc))
{}
pages)
blocks (map
(fn [block]
(assoc block :block/page
(get pages-map (:db/id (:block/page block)))))
(assoc block
:block/page
(or (get pages-map (:db/id (:block/page block)))
(:block/page block))))
blocks)]
blocks))

8
src/main/frontend/handler/db_based/rtc_background_tasks.cljs
View File

@@ -32,14 +32,14 @@
(c.m/<? (rtc-handler/<rtc-start! (state/get-current-repo) :stop-before-start? false)))))))
(run-background-task-when-not-publishing
;; stop rtc when [graph-switch user-logout]
;; stop rtc when [user-logout]
::stop-rtc-if-needed
(m/reduce
(constantly nil)
(m/ap
(let [logout-or-graph-switch (m/?> rtc-flows/logout-or-graph-switch-flow)]
(log/info :try-to-stop-rtc-if-needed logout-or-graph-switch)
(c.m/<? (rtc-handler/<rtc-stop!))))))
(m/?> rtc-flows/logout-flow)
(log/info :try-to-stop-rtc-if-needed :logout)
(c.m/<? (rtc-handler/<rtc-stop!)))))
(run-background-task-when-not-publishing
;; auto-start rtc when [user-login graph-switch]

14
src/main/frontend/handler/db_based/rtc_flows.cljs
View File

@@ -71,14 +71,10 @@ conditions:
{:graph-uuid graph-uuid :t (common-util/time-ms)})))))
(c.m/throttle 5000)))
(def logout-or-graph-switch-flow
(c.m/mix
(m/eduction
(filter #(= :logout %))
flows/current-login-user-flow)
(m/eduction
(keep (fn [repo] (when repo :graph-switch)))
flows/current-repo-flow)))
(def logout-flow
(m/eduction
(filter #(= :logout %))
flows/current-login-user-flow))
(def ^:private *rtc-start-trigger (atom nil))
(defn trigger-rtc-start
@@ -153,4 +149,4 @@ conditions:
(apply c.m/mix)
(m/latest vector flows/current-login-user-flow)
(m/eduction (keep (fn [[current-user trigger-event]] (when current-user trigger-event))))
(c.m/debounce 200)))
(c.m/debounce 50)))

56
src/main/frontend/handler/db_based/sync.cljs
View File

@@ -185,27 +185,65 @@
[repo]
(some #(= repo (:url %)) (state/get-rtc-graphs)))
(defn- graph-has-local-rtc-id?
[repo]
(boolean (some-> (db/get-db repo)
ldb/get-graph-rtc-uuid)))
(defn- remote-graphs-unknown?
[]
(not= false (:rtc/loading-graphs? @state/state)))
(defn- should-start-rtc?
[repo]
(or (graph-in-remote-list? repo)
;; During startup, remote graph list might not be fetched yet.
;; If local DB already has graph UUID, start optimistically to reduce cold-start latency.
(and (remote-graphs-unknown?)
(graph-has-local-rtc-id? repo))))
(defn- normalize-graph-e2ee?
[graph-e2ee?]
(if (nil? graph-e2ee?)
true
(true? graph-e2ee?)))
(defn <rtc-start!
[repo & {:keys [_stop-before-start?] :as _opts}]
(if (graph-in-remote-list? repo)
(do
(log/info :db-sync/start {:repo repo})
(state/<invoke-db-worker :thread-api/db-sync-start repo))
(do
(log/info :db-sync/skip-start {:repo repo :reason :graph-not-in-remote-list})
(p/resolved nil))))
(defn- <wait-for-db-worker-ready!
[]
(if @state/*db-worker
(p/resolved true)
(let [ready (p/deferred)
watch-key (keyword "frontend.handler.db-based.sync"
(str "wait-db-worker-ready-" (random-uuid)))]
(add-watch state/*db-worker watch-key
(fn [_ _ _ worker]
(when worker
(remove-watch state/*db-worker watch-key)
(p/resolve! ready true))))
;; If worker becomes ready between the initial check and add-watch.
(when @state/*db-worker
(remove-watch state/*db-worker watch-key)
(p/resolve! ready true))
ready)))
(defn <rtc-stop!
[]
(log/info :db-sync/stop true)
(state/<invoke-db-worker :thread-api/db-sync-stop))
(defn <rtc-start!
[repo & {:keys [_stop-before-start?] :as _opts}]
(p/let [_ (<wait-for-db-worker-ready!)]
(if (should-start-rtc? repo)
(do
(log/info :db-sync/start {:repo repo})
(state/<invoke-db-worker :thread-api/db-sync-start repo))
(do
(log/info :db-sync/skip-start {:repo repo :reason :graph-not-in-remote-list
:remote-graphs-loading? (:rtc/loading-graphs? @state/state)
:has-local-rtc-id? (graph-has-local-rtc-id? repo)})
(<rtc-stop!)))))
(defonce ^:private debounced-update-presence
(util/debounce
(fn [editing-block-uuid]

5
src/main/frontend/handler/editor.cljs
View File

@@ -1398,7 +1398,10 @@
:bottom? true
:sibling? (= edit-block target)
:replace-empty-target? true}))
(map (fn [b] (db/entity [:block/uuid (:block/uuid b)])) blocks)))))
(p/let [blocks (map (fn [b] (db/entity [:block/uuid (:block/uuid b)])) blocks)]
(when-let [block (some (fn [block] (when (= (:block/uuid block) (:block/uuid edit-block)) block)) blocks)]
(edit-block! block :max))
blocks)))))
(def insert-command! editor-common-handler/insert-command!)

37
src/main/frontend/handler/user.cljs
View File

@@ -37,6 +37,13 @@
(js->clj :keywordize-keys true)
(update :cognito:username decode-username)))
(defn- parse-jwt-safe
[jwt]
(try
(parse-jwt jwt)
(catch :default _
nil)))
(defn- expired? [parsed-jwt]
(some->
(* 1000 (:exp parsed-jwt))
@@ -191,12 +198,34 @@
"Refresh id-token&access-token, pull latest repos, returns nil when tokens are not available."
[]
(println "restore-tokens-from-localstorage")
(let [refresh-token (js/localStorage.getItem "refresh-token")]
(when refresh-token
(let [refresh-token (js/localStorage.getItem "refresh-token")
id-token (js/localStorage.getItem "id-token")
access-token (js/localStorage.getItem "access-token")
restored-from-cache?
(boolean
(when (and (string? refresh-token) (not (string/blank? refresh-token))
(string? id-token) (not (string/blank? id-token))
(string? access-token) (not (string/blank? access-token)))
(when-let [parsed (parse-jwt-safe id-token)]
(when-not (expired? parsed)
(set-tokens! id-token access-token refresh-token)
true))))
should-refresh?
(and (string? refresh-token)
(not (string/blank? refresh-token))
(or (not restored-from-cache?)
(some-> (state/get-auth-id-token)
parse-jwt-safe
almost-expired?)))]
(when restored-from-cache?
;; Publish login event immediately so sync can start without waiting token refresh request.
(state/pub-event! [:user/fetch-info-and-graphs]))
(when should-refresh?
(go
(<! (<refresh-id-token&access-token))
;; refresh remote graph list by pub login event
(when (user-uuid) (state/pub-event! [:user/fetch-info-and-graphs]))))))
;; If tokens were not restored from cache, this is the first chance to continue login flow.
(when (and (not restored-from-cache?) (user-uuid))
(state/pub-event! [:user/fetch-info-and-graphs]))))))
(defn login-callback
[session]

2
src/main/frontend/mobile/camera.cljs
View File

@@ -28,7 +28,7 @@
(string/includes? message "not authorized")
(string/includes? message "permission")))))
(defn- take-or-choose-photo []
(defn take-or-choose-photo []
(-> (*camera-get-photo*
(clj->js
{:allowEditing (get-in

428
src/main/frontend/worker/sync.cljs
View File

@@ -27,6 +27,7 @@
[promesa.core :as p]))
(defonce *repo->latest-remote-tx (atom {}))
(defonce *start-inflight-target (atom nil))
(defn- current-client
[repo]
@@ -499,6 +500,186 @@
(= :block/uuid (first x)))
(second x)))
(defn- canonical-entity-id
[db e]
(cond
(vector? e) (or (get-lookup-id e) e)
(and (number? e) (not (neg? e))) (or (:block/uuid (d/entity db e)) e)
:else e))
(defn- remote-updated-attr-keys
[db tx-data]
(->> tx-data
(keep (fn [item]
(when (and (vector? item)
(>= (count item) 4)
(contains? #{:db/add :db/retract} (first item)))
[(canonical-entity-id db (second item))
(nth item 2)])))
set))
(defn- drop-remote-conflicted-local-tx
[db remote-updated-keys tx-data]
(if (seq remote-updated-keys)
(remove (fn [item]
(and (vector? item)
(>= (count item) 4)
(contains? #{:db/add :db/retract} (first item))
(contains? remote-updated-keys
[(canonical-entity-id db (second item))
(nth item 2)])))
tx-data)
tx-data))
(defn- remote-temp-id?
[x]
(or (and (integer? x) (neg? x))
(string? x)))
(defn- remap-remote-batch-temp-ids
[batch-index tx-data]
(let [ops #{:db/add :db/retract :db/retractEntity}
entity-temp-ids (->> tx-data
(keep (fn [item]
(when (and (vector? item)
(>= (count item) 2)
(contains? ops (first item))
(remote-temp-id? (second item)))
(second item))))
distinct)
temp-id-map (when (seq entity-temp-ids)
(zipmap entity-temp-ids
(map-indexed (fn [idx _]
(str "remote-batch-" batch-index "-tempid-" idx))
entity-temp-ids)))]
(if (seq temp-id-map)
(mapv (fn [item]
(if (and (vector? item)
(>= (count item) 2)
(contains? ops (first item)))
(let [entity (second item)
item' (if-let [entity' (get temp-id-map entity)]
(assoc item 1 entity')
item)]
(cond-> item'
(>= (count item') 4)
(#(if-let [value' (get temp-id-map (nth % 3))]
(assoc % 3 value')
%))
(>= (count item') 5)
(#(if-let [tx' (get temp-id-map (nth % 4))]
(assoc % 4 tx')
%))))
item))
tx-data)
tx-data)))
(defn- lookup-ref?
[x]
(and (vector? x)
(= 2 (count x))
(keyword? (first x))))
(defn- created-lookup->temp-id
[tx-data]
(->> tx-data
(keep (fn [item]
(when (and (vector? item)
(= :db/add (first item))
(>= (count item) 4)
(contains? #{:block/uuid :db/ident} (nth item 2))
(remote-temp-id? (second item)))
[[(nth item 2) (nth item 3)]
(second item)])))
(into {})))
(defn- resolve-lookup-refs
[lookup->temp-id tx-data]
(if (seq lookup->temp-id)
(mapv (fn [item]
(if (and (vector? item)
(>= (count item) 2)
(= :db/add (first item)))
(let [entity (second item)
item' (if-let [entity' (and (lookup-ref? entity)
(get lookup->temp-id entity))]
(assoc item 1 entity')
item)]
(if (>= (count item') 4)
(let [value (nth item' 3)]
(if-let [value' (and (lookup-ref? value)
(get lookup->temp-id value))]
(assoc item' 3 value')
item'))
item'))
item))
tx-data)
tx-data))
(defn- flatten-batched-remote-tx-data
[tx-data*]
(loop [remaining (map-indexed vector tx-data*)
lookup->temp-id {}
acc []]
(if-let [[batch-index tx-data] (first remaining)]
(let [remapped-batch (remap-remote-batch-temp-ids batch-index tx-data)
lookup->temp-id (merge lookup->temp-id (created-lookup->temp-id remapped-batch))
resolved-batch (resolve-lookup-refs lookup->temp-id remapped-batch)]
(recur (rest remaining)
lookup->temp-id
(into acc resolved-batch)))
acc)))
(defn- batched-remote-tx-data?
[tx-data*]
(and (seq tx-data*)
(sequential? (first tx-data*))
(sequential? (first (first tx-data*)))))
(defn- drop-anonymous-temp-entity-datoms
"Drop malformed temp entities from remote txs.
A temp entity must declare one identity attr (:block/uuid or :db/ident)
in its :db/add datoms; otherwise it can create anonymous entities that fail validation."
[db tx-data]
(let [identity-attrs #{:block/uuid :db/ident}
temp-id? (fn [x]
(or (string? x)
(and (integer? x) (neg? x))))
add-attrs-by-entity
(reduce (fn [acc item]
(if (and (vector? item)
(= :db/add (first item))
(>= (count item) 4))
(update acc (second item) (fnil conj #{}) (nth item 2))
acc))
{}
tx-data)
dropped-entities
(->> add-attrs-by-entity
(keep (fn [[entity attrs]]
(when (and (temp-id? entity)
(empty? (set/intersection identity-attrs attrs)))
entity)))
set)]
(if (seq dropped-entities)
(let [tx-data' (->> tx-data
(remove (fn [item]
(and (vector? item)
(>= (count item) 2)
(contains? dropped-entities (second item)))))
(remove (fn [item]
(and (vector? item)
(>= (count item) 4)
(keyword? (nth item 2))
(= :db.type/ref (:db/valueType (d/entity db (nth item 2))))
(contains? dropped-entities (nth item 3))))))]
(log/warn :db-sync/drop-anonymous-temp-entities
{:count (count dropped-entities)
:entities dropped-entities})
tx-data')
tx-data)))
(defn- sanitize-tx-data
[db tx-data local-deleted-ids]
(let [sanitized-tx-data (->> tx-data
@@ -648,8 +829,8 @@
(p/recur (rest remaining) (conj acc item))))))))
(defn- rehydrate-large-titles!
[repo {:keys [graph-id download-fn aes-key tx-data]}]
(when-let [conn (worker-state/get-datascript-conn repo)]
[repo {:keys [graph-id download-fn aes-key tx-data conn]}]
(when-let [conn (or conn (worker-state/get-datascript-conn repo))]
(let [download-fn (or download-fn download-large-title!)
graph-id (or graph-id (get-graph-id repo))
items (if (seq tx-data)
@@ -985,11 +1166,25 @@
;; 3. rebase pending local txs
rebase-tx-report (when (seq local-txs)
(let [pending-tx-data (mapcat :tx local-txs)
rebased-tx-data (sanitize-tx-data
(or (:db-after remote-tx-report)
(:db-after reversed-tx-report))
pending-tx-data
(set (map :block/uuid local-deleted-blocks)))]
remote-db (or (:db-after remote-tx-report)
(:db-after reversed-tx-report))
remote-updated-keys (remote-updated-attr-keys remote-db safe-remote-tx-data)
remote-tx-data-set (->> safe-remote-tx-data
(map (fn [item]
(if (and (vector? item)
(= 5 (count item)))
(vec (butlast item))
item)))
set)
pending-tx-data (drop-remote-conflicted-local-tx
remote-db
remote-updated-keys
pending-tx-data)
rebased-tx-data (->> (sanitize-tx-data
remote-db
pending-tx-data
(set (map :block/uuid local-deleted-blocks)))
(remove remote-tx-data-set))]
(when (seq rebased-tx-data)
(ldb/transact! temp-conn rebased-tx-data (assoc tx-meta :op :rebase)))))
;; 4. fix tx data and delete nodes
@@ -1018,78 +1213,81 @@
(defn- apply-remote-tx!
[repo client tx-data*]
(if-let [conn (worker-state/get-datascript-conn repo)]
(let [tx-data (->> tx-data*
(db-normalize/remove-retract-entity-ref @conn))
local-txs (pending-txs repo)
reversed-tx-data (get-reverse-tx-data local-txs)
has-local-changes? (seq reversed-tx-data)
*remote-tx-report (atom nil)
*reversed-tx-report (atom nil)
*remote-deleted-ids (atom #{})
*rebase-tx-data (atom [])
db @conn
remote-deleted-blocks (->> tx-data
(keep (fn [item]
(when (= :db/retractEntity (first item))
(d/entity db (second item))))))
remote-deleted-block-ids (set (map :block/uuid remote-deleted-blocks))
safe-remote-tx-data (->> tx-data
(remove (fn [item]
(or (= :db/retractEntity (first item))
(contains? remote-deleted-block-ids (get-lookup-id (last item))))))
seq)
temp-tx-meta {:rtc-tx? true
:temp-conn? true
:gen-undo-ops? false
:persist-op? false}
apply-context {:conn conn
:local-txs local-txs
:reversed-tx-data reversed-tx-data
:safe-remote-tx-data safe-remote-tx-data
:remote-deleted-blocks remote-deleted-blocks
:remote-deleted-block-ids remote-deleted-block-ids
:temp-tx-meta temp-tx-meta
:*remote-tx-report *remote-tx-report
:*reversed-tx-report *reversed-tx-report
:*remote-deleted-ids *remote-deleted-ids
:*rebase-tx-data *rebase-tx-data}
tx-report (if has-local-changes?
(apply-remote-tx-with-local-changes! apply-context)
(apply-remote-tx-without-local-changes! apply-context))
remote-tx-report @*remote-tx-report]
;; persist rebase tx to client ops
(when has-local-changes?
(when-let [tx-data (seq @*rebase-tx-data)]
(let [remote-tx-data-set (set tx-data*)
normalized (->> tx-data
(normalize-tx-data (:db-after tx-report)
(or (:db-after remote-tx-report)
(:db-after @*reversed-tx-report)))
(remove (fn [[op _e a]]
(and (= op :db/retract)
(contains? #{:block/updated-at :block/created-at :block/title} a)))))
normalized-tx-data (remove remote-tx-data-set normalized)
reversed-datoms (reverse-tx-data tx-data)]
;; (prn :debug :normalized-tx-data normalized-tx-data)
;; (prn :debug :remote-tx-data tx-data*)
;; (prn :debug :diff (data/diff remote-tx-data-set
;; (set normalized)))
(when (seq normalized-tx-data)
(persist-local-tx! repo normalized-tx-data reversed-datoms {:op :rtc-rebase}))))
(remove-pending-txs! repo (map :tx-id local-txs)))
(if (batched-remote-tx-data? tx-data*)
(apply-remote-tx! repo client (flatten-batched-remote-tx-data tx-data*))
(if-let [conn (worker-state/get-datascript-conn repo)]
(let [tx-data (->> tx-data*
(db-normalize/remove-retract-entity-ref @conn)
(#(drop-anonymous-temp-entity-datoms @conn %)))
local-txs (pending-txs repo)
reversed-tx-data (get-reverse-tx-data local-txs)
has-local-changes? (seq reversed-tx-data)
*remote-tx-report (atom nil)
*reversed-tx-report (atom nil)
*remote-deleted-ids (atom #{})
*rebase-tx-data (atom [])
db @conn
remote-deleted-blocks (->> tx-data
(keep (fn [item]
(when (= :db/retractEntity (first item))
(d/entity db (second item))))))
remote-deleted-block-ids (set (map :block/uuid remote-deleted-blocks))
safe-remote-tx-data (->> tx-data
(remove (fn [item]
(or (= :db/retractEntity (first item))
(contains? remote-deleted-block-ids (get-lookup-id (last item))))))
seq)
temp-tx-meta {:rtc-tx? true
:temp-conn? true
:gen-undo-ops? false
:persist-op? false}
apply-context {:conn conn
:local-txs local-txs
:reversed-tx-data reversed-tx-data
:safe-remote-tx-data safe-remote-tx-data
:remote-deleted-blocks remote-deleted-blocks
:remote-deleted-block-ids remote-deleted-block-ids
:temp-tx-meta temp-tx-meta
:*remote-tx-report *remote-tx-report
:*reversed-tx-report *reversed-tx-report
:*remote-deleted-ids *remote-deleted-ids
:*rebase-tx-data *rebase-tx-data}
tx-report (if has-local-changes?
(apply-remote-tx-with-local-changes! apply-context)
(apply-remote-tx-without-local-changes! apply-context))
remote-tx-report @*remote-tx-report]
;; persist rebase tx to client ops
(when has-local-changes?
(when-let [tx-data (seq @*rebase-tx-data)]
(let [remote-tx-data-set (set tx-data*)
normalized (->> tx-data
(normalize-tx-data (:db-after tx-report)
(or (:db-after remote-tx-report)
(:db-after @*reversed-tx-report)))
(remove (fn [[op _e a]]
(and (= op :db/retract)
(contains? #{:block/updated-at :block/created-at :block/title} a)))))
normalized-tx-data (remove remote-tx-data-set normalized)
reversed-datoms (reverse-tx-data tx-data)]
;; (prn :debug :normalized-tx-data normalized-tx-data)
;; (prn :debug :remote-tx-data tx-data*)
;; (prn :debug :diff (data/diff remote-tx-data-set
;; (set normalized)))
(when (seq normalized-tx-data)
(persist-local-tx! repo normalized-tx-data reversed-datoms {:op :rtc-rebase}))))
(remove-pending-txs! repo (map :tx-id local-txs)))
(when-let [*inflight (:inflight client)]
(reset! *inflight []))
(when-let [*inflight (:inflight client)]
(reset! *inflight []))
(-> (rehydrate-large-titles! repo {:tx-data tx-data
:graph-id (:graph-id client)})
(p/catch (fn [error]
(log/error :db-sync/large-title-rehydrate-failed
{:repo repo :error error}))))
(-> (rehydrate-large-titles! repo {:tx-data tx-data
:graph-id (:graph-id client)})
(p/catch (fn [error]
(log/error :db-sync/large-title-rehydrate-failed
{:repo repo :error error}))))
(reset! *remote-tx-report nil))
(fail-fast :db-sync/missing-db {:repo repo :op :apply-remote-tx})))
(reset! *remote-tx-report nil))
(fail-fast :db-sync/missing-db {:repo repo :op :apply-remote-tx}))))
(defn- handle-message! [repo client raw]
(let [message (-> raw parse-message coerce-ws-server-message)]
@@ -1125,23 +1323,23 @@
(reset! (:inflight client) [])
(flush-pending! repo client))
;; Download response
;; Merge batch txs to one tx, does it really work? We'll see
"pull/ok" (when-not (= local-tx remote-tx)
"pull/ok" (when (> remote-tx local-tx)
(let [txs (:txs message)
_ (require-non-negative remote-tx {:repo repo :type "pull/ok"})
_ (require-seq txs {:repo repo :type "pull/ok" :field :txs})
txs-data (mapv (fn [data]
(parse-transit (:tx data) {:repo repo :type "pull/ok"}))
txs)
tx (distinct (mapcat identity txs-data))]
(when (seq tx)
txs)]
(when (seq txs-data)
(p/let [aes-key (sync-crypt/<ensure-graph-aes-key repo (:graph-id client))
_ (when (and (sync-crypt/graph-e2ee? repo) (nil? aes-key))
(fail-fast :db-sync/missing-field {:repo repo :field :aes-key}))
tx* (if aes-key
(sync-crypt/<decrypt-tx-data aes-key tx)
(p/resolved tx))]
(apply-remote-tx! repo client tx*)
tx-batches (if aes-key
(p/all (mapv (fn [tx-data]
(sync-crypt/<decrypt-tx-data aes-key tx-data))
txs-data))
(p/resolved txs-data))]
(apply-remote-tx! repo client tx-batches)
(client-op/update-local-tx repo remote-tx)
(broadcast-rtc-state! client)
(flush-pending! repo client)))))
@@ -1267,23 +1465,53 @@
(reset! worker-state/*db-sync-client nil))
(p/resolved nil))
(defn- active-client-for?
[client repo graph-id]
(when (and client (= repo (:repo client)) (= graph-id (:graph-id client)))
(let [ws (:ws client)
ws-state (some-> (:ws-state client) deref)
ws-ready-state (when ws (ready-state ws))]
(or (= :open ws-state)
(contains? #{0 1} ws-ready-state)))))
(defn start!
[repo]
(p/do!
(stop!)
(let [base (ws-base-url)
graph-id (get-graph-id repo)]
(if (and (string? base) (seq base) (seq graph-id))
(let [client (ensure-client-state! repo)
url (format-ws-url base graph-id)
_ (ensure-client-graph-uuid! repo graph-id)
connected (assoc client :graph-id graph-id)
connected (connect! repo connected url)]
(reset! worker-state/*db-sync-client connected)
(p/resolved nil))
(do
(log/info :db-sync/start-skipped {:repo repo :graph-id graph-id :base base})
(p/resolved nil))))))
(let [base (ws-base-url)
graph-id (get-graph-id repo)
start-target [repo graph-id]
inflight-target @*start-inflight-target
current @worker-state/*db-sync-client]
(cond
(not (and (string? base) (seq base) (seq graph-id)))
(do
(log/info :db-sync/start-skipped {:repo repo :graph-id graph-id :base base})
(p/resolved nil))
(= start-target inflight-target)
(p/resolved nil)
(active-client-for? current repo graph-id)
(do
(broadcast-rtc-state! current)
(p/resolved nil))
:else
(do
(reset! *start-inflight-target start-target)
(->
(p/do!
(stop!)
(let [client (ensure-client-state! repo)
url (format-ws-url base graph-id)
_ (ensure-client-graph-uuid! repo graph-id)
connected (assoc client :graph-id graph-id)
connected (connect! repo connected url)]
(reset! worker-state/*db-sync-client connected)
nil))
(p/finally
(fn []
(when (= start-target @*start-inflight-target)
(reset! *start-inflight-target nil)))))))))
(defn enqueue-local-tx!
[repo {:keys [tx-meta tx-data db-after db-before]}]

92
src/main/mobile/components/header.cljs
View File

@@ -152,13 +152,10 @@
(reset! native-top-bar-listener? true)))
(defn- configure-native-top-bar!
[repo {:keys [tab title route-name route-view sync-color favorited?]}]
[{:keys [tab title route-name route-view sync-color favorited? show-sync?]}]
(when (and (mobile-util/native-platform?)
mobile-util/native-top-bar)
(let [hidden? (and (mobile-util/native-ios?) (= tab "search"))
rtc-indicator? (and repo
(ldb/get-graph-rtc-uuid (db/get-db))
(user-handler/logged-in?))
base (cond->
{:hidden hidden?}
(not (mobile-util/native-ipad?))
@@ -179,7 +176,7 @@
(cond-> []
(nil? route-view)
(conj {:id "home-setting" :systemIcon "ellipsis"})
(and rtc-indicator? (not page?))
(and show-sync? (not page?))
(conj {:id "sync" :systemIcon "circle.fill" :color sync-color
:size "small"}))
@@ -208,13 +205,27 @@
"Select a Graph")
route-name (get-in route-match [:data :name])
route-view (get-in route-match [:data :view])
route-id (get-in route-match [:parameters :path :name])
page-route? (= route-name :page)
[*configure-top-bar-f _] (hooks/use-state (atom nil))
detail-info (hooks/use-flow-state (m/watch rtc-indicator/*detail-info))
_ (hooks/use-flow-state flows/current-login-user-flow)
online? (hooks/use-flow-state flows/network-online-event-flow)
rtc-state (:rtc-state detail-info)
graph-uuid (or (:graph-uuid detail-info)
(ldb/get-graph-rtc-uuid (db/get-db)))
show-sync? (and current-repo graph-uuid (user-handler/logged-in?))
unpushed-block-update-count (:pending-local-ops detail-info)
pending-asset-ops (:pending-asset-ops detail-info)
fallback-title (cond
(= tab "home")
short-repo-name
(= tab "search")
"Search"
:else
(string/capitalize tab))
sync-color (if (and online?
(= :open rtc-state)
(zero? unpushed-block-update-count)
@@ -228,33 +239,56 @@
(when (and (mobile-util/native-platform?)
mobile-util/native-top-bar)
(register-native-top-bar-events! *configure-top-bar-f)
(p/let [block (when (= route-name :page)
(let [id (get-in route-match [:parameters :path :name])]
(when (common-util/uuid-string? id)
(db-async/<get-block current-repo (uuid id) {:children? false}))))
favorited? (when block
(page-handler/favorited? (str (:block/uuid block))))
title (cond block
(:block/title block)
(= tab "home")
short-repo-name
(= tab "search")
"Search"
:else
(string/capitalize tab))
f (fn [favorited?]
(configure-native-top-bar!
current-repo
{:tab tab
:title title
:route-name route-name
:route-view route-view
:sync-color sync-color
:favorited? favorited?}))]
(let [block (when (and page-route?
(common-util/uuid-string? route-id))
(db/entity [:block/uuid (uuid route-id)]))
favorited? (when block
(page-handler/favorited? (str (:block/uuid block))))
title (or (:block/title block) fallback-title)
f (fn [favorited?]
(configure-native-top-bar!
{:tab tab
:title title
:route-name route-name
:route-view route-view
:sync-color sync-color
:show-sync? show-sync?
:favorited? favorited?}))]
(reset! *configure-top-bar-f f)
(f favorited?)))
nil)
[tab short-repo-name route-match sync-color])
[current-repo tab route-name route-view route-id fallback-title sync-color show-sync? page-route?])
(hooks/use-effect!
(fn []
(if (and (mobile-util/native-platform?)
mobile-util/native-top-bar
current-repo
page-route?
(common-util/uuid-string? route-id))
(let [cancelled? (atom false)
page-id (uuid route-id)]
(-> (db-async/<get-block current-repo page-id {:children? false})
(p/then
(fn [block]
(when (and block (not @cancelled?))
(let [favorited? (page-handler/favorited? (str (:block/uuid block)))
title (:block/title block)
f (fn [favorited?]
(configure-native-top-bar!
{:tab tab
:title title
:route-name route-name
:route-view route-view
:sync-color sync-color
:show-sync? show-sync?
:favorited? favorited?}))]
(reset! *configure-top-bar-f f)
(f favorited?)))))
(p/catch (fn [_] nil)))
#(reset! cancelled? true))
nil))
[current-repo tab route-name route-view route-id sync-color show-sync? page-route?])
[:<>]))

10
src/test/frontend/components/query_test.cljs Normal file
View File

@@ -0,0 +1,10 @@
(ns frontend.components.query-test
(:require [cljs.test :refer [deftest is]]
[frontend.components.query :as query]))
(deftest grouped-by-page-result-detection-supports-partial-page-refs
(let [result [[{:db/id 42}
[{:block/uuid (random-uuid)}]]]]
(is (true? (#'query/grouped-by-page-result? result true))
"Grouped query results with page refs that only include :db/id should still be recognized")
(is (false? (#'query/grouped-by-page-result? result false)))))

11
src/test/frontend/db/model_test.cljs
View File

@@ -4,6 +4,7 @@
[frontend.db :as db]
[frontend.db.conn :as conn]
[frontend.db.model :as model]
[frontend.db.utils]
[frontend.test.helper :as test-helper :refer [load-test-files]]))
(use-fixtures :each {:before test-helper/start-test-db!
@@ -117,3 +118,13 @@
(is (= ["child 1" "child 2" "child 3"]
(map :block/title
(model/get-block-immediate-children test-db (:block/uuid parent)))))))
(deftest with-pages-preserves-page-ref-when-ui-db-is-partial
(let [page-ref {:db/id 1}
block {:db/id 2
:block/uuid (random-uuid)
:block/page page-ref}]
(with-redefs [frontend.db.utils/pull-many (fn [& _] nil)]
(is (= page-ref
(:block/page (first (model/with-pages [block]))))
"When page entity details are unavailable locally, keep the original page ref instead of replacing it with nil"))))

48
src/test/frontend/handler/db_based/sync_test.cljs
View File

@@ -85,14 +85,14 @@
(deftest rtc-start-skips-when-graph-missing-from-remote-list-test
(async done
(let [called (atom nil)]
(let [called (atom [])]
(-> (p/with-redefs [state/get-rtc-graphs (fn [] [{:url "repo-other"}])
state/<invoke-db-worker (fn [& args]
(reset! called args)
(swap! called conj args)
(p/resolved :ok))]
(db-sync/<rtc-start! "repo-current"))
(p/then (fn [_]
(is (nil? @called))
(is (= [[:thread-api/db-sync-stop]] @called))
(done)))
(p/catch (fn [e]
(is false (str e))
@@ -113,6 +113,48 @@
(is false (str e))
(done)))))))
(deftest rtc-start-waits-for-db-worker-before-start-test
(async done
(let [worker (atom nil)
called (atom [])]
(-> (p/with-redefs [state/get-rtc-graphs (fn [] [{:url "repo-current"}])
state/*db-worker worker]
(p/let [start-p (db-sync/<rtc-start! "repo-current")
_ (p/delay 30)
_ (is (empty? @called))
_ (reset! worker
(fn [qkw direct-pass? & args]
(swap! called conj [qkw direct-pass? args])
(p/resolved :ok)))
_ start-p]
(is (= [[:thread-api/db-sync-start false ["repo-current"]]]
@called))
(done)))
(p/catch (fn [e]
(is false (str e))
(done)))))))
(deftest rtc-start-waits-for-db-worker-before-stop-test
(async done
(let [worker (atom nil)
called (atom [])]
(-> (p/with-redefs [state/get-rtc-graphs (fn [] [{:url "repo-other"}])
state/*db-worker worker]
(p/let [start-p (db-sync/<rtc-start! "repo-current")
_ (p/delay 30)
_ (is (empty? @called))
_ (reset! worker
(fn [qkw direct-pass? & args]
(swap! called conj [qkw direct-pass? args])
(p/resolved :ok)))
_ start-p]
(is (= [[:thread-api/db-sync-stop false []]]
@called))
(done)))
(p/catch (fn [e]
(is false (str e))
(done)))))))
(deftest rtc-create-graph-persists-disabled-e2ee-flag-test
(async done
(let [fetch-called (atom nil)

38
src/test/frontend/mobile/audio_recorder_test.cljs
View File

@@ -1,38 +0,0 @@
(ns frontend.mobile.audio-recorder-test
(:require [cljs.test :refer [is testing]]
[frontend.handler.notification :as notification]
[frontend.mobile.audio-recorder :as audio-recorder]
[frontend.test.helper :include-macros true :refer [deftest-async]]
[logseq.shui.ui :as shui]
[promesa.core :as p]))
(deftest-async start-recording-shows-warning-when-microphone-permission-denied
(testing "Shows actionable warning and closes recorder popup when mic permission is denied"
(let [warning (atom nil)
popup-hidden? (atom false)]
(p/with-redefs
[notification/show! (fn [content & _]
(reset! warning content))
shui/popup-hide! (fn []
(reset! popup-hidden? true))]
(p/let [_ (audio-recorder/start-recording! #js {:startRecording
(fn []
(p/rejected (js/Error. "Error accessing the microphone: Permission denied")))})]
(is (string? @warning))
(is (re-find #"Settings" @warning))
(is (true? @popup-hidden?)))))))
(deftest-async start-recording-does-not-show-warning-for-non-permission-errors
(testing "Avoids permission warning for unrelated start recording failures"
(let [warning (atom nil)
popup-hidden? (atom false)]
(p/with-redefs
[notification/show! (fn [content & _]
(reset! warning content))
shui/popup-hide! (fn []
(reset! popup-hidden? true))]
(p/let [_ (audio-recorder/start-recording! #js {:startRecording
(fn []
(p/rejected (js/Error. "Error: No microphone device found")))})]
(is (nil? @warning))
(is (false? @popup-hidden?)))))))

79
src/test/frontend/mobile/camera_test.cljs
View File

@@ -1,79 +0,0 @@
(ns frontend.mobile.camera-test
(:require [cljs.test :refer [is testing]]
[frontend.handler.editor :as editor-handler]
[frontend.handler.notification :as notification]
[frontend.mobile.camera :as mobile-camera]
[frontend.state :as state]
[frontend.test.helper :include-macros true :refer [deftest-async]]
[promesa.core :as p]))
(deftest-async embed-photo-uses-provided-id
(testing "Uses explicit editor id so capture/upload still works if focused input id is temporarily nil"
(let [upload-id (atom nil)]
(p/with-redefs
[mobile-camera/take-or-choose-photo (fn [] (p/resolved #js {:name "photo.jpeg"}))
state/get-edit-block (constantly {:block/format :markdown})
editor-handler/upload-asset! (fn [id _files _format _uploading? _drop-or-paste?]
(reset! upload-id id)
(p/resolved nil))]
(p/let [_ (mobile-camera/embed-photo "editor-id")]
(is (= "editor-id" @upload-id)))))))
(deftest-async embed-photo-skips-upload-when-no-photo
(testing "Doesn't trigger upload pipeline when camera returns nil photo"
(let [upload-called? (atom false)]
(p/with-redefs
[mobile-camera/take-or-choose-photo (fn [] (p/resolved nil))
state/get-edit-block (constantly {:block/format :markdown})
editor-handler/upload-asset! (fn [& _]
(reset! upload-called? true)
(p/resolved nil))]
(p/let [_ (mobile-camera/embed-photo "editor-id")]
(is (false? @upload-called?)))))))
(deftest-async embed-photo-still-allows-photo-picking-when-camera-permission-denied
(testing "Does not pre-block getPhoto by camera permission so users can still pick existing photos"
(let [upload-called? (atom false)
get-photo-called? (atom false)
warning (atom nil)]
(p/with-redefs
[mobile-camera/*camera-get-photo* (fn [_]
(reset! get-photo-called? true)
(p/resolved #js {:base64String "AA=="}))
state/get-edit-block (constantly {:block/format :markdown})
notification/show! (fn [content & _]
(reset! warning content))
editor-handler/upload-asset! (fn [& _]
(reset! upload-called? true)
(p/resolved nil))]
(p/let [_ (mobile-camera/embed-photo "editor-id")]
(is (true? @get-photo-called?))
(is (true? @upload-called?))
(is (nil? @warning)))))))
(deftest-async embed-photo-warns-only-for-camera-denied
(testing "Shows camera warning only for camera denied errors, not photo-library denied"
(let [warning (atom nil)]
(p/with-redefs
[mobile-camera/*camera-get-photo* (fn [_]
(p/rejected (js/Error. "User denied access to photos")))
state/get-edit-block (constantly {:block/format :markdown})
notification/show! (fn [content & _]
(reset! warning content))
editor-handler/upload-asset! (fn [& _] (p/resolved nil))]
(p/let [_ (mobile-camera/embed-photo "editor-id")]
(is (nil? @warning)))))))
(deftest-async embed-photo-warns-when-camera-access-denied
(testing "Shows camera warning when take picture is denied by camera permission"
(let [warning (atom nil)]
(p/with-redefs
[mobile-camera/*camera-get-photo* (fn [_]
(p/rejected (js/Error. "User denied access to camera")))
state/get-edit-block (constantly {:block/format :markdown})
notification/show! (fn [content & _]
(reset! warning content))
editor-handler/upload-asset! (fn [& _] (p/resolved nil))]
(p/let [_ (mobile-camera/embed-photo "editor-id")]
(is (string? @warning))
(is (re-find #"Settings" @warning)))))))

14
src/test/frontend/worker/db_sync_sim_test.cljs
View File

@@ -1,7 +1,6 @@
(ns frontend.worker.db-sync-sim-test
(:require [cljs.test :refer [deftest is testing]]
[clojure.data :as data]
[clojure.string :as string]
[datascript.core :as d]
[frontend.worker.handler.page :as worker-page]
[frontend.worker.state :as worker-state]
@@ -207,7 +206,7 @@
(defn- server-pull [server since]
(let [{:keys [txs]} @server]
(->> (filter (fn [{:keys [t]}] (> t since)) txs)
(mapcat :tx))))
(mapv :tx))))
(defn- server-upload! [server t-before tx-data]
(swap! server
@@ -234,10 +233,10 @@
server-t (:t @server)]
;; (prn :debug :repo repo :local-tx local-tx :server-t server-t)
(when (< local-tx server-t)
(let [tx (server-pull server local-tx)]
(let [txs (server-pull server local-tx)]
;; (prn :debug :apply-remote-tx :repo repo
;; :tx tx)
(#'db-sync/apply-remote-tx! repo client tx)
(#'db-sync/apply-remote-tx! repo client txs)
(client-op/update-local-tx repo server-t)
(reset! progress? true)))
(let [pending (#'db-sync/pending-txs repo)
@@ -371,9 +370,8 @@
(let [parent-uuid (:block/uuid parent)
parent (d/entity db [:block/uuid parent-uuid])]
(when parent
(let [uuid ((or gen-uuid random-uuid))
title (str "Block-" (rand-int! rng 1000000))]
(create-block! conn parent title uuid)
(let [uuid ((or gen-uuid random-uuid))]
(create-block! conn parent "" uuid)
(swap! state update :blocks conj uuid)
{:op :create-block :uuid uuid :parent parent-uuid}))))))
@@ -384,7 +382,7 @@
block (d/entity db [:block/uuid (:block/uuid ent)])]
(when (and block (not (ldb/page? block)))
(let [uuid (:block/uuid block)
new-title (string/replace (:block/title (d/entity @conn [:block/uuid uuid])) "block" "title")]
new-title (str "title-" (:db/id block))]
(update-title! conn uuid new-title)
{:op :update-title :uuid uuid :title new-title}))))

203
src/test/frontend/worker/db_sync_test.cljs
View File

@@ -10,6 +10,8 @@
[frontend.worker.sync.crypt :as sync-crypt]
[logseq.common.config :as common-config]
[logseq.db :as ldb]
[logseq.db.frontend.validate :as db-validate]
[logseq.db.sqlite.util :as sqlite-util]
[logseq.db.test.helper :as db-test]
[logseq.outliner.core :as outliner-core]
[logseq.outliner.op :as outliner-op]
@@ -114,6 +116,112 @@
@(:online-users client)))
(is (= 1 (count @broadcasts))))))
(deftest pull-ok-with-older-remote-tx-is-ignored-test
(testing "pull/ok with remote tx behind local tx does not apply stale tx data"
(let [{:keys [conn client-ops-conn parent]} (setup-parent-child)
parent-id (:db/id parent)
stale-tx (sqlite-util/write-transit-str [[:db/add parent-id :block/title "stale-title"]])
raw-message (js/JSON.stringify
(clj->js {:type "pull/ok"
:t 4
:txs [{:t 4 :tx stale-tx}]}))
latest-prev @db-sync/*repo->latest-remote-tx
client {:repo test-repo
:graph-id "graph-1"
:inflight (atom [])
:online-users (atom [])
:ws-state (atom :open)}]
(with-datascript-conns conn client-ops-conn
(fn []
(reset! db-sync/*repo->latest-remote-tx {})
(try
(client-op/update-local-tx test-repo 5)
(#'db-sync/handle-message! test-repo client raw-message)
(let [parent' (d/entity @conn parent-id)]
(is (= "parent" (:block/title parent')))
(is (= 5 (client-op/get-local-tx test-repo))))
(finally
(reset! db-sync/*repo->latest-remote-tx latest-prev))))))))
(deftest pull-ok-out-of-order-stale-response-is-ignored-test
(testing "late stale pull/ok should not overwrite a newer already-applied tx"
(async done
(let [{:keys [conn client-ops-conn parent]} (setup-parent-child)
parent-id (:db/id parent)
new-tx (sqlite-util/write-transit-str [[:db/add parent-id :block/title "remote-new-title"]])
stale-tx (sqlite-util/write-transit-str [[:db/add parent-id :block/title "stale-title"]])
raw-new (js/JSON.stringify
(clj->js {:type "pull/ok"
:t 2
:txs [{:t 2 :tx new-tx}]}))
raw-stale (js/JSON.stringify
(clj->js {:type "pull/ok"
:t 1
:txs [{:t 1 :tx stale-tx}]}))
latest-prev @db-sync/*repo->latest-remote-tx
client {:repo test-repo
:graph-id "graph-1"
:inflight (atom [])
:online-users (atom [])
:ws-state (atom :open)}]
(reset! db-sync/*repo->latest-remote-tx {})
(with-datascript-conns conn client-ops-conn
(fn []
(-> (p/let [_ (#'db-sync/handle-message! test-repo client raw-new)
_ (#'db-sync/handle-message! test-repo client raw-stale)
parent' (d/entity @conn parent-id)]
(is (= "remote-new-title" (:block/title parent')))
(is (= 2 (client-op/get-local-tx test-repo))))
(p/finally (fn []
(reset! db-sync/*repo->latest-remote-tx latest-prev)
(done))))))))))
(deftest pull-ok-batched-txs-preserve-tempid-boundaries-test
(testing "pull/ok applies tx batches without cross-tx tempid collisions"
(async done
(let [{:keys [conn client-ops-conn parent]} (setup-parent-child)
page-uuid (:block/uuid (:block/page parent))
block-uuid-a (random-uuid)
block-uuid-b (random-uuid)
now 1760000000000
tx-a (sqlite-util/write-transit-str
[[:db/add -1 :block/uuid block-uuid-a]
[:db/add -1 :block/title "remote-a"]
[:db/add -1 :block/parent [:block/uuid page-uuid]]
[:db/add -1 :block/page [:block/uuid page-uuid]]
[:db/add -1 :block/order 1]
[:db/add -1 :block/updated-at now]
[:db/add -1 :block/created-at now]])
tx-b (sqlite-util/write-transit-str
[[:db/add -1 :block/uuid block-uuid-b]
[:db/add -1 :block/title "remote-b"]
[:db/add -1 :block/parent [:block/uuid page-uuid]]
[:db/add -1 :block/page [:block/uuid page-uuid]]
[:db/add -1 :block/order 2]
[:db/add -1 :block/updated-at now]
[:db/add -1 :block/created-at now]])
raw-message (js/JSON.stringify
(clj->js {:type "pull/ok"
:t 2
:txs [{:t 1 :tx tx-a}
{:t 2 :tx tx-b}]}))
latest-prev @db-sync/*repo->latest-remote-tx
client {:repo test-repo
:graph-id "graph-1"
:inflight (atom [])
:online-users (atom [])
:ws-state (atom :open)}]
(with-datascript-conns conn client-ops-conn
(fn []
(reset! db-sync/*repo->latest-remote-tx {})
(-> (p/let [_ (client-op/update-local-tx test-repo 0)
_ (#'db-sync/handle-message! test-repo client raw-message)]
(is (= "remote-a" (:block/title (d/entity @conn [:block/uuid block-uuid-a]))))
(is (= "remote-b" (:block/title (d/entity @conn [:block/uuid block-uuid-b])))))
(p/finally (fn []
(reset! db-sync/*repo->latest-remote-tx latest-prev)
(done))))))))))
(deftest reaction-add-enqueues-pending-sync-tx-test
(testing "adding a reaction should enqueue tx for db-sync"
(let [{:keys [conn client-ops-conn parent]} (setup-parent-child)]
@@ -423,6 +531,100 @@
(let [block' (d/entity @conn (:db/id block))]
(is (= "test" (:block/title block'))))))))))
(deftest ^:long rebase-does-not-leave-anonymous-created-by-entities-test
(testing "rebase should not leave entities with timestamps/created-by but without identity attrs"
(let [{:keys [conn client-ops-conn parent child1]} (setup-parent-child)
child-id (:db/id child1)
page-id (:db/id (:block/page parent))]
(with-redefs [db-sync/enqueue-local-tx!
(let [orig db-sync/enqueue-local-tx!]
(fn [repo tx-report]
(when-not (:rtc-tx? (:tx-meta tx-report))
(orig repo tx-report))))]
(with-datascript-conns conn client-ops-conn
(fn []
;; Ensure the deleted block has the same created-by shape from production repros.
(d/transact! conn [[:db/add child-id :logseq.property/created-by-ref page-id]])
(outliner-core/delete-blocks! conn [(d/entity @conn child-id)] {})
(is (seq (#'db-sync/pending-txs test-repo)))
(#'db-sync/apply-remote-tx!
test-repo
nil
[[:db/add (:db/id parent) :block/title "parent remote"]])
(let [anonymous-ents (->> (d/datoms @conn :avet :logseq.property/created-by-ref)
(keep (fn [datom]
(let [ent (d/entity @conn (:e datom))]
(when (and (nil? (:block/uuid ent))
(nil? (:db/ident ent))
(some? (:block/created-at ent))
(some? (:block/updated-at ent)))
(select-keys ent [:db/id :block/created-at :block/updated-at :logseq.property/created-by-ref]))))))
validation (db-validate/validate-local-db! @conn)]
(is (empty? anonymous-ents) (str anonymous-ents))
(is (empty? (map :entity (:errors validation)))
(str (:errors validation))))))))))
(deftest ^:long rebase-create-then-delete-does-not-leave-anonymous-entities-test
(testing "create+delete before sync should not leave anonymous entities after rebase"
(let [{:keys [conn client-ops-conn parent]} (setup-parent-child)
page-id (:db/id (:block/page parent))]
(with-redefs [db-sync/enqueue-local-tx!
(let [orig db-sync/enqueue-local-tx!]
(fn [repo tx-report]
(when-not (:rtc-tx? (:tx-meta tx-report))
(orig repo tx-report))))]
(with-datascript-conns conn client-ops-conn
(fn []
(outliner-core/insert-blocks! conn [{:block/title "temp-rebase-case"}] parent {:sibling? false})
(let [temp-block (db-test/find-block-by-content @conn "temp-rebase-case")
temp-id (:db/id temp-block)]
(d/transact! conn [[:db/add temp-id :logseq.property/created-by-ref page-id]])
(outliner-core/delete-blocks! conn [temp-block] {})
(is (>= (count (#'db-sync/pending-txs test-repo)) 2))
(#'db-sync/apply-remote-tx!
test-repo
nil
[[:db/add (:db/id parent) :block/title "parent remote 2"]])
(let [anonymous-ents (->> (d/datoms @conn :avet :block/created-at)
(keep (fn [datom]
(let [ent (d/entity @conn (:e datom))]
(when (and (nil? (:block/uuid ent))
(nil? (:db/ident ent))
(some? (:block/updated-at ent)))
(select-keys ent [:db/id :block/created-at :block/updated-at :logseq.property/created-by-ref]))))))
validation (db-validate/validate-local-db! @conn)]
(is (empty? anonymous-ents) (str anonymous-ents))
(is (empty? (map :entity (:errors validation)))
(str (:errors validation)))))))))))
(deftest ^:long malformed-remote-anonymous-entity-tx-is-ignored-test
(testing "remote tx creating anonymous entities should be ignored instead of invalidating db"
(let [{:keys [conn parent]} (setup-parent-child)
created-by-id (:db/id (:block/page parent))
ts 1771435997392
malformed-tx [[:db/add "missing-uuid-entity" :block/created-at ts]
[:db/add "missing-uuid-entity" :block/updated-at ts]
[:db/add "missing-uuid-entity" :logseq.property/created-by-ref created-by-id]]]
(with-datascript-conns conn nil
(fn []
(is (nil? (try
(#'db-sync/apply-remote-tx! test-repo nil malformed-tx)
nil
(catch :default e
e))))
(let [anonymous-ents (->> (d/datoms @conn :avet :logseq.property/created-by-ref)
(keep (fn [datom]
(let [ent (d/entity @conn (:e datom))]
(when (and (nil? (:block/uuid ent))
(nil? (:db/ident ent))
(= ts (:block/created-at ent))
(= ts (:block/updated-at ent)))
(select-keys ent [:db/id :block/created-at :block/updated-at :logseq.property/created-by-ref]))))))
validation (db-validate/validate-local-db! @conn)]
(is (empty? anonymous-ents) (str anonymous-ents))
(is (empty? (map :entity (:errors validation)))
(str (:errors validation)))))))))
(deftest ^:long offload-large-title-test
(testing "large titles are offloaded to object storage with placeholder"
(async done
@@ -554,6 +756,7 @@
(-> (p/let [result (#'db-sync/rehydrate-large-titles!
test-repo
{:tx-data tx-data
:conn conn
:graph-id "graph-1"
:download-fn download-fn
:aes-key nil})