mirror of
https://github.com/logseq/logseq.git
synced 2026-05-29 15:09:41 +00:00
Merge remote-tracking branch 'origin/master' into refactor/vite-migration
This commit is contained in:
Mega Yu
@@ -754,7 +754,9 @@
|
||||
(last child)
|
||||
(let [{:keys [content children]} (last child)
|
||||
page-name (subs content 2 (- (count content) 2))]
|
||||
(rum/with-key (page-reference (assoc config :children children) page-name nil) page-name))))
|
||||
(rum/with-key (page-reference (assoc config :children children)
|
||||
(or (:block/uuid page-entity) page-name)
|
||||
nil) page-name))))
|
||||
(cond
|
||||
(and label
|
||||
(string? label)
|
||||
@@ -3018,7 +3020,7 @@
|
||||
:disable-preview? true)]
|
||||
(when (seq parents)
|
||||
(let [parents-props (doall
|
||||
(for [{:block/keys [uuid name title] :as block} parents]
|
||||
(for [{:block/keys [uuid name] :as block} parents]
|
||||
(if name
|
||||
[block (page-cp (cond-> {:disable-preview? true}
|
||||
disabled?
|
||||
@@ -3027,7 +3029,7 @@
|
||||
(let [result (block/parse-title-and-body
|
||||
uuid
|
||||
(get block :block/format :markdown)
|
||||
title)
|
||||
(:block/raw-title block))
|
||||
ast-body (:block.temp/ast-body result)
|
||||
ast-title (:block.temp/ast-title result)
|
||||
config (assoc config :block/uuid uuid)]
|
||||
|
||||
@@ -433,7 +433,7 @@
|
||||
(let [format (util/evalue e)]
|
||||
(when-not (string/blank? format)
|
||||
(p/do!
|
||||
(property-handler/set-block-property! :logseq.class/Journal
|
||||
(property-handler/set-block-property! (:block/uuid (db/entity :logseq.class/Journal))
|
||||
:logseq.property.journal/title-format
|
||||
format)
|
||||
(notification/show! (t :settings.general/refresh-required-feedback)))
|
||||
|
||||
@@ -215,7 +215,7 @@
|
||||
|
||||
(defn backup-db-graph
|
||||
[repo]
|
||||
(when-not (util/capacitor?)
|
||||
(when util/web-platform?
|
||||
(web-backup-db-graph repo)))
|
||||
|
||||
(defonce *backup-interval (atom nil))
|
||||
|
||||
@@ -25,6 +25,9 @@
|
||||
(defn- ->block-id
|
||||
[block-or-id]
|
||||
(cond
|
||||
(keyword? block-or-id)
|
||||
(:block/uuid (db-utils/entity block-or-id))
|
||||
|
||||
(de/entity? block-or-id)
|
||||
(:block/uuid block-or-id)
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
(not (ldb/journal? entity))
|
||||
(not (:logseq.property/built-in? entity))
|
||||
(not (= :logseq.property/query (:db/ident (:logseq.property/created-from-property entity)))))))
|
||||
(d/datom e a (str "debug " e) t)
|
||||
(d/datom e a (str "debug " e " " (apply str (repeat (count v) "x"))) t)
|
||||
|
||||
:else
|
||||
(d/datom e a v t))))))
|
||||
|
||||
@@ -594,8 +594,6 @@ DROP TRIGGER IF EXISTS blocks_au;
|
||||
(when-not (string/blank? q)
|
||||
(let [option (assoc option :enable-snippet? enable-snippet?)
|
||||
match-input (get-match-input q)
|
||||
page-count (count (d/datoms @conn :avet :block/name))
|
||||
large-graph? (> page-count 2500)
|
||||
non-match-input (when (<= (count q) 2)
|
||||
(str "%" (string/replace q #"\s+" "%") "%"))
|
||||
limit (or limit 100)
|
||||
@@ -613,11 +611,9 @@ DROP TRIGGER IF EXISTS blocks_au;
|
||||
(->> (search-blocks-aux search-db non-match-sql q non-match-input page limit-p)
|
||||
(map (fn [result]
|
||||
(assoc result :keyword-score (fuzzy/score q (:title result)))))))
|
||||
;; fuzzy is too slow for large graphs
|
||||
fuzzy-result (when-not (or page large-graph?)
|
||||
(->> (fuzzy-search repo @conn q option)
|
||||
(map (fn [result]
|
||||
(assoc result :keyword-score (fuzzy/score q (:title result)))))))
|
||||
fuzzy-result (->> (fuzzy-search repo @conn q option)
|
||||
(map (fn [result]
|
||||
(assoc result :keyword-score (fuzzy/score q (:title result))))))
|
||||
;; _ (prn :debug "Search results before combine:" enable-snippet? (map :snippet matched-result))
|
||||
;; _ (doseq [item (concat fuzzy-result matched-result)]
|
||||
;; (prn :debug :keyword-search-result item))
|
||||
|
||||
@@ -17,8 +17,11 @@
|
||||
[logseq.db.sqlite.util :as sqlite-util]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(def upload-kvs-batch-size 2000)
|
||||
(def upload-kvs-batch-size 500)
|
||||
(def upload-prepare-datoms-batch-size 100000)
|
||||
(def snapshot-upload-max-bytes 1000000)
|
||||
(def snapshot-frame-header-bytes 4)
|
||||
(def ignored-oversized-upload-attrs #{:logseq.property.tldraw/page})
|
||||
(def snapshot-content-type "application/transit+json")
|
||||
(def snapshot-content-encoding "gzip")
|
||||
(def snapshot-text-encoder (js/TextEncoder.))
|
||||
@@ -63,6 +66,66 @@
|
||||
[rows]
|
||||
(.encode snapshot-text-encoder (sqlite-util/write-transit-str rows)))
|
||||
|
||||
(defn- datom-value-byte-length
|
||||
[value]
|
||||
(.-byteLength ^js (.encode snapshot-text-encoder (sqlite-util/write-transit-str value))))
|
||||
|
||||
(defn- drop-oversized-upload-datoms
|
||||
[datoms]
|
||||
(let [threshold (- snapshot-upload-max-bytes snapshot-frame-header-bytes)]
|
||||
(reduce (fn [{:keys [kept dropped]} datom]
|
||||
(let [attr (:a datom)
|
||||
size (when (contains? ignored-oversized-upload-attrs attr)
|
||||
(datom-value-byte-length (:v datom)))]
|
||||
(if (and size (> size threshold))
|
||||
{:kept kept
|
||||
:dropped (conj dropped {:a attr
|
||||
:e (:e datom)
|
||||
:bytes size})}
|
||||
{:kept (conj kept datom)
|
||||
:dropped dropped})))
|
||||
{:kept []
|
||||
:dropped []}
|
||||
datoms)))
|
||||
|
||||
(defn- snapshot-rows-byte-length
|
||||
[rows]
|
||||
(+ snapshot-frame-header-bytes
|
||||
(.-byteLength ^js (encode-snapshot-rows rows))))
|
||||
|
||||
(defn- max-prefix-rows-within-bytes
|
||||
[rows max-bytes]
|
||||
(let [rows-count (count rows)]
|
||||
(loop [low 1
|
||||
high rows-count
|
||||
best 0]
|
||||
(if (> low high)
|
||||
best
|
||||
(let [mid (quot (+ low high) 2)
|
||||
rows' (subvec rows 0 mid)
|
||||
size (snapshot-rows-byte-length rows')]
|
||||
(if (<= size max-bytes)
|
||||
(recur (inc mid) high mid)
|
||||
(recur low (dec mid) best)))))))
|
||||
|
||||
(defn- split-snapshot-rows-by-max-bytes
|
||||
[rows max-bytes]
|
||||
(loop [remaining rows
|
||||
batches []]
|
||||
(if (empty? remaining)
|
||||
batches
|
||||
(let [prefix-count (max-prefix-rows-within-bytes remaining max-bytes)]
|
||||
(if (pos? prefix-count)
|
||||
(let [batch (subvec remaining 0 prefix-count)
|
||||
remaining' (subvec remaining prefix-count)]
|
||||
(recur remaining' (conj batches batch)))
|
||||
(let [row (first remaining)
|
||||
row-size (snapshot-rows-byte-length [row])]
|
||||
(fail-fast :db-sync/snapshot-row-too-large
|
||||
{:max-bytes max-bytes
|
||||
:row-size row-size
|
||||
:addr (first row)})))))))
|
||||
|
||||
(defn frame-bytes
|
||||
[^js data]
|
||||
(let [len (.-byteLength data)
|
||||
@@ -98,6 +161,30 @@
|
||||
{:body buf :encoding snapshot-content-encoding})
|
||||
(p/resolved {:body frame :encoding nil}))))
|
||||
|
||||
(defn- snapshot-upload-url
|
||||
[base graph-id reset? finished? checksum]
|
||||
(str base "/sync/" graph-id "/snapshot/upload?reset="
|
||||
(if reset? "true" "false")
|
||||
"&finished="
|
||||
(if finished? "true" "false")
|
||||
(when finished?
|
||||
(str "&checksum=" (js/encodeURIComponent checksum)))))
|
||||
|
||||
(defn- <upload-snapshot-rows-batches!
|
||||
[rows-batches {:keys [base graph-id first-batch? finished? checksum auth-fetch-f]}]
|
||||
(p/loop [remaining rows-batches
|
||||
first-request? first-batch?]
|
||||
(if-let [rows-batch (first remaining)]
|
||||
(let [last-request? (nil? (next remaining))
|
||||
finished-request? (and finished? last-request?)
|
||||
upload-url (snapshot-upload-url base graph-id first-request? finished-request? checksum)]
|
||||
(p/let [{:keys [body encoding]} (<snapshot-upload-body rows-batch)
|
||||
headers (cond-> {"content-type" snapshot-content-type}
|
||||
(string? encoding) (assoc "content-encoding" encoding))
|
||||
_ (auth-fetch-f upload-url headers body)]
|
||||
(p/recur (next remaining) false)))
|
||||
nil)))
|
||||
|
||||
(defn set-graph-sync-metadata!
|
||||
[repo graph-e2ee?]
|
||||
(when-let [conn (worker-state/get-datascript-conn repo)]
|
||||
@@ -126,9 +213,16 @@
|
||||
(fn [batch]
|
||||
(p/let [datoms* (sync-large-title/offload-large-titles-in-datoms-batch
|
||||
repo graph-id batch aes-key sync-apply/upload-large-title!)
|
||||
{:keys [kept dropped]} (drop-oversized-upload-datoms datoms*)
|
||||
_ (when (seq dropped)
|
||||
(prn :db-sync/drop-oversized-upload-datoms
|
||||
{:repo repo
|
||||
:count (count dropped)
|
||||
:attrs (vec (distinct (map :a dropped)))
|
||||
:max-bytes (apply max (map :bytes dropped))}))
|
||||
encrypted-datoms (if aes-key
|
||||
(sync-crypt/<encrypt-datoms aes-key datoms*)
|
||||
datoms*)
|
||||
(sync-crypt/<encrypt-datoms aes-key kept)
|
||||
kept)
|
||||
tx-data (mapv sync-large-title/datom->tx encrypted-datoms)]
|
||||
(d/transact! (:conn temp) tx-data {:initial-db? true})
|
||||
nil))
|
||||
@@ -186,25 +280,38 @@
|
||||
rows* (normalize-snapshot-rows rows)
|
||||
loaded' (+ loaded (count rows*))
|
||||
finished? (= loaded' total-rows)
|
||||
upload-url (str base "/sync/" graph-id "/snapshot/upload?reset="
|
||||
(if first-batch? "true" "false")
|
||||
"&finished="
|
||||
(if finished? "true" "false")
|
||||
(when finished?
|
||||
(str "&checksum=" (js/encodeURIComponent snapshot-checksum))))]
|
||||
(p/let [{:keys [body encoding]} (<snapshot-upload-body rows*)
|
||||
headers (cond-> {"content-type" snapshot-content-type}
|
||||
(string? encoding) (assoc "content-encoding" encoding))
|
||||
_ (sync-transport/fetch-json
|
||||
(fn [opts]
|
||||
(sync-auth/with-auth-headers
|
||||
#(sync-auth/auth-headers (worker-state/get-id-token))
|
||||
opts))
|
||||
upload-url
|
||||
{:method "POST"
|
||||
:headers headers
|
||||
:body body}
|
||||
{:response-schema :sync/snapshot-upload})]
|
||||
row-batches (split-snapshot-rows-by-max-bytes rows* snapshot-upload-max-bytes)
|
||||
batch-payloads
|
||||
(mapv (fn [rows-batch]
|
||||
{:rows (count rows-batch)
|
||||
:payload-bytes (snapshot-rows-byte-length rows-batch)})
|
||||
row-batches)]
|
||||
(prn :db-sync/upload-kvs-batch
|
||||
{:total-kvs-rows total-rows
|
||||
:fetched-kvs-rows (count rows*)
|
||||
:upload-kvs-batch-size upload-kvs-batch-size
|
||||
:split-batch-count (count row-batches)
|
||||
:split-batches batch-payloads
|
||||
:max-request-bytes snapshot-upload-max-bytes})
|
||||
(p/let [_ (<upload-snapshot-rows-batches!
|
||||
row-batches
|
||||
{:base base
|
||||
:graph-id graph-id
|
||||
:first-batch? first-batch?
|
||||
:finished? finished?
|
||||
:checksum snapshot-checksum
|
||||
:auth-fetch-f
|
||||
(fn [upload-url headers body]
|
||||
(sync-transport/fetch-json
|
||||
(fn [opts]
|
||||
(sync-auth/with-auth-headers
|
||||
#(sync-auth/auth-headers (worker-state/get-id-token))
|
||||
opts))
|
||||
upload-url
|
||||
{:method "POST"
|
||||
:headers headers
|
||||
:body body}
|
||||
{:response-schema :sync/snapshot-upload}))})]
|
||||
(update-progress {:sub-type :upload-progress
|
||||
:message (str "Uploading " loaded' "/" total-rows)})
|
||||
(p/recur max-addr false loaded'))))))
|
||||
|
||||
@@ -1663,11 +1663,11 @@
|
||||
:settings.sync-server/url "Sync Server URL"
|
||||
:settings.sync-server/url-desc "Set a custom HTTPS sync server URL for self-hosted sync. Your Logseq authentication tokens will be sent to this server, so only use a trusted URL. Leave empty to use the official Logseq Sync."
|
||||
:settings.sync-server/url-invalid-error "URL must start with https:// or http://"
|
||||
:settings-page/publish-server-url "Publish server URL"
|
||||
:settings-page/publish-server-url-desc "Set a custom HTTPS publish server URL for self-hosted single-page publishing. Your Logseq authentication tokens will be sent to this server, so only use a trusted URL. Leave empty to use the official Logseq publish service."
|
||||
:settings-page/publish-server-url "Publish Server URL"
|
||||
:settings-page/publish-server-url-desc "Set a custom HTTPS publish server URL for self-hosted single-page publishing. Your Logseq authentication tokens will be sent to this server, so only use a trusted URL. Leave empty to use the official Logseq Publish service."
|
||||
:settings-page/publish-server-url-saved "Publish server URL saved."
|
||||
:settings-page/publish-server-url-cleared "Publish server URL cleared. Using official Logseq publish."
|
||||
:settings-page/publish-server-url-default "Logseq publish"
|
||||
:settings-page/publish-server-url-cleared "Publish server URL cleared. Using official Logseq Publish."
|
||||
:settings-page/publish-server-url-default "Logseq Publish"
|
||||
:settings-page/publish-server-url-reset "Reset to default"
|
||||
|
||||
:shell/input-command-title "Input command"
|
||||
|
||||
@@ -2,9 +2,12 @@
|
||||
(:require [cljs.test :refer [deftest is testing]]
|
||||
[datascript.core :as d]
|
||||
[frontend.worker.pipeline :as worker-pipeline]
|
||||
[logseq.common.util :as common-util]
|
||||
[logseq.common.util.date-time :as date-time-util]
|
||||
[logseq.db :as ldb]
|
||||
[logseq.db.common.order :as db-order]
|
||||
[logseq.db.test.helper :as db-test]))
|
||||
[logseq.db.test.helper :as db-test]
|
||||
[logseq.outliner.page :as outliner-page]))
|
||||
|
||||
(deftest test-built-in-page-updates-that-should-be-reverted
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
@@ -145,6 +148,21 @@
|
||||
(is (= "page1-renamed"
|
||||
(:block/title (d/entity (:db-after result) (:db/id page1)))))))))
|
||||
|
||||
(deftest create-journal-page-name-uses-default-formatter-test
|
||||
(let [conn (db-test/create-conn)]
|
||||
(d/transact! conn [[:db/add :logseq.class/Journal :logseq.property.journal/title-format "yyyy-MM-dd EEEE"]])
|
||||
(let [[_ page-uuid] (outliner-page/create! conn "Dec 16th, 2024" {})
|
||||
page (d/entity @conn [:block/uuid page-uuid])
|
||||
journal-day (:block/journal-day page)
|
||||
expected-title (date-time-util/int->journal-title journal-day "yyyy-MM-dd EEEE")
|
||||
expected-name (-> journal-day
|
||||
(date-time-util/int->journal-title date-time-util/default-journal-title-formatter)
|
||||
common-util/page-name-sanity-lc)]
|
||||
(is (= expected-title (:block/title page))
|
||||
"Journal title follows configured title format")
|
||||
(is (= expected-name (:block/name page))
|
||||
"Journal block/name keeps the default formatter for stable identity"))))
|
||||
|
||||
(deftest built-in-tag-must-not-convert-page-child-block-to-class-test
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks [{:page {:block/title "page1"}}]})
|
||||
|
||||
93
src/test/frontend/worker/sync/upload_test.cljs
Normal file
93
src/test/frontend/worker/sync/upload_test.cljs
Normal file
@@ -0,0 +1,93 @@
|
||||
(ns frontend.worker.sync.upload-test
|
||||
(:require [cljs.test :refer [async deftest is]]
|
||||
[frontend.worker.sync.upload :as sync-upload]
|
||||
[promesa.core :as p]
|
||||
[clojure.string :as string]))
|
||||
|
||||
(deftest split-snapshot-rows-by-max-bytes-splits-rows-into-byte-capped-batches-test
|
||||
(let [sizes {:a 4
|
||||
:b 4
|
||||
:c 4
|
||||
:d 4}
|
||||
rows [[:a] [:b] [:c] [:d]]]
|
||||
(with-redefs [sync-upload/snapshot-rows-byte-length
|
||||
(fn [rows']
|
||||
(reduce + (map (fn [[addr]] (get sizes addr 0)) rows')))]
|
||||
(is (= [[[:a] [:b]]
|
||||
[[:c] [:d]]]
|
||||
(#'sync-upload/split-snapshot-rows-by-max-bytes rows 8))))))
|
||||
|
||||
(deftest split-snapshot-rows-by-max-bytes-fails-fast-for-oversized-single-row-test
|
||||
(let [sizes {:ok 3
|
||||
:too-big 11}
|
||||
rows [[:ok] [:too-big]]]
|
||||
(with-redefs [sync-upload/snapshot-rows-byte-length
|
||||
(fn [rows']
|
||||
(reduce + (map (fn [[addr]] (get sizes addr 0)) rows')))]
|
||||
(try
|
||||
(#'sync-upload/split-snapshot-rows-by-max-bytes rows 10)
|
||||
(is false "expected snapshot row too large error")
|
||||
(catch :default error
|
||||
(let [data (ex-data error)]
|
||||
(is (= "snapshot-row-too-large" (ex-message error)))
|
||||
(is (= 10 (:max-bytes data)))
|
||||
(is (= 11 (:row-size data)))
|
||||
(is (= :too-big (:addr data)))))))))
|
||||
|
||||
(deftest upload-snapshot-rows-batches-sets-reset-and-finished-flags-correctly-test
|
||||
(async done
|
||||
(let [calls* (atom [])
|
||||
rows-batches [[[1 "a" nil]]
|
||||
[[2 "b" nil]]
|
||||
[[3 "c" nil]]]]
|
||||
(-> (p/with-redefs [sync-upload/<snapshot-upload-body
|
||||
(fn [rows]
|
||||
(p/resolved {:body rows
|
||||
:encoding nil}))]
|
||||
(#'sync-upload/<upload-snapshot-rows-batches!
|
||||
rows-batches
|
||||
{:base "https://sync.example.test"
|
||||
:graph-id "graph-1"
|
||||
:first-batch? true
|
||||
:finished? true
|
||||
:checksum "abc+123="
|
||||
:auth-fetch-f
|
||||
(fn [url headers body]
|
||||
(swap! calls* conj {:url url
|
||||
:headers headers
|
||||
:body body})
|
||||
(p/resolved true))}))
|
||||
(p/then
|
||||
(fn [_]
|
||||
(is (= 3 (count @calls*)))
|
||||
(is (string/includes? (:url (nth @calls* 0)) "reset=true"))
|
||||
(is (string/includes? (:url (nth @calls* 0)) "finished=false"))
|
||||
(is (string/includes? (:url (nth @calls* 1)) "reset=false"))
|
||||
(is (string/includes? (:url (nth @calls* 1)) "finished=false"))
|
||||
(is (string/includes? (:url (nth @calls* 2)) "reset=false"))
|
||||
(is (string/includes? (:url (nth @calls* 2)) "finished=true"))
|
||||
(is (string/includes? (:url (nth @calls* 2)) "checksum=abc%2B123%3D"))
|
||||
(done)))
|
||||
(p/catch
|
||||
(fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest drop-oversized-upload-datoms-drops-large-tldraw-page-values-test
|
||||
(let [datoms [{:e 1 :a :block/title :v "safe"}
|
||||
{:e 2 :a :logseq.property.tldraw/page :v {:id "small"}}
|
||||
{:e 3 :a :logseq.property.tldraw/page :v {:id "huge"}}]]
|
||||
(with-redefs [sync-upload/datom-value-byte-length
|
||||
(fn [value]
|
||||
(case (:id value)
|
||||
"small" 32
|
||||
"huge" 1500000
|
||||
0))]
|
||||
(let [{:keys [kept dropped]} (#'sync-upload/drop-oversized-upload-datoms datoms)]
|
||||
(is (= 2 (count kept)))
|
||||
(is (= [1 2] (mapv :e kept)))
|
||||
(is (= 1 (count dropped)))
|
||||
(is (= {:a :logseq.property.tldraw/page
|
||||
:e 3
|
||||
:bytes 1500000}
|
||||
(first dropped)))))))
|
||||
Reference in New Issue
Block a user