mirror of
https://github.com/logseq/logseq.git
synced 2026-05-19 02:12:41 +00:00
feat(cli): add cmd 'sync asset download'
This commit is contained in:
rcmerci
@@ -330,6 +330,13 @@
|
||||
:base base
|
||||
:graph-id graph-id})))))
|
||||
|
||||
(defn log-request-asset-download-failed!
|
||||
[repo asset-uuid error]
|
||||
(log/error :db-sync/request-asset-download-failed
|
||||
{:repo repo
|
||||
:asset-uuid asset-uuid
|
||||
:error error}))
|
||||
|
||||
(defn request-asset-download!
|
||||
[repo asset-uuid {:keys [current-client-f enqueue-asset-task-f broadcast-rtc-state!-f]}]
|
||||
(when-let [client (current-client-f repo)]
|
||||
@@ -355,4 +362,5 @@
|
||||
(broadcast-rtc-state!-f client))]
|
||||
nil)
|
||||
(p/catch (fn [e]
|
||||
(js/console.error e)))))))))))
|
||||
(log-request-asset-download-failed! repo asset-uuid e)
|
||||
(p/rejected e)))))))))))
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
(ns logseq.cli.command.sync
|
||||
"Sync-related CLI commands."
|
||||
(:require [clojure.string :as string]
|
||||
(:require ["crypto" :as crypto]
|
||||
["fs" :as fs]
|
||||
["path" :as node-path]
|
||||
[clojure.string :as string]
|
||||
[lambdaisland.glogi :as log]
|
||||
[logseq.cli.auth :as cli-auth]
|
||||
[logseq.cli.command.core :as core]
|
||||
@@ -10,6 +13,7 @@
|
||||
[logseq.cli.server :as cli-server]
|
||||
[logseq.cli.transport :as transport]
|
||||
[logseq.common.cognito-config :as cognito-config]
|
||||
[logseq.common.graph-dir :as graph-dir]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(def ^:private sync-grant-access-spec
|
||||
@@ -30,6 +34,14 @@
|
||||
:e2ee-password {:desc "Verify and persist E2EE password before download"
|
||||
:coerce :string}})
|
||||
|
||||
(def ^:private sync-asset-download-spec
|
||||
{:id {:desc "Target asset node db/id"
|
||||
:coerce :long}
|
||||
:uuid {:desc "Target asset block UUID"
|
||||
:coerce :string
|
||||
:validate {:pred (comp parse-uuid str)
|
||||
:ex-msg (constantly "Option uuid must be a valid UUID string")}}})
|
||||
|
||||
(def ^:private sync-ensure-keys-spec
|
||||
{:e2ee-password {:desc "Verify and persist E2EE password before ensuring user RSA keys"
|
||||
:coerce :string}
|
||||
@@ -51,6 +63,9 @@
|
||||
{:examples ["logseq sync download --graph my-graph"
|
||||
"logseq sync download --graph my-graph --progress"
|
||||
"logseq sync download --graph my-graph --e2ee-password \"my-secret\""]})
|
||||
(core/command-entry ["sync" "asset" "download"] :sync-asset-download "Download remote asset" sync-asset-download-spec
|
||||
{:examples ["logseq sync asset download --graph my-graph --id 123"
|
||||
"logseq sync asset download --graph my-graph --uuid <asset-uuid>"]})
|
||||
(core/command-entry ["sync" "remote-graphs"] :sync-remote-graphs "List remote graphs" {})
|
||||
(core/command-entry ["sync" "ensure-keys"] :sync-ensure-keys "Ensure user RSA keys for sync/e2ee" sync-ensure-keys-spec
|
||||
{:examples ["logseq sync ensure-keys"
|
||||
@@ -77,6 +92,7 @@
|
||||
#{:sync-start
|
||||
:sync-upload
|
||||
:sync-download
|
||||
:sync-asset-download
|
||||
:sync-remote-graphs
|
||||
:sync-ensure-keys
|
||||
:sync-grant-access})
|
||||
@@ -97,6 +113,7 @@
|
||||
{:sync-start [:ws-url]
|
||||
:sync-upload [:http-base]
|
||||
:sync-download [:http-base]
|
||||
:sync-asset-download [:http-base]
|
||||
:sync-grant-access [:http-base]})
|
||||
|
||||
(defn- config-value-present?
|
||||
@@ -160,6 +177,18 @@
|
||||
[?e :db/ident :logseq.kv/graph-rtc-e2ee?]
|
||||
[?e :kv/value ?v]])
|
||||
|
||||
(def ^:private asset-tag-ident
|
||||
:logseq.class/Asset)
|
||||
|
||||
(def ^:private sync-asset-pull-selector
|
||||
[:db/id
|
||||
:block/uuid
|
||||
{:block/tags [:db/ident]}
|
||||
:logseq.property.asset/type
|
||||
:logseq.property.asset/checksum
|
||||
:logseq.property.asset/remote-metadata
|
||||
:logseq.property.asset/external-url])
|
||||
|
||||
(defn- missing-repo
|
||||
[label]
|
||||
{:ok? false
|
||||
@@ -305,6 +334,27 @@
|
||||
:progress-explicit? (contains? options :progress)
|
||||
:e2ee-password (:e2ee-password options)}}))
|
||||
|
||||
(defn- build-sync-asset-download-action
|
||||
[options repo]
|
||||
(let [id (:id options)
|
||||
asset-uuid (some-> (:uuid options) string/trim)
|
||||
id? (some? id)
|
||||
has-uuid? (seq asset-uuid)]
|
||||
(cond
|
||||
(not (seq repo))
|
||||
(missing-repo "sync asset download")
|
||||
|
||||
(not= 1 (count (filter true? [id? (boolean has-uuid?)])))
|
||||
(invalid-options "exactly one of --id or --uuid is required")
|
||||
|
||||
:else
|
||||
{:ok? true
|
||||
:action (cond-> {:type :sync-asset-download
|
||||
:repo repo
|
||||
:graph (core/repo->graph repo)}
|
||||
id? (assoc :id id)
|
||||
has-uuid? (assoc :uuid asset-uuid))})))
|
||||
|
||||
(defn- build-sync-ensure-keys-action
|
||||
[options]
|
||||
{:ok? true
|
||||
@@ -396,6 +446,9 @@
|
||||
:sync-download
|
||||
(build-sync-download-action options repo)
|
||||
|
||||
:sync-asset-download
|
||||
(build-sync-asset-download-action options repo)
|
||||
|
||||
:sync-remote-graphs
|
||||
{:ok? true
|
||||
:action {:type :sync-remote-graphs}}
|
||||
@@ -484,6 +537,167 @@
|
||||
result (transport/invoke cfg method args)]
|
||||
result))
|
||||
|
||||
(defn- asset-download-error
|
||||
[code message action extra]
|
||||
{:status :error
|
||||
:error (merge {:code code
|
||||
:message message
|
||||
:repo (:repo action)
|
||||
:graph (:graph action)}
|
||||
extra)})
|
||||
|
||||
(defn- sync-asset-lookup-ref
|
||||
[{:keys [id] :as action}]
|
||||
(let [asset-uuid (:uuid action)]
|
||||
(if (some? id)
|
||||
id
|
||||
[:block/uuid (uuid asset-uuid)])))
|
||||
|
||||
(defn- resolve-sync-asset
|
||||
[cfg action]
|
||||
(transport/invoke cfg :thread-api/pull
|
||||
[(:repo action)
|
||||
sync-asset-pull-selector
|
||||
(sync-asset-lookup-ref action)]))
|
||||
|
||||
(defn- asset-tag?
|
||||
[tag]
|
||||
(cond
|
||||
(= asset-tag-ident tag) true
|
||||
(map? tag) (= asset-tag-ident (:db/ident tag))
|
||||
:else false))
|
||||
|
||||
(defn- validate-sync-asset
|
||||
[action asset]
|
||||
(let [asset-uuid (:block/uuid asset)
|
||||
asset-type (:logseq.property.asset/type asset)
|
||||
checksum (:logseq.property.asset/checksum asset)]
|
||||
(cond
|
||||
(nil? asset)
|
||||
(asset-download-error :asset-not-found "asset not found" action nil)
|
||||
|
||||
(not-any? asset-tag? (:block/tags asset))
|
||||
(asset-download-error :not-asset "selected entity is not an asset" action {:asset-id (:db/id asset)})
|
||||
|
||||
(not (seq (some-> asset-uuid str string/trim)))
|
||||
(asset-download-error :asset-uuid-missing "asset uuid is missing" action {:asset-id (:db/id asset)})
|
||||
|
||||
(not (seq (some-> asset-type str string/trim)))
|
||||
(asset-download-error :asset-type-missing "asset type is missing" action {:asset-id (:db/id asset)
|
||||
:asset-uuid asset-uuid})
|
||||
|
||||
(not (seq (some-> checksum str string/trim)))
|
||||
(asset-download-error :asset-checksum-missing "asset checksum is missing" action {:asset-id (:db/id asset)
|
||||
:asset-uuid asset-uuid})
|
||||
|
||||
(nil? (:logseq.property.asset/remote-metadata asset))
|
||||
(asset-download-error :asset-not-remote "asset remote metadata is missing" action {:asset-id (:db/id asset)
|
||||
:asset-uuid asset-uuid})
|
||||
|
||||
(seq (some-> (:logseq.property.asset/external-url asset) str string/trim))
|
||||
(asset-download-error :external-asset "external URL assets cannot be downloaded through sync" action {:asset-id (:db/id asset)
|
||||
:asset-uuid asset-uuid})
|
||||
|
||||
:else
|
||||
{:status :ok
|
||||
:asset asset})))
|
||||
|
||||
(defn- sync-active?
|
||||
[status]
|
||||
(and (= :open (:ws-state status))
|
||||
(seq (some-> (:graph-id status) str string/trim))))
|
||||
|
||||
(defn- sync-not-started-error
|
||||
[action status]
|
||||
(asset-download-error :sync-not-started
|
||||
"sync is not started for this graph"
|
||||
action
|
||||
{:status status
|
||||
:hint (str "Run logseq sync start --graph " (:graph action) " first.")}))
|
||||
|
||||
(defn- asset-file-exists?
|
||||
[path]
|
||||
(and (seq path)
|
||||
(fs/existsSync path)))
|
||||
|
||||
(defn- asset-file-checksum
|
||||
[path]
|
||||
(-> (.createHash crypto "sha256")
|
||||
(.update (fs/readFileSync path))
|
||||
(.digest "hex")))
|
||||
|
||||
(defn- graph-asset-file-path
|
||||
[config repo asset-uuid asset-type]
|
||||
(if-let [graph-dir-name (graph-dir/repo->encoded-graph-dir-name repo)]
|
||||
(node-path/join (cli-server/graphs-dir config)
|
||||
graph-dir-name
|
||||
"assets"
|
||||
(str asset-uuid "." asset-type))
|
||||
(throw (ex-info "invalid repo"
|
||||
{:code :invalid-repo
|
||||
:repo repo}))))
|
||||
|
||||
(defn- asset-download-result-data
|
||||
[asset download-requested? checksum-status extra]
|
||||
(cond-> {:asset-uuid (str (:block/uuid asset))
|
||||
:asset-type (:logseq.property.asset/type asset)
|
||||
:download-requested? download-requested?
|
||||
:checksum-status checksum-status}
|
||||
(some? (:db/id asset)) (assoc :asset-id (:db/id asset))
|
||||
(seq extra) (merge extra)))
|
||||
|
||||
(defn- local-asset-checksum-status
|
||||
[config action asset]
|
||||
(let [asset-path (graph-asset-file-path config
|
||||
(:repo action)
|
||||
(:block/uuid asset)
|
||||
(:logseq.property.asset/type asset))]
|
||||
(if-not (asset-file-exists? asset-path)
|
||||
{:checksum-status :missing}
|
||||
(let [local-checksum (asset-file-checksum asset-path)]
|
||||
(if (= local-checksum (:logseq.property.asset/checksum asset))
|
||||
{:checksum-status :match}
|
||||
{:checksum-status :mismatch
|
||||
:local-path asset-path
|
||||
:local-checksum local-checksum})))))
|
||||
|
||||
(defn- remove-local-asset-file!
|
||||
[path]
|
||||
(when (asset-file-exists? path)
|
||||
(fs/rmSync path #js {:force true})))
|
||||
|
||||
(defn- request-asset-download-result
|
||||
[cfg action asset checksum-status extra]
|
||||
(p/let [_ (transport/invoke cfg :thread-api/db-sync-request-asset-download
|
||||
[(:repo action) (:block/uuid asset)])]
|
||||
{:status :ok
|
||||
:data (asset-download-result-data asset true checksum-status extra)}))
|
||||
|
||||
(defn- execute-sync-asset-download*
|
||||
[cfg config action asset]
|
||||
(let [validation (validate-sync-asset action asset)]
|
||||
(if (= :error (:status validation))
|
||||
(p/resolved validation)
|
||||
(p/let [status (transport/invoke cfg :thread-api/db-sync-status [(:repo action)])]
|
||||
(if-not (sync-active? status)
|
||||
(sync-not-started-error action status)
|
||||
(let [{:keys [checksum-status local-path]} (local-asset-checksum-status config action asset)]
|
||||
(case checksum-status
|
||||
:match
|
||||
{:status :ok
|
||||
:data (asset-download-result-data asset false :match {:skipped-reason :already-downloaded})}
|
||||
|
||||
:mismatch
|
||||
(do
|
||||
(remove-local-asset-file! local-path)
|
||||
(request-asset-download-result cfg
|
||||
action
|
||||
asset
|
||||
:mismatch
|
||||
{:hint "Local asset checksum mismatched; requested re-download."}))
|
||||
|
||||
(request-asset-download-result cfg action asset :missing nil))))))))
|
||||
|
||||
(defn- invoke-global
|
||||
[config method args]
|
||||
(let [base-url (:base-url config)]
|
||||
@@ -759,6 +973,21 @@
|
||||
(exception->error error {:repo (:repo action)
|
||||
:graph (:graph action)})))))
|
||||
|
||||
(defn- run-sync-asset-download
|
||||
[action config]
|
||||
(-> (p/let [config' (resolve-runtime-config! action config)
|
||||
missing-keys (missing-required-sync-config-keys (:type action) config')]
|
||||
(if (seq missing-keys)
|
||||
(missing-sync-config-error (:type action) missing-keys)
|
||||
(let [config* (assoc config' :http-base (effective-sync-config-value config' :http-base))]
|
||||
(p/let [cfg (cli-server/ensure-server! config* (:repo action))
|
||||
_ (<sync-worker-runtime! cfg config*)
|
||||
asset (resolve-sync-asset cfg action)]
|
||||
(execute-sync-asset-download* cfg config* action asset)))))
|
||||
(p/catch (fn [error]
|
||||
(exception->error error {:repo (:repo action)
|
||||
:graph (:graph action)})))))
|
||||
|
||||
(defn- run-sync-remote-graphs
|
||||
[action config]
|
||||
(-> (p/let [config' (resolve-runtime-config! action config)
|
||||
@@ -834,6 +1063,7 @@
|
||||
:sync-stop (run-sync-stop action config)
|
||||
:sync-upload (run-sync-upload action config)
|
||||
:sync-download (run-sync-download action config)
|
||||
:sync-asset-download (run-sync-asset-download action config)
|
||||
:sync-remote-graphs (run-sync-remote-graphs action config)
|
||||
:sync-ensure-keys (run-sync-ensure-keys action config)
|
||||
:sync-grant-access (run-sync-grant-access action config)
|
||||
|
||||
@@ -357,10 +357,15 @@
|
||||
(not (seq (:graph opts))))
|
||||
(missing-graph-result summary)
|
||||
|
||||
(and (= command :sync-download)
|
||||
(and (= :sync-download command)
|
||||
(not (seq (:graph opts))))
|
||||
(missing-graph-result summary)
|
||||
|
||||
(and (= command :sync-asset-download)
|
||||
(not= 1 (count (filter true? [(some? (:id opts))
|
||||
(boolean (seq (some-> (:uuid opts) string/trim)))]))))
|
||||
(command-core/invalid-options-result summary "exactly one of --id or --uuid is required")
|
||||
|
||||
(and (= command :completion)
|
||||
completion-shell-error)
|
||||
(command-core/invalid-options-result summary completion-shell-error)
|
||||
@@ -650,7 +655,7 @@
|
||||
(doctor-command/build-action options)
|
||||
|
||||
(:sync-status :sync-start :sync-stop :sync-upload :sync-download
|
||||
:sync-remote-graphs :sync-ensure-keys :sync-grant-access
|
||||
:sync-asset-download :sync-remote-graphs :sync-ensure-keys :sync-grant-access
|
||||
:sync-config-set :sync-config-get :sync-config-unset)
|
||||
(sync-command/build-action command options args repo)
|
||||
|
||||
@@ -740,7 +745,7 @@
|
||||
:server-stop (server-command/execute-stop action config)
|
||||
:server-restart (server-command/execute-restart action config)
|
||||
(:sync-status :sync-start :sync-stop :sync-upload :sync-download
|
||||
:sync-remote-graphs :sync-ensure-keys :sync-grant-access
|
||||
:sync-asset-download :sync-remote-graphs :sync-ensure-keys :sync-grant-access
|
||||
:sync-config-set :sync-config-get :sync-config-unset)
|
||||
(sync-command/execute action config)
|
||||
(:login :logout)
|
||||
|
||||
@@ -797,6 +797,19 @@
|
||||
:sync-grant-access (str "Sync access granted: " email " (repo: " repo ")")
|
||||
"Sync updated"))
|
||||
|
||||
(defn- format-sync-asset-download
|
||||
[{:keys [repo]} {:keys [asset-uuid download-requested? checksum-status hint]}]
|
||||
(cond
|
||||
(= :mismatch checksum-status)
|
||||
(str (or hint "Local asset checksum mismatched; requested re-download.")
|
||||
" " asset-uuid)
|
||||
|
||||
(false? download-requested?)
|
||||
(str "Sync asset already downloaded: " asset-uuid " (repo: " repo ")")
|
||||
|
||||
:else
|
||||
(str "Sync asset download requested: " asset-uuid " (repo: " repo ")")))
|
||||
|
||||
(defn- format-sync-config-get
|
||||
[{:keys [key value]}]
|
||||
(let [display-value (if (contains? #{:auth-token :e2ee-password} key)
|
||||
@@ -1001,6 +1014,7 @@
|
||||
:sync-remote-graphs (format-sync-remote-graphs (:graphs data))
|
||||
(:sync-start :sync-stop :sync-upload :sync-download :sync-ensure-keys :sync-grant-access)
|
||||
(format-sync-action command context)
|
||||
:sync-asset-download (format-sync-asset-download context data)
|
||||
:sync-config-get (format-sync-config-get data)
|
||||
:sync-config-set (format-sync-config-set data)
|
||||
:sync-config-unset (format-sync-config-unset data)
|
||||
|
||||
@@ -1,13 +1,137 @@
|
||||
(ns frontend.worker.sync.assets-test
|
||||
(:require [cljs.test :refer [async deftest is]]
|
||||
[datascript.core :as d]
|
||||
[frontend.common.crypt :as crypt]
|
||||
[frontend.worker.platform :as platform]
|
||||
[frontend.worker.shared-service :as shared-service]
|
||||
[frontend.worker.state :as worker-state]
|
||||
[frontend.worker.sync.assets :as sync-assets]
|
||||
[logseq.db :as ldb]
|
||||
[logseq.db.frontend.schema :as db-schema]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(defn- asset-conn
|
||||
[asset-uuid]
|
||||
(let [conn (d/create-conn db-schema/schema)]
|
||||
(ldb/transact! conn [{:block/uuid asset-uuid
|
||||
:logseq.property.asset/type "png"
|
||||
:logseq.property.asset/checksum "sha-256-value"
|
||||
:logseq.property.asset/remote-metadata {:checksum "sha-256-value"
|
||||
:type "png"}}])
|
||||
conn))
|
||||
|
||||
(defn- execute-enqueued-asset-task!
|
||||
[task]
|
||||
(if (fn? task)
|
||||
(task)
|
||||
(p/resolved nil)))
|
||||
|
||||
(deftest request-asset-download-skips-existing-local-asset-test
|
||||
(async done
|
||||
(let [repo "asset-download-repo"
|
||||
graph-id "graph-1"
|
||||
asset-uuid (random-uuid)
|
||||
conn (asset-conn asset-uuid)
|
||||
download-calls (atom [])
|
||||
asset-stat-calls (atom [])
|
||||
enqueued-task (atom nil)
|
||||
broadcast-calls (atom [])]
|
||||
(-> (p/with-redefs [worker-state/get-datascript-conn (fn [_repo]
|
||||
conn)
|
||||
platform/current (fn [] {})
|
||||
platform/asset-stat (fn [_platform repo' file-name]
|
||||
(swap! asset-stat-calls conj [repo' file-name])
|
||||
(p/resolved {:size 10}))
|
||||
sync-assets/download-remote-asset! (fn [& args]
|
||||
(swap! download-calls conj args)
|
||||
(p/resolved nil))]
|
||||
(sync-assets/request-asset-download!
|
||||
repo
|
||||
asset-uuid
|
||||
{:current-client-f (fn [_repo]
|
||||
{:graph-id graph-id})
|
||||
:enqueue-asset-task-f (fn [_client task]
|
||||
(reset! enqueued-task task)
|
||||
(execute-enqueued-asset-task! task))
|
||||
:broadcast-rtc-state!-f (fn [& args]
|
||||
(swap! broadcast-calls conj args))}))
|
||||
(p/then (fn [_]
|
||||
(is (= [[repo (str asset-uuid ".png")]] @asset-stat-calls))
|
||||
(is (fn? @enqueued-task))
|
||||
(is (= [] @download-calls))
|
||||
(is (= [] @broadcast-calls))))
|
||||
(p/catch (fn [error]
|
||||
(is false (str "unexpected error: " error))))
|
||||
(p/finally done)))))
|
||||
|
||||
(deftest request-asset-download-downloads-missing-local-asset-test
|
||||
(async done
|
||||
(let [repo "asset-download-repo"
|
||||
graph-id "graph-1"
|
||||
asset-uuid (random-uuid)
|
||||
conn (asset-conn asset-uuid)
|
||||
download-calls (atom [])
|
||||
asset-stat-calls (atom [])
|
||||
broadcast-calls (atom [])]
|
||||
(-> (p/with-redefs [worker-state/get-datascript-conn (fn [_repo]
|
||||
conn)
|
||||
platform/current (fn [] {})
|
||||
platform/asset-stat (fn [_platform repo' file-name]
|
||||
(swap! asset-stat-calls conj [repo' file-name])
|
||||
(p/resolved nil))
|
||||
sync-assets/download-remote-asset! (fn [& args]
|
||||
(swap! download-calls conj args)
|
||||
(p/resolved nil))]
|
||||
(sync-assets/request-asset-download!
|
||||
repo
|
||||
asset-uuid
|
||||
{:current-client-f (fn [_repo]
|
||||
{:graph-id graph-id})
|
||||
:enqueue-asset-task-f (fn [_client task]
|
||||
(execute-enqueued-asset-task! task))
|
||||
:broadcast-rtc-state!-f (fn [& args]
|
||||
(swap! broadcast-calls conj args))}))
|
||||
(p/then (fn [_]
|
||||
(is (= [[repo (str asset-uuid ".png")]] @asset-stat-calls))
|
||||
(is (= [[repo graph-id asset-uuid "png"]] @download-calls))
|
||||
(is (= 1 (count @broadcast-calls)))))
|
||||
(p/catch (fn [error]
|
||||
(is false (str "unexpected error: " error))))
|
||||
(p/finally done)))))
|
||||
|
||||
(deftest request-asset-download-propagates-and-logs-download-failure-test
|
||||
(async done
|
||||
(let [repo "asset-download-repo"
|
||||
graph-id "graph-1"
|
||||
asset-uuid (random-uuid)
|
||||
conn (asset-conn asset-uuid)
|
||||
download-error (ex-info "download failed" {:type :rtc.exception/download-asset-failed})
|
||||
log-calls (atom [])]
|
||||
(-> (p/with-redefs [worker-state/get-datascript-conn (fn [_repo]
|
||||
conn)
|
||||
platform/current (fn [] {})
|
||||
platform/asset-stat (fn [_platform _repo _file-name]
|
||||
(p/resolved nil))
|
||||
sync-assets/download-remote-asset! (fn [& _args]
|
||||
(p/rejected download-error))
|
||||
sync-assets/log-request-asset-download-failed!
|
||||
(fn [repo' asset-uuid' error']
|
||||
(swap! log-calls conj [repo' asset-uuid' error']))]
|
||||
(sync-assets/request-asset-download!
|
||||
repo
|
||||
asset-uuid
|
||||
{:current-client-f (fn [_repo]
|
||||
{:graph-id graph-id})
|
||||
:enqueue-asset-task-f (fn [_client task]
|
||||
(execute-enqueued-asset-task! task))
|
||||
:broadcast-rtc-state!-f (fn [& _args] nil)}))
|
||||
(p/then (fn [_]
|
||||
(is false "expected download failure to reject")))
|
||||
(p/catch (fn [error]
|
||||
(is (= download-error error))
|
||||
(is (= [[repo asset-uuid download-error]] @log-calls))))
|
||||
(p/finally done)))))
|
||||
|
||||
(deftest upload-remote-asset-serializes-resolved-encrypted-payload-test
|
||||
(async done
|
||||
(let [repo "asset-upload-repo"
|
||||
|
||||
@@ -1,17 +1,122 @@
|
||||
(ns logseq.cli.command.sync-test
|
||||
(:require [cljs.test :refer [async deftest is testing]]
|
||||
(:require ["crypto" :as crypto]
|
||||
["fs" :as fs]
|
||||
["os" :as os]
|
||||
["path" :as node-path]
|
||||
[cljs.test :refer [async deftest is testing]]
|
||||
[logseq.cli.auth :as cli-auth]
|
||||
[logseq.cli.command.sync :as sync-command]
|
||||
[logseq.cli.common :as cli-common]
|
||||
[logseq.cli.config :as cli-config]
|
||||
[logseq.cli.server :as cli-server]
|
||||
[logseq.cli.transport :as transport]
|
||||
[logseq.common.graph-dir :as graph-dir]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(defn- execute-with-runtime-auth
|
||||
[action config]
|
||||
(sync-command/execute action (assoc config :id-token "runtime-token")))
|
||||
|
||||
(def ^:private sync-asset-repo "logseq_db_demo")
|
||||
(def ^:private sync-asset-uuid "11111111-1111-1111-1111-111111111111")
|
||||
(def ^:private sync-asset-type "txt")
|
||||
|
||||
(defn- temp-root-dir
|
||||
[]
|
||||
(fs/mkdtempSync (node-path/join (os/tmpdir) "logseq-sync-asset-test-")))
|
||||
|
||||
(defn- remove-dir!
|
||||
[path]
|
||||
(when (and (seq path) (fs/existsSync path))
|
||||
(fs/rmSync path #js {:recursive true :force true})))
|
||||
|
||||
(defn- sha256
|
||||
[payload]
|
||||
(-> (.createHash crypto "sha256")
|
||||
(.update payload)
|
||||
(.digest "hex")))
|
||||
|
||||
(defn- graph-assets-dir
|
||||
[root-dir repo]
|
||||
(node-path/join (cli-server/graphs-dir {:root-dir root-dir})
|
||||
(graph-dir/repo->encoded-graph-dir-name repo)
|
||||
"assets"))
|
||||
|
||||
(defn- write-local-asset!
|
||||
[root-dir repo asset-uuid asset-type payload]
|
||||
(let [assets-dir (graph-assets-dir root-dir repo)
|
||||
asset-path (node-path/join assets-dir (str asset-uuid "." asset-type))]
|
||||
(fs/mkdirSync assets-dir #js {:recursive true})
|
||||
(fs/writeFileSync asset-path payload)
|
||||
asset-path))
|
||||
|
||||
(defn- remote-asset
|
||||
[checksum]
|
||||
{:db/id 123
|
||||
:block/uuid sync-asset-uuid
|
||||
:block/tags [{:db/ident :logseq.class/Asset}]
|
||||
:logseq.property.asset/type sync-asset-type
|
||||
:logseq.property.asset/checksum checksum
|
||||
:logseq.property.asset/remote-metadata {:checksum checksum
|
||||
:type sync-asset-type}})
|
||||
|
||||
(defn- active-sync-status
|
||||
[]
|
||||
{:repo sync-asset-repo
|
||||
:graph-id "graph-id"
|
||||
:ws-state :open
|
||||
:pending-local 0
|
||||
:pending-asset 0
|
||||
:pending-server 0})
|
||||
|
||||
(defn- sync-asset-download-action
|
||||
[]
|
||||
{:type :sync-asset-download
|
||||
:repo sync-asset-repo
|
||||
:graph "demo"
|
||||
:id 123})
|
||||
|
||||
(defn- run-sync-asset-download-scenario
|
||||
[{:keys [asset status local-payload action config]
|
||||
:or {status (active-sync-status)
|
||||
action (sync-asset-download-action)}}]
|
||||
(let [root-dir (temp-root-dir)
|
||||
calls (atom [])
|
||||
config' (merge {:root-dir root-dir
|
||||
:http-base "https://api.logseq.io"}
|
||||
config)]
|
||||
(when (some? local-payload)
|
||||
(write-local-asset! root-dir
|
||||
(:repo action)
|
||||
(or (:block/uuid asset) sync-asset-uuid)
|
||||
(or (:logseq.property.asset/type asset) sync-asset-type)
|
||||
local-payload))
|
||||
(-> (p/with-redefs [cli-server/ensure-server! (fn [config repo]
|
||||
(swap! calls conj [:ensure-server repo])
|
||||
(p/resolved (assoc config :base-url "http://example")))
|
||||
transport/invoke (fn [_ method args]
|
||||
(swap! calls conj [method args])
|
||||
(case method
|
||||
:thread-api/pull
|
||||
(p/resolved asset)
|
||||
|
||||
:thread-api/db-sync-status
|
||||
(p/resolved status)
|
||||
|
||||
:thread-api/db-sync-request-asset-download
|
||||
(p/resolved nil)
|
||||
|
||||
(p/resolved nil)))]
|
||||
(p/let [result (execute-with-runtime-auth action config')]
|
||||
{:result result
|
||||
:calls @calls}))
|
||||
(p/finally (fn []
|
||||
(remove-dir! root-dir))))))
|
||||
|
||||
(defn- called-method?
|
||||
[calls method]
|
||||
(boolean (some #(= method (first %)) calls)))
|
||||
|
||||
(deftest test-build-action-validation
|
||||
(testing "sync status requires repo"
|
||||
(let [result (sync-command/build-action :sync-status {} [] nil)]
|
||||
@@ -92,6 +197,190 @@
|
||||
(is (false? (:ok? missing-email)))
|
||||
(is (= :invalid-options (get-in missing-email [:error :code]))))))
|
||||
|
||||
(deftest test-build-sync-asset-download-action-validation
|
||||
(testing "sync asset download builds action with db id selector"
|
||||
(let [result (sync-command/build-action :sync-asset-download {:id 123} [] "logseq_db_demo")]
|
||||
(is (true? (:ok? result)))
|
||||
(is (= {:type :sync-asset-download
|
||||
:repo "logseq_db_demo"
|
||||
:graph "demo"
|
||||
:id 123}
|
||||
(:action result)))))
|
||||
|
||||
(testing "sync asset download builds action with uuid selector"
|
||||
(let [result (sync-command/build-action :sync-asset-download {:uuid sync-asset-uuid} [] "logseq_db_demo")]
|
||||
(is (true? (:ok? result)))
|
||||
(is (= {:type :sync-asset-download
|
||||
:repo "logseq_db_demo"
|
||||
:graph "demo"
|
||||
:uuid sync-asset-uuid}
|
||||
(:action result)))))
|
||||
|
||||
(testing "sync asset download requires repo"
|
||||
(let [result (sync-command/build-action :sync-asset-download {:id 123} [] nil)]
|
||||
(is (false? (:ok? result)))
|
||||
(is (= :missing-repo (get-in result [:error :code])))))
|
||||
|
||||
(testing "sync asset download requires one selector"
|
||||
(let [result (sync-command/build-action :sync-asset-download {} [] "logseq_db_demo")]
|
||||
(is (false? (:ok? result)))
|
||||
(is (= :invalid-options (get-in result [:error :code])))))
|
||||
|
||||
(testing "sync asset download rejects conflicting selectors"
|
||||
(let [result (sync-command/build-action :sync-asset-download {:id 123
|
||||
:uuid sync-asset-uuid}
|
||||
[]
|
||||
"logseq_db_demo")]
|
||||
(is (false? (:ok? result)))
|
||||
(is (= :invalid-options (get-in result [:error :code]))))))
|
||||
|
||||
(deftest test-execute-sync-asset-download-uses-uuid-lookup-value
|
||||
(async done
|
||||
(let [checksum (sha256 "remote asset payload")
|
||||
action {:type :sync-asset-download
|
||||
:repo sync-asset-repo
|
||||
:graph "demo"
|
||||
:uuid sync-asset-uuid}]
|
||||
(-> (run-sync-asset-download-scenario {:asset (remote-asset checksum)
|
||||
:action action})
|
||||
(p/then (fn [{:keys [calls]}]
|
||||
(is (some #(and (= :thread-api/pull (first %))
|
||||
(= [:block/uuid (uuid sync-asset-uuid)]
|
||||
(get-in % [1 2])))
|
||||
calls))
|
||||
(is (some #(= [:thread-api/db-sync-request-asset-download
|
||||
[sync-asset-repo sync-asset-uuid]]
|
||||
%)
|
||||
calls))))
|
||||
(p/catch (fn [e]
|
||||
(is false (str "unexpected error: " e))))
|
||||
(p/finally done)))))
|
||||
|
||||
(deftest test-execute-sync-asset-download-requests-missing-local-file
|
||||
(async done
|
||||
(let [checksum (sha256 "remote asset payload")]
|
||||
(-> (run-sync-asset-download-scenario {:asset (remote-asset checksum)})
|
||||
(p/then (fn [{:keys [result calls]}]
|
||||
(is (= :ok (:status result)))
|
||||
(is (= {:asset-id 123
|
||||
:asset-uuid sync-asset-uuid
|
||||
:asset-type sync-asset-type
|
||||
:download-requested? true
|
||||
:checksum-status :missing}
|
||||
(:data result)))
|
||||
(is (called-method? calls :thread-api/sync-app-state))
|
||||
(is (called-method? calls :thread-api/set-db-sync-config))
|
||||
(is (called-method? calls :thread-api/pull))
|
||||
(is (called-method? calls :thread-api/db-sync-status))
|
||||
(is (some #(= [:thread-api/db-sync-request-asset-download
|
||||
[sync-asset-repo sync-asset-uuid]]
|
||||
%)
|
||||
calls))))
|
||||
(p/catch (fn [e]
|
||||
(is false (str "unexpected error: " e))))
|
||||
(p/finally done)))))
|
||||
|
||||
(deftest test-execute-sync-asset-download-skips-matching-local-file
|
||||
(async done
|
||||
(let [payload "local asset payload"
|
||||
checksum (sha256 payload)]
|
||||
(-> (run-sync-asset-download-scenario {:asset (remote-asset checksum)
|
||||
:local-payload payload})
|
||||
(p/then (fn [{:keys [result calls]}]
|
||||
(is (= :ok (:status result)))
|
||||
(is (= {:asset-id 123
|
||||
:asset-uuid sync-asset-uuid
|
||||
:asset-type sync-asset-type
|
||||
:download-requested? false
|
||||
:checksum-status :match
|
||||
:skipped-reason :already-downloaded}
|
||||
(:data result)))
|
||||
(is (not (contains? (:data result) :local-path)))
|
||||
(is (not (called-method? calls :thread-api/db-sync-request-asset-download)))))
|
||||
(p/catch (fn [e]
|
||||
(is false (str "unexpected error: " e))))
|
||||
(p/finally done)))))
|
||||
|
||||
(deftest test-execute-sync-asset-download-requests-mismatched-local-file
|
||||
(async done
|
||||
(let [checksum (sha256 "remote asset payload")]
|
||||
(-> (run-sync-asset-download-scenario {:asset (remote-asset checksum)
|
||||
:local-payload "corrupted local payload"})
|
||||
(p/then (fn [{:keys [result calls]}]
|
||||
(is (= :ok (:status result)))
|
||||
(is (= 123 (get-in result [:data :asset-id])))
|
||||
(is (= sync-asset-uuid (get-in result [:data :asset-uuid])))
|
||||
(is (= sync-asset-type (get-in result [:data :asset-type])))
|
||||
(is (= true (get-in result [:data :download-requested?])))
|
||||
(is (= :mismatch (get-in result [:data :checksum-status])))
|
||||
(let [hint (get-in result [:data :hint])]
|
||||
(is (string? hint))
|
||||
(is (boolean (when (string? hint)
|
||||
(re-find #"checksum" hint)))))
|
||||
(is (not (contains? (:data result) :local-path)))
|
||||
(is (some #(= [:thread-api/db-sync-request-asset-download
|
||||
[sync-asset-repo sync-asset-uuid]]
|
||||
%)
|
||||
calls))))
|
||||
(p/catch (fn [e]
|
||||
(is false (str "unexpected error: " e))))
|
||||
(p/finally done)))))
|
||||
|
||||
(deftest test-execute-sync-asset-download-requires-active-sync
|
||||
(async done
|
||||
(let [checksum (sha256 "remote asset payload")]
|
||||
(-> (run-sync-asset-download-scenario {:asset (remote-asset checksum)
|
||||
:status {:repo sync-asset-repo
|
||||
:graph-id "graph-id"
|
||||
:ws-state :stopped}})
|
||||
(p/then (fn [{:keys [result calls]}]
|
||||
(is (= :error (:status result)))
|
||||
(is (= :sync-not-started (get-in result [:error :code])))
|
||||
(is (= "Run logseq sync start --graph demo first."
|
||||
(get-in result [:error :hint])))
|
||||
(is (not (called-method? calls :thread-api/db-sync-request-asset-download)))))
|
||||
(p/catch (fn [e]
|
||||
(is false (str "unexpected error: " e))))
|
||||
(p/finally done)))))
|
||||
|
||||
(deftest test-execute-sync-asset-download-validates-asset-metadata
|
||||
(async done
|
||||
(let [checksum (sha256 "remote asset payload")
|
||||
base-asset (remote-asset checksum)
|
||||
cases [{:label "missing asset"
|
||||
:asset nil
|
||||
:code :asset-not-found}
|
||||
{:label "non asset"
|
||||
:asset (assoc base-asset :block/tags [])
|
||||
:code :not-asset}
|
||||
{:label "missing uuid"
|
||||
:asset (dissoc base-asset :block/uuid)
|
||||
:code :asset-uuid-missing}
|
||||
{:label "missing type"
|
||||
:asset (dissoc base-asset :logseq.property.asset/type)
|
||||
:code :asset-type-missing}
|
||||
{:label "missing checksum"
|
||||
:asset (dissoc base-asset :logseq.property.asset/checksum)
|
||||
:code :asset-checksum-missing}
|
||||
{:label "missing remote metadata"
|
||||
:asset (dissoc base-asset :logseq.property.asset/remote-metadata)
|
||||
:code :asset-not-remote}
|
||||
{:label "external asset"
|
||||
:asset (assoc base-asset :logseq.property.asset/external-url "https://example.com/a.txt")
|
||||
:code :external-asset}]]
|
||||
(-> (reduce (fn [chain {:keys [label asset code]}]
|
||||
(p/then chain
|
||||
(fn []
|
||||
(p/let [{:keys [result calls]} (run-sync-asset-download-scenario {:asset asset})]
|
||||
(is (= :error (:status result)) label)
|
||||
(is (= code (get-in result [:error :code])) label)
|
||||
(is (not (called-method? calls :thread-api/db-sync-request-asset-download)) label)))))
|
||||
(p/resolved nil)
|
||||
cases)
|
||||
(p/catch (fn [e]
|
||||
(is false (str "unexpected error: " e))))
|
||||
(p/finally done)))))
|
||||
|
||||
(deftest test-execute-sync-start
|
||||
(async done
|
||||
(let [ensure-calls (atom [])
|
||||
|
||||
@@ -2304,6 +2304,48 @@
|
||||
(is (true? (:ok? enabled)))
|
||||
(is (= true (get-in enabled [:options :progress])))))
|
||||
|
||||
(testing "sync asset download parses db id selector"
|
||||
(let [result (commands/parse-args ["sync" "asset" "download" "--graph" "demo" "--id" "123"])]
|
||||
(is (true? (:ok? result)))
|
||||
(is (= :sync-asset-download (:command result)))
|
||||
(is (= "demo" (get-in result [:options :graph])))
|
||||
(is (= 123 (get-in result [:options :id])))))
|
||||
|
||||
(testing "sync asset download parses uuid selector"
|
||||
(let [asset-uuid "11111111-1111-1111-1111-111111111111"
|
||||
result (commands/parse-args ["sync" "asset" "download" "--graph" "demo" "--uuid" asset-uuid])]
|
||||
(is (true? (:ok? result)))
|
||||
(is (= :sync-asset-download (:command result)))
|
||||
(is (= "demo" (get-in result [:options :graph])))
|
||||
(is (= asset-uuid (get-in result [:options :uuid])))))
|
||||
|
||||
(testing "sync asset download rejects invalid uuid selector"
|
||||
(let [result (commands/parse-args ["sync" "asset" "download" "--graph" "demo" "--uuid" "asset-uuid"])]
|
||||
(is (false? (:ok? result)))
|
||||
(is (= :invalid-options (get-in result [:error :code])))))
|
||||
|
||||
(testing "sync asset download can use current graph"
|
||||
(let [parsed (commands/parse-args ["sync" "asset" "download" "--id" "123"])
|
||||
result (when (:ok? parsed)
|
||||
(commands/build-action parsed {:graph "demo"}))]
|
||||
(is (true? (:ok? parsed)))
|
||||
(is (true? (:ok? result)))
|
||||
(is (= {:type :sync-asset-download
|
||||
:repo "logseq_db_demo"
|
||||
:graph "demo"
|
||||
:id 123}
|
||||
(:action result)))))
|
||||
|
||||
(testing "sync asset download requires one selector"
|
||||
(let [missing-selector (commands/parse-args ["sync" "asset" "download" "--graph" "demo"])
|
||||
conflicting-selectors (commands/parse-args ["sync" "asset" "download" "--graph" "demo"
|
||||
"--id" "123"
|
||||
"--uuid" "11111111-1111-1111-1111-111111111111"])]
|
||||
(is (false? (:ok? missing-selector)))
|
||||
(is (= :invalid-options (get-in missing-selector [:error :code])))
|
||||
(is (false? (:ok? conflicting-selectors)))
|
||||
(is (= :invalid-options (get-in conflicting-selectors [:error :code])))))
|
||||
|
||||
(testing "sync ensure-keys accepts e2ee-password option"
|
||||
(let [result (commands/parse-args ["sync" "ensure-keys" "--e2ee-password" "pw"])]
|
||||
(is (true? (:ok? result)))
|
||||
|
||||
@@ -950,7 +950,72 @@
|
||||
:context {:repo "demo-graph"}}
|
||||
{:output-format nil})]
|
||||
(is (string/includes? result "Sync download"))
|
||||
(is (string/includes? result "demo-graph")))))
|
||||
(is (string/includes? result "demo-graph"))))
|
||||
|
||||
(testing "sync asset download renders requested output"
|
||||
(let [result (format/format-result {:status :ok
|
||||
:command :sync-asset-download
|
||||
:context {:repo "demo-graph"}
|
||||
:data {:asset-id 123
|
||||
:asset-uuid "asset-uuid"
|
||||
:asset-type "png"
|
||||
:download-requested? true
|
||||
:checksum-status :missing}}
|
||||
{:output-format nil})]
|
||||
(is (string/includes? result "Sync asset download requested"))
|
||||
(is (string/includes? result "asset-uuid"))
|
||||
(is (string/includes? result "demo-graph"))
|
||||
(is (not (string/includes? result "local-path")))))
|
||||
|
||||
(testing "sync asset download renders checksum mismatch hint"
|
||||
(let [result (format/format-result {:status :ok
|
||||
:command :sync-asset-download
|
||||
:context {:repo "demo-graph"}
|
||||
:data {:asset-id 123
|
||||
:asset-uuid "asset-uuid"
|
||||
:asset-type "png"
|
||||
:download-requested? true
|
||||
:checksum-status :mismatch
|
||||
:hint "Local asset checksum mismatched; requested re-download."}}
|
||||
{:output-format nil})]
|
||||
(is (string/includes? result "Local asset checksum mismatched"))
|
||||
(is (string/includes? result "asset-uuid"))
|
||||
(is (not (string/includes? result "local-path")))))
|
||||
|
||||
(testing "sync asset download renders skipped output"
|
||||
(let [result (format/format-result {:status :ok
|
||||
:command :sync-asset-download
|
||||
:context {:repo "demo-graph"}
|
||||
:data {:asset-id 123
|
||||
:asset-uuid "asset-uuid"
|
||||
:asset-type "png"
|
||||
:download-requested? false
|
||||
:checksum-status :match
|
||||
:skipped-reason :already-downloaded}}
|
||||
{:output-format nil})]
|
||||
(is (string/includes? result "Sync asset already downloaded"))
|
||||
(is (string/includes? result "asset-uuid"))
|
||||
(is (not (string/includes? result "local-path")))))
|
||||
|
||||
(testing "sync asset download structured output keeps raw data"
|
||||
(let [data {:asset-id 123
|
||||
:asset-uuid "asset-uuid"
|
||||
:asset-type "png"
|
||||
:download-requested? false
|
||||
:checksum-status :match
|
||||
:skipped-reason :already-downloaded}
|
||||
json-result (format/format-result {:status :ok
|
||||
:command :sync-asset-download
|
||||
:data data}
|
||||
{:output-format :json})
|
||||
edn-result (format/format-result {:status :ok
|
||||
:command :sync-asset-download
|
||||
:data data}
|
||||
{:output-format :edn})]
|
||||
(is (string/includes? json-result "download-requested?"))
|
||||
(is (string/includes? edn-result ":download-requested?"))
|
||||
(is (not (string/includes? json-result "local-path")))
|
||||
(is (not (string/includes? edn-result "local-path"))))))
|
||||
|
||||
(deftest test-human-output-sync-config-get-ws-url
|
||||
(testing "sync config get ws-url renders value in human output"
|
||||
|
||||
Reference in New Issue
Block a user