clone/logseq
1
0
Fork 0
mirror of https://github.com/logseq/logseq.git synced 2026-05-03 18:36:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(import): remove asset byte-array usage to prevent OOM

Browse Source 
This commit also combine read-asset and copy-asset to read-and-copy-asset
This commit is contained in:
Tienson Qin
2025-09-26 20:39:17 +08:00
committed by Gabriel Horner
parent b4d3fb4284
commit 971cec54bc
6 changed files with 130 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
deps/graph-parser/script/db_import.cljs vendored
View File

@@ -49,26 +49,31 @@
(p/let [s (fsp/readFile (:path file))]
(str s)))
(defn- <read-asset-file [file assets]
(p/let [buffer (fs/readFileSync (:path file))
checksum (db-asset/<get-file-array-buffer-checksum buffer)]
(swap! assets assoc
(gp-exporter/asset-path->name (:path file))
{:size (.-length buffer)
:checksum checksum
:type (db-asset/asset-path->type (:path file))
:path (:path file)})
buffer))
(defn- exceed-limit-size?
"Asset size no more than 100M"
[^js buffer]
(> (.-length buffer) (* 100 1024 1024)))
(defn- <copy-asset-file [asset-m db-graph-dir]
(p/let [parent-dir (node-path/join db-graph-dir common-config/local-assets-dir)
_ (fsp/mkdir parent-dir #js {:recursive true})]
(if (:block/uuid asset-m)
(fsp/copyFile (:path asset-m) (node-path/join parent-dir (str (:block/uuid asset-m) "." (:type asset-m))))
(when-not (:pdf-annotation? asset-m)
(println "[INFO]" "Copied asset" (pr-str (node-path/basename (:path asset-m)))
"by its name since it was unused.")
(fsp/copyFile (:path asset-m) (node-path/join parent-dir (node-path/basename (:path asset-m))))))))
(defn- <read-and-copy-asset [db-graph-dir file assets buffer-handler]
(p/let [buffer (fs/readFileSync (:path file))
checksum (db-asset/<get-file-array-buffer-checksum buffer)
asset-id (d/squuid)
asset-name (gp-exporter/asset-path->name (:path file))
asset-type (db-asset/asset-path->type (:path file))]
(if (exceed-limit-size? buffer)
(js/console.error (str "Asset size shouldn't be larger than 100M, path: " (:path file)))
(p/let [parent-dir (node-path/join db-graph-dir common-config/local-assets-dir)
{:keys [with-edn-content pdf-annotation?]} (buffer-handler buffer)]
(fsp/mkdir parent-dir #js {:recursive true})
(swap! assets assoc asset-name
(with-edn-content
{:size (.-length buffer)
:type asset-type
:path (:path file)
:checksum checksum
:asset-id asset-id}))
(when-not pdf-annotation?
(fsp/copyFile (:path file) (node-path/join parent-dir (str asset-id "." asset-type))))))))
(defn- notify-user [{:keys [continue debug]} m]
(println (:msg m))
@@ -119,9 +124,7 @@
options (merge options
(default-export-options options)
;; asset file options
{:<copy-asset (fn copy-asset [file]
(<copy-asset-file file db-graph-dir))
:<read-asset <read-asset-file})]
{:<read-and-copy-asset #(<read-and-copy-asset db-graph-dir %1 %2 %3)})]
(p/with-redefs [d/transact! dev-transact!]
(gp-exporter/export-file-graph conn conn config-file *files options))))