mirror of
https://github.com/logseq/logseq.git
synced 2026-04-25 22:54:48 +00:00
846f9307c3
so they are aware why asset was ignored. Also if asset ids are not present we should fail loudly like we everywhere else in the importer. Users should know when data is missing instead of hiding it in the console. Also refactor confusing usage of :block/uuid to :asset-created? as it was only used to track the first time an asset is built to be transacted
217 lines
9.7 KiB
Clojure
217 lines
9.7 KiB
Clojure
(ns db-import
|
|
"Imports given file(s) to a db graph. This script is primarily for
|
|
developing the import feature and for engineers who want to customize
|
|
the import process"
|
|
(:require ["fs" :as fs]
|
|
["fs/promises" :as fsp]
|
|
["path" :as node-path]
|
|
[babashka.cli :as cli]
|
|
[cljs.pprint :as pprint]
|
|
[clojure.set :as set]
|
|
[clojure.string :as string]
|
|
[datascript.core :as d]
|
|
[logseq.common.config :as common-config]
|
|
[logseq.common.graph :as common-graph]
|
|
[logseq.db.common.sqlite-cli :as sqlite-cli]
|
|
[logseq.db.frontend.asset :as db-asset]
|
|
[logseq.graph-parser.exporter :as gp-exporter]
|
|
[logseq.outliner.cli :as outliner-cli]
|
|
[logseq.outliner.pipeline :as outliner-pipeline]
|
|
[nbb.classpath :as cp]
|
|
[nbb.core :as nbb]
|
|
[promesa.core :as p]))
|
|
|
|
(def tx-queue (atom cljs.core/PersistentQueue.EMPTY))
|
|
(def original-transact! d/transact!)
|
|
(defn dev-transact! [conn tx-data tx-meta]
|
|
(swap! tx-queue (fn [queue]
|
|
(let [new-queue (conj queue {:tx-data tx-data :tx-meta tx-meta})]
|
|
;; Only care about last few so vary 10 as needed
|
|
(if (> (count new-queue) 10)
|
|
(pop new-queue)
|
|
new-queue))))
|
|
(original-transact! conn tx-data tx-meta))
|
|
|
|
(defn- build-graph-files
|
|
"Given a file graph directory, return all files including assets and adds relative paths
|
|
on ::rpath since paths are absolute by default and exporter needs relative paths for
|
|
some operations"
|
|
[dir*]
|
|
(let [dir (node-path/resolve dir*)]
|
|
(->> (common-graph/get-files dir)
|
|
(concat (when (fs/existsSync (node-path/join dir* "assets"))
|
|
(common-graph/readdir (node-path/join dir* "assets"))))
|
|
(mapv #(hash-map :path %
|
|
::rpath (node-path/relative dir* %))))))
|
|
|
|
(defn- <read-file
|
|
[file]
|
|
(p/let [s (fsp/readFile (:path file))]
|
|
(str s)))
|
|
|
|
(defn- exceed-limit-size?
|
|
"Asset size no more than 100M"
|
|
[^js buffer]
|
|
(> (.-length buffer) (* 100 1024 1024)))
|
|
|
|
(defn- <read-and-copy-asset [db-graph-dir file assets buffer-handler]
|
|
(p/let [buffer (fs/readFileSync (:path file))
|
|
checksum (db-asset/<get-file-array-buffer-checksum buffer)
|
|
asset-id (d/squuid)
|
|
asset-name (gp-exporter/asset-path->name (:path file))
|
|
asset-type (db-asset/asset-path->type (:path file))]
|
|
(if (exceed-limit-size? buffer)
|
|
(js/console.log (str "Skipped copying asset " (pr-str (:path file)) " because it is larger than the 100M max."))
|
|
(p/let [parent-dir (node-path/join db-graph-dir common-config/local-assets-dir)
|
|
{:keys [with-edn-content pdf-annotation?]} (buffer-handler buffer)]
|
|
(fsp/mkdir parent-dir #js {:recursive true})
|
|
(swap! assets assoc asset-name
|
|
(with-edn-content
|
|
{:size (.-length buffer)
|
|
:type asset-type
|
|
:path (:path file)
|
|
:checksum checksum
|
|
:asset-id asset-id}))
|
|
(when-not pdf-annotation?
|
|
(fsp/copyFile (:path file) (node-path/join parent-dir (str asset-id "." asset-type))))))))
|
|
|
|
(defn- notify-user [{:keys [continue debug]} m]
|
|
(println (:msg m))
|
|
(when (:ex-data m)
|
|
(println "Ex-data:" (pr-str (merge (dissoc (:ex-data m) :error)
|
|
(when-let [err (get-in m [:ex-data :error])]
|
|
{:original-error (ex-data (.-cause err))}))))
|
|
(println "\nStacktrace:")
|
|
(if-let [stack (some-> (get-in m [:ex-data :error]) ex-data :sci.impl/callstack deref)]
|
|
(println (string/join
|
|
"\n"
|
|
(map
|
|
#(str (:file %)
|
|
(when (:line %) (str ":" (:line %)))
|
|
(when (:sci.impl/f-meta %)
|
|
(str " calls #'" (get-in % [:sci.impl/f-meta :ns]) "/" (get-in % [:sci.impl/f-meta :name]))))
|
|
(reverse stack))))
|
|
(println (some-> (get-in m [:ex-data :error]) .-stack)))
|
|
(when debug
|
|
(when-let [matching-tx (seq (filter #(and (get-in m [:ex-data :path])
|
|
(or (= (get-in % [:tx-meta ::gp-exporter/path]) (get-in m [:ex-data :path]))
|
|
(= (get-in % [:tx-meta ::outliner-pipeline/original-tx-meta ::gp-exporter/path]) (get-in m [:ex-data :path]))))
|
|
@tx-queue))]
|
|
(println (str "\n" (count matching-tx)) "Tx Maps for failing path:")
|
|
(pprint/pprint matching-tx))))
|
|
(when (and (= :error (:level m)) (not continue))
|
|
(js/process.exit 1)))
|
|
|
|
(defn default-export-options
|
|
[options]
|
|
{;; common options
|
|
:rpath-key ::rpath
|
|
:notify-user (partial notify-user options)
|
|
:<read-file <read-file
|
|
;; :set-ui-state prn
|
|
;; config file options
|
|
;; TODO: Add actual default
|
|
:default-config {}})
|
|
|
|
(defn- import-file-graph-to-db
|
|
"Import a file graph dir just like UI does. However, unlike the UI the
|
|
exporter receives file maps containing keys :path and ::rpath since :path
|
|
are full paths"
|
|
[file-graph-dir db-graph-dir conn options]
|
|
(let [*files (build-graph-files file-graph-dir)
|
|
config-file (first (filter #(string/ends-with? (:path %) "logseq/config.edn") *files))
|
|
_ (assert config-file "No 'logseq/config.edn' found for file graph dir")
|
|
options (merge options
|
|
(default-export-options options)
|
|
;; asset file options
|
|
{:<read-and-copy-asset #(<read-and-copy-asset db-graph-dir %1 %2 %3)})]
|
|
(p/with-redefs [d/transact! dev-transact!]
|
|
(gp-exporter/export-file-graph conn conn config-file *files options))))
|
|
|
|
(defn- resolve-path
|
|
"If relative path, resolve with $ORIGINAL_PWD"
|
|
[path]
|
|
(if (node-path/isAbsolute path)
|
|
path
|
|
(node-path/join (or js/process.env.ORIGINAL_PWD ".") path)))
|
|
|
|
(defn- import-files-to-db
|
|
"Import specific doc files for dev purposes"
|
|
[file conn {:keys [files] :as options}]
|
|
(let [doc-options (gp-exporter/build-doc-options {:macros {}} (merge options (default-export-options options)))
|
|
files' (mapv #(hash-map :path %)
|
|
(into [file] (map resolve-path files)))]
|
|
(p/with-redefs [d/transact! dev-transact!]
|
|
(p/let [_ (gp-exporter/export-doc-files conn files' <read-file doc-options)]
|
|
{:import-state (:import-state doc-options)}))))
|
|
|
|
(def spec
|
|
"Options spec"
|
|
{:help {:alias :h
|
|
:desc "Print help"}
|
|
:verbose {:alias :v
|
|
:desc "Verbose mode"}
|
|
:debug {:alias :d
|
|
:desc "Debug mode"}
|
|
:continue {:alias :c
|
|
:desc "Continue past import failures"}
|
|
:all-tags {:alias :a
|
|
:desc "All tags convert to classes"}
|
|
:tag-classes {:alias :t
|
|
:coerce []
|
|
:desc "List of tags to convert to classes"}
|
|
:files {:alias :f
|
|
:coerce []
|
|
:desc "Additional files to import"}
|
|
:remove-inline-tags {:alias :r
|
|
:desc "Remove inline tags"}
|
|
:property-classes {:alias :p
|
|
:coerce []
|
|
:desc "List of properties whose values convert to classes"}
|
|
:property-parent-classes
|
|
{:alias :P
|
|
:coerce []
|
|
:desc "List of properties whose values convert to a parent class"}})
|
|
|
|
(defn -main [args]
|
|
(let [[file-graph db-graph-dir] args
|
|
options (cli/parse-opts args {:spec spec})
|
|
_ (when (or (< (count args) 2) (:help options))
|
|
(println (str "Usage: $0 FILE-GRAPH DB-GRAPH [OPTIONS]\nOptions:\n"
|
|
(cli/format-opts {:spec spec})))
|
|
(js/process.exit 1))
|
|
init-conn-args (sqlite-cli/->open-db-args db-graph-dir)
|
|
db-name (if (= 1 (count init-conn-args)) (first init-conn-args) (second init-conn-args))
|
|
db-full-dir (if (= 1 (count init-conn-args))
|
|
(node-path/dirname (first init-conn-args))
|
|
(apply node-path/join init-conn-args))
|
|
file-graph' (resolve-path file-graph)
|
|
conn (apply outliner-cli/init-conn (conj init-conn-args {:classpath (cp/get-classpath)
|
|
:import-type :cli/db-import}))
|
|
directory? (.isDirectory (fs/statSync file-graph'))
|
|
user-options (cond-> (merge {:all-tags false} (dissoc options :verbose :files :help :continue))
|
|
;; coerce option collection into strings
|
|
(:tag-classes options)
|
|
(update :tag-classes (partial mapv str))
|
|
true
|
|
(set/rename-keys {:all-tags :convert-all-tags? :remove-inline-tags :remove-inline-tags?}))
|
|
_ (when (:verbose options) (prn :options user-options))
|
|
options' (merge {:user-options user-options}
|
|
(select-keys options [:files :verbose :continue :debug]))]
|
|
(p/let [{:keys [import-state]}
|
|
(if directory?
|
|
(import-file-graph-to-db file-graph' db-full-dir conn options')
|
|
(import-files-to-db file-graph' conn options'))]
|
|
|
|
(when-let [ignored-props (seq @(:ignored-properties import-state))]
|
|
(println "Ignored properties:" (pr-str ignored-props)))
|
|
(when-let [ignored-assets (seq @(:ignored-assets import-state))]
|
|
(println "Ignored assets:" (pr-str ignored-assets)))
|
|
(when-let [ignored-files (seq @(:ignored-files import-state))]
|
|
(println (count ignored-files) "ignored file(s):" (pr-str (vec ignored-files))))
|
|
(when (:verbose options') (println "Transacted" (count (d/datoms @conn :eavt)) "datoms"))
|
|
(println "Created graph" (str db-name "!")))))
|
|
|
|
(when (= nbb/*file* (nbb/invoked-file))
|
|
(-main *command-line-args*))
|