clone/logseq
1
0
Fork 0
mirror of https://github.com/logseq/logseq.git synced 2026-04-24 22:25:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

enhance: full file graph import works from CLI

Browse Source 
Also cleaned up naming, added docstrings and fixed a bug that
failed import hard when user didn't input any tag-classes or
property-classes
This commit is contained in:
Gabriel Horner
2024-03-13 16:23:49 -04:00
parent 6c12f5ab33
commit 345fd96e2a
3 changed files with 179 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
deps/graph-parser/script/db_import.cljs vendored
View File

@@ -12,65 +12,53 @@
[babashka.cli :as cli]
[logseq.graph-parser.exporter :as gp-exporter]
[logseq.common.graph :as common-graph]
[logseq.common.config :as common-config]
[logseq.tasks.db-graph.create-graph :as create-graph]
[promesa.core :as p]))
(defn- remove-hidden-files [dir config files]
(if (seq (:hidden config))
(->> files
(map #(assoc % ::rel-path (node-path/relative dir (:rpath %))))
((fn [files] (common-config/remove-hidden-files files config ::rel-path)))
(map #(dissoc % ::rel-path)))
files))
(defn- build-graph-files
"Given a graph directory, return absolute, allowed file paths and their contents in preparation
for parsing"
"Given a file graph directory, return all files including assets and adds relative paths
on ::rpath since paths are absolute by default and exporter needs relative paths for
some operations"
[dir*]
(let [dir (node-path/resolve dir*)]
(->> (common-graph/get-files dir)
(mapv #(hash-map :rpath %)))))
(concat (when (fs/existsSync (node-path/join dir* "assets"))
(common-graph/readdir (node-path/join dir* "assets"))))
(mapv #(hash-map :path %
::rpath (node-path/relative dir* %))))))
(defn- <read-file
[file]
(p/let [s (fsp/readFile (:rpath file))]
(p/let [s (fsp/readFile (:path file))]
(str s)))
(defn- build-import-options [conn config options]
(-> (gp-exporter/setup-import-options
@conn
{}
(select-keys options [:tag-classes :property-classes])
{:notify-user prn :macros (:macros config)})
(assoc-in [:extract-options :verbose] (:verbose options))))
(defn- <copy-asset-file [file db-graph-dir file-graph-dir]
(p/let [parent-dir (node-path/dirname
(node-path/join db-graph-dir (node-path/relative file-graph-dir (:rpath file))))
(node-path/join db-graph-dir (node-path/relative file-graph-dir (:path file))))
_ (fsp/mkdir parent-dir #js {:recursive true})]
(fsp/copyFile (:rpath file) (node-path/join parent-dir (node-path/basename (:rpath file))))))
(fsp/copyFile (:path file) (node-path/join parent-dir (node-path/basename (:path file))))))
(defn- import-file-graph-to-db [file-graph-dir db-graph-dir conn options]
(p/let [*files (build-graph-files file-graph-dir)
config-file (first (filter #(string/ends-with? (:rpath %) "logseq/config.edn") *files))
_ (assert config-file "No 'logseq/config.edn' found for file graph dir")
;; TODO: Add :default-config option
config (gp-exporter/import-config-file! conn config-file <read-file {:notify-user prn})
files (remove-hidden-files file-graph-dir config *files)
import-options (build-import-options conn config options)
logseq-file? #(string/includes? (:rpath %) "logseq/")
asset-files (when (fs/existsSync (node-path/join file-graph-dir "assets"))
(map #(hash-map :rpath %) (common-graph/readdir (node-path/join file-graph-dir "assets"))))
doc-files (remove logseq-file? files)
logseq-files (filter logseq-file? files)]
(println "Importing" (count files) "files ...")
(p/do!
(gp-exporter/import-logseq-files conn logseq-files <read-file {:notify-user prn})
(gp-exporter/import-from-asset-files! asset-files #(<copy-asset-file % db-graph-dir file-graph-dir) {:notify-user prn})
(gp-exporter/import-from-doc-files! conn doc-files <read-file import-options)
(gp-exporter/import-favorites-from-config-edn! conn conn config {})
(gp-exporter/import-class-properties conn conn))))
(defn- import-file-graph-to-db
"Import a file graph dir just like UI does. However, unlike the UI the
exporter receives file maps containing keys :path and ::rpath since :path
are full paths"
[file-graph-dir db-graph-dir conn options]
(let [*files (build-graph-files file-graph-dir)
config-file (first (filter #(string/ends-with? (:path %) "logseq/config.edn") *files))
_ (assert config-file "No 'logseq/config.edn' found for file graph dir")
options (merge options
{;; common options
:rpath-key ::rpath
:notify-user prn
:<read-file <read-file
;; :set-ui-state prn
;; config file options
;; TODO: Add actual default
:default-config {}
;; asset file options
:<copy-asset (fn copy-asset [file]
(<copy-asset-file file db-graph-dir file-graph-dir))})]
(gp-exporter/export-file-graph conn conn config-file *files options)))
(defn- resolve-path
"If relative path, resolve with $ORIGINAL_PWD"
@@ -79,11 +67,13 @@
path
(node-path/join (or js/process.env.ORIGINAL_PWD ".") path)))
(defn- import-files-to-db [file conn {:keys [files] :as options}]
(let [import-options (build-import-options conn {:macros {}} options)
files' (mapv #(hash-map :rpath %)
(defn- import-files-to-db
"Import specific doc files for dev purposes"
[file conn {:keys [files] :as options}]
(let [doc-options (gp-exporter/build-doc-options conn {:macros {}} options)
files' (mapv #(hash-map :path %)
(into [file] (map resolve-path files)))]
(gp-exporter/import-from-doc-files! conn files' <read-file import-options)))
(gp-exporter/export-doc-files conn files' <read-file doc-options)))
(def spec
"Options spec"