mirror of
https://github.com/logseq/logseq.git
synced 2026-04-24 22:25:01 +00:00
Graph parser adds page metadata to a fresh db
This commit is contained in:
Gabriel Horner
253
src/main/frontend/handler/graph_parser.cljs
Normal file
253
src/main/frontend/handler/graph_parser.cljs
Normal file
@@ -0,0 +1,253 @@
|
||||
(ns frontend.handler.graph-parser
|
||||
(:require [clojure.string :as string]
|
||||
[clojure.set :as set]
|
||||
[clojure.edn :as edn]
|
||||
[datascript.core :as d]
|
||||
["fs" :as fs]
|
||||
["child_process" :as child-process]
|
||||
[frontend.db-schema :as db-schema]
|
||||
;; Disable for now since kondo can't pick it up
|
||||
; #?(:org.babashka/nbb [nbb.core :as nbb])
|
||||
[nbb.core :as nbb]))
|
||||
|
||||
;; Helpers
|
||||
;; =======
|
||||
|
||||
(defn- fline [and-form] (:line (meta and-form)))
|
||||
|
||||
(defmacro -spy [line name expr]
|
||||
`(let [result# ~expr
|
||||
line-str# ~(str "Line " line ":")]
|
||||
;; Subject to elision:
|
||||
;; (log* ~config ~level ~name "=>" result#) ; CLJ-865
|
||||
(apply println line-str# [~name "=>" result#])
|
||||
|
||||
;; NOT subject to elision:
|
||||
result#))
|
||||
|
||||
(defmacro spy
|
||||
"Modified version of taoensso.timbre/spy"
|
||||
[expr] `(-spy ~(fline &form) '~expr ~expr))
|
||||
|
||||
(defn slurp
|
||||
"Like clojure.core/slurp"
|
||||
[file]
|
||||
(str (fs/readFileSync file)))
|
||||
|
||||
(defn sh
|
||||
"Run shell cmd synchronously and print to inherited streams by default. Aims
|
||||
to be similar to babashka.tasks/shell"
|
||||
[cmd opts]
|
||||
(child-process/spawnSync (first cmd)
|
||||
(clj->js (rest cmd))
|
||||
(clj->js (merge {:stdio "inherit"} opts))))
|
||||
|
||||
;; From other namespace
|
||||
;; =====
|
||||
(defn get-all-pages
|
||||
[db]
|
||||
(d/q
|
||||
'[:find [(pull ?page [*]) ...]
|
||||
:where
|
||||
[?page :block/name]]
|
||||
db))
|
||||
|
||||
(defn get-file
|
||||
([db path]
|
||||
(when-let [conn db]
|
||||
(:file/content (d/entity conn [:file/path path])))))
|
||||
|
||||
(defn safe-read-string
|
||||
[content error-message-or-handler]
|
||||
(try
|
||||
;; reader/read-string
|
||||
(edn/read-string content)
|
||||
(catch :default e
|
||||
(js/console.error e)
|
||||
(if (fn? error-message-or-handler)
|
||||
(error-message-or-handler e)
|
||||
(println error-message-or-handler))
|
||||
{})))
|
||||
|
||||
#_(defn alter-file
|
||||
[repo path content {:keys [reset? re-render-root? from-disk? skip-compare? new-graph?]
|
||||
:or {reset? true
|
||||
re-render-root? false
|
||||
from-disk? false
|
||||
skip-compare? false}}]
|
||||
(let [original-content (get-file repo path)
|
||||
write-file! (if from-disk?
|
||||
#(p/resolved nil)
|
||||
#(fs/write-file! repo (config/get-repo-dir repo) path content
|
||||
(assoc (when original-content {:old-content original-content})
|
||||
:skip-compare? skip-compare?)))]
|
||||
(if reset?
|
||||
(do
|
||||
(when-let [page-id (db/get-file-page-id path)]
|
||||
(db/transact! repo
|
||||
[[:db/retract page-id :block/alias]
|
||||
[:db/retract page-id :block/tags]]))
|
||||
(reset-file! repo path content new-graph?))
|
||||
(db/set-file-content! repo path content))
|
||||
(util/p-handle (write-file!)
|
||||
(fn [_]
|
||||
(when (= path (config/get-config-path repo))
|
||||
(restore-config! repo true))
|
||||
(when (= path (config/get-custom-css-path repo))
|
||||
(ui-handler/add-style-if-exists!))
|
||||
(when re-render-root? (ui-handler/re-render-root!)))
|
||||
(fn [error]
|
||||
(println "Write file failed, path: " path ", content: " content)
|
||||
(log/error :write/failed error)))))
|
||||
|
||||
(defn normalize
|
||||
[format]
|
||||
(case (keyword format)
|
||||
:md :markdown
|
||||
:asciidoc :adoc
|
||||
;; default
|
||||
(keyword format)))
|
||||
(defn get-format
|
||||
[file]
|
||||
(when file
|
||||
(normalize (keyword (string/lower-case (last (string/split file #"\.")))))))
|
||||
|
||||
;; Main handler.repo parse fns
|
||||
;; ====
|
||||
(defonce local-db-prefix "logseq_local_")
|
||||
|
||||
(defn get-local-dir
|
||||
[s]
|
||||
(string/replace s local-db-prefix ""))
|
||||
|
||||
(defn get-repo-dir
|
||||
[repo-url]
|
||||
(cond
|
||||
true
|
||||
#_(and (util/electron?) (local-db? repo-url))
|
||||
(get-local-dir repo-url)
|
||||
|
||||
#_(and (mobile-util/is-native-platform?) (local-db? repo-url))
|
||||
#_(let [dir (get-local-dir repo-url)]
|
||||
(if (string/starts-with? dir "file:")
|
||||
dir
|
||||
(str "file:///" (string/replace dir #"^/+" ""))))
|
||||
|
||||
:else
|
||||
(str "/"
|
||||
(->> (take-last 2 (string/split repo-url #"/"))
|
||||
(string/join "_")))))
|
||||
|
||||
(defn path-normalize
|
||||
"Normalize file path (for reading paths from FS, not required by writting)"
|
||||
[s]
|
||||
(.normalize s "NFC"))
|
||||
|
||||
(defn get-file-path
|
||||
"Normalization happens here"
|
||||
[repo-url relative-path]
|
||||
(when (and repo-url relative-path)
|
||||
(let [path (cond
|
||||
true
|
||||
#_(and (util/electron?) (local-db? repo-url))
|
||||
(let [dir (get-repo-dir repo-url)]
|
||||
(if (string/starts-with? relative-path dir)
|
||||
relative-path
|
||||
(str dir "/"
|
||||
(string/replace relative-path #"^/" ""))))
|
||||
|
||||
(= "/" (first relative-path))
|
||||
(subs relative-path 1)
|
||||
|
||||
:else
|
||||
relative-path)]
|
||||
(path-normalize path))))
|
||||
|
||||
(def app-name "logseq")
|
||||
(def pages-metadata-file "pages-metadata.edn")
|
||||
|
||||
(defn get-pages-metadata-path
|
||||
[repo]
|
||||
(when repo
|
||||
(get-file-path repo (str app-name "/" pages-metadata-file))))
|
||||
|
||||
(defn- load-pages-metadata!
|
||||
"force?: if set true, skip the metadata timestamp range check"
|
||||
[conn repo _file-paths files _force?]
|
||||
(try
|
||||
(let [file (get-pages-metadata-path repo)]
|
||||
(when-let [content (some #(when (= (:file/path %) file) (:file/content %)) files)]
|
||||
(let [metadata (safe-read-string content "Parsing pages metadata file failed: ")
|
||||
;; pages (get-all-pages repo)
|
||||
;; pages (zipmap (map :block/name pages) pages)
|
||||
metadata (->> metadata
|
||||
#_(filter (fn [{:block/keys [name created-at updated-at]}]
|
||||
(when-let [page (get pages name)]
|
||||
(and
|
||||
(>= updated-at created-at) ;; metadata validation
|
||||
(or force? ;; when force is true, shortcut timestamp range check
|
||||
(and (or (nil? (:block/created-at page))
|
||||
(>= created-at (:block/created-at page)))
|
||||
(or (nil? (:block/updated-at page))
|
||||
(>= updated-at (:block/created-at page)))))
|
||||
(or ;; persistent metadata is the gold standard
|
||||
(not= created-at (:block/created-at page))
|
||||
(not= updated-at (:block/created-at page)))))))
|
||||
(remove nil?))]
|
||||
(when (seq metadata)
|
||||
(d/transact! conn metadata)))))
|
||||
(catch js/Error e
|
||||
(prn e)
|
||||
#_(log/error :exception e))))
|
||||
|
||||
(defonce mldoc-support-formats
|
||||
#{:org :markdown :md})
|
||||
|
||||
(defn parse
|
||||
"From parse-files-and-create-default-files-inner!"
|
||||
[conn repo-url files file-paths]
|
||||
(let [support-files (filter
|
||||
(fn [file]
|
||||
(let [format (get-format (:file/path file))]
|
||||
(contains? (set/union #{:edn :css} mldoc-support-formats) format)))
|
||||
files)
|
||||
_support-files (sort-by :file/path support-files)]
|
||||
;; TODO: Add file content
|
||||
#_(doseq [file support-files]
|
||||
(alter-file repo-url
|
||||
(:file/path file)
|
||||
(:file/content file)
|
||||
{:new-graph? (:new-graph? opts)
|
||||
:re-render-root? false
|
||||
:from-disk? true
|
||||
:metadata metadata}))
|
||||
(load-pages-metadata! conn repo-url file-paths files true)))
|
||||
|
||||
|
||||
(defn db-start
|
||||
;; TODO: Add frontend.db.default/built-in-pages
|
||||
[]
|
||||
(let [db-conn (d/create-conn db-schema/schema)]
|
||||
(d/transact! db-conn [{:schema/version db-schema/version}])
|
||||
db-conn))
|
||||
|
||||
(defn -main
|
||||
[args]
|
||||
(let [conn (db-start)
|
||||
repo-dir (or (first args)
|
||||
(throw (ex-info "Directory required" {})))
|
||||
files (->> (str (.-stdout (sh ["git" "ls-files"]
|
||||
{:cwd repo-dir :stdio nil})))
|
||||
string/split-lines
|
||||
(filter #(re-find #"^(pages|assets|journals|logseq)" %))
|
||||
(map #(str repo-dir "/" %)))
|
||||
file-maps (mapv #(hash-map :file/path % :file/content (slurp %)) files)]
|
||||
(parse conn
|
||||
(str "logseq_local_" repo-dir)
|
||||
file-maps
|
||||
files)
|
||||
(prn (d/q '[:find ?n :where [?b :block/name ?n]]
|
||||
@conn))))
|
||||
|
||||
(when (= nbb/*file* (:file (meta #'-main)))
|
||||
(-main *command-line-args*))
|
||||
Reference in New Issue
Block a user