clone/logseq
1
0
Fork 0
mirror of https://github.com/logseq/logseq.git synced 2026-06-01 19:01:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(fs): use rpath

Browse Source
This commit is contained in:
Andelf
2023-03-09 02:53:57 +08:00
parent 16f8188fd1
commit 31d7c8f836
19 changed files with 510 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/main/electron/listener.cljs
View File

@@ -10,6 +10,7 @@
[frontend.db.model :as db-model]
[frontend.fs.sync :as sync]
[frontend.fs.watcher-handler :as watcher-handler]
[frontend.fs2.path :as fs2-path]
[frontend.handler.editor :as editor-handler]
[frontend.handler.file-sync :as file-sync-handler]
[frontend.handler.notification :as notification]
@@ -19,6 +20,7 @@
[frontend.handler.user :as user]
[frontend.state :as state]
[frontend.ui :as ui]
[logseq.graph-parser.util :as gp-util]
[promesa.core :as p]))
(defn- safe-api-call
@@ -48,7 +50,10 @@
;; TODO: move "file-watcher" to electron.ipc.channels
(safe-api-call "file-watcher"
(fn [data]
(let [{:keys [type payload]} (bean/->clj data)]
(let [{:keys [type payload]} (bean/->clj data)
path (gp-util/path-normalize (:path payload))
dir (:dir payload)
payload (assoc payload :path (fs2-path/relative-path dir path))]
(watcher-handler/handle-changed! type payload)
(when (file-sync-handler/enable-sync?)
(sync/file-watch-handler type payload)))))

74
src/main/frontend/fs.cljs
View File

@@ -13,6 +13,7 @@
[lambdaisland.glogi :as log]
[promesa.core :as p]
[frontend.db :as db]
[frontend.fs2.path :as fs2-path]
[clojure.string :as string]
[frontend.state :as state]
[logseq.graph-parser.util :as gp-util]
@@ -117,16 +118,16 @@
(let [new-path (gp-util/path-normalize new-path)]
(cond
; See https://github.com/isomorphic-git/lightning-fs/issues/41
(= old-path new-path)
(p/resolved nil)
(= old-path new-path)
(p/resolved nil)
:else
(let [[old-path new-path]
(map #(if (or (util/electron?) (mobile-util/native-platform?))
%
(str (config/get-repo-dir repo) "/" %))
[old-path new-path])]
(protocol/rename! (get-fs old-path) repo old-path new-path)))))
:else
(let [[old-path new-path]
(map #(if (or (util/electron?) (mobile-util/native-platform?))
%
(str (config/get-repo-dir repo) "/" %))
[old-path new-path])]
(protocol/rename! (get-fs old-path) repo old-path new-path)))))
(defn copy!
[repo old-path new-path]
@@ -164,19 +165,36 @@
(p/let [result (protocol/open-dir record dir ok-handler)]
(if (or (util/electron?)
(mobile-util/native-platform?))
(let [[dir & paths] (bean/->clj result)]
[(:path dir) paths])
(let [[dir & paths] result
_ (prn ::open-dir result)
dir (:path dir)
_ (prn ::open-dir dir)
files (mapv (fn [entry]
(assoc entry :path (fs2-path/relative-path dir (:path entry))))
paths)]
(prn :got files)
{:path dir :files files})
result))))
(defn get-files
(defn list-files
"List all files in the directory, recursively.
{:path :files []}"
[path-or-handle ok-handler]
(let [record (get-record)
electron? (util/electron?)
mobile? (mobile-util/native-platform?)]
(p/let [result (protocol/get-files record path-or-handle ok-handler)]
(if (or electron? mobile?)
(let [result (bean/->clj result)]
(if electron? (rest result) result))
(let [record (get-record)]
(when ok-handler
(js/console.warn "ok-handler not nil"))
(p/let [result (protocol/list-files record path-or-handle ok-handler)]
(prn :t result)
(if (or (util/electron?)
(mobile-util/native-platform?))
(let [[dir & paths] result
dir (:path dir)
files (mapv (fn [entry]
(prn ::xx entry)
(assoc entry :path (fs2-path/relative-path dir (:path entry))))
paths)]
(prn :got files)
{:path dir :files files})
result))))
(defn watch-dir!
@@ -202,17 +220,13 @@
([repo dir path]
(create-if-not-exists repo dir path ""))
([repo dir path initial-content]
(let [path (if (util/absolute-path? path) path
(if (util/starts-with? path "/")
path
(str "/" path)))]
(->
(p/let [_stat (stat dir path)]
true)
(p/catch
(fn [_error]
(p/let [_ (write-file! repo dir path initial-content nil)]
false)))))))
(let []
(-> (p/let [_stat (stat dir path)]
true)
(p/catch
(fn [_error]
(p/let [_ (write-file! repo dir path initial-content nil)]
false)))))))
(defn file-exists?
[dir path]

2
src/main/frontend/fs/bfs.cljs
View File

@@ -32,7 +32,7 @@
(js/window.pfs.stat (str dir path)))
(open-dir [_this _dir _ok-handler]
nil)
(get-files [_this _path-or-handle _ok-handler]
(list-files [_this _path-or-handle _ok-handler]
nil)
(watch-dir! [_this _dir _options]
nil)

4
src/main/frontend/fs/capacitor_fs.cljs
View File

@@ -389,8 +389,8 @@
#(js->clj % :keywordize-keys true))))
(open-dir [_this dir _ok-handler]
(open-dir dir))
(get-files [_this path-or-handle _ok-handler]
(readdir path-or-handle))
(list-files [_this dir _ok-handler]
(readdir dir))
(watch-dir! [_this dir _options]
(p/do!
(.unwatch mobile-util/fs-watcher)

6
src/main/frontend/fs/nfs.cljs
View File

@@ -195,8 +195,8 @@
(nfs-saved-handler repo path file)))
(do
(notification/show! (str "The file " path " already exists, please append the content if you need it.\n Unsaved content: \n" content)
:warning
false)
:warning
false)
(state/pub-event! [:file/alter repo path text]))))
(println "Error: directory handle not exists: " handle-path)))
(p/catch (fn [error]
@@ -227,7 +227,7 @@
(open-dir [_this _dir ok-handler]
(utils/openDirectory #js {:recursive true}
ok-handler))
(get-files [_this path-or-handle ok-handler]
(list-files [_this path-or-handle ok-handler]
(utils/getFiles path-or-handle true ok-handler))
(watch-dir! [_this _dir _options]

151
src/main/frontend/fs/node.cljs
View File

@@ -1,6 +1,7 @@
(ns frontend.fs.node
"Implementation of fs protocol for desktop"
(:require [clojure.string :as string]
"Implementation of fs protocol for Electron, based on nodejs"
(:require [cljs-bean.core :as bean]
[clojure.string :as string]
[electron.ipc :as ipc]
[frontend.config :as config]
[frontend.db :as db]
@@ -9,7 +10,8 @@
[frontend.util :as util]
[goog.object :as gobj]
[lambdaisland.glogi :as log]
[promesa.core :as p]))
[promesa.core :as p]
[frontend.fs2.path :as fs2-path]))
(defn concat-path
[dir path]
@@ -30,58 +32,89 @@
(when (and (string? disk-content) (string? db-content))
(p/resolved (= (string/trim disk-content) (string/trim db-content)))))
(defn- write-file-without-backup
[repo dir path content ok-handler error-handler]
(p/catch
(p/let [result (ipc/ipc "writeFile" repo path content)]
(when ok-handler
(ok-handler repo path result)))
(fn [error]
(if error-handler
(error-handler error)
(log/error :write-file-failed error))))
)
(defn- write-file-impl!
[this repo dir path content {:keys [ok-handler error-handler old-content skip-compare?]} stat]
(if skip-compare?
(p/catch
(p/let [result (ipc/ipc "writeFile" repo path content)]
(when ok-handler
(ok-handler repo path result)))
(fn [error]
(if error-handler
(error-handler error)
(log/error :write-file-failed error))))
[this repo dir rpath content {:keys [ok-handler error-handler old-content skip-compare?]} stat]
(prn ::write-file-impl repo dir rpath)
(js/console.trace)
(let [file-path (fs2-path/path-join dir rpath)]
(if skip-compare?
(p/catch
(p/let [result (ipc/ipc "writeFile" repo file-path content)]
(when ok-handler
(prn ::fuck :why-are-you-using-ok-handler)
(ok-handler repo rpath result)))
(fn [error]
(if error-handler
(error-handler error)
(log/error :write-file-failed error))))
(p/let [disk-content (when (not= stat :not-found)
(-> (protocol/read-file this dir path nil)
(p/catch (fn [error]
(js/console.error error)
nil))))
disk-content (or disk-content "")
ext (string/lower-case (util/get-file-ext path))
db-content (or old-content (db/get-file repo path) "")
contents-matched? (contents-matched? disk-content db-content)]
(cond
(and
(not= stat :not-found) ; file on the disk was deleted
(not contents-matched?)
(not (contains? #{"excalidraw" "edn" "css"} ext))
(not (string/includes? path "/.recycle/")))
(state/pub-event! [:file/not-matched-from-disk path disk-content content])
(p/let [disk-content (when (not= stat :not-found)
(-> (protocol/read-file this dir file-path nil)
(p/catch (fn [error]
(js/console.error error)
nil))))
disk-content (or disk-content "")
ext (string/lower-case (util/get-file-ext rpath))
db-content (or old-content (db/get-file repo rpath) "")
contents-matched? (contents-matched? disk-content db-content)]
(prn ::disk disk-content ::db db-content ::new content)
(cond
(and
(not= stat :not-found) ; file on the disk was deleted
(not contents-matched?)
(not (contains? #{"excalidraw" "edn" "css"} ext))
(not (string/includes? rpath "/.recycle/")))
(do
(prn ::?????)
(state/pub-event! [:file/not-matched-from-disk rpath disk-content content]))
:else
(->
(p/let [result (ipc/ipc "writeFile" repo path content)
mtime (gobj/get result "mtime")]
(when-not contents-matched?
(ipc/ipc "backupDbFile" (config/get-local-dir repo) path disk-content content))
(db/set-file-last-modified-at! repo path mtime)
(db/set-file-content! repo path content)
(when ok-handler
(ok-handler repo path result))
result)
(p/catch (fn [error]
(if error-handler
(error-handler error)
(log/error :write-file-failed error)))))))))
:else
(->
(p/let [result (ipc/ipc "writeFile" repo file-path content)
mtime (gobj/get result "mtime")]
(when-not contents-matched?
(ipc/ipc "backupDbFile" (config/get-local-dir repo) rpath disk-content content))
(db/set-file-last-modified-at! repo rpath mtime)
(db/set-file-content! repo rpath content)
(when ok-handler
(ok-handler repo rpath result))
result)
(p/catch (fn [error]
(if error-handler
(error-handler error)
(log/error :write-file-failed error))))))))))
(defn- open-dir [dir]
(defn- open-dir
"Open a new directory"
[dir]
(p/let [dir-path (or dir (util/mocked-open-dir-path))
result (if dir-path
(ipc/ipc "getFiles" dir-path)
(ipc/ipc "openDir" {}))]
(prn ::open-dir result)
result))
(defn- <ensure-dir!
[fs dir]
(protocol/mkdir-recur! fs dir))
(defn- <exists
[fs path]
)
(defrecord Node []
protocol/Fs
(mkdir! [_this dir]
@@ -89,7 +122,8 @@
(mkdir-recur! [_this dir]
(ipc/ipc "mkdir-recur" dir))
(readdir [_this dir] ; recursive
(ipc/ipc "readdir" dir))
(p/then (ipc/ipc "readdir" dir)
bean/->clj))
(unlink! [_this repo path _opts]
(ipc/ipc "unlink"
(config/get-repo-dir repo)
@@ -98,25 +132,26 @@
;; Too dangerious!!! We'll never implement this.
nil)
(read-file [_this dir path _options]
(let [path (concat-path dir path)]
(let [path (fs2-path/path-join dir path)]
(ipc/ipc "readFile" path)))
(write-file! [this repo dir path content opts]
(let [path (concat-path dir path)]
(p/let [stat (p/catch
(protocol/stat this dir path)
(fn [_e] :not-found))
sub-dir (first (util/get-dir-and-basename path))
_ (protocol/mkdir-recur! this sub-dir)]
(write-file-impl! this repo dir path content opts stat))))
(p/let [stat (p/catch
(protocol/stat this dir path)
(fn [_e] :not-found))
sub-dir (first (util/get-dir-and-basename path)) ;; FIXME: todo dirname
_ (protocol/mkdir-recur! this sub-dir)]
(write-file-impl! this repo dir path content opts stat)))
(rename! [_this _repo old-path new-path]
(ipc/ipc "rename" old-path new-path))
(stat [_this dir path]
(let [path (concat-path dir path)]
(let [path (fs2-path/path-join dir path)]
(ipc/ipc "stat" path)))
(open-dir [_this dir _ok-handler]
(open-dir dir))
(get-files [_this path-or-handle _ok-handler]
(ipc/ipc "getFiles" path-or-handle))
(p/then (open-dir dir)
bean/->clj))
(list-files [_this dir _ok-handler]
(p/then (ipc/ipc "getFiles" dir)
bean/->clj))
(watch-dir! [_this dir options]
(ipc/ipc "addDirWatcher" dir options))
(unwatch-dir! [_this dir]

2
src/main/frontend/fs/protocol.cljs
View File

@@ -14,7 +14,7 @@
(copy! [this repo old-path new-path])
(stat [this dir path])
(open-dir [this dir ok-handler])
(get-files [this path-or-handle ok-handler])
(list-files [this dir ok-handler])
(watch-dir! [this dir options])
(unwatch-dir! [this dir])
;; Ensure the dir is watched, window agnostic.

5
src/main/frontend/fs/sync.cljs
View File

@@ -694,9 +694,8 @@
(- (.-size item)))))))
;;; ### path-normalize
(def path-normalize
(if (util/electron?)
gp-util/path-normalize
(partial capacitor-fs/normalize-file-protocol-path nil)))
gp-util/path-normalize)
;;; ### APIs

29
src/main/frontend/fs/watcher_handler.cljs
View File

@@ -1,25 +1,26 @@
(ns frontend.fs.watcher-handler
"Main ns that handles file watching events from electron's main process"
(:require [clojure.string :as string]
(:require [clojure.set :as set]
[clojure.string :as string]
[frontend.config :as config]
[frontend.db :as db]
[frontend.db.model :as model]
[frontend.fs :as fs]
[frontend.fs.capacitor-fs :as capacitor-fs]
[frontend.fs2.path :as fs2-path]
[frontend.handler.editor :as editor]
[frontend.handler.file :as file-handler]
[frontend.handler.page :as page-handler]
[frontend.handler.ui :as ui-handler]
[logseq.graph-parser.util :as gp-util]
[logseq.graph-parser.config :as gp-config]
[logseq.graph-parser.util.block-ref :as block-ref]
[frontend.mobile.util :as mobile-util]
[lambdaisland.glogi :as log]
[promesa.core :as p]
[frontend.state :as state]
[frontend.fs :as fs]
[frontend.fs.capacitor-fs :as capacitor-fs]
[frontend.util.fs :as fs-util]
[frontend.util :as util]
[clojure.set :as set]))
[frontend.util.fs :as fs-util]
[lambdaisland.glogi :as log]
[logseq.graph-parser.config :as gp-config]
[logseq.graph-parser.util :as gp-util]
[logseq.graph-parser.util.block-ref :as block-ref]
[promesa.core :as p]))
;; all IPC paths must be normalized! (via gp-util/path-normalize)
@@ -50,12 +51,9 @@
(defn handle-changed!
[type {:keys [dir path content stat global-dir] :as payload}]
(prn ::wather payload)
(when dir
(let [path (gp-util/path-normalize path)
path (if (mobile-util/native-platform?)
(capacitor-fs/normalize-file-protocol-path nil path)
path)
;; Global directory events don't know their originating repo so we rely
(let [;; Global directory events don't know their originating repo so we rely
;; on the client to correctly identify it
repo (if global-dir (state/get-current-repo) (config/get-local-repo dir))
{:keys [mtime]} stat
@@ -124,6 +122,7 @@
(map first)
(filter #(string/starts-with? % (config/get-repo-dir graph))))]
(p/let [files (fs/readdir dir :path-only? true)
files (map #(fs2-path/relative-path dir %) files) ;; FIXME: readdir returns full paths
files (remove #(fs-util/ignored-path? dir %) files)]
(let [deleted-files (set/difference (set db-files) (set files))]
(when (seq deleted-files)

158
src/main/frontend/fs2/path.cljs Normal file
View File

@@ -0,0 +1,158 @@
(ns frontend.fs2.path
"Path manipulation functions, use '/' on all platforms.
Also handles URL paths."
(:require [clojure.string :as string]
[goog :refer [Uri]]
[logseq.graph-parser.util :as gp-util]))
(defn is-file-url
[s]
(and (string? s)
(or (string/starts-with? s "file://")
(string/starts-with? s "content://")
(string/starts-with? s "s3://"))))
(defn file-name
"File name of a path or URL"
[path]
(let [fname (if (string/ends-with? path "/")
nil
(last (string/split path #"/")))]
(if (and (not-empty fname) (is-file-url path))
(gp-util/safe-decode-uri-component fname)
fname)))
(defn split-ext
"Split file name into stem and extension, for both path and URL"
[path]
(let [fname (file-name path)
pos (string/last-index-of fname ".")]
(if-not (or (nil? pos) (zero? pos))
[(subs fname 0 pos)
(subs fname (+ pos 1))]
[fname ""])))
(defn file-stem
"File name without extension"
[path]
(first (split-ext path)))
(defn file-ext
"File extension"
[path]
(second (split-ext path)))
(defn safe-file-name?
"Safe path on all platforms"
[fname]
(and (not (string/blank? fname))
(< (count fname) 255)
(not (or (re-find #"[\/?<>\\:*|\"]" fname)
(re-find #"^\.+$" fname)
(re-find #"[\. ]$" fname)
(re-find #"(?i)^(COM[0-9]|CON|LPT[0-9]|NUL|PRN|AUX|com[0-9]|con|lpt[0-9]|nul|prn|aux)\..+" fname)
(re-find #"[\u0000-\u001f\u0080-\u009f]" fname)))))
(comment defn inspect [x]
(prn ::inspect x)
x)
(defn path-join
"Joins the given path segments into a single path, handling relative paths,
'..' and '.' normalization."
[& segments]
(let [segments (remove nil? segments) ;; handle (path-join nil path)
segments (map #(string/replace % #"[/\\]+" "/") segments)
;; a fix for clojure.string/split
split-fn (fn [s]
(if (= s "/")
[""]
(string/split s #"/")))
join-fn (fn [segs]
(case segs
[] "."
[""] "/"
#_{:clj-kondo/ignore [:path-invalid-construct/string-join]}
(string/join "/" segs)))]
(->> (filter not-empty segments)
(mapcat split-fn)
(reduce (fn [acc segment]
(cond
(= "" segment)
[segment]
(= ".." segment)
(case (last acc)
".." (conj acc segment)
"" acc
nil [".."]
(pop acc))
(= "." segment)
acc
:else
(conj acc segment)))
[])
(join-fn))))
(defn url-join
[base-url & segments]
(let [^js url (.parse Uri base-url)
scheme (.getScheme url)
domain (.getDomain url)
path (.getPath url)
new-path (apply path-join path segments)
;; opt_scheme, opt_userInfo, opt_domain, opt_port, opt_path, opt_query, opt_fragment, opt_ignoreCase
new-url (.create Uri scheme nil domain nil new-path nil nil nil)]
(.toString new-url)))
(defn path-normalize
"Normalize path using path-join, break into segment and re-join"
[path]
(path-join path))
(defn url-normalize
[url]
(let [^js uri (.parse Uri url)
scheme (.getScheme uri)
domain (.getDomain uri)
path (.getPath uri)
new-path (path-normalize path)
;; opt_scheme, opt_userInfo, opt_domain, opt_port, opt_path, opt_query, opt_fragment, opt_ignoreCase
new-uri (.create Uri scheme nil domain nil new-path nil nil nil)]
(.toString new-uri)))
(defn relative-path
"Get relative path from base path"
[base path]
(let [base (path-normalize base)
path (path-normalize path)]
(if (string/starts-with? path base)
(string/replace (subs path (count base)) #"^/+", "")
(do
(js/console.error "unhandled relative path" base path)
path))))
(defn decoded-relative-uri
"Get relative uri from base url, url-decoded"
[base-url path-url]
(let [base-url (url-normalize base-url)
path-url (url-normalize path-url)]
(if (string/starts-with? path-url base-url)
(gp-util/safe-decode-uri-component (string/replace (subs path-url (count base-url)) #"^/+", ""))
(do
(js/console.error "unhandled relative path" base-url path-url)
path-url))))
(defn parent
[path]
;; ugly but works
(path-normalize (str path "/..")))

36
src/main/frontend/fs2/path_test.cljs Normal file
View File

@@ -0,0 +1,36 @@
(ns frontend.fs2.path-test
(:require [cljs.test :refer [deftest is testing]]
[frontend.fs2.path :as path]))
(deftest test-safe-file-name?
(testing "safe-file-name"
(is (path/safe-file-name? "foo"))
(is (path/safe-file-name? "foo bar"))
(is (path/safe-file-name? "foo-bar"))
(is (path/safe-file-name? "foo_bar"))
(is (path/safe-file-name? "foo.bar"))
(is (path/safe-file-name? "foo..bar"))
(is (path/safe-file-name? "foo...bar"))
(is (= nil (path/safe-file-name? "foo/bar")))
(is (not (path/safe-file-name? "foo?bar")))
(is (not (path/safe-file-name? "foo<bar")))
(is (not (path/safe-file-name? "foo>bar")))))
(deftest path-join
(testing "join-path")
(is (= "foo/bar" (path/path-join ["foo" "bar"])))
(is (= "foo/bar" (path/path-join ["foo/" "bar"])))
(is (= "/foo/bar/baz/asdf" (path/path-join ["/foo/bar//baz/asdf/quux/.."]))))
((deftest url-join-test
(testing "url-join"
(is (= "https://foo.bar/baz" (path/url-join "https://foo.bar" ["baz"])))
(is (= "https://foo.bar/baz" (path/url-join "https://foo.bar/" ["baz"])))
(is (= "https://foo.bar/baz" (path/url-join "https://foo.bar/" ["/baz"])))
(is (= "https://foo.bar/baz" (path/url-join "https://foo.bar" ["/baz"])))
(is (= "https://foo.bar/baz" (path/url-join "https://foo.bar" ["/baz/"])))
(is (= "https://foo.bar/baz" (path/url-join "https://foo.bar/" ["/baz/"])))
(is (= "https://foo.bar/baz" (path/url-join "https://foo.bar/" ["/baz"]))))))

22
src/main/frontend/handler/common/file.cljs
View File

@@ -50,14 +50,16 @@
(graph-parser/get-blocks-to-delete db file-page file-path retain-uuid-blocks))
(defn reset-file!
"Main fn for updating a db with the results of a parsed file"
([repo-url file content]
(reset-file! repo-url file content {}))
([repo-url file content {:keys [verbose] :as options}]
"Main fn for updating a db with the results of a parsed file.
"
([repo-url file-path content]
(reset-file! repo-url file-path content {}))
([repo-url file-path content {:keys [verbose] :as options}]
(let [electron-local-repo? (and (util/electron?)
(config/local-db? repo-url))
repo-dir (config/get-repo-dir repo-url)
file (cond
;; use relpath
_ (comment cond
(and electron-local-repo?
util/win32?
(utils/win32 file))
@@ -73,18 +75,18 @@
:else
file)
file (gp-util/path-normalize file)
new? (nil? (db/entity [:file/path file]))
_ (prn ::reset-file file-path)
new? (nil? (db/entity [:file/path file-path]))
options (merge (dissoc options :verbose)
{:new? new?
:delete-blocks-fn (partial validate-and-get-blocks-to-delete repo-url)
:extract-options (merge
{:user-config (state/get-config)
:date-formatter (state/get-date-formatter)
:block-pattern (config/get-block-pattern (gp-util/get-format file))
:block-pattern (config/get-block-pattern (gp-util/get-format file-path))
:supported-formats (gp-config/supported-formats)
:uri-encoded? (boolean (mobile-util/native-platform?))
;; :uri-encoded? (boolean (mobile-util/native-platform?))
:filename-format (state/get-filename-format repo-url)
:extracted-block-ids (:extracted-block-ids options)}
(when (some? verbose) {:verbose verbose}))})]
(:tx (graph-parser/parse-file (db/get-db repo-url false) file content options)))))
(:tx (graph-parser/parse-file (db/get-db repo-url false) file-path content options)))))

45
src/main/frontend/handler/file.cljs
View File

@@ -127,7 +127,6 @@
(when original-content {:old-content original-content}))]
(fs/write-file! repo path-dir path content write-file-options')))
;; TODO: Remove this function in favor of `alter-files`
(defn alter-file
[repo path content {:keys [reset? re-render-root? from-disk? skip-compare? new-graph? verbose
skip-db-transact? extracted-block-ids]
@@ -136,10 +135,13 @@
from-disk? false
skip-compare? false}}]
(let [path (gp-util/path-normalize path)
config-file? (string/ends-with? path config/config-file)
_ (when config-file? (detect-deprecations repo path content))
_ (prn ::alter-file path)
;; _ (js/console.trace)
config-file? (= path "logseq/config.edn") ; (string/ends-with? path config/config-file)
_ (when config-file?
(detect-deprecations repo path content))
config-valid? (and config-file? (validate-file repo path content))]
(when-not (and config-file? (not config-valid?)) ; non-config file or valid config
(when (or config-valid? (not config-file?)) ; non-config file or valid config
(let [opts {:new-graph? new-graph?
:from-disk? from-disk?
:skip-db-transact? skip-db-transact?
@@ -149,9 +151,9 @@
(when-not skip-db-transact?
(when-let [page-id (db/get-file-page-id path)]
(db/transact! repo
[[:db/retract page-id :block/alias]
[:db/retract page-id :block/tags]]
opts)))
[[:db/retract page-id :block/alias]
[:db/retract page-id :block/tags]]
opts)))
(file-common-handler/reset-file!
repo path content (merge opts
(when (some? verbose) {:verbose verbose}))))
@@ -161,34 +163,35 @@
(when re-render-root? (ui-handler/re-render-root!))
(cond
(= path (config/get-custom-css-path repo))
(= path "logseq/custom.css") ; (= path (config/get-custom-css-path repo))
(ui-handler/add-style-if-exists!)
(= path (config/get-repo-config-path repo))
(= path "logseq/config.edn") ; (= path (config/get-repo-config-path repo))
(p/let [_ (repo-config-handler/restore-repo-config! repo content)]
(state/pub-event! [:shortcut/refresh]))
;; FIXME: global config
(and (config/global-config-enabled?)
(= path (global-config-handler/global-config-path)))
(p/let [_ (global-config-handler/restore-global-config!)]
(state/pub-event! [:shortcut/refresh]))))
(p/catch
(fn [error]
(when (and (config/global-config-enabled?)
(fn [error]
(when (and (config/global-config-enabled?)
;; Global-config not started correctly but don't
;; know root cause yet
;; https://sentry.io/organizations/logseq/issues/3587411237/events/4b5da8b8e58b4f929bd9e43562213d32/events/?cursor=0%3A0%3A1&project=5311485&statsPeriod=14d
(global-config-handler/global-config-dir-exists?)
(= path (global-config-handler/global-config-path)))
(state/pub-event! [:notification/show
{:content (str "Failed to write to file " path)
:status :error}]))
(global-config-handler/global-config-dir-exists?)
(= path (global-config-handler/global-config-path)))
(state/pub-event! [:notification/show
{:content (str "Failed to write to file " path)
:status :error}]))
(println "Write file failed, path: " path ", content: " content)
(log/error :write/failed error)
(state/pub-event! [:capture-error
{:error error
:payload {:type :write-file/failed-for-alter-file}}]))))
(println "Write file failed, path: " path ", content: " content)
(log/error :write/failed error)
(state/pub-event! [:capture-error
{:error error
:payload {:type :write-file/failed-for-alter-file}}]))))
result))))
(defn set-file-content!

55
src/main/frontend/handler/repo.cljs
View File

@@ -45,27 +45,27 @@
contents-file-exist? (some #(fs/file-exists? repo-dir %) [org-path md-path])]
(when-not contents-file-exist?
(let [format (state/get-preferred-format)
path (str pages-dir "/contents."
(config/get-file-extension format))
file-path (str "/" path)
;;path (str pages-dir "/contents."
;; (config/get-file-extension format))
file-rpath (str "pages/" "contents." (config/get-file-extension format))
default-content (case (name format)
"org" (rc/inline "contents.org")
"markdown" (rc/inline "contents.md")
"")]
(p/let [_ (fs/mkdir-if-not-exists (util/safe-path-join repo-dir pages-dir))
file-exists? (fs/create-if-not-exists repo-url repo-dir file-path default-content)]
file-exists? (fs/create-if-not-exists repo-url repo-dir file-rpath default-content)]
(when-not file-exists?
(file-common-handler/reset-file! repo-url path default-content)))))))
(file-common-handler/reset-file! repo-url file-rpath default-content)))))))
(defn create-custom-theme
[repo-url]
(spec/validate :repos/url repo-url)
(let [repo-dir (config/get-repo-dir repo-url)
path (str config/app-name "/" config/custom-css-file)
file-path (str "/" path)
file-rpath path
default-content ""]
(p/let [_ (fs/mkdir-if-not-exists (util/safe-path-join repo-dir config/app-name))
file-exists? (fs/create-if-not-exists repo-url repo-dir file-path default-content)]
file-exists? (fs/create-if-not-exists repo-url repo-dir file-rpath default-content)]
(when-not file-exists?
(file-common-handler/reset-file! repo-url path default-content)))))
@@ -133,8 +133,10 @@
(defonce *file-tx (atom nil))
(defn- parse-and-load-file!
"Accept: .md, .org, .edn, .css"
[repo-url file {:keys [new-graph? verbose skip-db-transact? extracted-block-ids]
:or {skip-db-transact? true}}]
;; (prn ::parse-and-load-file file)
(try
(reset! *file-tx
(file-handler/alter-file repo-url
@@ -257,21 +259,52 @@
:or {re-render? true}}]
(parse-files-and-create-default-files! repo-url files delete-files delete-blocks re-render? re-render-opts opts))
(defn load-repo-to-db!
[repo-url {:keys [diffs nfs-files refresh? new-graph? empty-graph?]}]
(defn load-new-repo-to-db!
"load graph files to db"
[repo-url {:keys [file-objs new-graph? empty-graph?]}]
(spec/validate :repos/url repo-url)
(route-handler/redirect-to-home!)
(prn ::load---- file-objs repo-url)
(state/set-parsing-state! {:graph-loading? true})
(let [config (or (when-let [content (some-> (first (filter #(= (config/get-repo-config-path repo-url) (:file/path %)) nfs-files))
(let [repo-dir (config/get-local-dir repo-url)
_ (prn ::repo-dir repo-dir)
config (or (when-let [content (some-> (first (filter #(= "logseq/config.edn" (:file/path %)) file-objs))
:file/content)]
(repo-config-handler/read-repo-config content))
(state/get-config repo-url))
_ (prn ::repo-config config)
;; NOTE: Use config while parsing. Make sure it's the current journal title format
;; config should be loaded to state first
_ (state/set-config! repo-url config)
;; remove :hidden files from file-objs, :hidden
file-objs (common-handler/remove-hidden-files file-objs config :file/path)]
(when (seq file-objs)
(parse-files-and-load-to-db! repo-url file-objs {:new-graph? new-graph?
:empty-graph? empty-graph?}))))
(defn load-repo-to-db!
[repo-url {:keys [diffs file-objs refresh? new-graph? empty-graph?]}]
(spec/validate :repos/url repo-url)
(route-handler/redirect-to-home!)
(prn ::load---- file-objs)
(state/set-parsing-state! {:graph-loading? true})
(let [repo-dir (config/get-local-dir repo-url)
_ (prn ::repo-dir repo-dir)
config (or (when-let [content (some-> (first (filter #(= (config/get-repo-config-path repo-url) (:file/path %)) file-objs))
:file/content)]
(repo-config-handler/read-repo-config content))
(state/get-config repo-url))
_ (prn ::repo-config config)
;; NOTE: Use config while parsing. Make sure it's the current journal title format
_ (state/set-config! repo-url config)
relate-path-fn (fn [m k]
(some-> (get m k)
(string/replace (js/decodeURI (config/get-local-dir repo-url)) "")))
nfs-files (common-handler/remove-hidden-files nfs-files config #(relate-path-fn % :file/path))
nfs-files (common-handler/remove-hidden-files file-objs config #(relate-path-fn % :file/path))
diffs (common-handler/remove-hidden-files diffs config #(relate-path-fn % :path))
load-contents (fn [files option]
(file-handler/load-files-contents!

2
src/main/frontend/handler/repo_config.cljs
View File

@@ -44,7 +44,7 @@
(p/let [_ (fs/mkdir-if-not-exists dir)]
(let [default-content config/config-default-content
path (str app-dir "/" config/config-file)]
(p/let [file-exists? (fs/create-if-not-exists repo-url repo-dir (str app-dir "/" config/config-file) default-content)]
(p/let [file-exists? (fs/create-if-not-exists repo-url repo-dir "logseq/config.edn" default-content)]
(when-not file-exists?
(file-common-handler/reset-file! repo-url path default-content)
(set-repo-config-state! repo-url default-content)))))))

90
src/main/frontend/handler/web/nfs.cljs
View File

@@ -39,10 +39,10 @@
%) files)]
(if-let [file (:file/file ignore-file)]
(p/let [content (.text file)]
(when content
(let [paths (set (common-handler/ignore-files content (map :file/path files)))]
(when (seq paths)
(filter (fn [f] (contains? paths (:file/path f))) files)))))
(when content
(let [paths (set (common-handler/ignore-files content (map :file/path files)))]
(when (seq paths)
(filter (fn [f] (contains? paths (:file/path f))) files)))))
(p/resolved files))
(p/resolved files))))
@@ -129,9 +129,7 @@
nfs? (and (not electron?)
(not mobile-native?))
*repo (atom nil)
dir (or dir nil)
dir (some-> dir
(string/replace " " "%20"))]
dir (or dir nil)]
;; TODO: add ext filter to avoid loading .git or other ignored file handlers
(->
(p/let [result (if (fn? dir-result-fn)
@@ -143,12 +141,12 @@
_ (when-not (nil? empty-dir?-or-pred)
(cond
(boolean? empty-dir?-or-pred)
(and (not-empty (second result))
(and (not-empty (:files result))
(throw (js/Error. "EmptyDirOnly")))
(fn? empty-dir?-or-pred)
(empty-dir?-or-pred result)))
root-handle (first result)
root-handle (:path result)
_ (when (fn? picked-root-fn) (picked-root-fn root-handle))
dir-name (if nfs?
(gobj/get root-handle "name")
@@ -166,10 +164,11 @@
_ (when nfs?
(idb/set-item! root-handle-path root-handle)
(nfs/add-nfs-file-handle! root-handle-path root-handle))
result (nth result 1)
files (-> (->db-files mobile-native? electron? dir-name result)
files (:files result)
files (-> (->db-files mobile-native? electron? dir-name files)
(remove-ignore-files dir-name nfs?))
_ (when nfs?
;; only for browserfs
(let [file-paths (set (map :file/path files))]
(swap! path-handles (fn [handles]
(->> handles
@@ -189,12 +188,14 @@
(set-files! @path-handles))
markup-files (filter-markup-and-built-in-files files)]
(-> (p/all (map (fn [file]
;; read file for nfs
(p/let [content (if nfs?
(.text (:file/file file))
(:file/content file))]
(assoc file :file/content content))) markup-files))
(p/then (fn [result]
(p/let [files (map #(dissoc % :file/file) result)
;; handle graphs txid
(p/let [files (mapv #(dissoc % :file/file) result)
graphs-txid-meta (util-fs/read-graphs-txid-info dir-name)
graph-uuid (and (vector? graphs-txid-meta) (second graphs-txid-meta))]
(if-let [exists-graph (state/get-sync-graph-by-id graph-uuid)]
@@ -203,12 +204,13 @@
{:content (str "This graph already exists in \"" (:root exists-graph) "\"")
:status :warning}])
(do
(prn ::prepare-load-new-repo files)
(repo-handler/start-repo-db-if-not-exists! repo)
(async/go
(let [_finished? (async/<! (repo-handler/load-repo-to-db! repo
(let [_finished? (async/<! (repo-handler/load-new-repo-to-db! repo
{:new-graph? true
:empty-graph? (nil? (seq markup-files))
:nfs-files files}))]
:file-objs files}))]
(state/add-repo! {:url repo :nfs? true})
(state/set-loading-files! repo false)
(when ok-handler (ok-handler {:url repo}))
@@ -233,12 +235,12 @@
([path opts]
(when-let [dir-result-fn
(and path (fn [{:keys [path-handles nfs?]}]
(p/let [files-result (fs/get-files
(p/let [files-result (fs/list-files
path
(fn [path handle]
(when nfs?
(swap! path-handles assoc path handle))))]
[path files-result])))]
[path (:files files-result)])))]
(ls-dir-files-with-handler!
(:ok-handler opts)
(merge {:dir-result-fn dir-result-fn} opts)))))
@@ -269,6 +271,7 @@
:deleted deleted}))
(defn- handle-diffs!
"Compute directory diffs and handle them."
[repo nfs? old-files new-files handle-path path-handles re-index? ok-handler]
(let [get-last-modified-at (fn [path] (some (fn [file]
(when (= path (:file/path file))
@@ -341,24 +344,24 @@
(when (or handle electron? mobile-native?) ; electron doesn't store the file handle
(p/let [_ (when handle (nfs/verify-permission repo handle true))
local-files-result
(fs/get-files (if nfs? handle
(config/get-local-dir repo))
(fn [path handle]
(when nfs?
(swap! path-handles assoc path handle))))
new-local-files (-> (->db-files mobile-native? electron? dir-name local-files-result)
(fs/list-files (if nfs? handle
(config/get-local-dir repo))
(fn [path handle]
(when nfs?
(swap! path-handles assoc path handle))))
new-local-files (-> (->db-files mobile-native? electron? dir-name (:files local-files-result))
(remove-ignore-files dir-name nfs?))
new-global-files (if (and (config/global-config-enabled?)
;; Hack until we better understand failure in frontend.handler.file/alter-file
(global-config-handler/global-config-dir-exists?))
(p/let [global-files-result (fs/get-files
(global-config-handler/global-config-dir)
(constantly nil))
global-files (-> (->db-files mobile-native? electron? (global-config-handler/global-config-dir) global-files-result)
(remove-ignore-files (global-config-handler/global-config-dir) nfs?))]
global-files)
(p/resolved []))
new-files (concat new-local-files new-global-files)
;; new-global-files (if (and (config/global-config-enabled?)
;; ;; Hack until we better understand failure in frontend.handler.file/alter-file
;; (global-config-handler/global-config-dir-exists?))
;; (p/let [global-files-result (fs/list-files
;; (global-config-handler/global-config-dir)
;; (constantly nil))
;; global-files (-> (->db-files mobile-native? electron? (global-config-handler/global-config-dir) global-files-result)
;; (remove-ignore-files (global-config-handler/global-config-dir) nfs?))]
;; global-files)
;; (p/resolved []))
new-files new-local-files ;; (concat new-local-files new-global-files)
_ (when nfs?
(let [file-paths (set (map :file/path new-files))]
@@ -369,6 +372,7 @@
(string/replace-first path (str dir-name "/") ""))))
(into {})))))
(set-files! @path-handles))]
(prn ::going-to-handle new-files)
(handle-diffs! repo nfs? old-files new-files handle-path path-handles re-index? ok-handler))))
(p/catch (fn [error]
(log/error :nfs/load-files-error repo)
@@ -382,13 +386,13 @@
(ok-handler)
(state/set-nfs-refreshing! false))]
(when repo
(state/set-nfs-refreshing! true)
(search/reset-indice! repo)
(db/remove-conn! repo)
(db/clear-query-state!)
(db/start-db-conn! repo)
(reload-dir! repo {:re-index? true
:ok-handler ok-handler}))))
(state/set-nfs-refreshing! true)
(search/reset-indice! repo)
(db/remove-conn! repo)
(db/clear-query-state!)
(db/start-db-conn! repo)
(reload-dir! repo {:re-index? true
:ok-handler ok-handler}))))
;; TODO: move to frontend.handler.repo
(defn refresh!
@@ -397,9 +401,9 @@
(ok-handler)
(state/set-nfs-refreshing! false))]
(when (and repo
(not (state/unlinked-dir? (config/get-repo-dir repo))))
(state/set-nfs-refreshing! true)
(reload-dir! repo {:ok-handler ok-handler}))))
(not (state/unlinked-dir? (config/get-repo-dir repo))))
(state/set-nfs-refreshing! true)
(reload-dir! repo {:ok-handler ok-handler}))))
(defn supported?
[]

6
src/main/frontend/modules/file/core.cljs
View File

@@ -125,9 +125,9 @@
whiteboard-page? (config/get-whiteboards-directory)
:else (config/get-pages-directory))
ext (if (= format "markdown") "md" format)
file-path (config/get-page-file-path repo sub-dir filename ext)
file {:file/path file-path}
tx [{:file/path file-path}
file-rpath (str sub-dir "/" filename "." ext) ;; FIXME: use path-join
file {:file/path file-rpath}
tx [{:file/path file-rpath}
{:block/name (:block/name page)
:block/file file}]]
(db/transact! tx)

14
src/main/frontend/util/persist_var.cljs
View File

@@ -9,8 +9,10 @@
[promesa.core :as p]))
(defn- load-path [location]
(config/get-file-path (state/get-current-repo) (str config/app-name "/" location ".edn")))
(defn- load-rpath
"Returns the relative path to the file that stores the persist-var"
[location]
(str config/app-name "/" location ".edn"))
(defprotocol ILoad
(-load [this])
@@ -33,7 +35,7 @@
(p/resolved nil)
(let [repo (state/get-current-repo)
dir (config/get-repo-dir repo)
path (load-path location)]
path (load-rpath location)]
(p/let [file-exists? (fs/file-exists? dir path)]
(when file-exists?
(-> (p/chain (fs/stat dir path)
@@ -44,7 +46,7 @@
(when (not-empty content)
(try (reader/read-string content)
(catch :default e
(println (util/format "read persist-var failed: %s" (load-path location)))
(println (util/format "read persist-var failed: %s" (load-rpath location)))
(js/console.dir e)))))
(fn [value]
(when (some? value)
@@ -53,7 +55,7 @@
(assoc-in [repo :loaded?] true)
(assoc-in [repo :value] value)))))))
(p/catch (fn [e]
(println (util/format "load persist-var failed: %s: %s" (load-path location) e))))))))))
(println (util/format "load persist-var failed: %s: %s" (load-rpath location) e))))))))))
(-loaded? [_]
(get-in @*value [(state/get-current-repo) :loaded?]))
@@ -61,7 +63,7 @@
(-save [_]
(if (config/demo-graph?)
(p/resolved nil)
(let [path (load-path location)
(let [path (load-rpath location)
repo (state/get-current-repo)
content (str (get-in @*value [repo :value]))
dir (config/get-repo-dir repo)]