mirror of
https://github.com/logseq/logseq.git
synced 2026-05-02 18:06:32 +00:00
feat!: File Sync (#5355)
- file sync for electron/ios/android - age encryption of both file content and file path - massive UI enhancement - corresponding CI tasks Co-authored-by: llcc <lzhes43@gmail.com> Co-authored-by: rcmerci <rcmerci@gmail.com> Co-authored-by: Tienson Qin <tiensonqin@gmail.com> Co-authored-by: Andelf <andelf@gmail.com> Co-authored-by: Gabriel Horner <gabriel@logseq.com>
This commit is contained in:
Charlie
@@ -119,7 +119,8 @@
|
||||
|
||||
;; TODO: extract code for `ls-dir-files` and `reload-dir!`
|
||||
(defn ls-dir-files-with-handler!
|
||||
[ok-handler]
|
||||
([ok-handler] (ls-dir-files-with-handler! ok-handler nil))
|
||||
([ok-handler {:keys [empty-dir?-or-pred dir-result-fn]}]
|
||||
(let [path-handles (atom {})
|
||||
electron? (util/electron?)
|
||||
mobile-native? (mobile-util/native-platform?)
|
||||
@@ -128,72 +129,108 @@
|
||||
*repo (atom nil)]
|
||||
;; TODO: add ext filter to avoid loading .git or other ignored file handlers
|
||||
(->
|
||||
(p/let [result (fs/open-dir (fn [path handle]
|
||||
(when nfs?
|
||||
(swap! path-handles assoc path handle))))
|
||||
root-handle (first result)
|
||||
dir-name (if nfs?
|
||||
(gobj/get root-handle "name")
|
||||
root-handle)
|
||||
repo (str config/local-db-prefix dir-name)
|
||||
_ (state/set-loading-files! repo true)
|
||||
_ (when-not (or (state/home?) (state/setups-picker?))
|
||||
(route-handler/redirect-to-home! false))]
|
||||
(reset! *repo repo)
|
||||
(when-not (string/blank? dir-name)
|
||||
(p/let [root-handle-path (str config/local-handle-prefix dir-name)
|
||||
_ (when nfs?
|
||||
(idb/set-item! root-handle-path root-handle)
|
||||
(nfs/add-nfs-file-handle! root-handle-path root-handle))
|
||||
result (nth result 1)
|
||||
files (-> (->db-files mobile-native? electron? dir-name result)
|
||||
(remove-ignore-files dir-name nfs?))
|
||||
_ (when nfs?
|
||||
(let [file-paths (set (map :file/path files))]
|
||||
(swap! path-handles (fn [handles]
|
||||
(->> handles
|
||||
(filter (fn [[path _handle]]
|
||||
(or
|
||||
(contains? file-paths
|
||||
(string/replace-first path (str dir-name "/") ""))
|
||||
(let [last-part (last (string/split path "/"))]
|
||||
(contains? #{config/app-name
|
||||
gp-config/default-draw-directory
|
||||
(config/get-journals-directory)
|
||||
(config/get-pages-directory)}
|
||||
last-part)))))
|
||||
(into {})))))
|
||||
(p/let [result (if (fn? dir-result-fn)
|
||||
(dir-result-fn {:path-handles path-handles :nfs? nfs?})
|
||||
(fs/open-dir (fn [path handle]
|
||||
(when nfs?
|
||||
(swap! path-handles assoc path handle)))))
|
||||
_ (when-not (nil? empty-dir?-or-pred)
|
||||
(cond
|
||||
(boolean? empty-dir?-or-pred)
|
||||
(and (not-empty (second result))
|
||||
(throw (js/Error. "EmptyDirOnly")))
|
||||
|
||||
(set-files! @path-handles))
|
||||
markup-files (filter-markup-and-built-in-files files)]
|
||||
(-> (p/all (map (fn [file]
|
||||
(p/let [content (if nfs?
|
||||
(.text (:file/file file))
|
||||
(:file/content file))
|
||||
content (encrypt/decrypt content)]
|
||||
(assoc file :file/content content))) markup-files))
|
||||
(p/then (fn [result]
|
||||
(let [files (map #(dissoc % :file/file) result)]
|
||||
(repo-handler/start-repo-db-if-not-exists! repo)
|
||||
(async/go
|
||||
(let [_finished? (async/<! (repo-handler/load-repo-to-db! repo
|
||||
{:new-graph? true
|
||||
:empty-graph? (nil? (seq markup-files))
|
||||
:nfs-files files}))]
|
||||
(state/add-repo! {:url repo :nfs? true})
|
||||
(state/set-loading-files! repo false)
|
||||
(when ok-handler (ok-handler))
|
||||
(fs/watch-dir! dir-name)
|
||||
(db/persist-if-idle! repo))))))
|
||||
(p/catch (fn [error]
|
||||
(log/error :nfs/load-files-error repo)
|
||||
(log/error :exception error)))))))
|
||||
(p/catch (fn [error]
|
||||
(log/error :exception error)
|
||||
(when (contains? #{"AbortError" "Error"} (gobj/get error "name"))
|
||||
(when @*repo (state/set-loading-files! @*repo false))
|
||||
;; (log/error :nfs/open-dir-error error)
|
||||
))))))
|
||||
(fn? empty-dir?-or-pred)
|
||||
(empty-dir?-or-pred result)))
|
||||
root-handle (first result)
|
||||
dir-name (if nfs?
|
||||
(gobj/get root-handle "name")
|
||||
root-handle)
|
||||
repo (str config/local-db-prefix dir-name)
|
||||
_ (state/set-loading-files! repo true)
|
||||
_ (when-not (or (state/home?) (state/setups-picker?))
|
||||
(route-handler/redirect-to-home! false))]
|
||||
(reset! *repo repo)
|
||||
(when-not (string/blank? dir-name)
|
||||
(p/let [root-handle-path (str config/local-handle-prefix dir-name)
|
||||
_ (when nfs?
|
||||
(idb/set-item! root-handle-path root-handle)
|
||||
(nfs/add-nfs-file-handle! root-handle-path root-handle))
|
||||
result (nth result 1)
|
||||
files (-> (->db-files mobile-native? electron? dir-name result)
|
||||
(remove-ignore-files dir-name nfs?))
|
||||
_ (when nfs?
|
||||
(let [file-paths (set (map :file/path files))]
|
||||
(swap! path-handles (fn [handles]
|
||||
(->> handles
|
||||
(filter (fn [[path _handle]]
|
||||
(or
|
||||
(contains? file-paths
|
||||
(string/replace-first path (str dir-name "/") ""))
|
||||
(let [last-part (last (string/split path "/"))]
|
||||
(contains? #{config/app-name
|
||||
gp-config/default-draw-directory
|
||||
(config/get-journals-directory)
|
||||
(config/get-pages-directory)}
|
||||
last-part)))))
|
||||
(into {})))))
|
||||
|
||||
(set-files! @path-handles))
|
||||
markup-files (filter-markup-and-built-in-files files)]
|
||||
(-> (p/all (map (fn [file]
|
||||
(p/let [content (if nfs?
|
||||
(.text (:file/file file))
|
||||
(:file/content file))
|
||||
content (encrypt/decrypt content)]
|
||||
(assoc file :file/content content))) markup-files))
|
||||
(p/then (fn [result]
|
||||
(p/let [files (map #(dissoc % :file/file) result)
|
||||
graph-txid-meta (util-fs/read-graph-txid-info dir-name)
|
||||
graph-uuid (and (vector? graph-txid-meta) (second graph-txid-meta))]
|
||||
(if-let [exists-graph (state/get-sync-graph-by-uuid graph-uuid)]
|
||||
(state/pub-event!
|
||||
[:notification/show
|
||||
{:content (str "This graph already exists in \"" (:root exists-graph) "\"")
|
||||
:status :warning}])
|
||||
(do
|
||||
(repo-handler/start-repo-db-if-not-exists! repo)
|
||||
(async/go
|
||||
(let [_finished? (async/<! (repo-handler/load-repo-to-db! repo
|
||||
{:new-graph? true
|
||||
:empty-graph? (nil? (seq markup-files))
|
||||
:nfs-files files}))]
|
||||
(state/add-repo! {:url repo :nfs? true})
|
||||
(state/set-loading-files! repo false)
|
||||
(when ok-handler (ok-handler {:url repo}))
|
||||
(fs/watch-dir! dir-name)
|
||||
(db/persist-if-idle! repo))))))))
|
||||
(p/catch (fn [error]
|
||||
(log/error :nfs/load-files-error repo)
|
||||
(log/error :exception error)))))))
|
||||
(p/catch (fn [error]
|
||||
(log/error :exception error)
|
||||
(when mobile-native?
|
||||
(state/pub-event!
|
||||
[:notification/show {:content (str error) :status :error}]))
|
||||
(when (contains? #{"AbortError" "Error"} (gobj/get error "name"))
|
||||
(when @*repo (state/set-loading-files! @*repo false))
|
||||
(throw error)
|
||||
)))))))
|
||||
|
||||
(defn ls-dir-files-with-path!
|
||||
([path] (ls-dir-files-with-path! path nil))
|
||||
([path opts]
|
||||
(when-let [dir-result-fn
|
||||
(and path (fn [{:keys [path-handles nfs?]}]
|
||||
(p/let [files-result (fs/get-files
|
||||
path
|
||||
(fn [path handle]
|
||||
(when nfs?
|
||||
(swap! path-handles assoc path handle))))]
|
||||
[path files-result])))]
|
||||
(ls-dir-files-with-handler!
|
||||
(:ok-handler opts)
|
||||
(merge {:dir-result-fn dir-result-fn} opts)))))
|
||||
|
||||
(defn- compute-diffs
|
||||
[old-files new-files]
|
||||
|
||||
Reference in New Issue
Block a user