clone/logseq
1
0
Fork 0
mirror of https://github.com/logseq/logseq.git synced 2026-05-04 19:06:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

enhance(sync): add get-deletion-logs

Browse Source 
remove local files according to deletion-logs when local->remote-full-sync
This commit is contained in:
rcmerci
2022-09-15 13:17:49 +08:00
committed by Andelf
parent bada72fe28
commit ef8d74d68b
1 changed files with 65 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
src/main/frontend/fs/sync.cljs
View File

@@ -653,6 +653,7 @@
(<get-remote-graph [this graph-name-opt graph-uuid-opt] "get graph info by GRAPH-NAME-OPT or GRAPH-UUID-OPT")
(<get-remote-file-versions [this graph-uuid filepath] "get file's version list")
(<list-remote-graphs [this] "list all remote graphs")
(<get-deletion-logs [this graph-uuid from-txid] "get deletion logs from FROM-TXID")
(<get-diff [this graph-uuid from-txid] "get diff from FROM-TXID, return [txns, latest-txid, min-txid]")
(<create-graph [this graph-name] "create graph")
(<delete-graph [this graph-uuid] "delete graph")
@@ -1104,9 +1105,9 @@
(recur next-continuation-token)))))))]
(if (instance? ExceptionInfo exp-r)
exp-r
(let [file-meta-list* (persistent! file-meta-list)
encrypted-path-list* (persistent! encrypted-path-list)
path-list-or-exp (<! (<decrypt-fnames rsapi encrypted-path-list*))]
(let [file-meta-list* (persistent! file-meta-list)
encrypted-path-list* (persistent! encrypted-path-list)
path-list-or-exp (<! (<decrypt-fnames rsapi encrypted-path-list*))]
(if (instance? ExceptionInfo path-list-or-exp)
path-list-or-exp
(let [encrypted-path->path-map (zipmap encrypted-path-list* path-list-or-exp)]
@@ -1157,6 +1158,23 @@
(<list-remote-graphs [this]
(.<request this "list_graphs"))
(<get-deletion-logs [this graph-uuid from-txid]
(go
(let [r (<! (.<request this "get_deletion_log" {:GraphUUID graph-uuid :FromTXId from-txid}))]
(if (instance? ExceptionInfo r)
r
(let [txns-with-encrypted-paths (mapv #(update % :path remove-user-graph-uuid-prefix) (:Transactions r))
encrypted-paths (mapv :path txns-with-encrypted-paths)
encrypted-path->path-map
(zipmap
encrypted-paths
(<! (<decrypt-fnames rsapi encrypted-paths)))
txns
(mapv
(fn [txn] (update txn :path #(get encrypted-path->path-map %)))
txns-with-encrypted-paths)]
txns)))))
(<get-diff [this graph-uuid from-txid]
;; TODO: path in transactions should be relative path(now s3 key, which includes graph-uuid and user-uuid)
(go
@@ -2190,6 +2208,21 @@
(every? #(instance? FileChangeEvent %) es)]}
(filterv filter-too-huge-files-aux es))
(defn- filter-local-files-in-deletion-logs
[local-all-files-meta deletion-logs]
(let [deletion-logs-map (into {} (map (juxt :path identity)) deletion-logs)
*keep (transient #{})
*delete (transient #{})]
(doseq [f local-all-files-meta]
(let [epoch-long (some-> (get deletion-logs-map (:path f))
:epoch
(* 1000))]
(if (and epoch-long (> epoch-long (:last-modified f)))
(conj! *delete f)
(conj! *keep f))))
{:keep (persistent! *keep)
:delete (persistent! *delete)}))
(defrecord ^:large-vars/cleanup-todo
Local->RemoteSyncer [user-uuid graph-uuid base-path repo *sync-state remoteapi
@@ -2207,7 +2240,7 @@
true)
(or (string/starts-with? (.-dir e) base-path)
(string/starts-with? (str "file://" (.-dir e)) base-path)) ; valid path prefix
(not (ignored? e)) ;not ignored
(not (ignored? e)) ;not ignored
;; download files will also trigger file-change-events, ignore them
(let [r (not (contains? (:recent-remote->local-files @*sync-state)
(<! (<file-change-event=>recent-remote->local-file-item e))))]
@@ -2253,10 +2286,10 @@
(<sync-local->remote! [_ es]
(if (empty? es)
(go {:succ true})
(let [type (.-type ^FileChangeEvent (first es))
es->paths-xf (comp
(map #(relative-path %))
(remove ignored?))]
(let [type (.-type ^FileChangeEvent (first es))
es->paths-xf (comp
(map #(relative-path %))
(remove ignored?))]
(go
(let [es* (<! (<filter-checksum-not-consistent es))
_ (when (not= (count es*) (count es))
@@ -2316,7 +2349,9 @@
(go
(let [remote-all-files-meta-c (<get-remote-all-files-meta remoteapi graph-uuid)
local-all-files-meta-c (<get-local-all-files-meta rsapi graph-uuid base-path)
remote-all-files-meta-or-exp (<! remote-all-files-meta-c)]
deletion-logs-c (<get-deletion-logs remoteapi graph-uuid @*txid)
remote-all-files-meta-or-exp (<! remote-all-files-meta-c)
deletion-logs (<! deletion-logs-c)]
(if (or (storage-exceed-limit? remote-all-files-meta-or-exp)
(sync-stop-when-api-flying? remote-all-files-meta-or-exp)
(decrypt-exp? remote-all-files-meta-or-exp))
@@ -2327,6 +2362,8 @@
{:stop true})
(let [remote-all-files-meta remote-all-files-meta-or-exp
local-all-files-meta (<! local-all-files-meta-c)
{local-all-files-meta :keep delete-local-files :delete}
(filter-local-files-in-deletion-logs local-all-files-meta deletion-logs)
diff-local-files (diff-file-metadata-sets local-all-files-meta remote-all-files-meta)
change-events
(sequence
@@ -2342,7 +2379,25 @@
(partition-file-change-events 10)
(distinct-file-change-events change-events))]
(println "[full-sync(local->remote)]"
(count (flatten change-events-partitions)) "files need to sync")
(count (flatten change-events-partitions)) "files need to sync and"
(count delete-local-files) "local files need to delete")
;; 1. delete local files
(loop [[f & fs] delete-local-files]
(when f
(let [relative-p (relative-path f)]
(when-not (<! (<local-file-not-exist? rsapi base-path relative-p))
(let [fake-recent-remote->local-file-item {:remote->local-type :delete
:checksum nil
:path relative-p}]
(swap! *sync-state sync-state--add-recent-remote->local-files
[fake-recent-remote->local-file-item])
(<! (<delete-local-files rsapi graph-uuid base-path [(relative-path f)]))
(go (<! (timeout 5000))
(swap! *sync-state sync-state--remove-recent-remote->local-files
[fake-recent-remote->local-file-item])))))
(recur fs)))
;; 2. upload local files
(loop [es-partitions change-events-partitions]
(if @*stopped
{:stop true}