mirror of
https://github.com/logseq/logseq.git
synced 2026-02-01 14:43:56 +00:00
chore: rm unused file-sync bb tasks
This commit is contained in:
Gabriel Horner
3
bb.edn
3
bb.edn
@@ -209,9 +209,6 @@
|
||||
lang:validate-translations
|
||||
logseq.tasks.lang/validate-translations
|
||||
|
||||
file-sync:integration-tests
|
||||
logseq.tasks.file-sync/integration-tests
|
||||
|
||||
ai:check-common-errors
|
||||
logseq.tasks.common-errors/check-common-errors}
|
||||
|
||||
|
||||
@@ -1,201 +0,0 @@
|
||||
(ns logseq.tasks.file-sync
|
||||
"Run integration tests on file-sync service. Instructions:
|
||||
|
||||
* Login to electron app and toggle file-sync on
|
||||
* Set up file-sync-auth.json file per #'read-config
|
||||
* Run `bb file-sync:integration-tests`
|
||||
* Wait for test results. Each action takes 10-20s and prints results as it goes"
|
||||
(:require [clojure.string :as str]
|
||||
[cheshire.core :as json]
|
||||
[clojure.pprint :as pp]
|
||||
[babashka.fs :as fs]
|
||||
[babashka.curl :as curl]
|
||||
[clojure.data :as data]
|
||||
[clojure.test :as t :refer [deftest is]]
|
||||
[clj-commons.digest :as digest]
|
||||
[logseq.tasks.file-sync-actions :as file-sync-actions])
|
||||
(:import (java.net URLDecoder)))
|
||||
|
||||
;; Root directory for graph that is being tested
|
||||
(defonce root-dir (atom nil))
|
||||
|
||||
;; Graph id for given graph
|
||||
(defonce root-graph-id (atom nil))
|
||||
|
||||
(defn- read-config*
|
||||
[]
|
||||
(-> "file-sync-auth.json" slurp json/parse-string))
|
||||
|
||||
(def read-config
|
||||
"file-sync-auth.json file is populated by user right clicking on
|
||||
logseq.com/auth_callback request in Network tab, choosing Copy > 'Copy
|
||||
response' and saving"
|
||||
;; Only want to read this once
|
||||
(memoize read-config*))
|
||||
|
||||
(defn- post
|
||||
[url headers]
|
||||
(println "-> POST" url)
|
||||
(let [resp (curl/post url (merge headers {:throw false}))]
|
||||
(if (not= 200 (:status resp))
|
||||
(throw (ex-info (str "Response failed with: " (select-keys resp [:status :body])) (select-keys resp [:status :body])))
|
||||
resp)))
|
||||
|
||||
(defn- build-headers
|
||||
[]
|
||||
(let [{:strs [id_token]} (read-config)]
|
||||
{"authorization" (str "Bearer " id_token)}))
|
||||
|
||||
(defn- api-get-all-files
|
||||
[graph-id subdir]
|
||||
(let [body (json/generate-string {"GraphUUID" graph-id
|
||||
"Dir" subdir})
|
||||
resp (post "https://api-dev.logseq.com/file-sync/get_all_files"
|
||||
{:headers (build-headers)
|
||||
:body body})
|
||||
body (json/parse-string (:body resp) keyword)]
|
||||
(->> body
|
||||
:Objects
|
||||
(map (juxt (comp #(URLDecoder/decode %) fs/file-name :Key) :checksum)))))
|
||||
|
||||
(defn- get-local-all-files
|
||||
[dir subdir]
|
||||
(let [files (map fs/file (fs/list-dir (fs/file dir subdir)))
|
||||
f (juxt fs/file-name digest/md5)]
|
||||
(map f files)))
|
||||
|
||||
|
||||
(defn- api-post-get-graphs
|
||||
[]
|
||||
(let [resp (post "https://api-dev.logseq.com/file-sync/list_graphs"
|
||||
{:headers (build-headers)})
|
||||
body (json/parse-string (:body resp) keyword)]
|
||||
(->> body
|
||||
:Graphs
|
||||
(map (juxt :GraphName :GraphUUID))
|
||||
(into {}))))
|
||||
|
||||
(defmulti run-action* :action)
|
||||
|
||||
(defmethod run-action* :create-file
|
||||
[{{:keys [file blocks dir]} :args}]
|
||||
(spit (fs/file dir file)
|
||||
(->> blocks
|
||||
(map #(str "- " %))
|
||||
(str/join "\n"))))
|
||||
|
||||
(defmethod run-action* :delete-file
|
||||
[{{:keys [file dir]} :args}]
|
||||
(fs/delete (fs/file dir file)))
|
||||
|
||||
(defmethod run-action* :move-file
|
||||
[{{:keys [file dir new-file]} :args}]
|
||||
(fs/move (fs/file dir file)
|
||||
(fs/file dir new-file)))
|
||||
|
||||
(defmethod run-action* :update-file
|
||||
[{{:keys [file blocks dir]} :args}]
|
||||
(let [origin-content (slurp (fs/file dir file))
|
||||
new-content (str (str/trim-newline origin-content) "\n"
|
||||
(->> blocks
|
||||
(map #(str "- " %))
|
||||
(str/join "\n")))]
|
||||
(spit (fs/file dir file) new-content)))
|
||||
|
||||
(defn run-action [action-map]
|
||||
(println "===\nRUN")
|
||||
(pp/pprint ((juxt :action #(get-in % [:args :file])) action-map))
|
||||
(println "===")
|
||||
(run-action* action-map))
|
||||
|
||||
(defn- ensure-dir-is-synced!
|
||||
[dir graph-id subdir]
|
||||
(let [actual (set (get-local-all-files dir subdir))
|
||||
expected (set (api-get-all-files graph-id subdir))]
|
||||
(assert (= actual expected)
|
||||
(let [[local-only remote-only _] (data/diff actual expected)]
|
||||
(format "Files in '%s' are not synced yet:\nLocal only files: %s\nRemote only files: %s"
|
||||
subdir
|
||||
local-only
|
||||
remote-only)))))
|
||||
|
||||
(defn- try-fn-n-times
|
||||
"Tries a fn for max-attempts times, returning true if fn returns true.
|
||||
Otherwise returns false. Sleeps for 2 seconds between attempts by default"
|
||||
[f max-attempts & {:keys [sleep-duration] :or {sleep-duration 2000}}]
|
||||
(loop [attempt 1]
|
||||
(println "Try" attempt)
|
||||
(let [ret (f)]
|
||||
(cond
|
||||
(true? ret)
|
||||
true
|
||||
(= attempt max-attempts)
|
||||
false
|
||||
:else
|
||||
(do
|
||||
(Thread/sleep sleep-duration)
|
||||
(recur (inc attempt)))))))
|
||||
|
||||
(defn- files-are-in-sync?
|
||||
[dir graph-id subdir]
|
||||
(try (ensure-dir-is-synced! dir graph-id subdir)
|
||||
true
|
||||
(catch Throwable e
|
||||
(println (.getMessage e))
|
||||
false)))
|
||||
|
||||
(defn- wait&files-are-in-sync?
|
||||
[dir graph-id subdir & [msg]]
|
||||
;; Approximate polling time before file changes are picked up by client
|
||||
(if msg
|
||||
(println msg)
|
||||
(println "Wait 10s for logseq to pick up changes..."))
|
||||
(Thread/sleep 10000)
|
||||
(try-fn-n-times #(files-are-in-sync? dir graph-id subdir) 10))
|
||||
|
||||
(defn- clear-dir-pages
|
||||
[subdir]
|
||||
(fs/delete-tree (str @root-dir "/" subdir))
|
||||
(fs/create-dir (str @root-dir "/" subdir)))
|
||||
|
||||
(deftest rand-file-changes
|
||||
(let [subdir "pages"
|
||||
actions (:generated-action (file-sync-actions/generate-rand-actions 30))
|
||||
actions (mapv
|
||||
#(assoc-in % [:args :dir] @root-dir)
|
||||
actions)
|
||||
partitioned-actions (partition-all 3 actions)]
|
||||
(clear-dir-pages subdir)
|
||||
(wait&files-are-in-sync? @root-dir @root-graph-id subdir
|
||||
(format "clear dir [%s], and ensure it's in sync" subdir))
|
||||
(doseq [actions partitioned-actions]
|
||||
(doseq [action actions]
|
||||
(run-action action)
|
||||
(Thread/sleep 1000))
|
||||
(is (wait&files-are-in-sync? @root-dir @root-graph-id subdir)
|
||||
(str "Test " (mapv (juxt :action #(get-in % [:args :file])) actions))))))
|
||||
|
||||
(defn setup-vars
|
||||
[]
|
||||
(let [{:strs [dir]} (read-config)
|
||||
graph-names-to-ids (api-post-get-graphs)
|
||||
graph-id (get graph-names-to-ids (fs/file-name dir))]
|
||||
(assert dir "No graph id for given dir")
|
||||
(reset! root-dir dir)
|
||||
(reset! root-graph-id graph-id)))
|
||||
|
||||
(defn integration-tests
|
||||
"Run file-sync integration tests on graph directory
|
||||
requirements:
|
||||
|
||||
* file-sync-auth.json, and it looks like:
|
||||
```
|
||||
{\"id_token\":\"<id-token>\",
|
||||
\"dir\": \"/Users/me/Documents/untitled folder 31\"}
|
||||
```
|
||||
|
||||
* you also need to open logseq-app(or yarn electron-watch),
|
||||
and open <dir> and start file-sync"
|
||||
[& _args]
|
||||
(setup-vars)
|
||||
(t/run-tests 'logseq.tasks.file-sync))
|
||||
@@ -1,164 +0,0 @@
|
||||
(ns logseq.tasks.file-sync-actions
|
||||
(:require [clojure.test.check.generators :as gen]))
|
||||
|
||||
|
||||
(defmulti gen-action* (fn [& args] (first args)))
|
||||
|
||||
(defmethod gen-action* :create-file
|
||||
[_ page-index & _args]
|
||||
(gen/let [blocks (gen/vector gen/string-alphanumeric 1 10)]
|
||||
{:action :create-file
|
||||
:args {:file (format "pages/test.page-%d.md" page-index)
|
||||
:blocks blocks}}))
|
||||
|
||||
(defmethod gen-action* :move-file
|
||||
[_ origin-page-index & [moved?]]
|
||||
(let [page-name (if moved?
|
||||
(format "pages/test.page-move-%d.md" origin-page-index)
|
||||
(format "pages/test.page-%d.md" origin-page-index))]
|
||||
(gen/return
|
||||
{:action :move-file
|
||||
:args {:file page-name
|
||||
:new-file (format "pages/test.page-move-%d.md" origin-page-index)}})))
|
||||
|
||||
(defmethod gen-action* :update-file
|
||||
[_ page-index & [moved?]]
|
||||
(gen/let [append-blocks (gen/vector gen/string-alphanumeric 1 10)]
|
||||
(let [page-name (if moved?
|
||||
(format "pages/test.page-move-%d.md" page-index)
|
||||
(format "pages/test.page-%d.md" page-index))]
|
||||
{:action :update-file
|
||||
:args {:file page-name
|
||||
:blocks append-blocks}})))
|
||||
|
||||
(defmethod gen-action* :delete-file
|
||||
[_ page-index & [moved?]]
|
||||
(let [page-name (if moved?
|
||||
(format "pages/test.page-move-%d.md" page-index)
|
||||
(format "pages/test.page-%d.md" page-index))]
|
||||
(gen/return
|
||||
{:action :delete-file
|
||||
:args {:file page-name}})))
|
||||
|
||||
|
||||
(defmacro gen-actions-plan
|
||||
"state monad
|
||||
state: {:page-index [{:index 1 :moved? false}, ...]
|
||||
:generated-action [...]}
|
||||
|
||||
(gen-actions-plan
|
||||
[id+moved? get-rand-available-index-op
|
||||
_ (when-op id+moved? (apply action-op action id+moved?))]
|
||||
nil)"
|
||||
[binds val-expr]
|
||||
(let [binds (partition 2 binds)
|
||||
psym (gensym "state_")
|
||||
forms (reduce (fn [acc [id expr]]
|
||||
(concat acc `[[~id ~psym] (~expr ~psym)]))
|
||||
[]
|
||||
binds)]
|
||||
`(fn [~psym]
|
||||
(let [~@forms]
|
||||
[~val-expr ~psym]))))
|
||||
|
||||
(defn- all-indexes
|
||||
[state]
|
||||
(let [r (map :index (:page-index state))]
|
||||
(if (empty? r) '(0) r)))
|
||||
|
||||
(defn- add-index
|
||||
[state index moved?]
|
||||
(update state :page-index conj {:index index :moved? moved?}))
|
||||
|
||||
(defn- assign-page-index-op
|
||||
[state]
|
||||
(let [max-index (apply max (all-indexes state))
|
||||
next-index (inc max-index)]
|
||||
[next-index (add-index state next-index false)]))
|
||||
|
||||
(defn- get-rand-available-index-op
|
||||
[state]
|
||||
(let [indexes (:page-index state)]
|
||||
(if (empty? indexes)
|
||||
[nil state]
|
||||
(let [rand-index (rand-nth (vec indexes))]
|
||||
[((juxt :index :moved?) rand-index) state]))))
|
||||
|
||||
(defn- action-op
|
||||
[action id & args]
|
||||
(fn [state]
|
||||
(let [generated-action (gen/generate (apply gen-action* action id args))
|
||||
[moved?] args
|
||||
state* (update state :generated-action conj generated-action)
|
||||
state* (case action
|
||||
:move-file
|
||||
(update state* :page-index
|
||||
#(-> %
|
||||
(disj {:index id :moved? (boolean moved?)})
|
||||
(conj {:index id :moved? true})))
|
||||
:delete-file
|
||||
(update state* :page-index
|
||||
#(disj % {:index id :moved? (boolean moved?)}))
|
||||
state*)]
|
||||
[nil state*])))
|
||||
|
||||
(defmacro when-op
|
||||
[x f]
|
||||
`(fn [state#]
|
||||
(if ~x
|
||||
(~f state#)
|
||||
[nil state#])))
|
||||
|
||||
(defn- print-op
|
||||
[x]
|
||||
(fn [state]
|
||||
(println x)
|
||||
[nil state]))
|
||||
|
||||
|
||||
(defn rand-action-op
|
||||
[]
|
||||
(let [action (gen/generate
|
||||
(gen/frequency [[5 (gen/return :update-file)]
|
||||
[2 (gen/return :create-file)]
|
||||
[2 (gen/return :move-file)]
|
||||
[1 (gen/return :delete-file)]]))]
|
||||
(case action
|
||||
:create-file
|
||||
(gen-actions-plan
|
||||
[id assign-page-index-op
|
||||
_ (action-op action id)]
|
||||
nil)
|
||||
:update-file
|
||||
(gen-actions-plan
|
||||
[id+moved? get-rand-available-index-op
|
||||
_ (when-op id+moved? (apply action-op action id+moved?))]
|
||||
nil)
|
||||
:move-file
|
||||
(gen-actions-plan
|
||||
[id+moved? get-rand-available-index-op
|
||||
_ (when-op id+moved? (apply action-op action id+moved?))]
|
||||
nil)
|
||||
:delete-file
|
||||
(gen-actions-plan
|
||||
[id+moved? get-rand-available-index-op
|
||||
_ (when-op id+moved? (apply action-op action id+moved? ))]
|
||||
nil))))
|
||||
|
||||
(def empty-actions-plan {:page-index #{}
|
||||
:generated-action []})
|
||||
|
||||
|
||||
(defmacro generate-rand-actions
|
||||
[max-n & {:keys [pre-create-files-n] :or {pre-create-files-n 2}}]
|
||||
(let [pre-create-files-binds
|
||||
(for [id (map (fn [_] (gensym)) (range pre-create-files-n))]
|
||||
`[~id assign-page-index-op
|
||||
~'_ (action-op :create-file ~id)])
|
||||
binds (apply concat
|
||||
(concat pre-create-files-binds (repeat max-n `[~'_ (rand-action-op)])))]
|
||||
`(second
|
||||
((gen-actions-plan
|
||||
~binds
|
||||
nil)
|
||||
empty-actions-plan))))
|
||||
Reference in New Issue
Block a user