diff --git a/bb.edn b/bb.edn index 360ef27d3b..43ab0b6855 100644 --- a/bb.edn +++ b/bb.edn @@ -209,9 +209,6 @@ lang:validate-translations logseq.tasks.lang/validate-translations - file-sync:integration-tests - logseq.tasks.file-sync/integration-tests - ai:check-common-errors logseq.tasks.common-errors/check-common-errors} diff --git a/scripts/src/logseq/tasks/file_sync.clj b/scripts/src/logseq/tasks/file_sync.clj deleted file mode 100644 index 39a522336f..0000000000 --- a/scripts/src/logseq/tasks/file_sync.clj +++ /dev/null @@ -1,201 +0,0 @@ -(ns logseq.tasks.file-sync - "Run integration tests on file-sync service. Instructions: - -* Login to electron app and toggle file-sync on -* Set up file-sync-auth.json file per #'read-config -* Run `bb file-sync:integration-tests` -* Wait for test results. Each action takes 10-20s and prints results as it goes" - (:require [clojure.string :as str] - [cheshire.core :as json] - [clojure.pprint :as pp] - [babashka.fs :as fs] - [babashka.curl :as curl] - [clojure.data :as data] - [clojure.test :as t :refer [deftest is]] - [clj-commons.digest :as digest] - [logseq.tasks.file-sync-actions :as file-sync-actions]) - (:import (java.net URLDecoder))) - -;; Root directory for graph that is being tested -(defonce root-dir (atom nil)) - -;; Graph id for given graph -(defonce root-graph-id (atom nil)) - -(defn- read-config* - [] - (-> "file-sync-auth.json" slurp json/parse-string)) - -(def read-config - "file-sync-auth.json file is populated by user right clicking on - logseq.com/auth_callback request in Network tab, choosing Copy > 'Copy - response' and saving" - ;; Only want to read this once - (memoize read-config*)) - -(defn- post - [url headers] - (println "-> POST" url) - (let [resp (curl/post url (merge headers {:throw false}))] - (if (not= 200 (:status resp)) - (throw (ex-info (str "Response failed with: " (select-keys resp [:status :body])) (select-keys resp [:status :body]))) - resp))) - -(defn- build-headers - [] - (let [{:strs [id_token]} (read-config)] - {"authorization" (str "Bearer " id_token)})) - -(defn- api-get-all-files - [graph-id subdir] - (let [body (json/generate-string {"GraphUUID" graph-id - "Dir" subdir}) - resp (post "https://api-dev.logseq.com/file-sync/get_all_files" - {:headers (build-headers) - :body body}) - body (json/parse-string (:body resp) keyword)] - (->> body - :Objects - (map (juxt (comp #(URLDecoder/decode %) fs/file-name :Key) :checksum))))) - -(defn- get-local-all-files - [dir subdir] - (let [files (map fs/file (fs/list-dir (fs/file dir subdir))) - f (juxt fs/file-name digest/md5)] - (map f files))) - - -(defn- api-post-get-graphs - [] - (let [resp (post "https://api-dev.logseq.com/file-sync/list_graphs" - {:headers (build-headers)}) - body (json/parse-string (:body resp) keyword)] - (->> body - :Graphs - (map (juxt :GraphName :GraphUUID)) - (into {})))) - -(defmulti run-action* :action) - -(defmethod run-action* :create-file - [{{:keys [file blocks dir]} :args}] - (spit (fs/file dir file) - (->> blocks - (map #(str "- " %)) - (str/join "\n")))) - -(defmethod run-action* :delete-file - [{{:keys [file dir]} :args}] - (fs/delete (fs/file dir file))) - -(defmethod run-action* :move-file - [{{:keys [file dir new-file]} :args}] - (fs/move (fs/file dir file) - (fs/file dir new-file))) - -(defmethod run-action* :update-file - [{{:keys [file blocks dir]} :args}] - (let [origin-content (slurp (fs/file dir file)) - new-content (str (str/trim-newline origin-content) "\n" - (->> blocks - (map #(str "- " %)) - (str/join "\n")))] - (spit (fs/file dir file) new-content))) - -(defn run-action [action-map] - (println "===\nRUN") - (pp/pprint ((juxt :action #(get-in % [:args :file])) action-map)) - (println "===") - (run-action* action-map)) - -(defn- ensure-dir-is-synced! - [dir graph-id subdir] - (let [actual (set (get-local-all-files dir subdir)) - expected (set (api-get-all-files graph-id subdir))] - (assert (= actual expected) - (let [[local-only remote-only _] (data/diff actual expected)] - (format "Files in '%s' are not synced yet:\nLocal only files: %s\nRemote only files: %s" - subdir - local-only - remote-only))))) - -(defn- try-fn-n-times - "Tries a fn for max-attempts times, returning true if fn returns true. - Otherwise returns false. Sleeps for 2 seconds between attempts by default" - [f max-attempts & {:keys [sleep-duration] :or {sleep-duration 2000}}] - (loop [attempt 1] - (println "Try" attempt) - (let [ret (f)] - (cond - (true? ret) - true - (= attempt max-attempts) - false - :else - (do - (Thread/sleep sleep-duration) - (recur (inc attempt))))))) - -(defn- files-are-in-sync? - [dir graph-id subdir] - (try (ensure-dir-is-synced! dir graph-id subdir) - true - (catch Throwable e - (println (.getMessage e)) - false))) - -(defn- wait&files-are-in-sync? - [dir graph-id subdir & [msg]] - ;; Approximate polling time before file changes are picked up by client - (if msg - (println msg) - (println "Wait 10s for logseq to pick up changes...")) - (Thread/sleep 10000) - (try-fn-n-times #(files-are-in-sync? dir graph-id subdir) 10)) - -(defn- clear-dir-pages - [subdir] - (fs/delete-tree (str @root-dir "/" subdir)) - (fs/create-dir (str @root-dir "/" subdir))) - -(deftest rand-file-changes - (let [subdir "pages" - actions (:generated-action (file-sync-actions/generate-rand-actions 30)) - actions (mapv - #(assoc-in % [:args :dir] @root-dir) - actions) - partitioned-actions (partition-all 3 actions)] - (clear-dir-pages subdir) - (wait&files-are-in-sync? @root-dir @root-graph-id subdir - (format "clear dir [%s], and ensure it's in sync" subdir)) - (doseq [actions partitioned-actions] - (doseq [action actions] - (run-action action) - (Thread/sleep 1000)) - (is (wait&files-are-in-sync? @root-dir @root-graph-id subdir) - (str "Test " (mapv (juxt :action #(get-in % [:args :file])) actions)))))) - -(defn setup-vars - [] - (let [{:strs [dir]} (read-config) - graph-names-to-ids (api-post-get-graphs) - graph-id (get graph-names-to-ids (fs/file-name dir))] - (assert dir "No graph id for given dir") - (reset! root-dir dir) - (reset! root-graph-id graph-id))) - -(defn integration-tests - "Run file-sync integration tests on graph directory - requirements: - - * file-sync-auth.json, and it looks like: - ``` - {\"id_token\":\"\", - \"dir\": \"/Users/me/Documents/untitled folder 31\"} - ``` - - * you also need to open logseq-app(or yarn electron-watch), - and open and start file-sync" - [& _args] - (setup-vars) - (t/run-tests 'logseq.tasks.file-sync)) diff --git a/scripts/src/logseq/tasks/file_sync_actions.clj b/scripts/src/logseq/tasks/file_sync_actions.clj deleted file mode 100644 index 9889e5cc5b..0000000000 --- a/scripts/src/logseq/tasks/file_sync_actions.clj +++ /dev/null @@ -1,164 +0,0 @@ -(ns logseq.tasks.file-sync-actions - (:require [clojure.test.check.generators :as gen])) - - -(defmulti gen-action* (fn [& args] (first args))) - -(defmethod gen-action* :create-file - [_ page-index & _args] - (gen/let [blocks (gen/vector gen/string-alphanumeric 1 10)] - {:action :create-file - :args {:file (format "pages/test.page-%d.md" page-index) - :blocks blocks}})) - -(defmethod gen-action* :move-file - [_ origin-page-index & [moved?]] - (let [page-name (if moved? - (format "pages/test.page-move-%d.md" origin-page-index) - (format "pages/test.page-%d.md" origin-page-index))] - (gen/return - {:action :move-file - :args {:file page-name - :new-file (format "pages/test.page-move-%d.md" origin-page-index)}}))) - -(defmethod gen-action* :update-file - [_ page-index & [moved?]] - (gen/let [append-blocks (gen/vector gen/string-alphanumeric 1 10)] - (let [page-name (if moved? - (format "pages/test.page-move-%d.md" page-index) - (format "pages/test.page-%d.md" page-index))] - {:action :update-file - :args {:file page-name - :blocks append-blocks}}))) - -(defmethod gen-action* :delete-file - [_ page-index & [moved?]] - (let [page-name (if moved? - (format "pages/test.page-move-%d.md" page-index) - (format "pages/test.page-%d.md" page-index))] - (gen/return - {:action :delete-file - :args {:file page-name}}))) - - -(defmacro gen-actions-plan - "state monad - state: {:page-index [{:index 1 :moved? false}, ...] - :generated-action [...]} - - (gen-actions-plan - [id+moved? get-rand-available-index-op - _ (when-op id+moved? (apply action-op action id+moved?))] - nil)" - [binds val-expr] - (let [binds (partition 2 binds) - psym (gensym "state_") - forms (reduce (fn [acc [id expr]] - (concat acc `[[~id ~psym] (~expr ~psym)])) - [] - binds)] - `(fn [~psym] - (let [~@forms] - [~val-expr ~psym])))) - -(defn- all-indexes - [state] - (let [r (map :index (:page-index state))] - (if (empty? r) '(0) r))) - -(defn- add-index - [state index moved?] - (update state :page-index conj {:index index :moved? moved?})) - -(defn- assign-page-index-op - [state] - (let [max-index (apply max (all-indexes state)) - next-index (inc max-index)] - [next-index (add-index state next-index false)])) - -(defn- get-rand-available-index-op - [state] - (let [indexes (:page-index state)] - (if (empty? indexes) - [nil state] - (let [rand-index (rand-nth (vec indexes))] - [((juxt :index :moved?) rand-index) state])))) - -(defn- action-op - [action id & args] - (fn [state] - (let [generated-action (gen/generate (apply gen-action* action id args)) - [moved?] args - state* (update state :generated-action conj generated-action) - state* (case action - :move-file - (update state* :page-index - #(-> % - (disj {:index id :moved? (boolean moved?)}) - (conj {:index id :moved? true}))) - :delete-file - (update state* :page-index - #(disj % {:index id :moved? (boolean moved?)})) - state*)] - [nil state*]))) - -(defmacro when-op - [x f] - `(fn [state#] - (if ~x - (~f state#) - [nil state#]))) - -(defn- print-op - [x] - (fn [state] - (println x) - [nil state])) - - -(defn rand-action-op - [] - (let [action (gen/generate - (gen/frequency [[5 (gen/return :update-file)] - [2 (gen/return :create-file)] - [2 (gen/return :move-file)] - [1 (gen/return :delete-file)]]))] - (case action - :create-file - (gen-actions-plan - [id assign-page-index-op - _ (action-op action id)] - nil) - :update-file - (gen-actions-plan - [id+moved? get-rand-available-index-op - _ (when-op id+moved? (apply action-op action id+moved?))] - nil) - :move-file - (gen-actions-plan - [id+moved? get-rand-available-index-op - _ (when-op id+moved? (apply action-op action id+moved?))] - nil) - :delete-file - (gen-actions-plan - [id+moved? get-rand-available-index-op - _ (when-op id+moved? (apply action-op action id+moved? ))] - nil)))) - -(def empty-actions-plan {:page-index #{} - :generated-action []}) - - -(defmacro generate-rand-actions - [max-n & {:keys [pre-create-files-n] :or {pre-create-files-n 2}}] - (let [pre-create-files-binds - (for [id (map (fn [_] (gensym)) (range pre-create-files-n))] - `[~id assign-page-index-op - ~'_ (action-op :create-file ~id)]) - binds (apply concat - (concat pre-create-files-binds (repeat max-n `[~'_ (rand-action-op)])))] - `(second - ((gen-actions-plan - ~binds - nil) - empty-actions-plan))))