diff --git a/bb.edn b/bb.edn index da66cca6a0..c93b8cfbd9 100644 --- a/bb.edn +++ b/bb.edn @@ -193,6 +193,10 @@ {:doc "Run shell-first CLI end-to-end tests" :task (apply shell {:shutdown nil} "bb" "-f" "cli-e2e/bb.edn" "test" *command-line-args*)} + dev:cli-e2e-sync + {:doc "Run shell-first CLI sync end-to-end tests" + :task (apply shell {:shutdown nil} "bb" "-f" "cli-e2e/bb.edn" "test-sync" *command-line-args*)} + lint:dev logseq.tasks.dev.lint/dev diff --git a/cli-e2e/AGENTS.md b/cli-e2e/AGENTS.md index 0f77d810b8..6ba9df194c 100644 --- a/cli-e2e/AGENTS.md +++ b/cli-e2e/AGENTS.md @@ -5,10 +5,24 @@ Shell-first end-to-end tests for logseq CLI. ## Test cli-e2e itself - Run internal cli-e2e harness unit tests: `bb unit-test` -## Test cli-e2e cases -- List declared cli-e2e case ids: `bb list-cases` -- Run cli-e2e cases with build preflight unless --skip-build is provided: `bb test` - - `bb test --help` for more help info +## Run non-sync suite +- List declared non-sync case ids: `bb list-cases` +- Run non-sync cases with build preflight unless `--skip-build` is provided: `bb test` + - `bb test --help` for options + +## Run sync suite +- List declared sync case ids: `bb list-sync-cases` +- Run sync cases with build preflight unless `--skip-build` is provided: `bb test-sync` + - `bb test-sync --help` for options + - Run only sync MVP case: `bb test-sync --skip-build --case sync-upload-download-mvp` + +### Sync suite prerequisites +- CLI auth file must exist at `~/logseq/auth.json` (generated by `logseq login`). +- Sync suite starts/stops a local db-sync server inside the test case. +- Required build artifact for local db-sync server: + - `deps/db-sync/worker/dist/node-adapter.js` +- If artifact is missing, build it before running sync suite: + - `yarn --cwd deps/db-sync build:node-adapter` ## Clean up - Execute cleanup: terminate stale db-worker-node processes and remove cli-e2e temp graph directories: `bb cleanup` diff --git a/cli-e2e/bb.edn b/cli-e2e/bb.edn index 8edde28089..2c76216866 100644 --- a/cli-e2e/bb.edn +++ b/cli-e2e/bb.edn @@ -24,13 +24,21 @@ :task (main/build! cli-opts)} list-cases - {:doc "List declared cli-e2e case ids" + {:doc "List declared non-sync cli-e2e case ids" :task (main/list-cases! cli-opts)} + list-sync-cases + {:doc "List declared sync cli-e2e case ids" + :task (main/list-sync-cases! cli-opts)} + test - {:doc "Run cli-e2e cases with build preflight unless --skip-build is provided" + {:doc "Run non-sync cli-e2e cases with build preflight unless --skip-build is provided" :task (main/test! cli-opts)} + test-sync + {:doc "Run sync cli-e2e cases with build preflight unless --skip-build is provided" + :task (main/test-sync! cli-opts)} + cleanup {:doc "Terminate cli-e2e db-worker-node processes and remove temp graph dirs" :task (main/cleanup! cli-opts)}}} diff --git a/cli-e2e/scripts/compare_graph_queries.py b/cli-e2e/scripts/compare_graph_queries.py new file mode 100644 index 0000000000..183c7001c2 --- /dev/null +++ b/cli-e2e/scripts/compare_graph_queries.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Compare normalized query payloads between two cli graph contexts.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from pathlib import Path +from typing import Any, Dict + +IGNORED_KEYS = { + "db/id", + "db/created-at", + "db/updated-at", + "block/uuid", + "block/updated-at", + "block/created-at", +} + + +def fail(message: str, **context: object) -> None: + payload = {"status": "error", "message": message} + if context: + payload["context"] = context + print(json.dumps(payload), file=sys.stderr) + raise SystemExit(1) + + +def normalize(value: Any) -> Any: + if isinstance(value, dict): + normalized = {} + for key in sorted(value.keys(), key=str): + key_str = str(key) + if key_str in IGNORED_KEYS: + continue + normalized[key_str] = normalize(value[key]) + return normalized + if isinstance(value, list): + normalized_list = [normalize(item) for item in value] + try: + return sorted(normalized_list, key=lambda item: json.dumps(item, sort_keys=True)) + except TypeError: + return normalized_list + return value + + +def run_query(cli_path: Path, config_path: Path, data_dir: Path, graph: str, query: str) -> Dict[str, Any]: + command = [ + "node", + str(cli_path), + "--data-dir", + str(data_dir), + "--config", + str(config_path), + "--output", + "json", + "query", + "--graph", + graph, + "--query", + query, + ] + + result = subprocess.run(command, capture_output=True, text=True) + if result.returncode != 0: + fail( + "query command failed", + command=command, + exit=result.returncode, + stdout=result.stdout, + stderr=result.stderr, + ) + + try: + payload = json.loads(result.stdout) + except json.JSONDecodeError as error: + fail( + "query command did not return valid JSON", + command=command, + stdout=result.stdout, + stderr=result.stderr, + detail=str(error), + ) + + if payload.get("status") != "ok": + fail("query command returned non-ok status", command=command, payload=payload) + + data = payload.get("data") or {} + return { + "payload": payload, + "result": data.get("result"), + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Compare normalized query payloads between two cli contexts") + parser.add_argument("--cli", required=True, help="Path to static/logseq-cli.js") + parser.add_argument("--graph", required=True) + parser.add_argument("--query", required=True) + parser.add_argument("--config-a", required=True) + parser.add_argument("--data-dir-a", required=True) + parser.add_argument("--config-b", required=True) + parser.add_argument("--data-dir-b", required=True) + parser.add_argument("--require-result", action="store_true") + args = parser.parse_args() + + cli_path = Path(args.cli).expanduser().resolve() + if not cli_path.exists(): + fail("cli entry file does not exist", cli=str(cli_path)) + + left = run_query( + cli_path, + Path(args.config_a).expanduser().resolve(), + Path(args.data_dir_a).expanduser().resolve(), + args.graph, + args.query, + ) + right = run_query( + cli_path, + Path(args.config_b).expanduser().resolve(), + Path(args.data_dir_b).expanduser().resolve(), + args.graph, + args.query, + ) + + left_result = left["result"] + right_result = right["result"] + + if args.require_result and (left_result is None or right_result is None): + fail("query result is empty", left_result=left_result, right_result=right_result) + + left_normalized = normalize(left_result) + right_normalized = normalize(right_result) + + if left_normalized != right_normalized: + fail( + "normalized query results differ", + left_result=left_normalized, + right_result=right_normalized, + left_payload=left["payload"], + right_payload=right["payload"], + ) + + print( + json.dumps( + { + "status": "ok", + "result": left_normalized, + } + ) + ) + + +if __name__ == "__main__": + main() diff --git a/cli-e2e/scripts/db_sync_server.py b/cli-e2e/scripts/db_sync_server.py new file mode 100644 index 0000000000..ac573062dc --- /dev/null +++ b/cli-e2e/scripts/db_sync_server.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +"""Manage local db-sync server process for cli-e2e sync suite.""" + +from __future__ import annotations + +import argparse +import base64 +import json +import os +import signal +import subprocess +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any, Dict, Optional + +DEFAULT_COGNITO_ISSUER = "https://cognito-idp.us-east-2.amazonaws.com/us-east-2_kAqZcxIeM" +DEFAULT_COGNITO_CLIENT_ID = "1qi1uijg8b6ra70nejvbptis0q" + + +def fail(message: str, **context: object) -> None: + payload = {"status": "error", "message": message} + if context: + payload["context"] = context + print(json.dumps(payload), file=sys.stderr) + raise SystemExit(1) + + +def load_pid(pid_file: Path) -> Optional[int]: + if not pid_file.exists(): + return None + raw = pid_file.read_text(encoding="utf-8").strip() + if not raw: + return None + try: + return int(raw) + except ValueError: + return None + + +def process_running(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + else: + return True + + +def parse_json_file(path: Path) -> Dict[str, Any]: + if not path.exists(): + fail("sync auth file is missing", auth_path=str(path), hint="Run `logseq login` first.") + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as error: + fail("sync auth file is invalid JSON", auth_path=str(path), detail=str(error)) + if not isinstance(payload, dict): + fail("sync auth file must be a JSON object", auth_path=str(path)) + return payload + + +def decode_jwt_claims(token: str) -> Optional[Dict[str, Any]]: + if not isinstance(token, str): + return None + parts = token.split(".") + if len(parts) != 3: + return None + payload = parts[1] + padded = payload + "=" * ((4 - (len(payload) % 4)) % 4) + try: + decoded = base64.urlsafe_b64decode(padded.encode("utf-8")).decode("utf-8") + claims = json.loads(decoded) + except (ValueError, UnicodeDecodeError, json.JSONDecodeError): + return None + return claims if isinstance(claims, dict) else None + + +def auth_cognito_from_auth_file(auth_path: Path) -> Dict[str, str]: + payload = parse_json_file(auth_path) + token = payload.get("id-token") or payload.get("access-token") + claims = decode_jwt_claims(token) if isinstance(token, str) else None + + issuer = claims.get("iss") if isinstance(claims, dict) else None + client_id = None + if isinstance(claims, dict): + client_id = claims.get("aud") or claims.get("client_id") + + return { + "issuer": issuer if isinstance(issuer, str) and issuer else "", + "client_id": client_id if isinstance(client_id, str) and client_id else "", + } + + +def wait_health(base_url: str, timeout_s: float, interval_s: float) -> bool: + deadline = time.time() + timeout_s + url = base_url.rstrip("/") + "/health" + while time.time() < deadline: + try: + with urllib.request.urlopen(url, timeout=2) as response: + if response.status == 200: + return True + except (urllib.error.URLError, TimeoutError): + pass + time.sleep(interval_s) + return False + + +def start_server(args: argparse.Namespace) -> None: + repo_root = Path(args.repo_root).expanduser().resolve() + entry = repo_root / "deps" / "db-sync" / "worker" / "dist" / "node-adapter.js" + if not entry.exists(): + fail("db-sync node adapter build artifact is missing", entry=str(entry), hint="Run: yarn --cwd deps/db-sync build:node-adapter") + + pid_file = Path(args.pid_file).expanduser().resolve() + log_file = Path(args.log_file).expanduser().resolve() + data_dir = Path(args.data_dir).expanduser().resolve() + + pid_file.parent.mkdir(parents=True, exist_ok=True) + log_file.parent.mkdir(parents=True, exist_ok=True) + data_dir.mkdir(parents=True, exist_ok=True) + + existing_pid = load_pid(pid_file) + if existing_pid and process_running(existing_pid): + fail("db-sync server already running", pid=existing_pid, pid_file=str(pid_file)) + + auth_path = Path(args.auth_path).expanduser().resolve() if args.auth_path else None + auth_derived = auth_cognito_from_auth_file(auth_path) if auth_path else {"issuer": "", "client_id": ""} + + issuer = args.cognito_issuer or auth_derived.get("issuer") or DEFAULT_COGNITO_ISSUER + client_id = args.cognito_client_id or auth_derived.get("client_id") or DEFAULT_COGNITO_CLIENT_ID + jwks_url = args.cognito_jwks_url or f"{issuer}/.well-known/jwks.json" + + env = os.environ.copy() + env.update( + { + "DB_SYNC_PORT": str(args.port), + "DB_SYNC_DATA_DIR": str(data_dir), + "COGNITO_ISSUER": issuer, + "COGNITO_CLIENT_ID": client_id, + "COGNITO_JWKS_URL": jwks_url, + } + ) + + with log_file.open("a", encoding="utf-8") as stream: + stream.write(f"\n=== cli-e2e db-sync start {time.strftime('%Y-%m-%d %H:%M:%S')} ===\n") + stream.flush() + process = subprocess.Popen( + ["node", str(entry)], + stdout=stream, + stderr=stream, + cwd=str(repo_root), + env=env, + start_new_session=True, + ) + + base_url = f"http://{args.host}:{args.port}" + if not wait_health(base_url, args.startup_timeout_s, args.poll_interval_s): + try: + os.kill(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + fail( + "db-sync server failed health check before timeout", + base_url=base_url, + pid=process.pid, + log_file=str(log_file), + ) + + pid_file.write_text(f"{process.pid}\n", encoding="utf-8") + print( + json.dumps( + { + "status": "ok", + "pid": process.pid, + "pid_file": str(pid_file), + "log_file": str(log_file), + "base_url": base_url, + "data_dir": str(data_dir), + "cognito_issuer": issuer, + "cognito_client_id": client_id, + "auth_path": str(auth_path) if auth_path else None, + } + ) + ) + + +def stop_server(args: argparse.Namespace) -> None: + pid_file = Path(args.pid_file).expanduser().resolve() + pid = load_pid(pid_file) + + if pid is None: + print(json.dumps({"status": "ok", "stopped": False, "reason": "pid-file-missing-or-empty", "pid_file": str(pid_file)})) + return + + if not process_running(pid): + try: + pid_file.unlink(missing_ok=True) + except OSError: + pass + print(json.dumps({"status": "ok", "stopped": False, "reason": "process-not-running", "pid": pid, "pid_file": str(pid_file)})) + return + + os.kill(pid, signal.SIGTERM) + deadline = time.time() + args.shutdown_timeout_s + while time.time() < deadline: + if not process_running(pid): + pid_file.unlink(missing_ok=True) + print(json.dumps({"status": "ok", "stopped": True, "signal": "SIGTERM", "pid": pid, "pid_file": str(pid_file)})) + return + time.sleep(args.poll_interval_s) + + os.kill(pid, signal.SIGKILL) + pid_file.unlink(missing_ok=True) + print(json.dumps({"status": "ok", "stopped": True, "signal": "SIGKILL", "pid": pid, "pid_file": str(pid_file)})) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Manage db-sync local server for cli-e2e sync tests") + subparsers = parser.add_subparsers(dest="command", required=True) + + start = subparsers.add_parser("start", help="Start server and wait for /health") + start.add_argument("--repo-root", required=True) + start.add_argument("--pid-file", required=True) + start.add_argument("--log-file", required=True) + start.add_argument("--data-dir", required=True) + start.add_argument("--host", default="127.0.0.1") + start.add_argument("--port", type=int, default=8080) + start.add_argument("--startup-timeout-s", type=float, default=25.0) + start.add_argument("--poll-interval-s", type=float, default=0.5) + start.add_argument("--auth-path", default="~/logseq/auth.json") + start.add_argument("--cognito-issuer") + start.add_argument("--cognito-client-id") + start.add_argument("--cognito-jwks-url") + + stop = subparsers.add_parser("stop", help="Stop server if running") + stop.add_argument("--pid-file", required=True) + stop.add_argument("--shutdown-timeout-s", type=float, default=10.0) + stop.add_argument("--poll-interval-s", type=float, default=0.25) + + return parser + + +def main() -> None: + args = build_parser().parse_args() + if args.command == "start": + start_server(args) + elif args.command == "stop": + stop_server(args) + else: + fail("unknown command", command=args.command) + + +if __name__ == "__main__": + main() diff --git a/cli-e2e/scripts/prepare_sync_config.py b/cli-e2e/scripts/prepare_sync_config.py new file mode 100644 index 0000000000..9b750f6ae6 --- /dev/null +++ b/cli-e2e/scripts/prepare_sync_config.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Prepare per-case CLI config for sync e2e tests.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +def fail(message: str, **context: object) -> None: + payload = {"status": "error", "message": message} + if context: + payload["context"] = context + print(json.dumps(payload), file=sys.stderr) + raise SystemExit(1) + + +def read_auth(auth_path: Path) -> dict: + if not auth_path.exists(): + fail("sync auth file is missing", auth_path=str(auth_path), hint="Run `logseq login` first.") + try: + payload = json.loads(auth_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as error: + fail("sync auth file is invalid JSON", auth_path=str(auth_path), detail=str(error)) + + has_token = any(payload.get(key) for key in ("refresh-token", "id-token", "access-token")) + if not has_token: + fail( + "sync auth file does not contain usable tokens", + auth_path=str(auth_path), + required_any_of=["refresh-token", "id-token", "access-token"], + ) + return payload + + +def write_config(output_path: Path, http_base: str, ws_url: str) -> None: + output_path.parent.mkdir(parents=True, exist_ok=True) + payload = "\n".join( + [ + "{", + " :output-format :json", + f' :http-base "{http_base}"', + f' :ws-url "{ws_url}"', + "}", + "", + ] + ) + output_path.write_text(payload, encoding="utf-8") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Prepare cli.edn for sync e2e") + parser.add_argument("--output", required=True) + parser.add_argument("--auth-path", default="~/logseq/auth.json") + parser.add_argument("--http-base", required=True) + parser.add_argument("--ws-url", required=True) + args = parser.parse_args() + + auth_path = Path(args.auth_path).expanduser().resolve() + _auth = read_auth(auth_path) + + output_path = Path(args.output).expanduser().resolve() + write_config(output_path, args.http_base, args.ws_url) + + print( + json.dumps( + { + "status": "ok", + "auth_path": str(auth_path), + "config_path": str(output_path), + "http_base": args.http_base, + "ws_url": args.ws_url, + } + ) + ) + + +if __name__ == "__main__": + main() diff --git a/cli-e2e/scripts/wait_sync_status.py b/cli-e2e/scripts/wait_sync_status.py new file mode 100644 index 0000000000..bb44e45f2b --- /dev/null +++ b/cli-e2e/scripts/wait_sync_status.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +"""Poll `logseq sync status` until queues settle and tx converges.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import time +from pathlib import Path +from typing import Any, Dict + + +def fail(message: str, **context: object) -> None: + payload = {"status": "error", "message": message} + if context: + payload["context"] = context + print(json.dumps(payload), file=sys.stderr) + raise SystemExit(1) + + +def parse_int(value: Any) -> int: + if value is None: + return 0 + if isinstance(value, bool): + return int(value) + if isinstance(value, (int, float)): + return int(value) + try: + return int(str(value)) + except (TypeError, ValueError): + return 0 + + +def run_status(args: argparse.Namespace) -> Dict[str, Any]: + command = [ + "node", + str(Path(args.cli).expanduser().resolve()), + "--data-dir", + str(Path(args.data_dir).expanduser().resolve()), + "--config", + str(Path(args.config).expanduser().resolve()), + "--output", + "json", + "sync", + "status", + "--graph", + args.graph, + ] + + result = subprocess.run(command, capture_output=True, text=True) + if result.returncode != 0: + fail( + "sync status command failed", + command=command, + exit=result.returncode, + stdout=result.stdout, + stderr=result.stderr, + ) + + try: + payload = json.loads(result.stdout) + except json.JSONDecodeError as error: + fail( + "sync status command did not return valid JSON", + command=command, + stdout=result.stdout, + stderr=result.stderr, + detail=str(error), + ) + + if payload.get("status") != "ok": + fail("sync status returned non-ok status", payload=payload) + + return payload + + +def pending_counts(status_payload: Dict[str, Any]) -> Dict[str, int]: + data = status_payload.get("data") or {} + return { + "pending-local": parse_int(data.get("pending-local")), + "pending-asset": parse_int(data.get("pending-asset")), + "pending-server": parse_int(data.get("pending-server")), + } + + +def all_settled(counts: Dict[str, int]) -> bool: + required_keys = ("pending-local", "pending-asset", "pending-server") + return all(counts.get(key, 0) == 0 for key in required_keys) + + +def parse_required_int(value: Any) -> int | None: + if value is None: + return None + if isinstance(value, str) and value.strip() == "": + return None + if isinstance(value, bool): + return int(value) + if isinstance(value, (int, float)): + return int(value) + try: + return int(str(value)) + except (TypeError, ValueError): + return None + + +def tx_sync_status(status_payload: Dict[str, Any]) -> Dict[str, Any]: + data = status_payload.get("data") or {} + local_tx = parse_required_int(data.get("local-tx")) + remote_tx = parse_required_int(data.get("remote-tx")) + synced = ( + local_tx is not None + and remote_tx is not None + and local_tx == remote_tx + ) + return { + "local-tx": local_tx, + "remote-tx": remote_tx, + "synced": synced, + } + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Wait for sync status to settle" + ) + parser.add_argument( + "--cli", + required=True, + help="Path to static/logseq-cli.js", + ) + parser.add_argument("--data-dir", required=True) + parser.add_argument("--config", required=True) + parser.add_argument("--graph", required=True) + parser.add_argument("--timeout-s", type=float, default=120.0) + parser.add_argument("--interval-s", type=float, default=1.0) + args = parser.parse_args() + + started = time.time() + deadline = started + args.timeout_s + last_payload: Dict[str, Any] | None = None + + while time.time() < deadline: + payload = run_status(args) + last_payload = payload + data = payload.get("data") or {} + last_error = data.get("last-error") + + if last_error is not None: + fail("sync status reports last-error", payload=payload) + + counts = pending_counts(payload) + tx_status = tx_sync_status(payload) + if all_settled(counts) and tx_status["synced"]: + print( + json.dumps( + { + "status": "ok", + "elapsed_s": round(time.time() - started, 3), + "counts": counts, + "tx": { + "local-tx": tx_status["local-tx"], + "remote-tx": tx_status["remote-tx"], + }, + "payload": payload, + } + ) + ) + return + + time.sleep(max(args.interval_s, 0.0)) + + fail( + "sync status polling timed out before queues settled and tx synced", + timeout_s=args.timeout_s, + last_payload=last_payload, + last_tx=tx_sync_status(last_payload or {}), + ) + + +if __name__ == "__main__": + main() diff --git a/cli-e2e/spec/sync_cases.edn b/cli-e2e/spec/sync_cases.edn new file mode 100644 index 0000000000..4cb77e7109 --- /dev/null +++ b/cli-e2e/spec/sync_cases.edn @@ -0,0 +1,40 @@ +[ + {:id "sync-upload-download-mvp" + :graph "sync-e2e-mvp" + :vars {:sync-port "18080" + :sync-http-base "http://127.0.0.1:18080" + :sync-ws-url "ws://127.0.0.1:18080/sync/%s" + :marker-content "sync-e2e-marker" + :home-dir "{{tmp-dir}}/home" + :auth-path "{{tmp-dir}}/home/logseq/auth.json" + :cli-home "HOME='{{tmp-dir}}/home' {{cli}}"} + :setup ["mkdir -p '{{tmp-dir}}/graphs-b'" + "mkdir -p '{{tmp-dir}}/home/logseq'" + "cp ~/logseq/auth.json '{{tmp-dir}}/home/logseq/auth.json'" + "python3 '{{repo-root}}/cli-e2e/scripts/prepare_sync_config.py' --output '{{config-path}}' --auth-path '{{auth-path}}' --http-base '{{sync-http-base}}' --ws-url '{{sync-ws-url}}'" + "python3 '{{repo-root}}/cli-e2e/scripts/prepare_sync_config.py' --output '{{tmp-dir}}/cli-b.edn' --auth-path '{{auth-path}}' --http-base '{{sync-http-base}}' --ws-url '{{sync-ws-url}}'" + "python3 '{{repo-root}}/cli-e2e/scripts/db_sync_server.py' start --repo-root '{{repo-root}}' --pid-file '{{tmp-dir}}/db-sync-server.pid' --log-file '{{tmp-dir}}/db-sync-server.log' --data-dir '{{tmp-dir}}/db-sync-server-data' --port {{sync-port}} --auth-path '{{auth-path}}'" + "{{cli-home}} --data-dir {{data-dir-arg}} --config {{config-path-arg}} --output json graph create --graph {{graph-arg}} >/dev/null" + "{{cli-home}} --data-dir {{data-dir-arg}} --config {{config-path-arg}} --output json sync ensure-keys --graph {{graph-arg}} --e2ee-password '11111' --upload-keys" + "{{cli-home}} --data-dir {{data-dir-arg}} --config {{config-path-arg}} --output json upsert page --graph {{graph-arg}} --page SyncMvpHome >/dev/null" + "{{cli-home}} --data-dir {{data-dir-arg}} --config {{config-path-arg}} --output json upsert block --graph {{graph-arg}} --target-page SyncMvpHome --content '{{marker-content}}' --update-tags '[:logseq.class/Quote-block]' --update-properties '{:logseq.property/publishing-public? true}' >/dev/null"] + :cmds ["{{cli-home}} --data-dir {{data-dir-arg}} --config {{config-path-arg}} --output json sync upload --graph {{graph-arg}}" + "{{cli-home}} --data-dir {{data-dir-arg}} --config {{config-path-arg}} --output json sync start --graph {{graph-arg}} --e2ee-password '11111'" + "python3 '{{repo-root}}/cli-e2e/scripts/wait_sync_status.py' --cli '{{repo-root}}/static/logseq-cli.js' --data-dir '{{data-dir}}' --config '{{config-path}}' --graph '{{graph}}' --timeout-s 120 --interval-s 1" + "{{cli-home}} --data-dir '{{tmp-dir}}/graphs-b' --config '{{tmp-dir}}/cli-b.edn' --output json sync download --graph {{graph-arg}} --e2ee-password '11111'" + "{{cli-home}} --data-dir '{{tmp-dir}}/graphs-b' --config '{{tmp-dir}}/cli-b.edn' --output json sync start --graph {{graph-arg}} --e2ee-password '11111'" + "python3 '{{repo-root}}/cli-e2e/scripts/wait_sync_status.py' --cli '{{repo-root}}/static/logseq-cli.js' --data-dir '{{tmp-dir}}/graphs-b' --config '{{tmp-dir}}/cli-b.edn' --graph '{{graph}}' --timeout-s 120 --interval-s 1" + "python3 '{{repo-root}}/cli-e2e/scripts/compare_graph_queries.py' --cli '{{repo-root}}/static/logseq-cli.js' --graph '{{graph}}' --config-a '{{config-path}}' --data-dir-a '{{data-dir}}' --config-b '{{tmp-dir}}/cli-b.edn' --data-dir-b '{{tmp-dir}}/graphs-b' --query '[:find ?title ?public ?tag-ident :where [?b :block/title ?title] [(= ?title \"sync-e2e-marker\")] [?b :logseq.property/publishing-public? ?public] [?b :block/tags ?tag] [?tag :db/ident ?tag-ident] [(= ?tag-ident :logseq.class/Quote-block)]]' --require-result"] + :expect {:exit 0 + :stdout-json-paths {[:status] "ok"}} + :covers {:commands ["sync upload" + "sync download" + "sync status"] + :options {:global ["--config" + "--graph" + "--data-dir" + "--output"]}} + :cleanup ["{{cli}} --data-dir {{data-dir-arg}} --config {{config-path-arg}} --output json server stop --graph {{graph-arg}}" + "{{cli}} --data-dir '{{tmp-dir}}/graphs-b' --config '{{tmp-dir}}/cli-b.edn' --output json server stop --graph {{graph-arg}}" + "python3 '{{repo-root}}/cli-e2e/scripts/db_sync_server.py' stop --pid-file '{{tmp-dir}}/db-sync-server.pid'"] + :tags [:sync :mvp]}] diff --git a/cli-e2e/spec/sync_inventory.edn b/cli-e2e/spec/sync_inventory.edn new file mode 100644 index 0000000000..67d35df73f --- /dev/null +++ b/cli-e2e/spec/sync_inventory.edn @@ -0,0 +1,13 @@ +{:excluded-command-prefixes ["login" "logout"] + :scopes + {:global + {:options ["--config" + "--graph" + "--data-dir" + "--output"]} + + :sync + {:commands ["sync upload" + "sync download" + "sync status"] + :options []}}} diff --git a/cli-e2e/src/logseq/cli/e2e/main.clj b/cli-e2e/src/logseq/cli/e2e/main.clj index 0de2be031e..e6875be4b8 100644 --- a/cli-e2e/src/logseq/cli/e2e/main.clj +++ b/cli-e2e/src/logseq/cli/e2e/main.clj @@ -23,6 +23,12 @@ :else (vec cases))) +(def default-suite :non-sync) + +(defn- suite-from-opts + [opts] + (or (:suite opts) default-suite)) + (defn- elapsed-ms [started-at] (long (/ (- (System/nanoTime) started-at) 1000000))) @@ -68,6 +74,7 @@ :as opts}] (let [run-command (or run-command shell/run!) run-case (or (:run-case opts) runner/run-case!) + suite (suite-from-opts opts) targeted-run? (or (:case opts) (seq (:include opts))) on-preflight-start (:on-preflight-start opts) on-preflight-complete (:on-preflight-complete opts) @@ -76,8 +83,8 @@ (on-preflight-start {:skip-build skip-build})) (let [preflight-result (preflight/run! {:skip-build skip-build :run-command run-command}) - inventory (or inventory (manifests/load-inventory)) - cases (or cases (manifests/load-cases))] + inventory (or inventory (manifests/load-inventory suite)) + cases (or cases (manifests/load-cases suite))] (when on-preflight-complete (on-preflight-complete preflight-result)) (let [selected-cases (select-cases cases opts) @@ -100,10 +107,14 @@ (preflight/run! opts)) (defn list-cases! - [_opts] - (doseq [{:keys [id]} (manifests/load-cases)] + [opts] + (doseq [{:keys [id]} (manifests/load-cases (suite-from-opts opts))] (println id))) +(defn list-sync-cases! + [opts] + (list-cases! (assoc opts :suite :sync))) + (defn- print-cleanup-help! [] (println "Usage: bb -f cli-e2e/bb.edn cleanup [options]") @@ -165,8 +176,8 @@ (flush))) (defn- print-test-help! - [] - (println "Usage: bb -f cli-e2e/bb.edn test [options]") + [command-name] + (println (str "Usage: bb -f cli-e2e/bb.edn " command-name " [options]")) (println) (println "Options:") (println " -h, --help Show this help and exit") @@ -176,79 +187,93 @@ (println " --verbose Enable verbose output") (println) (println "Examples:") - (println " bb -f cli-e2e/bb.edn test --skip-build") - (println " bb -f cli-e2e/bb.edn test --skip-build -i smoke") - (println " bb -f cli-e2e/bb.edn test --skip-build --case global-help") + (println (str " bb -f cli-e2e/bb.edn " command-name " --skip-build")) + (println (str " bb -f cli-e2e/bb.edn " command-name " --skip-build -i smoke")) + (println (str " bb -f cli-e2e/bb.edn " command-name " --skip-build --case global-help")) (flush)) +(defn- test-suite! + [opts {:keys [suite command-name] + :or {suite default-suite + command-name "test"}}] + (let [suite (or suite default-suite) + opts (assoc opts :suite suite)] + (if (:help opts) + (do + (print-test-help! command-name) + {:status :help}) + (let [started-at (System/nanoTime) + passed (atom 0) + failed (atom 0) + total-count (atom 0) + detailed-case-log? (some? (:case opts)) + base-run-command (or (:run-command opts) shell/run!) + run-command (if detailed-case-log? + (fn [{:keys [cmd phase step-index step-total] :as command-opts}] + (let [prefix (case phase + :setup (format " [setup %d/%d]" step-index step-total) + :main (format " [main %d/%d]" (or step-index 1) (or step-total 1)) + :cleanup (format " [cleanup %d/%d]" step-index step-total) + " [command]")] + (println (str prefix " $ " cmd)) + (flush) + (base-run-command (assoc command-opts :stream-output? true)))) + base-run-command)] + (println "==> Running cli-e2e cases") + (when detailed-case-log? + (println (format "==> Detailed case logging enabled (--case %s)" (:case opts)))) + (flush) + (try + (run! (assoc opts + :run-command run-command + :detailed-log? detailed-case-log? + :on-preflight-start (fn [_] + (println "==> Build preflight: running...") + (flush)) + :on-preflight-complete (fn [{:keys [status]}] + (println (case status + :skipped "==> Build preflight: skipped (--skip-build)" + "==> Build preflight: completed")) + (flush)) + :on-cases-ready (fn [{:keys [total]}] + (reset! total-count total) + (println (format "==> Prepared %d case(s), starting execution" total)) + (flush)) + :on-case-start (fn [{:keys [index total case]}] + (println (format "[%d/%d] ▶ %s" index total (:id case))) + (flush)) + :on-case-success (fn [{:keys [index total result elapsed-ms]}] + (swap! passed inc) + (println (format "[%d/%d] ✓ %s (%dms)" + index + total + (:id result) + elapsed-ms)) + (flush)) + :on-case-failure (fn [{:keys [index total case error elapsed-ms]}] + (swap! failed inc) + (println (format "[%d/%d] ✗ %s (%dms)" + index + total + (:id case) + elapsed-ms)) + (print-failure-details! error)))) + (println (format "Summary: %d passed, %d failed" @passed @failed)) + (println (str "Selected cases: " @total-count)) + (println (str "Duration: " (format-duration started-at))) + (catch Exception error + (let [failed-count (max 1 @failed)] + (println (format "Summary: %d passed, %d failed" @passed failed-count)) + (println (str "Selected cases: " (max @total-count failed-count))) + (println (str "Duration: " (format-duration started-at)))) + (throw error))))))) + (defn test! [opts] - (if (:help opts) - (do - (print-test-help!) - {:status :help}) - (let [started-at (System/nanoTime) - passed (atom 0) - failed (atom 0) - total-count (atom 0) - detailed-case-log? (some? (:case opts)) - base-run-command (or (:run-command opts) shell/run!) - run-command (if detailed-case-log? - (fn [{:keys [cmd phase step-index step-total] :as command-opts}] - (let [prefix (case phase - :setup (format " [setup %d/%d]" step-index step-total) - :main (format " [main %d/%d]" (or step-index 1) (or step-total 1)) - :cleanup (format " [cleanup %d/%d]" step-index step-total) - " [command]")] - (println (str prefix " $ " cmd)) - (flush) - (base-run-command (assoc command-opts :stream-output? true)))) - base-run-command)] - (println "==> Running cli-e2e cases") - (when detailed-case-log? - (println (format "==> Detailed case logging enabled (--case %s)" (:case opts)))) - (flush) - (try - (run! (assoc opts - :run-command run-command - :detailed-log? detailed-case-log? - :on-preflight-start (fn [_] - (println "==> Build preflight: running...") - (flush)) - :on-preflight-complete (fn [{:keys [status]}] - (println (case status - :skipped "==> Build preflight: skipped (--skip-build)" - "==> Build preflight: completed")) - (flush)) - :on-cases-ready (fn [{:keys [total]}] - (reset! total-count total) - (println (format "==> Prepared %d case(s), starting execution" total)) - (flush)) - :on-case-start (fn [{:keys [index total case]}] - (println (format "[%d/%d] ▶ %s" index total (:id case))) - (flush)) - :on-case-success (fn [{:keys [index total result elapsed-ms]}] - (swap! passed inc) - (println (format "[%d/%d] ✓ %s (%dms)" - index - total - (:id result) - elapsed-ms)) - (flush)) - :on-case-failure (fn [{:keys [index total case error elapsed-ms]}] - (swap! failed inc) - (println (format "[%d/%d] ✗ %s (%dms)" - index - total - (:id case) - elapsed-ms)) - (print-failure-details! error)))) - (println (format "Summary: %d passed, %d failed" @passed @failed)) - (println (str "Selected cases: " @total-count)) - (println (str "Duration: " (format-duration started-at))) - (catch Exception error - (let [failed-count (max 1 @failed)] - (println (format "Summary: %d passed, %d failed" @passed failed-count)) - (println (str "Selected cases: " (max @total-count failed-count))) - (println (str "Duration: " (format-duration started-at)))) - (throw error)))))) + (test-suite! opts {:suite :non-sync + :command-name "test"})) + +(defn test-sync! + [opts] + (test-suite! opts {:suite :sync + :command-name "test-sync"})) diff --git a/cli-e2e/src/logseq/cli/e2e/manifests.clj b/cli-e2e/src/logseq/cli/e2e/manifests.clj index e784080ea7..941e97d19c 100644 --- a/cli-e2e/src/logseq/cli/e2e/manifests.clj +++ b/cli-e2e/src/logseq/cli/e2e/manifests.clj @@ -2,14 +2,43 @@ (:require [clojure.edn :as edn] [logseq.cli.e2e.paths :as paths])) +(def suite->manifest-files + {:non-sync {:inventory "non_sync_inventory.edn" + :cases "non_sync_cases.edn"} + :sync {:inventory "sync_inventory.edn" + :cases "sync_cases.edn"}}) + +(def default-suite :non-sync) + (defn read-edn-file [path] (edn/read-string (slurp path))) +(defn- normalize-suite + [suite] + (let [suite' (cond + (nil? suite) default-suite + (keyword? suite) suite + (string? suite) (keyword suite) + :else suite)] + (when-not (contains? suite->manifest-files suite') + (throw (ex-info "Unknown cli-e2e suite" + {:suite suite + :known-suites (sort (keys suite->manifest-files))}))) + suite')) + +(defn- manifest-file + [suite kind] + (get-in suite->manifest-files [(normalize-suite suite) kind])) + (defn load-inventory - [] - (read-edn-file (paths/spec-path "non_sync_inventory.edn"))) + ([] + (load-inventory nil)) + ([suite] + (read-edn-file (paths/spec-path (manifest-file suite :inventory))))) (defn load-cases - [] - (read-edn-file (paths/spec-path "non_sync_cases.edn"))) + ([] + (load-cases nil)) + ([suite] + (read-edn-file (paths/spec-path (manifest-file suite :cases))))) diff --git a/cli-e2e/test/logseq/cli/e2e/coverage_test.clj b/cli-e2e/test/logseq/cli/e2e/coverage_test.clj index 7c5615e584..883a27edef 100644 --- a/cli-e2e/test/logseq/cli/e2e/coverage_test.clj +++ b/cli-e2e/test/logseq/cli/e2e/coverage_test.clj @@ -1,6 +1,7 @@ (ns logseq.cli.e2e.coverage-test (:require [clojure.test :refer [deftest is testing]] - [logseq.cli.e2e.coverage :as coverage])) + [logseq.cli.e2e.coverage :as coverage] + [logseq.cli.e2e.manifests :as manifests])) (def sample-inventory {:excluded-command-prefixes ["sync" "login" "logout"] @@ -45,3 +46,13 @@ :options {:graph ["--file"]}}}] report (coverage/coverage-report sample-inventory cases)] (is (coverage/complete? report)))) + +(deftest sync-suite-manifests-cover-mvp-commands + (let [inventory (manifests/load-inventory :sync) + cases (manifests/load-cases :sync) + covered-commands (set (mapcat #(get-in % [:covers :commands]) cases)) + report (coverage/coverage-report inventory cases)] + (is (contains? covered-commands "sync upload")) + (is (contains? covered-commands "sync download")) + (is (contains? covered-commands "sync status")) + (is (coverage/complete? report)))) diff --git a/cli-e2e/test/logseq/cli/e2e/main_test.clj b/cli-e2e/test/logseq/cli/e2e/main_test.clj index a53fe965a9..6e2f729570 100644 --- a/cli-e2e/test/logseq/cli/e2e/main_test.clj +++ b/cli-e2e/test/logseq/cli/e2e/main_test.clj @@ -2,7 +2,8 @@ (:require [clojure.string :as string] [clojure.test :refer [deftest is testing]] [logseq.cli.e2e.cleanup :as cleanup] - [logseq.cli.e2e.main :as main])) + [logseq.cli.e2e.main :as main] + [logseq.cli.e2e.manifests :as manifests])) (def sample-cases [{:id "global-help" @@ -118,6 +119,75 @@ [:ok 2 2 "graph-list"]] @events)))) +(deftest run-loads-non-sync-suite-by-default + (let [suite-calls (atom [])] + (with-redefs [manifests/load-inventory (fn [suite] + (swap! suite-calls conj [:inventory suite]) + complete-inventory) + manifests/load-cases (fn [suite] + (swap! suite-calls conj [:cases suite]) + sample-cases)] + (let [result (main/run! {:skip-build true + :run-command (fn [_] + {:exit 0 + :out "" + :err ""}) + :run-case (fn [case _opts] + {:id (:id case) + :status :ok})})] + (is (= :ok (:status result)))) + (is (= [[:inventory :non-sync] + [:cases :non-sync]] + @suite-calls))))) + +(deftest run-loads-sync-suite-when-explicit + (let [suite-calls (atom []) + sync-inventory {:excluded-command-prefixes ["login" "logout"] + :scopes {:sync {:commands ["sync upload"] + :options []}}} + sync-cases [{:id "sync-upload" + :cmds ["node static/logseq-cli.js sync upload"] + :covers {:commands ["sync upload"]}}]] + (with-redefs [manifests/load-inventory (fn [suite] + (swap! suite-calls conj [:inventory suite]) + sync-inventory) + manifests/load-cases (fn [suite] + (swap! suite-calls conj [:cases suite]) + sync-cases)] + (let [result (main/run! {:suite :sync + :skip-build true + :run-command (fn [_] + {:exit 0 + :out "" + :err ""}) + :run-case (fn [case _opts] + {:id (:id case) + :status :ok})})] + (is (= :ok (:status result)))) + (is (= [[:inventory :sync] + [:cases :sync]] + @suite-calls))))) + +(deftest list-cases-defaults-to-non-sync + (let [selected-suite (atom nil) + output (with-out-str + (with-redefs [manifests/load-cases (fn [suite] + (reset! selected-suite suite) + [{:id "non-sync-case"}])] + (main/list-cases! {})))] + (is (= :non-sync @selected-suite)) + (is (string/includes? output "non-sync-case")))) + +(deftest list-sync-cases-uses-sync-suite + (let [selected-suite (atom nil) + output (with-out-str + (with-redefs [manifests/load-cases (fn [suite] + (reset! selected-suite suite) + [{:id "sync-case"}])] + (main/list-sync-cases! {})))] + (is (= :sync @selected-suite)) + (is (string/includes? output "sync-case")))) + (deftest test-prints-progress-and-summary (let [output (with-out-str (main/test! {:inventory complete-inventory @@ -164,6 +234,28 @@ (is (string/includes? output "--include TAG")) (is (string/includes? output "--case ID")))) +(deftest test-sync-help-prints-usage-and-skips-execution + (let [ran? (atom false) + result (atom nil) + output (with-out-str + (reset! result + (main/test-sync! {:help true + :run-command (fn [_] + (reset! ran? true) + {:exit 0 + :out "" + :err ""}) + :run-case (fn [_ _] + (reset! ran? true) + {:id "unexpected" + :status :ok})})))] + (is (= :help (:status @result))) + (is (false? @ran?)) + (is (string/includes? output "Usage: bb -f cli-e2e/bb.edn test-sync [options]")) + (is (string/includes? output "--skip-build")) + (is (string/includes? output "--include TAG")) + (is (string/includes? output "--case ID")))) + (deftest test-single-case-enables-detailed-command-logging (let [command-opts (atom nil) output (with-out-str diff --git a/docs/agent-guide/077-cli-e2e-sync-tests.md b/docs/agent-guide/077-cli-e2e-sync-tests.md new file mode 100644 index 0000000000..aa4b3b33e7 --- /dev/null +++ b/docs/agent-guide/077-cli-e2e-sync-tests.md @@ -0,0 +1,195 @@ +# CLI E2E Sync Suite Implementation Plan + +Goal: Add a dedicated sync-focused `cli-e2e` suite that is isolated from non-sync coverage and validates MVP upload and download behavior using two independent `db-worker-node` processes driven only through CLI commands. + +Architecture: Keep the existing non-sync suite as the default `cli-e2e` path and introduce a separate sync suite with its own manifest files, runner entrypoint, and preconditions. +Architecture: Model sync behavior with two distinct data directories that run two different `db-worker-node` processes against the same graph name, then assert health via `sync status` and data convergence via CLI queries. +Architecture: Ship MVP coverage for upload and download first, while leaving realtime sync-start convergence tests for a follow-up phase. + +Tech Stack: Babashka, EDN case manifests, `logseq-cli`, `db-worker-node`, JSON parsing via Python 3 in shell helpers, existing CLI sync commands. + +Related: Builds on `/Users/rcmerci/gh-repos/logseq/docs/agent-guide/064-logseq-cli-integration-test-shell-refactor.md`. +Related: Relates to `/Users/rcmerci/gh-repos/logseq/docs/agent-guide/047-logseq-cli-sync-command.md`. +Related: Relates to `/Users/rcmerci/gh-repos/logseq/docs/agent-guide/048-sync-download-start-reliability.md`. +Related: Relates to `/Users/rcmerci/gh-repos/logseq/docs/agent-guide/051-logseq-cli-sync-upload-fix.md`. + +## Problem statement + +`cli-e2e` currently excludes all `sync` commands by design. + +`/Users/rcmerci/gh-repos/logseq/cli-e2e/spec/non_sync_inventory.edn` explicitly excludes the `sync` prefix, and `/Users/rcmerci/gh-repos/logseq/cli-e2e/src/logseq/cli/e2e/manifests.clj` only loads `non_sync_*` manifests. + +This keeps non-sync coverage clean, but there is currently no shell-first `cli-e2e` coverage for sync upload and download behavior. + +Current sync integration checks in `/Users/rcmerci/gh-repos/logseq/src/test/logseq/cli/integration_test.cljs` are mostly mocked transport-level tests and do not validate real shell command orchestration in a two-process setup. + +The requested test architecture requires two independent `db-worker-node` processes in different directories, both operating on the same graph name, with CLI-only operations and status-driven verification. + +## Current implementation snapshot + +| Area | Current file | Current behavior | Gap for this plan | +| --- | --- | --- | --- | +| Suite manifests | `/Users/rcmerci/gh-repos/logseq/cli-e2e/src/logseq/cli/e2e/manifests.clj` | Loads only `non_sync_inventory.edn` and `non_sync_cases.edn`. | No sync suite loading path. | +| Non-sync inventory policy | `/Users/rcmerci/gh-repos/logseq/cli-e2e/spec/non_sync_inventory.edn` | Excludes `sync`, `login`, and `logout`. | Sync tests must live in separate manifests to avoid policy conflict. | +| CLI runner tasks | `/Users/rcmerci/gh-repos/logseq/cli-e2e/bb.edn` | Exposes `test`, `list-cases`, and `build` for one suite. | Need dedicated sync tasks and clearer suite-level ergonomics. | +| Case execution model | `/Users/rcmerci/gh-repos/logseq/cli-e2e/src/logseq/cli/e2e/runner.clj` | Supports shell-first setup and command chains with templating. | No built-in wait helper for polling `sync status` until pending queues are empty. | +| Sync command behavior | `/Users/rcmerci/gh-repos/logseq/src/main/logseq/cli/command/sync.cljs` | `sync upload`, `sync download`, and `sync status` are implemented and return structured JSON. | E2E harness does not yet assert these behaviors with two independent workers. | +| Server process isolation | `/Users/rcmerci/gh-repos/logseq/src/main/logseq/cli/server.cljs` | Data-dir and graph path determine lock ownership and process identity. | Need explicit test cases that prove two data dirs produce two independent worker processes for the same graph name. | + +## Scope and MVP boundaries + +MVP in this plan covers upload and download flows only. + +MVP does not add coverage for long-running `sync start` websocket convergence behavior. + +MVP requires CLI-only graph operations during test execution, including graph creation, mutation, upload, download, status checks, and data verification queries. + +MVP keeps non-sync test behavior unchanged and isolated. + +## Testing Plan + +I will add runner-level unit tests that fail first when sync suite manifests and tasks are missing, and pass only after suite separation is implemented. + +I will add sync suite manifest coverage tests that fail first when required sync command options are not covered by MVP cases. + +I will add shell-first sync E2E cases that fail first and validate the two-data-dir architecture, `sync status` health checks, pending queue convergence checks, and graph data parity assertions. + +I will validate command ergonomics by running non-sync and sync suites independently and ensuring their outputs and selection logic remain deterministic. + +I will follow @test-driven-development for every behavior slice in this plan. + +NOTE: I will write all tests before I add any implementation behavior. + +## Target sync suite architecture + +```text ++----------------------------------+ +----------------------------------+ +| data-dir A | | data-dir B | +| graph: sync-e2e-mvp | | graph: sync-e2e-mvp | +| db-worker-node process A | | db-worker-node process B | ++----------------+-----------------+ +----------------+-----------------+ + | | + | CLI commands only | CLI commands only + v v + logseq sync upload logseq sync download + | | + +-------------------> remote sync backend <----------+ + +Verification path: +1) mutate graph via CLI in A. +2) run sync upload via CLI in A. +3) poll sync status via CLI until pending queues settle and last-error remains nil. +4) run sync download via CLI in B. +5) compare graph data via CLI queries in A and B. +``` + +## Detailed implementation plan + +### Phase 1. Add explicit sync suite separation in `cli-e2e`. + +1. Add a failing unit test in `/Users/rcmerci/gh-repos/logseq/cli-e2e/test/logseq/cli/e2e/main_test.clj` that expects a dedicated sync test entrypoint to load sync manifests instead of non-sync manifests. +2. Add a failing unit test in `/Users/rcmerci/gh-repos/logseq/cli-e2e/test/logseq/cli/e2e/main_test.clj` that expects non-sync `test` to keep current behavior unchanged. +3. Add `sync_inventory.edn` loading support in `/Users/rcmerci/gh-repos/logseq/cli-e2e/src/logseq/cli/e2e/manifests.clj` with a suite selector API. +4. Add `sync_cases.edn` loading support in `/Users/rcmerci/gh-repos/logseq/cli-e2e/src/logseq/cli/e2e/manifests.clj` with the same suite selector API. +5. Add suite-aware run helpers in `/Users/rcmerci/gh-repos/logseq/cli-e2e/src/logseq/cli/e2e/main.clj` so non-sync and sync share execution plumbing but load different manifests. +6. Add new tasks in `/Users/rcmerci/gh-repos/logseq/cli-e2e/bb.edn` for `test-sync` and `list-sync-cases`. +7. Keep existing `test` and `list-cases` mapped to non-sync manifests. +8. Run `bb -f /Users/rcmerci/gh-repos/logseq/cli-e2e/bb.edn unit-test` and confirm failures turn green for the new suite selection behavior. + +### Phase 2. Define sync inventory and MVP case manifests. + +9. Create `/Users/rcmerci/gh-repos/logseq/cli-e2e/spec/sync_inventory.edn` with MVP required commands `sync upload`, `sync download`, and `sync status`. +10. Include only MVP-required sync options in `sync_inventory.edn` to avoid over-scoping phase one. +11. Add a failing coverage test in `/Users/rcmerci/gh-repos/logseq/cli-e2e/test/logseq/cli/e2e/coverage_test.clj` for missing sync command coverage. +12. Create `/Users/rcmerci/gh-repos/logseq/cli-e2e/spec/sync_cases.edn` with initial empty or placeholder MVP case definitions that intentionally fail coverage. +13. Run `bb -f /Users/rcmerci/gh-repos/logseq/cli-e2e/bb.edn test-sync --skip-build` and confirm coverage failure is clear and actionable. + +### Phase 3. Add reusable sync status and graph parity helper scripts. + +14. Add `/Users/rcmerci/gh-repos/logseq/cli-e2e/scripts/wait_sync_status.py` that repeatedly executes CLI `sync status --output json` until pending queues reach zero or timeout. +15. Make `wait_sync_status.py` fail immediately when `status` is not `ok` or when `data.last-error` is not `null`. +16. Add `/Users/rcmerci/gh-repos/logseq/cli-e2e/scripts/compare_graph_queries.py` that executes two CLI query commands and compares normalized payloads. +17. Keep helper scripts CLI-only by calling `node static/logseq-cli.js` commands rather than reading DB files directly. +18. Add shell-level tests for these helper scripts in `/Users/rcmerci/gh-repos/logseq/cli-e2e/test/logseq/cli/e2e/runner_test.clj` or a new helper test namespace using mocked command execution. + +### Phase 4. Implement MVP sync upload/download test case with two worker processes. + +19. Add one MVP case in `/Users/rcmerci/gh-repos/logseq/cli-e2e/spec/sync_cases.edn` that provisions two data dirs under one temp root. +20. In setup, create two separate config files for directory A and directory B with explicit sync endpoint keys and auth placeholders sourced from environment variables. +21. In setup, create the graph in directory A via CLI and add deterministic marker data via CLI `upsert` commands. +22. In setup, start `db-worker-node` for graph A via CLI `server start`. +23. In main commands, run CLI `sync upload` in directory A. +24. In main commands, run `wait_sync_status.py` against directory A to ensure `last-error` remains empty and pending counters settle. +25. In main commands, run CLI `sync download` in directory B for the same graph name. +26. In main commands, start `db-worker-node` for graph B via CLI `server start`. +27. In main commands, use `compare_graph_queries.py` to compare deterministic query outputs between A and B. +28. In cleanup, stop servers for both directory A and directory B via CLI `server stop`. +29. Ensure the case `:covers` map marks `sync upload`, `sync download`, and `sync status` coverage in `/Users/rcmerci/gh-repos/logseq/cli-e2e/spec/sync_cases.edn`. +30. Run `bb -f /Users/rcmerci/gh-repos/logseq/cli-e2e/bb.edn test-sync --skip-build` and verify the MVP case passes. + +### Phase 5. Keep non-sync suite stable and document operator workflow. + +31. Run `bb -f /Users/rcmerci/gh-repos/logseq/cli-e2e/bb.edn test --skip-build` and verify non-sync behavior is unchanged. +32. Update `/Users/rcmerci/gh-repos/logseq/cli-e2e/README.md` with separate commands for non-sync and sync suites. +33. Add required environment variable documentation in `/Users/rcmerci/gh-repos/logseq/cli-e2e/README.md` for sync suite execution. +34. Optionally add `dev:cli-e2e-sync` task in `/Users/rcmerci/gh-repos/logseq/bb.edn` that delegates to `bb -f cli-e2e/bb.edn test-sync`. +35. Run `bb -f /Users/rcmerci/gh-repos/logseq/cli-e2e/bb.edn list-sync-cases` and ensure the new case is discoverable. + +## Verification commands and expected outcomes + +| Command | Expected outcome | +| --- | --- | +| `bb -f /Users/rcmerci/gh-repos/logseq/cli-e2e/bb.edn test --skip-build` | Runs non-sync suite only and remains green. | +| `bb -f /Users/rcmerci/gh-repos/logseq/cli-e2e/bb.edn list-cases` | Lists non-sync case ids only. | +| `bb -f /Users/rcmerci/gh-repos/logseq/cli-e2e/bb.edn list-sync-cases` | Lists sync case ids only. | +| `bb -f /Users/rcmerci/gh-repos/logseq/cli-e2e/bb.edn test-sync --skip-build` | Runs sync suite only and validates MVP upload or download behavior. | +| `bb -f /Users/rcmerci/gh-repos/logseq/cli-e2e/bb.edn test-sync --skip-build --case sync-upload-download-mvp` | Runs one sync MVP case with deterministic status and parity checks. | + +## Edge cases to include in MVP case design + +The sync suite must fail with a clear message when required auth or endpoint environment variables are missing. + +The status polling helper must fail on timeout and print the last seen status payload for debugging. + +The status polling helper must fail when `last-error` appears even if pending counters reach zero. + +The graph parity helper must compare normalized query results, not raw command output strings that can differ by formatting. + +Cleanup must tolerate partially started state and still attempt to stop both servers. + +The sync suite must not mutate or depend on `non_sync_*` manifest files. + +## Open clarifications to resolve before implementation + +MVP sync suite will target local db-sync by default (`http://localhost:8080` plus local websocket). + +CI integration is intentionally out of scope for this phase and will be decided after MVP stabilizes. + +Confirm the minimum auth material for sync MVP in test environments, including whether a refresh token is strictly required or whether pre-seeded runtime tokens in config are sufficient. + +## Testing Details + +The new tests validate real shell behavior through compiled `logseq-cli` commands and real `db-worker-node` process lifecycle handling across two independent data directories. + +The MVP sync case verifies behavior outcomes by checking sync health status, pending queue convergence, and cross-directory graph data parity for deterministic query payloads. + +The suite separation tests ensure sync coverage does not destabilize non-sync command coverage expectations. + +## Implementation Details + +- Keep non-sync manifests and command coverage unchanged. +- Add sync manifests as a separate suite, not an extension of non-sync inventory. +- Reuse existing `main/run!` and runner infrastructure with suite-aware manifest loading. +- Keep all graph mutations and validations CLI-driven in case commands and helper scripts. +- Use two explicit data directories per sync case to guarantee two independent `db-worker-node` processes. +- Poll `sync status` until pending counters settle and fail on `last-error`. +- Compare graph parity through deterministic CLI query outputs. +- Document sync suite environment requirements in `cli-e2e/README.md`. +- Keep sync suite runnable independently with `test-sync` and `list-sync-cases` tasks. +- Defer `sync start` realtime scenarios to a follow-up plan after MVP upload and download stabilization. + +## Question + +Should MVP include only one-direction flow `A upload -> B download`, or should it also include the reverse-direction snapshot refresh in the same phase. + +--- diff --git a/src/main/frontend/worker/db_core.cljs b/src/main/frontend/worker/db_core.cljs index 62837cc6cd..d4b6e6dbd6 100644 --- a/src/main/frontend/worker/db_core.cljs +++ b/src/main/frontend/worker/db_core.cljs @@ -503,8 +503,8 @@ (sync-crypt/ (p/let [log-f (fn [payload] + (rtc-log-and-state/rtc-log :rtc.log/download payload)) + _ (log-f {:sub-type :download-progress + :graph-uuid graph-id + :message "Preparing graph snapshot download"}) + _ (reset! stage* :fetch-pull) + pull-resp (fetch-json (str base "/sync/" graph-id "/pull") + {:method "GET"} + :sync/pull) + remote-tx (:t pull-resp) + _ (when-not (integer? remote-tx) + (throw (ex-info "non-integer remote-tx when downloading graph" + {:repo repo + :remote-tx remote-tx}))) + _ (reset! stage* :fetch-snapshot-download) + snapshot-resp (fetch-json (str base "/sync/" graph-id "/snapshot/download") + {:method "GET"} + :sync/snapshot-download) + _ (reset! stage* :fetch-snapshot-stream) + resp (js/fetch (:url snapshot-resp) + (clj->js (with-auth-headers {:method "GET"}))) + _ (log-f {:sub-type :download-progress + :graph-uuid graph-id + :message "Start downloading graph snapshot"})] + (when-not (.-ok resp) + (throw (ex-info "snapshot download failed" + {:repo repo + :status (.-status resp)}))) + (let [import-id* (atom nil) + ensure-import! (fn [] + (if-let [import-id @import-id*] + (p/resolved import-id) + (p/let [_ (reset! stage* :prepare-import) + {:keys [import-id]} (prepare-import! repo true graph-id graph-e2ee?)] + (reset! import-id* import-id) + import-id)))] + (p/let [_ (do + (reset! stage* :stream-snapshot) + (js (with-auth-headers {:method "GET"}))) - _ (log-f {:sub-type :download-progress - :graph-uuid graph-id - :message "Start downloading graph snapshot"})] - (when-not (.-ok resp) - (throw (ex-info "snapshot download failed" - {:repo repo - :status (.-status resp)}))) - (let [import-id* (atom nil) - ensure-import! (fn [] - (if-let [import-id @import-id*] - (p/resolved import-id) - (p/let [{:keys [import-id]} (prepare-import! repo true graph-id graph-e2ee?)] - (reset! import-id* import-id) - import-id)))] - (p/let [_ ( raw-key string/trim string/lower-case) @@ -263,6 +282,13 @@ :progress-explicit? (contains? options :progress) :e2ee-password (:e2ee-password options)}})) +(defn- build-sync-ensure-keys-action + [options] + {:ok? true + :action {:type :sync-ensure-keys + :e2ee-password (:e2ee-password options) + :upload-keys (:upload-keys options)}}) + (defn- build-sync-grant-access-action [options repo] (if-not (seq repo) @@ -349,8 +375,7 @@ :action {:type :sync-remote-graphs}} :sync-ensure-keys - {:ok? true - :action {:type :sync-ensure-keys}} + (build-sync-ensure-keys-action options) :sync-grant-access (build-sync-grant-access-action options repo) @@ -659,10 +684,23 @@ (p/catch (fn [error] (exception->error error nil))))) +(defn- sync-ensure-keys-upload-options + [{:keys [upload-keys e2ee-password]}] + (when (true? upload-keys) + (cond-> {:ensure-server? true} + (seq e2ee-password) (assoc :password e2ee-password)))) + (defn- run-sync-ensure-keys [action config] (-> (p/let [config' (resolve-runtime-config! action config) - result (invoke-global config' :thread-api/db-sync-ensure-user-rsa-keys [])] + upload-options (sync-ensure-keys-upload-options action) + _ (when-not upload-options + ( (p/with-redefs [transport/invoke (fn [_ method direct-pass? args] + (swap! invoke-calls conj [method direct-pass? args]) + (p/resolved {:ok true}))] + (p/let [_ (sync-command/execute {:type :sync-ensure-keys + :e2ee-password "pw"} + {:base-url "http://example" + :data-dir "/tmp" + :refresh-token "refresh-token" + :id-token "runtime-token"})] + (is (= [:thread-api/sync-app-state + :thread-api/set-db-sync-config + :thread-api/verify-and-save-e2ee-password + :thread-api/sync-app-state + :thread-api/set-db-sync-config + :thread-api/db-sync-ensure-user-rsa-keys] + (mapv first @invoke-calls))) + (is (some #(= [:thread-api/verify-and-save-e2ee-password false ["refresh-token" "pw"]] + %) + @invoke-calls)))) + (p/catch (fn [e] + (is false (str "unexpected error: " e)))) + (p/finally done))))) + +(deftest test-execute-sync-ensure-keys-upload-keys-enables-server-upload-flow + (async done + (let [invoke-calls (atom [])] + (-> (p/with-redefs [transport/invoke (fn [_ method direct-pass? args] + (swap! invoke-calls conj [method direct-pass? args]) + (p/resolved {:ok true}))] + (p/let [_ (sync-command/execute {:type :sync-ensure-keys + :upload-keys true + :e2ee-password "pw"} + {:base-url "http://example" + :data-dir "/tmp" + :refresh-token "refresh-token" + :id-token "runtime-token"})] + (is (= [:thread-api/sync-app-state + :thread-api/set-db-sync-config + :thread-api/db-sync-ensure-user-rsa-keys + :thread-api/sync-app-state + :thread-api/set-db-sync-config + :thread-api/verify-and-save-e2ee-password] + (mapv first @invoke-calls))) + (is (some #(= [:thread-api/db-sync-ensure-user-rsa-keys false [{:ensure-server? true + :password "pw"}]] + %) + @invoke-calls)) + (is (some #(= [:thread-api/verify-and-save-e2ee-password false ["refresh-token" "pw"]] + %) + @invoke-calls)))) + (p/catch (fn [e] + (is false (str "unexpected error: " e)))) + (p/finally done))))) + (deftest test-execute-sync-grant-access-missing-http-base-is-error (async done (let [ensure-calls (atom []) diff --git a/src/test/logseq/cli/commands_test.cljs b/src/test/logseq/cli/commands_test.cljs index 8906da05c2..fc112ef923 100644 --- a/src/test/logseq/cli/commands_test.cljs +++ b/src/test/logseq/cli/commands_test.cljs @@ -1696,6 +1696,18 @@ (is (true? (:ok? enabled))) (is (= true (get-in enabled [:options :progress]))))) + (testing "sync ensure-keys accepts e2ee-password option" + (let [result (commands/parse-args ["sync" "ensure-keys" "--e2ee-password" "pw"])] + (is (true? (:ok? result))) + (is (= :sync-ensure-keys (:command result))) + (is (= "pw" (get-in result [:options :e2ee-password]))))) + + (testing "sync ensure-keys accepts upload-keys option" + (let [result (commands/parse-args ["sync" "ensure-keys" "--upload-keys"])] + (is (true? (:ok? result))) + (is (= :sync-ensure-keys (:command result))) + (is (= true (get-in result [:options :upload-keys]))))) + (testing "graph import rejects unknown type" (let [result (commands/parse-args ["graph" "import" "--type" "zip"