From d74fe5a2789d7316f0f5620fdfbcfbbcd6cb86b9 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Fri, 6 Mar 2026 08:22:17 +0800 Subject: [PATCH] e2b sandbox --- deps/workers/README.md | 22 +- .../docs/milestones/agents/00-index.md | 1 + .../24-m24-e2b-sandbox-runtime-default.md | 97 ++++ deps/workers/package.json | 1 + deps/workers/shadow-cljs.edn | 7 +- .../src/logseq/agents/runtime_provider.cljs | 541 +++++++++++++++++- deps/workers/src/logseq/sync/node/config.cljs | 11 + deps/workers/src/logseq/sync/node/server.cljs | 8 + .../agents/e2b_runtime_provider_test.cljs | 193 +++++++ .../e2b_runtime_provider_test_runner.cljs | 21 + .../logseq/agents/runtime_provider_test.cljs | 24 +- deps/workers/worker/wrangler.agents.toml | 6 +- deps/workers/yarn.lock | 218 ++++++- 13 files changed, 1133 insertions(+), 17 deletions(-) create mode 100644 deps/workers/docs/milestones/agents/24-m24-e2b-sandbox-runtime-default.md create mode 100644 deps/workers/test/logseq/agents/e2b_runtime_provider_test.cljs create mode 100644 deps/workers/test/logseq/agents/e2b_runtime_provider_test_runner.cljs diff --git a/deps/workers/README.md b/deps/workers/README.md index 0cc070ea4e..26f4ab4215 100644 --- a/deps/workers/README.md +++ b/deps/workers/README.md @@ -39,7 +39,7 @@ Local split note: `wrangler dev -c wrangler.toml -c wrangler.agents.toml --port 8787` and `/sessions*` will be forwarded via `AGENTS_SERVICE`. - If you run workers separately, call sessions on the agents port. -- `worker/wrangler.agents.toml` sets `AGENT_RUNTIME_PROVIDER=cloudflare` by default. +- `worker/wrangler.agents.toml` sets `AGENT_RUNTIME_PROVIDER=e2b` by default. - On localhost, `/sessions*` forwarding retries during agents startup (up to ~30s) to avoid transient `503`. Production routing note: @@ -164,12 +164,20 @@ Create session pinned to local runner: ### Runtime Provider Agent runtime is selected by `AGENT_RUNTIME_PROVIDER`: +- `e2b`: provisions/manages E2B sandboxes via the `e2b` SDK, then runs `sandbox-agent` inside the sandbox. - `sprites`: provisions a Sprite and runs `sandbox-agent` inside it. - `local-dev`: uses `SANDBOX_AGENT_URL` directly. - `local-runner`: selects a registered user runner endpoint from `AGENTS_DB` and runs through that endpoint. - `vercel`: provisions/manages Vercel sandboxes via `@vercel/sandbox`, then runs `sandbox-agent` inside the sandbox. - `cloudflare`: provisions a sandbox first, then connects to the sandbox-hosted `sandbox-agent`. -- Agents worker default is `cloudflare` (set in `worker/wrangler.agents.toml`). +- Agents worker default is `e2b` (set in `worker/wrangler.agents.toml`). + +E2B runtime flow: +- create or restore sandbox from template/snapshot +- bootstrap `sandbox-agent` in sandbox +- create runtime session via `sandbox-agent` HTTP API +- stream events/messages through sandbox host URL +- support runtime snapshot persistence and browser terminal open For `cloudflare`, bind and export `Sandbox` in the agents worker and configure the container image in `worker/wrangler.agents.toml` (`[[containers]] class_name = "Sandbox"`). @@ -195,7 +203,7 @@ Cloudflare runtime flow: | DB_SYNC_STATIC_USER_ID | Static user id for local dev | | DB_SYNC_STATIC_EMAIL | Static user email for local dev | | DB_SYNC_STATIC_USERNAME | Static username for local dev | -| AGENT_RUNTIME_PROVIDER | Runtime backend (`sprites`, `local-dev`, `local-runner`, `vercel`, `cloudflare`) | +| AGENT_RUNTIME_PROVIDER | Runtime backend (`e2b`, `sprites`, `local-dev`, `local-runner`, `vercel`, `cloudflare`) | | SENTRY_DSN | Sentry DSN | | SENTRY_RELEASE | Release identifier for Sentry events and sourcemaps | | SENTRY_ENVIRONMENT | Sentry environment name (prod, staging, etc.) | @@ -205,6 +213,14 @@ Cloudflare runtime flow: | COGNITO_JWKS_URL | Cognito JWKS URL | | SANDBOX_AGENT_URL | sandbox-agent base URL for agent sessions | | SANDBOX_AGENT_TOKEN | Optional bearer token for sandbox-agent | +| E2B_API_KEY | E2B API key for runtime provisioning | +| E2B_DOMAIN | Optional E2B API domain override | +| E2B_TEMPLATE | Optional E2B template name/id used for new sandboxes | +| E2B_REPO_CLONE_COMMAND | Optional repo clone command template for E2B runtime | +| E2B_SANDBOX_AGENT_PORT | sandbox-agent port inside E2B sandbox (default `2468`) | +| E2B_SANDBOX_TIMEOUT_MS | E2B sandbox timeout in milliseconds (default `1800000`) | +| E2B_HEALTH_RETRIES | E2B sandbox health check retry count | +| E2B_HEALTH_INTERVAL_MS | E2B sandbox health check retry interval (ms) | | SPRITE_TOKEN | Sprites API token (default runtime auth) | | SPRITES_TOKEN | Alias for `SPRITE_TOKEN` | | SPRITES_API_URL | Sprites API base URL override | diff --git a/deps/workers/docs/milestones/agents/00-index.md b/deps/workers/docs/milestones/agents/00-index.md index 3337810e94..502a0dac53 100644 --- a/deps/workers/docs/milestones/agents/00-index.md +++ b/deps/workers/docs/milestones/agents/00-index.md @@ -27,3 +27,4 @@ Milestones are tracked as separate files in this folder: - `21-m21-store-snapshots-metadata-in-d1.md` - `22-m22-persist-workspace-git-bundles.md` - `23-m23-local-runner-via-tunnel.md` +- `24-m24-e2b-sandbox-runtime-default.md` diff --git a/deps/workers/docs/milestones/agents/24-m24-e2b-sandbox-runtime-default.md b/deps/workers/docs/milestones/agents/24-m24-e2b-sandbox-runtime-default.md new file mode 100644 index 0000000000..75849e2e6a --- /dev/null +++ b/deps/workers/docs/milestones/agents/24-m24-e2b-sandbox-runtime-default.md @@ -0,0 +1,97 @@ +# M24: E2B Sandbox Runtime Provider (Default Runtime) + +Status: Proposed +Target: Add a first-class `e2b` runtime provider and make `e2b` the default runtime for new agent sessions. + +## Goal +Support `AGENT_RUNTIME_PROVIDER=e2b` end-to-end so sessions can provision E2B sandboxes, persist/restore workspace state, use template-based startup, and support browser terminal access. + +## Why M24 +- E2B supports sandbox persistence and resume semantics that align with long-running agent sessions and interruption recovery. +- E2B supports template-based environments, which reduces startup setup time and improves reproducibility. +- E2B exposes PTY APIs for interactive terminals, which can be used to provide browser terminal capabilities. +- Moving the default runtime to `e2b` gives a single hosted default while preserving explicit fallback providers. + +## Inputs +- E2B docs root: `https://e2b.dev/docs` +- E2B quickstart (`E2B_API_KEY`, sandbox creation): `https://e2b.dev/docs/quickstart` +- E2B sandbox persistence: `https://e2b.dev/docs/sandbox/persistence` +- E2B interactive PTY terminal docs: `https://e2b.dev/docs/sandbox/pty` +- E2B template quickstart: `https://e2b.dev/docs/template/quickstart` +- E2B JS SDK sandbox reference (`Sandbox.create/connect/getHost/createSnapshot`, `commands`, `pty`): `https://e2b.dev/docs/sdk-reference/js-sdk/v2.0.1/sandbox` + +## Scope +1) Add `E2BProvider` under `src/logseq/agents/runtime_provider.cljs` implementing runtime lifecycle: +- ` (get-in task [:runtime :template]) str string/trim not-empty) + (env-str env "E2B_TEMPLATE"))) + +(defn- e2b-agent-token + [^js env runtime] + (or (:agent-token runtime) + (env-str env "SANDBOX_AGENT_TOKEN"))) + +(defn- e2b-agent-port + [^js env runtime] + (or (:sandbox-port runtime) + (parse-int (env-str env "E2B_SANDBOX_AGENT_PORT") 2468))) + +(defn- e2b-health-retries + [^js env] + (parse-int (env-str env "E2B_HEALTH_RETRIES") 30)) + +(defn- e2b-health-interval-ms + [^js env] + (parse-int (env-str env "E2B_HEALTH_INTERVAL_MS") 300)) + +(defn- e2b-sandbox-timeout-ms + [^js env] + (parse-int (env-str env "E2B_SANDBOX_TIMEOUT_MS") (* 30 60 1000))) + +(defn- e2b-api-opts + [^js env] + (let [api-key (e2b-api-key env) + domain (e2b-domain env)] + (when-not (string? api-key) + (throw (ex-info "missing E2B_API_KEY for e2b runtime provider" + {:reason :missing-e2b-api-key}))) + (cond-> {:apiKey api-key} + (string? domain) (assoc :domain domain)))) + +(defn e2b-sandbox-class + [] + (let [sandbox-class (aget e2b "Sandbox")] + (when-not sandbox-class + (throw (ex-info "missing e2b Sandbox export" + {:reason :missing-e2b-sdk}))) + sandbox-class)) + +(defn- e2b-sandbox-id + [sandbox] + (or (aget sandbox "sandboxId") + (aget sandbox "id"))) + +(defn- e2b-sandbox-host + [sandbox port] + (let [get-host (js-method sandbox "getHost")] + (when-not (fn? get-host) + (throw (ex-info "e2b sandbox missing getHost method" + {:reason :missing-e2b-get-host}))) + (.call get-host sandbox port))) + +(defn js (merge (e2b-api-opts env) opts))] + (when-not (fn? create) + (throw (ex-info "e2b sdk missing Sandbox.create" + {:reason :missing-e2b-create}))) + (if (string? template) + (->promise (.call create sandbox-class template params)) + (->promise (.call create sandbox-class params))))) + +(defn js (e2b-api-opts env))] + (when-not (string? sandbox-id) + (throw (ex-info "missing sandbox-id on runtime" + {:reason :missing-sandbox-id}))) + (when-not (fn? connect) + (throw (ex-info "e2b sdk missing Sandbox.connect" + {:reason :missing-e2b-connect}))) + (->promise (.call connect sandbox-class sandbox-id opts)))) + +(defn js (e2b-api-opts env))] + (if (and (string? sandbox-id) (fn? kill)) + (->promise (.call kill sandbox-class sandbox-id opts)) + (p/resolved nil)))) + +(defn {} + (string? (:cwd opts)) (assoc :cwd (:cwd opts)) + (map? (:env opts)) (assoc :envs (:env opts)) + (number? (:timeout-ms opts)) (assoc :timeoutMs (:timeout-ms opts)) + (true? (:background opts)) (assoc :background true) + (fn? (:on-stdout opts)) (assoc :onStdout (:on-stdout opts)) + (fn? (:on-stderr opts)) (assoc :onStderr (:on-stderr opts)))] + (when-not (fn? run-command) + (throw (ex-info "e2b sandbox missing commands.run" + {:reason :missing-e2b-run-command}))) + (if (true? (:background opts)) + (->promise (.call run-command commands command (clj->js params))) + (p/let [result (->promise (.call run-command commands command (clj->js params))) + stdout (or (aget result "stdout") "") + stderr (or (aget result "stderr") "") + exit-code (or (aget result "exitCode") + (aget result "exit_code"))] + (when (and (number? exit-code) (not (zero? exit-code))) + (throw (ex-info "e2b sandbox command failed" + {:reason :e2b-command-failed + :command command + :exit-code exit-code + :stdout stdout + :stderr stderr}))) + {:stdout stdout + :stderr stderr + :exit-code exit-code})))) + +(defn- e2b-health-command + [port agent-token] + (str "if curl -fsS " + (curl-auth-arg agent-token) + " " + cloudflare-local-host + ":" + port + "/v1/health >/dev/null; then echo __HEALTH_OK__; else echo __HEALTH_FAIL__; fi")) + +(defn- ( (get-in task [:agent :api-token]) str string/trim not-empty)] + (p/let [github-token ( (merge base task-env) + (and (string? api-token) (= "codex" agent-id)) (assoc "OPENAI_API_KEY" api-token) + (and (string? api-token) (= "claude" agent-id)) (assoc "ANTHROPIC_API_KEY" api-token) + (string? github-token) (assoc-github-installation-token github-token))))) + +(defn- e2b-create-opts + [^js env session-id env-vars] + (let [timeout-ms (e2b-sandbox-timeout-ms env) + metadata (cond-> {"session-id" (or session-id "") + "runtime-provider" "e2b"} + (string? session-id) (assoc "runtime-session-id" session-id))] + (cond-> {:timeoutMs timeout-ms + :lifecycle {:onTimeout "pause" + :autoResume true}} + (seq env-vars) (assoc :envs env-vars) + (string? session-id) (assoc :metadata metadata)))) + +(defn- e2b-server-command + [^js env task session-id port agent-token env-vars] + (let [auth-json (get-in task [:agent :auth-json]) + write-auth (or (auth-json-write-command auth-json) "") + repo-cd (or (repo-cd-command session-id task "e2b") "") + token-exports (shell-export-command env-vars) + github-auth-setup (github-auth-setup-command (get env-vars "GITHUB_TOKEN"))] + (str (sandbox-agent-install-command env) + token-exports + github-auth-setup + write-auth + (when (string? repo-cd) (str repo-cd "; ")) + "nohup sandbox-agent server " + (if (string? agent-token) + (str "--token '" (escape-shell-single agent-token) "'") + "--no-token") + " --host 0.0.0.0 --port " port + " --no-telemetry >/tmp/sandbox-agent.log 2>&1 &"))) + +(defn- promise (.call create-snapshot sandbox)) + snapshot-id (or (aget snapshot "snapshotId") + (aget snapshot "snapshotID"))] + (if (string? snapshot-id) + {:snapshot-id snapshot-id} + (throw (ex-info "e2b snapshot create returned invalid id" + {:reason :invalid-snapshot-id + :snapshot snapshot})))))) + +(defn- ( (get-in task [:project :base-branch]) source-control/sanitize-branch-name) + "main") + head-branch (some-> (:head-branch opts) source-control/sanitize-branch-name)] + (when-not (string? sandbox-id) + (throw (ex-info "missing sandbox-id on runtime" + {:reason :missing-sandbox-id + :runtime runtime}))) + (when-not (string? repo-dir) + (throw (ex-info "missing repo dir for workspace bundle export" + {:reason :missing-repo-dir + :session-id session-id}))) + (p/let [sandbox ( (:head-sha opts) str string/trim not-empty) + bundle-base64 (some-> (:bundle-base64 opts) str string/trim not-empty) + head-branch (some-> (:head-branch opts) source-control/sanitize-branch-name)] + (when-not (string? sandbox-id) + (throw (ex-info "missing sandbox-id on runtime" + {:reason :missing-sandbox-id + :runtime runtime}))) + (when-not (string? repo-dir) + (throw (ex-info "missing repo dir for workspace bundle apply" + {:reason :missing-repo-dir + :session-id session-id}))) + (when-not (string? head-sha) + (throw (ex-info "missing bundle head sha" + {:reason :missing-bundle-head-sha + :provider "e2b"}))) + (when-not (string? bundle-base64) + (throw (ex-info "missing bundle payload" + {:reason :missing-bundle-payload + :provider "e2b"}))) + (p/let [sandbox ( result + (string? backup-key) (assoc :backup-key backup-key) + (string? backup-dir) (assoc :backup-dir backup-dir) + :always (assoc :provider "e2b"))))) + + ( (:push-token opts) str string/trim not-empty)) + run-opts (when (seq env-vars) + {:env env-vars})] + (-> (p/let [sandbox (E2BProvider env) "local-dev" (->LocalDevProvider env) "local-runner" (->LocalRunnerProvider env) "vercel" (->VercelProvider env) "cloudflare" (->CloudflareProvider env) - (->SpritesProvider env))) + (->E2BProvider env))) (defn resolve-provider [^js env runtime] @@ -2495,4 +3027,5 @@ (defn runtime-terminal-supported? [runtime] (let [provider (known-provider-kind (:provider runtime))] - (= "cloudflare" provider))) + (or (= "cloudflare" provider) + (= "e2b" provider)))) diff --git a/deps/workers/src/logseq/sync/node/config.cljs b/deps/workers/src/logseq/sync/node/config.cljs index 90ce3a5615..5e4d84c838 100644 --- a/deps/workers/src/logseq/sync/node/config.cljs +++ b/deps/workers/src/logseq/sync/node/config.cljs @@ -24,6 +24,14 @@ :agent-runtime-provider (env-value env "AGENT_RUNTIME_PROVIDER") :sandbox-agent-url (env-value env "SANDBOX_AGENT_URL") :sandbox-agent-token (env-value env "SANDBOX_AGENT_TOKEN") + :e2b-api-key (env-value env "E2B_API_KEY") + :e2b-domain (env-value env "E2B_DOMAIN") + :e2b-template (env-value env "E2B_TEMPLATE") + :e2b-repo-clone-command (env-value env "E2B_REPO_CLONE_COMMAND") + :e2b-sandbox-agent-port (env-value env "E2B_SANDBOX_AGENT_PORT") + :e2b-sandbox-timeout-ms (env-value env "E2B_SANDBOX_TIMEOUT_MS") + :e2b-health-retries (env-value env "E2B_HEALTH_RETRIES") + :e2b-health-interval-ms (env-value env "E2B_HEALTH_INTERVAL_MS") :sprite-token (env-value env "SPRITE_TOKEN") :sprites-token (env-value env "SPRITES_TOKEN") :sprites-api-url (env-value env "SPRITES_API_URL") @@ -72,6 +80,9 @@ [:port :base-url :data-dir :storage-driver :assets-driver :auth-driver :auth-token :static-user-id :static-email :static-username :agent-runtime-provider :sandbox-agent-url :sandbox-agent-token + :e2b-api-key :e2b-domain :e2b-template :e2b-repo-clone-command + :e2b-sandbox-agent-port :e2b-sandbox-timeout-ms + :e2b-health-retries :e2b-health-interval-ms :sprite-token :sprites-token :sprites-api-url :sprites-timeout-ms :sprites-name-prefix :sprites-ram-mb :sprites-cpus :sprites-region :sprites-storage-gb :sprites-bootstrap-command :sprites-repo-clone-command diff --git a/deps/workers/src/logseq/sync/node/server.cljs b/deps/workers/src/logseq/sync/node/server.cljs index 5a850628c9..84b7d25930 100644 --- a/deps/workers/src/logseq/sync/node/server.cljs +++ b/deps/workers/src/logseq/sync/node/server.cljs @@ -33,6 +33,14 @@ (aset "AGENT_RUNTIME_PROVIDER" (:agent-runtime-provider cfg)) (aset "SANDBOX_AGENT_URL" (:sandbox-agent-url cfg)) (aset "SANDBOX_AGENT_TOKEN" (:sandbox-agent-token cfg)) + (aset "E2B_API_KEY" (:e2b-api-key cfg)) + (aset "E2B_DOMAIN" (:e2b-domain cfg)) + (aset "E2B_TEMPLATE" (:e2b-template cfg)) + (aset "E2B_REPO_CLONE_COMMAND" (:e2b-repo-clone-command cfg)) + (aset "E2B_SANDBOX_AGENT_PORT" (:e2b-sandbox-agent-port cfg)) + (aset "E2B_SANDBOX_TIMEOUT_MS" (:e2b-sandbox-timeout-ms cfg)) + (aset "E2B_HEALTH_RETRIES" (:e2b-health-retries cfg)) + (aset "E2B_HEALTH_INTERVAL_MS" (:e2b-health-interval-ms cfg)) (aset "SPRITE_TOKEN" (:sprite-token cfg)) (aset "SPRITES_TOKEN" (:sprites-token cfg)) (aset "SPRITES_API_URL" (:sprites-api-url cfg)) diff --git a/deps/workers/test/logseq/agents/e2b_runtime_provider_test.cljs b/deps/workers/test/logseq/agents/e2b_runtime_provider_test.cljs new file mode 100644 index 0000000000..13cb8136a8 --- /dev/null +++ b/deps/workers/test/logseq/agents/e2b_runtime_provider_test.cljs @@ -0,0 +1,193 @@ +(ns logseq.agents.e2b-runtime-provider-test + (:require [cljs.test :refer [async deftest is testing]] + [clojure.string :as string] + [logseq.agents.runtime-provider :as runtime-provider])) + +(deftest e2b-default-provider-kind-test + (testing "e2b is the default and supports explicit normalization" + (is (= "e2b" (runtime-provider/provider-kind #js {}))) + (is (= "e2b" (runtime-provider/provider-kind #js {"AGENT_RUNTIME_PROVIDER" "E2B"}))) + (is (= "e2b" (runtime-provider/provider-kind #js {"AGENT_RUNTIME_PROVIDER" "unknown"}))))) + +(deftest e2b-provider-dispatch-test + (testing "create-provider and resolve-provider dispatch e2b" + (let [env #js {"AGENT_RUNTIME_PROVIDER" "e2b"}] + (is (= "e2b" + (runtime-provider/provider-id (runtime-provider/create-provider env "e2b")))) + (is (= "e2b" + (runtime-provider/provider-id (runtime-provider/create-provider env "unknown")))) + (is (= "e2b" + (runtime-provider/provider-id (runtime-provider/resolve-provider env {:provider "e2b"})))) + (is (= "e2b" + (runtime-provider/provider-id (runtime-provider/resolve-provider env nil))))))) + +(deftest e2b-terminal-supported-test + (testing "e2b runtime supports browser terminal" + (is (true? (runtime-provider/runtime-terminal-supported? {:provider "e2b"}))))) + +(deftest e2b-provider-provision-test + (async done + (let [calls (atom []) + env #js {"E2B_API_KEY" "e2b-key" + "SANDBOX_AGENT_TOKEN" "agent-token"} + provider (runtime-provider/create-provider env "e2b") + task {:agent {:provider "codex"}} + sandbox-class (runtime-provider/e2b-sandbox-class) + original-create (aget sandbox-class "create") + original-fetch js/fetch + restore! (fn [] + (aset sandbox-class "create" original-create) + (set! js/fetch original-fetch))] + (aset sandbox-class "create" + (fn [& args] + (let [opts (js->clj (last args) :keywordize-keys true)] + (is (= "e2b-key" (:apiKey opts))) + (is (= "pause" (get-in opts [:lifecycle :onTimeout]))) + (js/Promise.resolve + #js {:sandboxId "e2b-sbx-1" + :getHost (fn [port] + (swap! calls conj {:type :host :port port}) + "https://e2b-agent.local") + :commands + #js {:run (fn [cmd _opts] + (swap! calls conj {:type :command :cmd cmd}) + (if (string/includes? cmd "/v1/health") + (js/Promise.resolve #js {:stdout "__HEALTH_OK__" + :stderr "" + :exitCode 0}) + (js/Promise.resolve #js {:stdout "" + :stderr "" + :exitCode 0})))}})))) + (set! js/fetch + (fn [request] + (is (= "POST" (.-method request))) + (is (= "https://e2b-agent.local/v1/sessions/sess-e2b-1" (.-url request))) + (is (= "Bearer agent-token" + (.get (.-headers request) "authorization"))) + (js/Promise.resolve + (js/Response. + (js/JSON.stringify #js {:ok true}) + #js {:status 200 + :headers #js {"content-type" "application/json"}})))) + (-> (runtime-provider/ (runtime-provider/ (runtime-provider/ (runtime-provider/