add logseq sandbox docker file

This commit is contained in:
Tienson Qin
2026-03-08 21:01:08 +08:00
parent d96ed6ab58
commit 4acba65e45
3 changed files with 400 additions and 116 deletions

View File

@@ -398,7 +398,7 @@
(defn- e2b-template-name
[task]
(when-let [graph-id (task-graph-id task)]
(sanitize-name (str "logseq-" graph-id "-" (task-repo-name task)))))
(sanitize-name (str graph-id "-" (task-repo-name task)))))
(defn- e2b-agent-token
[^js env runtime]
@@ -470,6 +470,8 @@
{:reason :missing-e2b-get-host})))
(sandbox/normalize-base-url (.call get-host sandbox port))))
(declare <e2b-build-template! <e2b-resolve-template!)
(defn <e2b-create-sandbox!
[^js env template opts]
(let [sandbox-class (e2b-sandbox-class)
@@ -482,6 +484,36 @@
(->promise (.call create sandbox-class template params))
(->promise (.call create sandbox-class params)))))
(defn- e2b-template-not-found-error?
[error]
(let [message (or (some-> error (aget "message"))
(some-> error str))]
(boolean (and (string? message)
(string/includes? message "404:")
(string/includes? message "template")
(string/includes? message "not found")))))
(defn- <e2b-create-sandbox-with-template-recovery!
[^js env task create-opts]
(p/let [template (<e2b-resolve-template! env task nil)]
(-> (<e2b-create-sandbox! env template create-opts)
(p/then (fn [sandbox]
{:sandbox sandbox
:template template}))
(p/catch
(fn [error]
(if-not (and (string? template)
(e2b-template-not-found-error? error))
(throw error)
(if (project-docker-file task)
(p/let [rebuilt-template (<e2b-build-template! env task (project-docker-file task) {:force? true})
sandbox (<e2b-create-sandbox! env rebuilt-template create-opts)]
{:sandbox sandbox
:template rebuilt-template})
(p/let [sandbox (<e2b-create-sandbox! env nil create-opts)]
{:sandbox sandbox
:template nil}))))))))
(defn <e2b-connect-sandbox!
[^js env sandbox-id]
(let [sandbox-class (e2b-sandbox-class)
@@ -632,35 +664,72 @@
[]
(aget e2b "Template"))
(defn- e2b-template-missing-error?
[error]
(let [message (some-> error str)]
(boolean (and (string? message)
(string/starts-with? message "Error: 404:")))))
(defn- <e2b-existing-template!
(defn- <e2b-template-exists!
[^js env template-name]
(let [template-fn (e2b-template-fn)
get-tags (js-method template-fn "getTags")]
exists (js-method template-fn "exists")]
(when-not (fn? template-fn)
(throw (ex-info "e2b sdk missing Template"
{:reason :missing-e2b-template-sdk})))
(when-not (fn? get-tags)
(throw (ex-info "e2b sdk missing Template.getTags"
{:reason :missing-e2b-template-get-tags})))
(-> (->promise (.call get-tags template-fn template-name (clj->js (e2b-api-opts env))))
(p/then (fn [_tags] template-name))
(p/catch (fn [error]
(if (e2b-template-missing-error? error)
nil
(throw error)))))))
(when-not (fn? exists)
(throw (ex-info "e2b sdk missing Template.exists"
{:reason :missing-e2b-template-exists})))
(-> (->promise (.call exists template-fn template-name (clj->js (e2b-api-opts env))))
(p/then true?))))
(defn- <e2b-build-template!
[^js env task docker-file & [{:keys [force?]}]]
(let [template-fn (e2b-template-fn)
build (js-method template-fn "build")
wait-for-timeout (aget e2b "waitForTimeout")
default-build-logger (aget e2b "defaultBuildLogger")
template-name (e2b-template-name task)]
(when-not (fn? template-fn)
(throw (ex-info "e2b sdk missing Template"
{:reason :missing-e2b-template-sdk})))
(when-not (fn? build)
(throw (ex-info "e2b sdk missing Template.build"
{:reason :missing-e2b-template-build})))
(when-not (string? template-name)
(throw (ex-info "missing graph-scoped e2b template name"
{:reason :missing-e2b-template-name})))
(p/let [template-exists? (if force?
(p/resolved false)
(<e2b-template-exists! env template-name))]
(if template-exists?
template-name
(p/let [template-builder-fn (some-> (template-fn) (js-method "fromDockerfile"))
template-builder (when (fn? template-builder-fn)
(.call template-builder-fn (template-fn) docker-file))
set-start-cmd (js-method template-builder "setStartCmd")
template-builder (if (and (some? template-builder)
(fn? set-start-cmd)
(fn? wait-for-timeout))
(.call set-start-cmd
template-builder
"bash -lc 'while true; do sleep 3600; done'"
(.call wait-for-timeout nil 5000))
template-builder)
build-opts (cond-> (merge (e2b-api-opts env)
{:cpuCount 1
:memoryMB 2048
:skipCache false})
(fn? default-build-logger)
(assoc :onBuildLogs (.call default-build-logger nil)))]
(when-not (some? template-builder)
(throw (ex-info "e2b sdk missing Template().fromDockerfile"
{:reason :missing-e2b-from-dockerfile})))
(p/let [_build-info (->promise (.call build
template-fn
template-builder
template-name
(clj->js build-opts)))]
template-name))))))
(defn- <e2b-resolve-template!
[^js env task runtime]
(if (project-docker-file task)
(if-let [template-name (e2b-template-name task)]
(<e2b-existing-template! env template-name)
(p/resolved nil))
(<e2b-build-template! env task (project-docker-file task))
(p/resolved (e2b-template env task runtime))))
(defn- e2b-server-command
@@ -958,22 +1027,23 @@
checkpoint
create-opts
(fn []
(p/let [template (<e2b-resolve-template! env task nil)
sandbox (<e2b-create-sandbox! env template create-opts)]
(p/let [{:keys [sandbox template]} (<e2b-create-sandbox-with-template-recovery! env
task
create-opts)]
{:sandbox sandbox
:snapshot-id nil
:restored? false
:template template})))
_ (<e2b-ensure-running! env sandbox session-id task port agent-token env-vars)
_ (when-not restored?
(p/catch
(<e2b-clone-repo! env sandbox session-id task)
(fn [error]
(log/error :agent/e2b-repo-clone-failed
{:session-id session-id
:error (str error)
:error-data (ex-data error)})
nil)))
;; _ (when-not restored?
;; (p/catch
;; (<e2b-clone-repo! env sandbox session-id task)
;; (fn [error]
;; (log/error :agent/e2b-repo-clone-failed
;; {:session-id session-id
;; :error (str error)
;; :error-data (ex-data error)})
;; nil)))
base-url (e2b-sandbox-host sandbox port)
response (sandbox/<create-session base-url agent-token session-id payload)
sandbox-id (e2b-sandbox-id sandbox)]

View File

@@ -141,7 +141,7 @@
(is false (str "unexpected error: " error))
(done)))))))
(deftest e2b-provider-provision-uses-existing-template-from-project-docker-file-test
(deftest e2b-provider-provision-builds-template-from-project-docker-file-test
(async done
(let [calls (atom [])
env #js {"E2B_API_KEY" "e2b-key"
@@ -154,21 +154,54 @@
:docker-file "FROM node:20\nRUN corepack enable"}}
e2b-ns (js/require "e2b")
original-template (aget e2b-ns "Template")
original-wait-for-timeout (aget e2b-ns "waitForTimeout")
original-default-build-logger (aget e2b-ns "defaultBuildLogger")
sandbox-class (runtime-provider/e2b-sandbox-class)
original-create (aget sandbox-class "create")
original-fetch js/fetch
restore! (fn []
(aset e2b-ns "Template" original-template)
(aset e2b-ns "waitForTimeout" original-wait-for-timeout)
(aset e2b-ns "defaultBuildLogger" original-default-build-logger)
(aset sandbox-class "create" original-create)
(set! js/fetch original-fetch))
template-fn (fn [] #js {})]
(aset template-fn "getTags"
template-builder #js {}
template-fn (fn []
template-builder)]
(aset template-builder "fromDockerfile"
(fn [docker-file]
(swap! calls conj {:type :from-dockerfile
:docker-file docker-file})
template-builder))
(aset template-builder "setStartCmd"
(fn [command ready-cmd]
(swap! calls conj {:type :set-start-cmd
:command command
:ready-cmd ready-cmd})
template-builder))
(aset template-fn "exists"
(fn [name opts]
(swap! calls conj {:type :get-tags
(swap! calls conj {:type :exists
:name name
:opts (js->clj opts :keywordize-keys true)})
(js/Promise.resolve #js [])))
(js/Promise.resolve false)))
(aset template-fn "build"
(fn [builder name opts]
(swap! calls conj {:type :build
:builder (= builder template-builder)
:name name
:opts (js->clj opts :keywordize-keys true)})
(js/Promise.resolve #js {:templateId "tpl-from-docker"})))
(aset e2b-ns "Template" template-fn)
(aset e2b-ns "waitForTimeout"
(fn [timeout-ms]
(swap! calls conj {:type :wait-for-timeout
:timeout-ms timeout-ms})
#js {:timeoutMs timeout-ms}))
(aset e2b-ns "defaultBuildLogger"
(fn []
(swap! calls conj {:type :default-build-logger})
(fn [_entry] nil)))
(aset sandbox-class "create"
(fn [& args]
(let [template (first args)
@@ -201,7 +234,22 @@
(restore!)
(is (= "e2b" (:provider runtime)))
(is (= "logseq-graph-123-agent-test" (:template runtime)))
(is (some #(and (= :get-tags (:type %))
(is (some #(= {:type :from-dockerfile
:docker-file "FROM node:20\nRUN corepack enable"} %)
@calls))
(is (some #(and (= :set-start-cmd (:type %))
(= "bash -lc 'while true; do sleep 3600; done'" (:command %)))
@calls))
(is (some #(= {:type :wait-for-timeout
:timeout-ms 5000} %)
@calls))
(is (some #(= {:type :default-build-logger} %)
@calls))
(is (some #(and (= :exists (:type %))
(= "logseq-graph-123-agent-test" (:name %)))
@calls))
(is (some #(and (= :build (:type %))
(:builder %)
(= "logseq-graph-123-agent-test" (:name %))
(= "e2b-key" (get-in % [:opts :apiKey])))
@calls))
@@ -214,12 +262,11 @@
(is false (str "unexpected error: " error))
(done)))))))
(deftest e2b-provider-provision-falls-back-to-default-sandbox-when-template-missing-test
(deftest e2b-provider-provision-reuses-existing-template-before-building-test
(async done
(let [calls (atom [])
env #js {"E2B_API_KEY" "e2b-key"
"SANDBOX_AGENT_TOKEN" "agent-token"
"E2B_TEMPLATE" "fallback-template"}
"SANDBOX_AGENT_TOKEN" "agent-token"}
provider (runtime-provider/create-provider env "e2b")
task {:agent {:provider "codex"}
:project {:repo-url "https://github.com/logseq/agent-test"
@@ -235,29 +282,34 @@
(aset sandbox-class "create" original-create)
(set! js/fetch original-fetch))
template-fn (fn [] #js {})]
(aset template-fn "getTags"
(fn [_name _opts]
(js/Promise.reject (js/Error. "404: Not Found"))))
(aset template-fn "exists"
(fn [name _opts]
(swap! calls conj {:type :exists
:name name})
(js/Promise.resolve true)))
(aset template-fn "build"
(fn [& _args]
(swap! calls conj {:type :build})
(js/Promise.resolve #js {:templateId "tpl-should-not-build"})))
(aset e2b-ns "Template" template-fn)
(aset sandbox-class "create"
(fn [& args]
(swap! calls conj {:type :create
:argc (count args)
:first-arg (first args)
:last-arg (js->clj (last args) :keywordize-keys true)})
(js/Promise.resolve
#js {:sandboxId "e2b-sbx-docker-default"
:getHost (fn [_port]
"https://e2b-agent.local")
:commands
#js {:run (fn [cmd _opts]
(if (string/includes? cmd "/v1/health")
(js/Promise.resolve #js {:stdout "__HEALTH_OK__"
:stderr ""
:exitCode 0})
(js/Promise.resolve #js {:stdout ""
:stderr ""
:exitCode 0})))}})))
(let [template (first args)]
(swap! calls conj {:type :create
:template template})
(js/Promise.resolve
#js {:sandboxId "e2b-sbx-existing-template"
:getHost (fn [_port]
"https://e2b-agent.local")
:commands
#js {:run (fn [cmd _opts]
(if (string/includes? cmd "/v1/health")
(js/Promise.resolve #js {:stdout "__HEALTH_OK__"
:stderr ""
:exitCode 0})
(js/Promise.resolve #js {:stdout ""
:stderr ""
:exitCode 0})))}}))))
(set! js/fetch
(fn [_request]
(js/Promise.resolve
@@ -265,16 +317,186 @@
(js/JSON.stringify #js {:ok true})
#js {:status 200
:headers #js {"content-type" "application/json"}}))))
(-> (runtime-provider/<provision-runtime! provider "sess-e2b-docker-default" task)
(-> (runtime-provider/<provision-runtime! provider "sess-e2b-existing-template" task)
(.then (fn [runtime]
(restore!)
(is (= "e2b" (:provider runtime)))
(is (nil? (:template runtime)))
(is (some #(and (= :create (:type %))
(= 1 (:argc %))
(map? (:first-arg %))
(= "e2b-key" (get-in % [:first-arg :apiKey])))
(is (= "logseq-graph-123-agent-test" (:template runtime)))
(is (some #(= {:type :exists
:name "logseq-graph-123-agent-test"} %)
@calls))
(is (nil? (some #(when (= :build (:type %)) %) @calls)))
(is (some #(and (= :create (:type %))
(= "logseq-graph-123-agent-test" (:template %)))
@calls))
(done)))
(.catch (fn [error]
(restore!)
(is false (str "unexpected error: " error))
(done)))))))
(deftest e2b-provider-provision-falls-back-when-configured-template-is-missing-test
(async done
(let [calls (atom [])
env #js {"E2B_API_KEY" "e2b-key"
"SANDBOX_AGENT_TOKEN" "agent-token"
"E2B_TEMPLATE" "78fq0upgdyots2idb7fv"}
provider (runtime-provider/create-provider env "e2b")
task {:agent {:provider "codex"}
:project {:repo-url "https://github.com/logseq/agent-test"}}
sandbox-class (runtime-provider/e2b-sandbox-class)
original-create (aget sandbox-class "create")
original-fetch js/fetch
restore! (fn []
(aset sandbox-class "create" original-create)
(set! js/fetch original-fetch))]
(aset sandbox-class "create"
(fn [& args]
(swap! calls conj {:type :create
:argc (count args)
:template (when (= 2 (count args))
(first args))
:opts (js->clj (last args) :keywordize-keys true)})
(if (= 2 (count args))
(js/Promise.reject
#js {:name "SandboxError"
:message "404: template '78fq0upgdyots2idb7fv' not found"})
(js/Promise.resolve
#js {:sandboxId "e2b-sbx-fallback-template"
:getHost (fn [_port]
"https://e2b-agent.local")
:commands
#js {:run (fn [cmd _opts]
(if (string/includes? cmd "/v1/health")
(js/Promise.resolve #js {:stdout "__HEALTH_OK__"
:stderr ""
:exitCode 0})
(js/Promise.resolve #js {:stdout ""
:stderr ""
:exitCode 0})))}}))))
(set! js/fetch
(fn [_request]
(js/Promise.resolve
(js/Response.
(js/JSON.stringify #js {:ok true})
#js {:status 200
:headers #js {"content-type" "application/json"}}))))
(-> (runtime-provider/<provision-runtime! provider "sess-e2b-fallback-template" task)
(.then (fn [runtime]
(restore!)
(is (= "e2b-sbx-fallback-template" (:sandbox-id runtime)))
(is (nil? (:template runtime)))
(is (= 2 (count @calls)))
(is (= {:type :create
:argc 2
:template "78fq0upgdyots2idb7fv"}
(select-keys (first @calls) [:type :argc :template])))
(is (= {:type :create
:argc 1
:template nil}
(select-keys (second @calls) [:type :argc :template])))
(is (= "e2b-key" (get-in (first @calls) [:opts :apiKey])))
(is (= "pause" (get-in (first @calls) [:opts :lifecycle :onTimeout])))
(is (= "sess-e2b-fallback-template"
(get-in (first @calls) [:opts :metadata :session-id])))
(done)))
(.catch (fn [error]
(restore!)
(is false (str "unexpected error: " error))
(done)))))))
(deftest e2b-provider-provision-rebuilds-docker-template-when-create-reports-missing-template-test
(async done
(let [calls (atom [])
env #js {"E2B_API_KEY" "e2b-key"
"SANDBOX_AGENT_TOKEN" "agent-token"}
provider (runtime-provider/create-provider env "e2b")
task {:agent {:provider "codex"}
:project {:repo-url "https://github.com/logseq/logseq"
:graph-id "d86730bc-f157-4f91-808a-5b23532c51fc"
:docker-file "FROM node:20\nRUN corepack enable"}}
e2b-ns (js/require "e2b")
original-template (aget e2b-ns "Template")
original-wait-for-timeout (aget e2b-ns "waitForTimeout")
original-default-build-logger (aget e2b-ns "defaultBuildLogger")
sandbox-class (runtime-provider/e2b-sandbox-class)
original-create (aget sandbox-class "create")
original-fetch js/fetch
restore! (fn []
(aset e2b-ns "Template" original-template)
(aset e2b-ns "waitForTimeout" original-wait-for-timeout)
(aset e2b-ns "defaultBuildLogger" original-default-build-logger)
(aset sandbox-class "create" original-create)
(set! js/fetch original-fetch))
template-builder #js {}
template-fn (fn [] template-builder)]
(aset template-builder "fromDockerfile"
(fn [docker-file]
(swap! calls conj {:type :from-dockerfile
:docker-file docker-file})
template-builder))
(aset template-builder "setStartCmd"
(fn [command _ready-cmd]
(swap! calls conj {:type :set-start-cmd
:command command})
template-builder))
(aset template-fn "exists"
(fn [name _opts]
(swap! calls conj {:type :exists
:name name})
(js/Promise.resolve true)))
(aset template-fn "build"
(fn [_builder name _opts]
(swap! calls conj {:type :build
:name name})
(js/Promise.resolve #js {:templateId "tpl-rebuilt"})))
(aset e2b-ns "Template" template-fn)
(aset e2b-ns "waitForTimeout" (fn [_timeout-ms] #js {}))
(aset e2b-ns "defaultBuildLogger" (fn [] (fn [_entry] nil)))
(aset sandbox-class "create"
(fn [& args]
(let [template (first args)]
(swap! calls conj {:type :create
:argc (count args)
:template (when (= 2 (count args)) template)})
(if (= 1 (count (filter #(and (= :create (:type %))
(= "d86730bc-f157-4f91-808a-5b23532c51fc-logseq" (:template %)))
@calls)))
(js/Promise.reject
#js {:name "SandboxError"
:message "404: template '78fq0upgdyots2idb7fv' not found"})
(js/Promise.resolve
#js {:sandboxId "e2b-sbx-rebuilt-template"
:getHost (fn [_port]
"https://e2b-agent.local")
:commands
#js {:run (fn [cmd _opts]
(if (string/includes? cmd "/v1/health")
(js/Promise.resolve #js {:stdout "__HEALTH_OK__"
:stderr ""
:exitCode 0})
(js/Promise.resolve #js {:stdout ""
:stderr ""
:exitCode 0})))}})))))
(set! js/fetch
(fn [_request]
(js/Promise.resolve
(js/Response.
(js/JSON.stringify #js {:ok true})
#js {:status 200
:headers #js {"content-type" "application/json"}}))))
(-> (runtime-provider/<provision-runtime! provider "sess-e2b-rebuild-template" task)
(.then (fn [runtime]
(restore!)
(is (= "d86730bc-f157-4f91-808a-5b23532c51fc-logseq" (:template runtime)))
(is (= 1 (count (filter #(= {:type :exists
:name "d86730bc-f157-4f91-808a-5b23532c51fc-logseq"} %)
@calls))))
(is (= 1 (count (filter #(= {:type :build
:name "d86730bc-f157-4f91-808a-5b23532c51fc-logseq"} %)
@calls))))
(is (= 2 (count (filter #(and (= :create (:type %))
(= "d86730bc-f157-4f91-808a-5b23532c51fc-logseq" (:template %)))
@calls))))
(done)))
(.catch (fn [error]
(restore!)

View File

@@ -1,60 +1,52 @@
FROM docker.io/cloudflare/sandbox:0.7.6
FROM node:lts-trixie
ENV DEBIAN_FRONTEND=noninteractive
# System deps + Java 21
RUN apt-get update && apt-get install -y --no-install-recommends \
RUN DEBIAN_FRONTEND=noninteractive apt-get update \
&& apt-get install -y --no-install-recommends \
ca-certificates \
curl \
git \
gnupg \
openjdk-21-jdk \
rclone \
ripgrep \
fd-find \
jq \
tmux \
vim \
&& ln -sf /usr/bin/fdfind /usr/local/bin/fd \
&& mkdir -p /etc/apt/keyrings \
&& curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg \
-o /etc/apt/keyrings/githubcli-archive-keyring.gpg \
&& chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \
&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" \
> /etc/apt/sources.list.d/github-cli.list \
&& apt-get update \
&& apt-get install -y --no-install-recommends gh \
&& rm -rf /var/lib/apt/lists/*
# Install Node.js 22
# The base image has Node 20, we need to replace it with Node 22
# Using direct binary download for reliability
ENV NODE_VERSION=22.20.0
RUN ARCH="$(dpkg --print-architecture)" \
&& case "${ARCH}" in \
amd64) NODE_ARCH="x64" ;; \
arm64) NODE_ARCH="arm64" ;; \
*) echo "Unsupported architecture: ${ARCH}" >&2; exit 1 ;; \
esac \
&& apt-get update && apt-get install -y xz-utils ca-certificates rclone tmux vim \
&& curl -fsSLk https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${NODE_ARCH}.tar.xz -o /tmp/node.tar.xz \
&& tar -xJf /tmp/node.tar.xz -C /usr/local --strip-components=1 \
&& rm /tmp/node.tar.xz \
&& node --version \
&& npm --version
# GitHub CLI
RUN mkdir -p /etc/apt/keyrings \
&& curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg \
-o /etc/apt/keyrings/githubcli-archive-keyring.gpg \
&& chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \
&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" \
> /etc/apt/sources.list.d/github-cli.list \
&& apt-get update \
&& apt-get install -y --no-install-recommends gh \
&& gh --version \
&& rm -rf /var/lib/apt/lists/*
# Prefer corepack over "npm i -g yarn"
RUN corepack enable
# Optionally pin yarn (pick a version you want)
# RUN corepack prepare yarn@4.6.0 --activate
# Clojure
RUN curl -fsSL -o /tmp/linux-install.sh \
https://github.com/clojure/brew-install/releases/latest/download/linux-install.sh \
&& chmod +x /tmp/linux-install.sh \
&& /tmp/linux-install.sh \
&& rm -f /tmp/linux-install.sh
RUN curl -fsSL \
https://github.com/clojure/brew-install/releases/latest/download/linux-install.sh \
-o /tmp/clojure-install.sh \
&& chmod +x /tmp/clojure-install.sh \
&& /tmp/clojure-install.sh \
&& rm -f /tmp/clojure-install.sh
# Babashka (actually run installer)
RUN curl -fsSL https://raw.githubusercontent.com/babashka/babashka/master/install | bash
WORKDIR /workspace
# sandbox-agent + codex agent
RUN curl -fsSL https://releases.rivet.dev/sandbox-agent/0.1.5/install.sh | sh \
&& sandbox-agent install-agent codex
&& /usr/local/bin/sandbox-agent install-agent codex
WORKDIR /home/user/workspace
RUN git clone https://github.com/logseq/logseq
WORKDIR /home/user/workspace/logseq
RUN yarn install
RUN clojure -M:test compile test
EXPOSE 2468
CMD ["/bin/bash"]