diff --git a/deps/db/src/logseq/db/frontend/class.cljs b/deps/db/src/logseq/db/frontend/class.cljs index fb41f3ad33..d4949bb645 100644 --- a/deps/db/src/logseq/db/frontend/class.cljs +++ b/deps/db/src/logseq/db/frontend/class.cljs @@ -45,6 +45,7 @@ :logseq.class/Project {:title "Project" :schema {:properties [:logseq.property/git-repo + :logseq.property/project.docker-file :logseq.property/project-sandbox-init-setup] :required-properties [:logseq.property/git-repo]}} diff --git a/deps/db/src/logseq/db/frontend/property.cljs b/deps/db/src/logseq/db/frontend/property.cljs index b26ce091a3..b737ffc822 100644 --- a/deps/db/src/logseq/db/frontend/property.cljs +++ b/deps/db/src/logseq/db/frontend/property.cljs @@ -449,6 +449,12 @@ :public? true :view-context :page} :properties {:logseq.property/description "Runs after sandbox startup is ready. Store setup commands in a code block, e.g. `yarn install`."}} + :logseq.property/project.docker-file + {:title "Project Dockerfile" + :schema {:type :default + :public? true + :view-context :page} + :properties {:logseq.property/description "Store Dockerfile content in a code block. Used to build the sandbox image before startup."}} :logseq.property/pr {:title "PR" :schema {:type :url diff --git a/deps/db/src/logseq/db/frontend/schema.cljs b/deps/db/src/logseq/db/frontend/schema.cljs index 5a095064e4..08d4dab70e 100644 --- a/deps/db/src/logseq/db/frontend/schema.cljs +++ b/deps/db/src/logseq/db/frontend/schema.cljs @@ -30,7 +30,7 @@ (map (juxt :major :minor) [(parse-schema-version x) (parse-schema-version y)]))) -(def version (parse-schema-version "65.28")) +(def version (parse-schema-version "65.29")) (defn major-version "Return a number. diff --git a/deps/db/test/logseq/db/frontend/property_test.cljs b/deps/db/test/logseq/db/frontend/property_test.cljs index 71db03eb6f..4b343f78e3 100644 --- a/deps/db/test/logseq/db/frontend/property_test.cljs +++ b/deps/db/test/logseq/db/frontend/property_test.cljs @@ -63,3 +63,13 @@ (testing "logseq property namespace" (is (db-property/logseq-property? :logseq.property.reaction/emoji-id)) (is (db-property/logseq-property? :logseq.property.reaction/target))))) + +(deftest project-docker-file-built-in-property + (let [props db-property/built-in-properties] + (is (contains? props :logseq.property/project.docker-file)) + (is (= "Project Dockerfile" + (get-in props [:logseq.property/project.docker-file :title]))) + (is (= :default + (get-in props [:logseq.property/project.docker-file :schema :type]))) + (is (= true + (get-in props [:logseq.property/project.docker-file :schema :public?]))))) diff --git a/deps/workers/src/logseq/agents/do.cljs b/deps/workers/src/logseq/agents/do.cljs index bf709a40be..6c8483cb76 100644 --- a/deps/workers/src/logseq/agents/do.cljs +++ b/deps/workers/src/logseq/agents/do.cljs @@ -132,6 +132,30 @@ checkpoint (merge checkpoint (checkpoint-bundle-fields bundle)))) +(def ^:private checkpoint-bundle-ks + [:bundle-id + :bundle-seq + :bundle-object-key + :bundle-byte-size + :bundle-checksum + :bundle-head-sha + :bundle-base-sha + :bundle-head-branch]) + +(defn- strip-checkpoint-bundle + [checkpoint] + (if (map? checkpoint) + (apply dissoc checkpoint checkpoint-bundle-ks) + checkpoint)) + +(defn- runtime-provider-id + [runtime] + (some-> (:provider runtime) str string/lower-case)) + +(defn- workspace-bundle-required? + [runtime] + (not= "e2b" (runtime-provider-id runtime))) + (defn- runtime-checkpoint-payload [runtime result reason] (let [snapshot-id (runtime-snapshot-id result) @@ -423,10 +447,12 @@ (string? reason) (assoc :reason reason)) :ts (common/now-ms)})] (if (map? checkpoint) - (p/let [checkpoint ( {:by by - :reason reason} - (string? head-branch) - (assoc :head-branch head-branch))) + (p/let [checkpoint (if (workspace-bundle-required? (:runtime current-session)) + ( {:by by + :reason reason} + (string? head-branch) + (assoc :head-branch head-branch))) + (p/resolved (strip-checkpoint-bundle checkpoint))) _ ( (checkpoint-payload-with-reason d1-checkpoint "provisioned") + strip-checkpoint-bundle)) runtime-checkpoint (merge-checkpoint-bundle base-checkpoint restored-bundle) session (-> session (assoc :runtime runtime) diff --git a/deps/workers/src/logseq/agents/runtime_provider.cljs b/deps/workers/src/logseq/agents/runtime_provider.cljs index 40d2e41f0d..966818c8d1 100644 --- a/deps/workers/src/logseq/agents/runtime_provider.cljs +++ b/deps/workers/src/logseq/agents/runtime_provider.cljs @@ -949,6 +949,19 @@ (some-> (get-in task [:runtime :template]) str string/trim not-empty) (env-str env "E2B_TEMPLATE"))) +(defn- task-graph-id + [task] + (some-> (get-in task [:project :graph-id]) str string/trim not-empty)) + +(defn- project-docker-file + [task] + (some-> (get-in task [:project :docker-file]) str string/trim not-empty)) + +(defn- e2b-template-name + [task] + (when-let [graph-id (task-graph-id task)] + (sanitize-name (str "logseq-" graph-id "-" (task-repo-name task))))) + (defn- e2b-agent-token [^js env runtime] (or (:agent-token runtime) @@ -1163,6 +1176,41 @@ (seq env-vars) (assoc :envs env-vars) (string? session-id) (assoc :metadata metadata)))) +(defn- e2b-template-fn + [] + (aget e2b "Template")) + +(defn- e2b-template-missing-error? + [error] + (let [message (some-> error str)] + (boolean (and (string? message) + (string/starts-with? message "Error: 404:"))))) + +(defn- (->promise (.call get-tags template-fn template-name (clj->js (e2b-api-opts env)))) + (p/then (fn [_tags] template-name)) + (p/catch (fn [error] + (if (e2b-template-missing-error? error) + nil + (throw error))))))) + +(defn- (.put (.-storage self) "session" (clj->js session)) + (.then (fn [_] + (with-redefs [checkpoint-store/clj session-js :keywordize-keys true) + checkpoint (get-in stored [:task :sandbox-checkpoint])] + (is (= "e2b-snapshot-1" (:snapshot-id checkpoint))) + (is (= "e2b" (:provider checkpoint))) + (is (nil? (:bundle-id checkpoint))) + (is (nil? (:bundle-object-key checkpoint))) + (done)))))) + (.catch (fn [error] + (is false (str "unexpected e2b checkpoint error: " error)) + (done)))))))) + (deftest provision-runtime-restores-workspace-bundle-test (testing "provision runtime applies latest workspace bundle and stores bundle metadata in checkpoint" (async done @@ -814,6 +875,86 @@ (is false (str "unexpected restore bundle error: " error)) (done)))))))) +(deftest provision-runtime-skips-workspace-bundle-restore-for-e2b-test + (testing "e2b provision should not restore workspace bundle metadata from R2" + (async done + (let [env #js {"AGENT_RUNTIME_PROVIDER" "e2b" + "AGENTS_DB" #js {}} + self (make-self env) + task {:id "sess-restore-e2b" + :agent "codex" + :project {:repo-url "https://github.com/logseq/logseq" + :base-branch "main"}} + runtime {:provider "e2b" + :session-id "runtime-restore-e2b" + :sandbox-id "sbx-restore-e2b" + :snapshot-id "e2b-snapshot-restored"} + get-bundle-calls (atom 0) + apply-calls (atom 0) + provider (reify runtime-provider/RuntimeProvider + ( (.put (.-storage self) + "session" + (clj->js {:id "sess-restore-e2b" + :status "running" + :task task + :audit {} + :created-at 0 + :updated-at 0})) + (.then (fn [_] + (with-redefs [runtime-provider/resolve-provider (fn [_env _runtime] provider) + runtime-provider/provider-id (fn [_provider] "e2b") + agent-do/start-runtime-events-stream-background! (fn [& _] nil) + common/clj session-js :keywordize-keys true) + checkpoint (get-in stored [:task :sandbox-checkpoint])] + (is (= "e2b-snapshot-restored" (:snapshot-id checkpoint))) + (is (= "e2b" (:provider checkpoint))) + (is (nil? (:bundle-id checkpoint))) + (is (nil? (:bundle-object-key checkpoint))) + (done)))))) + (.catch (fn [error] + (is false (str "unexpected e2b restore provision error: " error)) + (done)))))))) + (deftest restore-workspace-bundle-skips-when-session-changed-test (testing "bundle apply should be skipped when expected session is no longer current" (async done diff --git a/deps/workers/test/logseq/agents/e2b_runtime_provider_test.cljs b/deps/workers/test/logseq/agents/e2b_runtime_provider_test.cljs index 144456174c..c473c8b3cc 100644 --- a/deps/workers/test/logseq/agents/e2b_runtime_provider_test.cljs +++ b/deps/workers/test/logseq/agents/e2b_runtime_provider_test.cljs @@ -141,6 +141,146 @@ (is false (str "unexpected error: " error)) (done))))))) +(deftest e2b-provider-provision-uses-existing-template-from-project-docker-file-test + (async done + (let [calls (atom []) + env #js {"E2B_API_KEY" "e2b-key" + "SANDBOX_AGENT_TOKEN" "agent-token" + "E2B_TEMPLATE" "fallback-template"} + provider (runtime-provider/create-provider env "e2b") + task {:agent {:provider "codex"} + :project {:repo-url "https://github.com/logseq/agent-test" + :graph-id "graph-123" + :docker-file "FROM node:20\nRUN corepack enable"}} + e2b-ns (js/require "e2b") + original-template (aget e2b-ns "Template") + sandbox-class (runtime-provider/e2b-sandbox-class) + original-create (aget sandbox-class "create") + original-fetch js/fetch + restore! (fn [] + (aset e2b-ns "Template" original-template) + (aset sandbox-class "create" original-create) + (set! js/fetch original-fetch)) + template-fn (fn [] #js {})] + (aset template-fn "getTags" + (fn [name opts] + (swap! calls conj {:type :get-tags + :name name + :opts (js->clj opts :keywordize-keys true)}) + (js/Promise.resolve #js []))) + (aset e2b-ns "Template" template-fn) + (aset sandbox-class "create" + (fn [& args] + (let [template (first args) + opts (js->clj (last args) :keywordize-keys true)] + (swap! calls conj {:type :create + :template template + :opts opts}) + (js/Promise.resolve + #js {:sandboxId "e2b-sbx-docker" + :getHost (fn [_port] + "https://e2b-agent.local") + :commands + #js {:run (fn [cmd _opts] + (if (string/includes? cmd "/v1/health") + (js/Promise.resolve #js {:stdout "__HEALTH_OK__" + :stderr "" + :exitCode 0}) + (js/Promise.resolve #js {:stdout "" + :stderr "" + :exitCode 0})))}})))) + (set! js/fetch + (fn [_request] + (js/Promise.resolve + (js/Response. + (js/JSON.stringify #js {:ok true}) + #js {:status 200 + :headers #js {"content-type" "application/json"}})))) + (-> (runtime-provider/clj (last args) :keywordize-keys true)}) + (js/Promise.resolve + #js {:sandboxId "e2b-sbx-docker-default" + :getHost (fn [_port] + "https://e2b-agent.local") + :commands + #js {:run (fn [cmd _opts] + (if (string/includes? cmd "/v1/health") + (js/Promise.resolve #js {:stdout "__HEALTH_OK__" + :stderr "" + :exitCode 0}) + (js/Promise.resolve #js {:stdout "" + :stderr "" + :exitCode 0})))}}))) + (set! js/fetch + (fn [_request] + (js/Promise.resolve + (js/Response. + (js/JSON.stringify #js {:ok true}) + #js {:status 200 + :headers #js {"content-type" "application/json"}})))) + (-> (runtime-provider/nil (or (pu/get-block-property-value project-page :logseq.property/git-repo) (:logseq.property/git-repo project-page))) + docker-file (blank->nil (or (pu/get-block-property-value project-page :logseq.property/project.docker-file) + (:logseq.property/project.docker-file project-page))) sandbox-init-setup (blank->nil (or (pu/get-block-property-value project-page :logseq.property/project-sandbox-init-setup) (:logseq.property/project-sandbox-init-setup project-page))) project-id (some-> (:block/uuid project-page) str) + graph-id (some-> (ldb/get-graph-rtc-uuid (db/get-db)) str blank->nil) title (blank->nil (:block/title project-page)) base-branch (blank->nil base-branch)] (when (and project-id title repo-url) @@ -115,6 +119,8 @@ :title title :repo-url repo-url} (string? base-branch) (assoc :base-branch base-branch) + (string? graph-id) (assoc :graph-id graph-id) + (string? docker-file) (assoc :docker-file docker-file) (string? sandbox-init-setup) (assoc :sandbox-init-setup sandbox-init-setup)))))) (defn- block-line-content diff --git a/src/main/frontend/worker/db/migrate.cljs b/src/main/frontend/worker/db/migrate.cljs index 9bd2ccab5d..ca54be19d0 100644 --- a/src/main/frontend/worker/db/migrate.cljs +++ b/src/main/frontend/worker/db/migrate.cljs @@ -86,7 +86,8 @@ :logseq.property/agent-auth-json]}] ["65.25" {:properties [:logseq.property/pr]}] ["65.27" {:properties [:logseq.property/agent-session-id]}] - ["65.28" {:properties [:logseq.property/sandbox-checkpoint]}]]) + ["65.28" {:properties [:logseq.property/sandbox-checkpoint]}] + ["65.29" {:properties [:logseq.property/project.docker-file]}]]) (let [[major minor] (last (sort (map (comp (juxt :major :minor) db-schema/parse-schema-version first) schema-version->updates)))] diff --git a/src/test/frontend/worker/migrate_test.cljs b/src/test/frontend/worker/migrate_test.cljs index ce1bbcfa21..0b1720e04b 100644 --- a/src/test/frontend/worker/migrate_test.cljs +++ b/src/test/frontend/worker/migrate_test.cljs @@ -82,3 +82,18 @@ (:kv/value (d/entity @conn :logseq.kv/schema-version)))) (is (some? property)) (is (= :map (:logseq.property/type property))))) + +(deftest migrate-adds-project-docker-file-property-builtin + (let [conn (db-test/create-conn) + property-ident :logseq.property/project.docker-file + _ (d/transact! conn [{:db/ident :logseq.kv/schema-version + :kv/value {:major 65 :minor 28}}]) + existing-eid (d/entid @conn property-ident) + _ (when existing-eid + (d/transact! conn [[:db/retractEntity existing-eid]])) + _ (db-migrate/migrate conn :target-version "65.29") + property (d/entity @conn property-ident)] + (is (= {:major 65 :minor 29} + (:kv/value (d/entity @conn :logseq.kv/schema-version)))) + (is (some? property)) + (is (= :default (:logseq.property/type property)))))