diff --git a/deps/workers/README.md b/deps/workers/README.md index 90d919a68b..25ef9f9b95 100644 --- a/deps/workers/README.md +++ b/deps/workers/README.md @@ -180,7 +180,10 @@ Cloudflare runtime flow: | CLOUDFLARE_REPO_CLONE_COMMAND | Optional repo clone command template for Cloudflare sandbox | | CLOUDFLARE_HEALTH_RETRIES | Cloudflare sandbox health check retry count | | CLOUDFLARE_HEALTH_INTERVAL_MS | Cloudflare sandbox health check retry interval (ms) | -| GITHUB_TOKEN | Fallback token used for both git push and PR API calls | +| GITHUB_APP_ID | GitHub App ID used to mint installation tokens | +| GITHUB_APP_INSTALLATION_ID | Optional fixed installation ID (if omitted, resolved from repo) | +| GITHUB_APP_PRIVATE_KEY | GitHub App private key PEM used for JWT signing | +| GITHUB_APP_SLUG | Optional app slug used to build install URL in setup prompts | | GITHUB_API_BASE | Optional GitHub API base URL override (default `https://api.github.com`) | | OPENAI_API_KEY | Passed into Cloudflare sandbox runtime env (if set) | | ANTHROPIC_API_KEY | Passed into Cloudflare sandbox runtime env (if set) | @@ -197,15 +200,24 @@ This endpoint is available to any authenticated collaborator and supports: - push only (`{"create-pr": false}`) - push + PR (`{"create-pr": true}` or omitted) +When creating a session, the worker verifies the GitHub App is installed on the target repo. +If not installed, `POST /sessions` returns `412` with an install prompt message. + +Agent chat messages can also trigger the same publish flow: +- `push` +- `submit PR` +- natural language requests like `please push this branch` or `can you submit a pull request?` + Response `status` values: - `pushed` - `pr-created` - `manual-pr-required` -If PR credentials are missing or PR API creation fails after a successful push, response includes `manual-pr-url`. +If PR API creation fails after a successful push, response includes `manual-pr-url`. For Cloudflare deploys, store tokens as agents worker secrets: -- `wrangler secret put GITHUB_TOKEN -c worker/wrangler.agents.toml --env ` +- `wrangler secret put GITHUB_APP_PRIVATE_KEY -c worker/wrangler.agents.toml --env ` +- set `GITHUB_APP_ID` and optional `GITHUB_APP_INSTALLATION_ID` in worker vars ## Notes - Protocol definitions live in `docs/agent-guide/db-sync/protocol.md`. diff --git a/deps/workers/src/logseq/agents/do.cljs b/deps/workers/src/logseq/agents/do.cljs index e62d0a62c8..5fcfe58903 100644 --- a/deps/workers/src/logseq/agents/do.cljs +++ b/deps/workers/src/logseq/agents/do.cljs @@ -267,21 +267,17 @@ [] "main") -(defn- github-default-branch-token - [^js env] - (or (source-control/push-token env) - (source-control/pr-token env))) - -(defn- github-branches-token - [^js env] - (or (source-control/push-token env) - (source-control/pr-token env))) - (defn- task-requested-base-branch [task] (or (some-> (get-in task [:project :base-branch]) source-control/sanitize-branch-name) (some-> (get-in task [:project :branch]) source-control/sanitize-branch-name))) +(defn- github-install-required-message + [install-url] + (if (string? install-url) + (str "GitHub App is not installed for this repository. Install it and retry: " install-url) + "GitHub App is not installed for this repository. Install it and retry.")) + (defn- (get-in task [:project :repo-url]) str string/trim not-empty) @@ -301,7 +297,7 @@ :else (p/let [detected-base (source-control/ (get-in task [:project :repo-url]) str string/trim not-empty) + install-status (-> (if (string? repo-url) + (source-control/ (:body body) str string/trim not-empty) - "Automated changes from agent session.") - title (or (some-> (:title body) str string/trim not-empty) - (description->pr-title description) - (str "Agent updates for session " (:id current-session))) - manual-url (source-control/manual-pr-url repo-url head-branch base-branch)] - (if-not (string? pr-token) - ( (:body body) str string/trim not-empty) + "Automated changes from agent session.") + title (or (some-> (:title body) str string/trim not-empty) + (description->pr-title description) + (str "Agent updates for session " (:id current-session))) + manual-url (source-control/manual-pr-url repo-url head-branch base-branch)] ( (:commit-message body) str string/trim not-empty) head-branch (source-control/resolve-head-branch (:head-branch body) (generated-head-branch current-session body))] (if-not (string? head-branch) (http/bad-request "invalid head branch") - (-> (p/let [_ ( (p/let [push-token (source-control/> branches diff --git a/deps/workers/src/logseq/agents/source_control.cljs b/deps/workers/src/logseq/agents/source_control.cljs index c23b123255..987645887c 100644 --- a/deps/workers/src/logseq/agents/source_control.cljs +++ b/deps/workers/src/logseq/agents/source_control.cljs @@ -98,21 +98,154 @@ (or (some-> (aget env "GITHUB_USER_AGENT") non-empty-str) "logseq-worker/1.0")) -(defn pr-token - [^js env] - (some-> (aget env "GITHUB_TOKEN") non-empty-str)) +(def ^:private github-api-version "2022-11-28") +(def ^:private token-refresh-skew-ms 60000) +(defonce ^:private installation-token-cache (atom {})) -(defn push-token - [^js env] - (some-> (aget env "GITHUB_TOKEN") non-empty-str)) +(defn- now-ms [] + (.now js/Date)) -(defn push-remote-url - [repo-url token] - (when-let [{:keys [provider owner name]} (repo-ref repo-url)] - (when (= "github" provider) - (if (string? token) - (str "https://x-access-token:" (js/encodeURIComponent token) "@github.com/" owner "/" name ".git") - (str "https://github.com/" owner "/" name ".git"))))) +(defn- parse-int-safe + [value] + (let [num (some-> value js/parseInt)] + (when (and (number? num) + (not (js/isNaN num))) + num))) + +(defn- parse-time-ms + [value] + (let [ms (some-> value js/Date.parse)] + (when (and (number? ms) + (not (js/isNaN ms))) + ms))) + +(defn- base64url + [value] + (-> (or value "") + (string/replace #"\+" "-") + (string/replace #"/" "_") + (string/replace #"=+$" ""))) + +(defn- uint8->base64url + [payload] + (base64url (js/btoa (apply str (map char payload))))) + +(defn- text->base64url + [text] + (-> (.encode (js/TextEncoder.) (or text "")) + uint8->base64url)) + +(defn- json->base64url + [payload] + (text->base64url (js/JSON.stringify (clj->js payload)))) + +(defn- github-app-id + [^js env] + (some-> (aget env "GITHUB_APP_ID") non-empty-str)) + +(defn- github-app-installation-id + [^js env] + (some-> (aget env "GITHUB_APP_INSTALLATION_ID") + non-empty-str + parse-int-safe)) + +(defn- github-app-private-key + [^js env] + (some-> (aget env "GITHUB_APP_PRIVATE_KEY") + non-empty-str + (string/replace #"\\n" "\n"))) + +(defn- github-app-slug + [^js env] + (or (some-> (aget env "GITHUB_APP_SLUG") non-empty-str) + (some-> (aget env "GITHUB_APP_NAME") non-empty-str))) + +(defn- node-subtle + [] + (let [require-fn (aget js/globalThis "require")] + (when (fn? require-fn) + (let [crypto-module (.call require-fn nil "node:crypto") + webcrypto (when crypto-module + (aget crypto-module "webcrypto"))] + (when webcrypto + (aget webcrypto "subtle")))))) + +(defn- subtle-crypto + [] + (or (some-> (aget js/globalThis "crypto") + .-subtle) + (node-subtle))) + +(defn- pem->array-buffer + [pem] + (let [body (-> (or pem "") + (string/replace #"-+BEGIN[^-]+-+" "") + (string/replace #"-+END[^-]+-+" "") + (string/replace #"\s+" "")) + binary (js/atob body) + length (.-length binary) + payload (js/Uint8Array. length)] + (dotimes [idx length] + (aset payload idx (.charCodeAt binary idx))) + (.-buffer payload))) + +(defn- array-buffer private-key) + #js {:name "RSASSA-PKCS1-v1_5" + :hash #js {:name "SHA-256"}} + false + #js ["sign"]) + (p/rejected (ex-info "webcrypto subtle unavailable" + {:reason :crypto-unavailable})))) + +(defn- base64url {:alg "RS256" + :typ "JWT"}) + claims (json->base64url {:iat (- issued-at 60) + :exp (+ issued-at 540) + :iss app-id}) + unsigned-token (str header "." claims) + encoded-token (.encode (js/TextEncoder.) unsigned-token)] + (p/let [private-key-obj ( (js/Uint8Array. signature) + uint8->base64url)] + (str unsigned-token "." signature-token)))))) + +(defn- request-headers + [^js env {:keys [token content-type]}] + (let [headers (js/Headers.)] + (.set headers "accept" "application/vnd.github+json") + (.set headers "user-agent" (user-agent env)) + (.set headers "x-github-api-version" github-api-version) + (when-let [auth-token (non-empty-str token)] + (.set headers "authorization" (str "Bearer " auth-token))) + (when-let [value (non-empty-str content-type)] + (.set headers "content-type" value)) + headers)) (defn- parse-json-safe [text] @@ -144,18 +277,182 @@ {} github-debug-response-headers)) +(defn- fetch-json-safe! + [url request] + (p/let [resp (js/fetch url request) + status (.-status resp) + text (.text resp) + payload (parse-json-safe text) + response-headers (select-response-headers resp)] + {:status status + :ok? (<= 200 status 299) + :payload payload + :raw-body text + :response-headers response-headers})) + +(defn- token-cache-key + [^js env repo-url installation-id] + (str (api-base-url env) "|" (or installation-id repo-url))) + +(defn- cached-installation-token + [cache-key] + (let [{:keys [token expires-at-ms]} (get @installation-token-cache cache-key)] + (when (and (string? token) + (number? expires-at-ms) + (> (- expires-at-ms (now-ms)) token-refresh-skew-ms)) + token))) + +(defn- (:id payload) parse-int-safe)] + (if (and ok? (number? installation-id)) + installation-id + (p/rejected (ex-info "resolve installation failed" + {:reason :installation-resolution-failed + :status status + :body payload + :raw-body raw-body + :response-headers response-headers})))))) + (p/rejected (ex-info "invalid repo url" + {:reason :invalid-repo-url + :repo-url repo-url}))))) + +(defn (:expires_at payload) parse-time-ms)] + (if (and ok? + (string? token) + (number? expires-at-ms)) + (do + (swap! installation-token-cache + assoc + cache-key + {:token token + :expires-at-ms expires-at-ms}) + token) + (p/rejected (ex-info "create installation token failed" + {:reason :token-mint-failed + :status status + :body payload + :raw-body raw-body + :response-headers response-headers}))))))))))) + +(defn app-install-url + [^js env repo-url] + (when-let [{:keys [provider]} (repo-ref repo-url)] + (when (= "github" provider) + (some-> (github-app-slug env) + (#(str "https://github.com/apps/" % "/installations/new")))))) + +(defn (:id payload) parse-int-safe)] + (cond + (and ok? (number? installation-id)) + {:provider provider + :installed? true + :check-required? true + :installation-id installation-id} + + (= status 404) + {:provider provider + :installed? false + :check-required? true + :reason :not-installed + :install-url (app-install-url env repo-url)} + + :else + (p/rejected (ex-info "check github app installation failed" + {:reason :installation-check-failed + :status status + :body payload + :raw-body raw-body + :response-headers response-headers}))))) + (p/resolved {:provider nil + :installed? true + :check-required? false}))) + +(defn pr-token + [_env] + nil) + +(defn push-token + [_env] + nil) + +(defn js payload))}) @@ -257,11 +545,7 @@ (string? base-branch) (str "&base=" (js/encodeURIComponent base-branch))) url (str (api-base-url env) "/repos/" owner "/" name "/pulls?" query) - headers (doto (js/Headers.) - (.set "accept" "application/vnd.github+json") - (.set "authorization" (str "Bearer " token)) - (.set "user-agent" (user-agent env)) - (.set "x-github-api-version" "2022-11-28"))] + headers (request-headers env {:token token})] (p/let [resp (js/fetch url #js {:method "GET" :headers headers}) status (.-status resp) text (.text resp) diff --git a/deps/workers/src/logseq/sync/node/config.cljs b/deps/workers/src/logseq/sync/node/config.cljs index eff4f86317..d8e09caa99 100644 --- a/deps/workers/src/logseq/sync/node/config.cljs +++ b/deps/workers/src/logseq/sync/node/config.cljs @@ -44,7 +44,10 @@ :cloudflare-repo-clone-command (env-value env "CLOUDFLARE_REPO_CLONE_COMMAND") :cloudflare-health-retries (env-value env "CLOUDFLARE_HEALTH_RETRIES") :cloudflare-health-interval-ms (env-value env "CLOUDFLARE_HEALTH_INTERVAL_MS") - :github-token (env-value env "GITHUB_TOKEN") + :github-app-id (env-value env "GITHUB_APP_ID") + :github-app-installation-id (env-value env "GITHUB_APP_INSTALLATION_ID") + :github-app-private-key (env-value env "GITHUB_APP_PRIVATE_KEY") + :github-app-slug (env-value env "GITHUB_APP_SLUG") :github-api-base (env-value env "GITHUB_API_BASE") :openai-api-key (env-value env "OPENAI_API_KEY") :anthropic-api-key (env-value env "ANTHROPIC_API_KEY") @@ -66,7 +69,7 @@ :cloudflare-sandbox-name-prefix :cloudflare-sandbox-agent-port :cloudflare-bootstrap-command :cloudflare-repo-clone-command :cloudflare-health-retries :cloudflare-health-interval-ms - :github-token :github-api-base + :github-app-id :github-app-installation-id :github-app-private-key :github-app-slug :github-api-base :openai-api-key :anthropic-api-key :openai-base-url :anthropic-base-url :log-level :cognito-issuer :cognito-client-id :cognito-jwks-url]) diff --git a/deps/workers/src/logseq/sync/node/server.cljs b/deps/workers/src/logseq/sync/node/server.cljs index c0f44473b7..26eaf82309 100644 --- a/deps/workers/src/logseq/sync/node/server.cljs +++ b/deps/workers/src/logseq/sync/node/server.cljs @@ -53,7 +53,10 @@ (aset "CLOUDFLARE_REPO_CLONE_COMMAND" (:cloudflare-repo-clone-command cfg)) (aset "CLOUDFLARE_HEALTH_RETRIES" (:cloudflare-health-retries cfg)) (aset "CLOUDFLARE_HEALTH_INTERVAL_MS" (:cloudflare-health-interval-ms cfg)) - (aset "GITHUB_TOKEN" (:github-token cfg)) + (aset "GITHUB_APP_ID" (:github-app-id cfg)) + (aset "GITHUB_APP_INSTALLATION_ID" (:github-app-installation-id cfg)) + (aset "GITHUB_APP_PRIVATE_KEY" (:github-app-private-key cfg)) + (aset "GITHUB_APP_SLUG" (:github-app-slug cfg)) (aset "GITHUB_API_BASE" (:github-api-base cfg)) (aset "OPENAI_API_KEY" (:openai-api-key cfg)) (aset "ANTHROPIC_API_KEY" (:anthropic-api-key cfg)) diff --git a/deps/workers/test/logseq/agents/do_test.cljs b/deps/workers/test/logseq/agents/do_test.cljs index cc5d044e23..1813a88700 100644 --- a/deps/workers/test/logseq/agents/do_test.cljs +++ b/deps/workers/test/logseq/agents/do_test.cljs @@ -844,8 +844,6 @@ :repo-url (:repo-url opts) :force (:force opts) :remote "origin"})) - source-control/pr-token - (fn [_env] "token-1") source-control/ (when (string? text) + (re-find #"https?://[^\s)\]}]+" text)) + (string/replace #"[,.;:!?]+$" ""))) + +(defn- start-session-error-message + [error] + (or (some-> (ex-data error) :body :error) + (some-> error ex-message) + "Failed to start agent session.")) + +(defn- start-session-install-url + [error] + (first-url-in-text (some-> (ex-data error) :body :error))) + +(defn- open-external-url! + [url] + (when (string? url) + (if (util/electron?) + (js/window.apis.openExternal url) + (js/window.open url "_blank" "noopener,noreferrer")))) + +(defn- show-github-install-required-notification! + [error retry-fn] + (let [message (start-session-error-message error) + install-url (start-session-install-url error) + can-retry? (fn? retry-fn)] + (notification/show! + [:div.space-y-2 + [:div.whitespace-pre-line message] + [:div.flex.flex-wrap.gap-2 + (when (string? install-url) + (shui/button + {:size :sm + :on-click (fn [] + (open-external-url! install-url))} + "Install GitHub App")) + (when can-retry? + (shui/button + {:variant :outline + :size :sm + :on-click (fn [] + (retry-fn))} + "retry"))]] + :warning false github-install-required-notification-uid))) + (defn- normalize-branches [branches] (->> branches @@ -536,32 +586,42 @@ (do (notification/show! "Invalid agent session payload." :error false) nil) - (p/let [resp (db-sync/fetch-json (str base "/sessions") - {:method "POST" - :headers {"content-type" "application/json"} - :body (js/JSON.stringify (clj->js body))} - {:response-schema :sessions/create}) - session-id (:session-id resp) - status (:status resp) - stream-url (:stream-url resp) - block-uuid (:block/uuid block) - _ (when-let [raw-message (message-body (:content raw-body))] - (let [coerced (coerce-http-request :sessions/message raw-message) - msg-body (if (map? coerced) coerced raw-message)] - (db-sync/fetch-json (str base "/sessions/" session-id "/messages") - {:method "POST" - :headers {"content-type" "application/json"} - :body (js/JSON.stringify (clj->js msg-body))} - {:response-schema :sessions/message})))] - (update-session-state! block-uuid {:session-id session-id - :status status - :runtime-provider (:runtime-provider resp) - :terminal-enabled (true? (:terminal-enabled resp)) - :stream-url stream-url - :started-at (util/time-ms)}) - (mark-task-session-created! block-uuid) - ( (p/let [resp (db-sync/fetch-json (str base "/sessions") + {:method "POST" + :headers {"content-type" "application/json"} + :body (js/JSON.stringify (clj->js body))} + {:response-schema :sessions/create}) + session-id (:session-id resp) + status (:status resp) + stream-url (:stream-url resp) + block-uuid (:block/uuid block) + _ (when-let [raw-message (message-body (:content raw-body))] + (let [coerced (coerce-http-request :sessions/message raw-message) + msg-body (if (map? coerced) coerced raw-message)] + (db-sync/fetch-json (str base "/sessions/" session-id "/messages") + {:method "POST" + :headers {"content-type" "application/json"} + :body (js/JSON.stringify (clj->js msg-body))} + {:response-schema :sessions/message})))] + (notification/clear! github-install-required-notification-uid) + (update-session-state! block-uuid {:session-id session-id + :status status + :runtime-provider (:runtime-provider resp) + :terminal-enabled (true? (:terminal-enabled resp)) + :stream-url stream-url + :started-at (util/time-ms)}) + (mark-task-session-created! block-uuid) + ( (