From 014f19af5f68e819e987a71f0d33bdfe5ee20a59 Mon Sep 17 00:00:00 2001 From: Channing Conger Date: Fri, 22 May 2026 09:42:08 -0700 Subject: [PATCH] ci: Use codex produced v8 artifacts for release builds (#23934) Updates our build script to pull down the artifacts like we do in CI for building v8 into our targets. This changes the flow so that we now pre-install rusty v8 assets for all of our release targets from pre-built in workflow. Secondarily if running it locally we now optionally pull the assets down on python run assuming the user hasn't set the proper values, it then provides them. Sorry for the miss here. --- .../action.yml | 23 +-- .github/workflows/rust-ci-full.yml | 6 +- .github/workflows/rust-release.yml | 5 +- scripts/codex_package/README.md | 7 + scripts/codex_package/cargo.py | 14 +- scripts/codex_package/v8.py | 173 ++++++++++++++++++ third_party/v8/README.md | 10 +- 7 files changed, 212 insertions(+), 26 deletions(-) rename .github/actions/{setup-rusty-v8-musl => setup-rusty-v8}/action.yml (71%) create mode 100644 scripts/codex_package/v8.py diff --git a/.github/actions/setup-rusty-v8-musl/action.yml b/.github/actions/setup-rusty-v8/action.yml similarity index 71% rename from .github/actions/setup-rusty-v8-musl/action.yml rename to .github/actions/setup-rusty-v8/action.yml index fbec1feb46..d9c4484657 100644 --- a/.github/actions/setup-rusty-v8-musl/action.yml +++ b/.github/actions/setup-rusty-v8/action.yml @@ -1,29 +1,20 @@ -name: setup-rusty-v8-musl -description: Download and verify musl rusty_v8 artifacts for Cargo builds. +name: setup-rusty-v8 +description: Download and verify Codex-built rusty_v8 artifacts for Cargo builds. inputs: target: - description: Rust musl target triple. + description: Rust target triple with Codex-built V8 release artifacts. required: true runs: using: composite steps: - - name: Configure musl rusty_v8 artifact overrides and verify checksums + - name: Configure rusty_v8 artifact overrides and verify checksums shell: bash env: TARGET: ${{ inputs.target }} run: | set -euo pipefail - case "${TARGET}" in - x86_64-unknown-linux-musl|aarch64-unknown-linux-musl) - ;; - *) - echo "Unsupported musl rusty_v8 target: ${TARGET}" >&2 - exit 1 - ;; - esac - version="$(python3 "${GITHUB_WORKSPACE}/.github/scripts/rusty_v8_bazel.py" resolved-v8-crate-version)" release_tag="rusty-v8-v${version}" base_url="https://github.com/openai/codex/releases/download/${release_tag}" @@ -42,6 +33,10 @@ runs: exit 1 fi - (cd "${binding_dir}" && sha256sum -c "${checksums_path}") + if command -v sha256sum >/dev/null 2>&1; then + (cd "${binding_dir}" && sha256sum -c "${checksums_path}") + else + (cd "${binding_dir}" && shasum -a 256 -c "${checksums_path}") + fi echo "RUSTY_V8_ARCHIVE=${archive_path}" >> "${GITHUB_ENV}" echo "RUSTY_V8_SRC_BINDING_PATH=${binding_path}" >> "${GITHUB_ENV}" diff --git a/.github/workflows/rust-ci-full.yml b/.github/workflows/rust-ci-full.yml index 08e0709e17..b65c2f5d8f 100644 --- a/.github/workflows/rust-ci-full.yml +++ b/.github/workflows/rust-ci-full.yml @@ -436,9 +436,9 @@ jobs: echo "CFLAGS=${cflags}" >> "$GITHUB_ENV" echo "CXXFLAGS=${cxxflags}" >> "$GITHUB_ENV" - - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl' }} - name: Configure musl rusty_v8 artifact overrides and verify checksums - uses: ./.github/actions/setup-rusty-v8-musl + - if: ${{ !contains(matrix.target, 'windows') }} + name: Configure rusty_v8 artifact overrides and verify checksums + uses: ./.github/actions/setup-rusty-v8 with: target: ${{ matrix.target }} diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index 4f10efa9dc..2cd11e66fe 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -340,9 +340,8 @@ jobs: echo "CFLAGS=${cflags}" >> "$GITHUB_ENV" echo "CXXFLAGS=${cxxflags}" >> "$GITHUB_ENV" - - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl' }} - name: Configure musl rusty_v8 artifact overrides and verify checksums - uses: ./.github/actions/setup-rusty-v8-musl + - name: Configure rusty_v8 artifact overrides and verify checksums + uses: ./.github/actions/setup-rusty-v8 with: target: ${{ matrix.target }} diff --git a/scripts/codex_package/README.md b/scripts/codex_package/README.md index 8a1b392b20..af070e2c44 100644 --- a/scripts/codex_package/README.md +++ b/scripts/codex_package/README.md @@ -55,6 +55,13 @@ corresponding resource flags: `--bwrap-bin` for Linux packages, and Windows packages. This keeps package archive creation as a pure staging step after signing instead of rebuilding resources. +When the builder source-builds an entrypoint for a Darwin or Linux target, it +downloads and verifies the matching Codex-built V8 release pair before invoking +Cargo and sets `RUSTY_V8_ARCHIVE` plus `RUSTY_V8_SRC_BINDING_PATH` for that +build. Windows targets keep Cargo's release-build MSVC artifact path. Explicit +overrides remain authoritative when both variables are already set. Set +`V8_FROM_SOURCE=1` to leave the build with the `v8` crate source-build path. + `rg` is not built from this repository, so the builder fetches it from the DotSlash manifest at `scripts/codex_package/rg`. Downloaded archives are cached under `$TMPDIR/codex-package/-rg` and are reused only after the recorded diff --git a/scripts/codex_package/cargo.py b/scripts/codex_package/cargo.py index f7fbf0f9ad..f2238dce53 100644 --- a/scripts/codex_package/cargo.py +++ b/scripts/codex_package/cargo.py @@ -8,6 +8,7 @@ from pathlib import Path from .targets import REPO_ROOT from .targets import PackageVariant from .targets import TargetSpec +from .v8 import resolve_codex_v8_cargo_env CODEX_RS_ROOT = REPO_ROOT / "codex-rs" @@ -60,8 +61,19 @@ def build_source_binaries( for binary in binaries: cmd.extend(["--bin", binary]) + cargo_env = None + if entrypoint_bin is None: + codex_v8_env = resolve_codex_v8_cargo_env(spec) + if codex_v8_env: + cargo_env = {**os.environ, **codex_v8_env} + print("+", " ".join(cmd)) - subprocess.run(cmd, cwd=CODEX_RS_ROOT, check=True) + subprocess.run( + cmd, + cwd=CODEX_RS_ROOT, + check=True, + env=cargo_env, + ) output_dir = cargo_profile_output_dir(spec, profile) outputs = SourceBuildOutputs( diff --git a/scripts/codex_package/v8.py b/scripts/codex_package/v8.py new file mode 100644 index 0000000000..43d0dcb611 --- /dev/null +++ b/scripts/codex_package/v8.py @@ -0,0 +1,173 @@ +"""Codex-built V8 artifact overrides for package Cargo builds.""" + +from __future__ import annotations + +import hashlib +import os +import shutil +import tempfile +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from urllib.request import urlopen + +from .targets import REPO_ROOT +from .targets import TargetSpec + + +DOWNLOAD_TIMEOUT_SECS = 120 + + +@dataclass(frozen=True) +class RustyV8ArtifactPair: + archive: Path + binding: Path + + +def resolve_codex_v8_cargo_env( + spec: TargetSpec, + *, + environ: Mapping[str, str] | None = None, + cache_root: Path | None = None, +) -> dict[str, str]: + if spec.is_windows: + return {} + + environ = os.environ if environ is None else environ + if environ.get("V8_FROM_SOURCE") in {"true", "1", "yes"}: + return {} + + archive_override = environ.get("RUSTY_V8_ARCHIVE") + binding_override = environ.get("RUSTY_V8_SRC_BINDING_PATH") + if archive_override and binding_override: + return {} + if archive_override or binding_override: + raise RuntimeError( + "Cargo package builds need RUSTY_V8_ARCHIVE and " + "RUSTY_V8_SRC_BINDING_PATH set together." + ) + + artifacts = fetch_codex_v8_artifacts(spec, cache_root=cache_root) + return { + "RUSTY_V8_ARCHIVE": str(artifacts.archive), + "RUSTY_V8_SRC_BINDING_PATH": str(artifacts.binding), + } + + +def fetch_codex_v8_artifacts( + spec: TargetSpec, + *, + version: str | None = None, + cache_root: Path | None = None, +) -> RustyV8ArtifactPair: + if spec.is_windows: + raise RuntimeError(f"No Codex-built V8 release artifacts for target: {spec.target}") + + version = version or resolved_v8_crate_version() + release_url = ( + "https://github.com/openai/codex/releases/download/" + f"rusty-v8-v{version}" + ) + target = spec.target + cache_dir = (cache_root or default_cache_root()) / f"rusty-v8-{version}-{target}" + archive = cache_dir / f"librusty_v8_release_{target}.a.gz" + binding = cache_dir / f"src_binding_release_{target}.rs" + checksums = cache_dir / f"rusty_v8_release_{target}.sha256" + + download_file(f"{release_url}/{checksums.name}", checksums) + expected_checksums = load_checksums(checksums, {archive.name, binding.name}) + for artifact in [archive, binding]: + ensure_valid_artifact( + artifact, + expected_checksums[artifact.name], + f"{release_url}/{artifact.name}", + ) + + return RustyV8ArtifactPair(archive=archive, binding=binding) + + +def resolved_v8_crate_version() -> str: + import tomllib + + cargo_lock = tomllib.loads((REPO_ROOT / "codex-rs" / "Cargo.lock").read_text()) + versions = sorted( + { + package["version"] + for package in cargo_lock["package"] + if package["name"] == "v8" + } + ) + if len(versions) != 1: + raise RuntimeError(f"Expected exactly one resolved v8 version, found: {versions}") + return versions[0] + + +def default_cache_root() -> Path: + return Path(tempfile.gettempdir()) / "codex-package" + + +def load_checksums(checksums_path: Path, artifact_names: set[str]) -> dict[str, str]: + checksums: dict[str, str] = {} + lines = checksums_path.read_text(encoding="utf-8").splitlines() + if len(lines) != len(artifact_names): + raise RuntimeError( + f"Expected {len(artifact_names)} V8 checksums in {checksums_path}, " + f"found {len(lines)}." + ) + + for line in lines: + parts = line.split(maxsplit=1) + if len(parts) != 2: + raise RuntimeError(f"Invalid V8 checksum line in {checksums_path}: {line!r}") + + digest, artifact_name = parts[0], parts[1].strip() + if len(digest) != 64 or any(char not in "0123456789abcdef" for char in digest): + raise RuntimeError(f"Invalid V8 checksum digest in {checksums_path}: {digest}") + if artifact_name not in artifact_names: + raise RuntimeError( + f"Unexpected V8 checksum artifact in {checksums_path}: {artifact_name}" + ) + checksums[artifact_name] = digest + + if checksums.keys() != artifact_names: + raise RuntimeError( + f"V8 checksum manifest {checksums_path} does not cover {artifact_names}." + ) + return checksums + + +def ensure_valid_artifact(artifact: Path, checksum: str, url: str) -> None: + if has_checksum(artifact, checksum): + return + + artifact.unlink(missing_ok=True) + download_file(url, artifact) + if has_checksum(artifact, checksum): + return + + artifact.unlink(missing_ok=True) + raise RuntimeError(f"Codex-built V8 artifact {artifact} failed checksum validation.") + + +def has_checksum(path: Path, expected: str) -> bool: + if not path.is_file(): + return False + + digest = hashlib.sha256() + with path.open("rb") as artifact: + for chunk in iter(lambda: artifact.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() == expected + + +def download_file(url: str, dest: Path) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + temp_path = dest.with_suffix(f"{dest.suffix}.tmp") + temp_path.unlink(missing_ok=True) + try: + with urlopen(url, timeout=DOWNLOAD_TIMEOUT_SECS) as response: + with temp_path.open("wb") as output: + shutil.copyfileobj(response, output) + temp_path.replace(dest) + finally: + temp_path.unlink(missing_ok=True) diff --git a/third_party/v8/README.md b/third_party/v8/README.md index 336d03c74b..8b512ecb19 100644 --- a/third_party/v8/README.md +++ b/third_party/v8/README.md @@ -100,11 +100,11 @@ hermetic Windows C++ platform is `windows-gnullvm`/`x86_64-w64-windows-gnu`, so it cannot truthfully reproduce upstream's `*-pc-windows-msvc` archives until we add a real MSVC-targeting C++ toolchain to the Bazel graph. -Cargo musl builds use `RUSTY_V8_ARCHIVE` plus a downloaded -`RUSTY_V8_SRC_BINDING_PATH` to point at those `openai/codex` release assets -directly. We do not use `RUSTY_V8_MIRROR` for musl because the upstream `v8` -crate hardcodes a `v` tag layout, while our musl artifacts are -published under `rusty-v8-v`. +Release and CI Cargo builds for Darwin and Linux use `RUSTY_V8_ARCHIVE` plus a +downloaded `RUSTY_V8_SRC_BINDING_PATH` to point at those `openai/codex` release +assets directly. We do not use `RUSTY_V8_MIRROR` because the upstream `v8` crate +hardcodes a `v` tag layout, while our artifacts are published +under `rusty-v8-v`. Do not mix artifacts across crate versions. The archive and binding must match the exact resolved `v8` crate version in `codex-rs/Cargo.lock`.