diff --git a/.github/actions/setup-rusty-v8-musl/action.yml b/.github/actions/setup-rusty-v8/action.yml similarity index 71% rename from .github/actions/setup-rusty-v8-musl/action.yml rename to .github/actions/setup-rusty-v8/action.yml index fbec1feb46..d9c4484657 100644 --- a/.github/actions/setup-rusty-v8-musl/action.yml +++ b/.github/actions/setup-rusty-v8/action.yml @@ -1,29 +1,20 @@ -name: setup-rusty-v8-musl -description: Download and verify musl rusty_v8 artifacts for Cargo builds. +name: setup-rusty-v8 +description: Download and verify Codex-built rusty_v8 artifacts for Cargo builds. inputs: target: - description: Rust musl target triple. + description: Rust target triple with Codex-built V8 release artifacts. required: true runs: using: composite steps: - - name: Configure musl rusty_v8 artifact overrides and verify checksums + - name: Configure rusty_v8 artifact overrides and verify checksums shell: bash env: TARGET: ${{ inputs.target }} run: | set -euo pipefail - case "${TARGET}" in - x86_64-unknown-linux-musl|aarch64-unknown-linux-musl) - ;; - *) - echo "Unsupported musl rusty_v8 target: ${TARGET}" >&2 - exit 1 - ;; - esac - version="$(python3 "${GITHUB_WORKSPACE}/.github/scripts/rusty_v8_bazel.py" resolved-v8-crate-version)" release_tag="rusty-v8-v${version}" base_url="https://github.com/openai/codex/releases/download/${release_tag}" @@ -42,6 +33,10 @@ runs: exit 1 fi - (cd "${binding_dir}" && sha256sum -c "${checksums_path}") + if command -v sha256sum >/dev/null 2>&1; then + (cd "${binding_dir}" && sha256sum -c "${checksums_path}") + else + (cd "${binding_dir}" && shasum -a 256 -c "${checksums_path}") + fi echo "RUSTY_V8_ARCHIVE=${archive_path}" >> "${GITHUB_ENV}" echo "RUSTY_V8_SRC_BINDING_PATH=${binding_path}" >> "${GITHUB_ENV}" diff --git a/.github/workflows/rust-ci-full.yml b/.github/workflows/rust-ci-full.yml index 08e0709e17..b65c2f5d8f 100644 --- a/.github/workflows/rust-ci-full.yml +++ b/.github/workflows/rust-ci-full.yml @@ -436,9 +436,9 @@ jobs: echo "CFLAGS=${cflags}" >> "$GITHUB_ENV" echo "CXXFLAGS=${cxxflags}" >> "$GITHUB_ENV" - - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl' }} - name: Configure musl rusty_v8 artifact overrides and verify checksums - uses: ./.github/actions/setup-rusty-v8-musl + - if: ${{ !contains(matrix.target, 'windows') }} + name: Configure rusty_v8 artifact overrides and verify checksums + uses: ./.github/actions/setup-rusty-v8 with: target: ${{ matrix.target }} diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index 4f10efa9dc..2cd11e66fe 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -340,9 +340,8 @@ jobs: echo "CFLAGS=${cflags}" >> "$GITHUB_ENV" echo "CXXFLAGS=${cxxflags}" >> "$GITHUB_ENV" - - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl' }} - name: Configure musl rusty_v8 artifact overrides and verify checksums - uses: ./.github/actions/setup-rusty-v8-musl + - name: Configure rusty_v8 artifact overrides and verify checksums + uses: ./.github/actions/setup-rusty-v8 with: target: ${{ matrix.target }} diff --git a/scripts/codex_package/README.md b/scripts/codex_package/README.md index 8a1b392b20..af070e2c44 100644 --- a/scripts/codex_package/README.md +++ b/scripts/codex_package/README.md @@ -55,6 +55,13 @@ corresponding resource flags: `--bwrap-bin` for Linux packages, and Windows packages. This keeps package archive creation as a pure staging step after signing instead of rebuilding resources. +When the builder source-builds an entrypoint for a Darwin or Linux target, it +downloads and verifies the matching Codex-built V8 release pair before invoking +Cargo and sets `RUSTY_V8_ARCHIVE` plus `RUSTY_V8_SRC_BINDING_PATH` for that +build. Windows targets keep Cargo's release-build MSVC artifact path. Explicit +overrides remain authoritative when both variables are already set. Set +`V8_FROM_SOURCE=1` to leave the build with the `v8` crate source-build path. + `rg` is not built from this repository, so the builder fetches it from the DotSlash manifest at `scripts/codex_package/rg`. Downloaded archives are cached under `$TMPDIR/codex-package/-rg` and are reused only after the recorded diff --git a/scripts/codex_package/cargo.py b/scripts/codex_package/cargo.py index f7fbf0f9ad..f2238dce53 100644 --- a/scripts/codex_package/cargo.py +++ b/scripts/codex_package/cargo.py @@ -8,6 +8,7 @@ from pathlib import Path from .targets import REPO_ROOT from .targets import PackageVariant from .targets import TargetSpec +from .v8 import resolve_codex_v8_cargo_env CODEX_RS_ROOT = REPO_ROOT / "codex-rs" @@ -60,8 +61,19 @@ def build_source_binaries( for binary in binaries: cmd.extend(["--bin", binary]) + cargo_env = None + if entrypoint_bin is None: + codex_v8_env = resolve_codex_v8_cargo_env(spec) + if codex_v8_env: + cargo_env = {**os.environ, **codex_v8_env} + print("+", " ".join(cmd)) - subprocess.run(cmd, cwd=CODEX_RS_ROOT, check=True) + subprocess.run( + cmd, + cwd=CODEX_RS_ROOT, + check=True, + env=cargo_env, + ) output_dir = cargo_profile_output_dir(spec, profile) outputs = SourceBuildOutputs( diff --git a/scripts/codex_package/v8.py b/scripts/codex_package/v8.py new file mode 100644 index 0000000000..43d0dcb611 --- /dev/null +++ b/scripts/codex_package/v8.py @@ -0,0 +1,173 @@ +"""Codex-built V8 artifact overrides for package Cargo builds.""" + +from __future__ import annotations + +import hashlib +import os +import shutil +import tempfile +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from urllib.request import urlopen + +from .targets import REPO_ROOT +from .targets import TargetSpec + + +DOWNLOAD_TIMEOUT_SECS = 120 + + +@dataclass(frozen=True) +class RustyV8ArtifactPair: + archive: Path + binding: Path + + +def resolve_codex_v8_cargo_env( + spec: TargetSpec, + *, + environ: Mapping[str, str] | None = None, + cache_root: Path | None = None, +) -> dict[str, str]: + if spec.is_windows: + return {} + + environ = os.environ if environ is None else environ + if environ.get("V8_FROM_SOURCE") in {"true", "1", "yes"}: + return {} + + archive_override = environ.get("RUSTY_V8_ARCHIVE") + binding_override = environ.get("RUSTY_V8_SRC_BINDING_PATH") + if archive_override and binding_override: + return {} + if archive_override or binding_override: + raise RuntimeError( + "Cargo package builds need RUSTY_V8_ARCHIVE and " + "RUSTY_V8_SRC_BINDING_PATH set together." + ) + + artifacts = fetch_codex_v8_artifacts(spec, cache_root=cache_root) + return { + "RUSTY_V8_ARCHIVE": str(artifacts.archive), + "RUSTY_V8_SRC_BINDING_PATH": str(artifacts.binding), + } + + +def fetch_codex_v8_artifacts( + spec: TargetSpec, + *, + version: str | None = None, + cache_root: Path | None = None, +) -> RustyV8ArtifactPair: + if spec.is_windows: + raise RuntimeError(f"No Codex-built V8 release artifacts for target: {spec.target}") + + version = version or resolved_v8_crate_version() + release_url = ( + "https://github.com/openai/codex/releases/download/" + f"rusty-v8-v{version}" + ) + target = spec.target + cache_dir = (cache_root or default_cache_root()) / f"rusty-v8-{version}-{target}" + archive = cache_dir / f"librusty_v8_release_{target}.a.gz" + binding = cache_dir / f"src_binding_release_{target}.rs" + checksums = cache_dir / f"rusty_v8_release_{target}.sha256" + + download_file(f"{release_url}/{checksums.name}", checksums) + expected_checksums = load_checksums(checksums, {archive.name, binding.name}) + for artifact in [archive, binding]: + ensure_valid_artifact( + artifact, + expected_checksums[artifact.name], + f"{release_url}/{artifact.name}", + ) + + return RustyV8ArtifactPair(archive=archive, binding=binding) + + +def resolved_v8_crate_version() -> str: + import tomllib + + cargo_lock = tomllib.loads((REPO_ROOT / "codex-rs" / "Cargo.lock").read_text()) + versions = sorted( + { + package["version"] + for package in cargo_lock["package"] + if package["name"] == "v8" + } + ) + if len(versions) != 1: + raise RuntimeError(f"Expected exactly one resolved v8 version, found: {versions}") + return versions[0] + + +def default_cache_root() -> Path: + return Path(tempfile.gettempdir()) / "codex-package" + + +def load_checksums(checksums_path: Path, artifact_names: set[str]) -> dict[str, str]: + checksums: dict[str, str] = {} + lines = checksums_path.read_text(encoding="utf-8").splitlines() + if len(lines) != len(artifact_names): + raise RuntimeError( + f"Expected {len(artifact_names)} V8 checksums in {checksums_path}, " + f"found {len(lines)}." + ) + + for line in lines: + parts = line.split(maxsplit=1) + if len(parts) != 2: + raise RuntimeError(f"Invalid V8 checksum line in {checksums_path}: {line!r}") + + digest, artifact_name = parts[0], parts[1].strip() + if len(digest) != 64 or any(char not in "0123456789abcdef" for char in digest): + raise RuntimeError(f"Invalid V8 checksum digest in {checksums_path}: {digest}") + if artifact_name not in artifact_names: + raise RuntimeError( + f"Unexpected V8 checksum artifact in {checksums_path}: {artifact_name}" + ) + checksums[artifact_name] = digest + + if checksums.keys() != artifact_names: + raise RuntimeError( + f"V8 checksum manifest {checksums_path} does not cover {artifact_names}." + ) + return checksums + + +def ensure_valid_artifact(artifact: Path, checksum: str, url: str) -> None: + if has_checksum(artifact, checksum): + return + + artifact.unlink(missing_ok=True) + download_file(url, artifact) + if has_checksum(artifact, checksum): + return + + artifact.unlink(missing_ok=True) + raise RuntimeError(f"Codex-built V8 artifact {artifact} failed checksum validation.") + + +def has_checksum(path: Path, expected: str) -> bool: + if not path.is_file(): + return False + + digest = hashlib.sha256() + with path.open("rb") as artifact: + for chunk in iter(lambda: artifact.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() == expected + + +def download_file(url: str, dest: Path) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + temp_path = dest.with_suffix(f"{dest.suffix}.tmp") + temp_path.unlink(missing_ok=True) + try: + with urlopen(url, timeout=DOWNLOAD_TIMEOUT_SECS) as response: + with temp_path.open("wb") as output: + shutil.copyfileobj(response, output) + temp_path.replace(dest) + finally: + temp_path.unlink(missing_ok=True) diff --git a/third_party/v8/README.md b/third_party/v8/README.md index 336d03c74b..8b512ecb19 100644 --- a/third_party/v8/README.md +++ b/third_party/v8/README.md @@ -100,11 +100,11 @@ hermetic Windows C++ platform is `windows-gnullvm`/`x86_64-w64-windows-gnu`, so it cannot truthfully reproduce upstream's `*-pc-windows-msvc` archives until we add a real MSVC-targeting C++ toolchain to the Bazel graph. -Cargo musl builds use `RUSTY_V8_ARCHIVE` plus a downloaded -`RUSTY_V8_SRC_BINDING_PATH` to point at those `openai/codex` release assets -directly. We do not use `RUSTY_V8_MIRROR` for musl because the upstream `v8` -crate hardcodes a `v` tag layout, while our musl artifacts are -published under `rusty-v8-v`. +Release and CI Cargo builds for Darwin and Linux use `RUSTY_V8_ARCHIVE` plus a +downloaded `RUSTY_V8_SRC_BINDING_PATH` to point at those `openai/codex` release +assets directly. We do not use `RUSTY_V8_MIRROR` because the upstream `v8` crate +hardcodes a `v` tag layout, while our artifacts are published +under `rusty-v8-v`. Do not mix artifacts across crate versions. The archive and binding must match the exact resolved `v8` crate version in `codex-rs/Cargo.lock`.