ci: Use codex produced v8 artifacts for release builds (#23934)

Updates our build script to pull down the artifacts like we do in CI for
building v8 into our targets.

This changes the flow so that we now pre-install rusty v8 assets for all
of our release targets from pre-built in workflow.
Secondarily if running it locally we now optionally pull the assets down
on python run assuming the user hasn't set the proper values, it then
provides them.

Sorry for the miss here.

.github/actions/setup-rusty-v8/action.yml

@@ -1,29 +1,20 @@
-name: setup-rusty-v8-musl
-description: Download and verify musl rusty_v8 artifacts for Cargo builds.
+name: setup-rusty-v8
+description: Download and verify Codex-built rusty_v8 artifacts for Cargo builds.
 inputs:
   target:
-    description: Rust musl target triple.
+    description: Rust target triple with Codex-built V8 release artifacts.
     required: true
 
 runs:
   using: composite
   steps:
-    - name: Configure musl rusty_v8 artifact overrides and verify checksums
+    - name: Configure rusty_v8 artifact overrides and verify checksums
       shell: bash
       env:
         TARGET: ${{ inputs.target }}
       run: |
         set -euo pipefail
 
-        case "${TARGET}" in
-          x86_64-unknown-linux-musl|aarch64-unknown-linux-musl)
-            ;;
-          *)
-            echo "Unsupported musl rusty_v8 target: ${TARGET}" >&2
-            exit 1
-            ;;
-        esac
-
         version="$(python3 "${GITHUB_WORKSPACE}/.github/scripts/rusty_v8_bazel.py" resolved-v8-crate-version)"
         release_tag="rusty-v8-v${version}"
         base_url="https://github.com/openai/codex/releases/download/${release_tag}"
@@ -42,6 +33,10 @@ runs:
           exit 1
         fi
 
-        (cd "${binding_dir}" && sha256sum -c "${checksums_path}")
+        if command -v sha256sum >/dev/null 2>&1; then
+          (cd "${binding_dir}" && sha256sum -c "${checksums_path}")
+        else
+          (cd "${binding_dir}" && shasum -a 256 -c "${checksums_path}")
+        fi
         echo "RUSTY_V8_ARCHIVE=${archive_path}" >> "${GITHUB_ENV}"
         echo "RUSTY_V8_SRC_BINDING_PATH=${binding_path}" >> "${GITHUB_ENV}"

.github/workflows/rust-ci-full.yml

@@ -436,9 +436,9 @@ jobs:
           echo "CFLAGS=${cflags}" >> "$GITHUB_ENV"
           echo "CXXFLAGS=${cxxflags}" >> "$GITHUB_ENV"
 
-      - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl' }}
-        name: Configure musl rusty_v8 artifact overrides and verify checksums
-        uses: ./.github/actions/setup-rusty-v8-musl
+      - if: ${{ !contains(matrix.target, 'windows') }}
+        name: Configure rusty_v8 artifact overrides and verify checksums
+        uses: ./.github/actions/setup-rusty-v8
         with:
           target: ${{ matrix.target }}
 

.github/workflows/rust-release.yml

@@ -340,9 +340,8 @@ jobs:
           echo "CFLAGS=${cflags}" >> "$GITHUB_ENV"
           echo "CXXFLAGS=${cxxflags}" >> "$GITHUB_ENV"
 
-      - if: ${{ matrix.target == 'x86_64-unknown-linux-musl' || matrix.target == 'aarch64-unknown-linux-musl' }}
-        name: Configure musl rusty_v8 artifact overrides and verify checksums
-        uses: ./.github/actions/setup-rusty-v8-musl
+      - name: Configure rusty_v8 artifact overrides and verify checksums
+        uses: ./.github/actions/setup-rusty-v8
         with:
           target: ${{ matrix.target }}
 

scripts/codex_package/README.md

@@ -55,6 +55,13 @@ corresponding resource flags: `--bwrap-bin` for Linux packages, and
 Windows packages. This keeps package archive creation as a pure staging step
 after signing instead of rebuilding resources.
 
+When the builder source-builds an entrypoint for a Darwin or Linux target, it
+downloads and verifies the matching Codex-built V8 release pair before invoking
+Cargo and sets `RUSTY_V8_ARCHIVE` plus `RUSTY_V8_SRC_BINDING_PATH` for that
+build. Windows targets keep Cargo's release-build MSVC artifact path. Explicit
+overrides remain authoritative when both variables are already set. Set
+`V8_FROM_SOURCE=1` to leave the build with the `v8` crate source-build path.
+
 `rg` is not built from this repository, so the builder fetches it from the
 DotSlash manifest at `scripts/codex_package/rg`. Downloaded archives are cached
 under `$TMPDIR/codex-package/<target>-rg` and are reused only after the recorded

scripts/codex_package/cargo.py

@@ -8,6 +8,7 @@ from pathlib import Path
 from .targets import REPO_ROOT
 from .targets import PackageVariant
 from .targets import TargetSpec
+from .v8 import resolve_codex_v8_cargo_env
 
 
 CODEX_RS_ROOT = REPO_ROOT / "codex-rs"
@@ -60,8 +61,19 @@ def build_source_binaries(
         for binary in binaries:
             cmd.extend(["--bin", binary])
 
+        cargo_env = None
+        if entrypoint_bin is None:
+            codex_v8_env = resolve_codex_v8_cargo_env(spec)
+            if codex_v8_env:
+                cargo_env = {**os.environ, **codex_v8_env}
+
         print("+", " ".join(cmd))
-        subprocess.run(cmd, cwd=CODEX_RS_ROOT, check=True)
+        subprocess.run(
+            cmd,
+            cwd=CODEX_RS_ROOT,
+            check=True,
+            env=cargo_env,
+        )
 
     output_dir = cargo_profile_output_dir(spec, profile)
     outputs = SourceBuildOutputs(

scripts/codex_package/v8.py

@@ -0,0 +1,173 @@
+"""Codex-built V8 artifact overrides for package Cargo builds."""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import shutil
+import tempfile
+from collections.abc import Mapping
+from dataclasses import dataclass
+from pathlib import Path
+from urllib.request import urlopen
+
+from .targets import REPO_ROOT
+from .targets import TargetSpec
+
+
+DOWNLOAD_TIMEOUT_SECS = 120
+
+
+@dataclass(frozen=True)
+class RustyV8ArtifactPair:
+    archive: Path
+    binding: Path
+
+
+def resolve_codex_v8_cargo_env(
+    spec: TargetSpec,
+    *,
+    environ: Mapping[str, str] | None = None,
+    cache_root: Path | None = None,
+) -> dict[str, str]:
+    if spec.is_windows:
+        return {}
+
+    environ = os.environ if environ is None else environ
+    if environ.get("V8_FROM_SOURCE") in {"true", "1", "yes"}:
+        return {}
+
+    archive_override = environ.get("RUSTY_V8_ARCHIVE")
+    binding_override = environ.get("RUSTY_V8_SRC_BINDING_PATH")
+    if archive_override and binding_override:
+        return {}
+    if archive_override or binding_override:
+        raise RuntimeError(
+            "Cargo package builds need RUSTY_V8_ARCHIVE and "
+            "RUSTY_V8_SRC_BINDING_PATH set together."
+        )
+
+    artifacts = fetch_codex_v8_artifacts(spec, cache_root=cache_root)
+    return {
+        "RUSTY_V8_ARCHIVE": str(artifacts.archive),
+        "RUSTY_V8_SRC_BINDING_PATH": str(artifacts.binding),
+    }
+
+
+def fetch_codex_v8_artifacts(
+    spec: TargetSpec,
+    *,
+    version: str | None = None,
+    cache_root: Path | None = None,
+) -> RustyV8ArtifactPair:
+    if spec.is_windows:
+        raise RuntimeError(f"No Codex-built V8 release artifacts for target: {spec.target}")
+
+    version = version or resolved_v8_crate_version()
+    release_url = (
+        "https://github.com/openai/codex/releases/download/"
+        f"rusty-v8-v{version}"
+    )
+    target = spec.target
+    cache_dir = (cache_root or default_cache_root()) / f"rusty-v8-{version}-{target}"
+    archive = cache_dir / f"librusty_v8_release_{target}.a.gz"
+    binding = cache_dir / f"src_binding_release_{target}.rs"
+    checksums = cache_dir / f"rusty_v8_release_{target}.sha256"
+
+    download_file(f"{release_url}/{checksums.name}", checksums)
+    expected_checksums = load_checksums(checksums, {archive.name, binding.name})
+    for artifact in [archive, binding]:
+        ensure_valid_artifact(
+            artifact,
+            expected_checksums[artifact.name],
+            f"{release_url}/{artifact.name}",
+        )
+
+    return RustyV8ArtifactPair(archive=archive, binding=binding)
+
+
+def resolved_v8_crate_version() -> str:
+    import tomllib
+
+    cargo_lock = tomllib.loads((REPO_ROOT / "codex-rs" / "Cargo.lock").read_text())
+    versions = sorted(
+        {
+            package["version"]
+            for package in cargo_lock["package"]
+            if package["name"] == "v8"
+        }
+    )
+    if len(versions) != 1:
+        raise RuntimeError(f"Expected exactly one resolved v8 version, found: {versions}")
+    return versions[0]
+
+
+def default_cache_root() -> Path:
+    return Path(tempfile.gettempdir()) / "codex-package"
+
+
+def load_checksums(checksums_path: Path, artifact_names: set[str]) -> dict[str, str]:
+    checksums: dict[str, str] = {}
+    lines = checksums_path.read_text(encoding="utf-8").splitlines()
+    if len(lines) != len(artifact_names):
+        raise RuntimeError(
+            f"Expected {len(artifact_names)} V8 checksums in {checksums_path}, "
+            f"found {len(lines)}."
+        )
+
+    for line in lines:
+        parts = line.split(maxsplit=1)
+        if len(parts) != 2:
+            raise RuntimeError(f"Invalid V8 checksum line in {checksums_path}: {line!r}")
+
+        digest, artifact_name = parts[0], parts[1].strip()
+        if len(digest) != 64 or any(char not in "0123456789abcdef" for char in digest):
+            raise RuntimeError(f"Invalid V8 checksum digest in {checksums_path}: {digest}")
+        if artifact_name not in artifact_names:
+            raise RuntimeError(
+                f"Unexpected V8 checksum artifact in {checksums_path}: {artifact_name}"
+            )
+        checksums[artifact_name] = digest
+
+    if checksums.keys() != artifact_names:
+        raise RuntimeError(
+            f"V8 checksum manifest {checksums_path} does not cover {artifact_names}."
+        )
+    return checksums
+
+
+def ensure_valid_artifact(artifact: Path, checksum: str, url: str) -> None:
+    if has_checksum(artifact, checksum):
+        return
+
+    artifact.unlink(missing_ok=True)
+    download_file(url, artifact)
+    if has_checksum(artifact, checksum):
+        return
+
+    artifact.unlink(missing_ok=True)
+    raise RuntimeError(f"Codex-built V8 artifact {artifact} failed checksum validation.")
+
+
+def has_checksum(path: Path, expected: str) -> bool:
+    if not path.is_file():
+        return False
+
+    digest = hashlib.sha256()
+    with path.open("rb") as artifact:
+        for chunk in iter(lambda: artifact.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest() == expected
+
+
+def download_file(url: str, dest: Path) -> None:
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    temp_path = dest.with_suffix(f"{dest.suffix}.tmp")
+    temp_path.unlink(missing_ok=True)
+    try:
+        with urlopen(url, timeout=DOWNLOAD_TIMEOUT_SECS) as response:
+            with temp_path.open("wb") as output:
+                shutil.copyfileobj(response, output)
+        temp_path.replace(dest)
+    finally:
+        temp_path.unlink(missing_ok=True)

third_party/v8/README.md

@@ -100,11 +100,11 @@ hermetic Windows C++ platform is `windows-gnullvm`/`x86_64-w64-windows-gnu`, so
 it cannot truthfully reproduce upstream's `*-pc-windows-msvc` archives until we
 add a real MSVC-targeting C++ toolchain to the Bazel graph.
 
-Cargo musl builds use `RUSTY_V8_ARCHIVE` plus a downloaded
-`RUSTY_V8_SRC_BINDING_PATH` to point at those `openai/codex` release assets
-directly. We do not use `RUSTY_V8_MIRROR` for musl because the upstream `v8`
-crate hardcodes a `v<crate_version>` tag layout, while our musl artifacts are
-published under `rusty-v8-v<crate_version>`.
+Release and CI Cargo builds for Darwin and Linux use `RUSTY_V8_ARCHIVE` plus a
+downloaded `RUSTY_V8_SRC_BINDING_PATH` to point at those `openai/codex` release
+assets directly. We do not use `RUSTY_V8_MIRROR` because the upstream `v8` crate
+hardcodes a `v<crate_version>` tag layout, while our artifacts are published
+under `rusty-v8-v<crate_version>`.
 
 Do not mix artifacts across crate versions. The archive and binding must match
 the exact resolved `v8` crate version in `codex-rs/Cargo.lock`.