diff --git a/.github/scripts/rusty_v8_bazel.py b/.github/scripts/rusty_v8_bazel.py index 712c2e3004..f2712d4a31 100644 --- a/.github/scripts/rusty_v8_bazel.py +++ b/.github/scripts/rusty_v8_bazel.py @@ -29,6 +29,8 @@ MUSL_RUNTIME_ARCHIVE_LABELS = [ ] LLVM_AR_LABEL = "@llvm//tools:llvm-ar" LLVM_RANLIB_LABEL = "@llvm//tools:llvm-ranlib" +RELEASE_ARTIFACT_PROFILE = "release" +SANDBOX_ARTIFACT_PROFILE = "ptrcomp_sandbox_release" def bazel_execroot() -> Path: @@ -126,9 +128,10 @@ def ensure_bazel_output_files( return outputs -def release_pair_label(target: str) -> str: +def release_pair_label(target: str, sandbox: bool = False) -> str: target_suffix = target.replace("-", "_") - return f"//third_party/v8:rusty_v8_release_pair_{target_suffix}" + pair_kind = "sandbox_release_pair" if sandbox else "release_pair" + return f"//third_party/v8:rusty_v8_{pair_kind}_{target_suffix}" def resolved_v8_crate_version() -> str: @@ -180,10 +183,18 @@ def command_manifest_path(manifest: Path | None, version: str) -> Path: return ROOT / manifest -def staged_archive_name(target: str, source_path: Path) -> str: +def staged_archive_name(target: str, source_path: Path, artifact_profile: str) -> str: if source_path.suffix == ".lib": - return f"rusty_v8_release_{target}.lib.gz" - return f"librusty_v8_release_{target}.a.gz" + return f"rusty_v8_{artifact_profile}_{target}.lib.gz" + return f"librusty_v8_{artifact_profile}_{target}.a.gz" + + +def staged_binding_name(target: str, artifact_profile: str) -> str: + return f"src_binding_{artifact_profile}_{target}.rs" + + +def staged_checksums_name(target: str, artifact_profile: str) -> str: + return f"rusty_v8_{artifact_profile}_{target}.sha256" def is_musl_archive_target(target: str, source_path: Path) -> bool: @@ -285,10 +296,11 @@ def stage_release_pair( output_dir: Path, compilation_mode: str = "fastbuild", bazel_configs: list[str] | None = None, + sandbox: bool = False, ) -> None: outputs = ensure_bazel_output_files( platform, - [release_pair_label(target)], + [release_pair_label(target, sandbox)], compilation_mode, bazel_configs, ) @@ -304,8 +316,9 @@ def stage_release_pair( raise SystemExit(f"missing Rust binding output for {target}") from exc output_dir.mkdir(parents=True, exist_ok=True) - staged_library = output_dir / staged_archive_name(target, lib_path) - staged_binding = output_dir / f"src_binding_release_{target}.rs" + artifact_profile = SANDBOX_ARTIFACT_PROFILE if sandbox else RELEASE_ARTIFACT_PROFILE + staged_library = output_dir / staged_archive_name(target, lib_path, artifact_profile) + staged_binding = output_dir / staged_binding_name(target, artifact_profile) source_archive = ( merged_musl_archive(platform, lib_path, compilation_mode, bazel_configs) if is_musl_archive_target(target, lib_path) @@ -324,7 +337,7 @@ def stage_release_pair( shutil.copyfile(binding_path, staged_binding) - staged_checksums = output_dir / f"rusty_v8_release_{target}.sha256" + staged_checksums = output_dir / staged_checksums_name(target, artifact_profile) with staged_checksums.open("w", encoding="utf-8") as checksums: for path in [staged_library, staged_binding]: digest = hashlib.sha256() @@ -346,6 +359,7 @@ def parse_args() -> argparse.Namespace: stage_release_pair_parser.add_argument("--platform", required=True) stage_release_pair_parser.add_argument("--target", required=True) stage_release_pair_parser.add_argument("--output-dir", required=True) + stage_release_pair_parser.add_argument("--sandbox", action="store_true") stage_release_pair_parser.add_argument( "--bazel-config", action="append", @@ -390,6 +404,7 @@ def main() -> int: output_dir=Path(args.output_dir), compilation_mode=args.compilation_mode, bazel_configs=args.bazel_configs, + sandbox=args.sandbox, ) return 0 if args.command == "resolved-v8-crate-version": diff --git a/.github/scripts/test_rusty_v8_bazel.py b/.github/scripts/test_rusty_v8_bazel.py index 95c3eac157..c013eb7ca8 100644 --- a/.github/scripts/test_rusty_v8_bazel.py +++ b/.github/scripts/test_rusty_v8_bazel.py @@ -13,6 +13,46 @@ import rusty_v8_module_bazel class RustyV8BazelTest(unittest.TestCase): + def test_release_pair_labels_and_staged_names_distinguish_sandbox_artifacts(self) -> None: + self.assertEqual( + "//third_party/v8:rusty_v8_release_pair_x86_64_unknown_linux_musl", + rusty_v8_bazel.release_pair_label("x86_64-unknown-linux-musl"), + ) + self.assertEqual( + "//third_party/v8:rusty_v8_sandbox_release_pair_x86_64_unknown_linux_musl", + rusty_v8_bazel.release_pair_label("x86_64-unknown-linux-musl", sandbox=True), + ) + self.assertEqual( + "librusty_v8_release_x86_64-unknown-linux-musl.a.gz", + rusty_v8_bazel.staged_archive_name( + "x86_64-unknown-linux-musl", + Path("libv8.a"), + rusty_v8_bazel.RELEASE_ARTIFACT_PROFILE, + ), + ) + self.assertEqual( + "rusty_v8_ptrcomp_sandbox_release_x86_64-pc-windows-msvc.lib.gz", + rusty_v8_bazel.staged_archive_name( + "x86_64-pc-windows-msvc", + Path("v8.lib"), + rusty_v8_bazel.SANDBOX_ARTIFACT_PROFILE, + ), + ) + self.assertEqual( + "src_binding_ptrcomp_sandbox_release_x86_64-unknown-linux-musl.rs", + rusty_v8_bazel.staged_binding_name( + "x86_64-unknown-linux-musl", + rusty_v8_bazel.SANDBOX_ARTIFACT_PROFILE, + ), + ) + self.assertEqual( + "rusty_v8_ptrcomp_sandbox_release_x86_64-unknown-linux-musl.sha256", + rusty_v8_bazel.staged_checksums_name( + "x86_64-unknown-linux-musl", + rusty_v8_bazel.SANDBOX_ARTIFACT_PROFILE, + ), + ) + @patch("rusty_v8_bazel.ensure_bazel_output_files") @patch("rusty_v8_bazel.subprocess.run") def test_host_runnable_bazel_output_file_selects_runnable_candidate( diff --git a/.github/workflows/rusty-v8-release.yml b/.github/workflows/rusty-v8-release.yml index 5a436722bb..2b59628885 100644 --- a/.github/workflows/rusty-v8-release.yml +++ b/.github/workflows/rusty-v8-release.yml @@ -46,14 +46,14 @@ jobs: expected_release_tag="rusty-v8-v${V8_VERSION}" release_tag="${GITHUB_REF_NAME}" if [[ "${release_tag}" != "${expected_release_tag}" ]]; then - echo "Tag ${release_tag} does not match resolved v8 crate version ${V8_VERSION}." >&2 + echo "Tag ${release_tag} does not match expected release tag ${expected_release_tag}." >&2 exit 1 fi echo "release_tag=${release_tag}" >> "$GITHUB_OUTPUT" build: - name: Build ${{ matrix.target }} + name: Build ${{ matrix.variant }} ${{ matrix.target }} needs: metadata runs-on: ${{ matrix.runner }} permissions: @@ -65,10 +65,34 @@ jobs: include: - runner: ubuntu-24.04 platform: linux_amd64_musl + sandbox: false target: x86_64-unknown-linux-musl + variant: release - runner: ubuntu-24.04-arm platform: linux_arm64_musl + sandbox: false target: aarch64-unknown-linux-musl + variant: release + - runner: ubuntu-24.04 + platform: linux_amd64_musl + sandbox: true + target: x86_64-unknown-linux-musl + variant: ptrcomp-sandbox + - runner: ubuntu-24.04-arm + platform: linux_arm64_musl + sandbox: true + target: aarch64-unknown-linux-musl + variant: ptrcomp-sandbox + - runner: ubuntu-24.04 + platform: windows_amd64 + sandbox: true + target: x86_64-pc-windows-msvc + variant: ptrcomp-sandbox + - runner: ubuntu-24.04-arm + platform: windows_arm64 + sandbox: true + target: aarch64-pc-windows-msvc + variant: ptrcomp-sandbox steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -89,13 +113,20 @@ jobs: env: BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} PLATFORM: ${{ matrix.platform }} + SANDBOX: ${{ matrix.sandbox }} TARGET: ${{ matrix.target }} shell: bash run: | set -euo pipefail target_suffix="${TARGET//-/_}" - pair_target="//third_party/v8:rusty_v8_release_pair_${target_suffix}" + pair_kind="release_pair" + bazel_config_args=(--config=v8-release-compat) + if [[ "${SANDBOX}" == "true" ]]; then + pair_kind="sandbox_release_pair" + bazel_config_args=() + fi + pair_target="//third_party/v8:rusty_v8_${pair_kind}_${target_suffix}" extra_targets=() if [[ "${TARGET}" == *-unknown-linux-musl ]]; then extra_targets=( @@ -109,7 +140,7 @@ jobs: -c opt "--platforms=@llvm//platforms:${PLATFORM}" - --config=v8-release-compat + "${bazel_config_args[@]}" "${pair_target}" "${extra_targets[@]}" --build_metadata=COMMIT_SHA=$(git rev-parse HEAD) @@ -124,28 +155,138 @@ jobs: - name: Stage release pair env: PLATFORM: ${{ matrix.platform }} + SANDBOX: ${{ matrix.sandbox }} TARGET: ${{ matrix.target }} shell: bash run: | set -euo pipefail - python3 .github/scripts/rusty_v8_bazel.py stage-release-pair \ - --platform "${PLATFORM}" \ - --target "${TARGET}" \ - --compilation-mode opt \ - --bazel-config v8-release-compat \ + stage_args=( + --platform "${PLATFORM}" + --target "${TARGET}" + --compilation-mode opt --output-dir "dist/${TARGET}" + ) + if [[ "${SANDBOX}" == "true" ]]; then + stage_args+=(--sandbox) + else + stage_args+=(--bazel-config v8-release-compat) + fi - - name: Upload staged musl artifacts + python3 .github/scripts/rusty_v8_bazel.py stage-release-pair "${stage_args[@]}" + + - name: Upload staged artifacts uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - name: rusty-v8-${{ needs.metadata.outputs.v8_version }}-${{ matrix.target }} + name: rusty-v8-${{ needs.metadata.outputs.v8_version }}-${{ matrix.variant }}-${{ matrix.target }} path: dist/${{ matrix.target }}/* + validate-sandbox: + name: Validate sandbox artifacts - ${{ matrix.target }} + needs: + - metadata + - build + runs-on: ${{ matrix.runs_on }} + defaults: + run: + working-directory: codex-rs + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-unknown-linux-musl + exe_suffix: "" + runs_on: + group: codex-runners + labels: codex-linux-x64 + - target: aarch64-unknown-linux-musl + exe_suffix: "" + runs_on: + group: codex-runners + labels: codex-linux-arm64 + - target: x86_64-pc-windows-msvc + exe_suffix: ".exe" + runs_on: + group: codex-runners + labels: codex-windows-x64 + - target: aarch64-pc-windows-msvc + exe_suffix: ".exe" + runs_on: + group: codex-runners + labels: codex-windows-arm64 + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - if: ${{ runner.os == 'Linux' }} + name: Install Linux build dependencies + shell: bash + run: | + set -euo pipefail + sudo apt-get update -y + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + pkg-config \ + libcap-dev + + - uses: dtolnay/rust-toolchain@a0b273b48ed29de4470960879e8381ff45632f26 # 1.93.0 + with: + targets: ${{ matrix.target }} + + - name: Download staged sandbox artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: rusty-v8-${{ needs.metadata.outputs.v8_version }}-ptrcomp-sandbox-${{ matrix.target }} + path: ${{ runner.temp }}/rusty_v8_sandbox + + - name: Configure sandboxed rusty_v8 artifact overrides + env: + ARTIFACT_DIR: ${{ runner.temp }}/rusty_v8_sandbox + TARGET: ${{ matrix.target }} + shell: bash + run: | + set -euo pipefail + + artifact_profile="ptrcomp_sandbox_release" + binding_path="${ARTIFACT_DIR}/src_binding_${artifact_profile}_${TARGET}.rs" + checksums_path="${ARTIFACT_DIR}/rusty_v8_${artifact_profile}_${TARGET}.sha256" + + if [[ "${TARGET}" == *-pc-windows-msvc ]]; then + archive_name="rusty_v8_${artifact_profile}_${TARGET}.lib.gz" + else + archive_name="librusty_v8_${artifact_profile}_${TARGET}.a.gz" + fi + archive_path="${ARTIFACT_DIR}/${archive_name}" + + if [[ "$(wc -l < "${checksums_path}")" -ne 2 ]]; then + echo "Expected exactly two checksums for ${TARGET} in ${checksums_path}" >&2 + exit 1 + fi + + (cd "${ARTIFACT_DIR}" && sha256sum -c "${checksums_path}") + echo "RUSTY_V8_ARCHIVE=${archive_path}" >> "${GITHUB_ENV}" + echo "RUSTY_V8_SRC_BINDING_PATH=${binding_path}" >> "${GITHUB_ENV}" + + - name: Run sandboxed V8 probe tests + shell: bash + run: cargo test -p codex-v8-poc --target "${{ matrix.target }}" --features sandbox + + - name: Run sandboxed code-mode tests + shell: bash + run: cargo test -p codex-code-mode --target "${{ matrix.target }}" --features sandbox + + - name: Build release binary against sandboxed artifacts + shell: bash + run: cargo build --target "${{ matrix.target }}" --release --bin codex + + - name: Smoke release binary + shell: bash + run: "target/${{ matrix.target }}/release/codex${{ matrix.exe_suffix }} --version" + publish-release: needs: - metadata - build + - validate-sandbox runs-on: ubuntu-latest permissions: contents: write diff --git a/.github/workflows/v8-canary.yml b/.github/workflows/v8-canary.yml index 6e71367d1f..32e529b731 100644 --- a/.github/workflows/v8-canary.yml +++ b/.github/workflows/v8-canary.yml @@ -71,10 +71,24 @@ jobs: include: - runner: ubuntu-24.04 platform: linux_amd64_musl + sandbox: false target: x86_64-unknown-linux-musl + variant: release + - runner: ubuntu-24.04 + platform: linux_amd64_musl + sandbox: true + target: x86_64-unknown-linux-musl + variant: ptrcomp-sandbox - runner: ubuntu-24.04-arm platform: linux_arm64_musl + sandbox: false target: aarch64-unknown-linux-musl + variant: release + - runner: ubuntu-24.04-arm + platform: linux_arm64_musl + sandbox: true + target: aarch64-unknown-linux-musl + variant: ptrcomp-sandbox steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -96,13 +110,20 @@ jobs: env: BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} PLATFORM: ${{ matrix.platform }} + SANDBOX: ${{ matrix.sandbox }} TARGET: ${{ matrix.target }} shell: bash run: | set -euo pipefail target_suffix="${TARGET//-/_}" - pair_target="//third_party/v8:rusty_v8_release_pair_${target_suffix}" + pair_kind="release_pair" + bazel_config_args=(--config=v8-release-compat) + if [[ "${SANDBOX}" == "true" ]]; then + pair_kind="sandbox_release_pair" + bazel_config_args=() + fi + pair_target="//third_party/v8:rusty_v8_${pair_kind}_${target_suffix}" extra_targets=( "@llvm//runtimes/libcxx:libcxx.static" "@llvm//runtimes/libcxx:libcxxabi.static" @@ -111,7 +132,7 @@ jobs: bazel_args=( build "--platforms=@llvm//platforms:${PLATFORM}" - --config=v8-release-compat + "${bazel_config_args[@]}" "${pair_target}" "${extra_targets[@]}" --build_metadata=COMMIT_SHA=$(git rev-parse HEAD) @@ -126,19 +147,27 @@ jobs: - name: Stage release pair env: PLATFORM: ${{ matrix.platform }} + SANDBOX: ${{ matrix.sandbox }} TARGET: ${{ matrix.target }} shell: bash run: | set -euo pipefail - python3 .github/scripts/rusty_v8_bazel.py stage-release-pair \ - --platform "${PLATFORM}" \ - --target "${TARGET}" \ - --bazel-config v8-release-compat \ + stage_args=( + --platform "${PLATFORM}" + --target "${TARGET}" --output-dir "dist/${TARGET}" + ) + if [[ "${SANDBOX}" == "true" ]]; then + stage_args+=(--sandbox) + else + stage_args+=(--bazel-config v8-release-compat) + fi + + python3 .github/scripts/rusty_v8_bazel.py stage-release-pair "${stage_args[@]}" - name: Upload staged musl artifacts uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - name: v8-canary-${{ needs.metadata.outputs.v8_version }}-${{ matrix.target }} + name: v8-canary-${{ needs.metadata.outputs.v8_version }}-${{ matrix.variant }}-${{ matrix.target }} path: dist/${{ matrix.target }}/* diff --git a/third_party/v8/BUILD.bazel b/third_party/v8/BUILD.bazel index d6aa4c2cc9..e2f1a19c9f 100644 --- a/third_party/v8/BUILD.bazel +++ b/third_party/v8/BUILD.bazel @@ -364,6 +364,18 @@ cc_static_library( features = V8_STATIC_LIBRARY_FEATURES, ) +cc_static_library( + name = "v8_147_4_0_aarch64_pc_windows_msvc_bazel", + deps = [":v8_147_4_0_binding"], + features = V8_STATIC_LIBRARY_FEATURES, +) + +cc_static_library( + name = "v8_147_4_0_x86_64_pc_windows_msvc_bazel", + deps = [":v8_147_4_0_binding"], + features = V8_STATIC_LIBRARY_FEATURES, +) + filegroup( name = "src_binding_release_aarch64_unknown_linux_musl_147_4_0_release", srcs = ["@v8_crate_147_4_0//:src_binding_release_aarch64_unknown_linux_gnu"], @@ -374,6 +386,16 @@ filegroup( srcs = ["@v8_crate_147_4_0//:src_binding_release_x86_64_unknown_linux_gnu"], ) +filegroup( + name = "src_binding_release_aarch64_pc_windows_msvc_147_4_0_release", + srcs = ["@v8_crate_147_4_0//:src_binding_release_aarch64_pc_windows_msvc"], +) + +filegroup( + name = "src_binding_release_x86_64_pc_windows_msvc_147_4_0_release", + srcs = ["@v8_crate_147_4_0//:src_binding_release_x86_64_pc_windows_msvc"], +) + filegroup( name = "rusty_v8_release_pair_x86_64_apple_darwin", srcs = [ @@ -437,3 +459,35 @@ filegroup( ":src_binding_release_aarch64_pc_windows_msvc", ], ) + +filegroup( + name = "rusty_v8_sandbox_release_pair_x86_64_unknown_linux_musl", + srcs = [ + ":v8_147_4_0_x86_64_unknown_linux_musl_release", + ":src_binding_release_x86_64_unknown_linux_musl_147_4_0_release", + ], +) + +filegroup( + name = "rusty_v8_sandbox_release_pair_aarch64_unknown_linux_musl", + srcs = [ + ":v8_147_4_0_aarch64_unknown_linux_musl_release", + ":src_binding_release_aarch64_unknown_linux_musl_147_4_0_release", + ], +) + +filegroup( + name = "rusty_v8_sandbox_release_pair_x86_64_pc_windows_msvc", + srcs = [ + ":v8_147_4_0_x86_64_pc_windows_msvc_bazel", + ":src_binding_release_x86_64_pc_windows_msvc_147_4_0_release", + ], +) + +filegroup( + name = "rusty_v8_sandbox_release_pair_aarch64_pc_windows_msvc", + srcs = [ + ":v8_147_4_0_aarch64_pc_windows_msvc_bazel", + ":src_binding_release_aarch64_pc_windows_msvc_147_4_0_release", + ], +) diff --git a/third_party/v8/README.md b/third_party/v8/README.md index bd4e108754..96be904ff2 100644 --- a/third_party/v8/README.md +++ b/third_party/v8/README.md @@ -40,7 +40,7 @@ The consumer-facing selectors are: - `//third_party/v8:rusty_v8_archive_for_target` - `//third_party/v8:rusty_v8_binding_for_target` -Musl release assets are expected at the tag: +Current musl release assets are expected at the tag: - `rusty-v8-v` @@ -49,13 +49,32 @@ with these raw asset names: - `librusty_v8_release_.a.gz` - `src_binding_release_.rs` +During the sandbox rollout, sandbox-enabled assets are published alongside those +current assets on the same tag, with the Rust crate's sandbox feature suffix in +their raw names: + +- `librusty_v8_ptrcomp_sandbox_release_.a.gz` +- `rusty_v8_ptrcomp_sandbox_release_.lib.gz` +- `src_binding_ptrcomp_sandbox_release_.rs` + The dedicated publishing workflow is `.github/workflows/rusty-v8-release.yml`. -It builds musl release pairs from source and keeps the release artifacts as the -statically linked form: +Every tagged run builds the current musl release pairs from source and keeps the +release artifacts as the statically linked form: - `//third_party/v8:rusty_v8_release_pair_x86_64_unknown_linux_musl` - `//third_party/v8:rusty_v8_release_pair_aarch64_unknown_linux_musl` +The same run also builds the matching sandbox pair targets: + +- `//third_party/v8:rusty_v8_sandbox_release_pair_x86_64_unknown_linux_musl` +- `//third_party/v8:rusty_v8_sandbox_release_pair_aarch64_unknown_linux_musl` +- `//third_party/v8:rusty_v8_sandbox_release_pair_x86_64_pc_windows_msvc` +- `//third_party/v8:rusty_v8_sandbox_release_pair_aarch64_pc_windows_msvc` + +The workflow validates the staged sandbox outputs before publication by checking +the emitted checksums, running the focused V8/code-mode sandbox tests, and smoke +starting a release `codex` binary on every supported artifact target. + Cargo musl builds use `RUSTY_V8_ARCHIVE` plus a downloaded `RUSTY_V8_SRC_BINDING_PATH` to point at those `openai/codex` release assets directly. We do not use `RUSTY_V8_MIRROR` for musl because the upstream `v8`