clone/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-05-17 01:32:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

bazel: build rusty_v8 release artifacts hermetically

Browse Source
This commit is contained in:
Channing Conger
2026-05-08 00:23:13 -07:00
parent c79864cb7a
commit b47b5dae0d
17 changed files with 917 additions and 601 deletions
Download Patch File Download Diff File Expand all files Collapse all files

174
.github/scripts/rusty_v8_bazel.py vendored
View File

@@ -9,7 +9,6 @@ import re
import shutil
import subprocess
import sys
import tempfile
import tomllib
from pathlib import Path
@@ -23,14 +22,9 @@ from rusty_v8_module_bazel import (
ROOT = Path(__file__).resolve().parents[2]
MODULE_BAZEL = ROOT / "MODULE.bazel"
RUSTY_V8_CHECKSUMS_DIR = ROOT / "third_party" / "v8"
STATIC_RUNTIME_ARCHIVE_LABELS = [
"@llvm//runtimes/libcxx:libcxx.static",
"@llvm//runtimes/libcxx:libcxxabi.static",
]
LLVM_AR_LABEL = "@llvm//tools:llvm-ar"
LLVM_RANLIB_LABEL = "@llvm//tools:llvm-ranlib"
RELEASE_ARTIFACT_PROFILE = "release"
SANDBOX_ARTIFACT_PROFILE = "ptrcomp_sandbox_release"
ARTIFACT_BAZEL_CONFIGS = ["rusty-v8-upstream-libcxx"]
def bazel_execroot() -> Path:
@@ -116,10 +110,8 @@ def ensure_bazel_output_files(
compilation_mode: str = "fastbuild",
bazel_configs: list[str] | None = None,
) -> list[Path]:
outputs = bazel_output_files(platform, labels, compilation_mode, bazel_configs)
if all(path.exists() for path in outputs):
return outputs
# Bazel output paths can be reused across config flips, so existence alone
# does not prove the files match the requested flags.
bazel_build(platform, labels, compilation_mode, bazel_configs)
outputs = bazel_output_files(platform, labels, compilation_mode, bazel_configs)
missing = [str(path) for path in outputs if not path.exists()]
@@ -128,6 +120,14 @@ def ensure_bazel_output_files(
return outputs
def artifact_bazel_configs(bazel_configs: list[str] | None = None) -> list[str]:
configured = list(ARTIFACT_BAZEL_CONFIGS)
for config in bazel_configs or []:
if config not in configured:
configured.append(config)
return configured
def release_pair_label(target: str, sandbox: bool = False) -> str:
target_suffix = target.replace("-", "_")
pair_kind = "sandbox_release_pair" if sandbox else "release_pair"
@@ -184,7 +184,7 @@ def command_manifest_path(manifest: Path | None, version: str) -> Path:
def staged_archive_name(target: str, source_path: Path, artifact_profile: str) -> str:
if source_path.suffix == ".lib":
if target.endswith("-pc-windows-msvc"):
return f"rusty_v8_{artifact_profile}_{target}.lib.gz"
return f"librusty_v8_{artifact_profile}_{target}.a.gz"
@@ -197,122 +197,6 @@ def staged_checksums_name(target: str, artifact_profile: str) -> str:
return f"rusty_v8_{artifact_profile}_{target}.sha256"
def needs_merged_runtime_archive(target: str, source_path: Path) -> bool:
return source_path.suffix == ".a" and target.endswith(
("-unknown-linux-gnu", "-unknown-linux-musl")
)
def single_bazel_output_file(
platform: str,
label: str,
compilation_mode: str = "fastbuild",
bazel_configs: list[str] | None = None,
) -> Path:
outputs = ensure_bazel_output_files(platform, [label], compilation_mode, bazel_configs)
if len(outputs) != 1:
raise SystemExit(f"expected exactly one output for {label}, found {outputs}")
return outputs[0]
def host_runnable_bazel_output_file(
platform: str,
label: str,
compilation_mode: str = "fastbuild",
bazel_configs: list[str] | None = None,
) -> Path:
outputs = ensure_bazel_output_files(platform, [label], compilation_mode, bazel_configs)
if len(outputs) == 1:
return outputs[0]
runnable_outputs = []
for output in outputs:
try:
result = subprocess.run(
[str(output), "--version"],
cwd=ROOT,
capture_output=True,
text=True,
)
except OSError:
continue
if result.returncode == 0:
runnable_outputs.append(output)
if len(runnable_outputs) != 1:
raise SystemExit(
f"expected exactly one host-runnable output for {label}, "
f"found {runnable_outputs} from {outputs}"
)
return runnable_outputs[0]
def merged_archive(
platform: str,
lib_path: Path,
extra_archives: list[Path],
compilation_mode: str = "fastbuild",
bazel_configs: list[str] | None = None,
) -> Path:
llvm_ar = host_runnable_bazel_output_file(
platform,
LLVM_AR_LABEL,
compilation_mode,
bazel_configs,
)
llvm_ranlib = host_runnable_bazel_output_file(
platform,
LLVM_RANLIB_LABEL,
compilation_mode,
bazel_configs,
)
temp_dir = Path(tempfile.mkdtemp(prefix="rusty-v8-runtime-stage-"))
merged_archive = temp_dir / lib_path.name
merge_commands = "\n".join(
[
f"create {merged_archive}",
f"addlib {lib_path}",
*[f"addlib {archive}" for archive in extra_archives],
"save",
"end",
]
)
subprocess.run(
[str(llvm_ar), "-M"],
cwd=ROOT,
check=True,
input=merge_commands,
text=True,
)
subprocess.run([str(llvm_ranlib), str(merged_archive)], cwd=ROOT, check=True)
return merged_archive
def merged_built_runtime_archive(
platform: str,
lib_path: Path,
compilation_mode: str = "fastbuild",
bazel_configs: list[str] | None = None,
) -> Path:
runtime_archives = [
single_bazel_output_file(platform, label, compilation_mode, bazel_configs)
for label in STATIC_RUNTIME_ARCHIVE_LABELS
]
return merged_archive(
platform,
lib_path,
runtime_archives,
compilation_mode,
bazel_configs,
)
def upstream_release_pair_paths(source_root: Path, target: str) -> tuple[Path, Path]:
lib_name = "rusty_v8.lib" if target.endswith("-pc-windows-msvc") else "librusty_v8.a"
gn_out = source_root / "target" / target / "release" / "gn_out"
return gn_out / "obj" / lib_name, gn_out / "src_binding.rs"
def stage_artifacts(
target: str,
lib_path: Path,
@@ -355,16 +239,6 @@ def stage_artifacts(
print(staged_checksums)
def stage_upstream_release_pair(
source_root: Path,
target: str,
output_dir: Path,
sandbox: bool = False,
) -> None:
lib_path, binding_path = upstream_release_pair_paths(source_root, target)
stage_artifacts(target, lib_path, binding_path, output_dir, sandbox)
def stage_release_pair(
platform: str,
target: str,
@@ -373,6 +247,7 @@ def stage_release_pair(
bazel_configs: list[str] | None = None,
sandbox: bool = False,
) -> None:
bazel_configs = artifact_bazel_configs(bazel_configs)
outputs = ensure_bazel_output_files(
platform,
[release_pair_label(target, sandbox)],
@@ -390,12 +265,7 @@ def stage_release_pair(
except StopIteration as exc:
raise SystemExit(f"missing Rust binding output for {target}") from exc
source_archive = (
merged_built_runtime_archive(platform, lib_path, compilation_mode, bazel_configs)
if needs_merged_runtime_archive(target, lib_path)
else lib_path
)
stage_artifacts(target, source_archive, binding_path, output_dir, sandbox)
stage_artifacts(target, lib_path, binding_path, output_dir, sandbox)
def parse_args() -> argparse.Namespace:
@@ -419,14 +289,6 @@ def parse_args() -> argparse.Namespace:
choices=["fastbuild", "opt", "dbg"],
)
stage_upstream_release_pair_parser = subparsers.add_parser(
"stage-upstream-release-pair"
)
stage_upstream_release_pair_parser.add_argument("--source-root", type=Path, required=True)
stage_upstream_release_pair_parser.add_argument("--target", required=True)
stage_upstream_release_pair_parser.add_argument("--output-dir", required=True)
stage_upstream_release_pair_parser.add_argument("--sandbox", action="store_true")
subparsers.add_parser("resolved-v8-crate-version")
check_module_bazel_parser = subparsers.add_parser("check-module-bazel")
@@ -462,14 +324,6 @@ def main() -> int:
sandbox=args.sandbox,
)
return 0
if args.command == "stage-upstream-release-pair":
stage_upstream_release_pair(
source_root=args.source_root,
target=args.target,
output_dir=Path(args.output_dir),
sandbox=args.sandbox,
)
return 0
if args.command == "resolved-v8-crate-version":
print(resolved_v8_crate_version())
return 0

171
.github/scripts/test_rusty_v8_bazel.py vendored
View File

@@ -4,9 +4,8 @@ from __future__ import annotations
import textwrap
import unittest
from tempfile import TemporaryDirectory
from pathlib import Path
from unittest.mock import Mock
from tempfile import TemporaryDirectory
from unittest.mock import patch
import rusty_v8_bazel
@@ -14,6 +13,22 @@ import rusty_v8_module_bazel
class RustyV8BazelTest(unittest.TestCase):
def test_artifact_bazel_configs_always_enable_upstream_libcxx(self) -> None:
self.assertEqual(
["rusty-v8-upstream-libcxx"],
rusty_v8_bazel.artifact_bazel_configs(),
)
self.assertEqual(
["rusty-v8-upstream-libcxx", "v8-release-compat"],
rusty_v8_bazel.artifact_bazel_configs(["v8-release-compat"]),
)
self.assertEqual(
["rusty-v8-upstream-libcxx", "v8-release-compat"],
rusty_v8_bazel.artifact_bazel_configs(
["rusty-v8-upstream-libcxx", "v8-release-compat"]
),
)
def test_release_pair_labels_and_staged_names_distinguish_sandbox_artifacts(self) -> None:
self.assertEqual(
"//third_party/v8:rusty_v8_release_pair_x86_64_unknown_linux_musl",
@@ -36,7 +51,7 @@ class RustyV8BazelTest(unittest.TestCase):
),
)
self.assertEqual(
"librusty_v8_ptrcomp_sandbox_release_x86_64-pc-windows-msvc.a.gz",
"rusty_v8_ptrcomp_sandbox_release_x86_64-pc-windows-msvc.lib.gz",
rusty_v8_bazel.staged_archive_name(
"x86_64-pc-windows-msvc",
Path("v8.a"),
@@ -58,77 +73,18 @@ class RustyV8BazelTest(unittest.TestCase):
),
)
def test_needs_merged_runtime_archive(self) -> None:
for target in [
"x86_64-unknown-linux-gnu",
"x86_64-unknown-linux-musl",
]:
self.assertTrue(rusty_v8_bazel.needs_merged_runtime_archive(target, Path("v8.a")))
self.assertFalse(
rusty_v8_bazel.needs_merged_runtime_archive(
"x86_64-apple-darwin",
Path("v8.a"),
)
)
self.assertFalse(
rusty_v8_bazel.needs_merged_runtime_archive(
"x86_64-pc-windows-msvc",
Path("v8.a"),
)
)
def test_upstream_release_pair_paths(self) -> None:
self.assertEqual(
(
Path(
"/tmp/rusty_v8/target/x86_64-apple-darwin/release/gn_out/obj/"
"librusty_v8.a"
),
Path(
"/tmp/rusty_v8/target/x86_64-apple-darwin/release/gn_out/"
"src_binding.rs"
),
),
rusty_v8_bazel.upstream_release_pair_paths(
Path("/tmp/rusty_v8"),
"x86_64-apple-darwin",
),
)
self.assertEqual(
(
Path(
"/tmp/rusty_v8/target/x86_64-pc-windows-msvc/release/gn_out/"
"obj/rusty_v8.lib"
),
Path(
"/tmp/rusty_v8/target/x86_64-pc-windows-msvc/release/gn_out/"
"src_binding.rs"
),
),
rusty_v8_bazel.upstream_release_pair_paths(
Path("/tmp/rusty_v8"),
"x86_64-pc-windows-msvc",
),
)
def test_stage_upstream_release_pair(self) -> None:
def test_stage_artifacts(self) -> None:
with TemporaryDirectory() as source_dir, TemporaryDirectory() as output_dir:
source_root = Path(source_dir)
gn_out = (
source_root
/ "target"
/ "aarch64-apple-darwin"
/ "release"
/ "gn_out"
)
(gn_out / "obj").mkdir(parents=True)
(gn_out / "obj" / "librusty_v8.a").write_bytes(b"archive")
(gn_out / "src_binding.rs").write_text("binding")
archive = source_root / "librusty_v8.a"
binding = source_root / "src_binding.rs"
archive.write_bytes(b"archive")
binding.write_text("binding")
rusty_v8_bazel.stage_upstream_release_pair(
source_root,
rusty_v8_bazel.stage_artifacts(
"aarch64-apple-darwin",
archive,
binding,
Path(output_dir),
sandbox=True,
)
@@ -142,53 +98,40 @@ class RustyV8BazelTest(unittest.TestCase):
{path.name for path in Path(output_dir).iterdir()},
)
@patch("rusty_v8_bazel.ensure_bazel_output_files")
@patch("rusty_v8_bazel.subprocess.run")
def test_host_runnable_bazel_output_file_selects_runnable_candidate(
self,
run: Mock,
ensure_outputs: Mock,
) -> None:
amd64_tool = Path("/tmp/llvm-amd64/bin/llvm-ar")
arm64_tool = Path("/tmp/llvm-arm64/bin/llvm-ar")
ensure_outputs.return_value = [amd64_tool, arm64_tool]
run.side_effect = [
OSError("Exec format error"),
Mock(returncode=0),
]
def test_ensure_bazel_output_files_rebuilds_existing_outputs(self) -> None:
with TemporaryDirectory() as output_dir:
output = Path(output_dir) / "libv8.a"
output.write_bytes(b"archive")
self.assertEqual(
arm64_tool,
rusty_v8_bazel.host_runnable_bazel_output_file(
"linux_arm64_musl",
"@llvm//tools:llvm-ar",
with (
patch.object(rusty_v8_bazel, "bazel_build") as bazel_build,
patch.object(
rusty_v8_bazel,
"bazel_output_files",
return_value=[output],
) as bazel_output_files,
):
self.assertEqual(
[output],
rusty_v8_bazel.ensure_bazel_output_files(
"macos_arm64",
["//third_party/v8:pair"],
"opt",
["rusty-v8-upstream-libcxx"],
),
)
bazel_build.assert_called_once_with(
"macos_arm64",
["//third_party/v8:pair"],
"opt",
),
)
@patch("rusty_v8_bazel.ensure_bazel_output_files")
@patch("rusty_v8_bazel.subprocess.run")
def test_host_runnable_bazel_output_file_rejects_ambiguous_candidates(
self,
run: Mock,
ensure_outputs: Mock,
) -> None:
amd64_tool = Path("/tmp/llvm-amd64/bin/llvm-ar")
arm64_tool = Path("/tmp/llvm-arm64/bin/llvm-ar")
ensure_outputs.return_value = [amd64_tool, arm64_tool]
run.side_effect = [
Mock(returncode=0),
Mock(returncode=0),
]
with self.assertRaisesRegex(
SystemExit,
"expected exactly one host-runnable output",
):
rusty_v8_bazel.host_runnable_bazel_output_file(
"linux_arm64_musl",
"@llvm//tools:llvm-ar",
["rusty-v8-upstream-libcxx"],
)
bazel_output_files.assert_called_once_with(
"macos_arm64",
["//third_party/v8:pair"],
"opt",
["rusty-v8-upstream-libcxx"],
)
def test_update_module_bazel_replaces_and_inserts_sha256(self) -> None: