mirror of
https://github.com/openai/codex.git
synced 2026-02-01 22:47:52 +00:00
This PR configures Codex CLI so it can be built with [Bazel](https://bazel.build) in addition to Cargo. The `.bazelrc` includes configuration so that remote builds can be done using [BuildBuddy](https://www.buildbuddy.io). If you are familiar with Bazel, things should work as you expect, e.g., run `bazel test //... --keep-going` to run all the tests in the repo, but we have also added some new aliases in the `justfile` for convenience: - `just bazel-test` to run tests locally - `just bazel-remote-test` to run tests remotely (currently, the remote build is for x86_64 Linux regardless of your host platform). Note we are currently seeing the following test failures in the remote build, so we still need to figure out what is happening here: ``` failures: suite::compact::manual_compact_twice_preserves_latest_user_messages suite::compact_resume_fork::compact_resume_after_second_compaction_preserves_history suite::compact_resume_fork::compact_resume_and_fork_preserve_model_history_view ``` - `just build-for-release` to build release binaries for all platforms/architectures remotely To setup remote execution: - [Create a buildbuddy account](https://app.buildbuddy.io/) (OpenAI employees should also request org access at https://openai.buildbuddy.io/join/ with their `@openai.com` email address.) - [Copy your API key](https://app.buildbuddy.io/docs/setup/) to `~/.bazelrc` (add the line `build --remote_header=x-buildbuddy-api-key=YOUR_KEY`) - Use `--config=remote` in your `bazel` invocations (or add `common --config=remote` to your `~/.bazelrc`, or use the `just` commands) ## CI In terms of CI, this PR introduces `.github/workflows/bazel.yml`, which uses Bazel to run the tests _locally_ on Mac and Linux GitHub runners (we are working on supporting Windows, but that is not ready yet). Note that the failures we are seeing in `just bazel-remote-test` do not occur on these GitHub CI jobs, so everything in `.github/workflows/bazel.yml` is green right now. The `bazel.yml` uses extra config in `.github/workflows/ci.bazelrc` so that macOS CI jobs build _remotely_ on Linux hosts (using the `docker://docker.io/mbolin491/codex-bazel` Docker image declared in the root `BUILD.bazel`) using cross-compilation to build the macOS artifacts. Then these artifacts are downloaded locally to GitHub's macOS runner so the tests can be executed natively. This is the relevant config that enables this: ``` common:macos --config=remote common:macos --strategy=remote common:macos --strategy=TestRunner=darwin-sandbox,local ``` Because of the remote caching benefits we get from BuildBuddy, these new CI jobs can be extremely fast! For example, consider these two jobs that ran all the tests on Linux x86_64: - Bazel 1m37s https://github.com/openai/codex/actions/runs/20861063212/job/59940545209?pr=8875 - Cargo 9m20s https://github.com/openai/codex/actions/runs/20861063192/job/59940559592?pr=8875 For now, we will continue to run both the Bazel and Cargo jobs for PRs, but once we add support for Windows and running Clippy, we should be able to cutover to using Bazel exclusively for PRs, which should still speed things up considerably. We will probably continue to run the Cargo jobs post-merge for commits that land on `main` as a sanity check. Release builds will also continue to be done by Cargo for now. Earlier attempt at this PR: https://github.com/openai/codex/pull/8832 Earlier attempt to add support for Buck2, now abandoned: https://github.com/openai/codex/pull/8504 --------- Co-authored-by: David Zbarsky <dzbarsky@gmail.com> Co-authored-by: Michael Bolin <mbolin@openai.com>
178 lines
6.7 KiB
Python
178 lines
6.7 KiB
Python
load("@crates//:data.bzl", "DEP_DATA")
|
|
load("@crates//:defs.bzl", "all_crate_deps")
|
|
load("@rules_platform//platform_data:defs.bzl", "platform_data")
|
|
load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library", "rust_test")
|
|
load("@rules_rust//cargo/private:cargo_build_script_wrapper.bzl", "cargo_build_script")
|
|
|
|
PLATFORMS = [
|
|
"linux_arm64_musl",
|
|
"linux_amd64_musl",
|
|
"macos_amd64",
|
|
"macos_arm64",
|
|
"windows_amd64",
|
|
"windows_arm64",
|
|
]
|
|
|
|
def multiplatform_binaries(name, platforms = PLATFORMS):
|
|
for platform in platforms:
|
|
platform_data(
|
|
name = name + "_" + platform,
|
|
platform = "@toolchains_llvm_bootstrapped//platforms:" + platform,
|
|
target = name,
|
|
tags = ["manual"],
|
|
)
|
|
|
|
native.filegroup(
|
|
name = "release_binaries",
|
|
srcs = [name + "_" + platform for platform in platforms],
|
|
tags = ["manual"],
|
|
)
|
|
|
|
def codex_rust_crate(
|
|
name,
|
|
crate_name,
|
|
crate_features = [],
|
|
crate_srcs = None,
|
|
crate_edition = None,
|
|
build_script_data = [],
|
|
compile_data = [],
|
|
deps_extra = [],
|
|
integration_deps_extra = [],
|
|
integration_compile_data_extra = [],
|
|
test_data_extra = [],
|
|
test_tags = [],
|
|
extra_binaries = []):
|
|
"""Defines a Rust crate with library, binaries, and tests wired for Bazel + Cargo parity.
|
|
|
|
The macro mirrors Cargo conventions: it builds a library when `src/` exists,
|
|
wires build scripts, exports `CARGO_BIN_EXE_*` for integration tests, and
|
|
creates unit + integration test targets. Dependency buckets map to the
|
|
Cargo.lock resolution in `@crates`.
|
|
|
|
Args:
|
|
name: Bazel target name for the library, should be the directory name.
|
|
Example: `app-server`.
|
|
crate_name: Cargo crate name from Cargo.toml
|
|
Example: `codex_app_server`.
|
|
crate_features: Cargo features to enable for this crate.
|
|
Crates are only compiled in a single configuration across the workspace, i.e.
|
|
with all features in this list enabled. So use sparingly, and prefer to refactor
|
|
optional functionality to a separate crate.
|
|
crate_srcs: Optional explicit srcs; defaults to `src/**/*.rs`.
|
|
crate_edition: Rust edition override, if not default.
|
|
You probably don't want this, it's only here for a single caller.
|
|
build_script_data: Data files exposed to the build script at runtime.
|
|
compile_data: Non-Rust compile-time data for the library target.
|
|
deps_extra: Extra normal deps beyond @crates resolution.
|
|
Typically only needed when features add additional deps.
|
|
integration_deps_extra: Extra deps for integration tests only.
|
|
integration_compile_data_extra: Extra compile_data for integration tests.
|
|
test_data_extra: Extra runtime data for tests.
|
|
test_tags: Tags applied to unit + integration test targets.
|
|
Typically used to disable the sandbox, but see https://bazel.build/reference/be/common-definitions#common.tags
|
|
extra_binaries: Additional binary labels to surface as test data and
|
|
`CARGO_BIN_EXE_*` environment variables. These are only needed for binaries from a different crate.
|
|
"""
|
|
deps = all_crate_deps(normal = True) + deps_extra
|
|
dev_deps = all_crate_deps(normal_dev = True)
|
|
proc_macro_deps = all_crate_deps(proc_macro = True)
|
|
proc_macro_dev_deps = all_crate_deps(proc_macro_dev = True)
|
|
|
|
test_env = {
|
|
"INSTA_WORKSPACE_ROOT": ".",
|
|
"INSTA_SNAPSHOT_PATH": "src",
|
|
}
|
|
|
|
rustc_env = {
|
|
"BAZEL_PACKAGE": native.package_name(),
|
|
}
|
|
|
|
binaries = DEP_DATA.get(native.package_name())["binaries"]
|
|
|
|
# TODO(zbarsky): cargo_build_script support?
|
|
|
|
lib_srcs = crate_srcs or native.glob(["src/**/*.rs"], exclude = binaries.values(), allow_empty = True)
|
|
|
|
if native.glob(["build.rs"], allow_empty = True):
|
|
cargo_build_script(
|
|
name = name + "-build-script",
|
|
srcs = ["build.rs"],
|
|
deps = all_crate_deps(build = True),
|
|
proc_macro_deps = all_crate_deps(build_proc_macro = True),
|
|
data = build_script_data,
|
|
# Some build script deps sniff version-related env vars...
|
|
version = "0.0.0",
|
|
)
|
|
|
|
deps = deps + [name + "-build-script"]
|
|
|
|
if lib_srcs:
|
|
rust_library(
|
|
name = name,
|
|
crate_name = crate_name,
|
|
crate_features = crate_features,
|
|
deps = deps,
|
|
proc_macro_deps = proc_macro_deps,
|
|
compile_data = compile_data,
|
|
srcs = lib_srcs,
|
|
edition = crate_edition,
|
|
rustc_env = rustc_env,
|
|
visibility = ["//visibility:public"],
|
|
)
|
|
|
|
rust_test(
|
|
name = name + "-unit-tests",
|
|
crate = name,
|
|
env = test_env,
|
|
deps = deps + dev_deps,
|
|
proc_macro_deps = proc_macro_deps + proc_macro_dev_deps,
|
|
rustc_env = rustc_env,
|
|
data = test_data_extra,
|
|
tags = test_tags,
|
|
)
|
|
|
|
maybe_lib = [name]
|
|
else:
|
|
maybe_lib = []
|
|
|
|
sanitized_binaries = []
|
|
cargo_env = {}
|
|
for binary, main in binaries.items():
|
|
#binary = binary.replace("-", "_")
|
|
sanitized_binaries.append(binary)
|
|
cargo_env["CARGO_BIN_EXE_" + binary] = "$(rootpath :%s)" % binary
|
|
|
|
rust_binary(
|
|
name = binary,
|
|
crate_name = binary.replace("-", "_"),
|
|
crate_root = main,
|
|
deps = maybe_lib + deps,
|
|
proc_macro_deps = proc_macro_deps,
|
|
edition = crate_edition,
|
|
srcs = native.glob(["src/**/*.rs"]),
|
|
visibility = ["//visibility:public"],
|
|
)
|
|
|
|
for binary_label in extra_binaries:
|
|
sanitized_binaries.append(binary_label)
|
|
binary = Label(binary_label).name
|
|
cargo_env["CARGO_BIN_EXE_" + binary] = "$(rootpath %s)" % binary_label
|
|
|
|
for test in native.glob(["tests/*.rs"], allow_empty = True):
|
|
test_name = name + "-" + test.removeprefix("tests/").removesuffix(".rs").replace("/", "-")
|
|
if not test_name.endswith("-test"):
|
|
test_name += "-test"
|
|
|
|
rust_test(
|
|
name = test_name,
|
|
crate_root = test,
|
|
srcs = [test],
|
|
data = native.glob(["tests/**"], allow_empty = True) + sanitized_binaries + test_data_extra,
|
|
compile_data = native.glob(["tests/**"], allow_empty = True) + integration_compile_data_extra,
|
|
deps = maybe_lib + deps + dev_deps + integration_deps_extra,
|
|
proc_macro_deps = proc_macro_deps + proc_macro_dev_deps,
|
|
rustc_env = rustc_env,
|
|
env = test_env | cargo_env,
|
|
tags = test_tags,
|
|
)
|