mirror of
https://github.com/openai/codex.git
synced 2026-02-02 06:57:03 +00:00
Compare commits
1 Commits
compaction
...
pr8502
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
53edb6ebd3 |
30
.buckconfig
Normal file
30
.buckconfig
Normal file
@@ -0,0 +1,30 @@
|
||||
[cells]
|
||||
root = .
|
||||
toolchains = codex-rs/toolchains
|
||||
prelude = prelude
|
||||
none = none
|
||||
|
||||
[cell_aliases]
|
||||
# Buck2 prelude expects some common aliases to exist in some environments.
|
||||
config = prelude
|
||||
ovr_config = prelude
|
||||
fbcode = none
|
||||
fbsource = none
|
||||
fbcode_macros = none
|
||||
buck = none
|
||||
|
||||
# Use the Buck2 prelude bundled with the buck2 binary.
|
||||
[external_cells]
|
||||
prelude = bundled
|
||||
|
||||
[parser]
|
||||
target_platform_detector_spec = target://...->prelude//platforms:default
|
||||
|
||||
[codex]
|
||||
# Local-only knob used by codex-rs/buck2 to approximate Cargo profiles.
|
||||
# Override on the command line with:
|
||||
# ./scripts/buck2 build -c codex.rust_profile=release //codex-rs/cli:codex
|
||||
rust_profile = dev
|
||||
|
||||
[build]
|
||||
execution_platforms = prelude//platforms:default
|
||||
56
.github/workflows/buck2.yml
vendored
Normal file
56
.github/workflows/buck2.yml
vendored
Normal file
@@ -0,0 +1,56 @@
|
||||
name: Buck2 (Experimental, non-blocking for PRs)
|
||||
|
||||
on:
|
||||
pull_request: {}
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: buck2-${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
buck2-test:
|
||||
name: buck2 test //codex-rs/...
|
||||
runs-on: ubuntu-24.04
|
||||
# Non-blocking while Buck2 support is still experimental.
|
||||
# continue-on-error: true
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# scripts/buck2, scripts/reindeer, etc. are DotSlash wrappers.
|
||||
- name: Install DotSlash
|
||||
uses: facebook/install-dotslash@v2
|
||||
|
||||
- name: Setup Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@1.90
|
||||
with:
|
||||
# Match codex-rs/rust-toolchain.toml (and include rust-src for Buck2 toolchains).
|
||||
components: rustfmt, clippy, rust-src
|
||||
|
||||
- name: Install system deps (Linux)
|
||||
shell: bash
|
||||
run: |
|
||||
set -euxo pipefail
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y pkg-config libssl-dev
|
||||
|
||||
- name: Setup Buck2 (local)
|
||||
shell: bash
|
||||
run: |
|
||||
set -euxo pipefail
|
||||
./scripts/setup_buck2_local.sh
|
||||
|
||||
- name: Run Buck2 tests
|
||||
shell: bash
|
||||
run: |
|
||||
set -euxo pipefail
|
||||
./scripts/buck2 test -c test.rule_timeout_ms=1800000 --test-executor-stdout=- --test-executor-stderr=- //codex-rs/...
|
||||
7
.gitignore
vendored
7
.gitignore
vendored
@@ -13,6 +13,12 @@ build/
|
||||
out/
|
||||
storybook-static/
|
||||
|
||||
# buck2
|
||||
buck-out/
|
||||
**/BUCK
|
||||
codex-rs/third-party/
|
||||
codex-rs/cli/Cargo.lock
|
||||
|
||||
# ignore README for publishing
|
||||
codex-cli/README.md
|
||||
|
||||
@@ -89,4 +95,3 @@ CHANGELOG.ignore.md
|
||||
# Python bytecode files
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
||||
|
||||
1
codex-rs/Cargo.lock
generated
1
codex-rs/Cargo.lock
generated
@@ -1502,6 +1502,7 @@ dependencies = [
|
||||
"clap",
|
||||
"codex-core",
|
||||
"codex-utils-absolute-path",
|
||||
"codex-utils-cargo-bin",
|
||||
"landlock",
|
||||
"libc",
|
||||
"seccompiler",
|
||||
|
||||
99
codex-rs/buck2/codex_rust_toolchain.bzl
Normal file
99
codex-rs/buck2/codex_rust_toolchain.bzl
Normal file
@@ -0,0 +1,99 @@
|
||||
load("@prelude//rust:rust_toolchain.bzl", "PanicRuntime", "RustToolchainInfo")
|
||||
|
||||
_DEFAULT_TRIPLE = select({
|
||||
"prelude//os:linux": select({
|
||||
"prelude//cpu:arm64": "aarch64-unknown-linux-gnu",
|
||||
"prelude//cpu:riscv64": "riscv64gc-unknown-linux-gnu",
|
||||
"prelude//cpu:x86_64": "x86_64-unknown-linux-gnu",
|
||||
}),
|
||||
"prelude//os:macos": select({
|
||||
"prelude//cpu:arm64": "aarch64-apple-darwin",
|
||||
"prelude//cpu:x86_64": "x86_64-apple-darwin",
|
||||
}),
|
||||
"prelude//os:windows": select({
|
||||
"prelude//cpu:arm64": select({
|
||||
# Rustup's default ABI for the host on Windows is MSVC, not GNU.
|
||||
"DEFAULT": "aarch64-pc-windows-msvc",
|
||||
"prelude//abi:gnu": "aarch64-pc-windows-gnu",
|
||||
"prelude//abi:msvc": "aarch64-pc-windows-msvc",
|
||||
}),
|
||||
"prelude//cpu:x86_64": select({
|
||||
"DEFAULT": "x86_64-pc-windows-msvc",
|
||||
"prelude//abi:gnu": "x86_64-pc-windows-gnu",
|
||||
"prelude//abi:msvc": "x86_64-pc-windows-msvc",
|
||||
}),
|
||||
}),
|
||||
})
|
||||
|
||||
|
||||
def _codex_rust_toolchain_impl(ctx):
|
||||
# Buck doesn't have a built-in notion of "Cargo profiles", but it's useful
|
||||
# to provide a simple local knob that roughly matches `cargo build` vs
|
||||
# `cargo build --release`.
|
||||
#
|
||||
# Default is "dev" to match local development expectations.
|
||||
rust_profile = read_config("codex", "rust_profile", "dev")
|
||||
extra_rustc_flags = []
|
||||
if rust_profile == "release":
|
||||
# Roughly mirrors Cargo's release defaults (not a perfect match).
|
||||
extra_rustc_flags = [
|
||||
"-C",
|
||||
"opt-level=3",
|
||||
"-C",
|
||||
"debuginfo=0",
|
||||
]
|
||||
|
||||
return [
|
||||
DefaultInfo(),
|
||||
RustToolchainInfo(
|
||||
allow_lints = ctx.attrs.allow_lints,
|
||||
clippy_driver = RunInfo(args = [ctx.attrs.clippy_driver]),
|
||||
clippy_toml = ctx.attrs.clippy_toml[DefaultInfo].default_outputs[0] if ctx.attrs.clippy_toml else None,
|
||||
compiler = RunInfo(args = [ctx.attrs.rustc]),
|
||||
default_edition = ctx.attrs.default_edition,
|
||||
deny_lints = ctx.attrs.deny_lints,
|
||||
doctests = ctx.attrs.doctests,
|
||||
nightly_features = ctx.attrs.nightly_features,
|
||||
panic_runtime = PanicRuntime("unwind"),
|
||||
report_unused_deps = ctx.attrs.report_unused_deps,
|
||||
rustc_binary_flags = ctx.attrs.rustc_binary_flags,
|
||||
rustc_flags = ctx.attrs.rustc_flags + extra_rustc_flags,
|
||||
rustc_target_triple = ctx.attrs.rustc_target_triple,
|
||||
rustc_test_flags = ctx.attrs.rustc_test_flags,
|
||||
rustdoc = RunInfo(args = [ctx.attrs.rustdoc]),
|
||||
rustdoc_flags = ctx.attrs.rustdoc_flags,
|
||||
warn_lints = ctx.attrs.warn_lints,
|
||||
# Enable the prelude's "metadata-only rlib" behavior consistently
|
||||
# across the crate graph. This avoids rustc "found possibly newer
|
||||
# version of crate ..." (E0460) mismatches between binaries and
|
||||
# libraries in large Rust graphs.
|
||||
advanced_unstable_linking = ctx.attrs.advanced_unstable_linking,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
codex_rust_toolchain = rule(
|
||||
impl = _codex_rust_toolchain_impl,
|
||||
attrs = {
|
||||
"advanced_unstable_linking": attrs.bool(default = True),
|
||||
"allow_lints": attrs.list(attrs.string(), default = []),
|
||||
# Prefer explicit tool paths so the Buck execution directory doesn't
|
||||
# affect rustup toolchain resolution.
|
||||
"clippy_driver": attrs.string(default = "clippy-driver"),
|
||||
"clippy_toml": attrs.option(attrs.dep(providers = [DefaultInfo]), default = None),
|
||||
"default_edition": attrs.option(attrs.string(), default = None),
|
||||
"deny_lints": attrs.list(attrs.string(), default = []),
|
||||
"doctests": attrs.bool(default = False),
|
||||
"nightly_features": attrs.bool(default = False),
|
||||
"report_unused_deps": attrs.bool(default = False),
|
||||
"rustc": attrs.string(default = "rustc"),
|
||||
"rustc_binary_flags": attrs.list(attrs.arg(), default = []),
|
||||
"rustc_flags": attrs.list(attrs.arg(), default = []),
|
||||
"rustc_target_triple": attrs.string(default = _DEFAULT_TRIPLE),
|
||||
"rustc_test_flags": attrs.list(attrs.arg(), default = []),
|
||||
"rustdoc": attrs.string(default = "rustdoc"),
|
||||
"rustdoc_flags": attrs.list(attrs.arg(), default = []),
|
||||
"warn_lints": attrs.list(attrs.string(), default = []),
|
||||
},
|
||||
is_toolchain_rule = True,
|
||||
)
|
||||
111
codex-rs/buck2/reindeer_macros.bzl
Normal file
111
codex-rs/buck2/reindeer_macros.bzl
Normal file
@@ -0,0 +1,111 @@
|
||||
load("@prelude//rust:cargo_buildscript.bzl", _prelude_buildscript_run = "buildscript_run")
|
||||
load("@prelude//rust:cargo_package.bzl", "cargo")
|
||||
|
||||
|
||||
def codex_noop_alias(**_kwargs):
|
||||
# Reindeer normally emits aliases like `alias(name = "rand", actual = ":rand-0.8.5")`
|
||||
# to provide stable unversioned target names. In a non-trivial workspace it's
|
||||
# common to have multiple versions of the same crate in one graph, which
|
||||
# leads to duplicate alias target names and buckification failures.
|
||||
#
|
||||
# For local Buck experiments (where we don't check in generated third-party
|
||||
# BUCK files), it's simplest to disable these aliases and depend on
|
||||
# versioned targets directly.
|
||||
pass
|
||||
|
||||
|
||||
def _codex_extra_srcs_for_manifest_dir(manifest_dir):
|
||||
if not manifest_dir:
|
||||
return []
|
||||
|
||||
# Use per-crate globs rooted at the crate's vendored manifest dir, which is
|
||||
# passed through by Reindeer as CARGO_MANIFEST_DIR (e.g. vendor/foo-1.2.3).
|
||||
#
|
||||
# We include *all* files under the crate root so `include_str!` and
|
||||
# `include_bytes!` work without per-crate Reindeer fixups. This is local-only
|
||||
# buckification, so we prefer robustness over a minimal srcs list.
|
||||
return glob(
|
||||
["{}/**".format(manifest_dir)],
|
||||
exclude = [
|
||||
"{}/target/**".format(manifest_dir),
|
||||
"{}/.git/**".format(manifest_dir),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def codex_rust_library(**kwargs):
|
||||
# Make generated third-party targets consumable from anywhere in the repo.
|
||||
kwargs["visibility"] = ["PUBLIC"]
|
||||
env = kwargs.get("env", {})
|
||||
manifest_dir = env.get("CARGO_MANIFEST_DIR")
|
||||
srcs = list(kwargs.get("srcs", []))
|
||||
srcs.extend(_codex_extra_srcs_for_manifest_dir(manifest_dir))
|
||||
kwargs["srcs"] = srcs
|
||||
cargo.rust_library(**kwargs)
|
||||
|
||||
|
||||
def codex_rust_binary(**kwargs):
|
||||
kwargs["visibility"] = ["PUBLIC"]
|
||||
env = kwargs.get("env", {})
|
||||
manifest_dir = env.get("CARGO_MANIFEST_DIR")
|
||||
srcs = list(kwargs.get("srcs", []))
|
||||
srcs.extend(_codex_extra_srcs_for_manifest_dir(manifest_dir))
|
||||
kwargs["srcs"] = srcs
|
||||
cargo.rust_binary(**kwargs)
|
||||
|
||||
|
||||
def codex_buildscript_run(**kwargs):
|
||||
# Many build scripts (especially those using `cc`/`cc-rs`) expect Cargo to
|
||||
# provide a handful of profile env vars. Buck does not set these by default.
|
||||
env = dict(kwargs.get("env", {}))
|
||||
rust_profile = read_config("codex", "rust_profile", "dev")
|
||||
if rust_profile == "release":
|
||||
env.setdefault("OPT_LEVEL", "3")
|
||||
env.setdefault("PROFILE", "release")
|
||||
env.setdefault("DEBUG", "false")
|
||||
else:
|
||||
env.setdefault("OPT_LEVEL", "0")
|
||||
env.setdefault("PROFILE", "debug")
|
||||
env.setdefault("DEBUG", "true")
|
||||
|
||||
# Provide common Cargo cfg env vars that some build scripts expect.
|
||||
env.setdefault(
|
||||
"CARGO_CFG_TARGET_OS",
|
||||
select({
|
||||
"prelude//os:linux": "linux",
|
||||
"prelude//os:macos": "macos",
|
||||
"prelude//os:windows": "windows",
|
||||
"DEFAULT": "",
|
||||
}),
|
||||
)
|
||||
env.setdefault(
|
||||
"CARGO_CFG_TARGET_ARCH",
|
||||
select({
|
||||
"prelude//cpu:arm64": "aarch64",
|
||||
"prelude//cpu:x86_64": "x86_64",
|
||||
"DEFAULT": "",
|
||||
}),
|
||||
)
|
||||
env.setdefault("CARGO_CFG_TARGET_ENDIAN", "little")
|
||||
env.setdefault(
|
||||
"CARGO_CFG_TARGET_ENV",
|
||||
select({
|
||||
"prelude//os:linux": "gnu",
|
||||
"DEFAULT": "",
|
||||
}),
|
||||
)
|
||||
|
||||
# Forward native link directives emitted by build scripts into rustc flags.
|
||||
# Without this, crates like `ring` and `tree-sitter-*` will compile but fail
|
||||
# to link due to missing native symbols.
|
||||
kwargs.setdefault("rustc_link_lib", True)
|
||||
kwargs.setdefault("rustc_link_search", True)
|
||||
|
||||
# `CARGO_MANIFEST_DIR` is expected by many build scripts. We can usually
|
||||
# derive it from the `manifest_dir` parameter that buildscript_run uses.
|
||||
manifest_dir = kwargs.get("manifest_dir")
|
||||
if type(manifest_dir) == type(""):
|
||||
env.setdefault("CARGO_MANIFEST_DIR", "$(location {})".format(manifest_dir))
|
||||
|
||||
kwargs["env"] = env
|
||||
_prelude_buildscript_run(**kwargs)
|
||||
@@ -24,6 +24,7 @@ libc = { workspace = true }
|
||||
seccompiler = { workspace = true }
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dev-dependencies]
|
||||
codex-utils-cargo-bin = { workspace = true }
|
||||
tempfile = { workspace = true }
|
||||
tokio = { workspace = true, features = [
|
||||
"io-std",
|
||||
|
||||
@@ -60,8 +60,10 @@ async fn run_cmd(cmd: &[&str], writable_roots: &[PathBuf], timeout_ms: u64) {
|
||||
exclude_tmpdir_env_var: true,
|
||||
exclude_slash_tmp: true,
|
||||
};
|
||||
let sandbox_program = env!("CARGO_BIN_EXE_codex-linux-sandbox");
|
||||
let codex_linux_sandbox_exe = Some(PathBuf::from(sandbox_program));
|
||||
let codex_linux_sandbox_exe = Some(
|
||||
codex_utils_cargo_bin::cargo_bin("codex-linux-sandbox")
|
||||
.expect("codex-linux-sandbox binary should be available for tests"),
|
||||
);
|
||||
let res = process_exec_tool_call(
|
||||
params,
|
||||
&sandbox_policy,
|
||||
@@ -154,8 +156,10 @@ async fn assert_network_blocked(cmd: &[&str]) {
|
||||
};
|
||||
|
||||
let sandbox_policy = SandboxPolicy::new_read_only_policy();
|
||||
let sandbox_program = env!("CARGO_BIN_EXE_codex-linux-sandbox");
|
||||
let codex_linux_sandbox_exe: Option<PathBuf> = Some(PathBuf::from(sandbox_program));
|
||||
let codex_linux_sandbox_exe: Option<PathBuf> = Some(
|
||||
codex_utils_cargo_bin::cargo_bin("codex-linux-sandbox")
|
||||
.expect("codex-linux-sandbox binary should be available for tests"),
|
||||
);
|
||||
let result = process_exec_tool_call(
|
||||
params,
|
||||
&sandbox_policy,
|
||||
|
||||
56
codex-rs/reindeer.toml
Normal file
56
codex-rs/reindeer.toml
Normal file
@@ -0,0 +1,56 @@
|
||||
##
|
||||
## Reindeer config for codex-rs (Cargo workspace) + Buck2.
|
||||
##
|
||||
## This file is checked in, but generated Buck artifacts (BUCK files and
|
||||
## vendored crates) are intentionally gitignored for now.
|
||||
##
|
||||
|
||||
# Focus buckification on the main CLI to avoid pulling in platform-specific
|
||||
# workspace members that can create duplicate "public alias" names.
|
||||
manifest_path = "cli/Cargo.toml"
|
||||
|
||||
# Place generated output under codex-rs/third-party/.
|
||||
third_party_dir = "third-party"
|
||||
|
||||
# Use vendored crates (no crates.io downloads during `buck2 build`).
|
||||
vendor = true
|
||||
|
||||
# Provide common Cargo env vars for crates/build scripts that use `env!()`.
|
||||
cargo_env = [
|
||||
"CARGO_CRATE_NAME",
|
||||
"CARGO_MANIFEST_DIR",
|
||||
"CARGO_PKG_AUTHORS",
|
||||
"CARGO_PKG_DESCRIPTION",
|
||||
"CARGO_PKG_NAME",
|
||||
"CARGO_PKG_REPOSITORY",
|
||||
"CARGO_PKG_VERSION",
|
||||
"CARGO_PKG_VERSION_MAJOR",
|
||||
"CARGO_PKG_VERSION_MINOR",
|
||||
"CARGO_PKG_VERSION_PATCH",
|
||||
"CARGO_PKG_VERSION_PRE",
|
||||
]
|
||||
|
||||
[buck]
|
||||
file_name = "BUCK"
|
||||
|
||||
# These resolve to macros defined in codex-rs/buck2/reindeer_macros.bzl.
|
||||
alias = "codex_noop_alias"
|
||||
alias_with_platforms = "codex_noop_alias"
|
||||
rust_library = "codex_rust_library"
|
||||
rust_binary = "codex_rust_binary"
|
||||
buildscript_genrule = "codex_buildscript_run"
|
||||
|
||||
buckfile_imports = """
|
||||
load("//codex-rs/buck2:reindeer_macros.bzl", "codex_buildscript_run", "codex_noop_alias", "codex_rust_binary", "codex_rust_library")
|
||||
"""
|
||||
|
||||
generated_file_header = """
|
||||
##
|
||||
## @generated by reindeer
|
||||
## Do not edit by hand.
|
||||
##
|
||||
## To regenerate:
|
||||
## (cd codex-rs && reindeer vendor && reindeer buckify)
|
||||
##
|
||||
"""
|
||||
|
||||
636
codex-rs/scripts/gen_buck_first_party.py
Normal file
636
codex-rs/scripts/gen_buck_first_party.py
Normal file
@@ -0,0 +1,636 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Generate BUCK files for first-party Rust crates in the codex-rs workspace.
|
||||
|
||||
Reindeer generates third-party targets (in `codex-rs/third-party/BUCK`). This
|
||||
script generates Buck targets for workspace members (local crates) that depend
|
||||
on those third-party targets.
|
||||
|
||||
The generated BUCK files are intentionally gitignored for now.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Optional
|
||||
|
||||
|
||||
CODEX_RS_ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
REPO_ROOT = CODEX_RS_ROOT.parent
|
||||
BUILDIFIER = REPO_ROOT / "scripts" / "buildifier"
|
||||
|
||||
# All labels are rooted at the repo root Buck project.
|
||||
CODEX_RS_LABEL_PREFIX = "//codex-rs"
|
||||
|
||||
# Some integration tests rely on Cargo's behavior of allowing `include_str!`
|
||||
# paths that traverse outside of a package directory. Buck does not allow `..`
|
||||
# paths in `srcs`, so we skip those tests for now.
|
||||
SKIP_INTEGRATION_TESTS_BY_PACKAGE: dict[str, set[str]] = {
|
||||
"codex-core": {"all"},
|
||||
}
|
||||
|
||||
|
||||
def cargo_metadata() -> dict[str, Any]:
|
||||
out = subprocess.check_output(
|
||||
["cargo", "metadata", "--format-version=1", "--locked"],
|
||||
cwd=CODEX_RS_ROOT,
|
||||
text=True,
|
||||
)
|
||||
return json.loads(out)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BuckDep:
|
||||
label: str
|
||||
local_name: str
|
||||
crate_name: str
|
||||
cfg: Optional[str]
|
||||
|
||||
|
||||
def parse_dep_cfg(cfg: Optional[str]) -> list[str]:
|
||||
"""
|
||||
Map a subset of Cargo cfg(...) strings to Buck2 constraint keys.
|
||||
|
||||
Returns a list of constraint keys that should include the dependency.
|
||||
An empty list means "unconditional".
|
||||
"""
|
||||
if cfg is None:
|
||||
return []
|
||||
|
||||
cfg = cfg.strip()
|
||||
m = re.fullmatch(r'cfg\(target_os\s*=\s*"([^"]+)"\)', cfg)
|
||||
if m:
|
||||
os_name = m.group(1)
|
||||
if os_name in ("linux", "macos", "windows"):
|
||||
return [f"prelude//os:{os_name}"]
|
||||
|
||||
if cfg == "cfg(windows)":
|
||||
return ["prelude//os:windows"]
|
||||
|
||||
if cfg == "cfg(unix)":
|
||||
return ["prelude//os:linux", "prelude//os:macos"]
|
||||
|
||||
# If we don't recognize the expression, treat it as unconditional to avoid
|
||||
# silently dropping deps. This may cause platform build issues, in which
|
||||
# case we can extend this mapping.
|
||||
return []
|
||||
|
||||
|
||||
def buckify_features(features: list[str]) -> list[str]:
|
||||
# Cargo metadata includes lots of feature names, including "default".
|
||||
# Keep stable output for diffs.
|
||||
return sorted(features)
|
||||
|
||||
|
||||
def package_lib_target(pkg: dict[str, Any]) -> Optional[dict[str, Any]]:
|
||||
for t in pkg.get("targets") or []:
|
||||
kind = t.get("kind") or []
|
||||
if "proc-macro" in kind or "lib" in kind:
|
||||
return t
|
||||
return None
|
||||
|
||||
|
||||
def package_bin_targets(pkg: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
return [t for t in (pkg.get("targets") or []) if "bin" in (t.get("kind") or [])]
|
||||
|
||||
def package_test_targets(pkg: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
# These correspond to Cargo integration tests under `tests/`.
|
||||
return [t for t in (pkg.get("targets") or []) if "test" in (t.get("kind") or [])]
|
||||
|
||||
|
||||
def relpath_from_codex_rs(path: str) -> str:
|
||||
return str(pathlib.Path(path).resolve().relative_to(CODEX_RS_ROOT))
|
||||
|
||||
|
||||
def write_buck_file(crate_dir: pathlib.Path, content: str) -> None:
|
||||
buck_path = crate_dir / "BUCK"
|
||||
buck_path.write_text(content, encoding="utf-8")
|
||||
|
||||
def buildifier(paths: list[pathlib.Path]) -> None:
|
||||
if not paths:
|
||||
return
|
||||
if not BUILDIFIER.exists():
|
||||
# Generated BUCK files are local-only; keep generation working even if
|
||||
# buildifier isn't available.
|
||||
return
|
||||
|
||||
# Run buildifier from the repo root so it can infer workspace-relative paths.
|
||||
proc = subprocess.run(
|
||||
[str(BUILDIFIER), "-lint=off", "-mode=fix", *[str(p) for p in paths]],
|
||||
cwd=REPO_ROOT,
|
||||
text=True,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
print(f"warning: buildifier failed (exit {proc.returncode}); leaving BUCK files unformatted", file=sys.stderr)
|
||||
|
||||
|
||||
def starlark_list(items: list[str], indent: str = " ") -> str:
|
||||
if not items:
|
||||
return "[]"
|
||||
lines = ["["]
|
||||
for it in items:
|
||||
lines.append(f'{indent}"{it}",')
|
||||
lines.append("]")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def starlark_dict(d: dict[str, str], indent: str = " ") -> str:
|
||||
if not d:
|
||||
return "{}"
|
||||
lines = ["{"] # insertion order is stable for our dict construction
|
||||
for k, v in d.items():
|
||||
lines.append(f'{indent}"{k}": "{v}",')
|
||||
lines.append("}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def starlark_deps_expr(base: list[str], conditional: dict[str, list[str]]) -> str:
|
||||
"""
|
||||
Render deps as a list plus (optional) `select()` fragments.
|
||||
"""
|
||||
expr = starlark_list(base)
|
||||
for constraint, deps in sorted(conditional.items()):
|
||||
expr = (
|
||||
f"{expr} + select({{\n"
|
||||
f' "{constraint}": {starlark_list(deps, indent=" ")},\n'
|
||||
f' "DEFAULT": [],\n'
|
||||
"})"
|
||||
)
|
||||
return expr
|
||||
|
||||
|
||||
def starlark_str(s: str) -> str:
|
||||
# Conservative escaping for readability (most strings here are already safe).
|
||||
return s.replace("\\", "\\\\").replace('"', '\\"')
|
||||
|
||||
|
||||
def parse_semver(version: str) -> tuple[str, str, str, str]:
|
||||
m = re.match(r"^(\d+)\.(\d+)\.(\d+)(?:-(.*))?$", version)
|
||||
if not m:
|
||||
return ("0", "0", "0", "")
|
||||
return (m.group(1), m.group(2), m.group(3), m.group(4) or "")
|
||||
|
||||
def group_deps(deps: list[BuckDep]) -> tuple[list[str], dict[str, list[str]], dict[str, str]]:
|
||||
"""
|
||||
Group deps into (base_deps, conditional_deps, named_deps) for emitting in BUCK.
|
||||
"""
|
||||
base_deps: list[str] = []
|
||||
conditional_deps: dict[str, list[str]] = {}
|
||||
named_deps: dict[str, str] = {}
|
||||
for d in deps:
|
||||
dep_constraints = parse_dep_cfg(d.cfg)
|
||||
if dep_constraints:
|
||||
for c in dep_constraints:
|
||||
conditional_deps.setdefault(c, []).append(d.label)
|
||||
else:
|
||||
base_deps.append(d.label)
|
||||
|
||||
# Handle renamed deps (Cargo `package = ...` style).
|
||||
if d.local_name != d.crate_name:
|
||||
named_deps[d.local_name] = d.label
|
||||
|
||||
base_deps = sorted(set(base_deps))
|
||||
for k in list(conditional_deps.keys()):
|
||||
conditional_deps[k] = sorted(set(conditional_deps[k]))
|
||||
|
||||
return (base_deps, conditional_deps, named_deps)
|
||||
|
||||
|
||||
def label_for_workspace_pkg(crate_rel_dir: str, rule_name: str) -> str:
|
||||
# crate_rel_dir is relative to codex-rs/, but buck labels are rooted at the
|
||||
# repo root, so we prefix with //codex-rs.
|
||||
return f"{CODEX_RS_LABEL_PREFIX}/{crate_rel_dir}:{rule_name}"
|
||||
|
||||
|
||||
def label_for_third_party(pkg_name: str, version: str) -> str:
|
||||
# Reindeer generates versioned third-party targets and we disable the
|
||||
# unversioned alias layer.
|
||||
return f"{CODEX_RS_LABEL_PREFIX}/third-party:{pkg_name}-{version}"
|
||||
|
||||
def collect_snap_resources(crate_dir: pathlib.Path) -> dict[str, str]:
|
||||
"""
|
||||
Collect `.snap` files for insta snapshot tests.
|
||||
|
||||
Buck's rust_test uses an external runner that executes tests from the
|
||||
project root with project-relative paths. Many snapshot tests expect to find
|
||||
their `.snap` files under `codex-rs/<crate>/...` from that root, which is
|
||||
exactly where the rust rules place `resources`.
|
||||
"""
|
||||
resources: dict[str, str] = {}
|
||||
for p in sorted(crate_dir.rglob("*.snap")):
|
||||
rel = str(p.relative_to(crate_dir))
|
||||
resources[rel] = rel
|
||||
return resources
|
||||
|
||||
def buck_bin_rule_name(pkg_name: str, bin_name: str) -> str:
|
||||
# Avoid collisions with the package's rust_library target, which we name
|
||||
# after the package (pkg_name).
|
||||
if bin_name == pkg_name:
|
||||
return f"{bin_name}-bin"
|
||||
return bin_name
|
||||
|
||||
|
||||
def main() -> int:
|
||||
meta = cargo_metadata()
|
||||
packages = meta.get("packages", [])
|
||||
|
||||
# Package ID -> package json
|
||||
by_id: dict[str, Any] = {p["id"]: p for p in packages}
|
||||
|
||||
# Workspace member IDs -> crate directory relative to codex-rs/
|
||||
workspace_members: list[str] = list(meta.get("workspace_members") or [])
|
||||
workspace_dirs: dict[str, str] = {}
|
||||
for pkg_id in workspace_members:
|
||||
pkg = by_id[pkg_id]
|
||||
manifest_path = pkg["manifest_path"]
|
||||
crate_rel = relpath_from_codex_rs(manifest_path)
|
||||
crate_dir = str(pathlib.Path(crate_rel).parent)
|
||||
workspace_dirs[pkg_id] = crate_dir
|
||||
|
||||
# Resolve graph nodes for dependency edges (including cfg/platform edges).
|
||||
resolve = meta.get("resolve") or {}
|
||||
nodes = resolve.get("nodes") or []
|
||||
node_by_id: dict[str, Any] = {n["id"]: n for n in nodes}
|
||||
|
||||
# Build a workspace-wide mapping of Cargo binary name -> Buck label for the
|
||||
# corresponding rust_binary target. Many integration tests use
|
||||
# assert_cmd/escargot helpers that look for CARGO_BIN_EXE_* env vars (set by
|
||||
# Cargo) to find binaries; we emulate that under Buck.
|
||||
cargo_bin_to_label: dict[str, str] = {}
|
||||
for pkg_id in workspace_members:
|
||||
pkg = by_id[pkg_id]
|
||||
crate_dir_rel = workspace_dirs[pkg_id]
|
||||
for bin_t in package_bin_targets(pkg):
|
||||
bin_name = bin_t["name"]
|
||||
cargo_bin_to_label[bin_name] = label_for_workspace_pkg(
|
||||
crate_dir_rel,
|
||||
buck_bin_rule_name(pkg["name"], bin_name),
|
||||
)
|
||||
|
||||
generated_buck_files: list[pathlib.Path] = []
|
||||
|
||||
for pkg_id in workspace_members:
|
||||
pkg = by_id[pkg_id]
|
||||
crate_dir_rel = workspace_dirs[pkg_id]
|
||||
crate_dir = CODEX_RS_ROOT / crate_dir_rel
|
||||
|
||||
lib_t = package_lib_target(pkg)
|
||||
bin_ts = package_bin_targets(pkg)
|
||||
test_ts = package_test_targets(pkg)
|
||||
if lib_t is None and not bin_ts:
|
||||
continue
|
||||
|
||||
node = node_by_id.get(pkg_id) or {}
|
||||
|
||||
normal_deps: list[BuckDep] = []
|
||||
dev_deps: list[BuckDep] = []
|
||||
for dep in node.get("deps") or []:
|
||||
dep_id = dep["pkg"]
|
||||
if dep_id == pkg_id:
|
||||
# Cargo sometimes models "self dependencies" (commonly as a
|
||||
# dev-dependency to enable features for tests). Buck should not
|
||||
# treat this as a real crate edge, and using it creates cycles.
|
||||
continue
|
||||
dep_pkg = by_id.get(dep_id)
|
||||
if not dep_pkg:
|
||||
continue
|
||||
|
||||
dep_kinds = dep.get("dep_kinds") or []
|
||||
|
||||
local_name = dep.get("name") or dep_pkg["name"]
|
||||
crate_name = dep_pkg.get("targets", [{}])[0].get("name") or local_name
|
||||
|
||||
if dep_id in workspace_dirs:
|
||||
dep_label = label_for_workspace_pkg(workspace_dirs[dep_id], dep_pkg["name"])
|
||||
else:
|
||||
dep_label = label_for_third_party(dep_pkg["name"], dep_pkg["version"])
|
||||
|
||||
# A single dependency may appear multiple times with different Cargo
|
||||
# cfg(...) selectors (e.g. target.'cfg(target_os = "...")'). Cargo
|
||||
# metadata encodes these as multiple dep_kind entries, so we emit a
|
||||
# Buck dep entry for each one.
|
||||
if not dep_kinds:
|
||||
dep_kinds = [{"kind": None, "target": None}]
|
||||
|
||||
for k in dep_kinds:
|
||||
kind = k.get("kind")
|
||||
if kind not in (None, "normal", "dev"):
|
||||
continue
|
||||
|
||||
dep_entry = BuckDep(
|
||||
label=dep_label,
|
||||
local_name=local_name,
|
||||
crate_name=crate_name,
|
||||
cfg=k.get("target"),
|
||||
)
|
||||
if kind is None or kind == "normal":
|
||||
normal_deps.append(dep_entry)
|
||||
elif kind == "dev":
|
||||
dev_deps.append(dep_entry)
|
||||
|
||||
# Deduplicate while preserving stable output.
|
||||
normal_deps = list({(d.label, d.local_name, d.crate_name, d.cfg): d for d in normal_deps}.values())
|
||||
dev_deps = list({(d.label, d.local_name, d.crate_name, d.cfg): d for d in dev_deps}.values())
|
||||
|
||||
base_deps, conditional_deps, named_deps = group_deps(normal_deps)
|
||||
test_base_deps, test_conditional_deps, test_named_deps = group_deps(normal_deps + dev_deps)
|
||||
|
||||
# Enabled features for this package (as resolved by Cargo).
|
||||
features = buckify_features(list(node.get("features") or []))
|
||||
|
||||
edition = pkg.get("edition") or "2021"
|
||||
ver_major, ver_minor, ver_patch, ver_pre = parse_semver(pkg.get("version") or "0.0.0")
|
||||
|
||||
# Emulate Cargo's CARGO_BIN_EXE_* env vars for all known workspace
|
||||
# binaries. This avoids test helpers trying to discover binaries via a
|
||||
# Cargo target directory layout.
|
||||
#
|
||||
# These are runtime-only and should be provided via `run_env` so we
|
||||
# don't accidentally make compile actions depend on executable outputs.
|
||||
cargo_bin_env = {f"CARGO_BIN_EXE_{k}": f"$(location {v})" for k, v in sorted(cargo_bin_to_label.items())}
|
||||
cargo_run_env = cargo_bin_env
|
||||
|
||||
lines: list[str] = []
|
||||
lines.append("# @generated by scripts/gen_buck_first_party.py")
|
||||
lines.append("# Regenerate with: (cd codex-rs && python3 scripts/gen_buck_first_party.py)")
|
||||
lines.append("")
|
||||
|
||||
# NOTE: We use a broad src glob to support include_str!/include_bytes!
|
||||
# and other build-time file reads without crate-specific buckification.
|
||||
src_glob = 'glob(["**"], exclude = ["BUCK", "target/**"])'
|
||||
snap_resources = collect_snap_resources(crate_dir)
|
||||
|
||||
deps_expr = starlark_deps_expr(base_deps, conditional_deps)
|
||||
|
||||
if lib_t is not None:
|
||||
crate_root = relpath_from_codex_rs(lib_t["src_path"])
|
||||
crate_root_rel = str(pathlib.Path(crate_root).relative_to(crate_dir.relative_to(CODEX_RS_ROOT)))
|
||||
|
||||
lines.append("rust_library(")
|
||||
lines.append(f' name = "{pkg["name"]}",')
|
||||
lines.append(f' crate = "{lib_t["name"]}",')
|
||||
lines.append(f' crate_root = "{crate_root_rel}",')
|
||||
lines.append(f" srcs = {src_glob},")
|
||||
lines.append(f' edition = "{edition}",')
|
||||
lines.append(
|
||||
" env = "
|
||||
+ starlark_dict(
|
||||
{
|
||||
"CARGO_CRATE_NAME": lib_t["name"],
|
||||
"CARGO_MANIFEST_DIR": ".",
|
||||
# Insta snapshots use a compile-time workspace root
|
||||
# (option_env!("INSTA_WORKSPACE_ROOT")) and otherwise
|
||||
# fall back to cargo metadata, which doesn't work under
|
||||
# Buck's test sandbox. Pin this to repo root.
|
||||
"INSTA_WORKSPACE_ROOT": ".",
|
||||
"CARGO_PKG_AUTHORS": "",
|
||||
"CARGO_PKG_DESCRIPTION": "",
|
||||
"CARGO_PKG_NAME": pkg["name"],
|
||||
"CARGO_PKG_REPOSITORY": "",
|
||||
"CARGO_PKG_VERSION": pkg.get("version") or "0.0.0",
|
||||
"CARGO_PKG_VERSION_MAJOR": ver_major,
|
||||
"CARGO_PKG_VERSION_MINOR": ver_minor,
|
||||
"CARGO_PKG_VERSION_PATCH": ver_patch,
|
||||
"CARGO_PKG_VERSION_PRE": ver_pre,
|
||||
},
|
||||
indent=" ",
|
||||
)
|
||||
+ ","
|
||||
)
|
||||
if features:
|
||||
lines.append(f" features = {starlark_list(features)},")
|
||||
lines.append(f" deps = {deps_expr},")
|
||||
if named_deps:
|
||||
items = [f'"{k}": "{v}"' for k, v in sorted(named_deps.items())]
|
||||
lines.append(" named_deps = {")
|
||||
for it in items:
|
||||
lines.append(f" {it},")
|
||||
lines.append(" },")
|
||||
lines.append(' visibility = ["PUBLIC"],')
|
||||
lines.append(")")
|
||||
|
||||
# Unit tests for the library (i.e., `#[cfg(test)]` within src/).
|
||||
test_deps_expr = starlark_deps_expr(test_base_deps, test_conditional_deps)
|
||||
lines.append("")
|
||||
lines.append("rust_test(")
|
||||
lines.append(f' name = "{pkg["name"]}-unit-tests",')
|
||||
lines.append(f' crate = "{lib_t["name"]}",')
|
||||
lines.append(f' crate_root = "{crate_root_rel}",')
|
||||
lines.append(f" srcs = {src_glob},")
|
||||
lines.append(f' edition = "{edition}",')
|
||||
lines.append(
|
||||
" env = "
|
||||
+ starlark_dict(
|
||||
{
|
||||
"CARGO_CRATE_NAME": lib_t["name"],
|
||||
"CARGO_MANIFEST_DIR": ".",
|
||||
"INSTA_WORKSPACE_ROOT": ".",
|
||||
"CARGO_PKG_AUTHORS": "",
|
||||
"CARGO_PKG_DESCRIPTION": "",
|
||||
"CARGO_PKG_NAME": pkg["name"],
|
||||
"CARGO_PKG_REPOSITORY": "",
|
||||
"CARGO_PKG_VERSION": pkg.get("version") or "0.0.0",
|
||||
"CARGO_PKG_VERSION_MAJOR": ver_major,
|
||||
"CARGO_PKG_VERSION_MINOR": ver_minor,
|
||||
"CARGO_PKG_VERSION_PATCH": ver_patch,
|
||||
"CARGO_PKG_VERSION_PRE": ver_pre,
|
||||
},
|
||||
indent=" ",
|
||||
)
|
||||
+ ","
|
||||
)
|
||||
if cargo_run_env:
|
||||
lines.append(" run_env = " + starlark_dict(cargo_run_env, indent=" ") + ",")
|
||||
if snap_resources:
|
||||
lines.append(" resources = " + starlark_dict(snap_resources, indent=" ") + ",")
|
||||
if features:
|
||||
lines.append(f" features = {starlark_list(features)},")
|
||||
lines.append(f" deps = {test_deps_expr},")
|
||||
if test_named_deps:
|
||||
items = [f'"{k}": "{v}"' for k, v in sorted(test_named_deps.items())]
|
||||
lines.append(" named_deps = {")
|
||||
for it in items:
|
||||
lines.append(f" {it},")
|
||||
lines.append(" },")
|
||||
lines.append(' visibility = ["PUBLIC"],')
|
||||
lines.append(")")
|
||||
|
||||
for bin_t in bin_ts:
|
||||
bin_rule_name = buck_bin_rule_name(pkg["name"], bin_t["name"])
|
||||
crate_root = relpath_from_codex_rs(bin_t["src_path"])
|
||||
crate_root_rel = str(pathlib.Path(crate_root).relative_to(crate_dir.relative_to(CODEX_RS_ROOT)))
|
||||
|
||||
# Cargo makes the package's library available to binaries in the
|
||||
# same package, and package dependencies are also visible.
|
||||
bin_base_deps = base_deps
|
||||
if lib_t is not None:
|
||||
bin_base_deps = [f":{pkg['name']}"] + base_deps
|
||||
bin_deps_expr = starlark_deps_expr(bin_base_deps, conditional_deps)
|
||||
|
||||
lines.append("")
|
||||
lines.append("rust_binary(")
|
||||
lines.append(f' name = "{bin_rule_name}",')
|
||||
lines.append(f' crate = "{bin_t["name"]}",')
|
||||
lines.append(f' crate_root = "{crate_root_rel}",')
|
||||
lines.append(f" srcs = {src_glob},")
|
||||
lines.append(f' edition = "{edition}",')
|
||||
lines.append(
|
||||
" env = "
|
||||
+ starlark_dict(
|
||||
{
|
||||
"CARGO_CRATE_NAME": bin_t["name"],
|
||||
"CARGO_MANIFEST_DIR": ".",
|
||||
"INSTA_WORKSPACE_ROOT": ".",
|
||||
"CARGO_PKG_AUTHORS": "",
|
||||
"CARGO_PKG_DESCRIPTION": "",
|
||||
"CARGO_PKG_NAME": pkg["name"],
|
||||
"CARGO_PKG_REPOSITORY": "",
|
||||
"CARGO_PKG_VERSION": pkg.get("version") or "0.0.0",
|
||||
"CARGO_PKG_VERSION_MAJOR": ver_major,
|
||||
"CARGO_PKG_VERSION_MINOR": ver_minor,
|
||||
"CARGO_PKG_VERSION_PATCH": ver_patch,
|
||||
"CARGO_PKG_VERSION_PRE": ver_pre,
|
||||
},
|
||||
indent=" ",
|
||||
)
|
||||
+ ","
|
||||
)
|
||||
if features:
|
||||
lines.append(f" features = {starlark_list(features)},")
|
||||
lines.append(f" deps = {bin_deps_expr},")
|
||||
if named_deps:
|
||||
items = [f'"{k}": "{v}"' for k, v in sorted(named_deps.items())]
|
||||
lines.append(" named_deps = {")
|
||||
for it in items:
|
||||
lines.append(f" {it},")
|
||||
lines.append(" },")
|
||||
lines.append(' visibility = ["PUBLIC"],')
|
||||
lines.append(")")
|
||||
|
||||
# Unit tests for the binary (i.e., `#[cfg(test)]` within src/main.rs).
|
||||
bin_test_base_deps = list(test_base_deps)
|
||||
if lib_t is not None:
|
||||
# Cargo makes the package's library available to binaries; the
|
||||
# unit test harness for `src/main.rs` needs that dependency too.
|
||||
bin_test_base_deps = [f":{pkg['name']}"] + bin_test_base_deps
|
||||
test_deps_expr = starlark_deps_expr(bin_test_base_deps, test_conditional_deps)
|
||||
lines.append("")
|
||||
lines.append("rust_test(")
|
||||
lines.append(f' name = "{bin_rule_name}-unit-tests",')
|
||||
lines.append(f' crate = "{bin_t["name"]}",')
|
||||
lines.append(f' crate_root = "{crate_root_rel}",')
|
||||
lines.append(f" srcs = {src_glob},")
|
||||
lines.append(f' edition = "{edition}",')
|
||||
lines.append(
|
||||
" env = "
|
||||
+ starlark_dict(
|
||||
{
|
||||
"CARGO_CRATE_NAME": bin_t["name"],
|
||||
"CARGO_MANIFEST_DIR": ".",
|
||||
"INSTA_WORKSPACE_ROOT": ".",
|
||||
"CARGO_PKG_AUTHORS": "",
|
||||
"CARGO_PKG_DESCRIPTION": "",
|
||||
"CARGO_PKG_NAME": pkg["name"],
|
||||
"CARGO_PKG_REPOSITORY": "",
|
||||
"CARGO_PKG_VERSION": pkg.get("version") or "0.0.0",
|
||||
"CARGO_PKG_VERSION_MAJOR": ver_major,
|
||||
"CARGO_PKG_VERSION_MINOR": ver_minor,
|
||||
"CARGO_PKG_VERSION_PATCH": ver_patch,
|
||||
"CARGO_PKG_VERSION_PRE": ver_pre,
|
||||
},
|
||||
indent=" ",
|
||||
)
|
||||
+ ","
|
||||
)
|
||||
if cargo_run_env:
|
||||
lines.append(" run_env = " + starlark_dict(cargo_run_env, indent=" ") + ",")
|
||||
if snap_resources:
|
||||
lines.append(" resources = " + starlark_dict(snap_resources, indent=" ") + ",")
|
||||
if features:
|
||||
lines.append(f" features = {starlark_list(features)},")
|
||||
lines.append(f" deps = {test_deps_expr},")
|
||||
if test_named_deps:
|
||||
items = [f'"{k}": "{v}"' for k, v in sorted(test_named_deps.items())]
|
||||
lines.append(" named_deps = {")
|
||||
for it in items:
|
||||
lines.append(f" {it},")
|
||||
lines.append(" },")
|
||||
lines.append(' visibility = ["PUBLIC"],')
|
||||
lines.append(")")
|
||||
|
||||
# Integration tests under `tests/` (Cargo `kind = ["test"]`).
|
||||
#
|
||||
# Cargo gives these crates access to both normal + dev dependencies, and
|
||||
# also makes the package's library available if it exists.
|
||||
if test_ts:
|
||||
test_deps_expr = starlark_deps_expr(test_base_deps, test_conditional_deps)
|
||||
for test_t in test_ts:
|
||||
test_name = test_t["name"]
|
||||
if test_name in SKIP_INTEGRATION_TESTS_BY_PACKAGE.get(pkg["name"], set()):
|
||||
continue
|
||||
crate_root = relpath_from_codex_rs(test_t["src_path"])
|
||||
crate_root_rel = str(pathlib.Path(crate_root).relative_to(crate_dir.relative_to(CODEX_RS_ROOT)))
|
||||
|
||||
integration_base_deps = list(test_base_deps)
|
||||
if lib_t is not None:
|
||||
integration_base_deps = [f":{pkg['name']}"] + integration_base_deps
|
||||
integration_deps_expr = starlark_deps_expr(integration_base_deps, test_conditional_deps)
|
||||
|
||||
lines.append("")
|
||||
lines.append("rust_test(")
|
||||
lines.append(f' name = "{test_name}-integration-test",')
|
||||
lines.append(f' crate = "{test_name}",')
|
||||
lines.append(f' crate_root = "{crate_root_rel}",')
|
||||
lines.append(f" srcs = {src_glob},")
|
||||
lines.append(f' edition = "{edition}",')
|
||||
lines.append(
|
||||
" env = "
|
||||
+ starlark_dict(
|
||||
{
|
||||
"CARGO_CRATE_NAME": test_name,
|
||||
"CARGO_MANIFEST_DIR": ".",
|
||||
"INSTA_WORKSPACE_ROOT": ".",
|
||||
"CARGO_PKG_AUTHORS": "",
|
||||
"CARGO_PKG_DESCRIPTION": "",
|
||||
"CARGO_PKG_NAME": pkg["name"],
|
||||
"CARGO_PKG_REPOSITORY": "",
|
||||
"CARGO_PKG_VERSION": pkg.get("version") or "0.0.0",
|
||||
"CARGO_PKG_VERSION_MAJOR": ver_major,
|
||||
"CARGO_PKG_VERSION_MINOR": ver_minor,
|
||||
"CARGO_PKG_VERSION_PATCH": ver_patch,
|
||||
"CARGO_PKG_VERSION_PRE": ver_pre,
|
||||
},
|
||||
indent=" ",
|
||||
)
|
||||
+ ","
|
||||
)
|
||||
if cargo_run_env:
|
||||
lines.append(" run_env = " + starlark_dict(cargo_run_env, indent=" ") + ",")
|
||||
if snap_resources:
|
||||
lines.append(" resources = " + starlark_dict(snap_resources, indent=" ") + ",")
|
||||
if features:
|
||||
lines.append(f" features = {starlark_list(features)},")
|
||||
lines.append(f" deps = {integration_deps_expr},")
|
||||
if test_named_deps:
|
||||
items = [f'"{k}": "{v}"' for k, v in sorted(test_named_deps.items())]
|
||||
lines.append(" named_deps = {")
|
||||
for it in items:
|
||||
lines.append(f" {it},")
|
||||
lines.append(" },")
|
||||
lines.append(' visibility = ["PUBLIC"],')
|
||||
lines.append(")")
|
||||
|
||||
write_buck_file(crate_dir, "\n".join(lines) + "\n")
|
||||
generated_buck_files.append(crate_dir / "BUCK")
|
||||
|
||||
buildifier(generated_buck_files)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
351
codex-rs/scripts/gen_reindeer_fixups.py
Normal file
351
codex-rs/scripts/gen_reindeer_fixups.py
Normal file
@@ -0,0 +1,351 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Generate Reindeer fixups for third-party crates.
|
||||
|
||||
Reindeer requires an explicit decision for each crate with a build script:
|
||||
either run it or ignore it. For most third-party crates we run build scripts.
|
||||
|
||||
We intentionally do not generate `extra_srcs` fixups here:
|
||||
- Reindeer validates fixup globs against its chosen crate source directory,
|
||||
and with vendoring enabled this can be a filtered view of the package that
|
||||
does not include top-level docs/fixtures (README, tests data, etc).
|
||||
- Instead, we include common non-Rust sources via the `codex_rust_*` wrapper
|
||||
macros in `codex-rs/buck2/reindeer_macros.bzl` using Buck `glob(...)`.
|
||||
|
||||
This script is checked in; its outputs are not.
|
||||
|
||||
The generated fixups live under `codex-rs/third-party/fixups/` and are
|
||||
intentionally gitignored for now (see the repo root `.gitignore`, which ignores
|
||||
`codex-rs/third-party/`). This script is designed to be re-run and will
|
||||
overwrite any existing generated fixups.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
|
||||
CODEX_RS_ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
THIRD_PARTY_DIR = CODEX_RS_ROOT / "third-party"
|
||||
VENDOR_DIR = THIRD_PARTY_DIR / "vendor"
|
||||
FIXUPS_DIR = THIRD_PARTY_DIR / "fixups"
|
||||
|
||||
|
||||
def cargo_metadata() -> dict[str, Any]:
|
||||
out = subprocess.check_output(
|
||||
["cargo", "metadata", "--format-version=1", "--locked"],
|
||||
cwd=CODEX_RS_ROOT,
|
||||
text=True,
|
||||
)
|
||||
return json.loads(out)
|
||||
|
||||
|
||||
def reindeer_reachable_package_ids(meta: dict[str, Any]) -> set[str]:
|
||||
"""
|
||||
Approximate the set of packages Reindeer cares about for `buckify`.
|
||||
|
||||
Reindeer buckifies dependencies of workspace members, but it does not need
|
||||
deps that are only reachable through `dev` edges (tests/benches/examples).
|
||||
"""
|
||||
|
||||
resolve = meta.get("resolve") or {}
|
||||
nodes = resolve.get("nodes") or []
|
||||
by_id: dict[str, Any] = {n["id"]: n for n in nodes}
|
||||
|
||||
roots = list(meta.get("workspace_members") or [])
|
||||
reachable: set[str] = set()
|
||||
stack = list(roots)
|
||||
while stack:
|
||||
pkg_id = stack.pop()
|
||||
if pkg_id in reachable:
|
||||
continue
|
||||
reachable.add(pkg_id)
|
||||
|
||||
node = by_id.get(pkg_id)
|
||||
if not node:
|
||||
continue
|
||||
|
||||
for dep in node.get("deps") or []:
|
||||
kinds = dep.get("dep_kinds") or []
|
||||
if kinds and all(k.get("kind") == "dev" for k in kinds):
|
||||
continue
|
||||
stack.append(dep["pkg"])
|
||||
|
||||
return reachable
|
||||
|
||||
|
||||
def vendored_links_value(name: str, version: str) -> str | None:
|
||||
"""
|
||||
If the vendored crate declares `links = "..."` in its Cargo.toml, return it.
|
||||
|
||||
Cargo provides this to build scripts via the CARGO_MANIFEST_LINKS env var.
|
||||
Some build scripts rely on it (e.g. ring).
|
||||
"""
|
||||
|
||||
cargo_toml = VENDOR_DIR / f"{name}-{version}" / "Cargo.toml"
|
||||
if not cargo_toml.exists():
|
||||
return None
|
||||
|
||||
text = cargo_toml.read_text(encoding="utf-8", errors="replace")
|
||||
|
||||
# Very small parser: search within the [package] section first, then fall back.
|
||||
pkg_idx = text.find("[package]")
|
||||
if pkg_idx != -1:
|
||||
rest = text[pkg_idx:]
|
||||
next_table = rest.find("\n[", 1)
|
||||
pkg_block = rest if next_table == -1 else rest[:next_table]
|
||||
m = re.search(r'(?m)^\s*links\s*=\s*"([^"]+)"\s*$', pkg_block)
|
||||
if m:
|
||||
return m.group(1)
|
||||
|
||||
m = re.search(r'(?m)^\s*links\s*=\s*"([^"]+)"\s*$', text)
|
||||
if m:
|
||||
return m.group(1)
|
||||
return None
|
||||
|
||||
|
||||
def openssl_dep_version_number_hex() -> str | None:
|
||||
"""
|
||||
Derive a DEP_OPENSSL_VERSION_NUMBER value (hex string, no 0x prefix).
|
||||
|
||||
Under Cargo, openssl-sys emits a `cargo:version_number=...` metadata line
|
||||
and Cargo converts it into DEP_OPENSSL_VERSION_NUMBER for dependents.
|
||||
Buck's buildscript runner does not currently propagate those DEP_* env vars,
|
||||
so we approximate the value here for Buck builds.
|
||||
"""
|
||||
|
||||
def pkg_config_modversion() -> str | None:
|
||||
try:
|
||||
out = subprocess.check_output(
|
||||
["pkg-config", "--modversion", "openssl"],
|
||||
cwd=CODEX_RS_ROOT,
|
||||
text=True,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
return None
|
||||
v = out.strip()
|
||||
return v if v else None
|
||||
|
||||
def parse_openssl_version(
|
||||
v: str,
|
||||
) -> tuple[int, int, int, str | None] | None:
|
||||
"""
|
||||
Parse versions like:
|
||||
- 3.0.13
|
||||
- 1.1.1w
|
||||
- 1.1.0h
|
||||
"""
|
||||
|
||||
m = re.match(r"^(\d+)\.(\d+)\.(\d+)([a-z])?$", v)
|
||||
if not m:
|
||||
return None
|
||||
major, minor, patch = (int(m.group(1)), int(m.group(2)), int(m.group(3)))
|
||||
suffix = m.group(4)
|
||||
return (major, minor, patch, suffix)
|
||||
|
||||
v = pkg_config_modversion()
|
||||
parsed = parse_openssl_version(v) if v else None
|
||||
|
||||
# In CI we install libssl-dev, so pkg-config should generally be available.
|
||||
# If it isn't, guess OpenSSL 3.x on modern Linux distros.
|
||||
if parsed is None:
|
||||
if os.name == "posix":
|
||||
# 3.0.0: (3<<28)|(0<<20)|(0<<4)
|
||||
return f"{(3 << 28):x}"
|
||||
return None
|
||||
|
||||
major, minor, patch, suffix = parsed
|
||||
|
||||
# OpenSSL 3 uses (major<<28)|(minor<<20)|(patch<<4).
|
||||
if major >= 3:
|
||||
version = (major << 28) | (minor << 20) | (patch << 4)
|
||||
return f"{version:x}"
|
||||
|
||||
# OpenSSL 1.x uses 0xMNNFFPPS where PP is the patch number and S is the
|
||||
# patch status nibble. For lettered patch releases, 'a' => 1, 'b' => 2, etc.
|
||||
if major == 1:
|
||||
patch_num = 0
|
||||
if suffix:
|
||||
patch_num = ord(suffix) - ord("a") + 1
|
||||
# Release status (0xf) matches Cargo/rust-openssl default behavior.
|
||||
version = (major << 28) | (minor << 20) | (patch << 12) | (patch_num << 4) | 0xF
|
||||
return f"{version:x}"
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def openssl_dep_conf() -> str | None:
|
||||
"""
|
||||
Derive a DEP_OPENSSL_CONF value (comma-separated macro names).
|
||||
|
||||
Under Cargo, openssl-sys emits `cargo:conf=...` and Cargo converts it into
|
||||
DEP_OPENSSL_CONF for dependents. The `openssl` crate's build script reads
|
||||
DEP_OPENSSL_CONF and re-emits per-macro `osslconf="..."` cfgs so it can
|
||||
conditionalize APIs that are disabled in the system OpenSSL build (e.g.
|
||||
OPENSSL_NO_IDEA).
|
||||
"""
|
||||
|
||||
def pkg_config_cflags() -> list[str] | None:
|
||||
try:
|
||||
out = subprocess.check_output(
|
||||
["pkg-config", "--cflags", "openssl"],
|
||||
cwd=CODEX_RS_ROOT,
|
||||
text=True,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
return None
|
||||
return [w for w in out.split() if w]
|
||||
|
||||
def preprocess_opensslconf(cflags: list[str]) -> str | None:
|
||||
snippet = "#include <openssl/opensslconf.h>\n"
|
||||
for cc in ("cc", "clang"):
|
||||
try:
|
||||
return subprocess.check_output(
|
||||
[cc, "-dM", "-E", "-xc", "-", *cflags],
|
||||
cwd=CODEX_RS_ROOT,
|
||||
input=snippet,
|
||||
text=True,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
continue
|
||||
return None
|
||||
|
||||
cflags = pkg_config_cflags()
|
||||
if not cflags:
|
||||
return None
|
||||
|
||||
out = preprocess_opensslconf(cflags)
|
||||
if not out:
|
||||
return None
|
||||
|
||||
# Match the behavior expected by openssl/build.rs: it splits DEP_OPENSSL_CONF
|
||||
# by ',' and emits cfgs `osslconf="..."` for each token.
|
||||
conf: set[str] = set()
|
||||
for line in out.splitlines():
|
||||
# Lines look like: "#define OPENSSL_NO_IDEA 1"
|
||||
m = re.match(r"^#define\s+(OPENSSL_[A-Z0-9_]+)\b", line)
|
||||
if not m:
|
||||
continue
|
||||
macro = m.group(1)
|
||||
if macro.startswith("OPENSSL_NO_"):
|
||||
conf.add(macro)
|
||||
return ",".join(sorted(conf)) if conf else ""
|
||||
|
||||
|
||||
def toml_inline_table(entries: dict[str, str]) -> str:
|
||||
# Emit stable TOML with double-quoted strings.
|
||||
parts = [f"{k} = {json.dumps(v)}" for k, v in sorted(entries.items())]
|
||||
return "{ " + ", ".join(parts) + " }"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
meta = cargo_metadata()
|
||||
packages = meta.get("packages", [])
|
||||
reachable_ids = reindeer_reachable_package_ids(meta)
|
||||
|
||||
# First-party packages whose build.rs we intentionally ignore under Buck.
|
||||
buildscript_run_overrides: dict[str, bool] = {
|
||||
# build.rs only adds rerun-if-changed
|
||||
"codex-execpolicy-legacy": False,
|
||||
# Windows-only resource compilation
|
||||
"codex-windows-sandbox": False,
|
||||
}
|
||||
|
||||
# Packages where Cargo metadata reports a build script, but Reindeer does not
|
||||
# require/accept a buildscript fixup in this workspace.
|
||||
skip_packages: set[str] = {
|
||||
"indexmap",
|
||||
"quinn",
|
||||
}
|
||||
|
||||
reachable_pkgs = [p for p in packages if p["id"] in reachable_ids]
|
||||
|
||||
# name -> version -> has_buildscript
|
||||
by_name: dict[str, dict[str, bool]] = {}
|
||||
for pkg in reachable_pkgs:
|
||||
name = pkg["name"]
|
||||
version = pkg["version"]
|
||||
has_buildscript = any(
|
||||
"custom-build" in (t.get("kind") or []) for t in (pkg.get("targets") or [])
|
||||
)
|
||||
by_name.setdefault(name, {})[version] = has_buildscript
|
||||
|
||||
# Nuke and regenerate: fixups are gitignored and generated.
|
||||
if FIXUPS_DIR.exists():
|
||||
shutil.rmtree(FIXUPS_DIR)
|
||||
FIXUPS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
wrote = 0
|
||||
for name, versions in sorted(by_name.items()):
|
||||
if name in skip_packages:
|
||||
continue
|
||||
|
||||
buildscript_versions = sorted([v for v, has in versions.items() if has])
|
||||
if not buildscript_versions:
|
||||
continue
|
||||
|
||||
run = buildscript_run_overrides.get(name, True)
|
||||
|
||||
fixup_path = FIXUPS_DIR / name / "fixups.toml"
|
||||
fixup_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
stanzas: list[str] = []
|
||||
for v in buildscript_versions:
|
||||
if run:
|
||||
links = vendored_links_value(name, v)
|
||||
stanzas.append(f"['cfg(version = \"={v}\")'.buildscript.run]")
|
||||
stanzas.append("rustc_link_lib = true")
|
||||
stanzas.append("rustc_link_search = true")
|
||||
env_vars: dict[str, str] = {}
|
||||
# Some crates rely on this being set when `links = "..."`
|
||||
# exists in Cargo.toml (Cargo passes it to build scripts).
|
||||
if links:
|
||||
env_vars["CARGO_MANIFEST_LINKS"] = links
|
||||
# The `openssl` crate's build script derives cfgs from
|
||||
# DEP_OPENSSL_VERSION_NUMBER (emitted by openssl-sys). Buck's
|
||||
# buildscript runner does not currently propagate those DEP_*
|
||||
# env vars, so approximate the value for Buck builds.
|
||||
if name == "openssl" and v == "0.10.73":
|
||||
dep_ver = openssl_dep_version_number_hex()
|
||||
if dep_ver:
|
||||
env_vars["DEP_OPENSSL_VERSION_NUMBER"] = dep_ver
|
||||
dep_conf = openssl_dep_conf()
|
||||
if dep_conf is not None:
|
||||
# Empty string is meaningful: "no conf macros enabled".
|
||||
env_vars["DEP_OPENSSL_CONF"] = dep_conf
|
||||
elif sys.platform.startswith("linux"):
|
||||
# If we can't determine the conf macros, be conservative
|
||||
# on Linux (Ubuntu CI commonly disables IDEA).
|
||||
env_vars["DEP_OPENSSL_CONF"] = "OPENSSL_NO_IDEA"
|
||||
if env_vars:
|
||||
stanzas.append(f"env = {toml_inline_table(env_vars)}")
|
||||
stanzas.append("")
|
||||
else:
|
||||
stanzas.append(f"['cfg(version = \"={v}\")']")
|
||||
stanzas.append("buildscript.run = false")
|
||||
stanzas.append("")
|
||||
|
||||
content = "\n".join(stanzas).rstrip() + "\n"
|
||||
fixup_path.write_text(content, encoding="utf-8")
|
||||
wrote += 1
|
||||
|
||||
if wrote:
|
||||
print(
|
||||
f"Wrote buildscript fixups.toml for {wrote} crates under "
|
||||
f"{os.path.relpath(FIXUPS_DIR, CODEX_RS_ROOT)}"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
510
codex-rs/scripts/patch_third_party_buck_for_tests.py
Normal file
510
codex-rs/scripts/patch_third_party_buck_for_tests.py
Normal file
@@ -0,0 +1,510 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Patch the Reindeer-generated `codex-rs/third-party/BUCK` to add a small overlay
|
||||
of third-party crates that are only used as *dev-dependencies* in the Cargo
|
||||
workspace.
|
||||
|
||||
Reindeer currently generates Buck targets for Cargo "normal" deps (plus build
|
||||
deps for build scripts), but it does not emit targets for dev-dependencies
|
||||
because it does not generate rules for Cargo test/example/bench targets.
|
||||
|
||||
For Buck2-driven `rust_test()` targets in first-party crates, we still need a
|
||||
handful of dev-only third-party crates to exist as targets in
|
||||
`//codex-rs/third-party`.
|
||||
|
||||
This script is run as part of `scripts/setup_buck2_local.sh` and is intended to
|
||||
be safe to run repeatedly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Optional
|
||||
|
||||
|
||||
CODEX_RS_ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
REPO_ROOT = CODEX_RS_ROOT.parent
|
||||
BUILDIFIER = REPO_ROOT / "scripts" / "buildifier"
|
||||
THIRD_PARTY_BUCK = CODEX_RS_ROOT / "third-party" / "BUCK"
|
||||
VENDOR_DIR = CODEX_RS_ROOT / "third-party" / "vendor"
|
||||
|
||||
BEGIN_MARKER = "# BEGIN CODEX LOCAL DEV TEST DEPS (generated)\n"
|
||||
END_MARKER = "# END CODEX LOCAL DEV TEST DEPS (generated)\n"
|
||||
|
||||
|
||||
def cargo_metadata() -> dict[str, Any]:
|
||||
out = subprocess.check_output(
|
||||
["cargo", "metadata", "--format-version=1", "--locked"],
|
||||
cwd=CODEX_RS_ROOT,
|
||||
text=True,
|
||||
)
|
||||
return json.loads(out)
|
||||
|
||||
|
||||
def parse_semver(version: str) -> tuple[str, str, str, str]:
|
||||
m = re.match(r"^(\d+)\.(\d+)\.(\d+)(?:-(.*))?$", version)
|
||||
if not m:
|
||||
return ("0", "0", "0", "")
|
||||
return (m.group(1), m.group(2), m.group(3), m.group(4) or "")
|
||||
|
||||
|
||||
def parse_dep_cfg(cfg: Optional[str]) -> list[str]:
|
||||
"""
|
||||
Best-effort mapping for cfg(...) strings to Buck2 constraint keys.
|
||||
This mirrors the mapping in scripts/gen_buck_first_party.py.
|
||||
"""
|
||||
if cfg is None:
|
||||
return []
|
||||
|
||||
cfg = cfg.strip()
|
||||
m = re.fullmatch(r'cfg\(target_os\s*=\s*"([^"]+)"\)', cfg)
|
||||
if m:
|
||||
os_name = m.group(1)
|
||||
if os_name in ("linux", "macos", "windows"):
|
||||
return [f"prelude//os:{os_name}"]
|
||||
|
||||
if cfg == "cfg(windows)":
|
||||
return ["prelude//os:windows"]
|
||||
|
||||
if cfg == "cfg(unix)":
|
||||
return ["prelude//os:linux", "prelude//os:macos"]
|
||||
|
||||
return []
|
||||
|
||||
|
||||
def starlark_list(items: list[str], indent: str = " ") -> str:
|
||||
if not items:
|
||||
return "[]"
|
||||
lines = ["["]
|
||||
for it in items:
|
||||
lines.append(f'{indent}"{it}",')
|
||||
lines.append("]")
|
||||
return "\n".join(lines)
|
||||
|
||||
def starlark_str(s: str) -> str:
|
||||
# Conservative escaping suitable for Starlark string literals.
|
||||
return (
|
||||
s.replace("\\", "\\\\")
|
||||
.replace('"', '\\"')
|
||||
.replace("\r", "\\r")
|
||||
.replace("\n", "\\n")
|
||||
)
|
||||
|
||||
|
||||
def starlark_dict(d: dict[str, str], indent: str = " ") -> str:
|
||||
if not d:
|
||||
return "{}"
|
||||
lines = ["{"]
|
||||
for k, v in d.items():
|
||||
lines.append(f'{indent}"{starlark_str(k)}": "{starlark_str(v)}",')
|
||||
lines.append("}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def starlark_deps_expr(base: list[str], conditional: dict[str, list[str]]) -> str:
|
||||
expr = starlark_list(base)
|
||||
for constraint, deps in sorted(conditional.items()):
|
||||
expr = (
|
||||
f"{expr} + select({{\n"
|
||||
f' "{constraint}": {starlark_list(deps, indent=" ")},\n'
|
||||
' "DEFAULT": [],\n'
|
||||
"})"
|
||||
)
|
||||
return expr
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BuckDep:
|
||||
label: str
|
||||
local_name: str
|
||||
crate_name: str
|
||||
cfg: Optional[str]
|
||||
|
||||
|
||||
def group_deps(deps: list[BuckDep]) -> tuple[list[str], dict[str, list[str]], dict[str, str]]:
|
||||
base_deps: list[str] = []
|
||||
conditional_deps: dict[str, list[str]] = {}
|
||||
named_deps: dict[str, str] = {}
|
||||
for d in deps:
|
||||
dep_constraints = parse_dep_cfg(d.cfg)
|
||||
if dep_constraints:
|
||||
for c in dep_constraints:
|
||||
conditional_deps.setdefault(c, []).append(d.label)
|
||||
else:
|
||||
base_deps.append(d.label)
|
||||
|
||||
if d.local_name != d.crate_name:
|
||||
named_deps[d.local_name] = d.label
|
||||
|
||||
base_deps = sorted(set(base_deps))
|
||||
for k in list(conditional_deps.keys()):
|
||||
conditional_deps[k] = sorted(set(conditional_deps[k]))
|
||||
|
||||
return (base_deps, conditional_deps, named_deps)
|
||||
|
||||
|
||||
def strip_existing_overlay(text: str) -> str:
|
||||
start = text.find(BEGIN_MARKER)
|
||||
if start == -1:
|
||||
return text
|
||||
end = text.find(END_MARKER, start)
|
||||
if end == -1:
|
||||
# If the file is corrupted, be conservative and do not try to patch.
|
||||
raise SystemExit(f"Found BEGIN marker but not END marker in {THIRD_PARTY_BUCK}")
|
||||
return text[:start] + text[end + len(END_MARKER) :]
|
||||
|
||||
|
||||
def patch_reindeer_output_for_test_features(text: str) -> str:
|
||||
"""
|
||||
Best-effort patching of Reindeer output to make certain dev-only crates build.
|
||||
|
||||
Today, our locally-generated targets for dev-dependencies (e.g. `insta`) may
|
||||
require feature combinations on their transitive deps that are not enabled
|
||||
by the normal (non-test) workspace graph that Reindeer buckifies.
|
||||
|
||||
We keep this narrowly-scoped and idempotent.
|
||||
"""
|
||||
# `insta` requires `similar` with the `inline` feature.
|
||||
similar_rule_name = "similar-2.7.0"
|
||||
lines = text.splitlines(keepends=True)
|
||||
for i, line in enumerate(lines):
|
||||
if "name" in line and similar_rule_name in line and "name" in line:
|
||||
# Walk until the end of this rule.
|
||||
j = i
|
||||
while j < len(lines) and lines[j].strip() != ")":
|
||||
j += 1
|
||||
if j >= len(lines):
|
||||
break
|
||||
|
||||
block = lines[i : j + 1]
|
||||
# Only treat it as patched if the *feature* is present. The crate
|
||||
# itself contains an `inline.rs` source file, so substring checks
|
||||
# would be a false positive.
|
||||
if any(l.strip() == '"inline",' for l in block):
|
||||
break
|
||||
|
||||
for k, bl in enumerate(block):
|
||||
if bl.lstrip().startswith("features = ["):
|
||||
block.insert(k + 1, ' "inline",\n')
|
||||
lines[i : j + 1] = block
|
||||
return "".join(lines)
|
||||
|
||||
# No existing features list; add one just before `visibility`.
|
||||
for k, bl in enumerate(block):
|
||||
if bl.lstrip().startswith("visibility ="):
|
||||
block.insert(k, ' features = ["inline"],\n')
|
||||
lines[i : j + 1] = block
|
||||
return "".join(lines)
|
||||
break
|
||||
|
||||
return text
|
||||
|
||||
|
||||
def present_target_names(text: str) -> set[str]:
|
||||
# This is intentionally simple: it's used only to decide whether a target
|
||||
# already exists so we don't duplicate it.
|
||||
# Reindeer emits string literals with escaped quotes (e.g. `name = \"foo\"`)
|
||||
# so we accept an optional backslash before both quote characters.
|
||||
return set(re.findall(r'(?m)^\s*name\s*=\s*\\?"([^"]+)\\?"', text))
|
||||
|
||||
|
||||
def vendor_crate_root(pkg_name: str, version: str) -> pathlib.Path:
|
||||
crate_dir = VENDOR_DIR / f"{pkg_name}-{version}"
|
||||
# Prefer src/lib.rs, but fall back to whichever crate root Cargo reports.
|
||||
lib_rs = crate_dir / "src" / "lib.rs"
|
||||
if lib_rs.exists():
|
||||
# Paths in `codex-rs/third-party/BUCK` are relative to the `third-party`
|
||||
# package directory, so use `vendor/...` rather than an absolute path or
|
||||
# a `third-party/vendor/...` path.
|
||||
return pathlib.Path(f"vendor/{pkg_name}-{version}/src/lib.rs")
|
||||
raise SystemExit(f"Could not find {crate_dir}/src/lib.rs (needed for {pkg_name} {version})")
|
||||
|
||||
|
||||
def append_overlay(meta: dict[str, Any], existing_buck: str) -> str:
|
||||
by_id: dict[str, Any] = {p["id"]: p for p in (meta.get("packages") or [])}
|
||||
workspace_members: set[str] = set(meta.get("workspace_members") or [])
|
||||
node_by_id: dict[str, Any] = {n["id"]: n for n in (meta.get("resolve") or {}).get("nodes") or []}
|
||||
|
||||
existing_targets = present_target_names(existing_buck)
|
||||
|
||||
def target_name_for_pkg_id(pid: str) -> str:
|
||||
p = by_id[pid]
|
||||
return f"{p['name']}-{p['version']}"
|
||||
|
||||
# Find third-party packages that are reachable via any dev-dependency edge
|
||||
# from any workspace member.
|
||||
dev_needed_pkg_ids: set[str] = set()
|
||||
for pkg_id in workspace_members:
|
||||
node = node_by_id.get(pkg_id) or {}
|
||||
for dep in node.get("deps") or []:
|
||||
dep_id = dep["pkg"]
|
||||
if dep_id in workspace_members:
|
||||
continue
|
||||
dep_kinds = dep.get("dep_kinds") or []
|
||||
if not any(k.get("kind") == "dev" for k in dep_kinds):
|
||||
continue
|
||||
dep_pkg = by_id.get(dep_id)
|
||||
if not dep_pkg:
|
||||
continue
|
||||
# Only patch registry crates. Workspace members have source=None.
|
||||
if dep_pkg.get("source") is None:
|
||||
continue
|
||||
dev_needed_pkg_ids.add(dep_id)
|
||||
|
||||
# Reindeer buckify does not emit targets for dev-deps. We patch in any dev
|
||||
# deps that are missing *and* any of their transitive normal deps that are
|
||||
# also missing (e.g., proc-macro helper crates).
|
||||
to_add: set[str] = {pid for pid in dev_needed_pkg_ids if target_name_for_pkg_id(pid) not in existing_targets}
|
||||
queue = list(to_add)
|
||||
|
||||
while queue:
|
||||
pid = queue.pop()
|
||||
node = node_by_id.get(pid) or {}
|
||||
for dep in node.get("deps") or []:
|
||||
dep_id = dep["pkg"]
|
||||
dep_pkg = by_id.get(dep_id)
|
||||
if not dep_pkg:
|
||||
continue
|
||||
if dep_pkg.get("source") is None:
|
||||
continue
|
||||
|
||||
dep_kinds = dep.get("dep_kinds") or []
|
||||
if not any(k.get("kind") is None or k.get("kind") == "normal" for k in dep_kinds):
|
||||
continue
|
||||
|
||||
tgt = target_name_for_pkg_id(dep_id)
|
||||
if tgt in existing_targets or dep_id in to_add:
|
||||
continue
|
||||
|
||||
to_add.add(dep_id)
|
||||
queue.append(dep_id)
|
||||
|
||||
missing_pkg_ids = sorted(to_add, key=lambda pid: (by_id[pid]["name"], by_id[pid]["version"]))
|
||||
if os.environ.get("CODEX_PATCH_THIRD_PARTY_DEBUG") == "1":
|
||||
missing_targets = [target_name_for_pkg_id(pid) for pid in missing_pkg_ids]
|
||||
preview = existing_buck[:120].replace("\n", "\\n")
|
||||
print(f"patch_third_party_buck_for_tests: buck_preview={preview!r}")
|
||||
print(f"patch_third_party_buck_for_tests: existing_targets={len(existing_targets)}")
|
||||
print(f"patch_third_party_buck_for_tests: dev_needed={len(dev_needed_pkg_ids)} missing={len(missing_targets)}")
|
||||
for t in sorted(missing_targets)[:50]:
|
||||
print(f" missing: {t}")
|
||||
if not missing_pkg_ids:
|
||||
return existing_buck
|
||||
|
||||
lines: list[str] = []
|
||||
lines.append("\n" + BEGIN_MARKER.rstrip("\n"))
|
||||
lines.append("# This section is appended by codex-rs/scripts/patch_third_party_buck_for_tests.py")
|
||||
lines.append("# to make Buck `rust_test()` targets usable for first-party dev-deps.")
|
||||
lines.append("")
|
||||
|
||||
for pkg_id in missing_pkg_ids:
|
||||
pkg = by_id[pkg_id]
|
||||
name = pkg["name"]
|
||||
version = pkg["version"]
|
||||
rule_name = f"{name}-{version}"
|
||||
if rule_name in existing_targets:
|
||||
continue
|
||||
|
||||
# Find the lib target to determine the Rust crate name and edition.
|
||||
lib_targets = [
|
||||
t
|
||||
for t in (pkg.get("targets") or [])
|
||||
if "lib" in (t.get("kind") or []) or "proc-macro" in (t.get("kind") or [])
|
||||
]
|
||||
if not lib_targets:
|
||||
# These dev deps should all be libraries.
|
||||
continue
|
||||
lib_t = lib_targets[0]
|
||||
crate_name = lib_t["name"]
|
||||
edition = pkg.get("edition") or "2021"
|
||||
ver_major, ver_minor, ver_patch, ver_pre = parse_semver(version)
|
||||
proc_macro = "proc-macro" in (lib_t.get("kind") or [])
|
||||
|
||||
# Compute deps for the library (normal deps only).
|
||||
node = node_by_id.get(pkg_id) or {}
|
||||
deps: list[BuckDep] = []
|
||||
for dep in node.get("deps") or []:
|
||||
dep_id = dep["pkg"]
|
||||
dep_pkg = by_id.get(dep_id)
|
||||
if not dep_pkg:
|
||||
continue
|
||||
|
||||
dep_kinds = dep.get("dep_kinds") or []
|
||||
local_name = dep.get("name") or dep_pkg["name"]
|
||||
dep_crate_name = dep_pkg.get("targets", [{}])[0].get("name") or local_name
|
||||
dep_label = f":{dep_pkg['name']}-{dep_pkg['version']}"
|
||||
|
||||
# One dependency may have multiple cfg(...) selectors; include them all.
|
||||
for k in dep_kinds:
|
||||
kind = k.get("kind")
|
||||
if kind is None or kind == "normal":
|
||||
deps.append(
|
||||
BuckDep(
|
||||
label=dep_label,
|
||||
local_name=local_name,
|
||||
crate_name=dep_crate_name,
|
||||
cfg=k.get("target"),
|
||||
)
|
||||
)
|
||||
|
||||
base_deps, conditional_deps, named_deps = group_deps(deps)
|
||||
deps_expr = starlark_deps_expr(base_deps, conditional_deps)
|
||||
|
||||
crate_root_rel = vendor_crate_root(name, version)
|
||||
|
||||
env = {
|
||||
"CARGO_CRATE_NAME": crate_name,
|
||||
"CARGO_MANIFEST_DIR": f"vendor/{name}-{version}",
|
||||
"CARGO_PKG_AUTHORS": ":".join(pkg.get("authors") or []),
|
||||
"CARGO_PKG_DESCRIPTION": pkg.get("description") or "",
|
||||
"CARGO_PKG_NAME": name,
|
||||
"CARGO_PKG_REPOSITORY": pkg.get("repository") or "",
|
||||
"CARGO_PKG_VERSION": version,
|
||||
"CARGO_PKG_VERSION_MAJOR": ver_major,
|
||||
"CARGO_PKG_VERSION_MINOR": ver_minor,
|
||||
"CARGO_PKG_VERSION_PATCH": ver_patch,
|
||||
"CARGO_PKG_VERSION_PRE": ver_pre,
|
||||
}
|
||||
|
||||
has_build_rs = (VENDOR_DIR / f"{name}-{version}" / "build.rs").exists()
|
||||
if has_build_rs:
|
||||
build_bin = f"{rule_name}-build-script-build"
|
||||
build_run = f"{rule_name}-build-script-run"
|
||||
build_rs_rel = pathlib.Path(f"vendor/{name}-{version}/build.rs")
|
||||
|
||||
# Build deps for compiling the build script itself.
|
||||
build_deps: list[BuckDep] = []
|
||||
for dep in node.get("deps") or []:
|
||||
dep_id = dep["pkg"]
|
||||
dep_pkg = by_id.get(dep_id)
|
||||
if not dep_pkg:
|
||||
continue
|
||||
dep_kinds = dep.get("dep_kinds") or []
|
||||
local_name = dep.get("name") or dep_pkg["name"]
|
||||
dep_crate_name = dep_pkg.get("targets", [{}])[0].get("name") or local_name
|
||||
dep_label = f":{dep_pkg['name']}-{dep_pkg['version']}"
|
||||
for k in dep_kinds:
|
||||
if k.get("kind") == "build":
|
||||
build_deps.append(
|
||||
BuckDep(
|
||||
label=dep_label,
|
||||
local_name=local_name,
|
||||
crate_name=dep_crate_name,
|
||||
cfg=k.get("target"),
|
||||
)
|
||||
)
|
||||
build_base_deps, build_conditional_deps, build_named_deps = group_deps(build_deps)
|
||||
build_deps_expr = starlark_deps_expr(build_base_deps, build_conditional_deps)
|
||||
|
||||
lines.append("codex_rust_binary(")
|
||||
lines.append(f' name = "{build_bin}",')
|
||||
lines.append(f' srcs = ["{build_rs_rel}"],')
|
||||
lines.append(' crate = "build_script_build",')
|
||||
lines.append(f' crate_root = "{build_rs_rel}",')
|
||||
lines.append(f' edition = "{edition}",')
|
||||
lines.append(" env = " + starlark_dict({**env, "CARGO_CRATE_NAME": "build_script_build"}, indent=" ") + ",")
|
||||
lines.append(' visibility = [],')
|
||||
if build_base_deps or build_conditional_deps:
|
||||
lines.append(f" deps = {build_deps_expr},")
|
||||
if build_named_deps:
|
||||
items = [f'"{k}": "{v}"' for k, v in sorted(build_named_deps.items())]
|
||||
lines.append(" named_deps = {")
|
||||
for it in items:
|
||||
lines.append(f" {it},")
|
||||
lines.append(" },")
|
||||
lines.append(")")
|
||||
lines.append("")
|
||||
|
||||
lines.append("codex_buildscript_run(")
|
||||
lines.append(f' name = "{build_run}",')
|
||||
lines.append(f' package_name = "{name}",')
|
||||
lines.append(f' buildscript_rule = ":{build_bin}",')
|
||||
# Keep env small; codex_buildscript_run fills in common profile vars.
|
||||
lines.append(
|
||||
" env = "
|
||||
+ starlark_dict(
|
||||
{
|
||||
"CARGO_PKG_AUTHORS": env["CARGO_PKG_AUTHORS"],
|
||||
"CARGO_PKG_DESCRIPTION": env["CARGO_PKG_DESCRIPTION"],
|
||||
"CARGO_PKG_REPOSITORY": env["CARGO_PKG_REPOSITORY"],
|
||||
"CARGO_PKG_VERSION_MAJOR": env["CARGO_PKG_VERSION_MAJOR"],
|
||||
"CARGO_PKG_VERSION_MINOR": env["CARGO_PKG_VERSION_MINOR"],
|
||||
"CARGO_PKG_VERSION_PATCH": env["CARGO_PKG_VERSION_PATCH"],
|
||||
"CARGO_PKG_VERSION_PRE": env["CARGO_PKG_VERSION_PRE"],
|
||||
},
|
||||
indent=" ",
|
||||
)
|
||||
+ ","
|
||||
)
|
||||
lines.append(" rustc_link_lib = True,")
|
||||
lines.append(" rustc_link_search = True,")
|
||||
lines.append(f' version = "{version}",')
|
||||
lines.append(")")
|
||||
lines.append("")
|
||||
|
||||
# Build-script outputs for crates that use `OUT_DIR`.
|
||||
env = {**env, "OUT_DIR": f"$(location :{build_run}[out_dir])"}
|
||||
|
||||
lines.append("codex_rust_library(")
|
||||
lines.append(f' name = "{rule_name}",')
|
||||
lines.append(f' srcs = ["{crate_root_rel}"],')
|
||||
lines.append(f' crate = "{crate_name}",')
|
||||
lines.append(f' crate_root = "{crate_root_rel}",')
|
||||
lines.append(f' edition = "{edition}",')
|
||||
lines.append(" env = " + starlark_dict(env, indent=" ") + ",")
|
||||
features = sorted([f for f in (node.get("features") or []) if not f.startswith("dep:")])
|
||||
if features:
|
||||
lines.append(f" features = {starlark_list(features)},")
|
||||
if proc_macro:
|
||||
lines.append(" proc_macro = True,")
|
||||
if has_build_rs:
|
||||
lines.append(f' rustc_flags = ["@$(location :{rule_name}-build-script-run[rustc_flags])"],')
|
||||
lines.append(" visibility = [],")
|
||||
if base_deps or conditional_deps:
|
||||
lines.append(f" deps = {deps_expr},")
|
||||
if named_deps:
|
||||
items = [f'"{k}": "{v}"' for k, v in sorted(named_deps.items())]
|
||||
lines.append(" named_deps = {")
|
||||
for it in items:
|
||||
lines.append(f" {it},")
|
||||
lines.append(" },")
|
||||
lines.append(")")
|
||||
lines.append("")
|
||||
|
||||
lines.append(END_MARKER.rstrip("\n") + "\n")
|
||||
return existing_buck + "\n".join(lines)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not THIRD_PARTY_BUCK.exists():
|
||||
raise SystemExit(f"Missing {THIRD_PARTY_BUCK}; run `reindeer buckify` first.")
|
||||
meta = cargo_metadata()
|
||||
existing = THIRD_PARTY_BUCK.read_text(encoding="utf-8")
|
||||
existing = strip_existing_overlay(existing)
|
||||
existing = patch_reindeer_output_for_test_features(existing)
|
||||
patched = append_overlay(meta, existing)
|
||||
if patched != existing:
|
||||
THIRD_PARTY_BUCK.write_text(patched, encoding="utf-8")
|
||||
if BUILDIFIER.exists():
|
||||
# Keep the local-only generated file readable.
|
||||
proc = subprocess.run(
|
||||
[str(BUILDIFIER), "-lint=off", "-mode=fix", str(THIRD_PARTY_BUCK)],
|
||||
cwd=REPO_ROOT,
|
||||
text=True,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
print(
|
||||
f"warning: buildifier failed (exit {proc.returncode}); leaving {THIRD_PARTY_BUCK} unformatted",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
104
docs/buck2.md
Normal file
104
docs/buck2.md
Normal file
@@ -0,0 +1,104 @@
|
||||
# Buck2 (Experimental, Local-Only)
|
||||
|
||||
This repo has an **experimental** Buck2 + Reindeer setup for `codex-rs`.
|
||||
|
||||
For now, this is intended for **local development only**. We are not ready to:
|
||||
|
||||
- commit generated `BUCK` files for first-party crates
|
||||
- commit the vendored third-party crate sources under `codex-rs/third-party/`
|
||||
|
||||
Those artifacts are intentionally gitignored to avoid repo bloat while we
|
||||
evaluate performance and developer experience.
|
||||
|
||||
## Prereqs
|
||||
|
||||
- A working Rust toolchain (see `codex-rs/rust-toolchain.toml`).
|
||||
- Buck2 + Reindeer are pinned via:
|
||||
- `./scripts/buck2`
|
||||
- `./scripts/reindeer`
|
||||
|
||||
## One-Time Setup (Generates Local-Only Files)
|
||||
|
||||
Run:
|
||||
|
||||
```sh
|
||||
./scripts/setup_buck2_local.sh
|
||||
```
|
||||
|
||||
This script will:
|
||||
|
||||
- vendor third-party crates into `codex-rs/third-party/vendor/` (large)
|
||||
- run `reindeer` to produce third-party Buck targets (gitignored)
|
||||
- generate first-party `BUCK` files for workspace members in `codex-rs/**/BUCK` (gitignored)
|
||||
- format generated `BUCK` files with the pinned `./scripts/buildifier`
|
||||
|
||||
Notes:
|
||||
|
||||
- `codex-rs/third-party/` is currently large (on the order of GBs) due to vendoring.
|
||||
- All generated artifacts live under gitignored paths (see `.gitignore`).
|
||||
|
||||
## Building
|
||||
|
||||
Buck targets are rooted at the repo root.
|
||||
|
||||
Dev build (default):
|
||||
|
||||
```sh
|
||||
./scripts/buck2 build //codex-rs/cli:codex
|
||||
```
|
||||
|
||||
Release-ish build:
|
||||
|
||||
```sh
|
||||
./scripts/buck2 build -c codex.rust_profile=release //codex-rs/cli:codex
|
||||
```
|
||||
|
||||
The `codex.rust_profile` config knob is defined in `.buckconfig` and is used to
|
||||
approximate Cargo profiles.
|
||||
|
||||
## Testing
|
||||
|
||||
This setup generates `rust_test()` rules for first-party crates, so you can run:
|
||||
|
||||
```sh
|
||||
./scripts/buck2 test //codex-rs/...
|
||||
```
|
||||
|
||||
Targeted runs are often more useful:
|
||||
|
||||
```sh
|
||||
./scripts/buck2 test //codex-rs/cli:codex-cli-unit-tests
|
||||
```
|
||||
|
||||
### Current Limitations
|
||||
|
||||
This is still a work-in-progress, so some tests may fail under Buck2 even if
|
||||
they pass under Cargo. Common reasons:
|
||||
|
||||
- Integration tests that assume `cargo test` semantics (e.g. using `escargot` to
|
||||
invoke Cargo during the test).
|
||||
- Snapshot tests (via `insta`) can behave differently because Buck2 executes
|
||||
tests under an isolated sandbox with project-relative paths.
|
||||
|
||||
If you see failures, prefer running a smaller target (like a single crate’s unit
|
||||
tests) while we iterate on broader compatibility.
|
||||
|
||||
## Cleaning Up
|
||||
|
||||
To delete Buck2 outputs/caches for this repo:
|
||||
|
||||
```sh
|
||||
./scripts/buck2 clean
|
||||
```
|
||||
|
||||
To reclaim disk space from the vendored crates, remove:
|
||||
|
||||
```sh
|
||||
rm -rf codex-rs/third-party
|
||||
```
|
||||
|
||||
## Repo Layout Notes
|
||||
|
||||
- The repo root is the Buck root (`.buckroot`).
|
||||
- `prelude/` is a tiny placeholder directory so `.buckconfig` can declare the
|
||||
`prelude` cell while using Buck2’s bundled prelude implementation.
|
||||
6
prelude/README.md
Normal file
6
prelude/README.md
Normal file
@@ -0,0 +1,6 @@
|
||||
This directory exists solely to give Buck2 a stable in-repo path for the
|
||||
`prelude` cell declared in `.buckconfig`.
|
||||
|
||||
The actual prelude implementation is provided by the Buck2 binary itself via
|
||||
`[external_cells] prelude = bundled`.
|
||||
|
||||
91
scripts/buck2
Executable file
91
scripts/buck2
Executable file
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env dotslash
|
||||
|
||||
{
|
||||
"name": "buck2",
|
||||
"platforms": {
|
||||
"macos-aarch64": {
|
||||
"size": 33146442,
|
||||
"hash": "blake3",
|
||||
"digest": "2940f7436659784c8e6e628ebdbe01c02b565901e37c0ccc22d910a549c8f56a",
|
||||
"format": "zst",
|
||||
"path": "buck2-aarch64-apple-darwin",
|
||||
"providers": [
|
||||
{
|
||||
"url": "https://github.com/facebook/buck2/releases/download/2025-12-15/buck2-aarch64-apple-darwin.zst"
|
||||
}
|
||||
]
|
||||
},
|
||||
"windows-aarch64": {
|
||||
"size": 28937384,
|
||||
"hash": "blake3",
|
||||
"digest": "1f91d6fd0db612b6ce62b18d9267dec87ea076d83f6bd316d29a3264f9dad4f7",
|
||||
"format": "zst",
|
||||
"path": "buck2-aarch64-pc-windows-msvc.exe",
|
||||
"providers": [
|
||||
{
|
||||
"url": "https://github.com/facebook/buck2/releases/download/2025-12-15/buck2-aarch64-pc-windows-msvc.exe.zst"
|
||||
}
|
||||
]
|
||||
},
|
||||
"linux-aarch64": {
|
||||
"size": 34435966,
|
||||
"hash": "blake3",
|
||||
"digest": "0e0a599cf3d9da6d81fe8b615fefa0d57d0b4c3117cbc9984a74d45aaf061cad",
|
||||
"format": "zst",
|
||||
"path": "buck2-aarch64-unknown-linux-musl",
|
||||
"providers": [
|
||||
{
|
||||
"url": "https://github.com/facebook/buck2/releases/download/2025-12-15/buck2-aarch64-unknown-linux-musl.zst"
|
||||
}
|
||||
]
|
||||
},
|
||||
"linux-riscv64": {
|
||||
"size": 36811888,
|
||||
"hash": "blake3",
|
||||
"digest": "52dcb7df8353283d95858cc03eb4acb4014d4ae06c3ad244f2c32c9ff5e4343c",
|
||||
"format": "zst",
|
||||
"path": "buck2-riscv64gc-unknown-linux-gnu",
|
||||
"providers": [
|
||||
{
|
||||
"url": "https://github.com/facebook/buck2/releases/download/2025-12-15/buck2-riscv64gc-unknown-linux-gnu.zst"
|
||||
}
|
||||
]
|
||||
},
|
||||
"macos-x86_64": {
|
||||
"size": 35331653,
|
||||
"hash": "blake3",
|
||||
"digest": "3834c662779ac9c3de22a8b0e6b85aacc2df21a716fa5a9316374e676ab69e64",
|
||||
"format": "zst",
|
||||
"path": "buck2-x86_64-apple-darwin",
|
||||
"providers": [
|
||||
{
|
||||
"url": "https://github.com/facebook/buck2/releases/download/2025-12-15/buck2-x86_64-apple-darwin.zst"
|
||||
}
|
||||
]
|
||||
},
|
||||
"windows-x86_64": {
|
||||
"size": 30576683,
|
||||
"hash": "blake3",
|
||||
"digest": "9df9a89e9ffba06eef0f3808dd802280380922cf26eba87aebb6313847c4754e",
|
||||
"format": "zst",
|
||||
"path": "buck2-x86_64-pc-windows-msvc.exe",
|
||||
"providers": [
|
||||
{
|
||||
"url": "https://github.com/facebook/buck2/releases/download/2025-12-15/buck2-x86_64-pc-windows-msvc.exe.zst"
|
||||
}
|
||||
]
|
||||
},
|
||||
"linux-x86_64": {
|
||||
"size": 35814139,
|
||||
"hash": "blake3",
|
||||
"digest": "3c78e830016fd68156913a830a5376b278ae366ffc976f155ba3d4b455a2f6e9",
|
||||
"format": "zst",
|
||||
"path": "buck2-x86_64-unknown-linux-musl",
|
||||
"providers": [
|
||||
{
|
||||
"url": "https://github.com/facebook/buck2/releases/download/2025-12-15/buck2-x86_64-unknown-linux-musl.zst"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
80
scripts/buildifier
Executable file
80
scripts/buildifier
Executable file
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env dotslash
|
||||
|
||||
// Buildifier is published as raw binaries (not archived), so this manifest omits
|
||||
// the optional `format` field.
|
||||
//
|
||||
// TODO: Add `windows-aarch64` once the buildtools releases include a Windows
|
||||
// arm64 artifact. Support was added in https://github.com/bazelbuild/buildtools/issues/874,
|
||||
// but as of v8.2.1, only windows-amd64 is available.
|
||||
{
|
||||
"name": "buildifier",
|
||||
"platforms": {
|
||||
"macos-aarch64": {
|
||||
"size": 7717890,
|
||||
"hash": "blake3",
|
||||
"digest": "1f01da46c314bd22890d92731b44bcfc555331c87bc113c72c9be13eaa27cb59",
|
||||
"path": "buildifier",
|
||||
"providers": [
|
||||
{
|
||||
"url": "https://github.com/bazelbuild/buildtools/releases/download/v8.2.1/buildifier-darwin-arm64"
|
||||
}
|
||||
]
|
||||
},
|
||||
"linux-aarch64": {
|
||||
"size": 7755744,
|
||||
"hash": "blake3",
|
||||
"digest": "9a35407ecf3d4aa0daf493a364c1a68a2a469f0f96c295f85ab29deb46265f16",
|
||||
"path": "buildifier",
|
||||
"providers": [
|
||||
{
|
||||
"url": "https://github.com/bazelbuild/buildtools/releases/download/v8.2.1/buildifier-linux-arm64"
|
||||
}
|
||||
]
|
||||
},
|
||||
"linux-riscv64": {
|
||||
"size": 7895176,
|
||||
"hash": "blake3",
|
||||
"digest": "90abd14b0eac635de23273588d2224624541cf6a45f548334252182a45a2f9a6",
|
||||
"path": "buildifier",
|
||||
"providers": [
|
||||
{
|
||||
"url": "https://github.com/bazelbuild/buildtools/releases/download/v8.2.1/buildifier-linux-riscv64"
|
||||
}
|
||||
]
|
||||
},
|
||||
"macos-x86_64": {
|
||||
"size": 7776816,
|
||||
"hash": "blake3",
|
||||
"digest": "f367d5e52779571e4da8363b5db2cb60b64c1c97d83ab53de3eaa60d598e8bf1",
|
||||
"path": "buildifier",
|
||||
"providers": [
|
||||
{
|
||||
"url": "https://github.com/bazelbuild/buildtools/releases/download/v8.2.1/buildifier-darwin-amd64"
|
||||
}
|
||||
]
|
||||
},
|
||||
"windows-x86_64": {
|
||||
"size": 8020480,
|
||||
"hash": "blake3",
|
||||
"digest": "1aa436cdcbda55ab32ac18bf76940c8bc83d548c424e9e7db660ee38fb9a91fc",
|
||||
"path": "buildifier.exe",
|
||||
"providers": [
|
||||
{
|
||||
"url": "https://github.com/bazelbuild/buildtools/releases/download/v8.2.1/buildifier-windows-amd64.exe"
|
||||
}
|
||||
]
|
||||
},
|
||||
"linux-x86_64": {
|
||||
"size": 7882884,
|
||||
"hash": "blake3",
|
||||
"digest": "0405f14324a9dae109a9fffb556b49adff9d6fd710f2648d2a6787e000f6b8f9",
|
||||
"path": "buildifier",
|
||||
"providers": [
|
||||
{
|
||||
"url": "https://github.com/bazelbuild/buildtools/releases/download/v8.2.1/buildifier-linux-amd64"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
67
scripts/reindeer
Executable file
67
scripts/reindeer
Executable file
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env dotslash
|
||||
|
||||
{
|
||||
"name": "reindeer",
|
||||
"platforms": {
|
||||
"macos-aarch64": {
|
||||
"size": 10333626,
|
||||
"hash": "blake3",
|
||||
"digest": "087035eba5af3addbbd79354b48ba8c3b5696d1d301ee68023b386e113a68b13",
|
||||
"format": "zst",
|
||||
"path": "reindeer-aarch64-apple-darwin",
|
||||
"providers": [
|
||||
{
|
||||
"url": "https://github.com/facebookincubator/reindeer/releases/download/v2025.11.24.00/reindeer-aarch64-apple-darwin.zst"
|
||||
}
|
||||
]
|
||||
},
|
||||
"linux-aarch64": {
|
||||
"size": 12023922,
|
||||
"hash": "blake3",
|
||||
"digest": "1c41919ddadbdd2390b43a6d4a97267e079038fcb3c42db14868627b0b6eaa6d",
|
||||
"format": "zst",
|
||||
"path": "reindeer-aarch64-unknown-linux-musl",
|
||||
"providers": [
|
||||
{
|
||||
"url": "https://github.com/facebookincubator/reindeer/releases/download/v2025.11.24.00/reindeer-aarch64-unknown-linux-musl.zst"
|
||||
}
|
||||
]
|
||||
},
|
||||
"macos-x86_64": {
|
||||
"size": 10444977,
|
||||
"hash": "blake3",
|
||||
"digest": "8242100bf3b07bd5665c07f42e082a2546b39a2d096c556c19db62d879604c3c",
|
||||
"format": "zst",
|
||||
"path": "reindeer-x86_64-apple-darwin",
|
||||
"providers": [
|
||||
{
|
||||
"url": "https://github.com/facebookincubator/reindeer/releases/download/v2025.11.24.00/reindeer-x86_64-apple-darwin.zst"
|
||||
}
|
||||
]
|
||||
},
|
||||
"windows-x86_64": {
|
||||
"size": 9285431,
|
||||
"hash": "blake3",
|
||||
"digest": "c90e79a55f15b20021eb5ee6ddd4149cb1f6c013cfca4d7ab3902e94b1de0dd3",
|
||||
"format": "zst",
|
||||
"path": "reindeer-x86_64-pc-windows-msvc.exe",
|
||||
"providers": [
|
||||
{
|
||||
"url": "https://github.com/facebookincubator/reindeer/releases/download/v2025.11.24.00/reindeer-x86_64-pc-windows-msvc.exe.zst"
|
||||
}
|
||||
]
|
||||
},
|
||||
"linux-x86_64": {
|
||||
"size": 12135295,
|
||||
"hash": "blake3",
|
||||
"digest": "ad5cdd028d244d47f6ba2368f9c8a9ab1c19451a046b917ce84b07acc7fee319",
|
||||
"format": "zst",
|
||||
"path": "reindeer-x86_64-unknown-linux-musl",
|
||||
"providers": [
|
||||
{
|
||||
"url": "https://github.com/facebookincubator/reindeer/releases/download/v2025.11.24.00/reindeer-x86_64-unknown-linux-musl.zst"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
218
scripts/setup_buck2_local.sh
Executable file
218
scripts/setup_buck2_local.sh
Executable file
@@ -0,0 +1,218 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage: scripts/setup_buck2_local.sh [--build]
|
||||
|
||||
Sets up local-only Buck2 + Reindeer artifacts for codex-rs:
|
||||
- vendors crates into codex-rs/third-party/vendor/
|
||||
- generates codex-rs/third-party/BUCK via reindeer buckify
|
||||
- generates codex-rs/**/BUCK for workspace crates
|
||||
- generates a toolchain definition under codex-rs/toolchains/BUCK
|
||||
|
||||
All generated Buck artifacts are expected to be gitignored (BUCK files,
|
||||
codex-rs/third-party, buck-out).
|
||||
|
||||
Environment:
|
||||
REINDEER_BIN Optional path to reindeer binary (default: scripts/reindeer)
|
||||
BUCK2_BIN Optional path to buck2 binary (default: scripts/buck2)
|
||||
|
||||
If invoked from within Codex CLI, you may want:
|
||||
env -u BASH_EXEC_WRAPPER -u CODEX_ESCALATE_SOCKET buck2 build //codex-rs/cli:codex
|
||||
EOF
|
||||
}
|
||||
|
||||
DO_BUILD=0
|
||||
if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then
|
||||
usage
|
||||
exit 0
|
||||
elif [[ "${1:-}" == "--build" ]]; then
|
||||
DO_BUILD=1
|
||||
elif [[ -n "${1:-}" ]]; then
|
||||
echo "Unknown argument: $1" >&2
|
||||
usage >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
REINDEER_BIN="${REINDEER_BIN:-}"
|
||||
if [[ -z "${REINDEER_BIN}" ]]; then
|
||||
if [[ -x "${REPO_ROOT}/scripts/reindeer" ]]; then
|
||||
REINDEER_BIN="${REPO_ROOT}/scripts/reindeer"
|
||||
elif command -v reindeer >/dev/null 2>&1; then
|
||||
REINDEER_BIN="$(command -v reindeer)"
|
||||
else
|
||||
echo "Could not find reindeer. Set REINDEER_BIN or use scripts/reindeer." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
BUCK2_BIN="${BUCK2_BIN:-}"
|
||||
if [[ -z "${BUCK2_BIN}" ]]; then
|
||||
if [[ -x "${REPO_ROOT}/scripts/buck2" ]]; then
|
||||
BUCK2_BIN="${REPO_ROOT}/scripts/buck2"
|
||||
elif command -v buck2 >/dev/null 2>&1; then
|
||||
BUCK2_BIN="$(command -v buck2)"
|
||||
else
|
||||
echo "Could not find buck2. Set BUCK2_BIN or use scripts/buck2." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
BUILDIFIER_BIN="${BUILDIFIER_BIN:-}"
|
||||
if [[ -z "${BUILDIFIER_BIN}" ]]; then
|
||||
if [[ -e "${REPO_ROOT}/scripts/buildifier" ]]; then
|
||||
BUILDIFIER_BIN="${REPO_ROOT}/scripts/buildifier"
|
||||
elif command -v buildifier >/dev/null 2>&1; then
|
||||
BUILDIFIER_BIN="$(command -v buildifier)"
|
||||
else
|
||||
BUILDIFIER_BIN=""
|
||||
fi
|
||||
fi
|
||||
|
||||
cd "${REPO_ROOT}"
|
||||
|
||||
# Resolve the Rust toolchain used by codex-rs so Buck uses the same compiler
|
||||
# regardless of the working directory Buck actions run in.
|
||||
RUST_TOOLCHAIN="$(
|
||||
python3 - <<'PY'
|
||||
import pathlib, re
|
||||
text = pathlib.Path("codex-rs/rust-toolchain.toml").read_text(encoding="utf-8")
|
||||
m = re.search(r'(?m)^channel\s*=\s*"([^"]+)"\s*$', text)
|
||||
if not m:
|
||||
raise SystemExit("Could not find toolchain channel in codex-rs/rust-toolchain.toml")
|
||||
print(m.group(1))
|
||||
PY
|
||||
)"
|
||||
|
||||
if command -v rustup >/dev/null 2>&1; then
|
||||
RUSTC_PATH="$(rustup which rustc --toolchain "${RUST_TOOLCHAIN}")"
|
||||
RUSTDOC_PATH="$(rustup which rustdoc --toolchain "${RUST_TOOLCHAIN}")"
|
||||
if rustup which clippy-driver --toolchain "${RUST_TOOLCHAIN}" >/dev/null 2>&1; then
|
||||
CLIPPY_DRIVER_PATH="$(rustup which clippy-driver --toolchain "${RUST_TOOLCHAIN}")"
|
||||
else
|
||||
CLIPPY_DRIVER_PATH="clippy-driver"
|
||||
fi
|
||||
else
|
||||
echo "rustup not found; falling back to rustc/rustdoc from PATH (may not match codex-rs toolchain)." >&2
|
||||
RUSTC_PATH="rustc"
|
||||
RUSTDOC_PATH="rustdoc"
|
||||
CLIPPY_DRIVER_PATH="clippy-driver"
|
||||
fi
|
||||
|
||||
# Reindeer canonicalizes third_party_dir up-front, so it must exist.
|
||||
mkdir -p codex-rs/third-party
|
||||
|
||||
# Ensure we have an ignored Cargo.lock next to the manifest Reindeer uses.
|
||||
if [[ ! -e "codex-rs/cli/Cargo.lock" ]]; then
|
||||
(
|
||||
cd codex-rs/cli
|
||||
# Prefer a symlink (fast), but fall back to copying if symlinks are unsupported.
|
||||
if ln -s ../Cargo.lock Cargo.lock 2>/dev/null; then
|
||||
true
|
||||
else
|
||||
cp ../Cargo.lock Cargo.lock
|
||||
fi
|
||||
)
|
||||
fi
|
||||
|
||||
# Ensure Buck can load bzl files from codex-rs/buck2 by creating an (ignored)
|
||||
# BUCK file to define the package.
|
||||
mkdir -p codex-rs/buck2
|
||||
if [[ ! -e "codex-rs/buck2/BUCK" ]]; then
|
||||
: > codex-rs/buck2/BUCK
|
||||
fi
|
||||
|
||||
# Generate a minimal toolchain package under the toolchains cell.
|
||||
mkdir -p codex-rs/toolchains
|
||||
# This BUCK file is local-only (gitignored), so we bake in absolute tool paths.
|
||||
cat > codex-rs/toolchains/BUCK <<EOF
|
||||
load("@prelude//tests:test_toolchain.bzl", "noop_test_toolchain")
|
||||
load("@prelude//toolchains:cxx.bzl", "system_cxx_toolchain")
|
||||
load("@prelude//toolchains:genrule.bzl", "system_genrule_toolchain")
|
||||
load(
|
||||
"@prelude//toolchains:python.bzl",
|
||||
"system_python_bootstrap_toolchain",
|
||||
"system_python_toolchain",
|
||||
)
|
||||
load("@prelude//toolchains:remote_test_execution.bzl", "remote_test_execution_toolchain")
|
||||
load("@root//codex-rs/buck2:codex_rust_toolchain.bzl", "codex_rust_toolchain")
|
||||
|
||||
system_cxx_toolchain(
|
||||
name = "cxx",
|
||||
visibility = ["PUBLIC"],
|
||||
)
|
||||
|
||||
system_genrule_toolchain(
|
||||
name = "genrule",
|
||||
visibility = ["PUBLIC"],
|
||||
)
|
||||
|
||||
system_python_toolchain(
|
||||
name = "python",
|
||||
visibility = ["PUBLIC"],
|
||||
)
|
||||
|
||||
system_python_bootstrap_toolchain(
|
||||
name = "python_bootstrap",
|
||||
visibility = ["PUBLIC"],
|
||||
)
|
||||
|
||||
codex_rust_toolchain(
|
||||
name = "rust",
|
||||
default_edition = "2024",
|
||||
rustc = "${RUSTC_PATH}",
|
||||
rustdoc = "${RUSTDOC_PATH}",
|
||||
clippy_driver = "${CLIPPY_DRIVER_PATH}",
|
||||
visibility = ["PUBLIC"],
|
||||
)
|
||||
|
||||
remote_test_execution_toolchain(
|
||||
name = "remote_test_execution",
|
||||
visibility = ["PUBLIC"],
|
||||
)
|
||||
|
||||
noop_test_toolchain(
|
||||
name = "test",
|
||||
visibility = ["PUBLIC"],
|
||||
)
|
||||
EOF
|
||||
|
||||
if [[ -n "${BUILDIFIER_BIN}" ]]; then
|
||||
# Keep local-only BUCK files readable for debugging.
|
||||
"${BUILDIFIER_BIN}" -lint=off -mode=fix codex-rs/toolchains/BUCK codex-rs/buck2/BUCK || true
|
||||
fi
|
||||
|
||||
echo "Using:"
|
||||
echo " reindeer: ${REINDEER_BIN}"
|
||||
echo " buck2: ${BUCK2_BIN}"
|
||||
echo " rustc: ${RUSTC_PATH} (toolchain: ${RUST_TOOLCHAIN})"
|
||||
|
||||
(
|
||||
cd codex-rs
|
||||
"${REINDEER_BIN}" vendor
|
||||
python3 scripts/gen_reindeer_fixups.py
|
||||
"${REINDEER_BIN}" buckify
|
||||
python3 scripts/patch_third_party_buck_for_tests.py
|
||||
python3 scripts/gen_buck_first_party.py
|
||||
)
|
||||
|
||||
echo ""
|
||||
echo "Third-party size:"
|
||||
if command -v du >/dev/null 2>&1; then
|
||||
du -sh codex-rs/third-party 2>/dev/null || true
|
||||
du -sh codex-rs/third-party/vendor 2>/dev/null || true
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Next:"
|
||||
echo " ${BUCK2_BIN} build //codex-rs/cli:codex"
|
||||
|
||||
if [[ "${DO_BUILD}" -eq 1 ]]; then
|
||||
echo ""
|
||||
echo "Building //codex-rs/cli:codex ..."
|
||||
# When running inside Codex CLI, these wrapper env vars can interfere with buck2.
|
||||
env -u BASH_EXEC_WRAPPER -u CODEX_ESCALATE_SOCKET "${BUCK2_BIN}" build //codex-rs/cli:codex
|
||||
fi
|
||||
Reference in New Issue
Block a user