clone/codex
1
0
Fork 0
mirror of https://github.com/openai/codex.git synced 2026-05-23 12:34:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

npm: ship platform packages in Codex package layout (#23637)

Browse Source 
## Summary

The npm platform packages should stop carrying a bespoke native layout
now that the release workflow builds canonical Codex package archives.
Keeping npm on the same `bin/`, `codex-resources/`, and `codex-path/`
structure lets the Rust package-layout detection behave consistently
across standalone, npm, and future DotSlash installs.

This changes platform npm packages to stage the `codex-package` artifact
for each target under `vendor/<target>`. The Node launcher now resolves
`bin/codex` and prepends `codex-path`, while retaining legacy
`vendor/<target>/codex` and `vendor/<target>/path` fallback support for
local development and migration. The npm staging helper downloads
`codex-package` archives instead of rebuilding the CLI payload from
individual `codex`, `rg`, `bwrap`, and sandbox helper artifacts.

CI still needs to stage npm packages from historical rust-release
workflow artifacts that predate package archives, so the staging scripts
expose an explicit `--allow-legacy-codex-package` fallback. That
fallback synthesizes the canonical package layout from legacy per-binary
artifacts and is wired only into the CI smoke path; release staging
remains strict and continues to require real package archives.

For direct local use, `install_native_deps.py` now points its built-in
default workflow at the same recent artifact run used by CI and
automatically enables legacy package synthesis only when
`--workflow-url` is omitted. Explicit workflow URLs remain strict unless
callers opt in with `--allow-legacy-codex-package`.

## Test plan

- `python3 -m py_compile codex-cli/scripts/build_npm_package.py
codex-cli/scripts/install_native_deps.py scripts/stage_npm_packages.py
scripts/codex_package/cli.py`
- `node --check codex-cli/bin/codex.js`
- `ruby -e 'require "yaml";
YAML.load_file(".github/workflows/rust-release.yml");
YAML.load_file(".github/workflows/ci.yml"); puts "ok"'`
- Staged a synthetic `codex-linux-x64` platform package from a canonical
vendor tree and verified it copied only `bin/`, `codex-path/`,
`codex-resources/`, and `codex-package.json`.
- Imported `install_native_deps.py` and extracted a synthetic
`codex-package-x86_64-unknown-linux-musl.tar.gz` into `vendor/<target>`.
- Ran legacy-layout conversion smokes for Linux, Windows, and unsigned
macOS artifact naming.
- Ran a synthetic `install_native_deps.py` default-workflow smoke that
verifies legacy package synthesis is automatic only when
`--workflow-url` is omitted.
- `NPM_CONFIG_CACHE="$tmp_dir/npm-cache" python3
./scripts/stage_npm_packages.py --release-version 0.125.0 --workflow-url
https://github.com/openai/codex/actions/runs/26131514935 --package codex
--allow-legacy-codex-package --output-dir "$tmp_dir"`
- `node codex-cli/bin/codex.js --version`


---
[//]: # (BEGIN SAPLING FOOTER)
Stack created with [Sapling](https://sapling-scm.com). Best reviewed
with [ReviewStack](https://reviewstack.dev/openai/codex/pull/23637).
* #23638
* __->__ #23637
This commit is contained in:
Michael Bolin
2026-05-20 12:02:32 -07:00
committed by GitHub
parent 7c3cc1db81
commit e389e01f83
7 changed files with 312 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
codex-cli/scripts/README.md
View File

@@ -11,13 +11,13 @@ example, to stage the CLI, responses proxy, and SDK packages for version `0.6.0`
--package codex-sdk
```
This downloads the native artifacts once, hydrates `vendor/` for each package, and writes
tarballs to `dist/npm/`.
This downloads the native package archive artifacts once, hydrates `vendor/` for each
package, and writes tarballs to `dist/npm/`.
When `--package codex` is provided, the staging helper builds the lightweight
`@openai/codex` meta package plus all platform-native `@openai/codex` variants
that are later published under platform-specific dist-tags.
If you need to invoke `build_npm_package.py` directly, run
`codex-cli/scripts/install_native_deps.py` first and pass `--vendor-src` pointing to the
directory that contains the populated `vendor/` tree.
`codex-cli/scripts/install_native_deps.py --component codex-package` first and pass
`--vendor-src` pointing to the directory that contains the populated `vendor/` tree.

70
codex-cli/scripts/build_npm_package.py
View File

@@ -15,6 +15,8 @@ REPO_ROOT = CODEX_CLI_ROOT.parent
RESPONSES_API_PROXY_NPM_ROOT = REPO_ROOT / "codex-rs" / "responses-api-proxy" / "npm"
CODEX_SDK_ROOT = REPO_ROOT / "sdk" / "typescript"
CODEX_NPM_NAME = "@openai/codex"
CODEX_PACKAGE_COMPONENT = "codex-package"
CODEX_PACKAGE_ENTRIES = ("codex-package.json", "bin", "codex-resources", "codex-path")
# `npm_name` is the local optional-dependency alias consumed by `bin/codex.js`.
# The underlying package published to npm is always `@openai/codex`.
@@ -69,12 +71,12 @@ PACKAGE_EXPANSIONS: dict[str, list[str]] = {
PACKAGE_NATIVE_COMPONENTS: dict[str, list[str]] = {
"codex": [],
"codex-linux-x64": ["bwrap", "codex", "rg"],
"codex-linux-arm64": ["bwrap", "codex", "rg"],
"codex-darwin-x64": ["codex", "rg"],
"codex-darwin-arm64": ["codex", "rg"],
"codex-win32-x64": ["codex", "rg", "codex-windows-sandbox-setup", "codex-command-runner"],
"codex-win32-arm64": ["codex", "rg", "codex-windows-sandbox-setup", "codex-command-runner"],
"codex-linux-x64": [CODEX_PACKAGE_COMPONENT],
"codex-linux-arm64": [CODEX_PACKAGE_COMPONENT],
"codex-darwin-x64": [CODEX_PACKAGE_COMPONENT],
"codex-darwin-arm64": [CODEX_PACKAGE_COMPONENT],
"codex-win32-x64": [CODEX_PACKAGE_COMPONENT],
"codex-win32-arm64": [CODEX_PACKAGE_COMPONENT],
"codex-responses-api-proxy": ["codex-responses-api-proxy"],
"codex-sdk": [],
}
@@ -383,7 +385,11 @@ def copy_native_binaries(
if not vendor_src.exists():
raise RuntimeError(f"Vendor source directory not found: {vendor_src}")
components_set = {component for component in components if component in COMPONENT_DEST_DIR}
components_set = {
component
for component in components
if component == CODEX_PACKAGE_COMPONENT or component in COMPONENT_DEST_DIR
}
allow_missing_components = allow_missing_components or set()
if not components_set:
return
@@ -402,11 +408,26 @@ def copy_native_binaries(
if target_filter is not None and target_dir.name not in target_filter:
continue
dest_target_dir = vendor_dest / target_dir.name
dest_target_dir.mkdir(parents=True, exist_ok=True)
copied_targets.add(target_dir.name)
for component in components_set:
dest_target_dir = vendor_dest / target_dir.name
if CODEX_PACKAGE_COMPONENT in components_set:
validate_codex_package_dir(target_dir)
if dest_target_dir.exists():
shutil.rmtree(dest_target_dir)
dest_target_dir.mkdir(parents=True, exist_ok=True)
for entry in CODEX_PACKAGE_ENTRIES:
src = target_dir / entry
dest = dest_target_dir / entry
if src.is_dir():
shutil.copytree(src, dest)
else:
shutil.copy2(src, dest)
else:
dest_target_dir.mkdir(parents=True, exist_ok=True)
for component in components_set - {CODEX_PACKAGE_COMPONENT}:
dest_dir_name = COMPONENT_DEST_DIR.get(component)
if dest_dir_name is None:
continue
@@ -431,6 +452,35 @@ def copy_native_binaries(
raise RuntimeError(f"Missing target directories in vendor source: {missing_list}")
def validate_codex_package_dir(package_dir: Path) -> None:
is_windows = "windows" in package_dir.name
required_files = [
Path("codex-package.json"),
Path("bin") / ("codex.exe" if is_windows else "codex"),
Path("codex-path") / ("rg.exe" if is_windows else "rg"),
]
if "linux" in package_dir.name:
required_files.append(Path("codex-resources") / "bwrap")
if is_windows:
required_files.extend(
[
Path("codex-resources") / "codex-command-runner.exe",
Path("codex-resources") / "codex-windows-sandbox-setup.exe",
]
)
missing_files = [
str(relative_path)
for relative_path in required_files
if not (package_dir / relative_path).is_file()
]
if missing_files:
missing = ", ".join(missing_files)
raise RuntimeError(f"Missing files in Codex package directory {package_dir}: {missing}")
def run_npm_pack(staging_dir: Path, output_path: Path) -> Path:
output_path = output_path.resolve()
output_path.parent.mkdir(parents=True, exist_ok=True)

210
codex-cli/scripts/install_native_deps.py
View File

@@ -1,5 +1,5 @@
#!/usr/bin/env python3
"""Install Codex native binaries (Rust CLI, bwrap, and ripgrep helpers)."""
"""Install Codex package archives and native helper binaries."""
import argparse
from contextlib import contextmanager
@@ -20,7 +20,7 @@ from urllib.request import urlopen
SCRIPT_DIR = Path(__file__).resolve().parent
CODEX_CLI_ROOT = SCRIPT_DIR.parent
DEFAULT_WORKFLOW_URL = "https://github.com/openai/codex/actions/runs/17952349351" # rust-v0.40.0
DEFAULT_WORKFLOW_URL = "https://github.com/openai/codex/actions/runs/26131514935" # rust-v0.132.0
VENDOR_DIR_NAME = "vendor"
RG_MANIFEST = CODEX_CLI_ROOT / "bin" / "rg"
BINARY_TARGETS = (
@@ -31,6 +31,7 @@ BINARY_TARGETS = (
"x86_64-pc-windows-msvc",
"aarch64-pc-windows-msvc",
)
CODEX_PACKAGE_COMPONENT = "codex-package"
@dataclass(frozen=True)
@@ -139,11 +140,20 @@ def parse_args() -> argparse.Namespace:
"--component",
dest="components",
action="append",
choices=tuple(list(BINARY_COMPONENTS) + ["rg"]),
choices=tuple([CODEX_PACKAGE_COMPONENT, *BINARY_COMPONENTS, "rg"]),
help=(
"Limit installation to the specified components."
" May be repeated. Defaults to bwrap, codex, codex-windows-sandbox-setup,"
" codex-command-runner, and rg."
" May be repeated. Defaults to codex-package and codex-responses-api-proxy."
),
)
parser.add_argument(
"--allow-legacy-codex-package",
action="store_true",
help=(
"Allow codex-package to be synthesized from legacy per-binary artifacts "
"when package archives are missing. Intended for CI compatibility only; "
"release staging should not use this. Automatically enabled for the "
"built-in default workflow."
),
)
parser.add_argument(
@@ -165,17 +175,11 @@ def main() -> int:
vendor_dir = codex_cli_root / VENDOR_DIR_NAME
vendor_dir.mkdir(parents=True, exist_ok=True)
components = args.components or [
"bwrap",
"codex",
"codex-windows-sandbox-setup",
"codex-command-runner",
"rg",
]
components = args.components or [CODEX_PACKAGE_COMPONENT, "codex-responses-api-proxy"]
workflow_url = (args.workflow_url or DEFAULT_WORKFLOW_URL).strip()
if not workflow_url:
workflow_url = DEFAULT_WORKFLOW_URL
workflow_override = (args.workflow_url or "").strip()
use_default_workflow = not workflow_override
workflow_url = workflow_override or DEFAULT_WORKFLOW_URL
workflow_id = workflow_url.rstrip("/").split("/")[-1]
print(f"Downloading native artifacts from workflow {workflow_id}...")
@@ -184,6 +188,18 @@ def main() -> int:
with tempfile.TemporaryDirectory(prefix="codex-native-artifacts-") as artifacts_dir_str:
artifacts_dir = Path(artifacts_dir_str)
_download_artifacts(workflow_id, artifacts_dir)
if CODEX_PACKAGE_COMPONENT in components:
try:
install_codex_package_archives(artifacts_dir, vendor_dir, BINARY_TARGETS)
except FileNotFoundError:
if not (args.allow_legacy_codex_package or use_default_workflow):
raise
install_legacy_codex_package_layouts(
artifacts_dir,
vendor_dir,
BINARY_TARGETS,
manifest_path=RG_MANIFEST,
)
install_binary_components(
artifacts_dir,
vendor_dir,
@@ -199,6 +215,135 @@ def main() -> int:
return 0
def install_codex_package_archives(
artifacts_dir: Path,
vendor_dir: Path,
targets: Sequence[str],
) -> None:
targets = list(targets)
if not targets:
return
print("Installing Codex package archives for targets: " + ", ".join(targets))
max_workers = min(len(targets), max(1, (os.cpu_count() or 1)))
with ThreadPoolExecutor(max_workers=max_workers) as executor:
futures = {
executor.submit(
_install_single_codex_package_archive,
artifacts_dir,
vendor_dir,
target,
): target
for target in targets
}
for future in as_completed(futures):
installed_path = future.result()
print(f" installed {installed_path}")
def _install_single_codex_package_archive(
artifacts_dir: Path,
vendor_dir: Path,
target: str,
) -> Path:
artifact_subdir = artifact_dir_for_target(artifacts_dir, target)
archive_path = artifact_subdir / f"codex-package-{target}.tar.gz"
if not archive_path.exists():
raise FileNotFoundError(f"Expected package archive not found: {archive_path}")
dest_dir = vendor_dir / target
if dest_dir.exists():
shutil.rmtree(dest_dir)
dest_dir.mkdir(parents=True, exist_ok=True)
with tarfile.open(archive_path, "r:gz") as archive:
archive.extractall(dest_dir, filter="data")
return dest_dir
def install_legacy_codex_package_layouts(
artifacts_dir: Path,
vendor_dir: Path,
targets: Sequence[str],
*,
manifest_path: Path,
) -> None:
targets = list(targets)
print(
"Synthesizing Codex package layouts from legacy artifacts for targets: "
+ ", ".join(targets)
)
with tempfile.TemporaryDirectory(prefix="codex-legacy-package-") as legacy_vendor_dir_str:
legacy_vendor_dir = Path(legacy_vendor_dir_str)
install_binary_components(
artifacts_dir,
legacy_vendor_dir,
[
BINARY_COMPONENTS["codex"],
BINARY_COMPONENTS["bwrap"],
BINARY_COMPONENTS["codex-windows-sandbox-setup"],
BINARY_COMPONENTS["codex-command-runner"],
],
)
fetch_rg(legacy_vendor_dir, targets, manifest_path=manifest_path)
for target in targets:
dest_dir = vendor_dir / target
if dest_dir.exists():
shutil.rmtree(dest_dir)
_build_legacy_codex_package_layout(legacy_vendor_dir / target, dest_dir, target)
print(f" synthesized {dest_dir}")
def _build_legacy_codex_package_layout(
legacy_target_dir: Path,
package_dir: Path,
target: str,
) -> None:
is_windows = "windows" in target
exe_suffix = ".exe" if is_windows else ""
package_dir.mkdir(parents=True)
bin_dir = package_dir / "bin"
resources_dir = package_dir / "codex-resources"
path_dir = package_dir / "codex-path"
bin_dir.mkdir()
resources_dir.mkdir()
path_dir.mkdir()
shutil.copy2(
legacy_target_dir / "codex" / f"codex{exe_suffix}",
bin_dir / f"codex{exe_suffix}",
)
shutil.copy2(
legacy_target_dir / "path" / f"rg{exe_suffix}",
path_dir / f"rg{exe_suffix}",
)
if is_windows:
for helper in [
"codex-command-runner.exe",
"codex-windows-sandbox-setup.exe",
]:
shutil.copy2(legacy_target_dir / "codex" / helper, resources_dir / helper)
elif "linux" in target:
shutil.copy2(legacy_target_dir / "codex-resources" / "bwrap", resources_dir / "bwrap")
write_json(
package_dir / "codex-package.json",
{
"layoutVersion": 1,
"version": "unknown",
"target": target,
"variant": "codex",
"entrypoint": f"bin/codex{exe_suffix}",
"resourcesDir": "codex-resources",
"pathDir": "codex-path",
},
)
def fetch_rg(
vendor_dir: Path,
targets: Sequence[str] | None = None,
@@ -319,11 +464,8 @@ def _install_single_binary(
target: str,
component: BinaryComponent,
) -> Path:
artifact_subdir = artifacts_dir / target
archive_name = _archive_name_for_target(component.artifact_prefix, target)
archive_path = artifact_subdir / archive_name
if not archive_path.exists():
raise FileNotFoundError(f"Expected artifact not found: {archive_path}")
artifact_subdir = artifact_dir_for_target(artifacts_dir, target)
archive_path = legacy_binary_archive_path(artifact_subdir, component.artifact_prefix, target)
dest_dir = vendor_dir / target / component.dest_dir
dest_dir.mkdir(parents=True, exist_ok=True)
@@ -345,6 +487,28 @@ def _archive_name_for_target(artifact_prefix: str, target: str) -> str:
return f"{artifact_prefix}-{target}.zst"
def legacy_binary_archive_path(artifact_dir: Path, artifact_prefix: str, target: str) -> Path:
archive_names = [_archive_name_for_target(artifact_prefix, target)]
if artifact_dir.name == f"{target}-unsigned":
archive_names.append(_archive_name_for_target(artifact_prefix, f"{target}-unsigned"))
for archive_name in archive_names:
archive_path = artifact_dir / archive_name
if archive_path.exists():
return archive_path
raise FileNotFoundError(f"Expected artifact not found: {artifact_dir / archive_names[0]}")
def artifact_dir_for_target(artifacts_dir: Path, target: str) -> Path:
for artifact_name in [target, f"{target}-unsigned"]:
artifact_dir = artifacts_dir / artifact_name
if artifact_dir.is_dir():
return artifact_dir
return artifacts_dir / target
def _fetch_single_rg(
vendor_dir: Path,
target: str,
@@ -477,6 +641,12 @@ def _load_manifest(manifest_path: Path) -> dict:
return manifest
def write_json(path: Path, value: object) -> None:
with open(path, "w", encoding="utf-8") as out:
json.dump(value, out, indent=2)
out.write("\n")
if __name__ == "__main__":
import sys