From b20e969f23bee39497d060431756283e17749e37 Mon Sep 17 00:00:00 2001 From: Michael Bolin Date: Thu, 21 May 2026 13:43:48 -0700 Subject: [PATCH] npm: remove legacy package artifact synthesis (#23836) ## Why `rust-release` now publishes `codex-package-.tar.gz` as the canonical native package payload. npm staging should consume those archives directly instead of keeping legacy synthesis code that fetched `rg`, copied standalone binaries, and rebuilt an approximate package layout. That also means the package builder should not know the internal shape of `codex-package`. It should extract and copy the target payload wholesale so future layout changes stay localized to the archive producer. The release job stages `codex`, `codex-responses-api-proxy`, and `codex-sdk` together, so native artifact download should be filtered, observable, and shared across component installs. Since that native hydration is now only used by release staging, keeping a separate `install_native_deps.py` CLI adds an extra wrapper without a real caller. ## What Changed - Removed legacy `codex-package` synthesis and related compatibility flags from npm staging. - Folded the remaining native artifact hydration code into `scripts/stage_npm_packages.py` and deleted `codex-cli/scripts/install_native_deps.py`. - Made platform package staging copy the full extracted target directory instead of enumerating package entries. - Kept non-`codex-package` native components under their component directory name instead of using a legacy destination map. - Split native staging by component set while sharing one workflow-artifact cache across the invocation. - Changed workflow artifact download to select target artifacts by name, print sizes/progress, and reuse cached artifacts. - Removed the implicit `CI=true` default from `build_npm_package.py`; local CI-shaped runs should set that environment explicitly. - Kept `npm pack` cache/log output in its temporary directory so packing does not write to the user npm cache. ## Verification - `python3 -m py_compile scripts/stage_npm_packages.py codex-cli/scripts/build_npm_package.py` - `python3 -m unittest discover -s scripts/codex_package -p "test_*.py"` - `scripts/stage_npm_packages.py --help` - `codex-cli/scripts/build_npm_package.py --help` - Ran the release-shaped staging command from `rust-release.yml` against workflow run https://github.com/openai/codex/actions/runs/26240748758 with `CI=true` set locally to match GitHub Actions: ```sh CI=true python3 ./scripts/stage_npm_packages.py \ --release-version 0.133.0 \ --workflow-url https://github.com/openai/codex/actions/runs/26240748758 \ --package codex \ --package codex-responses-api-proxy \ --package codex-sdk ``` That completed successfully, downloaded only the six target artifacts once, reused the cache for `codex-responses-api-proxy`, and produced all nine npm tarballs. Generated tarballs and staging/artifact temp dirs were cleaned afterward. --- .github/workflows/ci.yml | 11 +- codex-cli/scripts/README.md | 10 +- codex-cli/scripts/build_npm_package.py | 93 +--- codex-cli/scripts/install_native_deps.py | 654 ----------------------- scripts/stage_npm_packages.py | 432 +++++++++++++-- 5 files changed, 400 insertions(+), 800 deletions(-) delete mode 100755 codex-cli/scripts/install_native_deps.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 25dff134a7..b1ee1395e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,9 +42,6 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - # stage_npm_packages.py requires DotSlash when staging releases. - - uses: facebook/install-dotslash@1e4e7b3e07eaca387acb98f1d4720e0bee8dbb6a # v2 - - name: Stage npm package id: stage_npm_package env: @@ -55,17 +52,13 @@ jobs: # cross-platform native payload required by the npm package layout. # Passing the workflow URL directly avoids relying on old rust-v* # branches remaining discoverable via `gh run list --branch ...`. - CODEX_VERSION=0.125.0 - WORKFLOW_URL="https://github.com/openai/codex/actions/runs/26131514935" + CODEX_VERSION=0.133.0-alpha.4 + WORKFLOW_URL="https://github.com/openai/codex/actions/runs/26201494185" OUTPUT_DIR="${RUNNER_TEMP}" - # This reused workflow predates codex-package archive artifacts, so - # CI synthesizes the package layout from the older per-binary - # artifacts. Release staging must use real package archives. python3 ./scripts/stage_npm_packages.py \ --release-version "$CODEX_VERSION" \ --workflow-url "$WORKFLOW_URL" \ --package codex \ - --allow-legacy-codex-package \ --output-dir "$OUTPUT_DIR" PACK_OUTPUT="${OUTPUT_DIR}/codex-npm-${CODEX_VERSION}.tgz" echo "pack_output=$PACK_OUTPUT" >> "$GITHUB_OUTPUT" diff --git a/codex-cli/scripts/README.md b/codex-cli/scripts/README.md index ce58097db5..65923151a0 100644 --- a/codex-cli/scripts/README.md +++ b/codex-cli/scripts/README.md @@ -11,13 +11,13 @@ example, to stage the CLI, responses proxy, and SDK packages for version `0.6.0` --package codex-sdk ``` -This downloads the native package archive artifacts once, hydrates `vendor/` for each -package, and writes tarballs to `dist/npm/`. +This downloads the required native package archive artifacts, hydrates `vendor/` for +each package, and writes tarballs to `dist/npm/`. When `--package codex` is provided, the staging helper builds the lightweight `@openai/codex` meta package plus all platform-native `@openai/codex` variants that are later published under platform-specific dist-tags. -If you need to invoke `build_npm_package.py` directly, run -`codex-cli/scripts/install_native_deps.py --component codex-package` first and pass -`--vendor-src` pointing to the directory that contains the populated `vendor/` tree. +Direct `build_npm_package.py` invocations are still useful for package-specific +debugging, but native packages expect `--vendor-src` to point at a prehydrated +`vendor/` tree. Release packaging should use `scripts/stage_npm_packages.py`. diff --git a/codex-cli/scripts/build_npm_package.py b/codex-cli/scripts/build_npm_package.py index 261b9e0b61..60f6ca7a9d 100755 --- a/codex-cli/scripts/build_npm_package.py +++ b/codex-cli/scripts/build_npm_package.py @@ -3,6 +3,7 @@ import argparse import json +import os import shutil import subprocess import sys @@ -16,7 +17,6 @@ RESPONSES_API_PROXY_NPM_ROOT = REPO_ROOT / "codex-rs" / "responses-api-proxy" / CODEX_SDK_ROOT = REPO_ROOT / "sdk" / "typescript" CODEX_NPM_NAME = "@openai/codex" CODEX_PACKAGE_COMPONENT = "codex-package" -CODEX_PACKAGE_ENTRIES = ("codex-package.json", "bin", "codex-resources", "codex-path") # `npm_name` is the local optional-dependency alias consumed by `bin/codex.js`. # The underlying package published to npm is always `@openai/codex`. @@ -88,16 +88,6 @@ PACKAGE_TARGET_FILTERS: dict[str, str] = { PACKAGE_CHOICES = tuple(PACKAGE_NATIVE_COMPONENTS) -COMPONENT_DEST_DIR: dict[str, str] = { - "bwrap": "codex-resources", - "codex": "codex", - "codex-responses-api-proxy": "codex-responses-api-proxy", - "codex-windows-sandbox-setup": "codex", - "codex-command-runner": "codex", - "rg": "path", -} - - def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Build or stage the Codex CLI npm package.") parser.add_argument( @@ -140,16 +130,6 @@ def parse_args() -> argparse.Namespace: type=Path, help="Directory containing pre-installed native binaries to bundle (vendor root).", ) - parser.add_argument( - "--allow-missing-native-component", - dest="allow_missing_native_components", - action="append", - default=[], - help=( - "Native component that may be absent from --vendor-src. Intended for CI " - "compatibility with older artifact workflows; releases should not use this." - ), - ) return parser.parse_args() @@ -190,7 +170,6 @@ def main() -> int: staging_dir, native_components, target_filter={target_filter} if target_filter else None, - allow_missing_components=set(args.allow_missing_native_components), ) if release_version: @@ -346,7 +325,7 @@ def compute_platform_package_version(version: str, platform_tag: str) -> str: def run_command(cmd: list[str], cwd: Path | None = None) -> None: - print("+", " ".join(cmd)) + print("+", " ".join(cmd), flush=True) subprocess.run(cmd, cwd=cwd, check=True) @@ -376,18 +355,12 @@ def copy_native_binaries( staging_dir: Path, components: list[str], target_filter: set[str] | None = None, - allow_missing_components: set[str] | None = None, ) -> None: vendor_src = vendor_src.resolve() if not vendor_src.exists(): raise RuntimeError(f"Vendor source directory not found: {vendor_src}") - components_set = { - component - for component in components - if component == CODEX_PACKAGE_COMPONENT or component in COMPONENT_DEST_DIR - } - allow_missing_components = allow_missing_components or set() + components_set = set(components) if not components_set: return @@ -410,34 +383,20 @@ def copy_native_binaries( dest_target_dir = vendor_dest / target_dir.name if CODEX_PACKAGE_COMPONENT in components_set: - validate_codex_package_dir(target_dir) if dest_target_dir.exists(): shutil.rmtree(dest_target_dir) - dest_target_dir.mkdir(parents=True, exist_ok=True) - for entry in CODEX_PACKAGE_ENTRIES: - src = target_dir / entry - dest = dest_target_dir / entry - if src.is_dir(): - shutil.copytree(src, dest) - else: - shutil.copy2(src, dest) + shutil.copytree(target_dir, dest_target_dir) else: dest_target_dir.mkdir(parents=True, exist_ok=True) - for component in components_set - {CODEX_PACKAGE_COMPONENT}: - dest_dir_name = COMPONENT_DEST_DIR.get(component) - if dest_dir_name is None: - continue - - src_component_dir = target_dir / dest_dir_name + for component in sorted(components_set - {CODEX_PACKAGE_COMPONENT}): + src_component_dir = target_dir / component if not src_component_dir.exists(): - if component in allow_missing_components: - continue raise RuntimeError( f"Missing native component '{component}' in vendor source: {src_component_dir}" ) - dest_component_dir = dest_target_dir / dest_dir_name + dest_component_dir = dest_target_dir / component if dest_component_dir.exists(): shutil.rmtree(dest_component_dir) shutil.copytree(src_component_dir, dest_component_dir) @@ -448,45 +407,23 @@ def copy_native_binaries( missing_list = ", ".join(missing_targets) raise RuntimeError(f"Missing target directories in vendor source: {missing_list}") - -def validate_codex_package_dir(package_dir: Path) -> None: - is_windows = "windows" in package_dir.name - required_files = [ - Path("codex-package.json"), - Path("bin") / ("codex.exe" if is_windows else "codex"), - Path("codex-path") / ("rg.exe" if is_windows else "rg"), - ] - - if "linux" in package_dir.name: - required_files.append(Path("codex-resources") / "bwrap") - - if is_windows: - required_files.extend( - [ - Path("codex-resources") / "codex-command-runner.exe", - Path("codex-resources") / "codex-windows-sandbox-setup.exe", - ] - ) - - missing_files = [ - str(relative_path) - for relative_path in required_files - if not (package_dir / relative_path).is_file() - ] - if missing_files: - missing = ", ".join(missing_files) - raise RuntimeError(f"Missing files in Codex package directory {package_dir}: {missing}") - - def run_npm_pack(staging_dir: Path, output_path: Path) -> Path: output_path = output_path.resolve() output_path.parent.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(prefix="codex-npm-pack-") as pack_dir_str: pack_dir = Path(pack_dir_str) + npm_cache_dir = pack_dir / "npm-cache" + npm_logs_dir = pack_dir / "npm-logs" + npm_cache_dir.mkdir() + npm_logs_dir.mkdir() + env = os.environ.copy() + env["NPM_CONFIG_CACHE"] = str(npm_cache_dir) + env["NPM_CONFIG_LOGS_DIR"] = str(npm_logs_dir) stdout = subprocess.check_output( ["npm", "pack", "--json", "--pack-destination", str(pack_dir)], cwd=staging_dir, + env=env, text=True, ) try: diff --git a/codex-cli/scripts/install_native_deps.py b/codex-cli/scripts/install_native_deps.py deleted file mode 100755 index de157334cd..0000000000 --- a/codex-cli/scripts/install_native_deps.py +++ /dev/null @@ -1,654 +0,0 @@ -#!/usr/bin/env python3 -"""Install Codex package archives and native helper binaries.""" - -import argparse -from contextlib import contextmanager -import json -import os -import shutil -import subprocess -import tarfile -import tempfile -import zipfile -from dataclasses import dataclass -from concurrent.futures import ThreadPoolExecutor, as_completed -from pathlib import Path -import sys -from typing import Iterable, Sequence -from urllib.parse import urlparse -from urllib.request import urlopen - -SCRIPT_DIR = Path(__file__).resolve().parent -CODEX_CLI_ROOT = SCRIPT_DIR.parent -REPO_ROOT = CODEX_CLI_ROOT.parent -DEFAULT_WORKFLOW_URL = "https://github.com/openai/codex/actions/runs/26131514935" # rust-v0.132.0 -VENDOR_DIR_NAME = "vendor" -RG_MANIFEST = REPO_ROOT / "scripts" / "codex_package" / "rg" -BINARY_TARGETS = ( - "x86_64-unknown-linux-musl", - "aarch64-unknown-linux-musl", - "x86_64-apple-darwin", - "aarch64-apple-darwin", - "x86_64-pc-windows-msvc", - "aarch64-pc-windows-msvc", -) -CODEX_PACKAGE_COMPONENT = "codex-package" - - -@dataclass(frozen=True) -class BinaryComponent: - artifact_prefix: str # matches the artifact filename prefix (e.g. codex-.zst) - dest_dir: str # directory under vendor// where the binary is installed - binary_basename: str # executable name inside dest_dir (before optional .exe) - targets: tuple[str, ...] | None = None # limit installation to specific targets - - -WINDOWS_TARGETS = tuple(target for target in BINARY_TARGETS if "windows" in target) -LINUX_TARGETS = tuple(target for target in BINARY_TARGETS if "linux" in target) - -BINARY_COMPONENTS = { - "bwrap": BinaryComponent( - artifact_prefix="bwrap", - dest_dir="codex-resources", - binary_basename="bwrap", - targets=LINUX_TARGETS, - ), - "codex": BinaryComponent( - artifact_prefix="codex", - dest_dir="codex", - binary_basename="codex", - ), - "codex-responses-api-proxy": BinaryComponent( - artifact_prefix="codex-responses-api-proxy", - dest_dir="codex-responses-api-proxy", - binary_basename="codex-responses-api-proxy", - ), - "codex-windows-sandbox-setup": BinaryComponent( - artifact_prefix="codex-windows-sandbox-setup", - dest_dir="codex", - binary_basename="codex-windows-sandbox-setup", - targets=WINDOWS_TARGETS, - ), - "codex-command-runner": BinaryComponent( - artifact_prefix="codex-command-runner", - dest_dir="codex", - binary_basename="codex-command-runner", - targets=WINDOWS_TARGETS, - ), -} - -RG_TARGET_PLATFORM_PAIRS: list[tuple[str, str]] = [ - ("x86_64-unknown-linux-musl", "linux-x86_64"), - ("aarch64-unknown-linux-musl", "linux-aarch64"), - ("x86_64-apple-darwin", "macos-x86_64"), - ("aarch64-apple-darwin", "macos-aarch64"), - ("x86_64-pc-windows-msvc", "windows-x86_64"), - ("aarch64-pc-windows-msvc", "windows-aarch64"), -] -RG_TARGET_TO_PLATFORM = {target: platform for target, platform in RG_TARGET_PLATFORM_PAIRS} -DEFAULT_RG_TARGETS = [target for target, _ in RG_TARGET_PLATFORM_PAIRS] - -# urllib.request.urlopen() defaults to no timeout (can hang indefinitely), which is painful in CI. -DOWNLOAD_TIMEOUT_SECS = 60 - - -def _gha_enabled() -> bool: - # GitHub Actions supports "workflow commands" (e.g. ::group:: / ::error::) that make logs - # much easier to scan: groups collapse noisy sections and error annotations surface the - # failure in the UI without changing the actual exception/traceback output. - return os.environ.get("GITHUB_ACTIONS") == "true" - - -def _gha_escape(value: str) -> str: - # Workflow commands require percent/newline escaping. - return value.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") - - -def _gha_error(*, title: str, message: str) -> None: - # Emit a GitHub Actions error annotation. This does not replace stdout/stderr logs; it just - # adds a prominent summary line to the job UI so the root cause is easier to spot. - if not _gha_enabled(): - return - print( - f"::error title={_gha_escape(title)}::{_gha_escape(message)}", - flush=True, - ) - - -@contextmanager -def _gha_group(title: str): - # Wrap a block in a collapsible log group on GitHub Actions. Outside of GHA this is a no-op - # so local output remains unchanged. - if _gha_enabled(): - print(f"::group::{_gha_escape(title)}", flush=True) - try: - yield - finally: - if _gha_enabled(): - print("::endgroup::", flush=True) - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Install native Codex binaries.") - parser.add_argument( - "--workflow-url", - help=( - "GitHub Actions workflow URL that produced the artifacts. Defaults to a " - "known good run when omitted." - ), - ) - parser.add_argument( - "--component", - dest="components", - action="append", - choices=tuple([CODEX_PACKAGE_COMPONENT, *BINARY_COMPONENTS, "rg"]), - help=( - "Limit installation to the specified components." - " May be repeated. Defaults to codex-package and codex-responses-api-proxy." - ), - ) - parser.add_argument( - "--allow-legacy-codex-package", - action="store_true", - help=( - "Allow codex-package to be synthesized from legacy per-binary artifacts " - "when package archives are missing. Intended for CI compatibility only; " - "release staging should not use this. Automatically enabled for the " - "built-in default workflow." - ), - ) - parser.add_argument( - "root", - nargs="?", - type=Path, - help=( - "Directory containing package.json for the staged package. If omitted, the " - "repository checkout is used." - ), - ) - return parser.parse_args() - - -def main() -> int: - args = parse_args() - - codex_cli_root = (args.root or CODEX_CLI_ROOT).resolve() - vendor_dir = codex_cli_root / VENDOR_DIR_NAME - vendor_dir.mkdir(parents=True, exist_ok=True) - - components = args.components or [CODEX_PACKAGE_COMPONENT, "codex-responses-api-proxy"] - - workflow_override = (args.workflow_url or "").strip() - use_default_workflow = not workflow_override - workflow_url = workflow_override or DEFAULT_WORKFLOW_URL - - workflow_id = workflow_url.rstrip("/").split("/")[-1] - print(f"Downloading native artifacts from workflow {workflow_id}...") - - with _gha_group(f"Download native artifacts from workflow {workflow_id}"): - with tempfile.TemporaryDirectory(prefix="codex-native-artifacts-") as artifacts_dir_str: - artifacts_dir = Path(artifacts_dir_str) - _download_artifacts(workflow_id, artifacts_dir) - if CODEX_PACKAGE_COMPONENT in components: - try: - install_codex_package_archives(artifacts_dir, vendor_dir, BINARY_TARGETS) - except FileNotFoundError: - if not (args.allow_legacy_codex_package or use_default_workflow): - raise - install_legacy_codex_package_layouts( - artifacts_dir, - vendor_dir, - BINARY_TARGETS, - manifest_path=RG_MANIFEST, - ) - install_binary_components( - artifacts_dir, - vendor_dir, - [BINARY_COMPONENTS[name] for name in components if name in BINARY_COMPONENTS], - ) - - if "rg" in components: - with _gha_group("Fetch ripgrep binaries"): - print("Fetching ripgrep binaries...") - fetch_rg(vendor_dir, DEFAULT_RG_TARGETS, manifest_path=RG_MANIFEST) - - print(f"Installed native dependencies into {vendor_dir}") - return 0 - - -def install_codex_package_archives( - artifacts_dir: Path, - vendor_dir: Path, - targets: Sequence[str], -) -> None: - targets = list(targets) - if not targets: - return - - print("Installing Codex package archives for targets: " + ", ".join(targets)) - max_workers = min(len(targets), max(1, (os.cpu_count() or 1))) - with ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = { - executor.submit( - _install_single_codex_package_archive, - artifacts_dir, - vendor_dir, - target, - ): target - for target in targets - } - for future in as_completed(futures): - installed_path = future.result() - print(f" installed {installed_path}") - - -def _install_single_codex_package_archive( - artifacts_dir: Path, - vendor_dir: Path, - target: str, -) -> Path: - artifact_subdir = artifact_dir_for_target(artifacts_dir, target) - archive_path = artifact_subdir / f"codex-package-{target}.tar.gz" - if not archive_path.exists(): - raise FileNotFoundError(f"Expected package archive not found: {archive_path}") - - dest_dir = vendor_dir / target - if dest_dir.exists(): - shutil.rmtree(dest_dir) - dest_dir.mkdir(parents=True, exist_ok=True) - - with tarfile.open(archive_path, "r:gz") as archive: - archive.extractall(dest_dir, filter="data") - - return dest_dir - - -def install_legacy_codex_package_layouts( - artifacts_dir: Path, - vendor_dir: Path, - targets: Sequence[str], - *, - manifest_path: Path, -) -> None: - targets = list(targets) - print( - "Synthesizing Codex package layouts from legacy artifacts for targets: " - + ", ".join(targets) - ) - with tempfile.TemporaryDirectory(prefix="codex-legacy-package-") as legacy_vendor_dir_str: - legacy_vendor_dir = Path(legacy_vendor_dir_str) - install_binary_components( - artifacts_dir, - legacy_vendor_dir, - [ - BINARY_COMPONENTS["codex"], - BINARY_COMPONENTS["bwrap"], - BINARY_COMPONENTS["codex-windows-sandbox-setup"], - BINARY_COMPONENTS["codex-command-runner"], - ], - ) - fetch_rg(legacy_vendor_dir, targets, manifest_path=manifest_path) - - for target in targets: - dest_dir = vendor_dir / target - if dest_dir.exists(): - shutil.rmtree(dest_dir) - _build_legacy_codex_package_layout(legacy_vendor_dir / target, dest_dir, target) - print(f" synthesized {dest_dir}") - - -def _build_legacy_codex_package_layout( - legacy_target_dir: Path, - package_dir: Path, - target: str, -) -> None: - is_windows = "windows" in target - exe_suffix = ".exe" if is_windows else "" - package_dir.mkdir(parents=True) - - bin_dir = package_dir / "bin" - resources_dir = package_dir / "codex-resources" - path_dir = package_dir / "codex-path" - bin_dir.mkdir() - resources_dir.mkdir() - path_dir.mkdir() - - shutil.copy2( - legacy_target_dir / "codex" / f"codex{exe_suffix}", - bin_dir / f"codex{exe_suffix}", - ) - shutil.copy2( - legacy_target_dir / "path" / f"rg{exe_suffix}", - path_dir / f"rg{exe_suffix}", - ) - - if is_windows: - for helper in [ - "codex-command-runner.exe", - "codex-windows-sandbox-setup.exe", - ]: - shutil.copy2(legacy_target_dir / "codex" / helper, resources_dir / helper) - elif "linux" in target: - shutil.copy2(legacy_target_dir / "codex-resources" / "bwrap", resources_dir / "bwrap") - - write_json( - package_dir / "codex-package.json", - { - "layoutVersion": 1, - "version": "unknown", - "target": target, - "variant": "codex", - "entrypoint": f"bin/codex{exe_suffix}", - "resourcesDir": "codex-resources", - "pathDir": "codex-path", - }, - ) - - -def fetch_rg( - vendor_dir: Path, - targets: Sequence[str] | None = None, - *, - manifest_path: Path, -) -> list[Path]: - """Download ripgrep binaries described by the DotSlash manifest.""" - - if targets is None: - targets = DEFAULT_RG_TARGETS - - if not manifest_path.exists(): - raise FileNotFoundError(f"DotSlash manifest not found: {manifest_path}") - - manifest = _load_manifest(manifest_path) - platforms = manifest.get("platforms", {}) - - vendor_dir.mkdir(parents=True, exist_ok=True) - - targets = list(targets) - if not targets: - return [] - - task_configs: list[tuple[str, str, dict]] = [] - for target in targets: - platform_key = RG_TARGET_TO_PLATFORM.get(target) - if platform_key is None: - raise ValueError(f"Unsupported ripgrep target '{target}'.") - - platform_info = platforms.get(platform_key) - if platform_info is None: - raise RuntimeError(f"Platform '{platform_key}' not found in manifest {manifest_path}.") - - task_configs.append((target, platform_key, platform_info)) - - results: dict[str, Path] = {} - max_workers = min(len(task_configs), max(1, (os.cpu_count() or 1))) - - print("Installing ripgrep binaries for targets: " + ", ".join(targets)) - - with ThreadPoolExecutor(max_workers=max_workers) as executor: - future_map = { - executor.submit( - _fetch_single_rg, - vendor_dir, - target, - platform_key, - platform_info, - manifest_path, - ): target - for target, platform_key, platform_info in task_configs - } - - for future in as_completed(future_map): - target = future_map[future] - try: - results[target] = future.result() - except Exception as exc: - _gha_error( - title="ripgrep install failed", - message=f"target={target} error={exc!r}", - ) - raise RuntimeError(f"Failed to install ripgrep for target {target}.") from exc - print(f" installed ripgrep for {target}") - - return [results[target] for target in targets] - - -def _download_artifacts(workflow_id: str, dest_dir: Path) -> None: - cmd = [ - "gh", - "run", - "download", - "--dir", - str(dest_dir), - "--repo", - "openai/codex", - workflow_id, - ] - subprocess.check_call(cmd) - - -def install_binary_components( - artifacts_dir: Path, - vendor_dir: Path, - selected_components: Sequence[BinaryComponent], -) -> None: - if not selected_components: - return - - for component in selected_components: - component_targets = list(component.targets or BINARY_TARGETS) - - print( - f"Installing {component.binary_basename} binaries for targets: " - + ", ".join(component_targets) - ) - max_workers = min(len(component_targets), max(1, (os.cpu_count() or 1))) - with ThreadPoolExecutor(max_workers=max_workers) as executor: - futures = { - executor.submit( - _install_single_binary, - artifacts_dir, - vendor_dir, - target, - component, - ): target - for target in component_targets - } - for future in as_completed(futures): - installed_path = future.result() - print(f" installed {installed_path}") - - -def _install_single_binary( - artifacts_dir: Path, - vendor_dir: Path, - target: str, - component: BinaryComponent, -) -> Path: - artifact_subdir = artifact_dir_for_target(artifacts_dir, target) - archive_path = legacy_binary_archive_path(artifact_subdir, component.artifact_prefix, target) - - dest_dir = vendor_dir / target / component.dest_dir - dest_dir.mkdir(parents=True, exist_ok=True) - - binary_name = ( - f"{component.binary_basename}.exe" if "windows" in target else component.binary_basename - ) - dest = dest_dir / binary_name - dest.unlink(missing_ok=True) - extract_archive(archive_path, "zst", None, dest) - if "windows" not in target: - dest.chmod(0o755) - return dest - - -def _archive_name_for_target(artifact_prefix: str, target: str) -> str: - if "windows" in target: - return f"{artifact_prefix}-{target}.exe.zst" - return f"{artifact_prefix}-{target}.zst" - - -def legacy_binary_archive_path(artifact_dir: Path, artifact_prefix: str, target: str) -> Path: - archive_names = [_archive_name_for_target(artifact_prefix, target)] - if artifact_dir.name == f"{target}-unsigned": - archive_names.append(_archive_name_for_target(artifact_prefix, f"{target}-unsigned")) - - for archive_name in archive_names: - archive_path = artifact_dir / archive_name - if archive_path.exists(): - return archive_path - - raise FileNotFoundError(f"Expected artifact not found: {artifact_dir / archive_names[0]}") - - -def artifact_dir_for_target(artifacts_dir: Path, target: str) -> Path: - for artifact_name in [target, f"{target}-unsigned"]: - artifact_dir = artifacts_dir / artifact_name - if artifact_dir.is_dir(): - return artifact_dir - - return artifacts_dir / target - - -def _fetch_single_rg( - vendor_dir: Path, - target: str, - platform_key: str, - platform_info: dict, - manifest_path: Path, -) -> Path: - providers = platform_info.get("providers", []) - if not providers: - raise RuntimeError(f"No providers listed for platform '{platform_key}' in {manifest_path}.") - - url = providers[0]["url"] - archive_format = platform_info.get("format", "zst") - archive_member = platform_info.get("path") - digest = platform_info.get("digest") - expected_size = platform_info.get("size") - - dest_dir = vendor_dir / target / "path" - dest_dir.mkdir(parents=True, exist_ok=True) - - is_windows = platform_key.startswith("win") - binary_name = "rg.exe" if is_windows else "rg" - dest = dest_dir / binary_name - - with tempfile.TemporaryDirectory() as tmp_dir_str: - tmp_dir = Path(tmp_dir_str) - archive_filename = os.path.basename(urlparse(url).path) - download_path = tmp_dir / archive_filename - print( - f" downloading ripgrep for {target} ({platform_key}) from {url}", - flush=True, - ) - try: - _download_file(url, download_path) - except Exception as exc: - _gha_error( - title="ripgrep download failed", - message=f"target={target} platform={platform_key} url={url} error={exc!r}", - ) - raise RuntimeError( - "Failed to download ripgrep " - f"(target={target}, platform={platform_key}, format={archive_format}, " - f"expected_size={expected_size!r}, digest={digest!r}, url={url}, dest={download_path})." - ) from exc - - dest.unlink(missing_ok=True) - try: - extract_archive(download_path, archive_format, archive_member, dest) - except Exception as exc: - raise RuntimeError( - "Failed to extract ripgrep " - f"(target={target}, platform={platform_key}, format={archive_format}, " - f"member={archive_member!r}, url={url}, archive={download_path})." - ) from exc - - if not is_windows: - dest.chmod(0o755) - - return dest - - -def _download_file(url: str, dest: Path) -> None: - dest.parent.mkdir(parents=True, exist_ok=True) - dest.unlink(missing_ok=True) - - with urlopen(url, timeout=DOWNLOAD_TIMEOUT_SECS) as response, open(dest, "wb") as out: - shutil.copyfileobj(response, out) - - -def extract_archive( - archive_path: Path, - archive_format: str, - archive_member: str | None, - dest: Path, -) -> None: - dest.parent.mkdir(parents=True, exist_ok=True) - - if archive_format == "zst": - output_path = archive_path.parent / dest.name - subprocess.check_call( - ["zstd", "-f", "-d", str(archive_path), "-o", str(output_path)] - ) - shutil.move(str(output_path), dest) - return - - if archive_format == "tar.gz": - if not archive_member: - raise RuntimeError("Missing 'path' for tar.gz archive in DotSlash manifest.") - with tarfile.open(archive_path, "r:gz") as tar: - try: - member = tar.getmember(archive_member) - except KeyError as exc: - raise RuntimeError( - f"Entry '{archive_member}' not found in archive {archive_path}." - ) from exc - tar.extract(member, path=archive_path.parent, filter="data") - extracted = archive_path.parent / archive_member - shutil.move(str(extracted), dest) - return - - if archive_format == "zip": - if not archive_member: - raise RuntimeError("Missing 'path' for zip archive in DotSlash manifest.") - with zipfile.ZipFile(archive_path) as archive: - try: - with archive.open(archive_member) as src, open(dest, "wb") as out: - shutil.copyfileobj(src, out) - except KeyError as exc: - raise RuntimeError( - f"Entry '{archive_member}' not found in archive {archive_path}." - ) from exc - return - - raise RuntimeError(f"Unsupported archive format '{archive_format}'.") - - -def _load_manifest(manifest_path: Path) -> dict: - cmd = ["dotslash", "--", "parse", str(manifest_path)] - stdout = subprocess.check_output(cmd, text=True) - try: - manifest = json.loads(stdout) - except json.JSONDecodeError as exc: - raise RuntimeError(f"Invalid DotSlash manifest output from {manifest_path}.") from exc - - if not isinstance(manifest, dict): - raise RuntimeError( - f"Unexpected DotSlash manifest structure for {manifest_path}: {type(manifest)!r}" - ) - - return manifest - - -def write_json(path: Path, value: object) -> None: - with open(path, "w", encoding="utf-8") as out: - json.dump(value, out, indent=2) - out.write("\n") - - -if __name__ == "__main__": - import sys - - sys.exit(main()) diff --git a/scripts/stage_npm_packages.py b/scripts/stage_npm_packages.py index 4eb69053eb..d0cfccf37f 100755 --- a/scripts/stage_npm_packages.py +++ b/scripts/stage_npm_packages.py @@ -2,20 +2,32 @@ """Stage one or more Codex npm packages for release.""" import argparse +from concurrent.futures import ThreadPoolExecutor, as_completed +from contextlib import contextmanager +from dataclasses import dataclass import importlib.util import json import os import shutil import subprocess +import tarfile import tempfile from pathlib import Path +from typing import Sequence REPO_ROOT = Path(__file__).resolve().parent.parent BUILD_SCRIPT = REPO_ROOT / "codex-cli" / "scripts" / "build_npm_package.py" -INSTALL_NATIVE_DEPS = REPO_ROOT / "codex-cli" / "scripts" / "install_native_deps.py" WORKFLOW_NAME = ".github/workflows/rust-release.yml" GITHUB_REPO = "openai/codex" +BINARY_TARGETS = ( + "x86_64-unknown-linux-musl", + "aarch64-unknown-linux-musl", + "x86_64-apple-darwin", + "aarch64-apple-darwin", + "x86_64-pc-windows-msvc", + "aarch64-pc-windows-msvc", +) _SPEC = importlib.util.spec_from_file_location("codex_build_npm_package", BUILD_SCRIPT) if _SPEC is None or _SPEC.loader is None: @@ -25,6 +37,48 @@ _SPEC.loader.exec_module(_BUILD_MODULE) PACKAGE_NATIVE_COMPONENTS = getattr(_BUILD_MODULE, "PACKAGE_NATIVE_COMPONENTS", {}) PACKAGE_EXPANSIONS = getattr(_BUILD_MODULE, "PACKAGE_EXPANSIONS", {}) CODEX_PLATFORM_PACKAGES = getattr(_BUILD_MODULE, "CODEX_PLATFORM_PACKAGES", {}) +CODEX_PACKAGE_COMPONENT = getattr(_BUILD_MODULE, "CODEX_PACKAGE_COMPONENT", "codex-package") + + +@dataclass(frozen=True) +class BinaryComponent: + artifact_prefix: str + dest_dir: str + binary_basename: str + + +@dataclass(frozen=True) +class WorkflowArtifact: + name: str + size_in_bytes: int + + +BINARY_COMPONENTS = { + "codex-responses-api-proxy": BinaryComponent( + artifact_prefix="codex-responses-api-proxy", + dest_dir="codex-responses-api-proxy", + binary_basename="codex-responses-api-proxy", + ), +} + + +def _gha_enabled() -> bool: + return os.environ.get("GITHUB_ACTIONS") == "true" + + +def _gha_escape(value: str) -> str: + return value.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + + +@contextmanager +def _gha_group(title: str): + if _gha_enabled(): + print(f"::group::{_gha_escape(title)}", flush=True) + try: + yield + finally: + if _gha_enabled(): + print("::endgroup::", flush=True) def parse_args() -> argparse.Namespace: @@ -56,33 +110,23 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Retain temporary staging directories instead of deleting them.", ) - parser.add_argument( - "--allow-missing-native-component", - dest="allow_missing_native_components", - action="append", - default=[], - help=( - "Native component that may be absent from reused workflow artifacts. " - "Intended for CI compatibility only; release staging should not use this." - ), - ) - parser.add_argument( - "--allow-legacy-codex-package", - action="store_true", - help=( - "Allow codex-package layouts to be synthesized from legacy per-binary " - "workflow artifacts. Intended for CI compatibility only; release staging " - "should not use this." - ), - ) return parser.parse_args() -def collect_native_components(packages: list[str]) -> set[str]: - components: set[str] = set() +def native_components_for_package(package: str) -> tuple[str, ...]: + return tuple(sorted(PACKAGE_NATIVE_COMPONENTS.get(package, []))) + + +def collect_native_component_sets(packages: list[str]) -> list[tuple[str, ...]]: + component_sets: list[tuple[str, ...]] = [] + seen: set[tuple[str, ...]] = set() for package in packages: - components.update(PACKAGE_NATIVE_COMPONENTS.get(package, [])) - return components + components = native_components_for_package(package) + if not components or components in seen: + continue + seen.add(components) + component_sets.append(components) + return component_sets def expand_packages(packages: list[str]) -> list[str]: @@ -131,23 +175,280 @@ def install_native_components( workflow_url: str, components: set[str], vendor_root: Path, - *, - allow_legacy_codex_package: bool, + artifacts_dir: Path, ) -> None: if not components: return - cmd = [str(INSTALL_NATIVE_DEPS), "--workflow-url", workflow_url] - if allow_legacy_codex_package: - cmd.append("--allow-legacy-codex-package") - for component in sorted(components): - cmd.extend(["--component", component]) - cmd.append(str(vendor_root)) - run_command(cmd) + vendor_dir = vendor_root / "vendor" + vendor_dir.mkdir(parents=True, exist_ok=True) + + workflow_id = workflow_url.rstrip("/").split("/")[-1] + print(f"Downloading native artifacts from workflow {workflow_id}...", flush=True) + with _gha_group(f"Download native artifacts from workflow {workflow_id}"): + artifacts_dir.mkdir(parents=True, exist_ok=True) + install_from_workflow_artifacts( + workflow_id, + artifacts_dir, + sorted(components), + vendor_dir, + ) + print(f"Installed native dependencies into {vendor_dir}", flush=True) + + +def install_from_workflow_artifacts( + workflow_id: str, + artifacts_dir: Path, + components: Sequence[str], + vendor_dir: Path, +) -> None: + artifacts = select_target_artifacts(workflow_id, components) + download_artifacts(workflow_id, artifacts_dir, artifacts) + if CODEX_PACKAGE_COMPONENT in components: + install_codex_package_archives(artifacts_dir, vendor_dir, BINARY_TARGETS) + install_binary_components( + artifacts_dir, + vendor_dir, + [BINARY_COMPONENTS[name] for name in components if name in BINARY_COMPONENTS], + ) + + +def select_target_artifacts( + workflow_id: str, + components: Sequence[str], +) -> list[WorkflowArtifact]: + needs_target_artifacts = CODEX_PACKAGE_COMPONENT in components or any( + component in BINARY_COMPONENTS for component in components + ) + if not needs_target_artifacts: + return [] + + artifacts_by_name = { + artifact.name: artifact for artifact in list_workflow_artifacts(workflow_id) + } + selected_artifacts: list[WorkflowArtifact] = [] + for target in BINARY_TARGETS: + for artifact_name in [target, f"{target}-unsigned"]: + artifact = artifacts_by_name.get(artifact_name) + if artifact is not None: + selected_artifacts.append(artifact) + break + else: + raise FileNotFoundError( + f"Expected workflow artifact not found for target {target}" + ) + + return selected_artifacts + + +def list_workflow_artifacts(workflow_id: str) -> list[WorkflowArtifact]: + stdout = subprocess.check_output( + [ + "gh", + "api", + f"repos/{GITHUB_REPO}/actions/runs/{workflow_id}/artifacts", + "--paginate", + "--jq", + ".artifacts[] | [.name, .size_in_bytes] | @tsv", + ], + text=True, + ) + artifacts: list[WorkflowArtifact] = [] + for line in stdout.splitlines(): + name, size_in_bytes = line.split("\t", 1) + artifacts.append(WorkflowArtifact(name=name, size_in_bytes=int(size_in_bytes))) + return artifacts + + +def download_artifacts( + workflow_id: str, + dest_dir: Path, + artifacts: Sequence[WorkflowArtifact], +) -> None: + total_bytes = sum(artifact.size_in_bytes for artifact in artifacts) + print( + f"Downloading {len(artifacts)} artifacts ({format_bytes(total_bytes)})", + flush=True, + ) + for artifact in artifacts: + artifact_dir = dest_dir / artifact.name + if artifact_dir.is_dir() and any(artifact_dir.iterdir()): + print( + f" using cached {artifact.name} ({format_bytes(artifact.size_in_bytes)})", + flush=True, + ) + continue + + artifact_dir.mkdir(parents=True, exist_ok=True) + print( + f" downloading {artifact.name} ({format_bytes(artifact.size_in_bytes)})", + flush=True, + ) + subprocess.check_call( + [ + "gh", + "run", + "download", + "--name", + artifact.name, + "--dir", + str(artifact_dir), + "--repo", + GITHUB_REPO, + workflow_id, + ] + ) + + +def install_codex_package_archives( + artifacts_dir: Path, + vendor_dir: Path, + targets: Sequence[str], +) -> None: + if not targets: + return + + print( + "Installing Codex package archives for targets: " + ", ".join(targets), + flush=True, + ) + max_workers = min(len(targets), max(1, (os.cpu_count() or 1))) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = { + executor.submit( + install_single_codex_package_archive, + artifacts_dir, + vendor_dir, + target, + ): target + for target in targets + } + for future in as_completed(futures): + installed_path = future.result() + print(f" installed {installed_path}", flush=True) + + +def install_single_codex_package_archive( + artifacts_dir: Path, + vendor_dir: Path, + target: str, +) -> Path: + artifact_subdir = artifact_dir_for_target(artifacts_dir, target) + archive_path = artifact_subdir / f"codex-package-{target}.tar.gz" + if not archive_path.exists(): + raise FileNotFoundError(f"Expected package archive not found: {archive_path}") + + dest_dir = vendor_dir / target + if dest_dir.exists(): + shutil.rmtree(dest_dir) + dest_dir.mkdir(parents=True, exist_ok=True) + + with tarfile.open(archive_path, "r:gz") as archive: + archive.extractall(dest_dir, filter="data") + + return dest_dir + + +def install_binary_components( + artifacts_dir: Path, + vendor_dir: Path, + selected_components: Sequence[BinaryComponent], +) -> None: + for component in selected_components: + component_targets = list(BINARY_TARGETS) + + print( + f"Installing {component.binary_basename} binaries for targets: " + + ", ".join(component_targets), + flush=True, + ) + max_workers = min(len(component_targets), max(1, (os.cpu_count() or 1))) + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = { + executor.submit( + install_single_binary, + artifacts_dir, + vendor_dir, + target, + component, + ): target + for target in component_targets + } + for future in as_completed(futures): + installed_path = future.result() + print(f" installed {installed_path}", flush=True) + + +def install_single_binary( + artifacts_dir: Path, + vendor_dir: Path, + target: str, + component: BinaryComponent, +) -> Path: + artifact_subdir = artifact_dir_for_target(artifacts_dir, target) + archive_path = binary_archive_path(artifact_subdir, component.artifact_prefix, target) + + dest_dir = vendor_dir / target / component.dest_dir + dest_dir.mkdir(parents=True, exist_ok=True) + + binary_name = ( + f"{component.binary_basename}.exe" if "windows" in target else component.binary_basename + ) + dest = dest_dir / binary_name + dest.unlink(missing_ok=True) + extract_zstd_archive(archive_path, dest) + if "windows" not in target: + dest.chmod(0o755) + return dest + + +def binary_archive_path(artifact_dir: Path, artifact_prefix: str, target: str) -> Path: + archive_names = [archive_name_for_target(artifact_prefix, target)] + if artifact_dir.name == f"{target}-unsigned": + archive_names.append(archive_name_for_target(artifact_prefix, f"{target}-unsigned")) + + for archive_name in archive_names: + archive_path = artifact_dir / archive_name + if archive_path.exists(): + return archive_path + + raise FileNotFoundError(f"Expected artifact not found: {artifact_dir / archive_names[0]}") + + +def archive_name_for_target(artifact_prefix: str, target: str) -> str: + if "windows" in target: + return f"{artifact_prefix}-{target}.exe.zst" + return f"{artifact_prefix}-{target}.zst" + + +def artifact_dir_for_target(artifacts_dir: Path, target: str) -> Path: + for artifact_name in [target, f"{target}-unsigned"]: + artifact_dir = artifacts_dir / artifact_name + if artifact_dir.is_dir(): + return artifact_dir + + return artifacts_dir / target + + +def extract_zstd_archive(archive_path: Path, dest: Path) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + + output_path = archive_path.parent / dest.name + subprocess.check_call(["zstd", "-f", "-d", str(archive_path), "-o", str(output_path)]) + shutil.move(str(output_path), dest) + + +def format_bytes(size_in_bytes: int) -> str: + value = float(size_in_bytes) + for unit in ["B", "KiB", "MiB"]: + if value < 1024: + return f"{value:.1f} {unit}" + value /= 1024 + return f"{value:.1f} GiB" def run_command(cmd: list[str]) -> None: - print("+", " ".join(cmd)) + print("+", " ".join(cmd), flush=True) subprocess.run(cmd, cwd=REPO_ROOT, check=True) @@ -167,36 +468,58 @@ def main() -> int: runner_temp = Path(os.environ.get("RUNNER_TEMP", tempfile.gettempdir())) packages = expand_packages(list(args.packages)) - native_components = collect_native_components(packages) - allow_missing_native_components = set(args.allow_missing_native_components) - native_components_to_install = native_components - allow_missing_native_components + native_component_sets = collect_native_component_sets(packages) + print("Expanded packages: " + ", ".join(packages), flush=True) + if native_component_sets: + component_sets = [ + "(" + ", ".join(components) + ")" for components in native_component_sets + ] + print( + "Native component sets: " + ", ".join(component_sets), + flush=True, + ) - vendor_temp_root: Path | None = None - vendor_src: Path | None = None + vendor_temp_roots: list[Path] = [] + vendor_src_by_components: dict[tuple[str, ...], Path] = {} + artifacts_temp_root: Path | None = None resolved_head_sha: str | None = None final_messages = [] try: - if native_components_to_install: + if native_component_sets: workflow_url, resolved_head_sha = resolve_workflow_url( args.release_version, args.workflow_url ) - vendor_temp_root = Path(tempfile.mkdtemp(prefix="npm-native-", dir=runner_temp)) - install_native_components( - workflow_url, - native_components_to_install, - vendor_temp_root, - allow_legacy_codex_package=args.allow_legacy_codex_package, + print(f"Using native artifacts from {workflow_url}", flush=True) + artifacts_temp_root = Path( + tempfile.mkdtemp(prefix="npm-native-artifacts-", dir=runner_temp) ) - vendor_src = vendor_temp_root / "vendor" + print(f"Caching downloaded artifacts in {artifacts_temp_root}", flush=True) + for components in native_component_sets: + vendor_temp_root = Path(tempfile.mkdtemp(prefix="npm-native-", dir=runner_temp)) + vendor_temp_roots.append(vendor_temp_root) + print( + "Installing native components " + + ", ".join(components) + + f" into {vendor_temp_root}", + flush=True, + ) + install_native_components( + workflow_url, + set(components), + vendor_temp_root, + artifacts_temp_root, + ) + vendor_src_by_components[components] = vendor_temp_root / "vendor" if resolved_head_sha: - print(f"should `git checkout {resolved_head_sha}`") + print(f"should `git checkout {resolved_head_sha}`", flush=True) for package in packages: staging_dir = Path(tempfile.mkdtemp(prefix=f"npm-stage-{package}-", dir=runner_temp)) pack_output = output_dir / tarball_name_for_package(package, args.release_version) + print(f"Staging {package} in {staging_dir}", flush=True) cmd = [ str(BUILD_SCRIPT), @@ -210,12 +533,10 @@ def main() -> int: str(pack_output), ] + vendor_src = vendor_src_by_components.get(native_components_for_package(package)) if vendor_src is not None: cmd.extend(["--vendor-src", str(vendor_src)]) - for component in sorted(allow_missing_native_components): - cmd.extend(["--allow-missing-native-component", component]) - try: run_command(cmd) finally: @@ -224,11 +545,14 @@ def main() -> int: final_messages.append(f"Staged {package} at {pack_output}") finally: - if vendor_temp_root is not None and not args.keep_staging_dirs: - shutil.rmtree(vendor_temp_root, ignore_errors=True) + if not args.keep_staging_dirs: + for vendor_temp_root in vendor_temp_roots: + shutil.rmtree(vendor_temp_root, ignore_errors=True) + if artifacts_temp_root is not None: + shutil.rmtree(artifacts_temp_root, ignore_errors=True) for msg in final_messages: - print(msg) + print(msg, flush=True) return 0