diff --git a/.github/workflows/rust-release-windows.yml b/.github/workflows/rust-release-windows.yml index 12be586188..51412be0e0 100644 --- a/.github/workflows/rust-release-windows.yml +++ b/.github/workflows/rust-release-windows.yml @@ -258,16 +258,12 @@ jobs: stage_dir="${RUNNER_TEMP}/openai-codex-cli-bin-${{ matrix.target }}" wheel_dir="${GITHUB_WORKSPACE}/python-runtime-dist/${{ matrix.target }}" - # Keep the helpers next to codex.exe in the runtime wheel so Windows - # sandbox/elevation lookup matches the standalone release zip. python "${GITHUB_WORKSPACE}/sdk/python/scripts/update_sdk_artifacts.py" \ stage-runtime \ "$stage_dir" \ - "${GITHUB_WORKSPACE}/codex-rs/target/${{ matrix.target }}/release/codex.exe" \ + "dist/${{ matrix.target }}/codex-package-${{ matrix.target }}.tar.gz" \ --codex-version "${GITHUB_REF_NAME}" \ - --platform-tag "$platform_tag" \ - --resource-binary "${GITHUB_WORKSPACE}/codex-rs/target/${{ matrix.target }}/release/codex-command-runner.exe" \ - --resource-binary "${GITHUB_WORKSPACE}/codex-rs/target/${{ matrix.target }}/release/codex-windows-sandbox-setup.exe" + --platform-tag "$platform_tag" "${RUNNER_TEMP}/python-runtime-build-venv/Scripts/python.exe" -m build --wheel --outdir "$wheel_dir" "$stage_dir" - name: Upload Python runtime wheel diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index c88fede7fa..4f10efa9dc 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -569,18 +569,10 @@ jobs: "${GITHUB_WORKSPACE}/sdk/python/scripts/update_sdk_artifacts.py" stage-runtime "$stage_dir" - "${GITHUB_WORKSPACE}/codex-rs/target/${{ matrix.target }}/release/codex" + "dist/${{ matrix.target }}/codex-package-${{ matrix.target }}.tar.gz" --codex-version "${GITHUB_REF_NAME}" --platform-tag "$platform_tag" ) - if [[ "${{ matrix.target }}" == *linux* ]]; then - # Keep bwrap in the runtime wheel so Linux sandbox fallback behavior - # matches the standalone release bundle on hosts without system bwrap. - stage_runtime_args+=( - --resource-binary - "${GITHUB_WORKSPACE}/codex-rs/target/${{ matrix.target }}/release/bwrap" - ) - fi python3 "${stage_runtime_args[@]}" "${RUNNER_TEMP}/python-runtime-build-venv/bin/python" -m build --wheel --outdir "$wheel_dir" "$stage_dir" @@ -800,6 +792,20 @@ jobs: cp "$dmg_source" "$dest/$dmg_name" fi + - name: Build Codex package archive + shell: bash + env: + TARGET: ${{ matrix.target }} + BUNDLE: ${{ matrix.bundle }} + run: | + set -euo pipefail + bash "${GITHUB_WORKSPACE}/.github/scripts/build-codex-package-archive.sh" \ + --target "$TARGET" \ + --bundle "$BUNDLE" \ + --entrypoint-dir "dist/${TARGET}" \ + --archive-dir "dist/${TARGET}" \ + --target-suffixed-entrypoint + - name: Build Python runtime wheel if: ${{ matrix.bundle == 'primary' }} shell: bash @@ -828,25 +834,11 @@ jobs: "${GITHUB_WORKSPACE}/sdk/python/scripts/update_sdk_artifacts.py" \ stage-runtime \ "$stage_dir" \ - "${GITHUB_WORKSPACE}/codex-rs/dist/${{ matrix.target }}/codex-${{ matrix.target }}" \ + "dist/${{ matrix.target }}/codex-package-${{ matrix.target }}.tar.gz" \ --codex-version "${GITHUB_REF_NAME}" \ --platform-tag "$platform_tag" "${RUNNER_TEMP}/python-runtime-build-venv/bin/python" -m build --wheel --outdir "$wheel_dir" "$stage_dir" - - name: Build Codex package archive - shell: bash - env: - TARGET: ${{ matrix.target }} - BUNDLE: ${{ matrix.bundle }} - run: | - set -euo pipefail - bash "${GITHUB_WORKSPACE}/.github/scripts/build-codex-package-archive.sh" \ - --target "$TARGET" \ - --bundle "$BUNDLE" \ - --entrypoint-dir "dist/${TARGET}" \ - --archive-dir "dist/${TARGET}" \ - --target-suffixed-entrypoint - - name: Upload Python runtime wheel if: ${{ matrix.bundle == 'primary' }} uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 diff --git a/sdk/python-runtime/pyproject.toml b/sdk/python-runtime/pyproject.toml index 789453d059..e01a517037 100644 --- a/sdk/python-runtime/pyproject.toml +++ b/sdk/python-runtime/pyproject.toml @@ -36,7 +36,12 @@ exclude = [ [tool.hatch.build.targets.wheel] packages = ["src/codex_cli_bin"] -include = ["src/codex_cli_bin/bin/**"] +include = [ + "src/codex_cli_bin/codex-package.json", + "src/codex_cli_bin/bin/**", + "src/codex_cli_bin/codex-resources/**", + "src/codex_cli_bin/codex-path/**", +] [tool.hatch.build.targets.wheel.hooks.custom] diff --git a/sdk/python-runtime/src/codex_cli_bin/__init__.py b/sdk/python-runtime/src/codex_cli_bin/__init__.py index dbd9a6f660..4c563de8bb 100644 --- a/sdk/python-runtime/src/codex_cli_bin/__init__.py +++ b/sdk/python-runtime/src/codex_cli_bin/__init__.py @@ -1,14 +1,23 @@ -from __future__ import annotations - import os from pathlib import Path PACKAGE_NAME = "openai-codex-cli-bin" +PACKAGE_METADATA_FILENAME = "codex-package.json" + + +def bundled_package_dir() -> Path: + path = Path(__file__).resolve().parent + metadata_path = path / PACKAGE_METADATA_FILENAME + if not metadata_path.is_file(): + raise FileNotFoundError( + f"{PACKAGE_NAME} is installed but missing its package metadata at {metadata_path}" + ) + return path def bundled_codex_path() -> Path: exe = "codex.exe" if os.name == "nt" else "codex" - path = Path(__file__).resolve().parent / "bin" / exe + path = bundled_package_dir() / "bin" / exe if not path.is_file(): raise FileNotFoundError( f"{PACKAGE_NAME} is installed but missing its packaged codex binary at {path}" @@ -16,4 +25,14 @@ def bundled_codex_path() -> Path: return path -__all__ = ["PACKAGE_NAME", "bundled_codex_path"] +def bundled_path_dir() -> Path | None: + path = bundled_package_dir() / "codex-path" + return path if path.is_dir() else None + + +__all__ = [ + "PACKAGE_NAME", + "bundled_codex_path", + "bundled_package_dir", + "bundled_path_dir", +] diff --git a/sdk/python/_runtime_setup.py b/sdk/python/_runtime_setup.py index db7007fa64..2c04254982 100644 --- a/sdk/python/_runtime_setup.py +++ b/sdk/python/_runtime_setup.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import importlib import importlib.metadata import importlib.util @@ -10,11 +8,9 @@ import re import shutil import subprocess import sys -import tarfile import tempfile import urllib.error import urllib.request -import zipfile from pathlib import Path PACKAGE_NAME = "openai-codex-cli-bin" @@ -65,11 +61,10 @@ def ensure_runtime_package_installed( with tempfile.TemporaryDirectory(prefix="codex-python-runtime-") as temp_root_str: temp_root = Path(temp_root_str) archive_path = _download_release_archive(requested_version, temp_root) - runtime_binary = _extract_runtime_binary(archive_path, temp_root) staged_runtime_dir = _stage_runtime_package( sdk_python_dir, requested_version, - runtime_binary, + archive_path, temp_root / "runtime-stage", ) _install_runtime_package(python_executable, staged_runtime_dir, install_target) @@ -98,19 +93,19 @@ def platform_asset_name() -> str: if system == "darwin": if machine in {"arm64", "aarch64"}: - return "codex-aarch64-apple-darwin.tar.gz" + return "codex-package-aarch64-apple-darwin.tar.gz" if machine in {"x86_64", "amd64"}: - return "codex-x86_64-apple-darwin.tar.gz" + return "codex-package-x86_64-apple-darwin.tar.gz" elif system == "linux": if machine in {"aarch64", "arm64"}: - return "codex-aarch64-unknown-linux-musl.tar.gz" + return "codex-package-aarch64-unknown-linux-musl.tar.gz" if machine in {"x86_64", "amd64"}: - return "codex-x86_64-unknown-linux-musl.tar.gz" + return "codex-package-x86_64-unknown-linux-musl.tar.gz" elif system == "windows": if machine in {"aarch64", "arm64"}: - return "codex-aarch64-pc-windows-msvc.exe.zip" + return "codex-package-aarch64-pc-windows-msvc.tar.gz" if machine in {"x86_64", "amd64"}: - return "codex-x86_64-pc-windows-msvc.exe.zip" + return "codex-package-x86_64-pc-windows-msvc.tar.gz" raise RuntimeSetupError( f"Unsupported runtime artifact platform: system={platform.system()!r}, " @@ -118,10 +113,6 @@ def platform_asset_name() -> str: ) -def runtime_binary_name() -> str: - return "codex.exe" if platform.system().lower() == "windows" else "codex" - - def _installed_runtime_version(python_executable: str | Path) -> str | None: snippet = ( "import importlib.metadata, json, sys\n" @@ -260,49 +251,17 @@ def _download_release_archive(version: str, temp_root: Path) -> Path: return archive_path -def _extract_runtime_binary(archive_path: Path, temp_root: Path) -> Path: - extract_dir = temp_root / "extracted" - extract_dir.mkdir(parents=True, exist_ok=True) - if archive_path.name.endswith(".tar.gz"): - with tarfile.open(archive_path, "r:gz") as tar: - try: - tar.extractall(extract_dir, filter="data") - except TypeError: - tar.extractall(extract_dir) - elif archive_path.suffix == ".zip": - with zipfile.ZipFile(archive_path) as zip_file: - zip_file.extractall(extract_dir) - else: - raise RuntimeSetupError(f"Unsupported release archive format: {archive_path.name}") - - binary_name = runtime_binary_name() - archive_stem = archive_path.name.removesuffix(".tar.gz").removesuffix(".zip") - candidates = [ - path - for path in extract_dir.rglob("*") - if path.is_file() - and ( - path.name == binary_name or path.name == archive_stem or path.name.startswith("codex-") - ) - ] - if not candidates: - raise RuntimeSetupError( - f"Failed to find {binary_name} in extracted runtime archive {archive_path.name}." - ) - return candidates[0] - - def _stage_runtime_package( sdk_python_dir: Path, runtime_version: str, - runtime_binary: Path, + runtime_package_archive: Path, staging_dir: Path, ) -> Path: script_module = _load_update_script_module(sdk_python_dir) return script_module.stage_python_runtime_package( # type: ignore[no-any-return] staging_dir, runtime_version, - runtime_binary.resolve(), + runtime_package_archive.resolve(), ) diff --git a/sdk/python/scripts/update_sdk_artifacts.py b/sdk/python/scripts/update_sdk_artifacts.py index 5cad14cf5e..e88c45521f 100755 --- a/sdk/python/scripts/update_sdk_artifacts.py +++ b/sdk/python/scripts/update_sdk_artifacts.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -from __future__ import annotations import argparse import importlib @@ -8,9 +7,9 @@ import json import platform import re import shutil -import stat import subprocess import sys +import tarfile import tempfile import types import typing @@ -20,6 +19,8 @@ from typing import Any, Callable, Sequence, get_args, get_origin SDK_DISTRIBUTION_NAME = "openai-codex" RUNTIME_DISTRIBUTION_NAME = "openai-codex-cli-bin" +RUNTIME_PACKAGE_ROOT = Path("src") / "codex_cli_bin" +CODEX_PACKAGE_METADATA = "codex-package.json" def repo_root() -> Path: @@ -52,16 +53,8 @@ def runtime_binary_name() -> str: return "codex.exe" if _is_windows() else "codex" -def staged_runtime_bin_path(root: Path) -> Path: - return root / "src" / "codex_cli_bin" / "bin" / runtime_binary_name() - - -def staged_runtime_resource_path(root: Path, resource: Path) -> Path: - """Stage runtime helper binaries beside the main bundled Codex binary.""" - # Runtime wheels include the whole bin/ directory, so helper executables - # should be staged beside the main Codex binary instead of changing the - # package template for each platform. - return root / "src" / "codex_cli_bin" / "bin" / resource.name +def staged_runtime_package_root(root: Path) -> Path: + return root / RUNTIME_PACKAGE_ROOT def run(cmd: list[str], cwd: Path) -> None: @@ -259,9 +252,8 @@ def stage_python_sdk_package(staging_dir: Path, codex_version: str) -> Path: def stage_python_runtime_package( staging_dir: Path, codex_version: str, - binary_path: Path, + package_archive: Path, platform_tag: str | None = None, - resource_binaries: Sequence[Path] = (), ) -> Path: package_version = normalize_codex_version(codex_version) _copy_package_tree(python_runtime_root(), staging_dir) @@ -274,24 +266,39 @@ def stage_python_runtime_package( pyproject_text = _rewrite_runtime_platform_tag(pyproject_text, platform_tag) pyproject_path.write_text(pyproject_text) - out_bin = staged_runtime_bin_path(staging_dir) - out_bin.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(binary_path, out_bin) - if not _is_windows(): - out_bin.chmod(out_bin.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) - for resource_binary in resource_binaries: - # Some release targets need helper executables beside the main binary - # (for example Linux bwrap or Windows sandbox helpers). Keep this - # generic so release workflows own the platform-specific list. - out_resource = staged_runtime_resource_path(staging_dir, resource_binary) - shutil.copy2(resource_binary, out_resource) - if not _is_windows(): - out_resource.chmod( - out_resource.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH - ) + _extract_codex_package_archive(package_archive, staged_runtime_package_root(staging_dir)) return staging_dir +def _extract_codex_package_archive(package_archive: Path, runtime_package_root: Path) -> None: + if not package_archive.name.endswith(".tar.gz"): + raise RuntimeError(f"Expected a .tar.gz Codex package archive: {package_archive}") + + runtime_package_root.mkdir(parents=True, exist_ok=True) + with tarfile.open(package_archive, "r:gz") as archive: + try: + archive.extractall(runtime_package_root, filter="data") + except TypeError: + archive.extractall(runtime_package_root) + + _validate_codex_package_layout(runtime_package_root, package_archive) + + +def _validate_codex_package_layout(package_dir: Path, package_archive: Path) -> None: + missing_entries = [] + if not (package_dir / CODEX_PACKAGE_METADATA).is_file(): + missing_entries.append(CODEX_PACKAGE_METADATA) + for entry in ("bin", "codex-resources", "codex-path"): + if not (package_dir / entry).is_dir(): + missing_entries.append(entry) + package_binary = package_dir / "bin" / runtime_binary_name() + if not package_binary.is_file(): + missing_entries.append(str(Path("bin") / runtime_binary_name())) + if missing_entries: + missing = ", ".join(missing_entries) + raise RuntimeError(f"Missing Codex package layout entries in {package_archive}: {missing}") + + def _flatten_string_enum_one_of(definition: dict[str, Any]) -> bool: branches = definition.get("oneOf") if not isinstance(branches, list) or not branches: @@ -752,7 +759,7 @@ class PublicFieldSpec: class CliOps: generate_types: Callable[[], None] stage_python_sdk_package: Callable[[Path, str], Path] - stage_python_runtime_package: Callable[[Path, str, Path, str | None, Sequence[Path]], Path] + stage_python_runtime_package: Callable[[Path, str, Path, str | None], Path] current_sdk_version: Callable[[], str] @@ -1218,9 +1225,9 @@ def build_parser() -> argparse.ArgumentParser: help="Output directory for the staged runtime package", ) stage_runtime_parser.add_argument( - "runtime_binary", + "package_archive", type=Path, - help="Path to the codex binary to package for this platform", + help="Path to a Codex package .tar.gz archive for this platform.", ) stage_runtime_parser.add_argument( "--codex-version", @@ -1240,13 +1247,6 @@ def build_parser() -> argparse.ArgumentParser: "macosx_11_0_arm64 or musllinux_1_1_x86_64." ), ) - stage_runtime_parser.add_argument( - "--resource-binary", - action="append", - default=[], - type=Path, - help="Additional executable to package beside the codex runtime binary.", - ) return parser @@ -1297,9 +1297,8 @@ def run_command(args: argparse.Namespace, ops: CliOps) -> None: ops.stage_python_runtime_package( args.staging_dir, codex_version, - args.runtime_binary.resolve(), + args.package_archive.resolve(), args.platform_tag, - tuple(path.resolve() for path in args.resource_binary), ) diff --git a/sdk/python/src/openai_codex/client.py b/sdk/python/src/openai_codex/client.py index 6f2e2b8aac..794411d2a1 100644 --- a/sdk/python/src/openai_codex/client.py +++ b/sdk/python/src/openai_codex/client.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import json import os import subprocess @@ -103,6 +101,45 @@ def _installed_codex_path() -> Path: return bundled_codex_path() +def _installed_codex_path_dirs() -> tuple[Path, ...]: + try: + from codex_cli_bin import bundled_path_dir + except (ImportError, AttributeError): + return () + + path_dir = bundled_path_dir() + return (path_dir,) if path_dir is not None else () + + +def _prepend_path_dirs(env: dict[str, str], path_dirs: tuple[Path, ...]) -> None: + if not path_dirs: + return + + path_key = _path_env_key(env) + if os.name == "nt": + for key in list(env): + if key.upper() == "PATH" and key != path_key: + env.pop(key) + + path_sep = os.pathsep + existing_path = env.get(path_key, "") + path_dir_values = [str(path_dir) for path_dir in path_dirs] + existing_entries = [ + entry for entry in existing_path.split(path_sep) if entry and entry not in path_dir_values + ] + env[path_key] = path_sep.join([*path_dir_values, *existing_entries]) + + +def _path_env_key(env: dict[str, str]) -> str: + if os.name != "nt": + return "PATH" + + matching_keys = [key for key in env if key.upper() == "PATH"] + if "Path" in matching_keys: + return "Path" + return matching_keys[-1] if matching_keys else "PATH" + + @dataclass(frozen=True) class CodexBinResolverOps: installed_codex_path: Callable[[], Path] @@ -174,10 +211,13 @@ class AppServerClient: if self._proc is not None: return + path_dirs: tuple[Path, ...] = () if self.config.launch_args_override is not None: args = list(self.config.launch_args_override) else: codex_bin = _resolve_codex_bin(self.config) + if self.config.codex_bin is None: + path_dirs = _installed_codex_path_dirs() args = [str(codex_bin)] for kv in self.config.config_overrides: args.extend(["--config", kv]) @@ -186,6 +226,7 @@ class AppServerClient: env = os.environ.copy() if self.config.env: env.update(self.config.env) + _prepend_path_dirs(env, path_dirs) self._proc = subprocess.Popen( args, diff --git a/sdk/python/tests/test_artifact_workflow_and_binaries.py b/sdk/python/tests/test_artifact_workflow_and_binaries.py index b43cdd5703..44459d9f76 100644 --- a/sdk/python/tests/test_artifact_workflow_and_binaries.py +++ b/sdk/python/tests/test_artifact_workflow_and_binaries.py @@ -1,13 +1,12 @@ -from __future__ import annotations - import ast import importlib.util import io import json +import os import sys +import tarfile import urllib.error from pathlib import Path -from typing import Sequence import pytest import tomllib @@ -39,6 +38,30 @@ def _load_runtime_setup_module(): return module +def _write_fake_codex_package(package_dir: Path, script) -> Path: + (package_dir / "bin").mkdir(parents=True) + (package_dir / "codex-resources").mkdir() + (package_dir / "codex-path").mkdir() + (package_dir / "codex-package.json").write_text('{"variant":"codex"}\n') + (package_dir / "bin" / script.runtime_binary_name()).write_text("fake codex\n") + (package_dir / "codex-resources" / "bwrap").write_text("fake bwrap\n") + (package_dir / "codex-path" / "rg").write_text("fake rg\n") + return package_dir + + +def _write_fake_codex_package_archive(tmp_path: Path, script) -> Path: + package_dir = _write_fake_codex_package(tmp_path / "codex-package", script) + archive_path = tmp_path / "codex-package.tar.gz" + _write_package_archive(package_dir, archive_path) + return archive_path + + +def _write_package_archive(package_dir: Path, archive_path: Path) -> None: + with tarfile.open(archive_path, "w:gz") as archive: + for path in package_dir.rglob("*"): + archive.add(path, arcname=path.relative_to(package_dir)) + + def test_generation_has_single_maintenance_entrypoint_script() -> None: """Keep artifact workflows routed through one script instead of side entrypoints.""" scripts = sorted(p.name for p in (ROOT / "scripts").glob("*.py")) @@ -276,6 +299,27 @@ def test_runtime_setup_uses_pep440_package_version_and_codex_release_tags() -> N assert runtime_setup._release_tag("0.116.0a1") == "rust-v0.116.0-alpha.1" +@pytest.mark.parametrize( + ("system", "machine", "asset_name"), + [ + ("Darwin", "arm64", "codex-package-aarch64-apple-darwin.tar.gz"), + ("Linux", "x86_64", "codex-package-x86_64-unknown-linux-musl.tar.gz"), + ("Windows", "AMD64", "codex-package-x86_64-pc-windows-msvc.tar.gz"), + ], +) +def test_runtime_setup_downloads_codex_package_archives( + monkeypatch: pytest.MonkeyPatch, + system: str, + machine: str, + asset_name: str, +) -> None: + runtime_setup = _load_runtime_setup_module() + monkeypatch.setattr(runtime_setup.platform, "system", lambda: system) + monkeypatch.setattr(runtime_setup.platform, "machine", lambda: machine) + + assert runtime_setup.platform_asset_name() == asset_name + + def test_runtime_package_is_wheel_only_and_builds_platform_specific_wheels() -> None: pyproject = tomllib.loads((ROOT.parent / "python-runtime" / "pyproject.toml").read_text()) hook_source = (ROOT.parent / "python-runtime" / "hatch_build.py").read_text() @@ -324,7 +368,12 @@ def test_runtime_package_is_wheel_only_and_builds_platform_specific_wheels() -> assert pyproject["project"]["name"] == "openai-codex-cli-bin" assert pyproject["tool"]["hatch"]["build"]["targets"]["wheel"] == { "packages": ["src/codex_cli_bin"], - "include": ["src/codex_cli_bin/bin/**"], + "include": [ + "src/codex_cli_bin/codex-package.json", + "src/codex_cli_bin/bin/**", + "src/codex_cli_bin/codex-resources/**", + "src/codex_cli_bin/codex-path/**", + ], "hooks": {"custom": {}}, } assert pyproject["tool"]["hatch"]["build"]["targets"]["sdist"] == { @@ -338,19 +387,30 @@ def test_runtime_package_is_wheel_only_and_builds_platform_specific_wheels() -> } -def test_stage_runtime_release_copies_binary_and_sets_version(tmp_path: Path) -> None: +def test_stage_runtime_release_copies_package_layout_and_sets_version( + tmp_path: Path, +) -> None: script = _load_update_script_module() - fake_binary = tmp_path / script.runtime_binary_name() - fake_binary.write_text("fake codex\n") + package_archive = _write_fake_codex_package_archive(tmp_path, script) staged = script.stage_python_runtime_package( tmp_path / "runtime-stage", "1.2.3", - fake_binary, + package_archive, ) + package_root = script.staged_runtime_package_root(staged) - assert staged == tmp_path / "runtime-stage" - assert script.staged_runtime_bin_path(staged).read_text() == "fake codex\n" + assert { + "metadata": (package_root / "codex-package.json").read_text(), + "codex": (package_root / "bin" / script.runtime_binary_name()).read_text(), + "bwrap": (package_root / "codex-resources" / "bwrap").read_text(), + "rg": (package_root / "codex-path" / "rg").read_text(), + } == { + "metadata": '{"variant":"codex"}\n', + "codex": "fake codex\n", + "bwrap": "fake bwrap\n", + "rg": "fake rg\n", + } assert 'name = "openai-codex-cli-bin"' in (staged / "pyproject.toml").read_text() assert 'version = "1.2.3"' in (staged / "pyproject.toml").read_text() @@ -370,30 +430,28 @@ def test_stage_runtime_release_replaces_existing_staging_dir(tmp_path: Path) -> old_file = staging_dir / "stale.txt" old_file.parent.mkdir(parents=True) old_file.write_text("stale") - - fake_binary = tmp_path / script.runtime_binary_name() - fake_binary.write_text("fake codex\n") + package_archive = _write_fake_codex_package_archive(tmp_path, script) staged = script.stage_python_runtime_package( staging_dir, "1.2.3", - fake_binary, + package_archive, ) assert staged == staging_dir assert not old_file.exists() - assert script.staged_runtime_bin_path(staged).read_text() == "fake codex\n" + package_root = script.staged_runtime_package_root(staged) + assert (package_root / "bin" / script.runtime_binary_name()).read_text() == "fake codex\n" def test_stage_runtime_release_can_pin_wheel_platform_tag(tmp_path: Path) -> None: script = _load_update_script_module() - fake_binary = tmp_path / script.runtime_binary_name() - fake_binary.write_text("fake codex\n") + package_archive = _write_fake_codex_package_archive(tmp_path, script) staged = script.stage_python_runtime_package( tmp_path / "runtime-stage", "0.116.0a1", - fake_binary, + package_archive, platform_tag="musllinux_1_1_x86_64", ) @@ -401,58 +459,36 @@ def test_stage_runtime_release_can_pin_wheel_platform_tag(tmp_path: Path) -> Non assert 'platform-tag = "musllinux_1_1_x86_64"' in pyproject -def test_stage_runtime_release_copies_resource_binaries(tmp_path: Path) -> None: - """Runtime staging should copy every helper binary into the wheel bin dir.""" +def test_stage_runtime_release_rejects_incomplete_package_layout(tmp_path: Path) -> None: script = _load_update_script_module() - fake_binary = tmp_path / script.runtime_binary_name() - helper = tmp_path / "helper" - fallback = tmp_path / "fallback-helper" - fake_binary.write_text("fake codex\n") - helper.write_text("fake helper\n") - fallback.write_text("fake fallback\n") + package_dir = tmp_path / "codex-package" + (package_dir / "bin").mkdir(parents=True) + package_archive = tmp_path / "codex-package.tar.gz" + _write_package_archive(package_dir, package_archive) - staged = script.stage_python_runtime_package( - tmp_path / "runtime-stage", - "1.2.3", - fake_binary, - resource_binaries=(helper, fallback), - ) - - assert { - path.relative_to(staged / "src" / "codex_cli_bin" / "bin").as_posix(): path.read_text() - for path in (staged / "src" / "codex_cli_bin" / "bin").iterdir() - } == { - script.runtime_binary_name(): "fake codex\n", - "fallback-helper": "fake fallback\n", - "helper": "fake helper\n", - } + with pytest.raises(RuntimeError, match="Missing Codex package layout entries"): + script.stage_python_runtime_package(tmp_path / "runtime-stage", "1.2.3", package_archive) -def test_runtime_resource_binaries_are_included_by_wheel_config( +def test_runtime_package_layout_is_included_by_wheel_config( tmp_path: Path, ) -> None: - """The runtime wheel config should include helper binaries beside Codex.""" script = _load_update_script_module() - fake_binary = tmp_path / script.runtime_binary_name() - helper = tmp_path / "helper" - fake_binary.write_text("fake codex\n") - helper.write_text("fake helper\n") + package_archive = _write_fake_codex_package_archive(tmp_path, script) staged = script.stage_python_runtime_package( tmp_path / "runtime-stage", "1.2.3", - fake_binary, - resource_binaries=(helper,), + package_archive, ) pyproject = tomllib.loads((staged / "pyproject.toml").read_text()) - assert { - "include": pyproject["tool"]["hatch"]["build"]["targets"]["wheel"]["include"], - "helper": (staged / "src" / "codex_cli_bin" / "bin" / "helper").read_text(), - } == { - "include": ["src/codex_cli_bin/bin/**"], - "helper": "fake helper\n", - } + assert pyproject["tool"]["hatch"]["build"]["targets"]["wheel"]["include"] == [ + "src/codex_cli_bin/codex-package.json", + "src/codex_cli_bin/bin/**", + "src/codex_cli_bin/codex-resources/**", + "src/codex_cli_bin/codex-path/**", + ] def test_stage_sdk_release_injects_exact_runtime_pin(tmp_path: Path) -> None: @@ -492,8 +528,7 @@ def test_stage_sdk_release_replaces_existing_staging_dir(tmp_path: Path) -> None def test_staged_sdk_and_runtime_versions_match(tmp_path: Path) -> None: script = _load_update_script_module() - fake_binary = tmp_path / script.runtime_binary_name() - fake_binary.write_text("fake codex\n") + package_archive = _write_fake_codex_package_archive(tmp_path, script) sdk_stage = script.stage_python_sdk_package( tmp_path / "sdk-stage", @@ -502,7 +537,7 @@ def test_staged_sdk_and_runtime_versions_match(tmp_path: Path) -> None: runtime_stage = script.stage_python_runtime_package( tmp_path / "runtime-stage", "rust-v0.116.0-alpha.1", - fake_binary, + package_archive, ) sdk_pyproject = tomllib.loads((sdk_stage / "pyproject.toml").read_text()) @@ -537,9 +572,8 @@ def test_stage_sdk_runs_type_generation_before_staging(tmp_path: Path) -> None: def fake_stage_runtime_package( _staging_dir: Path, _runtime_version: str, - _runtime_binary: Path, + _package_dir: Path, _platform_tag: str | None, - _resource_binaries: Sequence[Path], ) -> Path: raise AssertionError("runtime staging should not run for stage-sdk") @@ -577,28 +611,19 @@ def test_stage_sdk_rejects_mismatched_legacy_versions(tmp_path: Path) -> None: script.run_command(args, script.default_cli_ops()) -def test_stage_runtime_stages_binary_without_type_generation(tmp_path: Path) -> None: +def test_stage_runtime_stages_package_without_type_generation(tmp_path: Path) -> None: script = _load_update_script_module() - fake_binary = tmp_path / script.runtime_binary_name() - helper = tmp_path / "helper" - fallback = tmp_path / "fallback-helper" - fake_binary.write_text("fake codex\n") - helper.write_text("fake helper\n") - fallback.write_text("fake fallback\n") + package_archive = _write_fake_codex_package_archive(tmp_path, script) calls: list[str] = [] args = script.parse_args( [ "stage-runtime", str(tmp_path / "runtime-stage"), - str(fake_binary), + str(package_archive), "--codex-version", "rust-v0.116.0-alpha.1", "--platform-tag", "musllinux_1_1_x86_64", - "--resource-binary", - str(helper), - "--resource-binary", - str(fallback), ] ) @@ -611,14 +636,10 @@ def test_stage_runtime_stages_binary_without_type_generation(tmp_path: Path) -> def fake_stage_runtime_package( _staging_dir: Path, codex_version: str, - _runtime_binary: Path, + package_archive: Path, platform_tag: str | None, - resource_binaries: Sequence[Path], ) -> Path: - calls.append( - f"stage_runtime:{codex_version}:{platform_tag}:" - f"{','.join(path.name for path in resource_binaries)}" - ) + calls.append(f"stage_runtime:{codex_version}:{platform_tag}:{package_archive.name}") return tmp_path / "runtime-stage" def fake_current_sdk_version() -> str: @@ -633,7 +654,7 @@ def test_stage_runtime_stages_binary_without_type_generation(tmp_path: Path) -> script.run_command(args, ops) - assert calls == ["stage_runtime:0.116.0a1:musllinux_1_1_x86_64:helper,fallback-helper"] + assert calls == ["stage_runtime:0.116.0a1:musllinux_1_1_x86_64:codex-package.tar.gz"] def test_default_runtime_is_resolved_from_installed_runtime_package( @@ -653,6 +674,35 @@ def test_default_runtime_is_resolved_from_installed_runtime_package( assert client_module.resolve_codex_bin(config, ops) == fake_binary +def test_runtime_path_dir_is_prepended_without_duplicates(tmp_path: Path) -> None: + from openai_codex import client as client_module + + path_dir = tmp_path / "codex-path" + env = {"PATH": os.pathsep.join(["/usr/bin", str(path_dir), "/bin"])} + + client_module._prepend_path_dirs(env, (path_dir,)) + + assert env["PATH"] == os.pathsep.join([str(path_dir), "/usr/bin", "/bin"]) + + +def test_runtime_path_dir_preserves_windows_path_key( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + from openai_codex import client as client_module + + path_dir = tmp_path / "codex-path" + monkeypatch.setattr(client_module.os, "name", "nt") + env = { + "PATH": "/usr/bin", + "Path": os.pathsep.join(["C\\Windows", str(path_dir)]), + } + + client_module._prepend_path_dirs(env, (path_dir,)) + + assert env == {"Path": os.pathsep.join([str(path_dir), "C\\Windows"])} + + def test_explicit_codex_bin_override_takes_priority(tmp_path: Path) -> None: from openai_codex import client as client_module diff --git a/sdk/typescript/src/exec.ts b/sdk/typescript/src/exec.ts index 3447a31fb4..26741bcc31 100644 --- a/sdk/typescript/src/exec.ts +++ b/sdk/typescript/src/exec.ts @@ -1,4 +1,5 @@ import { spawn } from "node:child_process"; +import { statSync } from "node:fs"; import path from "node:path"; import readline from "node:readline"; import { createRequire } from "node:module"; @@ -54,8 +55,14 @@ const PLATFORM_PACKAGE_BY_TARGET: Record = { const moduleRequire = createRequire(import.meta.url); +type CodexPathResolution = { + executablePath: string; + pathDirs: string[]; +}; + export class CodexExec { private executablePath: string; + private pathDirs: string[]; private envOverride?: Record; private configOverrides?: CodexConfigObject; @@ -64,7 +71,14 @@ export class CodexExec { env?: Record, configOverrides?: CodexConfigObject, ) { - this.executablePath = executablePath || findCodexPath(); + if (executablePath) { + this.executablePath = executablePath; + this.pathDirs = []; + } else { + const resolved = findCodexPath(); + this.executablePath = resolved.executablePath; + this.pathDirs = resolved.pathDirs; + } this.envOverride = env; this.configOverrides = configOverrides; } @@ -160,6 +174,9 @@ export class CodexExec { if (args.apiKey) { env.CODEX_API_KEY = args.apiKey; } + if (this.pathDirs.length > 0) { + prependPathDirs(env, this.pathDirs); + } const child = spawn(this.executablePath, commandArgs, { env, @@ -314,7 +331,7 @@ function isPlainObject(value: unknown): value is CodexConfigObject { return typeof value === "object" && value !== null && !Array.isArray(value); } -function findCodexPath() { +function findCodexPath(): CodexPathResolution { const { platform, arch } = process; let targetTriple = null; @@ -381,9 +398,87 @@ function findCodexPath() { ); } - const archRoot = path.join(vendorRoot, targetTriple); const codexBinaryName = process.platform === "win32" ? "codex.exe" : "codex"; - const binaryPath = path.join(archRoot, "codex", codexBinaryName); + const nativePackage = resolveNativePackage(vendorRoot, targetTriple, codexBinaryName); + if (!nativePackage) { + throw new Error( + `Unable to locate Codex CLI binaries for ${targetTriple}. Ensure ${CODEX_NPM_NAME} is installed with optional dependencies.`, + ); + } - return binaryPath; + return nativePackage; +} + +export function resolveNativePackage( + vendorRoot: string, + targetTriple: string, + codexBinaryName: string, +): CodexPathResolution | null { + const packageRoot = path.join(vendorRoot, targetTriple); + const packageBinaryPath = path.join(packageRoot, "bin", codexBinaryName); + if (isFile(packageBinaryPath) && isFile(path.join(packageRoot, "codex-package.json"))) { + return { + executablePath: packageBinaryPath, + pathDirs: existingDirs(path.join(packageRoot, "codex-path")), + }; + } + + const legacyBinaryPath = path.join(packageRoot, "codex", codexBinaryName); + if (isFile(legacyBinaryPath)) { + return { + executablePath: legacyBinaryPath, + pathDirs: existingDirs(path.join(packageRoot, "path")), + }; + } + + return null; +} + +function existingDirs(...dirs: string[]): string[] { + return dirs.filter(isDirectory); +} + +export function prependPathDirs( + env: Record, + pathDirs: string[], + platform: NodeJS.Platform = process.platform, +): void { + const pathKey = pathEnvKey(env, platform); + if (platform === "win32") { + for (const key of Object.keys(env)) { + if (key.toLowerCase() === "path" && key !== pathKey) { + delete env[key]; + } + } + } + + const existingEntries = (env[pathKey] ?? "") + .split(path.delimiter) + .filter((entry) => entry.length > 0 && !pathDirs.includes(entry)); + env[pathKey] = [...pathDirs, ...existingEntries].join(path.delimiter); +} + +function pathEnvKey(env: Record, platform: NodeJS.Platform): string { + if (platform !== "win32") { + return "PATH"; + } + + const matchingKeys = Object.keys(env).filter((key) => key.toLowerCase() === "path"); + return matchingKeys.includes("Path") ? "Path" : (matchingKeys.at(-1) ?? "PATH"); +} + +function isFile(filePath: string): boolean { + try { + return statSync(filePath).isFile(); + } catch { + return false; + } +} + +function isDirectory(filePath: string): boolean { + try { + return statSync(filePath).isDirectory(); + } catch { + return false; + } } diff --git a/sdk/typescript/tests/exec.test.ts b/sdk/typescript/tests/exec.test.ts index c8aa79646c..eb068f3f1a 100644 --- a/sdk/typescript/tests/exec.test.ts +++ b/sdk/typescript/tests/exec.test.ts @@ -1,5 +1,8 @@ import * as child_process from "node:child_process"; import { EventEmitter } from "node:events"; +import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; import { PassThrough } from "node:stream"; import { describe, expect, it } from "@jest/globals"; @@ -142,4 +145,57 @@ describe("CodexExec", () => { delete process.env.CODEX_ENV_SHOULD_NOT_LEAK; } }); + + it("resolves the package-layout binary and PATH directory", async () => { + const { resolveNativePackage } = await import("../src/exec"); + const vendorRoot = mkdtempSync(path.join(tmpdir(), "codex-sdk-vendor-")); + const packageRoot = path.join(vendorRoot, "x86_64-unknown-linux-musl"); + const binDir = path.join(packageRoot, "bin"); + const pathDir = path.join(packageRoot, "codex-path"); + mkdirSync(binDir, { recursive: true }); + mkdirSync(pathDir, { recursive: true }); + writeFileSync(path.join(packageRoot, "codex-package.json"), "{}"); + writeFileSync(path.join(binDir, "codex"), ""); + + expect(resolveNativePackage(vendorRoot, "x86_64-unknown-linux-musl", "codex")).toEqual({ + executablePath: path.join(binDir, "codex"), + pathDirs: [pathDir], + }); + }); + + it("falls back to the legacy binary layout", async () => { + const { resolveNativePackage } = await import("../src/exec"); + const vendorRoot = mkdtempSync(path.join(tmpdir(), "codex-sdk-vendor-")); + const packageRoot = path.join(vendorRoot, "x86_64-unknown-linux-musl"); + const binDir = path.join(packageRoot, "codex"); + const pathDir = path.join(packageRoot, "path"); + mkdirSync(binDir, { recursive: true }); + mkdirSync(pathDir, { recursive: true }); + writeFileSync(path.join(binDir, "codex"), ""); + + expect(resolveNativePackage(vendorRoot, "x86_64-unknown-linux-musl", "codex")).toEqual({ + executablePath: path.join(binDir, "codex"), + pathDirs: [pathDir], + }); + }); + + it("prepends package PATH entries without duplicating them", async () => { + const { prependPathDirs } = await import("../src/exec"); + const pathDir = path.join(tmpdir(), "codex-path"); + const env = { PATH: `/usr/bin${path.delimiter}${pathDir}` }; + + prependPathDirs(env, [pathDir]); + + expect(env).toEqual({ PATH: `${pathDir}${path.delimiter}/usr/bin` }); + }); + + it("preserves the Windows Path key when prepending package PATH entries", async () => { + const { prependPathDirs } = await import("../src/exec"); + const pathDir = path.join(tmpdir(), "codex-path"); + const env = { PATH: "/usr/bin", Path: `C\\Windows${path.delimiter}${pathDir}` }; + + prependPathDirs(env, [pathDir], "win32"); + + expect(env).toEqual({ Path: `${pathDir}${path.delimiter}C\\Windows` }); + }); });