diff --git a/.github/workflows/update-nix-hashes.yml b/.github/workflows/update-nix-hashes.yml index f9817fe1ea..6e937da527 100644 --- a/.github/workflows/update-nix-hashes.yml +++ b/.github/workflows/update-nix-hashes.yml @@ -10,129 +10,18 @@ on: - "bun.lock" - "package.json" - "packages/*/package.json" + - "flake.lock" - ".github/workflows/update-nix-hashes.yml" pull_request: paths: - "bun.lock" - "package.json" - "packages/*/package.json" + - "flake.lock" - ".github/workflows/update-nix-hashes.yml" jobs: - compute-node-modules-hash: - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - strategy: - fail-fast: false - matrix: - include: - - system: x86_64-linux - host: blacksmith-4vcpu-ubuntu-2404 - - system: aarch64-linux - host: blacksmith-4vcpu-ubuntu-2404-arm - - system: x86_64-darwin - host: macos-15-intel - - system: aarch64-darwin - host: macos-latest - runs-on: ${{ matrix.host }} - env: - SYSTEM: ${{ matrix.system }} - - steps: - - name: Checkout repository - uses: actions/checkout@v6 - with: - token: ${{ secrets.GITHUB_TOKEN }} - fetch-depth: 0 - ref: ${{ github.head_ref || github.ref_name }} - repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} - - - name: Setup Nix - uses: nixbuild/nix-quick-install-action@v34 - - - name: Compute node_modules hash - run: | - set -euo pipefail - - DUMMY="sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" - HASH_FILE="nix/hashes.json" - OUTPUT_FILE="hash-${SYSTEM}.txt" - - export NIX_KEEP_OUTPUTS=1 - export NIX_KEEP_DERIVATIONS=1 - - BUILD_LOG=$(mktemp) - TMP_JSON=$(mktemp) - trap 'rm -f "$BUILD_LOG" "$TMP_JSON"' EXIT - - if [ ! -f "$HASH_FILE" ]; then - mkdir -p "$(dirname "$HASH_FILE")" - echo '{"nodeModules":{}}' > "$HASH_FILE" - fi - - # Set dummy hash to force nix to rebuild and reveal correct hash - jq --arg system "$SYSTEM" --arg value "$DUMMY" \ - '.nodeModules = (.nodeModules // {}) | .nodeModules[$system] = $value' "$HASH_FILE" > "$TMP_JSON" - mv "$TMP_JSON" "$HASH_FILE" - - MODULES_ATTR=".#packages.${SYSTEM}.default.node_modules" - DRV_PATH="$(nix eval --raw "${MODULES_ATTR}.drvPath")" - - echo "Building node_modules for ${SYSTEM} to discover correct hash..." - echo "Attempting to realize derivation: ${DRV_PATH}" - REALISE_OUT=$(nix-store --realise "$DRV_PATH" --keep-failed 2>&1 | tee "$BUILD_LOG" || true) - - BUILD_PATH=$(echo "$REALISE_OUT" | grep "^/nix/store/" | head -n1 || true) - CORRECT_HASH="" - - if [ -n "$BUILD_PATH" ] && [ -d "$BUILD_PATH" ]; then - echo "Realized node_modules output: $BUILD_PATH" - CORRECT_HASH=$(nix hash path --sri "$BUILD_PATH" 2>/dev/null || true) - fi - - # Try to extract hash from build log - if [ -z "$CORRECT_HASH" ]; then - CORRECT_HASH="$(grep -E 'got:\s+sha256-[A-Za-z0-9+/=]+' "$BUILD_LOG" | awk '{print $2}' | head -n1 || true)" - fi - - if [ -z "$CORRECT_HASH" ]; then - CORRECT_HASH="$(grep -A2 'hash mismatch' "$BUILD_LOG" | grep 'got:' | awk '{print $2}' | sed 's/sha256:/sha256-/' || true)" - fi - - # Try to hash from kept failed build directory - if [ -z "$CORRECT_HASH" ]; then - KEPT_DIR=$(grep -oE "build directory.*'[^']+'" "$BUILD_LOG" | grep -oE "'/[^']+'" | tr -d "'" | head -n1 || true) - if [ -z "$KEPT_DIR" ]; then - KEPT_DIR=$(grep -oE '/nix/var/nix/builds/[^ ]+' "$BUILD_LOG" | head -n1 || true) - fi - - if [ -n "$KEPT_DIR" ] && [ -d "$KEPT_DIR" ]; then - HASH_PATH="$KEPT_DIR" - [ -d "$KEPT_DIR/build" ] && HASH_PATH="$KEPT_DIR/build" - - if [ -d "$HASH_PATH/node_modules" ]; then - CORRECT_HASH=$(nix hash path --sri "$HASH_PATH" 2>/dev/null || true) - fi - fi - fi - - if [ -z "$CORRECT_HASH" ]; then - echo "Failed to determine correct node_modules hash for ${SYSTEM}." - cat "$BUILD_LOG" - exit 1 - fi - - echo "$CORRECT_HASH" > "$OUTPUT_FILE" - echo "Hash for ${SYSTEM}: $CORRECT_HASH" - - - name: Upload hash artifact - uses: actions/upload-artifact@v6 - with: - name: hash-${{ matrix.system }} - path: hash-${{ matrix.system }}.txt - retention-days: 1 - - commit-node-modules-hashes: - needs: compute-node-modules-hash + update-node-modules-hashes: if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository runs-on: blacksmith-4vcpu-ubuntu-2404 env: @@ -147,6 +36,9 @@ jobs: ref: ${{ github.head_ref || github.ref_name }} repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + - name: Setup Nix + uses: nixbuild/nix-quick-install-action@v34 + - name: Configure git run: | git config --global user.email "action@github.com" @@ -159,54 +51,47 @@ jobs: BRANCH="${TARGET_BRANCH:-${GITHUB_REF_NAME}}" git pull --rebase --autostash origin "$BRANCH" - - name: Download all hash artifacts - uses: actions/download-artifact@v7 - with: - pattern: hash-* - merge-multiple: true - - - name: Merge hashes into hashes.json + - name: Compute all node_modules hashes run: | set -euo pipefail HASH_FILE="nix/hashes.json" + SYSTEMS="x86_64-linux aarch64-linux x86_64-darwin aarch64-darwin" if [ ! -f "$HASH_FILE" ]; then mkdir -p "$(dirname "$HASH_FILE")" echo '{"nodeModules":{}}' > "$HASH_FILE" fi - echo "Merging hashes into ${HASH_FILE}..." + for SYSTEM in $SYSTEMS; do + echo "Computing hash for ${SYSTEM}..." + BUILD_LOG=$(mktemp) + trap 'rm -f "$BUILD_LOG"' EXIT - shopt -s nullglob - files=(hash-*.txt) - if [ ${#files[@]} -eq 0 ]; then - echo "No hash files found, nothing to update" - exit 0 - fi + # The updater derivations use fakeHash, so they will fail and reveal the correct hash + UPDATER_ATTR=".#packages.x86_64-linux.${SYSTEM}_node_modules" - EXPECTED_SYSTEMS="x86_64-linux aarch64-linux x86_64-darwin aarch64-darwin" - for sys in $EXPECTED_SYSTEMS; do - if [ ! -f "hash-${sys}.txt" ]; then - echo "WARNING: Missing hash file for $sys" + nix build "$UPDATER_ATTR" --no-link 2>&1 | tee "$BUILD_LOG" || true + + CORRECT_HASH="$(grep -E 'got:\s+sha256-[A-Za-z0-9+/=]+' "$BUILD_LOG" | awk '{print $2}' | head -n1 || true)" + + if [ -z "$CORRECT_HASH" ]; then + CORRECT_HASH="$(grep -A2 'hash mismatch' "$BUILD_LOG" | grep 'got:' | awk '{print $2}' | sed 's/sha256:/sha256-/' || true)" fi - done - for f in "${files[@]}"; do - system="${f#hash-}" - system="${system%.txt}" - hash=$(cat "$f") - if [ -z "$hash" ]; then - echo "WARNING: Empty hash for $system, skipping" - continue + if [ -z "$CORRECT_HASH" ]; then + echo "Failed to determine correct node_modules hash for ${SYSTEM}." + cat "$BUILD_LOG" + exit 1 fi - echo " $system: $hash" - jq --arg sys "$system" --arg h "$hash" \ - '.nodeModules = (.nodeModules // {}) | .nodeModules[$sys] = $h' "$HASH_FILE" > "${HASH_FILE}.tmp" + + echo " ${SYSTEM}: ${CORRECT_HASH}" + jq --arg sys "$SYSTEM" --arg h "$CORRECT_HASH" \ + '.nodeModules[$sys] = $h' "$HASH_FILE" > "${HASH_FILE}.tmp" mv "${HASH_FILE}.tmp" "$HASH_FILE" done - echo "All hashes merged:" + echo "All hashes computed:" cat "$HASH_FILE" - name: Commit ${{ env.TITLE }} changes diff --git a/flake.nix b/flake.nix index 20833fc49e..0f42509374 100644 --- a/flake.nix +++ b/flake.nix @@ -33,17 +33,37 @@ packages = forEachSystem ( pkgs: let - opencode = pkgs.callPackage ./nix/opencode.nix { + node_modules = pkgs.callPackage ./nix/node_modules.nix { inherit rev; }; + opencode = pkgs.callPackage ./nix/opencode.nix { + inherit node_modules; + }; desktop = pkgs.callPackage ./nix/desktop.nix { inherit opencode; }; + # nixpkgs cpu naming to bun cpu naming + cpuMap = { x86_64 = "x64"; aarch64 = "arm64"; }; + # matrix of node_modules builds - these will always fail due to fakeHash usage + # but allow computation of the correct hash from any build machine for any cpu/os + # see the update-nix-hashes workflow for usage + moduleUpdaters = pkgs.lib.listToAttrs ( + pkgs.lib.concatMap (cpu: + map (os: { + name = "${cpu}_${os}_node_modules"; + value = node_modules.override { + bunCpu = cpuMap.${cpu}; + bunOs = os; + hash = pkgs.lib.fakeHash; + }; + }) [ "linux" "darwin" ] + ) [ "x86_64" "aarch64" ] + ); in { default = opencode; inherit opencode desktop; - } + } // moduleUpdaters ); }; } diff --git a/nix/node_modules.nix b/nix/node_modules.nix new file mode 100644 index 0000000000..981a60ef9b --- /dev/null +++ b/nix/node_modules.nix @@ -0,0 +1,85 @@ +{ + lib, + stdenvNoCC, + bun, + bunCpu ? if stdenvNoCC.hostPlatform.isAarch64 then "arm64" else "x64", + bunOs ? if stdenvNoCC.hostPlatform.isLinux then "linux" else "darwin", + rev ? "dirty", + hash ? + (lib.pipe ./hashes.json [ + builtins.readFile + builtins.fromJSON + ]).nodeModules.${stdenvNoCC.hostPlatform.system}, +}: +let + packageJson = lib.pipe ../packages/opencode/package.json [ + builtins.readFile + builtins.fromJSON + ]; +in +stdenvNoCC.mkDerivation { + pname = "opencode-node_modules"; + version = "${packageJson.version}-${rev}"; + + src = lib.fileset.toSource { + root = ../.; + fileset = lib.fileset.intersection (lib.fileset.fromSource (lib.sources.cleanSource ../.)) ( + lib.fileset.unions [ + ../packages + ../bun.lock + ../package.json + ../patches + ../install + ] + ); + }; + + impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [ + "GIT_PROXY_COMMAND" + "SOCKS_SERVER" + ]; + + nativeBuildInputs = [ + bun + ]; + + dontConfigure = true; + + buildPhase = '' + runHook preBuild + export HOME=$(mktemp -d) + export BUN_INSTALL_CACHE_DIR=$(mktemp -d) + bun install \ + --cpu="${bunCpu}" \ + --os="${bunOs}" \ + --frozen-lockfile \ + --ignore-scripts \ + --no-progress \ + --linker=isolated + bun --bun ${./scripts/canonicalize-node-modules.ts} + bun --bun ${./scripts/normalize-bun-binaries.ts} + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + + mkdir -p $out + find . -type d -name node_modules -exec cp -R --parents {} $out \; + + runHook postInstall + ''; + + dontFixup = true; + + outputHashAlgo = "sha256"; + outputHashMode = "recursive"; + outputHash = hash; + + meta.platforms = [ + "aarch64-linux" + "x86_64-linux" + "aarch64-darwin" + "x86_64-darwin" + ]; +} diff --git a/nix/opencode.nix b/nix/opencode.nix index 4d6f8e9b42..23d9fbe34e 100644 --- a/nix/opencode.nix +++ b/nix/opencode.nix @@ -1,6 +1,7 @@ { lib, stdenvNoCC, + callPackage, bun, sysctl, makeBinaryWrapper, @@ -9,81 +10,12 @@ installShellFiles, versionCheckHook, writableTmpDirAsHomeHook, - rev ? "dirty", + node_modules ? callPackage ./node-modules.nix { }, }: -let - packageJson = lib.pipe ../packages/opencode/package.json [ - builtins.readFile - builtins.fromJSON - ]; -in stdenvNoCC.mkDerivation (finalAttrs: { pname = "opencode"; - version = "${packageJson.version}-${rev}"; - - src = lib.fileset.toSource { - root = ../.; - fileset = lib.fileset.intersection (lib.fileset.fromSource (lib.sources.cleanSource ../.)) ( - lib.fileset.unions [ - ../packages - ../bun.lock - ../package.json - ../patches - ../install - ] - ); - }; - - node_modules = stdenvNoCC.mkDerivation { - pname = "${finalAttrs.pname}-node_modules"; - inherit (finalAttrs) version src; - - impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [ - "GIT_PROXY_COMMAND" - "SOCKS_SERVER" - ]; - - nativeBuildInputs = [ - bun - ]; - - dontConfigure = true; - - buildPhase = '' - runHook preBuild - export HOME=$(mktemp -d) - export BUN_INSTALL_CACHE_DIR=$(mktemp -d) - bun install \ - --cpu="${if stdenvNoCC.hostPlatform.isAarch64 then "arm64" else "x64"}" \ - --os="${if stdenvNoCC.hostPlatform.isLinux then "linux" else "darwin"}" \ - --frozen-lockfile \ - --ignore-scripts \ - --no-progress \ - --linker=isolated - bun --bun ${./scripts/canonicalize-node-modules.ts} - bun --bun ${./scripts/normalize-bun-binaries.ts} - runHook postBuild - ''; - - installPhase = '' - runHook preInstall - - mkdir -p $out - find . -type d -name node_modules -exec cp -R --parents {} $out \; - - runHook postInstall - ''; - - dontFixup = true; - - outputHashAlgo = "sha256"; - outputHashMode = "recursive"; - outputHash = - (lib.pipe ./hashes.json [ - builtins.readFile - builtins.fromJSON - ]).nodeModules.${stdenvNoCC.hostPlatform.system}; - }; + inherit (node_modules) version src; + inherit node_modules; nativeBuildInputs = [ bun @@ -159,11 +91,6 @@ stdenvNoCC.mkDerivation (finalAttrs: { homepage = "https://opencode.ai/"; license = lib.licenses.mit; mainProgram = "opencode"; - platforms = [ - "aarch64-linux" - "x86_64-linux" - "aarch64-darwin" - "x86_64-darwin" - ]; + inherit (node_modules.meta) platforms; }; })