import { spawn } from 'child_process'; import { createHash } from 'crypto'; import { existsSync, readdirSync, readFileSync, statSync, writeFileSync, } from 'fs'; import { join, resolve } from 'path'; import { cpus } from 'os'; const CHECKSUM_FILE = '.build-checksums.json'; const ALL_PACKAGES = ['core', ...getPackages()]; const BUILD_LEVELS = computeBuildLevels(ALL_PACKAGES); // Flatten levels → stable iteration order for checksum checks const WORKSPACE_PACKAGES = BUILD_LEVELS.flat(); function getPackages() { try { const packagesDir = resolve('packages'); if (!existsSync(packagesDir)) { return []; } const packages = readdirSync(packagesDir).filter((name) => { const packagePath = join(packagesDir, name); return statSync(packagePath).isDirectory() && !name.startsWith('.'); }); return packages.map((name) => `packages/${name}`); } catch (error) { console.error('Error reading packages directory:', error.message); return []; } } /** * Returns the @noco-integrations/* workspace dependency short-names for a package. * e.g. { "@noco-integrations/smtp-auth": "workspace:*" } → ["smtp-auth"] */ function getWorkspaceDependencies(packagePath) { const pkgJsonPath = join(packagePath, 'package.json'); if (!existsSync(pkgJsonPath)) return []; try { const pkg = JSON.parse(readFileSync(pkgJsonPath, 'utf8')); const allDeps = { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) }; return Object.entries(allDeps) .filter(([name, version]) => name.startsWith('@noco-integrations/') && String(version).startsWith('workspace:'), ) .map(([name]) => name.replace('@noco-integrations/', '')); } catch { return []; } } /** * Returns packages grouped into build levels via BFS. * All packages within a level have no unbuilt dependencies — they can be built * in parallel. Level N finishes completely before level N+1 starts. * * Example with 60 packages all depending only on core: * Level 0: [core] * Level 1: [all-60-packages] ← built in parallel */ function computeBuildLevels(packages) { const nameToPath = new Map(); for (const pkg of packages) { const shortName = pkg === 'core' ? 'core' : pkg.replace('packages/', ''); nameToPath.set(shortName, pkg); } const inDegree = new Map(); const dependents = new Map(); for (const pkg of packages) { inDegree.set(pkg, 0); dependents.set(pkg, []); } for (const pkg of packages) { for (const depName of getWorkspaceDependencies(pkg)) { const depPath = nameToPath.get(depName); if (depPath) { inDegree.set(pkg, inDegree.get(pkg) + 1); dependents.get(depPath).push(pkg); } } } const levels = []; const placed = new Set(); let current = packages.filter((pkg) => inDegree.get(pkg) === 0); current.forEach((pkg) => placed.add(pkg)); while (current.length > 0) { levels.push([...current]); const next = []; for (const pkg of current) { for (const dep of dependents.get(pkg) || []) { inDegree.set(dep, inDegree.get(dep) - 1); if (inDegree.get(dep) === 0 && !placed.has(dep)) { next.push(dep); placed.add(dep); } } } current = next; } // Any unplaced packages have a cycle — build them last with a warning const remaining = packages.filter((pkg) => !placed.has(pkg)); if (remaining.length > 0) { console.warn( `⚠️ Cycle detected, building last: ${remaining.join(', ')}`, ); levels.push(remaining); } return levels; } function getAllFiles(dir, files = []) { try { const items = readdirSync(dir); for (const item of items) { const fullPath = join(dir, item); const stat = statSync(fullPath); if (stat.isDirectory()) { if (item !== 'node_modules' && item !== 'dist' && !item.startsWith('.')) { getAllFiles(fullPath, files); } } else { files.push(fullPath); } } } catch { // Skip directories we can't read } return files; } function calculateChecksum(packagePath) { try { if (!existsSync(packagePath)) { return null; } const files = getAllFiles(packagePath); const hash = createHash('sha256'); files.sort(); for (const file of files) { try { const relativePath = file.replace(packagePath, '').replace(/\\/g, '/'); hash.update(relativePath); hash.update(readFileSync(file)); } catch { // Skip files we can't read } } return hash.digest('hex'); } catch (error) { console.error(`Error calculating checksum for ${packagePath}:`, error.message); return null; } } function loadChecksums() { if (!existsSync(CHECKSUM_FILE)) { return {}; } try { return JSON.parse(readFileSync(CHECKSUM_FILE, 'utf8')); } catch (error) { console.warn('Error reading checksum file, starting fresh:', error.message); return {}; } } function saveChecksums(checksums) { try { writeFileSync(CHECKSUM_FILE, JSON.stringify(checksums, null, 2)); } catch (error) { console.error('Error saving checksums:', error.message); } } function packageExists(packagePath) { return ( existsSync(packagePath) && existsSync(join(packagePath, 'package.json')) ); } function hasDistFolder(packagePath) { return existsSync(join(packagePath, 'dist')); } /** * Builds a single package asynchronously. Output is buffered and printed * atomically when the process exits so parallel builds don't interleave. */ function buildPackageAsync(packagePath) { return new Promise((resolve) => { const proc = spawn('pnpm', ['build'], { cwd: packagePath, shell: process.platform === 'win32', }); const chunks = []; proc.stdout?.on('data', (d) => chunks.push({ stream: 'out', d })); proc.stderr?.on('data', (d) => chunks.push({ stream: 'err', d })); proc.on('close', (code) => { const success = code === 0; if (success) { console.log(` ✅ ${packagePath}`); } else { console.error(` ❌ ${packagePath} (exit ${code})`); if (chunks.length) { console.error(` --- output for ${packagePath} ---`); for (const { stream, d } of chunks) { (stream === 'out' ? process.stdout : process.stderr).write(d); } console.error(` --- end ${packagePath} ---`); } } resolve(success); }); proc.on('error', (err) => { console.error(` ❌ ${packagePath}: spawn error — ${err.message}`); resolve(false); }); }); } /** * Builds an array of packages in parallel, capped at maxConcurrent workers. * Returns { successCount, failCount }. */ async function buildLevel(packages, maxConcurrent) { const queue = [...packages]; let successCount = 0; let failCount = 0; async function worker() { while (queue.length > 0) { const pkg = queue.shift(); if (!pkg) break; if (await buildPackageAsync(pkg)) { successCount++; } else { failCount++; } } } await Promise.all( Array.from({ length: Math.min(maxConcurrent, packages.length) }, worker), ); return { successCount, failCount }; } async function main() { const force = process.argv.includes('--force'); const verbose = process.argv.includes('--verbose'); const concurrencyArg = process.argv.find((a) => a.startsWith('--concurrency=')); const maxConcurrent = concurrencyArg ? Math.max(1, parseInt(concurrencyArg.split('=')[1], 10)) : cpus().length; if (force) { console.log('🔄 Force rebuild requested, building all packages...'); } const oldChecksums = loadChecksums(); const newChecksums = {}; const packagesNeedingBuild = []; console.log('🔍 Checking package changes...'); for (const packagePath of WORKSPACE_PACKAGES) { if (!packageExists(packagePath)) { if (verbose) { console.log(`⚠️ Package ${packagePath} does not exist, skipping...`); } continue; } const currentChecksum = calculateChecksum(packagePath); if (!currentChecksum) { console.log(`⚠️ Could not calculate checksum for ${packagePath}, skipping...`); continue; } newChecksums[packagePath] = currentChecksum; const hasChanged = oldChecksums[packagePath] !== currentChecksum; const missingDist = !hasDistFolder(packagePath); if (force || hasChanged || missingDist) { const reason = force ? 'forced' : hasChanged ? 'changed' : 'missing dist'; if (verbose || hasChanged || missingDist) { console.log(`📋 ${packagePath} needs build (${reason})`); } packagesNeedingBuild.push(packagePath); } else if (verbose) { console.log(`✅ ${packagePath} unchanged, skipping build`); } } if (packagesNeedingBuild.length === 0) { console.log('🎉 All packages are up to date, no builds needed!'); return; } const packagesNeedingBuildSet = new Set(packagesNeedingBuild); console.log( `\n🚀 Building ${packagesNeedingBuild.length} package(s) (up to ${maxConcurrent} parallel)...`, ); let totalSuccess = 0; let totalFail = 0; for (let i = 0; i < BUILD_LEVELS.length; i++) { const toBuild = BUILD_LEVELS[i].filter((pkg) => packagesNeedingBuildSet.has(pkg), ); if (toBuild.length === 0) continue; console.log( `\n⚡ Level ${i} — ${toBuild.length} package(s) in parallel`, ); if (verbose) { console.log(` ${toBuild.join(', ')}`); } const { successCount, failCount } = await buildLevel(toBuild, maxConcurrent); totalSuccess += successCount; totalFail += failCount; if (failCount > 0) { console.error( `\n❌ ${failCount} package(s) failed in level ${i} — stopping build`, ); process.exit(1); } } if (totalSuccess > 0) { saveChecksums(newChecksums); console.log(`\n✅ Successfully built ${totalSuccess} package(s)`); } if (totalFail > 0) { console.log(`❌ Failed to build ${totalFail} package(s)`); process.exit(1); } console.log('🎯 Build optimization complete!'); } if (process.argv.includes('--help')) { console.log(` Usage: node scripts/build-optimized.js [options] Options: --force Force rebuild all packages regardless of changes --verbose Show detailed output for all packages --concurrency=N Max parallel builds per level (default: CPU count) --help Show this help message Examples: node scripts/build-optimized.js # Build only changed packages node scripts/build-optimized.js --force # Force rebuild all node scripts/build-optimized.js --concurrency=4 # Limit to 4 parallel node scripts/build-optimized.js --verbose # Show per-package details `); process.exit(0); } main().catch((error) => { console.error('❌ Build script failed:', error.message); process.exit(1); });