From 23cf2b948dc626c234f81d6fee7d9978b4c43631 Mon Sep 17 00:00:00 2001 From: Christian Klotz Date: Tue, 6 Oct 2026 13:30:02 +0200 Subject: [PATCH] feat: unify package artifact validation (#10197) * feat: unify package artifact validation * fix: restrict local artifact output paths * fix: clarify package artifact module names * feat: support pnpm package artifact consumers * fix: preserve local release package installs Keep npm and Bun consumer installs as runnable release artifacts while retaining isolated validation for other packages. Simplify smoke fixtures and avoid redundant release work. --- .github/workflows/build-binaries.yml | 3 + README.md | 44 +++- package-lock.json | 23 +- package.json | 7 +- packages/evals/docker/install-runtime.mjs | 22 +- scripts/check-package-install.mjs | 22 ++ scripts/coding-agent-consumer.mjs | 138 ------------ scripts/coding-agent-consumer.test.mjs | 95 --------- scripts/coding-agent-smoke.mjs | 77 +++++++ scripts/coding-agent-smoke.test.mjs | 103 +++++++++ scripts/local-package-install.mjs | 205 ++++++++++++++++++ scripts/local-package-install.test.mjs | 210 ++++++++++++++++++ scripts/local-release.mjs | 248 +++++++--------------- scripts/npm-command.mjs | 10 + scripts/pack-packages.mjs | 48 +++++ scripts/package-artifact-cli.test.mjs | 31 +++ scripts/package-artifacts.mjs | 141 ++++++++++++ scripts/package-artifacts.test.mjs | 76 +++++++ scripts/publish.mjs | 136 ++++-------- scripts/release-packages.mjs | 4 +- scripts/use-local-packages.mjs | 41 ++++ 21 files changed, 1172 insertions(+), 512 deletions(-) create mode 100644 scripts/check-package-install.mjs delete mode 100644 scripts/coding-agent-consumer.mjs delete mode 100644 scripts/coding-agent-consumer.test.mjs create mode 100644 scripts/coding-agent-smoke.mjs create mode 100644 scripts/coding-agent-smoke.test.mjs create mode 100644 scripts/local-package-install.mjs create mode 100644 scripts/local-package-install.test.mjs create mode 100644 scripts/npm-command.mjs create mode 100644 scripts/pack-packages.mjs create mode 100644 scripts/package-artifact-cli.test.mjs create mode 100644 scripts/package-artifacts.mjs create mode 100644 scripts/package-artifacts.test.mjs create mode 100644 scripts/use-local-packages.mjs diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index 5bafe00ef..74f6847cf 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -380,6 +380,9 @@ jobs: npm install -g npm@11.16.0 --ignore-scripts npm --version + - name: Validate npm tarball publication with provenance + run: node scripts/publish.mjs --dry-run + - name: Publish npm packages run: node scripts/publish.mjs diff --git a/README.md b/README.md index 799d9fea5..9c88e5bf2 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,48 @@ npm run check # Lint, format, and type check ./pi-test.sh # Run pi from sources (can be run from any directory) ``` +### Using local packages outside the monorepo + +Build every public package into one coherent local artifact set: + +```bash +npm run pack:packages -- --out .artifacts/pi-packages +``` + +This refreshes model data before building `pi-ai`. To avoid network access when +model data is already hydrated, pass `--offline-model-data`. + +Then configure an external project to consume one package and resolve all of +its Pi dependencies from the same artifact set. npm is the default: + +```bash +node scripts/use-local-packages.mjs \ + --manifest .artifacts/pi-packages/manifest.json \ + --consumer ../my-project \ + --package @earendil-works/pi-durable \ + --package @earendil-works/pi-agent-core +cd ../my-project +npm install --ignore-scripts +``` + +For a pnpm project, point `--consumer` at the workspace root: + +```bash +node scripts/use-local-packages.mjs \ + --manifest .artifacts/pi-packages/manifest.json \ + --consumer ../my-project \ + --package @earendil-works/pi-agent-core \ + --package-manager pnpm +cd ../my-project +pnpm install --ignore-scripts +``` + +Repeat `--package` for each direct dependency. The command updates the +consumer's `package.json` with content-addressed local `file:` references. It +writes transitive overrides to `package.json` for npm or `pnpm-workspace.yaml` +for pnpm. Keep the artifact directory available while installing or updating +the consumer. Re-run both commands after changing Pi source. + ## Building standalone binaries from release source GitHub releases include a versioned source archive covered by the release's `SHA256SUMS` file. Extract it and run the same build script used for the official standalone binaries: @@ -132,7 +174,7 @@ We treat npm dependency changes as reviewed code changes. - `package-lock.json` is the dependency ground truth. Pre-commit blocks accidental lockfile commits unless `PI_ALLOW_LOCKFILE_CHANGE=1` is set. - `npm run check` verifies pinned direct deps, native TypeScript import compatibility, and the generated coding-agent install lock. - The pi.dev installer installs from `packages/coding-agent/install-lock/`, generated from the root lockfile, to pin transitive deps. The npm package does not pin transitive deps. -- Release smoke tests use `npm run release:local` to build, pack, and create isolated npm and Bun installs outside the repo before tagging a release. +- Local release smoke tests and npm publication use the same tarball packer; npm publishes the validated tarballs rather than repacking workspace directories. - Local release installs, documented npm installs, and `pi update --self` use `--ignore-scripts` where supported. - CI installs with `npm ci --ignore-scripts`, and a scheduled GitHub workflow runs `npm audit --omit=dev` plus `npm audit signatures --omit=dev`. - Install lock generation has an explicit allowlist for dependency lifecycle scripts; new lifecycle-script deps fail checks until reviewed. diff --git a/package-lock.json b/package-lock.json index c12187bf6..d0a3e933c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,8 +21,10 @@ "@types/node": "22.19.19", "esbuild": "0.28.2", "husky": "9.1.7", + "pnpm": "11.15.1", "shx": "0.4.0", - "typescript": "7.0.2" + "typescript": "7.0.2", + "yaml": "2.9.0" }, "engines": { "node": ">=22.19.0" @@ -4168,6 +4170,25 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/pnpm": { + "version": "11.15.1", + "resolved": "https://registry.npmjs.org/pnpm/-/pnpm-11.15.1.tgz", + "integrity": "sha512-gTULB+U8lTigLx8jA7QpD6LXvgTlbiqXDEzEtBfcdh3hlu2r1J1Vx9yVgNuBAHxEFD5OPX5GKzAA0jwlUSLQZQ==", + "dev": true, + "license": "MIT", + "bin": { + "pn": "bin/pnpm.mjs", + "pnpm": "bin/pnpm.mjs", + "pnpx": "bin/pnpx.mjs", + "pnx": "bin/pnpx.mjs" + }, + "engines": { + "node": ">=22.13" + }, + "funding": { + "url": "https://opencollective.com/pnpm" + } + }, "node_modules/postcss": { "version": "8.5.24", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.24.tgz", diff --git a/package.json b/package.json index fed2ac44e..dc82614d4 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "check:browser-smoke": "node scripts/check-browser-smoke.mjs", "check:pinned-deps": "node scripts/check-pinned-deps.mjs", "check:runtime-deps": "node scripts/check-runtime-deps.mjs", - "check:package-install": "node scripts/coding-agent-consumer.mjs", + "check:package-install": "node scripts/check-package-install.mjs", "check:install-lock:coding-agent": "node scripts/generate-coding-agent-install-lock.mjs --check", "check:ts-imports": "node scripts/check-ts-relative-imports.mjs", "check:entry-graphs": "node scripts/check-entry-graphs.mjs", @@ -46,6 +46,7 @@ "publish": "npm run prepublishOnly && node scripts/publish.mjs", "publish:dry": "npm run prepublishOnly && node scripts/publish.mjs --dry-run", "release:local": "node scripts/local-release.mjs", + "pack:packages": "node scripts/pack-packages.mjs", "install-lock:coding-agent": "node scripts/generate-coding-agent-install-lock.mjs", "release:patch": "node scripts/release.mjs patch", "release:minor": "node scripts/release.mjs minor", @@ -59,8 +60,10 @@ "@types/node": "22.19.19", "esbuild": "0.28.2", "husky": "9.1.7", + "pnpm": "11.15.1", "shx": "0.4.0", - "typescript": "7.0.2" + "typescript": "7.0.2", + "yaml": "2.9.0" }, "engines": { "node": ">=22.19.0" diff --git a/packages/evals/docker/install-runtime.mjs b/packages/evals/docker/install-runtime.mjs index 8afc0de43..93f7a3720 100644 --- a/packages/evals/docker/install-runtime.mjs +++ b/packages/evals/docker/install-runtime.mjs @@ -2,8 +2,9 @@ import { execFileSync } from "node:child_process"; import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, rmSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { installCodingAgentConsumer, packReleasePackages } from "../../../scripts/coding-agent-consumer.mjs"; -import { getPublicWorkspacePackages } from "../../../scripts/release-packages.mjs"; +import { installConsumer } from "../../../scripts/local-package-install.mjs"; +import { produceArtifactSet } from "../../../scripts/package-artifacts.mjs"; +import { codingAgentName } from "../../../scripts/coding-agent-smoke.mjs"; const repositoryRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../../.."); const outputDirectory = process.argv[2]; @@ -12,10 +13,17 @@ if (!outputDirectory || process.argv.length !== 3) { } const evalPackage = JSON.parse(readFileSync(join(repositoryRoot, "packages/evals/package.json"), "utf8")); -const tarballs = packReleasePackages(getPublicWorkspacePackages(), join(outputDirectory, "tarballs")); -const evaluatorDependencies = Object.entries(evalPackage.devDependencies).filter(([name]) => !tarballs.has(name)); +const artifactSet = produceArtifactSet({ + build: false, + outDir: join(outputDirectory, "artifacts"), + repoRoot: repositoryRoot, + // Docker build contexts may omit Git metadata; this artifact set is consumed only within this image build. + source: null, +}); +const artifactNames = new Set(artifactSet.packages.map((pkg) => pkg.name)); +const evaluatorDependencies = Object.entries(evalPackage.devDependencies).filter(([name]) => !artifactNames.has(name)); const installDirectory = join(outputDirectory, "install"); -installCodingAgentConsumer(installDirectory, tarballs); +installConsumer({ artifactSet, directory: installDirectory, packageNames: [codingAgentName] }); execFileSync( "npm", [ @@ -30,8 +38,8 @@ execFileSync( { cwd: installDirectory, stdio: "inherit" }, ); -for (const packageName of tarballs.keys()) { - if (packageName === "@earendil-works/pi-coding-agent") continue; +for (const packageName of artifactNames) { + if (packageName === codingAgentName) continue; const packageDirectory = join(installDirectory, "node_modules", ...packageName.split("/")); if (!existsSync(packageDirectory)) continue; for (const entry of readdirSync(packageDirectory, { withFileTypes: true })) { diff --git a/scripts/check-package-install.mjs b/scripts/check-package-install.mjs new file mode 100644 index 000000000..e900d9780 --- /dev/null +++ b/scripts/check-package-install.mjs @@ -0,0 +1,22 @@ +#!/usr/bin/env node + +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { installConsumer, packageConsumerDirectoryName, smokeTestNpmConsumer } from "./local-package-install.mjs"; +import { produceArtifactSet } from "./package-artifacts.mjs"; +import { codingAgentName, smokeTestCodingAgent } from "./coding-agent-smoke.mjs"; + +const root = mkdtempSync(join(tmpdir(), "pi-package-install-")); +try { + const artifactSet = produceArtifactSet({ build: false, outDir: join(root, "artifacts"), repoRoot: process.cwd() }); + for (const pkg of artifactSet.packages) { + const directory = join(root, "consumers", packageConsumerDirectoryName(pkg.name)); + installConsumer({ artifactSet, directory, packageNames: [pkg.name] }); + smokeTestNpmConsumer({ artifactSet, directory, packageName: pkg.name }); + if (pkg.name === codingAgentName) smokeTestCodingAgent(directory); + } + console.log(`Verified ${artifactSet.packages.length} isolated package consumers.`); +} finally { + rmSync(root, { force: true, recursive: true }); +} diff --git a/scripts/coding-agent-consumer.mjs b/scripts/coding-agent-consumer.mjs deleted file mode 100644 index 82c37960c..000000000 --- a/scripts/coding-agent-consumer.mjs +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/env node - -import { spawnSync } from "node:child_process"; -import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join, relative, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; -import { getPublicWorkspacePackages } from "./release-packages.mjs"; - -const codingAgentName = "@earendil-works/pi-coding-agent"; -const developmentPackages = new Set(["pi-client", "pi-protocol", "pi-server"].map((name) => `@earendil-works/${name}`)); - -function run(command, args, options = {}) { - console.log(`$ ${[command, ...args].join(" ")}`); - const result = spawnSync(command, args, { - encoding: "utf8", - shell: process.platform === "win32", - timeout: 300_000, - ...options, - }); - if (result.status !== 0) { - throw new Error(`Command failed: ${command} ${args.join(" ")}\n${result.stdout ?? ""}${result.stderr ?? ""}${result.error?.message ?? ""}`); - } - return result.stdout; -} - -export function packReleasePackages(packages, tarballDirectory) { - mkdirSync(tarballDirectory, { recursive: true }); - const tarballs = new Map(); - for (const pkg of packages) { - const manifest = JSON.parse(readFileSync(join(pkg.directory, "package.json"), "utf8")); - if (manifest.name !== pkg.name) throw new Error(`Unexpected package name in ${pkg.directory}`); - const output = run("npm", ["pack", "--ignore-scripts", "--json", "--pack-destination", tarballDirectory], { cwd: pkg.directory }); - // npm <11.6 returns an array; newer npm can return an object keyed by package name. - const parsed = JSON.parse(output); - const packed = Array.isArray(parsed) ? parsed[0] : Object.values(parsed)[0]; - tarballs.set(pkg.name, join(tarballDirectory, packed.filename)); - } - return tarballs; -} - -export function installCodingAgentConsumer(directory, tarballs, packageManager = "npm") { - mkdirSync(directory, { recursive: true }); - const overrides = Object.fromEntries([...tarballs].map(([name, path]) => [ - name, `file:./${relative(directory, path).replaceAll("\\", "/")}`, - ])); - if (!overrides[codingAgentName]) throw new Error("Missing coding-agent tarball"); - // Only coding-agent is a direct dependency. Overrides select local artifacts - // for declared transitive dependencies without installing undeclared packages. - const manifest = { - private: true, - dependencies: { [codingAgentName]: overrides[codingAgentName] }, - overrides, - }; - writeFileSync(join(directory, "package.json"), `${JSON.stringify(manifest, null, "\t")}\n`); - const installArgs = packageManager === "bun" ? ["--production"] : ["--omit=dev", "--no-audit", "--no-fund"]; - run(packageManager, ["install", "--ignore-scripts", ...installArgs], { cwd: directory }); -} - -function checkInstalledPackages(nodeModules, seen = new Set()) { - if (!existsSync(nodeModules)) return; - const directories = readdirSync(nodeModules) - .filter((name) => !name.startsWith(".")) - .flatMap((name) => name.startsWith("@") - ? readdirSync(join(nodeModules, name)).map((child) => join(nodeModules, name, child)) - : [join(nodeModules, name)]); - for (const directory of directories) { - if (!existsSync(join(directory, "package.json"))) continue; - const path = realpathSync(directory); - if (seen.has(path)) continue; - seen.add(path); - const manifest = JSON.parse(readFileSync(join(path, "package.json"), "utf8")); - if (developmentPackages.has(manifest.name)) throw new Error(`${manifest.name} must not be installed: ${path}`); - checkInstalledPackages(join(path, "node_modules"), seen); - } -} - -export function smokeTestCodingAgentConsumer(directory, runtime = process.execPath) { - checkInstalledPackages(join(directory, "node_modules")); - const packageDir = join(directory, "node_modules", codingAgentName); - const manifest = JSON.parse(readFileSync(join(packageDir, "package.json"), "utf8")); - for (const path of ["dist/client", "dist/experimental", "dist/cli/experimental", "dist/bundle/client.js", "dist/bundle/coordinator.js"]) { - if (existsSync(join(packageDir, path))) throw new Error(`Published package contains development-only code: ${path}`); - } - const home = mkdtempSync(join(directory, "smoke-home-")); - const entry = join(directory, "smoke-sdk.mjs"); - const env = { - PATH: process.env.PATH, - HOME: home, - USERPROFILE: home, - APPDATA: home, - LOCALAPPDATA: home, - XDG_CONFIG_HOME: home, - XDG_CACHE_HOME: home, - PI_CODING_AGENT_DIR: join(home, ".pi", "agent"), - PI_OFFLINE: "1", - PI_TELEMETRY: "0", - }; - for (const name of ["SystemRoot", "SYSTEMROOT", "WINDIR", "COMSPEC", "PATHEXT"]) { - if (process.env[name]) env[name] = process.env[name]; - } - try { - writeFileSync(entry, `import assert from "node:assert/strict"; -import { createAgentSession, SessionManager, ModelRuntime } from "${codingAgentName}"; -assert.equal(typeof createAgentSession, "function"); -assert.equal(typeof SessionManager.inMemory, "function"); -assert.equal(typeof ModelRuntime.create, "function"); -for (const name of ["pi-client", "pi-protocol", "pi-server"]) { - assert.throws(() => import.meta.resolve("@earendil-works/" + name), /Cannot find|cannot find/, name + " must not be installed"); -} -for (const subpath of ["/client", "/experimental/plugin"]) { - assert.throws(() => import.meta.resolve("${codingAgentName}" + subpath), /not exported|not defined|Cannot find|cannot find/); -} -`); - run(runtime, [entry], { cwd: directory, env, timeout: 30_000 }); - for (const cli of new Set([manifest.bin.pi, "dist/cli.js"])) { - const output = run(runtime, [join(packageDir, cli), "--version"], { cwd: directory, env, timeout: 30_000 }); - if (output.trim() !== manifest.version) throw new Error(`Unexpected version from ${cli}: ${output}`); - } - } finally { - rmSync(entry, { force: true }); - rmSync(home, { recursive: true, force: true }); - } - console.log(`Coding-agent SDK and CLI consumer smoke tests passed (${runtime}).`); -} - -if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { - if (process.argv.length !== 2) throw new Error("Usage: node scripts/coding-agent-consumer.mjs"); - const root = mkdtempSync(join(tmpdir(), "pi-package-consumer-")); - try { - const tarballs = packReleasePackages(getPublicWorkspacePackages(), join(root, "tarballs")); - const directory = join(root, "consumer"); - installCodingAgentConsumer(directory, tarballs); - smokeTestCodingAgentConsumer(directory); - } finally { - rmSync(root, { recursive: true, force: true }); - } -} diff --git a/scripts/coding-agent-consumer.test.mjs b/scripts/coding-agent-consumer.test.mjs deleted file mode 100644 index 3d125cce8..000000000 --- a/scripts/coding-agent-consumer.test.mjs +++ /dev/null @@ -1,95 +0,0 @@ -import assert from "node:assert/strict"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { dirname, join } from "node:path"; -import test from "node:test"; -import { installCodingAgentConsumer, packReleasePackages, smokeTestCodingAgentConsumer } from "./coding-agent-consumer.mjs"; - -const codingAgentName = "@earendil-works/pi-coding-agent"; -const devPackages = ["pi-client", "pi-protocol", "pi-server"].map((name) => `@earendil-works/${name}`); - -function createFixture(t, { importServer = false, declareServer = false } = {}) { - const root = mkdtempSync(join(tmpdir(), "pi-consumer-test-")); - t.after(() => rmSync(root, { recursive: true, force: true })); - const packages = [codingAgentName, "@earendil-works/chord", ...devPackages].map((name) => ({ - name, - directory: join(root, "packages", name.split("/")[1]), - })); - for (const pkg of packages) { - const isAgent = pkg.name === codingAgentName; - const manifest = { - name: pkg.name, - version: "1.0.0", - type: "module", - exports: isAgent ? { - ".": "./dist/index.js", - "./client": { source: "./src/client/index.ts" }, - "./experimental/plugin": { source: "./src/experimental/plugin.ts" }, - } : "./dist/index.js", - ...(isAgent ? { - bin: { pi: "dist/bundle/cli.js" }, - dependencies: { - "@earendil-works/chord": "1.0.0", - ...(declareServer ? { "@earendil-works/pi-server": "1.0.0" } : {}), - }, - devDependencies: Object.fromEntries(devPackages.map((name) => [name, "1.0.0"])), - } : {}), - }; - const files = { - "package.json": JSON.stringify(manifest), - "dist/index.js": isAgent ? ` -${importServer ? 'import "@earendil-works/pi-server";' : ""} -import { marker } from "@earendil-works/chord"; -if (marker !== "local tarball") throw new Error("Wrong Chord artifact"); -export function createAgentSession() {} -export class SessionManager { static inMemory() {} } -export class ModelRuntime { static create() {} } -` : 'export const marker = "local tarball";', - ...(isAgent ? { - "dist/cli.js": 'console.log("1.0.0");', - "dist/bundle/cli.js": 'console.log("1.0.0");', - } : {}), - }; - for (const [path, content] of Object.entries(files)) { - mkdirSync(dirname(join(pkg.directory, path)), { recursive: true }); - writeFileSync(join(pkg.directory, path), content); - } - } - const tarballs = packReleasePackages(packages, join(root, "tarballs")); - const directory = join(root, "consumer"); - installCodingAgentConsumer(directory, tarballs); - return directory; -} - -// #9132: installing every tarball directly hid undeclared runtime imports. -test("installs only coding-agent directly and uses overrides only for declared runtime dependencies", (t) => { - const directory = createFixture(t); - const manifest = JSON.parse(readFileSync(join(directory, "package.json"), "utf8")); - assert.deepEqual(Object.keys(manifest.dependencies), [codingAgentName]); - for (const name of devPackages) { - assert.ok(manifest.overrides[name]); - assert.equal(existsSync(join(directory, "node_modules", name)), false); - } - smokeTestCodingAgentConsumer(directory); - - const nested = join(directory, "node_modules", codingAgentName, "node_modules/@earendil-works/pi-server"); - mkdirSync(nested, { recursive: true }); - writeFileSync(join(nested, "package.json"), JSON.stringify({ name: "@earendil-works/pi-server", version: "1.0.0" })); - assert.throws(() => smokeTestCodingAgentConsumer(directory), /pi-server must not be installed/); - rmSync(nested, { recursive: true }); - - const experimental = join(directory, "node_modules", codingAgentName, "dist/experimental"); - mkdirSync(experimental); - assert.throws(() => smokeTestCodingAgentConsumer(directory), /contains development-only code/); -}); - -// #9132: smoke-test the public SDK, not just a bundled CLI that hides missing imports. -test("fails when the SDK imports an undeclared server despite a working CLI", (t) => { - const directory = createFixture(t, { importServer: true }); - assert.throws(() => smokeTestCodingAgentConsumer(directory), /Cannot find package '@earendil-works\/pi-server'/); -}); - -test("fails if a development-only dependency is added back to the published dependency tree", (t) => { - const directory = createFixture(t, { declareServer: true }); - assert.throws(() => smokeTestCodingAgentConsumer(directory), /pi-server must not be installed/); -}); diff --git a/scripts/coding-agent-smoke.mjs b/scripts/coding-agent-smoke.mjs new file mode 100644 index 000000000..cef0fa1bd --- /dev/null +++ b/scripts/coding-agent-smoke.mjs @@ -0,0 +1,77 @@ +import { execFileSync } from "node:child_process"; +import { existsSync, mkdtempSync, readdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +export const codingAgentName = "@earendil-works/pi-coding-agent"; +const developmentPackages = new Set(["pi-client", "pi-protocol", "pi-server"].map((name) => `@earendil-works/${name}`)); + +function checkInstalledPackages(nodeModules, seen = new Set()) { + if (!existsSync(nodeModules)) return; + const directories = readdirSync(nodeModules) + .filter((name) => !name.startsWith(".")) + .flatMap((name) => name.startsWith("@") + ? readdirSync(join(nodeModules, name)).map((child) => join(nodeModules, name, child)) + : [join(nodeModules, name)]); + for (const directory of directories) { + if (!existsSync(join(directory, "package.json"))) continue; + const path = realpathSync(directory); + if (seen.has(path)) continue; + seen.add(path); + const manifest = JSON.parse(readFileSync(join(path, "package.json"), "utf8")); + if (developmentPackages.has(manifest.name)) throw new Error(`${manifest.name} must not be installed: ${path}`); + checkInstalledPackages(join(path, "node_modules"), seen); + } +} + +export function smokeTestCodingAgent(directory, runtime = process.execPath) { + checkInstalledPackages(join(directory, "node_modules")); + const packageDir = join(directory, "node_modules", codingAgentName); + const manifest = JSON.parse(readFileSync(join(packageDir, "package.json"), "utf8")); + for (const path of ["dist/client", "dist/experimental", "dist/cli/experimental", "dist/bundle/client.js", "dist/bundle/coordinator.js"]) { + if (existsSync(join(packageDir, path))) throw new Error(`Published package contains development-only code: ${path}`); + } + const home = mkdtempSync(join(tmpdir(), "pi-coding-agent-smoke-home-")); + const entry = join(directory, "coding-agent-smoke.mjs"); + const env = { + PATH: process.env.PATH, + HOME: home, + USERPROFILE: home, + APPDATA: home, + LOCALAPPDATA: home, + XDG_CONFIG_HOME: home, + XDG_CACHE_HOME: home, + PI_CODING_AGENT_DIR: join(home, ".pi", "agent"), + PI_OFFLINE: "1", + PI_TELEMETRY: "0", + }; + for (const name of ["SystemRoot", "SYSTEMROOT", "WINDIR", "COMSPEC", "PATHEXT"]) { + if (process.env[name]) env[name] = process.env[name]; + } + try { + writeFileSync(entry, `import assert from "node:assert/strict"; +import { createAgentSession, SessionManager, ModelRuntime } from "${codingAgentName}"; +assert.equal(typeof createAgentSession, "function"); +assert.equal(typeof SessionManager.inMemory, "function"); +assert.equal(typeof ModelRuntime.create, "function"); +for (const subpath of ["/client", "/experimental/plugin"]) { + assert.throws(() => import.meta.resolve("${codingAgentName}" + subpath), /not exported|not defined|Cannot find|cannot find/); +} +`); + execFileSync(runtime, [entry], { cwd: directory, env, stdio: ["inherit", "pipe", "pipe"], timeout: 30_000 }); + for (const cli of new Set([manifest.bin.pi, "dist/cli.js"])) { + const output = execFileSync(runtime, [join(packageDir, cli), "--version"], { + cwd: directory, + encoding: "utf8", + env, + stdio: ["inherit", "pipe", "pipe"], + timeout: 30_000, + }); + if (output.trim() !== manifest.version) throw new Error(`Unexpected version from ${cli}: ${output}`); + } + } finally { + rmSync(entry, { force: true }); + rmSync(home, { recursive: true, force: true }); + } + console.log(`Coding-agent policy smoke tests passed (${runtime}).`); +} diff --git a/scripts/coding-agent-smoke.test.mjs b/scripts/coding-agent-smoke.test.mjs new file mode 100644 index 000000000..e4827921a --- /dev/null +++ b/scripts/coding-agent-smoke.test.mjs @@ -0,0 +1,103 @@ +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import test from "node:test"; +import { installConsumer } from "./local-package-install.mjs"; +import { produceArtifactSet } from "./package-artifacts.mjs"; +import { codingAgentName, smokeTestCodingAgent } from "./coding-agent-smoke.mjs"; + +const devPackages = ["pi-client", "pi-protocol", "pi-server"].map((name) => `@earendil-works/${name}`); + +function writePackage(directory, manifest, files) { + mkdirSync(directory, { recursive: true }); + writeFileSync(join(directory, "package.json"), `${JSON.stringify(manifest, null, "\t")}\n`); + for (const [path, content] of Object.entries(files)) { + mkdirSync(dirname(join(directory, path)), { recursive: true }); + writeFileSync(join(directory, path), content); + } +} + +function createFixture(t, { importServer = false, declareServer = false } = {}) { + const temporaryRoot = mkdtempSync(join(tmpdir(), "pi-coding-agent-smoke-test-")); + t.after(() => rmSync(temporaryRoot, { recursive: true, force: true })); + const root = join(temporaryRoot, "fixture with spaces"); + const repoRoot = join(root, "repo"); + mkdirSync(repoRoot, { recursive: true }); + writeFileSync(join(repoRoot, "package.json"), '{"name":"fixture","private":true}\n'); + const packageNames = [codingAgentName, ...devPackages]; + for (const name of packageNames) { + const isAgent = name === codingAgentName; + writePackage( + join(repoRoot, "packages", name.split("/")[1]), + { + name, + version: "1.0.0", + type: "module", + main: "./dist/index.js", + types: "./dist/index.d.ts", + exports: isAgent + ? { + ".": { types: "./dist/index.d.ts", import: "./dist/index.js" }, + "./client": { source: "./src/client/index.ts" }, + "./experimental/plugin": { source: "./src/experimental/plugin.ts" }, + } + : { ".": { types: "./dist/index.d.ts", import: "./dist/index.js" } }, + files: ["dist"], + ...(isAgent + ? { + bin: { pi: "dist/bundle/cli.js" }, + ...(declareServer ? { dependencies: { "@earendil-works/pi-server": "1.0.0" } } : {}), + devDependencies: Object.fromEntries(devPackages.map((packageName) => [packageName, "1.0.0"])), + } + : {}), + }, + { + "dist/index.js": isAgent + ? `${importServer ? 'import "@earendil-works/pi-server";' : ""} +export function createAgentSession() {} +export class SessionManager { static inMemory() {} } +export class ModelRuntime { static create() {} } +` + : "export {};\n", + "dist/index.d.ts": "export {};\n", + ...(isAgent + ? { + "dist/cli.js": 'console.log("1.0.0");\n', + "dist/bundle/cli.js": 'console.log("1.0.0");\n', + } + : {}), + }, + ); + } + const artifactSet = produceArtifactSet({ build: false, outDir: join(root, "artifacts"), repoRoot, source: null }); + const directory = join(root, "consumer"); + installConsumer({ artifactSet, directory, packageNames: [codingAgentName] }); + return directory; +} + +test("accepts a valid coding-agent package and rejects development-only packages and files", (t) => { + const directory = createFixture(t); + smokeTestCodingAgent(directory); + + const nested = join(directory, "node_modules", codingAgentName, "node_modules/@earendil-works/pi-server"); + mkdirSync(nested, { recursive: true }); + writeFileSync(join(nested, "package.json"), JSON.stringify({ name: "@earendil-works/pi-server", version: "1.0.0" })); + assert.throws(() => smokeTestCodingAgent(directory), /pi-server must not be installed/); + rmSync(nested, { recursive: true }); + + const experimental = join(directory, "node_modules", codingAgentName, "dist/experimental"); + mkdirSync(experimental); + assert.throws(() => smokeTestCodingAgent(directory), /contains development-only code/); +}); + +// #9132: smoke-test the public SDK, not just a bundled CLI that hides missing imports. +test("fails when the SDK imports an undeclared server despite a working CLI", (t) => { + const directory = createFixture(t, { importServer: true }); + assert.throws(() => smokeTestCodingAgent(directory), /Cannot find package '@earendil-works\/pi-server'/); +}); + +test("fails if a development-only dependency is added back to the published dependency tree", (t) => { + const directory = createFixture(t, { declareServer: true }); + assert.throws(() => smokeTestCodingAgent(directory), /pi-server must not be installed/); +}); diff --git a/scripts/local-package-install.mjs b/scripts/local-package-install.mjs new file mode 100644 index 000000000..f1dc3b872 --- /dev/null +++ b/scripts/local-package-install.mjs @@ -0,0 +1,205 @@ +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { isAbsolute, join, relative, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { parseDocument } from "yaml"; +import { execNpmSync } from "./npm-command.mjs"; + +const dependencySections = ["dependencies", "devDependencies", "optionalDependencies"]; + +function createFileSpecifier(fromDirectory, path) { + const relativePath = relative(fromDirectory, path); + return isAbsolute(relativePath) ? pathToFileURL(path).href : `file:./${relativePath.replaceAll("\\", "/")}`; +} + +function detectIndentation(contents) { + return contents.match(/\n([\t ]+)"/)?.[1] ?? "\t"; +} + +function packageDirectory(directory, packageName) { + return join(directory, "node_modules", ...packageName.split("/")); +} + +// Generic consumers run Node ESM, so other export conditions are outside this smoke test's scope. +function runtimeTarget(value) { + if (typeof value === "string") return value; + if (Array.isArray(value)) { + for (const candidate of value) { + const target = runtimeTarget(candidate); + if (target) return target; + } + return undefined; + } + if (!value || typeof value !== "object") return undefined; + for (const condition of ["import", "node", "default"]) { + const target = runtimeTarget(value[condition]); + if (target) return target; + } + return undefined; +} + +function typesTarget(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) return undefined; + return typeof value.types === "string" ? value.types : undefined; +} + +// Generic validation covers literal exports only; wildcard exports need package-specific test inputs. +function literalExports(manifest) { + // Packages without an exports map are still checked through main, types, and the root import. + if (manifest.exports === undefined) return []; + if (typeof manifest.exports === "string" || Array.isArray(manifest.exports)) return [[".", manifest.exports]]; + const keys = Object.keys(manifest.exports); + if (!keys.some((key) => key.startsWith("."))) return [[".", manifest.exports]]; + return Object.entries(manifest.exports).filter(([subpath]) => !subpath.includes("*")); +} + +function assertPackageFile(directory, target, description) { + if (typeof target !== "string") throw new Error(`${description} must be a package path, found ${String(target)}`); + const path = resolve(directory, target); + const relativePath = relative(directory, path); + if (relativePath.startsWith("..") || isAbsolute(relativePath)) { + throw new Error(`${description} must stay inside the package, found ${target}`); + } + if (!existsSync(path)) throw new Error(`${description} does not exist: ${path}`); +} + +function lockfilePackageName(path) { + const marker = "node_modules/"; + const index = path.lastIndexOf(marker); + return index === -1 ? undefined : path.slice(index + marker.length); +} + +function verifyLocalResolutions(directory, artifactSet, directPackageName) { + const lockPath = join(directory, "package-lock.json"); + const lock = JSON.parse(readFileSync(lockPath, "utf8")); + if (!lock.packages || typeof lock.packages !== "object" || Array.isArray(lock.packages)) { + throw new Error(`Invalid npm lockfile: packages object is missing from ${lockPath}`); + } + const artifacts = new Map(artifactSet.packages.map((pkg) => [pkg.name, pkg])); + const resolvedNames = new Set(); + for (const [path, installed] of Object.entries(lock.packages)) { + const name = lockfilePackageName(path); + const artifact = artifacts.get(name); + if (!artifact) continue; + if (typeof installed.resolved !== "string" || !installed.resolved.startsWith("file:")) { + throw new Error(`${name} did not resolve from a local artifact: ${String(installed.resolved)}`); + } + const resolvedPath = new URL(installed.resolved, pathToFileURL(`${directory}/`)); + if (resolvedPath.protocol !== "file:" || resolvedPath.pathname !== pathToFileURL(artifact.tarballPath).pathname) { + throw new Error(`${name} resolved from the wrong local artifact: ${installed.resolved}`); + } + resolvedNames.add(name); + } + if (!resolvedNames.has(directPackageName)) throw new Error(`${directPackageName} is missing from the consumer lockfile`); +} + +function preparePnpmWorkspace(consumerDirectory, specifiers) { + const workspacePath = join(consumerDirectory, "pnpm-workspace.yaml"); + const contents = existsSync(workspacePath) ? readFileSync(workspacePath, "utf8") : ""; + const document = parseDocument(contents); + if (document.errors.length > 0) { + throw new Error(`Invalid pnpm workspace file ${workspacePath}: ${document.errors.map((error) => error.message).join("; ")}`); + } + const workspace = document.toJS(); + if (workspace !== null && (typeof workspace !== "object" || Array.isArray(workspace))) { + throw new Error(`Invalid pnpm workspace file ${workspacePath}: root must be a mapping`); + } + if (workspace?.overrides !== undefined && (typeof workspace.overrides !== "object" || workspace.overrides === null || Array.isArray(workspace.overrides))) { + throw new Error(`Invalid pnpm workspace file ${workspacePath}: overrides must be a mapping`); + } + if (document.contents === null) document.contents = document.createNode({}); + for (const [name, specifier] of Object.entries(specifiers)) document.setIn(["overrides", name], specifier); + return { contents: String(document), path: workspacePath }; +} + +export function wireConsumer({ artifactSet, consumerDirectory, packageNames, packageManager = "npm" }) { + if (packageManager !== "npm" && packageManager !== "pnpm") throw new Error(`Unsupported package manager: ${packageManager}`); + if (packageNames.length === 0) throw new Error("At least one package is required"); + const packageJsonPath = join(consumerDirectory, "package.json"); + if (!existsSync(packageJsonPath)) throw new Error(`Consumer package.json does not exist: ${packageJsonPath}`); + const contents = readFileSync(packageJsonPath, "utf8"); + const manifest = JSON.parse(contents); + const artifacts = new Map(artifactSet.packages.map((pkg) => [pkg.name, pkg])); + for (const packageName of packageNames) artifactSet.getPackage(packageName); + const specifiers = Object.fromEntries( + artifactSet.packages.map((pkg) => [pkg.name, createFileSpecifier(consumerDirectory, pkg.tarballPath)]), + ); + + const directPackages = new Set(packageNames); + for (const section of dependencySections) { + for (const name of Object.keys(manifest[section] ?? {})) { + if (artifacts.has(name)) directPackages.add(name); + } + } + for (const name of directPackages) { + const targetSection = dependencySections.find((section) => Object.hasOwn(manifest[section] ?? {}, name)) ?? "dependencies"; + for (const section of dependencySections) { + if (section !== targetSection && manifest[section]) delete manifest[section][name]; + } + manifest[targetSection] = { ...manifest[targetSection], [name]: specifiers[name] }; + } + if (packageManager === "npm") manifest.overrides = { ...manifest.overrides, ...specifiers }; + const pnpmWorkspace = packageManager === "pnpm" ? preparePnpmWorkspace(consumerDirectory, specifiers) : undefined; + writeFileSync(packageJsonPath, `${JSON.stringify(manifest, null, detectIndentation(contents))}\n`); + if (pnpmWorkspace) writeFileSync(pnpmWorkspace.path, pnpmWorkspace.contents); + return manifest; +} + +export function installConsumer({ artifactSet, directory, packageNames, packageManager = "npm" }) { + if (packageManager !== "npm" && packageManager !== "bun") throw new Error(`Unsupported package manager: ${packageManager}`); + if (packageNames.length === 0) throw new Error("At least one package is required"); + for (const packageName of packageNames) artifactSet.getPackage(packageName); + mkdirSync(directory, { recursive: true }); + writeFileSync(join(directory, "package.json"), '{"private":true,"type":"module"}\n'); + wireConsumer({ artifactSet, consumerDirectory: directory, packageNames }); + const installArgs = packageManager === "bun" ? ["--production"] : ["--omit=dev", "--no-audit", "--no-fund"]; + const args = ["install", "--ignore-scripts", ...installArgs]; + const options = { cwd: directory, stdio: "inherit", timeout: 300_000 }; + if (packageManager === "npm") execNpmSync(args, options); + else execFileSync(packageManager, args, options); +} + +export function smokeTestNpmConsumer({ directory, artifactSet, packageName }) { + const artifact = artifactSet.getPackage(packageName); + const installedDirectory = packageDirectory(directory, packageName); + const manifest = JSON.parse(readFileSync(join(installedDirectory, "package.json"), "utf8")); + if (manifest.version !== artifact.version) { + throw new Error(`${packageName} version mismatch: expected ${artifact.version}, found ${manifest.version}`); + } + if (typeof manifest.main !== "string") throw new Error(`Public Pi package ${packageName} must declare a string main field`); + if (typeof manifest.types !== "string") throw new Error(`Public Pi package ${packageName} must declare a string types field`); + assertPackageFile(installedDirectory, manifest.main, `${packageName} main`); + assertPackageFile(installedDirectory, manifest.types, `${packageName} types`); + + const runtimeSpecifiers = []; + for (const [subpath, value] of literalExports(manifest)) { + const runtime = runtimeTarget(value); + const types = typesTarget(value); + if (runtime) { + assertPackageFile(installedDirectory, runtime, `${packageName} export ${subpath}`); + runtimeSpecifiers.push(subpath === "." ? packageName : `${packageName}${subpath.slice(1)}`); + } + if (types) assertPackageFile(installedDirectory, types, `${packageName} types export ${subpath}`); + } + for (const [name, target] of Object.entries(typeof manifest.bin === "string" ? { [packageName.split("/").at(-1)]: manifest.bin } : (manifest.bin ?? {}))) { + assertPackageFile(installedDirectory, target.startsWith("./") ? target : `./${target}`, `${packageName} bin ${name}`); + } + + if (!existsSync(join(directory, "package-lock.json"))) throw new Error(`npm consumer smoke test requires package-lock.json: ${directory}`); + verifyLocalResolutions(directory, artifactSet, packageName); + const smokePath = join(directory, "artifact-smoke.mjs"); + const resolveChecks = runtimeSpecifiers.map((specifier) => `assert.ok(existsSync(fileURLToPath(import.meta.resolve(${JSON.stringify(specifier)}))));`).join("\n"); + writeFileSync( + smokePath, + `import assert from "node:assert/strict";\nimport { existsSync } from "node:fs";\nimport { fileURLToPath } from "node:url";\nimport ${JSON.stringify(packageName)};\n${resolveChecks}\n`, + ); + try { + execFileSync(process.execPath, [smokePath], { cwd: directory, stdio: "inherit", timeout: 300_000 }); + } finally { + rmSync(smokePath, { force: true }); + } +} + +export function packageConsumerDirectoryName(packageName) { + return packageName.replace(/^@/, "").replaceAll("/", "-"); +} diff --git a/scripts/local-package-install.test.mjs b/scripts/local-package-install.test.mjs new file mode 100644 index 000000000..fe467d0fe --- /dev/null +++ b/scripts/local-package-install.test.mjs @@ -0,0 +1,210 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, dirname, join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { parse } from "yaml"; +import { installConsumer, smokeTestNpmConsumer, wireConsumer } from "./local-package-install.mjs"; +import { produceArtifactSet } from "./package-artifacts.mjs"; + +function writePackage(directory, manifest, files) { + mkdirSync(directory, { recursive: true }); + writeFileSync(join(directory, "package.json"), `${JSON.stringify(manifest, null, "\t")}\n`); + for (const [path, contents] of Object.entries(files)) { + mkdirSync(dirname(join(directory, path)), { recursive: true }); + writeFileSync(join(directory, path), contents); + } +} + +function createArtifactSet(t) { + const temporaryRoot = mkdtempSync(join(tmpdir(), "pi-local-package-install-test-")); + t.after(() => rmSync(temporaryRoot, { recursive: true, force: true })); + const root = join(temporaryRoot, "fixture with spaces"); + const repoRoot = join(root, "repo"); + mkdirSync(repoRoot, { recursive: true }); + writeFileSync(join(repoRoot, "package.json"), '{"name":"fixture","private":true}\n'); + writePackage( + join(repoRoot, "packages", "shared"), + { + name: "@pi-package-test/shared", + version: "1.0.0", + type: "module", + main: "./dist/index.js", + types: "./dist/index.d.ts", + exports: { ".": { types: "./dist/index.d.ts", import: "./dist/index.js" } }, + files: ["dist"], + }, + { + "dist/index.js": 'export const marker = "local artifact";\n', + "dist/index.d.ts": 'export declare const marker: "local artifact";\n', + }, + ); + writePackage( + join(repoRoot, "packages", "target"), + { + name: "@pi-package-test/target", + version: "1.0.0", + type: "module", + main: "./dist/index.js", + types: "./dist/index.d.ts", + exports: { + ".": { types: "./dist/index.d.ts", import: "./dist/index.js" }, + "./feature": { types: "./dist/feature.d.ts", import: "./dist/feature.js" }, + }, + bin: { target: "./dist/cli.js" }, + files: ["dist"], + dependencies: { "@pi-package-test/shared": "1.0.0" }, + }, + { + "dist/index.js": 'import { marker } from "@pi-package-test/shared";\nexport { marker };\n', + "dist/index.d.ts": 'export { marker } from "@pi-package-test/shared";\n', + "dist/feature.js": "export const feature = true;\n", + "dist/feature.d.ts": "export declare const feature: true;\n", + "dist/cli.js": "#!/usr/bin/env node\n", + }, + ); + writePackage( + join(repoRoot, "packages", "second"), + { + name: "@pi-package-test/second", + version: "1.0.0", + type: "module", + main: "./dist/index.js", + types: "./dist/index.d.ts", + files: ["dist"], + }, + { + "dist/index.js": "export const second = true;\n", + "dist/index.d.ts": "export declare const second: true;\n", + }, + ); + return { + artifactSet: produceArtifactSet({ build: false, outDir: join(root, "artifacts"), repoRoot, source: null }), + root, + }; +} + +test("wires multiple direct packages without registry fallbacks", (t) => { + const { artifactSet, root } = createArtifactSet(t); + const consumerDirectory = join(root, "consumer"); + mkdirSync(consumerDirectory); + writeFileSync( + join(consumerDirectory, "package.json"), + '{"private":true,"type":"module","dependencies":{"@pi-package-test/target":"1.0.0","@pi-package-test/second":"1.0.0"}}\n', + ); + wireConsumer({ + artifactSet, + consumerDirectory, + packageNames: ["@pi-package-test/target", "@pi-package-test/second"], + }); + execFileSync("npm", ["install", "--ignore-scripts", "--no-audit", "--no-fund"], { cwd: consumerDirectory, stdio: "pipe" }); + + const manifest = JSON.parse(readFileSync(join(consumerDirectory, "package.json"), "utf8")); + assert.match(manifest.dependencies["@pi-package-test/target"], /^file:/); + assert.match(manifest.dependencies["@pi-package-test/second"], /^file:/); + smokeTestNpmConsumer({ artifactSet, directory: consumerDirectory, packageName: "@pi-package-test/target" }); + smokeTestNpmConsumer({ artifactSet, directory: consumerDirectory, packageName: "@pi-package-test/second" }); + + const lockPath = join(consumerDirectory, "package-lock.json"); + const lockContents = readFileSync(lockPath, "utf8"); + const lock = JSON.parse(lockContents); + const targetLockEntry = Object.values(lock.packages).find((entry) => entry.resolved?.includes("pi-package-test-target")); + assert.ok(targetLockEntry); + targetLockEntry.resolved = "https://registry.npmjs.org/@pi-package-test/target/-/target-1.0.0.tgz"; + writeFileSync(lockPath, JSON.stringify(lock)); + assert.throws( + () => smokeTestNpmConsumer({ artifactSet, directory: consumerDirectory, packageName: "@pi-package-test/target" }), + /did not resolve from a local artifact/, + ); + writeFileSync(lockPath, lockContents); + + rmSync(join(consumerDirectory, "node_modules", "@pi-package-test", "target", "dist", "feature.js")); + assert.throws( + () => smokeTestNpmConsumer({ artifactSet, directory: consumerDirectory, packageName: "@pi-package-test/target" }), + /export \.\/feature does not exist/, + ); +}); + +test("installs a package as the only direct dependency", (t) => { + const { artifactSet, root } = createArtifactSet(t); + const unsupportedDirectory = join(root, "unsupported-consumer"); + assert.throws( + () => installConsumer({ artifactSet, directory: unsupportedDirectory, packageManager: "yarn", packageNames: ["@pi-package-test/target"] }), + /Unsupported package manager: yarn/, + ); + assert.equal(existsSync(unsupportedDirectory), false); + const consumerDirectory = join(root, "isolated-consumer"); + installConsumer({ artifactSet, directory: consumerDirectory, packageNames: ["@pi-package-test/target"] }); + const manifest = JSON.parse(readFileSync(join(consumerDirectory, "package.json"), "utf8")); + assert.deepEqual(Object.keys(manifest.dependencies), ["@pi-package-test/target"]); + smokeTestNpmConsumer({ artifactSet, directory: consumerDirectory, packageName: "@pi-package-test/target" }); + const installedManifestPath = join(consumerDirectory, "node_modules", "@pi-package-test", "target", "package.json"); + const installedManifestContents = readFileSync(installedManifestPath, "utf8"); + const installedManifest = JSON.parse(installedManifestContents); + delete installedManifest.main; + writeFileSync(installedManifestPath, JSON.stringify(installedManifest)); + assert.throws( + () => smokeTestNpmConsumer({ artifactSet, directory: consumerDirectory, packageName: "@pi-package-test/target" }), + /Public Pi package @pi-package-test\/target must declare a string main field/, + ); + writeFileSync(installedManifestPath, installedManifestContents); + const lockPath = join(consumerDirectory, "package-lock.json"); + const lockContents = readFileSync(lockPath, "utf8"); + rmSync(lockPath); + assert.throws( + () => smokeTestNpmConsumer({ artifactSet, directory: consumerDirectory, packageName: "@pi-package-test/target" }), + /npm consumer smoke test requires package-lock\.json/, + ); + writeFileSync(lockPath, lockContents); + const lock = JSON.parse(lockContents); + delete lock.packages; + writeFileSync(lockPath, JSON.stringify(lock)); + assert.throws( + () => smokeTestNpmConsumer({ artifactSet, directory: consumerDirectory, packageName: "@pi-package-test/target" }), + /Invalid npm lockfile/, + ); +}); + +test("wires and installs a pnpm consumer without registry fallbacks", (t) => { + const { artifactSet, root } = createArtifactSet(t); + const consumerDirectory = join(root, "pnpm-consumer"); + mkdirSync(consumerDirectory); + writeFileSync(join(consumerDirectory, "package.json"), '{"private":true,"type":"module"}\n'); + writeFileSync(join(consumerDirectory, "pnpm-workspace.yaml"), "# existing workspace comment\npackages: []\noverrides:\n existing: 1.2.3\n"); + wireConsumer({ + artifactSet, + consumerDirectory, + packageManager: "pnpm", + packageNames: ["@pi-package-test/target"], + }); + + const manifest = JSON.parse(readFileSync(join(consumerDirectory, "package.json"), "utf8")); + assert.deepEqual(Object.keys(manifest.dependencies), ["@pi-package-test/target"]); + assert.equal(manifest.overrides, undefined); + const workspaceContents = readFileSync(join(consumerDirectory, "pnpm-workspace.yaml"), "utf8"); + assert.match(workspaceContents, /^# existing workspace comment/m); + const workspace = parse(workspaceContents); + assert.equal(workspace.overrides.existing, "1.2.3"); + for (const artifact of artifactSet.packages) { + assert.ok(workspace.overrides[artifact.name].endsWith(basename(artifact.tarballPath)), `${artifact.name} override must reference its artifact`); + } + + const pnpmCli = fileURLToPath(new URL("../node_modules/pnpm/bin/pnpm.mjs", import.meta.url)); + execFileSync(process.execPath, [pnpmCli, "install", "--prod", "--ignore-scripts", "--offline"], { + cwd: consumerDirectory, + stdio: "pipe", + timeout: 300_000, + }); + const marker = execFileSync(process.execPath, ["--input-type=module", "--eval", 'import("@pi-package-test/target").then(({ marker }) => console.log(marker))'], { + cwd: consumerDirectory, + encoding: "utf8", + }); + assert.equal(marker.trim(), "local artifact"); + const lockContents = readFileSync(join(consumerDirectory, "pnpm-lock.yaml"), "utf8"); + assert.doesNotMatch(lockContents, /https?:\/\//); + for (const packageName of ["@pi-package-test/target", "@pi-package-test/shared"]) { + assert.ok(lockContents.includes(basename(artifactSet.getPackage(packageName).tarballPath)), `${packageName} lock entry must reference its artifact`); + } +}); diff --git a/scripts/local-release.mjs b/scripts/local-release.mjs index c93670d74..747435b66 100644 --- a/scripts/local-release.mjs +++ b/scripts/local-release.mjs @@ -1,146 +1,32 @@ #!/usr/bin/env node +import { execFileSync } from "node:child_process"; -import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { cpSync, existsSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { isAbsolute, join, relative, resolve } from "node:path"; -import { spawnSync } from "node:child_process"; -import { installCodingAgentConsumer, packReleasePackages, smokeTestCodingAgentConsumer } from "./coding-agent-consumer.mjs"; - -const packages = [ - { directory: "packages/chord", name: "@earendil-works/chord" }, - { directory: "packages/telemetry", name: "@earendil-works/pi-telemetry" }, - { directory: "packages/codemode", name: "@earendil-works/pi-codemode" }, - { directory: "packages/mcp", name: "@earendil-works/pi-mcp" }, - { directory: "packages/ai", name: "@earendil-works/pi-ai" }, - { directory: "packages/durable", name: "@earendil-works/pi-durable" }, - { directory: "packages/env", name: "@earendil-works/pi-env" }, - { directory: "packages/tui", name: "@earendil-works/pi-tui" }, - { directory: "packages/agent", name: "@earendil-works/pi-agent-core" }, - { directory: "packages/protocol", name: "@earendil-works/pi-protocol" }, - { directory: "packages/client", name: "@earendil-works/pi-client" }, - { directory: "packages/server", name: "@earendil-works/pi-server" }, - { directory: "packages/coding-agent", name: "@earendil-works/pi-coding-agent" }, -]; +import { join } from "node:path"; +import { parseArgs } from "node:util"; +import { installConsumer, packageConsumerDirectoryName, smokeTestNpmConsumer } from "./local-package-install.mjs"; +import { produceArtifactSet } from "./package-artifacts.mjs"; +import { codingAgentName, smokeTestCodingAgent } from "./coding-agent-smoke.mjs"; +import { execNpmSync } from "./npm-command.mjs"; function printUsage() { console.log(`Usage: node scripts/local-release.mjs [options] -Builds and packs the publishable packages, then installs the tarballs into an -isolated directory outside the repository for local release testing. +Builds and packs the publishable packages, then installs the tarballs into +isolated directories outside the repository for local release testing. Options: --out Output directory. Defaults to a new directory under ${tmpdir()} --force Remove --out first if it already exists --skip-check Do not run npm run check before building - --skip-test Do not run ./test.sh before building - --skip-install Only create tarballs; do not create isolated installs + --skip-test Do not run ./test.sh after building + --skip-install Only create the artifact set; do not create isolated installs --skip-bun-install Do not create the isolated Bun install --help Show this help `); } -function parseArgs() { - const options = { - force: false, - outDir: undefined, - skipBunInstall: false, - skipCheck: false, - skipInstall: false, - skipTest: false, - }; - const args = process.argv.slice(2); - - for (let i = 0; i < args.length; i++) { - const arg = args[i]; - if (arg === "--help") { - printUsage(); - process.exit(0); - } - if (arg === "--force") { - options.force = true; - continue; - } - if (arg === "--skip-check") { - options.skipCheck = true; - continue; - } - if (arg === "--skip-test") { - options.skipTest = true; - continue; - } - if (arg === "--skip-install") { - options.skipInstall = true; - continue; - } - if (arg === "--skip-bun-install") { - options.skipBunInstall = true; - continue; - } - if (arg === "--out") { - const value = args[++i]; - if (!value) { - throw new Error("--out requires a directory"); - } - options.outDir = value; - continue; - } - throw new Error(`Unknown option: ${arg}`); - } - - return options; -} - -function run(command, args, options = {}) { - console.log(`$ ${[command, ...args].join(" ")}`); - const result = spawnSync(command, args, { - cwd: options.cwd, - encoding: "utf8", - shell: process.platform === "win32", - stdio: options.capture ? ["inherit", "pipe", "inherit"] : "inherit", - }); - - if (result.status !== 0) { - throw new Error(`Command failed: ${[command, ...args].join(" ")}`); - } - - return result.stdout ?? ""; -} - -function readPackageJson(directory) { - return JSON.parse(readFileSync(join(directory, "package.json"), "utf8")); -} - -function commandExists(command) { - return spawnSync(command, ["--version"], { stdio: "ignore" }).status === 0; -} - -function isInsidePath(child, parent) { - const relativePath = relative(parent, child); - return relativePath === "" || (!relativePath.startsWith("..") && !isAbsolute(relativePath)); -} - -function prepareOutputDirectory(options, repoRoot) { - if (!options.outDir) { - return mkdtempSync(join(tmpdir(), "pi-local-release-")); - } - - const outDir = resolve(options.outDir); - - if (isInsidePath(outDir, repoRoot)) { - throw new Error(`Output directory must be outside the repository: ${outDir}`); - } - - if (existsSync(outDir)) { - if (!options.force) { - throw new Error(`Output directory already exists. Use --force to replace it: ${outDir}`); - } - rmSync(outDir, { force: true, recursive: true }); - } - - mkdirSync(outDir, { recursive: true }); - return outDir; -} - function currentBinaryPlatform() { if (process.platform === "win32") return process.arch === "arm64" ? "windows-arm64" : "windows-x64"; if (process.platform === "darwin") return process.arch === "arm64" ? "darwin-arm64" : "darwin-x64"; @@ -149,19 +35,16 @@ function currentBinaryPlatform() { } function buildBunBinaryRelease(targetDirectory, archiveDirectory) { - if (!commandExists("bun")) { - throw new Error("Bun is required for the local binary release build."); - } const platform = currentBinaryPlatform(); const binaryBuildDirectory = join(archiveDirectory, "binary-build"); - run("./scripts/build-binaries.sh", [ + execFileSync("bash", ["./scripts/build-binaries.sh", "--skip-install", "--skip-build", "--platform", platform, "--out", binaryBuildDirectory, - ]); + ], { stdio: "inherit" }); rmSync(targetDirectory, { force: true, recursive: true }); cpSync(join(binaryBuildDirectory, platform), targetDirectory, { recursive: true }); const archiveName = platform.startsWith("windows-") ? `pi-${platform}.zip` : `pi-${platform}.tar.gz`; @@ -184,64 +67,79 @@ function createPiShim(installDirectory) { symlinkSync(join("node_modules", ".bin", "pi"), join(installDirectory, "pi")); } -const options = parseArgs(); -const repoRoot = process.cwd(); -const rootPackageJson = readPackageJson(repoRoot); - -if (rootPackageJson.name !== "pi-monorepo") { - throw new Error("Run this script from the repository root"); +const { values } = parseArgs({ + options: { + force: { type: "boolean", default: false }, + help: { type: "boolean", default: false }, + out: { type: "string" }, + "skip-bun-install": { type: "boolean", default: false }, + "skip-check": { type: "boolean", default: false }, + "skip-install": { type: "boolean", default: false }, + "skip-test": { type: "boolean", default: false }, + }, +}); +if (values.help) { + printUsage(); + process.exit(0); } +const options = { + force: values.force, + outDir: values.out, + skipBunInstall: values["skip-bun-install"], + skipCheck: values["skip-check"], + skipInstall: values["skip-install"], + skipTest: values["skip-test"], +}; +const repoRoot = process.cwd(); +const rootPackageJson = JSON.parse(readFileSync(join(repoRoot, "package.json"), "utf8")); +if (rootPackageJson.name !== "pi-monorepo") throw new Error("Run this script from the repository root"); -const outDir = prepareOutputDirectory(options, repoRoot); -const tarballDirectory = join(outDir, "tarballs"); +execNpmSync(["run", "generate:models"], { cwd: repoRoot, stdio: "inherit" }); +if (!options.skipCheck) execNpmSync(["run", "check"], { cwd: repoRoot, stdio: "inherit" }); + +const artifactSet = produceArtifactSet({ + build: true, + force: options.force, + offlineModelData: true, + outDir: options.outDir, + repoRoot, +}); +const outDir = artifactSet.artifactDirectory; +const binaryDirectory = join(outDir, "bun"); const nodeInstallDirectory = join(outDir, "node"); const bunInstallDirectory = join(outDir, "bun-install"); -const binaryDirectory = join(outDir, "bun"); -mkdirSync(tarballDirectory, { recursive: true }); -// Release artifacts always use a freshly generated, strictly validated catalog, -// including when checks or tests are explicitly skipped. -run("npm", ["run", "generate:models"], { cwd: repoRoot }); - -if (!options.skipCheck) { - run("npm", ["run", "check"], { cwd: repoRoot }); -} - -for (const pkg of packages) { - run("npm", ["run", "clean"], { cwd: pkg.directory }); - run("npm", ["run", pkg.directory === "packages/ai" ? "build:offline" : "build"], { cwd: pkg.directory }); -} - -if (!options.skipTest) { - run("./test.sh", [], { cwd: repoRoot }); -} - -const tarballs = packReleasePackages(packages, tarballDirectory); +if (!options.skipTest) execFileSync("bash", ["./test.sh"], { cwd: repoRoot, stdio: "inherit" }); let binaryPlatform; if (!options.skipInstall) { binaryPlatform = buildBunBinaryRelease(binaryDirectory, outDir); - - installCodingAgentConsumer(nodeInstallDirectory, tarballs); - smokeTestCodingAgentConsumer(nodeInstallDirectory); - createPiShim(nodeInstallDirectory); - - if (!options.skipBunInstall) { - if (!commandExists("bun")) { - throw new Error("Bun is required for the isolated Bun install. Use --skip-bun-install to skip it."); + const validationRoot = mkdtempSync(join(tmpdir(), "pi-local-release-consumers-")); + try { + for (const pkg of artifactSet.packages) { + const directory = pkg.name === codingAgentName + ? nodeInstallDirectory + : join(validationRoot, "npm", packageConsumerDirectoryName(pkg.name)); + installConsumer({ artifactSet, directory, packageNames: [pkg.name] }); + smokeTestNpmConsumer({ artifactSet, directory, packageName: pkg.name }); + if (pkg.name === codingAgentName) smokeTestCodingAgent(directory); } - installCodingAgentConsumer(bunInstallDirectory, tarballs, "bun"); - smokeTestCodingAgentConsumer(bunInstallDirectory, "bun"); - createPiShim(bunInstallDirectory); + createPiShim(nodeInstallDirectory); + + if (!options.skipBunInstall) { + installConsumer({ artifactSet, directory: bunInstallDirectory, packageManager: "bun", packageNames: [codingAgentName] }); + smokeTestCodingAgent(bunInstallDirectory, "bun"); + createPiShim(bunInstallDirectory); + } + } finally { + rmSync(validationRoot, { force: true, recursive: true }); } } -console.log("\nLocal release artifacts created:"); -console.log(` ${outDir}`); +console.log(`\nLocal release artifacts created: ${outDir}`); +console.log(`Manifest: ${artifactSet.manifestPath}`); console.log("\nTarballs:"); -for (const tarball of tarballs.values()) { - console.log(` ${tarball}`); -} +for (const pkg of artifactSet.packages) console.log(` ${pkg.tarballPath}`); if (!options.skipInstall) { console.log("\nLocal Bun binary release:"); diff --git a/scripts/npm-command.mjs b/scripts/npm-command.mjs new file mode 100644 index 000000000..b5af68ba8 --- /dev/null +++ b/scripts/npm-command.mjs @@ -0,0 +1,10 @@ +import { execFileSync } from "node:child_process"; +import { existsSync } from "node:fs"; + +export function execNpmSync(args, options = {}) { + if (process.platform !== "win32") return execFileSync("npm", args, options); + const npmCli = process.env.npm_execpath; + if (!npmCli) throw new Error("Cannot locate npm on Windows. Run this command through its npm script."); + if (!existsSync(npmCli)) throw new Error(`Cannot locate the npm CLI: ${npmCli}`); + return execFileSync(process.execPath, [npmCli, ...args], options); +} diff --git a/scripts/pack-packages.mjs b/scripts/pack-packages.mjs new file mode 100644 index 000000000..b61a61320 --- /dev/null +++ b/scripts/pack-packages.mjs @@ -0,0 +1,48 @@ +#!/usr/bin/env node + +import { resolve } from "node:path"; +import { parseArgs } from "node:util"; +import { produceArtifactSet } from "./package-artifacts.mjs"; + +function printUsage() { + console.log(`Usage: node scripts/pack-packages.mjs --out [options] + +Builds every public package and writes one coherent local artifact set. + +Options: + --out Output directory; .artifacts/ is ignored by Git + --force Replace an existing output directory + --offline-model-data Build with already-hydrated model data + --skip-build Pack existing build output without rebuilding + --help Show this help +`); +} + +const { values } = parseArgs({ + options: { + force: { type: "boolean", default: false }, + help: { type: "boolean", default: false }, + "offline-model-data": { type: "boolean", default: false }, + out: { type: "string" }, + "skip-build": { type: "boolean", default: false }, + }, +}); +if (values.help) { + printUsage(); + process.exit(0); +} +if (!values.out) throw new Error("--out is required"); + +const artifactSet = produceArtifactSet({ + build: !values["skip-build"], + force: values.force, + offlineModelData: values["offline-model-data"], + outDir: values.out, + repoRoot: process.cwd(), +}); +console.log(`\nLocal package artifacts created: ${artifactSet.artifactDirectory}`); +console.log(`Manifest: ${artifactSet.manifestPath}`); +console.log("\nConnect an external npm project with:"); +console.log( + ` node ${resolve("scripts/use-local-packages.mjs")} --manifest ${artifactSet.manifestPath} --consumer --package `, +); diff --git a/scripts/package-artifact-cli.test.mjs b/scripts/package-artifact-cli.test.mjs new file mode 100644 index 000000000..f730319d7 --- /dev/null +++ b/scripts/package-artifact-cli.test.mjs @@ -0,0 +1,31 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { dirname, join, resolve } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const scriptsDirectory = dirname(fileURLToPath(import.meta.url)); +const repoRoot = resolve(scriptsDirectory, ".."); + +function runScript(name, args) { + return spawnSync(process.execPath, [join(scriptsDirectory, name), ...args], { cwd: repoRoot, encoding: "utf8" }); +} + +function assertFailsBeforeWork(result, message) { + assert.notEqual(result.status, 0); + assert.match(result.stderr, message); + assert.doesNotMatch(result.stdout, /generate:models|npm run clean/); +} + +test("rejects missing option values before doing work", () => { + assertFailsBeforeWork(runScript("local-release.mjs", ["--out"]), /--out/); + assertFailsBeforeWork(runScript("pack-packages.mjs", ["--out", "--force"]), /--out/); + assertFailsBeforeWork(runScript("use-local-packages.mjs", ["--manifest", "--consumer", "target"]), /--manifest/); +}); + +test("rejects an unsupported package manager before doing work", () => { + assertFailsBeforeWork( + runScript("use-local-packages.mjs", ["--manifest", "missing.json", "--consumer", "target", "--package", "example", "--package-manager", "yarn"]), + /Unsupported package manager: yarn/, + ); +}); diff --git a/scripts/package-artifacts.mjs b/scripts/package-artifacts.mjs new file mode 100644 index 000000000..5934b9faf --- /dev/null +++ b/scripts/package-artifacts.mjs @@ -0,0 +1,141 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, isAbsolute, join, relative, resolve } from "node:path"; +import { getPublicWorkspacePackages } from "./release-packages.mjs"; +import { execNpmSync } from "./npm-command.mjs"; + +const manifestSchemaVersion = 1; + +function isInsidePath(child, parent) { + const relativePath = relative(parent, child); + return relativePath === "" || (!relativePath.startsWith("..") && !isAbsolute(relativePath)); +} + +function getGitSource(repoRoot) { + const commit = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repoRoot, encoding: "utf8" }).trim(); + const status = execFileSync("git", ["status", "--porcelain"], { cwd: repoRoot, encoding: "utf8" }); + return { commit, dirty: status.length > 0 }; +} + +function packPackages(packages, tarballDirectory) { + mkdirSync(tarballDirectory, { recursive: true }); + const packedPackages = []; + for (const pkg of packages) { + const packageJson = JSON.parse(readFileSync(join(pkg.directory, "package.json"), "utf8")); + if (packageJson.name !== pkg.name) throw new Error(`Unexpected package name in ${pkg.directory}`); + const output = execNpmSync(["pack", "--ignore-scripts", "--json", "--pack-destination", tarballDirectory], { + cwd: pkg.directory, + encoding: "utf8", + stdio: ["inherit", "pipe", "inherit"], + }); + const parsed = JSON.parse(output); + if (!Array.isArray(parsed) || parsed.length !== 1 || !parsed[0]?.filename) { + throw new Error(`npm pack returned an unexpected result for ${pkg.name}`); + } + const packed = parsed[0]; + + const originalPath = join(tarballDirectory, packed.filename); + const contents = readFileSync(originalPath); + const digest = createHash("sha512").update(contents).digest(); + const hash = digest.toString("hex").slice(0, 12); + const tarballPath = originalPath.replace(/\.tgz$/, `-${hash}.tgz`); + renameSync(originalPath, tarballPath); + console.log( + ` ${pkg.name}@${pkg.version}: ${packed.files.length} files, ${packed.size} bytes packed, ${packed.unpackedSize} bytes unpacked`, + ); + packedPackages.push({ + name: pkg.name, + version: pkg.version, + tarballPath, + integrity: `sha512-${digest.toString("base64")}`, + }); + } + return packedPackages; +} + +function prepareOutputDirectory(outDir, options) { + const repoRoot = resolve(options.repoRoot); + if (!outDir) return mkdtempSync(join(tmpdir(), "pi-package-artifacts-")); + const outputDirectory = resolve(outDir); + if (dirname(outputDirectory) === outputDirectory || isInsidePath(repoRoot, outputDirectory)) { + throw new Error(`Output directory must not be the repository, its ancestor, or a filesystem root: ${outputDirectory}`); + } + if (isInsidePath(outputDirectory, repoRoot) && !isInsidePath(outputDirectory, join(repoRoot, ".artifacts"))) { + throw new Error(`Repository-local output directory must be inside ${join(repoRoot, ".artifacts")}: ${outputDirectory}`); + } + if (existsSync(outputDirectory)) { + if (!options.force) throw new Error(`Output directory already exists. Use --force to replace it: ${outputDirectory}`); + rmSync(outputDirectory, { force: true, recursive: true }); + } + mkdirSync(outputDirectory, { recursive: true }); + return outputDirectory; +} + +export function produceArtifactSet({ repoRoot, outDir, build = true, offlineModelData = false, force = false, source }) { + const root = resolve(repoRoot); + const artifactDirectory = prepareOutputDirectory(outDir, { force, repoRoot: root }); + if (build) { + execNpmSync(["run", "clean"], { cwd: root, stdio: "inherit" }); + execNpmSync(["run", offlineModelData ? "build:offline" : "build"], { cwd: root, stdio: "inherit" }); + } + const artifactSource = source === undefined ? getGitSource(root) : source; + const packages = getPublicWorkspacePackages(join(root, "packages")); + const packedPackages = packPackages(packages, join(artifactDirectory, "tarballs")); + const manifest = { + schemaVersion: manifestSchemaVersion, + source: artifactSource, + packages: packedPackages + .map(({ name, version, tarballPath, integrity }) => ({ + name, + version, + tarball: relative(artifactDirectory, tarballPath).replaceAll("\\", "/"), + integrity, + })) + .sort((left, right) => left.name.localeCompare(right.name)), + }; + const manifestPath = join(artifactDirectory, "manifest.json"); + writeFileSync(manifestPath, `${JSON.stringify(manifest, null, "\t")}\n`); + return readArtifactSet(manifestPath); +} + +export function readArtifactSet(manifestPath) { + const absoluteManifestPath = resolve(manifestPath); + const artifactDirectory = dirname(absoluteManifestPath); + const manifest = JSON.parse(readFileSync(absoluteManifestPath, "utf8")); + if (manifest.schemaVersion !== manifestSchemaVersion || !Array.isArray(manifest.packages)) { + throw new Error(`Unsupported package artifact manifest: ${absoluteManifestPath}`); + } + const names = new Set(); + const packages = manifest.packages.map((pkg) => { + if ( + typeof pkg?.name !== "string" || + typeof pkg.version !== "string" || + typeof pkg.tarball !== "string" || + typeof pkg.integrity !== "string" + ) { + throw new Error(`Invalid package entry in artifact manifest: ${absoluteManifestPath}`); + } + if (names.has(pkg.name)) throw new Error(`Duplicate package in artifact manifest: ${pkg.name}`); + names.add(pkg.name); + const tarballPath = resolve(artifactDirectory, pkg.tarball); + if (!isInsidePath(tarballPath, artifactDirectory) || !existsSync(tarballPath)) { + throw new Error(`Missing package tarball for ${pkg.name}: ${tarballPath}`); + } + const integrity = `sha512-${createHash("sha512").update(readFileSync(tarballPath)).digest("base64")}`; + if (integrity !== pkg.integrity) throw new Error(`Package tarball integrity mismatch for ${pkg.name}: ${tarballPath}`); + return { ...pkg, tarballPath }; + }); + return { + artifactDirectory, + manifestPath: absoluteManifestPath, + packages, + source: manifest.source, + getPackage(name) { + const pkg = packages.find((candidate) => candidate.name === name); + if (!pkg) throw new Error(`Package is not present in the artifact set: ${name}`); + return pkg; + }, + }; +} diff --git a/scripts/package-artifacts.test.mjs b/scripts/package-artifacts.test.mjs new file mode 100644 index 000000000..b5ec291f5 --- /dev/null +++ b/scripts/package-artifacts.test.mjs @@ -0,0 +1,76 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { appendFileSync, existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, parse } from "node:path"; +import test from "node:test"; +import { produceArtifactSet, readArtifactSet } from "./package-artifacts.mjs"; + +function writePackage(directory, manifest, files) { + mkdirSync(directory, { recursive: true }); + writeFileSync(join(directory, "package.json"), `${JSON.stringify(manifest, null, "\t")}\n`); + for (const [path, contents] of Object.entries(files)) { + mkdirSync(dirname(join(directory, path)), { recursive: true }); + writeFileSync(join(directory, path), contents); + } +} + +test("produces a verified, content-addressed artifact set", (t) => { + const temporaryRoot = mkdtempSync(join(tmpdir(), "pi-package-artifacts-test-")); + t.after(() => rmSync(temporaryRoot, { recursive: true, force: true })); + const repoRoot = join(temporaryRoot, "repo with spaces"); + mkdirSync(repoRoot); + writeFileSync(join(repoRoot, "package.json"), '{"name":"fixture","private":true}\n'); + writePackage( + join(repoRoot, "packages", "shared"), + { name: "@pi-package-test/shared", version: "1.0.0", files: ["dist"] }, + { "dist/index.js": 'export const marker = "artifact";\n' }, + ); + writePackage( + join(repoRoot, "packages", "target"), + { name: "@pi-package-test/target", version: "1.0.0", files: ["dist"] }, + { "dist/index.js": 'export const target = true;\n' }, + ); + execFileSync("git", ["init", "--quiet"], { cwd: repoRoot }); + execFileSync("git", ["config", "user.email", "test@example.com"], { cwd: repoRoot }); + execFileSync("git", ["config", "user.name", "Test"], { cwd: repoRoot }); + execFileSync("git", ["add", "."], { cwd: repoRoot }); + execFileSync("git", ["commit", "--quiet", "-m", "fixture"], { cwd: repoRoot }); + + const artifactSet = produceArtifactSet({ build: false, outDir: join(repoRoot, ".artifacts", "package set"), repoRoot }); + assert.deepEqual(artifactSet.packages.map((pkg) => pkg.name), ["@pi-package-test/shared", "@pi-package-test/target"]); + for (const pkg of artifactSet.packages) { + assert.match(pkg.tarball, /-[0-9a-f]{12}\.tgz$/); + assert.match(pkg.integrity, /^sha512-/); + } + assert.equal(artifactSet.source.dirty, false); + assert.equal(readArtifactSet(artifactSet.manifestPath).packages.length, 2); + + writeFileSync(join(repoRoot, "packages/shared/dist/index.js"), 'export const marker = "changed";\n'); + const changedArtifactSet = produceArtifactSet({ build: false, outDir: join(repoRoot, ".artifacts", "changed package set"), repoRoot }); + assert.notEqual( + changedArtifactSet.getPackage("@pi-package-test/shared").tarball, + artifactSet.getPackage("@pi-package-test/shared").tarball, + ); + + appendFileSync(artifactSet.packages[0].tarballPath, "corrupt"); + assert.throws(() => readArtifactSet(artifactSet.manifestPath), /integrity mismatch/); + const packageJsonPath = join(repoRoot, "packages", "shared", "package.json"); + assert.throws( + () => produceArtifactSet({ build: false, force: true, outDir: join(repoRoot, "packages", "shared"), repoRoot }), + /Repository-local output directory must be inside.*\.artifacts/, + ); + assert.equal(existsSync(packageJsonPath), true); + assert.throws( + () => produceArtifactSet({ build: false, force: true, outDir: repoRoot, repoRoot }), + /repository, its ancestor, or a filesystem root/, + ); + assert.throws( + () => produceArtifactSet({ build: false, force: true, outDir: temporaryRoot, repoRoot }), + /repository, its ancestor, or a filesystem root/, + ); + assert.throws( + () => produceArtifactSet({ build: false, force: true, outDir: parse(repoRoot).root, repoRoot }), + /repository, its ancestor, or a filesystem root/, + ); +}); diff --git a/scripts/publish.mjs b/scripts/publish.mjs index 0ce30944e..80957a869 100644 --- a/scripts/publish.mjs +++ b/scripts/publish.mjs @@ -1,110 +1,64 @@ #!/usr/bin/env node -import { spawnSync } from "node:child_process"; -import { existsSync } from "node:fs"; +import { existsSync, rmSync } from "node:fs"; import { join } from "node:path"; +import { produceArtifactSet } from "./package-artifacts.mjs"; import { getPublicWorkspacePackages } from "./release-packages.mjs"; - -const packages = getPublicWorkspacePackages(); +import { execNpmSync } from "./npm-command.mjs"; const dryRun = process.argv.includes("--dry-run"); const unknownArgs = process.argv.slice(2).filter((arg) => arg !== "--dry-run"); - -if (unknownArgs.length > 0) { - console.error(`Usage: node scripts/publish.mjs [--dry-run]`); - process.exit(1); -} - -function commandForPlatform(command) { - return process.platform === "win32" ? `${command}.cmd` : command; -} - -function run(command, args, options = {}) { - console.log(`$ ${[command, ...args].join(" ")}`); - const result = spawnSync(commandForPlatform(command), args, { - cwd: options.cwd, - encoding: "utf8", - stdio: options.capture ? ["inherit", "pipe", "pipe"] : "inherit", - }); - - if (result.status !== 0) { - const output = [result.stdout, result.stderr].filter(Boolean).join("\n"); - throw new Error(output ? `Command failed: ${command} ${args.join(" ")}\n${output}` : `Command failed: ${command} ${args.join(" ")}`); - } - - return result; -} - -function assertBuildOutputExists(directory) { - if (!existsSync(join(directory, "dist"))) { - throw new Error(`${directory}/dist does not exist. Run npm run build before publishing.`); - } -} - -function validatePack(directory) { - const result = run("npm", ["pack", "--dry-run", "--ignore-scripts", "--json"], { capture: true, cwd: directory }); - const packed = JSON.parse(result.stdout)[0]; - console.log(` ${packed.filename}: ${packed.files.length} files, ${packed.size} bytes packed, ${packed.unpackedSize} bytes unpacked`); -} +if (unknownArgs.length > 0) throw new Error("Usage: node scripts/publish.mjs [--dry-run]"); function isPublished(name, version) { - const result = spawnSync(commandForPlatform("npm"), ["view", `${name}@${version}`, "version", "--json"], { - encoding: "utf8", - stdio: ["inherit", "pipe", "pipe"], - }); - - if (result.status === 0 && result.stdout.trim()) { + try { + const output = execNpmSync(["view", `${name}@${version}`, "version", "--json"], { + encoding: "utf8", + stdio: ["inherit", "pipe", "pipe"], + }); + if (!output.trim()) throw new Error(`npm returned no version for ${name}@${version}`); return true; + } catch (error) { + const output = error && typeof error === "object" + ? ["stdout", "stderr"].map((key) => key in error ? String(error[key] ?? "") : "").filter(Boolean).join("\n") + : ""; + if (output.includes("E404") || output.includes("404 Not Found")) return false; + throw new Error(`Failed to query ${name}@${version}${output ? `\n${output}` : ""}`, { cause: error }); } - - const output = [result.stdout, result.stderr].filter(Boolean).join("\n"); - if (result.status !== 0 && (output.includes("E404") || output.includes("404 Not Found"))) { - return false; - } - - throw new Error(output ? `Failed to query ${name}@${version}\n${output}` : `Failed to query ${name}@${version}`); } -const packageVersions = new Map(packages.map((pkg) => [pkg.name, pkg.version])); - -const versions = [...new Set(packageVersions.values())]; -if (versions.length !== 1) { - throw new Error(`Publish packages are not lockstep versioned: ${versions.join(", ")}`); +const repoRoot = process.cwd(); +const packages = getPublicWorkspacePackages(); +const packageByName = new Map(packages.map((pkg) => [pkg.name, pkg])); +const versions = [...new Set(packages.map((pkg) => pkg.version))]; +if (versions.length !== 1) throw new Error(`Publish packages are not lockstep versioned: ${versions.join(", ")}`); +for (const pkg of packages) { + if (!existsSync(join(pkg.directory, "dist"))) { + throw new Error(`${pkg.directory}/dist does not exist. Run npm run build before publishing.`); + } } console.log(`Publishing pi packages at ${versions[0]}${dryRun ? " (dry run)" : ""}\n`); - -const packageStates = packages.map((pkg) => ({ - ...pkg, - published: false, - version: packageVersions.get(pkg.name), -})); - -for (const pkg of packageStates) { - assertBuildOutputExists(pkg.directory); - pkg.published = isPublished(pkg.name, pkg.version); - - if (pkg.published) { - console.log(`${pkg.name}@${pkg.version} is already published; validating package contents only.`); - } else { - console.log(`${pkg.name}@${pkg.version} is not published; validating package contents before publish.`); +const artifactSet = produceArtifactSet({ build: false, repoRoot }); +try { + const packageStates = artifactSet.packages.map((pkg) => ({ ...pkg, published: isPublished(pkg.name, pkg.version) })); + for (const pkg of packageStates) { + console.log(`${pkg.name}@${pkg.version} ${pkg.published ? "is already published" : "is ready to publish"}: ${pkg.tarballPath}`); } - validatePack(pkg.directory); - console.log(); -} - -if (dryRun) { - process.exit(0); -} - -console.log("All packages validated; starting publication.\n"); - -for (const pkg of packageStates) { - if (pkg.published) { - console.log(`Skipping ${pkg.name}@${pkg.version}: already published\n`); - continue; + console.log(dryRun ? "\nDry-running tarball publication with provenance.\n" : "\nAll packages packed; starting publication.\n"); + for (const pkg of packageStates) { + if (pkg.published) { + console.log(`Skipping ${pkg.name}@${pkg.version}: already published\n`); + continue; + } + const args = ["publish", pkg.tarballPath, "--access", "public", "--provenance", "--ignore-scripts"]; + if (dryRun) args.push("--dry-run"); + execNpmSync(args, { + cwd: packageByName.get(pkg.name).directory, + stdio: "inherit", + }); + console.log(); } - - run("npm", ["publish", "--access", "public", "--provenance", "--ignore-scripts"], { cwd: pkg.directory }); - console.log(); +} finally { + rmSync(artifactSet.artifactDirectory, { force: true, recursive: true }); } diff --git a/scripts/release-packages.mjs b/scripts/release-packages.mjs index fadc4fc08..cc1c77dfe 100644 --- a/scripts/release-packages.mjs +++ b/scripts/release-packages.mjs @@ -2,8 +2,8 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { findPackageDirectories } from "./package-workspaces.mjs"; -export function getPublicWorkspacePackages() { - return findPackageDirectories() +export function getPublicWorkspacePackages(root) { + return findPackageDirectories(root) .map((directory) => ({ directory, ...JSON.parse(readFileSync(join(directory, "package.json"), "utf8")), diff --git a/scripts/use-local-packages.mjs b/scripts/use-local-packages.mjs new file mode 100644 index 000000000..6641bc488 --- /dev/null +++ b/scripts/use-local-packages.mjs @@ -0,0 +1,41 @@ +#!/usr/bin/env node + +import { resolve } from "node:path"; +import { parseArgs } from "node:util"; +import { wireConsumer } from "./local-package-install.mjs"; +import { readArtifactSet } from "./package-artifacts.mjs"; + +function printUsage() { + console.log(`Usage: node scripts/use-local-packages.mjs --manifest --consumer --package [--package ...] [--package-manager npm|pnpm] + +Updates an external project's package configuration to use direct packages and +all transitive Pi packages from a local package artifact set. Defaults to npm. +`); +} + +const { values } = parseArgs({ + options: { + consumer: { type: "string" }, + help: { type: "boolean", default: false }, + manifest: { type: "string" }, + package: { type: "string", multiple: true, default: [] }, + "package-manager": { type: "string", default: "npm" }, + }, +}); +if (values.help) { + printUsage(); + process.exit(0); +} +if (!values.manifest) throw new Error("--manifest is required"); +if (!values.consumer) throw new Error("--consumer is required"); +if (values.package.length === 0) throw new Error("At least one --package is required"); +if (values["package-manager"] !== "npm" && values["package-manager"] !== "pnpm") { + throw new Error(`Unsupported package manager: ${values["package-manager"]}`); +} + +const artifactSet = readArtifactSet(resolve(values.manifest)); +const consumerDirectory = resolve(values.consumer); +wireConsumer({ artifactSet, consumerDirectory, packageManager: values["package-manager"], packageNames: values.package }); + +console.log(`Updated ${consumerDirectory}/package.json from ${artifactSet.manifestPath}`); +console.log(`Run ${values["package-manager"]} install --ignore-scripts in the consumer project.`);