diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index 792782066..d76e6f6e9 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -270,11 +270,51 @@ jobs: - name: Publish npm packages run: node scripts/publish.mjs + announce-pi-dev-release: + runs-on: ubuntu-latest + needs: publish-npm + environment: pi-model-upload + concurrency: + group: announce-pi-dev-release + cancel-in-progress: false + permissions: + contents: read + env: + RELEASE_TAG: ${{ github.event.inputs.tag || github.ref_name }} + SOURCE_REF: ${{ github.event.inputs.source_ref || github.event.inputs.tag || github.ref_name }} + AWS_ACCESS_KEY_ID: ${{ secrets.PI_ARTIFACTS_R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.PI_ARTIFACTS_R2_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: auto + AWS_EC2_METADATA_DISABLED: 'true' + R2_ENDPOINT: https://67c0d357268b0fca6e0b465bb9d01b84.r2.cloudflarestorage.com + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ env.SOURCE_REF }} + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22' + + - name: Verify AWS CLI + run: aws --version + + - name: Announce verified release on pi.dev + run: | + node scripts/publish-release-announcement.mjs \ + --version "${RELEASE_TAG#v}" \ + --bucket pi-artifacts \ + --endpoint "$R2_ENDPOINT" + publish-github-release: runs-on: ubuntu-latest needs: - stage-github-release - publish-npm + - announce-pi-dev-release permissions: contents: write env: @@ -305,8 +345,9 @@ jobs: - build - stage-github-release - publish-npm + - announce-pi-dev-release - publish-github-release - if: ${{ always() && needs.stage-github-release.result != 'skipped' && (needs.stage-github-release.result != 'success' || needs.publish-npm.result != 'success' || needs.publish-github-release.result != 'success') }} + if: ${{ always() && needs.stage-github-release.result != 'skipped' && (needs.stage-github-release.result != 'success' || needs.publish-npm.result != 'success' || needs.announce-pi-dev-release.result != 'success' || needs.publish-github-release.result != 'success') }} permissions: contents: write env: diff --git a/AGENTS.md b/AGENTS.md index 4c3a12d99..74b628083 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -154,9 +154,9 @@ Attribution: The release script bumps all package versions, updates changelogs, regenerates release artifacts, runs `npm run check`, commits `Release vX.Y.Z`, tags `vX.Y.Z`, adds fresh `## [Unreleased]` changelog sections, commits `Add [Unreleased] section for next cycle`, then pushes `main` and the tag. Do not rerun the release script after a tag was pushed. -4. **CI publishes npm packages**: pushing the `vX.Y.Z` tag triggers `.github/workflows/build-binaries.yml`. The `publish-npm` job uses npm trusted publishing through GitHub Actions OIDC with environment `npm-publish`; no local `npm publish`, `npm whoami`, OTP, or WebAuthn flow is required. +4. **CI verifies and announces the npm release**: pushing the `vX.Y.Z` tag triggers `.github/workflows/build-binaries.yml`. The `publish-npm` job uses npm trusted publishing through GitHub Actions OIDC with environment `npm-publish`; no local `npm publish`, `npm whoami`, OTP, or WebAuthn flow is required. After publishing, `announce-pi-dev-release` verifies every public workspace package resolves at the exact release version and that its npm tarball is available, then writes the verified release marker to R2. `pi.dev/api/latest-version` reads that marker; it must never announce a release from npm before this job succeeds. -5. **If CI publish fails**: inspect the failed `publish-npm` job. The publish helper is idempotent and skips package versions already present on npm, so rerun the tag workflow after fixing CI or transient npm issues. Do not rerun `npm run release:patch` or `npm run release:minor` for the same version. +5. **If CI publish or announcement fails**: inspect the failed job. The publish helper is idempotent and skips package versions already present on npm; the announcement job rechecks availability before updating the R2 marker. Rerun the failed job or workflow after fixing CI or transient npm issues. Do not rerun `npm run release:patch` or `npm run release:minor` for the same version. ## User Override diff --git a/scripts/publish-release-announcement.mjs b/scripts/publish-release-announcement.mjs new file mode 100644 index 000000000..238d868f8 --- /dev/null +++ b/scripts/publish-release-announcement.mjs @@ -0,0 +1,290 @@ +#!/usr/bin/env node + +import { execFileSync, spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { getPublicWorkspacePackages } from "./release-packages.mjs"; + +const RELEASES_PREFIX = "releases/v1"; +const REGISTRY_URL = "https://registry.npmjs.org"; +const RETRY_DELAY_MS = 5000; +const RETRY_TIMEOUT_MS = 10 * 60 * 1000; +const MAX_POINTER_UPDATE_ATTEMPTS = 5; +const STABLE_SEMVER_RE = /^(\d+)\.(\d+)\.(\d+)$/; + +function parseArgs(args) { + const options = { + bucket: undefined, + endpoint: undefined, + sourceCommit: undefined, + version: undefined, + }; + + for (let index = 0; index < args.length; index++) { + const arg = args[index]; + if (arg !== "--bucket" && arg !== "--endpoint" && arg !== "--source-commit" && arg !== "--version") { + throw new Error(`Unknown argument: ${arg}`); + } + const value = args[++index]; + if (!value) throw new Error(`${arg} requires a value`); + options[ + { "--bucket": "bucket", "--endpoint": "endpoint", "--source-commit": "sourceCommit", "--version": "version" }[arg] + ] = value; + } + + if (!options.bucket) throw new Error("--bucket is required"); + if (!options.endpoint) throw new Error("--endpoint is required"); + if (!options.version || !STABLE_SEMVER_RE.test(options.version)) { + throw new Error("--version must be a stable semver version"); + } + return options; +} + +function sleep(milliseconds) { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + +async function resolvePackageFromNpm(pkg) { + const packageUrl = `${REGISTRY_URL}/${encodeURIComponent(pkg.name)}/${pkg.version}`; + const response = await fetch(packageUrl, { headers: { accept: "application/json" } }); + if (!response.ok) { + throw new Error(`${pkg.name}@${pkg.version}: npm registry returned ${response.status}`); + } + + const data = await response.json(); + if ( + typeof data !== "object" || + data === null || + Array.isArray(data) || + data.version !== pkg.version || + typeof data.dist !== "object" || + data.dist === null || + Array.isArray(data.dist) || + typeof data.dist.tarball !== "string" || + typeof data.dist.integrity !== "string" + ) { + throw new Error(`${pkg.name}@${pkg.version}: npm registry returned invalid metadata`); + } + + const tarball = await fetch(data.dist.tarball, { method: "HEAD" }); + if (!tarball.ok) { + throw new Error(`${pkg.name}@${pkg.version}: tarball returned ${tarball.status}`); + } + + return { + name: pkg.name, + version: pkg.version, + tarball: data.dist.tarball, + integrity: data.dist.integrity, + }; +} + +async function verifyPackagesAreAvailable(packages) { + const deadline = Date.now() + RETRY_TIMEOUT_MS; + let attempt = 0; + let failures = []; + + do { + attempt++; + const results = await Promise.allSettled(packages.map(resolvePackageFromNpm)); + failures = results.flatMap((result, index) => + result.status === "rejected" ? [`${packages[index].name}: ${result.reason}`] : [], + ); + if (failures.length === 0) { + console.log(`All ${packages.length} Pi packages are available from npm (attempt ${attempt}).`); + return results.map((result) => result.value); + } + + console.log(`Waiting for ${failures.length} Pi package${failures.length === 1 ? "" : "s"} on npm (attempt ${attempt}):`); + for (const failure of failures) console.log(` ${failure}`); + if (Date.now() < deadline) await sleep(RETRY_DELAY_MS); + } while (Date.now() < deadline); + + throw new Error(`Timed out waiting for Pi packages to become available from npm:\n${failures.map((failure) => ` ${failure}`).join("\n")}`); +} + +function gitSourceCommit() { + return execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim(); +} + +function runAws(args, { allowNotFound = false, allowPreconditionFailure = false } = {}) { + const result = spawnSync("aws", args, { + encoding: "utf8", + env: { + ...process.env, + AWS_DEFAULT_REGION: process.env.AWS_DEFAULT_REGION || "auto", + AWS_EC2_METADATA_DISABLED: "true", + }, + stdio: ["ignore", "pipe", "pipe"], + }); + if (result.error) throw result.error; + if (result.status === 0) return result.stdout; + + const message = `${result.stdout}\n${result.stderr}`.trim(); + if (allowNotFound && /(?:404|NoSuchKey|Not Found)/i.test(message)) return undefined; + if (allowPreconditionFailure && /(?:412|PreconditionFailed|ConditionalRequestConflict)/i.test(message)) { + return undefined; + } + throw new Error(`aws ${args.slice(0, 2).join(" ")} failed:\n${message}`); +} + +function readLatestRelease(bucket, endpoint, outputPath) { + const head = runAws( + [ + "s3api", + "head-object", + "--bucket", + bucket, + "--key", + `${RELEASES_PREFIX}/latest.json`, + "--endpoint-url", + endpoint, + ], + { allowNotFound: true }, + ); + if (head === undefined) return undefined; + + const metadata = JSON.parse(head); + if (typeof metadata.ETag !== "string") { + throw new Error("Latest Pi release marker has no ETag."); + } + runAws([ + "s3api", + "get-object", + "--bucket", + bucket, + "--key", + `${RELEASES_PREFIX}/latest.json`, + "--endpoint-url", + endpoint, + outputPath, + ]); + const release = JSON.parse(readFileSync(outputPath, "utf8")); + if ( + typeof release !== "object" || + release === null || + Array.isArray(release) || + typeof release.version !== "string" || + !STABLE_SEMVER_RE.test(release.version) + ) { + throw new Error("Latest Pi release marker has an invalid version."); + } + return { etag: metadata.ETag, version: release.version }; +} + +function putJson(bucket, endpoint, path, key, cacheControl, condition) { + const args = [ + "s3api", + "put-object", + "--bucket", + bucket, + "--key", + key, + "--body", + path, + "--endpoint-url", + endpoint, + "--content-type", + "application/json; charset=utf-8", + "--cache-control", + cacheControl, + ]; + if (condition?.etag) args.push("--if-match", condition.etag); + if (condition?.missing) args.push("--if-none-match", "*"); + return runAws(args, { allowPreconditionFailure: Boolean(condition) }) !== undefined; +} + +export function compareReleaseVersions(left, right) { + const leftMatch = STABLE_SEMVER_RE.exec(left); + const rightMatch = STABLE_SEMVER_RE.exec(right); + if (!leftMatch || !rightMatch) throw new Error("Release versions must be stable semver versions."); + + for (const index of [1, 2, 3]) { + const difference = Number(leftMatch[index]) - Number(rightMatch[index]); + if (difference !== 0) return difference; + } + return 0; +} + +export async function advanceLatestRelease(version, readLatest, writeLatest) { + for (let attempt = 0; attempt < MAX_POINTER_UPDATE_ATTEMPTS; attempt++) { + const current = await readLatest(); + if (current && compareReleaseVersions(version, current.version) <= 0) { + return { advanced: false, version: current.version }; + } + + const updated = await writeLatest(current ? { etag: current.etag } : { missing: true }); + if (updated) { + return { advanced: true, version }; + } + } + throw new Error(`Could not advance the Pi release marker to ${version} after ${MAX_POINTER_UPDATE_ATTEMPTS} attempts.`); +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + const packages = getPublicWorkspacePackages(); + for (const pkg of packages) { + if (pkg.version !== options.version) { + throw new Error(`${pkg.name} is ${pkg.version}; expected ${options.version}`); + } + } + + const publishedPackages = await verifyPackagesAreAvailable(packages); + const release = { + schemaVersion: 1, + version: options.version, + sourceCommit: options.sourceCommit ?? gitSourceCommit(), + publishedAt: new Date().toISOString(), + packages: publishedPackages, + }; + const temporaryDirectory = mkdtempSync(join(tmpdir(), "pi-release-announcement-")); + try { + const releasePath = join(temporaryDirectory, "release.json"); + const latestPath = join(temporaryDirectory, "latest.json"); + writeFileSync(releasePath, `${JSON.stringify(release, null, "\t")}\n`); + if ( + !putJson( + options.bucket, + options.endpoint, + releasePath, + `${RELEASES_PREFIX}/releases/${options.version}.json`, + "public, max-age=31536000, immutable", + { missing: true }, + ) + ) { + console.log(`Release record ${options.version} already exists.`); + } + + writeFileSync(latestPath, `${JSON.stringify(release, null, "\t")}\n`); + const result = await advanceLatestRelease( + options.version, + () => readLatestRelease(options.bucket, options.endpoint, join(temporaryDirectory, "latest-current.json")), + (condition) => + putJson( + options.bucket, + options.endpoint, + latestPath, + `${RELEASES_PREFIX}/latest.json`, + "no-store", + condition, + ), + ); + console.log( + result.advanced + ? `Announced Pi ${options.version} through s3://${options.bucket}/${RELEASES_PREFIX}/latest.json` + : `Pi ${result.version} is already the latest announced release.`, + ); + } finally { + rmSync(temporaryDirectory, { force: true, recursive: true }); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +} diff --git a/scripts/publish-release-announcement.test.mjs b/scripts/publish-release-announcement.test.mjs new file mode 100644 index 000000000..1a51146dd --- /dev/null +++ b/scripts/publish-release-announcement.test.mjs @@ -0,0 +1,62 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { advanceLatestRelease, compareReleaseVersions } from "./publish-release-announcement.mjs"; + +test("compares stable release versions numerically", () => { + assert.ok(compareReleaseVersions("0.85.0", "0.84.9") > 0); + assert.ok(compareReleaseVersions("0.84.10", "0.84.9") > 0); + assert.equal(compareReleaseVersions("0.84.0", "0.84.0"), 0); + assert.throws(() => compareReleaseVersions("0.85.0-beta.1", "0.84.0")); +}); + +test("does not regress an existing newer release marker", async () => { + let writeCount = 0; + const result = await advanceLatestRelease( + "0.84.0", + async () => ({ etag: '"newer"', version: "0.85.0" }), + async () => { + writeCount++; + return true; + }, + ); + + assert.deepEqual(result, { advanced: false, version: "0.85.0" }); + assert.equal(writeCount, 0); +}); + +test("retries a lost conditional update and preserves a racing newer marker", async () => { + let readCount = 0; + let writeCount = 0; + const result = await advanceLatestRelease( + "0.84.0", + async () => { + readCount++; + return readCount === 1 + ? { etag: '"previous"', version: "0.83.0" } + : { etag: '"newer"', version: "0.85.0" }; + }, + async (condition) => { + writeCount++; + assert.deepEqual(condition, { etag: '"previous"' }); + return false; + }, + ); + + assert.deepEqual(result, { advanced: false, version: "0.85.0" }); + assert.equal(writeCount, 1); +}); + +test("creates a missing marker with an if-none-match condition", async () => { + let condition; + const result = await advanceLatestRelease( + "0.84.0", + async () => undefined, + async (value) => { + condition = value; + return true; + }, + ); + + assert.deepEqual(result, { advanced: true, version: "0.84.0" }); + assert.deepEqual(condition, { missing: true }); +}); diff --git a/scripts/publish.mjs b/scripts/publish.mjs index e680e25f9..0ce30944e 100644 --- a/scripts/publish.mjs +++ b/scripts/publish.mjs @@ -1,20 +1,11 @@ #!/usr/bin/env node import { spawnSync } from "node:child_process"; -import { existsSync, readFileSync } from "node:fs"; +import { existsSync } from "node:fs"; import { join } from "node:path"; +import { getPublicWorkspacePackages } from "./release-packages.mjs"; -const packages = [ - { directory: "packages/telemetry", name: "@earendil-works/pi-telemetry" }, - { directory: "packages/ai", name: "@earendil-works/pi-ai" }, - { directory: "packages/agent", name: "@earendil-works/pi-agent-core" }, - { directory: "packages/protocol", name: "@earendil-works/pi-protocol" }, - { directory: "packages/client", name: "@earendil-works/pi-client" }, - { directory: "packages/session-backends/sqlite-node", name: "@earendil-works/pi-session-backend-sqlite-node" }, - { directory: "packages/server", name: "@earendil-works/pi-server" }, - { directory: "packages/tui", name: "@earendil-works/pi-tui" }, - { directory: "packages/coding-agent", name: "@earendil-works/pi-coding-agent" }, -]; +const packages = getPublicWorkspacePackages(); const dryRun = process.argv.includes("--dry-run"); const unknownArgs = process.argv.slice(2).filter((arg) => arg !== "--dry-run"); @@ -44,10 +35,6 @@ function run(command, args, options = {}) { return result; } -function readPackageJson(directory) { - return JSON.parse(readFileSync(join(directory, "package.json"), "utf8")); -} - function assertBuildOutputExists(directory) { if (!existsSync(join(directory, "dist"))) { throw new Error(`${directory}/dist does not exist. Run npm run build before publishing.`); @@ -78,14 +65,7 @@ function isPublished(name, version) { throw new Error(output ? `Failed to query ${name}@${version}\n${output}` : `Failed to query ${name}@${version}`); } -const packageVersions = new Map(); -for (const pkg of packages) { - const packageJson = readPackageJson(pkg.directory); - if (packageJson.name !== pkg.name) { - throw new Error(`${pkg.directory}/package.json has name ${packageJson.name}, expected ${pkg.name}`); - } - packageVersions.set(pkg.name, packageJson.version); -} +const packageVersions = new Map(packages.map((pkg) => [pkg.name, pkg.version])); const versions = [...new Set(packageVersions.values())]; if (versions.length !== 1) { diff --git a/scripts/release-packages.mjs b/scripts/release-packages.mjs new file mode 100644 index 000000000..fadc4fc08 --- /dev/null +++ b/scripts/release-packages.mjs @@ -0,0 +1,13 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { findPackageDirectories } from "./package-workspaces.mjs"; + +export function getPublicWorkspacePackages() { + return findPackageDirectories() + .map((directory) => ({ + directory, + ...JSON.parse(readFileSync(join(directory, "package.json"), "utf8")), + })) + .filter((pkg) => pkg.private !== true) + .map(({ directory, name, version }) => ({ directory, name, version })); +} diff --git a/scripts/release.mjs b/scripts/release.mjs index 422c08a8a..02e1fd0fc 100755 --- a/scripts/release.mjs +++ b/scripts/release.mjs @@ -16,13 +16,14 @@ * 7. Commit and tag the release * 8. Add new [Unreleased] section to changelogs * 9. Commit next-cycle changelog updates - * 10. Push main and the tag to trigger CI publishing + * 10. Push main and the tag to trigger CI publication and verified pi.dev announcement */ import { execSync, spawnSync } from "node:child_process"; import { existsSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { findPackageDirectories } from "./package-workspaces.mjs"; +import { getPublicWorkspacePackages } from "./release-packages.mjs"; const RELEASE_TARGET = process.argv[2]; const BUMP_TYPES = new Set(["major", "minor", "patch"]); @@ -52,10 +53,7 @@ function getVersion() { } function assertPackagesAreRegisteredWithNpm() { - const packageNames = findPackageDirectories() - .map((directory) => JSON.parse(readFileSync(join(directory, "package.json"), "utf8"))) - .filter((pkg) => pkg.private !== true) - .map((pkg) => pkg.name); + const packageNames = getPublicWorkspacePackages().map((pkg) => pkg.name); const unregisteredPackages = []; console.log("Checking npm package registration..."); @@ -106,10 +104,7 @@ function shellQuote(value) { function removeStaleWorkspaceLockEntries() { const workspaceVersions = new Map( - findPackageDirectories() - .map((directory) => JSON.parse(readFileSync(join(directory, "package.json"), "utf8"))) - .filter((pkg) => pkg.private !== true) - .map((pkg) => [pkg.name, pkg.version]), + getPublicWorkspacePackages().map((pkg) => [pkg.name, pkg.version]), ); const lockPath = "package-lock.json"; const lock = JSON.parse(readFileSync(lockPath, "utf8")); @@ -283,4 +278,4 @@ run("git push origin main"); run(`git push origin v${version}`); console.log(); -console.log(`=== Prepared release v${version}; CI publishing starts after the tag push ===`); +console.log(`=== Prepared release v${version}; CI publication and pi.dev announcement start after the tag push ===`);