diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 302158278..a8c980e5a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,23 @@ jobs: - name: Build run: npm run build + # Previews are published for main and PRs from branches of this repo only (see preview-packages.yml). + - name: Pack preview packages + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository + env: + # Pull request runs check out a merge commit; tarballs are keyed by the PR head commit. + SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: node scripts/preview-packages.mjs pack --sha "$SHA" --out "$RUNNER_TEMP/preview-packages" + + - name: Upload preview packages + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: preview-packages + path: ${{ runner.temp }}/preview-packages + retention-days: 3 + if-no-files-found: error + - name: Check run: npm run check diff --git a/.github/workflows/preview-packages.yml b/.github/workflows/preview-packages.yml new file mode 100644 index 000000000..3b4121b79 --- /dev/null +++ b/.github/workflows/preview-packages.yml @@ -0,0 +1,117 @@ +# Publishes the preview packages that CI packed for main and for PRs from branches of this repo to pkg.earendil.com, +# moving the `main` or `pr-` ref, and deletes a PR's ref when the PR closes. It reads the CI artifact as data +# and never checks out or runs PR code. +# +# Uploads are only accepted if pkg.earendil.com allows this repository, this workflow on main, and every package it +# publishes. + +name: Preview packages + +on: + workflow_run: + workflows: [CI] + types: [completed] + pull_request_target: + types: [closed] + +permissions: {} + +jobs: + publish: + # Only CI runs for pushes to main and for PRs from branches of this repo. A branch can make its own copy of CI run + # on push; such runs are not reviewed and must not be published as push builds. + if: >- + github.event_name == 'workflow_run' && github.event.workflow_run.conclusion != 'cancelled' && + github.event.workflow_run.head_repository.full_name == github.repository && + ((github.event.workflow_run.event == 'push' && github.event.workflow_run.head_branch == 'main') || + github.event.workflow_run.event == 'pull_request') + # Publish and cleanup runs for the same branch execute one at a time, in order. The group is set per job, so runs + # whose job is skipped (forks) never wait in it. `queue: max` keeps every waiting run instead of replacing one. + concurrency: + group: preview-packages-${{ github.event.workflow_run.head_branch }} + queue: max + runs-on: ubuntu-latest + permissions: + actions: read # download the artifact from the CI run + contents: read # check out scripts/ + id-token: write # authenticate to pkg.earendil.com + pull-requests: read # check that the PR is still open + steps: + - name: Find packed packages + id: find + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ github.event.workflow_run.id }} + run: | + count=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$RUN_ID/artifacts" \ + --jq '[.artifacts[] | select(.name == "preview-packages")] | length') + if [ "$count" = 0 ]; then echo "::notice::CI produced no preview packages (it failed before packing)"; fi + echo "found=$([ "$count" != 0 ] && echo true || echo false)" >> "$GITHUB_OUTPUT" + + - name: Download packed packages + if: steps.find.outputs.found == 'true' + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: preview-packages + path: ${{ runner.temp }}/preview-packages + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + + - name: Check out scripts + if: steps.find.outputs.found == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: scripts + persist-credentials: false + + - name: Publish + if: steps.find.outputs.found == 'true' + env: + GH_TOKEN: ${{ github.token }} + # Event data is passed through env, never interpolated into the script: branch names are PR-controlled. + EVENT: ${{ github.event.workflow_run.event }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + PULL_REQUESTS: ${{ toJSON(github.event.workflow_run.pull_requests) }} + RUN_ID: ${{ github.event.workflow_run.id }} + DIR: ${{ runner.temp }}/preview-packages + run: | + args=(--dir "$DIR" --sha "$HEAD_SHA" --run "$RUN_ID" --event "$EVENT") + if [ "$EVENT" = push ] && [ "$HEAD_BRANCH" = main ]; then + args+=(--ref main) + else + # The branch's PR into main; a branch can also have PRs into other branches. It may have closed or received + # new commits since CI ran, so the ref moves only while the PR is open with this commit as its head. + pr=$(jq -r 'map(select(.base.ref == "main")) | .[0].number // empty' <<< "$PULL_REQUESTS") + state=$([ -z "$pr" ] || gh api "repos/$GITHUB_REPOSITORY/pulls/$pr" --jq '"\(.state) \(.base.ref) \(.head.sha)"') + if [ "$state" = "open main $HEAD_SHA" ]; then + args+=(--ref "pr-$pr") + else + echo "::notice::No open PR into main has head $HEAD_SHA; publishing without a pr- ref" + fi + fi + node scripts/preview-packages.mjs publish "${args[@]}" + + cleanup: + # Only PRs into main: pull_request_target runs this file from the PR's base branch, and only main is allowed. + if: >- + github.event_name == 'pull_request_target' && github.event.pull_request.base.ref == github.event.repository.default_branch && + github.event.pull_request.head.repo.full_name == github.repository + concurrency: + group: preview-packages-${{ github.event.pull_request.head.ref }} + queue: max + runs-on: ubuntu-latest + permissions: + contents: read # check out scripts/ from the base branch, not PR code + id-token: write # authenticate to pkg.earendil.com + steps: + - name: Check out scripts + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: scripts + persist-credentials: false + + - name: Delete PR ref + env: + PR: ${{ github.event.pull_request.number }} + run: node scripts/preview-packages.mjs delete-ref --ref "pr-$PR" diff --git a/scripts/preview-packages.mjs b/scripts/preview-packages.mjs new file mode 100644 index 000000000..bf18e9cf4 --- /dev/null +++ b/scripts/preview-packages.mjs @@ -0,0 +1,188 @@ +#!/usr/bin/env node +// Preview builds of pi's packages on pkg.earendil.com, served by the earendil-works/pkg worker. +// Used by .github/workflows/ci.yml (pack) and .github/workflows/preview-packages.yml (publish, delete-ref). +// +// pack Point internal dependencies at this commit's builds, then `npm pack` every public package. +// Runs in the regular CI job after the build. +// publish Upload the packed tarballs and move refs (main, pr-1234). Runs in the trusted workflow_run job and +// authenticates with the job's GitHub OIDC token (`permissions: id-token: write`). +// delete-ref Remove a ref, e.g. pr-1234 when the PR closes. + +import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises"; +import { basename, join, resolve } from "node:path"; +import { parseArgs } from "node:util"; +import { execNpmSync } from "./npm-command.mjs"; +import { getPublicWorkspacePackages } from "./release-packages.mjs"; + +const BASE_URL = "https://pkg.earendil.com"; +const FULL_SHA = /^[0-9a-f]{40}$/; +// Its daemon binaries are cross-compiled only by the release workflow; previews would ship without them. +const EXCLUDED = new Set(["@earendil-works/pi-env"]); + +class UsageError extends Error {} + +function assertSha(sha) { + if (typeof sha !== "string" || !FULL_SHA.test(sha)) { + throw new UsageError(`--sha must be a full commit SHA, got ${sha}`); + } +} + +async function pack(argv) { + const { values } = parseArgs({ + args: argv, + options: { sha: { type: "string" }, out: { type: "string", default: "preview-packages" } }, + }); + assertSha(values.sha); + const sha = values.sha; + const packages = getPublicWorkspacePackages().filter((pkg) => !EXCLUDED.has(pkg.name)); + const names = new Set(packages.map((pkg) => pkg.name)); + const outDir = resolve(values.out); + await mkdir(outDir, { recursive: true }); + + const packed = []; + for (const pkg of packages) { + const manifestPath = join(pkg.directory, "package.json"); + const original = await readFile(manifestPath, "utf8"); + const manifest = JSON.parse(original); + // Internal dependencies point at the same commit's builds, so one install resolves a consistent set. + for (const field of ["dependencies", "optionalDependencies"]) { + for (const name of Object.keys(manifest[field] ?? {})) { + if (names.has(name)) manifest[field][name] = `${BASE_URL}/${name}@${sha}`; + } + } + await writeFile(manifestPath, `${JSON.stringify(manifest, null, "\t")}\n`); + try { + const output = execNpmSync(["pack", "--json", "--ignore-scripts", "--pack-destination", outDir], { + cwd: pkg.directory, + encoding: "utf8", + stdio: ["inherit", "pipe", "inherit"], + maxBuffer: 64 * 1024 * 1024, + }); + const [info] = JSON.parse(output); + packed.push({ name: info.name, file: info.filename }); + console.error(`packed ${info.name}@${info.version} -> ${info.filename}`); + } finally { + await writeFile(manifestPath, original); + } + } + await writeFile(join(outDir, "manifest.json"), `${JSON.stringify({ sha, packages: packed }, null, "\t")}\n`); +} + +// The audience names the production worker, also when --base-url points elsewhere (tests). +async function getOidcToken() { + const url = process.env.ACTIONS_ID_TOKEN_REQUEST_URL; + const token = process.env.ACTIONS_ID_TOKEN_REQUEST_TOKEN; + if (!url || !token) { + throw new UsageError("No GitHub OIDC token available; the job needs `permissions: id-token: write`"); + } + const requestUrl = new URL(url); + requestUrl.searchParams.set("audience", BASE_URL); + const response = await fetch(requestUrl, { headers: { authorization: `Bearer ${token}` } }); + if (!response.ok) throw new Error(`Requesting OIDC token failed: ${response.status}`); + const { value } = await response.json(); + return value; +} + +async function call(method, url, token, body) { + const response = await fetch(url, { method, headers: { authorization: `Bearer ${token}` }, body }); + const text = await response.text(); + if (!response.ok) throw new Error(`${method} ${url} failed: ${response.status} ${text}`); + return JSON.parse(text); +} + +function renderSummary({ baseUrl, sha, refs, names }) { + const lines = ["### Preview packages", "", `Built from \`${sha.slice(0, 7)}\`.`, ""]; + if (refs.length > 0) { + lines.push(`Latest build of \`${refs[0]}\` (updates with new pushes):`, "", "```sh"); + for (const name of names) lines.push(`npm i ${baseUrl}/${name}@${refs[0]}`); + lines.push("```", ""); + } + lines.push("
Pinned to this commit", "", "```sh"); + for (const name of names) lines.push(`npm i ${baseUrl}/${name}@${sha}`); + lines.push("```", "", "
", ""); + lines.push(`Installs need a token in \`.npmrc\`: \`//${new URL(baseUrl).host}/:_authToken=...\``); + return `${lines.join("\n")}\n`; +} + +async function publish(argv) { + const { values } = parseArgs({ + args: argv, + options: { + dir: { type: "string", default: "preview-packages" }, + sha: { type: "string" }, + ref: { type: "string", multiple: true, default: [] }, + run: { type: "string" }, + event: { type: "string" }, + "base-url": { type: "string", default: BASE_URL }, + }, + }); + assertSha(values.sha); + const { sha, ref: refs, event } = values; + const run = Number(values.run); + if (!/^\d+$/.test(values.run ?? "") || !Number.isSafeInteger(run)) { + throw new UsageError(`--run must be the CI run ID, got ${values.run}`); + } + if (event !== "push" && event !== "pull_request") { + throw new UsageError(`--event must be push or pull_request, got ${event}`); + } + const baseUrl = values["base-url"]; + const dir = resolve(values.dir); + + // The artifact comes from an untrusted PR build: only read tarballs from it. The SHA, run, event, and refs + // come from the trusted workflow_run event; the worker authorizes package names against its config. + const manifest = JSON.parse(await readFile(join(dir, "manifest.json"), "utf8")); + if (manifest.sha !== sha) throw new Error(`Artifact was packed for ${manifest.sha}, expected ${sha}`); + if (!Array.isArray(manifest.packages) || manifest.packages.length === 0) throw new Error("Artifact has no packages"); + + const token = await getOidcToken(); + const names = []; + for (const { name, file } of manifest.packages) { + if (typeof file !== "string" || basename(file) !== file || !file.endsWith(".tgz")) { + throw new Error(`Invalid tarball name in artifact: ${file}`); + } + const url = `${baseUrl}/_/upload/${sha}/${encodeURIComponent(name)}?event=${event}`; + const result = await call("PUT", url, token, await readFile(join(dir, file))); + names.push(result.name); + console.error(`${result.existing ? "exists" : "uploaded"} ${result.name}@${sha}`); + } + if (refs.length > 0) { + const result = await call("POST", `${baseUrl}/_/refs`, token, JSON.stringify({ sha, refs, run })); + const newer = result.refs.filter((ref) => ref.sha !== sha); + if (newer.length > 0) { + // A newer run already published these refs; its summary has the current install commands. + console.error(`a newer run already published ${newer.map((ref) => `${ref.ref} (${ref.sha})`).join(", ")}`); + return; + } + console.error(`moved ${refs.join(", ")} to ${sha}`); + } + + const summary = renderSummary({ baseUrl, sha, refs, names: names.sort() }); + process.stdout.write(summary); + if (process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, summary); +} + +async function deleteRef(argv) { + const { values } = parseArgs({ + args: argv, + options: { ref: { type: "string" }, "base-url": { type: "string", default: BASE_URL } }, + }); + if (!values.ref) throw new UsageError("delete-ref needs --ref"); + await call("DELETE", `${values["base-url"]}/_/refs/${encodeURIComponent(values.ref)}`, await getOidcToken()); + console.error(`deleted ${values.ref}`); +} + +const USAGE = `Usage: + node scripts/preview-packages.mjs pack --sha [--out dir] + node scripts/preview-packages.mjs publish --sha --run --event + [--dir dir] [--ref name]... + node scripts/preview-packages.mjs delete-ref --ref `; + +const [command, ...argv] = process.argv.slice(2); +const commands = { pack, publish, "delete-ref": deleteRef }; +try { + if (!commands[command]) throw new UsageError(USAGE); + await commands[command](argv); +} catch (error) { + console.error(error instanceof UsageError ? error.message : `preview-packages: ${error.message}`); + process.exit(1); +} diff --git a/scripts/preview-packages.test.mjs b/scripts/preview-packages.test.mjs new file mode 100644 index 000000000..d287c0c49 --- /dev/null +++ b/scripts/preview-packages.test.mjs @@ -0,0 +1,127 @@ +import assert from "node:assert/strict"; +import { execFile, execFileSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +const run = promisify(execFile); +const script = join(dirname(fileURLToPath(import.meta.url)), "preview-packages.mjs"); +const sha = "a".repeat(40); +const newerSha = "b".repeat(40); + +function temporaryDirectory(t) { + const directory = mkdtempSync(join(tmpdir(), "pi-preview-packages-test-")); + t.after(() => rmSync(directory, { recursive: true, force: true })); + return directory; +} + +function writePackage(repo, directory, manifest) { + mkdirSync(join(repo, "packages", directory), { recursive: true }); + writeFileSync(join(repo, "packages", directory, "package.json"), `${JSON.stringify(manifest, null, "\t")}\n`); +} + +/** + * Stands in for the pkg.earendil.com worker and GitHub's OIDC token endpoint, recording requests. `refsSha` is what the + * worker reports each ref resolves to after the update; `uploadBody` overrides the upload response. + */ +async function startServer(t, { refsSha = sha, uploadBody } = {}) { + const requests = []; + const server = createServer(async (req, res) => { + let body = ""; + for await (const chunk of req) body += chunk; + requests.push({ method: req.method, url: req.url, authorization: req.headers.authorization, body }); + const url = new URL(req.url, "http://localhost"); + if (url.pathname === "/token") return res.end(JSON.stringify({ value: `oidc-for-${url.searchParams.get("audience")}` })); + if (req.method === "PUT") { + const name = decodeURIComponent(url.pathname.split("/").slice(4).join("/")); + return res.end(uploadBody ?? JSON.stringify({ name, sha, existing: false })); + } + const { refs } = JSON.parse(body); + res.end(JSON.stringify({ refs: refs.map((ref) => ({ ref, sha: refsSha })) })); + }); + await new Promise((ready) => server.listen(0, "127.0.0.1", ready)); + t.after(() => server.close()); + return { url: `http://127.0.0.1:${server.address().port}`, requests }; +} + +function writeArtifact(directory) { + mkdirSync(directory, { recursive: true }); + writeFileSync(join(directory, "scope-a-1.0.0.tgz"), "tarball a"); + const packages = [{ name: "@scope/a", file: "scope-a-1.0.0.tgz" }]; + writeFileSync(join(directory, "manifest.json"), JSON.stringify({ sha, packages })); +} + +function publish(server, dir) { + const { GITHUB_STEP_SUMMARY: _, ...env } = process.env; + const args = ["publish", "--sha", sha, "--run", "7", "--event", "pull_request", "--ref", "pr-1", "--dir", dir]; + return run(process.execPath, [script, ...args, "--base-url", server.url], { + env: { ...env, ACTIONS_ID_TOKEN_REQUEST_URL: `${server.url}/token`, ACTIONS_ID_TOKEN_REQUEST_TOKEN: "runner" }, + }); +} + +test("pack points internal dependencies at the commit's builds and restores package.json", async (t) => { + const repo = temporaryDirectory(t); + writePackage(repo, "a", { name: "@scope/a", version: "1.0.0", dependencies: { "@scope/b": "1.0.0", other: "2.0.0" } }); + writePackage(repo, "b", { name: "@scope/b", version: "1.0.0" }); + writePackage(repo, "internal", { name: "@scope/internal", version: "1.0.0", private: true }); + writePackage(repo, "env", { name: "@earendil-works/pi-env", version: "1.0.0" }); + const original = readFileSync(join(repo, "packages/a/package.json"), "utf8"); + + const out = join(repo, "out"); + await run(process.execPath, [script, "pack", "--sha", sha, "--out", out], { cwd: repo }); + + const manifest = JSON.parse(readFileSync(join(out, "manifest.json"), "utf8")); + assert.equal(manifest.sha, sha); + assert.deepEqual(manifest.packages.map((pkg) => pkg.name).sort(), ["@scope/a", "@scope/b"]); + assert.deepEqual(readdirSync(out).sort(), ["manifest.json", ...manifest.packages.map((pkg) => pkg.file)].sort()); + const fileA = manifest.packages.find((pkg) => pkg.name === "@scope/a").file; + const packedA = JSON.parse(execFileSync("tar", ["-xOzf", join(out, fileA), "package/package.json"], { encoding: "utf8" })); + assert.deepEqual(packedA.dependencies, { "@scope/b": `https://pkg.earendil.com/@scope/b@${sha}`, other: "2.0.0" }); + assert.equal(readFileSync(join(repo, "packages/a/package.json"), "utf8"), original); +}); + +test("publish uploads each tarball, moves refs and prints install commands", async (t) => { + const server = await startServer(t); + const dir = join(temporaryDirectory(t), "artifact"); + writeArtifact(dir); + + const { stdout } = await publish(server, dir); + + const [token, upload, refs] = server.requests; + assert.equal(token.url, "/token?audience=https%3A%2F%2Fpkg.earendil.com"); + assert.deepEqual( + { method: upload.method, url: upload.url, authorization: upload.authorization, body: upload.body }, + { + method: "PUT", + url: `/_/upload/${sha}/%40scope%2Fa?event=pull_request`, + authorization: "Bearer oidc-for-https://pkg.earendil.com", + body: "tarball a", + }, + ); + assert.deepEqual(JSON.parse(refs.body), { sha, refs: ["pr-1"], run: 7 }); + assert.match(stdout, new RegExp(`npm i ${server.url}/@scope/a@pr-1`)); + assert.match(stdout, new RegExp(`npm i ${server.url}/@scope/a@${sha}`)); +}); + +test("publish leaves the summary to a newer run that already owns the ref", async (t) => { + const server = await startServer(t, { refsSha: newerSha }); + const dir = join(temporaryDirectory(t), "artifact"); + writeArtifact(dir); + + const { stdout, stderr } = await publish(server, dir); + assert.equal(stdout, ""); + assert.match(stderr, new RegExp(`a newer run already published pr-1 \\(${newerSha}\\)`)); +}); + +test("publish fails on a successful response that is not JSON", async (t) => { + const server = await startServer(t, { uploadBody: "proxy" }); + const dir = join(temporaryDirectory(t), "artifact"); + writeArtifact(dir); + + await assert.rejects(publish(server, dir), /not valid JSON/); + assert.equal(server.requests.filter((request) => request.method === "POST").length, 0); +});