From db774055e3ecae05ab6eae8656454cde98e36780 Mon Sep 17 00:00:00 2001 From: Christian Klotz Date: Thu, 22 Oct 2026 12:43:08 +0300 Subject: [PATCH] Set up preview packages (#11260) * feat: publish preview packages to pkg.earendil.com CI packs every public package after the build, with internal dependencies pointing at the same commit's builds. A workflow_run job publishes them to the earendil-pkg worker with the job's GitHub OIDC token, moves main or pr-, and comments install commands on the PR. Closing a PR deletes its ref. pi-env is left out because its daemons are only built for releases. * feat: publish preview packages only for main and same-repo PRs pkg.earendil.com requires a consumer token, so fork authors could not install their previews anyway. Fork PRs are no longer packed or published, which keeps fork code out of the registry and removes the PR lookup: the PR number now comes from the workflow_run event. The PR comment is dropped; the job summary lists the install commands. * fix: pick the PR into main and describe only the workflow in preview-packages.yml - Read the PR number from the workflow_run pull request list with jq, selecting the PR into main; a branch can also have PRs into other branches, and the index expression tripped workflow linters. - Comments describe what the workflow does and what it needs from pkg.earendil.com, not how the service works. * fix: gate preview publishing to main pushes and open PRs into main - Publish only CI runs for pushes to main and for pull requests. A branch can make its own copy of CI run on push; those runs are unreviewed and were uploaded as push builds. - Move pr- only while the PR is open into main with this commit as its head, checked live, so a late publish cannot recreate the ref of a closed PR; otherwise publish without a ref and print a notice. - Set concurrency per job, so skipped fork runs never wait in a group. * docs: drop the service rule from the preview-packages.yml header * docs: name the earendil-works/pkg worker in the preview script --- .github/workflows/ci.yml | 17 +++ .github/workflows/preview-packages.yml | 117 +++++++++++++++ scripts/preview-packages.mjs | 188 +++++++++++++++++++++++++ scripts/preview-packages.test.mjs | 127 +++++++++++++++++ 4 files changed, 449 insertions(+) create mode 100644 .github/workflows/preview-packages.yml create mode 100644 scripts/preview-packages.mjs create mode 100644 scripts/preview-packages.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 302158278..a8c980e5a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,23 @@ jobs: - name: Build run: npm run build + # Previews are published for main and PRs from branches of this repo only (see preview-packages.yml). + - name: Pack preview packages + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository + env: + # Pull request runs check out a merge commit; tarballs are keyed by the PR head commit. + SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: node scripts/preview-packages.mjs pack --sha "$SHA" --out "$RUNNER_TEMP/preview-packages" + + - name: Upload preview packages + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: preview-packages + path: ${{ runner.temp }}/preview-packages + retention-days: 3 + if-no-files-found: error + - name: Check run: npm run check diff --git a/.github/workflows/preview-packages.yml b/.github/workflows/preview-packages.yml new file mode 100644 index 000000000..3b4121b79 --- /dev/null +++ b/.github/workflows/preview-packages.yml @@ -0,0 +1,117 @@ +# Publishes the preview packages that CI packed for main and for PRs from branches of this repo to pkg.earendil.com, +# moving the `main` or `pr-` ref, and deletes a PR's ref when the PR closes. It reads the CI artifact as data +# and never checks out or runs PR code. +# +# Uploads are only accepted if pkg.earendil.com allows this repository, this workflow on main, and every package it +# publishes. + +name: Preview packages + +on: + workflow_run: + workflows: [CI] + types: [completed] + pull_request_target: + types: [closed] + +permissions: {} + +jobs: + publish: + # Only CI runs for pushes to main and for PRs from branches of this repo. A branch can make its own copy of CI run + # on push; such runs are not reviewed and must not be published as push builds. + if: >- + github.event_name == 'workflow_run' && github.event.workflow_run.conclusion != 'cancelled' && + github.event.workflow_run.head_repository.full_name == github.repository && + ((github.event.workflow_run.event == 'push' && github.event.workflow_run.head_branch == 'main') || + github.event.workflow_run.event == 'pull_request') + # Publish and cleanup runs for the same branch execute one at a time, in order. The group is set per job, so runs + # whose job is skipped (forks) never wait in it. `queue: max` keeps every waiting run instead of replacing one. + concurrency: + group: preview-packages-${{ github.event.workflow_run.head_branch }} + queue: max + runs-on: ubuntu-latest + permissions: + actions: read # download the artifact from the CI run + contents: read # check out scripts/ + id-token: write # authenticate to pkg.earendil.com + pull-requests: read # check that the PR is still open + steps: + - name: Find packed packages + id: find + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ github.event.workflow_run.id }} + run: | + count=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$RUN_ID/artifacts" \ + --jq '[.artifacts[] | select(.name == "preview-packages")] | length') + if [ "$count" = 0 ]; then echo "::notice::CI produced no preview packages (it failed before packing)"; fi + echo "found=$([ "$count" != 0 ] && echo true || echo false)" >> "$GITHUB_OUTPUT" + + - name: Download packed packages + if: steps.find.outputs.found == 'true' + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: preview-packages + path: ${{ runner.temp }}/preview-packages + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + + - name: Check out scripts + if: steps.find.outputs.found == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: scripts + persist-credentials: false + + - name: Publish + if: steps.find.outputs.found == 'true' + env: + GH_TOKEN: ${{ github.token }} + # Event data is passed through env, never interpolated into the script: branch names are PR-controlled. + EVENT: ${{ github.event.workflow_run.event }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + PULL_REQUESTS: ${{ toJSON(github.event.workflow_run.pull_requests) }} + RUN_ID: ${{ github.event.workflow_run.id }} + DIR: ${{ runner.temp }}/preview-packages + run: | + args=(--dir "$DIR" --sha "$HEAD_SHA" --run "$RUN_ID" --event "$EVENT") + if [ "$EVENT" = push ] && [ "$HEAD_BRANCH" = main ]; then + args+=(--ref main) + else + # The branch's PR into main; a branch can also have PRs into other branches. It may have closed or received + # new commits since CI ran, so the ref moves only while the PR is open with this commit as its head. + pr=$(jq -r 'map(select(.base.ref == "main")) | .[0].number // empty' <<< "$PULL_REQUESTS") + state=$([ -z "$pr" ] || gh api "repos/$GITHUB_REPOSITORY/pulls/$pr" --jq '"\(.state) \(.base.ref) \(.head.sha)"') + if [ "$state" = "open main $HEAD_SHA" ]; then + args+=(--ref "pr-$pr") + else + echo "::notice::No open PR into main has head $HEAD_SHA; publishing without a pr- ref" + fi + fi + node scripts/preview-packages.mjs publish "${args[@]}" + + cleanup: + # Only PRs into main: pull_request_target runs this file from the PR's base branch, and only main is allowed. + if: >- + github.event_name == 'pull_request_target' && github.event.pull_request.base.ref == github.event.repository.default_branch && + github.event.pull_request.head.repo.full_name == github.repository + concurrency: + group: preview-packages-${{ github.event.pull_request.head.ref }} + queue: max + runs-on: ubuntu-latest + permissions: + contents: read # check out scripts/ from the base branch, not PR code + id-token: write # authenticate to pkg.earendil.com + steps: + - name: Check out scripts + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: scripts + persist-credentials: false + + - name: Delete PR ref + env: + PR: ${{ github.event.pull_request.number }} + run: node scripts/preview-packages.mjs delete-ref --ref "pr-$PR" diff --git a/scripts/preview-packages.mjs b/scripts/preview-packages.mjs new file mode 100644 index 000000000..bf18e9cf4 --- /dev/null +++ b/scripts/preview-packages.mjs @@ -0,0 +1,188 @@ +#!/usr/bin/env node +// Preview builds of pi's packages on pkg.earendil.com, served by the earendil-works/pkg worker. +// Used by .github/workflows/ci.yml (pack) and .github/workflows/preview-packages.yml (publish, delete-ref). +// +// pack Point internal dependencies at this commit's builds, then `npm pack` every public package. +// Runs in the regular CI job after the build. +// publish Upload the packed tarballs and move refs (main, pr-1234). Runs in the trusted workflow_run job and +// authenticates with the job's GitHub OIDC token (`permissions: id-token: write`). +// delete-ref Remove a ref, e.g. pr-1234 when the PR closes. + +import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises"; +import { basename, join, resolve } from "node:path"; +import { parseArgs } from "node:util"; +import { execNpmSync } from "./npm-command.mjs"; +import { getPublicWorkspacePackages } from "./release-packages.mjs"; + +const BASE_URL = "https://pkg.earendil.com"; +const FULL_SHA = /^[0-9a-f]{40}$/; +// Its daemon binaries are cross-compiled only by the release workflow; previews would ship without them. +const EXCLUDED = new Set(["@earendil-works/pi-env"]); + +class UsageError extends Error {} + +function assertSha(sha) { + if (typeof sha !== "string" || !FULL_SHA.test(sha)) { + throw new UsageError(`--sha must be a full commit SHA, got ${sha}`); + } +} + +async function pack(argv) { + const { values } = parseArgs({ + args: argv, + options: { sha: { type: "string" }, out: { type: "string", default: "preview-packages" } }, + }); + assertSha(values.sha); + const sha = values.sha; + const packages = getPublicWorkspacePackages().filter((pkg) => !EXCLUDED.has(pkg.name)); + const names = new Set(packages.map((pkg) => pkg.name)); + const outDir = resolve(values.out); + await mkdir(outDir, { recursive: true }); + + const packed = []; + for (const pkg of packages) { + const manifestPath = join(pkg.directory, "package.json"); + const original = await readFile(manifestPath, "utf8"); + const manifest = JSON.parse(original); + // Internal dependencies point at the same commit's builds, so one install resolves a consistent set. + for (const field of ["dependencies", "optionalDependencies"]) { + for (const name of Object.keys(manifest[field] ?? {})) { + if (names.has(name)) manifest[field][name] = `${BASE_URL}/${name}@${sha}`; + } + } + await writeFile(manifestPath, `${JSON.stringify(manifest, null, "\t")}\n`); + try { + const output = execNpmSync(["pack", "--json", "--ignore-scripts", "--pack-destination", outDir], { + cwd: pkg.directory, + encoding: "utf8", + stdio: ["inherit", "pipe", "inherit"], + maxBuffer: 64 * 1024 * 1024, + }); + const [info] = JSON.parse(output); + packed.push({ name: info.name, file: info.filename }); + console.error(`packed ${info.name}@${info.version} -> ${info.filename}`); + } finally { + await writeFile(manifestPath, original); + } + } + await writeFile(join(outDir, "manifest.json"), `${JSON.stringify({ sha, packages: packed }, null, "\t")}\n`); +} + +// The audience names the production worker, also when --base-url points elsewhere (tests). +async function getOidcToken() { + const url = process.env.ACTIONS_ID_TOKEN_REQUEST_URL; + const token = process.env.ACTIONS_ID_TOKEN_REQUEST_TOKEN; + if (!url || !token) { + throw new UsageError("No GitHub OIDC token available; the job needs `permissions: id-token: write`"); + } + const requestUrl = new URL(url); + requestUrl.searchParams.set("audience", BASE_URL); + const response = await fetch(requestUrl, { headers: { authorization: `Bearer ${token}` } }); + if (!response.ok) throw new Error(`Requesting OIDC token failed: ${response.status}`); + const { value } = await response.json(); + return value; +} + +async function call(method, url, token, body) { + const response = await fetch(url, { method, headers: { authorization: `Bearer ${token}` }, body }); + const text = await response.text(); + if (!response.ok) throw new Error(`${method} ${url} failed: ${response.status} ${text}`); + return JSON.parse(text); +} + +function renderSummary({ baseUrl, sha, refs, names }) { + const lines = ["### Preview packages", "", `Built from \`${sha.slice(0, 7)}\`.`, ""]; + if (refs.length > 0) { + lines.push(`Latest build of \`${refs[0]}\` (updates with new pushes):`, "", "```sh"); + for (const name of names) lines.push(`npm i ${baseUrl}/${name}@${refs[0]}`); + lines.push("```", ""); + } + lines.push("
Pinned to this commit", "", "```sh"); + for (const name of names) lines.push(`npm i ${baseUrl}/${name}@${sha}`); + lines.push("```", "", "
", ""); + lines.push(`Installs need a token in \`.npmrc\`: \`//${new URL(baseUrl).host}/:_authToken=...\``); + return `${lines.join("\n")}\n`; +} + +async function publish(argv) { + const { values } = parseArgs({ + args: argv, + options: { + dir: { type: "string", default: "preview-packages" }, + sha: { type: "string" }, + ref: { type: "string", multiple: true, default: [] }, + run: { type: "string" }, + event: { type: "string" }, + "base-url": { type: "string", default: BASE_URL }, + }, + }); + assertSha(values.sha); + const { sha, ref: refs, event } = values; + const run = Number(values.run); + if (!/^\d+$/.test(values.run ?? "") || !Number.isSafeInteger(run)) { + throw new UsageError(`--run must be the CI run ID, got ${values.run}`); + } + if (event !== "push" && event !== "pull_request") { + throw new UsageError(`--event must be push or pull_request, got ${event}`); + } + const baseUrl = values["base-url"]; + const dir = resolve(values.dir); + + // The artifact comes from an untrusted PR build: only read tarballs from it. The SHA, run, event, and refs + // come from the trusted workflow_run event; the worker authorizes package names against its config. + const manifest = JSON.parse(await readFile(join(dir, "manifest.json"), "utf8")); + if (manifest.sha !== sha) throw new Error(`Artifact was packed for ${manifest.sha}, expected ${sha}`); + if (!Array.isArray(manifest.packages) || manifest.packages.length === 0) throw new Error("Artifact has no packages"); + + const token = await getOidcToken(); + const names = []; + for (const { name, file } of manifest.packages) { + if (typeof file !== "string" || basename(file) !== file || !file.endsWith(".tgz")) { + throw new Error(`Invalid tarball name in artifact: ${file}`); + } + const url = `${baseUrl}/_/upload/${sha}/${encodeURIComponent(name)}?event=${event}`; + const result = await call("PUT", url, token, await readFile(join(dir, file))); + names.push(result.name); + console.error(`${result.existing ? "exists" : "uploaded"} ${result.name}@${sha}`); + } + if (refs.length > 0) { + const result = await call("POST", `${baseUrl}/_/refs`, token, JSON.stringify({ sha, refs, run })); + const newer = result.refs.filter((ref) => ref.sha !== sha); + if (newer.length > 0) { + // A newer run already published these refs; its summary has the current install commands. + console.error(`a newer run already published ${newer.map((ref) => `${ref.ref} (${ref.sha})`).join(", ")}`); + return; + } + console.error(`moved ${refs.join(", ")} to ${sha}`); + } + + const summary = renderSummary({ baseUrl, sha, refs, names: names.sort() }); + process.stdout.write(summary); + if (process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, summary); +} + +async function deleteRef(argv) { + const { values } = parseArgs({ + args: argv, + options: { ref: { type: "string" }, "base-url": { type: "string", default: BASE_URL } }, + }); + if (!values.ref) throw new UsageError("delete-ref needs --ref"); + await call("DELETE", `${values["base-url"]}/_/refs/${encodeURIComponent(values.ref)}`, await getOidcToken()); + console.error(`deleted ${values.ref}`); +} + +const USAGE = `Usage: + node scripts/preview-packages.mjs pack --sha [--out dir] + node scripts/preview-packages.mjs publish --sha --run --event + [--dir dir] [--ref name]... + node scripts/preview-packages.mjs delete-ref --ref `; + +const [command, ...argv] = process.argv.slice(2); +const commands = { pack, publish, "delete-ref": deleteRef }; +try { + if (!commands[command]) throw new UsageError(USAGE); + await commands[command](argv); +} catch (error) { + console.error(error instanceof UsageError ? error.message : `preview-packages: ${error.message}`); + process.exit(1); +} diff --git a/scripts/preview-packages.test.mjs b/scripts/preview-packages.test.mjs new file mode 100644 index 000000000..d287c0c49 --- /dev/null +++ b/scripts/preview-packages.test.mjs @@ -0,0 +1,127 @@ +import assert from "node:assert/strict"; +import { execFile, execFileSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +const run = promisify(execFile); +const script = join(dirname(fileURLToPath(import.meta.url)), "preview-packages.mjs"); +const sha = "a".repeat(40); +const newerSha = "b".repeat(40); + +function temporaryDirectory(t) { + const directory = mkdtempSync(join(tmpdir(), "pi-preview-packages-test-")); + t.after(() => rmSync(directory, { recursive: true, force: true })); + return directory; +} + +function writePackage(repo, directory, manifest) { + mkdirSync(join(repo, "packages", directory), { recursive: true }); + writeFileSync(join(repo, "packages", directory, "package.json"), `${JSON.stringify(manifest, null, "\t")}\n`); +} + +/** + * Stands in for the pkg.earendil.com worker and GitHub's OIDC token endpoint, recording requests. `refsSha` is what the + * worker reports each ref resolves to after the update; `uploadBody` overrides the upload response. + */ +async function startServer(t, { refsSha = sha, uploadBody } = {}) { + const requests = []; + const server = createServer(async (req, res) => { + let body = ""; + for await (const chunk of req) body += chunk; + requests.push({ method: req.method, url: req.url, authorization: req.headers.authorization, body }); + const url = new URL(req.url, "http://localhost"); + if (url.pathname === "/token") return res.end(JSON.stringify({ value: `oidc-for-${url.searchParams.get("audience")}` })); + if (req.method === "PUT") { + const name = decodeURIComponent(url.pathname.split("/").slice(4).join("/")); + return res.end(uploadBody ?? JSON.stringify({ name, sha, existing: false })); + } + const { refs } = JSON.parse(body); + res.end(JSON.stringify({ refs: refs.map((ref) => ({ ref, sha: refsSha })) })); + }); + await new Promise((ready) => server.listen(0, "127.0.0.1", ready)); + t.after(() => server.close()); + return { url: `http://127.0.0.1:${server.address().port}`, requests }; +} + +function writeArtifact(directory) { + mkdirSync(directory, { recursive: true }); + writeFileSync(join(directory, "scope-a-1.0.0.tgz"), "tarball a"); + const packages = [{ name: "@scope/a", file: "scope-a-1.0.0.tgz" }]; + writeFileSync(join(directory, "manifest.json"), JSON.stringify({ sha, packages })); +} + +function publish(server, dir) { + const { GITHUB_STEP_SUMMARY: _, ...env } = process.env; + const args = ["publish", "--sha", sha, "--run", "7", "--event", "pull_request", "--ref", "pr-1", "--dir", dir]; + return run(process.execPath, [script, ...args, "--base-url", server.url], { + env: { ...env, ACTIONS_ID_TOKEN_REQUEST_URL: `${server.url}/token`, ACTIONS_ID_TOKEN_REQUEST_TOKEN: "runner" }, + }); +} + +test("pack points internal dependencies at the commit's builds and restores package.json", async (t) => { + const repo = temporaryDirectory(t); + writePackage(repo, "a", { name: "@scope/a", version: "1.0.0", dependencies: { "@scope/b": "1.0.0", other: "2.0.0" } }); + writePackage(repo, "b", { name: "@scope/b", version: "1.0.0" }); + writePackage(repo, "internal", { name: "@scope/internal", version: "1.0.0", private: true }); + writePackage(repo, "env", { name: "@earendil-works/pi-env", version: "1.0.0" }); + const original = readFileSync(join(repo, "packages/a/package.json"), "utf8"); + + const out = join(repo, "out"); + await run(process.execPath, [script, "pack", "--sha", sha, "--out", out], { cwd: repo }); + + const manifest = JSON.parse(readFileSync(join(out, "manifest.json"), "utf8")); + assert.equal(manifest.sha, sha); + assert.deepEqual(manifest.packages.map((pkg) => pkg.name).sort(), ["@scope/a", "@scope/b"]); + assert.deepEqual(readdirSync(out).sort(), ["manifest.json", ...manifest.packages.map((pkg) => pkg.file)].sort()); + const fileA = manifest.packages.find((pkg) => pkg.name === "@scope/a").file; + const packedA = JSON.parse(execFileSync("tar", ["-xOzf", join(out, fileA), "package/package.json"], { encoding: "utf8" })); + assert.deepEqual(packedA.dependencies, { "@scope/b": `https://pkg.earendil.com/@scope/b@${sha}`, other: "2.0.0" }); + assert.equal(readFileSync(join(repo, "packages/a/package.json"), "utf8"), original); +}); + +test("publish uploads each tarball, moves refs and prints install commands", async (t) => { + const server = await startServer(t); + const dir = join(temporaryDirectory(t), "artifact"); + writeArtifact(dir); + + const { stdout } = await publish(server, dir); + + const [token, upload, refs] = server.requests; + assert.equal(token.url, "/token?audience=https%3A%2F%2Fpkg.earendil.com"); + assert.deepEqual( + { method: upload.method, url: upload.url, authorization: upload.authorization, body: upload.body }, + { + method: "PUT", + url: `/_/upload/${sha}/%40scope%2Fa?event=pull_request`, + authorization: "Bearer oidc-for-https://pkg.earendil.com", + body: "tarball a", + }, + ); + assert.deepEqual(JSON.parse(refs.body), { sha, refs: ["pr-1"], run: 7 }); + assert.match(stdout, new RegExp(`npm i ${server.url}/@scope/a@pr-1`)); + assert.match(stdout, new RegExp(`npm i ${server.url}/@scope/a@${sha}`)); +}); + +test("publish leaves the summary to a newer run that already owns the ref", async (t) => { + const server = await startServer(t, { refsSha: newerSha }); + const dir = join(temporaryDirectory(t), "artifact"); + writeArtifact(dir); + + const { stdout, stderr } = await publish(server, dir); + assert.equal(stdout, ""); + assert.match(stderr, new RegExp(`a newer run already published pr-1 \\(${newerSha}\\)`)); +}); + +test("publish fails on a successful response that is not JSON", async (t) => { + const server = await startServer(t, { uploadBody: "proxy" }); + const dir = join(temporaryDirectory(t), "artifact"); + writeArtifact(dir); + + await assert.rejects(publish(server, dir), /not valid JSON/); + assert.equal(server.requests.filter((request) => request.method === "POST").length, 0); +});