Set up preview packages (#11260)

* feat: publish preview packages to pkg.earendil.com

CI packs every public package after the build, with internal dependencies
pointing at the same commit's builds. A workflow_run job publishes them to
the earendil-pkg worker with the job's GitHub OIDC token, moves main or
pr-<n>, and comments install commands on the PR. Closing a PR deletes its
ref. pi-env is left out because its daemons are only built for releases.

* feat: publish preview packages only for main and same-repo PRs

pkg.earendil.com requires a consumer token, so fork authors could not
install their previews anyway. Fork PRs are no longer packed or
published, which keeps fork code out of the registry and removes the PR
lookup: the PR number now comes from the workflow_run event. The PR
comment is dropped; the job summary lists the install commands.

* fix: pick the PR into main and describe only the workflow in preview-packages.yml

- Read the PR number from the workflow_run pull request list with jq,
  selecting the PR into main; a branch can also have PRs into other
  branches, and the index expression tripped workflow linters.
- Comments describe what the workflow does and what it needs from
  pkg.earendil.com, not how the service works.

* fix: gate preview publishing to main pushes and open PRs into main

- Publish only CI runs for pushes to main and for pull requests. A branch
  can make its own copy of CI run on push; those runs are unreviewed and
  were uploaded as push builds.
- Move pr-<number> only while the PR is open into main with this commit
  as its head, checked live, so a late publish cannot recreate the ref of
  a closed PR; otherwise publish without a ref and print a notice.
- Set concurrency per job, so skipped fork runs never wait in a group.

* docs: drop the service rule from the preview-packages.yml header

* docs: name the earendil-works/pkg worker in the preview script
This commit is contained in:
Christian Klotz 2026-10-22 12:43:08 +03:00 • committed by GitHub
parent 2685de7a17
commit db774055e3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 449 additions and 0 deletions

View file

@ -35,6 +35,23 @@ jobs:
- name: Build
run: npm run build
# Previews are published for main and PRs from branches of this repo only (see preview-packages.yml).
- name: Pack preview packages
if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository
env:
# Pull request runs check out a merge commit; tarballs are keyed by the PR head commit.
SHA: ${{ github.event.pull_request.head.sha || github.sha }}
run: node scripts/preview-packages.mjs pack --sha "$SHA" --out "$RUNNER_TEMP/preview-packages"
- name: Upload preview packages
if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: preview-packages
path: ${{ runner.temp }}/preview-packages
retention-days: 3
if-no-files-found: error
- name: Check
run: npm run check

117
.github/workflows/preview-packages.yml vendored Normal file
View file

@ -0,0 +1,117 @@
# Publishes the preview packages that CI packed for main and for PRs from branches of this repo to pkg.earendil.com,
# moving the `main` or `pr-<number>` ref, and deletes a PR's ref when the PR closes. It reads the CI artifact as data
# and never checks out or runs PR code.
#
# Uploads are only accepted if pkg.earendil.com allows this repository, this workflow on main, and every package it
# publishes.
name: Preview packages
on:
workflow_run:
workflows: [CI]
types: [completed]
pull_request_target:
types: [closed]
permissions: {}
jobs:
publish:
# Only CI runs for pushes to main and for PRs from branches of this repo. A branch can make its own copy of CI run
# on push; such runs are not reviewed and must not be published as push builds.
if: >-
github.event_name == 'workflow_run' && github.event.workflow_run.conclusion != 'cancelled' &&
github.event.workflow_run.head_repository.full_name == github.repository &&
((github.event.workflow_run.event == 'push' && github.event.workflow_run.head_branch == 'main') ||
github.event.workflow_run.event == 'pull_request')
# Publish and cleanup runs for the same branch execute one at a time, in order. The group is set per job, so runs
# whose job is skipped (forks) never wait in it. `queue: max` keeps every waiting run instead of replacing one.
concurrency:
group: preview-packages-${{ github.event.workflow_run.head_branch }}
queue: max
runs-on: ubuntu-latest
permissions:
actions: read # download the artifact from the CI run
contents: read # check out scripts/
id-token: write # authenticate to pkg.earendil.com
pull-requests: read # check that the PR is still open
steps:
- name: Find packed packages
id: find
env:
GH_TOKEN: ${{ github.token }}
RUN_ID: ${{ github.event.workflow_run.id }}
run: |
count=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$RUN_ID/artifacts" \
--jq '[.artifacts[] | select(.name == "preview-packages")] | length')
if [ "$count" = 0 ]; then echo "::notice::CI produced no preview packages (it failed before packing)"; fi
echo "found=$([ "$count" != 0 ] && echo true || echo false)" >> "$GITHUB_OUTPUT"
- name: Download packed packages
if: steps.find.outputs.found == 'true'
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
name: preview-packages
path: ${{ runner.temp }}/preview-packages
run-id: ${{ github.event.workflow_run.id }}
github-token: ${{ github.token }}
- name: Check out scripts
if: steps.find.outputs.found == 'true'
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
sparse-checkout: scripts
persist-credentials: false
- name: Publish
if: steps.find.outputs.found == 'true'
env:
GH_TOKEN: ${{ github.token }}
# Event data is passed through env, never interpolated into the script: branch names are PR-controlled.
EVENT: ${{ github.event.workflow_run.event }}
HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}
PULL_REQUESTS: ${{ toJSON(github.event.workflow_run.pull_requests) }}
RUN_ID: ${{ github.event.workflow_run.id }}
DIR: ${{ runner.temp }}/preview-packages
run: |
args=(--dir "$DIR" --sha "$HEAD_SHA" --run "$RUN_ID" --event "$EVENT")
if [ "$EVENT" = push ] && [ "$HEAD_BRANCH" = main ]; then
args+=(--ref main)
else
# The branch's PR into main; a branch can also have PRs into other branches. It may have closed or received
# new commits since CI ran, so the ref moves only while the PR is open with this commit as its head.
pr=$(jq -r 'map(select(.base.ref == "main")) | .[0].number // empty' <<< "$PULL_REQUESTS")
state=$([ -z "$pr" ] || gh api "repos/$GITHUB_REPOSITORY/pulls/$pr" --jq '"\(.state) \(.base.ref) \(.head.sha)"')
if [ "$state" = "open main $HEAD_SHA" ]; then
args+=(--ref "pr-$pr")
else
echo "::notice::No open PR into main has head $HEAD_SHA; publishing without a pr-<number> ref"
fi
fi
node scripts/preview-packages.mjs publish "${args[@]}"
cleanup:
# Only PRs into main: pull_request_target runs this file from the PR's base branch, and only main is allowed.
if: >-
github.event_name == 'pull_request_target' && github.event.pull_request.base.ref == github.event.repository.default_branch &&
github.event.pull_request.head.repo.full_name == github.repository
concurrency:
group: preview-packages-${{ github.event.pull_request.head.ref }}
queue: max
runs-on: ubuntu-latest
permissions:
contents: read # check out scripts/ from the base branch, not PR code
id-token: write # authenticate to pkg.earendil.com
steps:
- name: Check out scripts
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
sparse-checkout: scripts
persist-credentials: false
- name: Delete PR ref
env:
PR: ${{ github.event.pull_request.number }}
run: node scripts/preview-packages.mjs delete-ref --ref "pr-$PR"

View file

@ -0,0 +1,188 @@
#!/usr/bin/env node
// Preview builds of pi's packages on pkg.earendil.com, served by the earendil-works/pkg worker.
// Used by .github/workflows/ci.yml (pack) and .github/workflows/preview-packages.yml (publish, delete-ref).
//
// pack Point internal dependencies at this commit's builds, then `npm pack` every public package.
// Runs in the regular CI job after the build.
// publish Upload the packed tarballs and move refs (main, pr-1234). Runs in the trusted workflow_run job and
// authenticates with the job's GitHub OIDC token (`permissions: id-token: write`).
// delete-ref Remove a ref, e.g. pr-1234 when the PR closes.
import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises";
import { basename, join, resolve } from "node:path";
import { parseArgs } from "node:util";
import { execNpmSync } from "./npm-command.mjs";
import { getPublicWorkspacePackages } from "./release-packages.mjs";
const BASE_URL = "https://pkg.earendil.com";
const FULL_SHA = /^[0-9a-f]{40}$/;
// Its daemon binaries are cross-compiled only by the release workflow; previews would ship without them.
const EXCLUDED = new Set(["@earendil-works/pi-env"]);
class UsageError extends Error {}
function assertSha(sha) {
if (typeof sha !== "string" || !FULL_SHA.test(sha)) {
throw new UsageError(`--sha must be a full commit SHA, got ${sha}`);
}
}
async function pack(argv) {
const { values } = parseArgs({
args: argv,
options: { sha: { type: "string" }, out: { type: "string", default: "preview-packages" } },
});
assertSha(values.sha);
const sha = values.sha;
const packages = getPublicWorkspacePackages().filter((pkg) => !EXCLUDED.has(pkg.name));
const names = new Set(packages.map((pkg) => pkg.name));
const outDir = resolve(values.out);
await mkdir(outDir, { recursive: true });
const packed = [];
for (const pkg of packages) {
const manifestPath = join(pkg.directory, "package.json");
const original = await readFile(manifestPath, "utf8");
const manifest = JSON.parse(original);
// Internal dependencies point at the same commit's builds, so one install resolves a consistent set.
for (const field of ["dependencies", "optionalDependencies"]) {
for (const name of Object.keys(manifest[field] ?? {})) {
if (names.has(name)) manifest[field][name] = `${BASE_URL}/${name}@${sha}`;
}
}
await writeFile(manifestPath, `${JSON.stringify(manifest, null, "\t")}\n`);
try {
const output = execNpmSync(["pack", "--json", "--ignore-scripts", "--pack-destination", outDir], {
cwd: pkg.directory,
encoding: "utf8",
stdio: ["inherit", "pipe", "inherit"],
maxBuffer: 64 * 1024 * 1024,
});
const [info] = JSON.parse(output);
packed.push({ name: info.name, file: info.filename });
console.error(`packed ${info.name}@${info.version} -> ${info.filename}`);
} finally {
await writeFile(manifestPath, original);
}
}
await writeFile(join(outDir, "manifest.json"), `${JSON.stringify({ sha, packages: packed }, null, "\t")}\n`);
}
// The audience names the production worker, also when --base-url points elsewhere (tests).
async function getOidcToken() {
const url = process.env.ACTIONS_ID_TOKEN_REQUEST_URL;
const token = process.env.ACTIONS_ID_TOKEN_REQUEST_TOKEN;
if (!url || !token) {
throw new UsageError("No GitHub OIDC token available; the job needs `permissions: id-token: write`");
}
const requestUrl = new URL(url);
requestUrl.searchParams.set("audience", BASE_URL);
const response = await fetch(requestUrl, { headers: { authorization: `Bearer ${token}` } });
if (!response.ok) throw new Error(`Requesting OIDC token failed: ${response.status}`);
const { value } = await response.json();
return value;
}
async function call(method, url, token, body) {
const response = await fetch(url, { method, headers: { authorization: `Bearer ${token}` }, body });
const text = await response.text();
if (!response.ok) throw new Error(`${method} ${url} failed: ${response.status} ${text}`);
return JSON.parse(text);
}
function renderSummary({ baseUrl, sha, refs, names }) {
const lines = ["### Preview packages", "", `Built from \`${sha.slice(0, 7)}\`.`, ""];
if (refs.length > 0) {
lines.push(`Latest build of \`${refs[0]}\` (updates with new pushes):`, "", "```sh");
for (const name of names) lines.push(`npm i ${baseUrl}/${name}@${refs[0]}`);
lines.push("```", "");
}
lines.push("<details><summary>Pinned to this commit</summary>", "", "```sh");
for (const name of names) lines.push(`npm i ${baseUrl}/${name}@${sha}`);
lines.push("```", "", "</details>", "");
lines.push(`Installs need a token in \`.npmrc\`: \`//${new URL(baseUrl).host}/:_authToken=...\``);
return `${lines.join("\n")}\n`;
}
async function publish(argv) {
const { values } = parseArgs({
args: argv,
options: {
dir: { type: "string", default: "preview-packages" },
sha: { type: "string" },
ref: { type: "string", multiple: true, default: [] },
run: { type: "string" },
event: { type: "string" },
"base-url": { type: "string", default: BASE_URL },
},
});
assertSha(values.sha);
const { sha, ref: refs, event } = values;
const run = Number(values.run);
if (!/^\d+$/.test(values.run ?? "") || !Number.isSafeInteger(run)) {
throw new UsageError(`--run must be the CI run ID, got ${values.run}`);
}
if (event !== "push" && event !== "pull_request") {
throw new UsageError(`--event must be push or pull_request, got ${event}`);
}
const baseUrl = values["base-url"];
const dir = resolve(values.dir);
// The artifact comes from an untrusted PR build: only read tarballs from it. The SHA, run, event, and refs
// come from the trusted workflow_run event; the worker authorizes package names against its config.
const manifest = JSON.parse(await readFile(join(dir, "manifest.json"), "utf8"));
if (manifest.sha !== sha) throw new Error(`Artifact was packed for ${manifest.sha}, expected ${sha}`);
if (!Array.isArray(manifest.packages) || manifest.packages.length === 0) throw new Error("Artifact has no packages");
const token = await getOidcToken();
const names = [];
for (const { name, file } of manifest.packages) {
if (typeof file !== "string" || basename(file) !== file || !file.endsWith(".tgz")) {
throw new Error(`Invalid tarball name in artifact: ${file}`);
}
const url = `${baseUrl}/_/upload/${sha}/${encodeURIComponent(name)}?event=${event}`;
const result = await call("PUT", url, token, await readFile(join(dir, file)));
names.push(result.name);
console.error(`${result.existing ? "exists" : "uploaded"} ${result.name}@${sha}`);
}
if (refs.length > 0) {
const result = await call("POST", `${baseUrl}/_/refs`, token, JSON.stringify({ sha, refs, run }));
const newer = result.refs.filter((ref) => ref.sha !== sha);
if (newer.length > 0) {
// A newer run already published these refs; its summary has the current install commands.
console.error(`a newer run already published ${newer.map((ref) => `${ref.ref} (${ref.sha})`).join(", ")}`);
return;
}
console.error(`moved ${refs.join(", ")} to ${sha}`);
}
const summary = renderSummary({ baseUrl, sha, refs, names: names.sort() });
process.stdout.write(summary);
if (process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, summary);
}
async function deleteRef(argv) {
const { values } = parseArgs({
args: argv,
options: { ref: { type: "string" }, "base-url": { type: "string", default: BASE_URL } },
});
if (!values.ref) throw new UsageError("delete-ref needs --ref");
await call("DELETE", `${values["base-url"]}/_/refs/${encodeURIComponent(values.ref)}`, await getOidcToken());
console.error(`deleted ${values.ref}`);
}
const USAGE = `Usage:
node scripts/preview-packages.mjs pack --sha <sha> [--out dir]
node scripts/preview-packages.mjs publish --sha <sha> --run <CI run id> --event <push|pull_request>
[--dir dir] [--ref name]...
node scripts/preview-packages.mjs delete-ref --ref <name>`;
const [command, ...argv] = process.argv.slice(2);
const commands = { pack, publish, "delete-ref": deleteRef };
try {
if (!commands[command]) throw new UsageError(USAGE);
await commands[command](argv);
} catch (error) {
console.error(error instanceof UsageError ? error.message : `preview-packages: ${error.message}`);
process.exit(1);
}

View file

@ -0,0 +1,127 @@
import assert from "node:assert/strict";
import { execFile, execFileSync } from "node:child_process";
import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { createServer } from "node:http";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import test from "node:test";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
const run = promisify(execFile);
const script = join(dirname(fileURLToPath(import.meta.url)), "preview-packages.mjs");
const sha = "a".repeat(40);
const newerSha = "b".repeat(40);
function temporaryDirectory(t) {
const directory = mkdtempSync(join(tmpdir(), "pi-preview-packages-test-"));
t.after(() => rmSync(directory, { recursive: true, force: true }));
return directory;
}
function writePackage(repo, directory, manifest) {
mkdirSync(join(repo, "packages", directory), { recursive: true });
writeFileSync(join(repo, "packages", directory, "package.json"), `${JSON.stringify(manifest, null, "\t")}\n`);
}
/**
* Stands in for the pkg.earendil.com worker and GitHub's OIDC token endpoint, recording requests. `refsSha` is what the
* worker reports each ref resolves to after the update; `uploadBody` overrides the upload response.
*/
async function startServer(t, { refsSha = sha, uploadBody } = {}) {
const requests = [];
const server = createServer(async (req, res) => {
let body = "";
for await (const chunk of req) body += chunk;
requests.push({ method: req.method, url: req.url, authorization: req.headers.authorization, body });
const url = new URL(req.url, "http://localhost");
if (url.pathname === "/token") return res.end(JSON.stringify({ value: `oidc-for-${url.searchParams.get("audience")}` }));
if (req.method === "PUT") {
const name = decodeURIComponent(url.pathname.split("/").slice(4).join("/"));
return res.end(uploadBody ?? JSON.stringify({ name, sha, existing: false }));
}
const { refs } = JSON.parse(body);
res.end(JSON.stringify({ refs: refs.map((ref) => ({ ref, sha: refsSha })) }));
});
await new Promise((ready) => server.listen(0, "127.0.0.1", ready));
t.after(() => server.close());
return { url: `http://127.0.0.1:${server.address().port}`, requests };
}
function writeArtifact(directory) {
mkdirSync(directory, { recursive: true });
writeFileSync(join(directory, "scope-a-1.0.0.tgz"), "tarball a");
const packages = [{ name: "@scope/a", file: "scope-a-1.0.0.tgz" }];
writeFileSync(join(directory, "manifest.json"), JSON.stringify({ sha, packages }));
}
function publish(server, dir) {
const { GITHUB_STEP_SUMMARY: _, ...env } = process.env;
const args = ["publish", "--sha", sha, "--run", "7", "--event", "pull_request", "--ref", "pr-1", "--dir", dir];
return run(process.execPath, [script, ...args, "--base-url", server.url], {
env: { ...env, ACTIONS_ID_TOKEN_REQUEST_URL: `${server.url}/token`, ACTIONS_ID_TOKEN_REQUEST_TOKEN: "runner" },
});
}
test("pack points internal dependencies at the commit's builds and restores package.json", async (t) => {
const repo = temporaryDirectory(t);
writePackage(repo, "a", { name: "@scope/a", version: "1.0.0", dependencies: { "@scope/b": "1.0.0", other: "2.0.0" } });
writePackage(repo, "b", { name: "@scope/b", version: "1.0.0" });
writePackage(repo, "internal", { name: "@scope/internal", version: "1.0.0", private: true });
writePackage(repo, "env", { name: "@earendil-works/pi-env", version: "1.0.0" });
const original = readFileSync(join(repo, "packages/a/package.json"), "utf8");
const out = join(repo, "out");
await run(process.execPath, [script, "pack", "--sha", sha, "--out", out], { cwd: repo });
const manifest = JSON.parse(readFileSync(join(out, "manifest.json"), "utf8"));
assert.equal(manifest.sha, sha);
assert.deepEqual(manifest.packages.map((pkg) => pkg.name).sort(), ["@scope/a", "@scope/b"]);
assert.deepEqual(readdirSync(out).sort(), ["manifest.json", ...manifest.packages.map((pkg) => pkg.file)].sort());
const fileA = manifest.packages.find((pkg) => pkg.name === "@scope/a").file;
const packedA = JSON.parse(execFileSync("tar", ["-xOzf", join(out, fileA), "package/package.json"], { encoding: "utf8" }));
assert.deepEqual(packedA.dependencies, { "@scope/b": `https://pkg.earendil.com/@scope/b@${sha}`, other: "2.0.0" });
assert.equal(readFileSync(join(repo, "packages/a/package.json"), "utf8"), original);
});
test("publish uploads each tarball, moves refs and prints install commands", async (t) => {
const server = await startServer(t);
const dir = join(temporaryDirectory(t), "artifact");
writeArtifact(dir);
const { stdout } = await publish(server, dir);
const [token, upload, refs] = server.requests;
assert.equal(token.url, "/token?audience=https%3A%2F%2Fpkg.earendil.com");
assert.deepEqual(
{ method: upload.method, url: upload.url, authorization: upload.authorization, body: upload.body },
{
method: "PUT",
url: `/_/upload/${sha}/%40scope%2Fa?event=pull_request`,
authorization: "Bearer oidc-for-https://pkg.earendil.com",
body: "tarball a",
},
);
assert.deepEqual(JSON.parse(refs.body), { sha, refs: ["pr-1"], run: 7 });
assert.match(stdout, new RegExp(`npm i ${server.url}/@scope/a@pr-1`));
assert.match(stdout, new RegExp(`npm i ${server.url}/@scope/a@${sha}`));
});
test("publish leaves the summary to a newer run that already owns the ref", async (t) => {
const server = await startServer(t, { refsSha: newerSha });
const dir = join(temporaryDirectory(t), "artifact");
writeArtifact(dir);
const { stdout, stderr } = await publish(server, dir);
assert.equal(stdout, "");
assert.match(stderr, new RegExp(`a newer run already published pr-1 \\(${newerSha}\\)`));
});
test("publish fails on a successful response that is not JSON", async (t) => {
const server = await startServer(t, { uploadBody: "<html>proxy</html>" });
const dir = join(temporaryDirectory(t), "artifact");
writeArtifact(dir);
await assert.rejects(publish(server, dir), /not valid JSON/);
assert.equal(server.requests.filter((request) => request.method === "POST").length, 0);
});