sscoble/pi-mono
1
0
Fork 0
mirror of https://github.com/badlogic/pi-mono.git synced 2026-11-05 23:37:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Set up preview packages (#11260)

Browse source 
* feat: publish preview packages to pkg.earendil.com

CI packs every public package after the build, with internal dependencies
pointing at the same commit's builds. A workflow_run job publishes them to
the earendil-pkg worker with the job's GitHub OIDC token, moves main or
pr-<n>, and comments install commands on the PR. Closing a PR deletes its
ref. pi-env is left out because its daemons are only built for releases.

* feat: publish preview packages only for main and same-repo PRs

pkg.earendil.com requires a consumer token, so fork authors could not
install their previews anyway. Fork PRs are no longer packed or
published, which keeps fork code out of the registry and removes the PR
lookup: the PR number now comes from the workflow_run event. The PR
comment is dropped; the job summary lists the install commands.

* fix: pick the PR into main and describe only the workflow in preview-packages.yml

- Read the PR number from the workflow_run pull request list with jq,
  selecting the PR into main; a branch can also have PRs into other
  branches, and the index expression tripped workflow linters.
- Comments describe what the workflow does and what it needs from
  pkg.earendil.com, not how the service works.

* fix: gate preview publishing to main pushes and open PRs into main

- Publish only CI runs for pushes to main and for pull requests. A branch
  can make its own copy of CI run on push; those runs are unreviewed and
  were uploaded as push builds.
- Move pr-<number> only while the PR is open into main with this commit
  as its head, checked live, so a late publish cannot recreate the ref of
  a closed PR; otherwise publish without a ref and print a notice.
- Set concurrency per job, so skipped fork runs never wait in a group.

* docs: drop the service rule from the preview-packages.yml header

* docs: name the earendil-works/pkg worker in the preview script
This commit is contained in:
Christian Klotz 2026-10-22 12:43:08 +03:00 • committed by GitHub
parent 2685de7a17
commit db774055e3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 449 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

188
scripts/preview-packages.mjs Normal file
View file

@ -0,0 +1,188 @@
#!/usr/bin/env node
// Preview builds of pi's packages on pkg.earendil.com, served by the earendil-works/pkg worker.
// Used by .github/workflows/ci.yml (pack) and .github/workflows/preview-packages.yml (publish, delete-ref).
//
// pack Point internal dependencies at this commit's builds, then `npm pack` every public package.
// Runs in the regular CI job after the build.
// publish Upload the packed tarballs and move refs (main, pr-1234). Runs in the trusted workflow_run job and
// authenticates with the job's GitHub OIDC token (`permissions: id-token: write`).
// delete-ref Remove a ref, e.g. pr-1234 when the PR closes.
import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises";
import { basename, join, resolve } from "node:path";
import { parseArgs } from "node:util";
import { execNpmSync } from "./npm-command.mjs";
import { getPublicWorkspacePackages } from "./release-packages.mjs";
const BASE_URL = "https://pkg.earendil.com";
const FULL_SHA = /^[0-9a-f]{40}$/;
// Its daemon binaries are cross-compiled only by the release workflow; previews would ship without them.
const EXCLUDED = new Set(["@earendil-works/pi-env"]);
class UsageError extends Error {}
function assertSha(sha) {
if (typeof sha !== "string" || !FULL_SHA.test(sha)) {
throw new UsageError(`--sha must be a full commit SHA, got ${sha}`);
}
}
async function pack(argv) {
const { values } = parseArgs({
args: argv,
options: { sha: { type: "string" }, out: { type: "string", default: "preview-packages" } },
});
assertSha(values.sha);
const sha = values.sha;
const packages = getPublicWorkspacePackages().filter((pkg) => !EXCLUDED.has(pkg.name));
const names = new Set(packages.map((pkg) => pkg.name));
const outDir = resolve(values.out);
await mkdir(outDir, { recursive: true });
const packed = [];
for (const pkg of packages) {
const manifestPath = join(pkg.directory, "package.json");
const original = await readFile(manifestPath, "utf8");
const manifest = JSON.parse(original);
// Internal dependencies point at the same commit's builds, so one install resolves a consistent set.
for (const field of ["dependencies", "optionalDependencies"]) {
for (const name of Object.keys(manifest[field] ?? {})) {
if (names.has(name)) manifest[field][name] = `${BASE_URL}/${name}@${sha}`;
}
}
await writeFile(manifestPath, `${JSON.stringify(manifest, null, "\t")}\n`);
try {
const output = execNpmSync(["pack", "--json", "--ignore-scripts", "--pack-destination", outDir], {
cwd: pkg.directory,
encoding: "utf8",
stdio: ["inherit", "pipe", "inherit"],
maxBuffer: 64 * 1024 * 1024,
});
const [info] = JSON.parse(output);
packed.push({ name: info.name, file: info.filename });
console.error(`packed ${info.name}@${info.version} -> ${info.filename}`);
} finally {
await writeFile(manifestPath, original);
}
}
await writeFile(join(outDir, "manifest.json"), `${JSON.stringify({ sha, packages: packed }, null, "\t")}\n`);
}
// The audience names the production worker, also when --base-url points elsewhere (tests).
async function getOidcToken() {
const url = process.env.ACTIONS_ID_TOKEN_REQUEST_URL;
const token = process.env.ACTIONS_ID_TOKEN_REQUEST_TOKEN;
if (!url || !token) {
throw new UsageError("No GitHub OIDC token available; the job needs `permissions: id-token: write`");
}
const requestUrl = new URL(url);
requestUrl.searchParams.set("audience", BASE_URL);
const response = await fetch(requestUrl, { headers: { authorization: `Bearer ${token}` } });
if (!response.ok) throw new Error(`Requesting OIDC token failed: ${response.status}`);
const { value } = await response.json();
return value;
}
async function call(method, url, token, body) {
const response = await fetch(url, { method, headers: { authorization: `Bearer ${token}` }, body });
const text = await response.text();
if (!response.ok) throw new Error(`${method} ${url} failed: ${response.status} ${text}`);
return JSON.parse(text);
}
function renderSummary({ baseUrl, sha, refs, names }) {
const lines = ["### Preview packages", "", `Built from \`${sha.slice(0, 7)}\`.`, ""];
if (refs.length > 0) {
lines.push(`Latest build of \`${refs[0]}\` (updates with new pushes):`, "", "```sh");
for (const name of names) lines.push(`npm i ${baseUrl}/${name}@${refs[0]}`);
lines.push("```", "");
}
lines.push("<details><summary>Pinned to this commit</summary>", "", "```sh");
for (const name of names) lines.push(`npm i ${baseUrl}/${name}@${sha}`);
lines.push("```", "", "</details>", "");
lines.push(`Installs need a token in \`.npmrc\`: \`//${new URL(baseUrl).host}/:_authToken=...\``);
return `${lines.join("\n")}\n`;
}
async function publish(argv) {
const { values } = parseArgs({
args: argv,
options: {
dir: { type: "string", default: "preview-packages" },
sha: { type: "string" },
ref: { type: "string", multiple: true, default: [] },
run: { type: "string" },
event: { type: "string" },
"base-url": { type: "string", default: BASE_URL },
},
});
assertSha(values.sha);
const { sha, ref: refs, event } = values;
const run = Number(values.run);
if (!/^\d+$/.test(values.run ?? "") || !Number.isSafeInteger(run)) {
throw new UsageError(`--run must be the CI run ID, got ${values.run}`);
}
if (event !== "push" && event !== "pull_request") {
throw new UsageError(`--event must be push or pull_request, got ${event}`);
}
const baseUrl = values["base-url"];
const dir = resolve(values.dir);
// The artifact comes from an untrusted PR build: only read tarballs from it. The SHA, run, event, and refs
// come from the trusted workflow_run event; the worker authorizes package names against its config.
const manifest = JSON.parse(await readFile(join(dir, "manifest.json"), "utf8"));
if (manifest.sha !== sha) throw new Error(`Artifact was packed for ${manifest.sha}, expected ${sha}`);
if (!Array.isArray(manifest.packages) || manifest.packages.length === 0) throw new Error("Artifact has no packages");
const token = await getOidcToken();
const names = [];
for (const { name, file } of manifest.packages) {
if (typeof file !== "string" || basename(file) !== file || !file.endsWith(".tgz")) {
throw new Error(`Invalid tarball name in artifact: ${file}`);
}
const url = `${baseUrl}/_/upload/${sha}/${encodeURIComponent(name)}?event=${event}`;
const result = await call("PUT", url, token, await readFile(join(dir, file)));
names.push(result.name);
console.error(`${result.existing ? "exists" : "uploaded"} ${result.name}@${sha}`);
}
if (refs.length > 0) {
const result = await call("POST", `${baseUrl}/_/refs`, token, JSON.stringify({ sha, refs, run }));
const newer = result.refs.filter((ref) => ref.sha !== sha);
if (newer.length > 0) {
// A newer run already published these refs; its summary has the current install commands.
console.error(`a newer run already published ${newer.map((ref) => `${ref.ref} (${ref.sha})`).join(", ")}`);
return;
}
console.error(`moved ${refs.join(", ")} to ${sha}`);
}
const summary = renderSummary({ baseUrl, sha, refs, names: names.sort() });
process.stdout.write(summary);
if (process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, summary);
}
async function deleteRef(argv) {
const { values } = parseArgs({
args: argv,
options: { ref: { type: "string" }, "base-url": { type: "string", default: BASE_URL } },
});
if (!values.ref) throw new UsageError("delete-ref needs --ref");
await call("DELETE", `${values["base-url"]}/_/refs/${encodeURIComponent(values.ref)}`, await getOidcToken());
console.error(`deleted ${values.ref}`);
}
const USAGE = `Usage:
node scripts/preview-packages.mjs pack --sha <sha> [--out dir]
node scripts/preview-packages.mjs publish --sha <sha> --run <CI run id> --event <push|pull_request>
[--dir dir] [--ref name]...
node scripts/preview-packages.mjs delete-ref --ref <name>`;
const [command, ...argv] = process.argv.slice(2);
const commands = { pack, publish, "delete-ref": deleteRef };
try {
if (!commands[command]) throw new UsageError(USAGE);
await commands[command](argv);
} catch (error) {
console.error(error instanceof UsageError ? error.message : `preview-packages: ${error.message}`);
process.exit(1);
}

127
scripts/preview-packages.test.mjs Normal file
View file

@ -0,0 +1,127 @@
import assert from "node:assert/strict";
import { execFile, execFileSync } from "node:child_process";
import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { createServer } from "node:http";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import test from "node:test";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
const run = promisify(execFile);
const script = join(dirname(fileURLToPath(import.meta.url)), "preview-packages.mjs");
const sha = "a".repeat(40);
const newerSha = "b".repeat(40);
function temporaryDirectory(t) {
const directory = mkdtempSync(join(tmpdir(), "pi-preview-packages-test-"));
t.after(() => rmSync(directory, { recursive: true, force: true }));
return directory;
}
function writePackage(repo, directory, manifest) {
mkdirSync(join(repo, "packages", directory), { recursive: true });
writeFileSync(join(repo, "packages", directory, "package.json"), `${JSON.stringify(manifest, null, "\t")}\n`);
}
/**
* Stands in for the pkg.earendil.com worker and GitHub's OIDC token endpoint, recording requests. `refsSha` is what the
* worker reports each ref resolves to after the update; `uploadBody` overrides the upload response.
*/
async function startServer(t, { refsSha = sha, uploadBody } = {}) {
const requests = [];
const server = createServer(async (req, res) => {
let body = "";
for await (const chunk of req) body += chunk;
requests.push({ method: req.method, url: req.url, authorization: req.headers.authorization, body });
const url = new URL(req.url, "http://localhost");
if (url.pathname === "/token") return res.end(JSON.stringify({ value: `oidc-for-${url.searchParams.get("audience")}` }));
if (req.method === "PUT") {
const name = decodeURIComponent(url.pathname.split("/").slice(4).join("/"));
return res.end(uploadBody ?? JSON.stringify({ name, sha, existing: false }));
}
const { refs } = JSON.parse(body);
res.end(JSON.stringify({ refs: refs.map((ref) => ({ ref, sha: refsSha })) }));
});
await new Promise((ready) => server.listen(0, "127.0.0.1", ready));
t.after(() => server.close());
return { url: `http://127.0.0.1:${server.address().port}`, requests };
}
function writeArtifact(directory) {
mkdirSync(directory, { recursive: true });
writeFileSync(join(directory, "scope-a-1.0.0.tgz"), "tarball a");
const packages = [{ name: "@scope/a", file: "scope-a-1.0.0.tgz" }];
writeFileSync(join(directory, "manifest.json"), JSON.stringify({ sha, packages }));
}
function publish(server, dir) {
const { GITHUB_STEP_SUMMARY: _, ...env } = process.env;
const args = ["publish", "--sha", sha, "--run", "7", "--event", "pull_request", "--ref", "pr-1", "--dir", dir];
return run(process.execPath, [script, ...args, "--base-url", server.url], {
env: { ...env, ACTIONS_ID_TOKEN_REQUEST_URL: `${server.url}/token`, ACTIONS_ID_TOKEN_REQUEST_TOKEN: "runner" },
});
}
test("pack points internal dependencies at the commit's builds and restores package.json", async (t) => {
const repo = temporaryDirectory(t);
writePackage(repo, "a", { name: "@scope/a", version: "1.0.0", dependencies: { "@scope/b": "1.0.0", other: "2.0.0" } });
writePackage(repo, "b", { name: "@scope/b", version: "1.0.0" });
writePackage(repo, "internal", { name: "@scope/internal", version: "1.0.0", private: true });
writePackage(repo, "env", { name: "@earendil-works/pi-env", version: "1.0.0" });
const original = readFileSync(join(repo, "packages/a/package.json"), "utf8");
const out = join(repo, "out");
await run(process.execPath, [script, "pack", "--sha", sha, "--out", out], { cwd: repo });
const manifest = JSON.parse(readFileSync(join(out, "manifest.json"), "utf8"));
assert.equal(manifest.sha, sha);
assert.deepEqual(manifest.packages.map((pkg) => pkg.name).sort(), ["@scope/a", "@scope/b"]);
assert.deepEqual(readdirSync(out).sort(), ["manifest.json", ...manifest.packages.map((pkg) => pkg.file)].sort());
const fileA = manifest.packages.find((pkg) => pkg.name === "@scope/a").file;
const packedA = JSON.parse(execFileSync("tar", ["-xOzf", join(out, fileA), "package/package.json"], { encoding: "utf8" }));
assert.deepEqual(packedA.dependencies, { "@scope/b": `https://pkg.earendil.com/@scope/b@${sha}`, other: "2.0.0" });
assert.equal(readFileSync(join(repo, "packages/a/package.json"), "utf8"), original);
});
test("publish uploads each tarball, moves refs and prints install commands", async (t) => {
const server = await startServer(t);
const dir = join(temporaryDirectory(t), "artifact");
writeArtifact(dir);
const { stdout } = await publish(server, dir);
const [token, upload, refs] = server.requests;
assert.equal(token.url, "/token?audience=https%3A%2F%2Fpkg.earendil.com");
assert.deepEqual(
{ method: upload.method, url: upload.url, authorization: upload.authorization, body: upload.body },
{
method: "PUT",
url: `/_/upload/${sha}/%40scope%2Fa?event=pull_request`,
authorization: "Bearer oidc-for-https://pkg.earendil.com",
body: "tarball a",
},
);
assert.deepEqual(JSON.parse(refs.body), { sha, refs: ["pr-1"], run: 7 });
assert.match(stdout, new RegExp(`npm i ${server.url}/@scope/a@pr-1`));
assert.match(stdout, new RegExp(`npm i ${server.url}/@scope/a@${sha}`));
});
test("publish leaves the summary to a newer run that already owns the ref", async (t) => {
const server = await startServer(t, { refsSha: newerSha });
const dir = join(temporaryDirectory(t), "artifact");
writeArtifact(dir);
const { stdout, stderr } = await publish(server, dir);
assert.equal(stdout, "");
assert.match(stderr, new RegExp(`a newer run already published pr-1 \\(${newerSha}\\)`));
});
test("publish fails on a successful response that is not JSON", async (t) => {
const server = await startServer(t, { uploadBody: "<html>proxy</html>" });
const dir = join(temporaryDirectory(t), "artifact");
writeArtifact(dir);
await assert.rejects(publish(server, dir), /not valid JSON/);
assert.equal(server.requests.filter((request) => request.method === "POST").length, 0);
});