diff --git a/.github/workflows/p2-cloud-bundle.yml b/.github/workflows/p2-cloud-bundle.yml index 9dbb074f8..1ae4aaf1f 100644 --- a/.github/workflows/p2-cloud-bundle.yml +++ b/.github/workflows/p2-cloud-bundle.yml @@ -1,9 +1,6 @@ -# Builds p2's web UI for Radius Cloud Brain after CI passes on main and uploads it to R2 under the commit's full SHA -# (apps/p2/scripts/cloud-bundle.ts). pkg.earendil.com publishes pi's packages under the same SHA, so Radius pins one -# SHA for the packages, the p2 contracts it copies and this UI. -# -# Builds are immutable: a SHA that already has one is skipped. Runs for pushes to main only, never for PRs: the job -# holds R2 write credentials. +# Builds p2's web UI for Radius Cloud Brain after CI passes on main (apps/p2/scripts/cloud-bundle.ts) and uploads it +# to the R2 bucket pi-ui under the commit's full SHA, which Radius serves under /pi/ui/. Builds are immutable: a SHA +# that has one is skipped. Never runs for PRs: the job holds R2 write credentials. name: p2 cloud bundle @@ -13,8 +10,8 @@ on: types: [completed] workflow_dispatch: inputs: - source_ref: - description: Commit (full SHA) to build and upload + sha: + description: Full commit SHA to build and upload required: true type: string @@ -31,32 +28,20 @@ jobs: github.event.workflow_run.head_repository.full_name == github.repository) runs-on: ubuntu-latest environment: pi-model-upload - concurrency: - group: p2-cloud-bundle-${{ github.event.workflow_run.head_sha || inputs.source_ref }} - cancel-in-progress: false env: - SOURCE_REF: ${{ github.event.workflow_run.head_sha || inputs.source_ref }} + SHA: ${{ github.event.workflow_run.head_sha || inputs.sha }} AWS_ACCESS_KEY_ID: ${{ secrets.PI_ARTIFACTS_R2_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.PI_ARTIFACTS_R2_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: auto AWS_EC2_METADATA_DISABLED: 'true' - R2_ENDPOINT: https://67c0d357268b0fca6e0b465bb9d01b84.r2.cloudflarestorage.com - # Holds only these builds, at `/`. The bucket is not public: Radius's control plane binds it and serves it - # under /pi/ui/ on its own host, so the page loads its UI same-origin (no CORS). - R2_BUCKET: pi-ui - # Built into the script (the version poll and asset URLs). Root-relative, so one build serves every Radius host. - BASE_PATH: /pi/ui + R2: --endpoint-url https://67c0d357268b0fca6e0b465bb9d01b84.r2.cloudflarestorage.com steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ env.SOURCE_REF }} + ref: ${{ env.SHA }} persist-credentials: false - - name: Resolve the commit - id: commit - run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - - name: Setup Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: @@ -70,40 +55,19 @@ jobs: - name: Hydrate model data run: npm run hydrate:model-data - - name: Verify AWS CLI - run: aws --version - - - name: Build the cloud bundle + - name: Build working-directory: apps/p2 - env: - SHA: ${{ steps.commit.outputs.sha }} - run: >- - node --import ./src/source.ts scripts/cloud-bundle.ts build - --sha "$SHA" - --base "$BASE_PATH/$SHA" - --out "$RUNNER_TEMP/cloud-bundle" + run: node --import ./src/source.ts scripts/cloud-bundle.ts "$SHA" "$RUNNER_TEMP/cloud-bundle" - - name: Upload to R2 - working-directory: apps/p2 - env: - SHA: ${{ steps.commit.outputs.sha }} - run: >- - node --import ./src/source.ts scripts/cloud-bundle.ts upload - --sha "$SHA" - --bucket "$R2_BUCKET" - --endpoint "$R2_ENDPOINT" - --out "$RUNNER_TEMP/cloud-bundle" - - - name: Summary - env: - SHA: ${{ steps.commit.outputs.sha }} + # The manifest goes last, so a build whose manifest exists is complete. + - name: Upload run: | - { - echo "### p2 cloud bundle" - echo - echo "Built from \`${SHA}\`, uploaded to \`$R2_BUCKET\`. Radius \`PI_UI_MANIFEST_URL\`, on its own host:" - echo - echo "\`\`\`" - echo "$BASE_PATH/$SHA/manifest.json" - echo "\`\`\`" - } >> "$GITHUB_STEP_SUMMARY" + if aws s3api head-object $R2 --bucket pi-ui --key "$SHA/manifest.json" >/dev/null 2>&1; then + echo "pi-ui/$SHA exists: kept" + exit 0 + fi + cache="public, max-age=31536000, immutable" + aws s3 cp $R2 --recursive --exclude manifest.json --cache-control "$cache" \ + "$RUNNER_TEMP/cloud-bundle/$SHA" "s3://pi-ui/$SHA" + aws s3 cp $R2 --cache-control "$cache" \ + "$RUNNER_TEMP/cloud-bundle/$SHA/manifest.json" "s3://pi-ui/$SHA/manifest.json" diff --git a/apps/p2/docs/p2-notes.md b/apps/p2/docs/p2-notes.md index aeab8cce5..1e4992072 100644 --- a/apps/p2/docs/p2-notes.md +++ b/apps/p2/docs/p2-notes.md @@ -48,15 +48,10 @@ P2_FAKE=1 npm start # http://127.0.0.1:8788, data in ~/.pi/p2 until loader model D provides scoped, per-facet `ClientBundle` pushes (`docs/plugin-loading.md` 5.3 and 12.2 step 4). The TUI launcher separately polls `/__p2_tui`. There is no Vite, source-module server or `index.html`. - **Bundles for Radius Cloud Brain:** after CI passes on main, `.github/workflows/p2-cloud-bundle.yml` builds this - web bundle with `scripts/cloud-bundle.ts` (imports `buildWebBundle`) and uploads it to the R2 bucket `pi-ui` under - the commit's full SHA, the SHA pkg.earendil.com publishes pi's packages under. Radius's control plane serves the - bucket under `/pi/ui/` on its own host, and the build uses that root-relative path. Radius pins one SHA for the packages, the p2 contracts it - copies and this UI. The script patches three places in the built script: the Chord socket URL - (`` new WebSocket(`${...}://${location.host}/chord`) ``), the version poll - (`fetch("/__p2_web",{cache:"no-store",credentials:"same-origin"})`) and the `/__p2_web/` asset prefix. It fails - when a target is missing, so changing those in `src/hosts/web/main.tsx` or `WEB_BUNDLE_PATH` means updating it. The - earlier home of these bundles, the public repo `earendil-works/bundled-ui` (served through jsDelivr), is replaced by - this. + web bundle with `scripts/cloud-bundle.ts` and uploads it to the R2 bucket `pi-ui` under the commit's full SHA, which + Radius serves under `/pi/ui/`. The script patches the built script's Chord socket URL, version poll and `/__p2_web/` + asset prefix, and fails when one is missing: changing those in `src/hosts/web/main.tsx` or `WEB_BUNDLE_PATH` means + updating it. - `npm run dev` also watches plugins and supervises runtime restarts; see **Dev mode** below. Under `npm start`, code changes (web included) take effect only after a restart. - **Data report** (`scripts/data-report.ts`, read-only, beside a live server): per session its title, location, diff --git a/apps/p2/scripts/cloud-bundle.ts b/apps/p2/scripts/cloud-bundle.ts index 93fba06af..66f59db67 100644 --- a/apps/p2/scripts/cloud-bundle.ts +++ b/apps/p2/scripts/cloud-bundle.ts @@ -1,251 +1,61 @@ -// Builds p2's web UI for hosts other than p2 itself, and uploads it to R2: the Radius Cloud Brain page, and p2 brains -// whose page loads it from there. The UI is p2's own (`plugins/*/web.tsx`, `src/web/`, `src/hosts/web/`): this builds -// it with `buildWebBundle` and patches the three places where the built script assumes p2 serves it: +// Builds p2's web UI for Radius Cloud Brain into `//`: the plugins' web facets, their stylesheet and a +// manifest.json naming both. Radius serves the `pi-ui` bucket under `/pi/ui/` and puts the Chord URL in the page's +// `#root` `data-pi-bootstrap`; the built script assumes p2 serves it, so its Chord socket and its version poll are +// patched. A missing patch target, or anything else still loaded from p2's `/__p2_web/`, fails the build: +// `src/hosts/web/main.tsx` or `WEB_BUNDLE_PATH` changed, and this script must follow. // -// - the Chord socket: the page bootstrap's `chordUrl` (`data-pi-bootstrap`), else `/chord` on the page's host; -// - the version poll: this build's manifest.json instead of `/__p2_web`; -// - asset URLs (the terminal's wasm): this build's directory instead of `/__p2_web/`. -// -// A patch whose target is missing stops the build: `src/hosts/web/main.tsx` or `WEB_BUNDLE_PATH` changed, and this -// script must follow. Source maps are dropped (they embed the sources; the bundle is served publicly). -// -// Each build lives under the full commit SHA it was built from, in two variants with a manifest each: -// / cloud: the plugins Radius Cloud Brain serves -// /brain/ brain: every p2 plugin with a web facet -// The SHA is the same one pkg.earendil.com publishes pi's packages under, so a consumer pins one SHA for the packages -// (and the p2 contracts they bind) and for this UI. Files are immutable once uploaded: an existing build is kept. -// -// node --import ./src/source.ts scripts/cloud-bundle.ts build --sha --base [--out dir] -// node --import ./src/source.ts scripts/cloud-bundle.ts upload --sha --bucket --endpoint -// [--prefix ] [--out dir] -// -// `--base` is where the page loads `/` from: a root-relative path (`/pi/ui/`), so one build works on every -// host that serves the bucket under that path (production and staging alike), or an absolute URL. It is built into the -// script: the version poll fetches `/manifest.json`, assets load from `/`. `upload` uses the AWS CLI against -// R2's S3 API, with credentials from the environment (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY). +// node --import ./src/source.ts scripts/cloud-bundle.ts -import { spawnSync } from "node:child_process"; -import { mkdir, readdir, rm, writeFile } from "node:fs/promises"; -import { extname, join, relative, resolve } from "node:path"; -import { parseArgs } from "node:util"; +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; import { buildWebBundle } from "../src/hosts/bundles.ts"; -const FULL_SHA = /^[0-9a-f]{40}$/; +/** The plugins Cloud Brain serves. `web-shell` registers the "app" slot; without it the page stays on "Loading…". */ +const PLUGINS = ["web-shell", "artifacts", "chat", "drafts", "models", "sessions", "status", "subagents", "workspace"]; -/** The plugins Radius Cloud Brain serves. `web-shell` registers the "app" slot; without it the page stays on "Loading…". */ -const CLOUD = ["web-shell", "artifacts", "chat", "drafts", "models", "sessions", "status", "subagents", "workspace"]; -/** Every p2 plugin with a web facet: a p2 brain's full UI. */ -const BRAIN = [ - "artifacts", - "chat", - "chat-mode", - "conversations", - "devices", - "drafts", - "envs", - "login", - "logs", - "models", - "ports", - "prompt", - "sessions", - "sharing", - "status", - "subagents", - "terminals", - "voice", - "web-shell", - "workspace", -]; +const [sha, out] = process.argv.slice(2); +if (!sha || !/^[0-9a-f]{40}$/.test(sha) || !out) throw new Error("usage: cloud-bundle.ts "); -const VARIANTS = [ - { name: "cloud", plugins: CLOUD, directory: "" }, - { name: "brain", plugins: BRAIN, directory: "brain" }, -] as const; +const base = `/pi/ui/${sha}`; +const dir = join(out, sha); +const bundle = await buildWebBundle(PLUGINS); +await mkdir(dir, { recursive: true }); -const TYPES: Readonly> = { - ".js": "text/javascript; charset=utf-8", - ".css": "text/css; charset=utf-8", - ".json": "application/json; charset=utf-8", - ".wasm": "application/wasm", +// Source maps embed the sources and the bundle is served publicly: they are left out, and so are the comments naming +// them (the last line of the script and of the stylesheet). +for (const [file, { body }] of bundle.files) { + if (file.endsWith(".map")) continue; + let text = new TextDecoder().decode(body); + if (file === bundle.script) text = patchScript(text.replace(/\n\/\/# sourceMappingURL=\S+\n?$/, "\n")); + else if (file === bundle.style) text = text.replace(/\n\/\*# sourceMappingURL=\S+ \*\/\n?$/, "\n"); + await writeFile(join(dir, file), text); +} +const manifest = { + version: `pi-${sha}`, + script: bundle.script, + stylesheets: bundle.style ? [bundle.style] : [], + commit: sha, }; -/** Content-named files and a build's manifest never change once uploaded. */ -const IMMUTABLE = "public, max-age=31536000, immutable"; +await writeFile(join(dir, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`); +console.log(`built ${dir}`); -/** The page bootstrap's `chordUrl`, else `/chord` on the page's host (as p2 serves it). */ -const CHORD_URL = - // biome-ignore lint/suspicious/noTemplateCurlyInString: source text spliced into the built script, not a template. - '(()=>{const fallback=`${location.protocol==="https:"?"wss":"ws"}://${location.host}/chord`;' + - 'try{const raw=document.getElementById("root")?.getAttribute("data-pi-bootstrap");' + - "return (raw?JSON.parse(raw):undefined)?.chordUrl??fallback}catch{return fallback}})()"; +function patchScript(script: string): string { + script = patch( + script, + // biome-ignore lint/suspicious/noTemplateCurlyInString: the built script's exact text, not a template. + 'new WebSocket(`${location.protocol==="https:"?"wss":"ws"}://${location.host}/chord`)', + 'new WebSocket(JSON.parse(document.getElementById("root").dataset.piBootstrap).chordUrl)', + ); + script = patch( + script, + 'fetch("/__p2_web",{cache:"no-store",credentials:"same-origin"})', + `fetch("${base}/manifest.json",{cache:"no-store"})`, + ); + if (script.includes("__p2_web")) throw new Error("the built script still names /__p2_web"); + return script; +} -class UsageError extends Error {} - -function patch(source: string, from: string, to: string, all = false): string { +function patch(source: string, from: string, to: string): string { if (!source.includes(from)) throw new Error(`patch target not found: ${from}`); - return all ? source.split(from).join(to) : source.replace(from, to); -} - -function assertSha(sha: string | undefined): string { - if (sha === undefined || !FULL_SHA.test(sha)) throw new UsageError(`--sha must be a full commit SHA, got ${sha}`); - return sha; -} - -/** Builds both variants into `//`. */ -async function build(argv: string[]): Promise { - const { values } = parseArgs({ - args: argv, - options: { - sha: { type: "string" }, - base: { type: "string" }, - out: { type: "string", default: "cloud-bundle" }, - }, - }); - const sha = assertSha(values.sha); - const baseUrl = values.base?.replace(/\/$/, ""); - if (!baseUrl) throw new UsageError("build needs --base"); - const root = join(resolve(values.out), sha); - await rm(root, { recursive: true, force: true }); - for (const { name, plugins, directory } of VARIANTS) { - const output = join(root, directory); - const url = directory ? `${baseUrl}/${directory}` : baseUrl; - const manifestUrl = `${url}/manifest.json`; - // Plugins without a web facet are skipped by the build. - const bundle = await buildWebBundle(plugins); - await mkdir(output, { recursive: true }); - let script: string | undefined; - let style: string | undefined; - for (const [file, content] of bundle.files) { - if (file.endsWith(".map")) continue; - let body: Uint8Array | string = content.body; - if (file === bundle.script) { - let text = new TextDecoder().decode(content.body); - text = patch( - text, - // biome-ignore lint/suspicious/noTemplateCurlyInString: the built script's exact text, not a template. - 'new WebSocket(`${location.protocol==="https:"?"wss":"ws"}://${location.host}/chord`)', - `new WebSocket(${CHORD_URL})`, - ); - text = patch( - text, - 'fetch("/__p2_web",{cache:"no-store",credentials:"same-origin"})', - `fetch(${JSON.stringify(manifestUrl)},{cache:"no-store"})`, - ); - if (text.includes("/__p2_web/")) text = patch(text, "/__p2_web/", `${url}/`, true); - text = text.replace(/\n?\/\/# sourceMappingURL=\S*\s*$/, "\n"); - if (text.includes("__p2_web")) throw new Error(`${name}: the script still names /__p2_web`); - body = text; - script = file; - } else if (file === bundle.style) { - body = new TextDecoder().decode(content.body).replace(/\n?\/\*# sourceMappingURL=[^*]*\*\/\s*$/, "\n"); - style = file; - } - await writeFile(join(output, file), body); - } - if (!script) throw new Error(`${name}: bundle has no script`); - // `version` is what the page compares with `data-web-version` (Radius sets it from this manifest). - const manifest = { - version: `pi-${sha}${directory ? `-${directory}` : ""}`, - script, - stylesheets: style ? [style] : [], - commit: sha, - plugins, - }; - await writeFile(join(output, "manifest.json"), `${JSON.stringify(manifest, null, 2)}\n`); - console.log(JSON.stringify({ variant: name, output, manifest: manifestUrl, script, style })); - } -} - -/** Uploads `//` to `s3:////`, unless that build exists already (builds are immutable). */ -async function upload(argv: string[]): Promise { - const { values } = parseArgs({ - args: argv, - options: { - sha: { type: "string" }, - bucket: { type: "string" }, - endpoint: { type: "string" }, - prefix: { type: "string", default: "" }, - out: { type: "string", default: "cloud-bundle" }, - }, - }); - const sha = assertSha(values.sha); - if (!values.bucket || !values.endpoint) throw new UsageError("upload needs --bucket and --endpoint"); - const prefix = values.prefix.replace(/^\/+/, ""); - const root = join(resolve(values.out), sha); - const key = (file: string) => `${prefix}${sha}/${relative(root, file).split("\\").join("/")}`; - const manifestKey = `${prefix}${sha}/manifest.json`; - if ( - aws( - ["s3api", "head-object", "--bucket", values.bucket, "--key", manifestKey, "--endpoint-url", values.endpoint], - true, - ) - ) { - console.log(`exists s3://${values.bucket}/${manifestKey}: kept`); - return; - } - const files = await listFiles(root); - // The manifests last: a manifest that is visible names files that are already there. - const ordered = [ - ...files.filter((file) => !file.endsWith("manifest.json")), - ...files.filter((file) => file.endsWith("manifest.json")), - ]; - for (const file of ordered) { - const type = TYPES[extname(file)] ?? "application/octet-stream"; - aws([ - "s3", - "cp", - file, - `s3://${values.bucket}/${key(file)}`, - "--endpoint-url", - values.endpoint, - "--content-type", - type, - "--cache-control", - IMMUTABLE, - "--only-show-errors", - ]); - console.log(`uploaded s3://${values.bucket}/${key(file)}`); - } -} - -async function listFiles(directory: string): Promise { - const entries = await readdir(directory, { withFileTypes: true }); - const files: string[] = []; - for (const entry of entries) { - const path = join(directory, entry.name); - if (entry.isDirectory()) files.push(...(await listFiles(path))); - else files.push(path); - } - return files.sort(); -} - -/** Runs the AWS CLI; with `allowNotFound`, a missing object is `false` instead of an error. */ -function aws(args: string[], allowNotFound = false): boolean { - const result = spawnSync("aws", args, { - encoding: "utf8", - env: { - ...process.env, - AWS_DEFAULT_REGION: process.env.AWS_DEFAULT_REGION || "auto", - AWS_EC2_METADATA_DISABLED: "true", - }, - }); - if (result.error) throw result.error; - if (result.status === 0) return true; - const message = `${result.stdout}\n${result.stderr}`.trim(); - if (allowNotFound && /(?:404|NoSuchKey|Not Found)/i.test(message)) return false; - throw new Error(`aws ${args.slice(0, 2).join(" ")} failed:\n${message}`); -} - -const USAGE = `Usage: - node --import ./src/source.ts scripts/cloud-bundle.ts build --sha --base [--out dir] - node --import ./src/source.ts scripts/cloud-bundle.ts upload --sha --bucket --endpoint [--prefix ] [--out dir]`; - -const [command, ...rest] = process.argv.slice(2); -try { - if (command === "build") await build(rest); - else if (command === "upload") await upload(rest); - else throw new UsageError(USAGE); -} catch (error) { - console.error(error instanceof UsageError ? error.message : `cloud-bundle: ${(error as Error).message}`); - process.exit(1); + return source.replaceAll(from, to); }