From fc8048d2370ab4252966aa3bf220373d1358dce2 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Fri, 24 Jul 2026 07:32:53 +0200 Subject: [PATCH] fix(coding-agent): bundle model data in release source --- .github/workflows/build-binaries.yml | 4 +++ README.md | 4 +-- packages/coding-agent/CHANGELOG.md | 1 + scripts/build-binaries.sh | 27 +++++++++++++------ scripts/create-source-archive.sh | 39 +++++++++++++++++++++++++--- 5 files changed, 62 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index f646791cb..792782066 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -49,6 +49,9 @@ jobs: node-version: '22' registry-url: 'https://registry.npmjs.org' + - name: Hydrate release model data + run: npm run hydrate:model-data + - name: Create source archive run: | set -euo pipefail @@ -69,6 +72,7 @@ jobs: trap 'rm -rf "${build_root}"' EXIT tar -xzf "release-assets/pi-${VERSION}-source.tar.gz" -C "${build_root}" "${build_root}/pi-${VERSION}/scripts/build-binaries.sh" \ + --offline-model-data \ --out "${GITHUB_WORKSPACE}/packages/coding-agent/binaries" - name: Prepare GitHub release payload diff --git a/README.md b/README.md index f140437d2..83a1ff8ba 100644 --- a/README.md +++ b/README.md @@ -67,10 +67,10 @@ GitHub releases include a versioned source archive covered by the release's `SHA VERSION="" tar -xzf "pi-${VERSION}-source.tar.gz" cd "pi-${VERSION}" -./scripts/build-binaries.sh --platform linux-x64 --out "$PWD/out" +./scripts/build-binaries.sh --offline-model-data --platform linux-x64 --out "$PWD/out" ``` -The script installs dependencies, builds the monorepo, compiles the Bun executable, and stages its runtime assets. Package maintainers who provide dependencies separately can pass `--skip-install --skip-deps`. +The source archive includes the generated provider model data used for the release. `--offline-model-data` builds with that snapshot instead of refreshing it from live provider catalogs. The script still installs dependencies, builds the monorepo, compiles the Bun executable, and stages its runtime assets. Package maintainers who provide dependencies separately can pass `--skip-install --skip-deps`. ## Supply-chain hardening diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index da101e5bb..6eb152dc0 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -36,6 +36,7 @@ - Fixed fresh installs from preferring bundled model catalogs over newer remote catalogs because package file mtimes were newer ([#7016](https://github.com/earendil-works/pi/pull/7016) by [@davidbrai](https://github.com/davidbrai)). - Fixed inherited editor scroll indicators overflowing narrow terminals ([#7015](https://github.com/earendil-works/pi/pull/7015) by [@christianklotz](https://github.com/christianklotz)). - Fixed llama.cpp models to use the loaded context window as their output token limit instead of capping it at 16K ([#7034](https://github.com/earendil-works/pi/pull/7034) by [@christianklotz](https://github.com/christianklotz)). +- Fixed release source archives to include the generated provider model data used to build standalone binaries. - Updated the packaged `protobufjs` dependency to 7.6.5 to address GHSA-j3f2-48v5-ccww ([#7005](https://github.com/earendil-works/pi/issues/7005)). ## [0.81.1] - 2026-07-21 diff --git a/scripts/build-binaries.sh b/scripts/build-binaries.sh index c261aaf20..4b4df3590 100755 --- a/scripts/build-binaries.sh +++ b/scripts/build-binaries.sh @@ -4,14 +4,15 @@ # Mirrors .github/workflows/build-binaries.yml # # Usage: -# ./scripts/build-binaries.sh [--skip-install] [--skip-deps] [--skip-build] [--platform ] [--out ] +# ./scripts/build-binaries.sh [--skip-install] [--skip-deps] [--skip-build] [--offline-model-data] [--platform ] [--out ] # # Options: -# --skip-install Skip npm ci -# --skip-deps Skip installing cross-platform dependencies -# --skip-build Skip npm run build -# --platform Build only for specified platform (darwin-arm64, darwin-x64, linux-x64, linux-arm64, windows-x64, windows-arm64) -# --out Output directory (default: packages/coding-agent/binaries) +# --skip-install Skip npm ci +# --skip-deps Skip installing cross-platform dependencies +# --skip-build Skip the package build +# --offline-model-data Build with bundled model data instead of refreshing it +# --platform Build only for specified platform (darwin-arm64, darwin-x64, linux-x64, linux-arm64, windows-x64, windows-arm64) +# --out Output directory (default: packages/coding-agent/binaries) # # Output: # packages/coding-agent/binaries/ @@ -29,6 +30,7 @@ cd "$(dirname "$0")/.." SKIP_INSTALL=false SKIP_DEPS=false SKIP_BUILD=false +OFFLINE_MODEL_DATA=false PLATFORM="" OUTPUT_DIR="" @@ -46,6 +48,10 @@ while [[ $# -gt 0 ]]; do SKIP_BUILD=true shift ;; + --offline-model-data) + OFFLINE_MODEL_DATA=true + shift + ;; --platform) PLATFORM="$2" shift 2 @@ -108,8 +114,13 @@ else fi if [[ "$SKIP_BUILD" == "false" ]]; then - echo "==> Building all packages..." - npm run build + if [[ "$OFFLINE_MODEL_DATA" == "true" ]]; then + echo "==> Building all packages with bundled model data..." + npm run build:offline + else + echo "==> Building all packages..." + npm run build + fi else echo "==> Skipping package build (--skip-build)" fi diff --git a/scripts/create-source-archive.sh b/scripts/create-source-archive.sh index 629df7ae9..24f5cc849 100755 --- a/scripts/create-source-archive.sh +++ b/scripts/create-source-archive.sh @@ -2,6 +2,7 @@ # Create the deterministic source archive uploaded with GitHub releases. # # Usage: +# npm run hydrate:model-data # ./scripts/create-source-archive.sh --version --ref --out set -euo pipefail @@ -78,18 +79,47 @@ fi mkdir -p "$(dirname "$output")" output="$(cd "$(dirname "$output")" && pwd)/$(basename "$output")" +model_data_dir="packages/ai/src/providers/data" +if [[ ! -f "${model_data_dir}/.manifest.json" ]]; then + echo "Generated model data is missing. Run npm run hydrate:model-data first." >&2 + exit 1 +fi + +shopt -s nullglob +model_data_files=("${model_data_dir}/.manifest.json" "${model_data_dir}"/*.json) +shopt -u nullglob +if [[ ${#model_data_files[@]} -eq 1 ]]; then + echo "Generated model data is missing from ${model_data_dir}" >&2 + exit 1 +fi + temporary_archive="$(mktemp "${output}.tmp.XXXXXX")" +temporary_index="$(mktemp "${output}.index.XXXXXX")" manifest="$(mktemp "${output}.manifest.XXXXXX")" -trap 'rm -f "$temporary_archive" "$manifest"' EXIT +validation_root="$(mktemp -d "${output}.validation.XXXXXX")" +rm -f "$temporary_index" +trap 'rm -f "$temporary_archive" "$temporary_index" "$manifest"; rm -rf "$validation_root"' EXIT + +# Add the ignored release model-data snapshot to a temporary index based on the +# release commit. Archiving the resulting tree keeps the source artifact +# deterministic for the same commit and generated model data. +GIT_INDEX_FILE="$temporary_index" git read-tree "$commit" +GIT_INDEX_FILE="$temporary_index" git add -f -- "${model_data_files[@]}" +archive_tree="$(GIT_INDEX_FILE="$temporary_index" git write-tree)" +archive_mtime="$(git show -s --format=%ct "$commit")" archive_root="pi-${version}" -git archive --format=tar --prefix="${archive_root}/" "$commit" | gzip -n -9 > "$temporary_archive" +git archive --format=tar --prefix="${archive_root}/" --mtime="@${archive_mtime}" "$archive_tree" \ + | gzip -n -9 > "$temporary_archive" tar -tzf "$temporary_archive" > "$manifest" required_paths=( "package.json" "package-lock.json" "scripts/build-binaries.sh" + "packages/ai/src/models.generated.ts" + "packages/ai/src/image-models.generated.ts" + "packages/ai/src/providers/data/.manifest.json" "packages/coding-agent/package.json" "packages/coding-agent/src/utils/image-resize-worker.ts" "packages/coding-agent/src/core/export-html/template.css" @@ -112,6 +142,9 @@ if grep -Eq '(^|/)node_modules/|(^|/)packages/coding-agent/binaries/' "$manifest exit 1 fi +tar -xzf "$temporary_archive" -C "$validation_root" +node "${validation_root}/${archive_root}/packages/ai/scripts/check-model-data.ts" + mv "$temporary_archive" "$output" -trap 'rm -f "$manifest"' EXIT +trap 'rm -f "$temporary_index" "$manifest"; rm -rf "$validation_root"' EXIT printf '%s\n' "$output"