diff --git a/scripts/validate-all.sh b/scripts/validate-all.sh new file mode 100755 index 0000000..e9f1b17 --- /dev/null +++ b/scripts/validate-all.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash +# validate-all.sh — Batch compile all Speckl examples and validate outputs +# Usage: ./scripts/validate-all.sh [--quick] [--verbose] +# --quick: only parse + TypeScript check (skip WASM/WAT) +# --verbose: show all errors +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +SPECKL_ROOT="$(dirname "$SCRIPT_DIR")" +COMPILER_DIR="$SPECKL_ROOT/compiler" +EXAMPLES_DIR="$SPECKL_ROOT/examples" +OUT_DIR="$COMPILER_DIR/out/ci-validation" +GREEN='\033[0;32m' +RED='\033[0;31m' +YELLOW='\033[1;33m' +NC='\033[0m' + +QUICK=false +VERBOSE=false + +for arg in "$@"; do + case $arg in + --quick) QUICK=true ;; + --verbose) VERBOSE=true ;; + esac +done + +# Ensure compiler builds clean +echo "Building compiler..." +cd "$COMPILER_DIR" +if ! npx tsc --noEmit 2>&1; then + echo -e "${RED}FAIL: Compiler has TypeScript errors${NC}" + exit 1 +fi + +# Run unit tests +echo "Running unit tests..." +TEST_OUTPUT=$(npm test 2>&1) || true +TEST_COUNT=$(echo "$TEST_OUTPUT" | grep -oP 'Tests\s+\K\d+(?=\s+passed)' | head -1 || echo "0") +FAIL_COUNT=$(echo "$TEST_OUTPUT" | grep -oP 'Tests\s+\K\d+(?=\s+failed)' | head -1 || echo "0") +if [ "$FAIL_COUNT" != "0" ] && [ "$FAIL_COUNT" != "" ]; then + echo -e "${RED}FAIL: $FAIL_COUNT test(s) failing${NC}" +else + echo -e "${GREEN}PASS: $TEST_COUNT tests passing${NC}" +fi + +# Prepare output directory +rm -rf "$OUT_DIR" +mkdir -p "$OUT_DIR" + +echo "" +echo "============================================" +echo " Speckl Compiler Validation" +echo " $(date '+%Y-%m-%d %H:%M:%S')" +echo "============================================" +echo "" + +# Results tracking +PASS=0 +FAIL=0 +PARSE_FAIL=0 +TS_CLEAN=0 +TS_ERRORS=0 +WAT_OK=0 + +declare -A RESULTS +declare -A TS_ERROR_COUNTS + +NODE_BIN="$COMPILER_DIR/dist/index.js" +if [ ! -f "$NODE_BIN" ]; then + echo -e "${RED}FATAL: Compiler dist not found at $NODE_BIN${NC}" + echo "Run 'cd compiler && npx tsc' first" + exit 1 +fi + +shopt -s nullglob +SPECS=( "$EXAMPLES_DIR"/*.speck ) + +for spec in "${SPECS[@]}"; do + NAME=$(basename "$spec" .speck) + EXAMPLE_OUT="$OUT_DIR/$NAME" + mkdir -p "$EXAMPLE_OUT" + + echo -n " $NAME ... " + + # Step 1: Parse + generate all artifacts + PARSE_OUT=$(node "$NODE_BIN" "$spec" -o "$EXAMPLE_OUT" 2>&1) || true + if echo "$PARSE_OUT" | grep -q "Error\|error:"; then + echo -e "${RED}PARSE FAIL${NC}" + if $VERBOSE; then echo "$PARSE_OUT"; fi + RESULTS[$NAME]="PARSE_FAIL" + PARSE_FAIL=$((PARSE_FAIL + 1)) + FAIL=$((FAIL + 1)) + continue + fi + + # Step 2: Count artifacts generated + ARTIFACT_COUNT=$(find "$EXAMPLE_OUT" -maxdepth 1 -type f \( -name "*.json" -o -name "*.jsonld" -o -name "*.ts" -o -name "*.wat" \) 2>/dev/null | wc -l) + + # Step 3: TypeScript check + TS_FILE="$EXAMPLE_OUT/$NAME.ts" + TS_OK=false + if [ -f "$TS_FILE" ]; then + TS_CHECK_OUT=$(npx tsc --noEmit --target ES2020 --module commonjs --strict --lib ES2020 --skipLibCheck \ + "$TS_FILE" 2>&1) && TS_OK=true || true + if [ "$TS_OK" = true ]; then + TS_CLEAN=$((TS_CLEAN + 1)) + else + TS_ERR_COUNT=$(echo "$TS_CHECK_OUT" | grep -c "error TS" 2>/dev/null || echo "0") + TS_ERROR_COUNTS[$NAME]=$TS_ERR_COUNT + TS_ERRORS=$((TS_ERRORS + 1)) + # Save error log + echo "$TS_CHECK_OUT" > "$EXAMPLE_OUT/ts-errors.txt" + fi + else + echo -e "${RED}NO TS OUTPUT${NC}" + RESULTS[$NAME]="NO_TS" + FAIL=$((FAIL + 1)) + continue + fi + + # Step 4: WASM/WAT validation + WAT_FILE="$EXAMPLE_OUT/$NAME.wat" + if [ "$QUICK" = false ] && [ -f "$WAT_FILE" ]; then + if command -v wat2wasm &>/dev/null; then + wat2wasm "$WAT_FILE" -o "$EXAMPLE_OUT/$NAME.wasm" 2>/dev/null && WAT_OK=$((WAT_OK + 1)) || true + fi + fi + + # Final status + if [ "$TS_OK" = true ]; then + echo -e "${GREEN}OK${NC} ($ARTIFACT_COUNT artifacts, TS clean)" + RESULTS[$NAME]="PASS" + PASS=$((PASS + 1)) + else + TS_ERR="${TS_ERROR_COUNTS[$NAME]:-?}" + echo -e "${YELLOW}TS ERRORS: $TS_ERR${NC} ($ARTIFACT_COUNT artifacts)" + RESULTS[$NAME]="TS_ERRORS" + FAIL=$((FAIL + 1)) + if $VERBOSE && [ -f "$EXAMPLE_OUT/ts-errors.txt" ]; then + head -20 "$EXAMPLE_OUT/ts-errors.txt" + fi + fi +done + +echo "" +echo "============================================" +echo " RESULTS" +echo "============================================" +TOTAL=${#SPECS[@]} +echo " Total examples: $TOTAL" +echo " Parse success: $((PASS + TS_ERRORS))/$((PASS + TS_ERRORS + PARSE_FAIL))" +echo " Parse failures: $PARSE_FAIL" +echo " TS clean: $TS_CLEAN" +echo " TS with errors: $TS_ERRORS" +echo " Overall pass: $PASS / $TOTAL" +if [ "$QUICK" = false ]; then + echo " WAT valid: $WAT_OK" +fi +echo "" + +# Detailed table +printf " %-30s %s\n" "EXAMPLE" "STATUS" +printf " %-30s %s\n" "-------" "------" +for spec in "${SPECS[@]}"; do + NAME=$(basename "$spec" .speck) + STATUS="${RESULTS[$NAME]:-UNKNOWN}" + case $STATUS in + PASS) COLOR="$GREEN" ;; + TS_ERRORS) COLOR="$YELLOW" ;; + PARSE_FAIL|NO_TS) COLOR="$RED" ;; + *) COLOR="$RED" ;; + esac + EXTRA="" + if [ "$STATUS" = "TS_ERRORS" ]; then + EXTRA=" (${TS_ERROR_COUNTS[$NAME]:-?} errors)" + fi + printf " ${COLOR}%-30s %-12s${NC}%s\n" "$NAME" "$STATUS" "$EXTRA" +done + +echo "" +echo " Artifacts in: $OUT_DIR" +echo " TS error logs: $OUT_DIR/*/ts-errors.txt" +echo "" + +# Exit code +if [ $FAIL -gt 0 ] || [ $PARSE_FAIL -gt 0 ]; then + exit 1 +fi +exit 0