From 702f2547bb59232a9050b9540d833f0204196ab2 Mon Sep 17 00:00:00 2001
From: Garry Tan <garrytan@gmail.com>
Date: Mon, 23 Mar 2026 21:21:20 -0700
Subject: [PATCH] fix: report job handles malformed eval JSON gracefully

Large eval transcripts (350k+ tokens) can produce JSON that jq chokes on.
Skip malformed files instead of crashing the entire report job.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 .github/workflows/evals.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/evals.yml b/.github/workflows/evals.yml
index a6f507a7..7fab3cb2 100644
--- a/.github/workflows/evals.yml
+++ b/.github/workflows/evals.yml
@@ -182,6 +182,10 @@ jobs:
           TOTAL=0; PASSED=0; FAILED=0; COST="0"
           SUITE_LINES=""
           for f in $RESULTS; do
+            if ! jq -e '.total_tests' "$f" >/dev/null 2>&1; then
+              echo "Skipping malformed JSON: $f"
+              continue
+            fi
             T=$(jq -r '.total_tests // 0' "$f")
             P=$(jq -r '.passed // 0' "$f")
             F=$(jq -r '.failed // 0' "$f")
@@ -214,9 +218,10 @@ jobs:
           if [ "$FAILED" -gt 0 ]; then
             FAILURES=""
             for f in $RESULTS; do
+              if ! jq -e '.failed' "$f" >/dev/null 2>&1; then continue; fi
               F=$(jq -r '.failed // 0' "$f")
               [ "$F" -eq 0 ] && continue
-              FAILS=$(jq -r '.tests[] | select(.passed == false) | "- ❌ \(.name): \(.exit_reason // "unknown")"' "$f")
+              FAILS=$(jq -r '.tests[] | select(.passed == false) | "- ❌ \(.name): \(.exit_reason // "unknown")"' "$f" 2>/dev/null || echo "- ⚠️ $(basename "$f"): parse error")
               FAILURES="${FAILURES}${FAILS}\n"
             done
             BODY="${BODY}