diff --git a/CHANGELOG.md b/CHANGELOG.md index 3291dc83f..6ba3000e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## [1.94.32.0] - 2026-11-03 + +### Fixed + +- **A RED paid screen now names its rule cases.** `gstack-ship-queue ready`, `verify --screen` and `train build` took a screen's red cases only from its behavior and judge panels, so rule cases that failed twice under the flaky-rerun rule never showed up. #3947's screen run 41662815506 printed `SCREEN_VERDICT: … RED flaky=ship-docsync-stale-after` with no `red=` list, and the refusal said "RED on its gate binding". That made it look like the one flaky case had turned the run red, when seven rule cases had actually failed twice. The verdict line and the `QUEUE_SCREEN_RED` refusal now list every case the report marks with ✗ (qa-quick, browse-basic and five more in that run). The verdict itself doesn't change. A case that passes on its one rerun is still green with a FLAKY line, and a case that fails twice is still red. + +### Tests + +- `test/ship-queue-screen-verdict.test.ts` reads the real gate binding and collector outcomes from run 41662815506. The seven twice-failed rule cases are named red, and the rerun-pass case stays only flaky. Negative control: with just the rerun-pass case left, the run reads GREEN with its FLAKY line. A red that never reran (INFRA, an unreported shard) is still named, and quarantined or passing split panel lines are not. + ## [1.94.31.0] - 2026-11-03 **gstack's browser now runs Chromium 156, which fixes CVE-2026-106378, a macOS sandbox escape ([#3907](https://github.com/garrytan/gstack/issues/3907)).** Playwright 1.62.1 bundled Chromium 151, which predates the fix. Your next `./setup` or `/gstack-upgrade` downloads the new browser. diff --git a/VERSION b/VERSION index 7b2979a0c..2da4db51a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.94.31.0 +1.94.32.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 600bfdbfc..41490b9d7 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.94.31.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.94.32.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index fe878b5cb..117f5b646 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -5709,7 +5709,8 @@ closed PRs. verdict is read from the run itself, never from a lane's report: the `gate-binding-a` artifact evals.yml's report job uploads (verdict, reconcile exit, flaky cases) and the `report-verdict-a` artifact's -`collector-outcomes.json` (the panels that failed the lane and the rule-16 +`collector-outcomes.json` (the panels that failed the lane, the report's `✗` +failure lines, which name rule cases that have no panel, and the rule-16 flaky lines). A case the binding calls red already failed twice inside the run (rule 16 reruns a first failure once), so the red is real. `ready` refuses with this code when the head's recorded screen run is red, whichever diff --git a/lib/ship-queue-screen.ts b/lib/ship-queue-screen.ts index febce6a21..76fc25829 100644 --- a/lib/ship-queue-screen.ts +++ b/lib/ship-queue-screen.ts @@ -47,7 +47,7 @@ function newestArtifact(ctx: QueueContext, slug: string, run: number, name: stri return hits[0]!.expired ? 'unreadable' : hits[0]!.name; } -interface Outcomes { panels?: Array<{ case?: string; status?: string; failsLane?: boolean; redClass?: string | null }>; flakyLines?: string[] } +interface Outcomes { panels?: Array<{ case?: string; status?: string; failsLane?: boolean; redClass?: string | null }>; failures?: unknown[]; flakyLines?: string[] } /** Read one screen run: status, plan, gate binding, panels. */ export function screenVerdict(ctx: QueueContext, slug: string, run: number): ScreenOutcome { @@ -78,7 +78,9 @@ export function screenVerdict(ctx: QueueContext, slug: string, run: number): Scr if (!binding) return { kind: 'unknown', detail: `run ${run}'s ${bindingName} artifact is unreadable` }; const flaky = [...(binding.flaky_cases ?? [])].sort(); const flakyLines = Array.isArray(outcomes?.flakyLines) ? outcomes!.flakyLines!.filter((l): l is string => typeof l === 'string') : []; - const red = (outcomes?.panels ?? []).filter(p => typeof p.case === 'string' && (p.failsLane === true || (p.status === 'FAIL' && p.redClass === 'VERDICT'))).map(p => p.case!).sort(); + const panelRed = (outcomes?.panels ?? []).filter(p => typeof p.case === 'string' && (p.failsLane === true || (p.status === 'FAIL' && p.redClass === 'VERDICT'))).map(p => p.case!); + const lineRed = (Array.isArray(outcomes?.failures) ? outcomes!.failures : []).flatMap(l => typeof l === 'string' ? /^✗ (\S+) {2}/.exec(l)?.[1] ?? [] : []); + const red = [...new Set([...panelRed, ...lineRed])].sort(); const base = `verdict ${binding.verdict}, reconcile exit ${binding.reconcile_exit}, run concluded ${view.conclusion ?? 'unknown'}${flaky.length ? `, flaky ${flaky.length}` : ''}`; if (binding.verdict === 'GREEN' && binding.reconcile_exit === 0 && view.conclusion === 'success') return { kind: 'green', detail: base, flaky, flakyLines }; if (binding.verdict === 'RED' || binding.reconcile_exit !== 0 || red.length) return { kind: 'red', detail: base, red, flaky, flakyLines }; diff --git a/package.json b/package.json index d19cceffc..5a7e670fd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.94.31", + "version": "1.94.32", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/test/fixtures/screen-verdict-41662815506/collector-outcomes.json b/test/fixtures/screen-verdict-41662815506/collector-outcomes.json new file mode 100644 index 000000000..1c6f4dec4 --- /dev/null +++ b/test/fixtures/screen-verdict-41662815506/collector-outcomes.json @@ -0,0 +1,144 @@ +{ + "version": 2, + "totals": { + "executed": 124, + "reused": 0, + "passed": 117, + "failed": 7, + "manual_accepted": 0, + "attempts": 124, + "total": 124, + "flaky": 0 + }, + "verdict": { + "lane": "gate/pr", + "verdict": "RED", + "attempt": 1, + "counts": { + "rule": { + "passed": 105, + "total": 112 + }, + "behavior": { + "passed": 1, + "total": 1, + "split": 0 + }, + "judge": { + "passed": 24, + "total": 24 + }, + "quarantined": { + "total": 0, + "failingLane": 0 + }, + "skipped": 0, + "infra": 0, + "incomplete": 0, + "unattributed": 0, + "flaky": 1 + }, + "reruns": { + "executions": 8, + "usd": 0.43, + "unknown": 7 + }, + "actionRequired": 4, + "wallMs": 607684, + "costUsd": 37.91, + "redispatchEligible": false + }, + "headline": [ + "[test:paid] VERDICT RED — lane gate/pr, attempt 1", + " rule 105/112 · behavior 1/1 · judge 24/24 · quarantined 0 (0 failing the lane)", + " SKIPPED 0 · INFRA 0 · INCOMPLETE 0 · unattributed 0 · ACTION REQUIRED 4", + " wall 10m08s · cost $37.91 known + 9 of 139 trial(s) cost unknown", + " FLAKY 1 (failed once, passed on its one rule-16 rerun) · 8 rerun execution(s), cost $0.43 known + 7 of 8 trial(s) cost unknown" + ], + "census": { + "skipped": [], + "deselected": { + "sibling case of a case shard (its own shard runs it)": 302, + "Bun hook placeholder of a skipped describe block (not a test)": 6, + "periodic-tier case (this lane runs gate)": 8 + }, + "unattributed": [] + }, + "panels": [ + { + "case": "ship-frozen-scope-convergence", + "kind": "behavior", + "status": "PASS", + "passed": 3, + "n": 3, + "k": 2, + "marks": "✓✓✓", + "split": false, + "quarantined": false, + "failsLane": false, + "redClass": null, + "reason": "3/3 passed", + "trials": [ + { + "trial": 1, + "outcome": "passed" + }, + { + "trial": 2, + "outcome": "passed" + }, + { + "trial": 3, + "outcome": "passed" + } + ] + } + ], + "failures": [ + "✗ qa-quick rule FAIL timeout at turn 10 — expect(received).toContain(expected) · cause session_timeout: armed 300000ms session timeout fired Expected: \"timeout\" · Received: [ \"success\", \"error_max_turns\" ] [slice 12, attempt 1] evidence: paid-slice-12-a1/shards/skill-e2e-qa-workflow--qa-quick (fetch: bun run eval:pass-rates --run 41662815506 --case qa-quick) after a repair: bun run scripts/test-paid-shards.ts --tier gate --case qa-quick", + "✗ browse-basic rule FAIL timeout — test timed out · cause session_timeout: armed 120000ms session timeout fired [slice 15, attempt 1] evidence: paid-slice-15-a1/shards/skill-e2e-bws (fetch: bun run eval:pass-rates --run 41662815506 --case browse-basic) after a repair: bun run scripts/test-paid-shards.ts --tier gate --case browse-basic", + "✗ browse-snapshot rule FAIL timeout — test timed out · cause session_timeout: armed 120000ms session timeout fired [slice 15, attempt 1] evidence: paid-slice-15-a1/shards/skill-e2e-bws (fetch: bun run eval:pass-rates --run 41662815506 --case browse-snapshot) after a repair: bun run scripts/test-paid-shards.ts --tier gate --case browse-snapshot", + "✗ skillmd-setup-discovery rule FAIL timeout — test timed out · cause session_timeout: armed 120000ms session timeout fired [slice 15, attempt 1] evidence: paid-slice-15-a1/shards/skill-e2e-bws (fetch: bun run eval:pass-rates --run 41662815506 --case skillmd-setup-discovery) after a repair: bun run scripts/test-paid-shards.ts --tier gate --case skillmd-setup-discovery", + "✗ skillmd-no-local-binary rule FAIL timeout — test timed out · cause session_timeout: armed 120000ms session timeout fired [slice 15, attempt 1] evidence: paid-slice-15-a1/shards/skill-e2e-bws (fetch: bun run eval:pass-rates --run 41662815506 --case skillmd-no-local-binary) after a repair: bun run scripts/test-paid-shards.ts --tier gate --case skillmd-no-local-binary", + "✗ skillmd-outside-git rule FAIL timeout — test timed out · cause unknown: case budget expired with no session evidence [slice 15, attempt 1] evidence: paid-slice-15-a1/shards/skill-e2e-bws (fetch: bun run eval:pass-rates --run 41662815506 --case skillmd-outside-git) after a repair: bun run scripts/test-paid-shards.ts --tier gate --case skillmd-outside-git", + "✗ diagram-triplet rule FAIL assertion — ENOENT: no such file or directory, open ' · cause assertion [slice 17, attempt 1] evidence: paid-slice-17-a1/shards/skill-e2e-diagram (fetch: bun run eval:pass-rates --run 41662815506 --case diagram-triplet) after a repair: bun run scripts/test-paid-shards.ts --tier gate --case diagram-triplet", + "✗ qa-quick rule FAILED TWICE (rule 16: a real red): attempt 1 (paid-slice-12-a1); again in attempt 1 rerun (slice 1 attempt 1) — timeout: expect(received).toContain(expected)", + "✗ browse-basic rule FAILED TWICE (rule 16: a real red): attempt 1 (paid-slice-15-a1); again in attempt 1 rerun (slice 1 attempt 1) — timeout: test timed out", + "✗ browse-snapshot rule FAILED TWICE (rule 16: a real red): attempt 1 (paid-slice-15-a1); again in attempt 1 rerun (slice 1 attempt 1) — timeout: test timed out", + "✗ skillmd-setup-discovery rule FAILED TWICE (rule 16: a real red): attempt 1 (paid-slice-15-a1); again in attempt 1 rerun (slice 1 attempt 1) — timeout: test timed out", + "✗ skillmd-no-local-binary rule FAILED TWICE (rule 16: a real red): attempt 1 (paid-slice-15-a1); again in attempt 1 rerun (slice 1 attempt 1) — timeout: test timed out", + "✗ skillmd-outside-git rule FAILED TWICE (rule 16: a real red): attempt 1 (paid-slice-15-a1); again in attempt 1 rerun (slice 1 attempt 1) — timeout: test timed out", + "✗ diagram-triplet rule FAILED TWICE (rule 16: a real red): attempt 1 (paid-slice-17-a1); again in attempt 1 rerun (slice 1 attempt 1) — assertion: ENOENT: no such file or directory, open '" + ], + "flaky": [ + { + "schema": 1, + "kind": "gate-flake", + "case": "ship-docsync-stale-after", + "case_kind": "rule", + "repository": "garrytan/gstack", + "pr": 3947, + "head_sha": "17ffac36004e20a63390cafab55521d2477b2b3d", + "tested_sha": "95fc2105ff24d114d5aa078c8a46f860a0e0a8d9", + "lane": "gate/pr", + "run_id": "41662815506", + "first": { + "attempt": 1, + "phase": "first", + "where": "attempt 1 (paid-slice-13-a1)" + }, + "rerun": { + "attempt": 1, + "phase": "rerun", + "where": "attempt 1 rerun (slice 1 attempt 1)" + }, + "symptom": "assertion: expect(received).toEqual(expected)", + "rerun_cost_usd": 0.43, + "recorded_at": "2026-11-03T15:43:25.811Z" + } + ], + "flakyLines": [ + "⚑ ship-docsync-stale-after rule FLAKY: failed in attempt 1 (paid-slice-13-a1), passed on its one rerun in attempt 1 rerun (slice 1 attempt 1); rerun cost $0.43; first failure: assertion: expect(received).toEqual(expected)" + ], + "flakyCollectorFailures": 1 +} diff --git a/test/fixtures/screen-verdict-41662815506/gate-binding.json b/test/fixtures/screen-verdict-41662815506/gate-binding.json new file mode 100644 index 000000000..1131c6ec9 --- /dev/null +++ b/test/fixtures/screen-verdict-41662815506/gate-binding.json @@ -0,0 +1,20 @@ +{ + "schema": 1, + "repository": "garrytan/gstack", + "pr": 3947, + "run_id": 41662815506, + "run_attempt": 1, + "head_sha": "17ffac36004e20a63390cafab55521d2477b2b3d", + "base_sha": "95fc2105ff24d114d5aa078c8a46f860a0e0a8d9", + "profile": "pr", + "reconcile_exit": 1, + "verdict": "RED", + "manual_accepted": 0, + "selection_sha256": "4e4ff8ea430bf850a9f88a7a473f353b8a4f6a99387aa592335ee38cd10270a4", + "flaky_cases": [ + "ship-docsync-stale-after" + ], + "purpose": "screen", + "tested_sha": "472c733bfbed9421df7859b07b902b98c07b79b2", + "screen_policy": "narrow" +} diff --git a/test/helpers/free-fixtures-data.ts b/test/helpers/free-fixtures-data.ts index d0362f72f..788a759cd 100644 --- a/test/helpers/free-fixtures-data.ts +++ b/test/helpers/free-fixtures-data.ts @@ -215,6 +215,7 @@ export const FREE_FIXTURES: Readonly> = { 'test/fixtures/review-design-lite-reports-ci-36633323521.json': ['test/review-finalization-budget.test.ts'], 'test/fixtures/review-gate-integrity-opus-reports.json': ['test/gate-integrity-templates.test.ts'], 'test/fixtures/review-wrong-tree/**': ['test/review-dispatch.test.ts'], + 'test/fixtures/screen-verdict-41662815506/**': ['test/ship-queue-screen-verdict.test.ts'], 'test/fixtures/sealed-guard-off-golden.json': ['test/sealed-guard.test.ts'], 'test/fixtures/session-update-heartbeat.sh': ['test/session-update-heartbeat.test.ts'], 'test/fixtures/shadow-fake-model.ts': ['test/autoplan-shadow-driver.test.ts', 'test/autoplan-shadow.test.ts'], diff --git a/test/ship-queue-screen-verdict.test.ts b/test/ship-queue-screen-verdict.test.ts new file mode 100644 index 000000000..a1ad4ffeb --- /dev/null +++ b/test/ship-queue-screen-verdict.test.ts @@ -0,0 +1,94 @@ +/** + * The screen verdict names every red case, rule cases included, from the + * run's real artifacts. Run 41662815506 (#3947's screen) is a correct RED: + * seven rule cases failed twice under rule 16, and ship-docsync-stale-after + * failed once and passed on its rerun. Before this fix the verdict line named + * only the flaky case, so the RED read as if the rerun-pass case caused it. + * Fixtures: test/fixtures/screen-verdict-41662815506 holds the run's + * gate-binding.json and its collector-outcomes.json minus the per-file list. + */ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { screenVerdict, verdictLines } from '../lib/ship-queue-screen'; +import type { QueueContext } from '../lib/ship-queue'; + +const RUN = 41662815506; +const SLUG = 'garrytan/gstack'; +const FIXTURE = path.join(import.meta.dir, 'fixtures', 'screen-verdict-41662815506'); +const realBinding = () => JSON.parse(fs.readFileSync(path.join(FIXTURE, 'gate-binding.json'), 'utf8')); +const realOutcomes = () => JSON.parse(fs.readFileSync(path.join(FIXTURE, 'collector-outcomes.json'), 'utf8')); +const TWICE = ['browse-basic', 'browse-snapshot', 'diagram-triplet', 'qa-quick', 'skillmd-no-local-binary', 'skillmd-outside-git', 'skillmd-setup-discovery']; + +function ctxFor(conclusion: string, binding: unknown, outcomes: unknown): QueueContext { + const files: Record> = { + 'paid-plan': { 'manifest.json': { entries: [{ status: 'planned' }] } }, + 'gate-binding-a1': { 'gate-binding.json': binding }, + 'report-verdict-a1': { 'paid-report/collector-outcomes.json': outcomes }, + }; + const artifacts = Object.keys(files).map(name => ({ name, expired: false })); + return { + repoRoot: '/nonexistent', base: 'main', + gh: (args) => { + if (args[0] === 'run' && args[1] === 'view') return { ok: true, out: JSON.stringify({ databaseId: RUN, status: 'completed', conclusion }) }; + if (args[0] === 'api') return { ok: true, out: JSON.stringify({ artifacts: args[1]!.includes('name=paid-plan') ? artifacts.filter(a => a.name === 'paid-plan') : artifacts }) }; + if (args[0] === 'run' && args[1] === 'download') { + const name = args[args.indexOf('-n') + 1]!; + const dir = args[args.indexOf('-D') + 1]!; + if (!files[name]) return { ok: false, out: '', err: 'no artifact' }; + for (const [rel, body] of Object.entries(files[name]!)) { + fs.mkdirSync(path.dirname(path.join(dir, rel)), { recursive: true }); + fs.writeFileSync(path.join(dir, rel), JSON.stringify(body)); + } + return { ok: true, out: '' }; + } + return { ok: false, out: '', err: `unexpected gh ${args.join(' ')}` }; + }, + }; +} + +describe('screen verdict from run 41662815506', () => { + test('a rule case that failed twice is named red; the rerun-pass case is only flaky', () => { + // Value: protects=SCREEN_VERDICT and QUEUE_SCREEN_RED name the rule reds; fails_when=red names come from behavior/judge panels only; why_new=the train fixture models reds as judge panels; seam=none; no_claim=gh is a stub serving the run's downloaded artifacts + const read = screenVerdict(ctxFor('failure', realBinding(), realOutcomes()), SLUG, RUN); + expect(read.kind).toBe('red'); + if (read.kind !== 'red') return; + expect(read.red).toEqual(TWICE); + expect(read.flaky).toEqual(['ship-docsync-stale-after']); + const lines = verdictLines('3947', RUN, read); + expect(lines[0]).toBe(`SCREEN_VERDICT: #3947 run ${RUN} RED red=${TWICE.join(',')} flaky=ship-docsync-stale-after (verdict RED, reconcile exit 1, run concluded failure, flaky 1)`); + expect(lines.slice(1)).toEqual([expect.stringMatching(/^FLAKY: ⚑ ship-docsync-stale-after {2}rule {2}FLAKY: failed in attempt 1/)]); + }); + + test('negative control: with only the rerun-pass case left, the run reads GREEN with its FLAKY line', () => { + const outcomes = { ...realOutcomes(), failures: [] }; + const read = screenVerdict(ctxFor('success', { ...realBinding(), verdict: 'GREEN', reconcile_exit: 0 }, outcomes), SLUG, RUN); + expect(read.kind).toBe('green'); + if (read.kind !== 'green') return; + expect(read.flaky).toEqual(['ship-docsync-stale-after']); + expect(verdictLines('3947', RUN, read)).toEqual([ + `SCREEN_VERDICT: #3947 run ${RUN} GREEN flaky=ship-docsync-stale-after (verdict GREEN, reconcile exit 0, run concluded success, flaky 1)`, + `FLAKY: ${realOutcomes().flakyLines[0]}`, + ]); + }); + + test('a red that never reran (INFRA, harness, unattributed shard) is still named red', () => { + const outcomes = { ...realOutcomes(), failures: [ + '✗ infra-case rule FAIL infra — runner lost [slice 3, attempt 1]', + '✗ test/skill-e2e-x.test.ts rule shard NOT REPORTED [slice 4, attempt 1]', + ] }; + const read = screenVerdict(ctxFor('failure', realBinding(), outcomes), SLUG, RUN); + expect(read.kind).toBe('red'); + if (read.kind === 'red') expect(read.red).toEqual(['infra-case', 'test/skill-e2e-x.test.ts']); + }); + + test('a quarantined or passing split panel line is not named red', () => { + const outcomes = { ...realOutcomes(), failures: [ + '◌ quiet-case behavior (quarantined) FAIL 1/3 (✓✗✗) [t1@slice 2, attempt 1]', + '⚠ split-case behavior PASS 2/3 (✓✓✗) [t1@slice 2, attempt 1]', + ] }; + const read = screenVerdict(ctxFor('failure', realBinding(), outcomes), SLUG, RUN); + expect(read.kind).toBe('red'); + if (read.kind === 'red') expect(read.red).toEqual([]); + }); +});