mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-12 19:44:48 +01:00
fix(ship-queue): a RED screen names its rule-case reds, not just panel reds (GSTA-258 L8, run 41662815506)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed
1 parent
1ccab0d8bc
commit
ee46634264
5 files changed
+269
-2
No files matched your search
@@ -0,0 +1,144 @@
|
||||
{
|
||||
"version": 2,
|
||||
"totals": {
|
||||
"executed": 124,
|
||||
"reused": 0,
|
||||
"passed": 117,
|
||||
"failed": 7,
|
||||
"manual_accepted": 0,
|
||||
"attempts": 124,
|
||||
"total": 124,
|
||||
"flaky": 0
|
||||
},
|
||||
"verdict": {
|
||||
"lane": "gate/pr",
|
||||
"verdict": "RED",
|
||||
"attempt": 1,
|
||||
"counts": {
|
||||
"rule": {
|
||||
"passed": 105,
|
||||
"total": 112
|
||||
},
|
||||
"behavior": {
|
||||
"passed": 1,
|
||||
"total": 1,
|
||||
"split": 0
|
||||
},
|
||||
"judge": {
|
||||
"passed": 24,
|
||||
"total": 24
|
||||
},
|
||||
"quarantined": {
|
||||
"total": 0,
|
||||
"failingLane": 0
|
||||
},
|
||||
"skipped": 0,
|
||||
"infra": 0,
|
||||
"incomplete": 0,
|
||||
"unattributed": 0,
|
||||
"flaky": 1
|
||||
},
|
||||
"reruns": {
|
||||
"executions": 8,
|
||||
"usd": 0.43,
|
||||
"unknown": 7
|
||||
},
|
||||
"actionRequired": 4,
|
||||
"wallMs": 607684,
|
||||
"costUsd": 37.91,
|
||||
"redispatchEligible": false
|
||||
},
|
||||
"headline": [
|
||||
"[test:paid] VERDICT RED — lane gate/pr, attempt 1",
|
||||
" rule 105/112 · behavior 1/1 · judge 24/24 · quarantined 0 (0 failing the lane)",
|
||||
" SKIPPED 0 · INFRA 0 · INCOMPLETE 0 · unattributed 0 · ACTION REQUIRED 4",
|
||||
" wall 10m08s · cost $37.91 known + 9 of 139 trial(s) cost unknown",
|
||||
" FLAKY 1 (failed once, passed on its one rule-16 rerun) · 8 rerun execution(s), cost $0.43 known + 7 of 8 trial(s) cost unknown"
|
||||
],
|
||||
"census": {
|
||||
"skipped": [],
|
||||
"deselected": {
|
||||
"sibling case of a case shard (its own shard runs it)": 302,
|
||||
"Bun hook placeholder of a skipped describe block (not a test)": 6,
|
||||
"periodic-tier case (this lane runs gate)": 8
|
||||
},
|
||||
"unattributed": []
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"case": "ship-frozen-scope-convergence",
|
||||
"kind": "behavior",
|
||||
"status": "PASS",
|
||||
"passed": 3,
|
||||
"n": 3,
|
||||
"k": 2,
|
||||
"marks": "✓✓✓",
|
||||
"split": false,
|
||||
"quarantined": false,
|
||||
"failsLane": false,
|
||||
"redClass": null,
|
||||
"reason": "3/3 passed",
|
||||
"trials": [
|
||||
{
|
||||
"trial": 1,
|
||||
"outcome": "passed"
|
||||
},
|
||||
{
|
||||
"trial": 2,
|
||||
"outcome": "passed"
|
||||
},
|
||||
{
|
||||
"trial": 3,
|
||||
"outcome": "passed"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"failures": [
|
||||
"✗ qa-quick rule FAIL timeout at turn 10 — expect(received).toContain(expected) · cause session_timeout: armed 300000ms session timeout fired Expected: \"timeout\" · Received: [ \"success\", \"error_max_turns\" ] [slice 12, attempt 1] evidence: paid-slice-12-a1/shards/skill-e2e-qa-workflow--qa-quick (fetch: bun run eval:pass-rates --run 41662815506 --case qa-quick) after a repair: bun run scripts/test-paid-shards.ts --tier gate --case qa-quick",
|
||||
"✗ browse-basic rule FAIL timeout — test timed out · cause session_timeout: armed 120000ms session timeout fired [slice 15, attempt 1] evidence: paid-slice-15-a1/shards/skill-e2e-bws (fetch: bun run eval:pass-rates --run 41662815506 --case browse-basic) after a repair: bun run scripts/test-paid-shards.ts --tier gate --case browse-basic",
|
||||
"✗ browse-snapshot rule FAIL timeout — test timed out · cause session_timeout: armed 120000ms session timeout fired [slice 15, attempt 1] evidence: paid-slice-15-a1/shards/skill-e2e-bws (fetch: bun run eval:pass-rates --run 41662815506 --case browse-snapshot) after a repair: bun run scripts/test-paid-shards.ts --tier gate --case browse-snapshot",
|
||||
"✗ skillmd-setup-discovery rule FAIL timeout — test timed out · cause session_timeout: armed 120000ms session timeout fired [slice 15, attempt 1] evidence: paid-slice-15-a1/shards/skill-e2e-bws (fetch: bun run eval:pass-rates --run 41662815506 --case skillmd-setup-discovery) after a repair: bun run scripts/test-paid-shards.ts --tier gate --case skillmd-setup-discovery",
|
||||
"✗ skillmd-no-local-binary rule FAIL timeout — test timed out · cause session_timeout: armed 120000ms session timeout fired [slice 15, attempt 1] evidence: paid-slice-15-a1/shards/skill-e2e-bws (fetch: bun run eval:pass-rates --run 41662815506 --case skillmd-no-local-binary) after a repair: bun run scripts/test-paid-shards.ts --tier gate --case skillmd-no-local-binary",
|
||||
"✗ skillmd-outside-git rule FAIL timeout — test timed out · cause unknown: case budget expired with no session evidence [slice 15, attempt 1] evidence: paid-slice-15-a1/shards/skill-e2e-bws (fetch: bun run eval:pass-rates --run 41662815506 --case skillmd-outside-git) after a repair: bun run scripts/test-paid-shards.ts --tier gate --case skillmd-outside-git",
|
||||
"✗ diagram-triplet rule FAIL assertion — ENOENT: no such file or directory, open '<REDACTED-internal.user_path> · cause assertion [slice 17, attempt 1] evidence: paid-slice-17-a1/shards/skill-e2e-diagram (fetch: bun run eval:pass-rates --run 41662815506 --case diagram-triplet) after a repair: bun run scripts/test-paid-shards.ts --tier gate --case diagram-triplet",
|
||||
"✗ qa-quick rule FAILED TWICE (rule 16: a real red): attempt 1 (paid-slice-12-a1); again in attempt 1 rerun (slice 1 attempt 1) — timeout: expect(received).toContain(expected)",
|
||||
"✗ browse-basic rule FAILED TWICE (rule 16: a real red): attempt 1 (paid-slice-15-a1); again in attempt 1 rerun (slice 1 attempt 1) — timeout: test timed out",
|
||||
"✗ browse-snapshot rule FAILED TWICE (rule 16: a real red): attempt 1 (paid-slice-15-a1); again in attempt 1 rerun (slice 1 attempt 1) — timeout: test timed out",
|
||||
"✗ skillmd-setup-discovery rule FAILED TWICE (rule 16: a real red): attempt 1 (paid-slice-15-a1); again in attempt 1 rerun (slice 1 attempt 1) — timeout: test timed out",
|
||||
"✗ skillmd-no-local-binary rule FAILED TWICE (rule 16: a real red): attempt 1 (paid-slice-15-a1); again in attempt 1 rerun (slice 1 attempt 1) — timeout: test timed out",
|
||||
"✗ skillmd-outside-git rule FAILED TWICE (rule 16: a real red): attempt 1 (paid-slice-15-a1); again in attempt 1 rerun (slice 1 attempt 1) — timeout: test timed out",
|
||||
"✗ diagram-triplet rule FAILED TWICE (rule 16: a real red): attempt 1 (paid-slice-17-a1); again in attempt 1 rerun (slice 1 attempt 1) — assertion: ENOENT: no such file or directory, open '<REDACTED-internal.user_path>"
|
||||
],
|
||||
"flaky": [
|
||||
{
|
||||
"schema": 1,
|
||||
"kind": "gate-flake",
|
||||
"case": "ship-docsync-stale-after",
|
||||
"case_kind": "rule",
|
||||
"repository": "garrytan/gstack",
|
||||
"pr": 3947,
|
||||
"head_sha": "17ffac36004e20a63390cafab55521d2477b2b3d",
|
||||
"tested_sha": "95fc2105ff24d114d5aa078c8a46f860a0e0a8d9",
|
||||
"lane": "gate/pr",
|
||||
"run_id": "41662815506",
|
||||
"first": {
|
||||
"attempt": 1,
|
||||
"phase": "first",
|
||||
"where": "attempt 1 (paid-slice-13-a1)"
|
||||
},
|
||||
"rerun": {
|
||||
"attempt": 1,
|
||||
"phase": "rerun",
|
||||
"where": "attempt 1 rerun (slice 1 attempt 1)"
|
||||
},
|
||||
"symptom": "assertion: expect(received).toEqual(expected)",
|
||||
"rerun_cost_usd": 0.43,
|
||||
"recorded_at": "2026-11-03T15:43:25.811Z"
|
||||
}
|
||||
],
|
||||
"flakyLines": [
|
||||
"⚑ ship-docsync-stale-after rule FLAKY: failed in attempt 1 (paid-slice-13-a1), passed on its one rerun in attempt 1 rerun (slice 1 attempt 1); rerun cost $0.43; first failure: assertion: expect(received).toEqual(expected)"
|
||||
],
|
||||
"flakyCollectorFailures": 1
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"repository": "garrytan/gstack",
|
||||
"pr": 3947,
|
||||
"run_id": 41662815506,
|
||||
"run_attempt": 1,
|
||||
"head_sha": "17ffac36004e20a63390cafab55521d2477b2b3d",
|
||||
"base_sha": "95fc2105ff24d114d5aa078c8a46f860a0e0a8d9",
|
||||
"profile": "pr",
|
||||
"reconcile_exit": 1,
|
||||
"verdict": "RED",
|
||||
"manual_accepted": 0,
|
||||
"selection_sha256": "4e4ff8ea430bf850a9f88a7a473f353b8a4f6a99387aa592335ee38cd10270a4",
|
||||
"flaky_cases": [
|
||||
"ship-docsync-stale-after"
|
||||
],
|
||||
"purpose": "screen",
|
||||
"tested_sha": "472c733bfbed9421df7859b07b902b98c07b79b2",
|
||||
"screen_policy": "narrow"
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
/**
|
||||
* The screen verdict names every red case, rule cases included, from the
|
||||
* run's real artifacts. Run 41662815506 (#3947's screen) is a correct RED:
|
||||
* seven rule cases failed twice under rule 16, and ship-docsync-stale-after
|
||||
* failed once and passed on its rerun. Before this fix the verdict line named
|
||||
* only the flaky case, so the RED read as if the rerun-pass case caused it.
|
||||
* Fixtures: test/fixtures/screen-verdict-41662815506 holds the run's
|
||||
* gate-binding.json and its collector-outcomes.json minus the per-file list.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { screenVerdict, verdictLines } from '../lib/ship-queue-screen';
|
||||
import type { QueueContext } from '../lib/ship-queue';
|
||||
|
||||
const RUN = 41662815506;
|
||||
const SLUG = 'garrytan/gstack';
|
||||
const FIXTURE = path.join(import.meta.dir, 'fixtures', 'screen-verdict-41662815506');
|
||||
const realBinding = () => JSON.parse(fs.readFileSync(path.join(FIXTURE, 'gate-binding.json'), 'utf8'));
|
||||
const realOutcomes = () => JSON.parse(fs.readFileSync(path.join(FIXTURE, 'collector-outcomes.json'), 'utf8'));
|
||||
const TWICE = ['browse-basic', 'browse-snapshot', 'diagram-triplet', 'qa-quick', 'skillmd-no-local-binary', 'skillmd-outside-git', 'skillmd-setup-discovery'];
|
||||
|
||||
function ctxFor(conclusion: string, binding: unknown, outcomes: unknown): QueueContext {
|
||||
const files: Record<string, Record<string, unknown>> = {
|
||||
'paid-plan': { 'manifest.json': { entries: [{ status: 'planned' }] } },
|
||||
'gate-binding-a1': { 'gate-binding.json': binding },
|
||||
'report-verdict-a1': { 'paid-report/collector-outcomes.json': outcomes },
|
||||
};
|
||||
const artifacts = Object.keys(files).map(name => ({ name, expired: false }));
|
||||
return {
|
||||
repoRoot: '/nonexistent', base: 'main',
|
||||
gh: (args) => {
|
||||
if (args[0] === 'run' && args[1] === 'view') return { ok: true, out: JSON.stringify({ databaseId: RUN, status: 'completed', conclusion }) };
|
||||
if (args[0] === 'api') return { ok: true, out: JSON.stringify({ artifacts: args[1]!.includes('name=paid-plan') ? artifacts.filter(a => a.name === 'paid-plan') : artifacts }) };
|
||||
if (args[0] === 'run' && args[1] === 'download') {
|
||||
const name = args[args.indexOf('-n') + 1]!;
|
||||
const dir = args[args.indexOf('-D') + 1]!;
|
||||
if (!files[name]) return { ok: false, out: '', err: 'no artifact' };
|
||||
for (const [rel, body] of Object.entries(files[name]!)) {
|
||||
fs.mkdirSync(path.dirname(path.join(dir, rel)), { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, rel), JSON.stringify(body));
|
||||
}
|
||||
return { ok: true, out: '' };
|
||||
}
|
||||
return { ok: false, out: '', err: `unexpected gh ${args.join(' ')}` };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe('screen verdict from run 41662815506', () => {
|
||||
test('a rule case that failed twice is named red; the rerun-pass case is only flaky', () => {
|
||||
// Value: protects=SCREEN_VERDICT and QUEUE_SCREEN_RED name the rule reds; fails_when=red names come from behavior/judge panels only; why_new=the train fixture models reds as judge panels; seam=none; no_claim=gh is a stub serving the run's downloaded artifacts
|
||||
const read = screenVerdict(ctxFor('failure', realBinding(), realOutcomes()), SLUG, RUN);
|
||||
expect(read.kind).toBe('red');
|
||||
if (read.kind !== 'red') return;
|
||||
expect(read.red).toEqual(TWICE);
|
||||
expect(read.flaky).toEqual(['ship-docsync-stale-after']);
|
||||
const lines = verdictLines('3947', RUN, read);
|
||||
expect(lines[0]).toBe(`SCREEN_VERDICT: #3947 run ${RUN} RED red=${TWICE.join(',')} flaky=ship-docsync-stale-after (verdict RED, reconcile exit 1, run concluded failure, flaky 1)`);
|
||||
expect(lines.slice(1)).toEqual([expect.stringMatching(/^FLAKY: ⚑ ship-docsync-stale-after {2}rule {2}FLAKY: failed in attempt 1/)]);
|
||||
});
|
||||
|
||||
test('negative control: with only the rerun-pass case left, the run reads GREEN with its FLAKY line', () => {
|
||||
const outcomes = { ...realOutcomes(), failures: [] };
|
||||
const read = screenVerdict(ctxFor('success', { ...realBinding(), verdict: 'GREEN', reconcile_exit: 0 }, outcomes), SLUG, RUN);
|
||||
expect(read.kind).toBe('green');
|
||||
if (read.kind !== 'green') return;
|
||||
expect(read.flaky).toEqual(['ship-docsync-stale-after']);
|
||||
expect(verdictLines('3947', RUN, read)).toEqual([
|
||||
`SCREEN_VERDICT: #3947 run ${RUN} GREEN flaky=ship-docsync-stale-after (verdict GREEN, reconcile exit 0, run concluded success, flaky 1)`,
|
||||
`FLAKY: ${realOutcomes().flakyLines[0]}`,
|
||||
]);
|
||||
});
|
||||
|
||||
test('a red that never reran (INFRA, harness, unattributed shard) is still named red', () => {
|
||||
const outcomes = { ...realOutcomes(), failures: [
|
||||
'✗ infra-case rule FAIL infra — runner lost [slice 3, attempt 1]',
|
||||
'✗ test/skill-e2e-x.test.ts rule shard NOT REPORTED [slice 4, attempt 1]',
|
||||
] };
|
||||
const read = screenVerdict(ctxFor('failure', realBinding(), outcomes), SLUG, RUN);
|
||||
expect(read.kind).toBe('red');
|
||||
if (read.kind === 'red') expect(read.red).toEqual(['infra-case', 'test/skill-e2e-x.test.ts']);
|
||||
});
|
||||
|
||||
test('a quarantined or passing split panel line is not named red', () => {
|
||||
const outcomes = { ...realOutcomes(), failures: [
|
||||
'◌ quiet-case behavior (quarantined) FAIL 1/3 (✓✗✗) [t1@slice 2, attempt 1]',
|
||||
'⚠ split-case behavior PASS 2/3 (✓✓✗) [t1@slice 2, attempt 1]',
|
||||
] };
|
||||
const read = screenVerdict(ctxFor('failure', realBinding(), outcomes), SLUG, RUN);
|
||||
expect(read.kind).toBe('red');
|
||||
if (read.kind === 'red') expect(read.red).toEqual([]);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user