feat: green-by-skip census — skip counts in the classifier, all-skipped labeling in the paid runner

bun's 'Ran N tests' line COUNTS skipped tests, so a codex/gemini shard
whose every test self-skipped (binary absent on the runner — true of every
CI runner today) exits 0, dodges the hollow-shard guard, and reads as
coverage in the weekly census. The classifier now parses bun's ' N skip' /
' N pass' recap lines; ShardOutcome carries skippedTests; formatSummary and
the fail-closed slices report label an all-skipped pass explicitly:
'all N tests SKIPPED — verified nothing'. Status stays 'passed' (external
service availability is host state, not a repo regression) but the census
can no longer mistake absence for coverage.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-31 04:06:45 +00:00
co-authored by Claude Fable 5
parent d96027c098
commit 7788f7197e
4 changed files with 136 additions and 5 deletions
+37
View File
@@ -22,6 +22,7 @@ import {
computePaidDiffSelection,
diffSkipDecisionForFile,
formatSummary,
isAllSkippedPass,
isPaidTestFile,
knownTestNamesInSource,
partitionShardsByDiffSelection,
@@ -354,3 +355,39 @@ describe('parent-side diff shard skipping', () => {
expect(summaryExitCode(withNeverStarted)).toBe(1);
});
});
// Green-by-skip census: "Ran N tests" counts skips, so a codex/gemini file
// whose every test self-skipped (binary absent on the runner) exits 0 and
// used to read as coverage in the weekly report. The census label keeps the
// pass (service availability is host state, not a repo regression) but must
// say the shard verified nothing.
describe('all-skipped pass census', () => {
const base = { shard: 1, files: ['test/codex-e2e.test.ts'], exitCode: 0, elapsedMs: 1200, groupPid: 1 };
test('isAllSkippedPass: pass with every test skipped → true', () => {
expect(isAllSkippedPass({ ...base, status: 'passed', executedTests: 8, skippedTests: 8 } as ShardOutcome)).toBe(true);
});
test('isAllSkippedPass: real work, a failure, or no data → false', () => {
// one test actually ran
expect(isAllSkippedPass({ ...base, status: 'passed', executedTests: 8, skippedTests: 7 } as ShardOutcome)).toBe(false);
// zero tests: that's the hollow-shard guard's territory, not this label's
expect(isAllSkippedPass({ ...base, status: 'passed', executedTests: 0, skippedTests: 0 } as ShardOutcome)).toBe(false);
// non-pass statuses never get the label
expect(isAllSkippedPass({ ...base, status: 'failed', executedTests: 8, skippedTests: 8 } as ShardOutcome)).toBe(false);
// stream gave no counts (crash/timeout) — unknown, not all-skipped
expect(isAllSkippedPass({ ...base, status: 'passed', executedTests: null, skippedTests: null } as ShardOutcome)).toBe(false);
});
test('formatSummary labels an all-skipped pass and leaves real passes alone', () => {
const lines = formatSummary(summarize([
{ ...base, status: 'passed', executedTests: 8, skippedTests: 8 } as ShardOutcome,
{ shard: 2, files: ['test/skill-e2e-review.test.ts'], status: 'passed', exitCode: 0, elapsedMs: 900, groupPid: 2, executedTests: 3, skippedTests: 0 } as ShardOutcome,
]));
const codexLine = lines.find((l) => l.includes('codex-e2e'));
const reviewLine = lines.find((l) => l.includes('skill-e2e-review'));
expect(codexLine).toContain('all 8 tests SKIPPED');
expect(codexLine).toContain('verified nothing');
expect(reviewLine).not.toContain('SKIPPED');
});
});
+32
View File
@@ -89,6 +89,38 @@ describe('BunTestOutputClassifier', () => {
expect(summary.terminalFileCounts).toEqual([2]);
expect(strictTestExitCode(0, summary, 2)).toBe(0);
});
// "Ran N tests" COUNTS skipped tests, so the skip/pass count lines are the
// only stream evidence separating verified work from green-by-skip (a
// codex/gemini file whose every test self-skips on a binary-less runner).
it('parses the skip and pass count lines from buns recap block', () => {
const c = new BunTestOutputClassifier();
c.write(' 1 pass\n 2 skip\n 0 fail\nRan 3 tests across 1 file. [7.00ms]\n');
const summary = c.end();
expect(summary.passedTests).toBe(1);
expect(summary.skippedTests).toBe(2);
expect(summary.terminalTestCounts).toEqual([3]);
// all-skipped is still exit-0 at the classifier layer — the census
// labeling happens in the paid runner, not here
expect(strictTestExitCode(0, summary, 1)).toBe(0);
});
it('skip/pass counts survive ANSI color and chunk shears', () => {
const c = new BunTestOutputClassifier();
c.write(' 4 pa');
c.write('ss\n 9 skip\n');
const summary = c.end();
expect(summary.passedTests).toBe(4);
expect(summary.skippedTests).toBe(9);
});
it('prose mentioning skip counts does not pollute the tally', () => {
const c = new BunTestOutputClassifier();
c.write('console.log said: 7 skip is what we expect later\n');
c.write('(fail) 3 skip handling [1.00ms]\n');
const summary = c.end();
expect(summary.skippedTests).toBe(0);
});
});
describe('installChildSignalForwarding — cancellation terminates the RUN', () => {