mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-13 00:19:03 +02:00
feat: green-by-skip census — skip counts in the classifier, all-skipped labeling in the paid runner
bun's 'Ran N tests' line COUNTS skipped tests, so a codex/gemini shard whose every test self-skipped (binary absent on the runner — true of every CI runner today) exits 0, dodges the hollow-shard guard, and reads as coverage in the weekly census. The classifier now parses bun's ' N skip' / ' N pass' recap lines; ShardOutcome carries skippedTests; formatSummary and the fail-closed slices report label an all-skipped pass explicitly: 'all N tests SKIPPED — verified nothing'. Status stays 'passed' (external service availability is host state, not a repo regression) but the census can no longer mistake absence for coverage. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
d96027c098
commit
7788f7197e
@@ -22,6 +22,7 @@ import {
|
||||
computePaidDiffSelection,
|
||||
diffSkipDecisionForFile,
|
||||
formatSummary,
|
||||
isAllSkippedPass,
|
||||
isPaidTestFile,
|
||||
knownTestNamesInSource,
|
||||
partitionShardsByDiffSelection,
|
||||
@@ -354,3 +355,39 @@ describe('parent-side diff shard skipping', () => {
|
||||
expect(summaryExitCode(withNeverStarted)).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
// Green-by-skip census: "Ran N tests" counts skips, so a codex/gemini file
|
||||
// whose every test self-skipped (binary absent on the runner) exits 0 and
|
||||
// used to read as coverage in the weekly report. The census label keeps the
|
||||
// pass (service availability is host state, not a repo regression) but must
|
||||
// say the shard verified nothing.
|
||||
describe('all-skipped pass census', () => {
|
||||
const base = { shard: 1, files: ['test/codex-e2e.test.ts'], exitCode: 0, elapsedMs: 1200, groupPid: 1 };
|
||||
|
||||
test('isAllSkippedPass: pass with every test skipped → true', () => {
|
||||
expect(isAllSkippedPass({ ...base, status: 'passed', executedTests: 8, skippedTests: 8 } as ShardOutcome)).toBe(true);
|
||||
});
|
||||
|
||||
test('isAllSkippedPass: real work, a failure, or no data → false', () => {
|
||||
// one test actually ran
|
||||
expect(isAllSkippedPass({ ...base, status: 'passed', executedTests: 8, skippedTests: 7 } as ShardOutcome)).toBe(false);
|
||||
// zero tests: that's the hollow-shard guard's territory, not this label's
|
||||
expect(isAllSkippedPass({ ...base, status: 'passed', executedTests: 0, skippedTests: 0 } as ShardOutcome)).toBe(false);
|
||||
// non-pass statuses never get the label
|
||||
expect(isAllSkippedPass({ ...base, status: 'failed', executedTests: 8, skippedTests: 8 } as ShardOutcome)).toBe(false);
|
||||
// stream gave no counts (crash/timeout) — unknown, not all-skipped
|
||||
expect(isAllSkippedPass({ ...base, status: 'passed', executedTests: null, skippedTests: null } as ShardOutcome)).toBe(false);
|
||||
});
|
||||
|
||||
test('formatSummary labels an all-skipped pass and leaves real passes alone', () => {
|
||||
const lines = formatSummary(summarize([
|
||||
{ ...base, status: 'passed', executedTests: 8, skippedTests: 8 } as ShardOutcome,
|
||||
{ shard: 2, files: ['test/skill-e2e-review.test.ts'], status: 'passed', exitCode: 0, elapsedMs: 900, groupPid: 2, executedTests: 3, skippedTests: 0 } as ShardOutcome,
|
||||
]));
|
||||
const codexLine = lines.find((l) => l.includes('codex-e2e'));
|
||||
const reviewLine = lines.find((l) => l.includes('skill-e2e-review'));
|
||||
expect(codexLine).toContain('all 8 tests SKIPPED');
|
||||
expect(codexLine).toContain('verified nothing');
|
||||
expect(reviewLine).not.toContain('SKIPPED');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -89,6 +89,38 @@ describe('BunTestOutputClassifier', () => {
|
||||
expect(summary.terminalFileCounts).toEqual([2]);
|
||||
expect(strictTestExitCode(0, summary, 2)).toBe(0);
|
||||
});
|
||||
|
||||
// "Ran N tests" COUNTS skipped tests, so the skip/pass count lines are the
|
||||
// only stream evidence separating verified work from green-by-skip (a
|
||||
// codex/gemini file whose every test self-skips on a binary-less runner).
|
||||
it('parses the skip and pass count lines from bun’s recap block', () => {
|
||||
const c = new BunTestOutputClassifier();
|
||||
c.write(' 1 pass\n 2 skip\n 0 fail\nRan 3 tests across 1 file. [7.00ms]\n');
|
||||
const summary = c.end();
|
||||
expect(summary.passedTests).toBe(1);
|
||||
expect(summary.skippedTests).toBe(2);
|
||||
expect(summary.terminalTestCounts).toEqual([3]);
|
||||
// all-skipped is still exit-0 at the classifier layer — the census
|
||||
// labeling happens in the paid runner, not here
|
||||
expect(strictTestExitCode(0, summary, 1)).toBe(0);
|
||||
});
|
||||
|
||||
it('skip/pass counts survive ANSI color and chunk shears', () => {
|
||||
const c = new BunTestOutputClassifier();
|
||||
c.write('[32m 4 pa');
|
||||
c.write('ss[0m\n[33m 9 skip[0m\n');
|
||||
const summary = c.end();
|
||||
expect(summary.passedTests).toBe(4);
|
||||
expect(summary.skippedTests).toBe(9);
|
||||
});
|
||||
|
||||
it('prose mentioning skip counts does not pollute the tally', () => {
|
||||
const c = new BunTestOutputClassifier();
|
||||
c.write('console.log said: 7 skip is what we expect later\n');
|
||||
c.write('(fail) 3 skip handling [1.00ms]\n');
|
||||
const summary = c.end();
|
||||
expect(summary.skippedTests).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('installChildSignalForwarding — cancellation terminates the RUN', () => {
|
||||
|
||||
Reference in New Issue
Block a user