mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-16 18:05:31 +02:00
feat: green-by-skip census — skip counts in the classifier, all-skipped labeling in the paid runner
bun's 'Ran N tests' line COUNTS skipped tests, so a codex/gemini shard whose every test self-skipped (binary absent on the runner — true of every CI runner today) exits 0, dodges the hollow-shard guard, and reads as coverage in the weekly census. The classifier now parses bun's ' N skip' / ' N pass' recap lines; ShardOutcome carries skippedTests; formatSummary and the fail-closed slices report label an all-skipped pass explicitly: 'all N tests SKIPPED — verified nothing'. Status stays 'passed' (external service availability is host state, not a repo regression) but the census can no longer mistake absence for coverage. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
d96027c098
commit
7788f7197e
@@ -20,6 +20,15 @@ const ANSI_ESCAPE = /\u001B\[[0-?]*[ -/]*[@-~]/g;
|
||||
const BUN_FAIL_RESULT = /^\(fail\) .+ \[(?:\d+(?:\.\d+)?)(?:ns|us|µs|ms|s)\]$/;
|
||||
const BUN_BETWEEN_TESTS_ERROR = '# Unhandled error between tests';
|
||||
const BUN_TERMINAL_SUMMARY = /^Ran (\d+) tests? across (\d+) files?\. \[(?:\d+(?:\.\d+)?)(?:ns|us|µs|ms|s)\]$/;
|
||||
// The counts block bun prints just before the terminal summary (" 1 pass",
|
||||
// " 2 skip", " 0 fail"). "Ran N tests" COUNTS skipped tests, so N alone
|
||||
// cannot distinguish a shard that verified work from one whose every test
|
||||
// self-skipped (external-service binary missing, tier mismatch) — the
|
||||
// green-by-skip class. Anchored to whole-line matches; nested bun-test
|
||||
// children can still contribute counts (same known limit as the terminal
|
||||
// summary — see the last-summary-anchoring TODO in the audit).
|
||||
const BUN_SKIP_COUNT = /^\s*(\d+) skip$/;
|
||||
const BUN_PASS_COUNT = /^\s*(\d+) pass$/;
|
||||
|
||||
export type BunTestOutputFinding = 'failed-test' | 'unhandled-between-tests';
|
||||
|
||||
@@ -29,6 +38,11 @@ export interface BunTestOutputSummary {
|
||||
terminalFileCounts: number[];
|
||||
/** Test counts from the same terminal lines — feeds the hollow-shard guard. */
|
||||
terminalTestCounts: number[];
|
||||
/** Sum of bun's " N skip" count lines. "Ran N tests" includes skips, so
|
||||
* this is what separates verified work from green-by-skip. */
|
||||
skippedTests: number;
|
||||
/** Sum of bun's " N pass" count lines. */
|
||||
passedTests: number;
|
||||
}
|
||||
|
||||
export type ForwardedTerminationSignal = 'SIGINT' | 'SIGTERM';
|
||||
@@ -229,6 +243,8 @@ export class BunTestOutputClassifier {
|
||||
private unhandledBetweenTests = 0;
|
||||
private terminalFileCounts: number[] = [];
|
||||
private terminalTestCounts: number[] = [];
|
||||
private skippedTests = 0;
|
||||
private passedTests = 0;
|
||||
|
||||
write(chunk: Uint8Array | string, origin: ClassifierOrigin = 'stdout'): void {
|
||||
this.pending[origin] += typeof chunk === 'string'
|
||||
@@ -252,6 +268,8 @@ export class BunTestOutputClassifier {
|
||||
unhandledBetweenTests: this.unhandledBetweenTests,
|
||||
terminalFileCounts: [...this.terminalFileCounts],
|
||||
terminalTestCounts: [...this.terminalTestCounts],
|
||||
skippedTests: this.skippedTests,
|
||||
passedTests: this.passedTests,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -268,6 +286,11 @@ export class BunTestOutputClassifier {
|
||||
const finding = classifyBunTestOutputLine(line);
|
||||
if (finding === 'failed-test') this.failedTests += 1;
|
||||
if (finding === 'unhandled-between-tests') this.unhandledBetweenTests += 1;
|
||||
const stripped = stripAnsiLine(line);
|
||||
const skip = BUN_SKIP_COUNT.exec(stripped);
|
||||
if (skip !== null) this.skippedTests += Number.parseInt(skip[1], 10);
|
||||
const pass = BUN_PASS_COUNT.exec(stripped);
|
||||
if (pass !== null) this.passedTests += Number.parseInt(pass[1], 10);
|
||||
const terminal = parseBunTerminalSummary(line);
|
||||
if (terminal !== null) {
|
||||
this.terminalFileCounts.push(terminal.files);
|
||||
|
||||
Reference in New Issue
Block a user