test(scorecard): cover --gate-event any, an empty window, a failed PR read and a bad --now

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-26 14:07:08 +00:00
1 parent 29ed557b05
commit 3f5e4bd0ed
2 files changed
+52 -2

No files matched your search

+1 -1
View File
@@ -10,7 +10,7 @@
- **`docs/recommended-harness.md`**, linked from the README docs table; `docs/standalone-binaries.md` lists the new CLI.
### Tests
- `test/factory-scorecard.test.ts`: the recorded window (`test/fixtures/factory-scorecard/recorded.json`) yields the expected merges, lead times, gate kinds and outcomes, artifact and comment cost sources, flake counts, free retries, issues and releases; a crew login list overrides the association rule; runs older than the window, non-dispatch events and draft releases are left out; the artifact cap leaves later runs without cost or flake data; a missing input marks only its lines unavailable; the markdown and CSV carry every row; gate classification, READY-marker matching, verdict-cost and flake-record parsing and nearest-rank percentiles; the bin end to end with `--fixtures`, `--crew`, `--out-dir`, `--csv`, `--ready-marker`, bad usage and `--help`; the retro template pins the fence.
- `test/factory-scorecard.test.ts`: the recorded window (`test/fixtures/factory-scorecard/recorded.json`) yields the expected merges, lead times, gate kinds and outcomes, artifact and comment cost sources, flake counts, free retries, issues and releases; a crew login list overrides the association rule; runs older than the window, non-dispatch events and draft releases are left out; the artifact cap leaves later runs without cost or flake data; a missing input marks only its lines unavailable; the markdown and CSV carry every row; gate classification, READY-marker matching, verdict-cost and flake-record parsing and nearest-rank percentiles; `--gate-event any` keeps non-dispatch runs; an empty window prints n/a instead of NaN; a failed merged-PR read leaves the gate rows standing; the bin end to end with `--fixtures`, `--crew`, `--out-dir`, `--csv`, `--ready-marker`, bad usage (including `--now`) and `--help`; the retro template pins the fence.
## [1.91.246.0] - 2026-10-26
+51 -1
View File
@@ -11,7 +11,7 @@ import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import {
BENCHMARKS, DEFAULTS, classifyGate, collectScorecard, fixtureSource, flakeRecords, gateOutcome, percentile, readComments, renderCsv,
BENCHMARKS, DEFAULTS, classifyGate, collectScorecard, fixtureSource, flakeRecords, fmtHours, gateOutcome, percentile, readComments, renderCsv,
renderMarkdown, scorecardRows, verdictCost, type FixtureFile, type ScorecardOptions,
} from '../lib/factory-scorecard';
@@ -112,6 +112,54 @@ describe('factory-scorecard: recorded window', () => {
expect(csv).toHaveLength(rows.length + 2);
expect(csv[1]).toStartWith('"acme/widgets","2026-10-19T12:00:00.000Z","2026-10-26T12:00:00.000Z","merged","Merged PRs","3 (crew 2, outside 1)"');
});
test('--gate-event any keeps non-dispatch runs and drops the event filter from the query', () => {
// Value: protects=gateEvent null counts every completed gate run and omits event= from the runs query; fails_when=the null branch filters or the query keeps event=; why_new=only the default dispatch path was covered; seam=none; no_claim=live GitHub query semantics
const fx = fixture();
const key = Object.keys(fx.responses).find(k => k.includes('/evals.yml/runs?'))!;
fx.responses[key.replace('&event=workflow_dispatch', '')] = fx.responses[key];
const source = fixtureSource(fx, scratch());
const card = collectScorecard(source, options({ gateEvent: null }));
expect(source.calls.find(c => c.includes('/evals.yml/runs?'))).not.toContain('event=');
expect(card.gates.map(g => g.id)).toEqual([9005, 9004, 9003, 9002, 9001]);
expect(card.gates.find(g => g.id === 9005)!.kind).toBe('other');
});
test('an empty window prints n/a on every ratio instead of NaN, and short durations read as minutes', () => {
// Value: protects=zero merges and zero trains render n/a and sub-hour lead times render as minutes; fails_when=a 0/0 guard is removed or fmtHours loses a branch; why_new=every other case has merges and hour-scale durations; seam=none; no_claim=none beyond protects
const fx = fixture();
for (const key of Object.keys(fx.responses)) {
if (key.includes('/pulls?') || key.includes('/releases?')) fx.responses[key] = [];
else if (key.includes('/runs?')) fx.responses[key] = { total_count: 0, workflow_runs: [] };
else if (key.startsWith('search/issues')) fx.responses[key] = { total_count: 0, items: [] };
}
const card = collectScorecard(fixtureSource(fx, scratch()), options());
const values = byId(scorecardRows(card));
expect(card.unavailable).toEqual([]);
expect(values.merged).toBe('0 (crew 0, outside 0)');
expect(values.lead_time).toBe('median n/a, p90 n/a');
expect(values.cost_per_merge).toBe('n/a (no merges)');
expect(values.red_train).toBe('n/a (0 red of 0 trains; 0 red backstop, 0 red screens)');
expect(values.releases).toBe('0');
expect(renderMarkdown(card, scorecardRows(card))).not.toContain('NaN');
expect(fmtHours(0.5)).toBe('30m');
expect(fmtHours(null)).toBe('n/a');
});
test('a failed merged-PR read leaves gate rows standing with comment costs unknown', () => {
// Value: protects=each input is guarded on its own so a PR-list failure still prints gates, free retries, issues and releases; fails_when=the guards are merged into one try or the gate read depends on the PR read succeeding; why_new=only the issues guard was exercised; seam=none; no_claim=none beyond protects
const fx = fixture();
for (const key of Object.keys(fx.responses)) if (key.includes('/pulls?')) delete fx.responses[key];
const card = collectScorecard(fixtureSource(fx, scratch()), options());
expect(card.unavailable).toHaveLength(1);
expect(card.unavailable[0]).toStartWith('merged PRs: ');
const values = byId(scorecardRows(card));
expect(values.merged).toStartWith('unavailable (');
expect(values.lead_time).toStartWith('unavailable (');
expect(values.gates).toBe('2 / 1 / 1; known cost $30.50 on 1 of 4 runs');
expect(card.gates.find(g => g.id === 9003)!.costSource).toBeNull();
expect(values.issues).toBe('5 / 7');
});
});
describe('factory-scorecard: pieces', () => {
@@ -187,6 +235,8 @@ describe('factory-scorecard: the bin', () => {
test('bad usage exits 2 with the usage text; --help exits 0', () => {
const cwd = scratch();
expect(run(['--since-days', 'soon'], cwd).status).toBe(2);
// Value: protects=an unparsable --now exits 2 before any read; fails_when=the --now validation branch is removed; why_new=the other flag validations were covered and this one was not; seam=none; no_claim=none beyond protects
expect(run(['--now', 'yesterday-ish', '--fixtures', FIXTURE], cwd).status).toBe(2);
expect(run(['--ready-marker', '(', '--fixtures', FIXTURE], cwd).status).toBe(2);
const stray = run(['stray'], cwd);
expect(stray.status).toBe(2);