mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-05 02:47:17 +02:00
test(evals): retire every paid automatic retry
Paid evals never retry (approved 2026-09-29): delete SHORT_CASE_RETRY_FILES and retriesWithinCaseCap, drop the retry fields from the registered wall rows (walls now cover one run plus reserve), make retriesForFiles return 0, pass --retry 0 explicitly, and drop --retry 1 from the package.json paid scripts. Add the eval:pass-rates alias. Tests that pinned the old retry allowance are updated as a policy change; review-finalization-budget now proves late-result recording under the production zero-retry arguments.
This commit is contained in:
1 parent
a427f05c58
commit
b1f5bc0032
12 files changed
+131
-209
No files matched your search
@@ -1,4 +1,4 @@
|
||||
/** The real review registrations must finish capture cleanup before Bun retries. */
|
||||
/** The real review registrations record late results and clean up before finalization, under the production zero-retry arguments. */
|
||||
import { expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
@@ -12,7 +12,7 @@ const CASES = [
|
||||
['review-design-lite', 400, 35],
|
||||
] as const;
|
||||
for (const [id, workMs, maxTurns] of CASES) {
|
||||
test.each(['recover', 'both-timeout'])(`${id} records late results before retry or finalization: %s`, scenario => {
|
||||
test.each(['success', 'timeout'])(`${id} records late results before finalization: %s`, scenario => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'review-finalization-'));
|
||||
const script = path.join(dir, 'registration.test.ts');
|
||||
const facts = path.join(dir, 'events.jsonl');
|
||||
@@ -41,7 +41,7 @@ mock.module(path.join(root, 'test/helpers/session-runner.ts'), () => ({
|
||||
runSkillTest: async opts => {
|
||||
const id = ++attempts;
|
||||
event({ kind: 'start', id, timeout: opts.timeout, maxTurns: opts.maxTurns, cwd: opts.workingDirectory });
|
||||
const timeout = id === 1 || ${JSON.stringify(scenario)} === 'both-timeout';
|
||||
const timeout = ${JSON.stringify(scenario)} === 'timeout';
|
||||
// Use the caller's actual work budget; only the provider and budget
|
||||
// constants are scaled. The actual registered Bun outer deadline stays.
|
||||
await new Promise(resolve => setTimeout(resolve, timeout ? opts.timeout + 50 : 80));
|
||||
@@ -62,27 +62,25 @@ await import(path.join(root, ${JSON.stringify(PAID_FILE)}));
|
||||
`);
|
||||
try {
|
||||
const retries = retriesForFiles([PAID_FILE]);
|
||||
expect(retries).toBe(1);
|
||||
expect(retries).toBe(0);
|
||||
const child = Bun.spawnSync([process.execPath, ...buildPaidShardArgs([script], resolvePaidShardTimeoutMs([PAID_FILE]), 2, retries)], {
|
||||
cwd: ROOT, timeout: 15_000, stdout: 'pipe', stderr: 'pipe',
|
||||
env: { ...process.env, EVALS: '', EVALS_ALL: '', TMPDIR: dir, TMP: dir, TEMP: dir },
|
||||
});
|
||||
const output = child.stdout.toString() + child.stderr.toString();
|
||||
expect(child.exitCode, output).toBe(scenario === 'recover' ? 0 : 1);
|
||||
expect(child.exitCode, output).toBe(scenario === 'success' ? 0 : 1);
|
||||
expect(output).not.toContain('Unhandled error between tests');
|
||||
const events = fs.readFileSync(facts, 'utf8').trim().split('\n').map(line => JSON.parse(line));
|
||||
const starts = events.filter(event => event.kind === 'start');
|
||||
const ready = events.filter(event => event.kind === 'ready');
|
||||
const records = events.filter(event => event.kind === 'record');
|
||||
expect(starts.map(event => event.id)).toEqual([1, 2]);
|
||||
expect(starts.map(event => event.id)).toEqual([1]);
|
||||
expect(starts.map(({ timeout, maxTurns }) => ({ timeout, maxTurns })))
|
||||
.toEqual([{ timeout: workMs, maxTurns }, { timeout: workMs, maxTurns }]);
|
||||
expect(ready).toEqual([{ kind: 'ready', id: 1, fixtureExists: true }, { kind: 'ready', id: 2, fixtureExists: true }]);
|
||||
.toEqual([{ timeout: workMs, maxTurns }]);
|
||||
expect(ready).toEqual([{ kind: 'ready', id: 1, fixtureExists: true }]);
|
||||
expect(records.map(event => [event.id, event.exitReason]))
|
||||
.toEqual([[1, 'timeout'], [2, scenario === 'recover' ? 'success' : 'timeout']]);
|
||||
.toEqual([[1, scenario === 'success' ? 'success' : 'timeout']]);
|
||||
expect(events.findIndex(event => event.kind === 'record' && event.id === 1))
|
||||
.toBeLessThan(events.findIndex(event => event.kind === 'start' && event.id === 2));
|
||||
expect(events.findIndex(event => event.kind === 'record' && event.id === 2))
|
||||
.toBeLessThan(events.findIndex(event => event.kind === 'finalized'));
|
||||
expect(events.filter(event => event.kind === 'finalized')).toHaveLength(1);
|
||||
expect(events.find(event => event.kind === 'registration')).toEqual({ kind: 'registration', name: id, outerMs: workMs + 50 + 5_000 });
|
||||
|
||||
Reference in new issue
Block a user