mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-05 19:07:12 +02:00
test: run seven paid evals on the current default capture model (B8)
skill-e2e-{auq-matrix,plan-format,qa-bugs,retro,workflow} pinned
claude-opus-4-7 and skill-e2e-office-hours plus -brain-writeback pinned
claude-sonnet-4-6; none tests a historical model, so they now capture with
resolveEvalModel('capture'), and the free harness tests that execute these
registrations receive the same resolver. The paid re-pin run passed all of
them. skill-e2e-{design,office-hours-phase4,plan-prosons,plan} keep
claude-opus-4-7: six of their cases failed on the default model (three
timeouts, a missing report file, a format miss and a posture score of 3), so
per the plan's fallback they keep their pins with a TODOS entry. The pre-spend
estimate and drop threshold are in docs/test-audit-2026-09.md.
This commit is contained in:
1 parent
799f6ec36c
commit
67084a5caf
11 files changed
+55
-23
No files matched your search
@@ -1,4 +1,5 @@
|
||||
import { expect, test } from 'bun:test';
|
||||
import { resolveEvalModel } from '../lib/eval-model';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
@@ -26,7 +27,7 @@ async function exercise(owner: Owner, scenarios: Scenario[]) {
|
||||
} };
|
||||
const args: Record<string, any> = {
|
||||
expect, beforeAll: (fn: () => void) => setups.push(fn), afterAll: (fn: () => void) => finalizers.push(fn),
|
||||
CAPTURE_MS, CAPTURE_LONG_MS, OFFICE_HOURS_BUN_GRACE_MS, runRecordedOfficeHoursAttempt,
|
||||
CAPTURE_MS, CAPTURE_LONG_MS, OFFICE_HOURS_BUN_GRACE_MS, runRecordedOfficeHoursAttempt, resolveEvalModel,
|
||||
ROOT: '/source', runId: 'synthetic-posture-run', evalsEnabled: true,
|
||||
describeIfSelected: (_title: string, _names: string[], fn: () => void) => fn(),
|
||||
testConcurrentIfSelected: (name: string, fn: () => Promise<void>, timeout: number) => { expect(timeout).toBe(CAPTURE_LONG_MS + OFFICE_HOURS_BUN_GRACE_MS); callbacks.set(name, fn); },
|
||||
@@ -44,7 +45,7 @@ async function exercise(owner: Owner, scenarios: Scenario[]) {
|
||||
runSkillTest: async (opts: any) => {
|
||||
index++; current = scenarios[index]!; expect(current).toBeDefined(); calls.push(opts);
|
||||
expect(opts.testName).toBe(owner); expect(opts.maxTurns).toBe(8); expect(opts.timeout).toBe(CAPTURE_MS);
|
||||
expect(opts.model).toBe('claude-sonnet-4-6'); expect(opts.runId).toBe('synthetic-posture-run');
|
||||
expect(opts.model).toBe(resolveEvalModel('capture')); expect(opts.runId).toBe('synthetic-posture-run');
|
||||
expect(opts.signal).toBeInstanceOf(AbortSignal);
|
||||
expect(opts.prompt).toContain('Skip any AskUserQuestion');
|
||||
const file = path.join(opts.workingDirectory, owner === owners[0] ? 'q3.md' : 'unlocks.md');
|
||||
|
||||
Reference in new issue
Block a user