W3 E1: add the plan-devex-review member to the plan-mode no-op eval; budgets and floors follow

This commit is contained in:
garrytan committed 2026-10-03 20:44:55 +00:00
1 parent 395e1e2ead
commit 769ed52eb1
4 files changed
+16 -13

No files matched your search

+1 -1
View File
@@ -189,7 +189,7 @@ test('current detach supervision covers the live-census floor', () => {
const pkg = JSON.parse(fs.readFileSync(path.join(import.meta.dir, '../package.json'), 'utf8'));
const periodicTimeout = Number(pkg.scripts['eval:bg:periodic'].match(/--timeout\s+(\d+)/)[1]);
const gateTimeout = Number(pkg.scripts['eval:bg:gate'].match(/--timeout\s+(\d+)/)[1]);
expect(floorFor('gate')).toBe(21_725);
expect(floorFor('gate')).toBe(22_355);
expect(gateTimeout).toBe(49_320);
expect(gateTimeout).toBeGreaterThanOrEqual(floorFor('gate'));
expect(floorFor('periodic')).toBe(35_742);
+1 -1
View File
@@ -109,7 +109,7 @@ export const FILE_RETRY_BUDGETS = [
{ file: 'test/skill-e2e-plan-eng-finding-floor.test.ts', attemptMs: PTY_MS, caseMs: PTY_MS },
{ file: 'test/skill-e2e-plan-design-finding-floor.test.ts', attemptMs: PTY_MS, caseMs: PTY_MS },
{ file: 'test/skill-e2e-plan-devex-finding-floor.test.ts', attemptMs: PTY_MS, caseMs: PTY_MS },
{ file: 'test/skill-e2e-plan-mode-no-op.test.ts', attemptMs: 5 * CAPTURE_LONG_MS, caseMs: CAPTURE_LONG_MS },
{ file: 'test/skill-e2e-plan-mode-no-op.test.ts', attemptMs: 6 * CAPTURE_LONG_MS, caseMs: CAPTURE_LONG_MS },
{ file: 'test/skill-e2e-plan-ceo-mode-routing.test.ts', attemptMs: 2 * CAPTURE_LONG_MS, caseMs: CAPTURE_LONG_MS },
{ file: 'test/skill-e2e-plan-eng-plan-mode.test.ts', attemptMs: 2 * CAPTURE_LONG_MS, caseMs: CAPTURE_LONG_MS },
{ file: 'test/skill-e2e-plan-prosons.test.ts', attemptMs: 4 * (CAPTURE_MS + 10_000), caseMs: CAPTURE_MS + 10_000 },
+5 -5
View File
@@ -30,7 +30,7 @@ const expectedWalls = {
'test/skill-e2e-plan-eng-finding-floor.test.ts': 1_020_000,
'test/skill-e2e-plan-design-finding-floor.test.ts': 1_020_000,
'test/skill-e2e-plan-devex-finding-floor.test.ts': 1_020_000,
'test/skill-e2e-plan-mode-no-op.test.ts': 3_120_000,
'test/skill-e2e-plan-mode-no-op.test.ts': 3_720_000,
'test/skill-e2e-plan-ceo-mode-routing.test.ts': 1_320_000,
'test/skill-e2e-plan-eng-plan-mode.test.ts': 1_320_000,
'test/skill-e2e-plan-prosons.test.ts': 1_360_000,
@@ -88,7 +88,7 @@ test('source allowances retain all captures, cases, and finalization grace', ()
if (file.endsWith('-finding-floor')) expect(read(`test/skill-e2e-${file}.test.ts`)).toContain('timeoutMs: CAPTURE_LONG_MS');
}
const noop = read('test/skill-e2e-plan-mode-no-op.test.ts');
expect(noop).toContain("['plan-ceo-review', 'plan-eng-review', 'plan-design-review'] as const");
expect(noop).toContain("['plan-ceo-review', 'plan-eng-review', 'plan-design-review', 'plan-devex-review'] as const");
expect(noop).toContain("['plan-eng-review', 'plan-design-review'] as const");
expect(timeoutExpressions('test/skill-e2e-plan-mode-no-op.test.ts')).toEqual(Array(2).fill('CAPTURE_LONG_MS'));
expect(read('test/skill-e2e-plan-ceo-mode-routing.test.ts').match(/^\s*{ mode: '/gm)).toHaveLength(2);
@@ -210,7 +210,7 @@ test('detached PR fallback and release commands cover their actual default worke
const prFloor = Math.ceil((Math.ceil(fullGateFiles.length / prWorkers) * 1_800_000 + fullGateFiles.reduce(
(total, file) => total + Math.max(0, resolvePaidShardBudget([file]).timeoutMs - 1_800_000), 0,
)) / 1000 * 1.05);
expect(prFloor).toBe(72_755);
expect(prFloor).toBe(73_385);
expect(prWall).toBe(92_820_000);
expect(prWall).toBeGreaterThanOrEqual(paidShardWallUpperBoundMs(files, prWorkers) + 120_000);
@@ -229,8 +229,8 @@ test('detached PR fallback and release commands cover their actual default worke
)) / 1000 * 1.05));
}
const detachedReleaseWall = Number(scripts['eval:bg:release'].match(/--timeout (\d+)/)?.[1]) * 1000;
expect(releaseFloors).toEqual([21_725, 35_742]);
expect(releaseFloors.reduce((total, floor) => total + floor, 0)).toBe(57_467);
expect(releaseFloors).toEqual([22_355, 35_742]);
expect(releaseFloors.reduce((total, floor) => total + floor, 0)).toBe(58_097);
expect(detachedReleaseWall).toBe(116_700_000);
expect(detachedReleaseWall).toBeGreaterThanOrEqual(releaseWall + 120_000);
});
+9 -6
View File
@@ -21,11 +21,13 @@
* ends in 'asked', the question that fired must be the scope gate itself
* (outside plan mode with no named target, the gate is the FIRST
* question by contract).
* - plan-devex-review: no no-target scope gate, so only the terminal
* outcome and the absent plan-mode reminder are asserted.
* - named-target case: a pasted draft (initialPlanContent) IS an
* explicitly-named target, so the gate must NOT ask — and the review
* must actually consume the pasted content.
*
* Cost note: 5 sequential PTY runs (~3-5 min each) in the gate lane, up
* Cost note: 6 sequential PTY runs (~3-5 min each) in the gate lane, up
* from 1 pre-bypass. Selected only when plan-ceo/eng/design or the runner
* change (see 'plan-mode-no-op' in touchfiles.ts).
*/
@@ -58,7 +60,8 @@ weekly export emails. One new component, one route, one test file.
`;
describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
for (const skillName of ['plan-ceo-review', 'plan-eng-review', 'plan-design-review'] as const) {
for (const skillName of ['plan-ceo-review', 'plan-eng-review', 'plan-design-review', 'plan-devex-review'] as const) {
const hasScopeGate = skillName === 'plan-eng-review' || skillName === 'plan-design-review';
test(`${skillName} reaches a terminal outcome outside plan mode`, async () => {
const obs = await runPlanSkillObservation({
skillName,
@@ -74,9 +77,9 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
// renders as a lettered menu, so a judge 'waiting' verdict on a
// spinner-only frame must not end the run as 'asked' before that
// menu has rendered.
...(skillName === 'plan-ceo-review'
? {}
: { extraArgs: ['--disallowedTools', 'AskUserQuestion'], requireProseEvidence: true }),
...(hasScopeGate
? { extraArgs: ['--disallowedTools', 'AskUserQuestion'], requireProseEvidence: true }
: {}),
});
if (obs.outcome === 'silent_write' || obs.outcome === 'exited' || obs.outcome === 'timeout') {
@@ -94,7 +97,7 @@ describeE2E('plan-mode-info no-op outside plan mode (gate regression)', () => {
// section is leaking outside plan mode.
expect(obs.evidence).not.toContain(PLAN_MODE_REMINDER);
if (skillName !== 'plan-ceo-review') {
if (hasScopeGate) {
// Scope-gate bypass must not misfire: no auto-select announcement
// outside plan mode.
expect(obs.scopeGateAutoSelectObserved ?? false).toBe(false);