From 289672264ab58bef80bdd71d011a7391cbcc92d9 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sun, 4 Oct 2026 00:26:38 +0000 Subject: [PATCH] evals2 final: plan-eng-review-artifact touchfiles point at its PTY file and helpers; plan file wall drops the moved case Applies the lane's touchfiles-data patch (its outside-voice-evidence hunks were already in 9a96fb3). skill-e2e-plan.test.ts lost one 600 s case to skill-e2e-plan-eng-artifact.test.ts, so its periodic wall is two 900 s + two 600 s (3,000,000 + reserve) and its timeout pin lists the remaining cases. --- test/helpers/eval-budgets.ts | 5 +++-- test/helpers/touchfiles-data.ts | 4 ++-- test/paid-retry-supervision.test.ts | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/test/helpers/eval-budgets.ts b/test/helpers/eval-budgets.ts index a195c6f99..1aa5489ab 100644 --- a/test/helpers/eval-budgets.ts +++ b/test/helpers/eval-budgets.ts @@ -113,8 +113,9 @@ export const FILE_RETRY_BUDGETS = [ { file: 'test/skill-e2e-plan-ceo-mode-routing.test.ts', attemptMs: 2 * CAPTURE_LONG_MS, caseMs: CAPTURE_LONG_MS }, { file: 'test/skill-e2e-plan-eng-plan-mode.test.ts', attemptMs: 2 * CAPTURE_LONG_MS, caseMs: CAPTURE_LONG_MS }, { file: 'test/skill-e2e-plan-prosons.test.ts', attemptMs: 4 * (CAPTURE_MS + 10_000), caseMs: CAPTURE_MS + 10_000 }, - // Gate: six 300s cases + one 610s case; periodic: two 900s + three 600s. - { file: 'test/skill-e2e-plan.test.ts', attemptMs: Math.max(6 * CAPTURE_MS + CAPTURE_LONG_MS + 10_000, 2 * PTY_MS + 3 * CAPTURE_LONG_MS), caseMs: PTY_MS }, + // Gate: six 300s cases + one 610s case; periodic: two 900s + two 600s + // (plan-eng-review-artifact runs from its own PTY file). + { file: 'test/skill-e2e-plan.test.ts', attemptMs: Math.max(6 * CAPTURE_MS + CAPTURE_LONG_MS + 10_000, 2 * PTY_MS + 2 * CAPTURE_LONG_MS), caseMs: PTY_MS }, ].map(({ file, attemptMs, caseMs }) => ({ file, attemptMs, caseMs, id: `${file.slice('test/'.length, -'.test.ts'.length)}-existing-retry-v1`, diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index 7bcfe5418..48d7f40c3 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -156,11 +156,11 @@ export const E2E_TOUCHFILES: Record = { "test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts", 'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], - 'plan-eng-review-artifact': [ + 'plan-eng-review-artifact': [ 'test/helpers/auto-decision-state.ts', 'test/helpers/autoplan-artifact-digest.ts', 'test/helpers/autoplan-artifact-permission.ts', 'test/helpers/autoplan-artifact-recorder.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/dx-selected-navigation.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/eng-cache-writer-decision.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/helpers/native-auto-decide.ts', 'test/helpers/owned-claude-transcript.ts', 'test/helpers/plan-count-artifacts.ts', 'test/helpers/plan-count-file-permission.ts', 'test/helpers/plan-count-fixture.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/helpers/plan-count-pending-question.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-floor-review.ts', 'test/helpers/plan-floor-target.ts', 'test/helpers/plan-scope-selection.ts', 'test/helpers/plan-seed-submission.ts', 'test/helpers/plan-skill-question-events.ts', 'test/helpers/plan-skill-question-hook-scope.ts', 'test/helpers/plan-skill-questions.ts', 'test/helpers/pty-screen.ts', 'test/helpers/pty-trust-dialog.ts', 'test/helpers/pty/**', 'test/helpers/pty/auq.ts', 'test/helpers/pty/binary.ts', 'test/helpers/pty/boundaries.ts', 'test/helpers/pty/classify.ts', 'test/helpers/pty/fake-session.ts', 'test/helpers/pty/judge.ts', 'test/helpers/pty/launch.ts', 'test/helpers/pty/plan-native.ts', 'test/helpers/pty/runners/counting.ts', 'test/helpers/pty/runners/floor.ts', 'test/helpers/pty/runners/observation.ts', 'test/helpers/pty/screen.ts', 'test/helpers/pty/session.ts', 'test/helpers/skill-census.ts', 'scripts/resolvers/learnings.ts', - "test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', + "test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'test/skill-e2e-plan-eng-artifact.test.ts', 'test/helpers/workflow-excerpt.ts', "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts", 'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], 'plan-review-report': [ diff --git a/test/paid-retry-supervision.test.ts b/test/paid-retry-supervision.test.ts index e5cec82d4..881678f3a 100644 --- a/test/paid-retry-supervision.test.ts +++ b/test/paid-retry-supervision.test.ts @@ -34,7 +34,7 @@ const expectedWalls = { 'test/skill-e2e-plan-ceo-mode-routing.test.ts': 1_320_000, 'test/skill-e2e-plan-eng-plan-mode.test.ts': 1_320_000, 'test/skill-e2e-plan-prosons.test.ts': 1_360_000, - 'test/skill-e2e-plan.test.ts': 3_720_000, + 'test/skill-e2e-plan.test.ts': 3_120_000, }; test('paid evals never retry: every paid file and registered row runs once', () => { @@ -99,7 +99,7 @@ test('source allowances retain all captures, cases, and finalization grace', () expect(read('test/skill-e2e-auq-matrix.test.ts').match(/^ skill: '/gm)).toHaveLength(6); expect(timeoutExpressions('test/skill-e2e-auq-matrix.test.ts')).toEqual(['CAPTURE_MS']); expect(timeoutExpressions('test/skill-e2e-plan.test.ts')).toEqual([ - 'PTY_MS', 'PTY_MS', 'CAPTURE_LONG_MS', 'CAPTURE_LONG_MS', 'CAPTURE_LONG_MS', + 'PTY_MS', 'PTY_MS', 'CAPTURE_LONG_MS', 'CAPTURE_LONG_MS', 'CAPTURE_LONG_MS + OFFICE_HOURS_BUN_GRACE_MS', ]); });