From fc9049a8b59e525d2067c4d55e5dba3a8efdaad6 Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 01:47:21 +0000 Subject: [PATCH] test(W7h): retire the paid SKILL.md-summary cases in skill-e2e-plan W7h retirement (t14 F6-F8): office-hours-spec-review, plan-ceo-review-benefits and codex-offered-{office-hours,ceo-review,design-review,eng-review}. Each asked a model to read a generated SKILL.md, write a summary, and grepped the summary for facts the generated file states directly. Registration cleanup in the same commit: E2E_TOUCHFILES, E2E_TIERS (all six gate) and E2E_KINDS (rule) in test/helpers/touchfiles-data.ts; six scripts/paid-test-durations.json case-shard entries; no test/helpers/periodic-exclude-data.ts entries existed. The fast PR profile (scripts/test-pr-profile.ts) drops plan-ceo-review-benefits and office-hours-spec-review from PR_PROFILE_CASE_IDS and PR_PROFILE_FILES. Tests that named the retired ids as live examples move to existing ones: paid-free-boundary and paid-pr-profile (a CEO template change now plans auq-format-gate plus the CEO judge; the case-shard fixture uses plan-review-report), test-pr-profile, touchfiles (CEO selection count 21 -> 19), the touchfile-moved-code w5 golden (retired ids removed), the behavior-lane case-shard census (78 -> 72) in cookie-validation-phases, and the skill-e2e-plan timeout list and full-gate PR floor (78,425 -> 72,755 s) in paid-retry-supervision. e2e-tier-alignment, eval-kinds, eval-detach-timeout-floor and paid-shards stay green. Coverage consequence: per the Eng Retirements item, plan-ceo-review-benefits was the only gate-tier CEO E2E in the fast PR profile, so CEO invariants are treated as periodic-only under the W7c gate (every CEO conversion in this branch keeps one free check per invariant); the gate judge 'plan-ceo-review/SKILL.md modes' still runs for CEO prompt changes. An office-hours prompt change now has no fast PR E2E and is reported as deferred to broad coverage rather than run. Retirement cards: - office-hours-spec-review: detects the Spec Review Loop's stated dimensions, dispatch tool, iteration cap and metrics. Replacement: gen-skill-docs.test.ts 'SPEC_REVIEW_LOOP resolver' (round-3 cap, prepare/check/finalize commands, verdict and round-file wiring) and test/office-hours-review.test.ts (verdict parsing). Control: changing the round limit in scripts/resolvers/spec-review.ts turns gen-skill-docs red. - plan-ceo-review-benefits: detects the CEO /office-hours prerequisite and mid-session offer. Replacement: gen-skill-docs.test.ts 'plan-ceo-review offers /office-hours when no design doc exists and detects a lost user mid-session' (previous commit). Control: 4/4 red mutations there. - codex-offered-* (x4): detects the `command -v codex` probe, a fallback and optionality. Replacement: gen-skill-docs.test.ts 'Codex outside voice is offered behind an executed availability probe' (probe + not_installed fallback for all four skills) and skill-validation.test.ts 'plan reviews run the Codex outside voice default-on'. Control: 4/4 red there. - non_test_callers: none; history: c118e24 v1.64.1.0 (#2572). - validation: the registration tests named above. Controls log: .context/mutation-skill-e2e-plan-retirements.md. --- scripts/paid-test-durations.json | 6 - scripts/test-pr-profile.ts | 6 +- test/cookie-validation-phases.test.ts | 2 +- .../w5-review-resolvers.json | 2 - test/helpers/touchfiles-data.ts | 45 ---- test/paid-free-boundary.test.ts | 2 +- test/paid-pr-profile.test.ts | 14 +- test/paid-retry-supervision.test.ts | 5 +- test/skill-e2e-plan.test.ts | 224 ------------------ test/test-pr-profile.test.ts | 1 - test/touchfiles.test.ts | 6 +- 11 files changed, 16 insertions(+), 297 deletions(-) diff --git a/scripts/paid-test-durations.json b/scripts/paid-test-durations.json index bbc63ae1d..105bbf2ae 100644 --- a/scripts/paid-test-durations.json +++ b/scripts/paid-test-durations.json @@ -40,12 +40,6 @@ "test/skill-e2e-plan-prosons.test.ts": 1000, "test/skill-e2e-plan-tune.test.ts": 63925, "test/skill-e2e-plan.test.ts": 251000, - "test/skill-e2e-plan.test.ts#codex-offered-ceo-review": 49880, - "test/skill-e2e-plan.test.ts#codex-offered-design-review": 54895, - "test/skill-e2e-plan.test.ts#codex-offered-eng-review": 59818, - "test/skill-e2e-plan.test.ts#codex-offered-office-hours": 50442, - "test/skill-e2e-plan.test.ts#office-hours-spec-review": 33866, - "test/skill-e2e-plan.test.ts#plan-ceo-review-benefits": 48811, "test/skill-e2e-plan.test.ts#plan-review-report": 76406, "test/skill-e2e-qa-bugs.test.ts": 1000, "test/skill-e2e-qa-callers.test.ts": 737164, diff --git a/scripts/test-pr-profile.ts b/scripts/test-pr-profile.ts index a0e7302fb..082e2777f 100644 --- a/scripts/test-pr-profile.ts +++ b/scripts/test-pr-profile.ts @@ -12,8 +12,8 @@ export const PR_PROFILE_CASE_IDS = [ 'qa-functional-cli-fix', 'qa-functional-webhook-fix', 'review-exploratory-small-cli', 'ship-exploratory-small-cli', 'ship-exploratory-unavailable', 'ship-exploratory-plan-checks', 'ship-exploratory-late-input', - 'plan-ceo-review-benefits', 'plan-eng-coverage-audit', 'plan-review-report', - 'auq-format-gate', 'plan-design-review-no-ui-scope', 'office-hours-spec-review', + 'plan-eng-coverage-audit', 'plan-review-report', + 'auq-format-gate', 'plan-design-review-no-ui-scope', 'tpa-present', 'tpa-absent-linux', 'ship-local-workflow', 'ship-coverage-audit', 'docsync-spawned', 'ship-docsync-completion', 'ship-docsync-current', 'ship-docsync-failure', 'ship-docsync-store', @@ -41,7 +41,7 @@ export const PR_PROFILE_FILES: Record = { 'test/skill-e2e-review.test.ts': ['review-sql-injection'], 'test/skill-e2e-coverage-audit.test.ts': ['review-coverage-audit', 'plan-eng-coverage-audit'], 'test/skill-e2e-test-value.test.ts': ['ship-coverage-value', 'review-test-value', 'test-audit-report-only'], - 'test/skill-e2e-plan.test.ts': ['plan-ceo-review-benefits', 'plan-review-report', 'office-hours-spec-review'], + 'test/skill-e2e-plan.test.ts': ['plan-review-report'], 'test/skill-e2e-ask-user-question-format-compliance.test.ts': ['auq-format-gate'], 'test/skill-e2e-design.test.ts': ['plan-design-review-no-ui-scope'], 'test/skill-e2e-third-party-actions.test.ts': ['tpa-present', 'tpa-absent-linux'], diff --git a/test/cookie-validation-phases.test.ts b/test/cookie-validation-phases.test.ts index 4e986b56b..0d68b33ff 100644 --- a/test/cookie-validation-phases.test.ts +++ b/test/cookie-validation-phases.test.ts @@ -47,7 +47,7 @@ test('the existing quality and behavior phases retain their complete separate sh // 45 files (first-task-scaffold registers no gate case, so the gate lane // skips it); the seven case-sharded files contribute one shard per gate case. expect(new Set(behaviorFiles.map(file => file.split('#')[0])).size).toBe(45); - expect(behaviorFiles).toHaveLength(78); + expect(behaviorFiles).toHaveLength(72); expect(behaviorFiles).toEqual(expect.arrayContaining([ ...['review-exploratory-small-cli', 'ship-exploratory-small-cli', 'ship-exploratory-unavailable', 'ship-exploratory-plan-checks', 'ship-exploratory-late-input'].map(id => `test/skill-e2e-qa-callers.test.ts#${id}`), diff --git a/test/fixtures/touchfile-moved-code/w5-review-resolvers.json b/test/fixtures/touchfile-moved-code/w5-review-resolvers.json index f8b8d3fec..859dffe47 100644 --- a/test/fixtures/touchfile-moved-code/w5-review-resolvers.json +++ b/test/fixtures/touchfile-moved-code/w5-review-resolvers.json @@ -6,10 +6,8 @@ "e2e": [ "autoplan-dual-voice", "carve-section-loading", - "codex-offered-eng-review", "llm-judge-recommendation", "office-hours-section-loading", - "office-hours-spec-review", "outside-plan-disabled-no-fallback", "outside-voice-claude-code-to-codex", "outside-voice-codex-to-claude-code", diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index e2cc5e4e1..8b1e7afb1 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -135,8 +135,6 @@ export const E2E_TOUCHFILES: Record = { 'test/helpers/office-hours-attempt.ts'], // Office Hours - 'office-hours-spec-review': [ 'office-hours/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', - 'bin/gstack-office-hours-review', 'lib/office-hours-review.ts', 'lib/fs-atomic.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/skill-fixture.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/workflow-judge-input.ts'], 'office-hours-forcing-energy': [ 'office-hours/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-office-hours.test.ts', 'test/helpers/office-hours-attempt.ts' ], @@ -149,8 +147,6 @@ export const E2E_TOUCHFILES: Record = { 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], 'plan-ceo-review-selective': [ 'plan-ceo-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], - 'plan-ceo-review-benefits': [ 'plan-ceo-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', - 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], 'plan-ceo-review-expansion-energy': [ 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'], 'plan-eng-review': [ @@ -612,33 +608,6 @@ export const E2E_TOUCHFILES: Record = { // /plan-tune cathedral (T16 — 5 E2E scenarios, all gate per D12) - // Codex offering verification - 'codex-offered-office-hours': [ 'office-hours/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', - 'test/helpers/codex-offering-fixture.ts', 'test/fixtures/codex-offering-cdd-public.json', 'test/fixtures/codex-offering-timeout-public.json', - 'test/helpers/workflow-judge-input.ts', 'test/helpers/workflow-excerpt.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts'], - 'codex-offered-ceo-review': [ 'plan-ceo-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', - 'test/helpers/codex-offering-fixture.ts', 'test/fixtures/codex-offering-cdd-public.json', 'test/fixtures/codex-offering-timeout-public.json', - 'test/helpers/workflow-judge-input.ts', 'test/helpers/workflow-excerpt.ts', - 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts'], - 'codex-offered-design-review': [ - 'test/helpers/codex-offering-fixture.ts', 'test/fixtures/codex-offering-cdd-public.json', 'test/fixtures/codex-offering-timeout-public.json', - 'test/helpers/workflow-judge-input.ts', 'test/helpers/workflow-excerpt.ts', - - "test/fixtures/plan-scope-recovery-av.json", - "test/fixtures/design-scope-checkpoint-at.json",'plan-design-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', - - - "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts", - 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts'], - 'codex-offered-eng-review': [ - 'test/helpers/codex-offering-fixture.ts', 'test/fixtures/codex-offering-cdd-public.json', 'test/fixtures/codex-offering-timeout-public.json', - 'test/helpers/workflow-judge-input.ts', 'test/helpers/workflow-excerpt.ts', - 'scripts/resolvers/learnings.ts', - - "test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', - "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts", - 'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts'], - // Ship 'ship-managed-hook-refresh': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'ship/**', 'bin/gstack-redact', 'bin/gstack-config', 'scripts/gen-skill-docs.ts', 'test/helpers/ship-hook-actor.ts', @@ -1191,7 +1160,6 @@ export const E2E_TIERS: Record = { 'review-army-simplification-precision': 'periodic', // False-flag noise benchmark // Office Hours - 'office-hours-spec-review': 'gate', // Brain-writeback E2E — periodic per cost (claude -p) + non-deterministic // (model interprets the gbrain instruction). Matches nearby // setup-gbrain-path4-* tier classification. @@ -1215,7 +1183,6 @@ export const E2E_TIERS: Record = { // Plan reviews — gate for cheap functional, periodic for Opus quality 'plan-ceo-review': 'periodic', 'plan-ceo-review-selective': 'periodic', - 'plan-ceo-review-benefits': 'gate', 'plan-ceo-review-expansion-energy': 'periodic', // Demoted from gate (2026-08 audit): Opus generator + subjective 2-axis >=4/5 LLM-judge threshold in the merge lane — the exact class siblings were demoted for (a +21-line preamble change once flipped the score). CLAUDE.md's own rule: Opus model test -> periodic. 'plan-eng-review': 'periodic', 'plan-eng-review-artifact': 'periodic', @@ -1311,12 +1278,6 @@ export const E2E_TIERS: Record = { // /plan-tune cathedral (T16 per D12 — all gate) - // Codex offering verification - 'codex-offered-office-hours': 'gate', - 'codex-offered-ceo-review': 'gate', - 'codex-offered-design-review': 'gate', - 'codex-offered-eng-review': 'gate', - // Session Intelligence — gate for data flow, periodic for agent integration 'timeline-event-flow': 'gate', // Binary data flow (no LLM needed) 'context-recovery-artifacts': 'gate', // Preamble reads seeded artifacts @@ -1643,7 +1604,6 @@ export const E2E_KINDS: Record = { 'review-army-consensus': 'behavior', 'review-army-simplification': 'behavior', 'review-army-simplification-precision': 'behavior', - 'office-hours-spec-review': 'rule', 'office-hours-brain-writeback': 'behavior', 'gbrain-roundtrip-local': 'rule', 'sync-gbrain-read-ready': 'rule', @@ -1652,7 +1612,6 @@ export const E2E_KINDS: Record = { 'office-hours-builder-wildness': 'behavior', 'plan-ceo-review': 'rule', 'plan-ceo-review-selective': 'rule', - 'plan-ceo-review-benefits': 'rule', 'plan-ceo-review-expansion-energy': 'behavior', 'plan-eng-review': 'rule', 'plan-eng-review-artifact': 'rule', @@ -1700,10 +1659,6 @@ export const E2E_KINDS: Record = { 'plan-review-prosons-hardstop-neg': 'behavior', 'plan-review-prosons-neutral-neg': 'behavior', 'plan-tune-inspect': 'rule', - 'codex-offered-office-hours': 'rule', - 'codex-offered-ceo-review': 'rule', - 'codex-offered-design-review': 'rule', - 'codex-offered-eng-review': 'rule', 'timeline-event-flow': 'rule', 'context-recovery-artifacts': 'rule', 'context-save-writes-file': 'rule', diff --git a/test/paid-free-boundary.test.ts b/test/paid-free-boundary.test.ts index 368a439ab..769f33443 100644 --- a/test/paid-free-boundary.test.ts +++ b/test/paid-free-boundary.test.ts @@ -182,7 +182,7 @@ describe('paid/free dependency boundary', () => { const real = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: [file, 'plan-ceo-review/SKILL.md.tmpl'] }); expect(real.coverage?.mode).toBe('pr'); - expect(real.selection.e2e).toContain('plan-ceo-review-benefits'); + expect(real.selection.e2e).toContain('auq-format-gate'); expect(real.selection.judges).toContain('plan-ceo-review/SKILL.md modes'); expect(real.selection.e2e?.every(id => PR_PROFILE_CASE_IDS.includes(id as typeof PR_PROFILE_CASE_IDS[number]))).toBe(true); }); diff --git a/test/paid-pr-profile.test.ts b/test/paid-pr-profile.test.ts index 8e46944df..4439e5b59 100644 --- a/test/paid-pr-profile.test.ts +++ b/test/paid-pr-profile.test.ts @@ -67,7 +67,7 @@ describe('PR profile paid-runner integration', () => { test('planner binds E2E and judge IDs and explicitly defers direct-describe broad probes', () => { const manifest = ceoManifest(); expect(manifest.profile).toBe('pr'); - expect(manifest.selection?.e2e).toContain('plan-ceo-review-benefits'); + expect(manifest.selection?.e2e).toContain('auq-format-gate'); expect(manifest.selection?.e2e).not.toContain('plan-ceo-review-plan-mode'); expect(manifest.selection?.judges).toContain('plan-ceo-review/SKILL.md modes'); expect(manifest.entries.find(entry => entry.file.includes('skill-e2e-retro'))?.status).toBe('skipped-by-diff'); @@ -150,8 +150,8 @@ describe('PR profile paid-runner integration', () => { expect(() => parseRunManifest(JSON.stringify(injected))).toThrow('outside its PR case selection'); for (const action of ['remove', 'skip', 'duplicate'] as const) { const missing = structuredClone(manifest); - // plan.test is case-sharded: its PR case runs as `#`. - const file = manifest.entries.find(entry => entry.status === 'planned' && entry.file.startsWith('test/skill-e2e-plan.test.ts#'))!.file; + // The CEO template's PR case runs from the AskUserQuestion format file. + const file = manifest.entries.find(entry => entry.status === 'planned' && entry.file.includes('ask-user-question'))!.file; if (action === 'remove') missing.entries = missing.entries.filter(entry => entry.file !== file); if (action === 'skip') missing.entries.find(entry => entry.file === file)!.status = 'skipped-by-diff'; if (action === 'duplicate') missing.entries.push({ ...missing.entries.find(entry => entry.file === file)! }); @@ -192,16 +192,16 @@ describe('PR profile paid-runner integration', () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'pr-profile-')); const file = 'test/skill-e2e-plan.test.ts'; const receipt = path.join(root, 'receipt.txt'); - const selection: PaidCaseSelection = { e2e: ['plan-ceo-review-benefits'], judges: [] }; + const selection: PaidCaseSelection = { e2e: ['plan-review-report'], judges: [] }; try { fs.mkdirSync(path.join(root, 'test')); fs.writeFileSync(path.join(root, file), ` import { test } from 'bun:test'; import { appendFileSync } from 'node:fs'; import { describeIfSelected, testIfSelected } from ${JSON.stringify(path.join(ROOT, 'test/helpers/e2e-helpers.ts'))}; - describeIfSelected('fixture', ['plan-ceo-review-benefits', 'plan-review-report'], () => { - testIfSelected('plan-ceo-review-benefits', async () => { appendFileSync(${JSON.stringify(receipt)}, 'selected\\n'); }, 5000); - testIfSelected('plan-review-report', async () => { throw new Error('unselected model boundary executed'); }, 5000); + describeIfSelected('fixture', ['plan-review-report', 'plan-ceo-review'], () => { + test('/plan-eng-review writes GSTACK REVIEW REPORT to plan file', async () => { appendFileSync(${JSON.stringify(receipt)}, 'selected\\n'); }, 5000); + testIfSelected('plan-ceo-review', async () => { throw new Error('unselected model boundary executed'); }, 5000); test('unexpected raw paid call', () => { throw new Error('raw model boundary executed'); }); }); `); diff --git a/test/paid-retry-supervision.test.ts b/test/paid-retry-supervision.test.ts index 3b967a507..b5de034b3 100644 --- a/test/paid-retry-supervision.test.ts +++ b/test/paid-retry-supervision.test.ts @@ -100,8 +100,7 @@ test('source allowances retain all captures, cases, and finalization grace', () expect(timeoutExpressions('test/skill-e2e-auq-matrix.test.ts')).toEqual(['CAPTURE_MS']); expect(timeoutExpressions('test/skill-e2e-plan.test.ts')).toEqual([ 'PTY_MS', 'PTY_MS', 'CAPTURE_LONG_MS', 'CAPTURE_LONG_MS', 'CAPTURE_LONG_MS', - 'CAPTURE_MS', 'CAPTURE_MS', 'CAPTURE_LONG_MS + OFFICE_HOURS_BUN_GRACE_MS', - 'CAPTURE_MS', 'CAPTURE_MS', 'CAPTURE_MS', 'CAPTURE_MS', + 'CAPTURE_LONG_MS + OFFICE_HOURS_BUN_GRACE_MS', ]); }); @@ -211,7 +210,7 @@ test('detached PR fallback and release commands cover their actual default worke const prFloor = Math.ceil((Math.ceil(fullGateFiles.length / prWorkers) * 1_800_000 + fullGateFiles.reduce( (total, file) => total + Math.max(0, resolvePaidShardBudget([file]).timeoutMs - 1_800_000), 0, )) / 1000 * 1.05); - expect(prFloor).toBe(78_425); + expect(prFloor).toBe(72_755); expect(prWall).toBe(92_820_000); expect(prWall).toBeGreaterThanOrEqual(paidShardWallUpperBoundMs(files, prWorkers) + 120_000); diff --git a/test/skill-e2e-plan.test.ts b/test/skill-e2e-plan.test.ts index cb91b20f8..1d862211c 100644 --- a/test/skill-e2e-plan.test.ts +++ b/test/skill-e2e-plan.test.ts @@ -11,9 +11,6 @@ import { finalizeEvalCollector, } from './helpers/e2e-helpers'; import { judgePosture } from './helpers/llm-judge'; -import { extractSkillSections } from './helpers/skill-fixture'; -import { buildCodexOfferingPrompt } from './helpers/codex-offering-fixture'; -import { validateOfficeHoursSpecSummary } from './helpers/office-hours-completion'; import { spawnSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; @@ -513,140 +510,6 @@ Write your review to ${planDir}/review-output.md`, }, CAPTURE_LONG_MS); }); -// --- Office Hours Spec Review E2E --- - -describeIfSelected('Office Hours Spec Review E2E', ['office-hours-spec-review'], () => { - let ohDir: string; - - beforeAll(() => { - ohDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-oh-spec-')); - const run = (cmd: string, args: string[]) => - spawnSync(cmd, args, { cwd: ohDir, stdio: 'pipe', timeout: 5000 }); - - run('git', ['init', '-b', 'main']); - run('git', ['config', 'user.email', 'test@test.com']); - run('git', ['config', 'user.name', 'Test']); - fs.writeFileSync(path.join(ohDir, 'README.md'), '# Test Project\n'); - run('git', ['add', '.']); - run('git', ['commit', '-m', 'init']); - - // This case explains the review procedure. Extract its actual section; - // the dedicated full-workflow case exercises skeleton/section discovery. - fs.mkdirSync(path.join(ohDir, 'office-hours'), { recursive: true }); - const fixturePath = path.join(ohDir, 'office-hours', 'spec-review.md'); - // The extractor requires the entry's real frontmatter. Recombine the entry - // and carved body locally, then keep only the review section for the model. - fs.writeFileSync( - fixturePath, - fs.readFileSync(path.join(ROOT, 'office-hours/SKILL.md'), 'utf-8') + '\n' - + fs.readFileSync(path.join(ROOT, 'office-hours/sections/design-and-handoff.md'), 'utf-8'), - ); - fs.writeFileSync(fixturePath, extractSkillSections(fixturePath, ['Spec Review Loop'])); - }); - - afterAll(() => { - try { fs.rmSync(ohDir, { recursive: true, force: true }); } catch {} - }); - - testConcurrentIfSelected('office-hours-spec-review', async () => { - const result = await runSkillTest({ - prompt: `Read office-hours/spec-review.md. This is a documentation question: explain the procedure without executing office hours. - -Summarize what the "Spec Review Loop" section does — specifically: -1. How many dimensions does the reviewer check? -2. What tool is used to dispatch the reviewer? -3. What's the maximum number of iterations? -4. What metrics are tracked? - -Write your summary to ${ohDir}/spec-review-summary.md`, - workingDirectory: ohDir, - // Preserve this case's existing turn/time allowances. The fixture now - // supplies the section it asks about instead of a carved skeleton. - maxTurns: 12, - timeout: JUDGE_MS, - testName: 'office-hours-spec-review', - runId, - }); - - logCost('/office-hours spec review', result); - let validationError: unknown; - try { - const summaryPath = path.join(ohDir, 'spec-review-summary.md'); - validateOfficeHoursSpecSummary(result.exitReason, - fs.existsSync(summaryPath) ? fs.readFileSync(summaryPath, 'utf-8') : null); - } catch (error) { - validationError = error; - throw error; - } finally { - recordE2E(evalCollector, '/office-hours-spec-review', 'Office Hours Spec Review E2E', result, - validationError ? { - passed: false, - exit_reason: result.exitReason === 'success' ? 'validation_failed' : result.exitReason, - output: String(validationError).slice(0, 2000), - } : undefined); - } - }, CAPTURE_MS); -}); - -// --- Plan CEO Review Benefits-From E2E --- - -describeIfSelected('Plan CEO Review Benefits-From E2E', ['plan-ceo-review-benefits'], () => { - let benefitsDir: string; - - beforeAll(() => { - benefitsDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-benefits-')); - const run = (cmd: string, args: string[]) => - spawnSync(cmd, args, { cwd: benefitsDir, stdio: 'pipe', timeout: 5000 }); - - run('git', ['init', '-b', 'main']); - run('git', ['config', 'user.email', 'test@test.com']); - run('git', ['config', 'user.name', 'Test']); - fs.writeFileSync(path.join(benefitsDir, 'README.md'), '# Test Project\n'); - run('git', ['add', '.']); - run('git', ['commit', '-m', 'init']); - - fs.mkdirSync(path.join(benefitsDir, 'plan-ceo-review'), { recursive: true }); - fs.copyFileSync( - path.join(ROOT, 'plan-ceo-review', 'SKILL.md'), - path.join(benefitsDir, 'plan-ceo-review', 'SKILL.md'), - ); - { const _sec = path.join(ROOT, 'plan-ceo-review', 'sections'); if (fs.existsSync(_sec)) fs.cpSync(_sec, path.join(benefitsDir, 'plan-ceo-review', 'sections'), { recursive: true }); } - }); - - afterAll(() => { - try { fs.rmSync(benefitsDir, { recursive: true, force: true }); } catch {} - }); - - testConcurrentIfSelected('plan-ceo-review-benefits', async () => { - const result = await runSkillTest({ - prompt: `Read plan-ceo-review/SKILL.md. Search for sections about "Prerequisite" or "office-hours" or "design doc found". - -Summarize what happens when no design doc is found — specifically: -1. Is /office-hours offered as a prerequisite? -2. What options does the user get? -3. Is there a mid-session detection for when the user seems lost? - -Write your summary to ${benefitsDir}/benefits-summary.md`, - workingDirectory: benefitsDir, - maxTurns: 8, - timeout: JUDGE_MS, - testName: 'plan-ceo-review-benefits', - runId, - }); - - logCost('/plan-ceo-review benefits-from', result); - recordE2E(evalCollector, '/plan-ceo-review-benefits', 'Plan CEO Review Benefits-From E2E', result); - expect(result.exitReason).toBe('success'); - - const summaryPath = path.join(benefitsDir, 'benefits-summary.md'); - if (fs.existsSync(summaryPath)) { - const summary = fs.readFileSync(summaryPath, 'utf-8').toLowerCase(); - expect(summary).toMatch(/office.hours/); - expect(summary).toMatch(/design doc|no design/i); - } - }, CAPTURE_MS); -}); - // --- Plan Review Report E2E --- // Verifies that plan-eng-review writes a "## GSTACK REVIEW REPORT" section // to the bottom of the plan file (the living review status footer). @@ -775,93 +638,6 @@ plan.md is the plan file for this review session.`, }); // --- Codex Offering E2E --- -// Verifies that Codex is properly offered (with availability check, user prompt, -// and fallback) in office-hours, plan-ceo-review, plan-design-review, plan-eng-review. - -describeIfSelected('Codex Offering E2E', [ - 'codex-offered-office-hours', 'codex-offered-ceo-review', - 'codex-offered-design-review', 'codex-offered-eng-review', -], () => { - let testDir: string; - - beforeAll(() => { - testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-codex-offer-')); - const run = (cmd: string, args: string[]) => - spawnSync(cmd, args, { cwd: testDir, stdio: 'pipe', timeout: 5000 }); - - run('git', ['init', '-b', 'main']); - run('git', ['config', 'user.email', 'test@test.com']); - run('git', ['config', 'user.name', 'Test']); - fs.writeFileSync(path.join(testDir, 'README.md'), '# Test Project\n'); - run('git', ['add', '.']); - run('git', ['commit', '-m', 'init']); - - // Copy all 4 SKILL.md files - for (const skill of ['office-hours', 'plan-ceo-review', 'plan-design-review', 'plan-eng-review']) { - fs.mkdirSync(path.join(testDir, skill), { recursive: true }); - fs.copyFileSync( - path.join(ROOT, skill, 'SKILL.md'), - path.join(testDir, skill, 'SKILL.md'), - ); - // Carved skills (v2 plan T9): copy sections/ so codex/outside-voice content - // (carved into review-sections.md) is present for the search. - const _sec = path.join(ROOT, skill, 'sections'); - if (fs.existsSync(_sec)) fs.cpSync(_sec, path.join(testDir, skill, 'sections'), { recursive: true }); - } - }); - - afterAll(() => { - try { fs.rmSync(testDir, { recursive: true, force: true }); } catch {} - }); - - async function checkCodexOffering(skill: string, testName: string, featureName: string) { - const result = await runSkillTest({ - prompt: buildCodexOfferingPrompt({ - root: testDir, skill, featureName, - summaryPath: path.join(testDir, `${testName}-summary.md`), - }), - workingDirectory: testDir, - maxTurns: 8, - timeout: JUDGE_MS, - testName, - runId, - }); - - logCost(`/${skill} codex offering`, result); - recordE2E(evalCollector, `/${testName}`, 'Codex Offering E2E', result); - expect(result.exitReason).toBe('success'); - - const summaryPath = path.join(testDir, `${testName}-summary.md`); - expect(fs.existsSync(summaryPath)).toBe(true); - - const summary = fs.readFileSync(summaryPath, 'utf-8').toLowerCase(); - // All skills should have codex availability check (command -v per #1197) - expect(summary).toMatch(/command -v codex/); - // All skills should have fallback behavior - expect(summary).toMatch(/fallback|subagent|unavailable|not available|skip/); - // All skills should show it's optional/non-blocking - expect(summary).toMatch(/optional|non.?blocking|skip|not.*required/); - - console.log(`${skill}: Codex offering verified`); - } - - testConcurrentIfSelected('codex-offered-office-hours', async () => { - await checkCodexOffering('office-hours', 'codex-offered-office-hours', 'second opinion'); - }, CAPTURE_MS); - - testConcurrentIfSelected('codex-offered-ceo-review', async () => { - await checkCodexOffering('plan-ceo-review', 'codex-offered-ceo-review', 'outside voice'); - }, CAPTURE_MS); - - testConcurrentIfSelected('codex-offered-design-review', async () => { - await checkCodexOffering('plan-design-review', 'codex-offered-design-review', 'design outside voices'); - }, CAPTURE_MS); - - testConcurrentIfSelected('codex-offered-eng-review', async () => { - await checkCodexOffering('plan-eng-review', 'codex-offered-eng-review', 'outside voice'); - }, CAPTURE_MS); -}); - // Module-level afterAll — finalize eval collector after all tests complete afterAll(async () => { await finalizeEvalCollector(evalCollector); diff --git a/test/test-pr-profile.test.ts b/test/test-pr-profile.test.ts index 2ec5ec0c7..55b4e984f 100644 --- a/test/test-pr-profile.test.ts +++ b/test/test-pr-profile.test.ts @@ -32,7 +32,6 @@ const select = (options: Partial[0]> = {}) => describe('fast PR coverage policy', () => { test('canonical short probes exist in the unchanged broad gate census', () => { expect(() => validatePrProfileInventory()).not.toThrow(); - expect(PR_PROFILE_CASE_IDS).toContain('plan-ceo-review-benefits'); expect(PR_PROFILE_CASE_IDS).toContain('plan-review-report'); expect(PR_PROFILE_CASE_IDS).toContain('auq-format-gate'); for (const tier of Object.values(PR_PROFILE_MAPS.tiers)) expect(['gate', 'periodic', 'marathon']).toContain(tier); diff --git a/test/touchfiles.test.ts b/test/touchfiles.test.ts index a006c7f96..c25afdb5b 100644 --- a/test/touchfiles.test.ts +++ b/test/touchfiles.test.ts @@ -278,9 +278,7 @@ describe('selectTests', () => { const result = selectTests(['plan-ceo-review/SKILL.md'], E2E_TOUCHFILES); expect(result.selected).toContain('plan-ceo-review'); expect(result.selected).toContain('plan-ceo-review-selective'); - expect(result.selected).toContain('plan-ceo-review-benefits'); expect(result.selected).toContain('plan-ceo-review-expansion-energy'); - expect(result.selected).toContain('codex-offered-ceo-review'); expect(result.selected).toContain('plan-ceo-review-format-mode'); expect(result.selected).toContain('plan-ceo-review-format-approach'); // v1.10.2.0 plan-mode handshake entries also depend on plan-ceo-review/** @@ -308,8 +306,8 @@ describe('selectTests', () => { // v2 plan Phase B carve: the section-loading E2E depends on plan-ceo-review/**. expect(result.selected).toContain('plan-ceo-section-loading'); expect(result.selected).toContain('outside-plan-disabled-no-fallback'); - expect(result.selected.length).toBe(21); - expect(result.skipped.length).toBe(Object.keys(E2E_TOUCHFILES).length - 21); + expect(result.selected.length).toBe(19); + expect(result.skipped.length).toBe(Object.keys(E2E_TOUCHFILES).length - 19); }); test('global touchfile triggers ALL tests', () => {