From 00ff213ddbae7e845253508a9422e6c23107cbeb Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Tue, 8 Sep 2026 06:41:50 +0000 Subject: [PATCH] test: classify collapsed review modes and isolate seeded findings Keep review questions out of the setup count when terminal cursor positioning removes spaces. State existing webhook safeguards so the five-finding control measures its seeded defects without accidental extra security and concurrency gaps. Preserve question bands and the paired control. Co-Authored-By: OpenAI Codex --- test/helpers/claude-pty-runner.ts | 9 +++++---- test/helpers/claude-pty-runner.unit.test.ts | 14 ++++++++++++++ test/helpers/touchfiles-data.ts | 4 ++-- test/skill-e2e-plan-ceo-finding-count.test.ts | 13 ++++++++++++- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/test/helpers/claude-pty-runner.ts b/test/helpers/claude-pty-runner.ts index ba56825ed..2a48bd946 100644 --- a/test/helpers/claude-pty-runner.ts +++ b/test/helpers/claude-pty-runner.ts @@ -817,16 +817,17 @@ export function parseNumberedOptions( } /** - * The four /plan-ceo-review modes. Used by `skill-e2e-plan-ceo-mode-routing` - * to detect Step 0F mode-selection AskUserQuestions, and by the upcoming - * finding-count tests as a Step-0 boundary signal: an AUQ whose options + * The four /plan-ceo-review modes, used by the finding-count tests as a + * Step-0 boundary signal: an AUQ whose options * match this regex IS the mode pick (the last Step-0 question for plan-ceo). * * Lifted out of the mode-routing test so multiple PTY tests can share one * source of truth — when /plan-ceo-review adds a fifth mode, one regex updates * everywhere instead of drifting per-test. */ -export const MODE_RE = /HOLD SCOPE|SCOPE EXPANSION|SELECTIVE EXPANSION|SCOPE REDUCTION/i; +// Cursor-positioning escapes render inter-word spaces that stripAnsi removes. +// Recognize both the spaced labels and their captured HOLDSCOPE-style forms. +export const MODE_RE = /HOLD\s*SCOPE|SCOPE\s*EXPANSION|SELECTIVE\s*EXPANSION|SCOPE\s*REDUCTION/i; /** * Stable signature for a parsed numbered-option list — used by tests to detect diff --git a/test/helpers/claude-pty-runner.unit.test.ts b/test/helpers/claude-pty-runner.unit.test.ts index b46a3faed..05f54d8ee 100644 --- a/test/helpers/claude-pty-runner.unit.test.ts +++ b/test/helpers/claude-pty-runner.unit.test.ts @@ -1089,6 +1089,20 @@ describe('Step0BoundaryPredicate per-skill', () => { expect(ceoStep0Boundary(f)).toBe(true); }); + test('FIRES on collapsed mode labels captured from the 2026-09-08 paid controls', () => { + // Test each captured label independently: a spaced sibling option can + // otherwise conceal the mismatch and leave every review AUQ in Step 0. + const labels = [ + 'HOLDSCOPE—makeitbulletproof(Recommended)', + 'SELECTIVEEXPANSION┌────────────────────────────────────────────────────────────────────────────────────┐\r (ecommnded) │SELECTIVEEXPANSION│', + 'SCOPEEXPANSION│Neutralposture:presentopportunities,stateeffort,youdecide.│\r │ Good for: substantialfeaturewithsolidfoundation,shippedbeforescopelock.│\r└────────────────────────────────────────────────────────────────────┘', + 'SCOPEREDUCTION—findtheminimalversion', + ]; + for (const label of labels) { + expect(ceoStep0Boundary(fp('Pick a mode', [label, 'Type something.']))).toBe(true); + } + }); + test('FIRES on scope-selection AUQ with "Skip interview" option (skip-interview path)', () => { // After calibration run 1: plan-ceo's first AUQ is scope-selection, // and we route via "Skip interview and plan immediately" to bypass diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index 32fc74e18..72e0b156b 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -160,7 +160,7 @@ export const E2E_TOUCHFILES: Record = { // Each test drives its skill end-to-end; touchfiles include preamble + // completion-status resolvers because they affect question cadence and // terminal output (the regression surface this test catches). - 'plan-ceo-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-ceo-finding-count.test.ts'], + 'plan-ceo-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-ceo-finding-count.test.ts'], 'plan-eng-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-eng-finding-count.test.ts'], 'plan-design-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-design-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-design-finding-count.test.ts'], 'plan-devex-finding-count': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-devex-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/skill-e2e-plan-devex-finding-count.test.ts'], @@ -181,7 +181,7 @@ export const E2E_TOUCHFILES: Record = { // confirm" plan write. runPlanSkillFloorCheck cannot detect that shape // (it exits on first AUQ); runPlanSkillCounting can. 'plan-eng-multi-finding-batching': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-eng-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-eng-multi-finding-batching.test.ts'], - 'plan-ceo-split-overflow': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-question-preference', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-split-overflow.test.ts'], + 'plan-ceo-split-overflow': ['plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-question-preference', 'test/helpers/claude-pty-runner.ts', 'test/helpers/claude-pty-runner.unit.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/helpers/plan-count-fixture.ts', 'test/plan-count-fixture.test.ts', 'test/fixtures/forcing-finding-seeds.ts', 'test/skill-e2e-plan-ceo-split-overflow.test.ts'], 'brain-privacy-gate': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'scripts/resolvers/preamble/generate-brain-sync-block.ts', 'scripts/resolvers/preamble.ts', 'bin/gstack-brain-sync', 'bin/gstack-artifacts-init', 'bin/gstack-config', 'test/helpers/agent-sdk-runner.ts', 'test/skill-e2e-brain-privacy-gate.test.ts'], // /setup-gbrain Path 4 (Remote MCP) — happy + bad-token end-to-end via diff --git a/test/skill-e2e-plan-ceo-finding-count.test.ts b/test/skill-e2e-plan-ceo-finding-count.test.ts index 6ed054bc2..b173a65e8 100644 --- a/test/skill-e2e-plan-ceo-finding-count.test.ts +++ b/test/skill-e2e-plan-ceo-finding-count.test.ts @@ -64,13 +64,24 @@ const N_PAIRED = 2; const FLOOR_PAIRED = 2; const CEILING_PAIRED = 4; +// Keep the five seeded defects distinct from already-satisfied surrounding +// contracts. The live control correctly found extra signature, deduplication, +// concurrency, and naming gaps when those baseline facts were unspecified. const planCeo5Findings = (planPath: string) => [ `Please review this plan thoroughly. As you go, write your plan-mode plan to ${planPath} (use Edit/Write to that exact path).`, '', '# Plan: Payment Processing Integration', '', + '## Existing contracts retained', + 'The existing ingress middleware verifies the Stripe signature against the', + 'raw request body and rejects invalid signatures before invoking handlers.', + 'The existing webhook event guard deduplicates deliveries by Stripe event ID,', + 'and an existing per-user transaction lock serializes payment updates.', + 'The new handler runs inside those unchanged guards; this plan does not', + 'replace signature verification, event deduplication, or update locking.', + '', '## Architecture', - "We're adding a new `PaymentService` class that will handle Stripe webhooks.", + "We're adding a new `StripePaymentWebhookHandler` class that will handle Stripe webhooks.", 'This bypasses the existing `WebhookDispatcher` module — we want a clean', 'namespace separation.', '',