From d636245c3849f74847b4e9cacfdd55928d2ede1a Mon Sep 17 00:00:00 2001 From: garrytan Date: Sat, 3 Oct 2026 01:35:14 +0000 Subject: [PATCH] test(W7h): retire the bws session-awareness (ELI16) paid case W7h retirement (t14 F5). Removes the case from test/skill-e2e-bws.test.ts and its E2E_TOUCHFILES / E2E_TIERS (gate) / E2E_KINDS (rule) entries in test/helpers/touchfiles-data.ts; scripts/paid-test-durations.json is per-file and test/helpers/periodic-exclude-data.ts had no entry. test/touchfiles.test.ts selection examples now name sibling cases with the same touchfiles (skillmd-outside-git, skillmd-no-local-binary). e2e-tier-alignment, eval-kinds and touchfiles stay green. Retirement card: - test: skill-e2e-bws.test.ts 'session-awareness' (gate, paid). - detects: nothing gstack ships. The prompt coaches a mode that no longer exists ("_SESSIONS=4, so ELI16 mode is active ... Re-ground them"; `git grep ELI16` outside test/ and CHANGELOG is empty) and says "DO NOT actually call AskUserQuestion"; every asserted word (billing/acme, payment/feature, stripe/checkout, recommend) comes from the prompt itself. - non_test_callers: none. - search_command: git grep -n "session-awareness\|ELI16" -- test scripts - stronger_proof / replacement: the AUQ format's grounding line 'Project/branch/task: <1 short grounding sentence using _BRANCH>' is pinned free by plan-review-calibration.test.ts (format tokens) and graded by gate E2E auq-format-gate; session identity in the preamble stays pinned by skill-validation.test.ts 'contains session awareness' and gen-skill-docs.test.ts 'session awareness lives in gstack-skill-start'. - history: c118e24 v1.64.1.0 (#2572). - validation: bun test test/touchfiles.test.ts test/e2e-tier-alignment.test.ts test/eval-kinds.test.ts. Control (scratch worktree, against the replacement): removing the 'Project/branch/task:' grounding line from the AUQ format resolver turns plan-review-calibration.test.ts red (1/1). Log: .context/mutation-session-awareness.md. --- test/helpers/touchfiles-data.ts | 3 - test/skill-e2e-bws.test.ts | 109 +------------------------------- test/touchfiles.test.ts | 6 +- 3 files changed, 4 insertions(+), 114 deletions(-) diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index ea7fc1b12..e2cc5e4e1 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -96,7 +96,6 @@ export const E2E_TOUCHFILES: Record = { 'skillmd-no-local-binary': [ 'SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-bws.test.ts'], 'skillmd-outside-git': [ 'SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-bws.test.ts'], - 'session-awareness': [ 'SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-bws.test.ts'], 'operational-learning': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'scripts/resolvers/preamble.ts', 'bin/gstack-learnings-log', 'test/skill-e2e-bws.test.ts'], // QA (+ test-server dependency). /qa drives Aside first (the resolver) and @@ -1150,7 +1149,6 @@ export const E2E_TIERS: Record = { 'skillmd-setup-discovery': 'gate', 'skillmd-no-local-binary': 'gate', 'skillmd-outside-git': 'gate', - 'session-awareness': 'gate', 'operational-learning': 'gate', // P4 first-run scaffold — periodic (onboarding, non-safety, model-touched marker) @@ -1613,7 +1611,6 @@ export const E2E_KINDS: Record = { 'skillmd-setup-discovery': 'rule', 'skillmd-no-local-binary': 'rule', 'skillmd-outside-git': 'rule', - 'session-awareness': 'rule', 'operational-learning': 'rule', 'qa-quick': 'rule', 'qa-b6-static': 'rule', diff --git a/test/skill-e2e-bws.test.ts b/test/skill-e2e-bws.test.ts index 4a8dbfdaf..c31846e5c 100644 --- a/test/skill-e2e-bws.test.ts +++ b/test/skill-e2e-bws.test.ts @@ -20,7 +20,7 @@ let tmpDir: string; describeIfSelected('Skill E2E tests', [ 'browse-basic', 'browse-snapshot', 'skillmd-setup-discovery', - 'skillmd-no-local-binary', 'skillmd-outside-git', 'session-awareness', + 'skillmd-no-local-binary', 'skillmd-outside-git', 'operational-learning', ], () => { beforeAll(() => { @@ -301,113 +301,6 @@ Log the operational learning now. Then say what you logged.`, try { fs.rmSync(opDir, { recursive: true, force: true }); } catch {} }, JUDGE_MS); - testConcurrentIfSelected('session-awareness', async () => { - const sessionDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-session-')); - - // Set up a git repo so there's project/branch context to reference - const run = (cmd: string, args: string[]) => - spawnSync(cmd, args, { cwd: sessionDir, stdio: 'pipe', timeout: 5000 }); - run('git', ['init', '-b', 'main']); - run('git', ['config', 'user.email', 'test@test.com']); - run('git', ['config', 'user.name', 'Test']); - fs.writeFileSync(path.join(sessionDir, 'app.rb'), '# my app\n'); - run('git', ['add', '.']); - run('git', ['commit', '-m', 'init']); - run('git', ['checkout', '-b', 'feature/add-payments']); - // Add a remote so the agent can derive a project name - run('git', ['remote', 'add', 'origin', 'https://github.com/acme/billing-app.git']); - - // Extract AskUserQuestion format instructions from a generated SKILL.md. - // ROOT/SKILL.md is the browse skill (Tier 1) and does NOT contain the - // "## AskUserQuestion Format" section — that block is only emitted for - // Tier 2+ skills by scripts/resolvers/preamble.ts. Use office-hours/SKILL.md - // (Tier 3) which always has the format guidance baked in. Falls back to - // the first SKILL.md that contains the header so a future template move - // doesn't break this test again. - let skillMdPath = path.join(ROOT, 'office-hours', 'SKILL.md'); - let skillMd = ''; - if (fs.existsSync(skillMdPath)) { - skillMd = fs.readFileSync(skillMdPath, 'utf-8'); - } - if (!skillMd.includes('## AskUserQuestion Format')) { - // Fallback: scan top-level skill dirs for the first match. - const skillDirs = fs.readdirSync(ROOT, { withFileTypes: true }) - .filter(d => d.isDirectory()) - .map(d => path.join(ROOT, d.name, 'SKILL.md')); - for (const candidate of skillDirs) { - if (!fs.existsSync(candidate)) continue; - const content = fs.readFileSync(candidate, 'utf-8'); - if (content.includes('## AskUserQuestion Format')) { - skillMd = content; - skillMdPath = candidate; - break; - } - } - } - const aqStart = skillMd.indexOf('## AskUserQuestion Format'); - const aqEnd = skillMd.indexOf('\n## ', aqStart + 1); - const aqBlock = aqStart >= 0 - ? skillMd.slice(aqStart, aqEnd > 0 ? aqEnd : undefined) - : ''; - - const outputPath = path.join(sessionDir, 'question-output.md'); - - const result = await runSkillTest({ - prompt: `You are running a gstack skill. The session preamble detected _SESSIONS=4 (the user has 4 gstack windows open). - -${aqBlock} - -You are on branch feature/add-payments in the billing-app project. You were reviewing a plan to add Stripe integration. - -You've hit a decision point: the plan doesn't specify whether to use Stripe Checkout (hosted) or Stripe Elements (embedded). You need to ask the user which approach to use. - -Since this is non-interactive, DO NOT actually call AskUserQuestion. Instead, write the EXACT text you would display to the user (the full AskUserQuestion content) to the file: ${outputPath} - -Remember: _SESSIONS=4, so ELI16 mode is active. The user is juggling multiple windows and may not remember what this conversation is about. Re-ground them.`, - workingDirectory: sessionDir, - maxTurns: 8, - timeout: JUDGE_MS, - testName: 'session-awareness', - runId, - }); - - logCost('session awareness', result); - recordE2E(evalCollector, 'session awareness ELI16', 'Skill E2E tests', result); - - // Verify the output contains ELI16 re-grounding context - if (fs.existsSync(outputPath)) { - const output = fs.readFileSync(outputPath, 'utf-8'); - const lower = output.toLowerCase(); - // Must mention project name - expect(lower.includes('billing') || lower.includes('acme')).toBe(true); - // Must mention branch - expect(lower.includes('payment') || lower.includes('feature')).toBe(true); - // Must mention what we're working on - expect(lower.includes('stripe') || lower.includes('checkout') || lower.includes('payment')).toBe(true); - // Must have a recommendation or structured options - expect( - output.includes('RECOMMENDATION') || - lower.includes('recommend') || - lower.includes('option a') || - lower.includes('which do you want') || - lower.includes('which approach') - ).toBe(true); - } else { - // Check agent output as fallback - const output = result.output || ''; - const lowerOut = output.toLowerCase(); - expect( - output.includes('RECOMMENDATION') || - lowerOut.includes('recommend') || - lowerOut.includes('option a') || - lowerOut.includes('which do you want') || - lowerOut.includes('which approach') - ).toBe(true); - } - - // Clean up - try { fs.rmSync(sessionDir, { recursive: true, force: true }); } catch {} - }, JUDGE_MS); }); // Module-level afterAll — finalize eval collector after all tests complete diff --git a/test/touchfiles.test.ts b/test/touchfiles.test.ts index a766d0ab8..a006c7f96 100644 --- a/test/touchfiles.test.ts +++ b/test/touchfiles.test.ts @@ -340,7 +340,7 @@ describe('selectTests', () => { expect(result.reason).toBe('diff'); // Should include tests that depend on gen-skill-docs.ts expect(result.selected).toContain('skillmd-setup-discovery'); - expect(result.selected).toContain('session-awareness'); + expect(result.selected).toContain('skillmd-outside-git'); expect(result.selected).toContain('journey-ideation'); // Should NOT include tests that don't depend on it expect(result.selected).not.toContain('retro'); @@ -389,8 +389,8 @@ describe('selectTests', () => { const result = selectTests(['SKILL.md.tmpl'], E2E_TOUCHFILES); // Should select the 7 tests that depend on root SKILL.md expect(result.selected).toContain('skillmd-setup-discovery'); - expect(result.selected).toContain('session-awareness'); - expect(result.selected).toContain('session-awareness'); + expect(result.selected).toContain('skillmd-no-local-binary'); + expect(result.selected).toContain('skillmd-outside-git'); // Also selects journey routing tests (SKILL.md.tmpl in their touchfiles) expect(result.selected).toContain('journey-ideation'); // Should NOT select unrelated non-routing tests