mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-12 19:44:48 +01:00
test(W7h): retire the bws session-awareness (ELI16) paid case
W7h retirement (t14 F5). Removes the case from test/skill-e2e-bws.test.ts and
its E2E_TOUCHFILES / E2E_TIERS (gate) / E2E_KINDS (rule) entries in
test/helpers/touchfiles-data.ts; scripts/paid-test-durations.json is
per-file and test/helpers/periodic-exclude-data.ts had no entry.
test/touchfiles.test.ts selection examples now name sibling cases with the
same touchfiles (skillmd-outside-git, skillmd-no-local-binary).
e2e-tier-alignment, eval-kinds and touchfiles stay green.
Retirement card:
- test: skill-e2e-bws.test.ts 'session-awareness' (gate, paid).
- detects: nothing gstack ships. The prompt coaches a mode that no longer
exists ("_SESSIONS=4, so ELI16 mode is active ... Re-ground them";
`git grep ELI16` outside test/ and CHANGELOG is empty) and says "DO NOT
actually call AskUserQuestion"; every asserted word (billing/acme,
payment/feature, stripe/checkout, recommend) comes from the prompt itself.
- non_test_callers: none.
- search_command: git grep -n "session-awareness\|ELI16" -- test scripts
- stronger_proof / replacement: the AUQ format's grounding line
'Project/branch/task: <1 short grounding sentence using _BRANCH>' is pinned
free by plan-review-calibration.test.ts (format tokens) and graded by gate
E2E auq-format-gate; session identity in the preamble stays pinned by
skill-validation.test.ts 'contains session awareness' and
gen-skill-docs.test.ts 'session awareness lives in gstack-skill-start'.
- history: c118e24 v1.64.1.0 (#2572).
- validation: bun test test/touchfiles.test.ts test/e2e-tier-alignment.test.ts
test/eval-kinds.test.ts.
Control (scratch worktree, against the replacement): removing the
'Project/branch/task:' grounding line from the AUQ format resolver turns
plan-review-calibration.test.ts red (1/1). Log:
.context/mutation-session-awareness.md.
This commit is contained in:
1 parent
8ac565d615
commit
d636245c38
3 files changed
+4
-114
No files matched your search
@@ -96,7 +96,6 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'skillmd-no-local-binary': [ 'SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-bws.test.ts'],
|
||||
'skillmd-outside-git': [ 'SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-bws.test.ts'],
|
||||
|
||||
'session-awareness': [ 'SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-bws.test.ts'],
|
||||
'operational-learning': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'scripts/resolvers/preamble.ts', 'bin/gstack-learnings-log', 'test/skill-e2e-bws.test.ts'],
|
||||
|
||||
// QA (+ test-server dependency). /qa drives Aside first (the resolver) and
|
||||
@@ -1150,7 +1149,6 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic' | 'marathon'> = {
|
||||
'skillmd-setup-discovery': 'gate',
|
||||
'skillmd-no-local-binary': 'gate',
|
||||
'skillmd-outside-git': 'gate',
|
||||
'session-awareness': 'gate',
|
||||
'operational-learning': 'gate',
|
||||
|
||||
// P4 first-run scaffold — periodic (onboarding, non-safety, model-touched marker)
|
||||
@@ -1613,7 +1611,6 @@ export const E2E_KINDS: Record<string, 'rule' | 'behavior' | 'judge'> = {
|
||||
'skillmd-setup-discovery': 'rule',
|
||||
'skillmd-no-local-binary': 'rule',
|
||||
'skillmd-outside-git': 'rule',
|
||||
'session-awareness': 'rule',
|
||||
'operational-learning': 'rule',
|
||||
'qa-quick': 'rule',
|
||||
'qa-b6-static': 'rule',
|
||||
|
||||
+1
-108
@@ -20,7 +20,7 @@ let tmpDir: string;
|
||||
|
||||
describeIfSelected('Skill E2E tests', [
|
||||
'browse-basic', 'browse-snapshot', 'skillmd-setup-discovery',
|
||||
'skillmd-no-local-binary', 'skillmd-outside-git', 'session-awareness',
|
||||
'skillmd-no-local-binary', 'skillmd-outside-git',
|
||||
'operational-learning',
|
||||
], () => {
|
||||
beforeAll(() => {
|
||||
@@ -301,113 +301,6 @@ Log the operational learning now. Then say what you logged.`,
|
||||
try { fs.rmSync(opDir, { recursive: true, force: true }); } catch {}
|
||||
}, JUDGE_MS);
|
||||
|
||||
testConcurrentIfSelected('session-awareness', async () => {
|
||||
const sessionDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-session-'));
|
||||
|
||||
// Set up a git repo so there's project/branch context to reference
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: sessionDir, stdio: 'pipe', timeout: 5000 });
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
fs.writeFileSync(path.join(sessionDir, 'app.rb'), '# my app\n');
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'init']);
|
||||
run('git', ['checkout', '-b', 'feature/add-payments']);
|
||||
// Add a remote so the agent can derive a project name
|
||||
run('git', ['remote', 'add', 'origin', 'https://github.com/acme/billing-app.git']);
|
||||
|
||||
// Extract AskUserQuestion format instructions from a generated SKILL.md.
|
||||
// ROOT/SKILL.md is the browse skill (Tier 1) and does NOT contain the
|
||||
// "## AskUserQuestion Format" section — that block is only emitted for
|
||||
// Tier 2+ skills by scripts/resolvers/preamble.ts. Use office-hours/SKILL.md
|
||||
// (Tier 3) which always has the format guidance baked in. Falls back to
|
||||
// the first SKILL.md that contains the header so a future template move
|
||||
// doesn't break this test again.
|
||||
let skillMdPath = path.join(ROOT, 'office-hours', 'SKILL.md');
|
||||
let skillMd = '';
|
||||
if (fs.existsSync(skillMdPath)) {
|
||||
skillMd = fs.readFileSync(skillMdPath, 'utf-8');
|
||||
}
|
||||
if (!skillMd.includes('## AskUserQuestion Format')) {
|
||||
// Fallback: scan top-level skill dirs for the first match.
|
||||
const skillDirs = fs.readdirSync(ROOT, { withFileTypes: true })
|
||||
.filter(d => d.isDirectory())
|
||||
.map(d => path.join(ROOT, d.name, 'SKILL.md'));
|
||||
for (const candidate of skillDirs) {
|
||||
if (!fs.existsSync(candidate)) continue;
|
||||
const content = fs.readFileSync(candidate, 'utf-8');
|
||||
if (content.includes('## AskUserQuestion Format')) {
|
||||
skillMd = content;
|
||||
skillMdPath = candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
const aqStart = skillMd.indexOf('## AskUserQuestion Format');
|
||||
const aqEnd = skillMd.indexOf('\n## ', aqStart + 1);
|
||||
const aqBlock = aqStart >= 0
|
||||
? skillMd.slice(aqStart, aqEnd > 0 ? aqEnd : undefined)
|
||||
: '';
|
||||
|
||||
const outputPath = path.join(sessionDir, 'question-output.md');
|
||||
|
||||
const result = await runSkillTest({
|
||||
prompt: `You are running a gstack skill. The session preamble detected _SESSIONS=4 (the user has 4 gstack windows open).
|
||||
|
||||
${aqBlock}
|
||||
|
||||
You are on branch feature/add-payments in the billing-app project. You were reviewing a plan to add Stripe integration.
|
||||
|
||||
You've hit a decision point: the plan doesn't specify whether to use Stripe Checkout (hosted) or Stripe Elements (embedded). You need to ask the user which approach to use.
|
||||
|
||||
Since this is non-interactive, DO NOT actually call AskUserQuestion. Instead, write the EXACT text you would display to the user (the full AskUserQuestion content) to the file: ${outputPath}
|
||||
|
||||
Remember: _SESSIONS=4, so ELI16 mode is active. The user is juggling multiple windows and may not remember what this conversation is about. Re-ground them.`,
|
||||
workingDirectory: sessionDir,
|
||||
maxTurns: 8,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'session-awareness',
|
||||
runId,
|
||||
});
|
||||
|
||||
logCost('session awareness', result);
|
||||
recordE2E(evalCollector, 'session awareness ELI16', 'Skill E2E tests', result);
|
||||
|
||||
// Verify the output contains ELI16 re-grounding context
|
||||
if (fs.existsSync(outputPath)) {
|
||||
const output = fs.readFileSync(outputPath, 'utf-8');
|
||||
const lower = output.toLowerCase();
|
||||
// Must mention project name
|
||||
expect(lower.includes('billing') || lower.includes('acme')).toBe(true);
|
||||
// Must mention branch
|
||||
expect(lower.includes('payment') || lower.includes('feature')).toBe(true);
|
||||
// Must mention what we're working on
|
||||
expect(lower.includes('stripe') || lower.includes('checkout') || lower.includes('payment')).toBe(true);
|
||||
// Must have a recommendation or structured options
|
||||
expect(
|
||||
output.includes('RECOMMENDATION') ||
|
||||
lower.includes('recommend') ||
|
||||
lower.includes('option a') ||
|
||||
lower.includes('which do you want') ||
|
||||
lower.includes('which approach')
|
||||
).toBe(true);
|
||||
} else {
|
||||
// Check agent output as fallback
|
||||
const output = result.output || '';
|
||||
const lowerOut = output.toLowerCase();
|
||||
expect(
|
||||
output.includes('RECOMMENDATION') ||
|
||||
lowerOut.includes('recommend') ||
|
||||
lowerOut.includes('option a') ||
|
||||
lowerOut.includes('which do you want') ||
|
||||
lowerOut.includes('which approach')
|
||||
).toBe(true);
|
||||
}
|
||||
|
||||
// Clean up
|
||||
try { fs.rmSync(sessionDir, { recursive: true, force: true }); } catch {}
|
||||
}, JUDGE_MS);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
|
||||
@@ -340,7 +340,7 @@ describe('selectTests', () => {
|
||||
expect(result.reason).toBe('diff');
|
||||
// Should include tests that depend on gen-skill-docs.ts
|
||||
expect(result.selected).toContain('skillmd-setup-discovery');
|
||||
expect(result.selected).toContain('session-awareness');
|
||||
expect(result.selected).toContain('skillmd-outside-git');
|
||||
expect(result.selected).toContain('journey-ideation');
|
||||
// Should NOT include tests that don't depend on it
|
||||
expect(result.selected).not.toContain('retro');
|
||||
@@ -389,8 +389,8 @@ describe('selectTests', () => {
|
||||
const result = selectTests(['SKILL.md.tmpl'], E2E_TOUCHFILES);
|
||||
// Should select the 7 tests that depend on root SKILL.md
|
||||
expect(result.selected).toContain('skillmd-setup-discovery');
|
||||
expect(result.selected).toContain('session-awareness');
|
||||
expect(result.selected).toContain('session-awareness');
|
||||
expect(result.selected).toContain('skillmd-no-local-binary');
|
||||
expect(result.selected).toContain('skillmd-outside-git');
|
||||
// Also selects journey routing tests (SKILL.md.tmpl in their touchfiles)
|
||||
expect(result.selected).toContain('journey-ideation');
|
||||
// Should NOT select unrelated non-routing tests
|
||||
|
||||
Reference in new issue
Block a user