mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-03 01:46:55 +02:00
test: fold per-incident replay series into their detector owners (D)
Twelve detector families move into one owner test each: 73 incident files become describe blocks in ceo-section-loading-fixture (stale-fill race), model-overlays, coverage-audit-evidence, autoplan-phase-observer, native-auto-decide, outside-voice-evidence, eng-first-review, plan-count-completion, plan-count-file-permission, ceo-mode-option, plan-scope-selection and plan-count-prerequisite. Each block keeps its original code and fixture, so every case still runs; only tests asserting the incident file's own touchfile registration are dropped (41). Touchfile lists that named an incident now name its owner.
This commit is contained in:
garrytan committed
1 parent
6415690a18
commit
0489bc9fe8
89 files changed
+8091
-8373
No files matched your search
@@ -11,6 +11,21 @@ import { nativeSeededPlanSelection } from './helpers/plan-scope-selection';
|
||||
import { isScopeGateQuestionVisible } from './helpers/claude-pty-runner';
|
||||
import { readPlanCountTranscript, type NativePublicToolEvent, type PlanCountTranscript } from './helpers/plan-count-transcript';
|
||||
import { selectTests, E2E_TOUCHFILES } from './helpers/touchfiles';
|
||||
import captured_design_scope_announcement_ao from './fixtures/design-scope-announcement-ao.json';
|
||||
import fixture_design_scope_declaration_ak from './fixtures/design-scope-declaration-ak.json';
|
||||
import fs_design_scope_entry_aq from 'node:fs';
|
||||
import path_design_scope_entry_aq from 'node:path';
|
||||
import { ALL_HOST_CONFIGS } from '../hosts';
|
||||
import { HOST_PATHS } from '../scripts/resolvers/types';
|
||||
import type { TemplateContext } from '../scripts/resolvers/types';
|
||||
import { generatePreamble } from '../scripts/resolvers/preamble';
|
||||
import { generateBaseBranchDetect } from '../scripts/resolvers/utility';
|
||||
import failedScopes_design_scope_entry_aq from './fixtures/design-scope-checkpoint-at.json';
|
||||
import { isScopeGateAutoSelectVisible } from './helpers/claude-pty-runner';
|
||||
import capture_design_scope_selection_aj from './fixtures/design-scope-selection-aj.json';
|
||||
import fixture_eng_option_b_scope_al from './fixtures/eng-option-b-scope-al.json';
|
||||
import observedFailures_plan_scope_recovery_av from './fixtures/plan-scope-recovery-av.json';
|
||||
import { describe } from 'bun:test';
|
||||
|
||||
const START = Date.parse('2026-09-10T00:25:00Z');
|
||||
const opts = { seed: '# Plan: Marketing landing page\n\n## Layout\nA draft.', skillName: 'plan-design-review', sessionId: 'owned', commandStartedAt: START };
|
||||
@@ -426,3 +441,520 @@ test('spoken skill names retain scope ownership, currentness and target boundari
|
||||
expect(verdict(spokenEngFixture(text),{...engScope,seed:opts.seed+'\n# Another plan'})).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
describe('design-scope-announcement-ao', () => {
|
||||
const captured = captured_design_scope_announcement_ao;
|
||||
const input = () => structuredClone(captured.projection);
|
||||
type Input = ReturnType<typeof input>;
|
||||
const announcement = (p: Input) => p.transcript.assistantMessages.find(m => m.sessionId === p.opts.sessionId && m.text.startsWith("I'll auto-select"))!;
|
||||
const verdict = (p: Input) => nativeSeededPlanSelection(p.transcript as PlanCountTranscript, p.tools as NativePublicToolEvent[], p.opts);
|
||||
|
||||
test('the exact owned post-load option B announcement selects the seeded title', () => {
|
||||
expect(captured.rawScopeGateAutoSelectObserved).toBe(false);
|
||||
expect(verdict(input())).toBe(true);
|
||||
});
|
||||
|
||||
test('equivalent explicit selection words and balanced title quotes retain identity', () => {
|
||||
for (const prefix of ["I'll auto-select", 'I will auto-select', "I'll auto select"]) {
|
||||
for (const title of ['Marketing landing page', '"Marketing landing page"', '“Marketing landing page”', '`Marketing landing page`']) {
|
||||
const p = input(), m = announcement(p);
|
||||
m.text = m.text.replace("I'll auto-select", prefix).replace('Marketing landing page', title);
|
||||
expect(verdict(p)).toBe(true);
|
||||
}
|
||||
}
|
||||
const p = input(), m = announcement(p);
|
||||
p.opts.seed = p.opts.seed.replace('Marketing landing page', 'Account settings');
|
||||
m.text = m.text.replace('Marketing landing page', 'Account settings');
|
||||
expect(verdict(p)).toBe(true);
|
||||
});
|
||||
|
||||
const rejected: Array<[string, (p: Input) => void]> = [
|
||||
['wrong option', p => { announcement(p).text = announcement(p).text.replace('option B', 'option A'); }],
|
||||
['wrong target', p => { announcement(p).text = announcement(p).text.replace('Marketing landing page', 'Account settings'); }],
|
||||
['target prefix only', p => { announcement(p).text = announcement(p).text.replace('page draft', 'page experiment draft'); }],
|
||||
['conditional selection', p => { announcement(p).text = 'If approved: ' + announcement(p).text; }],
|
||||
['source selection', p => { announcement(p).text = 'Source excerpt:\n' + announcement(p).text; }],
|
||||
['quoted selection', p => { announcement(p).text = '> ' + announcement(p).text; }],
|
||||
['wholly quoted selection', p => { announcement(p).text = '"' + announcement(p).text + '"'; }],
|
||||
['unbalanced target quotes', p => { announcement(p).text = announcement(p).text.replace('Marketing landing page', '"Marketing landing page'); }],
|
||||
['question instead of assertion', p => { announcement(p).text = announcement(p).text.replace(/\.$/, '?'); }],
|
||||
['conditional tail', p => { announcement(p).text = announcement(p).text.replace(', running', ' if approved, running'); }],
|
||||
['cancelled selection', p => { announcement(p).text += '\nCorrection: this selection is withdrawn.'; }],
|
||||
['quoted status cancellation', p => { announcement(p).text += '\nThis selection is "withdrawn".'; }],
|
||||
['replaced target', p => { announcement(p).text += '\nThe selected target is now the branch diff.'; }],
|
||||
['pre-invocation announcement', p => { announcement(p).timestamp = new Date(p.opts.commandStartedAt - 1).toISOString(); }],
|
||||
['foreign announcement', p => { announcement(p).sessionId = 'foreign'; }],
|
||||
['foreign load result', p => { p.tools[1]!.sessionId = 'foreign'; }],
|
||||
['failed skill load', p => { p.tools[1]!.isError = true; }],
|
||||
['wrong skill', p => { p.tools[0]!.input!.skill = 'plan-eng-review'; }],
|
||||
['late command start', p => { p.opts.commandStartedAt = Date.parse(p.tools[1]!.timestamp) + 1; }],
|
||||
['multiple seed titles', p => { p.opts.seed += '\n# Another plan\n'; }],
|
||||
];
|
||||
test.each(rejected)('%s supplies no scope selection', (_, change) => {
|
||||
const p = input(); p.transcript.assistantMessages = [announcement(p)]; change(p); expect(verdict(p)).toBe(false);
|
||||
});
|
||||
|
||||
test('quoted historical or foreign withdrawals do not replace the current selection', () => {
|
||||
for (const correction of ['> This selection is withdrawn.', 'Historical note: "This selection is withdrawn."']) {
|
||||
const p = input(); announcement(p).text += '\n' + correction; expect(verdict(p)).toBe(true);
|
||||
}
|
||||
const p = input(), m = announcement(p);
|
||||
p.transcript.assistantMessages.push({ ...m, sessionId: 'foreign', text: 'This selection is withdrawn.' });
|
||||
expect(verdict(p)).toBe(true);
|
||||
});
|
||||
|
||||
test('a later current withdrawal invalidates selection until a later reselection', () => {
|
||||
const p = input(), m = announcement(p);
|
||||
p.transcript.assistantMessages.push({ ...m, timestamp: new Date(Date.parse(m.timestamp) + 1000).toISOString(), text: 'This selection is withdrawn.' });
|
||||
expect(verdict(p)).toBe(false);
|
||||
p.transcript.assistantMessages.push({ ...m, timestamp: new Date(Date.parse(m.timestamp) + 2000).toISOString() });
|
||||
expect(verdict(p)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('design-scope-declaration-ak', () => {
|
||||
const fixture = fixture_design_scope_declaration_ak;
|
||||
const input = (attempt = 0) => structuredClone(fixture.attempts[attempt]!.projection);
|
||||
const verdict = (p = input()) => nativeSeededPlanSelection(p.transcript as PlanCountTranscript, p.tools as NativePublicToolEvent[], p.opts);
|
||||
const declaration = (p: ReturnType<typeof input>) => p.transcript.assistantMessages.find(m => /^(?:I'll proceed with reviewing|Scope gate confirms plan mode)/.test(m.text))!;
|
||||
|
||||
test('both exact owned post-load announcements select the named pasted draft', () => {
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
const p = input(attempt);
|
||||
expect(fixture.attempts[attempt]!.rawScopeGateAutoSelectObserved).toBe(false);
|
||||
expect(verdict(p)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test('the prior AJ fresh unique-draft introduction now binds without changing its recorded outcome', () => {
|
||||
const p = fixture.priorGenuineFailure.projection;
|
||||
expect(nativeSeededPlanSelection(p.transcript as PlanCountTranscript, p.tools as NativePublicToolEvent[], p.opts)).toBe(true);
|
||||
});
|
||||
|
||||
test('target identity and ordinary equivalent current review wording remain bound', () => {
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
const p = input(attempt); p.opts.seed = p.opts.seed.replace('Marketing landing page', 'Account settings');
|
||||
p.transcript.assistantMessages.forEach(m => { m.text = m.text.replaceAll('Marketing landing page', 'Account settings'); });
|
||||
for (const t of p.tools) if (t.input?.args) t.input.args = t.input.args.replaceAll('Marketing landing page', 'Account settings');
|
||||
expect(verdict(p)).toBe(true);
|
||||
}
|
||||
const p = input(); declaration(p).text = declaration(p).text.replace("I'll proceed", 'I will proceed'); expect(verdict(p)).toBe(true);
|
||||
});
|
||||
|
||||
test('source, historical, quoted, hypothetical and conditional introductions do not select', () => {
|
||||
for (let attempt = 0; attempt < 2; attempt++) for (const prefix of [
|
||||
'> ', ' ', 'Source excerpt:\n', 'Historical example only.\n', 'The following is hypothetical. ', 'If approved, ', '```\n', '"',
|
||||
]) {
|
||||
const p = input(attempt), m = declaration(p); p.transcript.assistantMessages = [m]; m.text = prefix + m.text;
|
||||
expect(verdict(p)).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test('a different target or conditional scope announcement cannot borrow the draft name', () => {
|
||||
for (let attempt = 0; attempt < 2; attempt++) for (const change of [
|
||||
(s: string) => s.replaceAll('Marketing landing page', 'Checkout redesign'),
|
||||
(s: string) => s.replace(/draft(?: plan)?/, 'draft plan if approved'),
|
||||
]) { const p = input(attempt), m = declaration(p); p.transcript.assistantMessages = [m]; m.text = change(m.text); expect(verdict(p)).toBe(false); }
|
||||
for (const prefix of ['Scope gate might confirm plan mode, so', 'Scope gate confirms branch mode, so']) {
|
||||
const p = input(1); p.transcript.assistantMessages = [declaration(p)]; declaration(p).text = declaration(p).text.replace('Scope gate confirms plan mode, so', prefix); expect(verdict(p)).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test('the same successful Skill load and post-command current session remain necessary', () => {
|
||||
for (let attempt = 0; attempt < 2; attempt++) for (const change of [
|
||||
(p: ReturnType<typeof input>) => { p.opts.sessionId = 'foreign'; },
|
||||
(p: ReturnType<typeof input>) => { p.tools[1]!.isError = true; },
|
||||
(p: ReturnType<typeof input>) => { p.tools[1]!.toolUseId = 'foreign'; },
|
||||
(p: ReturnType<typeof input>) => { p.tools[0]!.input!.skill = 'plan-eng-review'; },
|
||||
(p: ReturnType<typeof input>) => { p.opts.commandStartedAt = Date.parse(p.tools[1]!.timestamp) + 1; },
|
||||
(p: ReturnType<typeof input>) => { declaration(p).timestamp = new Date(p.opts.commandStartedAt - 1).toISOString(); p.transcript.assistantMessages = [declaration(p)]; },
|
||||
]) { const p = input(attempt); change(p); expect(verdict(p)).toBe(false); }
|
||||
});
|
||||
|
||||
test('same-message and later current withdrawals or replacement targets defeat selection', () => {
|
||||
for (let attempt = 0; attempt < 2; attempt++) for (const correction of [
|
||||
'Correction: this selection is withdrawn.',
|
||||
'The selected target is now the branch diff.',
|
||||
'This declaration has been retracted.',
|
||||
]) for (const later of [false, true]) {
|
||||
const p = input(attempt), m = declaration(p); p.transcript.assistantMessages = [m];
|
||||
if (later) p.transcript.assistantMessages.push({ ...m, timestamp: new Date(Date.parse(m.timestamp) + 1000).toISOString(), text: correction });
|
||||
else m.text += '\n' + correction;
|
||||
expect(verdict(p)).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test('literal or foreign corrections preserve the actual declaration and a later reselection is current', () => {
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
const p = input(attempt), m = declaration(p); p.transcript.assistantMessages = [m];
|
||||
p.transcript.assistantMessages.push({ ...m, sessionId: 'foreign', text: 'The selected target is now the branch diff.' });
|
||||
p.transcript.assistantMessages.push({ ...m, text: '> This selection is withdrawn.' }); expect(verdict(p)).toBe(true);
|
||||
p.transcript.assistantMessages.push({ ...m, timestamp: new Date(Date.parse(m.timestamp) + 1000).toISOString(), text: 'This selection is withdrawn.' }); expect(verdict(p)).toBe(false);
|
||||
p.transcript.assistantMessages.push({ ...m, timestamp: new Date(Date.parse(m.timestamp) + 2000).toISOString() }); expect(verdict(p)).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('design-scope-entry-aq', () => {
|
||||
const fs = fs_design_scope_entry_aq;
|
||||
const path = path_design_scope_entry_aq;
|
||||
const failedScopes = failedScopes_design_scope_entry_aq;
|
||||
const template = fs.readFileSync(path.join(import.meta.dir, '../plan-design-review/SKILL.md.tmpl'), 'utf8');
|
||||
const scope = template.slice(template.indexOf('## Scope gate'), template.indexOf('## Design Philosophy'));
|
||||
const announcement = 'Scope gate: plan mode — auto-selected B (reviewing <target>).';
|
||||
|
||||
test('Design resolves scope before either executable bootstrap placeholder', () => {
|
||||
const gate = template.indexOf('## Scope gate');
|
||||
expect(gate).toBeGreaterThan(0);
|
||||
for (const token of ['{{PREAMBLE}}', '{{BASE_BRANCH_DETECT}}']) {
|
||||
expect(template.split(token)).toHaveLength(2);
|
||||
expect(template.indexOf(announcement)).toBeLessThan(template.indexOf(token));
|
||||
expect(template.indexOf('Reply with A, B, or C. STOP and wait')).toBeLessThan(template.indexOf(token));
|
||||
}
|
||||
expect(template.indexOf('{{PREAMBLE}}')).toBeLessThan(template.indexOf('{{BASE_BRANCH_DETECT}}'));
|
||||
expect(template.indexOf('{{BASE_BRANCH_DETECT}}')).toBeLessThan(template.indexOf('## Design Philosophy'));
|
||||
});
|
||||
|
||||
test('every host expands its real bootstrap after the mandatory entry gate', () => {
|
||||
for (const host of ALL_HOST_CONFIGS) {
|
||||
const ctx: TemplateContext = {skillName: 'plan-design-review', tmplPath: 'plan-design-review/SKILL.md.tmpl',
|
||||
host: host.name, paths: HOST_PATHS[host.name]!, preambleTier: 3, interactive: true};
|
||||
const preamble = generatePreamble(ctx);
|
||||
const brain = host.suppressedResolvers?.includes('BASE_BRANCH_DETECT') ? '' : generateBaseBranchDetect(ctx);
|
||||
const expanded = template.replace('{{PREAMBLE}}', preamble).replace('{{BASE_BRANCH_DETECT}}', brain);
|
||||
expect(expanded.indexOf(announcement)).toBeLessThan(expanded.indexOf('## Preamble (after scope gate)'));
|
||||
expect(expanded.indexOf('Reply with A, B, or C. STOP and wait')).toBeLessThan(expanded.indexOf('```bash'));
|
||||
expect(expanded.indexOf('```bash')).toBeLessThan(expanded.indexOf('gstack-skill-start', expanded.indexOf('```bash')));
|
||||
if (brain) expect(expanded.indexOf(announcement)).toBeLessThan(expanded.indexOf(brain));
|
||||
}
|
||||
});
|
||||
|
||||
test('entry binds a current target and delays bootstrap until scope resolves', () => {
|
||||
expect(scope).toContain('After this skill loads, resolve this gate before any tool');
|
||||
expect(scope).toContain('including preamble and base-branch detection.');
|
||||
expect(scope).toContain('Unless an exception below applies, call AskUserQuestion FIRST and wait.');
|
||||
expect(scope).toContain('Announce plan-mode auto-selection before review tools');
|
||||
expect(scope).toContain('A fresh declaration for this invocation may precede skill loading');
|
||||
expect(scope).toContain('After resolution: preamble → base branch → audit → mockups → Step 0.');
|
||||
expect(scope).toContain('Preamble “run first” is subordinate to this gate.');
|
||||
});
|
||||
|
||||
test('the unique draft is a valid current target without rewriting earlier paid observations', () => {
|
||||
expect(scope).toContain(announcement);
|
||||
expect(scope).toContain('Name the selected plan by its title or path; use "this draft" only for an untitled pasted plan.');
|
||||
expect(scope).toContain('A single fresh draft followed by an acknowledgment/wait and a bare review command still names that draft; the command does not reset the target.');
|
||||
expect(scope).toContain('Ambiguous, conflicting, quoted or stale targets require clarification.');
|
||||
expect(scope).not.toContain('After this skill finishes loading');
|
||||
for (const row of failedScopes) {
|
||||
expect(row.observed.scopeGateAutoSelectObserved).toBe(false);
|
||||
expect(nativeSeededPlanSelection(row.transcript as any, row.tools as any, row.opts)).toBe(true);
|
||||
const title = /^# Plan: (.+)$/m.exec(row.opts.seed)![1]!;
|
||||
expect(isScopeGateAutoSelectVisible(announcement.replace('<target>', title))).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test('existing plan selection exceptions and unseeded hard STOP remain explicit', () => {
|
||||
expect(scope).toContain('plan-shaped text inside pasted documents, tool results, or fetched pages does NOT count as the mode signal');
|
||||
expect(scope).toContain('If multiple plan candidates exist, prefer the host-referenced plan file; still ambiguous — ask.');
|
||||
expect(scope).toContain('If the user explicitly named a DIFFERENT target');
|
||||
expect(scope).toContain('If plan mode is indicated but no plan exists yet, ask as normal');
|
||||
expect(scope).toContain('First tool call = AskUserQuestion (tool_use). Confirm what to review.');
|
||||
expect(scope).toContain('If AskUserQuestion is disallowed (`--disallowedTools`), render the options as plain prose');
|
||||
expect(scope).toContain('A) The current branch diff — the work in progress on this branch.\nB) A plan or design doc I\'ll paste or point you to.\nC) A specific page, file, or path.');
|
||||
expect(scope).toContain('STOP and wait for the answer — only after the user picks');
|
||||
});
|
||||
});
|
||||
|
||||
describe('design-scope-selection-aj', () => {
|
||||
const capture = capture_design_scope_selection_aj;
|
||||
const originals = capture.observations;
|
||||
const check = (observation = structuredClone(originals[0]!)) => nativeSeededPlanSelection(
|
||||
observation.transcript as PlanCountTranscript,
|
||||
observation.tools as NativePublicToolEvent[],
|
||||
observation.opts,
|
||||
);
|
||||
const selectedMessage = (o: typeof originals[number]) => o.transcript.assistantMessages.find(m => m.text.includes('"Marketing landing page"'))!;
|
||||
|
||||
test('both actual explicit draft selections bind the named seed after this session loaded the skill', () => {
|
||||
for (const o of originals) expect(check(o)).toBe(true);
|
||||
for (const verb of ["I'll review", 'I will review', "I'll go with reviewing", 'I will go with reviewing']) {
|
||||
const o = structuredClone(originals[1]!);
|
||||
selectedMessage(o).text = `${verb} the "Marketing landing page" draft, starting by checking the design system.`;
|
||||
expect(check(o)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test('a named target still requires the successful current skill and invocation', () => {
|
||||
for (const original of originals) {
|
||||
for (const mutate of [
|
||||
(o: typeof original) => { o.opts.seed = '# Plan: Other page'; },
|
||||
(o: typeof original) => { o.opts.seed += '\n# Plan: Another'; },
|
||||
(o: typeof original) => { o.opts.sessionId = 'foreign'; },
|
||||
(o: typeof original) => { o.opts.commandStartedAt = Date.parse(selectedMessage(o).timestamp) + 1; },
|
||||
(o: typeof original) => { o.tools[0]!.input!.skill = 'plan-ceo-review'; },
|
||||
(o: typeof original) => { o.tools[1]!.isError = true; },
|
||||
(o: typeof original) => { o.tools[1]!.toolUseId = 'foreign'; },
|
||||
(o: typeof original) => { o.tools.pop(); },
|
||||
]) {
|
||||
const o = structuredClone(original); o.transcript.assistantMessages = [selectedMessage(o)]; mutate(o); expect(check(o)).toBe(false);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('quoted, hypothetical, conditional and withdrawn selections do not select the seed', () => {
|
||||
for (const original of originals) {
|
||||
const text = selectedMessage(original).text.trim();
|
||||
for (const invalid of [
|
||||
'> ' + text, ' ' + text, '"' + text + '"', 'Example:\n' + text,
|
||||
'The following is a source excerpt.\n' + text, 'An unproven hypothesis.\n' + text,
|
||||
text.replace("I'll", 'I might'), text.replace("I'll", "I won't"),
|
||||
text.replace('Marketing landing page', 'Other page'),
|
||||
text.replace('draft', 'branch diff'), text.replace(/,$/, '?'),
|
||||
text.replace(', ', ', if approved, '),
|
||||
text + ' I retract that selection.', text + ' This selection is withdrawn.',
|
||||
text + ' Treat that declaration as a hypothetical example.',
|
||||
].filter(value => value !== text)) {
|
||||
const o = structuredClone(original); o.transcript.assistantMessages = [selectedMessage(o)]; selectedMessage(o).text = invalid;
|
||||
expect(check(o), invalid).toBe(false);
|
||||
}
|
||||
}
|
||||
});
|
||||
test('a complete owned observation cannot use a withdrawn selection or a replacement target', () => {
|
||||
for (const original of originals) {
|
||||
for (const correction of ['The selection has been withdrawn.', 'The selected target is now the branch diff.', 'I have withdrawn this selection.', 'Correction: The selected target is now the branch diff.']) {
|
||||
for (const separator of [' ', '\n\n']) {
|
||||
const o = structuredClone(original);
|
||||
selectedMessage(o).text = selectedMessage(o).text.trim() + separator + correction;
|
||||
expect(check(o)).toBe(false);
|
||||
}
|
||||
const o = structuredClone(original);
|
||||
o.transcript.assistantMessages.push({ sessionId: o.opts.sessionId, timestamp: new Date(Date.parse(selectedMessage(o).timestamp) + 1000).toISOString(), text: correction });
|
||||
expect(check(o)).toBe(false);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('old, unrelated, foreign and quoted assessments do not withdraw the current target', () => {
|
||||
for (const original of originals) {
|
||||
for (const text of [
|
||||
'Old note: "The selection has been withdrawn."',
|
||||
'> The selection has been withdrawn.',
|
||||
'```text\nThe selected target is now the branch diff.\n```',
|
||||
'Source excerpt:\nThe selection has been withdrawn.',
|
||||
'The following is a hypothetical example.\nThe selected target is now the branch diff.',
|
||||
'An unrelated payment selection has been withdrawn.',
|
||||
'The selected target is now the "Marketing landing page" draft.',
|
||||
'If approved, the selection has been withdrawn.',
|
||||
'The selected target is now the branch diff?',
|
||||
'The selected target is now the branch diff? This is a question.',
|
||||
'I have withdrawn this selection?',
|
||||
]) {
|
||||
const o = structuredClone(original);
|
||||
o.transcript.assistantMessages.push({ sessionId: o.opts.sessionId, timestamp: new Date(Date.parse(selectedMessage(o).timestamp) + 1000).toISOString(), text });
|
||||
expect(check(o), text).toBe(true);
|
||||
}
|
||||
for (const foreign of [false, true]) {
|
||||
const o = structuredClone(original);
|
||||
o.transcript.assistantMessages.push({ sessionId: foreign ? 'foreign' : o.opts.sessionId, timestamp: new Date(Date.parse(selectedMessage(o).timestamp) + (foreign ? 1000 : -1000)).toISOString(), text: 'The selection has been withdrawn.' });
|
||||
expect(check(o)).toBe(true);
|
||||
}
|
||||
const o = structuredClone(original), selected = structuredClone(selectedMessage(o));
|
||||
o.transcript.assistantMessages.push({ sessionId: o.opts.sessionId, timestamp: new Date(Date.parse(selected.timestamp) + 1000).toISOString(), text: 'The selection has been withdrawn.' });
|
||||
o.transcript.assistantMessages.push({ ...selected, timestamp: new Date(Date.parse(selected.timestamp) + 2000).toISOString() });
|
||||
expect(check(o)).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('eng-option-b-scope-al', () => {
|
||||
const fixture = fixture_eng_option_b_scope_al;
|
||||
const actualInput = (attempt = 1) => structuredClone(fixture.attempts[attempt]!.projection);
|
||||
const input = () => {
|
||||
const p = actualInput();
|
||||
// Mutate the named declaration alone; the earlier spoken introduction is
|
||||
// independently valid and remains present in the exact replays below.
|
||||
p.transcript.assistantMessages = p.transcript.assistantMessages.filter(m => m.text !== "I'll run the eng review skill on this draft plan.");
|
||||
return p;
|
||||
};
|
||||
type Input = ReturnType<typeof input>;
|
||||
const verdict = (p = input()) => nativeSeededPlanSelection(p.transcript as PlanCountTranscript, p.tools as NativePublicToolEvent[], p.opts);
|
||||
const declaration = (p: Input) => p.transcript.assistantMessages.find(m => m.text.startsWith("I've selected option B,"))!;
|
||||
|
||||
test('both named retry and fresh unique-draft first introduction bind; original outcomes stay intact', () => {
|
||||
expect(fixture.attempts.map(a => a.rawScopeGateAutoSelectObserved)).toEqual([false, false]);
|
||||
expect(verdict(actualInput(0))).toBe(true);
|
||||
expect(verdict(actualInput(1))).toBe(true);
|
||||
});
|
||||
|
||||
test('an unnamed option-B notice supplies no selection without a draft introduction', () => {
|
||||
const p = input(); p.transcript.assistantMessages = p.transcript.assistantMessages.filter(m => m !== declaration(p));
|
||||
expect(verdict(p)).toBe(false);
|
||||
for (const replacement of ['option A,', 'option C,', 'option B if approved,', 'option B, possibly']) {
|
||||
const p = input(); declaration(p).text = declaration(p).text.replace('option B,', replacement); expect(verdict(p)).toBe(false);
|
||||
}
|
||||
for (const target of ['Unrelated draft', 'branch diff']) {
|
||||
const p = input(); declaration(p).text = declaration(p).text.replace('Parallelize unit tests', target); expect(verdict(p)).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test('equivalent current wording and consistently renamed title preserve selection', () => {
|
||||
for (const change of [
|
||||
(s: string) => s.replace("I've", 'I have'),
|
||||
(s: string) => s.replace('Next I', 'Next, I'),
|
||||
(s: string) => s.replace('reviewing the pasted', 'to review the pasted'),
|
||||
(s: string) => s.replace(/\. Next.*$/, '.'),
|
||||
(s: string) => s.replace('Design Doc Check, brain context, and context recovery, along with the Aside probe', 'audit for DESIGN.md'),
|
||||
]) { const p = input(); declaration(p).text = change(declaration(p).text); expect(verdict(p)).toBe(true); }
|
||||
const p = input(); p.opts.seed = p.opts.seed.replace('Parallelize unit tests', 'Build cache invalidation');
|
||||
declaration(p).text = declaration(p).text.replace('Parallelize unit tests', 'Build cache invalidation'); expect(verdict(p)).toBe(true);
|
||||
});
|
||||
|
||||
test('quoted, source, hypothetical, historical and conditional first lines cannot select', () => {
|
||||
for (const prefix of ['> ', ' ', '\t', '```\n', 'Source excerpt:\n', 'Historical example only.\n', 'The following is a hypothetical example. ', 'If approved, ', '"']) {
|
||||
const p = input(); declaration(p).text = prefix + declaration(p).text; expect(verdict(p)).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test('the same successful post-command Skill completion and current native session are required', () => {
|
||||
for (const change of [
|
||||
(p: Input) => { p.opts.sessionId = 'foreign'; },
|
||||
(p: Input) => { p.transcript.status = 'unavailable'; },
|
||||
(p: Input) => { p.tools = []; },
|
||||
(p: Input) => { p.tools[0]!.input!.skill = 'plan-design-review'; },
|
||||
(p: Input) => { p.tools[1]!.isError = true; },
|
||||
(p: Input) => { p.tools[1]!.sessionId = 'foreign'; },
|
||||
(p: Input) => { p.tools[1]!.toolUseId = 'unrelated'; },
|
||||
(p: Input) => { p.opts.commandStartedAt = Date.parse(p.tools[0]!.timestamp) + 1; },
|
||||
(p: Input) => { declaration(p).timestamp = new Date(p.opts.commandStartedAt - 1).toISOString(); },
|
||||
(p: Input) => { declaration(p).sessionId = 'foreign'; },
|
||||
(p: Input) => { p.tools.push(structuredClone(p.tools[1]!)); },
|
||||
]) { const p = input(); change(p); expect(verdict(p)).toBe(false); }
|
||||
});
|
||||
|
||||
test('conditional, questioning and replacement continuations cannot borrow a completed selection', () => {
|
||||
for (const change of [
|
||||
(s: string) => s.replace('draft plan.', 'draft plan if approved.'),
|
||||
(s: string) => s.replace('Next I', 'If approved, I'),
|
||||
(s: string) => s.replace('Aside probe.', 'Aside probe?'),
|
||||
(s: string) => s.replace('Design Doc Check', 'branch diff review instead'),
|
||||
(s: string) => s.replace('Design Doc Check', 'unrelated work'),
|
||||
]) { const p = input(); declaration(p).text = change(declaration(p).text); expect(verdict(p)).toBe(false); }
|
||||
});
|
||||
|
||||
test('same-message or later owned withdrawals and target changes defeat the declaration', () => {
|
||||
for (const correction of ['Correction: this selection is withdrawn.', 'This declaration has been retracted.', 'The selected target is now the branch diff.']) {
|
||||
for (const placement of ['same-line', 'same-message', 'later']) {
|
||||
const p = input(), m = declaration(p);
|
||||
if (placement === 'later') p.transcript.assistantMessages.push({ ...m, timestamp: new Date(Date.parse(m.timestamp) + 1000).toISOString(), text: correction });
|
||||
else m.text += (placement === 'same-line' ? ' ' : '\n') + correction;
|
||||
expect(verdict(p)).toBe(false);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('foreign, historical and literal corrections do not retract a current named selection', () => {
|
||||
for (const text of ['> This selection is withdrawn.', 'Source excerpt:\nThis selection is withdrawn.', 'A prior assistant said "This selection is withdrawn."', 'The verification suite is withdrawn.', 'Is this selection withdrawn?']) {
|
||||
const p = input(), m = declaration(p); p.transcript.assistantMessages.push({ ...m, timestamp: new Date(Date.parse(m.timestamp) + 1000).toISOString(), text }); expect(verdict(p)).toBe(true);
|
||||
}
|
||||
const p = input(), m = declaration(p); p.transcript.assistantMessages.push({ ...m, sessionId: 'foreign', text: 'This selection is withdrawn.' }); expect(verdict(p)).toBe(true);
|
||||
});
|
||||
|
||||
test('a later explicit reselection follows the existing currentness rule', () => {
|
||||
const p = input(), m = declaration(p); p.transcript.assistantMessages.push({ ...m, timestamp: new Date(Date.parse(m.timestamp) + 1000).toISOString(), text: 'This selection is withdrawn.' }); expect(verdict(p)).toBe(false);
|
||||
p.transcript.assistantMessages.push({ ...m, timestamp: new Date(Date.parse(m.timestamp) + 2000).toISOString() }); expect(verdict(p)).toBe(true);
|
||||
});
|
||||
for (const owner of [
|
||||
'plan-ceo-review-plan-mode', 'plan-eng-review-plan-mode',
|
||||
'plan-design-review-plan-mode', 'plan-devex-review-plan-mode', 'plan-mode-no-op',
|
||||
]) test(`scope dependency registration is dense for ${owner}`, () => {
|
||||
const paths = E2E_TOUCHFILES[owner]!;
|
||||
for (let index = 0; index < paths.length; index++) {
|
||||
expect(Object.hasOwn(paths, index)).toBe(true);
|
||||
expect(typeof paths[index]).toBe('string');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('plan-scope-recovery-av', () => {
|
||||
const fs = fs_design_scope_entry_aq;
|
||||
const path = path_design_scope_entry_aq;
|
||||
const observedFailures = observedFailures_plan_scope_recovery_av;
|
||||
const skills = ['plan-eng-review', 'plan-design-review'] as const;
|
||||
const read = (skill: string) => fs.readFileSync(path.join(import.meta.dir, '..', skill, 'SKILL.md.tmpl'), 'utf8');
|
||||
const recovery = (text: string) => text.split('\n').find(line => line.startsWith('> Before ') && line.includes('publicly identified'))!;
|
||||
|
||||
test('the review handoff repairs a missing public declaration without claiming timely compliance', () => {
|
||||
for (const skill of skills) {
|
||||
const text = read(skill), check = recovery(text);
|
||||
expect(check).toBeDefined();
|
||||
expect(check).toContain('require resolved scope');
|
||||
expect(check).toContain('For plan-mode auto-selection, verify you publicly identified the selected plan for this invocation before review work');
|
||||
expect(check).toContain('If missing, send "Scope gate: plan mode — auto-selected B (reviewing <target>)." now');
|
||||
expect(check).toContain('do not claim an earlier announcement');
|
||||
const start = skill === 'plan-eng-review' ? '### Step 0: Scope Challenge' : '## PRE-REVIEW SYSTEM AUDIT';
|
||||
expect(text.indexOf(check)).toBeGreaterThan(text.indexOf('{{PREAMBLE}}'));
|
||||
expect(text.indexOf(check)).toBeGreaterThan(text.indexOf(start));
|
||||
const reviewStart = text.indexOf(skill === 'plan-eng-review' ? '{{SECTION:review-sections}}' : 'Before reviewing the plan, gather context');
|
||||
expect(reviewStart).toBeGreaterThanOrEqual(0);
|
||||
expect(text.indexOf(check)).toBeLessThan(reviewStart);
|
||||
if (skill === 'plan-eng-review') {
|
||||
const section = fs.readFileSync(path.join(import.meta.dir, '..', skill, 'sections/review-sections.md.tmpl'), 'utf8');
|
||||
expect(section).toContain('### A. Assess the target');
|
||||
expect(section).toContain('Complete these checks before the complexity decision in B');
|
||||
expect(section.indexOf('### A. Assess the target')).toBeLessThan(section.indexOf('### B. Resolve complexity selectors'));
|
||||
expect(section.indexOf('### B. Resolve complexity selectors')).toBeLessThan(section.indexOf('### C. Resolve findings'));
|
||||
expect(section).toContain('Run C whether B was completed or skipped');
|
||||
expect(text.slice(text.indexOf(check), reviewStart)).toContain('Scope Challenge is mandatory before Section 1');
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('unseeded, explicit-target and early announcement rules remain authoritative on every host', () => {
|
||||
for (const skill of skills) {
|
||||
const template = read(skill);
|
||||
const gate = template.slice(template.indexOf('## Scope gate'), template.indexOf('{{PREAMBLE}}'));
|
||||
const entry = skill === 'plan-eng-review'
|
||||
? 'Before tools or preamble, resolve from provided messages, listed tools and explicit host metadata only'
|
||||
: 'After this skill loads, resolve this gate before any tool';
|
||||
const announce = skill === 'plan-eng-review'
|
||||
? 'Announce an auto-selected plan in one line so the user can interrupt'
|
||||
: 'Announce plan-mode auto-selection before review tools';
|
||||
expect(gate).toContain(entry);
|
||||
expect(gate).toContain(announce);
|
||||
expect(gate).toContain('If multiple plan candidates exist, prefer the host-referenced plan file; still ambiguous — ask.');
|
||||
expect(gate).toContain('If the user explicitly named a DIFFERENT target');
|
||||
expect(gate).toContain('If plan mode is indicated but no plan exists yet, ask as normal');
|
||||
expect(gate).toContain('When no exception above applied:');
|
||||
expect(gate).toContain(skill === 'plan-eng-review'
|
||||
? 'First tool call = AskUserQuestion (tool_use). Send this exact menu and wait'
|
||||
: 'First tool call = AskUserQuestion (tool_use). Confirm what to review.');
|
||||
expect(gate).toContain('STOP and wait for the answer');
|
||||
for (const host of ALL_HOST_CONFIGS) {
|
||||
const ctx: TemplateContext = {skillName: skill, tmplPath: `${skill}/SKILL.md.tmpl`, host: host.name,
|
||||
paths: HOST_PATHS[host.name]!, preambleTier: 3, interactive: true};
|
||||
const expanded = template.replace('{{PREAMBLE}}', generatePreamble(ctx));
|
||||
expect(expanded.indexOf(announce)).toBeLessThan(expanded.indexOf('```bash'));
|
||||
expect(expanded.indexOf('STOP and wait for the answer')).toBeLessThan(expanded.indexOf('```bash'));
|
||||
expect(expanded.indexOf(recovery(template))).toBeGreaterThan(expanded.indexOf('```bash'));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('recorded failures stay intact while fresh unique-draft introductions now bind', () => {
|
||||
expect(observedFailures).toHaveLength(4);
|
||||
for (const [index, row] of observedFailures.entries()) {
|
||||
expect(row.observed.scopeGateAutoSelectObserved).toBe(false);
|
||||
expect(nativeSeededPlanSelection(row.transcript as any, row.tools as any, row.opts)).toBe(index !== 0);
|
||||
const title = /^#\s+(?:Plan:\s*)?(.+)$/m.exec(row.opts.seed)![1]!;
|
||||
const loaded = row.tools.find(tool => tool.kind === 'result')!;
|
||||
const timestamp = new Date(Date.parse(loaded.timestamp) + 1).toISOString();
|
||||
const message = {sessionId: row.opts.sessionId, timestamp, text: `I'll review "${title}" plan.`};
|
||||
const amended = {...row.transcript, assistantMessages: [message]};
|
||||
// Explicit synthetic control only: no changed message is paid evidence.
|
||||
expect(nativeSeededPlanSelection(amended as any, row.tools as any, row.opts)).toBe(true);
|
||||
for (const text of [`> ${message.text}`, `Example:\n${message.text}`, `I'll review "Another plan" plan.`]) {
|
||||
expect(nativeSeededPlanSelection({...amended, assistantMessages: [{...message, text}]} as any, row.tools as any, row.opts)).toBe(false);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user