test(plan-ceo): split-overflow candidate identity is the one E1-E5 token

The skill fixes no header or question wording, and shape-matching kept
missing new forms (header 'D1.1 Slack' with 'SCOPE-E1:', 'D1.1 E1 Slack',
'E1 Slack DM bot ...:', a misspelled 'Mattermst'). Identity is now exactly
one E1-E5 token across the header and the question's first line, with no
other candidate's platform named. Options rules, judge, call floor and
include/defer/cut validation are unchanged. Replays all 31 measured runs;
the quoted-row controls now quote a second ledger id.
This commit is contained in:
garrytan committed 2026-10-05 23:16:17 +00:00
1 parent 0ff760f5f8
commit 1a579bbf29
3 files changed
+5200 -168

No files matched your search

+23 -13
View File
@@ -84,7 +84,7 @@ test.each(['foreign_row', 'quoted_row', 'second_platform', 'held'])('row-ID coll
const state = rowIdCapture(), call = state.transcript.calls.at(-1)!, question = call.questions[0]!;
const selected = call.answers![question.question]!;
if (kind === 'foreign_row') question.question = question.question.replace('R-E5:', 'R-E4:');
if (kind === 'quoted_row') question.question = 'Example: ' + question.question;
if (kind === 'quoted_row') question.question = 'Example from R-E4: ' + question.question;
if (kind === 'second_platform') question.question = question.question.replace('?', ' or the Slack bot?');
call.answers = { [question.question]: kind === 'held' ? question.options[3]!.label : selected };
state.fingerprints = fromCalls(state.transcript.calls).fingerprints;
@@ -127,7 +127,7 @@ test.each(['pending', 'failed', 'missing_answer', 'custom_answer', 'hold', 'quot
if (kind === 'missing_answer') call.answers = {};
if (kind === 'custom_answer') call.answers = { [question.question]: 'Custom approval' };
if (kind === 'hold') call.answers = { [question.question]: question.options[3]!.label };
if (kind === 'quoted') question.question = 'Example: ' + question.question;
if (kind === 'quoted') question.question = 'Example from E4: ' + question.question;
if (kind === 'wrong_platform') question.header = 'E5 Slack';
if (kind === 'bundled') question.question = question.question.replace('?', ' and E4: Telegram?');
if (kind === 'missing_cut') question.options.splice(2, 1);
@@ -265,25 +265,35 @@ await import(${JSON.stringify(path.join(ROOT, 'test/skill-e2e-plan-ceo-split-ove
}, 20_000,
);
test.each(variants.runs.map(run => [run.source, run]))('captured candidate wording completes collection: %s', (_source, run) => {
test.each(variants.runs.map(run => [run.source, run]))('captured candidate wording completes collection at the fifth call: %s', (_source, run) => {
const calls = structuredClone(run.calls) as unknown as NativePlanQuestionCall[];
expect(run.originalOutcome).not.toBe('collection_complete');
expect(calls.map(call => ceoSplitCandidate(call.questions[0] as NativeQuestion))).toEqual(['E1', 'E2', 'E3', 'E4', 'E5']);
const state = fromCalls(calls);
expect(state.fingerprints.map(isCeoSplitCandidateCall)).toEqual([true, true, true, true, true]);
expect(isCeoSplitCollectionComplete(state.transcript, state.fingerprints)).toBe(true);
expect(isCeoSplitCollectionComplete(fromCalls(calls.slice(0, 4)).transcript, fromCalls(calls.slice(0, 4)).fingerprints)).toBe(false);
const four = fromCalls(calls.slice(0, 4));
expect(isCeoSplitCollectionComplete(four.transcript, four.fingerprints)).toBe(false);
});
test('wording variants stay bound to one candidate and its scope buckets', () => {
const base = structuredClone(variants.runs.find(run => run.source.startsWith('ci-'))!.calls[0]!.questions[0]!) as unknown as NativeQuestion;
const lead = (question: NativeQuestion, text: string) => ({ ...question, question: question.question.replace(/^[^\n]*/, text) });
for (const text of ['D1.1 — E1: Ship the Slack DM bot?', 'D1.1 — E1) Ship the Slack DM bot?',
'D1.1 — E1 Slack: ship the Slack DM bot?', 'D1.1 — E1-SLACK: Ship the Slack DM bot?']) expect(ceoSplitCandidate(lead(base, text))).toBe('E1');
for (const text of ['D1.1 — E1-DISCORD: Ship the Slack DM bot?', 'D1.1 — E1 Slack: ship Slack before Discord?',
'D1.1 — E1-SLACK: ship Slack with E2?', 'D1.1 — E2: Ship the Slack DM bot?']) expect(ceoSplitCandidate(lead(base, text))).toBeNull();
test('the replay covers all 31 measured runs, including every slow-path variant', () => {
expect(variants.runs).toHaveLength(31);
expect(variants.runs.filter(run => run.originalOutcome !== 'collection_complete')).toHaveLength(18);
});
test('identity is the one ledger token; another platform or a second token rejects', () => {
const base = structuredClone(variants.runs[0]!.calls[0]!.questions[0]!) as unknown as NativeQuestion;
const ask = (header: string, lead: string) => ({ ...base, header, question: lead });
for (const [header, lead] of [['E1 Slack', 'D1.1 — E1: Ship the Slack DM bot?'], ['E1 Slack', 'D1.1 — E1) Ship the Slack DM bot?'],
['E1 Slack', 'D1.1 — E1 Slack DM bot for incident alerts: include, defer, or cut?'], ['E1 Slack', 'D1.1 — E1-SLACK: Ship it?'],
['D1.1 Slack', 'D1.1 — SCOPE-E1: Include the Slack DM bot this quarter?'], ['D1.1 E1 Slack', 'D1.1 — E1-SLACK: Include it?'],
['E5 Mattermst', 'D1.5 — E5-MATTERMOST: Include the plugin?'], ['E5 Mattermst', 'D1.5 — Include the plugin this quarter?']] as const) {
expect(ceoSplitCandidate(ask(header, lead))).toBe(header.includes('Matter') ? 'E5' : 'E1');
}
for (const [header, lead] of [['D1.1 Slack', 'D1.1 — Ship the Slack DM bot this quarter?'], ['E1 Slack', 'D1.1 — E1: ship Slack before Discord?'],
['E1 Discord', 'D1.1 — E1: Ship the bot?'], ['E1 Slack', 'D1.1 — E1-SLACK: ship Slack with E2?'], ['E1 Slack', 'D1.1 — E2: Ship the bot?'],
['E6 Zulip', 'D1.6 — E6: Ship the Zulip bot?']] as const) expect(ceoSplitCandidate(ask(header, lead))).toBeNull();
const relabeled = (labels: string[]) => ({ ...base, options: base.options.map((option, i) => ({ ...option, label: labels[i]! })) });
expect(ceoSplitCandidate(relabeled(['Include in this quarter (recommended)', 'Defer with flip trigger', 'Cut entirely', 'Hold, stop and discuss']))).toBe('E1');
expect(ceoSplitCandidate(relabeled(["Include in this quarter's scope (recommended)", 'Defer with flip trigger', 'Cut entirely', 'Hold — discuss']))).toBe('E1');
expect(ceoSplitCandidate(relabeled(['Ship now', 'Defer', 'Cut', 'Hold']))).toBeNull();
expect(ceoSplitCandidate(relabeled(['Include', 'Included elsewhere', 'Cut', 'Hold']))).toBeNull();
});
File diff suppressed because it is too large. Load diff
+15 -15
View File
@@ -24,24 +24,24 @@ export function ceoSplitOptionAction(label: string): 'include' | 'defer' | 'cut'
/** Candidate-shaped menus for live progress only. Final coverage, subject and
* independence are established by evaluatePlanReviewDecisions over every call.
* Identity comes from the native header. The question opens with that
* candidate's ledger reference (E1 or a row ID ending in it, optionally joined to
* its platform as in "E1 Slack:" or "E1-SLACK:"), names only that candidate, and
* offers exactly one include, defer and cut disposition. */
* The skill fixes no header or question wording, so identity is the one ledger
* token E1-E5 across the header and the question's first line (SCOPE-E1,
* E1-SLACK and "E1 Slack" all count). Neither may name another candidate's
* platform; a misspelled own platform is harmless because the E-id carries the
* identity. The menu offers exactly one include, defer and cut disposition. */
export function ceoSplitCandidate(question: NativeQuestion): string | null {
const header = /^E([1-5])\s+(.+)$/.exec(question.header.trim());
if (!header || question.multiSelect || question.options.length < 3 || question.options.length > 4) return null;
const index = Number(header[1]) - 1;
const lead = question.question.split(/\r?\n/, 1)[0]!
.replace(/^D[1-9]\d*(?:\.[1-9]\d*)?\s*[—–:-]\s*/, '');
const names = (platform: string) => new RegExp(`\\b${platform}\\b`, 'i').test(lead);
if (!new RegExp(`^${platforms[index]}$`, 'i').test(header[2]!) ||
!new RegExp(`^\\S*\\bE${header[1]}(?:[-\\s]+${platforms[index]})?[):]\\s+.+\\?$`, 'i').test(lead) || !names(platforms[index]!) ||
platforms.some((platform, i) => i !== index && names(platform)) ||
[...lead.matchAll(/\bE([1-9]\d*)\b/g)].some(match => match[1] !== header[1])) return null;
if (question.multiSelect || question.options.length < 3 || question.options.length > 4) return null;
const header = question.header.trim();
const lead = question.question.split(/\r?\n/, 1)[0]!;
const ids = new Set([...`${header}\n${lead}`.matchAll(/\bE([1-9]\d*)\b/g)].map(match => Number(match[1])));
if (ids.size !== 1) return null;
const [id] = ids as Set<number>;
if (id! < 1 || id! > platforms.length) return null;
const names = (text: string, platform: string) => new RegExp(`\\b${platform}\\b`, 'i').test(text);
if (platforms.some((platform, i) => i !== id! - 1 && (names(header, platform) || names(lead, platform)))) return null;
const actions = question.options.map(option => ceoSplitOptionAction(option.label));
return ['include', 'defer', 'cut'].every(action => actions.filter(found => found === action).length === 1)
? `E${header[1]}` : null;
? `E${id}` : null;
}
export function isCeoSplitCandidateCall(fp: AskUserQuestionFingerprint): boolean {