gstack/test/eng-owned-explanation.test.ts

import { expect, test } from 'bun:test';
import { evaluateEngSeedCoverage } from './helpers/eng-seeded-coverage';
import { E2E_TOUCHFILES } from './helpers/touchfiles';
import type { NativePlanQuestion, NativePlanQuestionCall } from './helpers/plan-count-transcript';
import fixture from './fixtures/eng-owned-explanation.json';

const seeds = ['complexity', 'swallowed-errors', 'sequential-idp'] as const;
const evaluate = (calls: NativePlanQuestionCall[] = [], plan = '') =>
  evaluateEngSeedCoverage({ status: 'ready', calls, assistantMessages: [] }, plan, 0, 10);
function decision(i: number, change: (q: NativePlanQuestion) => void = () => {}) {
  const q = structuredClone(fixture.questions[i]!) as NativePlanQuestion;
  change(q);
  return { sessionId: 'one-session', toolUseId: `decision-${i}`, questions: [q],
    answered: true, failed: false, unansweredQuestionIndices: [], answeredAt: new Date(5).toISOString(),
    answers: { [q.question]: q.options[0]!.label } } satisfies NativePlanQuestionCall;
}

test.each([0, 1, 2])('a current explanation and one concrete offered repair own seed %i', i => {
  expect(evaluate([decision(i)]).decisions[seeds[i]!]).toBe(`one-session:decision-${i}`);
});

test.each([0, 1, 2])('equivalent title, ordinal, formatting and offered answer preserve seed %i', i => {
  const titles = ['Which auth components should we retain?', 'Which error boundary should validateAndDispatch() use?', 'How should we schedule the IDP calls?'];
  const c = decision(i, q => { q.question = q.question.replace(/^D\d+ — .*$/m, `D17 — ${titles[i]}`); });
  for (const option of c.questions[0]!.options) {
    c.answers[c.questions[0]!.question] = option.label;
    expect(evaluate([c]).decisions[seeds[i]!]).toBeDefined();
  }
  expect(evaluate([decision(i, q => { q.question = q.question.replaceAll('validateAndDispatch()', '`validateAndDispatch()`'); })]).decisions[seeds[i]!]).toBeDefined();
  expect(evaluate([decision(i, q => { q.question = q.question.replace('ELI10: ', '[P1] Current finding\nELI10: '); })]).decisions[seeds[i]!]).toBeDefined();
});

const questionControls: Array<[string, (s: string) => string]> = [
  ['missing metadata', s => s.replace(/^Project\/branch\/task:.*\n/m, '')],
  ['missing explanation', s => s.replace(/^ELI10:.*\n/m, '')],
  ['unrelated explanation', s => s.replace(/^ELI10:.*$/m, 'ELI10: This asks about naming conventions.')],
  ['duplicate explanation', s => s + '\nELI10: Another finding.'],
  ['duplicate metadata', s => s + '\nProject/branch/task: a different task.'],
  ['two severity lines', s => s.replace('ELI10: ', '[P1] First finding\n[P2] Second finding\nELI10: ')],
  ['source severity line', s => s.replace('ELI10: ', '[P1] Source: borrowed priority\nELI10: ')],
  ['quoted explanation', s => s.replace(/^ELI10: (.*)$/m, 'ELI10: "$1"')],
  ['literal explanation', s => s.replace(/^ELI10: (.*)$/m, 'ELI10: `$1`')],
  ['source explanation', s => s.replace('ELI10: ', 'ELI10: Source: ')],
  ['copied source explanation', s => s.replace('ELI10: ', 'ELI10: Copied source excerpt. ')],
  ['historical metadata', s => s.replace('Project/branch/task: ', 'Project/branch/task: Historical assessment. ')],
  ['quoted title', s => s.replace(/^(D\d+ — )(.*)$/m, '$1"$2"')],
  ['literal title', s => s.replace(/^(D\d+ — )(.*)$/m, '$1`$2`')],
  ['historical title', s => s.replace(/^(D\d+ — )/m, '$1Historical: ')],
  ['conditional explanation', s => s.replace('ELI10: ', 'ELI10: If approved, ')],
  ['withdrawn finding', s => s + '\nCorrection: This finding is withdrawn.'],
  ['quoted inactive status', s => s + '\nThis finding is "withdrawn".'],
  ['whole quotation', s => s.split('\n').map(l => '> ' + l).join('\n')],
  ['whole fence', s => '```\n' + s + '\n```'],
  ['defect only in stakes', s => s.replace(/^ELI10: (.*)$/m, 'ELI10: We are considering names.\nStakes: $1')],
];
test.each(questionControls)('%s cannot supply an explained decision', (_, change) => {
  for (let i = 0; i < seeds.length; i++) {
    const c = decision(i, q => { const before = q.question; q.question = change(before); expect(q.question).not.toBe(before); });
    expect(evaluate([c]).decisions[seeds[i]!]).toBeUndefined();
  }
});

test.each([0, 1, 2])('repair ownership and native completion remain required for seed %i', i => {
  for (const wrap of [(s: string) => `Source: ${s}`, (s: string) => `"${s}"`, (s: string) => `${s}\nThis option is withdrawn.`]) {
    const c = decision(i, q => { q.options = q.options.map(o => ({ label: wrap(o.label), description: wrap(o.description ?? '') })); });
    expect(evaluate([c]).decisions[seeds[i]!]).toBeUndefined();
  }
  const c = decision(i); c.answered = false;
  expect(evaluate([c]).decisions[seeds[i]!]).toBeUndefined();
  const splitRepairs = [
    [{ label: 'Reduce scope', description: 'Remove extra components.' }, { label: 'Keep shape', description: 'Keep AuthBroker, SessionMint and injected AuthCache over one backing store.' }],
    [{ label: 'Flatten into named helpers', description: 'Use a typed boundary.' }, { label: 'Keep nesting', description: 'One catch maps errors to 401 and rethrows unknowns.' }],
    [{ label: 'Promise.all', description: 'Discuss a name.' }, { label: 'Keep request schedule', description: 'Five calls with cancellation of siblings.' }],
  ];
  expect(evaluate([decision(i, q => { q.options = splitRepairs[i]!; })]).decisions[seeds[i]!]).toBeUndefined();
});

test('a combined native decision cannot supply three distinct seed decisions', () => {
  const c = decision(0);
  c.questions = [0, 1, 2].map(i => decision(i).questions[0]!);
  c.answers = Object.fromEntries(c.questions.map(q => [q.question, q.options[0]!.label]));
  expect(evaluate([c]).decisions).toEqual({});
});

// Minimal mandatory declaration and T1 task excerpt from the acknowledged final
// report. The complete retained report is used only for a private local replay.
const report = `# Current reviewed plan
### REGRESSION RULE (mandatory, no decision required)

**CRITICAL — T1:** write characterization (golden) tests for \`legacyAuthFlow()\`
BEFORE any rewrite. Corpus: valid token, expired token, revoked token, wrong
audience, wrong issuer, unknown tenant, suspended tenant, policy version
mismatch, malformed token, cache hit vs miss. Run the same corpus against the
\`AuthBroker\` path. Both must produce identical results before the flag opens to
any tenant. This test is authorized by the regression rule itself.

## Implementation Tasks
- [ ] **T1 (P1, human: ~1 day / CC: ~30 min)** — legacyAuthFlow — Write characterization (golden) tests for \`legacyAuthFlow()\` before touching it
  - Surfaced by: Test review — REGRESSION RULE, PLAN.md:27-28
  - Files: test/auth/legacy-auth-flow.characterization.test.ts
  - Verify: suite passes against legacy; later passes unchanged against AuthBroker
`;

test('mandatory declaration and unique task bind the old baseline to unchanged new-path parity', () => {
  expect(evaluate([], report).regression).toBe('plan');
  for (const change of [
    (s: string) => s.replaceAll('T1', 'T17'),
    (s: string) => s.replace('Corpus: ', 'T1 is mandatory. Corpus: '),
    (s: string) => s.replaceAll('test/auth/legacy-auth-flow.characterization.test.ts', 'spec/compatibility.test.js'),
    (s: string) => s.replaceAll('AuthBroker', 'ReplacementBroker'),
    (s: string) => s.replace('REGRESSION RULE (mandatory, no decision required)', 'Required characterization (mandatory)').replace('Implementation Tasks', 'Execution checklist'),
    (s: string) => s.replace('Run the same corpus against the', 'Replay the same corpus through the').replace('produce identical results', 'return matching outputs').replace('later passes unchanged against', 'then is green unchanged on'),
    (s: string) => s + '\n## History\nT1 is withdrawn.\n',
    (s: string) => s + '\n## Current assessment\n"T1 is withdrawn."\n',
    (s: string) => s + '\n## Payment regression suite\nThe regression suite is withdrawn.\n',
  ]) expect(evaluate([], change(report)).regression).toBe('plan');
});

const regressionControls: Array<[string, (s: string) => string]> = [
  ['missing declaration', s => s.replace(/### [\s\S]*?(?=## Implementation Tasks)/, '')],
  ['optional declaration', s => s.replace('mandatory', 'optional')],
  ['conditional declaration', s => s.replace('mandatory', 'mandatory if approved')],
  ['missing legacy subject', s => s.replaceAll('legacyAuthFlow', 'otherAuthFlow')],
  ['late declaration baseline', s => s.replace('BEFORE any rewrite', 'AFTER any rewrite')],
  ['late task baseline', s => s.replace('before touching it', 'after touching it')],
  ['missing task', s => s.replace(/- \[ \][\s\S]*/, '')],
  ['wrong task ID', s => s.replace('**T1 (P1', '**T9 (P1')],
  ['distinct declaration task IDs', s => s.replace('Corpus: ', 'T9 owns this corpus. Corpus: ')],
  ['duplicate task ID', s => s + s.slice(s.indexOf('- [ ]'))],
  ['duplicate declaration', s => s + s.slice(s.indexOf('### '), s.indexOf('## Implementation Tasks'))],
  ['missing file', s => s.replace(/^  - Files:.*\n/m, '')],
  ['two files', s => s.replace('.test.ts', '.test.ts, test/other.test.ts')],
  ['conflicting declared file', s => s.replace('Corpus: ', 'File: test/other.test.ts. Corpus: ')],
  ['missing verification', s => s.replace(/^  - Verify:.*\n/m, '')],
  ['foreign baseline', s => s.replace('passes against legacy;', 'passes against otherAuthFlow;')],
  ['foreign rewrite', s => s.replace('unchanged against AuthBroker', 'unchanged against OtherBroker')],
  ['ambiguous rewrite', s => s.replace('Both must', 'Run the same corpus against OtherBroker path. Both must')],
  ['new suite', s => s.replace('passes unchanged against', 'passes after updating expectations against')],
  ['missing parity', s => s.replace('Both must produce identical results', 'Both produce different results')],
  ['different corpus', s => s.replace('Run the same corpus', 'Run a new corpus')],
  ['quoted declaration', s => s.replace(/(### [^\n]+\n)([\s\S]*?)(?=\n## Implementation Tasks)/, '$1"$2"')],
  ['fenced report', s => '```\n' + s + '\n```'],
  ['historical report', s => s.replace('Current reviewed plan', 'Historical reviewed plan')],
  ['source declaration', s => s.replace('**CRITICAL', 'Source:\n**CRITICAL')],
  ['conditional task', s => s.replace('## Implementation Tasks\n', '## Implementation Tasks\nOnce approved,\n')],
  ['source task', s => s.replace('## Implementation Tasks\n', '## Implementation Tasks\nSource:\n')],
  ['withdrawn task', s => s + '\n## Current assessment\nT1 is withdrawn.\n'],
  ['cancelled verification', s => s + '\n## Current assessment\nT1 verification is optional.\n'],
  ['quoted status', s => s + '\n## Current assessment\nT1 is "withdrawn".\n'],
  ['changed before baseline', s => s + '\n## Current assessment\nlegacyAuthFlow() is rewritten before T1.\n'],
  ['late baseline correction', s => s + '\n## Current assessment\nRun T1 only after rewriting legacyAuthFlow().\n'],
  ['changed expectations', s => s + '\n## Current assessment\nUpdate T1 expectations to match the new path.\n'],
  ['changed suite expectations', s => s + '\n## Current assessment\nThe legacy regression suite expectations will be updated to match the new path.\n'],
];
test.each(regressionControls)('%s cannot supply the linked baseline', (_, change) => {
  const changed = change(report);
  expect(changed).not.toBe(report);
  expect(evaluate([], changed).regression).toBeUndefined();
});

// The retry states the unchanged-code baseline in its file-bound task's Verify
// line. Retain only that task and its mandatory declaration, not the full report.
const retryReport = `# Current reviewed plan
### CRITICAL regression (mandatory, regression rule)
\`legacyAuthFlow()\` is existing behavior being rewritten with no test of its prior behavior (PLAN.md:27-28).
Add \`test/auth/legacyAuthFlow.characterization.test.ts\` pinning current outputs for: valid token, expired token,
revoked token, wrong audience, wrong issuer, suspended tenant, malformed token. It must pass before and after
this refactor, and the shadow compare asserts \`AuthBroker\` agrees with it.

## Implementation Tasks
- [ ] **T2 (P1, human: ~4h / CC: ~10 min)** — legacyAuthFlow — Characterization suite pinning prior behavior (CRITICAL regression)
  - Surfaced by: Test review, regression rule — PLAN.md:27-28
  - Files: test/auth/legacyAuthFlow.characterization.test.ts
  - Verify: suite passes on main before any refactor commit, and after
`;

test('a file-bound task can carry its own pre-commit baseline and retained-output parity', () => {
  for (const change of [
    (s: string) => s,
    (s: string) => s.replaceAll('T2', 'T19').replaceAll('AuthBroker', 'ReplacementBroker'),
    (s: string) => s.replaceAll('test/auth/legacyAuthFlow.characterization.test.ts', 'spec/compatibility.test.js'),
    (s: string) => s.replace('Implementation Tasks', 'Execution checklist'),
    (s: string) => s.replace('passes on main before any refactor commit', 'green against the untouched code before the rewrite commit').replace('asserts', 'verifies').replace('agrees with', 'matches'),
    (s: string) => s + '\n## Payment regression suite\nThe regression suite expectations will be updated.\n',
  ]) expect(evaluate([], change(retryReport)).regression).toBe('plan');
});

test.each([
  ['missing capture', (s: string) => s.replace('pinning current outputs', 'describing current outputs')],
  ['future outputs', (s: string) => s.replace('pinning current outputs', 'pinning new outputs')],
  ['missing required parity', (s: string) => s.replace('asserts `AuthBroker` agrees with it', 'describes AuthBroker')],
  ['foreign file', (s: string) => s.replace('  - Files: test/auth/legacyAuthFlow.characterization.test.ts', '  - Files: test/other.test.ts')],
  ['ambiguous file task', (s: string) => s + s.slice(s.indexOf('- [ ]')).replace('T2', 'T19')],
  ['ambiguous parity target', (s: string) => s.replace('agrees with it.', 'agrees with it. It also asserts OtherBroker matches it.')],
  ['unbound branch baseline', (s: string) => s.replace('on main', 'on the rewritten branch')],
  ['baseline after refactor commit', (s: string) => s.replace('main before any refactor commit', 'main after any refactor commit')],
  ['baseline before rollout only', (s: string) => s.replace('refactor commit', 'rollout commit')],
  ['missing after check', (s: string) => s.replace(', and after', '')],
  ['source task', (s: string) => s.replace('## Implementation Tasks\n', '## Implementation Tasks\nSource:\n')],
  ['conditional verification', (s: string) => s.replace('Verify: ', 'Verify: If approved, ')],
  ['withdrawn task', (s: string) => s + '\n## Current assessment\nT2 is withdrawn.\n'],
  ['rewritten before baseline', (s: string) => s + '\n## Current assessment\nlegacyAuthFlow() is rewritten before T2.\n'],
  ['changed expectations', (s: string) => s + '\n## Current assessment\nUpdate T2 expectations to match the new path.\n'],
] as Array<[string, (s: string) => string]>)('%s cannot provide a pre-commit baseline', (_, change) => {
  const changed = change(retryReport);
  expect(changed).not.toBe(retryReport);
  expect(evaluate([], changed).regression).toBeUndefined();
});

test('the explanation regression selects the existing Eng finding-count workflow', () => {
  for (const path of ['test/eng-owned-explanation.test.ts', 'test/fixtures/eng-owned-explanation.json']) {
    expect(Object.entries(E2E_TOUCHFILES).filter(([, paths]) => paths.includes(path)).map(([name]) => name))
      .toEqual(['plan-eng-finding-count']);
  }
});