gstack/test/eng-scheduled-regression.test.ts

import { expect, test } from 'bun:test';
import { E2E_TOUCHFILES } from './helpers/touchfiles';
import { evaluateEngSeedCoverage } from './helpers/eng-seeded-coverage';
// Minimal verbatim public requirements/tasks/verification from the two failed
// September 11 Eng runs. Replays diagnose the oracle; they do not credit those runs.
const reports: string[] = [
  "# Current reviewed plan\n\n### REGRESSION RULE (mandatory, no decision needed)\n\n`legacyAuthFlow()` is existing behavior being rewritten with no existing test\non the changed path. **CRITICAL:** add\n`test/auth/legacyAuthFlow.regression.test` before any rewrite. It records\nthe observable outcomes (status, reason code, cache side effects) of\n`legacyAuthFlow()` for a fixture matrix (valid, expired, wrong issuer, wrong\naudience, revoked, suspended tenant, malformed) and asserts `AuthBroker`\nproduces identical outcomes on the same fixtures.\n\n## Implementation Tasks\n- [ ] **T4 (P1, human: ~4h / CC: ~10min)** — tests — CRITICAL regression test pinning legacyAuthFlow() behavior\n  - Surfaced by: Test review REGRESSION RULE — PLAN.md:27-28\n  - Files: test/auth/legacyAuthFlow.regression.test\n  - Verify: passes against legacy before any rewrite; passes against AuthBroker after\n\n## Verification\n1. Write T4 first and run it against the untouched `legacyAuthFlow()`; it must pass before any other change.\n",
  "# Current reviewed plan\n\n### CRITICAL: regression suite for `legacyAuthFlow()` (regression rule, mandatory)\n\n`test/auth/legacyAuthFlow.regression.test.ts`. Captures current behavior\nbefore any rewrite: every success path, every error path, cache interactions,\nand the invalidation hooks it triggers. Runs against the flag-off path after\nthe refactor. This is the highest-priority test in the plan.\n\n## Implementation Tasks\n- [ ] **T4 (P1, human: ~4 hr / CC: ~15 min)** — legacyAuthFlow — CRITICAL regression suite for prior behavior\n  - Surfaced by: Test review REGRESSION RULE — PLAN.md:14-16, 27-28\n  - Files: `test/auth/legacyAuthFlow.regression.test.ts`\n  - Verify: suite green before and after the refactor on the flag-off path\n\n## Verification\n1. Run the regression suite (T4) against the current `legacyAuthFlow()` before touching it; it must be green on the unmodified code.\n"
];
const regression = (plan: string) => evaluateEngSeedCoverage({ status: 'ready', calls: [], assistantMessages: [] }, plan, 0, 1).regression;

const inlineRequired = `# Current reviewed plan
## Required tests
- **CRITICAL regression (T4)** \`auth/legacy-parity.test.ts\`:
  Record legacyAuthFlow() outputs before any change. Run the same fixtures
  against the new path; assert identical session shape and identical rejection class.
- **Other test** unrelated.test.ts: tests another feature.
## Implementation Tasks
- [ ] **T4 (P1)** — auth/tests — Regression: pin legacyAuthFlow() behavior
  - Files: auth/legacy-parity.test.ts
  - Verify: suite green on legacy before any refactor commit; green on both paths before rollout
## Verification
1. Run T4 against the untouched legacy path and commit the fixtures first.
2. Land the replacement and run T4 on both paths.
`;
test('inline required regression binds its own task, baseline and same-fixture parity', () => {
  expect(regression(inlineRequired)).toBe('plan');
  expect(regression(inlineRequired.replaceAll('T4', 'T17').replaceAll('auth/legacy-parity.test.ts', 'spec/old-path.test.ts'))).toBe('plan');
  expect(regression(inlineRequired.replace('CRITICAL regression', 'MANDATORY characterization').replace('Record', 'Capture')
    .replace('Run the same', 'Replay the same').replace('identical session shape', 'matching outputs')
    .replace('suite green on legacy', 'tests pass on the legacy path').replace('untouched', 'unmodified'))).toBe('plan');
  for (const change of [
    (s: string) => s.replace('CRITICAL regression', 'Optional regression'),
    (s: string) => s.replace('CRITICAL regression', 'CRITICAL regression withdrawn'),
    (s: string) => s.replace('## Required tests', '## Historical required tests'),
    (s: string) => s.replace('  Record', '  If approved, record'),
    (s: string) => s.replace('outputs before', 'behavior after'),
    (s: string) => s.replace('same fixtures', 'different fixtures'),
    (s: string) => s.replace('new path', 'unrelated path'),
    (s: string) => s.replace('identical rejection class', 'unspecified behavior'),
    (s: string) => s.replace('  - Files: auth/legacy-parity.test.ts', '  - Files: auth/other.test.ts'),
    (s: string) => s.replace('green on legacy before', 'red on legacy before'),
    (s: string) => s.replace('green on both paths', 'green on new path'),
    (s: string) => s.replace('untouched legacy', 'rewritten legacy'),
    (s: string) => s.replace('1. Run T4', '1. Run T9'),
    (s: string) => s + '\n## Current status\nT4 is "withdrawn".\n',
    (s: string) => s + '\n## Current status\nChange T4 assertions to match the new behavior.\n',
    (s: string) => s.split('\n').map(line => '> ' + line).join('\n'),
  ]) expect(regression(change(inlineRequired))).toBeUndefined();
});

test('mandatory named regression suites bind the task to an untouched baseline', () => {
  for (const report of reports) {
    expect(regression(report)).toBe('plan');
    for (const change of [
      (s: string) => s.replaceAll('T4', 'T17'),
      (s: string) => s.replaceAll('test/auth/legacyAuthFlow.regression.test', 'specs/old-auth.test'),
      (s: string) => s.replaceAll('AuthBroker', 'ReplacementBroker'),
      (s: string) => s.replace(/[`*]/g, ''),
      (s: string) => s + '\n## Future cleanup\nAfter 100% rollout for two weeks, delete legacyAuthFlow() and replace the parity test with a behavioral test.\n',
      (s: string) => s + '\n## History\nT4 is withdrawn.\n',
      (s: string) => s + '\n## Current assessment\n"T4 is withdrawn."\n',
      (s: string) => s + '\n## Payment regression suite\nThe regression suite is withdrawn.\n',
      (s: string) => s + '\n## Current assessment\nAfter committing the green baseline, run T4 after rewriting legacyAuthFlow().\n',
      (s: string) => s.replace('before any rewrite:', 'before any rewrite: A token receives success if accepted by legacyAuthFlow(). Rejected inputs receive the recorded error.'),
    ]) expect(regression(change(report))).toBe('plan');
  }
});

const controls: Array<[string, (s: string) => string]> = [
  ['missing declaration', s => s.replace(/### [\s\S]*?(?=## Implementation Tasks)/, '')],
  ['optional declaration', s => s.replaceAll('mandatory', 'optional')],
  ['never mandatory', s => s.replaceAll('mandatory', 'never mandatory')],
  ['missing legacy subject', s => s.replaceAll('legacyAuthFlow', 'otherAuthFlow')],
  ['no baseline capture', s => s.replace(/records|Captures/g, 'describes')],
  ['late baseline', s => s.replaceAll('before any rewrite', 'after any rewrite')],
  ['missing task', s => s.replace(/- \[ \] \*\*T4 [\s\S]*?(?=## Verification)/, '')],
  ['different task file', s => s.replace(/  - Files: .*/, '  - Files: test/other.test.ts')],
  ['missing verification', s => s.replace(/  - Verify: .*/, '')],
  ['missing baseline step', s => s.replace(/^1\. .*$/m, '')],
  ['rewritten baseline', s => s.replace(/untouched|unmodified/g, 'rewritten')],
  ['wrong task in baseline', s => s.replace(/^(1\. .*)T4/m, '$1T9')],
  ['quoted report', s => s.split('\n').map(l => '> ' + l).join('\n')],
  ['quoted baseline', s => s.replace(/^(1\. )(.*)$/m, '$1"$2"')],
  ['quoted declaration', s => s.replace(/(### [^\n]+\n)([\s\S]*?)(?=\n## Implementation Tasks)/, '$1"$2"')],
  ['fenced report', s => '```\n' + s + '\n```'],
  ['historical report', s => s.replace('Current reviewed plan', 'Historical reviewed plan')],
  ['source declaration', s => s.replace(/(### [^\n]+\n)/, '$1Source:\n')],
  ['conditional declaration', s => s.replace(/(### [^\n]+\n)/, '$1If approved,\n')],
  ['conditional task', s => s.replace('## Implementation Tasks\n', '## Implementation Tasks\nOnce approved,\n')],
  ['source task', s => s.replace('## Implementation Tasks\n', '## Implementation Tasks\nSource:\n')],
  ['explicit cancelled task', s => s + '\n## Current assessment\nDo not run T4.\n'],
  ['withdrawn task', s => s + '\n## Current assessment\nT4 is withdrawn.\n'],
  ['withdrawn verification', s => s + '\n## Current assessment\nT4 verification is optional.\n'],
  ['withdrawn legacy suite', s => s + '\n## Current assessment\nThe legacy regression suite is not required.\n'],
  ['quoted status', s => s + '\n## Current assessment\nT4 is "withdrawn".\n'],
  ['changed before baseline', s => s + '\n## Current assessment\nlegacyAuthFlow() is rewritten before T4.\n'],
  ['conditional mandatory heading', s => s.replace('mandatory', 'mandatory if approved')],
  ['conditional numbered baseline', s => s.replace(/^1\. /m, '1. Once approved, ')],
  ['conditional verification line', s => s.replace('  - Verify: ', '  - Verify: If approved, ')],
  ['hypothetical baseline', s => s + '\n## Current assessment\nT4 baseline verification is hypothetical.\n'],
  ['current post-rewrite instruction', s => s + '\n## Current assessment\nRun T4 only after rewriting legacyAuthFlow().\n'],
  ['post-rewrite baseline row', s => s.replace(/^1\. .*$/m, '1. Rewrite legacyAuthFlow() first, then run T4 against the unmodified legacyAuthFlow() snapshot; it must be green before rollout.')],
];
test.each(controls)('%s cannot provide mandatory regression coverage', (_, change) => {
  for (const report of reports) {
    const changed = change(report);
    expect(changed).not.toBe(report);
    expect(regression(changed)).toBeUndefined();
  }
});

test('the regression evidence test selects its existing Eng workflow', () => {
  expect(Object.entries(E2E_TOUCHFILES).filter(([, paths]) => paths.includes('test/eng-scheduled-regression.test.ts')).map(([name]) => name)).toEqual(['plan-eng-finding-count']);
});

// Verbatim owned rule, T1 and ordered verification from the failed AZ report.
const orderedRuleReport = "# Current reviewed plan\n\n### REGRESSION RULE — CRITICAL, no approval needed (skill iron rule)\n\nPLAN.md:27-28 rewrites `legacyAuthFlow()` with no regression test;\nPLAN.md:14-16 excluded it from coverage. That is modified existing behavior\nwith no covering test. **Before** the rewrite, add\n`legacyAuthFlow.characterization.test.ts` capturing current outputs for:\nvalid token, expired token, wrong tenant, wrong audience, revoked token,\nIDP unavailable. The rewrite must pass the same suite unchanged.\n\n## Implementation Tasks\n- [ ] **T1 (P1, human: ~half day / CC: ~15min)** — legacyAuthFlow — Write characterization suite for 6 prior behaviors BEFORE rewrite\n  - Surfaced by: Test review — REGRESSION RULE, PLAN.md:27-28 and 14-16\n  - Files: `legacyAuthFlow.characterization.test.ts`\n  - Verify: suite green on current code; green again after rewrite\n\n## Verification (end to end)\n1. Run T1's characterization suite on unmodified code: green.\n2. Implement T2-T6; run unit suites: green, no shared-state ordering flakes (run with shuffled order).\n3. Run T7 E2E: A/B isolation, suspend-mid-mint denial, double-submit consistency all green.\n4. Re-run T1 after the rewrite: green, unchanged.\n";

test('an iron-rule declaration and ordered task verification establish the mandatory baseline',()=>{
 expect(regression(orderedRuleReport)).toBe('plan');
 for(const change of [(s:string)=>s.replaceAll('T1','T17'),(s:string)=>s.replaceAll('legacyAuthFlow.characterization.test.ts','spec/legacy-golden.test.ts'),(s:string)=>s.replace(/[`*]/g,''),
  (s:string)=>s+'\n## History\nT1 is withdrawn.\n',(s:string)=>s+'\n## Current assessment\n"T1 is withdrawn."\n',(s:string)=>s+'\n## Payment regression suite\nThe regression suite is withdrawn.\n'])expect(regression(change(orderedRuleReport))).toBe('plan');
});
test('the ordered baseline stays owned, required, and unchanged across the rewrite',()=>{
 for(const [before,after] of [
  ['no approval needed','optional if approved'],['skill iron rule','hypothetical example'],['legacyAuthFlow','otherAuthFlow'],
  ['**Before** the rewrite','After the rewrite'],['capturing current outputs','capturing expected outputs'],
  ['The rewrite must pass the same suite unchanged.','The rewrite may update the expectations.'],
  ['suite green on current code; green again after rewrite','suite green on changed code; green again after rewrite'],
  ['suite green on current code','suite is not green on current code'],['green again after rewrite','not green again after rewrite'],
  ['on unmodified code: green.','on unmodified code: not green.'],['after the rewrite: green, unchanged.','after the rewrite: failing, unchanged.'],
  ['1. Run T1','1. Run T9'],['on unmodified code: green','on changed code: green'],['1. Run ','1. If approved, Run '],
  ['4. Re-run T1','4. Re-run T9'],['green, unchanged.','green, with updated expectations.'],
  ['## Implementation Tasks\n','## Implementation Tasks\nSource:\n'],['## Implementation Tasks\n','## Implementation Tasks\nOnce approved,\n'],
  ['PLAN.md:27-28','Source:\nPLAN.md:27-28'],['Current reviewed plan','Historical reviewed plan'],
 ]){const changed=orderedRuleReport.replaceAll(before!,after!);expect(changed).not.toBe(orderedRuleReport);expect(regression(changed)).toBeUndefined();}
 for(const change of [(s:string)=>s.replace(/^  - Files: .*$/m,'  - Files: different.test.ts'),(s:string)=>s.replace(/^1\. .*$/m,''),
  (s:string)=>s.replace(/^1\. .*$/m,'1. Rewrite legacyAuthFlow() before recording T1.'),(s:string)=>s.replace(/^(1\. )(.*)$/m,'$1"$2"'),
  (s:string)=>s.replace(/^4\. .*$/m,''),(s:string)=>s.split('\n').map(l=>'> '+l).join('\n'),(s:string)=>'```\n'+s+'\n```',
  (s:string)=>s+'\n## Current assessment\nT1 is withdrawn.\n',(s:string)=>s+'\n## Current assessment\nT1 verification is "optional".\n',
  (s:string)=>s+'\n## Current assessment\nDo not run T1.\n',(s:string)=>s+'\n## Current assessment\nlegacyAuthFlow() is rewritten before T1.\n',
  (s:string)=>s+'\n## Current assessment\nUpdate T1 assertions.\n']){const changed=change(orderedRuleReport);expect(changed).not.toBe(orderedRuleReport);expect(regression(changed)).toBeUndefined();}
});

test('required regression relations survive heading, task and verification paraphrases',()=>{
 const changed=orderedRuleReport.replace('REGRESSION RULE — CRITICAL, no approval needed (skill iron rule)','Required characterization baseline')
  .replace('The rewrite must pass the same suite unchanged.','The same suite must remain green unchanged after the rewrite.')
  .replace('Write characterization suite for 6 prior behaviors BEFORE rewrite','Add characterization tests for existing outputs')
  .replace('suite green on current code; green again after rewrite','current implementation passes; after the rewrite the suite passes again')
  .replace("1. Run T1's characterization suite on unmodified code: green.",'1) Execute characterization task T1 against untouched code; it must pass.')
  .replace('4. Re-run T1 after the rewrite: green, unchanged.','4) Execute the same T1 tests unchanged after the refactor; they must pass.');
 expect(regression(changed)).toBe('plan');
});