mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-16 13:21:30 +01:00
198 lines
12 KiB
TypeScript
198 lines
12 KiB
TypeScript
import { expect, test } from 'bun:test';
|
|
import fixture from './fixtures/eng-declared-regression-ai.json';
|
|
import { evaluateEngSeedCoverage } from './helpers/eng-seeded-coverage';
|
|
import type { PlanCountTranscript } from './helpers/plan-count-transcript';
|
|
import { E2E_TOUCHFILES, selectTests } from './helpers/touchfiles';
|
|
|
|
const transcript = fixture.transcript as PlanCountTranscript;
|
|
const { start, end } = fixture.provenance.window;
|
|
const plan = [
|
|
'## Tests\n\n### CRITICAL regression (mandatory, regression rule)\n\n' + fixture.mandatory,
|
|
'## Implementation Tasks\n\n' + fixture.task,
|
|
'## Verification\n\n' + fixture.verification,
|
|
fixture.reviewReport,
|
|
].join('\n\n');
|
|
const evaluate = (text = plan, native = transcript) => evaluateEngSeedCoverage(native, text, start, end);
|
|
const retryPlan = [
|
|
'## Architecture\n\n' + fixture.retry.legacyHeading + '\n\n' + fixture.retry.legacy,
|
|
'## Tests\n\n' + fixture.retry.heading + '\n\n' + fixture.retry.mandatory,
|
|
'## Implementation Tasks\n\n' + fixture.retry.task,
|
|
fixture.retry.reviewReport,
|
|
].join('\n\n');
|
|
const evaluateRetry = (text = retryPlan) => evaluateEngSeedCoverage(fixture.retry.transcript as PlanCountTranscript,
|
|
text, fixture.retry.provenance.window.start, fixture.retry.provenance.window.end);
|
|
|
|
test('actual mandatory suite, numbered task and unchanged baseline bind legacy regression', () => {
|
|
const result = evaluate();
|
|
expect(Object.keys(result.decisions)).toHaveLength(4);
|
|
expect(result.missing).toEqual([]);
|
|
expect(result.regression).toBe('plan');
|
|
expect(result.ok).toBe(true);
|
|
expect(fixture.provenance.retrospectivePass).toBe(false);
|
|
});
|
|
|
|
test('retry same-fixture contract compares new behavior with the unchanged legacy release oracle', () => {
|
|
const result = evaluateRetry();
|
|
expect(result.missing).toEqual([]);
|
|
expect(result.regression).toBe('plan');
|
|
expect(result.ok).toBe(true);
|
|
expect(fixture.retry.provenance.retrospectivePass).toBe(false);
|
|
});
|
|
|
|
test('retry requires an unchanged legacy release oracle and actual result parity', () => {
|
|
for (const text of [
|
|
retryPlan.replace(fixture.retry.legacy, ''),
|
|
retryPlan.replace('stays callable and unchanged this release', 'will be rewritten this release'),
|
|
retryPlan.replace('stays callable and unchanged this release', 'might stay callable and unchanged this release'),
|
|
retryPlan.replace('- A tenant-keyed flag', 'If approved:\n\n- A tenant-keyed flag'),
|
|
retryPlan.replace('- A tenant-keyed flag', 'Unless rejected.\n\n- A tenant-keyed flag'),
|
|
retryPlan.replace('- A tenant-keyed flag', 'Proposed baseline:\n\n- A tenant-keyed flag'),
|
|
retryPlan.replace('legacyAuthFlow()` stays callable', 'newAuthFlow()` stays callable'),
|
|
retryPlan.replace('Run each fixture', 'Run different fixtures'),
|
|
retryPlan.replace('and assert identical `Session` shape on success', 'and document different `Session` shape on success'),
|
|
retryPlan.replace('identical error code on failure', 'similar error code on failure'),
|
|
retryPlan.replace('through `legacyAuthFlow()` and', 'through `newAuthFlow()` and'),
|
|
retryPlan.replace('AuthBroker.authenticate()', 'NewBroker.authenticate()'),
|
|
retryPlan.replace('and AuthBroker, identical', 'and NewBroker, identical'),
|
|
retryPlan.replace('identical Session / error codes', 'identical OtherResponse / error codes'),
|
|
retryPlan.replace('Files: src/auth/authFlow.contract.test.ts', 'Files: src/auth/other.contract.test.ts'),
|
|
retryPlan.replace('suite green on both paths', 'suite green on the new path'),
|
|
retryPlan.replace(fixture.retry.task, ''),
|
|
retryPlan.replace('This test\nis also the gate', 'This optional test\nis also the gate'),
|
|
]) { expect(text).not.toBe(retryPlan); expect(evaluateRetry(text).regression, text).toBeUndefined(); }
|
|
});
|
|
|
|
test('retry proposals and current withdrawals cannot supply parity coverage', () => {
|
|
for (const text of [
|
|
retryPlan.replace(fixture.retry.mandatory, 'If approved, ' + fixture.retry.mandatory),
|
|
retryPlan.replace(fixture.retry.mandatory, '"' + fixture.retry.mandatory + '"'),
|
|
retryPlan.replace(fixture.retry.mandatory, '```\n' + fixture.retry.mandatory + '\n```'),
|
|
retryPlan.replace(fixture.retry.mandatory, fixture.retry.mandatory + '\nThis test is withdrawn.'),
|
|
retryPlan.replace(fixture.retry.task, fixture.retry.task + '\nT4 is no longer required.'),
|
|
retryPlan.replace(fixture.retry.legacy, fixture.retry.legacy + '\nCorrection: legacyAuthFlow() is changed this release.'),
|
|
retryPlan.replace(fixture.retry.legacy, fixture.retry.legacy.split('\n').map(line => '> ' + line).join('\n')),
|
|
'# Hypothetical example\n\n' + retryPlan,
|
|
'# Proposed work\n\n' + retryPlan,
|
|
]) { expect(text).not.toBe(retryPlan); expect(evaluateRetry(text).regression, text).toBeUndefined(); }
|
|
});
|
|
|
|
test('consistent retry identities and unrelated negative outcomes retain parity evidence', () => {
|
|
for (const text of [
|
|
retryPlan.replaceAll('AuthBroker', 'SessionBroker').replaceAll('Session', 'Reply'),
|
|
retryPlan.replaceAll('authFlow.contract.test.ts', 'loginFlow.contract.test.js').replaceAll('T4', 'T14'),
|
|
retryPlan.replace(fixture.retry.task, fixture.retry.task + '\n - Verify revoked tokens are rejected.'),
|
|
retryPlan.replace(fixture.retry.legacy, fixture.retry.legacy + '\nRejected alternatives stay documented.'),
|
|
]) expect(evaluateRetry(text).regression).toBe('plan');
|
|
});
|
|
|
|
for (const prefix of [
|
|
'# Source\n\n',
|
|
'An unproven hypothesis.\n\n',
|
|
'The following is a hypothetical example.\n\n',
|
|
'The following is source material only, not the current reviewed plan.\n\n',
|
|
'# Current reviewed plan\n\nThe following sections reproduce source material only; they are not requirements of this plan.\n\n',
|
|
]) {
|
|
test('first and retry evidence retain enclosing source frame: ' + prefix.trim(), () => {
|
|
expect(evaluate(prefix + plan).regression).toBeUndefined();
|
|
expect(evaluateRetry(prefix + retryPlan).regression).toBeUndefined();
|
|
});
|
|
}
|
|
|
|
test('declaration wording and task identity can vary without changing the required baseline', () => {
|
|
for (const text of [
|
|
plan.replaceAll('T4', 'T12'),
|
|
plan.replace('capture current', 'pin existing'),
|
|
plan.replace('is added as a critical', 'is required as a mandatory'),
|
|
plan.replace('auth/legacy tests — ', 'core/auth — '),
|
|
plan.replace('wrong-audience, wrong-issuer,', 'wrong-audience, wrong-issuer, malformed,'),
|
|
plan.replace(fixture.task, fixture.task + '\n - Verify expired and revoked tokens are rejected.'),
|
|
plan.replace(fixture.mandatory, fixture.mandatory + '\nKeep a record of rejected alternatives.'),
|
|
'# Historical example\n\nA proposed suite was discussed.\n\n# Current reviewed plan\n\n' + plan,
|
|
]) expect(evaluate(text).regression, text).toBe('plan');
|
|
});
|
|
|
|
test('declaration, task, target and original baseline cannot lend each other missing evidence', () => {
|
|
for (const text of [
|
|
plan.replace(fixture.mandatory, ''),
|
|
plan.replace(fixture.task, ''),
|
|
plan.replace(fixture.verification, ''),
|
|
plan.replace('suite (T4)', 'suite (T5)'),
|
|
plan.replace('against the untouched', 'against the rewritten'),
|
|
plan.replace('first and commit it green. This is the baseline.', 'after rollout and document it.'),
|
|
plan.replace('capture current', 'describe future'),
|
|
plan.replace('is\nthe oracle the new path is compared to', 'is documentation the new path links to'),
|
|
plan.replace('characterization test suite** for `legacyAuthFlow()`', 'characterization test suite** for `newAuthFlow()`'),
|
|
plan.replace('suite for `legacyAuthFlow()` prior behavior', 'suite for `newAuthFlow()` prior behavior'),
|
|
plan.replace('untouched `legacyAuthFlow()`', 'untouched `newAuthFlow()`'),
|
|
]) { expect(text).not.toBe(plan); expect(evaluate(text).regression, text).toBeUndefined(); }
|
|
});
|
|
|
|
test('proposals, future work, conditional and quoted declarations are not required coverage', () => {
|
|
for (const text of [
|
|
plan.replace('is added as a critical', 'will be added as a critical'),
|
|
plan.replace('is added as a critical', 'might be added as a critical'),
|
|
plan.replace('is added as a critical', 'is not added as a critical'),
|
|
plan.replace(fixture.mandatory, 'If approved, ' + fixture.mandatory),
|
|
plan.replace(fixture.mandatory, 'Example: ' + fixture.mandatory),
|
|
plan.replace(fixture.mandatory, 'An unproven hypothesis. ' + fixture.mandatory),
|
|
plan.replace(fixture.mandatory, '"' + fixture.mandatory + '"'),
|
|
plan.replace(fixture.mandatory, "'" + fixture.mandatory + "'"),
|
|
plan.replace(fixture.mandatory, fixture.mandatory.split('\n').map(line => '> ' + line).join('\n')),
|
|
plan.replace(fixture.mandatory, '```\n' + fixture.mandatory + '\n```'),
|
|
'# Hypothetical example\n\n' + plan,
|
|
'# Quoted source\n\n' + plan,
|
|
'# Proposed work\n\n' + plan,
|
|
plan.replace('## Verification\n\n1. Run', '## Verification\n\n1. If approved, run'),
|
|
]) { expect(text).not.toBe(plan); expect(evaluate(text).regression, text).toBeUndefined(); }
|
|
});
|
|
|
|
test('withdrawal of the owned suite, task or baseline prevents credit', () => {
|
|
for (const [from, addition] of [
|
|
[fixture.mandatory, 'This suite is withdrawn.'],
|
|
[fixture.mandatory, 'The characterization suite is not required.'],
|
|
[fixture.mandatory, 'Do not run the suite.'],
|
|
[fixture.task, 'Correction: T4 is cancelled.'],
|
|
[fixture.task, 'This task is deferred.'],
|
|
[fixture.verification, 'Correction: T4 is cancelled.'],
|
|
[fixture.verification, 'Skip the characterization suite.'],
|
|
]) expect(evaluate(plan.replace(from!, from + '\n' + addition)).regression, addition).toBeUndefined();
|
|
});
|
|
|
|
test('the required suite cannot replace completed distinct native decisions or final report', () => {
|
|
for (let index = 0; index < transcript.calls.length; index++) {
|
|
const native = structuredClone(transcript);
|
|
native.calls.splice(index, 1);
|
|
expect(evaluate(plan, native).ok).toBe(false);
|
|
expect(evaluate(plan, native).missing).toHaveLength(1);
|
|
}
|
|
for (const mutate of [
|
|
(native: PlanCountTranscript) => { native.calls[0]!.failed = true; },
|
|
(native: PlanCountTranscript) => { native.calls[0]!.answeredAt = new Date(start - 1).toISOString(); },
|
|
(native: PlanCountTranscript) => { native.calls[0]!.sessionId = 'foreign-session'; },
|
|
(native: PlanCountTranscript) => { native.calls.push(structuredClone(native.calls[0]!)); },
|
|
]) {
|
|
const native = structuredClone(transcript); mutate(native);
|
|
expect(evaluate(plan, native).ok).toBe(false);
|
|
}
|
|
expect(evaluate(plan.replace(fixture.reviewReport, '')).problems).toContain('final review report absent or empty');
|
|
});
|
|
|
|
test('public declaration still requires the existing owned time and session interval', () => {
|
|
const native = structuredClone(transcript);
|
|
native.assistantMessages = [{ sessionId: native.calls[0]!.sessionId, timestamp: new Date(start).toISOString(), text: plan }];
|
|
expect(evaluate(fixture.reviewReport, native).regression).toBe('public-narration');
|
|
native.assistantMessages[0]!.timestamp = new Date(start - 1).toISOString();
|
|
expect(evaluate(fixture.reviewReport, native).regression).toBeUndefined();
|
|
native.assistantMessages[0]!.timestamp = new Date(end + 1).toISOString();
|
|
expect(evaluate(fixture.reviewReport, native).regression).toBeUndefined();
|
|
native.assistantMessages[0]!.timestamp = new Date(start).toISOString();
|
|
native.assistantMessages[0]!.sessionId = 'foreign-session';
|
|
expect(evaluate(fixture.reviewReport, native).regression).toBeUndefined();
|
|
});
|
|
|
|
test('new declaration evidence registers only the two existing engineering count owners', () => {
|
|
for (const file of ['test/eng-declared-regression-ai.test.ts', 'test/fixtures/eng-declared-regression-ai.json']) {
|
|
expect(selectTests([file], E2E_TOUCHFILES, []).selected.sort()).toEqual(['plan-eng-finding-count', 'plan-eng-multi-finding-batching']);
|
|
}
|
|
});
|