import { describe, expect, test, spyOn } from 'bun:test'; import { readFileSync } from 'node:fs'; import { MODEL_CATALOG, modelCatalogSha256 } from '../lib/model-catalog'; import { assessFreshness, freshnessCatalogIdentity, validLifecycle, validState } from '../lib/model-policy-freshness'; import { FRESHNESS_SOURCES, FRESHNESS_BOUNDS, sha256 } from '../lib/model-policy-freshness-sources'; import { FRESHNESS_ISSUE_AUTHOR, FRESHNESS_ISSUE_MARKER, FRESHNESS_REGION_BEGIN, FRESHNESS_REGION_END, publishFreshness, readFreshnessIssue, writeFreshnessIssue, type FreshnessGitHub, type TrackingIssue } from '../lib/model-policy-freshness-publication'; import { freshnessGitHub } from '../lib/model-policy-freshness-github'; import { runModelPolicyFreshness } from '../scripts/model-policy-freshness'; import { freshnessCatalog, freshnessFixture, freshnessObservation, freshnessReplacementObservation, retirementFixture } from './fixtures/model-policy-freshness.ts'; function mockGitHub(initial: (Omit & { author?: string })[] = []) { const issues: TrackingIssue[] = structuredClone(initial).map(issue => ({ author: FRESHNESS_ISSUE_AUTHOR, ...issue })); const calls: string[] = []; let identity = { defaultBranch: 'main', sourceSha256: freshnessCatalog.sourceSha256 }; const github: FreshnessGitHub = { async defaultCatalog() { calls.push('catalog'); return identity; }, async listIssues() { calls.push('list'); return structuredClone(issues); }, async readIssue(number) { calls.push('read'); return structuredClone(issues.find(issue => issue.number === number)!); }, async createIssue(body) { calls.push('create'); const issue: TrackingIssue = { number: issues.length + 1, body, state: 'open', author: FRESHNESS_ISSUE_AUTHOR }; issues.push(issue); return structuredClone(issue); }, async updateIssue(number, update) { calls.push('update'); Object.assign(issues.find(issue => issue.number === number)!, update); }, }; return { github, issues, calls, setIdentity(value: typeof identity) { identity = value; } }; } describe('stable-marker issue ownership and evidence lifecycle', () => { test('healthy bootstrap creates one closed issue, retains evidence when closed, and repeat publication is idempotent', async () => { const api = mockGitHub(); const observation = freshnessObservation(); const created = await publishFreshness(observation, api.github, 'refs/heads/main'); expect(created.action).toBe('created'); expect(api.issues).toHaveLength(1); expect(api.issues[0].state).toBe('closed'); expect(api.issues[0].body).toContain(FRESHNESS_ISSUE_MARKER); expect(readFreshnessIssue(api.issues[0].body).state?.lastSuccess?.checkedAt).toBe(observation.checkedAt); const body = api.issues[0].body; expect((await publishFreshness(observation, api.github, 'refs/heads/main')).action).toBe('rejected-older-observation'); expect(api.issues[0].body).toBe(body); const next = freshnessObservation('101', '2026-10-08T12:00:00.000Z'); expect((await publishFreshness(next, api.github, 'refs/heads/main')).action).toBe('updated'); expect(api.issues).toHaveLength(1); expect(api.issues[0].state).toBe('closed'); }); test('marker owns the issue, title collisions and human body/disposition text are untouched', async () => { const initial = writeFreshnessIssue('', assessFreshness(freshnessObservation())); const humanPrefix = 'Human notes: retain this default until qualification.\n'; const humanSuffix = '\nHuman disposition: mitigation pending; do not delete.\n'; const api = mockGitHub([ { number: 40, body: 'Model-policy freshness: advisory maintenance', state: 'open' }, { number: 41, body: humanPrefix + initial + humanSuffix, state: 'closed' }, ]); await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': retirementFixture() }), api.github, 'refs/heads/main'); expect(api.issues[0].body).toBe('Model-policy freshness: advisory maintenance'); expect(api.issues[1].body.startsWith(humanPrefix)).toBe(true); expect(api.issues[1].body.endsWith(humanSuffix)).toBe(true); expect(api.issues[1].state).toBe('open'); expect(readFreshnessIssue(api.issues[1].body).lifecycle[0].state).toBe('deprecated'); }); test('retirement survives source failure and partial recovery; issue never closes on either', async () => { const api = mockGitHub(); await publishFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }), api.github, 'refs/heads/main'); const evidence = readFreshnessIssue(api.issues[0].body).state!.lastSuccess; const failure = await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': null, 'openai-lifecycle': null }), api.github, 'refs/heads/main'); expect(failure.report?.status).toBe('unknown/source-unavailable'); expect(api.issues[0].state).toBe('open'); expect(readFreshnessIssue(api.issues[0].body).state?.lastSuccess).toEqual(evidence); await publishFreshness(freshnessObservation('102', '2026-10-09T12:00:00.000Z', { 'openai-lifecycle': null }), api.github, 'refs/heads/main'); expect(readFreshnessIssue(api.issues[0].body).lifecycle[0].deadline).toBe('2026-11-01'); expect(api.issues[0].state).toBe('open'); }); test('both complete and source-unavailable catalog rollbacks reopen the closed replacement issue', async () => { for (const unavailable of [false, true]) { const api = mockGitHub(); await publishFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }), api.github, 'refs/heads/main'); const replacement = freshnessReplacementObservation(); api.setIdentity({ defaultBranch: 'main', sourceSha256: replacement.catalog.sourceSha256 }); await publishFreshness(replacement, api.github, 'refs/heads/main'); expect(api.issues[0].state).toBe('closed'); expect(readFreshnessIssue(api.issues[0].body).lifecycle[0].resolvedByCatalog).toBe(replacement.catalog.sha256); api.setIdentity({ defaultBranch: 'main', sourceSha256: freshnessCatalog.sourceSha256 }); const rollback = freshnessObservation('102', '2026-10-09T12:00:00.000Z', unavailable ? { 'anthropic-lifecycle': null, 'openai-lifecycle': null } : {}); const result = await publishFreshness(rollback, api.github, 'refs/heads/main'); expect(result.report?.status).toBe(unavailable ? 'unknown/source-unavailable' : 'update-candidate'); expect(api.issues[0].state).toBe('open'); expect(api.issues).toHaveLength(1); const stored = readFreshnessIssue(api.issues[0].body); expect(stored.lifecycle[0]).toMatchObject({ state: 'deprecated', deadline: '2026-11-01' }); expect(stored.lifecycle[0].resolvedByCatalog).toBeUndefined(); if (unavailable) expect(stored.state?.lastSuccess?.catalog.sha256).toBe(replacement.catalog.sha256); } }); test('corrupt state salvages independent lifecycle history and requires explicit manual repair', async () => { const initial = writeFreshnessIssue('', assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }))); const broken = initial.replace(/(\n)[^\n]+/, '$1{broken'); const api = mockGitHub([{ number: 7, body: `Human retain decision\n${broken}\nHuman footer`, state: 'closed' }]); expect(readFreshnessIssue(api.issues[0].body).state).toBeNull(); expect(readFreshnessIssue(api.issues[0].body).lifecycle).toHaveLength(1); const result = await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main'); expect(result.report?.status).toBe('unknown/source-unavailable'); expect(result.report?.state.lastSuccess).toBeNull(); expect(result.report?.state.lifecycle).toHaveLength(1); expect(api.issues[0].state).toBe('open'); expect(api.issues[0].body).toContain('Human retain decision'); expect(api.issues[0].body).toContain('Human footer'); expect(api.issues[0].body).toContain(Buffer.from(broken.slice(broken.indexOf(FRESHNESS_REGION_BEGIN) + FRESHNESS_REGION_BEGIN.length, broken.indexOf(FRESHNESS_REGION_END)).trim(), 'utf8').toString('base64')); const next = await publishFreshness(freshnessObservation('102', '2026-10-09T12:00:00.000Z'), api.github, 'refs/heads/main'); expect(next.report?.state.recoveryRequired).toBe(true); }); test('valid empty state cannot erase independent retirement history during partial publication', async () => { const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() })); const state = { ...initial.state, lifecycle: [] }; const broken = writeFreshnessIssue('', initial).replace(/(\n)[^\n]+/, (_, marker) => marker + JSON.stringify(state)); const api = mockGitHub([{ number: 7, body: `Human retain decision\n${broken}\nHuman footer`, state: 'closed' }]); const recovered = readFreshnessIssue(api.issues[0].body); expect(validState(recovered.state)).toBe(true); expect(recovered.state?.lifecycle).toEqual([]); expect(recovered.lifecycle).toEqual(initial.state.lifecycle); expect(recovered.recoveryRequired).toBe(true); const result = await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': null, 'openai-lifecycle': null }), api.github, 'refs/heads/main'); expect(result.action).toBe('updated'); expect(result.report?.state.lifecycle).toEqual(initial.state.lifecycle); expect(result.report?.state.lastSuccess).toEqual(initial.state.lastSuccess); expect(result.report?.status).toBe('unknown/source-unavailable'); expect(result.report?.complete).toBe(false); const stored = readFreshnessIssue(api.issues[0].body); expect(stored.state?.lifecycle).toEqual(initial.state.lifecycle); expect(stored.lifecycle).toEqual(initial.state.lifecycle); expect(stored.recoveryRequired).toBe(true); expect(api.issues[0].state).toBe('open'); expect(api.issues[0].body).toContain('- URGENT: claude-fable-5-1 is deprecated; removal deadline 2026-11-01.'); expect(api.issues[0].body.startsWith('Human retain decision\n')).toBe(true); expect(api.issues[0].body.endsWith('\nHuman footer')).toBe(true); expect(api.issues[0].body).toContain(Buffer.from(broken.slice(broken.indexOf(FRESHNESS_REGION_BEGIN) + FRESHNESS_REGION_BEGIN.length, broken.indexOf(FRESHNESS_REGION_END)).trim(), 'utf8').toString('base64')); const next = await publishFreshness(freshnessObservation('102', '2026-10-09T12:00:00.000Z'), api.github, 'refs/heads/main'); expect(next.report?.state.lifecycle).toEqual(initial.state.lifecycle); expect(next.report?.state.lastSuccess).toEqual(initial.state.lastSuccess); expect(next.report?.state.recoveryRequired).toBe(true); expect(next.report?.complete).toBe(false); expect(api.issues[0].state).toBe('open'); }); test('distinct valid state and history findings are both retained regardless of copy ownership', async () => { const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture(), 'openai-lifecycle': freshnessFixture('openai-lifecycle').replace('`gpt-5.3-codex`', '`gpt-6-astra`'), })); expect(initial.state.lifecycle).toHaveLength(2); for (const reversed of [false, true]) { const findings = reversed ? initial.state.lifecycle.toReversed() : initial.state.lifecycle; const state = { ...initial.state, lifecycle: [findings[0]] }; const history = [findings[1]]; const body = writeFreshnessIssue('', initial) .replace(/(\n)[^\n]+/, (_, marker) => marker + JSON.stringify(state)) .replace(/(\n)[^\n]+/, (_, marker) => marker + JSON.stringify(history)); const recovered = readFreshnessIssue(body); expect(validState(recovered.state)).toBe(true); expect(validLifecycle(recovered.lifecycle)).toBe(true); expect(recovered.state?.lifecycle).toEqual(state.lifecycle); expect(recovered.lifecycle).toEqual(history); const api = mockGitHub([{ number: 7, body, state: 'open' }]); const result = await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': null, 'openai-lifecycle': null }), api.github, 'refs/heads/main'); expect(result.report?.state.lifecycle).toEqual(initial.state.lifecycle); expect(readFreshnessIssue(api.issues[0].body).lifecycle).toEqual(initial.state.lifecycle); expect(result.report?.state.recoveryRequired).toBe(true); expect(result.report?.state.lastSuccess).toEqual(initial.state.lastSuccess); expect(api.issues[0].body).toContain('- URGENT: claude-fable-5-1 is deprecated; removal deadline 2026-11-01.'); expect(api.issues[0].body).toContain('- URGENT: gpt-6-astra is deprecated; removal deadline 2027-04-01.'); expect(api.issues[0].state).toBe('open'); } }); test('conflicting copies retain the strongest severity, earliest deadline and full observation chronology', async () => { const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() })); const finding = initial.state.lifecycle[0]; const severe = { ...finding, state: 'removed' as const, deadline: '2026-10-01', firstObservedAt: '2026-09-15T12:00:00.000Z', lastObservedAt: '2026-10-05T12:00:00.000Z' }; const early = { ...finding, state: 'legacy' as const, deadline: '2026-09-30', firstObservedAt: '2026-09-01T12:00:00.000Z', lastObservedAt: '2026-10-06T12:00:00.000Z', resolvedByCatalog: 'b'.repeat(64) }; for (const reversed of [false, true]) { const state = { ...initial.state, lifecycle: [reversed ? early : severe] }; const history = [reversed ? severe : early]; const body = writeFreshnessIssue('', initial) .replace(/(\n)[^\n]+/, (_, marker) => marker + JSON.stringify(state)) .replace(/(\n)[^\n]+/, (_, marker) => marker + JSON.stringify(history)); expect(validState(readFreshnessIssue(body).state)).toBe(true); expect(validLifecycle(readFreshnessIssue(body).lifecycle)).toBe(true); const api = mockGitHub([{ number: 7, body, state: 'closed' }]); const result = await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': null }), api.github, 'refs/heads/main'); const expected = [{ ...severe, deadline: early.deadline, firstObservedAt: early.firstObservedAt, lastObservedAt: early.lastObservedAt }]; expect(result.report?.state.lifecycle).toEqual(expected); expect(validState(result.report?.state)).toBe(true); expect(readFreshnessIssue(api.issues[0].body).lifecycle).toEqual(expected); expect(result.report?.state.recoveryRequired).toBe(true); expect(result.report?.state.lastSuccess).toEqual(initial.state.lastSuccess); expect(api.issues[0].body).toContain('- URGENT: claude-fable-5-1 is removed; removal deadline 2026-09-30.'); expect(api.issues[0].state).toBe('open'); } }); test('conflicting historical dispositions stay unresolved until manual repair even after a complete replacement check', async () => { const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() })); const replacement = freshnessReplacementObservation(); const replaced = assessFreshness(replacement, initial.state); const finding = replaced.state.lifecycle[0]; expect(finding.resolvedByCatalog).toBe(replacement.catalog.sha256); for (const disposition of [undefined, 'd'.repeat(64)]) for (const reversed of [false, true]) for (const unavailable of [false, true]) { const disagreement = { ...finding, resolvedByCatalog: disposition }; const state = { ...replaced.state, lifecycle: [reversed ? disagreement : finding] }; const history = [reversed ? finding : disagreement]; const body = writeFreshnessIssue('', replaced) .replace(/(\n)[^\n]+/, (_, marker) => marker + JSON.stringify(state)) .replace(/(\n)[^\n]+/, (_, marker) => marker + JSON.stringify(history)); const api = mockGitHub([{ number: 7, body, state: 'closed' }]); api.setIdentity({ defaultBranch: 'main', sourceSha256: replacement.catalog.sourceSha256 }); const next = structuredClone(replacement); next.run.id = '102'; next.checkedAt = '2026-10-09T12:00:00.000Z'; if (unavailable) next.sources[3] = { id: 'openai-lifecycle', error: 'http-503' }; const result = await publishFreshness(next, api.github, 'refs/heads/main'); expect(result.report?.state.lifecycle).toHaveLength(1); expect(result.report?.state.lifecycle[0].resolvedByCatalog).toBeUndefined(); expect(readFreshnessIssue(api.issues[0].body).lifecycle[0].resolvedByCatalog).toBeUndefined(); expect(result.report?.state.recoveryRequired).toBe(true); expect(result.report?.state.lastSuccess).toEqual(replaced.state.lastSuccess); expect(result.report?.status).toBe('unknown/source-unavailable'); expect(result.report?.complete).toBe(false); expect(api.issues[0].body).toContain('- URGENT: claude-fable-5-1 is deprecated; removal deadline 2026-11-01.'); expect(api.issues[0].state).toBe('open'); } }); test('matching resolved history remains resolved without recovery and only complete evidence closes the issue', async () => { const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() })); const replacement = freshnessReplacementObservation(); const replaced = assessFreshness(replacement, initial.state); for (const unavailable of [false, true]) { const api = mockGitHub([{ number: 7, body: writeFreshnessIssue('', replaced), state: 'closed' }]); api.setIdentity({ defaultBranch: 'main', sourceSha256: replacement.catalog.sourceSha256 }); const next = structuredClone(replacement); next.run.id = '102'; next.checkedAt = '2026-10-09T12:00:00.000Z'; if (unavailable) next.sources[3] = { id: 'openai-lifecycle', error: 'http-503' }; const result = await publishFreshness(next, api.github, 'refs/heads/main'); expect(result.report?.state.lifecycle).toEqual(replaced.state.lifecycle); expect(result.report?.state.recoveryRequired).toBe(false); expect(result.report?.complete).toBe(!unavailable); expect(result.report?.status).toBe(unavailable ? 'unknown/source-unavailable' : 'current'); expect(api.issues[0].state).toBe(unavailable ? 'open' : 'closed'); expect(api.issues[0].body).not.toContain('- URGENT:'); } }); test('missing machine state is recovery, malformed owned boundary is not overwritten, and output is bounded', async () => { const api = mockGitHub([{ number: 9, body: `${FRESHNESS_ISSUE_MARKER}\nMaintainer notes`, state: 'closed' }]); const result = await publishFreshness(freshnessObservation(), api.github, 'refs/heads/main'); expect(result.report?.status).toBe('unknown/source-unavailable'); expect(api.issues[0].state).toBe('open'); expect(api.issues[0].body).toContain('Maintainer notes'); expect(() => writeFreshnessIssue(`${FRESHNESS_ISSUE_MARKER}\n${FRESHNESS_REGION_BEGIN}\nbroken`, assessFreshness(freshnessObservation()))).toThrow('corrupt-owned-region'); expect(() => writeFreshnessIssue('x'.repeat(FRESHNESS_BOUNDS.issueBytes), assessFreshness(freshnessObservation()))).toThrow('tracking-issue-size-limit'); expect(() => writeFreshnessIssue(`${FRESHNESS_REGION_END}\n${FRESHNESS_REGION_BEGIN}`, assessFreshness(freshnessObservation()))).toThrow('corrupt-owned-region'); }); test('duplicate markers fail visibly rather than selecting an arbitrary issue', async () => { const api = mockGitHub([{ number: 1, body: FRESHNESS_ISSUE_MARKER, state: 'open' }, { number: 2, body: FRESHNESS_ISSUE_MARKER, state: 'closed' }]); await expect(publishFreshness(freshnessObservation(), api.github, 'refs/heads/main')).rejects.toThrow('duplicate-tracking-marker'); expect(api.calls).not.toContain('update'); expect(api.calls).not.toContain('create'); }); test('user-created marker copies never become authority or trigger replacement bootstrap', async () => { const body = writeFreshnessIssue('', assessFreshness(freshnessObservation())); for (const includeOwned of [false, true]) { const api = mockGitHub([{ number: 1, body, state: 'closed', author: 'untrusted-user' }, ...(includeOwned ? [{ number: 2, body, state: 'closed' as const }] : [])]); await expect(publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main')).rejects.toThrow('untrusted-tracking-marker-owner'); expect(api.calls).not.toContain('read'); expect(api.calls).not.toContain('update'); expect(api.calls).not.toContain('create'); } }); test('CRLF and CR-only marker lines retain retirement history and publish to the same issue on source failure', async () => { const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() })); for (const newline of ['\r\n', '\r']) { const prefix = `Human prefix remains byte-identical.${newline}${newline}`; const suffix = `${newline}${newline}Human suffix remains byte-identical.${newline}`; const body = prefix + writeFreshnessIssue('', initial).replaceAll('\n', newline) + suffix; const api = mockGitHub([{ number: 17, body, state: 'open' }]); const result = await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': null, 'openai-lifecycle': null }), api.github, 'refs/heads/main'); expect(result.action).toBe('updated'); expect(result.issueNumber).toBe(17); expect(api.issues).toHaveLength(1); expect(api.calls).not.toContain('create'); expect(api.issues[0].body.startsWith(prefix)).toBe(true); expect(api.issues[0].body.endsWith(suffix)).toBe(true); const stored = readFreshnessIssue(api.issues[0].body); expect(stored.lifecycle).toEqual(initial.state.lifecycle); expect(stored.state?.lastSuccess).toEqual(initial.state.lastSuccess); expect(result.report?.status).toBe('unknown/source-unavailable'); } }); test('inline and quoted marker mentions preserve human prefix and suffix outside the real region', async () => { const initial = assessFreshness(freshnessObservation()); const prefix = `Human note mentioning ${FRESHNESS_REGION_BEGIN} inline; preserve this entire sentence.\r\n> ${FRESHNESS_REGION_END}\r\nQuoted '${FRESHNESS_REGION_BEGIN}' and '${FRESHNESS_REGION_END}' are human prose.\r\n\r\n`; const suffix = `\r\nHuman suffix mentions ${FRESHNESS_REGION_END} inline.\r\n> ${FRESHNESS_REGION_BEGIN}\r\n`; const body = prefix + writeFreshnessIssue('', initial) + suffix; const report = assessFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), initial.state); const rewritten = writeFreshnessIssue(body, report); expect(rewritten.startsWith(prefix)).toBe(true); expect(rewritten.endsWith(suffix)).toBe(true); expect(readFreshnessIssue(rewritten).state).toEqual(report.state); const api = mockGitHub([{ number: 19, body, state: 'closed' }]); const result = await publishFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z'), api.github, 'refs/heads/main'); expect(result.action).toBe('updated'); expect(api.issues).toHaveLength(1); expect(api.issues[0].body.startsWith(prefix)).toBe(true); expect(api.issues[0].body.endsWith(suffix)).toBe(true); }); test('malformed or missing top markers with owned evidence never become a healthy bootstrap', async () => { const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() })); for (const marker of ['', `${FRESHNESS_ISSUE_MARKER} `, FRESHNESS_ISSUE_MARKER.replace(':v1', ':v2'), FRESHNESS_ISSUE_MARKER.replace('