gstack/test/model-policy-freshness.test.ts

import { describe, expect, test, spyOn } from 'bun:test';
import { collectFreshnessSources, fetchOfficialSource, FRESHNESS_BOUNDS, FRESHNESS_SOURCES, lifecycleDate, parseOfficialSource, parseOfficialSourceBounded, withinDeadline } from '../lib/model-policy-freshness-sources';
import { assessFreshness, renderFreshnessReport, storedEvidenceStatus, validEvidence, validLifecycle, validState } from '../lib/model-policy-freshness';
import { freshnessFixture, freshnessObservation, freshnessReplacementObservation, retirementFixture } from './fixtures/model-policy-freshness.ts';

describe('official recommendation and lifecycle structures', () => {
  test('four official fixtures parse, with Mythos and replacement mentions excluded', () => {
    const observation = freshnessObservation();
    expect(assessFreshness(observation).status).toBe('current');
    const anthropic = parseOfficialSource('anthropic-models', freshnessFixture('anthropic-models'));
    expect(anthropic.recommendations).toEqual({ frontier: 'claude-fable-5-1', smart: 'claude-opus-5-5' });
    expect(anthropic.lineup).not.toContain('claude-mythos-5-1');
    const openai = parseOfficialSource('openai-lifecycle', freshnessFixture('openai-lifecycle'));
    expect(openai.lifecycle?.map(row => row.modelId)).toEqual(['gpt-5.3-codex', 'gpt-4o-2024-08-06', 'gpt-4o-2024-11-20', 'gpt-old', 'gpt-unicode-date', 'gpt-iso-date', 'gpt-date-floor']);
    expect(openai.lifecycle?.find(row => row.modelId === 'gpt-date-floor')).toMatchObject({ deadline: null, supportFloor: '2024-06-13' });
  });

  test('known-family and actual unknown-family lineup replacements become candidates', () => {
    const known = freshnessFixture('anthropic-models').replaceAll('fable-5-1', 'fable-5-2').replaceAll('Fable 5.1', 'Fable 5.2');
    const knownReport = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-models': known }));
    expect(knownReport.status).toBe('update-candidate');
    expect(knownReport.candidates[0].reason).toBe('changed-monitored-family-recommendation');
    const unknown = freshnessFixture('anthropic-models').replaceAll('fable-5-1', 'mythos-5-1').replaceAll('Fable 5.1', 'Mythos 5.1');
    const report = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-models': unknown }));
    expect(report.status).toBe('update-candidate');
    expect(report.candidates[0]).toMatchObject({ proposed: 'claude-mythos-5-1', reason: 'unknown-replacement-family-needs-triage' });
    expect(report.complete).toBe(true);
  });

  test('OpenAI full-catalog newer IDs alone are not recommendations', () => {
    const original = freshnessFixture('openai-models');
    expect(assessFreshness(freshnessObservation()).candidates).toEqual([]);
    const changed = original.replaceAll('gpt-6.1-sol', 'gpt-6.2-sol').replaceAll('GPT-6.1 Sol', 'GPT-6.2 Sol');
    expect(assessFreshness(freshnessObservation('100', undefined, { 'openai-models': changed })).candidates[0]).toMatchObject({ proposed: 'gpt-6.2-sol' });
  });

  test('active support floors are never retirement deadlines, even after the floor', () => {
    const parsed = parseOfficialSource('anthropic-lifecycle', freshnessFixture('anthropic-lifecycle'));
    expect(parsed.lifecycle?.[0]).toMatchObject({ state: 'active', deadline: null, supportFloor: '2027-09-01' });
    expect(assessFreshness(freshnessObservation('100', '2028-10-07T12:00:00.000Z')).state.lifecycle).toEqual([]);
  });

  test('actual API retirement and removal are urgent, incidental partner prose is ignored', () => {
    const report = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
    expect(report.status).toBe('update-candidate');
    expect(report.state.lifecycle[0]).toMatchObject({ state: 'deprecated', deadline: '2026-11-01' });
    expect(renderFreshnessReport(report)).toContain('URGENT');
    expect(renderFreshnessReport(report)).toContain('Bedrock/Vertex/custom');
    const removed = assessFreshness(freshnessObservation('100', '2026-11-02T12:00:00.000Z', { 'anthropic-lifecycle': retirementFixture() }));
    expect(removed.state.lifecycle[0].state).toBe('removed');
  });

  test('OpenAI shutdown rows apply to the deprecated column, not the replacement', () => {
    const text = freshnessFixture('openai-lifecycle').replace('`gpt-5.3-codex`', '`gpt-6-astra`');
    const report = assessFreshness(freshnessObservation('100', undefined, { 'openai-lifecycle': text }));
    expect(report.state.lifecycle).toHaveLength(1);
    expect(report.state.lifecycle[0]).toMatchObject({ modelId: 'gpt-6-astra', deadline: '2027-04-01' });
  });

  test('OpenAI deprecated cells with a display name before the backticked IDs yield every ID', () => {
    const openai = parseOfficialSource('openai-lifecycle', freshnessFixture('openai-lifecycle'));
    expect(openai.lifecycle?.filter(row => row.modelId.startsWith('gpt-4o-'))).toEqual([
      { modelId: 'gpt-4o-2024-08-06', state: 'deprecated', deadline: '2027-02-16', supportFloor: null },
      { modelId: 'gpt-4o-2024-11-20', state: 'deprecated', deadline: '2027-02-16', supportFloor: null },
    ]);
    const unticked = freshnessFixture('openai-lifecycle').replace('GPT-4o (`gpt-4o-2024-08-06`, `gpt-4o-2024-11-20`)', 'GPT-4o (gpt-4o-2024-08-06, gpt-4o-2024-11-20)');
    expect(() => parseOfficialSource('openai-lifecycle', unticked)).toThrow('unrecognized-deprecated-model-cell');
  });

  test('missing required structures and semantic drift fail closed', () => {
    expect(() => parseOfficialSource('anthropic-models', freshnessFixture('anthropic-models').replace('Claude API ID', 'Google Cloud ID'))).toThrow('missing-api-id-row');
    expect(() => parseOfficialSource('anthropic-models', freshnessFixture('anthropic-models').replace('for demanding reasoning', 'for unexplained goals'))).toThrow('unrecognized-recommendation');
    expect(() => parseOfficialSource('openai-models', freshnessFixture('openai-models').replace('Balance intelligence and cost.', 'Best new model.'))).toThrow('unrecognized-recommendation');
    expect(() => parseOfficialSource('openai-lifecycle', '# New deprecations\nNo model tables')).toThrow('missing-or-ambiguous-section');
    expect(() => parseOfficialSource('openai-lifecycle', freshnessFixture('openai-lifecycle').replace('`gpt-5.3-codex`', 'gpt-5.3-codex'))).toThrow('unrecognized-deprecated-model-cell');
    expect(() => parseOfficialSource('anthropic-lifecycle', freshnessFixture('anthropic-lifecycle').replace('Active', 'Probably active'))).toThrow('unrecognized-lifecycle-state');
    expect(() => lifecycleDate('February 30, 2027')).toThrow('malformed-lifecycle-date');
    expect(() => lifecycleDate('tomorrow')).toThrow('malformed-lifecycle-date');
    expect(() => parseOfficialSource('anthropic-models', `${freshnessFixture('anthropic-models')}\n## Compare models\nDuplicate`)).toThrow('ambiguous-section');
    expect(() => parseOfficialSource('anthropic-models', freshnessFixture('anthropic-models').replace('## Using the Models API', 'Use [Other](https://platform.claude.com/docs/en/models/opus-5-5/overview) for demanding reasoning.\n\n## Using the Models API'))).toThrow('unrecognized-recommendation');
  });
});

describe('bounded fixed-source fetching', () => {
  test('exact byte limit passes and a streamed byte beyond it fails without trusting content-length', async () => {
    const atCap = 'x'.repeat(FRESHNESS_BOUNDS.responseBytes);
    expect((await fetchOfficialSource('openai-models', async () => new Response(atCap))).bytes).toBe(2_097_152);
    await expect(fetchOfficialSource('openai-models', async () => new Response(`${atCap}x`))).rejects.toThrow('response-too-large');
    await expect(fetchOfficialSource('openai-models', async () => new Response('x', { headers: { 'content-length': '2097153' } }))).rejects.toThrow('response-too-large');
    expect(() => parseOfficialSource('openai-models', `${atCap}x`)).toThrow('response-too-large');
  });

  test('all redirect destinations are checked; three redirects pass, four fail', async () => {
    const credentialUrl = new URL(FRESHNESS_SOURCES[2].url);
    credentialUrl.username = 'user';
    credentialUrl.password = 'pass';
    for (const location of ['https://evil.example/models.md', 'http://developers.openai.com/api/docs/models.md', credentialUrl.href, 'https://developers.openai.com/api/docs/models.md?untrusted=1']) {
      let calls = 0;
      await expect(fetchOfficialSource('openai-models', async () => { calls++; return new Response(null, { status: 302, headers: { location } }); })).rejects.toThrow('redirect-not-allowlisted');
      expect(calls).toBe(1);
    }
    let calls = 0;
    expect((await fetchOfficialSource('openai-models', async () => ++calls <= 3 ? new Response(null, { status: 302, headers: { location: FRESHNESS_SOURCES[2].url } }) : new Response('data'))).text).toBe('data');
    calls = 0;
    await expect(fetchOfficialSource('openai-models', async () => { calls++; return new Response(null, { status: 302, headers: { location: FRESHNESS_SOURCES[2].url } }); })).rejects.toThrow('redirect-limit');
    expect(calls).toBe(4);
  });

  test('per-response and whole-check clocks are installed at the exact approved bounds', async () => {
    const installed: number[] = [];
    const real = globalThis.setTimeout;
    const timers = spyOn(globalThis, 'setTimeout').mockImplementation(((callback: () => void, milliseconds: number) => {
      installed.push(milliseconds);
      return real(callback, milliseconds === 30_000 ? 5 : milliseconds);
    }) as typeof setTimeout);
    try {
      const result = await collectFreshnessSources(async () => new Promise<Response>(() => {}));
      expect(result.every(source => source.error === 'deadline-exceeded')).toBe(true);
      expect(installed.filter(ms => ms === 30_000)).toHaveLength(4);
      expect(installed).toContain(180_000);
    } finally { timers.mockRestore(); }
    const controller = new AbortController();
    await expect(withinDeadline(new Promise(() => {}), 5, controller)).rejects.toThrow('deadline-exceeded');
    expect(controller.signal.aborted).toBe(true);
  });

  test('only four fixed sources are fetched and failures never become parseable success', async () => {
    const urls: string[] = [];
    const result = await collectFreshnessSources(async (url, init) => {
      urls.push(url);
      expect(init.redirect).toBe('manual');
      const id = FRESHNESS_SOURCES.find(source => source.url === url)!.id;
      return id === 'openai-lifecycle' ? new Response('authentication needed', { status: 403 }) : new Response(freshnessFixture(id));
    });
    expect(urls.sort()).toEqual(FRESHNESS_SOURCES.map(source => source.url).sort());
    expect(result.find(source => source.id === 'openai-lifecycle')?.error).toBe('http-403');
    const observation = freshnessObservation();
    observation.sources = result;
    expect(assessFreshness(observation).status).toBe('unknown/source-unavailable');
  });

  test('CPU-bound parsing is terminable and the complete-check deadline stops all blocked parsers', async () => {
    let terminations = 0;
    const createWorker = () => ({ postMessage() {}, terminate() { terminations++; }, onmessage: null, onerror: null }) as unknown as Worker;
    const controller = new AbortController();
    const blocked = parseOfficialSourceBounded('openai-models', freshnessFixture('openai-models'), controller.signal, createWorker);
    controller.abort();
    await expect(blocked).rejects.toThrow('deadline-exceeded');
    expect(terminations).toBe(1);
    const real = globalThis.setTimeout;
    const timers = spyOn(globalThis, 'setTimeout').mockImplementation(((callback: () => void, ms: number) => real(callback, ms === 180_000 ? 5 : ms)) as typeof setTimeout);
    try {
      const result = await collectFreshnessSources(async url => new Response(freshnessFixture(FRESHNESS_SOURCES.find(source => source.url === url)!.id)), createWorker);
      expect(result.every(source => source.error && !source.receipt)).toBe(true);
    } finally { timers.mockRestore(); }
    expect(terminations).toBe(5);
  });
});

describe('trustworthy evidence and deterministic statuses', () => {
  test('source-check evidence is separate from catalog qualification and survives artifact expiry', () => {
    const report = assessFreshness(freshnessObservation());
    expect(validState(report.state)).toBe(true);
    expect(validEvidence(report.state.lastSuccess)).toBe(true);
    expect(report.state.lastSuccess?.nextCheckDueBy).toBe('2026-10-14T12:00:00.000Z');
    expect(report.state.lastSuccess?.sources).toHaveLength(4);
    expect(report.state.lastSuccess?.catalog).toEqual(freshnessObservation().catalog);
    expect(storedEvidenceStatus(freshnessObservation().catalog, JSON.parse(JSON.stringify(report.state.lastSuccess)), '2026-10-08T12:00:00.000Z')).toBe('current');
    expect(renderFreshnessReport(report)).toContain('not automatic liveness monitoring');
  });

  test('missing/corrupt/mismatched/future evidence is unknown; thirty-day staleness has an exact boundary', () => {
    const observation = freshnessObservation();
    const evidence = assessFreshness(observation).state.lastSuccess!;
    expect(storedEvidenceStatus(observation.catalog, null, observation.checkedAt)).toBe('unknown/source-unavailable');
    expect(storedEvidenceStatus({ ...observation.catalog, sha256: 'b'.repeat(64) }, evidence, observation.checkedAt)).toBe('unknown/source-unavailable');
    expect(storedEvidenceStatus(observation.catalog, { ...evidence, parserVersion: 99 }, observation.checkedAt)).toBe('unknown/source-unavailable');
    expect(storedEvidenceStatus(observation.catalog, evidence, '2026-10-06T12:00:00.000Z')).toBe('unknown/source-unavailable');
    expect(storedEvidenceStatus(observation.catalog, evidence, '2026-11-06T12:00:00.000Z')).toBe('current');
    expect(storedEvidenceStatus(observation.catalog, evidence, '2026-11-06T12:00:00.001Z')).toBe('stale');
  });

  test('partial recovery cannot refresh evidence or erase an unresolved retirement', () => {
    const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
    const next = assessFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'openai-lifecycle': null }), initial.state);
    expect(next.status).toBe('unknown/source-unavailable');
    expect(next.state.lastSuccess).toEqual(initial.state.lastSuccess);
    expect(next.state.lifecycle).toEqual(initial.state.lifecycle);
    const old = assessFreshness(freshnessObservation('102', '2026-11-08T12:00:00.000Z', { 'openai-lifecycle': null }), next.state);
    expect(old.evidenceStatus).toBe('stale');
    expect(old.status).toBe('unknown/source-unavailable');
    const recovered = assessFreshness(freshnessObservation('103', '2026-11-09T12:00:00.000Z'), old.state);
    expect(recovered.status).toBe('update-candidate');
    expect(recovered.state.lifecycle).toEqual(old.state.lifecycle);
  });

  test('corrupt duplicate lifecycle keys and source receipts cannot certify current', () => {
    const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
    expect(validLifecycle([initial.state.lifecycle[0], initial.state.lifecycle[0]])).toBe(false);
    const observation = freshnessObservation();
    observation.sources[0].receipt!.finalUrl = 'https://evil.example/docs';
    expect(assessFreshness(observation).status).toBe('unknown/source-unavailable');
    const evidence = structuredClone(initial.state.lastSuccess!);
    evidence.sources[0] = evidence.sources[1];
    expect(validEvidence(evidence)).toBe(false);
    expect(validState({ ...initial.state, lastSuccess: evidence })).toBe(false);
  });

  test('a partial run cannot downgrade an unresolved retirement to legacy or extend its deadline', () => {
    const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
    const legacy = freshnessFixture('anthropic-lifecycle').replace('claude-fable-5-1 | Active', 'claude-fable-5-1 | Legacy');
    const report = assessFreshness(freshnessObservation('101', '2026-10-08T12:00:00.000Z', { 'anthropic-lifecycle': legacy, 'openai-lifecycle': null }), initial.state);
    expect(report.state.lifecycle[0]).toMatchObject({ state: 'deprecated', deadline: '2026-11-01' });
    expect(report.state.lastSuccess).toEqual(initial.state.lastSuccess);
  });

  test('only a complete check of a reviewed replacement catalog resolves removed identities', () => {
    const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
    const replacement = freshnessObservation('101', '2026-10-08T12:00:00.000Z');
    replacement.catalog.models[0].modelId = 'claude-fable-5-2';
    replacement.catalog.sha256 = 'b'.repeat(64);
    replacement.sources = freshnessObservation('101', replacement.checkedAt, {
      'anthropic-models': freshnessFixture('anthropic-models').replaceAll('fable-5-1', 'fable-5-2'),
      'anthropic-lifecycle': freshnessFixture('anthropic-lifecycle').replaceAll('fable-5-1', 'fable-5-2'),
    }).sources;
    const incomplete = structuredClone(replacement);
    incomplete.sources[3] = { id: 'openai-lifecycle', error: 'http-503' };
    expect(assessFreshness(incomplete, initial.state).state.lifecycle[0].resolvedByCatalog).toBeUndefined();
    const complete = assessFreshness(replacement, initial.state);
    expect(complete.status).toBe('current');
    expect(complete.state.lifecycle[0].resolvedByCatalog).toBe('b'.repeat(64));
  });

  test('complete catalog rollback reopens the historical retirement even when the current source row is active', () => {
    const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
    const replacement = assessFreshness(freshnessReplacementObservation(), initial.state);
    expect(replacement.status).toBe('current');
    expect(replacement.state.lifecycle[0].resolvedByCatalog).toBe('b'.repeat(64));
    const rollback = assessFreshness(freshnessObservation('102', '2026-10-09T12:00:00.000Z'), replacement.state);
    expect(rollback.status).toBe('update-candidate');
    expect(rollback.state.lifecycle[0]).toMatchObject({ state: 'deprecated', deadline: '2026-11-01', firstObservedAt: initial.state.lifecycle[0].firstObservedAt });
    expect(rollback.state.lifecycle[0].resolvedByCatalog).toBeUndefined();
    expect(replacement.state.lifecycle[0].resolvedByCatalog).toBe('b'.repeat(64));
  });

  test('source-unavailable catalog rollback reopens history without refreshing prior replacement evidence', () => {
    const initial = assessFreshness(freshnessObservation('100', undefined, { 'anthropic-lifecycle': retirementFixture() }));
    const replacement = assessFreshness(freshnessReplacementObservation(), initial.state);
    const rollback = assessFreshness(freshnessObservation('102', '2026-10-09T12:00:00.000Z', { 'anthropic-lifecycle': null, 'openai-lifecycle': null }), replacement.state);
    expect(rollback.status).toBe('unknown/source-unavailable');
    expect(rollback.state.lastSuccess).toEqual(replacement.state.lastSuccess);
    expect(rollback.state.lifecycle[0]).toMatchObject({ state: 'deprecated', deadline: '2026-11-01' });
    expect(rollback.state.lifecycle[0].resolvedByCatalog).toBeUndefined();
  });
});