gstack/test/skill-e2e-auq-consistency.test.ts

/**
 * AUQ consistency — same prompt, N runs, stable format + substance (periodic).
 *
 * The user's core anxiety: AUQ is fine one run and broken the next — sometimes
 * no ELI10, sometimes no recommendation, sometimes minimal context. A single
 * snapshot can't see drift. This drives the carved /plan-ceo-review mode-selection
 * AUQ N times via the native SDK tool capture (exact public fields, no answer) and asserts
 * the decision-brief format holds EVERY time and substance never craters.
 *
 * Pass bar:
 *   - Format: no element present in one run may be missing in another (that IS
 *     the inconsistency the user feels).
 *   - Substance: every run >= 3, spread (max-min) <= 2.
 *
 * Reports per-run scores so drift is visible even on a pass. Periodic tier
 * (N native SDK captures, ~$0.50-1 each).
 */
import { test } from 'bun:test';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs';
import {
  setupPlanCeoDir,
  captureModeSelectionAuq,
  AUQ_FORMAT_ELEMENTS,
  carvedSkill,
} from './helpers/auq-sdk-capture';
import { judgeRecommendation } from './helpers/llm-judge';

const describeE2E = describeE2ETier('periodic');
const N_RUNS = Number(process.env.AUQ_CONSISTENCY_RUNS ?? '3');
const runId = `auq-consistency-${process.env.EVALS_RUN_ID ?? 'local'}`;

describeE2E('AUQ consistency across runs (periodic)', () => {
  test(
    `carved /plan-ceo-review AUQ format + substance stable across ${N_RUNS} runs`,
    async () => {
      const caseDeadline = Date.now() + N_RUNS * CAPTURE_MS + 60_000;
      const runs: Array<{ i: number; present: Set<string>; substance: number; empty: boolean }> = [];
      const problems: string[] = [];
      const dirs: string[] = [];
      let captures: PromiseSettledResult<string>[];

      try {
        captures = await Promise.allSettled(Array.from({ length: N_RUNS }, async (_, i) => {
          const carved = carvedSkill();
          const dir = setupPlanCeoDir({
            skillMd: carved.skillMd,
            sectionsFrom: carved.sectionsFrom,
            tmpPrefix: `auq-consistency-${i}-`,
          });
          dirs.push(dir);
          return captureModeSelectionAuq({ planDir: dir, testName: `auq-consistency-${i}`, runId, caseDeadline });
        }));
      } finally {
        for (const dir of dirs) {
          try { fs.rmSync(dir, { recursive: true, force: true }); }
          catch (error) { problems.push(`fixture cleanup failed: ${error}`); }
        }
      }

      for (const [i, capture] of captures.entries()) {
        if (capture.status === 'rejected') problems.push(`run ${i + 1} capture failed: ${capture.reason}`);
        const text = capture.status === 'fulfilled' ? capture.value : '';
        const present = new Set(AUQ_FORMAT_ELEMENTS.filter(e => e.re.test(text)).map(e => e.field));
        let substance = 0;
        if (text.trim()) {
          try {
            substance = (await judgeRecommendation(text)).reason_substance;
          } catch { /* judge unavailable */ }
        }
        runs.push({ i, present, substance, empty: !text.trim() });
        // eslint-disable-next-line no-console
        console.log(
          `[AUQ-consistency run ${i + 1}/${N_RUNS}] present=${present.size}/${AUQ_FORMAT_ELEMENTS.length} ` +
            `missing=[${AUQ_FORMAT_ELEMENTS.filter(e => !present.has(e.field)).map(e => e.field).join(',')}] ` +
            `substance=${substance}${runs[i]?.empty ? ' (EMPTY CAPTURE)' : ''}`,
        );
      }

      const anyEmpty = runs.filter(r => r.empty).map(r => r.i + 1);
      if (anyEmpty.length > 0) problems.push(`run(s) produced no AUQ at all: ${anyEmpty.join(',')}`);

      // Inconsistency = an element present in SOME run but missing in another.
      const everPresent = new Set<string>();
      for (const r of runs) for (const f of r.present) everPresent.add(f);
      for (const f of everPresent) {
        const runsMissing = runs.filter(r => !r.present.has(f)).map(r => r.i + 1);
        if (runsMissing.length > 0) problems.push(`format element "${f}" missing in run(s) ${runsMissing.join(',')}`);
      }

      const subs = runs.map(r => r.substance);
      const minSub = Math.min(...subs);
      const maxSub = Math.max(...subs);
      if (minSub < 3) problems.push(`a run cratered: min substance ${minSub} < 3`);
      if (maxSub - minSub > 2) problems.push(`substance unstable: spread ${maxSub - minSub} > 2 (${subs.join(',')})`);

      if (problems.length > 0) {
        throw new Error(
          `AUQ inconsistency across ${N_RUNS} runs:\n` +
            problems.map(p => `  - ${p}`).join('\n') +
            `\nper-run: ` +
            runs.map(r => `[${r.i + 1}] fmt=${r.present.size}/${AUQ_FORMAT_ELEMENTS.length} sub=${r.substance}`).join(' '),
        );
      }

      // eslint-disable-next-line no-console
      console.log(
        `[AUQ-consistency] STABLE across ${N_RUNS} runs: all ${AUQ_FORMAT_ELEMENTS.length} ` +
          `format elements every run; substance ${minSub}-${maxSub}`,
      );
    },
    N_RUNS * CAPTURE_MS + 60_000,
  );
});