gstack/test/skill-e2e-auq-matrix.test.ts

/**
 * AUQ behavioral matrix — drive each AUQ-heavy skill to its first
 * AskUserQuestion and grade it to plan-ceo's bar (periodic, paid, native capture).
 *
 * Layer 0 (auq-format-always-loaded.test.ts) deterministically guarantees each
 * listed skill SHIPS the format spec in its always-loaded skeleton. This test
 * proves each skill's model OBEYS it: that the first real AUQ it fires carries
 * the fields software reads (a Recommendation: line and exactly one
 * (recommended) option) with a substantive recommendation (3-sample judge panel
 * mean >= 4 on the one capture; C4). The other
 * format elements are logged. One parametrized case per skill so a single weak skill
 * is an isolated failure, not a blocker for the rest.
 *
 * Capture records the actual public AskUserQuestion payload and verifies its
 * native display, with the skill pinned to an absolute path and only Read,
 * Write, and AskUserQuestion available. The exact fields are graded without
 * adding format labels or reading private reasoning. See
 * test/helpers/auq-native-capture.ts.
 *
 * Scope: skills whose first AUQ is reliably reachable from a text fixture. Skills
 * that gate their first decision on external resources (a running browser for
 * /qa, the design binary + comparison boards for /design-shotgun and
 * /design-html — which by project policy use $D compare, not AUQ, for variant
 * choices) are intentionally OUT of this matrix; Layer 0 covers their format
 * spec, and a fixture can't fairly trigger their AUQ.
 *
 * CSO is intentionally absent: its private startup omits the shared AUQ block,
 * and its dedicated E2E grades evidence, reporting, and proof behavior.
 * Run a subset in the foreground with AUQ_MATRIX_ONLY="plan-eng-review,spec".
 */
import { test } from 'bun:test';
import { resolveEvalModel } from '../lib/eval-model';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs';
import { captureNativeFirstAuq, type NativeAuqCapture } from './helpers/auq-native-capture';
import {
  setupSkillDir,
  scoreAuqFormat,
  auqMachineFormatProblems,
  skillFromWorktree,
} from './helpers/auq-sdk-capture';
import { auqSubstancePanel, AUQ_SUBSTANCE_MIN } from './helpers/auq-substance-panel';

const describeE2E = describeE2ETier('periodic');
const runId = `auq-matrix-${process.env.EVALS_RUN_ID ?? 'local'}`;
const ONLY = (process.env.AUQ_MATRIX_ONLY ?? '').split(',').map(s => s.trim()).filter(Boolean);

const FLAWED_PLAN = `# Plan: Launch a "developer-friendly" pricing tier

## Goal
Increase developer adoption.

## Success metric
More signups.

## Premise
We haven't talked to any developers about whether price is the barrier. The team
agreed it "feels like" it should be cheaper. We'll add a new Stripe tier, a React
pricing page, a Postgres entitlements table, and a Redis cache — no tests
mentioned, no rollout plan, no auth check on the upgrade endpoint.
`;

interface MatrixSkill {
  skill: string;
  fixtures: Record<string, string>;
  scenario: string;
  /** D1a regressor pin: explicit capture model when the Sonnet default measurably fails this entry. */
  model?: string;
}

const MATRIX: MatrixSkill[] = [
  {
    skill: 'plan-design-review',
    fixtures: { 'plan.md': FLAWED_PLAN + '\n## UI\nA new pricing page with a comparison table, plan cards, and an upgrade modal.\n' },
    scenario: 'Read plan.md — that is the plan to review (it has UI scope). Walk the review until the first AskUserQuestion.',
  },
  {
    skill: 'plan-devex-review',
    fixtures: { 'plan.md': FLAWED_PLAN + '\n## CLI\nShip a `mytool pricing` command and a setup wizard for the new tier.\n' },
    scenario: 'Read plan.md — that is the plan to review (developer-experience scope). Walk the review until the first AskUserQuestion.',
  },
  {
    skill: 'office-hours',
    fixtures: {},
    scenario: 'The founder says: "I am building an AI tool that auto-writes unit tests for any repo. I think it is a great idea but I have zero users. Should I build it, and how do I get my first users?" Run the office-hours diagnostic until the first AskUserQuestion.',
  },
  {
    skill: 'spec',
    fixtures: {},
    scenario: 'Turn this vague intent into a precise spec: "add email notifications when a task is assigned to someone." Walk the spec workflow until the first AskUserQuestion.',
    // D1a pin-on-regressors, with receipts (2026-08-16 re-baseline): under
    // the Sonnet capture default this entry failed twice ("never reached a
    // question in budget", 242s) while the six sibling entries passed; the
    // controlled Opus re-run passed cleanly (7/7 format, substance 5, 160s).
    // The spec workflow's long pre-question phase needs the stronger model
    // to reach its first AskUserQuestion inside the turn budget.
    model: resolveEvalModel('capture'),
  },
  {
    skill: 'design-consultation',
    fixtures: { 'product.md': '# Product\nA terminal-first task manager for developers. Audience: senior engineers. Stage: pre-launch.\n' },
    scenario: 'Read product.md. Run the design consultation for this product until the first AskUserQuestion.',
  },
];

const selected = ONLY.length ? MATRIX.filter(m => ONLY.includes(m.skill)) : MATRIX;

describeE2E('AUQ behavioral matrix (periodic)', () => {
  for (const m of selected) {
    test(
      `${m.skill}: first AUQ is a compliant decision brief (7/7 format, substance >=4)`,
      async () => {
        const wt = skillFromWorktree(m.skill);
        const dir = setupSkillDir({
          skillName: m.skill,
          skillMd: wt.skillMd,
          sectionsFrom: wt.sectionsFrom,
          fixtures: m.fixtures,
          tmpPrefix: `auq-matrix-${m.skill}-`,
        });
        let capture: NativeAuqCapture;
        try {
          capture = await captureNativeFirstAuq({
            planDir: dir,
            skillName: m.skill,
            scenario: m.scenario,
            testName: `auq-matrix-${m.skill}`,
            runId,
            model: m.model,
          });
        } finally {
          fs.rmSync(dir, { recursive: true, force: true });
        }

        const text = capture.text;
        const fmt = scoreAuqFormat(text);
        let substance = 0;
        let substanceMean = 0;
        let samples: number[] = [];
        let recPresent = false;
        let hadBecause = false;
        if (text.trim()) {
          const panel = await auqSubstancePanel(text);
          substance = panel.median;
          substanceMean = panel.mean;
          samples = panel.samples.map(g => g.substance);
          recPresent = panel.samples[0].present;
          hadBecause = panel.samples[0].hadLiteralBecause;
        }
        // eslint-disable-next-line no-console
        console.log(
          `[AUQ-matrix ${m.skill}] captured=${text.length}B format=${fmt.present}/${fmt.total} ` +
            `missing=[${fmt.missing.join(',')}] recPresent=${recPresent} substance median=${substance} mean=${substanceMean.toFixed(2)} [${samples.join(',')}] ` +
            `literalBecause=${hadBecause}`,
        );

        if (!text.trim()) {
          throw new Error(`${m.skill}: agent produced NO AUQ capture (never reached a question in budget).`);
        }
        const problems: string[] = [];
        // Presentation elements (ELI10, Pros / cons, ✅/❌, Net:) are logged above, not failed.
        problems.push(...auqMachineFormatProblems(capture.question));
        if (substance < AUQ_SUBSTANCE_MIN) {
          problems.push(`recommendation substance median ${substance} < ${AUQ_SUBSTANCE_MIN} over samples [${samples.join(',')}] (boilerplate/weak)`);
        }
        if (problems.length > 0) {
          throw new Error(
            `${m.skill} AUQ not at plan-ceo bar:\n  - ${problems.join('\n  - ')}\n--- captured AUQ ---\n${text}`,
          );
        }
      },
      CAPTURE_MS,
    );
  }
});