mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-19 14:44:44 +01:00
auqMachineFormatProblems() fails a first-question run when the Recommendation: line is missing or empty, or when the brief does not label exactly one option (recommended); recommendation substance >= 4 is unchanged. ELI10, Pros / cons, ✅/❌ and Net: are still scored and logged but no longer fail a paid case. scoreAuqFormat and the consistency suite are unchanged. Stored briefs: three known-good (including one without emoji or Net) pass; four known-bad (no or empty Recommendation, zero or two (recommended)) fail.
174 lines
7.4 KiB
TypeScript
174 lines
7.4 KiB
TypeScript
/**
|
|
* AUQ behavioral matrix — drive each AUQ-heavy skill to its first
|
|
* AskUserQuestion and grade it to plan-ceo's bar (periodic, paid, native capture).
|
|
*
|
|
* Layer 0 (auq-format-always-loaded.test.ts) deterministically guarantees each
|
|
* listed skill SHIPS the format spec in its always-loaded skeleton. This test
|
|
* proves each skill's model OBEYS it: that the first real AUQ it fires carries
|
|
* the fields software reads (a Recommendation: line and exactly one
|
|
* (recommended) option) with a substantive recommendation (>= 4). The other
|
|
* format elements are logged. One parametrized case per skill so a single weak skill
|
|
* is an isolated failure, not a blocker for the rest.
|
|
*
|
|
* Capture records the actual public AskUserQuestion payload and verifies its
|
|
* native display, with the skill pinned to an absolute path and only Read,
|
|
* Write, and AskUserQuestion available. The exact fields are graded without
|
|
* adding format labels or reading private reasoning. See
|
|
* test/helpers/auq-native-capture.ts.
|
|
*
|
|
* Scope: skills whose first AUQ is reliably reachable from a text fixture. Skills
|
|
* that gate their first decision on external resources (a running browser for
|
|
* /qa, the design binary + comparison boards for /design-shotgun and
|
|
* /design-html — which by project policy use $D compare, not AUQ, for variant
|
|
* choices) are intentionally OUT of this matrix; Layer 0 covers their format
|
|
* spec, and a fixture can't fairly trigger their AUQ.
|
|
*
|
|
* CSO is intentionally absent: its private startup omits the shared AUQ block,
|
|
* and its dedicated E2E grades evidence, reporting, and proof behavior.
|
|
* Run a subset in the foreground with AUQ_MATRIX_ONLY="plan-eng-review,spec".
|
|
*/
|
|
import { test } from 'bun:test';
|
|
import { resolveEvalModel } from '../lib/eval-model';
|
|
import { CAPTURE_MS } from './helpers/eval-budgets';
|
|
import { describeE2ETier } from './helpers/e2e-gate';
|
|
import * as fs from 'node:fs';
|
|
import {
|
|
setupSkillDir,
|
|
captureFirstAuq,
|
|
scoreAuqFormat,
|
|
auqMachineFormatProblems,
|
|
skillFromWorktree,
|
|
gradeAuqRecommendation,
|
|
} from './helpers/auq-sdk-capture';
|
|
|
|
const describeE2E = describeE2ETier('periodic');
|
|
const runId = `auq-matrix-${process.env.EVALS_RUN_ID ?? 'local'}`;
|
|
const ONLY = (process.env.AUQ_MATRIX_ONLY ?? '').split(',').map(s => s.trim()).filter(Boolean);
|
|
|
|
const FLAWED_PLAN = `# Plan: Launch a "developer-friendly" pricing tier
|
|
|
|
## Goal
|
|
Increase developer adoption.
|
|
|
|
## Success metric
|
|
More signups.
|
|
|
|
## Premise
|
|
We haven't talked to any developers about whether price is the barrier. The team
|
|
agreed it "feels like" it should be cheaper. We'll add a new Stripe tier, a React
|
|
pricing page, a Postgres entitlements table, and a Redis cache — no tests
|
|
mentioned, no rollout plan, no auth check on the upgrade endpoint.
|
|
`;
|
|
|
|
interface MatrixSkill {
|
|
skill: string;
|
|
fixtures: Record<string, string>;
|
|
scenario: string;
|
|
/** D1a regressor pin: explicit capture model when the Sonnet default measurably fails this entry. */
|
|
model?: string;
|
|
}
|
|
|
|
const MATRIX: MatrixSkill[] = [
|
|
{
|
|
skill: 'plan-eng-review',
|
|
fixtures: { 'plan.md': FLAWED_PLAN },
|
|
scenario: 'Read plan.md — that is the plan to review. It is a standalone plan document, not a codebase. Walk the review until the first AskUserQuestion (a per-issue finding or a scope decision).',
|
|
},
|
|
{
|
|
skill: 'plan-design-review',
|
|
fixtures: { 'plan.md': FLAWED_PLAN + '\n## UI\nA new pricing page with a comparison table, plan cards, and an upgrade modal.\n' },
|
|
scenario: 'Read plan.md — that is the plan to review (it has UI scope). Walk the review until the first AskUserQuestion.',
|
|
},
|
|
{
|
|
skill: 'plan-devex-review',
|
|
fixtures: { 'plan.md': FLAWED_PLAN + '\n## CLI\nShip a `mytool pricing` command and a setup wizard for the new tier.\n' },
|
|
scenario: 'Read plan.md — that is the plan to review (developer-experience scope). Walk the review until the first AskUserQuestion.',
|
|
},
|
|
{
|
|
skill: 'office-hours',
|
|
fixtures: {},
|
|
scenario: 'The founder says: "I am building an AI tool that auto-writes unit tests for any repo. I think it is a great idea but I have zero users. Should I build it, and how do I get my first users?" Run the office-hours diagnostic until the first AskUserQuestion.',
|
|
},
|
|
{
|
|
skill: 'spec',
|
|
fixtures: {},
|
|
scenario: 'Turn this vague intent into a precise spec: "add email notifications when a task is assigned to someone." Walk the spec workflow until the first AskUserQuestion.',
|
|
// D1a pin-on-regressors, with receipts (2026-08-16 re-baseline): under
|
|
// the Sonnet capture default this entry failed twice ("never reached a
|
|
// question in budget", 242s) while the six sibling entries passed; the
|
|
// controlled Opus re-run passed cleanly (7/7 format, substance 5, 160s).
|
|
// The spec workflow's long pre-question phase needs the stronger model
|
|
// to reach its first AskUserQuestion inside the turn budget.
|
|
model: resolveEvalModel('capture'),
|
|
},
|
|
{
|
|
skill: 'design-consultation',
|
|
fixtures: { 'product.md': '# Product\nA terminal-first task manager for developers. Audience: senior engineers. Stage: pre-launch.\n' },
|
|
scenario: 'Read product.md. Run the design consultation for this product until the first AskUserQuestion.',
|
|
},
|
|
];
|
|
|
|
const selected = ONLY.length ? MATRIX.filter(m => ONLY.includes(m.skill)) : MATRIX;
|
|
|
|
describeE2E('AUQ behavioral matrix (periodic)', () => {
|
|
for (const m of selected) {
|
|
test(
|
|
`${m.skill}: first AUQ is a compliant decision brief (7/7 format, substance >=4)`,
|
|
async () => {
|
|
const wt = skillFromWorktree(m.skill);
|
|
const dir = setupSkillDir({
|
|
skillName: m.skill,
|
|
skillMd: wt.skillMd,
|
|
sectionsFrom: wt.sectionsFrom,
|
|
fixtures: m.fixtures,
|
|
tmpPrefix: `auq-matrix-${m.skill}-`,
|
|
});
|
|
let text = '';
|
|
try {
|
|
text = await captureFirstAuq({
|
|
planDir: dir,
|
|
skillName: m.skill,
|
|
scenario: m.scenario,
|
|
testName: `auq-matrix-${m.skill}`,
|
|
runId,
|
|
model: m.model,
|
|
});
|
|
} finally {
|
|
fs.rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
|
|
const fmt = scoreAuqFormat(text);
|
|
let substance = 0;
|
|
let recPresent = false;
|
|
let hadBecause = false;
|
|
if (text.trim()) {
|
|
const g = await gradeAuqRecommendation(text);
|
|
substance = g.substance;
|
|
recPresent = g.present;
|
|
hadBecause = g.hadLiteralBecause;
|
|
}
|
|
// eslint-disable-next-line no-console
|
|
console.log(
|
|
`[AUQ-matrix ${m.skill}] captured=${text.length}B format=${fmt.present}/${fmt.total} ` +
|
|
`missing=[${fmt.missing.join(',')}] recPresent=${recPresent} substance=${substance} ` +
|
|
`literalBecause=${hadBecause}`,
|
|
);
|
|
|
|
if (!text.trim()) {
|
|
throw new Error(`${m.skill}: agent produced NO AUQ capture (never reached a question in budget).`);
|
|
}
|
|
const problems: string[] = [];
|
|
// Presentation elements (ELI10, Pros / cons, ✅/❌, Net:) are logged above, not failed.
|
|
problems.push(...auqMachineFormatProblems(text));
|
|
if (substance < 4) problems.push(`recommendation substance ${substance} < 4 (boilerplate/weak)`);
|
|
if (problems.length > 0) {
|
|
throw new Error(
|
|
`${m.skill} AUQ not at plan-ceo bar:\n - ${problems.join('\n - ')}\n--- captured AUQ ---\n${text}`,
|
|
);
|
|
}
|
|
},
|
|
CAPTURE_MS,
|
|
);
|
|
}
|
|
});
|