test: PTY harness handles clipped reviews and bundled setup tabs; AUQ judge uses structured output; design-consultation carve declines optional outside voices

- ceo mode routing: a Submit review taller than the viewport, a setup tab
  bundled after the mode tab, and a clip through the mode question each hung
  or misread the run; the native answer is still verified after Submit.
- judgeRecommendation requests a 1-5 enum schema; a malformed Haiku reply had
  scored substance 0 for a 4/5 brief. Judge failures now propagate.
- carve section-loading for design-consultation declines the optional outside
  voices (a supported path) and treats DESIGN.md as the report; timeout unchanged.
The Step 0E handoff defect is not fixed (0/15 samples across four wordings,
none shipped) and is filed in TODOS.
This commit is contained in:
garrytan committed 2026-09-30 12:52:56 +00:00
1 parent dfe5e733fb
commit a27365bffe
16 files changed
+893 -28

No files matched your search

+6 -2
View File
@@ -108,11 +108,15 @@ export function registerCarveSectionCase(skill: string): void {
? '- Proceed directly with the requested engineering review; skip the optional /office-hours prerequisite. You represent the plan author, whose scope and proposed steps are in PLAN.md. At each decision, choose the complete alternative that preserves those requirements and existing contracts; choose the recommended option only among alternatives within that scope. Do not authorize optional scope, extra public input guarantees, arbitrary size limits, or optional proof projects. Decline work explicitly listed out of scope, including creating TODOs for it. Record the decision and its actual authority as the skill requires, then continue without asking a human. A demonstrated incompatibility or missing required proof still requires resolution; do not hide it or claim approval when no offered alternative meets these constraints.'
: undefined,
// Both plan reviews persist their required report in the reviewed plan.
reportFile: ['plan-devex-review', 'plan-eng-review'].includes(guard.skill) ? 'PLAN.md' : undefined,
// design-consultation's final output is DESIGN.md itself; a second
// REPORT.md only duplicated the proposal (census 36641820398 timeout).
reportFile: ['plan-devex-review', 'plan-eng-review'].includes(guard.skill) ? 'PLAN.md'
: guard.skill === 'design-consultation' ? 'DESIGN.md' : undefined,
// This scenario produces an HTML implementation, whose complete
// document need not contain any of the prose report keywords.
reportMarker: guard.skill === 'design-html'
? /<!doctype\s+html\s*>\s*<html\b[^>]*>[\s\S]*?<head\b[^>]*>[\s\S]*?<\/head\s*>[\s\S]*?<body\b[^>]*>[\s\S]*?<\/body\s*>\s*<\/html\s*>/i
: guard.skill === 'design-consultation' ? /^# gstack: design-md-format=spec$/m
: /report|review|summary|design doc|handoff/i,
testName: `${guard.skill} section-loading`,
runId,
@@ -125,7 +129,7 @@ export function registerCarveSectionCase(skill: string): void {
});
// Require the HTML artifact itself; a terminal-only claim is insufficient.
// captureSectionReads already requires a successful native completion.
const reportProduced = completionMarked && (guard.skill !== 'design-html' || reportWritten);
const reportProduced = completionMarked && (!['design-html', 'design-consultation'].includes(guard.skill) || reportWritten);
const missing = guard.requiredReads.filter((s) => !readSections.has(s));
// Named failure output (codex #2): skill + expected + observed.