diff --git a/README.md b/README.md index b6de1e229..18c21f8d7 100644 --- a/README.md +++ b/README.md @@ -286,14 +286,14 @@ Each skill feeds into the next. `/office-hours` writes a design doc that `/plan- | `/plan-ceo-review` | **CEO / Founder** | Rethink the problem. Find the 10-star product hiding inside the request. Four modes: Expansion, Selective Expansion, Hold Scope, Reduction. | | `/plan-eng-review` | **Eng Manager** | Lock in architecture, data flow, diagrams, edge cases, and tests. Forces hidden assumptions into the open. | | `/plan-design-review` | **Senior Designer** | Rates each design dimension 0-10, explains what a 10 looks like, then edits the plan to get there. AI Slop detection. Interactive — one AskUserQuestion per design choice. | -| `/plan-devex-review` | **Developer Experience Lead** | Interactive DX review: explores developer personas, benchmarks against competitors' TTHW, designs your magical moment, traces friction points step by step. Three modes: DX EXPANSION, DX POLISH, DX TRIAGE. 20-45 forcing questions. | +| `/plan-devex-review` | **Developer Experience Lead** | Interactive DX review: explores developer personas, benchmarks against competitors' TTHW, designs your magical moment, traces friction points step by step. Scores error messages for the agents that read them ([agent-ready errors](docs/agent-ready-errors.md)). Three modes: DX EXPANSION, DX POLISH, DX TRIAGE. 20-45 forcing questions. | | `/design-consultation` | **Design Partner** | Build a complete design system from scratch. Researches the landscape, proposes creative risks, generates realistic product mockups. Writes `DESIGN.md` in the open DESIGN.md format, so impeccable, Google Stitch, and any tool that reads it share one file. | | `/review` | **Staff Engineer** | Find the bugs that pass CI but blow up in production. Auto-fixes the obvious ones. Flags completeness gaps and gate integrity: every weakened test, timeout, snapshot or suppression reaches a human by name. Advisory simplification lens flags over-built code — never blocks, never auto-applies. | | `/deslop-shared-libs` | **Shared Code Reviewer** | Find worthwhile shared-code extractions in recent work. Compares up to five opportunities and recommends the best three, with source evidence, reliability gains, and total code savings. Recommendations only. | | `/test-audit` | **Test Auditor** | Sweep existing tests for ones that cost more than they protect: source greps, duplicates, assertion-free probes, test-only exports. Every candidate carries an evidence card; report-only unless you approve a batch. | | `/investigate` | **Debugger** | Systematic root-cause debugging. Iron Law: no fixes without investigation. Traces data flow, tests hypotheses, stops after 3 failed fixes. | | `/design-review` | **Designer Who Codes** | Same audit as /plan-design-review, then fixes what it finds. Atomic commits, before/after screenshots. If you have impeccable installed, its engine runs first and every mechanical finding arrives tagged with its rule id. | -| `/devex-review` | **DX Tester** | Live developer experience audit. Actually tests your onboarding: navigates docs, tries the getting started flow, times TTHW, screenshots errors. Compares against `/plan-devex-review` scores — the boomerang that shows if your plan matched reality. | +| `/devex-review` | **DX Tester** | Live developer experience audit. Actually tests your onboarding: navigates docs, tries the getting started flow, times TTHW, triggers CLI errors the way an agent meets them (`gstack-error-capture`: no terminal, contained, redacted). Compares against `/plan-devex-review` scores — the boomerang that shows if your plan matched reality. | | `/design-shotgun` | **Design Explorer** | "Show me options." Generates 4-6 AI mockup variants, opens a comparison board in your browser, collects your feedback, and iterates. Taste memory learns what you like. Repeat until you love something, then hand it to `/design-html`. | | `/design-html` | **Design Engineer** | Turn a mockup into production HTML that actually works. Pretext computed layout: text reflows, heights adjust, layouts are dynamic. 30KB, zero deps. Detects React/Svelte/Vue. Smart API routing per design type (landing page vs dashboard vs form). One slop-gate pass through the impeccable engine when you have it. The output is shippable, not a demo. | | `/qa` | **QA Lead** | Explore browser, API, CLI, job and webhook behavior. Reproduce bugs, write failing regressions, fix the cause and re-verify before committing. | diff --git a/test/dx-error-contract-grader.test.ts b/test/dx-error-contract-grader.test.ts new file mode 100644 index 000000000..2f911ecf7 --- /dev/null +++ b/test/dx-error-contract-grader.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, test } from 'bun:test'; +import { DX_ERROR_CONTRACT_PLAN, gradeDxContractReview } from './fixtures/dx-error-contract-case'; + +const fence = (rows: Array>) => ['INPUT: dx abc', 'Review prose.', '```gstack-findings', ...rows.map(row => JSON.stringify(row)), '```'].join('\n'); +const PASSING = [ + { id: 'F1', severity: 'Critical', title: 'CAP4:P1:redaction --debug prints the bearer token' }, + { id: 'F2', severity: 'Critical', title: 'CAP4:P1:channel push failure exits 0 and blocks on a y/N prompt' }, + { id: 'F3', severity: 'Critical', title: 'CAP4:P1:fix --force skips the overwrite confirmation' }, + { id: 'F4', severity: 'High', title: 'CAP6:P1:context network error is swallowed' }, + { id: 'F5', severity: 'High', title: 'CAP6:P5:fix migrate has no error handling' }, + { id: 'F6', severity: 'Medium', title: 'Getting started takes four steps' }, +]; + +describe('dx-error-contract-review grader (free controls for the paid case)', () => { + test('the fixture plan labels five paths and keeps the decoy complete', () => { + for (const path of ['P1', 'P2', 'P3', 'P4', 'P5']) expect(DX_ERROR_CONTRACT_PLAN).toContain(`### Error path ${path} `); + expect(DX_ERROR_CONTRACT_PLAN).toContain('## Implementation plan'); + expect(DX_ERROR_CONTRACT_PLAN).toContain('(PROJECT_NOT_FOUND); fix: run'); + }); + + test('a review meeting every bar passes', () => { + const grade = gradeDxContractReview(fence(PASSING)); + expect(grade.reasons).toEqual([]); + expect(grade.pass).toBe(true); + }); + + test('command-named paths map to their labels', () => { + const rows = PASSING.map(row => ({ ...row, title: String(row.title).replace(':P1:', ':acmesync push:').replace(':P5:', ':acmesync migrate:') })); + expect(gradeDxContractReview(fence(rows)).pass).toBe(true); + }); + + test.each([ + ['decoy capped', [...PASSING, { id: 'F7', severity: 'High', title: 'CAP6:P2:trace no trace' }], 'P2 should not be capped'], + ['remote API penalized for trace', [...PASSING, { id: 'F7', severity: 'High', title: 'CAP6:P3:trace no stack trace' }], 'P3 should not be capped'], + ['usage error capped', [...PASSING, { id: 'F7', severity: 'High', title: 'CAP6:P4:context no versions' }], 'P4 should not be capped'], + ['secret leak missed', PASSING.filter(row => row.id !== 'F1'), 'P1 missing CAP4:redaction'], + ['unsafe fix missed', PASSING.filter(row => row.id !== 'F3'), 'P1 missing CAP4:fix'], + ['context gap missed', PASSING.filter(row => row.id !== 'F4'), 'P1 missing a CAP finding on context'], + ['silent path missed', PASSING.filter(row => row.id !== 'F5'), 'P5 (unspecified error handling) not capped'], + ['CAP4 under-rated', PASSING.map(row => row.id === 'F2' ? { ...row, severity: 'High' } : row), 'CAP4 labelled High, want Critical'], + ])('%s fails', (_name, rows, reason) => { + const grade = gradeDxContractReview(fence(rows as Array>)); + expect(grade.pass).toBe(false); + expect(grade.reasons.join('\n')).toContain(reason); + }); + + test('a review with no findings fence fails', () => { + const grade = gradeDxContractReview('INPUT: dx abc\nLooks fine to me.'); + expect(grade.pass).toBe(false); + expect(grade.reasons[0]).toContain('no gstack-findings fence'); + }); +}); diff --git a/test/fixtures/dx-error-contract-case.ts b/test/fixtures/dx-error-contract-case.ts new file mode 100644 index 000000000..552fc4fd9 --- /dev/null +++ b/test/fixtures/dx-error-contract-case.ts @@ -0,0 +1,113 @@ +/** + * Preregistered fixture and grader for the dx-error-contract-review behavior + * case (plan v3 item 11, arm A: the snapshot-built autoplan native DX reviewer + * prompt, run headless). The bars below were fixed before any paid trial and + * are not tuned after seeing results. + * + * A trial PASSES when the review's gstack-findings fence: + * - flags path P1 (agent-hostile push) with Critical CAP4 findings naming + * redaction, channel and fix, and names context in some CAP finding on P1; + * - raises no CAP finding on P2 (agent-ready decoy), P3 (remote API error with + * caller-safe context and a correlation id) or P4 (plain usage error); + * - raises at least one CAP finding on P5 (error handling never specified); + * - labels every CAP4 finding Critical and every CAP6 finding High. + */ +import { parseRecordFindings } from '../../lib/autoplan-reconcile'; + +export const DX_ERROR_CONTRACT_PLAN = `# Plan: acmesync 2.0 CLI error handling + +## Implementation plan + +acmesync is a CLI that syncs a local project directory with the Acme hosted API. Version 2.0 +ships these error paths. Each is labelled so reviewers can refer to it. + +### Error path P1 (acmesync push) +When a push fails, print \`Error: push failed\` to stdout and exit 0, so CI scripts keep going. +If the server returns 401, ask on the terminal: \`Retry with saved token? [y/N]\` and wait for an +answer. Network errors are caught and reported as \`something went wrong\`; the underlying error is +discarded. With \`--debug\`, print the full HTTP request including its Authorization header (the +bearer token from ACME_TOKEN) so users can paste it into a bug report. On a conflict, print +\`Run acmesync push --force to skip the conflict check\`; \`--force\` also skips the "overwrite remote +files?" confirmation. + +### Error path P2 (acmesync pull) +A missing project prints to stderr and exits 4: +\`acmesync pull: project 'web' not found in ./acme.toml (PROJECT_NOT_FOUND); fix: run 'acmesync projects list', then 'acmesync pull '. Nothing was changed.\` +Network failures print the operation, the underlying error, acmesync and API versions, the config +file in use and the names of the env vars read, plus "safe to retry; nothing was written; retries +at most 3 times with backoff, then stop". \`--json\` emits the same fields as an RFC 9457 object with +\`code\`, \`fix\`, \`recovery: {action, side_effects}\` and a \`type\` URL to +https://acme.example/errors#PROJECT_NOT_FOUND. Unexpected crashes write the stack trace to +~/.acme/logs/crash-.log and print that path. Tokens are masked in every message. + +### Error path P3 (GET /v1/projects/{id} on the Acme API) +An unknown id returns HTTP 404 with an RFC 9457 body: \`type\` +https://acme.example/errors#PROJECT_NOT_FOUND, \`title\` "Project not found", \`detail\` "No project +with id prj_123", \`instance\` req_8f2c (the correlation id), \`code\` PROJECT_NOT_FOUND, and \`fix\` +"List projects with GET /v1/projects". Server stack traces and internal hostnames stay in the +server logs, keyed by the request id. + +### Error path P4 (unknown flag) +\`acmesync --colour\` prints \`acmesync: unknown flag --colour; did you mean --color? (USAGE)\` to +stderr and exits 2 before doing anything. + +### Error path P5 (acmesync migrate) +\`acmesync migrate\` rewrites ./acme.toml from the v1 format to v2 in place. + +## Review record +`; + +export type CapLevel = 4 | 6; +export interface CapFinding { level: CapLevel; path: string; item: string; severity?: string; title: string } +export interface DxContractGrade { pass: boolean; reasons: string[]; caps: CapFinding[] } + +const ITEMS = new Set(['what', 'fix', 'recovery', 'code', 'context', 'trace', 'redaction', 'channel']); +const ALIASES: Record = { terminal: 'channel', exit: 'channel', 'exit-code': 'channel', secret: 'redaction' }; + +function pathId(raw: string): string | undefined { + const label = /\bP([1-5])\b/i.exec(raw); + if (label) return `P${label[1]}`; + const text = raw.toLowerCase(); + if (text.includes('push')) return 'P1'; + if (text.includes('pull')) return 'P2'; + if (text.includes('/v1/projects') || text.includes('api')) return 'P3'; + if (text.includes('colour') || text.includes('flag') || text.includes('usage')) return 'P4'; + if (text.includes('migrate')) return 'P5'; + return undefined; +} + +export function capFindings(review: string): CapFinding[] { + const caps: CapFinding[] = []; + for (const finding of parseRecordFindings(review).findings) { + const title = String(finding.title ?? ''); + const match = /^\s*CAP([46])\s*:\s*([^:]+?)\s*:\s*([A-Za-z_-]+)/.exec(title); + if (!match) continue; + const item = match[3]!.toLowerCase(); + caps.push({ level: Number(match[1]) as CapLevel, path: pathId(match[2]!) ?? match[2]!.trim(), + item: ALIASES[item] ?? item, severity: finding.severity, title }); + } + return caps; +} + +export function gradeDxContractReview(review: string): DxContractGrade { + const reasons: string[] = []; + const parsed = parseRecordFindings(review); + if (parsed.source !== 'fence') reasons.push(`no gstack-findings fence (source=${parsed.source})`); + const caps = capFindings(review); + for (const cap of caps) { + if (!ITEMS.has(cap.item)) reasons.push(`unknown item in "${cap.title}"`); + const want = cap.level === 4 ? 'Critical' : 'High'; + if (cap.severity !== want) reasons.push(`CAP${cap.level} labelled ${cap.severity ?? 'none'}, want ${want}: "${cap.title}"`); + } + const p1 = caps.filter(cap => cap.path === 'P1'); + for (const item of ['redaction', 'channel', 'fix']) { + if (!p1.some(cap => cap.level === 4 && cap.item === item)) reasons.push(`P1 missing CAP4:${item}`); + } + if (!p1.some(cap => cap.item === 'context')) reasons.push('P1 missing a CAP finding on context'); + for (const path of ['P2', 'P3', 'P4']) { + const hits = caps.filter(cap => cap.path === path); + if (hits.length) reasons.push(`${path} should not be capped: ${hits.map(cap => cap.title).join(' | ')}`); + } + if (!caps.some(cap => cap.path === 'P5')) reasons.push('P5 (unspecified error handling) not capped'); + return { pass: reasons.length === 0, reasons, caps }; +} diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index 3f3e54830..4fe2d1ab6 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -1100,6 +1100,7 @@ export const E2E_TOUCHFILES: Record = { 'office-hours-section-loading': [ 'office-hours/**', 'bin/gstack-office-hours-review', 'lib/office-hours-review.ts', 'lib/fs-atomic.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/carve-guards.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/llm-judge.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-office-hours-section-loading.test.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/auq-native-capture.ts', 'test/helpers/auto-decision-state.ts', 'test/helpers/autoplan-artifact-digest.ts', 'test/helpers/autoplan-artifact-permission.ts', 'test/helpers/autoplan-artifact-recorder.ts', 'test/helpers/capture-parity-baseline.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty/**', 'test/helpers/dx-selected-navigation.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/eng-cache-writer-decision.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/helpers/native-auto-decide.ts', 'test/helpers/owned-claude-transcript.ts', 'test/helpers/parity-harness.ts', 'test/helpers/plan-count-artifacts.ts', 'test/helpers/plan-count-file-permission.ts', 'test/helpers/plan-count-fixture.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/helpers/plan-count-pending-question.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-floor-review.ts', 'test/helpers/plan-floor-target.ts', 'test/helpers/plan-scope-selection.ts', 'test/helpers/plan-seed-submission.ts', 'test/helpers/plan-skill-question-events.ts', 'test/helpers/plan-skill-question-hook-scope.ts', 'test/helpers/plan-skill-questions.ts', 'test/helpers/pty-screen.ts', 'test/helpers/pty-trust-dialog.ts', 'test/helpers/skill-census.ts'], 'office-hours-design-draft': [ 'office-hours/**', 'lib/office-hours-review.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/carve-guards.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/llm-judge.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-office-hours-design-draft.test.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/auq-native-capture.ts', 'test/helpers/auto-decision-state.ts', 'test/helpers/autoplan-artifact-digest.ts', 'test/helpers/autoplan-artifact-permission.ts', 'test/helpers/autoplan-artifact-recorder.ts', 'test/helpers/capture-parity-baseline.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty/**', 'test/helpers/dx-selected-navigation.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/eng-cache-writer-decision.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/helpers/native-auto-decide.ts', 'test/helpers/owned-claude-transcript.ts', 'test/helpers/parity-harness.ts', 'test/helpers/plan-count-artifacts.ts', 'test/helpers/plan-count-file-permission.ts', 'test/helpers/plan-count-fixture.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/helpers/plan-count-pending-question.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-floor-review.ts', 'test/helpers/plan-floor-target.ts', 'test/helpers/plan-scope-selection.ts', 'test/helpers/plan-seed-submission.ts', 'test/helpers/plan-skill-question-events.ts', 'test/helpers/plan-skill-question-hook-scope.ts', 'test/helpers/plan-skill-questions.ts', 'test/helpers/pty-screen.ts', 'test/helpers/pty-trust-dialog.ts', 'test/helpers/skill-census.ts'], + 'dx-error-contract-review': ['test/skill-e2e-dx-error-contract.test.ts', 'test/fixtures/dx-error-contract-case.ts', 'lib/dx-error-contract.ts', 'scripts/resolvers/dx.ts', 'scripts/resolvers/index.ts', 'scripts/gen-skill-docs.ts', 'plan-devex-review/**', 'bin/gstack-autoplan-snapshot.ts', 'lib/autoplan-prompts.ts', 'lib/autoplan-reconcile.ts', 'lib/coordinator-contract.ts', 'test/helpers/session-runner.ts', 'test/helpers/eval-store.ts', 'test/helpers/eval-budgets.ts', 'lib/claude-bin.ts', 'lib/eval-model.ts'], 'plan-devex-peer-comparison-classification': [ @@ -1514,6 +1515,7 @@ export const E2E_TIERS: Record = { 'office-hours-design-draft': 'periodic', // Same interview through the design-creating Write (~5 min) 'plan-decision-classification': 'periodic', 'plan-devex-peer-comparison-classification': 'periodic', + 'dx-error-contract-review': 'periodic', // Native DX reviewer scores five seeded error paths against the agent-ready contract 'health-reporting': 'periodic', 'overlay-harness-claude-dedicated-tools-vs-bash': 'periodic', 'overlay-harness-opus-4-7-effort-match-trivial': 'periodic', @@ -1877,6 +1879,7 @@ export const E2E_KINDS: Record = { 'office-hours-design-draft': 'rule', 'plan-decision-classification': 'rule', 'plan-devex-peer-comparison-classification': 'rule', + 'dx-error-contract-review': 'behavior', 'health-reporting': 'rule', 'overlay-harness-claude-dedicated-tools-vs-bash': 'rule', 'overlay-harness-opus-4-7-effort-match-trivial': 'rule', @@ -1962,6 +1965,8 @@ export const BEHAVIOR_WHY: Record = { "How many scan passes the one-pass slop gate takes on a fake engine's fixed output is a judgment call.", 'scrape-match-path': "The /scrape fallback no longer prescribes the browser-skills match flow, so taking it is prompt compliance.", + 'dx-error-contract-review': + "Which contract items the live DX reviewer names on each seeded error path varies run to run; writing the bound result with a gstack-findings fence stays a contract.", 'scrape-prototype-path': "The /scrape fallback no longer prescribes the prototype flow, so taking it is prompt compliance.", }; diff --git a/test/skill-e2e-dx-error-contract.test.ts b/test/skill-e2e-dx-error-contract.test.ts new file mode 100644 index 000000000..1dc22be4a --- /dev/null +++ b/test/skill-e2e-dx-error-contract.test.ts @@ -0,0 +1,67 @@ +/** + * Paid behavior case dx-error-contract-review (plan v3 item 11, arm A): the + * snapshot-built autoplan native DX reviewer prompt, run headless against a + * five-path fixture plan. The bars are preregistered in + * test/fixtures/dx-error-contract-case.ts and replayed free by + * test/dx-error-contract-grader.test.ts. One trial per run; the paid runner + * supplies the 3-trial panel (E2E_KINDS 'behavior'). + */ +import { afterAll, afterEach, beforeEach, expect } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { initializePlan, prepareMethodology, createSnapshot } from '../bin/gstack-autoplan-snapshot'; +import { detectScope, sectionLoadFor, writePrompts } from '../lib/autoplan-prompts'; +import { CAPTURE_MS } from './helpers/eval-budgets'; +import { expectContract } from './helpers/eval-store'; +import { ROOT, runId, logCost, recordE2E, createEvalCollector, finalizeEvalCollector, describeIfSelected, testIfSelected } from './helpers/e2e-helpers'; +import { runSkillTest } from './helpers/session-runner'; +import { DX_ERROR_CONTRACT_PLAN, gradeDxContractReview } from './fixtures/dx-error-contract-case'; + +const collector = createEvalCollector('e2e-dx-error-contract'); +const NAME = 'dx error contract review'; + +describeIfSelected('DX error contract native reviewer (arm A)', ['dx-error-contract-review'], () => { + let workDir = ''; + + beforeEach(() => { workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'dx-error-contract-')); }); + afterEach(() => { if (workDir) fs.rmSync(workDir, { recursive: true, force: true }); }); + afterAll(() => finalizeEvalCollector(collector)); + + testIfSelected('dx-error-contract-review', async () => { + const source = path.join(workDir, 'plan.md'); + const active = path.join(workDir, 'active-plan.md'); + const restore = path.join(workDir, 'restore.md'); + fs.writeFileSync(source, DX_ERROR_CONTRACT_PLAN); + initializePlan(source, active, restore); + const methodology = prepareMethodology('dx', path.join(ROOT, 'plan-devex-review/SKILL.md'), restore); + const snapshot = createSnapshot('dx', active, restore, methodology.methodologyPath); + const outDir = path.join(workDir, 'out'); + fs.mkdirSync(outDir); + const sections = sectionLoadFor('dx', detectScope(active, workDir, {}), 'light'); + const prompts = writePrompts({ phase: 'dx', snapshot, outDir, priorConsensus: [], sections }); + + const result = await runSkillTest({ + testName: 'dx-error-contract-review', + workingDirectory: workDir, + prompt: `Read ${JSON.stringify(prompts.native)} completely and carry out the review it describes. Write the result file it names.`, + tools: ['Read', 'Write'], + allowedTools: ['Read', 'Write'], + maxTurns: 10, + timeout: CAPTURE_MS, + runId, + }); + logCost(NAME, result); + const review = fs.existsSync(prompts.nativeResult) ? fs.readFileSync(prompts.nativeResult, 'utf8') : ''; + const grade = gradeDxContractReview(review); + console.log(`dx-error-contract-review caps: ${grade.caps.map(cap => `${cap.severity}:${cap.title}`).join(' | ') || 'none'}`); + recordE2E(collector, NAME, 'DX error contract native reviewer', result, { + passed: result.exitReason === 'success' && grade.pass, + ...(grade.reasons.length ? { error: grade.reasons.join('; ') } : {}), + }); + expect(result.exitReason).toBe('success'); + expectContract(review.startsWith(`INPUT: dx ${snapshot.sha256}`) && review.includes('```gstack-findings'), + 'dx-error-contract-review: the reviewer did not write a bound result with a gstack-findings fence', { collector, name: NAME }); + expect(grade.reasons).toEqual([]); + }, CAPTURE_MS + 30_000); +});