mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-04 15:57:33 +01:00
test(dx): preregistered behavior case for the agent-ready error contract; README skill rows
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
c85094e2c7
commit
7958b78377
5 files changed
+239
-2
No files matched your search
@@ -286,14 +286,14 @@ Each skill feeds into the next. `/office-hours` writes a design doc that `/plan-
|
||||
| `/plan-ceo-review` | **CEO / Founder** | Rethink the problem. Find the 10-star product hiding inside the request. Four modes: Expansion, Selective Expansion, Hold Scope, Reduction. |
|
||||
| `/plan-eng-review` | **Eng Manager** | Lock in architecture, data flow, diagrams, edge cases, and tests. Forces hidden assumptions into the open. |
|
||||
| `/plan-design-review` | **Senior Designer** | Rates each design dimension 0-10, explains what a 10 looks like, then edits the plan to get there. AI Slop detection. Interactive — one AskUserQuestion per design choice. |
|
||||
| `/plan-devex-review` | **Developer Experience Lead** | Interactive DX review: explores developer personas, benchmarks against competitors' TTHW, designs your magical moment, traces friction points step by step. Three modes: DX EXPANSION, DX POLISH, DX TRIAGE. 20-45 forcing questions. |
|
||||
| `/plan-devex-review` | **Developer Experience Lead** | Interactive DX review: explores developer personas, benchmarks against competitors' TTHW, designs your magical moment, traces friction points step by step. Scores error messages for the agents that read them ([agent-ready errors](docs/agent-ready-errors.md)). Three modes: DX EXPANSION, DX POLISH, DX TRIAGE. 20-45 forcing questions. |
|
||||
| `/design-consultation` | **Design Partner** | Build a complete design system from scratch. Researches the landscape, proposes creative risks, generates realistic product mockups. Writes `DESIGN.md` in the open DESIGN.md format, so impeccable, Google Stitch, and any tool that reads it share one file. |
|
||||
| `/review` | **Staff Engineer** | Find the bugs that pass CI but blow up in production. Auto-fixes the obvious ones. Flags completeness gaps and gate integrity: every weakened test, timeout, snapshot or suppression reaches a human by name. Advisory simplification lens flags over-built code — never blocks, never auto-applies. |
|
||||
| `/deslop-shared-libs` | **Shared Code Reviewer** | Find worthwhile shared-code extractions in recent work. Compares up to five opportunities and recommends the best three, with source evidence, reliability gains, and total code savings. Recommendations only. |
|
||||
| `/test-audit` | **Test Auditor** | Sweep existing tests for ones that cost more than they protect: source greps, duplicates, assertion-free probes, test-only exports. Every candidate carries an evidence card; report-only unless you approve a batch. |
|
||||
| `/investigate` | **Debugger** | Systematic root-cause debugging. Iron Law: no fixes without investigation. Traces data flow, tests hypotheses, stops after 3 failed fixes. |
|
||||
| `/design-review` | **Designer Who Codes** | Same audit as /plan-design-review, then fixes what it finds. Atomic commits, before/after screenshots. If you have impeccable installed, its engine runs first and every mechanical finding arrives tagged with its rule id. |
|
||||
| `/devex-review` | **DX Tester** | Live developer experience audit. Actually tests your onboarding: navigates docs, tries the getting started flow, times TTHW, screenshots errors. Compares against `/plan-devex-review` scores — the boomerang that shows if your plan matched reality. |
|
||||
| `/devex-review` | **DX Tester** | Live developer experience audit. Actually tests your onboarding: navigates docs, tries the getting started flow, times TTHW, triggers CLI errors the way an agent meets them (`gstack-error-capture`: no terminal, contained, redacted). Compares against `/plan-devex-review` scores — the boomerang that shows if your plan matched reality. |
|
||||
| `/design-shotgun` | **Design Explorer** | "Show me options." Generates 4-6 AI mockup variants, opens a comparison board in your browser, collects your feedback, and iterates. Taste memory learns what you like. Repeat until you love something, then hand it to `/design-html`. |
|
||||
| `/design-html` | **Design Engineer** | Turn a mockup into production HTML that actually works. Pretext computed layout: text reflows, heights adjust, layouts are dynamic. 30KB, zero deps. Detects React/Svelte/Vue. Smart API routing per design type (landing page vs dashboard vs form). One slop-gate pass through the impeccable engine when you have it. The output is shippable, not a demo. |
|
||||
| `/qa` | **QA Lead** | Explore browser, API, CLI, job and webhook behavior. Reproduce bugs, write failing regressions, fix the cause and re-verify before committing. |
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { DX_ERROR_CONTRACT_PLAN, gradeDxContractReview } from './fixtures/dx-error-contract-case';
|
||||
|
||||
const fence = (rows: Array<Record<string, unknown>>) => ['INPUT: dx abc', 'Review prose.', '```gstack-findings', ...rows.map(row => JSON.stringify(row)), '```'].join('\n');
|
||||
const PASSING = [
|
||||
{ id: 'F1', severity: 'Critical', title: 'CAP4:P1:redaction --debug prints the bearer token' },
|
||||
{ id: 'F2', severity: 'Critical', title: 'CAP4:P1:channel push failure exits 0 and blocks on a y/N prompt' },
|
||||
{ id: 'F3', severity: 'Critical', title: 'CAP4:P1:fix --force skips the overwrite confirmation' },
|
||||
{ id: 'F4', severity: 'High', title: 'CAP6:P1:context network error is swallowed' },
|
||||
{ id: 'F5', severity: 'High', title: 'CAP6:P5:fix migrate has no error handling' },
|
||||
{ id: 'F6', severity: 'Medium', title: 'Getting started takes four steps' },
|
||||
];
|
||||
|
||||
describe('dx-error-contract-review grader (free controls for the paid case)', () => {
|
||||
test('the fixture plan labels five paths and keeps the decoy complete', () => {
|
||||
for (const path of ['P1', 'P2', 'P3', 'P4', 'P5']) expect(DX_ERROR_CONTRACT_PLAN).toContain(`### Error path ${path} `);
|
||||
expect(DX_ERROR_CONTRACT_PLAN).toContain('## Implementation plan');
|
||||
expect(DX_ERROR_CONTRACT_PLAN).toContain('(PROJECT_NOT_FOUND); fix: run');
|
||||
});
|
||||
|
||||
test('a review meeting every bar passes', () => {
|
||||
const grade = gradeDxContractReview(fence(PASSING));
|
||||
expect(grade.reasons).toEqual([]);
|
||||
expect(grade.pass).toBe(true);
|
||||
});
|
||||
|
||||
test('command-named paths map to their labels', () => {
|
||||
const rows = PASSING.map(row => ({ ...row, title: String(row.title).replace(':P1:', ':acmesync push:').replace(':P5:', ':acmesync migrate:') }));
|
||||
expect(gradeDxContractReview(fence(rows)).pass).toBe(true);
|
||||
});
|
||||
|
||||
test.each([
|
||||
['decoy capped', [...PASSING, { id: 'F7', severity: 'High', title: 'CAP6:P2:trace no trace' }], 'P2 should not be capped'],
|
||||
['remote API penalized for trace', [...PASSING, { id: 'F7', severity: 'High', title: 'CAP6:P3:trace no stack trace' }], 'P3 should not be capped'],
|
||||
['usage error capped', [...PASSING, { id: 'F7', severity: 'High', title: 'CAP6:P4:context no versions' }], 'P4 should not be capped'],
|
||||
['secret leak missed', PASSING.filter(row => row.id !== 'F1'), 'P1 missing CAP4:redaction'],
|
||||
['unsafe fix missed', PASSING.filter(row => row.id !== 'F3'), 'P1 missing CAP4:fix'],
|
||||
['context gap missed', PASSING.filter(row => row.id !== 'F4'), 'P1 missing a CAP finding on context'],
|
||||
['silent path missed', PASSING.filter(row => row.id !== 'F5'), 'P5 (unspecified error handling) not capped'],
|
||||
['CAP4 under-rated', PASSING.map(row => row.id === 'F2' ? { ...row, severity: 'High' } : row), 'CAP4 labelled High, want Critical'],
|
||||
])('%s fails', (_name, rows, reason) => {
|
||||
const grade = gradeDxContractReview(fence(rows as Array<Record<string, unknown>>));
|
||||
expect(grade.pass).toBe(false);
|
||||
expect(grade.reasons.join('\n')).toContain(reason);
|
||||
});
|
||||
|
||||
test('a review with no findings fence fails', () => {
|
||||
const grade = gradeDxContractReview('INPUT: dx abc\nLooks fine to me.');
|
||||
expect(grade.pass).toBe(false);
|
||||
expect(grade.reasons[0]).toContain('no gstack-findings fence');
|
||||
});
|
||||
});
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
/**
|
||||
* Preregistered fixture and grader for the dx-error-contract-review behavior
|
||||
* case (plan v3 item 11, arm A: the snapshot-built autoplan native DX reviewer
|
||||
* prompt, run headless). The bars below were fixed before any paid trial and
|
||||
* are not tuned after seeing results.
|
||||
*
|
||||
* A trial PASSES when the review's gstack-findings fence:
|
||||
* - flags path P1 (agent-hostile push) with Critical CAP4 findings naming
|
||||
* redaction, channel and fix, and names context in some CAP finding on P1;
|
||||
* - raises no CAP finding on P2 (agent-ready decoy), P3 (remote API error with
|
||||
* caller-safe context and a correlation id) or P4 (plain usage error);
|
||||
* - raises at least one CAP finding on P5 (error handling never specified);
|
||||
* - labels every CAP4 finding Critical and every CAP6 finding High.
|
||||
*/
|
||||
import { parseRecordFindings } from '../../lib/autoplan-reconcile';
|
||||
|
||||
export const DX_ERROR_CONTRACT_PLAN = `# Plan: acmesync 2.0 CLI error handling
|
||||
|
||||
## Implementation plan
|
||||
|
||||
acmesync is a CLI that syncs a local project directory with the Acme hosted API. Version 2.0
|
||||
ships these error paths. Each is labelled so reviewers can refer to it.
|
||||
|
||||
### Error path P1 (acmesync push)
|
||||
When a push fails, print \`Error: push failed\` to stdout and exit 0, so CI scripts keep going.
|
||||
If the server returns 401, ask on the terminal: \`Retry with saved token? [y/N]\` and wait for an
|
||||
answer. Network errors are caught and reported as \`something went wrong\`; the underlying error is
|
||||
discarded. With \`--debug\`, print the full HTTP request including its Authorization header (the
|
||||
bearer token from ACME_TOKEN) so users can paste it into a bug report. On a conflict, print
|
||||
\`Run acmesync push --force to skip the conflict check\`; \`--force\` also skips the "overwrite remote
|
||||
files?" confirmation.
|
||||
|
||||
### Error path P2 (acmesync pull)
|
||||
A missing project prints to stderr and exits 4:
|
||||
\`acmesync pull: project 'web' not found in ./acme.toml (PROJECT_NOT_FOUND); fix: run 'acmesync projects list', then 'acmesync pull <name>'. Nothing was changed.\`
|
||||
Network failures print the operation, the underlying error, acmesync and API versions, the config
|
||||
file in use and the names of the env vars read, plus "safe to retry; nothing was written; retries
|
||||
at most 3 times with backoff, then stop". \`--json\` emits the same fields as an RFC 9457 object with
|
||||
\`code\`, \`fix\`, \`recovery: {action, side_effects}\` and a \`type\` URL to
|
||||
https://acme.example/errors#PROJECT_NOT_FOUND. Unexpected crashes write the stack trace to
|
||||
~/.acme/logs/crash-<timestamp>.log and print that path. Tokens are masked in every message.
|
||||
|
||||
### Error path P3 (GET /v1/projects/{id} on the Acme API)
|
||||
An unknown id returns HTTP 404 with an RFC 9457 body: \`type\`
|
||||
https://acme.example/errors#PROJECT_NOT_FOUND, \`title\` "Project not found", \`detail\` "No project
|
||||
with id prj_123", \`instance\` req_8f2c (the correlation id), \`code\` PROJECT_NOT_FOUND, and \`fix\`
|
||||
"List projects with GET /v1/projects". Server stack traces and internal hostnames stay in the
|
||||
server logs, keyed by the request id.
|
||||
|
||||
### Error path P4 (unknown flag)
|
||||
\`acmesync --colour\` prints \`acmesync: unknown flag --colour; did you mean --color? (USAGE)\` to
|
||||
stderr and exits 2 before doing anything.
|
||||
|
||||
### Error path P5 (acmesync migrate)
|
||||
\`acmesync migrate\` rewrites ./acme.toml from the v1 format to v2 in place.
|
||||
|
||||
## Review record
|
||||
`;
|
||||
|
||||
export type CapLevel = 4 | 6;
|
||||
export interface CapFinding { level: CapLevel; path: string; item: string; severity?: string; title: string }
|
||||
export interface DxContractGrade { pass: boolean; reasons: string[]; caps: CapFinding[] }
|
||||
|
||||
const ITEMS = new Set(['what', 'fix', 'recovery', 'code', 'context', 'trace', 'redaction', 'channel']);
|
||||
const ALIASES: Record<string, string> = { terminal: 'channel', exit: 'channel', 'exit-code': 'channel', secret: 'redaction' };
|
||||
|
||||
function pathId(raw: string): string | undefined {
|
||||
const label = /\bP([1-5])\b/i.exec(raw);
|
||||
if (label) return `P${label[1]}`;
|
||||
const text = raw.toLowerCase();
|
||||
if (text.includes('push')) return 'P1';
|
||||
if (text.includes('pull')) return 'P2';
|
||||
if (text.includes('/v1/projects') || text.includes('api')) return 'P3';
|
||||
if (text.includes('colour') || text.includes('flag') || text.includes('usage')) return 'P4';
|
||||
if (text.includes('migrate')) return 'P5';
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function capFindings(review: string): CapFinding[] {
|
||||
const caps: CapFinding[] = [];
|
||||
for (const finding of parseRecordFindings(review).findings) {
|
||||
const title = String(finding.title ?? '');
|
||||
const match = /^\s*CAP([46])\s*:\s*([^:]+?)\s*:\s*([A-Za-z_-]+)/.exec(title);
|
||||
if (!match) continue;
|
||||
const item = match[3]!.toLowerCase();
|
||||
caps.push({ level: Number(match[1]) as CapLevel, path: pathId(match[2]!) ?? match[2]!.trim(),
|
||||
item: ALIASES[item] ?? item, severity: finding.severity, title });
|
||||
}
|
||||
return caps;
|
||||
}
|
||||
|
||||
export function gradeDxContractReview(review: string): DxContractGrade {
|
||||
const reasons: string[] = [];
|
||||
const parsed = parseRecordFindings(review);
|
||||
if (parsed.source !== 'fence') reasons.push(`no gstack-findings fence (source=${parsed.source})`);
|
||||
const caps = capFindings(review);
|
||||
for (const cap of caps) {
|
||||
if (!ITEMS.has(cap.item)) reasons.push(`unknown item in "${cap.title}"`);
|
||||
const want = cap.level === 4 ? 'Critical' : 'High';
|
||||
if (cap.severity !== want) reasons.push(`CAP${cap.level} labelled ${cap.severity ?? 'none'}, want ${want}: "${cap.title}"`);
|
||||
}
|
||||
const p1 = caps.filter(cap => cap.path === 'P1');
|
||||
for (const item of ['redaction', 'channel', 'fix']) {
|
||||
if (!p1.some(cap => cap.level === 4 && cap.item === item)) reasons.push(`P1 missing CAP4:${item}`);
|
||||
}
|
||||
if (!p1.some(cap => cap.item === 'context')) reasons.push('P1 missing a CAP finding on context');
|
||||
for (const path of ['P2', 'P3', 'P4']) {
|
||||
const hits = caps.filter(cap => cap.path === path);
|
||||
if (hits.length) reasons.push(`${path} should not be capped: ${hits.map(cap => cap.title).join(' | ')}`);
|
||||
}
|
||||
if (!caps.some(cap => cap.path === 'P5')) reasons.push('P5 (unspecified error handling) not capped');
|
||||
return { pass: reasons.length === 0, reasons, caps };
|
||||
}
|
||||
@@ -1100,6 +1100,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
|
||||
'office-hours-section-loading': [ 'office-hours/**', 'bin/gstack-office-hours-review', 'lib/office-hours-review.ts', 'lib/fs-atomic.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/carve-guards.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/llm-judge.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-office-hours-section-loading.test.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/auq-native-capture.ts', 'test/helpers/auto-decision-state.ts', 'test/helpers/autoplan-artifact-digest.ts', 'test/helpers/autoplan-artifact-permission.ts', 'test/helpers/autoplan-artifact-recorder.ts', 'test/helpers/capture-parity-baseline.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty/**', 'test/helpers/dx-selected-navigation.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/eng-cache-writer-decision.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/helpers/native-auto-decide.ts', 'test/helpers/owned-claude-transcript.ts', 'test/helpers/parity-harness.ts', 'test/helpers/plan-count-artifacts.ts', 'test/helpers/plan-count-file-permission.ts', 'test/helpers/plan-count-fixture.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/helpers/plan-count-pending-question.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-floor-review.ts', 'test/helpers/plan-floor-target.ts', 'test/helpers/plan-scope-selection.ts', 'test/helpers/plan-seed-submission.ts', 'test/helpers/plan-skill-question-events.ts', 'test/helpers/plan-skill-question-hook-scope.ts', 'test/helpers/plan-skill-questions.ts', 'test/helpers/pty-screen.ts', 'test/helpers/pty-trust-dialog.ts', 'test/helpers/skill-census.ts'],
|
||||
'office-hours-design-draft': [ 'office-hours/**', 'lib/office-hours-review.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/carve-guards.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/llm-judge.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-office-hours-design-draft.test.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/auq-native-capture.ts', 'test/helpers/auto-decision-state.ts', 'test/helpers/autoplan-artifact-digest.ts', 'test/helpers/autoplan-artifact-permission.ts', 'test/helpers/autoplan-artifact-recorder.ts', 'test/helpers/capture-parity-baseline.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty/**', 'test/helpers/dx-selected-navigation.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/eng-cache-writer-decision.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/helpers/native-auto-decide.ts', 'test/helpers/owned-claude-transcript.ts', 'test/helpers/parity-harness.ts', 'test/helpers/plan-count-artifacts.ts', 'test/helpers/plan-count-file-permission.ts', 'test/helpers/plan-count-fixture.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/helpers/plan-count-pending-question.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-floor-review.ts', 'test/helpers/plan-floor-target.ts', 'test/helpers/plan-scope-selection.ts', 'test/helpers/plan-seed-submission.ts', 'test/helpers/plan-skill-question-events.ts', 'test/helpers/plan-skill-question-hook-scope.ts', 'test/helpers/plan-skill-questions.ts', 'test/helpers/pty-screen.ts', 'test/helpers/pty-trust-dialog.ts', 'test/helpers/skill-census.ts'],
|
||||
'dx-error-contract-review': ['test/skill-e2e-dx-error-contract.test.ts', 'test/fixtures/dx-error-contract-case.ts', 'lib/dx-error-contract.ts', 'scripts/resolvers/dx.ts', 'scripts/resolvers/index.ts', 'scripts/gen-skill-docs.ts', 'plan-devex-review/**', 'bin/gstack-autoplan-snapshot.ts', 'lib/autoplan-prompts.ts', 'lib/autoplan-reconcile.ts', 'lib/coordinator-contract.ts', 'test/helpers/session-runner.ts', 'test/helpers/eval-store.ts', 'test/helpers/eval-budgets.ts', 'lib/claude-bin.ts', 'lib/eval-model.ts'],
|
||||
'plan-devex-peer-comparison-classification': [
|
||||
|
||||
|
||||
@@ -1514,6 +1515,7 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic' | 'marathon'> = {
|
||||
'office-hours-design-draft': 'periodic', // Same interview through the design-creating Write (~5 min)
|
||||
'plan-decision-classification': 'periodic',
|
||||
'plan-devex-peer-comparison-classification': 'periodic',
|
||||
'dx-error-contract-review': 'periodic', // Native DX reviewer scores five seeded error paths against the agent-ready contract
|
||||
'health-reporting': 'periodic',
|
||||
'overlay-harness-claude-dedicated-tools-vs-bash': 'periodic',
|
||||
'overlay-harness-opus-4-7-effort-match-trivial': 'periodic',
|
||||
@@ -1877,6 +1879,7 @@ export const E2E_KINDS: Record<string, 'rule' | 'behavior' | 'judge'> = {
|
||||
'office-hours-design-draft': 'rule',
|
||||
'plan-decision-classification': 'rule',
|
||||
'plan-devex-peer-comparison-classification': 'rule',
|
||||
'dx-error-contract-review': 'behavior',
|
||||
'health-reporting': 'rule',
|
||||
'overlay-harness-claude-dedicated-tools-vs-bash': 'rule',
|
||||
'overlay-harness-opus-4-7-effort-match-trivial': 'rule',
|
||||
@@ -1962,6 +1965,8 @@ export const BEHAVIOR_WHY: Record<string, string> = {
|
||||
"How many scan passes the one-pass slop gate takes on a fake engine's fixed output is a judgment call.",
|
||||
'scrape-match-path':
|
||||
"The /scrape fallback no longer prescribes the browser-skills match flow, so taking it is prompt compliance.",
|
||||
'dx-error-contract-review':
|
||||
"Which contract items the live DX reviewer names on each seeded error path varies run to run; writing the bound result with a gstack-findings fence stays a contract.",
|
||||
'scrape-prototype-path':
|
||||
"The /scrape fallback no longer prescribes the prototype flow, so taking it is prompt compliance.",
|
||||
};
|
||||
@@ -0,0 +1,67 @@
|
||||
/**
|
||||
* Paid behavior case dx-error-contract-review (plan v3 item 11, arm A): the
|
||||
* snapshot-built autoplan native DX reviewer prompt, run headless against a
|
||||
* five-path fixture plan. The bars are preregistered in
|
||||
* test/fixtures/dx-error-contract-case.ts and replayed free by
|
||||
* test/dx-error-contract-grader.test.ts. One trial per run; the paid runner
|
||||
* supplies the 3-trial panel (E2E_KINDS 'behavior').
|
||||
*/
|
||||
import { afterAll, afterEach, beforeEach, expect } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { initializePlan, prepareMethodology, createSnapshot } from '../bin/gstack-autoplan-snapshot';
|
||||
import { detectScope, sectionLoadFor, writePrompts } from '../lib/autoplan-prompts';
|
||||
import { CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { expectContract } from './helpers/eval-store';
|
||||
import { ROOT, runId, logCost, recordE2E, createEvalCollector, finalizeEvalCollector, describeIfSelected, testIfSelected } from './helpers/e2e-helpers';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import { DX_ERROR_CONTRACT_PLAN, gradeDxContractReview } from './fixtures/dx-error-contract-case';
|
||||
|
||||
const collector = createEvalCollector('e2e-dx-error-contract');
|
||||
const NAME = 'dx error contract review';
|
||||
|
||||
describeIfSelected('DX error contract native reviewer (arm A)', ['dx-error-contract-review'], () => {
|
||||
let workDir = '';
|
||||
|
||||
beforeEach(() => { workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'dx-error-contract-')); });
|
||||
afterEach(() => { if (workDir) fs.rmSync(workDir, { recursive: true, force: true }); });
|
||||
afterAll(() => finalizeEvalCollector(collector));
|
||||
|
||||
testIfSelected('dx-error-contract-review', async () => {
|
||||
const source = path.join(workDir, 'plan.md');
|
||||
const active = path.join(workDir, 'active-plan.md');
|
||||
const restore = path.join(workDir, 'restore.md');
|
||||
fs.writeFileSync(source, DX_ERROR_CONTRACT_PLAN);
|
||||
initializePlan(source, active, restore);
|
||||
const methodology = prepareMethodology('dx', path.join(ROOT, 'plan-devex-review/SKILL.md'), restore);
|
||||
const snapshot = createSnapshot('dx', active, restore, methodology.methodologyPath);
|
||||
const outDir = path.join(workDir, 'out');
|
||||
fs.mkdirSync(outDir);
|
||||
const sections = sectionLoadFor('dx', detectScope(active, workDir, {}), 'light');
|
||||
const prompts = writePrompts({ phase: 'dx', snapshot, outDir, priorConsensus: [], sections });
|
||||
|
||||
const result = await runSkillTest({
|
||||
testName: 'dx-error-contract-review',
|
||||
workingDirectory: workDir,
|
||||
prompt: `Read ${JSON.stringify(prompts.native)} completely and carry out the review it describes. Write the result file it names.`,
|
||||
tools: ['Read', 'Write'],
|
||||
allowedTools: ['Read', 'Write'],
|
||||
maxTurns: 10,
|
||||
timeout: CAPTURE_MS,
|
||||
runId,
|
||||
});
|
||||
logCost(NAME, result);
|
||||
const review = fs.existsSync(prompts.nativeResult) ? fs.readFileSync(prompts.nativeResult, 'utf8') : '';
|
||||
const grade = gradeDxContractReview(review);
|
||||
console.log(`dx-error-contract-review caps: ${grade.caps.map(cap => `${cap.severity}:${cap.title}`).join(' | ') || 'none'}`);
|
||||
recordE2E(collector, NAME, 'DX error contract native reviewer', result, {
|
||||
passed: result.exitReason === 'success' && grade.pass,
|
||||
...(grade.reasons.length ? { error: grade.reasons.join('; ') } : {}),
|
||||
});
|
||||
expect(result.exitReason).toBe('success');
|
||||
expectContract(review.startsWith(`INPUT: dx ${snapshot.sha256}`) && review.includes('```gstack-findings'),
|
||||
'dx-error-contract-review: the reviewer did not write a bound result with a gstack-findings fence', { collector, name: NAME });
|
||||
expect(grade.reasons).toEqual([]);
|
||||
}, CAPTURE_MS + 30_000);
|
||||
});
|
||||
Reference in new issue
Block a user