gstack/test/skill-e2e-dx-error-contract.test.ts

/**
 * Paid behavior case dx-error-contract-review (plan v3 item 11, arm A): the
 * snapshot-built autoplan native DX reviewer prompt, run headless against a
 * five-path fixture plan. The bars are preregistered in
 * test/fixtures/dx-error-contract-case.ts and replayed free by
 * test/dx-error-contract-grader.test.ts. One trial per run; the paid runner
 * supplies the 3-trial panel (E2E_KINDS 'behavior').
 */
import { afterAll, afterEach, beforeEach, expect } from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { initializePlan, prepareMethodology, createSnapshot } from '../bin/gstack-autoplan-snapshot';
import { detectScope, sectionLoadFor, writePrompts } from '../lib/autoplan-prompts';
import { CAPTURE_MS } from './helpers/eval-budgets';
import { expectContract } from './helpers/eval-store';
import { ROOT, runId, logCost, recordE2E, createEvalCollector, finalizeEvalCollector, describeIfSelected, testIfSelected } from './helpers/e2e-helpers';
import { runSkillTest } from './helpers/session-runner';
import { DX_ERROR_CONTRACT_PLAN, gradeDxContractReview } from './fixtures/dx-error-contract-case';

const collector = createEvalCollector('e2e-dx-error-contract');
const NAME = 'dx error contract review';

describeIfSelected('DX error contract native reviewer (arm A)', ['dx-error-contract-review'], () => {
  let workDir = '';

  beforeEach(() => { workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'dx-error-contract-')); });
  afterEach(() => { if (workDir) fs.rmSync(workDir, { recursive: true, force: true }); });
  afterAll(() => finalizeEvalCollector(collector));

  testIfSelected('dx-error-contract-review', async () => {
    const source = path.join(workDir, 'plan.md');
    const active = path.join(workDir, 'active-plan.md');
    const restore = path.join(workDir, 'restore.md');
    fs.writeFileSync(source, DX_ERROR_CONTRACT_PLAN);
    initializePlan(source, active, restore);
    const methodology = prepareMethodology('dx', path.join(ROOT, 'plan-devex-review/SKILL.md'), restore);
    const snapshot = createSnapshot('dx', active, restore, methodology.methodologyPath);
    const outDir = path.join(workDir, 'out');
    fs.mkdirSync(outDir);
    const sections = sectionLoadFor('dx', detectScope(active, workDir, {}), 'light');
    const prompts = writePrompts({ phase: 'dx', snapshot, outDir, priorConsensus: [], sections });

    const result = await runSkillTest({
      testName: 'dx-error-contract-review',
      workingDirectory: workDir,
      prompt: `Read ${JSON.stringify(prompts.native)} completely and carry out the review it describes. Write the result file it names.`,
      tools: ['Read', 'Write'],
      allowedTools: ['Read', 'Write'],
      maxTurns: 10,
      timeout: CAPTURE_MS,
      runId,
    });
    logCost(NAME, result);
    const review = fs.existsSync(prompts.nativeResult) ? fs.readFileSync(prompts.nativeResult, 'utf8') : '';
    const grade = gradeDxContractReview(review, snapshot.sha256);
    console.log(`dx-error-contract-review caps: ${grade.caps.map(cap => `${cap.severity}:${cap.title}`).join(' | ') || 'none'}`);
    recordE2E(collector, NAME, 'DX error contract native reviewer', result, {
      passed: result.exitReason === 'success' && grade.pass,
      ...(grade.reasons.length ? { error: grade.reasons.join('; ') } : {}),
    });
    expect(result.exitReason).toBe('success');
    expectContract(!grade.reasons.some(reason => reason.startsWith('result: ')),
      `dx-error-contract-review: the bound result does not parse: ${grade.reasons.filter(reason => reason.startsWith('result: ')).join('; ')}`, { collector, name: NAME });
    expect(grade.reasons).toEqual([]);
  }, CAPTURE_MS + 30_000);
});