gstack/test/carve-section-sharding.test.ts

import { describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { CARVE_GUARDS } from './helpers/carve-guards';
import { isPaidTestFile } from './helpers/paid-test-set';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { buildRunManifest, DEFAULT_SHARD_TIMEOUT_MS, retriesForFiles, selectPaidTestFiles } from '../scripts/test-paid-shards';

describe('carved-skill cases each get a complete paid process budget', () => {
  const files = fs.readdirSync(import.meta.dir).filter(name => /^carve-section-loading-.*\.test\.ts$/.test(name));
  test('every generic registry entry has exactly one periodic wrapper and no external entry does', () => {
    const covered = files.flatMap(file => {
      const source = fs.readFileSync(path.join(import.meta.dir, file), 'utf8');
      expect(source).toContain("describeE2ETier('periodic')");
      const calls = [...source.matchAll(/registerCarveSectionCase\('([^']+)'\)/g)];
      expect(calls).toHaveLength(1);
      expect(isPaidTestFile('test/' + file)).toBe(true);
      return calls.map(match => match[1]);
    });
    expect(covered.sort()).toEqual(Object.values(CARVE_GUARDS).filter(guard => guard.behavioral === 'plan' || guard.behavioral === 'prompt').map(guard => guard.skill).sort());
    expect(new Set(covered).size).toBe(covered.length);
    expect(selectPaidTestFiles(files.map(file => 'test/' + file), 'periodic').selected).toHaveLength(files.length);
    expect(selectPaidTestFiles(files.map(file => 'test/' + file), 'gate').selected).toHaveLength(0);
  });
  test('every case run plus teardown fits even with within-shard concurrency one', () => {
    for (const file of files) {
      const attempts = retriesForFiles(['test/' + file]) + 1;
      expect(CAPTURE_LONG_MS * attempts + 10_000).toBeLessThan(DEFAULT_SHARD_TIMEOUT_MS);
    }
  });
  test('explicit skill scope selects one process and records why the others are excluded', () => {
    const discovered = files.map(file => 'test/' + file);
    const env = { GSTACK_CARVE_SKILL: ' review ', EVALS_ALL: '1' };
    const root = path.resolve(import.meta.dir, '..');
    const result = selectPaidTestFiles(discovered, 'periodic', root, env);
    expect(result.selected).toEqual(['test/carve-section-loading-review.test.ts']);
    expect(result.excluded).toHaveLength(discovered.length - 1);
    expect(result.excluded.every(entry => entry.reason.includes('GSTACK_CARVE_SKILL=review'))).toBe(true);
    const manifest = buildRunManifest({ tier: 'periodic', sliceCount: 2, evalsAll: true, discovered, env, rootDir: root });
    expect(manifest.entries.filter(entry => entry.status === 'planned').map(entry => entry.file)).toEqual(result.selected);
    expect(manifest.entries.filter(entry => entry.status === 'excluded')).toHaveLength(result.excluded.length);
    expect(() => selectPaidTestFiles(discovered, 'periodic', root, { GSTACK_CARVE_SKILL: 'typo' })).toThrow('no generic section-loading wrapper');
  });
});

describe('design-consultation section completion (census 36641820398 slice 8)', () => {
const DC_ROOT = path.resolve(import.meta.dir, '..');
// DESIGN.md content from the Write call of the census 36641820398 slice 8
// design-consultation capture. That run Read its section at 11s and wrote
// DESIGN.md and CLAUDE.md, then timed out composing the duplicate REPORT.md
// the generic fixture requested.
const designConsultationMd = fs.readFileSync(path.join(import.meta.dir, 'fixtures/design-consultation-section-design-md.md'), 'utf8');
const genericReport = '# Design report\n' + 'The design review summary is complete. '.repeat(8);

interface Fixture {
  output?: string;
  file?: 'DESIGN.md' | 'REPORT.md' | null;
  exitReason?: string;
  missingRead?: boolean;
}

// Run the actual paid registration and capture helper in an isolated free
// child; only the session-runner/provider boundary is replaced.
function exercise(fixture: Fixture = {}) {
  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'design-consultation-completion-'));
  const script = path.join(dir, 'capture.test.ts');
  const facts = path.join(dir, 'facts.json');
  const input = { output: designConsultationMd, file: 'DESIGN.md', exitReason: 'success', missingRead: false, ...fixture };
  fs.writeFileSync(script, `
import { expect, mock } from 'bun:test';
import * as fs from 'node:fs';
import * as path from 'node:path';
import { CARVE_GUARDS } from ${JSON.stringify(path.join(DC_ROOT, 'test/helpers/carve-guards.ts'))};
const input = ${JSON.stringify(input)};
const guard = CARVE_GUARDS['design-consultation'];
mock.module(${JSON.stringify(path.join(DC_ROOT, 'test/helpers/session-runner.ts'))}, () => ({
  runSkillTest: async opts => {
    fs.writeFileSync(${JSON.stringify(facts)}, JSON.stringify({ prompt: opts.prompt, timeout: opts.timeout, allowedTools: opts.allowedTools }));
    if (input.file) fs.writeFileSync(path.join(opts.workingDirectory, input.file), input.output);
    return {
      exitReason: input.exitReason, output: 'Wrote DESIGN.md and CLAUDE.md.',
      toolCalls: input.missingRead ? [] : guard.requiredReads.map(section => ({
        tool: 'Read', input: { file_path: path.join(opts.workingDirectory, 'design-consultation', 'sections', section) },
      })), transcript: [],
    };
  },
}));
const { registerCarveSectionCase } = await import(${JSON.stringify(path.join(DC_ROOT, 'test/helpers/carve-section-case.ts'))});
registerCarveSectionCase('design-consultation');
`);
  try {
    const child = Bun.spawnSync([process.execPath, 'test', script], {
      cwd: DC_ROOT,
      env: {
        PATH: process.env.PATH ?? '', HOME: dir, TMPDIR: dir, TEMP: dir, TMP: dir,
        ...(process.env.SystemRoot ? { SystemRoot: process.env.SystemRoot } : {}),
      },
      timeout: 10_000,
    });
    expect(fs.existsSync(facts), child.stderr.toString()).toBe(true);
    expect(child.signalCode ?? null).toBeNull();
    return { code: child.exitCode, output: child.stdout.toString() + child.stderr.toString(),
      facts: JSON.parse(fs.readFileSync(facts, 'utf8')) as { prompt: string; timeout: number; allowedTools: string[] } };
  } finally {
    fs.rmSync(dir, { recursive: true, force: true });
  }
}

test('the captured DESIGN.md is the completed output; no duplicate report is requested', () => {
  expect(designConsultationMd.split('\n')[1]).toBe('# gstack: design-md-format=spec');
  const result = exercise();
  expect(result.code, result.output).toBe(0);
  expect(result.facts.timeout).toBe(480_000);
  expect(result.facts.prompt).toContain('declined the optional outside design voices');
  expect(result.facts.prompt).toMatch(/write the skill's final output[^\n]*DESIGN\.md/);
  expect(result.facts.prompt).not.toContain('REPORT.md');
}, 20_000);

test('the census timeout still fails even after DESIGN.md was written', () => {
  expect(exercise({ exitReason: 'timeout' }).code).not.toBe(0);
}, 20_000);

test('a generic report without the DESIGN.md format marker does not count as completion', () => {
  expect(exercise({ output: genericReport }).code).not.toBe(0);
  expect(exercise({ output: genericReport, file: 'REPORT.md' }).code).not.toBe(0);
});

test('a terminal-only claim without writing DESIGN.md fails', () => {
  expect(exercise({ file: null }).code).not.toBe(0);
}, 20_000);

test('skipping the section Read fails', () => {
  expect(exercise({ missingRead: true }).code).not.toBe(0);
}, 20_000);

});