gstack/test/skill-e2e-write-skill.test.ts

import { describe, expect, beforeAll, afterAll } from 'bun:test';
import { CAPTURE_MS, JUDGE_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
  ROOT, runId, describeIfSelected, testConcurrentIfSelected, copyDirSync, logCost, recordE2E,
  createEvalCollector, finalizeEvalCollector,
} from './helpers/e2e-helpers';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';

const evalCollector = createEvalCollector('e2e-write-skill');

const DEPLOY_SKILL = `---
name: deploy
description: Deploy the web app to production.
---

# Deploy

1. Build the release image with \`make release\`.
2. Run database migrations before restarting the web servers.
3. Restart the web servers one at a time.
4. Check the health endpoint returns 200.
`;

describeIfSelected('/write-skill E2E', ['write-skill-add-lesson'], () => {
  let workDir: string;

  beforeAll(() => {
    workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-write-skill-'));
    copyDirSync(path.join(ROOT, 'write-skill'), path.join(workDir, 'write-skill'));
    fs.mkdirSync(path.join(workDir, 'deploy'));
    fs.writeFileSync(path.join(workDir, 'deploy', 'SKILL.md'), DEPLOY_SKILL);
  });

  afterAll(() => {
    try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
    finalizeEvalCollector(evalCollector);
  });

  testConcurrentIfSelected('write-skill-add-lesson', async () => {
    const result = await runSkillTest({
      prompt: `Read the file write-skill/SKILL.md for the /write-skill skill instructions.

Add this lesson to the deploy skill in ./deploy: "Last Tuesday we restarted the web servers while migrations were still running and the site returned 500s for ten minutes. Migrations must finish before any restart."

IMPORTANT:
- The gstack bin scripts are at ${ROOT}/bin/, not at ~/.claude/skills/gstack/bin/; use that path.
- ./deploy is a skill folder the user owns, not a git repository.
- Do NOT use AskUserQuestion. Edit the file, then report what you changed and its size before and after.`.replace('${ROOT}', ROOT),
      workingDirectory: workDir,
      maxTurns: 15,
      allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
      timeout: JUDGE_MS,
      testName: 'write-skill-add-lesson',
      runId,
    });
    logCost('/write-skill add lesson', result);

    const after = fs.readFileSync(path.join(workDir, 'deploy', 'SKILL.md'), 'utf-8');
    const grewBy = after.trimEnd().split('\n').length - DEPLOY_SKILL.trimEnd().split('\n').length;
    const ruleLanded = /finish|complete|wait/i.test(after.split('\n').find(l => /migration/i.test(l)) ?? '');
    const storyKept = /tuesday|500s?|ten minutes/i.test(after);
    const reportedSize = /\d+\s*(B|bytes)?\s*(→|->|to)\s*\d+/i.test(result.output);
    const exitOk = ['success', 'error_max_turns'].includes(result.exitReason);

    recordE2E(evalCollector, '/write-skill', 'Add a lesson by editing the existing line', result, {
      passed: exitOk && grewBy <= 1 && ruleLanded && !storyKept && reportedSize,
    });
    expect(exitOk).toBe(true);
    expect(ruleLanded).toBe(true);
    expect(storyKept).toBe(false);
    expect(grewBy).toBeLessThanOrEqual(1);
    expect(reportedSize).toBe(true);
  }, CAPTURE_MS);
});