gstack/test/skill-e2e-skillify.test.ts

/**
 * Browser-skills Phase 2a — gate-tier E2E for /scrape and /skillify.
 *
 * Five scenarios cover the productivity loop and the contracts locked
 * during the v1.19.0.0 plan review:
 *
 *   D1 — /skillify provenance guard (scenario 4)
 *   D2 — synthesis input slice (covered indirectly by scenario 3 — the
 *        committed SKILL.md must not contain conversation prose)
 *   D3 — atomic write discipline (scenarios 3 and 5)
 *
 *   1. scrape-match-path — /scrape with intent matching bundled
 *      hackernews-frontpage routes via $B skill run, no prototype.
 *   2. scrape-prototype-path — /scrape against a local file:// fixture
 *      (no matching skill) drives $B primitives, returns JSON, suggests
 *      /skillify.
 *   3. skillify-happy-path — /scrape then /skillify in one session.
 *      Skill written to the global tier (<state root>/browser-skills/<name>/)
 *      with full file tree, $B skill test passes.
 *   4. skillify-provenance-refusal — cold /skillify with no prior
 *      /scrape refuses with the D1 message; nothing on disk.
 *   5. skillify-approval-reject — /scrape then /skillify but reject in
 *      the approval gate; temp dir is removed, nothing at final path.
 *
 * The three skillify keys run gate-tier; the two scrape keys are periodic
 * (/scrape is Aside-first and its fallback no longer prescribes the match +
 * prototype flow they assert — see E2E_TIERS). ~$0.50–$1.50 each.
 * Set EVALS=1 to enable. Set EVALS_MODEL to override (default frontier Claude).
 */

import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { runSkillTest } from './helpers/session-runner';
import {
  ROOT, browseBin, runId,
  describeIfSelected, testConcurrentIfSelected,
  setupBrowseShims, copyDirSync, logCost, recordE2E,
  createEvalCollector, finalizeEvalCollector,
} from './helpers/e2e-helpers';
import { extractSkillBody } from './helpers/skill-fixture';
import { resolveStateRoot } from '../lib/state-root';
import { spawnSync } from 'child_process';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { readShippedSkillRouting } from './helpers/shipped-skill-routing';
import { hasNarrationLeak } from './helpers/skill-body-narration';

const evalCollector = createEvalCollector('e2e-skillify');

// ─── Shared workdir setup ───────────────────────────────────────

interface Workdir {
  workDir: string;
  gstackHome: string;
  skillsDir: string;
}

/**
 * Build a working directory that has:
 *   - The /scrape and /skillify skills installed under .claude/skills/
 *   - The browse binary symlinked + find-browse shim (via setupBrowseShims)
 *   - bin/ scripts referenced by the preamble
 *   - A scoped GSTACK_HOME under the workdir so on-disk artifacts are
 *     contained and assertable
 *   - A CLAUDE.md with the ## Skill routing section gstack ships
 *
 * `installSkills` lets each test pick the minimum surface (e.g., the
 * provenance-refusal scenario doesn't need /scrape).
 */
function setupSkillifyWorkdir(suffix: string, installSkills: string[] = ['scrape', 'skillify']): Workdir {
  const workDir = fs.mkdtempSync(path.join(os.tmpdir(), `skill-e2e-skillify-${suffix}-`));
  const gstackHome = path.join(workDir, '.gstack-home');
  fs.mkdirSync(gstackHome, { recursive: true });

  const run = (cmd: string, args: string[]) =>
    spawnSync(cmd, args, { cwd: workDir, stdio: 'pipe', timeout: 5000 });
  run('git', ['init', '-b', 'main']);
  run('git', ['config', 'user.email', 'test@test.com']);
  run('git', ['config', 'user.name', 'Test']);
  fs.writeFileSync(path.join(workDir, 'README.md'), '# test\n');
  run('git', ['add', '.']);
  run('git', ['commit', '-m', 'initial']);

  setupBrowseShims(workDir);

  // Install requested skills. The tests exercise the full /scrape + /skillify
  // flows (all 11 skillify steps, D1-D3 contracts), so keep the whole
  // skill-specific body — but drop the ~780-line shared preamble the tests
  // never touch (CLAUDE.md: "E2E test fixtures: extract, don't copy").
  const skillsDir = path.join(workDir, '.claude', 'skills');
  for (const skill of installSkills) {
    const destDir = path.join(skillsDir, skill);
    fs.mkdirSync(destDir, { recursive: true });
    fs.writeFileSync(path.join(destDir, 'SKILL.md'), extractSkillBody(path.join(ROOT, skill)));
  }

  // bin/ scripts — preamble references several of these.
  const binDir = path.join(workDir, 'bin');
  fs.mkdirSync(binDir, { recursive: true });
  for (const script of [
    'gstack-timeline-log', 'gstack-slug', 'gstack-config', 'gstack-state-root.sh', 'gstack-remote-identity.sh',
    'gstack-update-check', 'gstack-repo-mode',
    'gstack-learnings-log', 'gstack-learnings-search',
  ]) {
    const src = path.join(ROOT, 'bin', script);
    if (fs.existsSync(src)) {
      fs.copyFileSync(src, path.join(binDir, script));
      fs.chmodSync(path.join(binDir, script), 0o755);
    }
  }

  fs.writeFileSync(path.join(workDir, 'CLAUDE.md'), `# Project Instructions

${readShippedSkillRouting().section}

## Test environment

- GSTACK_HOME="${gstackHome}" for all gstack bin scripts.
- bin scripts are at ./bin/ relative to this directory.
- Browse binary is at ${browseBin} — assign to $B (e.g., \`B=${browseBin}\`).
`);

  return { workDir, gstackHome, skillsDir };
}

/**
 * Install the bundled hackernews-frontpage browser-skill into the workdir's
 * project-tier (so $B skill list finds it for match-path tests). The skill
 * has to live under <workdir>/.gstack/browser-skills/ for the project-tier
 * lookup to find it (gstack's bundled tier resolves from the install dir,
 * which the test workdir doesn't have).
 */
function installBundledHackernewsSkill(workDir: string) {
  const src = path.join(ROOT, 'browser-skills', 'hackernews-frontpage');
  const dst = path.join(workDir, '.gstack', 'browser-skills', 'hackernews-frontpage');
  copyDirSync(src, dst);
}

/**
 * Where the child's skill writes resolve: the state root its env names (the
 * global tier and /skillify's .tmp staging, per lib/state-root.ts — GSTACK_HOME
 * outranks HOME) and the cwd's project tier. Since the global tier moved to
 * the state root (v1.91.11.0), sweeping only $HOME/.gstack failed the agent
 * that used commitSkill as written and passed agents that unset GSTACK_HOME
 * to land a skill $B skill list cannot see.
 */
function skillWriteRoots(workDir: string, childEnv: { GSTACK_HOME: string; HOME: string }) {
  return { global: resolveStateRoot(childEnv), project: path.join(workDir, '.gstack') };
}

/** Negative sweeps also cover $HOME/.gstack, where an agent that bypassed the state root would write. */
function everyWriteRoot(workDir: string, childEnv: { GSTACK_HOME: string; HOME: string }): string[] {
  const { global, project } = skillWriteRoots(workDir, childEnv);
  return [...new Set([global, project, path.join(childEnv.HOME, '.gstack')])];
}

/** Helper: every Bash invocation's command string from the agent. */
function bashCommands(result: { toolCalls: Array<{ tool: string; input: any }> }): string[] {
  return result.toolCalls
    .filter((tc) => tc.tool === 'Bash')
    .map((tc) => String(tc.input?.command ?? ''))
    .filter(Boolean);
}

/** Helper: the union of agent text + every tool input/output for matching. */
function fullSurface(result: any): string {
  const parts: string[] = [];
  if (result.output) parts.push(String(result.output));
  for (const tc of result.toolCalls || []) {
    parts.push(JSON.stringify(tc.input || {}));
    if (tc.output) parts.push(String(tc.output));
  }
  for (const entry of result.transcript || []) {
    try { parts.push(JSON.stringify(entry)); } catch { /* skip */ }
  }
  return parts.join('\n');
}

// ─── Test fixtures ──────────────────────────────────────────────

/**
 * Tiny HTML fixture for the prototype-path test. Stable structure with three
 * "items" the agent should be able to extract via $B html + parse.
 */
const PROTOTYPE_FIXTURE_HTML = `<!doctype html>
<html><body>
  <h1>Test Items</h1>
  <ul id="items">
    <li class="item"><a href="/a">First Title</a><span class="score">42</span></li>
    <li class="item"><a href="/b">Second Title</a><span class="score">17</span></li>
    <li class="item"><a href="/c">Third Title</a><span class="score">8</span></li>
  </ul>
</body></html>
`;

// ─── Live-fire suite ────────────────────────────────────────────

describeIfSelected('Browser-skills Phase 2a E2E (/scrape + /skillify)', [
  'scrape-match-path',
  'scrape-prototype-path',
  'skillify-happy-path',
  'skillify-provenance-refusal',
  'skillify-approval-reject',
], () => {
  afterAll(() => { finalizeEvalCollector(evalCollector); });

  // ── 1. /scrape match path: bundled hackernews-frontpage matches ──────
  testConcurrentIfSelected('scrape-match-path', async () => {
    const { workDir, gstackHome } = setupSkillifyWorkdir('match', ['scrape']);
    installBundledHackernewsSkill(workDir);

    const result = await runSkillTest({
      prompt: `Run /scrape latest hacker news stories.
This run is non-interactive; AskUserQuestion is unavailable.`,
      workingDirectory: workDir,
      env: { GSTACK_HOME: gstackHome },
      maxTurns: 12,
      allowedTools: ['Skill', 'Bash', 'Read'],
      timeout: JUDGE_MS,
      testName: 'scrape-match-path',
      runId,
    });

    logCost('scrape-match-path', result);

    // Outcome: the matcher ran the bundled skill instead of prototyping, and
    // nothing new was written. Listing first is one valid way to match.
    const cmds = bashCommands(result);
    const ranBundledSkill = cmds.some(c => /\bskill\s+run\s+hackernews-frontpage\b/.test(c));
    const prototyped = cmds.some(c => /\$B\s+goto\b|\/browse\s+goto\b/.test(c));
    const newSkills = [workDir, gstackHome].flatMap((root) => {
      const skillsRoot = path.join(root, '.gstack', 'browser-skills');
      return fs.existsSync(skillsRoot) ? fs.readdirSync(skillsRoot).filter(d => !d.startsWith('.') && d !== 'hackernews-frontpage') : [];
    });
    const staged = [workDir, gstackHome].some((root) => {
      const stagingRoot = path.join(root, '.gstack', '.tmp');
      return fs.existsSync(stagingRoot) && fs.readdirSync(stagingRoot).some(d => d.startsWith('skillify-'));
    });
    console.log(`scrape-match-path: listed skills first = ${cmds.some(c => /\bskill\s+list\b/.test(c))}`);
    const exitOk = ['success', 'error_max_turns'].includes(result.exitReason);

    recordE2E(evalCollector, 'scrape match-path routes to bundled skill', 'Phase 2a E2E', result, {
      passed: exitOk && ranBundledSkill && !prototyped && newSkills.length === 0 && !staged,
    });

    expect(exitOk).toBe(true);
    expect(ranBundledSkill).toBe(true);
    expect(prototyped, 'matched skill must run without prototyping').toBe(false);
    expect(newSkills).toEqual([]);
    expect(staged).toBe(false);
    try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
  }, CAPTURE_MS);

  // ── 2. /scrape prototype path: drive $B primitives against fixture ────
  testConcurrentIfSelected('scrape-prototype-path', async () => {
    const { workDir, gstackHome } = setupSkillifyWorkdir('prototype', ['scrape']);

    // Stage a local HTML fixture the agent can goto via file://
    const fixturePath = path.join(workDir, 'fixture.html');
    fs.writeFileSync(fixturePath, PROTOTYPE_FIXTURE_HTML);
    const fileUrl = `file://${fixturePath}`;

    const result = await runSkillTest({
      prompt: `Run /scrape titles and scores from ${fileUrl}.
Return JSON of the form {"items": [{"title": "...", "score": N}, ...], "count": N}.
This run is non-interactive; AskUserQuestion is unavailable.`,
      workingDirectory: workDir,
      env: { GSTACK_HOME: gstackHome },
      maxTurns: 18,
      allowedTools: ['Skill', 'Bash', 'Read'],
      timeout: CAPTURE_MS,
      testName: 'scrape-prototype-path',
      runId,
    });

    logCost('scrape-prototype-path', result);

    const cmds = bashCommands(result);
    const wentToFixture = cmds.some(c => c.includes(fileUrl));
    const fetchedHtml = cmds.some(c => /\bgoto\b|\bhtml\b|\btext\b/.test(c));
    const surface = fullSurface(result);
    const mentionsSkillify = /skillify/i.test(surface);
    // Accept JSON shape variants — the prompt asks for `"items": [...]` but
    // the model sometimes emits equivalent containers (`"results"`, `"data"`,
    // `"hits"`) or skips the wrapper entirely and emits a bare array of
    // objects with title+score keys. All of these satisfy the underlying
    // intent: "the agent produced parseable structured output naming the
    // scraped items". We assert the shape, not a literal key name.
    const hasJsonItems =
      /"(items|results|data|hits|entries)"\s*:\s*\[/i.test(surface) ||
      /'(items|results|data|hits|entries)'\s*:/i.test(surface) ||
      // Bare array of {title, score} objects (no outer wrapper key)
      /\[\s*\{[^}]*\btitle\b[^}]*\bscore\b/.test(surface);
    const exitOk = ['success', 'error_max_turns'].includes(result.exitReason);

    recordE2E(evalCollector, 'scrape prototype-path drives $B + emits JSON + nudges skillify', 'Phase 2a E2E', result, {
      passed: exitOk && wentToFixture && fetchedHtml && hasJsonItems && mentionsSkillify,
    });

    expect(exitOk).toBe(true);
    expect(wentToFixture).toBe(true);
    expect(fetchedHtml).toBe(true);
    expect(hasJsonItems).toBe(true);
    expect(mentionsSkillify).toBe(true);
    try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
  }, CAPTURE_MS);

  // ── 3. /skillify happy path: scrape then skillify in one session ─────
  testConcurrentIfSelected('skillify-happy-path', async () => {
    const { workDir, gstackHome } = setupSkillifyWorkdir('happy', ['scrape', 'skillify']);
    const fixturePath = path.join(workDir, 'fixture.html');
    fs.writeFileSync(fixturePath, PROTOTYPE_FIXTURE_HTML);
    const fileUrl = `file://${fixturePath}`;

    const childHome = path.join(workDir, 'home');
    fs.mkdirSync(childHome, { recursive: true });
    const childEnv = {
      GSTACK_HOME: gstackHome,
      // Fresh subdir, NEVER the cwd: with HOME == cwd, claude resolves
      // <cwd>/.claude/skills as the PERSONAL skills dir and the seeded
      // project-tier skills stop registering — this test's Skill() calls
      // silently errored ("Unknown skill") and only passed via the agent
      // self-recovering by Reading SKILL.md manually. Same fix as the
      // provenance-refusal test below.
      HOME: childHome,
    };

    const result = await runSkillTest({
      prompt: `Two steps in this session:

1. Run /scrape titles and scores from ${fileUrl} via the Skill tool.
   Drive the prototype path; return JSON with items[].

2. Run /skillify via the Skill tool. Follow ALL 11 steps including:
   - D1 provenance guard (you have a recent /scrape, proceed)
   - D2 synthesis: include ONLY the final-attempt $B calls (goto + html)
   - D3 atomic write: stage to temp dir, run test, then commit on approval
   - When AskUserQuestion fires, choose the recommended option (A)
     for both the name/tier question AND the approval gate.

The environment already points gstack's state root (GSTACK_HOME) at a
test sandbox; keep it as set so the write helpers and $B agree.

Do NOT halt for clarification.`,
      workingDirectory: workDir,
      env: childEnv,
      maxTurns: 40,
      allowedTools: ['Skill', 'Bash', 'Read', 'Write'],
      timeout: CAPTURE_LONG_MS,
      testName: 'skillify-happy-path',
      runId,
    });

    logCost('skillify-happy-path', result);

    // The skill must land in a tier $B resolves: global (the state root) or project.
    const tiers = skillWriteRoots(workDir, childEnv);
    const skillRoots = [tiers.global, tiers.project].map((r) => path.join(r, 'browser-skills'));
    const writtenSkills = skillRoots.flatMap((root) => (fs.existsSync(root)
      ? fs.readdirSync(root)
        .filter(d => !d.startsWith('.') && d !== 'hackernews-frontpage')
        .map((d) => path.join(root, d))
      : []));
    const skillDir = writtenSkills[0] ?? '';
    const hasAllFiles = !!skillDir
      && fs.existsSync(path.join(skillDir, 'SKILL.md'))
      && fs.existsSync(path.join(skillDir, 'script.ts'))
      && fs.existsSync(path.join(skillDir, 'script.test.ts'))
      && fs.existsSync(path.join(skillDir, '_lib', 'browse-client.ts'))
      && fs.existsSync(path.join(skillDir, 'fixtures'));

    // D2 enforcement: the SKILL.md prose body must not contain conversation
    // fragments (stored good/bad bodies: skill-fixture.test.ts).
    let prosesClean = false;
    if (hasAllFiles) {
      const skillMd = fs.readFileSync(path.join(skillDir, 'SKILL.md'), 'utf-8');
      const body = skillMd.split(/\n---\n/)[1] || '';
      prosesClean = !hasNarrationLeak(body);
    }

    const exitOk = ['success', 'error_max_turns'].includes(result.exitReason);

    recordE2E(evalCollector, 'skillify happy path writes well-formed skill on disk', 'Phase 2a E2E', result, {
      passed: exitOk && hasAllFiles && prosesClean,
    });

    expect(exitOk).toBe(true);
    expect(writtenSkills.length).toBeGreaterThan(0);
    expect(hasAllFiles).toBe(true);
    expect(prosesClean).toBe(true);
    try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
  }, CAPTURE_LONG_MS);

  // ── 4. /skillify provenance refusal: D1 contract ─────────────────────
  testConcurrentIfSelected('skillify-provenance-refusal', async () => {
    const { workDir, gstackHome } = setupSkillifyWorkdir('refusal', ['skillify']);
    // Child HOME must be a FRESH dir, never workDir itself: with HOME == cwd,
    // claude resolves <cwd>/.claude/skills as the PERSONAL skills dir and the
    // project-tier skills seeded there never register — the Skill tool then
    // errors "Unknown skill: skillify" (observed on claude 2.1.237). A
    // sibling home/ dir keeps the override's intent (any ~/.gstack write from
    // the child lands inside the assertable sandbox, not the operator's real
    // home) without colliding with project-skill discovery.
    const childHome = path.join(workDir, 'home');
    fs.mkdirSync(childHome, { recursive: true });
    const childEnv = { GSTACK_HOME: gstackHome, HOME: childHome };

    const result = await runSkillTest({
      prompt: `Run /skillify.
This run is non-interactive; AskUserQuestion is unavailable.`,
      workingDirectory: workDir,
      env: childEnv,
      maxTurns: 8,
      allowedTools: ['Skill', 'Bash', 'Read'],
      timeout: JUDGE_MS,
      testName: 'skillify-provenance-refusal',
      runId,
    });

    logCost('skillify-provenance-refusal', result);

    // Tripwire: the Skill tool must actually LOAD skillify. A not-loaded
    // skill (tool error "Unknown skill: skillify", or the agent narrating
    // "not registered" and improvising a refusal) must never pass as a D1
    // refusal. Neither phrase appears in the skillify fixture or the prompt,
    // so a hit can only come from a real load failure.
    const surface = fullSurface(result);
    const skillLoadFailed = /unknown skill|not registered/i.test(surface);

    // The refusal must be in the AGENT'S OWN words. When the Skill tool
    // loads skillify, the SKILL.md body — which contains the exact refusal
    // message — is injected into the transcript as a user message, so
    // matching the full surface would pass vacuously. Match only assistant
    // text blocks + the final result.
    const agentText = [
      result.output,
      ...result.transcript
        .filter((e: any) => e?.type === 'assistant')
        .flatMap((e: any) => ((e.message?.content ?? []) as any[])
          .filter((c: any) => c?.type === 'text')
          .map((c: any) => String(c.text ?? ''))),
    ].join('\n');
    const refusalText = /no recent \/?scrape result|run \/scrape.*first|no prior \/?scrape/i.test(agentText);

    // Critical: nothing on disk. No staged dir, no committed skill, in any
    // root a write could resolve to.
    const diskRoots = everyWriteRoot(workDir, childEnv);
    const noSkillsWritten = diskRoots.every((root) => {
      const skillsRoot = path.join(root, 'browser-skills');
      return !fs.existsSync(skillsRoot)
        || fs.readdirSync(skillsRoot).filter(d => !d.startsWith('.')).length === 0;
    });
    const noStaging = diskRoots.every((root) => {
      const stagingRoot = path.join(root, '.tmp');
      return !fs.existsSync(stagingRoot)
        || fs.readdirSync(stagingRoot).filter(d => d.startsWith('skillify-')).length === 0;
    });

    const exitOk = ['success', 'error_max_turns'].includes(result.exitReason);

    recordE2E(evalCollector, 'skillify D1 refusal — no on-disk write', 'Phase 2a E2E', result, {
      passed: exitOk && !skillLoadFailed && refusalText && noSkillsWritten && noStaging,
    });

    expect(exitOk).toBe(true);
    expect(skillLoadFailed).toBe(false);
    expect(refusalText).toBe(true);
    expect(noSkillsWritten).toBe(true);
    expect(noStaging).toBe(true);
    try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
  }, JUDGE_MS);

  // ── 5. /skillify approval-gate reject: D3 cleanup ────────────────────
  testConcurrentIfSelected('skillify-approval-reject', async () => {
    const { workDir, gstackHome } = setupSkillifyWorkdir('reject', ['scrape', 'skillify']);
    const fixturePath = path.join(workDir, 'fixture.html');
    fs.writeFileSync(fixturePath, PROTOTYPE_FIXTURE_HTML);
    const fileUrl = `file://${fixturePath}`;

    const childHome = path.join(workDir, 'home');
    fs.mkdirSync(childHome, { recursive: true });
    // Fresh subdir, never the cwd — see the happy-path comment.
    const childEnv = { GSTACK_HOME: gstackHome, HOME: childHome };

    const result = await runSkillTest({
      prompt: `Two steps:

1. Run /scrape titles and scores from ${fileUrl} via the Skill tool.

2. Run /skillify via the Skill tool. Follow steps 1-9. When the approval
   gate AskUserQuestion fires (Step 9), choose option C (Discard) instead
   of A (Commit). The D3 contract says the temp dir must be removed and
   nothing should land at the final tier path.

Do NOT commit the skill.`,
      workingDirectory: workDir,
      env: childEnv,
      maxTurns: 35,
      allowedTools: ['Skill', 'Bash', 'Read', 'Write'],
      timeout: CAPTURE_LONG_MS,
      testName: 'skillify-approval-reject',
      runId,
    });

    logCost('skillify-approval-reject', result);

    // D3 contract: nothing at the final tier path; staging dir is gone.
    const negativeRoots = everyWriteRoot(workDir, childEnv);
    const writtenSkills = negativeRoots.flatMap((root) => {
      const skillsRoot = path.join(root, 'browser-skills');
      return fs.existsSync(skillsRoot)
        ? fs.readdirSync(skillsRoot).filter(d => !d.startsWith('.'))
        : [];
    });
    const stagingLeftovers = negativeRoots.flatMap((root) => {
      const stagingRoot = path.join(root, '.tmp');
      return fs.existsSync(stagingRoot)
        ? fs.readdirSync(stagingRoot).filter(d => d.startsWith('skillify-'))
        : [];
    });

    const exitOk = ['success', 'error_max_turns'].includes(result.exitReason);

    recordE2E(evalCollector, 'skillify approval-reject leaves no on-disk artifact', 'Phase 2a E2E', result, {
      passed: exitOk && writtenSkills.length === 0 && stagingLeftovers.length === 0,
    });

    expect(exitOk).toBe(true);
    expect(writtenSkills.length).toBe(0);
    expect(stagingLeftovers.length).toBe(0);
    try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
  }, CAPTURE_LONG_MS);
});