gstack/test/skill-e2e-office-hours-brain-writeback.test.ts

/**
 * E2E: /office-hours brain-writeback path under fake gbrain CLI.
 *
 * The matched-pair check for v1.50.0.0's "brain-aware planning actually
 * works under Claude Code" headline: prove that when a user runs
 * /office-hours with gbrain on PATH, the agent actually calls
 * `gbrain put office-hours/<slug>` with valid frontmatter.
 *
 * Approach:
 *   1. Render with --respect-detection into a temporary --out-dir against
 *      a temp GSTACK_HOME that has detected:true. Point section links at
 *      the final fixture workdir with --link-root; never rewrite the repo.
 *   2. Copy the rendered office-hours skill and all its carved sections
 *      into the temp workdir.
 *      Also write docs/gbrain-write-surfaces.md so the agent can read the
 *      template on demand (the compact block points to it).
 *   3. Write a fake `gbrain` shell script into workdir/bin/ with robust
 *      argv quoting (printf %q) so heredoc payloads in --content survive
 *      shell-to-shell. The fake logs every invocation + writes the --content
 *      or stdin page (the template's form) to a per-slug file for inspection.
 *   4. Run /office-hours via runSkillTest with workdir/bin/ first on PATH.
 *      Feed a deterministic founder pitch + auto-decide instructions.
 *   5. Assert the argv log contains `gbrain put office-hours/<slug>`, the
 *      payload file exists with valid YAML frontmatter, and entity stubs
 *      were created.
 *
 * Periodic tier (~$0.50-1/run via claude -p, matches nearby
 * setup-gbrain-path4-* tests at touchfiles.ts:496-498).
 *
 * NOT verified by this test (out of scope, owned by docs/gbrain-write-surfaces.md):
 *   - That gbrain itself persists what `gbrain put` is told (gbrain's
 *     own contract)
 *   - That `.gbrain-source` doesn't re-route writes (gbrain's contract)
 *   - Source-targeting (no way to fake source resolution in a stub CLI)
 */

import { expect, beforeAll, afterAll } from 'bun:test';
import { resolveEvalModel } from '../lib/eval-model';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { execFileSync, spawnSync } from 'child_process';
import {
  chmodSync,
  copyFileSync,
  cpSync,
  existsSync,
  mkdirSync,
  mkdtempSync,
  readFileSync,
  readdirSync,
  rmSync,
  writeFileSync,
} from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';

import { runSkillTest } from './helpers/session-runner';
import { seedHermeticGstackHome } from './helpers/hermetic-env';
import {
  ROOT,
  runId,
  describeIfSelected,
  testConcurrentIfSelected,
  logCost,
  evalsEnabled,
  finalizeEvalCollector,
} from './helpers/e2e-helpers';

import { EvalCollector } from './helpers/eval-store';
import { OFFICE_HOURS_BUN_GRACE_MS, runRecordedOfficeHoursAttempt } from './helpers/office-hours-attempt';

const evalCollector = evalsEnabled ? new EvalCollector('e2e', undefined, 'office-hours-brain-writeback') : null;
afterAll(() => finalizeEvalCollector(evalCollector));

describeIfSelected(
  'Office Hours Brain Writeback E2E',
  ['office-hours-brain-writeback'],
  () => {
    let workDir: string;
    let callsLogPath: string;
    let payloadDir: string;
    let childEnv: Record<string, string>;

    beforeAll(() => {
      workDir = mkdtempSync(join(tmpdir(), 'skill-e2e-brain-writeback-'));
      const run = (cmd: string, args: string[]) =>
        spawnSync(cmd, args, { cwd: workDir, stdio: 'pipe', timeout: 5000 });
      run('git', ['init', '-b', 'main']);
      run('git', ['config', 'user.email', 'test@test.com']);
      run('git', ['config', 'user.name', 'Test']);

      // Copy the founder pitch fixture into the workdir.
      const briefSrc = join(
        ROOT,
        'test',
        'fixtures',
        'office-hours-brain-writeback',
        'brief.md',
      );
      copyFileSync(briefSrc, join(workDir, 'pitch.md'));

      // Generate the brain-aware variant outside the repository. The renderer
      // writes every Claude skill, so snapshotting just office-hours cannot
      // protect the rest of the source tree from a detection-enabled render.
      const tmpHome = mkdtempSync(join(tmpdir(), 'gbrain-detect-home-'));
      const renderDir = join(tmpHome, 'render');
      writeFileSync(
        join(tmpHome, 'gbrain-detection.json'),
        JSON.stringify({
          gbrain_local_status: 'ok',
          gbrain_on_path: true,
          gbrain_version: 'test-0.41.0',
        }),
      );
      // office-hours is carved (v2 plan T9): GBRAIN_SAVE_RESULTS moved into
      // sections/design-and-handoff.md. Check the UNION for the gbrain put
      // block, then copy all sections so every rendered section link resolves.
      const skillPath = join(renderDir, 'office-hours', 'SKILL.md');
      const sectionPath = join(renderDir, 'office-hours', 'sections', 'design-and-handoff.md');
      try {
        execFileSync(
          'bun',
          [
            'run',
            'scripts/gen-skill-docs.ts',
            '--host',
            'claude',
            '--respect-detection',
            '--out-dir',
            renderDir,
            '--link-root',
            workDir,
          ],
          {
            // Read repository templates; all generated output lands in renderDir.
            cwd: ROOT,
            env: { ...process.env, GSTACK_HOME: tmpHome },
            stdio: ['ignore', 'pipe', 'pipe'],
            timeout: 60_000,
          },
        );
        const brainAwareSkill = readFileSync(skillPath, 'utf-8');
        const hasSection = existsSync(sectionPath);
        const brainAwareSection = hasSection ? readFileSync(sectionPath, 'utf-8') : '';
        if (!(brainAwareSkill + brainAwareSection).includes('gbrain put "office-hours/')) {
          throw new Error(
            'Regenerated office-hours skeleton+section does not contain gbrain put block. ' +
              'Detection override may be broken — see test/gbrain-detection-override.test.ts.',
          );
        }
        cpSync(join(renderDir, 'office-hours'), join(workDir, 'office-hours'), { recursive: true });
      } finally {
        rmSync(tmpHome, { recursive: true, force: true });
      }

      // Copy docs/gbrain-write-surfaces.md so the compact resolver block's
      // on-demand reference resolves (the agent may read it for the full
      // template; we don't require this read but make it available).
      const docsSrc = join(ROOT, 'docs', 'gbrain-write-surfaces.md');
      const docsDst = join(workDir, 'docs', 'gbrain-write-surfaces.md');
      mkdirSync(join(workDir, 'docs'), { recursive: true });
      copyFileSync(docsSrc, docsDst);

      // Set up the fake gbrain CLI with robust argv quoting + payload capture.
      callsLogPath = join(workDir, 'gbrain-calls.log');
      payloadDir = join(workDir, 'gbrain-payloads');
      mkdirSync(payloadDir, { recursive: true });
      const binDir = join(workDir, 'bin');
      mkdirSync(binDir, { recursive: true });
      const fakeGbrain = `#!/bin/bash
# Fake gbrain CLI for E2E test. Logs every invocation with shell-safe quoting
# (printf %q) so --content "$(cat <<'EOF' ... EOF)" payloads survive intact.
{ printf 'gbrain'; for a in "$@"; do printf ' %q' "$a"; done; printf '\\n'; } \\
  >> "${callsLogPath}"
case "$1" in
  --version) echo "gbrain test-0.41.0"; exit 0 ;;
  search) echo "[]"; exit 0 ;;
  # Real gbrain: \`get <slug>\` reads a page back; \`get_page\` is an MCP tool
  # name, not a CLI verb.
  get) cat "${payloadDir}/$2.md" 2>/dev/null; exit 0 ;;
  get_page) echo "Unknown command: get_page" >&2; exit 1 ;;
  put)
    SLUG="$2"
    PAYLOAD="${payloadDir}/$SLUG.md"
    shift 2
    while [ -n "$1" ]; do
      if [ "$1" = "--content" ]; then
        mkdir -p "$(dirname "$PAYLOAD")"
        printf '%s' "$2" > "$PAYLOAD"
        exit 0
      fi
      shift
    done
    # The SAVE_RESULTS template pipes the page: gbrain put "<slug>" < "<page-file>".
    if [ -p /dev/stdin ] || [ -f /dev/stdin ]; then
      mkdir -p "$(dirname "$PAYLOAD")"
      cat > "$PAYLOAD"
    fi
    exit 0
    ;;
esac
exit 0
`;
      const fakePath = join(binDir, 'gbrain');
      writeFileSync(fakePath, fakeGbrain);
      chmodSync(fakePath, 0o755);

      // Keep runtime state inside this fixture. Reuse the standard seed so
      // isolating HOME does not introduce onboarding into the writeback case.
      const fixtureHome = join(workDir, '.fixture-home');
      const gstackHome = join(fixtureHome, '.gstack');
      mkdirSync(join(fixtureHome, '.gbrain'), { recursive: true });
      mkdirSync(gstackHome, { recursive: true });
      seedHermeticGstackHome(gstackHome);
      childEnv = {
        HOME: fixtureHome,
        GBRAIN_HOME: fixtureHome,
        GSTACK_HOME: gstackHome,
        PATH: `${binDir}:${process.env.PATH || ''}`,
      };

      run('git', ['add', '.']);
      run('git', ['commit', '-m', 'fixture']);
    });

    afterAll(() => {
      try {
        rmSync(workDir, { recursive: true, force: true });
      } catch {
        // best effort
      }
    });

    testConcurrentIfSelected(
      'office-hours-brain-writeback',
      async () => {
        await runRecordedOfficeHoursAttempt({
          collector: evalCollector,
          name: '/office-hours-brain-writeback',
          suite: 'Office Hours Brain Writeback E2E',
          model: resolveEvalModel('capture'),
          run: (signal) => runSkillTest({
            signal,
            prompt: `Read office-hours/SKILL.md for the workflow with the Read tool (it is long; Bash output truncates it). Skip its preamble bash block, onboarding, telemetry and contributor sections — go straight to the workflow.

Read pitch.md — that's a founder pitch coming to office hours. Select Startup Mode. Skip any AskUserQuestion — this is non-interactive; auto-decide the recommended option for any question.

For the diagnostic, assume the founder confirmed Q1 (strongest evidence = "230 from a single tweet + 51 paying creators in 6 weeks"), Q2 (status quo = "creators write ad-hoc checks or use opaque Patreon-style platforms"), and Q3 (forcing question already asked).

Generate the design doc per Phase 5. The feature-slug value to substitute into the SAVE_RESULTS template's \`<feature-slug>\` placeholder is exactly 'pixel-fund' (no path prefix — the template already provides the prefix). The \`gbrain\` binary is on PATH at ${workDir}/bin/gbrain. Apply the SAVE_RESULTS template literally: the slug should land at \`<prefix>/pixel-fund\` per the resolver shape, with the actual design doc markdown body as the page content. Then enrich entity stubs for any named people or companies mentioned in the pitch.

This is a test of the brain-writeback path. Do NOT skip the gbrain save step under any circumstance — the runtime guard ("skip if gbrain not on PATH") does NOT apply here because gbrain IS available. Do NOT explore gbrain --help; follow the SAVE_RESULTS template's exact CLI shape. If you encounter any AskUserQuestion, auto-decide recommended.`,
            workingDirectory: workDir,
            maxTurns: 12,
            timeout: CAPTURE_LONG_MS,
            testName: 'office-hours-brain-writeback',
            runId,
            model: resolveEvalModel('capture'),
            env: childEnv,
          }),
          validate: (result) => {
            logCost('/office-hours (BRAIN WRITEBACK)', result);
            expect(['success', 'error_max_turns']).toContain(result.exitReason);

            // The headline assertion: agent actually called gbrain put on the
            // expected slug.
            if (!existsSync(callsLogPath)) {
              throw new Error(
                `No gbrain calls log at ${callsLogPath}. ` +
                  `Agent likely did NOT invoke gbrain at all. ` +
                  `Check that office-hours/SKILL.md in the workdir contains the gbrain put block.`,
              );
            }
            const callsLog = readFileSync(callsLogPath, 'utf-8');
            console.log('--- gbrain calls log ---');
            console.log(callsLog);
            console.log('--- end calls log ---');

            expect(callsLog).toContain('gbrain put');
            // Agent obedience: the slug should contain 'pixel-fund' somewhere
            // (preferably under the office-hours/ prefix). The strict slug
            // SHAPE (office-hours/<slug>) is already pinned by the resolver
            // unit test (test/resolvers-gbrain-save-results.test.ts); this
            // E2E proves the agent actually invokes gbrain put with the
            // payload, not the resolver's literal output shape.
            expect(callsLog).toMatch(/gbrain put .*pixel-fund/);

            // Payload file exists. Agent may write to office-hours/pixel-fund.md
            // (resolver-faithful) OR pixel-fund.md (agent dropped prefix); both
            // are acceptable here because the YAML frontmatter is the real
            // contract test. Search the payload tree for any *.md file that
            // contains 'pixel-fund' in the path.
            const findPayload = (dir: string): string | null => {
              if (!existsSync(dir)) return null;
              for (const entry of readdirSync(dir, { withFileTypes: true })) {
                const full = join(dir, entry.name);
                if (entry.isDirectory()) {
                  const nested = findPayload(full);
                  if (nested) return nested;
                } else if (entry.name.includes('pixel-fund')) {
                  return full;
                }
              }
              return null;
            };
            const payloadPath = findPayload(payloadDir);
            if (!payloadPath) {
              throw new Error(
                `Agent called gbrain put but no payload file with 'pixel-fund' ` +
                  `in name was written to ${payloadDir}. Check the fake gbrain ` +
                  `--content and stdin page capture.`,
              );
            }
            const payload = readFileSync(payloadPath, 'utf-8');
            expect(payload).toMatch(/^---\s*\n/);
            expect(payload).toContain('title:');
            expect(payload).toContain('tags:');
            expect(payload.length).toBeGreaterThan(200);

            // Entity stubs: agents are inconsistent about whether they use
            // 'entities/<name>' (resolver doc) or 'entity/<name>' (singular).
            // We accept either — the test asserts that AT LEAST ONE entity
            // stub call exists, not the exact slug shape.
            const entityCallMatches =
              callsLog.match(/gbrain put entit(?:y|ies)\//g) || [];
            if (entityCallMatches.length === 0) {
              console.warn(
                'No entity stub calls in gbrain calls log. Resolver instructs ' +
                  'entity extraction but it is best-effort.',
              );
            } else {
              console.log(
                `Entity stub calls observed: ${entityCallMatches.length}`,
              );
            }
          },
        });
      },
      CAPTURE_LONG_MS + OFFICE_HOURS_BUN_GRACE_MS,
    );
  },
);