gstack/test/helpers/session-runner.test.ts

import { describe, test, expect } from 'bun:test';
import { spawnSync } from 'node:child_process';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { pathToFileURL } from 'node:url';
import { browseNavigatedTo, parseNDJSON } from './session-runner';
import captured_scrape_goto from '../fixtures/scrape-prototype-goto-42571575838.json';

test('runSkillTest launches a child without operator credentials', () => {
  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-hermetic-session-'));
  try {
    const bin = path.join(root, 'claude');
    fs.writeFileSync(bin, `#!/usr/bin/env node
const names = ['GITHUB_TOKEN', 'GITHUB_PERSONAL_ACCESS_TOKEN', 'GITHUB_APP_PRIVATE_KEY', 'GH_TOKEN', 'GITHUB_ACTIONS', 'GITHUB_PATH', 'GITHUB_TOKENIZER', 'EVALS_RUN_ID'];
const present = Object.fromEntries(names.map(name => [name, Object.hasOwn(process.env, name)]));
console.log(JSON.stringify({type: 'result', subtype: 'success', result: JSON.stringify(present)}));
`, { mode: 0o700 });
    const script = `import { runSkillTest } from ${JSON.stringify(pathToFileURL(path.join(import.meta.dir, 'session-runner.ts')).href)};
const result = await runSkillTest({prompt: 'synthetic fixture', workingDirectory: ${JSON.stringify(root)}, model: 'fixture', timeout: 5000, startupGraceMs: 5000, allowedTools: []});
console.log(JSON.stringify({exitReason: result.exitReason, child: JSON.parse(result.output)}));`;
    const result = spawnSync(process.execPath, ['-e', script], {
      cwd: path.resolve(import.meta.dir, '..', '..'),
      encoding: 'utf8',
      timeout: 30_000,
      env: {
        PATH: `${root}${path.delimiter}${process.env.PATH ?? '/usr/bin:/bin'}`,
        HOME: root,
        TMPDIR: os.tmpdir(),
        GITHUB_TOKEN: 'synthetic-token',
        GITHUB_PERSONAL_ACCESS_TOKEN: 'synthetic-pat',
        GITHUB_APP_PRIVATE_KEY: 'synthetic-private-key',
        GH_TOKEN: 'synthetic-gh-token',
        GITHUB_ACTIONS: 'true',
        GITHUB_PATH: '/tmp/actions-path',
        GITHUB_TOKENIZER: 'metadata-tokenizer',
        EVALS_RUN_ID: 'synthetic-run',
      },
    });
    expect(result.status, result.stderr).toBe(0);
    expect(JSON.parse(result.stdout)).toEqual({
      exitReason: 'success',
      child: {
        GITHUB_TOKEN: false,
        GITHUB_PERSONAL_ACCESS_TOKEN: false,
        GITHUB_APP_PRIVATE_KEY: false,
        GH_TOKEN: false,
        GITHUB_ACTIONS: true,
        GITHUB_PATH: true,
        GITHUB_TOKENIZER: true,
        EVALS_RUN_ID: true,
      },
    });
  } finally {
    fs.rmSync(root, { recursive: true, force: true });
  }
});

// Fixture: minimal NDJSON session (system init, assistant with tool_use, tool result, assistant text, result)
const FIXTURE_LINES = [
  '{"type":"system","subtype":"init","session_id":"test-123"}',
  '{"type":"assistant","message":{"content":[{"type":"tool_use","id":"tu1","name":"Bash","input":{"command":"echo hello"}}]}}',
  '{"type":"user","tool_use_result":{"tool_use_id":"tu1","stdout":"hello\\n","stderr":""}}',
  '{"type":"assistant","message":{"content":[{"type":"text","text":"The command printed hello."}]}}',
  '{"type":"assistant","message":{"content":[{"type":"text","text":"Let me also read a file."},{"type":"tool_use","id":"tu2","name":"Read","input":{"file_path":"/tmp/test"}}]}}',
  '{"type":"result","subtype":"success","total_cost_usd":0.05,"num_turns":3,"usage":{"input_tokens":100,"output_tokens":50},"result":"Done."}',
];

describe('parseNDJSON', () => {
  test('parses valid NDJSON with system + assistant + result events', () => {
    const parsed = parseNDJSON(FIXTURE_LINES);
    expect(parsed.transcript).toHaveLength(6);
    expect(parsed.transcript[0].type).toBe('system');
    expect(parsed.transcript[5].type).toBe('result');
  });

  test('extracts tool calls from assistant.message.content[].type === tool_use', () => {
    const parsed = parseNDJSON(FIXTURE_LINES);
    expect(parsed.toolCalls).toHaveLength(2);
    expect(parsed.toolCalls[0]).toEqual({
      tool: 'Bash',
      input: { command: 'echo hello' },
      output: '',
    });
    expect(parsed.toolCalls[1]).toEqual({
      tool: 'Read',
      input: { file_path: '/tmp/test' },
      output: '',
    });
    expect(parsed.toolCallCount).toBe(2);
  });

  test('skips malformed lines without throwing', () => {
    const lines = [
      '{"type":"system"}',
      'this is not json',
      '{"type":"assistant","message":{"content":[{"type":"text","text":"ok"}]}}',
      '{incomplete json',
      '{"type":"result","subtype":"success","result":"done"}',
    ];
    const parsed = parseNDJSON(lines);
    expect(parsed.transcript).toHaveLength(3); // system, assistant, result
    expect(parsed.resultLine?.subtype).toBe('success');
  });

  test('skips empty and whitespace-only lines', () => {
    const lines = [
      '',
      '  ',
      '{"type":"system"}',
      '\t',
      '{"type":"result","subtype":"success","result":"ok"}',
    ];
    const parsed = parseNDJSON(lines);
    expect(parsed.transcript).toHaveLength(2);
  });

  test('extracts resultLine from type: "result" event', () => {
    const parsed = parseNDJSON(FIXTURE_LINES);
    expect(parsed.resultLine).not.toBeNull();
    expect(parsed.resultLine.subtype).toBe('success');
    expect(parsed.resultLine.total_cost_usd).toBe(0.05);
    expect(parsed.resultLine.num_turns).toBe(3);
    expect(parsed.resultLine.result).toBe('Done.');
  });

  test('counts turns correctly — one per assistant event, not per text block', () => {
    const parsed = parseNDJSON(FIXTURE_LINES);
    // 3 assistant events in fixture (tool_use, text, text+tool_use)
    expect(parsed.turnCount).toBe(3);
  });

  test('handles empty input', () => {
    const parsed = parseNDJSON([]);
    expect(parsed.transcript).toHaveLength(0);
    expect(parsed.resultLine).toBeNull();
    expect(parsed.turnCount).toBe(0);
    expect(parsed.toolCallCount).toBe(0);
    expect(parsed.toolCalls).toHaveLength(0);
  });

  test('handles assistant event with no content array', () => {
    const lines = [
      '{"type":"assistant","message":{}}',
      '{"type":"assistant"}',
    ];
    const parsed = parseNDJSON(lines);
    expect(parsed.turnCount).toBe(2);
    expect(parsed.toolCalls).toHaveLength(0);
  });

  test('associates Agent verdict text with its tool-use ID without transport metadata or reasoning', () => {
    const lines = [
      { type: 'assistant', message: { content: [
        { type: 'tool_use', id: 'review', name: 'Agent', input: { prompt: 'Review the design.' } },
      ] } },
      { type: 'user', tool_use_result: { content: [
        { type: 'thinking', thinking: 'Private computation must not become tool output.' },
        { type: 'text', text: 'Completeness: missing failure handling.' },
        { type: 'text', text: 'Quality score: 7/10' },
      ] }, message: { content: [
        { type: 'tool_result', tool_use_id: 'review', content: [
          { type: 'text', text: 'Completeness: missing failure handling.\nQuality score: 7/10' },
          { type: 'text', text: 'agentId: child-review\n<usage>duration_ms: 1000</usage>' },
        ] },
      ] } },
    ].map(event => JSON.stringify(event));

    expect(parseNDJSON(lines).toolCalls).toEqual([{
      tool: 'Agent',
      input: { prompt: 'Review the design.' },
      output: 'Completeness: missing failure handling.\nQuality score: 7/10',
    }]);
  });

  test('preserves Task error-result diagnostics as output', () => {
    const lines = [
      { type: 'assistant', message: { content: [
        { type: 'tool_use', id: 'failed-review', name: 'Task', input: {} },
      ] } },
      { type: 'user', message: { content: [
        { type: 'tool_result', tool_use_id: 'failed-review', is_error: true, content: 'Reviewer failed: deadline exceeded.' },
      ] } },
    ].map(event => JSON.stringify(event));

    expect(parseNDJSON(lines).toolCalls[0].output).toBe('Reviewer failed: deadline exceeded.');
  });

  test('flattens only public text blocks from matching message results', () => {
    const lines = [
      { type: 'assistant', message: { content: [
        { type: 'tool_use', id: 'legacy-task', name: 'Task', input: {} },
        { type: 'tool_use', id: 'shell', name: 'Bash', input: { command: 'echo done' } },
      ] } },
      { type: 'user', message: { content: [
        { type: 'tool_result', tool_use_id: 'legacy-task', content: [
          { type: 'text', text: 'Consistency: PASS' },
          { type: 'image', source: { data: 'not-text' } },
          { type: 'thinking', thinking: 'Private computation.' },
          { type: 'redacted_thinking', data: 'opaque' },
          null,
          { type: 'text', text: 123 },
          { type: 'text', text: 'Quality score: 10/10' },
        ] },
        { type: 'tool_result', tool_use_id: 'shell', content: 'done\n' },
      ] } },
    ].map(event => JSON.stringify(event));

    const parsed = parseNDJSON(lines);
    expect(parsed.toolCalls.map(call => call.output)).toEqual([
      'Consistency: PASS\nQuality score: 10/10', 'done\n',
    ]);
    expect(parsed.toolCallCount).toBe(2);
    expect(parsed.turnCount).toBe(1);
  });

  test('leaves missing, unmatched, and malformed results empty', () => {
    const lines = [
      { type: 'assistant', message: { content: [
        { type: 'tool_use', id: 'missing', name: 'Agent', input: {} },
        { type: 'tool_use', id: 'malformed', name: 'Task', input: {} },
        { type: 'tool_use', name: 'Read', input: {} },
      ] } },
      { type: 'user', message: { content: [
        { type: 'tool_result', tool_use_id: 'unknown', content: 'Do not attach to the latest call.' },
        { type: 'tool_result', tool_use_id: 'malformed', content: { text: 'Not a public content block.' } },
        { type: 'tool_result', content: 'No tool-use ID.' },
      ] } },
      { type: 'user', message: { content: 'Not a tool-result array.' } },
    ].map(event => JSON.stringify(event));

    expect(parseNDJSON(lines).toolCalls.map(call => call.output)).toEqual(['', '', '']);
  });

  test('scopes repeated tool-use IDs to the parent so child results cannot replace the parent verdict', () => {
    const lines = [
      { type: 'assistant', parent_tool_use_id: null, message: { content: [
        { type: 'tool_use', id: 'shared', name: 'Agent', input: { prompt: 'Parent review' } },
      ] } },
      { type: 'assistant', parent_tool_use_id: 'shared', message: { content: [
        { type: 'tool_use', id: 'shared', name: 'Read', input: { file_path: '/tmp/design.md' } },
      ] } },
      { type: 'user', parent_tool_use_id: 'shared', message: { content: [
        { type: 'tool_result', tool_use_id: 'shared', content: 'Child file content' },
      ] } },
      { type: 'user', message: { content: [
        { type: 'tool_result', tool_use_id: 'shared', content: 'Parent review verdict' },
      ] } },
      { type: 'user', parent_tool_use_id: 'another-child', message: { content: [
        { type: 'tool_result', tool_use_id: 'shared', content: 'Unrelated child result' },
      ] } },
    ].map(event => JSON.stringify(event));

    const parsed = parseNDJSON(lines);
    expect(parsed.toolCalls.map(call => call.output)).toEqual(['Parent review verdict', 'Child file content']);
    expect(parsed.toolCallCount).toBe(2);
    expect(parsed.turnCount).toBe(2);
  });
});

describe('browseNavigatedTo', () => {
  const trials = captured_scrape_goto.trials.map(trial => ({
    url: `file://${trial.cwd}/fixture.html`,
    calls: parseNDJSON(trial.events.map(event => JSON.stringify(event))).toolCalls,
  }));
  const bash = (command: string, output: string) => [{ tool: 'Bash', input: { command }, output }];

  test('credits every captured scrape-prototype-path goto file://$PWD/fixture.html (run 42571575838)', () => {
    expect(trials).toHaveLength(3);
    for (const { url, calls } of trials) {
      expect(calls.some(call => String(call.input.command).includes(url))).toBe(false);
      expect(browseNavigatedTo(calls, url)).toBe(true);
    }
  });

  test('a different file, an echoed URL, another path or a failed navigation is not credited', () => {
    const { url, calls } = trials[0]!;
    expect(browseNavigatedTo(calls, url.replace('fixture.html', 'other.html'))).toBe(false);
    expect(browseNavigatedTo(bash(`echo "${url}"`, url), url)).toBe(false);
    expect(browseNavigatedTo(bash(`echo "Navigated to ${url} (200)"`, `Navigated to ${url} (200)`), url)).toBe(false);
    expect(browseNavigatedTo(bash('$B goto file:///tmp/elsewhere/fixture.html', 'Navigated to file:///tmp/elsewhere/fixture.html (200)'), url)).toBe(false);
    expect(browseNavigatedTo(bash(`$B goto ${url}`, `Error: page.goto: net::ERR_FILE_NOT_FOUND at ${url}`), url)).toBe(false);
    expect(browseNavigatedTo(bash(`$B goto ${url}`, `Navigated to ${url} (404)`), url)).toBe(false);
  });
});