gstack/test/e2e-shard-reuse.test.ts

import { describe, expect, test } from 'bun:test';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import {
  e2eReuseEnvironment, e2eReuseLaneProblem, e2eShardIdentity, e2eShardInputFiles, prepareE2EShardReuse,
  type E2EShardReuseRequest,
} from '../scripts/e2e-shard-reuse';
import { buildRunManifest, fileCaseRegistration, runPaidShard, verifySliceResults, type SliceResult } from '../scripts/test-paid-shards';

const ROOT = path.resolve(import.meta.dir, '..');
const FILE = 'test/skill-e2e-deploy.test.ts';
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-reuse-'));
const bin = path.join(scratch, 'bin');
fs.mkdirSync(bin);
fs.writeFileSync(path.join(bin, 'claude'), '#!/bin/sh\necho "9.9.9 (Claude Code)"\n', { mode: 0o755 });

const laneEnv = (over: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv => ({
  PATH: `${bin}${path.delimiter}${process.env.PATH}`, HOME: scratch,
  EVALS_TIER: 'gate', EVALS_PROFILE: 'pr', EVALS: '1',
  EVALS_CACHE_DIR: path.join(scratch, 'cache'), EVALS_CACHE_REPOSITORY: 'garrytan/gstack', EVALS_CACHE_PR: '42',
  EVALS_CACHE_RUNTIME_ID: 'a'.repeat(64), GITHUB_RUN_ID: '1001', GITHUB_RUN_ATTEMPT: '1', ANTHROPIC_API_KEY: 'sk-fixture',
  GSTACK_CLAUDE_CLI_VERSION: '9.9.9 (Claude Code)',
  ...over,
});

function request(over: Partial<E2EShardReuseRequest> = {}): E2EShardReuseRequest {
  const { registered, known } = fileCaseRegistration(FILE, fs.readFileSync(path.join(ROOT, FILE), 'utf8'));
  return { root: ROOT, key: FILE, file: FILE, caseIds: ['setup-deploy-workflow'], registeredIds: registered, registrationKnown: known,
    casePattern: '(?:^|\\s)(?:setup-deploy-workflow)$', expectedCases: 1, retries: 0, timeoutMs: 1_800_000,
    withinShardConcurrency: 2, tier: 'gate', profile: 'pr', env: laneEnv(), ...over };
}

describe('E2E shard reuse eligibility', () => {
  test('only the same-PR fast profile with an immutable runtime and default endpoint may reuse', () => {
    expect(e2eReuseLaneProblem(laneEnv(), 'pr')).toBeNull();
    for (const [env, mode, problem] of [
      [laneEnv(), 'full-fallback', 'Only the fast PR profile'],
      [laneEnv(), undefined, 'Only the fast PR profile'],
      [laneEnv({ EVALS_CACHE_PR: '' }), 'pr', 'same-PR cache scope'],
      [laneEnv({ EVALS_CACHE_RUNTIME_ID: 'latest' }), 'pr', 'immutable runtime'],
      [laneEnv({ EVALS_FRESH: '1' }), 'pr', 'Fresh validation'],
      [laneEnv({ EVALS_TIER: 'periodic' }), 'pr', 'Fresh validation'],
      [laneEnv({ EVALS_CACHE_PURPOSE: 'periodic' }), 'pr', 'execute fresh'],
      [laneEnv({ EVALS_CACHE_PURPOSE: 'marathon' }), 'pr', 'execute fresh'],
      [laneEnv({ EVALS_CACHE_PURPOSE: 'release' }), 'pr', 'execute fresh'],
      [laneEnv({ NODE_OPTIONS: '--require x' }), 'pr', 'Preload'],
      [laneEnv({ ANTHROPIC_BASE_URL: 'https://proxy.example' }), 'pr', 'Custom model endpoint'],
    ] as const) expect(e2eReuseLaneProblem(env, mode)).toContain(problem);
  });

  test('the identity binds the child environment except run-scoped transport, and never secret values', () => {
    const env = e2eReuseEnvironment(laneEnv({ EVALS_RUN_ID: 'run-1', GSTACK_EVAL_DIR: '/tmp/x', EVALS_SELECTION_JSON: '{}', EVALS_MODEL: 'm', UNRELATED: 'x' }));
    expect(env.ANTHROPIC_API_KEY).toBe('set');
    expect(env.EVALS_MODEL).toBe('m');
    for (const name of ['EVALS_RUN_ID', 'GSTACK_EVAL_DIR', 'EVALS_SELECTION_JSON', 'EVALS_CACHE_DIR', 'EVALS_CACHE_PR', 'UNRELATED', 'GITHUB_RUN_ID']) {
      expect(env[name], name).toBeUndefined();
    }
    expect(JSON.stringify(env)).not.toContain('sk-fixture');
  });

  test('consumed files cover the test closure, every registered touchfile, the globals and the harness', () => {
    const files = e2eShardInputFiles(request());
    for (const file of [FILE, 'test/helpers/e2e-helpers.ts', 'scripts/test-paid-shards.ts', 'scripts/e2e-shard-reuse.ts',
      'bun.lock', '.github/workflows/evals.yml', '.github/actions/register-gstack-skills/action.yml', '.github/docker/Dockerfile.ci',
      'setup-deploy/SKILL.md.tmpl', 'test/helpers/touchfiles-data.ts']) expect(files, file).toContain(file);
    expect(files).not.toContain('package.json');
    expect(files.some(file => file.startsWith('node_modules/'))).toBe(true);
    expect(() => e2eShardInputFiles({ ...request(), registeredIds: ['no-such-case'] })).not.toThrow();
  });

  test('unknown or unprovable inputs fail closed', () => {
    expect(e2eShardIdentity(request()).status).toBe('eligible');
    for (const [over, reason] of [
      [{ retries: 1 }, 'first attempt'],
      [{ registrationKnown: false }, 'statically complete'],
      [{ caseIds: [] }, 'exactly known'],
      [{ expectedCases: 2 }, 'exactly known'],
      [{ caseIds: ['not-registered'] }, 'exactly known'],
      [{ file: 'test/skill-llm-eval.test.ts', key: 'test/skill-llm-eval.test.ts' }, 'audited E2E file'],
      [{ env: laneEnv({ PATH: path.join(scratch, 'empty') }) }, 'Claude CLI version is unknown'],
    ] as const) {
      const result = e2eShardIdentity(request(over as Partial<E2EShardReuseRequest>));
      expect(result.status, reason).toBe('ineligible');
      expect(result.status === 'ineligible' ? result.reason : '').toContain(reason);
    }
  });

  test('any consumed parameter, pin or runtime change is a different identity', () => {
    const key = (over: Partial<E2EShardReuseRequest>) => {
      const result = e2eShardIdentity(request(over));
      if (result.status !== 'eligible') throw new Error(result.reason);
      return result.identity.key;
    };
    const base = key({});
    expect(key({})).toBe(base);
    expect(key({ env: laneEnv({ EVALS_RUN_ID: 'another-run', GITHUB_RUN_ID: '9' }) })).toBe(base);
    for (const over of [{ timeoutMs: 1_000 }, { withinShardConcurrency: 1 }, { casePattern: 'x' },
      { env: laneEnv({ EVALS_MODEL: 'other' }) }, { env: laneEnv({ EVALS_CACHE_RUNTIME_ID: 'b'.repeat(64) }) },
      { env: laneEnv({ EVALS_CACHE_PR: '43' }) }] as Array<Partial<E2EShardReuseRequest>>) expect(key(over)).not.toBe(base);
  });
});

describe('E2E shard reuse through the runner', () => {
  test('a fresh first-attempt pass publishes; identical inputs then reuse without launching; changed inputs run', async () => {
    const env = laneEnv({ EVALS_CACHE_DIR: path.join(scratch, 'roundtrip') });
    const first = prepareE2EShardReuse(request({ env }))!;
    expect(first.lookup()).toBeNull();
    first.publish();
    const hit = prepareE2EShardReuse(request({ env }))!.lookup();
    expect(hit?.source.runId).toBe('1001/1');
    expect(prepareE2EShardReuse(request({ env: { ...env, EVALS_MODEL: 'changed' } }))!.lookup()).toBeNull();
    expect(prepareE2EShardReuse(request({ env: { ...env, EVALS_FRESH: '1' } }))).toBeNull();

    const evalDir = path.join(scratch, 'evals');
    let launched = 0;
    const outcome = await runPaidShard([FILE], 1, 1, { rootDir: ROOT, logDir: scratch, evalDirBase: evalDir, env, log: () => {},
      expectedCaseIds: { [FILE]: ['setup-deploy-workflow'] },
      reuseFor: (files, childEnv) => prepareE2EShardReuse(request({ env: { ...childEnv } })),
      commandFor: () => { launched++; return { command: process.execPath, args: ['-e', 'process.exit(1)'] }; } });
    expect(launched).toBe(0);
    expect(outcome).toMatchObject({ status: 'passed', exitCode: 0, executedTests: 1, skippedTests: 0, reused: { runId: '1001/1' } });
    const recorded = JSON.parse(fs.readFileSync(path.join(evalDir, 'shards', 'skill-e2e-deploy', 'e2e-reused-skill-e2e-deploy.json'), 'utf8'));
    expect(recorded.tests).toEqual([expect.objectContaining({ name: 'setup-deploy-workflow', passed: true, execution: 'reused' })]);
  });

  test('a failed shard never publishes a receipt', async () => {
    let published = 0;
    const outcome = await runPaidShard([FILE], 1, 1, { rootDir: ROOT, logDir: scratch, env: laneEnv(), log: () => {},
      reuseFor: () => ({ lookup: () => null, publish: () => { published++; } }),
      commandFor: () => ({ command: process.execPath, args: ['-e', 'process.exit(1)'] }) });
    expect(outcome.status).toBe('failed');
    expect(published).toBe(0);
  });

  test('the report accepts reused results only in the fast PR profile', () => {
    const manifest = buildRunManifest({ tier: 'gate', sliceCount: 1, evalsAll: true, env: { EVALS_ALL: '1' } });
    const planned = manifest.entries.filter(entry => entry.status === 'planned');
    const reused = { inputKey: 'c'.repeat(64), runId: '1001/1', revision: 'd'.repeat(40), completedAt: 1 };
    const results: SliceResult[] = [{ version: 1, tier: 'gate', sliceIndex: 1, sliceCount: 1, outcomes: planned.map(entry => ({
      files: [entry.file], status: 'passed' as const, exitCode: 0, elapsedMs: 0, executedTests: 1, skippedTests: 0,
      ...(entry.budget ? { budget: entry.budget } : {}), ...(entry.file === FILE ? { reused } : {}) })) }];
    expect(verifySliceResults(manifest, results).problems).toContain(`${FILE}: only the fast PR profile may reuse results; this lane executes fresh`);
  });
});