import { expect, test } from 'bun:test'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { runPaidShard, shardSlug } from '../scripts/test-paid-shards'; const ROOT = path.resolve(import.meta.dir, '..'); const workflows = ['evals.yml', 'evals-periodic.yml'].map(name => ({ name, value: Bun.YAML.parse(fs.readFileSync(path.join(ROOT, '.github/workflows', name), 'utf8')) as any, })); const executors = workflows.flatMap(({ name, value }) => Object.entries(value.jobs) .flatMap(([jobName, job]: [string, any]) => job.steps .filter((step: any) => step.run?.includes('scripts/test-paid-shards.ts') && step.run.includes(' --slice ')) .map((step: any) => ({ name, jobName, job, step })))); function render(template: string, fields: Record): string { return template.replace(/\$\{\{\s*([^}]+?)\s*\}\}/g, (_, key) => { if (!(key in fields)) throw new Error(`Unbound CI expression: ${key}`); return fields[key]; }); } test('every direct CI paid executor binds a safe unique run/attempt/job/slice identity', () => { expect(executors.map(({ name, jobName }) => `${name}:${jobName}`)).toEqual([ 'evals.yml:eval-slices', 'evals-periodic.yml:eval-slices', 'evals-periodic.yml:gate-census', ]); const ids = new Set(); for (const [workflowIndex, { job, step }] of executors.entries()) { expect(job.container.options).toBe('--user runner'); const env = { ...job.env, ...step.env }; expect(env.EVALS_RUN_ID).toBeString(); for (const run of ['36302678692', '36302678693']) { for (const attempt of ['1', '2']) { for (const slice of job.strategy.matrix.slice) { const id = render(env.EVALS_RUN_ID, { 'github.run_id': `${run}${workflowIndex === 0 ? '0' : '1'}`, 'github.run_attempt': attempt, 'matrix.slice': String(slice), }); expect(id).toMatch(/^[A-Za-z0-9_-]+$/); expect(id.length).toBeLessThan(120); expect(ids.has(id)).toBe(false); ids.add(id); } } } } }); for (const { name, jobName, job, step } of executors) { test(`${name}:${jobName} passes its rendered identity through a real shard and retains native evidence after cleanup`, async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), 'ci-native-')); const home = path.join(root, 'home'); const bin = path.join(root, 'bin'); fs.mkdirSync(home); fs.mkdirSync(bin); fs.mkdirSync(path.join(root, 'test')); const configured = { ...job.env, ...step.env }; const runId = render(configured.EVALS_RUN_ID, { 'github.run_id': '36302678692', 'github.run_attempt': '2', 'matrix.slice': '4', }); const evalDir = path.join(root, path.basename(configured.GSTACK_EVAL_DIR)); const file = 'test/native-launch.test.ts'; fs.writeFileSync(path.join(bin, 'claude'), `#!${process.execPath} console.log(JSON.stringify({type: 'result', subtype: 'success', result: JSON.stringify({runId: process.env.EVALS_RUN_ID, leakedToken: !!process.env.GITHUB_TOKEN})})); `, { mode: 0o700 }); fs.writeFileSync(path.join(root, file), ` import { expect, test } from 'bun:test'; import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { runSkillTest } from ${JSON.stringify(path.join(ROOT, 'test/helpers/session-runner.ts'))}; import { fixtureDocs, preserveDocsEvidence } from ${JSON.stringify(path.join(ROOT, 'test/helpers/docsync-fixture.ts'))}; import { persistPlanCountSnapshot } from ${JSON.stringify(path.join(ROOT, 'test/helpers/plan-count-artifacts.ts'))}; test('native launch plumbing without a model', async () => { expect(process.env.EVALS_RUN_ID).toBe(${JSON.stringify(runId)}); const fixture = fixtureDocs('risky'); const result = await runSkillTest({prompt: 'fixture', workingDirectory: fixture.repo, model: 'fixture', timeout: 5000, startupGraceMs: 5000, allowedTools: [], testName: 'ci-native', runId: process.env.EVALS_RUN_ID, env: fixture.env}); expect(result.exitReason).toBe('success'); expect(JSON.parse(result.output)).toEqual({runId: process.env.EVALS_RUN_ID, leakedToken: false}); const docs = preserveDocsEvidence(fixture, result, process.env.EVALS_RUN_ID!, 'ci-native'); const snapshots = [0, 1].map(attempt => persistPlanCountSnapshot({skillName: 'ci-native', observation: {attempt}, raw: 'native-raw-' + attempt, visible: 'native-visible', cwd: fixture.repo, claudeConfigDir: fixture.env.CLAUDE_CONFIG_DIR})); fixture.clean(); expect(fs.existsSync(fixture.home)).toBe(false); expect(fs.existsSync(docs)).toBe(true); expect(snapshots[0].artifactDir).not.toBe(snapshots[1].artifactDir); for (const snapshot of snapshots) { expect(snapshot.artifactError).toBeUndefined(); expect(fs.existsSync(path.join(snapshot.artifactDir!, 'observation.json'))).toBe(true); } fs.writeFileSync(path.join(process.env.GSTACK_EVAL_DIR!, 'smoke.json'), JSON.stringify({ docs, snapshots, shardTmp: os.tmpdir(), fixtureRoot: fixture.home, runId: process.env.EVALS_RUN_ID, evalDir: process.env.GSTACK_EVAL_DIR, })); }); `); try { const outcome = await runPaidShard([file], 1, 1, { rootDir: root, evalDirBase: evalDir, logDir: root, jobs: 2, log: () => {}, timeoutMs: 30_000, env: { PATH: `${bin}${path.delimiter}${process.env.PATH}`, HOME: home, EVALS_RUN_ID: runId, GSTACK_EVAL_DIR: configured.GSTACK_EVAL_DIR, GSTACK_CLAUDE_CLI_VERSION: 'synthetic-no-model', GITHUB_TOKEN: 'synthetic-token' }, }); expect(outcome.status).toBe('passed'); const shardDir = path.join(evalDir, 'shards', shardSlug([file])); const result = JSON.parse(fs.readFileSync(path.join(shardDir, 'smoke.json'), 'utf8')); expect(result).toMatchObject({ runId, evalDir: shardDir }); expect(fs.existsSync(result.shardTmp)).toBe(false); expect(fs.existsSync(result.fixtureRoot)).toBe(false); expect(fs.existsSync(result.docs)).toBe(true); for (const snapshot of result.snapshots) { const record = JSON.parse(fs.readFileSync(path.join(snapshot.artifactDir, 'observation.json'), 'utf8')); expect(record.capture.runId).toBe(runId); expect(snapshot.artifactDir.startsWith(shardDir + path.sep)).toBe(true); } const upload = job.steps.find((candidate: any) => candidate.with?.path === configured.GSTACK_EVAL_DIR); expect(upload.if).toBe('always()'); const captures = job.steps.find((candidate: any) => candidate.name === 'Upload native capture evidence'); expect(captures.if).toBe('always()'); expect(captures.with['include-hidden-files']).toBe(true); expect(captures.with['retention-days']).toBe(90); const artifactName = render(captures.with.name, { 'env.EVALS_RUN_ID': runId }); expect(artifactName).toBe(`native-captures-${runId}`); expect(artifactName).not.toMatch(/^(paid-slice|gate-census)-[0-9]/); const patterns = captures.with.path.trim().split('\n'); expect(patterns).toEqual(['~/.gstack/projects/*/e2e-runs', '~/.gstack/projects/*/evals/qa-callers', '~/.gstack-dev/e2e-runs', '~/.gstack-dev/evals/qa-callers']); const uploaded = patterns.flatMap((pattern: string) => [...new Bun.Glob(`${pattern.replace(/^~\//, '')}/**/*`) .scanSync({ cwd: home, absolute: true, dot: true, onlyFiles: true })]); expect(uploaded).toContain(result.docs); expect(uploaded.some((file: string) => file.endsWith('ci-native.ndjson'))).toBe(true); } finally { fs.rmSync(root, { recursive: true, force: true }); } }, 60_000); }