mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-03 07:18:03 +01:00
Garry Tanandcapy-ai[bot]
85b0f531f5
* feat(ship-queue): merge trains with pattern bisect and train-record verify (GSTA-49 T1) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test(windows): curate ship-queue-train.test.ts out of the Windows subset (gh shim not resolved on windows-latest) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * feat(evals): merge-train workflow side, inert behind the train policy block (GSTA-49 T1b) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore(evals): T1b CHANGELOG under [Unreleased]; classify evals-main-backstop.yml as free-lane only Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(evals): plan-design-with-ui focus detector accepts Opus 5.5's 'Review focus' title with the step's rating and an all-7 option; replay run 39298269902 Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * v1.91.162.0 feat(evals): merge-train workflow side, inert behind the train policy block (GSTA-49 T1b) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
589 lines
42 KiB
TypeScript
589 lines
42 KiB
TypeScript
/**
|
|
* Sliced-lane wiring pins for the paid CI workflows — the successor to
|
|
* evals-workflow-matrix.test.ts, which enforced completeness of a
|
|
* hand-enumerated 17-row matrix (and carried KNOWN_MATRIX_GAPS /
|
|
* KNOWN_TIER_UNSET burn-down ratchets for the files that matrix missed).
|
|
* The matrix is deleted: the sliced lane's planner derives the gate census
|
|
* from the runner itself (collectPaidTestFiles + tier selection), so "every
|
|
* gate-hosting file is in the census" is true BY CONSTRUCTION and the
|
|
* burn-down ratchets retired with the rows.
|
|
*
|
|
* What still needs pinning is the WIRING — the yml plumbing that free tests
|
|
* are the only guard for:
|
|
* - the legacy matrix (and its `needs: evals` serialization) stays deleted,
|
|
* - planner/executor/report all run tier=gate and agree on the slice count,
|
|
* - both surviving lanes register skills through the SHARED composite that
|
|
* carries the fail-fast dangling-symlink/frontmatter verification loop
|
|
* (the sliced + periodic copies had silently dropped it — the loop was
|
|
* written after a silent "Unknown command" + 35-min-timeout incident),
|
|
* - the PR comment survives the matrix-report deletion (it moved into
|
|
* slices-report, keyed on the same "## E2E Evals" upsert marker).
|
|
*/
|
|
import { describe, test, expect } from 'bun:test';
|
|
import * as fs from 'fs';
|
|
import * as path from 'path';
|
|
import * as os from 'os';
|
|
import { spawnSync } from 'child_process';
|
|
import { buildRunManifest, isCodexShard, isOverlayTestFile, parseCliOptions, resolvePaidShardTimeoutMs, sliceExecutionOrder, sliceSupervisedWallMs, CI_SETUP_ALLOWANCE_MINUTES } from '../scripts/test-paid-shards';
|
|
import { CENSUS_ARMS, MONTHLY_CENSUS_ARMS, MONTHLY_CENSUS_SCHEDULE, WEEKLY_CENSUS_ARMS, WEEKLY_CENSUS_SCHEDULE, censusArmEnv, censusMatrix, selectCensus } from '../scripts/census-arms';
|
|
|
|
const ROOT = path.join(import.meta.dir, '..');
|
|
const read = (rel: string) => fs.readFileSync(path.join(ROOT, rel), 'utf-8');
|
|
|
|
const evalsYml = read('.github/workflows/evals.yml');
|
|
const periodicYml = read('.github/workflows/evals-periodic.yml');
|
|
const marathonYml = read('.github/workflows/evals-marathon.yml');
|
|
const registerAction = read('.github/actions/register-gstack-skills/action.yml');
|
|
|
|
/** Every planner site: its manifest path and budget (`--slice-budget S --jobs J`). */
|
|
function plannerSites(source: string): Array<{ manifest: string; budgetSeconds: number; jobs: number }> {
|
|
return [...source.matchAll(/--emit-plan\s+(\S+)\s+--slice-budget\s+(\d+)\s+--jobs\s+(\d+)/g)]
|
|
.map((m) => ({ manifest: m[1]!, budgetSeconds: Number(m[2]), jobs: Number(m[3]) }));
|
|
}
|
|
|
|
type Step = { id?: string; name?: string; run?: string; uses?: string; env?: Record<string, string>; with?: Record<string, string> };
|
|
type Job = { needs?: string[]; env?: Record<string, string>; outputs?: Record<string, string>; 'timeout-minutes': string | number;
|
|
strategy?: { 'max-parallel': number; matrix: { slice?: string; include?: string } }; steps: Step[] };
|
|
|
|
/**
|
|
* An executor's matrix and timeout must come from the planner step that wrote
|
|
* the manifest it downloads: `slices` from `[range(1; .sliceCount + 1)]` and
|
|
* `timeout-minutes` from the plan, never hand-written numbers: per slice from
|
|
* `.plan.sliceCiTimeoutMinutes` (indexed by matrix.slice, W2c/ENG-2), or one
|
|
* `.plan.ciTimeoutMinutes` (the largest per-slice ceiling) for every slice.
|
|
*/
|
|
function expectPlannedExecutor(source: string, executorName: string, prefix: string) {
|
|
const workflow = Bun.YAML.parse(source) as { jobs: Record<string, Job> };
|
|
const planner = workflow.jobs['plan-slices']!;
|
|
const executor = workflow.jobs[executorName]!;
|
|
expect(executor.needs).toContain('plan-slices');
|
|
expect(executor.strategy!.matrix.slice).toBe(`\${{ fromJSON(needs.plan-slices.outputs.${prefix}slices) }}`);
|
|
const perSlice = executor['timeout-minutes'] === `\${{ fromJSON(needs.plan-slices.outputs.${prefix}slice_timeouts)[matrix.slice] }}`;
|
|
if (!perSlice) expect(executor['timeout-minutes']).toBe(`\${{ fromJSON(needs.plan-slices.outputs.${prefix}timeout_minutes) }}`);
|
|
const [stepId] = /^\$\{\{ steps\.([\w-]+)\.outputs\.slices \}\}$/.exec(planner.outputs![`${prefix}slices`]!)!.slice(1);
|
|
const output = perSlice ? 'slice_timeouts' : 'timeout_minutes';
|
|
expect(planner.outputs![`${prefix}${output}`]).toBe(`\${{ steps.${stepId}.outputs.${output} }}`);
|
|
const matrixStep = planner.steps.find(step => step.id === stepId)!;
|
|
const manifest = /jq -c '\[range\(1; \.sliceCount \+ 1\)\](?: - \.plan\.codexSlices)?' (\S+)\)/.exec(matrixStep.run!)![1]!;
|
|
expect(matrixStep.run).toContain(perSlice ? `jq -ec '[0] + .plan.sliceCiTimeoutMinutes' ${manifest})` : `jq -e '.plan.ciTimeoutMinutes' ${manifest})`);
|
|
if (perSlice) {
|
|
// The deadline's clock: the executor's first step records the job start.
|
|
expect(executor.steps[0]!.run).toBe('echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV"');
|
|
}
|
|
const emit = planner.steps.filter(step => step.run?.includes(`--emit-plan ${manifest} `));
|
|
expect(emit).toHaveLength(1);
|
|
const execute = executor.steps.filter(step => step.run?.includes('--plan '));
|
|
expect(execute).toHaveLength(1);
|
|
expect(execute[0]!.run).toContain(`--plan ${manifest} --slice \${{ matrix.slice }}`);
|
|
expect(executor.steps.some(step => step.with?.path === manifest.replace(/\/manifest\.json$/, ''))).toBe(true);
|
|
// The planner packs for exactly the executor's worker count.
|
|
const site = plannerSites(emit[0]!.run!)[0]!;
|
|
expect(execute[0]!.env?.EVALS_JOBS).toBe(String(site.jobs));
|
|
return { site, emit: emit[0]!, execute: execute[0]!, executor, planner };
|
|
}
|
|
|
|
describe('evals.yml sliced-lane wiring (post-matrix)', () => {
|
|
test('the legacy matrix job stays deleted', () => {
|
|
// Row-enumeration shapes from the deleted matrix. Any reappearance means
|
|
// someone is re-growing a hand-maintained enumeration next to a lane
|
|
// whose census is derived — the drift class the deletion killed.
|
|
expect(evalsYml).not.toMatch(/^\s+suite:\s*$/m);
|
|
expect(evalsYml).not.toMatch(/^\s+file: test\//m);
|
|
expect(evalsYml).not.toContain('needs: [build-image, evals]');
|
|
expect(evalsYml).not.toMatch(/^\s+needs: evals\s*$/m);
|
|
});
|
|
|
|
test('no workflow-level EVALS_TIER env (each command sets its own)', () => {
|
|
// The workflow-level `EVALS_TIER: gate` was dead config once every
|
|
// consumer set its own; a resurrected copy would silently leak gate
|
|
// semantics into steps that must choose explicitly.
|
|
expect(evalsYml).not.toMatch(/^env:[\s\S]{0,120}^\s+EVALS_TIER:/m);
|
|
});
|
|
|
|
test('planner, executors, and report all run tier=gate on the shared runner', () => {
|
|
expect(evalsYml).toMatch(/EVALS_TIER=gate bun --no-install run scripts\/test-paid-shards\.ts --tier gate --emit-plan/);
|
|
expect(evalsYml).toMatch(/EVALS_TIER=gate bun run scripts\/test-paid-shards\.ts --tier gate --plan .* --slice /);
|
|
expect(evalsYml).toMatch(/EVALS_TIER=gate bun --no-install run scripts\/test-paid-shards\.ts --tier gate --report /);
|
|
});
|
|
|
|
test('executor matrix and timeout come from the one budget planner', () => {
|
|
expect(plannerSites(evalsYml), 'expected exactly one --emit-plan site in evals.yml').toHaveLength(1);
|
|
const { site } = expectPlannedExecutor(evalsYml, 'eval-slices', '');
|
|
expect(site).toEqual({ manifest: '/tmp/paid-plan/manifest.json', budgetSeconds: 420, jobs: 2 });
|
|
});
|
|
|
|
test('reconcile exit is captured via PIPESTATUS, never $? after a pipe', () => {
|
|
// GitHub's default run-step shell is `bash -e {0}` with NO pipefail, so
|
|
// `$?` after `... | tee` is tee's exit — always 0. That made the
|
|
// fail-closed reconcile gate silently fail-open (ship review army,
|
|
// 2026-08-31). Both lanes must read PIPESTATUS[0].
|
|
for (const [name, source] of [['evals.yml', evalsYml], ['evals-periodic.yml', periodicYml], ['evals-marathon.yml', marathonYml]] as const) {
|
|
const reconcileBlocks = [...source.matchAll(/--report[^\n]*\| tee[^\n]*\n([\s\S]{0,400}?)GITHUB_OUTPUT/g)];
|
|
expect(reconcileBlocks.length, `${name}: expected a tee'd reconcile step`).toBeGreaterThanOrEqual(1);
|
|
for (const block of reconcileBlocks) {
|
|
expect(block[1], `${name} reconcile captures tee's exit, not the runner's`).toContain('PIPESTATUS[0]');
|
|
expect(block[1]).not.toMatch(/exit=\$\?/);
|
|
}
|
|
}
|
|
});
|
|
|
|
test('the PR comment survived the matrix-report deletion (moved to slices-comment)', () => {
|
|
// Keyed on the upsert marker so the migration keeps updating the SAME
|
|
// comment; and the job holding it needs the issues permission (#1802).
|
|
expect(evalsYml).toContain('## E2E Evals');
|
|
expect(evalsYml).toMatch(/pull-requests: write/);
|
|
expect(evalsYml).toMatch(/issues: write/);
|
|
});
|
|
|
|
test('the write-token job runs ZERO repo code (token/exec separation)', () => {
|
|
// slices-report executes PR-authored code (bun install + the reconcile
|
|
// runner), so it must hold contents:read ONLY; the write token lives in
|
|
// slices-comment, which may only download artifacts and run jq/gh —
|
|
// $GITHUB_ENV persistence is job-scoped, so this split IS the trust
|
|
// boundary (codex adversarial, 2026-08-31; the matrix-era report job had
|
|
// this property and the consolidation briefly regressed it).
|
|
const commentJob = evalsYml.slice(evalsYml.indexOf(' slices-comment:'));
|
|
expect(commentJob.length).toBeGreaterThan(100);
|
|
expect(commentJob).not.toContain('actions/checkout');
|
|
expect(commentJob).not.toContain('bun install');
|
|
expect(commentJob).not.toMatch(/run: .*bun run/);
|
|
expect(commentJob).not.toContain('uses: ./');
|
|
// No checkout also means no git context: `gh pr comment` resolves the
|
|
// repo FROM git and dies with "not a git repository" here (PR #2746's
|
|
// first run). Every comment call must be explicit-repo REST (gh api).
|
|
expect(commentJob).not.toContain('gh pr comment');
|
|
// And the code-executing report job must NOT hold write scopes.
|
|
const reportJob = evalsYml.slice(evalsYml.indexOf(' slices-report:'), evalsYml.indexOf(' slices-comment:'));
|
|
expect(reportJob).not.toMatch(/pull-requests: write/);
|
|
expect(reportJob).not.toMatch(/issues: write/);
|
|
});
|
|
});
|
|
|
|
describe('evals-periodic.yml sliced-lane wiring', () => {
|
|
const lanes = [
|
|
{ source: periodicYml, name: 'evals-periodic.yml', executor: 'eval-slices', prefix: 'periodic_', tier: 'periodic' },
|
|
{ source: evalsYml, name: 'evals.yml', executor: 'eval-slices', prefix: '', tier: 'gate' },
|
|
{ source: marathonYml, name: 'evals-marathon.yml', executor: 'eval-slices', prefix: '', tier: 'marathon' },
|
|
] as const;
|
|
|
|
for (const lane of lanes) {
|
|
test(`${lane.name}:${lane.executor} — the planned CI job cap covers every slice's supervised wall plus setup, and every slice starts at once`, () => {
|
|
const { emit, execute, executor } = expectPlannedExecutor(lane.source, lane.executor, lane.prefix);
|
|
const cliArgs = (run: string) => {
|
|
const command = /\bbun(?: --no-install)? run scripts\/test-paid-shards\.ts /.exec(run);
|
|
expect(command).not.toBeNull();
|
|
return run.slice(command!.index + command![0].length)
|
|
.replace(/\$\{\{\s*matrix\.slice\s*\}\}/g, '1').trim().split(/\s+/);
|
|
};
|
|
const workflow = Bun.YAML.parse(lane.source) as { env?: Record<string, string> };
|
|
// The complete census (EVALS_ALL) is the largest plan any event can produce.
|
|
const plannerEnv = { ...workflow.env, ...emit.env, EVALS_ALL: '1', EVALS_PROFILE: 'full' };
|
|
const planned = parseCliOptions(cliArgs(emit.run!), plannerEnv);
|
|
const active = parseCliOptions(cliArgs(execute.run!), { ...workflow.env, ...executor.env, ...execute.env, EVALS_PROFILE: 'full' });
|
|
expect(planned.tier).toBe(lane.tier);
|
|
expect(active.tier).toBe(lane.tier);
|
|
expect(active.jobs).toBe(planned.jobs);
|
|
const manifest = buildRunManifest({ tier: planned.tier, profile: 'full', sliceBudgetMs: planned.sliceBudgetMs!, jobs: planned.jobs,
|
|
evalsAll: true, env: plannerEnv, rootDir: ROOT, skipJudges: planned.skipJudges });
|
|
// W2c/ENG-2: each slice's ceiling is max(2x budget, its longest shard's supervised wall, the
|
|
// serialized overlay envelope) + setup; the executor's deadline turns the rest into not_run.
|
|
const sliceFiles = Array.from({ length: manifest.sliceCount }, (_, i) => sliceExecutionOrder(
|
|
manifest.entries.filter(entry => entry.status === 'planned' && entry.slice === i + 1)).map(entry => entry.file));
|
|
const ceilings = sliceFiles.map(files => Math.ceil(Math.max(2 * planned.sliceBudgetMs!,
|
|
...files.map(file => resolvePaidShardTimeoutMs([file])),
|
|
files.some(isOverlayTestFile) ? sliceSupervisedWallMs(files, planned.jobs) : 0) / 60_000) + CI_SETUP_ALLOWANCE_MINUTES);
|
|
expect(CI_SETUP_ALLOWANCE_MINUTES).toBe(20);
|
|
expect(manifest.plan!.sliceCiTimeoutMinutes).toEqual(ceilings);
|
|
expect(manifest.plan!.ciTimeoutMinutes).toBe(Math.max(...ceilings));
|
|
// GitHub-hosted-style job ceiling: a plan past it must be split, not truncated.
|
|
expect(manifest.plan!.ciTimeoutMinutes).toBeLessThanOrEqual(360);
|
|
expect(manifest.sliceCount, `${lane.name}:${lane.executor} plans more slices than max-parallel starts at once`)
|
|
.toBeLessThanOrEqual(executor.strategy!['max-parallel']);
|
|
});
|
|
}
|
|
|
|
test('evals-periodic.yml:gate-census — one row per census arm and planned slice with its plan\'s ceiling, and every row of a scheduled census starts at once', () => {
|
|
const workflow = Bun.YAML.parse(periodicYml) as { env?: Record<string, string>; jobs: Record<string, Job> };
|
|
const planner = workflow.jobs['plan-slices']!;
|
|
const executor = workflow.jobs['gate-census']!;
|
|
expect(executor.needs).toContain('plan-slices');
|
|
expect(executor.strategy!.matrix).toEqual({ include: '${{ fromJSON(needs.plan-slices.outputs.gate_matrix) }}' });
|
|
expect(executor['timeout-minutes']).toBe('${{ matrix.timeout }}');
|
|
expect(executor.steps[0]!.run).toBe('echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV"');
|
|
expect(planner.outputs!.gate_matrix).toBe('${{ steps.gate-matrix.outputs.gate_matrix }}');
|
|
expect(planner.steps.find(step => step.id === 'gate-matrix')!.run).toBe('bun --no-install run scripts/census-arms.ts --matrix /tmp/gate-census-plan "$ARMS" >> "$GITHUB_OUTPUT"');
|
|
const emit = planner.steps.filter(step => step.run?.includes('--emit-plan "/tmp/gate-census-plan/$arm/manifest.json" '));
|
|
expect(emit).toHaveLength(1);
|
|
expect(emit[0]!.run).toContain('mapfile -t arm_env < <(bun --no-install run scripts/census-arms.ts --env "$arm")');
|
|
expect(emit[0]!.run).toContain('env "${arm_env[@]}" EVALS_TIER=gate bun run scripts/test-paid-shards.ts');
|
|
const execute = executor.steps.filter(step => step.run?.includes('--plan '));
|
|
expect(execute).toHaveLength(1);
|
|
expect(execute[0]!.run).toContain('--plan /tmp/gate-census-plan/${{ matrix.arm }}/manifest.json --slice ${{ matrix.slice }}');
|
|
// The executor runs with its arm's host and model, and downloads every arm's plan.
|
|
expect(executor.steps.find(step => step.name === "Select the census arm's host and model")!.run).toBe('bun --no-install run scripts/census-arms.ts --env "$ARM" >> "$GITHUB_ENV"');
|
|
expect(executor.steps.some(step => step.with?.path === '/tmp/gate-census-plan')).toBe(true);
|
|
const cliArgs = (run: string) => {
|
|
const command = /\bbun(?: --no-install)? run scripts\/test-paid-shards\.ts (.*)$/m.exec(run)!;
|
|
return command[1]!.replace(/\$\{\{\s*matrix\.slice\s*\}\}/g, '1').replace(/\$\{\{\s*matrix\.arm\s*\}\}/g, 'claude-opus').trim().split(/\s+/);
|
|
};
|
|
const planned = parseCliOptions(cliArgs(emit[0]!.run!), { ...workflow.env, ...emit[0]!.env, EVALS_ALL: '1', EVALS_PROFILE: 'full' });
|
|
const active = parseCliOptions(cliArgs(execute[0]!.run!), { ...workflow.env, ...executor.env, ...execute[0]!.env, EVALS_PROFILE: 'full' });
|
|
expect(planned.tier).toBe('gate');
|
|
expect(active.tier).toBe('gate');
|
|
expect(active.jobs).toBe(planned.jobs);
|
|
expect(execute[0]!.env?.EVALS_JOBS).toBe(String(planned.jobs));
|
|
const planDir = fs.mkdtempSync(path.join(os.tmpdir(), 'census-arms-'));
|
|
try {
|
|
for (const arm of Object.keys(CENSUS_ARMS)) {
|
|
const env = { ...workflow.env, ...emit[0]!.env, EVALS_ALL: '1', EVALS_PROFILE: 'full', ...censusArmEnv(arm) };
|
|
const manifest = buildRunManifest({ tier: 'gate', profile: 'full', sliceBudgetMs: planned.sliceBudgetMs!, jobs: planned.jobs,
|
|
evalsAll: true, env, rootDir: ROOT, skipJudges: planned.skipJudges });
|
|
expect(manifest.host ?? 'claude').toBe(CENSUS_ARMS[arm]!.host);
|
|
const sliceFiles = Array.from({ length: manifest.sliceCount }, (_, i) => sliceExecutionOrder(
|
|
manifest.entries.filter(entry => entry.status === 'planned' && entry.slice === i + 1)).map(entry => entry.file));
|
|
const ceilings = sliceFiles.map(files => Math.ceil(Math.max(2 * planned.sliceBudgetMs!,
|
|
...files.map(file => resolvePaidShardTimeoutMs([file])),
|
|
files.some(isOverlayTestFile) ? sliceSupervisedWallMs(files, planned.jobs) : 0) / 60_000) + CI_SETUP_ALLOWANCE_MINUTES);
|
|
expect(manifest.plan!.sliceCiTimeoutMinutes).toEqual(ceilings);
|
|
expect(manifest.plan!.ciTimeoutMinutes).toBeLessThanOrEqual(360);
|
|
fs.mkdirSync(path.join(planDir, arm));
|
|
fs.writeFileSync(path.join(planDir, arm, 'manifest.json'), JSON.stringify(manifest));
|
|
expect(censusMatrix([arm], planDir)).toEqual(ceilings.map((timeout, i) => ({ arm, slice: i + 1, timeout })));
|
|
}
|
|
for (const arms of [WEEKLY_CENSUS_ARMS, MONTHLY_CENSUS_ARMS]) {
|
|
const rows = censusMatrix(arms, planDir);
|
|
expect(new Set(rows.map(row => `${row.arm}/${row.slice}`)).size).toBe(rows.length);
|
|
expect(rows.length, `${arms.join('+')} rows exceed max-parallel`).toBeLessThanOrEqual(executor.strategy!['max-parallel']);
|
|
}
|
|
} finally {
|
|
fs.rmSync(planDir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test('planner/executor/report tier=periodic agree and plan with the ~7-minute budget (W5c)', () => {
|
|
expect(periodicYml).toMatch(/EVALS_TIER=periodic bun --no-install run scripts\/test-paid-shards\.ts --tier periodic --emit-plan/);
|
|
expect(periodicYml).toMatch(/EVALS_TIER=periodic bun run scripts\/test-paid-shards\.ts --tier periodic --plan .* --slice /);
|
|
expect(periodicYml).toMatch(/EVALS_TIER=periodic bun --no-install run scripts\/test-paid-shards\.ts --tier periodic --report /);
|
|
// Periodic work and the full gate census have distinct immutable plans.
|
|
expect(plannerSites(periodicYml)).toEqual([
|
|
{ manifest: '/tmp/paid-plan/manifest.json', budgetSeconds: 420, jobs: 2 },
|
|
{ manifest: '"/tmp/gate-census-plan/$arm/manifest.json"', budgetSeconds: 420, jobs: 2 },
|
|
]);
|
|
});
|
|
});
|
|
|
|
describe('evals-periodic.yml host-run Codex slices', () => {
|
|
type HostJob = Job & { if?: string; container?: unknown; 'runs-on': string; permissions?: Record<string, string>;
|
|
steps: Array<Step & { uses?: string; shell?: string }> };
|
|
const jobs = (Bun.YAML.parse(periodicYml) as { jobs: Record<string, HostJob> }).jobs;
|
|
const codex = jobs['eval-codex-slices']!;
|
|
const container = jobs['eval-slices']!;
|
|
const planner = jobs['plan-slices']!;
|
|
const uploadNames = (job: HostJob) => job.steps.filter(step => step.uses?.startsWith('actions/upload-artifact@')).map(step => step.with!.name);
|
|
|
|
test('the planner gives Codex slices to the host job and every other slice to the container job', () => {
|
|
const matrixStep = planner.steps.find(step => step.id === 'periodic-matrix')!;
|
|
expect(matrixStep.run).toContain(`echo "slices=$(jq -c '[range(1; .sliceCount + 1)] - .plan.codexSlices' /tmp/paid-plan/manifest.json)"`);
|
|
expect(matrixStep.run).toContain(`echo "codex_slices=$(jq -c '.plan.codexSlices' /tmp/paid-plan/manifest.json)"`);
|
|
expect(matrixStep.run).toContain('} >> "$GITHUB_OUTPUT"');
|
|
expect(planner.outputs!.periodic_codex_slices).toBe('${{ steps.periodic-matrix.outputs.codex_slices }}');
|
|
expect(codex.needs).toEqual(['build-image', 'plan-slices']);
|
|
expect(codex.if).toBe("${{ needs.plan-slices.outputs.periodic_lane == 'true' && needs.plan-slices.outputs.periodic_codex_slices != '[]' }}");
|
|
expect(codex.strategy!.matrix.slice).toBe('${{ fromJSON(needs.plan-slices.outputs.periodic_codex_slices) }}');
|
|
expect(codex['timeout-minutes']).toBe('${{ fromJSON(needs.plan-slices.outputs.periodic_slice_timeouts)[matrix.slice] }}');
|
|
expect(codex.steps[0]!.run).toBe('echo "GSTACK_SLICE_JOB_STARTED_AT=$(date +%s)" >> "$GITHUB_ENV"');
|
|
const manifest = buildRunManifest({ tier: 'periodic', profile: 'full', sliceBudgetMs: 540_000, jobs: 2, evalsAll: true, env: { EVALS_ALL: '1' }, rootDir: ROOT });
|
|
const codexSlices = manifest.plan!.codexSlices!;
|
|
expect(codexSlices.length).toBeGreaterThan(0);
|
|
expect(codex.strategy!['max-parallel']).toBeGreaterThanOrEqual(codexSlices.length);
|
|
for (const entry of manifest.entries.filter(entry => entry.status === 'planned')) {
|
|
expect(codexSlices.includes(entry.slice), entry.file).toBe(isCodexShard(entry.file, 'periodic'));
|
|
}
|
|
// Gate plans (evals.yml and the weekly gate census) never give a shard Codex, so they have no Codex slices.
|
|
expect(buildRunManifest({ tier: 'gate', profile: 'full', sliceBudgetMs: 540_000, jobs: 2, evalsAll: true, env: { EVALS_ALL: '1' }, rootDir: ROOT })
|
|
.plan!.codexSlices).toEqual([]);
|
|
});
|
|
|
|
test('runs on the runner VM, lifts the user-namespace restriction, then runs the CI image with seccomp and AppArmor unconfined', () => {
|
|
expect(codex.container).toBeUndefined();
|
|
expect(codex['runs-on']).toBe(container['runs-on']);
|
|
expect(codex.permissions).toEqual({ contents: 'read', packages: 'read' });
|
|
const sysctl = codex.steps.findIndex(step => step.run?.includes('sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0'));
|
|
const execute = codex.steps.findIndex(step => step.run?.includes('docker run '));
|
|
expect(sysctl).toBeGreaterThan(-1);
|
|
expect(execute).toBeGreaterThan(sysctl);
|
|
const step = codex.steps[execute]!;
|
|
expect(step.run).toContain('--security-opt seccomp=unconfined --security-opt apparmor=unconfined');
|
|
expect(step.run).toContain('--user "$(id -u):$(id -g)"');
|
|
expect(step.run).not.toMatch(/--privileged|--cap-add|--pid[ =]host|--network[ =]host|docker\.sock/);
|
|
expect(step.env!.EVAL_IMAGE).toBe('${{ needs.build-image.outputs.image-tag }}');
|
|
expect(step.run).toContain('"$EVAL_IMAGE" bash -c');
|
|
expect(step.env!.SLICE).toBe('${{ matrix.slice }}');
|
|
expect(step.run).toContain('EVALS_TIER=periodic bun run scripts/test-paid-shards.ts --tier periodic --plan /tmp/paid-plan/manifest.json --slice "$SLICE"');
|
|
expect(step.run).toContain('-v "$RUNNER_TEMP/paid-plan:/tmp/paid-plan:ro"');
|
|
expect(codex.steps.some(other => other.uses?.startsWith('actions/download-artifact@') && other.with?.name === 'paid-plan'
|
|
&& other.with?.path === '${{ runner.temp }}/paid-plan')).toBe(true);
|
|
const containerRun = container.steps.find(other => other.run?.includes('--plan '))!;
|
|
expect(step.env!.EVALS_JOBS).toBe(containerRun.env!.EVALS_JOBS);
|
|
for (const name of ['EVALS_CONCURRENCY', 'GSTACK_EVAL_DIR', 'PLAYWRIGHT_BROWSERS_PATH']) {
|
|
expect(step.run, name).toContain(`-e ${name}=${containerRun.env![name]}`);
|
|
}
|
|
});
|
|
|
|
test('uploads under the container executor\'s artifact names, and the report waits for it and fails on it', () => {
|
|
expect(uploadNames(codex)).toEqual(uploadNames(container));
|
|
const report = jobs.report!;
|
|
expect(report.needs).toContain('eval-codex-slices');
|
|
const guards = (report.steps as Array<Step & { if?: string }>).filter(step => step.if?.includes('needs.eval-slices.result'));
|
|
expect(guards.length).toBeGreaterThanOrEqual(3);
|
|
for (const guard of guards) expect(guard.if, guard.name).toContain('needs.eval-codex-slices.result');
|
|
});
|
|
});
|
|
|
|
describe('evals-marathon.yml non-blocking lane', () => {
|
|
const workflow = Bun.YAML.parse(marathonYml) as { on: Record<string, unknown>; env: Record<string, string>; jobs: Record<string, Job> };
|
|
|
|
test('runs weekly and on dispatch, always fresh, with its own fail-closed report and tracking issue', () => {
|
|
expect(Object.keys(workflow.on).sort()).toEqual(['schedule', 'workflow_dispatch']);
|
|
expect(workflow.env).toMatchObject({ EVALS_PROFILE: 'full', EVALS_FRESH: '1', EVALS_CACHE_PURPOSE: 'marathon' });
|
|
expect(marathonYml).not.toContain('actions/cache');
|
|
expect(marathonYml).toMatch(/EVALS_TIER=marathon bun --no-install run scripts\/test-paid-shards\.ts --tier marathon --emit-plan \/tmp\/marathon-plan\/manifest\.json --slice-budget 1 --jobs 1/);
|
|
expect(marathonYml).toMatch(/EVALS_TIER=marathon bun run scripts\/test-paid-shards\.ts --tier marathon --plan .* --slice /);
|
|
const report = workflow.jobs.report!;
|
|
expect(report.needs).toEqual(['plan-slices', 'eval-slices']);
|
|
const reconcile = report.steps.find(step => step.id === 'reconcile')!;
|
|
expect(reconcile.run).toContain('EVALS_TIER=marathon bun --no-install run scripts/test-paid-shards.ts --tier marathon --report /tmp/marathon-report');
|
|
const guards = report.steps.filter(step => /Upsert tracking|Fail the workflow/.test(step.name ?? ''));
|
|
expect(guards).toHaveLength(2);
|
|
for (const step of guards) {
|
|
expect((step as { if?: string }).if).toContain("steps.reconcile.outputs.exit != '0'");
|
|
expect((step as { if?: string }).if).toContain("needs.eval-slices.result != 'success'");
|
|
}
|
|
expect(marathonYml).toContain('Weekly marathon evals: red lane needs triage');
|
|
});
|
|
|
|
test('the blocking lanes never plan or execute the marathon tier', () => {
|
|
for (const source of [evalsYml, periodicYml]) {
|
|
expect(source).not.toContain('--tier marathon');
|
|
expect(source).not.toContain('EVALS_TIER=marathon');
|
|
}
|
|
});
|
|
});
|
|
|
|
describe('shared setup composites (every paid lane)', () => {
|
|
test('every lane registers skills through the shared composite', () => {
|
|
for (const [name, source] of [['evals.yml', evalsYml], ['evals-periodic.yml', periodicYml], ['evals-marathon.yml', marathonYml]] as const) {
|
|
expect(source, `${name} must use the register-gstack-skills composite`)
|
|
.toContain('uses: ./.github/actions/register-gstack-skills');
|
|
// No inline re-implementation creeping back beside the composite.
|
|
expect(source, `${name} re-inlines the skill registry instead of using the composite`)
|
|
.not.toContain('ln -snf "$REPO" "$SKILLS_DIR/gstack"');
|
|
}
|
|
});
|
|
|
|
test('the register composite carries the fail-fast verification loop', () => {
|
|
// The loop is the POINT of the composite: a dangling symlink or renamed
|
|
// committed target fails in seconds with a named path, never as a wedged
|
|
// PTY session at the shard wall. Pin its load-bearing markers.
|
|
expect(registerAction).toContain('skill registry OK');
|
|
expect(registerAction).toContain('skill-registry target missing');
|
|
expect(registerAction).toContain('gstack root symlink dangles');
|
|
expect(registerAction).toMatch(/grep -m1 "\^name: \$s\\\$"/);
|
|
});
|
|
|
|
test('seed/deps/temp composites exist and both lanes use them', () => {
|
|
for (const action of ['seed-claude-config', 'restore-deps', 'fix-bun-temp']) {
|
|
expect(fs.existsSync(path.join(ROOT, '.github', 'actions', action, 'action.yml')), `missing composite: ${action}`).toBe(true);
|
|
}
|
|
for (const [name, source] of [['evals.yml', evalsYml], ['evals-periodic.yml', periodicYml], ['evals-marathon.yml', marathonYml]] as const) {
|
|
expect(source, `${name} must use seed-claude-config`).toContain('uses: ./.github/actions/seed-claude-config');
|
|
expect(source, `${name} must use restore-deps`).toContain('uses: ./.github/actions/restore-deps');
|
|
expect(source, `${name} must use fix-bun-temp`).toContain('uses: ./.github/actions/fix-bun-temp');
|
|
}
|
|
});
|
|
});
|
|
|
|
describe('panel verdict surfaces (eval reliability policy)', () => {
|
|
type AnyJob = { if?: string; needs?: string[]; permissions?: Record<string, string>; outputs?: Record<string, string>;
|
|
strategy?: { 'max-parallel': number }; steps: Array<Step & { if?: string; uses?: string }> };
|
|
const jobsOf = (source: string) => (Bun.YAML.parse(source) as { jobs: Record<string, AnyJob> }).jobs;
|
|
|
|
test('planners size the capacity preflight with their executor cap', () => {
|
|
for (const [source, executor, manifest] of [[evalsYml, 'eval-slices', '/tmp/paid-plan/manifest.json'],
|
|
[periodicYml, 'eval-slices', '/tmp/paid-plan/manifest.json'], [periodicYml, 'gate-census', '"/tmp/gate-census-plan/$arm/manifest.json"']] as const) {
|
|
const jobs = jobsOf(source);
|
|
const emit = jobs['plan-slices']!.steps.find(step => step.run?.includes(`--emit-plan ${manifest} `))!;
|
|
const cap = Number(/--max-parallel (\d+)/.exec(emit.run!)?.[1]);
|
|
expect(cap, `${executor}: --max-parallel`).toBe(jobs[executor]!.strategy!['max-parallel']);
|
|
}
|
|
});
|
|
|
|
test('slice artifacts are attempt-scoped and never merged into one tree', () => {
|
|
for (const source of [evalsYml, periodicYml, marathonYml]) {
|
|
const jobs = jobsOf(source);
|
|
const uploads = Object.values(jobs).flatMap(job => job.steps).filter(step => step.uses?.startsWith('actions/upload-artifact@'))
|
|
.map(step => step.with?.name ?? '').filter(name => /slice|census-\$/.test(name))
|
|
// GSTA-49 T1b: the train record keeps the fixed name gstack-ship-queue reads; it is built from the
|
|
// first attempt's slice results only (scripts/train-gate.ts buildTrainRecord), so a re-run cannot change it.
|
|
.filter(name => name !== '${{ needs.plan-slices.outputs.record_artifact }}');
|
|
expect(uploads.length).toBeGreaterThan(0);
|
|
for (const name of uploads) expect(name, name).toContain('-a${{ github.run_attempt }}');
|
|
const downloads = Object.values(jobs).flatMap(job => job.steps).filter(step => step.uses?.startsWith('actions/download-artifact@') && step.with?.pattern);
|
|
for (const step of downloads) expect((step.with as Record<string, unknown>)['merge-multiple'], step.with!.pattern).toBeUndefined();
|
|
}
|
|
});
|
|
|
|
test('the PR comment reads collector-outcomes v2 and never recomputes a verdict', () => {
|
|
const comment = evalsYml.slice(evalsYml.indexOf(' slices-comment:'));
|
|
expect(comment).toContain('.version == 2');
|
|
expect(comment).toContain("jq -r '.failures[]'");
|
|
expect(comment).toContain('name: report-verdict-a${{ github.run_attempt }}');
|
|
expect(evalsYml).not.toContain('group_by(.name)');
|
|
expect(comment).not.toMatch(/paid-slice-/);
|
|
const report = jobsOf(evalsYml)['slices-report']!;
|
|
expect(report.steps.some(step => step.run?.includes('scripts/eval-trial-series.ts /tmp/paid-report/trial-outcomes.jsonl'))).toBe(true);
|
|
expect(report.steps.some(step => step.with?.name?.startsWith('trial-outcomes-'))).toBe(true);
|
|
});
|
|
|
|
test('the weekly report gates on pass-rate history, closes its issue on green, and re-dispatches INFRA-only reds once', () => {
|
|
const jobs = jobsOf(periodicYml);
|
|
const report = jobs.report!;
|
|
expect(report.permissions).toEqual({ contents: 'read', issues: 'write', actions: 'read' });
|
|
const gate = report.steps.find(step => step.id === 'pass-rates')!;
|
|
expect(gate.run).toContain('bun run eval:pass-rates --gate --runs 10');
|
|
expect(gate.if).toBe('always()');
|
|
for (const name of ['Write the census report', 'Upsert tracking issue on failure', 'Fail the workflow when reconciliation failed']) {
|
|
expect(report.steps.find(step => step.name === name)!.if).toContain("steps.pass-rates.outputs.exit != '0'");
|
|
}
|
|
// The report body is written once on every ref; the main-only upsert posts it (test/evals-tracking-issue.test.ts).
|
|
expect(report.steps.find(step => step.name === 'Write the census report')!.run).toContain('report-summary.md');
|
|
expect(report.steps.find(step => step.name === 'Close the tracking issue on a green run')!.run).toContain('gh issue close');
|
|
expect(report.steps.filter(step => step.with?.name?.startsWith('trial-outcomes-')).length).toBe(2);
|
|
const redispatch = jobs.redispatch!;
|
|
expect([redispatch.needs].flat()).toEqual(['report']);
|
|
expect(redispatch.permissions).toEqual({ actions: 'write' });
|
|
expect(redispatch.if).toBe("${{ !cancelled() && needs.report.outputs.redispatch == 'true' }}");
|
|
expect(redispatch.steps[0]!.run).toContain('-f redispatch_of="$GITHUB_RUN_ID"');
|
|
// A census-only run re-dispatches with its own arms, never as a weekly run.
|
|
expect(redispatch.steps[0]!.run).toContain('${CENSUS_ARMS:+-f census_arms="$CENSUS_ARMS"}');
|
|
expect(redispatch.steps[0]!.env?.CENSUS_ARMS).toBe('${{ needs.report.outputs.redispatch_census_arms }}');
|
|
expect(report.outputs!.redispatch_census_arms).toBe("${{ needs.plan-slices.outputs.periodic_lane != 'true' && join(fromJSON(needs.plan-slices.outputs.census_arms), ',') || '' }}");
|
|
const classify = report.steps.find(step => step.id === 'verdict')!;
|
|
expect(classify.run).toContain('.verdict.redispatchEligible == true');
|
|
expect(classify.run).toContain('[ -z "$REDISPATCH_OF" ]');
|
|
expect(periodicYml).toMatch(/group: evals-periodic-\$\{\{ github\.ref \}\}-\$\{\{ github\.event_name \}\}\$\{\{ inputs\.redispatch_of/);
|
|
});
|
|
});
|
|
|
|
describe('scheduled paid lanes: concurrency and branch dispatch scope', () => {
|
|
type Wf = { on: { workflow_dispatch?: { inputs?: Record<string, { type?: string; default?: unknown }> } };
|
|
concurrency: { group: string; 'cancel-in-progress': boolean }; jobs: Record<string, { if?: string; steps: Step[] }> };
|
|
const parse = (source: string) => Bun.YAML.parse(source) as Wf;
|
|
|
|
test('periodic and marathon groups key on ref and event, so a branch or manual dispatch never cancels the scheduled main run', () => {
|
|
for (const [name, source, prefix] of [['evals-periodic.yml', periodicYml, 'evals-periodic'], ['evals-marathon.yml', marathonYml, 'evals-marathon']] as const) {
|
|
const { concurrency } = parse(source);
|
|
expect(concurrency['cancel-in-progress'], name).toBe(true);
|
|
expect(concurrency.group.startsWith(`${prefix}-\${{ github.ref }}-\${{ github.event_name }}`), `${name}: ${concurrency.group}`).toBe(true);
|
|
}
|
|
// The re-dispatch keeps its own group so it never cancels its dispatcher.
|
|
expect(parse(periodicYml).concurrency.group).toContain("format('-redispatch-{0}', inputs.redispatch_of)");
|
|
});
|
|
|
|
test('a branch dispatch of evals-periodic runs the periodic lane only unless it opts into the gate census', () => {
|
|
const wf = parse(periodicYml);
|
|
expect(wf.on.workflow_dispatch!.inputs!.include_gate_census).toMatchObject({ type: 'boolean', default: false });
|
|
// The planner selects the census arms (scripts/census-arms.ts); no arm, no census job.
|
|
expect(wf.jobs['gate-census']!.if).toBe("${{ needs.plan-slices.outputs.gate_matrix != '[]' }}");
|
|
const select = wf.jobs['plan-slices']!.steps.find(step => step.id === 'census')!;
|
|
expect(select.env?.INCLUDE_GATE_CENSUS).toBe("${{ github.ref == 'refs/heads/main' || inputs.include_gate_census }}");
|
|
expect(select.run).toBe('bun --no-install run scripts/census-arms.ts --select >> "$GITHUB_OUTPUT"');
|
|
expect(selectCensus({ schedule: '', censusArms: '', includeGateCensus: false })).toEqual({ arms: [], periodicLane: true });
|
|
expect(selectCensus({ schedule: WEEKLY_CENSUS_SCHEDULE, includeGateCensus: true })).toEqual({ arms: [...WEEKLY_CENSUS_ARMS], periodicLane: true });
|
|
const report = wf.jobs.report!.steps;
|
|
const reconcile = report.find(step => step.name === 'Reconcile gate census against the manifest (fail-closed)')!;
|
|
expect(reconcile.env?.GATE_CENSUS).toBe('${{ needs.gate-census.result }}');
|
|
expect(reconcile.run).toContain('if [ "$GATE_CENSUS" = "skipped" ]');
|
|
// A skipped census is not a red lane; a failed or cancelled one still is.
|
|
const fail = report.find(step => step.name === 'Fail the workflow when reconciliation failed') as Step & { if?: string };
|
|
expect(fail.if).toContain(`!contains(fromJSON('["success","skipped"]'), needs.gate-census.result)`);
|
|
});
|
|
});
|
|
|
|
describe('evals-periodic.yml census arms: Monday two-arm census, monthly Fable/Astra census', () => {
|
|
type CensusStep = Step & { if?: string };
|
|
type CensusWf = { on: { schedule: Array<{ cron: string }>; workflow_dispatch: { inputs: Record<string, { type?: string; default?: unknown }> } };
|
|
concurrency: { group: string }; jobs: Record<string, { if?: string; outputs?: Record<string, string>; steps: CensusStep[] }> };
|
|
const wf = Bun.YAML.parse(periodicYml) as CensusWf;
|
|
const planner = wf.jobs['plan-slices']!;
|
|
const census = wf.jobs['gate-census']!;
|
|
const report = wf.jobs.report!;
|
|
|
|
test('each arm is one host and model: the capture default and gpt-6.1-sol weekly, claude-fable-5-1 and gpt-6-astra monthly', () => {
|
|
expect(WEEKLY_CENSUS_ARMS).toEqual(['claude-opus', 'codex-sol']);
|
|
expect(MONTHLY_CENSUS_ARMS).toEqual(['claude-fable', 'codex-astra']);
|
|
// claude-opus pins no model: the Claude runners' capture default (lib/eval-model.ts) applies.
|
|
expect(censusArmEnv('claude-opus')).toEqual({ EVALS_HOST: 'claude' });
|
|
expect(censusArmEnv('codex-sol')).toEqual({ EVALS_HOST: 'codex', GSTACK_CODEX_MODEL: 'gpt-6.1-sol' });
|
|
expect(censusArmEnv('claude-fable')).toEqual({ EVALS_HOST: 'claude', EVALS_MODEL: 'claude-fable-5-1' });
|
|
expect(censusArmEnv('codex-astra')).toEqual({ EVALS_HOST: 'codex', GSTACK_CODEX_MODEL: 'gpt-6-astra' });
|
|
expect(() => censusArmEnv('codex-luna')).toThrow('unknown census arm');
|
|
// The report files slice artifacts by arm suffix: no arm id may end another one.
|
|
const ids = Object.keys(CENSUS_ARMS);
|
|
for (const a of ids) for (const b of ids) if (a !== b) expect(a.endsWith(`-${b}`)).toBe(false);
|
|
for (const id of ids) expect(id).toMatch(/^[a-z]+-[a-z]+$/);
|
|
});
|
|
|
|
test('the monthly cron runs only the census, under its own concurrency group; the Monday and daily canary schedules are unchanged', () => {
|
|
expect(wf.on.schedule.map(entry => entry.cron)).toEqual([WEEKLY_CENSUS_SCHEDULE, '30 7 * * *', MONTHLY_CENSUS_SCHEDULE]);
|
|
expect(MONTHLY_CENSUS_SCHEDULE).toBe('0 6 1 * *');
|
|
expect(wf.concurrency.group.endsWith("${{ github.event.schedule == '0 6 1 * *' && '-monthly' || '' }}")).toBe(true);
|
|
expect(wf.concurrency.group).toContain("(github.event.schedule == '30 7 * * *' || inputs.schema_canary_only) && '-schema-canary'");
|
|
expect(selectCensus({ schedule: MONTHLY_CENSUS_SCHEDULE, includeGateCensus: true })).toEqual({ arms: [...MONTHLY_CENSUS_ARMS], periodicLane: false });
|
|
const select = planner.steps.find(step => step.id === 'census')!;
|
|
expect(select.env).toMatchObject({ SCHEDULE: '${{ github.event.schedule }}', CENSUS_ARMS: '${{ inputs.census_arms }}' });
|
|
expect(planner.outputs).toMatchObject({ periodic_lane: '${{ steps.census.outputs.periodic_lane }}', census_arms: '${{ steps.census.outputs.arms }}' });
|
|
// Without the periodic lane no periodic plan is emitted, executed or reconciled.
|
|
for (const step of planner.steps.filter(step => step.run?.includes('/tmp/paid-plan/manifest.json') || step.with?.name === 'paid-plan')) {
|
|
expect(step.if, step.name ?? step.with?.name).toBe("steps.census.outputs.periodic_lane == 'true'");
|
|
}
|
|
const reconcile = report.steps.find(step => step.id === 'reconcile')!;
|
|
expect(reconcile.env?.PERIODIC_LANE).toBe('${{ needs.plan-slices.outputs.periodic_lane }}');
|
|
expect(reconcile.run).toContain('if [ "$PERIODIC_LANE" != "true" ]; then');
|
|
// A census-only green run never closes the weekly tracking issue.
|
|
expect(report.steps.find(step => step.name === 'Close the tracking issue on a green run')!.if)
|
|
.toContain("needs.plan-slices.outputs.periodic_lane == 'true' && needs.eval-slices.result == 'success'");
|
|
});
|
|
|
|
test('a census_arms dispatch runs only the named arms and refuses unknown ones', () => {
|
|
expect(wf.on.workflow_dispatch.inputs.census_arms).toMatchObject({ type: 'string', default: '' });
|
|
expect(selectCensus({ censusArms: 'codex-sol', includeGateCensus: true })).toEqual({ arms: ['codex-sol'], periodicLane: false });
|
|
expect(selectCensus({ censusArms: 'claude-opus, codex-sol,codex-sol', includeGateCensus: false })).toEqual({ arms: ['claude-opus', 'codex-sol'], periodicLane: false });
|
|
expect(() => selectCensus({ censusArms: 'codex-sol,gpt', includeGateCensus: false })).toThrow('unknown census arm(s): gpt');
|
|
const cli = (args: string[], env: Record<string, string>) => {
|
|
const proc = spawnSync(process.execPath, ['--no-install', 'run', 'scripts/census-arms.ts', ...args],
|
|
{ cwd: ROOT, env: { PATH: process.env.PATH ?? '', ...env }, encoding: 'utf8', timeout: 15_000 });
|
|
expect(proc.status, proc.stderr).toBe(0);
|
|
return proc.stdout;
|
|
};
|
|
expect(cli(['--select'], { SCHEDULE: MONTHLY_CENSUS_SCHEDULE })).toBe('arms=["claude-fable","codex-astra"]\nperiodic_lane=false\n');
|
|
expect(cli(['--select'], { CENSUS_ARMS: 'codex-sol', INCLUDE_GATE_CENSUS: 'true' })).toBe('arms=["codex-sol"]\nperiodic_lane=false\n');
|
|
expect(cli(['--env', 'codex-sol'], {})).toBe('EVALS_HOST=codex\nGSTACK_CODEX_MODEL=gpt-6.1-sol\n');
|
|
});
|
|
|
|
test('census artifacts and reconciliation are per arm', () => {
|
|
const upload = census.steps.find(step => step.name === 'Upload census results')!;
|
|
expect(upload.with!.name).toBe('gate-census-${{ matrix.slice }}-a${{ github.run_attempt }}-${{ matrix.arm }}');
|
|
expect(census.steps.find(step => step.name?.startsWith('Run gate census slice'))!.name).toContain('(${{ matrix.arm }})');
|
|
expect(report.steps.some(step => step.with?.pattern === 'gate-census-[0-9]*' && step.with?.path === '/tmp/gate-census-slices')).toBe(true);
|
|
const reconcile = report.steps.find(step => step.id === 'gate-reconcile')!;
|
|
expect(reconcile.env?.ARMS).toBe('${{ needs.plan-slices.outputs.census_arms }}');
|
|
expect(reconcile.run).toContain('cp "/tmp/gate-census-plan/$arm/manifest.json" "$dir/manifest.json" || status=1');
|
|
expect(reconcile.run).toContain('for slice in /tmp/gate-census-slices/gate-census-*-"$arm"; do');
|
|
expect(reconcile.run).toContain('EVALS_TIER=gate bun run scripts/test-paid-shards.ts --tier gate --report "$dir"');
|
|
// Every arm's report counts: one red arm fails the census.
|
|
expect(reconcile.run).toContain('if [ "$code" -ne 0 ]; then status=$code; fi');
|
|
expect(reconcile.run).toContain('echo "exit=$status" >> "$GITHUB_OUTPUT"');
|
|
});
|
|
});
|