mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-02 17:40:02 +02:00
garrytan
b421bba2c9
Keep both intents: v1.91.7.0's functional QA, docsync and exploratory paid cases and their free owners stay; this branch's deletions stay deleted. main's new paid keys follow the derived-closure touchfile rule (free *.test.ts paths dropped, static helper/fixture closure added), its new helper-only tests join the ratchet baseline, and its free selection examples that named free test files now assert the derived selection. Periodic CI keeps seven slices without the retired Autoplan slice; the gate census keeps seven single-worker slices with --skip-judges. Wall and census literals are recomputed from the merged planner, durations are re-recorded on Ubicloud, and VERSION stays 1.91.8.0 above 1.91.7.0.
384 lines
22 KiB
TypeScript
384 lines
22 KiB
TypeScript
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
|
|
import * as fs from 'node:fs';
|
|
import * as os from 'node:os';
|
|
import * as path from 'node:path';
|
|
import { spawnSync } from 'node:child_process';
|
|
import { buildRunManifest, collectPaidTestFiles, type PaidRunManifest, type SliceResult } from '../scripts/test-paid-shards';
|
|
import { STRICT_RETRY_CASE_BUDGETS } from './helpers/eval-budgets';
|
|
import { approvedCookieWorkflowSource, manualReviewFixture } from './helpers/manual-judge-review-fixture';
|
|
|
|
const ROOT = path.resolve(import.meta.dir, '..');
|
|
type Step = { uses?: string; run?: string; if?: string; with?: Record<string, unknown> };
|
|
type Job = {
|
|
needs?: string | string[];
|
|
if?: string;
|
|
container?: unknown;
|
|
permissions: Record<string, string>;
|
|
steps: Step[];
|
|
};
|
|
const workflows = ['evals.yml', 'evals-periodic.yml'].map(name => ({
|
|
name,
|
|
jobs: (Bun.YAML.parse(fs.readFileSync(path.join(ROOT, '.github/workflows', name), 'utf8')) as {
|
|
jobs: Record<string, Job>;
|
|
}).jobs,
|
|
}));
|
|
|
|
describe('paid CI coordination stays off the eval image', () => {
|
|
test('the actual planner and reporter load from a checkout without installed packages', () => {
|
|
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'paid-offline-'));
|
|
const listed = spawnSync('git', ['ls-files', '-z'], { cwd: ROOT, encoding: 'utf8', timeout: 5_000 });
|
|
expect(listed.status, listed.stderr).toBe(0);
|
|
try {
|
|
for (const relative of listed.stdout.split('\0').filter(Boolean)) {
|
|
const source = path.join(ROOT, relative);
|
|
const destination = path.join(directory, relative);
|
|
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
|
expect(fs.realpathSync(path.dirname(destination)).startsWith(fs.realpathSync(directory) + path.sep)
|
|
|| fs.realpathSync(path.dirname(destination)) === fs.realpathSync(directory)).toBe(true);
|
|
if (fs.lstatSync(source).isSymbolicLink()) fs.symlinkSync(fs.readlinkSync(source), destination);
|
|
else fs.copyFileSync(source, destination);
|
|
}
|
|
const env = { ...process.env, ANTHROPIC_API_KEY: undefined, EVALS: undefined, EVALS_ALL: undefined,
|
|
EVALS_PROFILE: 'pr', EVALS_TIER: 'gate' };
|
|
for (const args of [['init', '-b', 'main'], ['add', '-A'], ['commit', '-m', 'Seed offline planner fixture'], ['checkout', '-b', 'fixture-head']]) {
|
|
const git = spawnSync('git', args, { cwd: directory, env, encoding: 'utf8', timeout: 15_000 });
|
|
expect(git.status, git.stderr).toBe(0);
|
|
}
|
|
fs.appendFileSync(path.join(directory, '.github/workflows/evals.yml'), '\n');
|
|
const report = path.join(directory, 'report');
|
|
const plan = spawnSync(process.execPath, ['--no-install', 'run', 'scripts/test-paid-shards.ts', '--tier', 'gate',
|
|
'--emit-plan', path.join(report, 'manifest.json'), '--slices', '6'], { cwd: directory, env, encoding: 'utf8', timeout: 15_000 });
|
|
expect(plan.status, plan.stdout + plan.stderr).toBe(0);
|
|
const manifest = JSON.parse(fs.readFileSync(path.join(report, 'manifest.json'), 'utf8'));
|
|
expect(manifest.sliceCount).toBe(6);
|
|
expect(manifest.entries.some((entry: any) => entry.status === 'planned')).toBe(true);
|
|
const reconcile = spawnSync(process.execPath, ['--no-install', 'run', 'scripts/test-paid-shards.ts', '--tier', 'gate',
|
|
'--report', report], { cwd: directory, env, encoding: 'utf8', timeout: 15_000 });
|
|
expect(reconcile.status, reconcile.stdout + reconcile.stderr).toBe(1);
|
|
expect(reconcile.stdout).toContain('report: 0/6 slices');
|
|
expect(reconcile.stderr.match(/slice \d\/6 reported NO result/g)).toHaveLength(6);
|
|
expect(reconcile.stderr).not.toContain('Cannot find module');
|
|
expect(fs.existsSync(path.join(directory, 'node_modules'))).toBe(false);
|
|
const provider = spawnSync(process.execPath, ['--no-install', '-e', 'import "./test/helpers/llm-judge.ts"'],
|
|
{ cwd: directory, env, encoding: 'utf8', timeout: 15_000 });
|
|
expect(provider.status).toBe(1);
|
|
expect(provider.stderr).toContain("Cannot find module '@anthropic-ai/sdk'");
|
|
} finally { fs.rmSync(directory, { recursive: true, force: true }); }
|
|
}, 60_000);
|
|
|
|
for (const { name, jobs } of workflows) {
|
|
test(`${name}: planning is independent of image startup and has no dependency install`, () => {
|
|
const planner = jobs['plan-slices'];
|
|
expect(planner.needs).toBeUndefined();
|
|
expect(planner.container).toBeUndefined();
|
|
expect(planner.permissions).toEqual({ contents: 'read' });
|
|
const checkout = planner.steps.find(step => step.uses?.startsWith('actions/checkout@'))!;
|
|
expect(checkout.with?.['persist-credentials']).toBe(false);
|
|
if (name === 'evals.yml') expect(checkout.with?.['fetch-depth']).toBe(0);
|
|
const setup = planner.steps.find(step => step.uses?.startsWith('oven-sh/setup-bun@'))!;
|
|
expect(setup.with?.['bun-version']).toBe('1.4.0');
|
|
expect(JSON.stringify(planner)).not.toMatch(/secrets\.|restore-deps|bun install|bun run build/);
|
|
expect(planner.steps.find(step => step.run?.includes('--emit-plan'))?.run).toContain('bun --no-install run');
|
|
});
|
|
|
|
test(`${name}: cancel-in-progress can stop every job (no job-level always())`, () => {
|
|
for (const [id, job] of Object.entries(jobs)) {
|
|
const condition = String((job as { if?: unknown }).if ?? '');
|
|
expect(`${id}: ${condition}`).not.toMatch(/(^|[^!])always\(\)/);
|
|
}
|
|
});
|
|
|
|
test(`${name}: executors still require both prerequisites and consume the image`, () => {
|
|
const executor = jobs['eval-slices'];
|
|
expect(executor.needs).toEqual(['build-image', 'plan-slices']);
|
|
expect(JSON.stringify(executor.container)).toContain('needs.build-image.outputs.image-tag');
|
|
if (name === 'evals.yml') {
|
|
expect(executor.if).toBe("${{ !cancelled() && needs.build-image.result == 'success' && needs.plan-slices.result == 'success' }}");
|
|
} else {
|
|
expect(executor.if).toBeUndefined();
|
|
}
|
|
expect(executor.steps.some(step => step.run === 'bun run build')).toBe(true);
|
|
expect(executor.steps.some(step => step.uses === './.github/actions/restore-deps')).toBe(true);
|
|
});
|
|
|
|
test(`${name}: report still reconciles failed executors without installing dependencies`, () => {
|
|
const report = jobs[name === 'evals.yml' ? 'slices-report' : 'report'];
|
|
expect(report.container).toBeUndefined();
|
|
expect(report.needs).toContain('plan-slices');
|
|
expect(report.needs).toContain('eval-slices');
|
|
expect(report.if).toBe("${{ !cancelled() && needs.plan-slices.result == 'success' }}");
|
|
expect(JSON.stringify(report.steps)).not.toMatch(/restore-deps|bun install/);
|
|
expect(report.steps.find(step => step.run?.includes('--report'))?.run).toContain('bun --no-install run');
|
|
if (name === 'evals.yml') expect(report.permissions).toEqual({ contents: 'read' });
|
|
});
|
|
|
|
test(`${name}: failure logs include the hidden spool directory without uploading the rest of the cache`, () => {
|
|
const logs = jobs['eval-slices'].steps.find(step => step.with?.name === 'paid-slice-${{ matrix.slice }}-logs');
|
|
expect(logs?.uses).toStartWith('actions/upload-artifact@');
|
|
expect(logs?.if).toBe('failure()');
|
|
expect(logs?.with?.['include-hidden-files']).toBe(true);
|
|
expect(String(logs?.with?.path).trim().split('\n')).toEqual([
|
|
'/home/runner/.cache/gstack-paid-shard-*.log',
|
|
'/tmp/gstack-paid-shard-*.log',
|
|
]);
|
|
const hiddenUploads = Object.values(jobs).flatMap(job => job.steps).filter(step => step.with?.['include-hidden-files']);
|
|
const captures = hiddenUploads.filter(step => step.with?.name === 'native-captures-${{ env.EVALS_RUN_ID }}');
|
|
expect(captures).toHaveLength(name === 'evals.yml' ? 1 : 2);
|
|
for (const capture of captures) {
|
|
expect(capture.if).toBe('always()');
|
|
expect(String(capture.with?.path).trim().split('\n')).toEqual([
|
|
'~/.gstack/projects/*/e2e-runs', '~/.gstack/projects/*/evals/qa-callers',
|
|
'~/.gstack-dev/e2e-runs', '~/.gstack-dev/evals/qa-callers',
|
|
]);
|
|
}
|
|
expect(hiddenUploads.filter(step => !captures.includes(step))).toEqual([logs]);
|
|
});
|
|
}
|
|
|
|
test('PR planning preserves the fork and Dependabot trust boundaries without the needs chain', () => {
|
|
expect(workflows[0].jobs['plan-slices'].if).toBe(
|
|
"github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)",
|
|
);
|
|
});
|
|
});
|
|
|
|
describe('dependency-free CI planner and report execution', () => {
|
|
let fixture: string;
|
|
|
|
beforeAll(() => {
|
|
fixture = fs.mkdtempSync(path.join(os.tmpdir(), 'ci-paid-coordination-'));
|
|
const sourceOnly = { recursive: true, filter: (file: string) => path.basename(file) !== 'node_modules' };
|
|
fs.cpSync(path.join(ROOT, 'scripts'), path.join(fixture, 'scripts'), sourceOnly);
|
|
fs.cpSync(path.join(ROOT, 'test/helpers'), path.join(fixture, 'test/helpers'), sourceOnly);
|
|
expect(fs.readFileSync(path.join(fixture, 'test/helpers/llm-judge.ts'), 'utf8'))
|
|
.toBe(fs.readFileSync(path.join(ROOT, 'test/helpers/llm-judge.ts'), 'utf8'));
|
|
fs.cpSync(path.join(ROOT, 'lib'), path.join(fixture, 'lib'), sourceOnly);
|
|
expect(fs.existsSync(path.join(fixture, 'lib/diagram-render/node_modules'))).toBe(false);
|
|
fs.mkdirSync(path.join(fixture, '.github'), { recursive: true });
|
|
for (const file of ['.github/cookie-workflow-manual-review.json', 'setup-browser-cookies/SKILL.md', 'BROWSER.md']) {
|
|
fs.mkdirSync(path.dirname(path.join(fixture, file)), { recursive: true });
|
|
fs.copyFileSync(path.join(ROOT, file), path.join(fixture, file));
|
|
}
|
|
for (const file of collectPaidTestFiles()) {
|
|
fs.copyFileSync(path.join(ROOT, file), path.join(fixture, file));
|
|
}
|
|
});
|
|
|
|
afterAll(() => {
|
|
fs.rmSync(fixture, { recursive: true, force: true });
|
|
});
|
|
|
|
function run(args: string[], tier: string, env: NodeJS.ProcessEnv = {}) {
|
|
return spawnSync(process.execPath, ['--no-install', 'run', 'scripts/test-paid-shards.ts', '--tier', tier, ...args], {
|
|
cwd: fixture,
|
|
env: { PATH: '', HOME: fixture, EVALS_ALL: '1', EVALS_TIER: tier, ...env },
|
|
encoding: 'utf8',
|
|
timeout: 10_000,
|
|
});
|
|
}
|
|
|
|
test('diff-selected host planning matches the same checkout with no installed dependencies', () => {
|
|
const git = Bun.which('git')!;
|
|
const gitDir = spawnSync(git, ['rev-parse', '--absolute-git-dir'], { cwd: ROOT, encoding: 'utf8', timeout: 10_000 });
|
|
expect(gitDir.status).toBe(0);
|
|
const manifestPath = path.join(fixture, 'diff-manifest.json');
|
|
const env = { EVALS_ALL: '', EVALS_BASE: 'HEAD' };
|
|
const planned = run(['--emit-plan', manifestPath, '--slices', '6'], 'gate', {
|
|
...env,
|
|
PATH: path.dirname(git),
|
|
GIT_DIR: gitDir.stdout.trim(),
|
|
GIT_WORK_TREE: ROOT,
|
|
});
|
|
expect(planned.status, planned.stderr).toBe(0);
|
|
const manifest: PaidRunManifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
|
|
expect(manifest).toEqual(buildRunManifest({ tier: 'gate', sliceCount: 6, evalsAll: false, env }));
|
|
expect(manifest.evalsAll).toBe(false);
|
|
});
|
|
|
|
for (const tier of ['gate', 'periodic'] as const) {
|
|
test(`${tier}: host planner preserves the complete manifest and report fails closed`, () => {
|
|
const sliceCount = tier === 'gate' ? 6 : 7;
|
|
const reportDir = path.join(fixture, tier);
|
|
const manifestPath = path.join(reportDir, 'manifest.json');
|
|
const planned = run(['--emit-plan', manifestPath, '--slices', String(sliceCount)], tier);
|
|
expect(planned.error).toBeUndefined();
|
|
expect(planned.status, planned.stderr).toBe(0);
|
|
const manifest: PaidRunManifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
|
|
expect(manifest).toEqual(buildRunManifest({
|
|
tier, sliceCount, evalsAll: true, env: { EVALS_ALL: '1' },
|
|
}));
|
|
expect(manifest.entries.filter(entry => entry.status === 'planned').length).toBeGreaterThan(0);
|
|
expect(fs.existsSync(path.join(fixture, 'node_modules'))).toBe(false);
|
|
for (let sliceIndex = 1; sliceIndex <= sliceCount; sliceIndex++) {
|
|
const result: SliceResult = {
|
|
version: 1, tier, sliceIndex, sliceCount,
|
|
outcomes: manifest.entries.filter(entry => entry.status === 'planned' && entry.slice === sliceIndex).map(entry => ({
|
|
files: [entry.file], status: 'passed', exitCode: 0, elapsedMs: 1,
|
|
executedTests: STRICT_RETRY_CASE_BUDGETS.find(budget => budget.file === entry.file)?.cases ?? 1,
|
|
skippedTests: 0,
|
|
...(entry.budget ? { budget: entry.budget } : {}),
|
|
})),
|
|
};
|
|
fs.writeFileSync(path.join(reportDir, `slice-${sliceIndex}.json`), JSON.stringify(result));
|
|
}
|
|
const clean = run(['--report', reportDir], tier);
|
|
expect(clean.status, clean.stderr).toBe(0);
|
|
expect(clean.stdout).toContain('every planned shard accounted and passed');
|
|
|
|
const lastSlice = path.join(reportDir, `slice-${sliceCount}.json`);
|
|
const saved = fs.readFileSync(lastSlice, 'utf8');
|
|
fs.rmSync(lastSlice);
|
|
const missing = run(['--report', reportDir], tier);
|
|
expect(missing.status).toBe(1);
|
|
expect(missing.stderr).toContain(`slice ${sliceCount}/${sliceCount} reported NO result`);
|
|
|
|
const failed: SliceResult = JSON.parse(saved);
|
|
expect(failed.outcomes.length).toBeGreaterThan(0);
|
|
failed.outcomes[0].status = 'failed';
|
|
failed.outcomes[0].exitCode = 1;
|
|
fs.writeFileSync(lastSlice, JSON.stringify(failed));
|
|
fs.writeFileSync(path.join(reportDir, 'retry-results.json'), JSON.stringify({
|
|
tests: [
|
|
{ name: 'recovered', passed: false }, { name: 'recovered', passed: true },
|
|
{ name: 'exhausted', passed: false }, { name: 'exhausted', passed: false },
|
|
{ name: 'regressed', passed: true }, { name: 'regressed', passed: false },
|
|
],
|
|
flaky_retries: ['recovered', 'exhausted', 'regressed'].map(name => ({ name, attempts: 2 })),
|
|
}));
|
|
const red = run(['--report', reportDir], tier);
|
|
expect(red.status).toBe(1);
|
|
expect(red.stderr).toContain(`${failed.outcomes[0].files[0]}: failed`);
|
|
expect(red.stdout).toContain('3 executed, 0 reused; 1 passed, 2 failed, 0 manual accepted (unscored; no score-cache credit) (6 attempt records from 1 collectors)');
|
|
expect(red.stdout).toContain('3 cases with multiple attempts this run:');
|
|
expect(red.stdout).not.toMatch(/passed only on retry|not blocking/);
|
|
|
|
fs.writeFileSync(manifestPath, '{');
|
|
const corrupt = run(['--report', reportDir], tier);
|
|
expect(corrupt.status).toBe(1);
|
|
});
|
|
}
|
|
|
|
test('report verifies every manual claim against current source, preserves attempts, and never masks a failed shard', () => {
|
|
const skillPath = path.join(fixture, 'setup-browser-cookies/SKILL.md');
|
|
const currentSkill = fs.readFileSync(skillPath, 'utf8');
|
|
fs.writeFileSync(skillPath, approvedCookieWorkflowSource(currentSkill));
|
|
const approvedBrowserPath = path.join(fixture, 'BROWSER.md');
|
|
fs.writeFileSync(approvedBrowserPath, approvedCookieWorkflowSource(fs.readFileSync(approvedBrowserPath, 'utf8')));
|
|
const reportDir = path.join(fixture, 'manual-report');
|
|
const manifestPath = path.join(reportDir, 'manifest.json');
|
|
const planned = run(['--emit-plan', manifestPath, '--slices', '1'], 'gate');
|
|
expect(planned.status, planned.stderr).toBe(0);
|
|
const manifest: PaidRunManifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
|
|
const slice: SliceResult = {
|
|
version: 1, tier: 'gate', sliceIndex: 1, sliceCount: 1,
|
|
outcomes: manifest.entries.filter(entry => entry.status === 'planned').map(entry => ({
|
|
files: [entry.file], status: 'passed', exitCode: 0, elapsedMs: 1,
|
|
executedTests: STRICT_RETRY_CASE_BUDGETS.find(budget => budget.file === entry.file)?.cases ?? 1,
|
|
skippedTests: 0, ...(entry.budget ? { budget: entry.budget } : {}),
|
|
})),
|
|
};
|
|
const slicePath = path.join(reportDir, 'slice-1.json');
|
|
const collectorPath = path.join(reportDir, 'judge-results.json');
|
|
const summaryPath = path.join(reportDir, 'collector-outcomes.json');
|
|
const receipt = manualReviewFixture(fixture);
|
|
const write = (tests: unknown[]) => fs.writeFileSync(collectorPath, JSON.stringify({
|
|
total_tests: tests.length, tier: 'llm-judge', shard: 1, total_cost_usd: 0,
|
|
tests, flaky_retries: [{ name: receipt.name, attempts: tests.length }],
|
|
}));
|
|
fs.writeFileSync(slicePath, JSON.stringify(slice));
|
|
write([{ ...receipt, passed: true }, receipt]);
|
|
const historical = run(['--report', reportDir], 'gate');
|
|
expect(historical.status).toBe(1);
|
|
expect(historical.stderr).toContain('attempt 1: Malformed manual-review claim');
|
|
expect(fs.existsSync(summaryPath)).toBe(false);
|
|
|
|
write([{ ...receipt, manual_review: { ...receipt.manual_review, refusal: {
|
|
...receipt.manual_review!.refusal, response_id: '',
|
|
} } }, receipt]);
|
|
const malformed = run(['--report', reportDir], 'gate');
|
|
expect(malformed.status).toBe(1);
|
|
expect(malformed.stderr).toContain('attempt 1: Malformed manual-review claim');
|
|
|
|
write([{ ...receipt, manual_review: undefined, passed: false }, receipt]);
|
|
const forgedFirstAttempt = run(['--report', reportDir], 'gate');
|
|
expect(forgedFirstAttempt.status).toBe(1);
|
|
expect(forgedFirstAttempt.stderr).toContain('attempt 2: manual review is only valid on the first case attempt');
|
|
expect(fs.existsSync(summaryPath)).toBe(false);
|
|
write([receipt, { ...receipt, attempt: 2 }]);
|
|
const retriedManual = run(['--report', reportDir], 'gate');
|
|
expect(retriedManual.status).toBe(1);
|
|
expect(retriedManual.stderr).toContain('attempt 2: Malformed manual-review claim');
|
|
expect(fs.existsSync(summaryPath)).toBe(false);
|
|
|
|
write([receipt]);
|
|
const secondCollector = path.join(reportDir, 'other-results.json');
|
|
fs.writeFileSync(secondCollector, JSON.stringify({ total_tests: 1, tests: [receipt] }));
|
|
const duplicateCollector = run(['--report', reportDir], 'gate');
|
|
expect(duplicateCollector.status).toBe(1);
|
|
expect(duplicateCollector.stderr).toContain('duplicate manual-review claim');
|
|
expect(fs.existsSync(summaryPath)).toBe(false);
|
|
fs.rmSync(secondCollector);
|
|
|
|
write([receipt, { name: 'automated', suite: 'other', passed: true, execution: 'reused' }]);
|
|
const clean = run(['--report', reportDir], 'gate');
|
|
expect(clean.status, clean.stderr).toBe(0);
|
|
expect(clean.stdout).toContain('1 passed, 0 failed, 1 manual accepted (unscored; no score-cache credit) (2 attempt records');
|
|
const summary = JSON.parse(fs.readFileSync(summaryPath, 'utf8'));
|
|
expect(summary.totals).toEqual({ executed: 1, reused: 1, passed: 1, failed: 0,
|
|
manual_accepted: 1, attempts: 2, total: 2, flaky: 1 });
|
|
expect(summary.files[0]).toMatchObject({ file: 'judge-results.json', total: 2, manual_accepted: 1, passed: 1 });
|
|
expect(JSON.parse(fs.readFileSync(collectorPath, 'utf8')).tests[0]).toEqual(receipt);
|
|
|
|
fs.writeFileSync(skillPath, currentSkill);
|
|
const obsoleteApproval = run(['--report', reportDir], 'gate');
|
|
expect(obsoleteApproval.status).toBe(1);
|
|
expect(obsoleteApproval.stderr).toContain('does not match current source and approval');
|
|
fs.writeFileSync(skillPath, approvedCookieWorkflowSource(currentSkill));
|
|
|
|
const browserPath = path.join(fixture, 'BROWSER.md');
|
|
const browserSource = fs.readFileSync(browserPath, 'utf8');
|
|
fs.writeFileSync(browserPath, browserSource.replace('#### Choosing a source and checking sign-in',
|
|
'#### Choosing a source and checking sign-in\nchanged approved source'));
|
|
const sourceDrift = run(['--report', reportDir], 'gate');
|
|
expect(sourceDrift.status).toBe(1);
|
|
expect(sourceDrift.stderr).toContain('does not match current source and approval');
|
|
expect(fs.existsSync(summaryPath)).toBe(false);
|
|
fs.writeFileSync(browserPath, browserSource);
|
|
|
|
write([receipt, { name: 'unapproved', passed: false, execution: 'executed' }]);
|
|
const failedCollector = run(['--report', reportDir], 'gate');
|
|
expect(failedCollector.status).toBe(1);
|
|
expect(failedCollector.stderr).toContain('1 unapproved final collector failure(s)');
|
|
expect(JSON.parse(fs.readFileSync(summaryPath, 'utf8')).totals).toMatchObject({ failed: 1, manual_accepted: 1 });
|
|
|
|
write([receipt, { passed: true }]);
|
|
const missingName = run(['--report', reportDir], 'gate');
|
|
expect(missingName.status).toBe(1);
|
|
expect(missingName.stderr).toContain('attempt 2: malformed collector entry (name/passed required)');
|
|
expect(fs.existsSync(summaryPath)).toBe(false);
|
|
write([receipt, { name: 'malformed', passed: 'true' }]);
|
|
const malformedPassed = run(['--report', reportDir], 'gate');
|
|
expect(malformedPassed.status).toBe(1);
|
|
expect(malformedPassed.stderr).toContain('attempt 2: malformed collector entry (name/passed required)');
|
|
expect(fs.existsSync(summaryPath)).toBe(false);
|
|
|
|
write([receipt, { name: 'automated', suite: 'other', passed: true, execution: 'reused' }]);
|
|
|
|
slice.outcomes[0].status = 'failed';
|
|
slice.outcomes[0].exitCode = 1;
|
|
fs.writeFileSync(slicePath, JSON.stringify(slice));
|
|
const failedShard = run(['--report', reportDir], 'gate');
|
|
expect(failedShard.status).toBe(1);
|
|
expect(failedShard.stderr).toContain(`${slice.outcomes[0].files[0]}: failed`);
|
|
expect(JSON.parse(fs.readFileSync(summaryPath, 'utf8')).totals.manual_accepted).toBe(1);
|
|
|
|
const stale = structuredClone(receipt);
|
|
stale.manual_review!.approval.prompt_sha256 = '0'.repeat(64);
|
|
write([stale]);
|
|
const mismatched = run(['--report', reportDir], 'gate');
|
|
expect(mismatched.status).toBe(1);
|
|
expect(mismatched.stderr).toContain('attempt 1: Malformed manual-review claim');
|
|
expect(fs.existsSync(summaryPath)).toBe(false);
|
|
});
|
|
});
|