mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-13 20:07:08 +01:00
feat(evals): readiness-gated paid dispatch (R1), restamp-only gate reuse (R2a), and a receipt store that restores
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
fd13cf25e8
commit
f15a34dc35
11 files changed
+1150
-72
No files matched your search
@@ -39,7 +39,7 @@ test('receipt transport: only recover-receipts restores this repository and PR\'
|
||||
const emit = planner.find((s: any) => s.run?.includes('--emit-plan /tmp/paid-plan/manifest.json'));
|
||||
expect(emit.env.EVALS_CACHE_DIR).toBe("${{ steps.reuse.outputs.reuse == 'on' && '/tmp/gstack-eval-input-cache' || '' }}");
|
||||
const upload = planner.find((s: any) => s.with?.name === 'paid-plan');
|
||||
expect(upload.with.path.trim().split('\n')).toEqual(['/tmp/paid-plan/manifest.json', '/tmp/paid-plan/receipts', '/tmp/paid-plan/store']);
|
||||
expect(upload.with.path.trim().split('\n')).toEqual(['/tmp/paid-plan/manifest.json', '/tmp/paid-plan/receipts', '/tmp/paid-plan/store', '/tmp/paid-plan/gate-reuse.json']);
|
||||
// Executors never restore or save a cache of their own: every slice sees the plan's one receipt set.
|
||||
const executor = paid.jobs['eval-slices'].steps;
|
||||
expect(executor.filter((s: any) => s.uses?.startsWith('actions/cache/'))).toHaveLength(0);
|
||||
@@ -54,6 +54,12 @@ test('receipt transport: only recover-receipts restores this repository and PR\'
|
||||
expect(save[0].with.key).toBe('eval-input-v1-${{ github.repository_id }}-pr-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.run_attempt }}-merged');
|
||||
// Only PR runs save; a pr_receipts dispatch reads the PR store read-only.
|
||||
expect(save[0].if).toBe("always() && github.event_name == 'pull_request'");
|
||||
// Restore and save share one cache backend (GSTA-48): an Ubicloud runner saves into Ubicloud's transparent
|
||||
// cache, which a GitHub-hosted restore never sees, so every restore missed while the report ran on Ubicloud.
|
||||
for (const job of Object.values(paid.jobs) as any[]) {
|
||||
if (!job.steps.some((s: any) => s.uses?.startsWith('actions/cache/') && s.with?.path === '/tmp/gstack-eval-input-cache')) continue;
|
||||
expect(job['runs-on']).toBe('ubuntu-24.04');
|
||||
}
|
||||
expect(report.indexOf(save[0])).toBeGreaterThan(report.indexOf(merge));
|
||||
expect(paid.jobs['eval-slices'].permissions).toEqual({ contents: 'read', packages: 'read' });
|
||||
expect(paid.jobs['slices-report'].permissions).toEqual({ contents: 'read' });
|
||||
|
||||
@@ -75,9 +75,9 @@ describe('paid CI coordination stays off the eval image', () => {
|
||||
test(`${name}: planning is independent of image startup and has no dependency install`, () => {
|
||||
const planner = jobs['plan-slices'];
|
||||
if (name === 'evals.yml') {
|
||||
// The PR lane plans after its two tiny coordination jobs (push-burst debounce, base-ref receipt recovery), never after the image.
|
||||
expect(planner.needs).toEqual(['debounce', 'recover-receipts']);
|
||||
for (const id of ['debounce', 'recover-receipts']) {
|
||||
// The PR lane plans after its two tiny coordination jobs (readiness, base-ref receipt recovery), never after the image.
|
||||
expect(planner.needs).toEqual(['readiness', 'recover-receipts']);
|
||||
for (const id of ['readiness', 'recover-receipts']) {
|
||||
const job = jobs[id] as Job & { 'runs-on'?: string };
|
||||
expect(job.container, id).toBeUndefined();
|
||||
expect(job['runs-on'], id).toBe('ubuntu-24.04');
|
||||
@@ -110,7 +110,8 @@ describe('paid CI coordination stays off the eval image', () => {
|
||||
expect(executor.needs).toEqual(['build-image', 'plan-slices']);
|
||||
expect(JSON.stringify(executor.container)).toContain('needs.build-image.outputs.image-tag');
|
||||
if (name === 'evals.yml') {
|
||||
expect(executor.if).toBe("${{ !cancelled() && needs.build-image.result == 'success' && needs.plan-slices.result == 'success' }}");
|
||||
// A confirmed restamp-only reuse (R2a) is the one PR run that executes no slice.
|
||||
expect(executor.if).toBe("${{ !cancelled() && needs.build-image.result == 'success' && needs.plan-slices.result == 'success' && needs.plan-slices.outputs.gate_reuse != 'true' }}");
|
||||
} else {
|
||||
expect(executor.if).toBeUndefined();
|
||||
}
|
||||
@@ -165,12 +166,13 @@ describe('paid CI coordination stays off the eval image', () => {
|
||||
test('PR planning preserves the fork and Dependabot trust boundaries across the coordination jobs', () => {
|
||||
const jobs = workflows[0].jobs;
|
||||
const trust = "github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)";
|
||||
// !cancelled() lets planning proceed when debounce/recovery were skipped or failed (reuse is then off); the trust clause still gates it.
|
||||
expect(jobs['plan-slices'].if).toBe(`\${{ !cancelled() && needs.debounce.outputs.superseded != 'true' && ${trust} }}`);
|
||||
// !cancelled() lets planning proceed when readiness was skipped (dispatch) or recovery was skipped or failed (reuse is then off);
|
||||
// a readiness failure (Free Tests red on this head) or a superseded head plans nothing; the trust clause still gates it.
|
||||
expect(jobs['plan-slices'].if).toBe(`\${{ !cancelled() && (needs.readiness.result == 'success' || needs.readiness.result == 'skipped') && needs.readiness.outputs.superseded != 'true' && ${trust} }}`);
|
||||
// The coordination jobs never run for Dependabot or for a fork PR either (recovery also serves an explicit pr_receipts dispatch).
|
||||
expect(jobs.debounce.if).toContain("github.actor != 'dependabot[bot]'");
|
||||
expect(jobs.debounce.if).toContain("github.event_name == 'pull_request'");
|
||||
expect(jobs.debounce.if).toContain('github.event.pull_request.head.repo.full_name == github.repository');
|
||||
expect(jobs.readiness.if).toContain("github.actor != 'dependabot[bot]'");
|
||||
expect(jobs.readiness.if).toContain("github.event_name == 'pull_request'");
|
||||
expect(jobs.readiness.if).toContain('github.event.pull_request.head.repo.full_name == github.repository');
|
||||
expect(jobs['recover-receipts'].if).toContain("github.actor != 'dependabot[bot]'");
|
||||
expect(jobs['recover-receipts'].if).toContain("(github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository");
|
||||
expect(jobs['recover-receipts'].if).toContain("(github.event_name == 'workflow_dispatch' && inputs.pr_receipts != '')");
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
/**
|
||||
* Readiness-gated dispatch (R1, GSTA-48), driven from scripted gh responses
|
||||
* and a fake clock: no network, no sleeping.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { freeTestsState, readinessDecision, waitForReadiness, type FreeTestsRun } from '../scripts/eval-readiness';
|
||||
import type { GhRunner } from '../scripts/recover-receipts';
|
||||
|
||||
const HEAD = 'a'.repeat(40);
|
||||
const NEWER = 'b'.repeat(40);
|
||||
const CREATED = Date.parse('2026-10-18T00:00:00Z');
|
||||
const free = (id: number, status: string, conclusion: string | null, over: Partial<FreeTestsRun> = {}): FreeTestsRun =>
|
||||
({ id, status, conclusion, head_sha: HEAD, event: 'pull_request', ...over });
|
||||
|
||||
/** A gh that answers the run, Free Tests and PR reads from a per-poll script, recording each call. */
|
||||
function fakeGh(polls: Array<{ free: FreeTestsRun[]; prHead?: string } | 'error'>, createdAt: string | null = new Date(CREATED).toISOString()) {
|
||||
const calls: string[] = [];
|
||||
let poll = -1;
|
||||
const gh: GhRunner = (args) => {
|
||||
const target = args.find(a => a.startsWith('repos/')) ?? '';
|
||||
calls.push(target);
|
||||
const ok = (body: unknown) => ({ status: 0, stdout: JSON.stringify(body), stderr: '' });
|
||||
if (/\/actions\/runs\/\d+$/.test(target)) return createdAt ? ok({ created_at: createdAt }) : { status: 1, stdout: '', stderr: 'HTTP 502' };
|
||||
if (target.endsWith('/actions/workflows/free-tests.yml/runs')) {
|
||||
poll++;
|
||||
const step = polls[Math.min(poll, polls.length - 1)]!;
|
||||
return step === 'error' ? { status: 1, stdout: '', stderr: 'HTTP 502' } : ok({ workflow_runs: step.free });
|
||||
}
|
||||
if (/\/pulls\/\d+$/.test(target)) {
|
||||
const step = polls[Math.min(poll, polls.length - 1)]!;
|
||||
return ok({ head: { sha: step === 'error' ? HEAD : step.prHead ?? HEAD } });
|
||||
}
|
||||
return { status: 1, stdout: '', stderr: 'unexpected' };
|
||||
};
|
||||
return { gh, calls };
|
||||
}
|
||||
|
||||
function clock(start: number) {
|
||||
let t = start;
|
||||
const sleeps: number[] = [];
|
||||
return { now: () => t, sleep: (ms: number) => { sleeps.push(ms); t += ms; }, sleeps, at: () => t };
|
||||
}
|
||||
|
||||
const base = { repo: 'garrytan/gstack', pr: 42, headSha: HEAD, runId: 900, quietMs: 10 * 60_000, deadlineMs: 80 * 60_000, pollMs: 60_000 };
|
||||
|
||||
describe('readiness decision', () => {
|
||||
test('the newest Free Tests run on this exact head decides; other heads and events are ignored', () => {
|
||||
expect(freeTestsState([], HEAD)).toEqual({ state: 'missing' });
|
||||
expect(freeTestsState([free(1, 'completed', 'success', { head_sha: NEWER })], HEAD)).toEqual({ state: 'missing' });
|
||||
expect(freeTestsState([free(1, 'completed', 'success', { event: 'push' })], HEAD)).toEqual({ state: 'missing' });
|
||||
expect(freeTestsState([free(1, 'completed', 'failure'), free(2, 'in_progress', null)], HEAD)).toEqual({ state: 'running', runId: 2 });
|
||||
expect(freeTestsState([free(2, 'completed', 'success'), free(1, 'completed', 'failure')], HEAD)).toEqual({ state: 'success', runId: 2 });
|
||||
expect(freeTestsState([free(3, 'completed', 'cancelled')], HEAD)).toEqual({ state: 'failed', runId: 3, conclusion: 'cancelled' });
|
||||
});
|
||||
|
||||
test('ready needs both a green Free Tests on the head and an elapsed window; a moved head is superseded', () => {
|
||||
const green = { state: 'success', runId: 7 } as const;
|
||||
expect(readinessDecision({ headSha: HEAD, prHead: HEAD, free: green, quietLeftMs: 0 }).kind).toBe('ready');
|
||||
expect(readinessDecision({ headSha: HEAD, prHead: HEAD, free: green, quietLeftMs: 1 }).kind).toBe('wait');
|
||||
expect(readinessDecision({ headSha: HEAD, prHead: HEAD, free: { state: 'running', runId: 7 }, quietLeftMs: 0 }).kind).toBe('wait');
|
||||
expect(readinessDecision({ headSha: HEAD, prHead: HEAD, free: { state: 'missing' }, quietLeftMs: 0 }).kind).toBe('wait');
|
||||
expect(readinessDecision({ headSha: HEAD, prHead: NEWER, free: green, quietLeftMs: 0 })).toEqual({ kind: 'superseded', by: NEWER });
|
||||
const blocked = readinessDecision({ headSha: HEAD, prHead: HEAD, free: { state: 'failed', runId: 7, conclusion: 'failure' }, quietLeftMs: 0 });
|
||||
expect(blocked.kind).toBe('blocked');
|
||||
expect(blocked.kind === 'blocked' && blocked.reason).toContain('re-run Free Tests and then re-run this workflow');
|
||||
});
|
||||
});
|
||||
|
||||
describe('waitForReadiness', () => {
|
||||
test('sleeps out the window with no API calls, then starts once Free Tests passes', () => {
|
||||
const { gh, calls } = fakeGh([{ free: [free(5, 'in_progress', null)] }, { free: [free(5, 'completed', 'success')] }]);
|
||||
const c = clock(CREATED + 5_000);
|
||||
const { decision, lines } = waitForReadiness({ ...base, gh, now: c.now, sleep: c.sleep });
|
||||
expect(decision.kind).toBe('ready');
|
||||
// One run read, then nothing until the window closed at created + 10 min.
|
||||
expect(calls[0]).toBe('repos/garrytan/gstack/actions/runs/900');
|
||||
expect(c.sleeps[0]).toBe(10 * 60_000 - 5_000);
|
||||
expect(calls.filter(call => call.endsWith('free-tests.yml/runs'))).toHaveLength(2);
|
||||
expect(c.at()).toBe(CREATED + 11 * 60_000);
|
||||
expect(lines.join('\n')).toContain('Free Tests run 5 passed');
|
||||
});
|
||||
|
||||
test('a re-run keeps the original creation time, so an elapsed window is not waited again', () => {
|
||||
const { gh } = fakeGh([{ free: [free(5, 'completed', 'success')] }]);
|
||||
const c = clock(CREATED + 3 * 3600_000);
|
||||
expect(waitForReadiness({ ...base, gh, now: c.now, sleep: c.sleep }).decision.kind).toBe('ready');
|
||||
expect(c.sleeps).toEqual([]);
|
||||
});
|
||||
|
||||
test('a quiet window of 0 (the evals-no-debounce label) still waits for Free Tests', () => {
|
||||
const { gh } = fakeGh([{ free: [] }, { free: [free(5, 'queued', null)] }, { free: [free(5, 'completed', 'success')] }]);
|
||||
const c = clock(CREATED);
|
||||
expect(waitForReadiness({ ...base, quietMs: 0, gh, now: c.now, sleep: c.sleep }).decision.kind).toBe('ready');
|
||||
expect(c.sleeps).toEqual([60_000, 60_000]);
|
||||
});
|
||||
|
||||
test('Free Tests red on this head blocks the paid gate', () => {
|
||||
const { gh } = fakeGh([{ free: [free(5, 'completed', 'failure')] }]);
|
||||
const c = clock(CREATED);
|
||||
const { decision } = waitForReadiness({ ...base, gh, now: c.now, sleep: c.sleep });
|
||||
expect(decision.kind).toBe('blocked');
|
||||
});
|
||||
|
||||
test('a newer head is reported superseded instead of starting', () => {
|
||||
const { gh } = fakeGh([{ free: [free(5, 'completed', 'success')], prHead: NEWER }]);
|
||||
const c = clock(CREATED);
|
||||
const { decision, lines } = waitForReadiness({ ...base, gh, now: c.now, sleep: c.sleep });
|
||||
expect(decision).toEqual({ kind: 'superseded', by: NEWER });
|
||||
expect(lines.join('\n')).toContain('this run plans no slices');
|
||||
});
|
||||
|
||||
test('a Free Tests run that never finishes, or a gh outage, fails at the deadline with the last reason', () => {
|
||||
for (const polls of [[{ free: [free(5, 'in_progress', null)] }], ['error' as const]]) {
|
||||
const { gh } = fakeGh(polls);
|
||||
const c = clock(CREATED);
|
||||
const { decision } = waitForReadiness({ ...base, gh, now: c.now, sleep: c.sleep });
|
||||
expect(decision.kind).toBe('blocked');
|
||||
expect(decision.kind === 'blocked' && decision.reason).toContain('not ready after 80 min');
|
||||
expect(c.at() - CREATED).toBeLessThanOrEqual(80 * 60_000);
|
||||
}
|
||||
});
|
||||
|
||||
test('an unreadable run creation time measures the window from now', () => {
|
||||
const { gh } = fakeGh([{ free: [free(5, 'completed', 'success')] }], null);
|
||||
const c = clock(CREATED + 60_000);
|
||||
expect(waitForReadiness({ ...base, gh, now: c.now, sleep: c.sleep }).decision.kind).toBe('ready');
|
||||
expect(c.sleeps[0]).toBe(10 * 60_000);
|
||||
});
|
||||
});
|
||||
@@ -1,7 +1,8 @@
|
||||
/**
|
||||
* PR paid-lane workflow pins (W1, CEO-13/14/34, ENG-7/11/12, DX-7, W8f):
|
||||
* coverage summaries, cancellation recovery off PR code, the push-burst
|
||||
* debounce, dispatch inputs, and the absence of the old inline subset program.
|
||||
* coverage summaries, cancellation recovery off PR code, readiness-gated
|
||||
* dispatch (R1), restamp-only reuse wiring (R2a), dispatch inputs, and the
|
||||
* absence of the old inline subset program.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
@@ -77,24 +78,30 @@ describe('evals.yml receipt recovery (CEO-14/34, ENG-7)', () => {
|
||||
});
|
||||
|
||||
|
||||
describe('evals.yml push-burst debounce (CEO-15, ENG-11, DX-7)', () => {
|
||||
const debounce = workflow.jobs.debounce!;
|
||||
describe('evals.yml readiness-gated dispatch (R1, GSTA-48)', () => {
|
||||
const readiness = workflow.jobs.readiness!;
|
||||
|
||||
test('a tiny job debounces only synchronize pushes, skippable by label, before recovery and planning', () => {
|
||||
expect(debounce['runs-on']).toBe('ubuntu-24.04');
|
||||
expect(debounce.if).toContain("github.event.action == 'synchronize'");
|
||||
expect(debounce.if).toContain("!contains(github.event.pull_request.labels.*.name, 'evals-no-debounce')");
|
||||
expect(debounce.permissions).toEqual({ actions: 'read', 'pull-requests': 'read' });
|
||||
const wait = step('debounce', 'Wait out a push burst')!.run!;
|
||||
expect(wait).toContain("date -u -d '15 minutes ago'");
|
||||
expect(wait).toContain('sleep 90');
|
||||
expect(wait).toContain('superseded=true');
|
||||
expect(wait).toContain('superseded by ${head}');
|
||||
test('a tiny PR-only job waits for a quiet PR and green Free Tests before recovery and planning', () => {
|
||||
expect(readiness['runs-on']).toBe('ubuntu-24.04');
|
||||
expect(readiness.if).toBe("github.event_name == 'pull_request' && github.actor != 'dependabot[bot]' && github.event.pull_request.head.repo.full_name == github.repository");
|
||||
expect(readiness.permissions).toEqual({ contents: 'read', actions: 'read', 'pull-requests': 'read' });
|
||||
expect(Number(readiness['timeout-minutes'])).toBeGreaterThan(80);
|
||||
const wait = step('readiness', 'Wait for a quiet PR and green Free Tests on this head')!;
|
||||
expect(wait.id).toBe('wait');
|
||||
expect(wait.run).toContain('scripts/eval-readiness.ts wait');
|
||||
expect(wait.run).toContain('--deadline-minutes 80');
|
||||
expect(wait.run).toContain('--github-output "$GITHUB_OUTPUT"');
|
||||
expect(wait.env?.HEAD_SHA).toBe('${{ github.event.pull_request.head.sha }}');
|
||||
// The window is configurable and the old debounce label still skips it; nothing skips the Free Tests requirement.
|
||||
expect(wait.env?.QUIET_MINUTES).toBe("${{ contains(github.event.pull_request.labels.*.name, 'evals-no-debounce') && '0' || vars.EVALS_QUIET_MINUTES || '10' }}");
|
||||
expect(readiness.outputs?.superseded).toBe('${{ steps.wait.outputs.superseded }}');
|
||||
for (const job of ['recover-receipts', 'plan-slices']) {
|
||||
expect(workflow.jobs[job]!.needs).toContain('debounce');
|
||||
expect(workflow.jobs[job]!.if).toContain("needs.debounce.outputs.superseded != 'true'");
|
||||
expect(workflow.jobs[job]!.needs).toContain('readiness');
|
||||
expect(workflow.jobs[job]!.if).toContain("(needs.readiness.result == 'success' || needs.readiness.result == 'skipped')");
|
||||
expect(workflow.jobs[job]!.if).toContain("needs.readiness.outputs.superseded != 'true'");
|
||||
expect(workflow.jobs[job]!.if).toContain('!cancelled()');
|
||||
}
|
||||
expect(workflow.jobs.debounce).toBeUndefined();
|
||||
});
|
||||
|
||||
test('cancel-in-progress stays keyed on the PR, and evals-fresh turns receipt reuse off', () => {
|
||||
@@ -104,6 +111,41 @@ describe('evals.yml push-burst debounce (CEO-15, ENG-11, DX-7)', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('evals.yml restamp-only gate reuse wiring (R2a, GSTA-48)', () => {
|
||||
test('base-ref recovery proposes a candidate; it never fails the run', () => {
|
||||
const find = step('recover-receipts', 'Find a restamp-only gate to reuse (R2a, base-ref code)')!;
|
||||
expect(find.if).toBe("github.event_name == 'pull_request'");
|
||||
expect(find.env).toMatchObject({ GH_TOKEN: '${{ github.token }}', HEAD_SHA: '${{ github.event.pull_request.head.sha }}', BASE_SHA: '${{ github.event.pull_request.base.sha }}' });
|
||||
expect(find.run).toContain('scripts/gate-reuse.ts find');
|
||||
expect(find.run).toContain('exit 0');
|
||||
expect(find.run).toContain('|| echo');
|
||||
expect(step('recover-receipts', 'Hand a restamp-only reuse candidate to plan-slices')?.with).toMatchObject({ name: 'gate-reuse-candidate', path: '/tmp/gate-reuse' });
|
||||
});
|
||||
|
||||
test('the planner confirms the selection after its own plan, and a confirmed reuse runs no slice', () => {
|
||||
const planner = workflow.jobs['plan-slices']!;
|
||||
const names = planner.steps.map(s => s.name);
|
||||
expect(names.indexOf('Confirm restamp-only gate reuse (R2a)')).toBeGreaterThan(names.indexOf('Emit run manifest'));
|
||||
expect(step('plan-slices', 'Confirm restamp-only gate reuse (R2a)')!.run).toContain('gate-reuse.ts confirm /tmp/gate-reuse /tmp/paid-plan/manifest.json --out /tmp/paid-plan');
|
||||
expect(planner.outputs?.gate_reuse).toBe('${{ steps.gate_reuse.outputs.gate_reuse }}');
|
||||
expect(workflow.jobs['eval-slices']!.if).toContain("needs.plan-slices.outputs.gate_reuse != 'true'");
|
||||
});
|
||||
|
||||
test('the report prints a reused verdict fail-closed and only executed runs bind a verdict', () => {
|
||||
const report = workflow.jobs['slices-report']!;
|
||||
expect(step('slices-report', 'Reconcile slices against the manifest (fail-closed)')!.if).toBe("needs.plan-slices.outputs.gate_reuse != 'true'");
|
||||
const reused = step('slices-report', 'Report the reused restamp-only verdict (R2a)')!;
|
||||
expect(reused.if).toBe("needs.plan-slices.outputs.gate_reuse == 'true'");
|
||||
expect(reused.run).toContain('echo "exit=$code"');
|
||||
expect(report.outputs?.['reconcile-exit']).toBe('${{ steps.reconcile.outputs.exit || steps.reused.outputs.exit }}');
|
||||
expect(step('slices-report', 'Fail the workflow when reconciliation failed')!.if).toBe("(steps.reconcile.outputs.exit || steps.reused.outputs.exit) != '0'");
|
||||
const bind = step('slices-report', "Bind this run's verdict for restamp-only reuse (R2a)")!;
|
||||
expect(bind.if).toBe("always() && github.event_name == 'pull_request' && needs.plan-slices.outputs.gate_reuse != 'true'");
|
||||
expect(bind.env?.RECONCILE_EXIT).toBe('${{ steps.reconcile.outputs.exit }}');
|
||||
expect(step('slices-report', 'Upload the gate binding')?.with).toMatchObject({ name: 'gate-binding-a${{ github.run_attempt }}', path: '/tmp/gate-binding/gate-binding.json' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('evals.yml carries no inline planner program (W8f)', () => {
|
||||
test('the finished validation_phase dispatch and its inline subset planner are gone', () => {
|
||||
expect(source).not.toContain('validation_phase');
|
||||
|
||||
@@ -0,0 +1,309 @@
|
||||
/**
|
||||
* Restamp-only gate reuse (R2a, GSTA-48): a gate edit, so every refusal is a
|
||||
* negative control. Real git fixtures (a bare origin, the base-ref receipt
|
||||
* checkout) and scripted gh responses: no network, no API spend.
|
||||
*/
|
||||
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import {
|
||||
BINDING_FILE, CANDIDATE_FILE, REUSE_FILE, buildBinding, confirmReuse, findReusableGate, parseReuse, reuseReport, selectionFingerprint,
|
||||
type GateBinding, type GateReuse,
|
||||
} from '../scripts/gate-reuse';
|
||||
import type { GhRunner, WorkflowRun } from '../scripts/recover-receipts';
|
||||
import type { PaidRunManifest } from '../scripts/lib/paid-plan';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const REPO = 'garrytan/gstack';
|
||||
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'gate-reuse-'));
|
||||
afterAll(() => fs.rmSync(scratch, { recursive: true, force: true }));
|
||||
|
||||
function git(cwd: string, ...args: string[]): string {
|
||||
const r = spawnSync('git', ['-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.com', '-c', 'commit.gpgsign=false', ...args],
|
||||
{ cwd, encoding: 'utf8', timeout: 30_000 });
|
||||
if (r.status !== 0) throw new Error('git ' + args.join(' ') + ': ' + r.stderr);
|
||||
return r.stdout.trim();
|
||||
}
|
||||
|
||||
const changelog = (sections: string[]) => ['# Changelog', '', ...sections, '## [1.0.0.0] - 2026-10-01', '', '- base release', ''].join('\n');
|
||||
const files = (version: string, over: Record<string, string> = {}): Record<string, string> => ({
|
||||
VERSION: version + '\n',
|
||||
'package.json': JSON.stringify({ name: 'fixture', version, scripts: { test: 'bun test' } }, null, 2) + '\n',
|
||||
'agents-digest/gstack-AGENTS.md': '# gstack digest v' + version + ' — regenerate after upgrading\n\nbody\n',
|
||||
'src/feature.ts': 'export const feature = 1;\n',
|
||||
...over,
|
||||
});
|
||||
|
||||
let origin: string;
|
||||
let receipt: string;
|
||||
const sha: Record<string, string> = {};
|
||||
|
||||
function commit(work: string, branch: string, from: string, content: Record<string, string>, message: string): string {
|
||||
if (from) git(work, 'checkout', '-q', '-B', branch, from);
|
||||
else git(work, 'checkout', '-q', '--orphan', branch);
|
||||
for (const [rel, text] of Object.entries(content)) {
|
||||
fs.mkdirSync(path.dirname(path.join(work, rel)), { recursive: true });
|
||||
fs.writeFileSync(path.join(work, rel), text);
|
||||
}
|
||||
git(work, 'add', '-A');
|
||||
git(work, 'commit', '-q', '-m', message);
|
||||
git(work, 'push', '-q', 'origin', branch);
|
||||
return git(work, 'rev-parse', 'HEAD');
|
||||
}
|
||||
|
||||
beforeAll(() => {
|
||||
origin = path.join(scratch, 'origin.git');
|
||||
git(scratch, 'init', '-q', '--bare', origin);
|
||||
git(origin, 'config', 'uploadpack.allowAnySHA1InWant', 'true');
|
||||
const work = path.join(scratch, 'work');
|
||||
git(scratch, 'clone', '-q', origin, work);
|
||||
const policy = fs.readFileSync(path.join(ROOT, '.gstack/ship-policy.json'), 'utf8');
|
||||
sha.base = commit(work, 'main', '', { ...files('1.0.0.0', { 'src/feature.ts': 'export const feature = 0;\n' }), 'CHANGELOG.md': changelog([]),
|
||||
'.gstack/ship-policy.json': policy }, 'base');
|
||||
const entry = ['## [1.0.1.0] - 2026-10-02', '', '- the feature', ''];
|
||||
sha.gated = commit(work, 'feature', sha.base, { ...files('1.0.1.0'), 'CHANGELOG.md': changelog(entry) }, 'feature, tentative stamp');
|
||||
sha.restamp = commit(work, 'restamp', sha.gated, { ...files('1.0.2.0'), 'CHANGELOG.md': changelog(['## [1.0.2.0] - 2026-10-03', '', '- the feature', '']) }, 'restamp');
|
||||
sha.code = commit(work, 'code', sha.restamp, { 'src/feature.ts': 'export const feature = 2;\n' }, 'restamp plus a code byte');
|
||||
sha.entry = commit(work, 'entry', sha.gated, { ...files('1.0.2.0'), 'CHANGELOG.md': changelog(['## [1.0.2.0] - 2026-10-03', '', '- the feature, reworded', '']) }, 'restamp plus entry text');
|
||||
sha.lock = commit(work, 'lock', sha.gated, { 'bun.lock': '{\n "lockfileVersion": 1\n}\n' }, 'a non-stamp file beside the stamp');
|
||||
const lock = (version: string) => '{\n "lockfileVersion": 1,\n "workspaces": {\n "": {\n "name": "fixture",\n "version": "' + version + '",\n },\n },\n}\n';
|
||||
sha.gatedLock = commit(work, 'gated-lock', sha.gated, { 'bun.lock': lock('1.0.1.0') }, 'a lockfile carrying the root version');
|
||||
sha.lockStamp = commit(work, 'lock-stamp', sha.gatedLock, { ...files('1.0.2.0'), 'CHANGELOG.md': changelog(['## [1.0.2.0] - 2026-10-03', '', '- the feature', '']),
|
||||
'bun.lock': lock('1.0.2.0') }, 'restamp that also moves the lockfile root version');
|
||||
sha.moved = commit(work, 'main', sha.base, { 'README.md': 'main moved\n' }, 'main moves');
|
||||
receipt = path.join(scratch, 'receipt');
|
||||
git(scratch, 'clone', '-q', '--depth=1', 'file://' + origin, receipt);
|
||||
git(receipt, 'fetch', '-q', '--depth=1', 'origin', sha.base);
|
||||
git(receipt, 'checkout', '-q', '--detach', sha.base);
|
||||
});
|
||||
|
||||
const binding = (over: Partial<GateBinding> = {}): GateBinding => ({ schema: 1, repository: REPO, pr: 42, run_id: 300, run_attempt: 1,
|
||||
head_sha: sha.gated!, base_sha: sha.base!, profile: 'pr', reconcile_exit: 0, verdict: 'GREEN', selection_sha256: 'f'.repeat(64), ...over });
|
||||
const run = (id: number, head: string, conclusion: string | null = 'success', over: Partial<WorkflowRun> = {}): WorkflowRun => ({
|
||||
id, status: 'completed', conclusion, event: 'pull_request', head_sha: head, run_attempt: 1, head_repository: { full_name: REPO },
|
||||
pull_requests: [{ number: 42 }], ...over });
|
||||
|
||||
/** gh: the run list, each run's artifact list, and downloads that write the scripted binding (or raw text). */
|
||||
function fakeGh(runs: WorkflowRun[], bindings: Record<number, GateBinding | string>, failList = ''): GhRunner {
|
||||
return (args) => {
|
||||
const ok = (body: unknown) => ({ status: 0, stdout: JSON.stringify(body), stderr: '' });
|
||||
if (args.some(a => a.endsWith('/workflows/evals.yml/runs'))) return failList ? { status: 1, stdout: '', stderr: failList } : ok({ workflow_runs: runs });
|
||||
const listed = args.find(a => /\/runs\/\d+\/artifacts$/.test(a));
|
||||
if (listed) {
|
||||
const id = Number(/runs\/(\d+)\//.exec(listed)![1]);
|
||||
return ok({ artifacts: id in bindings ? [{ name: 'gate-binding-a1', expired: false }, { name: 'paid-plan', expired: false }] : [{ name: 'paid-plan', expired: false }] });
|
||||
}
|
||||
if (args[0] === 'run' && args[1] === 'download') {
|
||||
const dir = args[args.indexOf('-D') + 1]!;
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
const b = bindings[Number(args[2])]!;
|
||||
fs.writeFileSync(path.join(dir, BINDING_FILE), typeof b === 'string' ? b : JSON.stringify(b));
|
||||
return { status: 0, stdout: '', stderr: '' };
|
||||
}
|
||||
return { status: 1, stdout: '', stderr: 'unexpected gh ' + args.join(' ') };
|
||||
};
|
||||
}
|
||||
|
||||
let outCount = 0;
|
||||
function find(head: string, runs: WorkflowRun[], bindings: Record<number, GateBinding | string>, over: { baseSha?: string; failList?: string } = {}) {
|
||||
const out = path.join(scratch, 'out-' + ++outCount);
|
||||
const result = findReusableGate({ repo: REPO, pr: 42, headRepo: REPO, branch: 'feature', selfRunId: 500, headSha: head,
|
||||
baseSha: over.baseSha ?? sha.base!, repoRoot: receipt, out, gh: fakeGh(runs, bindings, over.failList) });
|
||||
return { ...result, out, text: result.lines.join('\n') };
|
||||
}
|
||||
|
||||
describe('find: the positive case', () => {
|
||||
test('a restamp of a GREEN head on the same base yields a candidate with the receipt', () => {
|
||||
const { candidate, out, text } = find(sha.restamp!, [run(300, sha.gated!)], { 300: binding() });
|
||||
expect(text).toContain('TREE: same-modulo-stamps');
|
||||
expect(candidate).not.toBeNull();
|
||||
expect(candidate!.stamp_files.sort()).toEqual(['CHANGELOG.md', 'VERSION', 'agents-digest/gstack-AGENTS.md', 'package.json']);
|
||||
expect(candidate!).toMatchObject({ reused_run_id: 300, gated_head: sha.gated, current_head: sha.restamp, base_sha: sha.base,
|
||||
old_version: '1.0.1.0', new_version: '1.0.2.0', selection_sha256: 'f'.repeat(64) });
|
||||
expect(candidate!.receipt_sha256).toMatch(/^[0-9a-f]{64}$/);
|
||||
expect(parseReuse(JSON.parse(fs.readFileSync(path.join(out, CANDIDATE_FILE), 'utf8')))).toEqual(candidate);
|
||||
});
|
||||
|
||||
test('cancelled or skipped runs that bound nothing are passed over to the newest executed gate', () => {
|
||||
const { candidate } = find(sha.restamp!, [run(320, sha.gated!, 'cancelled'), run(310, sha.gated!, 'success'), run(300, sha.gated!)], { 300: binding() });
|
||||
expect(candidate?.reused_run_id).toBe(300);
|
||||
});
|
||||
});
|
||||
|
||||
describe('find: negative controls (each forces the full gate)', () => {
|
||||
const refused = (r: { candidate: GateReuse | null; out: string; text: string }, reason: string | RegExp) => {
|
||||
expect(r.candidate).toBeNull();
|
||||
expect(fs.existsSync(path.join(r.out, CANDIDATE_FILE))).toBe(false);
|
||||
if (typeof reason === 'string') expect(r.text).toContain(reason);
|
||||
else expect(r.text).toMatch(reason);
|
||||
expect(r.text).toContain('the full gate runs');
|
||||
};
|
||||
|
||||
test('a non-stamp byte change', () => {
|
||||
refused(find(sha.code!, [run(300, sha.gated!)], { 300: binding() }), 'differs from the gated head beyond stamps: src/feature.ts');
|
||||
});
|
||||
|
||||
test('CHANGELOG entry text, not only its heading', () => {
|
||||
refused(find(sha.entry!, [run(300, sha.gated!)], { 300: binding() }), 'beyond stamps: CHANGELOG.md');
|
||||
});
|
||||
|
||||
test('a file outside the stamp set', () => {
|
||||
refused(find(sha.lock!, [run(300, sha.gated!)], { 300: binding() }), 'bun.lock');
|
||||
// The receipt calls a lockfile root-version bump a stamp; R2a's stamp set is narrower and still refuses it.
|
||||
refused(find(sha.lockStamp!, [run(300, sha.gatedLock!)], { 300: binding({ head_sha: sha.gatedLock }) }),
|
||||
'stamp-classified files outside the version file, package.json, CHANGELOG and release outputs: bun.lock');
|
||||
});
|
||||
|
||||
test('a different base: main moved between the gate and this run', () => {
|
||||
refused(find(sha.restamp!, [run(300, sha.gated!)], { 300: binding() }, { baseSha: sha.moved }), 'main moved');
|
||||
});
|
||||
|
||||
test('a missing receipt: no executed gate bound a verdict', () => {
|
||||
refused(find(sha.restamp!, [run(300, sha.gated!)], {}), 'no earlier executed gate on this PR bound a verdict');
|
||||
refused(find(sha.restamp!, [], {}), 'no earlier executed gate');
|
||||
});
|
||||
|
||||
test('a missing receipt: a malformed or mismatched binding', () => {
|
||||
refused(find(sha.restamp!, [run(300, sha.gated!)], { 300: '{"schema":1' }), 'missing or malformed');
|
||||
refused(find(sha.restamp!, [run(300, sha.gated!)], { 300: binding({ run_id: 299 }) }), 'does not match the run it came from');
|
||||
refused(find(sha.restamp!, [run(300, sha.restamp!)], { 300: binding() }), 'does not match the run it came from');
|
||||
});
|
||||
|
||||
test('a missing receipt: the gated head cannot be fetched, so no tree receipt exists', () => {
|
||||
const ghost = 'e'.repeat(40);
|
||||
refused(find(sha.restamp!, [run(300, ghost)], { 300: binding({ head_sha: ghost }) }), /cannot fetch|no tree receipt/);
|
||||
});
|
||||
|
||||
test('a red, partial or failed gate, and a newer red that blocks an older green', () => {
|
||||
refused(find(sha.restamp!, [run(300, sha.gated!)], { 300: binding({ verdict: 'RED' }) }), 'is RED');
|
||||
refused(find(sha.restamp!, [run(300, sha.gated!)], { 300: binding({ reconcile_exit: 1 }) }), 'reconcile exit 1');
|
||||
refused(find(sha.restamp!, [run(300, sha.gated!, 'failure')], { 300: binding() }), 'conclusion failure');
|
||||
refused(find(sha.restamp!, [run(301, sha.gated!), run(300, sha.gated!)], { 301: binding({ run_id: 301, verdict: 'RED' }), 300: binding() }), 'run 301');
|
||||
});
|
||||
|
||||
test('a full-profile binding, the same head, another PR, and a gh outage', () => {
|
||||
refused(find(sha.restamp!, [run(300, sha.gated!)], { 300: binding({ profile: 'full' }) }), 'not the PR gate');
|
||||
refused(find(sha.gated!, [run(300, sha.gated!)], { 300: binding() }), 'gated this same head');
|
||||
refused(find(sha.restamp!, [run(300, sha.gated!, 'success', { pull_requests: [{ number: 7 }] })], { 300: binding() }), 'no earlier executed gate');
|
||||
refused(find(sha.restamp!, [run(300, sha.gated!)], { 300: binding() }, { failList: 'HTTP 403: Resource not accessible' }), 'not authorized');
|
||||
});
|
||||
});
|
||||
|
||||
const manifest = (over: Partial<PaidRunManifest> = {}): PaidRunManifest => ({ version: 1, tier: 'gate', evalsAll: false, sliceCount: 2,
|
||||
selectionReason: 'diff', profile: 'pr', selection: { e2e: ['case-a'], judges: [] }, prCoverage: { mode: 'pr' } as PaidRunManifest['prCoverage'],
|
||||
entries: [
|
||||
{ file: 'test/a.test.ts', slice: 1, status: 'planned', estimatedMs: 1000 },
|
||||
{ file: 'test/b.test.ts#case-b~t1', slice: 2, status: 'planned', trial: { kind: 'behavior', panel: { n: 3, k: 2 }, quarantined: false } },
|
||||
{ file: 'test/c.test.ts', slice: 0, status: 'skipped-by-diff', reason: 'no input changed' },
|
||||
], ...over } as PaidRunManifest);
|
||||
|
||||
describe('selection fingerprint and confirm', () => {
|
||||
test('slice packing and estimates do not change the fingerprint; cases, statuses and trial plans do', () => {
|
||||
const fp = selectionFingerprint(manifest());
|
||||
const repacked = manifest();
|
||||
repacked.entries = [...repacked.entries].reverse().map(e => ({ ...e, slice: e.slice ? 3 - e.slice : 0, estimatedMs: 5, reason: 'other' }));
|
||||
expect(selectionFingerprint(repacked)).toBe(fp);
|
||||
const extra = manifest();
|
||||
extra.entries = [...extra.entries, { file: 'test/d.test.ts', slice: 1, status: 'planned' }];
|
||||
const quarantined = manifest();
|
||||
quarantined.entries = quarantined.entries.map(e => e.trial ? { ...e, trial: { ...e.trial, quarantined: true } } : e);
|
||||
const skipped = manifest();
|
||||
skipped.entries = skipped.entries.map(e => e.file === 'test/a.test.ts' ? { ...e, status: 'skipped-by-diff' as const } : e);
|
||||
for (const changed of [extra, quarantined, skipped, manifest({ selection: { e2e: ['case-a', 'case-z'], judges: [] } }), manifest({ evalsAll: true })]) {
|
||||
expect(selectionFingerprint(changed)).not.toBe(fp);
|
||||
}
|
||||
});
|
||||
|
||||
test('confirm reuses only when the candidate exists and the selection is identical', () => {
|
||||
const { out } = find(sha.restamp!, [run(300, sha.gated!)], { 300: binding({ selection_sha256: selectionFingerprint(manifest()) }) });
|
||||
const same = confirmReuse(out, manifest());
|
||||
expect(same.reuse?.reused_run_id).toBe(300);
|
||||
expect(same.reason).toContain('same selection');
|
||||
const extra = manifest();
|
||||
extra.entries = [...extra.entries, { file: 'test/d.test.ts', slice: 1, status: 'planned' }];
|
||||
expect(confirmReuse(out, extra).reuse).toBeNull();
|
||||
expect(confirmReuse(out, extra).reason).toContain('differs from the reused gate');
|
||||
expect(confirmReuse(out, null).reuse).toBeNull();
|
||||
expect(confirmReuse(path.join(scratch, 'no-candidate'), manifest()).reuse).toBeNull();
|
||||
const forged = path.join(scratch, 'forged');
|
||||
fs.mkdirSync(forged, { recursive: true });
|
||||
fs.writeFileSync(path.join(forged, CANDIDATE_FILE), JSON.stringify({ ...JSON.parse(fs.readFileSync(path.join(out, CANDIDATE_FILE), 'utf8')), stamp_files: [] }));
|
||||
expect(confirmReuse(forged, manifest()).reuse).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('bind and report', () => {
|
||||
const reportDir = (name: string, withReuse = false) => {
|
||||
const dir = path.join(scratch, name);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, 'manifest.json'), JSON.stringify(manifest()));
|
||||
fs.writeFileSync(path.join(dir, 'collector-outcomes.json'), JSON.stringify({ verdict: { verdict: 'GREEN' } }));
|
||||
if (withReuse) fs.writeFileSync(path.join(dir, REUSE_FILE), '{}');
|
||||
return dir;
|
||||
};
|
||||
const bindArgs = { repository: REPO, pr: 42, runId: 300, runAttempt: 1, headSha: 'a'.repeat(40), baseSha: 'b'.repeat(40), profile: 'pr', reconcileExit: 0 };
|
||||
|
||||
test('an executed run binds its verdict, reconcile exit and selection; a reused run binds nothing', () => {
|
||||
const { binding: bound } = buildBinding({ ...bindArgs, reportDir: reportDir('bind-ok') });
|
||||
expect(bound).toMatchObject({ verdict: 'GREEN', reconcile_exit: 0, selection_sha256: selectionFingerprint(manifest()) });
|
||||
expect(buildBinding({ ...bindArgs, reportDir: reportDir('bind-reused', true) }).binding).toBeNull();
|
||||
const empty = path.join(scratch, 'bind-empty');
|
||||
fs.mkdirSync(empty, { recursive: true });
|
||||
expect(buildBinding({ ...bindArgs, reportDir: empty }).binding).toBeNull();
|
||||
});
|
||||
|
||||
test('the CLI binds a missing reconcile exit as -1, never as a pass', () => {
|
||||
const out = path.join(scratch, 'bind-cli');
|
||||
const r = spawnSync(process.execPath, ['run', path.join(ROOT, 'scripts/gate-reuse.ts'), 'bind', reportDir('bind-cli-report'), '--out', out,
|
||||
'--repo', REPO, '--pr', '42', '--run-id', '300', '--run-attempt', '1', '--head-sha', 'a'.repeat(40), '--base-sha', 'b'.repeat(40),
|
||||
'--profile', 'pr', '--reconcile-exit', ''], { encoding: 'utf8', timeout: 30_000 });
|
||||
expect(r.status, r.stderr).toBe(0);
|
||||
expect(JSON.parse(fs.readFileSync(path.join(out, BINDING_FILE), 'utf8')).reconcile_exit).toBe(-1);
|
||||
});
|
||||
|
||||
test('report prints the reused verdict with its receipt, and fails closed without a valid record', () => {
|
||||
const { out } = find(sha.restamp!, [run(300, sha.gated!)], { 300: binding() });
|
||||
const reuse = parseReuse(JSON.parse(fs.readFileSync(path.join(out, CANDIDATE_FILE), 'utf8')))!;
|
||||
const text = reuseReport(reuse, REPO);
|
||||
expect(text).toContain('GATE REUSED (R2a, restamp-only)');
|
||||
expect(text).toContain('TREE: same-modulo-stamps receipt=' + reuse.receipt_sha256);
|
||||
expect(text).toContain('https://github.com/garrytan/gstack/actions/runs/300');
|
||||
const bad = spawnSync(process.execPath, ['run', path.join(ROOT, 'scripts/gate-reuse.ts'), 'report', path.join(scratch, 'missing.json'), '--repo', REPO],
|
||||
{ encoding: 'utf8', timeout: 30_000 });
|
||||
expect(bad.status).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!Bun.which('jq') || !Bun.which('bash'))('the PR comment for a reused verdict', () => {
|
||||
test('runs no repository code and names the reused run and receipt; without reconcile exit 0 it never says PASS (reused)', () => {
|
||||
const workflow = Bun.YAML.parse(fs.readFileSync(path.join(ROOT, '.github/workflows/evals.yml'), 'utf8')) as any;
|
||||
const script = (workflow.jobs['slices-comment'].steps.find((s: any) => s.name === 'Post PR comment').run as string);
|
||||
const root = path.join(scratch, 'comment');
|
||||
const paidDir = path.join(root, 'paid-report');
|
||||
const binDir = path.join(root, 'bin');
|
||||
fs.mkdirSync(paidDir, { recursive: true });
|
||||
fs.mkdirSync(binDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(binDir, 'gh'), '#!/bin/sh\ncase "$*" in *--jq*) exit 0;; esac\nfor arg do case "$arg" in body=*) printf "%s\\n" "${arg#body=}";; esac; done\n');
|
||||
fs.chmodSync(path.join(binDir, 'gh'), 0o755);
|
||||
fs.writeFileSync(path.join(binDir, 'bc'), '#!/bin/sh\nread -r expression\nprintf "0\\n"\n');
|
||||
fs.chmodSync(path.join(binDir, 'bc'), 0o755);
|
||||
const { out } = find(sha.restamp!, [run(300, sha.gated!)], { 300: binding() });
|
||||
fs.copyFileSync(path.join(out, CANDIDATE_FILE), path.join(paidDir, REUSE_FILE));
|
||||
fs.writeFileSync(path.join(paidDir, 'manifest.json'), JSON.stringify({ profile: 'pr', selection: { e2e: [], judges: [] } }));
|
||||
const body = script.replaceAll('/tmp/paid-report', paidDir).replaceAll('/tmp/verdict', path.join(root, 'verdict'))
|
||||
.replaceAll('$' + '{{ github.repository }}', REPO).replaceAll('$' + '{{ github.event.pull_request.number }}', '42');
|
||||
const exec = (reconcile: string) => spawnSync('bash', ['-e', '-c', body], { cwd: root, encoding: 'utf8', timeout: 10_000,
|
||||
env: { ...process.env, PATH: binDir + ':' + process.env.PATH, RECONCILE_EXIT: reconcile, GITHUB_REPOSITORY: REPO } });
|
||||
const reused = exec('0');
|
||||
expect(reused.status, reused.stderr).toBe(0);
|
||||
expect(reused.stdout).toContain('## E2E Evals: ✅ PASS (reused: restamp-only, R2a)');
|
||||
expect(reused.stdout).toContain('TREE: same-modulo-stamps receipt=');
|
||||
expect(reused.stdout).toContain('actions/runs/300');
|
||||
const failed = exec('1');
|
||||
expect(failed.status, failed.stderr).toBe(0);
|
||||
expect(failed.stdout).not.toContain('PASS (reused');
|
||||
expect(failed.stdout).toContain('❌ FAIL');
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user