mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-15 04:47:23 +01:00
Let the CI carry accept a run where only an unrelated job was rerun (GST-2284) (#4274)
* Let the CI carry accept a run where only an unrelated job was rerun (GST-2284) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Check artifact ids only when a run holds several same-named artifacts (GST-2284) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Date a carried CI run by its own test jobs so an unrelated rerun cannot refresh old results (GST-2284) Freshness now reads the newest completed_at among the profile's jobs, not every job in the run, so rerunning a job outside the profile no longer makes days-old test results look fresh. The no-dependencies hint now matches only a missing package or module error. Review-Finding: p1/red-team#1 reader=red-team Review-Finding: p1/specialist-batched#1 reader=testing Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Say in the changelog and pregate docs that a carried run is dated by its own test jobs (GST-2284) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Fix the test type error in the new freshness test (GST-2284) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
Garry Tanandcapy-ai[bot] authored and GitHub committed
1 parent
b1702eea91
commit
56c5aa62ea
6 files changed
+183
-24
No files matched your search
@@ -138,6 +138,9 @@ describe.skipIf(process.platform === 'win32')('clear --run with the Free Tests p
|
||||
expect(clear({ head, files: files.filter(f => f !== 'test/other.test.js') })).toContain("reason=inventory_uncovered — 1 of the");
|
||||
expect(clear({ head, files: files.filter(f => f !== 'test/other.test.js') }, { gateRuns: { profiles: [PROFILE], inventory: INVENTORY, skipped: `${SKIPPED}; echo test/other.test.js` } })).toContain('(test/other.test.js)');
|
||||
expect(clear({ head, files }, { gateRuns: { profiles: [PROFILE], inventory: INVENTORY, skipped: 'exit 3' } })).toContain('reason=skipped_failed');
|
||||
expect(clear({ head, files }, { gateRuns: { profiles: [PROFILE], inventory: INVENTORY, skipped: 'exit 3' } })).not.toContain('no installed dependencies');
|
||||
expect(clear({ head, files }, { gateRuns: { profiles: [PROFILE], inventory: INVENTORY, skipped: "echo 'TypeError: x at /repo/node_modules/zod/index.js:1' >&2; exit 1" } })).not.toContain('no installed dependencies');
|
||||
expect(clear({ head, files }, { gateRuns: { profiles: [PROFILE], inventory: INVENTORY, skipped: "echo \"error: Cannot find package 'zod' from '/x/test/helpers/paid-test-set.ts'\" >&2; exit 1" } })).toContain('reason=skipped_failed — the remote gate\'s skipped command failed because this checkout has no installed dependencies (install them, for example with bun install, then clear again): ');
|
||||
const paidOnly = structuredClone(file);
|
||||
paidOnly.requires_remote!.find(r => r.lane === LANE)!.tests = [PAID];
|
||||
expect(msg(clearDecision(paidOnly, LANE, { runs: [RUN], evidence: gateCheck(repo, { head, files }) }, repo.work))).toBe(`PREGATE_RECEIPT_INVALID: REUSE: refused reason=paid_only — the remote gate skips every selected test file (1), so no selected test ran; like the gate's own tests=0 report, that never clears; ${LANE} stays uncleared`);
|
||||
|
||||
+111
-2
@@ -14,7 +14,7 @@ const TREE = 'b'.repeat(40);
|
||||
const NOW = new Date('2026-10-29T12:00:00Z');
|
||||
const PROFILE: RunProfile = { workflow: 'test.yml', events: ['workflow_dispatch', 'push'], jobs: ['full-suite'], artifact: 'full-suite-tree' };
|
||||
|
||||
interface World { run: ApiRun; others: ApiRun[]; jobs: ApiJob[]; artifact: TreeRecord | Error; trees: Record<string, string | null>; throwOn?: 'run'; later?: ApiRun }
|
||||
interface World { run: ApiRun; others: ApiRun[]; jobs: ApiJob[]; jobsByAttempt?: Record<number, ApiJob[]>; artifact: TreeRecord | Error; trees: Record<string, string | null>; throwOn?: 'run'; later?: ApiRun }
|
||||
function world(over: Partial<World> = {}): World {
|
||||
const run: ApiRun = {
|
||||
id: 100, run_attempt: 1, status: 'completed', conclusion: 'success', event: 'workflow_dispatch', head_sha: HEAD,
|
||||
@@ -32,7 +32,7 @@ function deps(w: World): RunDeps {
|
||||
return {
|
||||
run: id => { if (w.throwOn === 'run') throw new Error('HTTP 502'); if (id !== w.run.id) throw new Error('HTTP 404'); return reads++ && w.later ? w.later : w.run; },
|
||||
runsForHead: (_wf, sha) => [w.run, ...w.others].filter(r => r.head_sha === sha),
|
||||
jobs: () => w.jobs,
|
||||
jobs: (_id, attempt) => w.jobsByAttempt?.[attempt] ?? w.jobs,
|
||||
artifact: () => { if (w.artifact instanceof Error) throw w.artifact; return w.artifact; },
|
||||
commitTree: sha => w.trees[sha] ?? null,
|
||||
};
|
||||
@@ -320,3 +320,112 @@ describe.skipIf(process.platform === 'win32')('ghRunDeps jobs read through a pro
|
||||
expect(missing.calls()).toEqual([page(50)]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('job-level reruns (GST-2284, HARN-56 run 42724981450)', () => {
|
||||
// Shapes from harness-evals run 42724981450: GitHub relists a job it did not rerun under each later attempt with its old
|
||||
// start and finish times, and a rerun of the uploading job leaves one checks-tree artifact per attempt.
|
||||
const MERGE = 'e'.repeat(40);
|
||||
const LATER = new Date('2026-11-10T02:00:00Z');
|
||||
const CHECKS: RunProfile = { workflow: 'test.yml', events: ['pull_request', 'push'], jobs: ['checks'], artifact: 'checks-tree', os: 'Linux' };
|
||||
const checks1: ApiJob = { name: 'checks', conclusion: 'success', started_at: '2026-11-10T00:02:31Z', completed_at: '2026-11-10T00:19:51Z' };
|
||||
const checks3: ApiJob = { name: 'checks', conclusion: 'success', started_at: '2026-11-10T01:27:17Z', completed_at: '2026-11-10T01:46:47Z' };
|
||||
const export1: ApiJob = { name: 'public-export', conclusion: 'failure', started_at: '2026-11-10T00:03:28Z', completed_at: '2026-11-10T00:40:49Z' };
|
||||
const export2: ApiJob = { name: 'public-export', conclusion: 'success', started_at: '2026-11-10T00:45:04Z', completed_at: '2026-11-10T01:20:31Z' };
|
||||
const record = (attempt: number, over: Partial<TreeRecord> = {}): TreeRecord => ({ run_id: 100, run_attempt: attempt, event: 'pull_request', sha: MERGE, head_sha: HEAD, tree: TREE, os: 'Linux', files: ['test/a.test.ts'], tests: 4, ...over });
|
||||
const rerun = (attempt: number, jobsByAttempt: Record<number, ApiJob[]>, artifact: TreeRecord, trees: Record<string, string | null> = { [HEAD]: TREE, [MERGE]: TREE }) => {
|
||||
const base = world();
|
||||
return world({ run: { ...base.run, event: 'pull_request', run_attempt: attempt, created_at: '2026-11-10T00:00:48Z', run_started_at: '2026-11-10T00:42:13Z', updated_at: '2026-11-10T01:20:31Z' }, jobs: jobsByAttempt[attempt]!, jobsByAttempt, artifact, trees });
|
||||
};
|
||||
const verify = (w: World, d: RunDeps = deps(w)) => verifyRun(d, { repo: REPO, runRef: '100', profileName: 'ci/checks', profile: CHECKS, now: LATER });
|
||||
const why = (w: World) => { const v = verify(w); return v.ok ? 'ok' : v.reason; };
|
||||
const onlyExportRerun = { 1: [checks1, export1], 2: [export2, checks1] };
|
||||
|
||||
// Value: protects=Case A, a rerun of an unrelated job keeps the carry; fails_when=the artifact must record the run's newest attempt
|
||||
test('a checks job carried unchanged from attempt 1 verifies its attempt-1 artifact in attempt 2', () => {
|
||||
const v = verify(rerun(2, onlyExportRerun, record(1)));
|
||||
expect(v.ok ? [v.evidence.run_attempt, v.evidence.tree] : v.detail).toEqual([2, TREE]);
|
||||
});
|
||||
|
||||
// Value: protects=freshness dates a carried run by its profile jobs; fails_when=rerunning an unrelated job refreshes old test results
|
||||
test('rerunning an unrelated job does not refresh a carried run whose checks job finished long ago', () => {
|
||||
const old = { ...checks1, started_at: '2026-11-02T00:02:31Z', completed_at: '2026-11-02T00:19:51Z' };
|
||||
expect(why(rerun(2, { 1: [old, export1], 2: [export2, old] }, record(1)))).toBe('run_stale');
|
||||
const v = verify(rerun(2, onlyExportRerun, record(1)));
|
||||
expect(v.ok && v.evidence.finished_at).toBe(checks1.completed_at!);
|
||||
});
|
||||
|
||||
// Value: protects=the carried attempt binds only the job's own attempt and every other field; fails_when=any recorded attempt or field is accepted
|
||||
test('a carried artifact still refuses a different tree, head, merge tree, event, run or job conclusion', () => {
|
||||
expect(why(rerun(2, onlyExportRerun, record(1, { tree: 'd'.repeat(40) })))).toBe('tree_mismatch');
|
||||
expect(why(rerun(2, onlyExportRerun, record(1, { head_sha: 'c'.repeat(40) })))).toBe('artifact_mismatch');
|
||||
expect(why(rerun(2, onlyExportRerun, record(1), { [HEAD]: TREE, [MERGE]: 'f'.repeat(40) }))).toBe('tree_mismatch');
|
||||
expect(why(rerun(2, onlyExportRerun, record(1, { event: 'push' })))).toBe('artifact_mismatch');
|
||||
expect(why(rerun(2, onlyExportRerun, record(1, { run_id: 99 })))).toBe('artifact_mismatch');
|
||||
expect(why(rerun(2, { 1: [checks1, export1], 2: [export2, { ...checks1, conclusion: 'failure' }] }, record(1)))).toBe('job_not_success');
|
||||
expect(why(rerun(2, onlyExportRerun, record(2)))).toBe('artifact_mismatch');
|
||||
});
|
||||
|
||||
// Value: protects=a rerun checks job never counts an older attempt's artifact; fails_when=any earlier attempt the run had is accepted
|
||||
test('a checks job rerun in a later attempt needs the artifact from that attempt', () => {
|
||||
const rerunChecks = { 1: [checks1, export1], 2: [export2, checks1], 3: [checks3, export2] };
|
||||
expect(why(rerun(3, rerunChecks, record(3)))).toBe('ok');
|
||||
expect(why(rerun(3, rerunChecks, record(1)))).toBe('artifact_mismatch');
|
||||
expect(why(rerun(3, rerunChecks, record(2)))).toBe('artifact_mismatch');
|
||||
expect(why(rerun(3, { 1: [checks1, export1], 2: [export2, checks1], 3: [checks1, export2] }, record(1)))).toBe('ok');
|
||||
expect(why(rerun(2, { 1: [checks1, export1], 2: [export2, { ...checks1, started_at: undefined }] }, record(1)))).toBe('artifact_mismatch');
|
||||
});
|
||||
|
||||
describe.skipIf(process.platform === 'win32')('ghRunDeps with one checks-tree artifact per attempt (Case B)', () => {
|
||||
const dirs: string[] = [];
|
||||
afterAll(() => { for (const d of dirs) fs.rmSync(d, { recursive: true, force: true }); });
|
||||
const storedZip = (name: string, body: string): Buffer => {
|
||||
const data = Buffer.from(body), file = Buffer.from(name), crc = Bun.hash.crc32(data) >>> 0;
|
||||
const local = Buffer.alloc(30); local.writeUInt32LE(0x04034b50, 0); local.writeUInt16LE(20, 4); local.writeUInt32LE(crc, 14); local.writeUInt32LE(data.length, 18); local.writeUInt32LE(data.length, 22); local.writeUInt16LE(file.length, 26);
|
||||
const central = Buffer.alloc(46); central.writeUInt32LE(0x02014b50, 0); central.writeUInt16LE(20, 4); central.writeUInt16LE(20, 6); central.writeUInt32LE(crc, 16); central.writeUInt32LE(data.length, 20); central.writeUInt32LE(data.length, 24); central.writeUInt16LE(file.length, 28);
|
||||
const end = Buffer.alloc(22); end.writeUInt32LE(0x06054b50, 0); end.writeUInt16LE(1, 8); end.writeUInt16LE(1, 10); end.writeUInt32LE(46 + file.length, 12); end.writeUInt32LE(30 + file.length + data.length, 16);
|
||||
return Buffer.concat([local, file, data, central, file, end]);
|
||||
};
|
||||
function stubGh(records: TreeRecord[]) {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gst2284-'));
|
||||
dirs.push(dir);
|
||||
records.forEach((r, i) => fs.writeFileSync(path.join(dir, `${i + 1}.zip`), storedZip('tree.json', JSON.stringify(r))));
|
||||
fs.writeFileSync(path.join(dir, 'gh'), [
|
||||
'#!/usr/bin/env bash',
|
||||
'url=""; for a in "$@"; do case "$a" in repos/*) url="$a";; esac; done',
|
||||
'case "$url" in',
|
||||
` */actions/runs/100/artifacts\\?name=checks-tree*) for i in ${records.map((_, i) => i + 1).join(' ')}; do printf '{"id":%d,"name":"checks-tree","size_in_bytes":200,"expired":false}\\n' $i; done;;`,
|
||||
` */actions/artifacts/*/zip) id="${'$'}{url%/zip}"; cat '${dir}'/"${'$'}{id##*/}".zip;;`,
|
||||
' *) echo "gh: unexpected $*" >&2; exit 1;;',
|
||||
'esac',
|
||||
].join('\n'), { mode: 0o755 });
|
||||
return dir;
|
||||
}
|
||||
function verifyThrough(dir: string, w: World) {
|
||||
const saved = { PATH: process.env.PATH, GSTACK_STATE_ROOT: process.env.GSTACK_STATE_ROOT };
|
||||
process.env.PATH = `${dir}:${saved.PATH}`;
|
||||
process.env.GSTACK_STATE_ROOT = path.join(dir, 'state');
|
||||
try { return verify(w, { ...deps(w), artifact: ghRunDeps(REPO, dir).artifact }); } finally {
|
||||
process.env.PATH = saved.PATH;
|
||||
if (saved.GSTACK_STATE_ROOT === undefined) delete process.env.GSTACK_STATE_ROOT; else process.env.GSTACK_STATE_ROOT = saved.GSTACK_STATE_ROOT;
|
||||
}
|
||||
}
|
||||
const rerunChecks = { 1: [checks1, export1], 2: [export2, checks1], 3: [checks3, export2] };
|
||||
|
||||
// Value: protects=Case B, rerunning the checks job to work around Case A keeps the carry; fails_when=two same-named artifacts refuse as artifact_missing
|
||||
test('two checks-tree artifacts: the one from the checks job\'s newest attempt is the evidence', () => {
|
||||
const v = verifyThrough(stubGh([record(1), record(3, { tests: 7 })]), rerun(3, rerunChecks, record(0)));
|
||||
expect(v.ok ? [v.evidence.run_attempt, v.evidence.tests] : v.detail).toEqual([3, 7]);
|
||||
});
|
||||
|
||||
// Value: protects=no fallback to an older artifact; fails_when=a picked artifact with another tree is passed over for an older matching one
|
||||
test('the newest attempt\'s artifact with a different tree refuses, even beside an older one with the right tree', () => {
|
||||
expect(verifyThrough(stubGh([record(1), record(3, { tree: 'd'.repeat(40) })]), rerun(3, rerunChecks, record(0)))).toMatchObject({ ok: false, reason: 'tree_mismatch' });
|
||||
});
|
||||
|
||||
// Value: protects=fail closed when no artifact belongs to the job's attempt; fails_when=the newest or any artifact is taken regardless
|
||||
test('no artifact from the checks job\'s own attempt refuses as artifact_missing', () => {
|
||||
const v = verifyThrough(stubGh([record(1), record(2)]), rerun(3, rerunChecks, record(0)));
|
||||
expect(v.ok ? 'ok' : [v.reason, v.detail.includes('0 of them record run_attempt 3')]).toEqual(['artifact_missing', true]);
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user