mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-16 13:21:30 +01:00
* Keep Free Tests planning under the job cap when the duration seed is stale (GST-2193) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Refresh the free-suite duration seed from run 42691699570 (GST-2193) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Admit the eight test files the fresh seed records over 60 seconds as annotated exceptions (GST-2193) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
221 lines
15 KiB
TypeScript
221 lines
15 KiB
TypeScript
/**
|
|
* Static pins and a functional check for the weekly duration-seed recorder
|
|
* (.github/workflows/free-durations-record.yml): Linux on the free-suite
|
|
* runner, Windows through the record_durations path, and one bot seed PR only
|
|
* when a seed's file set drifts.
|
|
*/
|
|
import { afterEach, expect, test } from 'bun:test';
|
|
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
import os from 'node:os';
|
|
import path from 'node:path';
|
|
import { spawnSync } from 'node:child_process';
|
|
|
|
const root = path.resolve(import.meta.dir, '..');
|
|
const load = (file: string) => Bun.YAML.parse(readFileSync(path.join(root, '.github/workflows', file), 'utf8')) as any;
|
|
const recorder = load('free-durations-record.yml');
|
|
const freeTests = load('free-tests.yml');
|
|
const proposeStep = recorder.jobs.propose.steps.find((step: any) => step.name === 'Open or refresh the seed PR when either seed\'s file set changed');
|
|
const dirs: string[] = [];
|
|
afterEach(() => { for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); });
|
|
|
|
test('the recorder runs weekly before test-health and on dispatch, read-only except the propose job', () => {
|
|
expect(recorder.on.schedule).toEqual([{ cron: '0 4 * * 1' }]);
|
|
expect(Object.keys(recorder.on).sort()).toEqual(['pull_request', 'schedule', 'workflow_dispatch']);
|
|
for (const job of ['linux', 'windows']) expect(recorder.jobs[job].if).toBe("github.event_name != 'pull_request'");
|
|
expect(Object.keys(recorder.on.workflow_dispatch.inputs)).toEqual(['open_pr']);
|
|
expect(load('test-health.yml').on.schedule[0].cron).toBe('0 10 * * 1');
|
|
expect(recorder.permissions).toEqual({ contents: 'read' });
|
|
expect(recorder.concurrency['cancel-in-progress']).toBe(false);
|
|
expect(recorder.jobs.linux.permissions).toBeUndefined();
|
|
expect(recorder.jobs.propose.permissions).toEqual({ contents: 'write', 'pull-requests': 'write', actions: 'write' });
|
|
});
|
|
|
|
test('Linux records on the free-suite runner image, turns red on any red file, and uploads only a complete recording', () => {
|
|
const linux = recorder.jobs.linux;
|
|
expect(linux['runs-on']).toMatch(/^ubicloud-standard-\d+$/);
|
|
expect(freeTests.env.FREE_SUITE_RUNNER).toMatch(/^ubicloud-standard-\d+$/);
|
|
const runs = linux.steps.map((step: any) => step.run).filter(Boolean).join('\n');
|
|
expect(runs).toContain('bash scripts/ubicloud/setup-free-suite.sh');
|
|
const record = linux.steps.find((step: any) => step.name === 'Time every free file alone');
|
|
expect(record.run).toBe('set -o pipefail\nxvfb-run -a bun run test:free --record-durations 2>&1 | tee "$RUNNER_TEMP/record.log"\n');
|
|
expect(record['continue-on-error']).toBeUndefined();
|
|
const keep = linux.steps.find((step: any) => step.id === 'seed');
|
|
expect(keep.if).toBe("always() && steps.record.outcome != 'skipped'");
|
|
expect(keep.run).toContain('grep -q \'wrote [0-9]* durations to\' "$RUNNER_TEMP/record.log" || {');
|
|
expect(linux.steps.at(-1).if).toBe("always() && steps.seed.outcome == 'success'");
|
|
expect(linux.steps.at(-1).with).toMatchObject({ name: 'free-test-durations', path: '${{ runner.temp }}/seed/' });
|
|
});
|
|
|
|
test('Windows records through the windows-free-tests record_durations path, and propose needs both complete recordings', () => {
|
|
expect(recorder.jobs.windows).toEqual({ if: "github.event_name != 'pull_request'", uses: './.github/workflows/windows-free-tests.yml', permissions: { contents: 'read', actions: 'read' }, with: { record_durations: true } });
|
|
expect(recorder.jobs.propose.needs).toEqual(['linux', 'windows']);
|
|
expect(recorder.jobs.propose.if).toBe("${{ !cancelled() && github.event_name != 'pull_request' }}");
|
|
const downloads = recorder.jobs.propose.steps.filter((step: any) => step.uses?.startsWith('actions/download-artifact@'));
|
|
expect(downloads.map((step: any) => step.with.name)).toEqual(['free-test-durations', 'free-test-durations-windows']);
|
|
expect(downloads.every((step: any) => step['continue-on-error'] === undefined)).toBe(true);
|
|
});
|
|
|
|
test('both recorders list exactly the files the recorder reported red, and more than 5 on either OS proposes nothing', () => {
|
|
const windowsRecord = load('windows-free-tests.yml').jobs['windows-record-durations'].steps.find((step: any) => step.id === 'seed').run as string;
|
|
const linuxRecord = recorder.jobs.linux.steps.find((step: any) => step.id === 'seed').run as string;
|
|
const extract = (run: string) => run.split('\n').find(line => line.startsWith("sed -n '/failed while recording:/"))!;
|
|
expect(extract(linuxRecord).replace('seed/record-failures.txt', 'record-failures.txt')).toBe(extract(windowsRecord));
|
|
const dir = mkdtempSync(path.join(os.tmpdir(), 'seed-failures-'));
|
|
dirs.push(dir);
|
|
const list = (log: string) => {
|
|
writeFileSync(path.join(dir, 'record.log'), log);
|
|
const result = spawnSync('bash', ['-c', `set -euo pipefail\n${extract(windowsRecord)}\ncat "$RUNNER_TEMP/record-failures.txt"`], { encoding: 'utf8', timeout: 10_000, env: { ...process.env, RUNNER_TEMP: dir } });
|
|
expect(result.status, result.stderr).toBe(0);
|
|
return result.stdout;
|
|
};
|
|
const red = ' ✗ test/a.test.ts — suite > case\r\n[test:free] wrote 3 durations to scripts\\free-test-durations.json\r\n[test:free] WARNING: 2 file(s) failed while recording:\r\n ✗ test/a.test.ts\r\n ✗ browse/test/b.test.ts\r\n';
|
|
expect(list(red)).toBe('test/a.test.ts\nbrowse/test/b.test.ts\n');
|
|
expect(list(' ✗ test/a.test.ts — suite > flaky then fixed\n[test:free] wrote 3 durations to scripts/free-test-durations.json\n')).toBe('');
|
|
const script: string = proposeStep.run;
|
|
expect(script).toContain('if [ "$linux_failed" -gt 5 ] || [ "$windows_failed" -gt 5 ]; then');
|
|
expect(script.indexOf('-gt 5 ]')).toBeLessThan(script.indexOf('drift() {'));
|
|
expect(script).toContain('Files that failed while recording (durations kept; this PR\'s own free checks decide): $failed_line');
|
|
});
|
|
|
|
test('the propose step touches only the two seeds, never the allowlist, on one bot branch', () => {
|
|
const script: string = proposeStep.run;
|
|
expect(script).toContain('git add scripts/free-test-durations.json scripts/free-test-durations-windows.json\n');
|
|
expect(script).not.toContain('free-test-seed-allowlist.json');
|
|
expect(script).toContain('branch=bot/free-duration-seeds');
|
|
expect(script).toContain('bun test test/free-seed-ratchet.test.ts');
|
|
for (const workflow of ['free-tests.yml', 'windows-smoke.yml', 'quality-gate.yml']) {
|
|
expect(script).toContain(`gh workflow run ${workflow} --repo "$GITHUB_REPOSITORY" --ref "$branch"`);
|
|
expect(load(workflow).on).toHaveProperty('workflow_dispatch');
|
|
}
|
|
expect(script).toContain('those are this PR\'s checks');
|
|
expect(proposeStep.env.OPEN_PR).toBe("${{ github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || inputs.open_pr) }}");
|
|
expect(script.indexOf('if [ "$OPEN_PR" != true ]')).toBeLessThan(script.indexOf('git push'));
|
|
expect(recorder.on.workflow_dispatch.inputs.open_pr).toMatchObject({ type: 'boolean', default: true });
|
|
});
|
|
|
|
test('a long ratchet report reaches the PR body instead of aborting the step under pipefail', () => {
|
|
const script: string = proposeStep.run;
|
|
const line = script.split('\n').find(entry => entry.trimStart().startsWith('ratchet_line="FAILS'))!.trim();
|
|
const ratchet = ['error: Free seed ratchet:', ...Array.from({ length: 3000 }, (_, i) => `- test/f${i}.test.ts is seeded at 61s, over the 60s ceiling. Fix: split it, then rerun the recording.`)].join('\n');
|
|
const dir = mkdtempSync(path.join(os.tmpdir(), 'seed-ratchet-'));
|
|
dirs.push(dir);
|
|
writeFileSync(path.join(dir, 'ratchet.txt'), ratchet);
|
|
const result = spawnSync('bash', ['-c', `set -euo pipefail\nratchet=$(cat "$1")\n${line}\nprintf '%s' "$ratchet_line"`, 'ratchet', path.join(dir, 'ratchet.txt')], { encoding: 'utf8', timeout: 10_000 });
|
|
expect(result.status, result.stderr).toBe(0);
|
|
expect(result.stdout).toContain('- test/f18.test.ts is seeded at 61s, over');
|
|
expect(result.stdout).not.toContain('- test/f19.test.ts');
|
|
});
|
|
|
|
test('a refresh keeps a maintainer\'s commits on the open seed PR and leases the bot branch it replaces', () => {
|
|
const script: string = proposeStep.run;
|
|
const publish = script.slice(script.indexOf('branch=bot/free-duration-seeds'), script.indexOf('cat > pr-body.md'));
|
|
const dir = mkdtempSync(path.join(os.tmpdir(), 'seed-publish-'));
|
|
dirs.push(dir);
|
|
const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith('GIT_')));
|
|
const git = (cwd: string, ...args: string[]) => {
|
|
const result = spawnSync('git', args, { cwd, encoding: 'utf8', timeout: 10_000, env });
|
|
expect(result.status, result.stderr).toBe(0);
|
|
return result.stdout.trim();
|
|
};
|
|
const origin = path.join(dir, 'origin.git');
|
|
const work = path.join(dir, 'work');
|
|
git(dir, 'init', '-q', '--bare', '-b', 'main', origin);
|
|
git(dir, 'clone', '-q', origin, work);
|
|
const commit = (email: string, file: string, message: string) => {
|
|
git(work, 'config', 'user.name', email);
|
|
git(work, 'config', 'user.email', email);
|
|
writeFileSync(path.join(work, file), message);
|
|
git(work, 'add', file);
|
|
git(work, 'commit', '-q', '-m', message);
|
|
};
|
|
for (const file of ['free-test-durations.json', 'free-test-durations-windows.json']) {
|
|
writeFileSync(path.join(work, file), '{}');
|
|
}
|
|
git(work, 'checkout', '-q', '-b', 'main');
|
|
commit('dev@example.com', 'README', 'main');
|
|
git(work, 'push', '-q', 'origin', 'main');
|
|
const subject = 'test(seed): refresh the free-suite duration seeds from the weekly recording';
|
|
git(work, 'checkout', '-q', '-b', 'bot/free-duration-seeds');
|
|
commit('bot@example.com', 'seed.txt', subject);
|
|
git(work, 'push', '-q', 'origin', 'bot/free-duration-seeds');
|
|
const stub = path.join(dir, 'bin');
|
|
spawnSync('mkdir', [stub], { timeout: 10_000 });
|
|
writeFileSync(path.join(stub, 'gh'), '#!/bin/sh\necho 7\n', { mode: 0o755 });
|
|
const run = () => {
|
|
git(work, 'checkout', '-q', 'main');
|
|
spawnSync('mkdir', ['-p', path.join(work, 'scripts')], { timeout: 10_000 });
|
|
writeFileSync(path.join(work, 'scripts/free-test-durations.json'), `{"run":${Date.now()}}`);
|
|
writeFileSync(path.join(work, 'scripts/free-test-durations-windows.json'), '{}');
|
|
const result = spawnSync('bash', ['-c', `set -euo pipefail\n${publish}`], { cwd: work, encoding: 'utf8', timeout: 20_000,
|
|
env: { ...env, PATH: `${stub}:${process.env.PATH}`, GITHUB_STEP_SUMMARY: path.join(dir, 'summary.md') } });
|
|
expect(result.status, result.stderr).toBe(0);
|
|
return git(origin, 'rev-parse', 'refs/heads/bot/free-duration-seeds');
|
|
};
|
|
const replaced = run();
|
|
expect(git(origin, 'log', '--format=%s', `main..${replaced}`)).toBe(subject);
|
|
expect(git(origin, 'show', `${replaced}:scripts/free-test-durations.json`)).toContain('"run"');
|
|
git(work, 'checkout', '-q', '-B', 'bot/free-duration-seeds', replaced);
|
|
commit('maintainer@example.com', 'allowlist.json', 'maintainer repair');
|
|
git(work, 'push', '-q', 'origin', 'HEAD:refs/heads/bot/free-duration-seeds');
|
|
const repaired = git(origin, 'rev-parse', 'refs/heads/bot/free-duration-seeds');
|
|
expect(run()).toBe(repaired);
|
|
expect(readFileSync(path.join(dir, 'summary.md'), 'utf8')).toContain("Seed PR #7 has commits besides the bot's one refresh commit");
|
|
});
|
|
|
|
test('the drift check counts added and removed files, so an unchanged file set proposes nothing', () => {
|
|
const script: string = proposeStep.run;
|
|
const drift = script.slice(script.indexOf('drift() {'), script.indexOf('\n}', script.indexOf('drift() {')) + 2);
|
|
const dir = mkdtempSync(path.join(os.tmpdir(), 'seed-drift-'));
|
|
dirs.push(dir);
|
|
const seed = (name: string, files: Record<string, number>) => {
|
|
const file = path.join(dir, name);
|
|
writeFileSync(file, JSON.stringify({ version: 1, recordedAt: '2026-10-17T00:00:00.000Z', durations: files }));
|
|
return file;
|
|
};
|
|
const committed = seed('committed.json', { 'test/a.test.ts': 100, 'test/b.test.ts': 200 });
|
|
const run = (recorded: string) => spawnSync('bash', ['-c', `${drift}\ndrift "$1" "$2"`, 'drift', committed, recorded], { encoding: 'utf8', timeout: 10_000 });
|
|
const same = run(seed('same.json', { 'test/a.test.ts': 900, 'test/b.test.ts': 50 }));
|
|
expect(same.status, same.stderr).toBe(0);
|
|
expect(same.stdout.trim()).toBe('"0 0"');
|
|
const drifted = run(seed('drifted.json', { 'test/a.test.ts': 100, 'test/c.test.ts': 1, 'test/d.test.ts': 2 }));
|
|
expect(drifted.status, drifted.stderr).toBe(0);
|
|
expect(drifted.stdout.trim()).toBe('"2 1"');
|
|
});
|
|
|
|
test('a seed PR closed unmerged raises one blocker issue and a red run; a merged seed PR clears it (GST-2193)', () => {
|
|
expect(recorder.on.pull_request).toEqual({ types: ['closed'], branches: ['main'],
|
|
paths: ['scripts/free-test-durations.json', 'scripts/free-test-durations-windows.json'] });
|
|
expect(recorder.concurrency.group).toContain("format('free-durations-seed-pr-{0}', github.event.pull_request.number)");
|
|
const job = recorder.jobs['seed-pr-closed'];
|
|
expect(job.if).toBe("github.event_name == 'pull_request' && github.event.pull_request.head.ref == 'bot/free-duration-seeds' && github.event.pull_request.head.repo.full_name == github.repository");
|
|
expect(job.permissions).toEqual({ issues: 'write' });
|
|
expect(job.steps.some((step: any) => step.uses?.startsWith('actions/checkout@'))).toBe(false);
|
|
const step = job.steps[0];
|
|
const dir = mkdtempSync(path.join(os.tmpdir(), 'seed-blocker-'));
|
|
dirs.push(dir);
|
|
const run = (merged: boolean, existing: string) => {
|
|
const log = path.join(dir, `gh-${merged}-${existing || 'none'}.log`);
|
|
writeFileSync(path.join(dir, 'gh'), `#!/bin/sh\necho "$*" >> "${log}"\nif [ "$1 $2" = "issue list" ]; then echo "${existing}"; fi\n`, { mode: 0o755 });
|
|
const result = spawnSync('bash', ['-c', step.run], { encoding: 'utf8', timeout: 10_000,
|
|
env: { ...process.env, PATH: `${dir}:${process.env.PATH}`, RUNNER_TEMP: dir, GITHUB_REPOSITORY: 'garrytan/gstack', MERGED: String(merged), PR_NUMBER: '3835' } });
|
|
const calls = (() => { try { return readFileSync(log, 'utf8').trim().split('\n'); } catch { return []; } })();
|
|
return { status: result.status, stdout: result.stdout, calls };
|
|
};
|
|
const raised = run(false, '');
|
|
expect(raised.status).toBe(1);
|
|
expect(raised.stdout).toContain('::error::Seed PR #3835 closed unmerged');
|
|
expect(raised.calls[1]).toBe(`issue create --repo garrytan/gstack --title Blocker: free-suite duration seed PR closed unmerged --body-file ${dir}/blocker.md`);
|
|
const body = readFileSync(path.join(dir, 'blocker.md'), 'utf8');
|
|
expect(body).toContain('Seed PR #3835 was closed without merging');
|
|
expect(body).toContain('`gh workflow run free-durations-record.yml --ref main`');
|
|
const again = run(false, '41');
|
|
expect(again.status).toBe(1);
|
|
expect(again.calls[1]).toBe(`issue comment 41 --repo garrytan/gstack --body-file ${dir}/blocker.md`);
|
|
const cleared = run(true, '41');
|
|
expect(cleared.status).toBe(0);
|
|
expect(cleared.calls[1]).toBe('issue close 41 --repo garrytan/gstack --comment Seed PR #3835 merged, so the seed is current again.');
|
|
const quiet = run(true, '');
|
|
expect(quiet.status).toBe(0);
|
|
expect(quiet.calls).toHaveLength(1);
|
|
});
|