gstack/test/free-durations-record-workflow.test.ts

/**
 * Static pins and a functional check for the weekly duration-seed recorder
 * (.github/workflows/free-durations-record.yml): Linux on the free-suite
 * runner, Windows through the record_durations path, and one bot seed PR only
 * when a seed's file set drifts.
 */
import { afterEach, expect, test } from 'bun:test';
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import { spawnSync } from 'node:child_process';

const root = path.resolve(import.meta.dir, '..');
const load = (file: string) => Bun.YAML.parse(readFileSync(path.join(root, '.github/workflows', file), 'utf8')) as any;
const recorder = load('free-durations-record.yml');
const freeTests = load('free-tests.yml');
const proposeStep = recorder.jobs.propose.steps.find((step: any) => step.name === 'Open or refresh the seed PR when either seed\'s file set changed');
const dirs: string[] = [];
afterEach(() => { for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); });

test('the recorder runs weekly before test-health and on dispatch, read-only except the propose job', () => {
  expect(recorder.on.schedule).toEqual([{ cron: '0 4 * * 1' }]);
  expect(Object.keys(recorder.on).sort()).toEqual(['schedule', 'workflow_dispatch']);
  expect(Object.keys(recorder.on.workflow_dispatch.inputs)).toEqual(['open_pr']);
  expect(load('test-health.yml').on.schedule[0].cron).toBe('0 10 * * 1');
  expect(recorder.permissions).toEqual({ contents: 'read' });
  expect(recorder.concurrency['cancel-in-progress']).toBe(false);
  expect(recorder.jobs.linux.permissions).toBeUndefined();
  expect(recorder.jobs.propose.permissions).toEqual({ contents: 'write', 'pull-requests': 'write', actions: 'write' });
});

test('Linux records on the free-suite runner, turns red on any red file, and uploads only a complete recording', () => {
  const linux = recorder.jobs.linux;
  expect(linux['runs-on']).toBe(freeTests.jobs['free-suite']['runs-on']);
  const runs = linux.steps.map((step: any) => step.run).filter(Boolean).join('\n');
  expect(runs).toContain('bash scripts/ubicloud/setup-free-suite.sh');
  const record = linux.steps.find((step: any) => step.name === 'Time every free file alone');
  expect(record.run).toBe('set -o pipefail\nxvfb-run -a bun run test:free --record-durations 2>&1 | tee "$RUNNER_TEMP/record.log"\n');
  expect(record['continue-on-error']).toBeUndefined();
  const keep = linux.steps.find((step: any) => step.id === 'seed');
  expect(keep.if).toBe("always() && steps.record.outcome != 'skipped'");
  expect(keep.run).toContain('grep -q \'wrote [0-9]* durations to\' "$RUNNER_TEMP/record.log" || {');
  expect(linux.steps.at(-1).if).toBe("always() && steps.seed.outcome == 'success'");
  expect(linux.steps.at(-1).with).toMatchObject({ name: 'free-test-durations', path: '${{ runner.temp }}/seed/' });
});

test('Windows records through the windows-free-tests record_durations path, and propose needs both complete recordings', () => {
  expect(recorder.jobs.windows).toEqual({ uses: './.github/workflows/windows-free-tests.yml', with: { record_durations: true } });
  expect(recorder.jobs.propose.needs).toEqual(['linux', 'windows']);
  expect(recorder.jobs.propose.if).toBe('${{ !cancelled() }}');
  const downloads = recorder.jobs.propose.steps.filter((step: any) => step.uses?.startsWith('actions/download-artifact@'));
  expect(downloads.map((step: any) => step.with.name)).toEqual(['free-test-durations', 'free-test-durations-windows']);
  expect(downloads.every((step: any) => step['continue-on-error'] === undefined)).toBe(true);
});

test('both recorders list exactly the files the recorder reported red, and more than 5 on either OS proposes nothing', () => {
  const windowsRecord = load('windows-free-tests.yml').jobs['windows-record-durations'].steps.find((step: any) => step.id === 'seed').run as string;
  const linuxRecord = recorder.jobs.linux.steps.find((step: any) => step.id === 'seed').run as string;
  const extract = (run: string) => run.split('\n').find(line => line.startsWith("sed -n '/failed while recording:/"))!;
  expect(extract(linuxRecord).replace('seed/record-failures.txt', 'record-failures.txt')).toBe(extract(windowsRecord));
  const dir = mkdtempSync(path.join(os.tmpdir(), 'seed-failures-'));
  dirs.push(dir);
  const list = (log: string) => {
    writeFileSync(path.join(dir, 'record.log'), log);
    const result = spawnSync('bash', ['-c', `set -euo pipefail\n${extract(windowsRecord)}\ncat "$RUNNER_TEMP/record-failures.txt"`], { encoding: 'utf8', timeout: 10_000, env: { ...process.env, RUNNER_TEMP: dir } });
    expect(result.status, result.stderr).toBe(0);
    return result.stdout;
  };
  const red = '  ✗ test/a.test.ts — suite > case\r\n[test:free] wrote 3 durations to scripts\\free-test-durations.json\r\n[test:free] WARNING: 2 file(s) failed while recording:\r\n  ✗ test/a.test.ts\r\n  ✗ browse/test/b.test.ts\r\n';
  expect(list(red)).toBe('test/a.test.ts\nbrowse/test/b.test.ts\n');
  expect(list('  ✗ test/a.test.ts — suite > flaky then fixed\n[test:free] wrote 3 durations to scripts/free-test-durations.json\n')).toBe('');
  const script: string = proposeStep.run;
  expect(script).toContain('if [ "$linux_failed" -gt 5 ] || [ "$windows_failed" -gt 5 ]; then');
  expect(script.indexOf('-gt 5 ]')).toBeLessThan(script.indexOf('drift() {'));
  expect(script).toContain('Files that failed while recording (durations kept; this PR\'s own free checks decide): $failed_line');
});

test('the propose step touches only the two seeds, never the allowlist, on one bot branch', () => {
  const script: string = proposeStep.run;
  expect(script).toContain('git add scripts/free-test-durations.json scripts/free-test-durations-windows.json\n');
  expect(script).not.toContain('free-test-seed-allowlist.json');
  expect(script).toContain('branch=bot/free-duration-seeds');
  expect(script).toContain('bun test test/free-seed-ratchet.test.ts');
  for (const workflow of ['free-tests.yml', 'windows-free-tests.yml', 'quality-gate.yml']) {
    expect(script).toContain(`gh workflow run ${workflow} --repo "$GITHUB_REPOSITORY" --ref "$branch"`);
    expect(load(workflow).on).toHaveProperty('workflow_dispatch');
  }
  expect(script).toContain('those are this PR\'s checks');
  expect(proposeStep.env.OPEN_PR).toBe("${{ github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || inputs.open_pr) }}");
  expect(script.indexOf('if [ "$OPEN_PR" != true ]')).toBeLessThan(script.indexOf('git push'));
  expect(recorder.on.workflow_dispatch.inputs.open_pr).toMatchObject({ type: 'boolean', default: true });
});

test('a long ratchet report reaches the PR body instead of aborting the step under pipefail', () => {
  const script: string = proposeStep.run;
  const line = script.split('\n').find(entry => entry.trimStart().startsWith('ratchet_line="FAILS'))!.trim();
  const ratchet = ['error: Free seed ratchet:', ...Array.from({ length: 3000 }, (_, i) => `- test/f${i}.test.ts is seeded at 61s, over the 60s ceiling. Fix: split it, then rerun the recording.`)].join('\n');
  const dir = mkdtempSync(path.join(os.tmpdir(), 'seed-ratchet-'));
  dirs.push(dir);
  writeFileSync(path.join(dir, 'ratchet.txt'), ratchet);
  const result = spawnSync('bash', ['-c', `set -euo pipefail\nratchet=$(cat "$1")\n${line}\nprintf '%s' "$ratchet_line"`, 'ratchet', path.join(dir, 'ratchet.txt')], { encoding: 'utf8', timeout: 10_000 });
  expect(result.status, result.stderr).toBe(0);
  expect(result.stdout).toContain('- test/f18.test.ts is seeded at 61s, over');
  expect(result.stdout).not.toContain('- test/f19.test.ts');
});

test('a refresh keeps a maintainer\'s commits on the open seed PR and leases the bot branch it replaces', () => {
  const script: string = proposeStep.run;
  const publish = script.slice(script.indexOf('branch=bot/free-duration-seeds'), script.indexOf('cat > pr-body.md'));
  const dir = mkdtempSync(path.join(os.tmpdir(), 'seed-publish-'));
  dirs.push(dir);
  const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith('GIT_')));
  const git = (cwd: string, ...args: string[]) => {
    const result = spawnSync('git', args, { cwd, encoding: 'utf8', timeout: 10_000, env });
    expect(result.status, result.stderr).toBe(0);
    return result.stdout.trim();
  };
  const origin = path.join(dir, 'origin.git');
  const work = path.join(dir, 'work');
  git(dir, 'init', '-q', '--bare', '-b', 'main', origin);
  git(dir, 'clone', '-q', origin, work);
  const commit = (email: string, file: string, message: string) => {
    git(work, 'config', 'user.name', email);
    git(work, 'config', 'user.email', email);
    writeFileSync(path.join(work, file), message);
    git(work, 'add', file);
    git(work, 'commit', '-q', '-m', message);
  };
  for (const file of ['free-test-durations.json', 'free-test-durations-windows.json']) {
    writeFileSync(path.join(work, file), '{}');
  }
  git(work, 'checkout', '-q', '-b', 'main');
  commit('dev@example.com', 'README', 'main');
  git(work, 'push', '-q', 'origin', 'main');
  const subject = 'test(seed): refresh the free-suite duration seeds from the weekly recording';
  git(work, 'checkout', '-q', '-b', 'bot/free-duration-seeds');
  commit('bot@example.com', 'seed.txt', subject);
  git(work, 'push', '-q', 'origin', 'bot/free-duration-seeds');
  const stub = path.join(dir, 'bin');
  spawnSync('mkdir', [stub], { timeout: 10_000 });
  writeFileSync(path.join(stub, 'gh'), '#!/bin/sh\necho 7\n', { mode: 0o755 });
  const run = () => {
    git(work, 'checkout', '-q', 'main');
    spawnSync('mkdir', ['-p', path.join(work, 'scripts')], { timeout: 10_000 });
    writeFileSync(path.join(work, 'scripts/free-test-durations.json'), `{"run":${Date.now()}}`);
    writeFileSync(path.join(work, 'scripts/free-test-durations-windows.json'), '{}');
    const result = spawnSync('bash', ['-c', `set -euo pipefail\n${publish}`], { cwd: work, encoding: 'utf8', timeout: 20_000,
      env: { ...env, PATH: `${stub}:${process.env.PATH}`, GITHUB_STEP_SUMMARY: path.join(dir, 'summary.md') } });
    expect(result.status, result.stderr).toBe(0);
    return git(origin, 'rev-parse', 'refs/heads/bot/free-duration-seeds');
  };
  const replaced = run();
  expect(git(origin, 'log', '--format=%s', `main..${replaced}`)).toBe(subject);
  expect(git(origin, 'show', `${replaced}:scripts/free-test-durations.json`)).toContain('"run"');
  git(work, 'checkout', '-q', '-B', 'bot/free-duration-seeds', replaced);
  commit('maintainer@example.com', 'allowlist.json', 'maintainer repair');
  git(work, 'push', '-q', 'origin', 'HEAD:refs/heads/bot/free-duration-seeds');
  const repaired = git(origin, 'rev-parse', 'refs/heads/bot/free-duration-seeds');
  expect(run()).toBe(repaired);
  expect(readFileSync(path.join(dir, 'summary.md'), 'utf8')).toContain("Seed PR #7 has commits besides the bot's one refresh commit");
});

test('the drift check counts added and removed files, so an unchanged file set proposes nothing', () => {
  const script: string = proposeStep.run;
  const drift = script.slice(script.indexOf('drift() {'), script.indexOf('\n}', script.indexOf('drift() {')) + 2);
  const dir = mkdtempSync(path.join(os.tmpdir(), 'seed-drift-'));
  dirs.push(dir);
  const seed = (name: string, files: Record<string, number>) => {
    const file = path.join(dir, name);
    writeFileSync(file, JSON.stringify({ version: 1, recordedAt: '2026-10-17T00:00:00.000Z', durations: files }));
    return file;
  };
  const committed = seed('committed.json', { 'test/a.test.ts': 100, 'test/b.test.ts': 200 });
  const run = (recorded: string) => spawnSync('bash', ['-c', `${drift}\ndrift "$1" "$2"`, 'drift', committed, recorded], { encoding: 'utf8', timeout: 10_000 });
  const same = run(seed('same.json', { 'test/a.test.ts': 900, 'test/b.test.ts': 50 }));
  expect(same.status, same.stderr).toBe(0);
  expect(same.stdout.trim()).toBe('"0 0"');
  const drifted = run(seed('drifted.json', { 'test/a.test.ts': 100, 'test/c.test.ts': 1, 'test/d.test.ts': 2 }));
  expect(drifted.status, drifted.stderr).toBe(0);
  expect(drifted.stdout.trim()).toBe('"2 1"');
});