mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 14:06:50 +01:00
v1.91.128.0 ci(seed): weekly standing recorder for the Linux and Windows duration seeds (#3253)
* ci(seed): weekly standing recorder for the Linux and Windows duration seeds (W4) free-durations-record.yml runs Mondays 04:00 UTC (before test-health) and on dispatch. Linux records on ubicloud-standard-8, the free-suite shard runner, with scripts/ubicloud/setup-free-suite.sh; Windows records through windows-free-tests.yml's record_durations path (now also workflow_call). When either seed's file set drifts from main, the propose job force-updates bot/free-duration-seeds, opens or refreshes one seed PR stating the seed ratchet result and the files that failed while recording, and dispatches the free lanes on it. A red file keeps its duration; an interrupted recording uploads no seed (this also stops the Windows dispatch from uploading a stale copy), and more than 5 red files on either OS proposes nothing. The bot never edits the allowlist. open_pr=false on dispatch records and reports only. The workflow is classified free-lane only in scripts/test-pr-profile.ts, with a non-behavioral HARNESS_VERSION decision. Intentional pin change: test/windows-native-workflows.test.ts now expects workflow_call and the complete-recording guard on the Windows record job. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * ci(seed): keep maintainer commits on the open seed PR, publish only from main, survive long ratchet reports /review on #3253 (gpt-6.1-sol): a refresh force-pushed over maintainer repairs on the seed PR, a dispatch from a feature branch would publish that branch's commits, and grep | head under pipefail could abort the step on a long ratchet report. The refresh now leaves an open seed PR alone when it holds any commit besides the bot's one refresh commit and pushes with an exact lease; only main publishes; the report is cut with sed. Each has a functional test that fails on the old workflow. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(changelog): unreleased entry for the weekly duration-seed recorder Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore: stamp v1.91.128.0 Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
77d125e0bc
commit
c5c2f28994
11 files changed
+413
-18
No files matched your search
@@ -0,0 +1,172 @@
|
||||
name: Free-suite duration seeds
|
||||
|
||||
# Weekly standing recorder for the two free-suite duration seeds that the
|
||||
# shard packers read: scripts/free-test-durations.json (Linux, recorded on the
|
||||
# same ubicloud-standard-8 runner as the free-suite shards) and
|
||||
# scripts/free-test-durations-windows.json (windows-latest, through the
|
||||
# record_durations path of windows-free-tests.yml). When either recording adds
|
||||
# or drops a test file, it force-updates one bot branch and opens or refreshes
|
||||
# a seed PR. A file that fails while recording turns its job red but keeps its
|
||||
# duration (a red file is still a real cost) and is named in the PR body; an
|
||||
# interrupted recording, or more than 5 red files on either OS, proposes
|
||||
# nothing (docs/TESTING_INTERNALS.md#free-suite-duration-seed).
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 4 * * 1' # Monday 04:00 UTC, before the 10:00 test-health run
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
open_pr:
|
||||
description: Push the bot branch and open or refresh the seed PR (false records and reports only)
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
concurrency:
|
||||
group: free-durations-record
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
linux:
|
||||
runs-on: ubicloud-standard-8
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Prepare the same inputs as a free-suite shard
|
||||
run: |
|
||||
bash scripts/ubicloud/setup-free-suite.sh
|
||||
echo "$HOME/.bun/bin" >> "$GITHUB_PATH"
|
||||
- name: Time every free file alone
|
||||
id: record
|
||||
env:
|
||||
GSTACK_EXPECT_BINARIES: '1'
|
||||
run: |
|
||||
set -o pipefail
|
||||
xvfb-run -a bun run test:free --record-durations 2>&1 | tee "$RUNNER_TEMP/record.log"
|
||||
- name: Keep the recording only when every file was timed
|
||||
id: seed
|
||||
if: always() && steps.record.outcome != 'skipped'
|
||||
run: |
|
||||
grep -q 'wrote [0-9]* durations to' "$RUNNER_TEMP/record.log" || { echo "::error::The recording was interrupted; no Linux seed to upload."; exit 1; }
|
||||
mkdir -p "$RUNNER_TEMP/seed"
|
||||
cp scripts/free-test-durations.json "$RUNNER_TEMP/seed/free-test-durations.json"
|
||||
sed -n '/failed while recording:/,$p' "$RUNNER_TEMP/record.log" | sed 's/\r$//' | { grep -E '^ ✗ [^ ]+$' || true; } | sed 's/^ ✗ //' > "$RUNNER_TEMP/seed/record-failures.txt"
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
if: always()
|
||||
with:
|
||||
name: linux-record-durations-logs
|
||||
path: .context/free-test-logs/gstack-free-test-*.log
|
||||
include-hidden-files: true
|
||||
if-no-files-found: ignore
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
if: always() && steps.seed.outcome == 'success'
|
||||
with:
|
||||
name: free-test-durations
|
||||
path: ${{ runner.temp }}/seed/
|
||||
if-no-files-found: error
|
||||
|
||||
windows:
|
||||
uses: ./.github/workflows/windows-free-tests.yml
|
||||
with:
|
||||
record_durations: true
|
||||
|
||||
propose:
|
||||
needs: [linux, windows]
|
||||
if: ${{ !cancelled() }}
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
actions: write
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
|
||||
with:
|
||||
bun-version: 1.4.2
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
||||
with:
|
||||
name: free-test-durations
|
||||
path: ${{ runner.temp }}/linux
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
|
||||
with:
|
||||
name: free-test-durations-windows
|
||||
path: ${{ runner.temp }}/windows
|
||||
- name: Open or refresh the seed PR when either seed's file set changed
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
OPEN_PR: ${{ github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || inputs.open_pr) }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
linux_failed=$(wc -l < "$RUNNER_TEMP/linux/record-failures.txt")
|
||||
windows_failed=$(wc -l < "$RUNNER_TEMP/windows/record-failures.txt")
|
||||
if [ "$linux_failed" -gt 5 ] || [ "$windows_failed" -gt 5 ]; then
|
||||
echo "::error::$linux_failed Linux and $windows_failed Windows files failed while recording; more than 5 on either OS means a broken tree, so no seed is proposed."
|
||||
exit 1
|
||||
fi
|
||||
failures() { if [ -s "$1" ]; then paste -sd, "$1" | sed 's/,/, /g'; else echo none; fi; }
|
||||
drift() { # committed recorded -> "<added> <removed>"
|
||||
jq -n --slurpfile old "$1" --slurpfile new "$2" \
|
||||
'[($new[0].durations|keys) - ($old[0].durations|keys), ($old[0].durations|keys) - ($new[0].durations|keys)] | map(length) | join(" ")'
|
||||
}
|
||||
read -r linux_added linux_removed <<<"$(drift scripts/free-test-durations.json "$RUNNER_TEMP/linux/free-test-durations.json" | tr -d '"')"
|
||||
read -r windows_added windows_removed <<<"$(drift scripts/free-test-durations-windows.json "$RUNNER_TEMP/windows/free-test-durations-windows.json" | tr -d '"')"
|
||||
echo "Linux seed: +$linux_added -$linux_removed files; Windows seed: +$windows_added -$windows_removed files" | tee -a "$GITHUB_STEP_SUMMARY"
|
||||
if [ "$((linux_added + linux_removed + windows_added + windows_removed))" -eq 0 ]; then
|
||||
echo "Both seeds cover the current test files; no PR." | tee -a "$GITHUB_STEP_SUMMARY"
|
||||
exit 0
|
||||
fi
|
||||
cp "$RUNNER_TEMP/linux/free-test-durations.json" scripts/free-test-durations.json
|
||||
cp "$RUNNER_TEMP/windows/free-test-durations-windows.json" scripts/free-test-durations-windows.json
|
||||
if ratchet=$(GSTACK_FREE_SEED_BASE="$(git rev-parse HEAD)" bun test test/free-seed-ratchet.test.ts 2>&1); then
|
||||
ratchet_line="passes"
|
||||
else
|
||||
ratchet_line="FAILS; a maintainer must split the named file or add an annotated allowlist entry on this branch before merge:"$'\n\n'"\`\`\`"$'\n'"$(printf '%s\n' "$ratchet" | sed -n -E '/Free seed ratchet|^- /p' | sed -n '1,20p')"$'\n'"\`\`\`"
|
||||
fi
|
||||
failed_line="Linux: $(failures "$RUNNER_TEMP/linux/record-failures.txt"); Windows: $(failures "$RUNNER_TEMP/windows/record-failures.txt")"
|
||||
echo "Seed growth ratchet: $ratchet_line" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "Failed while recording: $failed_line" | tee -a "$GITHUB_STEP_SUMMARY"
|
||||
if [ "$OPEN_PR" != true ]; then
|
||||
echo "Report only (open_pr=false, or a ref other than main): no branch pushed." | tee -a "$GITHUB_STEP_SUMMARY"
|
||||
exit 0
|
||||
fi
|
||||
branch=bot/free-duration-seeds
|
||||
subject="test(seed): refresh the free-suite duration seeds from the weekly recording"
|
||||
number=$(gh pr list --head "$branch" --state open --json number -q '.[0].number // empty')
|
||||
remote_head=
|
||||
if git fetch -q origin "+refs/heads/$branch:refs/remotes/origin/$branch" 2>/dev/null; then
|
||||
remote_head=$(git rev-parse "origin/$branch")
|
||||
fi
|
||||
if [ -n "$number" ] && [ -n "$remote_head" ] && [ "$(git log --format=%s "HEAD..$remote_head")" != "$subject" ]; then
|
||||
echo "::warning::Seed PR #$number has commits besides the bot's one refresh commit; this run leaves them in place. Merge or close that PR and the next run refreshes the seeds." | tee -a "$GITHUB_STEP_SUMMARY"
|
||||
exit 0
|
||||
fi
|
||||
git switch -C "$branch"
|
||||
git config user.name github-actions[bot]
|
||||
git config user.email 41898282+github-actions[bot]@users.noreply.github.com
|
||||
git add scripts/free-test-durations.json scripts/free-test-durations-windows.json
|
||||
git commit -m "$subject"
|
||||
git push --force-with-lease="refs/heads/$branch:$remote_head" origin "HEAD:refs/heads/$branch"
|
||||
cat > pr-body.md <<EOF
|
||||
Weekly recording from run $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID on $GITHUB_SHA.
|
||||
|
||||
- Linux seed (ubicloud-standard-8): +$linux_added / -$linux_removed files
|
||||
- Windows seed (windows-latest): +$windows_added / -$windows_removed files
|
||||
- Files that failed while recording (durations kept; this PR's own free checks decide): $failed_line
|
||||
- Seed growth ratchet (60 s ceiling, shrink-only allowlist): $ratchet_line
|
||||
|
||||
Data only. Stamp and merge per docs/ship-policy.md; the stamp push runs the full PR checks.
|
||||
EOF
|
||||
if [ -n "$number" ]; then
|
||||
gh pr edit "$number" --body-file pr-body.md
|
||||
else
|
||||
gh pr create --base main --head "$branch" --title "test(seed): refresh the free-suite duration seeds" --body-file pr-body.md
|
||||
fi
|
||||
gh workflow run free-tests.yml --repo "$GITHUB_REPOSITORY" --ref "$branch"
|
||||
gh workflow run windows-free-tests.yml --repo "$GITHUB_REPOSITORY" --ref "$branch"
|
||||
@@ -29,6 +29,12 @@ on:
|
||||
description: Record per-file Windows durations (artifact) instead of running the lane
|
||||
type: boolean
|
||||
default: false
|
||||
# free-durations-record.yml calls the recording path weekly.
|
||||
workflow_call:
|
||||
inputs:
|
||||
record_durations:
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
concurrency:
|
||||
group: windows-free-${{ github.event.pull_request.number || github.run_id }}
|
||||
@@ -231,15 +237,22 @@ jobs:
|
||||
# would leak into every timed test, and seed-reading tests would then read
|
||||
# the half-written recording instead of their fixtures.
|
||||
- name: Time every Windows-safe file alone
|
||||
id: record
|
||||
shell: bash
|
||||
env:
|
||||
TEMP: ${{ runner.temp }}
|
||||
TMP: ${{ runner.temp }}
|
||||
run: bun run test:windows --record-durations --wall-timeout 1200
|
||||
run: |
|
||||
set -o pipefail
|
||||
bun run test:windows --record-durations --wall-timeout 1200 2>&1 | tee "$RUNNER_TEMP/record.log"
|
||||
- name: Name the recording as the Windows seed
|
||||
if: always()
|
||||
id: seed
|
||||
if: always() && steps.record.outcome != 'skipped'
|
||||
shell: bash
|
||||
run: cp scripts/free-test-durations.json "$RUNNER_TEMP/free-test-durations-windows.json"
|
||||
run: |
|
||||
grep -q 'wrote [0-9]* durations to' "$RUNNER_TEMP/record.log" || { echo "::error::The recording was interrupted; no Windows seed to upload."; exit 1; }
|
||||
cp scripts/free-test-durations.json "$RUNNER_TEMP/free-test-durations-windows.json"
|
||||
sed -n '/failed while recording:/,$p' "$RUNNER_TEMP/record.log" | sed 's/\r$//' | { grep -E '^ ✗ [^ ]+$' || true; } | sed 's/^ ✗ //' > "$RUNNER_TEMP/record-failures.txt"
|
||||
- name: Upload full recording logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
@@ -251,8 +264,10 @@ jobs:
|
||||
include-hidden-files: true
|
||||
if-no-files-found: ignore
|
||||
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
|
||||
if: always()
|
||||
if: always() && steps.seed.outcome == 'success'
|
||||
with:
|
||||
name: free-test-durations-windows
|
||||
path: ${{ runner.temp }}/free-test-durations-windows.json
|
||||
path: |
|
||||
${{ runner.temp }}/free-test-durations-windows.json
|
||||
${{ runner.temp }}/record-failures.txt
|
||||
if-no-files-found: error
|
||||
@@ -1,5 +1,16 @@
|
||||
# Changelog
|
||||
|
||||
## [1.91.128.0] - 2026-10-18
|
||||
|
||||
**The two free-suite duration seeds now refresh themselves weekly.** The shard packers read a Linux and a Windows duration seed, and both went stale as test files came and went. They were refreshed by hand twice this week. A new weekly workflow records both seeds and opens one seed-refresh PR when either seed's file set changes. That PR goes through the normal review.
|
||||
|
||||
### Added
|
||||
- **`free-durations-record.yml`** runs Mondays at 04:00 UTC, before test-health, and on dispatch. It records the Linux seed on `ubicloud-standard-8`, the free-suite shard runner, and the Windows seed through `windows-free-tests.yml`'s `record_durations` path (now also callable with `workflow_call`). When either seed adds or drops a file, it opens or refreshes one PR on `bot/free-duration-seeds` and runs the free lanes on it. The PR body states the seed-ratchet result and names every file that failed while recording.
|
||||
- **Guards:** the bot never edits `scripts/free-test-seed-allowlist.json`. While the open seed PR holds any commit besides the bot's own refresh, later runs leave it alone. Only runs on `main` publish, and `open_pr=false` records and reports only. An interrupted recording, or more than 5 red files on either OS, proposes nothing.
|
||||
|
||||
### Fixed
|
||||
- **An interrupted manual Windows recording no longer uploads the unchanged committed seed** under the Windows seed name.
|
||||
|
||||
## [1.91.127.0] - 2026-10-17
|
||||
|
||||
**The paid PR gate stops failing on its own fixtures.** A 132-verdict PR gate was fully green only about a quarter of the time. Every red verdict since #3231 was diagnosed from its transcript and replayed through the real detector. This release fixes the remaining causes where they start: fixture actors that broke the production protocol, detectors that misread a correct run, and skill prompts that left the agent guessing. No threshold, trial count, case kind or quarantine changed.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# gstack digest v1.91.127.0 — regenerate/re-copy after upgrading gstack
|
||||
# gstack digest v1.91.128.0 — regenerate/re-copy after upgrading gstack
|
||||
|
||||
Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed
|
||||
for agent hosts without a full skill install. The full skills add workflows,
|
||||
|
||||
@@ -1164,10 +1164,11 @@ changed. Cut or unstarted reruns are incomplete, not passes.
|
||||
### Free suite duration seed
|
||||
|
||||
`scripts/free-test-durations.json` drives shard packing; files missing from it
|
||||
pack at the seed's 99th percentile. Acceptable seeds come only from
|
||||
`bun run test:ubicloud --record-durations` (each file timed in its own child on a
|
||||
VM with the CI lane's environment; it needs `UBICLOUD_API_KEY` and copies the
|
||||
refreshed seed back into the checkout). A 4-core laptop or small-sandbox
|
||||
pack at the seed's 99th percentile. Acceptable seeds come only from a recording
|
||||
in the CI lane's environment, each file timed in its own child: the weekly
|
||||
`free-durations-record.yml` run, or `bun run test:ubicloud --record-durations`
|
||||
(it needs `UBICLOUD_API_KEY` and copies the refreshed seed back into the
|
||||
checkout). A 4-core laptop or small-sandbox
|
||||
recording is not acceptable: browser and display tests skip or slow down there,
|
||||
so the seed misstates them. Contributors without `UBICLOUD_API_KEY` leave the
|
||||
refresh to a maintainer.
|
||||
@@ -1188,6 +1189,20 @@ refresh to a maintainer.
|
||||
`{file, reason, todo}` and the test prints it. A merge-base without the
|
||||
allowlist file counts as the first allowlist. Fix a failure by splitting the
|
||||
file, then rerun `bun run test:ubicloud --record-durations`.
|
||||
- `free-durations-record.yml` (Mondays 04:00 UTC, before test-health, and on
|
||||
dispatch) records the Linux seed on `ubicloud-standard-8` with
|
||||
`scripts/ubicloud/setup-free-suite.sh` and the Windows seed through
|
||||
`windows-free-tests.yml`'s `record_durations` path. When either recording adds
|
||||
or drops a test file, it force-updates the `bot/free-duration-seeds` branch,
|
||||
opens or refreshes one seed PR whose body states the ratchet result, and
|
||||
dispatches the free lanes on it. A file that fails while recording turns its
|
||||
job red but keeps its duration, and the PR body names it; the seed PR's own
|
||||
free checks decide whether the tree is healthy. An interrupted recording
|
||||
uploads no seed, and more than 5 red files on either OS proposes nothing. The
|
||||
bot never edits the allowlist: a ratchet failure is fixed on the branch by a
|
||||
maintainer before the PR merges, and while the open seed PR holds any commit
|
||||
besides the bot's one refresh commit, later runs leave it alone. Only runs on
|
||||
`main` push the branch; a dispatch from another ref reports only.
|
||||
|
||||
### Windows free lane
|
||||
|
||||
@@ -1195,7 +1210,8 @@ refresh to a maintainer.
|
||||
runs one strict serial shard per `windows-latest` job
|
||||
(`bun run test:windows --ci-run`), and verifies every shard receipt on Linux
|
||||
(`--windows-only --ci-verify`). Packing uses Windows-measured durations from
|
||||
`scripts/free-test-durations-windows.json`; refresh them with
|
||||
`scripts/free-test-durations-windows.json`; the weekly
|
||||
`free-durations-record.yml` run refreshes them, or by hand run
|
||||
`gh workflow run windows-free-tests.yml --ref <branch> -f record_durations=true`,
|
||||
then download the `free-test-durations-windows` artifact and commit it.
|
||||
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "gstack",
|
||||
"version": "1.91.127",
|
||||
"version": "1.91.128",
|
||||
"description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.",
|
||||
"license": "MIT",
|
||||
"type": "module",
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
"scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":5},
|
||||
"scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":5},
|
||||
"scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":5},
|
||||
"scripts/test-pr-profile.ts": {"blob":"ddc1e5991636c8c94722cd32472edce26489b26d","since":5},
|
||||
"scripts/test-pr-profile.ts": {"blob":"67d3e163dd443bfc5e381586e28f7e83ed59125a","since":5,"nonBehavioral":"Classifies the new free-durations-record.yml workflow as free-lane only; no paid case reads it, so no verdict can change"},
|
||||
"scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":5},
|
||||
"test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":5},
|
||||
"test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":5},
|
||||
|
||||
@@ -230,6 +230,7 @@ export const FREE_ONLY_PR_FILES = [
|
||||
'.github/workflows/cso-runtime-qualification.yml',
|
||||
'.github/workflows/cso-scanner-images.yml',
|
||||
'.github/workflows/dependency-review.yml',
|
||||
'.github/workflows/free-durations-record.yml',
|
||||
'.github/workflows/free-tests.yml',
|
||||
'.github/workflows/make-pdf-gate.yml',
|
||||
'.github/workflows/model-policy-freshness.yml',
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
/**
|
||||
* Static pins and a functional check for the weekly duration-seed recorder
|
||||
* (.github/workflows/free-durations-record.yml): Linux on the free-suite
|
||||
* runner, Windows through the record_durations path, and one bot seed PR only
|
||||
* when a seed's file set drifts.
|
||||
*/
|
||||
import { afterEach, expect, test } from 'bun:test';
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
|
||||
const root = path.resolve(import.meta.dir, '..');
|
||||
const load = (file: string) => Bun.YAML.parse(readFileSync(path.join(root, '.github/workflows', file), 'utf8')) as any;
|
||||
const recorder = load('free-durations-record.yml');
|
||||
const freeTests = load('free-tests.yml');
|
||||
const proposeStep = recorder.jobs.propose.steps.find((step: any) => step.name === 'Open or refresh the seed PR when either seed\'s file set changed');
|
||||
const dirs: string[] = [];
|
||||
afterEach(() => { for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); });
|
||||
|
||||
test('the recorder runs weekly before test-health and on dispatch, read-only except the propose job', () => {
|
||||
expect(recorder.on.schedule).toEqual([{ cron: '0 4 * * 1' }]);
|
||||
expect(Object.keys(recorder.on).sort()).toEqual(['schedule', 'workflow_dispatch']);
|
||||
expect(Object.keys(recorder.on.workflow_dispatch.inputs)).toEqual(['open_pr']);
|
||||
expect(load('test-health.yml').on.schedule[0].cron).toBe('0 10 * * 1');
|
||||
expect(recorder.permissions).toEqual({ contents: 'read' });
|
||||
expect(recorder.concurrency['cancel-in-progress']).toBe(false);
|
||||
expect(recorder.jobs.linux.permissions).toBeUndefined();
|
||||
expect(recorder.jobs.propose.permissions).toEqual({ contents: 'write', 'pull-requests': 'write', actions: 'write' });
|
||||
});
|
||||
|
||||
test('Linux records on the free-suite runner, turns red on any red file, and uploads only a complete recording', () => {
|
||||
const linux = recorder.jobs.linux;
|
||||
expect(linux['runs-on']).toBe(freeTests.jobs['free-suite']['runs-on']);
|
||||
const runs = linux.steps.map((step: any) => step.run).filter(Boolean).join('\n');
|
||||
expect(runs).toContain('bash scripts/ubicloud/setup-free-suite.sh');
|
||||
const record = linux.steps.find((step: any) => step.name === 'Time every free file alone');
|
||||
expect(record.run).toBe('set -o pipefail\nxvfb-run -a bun run test:free --record-durations 2>&1 | tee "$RUNNER_TEMP/record.log"\n');
|
||||
expect(record['continue-on-error']).toBeUndefined();
|
||||
const keep = linux.steps.find((step: any) => step.id === 'seed');
|
||||
expect(keep.if).toBe("always() && steps.record.outcome != 'skipped'");
|
||||
expect(keep.run).toContain('grep -q \'wrote [0-9]* durations to\' "$RUNNER_TEMP/record.log" || {');
|
||||
expect(linux.steps.at(-1).if).toBe("always() && steps.seed.outcome == 'success'");
|
||||
expect(linux.steps.at(-1).with).toMatchObject({ name: 'free-test-durations', path: '${{ runner.temp }}/seed/' });
|
||||
});
|
||||
|
||||
test('Windows records through the windows-free-tests record_durations path, and propose needs both complete recordings', () => {
|
||||
expect(recorder.jobs.windows).toEqual({ uses: './.github/workflows/windows-free-tests.yml', with: { record_durations: true } });
|
||||
expect(recorder.jobs.propose.needs).toEqual(['linux', 'windows']);
|
||||
expect(recorder.jobs.propose.if).toBe('${{ !cancelled() }}');
|
||||
const downloads = recorder.jobs.propose.steps.filter((step: any) => step.uses?.startsWith('actions/download-artifact@'));
|
||||
expect(downloads.map((step: any) => step.with.name)).toEqual(['free-test-durations', 'free-test-durations-windows']);
|
||||
expect(downloads.every((step: any) => step['continue-on-error'] === undefined)).toBe(true);
|
||||
});
|
||||
|
||||
test('both recorders list exactly the files the recorder reported red, and more than 5 on either OS proposes nothing', () => {
|
||||
const windowsRecord = load('windows-free-tests.yml').jobs['windows-record-durations'].steps.find((step: any) => step.id === 'seed').run as string;
|
||||
const linuxRecord = recorder.jobs.linux.steps.find((step: any) => step.id === 'seed').run as string;
|
||||
const extract = (run: string) => run.split('\n').find(line => line.startsWith("sed -n '/failed while recording:/"))!;
|
||||
expect(extract(linuxRecord).replace('seed/record-failures.txt', 'record-failures.txt')).toBe(extract(windowsRecord));
|
||||
const dir = mkdtempSync(path.join(os.tmpdir(), 'seed-failures-'));
|
||||
dirs.push(dir);
|
||||
const list = (log: string) => {
|
||||
writeFileSync(path.join(dir, 'record.log'), log);
|
||||
const result = spawnSync('bash', ['-c', `set -euo pipefail\n${extract(windowsRecord)}\ncat "$RUNNER_TEMP/record-failures.txt"`], { encoding: 'utf8', timeout: 10_000, env: { ...process.env, RUNNER_TEMP: dir } });
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
return result.stdout;
|
||||
};
|
||||
const red = ' ✗ test/a.test.ts — suite > case\r\n[test:free] wrote 3 durations to scripts\\free-test-durations.json\r\n[test:free] WARNING: 2 file(s) failed while recording:\r\n ✗ test/a.test.ts\r\n ✗ browse/test/b.test.ts\r\n';
|
||||
expect(list(red)).toBe('test/a.test.ts\nbrowse/test/b.test.ts\n');
|
||||
expect(list(' ✗ test/a.test.ts — suite > flaky then fixed\n[test:free] wrote 3 durations to scripts/free-test-durations.json\n')).toBe('');
|
||||
const script: string = proposeStep.run;
|
||||
expect(script).toContain('if [ "$linux_failed" -gt 5 ] || [ "$windows_failed" -gt 5 ]; then');
|
||||
expect(script.indexOf('-gt 5 ]')).toBeLessThan(script.indexOf('drift() {'));
|
||||
expect(script).toContain('Files that failed while recording (durations kept; this PR\'s own free checks decide): $failed_line');
|
||||
});
|
||||
|
||||
test('the propose step touches only the two seeds, never the allowlist, on one bot branch', () => {
|
||||
const script: string = proposeStep.run;
|
||||
expect(script).toContain('git add scripts/free-test-durations.json scripts/free-test-durations-windows.json\n');
|
||||
expect(script).not.toContain('free-test-seed-allowlist.json');
|
||||
expect(script).toContain('branch=bot/free-duration-seeds');
|
||||
expect(script).toContain('bun test test/free-seed-ratchet.test.ts');
|
||||
expect(script).toContain('gh workflow run free-tests.yml');
|
||||
expect(proposeStep.env.OPEN_PR).toBe("${{ github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || inputs.open_pr) }}");
|
||||
expect(script.indexOf('if [ "$OPEN_PR" != true ]')).toBeLessThan(script.indexOf('git push'));
|
||||
expect(recorder.on.workflow_dispatch.inputs.open_pr).toMatchObject({ type: 'boolean', default: true });
|
||||
});
|
||||
|
||||
test('a long ratchet report reaches the PR body instead of aborting the step under pipefail', () => {
|
||||
const script: string = proposeStep.run;
|
||||
const line = script.split('\n').find(entry => entry.trimStart().startsWith('ratchet_line="FAILS'))!.trim();
|
||||
const ratchet = ['error: Free seed ratchet:', ...Array.from({ length: 3000 }, (_, i) => `- test/f${i}.test.ts is seeded at 61s, over the 60s ceiling. Fix: split it, then rerun the recording.`)].join('\n');
|
||||
const dir = mkdtempSync(path.join(os.tmpdir(), 'seed-ratchet-'));
|
||||
dirs.push(dir);
|
||||
writeFileSync(path.join(dir, 'ratchet.txt'), ratchet);
|
||||
const result = spawnSync('bash', ['-c', `set -euo pipefail\nratchet=$(cat "$1")\n${line}\nprintf '%s' "$ratchet_line"`, 'ratchet', path.join(dir, 'ratchet.txt')], { encoding: 'utf8', timeout: 10_000 });
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
expect(result.stdout).toContain('- test/f18.test.ts is seeded at 61s, over');
|
||||
expect(result.stdout).not.toContain('- test/f19.test.ts');
|
||||
});
|
||||
|
||||
test('a refresh keeps a maintainer\'s commits on the open seed PR and leases the bot branch it replaces', () => {
|
||||
const script: string = proposeStep.run;
|
||||
const publish = script.slice(script.indexOf('branch=bot/free-duration-seeds'), script.indexOf('cat > pr-body.md'));
|
||||
const dir = mkdtempSync(path.join(os.tmpdir(), 'seed-publish-'));
|
||||
dirs.push(dir);
|
||||
const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith('GIT_')));
|
||||
const git = (cwd: string, ...args: string[]) => {
|
||||
const result = spawnSync('git', args, { cwd, encoding: 'utf8', timeout: 10_000, env });
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
return result.stdout.trim();
|
||||
};
|
||||
const origin = path.join(dir, 'origin.git');
|
||||
const work = path.join(dir, 'work');
|
||||
git(dir, 'init', '-q', '--bare', '-b', 'main', origin);
|
||||
git(dir, 'clone', '-q', origin, work);
|
||||
const commit = (email: string, file: string, message: string) => {
|
||||
git(work, 'config', 'user.name', email);
|
||||
git(work, 'config', 'user.email', email);
|
||||
writeFileSync(path.join(work, file), message);
|
||||
git(work, 'add', file);
|
||||
git(work, 'commit', '-q', '-m', message);
|
||||
};
|
||||
for (const file of ['free-test-durations.json', 'free-test-durations-windows.json']) {
|
||||
writeFileSync(path.join(work, file), '{}');
|
||||
}
|
||||
git(work, 'checkout', '-q', '-b', 'main');
|
||||
commit('dev@example.com', 'README', 'main');
|
||||
git(work, 'push', '-q', 'origin', 'main');
|
||||
const subject = 'test(seed): refresh the free-suite duration seeds from the weekly recording';
|
||||
git(work, 'checkout', '-q', '-b', 'bot/free-duration-seeds');
|
||||
commit('bot@example.com', 'seed.txt', subject);
|
||||
git(work, 'push', '-q', 'origin', 'bot/free-duration-seeds');
|
||||
const stub = path.join(dir, 'bin');
|
||||
spawnSync('mkdir', [stub], { timeout: 10_000 });
|
||||
writeFileSync(path.join(stub, 'gh'), '#!/bin/sh\necho 7\n', { mode: 0o755 });
|
||||
const run = () => {
|
||||
git(work, 'checkout', '-q', 'main');
|
||||
spawnSync('mkdir', ['-p', path.join(work, 'scripts')], { timeout: 10_000 });
|
||||
writeFileSync(path.join(work, 'scripts/free-test-durations.json'), `{"run":${Date.now()}}`);
|
||||
writeFileSync(path.join(work, 'scripts/free-test-durations-windows.json'), '{}');
|
||||
const result = spawnSync('bash', ['-c', `set -euo pipefail\n${publish}`], { cwd: work, encoding: 'utf8', timeout: 20_000,
|
||||
env: { ...env, PATH: `${stub}:${process.env.PATH}`, GITHUB_STEP_SUMMARY: path.join(dir, 'summary.md') } });
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
return git(origin, 'rev-parse', 'refs/heads/bot/free-duration-seeds');
|
||||
};
|
||||
const replaced = run();
|
||||
expect(git(origin, 'log', '--format=%s', `main..${replaced}`)).toBe(subject);
|
||||
expect(git(origin, 'show', `${replaced}:scripts/free-test-durations.json`)).toContain('"run"');
|
||||
git(work, 'checkout', '-q', '-B', 'bot/free-duration-seeds', replaced);
|
||||
commit('maintainer@example.com', 'allowlist.json', 'maintainer repair');
|
||||
git(work, 'push', '-q', 'origin', 'HEAD:refs/heads/bot/free-duration-seeds');
|
||||
const repaired = git(origin, 'rev-parse', 'refs/heads/bot/free-duration-seeds');
|
||||
expect(run()).toBe(repaired);
|
||||
expect(readFileSync(path.join(dir, 'summary.md'), 'utf8')).toContain("Seed PR #7 has commits besides the bot's one refresh commit");
|
||||
});
|
||||
|
||||
test('the drift check counts added and removed files, so an unchanged file set proposes nothing', () => {
|
||||
const script: string = proposeStep.run;
|
||||
const drift = script.slice(script.indexOf('drift() {'), script.indexOf('\n}', script.indexOf('drift() {')) + 2);
|
||||
const dir = mkdtempSync(path.join(os.tmpdir(), 'seed-drift-'));
|
||||
dirs.push(dir);
|
||||
const seed = (name: string, files: Record<string, number>) => {
|
||||
const file = path.join(dir, name);
|
||||
writeFileSync(file, JSON.stringify({ version: 1, recordedAt: '2026-10-17T00:00:00.000Z', durations: files }));
|
||||
return file;
|
||||
};
|
||||
const committed = seed('committed.json', { 'test/a.test.ts': 100, 'test/b.test.ts': 200 });
|
||||
const run = (recorded: string) => spawnSync('bash', ['-c', `${drift}\ndrift "$1" "$2"`, 'drift', committed, recorded], { encoding: 'utf8', timeout: 10_000 });
|
||||
const same = run(seed('same.json', { 'test/a.test.ts': 900, 'test/b.test.ts': 50 }));
|
||||
expect(same.status, same.stderr).toBe(0);
|
||||
expect(same.stdout.trim()).toBe('"0 0"');
|
||||
const drifted = run(seed('drifted.json', { 'test/a.test.ts': 100, 'test/c.test.ts': 1, 'test/d.test.ts': 2 }));
|
||||
expect(drifted.status, drifted.stderr).toBe(0);
|
||||
expect(drifted.stdout.trim()).toBe('"2 1"');
|
||||
});
|
||||
@@ -23,8 +23,9 @@ const native = load('native-qualification.yml');
|
||||
const WINDOWS_CURATED_FLOOR = 638;
|
||||
|
||||
test('the Windows lane plans, runs one strict shard per job and verifies every result', () => {
|
||||
expect(Object.keys(windows.on).sort()).toEqual(['pull_request', 'workflow_dispatch']);
|
||||
expect(Object.keys(windows.on).sort()).toEqual(['pull_request', 'workflow_call', 'workflow_dispatch']);
|
||||
expect(Object.keys(windows.on.workflow_dispatch.inputs)).toEqual(['record_durations']);
|
||||
expect(Object.keys(windows.on.workflow_call.inputs)).toEqual(['record_durations']);
|
||||
const plan = windows.jobs['windows-plan'];
|
||||
expect(plan['runs-on']).toBe('ubuntu-24.04');
|
||||
const planStep = plan.steps.find((step: any) => step.id === 'plan');
|
||||
@@ -62,15 +63,17 @@ test('a record_durations dispatch times each Windows-safe file alone and uploads
|
||||
expect(job.if).toBe('${{ inputs.record_durations }}');
|
||||
expect(job['runs-on']).toBe('windows-latest');
|
||||
const record = job.steps.find((step: any) => step.name === 'Time every Windows-safe file alone');
|
||||
expect(record.run).toBe('bun run test:windows --record-durations --wall-timeout 1200');
|
||||
expect(record.run).toBe('set -o pipefail\nbun run test:windows --record-durations --wall-timeout 1200 2>&1 | tee "$RUNNER_TEMP/record.log"\n');
|
||||
expect(record.env.GSTACK_FREE_TEST_DURATIONS).toBeUndefined();
|
||||
const rename = job.steps.find((step: any) => step.name === 'Name the recording as the Windows seed');
|
||||
expect(rename.if).toBe('always()');
|
||||
expect(rename.run).toBe('cp scripts/free-test-durations.json "$RUNNER_TEMP/free-test-durations-windows.json"');
|
||||
expect(rename.if).toBe("always() && steps.record.outcome != 'skipped'");
|
||||
expect(rename.run).toContain('grep -q \'wrote [0-9]* durations to\' "$RUNNER_TEMP/record.log" || {');
|
||||
expect(rename.run).toContain('cp scripts/free-test-durations.json "$RUNNER_TEMP/free-test-durations-windows.json"');
|
||||
const logs = job.steps.find((step: any) => step.name === 'Upload full recording logs');
|
||||
expect(logs.if).toBe('always()');
|
||||
expect(logs.with['include-hidden-files']).toBe(true);
|
||||
expect(job.steps.at(-1).with.name).toBe('free-test-durations-windows');
|
||||
expect(job.steps.at(-1).if).toBe("always() && steps.seed.outcome == 'success'");
|
||||
});
|
||||
|
||||
test('Windows shards use the pinned Node runtime and retain complete logs on every run', () => {
|
||||
|
||||
Reference in new issue
Block a user