mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-05 08:23:59 +01:00
fix(pregate): remote gate leaves paid test files to the paid gate, adds reason codes and the keyless backend line
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed
1 parent
12968d6db7
commit
3ff7671b5f
2 files changed
+67
-29
No files matched your search
@@ -5,7 +5,7 @@
|
||||
#
|
||||
# gstack-pregate --stage tests --remote --spend-cap 2 --allow-repo-commands
|
||||
#
|
||||
# with PREGATE_FILES_FILE (the selected free test files, one per line),
|
||||
# with PREGATE_FILES_FILE (the selected test files, one per line),
|
||||
# PREGATE_TREE (the working-tree fingerprint being tested) and
|
||||
# PREGATE_SPEND_CAP_USD. It prints the contract line pregate reads
|
||||
# (docs/pregate.md#remote-gate):
|
||||
@@ -40,40 +40,58 @@ root=$(cd "$here/../.." && pwd)
|
||||
rerun='gstack-pregate --stage tests --remote --spend-cap <usd> --allow-repo-commands'
|
||||
|
||||
say() { printf 'pregate-remote: %s\n' "$@"; }
|
||||
stop() { local code=$1; shift; say "$@"; exit "$code"; }
|
||||
stop() { local code=$1 reason=$2; shift 2; say "$reason: $1"; shift; [ $# -eq 0 ] || say "$@"; say "see docs/pregate.md#remote-gate"; exit "$code"; }
|
||||
|
||||
[ -n "${PREGATE_FILES_FILE:-}" ] || stop 2 "PREGATE_FILES_FILE is not set; run: $rerun"
|
||||
[ -n "${PREGATE_TREE:-}" ] || stop 2 "PREGATE_TREE is not set; run: $rerun"
|
||||
[ -f "$PREGATE_FILES_FILE" ] || stop 2 "PREGATE_FILES_FILE=$PREGATE_FILES_FILE does not exist; run: $rerun"
|
||||
[ -n "${PREGATE_FILES_FILE:-}" ] || stop 2 PREGATE_REMOTE_USAGE "PREGATE_FILES_FILE is not set; run: $rerun"
|
||||
[ -n "${PREGATE_TREE:-}" ] || stop 2 PREGATE_REMOTE_USAGE "PREGATE_TREE is not set; run: $rerun"
|
||||
[ -f "$PREGATE_FILES_FILE" ] || stop 2 PREGATE_REMOTE_USAGE "PREGATE_FILES_FILE=$PREGATE_FILES_FILE does not exist; run: $rerun"
|
||||
|
||||
wtree() { (cd "$root" && bin/gstack-wtree); }
|
||||
tree=$(wtree) || stop 3 "bin/gstack-wtree could not fingerprint $root" "nothing uploaded, nothing spent" "rerun: $rerun"
|
||||
tree=$(wtree) || stop 3 PREGATE_REMOTE_TREE_MISMATCH "bin/gstack-wtree could not fingerprint $root" "nothing uploaded, nothing spent" "rerun: $rerun"
|
||||
if [ "$tree" != "$PREGATE_TREE" ]; then
|
||||
stop 3 "the working tree is ${tree:0:12}, not the planned tree ${PREGATE_TREE:0:12}" "nothing uploaded, nothing spent" "rerun: $rerun"
|
||||
stop 3 PREGATE_REMOTE_TREE_MISMATCH "the working tree is ${tree:0:12}, not the planned tree ${PREGATE_TREE:0:12}" "nothing uploaded, nothing spent" "rerun: $rerun"
|
||||
fi
|
||||
|
||||
count=$(grep -c . "$PREGATE_FILES_FILE" || true)
|
||||
logs="$root/.context/pregate-remote/$(date -u +%Y%m%dT%H%M%SZ)-$$"
|
||||
mkdir -p "$logs"
|
||||
log="$logs/run.log"
|
||||
|
||||
# pregate's selection can name paid test files (skill-e2e and other paid
|
||||
# evals); the paid PR gate covers those, and test-free-shards --files-from
|
||||
# rejects them, so only the free files go on.
|
||||
files="$logs/free-files.txt"
|
||||
paid=$(cd "$root" && bun -e 'import { isPaidTestFile } from "./test/helpers/paid-test-set.ts"; import * as fs from "node:fs";
|
||||
const lines = fs.readFileSync(process.argv[1], "utf8").split(/\r?\n/).filter(Boolean);
|
||||
fs.writeFileSync(process.argv[2], lines.filter(f => !isPaidTestFile(f)).map(f => f + "\n").join(""));
|
||||
console.log(lines.filter(f => isPaidTestFile(f)).join("\n"));' "$PREGATE_FILES_FILE" "$files") || stop 2 PREGATE_REMOTE_USAGE "could not split $PREGATE_FILES_FILE into free and paid test files"
|
||||
if [ -n "$paid" ]; then
|
||||
say "$(grep -c . <<<"$paid") paid test file(s) are covered by the paid PR gate, not the free tier: $(head -5 <<<"$paid" | paste -sd, - | sed 's/,/, /g')$( [ "$(grep -c . <<<"$paid")" -le 5 ] || echo " ...")"
|
||||
fi
|
||||
count=$(grep -c . "$files" || true)
|
||||
if [ "$count" -eq 0 ]; then
|
||||
say "PREGATE_REMOTE_PAID_ONLY: every selected file is a paid test file; no free test ran, so this receipt cannot clear the obligation (tests=0)" "see docs/pregate.md#remote-gate"
|
||||
echo "PREGATE_REMOTE: tree=$tree tests=0 failed=0 spent=0"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
result_line() { grep '^\[test:free\] FILES-FROM RESULT: ' "$1" | tail -1 || true; }
|
||||
field() { sed -n "s/.* $1=\([0-9][0-9]*\).*/\1/p" <<<"$2"; }
|
||||
|
||||
if [ -z "${UBICLOUD_API_KEY:-${UBICLOUD_API_TOKEN:-}}" ]; then
|
||||
say "no UBICLOUD_API_KEY: running $count files locally in parallel shards"
|
||||
set +e
|
||||
(cd "$root" && bun run scripts/test-free-shards.ts --files-from "$PREGATE_FILES_FILE") 2>&1 | tee "$log"
|
||||
(cd "$root" && bun run scripts/test-free-shards.ts --files-from "$files") 2>&1 | tee "$log"
|
||||
rc=${PIPESTATUS[0]}
|
||||
set -e
|
||||
after=$(wtree) || after=unavailable
|
||||
if [ "$after" != "$tree" ]; then
|
||||
(cd "$root" && git diff --name-status "$tree" "$after" 2>&1 | head -20 | sed 's/^/ moved: /') || true
|
||||
stop 3 "the working tree moved during the run (${tree:0:12} before, ${after:0:12} after); no result reported" "log: $log" "rerun on a tree nothing else edits: $rerun"
|
||||
stop 3 PREGATE_REMOTE_TREE_MOVED "the working tree moved during the run (${tree:0:12} before, ${after:0:12} after); no result reported" "log: $log" "rerun on a tree nothing else edits: $rerun"
|
||||
fi
|
||||
result=$(result_line "$log")
|
||||
[ -n "$result" ] || stop "$(( rc == 0 ? 1 : rc ))" "test-free-shards exited $rc without a FILES-FROM RESULT line" "log: $log"
|
||||
[ -n "$result" ] || stop "$(( rc == 0 ? 1 : rc ))" PREGATE_REMOTE_NO_RESULT "test-free-shards exited $rc without a FILES-FROM RESULT line" "log: $log"
|
||||
say "log: $log"
|
||||
echo "backend=local ($(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu) cores; not the CI environment)"
|
||||
echo "PREGATE_REMOTE: tree=$tree tests=$(field tests "$result") failed=$(field failed "$result") spent=0"
|
||||
exit "$rc"
|
||||
fi
|
||||
@@ -82,19 +100,19 @@ size=${UBI_SIZE:-standard-16}
|
||||
case $size in
|
||||
standard-2) ppm=0.00104 ;; standard-4) ppm=0.00208 ;; standard-8) ppm=0.00416 ;;
|
||||
standard-16) ppm=0.00831 ;; standard-30) ppm=0.01559 ;; standard-60) ppm=0.03117 ;;
|
||||
*) stop 5 "no price for UBI_SIZE=$size (priced: standard-2, -4, -8, -16, -30, -60; Ubicloud list price per minute)" "nothing created, nothing spent" ;;
|
||||
*) stop 5 PREGATE_REMOTE_NO_PRICE "no price for UBI_SIZE=$size (priced: standard-2, -4, -8, -16, -30, -60; Ubicloud list price per minute)" "nothing created, nothing spent" ;;
|
||||
esac
|
||||
cap=${PREGATE_SPEND_CAP_USD:-}
|
||||
[[ "$cap" =~ ^[0-9]+(\.[0-9]+)?$ ]] || stop 2 "PREGATE_SPEND_CAP_USD is not a dollar amount ('$cap'); run: $rerun"
|
||||
[[ "$cap" =~ ^[0-9]+(\.[0-9]+)?$ ]] || stop 2 PREGATE_REMOTE_USAGE "PREGATE_SPEND_CAP_USD is not a dollar amount ('$cap'); run: $rerun"
|
||||
max_min=45 min_min=20
|
||||
wall_min=$(awk -v c="$cap" -v p="$ppm" -v m="$max_min" 'BEGIN { w = int(c / p); print (w < m ? w : m) }')
|
||||
usd() { awk -v m="$1" -v p="$ppm" 'BEGIN { s = m * p; c = int(s * 100); if (c < s * 100) c++; printf "%.2f", c / 100 }'; }
|
||||
if [ "$wall_min" -lt "$min_min" ]; then
|
||||
stop 5 "worst case \$$(usd "$min_min") ($min_min min x \$$ppm/min on $size) exceeds the spend cap \$$cap" "nothing created, nothing spent" "rerun with --spend-cap $(usd "$min_min") or more"
|
||||
stop 5 PREGATE_REMOTE_CAP_TOO_LOW "worst case \$$(usd "$min_min") ($min_min min x \$$ppm/min on $size) exceeds the spend cap \$$cap" "nothing created, nothing spent" "rerun with --spend-cap $(usd "$min_min") or more"
|
||||
fi
|
||||
say "$count files on one $size VM; wall timeout $wall_min min, worst case \$$(usd "$wall_min") under the cap \$$cap"
|
||||
|
||||
digest=$(tr -d '\r' <"$PREGATE_FILES_FILE" | grep . | LC_ALL=C sort | sha256sum | cut -d' ' -f1)
|
||||
digest=$(tr -d '\r' <"$files" | grep . | LC_ALL=C sort | sha256sum | cut -d' ' -f1)
|
||||
prelude="$logs/vm-setup.sh"
|
||||
{
|
||||
printf '%s\n' 'set -euo pipefail' 'BASH_COMPAT=50' \
|
||||
@@ -103,7 +121,7 @@ prelude="$logs/vm-setup.sh"
|
||||
"if [ \"\$vm_tree\" != $tree ]; then echo \"PREGATE_VM_TREE_MISMATCH vm=\$vm_tree\"; exit 3; fi" \
|
||||
'echo "PREGATE_VM_TREE $vm_tree"' \
|
||||
"cat > ~/.pregate-files.txt <<'PREGATE_FILES'"
|
||||
tr -d '\r' <"$PREGATE_FILES_FILE" | grep .
|
||||
tr -d '\r' <"$files" | grep .
|
||||
printf '%s\n' 'PREGATE_FILES' "cat > ~/.pregate-run.sh <<'PREGATE_RUN'" \
|
||||
'planned=$(bun run scripts/test-free-shards.ts --files-from ~/.pregate-files.txt --dry-run | sed -n "s/^\[test:free\] files-from .*: [0-9]* files, sha256=\([0-9a-f]*\)$/\1/p")' \
|
||||
"if [ \"\$planned\" != $digest ]; then echo \"PREGATE_VM_PLAN_MISMATCH planned=\${planned:-none} expected=$digest\"; exit 3; fi" \
|
||||
@@ -156,7 +174,7 @@ started=$(date +%s)
|
||||
heartbeat=$!
|
||||
vmrun="" tailer=""
|
||||
trap 'kill "$heartbeat" $tailer 2>/dev/null || true' EXIT
|
||||
trap '[ -z "$vmrun" ] || kill -TERM "$(cat "$log.pid" 2>/dev/null)" 2>/dev/null; stop 130 "interrupted: ubi-runner is tearing the VM down on its own (log: $log)"' INT TERM
|
||||
trap '[ -z "$vmrun" ] || kill -TERM "$(cat "$log.pid" 2>/dev/null)" 2>/dev/null; stop 130 PREGATE_REMOTE_INTERRUPTED "interrupted: ubi-runner is tearing the VM down on its own (log: $log)"' INT TERM
|
||||
|
||||
billed=0 attempt=0
|
||||
while :; do
|
||||
@@ -202,7 +220,7 @@ say "log: $log"
|
||||
vm_tree=$(sed -n 's/^[0-9]* PREGATE_VM_TREE \([0-9a-f]*\)$/\1/p' "$log" | tail -1)
|
||||
mismatch=$(grep -m1 -oE 'PREGATE_VM_(TREE|PLAN)_MISMATCH .*' "$log" || true)
|
||||
if [ -n "$mismatch" ]; then
|
||||
stop 3 "the VM refused before running tests: $mismatch (expected tree ${tree:0:12})" "no tests ran; spent=$spent ($billed billed min on $size)" "rerun: $rerun"
|
||||
stop 3 PREGATE_REMOTE_VM_TREE "the VM refused before running tests: $mismatch (expected tree ${tree:0:12})" "no tests ran; spent=$spent ($billed billed min on $size)" "rerun: $rerun"
|
||||
fi
|
||||
result=$(result_line <(cut -d' ' -f2- "$log"))
|
||||
if [ -n "$result" ] && [ "$rc" -ne 124 ]; then
|
||||
@@ -211,7 +229,7 @@ if [ -n "$result" ] && [ "$rc" -ne 124 ]; then
|
||||
sed -n 's/^[0-9]* PREGATE_VM_REBUILT / setup rebuilt (committed build output, as the CI free lane rebuilds it): /p' "$log"
|
||||
if [ "$after" != "$built" ]; then
|
||||
sed -n 's/^[0-9]* PREGATE_VM_MOVED / moved: /p' "$log"
|
||||
stop 3 "the VM's tree moved during the run (${built:0:12} after setup, ${after:0:12} after tests); no result reported" "spent=$spent ($billed billed min on $size)" "rerun: $rerun"
|
||||
stop 3 PREGATE_REMOTE_TREE_MOVED "the VM's tree moved during the run (${built:0:12} after setup, ${after:0:12} after tests); no result reported" "spent=$spent ($billed billed min on $size)" "rerun: $rerun"
|
||||
fi
|
||||
echo "PREGATE_REMOTE: tree=$vm_tree tests=$(field tests "$result") failed=$(field failed "$result") spent=$spent"
|
||||
exit $(( rc == 0 ? 0 : 1 ))
|
||||
@@ -220,5 +238,5 @@ vm=$(sed -n 's/^[0-9]* ubi-runner: creating \([^ ]*\) .*/\1/p' "$log" | tail -1)
|
||||
down=$(grep -E 'ubi-runner: (destroyed|gone:|WARNING|FAILED to destroy)' "$log" | tail -1 | cut -d' ' -f2- || true)
|
||||
cause=$(grep -E 'ubi-runner: ' "$log" | grep -vE 'ubi-runner: (destroyed|gone:|destroy requested|waiting for an in-flight)' | tail -1 | cut -d' ' -f2- || true)
|
||||
[ "$rc" -ne 124 ] || cause="wall timeout of $(( wall_min - 3 )) min reached (cap \$$cap); the VM was torn down"
|
||||
stop 4 "infrastructure failure (ubi-runner exit $rc): ${cause:-no ubi-runner message}" \
|
||||
stop 4 PREGATE_REMOTE_INFRA "infrastructure failure (ubi-runner exit $rc): ${cause:-no ubi-runner message}" \
|
||||
"VM: ${vm:-none created}${down:+; $down}" "log: $log" "spent=$spent ($billed billed min on $size)"
|
||||
@@ -156,7 +156,8 @@ describe('pregate-remote.sh inputs and the local tree check', () => {
|
||||
const f = fixture();
|
||||
const r = gate(f, null, KEY);
|
||||
expect(r.exit).toBe(2);
|
||||
expect(r.out).toContain('PREGATE_FILES_FILE is not set; run: gstack-pregate --stage tests --remote --spend-cap <usd> --allow-repo-commands');
|
||||
expect(r.out).toContain('pregate-remote: PREGATE_REMOTE_USAGE: PREGATE_FILES_FILE is not set; run: gstack-pregate --stage tests --remote --spend-cap <usd> --allow-repo-commands');
|
||||
expect(r.out).toContain('see docs/pregate.md#remote-gate');
|
||||
expect(r.calls).toEqual([]);
|
||||
expect(fs.existsSync(path.join(f.repo, '.context'))).toBe(false);
|
||||
}, 60_000);
|
||||
@@ -165,7 +166,7 @@ describe('pregate-remote.sh inputs and the local tree check', () => {
|
||||
const f = fixture();
|
||||
const r = gate(f, ['test/alpha.test.ts'], { ...KEY, PREGATE_TREE: 'f'.repeat(40) });
|
||||
expect(r.exit).toBe(3);
|
||||
expect(r.out).toContain(`the working tree is ${f.tree.slice(0, 12)}, not the planned tree ffffffffffff`);
|
||||
expect(r.out).toContain(`PREGATE_REMOTE_TREE_MISMATCH: the working tree is ${f.tree.slice(0, 12)}, not the planned tree ffffffffffff`);
|
||||
expect(r.out).toContain('nothing uploaded, nothing spent');
|
||||
expect(r.out).toContain('rerun: gstack-pregate --stage tests --remote --spend-cap <usd> --allow-repo-commands');
|
||||
expect(r.calls).toEqual([]);
|
||||
@@ -210,7 +211,7 @@ describe('pregate-remote.sh keyed path (stub ubi-runner)', () => {
|
||||
const f = fixture();
|
||||
const low = gate(f, ['test/alpha.test.ts'], { ...KEY, PREGATE_SPEND_CAP_USD: '0.1' });
|
||||
expect(low.exit).toBe(5);
|
||||
expect(low.out).toContain('worst case $0.17 (20 min x $0.00831/min on standard-16) exceeds the spend cap $0.1');
|
||||
expect(low.out).toContain('PREGATE_REMOTE_CAP_TOO_LOW: worst case $0.17 (20 min x $0.00831/min on standard-16) exceeds the spend cap $0.1');
|
||||
expect(low.out).toContain('nothing created, nothing spent');
|
||||
expect(low.out).toContain('rerun with --spend-cap 0.17 or more');
|
||||
const big = gate(f, ['test/alpha.test.ts'], { ...KEY, PREGATE_SPEND_CAP_USD: '0.5', UBI_SIZE: 'standard-60' });
|
||||
@@ -218,7 +219,7 @@ describe('pregate-remote.sh keyed path (stub ubi-runner)', () => {
|
||||
expect(big.out).toContain('worst case $0.63 (20 min x $0.03117/min on standard-60) exceeds the spend cap $0.5');
|
||||
const unpriced = gate(f, ['test/alpha.test.ts'], { ...KEY, UBI_SIZE: 'burstable-1' });
|
||||
expect(unpriced.exit).toBe(5);
|
||||
expect(unpriced.out).toContain('no price for UBI_SIZE=burstable-1');
|
||||
expect(unpriced.out).toContain('PREGATE_REMOTE_NO_PRICE: no price for UBI_SIZE=burstable-1');
|
||||
const noCap = gate(f, ['test/alpha.test.ts'], { ...KEY, PREGATE_SPEND_CAP_USD: '' });
|
||||
expect(noCap.exit).toBe(2);
|
||||
expect([low, big, unpriced, noCap].every(r => r.report === null)).toBe(true);
|
||||
@@ -228,7 +229,7 @@ describe('pregate-remote.sh keyed path (stub ubi-runner)', () => {
|
||||
test('create, setup and SSH failures exit 4 with the ubi-runner message, the VM, its teardown line, the log and spent=', () => {
|
||||
const create = gate(fixture(), ['test/alpha.test.ts'], { ...KEY, STUB_FAIL: 'create' });
|
||||
expect(create.exit, create.out).toBe(4);
|
||||
expect(create.out).toContain('infrastructure failure (ubi-runner exit 1): ubi-runner: create failed');
|
||||
expect(create.out).toContain('PREGATE_REMOTE_INFRA: infrastructure failure (ubi-runner exit 1): ubi-runner: create failed');
|
||||
expect(create.out).toContain('VM: eu-central-h1/ubirun-stub-1700000000-abcd; ubi-runner: gone: eu-central-h1/ubirun-stub-1700000000-abcd (never existed or already destroyed)');
|
||||
expect(create.out).toMatch(/log: \S+\/\.context\/pregate-remote\/\S+\/run\.log/);
|
||||
expect(create.out).toContain('spent=0.00 (0 billed min on standard-16)');
|
||||
@@ -257,14 +258,14 @@ describe('pregate-remote.sh keyed path (stub ubi-runner)', () => {
|
||||
test('the VM refuses a tree that differs after unpack, a planned set that differs from the list, and a tree that moved during the run', () => {
|
||||
const edited = gate(fixture(), ['test/alpha.test.ts'], { ...KEY, STUB_VM_EDIT: 'test/alpha.test.ts' });
|
||||
expect(edited.exit, edited.out).toBe(3);
|
||||
expect(edited.out).toContain('the VM refused before running tests: PREGATE_VM_TREE_MISMATCH vm=');
|
||||
expect(edited.out).toContain('PREGATE_REMOTE_VM_TREE: the VM refused before running tests: PREGATE_VM_TREE_MISMATCH vm=');
|
||||
expect(edited.out).toContain('no tests ran; spent=0.01');
|
||||
const listed = gate(fixture(), ['test/alpha.test.ts'], { ...KEY, STUB_VM_LIST_ADD: 'test/beta.test.ts' });
|
||||
expect(listed.exit, listed.out).toBe(3);
|
||||
expect(listed.out).toContain('PREGATE_VM_PLAN_MISMATCH planned=');
|
||||
const moved = gate(fixture(), ['test/alpha.test.ts', 'test/mover.test.ts'], KEY);
|
||||
expect(moved.exit, moved.out).toBe(3);
|
||||
expect(moved.out).toContain("the VM's tree moved during the run");
|
||||
expect(moved.out).toContain("PREGATE_REMOTE_TREE_MOVED: the VM's tree moved during the run");
|
||||
expect(moved.out).toContain(' moved: A\tmoved.txt');
|
||||
expect([edited, listed, moved].every(r => r.report === null)).toBe(true);
|
||||
}, 90_000);
|
||||
@@ -318,6 +319,7 @@ describe('pregate-remote.sh keyless path', () => {
|
||||
const r = gate(f, ['test/alpha.test.ts', 'test/beta.test.ts']);
|
||||
expect(r.exit, r.out).toBe(0);
|
||||
expect(r.out).toContain('no UBICLOUD_API_KEY: running 2 files locally in parallel shards');
|
||||
expect(r.out).toMatch(/\nbackend=local \(\d+ cores; not the CI environment\)\nPREGATE_REMOTE: /);
|
||||
expect(r.report).toEqual({ tree: f.tree, tests: 2, failed: 0, spent: '0' });
|
||||
expect(remoteReportProblem(r.report!, f.tree)).toBeNull();
|
||||
expect(fs.existsSync(f.calls)).toBe(false);
|
||||
@@ -326,16 +328,34 @@ describe('pregate-remote.sh keyless path', () => {
|
||||
test('a tree that moves during the local run fails without a report', () => {
|
||||
const r = gate(fixture(), ['test/alpha.test.ts', 'test/mover.test.ts']);
|
||||
expect(r.exit, r.out).toBe(3);
|
||||
expect(r.out).toContain('the working tree moved during the run');
|
||||
expect(r.out).toContain('PREGATE_REMOTE_TREE_MOVED: the working tree moved during the run');
|
||||
expect(r.out).toContain(' moved: A\tmoved.txt');
|
||||
expect(r.report).toBeNull();
|
||||
}, 90_000);
|
||||
|
||||
test('paid test files in the selection are named and left to the paid PR gate; a paid-only selection reports tests=0 and cannot clear', () => {
|
||||
const f = fixture();
|
||||
const paid = ['test/skill-e2e-a.test.ts', 'test/skill-e2e-b.test.ts', 'test/skill-llm-eval.test.ts', 'test/codex-e2e-x.test.ts', 'test/carve-section-loading-y.test.ts', 'test/skill-routing-e2e.test.ts'];
|
||||
const mixed = gate(f, ['test/alpha.test.ts', ...paid]);
|
||||
expect(mixed.exit, mixed.out).toBe(0);
|
||||
expect(mixed.out).toContain('6 paid test file(s) are covered by the paid PR gate, not the free tier: test/skill-e2e-a.test.ts, test/skill-e2e-b.test.ts, test/skill-llm-eval.test.ts, test/codex-e2e-x.test.ts, test/carve-section-loading-y.test.ts ...');
|
||||
expect(mixed.out).toContain('no UBICLOUD_API_KEY: running 1 files locally in parallel shards');
|
||||
expect(mixed.report).toEqual({ tree: f.tree, tests: 1, failed: 0, spent: '0' });
|
||||
const keyed = gate(f, ['test/beta.test.ts', 'test/skill-e2e-a.test.ts'], KEY);
|
||||
expect(keyed.exit, keyed.out).toBe(0);
|
||||
expect(fs.readFileSync(path.join(f.vm, '.pregate-files.txt'), 'utf8')).toBe('test/beta.test.ts\n');
|
||||
const only = gate(f, paid.slice(0, 2));
|
||||
expect(only.exit, only.out).toBe(0);
|
||||
expect(only.out).toContain('PREGATE_REMOTE_PAID_ONLY: every selected file is a paid test file');
|
||||
expect(only.report).toEqual({ tree: f.tree, tests: 0, failed: 0, spent: '0' });
|
||||
expect(remoteReportProblem(only.report!, f.tree)).toBe('the remote gate reported 0 tests');
|
||||
}, 90_000);
|
||||
|
||||
test('a list test-free-shards rejects exits 1 without a report', () => {
|
||||
const r = gate(fixture(), ['test/alpha.test.ts', 'test/missing.test.ts']);
|
||||
expect(r.exit, r.out).toBe(1);
|
||||
expect(r.out).toContain('1 path(s) not in the free planned set: test/missing.test.ts');
|
||||
expect(r.out).toContain('without a FILES-FROM RESULT line');
|
||||
expect(r.out).toContain('PREGATE_REMOTE_NO_RESULT: test-free-shards exited 1 without a FILES-FROM RESULT line');
|
||||
expect(r.report).toBeNull();
|
||||
}, 90_000);
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user