mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-05 16:34:00 +01:00
fix(flaky): ubi-runner quota wait counts its budget down; pregate TERM teardown and design-lint linearity checks measure what they mean (GSTA-128)
Daily flaky pass, free-tier timing entries from the 2026-10-27 ledger. - test/ubicloud-runner.test.ts quota case (Free Tests 40474956830, 40480063084, 40488875742): quota_preflight pinned a wall-clock deadline before its first usage call, so a slow fake-CLI spawn on a loaded runner shaved the printed budget to "waiting up to 2s more". The budget now counts seconds spent waiting for room; usage reads between sleeps no longer eat it. Loaded local loop: 8/12 and 6/12 green before, 12/12 after. - test/ubicloud-pregate-remote.test.ts TERM teardown (40487321907 shard 13): the stub's `>>` creates the .down file before echo writes the line, and the test polled the inode, then read "". Both teardown waits now wait for the written line. - test/design-lint-fifth-cycle.test.ts tracked-caps linearity (40494745147 shard 15): the 8000-label run was timed once by wall clock on a shared runner (515 ms against the 500 ms floor). It now measures the lint's own CPU time, fastest of three runs, against the same unchanged bound. - The hung-setup watchdog assertion now carries the runner's stderr, so its next red names the cause (one unexplained exit 2 in 46 local runs). Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
6ff98c6616
commit
b811ee2dbb
4 files changed
+18
-10
No files matched your search
@@ -393,7 +393,7 @@ vcpus() { [[ "$1" =~ -([0-9]+)$ ]] && echo "${BASH_REMATCH[1]}"; }
|
||||
# maximum the last refusal printed; without either the API's own refusal rules.
|
||||
QUOTA_CACHE_MAX_AGE=86400
|
||||
quota_preflight() {
|
||||
local location=$1 name=$2 size=$3 need max used wait_until remaining source="UBI_QUOTA_VCPUS" cache="$STATE_ROOT/quota-max"
|
||||
local location=$1 name=$2 size=$3 need max used remaining step source="UBI_QUOTA_VCPUS" cache="$STATE_ROOT/quota-max"
|
||||
need=$(vcpus "$size") || return 0
|
||||
max=$QUOTA_VCPUS
|
||||
if [ -z "$max" ] && [ -f "$cache" ]; then
|
||||
@@ -406,19 +406,22 @@ quota_preflight() {
|
||||
fi
|
||||
fi
|
||||
[[ "$max" =~ ^[0-9]+$ ]] || return 0
|
||||
wait_until=$(( $(date +%s) + QUOTA_WAIT ))
|
||||
# The budget counts seconds spent waiting for room, not the usage calls
|
||||
# between them: a slow first usage read must not shave the printed budget.
|
||||
remaining=$QUOTA_WAIT
|
||||
while :; do
|
||||
used=$(cmd_usage 2>/dev/null | awk '$1=="total" {print $3}')
|
||||
[[ "$used" =~ ^[0-9]+$ ]] || return 0
|
||||
[ $(( used + need )) -gt "$max" ] || return 0
|
||||
remaining=$(( wait_until - $(date +%s) ))
|
||||
if [ "$remaining" -le 0 ]; then
|
||||
log "quota preflight refused $location/$name ($size): it needs $need vCPUs and the project's VMs already use $used of $max (ceiling from $source; other usage such as managed runners counts on top)"
|
||||
cmd_usage >&2 || true
|
||||
die "create refused before any request: vCPU quota exhausted (quota full); wait for running VMs to finish (UBI_QUOTA_WAIT=<seconds> waits here), run $0 reap, use a smaller size, or set UBI_QUOTA_VCPUS to the project's real ceiling"
|
||||
fi
|
||||
log "quota: $used of $max vCPUs in use and $need needed; waiting up to ${remaining}s more for room"
|
||||
sleep $(( remaining < 30 ? remaining : 30 ))
|
||||
step=$(( remaining < 30 ? remaining : 30 ))
|
||||
sleep "$step"
|
||||
remaining=$(( remaining - step ))
|
||||
done
|
||||
}
|
||||
|
||||
|
||||
@@ -36,6 +36,9 @@ const quiet = { warn: () => {} };
|
||||
const run = (html: string, files: Record<string, string> = {}): Finding[] => { const dir = tree({ 'index.html': html, ...files }); return lintStatic([dir], { cwd: dir, ...quiet }).findings; };
|
||||
const only = (f: Finding[], rule: string) => f.filter((x) => x.rule === rule);
|
||||
const timed = (fn: () => void): number => { const t = performance.now(); fn(); return performance.now() - t; };
|
||||
// A linearity check measures the lint's own CPU time, fastest of three runs: wall time on a loaded shard carries scheduling waits that are not the lint's cost.
|
||||
const cpuTimed = (fn: () => void): number => { const c = process.cpuUsage(); fn(); const d = process.cpuUsage(c); return (d.user + d.system) / 1000; };
|
||||
const fastest = (fn: () => void, trials = 3): number => Math.min(...Array.from({ length: trials }, () => cpuTimed(fn)));
|
||||
|
||||
describe('design-lint fifth cycle: dead markup and hostile inputs', () => {
|
||||
test('an emoji placeholder inside an HTML comment is dead', () => {
|
||||
@@ -63,7 +66,7 @@ describe('design-lint fifth cycle: dead markup and hostile inputs', () => {
|
||||
});
|
||||
|
||||
test('a page of many tracked-caps labels lints in linear time and a well-formed JSX file still yields its classes', () => {
|
||||
const times = [4000, 8000].map((n) => { const dir = tree({ 'index.html': page('', '<main><h1>Hello world</h1>' + '<span class="text-xs uppercase tracking-widest">Label one</span>\n'.repeat(n) + '</main>') }); return timed(() => lintStatic([dir], { cwd: dir, ...quiet })); });
|
||||
const times = [4000, 8000].map((n) => { const dir = tree({ 'index.html': page('', '<main><h1>Hello world</h1>' + '<span class="text-xs uppercase tracking-widest">Label one</span>\n'.repeat(n) + '</main>') }); return fastest(() => lintStatic([dir], { cwd: dir, ...quiet })); });
|
||||
expect(times[1]).toBeLessThan(Math.max(times[0] * 2.5, 500));
|
||||
const doc = parseDoc('Page.tsx', 'export default function P(){ return (<div className="p-4"><Card className="rounded-xl">x</Card></div>); }');
|
||||
expect(doc.classAt.size + doc.virtuals.length).toBeGreaterThan(0);
|
||||
|
||||
@@ -285,21 +285,23 @@ describe('pregate-remote.sh keyed path (stub ubi-runner)', () => {
|
||||
while (!cond()) { if (Date.now() > deadline) throw new Error('condition not met in time'); await Bun.sleep(50); }
|
||||
};
|
||||
const vmLog = (f: Fixture) => { try { return fs.readFileSync(fs.readdirSync(path.join(f.repo, '.context/pregate-remote')).map(d => path.join(f.repo, '.context/pregate-remote', d, 'run.log'))[0]!, 'utf8'); } catch { return ''; } };
|
||||
// The stub's `>>` creates the file before echo writes the line; wait for the line itself.
|
||||
const down = (f: Fixture) => { try { return fs.readFileSync(f.calls + '.down', 'utf8'); } catch { return ''; } };
|
||||
const killed = fixture();
|
||||
const a = run(killed);
|
||||
await until(() => vmLog(killed).includes('ubi-runner: running on'), 30_000);
|
||||
process.kill(-a.pid, 'SIGKILL');
|
||||
await a.exited;
|
||||
await until(() => fs.existsSync(killed.calls + '.down'), 30_000);
|
||||
expect(fs.readFileSync(killed.calls + '.down', 'utf8')).toMatch(/^exit ubirun-stub-\d+-[0-9a-f]{8}\n$/);
|
||||
await until(() => down(killed).endsWith('\n'), 30_000);
|
||||
expect(down(killed)).toMatch(/^exit ubirun-stub-\d+-[0-9a-f]{8}\n$/);
|
||||
const termed = fixture();
|
||||
const b = run(termed);
|
||||
await until(() => vmLog(termed).includes('ubi-runner: running on'), 30_000);
|
||||
process.kill(b.pid, 'SIGTERM');
|
||||
expect(await b.exited).toBe(130);
|
||||
expect(await new Response(b.stdout).text()).toContain('interrupted: ubi-runner is tearing the VM down on its own');
|
||||
await until(() => fs.existsSync(termed.calls + '.down'), 10_000);
|
||||
expect(fs.readFileSync(termed.calls + '.down', 'utf8')).toMatch(/^term ubirun-stub-\d+-[0-9a-f]{8}\n$/);
|
||||
await until(() => down(termed).endsWith('\n'), 10_000);
|
||||
expect(down(termed)).toMatch(/^term ubirun-stub-\d+-[0-9a-f]{8}\n$/);
|
||||
const edited = fixture();
|
||||
const c = run(edited);
|
||||
await until(() => vmLog(edited).includes('ubi-runner: running on'), 30_000);
|
||||
|
||||
@@ -395,7 +395,7 @@ describe('ubi-runner run, verify and quota against a fake API and a fake ssh', (
|
||||
const setup = join(f.root, 'setup.sh'); writeFileSync(setup, 'echo hi\n');
|
||||
const r = run(f, ['run', '--src', f.root, '--setup', setup, '--', 'make test'], { UBI_IDLE_TIMEOUT: '2' });
|
||||
const err = r.stderr.toString();
|
||||
expect(r.exitCode).toBe(124);
|
||||
expect(r.exitCode, err).toBe(124);
|
||||
expect(err).toMatch(/watchdog: setup printed nothing for \d+s \(UBI_IDLE_TIMEOUT=2\)/);
|
||||
expect(err).toContain('setup failed (exit 124)');
|
||||
expect(f.ssh()).not.toContain('bash -lc');
|
||||
|
||||
Reference in new issue
Block a user