mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-01 10:50:39 +02:00
refactor(test): mechanical sweep — 298 paid-test timeouts onto eval-budget tiers
69 files, both shapes (trailing bun-test budgets and runner timeout/timeoutMs options), ROUND-UP ONLY so nothing that passed can start failing: 75 → JUDGE_MS, 137 → CAPTURE_MS, 74 → CAPTURE_LONG_MS, 9 → PTY_MS, 3 → PTY_LONG_MS. Raw >=60s literal count in the paid scope: 395 → 97, of which 51 are non-timeout noise (fixture dates, run IDs) and 46 are enumerated justified holds (comment-carrying calibrated budgets, poll-loop constants, utility spawn waits, and the seven physical-ceiling 1_500_000 sites). The eval-budgets policy ratchet keeps the residue from regrowing. Known collapse: where an inner runner budget and its enclosing test budget now share a tier, the old stagger is gone — an overrun surfaces as a bun test timeout instead of a graceful runner timeout (diagnosability trade, not a correctness one). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
74ae357e0f
commit
6841183c35
@@ -1,4 +1,5 @@
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
ROOT, browseBin, runId, evalsEnabled,
|
||||
@@ -67,7 +68,7 @@ Do NOT use AskUserQuestion. Do NOT run gh or fly commands.`,
|
||||
workingDirectory: landDir,
|
||||
maxTurns: 20,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'land-and-deploy-workflow',
|
||||
runId,
|
||||
});
|
||||
@@ -85,7 +86,7 @@ Do NOT use AskUserQuestion. Do NOT run gh or fly commands.`,
|
||||
|
||||
const reportDir = path.join(landDir, '.gstack', 'deploy-reports');
|
||||
expect(fs.existsSync(reportDir)).toBe(true);
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Land-and-Deploy First-Run E2E ---
|
||||
@@ -148,7 +149,7 @@ Just demonstrate the first-run dry-run output.`,
|
||||
workingDirectory: firstRunDir,
|
||||
maxTurns: 20,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'land-and-deploy-first-run',
|
||||
runId,
|
||||
});
|
||||
@@ -167,7 +168,7 @@ Just demonstrate the first-run dry-run output.`,
|
||||
const reportContent = fs.readFileSync(path.join(reportDir, reportFiles[0]), 'utf-8');
|
||||
const hasPlatform = reportContent.toLowerCase().includes('fly') || reportContent.toLowerCase().includes('first-run-app');
|
||||
expect(hasPlatform).toBe(true);
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Land-and-Deploy Review Gate E2E ---
|
||||
@@ -226,7 +227,7 @@ Show what the readiness gate output would look like.`,
|
||||
workingDirectory: reviewDir,
|
||||
maxTurns: 15,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'land-and-deploy-review-gate',
|
||||
runId,
|
||||
});
|
||||
@@ -246,7 +247,7 @@ Show what the readiness gate output would look like.`,
|
||||
const hasReviewMention = reportContent.toLowerCase().includes('review') ||
|
||||
reportContent.toLowerCase().includes('not run');
|
||||
expect(hasReviewMention).toBe(true);
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Canary skill E2E ---
|
||||
@@ -294,7 +295,7 @@ Just create the directory structure and report files showing the correct schema.
|
||||
workingDirectory: canaryDir,
|
||||
maxTurns: 15,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'canary-workflow',
|
||||
runId,
|
||||
});
|
||||
@@ -307,7 +308,7 @@ Just create the directory structure and report files showing the correct schema.
|
||||
const reportDir = path.join(canaryDir, '.gstack', 'canary-reports');
|
||||
const files = fs.readdirSync(reportDir, { recursive: true }) as string[];
|
||||
expect(files.length).toBeGreaterThan(0);
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Benchmark skill E2E ---
|
||||
@@ -357,7 +358,7 @@ Just create the files showing the correct schema and report format.`,
|
||||
workingDirectory: benchDir,
|
||||
maxTurns: 15,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'benchmark-workflow',
|
||||
runId,
|
||||
});
|
||||
@@ -372,7 +373,7 @@ Just create the files showing the correct schema and report format.`,
|
||||
const files = fs.readdirSync(baselineDir);
|
||||
expect(files.length).toBeGreaterThan(0);
|
||||
}
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// --- Setup-Deploy skill E2E ---
|
||||
@@ -418,7 +419,7 @@ Just detect the platform and write the config.`,
|
||||
workingDirectory: setupDir,
|
||||
maxTurns: 15,
|
||||
allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'],
|
||||
timeout: 120_000,
|
||||
timeout: JUDGE_MS,
|
||||
testName: 'setup-deploy-workflow',
|
||||
runId,
|
||||
});
|
||||
@@ -434,7 +435,7 @@ Just detect the platform and write the config.`,
|
||||
expect(content.toLowerCase()).toContain('fly');
|
||||
expect(content).toContain('my-cool-app');
|
||||
expect(content).toContain('Deploy Configuration');
|
||||
}, 180_000);
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
// Module-level afterAll — finalize eval collector after all tests complete
|
||||
|
||||
Reference in New Issue
Block a user