mirror of
https://github.com/garrytan/gstack.git
synced 2026-05-01 19:25:10 +02:00
feat: CI evals on Ubicloud — 12 parallel runners + Docker image (v0.11.10.0) (#360)
* feat: enable within-file E2E test concurrency for 3x faster runs Switch all E2E tests from serial test() to testConcurrentIfSelected() so tests within each file run in parallel. Wall clock drops from ~18min to ~6min (limited by the longest single test, not sequential sum). The concurrent helper was already built in e2e-helpers.ts but never wired up. Each test runs in its own describe block with its own beforeAll/tmpdir — no shared state conflicts. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * feat: add CI eval workflow on Ubicloud runners Single-job GitHub Actions workflow that runs E2E evals on every PR using Ubicloud runners ($0.006/run — 10x cheaper than GitHub standard). Uses EVALS_CONCURRENCY=40 with the new within-file concurrency for ~6min wall clock. Downloads previous eval artifact from main for comparison, uploads results, and posts a PR comment with pass/fail + cost. Ubicloud setup required: connect GitHub repo via ubicloud.com dashboard, add ANTHROPIC_API_KEY, OPENAI_API_KEY, GEMINI_API_KEY as repo secrets. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * chore: bump version and changelog (v0.11.6.0) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * chore: optimize CI eval PR comment — aggregate all suites, update-not-duplicate Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * feat: parallelize CI evals — 12 runners (1 per suite) for ~3min wall clock Matrix strategy spins up 12 ubicloud-standard-2 runners simultaneously, one per test file. Separate report job aggregates all artifacts into a single PR comment. Bun dependency cache cuts install from ~30s to ~3s. Runner cost: ~$0.048 (from $0.024) — negligible vs $3-4 API costs. Wall clock: ~3-4min (from ~8min). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * feat: add Docker CI image with pre-baked toolchain + deps Dockerfile.ci pre-installs bun, node, claude CLI, gh CLI, and node_modules so eval runners skip all setup. Image rebuilds weekly and on lockfile/Dockerfile changes via ci-image.yml. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * feat: parallelize CI evals — 12 runners (1 per suite) for ~3min wall clock Switch eval workflow to use Docker container image with pre-baked toolchain. Each of 12 matrix runners pulls the image, hardlinks cached node_modules, builds browse, and runs one test suite. Setup drops from ~70s to ~19s per runner. Wall clock is dominated by the slowest individual test, not sequential sum. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * chore: self-bootstrapping CI — build Docker image inline, cache by content hash Move Docker image build into the evals workflow as a dependency job. Image tag is keyed on hash of Dockerfile+lockfile+package.json — only rebuilds when those change. Eliminates chicken-and-egg problem where the image must exist before the first PR run. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: bun.lockb → bun.lock + auth before manifest check This project uses bun.lock (text format), not bun.lockb (binary). Also move Docker login before manifest inspect so GHCR auth works. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: bun.lock is gitignored — use package.json only for Docker cache bun.lock is in .gitignore so it doesn't exist after checkout. Dockerfile and workflows now use package.json only for deps caching. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: symlink node_modules instead of hardlink (cross-device) Docker image layers and workspace are on different filesystems, so cp -al (hardlink) fails. Use ln -s (symlink) instead — zero copy overhead. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * debug: add claude CLI smoke test step to diagnose exit_code_1 Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * ci: retrigger eval workflow * ci: add workflow_dispatch trigger for manual runs * debug: more verbose claude CLI diagnostics * fix: run eval container as non-root — claude CLI rejects --dangerously-skip-permissions as root Claude Code CLI blocks --dangerously-skip-permissions when running as uid=0 for security. Add a 'runner' user to the Docker image and set --user runner on the container. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: install bun to /usr/local so non-root runner user can access it Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: unset CI/GITHUB_ACTIONS env vars for eval runs Claude CLI routing behavior changes when CI=true — it skips skill invocation and uses Bash directly. Unsetting these markers makes Claude behave like a local environment for consistent eval results. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * revert: remove CI env unset — didn't fix routing Unsetting CI/GITHUB_ACTIONS didn't improve routing test results (still 1/11 in container). The issue is model behavior in containerized environments, not env vars. Routing tests will be tracked as a known CI gap. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: copy CLAUDE.md into routing test tmpDirs for skill context In containerized CI, Claude lacks the project context (CLAUDE.md) that guides routing decisions locally. Without it, Claude answers directly with Bash/Agent instead of invoking specific skills. Copying CLAUDE.md gives Claude the same context it has locally. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: routing tests use createRoutingWorkDir with full project context Routing tests now copy CLAUDE.md, README.md, package.json, ETHOS.md, and all SKILL.md files into each test tmpDir. This gives Claude the same project context it has locally, which is needed for correct skill routing decisions in containerized CI environments. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: install skills at top-level .claude/skills/ for CI discovery Claude Code discovers project skills from .claude/skills/<name>/SKILL.md at the top level only. Nesting under .claude/skills/gstack/<name>/ caused Claude to see only one "gstack" skill instead of individual skills like /ship, /qa, /review. This explains 10/11 routing failures in CI — Claude invoked "gstack" or used Bash directly instead of routing to specific skills. Also adds workflow_dispatch trigger and --user runner container option. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * chore: bump version and changelog (v0.11.10.0) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * fix: CI report needs checkout + routing needs user-level skill install Two fixes: 1. Report job: add actions/checkout so `gh pr comment` has git context. Also add pull-requests:write permission for comment posting. 2. Routing tests: install skills to BOTH project-level (.claude/skills/) AND user-level (~/.claude/skills/) since Claude Code discovers from both locations. In CI containers, $HOME differs from workdir. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
@@ -44,7 +44,11 @@ if (evalsEnabled && !process.env.EVALS_ALL) {
|
||||
|
||||
// --- Helper functions ---
|
||||
|
||||
/** Copy all SKILL.md files into tmpDir/.claude/skills/gstack/ for auto-discovery */
|
||||
/** Copy all SKILL.md files for auto-discovery.
|
||||
* Install to BOTH project-level (.claude/skills/) AND user-level (~/.claude/skills/)
|
||||
* because Claude Code discovers skills from both locations. In CI containers,
|
||||
* $HOME may differ from the working directory, so we need both paths to ensure
|
||||
* the Skill tool appears in Claude's available tools list. */
|
||||
function installSkills(tmpDir: string) {
|
||||
const skillDirs = [
|
||||
'', // root gstack SKILL.md
|
||||
@@ -54,15 +58,30 @@ function installSkills(tmpDir: string) {
|
||||
'gstack-upgrade', 'humanizer',
|
||||
];
|
||||
|
||||
// Install to both project-level and user-level skill directories
|
||||
const homeDir = process.env.HOME || os.homedir();
|
||||
const installTargets = [
|
||||
path.join(tmpDir, '.claude', 'skills'), // project-level
|
||||
path.join(homeDir, '.claude', 'skills'), // user-level (~/.claude/skills/)
|
||||
];
|
||||
|
||||
for (const skill of skillDirs) {
|
||||
const srcPath = path.join(ROOT, skill, 'SKILL.md');
|
||||
if (!fs.existsSync(srcPath)) continue;
|
||||
|
||||
const destDir = skill
|
||||
? path.join(tmpDir, '.claude', 'skills', 'gstack', skill)
|
||||
: path.join(tmpDir, '.claude', 'skills', 'gstack');
|
||||
fs.mkdirSync(destDir, { recursive: true });
|
||||
fs.copyFileSync(srcPath, path.join(destDir, 'SKILL.md'));
|
||||
const skillName = skill || 'gstack';
|
||||
|
||||
for (const targetBase of installTargets) {
|
||||
const destDir = path.join(targetBase, skillName);
|
||||
fs.mkdirSync(destDir, { recursive: true });
|
||||
fs.copyFileSync(srcPath, path.join(destDir, 'SKILL.md'));
|
||||
}
|
||||
}
|
||||
|
||||
// Copy CLAUDE.md so Claude has project context for skill routing.
|
||||
const claudeMdSrc = path.join(ROOT, 'CLAUDE.md');
|
||||
if (fs.existsSync(claudeMdSrc)) {
|
||||
fs.copyFileSync(claudeMdSrc, path.join(tmpDir, 'CLAUDE.md'));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,6 +94,31 @@ function initGitRepo(dir: string) {
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a routing test working directory.
|
||||
* Uses the actual repo checkout (ROOT) which has CLAUDE.md, .claude/skills/,
|
||||
* and full project context. This matches the local environment where routing
|
||||
* tests pass reliably. In containerized CI, bare tmpDirs lack the context
|
||||
* Claude needs to make correct routing decisions.
|
||||
*/
|
||||
function createRoutingWorkDir(suffix: string): string {
|
||||
// Clone the repo checkout into a tmpDir so concurrent tests don't interfere
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), `routing-${suffix}-`));
|
||||
// Copy essential context files
|
||||
const filesToCopy = ['CLAUDE.md', 'README.md', 'package.json', 'ETHOS.md'];
|
||||
for (const f of filesToCopy) {
|
||||
const src = path.join(ROOT, f);
|
||||
if (fs.existsSync(src)) fs.copyFileSync(src, path.join(tmpDir, f));
|
||||
}
|
||||
// Copy skill files
|
||||
installSkills(tmpDir);
|
||||
// Init git
|
||||
initGitRepo(tmpDir);
|
||||
spawnSync('git', ['add', '.'], { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
|
||||
spawnSync('git', ['commit', '-m', 'initial'], { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
|
||||
return tmpDir;
|
||||
}
|
||||
|
||||
function logCost(label: string, result: { costEstimate: { turnsUsed: number; estimatedTokens: number; estimatedCost: number }; duration: number }) {
|
||||
const { turnsUsed, estimatedTokens, estimatedCost } = result.costEstimate;
|
||||
const durationSec = Math.round(result.duration / 1000);
|
||||
@@ -104,13 +148,8 @@ describeE2E('Skill Routing E2E — Developer Journey', () => {
|
||||
});
|
||||
|
||||
test.concurrent('journey-ideation', async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-ideation-'));
|
||||
const tmpDir = createRoutingWorkDir('ideation');
|
||||
try {
|
||||
initGitRepo(tmpDir);
|
||||
installSkills(tmpDir);
|
||||
fs.writeFileSync(path.join(tmpDir, 'README.md'), '# New Project\n');
|
||||
spawnSync('git', ['add', '.'], { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
|
||||
spawnSync('git', ['commit', '-m', 'initial'], { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
const testName = 'journey-ideation';
|
||||
const expectedSkill = 'office-hours';
|
||||
@@ -138,10 +177,8 @@ describeE2E('Skill Routing E2E — Developer Journey', () => {
|
||||
}, 150_000);
|
||||
|
||||
test.concurrent('journey-plan-eng', async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-plan-eng-'));
|
||||
const tmpDir = createRoutingWorkDir('plan-eng');
|
||||
try {
|
||||
initGitRepo(tmpDir);
|
||||
installSkills(tmpDir);
|
||||
fs.writeFileSync(path.join(tmpDir, 'plan.md'), `# Waitlist App Architecture
|
||||
|
||||
## Components
|
||||
@@ -190,10 +227,8 @@ describeE2E('Skill Routing E2E — Developer Journey', () => {
|
||||
}, 150_000);
|
||||
|
||||
test.concurrent('journey-think-bigger', async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-think-bigger-'));
|
||||
const tmpDir = createRoutingWorkDir('think-bigger');
|
||||
try {
|
||||
initGitRepo(tmpDir);
|
||||
installSkills(tmpDir);
|
||||
fs.writeFileSync(path.join(tmpDir, 'plan.md'), `# Waitlist App Architecture
|
||||
|
||||
## Components
|
||||
@@ -242,11 +277,8 @@ describeE2E('Skill Routing E2E — Developer Journey', () => {
|
||||
}, 180_000);
|
||||
|
||||
test.concurrent('journey-debug', async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-debug-'));
|
||||
const tmpDir = createRoutingWorkDir('debug');
|
||||
try {
|
||||
initGitRepo(tmpDir);
|
||||
installSkills(tmpDir);
|
||||
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
@@ -302,11 +334,8 @@ export default app;
|
||||
}, 150_000);
|
||||
|
||||
test.concurrent('journey-qa', async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-qa-'));
|
||||
const tmpDir = createRoutingWorkDir('qa');
|
||||
try {
|
||||
initGitRepo(tmpDir);
|
||||
installSkills(tmpDir);
|
||||
|
||||
fs.writeFileSync(path.join(tmpDir, 'package.json'), JSON.stringify({ name: 'waitlist-app', scripts: { dev: 'next dev' } }, null, 2));
|
||||
fs.mkdirSync(path.join(tmpDir, 'src'), { recursive: true });
|
||||
fs.writeFileSync(path.join(tmpDir, 'src/index.html'), '<html><body><h1>Waitlist App</h1></body></html>');
|
||||
@@ -341,17 +370,14 @@ export default app;
|
||||
}, 150_000);
|
||||
|
||||
test.concurrent('journey-code-review', async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-code-review-'));
|
||||
const tmpDir = createRoutingWorkDir('code-review');
|
||||
try {
|
||||
initGitRepo(tmpDir);
|
||||
installSkills(tmpDir);
|
||||
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
fs.writeFileSync(path.join(tmpDir, 'app.ts'), '// base\n');
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'initial']);
|
||||
run('git', ['commit', '-m', 'add base app']);
|
||||
run('git', ['checkout', '-b', 'feature/add-waitlist']);
|
||||
fs.writeFileSync(path.join(tmpDir, 'app.ts'), '// updated with waitlist feature\nimport { WaitlistService } from "./waitlist";\n');
|
||||
fs.writeFileSync(path.join(tmpDir, 'waitlist.ts'), 'export class WaitlistService {\n async addParty(name: string, size: number) {\n // TODO: implement\n }\n}\n');
|
||||
@@ -384,17 +410,14 @@ export default app;
|
||||
}, 150_000);
|
||||
|
||||
test.concurrent('journey-ship', async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-ship-'));
|
||||
const tmpDir = createRoutingWorkDir('ship');
|
||||
try {
|
||||
initGitRepo(tmpDir);
|
||||
installSkills(tmpDir);
|
||||
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
fs.writeFileSync(path.join(tmpDir, 'app.ts'), '// base\n');
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'initial']);
|
||||
run('git', ['commit', '-m', 'add base app']);
|
||||
run('git', ['checkout', '-b', 'feature/waitlist']);
|
||||
fs.writeFileSync(path.join(tmpDir, 'app.ts'), '// waitlist feature\n');
|
||||
run('git', ['add', '.']);
|
||||
@@ -426,11 +449,8 @@ export default app;
|
||||
}, 150_000);
|
||||
|
||||
test.concurrent('journey-docs', async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-docs-'));
|
||||
const tmpDir = createRoutingWorkDir('docs');
|
||||
try {
|
||||
initGitRepo(tmpDir);
|
||||
installSkills(tmpDir);
|
||||
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
@@ -466,11 +486,8 @@ export default app;
|
||||
}, 150_000);
|
||||
|
||||
test.concurrent('journey-retro', async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-retro-'));
|
||||
const tmpDir = createRoutingWorkDir('retro');
|
||||
try {
|
||||
initGitRepo(tmpDir);
|
||||
installSkills(tmpDir);
|
||||
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
@@ -512,17 +529,8 @@ export default app;
|
||||
}, 150_000);
|
||||
|
||||
test.concurrent('journey-design-system', async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-design-system-'));
|
||||
const tmpDir = createRoutingWorkDir('design-system');
|
||||
try {
|
||||
initGitRepo(tmpDir);
|
||||
installSkills(tmpDir);
|
||||
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
fs.writeFileSync(path.join(tmpDir, 'package.json'), JSON.stringify({ name: 'waitlist-app' }, null, 2));
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'initial']);
|
||||
|
||||
const testName = 'journey-design-system';
|
||||
const expectedSkill = 'design-consultation';
|
||||
@@ -550,11 +558,8 @@ export default app;
|
||||
}, 150_000);
|
||||
|
||||
test.concurrent('journey-visual-qa', async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'routing-visual-qa-'));
|
||||
const tmpDir = createRoutingWorkDir('visual-qa');
|
||||
try {
|
||||
initGitRepo(tmpDir);
|
||||
installSkills(tmpDir);
|
||||
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: tmpDir, stdio: 'pipe', timeout: 5000 });
|
||||
|
||||
|
||||
Reference in New Issue
Block a user