From f3151839d862073d856baec8ee4698c7c413814e Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 23 Mar 2026 10:52:46 -0700 Subject: [PATCH] feat: migrate eval storage to project-scoped paths Move eval results and E2E run artifacts from ~/.gstack-dev/evals/ to ~/.gstack/projects/$SLUG/evals/ so each project's eval history lives alongside its other gstack data. Falls back to legacy path if slug detection fails. Co-Authored-By: Claude Opus 4.6 (1M context) --- scripts/eval-compare.ts | 3 ++- scripts/eval-list.ts | 3 ++- scripts/eval-summary.ts | 3 ++- test/helpers/eval-store.ts | 29 +++++++++++++++++++++++++++-- test/helpers/session-runner.ts | 6 ++++-- 5 files changed, 37 insertions(+), 7 deletions(-) diff --git a/scripts/eval-compare.ts b/scripts/eval-compare.ts index 6e2f6a8c..3cb30d5f 100644 --- a/scripts/eval-compare.ts +++ b/scripts/eval-compare.ts @@ -15,10 +15,11 @@ import { findPreviousRun, compareEvalResults, formatComparison, + getProjectEvalDir, } from '../test/helpers/eval-store'; import type { EvalResult } from '../test/helpers/eval-store'; -const EVAL_DIR = path.join(os.homedir(), '.gstack-dev', 'evals'); +const EVAL_DIR = getProjectEvalDir(); function loadResult(filepath: string): EvalResult { // Resolve relative to EVAL_DIR if not absolute diff --git a/scripts/eval-list.ts b/scripts/eval-list.ts index b34e11f0..12c5f0a9 100644 --- a/scripts/eval-list.ts +++ b/scripts/eval-list.ts @@ -8,8 +8,9 @@ import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; +import { getProjectEvalDir } from '../test/helpers/eval-store'; -const EVAL_DIR = path.join(os.homedir(), '.gstack-dev', 'evals'); +const EVAL_DIR = getProjectEvalDir(); // Parse args const args = process.argv.slice(2); diff --git a/scripts/eval-summary.ts b/scripts/eval-summary.ts index 776a0a8d..fba682c2 100644 --- a/scripts/eval-summary.ts +++ b/scripts/eval-summary.ts @@ -9,8 +9,9 @@ import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; import type { EvalResult } from '../test/helpers/eval-store'; +import { getProjectEvalDir } from '../test/helpers/eval-store'; -const EVAL_DIR = path.join(os.homedir(), '.gstack-dev', 'evals'); +const EVAL_DIR = getProjectEvalDir(); let files: string[]; try { diff --git a/test/helpers/eval-store.ts b/test/helpers/eval-store.ts index f2f13fce..ffe82625 100644 --- a/test/helpers/eval-store.ts +++ b/test/helpers/eval-store.ts @@ -2,7 +2,7 @@ * Eval result persistence and comparison. * * EvalCollector accumulates test results, writes them to - * ~/.gstack-dev/evals/{version}-{branch}-{tier}-{timestamp}.json, + * ~/.gstack/projects/$SLUG/evals/{version}-{branch}-{tier}-{timestamp}.json, * prints a summary table, and auto-compares with the previous run. * * Comparison functions are exported for reuse by the eval:compare CLI. @@ -14,7 +14,32 @@ import * as os from 'os'; import { spawnSync } from 'child_process'; const SCHEMA_VERSION = 1; -const DEFAULT_EVAL_DIR = path.join(os.homedir(), '.gstack-dev', 'evals'); +const LEGACY_EVAL_DIR = path.join(os.homedir(), '.gstack-dev', 'evals'); + +/** + * Detect project-scoped eval dir via gstack-slug. + * Falls back to legacy ~/.gstack-dev/evals/ if slug detection fails. + */ +export function getProjectEvalDir(): string { + try { + // Try repo-local gstack-slug first, then global install + const localSlug = spawnSync('bash', ['-c', '.claude/skills/gstack/bin/gstack-slug 2>/dev/null || ~/.claude/skills/gstack/bin/gstack-slug 2>/dev/null'], { + stdio: 'pipe', timeout: 3000, + }); + const output = localSlug.stdout?.toString().trim(); + if (output) { + const slugMatch = output.match(/^SLUG=(.+)$/m); + if (slugMatch && slugMatch[1]) { + const dir = path.join(os.homedir(), '.gstack', 'projects', slugMatch[1], 'evals'); + fs.mkdirSync(dir, { recursive: true }); + return dir; + } + } + } catch { /* fall through */ } + return LEGACY_EVAL_DIR; +} + +const DEFAULT_EVAL_DIR = getProjectEvalDir(); // --- Interfaces --- diff --git a/test/helpers/session-runner.ts b/test/helpers/session-runner.ts index ab9e2ee5..7101e30c 100644 --- a/test/helpers/session-runner.ts +++ b/test/helpers/session-runner.ts @@ -9,9 +9,11 @@ import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; +import { getProjectEvalDir } from './eval-store'; const GSTACK_DEV_DIR = path.join(os.homedir(), '.gstack-dev'); -const HEARTBEAT_PATH = path.join(GSTACK_DEV_DIR, 'e2e-live.json'); +const HEARTBEAT_PATH = path.join(GSTACK_DEV_DIR, 'e2e-live.json'); // heartbeat stays global +const PROJECT_DIR = path.dirname(getProjectEvalDir()); // ~/.gstack/projects/$SLUG/ /** Sanitize test name for use as filename: strip leading slashes, replace / with - */ export function sanitizeTestName(name: string): string { @@ -144,7 +146,7 @@ export async function runSkillTest(options: { const safeName = testName ? sanitizeTestName(testName) : null; if (runId) { try { - runDir = path.join(GSTACK_DEV_DIR, 'e2e-runs', runId); + runDir = path.join(PROJECT_DIR, 'e2e-runs', runId); fs.mkdirSync(runDir, { recursive: true }); } catch { /* non-fatal */ } }