CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-05-09 06:45:46 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: worktree isolation for E2E tests + infrastructure elegance (v0.11.12.0) (#425)

Browse Source 
* refactor: extract gen-skill-docs into modular resolver architecture

Break the 3000-line monolith into 10 domain modules under scripts/resolvers/:
types, constants, preamble, utility, browse, design, testing, review,
codex-helpers, and index. Each module owns one domain of template generation.

The preamble module introduces a 4-tier composition system (T1-T4) so skills
only pay for the preamble sections they actually need, reducing token usage
for lightweight skills by ~40%.

Adds a token budget dashboard that prints after every generation run showing
per-skill and total token counts.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* feat: tiered preamble — skills only pay for what they use

Tag all 23 templates with preamble-tier (T1-T4). Lightweight skills
like /browse and /benchmark get a minimal preamble (~40% fewer tokens),
while review skills get the full stack. Regenerate all SKILL.md files.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* feat: migrate eval storage to project-scoped paths

Move eval results and E2E run artifacts from ~/.gstack-dev/evals/ to
~/.gstack/projects/$SLUG/evals/ so each project's eval history lives
alongside its other gstack data. Falls back to legacy path if slug
detection fails.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* fix: sync package.json version with VERSION after merge

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* feat: add WorktreeManager for isolated test environments

Reusable platform module (lib/worktree.ts) that creates git worktrees
for test isolation and harvests useful changes as patches. Includes
SHA-256 dedup, original SHA tracking for committed change detection,
and automatic gitignored artifact copying (.agents/, browse/dist/).

12 unit tests covering lifecycle, harvest, dedup, and error handling.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* feat: integrate worktree isolation into E2E test infrastructure

Add createTestWorktree(), harvestAndCleanup(), and describeWithWorktree()
helpers to e2e-helpers.ts. Add harvest field to EvalTestEntry for
eval-store integration. Register lib/worktree.ts as a global touchfile.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* feat: run Gemini and Codex E2E tests in worktrees

Switch both test suites from cwd: ROOT to worktree isolation.
Gemini (--yolo) no longer pollutes the working tree. Codex
(read-only) gets worktree for consistency. Useful changes are
harvested as patches for cherry-picking.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* fix: skip symlinks in copyDirSync to prevent infinite recursion

Adversarial review caught that .claude/skills/gstack may be a symlink
back to the repo root, causing copyDirSync to recurse infinitely
when copying gitignored artifacts into worktrees.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* chore: bump version and changelog (v0.11.12.0)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix: relax session-awareness assertion to accept structured options

The LLM consistently presents well-formatted A/B choices with pros/cons
but doesn't always use the exact string "RECOMMENDATION". Accept
case-insensitive "recommend", "option a", "which do you want", or
"which approach" as equivalent signals of a structured recommendation.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-03-23 23:05:22 -07:00
committed by GitHub
parent 2c5ae38542
commit dc5e0538e5
79 changed files with 4217 additions and 4681 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/helpers/e2e-helpers.ts
+56 -1
View File
@@ -5,11 +5,13 @@
* tests across multiple files by category.
*/
import { describe, test, afterAll } from 'bun:test';
import { describe, test, beforeAll, afterAll } from 'bun:test';
import type { SkillTestResult } from './session-runner';
import { EvalCollector, judgePassed } from './eval-store';
import type { EvalTestEntry } from './eval-store';
import { selectTests, detectBaseBranch, getChangedFiles, E2E_TOUCHFILES, GLOBAL_TOUCHFILES } from './touchfiles';
import { WorktreeManager } from '../../lib/worktree';
import type { HarvestResult } from '../../lib/worktree';
import { spawnSync } from 'child_process';
import * as fs from 'fs';
import * as path from 'path';
@@ -234,6 +236,59 @@ export function testConcurrentIfSelected(testName: string, fn: () => Promise<voi
(shouldRun ? test.concurrent : test.skip)(testName, fn, timeout);
}
// --- Worktree isolation ---
let worktreeManager: WorktreeManager | null = null;
export function getWorktreeManager(): WorktreeManager {
if (!worktreeManager) {
worktreeManager = new WorktreeManager();
worktreeManager.pruneStale();
}
return worktreeManager;
}
/** Create an isolated worktree for a test. Returns the worktree path. */
export function createTestWorktree(testName: string): string {
return getWorktreeManager().create(testName);
}
/** Harvest changes and clean up. Call in afterAll(). Returns HarvestResult for eval integration. */
export function harvestAndCleanup(testName: string): HarvestResult | null {
const mgr = getWorktreeManager();
const result = mgr.harvest(testName);
if (result) {
if (result.isDuplicate) {
process.stderr.write(`\n HARVEST [${testName}]: duplicate patch (skipped)\n`);
} else {
process.stderr.write(`\n HARVEST [${testName}]: ${result.changedFiles.length} files changed\n`);
process.stderr.write(` Patch: ${result.patchPath}\n`);
process.stderr.write(` ${result.diffStat}\n\n`);
}
}
mgr.cleanup(testName);
return result;
}
/**
* Convenience: describe block with automatic worktree isolation + harvest.
* Any test file can use this to get real repo context instead of a tmpdir.
* Note: tests with planted-bug fixtures should NOT use this — they need their fixture repos.
*/
export function describeWithWorktree(
name: string,
testNames: string[],
fn: (getWorktreePath: () => string) => void,
) {
describeIfSelected(name, testNames, () => {
let worktreePath: string;
beforeAll(() => { worktreePath = createTestWorktree(name); });
afterAll(() => { harvestAndCleanup(name); });
fn(() => worktreePath);
});
}
export { judgePassed } from './eval-store';
export { EvalCollector } from './eval-store';
export type { EvalTestEntry } from './eval-store';
export type { HarvestResult } from '../../lib/worktree';
test/helpers/eval-store.ts
+34 -2
View File
@@ -2,7 +2,7 @@
* Eval result persistence and comparison.
*
* EvalCollector accumulates test results, writes them to
* ~/.gstack-dev/evals/{version}-{branch}-{tier}-{timestamp}.json,
* ~/.gstack/projects/$SLUG/evals/{version}-{branch}-{tier}-{timestamp}.json,
* prints a summary table, and auto-compares with the previous run.
*
* Comparison functions are exported for reuse by the eval:compare CLI.
@@ -14,7 +14,32 @@ import * as os from 'os';
import { spawnSync } from 'child_process';
const SCHEMA_VERSION = 1;
const DEFAULT_EVAL_DIR = path.join(os.homedir(), '.gstack-dev', 'evals');
const LEGACY_EVAL_DIR = path.join(os.homedir(), '.gstack-dev', 'evals');
/**
* Detect project-scoped eval dir via gstack-slug.
* Falls back to legacy ~/.gstack-dev/evals/ if slug detection fails.
*/
export function getProjectEvalDir(): string {
try {
// Try repo-local gstack-slug first, then global install
const localSlug = spawnSync('bash', ['-c', '.claude/skills/gstack/bin/gstack-slug 2>/dev/null || ~/.claude/skills/gstack/bin/gstack-slug 2>/dev/null'], {
stdio: 'pipe', timeout: 3000,
});
const output = localSlug.stdout?.toString().trim();
if (output) {
const slugMatch = output.match(/^SLUG=(.+)$/m);
if (slugMatch && slugMatch[1]) {
const dir = path.join(os.homedir(), '.gstack', 'projects', slugMatch[1], 'evals');
fs.mkdirSync(dir, { recursive: true });
return dir;
}
}
} catch { /* fall through */ }
return LEGACY_EVAL_DIR;
}
const DEFAULT_EVAL_DIR = getProjectEvalDir();
// --- Interfaces ---
@@ -55,6 +80,13 @@ export interface EvalTestEntry {
missed_bugs?: string[];
error?: string;
// Worktree harvest data
harvest?: {
filesChanged: number;
patchPath: string;
isDuplicate: boolean;
};
}
export interface EvalResult {
test/helpers/session-runner.ts
+4 -2
View File
@@ -9,9 +9,11 @@
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { getProjectEvalDir } from './eval-store';
const GSTACK_DEV_DIR = path.join(os.homedir(), '.gstack-dev');
const HEARTBEAT_PATH = path.join(GSTACK_DEV_DIR, 'e2e-live.json');
const HEARTBEAT_PATH = path.join(GSTACK_DEV_DIR, 'e2e-live.json'); // heartbeat stays global
const PROJECT_DIR = path.dirname(getProjectEvalDir()); // ~/.gstack/projects/$SLUG/
/** Sanitize test name for use as filename: strip leading slashes, replace / with - */
export function sanitizeTestName(name: string): string {
@@ -144,7 +146,7 @@ export async function runSkillTest(options: {
const safeName = testName ? sanitizeTestName(testName) : null;
if (runId) {
try {
runDir = path.join(GSTACK_DEV_DIR, 'e2e-runs', runId);
runDir = path.join(PROJECT_DIR, 'e2e-runs', runId);
fs.mkdirSync(runDir, { recursive: true });
} catch { /* non-fatal */ }
}
test/helpers/touchfiles.ts
+1
View File
@@ -200,6 +200,7 @@ export const GLOBAL_TOUCHFILES = [
'scripts/gen-skill-docs.ts',
'test/helpers/touchfiles.ts',
'browse/test/test-server.ts',
'lib/worktree.ts',
];
// --- Base branch detection ---