CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-09-09 14:38:59 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix: skill-e2e-opus-47 renders SKILL.md fixtures into a mkdtemp — never the live tree

Browse Source 
mkEvalRoot ran gen-skill-docs with cwd=ROOT, regenerating every in-repo
SKILL.md mid-run while concurrent paid shards copyFileSync those same files
in their beforeAll (EVALS_JOBS>=4 locally, 2 per CI slice) — a sibling
could capture a half-regenerated or opus-rendered SKILL.md, and a timeout
before afterAll stranded the whole tree at the wrong model for every later
shard. A cross-shard race that could flake ANY concurrent paid test.

Render via the --out-dir flag gen-skill-docs grew for exactly this reason
(mirrors the repo layout, which is all the fixture reads), read the skill
heads from the render dir, delete it, and drop the afterAll restore-regen
entirely.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-31 04:21:00 +00:00
co-authored by Claude Fable 5
parent 2e53b18670
commit cfcc9ecc6f
1 changed files with 16 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/skill-e2e-opus-47.test.ts
+16 -19
View File
@@ -61,15 +61,18 @@ const INSTALLED_SKILLS = [
function mkEvalRoot(suffix: string, includeOverlay: boolean): string {
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), `opus47-${suffix}-`));
// Regenerate at opus-4-7 so the per-skill SKILL.md files reflect that
// model's overlay. If includeOverlay is false we'll re-regen at default
// later just for the root SKILL.md copy. For individual skills, opus-4-7
// content doesn't matter for the routing test (we only need discovery).
// Render at opus-4-7 INTO A MKDTEMP via --out-dir — never the live tree.
// The previous cwd=ROOT regeneration rewrote every in-repo SKILL.md
// mid-run while concurrent paid shards copyFileSync those same files in
// their beforeAll (EVALS_JOBS>=4 locally, 2 per CI slice): a sibling could
// capture a half-regenerated or opus-rendered SKILL.md, and a timeout
// before afterAll left the whole tree rendered at the wrong model for
// every later shard. --out-dir mirrors the repo layout (<out>/<skill>/
// SKILL.md), which is all this fixture reads.
const renderDir = fs.mkdtempSync(path.join(os.tmpdir(), `opus47-render-${suffix}-`));
const result = spawnSync(
'bun',
['run', 'scripts/gen-skill-docs.ts', '--model', includeOverlay ? 'opus-4-7' : 'claude'],
// LIVE-REPO CWD: gen-skill-docs reads .tmpl sources and regenerates the
// in-repo SKILL.md files (restored to default in afterAll below).
['run', 'scripts/gen-skill-docs.ts', '--model', includeOverlay ? 'opus-4-7' : 'claude', '--out-dir', renderDir],
{ cwd: ROOT, stdio: 'pipe', encoding: 'utf-8', timeout: 60_000 },
);
if (result.status !== 0) {
@@ -82,12 +85,13 @@ function mkEvalRoot(suffix: string, includeOverlay: boolean): string {
// (CLAUDE.md: "E2E test fixtures: extract, don't copy").
const skillsDir = path.join(tmp, '.claude', 'skills');
for (const skill of INSTALLED_SKILLS) {
const src = path.join(ROOT, skill, 'SKILL.md');
const src = path.join(renderDir, skill, 'SKILL.md');
if (!fs.existsSync(src)) continue;
const destDir = path.join(skillsDir, skill);
fs.mkdirSync(destDir, { recursive: true });
fs.writeFileSync(path.join(destDir, 'SKILL.md'), extractSkillHead(src));
}
fs.rmSync(renderDir, { recursive: true, force: true });
// Extract the opus-4-7 model-overlay content from the checked-in file
// so we can inline it into CLAUDE.md when includeOverlay is true.
@@ -167,17 +171,10 @@ const ROUTING_CASES: RoutingCase[] = [
describeE2E('Opus 4.7 overlay behavior evals', () => {
afterAll(() => {
evalCollector?.finalize();
// Restore working tree: mkEvalRoot runs `gen-skill-docs` with various
// --model flags, leaving the in-repo SKILL.md files generated at
// whichever model ran last. Reset to the default (claude) so the tree
// matches what would be checked in.
spawnSync('bun', ['run', 'scripts/gen-skill-docs.ts'], {
// LIVE-REPO CWD: restores the in-repo SKILL.md files to the default
// model render after mkEvalRoot's --model regens.
cwd: ROOT,
stdio: 'pipe',
timeout: 60_000,
});
// No tree restore needed: mkEvalRoot renders into a mkdtemp via
// --out-dir, so the live repo's SKILL.md files are never touched — a
// timeout mid-run can no longer strand the tree at the wrong model for
// concurrent shards.
});
test(