CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-09-01 19:00:40 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix(test): the two expect(true) paid stubs become test.todo

Browse Source 
skill-e2e-spec-execute (600s budget) and skill-llm-eval-spec (300s)
reported PASS on every periodic run while asserting nothing. Deleting
them would remove the periodic-tier selector surface they exist to
register (diff-based selection for spec/ changes), so they become
test.todo — reported as todo/skip, never pass — with the v1.1
implementation specs kept in-file.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-29 04:49:33 +00:00
co-authored by Claude Fable 5
parent e9643e131f
commit 9eaf15564c
2 changed files with 24 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/skill-e2e-spec-execute.test.ts
+10 -21
View File
@@ -16,30 +16,19 @@
* minimum smoke that proves --execute end-to-end works. * minimum smoke that proves --execute end-to-end works.
*/ */
import { test, expect } from 'bun:test'; import { test } from 'bun:test';
import { describeE2ETier } from './helpers/e2e-gate'; import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'fs';
import * as path from 'path';
const describeE2E = describeE2ETier('periodic'); const describeE2E = describeE2ETier('periodic');
const ROOT = path.resolve(import.meta.dir, '..');
describeE2E('/spec --execute end-to-end (periodic)', () => { describeE2E('/spec --execute end-to-end (periodic)', () => {
test('phase gating + magical Phase 3 + quality gate + spawn — full pipeline', async () => { // test.todo, not expect(true): the placeholder reported PASS on every
// Sanity: spec template + generated SKILL.md exist at expected paths. // periodic run while asserting nothing — a lying green with a 600s budget.
expect(fs.existsSync(path.join(ROOT, 'spec', 'SKILL.md.tmpl'))).toBe(true); // The file itself stays: it is the periodic-tier surface registered in
expect(fs.existsSync(path.join(ROOT, 'spec', 'SKILL.md'))).toBe(true); // E2E_TIERS so the diff-based selector runs it when spec/ changes, and
// the deterministic template-invariant coverage in
// Full PTY-driven E2E lives in a follow-up. For now this test exists as // spec-template-invariants.test.ts + spec-template-sync.test.ts gates the
// the periodic-tier surface registered in E2E_TIERS so the diff-based // gate tier. Implementation spec for the real PTY-driven test lives in
// selector knows to run it when spec/ changes. The deterministic // the header TODO ("/spec --execute E2E full pipeline test (v1.1)").
// template-invariant coverage in spec-template-invariants.test.ts + test.todo('phase gating + magical Phase 3 + quality gate + spawn — full pipeline');
// spec-template-sync.test.ts gates the gate tier; this stub is the
// periodic-tier hook for the full claude-pty-runner driven test.
// Mark as pending — replace with full PTY driver in follow-up TODO:
// "/spec --execute E2E full pipeline test (v1.1)"
expect(true).toBe(true);
}, 600_000);
}); });
test/skill-llm-eval-spec.test.ts
+14 -26
View File
@@ -13,35 +13,23 @@
* Phase 3 fallback path). * Phase 3 fallback path).
*/ */
import { describe, test, expect } from 'bun:test'; import { describe, test } from 'bun:test';
import * as fs from 'fs';
import * as path from 'path';
const evalsEnabled = !!process.env.EVALS; const evalsEnabled = !!process.env.EVALS;
const describeEval = evalsEnabled ? describe : describe.skip; const describeEval = evalsEnabled ? describe : describe.skip;
const ROOT = path.resolve(import.meta.dir, '..');
describeEval('/spec LLM-judge eval (periodic)', () => { describeEval('/spec LLM-judge eval (periodic)', () => {
test('spec body scores >= 8/10 against 14-standard rubric on fixture request', async () => { // test.todo, not expect(true): the placeholder reported PASS on every
// Sanity: required files exist for the eval. // run while asserting nothing — a lying green with a 300s budget. The
expect(fs.existsSync(path.join(ROOT, 'spec', 'SKILL.md.tmpl'))).toBe(true); // file stays as the periodic-tier selector surface for spec/ changes.
//
// Full LLM-judge run lives in a follow-up. This file registers the // Expected v1.1 implementation:
// periodic-tier surface so the diff-based selector picks it up when // 1. Pick fixture prompt from test/fixtures/spec/vague-bug.md
// spec/ changes. Deterministic invariants are gate-tier; the LLM-judge // 2. Spawn `claude -p` with /spec loaded, send the prompt + role-play
// is for measuring authored-spec quality, which is non-deterministic // five Phase 1 answers (from test/fixtures/spec/vague-bug-answers.json)
// by nature. // 3. Capture final spec body
// // 4. Dispatch to Claude judge with prompt encoding the 14 Quality
// Expected v1.1 implementation: // Standards from spec/SKILL.md.tmpl
// 1. Pick fixture prompt from test/fixtures/spec/vague-bug.md // 5. Assert numeric score >= 8
// 2. Spawn `claude -p` with /spec loaded, send the prompt + role-play test.todo('spec body scores >= 8/10 against 14-standard rubric on fixture request');
// five Phase 1 answers (from test/fixtures/spec/vague-bug-answers.json)
// 3. Capture final spec body
// 4. Dispatch to Claude judge with prompt encoding the 14 Quality
// Standards from spec/SKILL.md.tmpl
// 5. Assert numeric score >= 8
expect(true).toBe(true);
}, 300_000);
}); });