From 9eaf15564c12af4630eaed36a870c57b334f5e3c Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 29 Aug 2026 04:49:33 +0000 Subject: [PATCH] fix(test): the two expect(true) paid stubs become test.todo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit skill-e2e-spec-execute (600s budget) and skill-llm-eval-spec (300s) reported PASS on every periodic run while asserting nothing. Deleting them would remove the periodic-tier selector surface they exist to register (diff-based selection for spec/ changes), so they become test.todo — reported as todo/skip, never pass — with the v1.1 implementation specs kept in-file. Co-Authored-By: Claude Fable 5 --- test/skill-e2e-spec-execute.test.ts | 31 ++++++++-------------- test/skill-llm-eval-spec.test.ts | 40 ++++++++++------------------- 2 files changed, 24 insertions(+), 47 deletions(-) diff --git a/test/skill-e2e-spec-execute.test.ts b/test/skill-e2e-spec-execute.test.ts index 4d99c957c..787b91c72 100644 --- a/test/skill-e2e-spec-execute.test.ts +++ b/test/skill-e2e-spec-execute.test.ts @@ -16,30 +16,19 @@ * minimum smoke that proves --execute end-to-end works. */ -import { test, expect } from 'bun:test'; +import { test } from 'bun:test'; import { describeE2ETier } from './helpers/e2e-gate'; -import * as fs from 'fs'; -import * as path from 'path'; const describeE2E = describeE2ETier('periodic'); -const ROOT = path.resolve(import.meta.dir, '..'); - describeE2E('/spec --execute end-to-end (periodic)', () => { - test('phase gating + magical Phase 3 + quality gate + spawn — full pipeline', async () => { - // Sanity: spec template + generated SKILL.md exist at expected paths. - expect(fs.existsSync(path.join(ROOT, 'spec', 'SKILL.md.tmpl'))).toBe(true); - expect(fs.existsSync(path.join(ROOT, 'spec', 'SKILL.md'))).toBe(true); - - // Full PTY-driven E2E lives in a follow-up. For now this test exists as - // the periodic-tier surface registered in E2E_TIERS so the diff-based - // selector knows to run it when spec/ changes. The deterministic - // template-invariant coverage in spec-template-invariants.test.ts + - // spec-template-sync.test.ts gates the gate tier; this stub is the - // periodic-tier hook for the full claude-pty-runner driven test. - - // Mark as pending — replace with full PTY driver in follow-up TODO: - // "/spec --execute E2E full pipeline test (v1.1)" - expect(true).toBe(true); - }, 600_000); + // test.todo, not expect(true): the placeholder reported PASS on every + // periodic run while asserting nothing — a lying green with a 600s budget. + // The file itself stays: it is the periodic-tier surface registered in + // E2E_TIERS so the diff-based selector runs it when spec/ changes, and + // the deterministic template-invariant coverage in + // spec-template-invariants.test.ts + spec-template-sync.test.ts gates the + // gate tier. Implementation spec for the real PTY-driven test lives in + // the header TODO ("/spec --execute E2E full pipeline test (v1.1)"). + test.todo('phase gating + magical Phase 3 + quality gate + spawn — full pipeline'); }); diff --git a/test/skill-llm-eval-spec.test.ts b/test/skill-llm-eval-spec.test.ts index 1ab6183be..87922f365 100644 --- a/test/skill-llm-eval-spec.test.ts +++ b/test/skill-llm-eval-spec.test.ts @@ -13,35 +13,23 @@ * Phase 3 fallback path). */ -import { describe, test, expect } from 'bun:test'; -import * as fs from 'fs'; -import * as path from 'path'; +import { describe, test } from 'bun:test'; const evalsEnabled = !!process.env.EVALS; const describeEval = evalsEnabled ? describe : describe.skip; -const ROOT = path.resolve(import.meta.dir, '..'); - describeEval('/spec LLM-judge eval (periodic)', () => { - test('spec body scores >= 8/10 against 14-standard rubric on fixture request', async () => { - // Sanity: required files exist for the eval. - expect(fs.existsSync(path.join(ROOT, 'spec', 'SKILL.md.tmpl'))).toBe(true); - - // Full LLM-judge run lives in a follow-up. This file registers the - // periodic-tier surface so the diff-based selector picks it up when - // spec/ changes. Deterministic invariants are gate-tier; the LLM-judge - // is for measuring authored-spec quality, which is non-deterministic - // by nature. - // - // Expected v1.1 implementation: - // 1. Pick fixture prompt from test/fixtures/spec/vague-bug.md - // 2. Spawn `claude -p` with /spec loaded, send the prompt + role-play - // five Phase 1 answers (from test/fixtures/spec/vague-bug-answers.json) - // 3. Capture final spec body - // 4. Dispatch to Claude judge with prompt encoding the 14 Quality - // Standards from spec/SKILL.md.tmpl - // 5. Assert numeric score >= 8 - - expect(true).toBe(true); - }, 300_000); + // test.todo, not expect(true): the placeholder reported PASS on every + // run while asserting nothing — a lying green with a 300s budget. The + // file stays as the periodic-tier selector surface for spec/ changes. + // + // Expected v1.1 implementation: + // 1. Pick fixture prompt from test/fixtures/spec/vague-bug.md + // 2. Spawn `claude -p` with /spec loaded, send the prompt + role-play + // five Phase 1 answers (from test/fixtures/spec/vague-bug-answers.json) + // 3. Capture final spec body + // 4. Dispatch to Claude judge with prompt encoding the 14 Quality + // Standards from spec/SKILL.md.tmpl + // 5. Assert numeric score >= 8 + test.todo('spec body scores >= 8/10 against 14-standard rubric on fixture request'); });