diff --git a/scripts/paid-test-durations.json b/scripts/paid-test-durations.json index 58e175ff5..bbc63ae1d 100644 --- a/scripts/paid-test-durations.json +++ b/scripts/paid-test-durations.json @@ -18,7 +18,6 @@ "test/skill-e2e-design.test.ts#plan-design-review-no-ui-scope": 24431, "test/skill-e2e-diagram.test.ts": 29998, "test/skill-e2e-docsync-spawned.test.ts": 137929, - "test/skill-e2e-first-task-scaffold.test.ts": 1000, "test/skill-e2e-gbrain-roundtrip-local.test.ts": 1000, "test/skill-e2e-hermetic-canary.test.ts": 8813, "test/skill-e2e-investigate-owned-completion.test.ts": 45178, @@ -143,7 +142,6 @@ "test/skill-e2e-design.test.ts#design-html-slop-gate": 158970, "test/skill-e2e-design.test.ts#plan-design-review-plan-mode": 300166, "test/skill-e2e-diagram.test.ts": 55187, - "test/skill-e2e-first-task-scaffold.test.ts": 8563, "test/skill-e2e-gbrain-roundtrip-local.test.ts": 1000, "test/skill-e2e-health.test.ts": 188584, "test/skill-e2e-hermetic-canary.test.ts": 1000, diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index a7e5ccebe..ea7fc1b12 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -90,7 +90,6 @@ export const E2E_TOUCHFILES: Record = { // through the real runner, plus the script wiring that gates + maps it // (token-reduction Phase 2: generate-first-run-guidance.ts was deleted; the // gate + token→tip map live in bin/gstack-skill-start's emission layer). - 'first-task-scaffold': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-first-task-detect', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'test/skill-e2e-first-task-scaffold.test.ts', 'test/helpers/session-runner.ts'], // SKILL.md setup + preamble (depend on ROOT SKILL.md + gen-skill-docs) 'skillmd-setup-discovery': [ 'SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-bws.test.ts'], @@ -1155,7 +1154,6 @@ export const E2E_TIERS: Record = { 'operational-learning': 'gate', // P4 first-run scaffold — periodic (onboarding, non-safety, model-touched marker) - 'first-task-scaffold': 'periodic', // QA — gate for functional, periodic for quality/benchmarks 'qa-quick': 'gate', @@ -1617,7 +1615,6 @@ export const E2E_KINDS: Record = { 'skillmd-outside-git': 'rule', 'session-awareness': 'rule', 'operational-learning': 'rule', - 'first-task-scaffold': 'rule', 'qa-quick': 'rule', 'qa-b6-static': 'rule', 'qa-b7-spa': 'rule', diff --git a/test/skill-e2e-first-task-scaffold.test.ts b/test/skill-e2e-first-task-scaffold.test.ts deleted file mode 100644 index ccb4df40d..000000000 --- a/test/skill-e2e-first-task-scaffold.test.ts +++ /dev/null @@ -1,98 +0,0 @@ -/** - * P4 first-run scaffold — E2E (periodic tier, ~$0.02 each, deterministic). - * - * Exercises bin/gstack-first-task-detect END-TO-END through the real runner + - * hermetic env (path resolution, execution, git-in-cwd), not just the unit - * harness. Deterministic by construction: it asserts the binary's enum token - * from the Bash tool_result in the stream-json transcript (never the model's - * prose), so it pins the detector's integration contract without depending on - * non-deterministic model phrasing. - * - * Periodic (not gate): onboarding behavior is non-safety, and the scaffold - * marker is model-touched (best-effort). The deterministic bucket logic itself - * is fully covered by the unit test (test/preamble-first-task-scaffold.test.ts). - */ - -import { expect, afterAll } from 'bun:test'; -import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets'; -import * as fs from 'fs'; -import * as path from 'path'; -import * as os from 'os'; -import { execSync } from 'node:child_process'; -import { runSkillTest } from './helpers/session-runner'; -import { - describeIfSelected, testIfSelected, createEvalCollector, finalizeEvalCollector, - recordE2E, runId, logCost, -} from './helpers/e2e-helpers'; - -const ROOT = path.join(import.meta.dir, '..'); -const DETECT = path.join(ROOT, 'bin', 'gstack-first-task-detect'); -const evalCollector = createEvalCollector('e2e-first-task-scaffold'); -const MODEL = 'claude-haiku-4-5-20251001'; - -const GIT_ENV = { - ...process.env, - GIT_AUTHOR_NAME: 'T', GIT_AUTHOR_EMAIL: 't@e.x', - GIT_COMMITTER_NAME: 'T', GIT_COMMITTER_EMAIL: 't@e.x', -}; - -/** Concatenated Bash tool_result text from the stream-json transcript. */ -function toolResultText(transcript: any[]): string { - const chunks: string[] = []; - for (const event of transcript) { - if (event.type !== 'user') continue; - for (const item of event.message?.content ?? []) { - if (item.type !== 'tool_result') continue; - if (typeof item.content === 'string') chunks.push(item.content); - else for (const c of item.content ?? []) if (c.type === 'text') chunks.push(c.text); - } - } - return chunks.join('\n'); -} - -async function detectVia(workDir: string, testName: string): Promise { - const result = await runSkillTest({ - prompt: `Run exactly this one bash command and then stop, printing its output verbatim: ${DETECT}`, - workingDirectory: workDir, - maxTurns: 3, - allowedTools: ['Bash'], - timeout: JUDGE_MS, - testName, - runId, - model: MODEL, - }); - logCost(testName, result); - recordE2E(evalCollector, testName, 'e2e-first-task-scaffold', result); - expect(result.exitReason).toBe('success'); - return toolResultText(result.transcript); -} - -describeIfSelected('first-run scaffold detection (E2E)', ['first-task-scaffold'], () => { - testIfSelected('first-task-scaffold', async () => { - if (!process.env.ANTHROPIC_API_KEY) { - throw new Error('first-task-scaffold requires ANTHROPIC_API_KEY (source ~/.zshrc); refusing to skip'); - } - - // code_node bucket: package.json + a commit, on the default branch. - const nodeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'fts-node-')); - // greenfield bucket: git repo, zero commits. - const greenDir = fs.mkdtempSync(path.join(os.tmpdir(), 'fts-green-')); - try { - execSync('git init -q -b main', { cwd: nodeDir, env: GIT_ENV, timeout: 30_000 }); - fs.writeFileSync(path.join(nodeDir, 'package.json'), '{"name":"x"}'); - execSync('git add -A && git commit -qm init', { cwd: nodeDir, env: GIT_ENV, timeout: 30_000 }); - execSync('git init -q -b main', { cwd: greenDir, env: GIT_ENV, timeout: 30_000 }); - - const nodeOut = await detectVia(nodeDir, 'first-task-scaffold'); - expect(nodeOut).toContain('code_node'); - - const greenOut = await detectVia(greenDir, 'first-task-scaffold-greenfield'); - expect(greenOut).toContain('greenfield'); - } finally { - fs.rmSync(nodeDir, { recursive: true, force: true }); - fs.rmSync(greenDir, { recursive: true, force: true }); - } - }, CAPTURE_MS); -}); - -afterAll(() => finalizeEvalCollector(evalCollector));