test(W7h): retire the paid first-task-scaffold E2E duplicate

W7h retirement (t12 F12). Removes the case from test/helpers/touchfiles-data.ts
(E2E_TOUCHFILES, E2E_TIERS periodic, E2E_KINDS rule) and both
scripts/paid-test-durations.json entries; test/helpers/periodic-exclude-data.ts
had no entry. e2e-tier-alignment, eval-kinds, eval-detach-timeout-floor,
periodic-exclude-policy, touchfiles and paid-shards tests stay green.

Retirement card:
- test: test/skill-e2e-first-task-scaffold.test.ts ('first-task-scaffold',
  periodic, ~$0.02): Haiku is told to run exactly one detector command and the
  test greps the tool result for code_node / greenfield.
- detects: detector bucket regressions in bin/gstack-first-task-detect.
- non_test_callers: none.
- search_command: grep -rn "first-task-scaffold" test scripts .github
- stronger_proof / replacement: test/preamble-first-task-scaffold.test.ts
  (free; executes the real detector on fixture repos for every bucket,
  including code_node and greenfield, plus first-run gating and telemetry in
  bin/gstack-skill-start). The model added nothing: the prompt dictated the
  exact command.
- history: c118e24 v1.64.1.0.
- validation: bun test test/preamble-first-task-scaffold.test.ts plus the
  registration tests above.

Control (scratch worktree, against the replacement): making the detector emit
the wrong bucket for a Node project or a commit-less repo turns
preamble-first-task-scaffold.test.ts red (2/2). Log:
.context/mutation-first-task-scaffold.md.
This commit is contained in:
garrytan committed 2026-10-03 01:29:02 +00:00
1 parent 958f27a2b8
commit af697dc6ec
3 files changed
-103

No files matched your search

-2
View File
@@ -18,7 +18,6 @@
"test/skill-e2e-design.test.ts#plan-design-review-no-ui-scope": 24431,
"test/skill-e2e-diagram.test.ts": 29998,
"test/skill-e2e-docsync-spawned.test.ts": 137929,
"test/skill-e2e-first-task-scaffold.test.ts": 1000,
"test/skill-e2e-gbrain-roundtrip-local.test.ts": 1000,
"test/skill-e2e-hermetic-canary.test.ts": 8813,
"test/skill-e2e-investigate-owned-completion.test.ts": 45178,
@@ -143,7 +142,6 @@
"test/skill-e2e-design.test.ts#design-html-slop-gate": 158970,
"test/skill-e2e-design.test.ts#plan-design-review-plan-mode": 300166,
"test/skill-e2e-diagram.test.ts": 55187,
"test/skill-e2e-first-task-scaffold.test.ts": 8563,
"test/skill-e2e-gbrain-roundtrip-local.test.ts": 1000,
"test/skill-e2e-health.test.ts": 188584,
"test/skill-e2e-hermetic-canary.test.ts": 1000,
-3
View File
@@ -90,7 +90,6 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
// through the real runner, plus the script wiring that gates + maps it
// (token-reduction Phase 2: generate-first-run-guidance.ts was deleted; the
// gate + token→tip map live in bin/gstack-skill-start's emission layer).
'first-task-scaffold': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-first-task-detect', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'test/skill-e2e-first-task-scaffold.test.ts', 'test/helpers/session-runner.ts'],
// SKILL.md setup + preamble (depend on ROOT SKILL.md + gen-skill-docs)
'skillmd-setup-discovery': [ 'SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-bws.test.ts'],
@@ -1155,7 +1154,6 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic' | 'marathon'> = {
'operational-learning': 'gate',
// P4 first-run scaffold — periodic (onboarding, non-safety, model-touched marker)
'first-task-scaffold': 'periodic',
// QA — gate for functional, periodic for quality/benchmarks
'qa-quick': 'gate',
@@ -1617,7 +1615,6 @@ export const E2E_KINDS: Record<string, 'rule' | 'behavior' | 'judge'> = {
'skillmd-outside-git': 'rule',
'session-awareness': 'rule',
'operational-learning': 'rule',
'first-task-scaffold': 'rule',
'qa-quick': 'rule',
'qa-b6-static': 'rule',
'qa-b7-spa': 'rule',
@@ -1,98 +0,0 @@
/**
* P4 first-run scaffold — E2E (periodic tier, ~$0.02 each, deterministic).
*
* Exercises bin/gstack-first-task-detect END-TO-END through the real runner +
* hermetic env (path resolution, execution, git-in-cwd), not just the unit
* harness. Deterministic by construction: it asserts the binary's enum token
* from the Bash tool_result in the stream-json transcript (never the model's
* prose), so it pins the detector's integration contract without depending on
* non-deterministic model phrasing.
*
* Periodic (not gate): onboarding behavior is non-safety, and the scaffold
* marker is model-touched (best-effort). The deterministic bucket logic itself
* is fully covered by the unit test (test/preamble-first-task-scaffold.test.ts).
*/
import { expect, afterAll } from 'bun:test';
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { execSync } from 'node:child_process';
import { runSkillTest } from './helpers/session-runner';
import {
describeIfSelected, testIfSelected, createEvalCollector, finalizeEvalCollector,
recordE2E, runId, logCost,
} from './helpers/e2e-helpers';
const ROOT = path.join(import.meta.dir, '..');
const DETECT = path.join(ROOT, 'bin', 'gstack-first-task-detect');
const evalCollector = createEvalCollector('e2e-first-task-scaffold');
const MODEL = 'claude-haiku-4-5-20251001';
const GIT_ENV = {
...process.env,
GIT_AUTHOR_NAME: 'T', GIT_AUTHOR_EMAIL: 't@e.x',
GIT_COMMITTER_NAME: 'T', GIT_COMMITTER_EMAIL: 't@e.x',
};
/** Concatenated Bash tool_result text from the stream-json transcript. */
function toolResultText(transcript: any[]): string {
const chunks: string[] = [];
for (const event of transcript) {
if (event.type !== 'user') continue;
for (const item of event.message?.content ?? []) {
if (item.type !== 'tool_result') continue;
if (typeof item.content === 'string') chunks.push(item.content);
else for (const c of item.content ?? []) if (c.type === 'text') chunks.push(c.text);
}
}
return chunks.join('\n');
}
async function detectVia(workDir: string, testName: string): Promise<string> {
const result = await runSkillTest({
prompt: `Run exactly this one bash command and then stop, printing its output verbatim: ${DETECT}`,
workingDirectory: workDir,
maxTurns: 3,
allowedTools: ['Bash'],
timeout: JUDGE_MS,
testName,
runId,
model: MODEL,
});
logCost(testName, result);
recordE2E(evalCollector, testName, 'e2e-first-task-scaffold', result);
expect(result.exitReason).toBe('success');
return toolResultText(result.transcript);
}
describeIfSelected('first-run scaffold detection (E2E)', ['first-task-scaffold'], () => {
testIfSelected('first-task-scaffold', async () => {
if (!process.env.ANTHROPIC_API_KEY) {
throw new Error('first-task-scaffold requires ANTHROPIC_API_KEY (source ~/.zshrc); refusing to skip');
}
// code_node bucket: package.json + a commit, on the default branch.
const nodeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'fts-node-'));
// greenfield bucket: git repo, zero commits.
const greenDir = fs.mkdtempSync(path.join(os.tmpdir(), 'fts-green-'));
try {
execSync('git init -q -b main', { cwd: nodeDir, env: GIT_ENV, timeout: 30_000 });
fs.writeFileSync(path.join(nodeDir, 'package.json'), '{"name":"x"}');
execSync('git add -A && git commit -qm init', { cwd: nodeDir, env: GIT_ENV, timeout: 30_000 });
execSync('git init -q -b main', { cwd: greenDir, env: GIT_ENV, timeout: 30_000 });
const nodeOut = await detectVia(nodeDir, 'first-task-scaffold');
expect(nodeOut).toContain('code_node');
const greenOut = await detectVia(greenDir, 'first-task-scaffold-greenfield');
expect(greenOut).toContain('greenfield');
} finally {
fs.rmSync(nodeDir, { recursive: true, force: true });
fs.rmSync(greenDir, { recursive: true, force: true });
}
}, CAPTURE_MS);
});
afterAll(() => finalizeEvalCollector(evalCollector));