mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-12 19:44:48 +01:00
test(W7h): retire the paid first-task-scaffold E2E duplicate
W7h retirement (t12 F12). Removes the case from test/helpers/touchfiles-data.ts
(E2E_TOUCHFILES, E2E_TIERS periodic, E2E_KINDS rule) and both
scripts/paid-test-durations.json entries; test/helpers/periodic-exclude-data.ts
had no entry. e2e-tier-alignment, eval-kinds, eval-detach-timeout-floor,
periodic-exclude-policy, touchfiles and paid-shards tests stay green.
Retirement card:
- test: test/skill-e2e-first-task-scaffold.test.ts ('first-task-scaffold',
periodic, ~$0.02): Haiku is told to run exactly one detector command and the
test greps the tool result for code_node / greenfield.
- detects: detector bucket regressions in bin/gstack-first-task-detect.
- non_test_callers: none.
- search_command: grep -rn "first-task-scaffold" test scripts .github
- stronger_proof / replacement: test/preamble-first-task-scaffold.test.ts
(free; executes the real detector on fixture repos for every bucket,
including code_node and greenfield, plus first-run gating and telemetry in
bin/gstack-skill-start). The model added nothing: the prompt dictated the
exact command.
- history: c118e24 v1.64.1.0.
- validation: bun test test/preamble-first-task-scaffold.test.ts plus the
registration tests above.
Control (scratch worktree, against the replacement): making the detector emit
the wrong bucket for a Node project or a commit-less repo turns
preamble-first-task-scaffold.test.ts red (2/2). Log:
.context/mutation-first-task-scaffold.md.
This commit is contained in:
1 parent
958f27a2b8
commit
af697dc6ec
3 files changed
-103
No files matched your search
@@ -18,7 +18,6 @@
|
||||
"test/skill-e2e-design.test.ts#plan-design-review-no-ui-scope": 24431,
|
||||
"test/skill-e2e-diagram.test.ts": 29998,
|
||||
"test/skill-e2e-docsync-spawned.test.ts": 137929,
|
||||
"test/skill-e2e-first-task-scaffold.test.ts": 1000,
|
||||
"test/skill-e2e-gbrain-roundtrip-local.test.ts": 1000,
|
||||
"test/skill-e2e-hermetic-canary.test.ts": 8813,
|
||||
"test/skill-e2e-investigate-owned-completion.test.ts": 45178,
|
||||
@@ -143,7 +142,6 @@
|
||||
"test/skill-e2e-design.test.ts#design-html-slop-gate": 158970,
|
||||
"test/skill-e2e-design.test.ts#plan-design-review-plan-mode": 300166,
|
||||
"test/skill-e2e-diagram.test.ts": 55187,
|
||||
"test/skill-e2e-first-task-scaffold.test.ts": 8563,
|
||||
"test/skill-e2e-gbrain-roundtrip-local.test.ts": 1000,
|
||||
"test/skill-e2e-health.test.ts": 188584,
|
||||
"test/skill-e2e-hermetic-canary.test.ts": 1000,
|
||||
|
||||
@@ -90,7 +90,6 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// through the real runner, plus the script wiring that gates + maps it
|
||||
// (token-reduction Phase 2: generate-first-run-guidance.ts was deleted; the
|
||||
// gate + token→tip map live in bin/gstack-skill-start's emission layer).
|
||||
'first-task-scaffold': ['bin/gstack-state-root.sh', 'lib/state-root.ts', 'bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-first-task-detect', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'test/skill-e2e-first-task-scaffold.test.ts', 'test/helpers/session-runner.ts'],
|
||||
|
||||
// SKILL.md setup + preamble (depend on ROOT SKILL.md + gen-skill-docs)
|
||||
'skillmd-setup-discovery': [ 'SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-bws.test.ts'],
|
||||
@@ -1155,7 +1154,6 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic' | 'marathon'> = {
|
||||
'operational-learning': 'gate',
|
||||
|
||||
// P4 first-run scaffold — periodic (onboarding, non-safety, model-touched marker)
|
||||
'first-task-scaffold': 'periodic',
|
||||
|
||||
// QA — gate for functional, periodic for quality/benchmarks
|
||||
'qa-quick': 'gate',
|
||||
@@ -1617,7 +1615,6 @@ export const E2E_KINDS: Record<string, 'rule' | 'behavior' | 'judge'> = {
|
||||
'skillmd-outside-git': 'rule',
|
||||
'session-awareness': 'rule',
|
||||
'operational-learning': 'rule',
|
||||
'first-task-scaffold': 'rule',
|
||||
'qa-quick': 'rule',
|
||||
'qa-b6-static': 'rule',
|
||||
'qa-b7-spa': 'rule',
|
||||
|
||||
@@ -1,98 +0,0 @@
|
||||
/**
|
||||
* P4 first-run scaffold — E2E (periodic tier, ~$0.02 each, deterministic).
|
||||
*
|
||||
* Exercises bin/gstack-first-task-detect END-TO-END through the real runner +
|
||||
* hermetic env (path resolution, execution, git-in-cwd), not just the unit
|
||||
* harness. Deterministic by construction: it asserts the binary's enum token
|
||||
* from the Bash tool_result in the stream-json transcript (never the model's
|
||||
* prose), so it pins the detector's integration contract without depending on
|
||||
* non-deterministic model phrasing.
|
||||
*
|
||||
* Periodic (not gate): onboarding behavior is non-safety, and the scaffold
|
||||
* marker is model-touched (best-effort). The deterministic bucket logic itself
|
||||
* is fully covered by the unit test (test/preamble-first-task-scaffold.test.ts).
|
||||
*/
|
||||
|
||||
import { expect, afterAll } from 'bun:test';
|
||||
import { JUDGE_MS, CAPTURE_MS } from './helpers/eval-budgets';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import { execSync } from 'node:child_process';
|
||||
import { runSkillTest } from './helpers/session-runner';
|
||||
import {
|
||||
describeIfSelected, testIfSelected, createEvalCollector, finalizeEvalCollector,
|
||||
recordE2E, runId, logCost,
|
||||
} from './helpers/e2e-helpers';
|
||||
|
||||
const ROOT = path.join(import.meta.dir, '..');
|
||||
const DETECT = path.join(ROOT, 'bin', 'gstack-first-task-detect');
|
||||
const evalCollector = createEvalCollector('e2e-first-task-scaffold');
|
||||
const MODEL = 'claude-haiku-4-5-20251001';
|
||||
|
||||
const GIT_ENV = {
|
||||
...process.env,
|
||||
GIT_AUTHOR_NAME: 'T', GIT_AUTHOR_EMAIL: 't@e.x',
|
||||
GIT_COMMITTER_NAME: 'T', GIT_COMMITTER_EMAIL: 't@e.x',
|
||||
};
|
||||
|
||||
/** Concatenated Bash tool_result text from the stream-json transcript. */
|
||||
function toolResultText(transcript: any[]): string {
|
||||
const chunks: string[] = [];
|
||||
for (const event of transcript) {
|
||||
if (event.type !== 'user') continue;
|
||||
for (const item of event.message?.content ?? []) {
|
||||
if (item.type !== 'tool_result') continue;
|
||||
if (typeof item.content === 'string') chunks.push(item.content);
|
||||
else for (const c of item.content ?? []) if (c.type === 'text') chunks.push(c.text);
|
||||
}
|
||||
}
|
||||
return chunks.join('\n');
|
||||
}
|
||||
|
||||
async function detectVia(workDir: string, testName: string): Promise<string> {
|
||||
const result = await runSkillTest({
|
||||
prompt: `Run exactly this one bash command and then stop, printing its output verbatim: ${DETECT}`,
|
||||
workingDirectory: workDir,
|
||||
maxTurns: 3,
|
||||
allowedTools: ['Bash'],
|
||||
timeout: JUDGE_MS,
|
||||
testName,
|
||||
runId,
|
||||
model: MODEL,
|
||||
});
|
||||
logCost(testName, result);
|
||||
recordE2E(evalCollector, testName, 'e2e-first-task-scaffold', result);
|
||||
expect(result.exitReason).toBe('success');
|
||||
return toolResultText(result.transcript);
|
||||
}
|
||||
|
||||
describeIfSelected('first-run scaffold detection (E2E)', ['first-task-scaffold'], () => {
|
||||
testIfSelected('first-task-scaffold', async () => {
|
||||
if (!process.env.ANTHROPIC_API_KEY) {
|
||||
throw new Error('first-task-scaffold requires ANTHROPIC_API_KEY (source ~/.zshrc); refusing to skip');
|
||||
}
|
||||
|
||||
// code_node bucket: package.json + a commit, on the default branch.
|
||||
const nodeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'fts-node-'));
|
||||
// greenfield bucket: git repo, zero commits.
|
||||
const greenDir = fs.mkdtempSync(path.join(os.tmpdir(), 'fts-green-'));
|
||||
try {
|
||||
execSync('git init -q -b main', { cwd: nodeDir, env: GIT_ENV, timeout: 30_000 });
|
||||
fs.writeFileSync(path.join(nodeDir, 'package.json'), '{"name":"x"}');
|
||||
execSync('git add -A && git commit -qm init', { cwd: nodeDir, env: GIT_ENV, timeout: 30_000 });
|
||||
execSync('git init -q -b main', { cwd: greenDir, env: GIT_ENV, timeout: 30_000 });
|
||||
|
||||
const nodeOut = await detectVia(nodeDir, 'first-task-scaffold');
|
||||
expect(nodeOut).toContain('code_node');
|
||||
|
||||
const greenOut = await detectVia(greenDir, 'first-task-scaffold-greenfield');
|
||||
expect(greenOut).toContain('greenfield');
|
||||
} finally {
|
||||
fs.rmSync(nodeDir, { recursive: true, force: true });
|
||||
fs.rmSync(greenDir, { recursive: true, force: true });
|
||||
}
|
||||
}, CAPTURE_MS);
|
||||
});
|
||||
|
||||
afterAll(() => finalizeEvalCollector(evalCollector));
|
||||
Reference in new issue
Block a user