mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-03 01:46:55 +02:00
test: clean up the paid eval lane (B1-B4, B6, B7)
- B1: delete paid files that assert nothing or cannot pass meaningfully: skill-llm-eval-spec and skill-e2e-spec-execute (test.todo), gemini-e2e (+ gemini-session-runner; no gemini CLI in CI), ship-idempotency (red since v1.63), the two opus-4-7 *-sonnet overlay wrappers, conductor-prose (+ its source-evaluation replay), codex-e2e-plan-format; drop their keys, scripts and census rows. - B2: skill-llm-eval grades browse/sections/command-list.md with one union judge that also carries the baseline score pin; regression-vs-baseline deleted (paid run: pass, c4/c4/a4). - B3: memory-pipeline, ios-qa, ios-qa-swift-build and plan-tune-cathedral make no model calls; renamed out of the paid glob so they run on every PR. Swift builds need GSTACK_TEST_SWIFT=1; device stub deleted. - B4: codex-e2e*, outside-voice, aside and ios-device cannot run in the CI image; excluded from the weekly lane with a tracked re-entry condition. - B6: fold opus-47's negative routing controls into skill-routing-e2e journey-negatives (paid run: 3/3 unrouted) and delete the file. - B7: delete the never-green brain-privacy-gate eval; a free gstack-skill-start test now proves consent precedes artifacts egress.
This commit is contained in:
garrytan committed
1 parent
5d032ef299
commit
53e7f3212f
58 files changed
+273
-2594
No files matched your search
@@ -106,9 +106,8 @@ export const FILE_RETRY_BUDGETS = [
|
||||
...STRICT_RETRY_CASE_BUDGETS,
|
||||
...[
|
||||
// Sixteen workflow judges include their 10s recording grace; the other
|
||||
// eleven judges retain 120s. Supervise all 27 and the existing one retry.
|
||||
{ file: 'test/skill-llm-eval.test.ts', attemptMs: 16 * (JUDGE_MS + 10_000) + 11 * JUDGE_MS, retries: 1 },
|
||||
{ file: 'test/codex-e2e-plan-format.test.ts', attemptMs: 4 * (CAPTURE_LONG_MS + 10_000), retries: 1 },
|
||||
// seven judges retain 120s. Supervise all 23 and the existing one retry.
|
||||
{ file: 'test/skill-llm-eval.test.ts', attemptMs: 16 * (JUDGE_MS + 10_000) + 7 * JUDGE_MS, retries: 1 },
|
||||
{ file: 'test/skill-e2e-auq-matrix.test.ts', attemptMs: 6 * CAPTURE_MS, retries: 1 },
|
||||
{ file: 'test/skill-e2e-plan-format.test.ts', attemptMs: 4 * (CAPTURE_MS + 10_000), retries: 1 },
|
||||
{ file: 'test/skill-e2e-auto-decide-preserved.test.ts', attemptMs: PTY_MS, retries: 1 },
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { parseGeminiJSONL } from './gemini-session-runner';
|
||||
|
||||
// Fixture: actual Gemini CLI stream-json output with tool use
|
||||
const FIXTURE_LINES = [
|
||||
'{"type":"init","timestamp":"2026-03-20T15:14:46.455Z","session_id":"test-session-123","model":"auto-gemini-3"}',
|
||||
'{"type":"message","timestamp":"2026-03-20T15:14:46.456Z","role":"user","content":"list the files"}',
|
||||
'{"type":"message","timestamp":"2026-03-20T15:14:49.650Z","role":"assistant","content":"I will list the files.","delta":true}',
|
||||
'{"type":"tool_use","timestamp":"2026-03-20T15:14:49.690Z","tool_name":"run_shell_command","tool_id":"cmd_1","parameters":{"command":"ls"}}',
|
||||
'{"type":"tool_result","timestamp":"2026-03-20T15:14:49.931Z","tool_id":"cmd_1","status":"success","output":"file1.ts\\nfile2.ts"}',
|
||||
'{"type":"message","timestamp":"2026-03-20T15:14:51.945Z","role":"assistant","content":"Here are the files.","delta":true}',
|
||||
'{"type":"result","timestamp":"2026-03-20T15:14:52.030Z","status":"success","stats":{"total_tokens":27147,"input_tokens":26928,"output_tokens":87,"cached":0,"duration_ms":5575,"tool_calls":1}}',
|
||||
];
|
||||
|
||||
describe('parseGeminiJSONL', () => {
|
||||
test('extracts session ID from init event', () => {
|
||||
const parsed = parseGeminiJSONL(FIXTURE_LINES);
|
||||
expect(parsed.sessionId).toBe('test-session-123');
|
||||
});
|
||||
|
||||
test('concatenates assistant message deltas into output', () => {
|
||||
const parsed = parseGeminiJSONL(FIXTURE_LINES);
|
||||
expect(parsed.output).toBe('I will list the files.Here are the files.');
|
||||
});
|
||||
|
||||
test('ignores user messages', () => {
|
||||
const lines = [
|
||||
'{"type":"message","role":"user","content":"this should be ignored"}',
|
||||
'{"type":"message","role":"assistant","content":"this should be kept","delta":true}',
|
||||
];
|
||||
const parsed = parseGeminiJSONL(lines);
|
||||
expect(parsed.output).toBe('this should be kept');
|
||||
});
|
||||
|
||||
test('extracts tool names from tool_use events', () => {
|
||||
const parsed = parseGeminiJSONL(FIXTURE_LINES);
|
||||
expect(parsed.toolCalls).toHaveLength(1);
|
||||
expect(parsed.toolCalls[0]).toBe('run_shell_command');
|
||||
});
|
||||
|
||||
test('extracts total tokens from result stats', () => {
|
||||
const parsed = parseGeminiJSONL(FIXTURE_LINES);
|
||||
expect(parsed.tokens).toBe(27147);
|
||||
});
|
||||
|
||||
test('skips malformed lines without throwing', () => {
|
||||
const lines = [
|
||||
'{"type":"init","session_id":"ok"}',
|
||||
'this is not json',
|
||||
'{"type":"message","role":"assistant","content":"hello","delta":true}',
|
||||
'{incomplete json',
|
||||
'{"type":"result","status":"success","stats":{"total_tokens":100}}',
|
||||
];
|
||||
const parsed = parseGeminiJSONL(lines);
|
||||
expect(parsed.sessionId).toBe('ok');
|
||||
expect(parsed.output).toBe('hello');
|
||||
expect(parsed.tokens).toBe(100);
|
||||
});
|
||||
|
||||
test('skips empty and whitespace-only lines', () => {
|
||||
const lines = [
|
||||
'',
|
||||
' ',
|
||||
'{"type":"init","session_id":"s1"}',
|
||||
'\t',
|
||||
'{"type":"result","status":"success","stats":{"total_tokens":50}}',
|
||||
];
|
||||
const parsed = parseGeminiJSONL(lines);
|
||||
expect(parsed.sessionId).toBe('s1');
|
||||
expect(parsed.tokens).toBe(50);
|
||||
});
|
||||
|
||||
test('handles empty input', () => {
|
||||
const parsed = parseGeminiJSONL([]);
|
||||
expect(parsed.output).toBe('');
|
||||
expect(parsed.toolCalls).toHaveLength(0);
|
||||
expect(parsed.tokens).toBe(0);
|
||||
expect(parsed.sessionId).toBeNull();
|
||||
});
|
||||
|
||||
test('handles missing fields gracefully', () => {
|
||||
const lines = [
|
||||
'{"type":"init"}', // no session_id
|
||||
'{"type":"message","role":"assistant"}', // no content
|
||||
'{"type":"tool_use"}', // no tool_name
|
||||
'{"type":"result","status":"success"}', // no stats
|
||||
];
|
||||
const parsed = parseGeminiJSONL(lines);
|
||||
expect(parsed.sessionId).toBeNull();
|
||||
expect(parsed.output).toBe('');
|
||||
expect(parsed.toolCalls).toHaveLength(0);
|
||||
expect(parsed.tokens).toBe(0);
|
||||
});
|
||||
|
||||
test('handles multiple tool_use events', () => {
|
||||
const lines = [
|
||||
'{"type":"tool_use","tool_name":"run_shell_command","tool_id":"cmd_1","parameters":{"command":"ls"}}',
|
||||
'{"type":"tool_use","tool_name":"read_file","tool_id":"cmd_2","parameters":{"path":"foo.ts"}}',
|
||||
'{"type":"tool_use","tool_name":"run_shell_command","tool_id":"cmd_3","parameters":{"command":"cat bar.ts"}}',
|
||||
];
|
||||
const parsed = parseGeminiJSONL(lines);
|
||||
expect(parsed.toolCalls).toEqual(['run_shell_command', 'read_file', 'run_shell_command']);
|
||||
});
|
||||
});
|
||||
@@ -1,262 +0,0 @@
|
||||
/**
|
||||
* Gemini CLI subprocess runner for skill E2E testing.
|
||||
*
|
||||
* Spawns `gemini -p` as an independent process, parses its stream-json
|
||||
* output, and returns structured results. Follows the same pattern as
|
||||
* codex-session-runner.ts but adapted for the Gemini CLI.
|
||||
*
|
||||
* Key differences from Codex session-runner:
|
||||
* - Uses `gemini -p` instead of `codex exec`
|
||||
* - Output is NDJSON with event types: init, message, tool_use, tool_result, result
|
||||
* - Uses `--output-format stream-json --yolo` instead of `--json -s read-only`
|
||||
* (`--skip-trust` was removed in gemini-cli 0.34; folder trust is settings-driven now)
|
||||
* - No temp HOME needed — Gemini discovers skills from `.agents/skills/` in cwd
|
||||
* - Message events are streamed with `delta: true` — must concatenate
|
||||
*/
|
||||
|
||||
import * as path from 'path';
|
||||
import { spawn } from 'child_process';
|
||||
import { Readable } from 'node:stream';
|
||||
import { hermeticChildEnv } from './hermetic-env';
|
||||
import { killProcessGroup } from '../../scripts/test-strict-output';
|
||||
|
||||
// --- Interfaces ---
|
||||
|
||||
export interface GeminiResult {
|
||||
output: string; // Full assistant message text (concatenated deltas)
|
||||
toolCalls: string[]; // Tool names from tool_use events
|
||||
tokens: number; // Total tokens used
|
||||
exitCode: number; // Process exit code
|
||||
durationMs: number; // Wall clock time
|
||||
sessionId: string | null; // Session ID from init event
|
||||
rawLines: string[]; // Raw JSONL lines for debugging
|
||||
}
|
||||
|
||||
// --- JSONL parser ---
|
||||
|
||||
export interface ParsedGeminiJSONL {
|
||||
output: string;
|
||||
toolCalls: string[];
|
||||
tokens: number;
|
||||
sessionId: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an array of JSONL lines from `gemini -p --output-format stream-json`.
|
||||
* Pure function — no I/O, no side effects.
|
||||
*
|
||||
* Handles these Gemini event types:
|
||||
* - init → extract session_id
|
||||
* - message (role=assistant, delta=true) → concatenate content into output
|
||||
* - tool_use → extract tool_name
|
||||
* - tool_result → logged but not extracted
|
||||
* - result → extract token usage from stats
|
||||
*/
|
||||
export function parseGeminiJSONL(lines: string[]): ParsedGeminiJSONL {
|
||||
const outputParts: string[] = [];
|
||||
const toolCalls: string[] = [];
|
||||
let tokens = 0;
|
||||
let sessionId: string | null = null;
|
||||
|
||||
for (const line of lines) {
|
||||
if (!line.trim()) continue;
|
||||
try {
|
||||
const obj = JSON.parse(line);
|
||||
const t = obj.type || '';
|
||||
|
||||
if (t === 'init') {
|
||||
const sid = obj.session_id || '';
|
||||
if (sid) sessionId = sid;
|
||||
} else if (t === 'message') {
|
||||
if (obj.role === 'assistant' && obj.content) {
|
||||
outputParts.push(obj.content);
|
||||
}
|
||||
} else if (t === 'tool_use') {
|
||||
const name = obj.tool_name || '';
|
||||
if (name) toolCalls.push(name);
|
||||
} else if (t === 'result') {
|
||||
const stats = obj.stats || {};
|
||||
tokens = (stats.total_tokens || 0);
|
||||
}
|
||||
} catch { /* skip malformed lines */ }
|
||||
}
|
||||
|
||||
return {
|
||||
output: outputParts.join(''),
|
||||
toolCalls,
|
||||
tokens,
|
||||
sessionId,
|
||||
};
|
||||
}
|
||||
|
||||
// --- Main runner ---
|
||||
|
||||
/**
|
||||
* Run a prompt via `gemini -p` and return structured results.
|
||||
*
|
||||
* Spawns gemini with stream-json output, parses JSONL events,
|
||||
* and returns a GeminiResult. Skips gracefully if gemini binary is not found.
|
||||
*/
|
||||
export async function runGeminiSkill(opts: {
|
||||
prompt: string; // What to ask Gemini
|
||||
timeoutMs?: number; // Default 300000 (5 min)
|
||||
cwd?: string; // Working directory (where .agents/skills/ lives)
|
||||
}): Promise<GeminiResult> {
|
||||
const {
|
||||
prompt,
|
||||
timeoutMs = 300_000,
|
||||
cwd,
|
||||
} = opts;
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
// Check if gemini binary exists
|
||||
const whichResult = Bun.spawnSync(['which', 'gemini'], { timeout: 30_000 });
|
||||
if (whichResult.exitCode !== 0) {
|
||||
return {
|
||||
output: 'SKIP: gemini binary not found',
|
||||
toolCalls: [],
|
||||
tokens: 0,
|
||||
exitCode: -1,
|
||||
durationMs: Date.now() - startTime,
|
||||
sessionId: null,
|
||||
rawLines: [],
|
||||
};
|
||||
}
|
||||
|
||||
// Build gemini command.
|
||||
// --skip-trust was REMOVED in gemini-cli 0.34 ("Unknown arguments:
|
||||
// skip-trust"); folder trust moved to settings and no longer needs a flag
|
||||
// for headless runs. --yolo still auto-approves tool actions.
|
||||
const args = ['-p', prompt, '--output-format', 'stream-json', '--yolo'];
|
||||
|
||||
// Spawn gemini — uses real HOME for auth (~/.gemini; HOME is allowlisted),
|
||||
// cwd for skill discovery. Hermetic scrub with gemini's auth surface
|
||||
// re-admitted (previously this spawn inherited the full operator env).
|
||||
// node:child_process spawn with `detached` (own process group) — mirrors
|
||||
// session-runner.ts. A bare kill signalled only gemini itself; tool
|
||||
// subprocesses survived as orphans holding our pipes open (the same
|
||||
// blocked-drain hang the claude runner fixed — this copy lacked it).
|
||||
const proc = spawn('gemini', args, {
|
||||
cwd: cwd || process.cwd(),
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
detached: process.platform !== 'win32',
|
||||
env: hermeticChildEnv(undefined, {
|
||||
extraAllow: ['GEMINI_API_KEY', 'GOOGLE_API_KEY', 'GOOGLE_APPLICATION_CREDENTIALS', 'GOOGLE_CLOUD_*', 'GEMINI_*'],
|
||||
}),
|
||||
});
|
||||
const stdoutWeb = Readable.toWeb(proc.stdout!) as ReadableStream<Uint8Array>;
|
||||
const stderrWeb = Readable.toWeb(proc.stderr!) as ReadableStream<Uint8Array>;
|
||||
const procExited: Promise<number> = new Promise((resolve) => {
|
||||
proc.on('close', (code) => resolve(code ?? 1));
|
||||
proc.on('error', () => resolve(1));
|
||||
});
|
||||
|
||||
// Race against timeout
|
||||
let timedOut = false;
|
||||
const timeoutId = setTimeout(() => {
|
||||
timedOut = true;
|
||||
// Group SIGKILL + reader cancel: kill the whole tree AND unblock the
|
||||
// read loop even if a stray grandchild survives the group kill.
|
||||
killProcessGroup(proc, 'SIGKILL');
|
||||
reader.cancel().catch(() => { /* stream already closed */ });
|
||||
}, timeoutMs);
|
||||
|
||||
// Stream and collect JSONL from stdout
|
||||
const collectedLines: string[] = [];
|
||||
const stderrPromise = new Response(stderrWeb).text();
|
||||
|
||||
const reader = stdoutWeb.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buf = '';
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
buf += decoder.decode(value, { stream: true });
|
||||
const lines = buf.split('\n');
|
||||
buf = lines.pop() || '';
|
||||
for (const line of lines) {
|
||||
if (!line.trim()) continue;
|
||||
collectedLines.push(line);
|
||||
|
||||
// Real-time progress to stderr
|
||||
try {
|
||||
const event = JSON.parse(line);
|
||||
if (event.type === 'tool_use' && event.tool_name) {
|
||||
const elapsed = Math.round((Date.now() - startTime) / 1000);
|
||||
process.stderr.write(` [gemini ${elapsed}s] tool: ${event.tool_name}\n`);
|
||||
} else if (event.type === 'message' && event.role === 'assistant' && event.content) {
|
||||
const elapsed = Math.round((Date.now() - startTime) / 1000);
|
||||
process.stderr.write(` [gemini ${elapsed}s] message: ${event.content.slice(0, 100)}\n`);
|
||||
}
|
||||
} catch { /* skip — parseGeminiJSONL will handle it later */ }
|
||||
}
|
||||
}
|
||||
} catch { /* stream read error — fall through to exit code handling */ }
|
||||
|
||||
// Flush remaining buffer
|
||||
if (buf.trim()) {
|
||||
collectedLines.push(buf);
|
||||
}
|
||||
|
||||
// Same orphan hazard as stdout: a grandchild holding stderr open would
|
||||
// block this drain forever. Race against child exit + a short grace window
|
||||
// (ported from session-runner.ts — the gemini copy lacked it).
|
||||
const stderr = await Promise.race([
|
||||
stderrPromise,
|
||||
(async () => {
|
||||
await procExited;
|
||||
await new Promise((r) => setTimeout(r, 5_000));
|
||||
return '';
|
||||
})(),
|
||||
]);
|
||||
const exitCode = await procExited;
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
const durationMs = Date.now() - startTime;
|
||||
|
||||
// Parse all collected JSONL lines
|
||||
const parsed = parseGeminiJSONL(collectedLines);
|
||||
|
||||
// Log stderr if non-empty (may contain auth errors, etc.)
|
||||
if (stderr.trim()) {
|
||||
process.stderr.write(` [gemini stderr] ${stderr.trim().slice(0, 200)}\n`);
|
||||
}
|
||||
|
||||
// Environment-unusable classification: these are Google-side conditions no
|
||||
// test assertion can act on — the deprecated individual code-assist auth
|
||||
// path ("migrate to the Antigravity suite") and argv drift on older/newer
|
||||
// CLIs. Return the same SKIP shape as binary-not-found so callers report
|
||||
// SKIPPED instead of a false FAIL.
|
||||
const unusableMarkers = [
|
||||
'no longer supported for Gemini Code Assist',
|
||||
'antigravity',
|
||||
'Unknown arguments: skip-trust',
|
||||
];
|
||||
if (exitCode !== 0 && parsed.tokens === 0) {
|
||||
const marker = unusableMarkers.find((m) => stderr.toLowerCase().includes(m.toLowerCase()));
|
||||
if (marker) {
|
||||
return {
|
||||
output: `SKIP: gemini CLI unusable (${marker})`,
|
||||
toolCalls: [],
|
||||
tokens: 0,
|
||||
exitCode: -1,
|
||||
durationMs,
|
||||
sessionId: null,
|
||||
rawLines: collectedLines,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
output: parsed.output,
|
||||
toolCalls: parsed.toolCalls,
|
||||
tokens: parsed.tokens,
|
||||
exitCode: timedOut ? 124 : exitCode,
|
||||
durationMs,
|
||||
sessionId: parsed.sessionId,
|
||||
rawLines: collectedLines,
|
||||
};
|
||||
}
|
||||
@@ -21,6 +21,4 @@ export const OVERLAY_CASE_FILES: Record<string, string> = {
|
||||
'test/skill-e2e-overlay-harness-opus-4-7-effort-match-trivial.test.ts': 'opus-4-7-effort-match-trivial',
|
||||
'test/skill-e2e-overlay-harness-opus-4-7-literal-interpretation.test.ts': 'opus-4-7-literal-interpretation',
|
||||
'test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-sonnet.test.ts': 'claude-dedicated-tools-vs-bash-sonnet',
|
||||
'test/skill-e2e-overlay-harness-opus-4-7-effort-match-trivial-sonnet.test.ts': 'opus-4-7-effort-match-trivial-sonnet',
|
||||
'test/skill-e2e-overlay-harness-opus-4-7-literal-interpretation-sonnet.test.ts': 'opus-4-7-literal-interpretation-sonnet',
|
||||
};
|
||||
@@ -11,8 +11,8 @@ import { matchGlob } from './touchfiles';
|
||||
|
||||
/** The exact globs package.json's `test:gate` passes to `bun test`. */
|
||||
export const PAID_TEST_GLOBS = [
|
||||
// skill-llm-eval* (not just the base file): skill-llm-eval-spec.test.ts
|
||||
// fell outside the exact glob and could never run in any lane.
|
||||
// skill-llm-eval* (not just the base file): a sibling judge file outside
|
||||
// the exact glob could never run in any lane.
|
||||
'test/skill-llm-eval*.test.ts',
|
||||
'test/skill-e2e-*.test.ts',
|
||||
'test/skill-routing-e2e.test.ts',
|
||||
@@ -22,7 +22,6 @@ export const PAID_TEST_GLOBS = [
|
||||
// entered the paid census. The same bug class as the deleted pre-split
|
||||
// monolith (see test/paid-shards.test.ts's regression pin).
|
||||
'test/codex-e2e*.test.ts',
|
||||
'test/gemini-e2e.test.ts',
|
||||
'test/llm-judge-recommendation.test.ts',
|
||||
'test/carve-section-loading*.test.ts',
|
||||
] as const;
|
||||
|
||||
@@ -15,20 +15,32 @@
|
||||
* the re-entry mechanism.
|
||||
*/
|
||||
export const PERIODIC_CI_EXCLUDE: Record<string, { reason: string; tracking: string }> = {
|
||||
'test/skill-e2e-ship-idempotency.test.ts': {
|
||||
reason:
|
||||
'documented-red: the PTY child sits at the Claude Code welcome screen for the full budget '
|
||||
+ '(readiness/typing race vs CLI 2.1.x); never green since it was born in v1.63',
|
||||
tracking: 'TODOS.md "periodic tier — three documented-red tests need structural repair" (1 of 3 resolved: sidebar trio already deleted)',
|
||||
'test/codex-e2e.test.ts': {
|
||||
reason: 'the codex CLI is not installed in the CI image (Dockerfile.ci ships only claude-code); every case self-skips',
|
||||
tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)',
|
||||
},
|
||||
'test/skill-e2e-brain-privacy-gate.test.ts': {
|
||||
reason:
|
||||
'documented-red: the artifacts-sync stop-gate preconditions do not survive the hermetic env '
|
||||
+ 'even with per-test HOME/GSTACK_HOME injection; never green anywhere',
|
||||
tracking: 'TODOS.md "periodic tier — three documented-red tests need structural repair"',
|
||||
'test/codex-e2e-sol-scope.test.ts': {
|
||||
reason: 'the codex CLI is not installed in the CI image (Dockerfile.ci ships only claude-code); every case self-skips',
|
||||
tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)',
|
||||
},
|
||||
'test/skill-e2e-ios.test.ts': {
|
||||
reason: 'requires a live iOS device/simulator toolchain (xcodebuild, devicectl) — manual hardware, not a CI runner capability',
|
||||
tracking: 'TODOS.md "skill-e2e-ios CI story" (device/runner decision)',
|
||||
'test/codex-e2e-shared-libs.test.ts': {
|
||||
reason: 'the codex CLI is not installed in the CI image (Dockerfile.ci ships only claude-code); every case self-skips',
|
||||
tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)',
|
||||
},
|
||||
'test/codex-e2e-recommendation-substance.test.ts': {
|
||||
reason: 'the codex CLI is not installed in the CI image (Dockerfile.ci ships only claude-code); every case self-skips',
|
||||
tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)',
|
||||
},
|
||||
'test/skill-e2e-outside-voice.test.ts': {
|
||||
reason: 'needs both the claude and codex CLIs; codex is not in the CI image, so every case self-skips',
|
||||
tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)',
|
||||
},
|
||||
'test/skill-e2e-aside.test.ts': {
|
||||
reason: 'needs macOS with the Aside app open (asideAvailable()); CI runners are Linux, so every case self-skips',
|
||||
tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)',
|
||||
},
|
||||
'test/skill-e2e-ios-device.test.ts': {
|
||||
reason: 'needs a physical iPhone over USB/devicectl — manual hardware, not a CI runner capability',
|
||||
tracking: 'TODOS.md "CI-unrunnable paid evals" (re-entry: the CLI/device is available in the CI image; review by 2026-12-28)',
|
||||
},
|
||||
};
|
||||
@@ -3,8 +3,8 @@ import * as path from 'node:path';
|
||||
/**
|
||||
* Provider adapter interface — uniform contract for Claude, GPT, Gemini.
|
||||
*
|
||||
* Each adapter wraps an existing runner (session-runner.ts, codex-session-runner.ts,
|
||||
* gemini-session-runner.ts) and normalizes its per-provider result shape into the
|
||||
* Each adapter wraps an existing runner or CLI (session-runner.ts,
|
||||
* codex-session-runner.ts, the gemini CLI) and normalizes its per-provider result shape into the
|
||||
* RunResult below. The benchmark harness only talks to adapters through this
|
||||
* interface, never to the underlying runners directly.
|
||||
*/
|
||||
|
||||
@@ -334,23 +334,6 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// Conductor → prose decision brief (Conductor signal makes prose the default;
|
||||
// the PreToolUse hook denies the flaky tool). Touches the resolver that owns
|
||||
// the Conductor rule, the preamble signal, the hook, and the detection helper.
|
||||
'conductor-prose': [
|
||||
'lib/claude-public-transcript.ts',
|
||||
'test/auto-decide-recommendation-scope.test.ts',
|
||||
'test/fixtures/auto-decide-recommendation-361c.json',
|
||||
'test/auto-decide-target-identity.test.ts',
|
||||
'test/fixtures/auto-decide-target-361c.json',
|
||||
|
||||
'test/autoplan-public-narration.test.ts', 'test/fixtures/autoplan-public-narration-ad.json',
|
||||
'test/helpers/plan-count-transcript.ts', 'test/plan-count-cross-cwd-ancestry.test.ts', 'test/fixtures/plan-count-cross-cwd-ancestry-0bcd.json', 'test/plan-count-session-cwd.test.ts',
|
||||
'scripts/resolvers/learnings.ts',
|
||||
"test/plan-scope-recovery-av.test.ts",
|
||||
"test/fixtures/plan-scope-recovery-av.json", 'test/pty-screen-unicode-ap.test.ts', 'test/eng-scope-entry-ap.test.ts',
|
||||
'test/conductor-prose-observation-ao.test.ts', 'test/fixtures/conductor-prose-ao.json',
|
||||
'test/auto-decide-saved-ai.test.ts', 'test/fixtures/auto-decide-saved-ai.json', 'test/fixtures/auto-decide-retry-ai.json','bin/gstack-skill-start', 'bin/gstack-skill-end', 'bin/gstack-session-kind', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble.ts', 'plan-eng-review/**', 'hosts/claude/hooks/question-preference-hook.ts', 'hosts/claude/hooks/spawned-directive.ts', 'lib/is-conductor.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-design-ui-recovery.test.ts', 'test/fixtures/design-ui-boxed-question.json', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-conductor-prose.test.ts', 'test/plan-count-truncated-border.test.ts', 'test/fixtures/eng-d2-truncated-border-0bcd.json', 'test/plan-count-truncated-question.test.ts', 'test/plan-count-clipped-elision.test.ts', 'test/fixtures/eng-d1-clipped-elision-1579.json', 'test/fixtures/eng-d2-planning-prelude-4d.json', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt', 'test/helpers/native-auto-decide.ts', 'test/auto-decide-current-declaration.test.ts', 'test/fixtures/auto-decide-current-declaration-6aef.json', 'test/auto-decide-explanatory-mode.test.ts', 'test/fixtures/auto-decide-explanatory-mode-043a.json', 'test/fixtures/auto-decide-explanatory-mode-749df.json', 'test/auto-decide-structured.test.ts', 'test/fixtures/auto-decide-structured-77.json', 'test/helpers/auto-decision-state.ts', 'test/auto-decision-state.test.ts', 'test/fixtures/auto-decide-state-cab3.json', 'bin/gstack-question-log', 'bin/gstack-question-preference', 'test/native-auto-decide.test.ts', 'test/native-auto-decide-pty.test.ts', 'test/helpers/fake-plan-seed.ts', 'test/fixtures/native-auto-decide-ag.json', 'test/eng-seeded-completion-ai.test.ts', 'test/fixtures/eng-seeded-completion-ai.json', 'test/helpers/plan-count-pending-exit.ts', 'test/plan-count-pending-exit.test.ts', 'test/helpers/pty-screen.ts', 'test/pty-screen.test.ts', 'test/pty-screen-session.test.ts', 'test/fixtures/pty-screen/**',
|
||||
"test/review-entry-and-design-clarity-au.test.ts", "scripts/resolvers/preamble/generate-completion-status.ts",
|
||||
'test/pty-workspace-trust.test.ts', 'test/fixtures/pty-companion-cli.ts', 'test/helpers/plan-seed-submission.ts', 'test/plan-seed-submission.test.ts', 'test/fixtures/plan-seed-cli.ts', 'test/helpers/owned-claude-transcript.ts', 'lib/fs-atomic.ts', 'test/helpers/plan-skill-questions.ts', 'test/fixtures/eng-auq-validation-error.json', 'test/fixtures/bash-directory-permission.json', 'test/fixtures/design-tasks-bash-permission.json', 'test/plan-skill-read-permission.test.ts', 'test/fixtures/read-permission.json', 'test/pty-numbered-option-indent-native.test.ts', 'test/fixtures/ceo-split-e5-numbered-description-491.json', 'test/plan-skill-questions.test.ts', 'test/helpers/plan-skill-question-events.ts', 'test/plan-skill-question-events.test.ts', 'test/helpers/plan-skill-question-hook-scope.ts', 'test/helpers/skill-census.ts', 'test/plan-skill-question-hook-scope.test.ts', 'scripts/resolvers/testing.ts', 'scripts/resolvers/review.ts', 'test/plan-review-cases.test.ts'
|
||||
],
|
||||
|
||||
// Native question capture and interactive workflow probes.
|
||||
'auq-format-gate': ['test/session-runner-stream-lifecycle.test.ts', 'plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-ask-user-question-format-compliance.test.ts',
|
||||
@@ -393,9 +376,6 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
"test/review-entry-and-design-clarity-au.test.ts", "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts",
|
||||
'test/pty-workspace-trust.test.ts', 'test/fixtures/pty-companion-cli.ts', 'lib/fs-atomic.ts', 'test/plan-design-with-ui-fixture.test.ts', 'test/helpers/ceo-finding-fixture.ts', 'test/helpers/plan-review-cases.ts', 'test/helpers/plan-review-board-feedback.ts', 'test/plan-review-board-feedback.test.ts', 'test/fixtures/design-board-questions.json', 'test/fixtures/design-outside-voices-question.json', 'design/src/daemon-state.ts', 'design/src/daemon.ts', 'design/test/daemon-tests-fixtures.ts', 'design/src/daemon-client.ts', 'test/helpers/owned-claude-transcript.ts', 'test/helpers/plan-skill-questions.ts', 'test/fixtures/eng-auq-validation-error.json', 'test/fixtures/bash-directory-permission.json', 'test/fixtures/design-tasks-bash-permission.json', 'test/plan-skill-read-permission.test.ts', 'test/fixtures/read-permission.json', 'test/pty-numbered-option-indent-native.test.ts', 'test/fixtures/ceo-split-e5-numbered-description-491.json', 'test/plan-skill-questions.test.ts', 'test/helpers/plan-skill-question-events.ts', 'test/plan-skill-question-events.test.ts', 'test/helpers/plan-skill-question-hook-scope.ts', 'test/helpers/skill-census.ts', 'test/plan-skill-question-hook-scope.test.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'bin/gstack-paths', 'bin/gstack-slug', 'scripts/resolvers/design.ts'
|
||||
],
|
||||
'ship-idempotency-pty': ['ship/**', 'bin/gstack-next-version', 'bin/gstack-version-bump', 'scripts/resolvers/sections.ts', 'lib/worktree.ts', 'test/helpers/claude-pty-runner.ts', 'test/plan-count-design-ui-recovery.test.ts', 'test/fixtures/design-ui-boxed-question.json', 'test/helpers/hermetic-skill-runtime.ts', 'test/hermetic-skill-runtime.test.ts', 'test/helpers/pty-trust-dialog.ts', 'test/pty-trust-dialog.test.ts', 'test/skill-e2e-ship-idempotency.test.ts', 'test/plan-count-truncated-border.test.ts', 'test/fixtures/eng-d2-truncated-border-0bcd.json', 'test/plan-count-truncated-question.test.ts', 'test/plan-count-clipped-elision.test.ts', 'test/fixtures/eng-d1-clipped-elision-1579.json', 'test/fixtures/eng-d2-planning-prelude-4d.json', 'test/fixtures/ceo-approach-z-call.json', 'test/fixtures/ceo-approach-z-screen.txt',
|
||||
'test/pty-workspace-trust.test.ts', 'test/fixtures/pty-companion-cli.ts', 'lib/fs-atomic.ts', 'test/helpers/plan-skill-questions.ts', 'test/fixtures/eng-auq-validation-error.json', 'test/fixtures/bash-directory-permission.json', 'test/fixtures/design-tasks-bash-permission.json', 'test/plan-skill-read-permission.test.ts', 'test/fixtures/read-permission.json', 'test/pty-numbered-option-indent-native.test.ts', 'test/fixtures/ceo-split-e5-numbered-description-491.json', 'test/plan-skill-questions.test.ts', 'test/helpers/plan-skill-question-events.ts', 'test/plan-skill-question-events.test.ts', 'test/helpers/plan-skill-question-hook-scope.ts', 'test/helpers/skill-census.ts', 'test/plan-skill-question-hook-scope.test.ts', 'scripts/resolvers/testing.ts'
|
||||
],
|
||||
'tpa-present': ['test/session-runner-stream-lifecycle.test.ts', 'scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts', 'test/helpers/third-party-actions.ts',
|
||||
'test/third-party-actions-recording.test.ts', 'lib/eval-model.ts'
|
||||
],
|
||||
@@ -784,9 +764,6 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'test/fixtures/plan-count-quoted-frame-ak.json',
|
||||
'docs/askuserquestion-split.md', 'test/resolver-ask-user-format.test.ts', 'bin/gstack-slug', 'test/helpers/hermetic-env.test.ts', 'test/pty-workspace-trust.test.ts', 'test/fixtures/pty-companion-cli.ts', 'lib/fs-atomic.ts', 'test/helpers/ceo-finding-fixture.ts', 'test/ceo-finding-fixture.test.ts', 'test/helpers/owned-claude-transcript.ts', 'test/eval-budgets-policy.test.ts', 'test/fixtures/webfetch-permission.json', 'test/plan-skill-webfetch-permission.test.ts', 'test/helpers/plan-skill-questions.ts', 'test/fixtures/eng-auq-validation-error.json', 'test/fixtures/bash-directory-permission.json', 'test/fixtures/design-tasks-bash-permission.json', 'test/plan-skill-read-permission.test.ts', 'test/fixtures/read-permission.json', 'test/pty-numbered-option-indent-native.test.ts', 'test/fixtures/ceo-split-e5-numbered-description-491.json', 'test/plan-skill-questions.test.ts', 'test/helpers/plan-skill-question-events.ts', 'test/plan-skill-question-events.test.ts', 'test/helpers/plan-skill-question-hook-scope.ts', 'test/helpers/skill-census.ts', 'test/plan-skill-question-hook-scope.test.ts', 'test/helpers/plan-review-decisions.ts', 'test/plan-review-decisions.test.ts', 'test/helpers/plan-review-cases.ts', 'test/plan-review-cases.test.ts', 'test/helpers/llm-judge.ts', 'lib/eval-model.ts', 'test/skill-e2e-plan-decision-classification.test.ts', 'test/fixtures/plan-decision-classification.ts', 'test/plan-review-calibration.test.ts', 'test/helpers/ceo-split-question-policy.ts', 'test/fixtures/ceo-split-actor-6aef.json', 'test/helpers/ceo-mode-option.ts', 'test/ceo-mode-option.test.ts', 'test/ceo-split-collection.test.ts', 'test/fixtures/ceo-split-collection-0bcd.json', 'test/ceo-split-question-policy.test.ts', 'scripts/resolvers/review.ts', 'scripts/resolvers/tasks-section.ts'
|
||||
],
|
||||
'brain-privacy-gate': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'scripts/resolvers/preamble/generate-brain-sync-block.ts', 'scripts/resolvers/preamble.ts', 'bin/gstack-brain-sync', 'bin/gstack-artifacts-init', 'bin/gstack-config', 'test/helpers/agent-sdk-runner.ts', 'test/skill-e2e-brain-privacy-gate.test.ts',
|
||||
'test/agent-sdk-runner.test.ts'
|
||||
],
|
||||
|
||||
// /setup-gbrain Path 4 (Remote MCP) — happy + bad-token end-to-end via
|
||||
// Agent SDK. Gate-tier (deterministic stub server, fixed inputs); fires
|
||||
@@ -864,11 +841,6 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'plan-tune-inspect': ['test/session-runner-stream-lifecycle.test.ts', 'plan-tune/**', 'scripts/question-registry.ts', 'scripts/psychographic-signals.ts', 'scripts/one-way-doors.ts', 'bin/gstack-question-log', 'bin/gstack-question-preference', 'bin/gstack-developer-profile', 'test/skill-e2e-plan-tune.test.ts'],
|
||||
|
||||
// /plan-tune cathedral (T16 — 5 E2E scenarios, all gate per D12)
|
||||
'plan-tune-hook-capture': ['hosts/claude/hooks/**', 'bin/gstack-question-log', 'bin/gstack-developer-profile', 'plan-tune/**', 'test/skill-e2e-plan-tune-cathedral.test.ts', 'lib/jsonl-store.ts', 'lib/is-conductor.ts', 'test/plan-tune-cathedral-fixture.test.ts'],
|
||||
'plan-tune-enforcement': ['hosts/claude/hooks/**', 'bin/gstack-question-preference', 'scripts/question-registry.ts', 'test/skill-e2e-plan-tune-cathedral.test.ts', 'lib/jsonl-store.ts', 'lib/is-conductor.ts', 'test/plan-tune-cathedral-fixture.test.ts'],
|
||||
'plan-tune-annotation': ['hosts/claude/hooks/**', 'scripts/declared-annotation.ts', 'scripts/psychographic-signals.ts', 'scripts/question-registry.ts', 'test/skill-e2e-plan-tune-cathedral.test.ts', 'lib/jsonl-store.ts', 'lib/is-conductor.ts', 'test/plan-tune-cathedral-fixture.test.ts'],
|
||||
'plan-tune-codex-import': ['bin/gstack-codex-session-import', 'bin/gstack-question-log', 'docs/spikes/codex-session-format.md', 'test/skill-e2e-plan-tune-cathedral.test.ts', 'lib/jsonl-store.ts', 'lib/is-conductor.ts', 'test/plan-tune-cathedral-fixture.test.ts'],
|
||||
'plan-tune-dream-cycle': ['bin/gstack-distill-free-text', 'bin/gstack-distill-apply', 'hosts/claude/hooks/**', 'plan-tune/**', 'test/skill-e2e-plan-tune-cathedral.test.ts', 'lib/jsonl-store.ts', 'lib/is-conductor.ts', 'test/plan-tune-cathedral-fixture.test.ts'],
|
||||
|
||||
// Codex offering verification
|
||||
'codex-offered-office-hours': ['test/session-runner-stream-lifecycle.test.ts', 'test/paid-retry-supervision.test.ts', 'office-hours/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts',
|
||||
@@ -1013,7 +985,6 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
],
|
||||
|
||||
// Gemini E2E — smoke test only (Gemini gets lost in worktrees on complex tasks)
|
||||
'gemini-smoke': ['scripts/gen-skill-docs.ts', 'test/helpers/gemini-session-runner.ts', 'lib/worktree.ts', 'test/gemini-e2e.test.ts'],
|
||||
|
||||
|
||||
// Coverage audit (shared fixture) + triage + gates
|
||||
@@ -1264,15 +1235,16 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
|
||||
"test/review-entry-and-design-clarity-au.test.ts", "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts",
|
||||
],
|
||||
'journey-negatives': ['test/session-runner-stream-lifecycle.test.ts',
|
||||
'test/skill-fixture.test.ts',
|
||||
"test/plan-scope-recovery-av.test.ts",
|
||||
"test/fixtures/plan-scope-recovery-av.json",
|
||||
"test/fixtures/design-scope-checkpoint-at.json",'test/eng-scope-entry-ap.test.ts', '*/SKILL.md.tmpl', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-routing-e2e.test.ts',
|
||||
"test/design-scope-entry-aq.test.ts",
|
||||
|
||||
"test/review-entry-and-design-clarity-au.test.ts", "scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts",
|
||||
],
|
||||
|
||||
// Opus 4.7 behavior evals — keys match testName: values in the test file.
|
||||
// Routing sub-tests use template literal `routing-${c.name}` testNames,
|
||||
// which the touchfile completeness scanner skips; they inherit selection
|
||||
// from the file-level touchfile entry via GLOBAL_TOUCHFILES.
|
||||
'fanout-arm-overlay-on':
|
||||
['test/session-runner-stream-lifecycle.test.ts', 'model-overlays/claude.md', 'model-overlays/opus-4-7.md', 'scripts/models.ts', 'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-opus-47.test.ts'],
|
||||
'fanout-arm-overlay-off':
|
||||
['test/session-runner-stream-lifecycle.test.ts', 'model-overlays/claude.md', 'model-overlays/opus-4-7.md', 'scripts/models.ts', 'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-opus-47.test.ts'],
|
||||
|
||||
// Overlay efficacy harness (SDK) — measures whether overlay nudges change
|
||||
// behavior under @anthropic-ai/claude-agent-sdk (closer to real Claude Code
|
||||
@@ -1280,22 +1252,10 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// completeness scanner doesn't require them; these entries exist for
|
||||
// diff-based selection accuracy.
|
||||
|
||||
// /ios-qa — agent flow E2E. Daemon + stub StateServer + codegen
|
||||
// exercised end-to-end. The no-device path is gate-tier; the with-device
|
||||
// path requires GSTACK_HAS_IOS_DEVICE=1 and is periodic-tier.
|
||||
'ios-qa-e2e': ['ios-qa/**', 'ios-fix/**', 'ios-design-review/**', 'ios-clean/**', 'ios-sync/**', 'test/skill-e2e-ios.test.ts'],
|
||||
// Swift-build invariant test — requires the Swift toolchain. Compiles the
|
||||
// fixture SPM package + runs the XCTest suite that validates the real
|
||||
// Swift StateServer implementation (loopback bind, boot token rotation,
|
||||
// session lock). Periodic-tier — Swift build is heavier than TS unit tests.
|
||||
'ios-qa-swift-build': ['ios-qa/templates/**', 'test/fixtures/ios-qa/FixtureApp/**', 'test/skill-e2e-ios-swift-build.test.ts'],
|
||||
// Real-device path — only runs with GSTACK_HAS_IOS_DEVICE=1 + a paired
|
||||
// iPhone. Validates the CoreDevice agent + iOS SDK toolchain. Periodic-tier.
|
||||
'ios-qa-device': ['ios-qa/templates/**', 'test/fixtures/ios-qa/FixtureApp/**', 'test/skill-e2e-ios-device.test.ts'],
|
||||
|
||||
// /spec end-to-end via PTY — exercises the full Phase 1→5 pipeline
|
||||
// including --execute spawn. Periodic-tier — paid + non-deterministic.
|
||||
'spec-execute': ['spec/**', 'scripts/resolvers/redact-doc.ts', 'scripts/resolvers/outside-voice.ts', 'scripts/resolvers/constants.ts', 'lib/outside-review-result.ts', 'bin/gstack-redact', 'lib/redact-engine.ts', 'test/skill-e2e-spec-execute.test.ts'],
|
||||
|
||||
// /office-hours brain-writeback path under fake gbrain CLI (v1.50.0.0
|
||||
// T7). Drives /office-hours with a regenerated SKILL.md that has the
|
||||
@@ -1370,16 +1330,10 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'test/fixtures/eng-omitted-select-361c.json',
|
||||
'test/skill-e2e-plan-decision-classification.test.ts', 'test/fixtures/plan-decision-classification.ts', 'test/plan-review-calibration.test.ts', 'test/helpers/plan-review-decisions.ts', 'test/plan-review-decisions.test.ts', 'test/helpers/plan-review-cases.ts', 'test/plan-review-cases.test.ts', 'test/helpers/llm-judge.ts', 'lib/eval-model.ts', 'test/helpers/e2e-helpers.ts', 'test/helpers/eval-store.ts', 'test/helpers/eval-budgets.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'docs/askuserquestion-split.md', 'plan-ceo-review/SKILL.md.tmpl', 'plan-ceo-review/sections/review-sections.md.tmpl', 'scripts/resolvers/tasks-section.ts'],
|
||||
'health-reporting': ['health/**', 'test/skill-e2e-health.test.ts', 'test/helpers/health-eval-fixture.ts'],
|
||||
'codex-plan-ceo-format-mode': ['test/paid-retry-supervision.test.ts', 'plan-ceo-review/**', 'scripts/gen-skill-docs.ts', 'scripts/resolvers/**', 'model-overlays/gpt.md', 'model-overlays/gpt-5.4.md', 'test/helpers/codex-session-runner.ts', 'test/helpers/codex-eval.ts', 'test/codex-e2e-plan-format.test.ts'],
|
||||
'codex-plan-ceo-format-approach': ['test/paid-retry-supervision.test.ts', 'plan-ceo-review/**', 'scripts/gen-skill-docs.ts', 'scripts/resolvers/**', 'model-overlays/gpt.md', 'model-overlays/gpt-5.4.md', 'test/helpers/codex-session-runner.ts', 'test/helpers/codex-eval.ts', 'test/codex-e2e-plan-format.test.ts'],
|
||||
'codex-plan-eng-format-coverage': ['test/paid-retry-supervision.test.ts', 'scripts/resolvers/learnings.ts', 'test/review-entry-and-design-clarity-au.test.ts', 'test/fixtures/plan-scope-recovery-av.json', 'test/plan-scope-recovery-av.test.ts', 'test/eng-scope-entry-ap.test.ts', 'plan-eng-review/**', 'scripts/gen-skill-docs.ts', 'scripts/resolvers/**', 'model-overlays/gpt.md', 'model-overlays/gpt-5.4.md', 'test/helpers/codex-session-runner.ts', 'test/helpers/codex-eval.ts', 'test/codex-e2e-plan-format.test.ts', 'test/plan-review-cases.test.ts'],
|
||||
'codex-plan-eng-format-kind': ['test/paid-retry-supervision.test.ts', 'scripts/resolvers/learnings.ts', 'test/review-entry-and-design-clarity-au.test.ts', 'test/fixtures/plan-scope-recovery-av.json', 'test/plan-scope-recovery-av.test.ts', 'test/eng-scope-entry-ap.test.ts', 'plan-eng-review/**', 'scripts/gen-skill-docs.ts', 'scripts/resolvers/**', 'model-overlays/gpt.md', 'model-overlays/gpt-5.4.md', 'test/helpers/codex-session-runner.ts', 'test/helpers/codex-eval.ts', 'test/codex-e2e-plan-format.test.ts', 'test/plan-review-cases.test.ts'],
|
||||
'overlay-harness-claude-dedicated-tools-vs-bash': ['model-overlays/**', 'test/fixtures/overlay-nudges.ts', 'test/helpers/agent-sdk-runner.ts', 'test/agent-sdk-runner.test.ts', 'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash.test.ts', 'test/helpers/overlay-measurement.ts', 'test/helpers/overlay-workspace.ts', 'test/helpers/overlay-attempt.ts', 'test/overlay-measurement.test.ts', 'test/helpers/overlay-case.ts', 'test/helpers/overlay-case-policy.ts', 'test/helpers/overlay-lifecycle.ts', 'test/overlay-lifecycle.test.ts', 'test/overlay-sdk-cancel-eof.test.ts', 'test/overlay-recording-order.test.ts', 'test/paid-overlay-scheduling.test.ts', 'test/fixtures/overlay-admission-child.ts'],
|
||||
'overlay-harness-opus-4-7-effort-match-trivial': ['model-overlays/**', 'test/fixtures/overlay-nudges.ts', 'test/helpers/agent-sdk-runner.ts', 'test/agent-sdk-runner.test.ts', 'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-overlay-harness-opus-4-7-effort-match-trivial.test.ts', 'test/helpers/overlay-measurement.ts', 'test/helpers/overlay-workspace.ts', 'test/helpers/overlay-attempt.ts', 'test/overlay-measurement.test.ts', 'test/helpers/overlay-case.ts', 'test/helpers/overlay-case-policy.ts', 'test/helpers/overlay-lifecycle.ts', 'test/overlay-lifecycle.test.ts', 'test/overlay-sdk-cancel-eof.test.ts', 'test/overlay-recording-order.test.ts', 'test/paid-overlay-scheduling.test.ts', 'test/fixtures/overlay-admission-child.ts'],
|
||||
'overlay-harness-opus-4-7-literal-interpretation': ['model-overlays/**', 'test/fixtures/overlay-nudges.ts', 'test/helpers/agent-sdk-runner.ts', 'test/agent-sdk-runner.test.ts', 'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-overlay-harness-opus-4-7-literal-interpretation.test.ts', 'test/helpers/overlay-measurement.ts', 'test/helpers/overlay-workspace.ts', 'test/helpers/overlay-attempt.ts', 'test/overlay-measurement.test.ts', 'test/helpers/overlay-case.ts', 'test/helpers/overlay-case-policy.ts', 'test/helpers/overlay-lifecycle.ts', 'test/overlay-lifecycle.test.ts', 'test/overlay-sdk-cancel-eof.test.ts', 'test/overlay-recording-order.test.ts', 'test/paid-overlay-scheduling.test.ts', 'test/fixtures/overlay-admission-child.ts'],
|
||||
'overlay-harness-claude-dedicated-tools-vs-bash-sonnet': ['model-overlays/**', 'test/fixtures/overlay-nudges.ts', 'test/helpers/agent-sdk-runner.ts', 'test/agent-sdk-runner.test.ts', 'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-overlay-harness-claude-dedicated-tools-vs-bash-sonnet.test.ts', 'test/helpers/overlay-measurement.ts', 'test/helpers/overlay-workspace.ts', 'test/helpers/overlay-attempt.ts', 'test/overlay-measurement.test.ts', 'test/helpers/overlay-case.ts', 'test/helpers/overlay-case-policy.ts', 'test/helpers/overlay-lifecycle.ts', 'test/overlay-lifecycle.test.ts', 'test/overlay-sdk-cancel-eof.test.ts', 'test/overlay-recording-order.test.ts', 'test/paid-overlay-scheduling.test.ts', 'test/fixtures/overlay-admission-child.ts'],
|
||||
'overlay-harness-opus-4-7-effort-match-trivial-sonnet': ['model-overlays/**', 'test/fixtures/overlay-nudges.ts', 'test/helpers/agent-sdk-runner.ts', 'test/agent-sdk-runner.test.ts', 'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-overlay-harness-opus-4-7-effort-match-trivial-sonnet.test.ts', 'test/helpers/overlay-measurement.ts', 'test/helpers/overlay-workspace.ts', 'test/helpers/overlay-attempt.ts', 'test/overlay-measurement.test.ts', 'test/helpers/overlay-case.ts', 'test/helpers/overlay-case-policy.ts', 'test/helpers/overlay-lifecycle.ts', 'test/overlay-lifecycle.test.ts', 'test/overlay-sdk-cancel-eof.test.ts', 'test/overlay-recording-order.test.ts', 'test/paid-overlay-scheduling.test.ts', 'test/fixtures/overlay-admission-child.ts'],
|
||||
'overlay-harness-opus-4-7-literal-interpretation-sonnet': ['model-overlays/**', 'test/fixtures/overlay-nudges.ts', 'test/helpers/agent-sdk-runner.ts', 'test/agent-sdk-runner.test.ts', 'scripts/resolvers/model-overlay.ts', 'test/skill-e2e-overlay-harness-opus-4-7-literal-interpretation-sonnet.test.ts', 'test/helpers/overlay-measurement.ts', 'test/helpers/overlay-workspace.ts', 'test/helpers/overlay-attempt.ts', 'test/overlay-measurement.test.ts', 'test/helpers/overlay-case.ts', 'test/helpers/overlay-case-policy.ts', 'test/helpers/overlay-lifecycle.ts', 'test/overlay-lifecycle.test.ts', 'test/overlay-sdk-cancel-eof.test.ts', 'test/overlay-recording-order.test.ts', 'test/paid-overlay-scheduling.test.ts', 'test/fixtures/overlay-admission-child.ts'],
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -1503,7 +1457,6 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
|
||||
// v1.21+ auto-mode regression tests
|
||||
'office-hours-auto-mode': 'gate',
|
||||
'auto-decide-preserved': 'periodic',
|
||||
'conductor-prose': 'periodic',
|
||||
|
||||
// Real-PTY E2E batch — tier classification:
|
||||
// gate: cheap, deterministic, run on every PR
|
||||
@@ -1511,7 +1464,6 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
|
||||
'auq-format-gate': 'gate', // ~$0.50/run, native SDK question capture, single skill probe
|
||||
'plan-ceo-mode-routing': 'periodic', // ~$3/run, deep navigation through 8-12 prior AskUserQuestions
|
||||
'plan-design-with-ui-scope': 'gate', // ~$0.80/run
|
||||
'ship-idempotency-pty': 'periodic', // ~$3/run, real /ship in plan mode
|
||||
'tpa-present': 'gate', // consent/credential safety guardrail; deterministic shims + grep asserts
|
||||
'tpa-absent-linux': 'gate', // consent/credential safety guardrail; deterministic shims + grep asserts
|
||||
'tpa-broken': 'gate', // consent/credential safety guardrail; deterministic shims + grep asserts
|
||||
@@ -1540,7 +1492,6 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
|
||||
|
||||
// Privacy gate for gstack-brain-sync — periodic (non-deterministic LLM call,
|
||||
// costs ~$0.30-$0.50 per run, not needed on every commit)
|
||||
'brain-privacy-gate': 'periodic',
|
||||
|
||||
// /setup-gbrain Path 4 (Remote MCP) — periodic-tier. The stub HTTP
|
||||
// server is deterministic but the model's interpretation of "follow
|
||||
@@ -1578,11 +1529,6 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
|
||||
'plan-tune-inspect': 'gate',
|
||||
|
||||
// /plan-tune cathedral (T16 per D12 — all gate)
|
||||
'plan-tune-hook-capture': 'gate',
|
||||
'plan-tune-enforcement': 'gate',
|
||||
'plan-tune-annotation': 'gate',
|
||||
'plan-tune-codex-import': 'gate',
|
||||
'plan-tune-dream-cycle': 'gate',
|
||||
|
||||
// Codex offering verification
|
||||
'codex-offered-office-hours': 'gate',
|
||||
@@ -1646,7 +1592,6 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
|
||||
'outside-voice-claude-code-to-codex': 'periodic',
|
||||
'outside-plan-disabled-no-fallback': 'periodic',
|
||||
'codex-sol-scope-termination': 'periodic',
|
||||
'gemini-smoke': 'periodic',
|
||||
|
||||
// Design — gate for cheap functional, periodic for Opus/quality
|
||||
'design-consultation-core': 'periodic',
|
||||
@@ -1706,10 +1651,9 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
|
||||
'journey-retro': 'periodic',
|
||||
'journey-design-system': 'periodic',
|
||||
'journey-visual-qa': 'periodic',
|
||||
'journey-negatives': 'periodic',
|
||||
|
||||
// Opus 4.7 overlay evals — periodic (non-deterministic LLM behavior + Opus cost)
|
||||
'fanout-arm-overlay-on': 'periodic',
|
||||
'fanout-arm-overlay-off': 'periodic',
|
||||
|
||||
// Overlay efficacy harness (SDK, paid) — periodic only
|
||||
|
||||
@@ -1720,13 +1664,10 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
|
||||
// on every Linux PR. Periodic keeps it in the weekly census on capable
|
||||
// hosts; re-promote if a macOS runner lands (flagged decision in the
|
||||
// test-infra overhaul plan).
|
||||
'ios-qa-e2e': 'periodic',
|
||||
// Swift toolchain only, no device required, but heavier than TS unit tests.
|
||||
'ios-qa-swift-build': 'periodic',
|
||||
// Requires a real connected + paired iPhone. Manual-trigger only.
|
||||
'ios-qa-device': 'periodic',
|
||||
// /spec end-to-end PTY pipeline (paid, non-deterministic — periodic-tier).
|
||||
'spec-execute': 'periodic',
|
||||
|
||||
// WS2 arm benchmark — periodic: full build-shaped agentic workflows, paid,
|
||||
// non-deterministic by construction (research instrument, not a gate).
|
||||
@@ -1737,16 +1678,10 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
|
||||
'plan-decision-classification': 'periodic',
|
||||
'plan-devex-peer-comparison-classification': 'periodic',
|
||||
'health-reporting': 'periodic',
|
||||
'codex-plan-ceo-format-mode': 'periodic',
|
||||
'codex-plan-ceo-format-approach': 'periodic',
|
||||
'codex-plan-eng-format-coverage': 'periodic',
|
||||
'codex-plan-eng-format-kind': 'periodic',
|
||||
'overlay-harness-claude-dedicated-tools-vs-bash': 'periodic',
|
||||
'overlay-harness-opus-4-7-effort-match-trivial': 'periodic',
|
||||
'overlay-harness-opus-4-7-literal-interpretation': 'periodic',
|
||||
'overlay-harness-claude-dedicated-tools-vs-bash-sonnet': 'periodic',
|
||||
'overlay-harness-opus-4-7-effort-match-trivial-sonnet': 'periodic',
|
||||
'overlay-harness-opus-4-7-literal-interpretation-sonnet': 'periodic',
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -1754,16 +1689,12 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
|
||||
*/
|
||||
export const LLM_JUDGE_TOUCHFILES: Record<string, string[]> = {
|
||||
'setup-browser-cookies/SKILL.md workflow': ['setup-browser-cookies/SKILL.md.tmpl', 'setup-browser-cookies/SKILL.md', 'BROWSER.md', 'test/helpers/cookie-workflow-judge-input.ts', 'test/cookie-workflow-judge-input.test.ts', 'test/helpers/cookie-workflow-manual-review.ts', 'test/cookie-workflow-manual-review.test.ts', 'test/helpers/manual-judge-review-fixture.ts', '.github/cookie-workflow-manual-review.json', 'test/helpers/workflow-judge-input.ts', 'test/skill-llm-eval.test.ts'],
|
||||
'command reference table': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts', 'gstack/llms.txt', 'test/skill-llm-eval.test.ts'],
|
||||
'snapshot flags reference': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/snapshot.ts', 'test/skill-llm-eval.test.ts'],
|
||||
'browse/SKILL.md reference': ['browse/sections/**', 'browse/SKILL.md', 'browse/SKILL.md.tmpl', 'browse/src/**', 'test/skill-llm-eval.test.ts'],
|
||||
'browse/SKILL.md reference': ['browse/sections/**', 'browse/SKILL.md', 'browse/SKILL.md.tmpl', 'browse/src/**', 'SKILL.md', 'SKILL.md.tmpl', 'gstack/llms.txt', 'test/fixtures/eval-baselines.json', 'test/skill-llm-eval.test.ts'],
|
||||
'setup block': ['browse/SKILL.md', 'browse/SKILL.md.tmpl', 'scripts/resolvers/aside.ts', 'scripts/resolvers/browse.ts', 'test/skill-llm-eval.test.ts'],
|
||||
'regression vs baseline': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts', 'test/fixtures/eval-baselines.json', 'test/skill-llm-eval.test.ts'],
|
||||
'qa/SKILL.md workflow': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'qa/SKILL.md health rubric': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'qa/SKILL.md anti-refusal': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl', 'qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'cross-skill greptile consistency': ['review/SKILL.md', 'review/SKILL.md.tmpl', 'ship/SKILL.md', 'ship/SKILL.md.tmpl', 'review/greptile-triage.md', 'retro/SKILL.md', 'retro/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'baseline score pinning': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'test/fixtures/eval-baselines.json', 'test/skill-llm-eval.test.ts'],
|
||||
|
||||
// Ship & Release
|
||||
'ship/SKILL.md workflow': ['ship/SKILL.md', 'ship/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-judge-input.ts', 'test/helpers/workflow-judge-cache.ts', 'test/workflow-judge-cache.test.ts', 'scripts/eval-input-cache.ts', 'test/eval-input-cache.test.ts', 'test/workflow-judge-input.test.ts', 'test/helpers/workflow-excerpt.ts',
|
||||
|
||||
Reference in new issue
Block a user