mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-12 16:08:59 +02:00
v1.84.1.0 fix: default Codex and Claude to frontier models (#2835)
* fix: default cross-model workflows to frontier models * chore: bump version and changelog (v1.82.1.0) Co-Authored-By: OpenAI Codex <noreply@openai.com> * fix: repair frontier eval budgets and workflow instructions Preserve frontier models and quality thresholds while fixing truncated judge output, ordered section expansion, consent checks, QA scoring, and ship audit gates. Add regression coverage and refresh generated docs. Co-Authored-By: OpenAI Codex <noreply@openai.com> * fix: resolve workflow gaps exposed by frontier evals Clarify plan-review ordering and fallback modes, preserve deploy readiness gates, honor configured merge methods, correct benchmark and canary contracts, and restore vendored installs on setup failure. Cover recovery with real-shell regressions. Co-Authored-By: OpenAI Codex <noreply@openai.com> * fix: use agent capture budgets for deploy evals Multi-turn deploy and benchmark sessions were incorrectly limited to the single-call judge timeout. Use the existing capture tier and leave outer-test cleanup headroom, with a free policy regression test. Keep all behavioral assertions and frontier models unchanged. Co-Authored-By: OpenAI Codex <noreply@openai.com> * fix: clarify retro workflow and evaluate compare instructions Include compare mode in the frontier judge excerpt, define metric sources and snapshot ordering, and preserve the existing prompt-size budget. Co-authored-by: OpenAI Codex <noreply@openai.com> * fix: make documentation release review and publication consistent Review before commit, clarify changelog safeguards and unavailable reviewer modes, and preserve raw PR bodies across separate shell calls. Keep title sync in one shell and add regression coverage. Co-authored-by: OpenAI Codex <noreply@openai.com> --------- Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Garry Tan
co-authored by
OpenAI Codex
OpenAI Codex
parent
c8f0c4e368
commit
71f6048e8a
@@ -36,6 +36,7 @@ import {
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { resolveClaudeBinary as resolveClaudeBinaryShared } from '../../lib/claude-bin';
|
||||
import { resolveEvalModel } from '../../lib/eval-model';
|
||||
import { hermeticChildEnv } from './hermetic-env';
|
||||
import type { SkillTestResult } from './session-runner';
|
||||
|
||||
@@ -299,10 +300,9 @@ export async function runAgentSdkTest(
|
||||
const sem = getApiSemaphore();
|
||||
const maxRetries = opts.maxRetries ?? 3;
|
||||
const queryImpl: QueryProvider = opts.queryProvider ?? query;
|
||||
// Default matches session-runner's Sonnet (D1a, 2026-08): the old Opus
|
||||
// default was an inconsistency between the two runners, not a choice —
|
||||
// tests that need Opus pin it via opts.model (30+ already do).
|
||||
const model = opts.model ?? 'claude-sonnet-4-6';
|
||||
// Default matches session-runner's frontier eval fallback. Tests that need a
|
||||
// cheaper or historical model pin it via opts.model or EVALS_MODEL.
|
||||
const model = opts.model ?? process.env.EVALS_MODEL ?? resolveEvalModel('capture');
|
||||
|
||||
// NOTE on env: the SDK child gets the COMPLETE hermetic env (allowlist
|
||||
// scrub + ANTHROPIC_API_KEY + hermetic CLAUDE_CONFIG_DIR/GSTACK_HOME), with
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
/**
|
||||
* Benchmark quality judge — wraps llm-judge.ts for multi-provider scoring.
|
||||
*
|
||||
* The judge is always Anthropic SDK (claude-sonnet-4-6) for stability. It sees
|
||||
* The judge uses the shared frontier Claude eval default. It sees
|
||||
* the prompt + N provider outputs and scores each on: correctness, completeness,
|
||||
* code quality, edge case handling. 0-10 per dimension; overall = average.
|
||||
*
|
||||
@@ -9,6 +9,7 @@
|
||||
*/
|
||||
|
||||
import type { BenchmarkReport, BenchmarkEntry } from './benchmark-runner';
|
||||
import { resolveEvalModel } from '../../lib/eval-model';
|
||||
|
||||
export async function judgeEntries(report: BenchmarkReport): Promise<void> {
|
||||
if (!process.env.ANTHROPIC_API_KEY) {
|
||||
@@ -26,7 +27,7 @@ export async function judgeEntries(report: BenchmarkReport): Promise<void> {
|
||||
|
||||
const judgePrompt = buildJudgePrompt(report.prompt, successful);
|
||||
const msg = await client.messages.create({
|
||||
model: 'claude-sonnet-4-6',
|
||||
model: resolveEvalModel('judge'),
|
||||
max_tokens: 2048,
|
||||
messages: [{ role: 'user', content: judgePrompt }],
|
||||
});
|
||||
|
||||
@@ -471,7 +471,7 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
|
||||
mustPrecedeStop: ['land-deploy-confirmed'],
|
||||
mustMoveToSection: [
|
||||
'PRE-MERGE READINESS REPORT',
|
||||
'gh pr merge --squash --auto --delete-branch',
|
||||
'gh pr merge "$MERGE_FLAG" --auto --delete-branch',
|
||||
'DEPLOY INFRASTRUCTURE VALIDATION',
|
||||
],
|
||||
gateAfterStop: undefined, // operational skill
|
||||
|
||||
@@ -80,9 +80,8 @@ export interface ClaudePtyOptions {
|
||||
/**
|
||||
* Model for the spawned interactive `claude`. Without an explicit --model the
|
||||
* child inherits the operator's ~/.claude/settings.json model (e.g.
|
||||
* claude-fable-5[1m]), which can spend 5+ min in extended thinking on an empty
|
||||
* plan-mode context and blow every smoke budget. Resolution mirrors
|
||||
* session-runner.ts:144 exactly: opts.model ?? EVALS_MODEL ?? 'claude-sonnet-4-6'.
|
||||
* the operator's own settings. Resolution mirrors session-runner.ts exactly:
|
||||
* opts.model ?? EVALS_MODEL ?? resolveEvalModel('capture').
|
||||
* Pushed BEFORE extraArgs so a test-supplied --model still wins (last flag wins).
|
||||
*/
|
||||
model?: string;
|
||||
@@ -1306,10 +1305,10 @@ export async function launchClaudePty(
|
||||
|
||||
const args: string[] = [];
|
||||
// Pin the model so smokes don't inherit the operator's settings.json model
|
||||
// (see ClaudePtyOptions.model). Chain mirrors session-runner.ts:144 so PTY and
|
||||
// (see ClaudePtyOptions.model). Chain mirrors session-runner.ts so PTY and
|
||||
// `claude -p` evals always agree. Pushed before extraArgs => a test-supplied
|
||||
// --model wins (last flag wins).
|
||||
const model = opts.model ?? process.env.EVALS_MODEL ?? 'claude-sonnet-4-6';
|
||||
const model = opts.model ?? process.env.EVALS_MODEL ?? resolveEvalModel('capture');
|
||||
args.push('--model', model);
|
||||
// Permission mode: 'plan' default, null => omit flag entirely.
|
||||
const permissionMode = opts.permissionMode === undefined ? 'plan' : opts.permissionMode;
|
||||
@@ -1699,7 +1698,7 @@ export async function runPlanSkillObservation(opts: {
|
||||
*/
|
||||
initialPlanContent?: string;
|
||||
/** Override the spawned model. Defaults via launchClaudePty's chain
|
||||
* (opts.model ?? EVALS_MODEL ?? 'claude-sonnet-4-6'). */
|
||||
* (opts.model ?? EVALS_MODEL ?? resolveEvalModel('capture')). */
|
||||
model?: string;
|
||||
/** Literal tokens to track as high-water marks over the CUMULATIVE visible
|
||||
* buffer (case-sensitive). Results land in obs.tokensObserved. Use for
|
||||
|
||||
@@ -783,9 +783,9 @@ describe('launchClaudePty model pin (static tripwire)', () => {
|
||||
|
||||
test('spawn args push --model from the EVALS_MODEL fallback chain', () => {
|
||||
expect(src).toContain("args.push('--model', model)");
|
||||
// opts.model -> EVALS_MODEL -> 'claude-sonnet-4-6' (mirrors session-runner.ts:144)
|
||||
// opts.model -> EVALS_MODEL -> resolveEvalModel('capture') (mirrors session-runner.ts)
|
||||
expect(src).toMatch(
|
||||
/opts\.model\s*\?\?\s*process\.env\.EVALS_MODEL\s*\?\?\s*'claude-sonnet-4-6'/,
|
||||
/opts\.model\s*\?\?\s*process\.env\.EVALS_MODEL\s*\?\?\s*resolveEvalModel\('capture'\)/,
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ import { Readable } from 'node:stream';
|
||||
import { hermeticChildEnv } from './hermetic-env';
|
||||
import { extractSkillSections } from './skill-fixture';
|
||||
import { killProcessGroup } from '../../scripts/test-strict-output';
|
||||
import { CODEX_FRONTIER_MODEL } from '../../scripts/resolvers/constants';
|
||||
|
||||
// --- Interfaces ---
|
||||
|
||||
@@ -227,7 +228,7 @@ export async function runCodexSkill(opts: {
|
||||
// exactly that. Empirically verified against codex on this machine.
|
||||
const args = ['exec', '--json', '-s', sandbox, '--skip-git-repo-check'];
|
||||
if (ignoreUserConfig) args.push('--ignore-user-config');
|
||||
if (model) args.push('--model', model);
|
||||
args.push('--model', model ?? process.env.GSTACK_CODEX_MODEL ?? CODEX_FRONTIER_MODEL);
|
||||
for (const override of configOverrides) args.push('-c', override);
|
||||
args.push(prompt);
|
||||
|
||||
|
||||
+20
-19
@@ -11,7 +11,7 @@
|
||||
|
||||
import Anthropic from '@anthropic-ai/sdk';
|
||||
|
||||
import { resolveEvalModel } from '../../lib/eval-model';
|
||||
import { CLAUDE_FRONTIER_EVAL_MODEL, resolveEvalModel } from '../../lib/eval-model';
|
||||
|
||||
export interface JudgeScore {
|
||||
clarity: number; // 1-5
|
||||
@@ -55,37 +55,32 @@ export interface RecommendationScore {
|
||||
/**
|
||||
* Call an Anthropic model with a prompt, extract JSON response.
|
||||
* Jittered exponential backoff over three 429 retries. Model resolves via
|
||||
* lib/eval-model's `judge` kind (Sonnet default); pass a model id
|
||||
* lib/eval-model's `judge` kind (frontier Claude default); pass a model id
|
||||
* (e.g. claude-haiku-4-5-20251001) for cheaper bounded judgments like
|
||||
* judgeRecommendation.
|
||||
*/
|
||||
// Default judge model: Sonnet. D1a tried Haiku 4.5 here and the first live
|
||||
// run regressed the doc-rubric family — a controlled A/B on the identical
|
||||
// health-rubric prompt scored 2/2/2 under Haiku vs 4/3/4 under Sonnet (both
|
||||
// with coherent reasoning; Haiku is simply a harsher grader on long-document
|
||||
// rubrics, and every >=4 threshold in skill-llm-eval was calibrated against
|
||||
// months of Sonnet baselines). Per D1a's pin-on-regressors protocol the
|
||||
// default stays Sonnet; recalibrating the 25 rubrics for Haiku is separately
|
||||
// scoped work. Override per run with GSTACK_EVAL_MODEL_JUDGE; Haiku remains
|
||||
// the right default for classifier-grade duties (pty hung/working, warmup,
|
||||
// distill — see lib/eval-model.ts).
|
||||
// Default judge model: the current frontier Claude eval model. Override per run
|
||||
// with GSTACK_EVAL_MODEL_JUDGE; Haiku remains the right default for
|
||||
// classifier-grade duties (pty hung/working, warmup, distill — see
|
||||
// lib/eval-model.ts).
|
||||
export async function callJudge<T>(
|
||||
prompt: string,
|
||||
model?: string,
|
||||
opts?: { temperature?: number; max_tokens?: number },
|
||||
): Promise<T> {
|
||||
// Routed through the documented single resolution point: explicit arg >
|
||||
// GSTACK_EVAL_MODEL_JUDGE > GSTACK_EVAL_MODEL > sonnet default. The old
|
||||
// GSTACK_EVAL_MODEL_JUDGE > GSTACK_EVAL_MODEL > frontier default. The old
|
||||
// inline `GSTACK_EVAL_MODEL_JUDGE || sonnet` silently ignored the global
|
||||
// GSTACK_EVAL_MODEL override that every other eval call site honors.
|
||||
// opts (temperature/max_tokens) exist for bounded judgments like armJudge;
|
||||
// defaults preserve prior behavior.
|
||||
// Thinking and answer text share max_tokens. The old 1024-token budget
|
||||
// could be exhausted before a frontier judge emitted any JSON.
|
||||
const resolvedModel = resolveEvalModel('judge', model);
|
||||
const maxTokens = opts?.max_tokens ?? 8192;
|
||||
const client = new Anthropic();
|
||||
|
||||
const makeRequest = () => client.messages.create({
|
||||
model: resolvedModel,
|
||||
max_tokens: opts?.max_tokens ?? 1024,
|
||||
max_tokens: maxTokens,
|
||||
...(opts?.temperature !== undefined ? { temperature: opts.temperature } : {}),
|
||||
messages: [{ role: 'user', content: prompt }],
|
||||
});
|
||||
@@ -110,7 +105,13 @@ export async function callJudge<T>(
|
||||
}
|
||||
}
|
||||
|
||||
const text = response.content[0].type === 'text' ? response.content[0].text : '';
|
||||
if (response.stop_reason === 'max_tokens') {
|
||||
throw new Error(`Judge response truncated at max_tokens=${maxTokens} (model=${resolvedModel})`);
|
||||
}
|
||||
const text = response.content
|
||||
.filter(block => block.type === 'text')
|
||||
.map(block => block.text)
|
||||
.join('\n');
|
||||
const jsonMatch = text.match(/\{[\s\S]*\}/);
|
||||
if (!jsonMatch) throw new Error(`Judge returned non-JSON: ${text.slice(0, 200)}`);
|
||||
return JSON.parse(jsonMatch[0]) as T;
|
||||
@@ -369,7 +370,7 @@ export interface ArmJudgeScore {
|
||||
* point of a research instrument; a per-run judge swap silently moves the
|
||||
* ruler.
|
||||
*/
|
||||
export const ARM_JUDGE_MODEL = 'claude-sonnet-4-6';
|
||||
export const ARM_JUDGE_MODEL = CLAUDE_FRONTIER_EVAL_MODEL;
|
||||
|
||||
/** Bounded retry-on-malformed loop: total attempts, not extra retries. */
|
||||
export const ARM_JUDGE_ATTEMPTS = 2;
|
||||
@@ -468,7 +469,7 @@ export async function armJudge(
|
||||
let lastError: unknown;
|
||||
for (let attempt = 1; attempt <= ARM_JUDGE_ATTEMPTS; attempt++) {
|
||||
try {
|
||||
const raw = await call<Record<string, unknown>>(prompt, ARM_JUDGE_MODEL, { temperature: 0 });
|
||||
const raw = await call<Record<string, unknown>>(prompt, ARM_JUDGE_MODEL);
|
||||
return parseArmJudgeResponse(raw);
|
||||
} catch (err) {
|
||||
lastError = err;
|
||||
|
||||
@@ -5,6 +5,7 @@ import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import { resolveClaudeCommand } from '../../../lib/claude-bin';
|
||||
import { resolveEvalModel } from '../../../lib/eval-model';
|
||||
|
||||
/**
|
||||
* Claude adapter — wraps the `claude` CLI via claude -p.
|
||||
@@ -60,8 +61,9 @@ export class ClaudeAdapter implements ProviderAdapter {
|
||||
if (!resolved) {
|
||||
throw new Error('claude CLI not resolvable (set GSTACK_CLAUDE_BIN or install)');
|
||||
}
|
||||
const model = opts.model ?? process.env.EVALS_MODEL ?? resolveEvalModel('capture');
|
||||
const args = [...resolved.argsPrefix, '-p', '--output-format', 'json'];
|
||||
if (opts.model) args.push('--model', opts.model);
|
||||
args.push('--model', model);
|
||||
if (opts.extraArgs) args.push(...opts.extraArgs);
|
||||
|
||||
try {
|
||||
@@ -81,27 +83,27 @@ export class ClaudeAdapter implements ProviderAdapter {
|
||||
tokens: parsed.tokens,
|
||||
durationMs: Date.now() - start,
|
||||
toolCalls: parsed.toolCalls,
|
||||
modelUsed: parsed.modelUsed || opts.model || 'claude-opus-4-7',
|
||||
modelUsed: parsed.modelUsed || model,
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
const durationMs = Date.now() - start;
|
||||
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
|
||||
const stderr = e.stderr?.toString() ?? '';
|
||||
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') {
|
||||
return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model);
|
||||
return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, model);
|
||||
}
|
||||
if (/unauthorized|auth|login/i.test(stderr)) {
|
||||
return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model);
|
||||
return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, model);
|
||||
}
|
||||
if (/rate[- ]?limit|429/i.test(stderr)) {
|
||||
return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model);
|
||||
return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, model);
|
||||
}
|
||||
return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model);
|
||||
return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, model);
|
||||
}
|
||||
}
|
||||
|
||||
estimateCost(tokens: { input: number; output: number; cached?: number }, model?: string): number {
|
||||
return estimateCostUsd(tokens, model ?? 'claude-opus-4-7');
|
||||
return estimateCostUsd(tokens, model ?? resolveEvalModel('capture'));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -137,7 +139,7 @@ export class ClaudeAdapter implements ProviderAdapter {
|
||||
tokens: { input: 0, output: 0 },
|
||||
durationMs,
|
||||
toolCalls: 0,
|
||||
modelUsed: model ?? 'claude-opus-4-7',
|
||||
modelUsed: model ?? resolveEvalModel('capture'),
|
||||
error,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import { execFileSync, spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import { CODEX_FRONTIER_MODEL } from '../../../scripts/resolvers/constants';
|
||||
|
||||
/**
|
||||
* GPT adapter — wraps the OpenAI `codex` CLI (codex exec with --json output).
|
||||
@@ -36,8 +37,8 @@ export class GptAdapter implements ProviderAdapter {
|
||||
// often run in temp dirs / non-git paths), so the read-only sandbox is now
|
||||
// the only boundary preventing codex from mutating the workdir. If you ever
|
||||
// remove `-s read-only`, drop `--skip-git-repo-check` too.
|
||||
const args = ['exec', opts.prompt, '-C', opts.workdir, '-s', 'read-only', '--skip-git-repo-check', '--json'];
|
||||
if (opts.model) args.push('-m', opts.model);
|
||||
const model = opts.model ?? process.env.GSTACK_CODEX_MODEL ?? CODEX_FRONTIER_MODEL;
|
||||
const args = ['exec', opts.prompt, '-C', opts.workdir, '-s', 'read-only', '--skip-git-repo-check', '--json', '-m', model];
|
||||
if (opts.extraArgs) args.push(...opts.extraArgs);
|
||||
|
||||
try {
|
||||
@@ -53,27 +54,27 @@ export class GptAdapter implements ProviderAdapter {
|
||||
tokens: parsed.tokens,
|
||||
durationMs: Date.now() - start,
|
||||
toolCalls: parsed.toolCalls,
|
||||
modelUsed: parsed.modelUsed || opts.model || 'gpt-5.4',
|
||||
modelUsed: parsed.modelUsed || model,
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
const durationMs = Date.now() - start;
|
||||
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
|
||||
const stderr = e.stderr?.toString() ?? '';
|
||||
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') {
|
||||
return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model);
|
||||
return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, model);
|
||||
}
|
||||
if (/unauthorized|auth|login/i.test(stderr)) {
|
||||
return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model);
|
||||
return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, model);
|
||||
}
|
||||
if (/rate[- ]?limit|429/i.test(stderr)) {
|
||||
return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model);
|
||||
return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, model);
|
||||
}
|
||||
return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model);
|
||||
return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, model);
|
||||
}
|
||||
}
|
||||
|
||||
estimateCost(tokens: { input: number; output: number; cached?: number }, model?: string): number {
|
||||
return estimateCostUsd(tokens, model ?? 'gpt-5.4');
|
||||
return estimateCostUsd(tokens, model ?? CODEX_FRONTIER_MODEL);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -120,7 +121,7 @@ export class GptAdapter implements ProviderAdapter {
|
||||
tokens: { input: 0, output: 0 },
|
||||
durationMs,
|
||||
toolCalls: 0,
|
||||
modelUsed: model ?? 'gpt-5.4',
|
||||
modelUsed: model ?? CODEX_FRONTIER_MODEL,
|
||||
error,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import { Readable } from 'node:stream';
|
||||
import { getProjectEvalDir } from './eval-store';
|
||||
import { hermeticChildEnv, isHermeticEnabled } from './hermetic-env';
|
||||
import { killProcessGroup } from '../../scripts/test-strict-output';
|
||||
import { resolveEvalModel } from '../../lib/eval-model';
|
||||
|
||||
const GSTACK_DEV_DIR = path.join(os.homedir(), '.gstack-dev');
|
||||
const HEARTBEAT_PATH = path.join(GSTACK_DEV_DIR, 'e2e-live.json'); // heartbeat stays global
|
||||
@@ -136,7 +137,7 @@ export async function runSkillTest(options: {
|
||||
timeout?: number;
|
||||
testName?: string;
|
||||
runId?: string;
|
||||
/** Model to use. Defaults to claude-sonnet-4-6 (overridable via EVALS_MODEL env). */
|
||||
/** Model to use. Defaults to the frontier eval model (overridable via EVALS_MODEL env). */
|
||||
model?: string;
|
||||
/** Extra env vars merged into the spawned claude -p process. Useful for
|
||||
* per-test GSTACK_HOME overrides so the test doesn't have to spell out
|
||||
@@ -171,7 +172,7 @@ export async function runSkillTest(options: {
|
||||
process.env.CI ? Math.max(requestedGrace, STARTUP_GRACE_CI_FLOOR_MS) : requestedGrace,
|
||||
timeout,
|
||||
);
|
||||
const model = options.model ?? process.env.EVALS_MODEL ?? 'claude-sonnet-4-6';
|
||||
const model = options.model ?? process.env.EVALS_MODEL ?? resolveEvalModel('capture');
|
||||
|
||||
const startTime = Date.now();
|
||||
const startedAt = new Date().toISOString();
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
/** Inspect lettered consent options, not narration that quotes an unavailable option. */
|
||||
export function asideDriveOptions(text: string): string[] {
|
||||
const options: string[] = [];
|
||||
let current: string | undefined;
|
||||
for (const line of text.replaceAll('**', '').split('\n')) {
|
||||
const option = line.match(/^[ \t]*(?:[-*+][ \t]+)?[A-D][).][ \t]+(.*)$/);
|
||||
if (option) {
|
||||
if (current !== undefined) options.push(current);
|
||||
current = option[1];
|
||||
} else if (current !== undefined && /^[ \t]+\S/.test(line)) {
|
||||
current += ` ${line.trim()}`;
|
||||
} else if (current !== undefined) {
|
||||
options.push(current);
|
||||
current = undefined;
|
||||
}
|
||||
}
|
||||
if (current !== undefined) options.push(current);
|
||||
return options.filter(option => /\bAside\b/i.test(option) && /\b(?:drive|driving|browse|browsing|navigate|click)\b/i.test(option));
|
||||
}
|
||||
@@ -142,11 +142,11 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-ceo-mode-routing.test.ts'],
|
||||
'plan-design-with-ui-scope': ['plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-with-ui.test.ts'],
|
||||
'ship-idempotency-pty': ['ship/**', 'bin/gstack-next-version', 'bin/gstack-version-bump', 'scripts/resolvers/sections.ts', 'lib/worktree.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-ship-idempotency.test.ts'],
|
||||
'tpa-present': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'],
|
||||
'tpa-absent-linux': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'],
|
||||
'tpa-broken': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'],
|
||||
'tpa-absent-darwin': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'],
|
||||
'tpa-apple-ban': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts'],
|
||||
'tpa-present': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts', 'test/helpers/third-party-actions.ts'],
|
||||
'tpa-absent-linux': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts', 'test/helpers/third-party-actions.ts'],
|
||||
'tpa-broken': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts', 'test/helpers/third-party-actions.ts'],
|
||||
'tpa-absent-darwin': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts', 'test/helpers/third-party-actions.ts'],
|
||||
'tpa-apple-ban': ['scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts', 'test/helpers/third-party-actions.ts'],
|
||||
'ship-section-loading': ['ship/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-ship-section-loading.test.ts'],
|
||||
'plan-ceo-section-loading': ['plan-ceo-review/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-plan-ceo-review-section-loading.test.ts'],
|
||||
// Data-driven behavioral guard for the 'plan'/'prompt' carves (eng, design,
|
||||
@@ -839,30 +839,30 @@ export const LLM_JUDGE_TOUCHFILES: Record<string, string[]> = {
|
||||
'baseline score pinning': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'test/fixtures/eval-baselines.json', 'test/skill-llm-eval.test.ts'],
|
||||
|
||||
// Ship & Release
|
||||
'ship/SKILL.md workflow': ['ship/SKILL.md', 'ship/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'document-release/SKILL.md workflow': ['document-release/SKILL.md', 'document-release/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'ship/SKILL.md workflow': ['ship/SKILL.md', 'ship/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-excerpt.ts'],
|
||||
'document-release/SKILL.md workflow': ['document-release/SKILL.md', 'document-release/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-excerpt.ts'],
|
||||
|
||||
// Plan Reviews
|
||||
'plan-ceo-review/SKILL.md modes': ['plan-ceo-review/SKILL.md', 'plan-ceo-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'plan-eng-review/SKILL.md sections': ['plan-eng-review/SKILL.md', 'plan-eng-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'plan-ceo-review/SKILL.md modes': ['plan-ceo-review/SKILL.md', 'plan-ceo-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-excerpt.ts'],
|
||||
'plan-eng-review/SKILL.md sections': ['plan-eng-review/SKILL.md', 'plan-eng-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-excerpt.ts'],
|
||||
|
||||
// /spec authored-spec quality (paid LLM-judge — periodic-tier).
|
||||
'plan-design-review/SKILL.md passes': ['plan-design-review/SKILL.md', 'plan-design-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'plan-design-review/SKILL.md passes': ['plan-design-review/SKILL.md', 'plan-design-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-excerpt.ts'],
|
||||
|
||||
// Design skills
|
||||
'design-review/SKILL.md fix loop': ['design-review/SKILL.md', 'design-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'design-consultation/SKILL.md research': ['design-consultation/SKILL.md', 'design-consultation/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'design-review/SKILL.md fix loop': ['design-review/SKILL.md', 'design-review/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-excerpt.ts'],
|
||||
'design-consultation/SKILL.md research': ['design-consultation/SKILL.md', 'design-consultation/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-excerpt.ts'],
|
||||
|
||||
// Deploy skills
|
||||
'land-and-deploy/SKILL.md workflow': ['land-and-deploy/SKILL.md', 'land-and-deploy/SKILL.md.tmpl', 'land-and-deploy/sections/**', 'test/skill-llm-eval.test.ts'],
|
||||
'canary/SKILL.md monitoring loop': ['canary/SKILL.md', 'canary/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'benchmark/SKILL.md perf collection': ['benchmark/SKILL.md', 'benchmark/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'setup-deploy/SKILL.md platform setup': ['setup-deploy/SKILL.md', 'setup-deploy/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'land-and-deploy/SKILL.md workflow': ['land-and-deploy/SKILL.md', 'land-and-deploy/SKILL.md.tmpl', 'land-and-deploy/sections/**', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-excerpt.ts'],
|
||||
'canary/SKILL.md monitoring loop': ['canary/SKILL.md', 'canary/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-excerpt.ts'],
|
||||
'benchmark/SKILL.md perf collection': ['benchmark/SKILL.md', 'benchmark/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-excerpt.ts'],
|
||||
'setup-deploy/SKILL.md platform setup': ['setup-deploy/SKILL.md', 'setup-deploy/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-excerpt.ts'],
|
||||
|
||||
// Other skills
|
||||
'retro/SKILL.md instructions': ['retro/sections/**', 'retro/SKILL.md', 'retro/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'qa-only/SKILL.md workflow': ['qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'gstack-upgrade/SKILL.md upgrade flow': ['gstack-upgrade/SKILL.md', 'gstack-upgrade/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts'],
|
||||
'retro/SKILL.md instructions': ['retro/sections/**', 'retro/SKILL.md', 'retro/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-excerpt.ts'],
|
||||
'qa-only/SKILL.md workflow': ['qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-excerpt.ts'],
|
||||
'gstack-upgrade/SKILL.md upgrade flow': ['gstack-upgrade/SKILL.md', 'gstack-upgrade/SKILL.md.tmpl', 'test/skill-llm-eval.test.ts', 'test/helpers/workflow-excerpt.ts'],
|
||||
|
||||
// Voice directive
|
||||
'voice directive tone': ['scripts/resolvers/preamble.ts', 'review/SKILL.md', 'review/SKILL.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/skill-llm-eval.test.ts'],
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..', '..');
|
||||
|
||||
// Same generated two-line pointer consumed by setup-gbrain-fixture.ts.
|
||||
const STOP_POINTER =
|
||||
/^> \*\*STOP\.\*\* Before [^\n]*sections\/([a-z0-9-]+\.md)[^\n]*\n> in full\.[^\n]*/gm;
|
||||
|
||||
/** Expand on-demand sections where the agent reads them, then take the requested excerpt. */
|
||||
export function readWorkflowExcerpt(skillPath: string, startMarker: string, endMarker: string | null): string {
|
||||
const secDir = path.join(ROOT, path.dirname(skillPath), 'sections');
|
||||
const content = fs.readFileSync(path.join(ROOT, skillPath), 'utf-8').replace(STOP_POINTER, (_pointer, file: string) => {
|
||||
const body = fs.readFileSync(path.join(secDir, file), 'utf-8')
|
||||
.replace(/^<!--[^\n]*-->\n/gm, '').trim();
|
||||
if (body.length < 200) throw new Error(`${skillPath}: section ${file} is empty/stub`);
|
||||
return body;
|
||||
});
|
||||
const start = content.indexOf(startMarker);
|
||||
if (start < 0) throw new Error(`Start marker not found in ${skillPath}: "${startMarker}"`);
|
||||
const end = endMarker ? content.indexOf(endMarker, start) : content.length;
|
||||
if (end < 0) throw new Error(`End marker not found in ${skillPath}: "${endMarker}"`);
|
||||
return content.slice(start, end);
|
||||
}
|
||||
Reference in New Issue
Block a user