mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-09 02:08:19 +01:00
Merge remote-tracking branch 'origin/main' into capy/merge-train-workflow-side
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> # Conflicts: # lib/ship-policy.ts
This commit is contained in:
garrytan committed
commit
d9cfeba756
91 files changed
+7433
-328
No files matched your search
@@ -180,7 +180,7 @@ function emptyArtifactInventory(): ProducerArtifactInventory {
|
||||
}
|
||||
|
||||
/** A whole path component that is one of the helper's own identifiers: `<epoch ms>-<hex>` run and replay IDs, retired runs, hex artifact IDs with an optional lowercase-word prefix (`gitleaks-<hex>-<hex>.json`), and dependency-closure files (`<finding id>-before|after-<plan hash>.json`). */
|
||||
const HELPER_ID_COMPONENT = /^(?:\d{13}-[a-f0-9]{16}|\.retired-\d{13}-[a-f0-9]{16}-[a-f0-9]{32}|(?:[a-z]+-)*[a-f0-9]{16,64}(?:-[a-f0-9]{16})*(?:\.[a-z]+)?|[a-f0-9]{64}-(?:before|after)-[a-f0-9]{16}\.json)$/;
|
||||
const HELPER_ID_COMPONENT = /^(?:\d{13}-[a-f0-9]{16}|\.retired-\d{13}-[a-f0-9]{16}-[a-f0-9]{32}|(?:[a-z]+-)*[a-f0-9]{16,64}(?:-[a-f0-9]{16})*(?:\.[a-z]+)?|(?:[a-f0-9]{32}){1,2}-(?:before|after)-[a-f0-9]{16}\.json)$/;
|
||||
|
||||
/**
|
||||
* Helper artifact paths are `<repoId>/<runId>/...` plus the helper's public-cache/ and legacy-imports/. Components that
|
||||
|
||||
+271
-12
@@ -10,7 +10,12 @@
|
||||
* bun run scripts/gate-cost-replay.ts --runs <id,id,...> [--since YYYY-MM-DD --until YYYY-MM-DD | --prs 3201,3202]
|
||||
* [--ref origin/main] [--repo owner/name] [--cache .context/gate-cost-cache] [--window-days N]
|
||||
* [--usd-per-second R]
|
||||
* [--train-size 5] [--reps 400] [--seed 1] [--cases] [--json]
|
||||
* [--train-size 5] [--reps 400] [--seed 1] [--cases] [--history] [--json]
|
||||
*
|
||||
* --history (needs --since/--until) adds the red-head replay and the
|
||||
* READY-arrival train replay (lib/gate-cost-history.ts): it lists the
|
||||
* window's E2E Evals and Free Tests runs and PRs, downloads report-verdict
|
||||
* and trial-outcomes artifacts of red runs, and fetches PR heads by SHA.
|
||||
*
|
||||
* --runs names complete full-gate runs (108 rule cases + 24 judges). Bare
|
||||
* dates are UTC days; --until is inclusive. Cases without a recorded cost are
|
||||
@@ -23,14 +28,19 @@ import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import {
|
||||
SCREEN_LABEL, TODAY_WEEKLY_USD, AFTER_GSTA48_WEEKLY_USD, caseCostTable, changedLinesByFile, fullGateUsd, isSkillMd,
|
||||
parseEvalReport, parseTrialOutcomes, prSelectionCost, simulateTrains, summarizeGates, summarizeSelector,
|
||||
parseEvalReport, parseTrialOutcomes, prSelectionCost, screenSelection, simulateTrains, summarizeGates, summarizeSelector, todaySelection,
|
||||
type CaseCostEntry, type GateRecord, type GateSummary, type PrDiff, type PrSelectionCost, type ScreenPolicy,
|
||||
type SelectionDeps, type SelectorSummary, type TrainModelResult,
|
||||
} from '../lib/gate-cost';
|
||||
import { GH_HISTORY, downloadRunArtifacts, repoSlug } from './lib/ci-history';
|
||||
import {
|
||||
STAMP_FILES, classifyReds, headGreenAt, latencySummary, parseReportVerdict, prForRun, readyTimes, screenHit, simulateArrivalTrains,
|
||||
stampFreeSignature, summarizeRedHeads, SCREEN_CANARY_CASES,
|
||||
type ArrivalTrainResult, type CheckRun, type GatedRun, type HeadDiff, type LatencySummary, type ReadyRow, type RedHeadRow, type RedHeadSummary, type RedRun,
|
||||
} from '../lib/gate-cost-history';
|
||||
import { GH_CLIENT, GH_HISTORY, downloadRunArtifacts, repoSlug } from './lib/ci-history';
|
||||
import { computePaidCaseSelection } from './lib/paid-select';
|
||||
import { packageChangeOnlyVersion } from './test-pr-profile';
|
||||
import { E2E_TIERS, GLOBAL_TOUCHFILES, LLM_JUDGE_TOUCHFILES, matchGlob } from '../test/helpers/touchfiles';
|
||||
import { E2E_TIERS, E2E_TOUCHFILES, GLOBAL_TOUCHFILES, LLM_JUDGE_TOUCHFILES, matchGlob } from '../test/helpers/touchfiles';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const DAY_MS = 86_400_000;
|
||||
@@ -55,12 +65,13 @@ export interface ReplayArgs {
|
||||
reps: number;
|
||||
seed: number;
|
||||
cases: boolean;
|
||||
history: boolean;
|
||||
json: boolean;
|
||||
}
|
||||
|
||||
export function parseArgs(argv: string[]): ReplayArgs {
|
||||
const args: ReplayArgs = { runs: [], prs: [], ref: 'origin/main', repo: '', cache: path.join(ROOT, '.context', 'gate-cost-cache'),
|
||||
trainSize: 5, reps: 400, seed: 1, cases: false, json: false };
|
||||
trainSize: 5, reps: 400, seed: 1, cases: false, history: false, json: false };
|
||||
const list = (value: string) => value.split(/[\s,]+/).filter(Boolean);
|
||||
const positive = (flag: string, value: string) => {
|
||||
const n = Number(value);
|
||||
@@ -71,6 +82,7 @@ export function parseArgs(argv: string[]): ReplayArgs {
|
||||
const flag = argv[i]!;
|
||||
if (flag === '--json') { args.json = true; continue; }
|
||||
if (flag === '--cases') { args.cases = true; continue; }
|
||||
if (flag === '--history') { args.history = true; continue; }
|
||||
const value = argv[i + 1];
|
||||
if (value === undefined || value.startsWith('--')) throw new Error(`${flag} needs a value`);
|
||||
i += 1;
|
||||
@@ -93,6 +105,7 @@ export function parseArgs(argv: string[]): ReplayArgs {
|
||||
if (args.prs.some(pr => !Number.isInteger(pr) || pr <= 0)) throw new Error('--prs takes PR numbers');
|
||||
if (args.prs.length && (args.since || args.until)) throw new Error('use either --prs or --since/--until, not both');
|
||||
if (!!args.since !== !!args.until) throw new Error('--since and --until go together');
|
||||
if (args.history && !args.since) throw new Error('--history needs a window: --since and --until');
|
||||
return args;
|
||||
}
|
||||
|
||||
@@ -184,20 +197,23 @@ function mergedPrs(args: ReplayArgs): { prs: MergedPr[]; windowDays: number | nu
|
||||
return { prs: args.prs.map(pr => all.get(pr)!), windowDays: args.windowDays ?? null };
|
||||
}
|
||||
|
||||
function prDiff(merge: MergedPr): PrDiff {
|
||||
const files = git(['diff', '--name-only', '--no-renames', `${merge.sha}^`, merge.sha]).split('\n').filter(Boolean);
|
||||
/** The PR diff the selectors see between two revisions. */
|
||||
function revisionDiff(pr: number, from: string, to: string): PrDiff {
|
||||
const files = git(['diff', '--name-only', '--no-renames', from, to]).split('\n').filter(Boolean);
|
||||
let packageVersionOnly = false;
|
||||
if (files.includes('package.json')) {
|
||||
const show = (rev: string) => spawnSync('git', ['show', `${rev}:package.json`], { cwd: ROOT, encoding: 'utf8', timeout: 30_000 });
|
||||
const before = show(`${merge.sha}^`), after = show(merge.sha);
|
||||
const before = show(from), after = show(to);
|
||||
packageVersionOnly = before.status === 0 && after.status === 0 && packageChangeOnlyVersion(before.stdout, after.stdout);
|
||||
}
|
||||
const skillMds = files.filter(isSkillMd);
|
||||
const skillMdChangedLines = skillMds.length >= 10
|
||||
? changedLinesByFile(git(['diff', '-U0', '--no-renames', `${merge.sha}^`, merge.sha, '--', ...skillMds], 120_000)) : {};
|
||||
return { pr: merge.pr, sha: merge.sha.slice(0, 8), files, packageVersionOnly, skillMdChangedLines };
|
||||
? changedLinesByFile(git(['diff', '-U0', '--no-renames', from, to, '--', ...skillMds], 120_000)) : {};
|
||||
return { pr, sha: to.slice(0, 8), files, packageVersionOnly, skillMdChangedLines };
|
||||
}
|
||||
|
||||
const prDiff = (merge: MergedPr): PrDiff => revisionDiff(merge.pr, `${merge.sha}^`, merge.sha);
|
||||
|
||||
/** The selectors in this checkout: today's PR profile (with R4's shadow) and the plain touchfile selection. */
|
||||
export function checkoutSelectionDeps(rootDir = ROOT): SelectionDeps {
|
||||
const gateCases = Object.keys(E2E_TIERS).filter(id => E2E_TIERS[id] === 'gate');
|
||||
@@ -236,6 +252,7 @@ export interface ReplayReport {
|
||||
prs: PrSelectionCost[];
|
||||
selection: { today: SelectorSummary; shadow: SelectorSummary; screenProvisional: SelectorSummary } | null;
|
||||
model: { screenScale: number; fullGateUsd: number; trainSize: number; reps: number; seed: number; rows: TrainModelResult[] } | null;
|
||||
history?: HistoryReport;
|
||||
}
|
||||
|
||||
export function buildReport(args: ReplayArgs, gates: GateRecord[], merges: ReturnType<typeof mergedPrs>, deps: SelectionDeps,
|
||||
@@ -333,6 +350,244 @@ export function formatReport(report: ReplayReport, options: { cases?: boolean }
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
// --- History (--history): red-head replay and READY-arrival trains ---
|
||||
|
||||
const REPORT_ARTIFACT = /^report-verdict-a\d+$/;
|
||||
const TRIALS_ARTIFACT = /^trial-outcomes[A-Za-z0-9_-]*?-a\d+$/;
|
||||
const TRAIN_POLICIES: ScreenPolicy[] = [{ kind: 'all' }, { kind: 'random', fraction: 0.5 }, { kind: 'none' }];
|
||||
const GH_RUN_PAGE = 100;
|
||||
const HISTORY_GIT_TIMEOUT_MS = 120_000;
|
||||
const FETCH_TIMEOUT_MS = 300_000;
|
||||
|
||||
interface HistoryRun { id: number; sha: string; branch: string; createdAt: string; updatedAt: string; conclusion: string | null; prNumbers: number[] }
|
||||
interface PullRow { number: number; ref: string; head: string; createdAt: string; mergedAt: string | null; closedAt: string | null }
|
||||
type Merges = ReturnType<typeof mergedPrs>;
|
||||
|
||||
export interface HistoryReport {
|
||||
window: { since: string; until: string };
|
||||
runs: { evals: number; gated: number; red: number; free: number; unmapped: number };
|
||||
redRuns: RedRun[];
|
||||
redHeads: RedHeadRow[];
|
||||
redSummary: RedHeadSummary;
|
||||
perPr: Array<{ pr: number; paidRelevant: boolean; prCausedRed: boolean; screenedMiss: boolean }>;
|
||||
ready: ReadyRow[];
|
||||
baseline: LatencySummary & { gatesAfterReady: number; sources: Record<string, number> };
|
||||
trainParams: { trainSize: number; fillLimitHours: number; gateMinutes: number; bisectMinutes: number; redScreened: number; redUnscreened: number; reps: number; seed: number };
|
||||
trains: ArrivalTrainResult[];
|
||||
}
|
||||
|
||||
function tryGit(args: string[], timeout = HISTORY_GIT_TIMEOUT_MS): string | null {
|
||||
const result = spawnSync('git', args, { cwd: ROOT, encoding: 'utf8', timeout, maxBuffer: 256 * 1024 * 1024 });
|
||||
return result.status === 0 ? result.stdout : null;
|
||||
}
|
||||
|
||||
function cachedJson<T>(file: string, load: () => T): T {
|
||||
if (fs.existsSync(file)) return JSON.parse(fs.readFileSync(file, 'utf8')) as T;
|
||||
const value = load();
|
||||
fs.writeFileSync(file, JSON.stringify(value));
|
||||
return value;
|
||||
}
|
||||
|
||||
/** Completed runs of one workflow, one UTC day per query: GitHub caps a filtered run listing at 1,000 results. */
|
||||
function listRunsByDay(repo: string, workflow: string, since: Date, until: Date, event?: string): HistoryRun[] {
|
||||
const out: HistoryRun[] = [];
|
||||
for (let day = new Date(since.toISOString().slice(0, 10)); day < until; day = new Date(day.getTime() + DAY_MS)) {
|
||||
const date = day.toISOString().slice(0, 10);
|
||||
for (let page = 1; page <= 10; page += 1) {
|
||||
const query = [`status=completed`, `per_page=${GH_RUN_PAGE}`, `page=${page}`, `created=${date}..${date}`, ...(event ? [`event=${event}`] : [])];
|
||||
const body = GH_CLIENT.getJson(`repos/${repo}/actions/workflows/${workflow}/runs?${query.join('&')}`) as { workflow_runs?: any[] };
|
||||
const runs = Array.isArray(body?.workflow_runs) ? body.workflow_runs : [];
|
||||
out.push(...runs.map(run => ({ id: Number(run.id), sha: String(run.head_sha ?? ''), branch: String(run.head_branch ?? ''), createdAt: String(run.created_at ?? ''),
|
||||
updatedAt: String(run.updated_at ?? ''), conclusion: run.conclusion ?? null,
|
||||
prNumbers: Array.isArray(run.pull_requests) ? run.pull_requests.map((pr: any) => Number(pr.number)).filter(Number.isFinite) : [] })));
|
||||
if (runs.length < GH_RUN_PAGE) break;
|
||||
}
|
||||
}
|
||||
return out.filter(run => run.createdAt >= since.toISOString() && run.createdAt < until.toISOString());
|
||||
}
|
||||
|
||||
function listPulls(repo: string, since: Date): PullRow[] {
|
||||
const out: PullRow[] = [];
|
||||
for (let page = 1; page <= 30; page += 1) {
|
||||
const body = GH_CLIENT.getJson(`repos/${repo}/pulls?state=all&sort=updated&direction=desc&per_page=100&page=${page}`) as any[];
|
||||
if (!Array.isArray(body)) break;
|
||||
out.push(...body.map(p => ({ number: Number(p.number), ref: String(p.head?.ref ?? ''), head: String(p.head?.sha ?? ''), createdAt: String(p.created_at ?? ''),
|
||||
mergedAt: p.merged_at ?? null, closedAt: p.closed_at ?? null })));
|
||||
if (body.length < 100 || String(body[body.length - 1]?.updated_at ?? '') < since.toISOString()) break;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Artifacts of one run matching a name pattern, downloaded once (a .history marker records that the run was listed). */
|
||||
function runArtifactDirs(runId: number, repo: string, cache: string, match: RegExp): string[] {
|
||||
const dir = path.join(cache, String(runId));
|
||||
const marker = path.join(dir, `.history-${match.source.replace(/[^a-z]/gi, '')}`);
|
||||
if (!fs.existsSync(marker)) {
|
||||
downloadRunArtifacts({ repo, run: { id: runId }, cacheDir: cache, match: name => match.test(name), maxBytes: 16 * 1024 * 1024, fetcher: GH_HISTORY });
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(marker, '');
|
||||
}
|
||||
return fs.readdirSync(dir).filter(name => match.test(name)).sort((a, b) => Number(a.split('-a').pop()) - Number(b.split('-a').pop())).map(name => path.join(dir, name));
|
||||
}
|
||||
|
||||
/** The run's report verdict over its attempts: the last attempt's verdict, every attempt's failures; null when no report exists. */
|
||||
function runVerdict(runId: number, repo: string, cache: string): { green: boolean; failures: GatedRun['failures'] } | null {
|
||||
const reports = runArtifactDirs(runId, repo, cache, REPORT_ARTIFACT).map(dir => path.join(dir, 'report.txt')).filter(file => fs.existsSync(file))
|
||||
.map(file => parseReportVerdict(fs.readFileSync(file, 'utf8')));
|
||||
if (!reports.length) return null;
|
||||
return { green: reports[reports.length - 1]!.verdict === 'GREEN', failures: reports.flatMap(r => r.failures) };
|
||||
}
|
||||
|
||||
/** The tested merge commit of a run, from its trial outcomes. */
|
||||
function testedMergeSha(runId: number, repo: string, cache: string): string | null {
|
||||
for (const dir of runArtifactDirs(runId, repo, cache, TRIALS_ARTIFACT)) {
|
||||
const file = path.join(dir, 'trial-outcomes.jsonl');
|
||||
const first = fs.existsSync(file) ? fs.readFileSync(file, 'utf8').split('\n').find(Boolean) : undefined;
|
||||
try { const sha = first ? JSON.parse(first).sha : null; if (typeof sha === 'string' && /^[0-9a-f]{40}$/.test(sha)) return sha; } catch { /* torn line */ }
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Fetch commits this clone lacks by SHA (GitHub serves PR heads and merge refs that no branch holds anymore). */
|
||||
function ensureCommits(shas: string[]): void {
|
||||
const missing = [...new Set(shas)].filter(sha => tryGit(['cat-file', '-e', `${sha}^{commit}`], 30_000) === null);
|
||||
for (let i = 0; i < missing.length; i += 20) spawnSync('git', ['fetch', '-q', 'origin', ...missing.slice(i, i + 20)], { cwd: ROOT, timeout: FETCH_TIMEOUT_MS });
|
||||
}
|
||||
|
||||
function headDiff(sha: string, ref: string): HeadDiff | null {
|
||||
const mergeBase = tryGit(['merge-base', ref, sha])?.trim();
|
||||
if (!mergeBase) return null;
|
||||
const files = tryGit(['diff', '--name-only', '--no-renames', mergeBase, sha]);
|
||||
const diff = tryGit(['diff', '-U0', '--no-color', '--no-renames', '--no-ext-diff', mergeBase, sha, '--', '.', ...STAMP_FILES.map(file => `:(exclude)${file}`)]);
|
||||
return files === null || diff === null ? null : { ownSig: stampFreeSignature(diff), files: files.split('\n').filter(Boolean) };
|
||||
}
|
||||
|
||||
/** The base a red run tested: its merge commit's first parent, else main's first-parent tip when the run was created. */
|
||||
function testedBase(run: RedRun, args: ReplayArgs, repo: string): { base: string; source: RedHeadRow['baseSource'] } | null {
|
||||
const merge = testedMergeSha(run.runId, repo, args.cache);
|
||||
if (merge) {
|
||||
ensureCommits([merge]);
|
||||
const parents = (tryGit(['rev-list', '--parents', '-n1', merge]) ?? '').trim().split(' ').slice(1);
|
||||
if (parents.length === 2 && parents[1] === run.head) return { base: parents[0]!, source: 'tested-merge' };
|
||||
}
|
||||
const tip = tryGit(['rev-list', '-1', '--first-parent', `--before=${run.createdAt}`, args.ref])?.trim();
|
||||
return tip ? { base: tip, source: 'main-at-run' } : null;
|
||||
}
|
||||
|
||||
function collectHistory(args: ReplayArgs, repo: string, merges: Merges, body: Omit<ReplayReport, 'repo' | 'ref' | 'head'>, deps: SelectionDeps, gates: GateRecord[]): HistoryReport {
|
||||
const since = merges.since!, until = merges.until!;
|
||||
const key = `${since.toISOString()}_${until.toISOString()}`.replace(/[^0-9TZ_]/g, '');
|
||||
const cached = <T>(name: string, load: () => T) => cachedJson(path.join(args.cache, `history-${key}-${name}.json`), load);
|
||||
const evals = cached('evals', () => listRunsByDay(repo, 'evals.yml', since, until));
|
||||
const free = cached('free', () => listRunsByDay(repo, 'free-tests.yml', since, until, 'pull_request'));
|
||||
const pulls = cached('pulls', () => listPulls(repo, since));
|
||||
const prOf = new Map([...evals, ...free].map(run => [run.id, prForRun(run, pulls)]));
|
||||
const gated: GatedRun[] = evals.flatMap(run => {
|
||||
const pr = prOf.get(run.id) ?? null;
|
||||
if (run.conclusion === 'success') return [{ id: run.id, pr, head: run.sha, createdAt: run.createdAt, green: true, failures: [] }];
|
||||
const verdict = run.conclusion === 'failure' ? runVerdict(run.id, repo, args.cache) : null;
|
||||
return verdict ? [{ id: run.id, pr, head: run.sha, createdAt: run.createdAt, ...verdict }] : [];
|
||||
});
|
||||
const mergedSet = new Set(merges.prs.map(m => m.pr));
|
||||
const pullOf = new Map(pulls.map(p => [p.number, p]));
|
||||
const checkRuns = [...evals.map(run => ({ run, workflow: 'evals' as const })), ...free.map(run => ({ run, workflow: 'free' as const }))]
|
||||
.filter(({ run }) => mergedSet.has(prOf.get(run.id) ?? -1));
|
||||
const heads = [...new Set([...gated.filter(r => r.pr !== null).map(r => r.head), ...checkRuns.map(({ run }) => run.sha),
|
||||
...merges.prs.map(m => pullOf.get(m.pr)?.head ?? '')].filter(Boolean))];
|
||||
ensureCommits(heads);
|
||||
const diffs = cached('heads', () => Object.fromEntries(heads.map(sha => [sha, headDiff(sha, args.ref)])));
|
||||
const headMap = new Map(Object.entries(diffs).filter((e): e is [string, HeadDiff] => e[1] !== null));
|
||||
const caseFiles = new Map(gates.flatMap(g => g.trials.map(t => [t.caseId, t.file] as const)));
|
||||
const caseFile = (id: string) => caseFiles.get(id) ?? null;
|
||||
const inputsOf = (id: string) => /^test\/.+\.test\.ts$/.test(id)
|
||||
? [id, ...[...caseFiles].filter(([, file]) => file === id).flatMap(([c]) => E2E_TOUCHFILES[c] ?? [])]
|
||||
: [...(E2E_TOUCHFILES[id] ?? []), ...(caseFile(id) ? [caseFile(id)!] : [])];
|
||||
const redRuns = classifyReds(gated, {
|
||||
heads: headMap,
|
||||
delta: (from, to) => tryGit(['diff', '--name-only', '--no-renames', from, to])?.split('\n').filter(Boolean) ?? null,
|
||||
touches: (id, files) => files.filter(file => [...inputsOf(id), ...GLOBAL_TOUCHFILES].some(pattern => matchGlob(file, pattern))),
|
||||
});
|
||||
const redHeads: RedHeadRow[] = redRuns.filter(run => run.cause === 'pr').flatMap(run => {
|
||||
const tested = testedBase(run, args, repo);
|
||||
const mergeBase = tested && tryGit(['merge-base', tested.base, run.head])?.trim();
|
||||
if (!tested || !mergeBase) return [];
|
||||
const diff = revisionDiff(run.pr, mergeBase, run.head);
|
||||
const selection = screenSelection(diff, deps, todaySelection(diff, deps));
|
||||
return run.cases.filter(c => c.cause === 'pr').map(c => ({ caseId: c.caseId, kind: c.kind, pr: run.pr, runId: run.runId, head: run.head.slice(0, 8),
|
||||
base: tested.base.slice(0, 8), baseSource: tested.source, files: diff.files.length, hit: screenHit(c, selection, caseFile), canaryReasons: selection.canaryReasons }));
|
||||
});
|
||||
const selected = new Map(body.prs.map(row => [row.pr, row]));
|
||||
const merged = merges.prs.map(m => ({ pr: m.pr, paidRelevant: (selected.get(m.pr)?.today.usd ?? 0) > 0 }));
|
||||
const redSummary = summarizeRedHeads(redHeads, redRuns, merged);
|
||||
const ready = readyTimes(merges.prs.flatMap(m => {
|
||||
const pull = pullOf.get(m.pr);
|
||||
if (!pull?.mergedAt) return [];
|
||||
const prHeads = [...new Set([...checkRuns.filter(({ run }) => prOf.get(run.id) === m.pr).map(({ run }) => run.sha), pull.head])];
|
||||
return [{ pr: m.pr, mergedAt: pull.mergedAt, finalHead: pull.head, heads: prHeads.map(sha => ({ sha, ownSig: headMap.get(sha)?.ownSig ?? null })),
|
||||
gatedRuns: gated.filter(r => r.pr === m.pr).map(r => r.createdAt) }];
|
||||
}), headGreenAt(checkRuns.map(({ run, workflow }): CheckRun => ({ head: run.sha, workflow, createdAt: run.createdAt, completedAt: run.updatedAt, conclusion: run.conclusion }))));
|
||||
const withReady = ready.filter(row => row.ready !== null);
|
||||
const sources: Record<string, number> = {};
|
||||
for (const row of ready) sources[row.source] = (sources[row.source] ?? 0) + 1;
|
||||
const scale = body.model?.screenScale ?? 1;
|
||||
const arrivals = withReady.map(row => ({ pr: row.pr, readyMs: Date.parse(row.ready!), paidRelevant: (selected.get(row.pr)?.today.usd ?? 0) > 0,
|
||||
screenUsd: (selected.get(row.pr)?.screenProvisional.usd ?? 0) * scale }));
|
||||
const trainParams = { trainSize: args.trainSize, fillLimitHours: 3, gateMinutes: 45, bisectMinutes: 45, redScreened: 0.06, redUnscreened: 0.25, reps: args.reps, seed: args.seed };
|
||||
return {
|
||||
window: { since: since.toISOString(), until: until.toISOString() },
|
||||
runs: { evals: evals.length, gated: gated.length, red: gated.filter(r => !r.green).length, free: free.length, unmapped: evals.filter(run => prOf.get(run.id) === null).length },
|
||||
redRuns, redHeads, redSummary,
|
||||
perPr: merged.map(m => ({ ...m, prCausedRed: redSummary.prsWithPrRed.includes(m.pr), screenedMiss: redSummary.missedPrs.includes(m.pr) })),
|
||||
ready, baseline: { ...latencySummary(withReady.map(row => row.readyToMergeHours!)), gatesAfterReady: withReady.reduce((sum, row) => sum + row.gatesAfterReady, 0), sources },
|
||||
trainParams,
|
||||
trains: TRAIN_POLICIES.flatMap(policy => MODEL_OUTSIDE_RED.map(outsideRed => simulateArrivalTrains({ ...trainParams, arrivals, policy, outsideRed, windowDays: merges.windowDays! }))),
|
||||
};
|
||||
}
|
||||
|
||||
export function formatHistory(h: HistoryReport): string {
|
||||
const s = h.redSummary;
|
||||
const label = SCREEN_LABEL;
|
||||
const share = (n: number, d: number) => `${n} of ${d} (${pct(n, d)})`;
|
||||
const causes = (cause: string) => h.redRuns.filter(run => run.cause === cause).length;
|
||||
const out = [`History replays, E2E Evals and Free Tests runs created ${h.window.since} to ${h.window.until}:`,
|
||||
` E2E Evals runs ${h.runs.evals} (${h.runs.unmapped} not mapped to a PR), gated ${h.runs.gated}, red ${h.runs.red}: PR-caused ${causes('pr')}, outside ${causes('outside')}, unknown ${causes('unknown')}; Free Tests PR runs ${h.runs.free}`,
|
||||
' Classification (GSTA-47): per failed case, first match wins: transcript diagnosis; red in another PR within 72 h -> outside;',
|
||||
' no later gated run -> unknown; next gated run has the same own diff -> outside; the push to it changed the case\'s inputs',
|
||||
' (touchfiles, test file, harness globals) among either head\'s own files -> PR-caused; else outside. A run with any PR-caused case is PR-caused.',
|
||||
'', `Red-head replay (${label} on the failing head's diff from its tested base; canary set fixed before the replay: ${Object.values(SCREEN_CANARY_CASES).join(', ')}):`,
|
||||
` ${pad(['PR', 'run', 'head', 'base', 'files', 'case', 'kind', `${label} hit`], [6, 12, 9, 22, 5, 46, 6])}`];
|
||||
for (const row of h.redHeads) {
|
||||
out.push(` ${pad([`#${row.pr}`, row.runId, row.head, `${row.base} ${row.baseSource}`, row.files, row.caseId, row.kind, row.hit ?? 'MISSED'], [6, 12, 9, 22, 5, 46, 6])}`);
|
||||
}
|
||||
out.push(` ${label} hits by kind: ${Object.entries(s.byHit).map(([k, n]) => `${k} ${n}`).join(', ') || 'none'}`,
|
||||
` ${label} misses: ${share(s.missedPrs.length, s.prsWithPrRed.length)} merged PRs with a PR-caused red (${s.missedCases} of ${s.prCausedCases} case results)${s.missedPrs.length ? `: #${s.missedPrs.join(', #')}` : ''}`,
|
||||
` ${label} misses with an empty canary set: ${s.missedPrsWithoutCanary.length}${s.missedPrsWithoutCanary.length ? ` (#${s.missedPrsWithoutCanary.join(', #')})` : ''}`,
|
||||
` P-i (${label} misses over paid-relevant merged PRs): ${share(s.missedPrs.length, s.paidRelevantPrs)}`,
|
||||
` P-k (merged PRs with any PR-caused red run): ${share(s.prsWithPrRed.length, s.mergedPrs)}${s.prsWithPrRed.length ? `: #${s.prsWithPrRed.join(', #')}` : ''}; ${s.prCausedRuns} PR-caused red runs`,
|
||||
` PR-caused red runs on PRs that did not merge in the window (not counted): ${s.otherPrCausedRuns}`,
|
||||
'', `Per merged PR (paid-relevant: today's selector picks a case on the squash; red: any PR-caused red run; miss: a ${label} miss):`);
|
||||
for (let i = 0; i < h.perPr.length; i += 6) {
|
||||
out.push(` ${h.perPr.slice(i, i + 6).map(p => `#${p.pr} ${p.paidRelevant ? 'paid' : '----'} ${p.prCausedRed ? 'red ' : '--- '}${p.screenedMiss ? 'miss' : '----'}`).join(' ')}`);
|
||||
}
|
||||
const b = h.baseline;
|
||||
const hrs = (v: number) => v.toFixed(1);
|
||||
const p = h.trainParams;
|
||||
out.push('', 'READY-to-merge. READY = when the merged own diff (modulo main merges and stamps) first had green Free Tests and E2E Evals on a',
|
||||
' head, before the merge (fallback: the final head\'s green time; READY reports are not machine-readable).',
|
||||
` READY signal: ${Object.entries(b.sources).map(([k, n]) => `${k} ${n}`).join(', ')}`,
|
||||
` today (one PR per gate): ${b.prs} PRs, READY-to-merge p50 ${hrs(b.p50Hours)} h, p90 ${hrs(b.p90Hours)} h, mean ${hrs(b.meanHours)} h; ${b.gatesAfterReady} gated E2E Evals runs after READY`,
|
||||
` trains (${p.reps} reps, seed ${p.seed}; size ${p.trainSize}, fill limit ${p.fillLimitHours} h, screened first, unscreened alone when ${p.trainSize}+ wait; ${p.gateMinutes} min a gate,`,
|
||||
` ${p.bisectMinutes} min a bisect, one train gating at a time; defects ${p.redScreened * 100}% screened / ${p.redUnscreened * 100}% unscreened; a dropped PR rejoins at once):`,
|
||||
` ${pad(['policy', 'o', 'trains/w', 'occupancy', 'full', 'sizes 1..k per window', 'gates/w', 'drop+gate/w', 'rebld/w', 'stalls/w', 'p50 h', 'p90 h'], [10, 5, 8, 9, 5, 22, 7, 11, 7, 8, 6])} mean h`);
|
||||
for (const r of h.trains) {
|
||||
out.push(` ${pad([r.policy, r.outsideRed.toFixed(2), r.trainsWeekly.toFixed(2), r.meanOccupancy.toFixed(2), pct(r.fullTrainShare, 1), r.occupancy.map(n => n.toFixed(1)).join('/'),
|
||||
r.gateAttemptsWeekly.toFixed(2), r.attributedRebuildsWeekly.toFixed(2), r.rebuildOnceWeekly.toFixed(2), r.stallsWeekly.toFixed(2), hrs(r.readyToMerge.p50Hours), hrs(r.readyToMerge.p90Hours)],
|
||||
[10, 5, 8, 9, 5, 22, 7, 11, 7, 8, 6])} ${hrs(r.readyToMerge.meanHours)}`);
|
||||
}
|
||||
out.push(` "screened" follows the ${label} policy column; PRs with no paid case ride as screened.`);
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
export function main(argv: string[]): number {
|
||||
let args: ReplayArgs;
|
||||
try { args = parseArgs(argv); } catch (error) {
|
||||
@@ -346,8 +601,12 @@ export function main(argv: string[]): number {
|
||||
const empty = gates.filter(g => g.trials.length === 0);
|
||||
if (empty.length) throw new Error(`run(s) ${empty.map(g => g.runId).join(', ')} have no trial outcomes (expired artifacts, or not a paid gate run)`);
|
||||
const merges = mergedPrs(args);
|
||||
const report: ReplayReport = { repo, ref: args.ref, head: git(['rev-parse', args.ref]).trim(), ...buildReport(args, gates, merges, checkoutSelectionDeps()) };
|
||||
console.log(args.json ? JSON.stringify(report, null, 2) : formatReport(report, { cases: args.cases }));
|
||||
const deps = checkoutSelectionDeps();
|
||||
const body = buildReport(args, gates, merges, deps);
|
||||
const report: ReplayReport = { repo, ref: args.ref, head: git(['rev-parse', args.ref]).trim(), ...body,
|
||||
...(args.history ? { history: collectHistory(args, repo, merges, body, deps, gates) } : {}) };
|
||||
const text = formatReport(report, { cases: args.cases }) + (report.history ? `\n\n${formatHistory(report.history)}` : '');
|
||||
console.log(args.json ? JSON.stringify(report, null, 2) : text);
|
||||
return 0;
|
||||
} catch (error) {
|
||||
console.error(`gate-cost-replay: ${(error as Error).message}`);
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"version": 5,
|
||||
"version": 6,
|
||||
"patterns": [
|
||||
"test/helpers/agent-sdk-runner.ts",
|
||||
"test/helpers/codex-session-runner.ts",
|
||||
@@ -10,44 +10,44 @@
|
||||
"test/helpers/llm-judge.ts"
|
||||
],
|
||||
"files": {
|
||||
"scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":5},
|
||||
"scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":5},
|
||||
"scripts/lib/paid-report.ts": {"blob":"b66c41237611ccecaaa7bd6a4d7b7aac8dc548cd","since":5,"nonBehavioral":"A2 exercised-path map: test-pr-profile.ts adds a shadow block and classifies the shadow-only map file as selecting no case (selection only); paid-report.ts writes non-blocking MAP_GAP lines after the verdict inputs, inside try/catch, never a verdict or lane problem"},
|
||||
"scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":5},
|
||||
"scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":5},
|
||||
"scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":5},
|
||||
"scripts/test-pr-profile.ts": {"blob":"2029a89e14f511866cc91bebfee00b4fe9e7cf63","since":5,"nonBehavioral":"A2 exercised-path map: test-pr-profile.ts adds a shadow block and classifies the shadow-only map file as selecting no case (selection only); paid-report.ts writes non-blocking MAP_GAP lines after the verdict inputs, inside try/catch, never a verdict or lane problem"},
|
||||
"scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":5},
|
||||
"test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":5},
|
||||
"test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":5},
|
||||
"test/helpers/claude-pty-runner.ts": {"blob":"20cc5894532873aa6145fc6617fdeec8cef94a2d","since":5},
|
||||
"test/helpers/codex-session-runner.ts": {"blob":"4eea35931a35a0afb006363402e3d4191e379737","since":5},
|
||||
"test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":5},
|
||||
"test/helpers/e2e-helpers.ts": {"blob":"2d4a2a53063cfc1f003db366d2c861ab1e379e6e","since":5},
|
||||
"test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":5},
|
||||
"test/helpers/eval-store.ts": {"blob":"609c6d099108ff2696eb014b47aa318a618dd2af","since":5},
|
||||
"test/helpers/hermetic-env.ts": {"blob":"cc072895461dc52ea317d1beff72121342420d30","since":5},
|
||||
"test/helpers/llm-judge.ts": {"blob":"54e3306b494fb5d7f132ccdaaae5c680465028f9","since":5},
|
||||
"test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":5},
|
||||
"test/helpers/pty-screen.ts": {"blob":"4f77569f9ce11f9cee9f347216e468cd7c956820","since":5},
|
||||
"test/helpers/pty/auq.ts": {"blob":"16e5ea2bb5b9f28f44808a1a09024dbd3e6f28c4","since":5},
|
||||
"test/helpers/pty/binary.ts": {"blob":"404d7515076d5c00804f89e23ea69296312c00e1","since":5},
|
||||
"test/helpers/pty/boundaries.ts": {"blob":"53eb76e57c0558c00254366d6334a1c833126575","since":5},
|
||||
"test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":5},
|
||||
"test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":5},
|
||||
"test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":5},
|
||||
"test/helpers/pty/launch.ts": {"blob":"884d1008b3349e8b16d6c38f9f0aa7ae514f0f37","since":5},
|
||||
"test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":5},
|
||||
"test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":5},
|
||||
"test/helpers/pty/runners/floor.ts": {"blob":"6faa013d55901f59ae7b7d1ef6f3a9ee6c9a3034","since":5},
|
||||
"test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":5},
|
||||
"test/helpers/pty/screen.ts": {"blob":"ba7a80bb68b1959296ab2a827cfacdaa46f4ba2d","since":5},
|
||||
"test/helpers/pty/session.ts": {"blob":"58533cea9d81cc87bb3b499bdfe0db8e3a8709c5","since":5},
|
||||
"test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":5},
|
||||
"test/helpers/session-ledger.ts": {"blob":"6e3c9e837e46f98addba2d12eaebff802da320bb","since":5},
|
||||
"test/helpers/session-runner.ts": {"blob":"d40eb29e8b8fdc0675b511693e4a90823b5963fc","since":5},
|
||||
"test/helpers/skill-fixture.ts": {"blob":"b188064594eb4201a54a80ba1bc6f28e31d7f49e","since":5},
|
||||
"test/helpers/test-selection.ts": {"blob":"51655f89f66be7a5ec487e2abfe4744300ebd5f6","since":5},
|
||||
"test/helpers/touchfiles.ts": {"blob":"15f369056d41b9dc42dcd86f81f60c878074704c","since":5}
|
||||
"scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":6},
|
||||
"scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":6},
|
||||
"scripts/lib/paid-report.ts": {"blob":"b66c41237611ccecaaa7bd6a4d7b7aac8dc548cd","since":6},
|
||||
"scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6},
|
||||
"scripts/lib/shard-engine.ts": {"blob":"3cd0b1aaa84018b00bf9ea0ce48093f3c8492f66","since":6},
|
||||
"scripts/test-paid-shards.ts": {"blob":"8ddee5f11ed7856876135520dd8025ab989e2db9","since":6},
|
||||
"scripts/test-pr-profile.ts": {"blob":"2029a89e14f511866cc91bebfee00b4fe9e7cf63","since":6},
|
||||
"scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6},
|
||||
"test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":6},
|
||||
"test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":6},
|
||||
"test/helpers/claude-pty-runner.ts": {"blob":"20cc5894532873aa6145fc6617fdeec8cef94a2d","since":6},
|
||||
"test/helpers/codex-session-runner.ts": {"blob":"4eea35931a35a0afb006363402e3d4191e379737","since":6},
|
||||
"test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":6},
|
||||
"test/helpers/e2e-helpers.ts": {"blob":"2d4a2a53063cfc1f003db366d2c861ab1e379e6e","since":6},
|
||||
"test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":6},
|
||||
"test/helpers/eval-store.ts": {"blob":"609c6d099108ff2696eb014b47aa318a618dd2af","since":6},
|
||||
"test/helpers/hermetic-env.ts": {"blob":"cc072895461dc52ea317d1beff72121342420d30","since":6},
|
||||
"test/helpers/llm-judge.ts": {"blob":"d0bc6bbbe9f478603d11a1ece95e66c812ad34e6","since":6},
|
||||
"test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":6},
|
||||
"test/helpers/pty-screen.ts": {"blob":"4f77569f9ce11f9cee9f347216e468cd7c956820","since":6},
|
||||
"test/helpers/pty/auq.ts": {"blob":"16e5ea2bb5b9f28f44808a1a09024dbd3e6f28c4","since":6},
|
||||
"test/helpers/pty/binary.ts": {"blob":"404d7515076d5c00804f89e23ea69296312c00e1","since":6},
|
||||
"test/helpers/pty/boundaries.ts": {"blob":"53eb76e57c0558c00254366d6334a1c833126575","since":6},
|
||||
"test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":6},
|
||||
"test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":6},
|
||||
"test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":6},
|
||||
"test/helpers/pty/launch.ts": {"blob":"884d1008b3349e8b16d6c38f9f0aa7ae514f0f37","since":6},
|
||||
"test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":6},
|
||||
"test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":6},
|
||||
"test/helpers/pty/runners/floor.ts": {"blob":"57fa44c31300754b00473276cf7b972d4e9e47d8","since":6},
|
||||
"test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":6},
|
||||
"test/helpers/pty/screen.ts": {"blob":"ba7a80bb68b1959296ab2a827cfacdaa46f4ba2d","since":6},
|
||||
"test/helpers/pty/session.ts": {"blob":"58533cea9d81cc87bb3b499bdfe0db8e3a8709c5","since":6},
|
||||
"test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6},
|
||||
"test/helpers/session-ledger.ts": {"blob":"6e3c9e837e46f98addba2d12eaebff802da320bb","since":6},
|
||||
"test/helpers/session-runner.ts": {"blob":"d40eb29e8b8fdc0675b511693e4a90823b5963fc","since":6},
|
||||
"test/helpers/skill-fixture.ts": {"blob":"b188064594eb4201a54a80ba1bc6f28e31d7f49e","since":6},
|
||||
"test/helpers/test-selection.ts": {"blob":"51655f89f66be7a5ec487e2abfe4744300ebd5f6","since":6},
|
||||
"test/helpers/touchfiles.ts": {"blob":"15f369056d41b9dc42dcd86f81f60c878074704c","since":6}
|
||||
}
|
||||
}
|
||||
@@ -13,7 +13,7 @@ import {
|
||||
judgePanelMedian, judgePanelMajority,
|
||||
} from '../test/helpers/llm-judge';
|
||||
import { buildWorkflowJudgePrompt, WORKFLOW_JUDGE_RESPONSE_SCHEMA } from '../test/helpers/workflow-judge-input';
|
||||
import { CLAUDE_FRONTIER_EVAL_MODEL } from '../lib/eval-model';
|
||||
import { CLAUDE_JUDGE_EVAL_MODEL } from '../lib/eval-model';
|
||||
import type { CalibrationConfig, Verdict } from './judge-calibration';
|
||||
|
||||
type Sample = Record<string, unknown>;
|
||||
@@ -40,7 +40,7 @@ const scoreJudge = (floors: Record<string, number>) => ({
|
||||
|
||||
const workflow = (id: string, priority: number, cases: string[], extra: { agentCapability?: 'frontier'; model?: string; floors?: Record<string, number> } = {}): CalibrationConfig => ({
|
||||
id, priority, cases,
|
||||
model: extra.model ?? CLAUDE_FRONTIER_EVAL_MODEL,
|
||||
model: extra.model ?? CLAUDE_JUDGE_EVAL_MODEL,
|
||||
build: inputs => buildWorkflowJudgePrompt({ judgeContext: inputs.judgeContext!, judgeGoal: inputs.judgeGoal!, agentCapability: extra.agentCapability },
|
||||
{ files: [], text: inputs.bundle! }),
|
||||
oldOptions: { max_tokens: DEFAULT_JUDGE_MAX_TOKENS },
|
||||
@@ -63,19 +63,19 @@ export const CALIBRATION_CONFIGS: CalibrationConfig[] = [
|
||||
estimatedOutputTokens: 300,
|
||||
},
|
||||
{
|
||||
id: 'qa-workflow', priority: 2, cases: ['qa/SKILL.md workflow'], model: CLAUDE_FRONTIER_EVAL_MODEL,
|
||||
id: 'qa-workflow', priority: 2, cases: ['qa/SKILL.md workflow'], model: CLAUDE_JUDGE_EVAL_MODEL,
|
||||
build: inputs => buildQaWorkflowJudgePrompt(inputs.section!),
|
||||
oldOptions: {}, newOptions: { jsonSchema: JUDGE_SCORE_SCHEMA },
|
||||
...scoreJudge(QA_FLOORS), estimatedOutputTokens: 1200,
|
||||
},
|
||||
{
|
||||
id: 'qa-health-rubric', priority: 3, cases: ['qa/SKILL.md health rubric'], model: CLAUDE_FRONTIER_EVAL_MODEL,
|
||||
id: 'qa-health-rubric', priority: 3, cases: ['qa/SKILL.md health rubric'], model: CLAUDE_JUDGE_EVAL_MODEL,
|
||||
build: inputs => buildQaHealthRubricJudgePrompt(inputs.section!),
|
||||
oldOptions: {}, newOptions: { jsonSchema: JUDGE_SCORE_SCHEMA },
|
||||
...scoreJudge(QA_FLOORS), estimatedOutputTokens: 1200,
|
||||
},
|
||||
{
|
||||
id: 'qa-anti-refusal', priority: 4, cases: ['qa/SKILL.md anti-refusal'], model: CLAUDE_FRONTIER_EVAL_MODEL,
|
||||
id: 'qa-anti-refusal', priority: 4, cases: ['qa/SKILL.md anti-refusal'], model: CLAUDE_JUDGE_EVAL_MODEL,
|
||||
build: inputs => buildQaAntiRefusalJudgePrompt(inputs.diffAwareSection!, inputs.rulesSection!),
|
||||
oldOptions: {}, newOptions: { jsonSchema: QA_ANTI_REFUSAL_JUDGE_SCHEMA },
|
||||
validate: value => requireFields(value, ['confidence'], ['would_browse']),
|
||||
@@ -84,7 +84,7 @@ export const CALIBRATION_CONFIGS: CalibrationConfig[] = [
|
||||
estimatedOutputTokens: 1200,
|
||||
},
|
||||
{
|
||||
id: 'cross-skill', priority: 5, cases: ['cross-skill greptile consistency'], model: CLAUDE_FRONTIER_EVAL_MODEL,
|
||||
id: 'cross-skill', priority: 5, cases: ['cross-skill greptile consistency'], model: CLAUDE_JUDGE_EVAL_MODEL,
|
||||
build: inputs => buildCrossSkillConsistencyJudgePrompt(inputs.collected!),
|
||||
oldOptions: {}, newOptions: { jsonSchema: CROSS_SKILL_CONSISTENCY_JUDGE_SCHEMA },
|
||||
validate: value => requireFields(value, ['score'], ['consistent']),
|
||||
@@ -93,7 +93,7 @@ export const CALIBRATION_CONFIGS: CalibrationConfig[] = [
|
||||
estimatedOutputTokens: 1200,
|
||||
},
|
||||
{
|
||||
id: 'voice', priority: 6, cases: ['voice directive tone'], model: CLAUDE_FRONTIER_EVAL_MODEL,
|
||||
id: 'voice', priority: 6, cases: ['voice directive tone'], model: CLAUDE_JUDGE_EVAL_MODEL,
|
||||
build: inputs => buildVoiceDirectiveJudgePrompt(inputs.voiceSection!),
|
||||
oldOptions: {}, newOptions: { jsonSchema: VOICE_DIRECTIVE_JUDGE_SCHEMA },
|
||||
validate: value => requireFields(value, VOICE_DIRECTIVE_DIMENSIONS),
|
||||
|
||||
+4
-4
@@ -53,10 +53,10 @@ export type Model = (typeof ALL_MODEL_NAMES)[number];
|
||||
* with or without a date (`claude-opus-4-7-1`, `claude-sonnet-5-5`,
|
||||
* `claude-sonnet-5-5-20261001`), falls to `claude`: a newer model should
|
||||
* not read nudges written for an older one.
|
||||
* - Fable is the exception: any `claude-fable-5-*` ID keeps `fable-5`,
|
||||
* because `claude-fable-5-1` is CLAUDE_FRONTIER_EVAL_MODEL
|
||||
* (lib/eval-model.ts) and moving it would shift every eval baseline.
|
||||
* The exception ends when CLAUDE_FRONTIER_EVAL_MODEL changes.
|
||||
* - Fable is the exception: any `claude-fable-5-*` ID keeps `fable-5`
|
||||
* while a Fable 5 point release is one of gstack's eval models (the
|
||||
* judge, CLAUDE_JUDGE_EVAL_MODEL in lib/eval-model.ts). The exception
|
||||
* ends when CLAUDE_JUDGE_EVAL_MODEL leaves Fable 5.
|
||||
* - Any other `claude-*` (including `claude-opus-5-5`) → `claude`
|
||||
* - `gemini-*` (2.5-pro, flash, etc.) → `gemini`
|
||||
* 3. Unknown input → returns null (caller decides: error, or fall back).
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { TemplateContext } from './types';
|
||||
import { CODEX_MODEL_CONFIG_FLAG, CODEX_REVIEW_MODEL_CONFIG_FLAG, CODEX_WEB_SEARCH_FLAG } from './constants';
|
||||
import { CLAUDE_FRONTIER_EVAL_MODEL } from '../../lib/eval-model';
|
||||
import { CLAUDE_CAPTURE_EVAL_MODEL } from '../../lib/eval-model';
|
||||
|
||||
/**
|
||||
* {{CODEX_WEB_SEARCH_FLAG}} — the non-deprecated codex web-search flag
|
||||
@@ -25,7 +25,7 @@ export function generateCodexReviewModelConfigFlag(_ctx: TemplateContext): strin
|
||||
}
|
||||
|
||||
export function generateClaudeModelFlag(_ctx: TemplateContext): string {
|
||||
return `--model "\${GSTACK_CLAUDE_MODEL:-${CLAUDE_FRONTIER_EVAL_MODEL}}"`;
|
||||
return `--model "\${GSTACK_CLAUDE_MODEL:-${CLAUDE_CAPTURE_EVAL_MODEL}}"`;
|
||||
}
|
||||
|
||||
export function generateSlugEval(ctx: TemplateContext): string {
|
||||
|
||||
@@ -226,7 +226,6 @@
|
||||
"test/helpers/hermetic-env.test.ts\tTS7053\tElement implicitly has an 'any' type because expression of type 'string' can't be used to index type '{ ... 6 more ...; EVALS_SELECTION_JSON: string; GITHUB_APP_PRIVATE_KEY: string; GITHUB_CLIENT_SECRET: string; GITHUB_PAT: string; GITHUB_PERSONAL_ACCESS_TOKEN: string; GITHUB_TOKEN: string; NODE_ENV?: string | undefined; TZ?: string | undefined; }'. No index signature with a parameter of type 'string' was found on type '{ ... 6 more ...; EVALS_SELECTION_JSON: string; GITHUB_APP_PRIVATE_KEY: string; GITHUB_CLIENT_SECRET: string; GITHUB_PAT: string; GITHUB_PERSONAL_ACCESS_TOKEN: string; GITHUB_TOKEN: string; NODE_ENV?: string | undefined; TZ?: string | undefined; }'.": 1,
|
||||
"test/helpers/office-hours-completion.ts\tTS18047\t'disposition' is possibly 'null'.": 2,
|
||||
"test/helpers/plan-count-file-permission.ts\tTS18048\t'event.input' is possibly 'undefined'.": 3,
|
||||
"test/helpers/plan-count-file-permission.ts\tTS18048\t'input' is possibly 'undefined'.": 12,
|
||||
"test/helpers/plan-review-decisions.ts\tTS2339\tProperty 'questions' does not exist on type 'AskUserQuestionFingerprint'.": 4,
|
||||
"test/helpers/plan-review-decisions.ts\tTS2339\tProperty 'selectedOptions' does not exist on type 'AskUserQuestionFingerprint'.": 4,
|
||||
"test/helpers/plan-review-decisions.ts\tTS2339\tProperty 'toolUseId' does not exist on type 'AskUserQuestionFingerprint'.": 8,
|
||||
|
||||
Reference in new issue
Block a user