Merge main 68410828f (v1.93.17.0, train 92) into wave/review-53cfb5bfe011 (rule 19 grant)

Conflicts resolved keeping both sides: bin/gstack-review-plan (baseContainsHead and approval-record imports); scripts/harness-version.json (main's session-ledger and session-runner pins, the wave's skill-fixture pin); ship/DRIVER.md.tmpl (main's done-or-voices wording, regenerated DRIVER.md 12,278 B); test/review-packets-removed.test.ts (merged review-army.md, adversarial.md and review-depth.ts pinned under combined GST-508+GST-464+GST-967 and GST-508+R51+GST-911+GST-967 entries).
This commit is contained in:
garrytan committed 2026-11-02 19:52:15 +00:00
commit f0ce23418c
99 files changed
+3505 -295

No files matched your search

+4 -2
View File
@@ -55,6 +55,8 @@ export interface GenerationOptions {
catalogMode?: 'trim' | 'full';
explainLevel?: 'default' | 'terse';
respectDetection?: boolean;
/** A detection result the caller already read; wins over respectDetection so one read serves identity and render. */
gbrainDetected?: boolean;
log?: (message: string) => void;
planMode?: boolean;
/** Module loadout (scripts/modules.ts selectModules); null/undefined = `full`. */
@@ -94,7 +96,7 @@ export interface GenerationResult {
}
/** Canonical generation never reads local detection state unless opted in. */
function loadGbrainOverride(respectDetection: boolean): boolean {
export function loadGbrainOverride(respectDetection: boolean): boolean {
if (!respectDetection) return false;
const stateDir = resolveStateRoot();
try {
@@ -989,7 +991,7 @@ export async function runGeneration(settings: GenerationOptions = {}): Promise<G
model: settings.model ?? null,
catalogMode: settings.catalogMode ?? 'trim',
explainLevel: settings.explainLevel ?? 'default',
gbrainDetected: loadGbrainOverride(settings.respectDetection ?? false),
gbrainDetected: settings.gbrainDetected ?? loadGbrainOverride(settings.respectDetection ?? false),
planMode: settings.planMode ?? true,
modules: settings.modules ?? null,
};
+9 -8
View File
@@ -13,21 +13,21 @@
"files": {
"scripts/lib/paid-cases.ts": {"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
"scripts/lib/paid-plan.ts": {"blob":"e8ce3cb98474dded38c5128a631d283b0ca35b63","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"},
"scripts/lib/paid-report.ts": {"blob":"224dc5e8ba23a3589a37bad94db72ed1ca3c7c77","since":6,"nonBehavioral":"a case shard's result records are attributed to its one case: the trial's cost_usd and cost_known come from them, and a failed trial's failure_class, exit_reason and error come from its record instead of a bare assertion with the JUnit message, so a red whose record says infra or timeout is classed by that record (INFRA instead of VERDICT: rerun and bisect eligibility, not a pass rate); no case's outcome or selection changes and pass rates count outcomes only, so no pass-rate series resets"},
"scripts/lib/paid-report.ts": {"blob":"e0d18da4ea02ac443835a68992abeea2b47f22fc","since":6,"nonBehavioral":"the headline adds only streamed session cost; a billed session's cost is already in its collector total; cache writes without a 5m key fall back to the total minus 1h; cost fields only"},
"scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6},
"scripts/lib/shard-engine.ts": {"blob":"dc714d7d897aed185c0ec3d2c92ad22d954039e5","since":6,"nonBehavioral":"createShardSandbox drops the Capy launch markers (GSTACK_SESSION_KIND, GSTACK_CAPY, GSTACK_EPHEMERAL, GSTACK_INSTALL_VERDICT) from shard environments; CI never sets them, so no case's input or verdict changes there, and on a Capy machine fixtures now see the session kind they set themselves instead of the installer's"},
"scripts/test-paid-shards.ts": {"blob":"5439d0a8f6e1569fa7c924af20d67c30c81d5d30","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
"scripts/test-pr-profile.ts": {"blob":"44ae88bebf8f716f70a3bd836c8bedb03e6653e8","since":6,"nonBehavioral":"FREE_ONLY_PR_FILES gains scripts/lib/shard-jobs.ts (free-runner module) and .github/workflows/windows-smoke.yml (a workflow no paid case reads); both change only which PR paths select paid cases, never how a case runs or is judged"},
"scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6},
"test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":6},
"test/helpers/agent-sdk-runner.ts": {"blob":"d66dac458b8c13250973e550edf8087d4044cd0b","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets. Also: runAgentSdkTest now validates systemPrompt before starting; every live caller already passed the claude_code preset, so the prompt the model sees is unchanged and no verdict can change"},
"test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":6},
"test/helpers/claude-pty-runner.ts": {"blob":"ce7a5eaa66e502a052ea9c53348dd2ab25db6148","since":6,"nonBehavioral":"Removes the plan-design 0D focus picker from the shared harness; floor.ts gains an optional pickSetupChoice hook that no other caller passes, so no other case's input or verdict can change. plan-design-finding-floor, the only case whose seed raises that menu, passes the picker from its owned fixture test/fixtures/design-floor-focus-picker.ts, so its owned bytes start its own new series."},
"test/helpers/codex-host-runner.ts": {"blob":"bff225017e235f0be6ece29f04971ac04b95b544","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
"test/helpers/codex-session-runner.ts": {"blob":"c040ebbfa815b8cce19ef80c6133bd5a5474149a","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
"test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":6},
"test/helpers/e2e-helpers.ts": {"blob":"6ee6937e6c3ca081c77131b5a755278c7b714834","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
"test/helpers/e2e-helpers.ts": {"blob":"91996ff2bc303ee29f4e6f1705d1468b2536b32e","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"},
"test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":6},
"test/helpers/eval-store.ts": {"blob":"a50ea2c28f508ad9f35ea9353dd2f502ee0dcfc2","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
"test/helpers/eval-store.ts": {"blob":"4085ad18cd99698f675deadad2a76a796427b693","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"},
"test/helpers/hermetic-env.ts": {"blob":"cc072895461dc52ea317d1beff72121342420d30","since":6},
"test/helpers/llm-judge.ts": {"blob":"ff32d03f32e4af76c6c7314a79cd1b35263ed0dd","since":6,"nonBehavioral":"llm-judge.ts adds a pass-through byte counter (bytesReceived, lastByteMs) to the aborted-judge diagnostic; the request, response bytes, deadline and scoring are unchanged. GSTA-224 B2 (#3122): skill-fixture.ts follows review's renamed Step 2.5 heading so the review E2E cases extract the same section (now the skill's own reworded bytes); llm-judge.ts changes one bullet of the cross-skill consistency judge's intended architecture to name review-bot-history.md beside the older greptile-history.md name, so its calibration corpus (greptile-history inputs) stays valid. No other case's input or verdict can change; the cross-skill judge's judged files changed in this PR, so its owned inputs start its own series"},
"test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":6},
@@ -38,16 +38,17 @@
"test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":6},
"test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":6},
"test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":6},
"test/helpers/pty/launch.ts": {"blob":"e4413de380b799659170e7da248d51afda3b0e3f","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
"test/helpers/pty/launch.ts": {"blob":"4f0731ef39bc3a6a7c5c592ce98690a24cb1ed7f","since":6,"nonBehavioral":"PTY transcript pricing finds the session by the caller's --session-id or the one new transcript for the launch cwd; the launcher adds no argument; native AUQ capture opts out; ledger cost fields only, no prompt, argv, answer, timeout or verdict changes"},
"test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":6},
"test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":6},
"test/helpers/pty/runners/floor.ts": {"blob":"a9938cb98f285474ebc4860cd61012504a36466e","since":6,"nonBehavioral":"The PTY floor and observation runners end a plan-mode turn at an idle prompt (the plan-mode footer without its running 'esc to interrupt' segment, after a 10 s settle) instead of waiting out the deadline. The floor runner exits as api_error only after the idle screen with its error panel held still for 10 s and every tick's question and assessment steps found nothing to credit (the transcript is re-read each tick), so a native or prose question already on screen is credited as before, even when its transcript line lands after the screen draws done (test/helpers/claude-pty-runner.runners.unit.test.ts, the prose-question-before-panel case). With no qualifying question an idle turn stays idle, so a trial that failed at the deadline as observer_timeout now fails earlier as api_error. An observation returns what the deadline would have returned. A running footer, extra footer text and a cut footer still wait for the deadline. No pass becomes a fail or a fail a pass. The session ledger may now record failure_cause api_error or refusal instead of observer_timeout or session_timeout for a plan-mode session whose last idle turn shows a panel; failure_class, the gate red class and pass rates are unchanged, so no pass-rate series resets"},
"test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":6},
"test/helpers/pty/screen.ts": {"blob":"1384f765e48e511d5f843df2660b2ec70bbcb6b2","since":6,"nonBehavioral":"The PTY floor and observation runners end a plan-mode turn at an idle prompt (the plan-mode footer without its running 'esc to interrupt' segment, after a 10 s settle) instead of waiting out the deadline. The floor runner exits as api_error only after the idle screen with its error panel held still for 10 s and every tick's question and assessment steps found nothing to credit (the transcript is re-read each tick), so a native or prose question already on screen is credited as before, even when its transcript line lands after the screen draws done (test/helpers/claude-pty-runner.runners.unit.test.ts, the prose-question-before-panel case). With no qualifying question an idle turn stays idle, so a trial that failed at the deadline as observer_timeout now fails earlier as api_error. An observation returns what the deadline would have returned. A running footer, extra footer text and a cut footer still wait for the deadline. No pass becomes a fail or a fail a pass. The session ledger may now record failure_cause api_error or refusal instead of observer_timeout or session_timeout for a plan-mode session whose last idle turn shows a panel; failure_class, the gate red class and pass rates are unchanged, so no pass-rate series resets"},
"test/helpers/pty/session.ts": {"blob":"54fe04b61d953e4e55e2fc251532c04a49fab66f","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
"test/helpers/pty/session.ts": {"blob":"8fd1749a62f24dc37ae186c8a21c2165f7f7645e","since":6,"nonBehavioral":"PTY sessions price their ledger row (streamed_cost_usd) from the session transcript found by the caller's --session-id or the one new transcript for the launch cwd; ledger cost fields only, no prompt, argv, answer, timeout or verdict changes"},
"test/helpers/pty/transcript-cost.ts": {"blob":"08f7ccbdbd9d71a5dc86edd2f8fced423a78ad9a","since":6,"nonBehavioral":"PTY transcript pricing finds the session by the caller's --session-id or the one new transcript for the launch cwd; the launcher adds no argument; native AUQ capture opts out; ledger cost fields only, no prompt, argv, answer, timeout or verdict changes"},
"test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6},
"test/helpers/session-ledger.ts": {"blob":"a10bf28bf100f135b93c73818670dd147253cdd3","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
"test/helpers/session-runner.ts": {"blob":"cc65a66adafa9c053e3da3c6f17b5ac86c0eade7","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
"test/helpers/session-ledger.ts": {"blob":"c99baaaa0a77a5a72384493fd8a85a4253ecb05d","since":6,"nonBehavioral":"PTY sessions price their ledger row (streamed_cost_usd) from the session transcript found by the caller's --session-id or the one new transcript for the launch cwd; ledger cost fields only, no prompt, argv, answer, timeout or verdict changes"},
"test/helpers/session-runner.ts": {"blob":"c73b2596d9eb5377530a6bf1d284e0b6ea1556a0","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"},
"test/helpers/skill-fixture.ts": {"blob":"ce7609eacc7a8265a4a211358d1b4fb49ea85bc5","since":6,"nonBehavioral":"GSTA-224 B2 (#3122): skill-fixture.ts follows review's renamed Step 2.5 heading ('Check for review bot comments') so the review E2E cases extract the same section, now the skill's own reworded bytes; llm-judge.ts updates only the cross-skill consistency judge's intended architecture to the renamed review-bot-history.md and its record helper. No other case's input or verdict can change; the cross-skill judge's judged files changed in this PR, so its owned inputs start its own new series"},
"test/helpers/skill-startup.ts": {"blob":"a8d21e3817c3f35a1d06af325c92fde7d420e377","since":6,"nonBehavioral":"readSkillEffective also appends a variant's added ### subsections (office-hours' Open-ended questions); readSkillStartup, readSharedSections and readSkillEffective return bytes identical to the pinned helper for every tier 2+ render without added subsections (735 across all hosts) and for office-hours' SKILL.md before this PR, so office-hours' fixture input changes only because its prompt changed in this PR, not because the helper did"},
"test/helpers/test-selection.ts": {"blob":"e1b54ad5e3d3eadd8027de1b44e57554d6d99a41","since":6,"nonBehavioral":"selection only: fragment exemption narrowed to .md files directly in the fragments dir; changed-file lists include a rename's old path; no case prompt, fixture, judge or verdict logic changes (GST-400)"},
+33 -8
View File
@@ -9,7 +9,7 @@ import { STALL_WINDOW_MS } from '../../test/helpers/eval-budgets';
import {
isFinalizedEvalResultFile, credentialRejection, failureClassOf, panelVerdict, sanitizeTrialError, formatTrialOutcomes, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE,
type EvalCaseKind, type PanelVerdict, type TrialFailureClass, type TrialOutcome, type TrialOutcomeRecord,
trialCostKnown, trialFailureFields, trialSessions, type TrialFailureDetail,
trialFailureFields, trialSessions, type TrialFailureDetail,
} from '../../test/helpers/eval-store';
import { readSessionLedger, type SessionLedgerRow } from '../../test/helpers/session-ledger';
import { publishedFence, sanitizeFixedFenceLine } from './published-text';
@@ -230,6 +230,8 @@ export interface JUnitCensus {
deselected: Map<string, number>;
/** Per shard key: skip reason -> skipped testcase count. */
shardSkipReasons: Map<string, Map<string, number>>;
/** Streamed session cost (sessions without a terminal result), which no collector record carries; the headline adds it to the collectors' total_cost_usd. */
ledgerCostUsd: number;
/** `<shard key> :: <test name> (<outcome>)` for every testcase no rule attributes. */
unattributed: string[];
}
@@ -241,6 +243,24 @@ function caseSessions(rows: SessionLedgerRow[], id: string, onlyCase: boolean):
return rows.filter(row => row.case === id || (!row.case && (onlyCase || row.test_name === id || row.test_name === CASE_TEST_NAMES[id])));
}
/** Cases that run no model (a binary data flow): with no record and no session they cost a known $0. test/paid-report-cost.test.ts pins each body model-free. */
export const MODEL_FREE_CASES: ReadonlySet<string> = new Set(['timeline-event-flow']);
/**
* A trial's cost: its records' sum plus what no record carries, a session without a terminal result priced from its
* streamed usage (streamed_cost_usd). With no record, a billed session counts its own cost_usd (Codex). Unknown when any
* record or session cost is; a listed model-free case with neither record nor session is a known $0.
*/
export function caseCost(id: string, records: any[], sessions: SessionLedgerRow[]): { cost_usd: number; cost_known?: false } {
if (!records.length && !sessions.length) return MODEL_FREE_CASES.has(id) ? { cost_usd: 0 } : { cost_usd: 0, cost_known: false };
const unrecorded = sessions.map(s => s.billed !== false ? (records.length ? 0 : s.cost_usd) : s.streamed_cost_usd);
const known = records.every(r => r?.cost_known !== false) && unrecorded.every(c => typeof c === 'number' && Number.isFinite(c));
const total = records.reduce((sum, r) => sum + (Number(r?.cost_usd) || 0), 0) + knownSum(unrecorded);
return { cost_usd: Math.round(total * 100) / 100, ...(known ? {} : { cost_known: false as const }) };
}
const knownSum = (costs: Array<number | undefined>) => costs.reduce<number>((sum, c) => sum + (typeof c === 'number' && Number.isFinite(c) ? c : 0), 0);
/** The artifact holding a slice's shards: the downloaded directory's own name when it is one. */
/** The red-census guide the report and every eval:pass-rates view link. */
export const CENSUS_RED_GUIDE = 'docs/evals/census-red.md';
@@ -252,7 +272,8 @@ export function sliceArtifactName(root: string, slice: number, attempt: number):
export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact[], recordsByShard: Map<string, any[]>,
base: Record<string, unknown>, cliVersion?: string): JUnitCensus {
const census: JUnitCensus = { ruleCases: [], history: [], failedShards: new Set(), skipped: [], deselected: new Map(), shardSkipReasons: new Map(), unattributed: [] };
const census: JUnitCensus = { ruleCases: [], history: [], failedShards: new Set(), skipped: [], deselected: new Map(), shardSkipReasons: new Map(), unattributed: [],
ledgerCostUsd: 0 };
const entries = new Map(manifest.entries.map(entry => [normalizeRelativePath(entry.file), entry]));
for (const { root, result } of artifacts) {
for (const outcome of result.outcomes) {
@@ -277,11 +298,15 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
const caseOutcome: TrialOutcome = failed ? 'failed' : cases.some(tc => tc.outcome === 'passed') ? 'passed' : 'skipped';
// A record is named by its case id or carries case_id; an E2E record named by its display label ("/qa fix loop") is
// the shard's only case's when the shard ran one case, else it stays unattributed and the trial's cost unknown.
const mine = records.filter((r: any) => r?.name === id || r?.case_id === id || (byCase.size === 1 && !r?.case_id));
const failedRecord = mine.find((r: any) => r.passed === false);
// A display-label record whose runSkillTest testName is this case (cost_case_id) adds its cost only: the failed record stays the named one.
const named = records.filter((r: any) => r?.name === id || r?.case_id === id || (byCase.size === 1 && !r?.case_id));
const mine = [...named, ...records.filter((r: any) => r?.cost_case_id === id && !named.includes(r) && !byCase.has(r?.name))];
const failedRecord = named.find((r: any) => r.passed === false);
const recorded: TrialFailureClass | undefined = !failed ? undefined
: failed.failureType === 'TimeoutError' ? 'timeout' : failedRecord ? failureClassOf(failedRecord) : 'assertion';
const sessions = caseSessions(ledger, id, byCase.size === 1);
const cost = caseCost(id, mine, sessions);
census.ledgerCostUsd += knownSum(sessions.map(s => s.billed === false ? s.streamed_cost_usd : 0));
const credentials = failed && recorded !== 'contract' ? credentialRejection({ error: `${failedRecord?.error ?? ''}\n${failed.message ?? ''}`, sessions }) : undefined;
const failureClass = credentials ? 'infra' : recorded;
const rawError = credentials ? `credentials rejected: ${credentials}` : failedRecord?.error ?? failed?.message;
@@ -295,7 +320,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
census.skipped.push(`◌ ${id} ${key} (${cases.map(tc => tc.name).join('; ')}): ${RUNTIME_SKIP_REASON}`);
}
census.ruleCases.push({ id, kind, outcome: caseOutcome, key, slice: result.sliceIndex, ...(failureClass ? { failureClass } : {}), ...(error && failed ? { error } : {}),
costUsd: Math.round(mine.reduce((sum: number, r: any) => sum + (Number(r.cost_usd) || 0), 0) * 100) / 100, costKnown: trialCostKnown(mine, sessions).cost_known !== false,
costUsd: cost.cost_usd, costKnown: cost.cost_known !== false,
...(failed ? { line: `✗ ${id} ${kind} FAIL ${failureClass}${failedRecord?.exit_reason === 'timeout' && failedRecord?.timeout_at_turn !== undefined ? ` at turn ${failedRecord.timeout_at_turn}` : ''}${error ? ` — ${error}` : ''}`
+ `${diagnosisText(diagnosis, error)} [slice ${result.sliceIndex}, attempt ${result.attempt ?? 1}]`
+ `${evidencePointer(sliceArtifactName(root, result.sliceIndex, result.attempt ?? 1), key, typeof base.run_id === 'string' ? base.run_id : undefined, id)}`
@@ -303,8 +328,8 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
census.history.push({ ...base, case: id, file: shardFile(key), kind, trial: 1, panel: { n: 1, k: 1 }, outcome: caseOutcome,
...(failureClass ? { failure_class: failureClass } : {}), ...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}),
...(error && failed ? { error } : {}), ...diagnosis, ...trialSessions(sessions), duration_ms: Math.min(cases.reduce((sum, tc) => sum + tc.timeMs, 0), outcome.elapsedMs),
cost_usd: Math.round(mine.reduce((sum: number, r: any) => sum + (Number(r.cost_usd) || 0), 0) * 100) / 100, ...trialCostKnown(mine, sessions),
...(typeof mine[0]?.model === 'string' ? { model: mine[0].model } : {}), ...(cliVersion ? { cli_version: cliVersion } : {}),
...cost,
...(typeof named[0]?.model === 'string' ? { model: named[0].model } : {}), ...(cliVersion ? { cli_version: cliVersion } : {}),
quarantined: false, execution: outcome.reused ? 'reused' : 'executed', source: 'junit' } as TrialOutcomeRecord);
}
census.shardSkipReasons.set(key, reasons);
@@ -776,7 +801,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
...(spend.executions ? { reruns: spend } : {}),
actionRequired: verdict.problems.length,
wallMs: wall.length ? Math.max(...wall.map(r => r.finishedAt!)) - Math.min(...wall.map(r => r.startedAt!)) : null,
costUsd: Math.round(costUsd * 100) / 100,
costUsd: Math.round((costUsd + census.ledgerCostUsd) * 100) / 100,
redispatchEligible: red && infraOnly(verdict.problems),
};
const executed = history.filter(r => r.attempt === primary && r.execution === 'executed' && r.outcome !== 'skipped');
+9 -2
View File
@@ -26,7 +26,8 @@ import { createHash, randomBytes } from 'node:crypto';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { runGeneration } from './gen-skill-docs';
import { readConfigKey } from '../lib/state-root';
import { loadGbrainOverride, runGeneration } from './gen-skill-docs';
import { resolveModel } from './models';
import { ModuleError, type ModuleSelection } from './modules';
import { LOCK_FILE, danglingReferences } from './modules-render';
@@ -148,6 +149,10 @@ export async function installLoadout(request: InstallRequest): Promise<InstallRe
const planMode = request.planMode ?? true;
const model = request.boundModel ? resolveModel(request.boundModel) : null;
const selection: ModuleSelection = { ...request.selection, boundModel: request.boundModel ?? undefined, fullRoot: request.boundModel ? request.fullRoot : undefined };
const gbrainDetected = loadGbrainOverride(true);
const disabledSkills = [...new Set((readConfigKey('disabled_skills') ?? '').replace(/["']/g, '').split(/[\s,]+/)
.map(n => n.replace(/^\//, '')).filter(n => n !== 'gstack' && n !== 'gstack-upgrade')
.map(n => n.replace(/^gstack-/, '')).filter(n => /^[a-z0-9-]+$/.test(n)))].sort();
const identity = {
loadout: selection.name,
excluded: [...selection.excluded].sort(),
@@ -157,6 +162,8 @@ export async function installLoadout(request: InstallRequest): Promise<InstallRe
host,
overlay_model: model,
plan_mode: planMode,
gbrain_detected: gbrainDetected,
disabled_skills: disabledSkills,
source_tree: sourceTree(source),
};
const identitySha = sha256(JSON.stringify(identity));
@@ -174,7 +181,7 @@ export async function installLoadout(request: InstallRequest): Promise<InstallRe
}
const stage = `${final}.stage-${process.pid}-${randomBytes(4).toString('hex')}`;
try {
const result = await runGeneration({ host, outputRoot: stage, contentLinkRoot: final, installRoot: request.fullRoot && request.fullRoot !== path.join(os.homedir(), '.claude/skills') ? path.join(request.fullRoot, 'gstack') : null, model, planMode, modules: selection });
const result = await runGeneration({ host, outputRoot: stage, contentLinkRoot: final, installRoot: request.fullRoot && request.fullRoot !== path.join(os.homedir(), '.claude/skills') ? path.join(request.fullRoot, 'gstack') : null, model, planMode, modules: selection, gbrainDetected, disabledSkills });
const errors = result.diagnostics.filter(d => d.kind === 'error').map(d => d.message);
if (result.exitCode !== 0 || errors.length) {
throw new ModuleError('RENDER_FAILED', `gen-skill-docs failed: ${errors.join('; ') || 'exit 1'}`, 'fix the named template or module, then rerun; side effects: none, safe to retry');
+2 -1
View File
@@ -21,6 +21,7 @@ import '../lib/conductor-env-shim';
import { query, type SDKMessage } from '@anthropic-ai/claude-agent-sdk';
import { readOverlay } from './resolvers/model-overlay';
import { resolveClaudeBinary } from '../lib/claude-bin';
import { sdkSystemPrompt } from '../test/helpers/sdk-system-prompt';
const USAGE = `Usage: bun run scripts/preflight-agent-sdk.ts [--live]
@@ -94,7 +95,7 @@ async function main() {
prompt: 'say pong',
options: {
model: 'claude-opus-4-7',
systemPrompt: '',
...sdkSystemPrompt({ type: 'sdk-default', reason: 'connectivity probe with no tools: checks the event stream shape, not skill behavior' }),
tools: [],
permissionMode: 'bypassPermissions',
allowDangerouslySkipPermissions: true,
+1
View File
@@ -85,6 +85,7 @@ function newShip(f: Fixture, sc: Scenario): { calls: Call[]; ending: string } {
const diffBytes = plan.next.step === 'fix-check' ? f.diff_bytes * FIX_CHECK_FRACTION : h.fullPasses === 0 ? f.diff_bytes : f.diff_bytes * RANGE_FRACTION;
for (const d of plan.dispatches) calls.push({ role: d.role, name: d.name, effort: d.effort, diffBytes: d.role === 'outside-voice' ? f.diff_bytes : diffBytes });
adversarialRan ||= plan.dispatches.some(d => d.role === 'adversarial-native');
if (plan.next.step === 'voices') continue;
const o = outcomes[sc][Math.min(i, outcomes[sc].length - 1)] ?? {};
const last = plan.next.step === 'scoped' ? 'full' : plan.next.step;
h = { fullPasses: h.fullPasses + (last === 'full' ? 1 : 0), last, ...o };
+47 -5
View File
@@ -201,6 +201,24 @@ export function wallTimeoutForPackedShard(predictedMs: number, baseMs = DEFAULT_
// tighter — it keeps the per-file floor the runner has always guaranteed.
return Math.max(baseMs, Math.ceil(predictedMs * 3), fileCount * PER_FILE_WALL_MS);
}
/** The flaky-retry pass re-runs at most this many failing files, once, serially, in the same CI step. */
export const FLAKY_RETRY_CAP = 5;
/** No free-suite GitHub job gets a longer timeout; a plan whose shard would need more fails instead. */
export const MAX_CI_JOB_TIMEOUT_MINUTES = 90;
/**
* The GitHub job timeout for one planned CI shard (#3850): the wall --ci-run
* gives the shard, plus the flaky-retry pass's wall, plus the job's measured
* setup and teardown. GitHub must never kill the job before the runner can
* name a hung file and retry it; that kill uploads no result.
*/
export function ciShardJobTimeoutMinutes(shard: { shard: number; files: string[]; predictedMs: number }, setupSeconds: number): number {
const runnerMs = wallTimeoutForPackedShard(shard.predictedMs, DEFAULT_WALL_TIMEOUT_MS, shard.files.length) + wallTimeoutForShard(FLAKY_RETRY_CAP);
const minutes = Math.ceil((runnerMs / 1000 + setupSeconds) / 60);
if (minutes > MAX_CI_JOB_TIMEOUT_MINUTES) {
throw new Error(`FREE_JOB_TIMEOUT_OVER_CAP: CI shard ${shard.shard} needs a ${minutes}-minute job (runner wall plus retry plus ${setupSeconds} s setup), over the ${MAX_CI_JOB_TIMEOUT_MINUTES}-minute cap; plan more shards or a higher --shard-jobs instead of a longer job`);
}
return minutes;
}
/**
* Full-suite parallelism: use all available CPUs, with a floor of one and a
* per-platform cap (maxFullSuiteJobs). Shards stay serial internally; separate
@@ -453,6 +471,16 @@ export function createFreeCiPlan(files: string[], count: number, durations: Reco
return { ...body, id: planDigest(body) };
}
/**
* --ci-plan's stdout, the GitHub matrix. With --job-setup-seconds each entry
* carries the shard's job timeout (ciShardJobTimeoutMinutes); without it, the
* bare shard list the Windows lane reads.
*/
export function ciPlanMatrix(plan: FreeCiPlan, setupSeconds?: number): { shard: number[] } | { include: Array<{ shard: number; timeout_minutes: number }> } {
if (setupSeconds === undefined) return { shard: plan.shards.map(shard => shard.shard) };
return { include: plan.shards.map(shard => ({ shard: shard.shard, timeout_minutes: ciShardJobTimeoutMinutes(shard, setupSeconds) })) };
}
export function validateFreeCiPlan(plan: FreeCiPlan, files: string[], revision: string): void {
const { id, version, shards } = plan;
if (version !== 1 || plan.revision !== revision || !Array.isArray(shards) || !shards.length
@@ -575,6 +603,8 @@ type CliOptions = {
jobs?: number;
/** Set only by --shard-jobs (with --ci-plan); absent means k = 1 in every shard. */
shardJobs?: number;
/** Set only by --job-setup-seconds (with --ci-plan): the matrix then carries each shard's job timeout. */
jobSetupSeconds?: number;
};
export function parseCliOptions(argv: string[]): CliOptions {
@@ -591,6 +621,7 @@ export function parseCliOptions(argv: string[]): CliOptions {
let attributeHome = false;
let jobs: number | undefined;
let shardJobs: number | undefined;
let jobSetupSeconds: number | undefined;
const paths: Record<'ciPlan' | 'ciRun' | 'ciVerify' | 'result' | 'results' | 'filesFrom', string | null> = {
ciPlan: null, ciRun: null, ciVerify: null, result: null, results: null, filesFrom: null,
};
@@ -616,6 +647,11 @@ export function parseCliOptions(argv: string[]): CliOptions {
'--files-from': pathFlag('--files-from', 'filesFrom'),
'--jobs': (next) => { jobs = parseJobsFlag(next()); },
'--shard-jobs': (next) => { shardJobs = parseJobsFlag(next(), '--shard-jobs'); },
'--job-setup-seconds': (next) => {
const value = next() ?? '';
jobSetupSeconds = /^\d+$/.test(value) ? Number(value) : NaN;
if (!(jobSetupSeconds > 0 && jobSetupSeconds <= 3600)) throw new Error('--job-setup-seconds needs a whole number of seconds from 1 to 3600');
},
'--shards': (next) => {
const value = next();
if (!value) throw new Error('Missing value for --shards');
@@ -641,8 +677,9 @@ export function parseCliOptions(argv: string[]): CliOptions {
if (paths.ciVerify && !paths.results) throw new Error('--ci-verify requires --results');
if (jobs !== undefined && !paths.ciRun) rejectJobsWithoutCiRun();
if (shardJobs !== undefined && !paths.ciPlan) rejectShardJobsWithoutCiPlan();
if (jobSetupSeconds !== undefined && !paths.ciPlan) throw new Error('--job-setup-seconds applies only to --ci-plan');
if (quick && (recordDurations || windowsOnly || shardIndex !== null)) throw new Error('--quick cannot change recording, Windows or shard selection');
return { dryRun, listOnly, recordDurations, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit, quick, attributeHome, ...paths, ...(jobs === undefined ? {} : { jobs }), ...(shardJobs === undefined ? {} : { shardJobs }) };
return { dryRun, listOnly, recordDurations, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit, quick, attributeHome, ...paths, ...(jobs === undefined ? {} : { jobs }), ...(shardJobs === undefined ? {} : { shardJobs }), ...(jobSetupSeconds === undefined ? {} : { jobSetupSeconds }) };
}
const describeLines = (lines: string[]) => `${lines.slice(0, 5).join(', ')}${lines.length > 5 ? ` (+${lines.length - 5} more)` : ''}`;
@@ -1891,7 +1928,6 @@ async function retryFailedFreeFiles(
// Exists for syscall-supervised sandboxes (see fullSuiteJobs) where a run
// lands 0-1 spurious browser-timing failures under an otherwise-green
// suite. Capped so a genuinely broken tree never masquerades as flaky.
const RETRY_CAP = 5;
if (
worst !== 0
&& process.env.GSTACK_FREE_RETRY_FLAKY === '1'
@@ -1906,7 +1942,7 @@ async function retryFailedFreeFiles(
// mask the unattributable evidence as a FLAKY-PASS.
const allAttributed = outcomes.every((o) => hasScopedFailureAttribution(o) && (o.status === 'passed'
|| (o.failingFiles.length > 0 && o.unattributedFailures === 0)));
if (allAttributed && flakyFiles.length > 0 && flakyFiles.length <= RETRY_CAP) {
if (allAttributed && flakyFiles.length > 0 && flakyFiles.length <= FLAKY_RETRY_CAP) {
console.log(`[test:free] flaky-retry: re-running ${flakyFiles.length} failing file(s) once, serially: ${flakyFiles.join(', ')}`);
const retryOutcome = await runFreeShard(flakyFiles, totalShards + 1, totalShards + 1, {
wallTimeoutMs: shardTimeout(flakyFiles.length),
@@ -1942,7 +1978,7 @@ async function retryFailedFreeFiles(
console.error('[test:free] flaky-retry FAILED — the failures reproduce serially; not flaky.');
}
} else {
console.log(`[test:free] flaky-retry skipped: ${allAttributed ? `${flakyFiles.length} failing file(s) exceeds cap ${RETRY_CAP}` : 'failures not fully attributed'}.`);
console.log(`[test:free] flaky-retry skipped: ${allAttributed ? `${flakyFiles.length} failing file(s) exceeds cap ${FLAKY_RETRY_CAP}` : 'failures not fully attributed'}.`);
}
}
return { exitCode: worst, retry };
@@ -1978,8 +2014,14 @@ async function main(): Promise<number> {
warnUnseededFreeFiles(ciFiles, durations);
const plan = createFreeCiPlan(ciFiles, options.shardCount, durations, revision, options.shardJobs);
validateFreeCiPlan(plan, ciFiles, revision);
const matrix = ciPlanMatrix(plan, options.jobSetupSeconds);
if ('include' in matrix) {
const line = `job timeout per shard (runner wall + flaky retry + ${options.jobSetupSeconds} s setup): ${matrix.include.map(entry => `${entry.shard}=${entry.timeout_minutes}m`).join(' ')}`;
console.error(`[test:free] ${line}`);
appendStepSummary(line);
}
writeJson(options.ciPlan, plan);
console.log(JSON.stringify({ shard: plan.shards.map(shard => shard.shard) }));
console.log(JSON.stringify(matrix));
return 0;
}
const plan = JSON.parse(fs.readFileSync((options.ciRun ?? options.ciVerify)!, 'utf8')) as FreeCiPlan;