mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-11 11:10:13 +01:00
v1.93.14.0 stamp: merge main 7f2cfa6b81 into #3877 (land)
This commit is contained in:
garrytan committed
commit
9d4b41ba2b
163 files changed
+3400
-431
No files matched your search
@@ -55,6 +55,8 @@ export interface GenerationOptions {
|
||||
catalogMode?: 'trim' | 'full';
|
||||
explainLevel?: 'default' | 'terse';
|
||||
respectDetection?: boolean;
|
||||
/** A detection result the caller already read; wins over respectDetection so one read serves identity and render. */
|
||||
gbrainDetected?: boolean;
|
||||
log?: (message: string) => void;
|
||||
planMode?: boolean;
|
||||
/** Module loadout (scripts/modules.ts selectModules); null/undefined = `full`. */
|
||||
@@ -94,7 +96,7 @@ export interface GenerationResult {
|
||||
}
|
||||
|
||||
/** Canonical generation never reads local detection state unless opted in. */
|
||||
function loadGbrainOverride(respectDetection: boolean): boolean {
|
||||
export function loadGbrainOverride(respectDetection: boolean): boolean {
|
||||
if (!respectDetection) return false;
|
||||
const stateDir = resolveStateRoot();
|
||||
try {
|
||||
@@ -989,7 +991,7 @@ export async function runGeneration(settings: GenerationOptions = {}): Promise<G
|
||||
model: settings.model ?? null,
|
||||
catalogMode: settings.catalogMode ?? 'trim',
|
||||
explainLevel: settings.explainLevel ?? 'default',
|
||||
gbrainDetected: loadGbrainOverride(settings.respectDetection ?? false),
|
||||
gbrainDetected: settings.gbrainDetected ?? loadGbrainOverride(settings.respectDetection ?? false),
|
||||
planMode: settings.planMode ?? true,
|
||||
modules: settings.modules ?? null,
|
||||
};
|
||||
|
||||
@@ -13,41 +13,42 @@
|
||||
"files": {
|
||||
"scripts/lib/paid-cases.ts": {"blob":"f862cf5241580c7593b9ee62eacf5ef046248d26","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
|
||||
"scripts/lib/paid-plan.ts": {"blob":"e8ce3cb98474dded38c5128a631d283b0ca35b63","since":6,"nonBehavioral":"F1 rule-16 flaky rerun: report aggregation, rerun planning and the gate slice exit code; no case's execution, trial outcome, trial record or panel verdict changes, so no pass-rate series resets"},
|
||||
"scripts/lib/paid-report.ts": {"blob":"224dc5e8ba23a3589a37bad94db72ed1ca3c7c77","since":6,"nonBehavioral":"a case shard's result records are attributed to its one case: the trial's cost_usd and cost_known come from them, and a failed trial's failure_class, exit_reason and error come from its record instead of a bare assertion with the JUnit message, so a red whose record says infra or timeout is classed by that record (INFRA instead of VERDICT: rerun and bisect eligibility, not a pass rate); no case's outcome or selection changes and pass rates count outcomes only, so no pass-rate series resets"},
|
||||
"scripts/lib/paid-report.ts": {"blob":"e0d18da4ea02ac443835a68992abeea2b47f22fc","since":6,"nonBehavioral":"the headline adds only streamed session cost; a billed session's cost is already in its collector total; cache writes without a 5m key fall back to the total minus 1h; cost fields only"},
|
||||
"scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6},
|
||||
"scripts/lib/shard-engine.ts": {"blob":"dc714d7d897aed185c0ec3d2c92ad22d954039e5","since":6,"nonBehavioral":"createShardSandbox drops the Capy launch markers (GSTACK_SESSION_KIND, GSTACK_CAPY, GSTACK_EPHEMERAL, GSTACK_INSTALL_VERDICT) from shard environments; CI never sets them, so no case's input or verdict changes there, and on a Capy machine fixtures now see the session kind they set themselves instead of the installer's"},
|
||||
"scripts/test-paid-shards.ts": {"blob":"5439d0a8f6e1569fa7c924af20d67c30c81d5d30","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
|
||||
"scripts/test-pr-profile.ts": {"blob":"44ae88bebf8f716f70a3bd836c8bedb03e6653e8","since":6,"nonBehavioral":"FREE_ONLY_PR_FILES gains scripts/lib/shard-jobs.ts (free-runner module) and .github/workflows/windows-smoke.yml (a workflow no paid case reads); both change only which PR paths select paid cases, never how a case runs or is judged"},
|
||||
"scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6},
|
||||
"test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":6},
|
||||
"test/helpers/agent-sdk-runner.ts": {"blob":"d66dac458b8c13250973e550edf8087d4044cd0b","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets. Also: runAgentSdkTest now validates systemPrompt before starting; every live caller already passed the claude_code preset, so the prompt the model sees is unchanged and no verdict can change"},
|
||||
"test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":6},
|
||||
"test/helpers/claude-pty-runner.ts": {"blob":"ce7a5eaa66e502a052ea9c53348dd2ab25db6148","since":6,"nonBehavioral":"Removes the plan-design 0D focus picker from the shared harness; floor.ts gains an optional pickSetupChoice hook that no other caller passes, so no other case's input or verdict can change. plan-design-finding-floor, the only case whose seed raises that menu, passes the picker from its owned fixture test/fixtures/design-floor-focus-picker.ts, so its owned bytes start its own new series."},
|
||||
"test/helpers/codex-host-runner.ts": {"blob":"bff225017e235f0be6ece29f04971ac04b95b544","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
|
||||
"test/helpers/codex-session-runner.ts": {"blob":"c040ebbfa815b8cce19ef80c6133bd5a5474149a","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
|
||||
"test/helpers/e2e-gate.ts": {"blob":"efb7e197dd5f66c3ad0e34074b14e640754f7a45","since":6},
|
||||
"test/helpers/e2e-helpers.ts": {"blob":"6ee6937e6c3ca081c77131b5a755278c7b714834","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
|
||||
"test/helpers/e2e-helpers.ts": {"blob":"91996ff2bc303ee29f4e6f1705d1468b2536b32e","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"},
|
||||
"test/helpers/eval-budgets.ts": {"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":6},
|
||||
"test/helpers/eval-store.ts": {"blob":"a50ea2c28f508ad9f35ea9353dd2f502ee0dcfc2","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
|
||||
"test/helpers/eval-store.ts": {"blob":"4085ad18cd99698f675deadad2a76a796427b693","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"},
|
||||
"test/helpers/hermetic-env.ts": {"blob":"cc072895461dc52ea317d1beff72121342420d30","since":6},
|
||||
"test/helpers/llm-judge.ts": {"blob":"f14ca0db6504480698acd662351b6f0d6ef31a7b","since":6,"nonBehavioral":"llm-judge.ts adds a pass-through byte counter (bytesReceived, lastByteMs) to the aborted-judge diagnostic; the request, response bytes, deadline and scoring are unchanged"},
|
||||
"test/helpers/paid-test-set.ts": {"blob":"ff9bfe43c43ec2bdd11114f7b8ec7c5736116f75","since":6},
|
||||
"test/helpers/pty-screen.ts": {"blob":"4f77569f9ce11f9cee9f347216e468cd7c956820","since":6},
|
||||
"test/helpers/pty/auq.ts": {"blob":"16e5ea2bb5b9f28f44808a1a09024dbd3e6f28c4","since":6},
|
||||
"test/helpers/pty/binary.ts": {"blob":"404d7515076d5c00804f89e23ea69296312c00e1","since":6},
|
||||
"test/helpers/pty/boundaries.ts": {"blob":"c5520118da36f96f60a3c9720751bb57a669865c","since":6,"nonBehavioral":"Widens the plan-eng-review first-review severity annotation from [P0-3] to [P0-3|CRITICAL|INFORMATIONAL] (#3349). Only a question carrying the new labels can change verdict, and only the changed plan-eng-review prompt asks for them; every case that runs it owns that generated SKILL.md, which this change also edits, so those cases already start their own series. No other case's input or verdict can change."},
|
||||
"test/helpers/pty/boundaries.ts": {"blob":"c09b69611fb22ddf093adb2baa83434494e5f8fa","since":6,"nonBehavioral":"Widens the two plan-eng-review boundary predicates' qid regex to also accept the optional ;signal=<key> hint suffix (#3442). Only a question carrying that suffix can change verdict, and only the changed question-tuning resolver asks for it; every case that runs a skill with that resolver owns its generated SKILL.md, which this change also edits, so those cases already start their own series. No other case's input or verdict can change."},
|
||||
"test/helpers/pty/classify.ts": {"blob":"3f2daa12ff6484f96b8d28d910a50231c4bb3ae8","since":6},
|
||||
"test/helpers/pty/fake-session.ts": {"blob":"554e064ef259b8444280eddfe9b0db93c66a8202","since":6},
|
||||
"test/helpers/pty/judge.ts": {"blob":"81c5bb4c48ab928aca2f186a1ade9df7169d7fbe","since":6},
|
||||
"test/helpers/pty/launch.ts": {"blob":"e4413de380b799659170e7da248d51afda3b0e3f","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
|
||||
"test/helpers/pty/launch.ts": {"blob":"4f0731ef39bc3a6a7c5c592ce98690a24cb1ed7f","since":6,"nonBehavioral":"PTY transcript pricing finds the session by the caller's --session-id or the one new transcript for the launch cwd; the launcher adds no argument; native AUQ capture opts out; ledger cost fields only, no prompt, argv, answer, timeout or verdict changes"},
|
||||
"test/helpers/pty/plan-native.ts": {"blob":"ddba9cb68c7c0da935cff7c1cea258dee2dff740","since":6},
|
||||
"test/helpers/pty/runners/counting.ts": {"blob":"8c5c05e54abf961259d34d9c3d77db67138d7794","since":6},
|
||||
"test/helpers/pty/runners/floor.ts": {"blob":"a9938cb98f285474ebc4860cd61012504a36466e","since":6,"nonBehavioral":"The PTY floor and observation runners end a plan-mode turn at an idle prompt (the plan-mode footer without its running 'esc to interrupt' segment, after a 10 s settle) instead of waiting out the deadline. The floor runner exits as api_error only after the idle screen with its error panel held still for 10 s and every tick's question and assessment steps found nothing to credit (the transcript is re-read each tick), so a native or prose question already on screen is credited as before, even when its transcript line lands after the screen draws done (test/helpers/claude-pty-runner.runners.unit.test.ts, the prose-question-before-panel case). With no qualifying question an idle turn stays idle, so a trial that failed at the deadline as observer_timeout now fails earlier as api_error. An observation returns what the deadline would have returned. A running footer, extra footer text and a cut footer still wait for the deadline. No pass becomes a fail or a fail a pass. The session ledger may now record failure_cause api_error or refusal instead of observer_timeout or session_timeout for a plan-mode session whose last idle turn shows a panel; failure_class, the gate red class and pass rates are unchanged, so no pass-rate series resets"},
|
||||
"test/helpers/pty/runners/observation.ts": {"blob":"1c2b75fbb984708725abaa8939125f7cab969f9f","since":6},
|
||||
"test/helpers/pty/screen.ts": {"blob":"1384f765e48e511d5f843df2660b2ec70bbcb6b2","since":6,"nonBehavioral":"The PTY floor and observation runners end a plan-mode turn at an idle prompt (the plan-mode footer without its running 'esc to interrupt' segment, after a 10 s settle) instead of waiting out the deadline. The floor runner exits as api_error only after the idle screen with its error panel held still for 10 s and every tick's question and assessment steps found nothing to credit (the transcript is re-read each tick), so a native or prose question already on screen is credited as before, even when its transcript line lands after the screen draws done (test/helpers/claude-pty-runner.runners.unit.test.ts, the prose-question-before-panel case). With no qualifying question an idle turn stays idle, so a trial that failed at the deadline as observer_timeout now fails earlier as api_error. An observation returns what the deadline would have returned. A running footer, extra footer text and a cut footer still wait for the deadline. No pass becomes a fail or a fail a pass. The session ledger may now record failure_cause api_error or refusal instead of observer_timeout or session_timeout for a plan-mode session whose last idle turn shows a panel; failure_class, the gate red class and pass rates are unchanged, so no pass-rate series resets"},
|
||||
"test/helpers/pty/session.ts": {"blob":"54fe04b61d953e4e55e2fc251532c04a49fab66f","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
|
||||
"test/helpers/pty/session.ts": {"blob":"8fd1749a62f24dc37ae186c8a21c2165f7f7645e","since":6,"nonBehavioral":"PTY sessions price their ledger row (streamed_cost_usd) from the session transcript found by the caller's --session-id or the one new transcript for the launch cwd; ledger cost fields only, no prompt, argv, answer, timeout or verdict changes"},
|
||||
"test/helpers/pty/transcript-cost.ts": {"blob":"08f7ccbdbd9d71a5dc86edd2f8fced423a78ad9a","since":6,"nonBehavioral":"PTY transcript pricing finds the session by the caller's --session-id or the one new transcript for the launch cwd; the launcher adds no argument; native AUQ capture opts out; ledger cost fields only, no prompt, argv, answer, timeout or verdict changes"},
|
||||
"test/helpers/session-drain-policy.ts": {"blob":"6211239dd0485e6495a700b824a7e0e61d1bcd97","since":6},
|
||||
"test/helpers/session-ledger.ts": {"blob":"a10bf28bf100f135b93c73818670dd147253cdd3","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"},
|
||||
"test/helpers/session-runner.ts": {"blob":"cc65a66adafa9c053e3da3c6f17b5ac86c0eade7","since":6,"nonBehavioral":"Claude-host verdicts are unchanged: the Codex host (EVALS_HOST=codex, codex-host-runner) runs only when selected and records its own per-model series; the Codex runner refactor (execCodex extraction, onSpawn hook) keeps its events and verdicts and adds token cost accounting; host fields on the plan/report are absent for claude"},
|
||||
"test/helpers/session-ledger.ts": {"blob":"c99baaaa0a77a5a72384493fd8a85a4253ecb05d","since":6,"nonBehavioral":"PTY sessions price their ledger row (streamed_cost_usd) from the session transcript found by the caller's --session-id or the one new transcript for the launch cwd; ledger cost fields only, no prompt, argv, answer, timeout or verdict changes"},
|
||||
"test/helpers/session-runner.ts": {"blob":"c73b2596d9eb5377530a6bf1d284e0b6ea1556a0","since":6,"nonBehavioral":"Cost attribution only (GSTA-150 PR 2): a display-label record carries cost_case_id from runSkillTest's testName and the report binds its cost by it while the failed record and failure class still come from the named records (test/paid-report-cost.test.ts pins both); an aborted Agent SDK session logs its streamed usage as streamed_cost_usd, used only when a case has no record; a listed model-free case with no record or session is a known $0. No case's execution, outcome, failure class, rerun eligibility or panel verdict changes, so no pass-rate series resets"},
|
||||
"test/helpers/skill-fixture.ts": {"blob":"8a654dae6b0a396d839b02179119ae3e3bb13f69","since":6,"nonBehavioral":"extractSkillBody and extractSkillSections follow the shared-preamble hoist (pointer plus sections/preamble.md); every fixture they build is byte-identical to the pre-hoist output (compared for 8 body and 4 section extractions), so no case's input or verdict changes; the /ship review then reworded only comments and two error messages, so no fixture byte changes"},
|
||||
"test/helpers/skill-startup.ts": {"blob":"a8d21e3817c3f35a1d06af325c92fde7d420e377","since":6,"nonBehavioral":"readSkillEffective also appends a variant's added ### subsections (office-hours' Open-ended questions); readSkillStartup, readSharedSections and readSkillEffective return bytes identical to the pinned helper for every tier 2+ render without added subsections (735 across all hosts) and for office-hours' SKILL.md before this PR, so office-hours' fixture input changes only because its prompt changed in this PR, not because the helper did"},
|
||||
"test/helpers/test-selection.ts": {"blob":"e1b54ad5e3d3eadd8027de1b44e57554d6d99a41","since":6,"nonBehavioral":"selection only: fragment exemption narrowed to .md files directly in the fragments dir; changed-file lists include a rename's old path; no case prompt, fixture, judge or verdict logic changes (GST-400)"},
|
||||
|
||||
@@ -9,7 +9,7 @@ import { STALL_WINDOW_MS } from '../../test/helpers/eval-budgets';
|
||||
import {
|
||||
isFinalizedEvalResultFile, credentialRejection, failureClassOf, panelVerdict, sanitizeTrialError, formatTrialOutcomes, TRIAL_OUTCOME_SCHEMA, TRIAL_OUTCOMES_FILE,
|
||||
type EvalCaseKind, type PanelVerdict, type TrialFailureClass, type TrialOutcome, type TrialOutcomeRecord,
|
||||
trialCostKnown, trialFailureFields, trialSessions, type TrialFailureDetail,
|
||||
trialFailureFields, trialSessions, type TrialFailureDetail,
|
||||
} from '../../test/helpers/eval-store';
|
||||
import { readSessionLedger, type SessionLedgerRow } from '../../test/helpers/session-ledger';
|
||||
import { publishedFence, sanitizeFixedFenceLine } from './published-text';
|
||||
@@ -230,6 +230,8 @@ export interface JUnitCensus {
|
||||
deselected: Map<string, number>;
|
||||
/** Per shard key: skip reason -> skipped testcase count. */
|
||||
shardSkipReasons: Map<string, Map<string, number>>;
|
||||
/** Streamed session cost (sessions without a terminal result), which no collector record carries; the headline adds it to the collectors' total_cost_usd. */
|
||||
ledgerCostUsd: number;
|
||||
/** `<shard key> :: <test name> (<outcome>)` for every testcase no rule attributes. */
|
||||
unattributed: string[];
|
||||
}
|
||||
@@ -241,6 +243,24 @@ function caseSessions(rows: SessionLedgerRow[], id: string, onlyCase: boolean):
|
||||
return rows.filter(row => row.case === id || (!row.case && (onlyCase || row.test_name === id || row.test_name === CASE_TEST_NAMES[id])));
|
||||
}
|
||||
|
||||
/** Cases that run no model (a binary data flow): with no record and no session they cost a known $0. test/paid-report-cost.test.ts pins each body model-free. */
|
||||
export const MODEL_FREE_CASES: ReadonlySet<string> = new Set(['timeline-event-flow']);
|
||||
|
||||
/**
|
||||
* A trial's cost: its records' sum plus what no record carries, a session without a terminal result priced from its
|
||||
* streamed usage (streamed_cost_usd). With no record, a billed session counts its own cost_usd (Codex). Unknown when any
|
||||
* record or session cost is; a listed model-free case with neither record nor session is a known $0.
|
||||
*/
|
||||
export function caseCost(id: string, records: any[], sessions: SessionLedgerRow[]): { cost_usd: number; cost_known?: false } {
|
||||
if (!records.length && !sessions.length) return MODEL_FREE_CASES.has(id) ? { cost_usd: 0 } : { cost_usd: 0, cost_known: false };
|
||||
const unrecorded = sessions.map(s => s.billed !== false ? (records.length ? 0 : s.cost_usd) : s.streamed_cost_usd);
|
||||
const known = records.every(r => r?.cost_known !== false) && unrecorded.every(c => typeof c === 'number' && Number.isFinite(c));
|
||||
const total = records.reduce((sum, r) => sum + (Number(r?.cost_usd) || 0), 0) + knownSum(unrecorded);
|
||||
return { cost_usd: Math.round(total * 100) / 100, ...(known ? {} : { cost_known: false as const }) };
|
||||
}
|
||||
|
||||
const knownSum = (costs: Array<number | undefined>) => costs.reduce<number>((sum, c) => sum + (typeof c === 'number' && Number.isFinite(c) ? c : 0), 0);
|
||||
|
||||
/** The artifact holding a slice's shards: the downloaded directory's own name when it is one. */
|
||||
/** The red-census guide the report and every eval:pass-rates view link. */
|
||||
export const CENSUS_RED_GUIDE = 'docs/evals/census-red.md';
|
||||
@@ -252,7 +272,8 @@ export function sliceArtifactName(root: string, slice: number, attempt: number):
|
||||
|
||||
export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact[], recordsByShard: Map<string, any[]>,
|
||||
base: Record<string, unknown>, cliVersion?: string): JUnitCensus {
|
||||
const census: JUnitCensus = { ruleCases: [], history: [], failedShards: new Set(), skipped: [], deselected: new Map(), shardSkipReasons: new Map(), unattributed: [] };
|
||||
const census: JUnitCensus = { ruleCases: [], history: [], failedShards: new Set(), skipped: [], deselected: new Map(), shardSkipReasons: new Map(), unattributed: [],
|
||||
ledgerCostUsd: 0 };
|
||||
const entries = new Map(manifest.entries.map(entry => [normalizeRelativePath(entry.file), entry]));
|
||||
for (const { root, result } of artifacts) {
|
||||
for (const outcome of result.outcomes) {
|
||||
@@ -277,11 +298,15 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
|
||||
const caseOutcome: TrialOutcome = failed ? 'failed' : cases.some(tc => tc.outcome === 'passed') ? 'passed' : 'skipped';
|
||||
// A record is named by its case id or carries case_id; an E2E record named by its display label ("/qa fix loop") is
|
||||
// the shard's only case's when the shard ran one case, else it stays unattributed and the trial's cost unknown.
|
||||
const mine = records.filter((r: any) => r?.name === id || r?.case_id === id || (byCase.size === 1 && !r?.case_id));
|
||||
const failedRecord = mine.find((r: any) => r.passed === false);
|
||||
// A display-label record whose runSkillTest testName is this case (cost_case_id) adds its cost only: the failed record stays the named one.
|
||||
const named = records.filter((r: any) => r?.name === id || r?.case_id === id || (byCase.size === 1 && !r?.case_id));
|
||||
const mine = [...named, ...records.filter((r: any) => r?.cost_case_id === id && !named.includes(r) && !byCase.has(r?.name))];
|
||||
const failedRecord = named.find((r: any) => r.passed === false);
|
||||
const recorded: TrialFailureClass | undefined = !failed ? undefined
|
||||
: failed.failureType === 'TimeoutError' ? 'timeout' : failedRecord ? failureClassOf(failedRecord) : 'assertion';
|
||||
const sessions = caseSessions(ledger, id, byCase.size === 1);
|
||||
const cost = caseCost(id, mine, sessions);
|
||||
census.ledgerCostUsd += knownSum(sessions.map(s => s.billed === false ? s.streamed_cost_usd : 0));
|
||||
const credentials = failed && recorded !== 'contract' ? credentialRejection({ error: `${failedRecord?.error ?? ''}\n${failed.message ?? ''}`, sessions }) : undefined;
|
||||
const failureClass = credentials ? 'infra' : recorded;
|
||||
const rawError = credentials ? `credentials rejected: ${credentials}` : failedRecord?.error ?? failed?.message;
|
||||
@@ -295,7 +320,7 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
|
||||
census.skipped.push(`◌ ${id} ${key} (${cases.map(tc => tc.name).join('; ')}): ${RUNTIME_SKIP_REASON}`);
|
||||
}
|
||||
census.ruleCases.push({ id, kind, outcome: caseOutcome, key, slice: result.sliceIndex, ...(failureClass ? { failureClass } : {}), ...(error && failed ? { error } : {}),
|
||||
costUsd: Math.round(mine.reduce((sum: number, r: any) => sum + (Number(r.cost_usd) || 0), 0) * 100) / 100, costKnown: trialCostKnown(mine, sessions).cost_known !== false,
|
||||
costUsd: cost.cost_usd, costKnown: cost.cost_known !== false,
|
||||
...(failed ? { line: `✗ ${id} ${kind} FAIL ${failureClass}${failedRecord?.exit_reason === 'timeout' && failedRecord?.timeout_at_turn !== undefined ? ` at turn ${failedRecord.timeout_at_turn}` : ''}${error ? ` — ${error}` : ''}`
|
||||
+ `${diagnosisText(diagnosis, error)} [slice ${result.sliceIndex}, attempt ${result.attempt ?? 1}]`
|
||||
+ `${evidencePointer(sliceArtifactName(root, result.sliceIndex, result.attempt ?? 1), key, typeof base.run_id === 'string' ? base.run_id : undefined, id)}`
|
||||
@@ -303,8 +328,8 @@ export function junitCensus(manifest: PaidRunManifest, artifacts: ReportArtifact
|
||||
census.history.push({ ...base, case: id, file: shardFile(key), kind, trial: 1, panel: { n: 1, k: 1 }, outcome: caseOutcome,
|
||||
...(failureClass ? { failure_class: failureClass } : {}), ...(failedRecord?.exit_reason ? { exit_reason: String(failedRecord.exit_reason) } : {}),
|
||||
...(error && failed ? { error } : {}), ...diagnosis, ...trialSessions(sessions), duration_ms: Math.min(cases.reduce((sum, tc) => sum + tc.timeMs, 0), outcome.elapsedMs),
|
||||
cost_usd: Math.round(mine.reduce((sum: number, r: any) => sum + (Number(r.cost_usd) || 0), 0) * 100) / 100, ...trialCostKnown(mine, sessions),
|
||||
...(typeof mine[0]?.model === 'string' ? { model: mine[0].model } : {}), ...(cliVersion ? { cli_version: cliVersion } : {}),
|
||||
...cost,
|
||||
...(typeof named[0]?.model === 'string' ? { model: named[0].model } : {}), ...(cliVersion ? { cli_version: cliVersion } : {}),
|
||||
quarantined: false, execution: outcome.reused ? 'reused' : 'executed', source: 'junit' } as TrialOutcomeRecord);
|
||||
}
|
||||
census.shardSkipReasons.set(key, reasons);
|
||||
@@ -776,7 +801,7 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
...(spend.executions ? { reruns: spend } : {}),
|
||||
actionRequired: verdict.problems.length,
|
||||
wallMs: wall.length ? Math.max(...wall.map(r => r.finishedAt!)) - Math.min(...wall.map(r => r.startedAt!)) : null,
|
||||
costUsd: Math.round(costUsd * 100) / 100,
|
||||
costUsd: Math.round((costUsd + census.ledgerCostUsd) * 100) / 100,
|
||||
redispatchEligible: red && infraOnly(verdict.problems),
|
||||
};
|
||||
const executed = history.filter(r => r.attempt === primary && r.execution === 'executed' && r.outcome !== 'skipped');
|
||||
|
||||
@@ -26,7 +26,8 @@ import { createHash, randomBytes } from 'node:crypto';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { runGeneration } from './gen-skill-docs';
|
||||
import { readConfigKey } from '../lib/state-root';
|
||||
import { loadGbrainOverride, runGeneration } from './gen-skill-docs';
|
||||
import { resolveModel } from './models';
|
||||
import { ModuleError, type ModuleSelection } from './modules';
|
||||
import { LOCK_FILE, danglingReferences } from './modules-render';
|
||||
@@ -148,6 +149,10 @@ export async function installLoadout(request: InstallRequest): Promise<InstallRe
|
||||
const planMode = request.planMode ?? true;
|
||||
const model = request.boundModel ? resolveModel(request.boundModel) : null;
|
||||
const selection: ModuleSelection = { ...request.selection, boundModel: request.boundModel ?? undefined, fullRoot: request.boundModel ? request.fullRoot : undefined };
|
||||
const gbrainDetected = loadGbrainOverride(true);
|
||||
const disabledSkills = [...new Set((readConfigKey('disabled_skills') ?? '').replace(/["']/g, '').split(/[\s,]+/)
|
||||
.map(n => n.replace(/^\//, '')).filter(n => n !== 'gstack' && n !== 'gstack-upgrade')
|
||||
.map(n => n.replace(/^gstack-/, '')).filter(n => /^[a-z0-9-]+$/.test(n)))].sort();
|
||||
const identity = {
|
||||
loadout: selection.name,
|
||||
excluded: [...selection.excluded].sort(),
|
||||
@@ -157,6 +162,8 @@ export async function installLoadout(request: InstallRequest): Promise<InstallRe
|
||||
host,
|
||||
overlay_model: model,
|
||||
plan_mode: planMode,
|
||||
gbrain_detected: gbrainDetected,
|
||||
disabled_skills: disabledSkills,
|
||||
source_tree: sourceTree(source),
|
||||
};
|
||||
const identitySha = sha256(JSON.stringify(identity));
|
||||
@@ -174,7 +181,7 @@ export async function installLoadout(request: InstallRequest): Promise<InstallRe
|
||||
}
|
||||
const stage = `${final}.stage-${process.pid}-${randomBytes(4).toString('hex')}`;
|
||||
try {
|
||||
const result = await runGeneration({ host, outputRoot: stage, contentLinkRoot: final, installRoot: request.fullRoot && request.fullRoot !== path.join(os.homedir(), '.claude/skills') ? path.join(request.fullRoot, 'gstack') : null, model, planMode, modules: selection });
|
||||
const result = await runGeneration({ host, outputRoot: stage, contentLinkRoot: final, installRoot: request.fullRoot && request.fullRoot !== path.join(os.homedir(), '.claude/skills') ? path.join(request.fullRoot, 'gstack') : null, model, planMode, modules: selection, gbrainDetected, disabledSkills });
|
||||
const errors = result.diagnostics.filter(d => d.kind === 'error').map(d => d.message);
|
||||
if (result.exitCode !== 0 || errors.length) {
|
||||
throw new ModuleError('RENDER_FAILED', `gen-skill-docs failed: ${errors.join('; ') || 'exit 1'}`, 'fix the named template or module, then rerun; side effects: none, safe to retry');
|
||||
|
||||
@@ -21,6 +21,7 @@ import '../lib/conductor-env-shim';
|
||||
import { query, type SDKMessage } from '@anthropic-ai/claude-agent-sdk';
|
||||
import { readOverlay } from './resolvers/model-overlay';
|
||||
import { resolveClaudeBinary } from '../lib/claude-bin';
|
||||
import { sdkSystemPrompt } from '../test/helpers/sdk-system-prompt';
|
||||
|
||||
const USAGE = `Usage: bun run scripts/preflight-agent-sdk.ts [--live]
|
||||
|
||||
@@ -94,7 +95,7 @@ async function main() {
|
||||
prompt: 'say pong',
|
||||
options: {
|
||||
model: 'claude-opus-4-7',
|
||||
systemPrompt: '',
|
||||
...sdkSystemPrompt({ type: 'sdk-default', reason: 'connectivity probe with no tools: checks the event stream shape, not skill behavior' }),
|
||||
tools: [],
|
||||
permissionMode: 'bypassPermissions',
|
||||
allowDangerouslySkipPermissions: true,
|
||||
|
||||
@@ -306,6 +306,42 @@ export function resolveOptionKey(question_id: string, choice: string): string |
|
||||
return tie ? undefined : best;
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a choice on an unregistered question to an option key of its hinted
|
||||
* signal (#3442). Only the exact key or the key's own words count, never
|
||||
* OPTION_LABEL_CUES: a signal's keys mix several questions' options, so a
|
||||
* cue such as "accept" could map a "yes" to the opposite delta. The longest
|
||||
* matching key wins; a tie or no match returns undefined.
|
||||
*/
|
||||
export function resolveHintedOptionKey(signal_key: string, choice: string): string | undefined {
|
||||
if (!Object.hasOwn(SIGNAL_MAP, signal_key)) return undefined;
|
||||
const keys = Object.keys(SIGNAL_MAP[signal_key]!);
|
||||
if (keys.includes(choice)) return choice;
|
||||
const label = normalizeOptionLabel(choice);
|
||||
if (!label) return undefined;
|
||||
let best: string | undefined;
|
||||
let bestWords = 0;
|
||||
let tie = false;
|
||||
for (const key of keys) {
|
||||
const words = key.replace(/-/g, ' ');
|
||||
if (label !== words && !label.startsWith(words + ' ')) continue;
|
||||
const n = words.split(' ').length;
|
||||
if (n > bestWords) {
|
||||
best = key;
|
||||
bestWords = n;
|
||||
tie = false;
|
||||
} else if (n === bestWords) {
|
||||
tie = true;
|
||||
}
|
||||
}
|
||||
return tie ? undefined : best;
|
||||
}
|
||||
|
||||
/** One line per signal key with its option keys, as `--signal-keys` prints it. */
|
||||
export function signalKeyLines(): string[] {
|
||||
return Object.entries(SIGNAL_MAP).map(([key, byChoice]) => `${key}: ${Object.keys(byChoice).join(' | ')}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate that every signal_key referenced in the registry has a matching
|
||||
* entry in SIGNAL_MAP. Called by tests to catch drift.
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
/**
|
||||
* The <gstack-qid:...> marker a skill puts in an asked question (#3442).
|
||||
*
|
||||
* `<gstack-qid:{id}>` names the question; an unregistered id may add a
|
||||
* signal hint, `<gstack-qid:{id};signal={signal_key}>`, naming the
|
||||
* SIGNAL_MAP key its answer speaks to. Ids and keys are read lowercase.
|
||||
* Whether a hint is a real SIGNAL_MAP key is checked where the row is
|
||||
* logged (bin/gstack-question-log), not here.
|
||||
*/
|
||||
export const QID_MARKER_RE = /<gstack-qid:([a-z0-9-]{1,64})(?:;signal=([a-z0-9-]{1,64}))?>/i;
|
||||
|
||||
export interface QidMarker {
|
||||
id: string;
|
||||
signal_key?: string;
|
||||
stripped: string;
|
||||
}
|
||||
|
||||
export function parseQidMarker(text: string): QidMarker | undefined {
|
||||
const m = QID_MARKER_RE.exec(text);
|
||||
if (!m) return undefined;
|
||||
return {
|
||||
id: m[1]!.toLowerCase(),
|
||||
...(m[2] ? { signal_key: m[2].toLowerCase() } : {}),
|
||||
stripped: text.replace(QID_MARKER_RE, '').trim(),
|
||||
};
|
||||
}
|
||||
@@ -27,9 +27,9 @@ export function generateQuestionTuning(ctx: TemplateContext): string {
|
||||
const registry = `${ctx.paths.skillRoot}/scripts/question-registry.ts`;
|
||||
return `## Question Tuning (skip entirely if \`QUESTION_TUNING: false\`)
|
||||
|
||||
Before each decision brief (AskUserQuestion or Conductor/fallback prose), choose \`question_id\` from \`${registry}\` or \`{skill}-{slug}\`, then run \`${bin}/gstack-question-preference --check "<id>"\`; for an unregistered id, write the question summary to \`.gstack/tmp/qt.txt\` (file-write tool) and append \`--summary-file .gstack/tmp/qt.txt\` (one-way keyword check). \`AUTO_DECIDE\` means choose the recommended option and say "Auto-decided [summary] → [option] (your preference). Change with /plan-tune." \`ASK_NORMALLY\` means ask.
|
||||
Before each decision brief (AskUserQuestion or Conductor/fallback prose), choose \`question_id\` from \`${registry}\` or \`{skill}-{slug}\`, then run \`${bin}/gstack-question-preference --check "<id>"\`; for an unregistered id, write the question summary to \`.gstack/tmp/qt.txt\` (file-write tool) and append \`--summary-file .gstack/tmp/qt.txt\` (one-way keyword check); its \`--signal-keys\` prints an optional signal-key hint. \`AUTO_DECIDE\` means choose the recommended option and say "Auto-decided [summary] → [option] (your preference). Change with /plan-tune." \`ASK_NORMALLY\` means ask.
|
||||
|
||||
**Embed the question_id as a marker in every asked brief**, ad hoc IDs included, with one ID for check, marker and log. Include \`<gstack-qid:{question_id}>\` once in the question text itself, not only a command or log. On prose paths, use the explicit reply line. Without the marker, the PreToolUse hook treats AskUserQuestion as observed-only and never auto-decides.
|
||||
**Embed the question_id as a marker in every asked brief**, ad hoc IDs included, with one ID for check, marker and log. Include \`<gstack-qid:{question_id}>\` once in the question text itself, not only a command or log. On prose paths, use the explicit reply line. Without the marker the hook never auto-decides.
|
||||
|
||||
**Embed the option recommendation via the \`(recommended)\` label suffix** on exactly one option per AUQ. The PreToolUse hook parses it first, falls back to "Recommendation: X" prose, and refuses when ambiguous (two labels = refuse).
|
||||
|
||||
|
||||
@@ -85,6 +85,7 @@ function newShip(f: Fixture, sc: Scenario): { calls: Call[]; ending: string } {
|
||||
const diffBytes = plan.next.step === 'fix-check' ? f.diff_bytes * FIX_CHECK_FRACTION : h.fullPasses === 0 ? f.diff_bytes : f.diff_bytes * RANGE_FRACTION;
|
||||
for (const d of plan.dispatches) calls.push({ role: d.role, name: d.name, effort: d.effort, diffBytes: d.role === 'outside-voice' ? f.diff_bytes : diffBytes });
|
||||
adversarialRan ||= plan.dispatches.some(d => d.role === 'adversarial-native');
|
||||
if (plan.next.step === 'voices') continue;
|
||||
const o = outcomes[sc][Math.min(i, outcomes[sc].length - 1)] ?? {};
|
||||
const last = plan.next.step === 'scoped' ? 'full' : plan.next.step;
|
||||
h = { fullPasses: h.fullPasses + (last === 'full' ? 1 : 0), last, ...o };
|
||||
|
||||
Reference in new issue
Block a user