Merge origin/main (v1.65.0.0 fork port wave 2) into test-evals-ci-speedup

Second overlapping-wave merge; resolutions compose intent:

- TEST_ROOTS: ours is the superset (main also wired ios-qa/daemon/test;
  ours additionally has ios-qa/scripts + browser-skills). package.json
  'test' keeps routing through the canonical strict runner.
- gbrainAvailable: main fixed the same load-flake with a strictly better
  mechanism (memoized stat-based PATH scan, no subprocess at all) —
  theirs supersedes this branch's memoized-exec probe. Main also made
  the query timeout env-overridable (GSTACK_BRAIN_TIMEOUT_MS).
- Model defaults: adopted main's lib/eval-model.ts abstraction (one
  resolution point, env-overridable per kind) and applied decision D1a
  inside it: capture defaults to Sonnet (Opus opt-in via explicit arg or
  GSTACK_EVAL_MODEL_CAPTURE); test pins updated to follow.
- Parent watchdog: main's rewrite (named parameterized tick, driven
  deterministically by its test via __testInternals__, plus handoff
  suppression semantics from session persistence) supersedes this
  branch's env-tunable interval; adopted their server + test wholesale.
- windows-free-tests: ours (curated bun run test:windows) — main's
  hand-list grew by one more file, which the curated runner subsumes
  automatically; that drift is the reason for D11.
- context-skills 0-for-26 fix: both waves made the IDENTICAL fix; kept
  this branch's comment (carries the receipts).
- .gitignore: main's superset (also ignores Package.resolved — their
  never-commit call; untracked the copy this branch had committed).

Verified: 239-test merge battery green, watchdog 8/8, eval-model 5/5,
actionlint clean, eval:select works.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-15 11:56:34 -07:00
co-authored by Claude Fable 5
200 changed files with 11004 additions and 1207 deletions
+4 -3
View File
@@ -11,6 +11,7 @@
* zero rendering loss. The TTY rendering layer is identical for fat and slim
* skills, so it is not where token-reduction degradation can hide.
*/
import { resolveEvalModel } from '../../lib/eval-model';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
@@ -191,7 +192,7 @@ This is a capture test, not an interactive session. Skip any system-audit / envi
timeout: 240_000,
testName: opts.testName,
runId: opts.runId,
model: opts.model ?? 'claude-sonnet-4-6', // D1a: align with session-runner default; Opus is opt-in
model: resolveEvalModel('capture', opts.model),
});
try {
@@ -253,7 +254,7 @@ Rules for this run:
timeout: opts.timeout ?? 300_000,
testName: opts.testName,
runId: opts.runId,
model: opts.model ?? 'claude-sonnet-4-6', // D1a: align with session-runner default; Opus is opt-in
model: resolveEvalModel('capture', opts.model),
});
const readSections = new Set<string>();
@@ -334,7 +335,7 @@ Write the verbatim text of that AskUserQuestion (the full decision brief: title,
timeout: 240_000,
testName: opts.testName,
runId: opts.runId,
model: opts.model ?? 'claude-sonnet-4-6', // D1a: align with session-runner default; Opus is opt-in
model: resolveEvalModel('capture', opts.model),
});
try {
+43 -22
View File
@@ -99,6 +99,7 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
ship: {
skill: 'ship',
expectedSections: [
'apple-release.md',
'tests.md',
'test-coverage.md',
'plan-completion.md',
@@ -129,7 +130,16 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
minUnionBytes: 120_000,
mustContain: ['VERSION', 'CHANGELOG', 'review', 'merge', 'PR'],
// v1.58.5.0: pre-push-guard install (#2077) stacks on the shared first-run-guidance preamble.
maxSizeRatio: 1.08,
// Fork port wave 2: multi-ecosystem test-detection evidence (Django/JVM
// markers, test-file census — e3259078 port) + the #1079 gh pr edit REST
// fallback grew the union to 1.090x; the third-party web-actions
// contract (consent-gated browser drive for API-key registration etc.)
// adds ~2.3KB inline judgment, measured 1.103x. The Apple release
// adapter (14.8KB carved section, 21 live releases of judgment — the
// wave's headline capability) grows the union to 1.195x. Deliberate:
// the section is on-demand (loads only for Apple store targets), so
// per-invocation cost for non-iOS ships is one manifest line.
maxSizeRatio: 1.22,
},
'plan-ceo-review': {
skill: 'plan-ceo-review',
@@ -144,9 +154,10 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
},
behavioral: 'external',
externalTest: 'test/skill-e2e-plan-ceo-review-section-loading.test.ts',
// v1.64.1.0: shared-preamble prose from the two parallel v1.64 waves lands
// the skeleton at 90,280 B; +1 KB headroom.
maxSkeletonBytes: 91_000,
// v1.65 merge: provisional larger-of-both-waves budget; re-measured below.
// Fork port wave 2 (#703): the repo-doc-preference block in the design
// check grew every plan-review skeleton ~0.7KB. Measured values noted.
maxSkeletonBytes: 92_500, // v1.64+v1.65 merge: both waves' preamble growth; measured 92,004
minUnionBytes: 80_000,
mustContain: ['SCOPE EXPANSION', 'SELECTIVE EXPANSION', 'HOLD SCOPE', 'SCOPE REDUCTION'],
// Default-on Codex outside-voice (codexPreflight block + CODEX_MODE branch
@@ -167,9 +178,10 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
behavioral: 'plan',
// v1.2.0 activation lift (shared first-run-guidance preamble) + #2077 ask-first scope gate.
// +~1 KB: plan-mode auto-select-B scope-gate exceptions (2026-08).
// v1.64.1.0: shared-preamble prose from the two parallel v1.64 waves lands
// the skeleton at 68,163 B; +~1 KB headroom.
maxSkeletonBytes: 69_000,
// v1.65 merge: provisional larger-of-both-waves budget; re-measured below.
// Fork port wave 2 (#703): the repo-doc-preference block in the design
// check grew every plan-review skeleton ~0.7KB. Measured values noted.
maxSkeletonBytes: 70_000, // measured 68,780
minUnionBytes: 70_000,
mustContain: ['Architecture', 'Code Quality', 'Test', 'Performance'],
// Cross-cutting preamble growth (v1.57.2.0 AUQ-failure prose fallback + the
@@ -180,7 +192,7 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
// 1.08 → 1.10: the scope-gate exceptions block (+ its adversarial-review
// hardening: host-anchored mode signal, precedence, passing-mention
// guards) and the plan-mode preamble reword land the union at 1.092.
maxSizeRatio: 1.10,
maxSizeRatio: 1.12, // measured 1.103
},
'plan-design-review': {
skill: 'plan-design-review',
@@ -198,13 +210,14 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
// always-loaded AskUserQuestion Format section.
// v1.2.0 activation lift (shared first-run-guidance preamble) + #2077 ask-first scope gate.
// +~1.3 KB: plan-mode auto-select-B scope-gate exceptions (2026-08).
// +~340 B: telemetry --error-message/--failed-step flags + prose in the
// shared completion-status preamble (PR #769, 2026-08); this skill was the
// closest to its ceiling (landed 89040 / ratio 1.072).
maxSkeletonBytes: 89_400,
// Fork port wave 2 (D1): evidence directive adds ~0.45KB to every
// tier-2+ skeleton (measured 89,184). Main's v1.64.0.0 adds ~340 B more
// (telemetry --error-message/--failed-step preamble prose, PR #769).
// Budget covers the sum of both waves.
maxSkeletonBytes: 91_000,
minUnionBytes: 70_000,
mustContain: ['design', 'visual'],
maxSizeRatio: 1.08,
maxSizeRatio: 1.12, // D1 1.104 + main's ~0.008
},
'plan-devex-review': {
skill: 'plan-devex-review',
@@ -221,7 +234,9 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
// +Conductor AUQ-default-prose rule + one-way/destructive prose safety +
// continuation protocol in the always-loaded AskUserQuestion Format section.
// v1.2.0 activation lift: first-run-guidance section in the shared preamble.
maxSkeletonBytes: 80_000,
// Fork port wave 2 (#703): the repo-doc-preference block in the design
// check grew every plan-review skeleton ~0.7KB. Measured values noted.
maxSkeletonBytes: 82_000, // measured 80,493
minUnionBytes: 70_000,
mustContain: ['developer experience', 'Getting Started'],
// Default-on Codex outside-voice (codexPreflight block + CODEX_MODE branch
@@ -244,12 +259,15 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
behavioral: 'prompt',
// v1.2.0 activation lift: first-run-guidance section in the shared preamble,
// plus the P1 office-hours closing handoff (AUQ that launches the next skill).
// v1.64.1.0: shared-preamble prose from the two parallel v1.64 waves lands
// the skeleton at 98,193 B; +~1 KB headroom.
maxSkeletonBytes: 99_000,
// v1.65 merge: provisional larger-of-both-waves budget; re-measured below.
// Fork port wave 2: the third-party web-actions contract sits inline
// (judgment must be visible before the workflow directs the user to a
// vendor site), plus the #703 dual-write + repo-doc-preference block and
// the #538 opt-out + D1 evidence directive — ratio 1.104 measured.
maxSkeletonBytes: 101_000,
minUnionBytes: 70_000,
mustContain: ['design doc', 'problem statement'],
maxSizeRatio: 1.07,
maxSizeRatio: 1.12,
},
'document-release': {
skill: 'document-release',
@@ -267,7 +285,7 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
// +Conductor AUQ-default-prose rule + one-way/continuation safety in the
// always-loaded AskUserQuestion Format section.
// v1.2.0 activation lift: first-run-guidance section in the shared preamble.
maxSkeletonBytes: 56_000,
maxSkeletonBytes: 56_500, // v1.64+v1.65 merge; measured 56,044
minUnionBytes: 55_000,
mustContain: ['CHANGELOG', 'Diataxis', 'coverage'],
// Two intentional additions stack on this small skill: the AUQ-failure prose
@@ -295,6 +313,7 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
// +Conductor AUQ-default-prose rule + one-way/continuation safety in the
// always-loaded AskUserQuestion Format section.
// v1.2.0 activation lift: first-run-guidance section in the shared preamble.
// v1.65 merge: provisional larger-of-both-waves budget; re-measured below.
// v1.64.1.0: shared-preamble prose from the two parallel v1.64 waves lands
// the skeleton at 69,022 B; +~1 KB headroom.
maxSkeletonBytes: 70_000,
@@ -303,7 +322,8 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
// Cross-cutting preamble growth (v1.57.2.0 AUQ-failure prose fallback ~2KB +
// the cross-session decision-memory nudge) lands this carved skeleton just over
// the strict 1.05; headroom for the shared preamble additions.
maxSizeRatio: 1.07,
// v1.64+v1.65 merge sums both waves' preamble growth; measured 1.073.
maxSizeRatio: 1.08,
},
cso: {
skill: 'cso',
@@ -336,13 +356,14 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
// +Conductor AUQ-default-prose rule + one-way/continuation safety in the
// always-loaded AskUserQuestion Format section.
// v1.2.0 activation lift: first-run-guidance section in the shared preamble.
maxSkeletonBytes: 75_000,
maxSkeletonBytes: 75_800, // v1.64+v1.65 merge; measured 75,364
minUnionBytes: 72_000,
mustContain: ['OWASP', 'STRIDE', 'daily', 'comprehensive', 'verif'],
// cso keeps its mode-dispatch + FP-filtering phases always-loaded, so the
// cross-cutting preamble growth (v1.57.2.0 AUQ-failure prose fallback ~2KB + the
// decision-memory nudge) lands it just over 1.05; headroom for the shared additions.
maxSizeRatio: 1.07,
// v1.64+v1.65 merge sums both waves' preamble growth; measured 1.073.
maxSizeRatio: 1.08,
},
};
+11 -2
View File
@@ -21,6 +21,7 @@
* tests don't need it).
*/
import { resolveEvalModel } from '../../lib/eval-model';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
@@ -459,7 +460,7 @@ ${tail}
// below resolveClaudeBinary(), breaking under hermetic PATHs.
const result = nodeSpawnSync(
resolveClaudeBinary() ?? 'claude',
['-p', '--model', 'claude-haiku-4-5', '--max-turns', '1'],
['-p', '--model', resolveEvalModel('warmup'), '--max-turns', '1'],
{
input: prompt,
stdio: ['pipe', 'pipe', 'pipe'],
@@ -1256,7 +1257,15 @@ export const ceoStep0Boundary: Step0BoundaryPredicate = (fp) =>
export const engStep0Boundary: Step0BoundaryPredicate = (fp) =>
/scope reduction recommendation|cross[\s-]?project learnings/i.test(
fp.promptSnippet,
);
) ||
// plan-eng-review's Step 0 may legitimately end with NO scope-reduction /
// learnings AUQ. When it does, the first answered review-phase question —
// tagged <gstack-qid:plan-eng-review-...> ({skill}-{slug} convention) —
// must fire the boundary, or every per-finding AUQ stays classified
// preReview and the multi-finding batching counter reads 0. Anchor allows
// the skill-name prefix; live qids observed: plan-eng-review-jitter,
// plan-eng-review-idempotency, plan-eng-review-todos-e2e-concurrent.
/gstack-qid:\s*(?:plan-)?eng-review-/i.test(fp.promptSnippet);
export const designStep0Boundary: Step0BoundaryPredicate = (fp) =>
/design system|design posture|design score|first dimension/i.test(
+14 -4
View File
@@ -214,7 +214,9 @@ const MONOLITH_INVARIANTS: ParityInvariant[] = [
// codexPreflight() block (install + auth tri-state + CODEX_MODE branch prose),
// landing ~6.3% over the v1.53.0.0 baseline. Intentional: it adds proper
// not-installed vs not-authed handling, not slop.
maxSizeRatio: 1.08,
// v1.64+v1.65 merge: both waves grew the shared preamble (evidence
// directive + telemetry failure flags); measured 1.094.
maxSizeRatio: 1.10,
minBytes: 70_000,
},
{
@@ -223,7 +225,11 @@ const MONOLITH_INVARIANTS: ParityInvariant[] = [
mustHaveHeadings: ['## Preamble', '## When to invoke'],
// v1.2.0 activation lift: the unified first-run-guidance section (P4 scaffold +
// P3 loop tip) is added to every skill's shared preamble — intentional, ~1KB.
maxSizeRatio: 1.07,
// Fork port wave 2: the shared coverage-audit detection block gained the
// multi-ecosystem markers (Django/JVM, script/target/test-file census —
// e3259078 port); measured 1.111x. v1.64+v1.65 merge sums both waves'
// preamble growth; measured 1.125.
maxSizeRatio: 1.13,
minBytes: 50_000,
},
{
@@ -237,7 +243,9 @@ const MONOLITH_INVARIANTS: ParityInvariant[] = [
// 1.09 → 1.10: the plan-mode preamble reword (scope-gate auto-select-B
// change) adds ~250 B to every skill's shared preamble; investigate was
// the closest to its ceiling (landed 1.092).
maxSizeRatio: 1.10,
// Fork port wave 2 (D1): the evidence-before-claimed-limitations preamble
// directive adds ~0.45KB to every tier-2+ skill. Measured values noted.
maxSizeRatio: 1.12, // D1 measured
minBytes: 30_000,
},
{
@@ -245,7 +253,9 @@ const MONOLITH_INVARIANTS: ParityInvariant[] = [
mustContain: ['ceo', 'eng', 'design'],
mustHaveHeadings: ['## Preamble', '## When to invoke'],
// v1.2.0 activation lift: shared first-run-guidance preamble section.
maxSizeRatio: 1.07,
// Fork port wave 2 (D1): the evidence-before-claimed-limitations preamble
// directive adds ~0.45KB to every tier-2+ skill. Measured values noted.
maxSizeRatio: 1.09, // D1 measured
minBytes: 70_000,
},
];
+22 -3
View File
@@ -1,6 +1,6 @@
import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types';
import { estimateCostUsd } from '../pricing';
import { execFileSync } from 'child_process';
import { execFileSync, spawnSync } from 'child_process';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
@@ -25,11 +25,30 @@ export class ClaudeAdapter implements ProviderAdapter {
if (!resolved) {
return { ok: false, reason: 'claude CLI not found on PATH. Install from https://claude.ai/download or npm i -g @anthropic-ai/claude-code (or set GSTACK_CLAUDE_BIN)' };
}
// Auth sniff: ~/.claude/.credentials.json OR ANTHROPIC_API_KEY
// Auth sniff: ~/.claude/.credentials.json OR ANTHROPIC_API_KEY OR (macOS)
// the Keychain entry subscription installs use instead of the creds file.
// #1890: the default macOS install stores OAuth under the generic-password
// service "Claude Code-credentials" and never writes .credentials.json,
// so the file-or-env sniff reported "No Claude auth found" while
// `claude -p` worked fine. Metadata probe only (no -w — never reads the
// secret), and any failure of `security` itself falls through to the
// not-found reason rather than throwing.
const credsPath = path.join(os.homedir(), '.claude', '.credentials.json');
const hasCreds = fs.existsSync(credsPath);
const hasKey = !!process.env.ANTHROPIC_API_KEY;
if (!hasCreds && !hasKey) {
let hasKeychain = false;
if (!hasCreds && !hasKey && process.platform === 'darwin') {
try {
const probe = spawnSync('security', ['find-generic-password', '-s', 'Claude Code-credentials'], {
stdio: 'ignore',
timeout: 5000,
});
hasKeychain = probe.status === 0;
} catch {
hasKeychain = false;
}
}
if (!hasCreds && !hasKey && !hasKeychain) {
return { ok: false, reason: 'No Claude auth found. Log in via `claude` interactive session, or export ANTHROPIC_API_KEY.' };
}
return { ok: true };