From 2d9d155dd5482ed75ff1df38c0032d9b4c95fe39 Mon Sep 17 00:00:00 2001 From: garrytan Date: Fri, 2 Oct 2026 23:40:01 +0000 Subject: [PATCH 01/12] fix(models): allowlist routing for model-pinned Claude families (W1) resolveModel lowercases input and strips one trailing [...] context marker. Opus 4.7, Opus 4.8 and Sonnet 5 keep their pinned overlay only for the bare ID, the bare ID plus an 8-digit date, and the bare ID plus -latest; any other suffix (point releases such as claude-sonnet-5-5, claude-opus-4-7-1) falls to the generic claude overlay. claude-opus-5-5 routes to claude explicitly (no new overlay, no new ALL_MODEL_NAMES entry). Fable keeps its prefix match while CLAUDE_FRONTIER_EVAL_MODEL is claude-fable-5-1, guarded by a test. Tests: one test/model-overlays.test.ts case per shape in the plan, plus the lowercase / [1m] marker / family-name cases and the Fable guard. --- scripts/models.ts | 25 ++++++++++++++++----- test/model-overlays.test.ts | 45 +++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/scripts/models.ts b/scripts/models.ts index cfa462032..f80a14cd1 100644 --- a/scripts/models.ts +++ b/scripts/models.ts @@ -34,14 +34,30 @@ export type Model = (typeof ALL_MODEL_NAMES)[number]; /** * Resolve a model argument from CLI input to a known Model family. * + * Input is trimmed and lowercased, and one trailing `[...]` context marker + * (for example `claude-sonnet-5[1m]`) is stripped before matching. + * * Precedence rules: * 1. Exact match against ALL_MODEL_NAMES → return as-is. This is the ONLY * path that selects `gpt-5.6-sol` — Sol is intentionally exact-only. + * It is also how to force a model-pinned profile: `--model opus-4-7`, + * `--model sonnet-5`, and so on. * 2. Family heuristics for common variants: * - `gpt-5.4-mini`, `gpt-5.4-turbo`, `gpt-5.4-*` → `gpt-5.4` * - `gpt-*` (anything else GPT, including other 5.6 variants) → `gpt` * - `o3`, `o4`, `o4-mini`, `o1`, `o1-mini`, `o1-pro` → `o-series` - * - `claude-*` (sonnet, opus, haiku, any version) → `claude` + * - Model-pinned Claude families (Opus 4.7, Opus 4.8, Sonnet 5) use an + * allowlist: the bare ID (`claude--`), the bare ID plus + * an 8-digit date snapshot (`-YYYYMMDD`), and the bare ID plus `-latest` + * keep the pinned profile. Any other suffix, including a point release + * with or without a date (`claude-opus-4-7-1`, `claude-sonnet-5-5`, + * `claude-sonnet-5-5-20261001`), falls to `claude`: a newer model should + * not read nudges written for an older one. + * - Fable is the exception: any `claude-fable-5-*` ID keeps `fable-5`, + * because `claude-fable-5-1` is CLAUDE_FRONTIER_EVAL_MODEL + * (lib/eval-model.ts) and moving it would shift every eval baseline. + * The exception ends when CLAUDE_FRONTIER_EVAL_MODEL changes. + * - Any other `claude-*` (including `claude-opus-5-5`) → `claude` * - `gemini-*` (2.5-pro, flash, etc.) → `gemini` * 3. Unknown input → returns null (caller decides: error, or fall back). * @@ -50,7 +66,7 @@ export type Model = (typeof ALL_MODEL_NAMES)[number]; * normalizes CLI input to a family name. */ export function resolveModel(input: string): Model | null { - const s = input.trim(); + const s = input.trim().toLowerCase().replace(/\[[^\]]*\]$/, '').trim(); if (!s) return null; // Exact match first @@ -67,10 +83,9 @@ export function resolveModel(input: string): Model | null { if (/^gpt-5\.4(-|$)/.test(s)) return 'gpt-5.4'; if (/^gpt(-|$)/.test(s)) return 'gpt'; if (/^o[0-9]+(-|$)/.test(s)) return 'o-series'; - if (/^claude-opus-4-7(-|$)/.test(s)) return 'opus-4-7'; + const pinned = /^claude-(opus-4-7|opus-4-8|sonnet-5)(-\d{8}|-latest)?$/.exec(s); + if (pinned) return pinned[1] as Model; if (/^claude-fable-5(-|$)/.test(s)) return 'fable-5'; - if (/^claude-opus-4-8(-|$)/.test(s)) return 'opus-4-8'; - if (/^claude-sonnet-5(-|$)/.test(s)) return 'sonnet-5'; if (/^claude(-|$)/.test(s)) return 'claude'; if (/^gemini(-|$)/.test(s)) return 'gemini'; diff --git a/test/model-overlays.test.ts b/test/model-overlays.test.ts index e8ec21d19..452cede5c 100644 --- a/test/model-overlays.test.ts +++ b/test/model-overlays.test.ts @@ -10,6 +10,7 @@ import type { TemplateContext } from '../scripts/resolvers/types'; import { HOST_PATHS } from '../scripts/resolvers/types'; import { generateModelOverlay } from '../scripts/resolvers/model-overlay'; import { resolveModel } from '../scripts/models'; +import { CLAUDE_FRONTIER_EVAL_MODEL } from '../lib/eval-model'; import { readOverlay } from '../scripts/resolvers/model-overlay'; import { generateCompletenessSection } from '../scripts/resolvers/preamble/generate-completeness-section'; import { generateSetupCommand } from '../scripts/resolvers/utility'; @@ -368,3 +369,47 @@ describe('Sonnet 5 overlay — family nudges', () => { }); }); }); + +describe('model-routing-claude', () => { + test.each([ + ['claude-opus-4-7', 'opus-4-7'], + ['claude-opus-4-7-20260416', 'opus-4-7'], + ['claude-opus-4-7-latest', 'opus-4-7'], + ['claude-opus-4-7-1', 'claude'], + ['claude-opus-4-7-1-20260601', 'claude'], + ['claude-opus-4-8', 'opus-4-8'], + ['claude-opus-4-8-20260701', 'opus-4-8'], + ['claude-opus-4-8-latest', 'opus-4-8'], + ['claude-opus-4-8-1', 'claude'], + ['claude-sonnet-5', 'sonnet-5'], + ['claude-sonnet-5-20260115', 'sonnet-5'], + ['claude-sonnet-5-latest', 'sonnet-5'], + ['claude-sonnet-5-5', 'claude'], + ['claude-sonnet-5-5-20261001', 'claude'], + ['claude-sonnet-5-10', 'claude'], + ['claude-sonnet-5-latest-20260101', 'claude'], + ['claude-opus-5-5', 'claude'], + ['claude-fable-5', 'fable-5'], + ['claude-fable-5-1', 'fable-5'], + ])('%s routes to %s', (input, expected) => { + expect(resolveModel(input)).toBe(expected); + }); + + test('input is lowercased and one trailing [...] marker is stripped', () => { + expect(resolveModel('CLAUDE-SONNET-5')).toBe('sonnet-5'); + expect(resolveModel('claude-sonnet-5[1m]')).toBe('sonnet-5'); + expect(resolveModel('claude-opus-5-5[1m]')).toBe('claude'); + }); + + test('family names force the pinned profile', () => { + expect(resolveModel('opus-4-7')).toBe('opus-4-7'); + expect(resolveModel('opus-4-8')).toBe('opus-4-8'); + expect(resolveModel('sonnet-5')).toBe('sonnet-5'); + }); + + test('Fable prefix exception holds only while the frontier eval model is a Fable 5 point release', () => { + if (resolveModel('claude-fable-5-99') === 'fable-5') { + expect(CLAUDE_FRONTIER_EVAL_MODEL.startsWith('claude-fable-5-')).toBe(true); + } + }); +}); From 66e7e3d811ee53f6535dc6ae7a73f57994d92812 Mon Sep 17 00:00:00 2001 From: garrytan Date: Fri, 2 Oct 2026 23:40:26 +0000 Subject: [PATCH 02/12] docs(CLAUDE.md): define when prompt bytes are a test contract (W7 prompt-byte rule) Adds the gstack-repo definition to the Testing section at normal volume: prompt bytes are a contract only when software reads them or a recorded eval shows the wording matters. Amends the test-value-bar line so 'prompt bytes' refers to that definition. scripts/resolvers/test-value.ts and docs/test-value-bar.md are unchanged. --- CLAUDE.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index db5326d4c..6c0b40bab 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -121,7 +121,18 @@ engine's own tests run everywhere. New or changed tests follow the [test value bar](docs/test-value-bar.md): each one protects behavior a real regression would break, and contract tests (SKILL.md -goldens, prompt bytes) stay. The bar's source is `scripts/resolvers/test-value.ts`. +goldens, and prompt bytes that are a contract as defined below) stay. The bar's +source is `scripts/resolvers/test-value.ts`. + +In this repo, prompt bytes are a contract only when software reads them (a +parser, hook, grader or another skill consumes the exact text) or a recorded +eval shows the wording matters. Tests on skill templates and generated SKILL.md +pin structure, step order, routing tables, machine-read markers and +safety-critical lines; check safety lines case-insensitively, on meaning rather +than capitals. Leave behavior to E2E cases and judges. Don't pin emphasis, +capitalization, issue numbers, or a sentence a behavioral check already covers, +and when a rewrite changes a pinned sentence, replace the pin with a structural +or meaning-level check instead of pinning the new sentence. Projects tune `/ship`'s coverage gate with optional CLAUDE.md `## Test Coverage` keys, all absent by default: `Minimum:`, `Target:`, `Generation cap:` (default 5), `Base control:` (`auto` or `off`), `Base control budget:` (seconds, default 90) and From 4ceb1dfd86be6537b78c032d3c2f90db04aa3228 Mon Sep 17 00:00:00 2001 From: garrytan Date: Fri, 2 Oct 2026 23:41:48 +0000 Subject: [PATCH 03/12] fix(router): route telemetry uses the echoed SESSION_ID (s01 H1) $_SESSION_ID is never set in the router's bash block since bin/gstack-skill-start took over session bookkeeping, so every /gstack route event logged an empty session id. Use the literal SESSION_ID placeholder and tell the model to substitute the echoed value, matching the completion-status telemetry block. No W7a pins on the changed line. --- SKILL.md.tmpl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/SKILL.md.tmpl b/SKILL.md.tmpl index 5dad48a33..16b0b3428 100644 --- a/SKILL.md.tmpl +++ b/SKILL.md.tmpl @@ -40,9 +40,10 @@ This is the gstack router. Its one job is to send the request to the right skill Best-effort, record which way you routed (never block on it). Set `ROUTE_OUTCOME` to `browse` (sent to /browse), `routed` (sent to another skill), or `direct` (answered -directly, no skill matched): +directly, no skill matched), and replace `SESSION_ID` with the value the skill-start +output echoed: ```bash -~/.claude/skills/gstack/bin/gstack-telemetry-log --event-type route --skill gstack --outcome ROUTE_OUTCOME --session-id "$_SESSION_ID" 2>/dev/null || true +~/.claude/skills/gstack/bin/gstack-telemetry-log --event-type route --skill gstack --outcome ROUTE_OUTCOME --session-id "SESSION_ID" 2>/dev/null || true ``` If `PROACTIVE` is `false`: do NOT proactively invoke or suggest other gstack skills during From eee6811f9fef2ac8e61338a586aeda9358027456 Mon Sep 17 00:00:00 2001 From: garrytan Date: Fri, 2 Oct 2026 23:42:04 +0000 Subject: [PATCH 04/12] docs(CLAUDE.md): correct stale test-infra facts (s01 H2, H3, H4, H5) H2: TREE_MUTATING is not empty; it still holds bootstrap-retention. H3: catalog ceiling is CATALOG_BUDGET_TOKEN_EQUIVALENTS (1,194), not 1,171. H4: detach timeouts point at package.json's --timeout values (gate is 49320s). H5: Writing Style lives in preamble/generate-writing-style.ts and the jargon list is Read at runtime, not baked at generation time. No W7a pins on the changed lines. --- CLAUDE.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 6c0b40bab..887bd24b7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -92,9 +92,10 @@ shard processes, serial within each, packed by recorded per-file durations when `scripts/free-test-durations.json` exists — refresh occasionally with `bun run test:free --record-durations`; strict-output classification per shard: a shard without bun's terminal summary line FAILS — silent truncation -cannot report green). The former trailing serial tree-mutating shard is -gone: `TREE_MUTATING` is empty (gen-skill-docs has a main() guard and -`--out-dir` renders every host, so tests render into mkdtemps — see +cannot report green). `TREE_MUTATING` lists the files that still run in their +own trailing serial shard (today only `test/bootstrap-retention.test.ts`, for +host-wide procfs visibility); gen-skill-docs tests no longer need it (main() +guard, and `--out-dir` renders every host into mkdtemps — see docs/TESTING_INTERNALS.md). Never type bare `bun test` for the suite: it walks the whole repo, loading paid eval files and missing the strict classifier. @@ -199,7 +200,8 @@ or as a reference doc, (3) only compress carefully-tuned prose as a last resort cuts to the coverage audit, review army, or voice directive have real quality cost. A second, harder ceiling guards the DISCOVERY surface: `test/catalog-budget.test.ts` -caps the aggregate frontmatter `name` + `description` across all skills at 1,171 +caps the aggregate frontmatter `name` + `description` across all skills at +`CATALOG_BUDGET_TOKEN_EQUIVALENTS` (1,194 today; each new skill ratchets it) token-equivalents (260-byte per-skill sub-cap), counted through the shared census in `test/helpers/skill-census.ts`. This one is enforced, not a warning — every host loads the full catalog every session, so growth here taxes every @@ -257,8 +259,9 @@ Rules: ## Writing style (V1) Default output from every tier-≥2 skill follows the Writing Style section in -`scripts/resolvers/preamble.ts`: jargon glossed on first use (curated list in -`scripts/jargon-list.json`, baked at gen-skill-docs time), questions framed in +`scripts/resolvers/preamble/generate-writing-style.ts`: jargon glossed on first +use (curated list in `scripts/jargon-list.json`, which the skill Reads at runtime +on the first jargon term), questions framed in outcome terms ("what breaks for your users if...") not implementation terms, short sentences, decisions close with user impact. Power users who want the tighter V0 prose set `gstack-config set explain_level terse` (binary switch, @@ -748,7 +751,8 @@ the run can also die to idle-sleep. `gstack-detach` fixes both: a fresh session (stray `claude`/`codex` grandchildren included), a per-shard `GSTACK_EVAL_DIR=/shards//` honored by the `EvalCollector` constructor, and an aggregate that separates failed vs timed-out vs - never-started shards — the detach timeouts (47340s gate / 67380s periodic; + never-started shards — the detach timeouts (the `--timeout` values on + package.json's `eval:bg:gate` / `eval:bg:periodic`; floor enforced against the live shard census by test/eval-detach-timeout-floor.test.ts) are sized against worst-case shard wall clock. `EVALS_JOBS` sets the shard From c24344af37ef571dab2b78c05b109c4953fa39b4 Mon Sep 17 00:00:00 2001 From: garrytan Date: Fri, 2 Oct 2026 23:42:12 +0000 Subject: [PATCH 05/12] fix(preamble): drop the stale jargon-list count (s01 H6) The list has 77 terms, not 80+, and the model Reads the file anyway. No W7a pins on the changed text. --- scripts/resolvers/preamble/generate-writing-style.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/resolvers/preamble/generate-writing-style.ts b/scripts/resolvers/preamble/generate-writing-style.ts index 812571ce2..629b1d79a 100644 --- a/scripts/resolvers/preamble/generate-writing-style.ts +++ b/scripts/resolvers/preamble/generate-writing-style.ts @@ -31,6 +31,6 @@ Applies to AskUserQuestion, user replies, and findings. AskUserQuestion Format i - User-turn override wins: if the current message asks for terse / no explanations / just the answer, skip this section. - Terse mode (EXPLAIN_LEVEL: terse): no glosses, no outcome-framing layer, shorter responses. -Curated jargon list lives at \`${jargonPath}\` (80+ terms). On the first jargon term you encounter this session, Read that file once; treat the \`terms\` array as the canonical list. The list is repo-owned and may grow between releases. +Curated jargon list lives at \`${jargonPath}\`. On the first jargon term you encounter this session, Read that file once; treat the \`terms\` array as the canonical list. The list is repo-owned and may grow between releases. `; } From 949200ec8fb2acd01f141cbad54b79ce654377be Mon Sep 17 00:00:00 2001 From: garrytan Date: Fri, 2 Oct 2026 23:42:27 +0000 Subject: [PATCH 06/12] fix(overlay): claude.md dedicated-tools nudge stops naming tools the host may lack (s01 M1) The nudge named Glob and Grep, which some Claude hosts do not expose. It now names the host's dedicated file tools and its search tools when it has them, keeping the prefer-dedicated-tools behavior the claude-dedicated-tools-vs-bash overlay eval measures. Updates the prose comment at test/gen-skill-docs.test.ts:643 (comment only, no assertion change). The matching comment in test/fixtures/overlay-nudges.ts is left to its owner. --- model-overlays/claude.md | 5 +++-- test/gen-skill-docs.test.ts | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/model-overlays/claude.md b/model-overlays/claude.md index 95943af5b..289b781e7 100644 --- a/model-overlays/claude.md +++ b/model-overlays/claude.md @@ -6,5 +6,6 @@ turns out to be unnecessary, mark it skipped with a one-line reason. non-trivial new features), briefly state your approach before executing. This lets the user course-correct cheaply instead of mid-flight. -**Dedicated tools over Bash.** Prefer Read, Edit, Write, Glob, Grep over shell -equivalents (cat, sed, find, grep). The dedicated tools are cheaper and clearer. +**Dedicated tools over Bash.** Prefer the host's dedicated file tools (Read, Edit, +Write, and its search tools when it has them) over shell equivalents (cat, sed, +find, grep). The dedicated tools are cheaper and clearer. diff --git a/test/gen-skill-docs.test.ts b/test/gen-skill-docs.test.ts index 533697657..709461d2a 100644 --- a/test/gen-skill-docs.test.ts +++ b/test/gen-skill-docs.test.ts @@ -640,7 +640,7 @@ describe('gen-skill-docs', () => { // Should not have Edit, Glob, or Grep in allowed-tools. // Scope to frontmatter (between the first two --- lines) — the body can // legitimately mention these tool names in prose (e.g., Claude model - // overlay says "prefer Read, Edit, Write, Glob, Grep over Bash"). + // overlay names the host's dedicated file and search tools). const fmMatch = qaOnlyContent.match(/^---\n([\s\S]*?)\n---/); expect(fmMatch).not.toBeNull(); const frontmatter = fmMatch![1]; From d4fba5beb43dc1cd8c4376ce50f455ec2e2bfcbd Mon Sep 17 00:00:00 2001 From: garrytan Date: Fri, 2 Oct 2026 23:42:38 +0000 Subject: [PATCH 07/12] fix(preamble): progress notes on phase changes, not a [PROGRESS] cadence (s01 M2) Context Health told the model to periodically write a tagged [PROGRESS] summary. No code reads that tag; the line now says when to tell the user (finishing a phase or changing direction). Loop detection stays. W7a pin: test/terse-build.test.ts:103 pinned 'PROGRESS'; it now checks the default build emits a non-empty Context Health body (terse still emits ''). --- scripts/resolvers/preamble/generate-context-health.ts | 2 +- test/terse-build.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/resolvers/preamble/generate-context-health.ts b/scripts/resolvers/preamble/generate-context-health.ts index 88b058406..f0e1d402e 100644 --- a/scripts/resolvers/preamble/generate-context-health.ts +++ b/scripts/resolvers/preamble/generate-context-health.ts @@ -4,7 +4,7 @@ export function generateContextHealth(ctx?: TemplateContext): string { if (ctx?.explainLevel === 'terse') return ''; return `## Context Health (soft directive) -During long-running skill sessions, periodically write a brief \`[PROGRESS]\` summary: done, next, surprises. +During long-running skill sessions, when you finish a phase or change direction, tell the user in a sentence or two what is done, what is next, and anything surprising. If you are looping on the same diagnostic, same file, or failed fix variants, STOP and reassess. Consider escalation or /context-save. Progress summaries must NEVER mutate git state.`; } diff --git a/test/terse-build.test.ts b/test/terse-build.test.ts index 58ed10396..ea5c69529 100644 --- a/test/terse-build.test.ts +++ b/test/terse-build.test.ts @@ -100,7 +100,7 @@ describe('terse build — per-resolver behavior', () => { test('default: emits full section', () => { const out = generateContextHealth(makeCtx('default')); expect(out).toContain('## Context Health'); - expect(out).toContain('PROGRESS'); + expect(out.replace(/^## Context Health.*$/m, '').trim().length).toBeGreaterThan(80); }); test('terse: returns empty string', () => { From 6709bad3c98160c1daad2f183c0a6aa9fe531573 Mon Sep 17 00:00:00 2001 From: garrytan Date: Fri, 2 Oct 2026 23:42:49 +0000 Subject: [PATCH 08/12] fix(preamble): learnings review without incident archaeology (s01 M3) Drops the issue number, the 43-of-44 statistic and the ALWAYS caps from the Operational Self-Improvement step; keeps the rule that the review runs every time and that an empty result is stated explicitly. No W7a pins on the removed text. --- .../preamble/generate-completion-status.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/scripts/resolvers/preamble/generate-completion-status.ts b/scripts/resolvers/preamble/generate-completion-status.ts index 422ce2968..f07009020 100644 --- a/scripts/resolvers/preamble/generate-completion-status.ts +++ b/scripts/resolvers/preamble/generate-completion-status.ts @@ -48,13 +48,12 @@ Escalate after 3 failed attempts, uncertain security-sensitive changes, or scope ## Operational Self-Improvement -Before completing, review the session for durable learnings and log each one — -this step ALWAYS runs, it is not conditional on something feeling noteworthy -(#2402: 43 of 44 learnings came from explicit /learn because "if you -discovered" read as optional). A durable learning is a project quirk, command -fix, pitfall, or pattern that would save 5+ minutes in a future session. If -the review genuinely surfaces none, state "No durable learnings this session" -in your completion summary — an explicit empty result, not a skipped step. +Before completing, review the session for durable learnings and log each one. +The review runs every time, not only when something felt noteworthy. A durable +learning is a project quirk, command fix, pitfall, or pattern that would save +5+ minutes in a future session. If the review genuinely surfaces none, state +"No durable learnings this session" in your completion summary — an explicit +empty result, not a skipped step. \`\`\`bash ${ctx.paths.binDir}/gstack-learnings-log '{"skill":"SKILL_NAME","type":"operational","key":"SHORT_KEY","insight":"DESCRIPTION","confidence":N,"source":"observed"}' From 7a558a1dbc046097319cc1d7d2d3c7daadaf743a Mon Sep 17 00:00:00 2001 From: garrytan Date: Fri, 2 Oct 2026 23:43:04 +0000 Subject: [PATCH 09/12] docs(CLAUDE.md): fold 'Long-running tasks' into the detach section (s01 M4, CEO W4) The 'don't give up' section told agents to poll a plain background task with sleep 180 + TaskOutput, which the next section forbids for evals. Its one durable rule (keep checking until the run finishes, report progress at each check) now lives in the detach section's poll step, pointed at the gstack-detach EXIT sentinel, at normal volume. No tests pin the removed text. --- CLAUDE.md | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 887bd24b7..e196b6a1a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -716,19 +716,6 @@ regenerated SKILL.md shifts prompt context. "Pre-existing" without receipts is a lazy claim. Prove it or don't say it. -## Long-running tasks: don't give up - -When running evals, E2E tests, or any long-running background task, **poll until -completion**. Use `sleep 180 && echo "ready"` + `TaskOutput` in a loop every 3 -minutes. Never switch to blocking mode and give up when the poll times out. Never -say "I'll be notified when it completes" and stop checking — keep the loop going -until the task finishes or the user tells you to stop. - -The full E2E suite can take 30-45 minutes. That's 10-15 polling cycles. Do all of -them. Report progress at each check (which tests passed, which are running, any -failures so far). The user wants to see the run complete, not a promise that -you'll check later. - ## Running evals as an agent: always detach (SIGTERM-proof) When **you (an agent/harness)** launch a long eval/benchmark run, run it through @@ -765,7 +752,10 @@ the run can also die to idle-sleep. `gstack-detach` fixes both: a fresh session - Then **poll the printed logfile** with a death-aware watcher: break on the guaranteed `### gstack-detach EXIT= ###` sentinel (success AND failure are both marked, so silence is never mistaken for success). The detached run survives - even if your watcher gets reaped, so re-checking the log always works. + even if your watcher gets reaped, so re-checking the log always works. Keep + checking until the sentinel appears or the user tells you to stop; a long run is + expected, and a promise to check later is not a result. At each check, report + which tests passed, which are still running, and any failures so far. - Why the lock: a shared dev box with several Conductor worktrees will rate-limit the model API if two eval suites run at once (15-way concurrency each), which mass-times-out E2E tests. The lock makes the second run WAIT, not collide. From c9bd3d69a815ef3f29e77470a697911ad7014574 Mon Sep 17 00:00:00 2001 From: garrytan Date: Fri, 2 Oct 2026 23:43:23 +0000 Subject: [PATCH 10/12] fix(preamble): test-failure triage questions defer to the AskUserQuestion Format (s01 M5) The two quoted pseudo-briefs pinned a superseded question shape (RECOMMENDATION: Choose X, a completeness score on the recommendation, no D/ELI10/Net). The triage step now gives the facts, the options with their completeness values, and the recommended option as data, and defers the shape to the Format section. Option labels that Step T4 and test/skill-validation.test.ts:2004-2016 depend on are verbatim. No W7a pins on the removed text. --- .../preamble/generate-test-failure-triage.ts | 34 +++++-------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/scripts/resolvers/preamble/generate-test-failure-triage.ts b/scripts/resolvers/preamble/generate-test-failure-triage.ts index 1b680ec10..1a1262b2c 100644 --- a/scripts/resolvers/preamble/generate-test-failure-triage.ts +++ b/scripts/resolvers/preamble/generate-test-failure-triage.ts @@ -31,34 +31,18 @@ Check \`REPO_MODE\` from the preamble output. **If REPO_MODE is \`solo\`:** -Use AskUserQuestion: - -> These test failures appear pre-existing (not caused by your branch changes): -> -> [list each failure with file:line and brief error description] -> -> Since this is a solo repo, you're the only one who will fix these. -> -> RECOMMENDATION: Choose A — fix now while the context is fresh. Completeness: 9/10. -> A) Investigate and fix now (human: ~2-4h / CC: ~15min) — Completeness: 10/10 -> B) Add as P0 TODO — fix after this branch lands — Completeness: 7/10 -> C) Skip — I know about this, ship anyway — Completeness: 3/10 +Ask with AskUserQuestion in the AskUserQuestion Format. List each failure with file:line and a brief error description, say the failures appear pre-existing (not caused by this branch), and say that in a solo repo nobody else will fix them. Options, with A recommended because the context is fresh: +- A) Investigate and fix now (human: ~2-4h / CC: ~15min) — Completeness 10/10 +- B) Add as P0 TODO — fix after this branch lands — Completeness 7/10 +- C) Skip — I know about this, ship anyway — Completeness 3/10 **If REPO_MODE is \`collaborative\` or \`unknown\`:** -Use AskUserQuestion: - -> These test failures appear pre-existing (not caused by your branch changes): -> -> [list each failure with file:line and brief error description] -> -> This is a collaborative repo — these may be someone else's responsibility. -> -> RECOMMENDATION: Choose B — assign it to whoever broke it so the right person fixes it. Completeness: 9/10. -> A) Investigate and fix now anyway — Completeness: 10/10 -> B) Blame + assign GitHub issue to the author — Completeness: 9/10 -> C) Add as P0 TODO — Completeness: 7/10 -> D) Skip — ship anyway — Completeness: 3/10 +Ask with AskUserQuestion in the AskUserQuestion Format. List each failure the same way, and say that in a collaborative repo these may be someone else's responsibility. Options, with B recommended so the person who broke it fixes it: +- A) Investigate and fix now anyway — Completeness 10/10 +- B) Blame + assign GitHub issue to the author — Completeness 9/10 +- C) Add as P0 TODO — Completeness 7/10 +- D) Skip — ship anyway — Completeness 3/10 ### Step T4: Execute the chosen action From b00a0f19b0ff1ce3f9a52ef12030103b7022350a Mon Sep 17 00:00:00 2001 From: garrytan Date: Fri, 2 Oct 2026 23:43:51 +0000 Subject: [PATCH 11/12] fix(proactive): PROACTIVE=false means no suggestions, including 'want me to run /X?' (W5.1) The router forbade suggesting skills when PROACTIVE is false while the preamble told the model to ask 'want me to run it?'. Decision: false means never auto-invoke and never suggest, including by asking. The preamble line in generate-upgrade-check.ts now matches the router, and both quote the value as `false`, the way gstack-skill-start prints it. Restore with gstack-config set proactive true. W7a pin: test/gen-skill-docs.test.ts:4488 used the old quoted sentence as a preamble-order marker; it now finds the PROACTIVE-false rule with a quote-tolerant, case-insensitive pattern. --- SKILL.md.tmpl | 7 ++++--- scripts/resolvers/preamble/generate-upgrade-check.ts | 2 +- test/gen-skill-docs.test.ts | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/SKILL.md.tmpl b/SKILL.md.tmpl index 16b0b3428..e0c33a2da 100644 --- a/SKILL.md.tmpl +++ b/SKILL.md.tmpl @@ -46,9 +46,10 @@ output echoed: ~/.claude/skills/gstack/bin/gstack-telemetry-log --event-type route --skill gstack --outcome ROUTE_OUTCOME --session-id "SESSION_ID" 2>/dev/null || true ``` -If `PROACTIVE` is `false`: do NOT proactively invoke or suggest other gstack skills during -this session. Only run skills the user explicitly invokes. This preference persists across -sessions via `gstack-config`. +If `PROACTIVE` is `false`: do not proactively invoke or suggest other gstack skills during +this session, including by asking whether to run one ("want me to run /X?"). Only run +skills the user explicitly invokes. This preference persists across sessions via +`gstack-config`. If `PROACTIVE` is `true` (default): **invoke the Skill tool** when the user's request matches a skill's purpose. Do NOT answer directly when a skill exists for the task. diff --git a/scripts/resolvers/preamble/generate-upgrade-check.ts b/scripts/resolvers/preamble/generate-upgrade-check.ts index ff226f140..bf77ddb7b 100644 --- a/scripts/resolvers/preamble/generate-upgrade-check.ts +++ b/scripts/resolvers/preamble/generate-upgrade-check.ts @@ -8,7 +8,7 @@ import type { TemplateContext } from '../types'; * the fence prose scopes and the model follows. */ export function generateUpgradeCheck(ctx: TemplateContext): string { - return `If \`PROACTIVE\` is \`"false"\`, do not auto-invoke or proactively suggest skills. If a skill seems useful, ask: "I think /skillname might help here — want me to run it?" + return `If \`PROACTIVE\` is \`false\`, do not auto-invoke or suggest skills, including by asking whether to run one. Only run skills the user explicitly invokes. If \`SKILL_PREFIX\` is \`"true"\`, suggest/invoke \`/gstack-*\` names. Disk paths stay \`${ctx.paths.skillRoot}/[skill-name]/SKILL.md\`.`; } diff --git a/test/gen-skill-docs.test.ts b/test/gen-skill-docs.test.ts index 709461d2a..0d12ac2f4 100644 --- a/test/gen-skill-docs.test.ts +++ b/test/gen-skill-docs.test.ts @@ -4485,7 +4485,7 @@ describe('plan-mode-info resolver (handshake-replacement)', () => { 'utf-8', ); const planModeIdx = content.indexOf(PLAN_MODE_INFO_MARKER); - const upgradeIdx = content.indexOf('If `PROACTIVE` is `"false"`'); + const upgradeIdx = content.search(/If `PROACTIVE` is `"?false"?`/i); expect(planModeIdx).toBeGreaterThan(0); expect(upgradeIdx).toBeGreaterThan(0); expect(planModeIdx).toBeLessThan(upgradeIdx); From e9d1b162c9621de425b47abbb42154db54c97f1d Mon Sep 17 00:00:00 2001 From: garrytan Date: Fri, 2 Oct 2026 23:44:08 +0000 Subject: [PATCH 12/12] docs(ETHOS.md): correct the claim that ETHOS is injected into every preamble (W5.7) Nothing injects ETHOS.md. Preambles carry condensed principles (completeness, voice, search before building) and the Search Before Building section links to this file. Edit approved by Garry for this PR. --- ETHOS.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ETHOS.md b/ETHOS.md index 3dbd5e570..a0fd1b845 100644 --- a/ETHOS.md +++ b/ETHOS.md @@ -1,8 +1,9 @@ # gstack Builder Ethos These are the principles that shape how gstack thinks, recommends, and builds. -They are injected into every workflow skill's preamble automatically. They -reflect what we believe about building software in 2026. +Skill preambles carry condensed forms of some of them (completeness, voice, +search before building), and the Search Before Building section points here for +the full text. They reflect what we believe about building software in 2026. ---