From e8c8625bd65fdcc0f0650f982830649d326c4bf4 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Tue, 17 Nov 2026 20:51:54 +0800 Subject: [PATCH] Keep checking custom skill contracts that still list the removed test-after-edit item (GST-2811) (#4439) * Keep checking a custom skill's contract when it still lists the removed test-after-edit item The Stop hook used to treat such a contract as invalid and check none of its items. It now skips only that item, warns with a line naming the file and item, and checks the rest. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Move the changelog note to its own entry, since a landed entry is never edited Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix: pre-landing review fixes Test that gstack-skill-contract check warns about a removed-kind item and checks the rest. Review-Finding: p1/specialist-batched#1 reader=specialist-batched Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * chore: changelog and TODOS Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- bin/gstack-skill-contract | 4 +++- ...keep-custom-contracts-with-removed-item.md | 1 + docs/skill-routing-and-contracts.md | 2 +- .../claude/hooks/skill-contract-stop-hook.ts | 8 ++++++-- lib/skill-contract.ts | 17 ++++++++++++++-- test/skill-contract-stop-hook.test.ts | 13 +++++++++++- test/skill-contract.test.ts | 20 ++++++++++++++++++- 7 files changed, 57 insertions(+), 8 deletions(-) create mode 100644 changelog.d/keep-custom-contracts-with-removed-item.md diff --git a/bin/gstack-skill-contract b/bin/gstack-skill-contract index 27ba75adf..000c5ea45 100755 --- a/bin/gstack-skill-contract +++ b/bin/gstack-skill-contract @@ -19,7 +19,7 @@ */ import { EXIT, StateError, UnknownShape, blockMessage, checkContract, isMiss, lookupContract, parseTranscript, readAllRecords, - readTranscriptFile, reportRows, type CheckResult, + readTranscriptFile, reportRows, skippedLine, type CheckResult, } from '../lib/skill-contract'; import { fileURLToPath } from 'node:url'; import { installedSkillsDir } from '../lib/skill-route'; @@ -69,6 +69,8 @@ function check(args: string[]): never { if (!transcriptPath || !skill) die(EXIT.USAGE, `SKILL_CONTRACT_USAGE: check needs --transcript and --skill\n${USAGE}`); const found = lookupContract(skill, installedSkillsDir(), flag(args, '--skill-file')); if (!('contract' in found)) die(EXIT.NO_CONTRACT, `SKILL_CONTRACT_NO_CONTRACT ${skill}: ${found.error}`); + for (const why of found.contract.skipped ?? []) process.stderr.write(`${skippedLine(found.file, why)} +`); let result: CheckResult; try { result = checkContract(parseTranscript(readTranscriptFile(transcriptPath)), skill, found.contract); diff --git a/changelog.d/keep-custom-contracts-with-removed-item.md b/changelog.d/keep-custom-contracts-with-removed-item.md new file mode 100644 index 000000000..a8789f001 --- /dev/null +++ b/changelog.d/keep-custom-contracts-with-removed-item.md @@ -0,0 +1 @@ +- **A skill you wrote whose contract still lists `check_after_last_edit` keeps the rest of its contract.** This corrects the entry in this release that says such a contract is now invalid and none of its items are checked. The optional Stop hook now skips only that item and keeps checking the skill's other items. Until you delete the item, the hook prints a `SKILL_CONTRACT_SKIPPED` line naming the file and the item. A contract whose only items use the removed kind is still invalid, so the hook checks nothing for that skill. diff --git a/docs/skill-routing-and-contracts.md b/docs/skill-routing-and-contracts.md index c85997cc9..48139e563 100644 --- a/docs/skill-routing-and-contracts.md +++ b/docs/skill-routing-and-contracts.md @@ -76,7 +76,7 @@ Every item has an `id` (lowercase letters, digits, `.`, `_`, `-`) and a `kind`, | `selected_specialists_dispatched` | optional `marker`, `empty_marker` | The selection line was printed in an assistant message or a Bash result (never a fetched page or file), its names are plain (letters, digits, spaces, `_`, `.`, `-`, at most 40 characters), and each specialist it names has an Agent dispatch that mentions it. The marker is literal text in which a whole-word N stands for a number; the default is `Dispatching N specialists:`. N = 0 (`Dispatching 0 specialists: (small diff)`) or a "specialists skipped" line means zero specialists were selected, which passes. Text in parentheses is not a name (`Dispatching 1 specialists: State Machines (small diff)` selects State Machines) | | `agent_dispatch` | `match` (regular expression, case-insensitive) | An Agent dispatch's description or prompt matches `match`. A `marker` is rejected until the checker can bind it to the dispatch it names (GST-1242) | -There is no item for "run a test after your last edit". /verify asks for that in its text, and Opus 5.5 and GPT-6.1 Sol already did it in 96% and 89% of runs; a Stop-hook item for it added no measurable gain (gstack-evals#356), so it was removed. +There is no item for "run a test after your last edit". /verify asks for that in its text, and Opus 5.5 and GPT-6.1 Sol already did it in 96% and 89% of runs; a Stop-hook item for it added no measurable gain (gstack-evals#356), so it was removed. A contract that still lists `check_after_last_edit` keeps its other items. The checker skips that item and prints `SKILL_CONTRACT_SKIPPED : : ...` on stderr each time it reads the contract (the Stop hook and `check`). A contract whose only items use that kind is invalid. A run the skill ended early counts as met: a final `STATUS: BLOCKED` (or `Status: BLOCKED` in the skill's report) or a final message with a line that starts "Nothing to review" (/review's early exit). diff --git a/hosts/claude/hooks/skill-contract-stop-hook.ts b/hosts/claude/hooks/skill-contract-stop-hook.ts index 709a4f8e0..80aba4574 100755 --- a/hosts/claude/hooks/skill-contract-stop-hook.ts +++ b/hosts/claude/hooks/skill-contract-stop-hook.ts @@ -33,7 +33,7 @@ import { binPath, sessionKind } from './spawn-bin'; import { appendHookTrace, installedSkillsDir, readSkillSwitch, readStdinBounded } from '../../../lib/skill-route'; import { - AGENT_STALL_MS, StateError, UnknownShape, blockMessage, bumpCanary, checkContract, isFinalTurn, lookupContract, nextOutcome, + AGENT_STALL_MS, StateError, UnknownShape, blockMessage, bumpCanary, checkContract, isFinalTurn, lookupContract, nextOutcome, skippedLine, readRecord, recordPath, runKey, sameSkill, skillLoads, stateErrorLine, writeRecord, type Contract, type Transcript, } from '../../../lib/skill-contract'; @@ -67,7 +67,11 @@ function target(t: Transcript): { skill: string; contract: Contract } | null { if (tried.some(s => sameSkill(s, load.skill))) continue; tried.push(load.skill); const found = lookupContract(load.skill, dir); - if ('contract' in found) return { skill: load.skill, contract: found.contract }; + if ('contract' in found) { + for (const why of found.contract.skipped ?? []) process.stderr.write(`${skippedLine(found.file, why)} +`); + return { skill: load.skill, contract: found.contract }; + } if (found.reason === 'invalid' && found.file) throw new StateError(found.file, found.error); } return null; diff --git a/lib/skill-contract.ts b/lib/skill-contract.ts index cdcab0fe2..a6996e2db 100644 --- a/lib/skill-contract.ts +++ b/lib/skill-contract.ts @@ -24,6 +24,8 @@ import { resolveStateRoot } from './state-root'; export const TRANSCRIPT_MAX_BYTES = 32 * 1024 * 1024; export const ITEM_KINDS = ['report_labels', 'selected_specialists_dispatched', 'agent_dispatch'] as const; export type ItemKind = typeof ITEM_KINDS[number]; +/** Kinds a contract may still list from an older gstack: each such item is skipped with a warning, not checked. */ +const REMOVED_KINDS = ['check_after_last_edit']; const ITEM_ID = /^[a-z0-9][a-z0-9_.-]{0,63}$/; const MAX_ITEMS = 20; @@ -42,13 +44,18 @@ export interface ContractItem { fix?: string; summary?: string; } -export interface Contract { version: 1; items: ContractItem[] } +/** skipped: one warning per item whose kind was removed from gstack; the item is not checked. */ +export interface Contract { version: 1; items: ContractItem[]; skipped?: string[] } export class StateError extends Error { constructor(readonly file: string, readonly why: string) { super(`${file}: ${why}`); } } export class UnknownShape extends Error {} +export function skippedLine(file: string, why: string): string { + return `gstack-skill-contract: SKILL_CONTRACT_SKIPPED ${file}: ${why}`; +} + export function stateErrorLine(file: string, why: string): string { return `gstack-skill-contract: SKILL_CONTRACT_STATE_ERROR ${file}: ${why}; stop allowed, miss not recorded (see ${STATE_ERROR_DOC})`; } @@ -112,11 +119,16 @@ export function validateContract(raw: unknown): { contract: Contract } | { error return { error: `contract items must be a list of 1 to ${MAX_ITEMS} items` }; } const items: ContractItem[] = []; + const skipped: string[] = []; for (const [n, item] of raw.items.entries()) { if (!isObject(item)) return { error: `items[${n}] must be a mapping` }; const id = item.id; if (typeof id !== 'string' || !ITEM_ID.test(id)) return { error: `items[${n}].id must match ${ITEM_ID}` }; if (items.some(i => i.id === id)) return { error: `duplicate item id ${id}` }; + if (REMOVED_KINDS.includes(item.kind)) { + skipped.push(`${id}: kind ${item.kind} was removed from gstack, so this item is not checked; delete it from the contract`); + continue; + } if (!(ITEM_KINDS as readonly string[]).includes(item.kind)) { return { error: `${id}: unknown kind ${JSON.stringify(item.kind)} (expected one of ${ITEM_KINDS.join(', ')})` }; } @@ -141,7 +153,8 @@ export function validateContract(raw: unknown): { contract: Contract } | { error if (out.kind === 'agent_dispatch' && out.marker) return { error: `${id}: agent_dispatch takes no marker until the checker can bind a marker to the dispatch it names (GST-1242)` }; items.push(out); } - return { contract: { version: 1, items } }; + if (!items.length) return { error: `every item uses a removed kind (${skipped.join('; ')})` }; + return { contract: skipped.length ? { version: 1, items, skipped } : { version: 1, items } }; } /** One plain phrase per item, for the routing hint's "Done means" variant. */ diff --git a/test/skill-contract-stop-hook.test.ts b/test/skill-contract-stop-hook.test.ts index adffa02a7..22e2d041d 100644 --- a/test/skill-contract-stop-hook.test.ts +++ b/test/skill-contract-stop-hook.test.ts @@ -11,10 +11,11 @@ import { bunIsolationArgs } from '../lib/bun-isolation'; import { AGENT_STALL_MS, STATE_ERROR_DOC, stateErrorLine } from '../lib/skill-contract'; import { SWITCH_DEFAULT } from '../lib/skill-route'; import { - CONTRACT_HOOK, DEBUG_REPORT, INTERACTIVE_TURNS, J1_TRANSCRIPT, ROOT, hookEnv, meta, result, runHook, say, skillsDir, slash, tmpDir, tool, user, + CONTRACT_HOOK, DEBUG_REPORT, INTERACTIVE_TURNS, J1_TRANSCRIPT, ROOT, SKILLS, hookEnv, meta, result, runHook, say, skillsDir, slash, tmpDir, tool, user, } from './helpers/skill-contract-fixture'; const BIN = path.join(ROOT, 'bin', 'gstack-skill-contract'); +const SKILLS_LEGACY = SKILLS.verify.replace(' labels: [Symptom, Root cause, Fix, Evidence]\n', ' labels: [Symptom, Root cause, Fix, Evidence]\n - id: repro-after-fix\n kind: check_after_last_edit\n'); const PINNED = fs.readFileSync(path.join(ROOT, 'test', 'fixtures', 'skill-contract', 'block-message.txt'), 'utf8').trimEnd().replace('{bin}', BIN); const STATE_ERROR = /^gstack-skill-contract: SKILL_CONTRACT_STATE_ERROR .+: .+; stop allowed, miss not recorded \(see docs\/skill-routing-and-contracts\.md#state-error\)\n$/; @@ -149,6 +150,16 @@ describe('block once, with the pinned text', () => { expect((Object.values(record().runs)[0] as any).outcome).toBe('met'); }); + test('a custom skill whose contract still lists the removed check_after_last_edit kind: warned once, the rest still blocks', () => { + const legacy = SKILLS_LEGACY.replace('name: verify', 'name: legacy'); + skills = skillsDir([], { legacy }); + const r = stop(transcript(J1_TRANSCRIPT.map(l => l.replace('"skill":"verify"', '"skill":"legacy"')))); + expect(JSON.parse(r.stdout).decision).toBe('block'); + expect(r.stderr).toBe(`gstack-skill-contract: SKILL_CONTRACT_SKIPPED ${path.join(skills, 'legacy', 'SKILL.md')}: repro-after-fix: kind check_after_last_edit was removed from gstack, so this item is not checked; delete it from the contract +`); + expect(Object.values(record().runs)[0]).toMatchObject({ skill: 'legacy', outcome: 'blocked', missing: ['debug-report'] }); + }); + test('a waiver read from the transcript satisfies the block but counts against follow', () => { const lines = [...J1_TRANSCRIPT.slice(0, -1), tool('w1', 'Bash', { command: 'gstack-skill-contract waive debug-report --reason "a docs-only change, no bug to report"' }), diff --git a/test/skill-contract.test.ts b/test/skill-contract.test.ts index 7090e5902..b87e6f8be 100644 --- a/test/skill-contract.test.ts +++ b/test/skill-contract.test.ts @@ -43,7 +43,7 @@ describe('schema v1', () => { ['contract:\n version: 1\n items: [{id: a, kind: agent_dispatch}]', 'agent_dispatch needs match'], ['contract:\n version: 1\n items: [{id: a, kind: report_labels, heading: X, labels: []}]', 'needs labels'], ['contract:\n version: 1\n items: [{id: Bad_ID, kind: agent_dispatch, match: x}]', 'id must match'], - ['contract:\n version: 1\n items: [{id: a, kind: check_after_last_edit}]', 'unknown kind'], + ['contract:\n version: 1\n items: [{id: a, kind: check_after_last_edit}]', 'every item uses a removed kind'], ['contract: [unclosed', 'not valid YAML'], ]; for (const [fm, msg] of bad) { @@ -53,6 +53,14 @@ describe('schema v1', () => { }); }); +test('a contract that still lists the removed check_after_last_edit kind keeps its other items and names the skipped one', () => { + const parsed = parseContract('contract:\n version: 1\n items:\n - {id: debug-report, kind: report_labels, heading: DEBUG REPORT, labels: [Symptom]}\n - {id: repro-after-fix, kind: check_after_last_edit}'); + if (!parsed || !('contract' in parsed)) throw new Error(JSON.stringify(parsed)); + expect(parsed.contract.items.map(i => i.id)).toEqual(['debug-report']); + expect(parsed.contract.skipped).toEqual(['repro-after-fix: kind check_after_last_edit was removed from gstack, so this item is not checked; delete it from the contract']); + expect(parseContract('contract:\n version: 1\n items: [{id: a, kind: agent_dispatch, match: x}]')).toEqual({ contract: { version: 1, items: [{ id: 'a', kind: 'agent_dispatch', match: 'x' }] } }); +}); + describe('transcript parsing', () => { test('skill loads: Skill tool, typed /command, Read of a SKILL.md; sidechain records ignored', () => { const t = parseTranscript([ @@ -243,6 +251,16 @@ describe('gstack-skill-contract CLI', () => { expect(JSON.parse(unmet.stdout)).toMatchObject({ exit: 1, skill: 'verify', missing: ['debug-report'], follow: false }); }); + test('check: a contract that still lists the removed kind warns once on stderr and checks the rest', () => { + const file = path.join(work, 'SKILL.md'); + fs.writeFileSync(file, SKILLS.verify.replace(' labels: [Symptom, Root cause, Fix, Evidence]\n', ' labels: [Symptom, Root cause, Fix, Evidence]\n - id: repro-after-fix\n kind: check_after_last_edit\n')); + const r = cli('check', '--transcript', write(J1_TRANSCRIPT), '--skill', 'verify', '--skill-file', file, '--json'); + expect(r.code).toBe(EXIT.UNMET); + expect(JSON.parse(r.stdout)).toMatchObject({ missing: ['debug-report'] }); + expect(r.stderr).toBe(`gstack-skill-contract: SKILL_CONTRACT_SKIPPED ${file}: repro-after-fix: kind check_after_last_edit was removed from gstack, so this item is not checked; delete it from the contract +`); + }); + test('check: stable error exits 2, 3, 4, 5, 6', () => { expect(cli('check', '--skill', 'verify').code).toBe(EXIT.USAGE); expect(cli('check', '--transcript', path.join(work, 'none.jsonl'), '--skill', 'verify')).toMatchObject({ code: EXIT.STATE_ERROR });