Keep checking custom skill contracts that still list the removed test-after-edit item (GST-2811) (#4439)

* Keep checking a custom skill's contract when it still lists the removed test-after-edit item

The Stop hook used to treat such a contract as invalid and check none of
its items. It now skips only that item, warns with a line naming the file
and item, and checks the rest.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* Move the changelog note to its own entry, since a landed entry is never edited

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* fix: pre-landing review fixes

Test that gstack-skill-contract check warns about a removed-kind item and checks the rest.

Review-Finding: p1/specialist-batched#1 reader=specialist-batched
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

* chore: changelog and TODOS

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>

---------

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
Garry Tanandcapy-ai[bot] authored and GitHub committed 2026-11-17 12:51:54 +00:00
1 parent ab114c5308
commit e8c8625bd6
7 files changed
+57 -8

No files matched your search

+3 -1
View File
@@ -19,7 +19,7 @@
*/
import {
EXIT, StateError, UnknownShape, blockMessage, checkContract, isMiss, lookupContract, parseTranscript, readAllRecords,
readTranscriptFile, reportRows, type CheckResult,
readTranscriptFile, reportRows, skippedLine, type CheckResult,
} from '../lib/skill-contract';
import { fileURLToPath } from 'node:url';
import { installedSkillsDir } from '../lib/skill-route';
@@ -69,6 +69,8 @@ function check(args: string[]): never {
if (!transcriptPath || !skill) die(EXIT.USAGE, `SKILL_CONTRACT_USAGE: check needs --transcript and --skill\n${USAGE}`);
const found = lookupContract(skill, installedSkillsDir(), flag(args, '--skill-file'));
if (!('contract' in found)) die(EXIT.NO_CONTRACT, `SKILL_CONTRACT_NO_CONTRACT ${skill}: ${found.error}`);
for (const why of found.contract.skipped ?? []) process.stderr.write(`${skippedLine(found.file, why)}
`);
let result: CheckResult;
try {
result = checkContract(parseTranscript(readTranscriptFile(transcriptPath)), skill, found.contract);
@@ -0,0 +1 @@
- **A skill you wrote whose contract still lists `check_after_last_edit` keeps the rest of its contract.** This corrects the entry in this release that says such a contract is now invalid and none of its items are checked. The optional Stop hook now skips only that item and keeps checking the skill's other items. Until you delete the item, the hook prints a `SKILL_CONTRACT_SKIPPED` line naming the file and the item. A contract whose only items use the removed kind is still invalid, so the hook checks nothing for that skill.
+1 -1
View File
@@ -76,7 +76,7 @@ Every item has an `id` (lowercase letters, digits, `.`, `_`, `-`) and a `kind`,
| `selected_specialists_dispatched` | optional `marker`, `empty_marker` | The selection line was printed in an assistant message or a Bash result (never a fetched page or file), its names are plain (letters, digits, spaces, `_`, `.`, `-`, at most 40 characters), and each specialist it names has an Agent dispatch that mentions it. The marker is literal text in which a whole-word N stands for a number; the default is `Dispatching N specialists:`. N = 0 (`Dispatching 0 specialists: (small diff)`) or a "specialists skipped" line means zero specialists were selected, which passes. Text in parentheses is not a name (`Dispatching 1 specialists: State Machines (small diff)` selects State Machines) |
| `agent_dispatch` | `match` (regular expression, case-insensitive) | An Agent dispatch's description or prompt matches `match`. A `marker` is rejected until the checker can bind it to the dispatch it names (GST-1242) |
There is no item for "run a test after your last edit". /verify asks for that in its text, and Opus 5.5 and GPT-6.1 Sol already did it in 96% and 89% of runs; a Stop-hook item for it added no measurable gain (gstack-evals#356), so it was removed.
There is no item for "run a test after your last edit". /verify asks for that in its text, and Opus 5.5 and GPT-6.1 Sol already did it in 96% and 89% of runs; a Stop-hook item for it added no measurable gain (gstack-evals#356), so it was removed. A contract that still lists `check_after_last_edit` keeps its other items. The checker skips that item and prints `SKILL_CONTRACT_SKIPPED <SKILL.md>: <item id>: ...` on stderr each time it reads the contract (the Stop hook and `check`). A contract whose only items use that kind is invalid.
A run the skill ended early counts as met: a final `STATUS: BLOCKED` (or `Status: BLOCKED` in the skill's report) or a final message with a line that starts "Nothing to review" (/review's early exit).
@@ -33,7 +33,7 @@
import { binPath, sessionKind } from './spawn-bin';
import { appendHookTrace, installedSkillsDir, readSkillSwitch, readStdinBounded } from '../../../lib/skill-route';
import {
AGENT_STALL_MS, StateError, UnknownShape, blockMessage, bumpCanary, checkContract, isFinalTurn, lookupContract, nextOutcome,
AGENT_STALL_MS, StateError, UnknownShape, blockMessage, bumpCanary, checkContract, isFinalTurn, lookupContract, nextOutcome, skippedLine,
readRecord, recordPath, runKey, sameSkill, skillLoads, stateErrorLine, writeRecord,
type Contract, type Transcript,
} from '../../../lib/skill-contract';
@@ -67,7 +67,11 @@ function target(t: Transcript): { skill: string; contract: Contract } | null {
if (tried.some(s => sameSkill(s, load.skill))) continue;
tried.push(load.skill);
const found = lookupContract(load.skill, dir);
if ('contract' in found) return { skill: load.skill, contract: found.contract };
if ('contract' in found) {
for (const why of found.contract.skipped ?? []) process.stderr.write(`${skippedLine(found.file, why)}
`);
return { skill: load.skill, contract: found.contract };
}
if (found.reason === 'invalid' && found.file) throw new StateError(found.file, found.error);
}
return null;
+15 -2
View File
@@ -24,6 +24,8 @@ import { resolveStateRoot } from './state-root';
export const TRANSCRIPT_MAX_BYTES = 32 * 1024 * 1024;
export const ITEM_KINDS = ['report_labels', 'selected_specialists_dispatched', 'agent_dispatch'] as const;
export type ItemKind = typeof ITEM_KINDS[number];
/** Kinds a contract may still list from an older gstack: each such item is skipped with a warning, not checked. */
const REMOVED_KINDS = ['check_after_last_edit'];
const ITEM_ID = /^[a-z0-9][a-z0-9_.-]{0,63}$/;
const MAX_ITEMS = 20;
@@ -42,13 +44,18 @@ export interface ContractItem {
fix?: string;
summary?: string;
}
export interface Contract { version: 1; items: ContractItem[] }
/** skipped: one warning per item whose kind was removed from gstack; the item is not checked. */
export interface Contract { version: 1; items: ContractItem[]; skipped?: string[] }
export class StateError extends Error {
constructor(readonly file: string, readonly why: string) { super(`${file}: ${why}`); }
}
export class UnknownShape extends Error {}
export function skippedLine(file: string, why: string): string {
return `gstack-skill-contract: SKILL_CONTRACT_SKIPPED ${file}: ${why}`;
}
export function stateErrorLine(file: string, why: string): string {
return `gstack-skill-contract: SKILL_CONTRACT_STATE_ERROR ${file}: ${why}; stop allowed, miss not recorded (see ${STATE_ERROR_DOC})`;
}
@@ -112,11 +119,16 @@ export function validateContract(raw: unknown): { contract: Contract } | { error
return { error: `contract items must be a list of 1 to ${MAX_ITEMS} items` };
}
const items: ContractItem[] = [];
const skipped: string[] = [];
for (const [n, item] of raw.items.entries()) {
if (!isObject(item)) return { error: `items[${n}] must be a mapping` };
const id = item.id;
if (typeof id !== 'string' || !ITEM_ID.test(id)) return { error: `items[${n}].id must match ${ITEM_ID}` };
if (items.some(i => i.id === id)) return { error: `duplicate item id ${id}` };
if (REMOVED_KINDS.includes(item.kind)) {
skipped.push(`${id}: kind ${item.kind} was removed from gstack, so this item is not checked; delete it from the contract`);
continue;
}
if (!(ITEM_KINDS as readonly string[]).includes(item.kind)) {
return { error: `${id}: unknown kind ${JSON.stringify(item.kind)} (expected one of ${ITEM_KINDS.join(', ')})` };
}
@@ -141,7 +153,8 @@ export function validateContract(raw: unknown): { contract: Contract } | { error
if (out.kind === 'agent_dispatch' && out.marker) return { error: `${id}: agent_dispatch takes no marker until the checker can bind a marker to the dispatch it names (GST-1242)` };
items.push(out);
}
return { contract: { version: 1, items } };
if (!items.length) return { error: `every item uses a removed kind (${skipped.join('; ')})` };
return { contract: skipped.length ? { version: 1, items, skipped } : { version: 1, items } };
}
/** One plain phrase per item, for the routing hint's "Done means" variant. */
+12 -1
View File
@@ -11,10 +11,11 @@ import { bunIsolationArgs } from '../lib/bun-isolation';
import { AGENT_STALL_MS, STATE_ERROR_DOC, stateErrorLine } from '../lib/skill-contract';
import { SWITCH_DEFAULT } from '../lib/skill-route';
import {
CONTRACT_HOOK, DEBUG_REPORT, INTERACTIVE_TURNS, J1_TRANSCRIPT, ROOT, hookEnv, meta, result, runHook, say, skillsDir, slash, tmpDir, tool, user,
CONTRACT_HOOK, DEBUG_REPORT, INTERACTIVE_TURNS, J1_TRANSCRIPT, ROOT, SKILLS, hookEnv, meta, result, runHook, say, skillsDir, slash, tmpDir, tool, user,
} from './helpers/skill-contract-fixture';
const BIN = path.join(ROOT, 'bin', 'gstack-skill-contract');
const SKILLS_LEGACY = SKILLS.verify.replace(' labels: [Symptom, Root cause, Fix, Evidence]\n', ' labels: [Symptom, Root cause, Fix, Evidence]\n - id: repro-after-fix\n kind: check_after_last_edit\n');
const PINNED = fs.readFileSync(path.join(ROOT, 'test', 'fixtures', 'skill-contract', 'block-message.txt'), 'utf8').trimEnd().replace('{bin}', BIN);
const STATE_ERROR = /^gstack-skill-contract: SKILL_CONTRACT_STATE_ERROR .+: .+; stop allowed, miss not recorded \(see docs\/skill-routing-and-contracts\.md#state-error\)\n$/;
@@ -149,6 +150,16 @@ describe('block once, with the pinned text', () => {
expect((Object.values(record().runs)[0] as any).outcome).toBe('met');
});
test('a custom skill whose contract still lists the removed check_after_last_edit kind: warned once, the rest still blocks', () => {
const legacy = SKILLS_LEGACY.replace('name: verify', 'name: legacy');
skills = skillsDir([], { legacy });
const r = stop(transcript(J1_TRANSCRIPT.map(l => l.replace('"skill":"verify"', '"skill":"legacy"'))));
expect(JSON.parse(r.stdout).decision).toBe('block');
expect(r.stderr).toBe(`gstack-skill-contract: SKILL_CONTRACT_SKIPPED ${path.join(skills, 'legacy', 'SKILL.md')}: repro-after-fix: kind check_after_last_edit was removed from gstack, so this item is not checked; delete it from the contract
`);
expect(Object.values(record().runs)[0]).toMatchObject({ skill: 'legacy', outcome: 'blocked', missing: ['debug-report'] });
});
test('a waiver read from the transcript satisfies the block but counts against follow', () => {
const lines = [...J1_TRANSCRIPT.slice(0, -1),
tool('w1', 'Bash', { command: 'gstack-skill-contract waive debug-report --reason "a docs-only change, no bug to report"' }),
+19 -1
View File
@@ -43,7 +43,7 @@ describe('schema v1', () => {
['contract:\n version: 1\n items: [{id: a, kind: agent_dispatch}]', 'agent_dispatch needs match'],
['contract:\n version: 1\n items: [{id: a, kind: report_labels, heading: X, labels: []}]', 'needs labels'],
['contract:\n version: 1\n items: [{id: Bad_ID, kind: agent_dispatch, match: x}]', 'id must match'],
['contract:\n version: 1\n items: [{id: a, kind: check_after_last_edit}]', 'unknown kind'],
['contract:\n version: 1\n items: [{id: a, kind: check_after_last_edit}]', 'every item uses a removed kind'],
['contract: [unclosed', 'not valid YAML'],
];
for (const [fm, msg] of bad) {
@@ -53,6 +53,14 @@ describe('schema v1', () => {
});
});
test('a contract that still lists the removed check_after_last_edit kind keeps its other items and names the skipped one', () => {
const parsed = parseContract('contract:\n version: 1\n items:\n - {id: debug-report, kind: report_labels, heading: DEBUG REPORT, labels: [Symptom]}\n - {id: repro-after-fix, kind: check_after_last_edit}');
if (!parsed || !('contract' in parsed)) throw new Error(JSON.stringify(parsed));
expect(parsed.contract.items.map(i => i.id)).toEqual(['debug-report']);
expect(parsed.contract.skipped).toEqual(['repro-after-fix: kind check_after_last_edit was removed from gstack, so this item is not checked; delete it from the contract']);
expect(parseContract('contract:\n version: 1\n items: [{id: a, kind: agent_dispatch, match: x}]')).toEqual({ contract: { version: 1, items: [{ id: 'a', kind: 'agent_dispatch', match: 'x' }] } });
});
describe('transcript parsing', () => {
test('skill loads: Skill tool, typed /command, Read of a SKILL.md; sidechain records ignored', () => {
const t = parseTranscript([
@@ -243,6 +251,16 @@ describe('gstack-skill-contract CLI', () => {
expect(JSON.parse(unmet.stdout)).toMatchObject({ exit: 1, skill: 'verify', missing: ['debug-report'], follow: false });
});
test('check: a contract that still lists the removed kind warns once on stderr and checks the rest', () => {
const file = path.join(work, 'SKILL.md');
fs.writeFileSync(file, SKILLS.verify.replace(' labels: [Symptom, Root cause, Fix, Evidence]\n', ' labels: [Symptom, Root cause, Fix, Evidence]\n - id: repro-after-fix\n kind: check_after_last_edit\n'));
const r = cli('check', '--transcript', write(J1_TRANSCRIPT), '--skill', 'verify', '--skill-file', file, '--json');
expect(r.code).toBe(EXIT.UNMET);
expect(JSON.parse(r.stdout)).toMatchObject({ missing: ['debug-report'] });
expect(r.stderr).toBe(`gstack-skill-contract: SKILL_CONTRACT_SKIPPED ${file}: repro-after-fix: kind check_after_last_edit was removed from gstack, so this item is not checked; delete it from the contract
`);
});
test('check: stable error exits 2, 3, 4, 5, 6', () => {
expect(cli('check', '--skill', 'verify').code).toBe(EXIT.USAGE);
expect(cli('check', '--transcript', path.join(work, 'none.jsonl'), '--skill', 'verify')).toMatchObject({ code: EXIT.STATE_ERROR });