mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-18 22:24:17 +01:00
Unattended /autoplan records the owner's relayed gate reply instead of refusing it (GST-3068) (#4484)
An unattended /autoplan run that's handed the owner's reply at the final gate now records it and finishes, the same way whichever instruction file the agent reads. Before, autoplan/SKILL.md said "never approve" while autoplan/DRIVER.md said to record the relayed reply, so the outcome depended on which file the model followed. Nothing to do; this fixes existing instructions and adds no new feature. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
44170d6791
commit
bbf48aa7ef
15 files changed
+119
-53
No files matched your search
@@ -0,0 +1,61 @@
|
||||
/**
|
||||
* GST-3068 (GEVA-549, day-one eval at e671a531): unattended /autoplan read two rules at the
|
||||
* final gate. autoplan/SKILL.md said "end here with its GSTACK_RESULT line and never approve";
|
||||
* autoplan/DRIVER.md said to record a relayed owner reply with --by agent:<id> --owner-reply.
|
||||
* An A session followed DRIVER.md and reached status=complete; an S-full session followed
|
||||
* SKILL.md, refused the owner's relayed "rec", told the owner to run the command and stayed
|
||||
* gate_pending. One rule (AUTOPLAN_UNATTENDED_GATE_RULE) now renders into the unattended
|
||||
* profile and DRIVER.md, and SKILL.md's gate step defers to it. These checks read the
|
||||
* generated files, so a template edit that brings back an unconditional "never approve" fails.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { AUTOPLAN_UNATTENDED_GATE_RULE } from '../scripts/resolvers/autoplan-driver';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const read = (rel: string) => fs.readFileSync(path.join(ROOT, rel), 'utf8');
|
||||
const flat = (s: string) => s.replace(/\s+/g, ' ');
|
||||
const rule = flat(AUTOPLAN_UNATTENDED_GATE_RULE);
|
||||
|
||||
/** Sentences that forbid approving with no condition: a "never approve" that neither limits itself to the agent's own judgment nor names an exception. */
|
||||
const unconditionalNeverApprove = (text: string) =>
|
||||
flat(text).split(/(?<=[.;])\s+/).filter(s => /never approve/i.test(s) && !/own judgment|only exception/i.test(s));
|
||||
|
||||
const GEVA_549_SKILL_SENTENCE = 'Under `SESSION_KIND: unattended` Read `sections/unattended.md`, end here with its `GSTACK_RESULT` line and never approve.';
|
||||
const GEVA_549_PROFILE_SENTENCE = 'and the run stops at `status=gate_pending`: never approve.';
|
||||
|
||||
describe('unattended /autoplan states one gate rule', () => {
|
||||
test('the detector flags the two sentences GEVA-549 ran on (negative control)', () => {
|
||||
expect(unconditionalNeverApprove(GEVA_549_SKILL_SENTENCE)).toHaveLength(1);
|
||||
expect(unconditionalNeverApprove(GEVA_549_PROFILE_SENTENCE)).toHaveLength(1);
|
||||
expect(unconditionalNeverApprove(rule)).toEqual([]);
|
||||
});
|
||||
|
||||
test('the rule forbids approving on the agent\'s own judgment and records a relayed owner reply with the words bound', () => {
|
||||
expect(rule).toContain('Never approve it on your own judgment: with no owner reply, the run stops at `status=gate_pending`.');
|
||||
expect(rule).toContain("Record an owner's reply relayed to you yourself, never handing them the command");
|
||||
expect(rule).toContain('--by agent:<id> --owner-reply-file <file of their words>');
|
||||
});
|
||||
|
||||
test('the unattended profile and DRIVER.md carry the rule verbatim, and SKILL.md\'s gate step defers to it', () => {
|
||||
for (const rel of ['autoplan/sections/unattended.md', 'autoplan/DRIVER.md']) expect({ rel, has: flat(read(rel)).includes(rule) }).toEqual({ rel, has: true });
|
||||
const skill = flat(read('autoplan/SKILL.md'));
|
||||
expect(skill).toContain('else = A. Under `SESSION_KIND: unattended` Read `sections/unattended.md` and settle the gate by its gate rule instead.');
|
||||
expect(skill).not.toContain(GEVA_549_SKILL_SENTENCE);
|
||||
expect(flat(read('autoplan/sections/unattended.md'))).not.toContain(GEVA_549_PROFILE_SENTENCE);
|
||||
});
|
||||
|
||||
test('no generated text an unattended /autoplan session reads says never approve without the condition', () => {
|
||||
for (const rel of ['autoplan/SKILL.md', 'autoplan/sections/unattended.md', 'autoplan/DRIVER.md']) {
|
||||
expect({ rel, unconditional: unconditionalNeverApprove(read(rel)) }).toEqual({ rel, unconditional: [] });
|
||||
}
|
||||
});
|
||||
|
||||
test('every flag the rule names is one gstack-autoplan answer parses', () => {
|
||||
const bin = read('bin/gstack-autoplan');
|
||||
const flags = [...new Set(AUTOPLAN_UNATTENDED_GATE_RULE.match(/--[a-z][a-z-]+/g) ?? [])];
|
||||
expect(flags).toEqual(['--out', '--gate-rev', '--reply', '--by', '--owner-reply-file']);
|
||||
for (const f of flags) expect({ f, parsed: bin.includes(`'${f}'`) }).toEqual({ f, parsed: true });
|
||||
});
|
||||
});
|
||||
+12
-12
@@ -2,53 +2,53 @@
|
||||
"_comment": "Generated SKILL.md files over the limit, pinned at their exact character count (#3199). test/skill-md-size-guard.test.ts fails when a pinned skill grows past its pin or shrinks below it: lower the pin when a skill shrinks and delete it once the skill is within the limit. Raising a pin is an owner decision. The shared-preamble hoist (GSTA-96) moved every tier 2+ skill's shared preamble into sections/preamble.md; the pins below are what remains.",
|
||||
"limit": 100000,
|
||||
"pins": {
|
||||
"agy/gstack-autoplan": 109449,
|
||||
"agy/gstack-autoplan": 109152,
|
||||
"agy/gstack-plan-design-review": 109103,
|
||||
"agy/gstack-plan-devex-review": 114944,
|
||||
"agy/gstack-plan-eng-review": 143530,
|
||||
"agy/gstack-review": 104723,
|
||||
"codex/gstack-autoplan": 108192,
|
||||
"codex/gstack-autoplan": 107895,
|
||||
"codex/gstack-plan-design-review": 110460,
|
||||
"codex/gstack-plan-devex-review": 116452,
|
||||
"codex/gstack-plan-eng-review": 144695,
|
||||
"codex/gstack-review": 104107,
|
||||
"copilot/gstack-autoplan": 108892,
|
||||
"copilot/gstack-autoplan": 108595,
|
||||
"copilot/gstack-plan-design-review": 109369,
|
||||
"copilot/gstack-plan-devex-review": 114406,
|
||||
"copilot/gstack-plan-eng-review": 143695,
|
||||
"copilot/gstack-review": 104304,
|
||||
"cursor/gstack-autoplan": 108124,
|
||||
"cursor/gstack-autoplan": 107827,
|
||||
"cursor/gstack-plan-design-review": 107911,
|
||||
"cursor/gstack-plan-devex-review": 113676,
|
||||
"cursor/gstack-plan-eng-review": 142285,
|
||||
"cursor/gstack-review": 122776,
|
||||
"factory/gstack-autoplan": 110402,
|
||||
"factory/gstack-autoplan": 110105,
|
||||
"factory/gstack-plan-design-review": 111009,
|
||||
"factory/gstack-plan-devex-review": 117039,
|
||||
"factory/gstack-plan-eng-review": 145393,
|
||||
"factory/gstack-review": 126549,
|
||||
"gbrain/gstack-autoplan": 108531,
|
||||
"gbrain/gstack-autoplan": 108234,
|
||||
"gbrain/gstack-plan-eng-review": 124932,
|
||||
"gemini/gstack-autoplan": 108682,
|
||||
"gemini/gstack-autoplan": 108385,
|
||||
"gemini/gstack-plan-design-review": 109197,
|
||||
"gemini/gstack-plan-devex-review": 114234,
|
||||
"gemini/gstack-plan-eng-review": 143528,
|
||||
"gemini/gstack-review": 104126,
|
||||
"hermes/gstack-autoplan": 108536,
|
||||
"hermes/gstack-autoplan": 108239,
|
||||
"hermes/gstack-plan-eng-review": 124885,
|
||||
"kiro/gstack-autoplan": 107898,
|
||||
"kiro/gstack-autoplan": 107601,
|
||||
"kiro/gstack-plan-design-review": 107757,
|
||||
"kiro/gstack-plan-devex-review": 113528,
|
||||
"kiro/gstack-plan-eng-review": 142147,
|
||||
"kiro/gstack-review": 122600,
|
||||
"openclaw/gstack-autoplan": 108354,
|
||||
"openclaw/gstack-autoplan": 108057,
|
||||
"openclaw/gstack-plan-eng-review": 123428,
|
||||
"opencode/gstack-autoplan": 108873,
|
||||
"opencode/gstack-autoplan": 108576,
|
||||
"opencode/gstack-plan-design-review": 109225,
|
||||
"opencode/gstack-plan-devex-review": 114284,
|
||||
"opencode/gstack-plan-eng-review": 143554,
|
||||
"opencode/gstack-review": 123398,
|
||||
"slate/gstack-autoplan": 108011,
|
||||
"slate/gstack-autoplan": 107714,
|
||||
"slate/gstack-plan-design-review": 107834,
|
||||
"slate/gstack-plan-devex-review": 113602,
|
||||
"slate/gstack-plan-eng-review": 142216,
|
||||
|
||||
@@ -425,6 +425,9 @@ describe('gstack-skill-start behavior', () => {
|
||||
expect(autoplan).toContain('gate list (gstack-gate render)');
|
||||
expect(autoplan).toMatch(/decisions\.jsonl with kind "auto"/);
|
||||
expect(autoplan).toContain('gate.json');
|
||||
// GST-3068 (GEVA-549): the general "never approve a gate" rule defers to the record rule's exception; autoplan's names the relayed owner reply.
|
||||
expect(autoplan).toContain("The one exception: when the owner's own reply to that gate is relayed to you, record it with `gstack-autoplan answer --by agent:<id> --owner-reply-file <file>` as sections/unattended.md says, never on your own judgment and never handed back to the owner to run.");
|
||||
for (const [skill, text] of Object.entries(blocks)) if (skill !== 'autoplan') expect({ skill, relays: text.includes('--owner-reply') }).toEqual({ skill, relays: false });
|
||||
const evalPlan = blocks['eval-plan'];
|
||||
expect(evalPlan).toContain('`GSTACK_RESULT: skill=eval-plan status=gate_pending run=<outdir>`');
|
||||
expect(evalPlan).toContain('No helper writes a gate.json');
|
||||
|
||||
@@ -224,7 +224,7 @@ describe.skipIf(process.platform === 'win32')('host conformance kit', () => {
|
||||
expect(runSetup(f, join(src, 'setup'), ['--host', 'claude']).status).toBe(0);
|
||||
const sections = join(f.home, '.claude/skills/autoplan/sections');
|
||||
expect(lstatSync(sections).isSymbolicLink(), sections).toBe(true);
|
||||
expect(readFileSync(join(f.home, '.claude/skills/autoplan/SKILL.md'), 'utf8')).toContain('Read `sections/unattended.md`, end here');
|
||||
expect(readFileSync(join(f.home, '.claude/skills/autoplan/SKILL.md'), 'utf8')).toContain('Read `sections/unattended.md` and settle the gate');
|
||||
rmSync(join(src, 'autoplan/sections/unattended.md'));
|
||||
const status = runSetup(f, join(src, 'setup'), ['--status']);
|
||||
expect(status.status, status.stdout + status.stderr).toBe(1);
|
||||
|
||||
Reference in new issue
Block a user