mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-13 20:07:08 +01:00
test(flaky): plan-design no-mockup Voices text, docsync private mkdir, autoplan-guard-pty journal on unverified allow (GSTA-279) (#3997)
Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
4b9d74faf9
commit
f86c9491b2
11 files changed
+323
-5
No files matched your search
@@ -0,0 +1,6 @@
|
||||
### Fixed
|
||||
|
||||
- **Two paid flaky cases now accept what their fixtures already allow, and a third prints the evidence it was missing.** These come from main backstop run 41817566885 (main b244e6d1). Each case failed once and passed its rerun.
|
||||
- `plan-design-with-ui-scope`: in CI the image generator has no API key, so no mockup ever gets approved. The fixture's declared Voices menu still said "with the approved mockup as anchor". One run told the truth and wrote "(text-only this time, since mockups weren't available)" instead, and the picker refused it. The fixture now declares that one replacement. A free test replays the captured question; it fails on main and passes with the fix, and six variations that are not the declared replacement are still refused.
|
||||
- `ship-docsync-current`: the docs interface says a private artifact can live "in a new directory you create there" under the fixture home, and `gstack-docs-candidate snapshot` does not create the folder for its `--out` path. The parent ran `mkdir -p <home>/ship2` as its own call, and the detector refused it. The detector now accepts a plain `mkdir` or `mkdir -p` of one new folder directly under the home, as its own call. A free test replays the whole captured run, which fails on main and passes now. The test also shows that 25 other `mkdir` forms are still refused: the repository, protected roots, nested paths, `..`, symlinks, globs, variables, options and joined lines.
|
||||
- `autoplan-guard-pty`: the first guarded call got an unverified allow (`journal_lag`), because Claude Code didn't write its journal within the guard's 2-second window. The run had all four allows, so it never printed the journal, and cleanup deleted it. Now any run that falls short of four verified allows prints the journal and the time of each decision. The guard and its window are unchanged.
|
||||
@@ -2,11 +2,26 @@ import { expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { closePacketReadAfterNotice, dispatchEventRewriter, dispatchPromptDelta, dispatchToken, prepareGuardSession, startDispatchRelay, stoppedAtPhaseBoundary } from './helpers/autoplan-guard-pty';
|
||||
import { closePacketReadAfterNotice, dispatchEventRewriter, dispatchPromptDelta, dispatchToken, guardRunNeedsJournal, prepareGuardSession, startDispatchRelay, stoppedAtPhaseBoundary } from './helpers/autoplan-guard-pty';
|
||||
import { codeOf, guardFixture } from './helpers/autoplan-guard-fixture';
|
||||
|
||||
const base = '/home/runner/.cache/gstack-paid-shard-x/tmp/ap-guard-pty-y';
|
||||
|
||||
// Main backstop 41817566885 (b244e6d1, paid-slice-1-a1, green on the in-run rerun): four allows, the first unverified
|
||||
// (journal_lag, payload path), so the run failed its verified-allow assertion without printing the journal.
|
||||
test('an unverified allow prints the journal before cleanup; four verified allows with no card do not', () => {
|
||||
const verified = { decision: 'allow', disposition: 'allow', code: null, path: 'payload', claude_code_version: '2.1.292' };
|
||||
const captured = [{ ...verified, disposition: 'unverified', code: 'journal_lag' }, verified, verified, verified];
|
||||
expect(guardRunNeedsJournal(captured, 0)).toBe(true);
|
||||
expect(guardRunNeedsJournal([verified, verified, verified, verified], 0)).toBe(false);
|
||||
expect(guardRunNeedsJournal([verified, verified, verified, verified], 1)).toBe(true);
|
||||
expect(guardRunNeedsJournal([verified, verified, verified], 0)).toBe(true);
|
||||
expect(guardRunNeedsJournal([verified, verified, verified, { ...verified, decision: 'deny', disposition: 'deny' }], 0)).toBe(true);
|
||||
expect(guardRunNeedsJournal([verified, verified, { ...verified, disposition: 'fallback' }, verified], 0)).toBe(true);
|
||||
const live = fs.readFileSync(path.join(import.meta.dir, 'skill-e2e-autoplan-guard-pty.test.ts'), 'utf8');
|
||||
expect(live).toContain('if (guardRunNeedsJournal(log, prompts.length))');
|
||||
});
|
||||
|
||||
test('the close-packet check accepts a Read in the notice turn and rejects one made before the notice', () => {
|
||||
// PR gate 39069171159 (b789898, pass): the CEO reviewer's notice started the turn that read the close packet.
|
||||
expect(closePacketReadAfterNotice([
|
||||
|
||||
@@ -675,3 +675,34 @@ test.skipIf(process.platform === 'win32')('the captured lines joining a read com
|
||||
expect(docsCommandAllowed(line, fixture, [publish]), line).toBe(false);
|
||||
} finally { fixture.clean(); }
|
||||
});
|
||||
|
||||
// Main backstop 41817566885 (b244e6d1, paid-slice-5-a1, ship-docsync-current, green on the in-run rerun): the parent ran
|
||||
// `mkdir -p <home>/ship2` as its own call, then snapshotted into it. The interface lets a private artifact live "in a new
|
||||
// directory you create there", and gstack-docs-candidate snapshot does not create the parent of its --out.
|
||||
test.skipIf(process.platform === 'win32')('the captured private-directory mkdir replays clean, and mkdir grants nothing else', () => {
|
||||
const captured = JSON.parse(fs.readFileSync(path.join(import.meta.dir, 'fixtures/docsync-private-mkdir-public.json'), 'utf8')) as
|
||||
{ scenario: 'current'; calls: SkillTestResult['toolCalls'] };
|
||||
const fixture = fixtureDocs(captured.scenario);
|
||||
try {
|
||||
const publish = path.join(fixture.home, 'fixture-publish.ts');
|
||||
const fill = (text: string) => text.replaceAll('{home}', fixture.home).replaceAll('{shard}', path.dirname(path.dirname(fixture.home)));
|
||||
const replayed = captured.calls.map(call => ({ ...call, input: Object.fromEntries(Object.entries(call.input).map(([key, value]) =>
|
||||
[key, typeof value === 'string' ? fill(value) : value])), output: fill(call.output) }));
|
||||
expect(replayed.filter(call => /^mkdir /.test(String(call.input?.command ?? ''))).map(call => call.input.command))
|
||||
.toEqual([`mkdir -p ${fixture.home}/ship2`]);
|
||||
expect(docsToolFailures({ toolCalls: replayed, exitReason: 'success' } as SkillTestResult, fixture, [publish], true)).toEqual([]);
|
||||
for (const line of [`mkdir -p ${fixture.home}/ship2`, `mkdir ${fixture.home}/docs-audit`, `mkdir -p '${fixture.home}/audit-a1'`])
|
||||
expect(docsCommandAllowed(line, fixture, [publish]), line).toBe(true);
|
||||
const outside = fs.mkdtempSync(path.join(path.dirname(fixture.home), 'ds-mkdir-outside-'));
|
||||
try {
|
||||
fs.symlinkSync(outside, path.join(fixture.home, 'linked'));
|
||||
for (const line of [`mkdir -p ${fixture.repo}/notes`, `mkdir -p ${fixture.repo}`, `mkdir -p ${fixture.home}/state`, `mkdir -p ${fixture.home}/state/x`,
|
||||
`mkdir -p ${fixture.home}/.claude`, `mkdir -p ${fixture.skills}/x`, `mkdir -p ${fixture.env.CLAUDE_CONFIG_DIR}`, `mkdir -p ${fixture.home}/remote.git`,
|
||||
`mkdir -p ${fixture.home}/a/b`, `mkdir -p ${fixture.home}/..`, `mkdir -p ${fixture.home}/x/..`, `mkdir -p ${fixture.home}`, `mkdir -p ${fixture.home}/linked`,
|
||||
`mkdir -p ${outside}/x`, 'mkdir -p ship2', 'mkdir -p ~/ship2', 'mkdir -p $HOME/ship2', `mkdir -p ${fixture.home}/ship*`, `mkdir -m 777 ${fixture.home}/x`,
|
||||
`mkdir -p ${fixture.home}/x ${fixture.home}/y`, `mkdir -p ${fixture.home}/x && git status`, `git status; mkdir -p ${fixture.home}/x`,
|
||||
`mkdir -p ${fixture.home}/x; rm -rf ${fixture.repo}`, `mkdir -p ${fixture.home}/x > ${fixture.repo}/out.md`, `mkdir -pv ${fixture.home}/x`])
|
||||
expect(docsCommandAllowed(line, fixture, [publish]), line).toBe(false);
|
||||
} finally { fs.rmSync(outside, { recursive: true, force: true }); }
|
||||
} finally { fixture.clean(); }
|
||||
});
|
||||
@@ -62,5 +62,26 @@
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
},
|
||||
"textOnlyNoDescription41817566885": {
|
||||
"source": "Main backstop gate 41817566885 (main b244e6d1), paid-slice-6-a1, observation.json pendingQuestion",
|
||||
"cliVersion": "2.1.292",
|
||||
"toolUseId": "toolu_012UXuVt8LBywg7ePoFVmSrT",
|
||||
"note": "The image generator had no API key, so no mockup was approved; the native question replaced \"with the approved mockup as anchor\" in the No description with \"(text-only this time, since mockups weren't available)\" and otherwise kept the declared menu byte for byte. The passing rerun (paid-rerun-2-a1) kept the declared text although its mockups were unavailable too.",
|
||||
"question": {
|
||||
"question": "D3: Want outside design voices before the detailed review? Codex checks the plan against OpenAI's design hard rules and litmus checks, and a separate Claude subagent does an independent completeness review. Stakes: a second opinion on AI-slop risk and hierarchy, at the cost of a few extra minutes. Recommendation: No. The plan is a small app UI and the gaps are already clear, and with no mockup to anchor them, outside voices add less. This is a scope choice; it doesn't affect completeness.",
|
||||
"header": "Voices",
|
||||
"options": [
|
||||
{
|
||||
"label": "Yes, run outside design voices",
|
||||
"description": "✅ Codex hard-rule + litmus scorecard plus an independent Claude completeness read. ✅ Disagreements surface with both perspectives in the relevant pass. ❌ Adds a few minutes; Codex may be missing here, in which case only the Claude voice runs and it's tagged single-model."
|
||||
},
|
||||
{
|
||||
"label": "No, proceed without (recommended)",
|
||||
"description": "✅ Straight into the 7 passes (text-only this time, since mockups weren't available). ✅ Fewest steps to a fixed plan. ❌ Findings rest on one reviewer; no cross-model check on slop risk or hierarchy."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
}
|
||||
}
|
||||
+190
@@ -0,0 +1,190 @@
|
||||
{
|
||||
"note": "Every tool call of the ship-docsync-current parent in main backstop gate 41817566885 (main b244e6d1, paid-slice-5-a1), red on \"command outside declared docs observation interface\" and green on its in-run rerun. {home} is the fixture home and {shard} its shard root. Inputs keep only command, file_path or run_in_background; outputs are kept only for installed gstack helper calls and failed calls. Call 10 is mkdir -p {home}/ship2: the interface lets a private artifact live in a new directory the agent creates under the home, and the next call snapshots into it.",
|
||||
"sourceRun": "41817566885 slice 5 ship-docsync-current (main backstop gate.backstop-b244e6d11a64)",
|
||||
"scenario": "current",
|
||||
"sourceFileSha256": "8515e5e5e437f06597dad4bd0f110f5bb0254eee79bee8b70beecbd82cae63ed",
|
||||
"calls": [
|
||||
{
|
||||
"tool": "Bash",
|
||||
"input": {
|
||||
"command": "cat {home}/phase.md"
|
||||
},
|
||||
"output": ""
|
||||
},
|
||||
{
|
||||
"tool": "Bash",
|
||||
"input": {
|
||||
"command": "cat {home}/.claude/skills/gstack/ship/sections/documentation.md"
|
||||
},
|
||||
"output": ""
|
||||
},
|
||||
{
|
||||
"tool": "Bash",
|
||||
"input": {
|
||||
"command": "cat {home}/fixture-interface.md | head -c 400; echo; ls -la {home} {home}/.claude/skills/gstack/document-release {home}/.claude/skills/gstack/bin"
|
||||
},
|
||||
"output": ""
|
||||
},
|
||||
{
|
||||
"tool": "Read",
|
||||
"input": {
|
||||
"file_path": "{home}/.claude/skills/gstack/document-release/SKILL.md"
|
||||
},
|
||||
"output": ""
|
||||
},
|
||||
{
|
||||
"tool": "Bash",
|
||||
"input": {
|
||||
"command": "ls -la {home}/.claude/skills/gstack/document-release/sections; git status; git log --oneline --all; git diff main...HEAD --stat; ls -la; find . -path ./.git -prune -o -type f -print"
|
||||
},
|
||||
"output": ""
|
||||
},
|
||||
{
|
||||
"tool": "Read",
|
||||
"input": {
|
||||
"file_path": "{home}/.claude/skills/gstack/document-release/sections/audit-scope.md"
|
||||
},
|
||||
"output": ""
|
||||
},
|
||||
{
|
||||
"tool": "Bash",
|
||||
"input": {
|
||||
"command": "git diff main...HEAD; cat handbook/reference/commands/widget.md.tmpl; cat README.md AGENTS.md CHANGELOG.md TODOS.md VERSION; git branch -a -v; git merge-base main HEAD; git rev-parse origin/main main HEAD"
|
||||
},
|
||||
"output": ""
|
||||
},
|
||||
{
|
||||
"tool": "Bash",
|
||||
"input": {
|
||||
"command": "_EX=$(git rev-parse --git-path info/exclude 2>/dev/null) && mkdir -p \"$(dirname \"$_EX\")\" && { grep -qxF '/.gstack/tmp/' \"$_EX\" 2>/dev/null || echo '/.gstack/tmp/' >> \"$_EX\"; }"
|
||||
},
|
||||
"output": ""
|
||||
},
|
||||
{
|
||||
"tool": "Bash",
|
||||
"input": {
|
||||
"command": "rm -f .gstack/tmp/docs-audit.json"
|
||||
},
|
||||
"output": ""
|
||||
},
|
||||
{
|
||||
"tool": "Bash",
|
||||
"input": {
|
||||
"command": "{home}/.claude/skills/gstack/bin/gstack-docs-candidate --help"
|
||||
},
|
||||
"output": "usage: gstack-docs-candidate snapshot --out <candidate.json> --audit-id <id> --mode edit|read-only --base <rev> [--select <path>]... [--docs <path>[,<path>...]]... [--exception <docs-audit.json>]\n gstack-docs-candidate compare <candidate.json>\n gstack-docs-candidate attempt <child-output> --state <attempts.json>"
|
||||
},
|
||||
{
|
||||
"tool": "Bash",
|
||||
"input": {
|
||||
"command": "mkdir -p {home}/ship2"
|
||||
},
|
||||
"output": ""
|
||||
},
|
||||
{
|
||||
"tool": "Bash",
|
||||
"input": {
|
||||
"command": "{home}/.claude/skills/gstack/bin/gstack-docs-candidate snapshot --out {home}/ship2/ship2-docs-a1.json --audit-id ship2-docs-a1 --mode edit --base b0dca411d5ad6012553218e04861917e7c3eef5c --docs handbook --docs README.md --docs AGENTS.md"
|
||||
},
|
||||
"output": "CANDIDATE: {home}/ship2/ship2-docs-a1.json\nbase=b0dca411d5ad6012553218e04861917e7c3eef5c head=5b0984bee296e78b007eac9afa366df4c3dcf07b branch=feature/docs mode=edit\ncommitted=1 staged=0 unstaged=0 untracked=1 selected=2 docs=3 hashed=4\ncommitted: \"handbook/reference/commands/widget.md.tmpl\"\nuntracked: \"personal-note.txt\"\ndocs: \"AGENTS.md\" \"README.md\" \"handbook/reference/commands/widget.md.tmpl\""
|
||||
},
|
||||
{
|
||||
"tool": "Bash",
|
||||
"input": {
|
||||
"command": "cat personal-note.txt; cat {home}/ship2/ship2-docs-a1.json"
|
||||
},
|
||||
"output": ""
|
||||
},
|
||||
{
|
||||
"tool": "Agent",
|
||||
"input": {
|
||||
"run_in_background": false
|
||||
},
|
||||
"output": ""
|
||||
},
|
||||
{
|
||||
"tool": "Bash",
|
||||
"input": {
|
||||
"command": "{home}/.claude/skills/gstack/bin/gstack-docs-candidate compare {home}/ship2/ship2-docs-a1.json"
|
||||
},
|
||||
"output": "{\"audit_id\":\"ship2-docs-a1\",\"head_changed\":false,\"index_changed\":false,\"content_changed\":[],\"newly_dirty\":[]}"
|
||||
},
|
||||
{
|
||||
"tool": "Bash",
|
||||
"input": {
|
||||
"command": "cat app.ts package.json; git status --porcelain; git rev-parse HEAD; git diff --cached --stat; sha256sum personal-note.txt; git hash-object personal-note.txt"
|
||||
},
|
||||
"output": ""
|
||||
},
|
||||
{
|
||||
"tool": "Bash",
|
||||
"input": {
|
||||
"command": "{home}/.claude/skills/gstack/bin/gstack-ship-receipt docs-caps --base main"
|
||||
},
|
||||
"output": "DOCS_CAP: none declared"
|
||||
},
|
||||
{
|
||||
"tool": "Bash",
|
||||
"input": {
|
||||
"command": "{home}/.claude/skills/gstack/bin/gstack-docs-candidate snapshot --out {home}/ship2/ship2-docs-a1-post.json --audit-id ship2-docs-a1 --mode edit --base b0dca411d5ad6012553218e04861917e7c3eef5c --docs handbook --docs README.md --docs AGENTS.md"
|
||||
},
|
||||
"output": "CANDIDATE: {home}/ship2/ship2-docs-a1-post.json\nbase=b0dca411d5ad6012553218e04861917e7c3eef5c head=5b0984bee296e78b007eac9afa366df4c3dcf07b branch=feature/docs mode=edit\ncommitted=1 staged=0 unstaged=0 untracked=1 selected=2 docs=3 hashed=4\ncommitted: \"handbook/reference/commands/widget.md.tmpl\"\nuntracked: \"personal-note.txt\"\ndocs: \"AGENTS.md\" \"README.md\" \"handbook/reference/commands/widget.md.tmpl\""
|
||||
},
|
||||
{
|
||||
"tool": "Write",
|
||||
"input": {
|
||||
"file_path": "{home}/ship2/ship2-docs-a1-result.json"
|
||||
},
|
||||
"output": ""
|
||||
},
|
||||
{
|
||||
"tool": "Write",
|
||||
"input": {
|
||||
"file_path": "{home}/ship2/ship2-docs-a1-documentation.md"
|
||||
},
|
||||
"output": ""
|
||||
},
|
||||
{
|
||||
"tool": "Write",
|
||||
"input": {
|
||||
"file_path": "{home}/repo/.gstack/tmp/docs-audit.json"
|
||||
},
|
||||
"output": ""
|
||||
},
|
||||
{
|
||||
"tool": "Bash",
|
||||
"input": {
|
||||
"command": "git check-ignore -v .gstack/tmp/docs-audit.json; git status --porcelain; cat {home}/fixture-publish.ts"
|
||||
},
|
||||
"output": ""
|
||||
},
|
||||
{
|
||||
"tool": "Write",
|
||||
"input": {
|
||||
"file_path": "{home}/ship-report.md"
|
||||
},
|
||||
"output": ""
|
||||
},
|
||||
{
|
||||
"tool": "Bash",
|
||||
"input": {
|
||||
"command": "cat {home}/ship2/ship2-docs-a1-documentation.md >> {home}/ship-report.md"
|
||||
},
|
||||
"output": ""
|
||||
},
|
||||
{
|
||||
"tool": "Bash",
|
||||
"input": {
|
||||
"command": "bun {home}/fixture-publish.ts"
|
||||
},
|
||||
"output": ""
|
||||
},
|
||||
{
|
||||
"tool": "Bash",
|
||||
"input": {
|
||||
"command": "ls -la {home}/publication.json; git status --porcelain"
|
||||
},
|
||||
"output": ""
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -54,6 +54,14 @@ export function stoppedAtPhaseBoundary(screen: string): boolean {
|
||||
return /✻[^\n]*\bdone\b/.test(after) && !/Step\s*7|design-phase|Design\s*review/i.test(after);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the run must print its journal before cleanup: any run short of four verified allows and no permission card.
|
||||
* Main backstop 41817566885 kept four allows, but the first was unverified (journal_lag on the payload path), so the
|
||||
* journal that names the lagging record was deleted unread.
|
||||
*/
|
||||
export const guardRunNeedsJournal = (log: Array<Record<string, any>>, promptCount: number): boolean =>
|
||||
log.length !== 4 || promptCount > 0 || log.some(e => e.decision !== 'allow' || e.disposition !== 'allow');
|
||||
|
||||
/** The close packet Read (step 6) came after the CEO reviewer's completion notice, so it ran in the notice's new turn. */
|
||||
export const closePacketReadAfterNotice = (afterNotice: string[]): boolean => afterNotice.some(c => /^Read .*close-packet\.md$/.test(c));
|
||||
|
||||
|
||||
@@ -258,6 +258,10 @@ export function docsCommandAllowed(command: string, fixture: ReturnType<typeof f
|
||||
const insert = /^cat (['"]?)([^\s'"]+)\1 >> (['"]?)([^\s'"]+)\3$/.exec(text);
|
||||
if (insert) return [insert[2], insert[4]].every(file => path.isAbsolute(file) && /\.md$/i.test(file) &&
|
||||
!/[\x00-\x1f\x7f;&|<>`$\\()*?\[\]{}~#]/.test(file) && docsPrivateArtifact(file, fixture, scripts));
|
||||
// The interface lets a private artifact live "in a new directory you create there" (under the home), and snapshot --out
|
||||
// does not create its parent (gate 41817566885: the parent ran `mkdir -p <home>/ship2` before its snapshot).
|
||||
const directory = /^mkdir (?:-p )?(['"]?)([^\s'"]+)\1$/.exec(text);
|
||||
if (directory) return docsPrivateDirectory(directory[2]!, fixture);
|
||||
// Harmless wrappers are accepted on read-only commands only; lifecycle and
|
||||
// script calls must stay literal so their failures are not suppressed.
|
||||
const unwrapped = unwrapHarmlessRead(text, fixture);
|
||||
@@ -461,6 +465,15 @@ function docsPrivateArtifact(file: string, fixture: ReturnType<typeof fixtureDoc
|
||||
path.basename(file) !== 'actor-state.json' && path.basename(actual) !== 'actor-state.json';
|
||||
}
|
||||
|
||||
/** True when `dir` is one new directory directly under the fixture home, clear of the repository and every protected root. */
|
||||
function docsPrivateDirectory(dir: string, fixture: ReturnType<typeof fixtureDocs>): boolean {
|
||||
if (!path.isAbsolute(dir) || path.normalize(dir) !== dir || path.dirname(dir) !== fixture.home ||
|
||||
/[\x00-\x1f\x7f;&|<>`$\\()*?\[\]{}~#]/.test(dir)) return false;
|
||||
const roots = [fixture.repo, fixture.skills, fixture.env.CLAUDE_CONFIG_DIR, fixture.env.GSTACK_HOME, path.join(fixture.home, 'remote.git')];
|
||||
return actualWritePath(dir) === path.join(fs.realpathSync(fixture.home), path.basename(dir)) &&
|
||||
!roots.some(root => within(dir, root) || within(root, dir));
|
||||
}
|
||||
|
||||
/**
|
||||
* A call to one of Step 14.5's own helpers (the docs candidate or the cap gate; never a lifecycle helper) at a wrong
|
||||
* binary path that does not exist exited 127 before anything ran; it is ignored only when the same command at the
|
||||
|
||||
@@ -117,6 +117,7 @@ export const FREE_FIXTURES: Readonly<Record<string, readonly string[]>> = {
|
||||
'test/fixtures/docsync-invalid-first-attempt-public.json': ['test/ship-document-release-dispatch.test.ts'],
|
||||
'test/fixtures/docsync-observer-commands.json': ['test/docsync-lifecycle-interface.test.ts'],
|
||||
'test/fixtures/docsync-phase-read-aborted-public.json': ['test/docsync-authority.test.ts'],
|
||||
'test/fixtures/docsync-private-mkdir-public.json': ['test/docsync-command-grammar.test.ts'],
|
||||
'test/fixtures/docsync-read-help-public.json': ['test/docsync-command-grammar.test.ts'],
|
||||
'test/fixtures/docsync-replay/37237194905-late-result-calls.json': ['test/docsync-authority.test.ts'],
|
||||
'test/fixtures/docsync-replay/37239016858-completion-native.json': ['test/docsync-nested-writes.test.ts'],
|
||||
|
||||
@@ -24,6 +24,8 @@ const DESIGN_OUTSIDE_VOICE_OPTIONS = [
|
||||
"description": "✅ Straight into the 7 passes with the approved mockup as anchor. ✅ Fewest steps to a fixed plan. ❌ Findings rest on one reviewer; no cross-model check on slop risk or hierarchy."
|
||||
}
|
||||
] as const;
|
||||
const DESIGN_OUTSIDE_VOICE_MOCKUP_ANCHOR = 'with the approved mockup as anchor';
|
||||
const DESIGN_OUTSIDE_VOICE_TEXT_ONLY = "(text-only this time, since mockups weren't available)";
|
||||
|
||||
export const DESIGN_BOARD_ACTOR_PROTOCOL = [
|
||||
'## Fixture user: native Design review',
|
||||
@@ -37,6 +39,8 @@ export const DESIGN_BOARD_ACTOR_PROTOCOL = [
|
||||
...DESIGN_OUTSIDE_VOICE_OPTIONS.map(option => `- ${option.label}: ${option.description}`),
|
||||
'Option order, a letter prefix, a trailing (recommended) marker, and',
|
||||
'keeping or dropping every ✅/❌ bullet in a description may vary.',
|
||||
`When no mockup was approved, "${DESIGN_OUTSIDE_VOICE_TEXT_ONLY}" may replace`,
|
||||
`"${DESIGN_OUTSIDE_VOICE_MOCKUP_ANCHOR}" in the No description.`,
|
||||
'Keep this choice separate from other approvals; no previews or extra actions.',
|
||||
'The fixture user chooses No, proceed without. Observe the actual native',
|
||||
'answer before continuing when this question is offered.',
|
||||
@@ -121,7 +125,8 @@ export function createDesignReviewPicker({ cwd, deadlineAt }: { cwd: string; dea
|
||||
if (claimsVoices) {
|
||||
const voiceActions = question.options.map((option, index) => DESIGN_OUTSIDE_VOICE_OPTIONS.findIndex(expected =>
|
||||
labels[index] === expected.label && option.preview === undefined
|
||||
&& [expected.description, expected.description.replace(/(^| )[✅❌] /g, '$1')].includes(option.description)));
|
||||
&& [expected.description, expected.description.replace(DESIGN_OUTSIDE_VOICE_MOCKUP_ANCHOR, DESIGN_OUTSIDE_VOICE_TEXT_ONLY)]
|
||||
.flatMap(description => [description, description.replace(/(^| )[✅❌] /g, '$1')]).includes(option.description)));
|
||||
if (question.header.trim() !== 'Voices'
|
||||
|| !/^(?:Want outside design voices before the detailed review|Run outside design voices before the 7 passes)\?(?:\s|$)/i.test(lead)
|
||||
|| question.multiSelect || voiceActions.length !== 2 || voiceActions.includes(-1)
|
||||
|
||||
@@ -142,6 +142,34 @@ test('the native-only actor declines the declared menu with every ✅/❌ bullet
|
||||
await expectUnsubmitted(published);
|
||||
});
|
||||
|
||||
// Main backstop 41817566885 (b244e6d1, paid-slice-6-a1, passed on the in-run rerun): the image generator had no API key,
|
||||
// so no mockup was approved, and the native question replaced the No description's "with the approved mockup as anchor"
|
||||
// with "(text-only this time, since mockups weren't available)". The protocol declares that one replacement.
|
||||
test('the native-only actor declines the declared menu whose No description says no mockup was approved', async () => {
|
||||
const published = await board();
|
||||
const textOnly = structuredClone(outsideVoices.textOnlyNoDescription41817566885.question) as NativeQuestion;
|
||||
expect(textOnly.options.map(option => option.description).join(' ')).not.toContain('approved mockup');
|
||||
expect(picker()(textOnly)).toBe(2);
|
||||
expect(DESIGN_BOARD_ACTOR_PROTOCOL).toContain(`"(text-only this time, since mockups weren't available)" may replace`);
|
||||
const bulletFree = structuredClone(textOnly);
|
||||
for (const option of bulletFree.options) option.description = option.description.replace(/(^| )[✅❌] /g, '$1');
|
||||
expect(picker()(bulletFree)).toBe(2);
|
||||
const changes: ((menu: NativeQuestion) => void)[] = [
|
||||
menu => { menu.options[1]!.description = menu.options[1]!.description.replace('this time', 'for now'); },
|
||||
menu => { menu.options[1]!.description = menu.options[1]!.description.replace(" (text-only this time, since mockups weren't available)", ''); },
|
||||
menu => { menu.options[1]!.description = menu.options[1]!.description.replace('available)', 'available) with the approved mockup as anchor'); },
|
||||
menu => { menu.options[0]!.description = menu.options[0]!.description.replace('read.', "read (text-only this time, since mockups weren't available)."); },
|
||||
menu => { menu.options[1]!.description = menu.options[1]!.description.replace('✅ Fewest', 'Fewest'); },
|
||||
menu => { menu.options[1]!.description += ' Also approve the plan.'; },
|
||||
];
|
||||
for (const change of changes) {
|
||||
const menu = structuredClone(textOnly);
|
||||
change(menu);
|
||||
expect(() => picker()(menu)).toThrow('no unambiguous declared native-only action');
|
||||
}
|
||||
await expectUnsubmitted(published);
|
||||
});
|
||||
|
||||
test('the native-only actor accepts the declared question with a same-line explanation', async () => {
|
||||
const published = await board();
|
||||
const retained = structuredClone(outsideVoices.sameLineExplanation77.question) as NativeQuestion;
|
||||
|
||||
@@ -22,7 +22,7 @@ import { expect } from 'bun:test';
|
||||
import * as path from 'node:path';
|
||||
import { describeIfSelected, testIfSelected } from './helpers/e2e-helpers';
|
||||
import { launchClaudePty, selectPtyNumberedOption } from './helpers/claude-pty-runner';
|
||||
import { closePacketReadAfterNotice, prepareGuardSession, startDispatchRelay, stoppedAtPhaseBoundary } from './helpers/autoplan-guard-pty';
|
||||
import { closePacketReadAfterNotice, guardRunNeedsJournal, prepareGuardSession, startDispatchRelay, stoppedAtPhaseBoundary } from './helpers/autoplan-guard-pty';
|
||||
|
||||
const MODEL = 'claude-haiku-4-5-20251001';
|
||||
|
||||
@@ -59,10 +59,10 @@ describeIfSelected('autoplan publication guard live PTY', ['autoplan-guard-pty']
|
||||
await Bun.sleep(3_000);
|
||||
const log = s.readLog(), evidence = s.noticeEvidence();
|
||||
const summary = { prompts: prompts.length, substituted: relay.substituted, phase1Ms, totalMs: Math.round(performance.now() - started), evidence,
|
||||
decisions: log.map(e => ({ decision: e.decision, code: e.code, path: e.path, version: e.claude_code_version })) };
|
||||
decisions: log.map(e => ({ decision: e.decision, code: e.code, path: e.path, version: e.claude_code_version, ts: e.ts })) };
|
||||
console.log(`[autoplan-guard-pty] ${JSON.stringify(summary)}`);
|
||||
if (log.length < 4 && stoppedAtPhaseBoundary(session.visibleText())) console.log('[autoplan-guard-pty] PHASE_BOUNDARY_STOP: the actor ended its turn after "Phase 1 complete." instead of going on to step 7');
|
||||
if (log.length !== 4 || log.some(e => e.decision === 'deny') || prompts.length)
|
||||
if (guardRunNeedsJournal(log, prompts.length))
|
||||
console.log(`[autoplan-guard-pty] journal:\n${s.journalSummary().join('\n')}\n[autoplan-guard-pty] screen:\n${session.visibleText().slice(-2_000)}\n[autoplan-guard-pty] last non-blank screen:\n${session.lastScreen?.() ?? ''}`);
|
||||
if (log.some(e => e.code === 'dispatch_prompt')) console.log(`[autoplan-guard-pty] dispatch prompt deltas:\n${s.dispatchPromptDeltas().join('\n')}`);
|
||||
for (const prompt of prompts) console.log(`[autoplan-guard-pty] permission card approved:\n${prompt}`);
|
||||
|
||||
Reference in new issue
Block a user