mirror of
https://github.com/garrytan/gstack.git
synced 2026-06-18 07:40:09 +02:00
Merge remote-tracking branch 'origin/main' into garrytan/triage-open-issues
This commit is contained in:
Garry Tan
@@ -0,0 +1,131 @@
|
||||
/**
|
||||
* auq-error-fallback-hook — the OV3:B runtime reliability layer.
|
||||
*
|
||||
* Two layers of testing:
|
||||
* - PURE functions (isErrorResponse, directiveFor): deterministic, the core logic.
|
||||
* - INTEGRATION: spawn the hook as a PostToolUse process with synthetic stdin and
|
||||
* a controlled env, assert it injects the right directive on an error result and
|
||||
* stays inert on a real answer.
|
||||
*
|
||||
* NOTE: whether the Claude Code PLATFORM invokes PostToolUse on an MCP
|
||||
* transport/missing-result error is unverified (could not force the Conductor
|
||||
* bug in a harness — see docs/spikes/claude-code-hook-mutation.md). These tests
|
||||
* pin the hook's BEHAVIOR given it is invoked; the platform trigger is the
|
||||
* documented residual risk. The hook is inert if never invoked.
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as path from 'path';
|
||||
import { isErrorResponse, directiveFor } from '../hosts/claude/hooks/auq-error-fallback-hook.ts';
|
||||
|
||||
const HOOK = path.resolve(__dirname, '..', 'hosts', 'claude', 'hooks', 'auq-error-fallback-hook.ts');
|
||||
|
||||
describe('isErrorResponse — only clear failures, never a real answer', () => {
|
||||
test('null / undefined / empty string are failures', () => {
|
||||
expect(isErrorResponse(null)).toBe(true);
|
||||
expect(isErrorResponse(undefined)).toBe(true);
|
||||
expect(isErrorResponse('')).toBe(true);
|
||||
expect(isErrorResponse(' ')).toBe(true);
|
||||
});
|
||||
|
||||
test('the Conductor missing-result string is a failure', () => {
|
||||
expect(isErrorResponse('[Tool result missing due to internal error]')).toBe(true);
|
||||
});
|
||||
|
||||
test('is_error: true / error-field / sentinel-in-content are failures', () => {
|
||||
expect(isErrorResponse({ is_error: true })).toBe(true);
|
||||
expect(isErrorResponse({ isError: true })).toBe(true);
|
||||
expect(isErrorResponse({ error: 'boom' })).toBe(true);
|
||||
expect(isErrorResponse({ content: 'Tool result missing due to internal error' })).toBe(true);
|
||||
});
|
||||
|
||||
test('a real answer is NOT a failure (no false trigger)', () => {
|
||||
expect(isErrorResponse({ answers: [{ option_label: 'A' }] })).toBe(false);
|
||||
expect(isErrorResponse('A')).toBe(false);
|
||||
// a choice that coincidentally contains "error" must not trip it
|
||||
expect(isErrorResponse({ answers: [{ option_label: 'Fix the error' }] })).toBe(false);
|
||||
expect(isErrorResponse('Investigate the login error')).toBe(false);
|
||||
});
|
||||
|
||||
test('Codex review: narrow detection — generic "error"/"is_error" substrings do NOT trigger', () => {
|
||||
// A real answer mentioning "internal error" must not be read as a failure.
|
||||
expect(isErrorResponse('Investigate the internal error')).toBe(false);
|
||||
// A serialized success payload containing the substring is_error:false must not trigger.
|
||||
expect(isErrorResponse('{"is_error": false, "answer": "A"}')).toBe(false);
|
||||
expect(isErrorResponse({ is_error: false })).toBe(false);
|
||||
expect(isErrorResponse({ content: 'The page had an internal error we fixed' })).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('directiveFor — per-session-kind instruction', () => {
|
||||
test('interactive directive demands the prose triad', () => {
|
||||
const d = directiveFor('interactive');
|
||||
expect(d).toMatch(/ELI10/);
|
||||
expect(d).toMatch(/Completeness: X\/10/);
|
||||
expect(d).toMatch(/\(recommended\)/);
|
||||
expect(d).toMatch(/reply with a letter/i);
|
||||
expect(d).toMatch(/STOP/);
|
||||
});
|
||||
|
||||
test('headless directive BLOCKs', () => {
|
||||
expect(directiveFor('headless')).toMatch(/BLOCKED — AskUserQuestion unavailable/);
|
||||
});
|
||||
|
||||
test('spawned directive auto-chooses', () => {
|
||||
expect(directiveFor('spawned')).toMatch(/auto-choose/i);
|
||||
});
|
||||
});
|
||||
|
||||
/** Spawn the hook with synthetic stdin + controlled env; parse its JSON stdout. */
|
||||
function runHook(stdin: object, env: Record<string, string>): { additionalContext?: string } {
|
||||
const res = spawnSync('bun', [HOOK], {
|
||||
input: JSON.stringify(stdin),
|
||||
encoding: 'utf-8',
|
||||
env: { PATH: process.env.PATH ?? '/usr/bin:/bin', ...env },
|
||||
});
|
||||
const parsed = JSON.parse(res.stdout || '{}');
|
||||
return parsed.hookSpecificOutput ?? {};
|
||||
}
|
||||
|
||||
describe('hook integration — invoked as PostToolUse', () => {
|
||||
test('error result + headless env → injects BLOCK directive', () => {
|
||||
const out = runHook(
|
||||
{ tool_name: 'mcp__conductor__AskUserQuestion', tool_response: '[Tool result missing due to internal error]' },
|
||||
{ GSTACK_HEADLESS: '1' },
|
||||
);
|
||||
expect(out.additionalContext).toMatch(/BLOCKED — AskUserQuestion unavailable/);
|
||||
});
|
||||
|
||||
test('error result + interactive env → injects prose-triad directive', () => {
|
||||
const out = runHook(
|
||||
{ tool_name: 'AskUserQuestion', tool_response: null },
|
||||
{ CONDUCTOR_PORT: '55010' },
|
||||
);
|
||||
expect(out.additionalContext).toMatch(/render the decision as a PROSE message/i);
|
||||
expect(out.additionalContext).toMatch(/Completeness: X\/10/);
|
||||
});
|
||||
|
||||
test('error result + spawned env → injects auto-choose directive', () => {
|
||||
const out = runHook(
|
||||
{ tool_name: 'AskUserQuestion', tool_response: { is_error: true } },
|
||||
{ OPENCLAW_SESSION: '1' },
|
||||
);
|
||||
expect(out.additionalContext).toMatch(/auto-choose/i);
|
||||
});
|
||||
|
||||
test('SUCCESSFUL answer → no injection (inert on real answers)', () => {
|
||||
const out = runHook(
|
||||
{ tool_name: 'AskUserQuestion', tool_response: { answers: [{ option_label: 'A' }] } },
|
||||
{ GSTACK_HEADLESS: '1' },
|
||||
);
|
||||
expect(out.additionalContext).toBeUndefined();
|
||||
});
|
||||
|
||||
test('non-AUQ tool → defers (no injection)', () => {
|
||||
const out = runHook(
|
||||
{ tool_name: 'Bash', tool_response: null },
|
||||
{ GSTACK_HEADLESS: '1' },
|
||||
);
|
||||
expect(out.additionalContext).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -47,6 +47,11 @@ const MANDATORY: Array<{ name: string; re: RegExp }> = [
|
||||
{ name: 'Completeness coverage rule', re: /Completeness\s*:/i },
|
||||
{ name: 'kind-vs-coverage rule', re: /options differ in kind/i },
|
||||
{ name: 'Self-check checklist', re: /Self-check before emitting/i },
|
||||
// The runtime-failure fallback must be ALWAYS-LOADED too: when an AUQ call errors
|
||||
// mid-skill, the model needs the prose-fallback rule in context that instant, not
|
||||
// stranded in an on-demand section. Same guarantee as the format spec above.
|
||||
{ name: 'AUQ-failure fallback subsection', re: /When AskUserQuestion is unavailable or a call fails/i },
|
||||
{ name: 'fallback SESSION_KIND branch', re: /SESSION_KIND/ },
|
||||
];
|
||||
|
||||
/** Per-skill AUQ rules that govern review-finding cadence. A carve may move
|
||||
|
||||
+27
-4
@@ -50,6 +50,9 @@ echo "SKILL_PREFIX: $_SKILL_PREFIX"
|
||||
source <(~/.claude/skills/gstack/bin/gstack-repo-mode 2>/dev/null) || true
|
||||
REPO_MODE=${REPO_MODE:-unknown}
|
||||
echo "REPO_MODE: $REPO_MODE"
|
||||
_SESSION_KIND=$(~/.claude/skills/gstack/bin/gstack-session-kind 2>/dev/null || echo "interactive")
|
||||
case "$_SESSION_KIND" in spawned|headless|interactive) ;; *) _SESSION_KIND="interactive" ;; esac
|
||||
echo "SESSION_KIND: $_SESSION_KIND"
|
||||
_LAKE_SEEN=$([ -f ~/.gstack/.completeness-intro-seen ] && echo "yes" || echo "no")
|
||||
echo "LAKE_INTRO: $_LAKE_SEEN"
|
||||
_TEL=$(~/.claude/skills/gstack/bin/gstack-config get telemetry 2>/dev/null || true)
|
||||
@@ -129,7 +132,7 @@ In plan mode, allowed because they inform the plan: `$B`, `$D`, `codex exec`/`co
|
||||
|
||||
## Skill Invocation During Plan Mode
|
||||
|
||||
If the user invokes a skill in plan mode, the skill takes precedence over generic plan mode behavior. **Treat the skill file as executable instructions, not reference.** Follow it step by step starting from Step 0; the first AskUserQuestion is the workflow entering plan mode, not a violation of it. AskUserQuestion (any variant — `mcp__*__AskUserQuestion` or native; see "AskUserQuestion Format → Tool resolution") satisfies plan mode's end-of-turn requirement. If no variant is callable, the skill is BLOCKED — stop and report `BLOCKED — AskUserQuestion unavailable` per the AskUserQuestion Format rule. At a STOP point, stop immediately. Do not continue the workflow or call ExitPlanMode there. Commands marked "PLAN MODE EXCEPTION — ALWAYS RUN" execute. Call ExitPlanMode only after the skill workflow completes, or if the user tells you to cancel the skill or leave plan mode.
|
||||
If the user invokes a skill in plan mode, the skill takes precedence over generic plan mode behavior. **Treat the skill file as executable instructions, not reference.** Follow it step by step starting from Step 0; the first AskUserQuestion is the workflow entering plan mode, not a violation of it. AskUserQuestion (any variant — `mcp__*__AskUserQuestion` or native; see "AskUserQuestion Format → Tool resolution") satisfies plan mode's end-of-turn requirement. If AskUserQuestion is unavailable or a call fails, follow the AskUserQuestion Format failure fallback: `headless` → BLOCKED; `interactive` → the prose fallback (also satisfies end-of-turn). At a STOP point, stop immediately. Do not continue the workflow or call ExitPlanMode there. Commands marked "PLAN MODE EXCEPTION — ALWAYS RUN" execute. Call ExitPlanMode only after the skill workflow completes, or if the user tells you to cancel the skill or leave plan mode.
|
||||
|
||||
If `PROACTIVE` is `"false"`, do not auto-invoke or proactively suggest skills. If a skill seems useful, ask: "I think /skillname might help here — want me to run it?"
|
||||
|
||||
@@ -301,11 +304,31 @@ AI orchestrator (e.g., OpenClaw). In spawned sessions:
|
||||
|
||||
**Rule:** if any `mcp__*__AskUserQuestion` variant is in your tool list, prefer it. Hosts may disable native AUQ via `--disallowedTools AskUserQuestion` (Conductor does, by default) and route through their MCP variant; calling native there silently fails. Same questions/options shape; same decision-brief format applies.
|
||||
|
||||
**If no AskUserQuestion variant appears in your tool list, this skill is BLOCKED.** Stop, report `BLOCKED — AskUserQuestion unavailable`, and wait for the user. Do not write decisions to the plan file as a substitute, do not emit them as prose and stop, and do not silently auto-decide (only `/plan-tune` AUTO_DECIDE opt-ins authorize auto-picking).
|
||||
If AskUserQuestion is unavailable (no variant in your tool list) OR a call to it fails, do NOT silently auto-decide or write the decision to the plan file as a substitute. Follow the **failure fallback** below.
|
||||
|
||||
### When AskUserQuestion is unavailable or a call fails
|
||||
|
||||
Tell three outcomes apart:
|
||||
|
||||
1. **Auto-decide denial (NOT a failure).** The result contains `[plan-tune auto-decide] <id> → <option>` — the preference hook working as designed. Proceed with that option. Do NOT retry, do NOT fall back to prose.
|
||||
2. **Genuine failure** — no variant in your tool list, OR the variant is present but the call returns an error / missing result (MCP transport error, empty result, host bug — e.g. Conductor's MCP AskUserQuestion is flaky and returns `[Tool result missing due to internal error]`).
|
||||
- If it was present and **errored** (not absent), retry the SAME call **once** — but only if no answer could have surfaced (a missing-result error can arrive after the user already saw the question; retrying would double-prompt, so if it may have reached them, treat as pending, don't retry).
|
||||
- Then branch on `SESSION_KIND` (echoed by the preamble; empty/absent ⇒ `interactive`):
|
||||
- `spawned` → defer to the **Spawned session** block: auto-choose the recommended option. Never prose, never BLOCKED.
|
||||
- `headless` → `BLOCKED — AskUserQuestion unavailable`; stop and wait (no human can answer).
|
||||
- `interactive` → **prose fallback** (below).
|
||||
|
||||
**Prose fallback — render the decision brief as a markdown message, not a tool call.** Same information as the tool format below, different structure (paragraphs, not ✅/❌ bullets). It MUST surface this triad:
|
||||
|
||||
1. **A clear ELI10 of the issue itself** — plain English on what's being decided and why it matters (the question, not per-choice), naming the stakes. Lead with it.
|
||||
2. **Completeness scores per choice** — explicit `Completeness: X/10` on EACH choice (10 complete, 7 happy-path, 3 shortcut); use the kind-note when options differ in kind not coverage, but never silently drop the score.
|
||||
3. **The recommendation and why** — a `Recommendation: <choice> because <reason>` line plus the `(recommended)` marker on that choice.
|
||||
|
||||
Layout: a `D<N>` title + a one-line note that AskUserQuestion failed and to reply with a letter; the issue ELI10; the Recommendation line; then ONE paragraph per choice carrying its `(recommended)` marker, its `Completeness: X/10`, and 2-4 sentences of reasoning — never a bare bullet list; a closing `Net:` line. Split chains / 5+ options: one prose block per per-option call, in sequence. Then STOP and wait — the user's typed answer is the decision. In plan mode this satisfies end-of-turn like a tool call.
|
||||
|
||||
### Format
|
||||
|
||||
Every AskUserQuestion is a decision brief and must be sent as tool_use, not prose.
|
||||
Every AskUserQuestion is a decision brief and must be sent as tool_use, not prose — unless the documented failure fallback above applies (interactive session + the call is unavailable/erroring), in which case the prose fallback is the correct output.
|
||||
|
||||
```
|
||||
D<N> — <one-line question title>
|
||||
@@ -385,7 +408,7 @@ Before calling AskUserQuestion, verify:
|
||||
- [ ] (recommended) label on one option (even for neutral-posture)
|
||||
- [ ] Dual-scale effort labels on effort-bearing options (human / CC)
|
||||
- [ ] Net line closes the decision
|
||||
- [ ] You are calling the tool, not writing prose
|
||||
- [ ] You are calling the tool, not writing prose — unless the documented failure fallback applies (then: prose with the mandatory triad — issue ELI10, per-choice Completeness, Recommendation + `(recommended)` — and a "reply with a letter" instruction, then STOP)
|
||||
- [ ] Non-ASCII characters (CJK / accents) written directly, NOT \u-escaped
|
||||
- [ ] If you had 5+ options, you split (or batched into ≤4-groups) — did NOT drop any
|
||||
- [ ] If you split, you checked dependencies between options before firing the chain
|
||||
|
||||
+27
-4
@@ -36,6 +36,9 @@ echo "SKILL_PREFIX: $_SKILL_PREFIX"
|
||||
source <($GSTACK_BIN/gstack-repo-mode 2>/dev/null) || true
|
||||
REPO_MODE=${REPO_MODE:-unknown}
|
||||
echo "REPO_MODE: $REPO_MODE"
|
||||
_SESSION_KIND=$($GSTACK_BIN/gstack-session-kind 2>/dev/null || echo "interactive")
|
||||
case "$_SESSION_KIND" in spawned|headless|interactive) ;; *) _SESSION_KIND="interactive" ;; esac
|
||||
echo "SESSION_KIND: $_SESSION_KIND"
|
||||
_LAKE_SEEN=$([ -f ~/.gstack/.completeness-intro-seen ] && echo "yes" || echo "no")
|
||||
echo "LAKE_INTRO: $_LAKE_SEEN"
|
||||
_TEL=$($GSTACK_BIN/gstack-config get telemetry 2>/dev/null || true)
|
||||
@@ -115,7 +118,7 @@ In plan mode, allowed because they inform the plan: `$B`, `$D`, `codex exec`/`co
|
||||
|
||||
## Skill Invocation During Plan Mode
|
||||
|
||||
If the user invokes a skill in plan mode, the skill takes precedence over generic plan mode behavior. **Treat the skill file as executable instructions, not reference.** Follow it step by step starting from Step 0; the first AskUserQuestion is the workflow entering plan mode, not a violation of it. AskUserQuestion (any variant — `mcp__*__AskUserQuestion` or native; see "AskUserQuestion Format → Tool resolution") satisfies plan mode's end-of-turn requirement. If no variant is callable, the skill is BLOCKED — stop and report `BLOCKED — AskUserQuestion unavailable` per the AskUserQuestion Format rule. At a STOP point, stop immediately. Do not continue the workflow or call ExitPlanMode there. Commands marked "PLAN MODE EXCEPTION — ALWAYS RUN" execute. Call ExitPlanMode only after the skill workflow completes, or if the user tells you to cancel the skill or leave plan mode.
|
||||
If the user invokes a skill in plan mode, the skill takes precedence over generic plan mode behavior. **Treat the skill file as executable instructions, not reference.** Follow it step by step starting from Step 0; the first AskUserQuestion is the workflow entering plan mode, not a violation of it. AskUserQuestion (any variant — `mcp__*__AskUserQuestion` or native; see "AskUserQuestion Format → Tool resolution") satisfies plan mode's end-of-turn requirement. If AskUserQuestion is unavailable or a call fails, follow the AskUserQuestion Format failure fallback: `headless` → BLOCKED; `interactive` → the prose fallback (also satisfies end-of-turn). At a STOP point, stop immediately. Do not continue the workflow or call ExitPlanMode there. Commands marked "PLAN MODE EXCEPTION — ALWAYS RUN" execute. Call ExitPlanMode only after the skill workflow completes, or if the user tells you to cancel the skill or leave plan mode.
|
||||
|
||||
If `PROACTIVE` is `"false"`, do not auto-invoke or proactively suggest skills. If a skill seems useful, ask: "I think /skillname might help here — want me to run it?"
|
||||
|
||||
@@ -287,11 +290,31 @@ AI orchestrator (e.g., OpenClaw). In spawned sessions:
|
||||
|
||||
**Rule:** if any `mcp__*__AskUserQuestion` variant is in your tool list, prefer it. Hosts may disable native AUQ via `--disallowedTools AskUserQuestion` (Conductor does, by default) and route through their MCP variant; calling native there silently fails. Same questions/options shape; same decision-brief format applies.
|
||||
|
||||
**If no AskUserQuestion variant appears in your tool list, this skill is BLOCKED.** Stop, report `BLOCKED — AskUserQuestion unavailable`, and wait for the user. Do not write decisions to the plan file as a substitute, do not emit them as prose and stop, and do not silently auto-decide (only `/plan-tune` AUTO_DECIDE opt-ins authorize auto-picking).
|
||||
If AskUserQuestion is unavailable (no variant in your tool list) OR a call to it fails, do NOT silently auto-decide or write the decision to the plan file as a substitute. Follow the **failure fallback** below.
|
||||
|
||||
### When AskUserQuestion is unavailable or a call fails
|
||||
|
||||
Tell three outcomes apart:
|
||||
|
||||
1. **Auto-decide denial (NOT a failure).** The result contains `[plan-tune auto-decide] <id> → <option>` — the preference hook working as designed. Proceed with that option. Do NOT retry, do NOT fall back to prose.
|
||||
2. **Genuine failure** — no variant in your tool list, OR the variant is present but the call returns an error / missing result (MCP transport error, empty result, host bug — e.g. Conductor's MCP AskUserQuestion is flaky and returns `[Tool result missing due to internal error]`).
|
||||
- If it was present and **errored** (not absent), retry the SAME call **once** — but only if no answer could have surfaced (a missing-result error can arrive after the user already saw the question; retrying would double-prompt, so if it may have reached them, treat as pending, don't retry).
|
||||
- Then branch on `SESSION_KIND` (echoed by the preamble; empty/absent ⇒ `interactive`):
|
||||
- `spawned` → defer to the **Spawned session** block: auto-choose the recommended option. Never prose, never BLOCKED.
|
||||
- `headless` → `BLOCKED — AskUserQuestion unavailable`; stop and wait (no human can answer).
|
||||
- `interactive` → **prose fallback** (below).
|
||||
|
||||
**Prose fallback — render the decision brief as a markdown message, not a tool call.** Same information as the tool format below, different structure (paragraphs, not ✅/❌ bullets). It MUST surface this triad:
|
||||
|
||||
1. **A clear ELI10 of the issue itself** — plain English on what's being decided and why it matters (the question, not per-choice), naming the stakes. Lead with it.
|
||||
2. **Completeness scores per choice** — explicit `Completeness: X/10` on EACH choice (10 complete, 7 happy-path, 3 shortcut); use the kind-note when options differ in kind not coverage, but never silently drop the score.
|
||||
3. **The recommendation and why** — a `Recommendation: <choice> because <reason>` line plus the `(recommended)` marker on that choice.
|
||||
|
||||
Layout: a `D<N>` title + a one-line note that AskUserQuestion failed and to reply with a letter; the issue ELI10; the Recommendation line; then ONE paragraph per choice carrying its `(recommended)` marker, its `Completeness: X/10`, and 2-4 sentences of reasoning — never a bare bullet list; a closing `Net:` line. Split chains / 5+ options: one prose block per per-option call, in sequence. Then STOP and wait — the user's typed answer is the decision. In plan mode this satisfies end-of-turn like a tool call.
|
||||
|
||||
### Format
|
||||
|
||||
Every AskUserQuestion is a decision brief and must be sent as tool_use, not prose.
|
||||
Every AskUserQuestion is a decision brief and must be sent as tool_use, not prose — unless the documented failure fallback above applies (interactive session + the call is unavailable/erroring), in which case the prose fallback is the correct output.
|
||||
|
||||
```
|
||||
D<N> — <one-line question title>
|
||||
@@ -371,7 +394,7 @@ Before calling AskUserQuestion, verify:
|
||||
- [ ] (recommended) label on one option (even for neutral-posture)
|
||||
- [ ] Dual-scale effort labels on effort-bearing options (human / CC)
|
||||
- [ ] Net line closes the decision
|
||||
- [ ] You are calling the tool, not writing prose
|
||||
- [ ] You are calling the tool, not writing prose — unless the documented failure fallback applies (then: prose with the mandatory triad — issue ELI10, per-choice Completeness, Recommendation + `(recommended)` — and a "reply with a letter" instruction, then STOP)
|
||||
- [ ] Non-ASCII characters (CJK / accents) written directly, NOT \u-escaped
|
||||
- [ ] If you had 5+ options, you split (or batched into ≤4-groups) — did NOT drop any
|
||||
- [ ] If you split, you checked dependencies between options before firing the chain
|
||||
|
||||
+27
-4
@@ -38,6 +38,9 @@ echo "SKILL_PREFIX: $_SKILL_PREFIX"
|
||||
source <($GSTACK_BIN/gstack-repo-mode 2>/dev/null) || true
|
||||
REPO_MODE=${REPO_MODE:-unknown}
|
||||
echo "REPO_MODE: $REPO_MODE"
|
||||
_SESSION_KIND=$($GSTACK_BIN/gstack-session-kind 2>/dev/null || echo "interactive")
|
||||
case "$_SESSION_KIND" in spawned|headless|interactive) ;; *) _SESSION_KIND="interactive" ;; esac
|
||||
echo "SESSION_KIND: $_SESSION_KIND"
|
||||
_LAKE_SEEN=$([ -f ~/.gstack/.completeness-intro-seen ] && echo "yes" || echo "no")
|
||||
echo "LAKE_INTRO: $_LAKE_SEEN"
|
||||
_TEL=$($GSTACK_BIN/gstack-config get telemetry 2>/dev/null || true)
|
||||
@@ -117,7 +120,7 @@ In plan mode, allowed because they inform the plan: `$B`, `$D`, `codex exec`/`co
|
||||
|
||||
## Skill Invocation During Plan Mode
|
||||
|
||||
If the user invokes a skill in plan mode, the skill takes precedence over generic plan mode behavior. **Treat the skill file as executable instructions, not reference.** Follow it step by step starting from Step 0; the first AskUserQuestion is the workflow entering plan mode, not a violation of it. AskUserQuestion (any variant — `mcp__*__AskUserQuestion` or native; see "AskUserQuestion Format → Tool resolution") satisfies plan mode's end-of-turn requirement. If no variant is callable, the skill is BLOCKED — stop and report `BLOCKED — AskUserQuestion unavailable` per the AskUserQuestion Format rule. At a STOP point, stop immediately. Do not continue the workflow or call ExitPlanMode there. Commands marked "PLAN MODE EXCEPTION — ALWAYS RUN" execute. Call ExitPlanMode only after the skill workflow completes, or if the user tells you to cancel the skill or leave plan mode.
|
||||
If the user invokes a skill in plan mode, the skill takes precedence over generic plan mode behavior. **Treat the skill file as executable instructions, not reference.** Follow it step by step starting from Step 0; the first AskUserQuestion is the workflow entering plan mode, not a violation of it. AskUserQuestion (any variant — `mcp__*__AskUserQuestion` or native; see "AskUserQuestion Format → Tool resolution") satisfies plan mode's end-of-turn requirement. If AskUserQuestion is unavailable or a call fails, follow the AskUserQuestion Format failure fallback: `headless` → BLOCKED; `interactive` → the prose fallback (also satisfies end-of-turn). At a STOP point, stop immediately. Do not continue the workflow or call ExitPlanMode there. Commands marked "PLAN MODE EXCEPTION — ALWAYS RUN" execute. Call ExitPlanMode only after the skill workflow completes, or if the user tells you to cancel the skill or leave plan mode.
|
||||
|
||||
If `PROACTIVE` is `"false"`, do not auto-invoke or proactively suggest skills. If a skill seems useful, ask: "I think /skillname might help here — want me to run it?"
|
||||
|
||||
@@ -289,11 +292,31 @@ AI orchestrator (e.g., OpenClaw). In spawned sessions:
|
||||
|
||||
**Rule:** if any `mcp__*__AskUserQuestion` variant is in your tool list, prefer it. Hosts may disable native AUQ via `--disallowedTools AskUserQuestion` (Conductor does, by default) and route through their MCP variant; calling native there silently fails. Same questions/options shape; same decision-brief format applies.
|
||||
|
||||
**If no AskUserQuestion variant appears in your tool list, this skill is BLOCKED.** Stop, report `BLOCKED — AskUserQuestion unavailable`, and wait for the user. Do not write decisions to the plan file as a substitute, do not emit them as prose and stop, and do not silently auto-decide (only `/plan-tune` AUTO_DECIDE opt-ins authorize auto-picking).
|
||||
If AskUserQuestion is unavailable (no variant in your tool list) OR a call to it fails, do NOT silently auto-decide or write the decision to the plan file as a substitute. Follow the **failure fallback** below.
|
||||
|
||||
### When AskUserQuestion is unavailable or a call fails
|
||||
|
||||
Tell three outcomes apart:
|
||||
|
||||
1. **Auto-decide denial (NOT a failure).** The result contains `[plan-tune auto-decide] <id> → <option>` — the preference hook working as designed. Proceed with that option. Do NOT retry, do NOT fall back to prose.
|
||||
2. **Genuine failure** — no variant in your tool list, OR the variant is present but the call returns an error / missing result (MCP transport error, empty result, host bug — e.g. Conductor's MCP AskUserQuestion is flaky and returns `[Tool result missing due to internal error]`).
|
||||
- If it was present and **errored** (not absent), retry the SAME call **once** — but only if no answer could have surfaced (a missing-result error can arrive after the user already saw the question; retrying would double-prompt, so if it may have reached them, treat as pending, don't retry).
|
||||
- Then branch on `SESSION_KIND` (echoed by the preamble; empty/absent ⇒ `interactive`):
|
||||
- `spawned` → defer to the **Spawned session** block: auto-choose the recommended option. Never prose, never BLOCKED.
|
||||
- `headless` → `BLOCKED — AskUserQuestion unavailable`; stop and wait (no human can answer).
|
||||
- `interactive` → **prose fallback** (below).
|
||||
|
||||
**Prose fallback — render the decision brief as a markdown message, not a tool call.** Same information as the tool format below, different structure (paragraphs, not ✅/❌ bullets). It MUST surface this triad:
|
||||
|
||||
1. **A clear ELI10 of the issue itself** — plain English on what's being decided and why it matters (the question, not per-choice), naming the stakes. Lead with it.
|
||||
2. **Completeness scores per choice** — explicit `Completeness: X/10` on EACH choice (10 complete, 7 happy-path, 3 shortcut); use the kind-note when options differ in kind not coverage, but never silently drop the score.
|
||||
3. **The recommendation and why** — a `Recommendation: <choice> because <reason>` line plus the `(recommended)` marker on that choice.
|
||||
|
||||
Layout: a `D<N>` title + a one-line note that AskUserQuestion failed and to reply with a letter; the issue ELI10; the Recommendation line; then ONE paragraph per choice carrying its `(recommended)` marker, its `Completeness: X/10`, and 2-4 sentences of reasoning — never a bare bullet list; a closing `Net:` line. Split chains / 5+ options: one prose block per per-option call, in sequence. Then STOP and wait — the user's typed answer is the decision. In plan mode this satisfies end-of-turn like a tool call.
|
||||
|
||||
### Format
|
||||
|
||||
Every AskUserQuestion is a decision brief and must be sent as tool_use, not prose.
|
||||
Every AskUserQuestion is a decision brief and must be sent as tool_use, not prose — unless the documented failure fallback above applies (interactive session + the call is unavailable/erroring), in which case the prose fallback is the correct output.
|
||||
|
||||
```
|
||||
D<N> — <one-line question title>
|
||||
@@ -373,7 +396,7 @@ Before calling AskUserQuestion, verify:
|
||||
- [ ] (recommended) label on one option (even for neutral-posture)
|
||||
- [ ] Dual-scale effort labels on effort-bearing options (human / CC)
|
||||
- [ ] Net line closes the decision
|
||||
- [ ] You are calling the tool, not writing prose
|
||||
- [ ] You are calling the tool, not writing prose — unless the documented failure fallback applies (then: prose with the mandatory triad — issue ELI10, per-choice Completeness, Recommendation + `(recommended)` — and a "reply with a letter" instruction, then STOP)
|
||||
- [ ] Non-ASCII characters (CJK / accents) written directly, NOT \u-escaped
|
||||
- [ ] If you had 5+ options, you split (or batched into ≤4-groups) — did NOT drop any
|
||||
- [ ] If you split, you checked dependencies between options before firing the chain
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { describe, test, expect, beforeAll } from 'bun:test';
|
||||
import { COMMAND_DESCRIPTIONS } from '../browse/src/commands';
|
||||
import { SNAPSHOT_FLAGS } from '../browse/src/snapshot';
|
||||
import * as fs from 'fs';
|
||||
@@ -2125,6 +2125,21 @@ describe('Factory generation (--host factory)', () => {
|
||||
import { ALL_HOST_CONFIGS, getExternalHosts } from '../hosts/index';
|
||||
|
||||
describe('Parameterized host smoke tests', () => {
|
||||
// Regenerate every external host up front so the per-host `--dry-run` freshness
|
||||
// checks are deterministic. These host dirs (.agents/.factory/.cursor/...) are
|
||||
// gitignored regenerated artifacts, so the freshness check is really an
|
||||
// idempotency/determinism check — it still catches non-deterministic gen, but no
|
||||
// longer flakes on stale-on-disk state left by a missing `gen --host all` prestep
|
||||
// (the canonical `bun test` does not run one). The tracked-claude freshness test
|
||||
// (`generated files are fresh`) runs earlier and is unaffected.
|
||||
beforeAll(() => {
|
||||
for (const h of getExternalHosts()) {
|
||||
Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', h.name], {
|
||||
cwd: ROOT, stdout: 'pipe', stderr: 'pipe',
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
for (const hostConfig of getExternalHosts()) {
|
||||
describe(`${hostConfig.displayName} (--host ${hostConfig.name})`, () => {
|
||||
const hostDir = path.join(ROOT, hostConfig.hostSubdir, 'skills');
|
||||
@@ -2208,6 +2223,16 @@ describe('Parameterized host smoke tests', () => {
|
||||
// ─── --host all tests ────────────────────────────────────────
|
||||
|
||||
describe('--host all', () => {
|
||||
// Same determinism guard as the parameterized block: make external hosts fresh on
|
||||
// disk so `--host all --dry-run` reports FRESH regardless of prior state.
|
||||
beforeAll(() => {
|
||||
for (const h of getExternalHosts()) {
|
||||
Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', h.name], {
|
||||
cwd: ROOT, stdout: 'pipe', stderr: 'pipe',
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test('--host all generates for all registered hosts', () => {
|
||||
const result = Bun.spawnSync(['bun', 'run', 'scripts/gen-skill-docs.ts', '--host', 'all', '--dry-run'], {
|
||||
cwd: ROOT, stdout: 'pipe', stderr: 'pipe',
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
/**
|
||||
* gstack-session-kind — classifies the session so skills know whether a human can
|
||||
* answer an AskUserQuestion. Drives the AUQ-failure fallback branch:
|
||||
* spawned → auto-choose (orchestrator)
|
||||
* headless → BLOCK on AUQ failure
|
||||
* interactive → prose fallback on AUQ failure
|
||||
*
|
||||
* These permutations are the contract the resolver rule depends on. Run with a
|
||||
* SCRUBBED env (the test process itself runs inside Conductor, so CONDUCTOR_* /
|
||||
* CLAUDE_CODE_* would leak in and contaminate the classification).
|
||||
*
|
||||
* Free, deterministic, gate-tier.
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { execFileSync } from 'child_process';
|
||||
import * as path from 'path';
|
||||
|
||||
const BIN = path.resolve(__dirname, '..', 'bin', 'gstack-session-kind');
|
||||
|
||||
/** Run the helper with ONLY the supplied env (plus PATH so bash resolves). */
|
||||
function kind(env: Record<string, string>): string {
|
||||
return execFileSync(BIN, [], {
|
||||
env: { PATH: process.env.PATH ?? '/usr/bin:/bin', ...env },
|
||||
encoding: 'utf-8',
|
||||
}).trim();
|
||||
}
|
||||
|
||||
describe('gstack-session-kind', () => {
|
||||
test('OPENCLAW_SESSION → spawned (highest precedence)', () => {
|
||||
expect(kind({ OPENCLAW_SESSION: '1' })).toBe('spawned');
|
||||
// spawned wins even when other markers are also present
|
||||
expect(kind({ OPENCLAW_SESSION: '1', GSTACK_HEADLESS: '1', CONDUCTOR_PORT: '5' })).toBe('spawned');
|
||||
});
|
||||
|
||||
test('GSTACK_HEADLESS → headless', () => {
|
||||
expect(kind({ GSTACK_HEADLESS: '1' })).toBe('headless');
|
||||
});
|
||||
|
||||
test('CONDUCTOR_* → interactive (a human host is present)', () => {
|
||||
expect(kind({ CONDUCTOR_WORKSPACE_PATH: '/tmp/ws' })).toBe('interactive');
|
||||
expect(kind({ CONDUCTOR_PORT: '55010' })).toBe('interactive');
|
||||
});
|
||||
|
||||
test('CLAUDE_CODE_ENTRYPOINT=cli → interactive', () => {
|
||||
expect(kind({ CLAUDE_CODE_ENTRYPOINT: 'cli' })).toBe('interactive');
|
||||
});
|
||||
|
||||
test('interactive host beats CI markers', () => {
|
||||
expect(kind({ CONDUCTOR_PORT: '5', CI: '1' })).toBe('interactive');
|
||||
});
|
||||
|
||||
test('CI / GITHUB_ACTIONS with no host → headless', () => {
|
||||
expect(kind({ CI: '1' })).toBe('headless');
|
||||
expect(kind({ GITHUB_ACTIONS: 'true' })).toBe('headless');
|
||||
});
|
||||
|
||||
test('GSTACK_HEADLESS beats CONDUCTOR (explicit override wins)', () => {
|
||||
expect(kind({ GSTACK_HEADLESS: '1', CONDUCTOR_PORT: '5' })).toBe('headless');
|
||||
});
|
||||
|
||||
test('bare env → interactive (degrade-safe default)', () => {
|
||||
expect(kind({})).toBe('interactive');
|
||||
});
|
||||
|
||||
test('empty GSTACK_HEADLESS is treated as unset (interactive)', () => {
|
||||
// The resolver/helper guard on -n, so an empty string must NOT mean headless —
|
||||
// this is the opt-out path harness suites use to exercise the interactive branch.
|
||||
expect(kind({ GSTACK_HEADLESS: '' })).toBe('interactive');
|
||||
});
|
||||
});
|
||||
@@ -300,6 +300,13 @@ export async function runAgentSdkTest(
|
||||
const queryImpl: QueryProvider = opts.queryProvider ?? query;
|
||||
const model = opts.model ?? 'claude-opus-4-7';
|
||||
|
||||
// NOTE on GSTACK_HEADLESS: the SDK child inherits process.env, so headless
|
||||
// classification for eval/E2E runs is set by the `test:gate` / `test:evals`
|
||||
// package.json scripts (scoped to that invocation), NOT mutated here. We must not
|
||||
// pass sdkOpts.env (it breaks the SDK auth pipeline — see CLAUDE.md) and must not
|
||||
// mutate process.env ambiently (it would leak headless into later interactive-path
|
||||
// tests in the same Bun process — Codex review finding).
|
||||
|
||||
let attempt = 0;
|
||||
let lastErr: unknown = null;
|
||||
|
||||
|
||||
@@ -87,6 +87,12 @@ export interface CarveGuard {
|
||||
minUnionBytes: number;
|
||||
/** Parity: content phrases the union must preserve. */
|
||||
mustContain: string[];
|
||||
/**
|
||||
* Parity: optional per-skill override for the union size-growth ceiling vs the
|
||||
* v1.53.0.0 baseline (default 1.05). Bumped only when a deliberate cross-cutting
|
||||
* preamble feature legitimately grows a smaller carved skeleton past 5%.
|
||||
*/
|
||||
maxSizeRatio?: number;
|
||||
}
|
||||
|
||||
export const CARVE_GUARDS: Record<string, CarveGuard> = {
|
||||
@@ -106,8 +112,14 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
|
||||
scenario:
|
||||
'This is a FRESH version-changing ship: the branch has a real code change, VERSION still equals the base version (needs a bump), and CHANGELOG.md needs a new entry. Follow the skill flow for a version-changing ship: run the pre-landing review and prepare the CHANGELOG entry. Produce the ship plan / review report. Do NOT actually commit, push, or open a PR.',
|
||||
staticInvariants: {
|
||||
mustStayInSkeleton: [],
|
||||
mustMoveToSection: [],
|
||||
// The PR-title-version invariant MUST stay always-loaded: the v1.54.0.0
|
||||
// carve stranded it in pr-body.md and PRs started landing with bare titles
|
||||
// (CI backstop: test/pr-title-sync-workflow-safety.test.ts).
|
||||
mustStayInSkeleton: ['v$NEW_VERSION', 'gstack-pr-title-rewrite'],
|
||||
// ...while the full create/update procedure stays carved into pr-body.md
|
||||
// (out of the skeleton, present in the union). Asserts BOTH PR paths
|
||||
// survive: the create path and the idempotent update path.
|
||||
mustMoveToSection: ['gh pr create --base', 'gh pr edit --title'],
|
||||
// ship is operational (multi-STOP, not a plan review); no single post-STOP gate.
|
||||
gateAfterStop: undefined,
|
||||
},
|
||||
@@ -216,6 +228,11 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
|
||||
maxSkeletonBytes: 50_000,
|
||||
minUnionBytes: 55_000,
|
||||
mustContain: ['CHANGELOG', 'Diataxis', 'coverage'],
|
||||
// The AUQ-failure prose fallback (v1.57.2.0) adds ~2KB to every skill's
|
||||
// always-loaded preamble; on this small carved skeleton that lands at ~5.9%
|
||||
// over the pre-carve/pre-AUQ v1.53.0.0 baseline. Headroom for the
|
||||
// cross-cutting addition; all other skills keep the strict 1.05 ceiling.
|
||||
maxSizeRatio: 1.08,
|
||||
},
|
||||
'design-consultation': {
|
||||
skill: 'design-consultation',
|
||||
|
||||
@@ -252,7 +252,7 @@ const CARVED_INVARIANTS: ParityInvariant[] = Object.values(CARVE_GUARDS).map((g)
|
||||
minBytes: g.minUnionBytes,
|
||||
mustContain: g.mustContain,
|
||||
mustHaveHeadings: ['## Preamble', '## When to invoke'],
|
||||
maxSizeRatio: 1.05,
|
||||
maxSizeRatio: g.maxSizeRatio ?? 1.05,
|
||||
}));
|
||||
|
||||
export const PARITY_INVARIANTS: ParityInvariant[] = [
|
||||
|
||||
@@ -52,6 +52,9 @@ export class ClaudeAdapter implements ProviderAdapter {
|
||||
timeout: opts.timeoutMs,
|
||||
encoding: 'utf-8',
|
||||
maxBuffer: 32 * 1024 * 1024,
|
||||
// Default GSTACK_HEADLESS=1 so a benchmark run classifies as headless (an
|
||||
// AskUserQuestion failure BLOCKs rather than emitting unanswerable prose).
|
||||
env: { ...process.env, GSTACK_HEADLESS: '1' },
|
||||
});
|
||||
const parsed = this.parseOutput(out);
|
||||
return {
|
||||
|
||||
@@ -176,7 +176,11 @@ export async function runSkillTest(options: {
|
||||
|
||||
const proc = Bun.spawn(['sh', '-c', `cat "${promptFile}" | claude ${args.map(a => `"${a}"`).join(' ')}`], {
|
||||
cwd: workingDirectory,
|
||||
env: extraEnv ? { ...process.env, ...extraEnv } : undefined,
|
||||
// Default GSTACK_HEADLESS=1 so eval/E2E runs classify as headless (BLOCK on an
|
||||
// AskUserQuestion failure rather than emit a prose question no human reads). A
|
||||
// suite exercising the INTERACTIVE prose-fallback path opts out by passing
|
||||
// `env: { GSTACK_HEADLESS: '' }` — extraEnv wins because it spreads last.
|
||||
env: { ...process.env, GSTACK_HEADLESS: '1', ...extraEnv },
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe',
|
||||
});
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
/**
|
||||
* pr-title-sync.yml is a `pull_request_target` workflow — static injection
|
||||
* tripwire (gate, free).
|
||||
*
|
||||
* The anxiety this kills: `pull_request_target` runs with a WRITE token in the
|
||||
* base-repo context, even for fork PRs. That is what lets this workflow rewrite
|
||||
* fork-PR titles (the backstop). It is also the single most dangerous workflow
|
||||
* trigger in GitHub Actions. Two classic footguns turn it into remote code
|
||||
* execution / token theft, and `actionlint` catches NEITHER:
|
||||
*
|
||||
* 1. Checking out the PR head (`actions/checkout` with a `ref:` pointing at
|
||||
* `pull_request.head` / `head_ref`) and then running anything from it —
|
||||
* that executes attacker-controlled fork code with the write token.
|
||||
* 2. Interpolating an attacker-controlled `${{ github.event.pull_request.* }}`
|
||||
* field directly INSIDE a `run:` block — the title/body are attacker-
|
||||
* controlled and the `${{ }}` is expanded into the shell before execution,
|
||||
* so a crafted title runs as code. Those fields MUST arrive via `env:` and
|
||||
* be referenced as `"$VAR"` (shell-quoted), never inlined.
|
||||
*
|
||||
* This tripwire reads the workflow file directly and fails CI if either pattern
|
||||
* reappears. Mirrors the static-grep invariant tests in browse/test
|
||||
* (terminal-agent-pid-identity, server-sanitize-surrogates).
|
||||
*
|
||||
* Note: `gh api ... -q '.head.sha'` inside a run block is SAFE (reading PR
|
||||
* metadata as data via a jq filter string, not `${{ }}` interpolation), so we
|
||||
* ban the interpolation form specifically, not the literal substring `head.sha`.
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
const WORKFLOW = path.resolve(__dirname, '..', '.github', 'workflows', 'pr-title-sync.yml');
|
||||
|
||||
/** Indentation width (count of leading spaces) of a line. */
|
||||
function indent(line: string): number {
|
||||
const m = line.match(/^( *)/);
|
||||
return m ? m[1].length : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the lines that live inside a `run:` block, each tagged with its 1-based
|
||||
* line number. Handles both `run: |` (multiline) and `run: <inline command>`.
|
||||
*/
|
||||
function runBlockLines(content: string): Array<{ n: number; text: string }> {
|
||||
const lines = content.split('\n');
|
||||
const out: Array<{ n: number; text: string }> = [];
|
||||
let inRun = false;
|
||||
let runIndent = -1;
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
const n = i + 1;
|
||||
const inlineRun = line.match(/^(\s*)run:\s*(\S.*)$/); // `run: echo foo`
|
||||
const blockRun = /^(\s*)run:\s*(\|>?[+-]?)?\s*$/.test(line); // `run: |`
|
||||
if (inlineRun && !/^\|/.test(inlineRun[2])) {
|
||||
out.push({ n, text: inlineRun[2] });
|
||||
inRun = false;
|
||||
continue;
|
||||
}
|
||||
if (blockRun) {
|
||||
inRun = true;
|
||||
runIndent = indent(line);
|
||||
continue;
|
||||
}
|
||||
if (inRun) {
|
||||
if (line.trim() === '') {
|
||||
out.push({ n, text: line });
|
||||
continue;
|
||||
}
|
||||
// Block ends when a non-empty line is indented at or below the `run:` key.
|
||||
if (indent(line) <= runIndent) {
|
||||
inRun = false;
|
||||
} else {
|
||||
out.push({ n, text: line });
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
describe('pr-title-sync.yml pull_request_target safety', () => {
|
||||
const content = fs.readFileSync(WORKFLOW, 'utf-8');
|
||||
|
||||
test('workflow file exists', () => {
|
||||
expect(fs.existsSync(WORKFLOW)).toBe(true);
|
||||
});
|
||||
|
||||
test('does NOT check out the PR head ref (no fork-code execution)', () => {
|
||||
const offenders: string[] = [];
|
||||
content.split('\n').forEach((line, i) => {
|
||||
// A checkout `ref:` (or any `ref:`) pointing at the PR head is the footgun.
|
||||
if (/ref:\s*\$\{\{[^}]*(pull_request\.head|head_ref)/.test(line)) {
|
||||
offenders.push(` L${i + 1}: ${line.trim()}`);
|
||||
}
|
||||
});
|
||||
if (offenders.length > 0) {
|
||||
throw new Error(
|
||||
`pr-title-sync.yml checks out the PR head under pull_request_target — that ` +
|
||||
`runs attacker-controlled fork code with a write token. Check out the base ` +
|
||||
`repo (no ref:) and read PR-head data via the API instead.\n` +
|
||||
offenders.join('\n'),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('does NOT interpolate ${{ github.event.pull_request.* }} inside a run: block', () => {
|
||||
const offenders: string[] = [];
|
||||
for (const { n, text } of runBlockLines(content)) {
|
||||
if (/\$\{\{\s*github\.event\.pull_request/.test(text)) {
|
||||
offenders.push(` L${n}: ${text.trim()}`);
|
||||
}
|
||||
}
|
||||
if (offenders.length > 0) {
|
||||
throw new Error(
|
||||
`pr-title-sync.yml inlines an attacker-controlled PR field into a run: block ` +
|
||||
`— a crafted PR title/body executes as shell. Pass it via env: and ` +
|
||||
`reference "$VAR" (shell-quoted) instead.\n` +
|
||||
offenders.join('\n'),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('uses pull_request_target (the hardening is actually present)', () => {
|
||||
// Positive assertion: if someone reverts to plain pull_request, the fork
|
||||
// backstop silently stops working (read-only token). Keep it intentional.
|
||||
expect(/^on:\s*$/m.test(content) || /\bpull_request_target\b/.test(content)).toBe(true);
|
||||
expect(content).toMatch(/\bpull_request_target\b/);
|
||||
});
|
||||
|
||||
test('passes the PR title through env:, not raw interpolation', () => {
|
||||
// The safe pattern: OLD_TITLE: ${{ github.event.pull_request.title }} in an
|
||||
// env: mapping, consumed as "$OLD_TITLE" in script.
|
||||
expect(content).toMatch(/env:/);
|
||||
expect(content).toMatch(/github\.event\.pull_request\.title/);
|
||||
});
|
||||
});
|
||||
@@ -16,6 +16,8 @@
|
||||
* for the weekly periodic eval to notice.
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import type { TemplateContext } from '../scripts/resolvers/types';
|
||||
import { HOST_PATHS } from '../scripts/resolvers/types';
|
||||
import { generateAskUserFormat } from '../scripts/resolvers/preamble/generate-ask-user-format';
|
||||
@@ -161,3 +163,88 @@ describe('generateAskUserFormat — 5+ option split rule (slim inline + docs poi
|
||||
expect(out).toContain('**Non-ASCII characters');
|
||||
});
|
||||
});
|
||||
|
||||
describe('generateAskUserFormat — runtime-failure prose fallback', () => {
|
||||
const out = generateAskUserFormat(makeCtx());
|
||||
|
||||
test('documents the unavailable/failed subsection', () => {
|
||||
expect(out).toMatch(/When AskUserQuestion is unavailable or a call fails/i);
|
||||
});
|
||||
|
||||
test('carves out the auto-decide denial as NOT a failure', () => {
|
||||
expect(out).toContain('[plan-tune auto-decide]');
|
||||
expect(out).toMatch(/NOT a failure/i);
|
||||
// and explicitly: do not fall back to prose on an auto-decide denial
|
||||
expect(out).toMatch(/Do NOT[\s\S]{0,40}fall back to prose|never prose/i);
|
||||
});
|
||||
|
||||
test('retries the errored call exactly once before degrading', () => {
|
||||
expect(out).toMatch(/retry the SAME call \*\*once\*\*|retry the same call.*once/i);
|
||||
// idempotency guard against double-prompting
|
||||
expect(out).toMatch(/double-prompt|no answer could have surfaced/i);
|
||||
});
|
||||
|
||||
test('branches on SESSION_KIND: spawned / headless / interactive', () => {
|
||||
expect(out).toContain('SESSION_KIND');
|
||||
expect(out).toMatch(/`spawned`[\s\S]*auto-choose/);
|
||||
expect(out).toMatch(/`headless`[\s\S]*BLOCKED/);
|
||||
expect(out).toMatch(/`interactive`[\s\S]*prose fallback/);
|
||||
// empty/absent SESSION_KIND degrades to interactive
|
||||
expect(out).toMatch(/empty\/absent[\s\S]{0,40}interactive/i);
|
||||
});
|
||||
|
||||
// The mandatory triad the user explicitly required for the plain-text output.
|
||||
test('prose fallback mandates the triad: issue ELI10', () => {
|
||||
expect(out).toMatch(/ELI10 of the issue itself/i);
|
||||
});
|
||||
|
||||
test('prose fallback mandates the triad: per-choice Completeness score', () => {
|
||||
expect(out).toMatch(/Completeness scores per choice/i);
|
||||
expect(out).toMatch(/Completeness: X\/10.*EACH choice|on EACH choice/i);
|
||||
});
|
||||
|
||||
test('prose fallback mandates the triad: recommendation + (recommended) marker', () => {
|
||||
expect(out).toMatch(/Recommendation: <choice> because/);
|
||||
expect(out).toMatch(/\(recommended\)`? marker on that choice/);
|
||||
});
|
||||
|
||||
test('prose fallback is one paragraph per choice, not a bare bullet list', () => {
|
||||
expect(out).toMatch(/ONE paragraph per choice/i);
|
||||
expect(out).toMatch(/never a bare bullet list/i);
|
||||
});
|
||||
|
||||
test('prose fallback tells the user to reply with a letter, then STOP', () => {
|
||||
expect(out).toMatch(/reply with a letter/i);
|
||||
expect(out).toMatch(/STOP and wait/i);
|
||||
});
|
||||
|
||||
// OV2: the former "tool_use, not prose" assertions must carry the qualifier so the
|
||||
// fallback is not self-contradicting. Guards against the instruction collision
|
||||
// silently returning on a future edit.
|
||||
test('OV2: the Format line qualifies "not prose" with the fallback exception', () => {
|
||||
expect(out).toMatch(/must be sent as tool_use, not prose — unless the documented failure fallback/);
|
||||
});
|
||||
|
||||
test('OV2: the self-check "not writing prose" line carries the fallback qualifier', () => {
|
||||
expect(out).toMatch(/not writing prose — unless the documented failure fallback applies/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('CQ2 — cross-file invariant: auto-decide prefix matches the hook', () => {
|
||||
const out = generateAskUserFormat(makeCtx());
|
||||
const hookSrc = fs.readFileSync(
|
||||
path.resolve(__dirname, '..', 'hosts', 'claude', 'hooks', 'question-preference-hook.ts'),
|
||||
'utf-8',
|
||||
);
|
||||
|
||||
test('the hook actually emits the [plan-tune auto-decide] prefix', () => {
|
||||
expect(hookSrc).toContain('[plan-tune auto-decide]');
|
||||
});
|
||||
|
||||
test('the resolver references the exact same prefix the hook emits', () => {
|
||||
// If a future edit reworded the hook reason, this catches the drift: the prose
|
||||
// fallback would stop recognizing the auto-decide denial as not-a-failure.
|
||||
const PREFIX = '[plan-tune auto-decide]';
|
||||
expect(hookSrc.includes(PREFIX) && out.includes(PREFIX)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user