CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-09-22 12:50:50 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix(pty-runner): positional floor exclusion, flag builder, outcome union, token tracking

Browse Source 
Review-army + adversarial findings on the scope-gate observability work,
all verified before fixing:

- Floor check: acceptance scanned the CUMULATIVE buffer while the scope-gate
  exclusion scanned only the 1500-byte tail, so an early gate render satisfied
  the floor vacuously once ~1.5KB of output accumulated (found independently
  by 4 review passes; predicate reproduced). Acceptance now scans only content
  APPENDED after the first gate render (positional anchor), and the LLM-judge
  'waiting' shortcut no longer fires while the gate menu is the pending render.
- High-water flags are built once and spread at every return path — the
  hand-spread pattern had already drifted (judge-waiting return omitted two
  flags), which made must-stay-false asserts vacuous on those paths.
- isScopeGateAutoSelectVisible: tense-tolerant selected/selecting/selects
  token (must-be-TRUE asserts shouldn't fail semantically-perfect paraphrases)
  and quoted-occurrence rejection (a model verbatim-quoting the announcement
  while declining must not trip must-stay-FALSE asserts). Fixtures added for
  both directions.
- PlanSkillObservation outcome union gains 'wrote_findings_before_asking'
  (returned at runtime via classifyVisible but missing from the type).
- trackTokens/tokensObserved: cumulative-buffer token high-water for
  consumption asserts (the 2KB evidence tail is lossy and the plan-file
  fallback is unreachable outside plan mode).
- New scope-gate-floor unit pins (from the ship coverage audit): both gate
  render forms trip acceptance and exclusion; a genuine finding AUQ is not
  excluded; tail-scoping semantics pinned.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-08-12 09:09:32 -07:00
co-authored by Claude Fable 5
parent 57c53e7913
commit 4e61233021
3 changed files with 236 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/helpers/claude-pty-runner.ts
+87 -30
View File
@@ -644,15 +644,27 @@ export function isScopeGateQuestionVisible(visible: string): boolean {
/**
* True when the plan-mode auto-select announcement is rendered:
* "Scope gate: plan mode — auto-selected B (reviewing <target>)."
* Requires BOTH the announcement prefix and the selected-B token so
* narration ("in plan mode I'd auto-select B") stays false.
* Requires BOTH the announcement prefix and an auto-select-B token so
* narration ("in plan mode I'd auto-select B") stays false. The token is
* tense-tolerant (selected/selecting/selects) because the smokes assert
* must-be-TRUE on it — a semantically-perfect paraphrase must not fail a
* paid run — while the prefix stays exact so paraphrase narration without
* the announcement frame stays false. A prefix immediately preceded by a
* quote character is a QUOTATION (e.g. the model explaining why it is NOT
* announcing), not a render — the announcement line itself never renders
* quoted.
*/
export function isScopeGateAutoSelectVisible(visible: string): boolean {
const squished = visible.replace(/\s+/g, '').toLowerCase();
return (
squished.includes('scopegate:planmode') &&
(squished.includes('auto-selectedb') || squished.includes('autoselectedb'))
);
const QUOTES = ['"', "'", '`', '“', '‘'];
const re = /scopegate:planmode/g;
let m: RegExpExecArray | null;
while ((m = re.exec(squished)) !== null) {
const before = m.index > 0 ? squished[m.index - 1]! : '';
if (QUOTES.includes(before)) continue; // quoted occurrence — narration, keep scanning
if (/auto-?select(?:ed|ing|s)?b/.test(squished.slice(m.index))) return true;
}
return false;
}
/**
@@ -1513,10 +1525,20 @@ export interface PlanSkillObservation {
* "Ready to execute" confirmation
* - 'silent_write' — a Write/Edit landed BEFORE any prompt, to a path
* outside the sanctioned plan/project directories
* - 'wrote_findings_before_asking' — strictPlanWrites only (seeded runs):
* the plan file was rewritten with findings before any
* AskUserQuestion render (the May-2026 transcript bug)
* - 'exited' — claude process died before any of the above
* - 'timeout' — none of the above within budget
*/
outcome: 'asked' | 'auto_decided' | 'plan_ready' | 'silent_write' | 'exited' | 'timeout';
outcome:
| 'asked'
| 'auto_decided'
| 'plan_ready'
| 'silent_write'
| 'wrote_findings_before_asking'
| 'exited'
| 'timeout';
/** Human-readable summary. */
summary: string;
/** Visible terminal text since the slash command was sent (last 2KB). */
@@ -1567,6 +1589,14 @@ export interface PlanSkillObservation {
* plan-mode smokes assert true; the no-op regression asserts false.
*/
scopeGateAutoSelectObserved?: boolean;
/**
* High-water map for opts.trackTokens: token → did it EVER appear in the
* cumulative visible buffer? Consumption asserts (e.g. "the pasted target's
* distinctive token shows up in the review output") must not depend on the
* lossy 2KB evidence tail — plan-file fallbacks are unreachable outside
* plan mode (extractPlanFilePath only matches plan-mode save renders).
*/
tokensObserved?: Record<string, boolean>;
}
/**
@@ -1627,6 +1657,10 @@ export async function runPlanSkillObservation(opts: {
/** Override the spawned model. Defaults via launchClaudePty's chain
* (opts.model ?? EVALS_MODEL ?? 'claude-sonnet-4-6'). */
model?: string;
/** Literal tokens to track as high-water marks over the CUMULATIVE visible
* buffer (case-sensitive). Results land in obs.tokensObserved. Use for
* consumption asserts that must survive the 2KB evidence tail. */
trackTokens?: string[];
}): Promise<PlanSkillObservation> {
const startedAt = Date.now();
const session = await launchClaudePty({
@@ -1672,6 +1706,19 @@ export async function runPlanSkillObservation(opts: {
let waitingEverObserved = false;
let scopeGateQuestionObserved = false;
let scopeGateAutoSelectObserved = false;
const tokensObserved: Record<string, boolean> = {};
for (const t of opts.trackTokens ?? []) tokensObserved[t] = false;
// Single source for the high-water flags at EVERY return site. Hand-
// spreading them per-site already drifted once (the judge-waiting return
// omitted the prose/waiting flags); a site that forgets a must-stay-false
// flag makes `obs.flag ?? false` negative assertions pass vacuously.
const highWaterFlags = () => ({
proseAUQEverObserved,
waitingEverObserved,
scopeGateQuestionObserved,
scopeGateAutoSelectObserved,
...(opts.trackTokens?.length ? { tokensObserved } : {}),
});
const JUDGE_AFTER_MS = 60_000;
const JUDGE_INTERVAL_MS = 30_000;
while (Date.now() - start < budgetMs) {
@@ -1684,8 +1731,7 @@ export async function runPlanSkillObservation(opts: {
summary: `claude exited (code=${session.exitCode()}) before reaching a terminal outcome`,
evidence: visible.slice(-2000),
elapsedMs: Date.now() - startedAt,
scopeGateQuestionObserved,
scopeGateAutoSelectObserved,
...highWaterFlags(),
};
}
if (visible.includes('Unknown command:')) {
@@ -1694,8 +1740,7 @@ export async function runPlanSkillObservation(opts: {
summary: `claude rejected /${opts.skillName} as unknown command (skill not registered in this cwd)`,
evidence: visible.slice(-2000),
elapsedMs: Date.now() - startedAt,
scopeGateQuestionObserved,
scopeGateAutoSelectObserved,
...highWaterFlags(),
};
}
@@ -1718,6 +1763,9 @@ export async function runPlanSkillObservation(opts: {
if (!scopeGateAutoSelectObserved && isScopeGateAutoSelectVisible(visible)) {
scopeGateAutoSelectObserved = true;
}
for (const t of opts.trackTokens ?? []) {
if (!tokensObserved[t] && visible.includes(t)) tokensObserved[t] = true;
}
const classified = classifyVisible(visible, {
strictPlanWrites: !!opts.initialPlanContent,
@@ -1727,10 +1775,7 @@ export async function runPlanSkillObservation(opts: {
...classified,
evidence: visible.slice(-2000),
elapsedMs: Date.now() - startedAt,
proseAUQEverObserved,
waitingEverObserved,
scopeGateQuestionObserved,
scopeGateAutoSelectObserved,
...highWaterFlags(),
};
// Capture the plan file path on any outcome where one may have been
// written. Gating only on 'plan_ready' missed two cases: (1) the
@@ -1761,8 +1806,7 @@ export async function runPlanSkillObservation(opts: {
summary: `LLM judge: ${lastJudgeVerdict.reasoning} (state=waiting after ${Math.round(elapsed / 1000)}s)`,
evidence: visible.slice(-2000),
elapsedMs: Date.now() - startedAt,
scopeGateQuestionObserved,
scopeGateAutoSelectObserved,
...highWaterFlags(),
};
}
}
@@ -1784,10 +1828,7 @@ export async function runPlanSkillObservation(opts: {
: ''),
evidence: finalVisible.slice(-2000),
elapsedMs: Date.now() - startedAt,
proseAUQEverObserved,
waitingEverObserved,
scopeGateQuestionObserved,
scopeGateAutoSelectObserved,
...highWaterFlags(),
};
}
return {
@@ -1799,10 +1840,7 @@ export async function runPlanSkillObservation(opts: {
: ''),
evidence: finalVisible.slice(-2000),
elapsedMs: Date.now() - startedAt,
proseAUQEverObserved,
waitingEverObserved,
scopeGateQuestionObserved,
scopeGateAutoSelectObserved,
...highWaterFlags(),
};
} finally {
await session.close();
@@ -2173,11 +2211,23 @@ export async function runPlanSkillFloorCheck(opts: {
const start = Date.now();
let lastJudgeAt = 0;
let lastJudgeVerdict: PtyStateVerdict | null = null;
// Positional anchor for the scope-gate exclusion. The visible buffer is
// append-only (old renders never leave scrollback), so a gate question
// rendered in the 3s pre-target window would keep satisfying the
// full-buffer acceptance checks forever while a tail-only exclusion
// stops seeing it after ~TAIL_SCAN_BYTES of output — a vacuous
// auq_observed (found independently by 4 review passes). Once the gate
// render is seen, acceptance only counts AUQ renders in content APPENDED
// after that point.
let gateSeenIdx = -1;
const JUDGE_AFTER_MS = 60_000;
const JUDGE_INTERVAL_MS = 30_000;
while (Date.now() - start < timeoutMs) {
await Bun.sleep(2000);
const visible = session.visibleSince(since);
if (gateSeenIdx === -1 && isScopeGateQuestionVisible(visible)) {
gateSeenIdx = visible.length;
}
if (session.exited()) {
return {
@@ -2206,12 +2256,16 @@ export async function runPlanSkillFloorCheck(opts: {
//
// Scope-gate renders do NOT count: the gate's "What should I review?"
// can fire inside the 3s pre-target window and would trivially satisfy
// the floor, but the floor measures FINDING-driven questions. The
// exclusion is TAIL-scoped so an early gate render that has scrolled
// out doesn't suppress a later, real finding AUQ.
// the floor, but the floor measures FINDING-driven questions. Once a
// gate render has been seen, acceptance scans only the content APPENDED
// after it (positional anchor above) — the buffer is append-only, so a
// whole-buffer acceptance would keep matching the stale gate render
// forever. The tail exclusion additionally covers the window where the
// gate menu is still the active render.
const tail = visible.slice(-TAIL_SCAN_BYTES);
const acceptWindow = gateSeenIdx === -1 ? visible : visible.slice(gateSeenIdx);
if (
(isNumberedOptionListVisible(visible) || isProseAUQVisible(visible)) &&
(isNumberedOptionListVisible(acceptWindow) || isProseAUQVisible(acceptWindow)) &&
!isPermissionDialogVisible(tail) &&
!isScopeGateQuestionVisible(tail)
) {
@@ -2235,7 +2289,10 @@ export async function runPlanSkillFloorCheck(opts: {
lastJudgeAt = Date.now();
logPtySnapshot(visible, { testName: opts.skillName, elapsedMs: elapsed, tag: 'floor-judge-tick' });
lastJudgeVerdict = judgePtyState(visible, { testName: opts.skillName });
if (lastJudgeVerdict.state === 'waiting') {
// The judge can't tell a scope-gate question from a finding question,
// so a 'waiting' verdict while the gate menu is the pending render
// must NOT satisfy the floor — same exclusion as the regex path.
if (lastJudgeVerdict.state === 'waiting' && !isScopeGateQuestionVisible(tail)) {
return {
auqObserved: true,
outcome: 'auq_observed',