CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-09-22 21:00:51 +02:00
Code Issues Packages Projects Releases Wiki Activity

v1.87.5.0 perf: remove idle waits from tests and CI planning (#2897)

Browse Source 
* v1.87.5.0 perf: remove idle waits from tests and CI planning

* fix: settle split PTY redraws before routing input

* docs: record final burst-safe test benchmarks

* fix: keep cold-setup snapshot metadata dependency-free

* fix: avoid early-reader pipe races in artifact URL parsing

* fix: preserve safety matches for multiline command payloads

* fix: recognize concurrent CSO publication removal

* test: preload the UI design-review target before invocation

* docs: record validation blocker fixes

* fix: bind plan observer rejection to the invoked command

* fix: count only native design decisions in the UI gate

* docs: clarify UI-positive eval evidence requirements

* test: recognize native UI decisions without weakening finding counts

* test: decouple native UI evidence from question punctuation

* test: recognize concrete native UI decisions independently of prose format

* fix: retain failed eval logs under the hidden CI cache

* test: await telemetry completion instead of racing disk writes
This commit is contained in:
Garry Tan
2026-09-21 12:27:25 -04:00
committed by GitHub
parent a6b3a57512
commit 35dd014c58
42 changed files with 1583 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/skill-e2e-plan-design-with-ui.test.ts
+31 -114
View File
@@ -11,25 +11,21 @@
* exit — would pass the no-UI test (vacuously) and ship undetected. This
* test is the positive coverage.
*
* How: launch claude in plan mode in the gstack repo cwd (so the skill
* registry is loaded). Send /plan-design-review with the fixture path
* inline so the skill reviews the UI-heavy plan rather than git diff or
* .claude/plans/. Drive past permission dialogs. Wait for a numbered-
* option list that is NOT a permission dialog. Assert evidence does NOT
* contain "no UI scope".
*/
import { test } from 'bun:test';
import { PTY_MS } from './helpers/eval-budgets';
import { describeE2ETier } from './helpers/e2e-gate';
import * as fs from 'node:fs';
import * as path from 'path';
import {
launchClaudePty,
isNumberedOptionListVisible,
isPermissionDialogVisible,
parseNumberedOptions,
isPlanReadyVisible,
runPlanSkillCounting,
designStep0Boundary,
nativePlanCallFingerprint,
} from './helpers/claude-pty-runner';
import { isDesignCountSetup, isDesignCompletionHandoff, pickDesignCountQuestion } from './helpers/design-count-review';
import { isDesignArtifactGeneration } from './helpers/design-artifact-question';
import { isDesignUIScopeReview } from './helpers/design-ui-scope';
const describeE2E = describeE2ETier('gate');
@@ -40,116 +36,37 @@ describeE2E('/plan-design-review with UI scope (gate)', () => {
test(
'reaches a real skill AskUserQuestion (or plan_ready) without echoing the no-UI early-exit phrase',
async () => {
const fixtureRelPath = path.relative(ROOT, FIXTURE);
const session = await launchClaudePty({
permissionMode: 'plan',
// LIVE-REPO CWD: PTY session needs the repo cwd — skill registry,
// hermetic pre-trusted dir, and the repo-relative fixture path above.
cwd: ROOT,
timeoutMs: PTY_MS,
seedSkills: true,
const observation = await runPlanSkillCounting({
skillName: 'plan-design-review',
slashCommand: '/plan-design-review PLAN.md',
followUpPrompt: fs.readFileSync(FIXTURE, 'utf8'),
isLastStep0AUQ: designStep0Boundary,
isFirstReviewAUQ: isDesignUIScopeReview,
isReviewAUQ: isDesignUIScopeReview,
isSetupAUQ: isDesignCountSetup,
isCompletionHandoffAUQ: isDesignCompletionHandoff,
isArtifactGenerationAUQ: isDesignArtifactGeneration,
pickAUQ: pickDesignCountQuestion,
reviewCountCeiling: 1,
timeoutMs: 600_000,
});
let outcome: 'real_question' | 'plan_ready' | 'timeout' | 'exited' = 'timeout';
let evidence = '';
let debugBuffer = ''; // captured at end so timeout error has data
try {
await Bun.sleep(8000);
const since = session.mark();
// Send the slash command alone first; then provide the UI-heavy
// plan content as a follow-up message. Claude Code rejects slash
// commands with trailing arguments unless the skill defines them.
session.send('/plan-design-review\r');
await Bun.sleep(3000);
session.send(
`Please review this plan for UI scope:\n\n` +
`Title: User Dashboard Page\n` +
`New React page UserDashboard.tsx with three subcomponents: ` +
`ActivityFeed, NotificationsPanel, QuickActions. ` +
`Tailwind CSS responsive layout (mobile/desktop breakpoints), ` +
`loading skeletons, empty states, hover states on every interactive element, ` +
`modal dialog for "mark all read", toast notifications for action feedback. ` +
`Reference plan file: ${fixtureRelPath}\r`
);
// 600s, not 360s: the skill preamble (update-check, session bookkeeping,
// learnings) plus extended model thinking can take ~6 minutes before the
// scope-gate AskUserQuestion renders — a 360s budget expired seconds
// before the (correct) AUQ appeared in the observed failure transcript.
const budgetMs = 600_000;
const start = Date.now();
let lastPermSig = '';
while (Date.now() - start < budgetMs) {
await Bun.sleep(2500);
if (session.exited()) {
outcome = 'exited';
evidence = session.visibleSince(since).slice(-3000);
break;
}
const visible = session.visibleSince(since);
// Classify the recent tail only — old permission text persists
// in visibleSince(since) and would otherwise re-trigger forever.
// 5KB window: plan-design-review Step 0 renders a numbered AUQ with
// box dividers + per-option descriptions + footer prompt. The full
// rendering frequently exceeds 2.5KB, especially after TTY cursor-
// positioning escapes resolve through stripAnsi. A 2.5KB tail can
// capture the cursor `❯1.` line without capturing the line that has
// `2.`, defeating isNumberedOptionListVisible. 5KB comfortably
// covers the full AUQ block without including stale scrollback.
const recentTail = visible.slice(-5000);
// Real skill AskUserQuestion visible (not a permission dialog)?
if (
isNumberedOptionListVisible(recentTail) &&
parseNumberedOptions(recentTail).length >= 2 &&
!isPermissionDialogVisible(recentTail)
) {
outcome = 'real_question';
evidence = visible.slice(-3000);
break;
}
// Permission dialog: grant once per unique rendering.
if (isPermissionDialogVisible(recentTail)) {
const sig = visible.slice(-500);
if (sig !== lastPermSig) {
lastPermSig = sig;
session.send('1\r');
await Bun.sleep(1500);
continue;
}
}
// Plan-ready terminal — also acceptable (skill ran end-to-end
// and surfaced claude's "Ready to execute" prompt).
if (isPlanReadyVisible(visible)) {
outcome = 'plan_ready';
evidence = visible.slice(-3000);
break;
}
}
// Capture buffer state at end so a timeout error has diagnostic data.
debugBuffer = session.visibleSince(since).slice(-4000);
} finally {
await session.close();
}
// PASS: real_question or plan_ready, AND evidence does NOT echo the
// early-exit phrase.
if (outcome === 'exited' || outcome === 'timeout') {
const designQuestionObserved = observation.fingerprints.some(fp =>
!fp.preReview && !fp.administrative && fp.nativeCall &&
isDesignUIScopeReview(nativePlanCallFingerprint(fp.nativeCall, fp.observedAtMs, fp.preReview)));
if ((observation.outcome !== 'ceiling_reached' && observation.outcome !== 'plan_ready') ||
observation.reviewCount < 1 || !designQuestionObserved) {
throw new Error(
`plan-design-review with UI scope FAILED: outcome=${outcome}\n` +
`--- buffer at timeout (last 4KB) ---\n${debugBuffer || evidence}`,
`plan-design-review with UI scope FAILED: outcome=${observation.outcome}\n` +
`step0=${observation.step0Count} review=${observation.reviewCount}\n` +
`${observation.summary}\n--- questions ---\n${JSON.stringify(observation.fingerprints, null, 2)}\n` +
`--- evidence ---\n${observation.evidence}`,
);
}
const NO_UI_PHRASE = /no\s+UI\s+scope|isn'?t\s+applicable/i;
if (NO_UI_PHRASE.test(evidence)) {
if (NO_UI_PHRASE.test(observation.evidence)) {
throw new Error(
`plan-design-review early-exited despite UI-heavy fixture.\n` +
`--- evidence (last 3KB) ---\n${evidence}`,
`--- evidence (last 3KB) ---\n${observation.evidence}`,
);
}
},