mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-22 04:40:44 +02:00
Garry Tan
35dd014c58
* v1.87.5.0 perf: remove idle waits from tests and CI planning * fix: settle split PTY redraws before routing input * docs: record final burst-safe test benchmarks * fix: keep cold-setup snapshot metadata dependency-free * fix: avoid early-reader pipe races in artifact URL parsing * fix: preserve safety matches for multiline command payloads * fix: recognize concurrent CSO publication removal * test: preload the UI design-review target before invocation * docs: record validation blocker fixes * fix: bind plan observer rejection to the invoked command * fix: count only native design decisions in the UI gate * docs: clarify UI-positive eval evidence requirements * test: recognize native UI decisions without weakening finding counts * test: decouple native UI evidence from question punctuation * test: recognize concrete native UI decisions independently of prose format * fix: retain failed eval logs under the hidden CI cache * test: await telemetry completion instead of racing disk writes
76 lines
3.3 KiB
TypeScript
76 lines
3.3 KiB
TypeScript
/**
|
|
* /plan-design-review with UI scope (gate, paid, real-PTY).
|
|
*
|
|
* Counterpart to the existing no-UI early-exit test. When the input plan
|
|
* DOES describe UI changes, /plan-design-review must NOT early-exit and
|
|
* must reach a real skill numbered-option AskUserQuestion (its first design-rating
|
|
* question), with the captured evidence NOT echoing the early-exit phrase.
|
|
*
|
|
* Why: today we only test the negative path (no-UI → early-exit). A
|
|
* regression that flips the UI-detection logic — making EVERY plan early-
|
|
* exit — would pass the no-UI test (vacuously) and ship undetected. This
|
|
* test is the positive coverage.
|
|
*
|
|
*/
|
|
|
|
import { test } from 'bun:test';
|
|
import { PTY_MS } from './helpers/eval-budgets';
|
|
import { describeE2ETier } from './helpers/e2e-gate';
|
|
import * as fs from 'node:fs';
|
|
import * as path from 'path';
|
|
import {
|
|
runPlanSkillCounting,
|
|
designStep0Boundary,
|
|
nativePlanCallFingerprint,
|
|
} from './helpers/claude-pty-runner';
|
|
import { isDesignCountSetup, isDesignCompletionHandoff, pickDesignCountQuestion } from './helpers/design-count-review';
|
|
import { isDesignArtifactGeneration } from './helpers/design-artifact-question';
|
|
import { isDesignUIScopeReview } from './helpers/design-ui-scope';
|
|
|
|
const describeE2E = describeE2ETier('gate');
|
|
|
|
const ROOT = path.resolve(import.meta.dir, '..');
|
|
const FIXTURE = path.join(ROOT, 'test', 'fixtures', 'plans', 'ui-heavy-feature.md');
|
|
|
|
describeE2E('/plan-design-review with UI scope (gate)', () => {
|
|
test(
|
|
'reaches a real skill AskUserQuestion (or plan_ready) without echoing the no-UI early-exit phrase',
|
|
async () => {
|
|
const observation = await runPlanSkillCounting({
|
|
skillName: 'plan-design-review',
|
|
slashCommand: '/plan-design-review PLAN.md',
|
|
followUpPrompt: fs.readFileSync(FIXTURE, 'utf8'),
|
|
isLastStep0AUQ: designStep0Boundary,
|
|
isFirstReviewAUQ: isDesignUIScopeReview,
|
|
isReviewAUQ: isDesignUIScopeReview,
|
|
isSetupAUQ: isDesignCountSetup,
|
|
isCompletionHandoffAUQ: isDesignCompletionHandoff,
|
|
isArtifactGenerationAUQ: isDesignArtifactGeneration,
|
|
pickAUQ: pickDesignCountQuestion,
|
|
reviewCountCeiling: 1,
|
|
timeoutMs: 600_000,
|
|
});
|
|
const designQuestionObserved = observation.fingerprints.some(fp =>
|
|
!fp.preReview && !fp.administrative && fp.nativeCall &&
|
|
isDesignUIScopeReview(nativePlanCallFingerprint(fp.nativeCall, fp.observedAtMs, fp.preReview)));
|
|
if ((observation.outcome !== 'ceiling_reached' && observation.outcome !== 'plan_ready') ||
|
|
observation.reviewCount < 1 || !designQuestionObserved) {
|
|
throw new Error(
|
|
`plan-design-review with UI scope FAILED: outcome=${observation.outcome}\n` +
|
|
`step0=${observation.step0Count} review=${observation.reviewCount}\n` +
|
|
`${observation.summary}\n--- questions ---\n${JSON.stringify(observation.fingerprints, null, 2)}\n` +
|
|
`--- evidence ---\n${observation.evidence}`,
|
|
);
|
|
}
|
|
const NO_UI_PHRASE = /no\s+UI\s+scope|isn'?t\s+applicable/i;
|
|
if (NO_UI_PHRASE.test(observation.evidence)) {
|
|
throw new Error(
|
|
`plan-design-review early-exited despite UI-heavy fixture.\n` +
|
|
`--- evidence (last 3KB) ---\n${observation.evidence}`,
|
|
);
|
|
}
|
|
},
|
|
PTY_MS,
|
|
);
|
|
});
|