mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-15 01:15:29 +02:00
test: register wave-3 carves (qa, browse, retro) — guards, touchfiles, pins, baselines
CARVE_GUARDS gains the three entries; qa's monolith invariant retires. auq-format carve-safety now keys on the skeleton+sections union shipping the AUQ block (first tier-1 carve: browse never renders it by design). Baselines: parity v1.69.1.0 at 18 sectioned skills; ratchet recaptured. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
co-authored by
Claude Fable 5
Claude Fable 5
parent
e0250aa128
commit
07b225a40c
@@ -550,6 +550,77 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
|
||||
mustContain: ['PGLite', 'Supabase', 'claude mcp add', 'read_secret_to_env', 'pooler'],
|
||||
maxSizeRatio: 1.07, // measured 1.051 vs the branch monolith: index + stubs + 4 STOP pointers
|
||||
},
|
||||
// ── Token-reduction Phase 4 wave 3 (v1.69.x branch) ──────────────────────
|
||||
qa: {
|
||||
skill: 'qa',
|
||||
expectedSections: ['test-bootstrap.md', 'qa-patterns.md'],
|
||||
requiredReads: ['qa-patterns.md'],
|
||||
scenario:
|
||||
'Walk /qa in SIMULATION — do not launch a browser, run any $B command, or execute bash; treat the working tree as clean, the tier as Quick, and the target app as http://localhost:3000 with a small feature-branch diff touching one page. Skip the test-framework bootstrap (assume CLAUDE.md documents the test command). Read each pointed section before doing its step, then produce the QA plan as the report: the mode you selected and why, the Phase 1-6 steps you would run, and a worked health-score computation from the rubric. Do NOT use AskUserQuestion.',
|
||||
staticInvariants: {
|
||||
mustStayInSkeleton: [
|
||||
'## Setup',
|
||||
'## SETUP (run this check BEFORE any browse command)',
|
||||
'## Phases 1-6: QA Baseline',
|
||||
'## Phase 7: Triage',
|
||||
'## Phase 8: Fix Loop',
|
||||
'8e.5. Regression Test',
|
||||
'WTF-LIKELIHOOD',
|
||||
'## Additional Rules (qa-specific)',
|
||||
'## Output Structure',
|
||||
],
|
||||
mustPrecedeStop: ['## Setup'],
|
||||
mustMoveToSection: [
|
||||
'## Test Framework Bootstrap',
|
||||
'BOOTSTRAP_DECLINED',
|
||||
'## Health Score Rubric',
|
||||
'### Diff-aware (automatic when on a feature branch with no URL)',
|
||||
'Never refuse to use the browser',
|
||||
],
|
||||
gateAfterStop: undefined,
|
||||
},
|
||||
behavioral: 'prompt',
|
||||
maxSkeletonBytes: 48_750, // Phase 4 wave 3; measured 48,151
|
||||
minUnionBytes: 69_500, // measured union 70,385
|
||||
mustContain: ['bug', 'browse', 'fix', 'Health Score Rubric', 'regression'],
|
||||
},
|
||||
browse: {
|
||||
skill: 'browse',
|
||||
expectedSections: ['command-list.md'],
|
||||
requiredReads: ['command-list.md'],
|
||||
scenario:
|
||||
'QA a static page: before driving it, plan the full audit — enumerate which browse commands and snapshot flags you would use, including extraction/tab/dialog commands beyond the Most-Used table, reading the full command reference first. Do not launch the browser or run any $B command; produce the command plan as the report.',
|
||||
staticInvariants: {
|
||||
mustStayInSkeleton: ['## SETUP', '## Core QA Patterns', '## CSS Inspector', '## Most-Used Commands'],
|
||||
mustPrecedeStop: ['## SETUP'],
|
||||
mustMoveToSection: ['## Full Command List', '## Snapshot Flags', '### Navigation'],
|
||||
gateAfterStop: undefined,
|
||||
},
|
||||
behavioral: 'prompt',
|
||||
maxSkeletonBytes: 27_500, // Phase 4 wave 3; measured 26,875
|
||||
minUnionBytes: 39_500, // measured union 41,115
|
||||
// 'BEGIN/END UNTRUSTED EXTERNAL' pins the untrusted-content warning; the full
|
||||
// envelope phrase wraps across lines in the rendered blockquote, so the
|
||||
// contiguous-substring check needs the single-line prefix form.
|
||||
mustContain: ['BEGIN/END UNTRUSTED EXTERNAL', 'snapshot -i', '@e refs', 'deviceScaleFactor', 'handoff'],
|
||||
},
|
||||
retro: {
|
||||
skill: 'retro',
|
||||
expectedSections: ['report-format.md'],
|
||||
requiredReads: ['report-format.md'],
|
||||
scenario:
|
||||
'Run the repo-scoped weekly retrospective for the last 7 days on this repo. There is no origin remote — proceed with the local branch per the guard disclosure rules. The gstack-retro-metrics script is not installed, so follow the degraded path (compute the metrics manually with git). Skip any AskUserQuestion calls — this is non-interactive. Produce the full narrative retrospective report.',
|
||||
staticInvariants: {
|
||||
mustStayInSkeleton: ['gstack-retro-metrics', '### Step 2: Compute Metrics', '### Step 13: Save Retro History'],
|
||||
mustPrecedeStop: ['### Step 2: Compute Metrics'],
|
||||
mustMoveToSection: ['## Engineering Retro: [date range]', '### Team Breakdown', 'Plan Completion This Period'],
|
||||
gateAfterStop: undefined,
|
||||
},
|
||||
behavioral: 'prompt',
|
||||
maxSkeletonBytes: 69_500, // Phase 4 wave 3; measured 68,483 (script absorption -5.4KB)
|
||||
minUnionBytes: 66_000, // measured union 73,496
|
||||
mustContain: ['retrospective', '45-minute gap', 'Ship of the week', 'Praise'],
|
||||
},
|
||||
};
|
||||
|
||||
/** Sorted carved-skill names. Consumers derive their lists from this — no parallel lists. */
|
||||
|
||||
@@ -206,21 +206,8 @@ export function runParityChecks(opts: {
|
||||
*/
|
||||
const MONOLITH_INVARIANTS: ParityInvariant[] = [
|
||||
// cso is now carved — its invariant is generated from CARVE_GUARDS below.
|
||||
// review, codex, land-and-deploy (wave 1) and autoplan (wave 2) carved in token-reduction Phase 4
|
||||
// review, codex, land-and-deploy (w1), autoplan (w2), qa (w3) carved in token-reduction Phase 4
|
||||
// wave 1 (v1.69.x branch) — their invariants generate from CARVE_GUARDS too.
|
||||
{
|
||||
skill: 'qa',
|
||||
mustContain: ['bug', 'browse', 'fix'],
|
||||
mustHaveHeadings: ['## Preamble', '## When to invoke'],
|
||||
// v1.2.0 activation lift: the unified first-run-guidance section (P4 scaffold +
|
||||
// P3 loop tip) is added to every skill's shared preamble — intentional, ~1KB.
|
||||
// Fork port wave 2: the shared coverage-audit detection block gained the
|
||||
// multi-ecosystem markers (Django/JVM, script/target/test-file census —
|
||||
// e3259078 port); measured 1.111x. v1.64+v1.65 merge sums both waves'
|
||||
// preamble growth; measured 1.125.
|
||||
maxSizeRatio: 1.13,
|
||||
minBytes: 50_000,
|
||||
},
|
||||
{
|
||||
skill: 'investigate',
|
||||
mustContain: ['root cause', 'hypothes'],
|
||||
|
||||
@@ -139,7 +139,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// devex, office-hours + future PR2 carves). One file iterating CARVE_GUARDS;
|
||||
// the selector sets GSTACK_CARVE_SKILL=<name> to scope cost to the changed
|
||||
// skill (D-CODEX A). Touching the registry/helper or sections.ts runs all.
|
||||
'carve-section-loading': ['review/**', 'codex/**', 'land-and-deploy/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'office-hours/**', 'document-release/**', 'design-consultation/**', 'cso/**', 'test/helpers/carve-guards.ts', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'],
|
||||
'carve-section-loading': ['qa/**', 'browse/**', 'retro/**', 'autoplan/**', 'spec/**', 'setup-gbrain/**', 'review/**', 'codex/**', 'land-and-deploy/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'office-hours/**', 'document-release/**', 'design-consultation/**', 'cso/**', 'test/helpers/carve-guards.ts', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'],
|
||||
'autoplan-chain-pty': ['autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-autoplan-chain.test.ts'],
|
||||
'e2e-harness-audit': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/claude-pty-runner.ts'],
|
||||
|
||||
@@ -229,8 +229,8 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'review-dashboard-via': ['ship/**', 'scripts/resolvers/review.ts', 'codex/**', 'autoplan/**', 'land-and-deploy/**', 'test/skill-e2e-review-attribution.test.ts'],
|
||||
|
||||
// Retro
|
||||
'retro': ['retro/**', 'test/skill-e2e-retro.test.ts'],
|
||||
'retro-base-branch': ['retro/**', 'test/skill-e2e-retro.test.ts'],
|
||||
'retro': ['bin/gstack-retro-metrics', 'retro/**', 'test/skill-e2e-retro.test.ts'],
|
||||
'retro-base-branch': ['bin/gstack-retro-metrics', 'retro/**', 'test/skill-e2e-retro.test.ts'],
|
||||
|
||||
// Global discover
|
||||
'global-discover': ['bin/gstack-global-discover.ts', 'test/global-discover.test.ts'],
|
||||
@@ -752,16 +752,16 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
|
||||
* LLM-judge test touchfiles — keyed by test description string.
|
||||
*/
|
||||
export const LLM_JUDGE_TOUCHFILES: Record<string, string[]> = {
|
||||
'command reference table': ['SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts'],
|
||||
'snapshot flags reference': ['SKILL.md', 'SKILL.md.tmpl', 'browse/src/snapshot.ts'],
|
||||
'browse/SKILL.md reference': ['browse/SKILL.md', 'browse/SKILL.md.tmpl', 'browse/src/**'],
|
||||
'command reference table': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts'],
|
||||
'snapshot flags reference': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/snapshot.ts'],
|
||||
'browse/SKILL.md reference': ['browse/sections/**', 'browse/SKILL.md', 'browse/SKILL.md.tmpl', 'browse/src/**'],
|
||||
'setup block': ['SKILL.md', 'SKILL.md.tmpl'],
|
||||
'regression vs baseline': ['SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts', 'test/fixtures/eval-baselines.json'],
|
||||
'qa/SKILL.md workflow': ['qa/SKILL.md', 'qa/SKILL.md.tmpl'],
|
||||
'qa/SKILL.md health rubric': ['qa/SKILL.md', 'qa/SKILL.md.tmpl'],
|
||||
'qa/SKILL.md anti-refusal': ['qa/SKILL.md', 'qa/SKILL.md.tmpl', 'qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl'],
|
||||
'regression vs baseline': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'browse/src/commands.ts', 'test/fixtures/eval-baselines.json'],
|
||||
'qa/SKILL.md workflow': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl'],
|
||||
'qa/SKILL.md health rubric': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl'],
|
||||
'qa/SKILL.md anti-refusal': ['qa/sections/**', 'qa/SKILL.md', 'qa/SKILL.md.tmpl', 'qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl'],
|
||||
'cross-skill greptile consistency': ['review/SKILL.md', 'review/SKILL.md.tmpl', 'ship/SKILL.md', 'ship/SKILL.md.tmpl', 'review/greptile-triage.md', 'retro/SKILL.md', 'retro/SKILL.md.tmpl'],
|
||||
'baseline score pinning': ['SKILL.md', 'SKILL.md.tmpl', 'test/fixtures/eval-baselines.json'],
|
||||
'baseline score pinning': ['browse/sections/**', 'SKILL.md', 'SKILL.md.tmpl', 'test/fixtures/eval-baselines.json'],
|
||||
|
||||
// Ship & Release
|
||||
'ship/SKILL.md workflow': ['ship/SKILL.md', 'ship/SKILL.md.tmpl'],
|
||||
@@ -789,7 +789,7 @@ export const LLM_JUDGE_TOUCHFILES: Record<string, string[]> = {
|
||||
'setup-deploy/SKILL.md platform setup': ['setup-deploy/SKILL.md', 'setup-deploy/SKILL.md.tmpl'],
|
||||
|
||||
// Other skills
|
||||
'retro/SKILL.md instructions': ['retro/SKILL.md', 'retro/SKILL.md.tmpl'],
|
||||
'retro/SKILL.md instructions': ['retro/sections/**', 'retro/SKILL.md', 'retro/SKILL.md.tmpl'],
|
||||
'qa-only/SKILL.md workflow': ['qa-only/SKILL.md', 'qa-only/SKILL.md.tmpl'],
|
||||
'gstack-upgrade/SKILL.md upgrade flow': ['gstack-upgrade/SKILL.md', 'gstack-upgrade/SKILL.md.tmpl'],
|
||||
|
||||
|
||||
Reference in New Issue
Block a user