test: cover first-run detection + repoint browse-content assertions to /browse

- New unit tests for every detection bucket, the eval-safe enum contract, and the
  first-run gating (test/preamble-first-task-scaffold.test.ts); periodic E2E that runs
  the detector through the real harness (test/skill-e2e-first-task-scaffold.test.ts).
- Repoint browse-content assertions (gen-skill-docs, audit-compliance, skill-validation,
  LLM-judge eval) from the root skill to browse/SKILL.md following the router split;
  add a regression pinning that the router carries no browse body.
- Register first-task-scaffold touchfiles + periodic tier; bump parity/carve size caps
  ~1-2KB per skill for the shared first-run-guidance preamble section.
- Refresh ship golden fixtures for the preamble addition.
This commit is contained in:
Garry Tan
2026-06-21 07:16:24 -07:00
parent 938fa4a035
commit 866159861d
12 changed files with 438 additions and 51 deletions
+16 -7
View File
@@ -161,7 +161,8 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
gateAfterStop: 'EXIT PLAN MODE GATE',
},
behavioral: 'plan',
maxSkeletonBytes: 62_000,
// v1.2.0 activation lift: first-run-guidance section in the shared preamble.
maxSkeletonBytes: 65_000,
minUnionBytes: 70_000,
mustContain: ['Architecture', 'Code Quality', 'Test', 'Performance'],
// Cross-cutting preamble growth (v1.57.2.0 AUQ-failure prose fallback + the
@@ -185,7 +186,8 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
behavioral: 'plan',
// +Conductor AUQ-default-prose rule + one-way/continuation safety in the
// always-loaded AskUserQuestion Format section.
maxSkeletonBytes: 84_000,
// v1.2.0 activation lift: first-run-guidance section in the shared preamble.
maxSkeletonBytes: 86_000,
minUnionBytes: 70_000,
mustContain: ['design', 'visual'],
},
@@ -203,7 +205,8 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
behavioral: 'plan',
// +Conductor AUQ-default-prose rule + one-way/destructive prose safety +
// continuation protocol in the always-loaded AskUserQuestion Format section.
maxSkeletonBytes: 78_000,
// v1.2.0 activation lift: first-run-guidance section in the shared preamble.
maxSkeletonBytes: 80_000,
minUnionBytes: 70_000,
mustContain: ['developer experience', 'Getting Started'],
// Default-on Codex outside-voice (codexPreflight block + CODEX_MODE branch
@@ -224,9 +227,12 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
gateAfterStop: undefined,
},
behavioral: 'prompt',
maxSkeletonBytes: 96_000,
// v1.2.0 activation lift: first-run-guidance section in the shared preamble,
// plus the P1 office-hours closing handoff (AUQ that launches the next skill).
maxSkeletonBytes: 98_000,
minUnionBytes: 70_000,
mustContain: ['design doc', 'problem statement'],
maxSizeRatio: 1.07,
},
'document-release': {
skill: 'document-release',
@@ -243,7 +249,8 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
behavioral: 'prompt',
// +Conductor AUQ-default-prose rule + one-way/continuation safety in the
// always-loaded AskUserQuestion Format section.
maxSkeletonBytes: 53_000,
// v1.2.0 activation lift: first-run-guidance section in the shared preamble.
maxSkeletonBytes: 56_000,
minUnionBytes: 55_000,
mustContain: ['CHANGELOG', 'Diataxis', 'coverage'],
// Two intentional additions stack on this small skill: the AUQ-failure prose
@@ -270,7 +277,8 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
behavioral: 'prompt',
// +Conductor AUQ-default-prose rule + one-way/continuation safety in the
// always-loaded AskUserQuestion Format section.
maxSkeletonBytes: 67_000,
// v1.2.0 activation lift: first-run-guidance section in the shared preamble.
maxSkeletonBytes: 69_000,
minUnionBytes: 72_000,
mustContain: ['Typography', 'Color', 'Aesthetic Direction'],
// Cross-cutting preamble growth (v1.57.2.0 AUQ-failure prose fallback ~2KB +
@@ -308,7 +316,8 @@ export const CARVE_GUARDS: Record<string, CarveGuard> = {
behavioral: 'prompt',
// +Conductor AUQ-default-prose rule + one-way/continuation safety in the
// always-loaded AskUserQuestion Format section.
maxSkeletonBytes: 73_000,
// v1.2.0 activation lift: first-run-guidance section in the shared preamble.
maxSkeletonBytes: 75_000,
minUnionBytes: 72_000,
mustContain: ['OWASP', 'STRIDE', 'daily', 'comprehensive', 'verif'],
// cso keeps its mode-dispatch + FP-filtering phases always-loaded, so the
+7 -3
View File
@@ -221,7 +221,9 @@ const MONOLITH_INVARIANTS: ParityInvariant[] = [
skill: 'qa',
mustContain: ['bug', 'browse', 'fix'],
mustHaveHeadings: ['## Preamble', '## When to invoke'],
maxSizeRatio: 1.05,
// v1.2.0 activation lift: the unified first-run-guidance section (P4 scaffold +
// P3 loop tip) is added to every skill's shared preamble — intentional, ~1KB.
maxSizeRatio: 1.07,
minBytes: 50_000,
},
{
@@ -231,14 +233,16 @@ const MONOLITH_INVARIANTS: ParityInvariant[] = [
// Cross-cutting preamble growth (v1.57.2.0 AUQ-failure prose fallback ~2KB + the
// cross-session decision-memory nudge) lands this skill just over the strict 1.05;
// headroom for the shared preamble additions (matches the carved-skill overrides).
maxSizeRatio: 1.07,
// v1.2.0 activation lift adds the first-run-guidance section on top.
maxSizeRatio: 1.09,
minBytes: 30_000,
},
{
skill: 'autoplan',
mustContain: ['ceo', 'eng', 'design'],
mustHaveHeadings: ['## Preamble', '## When to invoke'],
maxSizeRatio: 1.05,
// v1.2.0 activation lift: shared first-run-guidance preamble section.
maxSizeRatio: 1.07,
minBytes: 70_000,
},
];
+7
View File
@@ -41,6 +41,10 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
'hermetic-canary': ['test/helpers/hermetic-env.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-hermetic-canary.test.ts', 'lib/conductor-env-shim.ts'],
'hermetic-sentinel': ['test/helpers/hermetic-env.ts', 'test/helpers/session-runner.ts', 'test/skill-e2e-hermetic-canary.test.ts', 'lib/conductor-env-shim.ts'],
// P4 first-run scaffold (activation lift) — the detection binary end-to-end
// through the real runner, plus the preamble wiring that gates + maps it.
'first-task-scaffold': ['bin/gstack-first-task-detect', 'scripts/resolvers/preamble/generate-first-run-guidance.ts', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'test/skill-e2e-first-task-scaffold.test.ts', 'test/helpers/session-runner.ts'],
// SKILL.md setup + preamble (depend on ROOT SKILL.md + gen-skill-docs)
'skillmd-setup-discovery': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
'skillmd-no-local-binary': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
@@ -459,6 +463,9 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
'session-awareness': 'gate',
'operational-learning': 'gate',
// P4 first-run scaffold — periodic (onboarding, non-safety, model-touched marker)
'first-task-scaffold': 'periodic',
// QA — gate for functional, periodic for quality/benchmarks
'qa-quick': 'gate',
'qa-b6-static': 'periodic',