diff --git a/test/carve-section-loading.test.ts b/test/carve-section-loading.test.ts new file mode 100644 index 000000000..0aaea779f --- /dev/null +++ b/test/carve-section-loading.test.ts @@ -0,0 +1,97 @@ +/** + * T2 — data-driven behavioral section-loading guard (PERIODIC tier, paid, SDK capture). + * + * The behavioral proof that a REAL agent actually Reads each carved skill's + * required sections at runtime — not just that the skeleton structure looks right + * (that's E2, free, per-PR). One file iterating the canonical CARVE_GUARDS + * registry (EQ2): registry membership IS the test, so "registered ⇒ asserted" is + * structural — a carve can't be registered yet behaviorally unguarded. + * + * Per codex refined-plan pass: + * #2 — ONE test() per skill, each with its own timeout + named failure output; + * a hung claude -p fails only its skill, not the whole file. + * #3 / D-CODEX(A) — GSTACK_CARVE_SKILL= runs only that skill's case, so + * the touchfile selector can scope cost to the changed skill; unset runs all. + * #7 — each case drives the run with the registry's `scenario` (built to force + * the STOP-Read path) and asserts the required sections were Read. + * + * 'external' skills (ship, plan-ceo-review) have bespoke fixtures (git state, + * Step-0 mode loop) and keep their dedicated tests; E1 asserts those exist. + */ + +import { describe, test, expect } from 'bun:test'; +import { setupSkillDir, skillFromWorktree, captureSectionReads } from './helpers/auq-sdk-capture'; +import { CARVE_GUARDS } from './helpers/carve-guards'; + +const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; +const describeE2E = shouldRun ? describe : describe.skip; +const runId = `carve-section-loading-${process.env.EVALS_RUN_ID ?? 'local'}`; +const only = process.env.GSTACK_CARVE_SKILL?.trim(); + +// A generic plan fixture for 'plan' behavioral skills (the review family). +const PLAN_MD = [ + '# Plan: add an in-memory cache layer', + '', + '## Context', + 'Reads hit the DB on every request. Add a process-local LRU cache in front of the', + 'read path to cut DB load.', + '', + '## Approach', + '- Wrap the read repository in a cache that stores the last 1000 keys.', + '- Invalidate on write.', + '', + '## Out of scope', + 'Distributed cache, cross-process coherence.', + '', +].join('\n'); + +describeE2E('carve behavioral section-loading (periodic, SDK capture)', () => { + for (const guard of Object.values(CARVE_GUARDS)) { + // 'external' carves keep their dedicated bespoke tests (E1 verifies those exist). + if (guard.behavioral === 'external') continue; + // Cost-scoped selection: when GSTACK_CARVE_SKILL is set, run only that skill. + if (only && only !== guard.skill) continue; + + test( + `${guard.skill}: a real run Reads ${guard.requiredReads.join(', ')}`, + async () => { + const { skillMd, sectionsFrom } = skillFromWorktree(guard.skill); + const fixtures = guard.behavioral === 'plan' ? { 'PLAN.md': PLAN_MD } : {}; + const planDir = setupSkillDir({ + skillName: guard.skill, + skillMd, + sectionsFrom, + fixtures, + tmpPrefix: `gstack-${guard.skill}-secload-`, + }); + + const { readSections, reportProduced, output } = await captureSectionReads({ + planDir, + skillName: guard.skill, + scenario: guard.scenario, + reportMarker: /report|review|summary|design doc|handoff/i, + testName: `${guard.skill} section-loading`, + runId, + }); + + const missing = guard.requiredReads.filter((s) => !readSections.has(s)); + // Named failure output (codex #2): skill + expected + observed. + expect({ + skill: guard.skill, + reportProduced, + expected: guard.requiredReads, + observed: [...readSections], + missing, + }).toEqual({ + skill: guard.skill, + reportProduced: true, + expected: guard.requiredReads, + observed: expect.any(Array), + missing: [], + }); + expect(output.trim().length).toBeGreaterThan(200); + }, + 360_000, + ); + } +}); diff --git a/test/helpers/touchfiles.ts b/test/helpers/touchfiles.ts index 5c49e46cd..ca9957c0e 100644 --- a/test/helpers/touchfiles.ts +++ b/test/helpers/touchfiles.ts @@ -123,6 +123,11 @@ export const E2E_TOUCHFILES: Record = { 'ship-idempotency-pty': ['ship/**', 'bin/gstack-next-version', 'bin/gstack-version-bump', 'scripts/resolvers/sections.ts', 'lib/worktree.ts', 'test/helpers/claude-pty-runner.ts'], 'ship-section-loading': ['ship/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'], 'plan-ceo-section-loading': ['plan-ceo-review/**', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'], + // Data-driven behavioral guard for the 'plan'/'prompt' carves (eng, design, + // devex, office-hours + future PR2 carves). One file iterating CARVE_GUARDS; + // the selector sets GSTACK_CARVE_SKILL= to scope cost to the changed + // skill (D-CODEX A). Touching the registry/helper or sections.ts runs all. + 'carve-section-loading': ['plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'office-hours/**', 'document-release/**', 'design-consultation/**', 'cso/**', 'test/helpers/carve-guards.ts', 'scripts/resolvers/sections.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts'], 'autoplan-chain-pty': ['autoplan/**', 'plan-ceo-review/**', 'plan-design-review/**', 'plan-eng-review/**', 'plan-devex-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts'], 'e2e-harness-audit': ['plan-ceo-review/**', 'plan-eng-review/**', 'plan-design-review/**', 'plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'test/helpers/agent-sdk-runner.ts', 'test/helpers/claude-pty-runner.ts'], @@ -512,6 +517,7 @@ export const E2E_TIERS: Record = { 'ship-idempotency-pty': 'periodic', // ~$3/run, real /ship in plan mode 'ship-section-loading': 'periodic', // ~$3/run, real /ship; asserts section reads 'plan-ceo-section-loading': 'periodic', // ~$3-5/run, real /plan-ceo-review; asserts section read + 'carve-section-loading': 'periodic', // ~$1-2/skill, data-driven; GSTACK_CARVE_SKILL scopes to one 'autoplan-chain-pty': 'periodic', // ~$8/run, all 3 phases sequential // Per-finding count + review-report-at-bottom — periodic because each diff --git a/test/skill-e2e-plan-ceo-review-section-loading.test.ts b/test/skill-e2e-plan-ceo-review-section-loading.test.ts index a4f96e080..6cac8960e 100644 --- a/test/skill-e2e-plan-ceo-review-section-loading.test.ts +++ b/test/skill-e2e-plan-ceo-review-section-loading.test.ts @@ -74,7 +74,7 @@ describeE2E('/plan-ceo-review section-loading E2E (periodic, SDK capture)', () = 'Review the plan in PLAN.md. Hold the current scope (HOLD SCOPE mode) — do not challenge or expand scope. Run the full CEO review and produce the review report.', requiredSections: REQUIRED_SECTIONS, reportMarker: /GSTACK REVIEW REPORT|COMPLETION SUMMARY|review/i, - testName: '/plan-ceo-review section-loading', + testName: 'plan-ceo-section-loading', runId, }); diff --git a/test/skill-e2e-ship-section-loading.test.ts b/test/skill-e2e-ship-section-loading.test.ts index a681ff114..4e6a717cf 100644 --- a/test/skill-e2e-ship-section-loading.test.ts +++ b/test/skill-e2e-ship-section-loading.test.ts @@ -65,7 +65,7 @@ describeE2E('/ship section-loading E2E (periodic, SDK capture)', () => { 'This is a FRESH version-changing ship: the branch has a real code change (app.js gained a new function with a test), VERSION still equals the base version (0.0.1, so it needs a bump), and CHANGELOG.md needs a new entry. Follow the skill\'s flow for a version-changing ship: run the pre-landing review and prepare the CHANGELOG entry. Produce the ship plan / review report. Do NOT actually commit, push, or open a PR.', requiredSections: REQUIRED_SECTIONS, reportMarker: /version|changelog|review|ship/i, - testName: '/ship section-loading', + testName: 'ship-section-loading', runId, });