mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-27 23:21:53 +02:00
Garry TanandOpenAI Codex
06ed920a97
* feat: bind shared-code review advice to source and branch * feat: add shared-code extraction audit and scoped review checks * test: recognize complete source reads and explicit coverage legends * chore: bump version and changelog (v1.88.0.0) Co-Authored-By: OpenAI Codex <noreply@openai.com> * test: capture native review questions and retain public evidence Capture the actual first public native question with strict ownership and display matching. Preserve terminal failures and raw evidence, and retain SDK completion checks. * test: recognize verified review evidence and complete fixtures Recognize complete source and diagram evidence, concrete design and developer-experience decisions, and the complete planted scenario contracts. Preserve negative controls and grading thresholds. * fix: preserve decision brief structure in native questions Keep the required pros-and-cons heading and final Net field in native question text. Regenerate host outputs and document the release and evaluation repairs. Co-Authored-By: OpenAI Codex <noreply@openai.com> * docs: update project documentation for v1.88.0.0 Co-Authored-By: OpenAI Codex <noreply@openai.com> * fix: correct eval retry accounting and ship workflow gates * fix: capture native eval evidence and stabilize CI fixtures * fix: keep shared-code eval skips read-only Choose explicit no-change answers instead of mixed fix/preservation options. Reuse the bounded revalidation prompt for path fixtures so required review metadata is available without repeated discovery. Preserve source checks, retry limits, and failed native terminal outcomes. Add captured-question and callback regressions, plus evaluation selection coverage for the affected fixtures. --------- Co-authored-by: OpenAI Codex <noreply@openai.com>
227 lines
12 KiB
TypeScript
227 lines
12 KiB
TypeScript
/**
|
|
* Skill coverage matrix (v1.45.0.0 T1, cathedral Phase 0).
|
|
*
|
|
* Single source of truth mapping each gstack skill to its E2E test files.
|
|
* The CI gate at test/skill-coverage-matrix.test.ts fails if a skill has
|
|
* no gate-tier entry, ensuring the eval-first foundation holds: every
|
|
* skill has at least one CI-blocking check that asserts must-have
|
|
* behavior.
|
|
*
|
|
* Two tiers per entry:
|
|
* gate CI-blocking, runs on every PR, target <$0.50/test or free.
|
|
* periodic Weekly cron, deeper coverage, can cost ~$1-$3/test.
|
|
*
|
|
* The 'floor' entry refers to test/skill-coverage-floor.test.ts —
|
|
* a structural-compliance smoke test that covers every skill with
|
|
* file-IO checks (free, no LLM cost). When a skill has only 'floor'
|
|
* coverage, that's the eval-first minimum; future work can layer
|
|
* behavioral checks on top.
|
|
*/
|
|
|
|
export interface SkillCoverage {
|
|
/** Gate-tier test file paths (relative to repo root). At least one required per skill. */
|
|
gate: string[];
|
|
/** Periodic-tier test file paths. Optional but recommended. */
|
|
periodic: string[];
|
|
/** Brief note on why this coverage is the right shape for this skill. */
|
|
rationale?: string;
|
|
}
|
|
|
|
/**
|
|
* Per-skill coverage. Keys MUST match the top-level skill directory name.
|
|
* The CI test asserts every skill in the repo has an entry here AND that
|
|
* gate[] is non-empty.
|
|
*
|
|
* Adding a new skill: add an entry here AND either reference an existing
|
|
* test that covers it OR add 'test/skill-coverage-floor.test.ts' as the
|
|
* minimum gate-tier check.
|
|
*/
|
|
export const SKILL_COVERAGE: Record<string, SkillCoverage> = {
|
|
// ─── Core loop ──────────────────────────────────────────────
|
|
ship: {
|
|
gate: ['test/skill-e2e-ship-idempotency.test.ts', 'test/skill-coverage-floor.test.ts'],
|
|
periodic: ['test/skill-e2e-workflow.test.ts'],
|
|
},
|
|
review: {
|
|
gate: ['test/skill-e2e-review.test.ts', 'test/skill-e2e-shared-libs.test.ts', 'test/skill-e2e-shared-libs-paths.test.ts', 'test/shared-libs-evidence.test.ts', 'test/shared-libs-rendering.test.ts', 'test/skill-coverage-floor.test.ts'],
|
|
periodic: ['test/skill-e2e-review-army.test.ts', 'test/regression-1539-review-self-verify.test.ts'],
|
|
},
|
|
'deslop-shared-libs': {
|
|
gate: ['test/shared-libs-rendering.test.ts', 'test/skill-e2e-shared-libs.test.ts', 'test/skill-coverage-floor.test.ts'],
|
|
periodic: ['test/skill-e2e-shared-libs-periodic.test.ts', 'test/codex-e2e-shared-libs.test.ts'],
|
|
rationale: 'Free host/discovery checks; native gate traces enforce read-only source access and the actual review advisory lifecycle. Periodic evaluates opportunity and PR coverage judgment.',
|
|
},
|
|
qa: {
|
|
gate: ['test/skill-e2e-qa-workflow.test.ts', 'test/skill-coverage-floor.test.ts'],
|
|
periodic: ['test/skill-e2e-qa-workflow.test.ts', 'test/skill-e2e-qa-bugs.test.ts', 'test/skill-e2e-aside.test.ts'],
|
|
rationale: 'qa-quick / qa-only-no-fix / qa-bootstrap are gate: the skill drives Aside when it is live and the gstack browse binary otherwise, so CI runs the fallback path. The planted-bug benchmarks, the fix loop and the live-Aside run (aside-qa-quick) are periodic.',
|
|
},
|
|
'qa-only': {
|
|
gate: ['test/skill-coverage-floor.test.ts'],
|
|
periodic: [],
|
|
rationale: 'qa-only is qa with --report-only; behavior tested via /qa coverage.',
|
|
},
|
|
investigate: {
|
|
gate: ['test/skill-coverage-floor.test.ts'],
|
|
periodic: [],
|
|
},
|
|
browse: {
|
|
gate: ['test/skill-e2e-bws.test.ts', 'test/skill-coverage-floor.test.ts'],
|
|
periodic: ['test/skill-e2e-aside.test.ts'],
|
|
rationale: '/browse drives the Aside browser first (the live E2E aside-browse-basic / aside-browse-flow needs a running Aside, so it is periodic) and the gstack browse binary as fallback (browse-basic / browse-snapshot exercise it, gate; the binary has its own integration suite under browse/test/). Local-HTML rendering (lib/aside-render.ts, bin/gstack-render.ts) is a library, covered by test/aside-render.test.ts.',
|
|
},
|
|
spec: {
|
|
gate: [
|
|
'test/spec-template-invariants.test.ts',
|
|
'test/spec-template-sync.test.ts',
|
|
'test/skill-coverage-floor.test.ts',
|
|
],
|
|
periodic: [
|
|
'test/skill-e2e-spec-execute.test.ts',
|
|
'test/skill-llm-eval-spec.test.ts',
|
|
],
|
|
rationale: '37 deterministic invariants pin Phase 1/3 gating, --execute race/security hardening, quality-gate redaction, archive contract, plan-mode-aware Phase 5. Periodic adds full PTY pipeline + LLM-judge.',
|
|
},
|
|
|
|
// ─── Plan triad ─────────────────────────────────────────────
|
|
'plan-ceo-review': {
|
|
gate: [
|
|
'test/skill-e2e-plan-ceo-finding-floor.test.ts',
|
|
'test/skill-e2e-plan-ceo-plan-mode.test.ts',
|
|
'test/skill-coverage-floor.test.ts',
|
|
],
|
|
periodic: [
|
|
'test/skill-e2e-plan-ceo-finding-count.test.ts',
|
|
'test/skill-e2e-plan-ceo-mode-routing.test.ts',
|
|
],
|
|
},
|
|
'plan-eng-review': {
|
|
gate: [
|
|
'test/shared-libs-rendering.test.ts',
|
|
'test/skill-e2e-plan-eng-finding-floor.test.ts',
|
|
'test/skill-e2e-plan-eng-plan-mode.test.ts',
|
|
'test/skill-coverage-floor.test.ts',
|
|
],
|
|
periodic: [
|
|
'test/skill-e2e-shared-libs-periodic.test.ts',
|
|
'test/skill-e2e-plan-eng-finding-count.test.ts',
|
|
'test/skill-e2e-plan-eng-multi-finding-batching.test.ts',
|
|
],
|
|
},
|
|
'plan-design-review': {
|
|
gate: [
|
|
'test/skill-e2e-plan-design-finding-floor.test.ts',
|
|
'test/skill-e2e-plan-design-plan-mode.test.ts',
|
|
'test/skill-e2e-plan-design-with-ui.test.ts',
|
|
'test/skill-coverage-floor.test.ts',
|
|
],
|
|
periodic: ['test/skill-e2e-plan-design-finding-count.test.ts'],
|
|
},
|
|
'plan-devex-review': {
|
|
gate: [
|
|
'test/skill-e2e-plan-devex-finding-floor.test.ts',
|
|
'test/skill-e2e-plan-devex-plan-mode.test.ts',
|
|
'test/skill-coverage-floor.test.ts',
|
|
],
|
|
periodic: ['test/skill-e2e-plan-devex-finding-count.test.ts'],
|
|
},
|
|
autoplan: {
|
|
gate: ['test/skill-coverage-floor.test.ts'],
|
|
periodic: ['test/skill-e2e-autoplan-chain.test.ts', 'test/skill-e2e-autoplan-dual-voice.test.ts'],
|
|
},
|
|
'office-hours': {
|
|
gate: ['test/skill-e2e-office-hours.test.ts', 'test/skill-coverage-floor.test.ts'],
|
|
periodic: ['test/skill-e2e-office-hours-auto-mode.test.ts', 'test/skill-e2e-office-hours-phase4.test.ts'],
|
|
},
|
|
|
|
// ─── Polish + design ────────────────────────────────────────
|
|
'design-review': {
|
|
gate: ['test/skill-coverage-floor.test.ts'],
|
|
periodic: ['test/skill-e2e-design.test.ts'],
|
|
rationale: 'design-review-fix drives the Aside browser (periodic; skips without one).',
|
|
},
|
|
'design-consultation': { gate: ['test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
'design-shotgun': { gate: ['test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
'design-html': { gate: ['test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
diagram: {
|
|
gate: ['test/skill-e2e-diagram.test.ts', 'test/skill-coverage-floor.test.ts'],
|
|
periodic: ['test/skill-e2e-diagram.test.ts'],
|
|
rationale: 'Triplet contract is gate-tier deterministic (gstack-render drives Aside when live, the browse daemon otherwise, so CI runs it); authoring-quality judge is periodic (E2E_TIERS: diagram-triplet/diagram-authoring-quality). The renderer itself is pinned free by test/aside-render.test.ts.',
|
|
},
|
|
cso: {
|
|
gate: ['test/skill-e2e-cso.test.ts', 'test/cso-preserved.test.ts', 'test/skill-coverage-floor.test.ts'],
|
|
periodic: [],
|
|
rationale: 'cso-preserved.test.ts pins must-not-strip security guidance phrases.',
|
|
},
|
|
'document-release': { gate: ['test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
'document-generate': { gate: ['test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
|
|
// ─── Ops + integrations ─────────────────────────────────────
|
|
'land-and-deploy': { gate: ['test/skill-e2e-deploy.test.ts', 'test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
canary: {
|
|
gate: ['test/skill-e2e-deploy.test.ts', 'test/skill-coverage-floor.test.ts'],
|
|
periodic: ['test/skill-e2e-aside.test.ts'],
|
|
rationale: 'canary-workflow (gate) runs the skill in simulation without a browser; aside-canary-quick drives Aside live (periodic).',
|
|
},
|
|
benchmark: {
|
|
gate: ['test/skill-e2e-deploy.test.ts', 'test/skill-e2e-benchmark-providers.test.ts', 'test/skill-coverage-floor.test.ts'],
|
|
periodic: [],
|
|
rationale: 'benchmark-workflow (gate) runs the skill in simulation without a browser.',
|
|
},
|
|
'benchmark-models': { gate: ['test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
codex: { gate: ['test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
retro: {
|
|
gate: ['test/skill-coverage-floor.test.ts'],
|
|
periodic: ['test/regression-1624-retro-stale-base.test.ts'],
|
|
},
|
|
'gstack-upgrade': { gate: ['test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
'context-save': { gate: ['test/skill-e2e-context-skills.test.ts', 'test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
'context-restore': { gate: ['test/skill-e2e-context-skills.test.ts', 'test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
'setup-deploy': { gate: ['test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
'setup-browser-cookies': { gate: ['test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
'setup-gbrain': {
|
|
gate: [
|
|
'test/skill-e2e-setup-gbrain-bad-token.test.ts',
|
|
'test/skill-e2e-setup-gbrain-path4-local-pglite.test.ts',
|
|
'test/skill-e2e-setup-gbrain-remote.test.ts',
|
|
'test/skill-coverage-floor.test.ts',
|
|
],
|
|
periodic: [],
|
|
},
|
|
'sync-gbrain': {
|
|
gate: ['test/skill-coverage-floor.test.ts'],
|
|
periodic: ['test/regression-1611-gbrain-sync-resume.test.ts'],
|
|
},
|
|
'open-gstack-browser': { gate: ['test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
'pair-agent': { gate: ['test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
scrape: {
|
|
gate: ['test/skill-coverage-floor.test.ts'],
|
|
periodic: ['test/skill-e2e-skillify.test.ts', 'test/skill-e2e-aside.test.ts'],
|
|
rationale: '/scrape is Aside-first: aside-scrape-json drives Aside live and checks the JSON-only output discipline (periodic; skips without Aside). scrape-match-path / scrape-prototype-path assert the browser-skills `$B skill list` / `skill run` flow, which the Aside-first template no longer prescribes in its fallback — periodic until the fallback carries it again.',
|
|
},
|
|
skillify: { gate: ['test/skill-e2e-skillify.test.ts', 'test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
learn: { gate: ['test/skill-e2e-learnings.test.ts', 'test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
'plan-tune': { gate: ['test/skill-e2e-plan-tune.test.ts', 'test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
|
|
// ─── iOS family ─────────────────────────────────────────────
|
|
'ios-qa': { gate: ['test/skill-e2e-ios.test.ts', 'test/skill-coverage-floor.test.ts'], periodic: ['test/skill-e2e-ios-device.test.ts', 'test/skill-e2e-ios-swift-build.test.ts'] },
|
|
'ios-fix': { gate: ['test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
'ios-clean': { gate: ['test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
'ios-sync': { gate: ['test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
'ios-design-review': { gate: ['test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
|
|
// ─── Safety / housekeeping ──────────────────────────────────
|
|
careful: { gate: ['test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
freeze: { gate: ['test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
unfreeze: { gate: ['test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
guard: { gate: ['test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
'landing-report': { gate: ['test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
health: { gate: ['test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
'make-pdf': {
|
|
gate: ['test/skill-coverage-floor.test.ts'],
|
|
periodic: [],
|
|
rationale: 'make-pdf is a binary with its own free suite under make-pdf/test/ (print pipeline via lib/aside-render.ts); the skill doc is structure-checked by the floor.',
|
|
},
|
|
'devex-review': { gate: ['test/skill-coverage-floor.test.ts'], periodic: [] },
|
|
};
|