mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-12 16:08:59 +02:00
fix: retire decided A/B experiments; vendor the pre-cut fixture; ban raw-SHA fixtures
Three one-shot decision experiments kept re-running weekly as N=1
stochastic comparisons — flaky by construction with near-zero remaining
information: skill-e2e-auq-repetition-cut-ab (its own header: gate "passed
pre-landing, approved 2026-08-25"), skill-e2e-preamble-script-ab ("demoted
post-Phase-3"), and opus-47's fanout arm-vs-arm (parA >= parB across two
SINGLE stochastic runs — a coin flip). Deleted, with their selection keys;
the SDK overlay-harness stays as the maintained instrument for the next
experiment, and opus-47 keeps its routing-precision cases.
verboseSkill() now reads the VENDORED test/fixtures/auq-pre-cut-...-SKILL.md
instead of `git show ab66193e^:...` — a branch-local ref that dies on
branch prune and already failed on shallow clones. New free tripwire
(test/git-ref-fixture-tripwire.test.ts) bans the raw-SHA fixture class
outright: quoted SHA:path rev-specs and gitRef-style hex defaults in the
test trees fail the suite with the vendor-instead instruction.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
99b5fa2e46
commit
d6df2af7dd
@@ -281,9 +281,17 @@ export function carvedSkill(): { skillMd: string; sectionsFrom: string | null }
|
||||
};
|
||||
}
|
||||
|
||||
/** Read the pre-carve verbose monolith plan-ceo SKILL.md from git. */
|
||||
export function verboseSkill(gitRef = 'ab66193e^'): string {
|
||||
return execGit(['show', `${gitRef}:plan-ceo-review/SKILL.md`]);
|
||||
/** Read the pre-carve verbose monolith plan-ceo SKILL.md.
|
||||
* VENDORED fixture (v1.75 precedent), not a git ref: the old default
|
||||
* `git show ab66193e^:...` pinned a BRANCH-LOCAL commit — it dies the day
|
||||
* that branch is pruned and already fails on shallow clones. The fixture
|
||||
* is the frozen pre-cut render; test/git-ref-fixture-tripwire.test.ts
|
||||
* keeps this class from coming back. */
|
||||
export function verboseSkill(): string {
|
||||
return fs.readFileSync(
|
||||
path.join(ROOT, 'test', 'fixtures', 'auq-pre-cut-plan-ceo-review-SKILL.md'),
|
||||
'utf-8',
|
||||
);
|
||||
}
|
||||
|
||||
function execGit(args: string[]): string {
|
||||
|
||||
@@ -129,9 +129,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
// Real-PTY E2E batch (#6 new tests on the harness).
|
||||
// Each one tests behavior the SDK harness can't observe (rendered TTY,
|
||||
// numbered-option lists, multi-phase ordering, idempotency state echo).
|
||||
'preamble-script-ab': ['bin/gstack-skill-start', 'bin/gstack-skill-end', 'scripts/resolvers/preamble/generate-preamble-bash.ts', 'scripts/resolvers/preamble/generate-brain-sync-block.ts', 'scripts/resolvers/preamble.ts', 'plan-ceo-review/**', 'test/helpers/auq-sdk-capture.ts', 'test/skill-e2e-preamble-script-ab.test.ts'],
|
||||
'auq-format-gate': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble/generate-completeness-section.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/session-runner.ts', 'test/helpers/llm-judge.ts', 'test/skill-e2e-ask-user-question-format-compliance.test.ts'],
|
||||
'auq-repetition-cut-ab': ['scripts/resolvers/preamble/generate-ask-user-format.ts', 'plan-ceo-review/**', 'test/helpers/auq-sdk-capture.ts', 'test/helpers/llm-judge.ts', 'test/fixtures/auq-pre-cut-plan-ceo-review-SKILL.md', 'test/skill-e2e-auq-repetition-cut-ab.test.ts'],
|
||||
'plan-ceo-mode-routing': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-ceo-mode-routing.test.ts'],
|
||||
'plan-design-with-ui-scope': ['plan-design-review/**', 'test/fixtures/plans/ui-heavy-feature.md', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-with-ui.test.ts'],
|
||||
'ship-idempotency-pty': ['ship/**', 'bin/gstack-next-version', 'bin/gstack-version-bump', 'scripts/resolvers/sections.ts', 'lib/worktree.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-ship-idempotency.test.ts'],
|
||||
@@ -569,8 +567,6 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
|
||||
// Real-PTY E2E batch — tier classification:
|
||||
// gate: cheap, deterministic, run on every PR
|
||||
// periodic: long-running or expensive (>$3/run), run weekly
|
||||
'preamble-script-ab': 'periodic', // Phase 1-3 A/B: script vs inline preamble; demoted post-Phase-3 (OV7)
|
||||
'auq-repetition-cut-ab': 'periodic', // AUQ repetition-cut NOT-WORSE gate (passed pre-landing). Periodic runs force EVALS_ALL, so the dep list cannot auto-trigger it — an AUQ format edit carries a MANUAL re-run obligation (bun test test/skill-e2e-auq-repetition-cut-ab.test.ts with EVALS=1 EVALS_TIER=periodic)
|
||||
'auq-format-gate': 'gate', // ~$0.50/run, SDK capture, single skill probe
|
||||
'plan-ceo-mode-routing': 'periodic', // ~$3/run, deep navigation through 8-12 prior AskUserQuestions
|
||||
'plan-design-with-ui-scope': 'gate', // ~$0.80/run
|
||||
|
||||
Reference in New Issue
Block a user