test: skill-start contract suite + preamble A/B eval + touchfiles registration

test/gstack-skill-start.test.ts (11 free tests): STATUS-key contract vs the prose (F2), per-host fence resolution shapes (E1), proto-first, OV4 marker sanitization, --parent-pid identity, headless suppression, skill-end duration math + pending cleanup. test/skill-e2e-preamble-script-ab.test.ts (gate tier, OV7): inline-bash render (pinned from 29785978) vs script render with the fence redirected at the worktree bin (EOV2 — hermetic evals otherwise resolve the operator install and silently exercise degraded mode). 21 touchfiles dep lists gain the two bin scripts (EOV9) so future script edits select the preamble evals; selection-count pin updated 23->24. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-09-12 16:08:59 +02:00 · 2026-08-25 15:37:00 +00:00
parent 23c806d4e7
commit eb1607aaf8
4 changed files with 337 additions and 23 deletions
@@ -0,0 +1,106 @@
+/**
+ * Preamble script-consolidation A/B: inline-bash render vs script render —
+ * gate during token-reduction Phases 1-3 (demote to periodic after Phase 3,
+ * plan OV7), paid, SDK capture.
+ *
+ * Phase 1 moved ~13KB of inline preamble bash per skill into
+ * bin/gstack-skill-start. Layer 0 (test/gstack-skill-start.test.ts) proves the
+ * script emits the same STATUS lines deterministically; THIS proves the model
+ * driven by the slim render still runs the preamble and produces an
+ * equal-quality decision brief on the same prompt.
+ *
+ * Arms (precedent: skill-e2e-auq-verbose-vs-carved-ab.test.ts):
+ *   - INLINE : pre-Phase-1 plan-ceo-review/SKILL.md read from git
+ *              (29785978 = the v1.69.1.0 bump, the last inline-bash render).
+ *   - SCRIPT : this worktree's render, with the fence's install-root bin path
+ *              rewritten to THIS WORKTREE's bin/ (plan EOV2: hermetic evals
+ *              resolve $HOME/.claude/skills/gstack/bin to the operator
+ *              install, which would silently exercise the degraded path;
+ *              the rewrite makes the branch's script the subject under test).
+ *
+ * Both arms pin GSTACK_HOME to the fixture dir (EOV7: onboarding state is
+ * hermetic now that the script honors GSTACK_HOME).
+ */
+import { test, expect } from 'bun:test';
+import { describeE2ETier } from './helpers/e2e-gate';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { execSync } from 'node:child_process';
+import {
+  setupPlanCeoDir,
+  captureModeSelectionAuq,
+  scoreAuqFormat,
+  carvedSkill,
+} from './helpers/auq-sdk-capture';
+
+const describeE2E = describeE2ETier('gate');
+const runId = `preamble-ab-${process.env.EVALS_RUN_ID ?? 'local'}`;
+const ROOT = path.resolve(import.meta.dir, '..');
+const INLINE_REF = '29785978'; // last pre-Phase-1 commit (v1.69.1.0 bump)
+
+function inlineSkill(): string {
+  return execSync(`git show ${INLINE_REF}:plan-ceo-review/SKILL.md`, {
+    cwd: ROOT,
+    encoding: 'utf-8',
+    maxBuffer: 8 * 1024 * 1024,
+  });
+}
+
+/** EOV2 redirection: point the fence at the worktree's bin. */
+function scriptSkillWorktreeBin(): string {
+  const current = carvedSkill();
+  const rewritten = current.skillMd.replaceAll(
+    '$HOME/.claude/skills/gstack/bin/gstack-skill-start',
+    path.join(ROOT, 'bin', 'gstack-skill-start'),
+  );
+  if (!rewritten.includes(path.join(ROOT, 'bin', 'gstack-skill-start'))) {
+    throw new Error('binDir rewrite matched nothing — fence shape changed; update the A/B redirection');
+  }
+  return rewritten;
+}
+
+async function grade(label: string, dir: string) {
+  const text = await captureModeSelectionAuq({ planDir: dir, testName: `preamble-ab-${label}`, runId });
+  const fmt = scoreAuqFormat(text);
+  // eslint-disable-next-line no-console
+  console.log(`[PREAMBLE-AB ${label}] captured=${text.length}B format=${fmt.present}/${fmt.total} missing=[${fmt.missing.join(',')}]`);
+  return { text, fmt };
+}
+
+describeE2E('Preamble consolidation no-degradation: inline bash vs script (gate)', () => {
+  test(
+    'script-render plan-ceo-review AUQ is not worse than the inline-bash render on the same prompt',
+    async () => {
+      const sections = carvedSkill().sectionsFrom;
+      const scriptDir = setupPlanCeoDir({
+        skillMd: scriptSkillWorktreeBin(),
+        sectionsFrom: sections,
+        tmpPrefix: 'preamble-ab-script-',
+      });
+      const inlineDir = setupPlanCeoDir({
+        skillMd: inlineSkill(),
+        sectionsFrom: sections,
+        tmpPrefix: 'preamble-ab-inline-',
+      });
+
+      let s, i;
+      try {
+        s = await grade('SCRIPT', scriptDir);
+        i = await grade('INLINE', inlineDir);
+      } finally {
+        fs.rmSync(scriptDir, { recursive: true, force: true });
+        fs.rmSync(inlineDir, { recursive: true, force: true });
+      }
+
+      // Both arms must produce a capture at all (an empty script-arm capture
+      // means the preamble derailed the workflow — exactly the regression this
+      // guards against).
+      expect(s.text.length).toBeGreaterThan(100);
+      expect(i.text.length).toBeGreaterThan(100);
+      // Relative parity: the script render is NOT WORSE on decision-brief
+      // format elements (absolute compliance is auq-format-gate's job).
+      expect(s.fmt.present).toBeGreaterThanOrEqual(i.fmt.present);
+    },
+    20 * 60 * 1000,
+  );
+});