v1.87.4.0 fix: preserve health failures and disclose coverage (#2882)

* fix: report health failures and coverage accurately * chore: prepare health reporting release 1.87.4.0 * test: restrict routing evaluations to installed project skills * test: stabilize health selection and terminal fixtures
2026-09-16 18:05:31 +02:00 · 2026-09-15 21:35:11 -07:00
parent 85b8c038fc
commit a6b3a57512
15 changed files with 728 additions and 43 deletions
@@ -0,0 +1,209 @@
+/** Isolated fixtures and assertions for the single periodic /health capture. */
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { extractSkillSections } from './skill-fixture';
+import type { SkillTestResult } from './session-runner';
+import type { EvalTestEntry } from './eval-store';
+
+export const HEALTH_EVAL_ID = 'health-reporting';
+export const HEALTH_EVAL_SECTIONS = [
+  'Step 1: Detect Health Stack',
+  'Step 2: Run Tools',
+  'Step 3: Score Each Category',
+  'Step 4: Present Dashboard',
+  'Step 5: Persist to Health History',
+  'Step 6: Trend Analysis + Recommendations',
+  'Important Rules',
+];
+
+const PRIOR_HISTORY = JSON.stringify({
+  ts: '2026-01-01T00:00:00Z', branch: 'unknown', score: 10,
+  typecheck: null, lint: null, test: 10, deadcode: null, shell: null,
+  gbrain: null, duration_s: 1,
+}) + '\n';
+
+export interface HealthEvalFixture {
+  dir: string;
+  gstackHome: string;
+  receipts: string;
+  prompt: string;
+  projectFiles: Record<string, string>;
+}
+
+function projectSnapshot(dir: string): Record<string, string> {
+  const files: Record<string, string> = {};
+  const walk = (relative: string) => {
+    for (const entry of fs.readdirSync(path.join(dir, relative), { withFileTypes: true })) {
+      const name = path.join(relative, entry.name);
+      if (entry.isDirectory()) { files[name + '/'] = '<directory>'; walk(name); }
+      else files[name] = entry.isSymbolicLink() ? `<symlink:${fs.readlinkSync(path.join(dir, name))}>` : fs.readFileSync(path.join(dir, name), 'utf-8');
+    }
+  };
+  for (const project of ['partial', 'no-tools']) walk(project);
+  return files;
+}
+
+export function createHealthEvalFixture(dir: string, repoRoot: string): HealthEvalFixture {
+  const gstackHome = path.join(dir, 'gstack-state');
+  const receipts = path.join(dir, 'checker-runs.txt');
+  // Instrumentation belongs to the fixture, not to the model's shell setup.
+  // Single-quote escaping also covers temporary paths containing apostrophes.
+  const quotedReceipts = "'" + receipts.replaceAll("'", "'\"'\"'") + "'";
+  fs.mkdirSync(path.join(dir, 'bin'), { recursive: true });
+  fs.copyFileSync(path.join(repoRoot, 'bin', 'gstack-slug'), path.join(dir, 'bin', 'gstack-slug'));
+  fs.chmodSync(path.join(dir, 'bin', 'gstack-slug'), 0o755);
+
+  // Only redirect installed paths. The workflow and score/history rules come
+  // from the real generated skill, without its unrelated shared preamble.
+  const skill = extractSkillSections(path.join(repoRoot, 'health'), HEALTH_EVAL_SECTIONS)
+    .replaceAll('~/.claude/skills/gstack/bin/gstack-slug', path.join(dir, 'bin', 'gstack-slug'))
+    .replaceAll('~/.gstack', gstackHome);
+  fs.writeFileSync(path.join(dir, 'health-SKILL.md'), skill);
+
+  for (const project of ['partial', 'no-tools']) {
+    const projectDir = path.join(dir, project);
+    fs.mkdirSync(projectDir);
+    fs.writeFileSync(path.join(projectDir, '.project.yaml'), `name: ${project}\n`);
+    const historyDir = path.join(gstackHome, 'projects', project);
+    fs.mkdirSync(historyDir, { recursive: true });
+    fs.writeFileSync(path.join(historyDir, 'health-history.jsonl'), PRIOR_HISTORY);
+  }
+  fs.writeFileSync(path.join(dir, 'partial', 'CLAUDE.md'), `# Partial project
+
+## Health Stack
+
+- typecheck: bash ./check-typecheck.sh
+- test: bash ./check-tests.sh
+
+Only the listed tools are available. Lint, dead-code, shell-lint, and GBrain
+tools are not installed. This configuration is final; do not install tools.
+`);
+  fs.writeFileSync(path.join(dir, 'no-tools', 'CLAUDE.md'), `# No-tools project
+
+## Health Stack
+
+No health tools are configured or installed for any category.
+This configuration is final; do not install tools or substitute other checks.
+`);
+  fs.writeFileSync(path.join(dir, 'partial', 'check-typecheck.sh'), `#!/usr/bin/env bash
+printf 'typecheck\\n' >> ${quotedReceipts}
+for ((i = 1; i <= 60; i++)); do
+  printf 'src/file%s.ts(1,1): error TS2322: Type mismatch.\\n' "$i" >&2
+done
+for ((i = 1; i <= 80; i++)); do
+  printf 'Additional diagnostic context %s\\n' "$i"
+done
+exit 2
+`);
+  fs.writeFileSync(path.join(dir, 'partial', 'check-tests.sh'), `#!/usr/bin/env bash
+printf 'test\\n' >> ${quotedReceipts}
+printf '5 pass\\n0 fail\\n'
+`);
+
+  return {
+    dir, gstackHome, receipts, projectFiles: projectSnapshot(dir),
+    prompt: `Read health-SKILL.md and run its /health workflow, Steps 1–6, for
+the partial project first and the no-tools project second. Each has its own
+CLAUDE.md with the final Health Stack configuration. Run commands from the
+corresponding project directory. These are local fixtures without Git remotes.
+
+Save each complete dashboard, details, trends, and recommendations as
+partial-report.md or no-tools-report.md in ${dir}. Keep the dashboard's
+COMPOSITE SCORE label. Existing health histories are available under
+${gstackHome}/projects/<project>/health-history.jsonl; apply the skill's normal
+history rules. GSTACK_HOME already points at this isolated state directory.
+
+Do not modify project files, install tools, or ask to change either Health Stack.
+Only write the reports and any history updates required by the supplied skill.
+Finish after producing both reports.`,
+  };
+}
+
+/** Validate behavior, allowing ordinary Markdown/wording variation in reports. */
+export function healthReportingFailures(fixture: HealthEvalFixture): string[] {
+  const failures: string[] = [];
+  const check = (ok: boolean, message: string) => { if (!ok) failures.push(message); };
+  const read = (file: string) => fs.existsSync(file) ? fs.readFileSync(file, 'utf-8') : '';
+  const plain = (text: string) => text.replace(/[*_`]/g, '');
+  const partial = plain(read(path.join(fixture.dir, 'partial-report.md')));
+  const empty = plain(read(path.join(fixture.dir, 'no-tools-report.md')));
+  const composite = (report: string) => report.match(/composite\s+score\s*[:|]?\s*(N\/A|\d+(?:\.\d+)?)/i)?.[1];
+  check(composite(partial) === '5.6', 'partial coverage must score 5.6, preserving the failing typecheck');
+  check(/partial\s+coverage/i.test(partial), 'numeric score must be labeled as partial coverage');
+  const typecheckRows = partial.split('\n').filter(line => /\btype\s*check\b/i.test(line));
+  check(typecheckRows.some(line => /(?:\b0\s*\/\s*10\b|\|\s*0\s*\|)/.test(line)
+    && /\b(?:critical|fail(?:ed|ure)?|error)\b/i.test(line) && !/\bclean\b/i.test(line)),
+  'dashboard must report the failing typecheck as 0/10, not clean');
+  check(/\b60\s+(?:\w+\s+){0,2}(?:errors|diagnostics|findings)\b/i.test(partial)
+    || /(?:errors|diagnostics|findings)[^\n]{0,20}\b60\b/i.test(partial),
+  'report must count all 60 errors before the final 50 output lines');
+  check(/(?:coverage|checked|executed)/i.test(partial), 'partial report must disclose checked coverage');
+  check(/type\s*check/i.test(partial) && /tests?/i.test(partial), 'checked category names must be visible');
+  for (const category of ['lint', 'dead[ -]?code', 'shell(?:[ -]?lint)?', 'gbrain']) {
+    const unavailable = '(?:unavailable|skipped|not (?:installed|configured|available|found))';
+    check(new RegExp(`${unavailable}[\\s\\S]{0,240}${category}|${category}[^\\n]{0,120}${unavailable}`, 'i').test(partial),
+      `partial report must name unavailable ${category}`);
+  }
+  check(/(?:coverage|categor(?:y|ies)|checks)[\s\S]{0,160}(?:chang|differ|not compar)/i.test(partial)
+    || /(?:chang|differ|not compar)[\s\S]{0,160}(?:coverage|categor(?:y|ies)|checks)/i.test(partial),
+  'partial report must flag changed coverage instead of comparing unlike histories');
+  check(!/[+-]\s*4\.4\b|trend\s*:\s*(?:improving|regressing|worsening)/i.test(partial),
+    'partial report must not calculate a trend delta against different coverage');
+  check(composite(empty)?.toUpperCase() === 'N/A', 'no-tools composite must be N/A');
+  check(/(?:no|zero|0)\s+(?:health\s+)?checks?\s+(?:ran|run|executed|available)|no\s+tools/i.test(empty),
+    'no-tools report must explain that no checks ran');
+
+  const history = read(path.join(fixture.gstackHome, 'projects', 'partial', 'health-history.jsonl'));
+  check(history.startsWith(PRIOR_HISTORY), 'partial run must preserve its prior history row');
+  const rows = history.trim().split('\n').filter(Boolean);
+  check(rows.length === 2, 'partial run must append exactly one history row');
+  try {
+    const row = JSON.parse(rows.at(-1) || '{}');
+    check(row.score === 5.6 && row.typecheck === 0 && row.test === 10,
+      'persisted partial scores must reflect all diagnostics and the actual exit status');
+    check(['lint', 'deadcode', 'shell', 'gbrain'].every(category => row[category] === null),
+      'unavailable categories must persist as null');
+  } catch {
+    failures.push('partial history must remain valid JSONL');
+  }
+  check(read(path.join(fixture.gstackHome, 'projects', 'no-tools', 'health-history.jsonl')) === PRIOR_HISTORY,
+    'no-tools run must leave its history unchanged');
+  const runs = read(fixture.receipts).trim().split('\n');
+  check(runs.includes('typecheck') && runs.includes('test'), 'both configured checkers must actually run');
+  check(JSON.stringify(projectSnapshot(fixture.dir)) === JSON.stringify(fixture.projectFiles),
+    'health must leave project files unchanged');
+  return failures;
+}
+
+/** One attempt records once, after all assertions; throws remain failures. */
+export async function recordHealthAttempt(
+  record: (entry: EvalTestEntry) => void,
+  capture: () => Promise<SkillTestResult>,
+  verify: (result: SkillTestResult) => void,
+): Promise<void> {
+  const started = Date.now();
+  let result: SkillTestResult | undefined;
+  let passed = false;
+  let failure: unknown;
+  try {
+    result = await capture();
+    if (result.exitReason !== 'success') throw new Error(`health capture ended: ${result.exitReason}`);
+    verify(result);
+    passed = true;
+  } catch (error) {
+    failure = error;
+    throw error;
+  } finally {
+    record({
+      name: HEALTH_EVAL_ID, suite: 'health', tier: 'e2e', passed,
+      duration_ms: result?.duration ?? Date.now() - started,
+      cost_usd: result?.costEstimate.estimatedCost ?? 0,
+      turns_used: result?.costEstimate.turnsUsed,
+      tokens_used: result?.costEstimate.estimatedTokens,
+      transcript: result?.transcript,
+      output: [result?.output, failure === undefined ? '' : String(failure)].filter(Boolean).join('\n').slice(-4000),
+      exit_reason: result?.exitReason === 'success' && !passed ? 'assertion_failed' : result?.exitReason ?? 'runner_error',
+      model: result?.model,
+    });
+  }
+}
@@ -582,6 +582,9 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
  // Document-release
  'document-release': ['document-release/**', 'test/skill-e2e-workflow.test.ts'],

+  // /health result capture, coverage, and comparable history (model behavior).
+  'health-reporting': ['health/**', 'test/skill-e2e-health.test.ts', 'test/helpers/health-eval-fixture.ts'],
+
  // Codex (Claude E2E — tests /codex skill via Claude)
  'codex-review': ['codex/**', 'test/skill-e2e-workflow.test.ts'],

@@ -1186,6 +1189,7 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {

  // Document-release — gate (CHANGELOG guardrail)
  'document-release': 'gate',
+  'health-reporting': 'periodic',

  // Codex — periodic (Opus, requires codex CLI)
  'codex-review': 'periodic',