v1.87.4.0 fix: preserve health failures and disclose coverage (#2882)

* fix: report health failures and coverage accurately * chore: prepare health reporting release 1.87.4.0 * test: restrict routing evaluations to installed project skills * test: stabilize health selection and terminal fixtures
2026-09-16 18:05:31 +02:00 · 2026-09-15 21:35:11 -07:00
parent 85b8c038fc
commit a6b3a57512
15 changed files with 728 additions and 43 deletions
@@ -0,0 +1,56 @@
+/**
+ * Periodic /health behavior: full-log failure counts, visible partial coverage,
+ * comparable histories, and an unscored no-tools run. One bounded capture;
+ * wording follows the model, while process receipts and history are asserted.
+ */
+import { afterAll, expect, test } from 'bun:test';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
+import { describeE2ETier, e2eTierEnabled } from './helpers/e2e-gate';
+import { EvalCollector } from './helpers/eval-store';
+import {
+  createHealthEvalFixture, healthReportingFailures, recordHealthAttempt,
+} from './helpers/health-eval-fixture';
+
+const ROOT = path.resolve(import.meta.dir, '..');
+const describeE2E = describeE2ETier('periodic');
+const collector = e2eTierEnabled('periodic') ? new EvalCollector('e2e') : null;
+
+describeE2E('/health trustworthy reporting (periodic)', () => {
+  test('health-reporting', async () => {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-health-eval-'));
+    try {
+      let fixture: ReturnType<typeof createHealthEvalFixture>;
+      await recordHealthAttempt(
+        entry => collector!.addTest(entry),
+        async () => {
+          fixture = createHealthEvalFixture(dir, ROOT);
+          // The runner resolves its transcript directory on import. Keep a
+          // skipped paid file free of that operator-state lookup/write.
+          const { runSkillTest } = await import('./helpers/session-runner');
+          return runSkillTest({
+            prompt: fixture.prompt,
+            workingDirectory: dir,
+            maxTurns: 18,
+            allowedTools: ['Bash', 'Read', 'Write', 'Glob', 'Grep'],
+            timeout: CAPTURE_MS,
+            testName: 'health-reporting',
+            // The collector retains the transcript in GSTACK_EVAL_DIR. Omit
+            // runId so the runner does not write a global heartbeat/run log.
+            env: { GSTACK_HOME: fixture.gstackHome },
+          });
+        },
+        result => {
+          expect(result.browseErrors).toEqual([]);
+          expect(healthReportingFailures(fixture)).toEqual([]);
+        },
+      );
+    } finally {
+      fs.rmSync(dir, { recursive: true, force: true });
+    }
+  }, CAPTURE_LONG_MS);
+});
+
+afterAll(async () => { await collector?.finalize(); });