test(coverage): fill three remaining v1.46.0.0 test gaps

Three untested surfaces from the v1.46.0.0 work. All three would have
caught real bugs we shipped (and fixed) on this branch.

1. test/helpers/budget-override.test.ts — 7 tests pin the audit-trail
   contract for EVALS_BUDGET_OVERRIDE_REASON and
   GSTACK_SIZE_BUDGET_OVERRIDE_REASON. Without this, the audit logger
   could silently drop events and overrides become invisible. Tests
   cover: required fields per JSONL line, CI provenance capture
   (CI/GITHUB_ACTIONS/branch/commit), local-runner defaults,
   append-only behavior, missing-directory recovery, and unwritable-
   path resilience (logs warning instead of throwing).

2. test/terse-build.test.ts — 16 tests pin --explain-level=terse
   behavior across the 4 gated resolvers and the composed preamble.
   Default vs terse vs undefined-ctx all asserted. Without this, a
   refactor that breaks the explainLevel threading silently regresses
   the opt-in compression path; the runtime EXPLAIN_LEVEL: terse gate
   still works so users wouldn't notice. Tier-1 invariant pinned
   (terse-only-affects-tier-2+).

3. test/gen-skill-docs-idempotency.test.ts — 2 tests catch the class
   of bug behind the v1.45.0.0 timestamp flap. Two consecutive
   gen-skill-docs runs must produce byte-identical outputs across
   STABLE_OUTPUTS (proactive-suggestions.json, SKILL.md, ship/SKILL.md,
   plan-ceo-review/SKILL.md, office-hours/SKILL.md, gstack/llms.txt).
   --dry-run reports zero stale files after a fresh gen. CI freshness
   regressions surface as test failures BEFORE a PR is opened.

Test plan:
- bun test test/helpers/budget-override.test.ts: 7 pass
- bun test test/terse-build.test.ts: 16 pass
- bun test test/gen-skill-docs-idempotency.test.ts: 2 pass
- Full focused suite (15 test files): 1179 pass, 0 fail (+45 new tests
  vs the pre-fill baseline of 1134)

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-05-26 08:41:08 -07:00
parent c88825dba0
commit 8b94e6d993
3 changed files with 377 additions and 0 deletions
+116
View File
@@ -0,0 +1,116 @@
/**
* Unit tests for budget-override audit logger.
*
* The audit trail is the only check on `EVALS_BUDGET_OVERRIDE_REASON` and
* `GSTACK_SIZE_BUDGET_OVERRIDE_REASON` — if the logger silently drops events,
* overrides become invisible and the budget gates are theater. These tests
* pin the contract: every override produces exactly one JSONL line with
* timestamp + scope + reason + CI provenance.
*/
import { describe, test, expect, beforeEach } from 'bun:test';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { logBudgetOverride } from './budget-override';
const TMP_HOME = fs.mkdtempSync(path.join(os.tmpdir(), 'budget-override-test-'));
process.env.GSTACK_HOME = TMP_HOME;
const AUDIT_PATH = path.join(TMP_HOME, 'analytics', 'spend-overrides.jsonl');
describe('logBudgetOverride', () => {
beforeEach(() => {
// Start each test with a clean audit file
try { fs.unlinkSync(AUDIT_PATH); } catch { /* doesn't exist */ }
});
test('writes one JSONL line per call with required fields', () => {
logBudgetOverride({
scope: 'evals-cost-cap-e2e',
reason: 'model price went up, will rebase the cap next sprint',
details: { tier: 'e2e', cap: 25, observed_cost_usd: 31.4 },
});
expect(fs.existsSync(AUDIT_PATH)).toBe(true);
const lines = fs.readFileSync(AUDIT_PATH, 'utf-8').split('\n').filter(Boolean);
expect(lines.length).toBe(1);
const entry = JSON.parse(lines[0]!);
expect(entry.scope).toBe('evals-cost-cap-e2e');
expect(entry.reason).toBe('model price went up, will rebase the cap next sprint');
expect(entry.details).toEqual({ tier: 'e2e', cap: 25, observed_cost_usd: 31.4 });
expect(typeof entry.timestamp).toBe('string');
expect(entry.timestamp).toMatch(/^\d{4}-\d{2}-\d{2}T/);
});
test('captures CI provenance when CI env is set', () => {
process.env.CI = 'true';
process.env.GITHUB_ACTIONS = 'true';
process.env.GITHUB_REF_NAME = 'feature/x';
process.env.GITHUB_SHA = 'deadbeefcafe1234';
logBudgetOverride({ scope: 'skill-size-budget', reason: 'big diff bake-in' });
const entry = JSON.parse(fs.readFileSync(AUDIT_PATH, 'utf-8').trim());
expect(entry.ci).toBe(true);
expect(entry.runner).toBe('github-actions');
expect(entry.branch).toBe('feature/x');
expect(entry.commit).toBe('deadbeef');
delete process.env.CI;
delete process.env.GITHUB_ACTIONS;
delete process.env.GITHUB_REF_NAME;
delete process.env.GITHUB_SHA;
});
test('defaults provenance to local when CI is unset', () => {
delete process.env.CI;
delete process.env.GITHUB_ACTIONS;
delete process.env.GITHUB_REF_NAME;
delete process.env.GITHUB_SHA;
delete process.env.CI_RUNNER;
delete process.env.CI_COMMIT_REF_NAME;
delete process.env.CI_COMMIT_SHORT_SHA;
logBudgetOverride({ scope: 'skill-size-budget-corpus', reason: 'local dev test' });
const entry = JSON.parse(fs.readFileSync(AUDIT_PATH, 'utf-8').trim());
expect(entry.ci).toBe(false);
expect(entry.runner).toBe('local');
expect(entry.branch).toBe('unknown');
expect(entry.commit).toBe('unknown');
});
test('append-only: multiple calls produce multiple lines', () => {
logBudgetOverride({ scope: 's1', reason: 'r1' });
logBudgetOverride({ scope: 's2', reason: 'r2' });
logBudgetOverride({ scope: 's3', reason: 'r3' });
const lines = fs.readFileSync(AUDIT_PATH, 'utf-8').split('\n').filter(Boolean);
expect(lines.length).toBe(3);
const scopes = lines.map(l => JSON.parse(l).scope);
expect(scopes).toEqual(['s1', 's2', 's3']);
});
test('omits details key when entry.details is absent (uses empty object)', () => {
logBudgetOverride({ scope: 'plain', reason: 'no details' });
const entry = JSON.parse(fs.readFileSync(AUDIT_PATH, 'utf-8').trim());
expect(entry.details).toEqual({});
});
test('never throws even when audit directory is missing — creates it', () => {
// Remove the analytics dir to force mkdir
try { fs.rmSync(path.join(TMP_HOME, 'analytics'), { recursive: true, force: true }); } catch { /* */ }
expect(() => logBudgetOverride({ scope: 'recreate', reason: 'test' })).not.toThrow();
expect(fs.existsSync(AUDIT_PATH)).toBe(true);
});
test('survives an unwritable audit path (logs warning, does not throw)', () => {
// Point GSTACK_HOME at a path inside a file (illegal directory location)
const originalHome = process.env.GSTACK_HOME;
const bogusFile = path.join(TMP_HOME, 'not-a-dir.txt');
fs.writeFileSync(bogusFile, 'just a file');
process.env.GSTACK_HOME = bogusFile;
expect(() => logBudgetOverride({ scope: 'unwritable', reason: 'fs error path' })).not.toThrow();
process.env.GSTACK_HOME = originalHome;
});
});