mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-09 18:28:15 +01:00
test: safety-rule registry ties each held rule to its eval (E3 T2, W1.1)
test/safety-rule-registry.test.ts locates each of the six safety rules by its anchor and fails when the rule's normalized-text hash changes or its eval leaves E2E_TOUCHFILES / E2E_TIERS / E2E_KINDS while the rule remains. Failures name the source file, the anchor, the old and new hash, the eval ID, and the fix (update the hash in test/helpers/safety-rules.ts in the rewording commit). Quarantine does not remove an eval from the registry, so quarantined evals still count as present. It also asserts that no outside-voice resolver grants read-only access to gstack-claude-code and that the codex-host dispatch passes --access none, so the codex-host reviewer stays tool-less.
This commit is contained in:
1 parent
8088bf7ab0
commit
0b0c99c0da
1 file changed
+82
@@ -0,0 +1,82 @@
|
||||
/**
|
||||
* E3 safety-rule registry: every held safety rule stays tied to the paid eval
|
||||
* that measures it. A rule's text cannot change without its hash being
|
||||
* updated in the same rewording commit, and its eval cannot be removed while
|
||||
* the rule remains. Quarantined evals still count as present.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { E2E_KINDS, E2E_TIERS, E2E_TOUCHFILES } from './helpers/touchfiles-data';
|
||||
import { matchGlob } from './helpers/test-selection';
|
||||
import { SAFETY_RULES, locateRule, normalizeRuleText, removeRule, ruleHash } from './helpers/safety-rules';
|
||||
import { outsideVoiceCommand } from '../scripts/resolvers/outside-voice';
|
||||
import { HOST_PATHS, type TemplateContext } from '../scripts/resolvers/types';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const REGISTRY = 'test/helpers/safety-rules.ts';
|
||||
|
||||
describe('safety-rule registry', () => {
|
||||
for (const rule of SAFETY_RULES) {
|
||||
test(`${rule.id}: rule located in ${rule.source}`, () => {
|
||||
const text = fs.readFileSync(path.join(ROOT, rule.source), 'utf8');
|
||||
const hit = locateRule(text, rule);
|
||||
if (!hit) {
|
||||
throw new Error(`${rule.id}: anchor ${rule.match} not found in ${rule.source}${rule.section ? ` under "${rule.section}"` : ''}. `
|
||||
+ `If the rule was deleted, remove its row from ${REGISTRY} and retire its eval ${rule.evalId}; `
|
||||
+ `if it moved, update the anchor in ${REGISTRY} in the same commit.`);
|
||||
}
|
||||
const current = ruleHash(hit.paragraph);
|
||||
if (current !== rule.sha256) {
|
||||
throw new Error(`${rule.id}: rule text changed in ${rule.source} (anchor ${rule.match}).\n`
|
||||
+ ` old hash: ${rule.sha256}\n new hash: ${current}\n eval: ${rule.evalId}\n`
|
||||
+ `A rewording lands only after ${rule.evalId} discriminates and its "after" runs are clean on every render that carries the rule. `
|
||||
+ `Then update the hash in ${REGISTRY} in the rewording commit.`);
|
||||
}
|
||||
});
|
||||
|
||||
test(`${rule.id}: eval ${rule.evalId} is registered`, () => {
|
||||
const missing = [
|
||||
!(rule.evalId in E2E_TOUCHFILES) && 'E2E_TOUCHFILES',
|
||||
!(rule.evalId in E2E_TIERS) && 'E2E_TIERS',
|
||||
!(rule.evalId in E2E_KINDS) && 'E2E_KINDS',
|
||||
].filter(Boolean);
|
||||
if (missing.length) {
|
||||
throw new Error(`${rule.id}: eval ${rule.evalId} is missing from ${missing.join(', ')} (test/helpers/touchfiles-data.ts) `
|
||||
+ `while its rule remains in ${rule.source}. Restore the eval, or remove the rule and its row in ${REGISTRY} together.`);
|
||||
}
|
||||
const covered = E2E_TOUCHFILES[rule.evalId]!.some(pattern => matchGlob(rule.source, pattern));
|
||||
if (!covered) throw new Error(`${rule.id}: E2E_TOUCHFILES['${rule.evalId}'] does not cover ${rule.source}, so a rewording would not select its eval. Add the source to that entry.`);
|
||||
});
|
||||
}
|
||||
|
||||
test('anchors are unique and rule-removed arms delete exactly the rule', () => {
|
||||
expect(new Set(SAFETY_RULES.map(r => r.id)).size).toBe(SAFETY_RULES.length);
|
||||
expect(new Set(SAFETY_RULES.map(r => r.evalId)).size).toBe(SAFETY_RULES.length);
|
||||
for (const rule of SAFETY_RULES) {
|
||||
const text = fs.readFileSync(path.join(ROOT, rule.source), 'utf8');
|
||||
const removed = removeRule(text, rule);
|
||||
const hit = locateRule(text, rule)!;
|
||||
expect(removed.length).toBeLessThan(text.length);
|
||||
expect(normalizeRuleText(removed)).not.toContain(normalizeRuleText(hit.paragraph));
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('codex-host outside reviewer stays tool-less', () => {
|
||||
test('no outside-voice resolver grants read-only access to gstack-claude-code', () => {
|
||||
const dir = path.join(ROOT, 'scripts', 'resolvers');
|
||||
const offenders = fs.readdirSync(dir, { recursive: true, encoding: 'utf8' })
|
||||
.filter(file => file.endsWith('.ts'))
|
||||
.filter(file => /access\s*:\s*['"`]read-only['"`]/.test(fs.readFileSync(path.join(dir, file), 'utf8')));
|
||||
expect(offenders).toEqual([]);
|
||||
});
|
||||
|
||||
test('the codex-host dispatch runs Claude Code with --access none', () => {
|
||||
const ctx: TemplateContext = { skillName: 'review', tmplPath: 'review/SKILL.md.tmpl', host: 'codex', paths: HOST_PATHS.codex };
|
||||
const command = outsideVoiceCommand(ctx, { timeoutMs: 1000 });
|
||||
expect(command).toContain('gstack-claude-code');
|
||||
expect(command).toContain('--access none');
|
||||
expect(command).not.toContain('--access read-only');
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user