mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-11 07:29:00 +02:00
Garry TanandClaude Fable 5
4328748136
The L4b Haiku transcript classifier and the opt-in DeBERTa ensemble
(GSTACK_SECURITY_ENSEMBLE=deberta, a documented 721MB download) had ZERO
production callers since the chat-path agent that invoked them was ripped.
The only live ML path is scanPageContent (testsavant) inside the security
sidecar subprocess. Deleted by import graph:
- security-classifier.ts 614 -> 265 lines: HAIKU_MODEL, checkTranscript,
shouldRunTranscriptCheck, loadDeberta, scanPageContentDeberta, ToolCallInput,
all DEBERTA_* consts + load state. Header now states the live truth
(imported only by security-sidecar-entry.ts). downloadFile kept, name
intact — it is an enumerated egress sink (HF model download).
- security-bunnative.ts + test: a research skeleton self-described as 'NOT a
production replacement', shipped into src/ with zero importers.
- security-bench-ensemble{,-live}.test.ts + the Haiku response fixture: a
paid live-model benchmark for a layer that could not fire. The
security-classifier-tdz test's only case exercised checkTranscript — gone.
- security.ts: layer-model header rewritten to the live architecture;
StatusDetail.layers -> {testsavant, canary}; getStatus() no longer requires
the impossible transcript==='ok' for 'protected' (old on-disk session state
with a transcript key is tolerated on read, never re-emitted).
- security-sidecar-entry.ts needed zero changes: it serializes
getClassifierStatus() verbatim and no consumer read .transcript (verified
in sidecar-client + server.ts).
- BROWSER.md security section matches reality (ensemble knob gone, 112MB not
22MB, sidecar hosting documented). combineVerdict/THRESHOLDS retained as
the pure, tested combiner of record — comments now flag transcript/deberta
votes as producer-less.
Net: 26 pass in security.test.ts incl. a NEW regression test for stale-
transcript disk tolerance; egress-receipt tripwire green.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
102 lines
3.8 KiB
TypeScript
102 lines
3.8 KiB
TypeScript
/**
|
|
* Regression tests for the 4 adversarial findings fixed during /ship:
|
|
*
|
|
* 1. Canary stream-chunk split bypass — rolling-buffer detection across
|
|
* consecutive text_delta / input_json_delta events.
|
|
* 2. Tool-output ensemble rule — single ML classifier >= BLOCK blocks
|
|
* directly when the content is tool output (not user input).
|
|
* 3. escapeHtml quote escaping (unit-level check on the shape we expect).
|
|
* 4. snapshot command added to PAGE_CONTENT_COMMANDS.
|
|
*
|
|
* These tests pin the fixes so future refactors don't silently re-open
|
|
* the bypasses both adversarial reviewers (Claude + Codex) flagged.
|
|
*/
|
|
import { describe, test, expect } from 'bun:test';
|
|
import * as fs from 'fs';
|
|
import * as path from 'path';
|
|
import { combineVerdict, THRESHOLDS } from '../src/security';
|
|
import { PAGE_CONTENT_COMMANDS } from '../src/commands';
|
|
|
|
const REPO_ROOT = path.resolve(__dirname, '..', '..');
|
|
|
|
// canary stream-chunk split detection — tested detectCanaryLeak inside
|
|
// sidebar-agent.ts. Both the chat-stream pipeline and the function are
|
|
// gone (Terminal pane uses an interactive PTY; user keystrokes are the
|
|
// trust source, no chunked LLM stream to canary-scan).
|
|
|
|
describe('tool-output ensemble rule (single-layer BLOCK)', () => {
|
|
test('user-input context: single layer at BLOCK degrades to WARN', () => {
|
|
const result = combineVerdict([
|
|
{ layer: 'testsavant_content', confidence: 0.95 },
|
|
{ layer: 'transcript_classifier', confidence: 0 },
|
|
]);
|
|
expect(result.verdict).toBe('warn');
|
|
expect(result.reason).toBe('single_layer_high');
|
|
});
|
|
|
|
test('tool-output context: single layer at BLOCK blocks directly', () => {
|
|
const result = combineVerdict(
|
|
[
|
|
{ layer: 'testsavant_content', confidence: 0.95 },
|
|
{ layer: 'transcript_classifier', confidence: 0, meta: { degraded: true } },
|
|
],
|
|
{ toolOutput: true },
|
|
);
|
|
expect(result.verdict).toBe('block');
|
|
expect(result.reason).toBe('single_layer_tool_output');
|
|
});
|
|
|
|
test('tool-output context still respects ensemble path when 2 agree', () => {
|
|
const result = combineVerdict(
|
|
[
|
|
{ layer: 'testsavant_content', confidence: 0.80 },
|
|
{ layer: 'transcript_classifier', confidence: 0.75, meta: { verdict: 'block' } },
|
|
],
|
|
{ toolOutput: true },
|
|
);
|
|
expect(result.verdict).toBe('block');
|
|
expect(result.reason).toBe('ensemble_agreement');
|
|
});
|
|
|
|
test('tool-output context: below BLOCK threshold still WARN, not BLOCK', () => {
|
|
const result = combineVerdict(
|
|
[{ layer: 'testsavant_content', confidence: THRESHOLDS.WARN }],
|
|
{ toolOutput: true },
|
|
);
|
|
expect(result.verdict).toBe('warn');
|
|
});
|
|
});
|
|
|
|
describe('sidepanel escapeHtml quote escaping', () => {
|
|
test('escapeHtml helper replaces double + single quotes', () => {
|
|
const src = fs.readFileSync(
|
|
path.join(REPO_ROOT, 'extension', 'sidepanel.js'),
|
|
'utf-8',
|
|
);
|
|
expect(src).toContain(".replace(/\"/g, '"')");
|
|
expect(src).toContain(".replace(/'/g, ''')");
|
|
});
|
|
});
|
|
|
|
describe('snapshot in PAGE_CONTENT_COMMANDS', () => {
|
|
test('snapshot is wrapped by untrusted-content envelope', () => {
|
|
expect(PAGE_CONTENT_COMMANDS.has('snapshot')).toBe(true);
|
|
});
|
|
});
|
|
|
|
// The transcript classifier (Haiku) and its tool_output parameter were
|
|
// removed along with sidebar-agent.ts's tool-result scan pipeline. The
|
|
// combineVerdict tests above retain the transcript_classifier vote-handling
|
|
// coverage — the combiner still accepts those signals even though no live
|
|
// layer produces them.
|
|
|
|
describe('GSTACK_SECURITY_OFF kill switch', () => {
|
|
test('loadTestsavant honors env var early', () => {
|
|
const src = fs.readFileSync(
|
|
path.join(REPO_ROOT, 'browse', 'src', 'security-classifier.ts'),
|
|
'utf-8',
|
|
);
|
|
expect(src).toContain("process.env.GSTACK_SECURITY_OFF === '1'");
|
|
});
|
|
});
|