mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-19 03:12:21 +02:00
fix(security): delete the dead ML layers — transcript classifier and DeBERTa ensemble
The L4b Haiku transcript classifier and the opt-in DeBERTa ensemble
(GSTACK_SECURITY_ENSEMBLE=deberta, a documented 721MB download) had ZERO
production callers since the chat-path agent that invoked them was ripped.
The only live ML path is scanPageContent (testsavant) inside the security
sidecar subprocess. Deleted by import graph:
- security-classifier.ts 614 -> 265 lines: HAIKU_MODEL, checkTranscript,
shouldRunTranscriptCheck, loadDeberta, scanPageContentDeberta, ToolCallInput,
all DEBERTA_* consts + load state. Header now states the live truth
(imported only by security-sidecar-entry.ts). downloadFile kept, name
intact — it is an enumerated egress sink (HF model download).
- security-bunnative.ts + test: a research skeleton self-described as 'NOT a
production replacement', shipped into src/ with zero importers.
- security-bench-ensemble{,-live}.test.ts + the Haiku response fixture: a
paid live-model benchmark for a layer that could not fire. The
security-classifier-tdz test's only case exercised checkTranscript — gone.
- security.ts: layer-model header rewritten to the live architecture;
StatusDetail.layers -> {testsavant, canary}; getStatus() no longer requires
the impossible transcript==='ok' for 'protected' (old on-disk session state
with a transcript key is tolerated on read, never re-emitted).
- security-sidecar-entry.ts needed zero changes: it serializes
getClassifierStatus() verbatim and no consumer read .transcript (verified
in sidecar-client + server.ts).
- BROWSER.md security section matches reality (ensemble knob gone, 112MB not
22MB, sidecar hosting documented). combineVerdict/THRESHOLDS retained as
the pure, tested combiner of record — comments now flag transcript/deberta
votes as producer-less.
Net: 26 pass in security.test.ts incl. a NEW regression test for stale-
transcript disk tolerance; egress-receipt tripwire green.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
ef186cccb3
commit
4328748136
@@ -1,91 +1,29 @@
|
||||
/**
|
||||
* Unit tests for browse/src/security-classifier.ts pure functions.
|
||||
*
|
||||
* Scope: functions that do NOT require model download, claude CLI, or
|
||||
* network access. Model-dependent behavior (loadTestsavant inference,
|
||||
* checkTranscript Haiku calls) belongs in a smoke harness that pulls
|
||||
* the cached model — filed as a P1 follow-up.
|
||||
* Scope: functions that do NOT require model download or network access.
|
||||
* Model-dependent behavior (loadTestsavant inference via scanPageContent)
|
||||
* is covered by security-bench.test.ts and security-live-playwright.test.ts,
|
||||
* which gate on the cached model being present.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import {
|
||||
shouldRunTranscriptCheck,
|
||||
getClassifierStatus,
|
||||
} from '../src/security-classifier';
|
||||
import { THRESHOLDS, type LayerSignal } from '../src/security';
|
||||
|
||||
describe('shouldRunTranscriptCheck — Haiku gating optimization', () => {
|
||||
test('returns false when no layer has fired at >= LOG_ONLY', () => {
|
||||
// Clean pre-tool-call: no classifier saw anything interesting.
|
||||
// Skipping Haiku here is the 70% savings described in plan §E1.
|
||||
const signals: LayerSignal[] = [
|
||||
{ layer: 'testsavant_content', confidence: 0 },
|
||||
{ layer: 'aria_regex', confidence: 0 },
|
||||
];
|
||||
expect(shouldRunTranscriptCheck(signals)).toBe(false);
|
||||
});
|
||||
|
||||
test('returns true when testsavant_content fires at LOG_ONLY threshold', () => {
|
||||
// Exactly at 0.40 — should trigger Haiku follow-up.
|
||||
const signals: LayerSignal[] = [
|
||||
{ layer: 'testsavant_content', confidence: THRESHOLDS.LOG_ONLY },
|
||||
];
|
||||
expect(shouldRunTranscriptCheck(signals)).toBe(true);
|
||||
});
|
||||
|
||||
test('returns true when aria_regex alone fires above LOG_ONLY', () => {
|
||||
// Regex hit on its own is suspicious enough to warrant Haiku second opinion.
|
||||
const signals: LayerSignal[] = [
|
||||
{ layer: 'aria_regex', confidence: 0.6 },
|
||||
];
|
||||
expect(shouldRunTranscriptCheck(signals)).toBe(true);
|
||||
});
|
||||
|
||||
test('does NOT gate on transcript_classifier itself (no recursion)', () => {
|
||||
// If the transcript classifier already reported (e.g., prior tool call),
|
||||
// the new tool call shouldn't re-trigger Haiku based on the previous
|
||||
// transcript signal alone — we need a fresh content signal. This
|
||||
// prevents feedback loops where one Haiku hit forever gates future calls.
|
||||
const signals: LayerSignal[] = [
|
||||
{ layer: 'transcript_classifier', confidence: 0.9 },
|
||||
];
|
||||
expect(shouldRunTranscriptCheck(signals)).toBe(false);
|
||||
});
|
||||
|
||||
test('empty signals list returns false (no reason to call Haiku)', () => {
|
||||
expect(shouldRunTranscriptCheck([])).toBe(false);
|
||||
});
|
||||
|
||||
test('confidence just below LOG_ONLY → false', () => {
|
||||
const signals: LayerSignal[] = [
|
||||
{ layer: 'testsavant_content', confidence: THRESHOLDS.LOG_ONLY - 0.01 },
|
||||
];
|
||||
expect(shouldRunTranscriptCheck(signals)).toBe(false);
|
||||
});
|
||||
|
||||
test('mixed low signals — any one >= LOG_ONLY gates true', () => {
|
||||
const signals: LayerSignal[] = [
|
||||
{ layer: 'testsavant_content', confidence: 0.1 },
|
||||
{ layer: 'aria_regex', confidence: 0.45 }, // just above LOG_ONLY
|
||||
];
|
||||
expect(shouldRunTranscriptCheck(signals)).toBe(true);
|
||||
});
|
||||
});
|
||||
import { getClassifierStatus } from '../src/security-classifier';
|
||||
|
||||
describe('getClassifierStatus — pre-load state', () => {
|
||||
test('returns testsavant=off before loadTestsavant has been called', () => {
|
||||
// Before any warmup has started, both classifiers report off.
|
||||
// Before any warmup has started, the classifier reports off.
|
||||
// (This test runs in fresh-module state; if another test already
|
||||
// loaded the classifier, status would be 'ok' — but this file runs
|
||||
// before model loads in typical CI.)
|
||||
const s = getClassifierStatus();
|
||||
// transcript starts 'off' until first checkHaikuAvailable() call
|
||||
expect(['ok', 'degraded', 'off']).toContain(s.testsavant);
|
||||
expect(['ok', 'degraded', 'off']).toContain(s.transcript);
|
||||
});
|
||||
|
||||
test('status shape contract — exactly two keys', () => {
|
||||
test('status shape contract — exactly one key (testsavant)', () => {
|
||||
// The sidecar's `status` op serializes this object verbatim onto the
|
||||
// NDJSON wire — pin the shape so accidental additions are deliberate.
|
||||
const s = getClassifierStatus();
|
||||
expect(Object.keys(s).sort()).toEqual(['testsavant', 'transcript']);
|
||||
expect(Object.keys(s).sort()).toEqual(['testsavant']);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user