From 43e77d84afb103d1d0e8ce56795ac85a3fd02b8f Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 15 Aug 2026 08:05:22 -0700 Subject: [PATCH] test: gate security-bench on SECURITY_BENCH=1, not model-cache existence The existsSync gate ran ~12s of ONNX inference (plus a HuggingFace dataset fetch) on every free-suite run on any dev box that had ever warmed the classifier, while CI (no cache) silently skipped it. Now explicit opt-in: SECURITY_BENCH=1 bun test browse/test/security-bench.test.ts. Co-Authored-By: Claude Fable 5 --- browse/test/security-bench.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/browse/test/security-bench.test.ts b/browse/test/security-bench.test.ts index 69ebec6cc..3571c01ea 100644 --- a/browse/test/security-bench.test.ts +++ b/browse/test/security-bench.test.ts @@ -33,7 +33,11 @@ const MODEL_CACHE = path.join( 'onnx', 'model.onnx', ); -const ML_AVAILABLE = fs.existsSync(MODEL_CACHE); +// Opt-in only (SECURITY_BENCH=1): ~12s of ONNX inference plus a HuggingFace +// dataset fetch. Gating on model-cache existence alone meant every dev box +// that had ever warmed the classifier paid this on every `bun run test`, +// while CI (no cache) silently skipped it — the worst of both. +const ML_AVAILABLE = process.env.SECURITY_BENCH === '1' && fs.existsSync(MODEL_CACHE); const CACHE_DIR = path.join(os.homedir(), '.gstack', 'cache', 'browsesafe-bench-smoke'); const CACHE_FILE = path.join(CACHE_DIR, 'test-rows.json');