diff --git a/browse/src/security-classifier.ts b/browse/src/security-classifier.ts index 6478eaed..62493e56 100644 --- a/browse/src/security-classifier.ts +++ b/browse/src/security-classifier.ts @@ -157,6 +157,17 @@ export function loadTestsavant(onProgress?: (msg: string) => void): Promise void): Promise]*>[\s\S]*?<\/\1>/gi, ' ') // drop script/style bodies entirely + .replace(/<[^>]+>/g, ' ') // drop tags + .replace(/ /g, ' ') + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/\s+/g, ' ') + .trim(); +} + export async function scanPageContent(text: string): Promise { if (!text || text.length === 0) { return { layer: 'testsavant_content', confidence: 0 }; @@ -187,10 +220,16 @@ export async function scanPageContent(text: string): Promise { return { layer: 'testsavant_content', confidence: 0, meta: { degraded: true } }; } try { - // Classify only the first 512 tokens worth of text (~2000 chars). - // Longer inputs get truncated by the tokenizer anyway, but explicit - // slicing avoids token-overflow warnings. - const input = text.slice(0, 2000); + // Normalize to plain text first — the classifier is trained on natural + // language, not HTML markup. A page with an injection buried in tag + // soup won't fire until we strip the noise. + const plain = htmlToPlainText(text); + // Character-level cap to avoid pathological memory use. The pipeline + // applies tokenizer truncation at 512 tokens (the BERT-small context + // limit — enforced via the model_max_length override in loadTestsavant) + // so the 4000-char cap is just a cheap upper bound. Real-world + // injection signals land in the first few hundred tokens anyway. + const input = plain.slice(0, 4000); const raw = await testsavantClassifier(input); const top = Array.isArray(raw) ? raw[0] : raw; const label = top?.label ?? 'SAFE';