From 85d5cbd65702426be01b70b12a9f3b937636b807 Mon Sep 17 00:00:00 2001 From: ajmallesh Date: Wed, 26 Aug 2026 20:18:44 -0700 Subject: [PATCH] feat(worker): disclose scan coverage and make reporting auditable Build on the retry-safe finalization foundation to preserve correct identities, source locations, scan dates, partial-coverage limitations, and consistent report JSON, Markdown, SARIF, and PDF output. Report Agentic SAST, reconciliation wall-clock time, stage usage, retry spend, and background work without duplicate or hardcoded totals. Keep report findings canonical, drop cross-class restatements, name enrichment losses, and render the executive-summary narrative in the PDF. --- apps/cli/src/scan/derive.ts | 32 ++- apps/cli/src/scan/pipeline.ts | 2 + apps/cli/src/scan/safe-fields.ts | 54 +++- apps/cli/src/scan/status-json.ts | 9 +- apps/worker/prompts/report-executive.txt | 61 +++- .../src/ai/sast/capella/sarif-exporter.ts | 95 +++++- .../src/ai/sast/capella/stages/export.ts | 18 +- .../ai/sast/capella/temporal/activities.ts | 2 +- .../sast/capella/temporal/activity-types.ts | 2 +- .../src/ai/sast/capella/temporal/workflow.ts | 2 +- apps/worker/src/audit/audit-session.ts | 22 +- apps/worker/src/audit/metrics-tracker.ts | 271 ++++++++++++++++-- apps/worker/src/audit/operational-summary.ts | 136 +++++++++ apps/worker/src/audit/workflow-logger.ts | 64 ++++- .../src/collectors/finding-collector.ts | 22 +- apps/worker/src/services/agent-execution.ts | 12 + apps/worker/src/services/agent-git-paths.ts | 26 +- .../worker/src/services/code-location-join.ts | 101 +++++-- apps/worker/src/services/compaction-core.ts | 6 + apps/worker/src/services/finding-order.ts | 10 +- apps/worker/src/services/findings-renderer.ts | 2 + apps/worker/src/services/pdf-renderer.ts | 2 + apps/worker/src/services/prompt-manager.ts | 30 +- apps/worker/src/services/queue-validation.ts | 7 +- apps/worker/src/services/renumber-core.ts | 17 +- .../worker/src/services/report-checkpoints.ts | 26 +- .../src/services/report-finalization.ts | 34 ++- .../src/services/report-json-adapter.ts | 23 +- .../src/services/report-output-schema.ts | 16 +- .../src/services/report-output-surface.ts | 14 +- apps/worker/src/services/report-renderer.ts | 105 +++++-- apps/worker/src/services/reporting.ts | 28 +- apps/worker/src/services/sarif-renderer.ts | 59 +++- .../src/services/validate-authentication.ts | 5 + apps/worker/src/temporal/activities.ts | 70 +++-- apps/worker/src/temporal/shared.ts | 11 +- apps/worker/src/temporal/summary-mapper.ts | 32 ++- apps/worker/src/temporal/worker.ts | 21 +- apps/worker/src/temporal/workflows.ts | 50 +++- apps/worker/templates/typst/report.typ | 22 +- 40 files changed, 1336 insertions(+), 185 deletions(-) create mode 100644 apps/worker/src/audit/operational-summary.ts diff --git a/apps/cli/src/scan/derive.ts b/apps/cli/src/scan/derive.ts index 5c24585c..0e5fa6f5 100644 --- a/apps/cli/src/scan/derive.ts +++ b/apps/cli/src/scan/derive.ts @@ -248,19 +248,35 @@ export function derivePipeline(input: RenderInput, now: number): DerivedPhase[] }; }); - // The synthetic phase appears only when there is operational work to show, so a scan + // The synthetic phase(s) appear only when there is operational work to show, so a scan // with no recorded operational stages keeps the plain agent tree. if (operationalAgents.length === 0) return agentPhases; - return [ - ...agentPhases, - { + + // Agentic SAST is a pluggable analysis engine — a peer to the pentest, not background plumbing — + // so it stands in its own phase; reconciliation and report steps remain under "Background work". + const engineAgents = operationalAgents.filter((agent) => operationFamilyKey(agent.name) === 'agentic-sast'); + const backgroundAgents = operationalAgents.filter((agent) => operationFamilyKey(agent.name) !== 'agentic-sast'); + + const syntheticPhases: DerivedPhase[] = []; + if (engineAgents.length > 0) { + syntheticPhases.push({ + key: 'analysis-engines', + label: 'Analysis Engines', + parallel: true, + state: phaseGlyphState(engineAgents.map((operation) => operation.state)), + agents: engineAgents, + }); + } + if (backgroundAgents.length > 0) { + syntheticPhases.push({ key: 'operational-work', label: 'Background work', parallel: true, - state: phaseGlyphState(operationalAgents.map((operation) => operation.state)), - agents: operationalAgents, - }, - ]; + state: phaseGlyphState(backgroundAgents.map((operation) => operation.state)), + agents: backgroundAgents, + }); + } + return [...agentPhases, ...syntheticPhases]; } export { agentError }; diff --git a/apps/cli/src/scan/pipeline.ts b/apps/cli/src/scan/pipeline.ts index a18bbe3e..7b0a5d4d 100644 --- a/apps/cli/src/scan/pipeline.ts +++ b/apps/cli/src/scan/pipeline.ts @@ -344,6 +344,8 @@ export interface PipelineState { readonly failedStageLabel?: string; readonly error?: string; readonly errorCode?: string; + /** Usage-accounting warnings projected by the worker; empty when the ledger reconciled. */ + readonly warnings?: readonly string[]; }; readonly nonFatalFailures?: { readonly phase: string; readonly error: string }[]; /** Ordered durable degradation reasons with safe messages; empty or absent for full success. */ diff --git a/apps/cli/src/scan/safe-fields.ts b/apps/cli/src/scan/safe-fields.ts index 60ef5aa3..1d0bed65 100644 --- a/apps/cli/src/scan/safe-fields.ts +++ b/apps/cli/src/scan/safe-fields.ts @@ -1,4 +1,12 @@ -/** Closed-field projection for Temporal values displayed by the CLI. */ +/** + * Closed-field projection for Temporal values displayed by the CLI. + * + * PipelineState travels through Temporal from a worker container this process does not + * control, so free-text fields are treated as unvetted: this module either matches a + * value against a known closed set (safe to print as-is) or collapses it to a fixed, + * bounded message. A value with no case here should fail closed to something generic, + * never pass through untouched. + */ import type { PartialReasonView, PipelineState } from './pipeline.js'; @@ -49,6 +57,23 @@ const CAPELLA_FAILURE_MESSAGES = new Set([ 'Agentic SAST had not finished when the scan stopped.', ]); +// Mirrors apps/worker/src/types/errors.ts. The CLI cannot import from the worker package, +// so keep this exact closed set in sync with ProviderFailureCategory. +const PROVIDER_FAILURE_CATEGORIES = new Set([ + 'rate_limit', + 'overloaded', + 'transport', + 'context_limit', + 'quota', + 'authentication', + 'configuration', + 'unknown', +]); + +function isProviderFailureCategory(value: unknown): value is string { + return typeof value === 'string' && PROVIDER_FAILURE_CATEGORIES.has(value); +} + const OPERATION_LABELS = new Set([ 'Agentic SAST', 'Miscellaneous findings', @@ -141,12 +166,27 @@ export function safePartialReasons(reasons: readonly PartialReasonView[]): reado }); } +/** Upper bounds on the warning array crossing into cli.status.json, so a malformed state cannot bloat it. */ +const MAX_AGENTIC_SAST_WARNINGS = 20; +const MAX_AGENTIC_SAST_WARNING_LENGTH = 2_000; + +/** Sanitize the worker's usage-accounting warnings: strings only, bounded count and length. */ +function safeAgenticSastWarnings(value: PipelineState['agenticSast']): readonly string[] { + const warnings = value?.warnings; + if (!Array.isArray(warnings)) return []; + return warnings + .filter((warning): warning is string => typeof warning === 'string') + .slice(0, MAX_AGENTIC_SAST_WARNINGS) + .map((warning) => warning.slice(0, MAX_AGENTIC_SAST_WARNING_LENGTH)); +} + export function safeAgenticSast(value: PipelineState['agenticSast']): | { readonly status: string; readonly failedStageLabel?: string; readonly error?: string; readonly errorCode?: string; + readonly warnings: readonly string[]; } | undefined { if (value === undefined || !['disabled', 'running', 'succeeded', 'failed'].includes(value.status)) return undefined; @@ -158,12 +198,16 @@ export function safeAgenticSast(value: PipelineState['agenticSast']): error = 'An agentic SAST step failed.'; } const errorCode = - value.errorCode !== undefined && /^[A-Z][A-Z0-9_]{0,63}$/u.test(value.errorCode) ? value.errorCode : undefined; + value.errorCode !== undefined && + (/^[A-Z][A-Z0-9_]{0,63}$/u.test(value.errorCode) || isProviderFailureCategory(value.errorCode)) + ? value.errorCode + : undefined; return { status: value.status, ...(failedStageLabel !== undefined && { failedStageLabel }), ...(error !== undefined && { error }), ...(errorCode !== undefined && { errorCode }), + warnings: safeAgenticSastWarnings(value), }; } @@ -183,6 +227,11 @@ export function safeOperationKey(value: string): string { return 'background-task'; } +/** + * A workspace or workflow id is printed straight into the progress display, so this + * confines it to a plain identifier charset before that happens: no control or escape + * characters survive to reach the terminal. + */ export function safeCliIdentifier(value: string): string { return /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u.test(value) ? value : 'unknown'; } @@ -210,6 +259,7 @@ export function safeFailureDetail(hasFailure: boolean): string | undefined { return hasFailure ? 'This scan step could not be completed.' : undefined; } +/** Same closed-set trade-off as safeFailureDetail, for the scan-level (not per-agent) failure. */ export function safeTerminalFailure(hasFailure: boolean): string | undefined { return hasFailure ? 'The scan could not be completed.' : undefined; } diff --git a/apps/cli/src/scan/status-json.ts b/apps/cli/src/scan/status-json.ts index 9702f798..1616e9e3 100644 --- a/apps/cli/src/scan/status-json.ts +++ b/apps/cli/src/scan/status-json.ts @@ -38,7 +38,13 @@ export interface StatusJson { /** Ordered durable degradation reasons with safe messages; present only when non-empty. */ readonly partialReasons?: readonly PartialReasonView[]; /** Agentic SAST outcome, with the worker's sanitized failure sentence and bounded code. */ - readonly agenticSast?: { readonly status: string; readonly error?: string; readonly errorCode?: string }; + readonly agenticSast?: { + readonly status: string; + readonly error?: string; + readonly errorCode?: string; + /** Usage-accounting warnings; always present (empty when the ledger reconciled) so it is never null. */ + readonly warnings: readonly string[]; + }; /** False when operational (Capella/reconciliation) spend is known to be incomplete. */ readonly usageAccountingComplete?: boolean; readonly phases: readonly DerivedPhase[]; @@ -90,6 +96,7 @@ export function toStatusJson(input: RenderInput, now: number): StatusJson { status: agenticSast.status, ...(agenticSast.error !== undefined && { error: agenticSast.error }), ...(agenticSast.errorCode !== undefined && { errorCode: agenticSast.errorCode }), + warnings: [...agenticSast.warnings], }, }), ...(usageAccountingComplete !== undefined && { usageAccountingComplete }), diff --git a/apps/worker/prompts/report-executive.txt b/apps/worker/prompts/report-executive.txt index 96ee039c..d93b5eef 100644 --- a/apps/worker/prompts/report-executive.txt +++ b/apps/worker/prompts/report-executive.txt @@ -11,7 +11,7 @@ You are the Security Report Writer for a multi-agent security assessment pipelin Record all findings as structured data using the `add_finding` tool. You do NOT write a markdown report — a downstream renderer produces the report from your structured output. 1. **Orient yourself** — read the assembled deliverables and understand what was found (see ). -2. **Filter and clean** — identify real findings, remove noise, rewrite weak titles (see ). +2. **Filter and clean** — identify real findings, remove noise, rewrite weak titles, drop restatements of findings already selected (see ). 3. **Record report metadata** — run `set-report-meta` once (see ). 4. **Record each finding** — call `add_finding` once per finding (see ). @@ -45,7 +45,8 @@ Read these files: - `.shannon/deliverables/recon_deliverable.md` — Attack surface mapping and endpoint discovery (for executive summary context). ### Vulnerability ID patterns -Findings have IDs matching `[TYPE]-VULN-[NUMBER]` (e.g., INJ-VULN-01, AUTH-VULN-03). +Findings have stable report IDs matching `[TYPE]-[NUMBER]` (e.g., INJ-01, AUTH-03, MISC-01). +Preserve each ID exactly as supplied. Do not mint a new ID or insert a `VULN` segment. ### Context Target URL: {{WEB_URL}} @@ -62,7 +63,7 @@ Exploitation: {{EXPLOITATION}} Read through the concatenated report and identify which vulnerability entries to record. Apply these rules: ### KEEP — these are real findings to record via `add_finding` -- Vulnerability entries under `## {{REPORT_VULN_SUBHEADING}}` sections with IDs matching `### [TYPE]-VULN-[NUMBER]` +- Vulnerability entries under `## {{REPORT_VULN_SUBHEADING}}` sections with IDs matching `### [TYPE]-[NUMBER]` {{REPORT_FILTER_RULES}} ### SKIP — do not record these @@ -73,9 +74,30 @@ Read through the concatenated report and identify which vulnerability entries to - False positives sections - Introductory text, vulnerability counts, or meta-commentary without vulnerability IDs - Any section that does not contain a finding with a valid vulnerability ID +- Entries that restate a finding you have already selected (see DROP below, applied to cleaned titles) ### Title cleanup -If a finding's title (the text after the colon in `### TYPE-VULN-NN: Title`) is only a short category label rather than a descriptive phrase, rewrite it to a concise descriptor derived from the finding's "Vulnerable location" and "Overview" fields. Use the improved title when calling `add_finding`. +If a finding's title (the text after the colon in `### : Title`, whatever the ID form) is only a short category label rather than a descriptive phrase, rewrite it to a concise descriptor derived from the finding's "Vulnerable location" and "Overview" fields. Use the improved title when calling `add_finding`. + +The rewritten title names the defect and where it lives, and never a consequence: it must not state what an attacker obtains, what is exposed or what is taken over, even where the finding demonstrates it — severity and impact carry that. Do not introduce hedges ("Theoretical", "Potential", "Precondition"). Where a supplied title already states a consequence, remove it. This cleanup only ever makes a title more precise, never louder. + +Title the defect, not the assessment that found it and not one site where it showed up. Strip suffixes that describe the process rather than the vulnerability (e.g. `— Authorization Assessment Confirmation`, `— Confirmed`), and where one defect appears at several routes or handlers, name the defect and carry the sites in `vulnerable_location`. + +Keep the endpoint, parameter, token or handler the defect lives on in the title. Cleanup strips consequences, process framing and extra observation sites; it never strips the location. `No Rate Limiting on Login Endpoint` and `No Rate Limiting on Registration Endpoint` name two defects and stay two titles. + +Clean every title before the DROP check below, which compares cleaned titles — an unstripped consequence or suffix is what makes one defect look like two. + +### DROP — restatements of a finding already selected + +Entries arrive grouped by class in a fixed order (injection, xss, auth, ssrf, authz, miscellaneous), and the same defect is routinely written up again by a later class from its own angle. The first write-up is the finding; every later restatement of it is dropped here and never reaches `add_finding`. + +Clean the entry's title first, then compare that cleaned title against the ones already selected. Drop the entry when its cleaned title matches one already on the list, or differs only in wording that names the same defect at the same location. Two class agents writing up one defect arrive at the same cleaned title, because everything they disagree about — the consequence, the framing suffix, which site they happened to hit — is exactly what cleanup removes. + +Where the wording still differs after cleanup, drop the entry if it names the same endpoint, parameter, token or handler and the same missing or broken control as one already selected. Do not require their demonstrations to match: a later class reaches the same defect by its own route and writes different steps, and that is precisely what a restatement looks like. + +Keep a running list of the cleaned titles selected so far. Check each new entry against that short list only. Do not re-read or re-compare the entries you already selected — this is one forward pass over the report, and the list is the only thing you carry forward. + +Dropping a restatement never drops coverage. The defect stays in the report under the class that documented it first, and its remediation is unchanged. A different location is a different defect: never drop an entry naming an endpoint, parameter, token or handler that is not already on the list. Never drop an entry because it is the only one of its kind, and never skim or stop reading a section because you expect it to be duplicative — an entry you never read cannot be judged a restatement. @@ -83,30 +105,41 @@ Run `set-report-meta` once before recording any individual findings (see -- `executive_summary`: 2-3 sentences summarizing the security posture for technical leadership (CTOs, CISOs, Engineering VPs). Must include the target URL and assessment date. Provide a high-level characterization based on the findings — severity distribution, most critical issues, and overall risk demonstrated by exploitation. If no vulnerabilities were confirmed in the assessed classes, state that scope clearly. A clean report is valid only when no block is present. If that block is present, explicitly say the listed classes were not assessed and do not assert they are free of vulnerabilities. +- `executive_summary`: 2-3 sentences summarizing the security posture for technical leadership (CTOs, CISOs, Engineering VPs). Must include the target URL and copy the assessment date `{{ASSESSMENT_DATE}}` exactly. Provide a high-level characterization based on the findings — severity distribution, most critical issues, and overall risk demonstrated by exploitation. If no vulnerabilities were confirmed in the assessed classes, state that scope clearly. A clean report is valid only when no block is present. If that block is present, explicitly say the listed classes were not assessed and do not assert they are free of vulnerabilities. -- `executive_summary`: 2-3 sentences summarizing the security posture for technical leadership (CTOs, CISOs, Engineering VPs). Must include the target URL and assessment date. Provide a high-level characterization based on the findings — severity and confidence distribution, the most serious weaknesses identified, and overall risk. State plainly that this was an analysis-only assessment and that no finding was confirmed by exploitation; do not describe risk as demonstrated or proven, and present severity as assessed rather than measured. If no vulnerabilities were identified in the assessed classes, state that scope clearly. A clean report is valid only when no block is present. If that block is present, explicitly say the listed classes were not assessed and do not assert they are free of vulnerabilities. +- `executive_summary`: 2-3 sentences summarizing the security posture for technical leadership (CTOs, CISOs, Engineering VPs). Must include the target URL and copy the assessment date `{{ASSESSMENT_DATE}}` exactly. Provide a high-level characterization based on the findings — severity and confidence distribution, the most serious weaknesses identified, and overall risk. State plainly that this was an analysis-only assessment and that no finding was confirmed by exploitation; do not describe risk as demonstrated or proven, and present severity as assessed rather than measured. If no vulnerabilities were identified in the assessed classes, state that scope clearly. A clean report is valid only when no block is present. If that block is present, explicitly say the listed classes were not assessed and do not assert they are free of vulnerabilities. -For each finding identified in , call `add_finding` once. +For each finding selected in — restatements already dropped there — call `add_finding` once. -Record findings in the order they appear in the concatenated report (which groups by vulnerability class: injection, xss, auth, ssrf, authz). +Record findings in the order they appear in the concatenated report. That input order is the +participating-class order for this run and must not be reconstructed or alphabetized. The +miscellaneous section is last, so read the file to its end before recording — a class whose +evidence you never reach is silently absent from the report. -Each `finding_id` may only be recorded once — duplicate calls are rejected. +Each `finding_id` may only be recorded once — duplicate calls are rejected. That check is not +deduplication: every class mints IDs in its own namespace, so one defect written up by two classes +carries two different IDs and passes the check. Restatements are stopped by the DROP rule in +, never by the tool. + +Carry the short list of cleaned titles from forward as you record, and check +each entry against it before calling `add_finding`. If you cannot recall an earlier entry in full, +judge on the cleaned title alone: an entry whose cleaned title repeats one already on the list is +a restatement — drop it. ### How to fill in each field Map the finding's content from the per-class deliverable sections to `add_finding` fields: -- `finding_id`: The vulnerability ID exactly as it appears (e.g., `"INJ-VULN-01"`, `"AUTH-VULN-07"`) +- `finding_id`: The stable vulnerability ID exactly as it appears (e.g., `"INJ-01"`, `"AUTH-07"`, `"MISC-01"`) - `title`: The cleaned-up title (see title cleanup rules in ) -- `category`: Derived from the finding type prefix — `INJ` → `"Injection"`, `XSS` → `"XSS"`, `AUTH` → `"Authentication"`, `AUTHZ` → `"Authorization"`, `SSRF` → `"SSRF"` +- `category`: Derived from the finding type prefix — `INJ` → `"Injection"`, `XSS` → `"XSS"`, `AUTH` → `"Authentication"`, `AUTHZ` → `"Authorization"`, `SSRF` → `"SSRF"`, `MISC` → `"Miscellaneous"` - `severity`: From the finding's "Severity" field. Use as-is; do not reassess. @@ -166,13 +199,15 @@ If no valid findings exist after filtering, do not call `add_finding` at all. Th - **No Speculation:** Only record findings that appear in the deliverables with valid vulnerability IDs. Do not add your own assessments. - **OWASP 2025:** Map all findings to OWASP Top 10 (2025) categories. - **Remediation Quality:** Provide specific, actionable remediation — code-level or configuration-level fixes. Avoid generic advice like "validate input" or "follow best practices". +- **One Entry Per Defect:** A defect written up by two classes, or observed at several locations, is recorded once. Restatements are dropped in ; the tool's `finding_id` check does not catch them. Before finalizing, verify: - [ ] Did I run `set-report-meta` exactly once with target, assessment_date, scope, and executive_summary? -- [ ] For each valid finding in the deliverables, did I call `add_finding` exactly once with the correct `finding_id`? +- [ ] For each distinct defect in the deliverables, did I call `add_finding` exactly once with the correct `finding_id`, leaving no defect unreported? +- [ ] Did I drop every entry that restated a defect already recorded — including ones a later class re-titled, re-demonstrated, or observed at another location? - [ ] Did I skip all entries from "Potential Vulnerabilities (Validation Blocked)", false positives, and meta-commentary sections? diff --git a/apps/worker/src/ai/sast/capella/sarif-exporter.ts b/apps/worker/src/ai/sast/capella/sarif-exporter.ts index 4f4fd94c..585948b9 100644 --- a/apps/worker/src/ai/sast/capella/sarif-exporter.ts +++ b/apps/worker/src/ai/sast/capella/sarif-exporter.ts @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Keygraph, Inc. +// Copyright (C) 2026 Keygraph, Inc. // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License version 3 @@ -18,7 +18,7 @@ import { type CapellaSarifSeverity, validateCapellaSarif, } from '../sarif-profile.js'; -import type { SarifRef } from '../types.js'; +import type { AgenticSastOmission, AgenticSastReduction, SarifRef } from '../types.js'; import { atomicPublishBytes, sha256Bytes, stableJson } from './artifacts.js'; import { SastContractError } from './errors.js'; import type { CapellaFinding, CapellaSeverity } from './finding-types.js'; @@ -33,6 +33,7 @@ export interface CapellaExportResult { readonly coverage: 'complete' | 'reduced'; readonly warnings: string[]; readonly reportPath: string; + readonly reduction?: AgenticSastReduction; } export interface CapellaExportOptions { @@ -43,6 +44,10 @@ export interface CapellaExportOptions { readonly cancellationSignal?: AbortSignal; } +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + function throwIfExportCancelled(signal: AbortSignal | undefined): void { if (!signal?.aborted) return; if (signal.reason instanceof Error) throw signal.reason; @@ -91,13 +96,54 @@ function threadLocationLabel(index: number, locationCount: number): 'Source' | ' return 'Step'; } -function isExportableFinding(value: unknown): value is CapellaFinding { - if (!isCapellaFinding(value)) return false; - if (value.code_paths.length === 0) return false; - return value.code_paths.every((entry) => { +function classifyExportCandidate( + value: unknown, +): { readonly finding: CapellaFinding } | { readonly omission: AgenticSastOmission } { + if (!isCapellaFinding(value)) { + return { omission: buildOmission(value, 'invalid_finding_record') }; + } + if (value.code_paths.length === 0) { + return { omission: buildOmission(value, 'missing_code_path') }; + } + const codePathsAreValid = value.code_paths.every((entry) => { const parsed = parseCodePath(entry); return parsed !== undefined && isNormalizedRepositoryPath(parsed.file); }); + if (!codePathsAreValid) { + return { omission: buildOmission(value, 'invalid_code_path') }; + } + return { finding: value }; +} + +function buildOmission(value: unknown, reason: AgenticSastOmission['reason']): AgenticSastOmission { + if (!isRecord(value)) return { reason }; + const findingId = safeFindingId(value.id); + const displayName = safeFindingDisplayName(value.cwe, value.title); + return { + reason, + ...(findingId !== undefined && { findingId }), + ...(displayName !== undefined && { displayName }), + }; +} + +function safeFindingId(value: unknown): string | undefined { + if (typeof value !== 'string' || !/^[a-z0-9-]{1,256}$/.test(value)) return undefined; + return value; +} + +function safeFindingDisplayName(cwe: unknown, title: unknown): string | undefined { + if (typeof cwe !== 'string' || !/^CWE-\d+$/.test(cwe) || typeof title !== 'string') return undefined; + const normalizedTitle = [...title] + .map((character) => { + const code = character.charCodeAt(0); + return code <= 31 || code === 127 ? ' ' : character; + }) + .join('') + .replace(/\s+/g, ' ') + .trim(); + if (normalizedTitle.length === 0) return undefined; + const prefix = `${cwe}: `; + return `${prefix}${normalizedTitle.slice(0, 160 - prefix.length)}`; } function buildRule(finding: CapellaFinding): CapellaSarifRule { @@ -186,6 +232,13 @@ export function buildCapellaSarif(findings: readonly CapellaFinding[], repositor }; } +/** + * Read the immutable SARIF reference from a prior successful run, if any. + * + * A missing or unreadable run.json reads as no prior success, and the export + * publishes fresh. A record that claims success but carries a bad reference + * throws: that is corruption of an immutability promise, not a fresh run. + */ async function readSuccessfulSarifRef(artifactRoot: string, expectedPath: string): Promise { try { const run = JSON.parse(await readFile(resolve(artifactRoot, 'run.json'), 'utf8')) as Record; @@ -212,9 +265,24 @@ export async function exportCapellaFindings( throwIfExportCancelled(options.cancellationSignal); const warnings: string[] = []; - const validFindings = rawFindings.filter(isExportableFinding); - const invalidCount = rawFindings.length - validFindings.length; - if (invalidCount > 0) warnings.push(`${invalidCount} agentic SAST findings were malformed and left out.`); + const classified = rawFindings.map(classifyExportCandidate); + const validFindings = classified.flatMap((entry) => ('finding' in entry ? [entry.finding] : [])); + const omissions = classified.flatMap((entry) => ('omission' in entry ? [entry.omission] : [])); + const invalidCount = omissions.length; + if (invalidCount > 0) { + const finding = invalidCount === 1 ? 'finding was' : 'findings were'; + warnings.push(`${invalidCount} agentic SAST ${finding} malformed and left out.`); + } + const reduction: AgenticSastReduction | undefined = + invalidCount > 0 + ? { + stage: 'export', + reason: 'malformed_findings', + omittedCount: invalidCount, + consideredCount: rawFindings.length, + omissions, + } + : undefined; const gated = validFindings.filter(passesExportGate); const exported = gated @@ -247,6 +315,9 @@ export async function exportCapellaFindings( const reportPath = resolve(options.artifactRoot, 'report.md'); const sarifPath = resolve(options.artifactRoot, 'capella.sarif'); + // Adoption path: once run.json says succeeded, the export is immutable. A + // re-driven activity verifies the published bytes still match what this input + // would produce and returns the existing reference instead of republishing. const successful = await readSuccessfulSarifRef(options.artifactRoot, sarifPath); if (successful) { let existingBytes: Buffer; @@ -269,12 +340,17 @@ export async function exportCapellaFindings( return { sarif: successful, findingCount: exported.length, + // Coverage is 'reduced' only when records were invalid: gate-dropped and + // operator-excluded findings are normal outcomes, not lost data. coverage: invalidCount > 0 ? 'reduced' : 'complete', warnings: [...warnings].sort(), reportPath, + ...(reduction !== undefined && { reduction }), }; } + // Cancellation is re-checked immediately before each publish so an aborted + // activity stops without writing new bytes. throwIfExportCancelled(options.cancellationSignal); await atomicPublishBytes( options.artifactRoot, @@ -298,5 +374,6 @@ export async function exportCapellaFindings( coverage: invalidCount > 0 ? 'reduced' : 'complete', warnings: [...warnings].sort(), reportPath, + ...(reduction !== undefined && { reduction }), }; } diff --git a/apps/worker/src/ai/sast/capella/stages/export.ts b/apps/worker/src/ai/sast/capella/stages/export.ts index ae03f30b..5baea10f 100644 --- a/apps/worker/src/ai/sast/capella/stages/export.ts +++ b/apps/worker/src/ai/sast/capella/stages/export.ts @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Keygraph, Inc. +// Copyright (C) 2026 Keygraph, Inc. // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License version 3 @@ -20,6 +20,12 @@ import { type CompletedStage, type ExportStageInput, type ExportValue, ZERO_CAPE import { isExportValue, isRawFindingSetValue } from '../validation.js'; import { artifactLineage, buildStageFingerprint, completeStage, resolveStageIdentity } from './shared.js'; +/** + * A cached export envelope proves only that an export once completed. The SARIF and + * report files live outside that envelope, so their presence and the SARIF digest are + * re-verified before the cache is adopted; adopting a gutted artifact root would + * report success with nothing on disk. + */ async function verifyPublishedExport(input: ExportStageInput, value: ExportValue): Promise { const expectedSarifPath = resolve(input.artifactRoot, 'capella.sarif'); const expectedReportPath = resolve(input.artifactRoot, 'report.md'); @@ -39,6 +45,9 @@ async function verifyPublishedExport(input: ExportStageInput, value: ExportValue } } +// The findings source is all-or-nothing: an export with no source is the legitimate +// short circuit for a run with nothing to report, while a half-specified source is +// always a caller bug. function assertExportSource(input: ExportStageInput): void { const hasArtifact = input.findingsArtifact !== undefined; const hasStage = input.findingsStage !== undefined; @@ -60,6 +69,8 @@ function assertExportSource(input: ExportStageInput): void { function throwIfCancelled(signal: AbortSignal | undefined): void { if (!signal?.aborted) return; + // Rethrowing the signal's own reason keeps a Temporal CancelledFailure's identity + // intact through this boundary instead of degrading it into a generic abort. if (signal.reason instanceof Error) throw signal.reason; throw new DOMException('Capella export cancelled.', 'AbortError'); } @@ -100,6 +111,8 @@ export async function runExportStage( const cached = await loadCompletedArtifact(input.artifactRoot, completedPath, 'export', fingerprint, isExportValue); if (cached) { await verifyPublishedExport(input, cached.value); + // Last checkpoint before the durable run-record write; a cancelled scan must not + // re-record completion. throwIfCancelled(cancellationSignal); await recordStageCompletion( input, @@ -137,6 +150,8 @@ export async function runExportStage( codePathAvoids: input.codePathAvoids, ...(cancellationSignal && { cancellationSignal }), }); + // The exporter has written capella.sarif, but completion is not yet recorded. + // Cancelling here leaves a reusable artifact without marking the stage complete. throwIfCancelled(cancellationSignal); const value: ExportValue = { sarif: exported.sarif, @@ -144,6 +159,7 @@ export async function runExportStage( coverage: exported.coverage, warnings: [...exported.warnings], reportPath: exported.reportPath, + ...(exported.reduction !== undefined && { reduction: exported.reduction }), }; const completed = await completeStage( input, diff --git a/apps/worker/src/ai/sast/capella/temporal/activities.ts b/apps/worker/src/ai/sast/capella/temporal/activities.ts index 19d5e976..485dbfaa 100644 --- a/apps/worker/src/ai/sast/capella/temporal/activities.ts +++ b/apps/worker/src/ai/sast/capella/temporal/activities.ts @@ -553,8 +553,8 @@ async function runStageActivity( const startedAt = Date.now(); let heartbeatInterval: NodeJS.Timeout | undefined; let inputFingerprint: string | undefined; - let completedStageReturned = false; let stageTrace: CapellaStageTrace | undefined; + let completedStageReturned = false; // Hold the stage's per-agent file open for the life of the activity so its concurrent sessions' // trace lines ride one reference count. openStageAgentLog never throws (it returns null on diff --git a/apps/worker/src/ai/sast/capella/temporal/activity-types.ts b/apps/worker/src/ai/sast/capella/temporal/activity-types.ts index c0120615..24841066 100644 --- a/apps/worker/src/ai/sast/capella/temporal/activity-types.ts +++ b/apps/worker/src/ai/sast/capella/temporal/activity-types.ts @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Keygraph, Inc. +// Copyright (C) 2026 Keygraph, Inc. // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License version 3 diff --git a/apps/worker/src/ai/sast/capella/temporal/workflow.ts b/apps/worker/src/ai/sast/capella/temporal/workflow.ts index e50cdfd7..79331d8a 100644 --- a/apps/worker/src/ai/sast/capella/temporal/workflow.ts +++ b/apps/worker/src/ai/sast/capella/temporal/workflow.ts @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Keygraph, Inc. +// Copyright (C) 2026 Keygraph, Inc. // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License version 3 diff --git a/apps/worker/src/audit/audit-session.ts b/apps/worker/src/audit/audit-session.ts index 50f01544..0c626f9b 100644 --- a/apps/worker/src/audit/audit-session.ts +++ b/apps/worker/src/audit/audit-session.ts @@ -26,7 +26,11 @@ import { } from '../types/run-state.js'; import { SessionMutex } from '../utils/concurrency.js'; import { fileExists } from '../utils/file-io.js'; -import { MetricsTracker } from './metrics-tracker.js'; +import { + MetricsTracker, + type TerminalWorkflowMetricsInput, + type TerminalWorkflowMetricTotals, +} from './metrics-tracker.js'; import type { LoggableAgentName, WorkflowPhase } from './safe-fields.js'; import { generateSessionJsonPath, @@ -335,6 +339,22 @@ export class AuditSession { } } + /** Persist the terminal workflow projection under its retry-stable workflow id. */ + async recordTerminalWorkflowMetrics( + workflowId: string, + input: TerminalWorkflowMetricsInput, + ): Promise { + await this.ensureInitialized(); + + const unlock = await sessionMutex.lock(this.sessionId); + try { + await this.metricsTracker.reload(); + return await this.metricsTracker.recordTerminalWorkflowMetrics(workflowId, input); + } finally { + unlock(); + } + } + /** * Get current metrics (read-only) */ diff --git a/apps/worker/src/audit/metrics-tracker.ts b/apps/worker/src/audit/metrics-tracker.ts index 4fef2cdc..bbaf4774 100644 --- a/apps/worker/src/audit/metrics-tracker.ts +++ b/apps/worker/src/audit/metrics-tracker.ts @@ -32,6 +32,7 @@ import { } from '../types/run-state.js'; import { atomicWrite, fileExists, readJson } from '../utils/file-io.js'; import { calculatePercentage, formatTimestamp } from '../utils/formatting.js'; +import { mergeIntervalsDurationMs, type OperationalStageTiming } from './operational-summary.js'; import { safeErrorFromCode } from './safe-fields.js'; import { generateSessionJsonPath, type SessionMetadata } from './utils.js'; @@ -71,6 +72,83 @@ interface PhaseMetrics { agent_count: number; } +interface OperationalAuditMetrics { + duration_ms: number; + input_tokens: number; + output_tokens: number; + cache_read_tokens: number; + cache_write_tokens: number; + cost_usd: number; + turns: number; + usage_complete: boolean; +} + +/** One operational stage's wall-clock span, as persisted per run. */ +interface StageSpan { + started_at_ms: number; + duration_ms: number; +} + +/** + * The stage families `total_operational_duration_ms` and the `background` phase are defined over: + * agentic SAST and finding reconciliation. Report steps and the miscellaneous lane are pipeline + * work, not operational spend, so their spans are excluded and those two fields keep the meaning + * they document. + */ +const OPERATIONAL_STAGE_FAMILIES: readonly string[] = ['agentic-sast', 'reconciliation']; + +function isOperationalStageKey(stageKey: string): boolean { + return OPERATIONAL_STAGE_FAMILIES.some((family) => stageKey === family || stageKey.startsWith(`${family}:`)); +} + +interface TerminalRunMetrics { + status: 'completed' | 'failed' | 'cancelled' | 'partial'; + started_at: string; + ended_at: string; + wall_duration_ms: number; + usage_accounting_complete: boolean; + /** Usage-accounting warnings for this run; always an array, empty when the ledger reconciled. */ + usage_accounting_warnings: string[]; +} + +/** One workflow's usage for a single operational metric key (an agentic-SAST stage, a reconciliation class). */ +export interface WorkflowOperationalMetric { + readonly durationMs: number; + readonly inputTokens: number | null; + readonly outputTokens: number | null; + readonly cacheReadTokens: number | null; + readonly cacheWriteTokens: number | null; + readonly costUsd: number | null; + readonly numTurns: number | null; + readonly usageComplete?: boolean; +} + +/** The terminal projection recorded for one workflow execution, keyed by its retry-stable workflow id. */ +export interface TerminalWorkflowMetricsInput { + readonly status: TerminalRunMetrics['status']; + readonly startedAtMs: number; + readonly endedAtMs: number; + readonly usageAccountingComplete: boolean; + readonly usageAccountingWarnings: readonly string[]; + readonly operationalMetrics: Readonly>; + /** + * Real wall-clock spans for this run's operational stages. Priced metrics carry no faithful + * duration — a reconciliation stage's `StageMetrics` records cost and tokens only — so this is + * the sole source of operational timing. + */ + readonly operationalStages: Readonly>; +} + +/** Workspace-wide totals recomputed across every recorded run, returned after a terminal metrics write. */ +export interface TerminalWorkflowMetricTotals { + readonly totalDurationMs: number; + readonly totalCostUsd: number; + readonly totalTurns: number; + readonly runCount: number; + readonly usageAccountingComplete: boolean; +} + +/** One recorded resume of a workspace, with the prior workflows it terminated and the checkpoint it restored. */ export interface ResumeAttempt { workflowId: string; timestamp: string; @@ -91,9 +169,18 @@ interface SessionData { }; metrics: { total_duration_ms: number; + total_agent_duration_ms?: number; + /** Wall time attributed to operational (non-agent) work — agentic SAST and reconciliation. */ + total_operational_duration_ms?: number; total_cost_usd: number; + total_turns?: number; + usage_accounting_complete?: boolean; phases: Record; agents: Record; + operational?: Record>; + /** Operational stage spans per run, keyed by workflow id then stage key. */ + stages?: Record>; + runs?: Record; }; durableScanState?: DurableScanState; } @@ -152,9 +239,16 @@ export class MetricsTracker { }, metrics: { total_duration_ms: 0, + total_agent_duration_ms: 0, + total_operational_duration_ms: 0, total_cost_usd: 0, + total_turns: 0, + usage_accounting_complete: true, phases: {}, // Phase-level aggregations agents: {}, // Agent-level metrics + operational: {}, + stages: {}, + runs: {}, }, }; @@ -194,6 +288,9 @@ export class MetricsTracker { ); } + // The report agent never reaches success through this ordinary path. Its model attempt is + // recorded as a nonterminal draft, and only verified finalization promotes it to success, + // so a success here would let an unfinalized report look complete. if (agentName === 'report' && result.success) { throw new RunStateError('DurableStateConflictError', 'report-success-requires-terminal-promotion'); } @@ -438,14 +535,13 @@ export class MetricsTracker { if (current.sarif_disposition !== terminal.sarifDisposition) { throw new RunStateError('DurableStateConflictError', 'report-final-disposition-conflict'); } - const adopted: ReportProgress = { - ...current, + const { pdf_provenance: _priorProvenance, ...currentWithoutProvenance } = current; + let adopted: ReportProgress = { + ...currentWithoutProvenance, partial_reasons: appendPartialReasons(current.partial_reasons, terminal.partialReasons), - ...(terminal.pdfProvenance !== null ? { pdf_provenance: terminal.pdfProvenance } : {}), }; - if (terminal.pdfProvenance === null && 'pdf_provenance' in adopted) { - const { pdf_provenance: _removed, ...withoutProvenance } = adopted; - return await this.persistFinalizedReport(data, durableState, withoutProvenance as ReportProgress); + if (terminal.pdfProvenance !== null) { + adopted = { ...adopted, pdf_provenance: terminal.pdfProvenance }; } return await this.persistFinalizedReport(data, durableState, adopted); } @@ -562,6 +658,67 @@ export class MetricsTracker { await this.save(); } + /** Upsert one workflow's terminal wall time and operational usage, then recompute workspace totals. */ + async recordTerminalWorkflowMetrics( + workflowId: string, + input: TerminalWorkflowMetricsInput, + ): Promise { + const data = this.requireData(); + if ( + !Number.isSafeInteger(input.startedAtMs) || + !Number.isSafeInteger(input.endedAtMs) || + input.startedAtMs < 0 || + input.endedAtMs < input.startedAtMs + ) { + throw new RunStateError('DurableStateConflictError', 'terminal-metric-time-invalid'); + } + + data.metrics.operational ??= {}; + const operational = data.metrics.operational; + operational[workflowId] = Object.fromEntries( + Object.entries(input.operationalMetrics).map(([key, metric]) => [ + key, + { + duration_ms: metric.durationMs, + input_tokens: metric.inputTokens ?? 0, + output_tokens: metric.outputTokens ?? 0, + cache_read_tokens: metric.cacheReadTokens ?? 0, + cache_write_tokens: metric.cacheWriteTokens ?? 0, + cost_usd: metric.costUsd ?? 0, + turns: metric.numTurns ?? 0, + usage_complete: metric.usageComplete !== false, + }, + ]), + ); + + data.metrics.stages ??= {}; + data.metrics.stages[workflowId] = this.collectOperationalSpans(input.operationalStages); + + data.metrics.runs ??= {}; + const runs = data.metrics.runs; + runs[workflowId] = { + status: input.status, + started_at: new Date(input.startedAtMs).toISOString(), + ended_at: new Date(input.endedAtMs).toISOString(), + wall_duration_ms: input.endedAtMs - input.startedAtMs, + usage_accounting_complete: input.usageAccountingComplete, + usage_accounting_warnings: [...input.usageAccountingWarnings], + }; + + data.session.status = input.status; + data.session.completedAt = runs[workflowId].ended_at; + this.recalculateAggregations(); + await this.save(); + + return { + totalDurationMs: data.metrics.total_duration_ms, + totalCostUsd: data.metrics.total_cost_usd, + totalTurns: data.metrics.total_turns ?? 0, + runCount: Object.keys(runs).length, + usageAccountingComplete: data.metrics.usage_accounting_complete ?? false, + }; + } + /** * Add a resume attempt to the session * @@ -626,22 +783,82 @@ export class MetricsTracker { // Only count successful agents const successfulAgents = Object.entries(agents).filter(([, data]) => data.status === 'success'); - // Calculate total duration and cost - const totalDuration = successfulAgents.reduce((sum, [, data]) => sum + data.final_duration_ms, 0); + const totalAgentDuration = successfulAgents.reduce((sum, [, data]) => sum + data.final_duration_ms, 0); + const operational = Object.values(this.data.metrics.operational ?? {}).flatMap((metrics) => Object.values(metrics)); + const runs = Object.values(this.data.metrics.runs ?? {}); - const totalCost = successfulAgents.reduce((sum, [, data]) => sum + data.total_cost_usd, 0); - - this.data.metrics.total_duration_ms = totalDuration; - this.data.metrics.total_cost_usd = totalCost; + this.data.metrics.total_agent_duration_ms = totalAgentDuration; + this.data.metrics.total_operational_duration_ms = this.operationalWallClockMs(this.data); + this.data.metrics.total_duration_ms = runs.reduce((sum, run) => sum + run.wall_duration_ms, 0); + this.data.metrics.total_cost_usd = + Object.values(agents).reduce((sum, agent) => sum + agent.total_cost_usd, 0) + + operational.reduce((sum, metric) => sum + metric.cost_usd, 0); + this.data.metrics.total_turns = + Object.values(agents).reduce( + (sum, agent) => sum + agent.attempts.reduce((attemptSum, attempt) => attemptSum + (attempt.turns ?? 0), 0), + 0, + ) + operational.reduce((sum, metric) => sum + metric.turns, 0); + this.data.metrics.usage_accounting_complete = + runs.every((run) => run.usage_accounting_complete) && operational.every((metric) => metric.usage_complete); // Calculate phase-level metrics - this.data.metrics.phases = this.calculatePhaseMetrics(successfulAgents); + this.data.metrics.phases = this.calculatePhaseMetrics(successfulAgents, operational); + } + + /** + * Keep the operational stage spans this run can place on a timeline. A stage that never ran, or + * that was still running, has no complete span and is dropped rather than guessed at; a present + * but nonsensical value is corruption and fails closed. + */ + private collectOperationalSpans( + operationalStages: Readonly>, + ): Record { + const spans: Record = {}; + for (const [stageKey, timing] of Object.entries(operationalStages)) { + if (!isOperationalStageKey(stageKey)) continue; + if (timing.startedAt === undefined || timing.durationMs === undefined) continue; + const spanIsWellFormed = + Number.isSafeInteger(timing.startedAt) && + Number.isSafeInteger(timing.durationMs) && + timing.startedAt >= 0 && + timing.durationMs >= 0; + if (!spanIsWellFormed) { + throw new RunStateError('DurableStateConflictError', 'terminal-stage-span-invalid'); + } + spans[stageKey] = { started_at_ms: timing.startedAt, duration_ms: timing.durationMs }; + } + return spans; + } + + /** + * Wall time the workspace actually spent on operational work. Reconciliation stages carry no + * duration in their priced metrics, so the timing comes from the stage spans, merged so classes + * that overlapped count once instead of once each. A run recorded before spans were persisted has + * none, so it falls back to summing its priced durations — the same fallback + * `summarizeOperationalMetrics` applies to a metrics-only view. The two sets never intersect, so + * no run is counted twice. + */ + private operationalWallClockMs(data: SessionData): number { + const spansByRun = data.metrics.stages ?? {}; + const spans = Object.values(spansByRun).flatMap((stages) => + Object.values(stages).map((span) => ({ startedAt: span.started_at_ms, durationMs: span.duration_ms })), + ); + const spanlessRunDuration = Object.entries(data.metrics.operational ?? {}) + .filter(([workflowId]) => spansByRun[workflowId] === undefined) + .reduce( + (sum, [, metrics]) => sum + Object.values(metrics).reduce((inner, metric) => inner + metric.duration_ms, 0), + 0, + ); + return mergeIntervalsDurationMs(spans) + spanlessRunDuration; } /** * Calculate phase-level metrics */ - private calculatePhaseMetrics(successfulAgents: Array<[string, AgentAuditMetrics]>): Record { + private calculatePhaseMetrics( + successfulAgents: Array<[string, AgentAuditMetrics]>, + operational: OperationalAuditMetrics[], + ): Record { const phases: Record = { 'pre-recon': [], recon: [], @@ -660,8 +877,11 @@ export class MetricsTracker { // Calculate metrics per phase const phaseMetrics: Record = {}; - // biome-ignore lint/style/noNonNullAssertion: called from recalculateAggregations which guards this.data - const totalDuration = this.data!.metrics.total_duration_ms; + // Percentages share one basis — agent plus operational duration — so the synthetic background + // phase below is comparable to the agent phases rather than measured against a different total. + // (`this.data` is guaranteed by the recalculateAggregations caller; optional chaining keeps it lint-clean.) + const operationalDuration = this.data?.metrics.total_operational_duration_ms ?? 0; + const totalDuration = (this.data?.metrics.total_agent_duration_ms ?? 0) + operationalDuration; for (const [phaseName, agentList] of Object.entries(phases)) { if (agentList.length === 0) continue; @@ -677,6 +897,19 @@ export class MetricsTracker { }; } + // Operational work (agentic SAST, reconciliation) runs concurrently with the agent phases and is + // otherwise absent from this breakdown; surface it as one `background` phase. `agent_count` here + // is the number of operational metric entries, not agents. + if (operational.length > 0) { + const backgroundCost = operational.reduce((sum, metric) => sum + metric.cost_usd, 0); + phaseMetrics.background = { + duration_ms: operationalDuration, + duration_percentage: calculatePercentage(operationalDuration, totalDuration), + cost_usd: backgroundCost, + agent_count: operational.length, + }; + } + return phaseMetrics; } @@ -720,6 +953,12 @@ export class MetricsTracker { return durableState; } + /** + * Append one attempt and recompute the agent's cumulative totals from the full attempt list, + * rather than incrementing them. A reload-then-write cycle can replay this on the same agent + * more than once across a retry, and recomputing from the stored attempts keeps the totals + * correct regardless of how many times that happens. + */ private appendAttempt(agentName: string, result: AgentEndResult): AgentAuditMetrics { const data = this.requireData(); const existingAgent = data.metrics.agents[agentName]; diff --git a/apps/worker/src/audit/operational-summary.ts b/apps/worker/src/audit/operational-summary.ts new file mode 100644 index 00000000..c326d3cd --- /dev/null +++ b/apps/worker/src/audit/operational-summary.ts @@ -0,0 +1,136 @@ +// Copyright (C) 2026 Keygraph, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License version 3 +// as published by the Free Software Foundation. + +/** + * Rolls a scan's non-agent (operational) spend up into the small, ordered set of labelled groups + * the completion summary and live status view show. Pure and self-contained (no I/O) so it can be + * unit-tested directly. Cost comes from the priced metrics; duration comes from the stage spans' + * real wall-clock, which is why concurrent reconciliation classes read as their union, not a sum. + */ + +/** The buckets operational (non-agent) spend is grouped into for the completion summary. */ +export type OperationalGroupKey = 'agentic-sast' | 'reconciliation' | 'other'; + +export interface OperationalGroupTotal { + readonly key: OperationalGroupKey; + readonly label: string; + readonly durationMs: number; + /** Null only when every metric in the group has an unknown cost, matching the agent breakdown's N/A. */ + readonly costUsd: number | null; +} + +/** + * The wall-clock span of one operational stage. Sourced from `operationalStages` (not the priced + * metrics), it is how the summary reports a group's real elapsed time — reconciliation classes run + * concurrently, so their true duration is the union of these spans, not a sum. + */ +export interface OperationalStageTiming { + readonly startedAt?: number; + readonly durationMs?: number; +} + +const OPERATIONAL_GROUP_LABELS: Readonly> = { + 'agentic-sast': 'Agentic SAST', + reconciliation: 'Finding reconciliation', + other: 'Background task', +}; + +// Stable render order; a group only appears when it has at least one metric. +const OPERATIONAL_GROUP_ORDER: readonly OperationalGroupKey[] = ['agentic-sast', 'reconciliation', 'other']; + +/** Classify an operational metric key by its generated prefix; anything unexpected folds into `other`. */ +function operationalGroupKey(metricKey: string): OperationalGroupKey { + if (metricKey.startsWith('agentic-sast:')) return 'agentic-sast'; + if (metricKey.startsWith('reconciliation:')) return 'reconciliation'; + return 'other'; +} + +/** + * Classify an operational *stage* key. Stage keys differ from metric keys: the agentic-SAST stage + * is the bare `agentic-sast` (its metric is `agentic-sast:export`), and reconciliation stages are + * `reconciliation:`. So match the bare family name as well as its colon-prefixed children. + */ +function operationalStageGroupKey(stageKey: string): OperationalGroupKey { + if (stageKey === 'agentic-sast' || stageKey.startsWith('agentic-sast:')) return 'agentic-sast'; + if (stageKey === 'reconciliation' || stageKey.startsWith('reconciliation:')) return 'reconciliation'; + return 'other'; +} + +/** + * Total wall-clock covered by a set of `[startedAt, startedAt + durationMs)` spans, merging any + * overlap. This is what keeps a group's duration faithful when its stages run concurrently: several + * reconciliation classes overlap in time, so their real elapsed time is the union, never the sum. + * Spans missing a start or a positive duration cannot be placed on the timeline and are ignored. + */ +export function mergeIntervalsDurationMs(spans: readonly OperationalStageTiming[]): number { + const intervals = spans + .filter( + (span): span is { startedAt: number; durationMs: number } => + span.startedAt !== undefined && span.durationMs !== undefined && span.durationMs > 0, + ) + .map((span) => ({ start: span.startedAt, end: span.startedAt + span.durationMs })) + .sort((a, b) => a.start - b.start); + + let total = 0; + let cursor = Number.NEGATIVE_INFINITY; + for (const interval of intervals) { + const start = Math.max(interval.start, cursor); + if (interval.end > start) total += interval.end - start; + cursor = Math.max(cursor, interval.end); + } + return total; +} + +/** + * Roll the per-key operational metrics up into a small, ordered set of labelled group totals. + * Grouping by prefix (rather than itemizing raw keys) keeps the summary robust to key drift and + * needs no per-key label table. A group's presence and cost come from the priced metrics — cost + * stays null for a group only when no metric in it reported one, so a partially-known group still + * shows its known spend. Duration comes from the group's stage spans (their real wall-clock union) + * when `operationalStages` is supplied; without it, it falls back to summing the metric durations. + */ +export function summarizeOperationalMetrics( + operationalMetrics: Readonly>, + operationalStages?: Readonly>, +): OperationalGroupTotal[] { + const metricDurationByGroup = new Map(); + const costByGroup = new Map(); + + for (const [metricKey, metrics] of Object.entries(operationalMetrics)) { + const group = operationalGroupKey(metricKey); + metricDurationByGroup.set(group, (metricDurationByGroup.get(group) ?? 0) + Math.max(0, metrics.durationMs)); + if (metrics.costUsd !== null) { + const priorCost = costByGroup.get(group); + costByGroup.set(group, (priorCost ?? 0) + Math.max(0, metrics.costUsd)); + } else if (!costByGroup.has(group)) { + costByGroup.set(group, null); + } + } + + const spansByGroup = new Map(); + for (const [stageKey, timing] of Object.entries(operationalStages ?? {})) { + const group = operationalStageGroupKey(stageKey); + const spans = spansByGroup.get(group) ?? []; + spans.push(timing); + spansByGroup.set(group, spans); + } + + const totals: OperationalGroupTotal[] = []; + for (const key of OPERATIONAL_GROUP_ORDER) { + if (!metricDurationByGroup.has(key)) continue; + const spans = spansByGroup.get(key); + // Real wall-clock from the stage spans; fall back to the summed metric durations when no spans + // were supplied (e.g. the console path passes metrics only). + const durationMs = spans !== undefined ? mergeIntervalsDurationMs(spans) : (metricDurationByGroup.get(key) ?? 0); + totals.push({ + key, + label: OPERATIONAL_GROUP_LABELS[key], + durationMs, + costUsd: costByGroup.get(key) ?? null, + }); + } + return totals; +} diff --git a/apps/worker/src/audit/workflow-logger.ts b/apps/worker/src/audit/workflow-logger.ts index c2970953..e2bea653 100644 --- a/apps/worker/src/audit/workflow-logger.ts +++ b/apps/worker/src/audit/workflow-logger.ts @@ -10,7 +10,7 @@ import { promises as fsPromises } from 'node:fs'; import path from 'node:path'; import { isCapellaSafeFailureMessage, isCapellaTerminalStageLabel } from '../ai/sast/capella/safe-failures.js'; import type { CapellaStage } from '../ai/sast/types.js'; -import type { ErrorCode } from '../types/errors.js'; +import { type ErrorCode, isProviderFailureCategory } from '../types/errors.js'; import { isPartialReason, type PartialReasonView, projectPartialReasons } from '../types/run-state.js'; import { formatDuration, formatTimestamp } from '../utils/formatting.js'; import { @@ -22,6 +22,7 @@ import { type TraceActor, } from './actor-projection.js'; import { LogStream, warnAgentLoggingFailure, warnLoggingFailure } from './log-stream.js'; +import { type OperationalStageTiming, summarizeOperationalMetrics } from './operational-summary.js'; import { isLoggableAgentName, isWorkflowPhase, @@ -49,18 +50,39 @@ export interface AgentMetricsSummary { readonly costUsd: number | null; } +export interface OperationalMetricsSummary { + readonly durationMs: number; + readonly costUsd: number | null; + readonly inputTokens: number | null; + readonly outputTokens: number | null; + readonly cacheReadTokens: number | null; + readonly cacheWriteTokens: number | null; + readonly numTurns: number | null; + readonly usageComplete: boolean; +} + export interface WorkflowSummary { readonly status: 'completed' | 'failed' | 'cancelled' | 'partial'; + readonly startedAtMs: number; + readonly endedAtMs: number; readonly totalDurationMs: number; readonly totalCostUsd: number; readonly completedAgents: readonly string[]; readonly skippedAgents?: readonly string[]; readonly agentMetrics: Readonly>; + readonly operationalMetrics: Readonly>; + /** Per-stage wall-clock spans, keyed as `operationalStages` is; feeds each group's real duration. */ + readonly operationalStages: Readonly>; readonly partialReasons?: readonly PartialReasonView[]; readonly usageAccountingComplete?: boolean; + /** Usage-accounting warnings from the Capella run; empty when the ledger reconciled. */ + readonly usageAccountingWarnings?: readonly string[]; readonly agenticSastFailedStage?: string; readonly agenticSastFailureMessage?: string; readonly agenticSastErrorCode?: string; + /** Terminal disposition of the agentic-SAST child, so a successful run is visible, not just a failed one. */ + readonly agenticSastStatus?: 'disabled' | 'running' | 'succeeded' | 'failed'; + readonly agenticSastCoverage?: 'complete' | 'reduced'; readonly errorCode?: ErrorCode; } @@ -102,7 +124,7 @@ function safeReasonMessage(reason: PartialReasonView): string | undefined { } function safeAgenticSastCode(code: string | undefined): string | undefined { - if (code !== undefined && /^[A-Z][A-Z0-9_]{0,63}$/u.test(code)) return code; + if (code !== undefined && (/^[A-Z][A-Z0-9_]{0,63}$/u.test(code) || isProviderFailureCategory(code))) return code; return undefined; } @@ -110,6 +132,11 @@ function safeAgenticSastStageLabel(label: string | undefined): string | undefine return label !== undefined && isCapellaTerminalStageLabel(label) ? label : undefined; } +/** Render a cost the same way the agent breakdown does: N/A when unknown, else a fixed 4-dp dollar value. */ +function formatCostUsd(costUsd: number | null): string { + return costUsd === null ? 'N/A' : `$${Math.max(0, costUsd).toFixed(4)}`; +} + /** Keep normal PI names readable and losslessly quote any unexpected name. */ function formatToolName(tool: string): string { return /^[A-Za-z][A-Za-z0-9_-]{0,63}$/u.test(tool) ? tool : JSON.stringify(tool); @@ -646,6 +673,8 @@ export class WorkflowLogger { const status = statusHeaders[summary.status]; const completedAgents = summary.completedAgents.filter(isLoggableAgentName); const skippedAgents = (summary.skippedAgents ?? []).filter(isLoggableAgentName); + const operationalGroups = summarizeOperationalMetrics(summary.operationalMetrics, summary.operationalStages); + const sastGroup = operationalGroups.find((group) => group.key === 'agentic-sast'); const lines = [ '', '================================================================================', @@ -691,11 +720,38 @@ export class WorkflowLogger { lines.push(` - ${agentName}`); continue; } - const cost = metrics.costUsd === null ? 'N/A' : `$${Math.max(0, metrics.costUsd).toFixed(4)}`; - lines.push(` - ${agentName} (${formatDuration(Math.max(0, metrics.durationMs))}, ${cost})`); + lines.push( + ` - ${agentName} (${formatDuration(Math.max(0, metrics.durationMs))}, ${formatCostUsd(metrics.costUsd)})`, + ); } for (const agentName of skippedAgents) lines.push(` - ${agentName} (skipped — nothing to exploit)`); } + + // Agentic SAST is a pluggable analysis engine, a peer to the pentest rather than background + // plumbing, so it gets its own section. Its spend sits in Total Cost but has no agent line; the + // detailed "stopped at" block above narrates a failure, this line accounts for its time and cost. + if ( + sastGroup !== undefined && + summary.agenticSastStatus !== undefined && + summary.agenticSastStatus !== 'disabled' + ) { + const outcome = summary.agenticSastStatus === 'failed' ? 'failed' : 'completed'; + const coverageSuffix = summary.agenticSastCoverage === 'reduced' ? ' — reduced coverage' : ''; + lines.push('', 'Analysis Engines:'); + lines.push( + ` - Agentic SAST — ${outcome} (${formatDuration(sastGroup.durationMs)}, ${formatCostUsd(sastGroup.costUsd)})${coverageSuffix}`, + ); + } + + // Concurrent, non-agent plumbing (finding reconciliation, and a rare catch-all) whose spend is + // inside Total Cost but never itemized above. Grouped so it stays legible as keys evolve. + const backgroundGroups = operationalGroups.filter((group) => group.key !== 'agentic-sast'); + if (backgroundGroups.length > 0) { + lines.push('', 'Background Work:'); + for (const group of backgroundGroups) { + lines.push(` - ${group.label} (${formatDuration(group.durationMs)}, ${formatCostUsd(group.costUsd)})`); + } + } lines.push('================================================================================'); const marker = `Scan ${status}`; diff --git a/apps/worker/src/collectors/finding-collector.ts b/apps/worker/src/collectors/finding-collector.ts index d6e6ea85..8df92bd6 100644 --- a/apps/worker/src/collectors/finding-collector.ts +++ b/apps/worker/src/collectors/finding-collector.ts @@ -24,6 +24,8 @@ import { cleanInput, stringEnum } from './schema.js'; // SCHEMA // ============================================================================ +const CATEGORY_VALUES = ['Injection', 'XSS', 'Authentication', 'SSRF', 'Authorization', 'Miscellaneous'] as const; + const OWASP_CATEGORY_VALUES = [ 'A01:2025 — Broken Access Control', 'A02:2025 — Security Misconfiguration', @@ -117,6 +119,13 @@ const AdditionalSectionSchema = Type.Object({ }), }); +const SASTSourceLocationSchema = Type.Object({ + file: Type.String({ minLength: 1, description: 'Source file path relative to the repository root.' }), + line: Type.Integer({ minimum: 1, description: 'One-based source line.' }), + column: Type.Integer({ minimum: 0, description: 'Zero-based source column.' }), + rule_id: Type.String({ minLength: 1, description: 'Validated SAST rule or CWE identifier.' }), +}); + /** * `severity` is recorded in both modes, but it does not mean the same thing in each: an exploit * run measures it from what the exploit demonstrated, an analysis run assesses it from the class @@ -131,18 +140,17 @@ function identityFields(exploit: boolean) { severity: stringEnum(SEVERITY_VALUES, { description: severityDescription }), finding_id: Type.String({ minLength: 1, - description: 'Finding identifier (e.g., "AUTH-VULN-07", "INJ-VULN-03"). Must be unique per report.', + description: 'Stable finding identifier (e.g., "AUTH-07", "INJ-03", "MISC-01"). Must be unique per report.', }), title: Type.String({ minLength: 1, description: 'Descriptive name (e.g., "SQL Injection — User Search", "IDOR — Unauthorized Access to User Orders").', }), - category: stringEnum(['Injection', 'XSS', 'Authentication', 'Authorization', 'SSRF'], { + category: stringEnum(CATEGORY_VALUES, { description: - 'From the finding_id prefix: INJ-VULN-xxx Injection, ' + - 'XSS-VULN-xxx XSS, AUTH-VULN-xxx Authentication, AUTHZ-VULN-xxx Authorization, ' + - 'SSRF-VULN-xxx SSRF.', + 'From the finding_id prefix: INJ-* Injection, XSS-* XSS, AUTH-* Authentication, ' + + 'AUTHZ-* Authorization, SSRF-* SSRF, MISC-* Miscellaneous.', }), owasp_category: stringEnum(OWASP_CATEGORY_VALUES, { description: 'OWASP Top Ten 2025 category.', @@ -256,6 +264,9 @@ export function buildAddFindingSchema(exploit: boolean) { const AddFindingSupersetSchema = Type.Object({ ...identityFields(true), code_locations: Type.Optional(Type.Array(CodeLocationSchema)), + // Join-only, like code_locations: set by attachQueueCodeLocations from Capella's committed + // output. Absent from buildAddFindingSchema so the report agent cannot author (fabricate) it. + sast_source_location: Type.Optional(Type.Union([SASTSourceLocationSchema, Type.Null()])), auth_state: Type.Optional(Type.String()), prerequisites: Type.Optional(Type.String()), exploitation_steps: Type.Optional(Type.Array(StructuredStepSchema)), @@ -275,6 +286,7 @@ export type HttpLocation = Static; export type StepItem = Static; export type StructuredStep = Static; export type AdditionalSection = Static; +export type SASTSourceLocation = Static; // ============================================================================ // RESPONSE HELPERS diff --git a/apps/worker/src/services/agent-execution.ts b/apps/worker/src/services/agent-execution.ts index ff076ba2..caaf3969 100644 --- a/apps/worker/src/services/agent-execution.ts +++ b/apps/worker/src/services/agent-execution.ts @@ -47,6 +47,8 @@ import { loadPrompt } from './prompt-manager.js'; export interface AgentExecutionInput { webUrl: string; repoPath: string; + /** Workflow-owned UTC date used by the report prompt. */ + assessmentDate?: string | undefined; deliverablesPath: string; configPath?: string | undefined; configData?: import('../types/config.js').DistributedConfig | undefined; @@ -60,6 +62,12 @@ export interface AgentExecutionInput { failedClasses?: readonly import('../types/config.js').VulnClass[] | undefined; // Renders the deliverable to disk; invoked after validation, before the success commit. writeDeliverable?: (deliverablesPath: string, execution: { readonly model?: string }) => Promise; + /** + * 'report-draft' routes a successful attempt through `AuditSession.endReportDraft` instead of + * `endAgent`, recording a nonterminal checkpoint. The report agent only becomes terminal once + * finalization verifies and promotes the draft; `MetricsTracker.endAgent` rejects a successful + * report attempt outright, so this must stay 'report-draft' for that agent or execution fails. + */ successDisposition?: 'terminal' | 'report-draft'; cancellationSignal?: AbortSignal | undefined; } @@ -150,6 +158,7 @@ export class AgentExecutionService { configYAML, pipelineTestingMode = false, attemptNumber, + assessmentDate, analysisClasses, promptDir, customTools, @@ -160,6 +169,8 @@ export class AgentExecutionService { } = input; const gitPaths = getAgentGitPaths(agentName); + // The parameter type only constrains element type, not that this is exactly the fixed five + // classes in order; this runtime check catches a caller that bypasses the workflow's contract. assertFixedAnalysisScope(analysisClasses); if (successDisposition === 'report-draft' && agentName !== 'report') { return err( @@ -189,6 +200,7 @@ export class AgentExecutionService { { webUrl, repoPath, + ...(assessmentDate !== undefined && { assessmentDate }), AUTH_STATE_FILE: authStateFile(auditSession.sessionMetadata), analysisClasses, ...(failedClasses !== undefined && { failedClasses }), diff --git a/apps/worker/src/services/agent-git-paths.ts b/apps/worker/src/services/agent-git-paths.ts index f190df55..ceb7bc5f 100644 --- a/apps/worker/src/services/agent-git-paths.ts +++ b/apps/worker/src/services/agent-git-paths.ts @@ -14,10 +14,24 @@ */ import { getQueueFilename } from '../ai/queue-schemas.js'; -import { REPORT_JSON_FILENAME, SARIF_FILENAME } from '../paths.js'; +import { REPORT_JSON_FILENAME } from '../paths.js'; import { AGENTS } from '../session-manager.js'; import type { AgentName } from '../types/agents.js'; +// These names must match `sparseExploitCollectorPath` in renumber-core.ts, which derives the +// same filename from the bare ReconciliationClass at read time. If the two fall out of sync, +// git scoping silently drops the collector file from an agent's checkpoint/commit/rollback set: +// the file sits uncommitted in the working tree, or is not restored on rollback, with neither +// side raising an error. +const EXPLOIT_COLLECTOR_PATHS: Readonly>> = Object.freeze({ + 'injection-exploit': 'injection_exploit_collector.json', + 'xss-exploit': 'xss_exploit_collector.json', + 'auth-exploit': 'auth_exploit_collector.json', + 'ssrf-exploit': 'ssrf_exploit_collector.json', + 'authz-exploit': 'authz_exploit_collector.json', + 'miscellaneous-exploit': 'miscellaneous_exploit_collector.json', +}); + /** * Deliverable files an agent writes into the deliverables directory. Used to * scope git operations so one agent never touches a sibling agent's output. @@ -28,12 +42,14 @@ export function getAgentGitPaths(agentName: AgentName): string[] { if (queueFilename) { paths.push(queueFilename); } - // The report agent also emits the structured findings the markdown is rendered from, and the - // SARIF log when produced. Listing the log unconditionally is harmless when it was not written, - // and keeps a stale one from surviving the rollback of a failed attempt. + const exploitCollectorPath = EXPLOIT_COLLECTOR_PATHS[agentName]; + if (exploitCollectorPath !== undefined) { + paths.push(exploitCollectorPath); + } + // The report agent owns only its provisional markdown input and structured JSON. Canonical + // Markdown/SARIF ownership transfers to the finalization transaction after report completion. if (agentName === 'report') { paths.push(REPORT_JSON_FILENAME); - paths.push(SARIF_FILENAME); } return [...new Set(paths)]; } diff --git a/apps/worker/src/services/code-location-join.ts b/apps/worker/src/services/code-location-join.ts index 5cbfc3d6..41156529 100644 --- a/apps/worker/src/services/code-location-join.ts +++ b/apps/worker/src/services/code-location-join.ts @@ -11,7 +11,9 @@ import type { SastSourceLocation } from '../ai/reconciliation/contracts.js'; import type { AddFindingInput } from '../collectors/finding-collector.js'; import type { ActivityLogger } from '../types/activity-logger.js'; import { ALL_VULN_CLASSES } from '../types/config.js'; +import { ErrorCode } from '../types/errors.js'; import type { ReconciliationClass } from '../types/reconciliation.js'; +import { PentestError } from './error-handling.js'; import { readCommittedFile } from './git-manager.js'; import { renumberMapPath } from './renumber-core.js'; @@ -26,10 +28,17 @@ interface JoinedLocations { readonly sastSourceLocation?: SastSourceLocation; } +interface LocationIndexes { + readonly stable: ReadonlyMap; + readonly current: ReadonlyMap; +} + function isRecord(value: unknown): value is Record { return value !== null && typeof value === 'object' && !Array.isArray(value); } +// The queue entry is untrusted committed JSON, not a value this process just produced, so its +// shape is re-validated field by field rather than trusted from the type annotation alone. function isSastSourceLocation(value: unknown): value is SastSourceLocation { if (!isRecord(value)) return false; return ( @@ -44,11 +53,29 @@ function isSastSourceLocation(value: unknown): value is SastSourceLocation { ); } -function parseJson(contents: string, description: string): unknown { +function committedLocationError( + message: string, + checkCode: string, + vulnerabilityClass: ReconciliationClass, +): PentestError { + return new PentestError( + message, + 'validation', + false, + { checkCode, vulnerabilityClass }, + ErrorCode.AGENT_EXECUTION_FAILED, + ); +} + +function parseJson(contents: string, description: string, vulnerabilityClass: ReconciliationClass): unknown { try { return JSON.parse(contents) as unknown; } catch { - throw new Error(`${description} is not valid JSON`); + throw committedLocationError( + `${description} is not valid JSON`, + 'committed-location-json-invalid', + vulnerabilityClass, + ); } } @@ -58,14 +85,30 @@ async function currentReferenceMap( ): Promise> { const mapRead = await readCommittedFile(deliverablesPath, renumberMapPath(vulnerabilityClass)); if (mapRead.state === 'absent') return new Map(); - if (mapRead.state !== 'present') throw new Error(`${vulnerabilityClass} report-reference map is unreadable`); - const decoded = parseJson(mapRead.contents, `${vulnerabilityClass} report-reference map`); + if (mapRead.state !== 'present') { + throw committedLocationError( + `${vulnerabilityClass} report-reference map is unreadable`, + 'committed-reference-map-unreadable', + vulnerabilityClass, + ); + } + const decoded = parseJson(mapRead.contents, `${vulnerabilityClass} report-reference map`, vulnerabilityClass); if (!isRecord(decoded) || !isRecord(decoded.map)) { - throw new Error(`${vulnerabilityClass} report-reference map is malformed`); + throw committedLocationError( + `${vulnerabilityClass} report-reference map is malformed`, + 'committed-reference-map-malformed', + vulnerabilityClass, + ); } const references = new Map(); for (const [stable, current] of Object.entries(decoded.map)) { - if (typeof current !== 'string') throw new Error(`${vulnerabilityClass} report-reference map is malformed`); + if (typeof current !== 'string') { + throw committedLocationError( + `${vulnerabilityClass} report-reference map is malformed`, + 'committed-reference-map-entry-malformed', + vulnerabilityClass, + ); + } references.set(stable, current); } return references; @@ -74,15 +117,26 @@ async function currentReferenceMap( async function loadQueueLocations( deliverablesPath: string, participatingClasses: readonly ReconciliationClass[], -): Promise> { - const locations = new Map(); +): Promise { + const stableLocations = new Map(); + const currentLocations = new Map(); for (const vulnerabilityClass of participatingClasses) { const queueRead = await readCommittedFile(deliverablesPath, `${vulnerabilityClass}_exploitation_queue.json`); if (queueRead.state === 'absent') continue; - if (queueRead.state !== 'present') throw new Error(`${vulnerabilityClass} queue is unreadable`); - const decoded = parseJson(queueRead.contents, `${vulnerabilityClass} queue`); + if (queueRead.state !== 'present') { + throw committedLocationError( + `${vulnerabilityClass} queue is unreadable`, + 'committed-queue-unreadable', + vulnerabilityClass, + ); + } + const decoded = parseJson(queueRead.contents, `${vulnerabilityClass} queue`, vulnerabilityClass); if (!isRecord(decoded) || !Array.isArray(decoded.vulnerabilities)) { - throw new Error(`${vulnerabilityClass} queue is malformed`); + throw committedLocationError( + `${vulnerabilityClass} queue is malformed`, + 'committed-queue-malformed', + vulnerabilityClass, + ); } const referenceMap = await currentReferenceMap(deliverablesPath, vulnerabilityClass); for (const rawEntry of decoded.vulnerabilities) { @@ -96,12 +150,12 @@ async function loadQueueLocations( ...(isSastSourceLocation(entry.sast_source_location) ? { sastSourceLocation: entry.sast_source_location } : {}), }; if (bundle.codeLocations === undefined && bundle.sastSourceLocation === undefined) continue; - locations.set(stableReference, bundle); + stableLocations.set(stableReference, bundle); const currentReference = referenceMap.get(stableReference); - if (currentReference !== undefined) locations.set(currentReference, bundle); + if (currentReference !== undefined) currentLocations.set(currentReference, bundle); } } - return locations; + return { stable: stableLocations, current: currentLocations }; } /** Attach the two independent location fields without consulting exploit-inspection locations. */ @@ -115,14 +169,19 @@ export async function attachQueueCodeLocations( let analysisMatches = 0; let sastMatches = 0; const joined = findings.map((finding) => { - const locations = byReference.get(finding.finding_id); - if (locations === undefined) return finding; - if (locations.codeLocations !== undefined) analysisMatches++; - if (locations.sastSourceLocation !== undefined) sastMatches++; + // Report findings normally carry the renumbered current reference. Resolve that lane first, + // then fall back to the stable lane for unrenumbered classes. Keeping the lanes separate + // prevents one entry's stable reference from overwriting another entry's current reference. + const locations = byReference.current.get(finding.finding_id) ?? byReference.stable.get(finding.finding_id); + if (locations?.codeLocations !== undefined) analysisMatches++; + if (locations?.sastSourceLocation !== undefined) sastMatches++; + // The queue is the sole authority for sast_source_location: strip any value the finding + // carries so it is present iff Capella committed one, never a report-agent fabrication. + const { sast_source_location: _dropped, ...withoutSast } = finding; return { - ...finding, - ...(locations.codeLocations !== undefined && { code_locations: locations.codeLocations }), - ...(locations.sastSourceLocation !== undefined && { sast_source_location: locations.sastSourceLocation }), + ...withoutSast, + ...(locations?.codeLocations !== undefined && { code_locations: locations.codeLocations }), + ...(locations?.sastSourceLocation !== undefined && { sast_source_location: locations.sastSourceLocation }), }; }); logger.info('Attached committed report-task locations', { diff --git a/apps/worker/src/services/compaction-core.ts b/apps/worker/src/services/compaction-core.ts index 4bd0e775..2bd623ee 100644 --- a/apps/worker/src/services/compaction-core.ts +++ b/apps/worker/src/services/compaction-core.ts @@ -48,9 +48,12 @@ export interface RenumberMapFile { export interface ClassCompaction { readonly vulnerabilityClass: ReconciliationClass; + /** Dense renumbered reference to its gapless replacement; the map applied to artifact text. */ readonly gapMap: ReadonlyMap; + /** Original stable reference straight to the gapless reference, composed across both hops. */ readonly composedMap: ReadonlyMap; readonly excluded: readonly ExcludedEntry[]; + /** Replacement renumber-map artifact, so the committed map always records stable-to-current. */ readonly renumberMapFile: RenumberMapFile; } @@ -169,6 +172,9 @@ export function remapExploitCollector( throw new RenumberError('key-set-divergence', false, { checkCode: 'compaction-collector-reference-missing' }); } const gapless = classGapMap.get(reference); + // A collector entry with no gap mapping is one the final report did not keep, so the + // compacted collector drops it too instead of carrying a reference the report no longer + // contains. if (gapless === undefined) continue; const rewritten = deepRemapStrings(entry, allGapMap) as Record; remapped.push({ ...rewritten, vulnerability_id: gapless } as unknown as AddExploitInput); diff --git a/apps/worker/src/services/finding-order.ts b/apps/worker/src/services/finding-order.ts index bd7b790b..eb763365 100644 --- a/apps/worker/src/services/finding-order.ts +++ b/apps/worker/src/services/finding-order.ts @@ -21,8 +21,14 @@ export const CONFIDENCE_RANK: Readonly> = Object.freeze({ low: 2, }); -/** Recognized report categories. Unrecognized values form a sentinel group after `Other`. */ -export const CATEGORY_ORDER = ['Injection', 'XSS', 'Authentication', 'SSRF', 'Authorization', 'Other'] as const; +/** + * Recognized report categories. Unrecognized values form a sentinel group after `Miscellaneous`. + * + * Mirrors the `TypstCategory` union in report-output-schema.ts as plain strings rather than + * importing the type. A category added there without a matching entry here will not fail to + * compile; it will just sort into the unrecognized/lexical tail instead of its intended position. + */ +export const CATEGORY_ORDER = ['Injection', 'XSS', 'Authentication', 'SSRF', 'Authorization', 'Miscellaneous'] as const; export function severityRank(severity: string | null | undefined): number { if (severity == null) return UNRANKED; diff --git a/apps/worker/src/services/findings-renderer.ts b/apps/worker/src/services/findings-renderer.ts index 60f1c899..33fda53f 100644 --- a/apps/worker/src/services/findings-renderer.ts +++ b/apps/worker/src/services/findings-renderer.ts @@ -286,6 +286,8 @@ export async function renderFindingsFromQueues( await fs.writeFile(findingsPath, markdown); logger.info(`${config.heading}: rendered ${entries.length} finding(s) to ${config.findingsFile}`); } catch (error) { + // One class's render failure does not abort the others. The failed class is returned so + // the report can mark it not_assessed rather than presenting it as a clean result. const err = error as Error; failedClasses.push(vulnerabilityClass); logger.warn(`${config.heading}: failed to render findings from ${config.queueFile}: ${err.message}`); diff --git a/apps/worker/src/services/pdf-renderer.ts b/apps/worker/src/services/pdf-renderer.ts index 4ab42d2f..de0ea60e 100644 --- a/apps/worker/src/services/pdf-renderer.ts +++ b/apps/worker/src/services/pdf-renderer.ts @@ -181,6 +181,8 @@ export async function renderReportPdf(options: RenderReportPdfOptions): Promise< ]); await mkdir(path.dirname(outputPath), { recursive: true }); + // Publish the compiled PDF by atomic rename, so a reader never observes a half-written file + // at outputPath and provenance can be recorded only after the complete bytes are in place. const outputAttemptPath = `${outputPath}.tmp-${randomUUID()}`; try { await copyFile(pdfInWorkDir, outputAttemptPath); diff --git a/apps/worker/src/services/prompt-manager.ts b/apps/worker/src/services/prompt-manager.ts index eb46d932..89b7040d 100644 --- a/apps/worker/src/services/prompt-manager.ts +++ b/apps/worker/src/services/prompt-manager.ts @@ -39,6 +39,21 @@ const VULN_CLASS_HEADINGS: Record = { ssrf: 'Server-Side Request Forgery (SSRF) Vulnerabilities', }; +// Inline report scope labels are intentionally separate from section headings. The report +// needs formal names without the repeated "Vulnerabilities" suffix. +const VULN_CLASS_SCOPE_LABELS: Readonly> = Object.freeze({ + injection: 'Injection', + xss: 'Cross-Site Scripting (XSS)', + auth: 'Authentication', + authz: 'Authorization', + ssrf: 'Server-Side Request Forgery (SSRF)', +}); + +/** Render the fixed workflow-owned vulnerability scope for a reader. */ +export function formatVulnClassScope(classes: readonly VulnClass[]): string { + return classes.map((vulnerabilityClass) => VULN_CLASS_SCOPE_LABELS[vulnerabilityClass]).join(', '); +} + /** * Renders the block. Empty when every class completed. * @@ -139,6 +154,8 @@ function renderReportFilterRules(report: DistributedReportConfig | undefined, ex interface PromptVariables { webUrl: string; repoPath: string; + /** Workflow-owned UTC date for report metadata and prose. */ + assessmentDate?: string; /** Classes whose analysis did not complete, so the report can mark them not assessed. */ failedClasses?: readonly VulnClass[]; /** Explicit workflow-owned analysis scope for prompts that describe tested classes. */ @@ -335,6 +352,14 @@ async function interpolateVariables( let result = template; result = replaceLiteral(result, /{{WEB_URL}}/g, variables.webUrl); result = replaceLiteral(result, /{{REPO_PATH}}/g, variables.repoPath); + if (result.includes('{{ASSESSMENT_DATE}}')) { + if (variables.assessmentDate === undefined) { + throw new PentestError('Prompt requires a workflow-owned assessment date', 'prompt', false, { + placeholder: 'ASSESSMENT_DATE', + }); + } + result = replaceLiteral(result, /{{ASSESSMENT_DATE}}/g, variables.assessmentDate); + } result = replaceLiteral(result, /{{PLAYWRIGHT_SESSION}}/g, variables.PLAYWRIGHT_SESSION || 'agent1'); result = replaceLiteral(result, /{{AUTH_CONTEXT}}/g, buildAuthContext(config)); result = replaceLiteral( @@ -384,13 +409,16 @@ async function interpolateVariables( } if (result.includes('{{VULN_CLASSES_TESTED}}')) { + // Fails closed instead of falling back to a guessed or hardcoded class list: a template + // that describes tested classes must receive the workflow's resolved scope explicitly, so + // a caller that forgets to pass analysisClasses cannot silently render a stale scope. if (variables.analysisClasses === undefined) { throw new PentestError('Prompt requires an explicit workflow-owned analysis scope', 'prompt', false, { placeholder: 'VULN_CLASSES_TESTED', }); } assertFixedAnalysisScope(variables.analysisClasses); - result = replaceLiteral(result, /{{VULN_CLASSES_TESTED}}/g, variables.analysisClasses.join(', ')); + result = replaceLiteral(result, /{{VULN_CLASSES_TESTED}}/g, formatVulnClassScope(variables.analysisClasses)); } result = replaceLiteral( result, diff --git a/apps/worker/src/services/queue-validation.ts b/apps/worker/src/services/queue-validation.ts index 4fa342ce..67ac1ab6 100644 --- a/apps/worker/src/services/queue-validation.ts +++ b/apps/worker/src/services/queue-validation.ts @@ -73,6 +73,9 @@ interface QueueValidationResult { error: string | null; } +// Filesystem faults are retryable by default: a transient I/O blip should not fail the run. +// The listed codes are the deterministic ones (bad path, wrong type, permissions) that will +// never resolve on retry, so they classify as non-retryable instead. function isRetryableQueueFileSystemError(error: unknown): boolean { if (!(error instanceof Error)) return false; const code = (error as NodeJS.ErrnoException).code; @@ -154,7 +157,7 @@ const PARTIAL_RESULTS_MESSAGE = /** * Name the outcome the reader can act on rather than the files behind it: nothing landed at - * all, or only some of what the class owes. The analysis-less `other` class has no + * all, or only some of what the class owes. The analysis-less `miscellaneous` class has no * deliverable, so its queue alone decides which of the two applies. */ function getExistenceErrorMessage({ existence, deliverableRequired, vulnerabilityClass }: ExistenceContext): string { @@ -358,6 +361,8 @@ export async function validateQueueSafe( return ok(result); } catch (error) { if (error instanceof PentestError) return err(error); + // Fail closed: an unexpected non-PentestError becomes a non-retryable error rather than + // being treated as a passed validation that would let exploitation proceed on bad input. return err( new PentestError('Queue validation failed closed on an internal invariant.', 'unknown', false, { vulnType }), ); diff --git a/apps/worker/src/services/renumber-core.ts b/apps/worker/src/services/renumber-core.ts index 41fcfffe..467bf15f 100644 --- a/apps/worker/src/services/renumber-core.ts +++ b/apps/worker/src/services/renumber-core.ts @@ -51,6 +51,8 @@ export function remapTaskReferences(text: string, oldToNew: ReadonlyMap second.length - first.length); if (oldReferences.length === 0) return text; const alternation = oldReferences.map(escapeForRegExp).join('|'); + // Longest-first alternation plus the digit lookahead keep a shorter reference from matching + // as a prefix of a longer one (INJ-1 inside INJ-10 would otherwise corrupt the longer ref). const referencePattern = new RegExp(`(?:${alternation})(?![0-9])`, 'g'); return text.replace(referencePattern, (oldReference) => oldToNew.get(oldReference) as string); } @@ -70,6 +72,7 @@ function scrubEntryText(value: unknown, oldToNew: ReadonlyMap): return value; } +/** Accepts only the exact zero-padded form the pipeline mints (INJ-01, not INJ-1 or INJ-001). */ export function parseRefNumber(reference: string, vulnerabilityClass: ReconciliationClass): number | null { const prefix = REF_PREFIX[vulnerabilityClass]; const match = new RegExp(`^${escapeForRegExp(prefix)}-(\\d+)$`).exec(reference); @@ -179,6 +182,8 @@ export function remapSastProvenance( const seen = new Set(); for (const entry of provenance.entries) { const nextReference = oldToNew.get(entry.exploit_ref); + // Entries with no mapping belong to findings the remap dropped (blocked or filtered out), + // so their provenance is dropped with them rather than kept pointing at a dead reference. if (nextReference === undefined) continue; if (seen.has(nextReference)) throw new RenumberError('key-set-divergence', false, { checkCode: 'provenance-duplicate' }); @@ -239,6 +244,14 @@ function parseProvenance(value: unknown, vulnerabilityClass: ReconciliationClass return value as SastProvenanceFile; } +/** + * Trust SAST provenance only after proving the whole committed publication around it: the class + * manifest is present, every consumer file matches its recorded digest, the queue's references + * agree with the manifest lineage in order, the collector references all fall inside that + * lineage, and every provenance entry points at a SAST-backed task. Returns undefined when the + * class published no provenance file; any other gap is a divergence, since partial trust here + * would let a stale or tampered file relabel findings in the customer report. + */ async function loadAndValidateProvenance( dir: string, vulnerabilityClass: ReconciliationClass, @@ -336,6 +349,8 @@ export async function computeRenumber( ...map.excluded.map((entry) => entry.source_ref), ]); const sparseProvenance = await loadAndValidateProvenance(dir, vulnerabilityClass, collectorReferences); + // The renderer's empty-state banner describes an empty queue, but an empty renumbered set + // means every queued finding failed exploitation, so the wording is corrected here. const evidenceMarkdown = renderExploitDeliverable( vulnerabilityClass, map.renumbered, @@ -392,7 +407,7 @@ export interface RenumberClassResult { readonly commit?: ExactOutputCommit; } -/** Service boundary used by the later Temporal activity wrapper. */ +/** Service boundary behind the Temporal activity wrapper. Skips when the class has no collector. */ export async function renumberClassFindings(args: { readonly deliverablesDir: string; readonly vulnerabilityClass: ReconciliationClass; diff --git a/apps/worker/src/services/report-checkpoints.ts b/apps/worker/src/services/report-checkpoints.ts index f54b2cf4..958edb57 100644 --- a/apps/worker/src/services/report-checkpoints.ts +++ b/apps/worker/src/services/report-checkpoints.ts @@ -1,4 +1,4 @@ -// Copyright (C) 2025 Keygraph, Inc. +// Copyright (C) 2026 Keygraph, Inc. // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License version 3 @@ -115,6 +115,12 @@ async function checkpointIsReachable(deliverablesPath: string, checkpoint: strin return head !== null && (await checkpointIsAncestor(checkpoint, head, deliverablesPath)); } +/** + * Prove one report checkpoint is trustworthy: its report.json parses with meta and findings, + * its recorded failed-class set equals the durable `failedClasses` list, and the commit is + * still reachable from HEAD. Reachability matters because rollback can rewrite the branch; + * a dangling checkpoint would validate content that publication has since discarded. + */ export async function reportCheckpointIsCoherent( deliverablesPath: string, checkpoint: string, @@ -139,6 +145,12 @@ export async function reportCheckpointIsCoherent( export type DraftValidation = 'coherent' | 'invalid-model' | 'invalid-canonical'; +/** + * Grade a durable draft for resume. `invalid-model` means the model-authored checkpoint itself + * cannot be trusted and the report stage must re-run; `invalid-canonical` means the model + * checkpoint stands but the canonical rebuild on top of it does not, so only that later work + * repeats. A pending draft is vacuously coherent. + */ export async function validateDraftProgress( deliverablesPath: string, progress: ReportProgress, @@ -153,13 +165,12 @@ export async function validateDraftProgress( if (!(await checkpointIsAncestor(progress.model_checkpoint, progress.canonical_checkpoint, deliverablesPath))) { return 'invalid-canonical'; } - return (await reportCheckpointIsCoherent( + const canonicalCoherent = await reportCheckpointIsCoherent( deliverablesPath, progress.canonical_checkpoint, progress.renumber_failed_classes, - )) - ? 'coherent' - : 'invalid-canonical'; + ); + return canonicalCoherent ? 'coherent' : 'invalid-canonical'; } export async function draftProgressIsCoherent(deliverablesPath: string, progress: ReportProgress): Promise { @@ -186,6 +197,9 @@ export async function finalProgressIsCoherent(deliverablesPath: string, progress } catch { return false; } + // The durable digest covers the manifest's exact bytes, so the committed file must also be in + // canonical serialization; a manifest that parses but is formatted differently is not the one + // finalization wrote. if (!isReportFinalizationManifest(manifest) || manifestContents !== `${JSON.stringify(manifest, null, 2)}\n`) { return false; } @@ -210,6 +224,8 @@ export async function finalProgressIsCoherent(deliverablesPath: string, progress const sarif = await checkpointFileContents(deliverablesPath, progress.final_checkpoint, SARIF_FILENAME); if (manifest.artifacts.sarif.disposition !== 'committed') { + // A non-committed disposition also requires the worktree to be clean of SARIF: a stray + // uncommitted file would otherwise be surfaced to the customer as though it were finalized. return sarif === null && !(await fileExists(path.join(deliverablesPath, SARIF_FILENAME))); } return sarif !== null && sha256(sarif) === manifest.artifacts.sarif.sha256; diff --git a/apps/worker/src/services/report-finalization.ts b/apps/worker/src/services/report-finalization.ts index 3606f5ee..a263222d 100644 --- a/apps/worker/src/services/report-finalization.ts +++ b/apps/worker/src/services/report-finalization.ts @@ -20,15 +20,18 @@ import { import type { ActivityLogger } from '../types/activity-logger.js'; import type { DistributedReportConfig } from '../types/config.js'; import type { ReconciliationClass } from '../types/reconciliation.js'; +import { isOrderedPartialReasonSet, type PartialReason } from '../types/run-state.js'; import type { ExactOutputCommit, ExactOutputFile } from './exact-output-commit.js'; import { writeAndCommitExactFiles } from './exact-output-commit.js'; import { orderFindings } from './finding-order.js'; import { readCommittedFile, withGitRepoLock } from './git-manager.js'; import { type PdfProvenance, pdfProvenanceIsCurrent, readPdfProvenance, renderReportPdf } from './pdf-renderer.js'; -import { type ReportData, renderReport } from './report-renderer.js'; +import { buildReportCoverage, type ReportData, renderReport } from './report-renderer.js'; import { renderSarif } from './sarif-renderer.js'; const FINALIZATION_SCHEMA_VERSION = 1; +// Renderer versions feed the input fingerprint: bumping one invalidates adoption of an existing +// finalization, forcing a re-render instead of adopting output from an older renderer. const REPORT_RENDERER_VERSION = '4.13.1'; const SARIF_RENDERER_VERSION = '4.13.1'; @@ -131,7 +134,7 @@ function isSha256(value: unknown): value is string { return typeof value === 'string' && /^[a-f0-9]{64}$/.test(value); } -/** Runtime guard consumed by resume/terminal-state wiring in the next task. */ +/** Runtime guard for manifests read back from the deliverables repo during resume and repair. */ export function isReportFinalizationManifest(value: unknown): value is ReportFinalizationManifest { if (!isRecord(value) || value.schema_version !== FINALIZATION_SCHEMA_VERSION || !isSha256(value.input_fingerprint)) { return false; @@ -174,8 +177,12 @@ export function isReportFinalizationManifest(value: unknown): value is ReportFin function canonicalizeReport(args: { readonly report: ReportData; readonly exploit: boolean; + readonly partialReasons: readonly PartialReason[]; readonly reconciliationFailedClasses?: readonly ReconciliationClass[]; }): ReportData { + if (!isOrderedPartialReasonSet(args.partialReasons)) { + throw new ReportFinalizationIntegrityError('finalization-partial-reasons-invalid'); + } const reconciliationFailed = args.reconciliationFailedClasses ?? args.report.reconciliation_failed ?? []; if (new Set(reconciliationFailed).size !== reconciliationFailed.length) { throw new ReportFinalizationIntegrityError('finalization-failed-class-duplicate'); @@ -188,12 +195,19 @@ function canonicalizeReport(args: { } return { ...args.report, - report_meta: { ...args.report.report_meta, exploit: args.exploit }, + report_meta: { + ...args.report.report_meta, + exploit: args.exploit, + coverage: buildReportCoverage(args.partialReasons), + }, findings: orderFindings(args.report.findings), reconciliation_failed: [...reconciliationFailed], }; } +// The fingerprint covers everything that determines the finalized bytes. Degraded-SARIF mode is +// deliberately excluded: a degraded re-drive must produce the same fingerprint as the ordinary +// attempt so it can adopt an existing coherent commit instead of publishing a rival one. function buildInputFingerprint(args: { readonly canonicalJson: string; readonly exploit: boolean; @@ -249,6 +263,13 @@ function buildManifest(args: { }; } +/** + * Load a previously committed finalization for adoption. Returns null when none exists, the + * verified manifest and SARIF when the committed state matches the current inputs byte for + * byte, and throws a terminal integrity error on any conflict or corruption. This runs before + * any new render, so a lost acknowledgement re-drive (including a degraded one) adopts the + * earlier coherent commit rather than replacing it. + */ async function readExistingFinalization(args: { readonly deliverablesDir: string; readonly canonicalJson: string; @@ -313,6 +334,7 @@ async function readExistingFinalization(args: { export async function finalizeReport(args: { readonly deliverablesDir: string; readonly exploit: boolean; + readonly partialReasons: readonly PartialReason[]; readonly reconciliationFailedClasses?: readonly ReconciliationClass[]; readonly reportConfig: DistributedReportConfig; readonly workspaceName: string; @@ -335,6 +357,7 @@ export async function finalizeReport(args: { const canonicalReport = canonicalizeReport({ report: parseReportData(reportRead.contents), exploit: args.exploit, + partialReasons: args.partialReasons, ...(args.reconciliationFailedClasses !== undefined && { reconciliationFailedClasses: args.reconciliationFailedClasses, }), @@ -369,6 +392,9 @@ export async function finalizeReport(args: { const sarifRequested = args.exploit && args.reportConfig.sarif; let sarif: string | null = null; let sarifDisposition: ReportSarifDisposition = 'absent'; + // Reaching this point in degraded mode means no coherent finalization existed to adopt, so + // the manifest records `render_failed` and the null SARIF contents below make the exact-path + // commit delete any SARIF a partial earlier attempt left behind. if (sarifRequested && args.degradedSarif === true) { sarifDisposition = 'render_failed'; } else if (sarifRequested) { @@ -428,6 +454,8 @@ export async function finalizeReport(args: { const warning = `The PDF report could not be produced (${label}). The Markdown report and the structured findings are unaffected.`; warnings.push(warning); args.logger.warn(warning); + // Keep an old PDF only when its bytes still prove out against the current canonical report; + // otherwise remove it so a stale PDF is never served alongside fresh JSON and Markdown. const canPreservePriorPdf = args.priorPdfProvenance !== undefined && (await pdfProvenanceIsCurrent({ diff --git a/apps/worker/src/services/report-json-adapter.ts b/apps/worker/src/services/report-json-adapter.ts index 4becb787..8a97133a 100644 --- a/apps/worker/src/services/report-json-adapter.ts +++ b/apps/worker/src/services/report-json-adapter.ts @@ -17,6 +17,7 @@ */ import type { AddFindingInput, AdditionalSection, StepItem, StructuredStep } from '../collectors/finding-collector.js'; +import { orderFindings } from './finding-order.js'; import type { ExploitsReportData, FindingsReportData, @@ -28,6 +29,8 @@ import type { } from './report-output-schema.js'; import type { ReportData } from './report-renderer.js'; +const COMPLETE_COVERAGE = { status: 'complete' as const, limitations: [] as const }; + // ============================================================================ // CASING TRANSFORMS // ============================================================================ @@ -58,7 +61,7 @@ const VALID_CATEGORIES = new Set([ 'XSS', 'Injection', 'SSRF', - 'Other', + 'Miscellaneous', ]); function toTypstSeverity(s: string): TypstSeverity { @@ -75,13 +78,16 @@ function toTypstConfidence(s: string): TypstConfidence { function toTypstCategory(s: string): TypstCategory { if (VALID_CATEGORIES.has(s as TypstCategory)) return s as TypstCategory; - return 'Other'; + return 'Miscellaneous'; } // ============================================================================ // STEP / ITEM TRANSFORMS // ============================================================================ +// StepItem is currently identical on both sides of the adapter boundary, so this is a no-op. +// It stays as an explicit seam rather than being inlined so a future divergence between the +// collector's StepItem and the Typst schema's StepItem has a single place to add the conversion. function adaptStepItem(item: StepItem): StepItem { return item; } @@ -110,6 +116,9 @@ interface CategoryGroup { findings: AddFindingInput[]; } +// Groups in insertion order, so callers must pass findings already ordered by category +// (orderFindings) for the resulting groups to come out in CATEGORY_ORDER: this function does +// not re-sort the groups it produces. function groupByCategory(findings: readonly AddFindingInput[]): CategoryGroup[] { const map = new Map(); for (const f of findings) { @@ -140,7 +149,8 @@ function countBySeverity(findings: readonly AddFindingInput[]): Record { const exploited = g.findings.filter((f) => (f.status ?? 'exploited') === 'exploited'); if (exploited.length === 0) { @@ -218,7 +230,8 @@ function adaptExploitsMode(data: ReportData): ExploitsReportData { // ============================================================================ function adaptFindingsMode(data: ReportData): FindingsReportData { - const { report_meta, findings } = data; + const { report_meta } = data; + const findings = orderFindings(data.findings); const groups = groupByCategory(findings); const sevCounts = countBySeverity(findings); @@ -235,7 +248,9 @@ function adaptFindingsMode(data: ReportData): FindingsReportData { assessmentDate: report_meta.assessment_date, classification: 'CONFIDENTIAL', }, + executiveSummary: report_meta.executive_summary, scope: report_meta.scope, + coverage: report_meta.coverage ?? COMPLETE_COVERAGE, identifiedByType: groups.map((g) => { if (g.findings.length === 0) { return { diff --git a/apps/worker/src/services/report-output-schema.ts b/apps/worker/src/services/report-output-schema.ts index 3f53ac90..8d4555a9 100644 --- a/apps/worker/src/services/report-output-schema.ts +++ b/apps/worker/src/services/report-output-schema.ts @@ -16,7 +16,7 @@ export type TypstSeverity = 'Critical' | 'High' | 'Medium' | 'Low'; export type TypstStatus = 'Exploited' | 'OutOfScope' | 'BlockedByConstraints' | 'FalsePositive'; export type TypstConfidence = 'High' | 'Medium' | 'Low'; -export type TypstCategory = 'Authentication' | 'Authorization' | 'XSS' | 'Injection' | 'SSRF' | 'Other'; +export type TypstCategory = 'Authentication' | 'Authorization' | 'XSS' | 'Injection' | 'SSRF' | 'Miscellaneous'; export interface CodeBlock { readonly language: string; @@ -52,6 +52,16 @@ export interface Meta { readonly classification: string; } +export interface ReportLimitation { + readonly code: string; + readonly message: string; +} + +export interface ReportCoverage { + readonly status: 'complete' | 'partial'; + readonly limitations: readonly ReportLimitation[]; +} + export interface CategoryCount { readonly category: TypstCategory; readonly count: number; @@ -92,7 +102,9 @@ export interface ExploitedByTypeEntry { export interface ExploitsReportData { readonly mode: 'exploits'; readonly meta: Meta; + readonly executiveSummary: string; readonly scope: string; + readonly coverage: ReportCoverage; readonly exploitedByType: readonly ExploitedByTypeEntry[]; readonly summary: { readonly totalIdentified: number; @@ -138,7 +150,9 @@ export interface IdentifiedByTypeEntry { export interface FindingsReportData { readonly mode: 'findings'; readonly meta: Meta; + readonly executiveSummary: string; readonly scope: string; + readonly coverage: ReportCoverage; readonly identifiedByType: readonly IdentifiedByTypeEntry[]; readonly summary: { readonly totalIdentified: number; diff --git a/apps/worker/src/services/report-output-surface.ts b/apps/worker/src/services/report-output-surface.ts index 85bb5819..d1d085e3 100644 --- a/apps/worker/src/services/report-output-surface.ts +++ b/apps/worker/src/services/report-output-surface.ts @@ -59,6 +59,8 @@ async function atomicCopy(sourcePath: string, destinationPath: string): Promise< } catch (error) { if (!isErrno(error, 'ENOENT')) throw error; } + // Write to a unique temp path then rename, so a crash mid-write never leaves a partial file + // at the customer destination. `wx` fails rather than reusing a leftover temp file. const temporaryPath = `${destinationPath}.tmp-${randomUUID()}`; try { await writeFile(temporaryPath, contents, { flag: 'wx' }); @@ -75,6 +77,12 @@ async function atomicCopy(sourcePath: string, destinationPath: string): Promise< } } +/** + * Surface the PDF only when its bytes verify against the current canonical report's provenance. + * A matching deliverables PDF is copied out; otherwise the unverifiable source is removed, and + * an already-verified customer PDF is preserved rather than deleted. Every failure here is a + * warning: the PDF is a secondary artifact and cannot fail the run or change its status. + */ async function surfaceProvenancedPdf(args: { readonly deliverablesDir: string; readonly customerDir: string; @@ -159,15 +167,17 @@ export async function surfaceReportOutputs(args: { destination: FINAL_REPORT_MD_FILENAME, removeWhenSourceMissing: false, }, - { source: SARIF_FILENAME, destination: SARIF_FILENAME, removeWhenSourceMissing: true }, ]; + // With provenance supplied, the PDF is surfaced by surfaceProvenancedPdf under a byte check; + // without it, the PDF falls back to the same unconditional copy as the other outputs. if (args.pdfVerification === undefined) { - outputs.splice(1, 0, { + outputs.push({ source: ASSEMBLED_REPORT_PDF_FILENAME, destination: FINAL_REPORT_PDF_FILENAME, removeWhenSourceMissing: true, }); } + outputs.push({ source: SARIF_FILENAME, destination: SARIF_FILENAME, removeWhenSourceMissing: true }); const copyOutput = args.copyOutput ?? atomicCopy; const surfaced: string[] = []; const removedStale: string[] = []; diff --git a/apps/worker/src/services/report-renderer.ts b/apps/worker/src/services/report-renderer.ts index 953b48f4..2d8d6139 100644 --- a/apps/worker/src/services/report-renderer.ts +++ b/apps/worker/src/services/report-renderer.ts @@ -15,6 +15,8 @@ import type { AddFindingInput, AdditionalSection, StepItem, StructuredStep } from '../collectors/finding-collector.js'; import type { VulnClass } from '../types/config.js'; import type { ReconciliationClass } from '../types/reconciliation.js'; +import { classDisplayName, type PartialReason, type PartialReasonCode } from '../types/run-state.js'; +import { compareCategories, orderFindings } from './finding-order.js'; // ============================================================================ // TYPES @@ -27,6 +29,18 @@ export interface ReportMeta { readonly executive_summary: string; readonly exploit?: boolean; readonly model?: string; + /** Added deterministically during finalization; model-authored values are ignored. */ + readonly coverage?: ReportCoverage; +} + +export interface ReportLimitation { + readonly code: PartialReasonCode; + readonly message: string; +} + +export interface ReportCoverage { + readonly status: 'complete' | 'partial'; + readonly limitations: readonly ReportLimitation[]; } export interface ReportData { @@ -35,7 +49,11 @@ export interface ReportData { // Vuln classes whose pipeline failed and were not assessed this run. Rendered as an explicit // caveat so an un-assessed class is never presented as a clean result. readonly not_assessed?: readonly VulnClass[]; - /** Exploit classes excluded from compaction after a renumber failure, in workflow order. */ + /** + * Exploit classes excluded from compaction after a renumber failure, in workflow order. Read + * by report finalization and its checkpoint idempotency check, not by renderReport: this + * field carries no markdown output of its own. + */ readonly reconciliation_failed?: readonly ReconciliationClass[]; } @@ -46,6 +64,40 @@ const ANALYSIS_ONLY_DISCLAIMER = [ '> exploitation steps or proof of impact are included.', ].join('\n'); +const COMPLETE_COVERAGE: ReportCoverage = Object.freeze({ status: 'complete', limitations: Object.freeze([]) }); + +function limitationMessage(reason: PartialReason): string { + switch (reason.code) { + case 'agentic_sast_failed': + return 'Agentic SAST did not complete, so its findings are not included.'; + case 'agentic_sast_reduced': + if (reason.stage === 'research') return 'Some files were not reviewed by Agentic SAST.'; + if (reason.stage === 'export') return 'Some Agentic SAST findings are not included.'; + return 'Some Agentic SAST results are not included.'; + case 'class_pipeline_failed': + return `${classDisplayName(reason.vulnerabilityClass as ReconciliationClass)} was not fully assessed.`; + case 'class_reconciliation_failed': + return `${classDisplayName(reason.vulnerabilityClass as ReconciliationClass)} was analyzed but not exploited.`; + case 'report_renumber_failed': + return `${classDisplayName(reason.vulnerabilityClass as ReconciliationClass)} finding numbers may contain gaps; all findings are included.`; + case 'report_compaction_failed': + return 'Finding numbers may contain gaps; all findings are included.'; + case 'report_class_omitted': + return `${classDisplayName(reason.vulnerabilityClass as ReconciliationClass)} was assessed, but its results are not included.`; + case 'report_sarif_failed': + return 'The SARIF report could not be generated.'; + } +} + +/** Project durable degradation state into the deliberately small customer report contract. */ +export function buildReportCoverage(reasons: readonly PartialReason[]): ReportCoverage { + if (reasons.length === 0) return COMPLETE_COVERAGE; + return { + status: 'partial', + limitations: reasons.map((reason) => ({ code: reason.code, message: limitationMessage(reason) })), + }; +} + const NOT_ASSESSED_LABELS: Record = { auth: 'Authentication', authz: 'Authorization', @@ -150,8 +202,8 @@ function renderFinding(finding: AddFindingInput, exploitEnabled: boolean): strin if (finding.exploitation_steps && finding.exploitation_steps.length > 0) { lines.push('**Exploitation Steps:**'); lines.push(''); - for (let i = 0; i < finding.exploitation_steps.length; i++) { - lines.push(renderStructuredStep(finding.exploitation_steps[i]!, i)); + for (const [index, step] of finding.exploitation_steps.entries()) { + lines.push(renderStructuredStep(step, index)); lines.push(''); } } @@ -187,29 +239,16 @@ function renderFinding(finding: AddFindingInput, exploitEnabled: boolean): strin return lines.join('\n').trimEnd(); } -// ============================================================================ -// CATEGORY GROUPING -// ============================================================================ - -const CATEGORY_ORDER: readonly string[] = ['Injection', 'XSS', 'Authentication', 'SSRF', 'Authorization']; - -function categorySort(a: string, b: string): number { - const ai = CATEGORY_ORDER.indexOf(a); - const bi = CATEGORY_ORDER.indexOf(b); - if (ai !== -1 && bi !== -1) return ai - bi; - if (ai !== -1) return -1; - if (bi !== -1) return 1; - return a.localeCompare(b); -} - // ============================================================================ // REPORT RENDERING // ============================================================================ export function renderReport(data: ReportData): string { - const { report_meta, findings, not_assessed = [] } = data; + const { report_meta, not_assessed = [] } = data; + const findings = orderFindings(data.findings); const notAssessedClasses = [...new Set(not_assessed)]; const exploitEnabled = report_meta.exploit ?? true; + const coverage = report_meta.coverage ?? COMPLETE_COVERAGE; const sections: string[] = []; // 1. Executive Summary @@ -220,6 +259,7 @@ export function renderReport(data: ReportData): string { sections.push(`- Assessment Date: ${report_meta.assessment_date}`); sections.push(`- Scope: ${report_meta.scope}`); sections.push(`- Exploitation: ${exploitEnabled ? 'enabled' : 'disabled'}`); + sections.push(`- Assessment Status: ${coverage.status === 'partial' ? 'Completed with limitations' : 'Complete'}`); if (report_meta.model) { sections.push(`- Model: ${report_meta.model}`); } @@ -231,20 +271,31 @@ export function renderReport(data: ReportData): string { sections.push(''); } + if (coverage.status === 'partial' && coverage.limitations.length > 0) { + sections.push('## Limitations'); + sections.push(''); + for (const limitation of coverage.limitations) { + sections.push(`- ${limitation.message}`); + } + sections.push(''); + } + if (findings.length === 0) { if (notAssessedClasses.length > 0) { // Some classes were not assessed — a blanket "no vulnerabilities" statement would be a false // clean bill of health. Scope the clean statement to assessed classes and list the gaps. sections.push('No vulnerabilities were identified in the classes that were assessed.'); - sections.push(''); - sections.push(renderNotAssessedSection(notAssessedClasses)); + if (coverage.status !== 'partial' || coverage.limitations.length === 0) { + sections.push(''); + sections.push(renderNotAssessedSection(notAssessedClasses)); + } } else { sections.push('No vulnerabilities were identified during this assessment.'); } - return sections.join('\n').trimEnd() + '\n'; + return `${sections.join('\n').trimEnd()}\n`; } - if (notAssessedClasses.length > 0) { + if (notAssessedClasses.length > 0 && (coverage.status !== 'partial' || coverage.limitations.length === 0)) { sections.push(renderNotAssessedSection(notAssessedClasses)); sections.push(''); } @@ -257,12 +308,12 @@ export function renderReport(data: ReportData): string { byCategory.set(f.category, list); } - const sortedCategories = [...byCategory.keys()].sort(categorySort); + const sortedCategories = [...byCategory.keys()].sort(compareCategories); sections.push('## Summary by Vulnerability Type'); sections.push(''); for (const cat of sortedCategories) { - const catFindings = byCategory.get(cat)!; + const catFindings = byCategory.get(cat) ?? []; sections.push(`### ${cat}`); sections.push(''); for (const f of catFindings) { @@ -286,7 +337,7 @@ export function renderReport(data: ReportData): string { const heading = exploitEnabled ? 'Exploitation Evidence' : 'Findings'; for (const cat of sortedCategories) { - const catFindings = byCategory.get(cat)!; + const catFindings = byCategory.get(cat) ?? []; sections.push(`# ${cat} ${heading}`); sections.push(''); sections.push(`## ${subheading}`); @@ -297,5 +348,5 @@ export function renderReport(data: ReportData): string { } } - return sections.join('\n').trimEnd() + '\n'; + return `${sections.join('\n').trimEnd()}\n`; } diff --git a/apps/worker/src/services/reporting.ts b/apps/worker/src/services/reporting.ts index bc527891..18f115fa 100644 --- a/apps/worker/src/services/reporting.ts +++ b/apps/worker/src/services/reporting.ts @@ -49,6 +49,8 @@ const DELIVERABLE_BY_CLASS: Readonly< }, }); +// `miscellaneous` is absent on purpose: it joins the report only when the workflow passes it in +// `participatingClasses`, since the class exists only on runs that produced miscellaneous-class tasks. const DEFAULT_REPORT_CLASS_ORDER = [ 'injection', 'xss', @@ -108,9 +110,14 @@ async function assembleFinalReportInternal( const participatingClasses = options.participatingClasses ?? DEFAULT_REPORT_CLASS_ORDER; const deliverableFiles: readonly DeliverableFile[] = participatingClasses.map((vulnerabilityClass) => { const definition = DELIVERABLE_BY_CLASS[vulnerabilityClass]; - let paths: readonly string[] = [definition.exploit, definition.analysis]; - if (options.exploit === true) paths = [definition.exploit]; - if (options.exploit === false) paths = [definition.analysis]; + let paths: readonly string[]; + if (options.exploit === true) { + paths = [definition.exploit]; + } else if (options.exploit === false) { + paths = [definition.analysis]; + } else { + paths = [definition.exploit, definition.analysis]; + } return { vulnerabilityClass, name: definition.name, paths, required: false }; }); @@ -126,6 +133,9 @@ async function assembleFinalReportInternal( let added = false; for (const candidate of file.paths) { try { + // Exploit runs assemble from committed Git state, not the worktree: evidence is only + // trustworthy once checkpointed, and a corrupt committed object is a class failure + // rather than a silently skipped section. if (options.exploit === true) { const committed = await readCommittedFile(dir, candidate); if (committed.state === 'corrupt') { @@ -211,9 +221,13 @@ export async function assembleFinalReportWithEvidence( return assembleFinalReportInternal(sourceDir, deliverablesSubdir, logger, options, true); } -// Pure function: Assemble final report from specialist deliverables. -// Per class, prefer the exploit-agent's evidence file; fall back to renderer-produced findings. -// Both never coexist for a workspace because scope (exploit flag) is locked. +/** + * Assemble the final report from per-class deliverables and return only the content. With an + * explicit exploit mode each class reads exactly its evidence or findings file; without one, + * evidence is preferred and findings are the fallback. The boolean form of the last parameter + * is shorthand for `{ exploit }`. In exploit mode a read failure throws instead of being + * collected, because canonical evidence assembly must not silently omit a class. + */ export async function assembleFinalReport( sourceDir: string, deliverablesSubdir: string | undefined, @@ -240,7 +254,7 @@ export async function assembleFinalReport( * * The SARIF log is surfaced beside it when present, since a CI step consuming it needs a stable * path and cannot be expected to reach into the internals directory. It is absent whenever the - * run was analysis-only or `report.sarif` was not enabled. + * run was analysis-only or `report.sarif` was set to false. */ export async function copyReportToRunRoot( repoPath: string, diff --git a/apps/worker/src/services/sarif-renderer.ts b/apps/worker/src/services/sarif-renderer.ts index 3a892a89..9ce0216f 100644 --- a/apps/worker/src/services/sarif-renderer.ts +++ b/apps/worker/src/services/sarif-renderer.ts @@ -6,7 +6,8 @@ /** Deterministic report.json to SARIF 2.1.0 renderer, for `exploit=true` runs only. */ -import type { AddFindingInput, CodeLocation } from '../collectors/finding-collector.js'; +import type { AddFindingInput, CodeLocation, SASTSourceLocation } from '../collectors/finding-collector.js'; +import { CATEGORY_ORDER, orderFindings } from './finding-order.js'; import type { ReportData } from './report-renderer.js'; export interface SarifOptions { @@ -96,10 +97,20 @@ const RULES: Record = { }, properties: { tags: ['security', 'shannon'] }, }, + Miscellaneous: { + id: 'shannon/miscellaneous', + name: 'Miscellaneous Security Vulnerability', + shortDescription: { text: 'Miscellaneous Security Vulnerability' }, + fullDescription: { + text: "A security weakness outside Shannon's named vulnerability classes that can affect confidentiality, integrity, or availability.", + }, + help: { + text: 'Apply the finding-specific remediation, add a regression test at the affected trust boundary, and verify that equivalent entry points enforce the same control.', + }, + properties: { tags: ['security', 'shannon'] }, + }, }; -const CATEGORY_ORDER: readonly string[] = ['Injection', 'XSS', 'Authentication', 'SSRF', 'Authorization']; - /** * Five severities collapse into SARIF's three usable levels, so `critical` and `high` are * indistinguishable. `security-severity` would separate them but lives on the rule, which would @@ -132,6 +143,17 @@ function toPhysicalLocation(location: CodeLocation) { }; } +function toSastPhysicalLocation(location: SASTSourceLocation) { + return { + physicalLocation: { + artifactLocation: { uri: location.file }, + // SAST source locations store zero-based columns; SARIF regions are one-based. + region: { startLine: location.line, startColumn: location.column + 1 }, + }, + message: { text: `Validated SAST source location (${location.rule_id})` }, + }; +} + /** * Fall back to the HTTP entry point when a finding names no file: a result with no location is * silently discarded downstream. No `uriBaseId`, since the path does not resolve in the repo. @@ -174,13 +196,22 @@ interface RenderedResult { readonly owaspId: string; } -function renderResult(finding: AddFindingInput, ruleId: string): RenderedResult | null { +function renderResult(finding: AddFindingInput, ruleId: string, ruleCategory: string): RenderedResult | null { const codeLocations = finding.code_locations ?? []; const sinks = codeLocations.filter((l) => l.role === 'sink'); const related = codeLocations.filter((l) => l.role !== 'sink'); const primary = sinks[0] ?? codeLocations[0]; - const locations = primary ? [toPhysicalLocation(primary)] : [syntheticLocationFromHttp(finding)].filter(Boolean); + // Location precedence: analysis-authored code locations, then the validated SAST source + // location, then the synthetic HTTP entry point as the last resort before omission. + let locations: unknown[] = []; + if (primary !== undefined) { + locations = [toPhysicalLocation(primary)]; + } else if (finding.sast_source_location) { + locations = [toSastPhysicalLocation(finding.sast_source_location)]; + } else { + locations = [syntheticLocationFromHttp(finding)].filter((location) => location !== undefined); + } if (locations.length === 0) return null; const properties: Record = { findingId: finding.finding_id }; @@ -188,11 +219,12 @@ function renderResult(finding: AddFindingInput, ruleId: string): RenderedResult if (finding.status) properties.status = finding.status; if (finding.auth_state) properties.authState = finding.auth_state; if (finding.prerequisites) properties.prerequisites = finding.prerequisites; + if (finding.sast_source_location) properties.sastRuleId = finding.sast_source_location.rule_id; const owaspId = splitOwaspCategory(finding.owasp_category).id; return { - category: finding.category, + category: ruleCategory, owaspId, result: { ruleId, @@ -223,14 +255,15 @@ function renderResult(finding: AddFindingInput, ruleId: string): RenderedResult /** Render a SARIF 2.1.0 log from the structured report. Findings with no location are omitted. */ export function renderSarif(data: ReportData, options: SarifOptions): string { - const { report_meta, findings, not_assessed = [] } = data; + const { report_meta, not_assessed = [] } = data; + const findings = orderFindings(data.findings); const rendered: RenderedResult[] = []; for (const finding of findings) { - const rule = RULES[finding.category]; - if (!rule) continue; - const result = renderResult(finding, rule.id); + const ruleCategory = RULES[finding.category] === undefined ? 'Miscellaneous' : finding.category; + const rule = RULES[ruleCategory] as SarifRule; + const result = renderResult(finding, rule.id, ruleCategory); if (result !== null) rendered.push(result); } @@ -284,7 +317,11 @@ export function renderSarif(data: ReportData, options: SarifOptions): string { ], }), results, - properties: { target: report_meta.target, assessmentDate: report_meta.assessment_date }, + properties: { + target: report_meta.target, + assessmentDate: report_meta.assessment_date, + ...(report_meta.model && { model: report_meta.model }), + }, }, ], }; diff --git a/apps/worker/src/services/validate-authentication.ts b/apps/worker/src/services/validate-authentication.ts index 6c90f72f..3d5536b7 100644 --- a/apps/worker/src/services/validate-authentication.ts +++ b/apps/worker/src/services/validate-authentication.ts @@ -171,6 +171,11 @@ export async function validateAuthentication( attemptNumber, duration_ms: durationMs, cost_usd: result.cost || 0, + ...(result.inputTokens !== undefined && { input_tokens: result.inputTokens }), + ...(result.outputTokens !== undefined && { output_tokens: result.outputTokens }), + ...(result.cacheReadTokens !== undefined && { cache_read_tokens: result.cacheReadTokens }), + ...(result.cacheWriteTokens !== undefined && { cache_write_tokens: result.cacheWriteTokens }), + ...(result.turns !== undefined && { turns: result.turns }), success: classification.ok, ...(result.model !== undefined && { model: result.model }), ...(safeError !== undefined && { error: safeError.message, errorCode: safeError.code }), diff --git a/apps/worker/src/temporal/activities.ts b/apps/worker/src/temporal/activities.ts index e4d975ac..a2fd7d80 100644 --- a/apps/worker/src/temporal/activities.ts +++ b/apps/worker/src/temporal/activities.ts @@ -46,6 +46,7 @@ import { renderFindingsFromQueues } from '../services/findings-renderer.js'; import { executeGitCommandWithRetry } from '../services/git-manager.js'; import { pdfProvenanceIsCurrent } from '../services/pdf-renderer.js'; import { runPreflightChecks } from '../services/preflight.js'; +import { formatVulnClassScope } from '../services/prompt-manager.js'; import type { ExploitationDecision, VulnType } from '../services/queue-validation.js'; import { renumberClassFindings as renumberClassFindingsService, @@ -125,6 +126,8 @@ const HEARTBEAT_INTERVAL_MS = 2000; export interface ActivityInput { webUrl: string; repoPath: string; + /** Workflow-owned UTC date for every customer-facing assessment date. */ + assessmentDate?: string; configPath?: string; outputPath?: string; pipelineTestingMode?: boolean; @@ -413,6 +416,7 @@ async function runAgentActivity( configPath, pipelineTestingMode, attemptNumber, + assessmentDate: input.assessmentDate, analysisClasses: resolveAnalysisClasses(input), ...(input.promptDir !== undefined && { promptDir: input.promptDir }), ...(input.configYAML !== undefined && { configYAML: input.configYAML }), @@ -683,6 +687,12 @@ export async function runMiscellaneousExploitAgent(input: ActivityInput): Promis export async function runReportAgent(input: ActivityInput, exploit: boolean): Promise { const { createFindingCollector } = await import('../collectors/finding-collector.js'); + const assessmentDate = input.assessmentDate; + if (assessmentDate === undefined) { + throw ApplicationFailure.nonRetryable('The report assessment date is missing.', 'ConfigurationError', [ + { checkCode: 'report-assessment-date-missing' }, + ]); + } const auditSession = new AuditSession(buildSessionMetadata(input)); await auditSession.initialize(input.workflowId); @@ -734,7 +744,7 @@ export async function runReportAgent(input: ActivityInput, exploit: boolean): Pr const reportJsonPath = path.join(deliverablesPath, REPORT_JSON_FILENAME); let reportMeta: ReportMeta = { target: input.webUrl, - assessment_date: new Date().toISOString().slice(0, 10), + assessment_date: assessmentDate, scope: '', executive_summary: '', exploit, @@ -761,6 +771,15 @@ export async function runReportAgent(input: ActivityInput, exploit: boolean): Pr } else { logger.warn('Report execution returned no model identifier; canonical report metadata omits model'); } + reportMeta = { + ...reportMeta, + // The workflow supplies the scan date to the prompt and owns the final field. A model-written + // value cannot move the assessment to a different day. + assessment_date: assessmentDate, + // The workflow owns the assessed class set. The model may summarize the engagement, + // but it cannot rename or omit the canonical scope in customer output. + scope: formatVulnClassScope(resolveAnalysisClasses(input)), + }; const reportData: ReportData = { report_meta: reportMeta, @@ -1231,13 +1250,15 @@ export async function finalizeReportOutputs( const container = getOrCreateContainer(input.workflowId, buildSessionMetadata(input), buildContainerConfig(input)); const configResult = await container.configLoader.loadOptional(input.configPath, undefined, input.configYAML); if (isErr(configResult)) throw deterministicActivityFailure(configResult.error, 'ReportFinalizationError'); - // Preserve public main's default-on exploit SARIF behavior when no report config is present. + // A run with no config file at all still leaves `report.sarif` omitted, which means on: the + // default must match `distributeConfig`, or the commonest launch path would silently opt out. const reportConfig = configResult.value?.report ?? { sarif: true }; let result: Awaited>; try { result = await finalizeReport({ deliverablesDir: deliverablesPath, exploit: state.exploit, + partialReasons: state.report.partial_reasons, reconciliationFailedClasses: state.report.renumber_failed_classes, reportConfig, workspaceName: input.sessionId, @@ -1855,21 +1876,33 @@ export async function logWorkflowComplete(input: ActivityInput, summary: Workflo const { workflowId } = input; const sessionMetadata = buildSessionMetadata(input); - // 1. Initialize audit session and mark final status + // 1. Initialize the audit session. The terminal write below owns both status and totals. const auditSession = new AuditSession(sessionMetadata); await auditSession.initialize(workflowId); - await auditSession.updateSessionStatus(summary.status); - // 2. Load cumulative metrics from session.json + // 2. Operational work has no independent audit-session writer, so this terminal + // projection is its durable source of truth. + const totals = await auditSession.recordTerminalWorkflowMetrics(workflowId, { + status: summary.status, + startedAtMs: summary.startedAtMs, + endedAtMs: summary.endedAtMs, + usageAccountingComplete: summary.usageAccountingComplete !== false, + usageAccountingWarnings: summary.usageAccountingWarnings ?? [], + operationalMetrics: summary.operationalMetrics, + // The priced metrics carry cost but no faithful duration, so the stage spans are the only + // source of operational wall-clock. The tracker keeps the operational families and drops + // the rest. + operationalStages: summary.operationalStages, + }); + + // 3. Load the now-current cumulative agent ledger from session.json. const sessionData = (await auditSession.getMetrics()) as { metrics: { - total_duration_ms: number; - total_cost_usd: number; agents: Record; }; }; - // 3. Fill in metrics for skipped agents (resumed from previous run) + // 4. Fill in metrics for skipped agents that completed in an earlier workflow run. const agentMetrics = { ...summary.agentMetrics }; for (const agentName of summary.completedAgents) { if (!agentMetrics[agentName]) { @@ -1883,25 +1916,20 @@ export async function logWorkflowComplete(input: ActivityInput, summary: Workflo } } - // 4. Build cumulative totals: session.json carries cross-run agent spend only, so this - // run's operational (Capella/reconciliation) entries are added on top instead of being - // silently replaced by agent-only session totals. - const operationalEntries = Object.entries(agentMetrics).filter( - ([name]) => sessionData.metrics.agents[name] === undefined, - ); - const operationalCostUsd = operationalEntries.reduce((sum, [, metrics]) => sum + (metrics.costUsd ?? 0), 0); - const operationalDurationMs = operationalEntries.reduce((sum, [, metrics]) => sum + metrics.durationMs, 0); + // 5. Log the cumulative workspace totals. Duration is elapsed workflow time, while the + // separate total_agent_duration_ms in session.json remains the sum of agent work. const cumulativeSummary: WorkflowSummary = { ...summary, - totalDurationMs: sessionData.metrics.total_duration_ms + operationalDurationMs, - totalCostUsd: sessionData.metrics.total_cost_usd + operationalCostUsd, + totalDurationMs: totals.totalDurationMs, + totalCostUsd: totals.totalCostUsd, + usageAccountingComplete: totals.usageAccountingComplete, agentMetrics, }; - // 5. Write completion entry to workflow.log + // 6. Write completion entry to workflow.log await auditSession.logWorkflowComplete(cumulativeSummary); - // 6. Drop the authenticated browser session. auth-state.json holds live cookies/storage for + // 7. Drop the authenticated browser session. auth-state.json holds live cookies/storage for // the lifetime of the scan only; leaving it on disk past workflow end would let a session // outlive the run that created it. The removal is best-effort: a failure here is logged and // swallowed rather than failing a scan that otherwise completed successfully. @@ -1912,7 +1940,7 @@ export async function logWorkflowComplete(input: ActivityInput, summary: Workflo console.warn(`Failed to clean up auth-state.json: ${detail}`); } - // 7. Clean up container + // 8. Clean up container removeContainer(workflowId); } diff --git a/apps/worker/src/temporal/shared.ts b/apps/worker/src/temporal/shared.ts index fdef21a6..a9e1c91d 100644 --- a/apps/worker/src/temporal/shared.ts +++ b/apps/worker/src/temporal/shared.ts @@ -2,7 +2,13 @@ import { defineQuery } from '@temporalio/workflow'; export type { AgentMetrics } from '../types/metrics.js'; -import type { CapellaFailurePoint, CapellaStage, SarifRef } from '../ai/sast/types.js'; +import type { + AgenticSastReduction, + CapellaFailurePoint, + CapellaRecoveredFailure, + CapellaStage, + SarifRef, +} from '../ai/sast/types.js'; import type { VulnClass } from '../types/config.js'; import type { ErrorCode } from '../types/errors.js'; import type { AgentMetrics } from '../types/metrics.js'; @@ -45,6 +51,8 @@ export type AgenticSastState = readonly coverage: 'complete' | 'reduced'; readonly warnings: readonly string[]; readonly durationMs: number; + readonly reductions?: readonly AgenticSastReduction[]; + readonly recoveredFailure?: CapellaRecoveredFailure; } | { readonly status: 'failed'; @@ -55,6 +63,7 @@ export type AgenticSastState = /** Bounded machine code preserved from the failing Capella activity, when one crossed the child. */ readonly errorCode?: string; readonly completedStages: readonly CapellaStage[]; + readonly warnings: readonly string[]; readonly durationMs: number; }; diff --git a/apps/worker/src/temporal/summary-mapper.ts b/apps/worker/src/temporal/summary-mapper.ts index 5c1c89ec..ae428585 100644 --- a/apps/worker/src/temporal/summary-mapper.ts +++ b/apps/worker/src/temporal/summary-mapper.ts @@ -35,20 +35,50 @@ export function toWorkflowSummary( const agenticSastErrorCode = agenticSastFailure?.errorCode; const agenticSastFailureMessage = agenticSastFailure?.error; const agenticSastFailedStage = agenticSastFailure?.failedStageLabel; + // Both terminal Capella variants carry a warnings array; a disabled or still-running run has none. + const agenticSast = state.agenticSast; + const usageAccountingWarnings = + agenticSast.status === 'succeeded' || agenticSast.status === 'failed' ? agenticSast.warnings : []; + // Carry the terminal disposition so a successful (or reduced-coverage) run is visible in the summary, + // not only a failed one. Coverage is meaningful only on success. + const agenticSastCoverage = agenticSast.status === 'succeeded' ? agenticSast.coverage : undefined; + const endedAtMs = state.startTime + summary.totalDurationMs; return { status, + startedAtMs: state.startTime, + endedAtMs, totalDurationMs: summary.totalDurationMs, totalCostUsd: summary.totalCostUsd, completedAgents: state.completedAgents, skippedAgents: state.skippedAgents, agentMetrics: Object.fromEntries( - [...Object.entries(state.agentMetrics), ...Object.entries(state.operationalMetrics)].map(([name, metrics]) => [ + Object.entries(state.agentMetrics).map(([name, metrics]) => [ name, { durationMs: metrics.durationMs, costUsd: metrics.costUsd }, ]), ), + operationalMetrics: Object.fromEntries( + Object.entries(state.operationalMetrics).map(([name, metrics]) => [ + name, + { ...metrics, usageComplete: metrics.usageComplete !== false }, + ]), + ), + // The stage wall-clocks the summary reads to report each group's real elapsed time; the priced + // metrics carry cost but no faithful duration (reconciliation stages record 0). + operationalStages: Object.fromEntries( + Object.entries(state.operationalStages).map(([key, stage]) => [ + key, + { + ...(stage.startedAt !== undefined && { startedAt: stage.startedAt }), + ...(stage.durationMs !== undefined && { durationMs: stage.durationMs }), + }, + ]), + ), partialReasons: state.partialReasons, usageAccountingComplete: summary.usageAccountingComplete, + usageAccountingWarnings: [...usageAccountingWarnings], + agenticSastStatus: agenticSast.status, + ...(agenticSastCoverage !== undefined && { agenticSastCoverage }), ...(agenticSastFailedStage !== undefined && { agenticSastFailedStage }), ...(agenticSastFailureMessage !== undefined && { agenticSastFailureMessage }), ...(agenticSastErrorCode !== undefined && { agenticSastErrorCode }), diff --git a/apps/worker/src/temporal/worker.ts b/apps/worker/src/temporal/worker.ts index 77d37880..d3f5a574 100644 --- a/apps/worker/src/temporal/worker.ts +++ b/apps/worker/src/temporal/worker.ts @@ -36,9 +36,11 @@ import { DEFAULT_MODEL_SPEC } from '../ai/models.js'; import { capellaTerminalStageLabel, isCapellaSafeFailureMessage } from '../ai/sast/capella/safe-failures.js'; import { capellaActivities, mergeActivityRegistries } from '../ai/sast/capella/temporal/registry.js'; import { CAPELLA_FORMAT_VERSION, CAPELLA_PROMPT_SET_VERSION } from '../ai/sast/capella/types.js'; +import { summarizeOperationalMetrics } from '../audit/operational-summary.js'; import { sanitizeHostname } from '../audit/utils.js'; import { distributeConfig, parseConfig } from '../config-parser.js'; import { deliverablesDir, resolveSessionJsonPath } from '../paths.js'; +import { isProviderFailureCategory } from '../types/errors.js'; import { ACCEPTED_CAPELLA_FAILURE_STAGES, isPartialReason, @@ -95,11 +97,15 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url)); const PROGRESS_QUERY = 'getProgress'; +/** Accept only a code shaped like a stable error code, or a known provider-failure category; anything else is treated as absent rather than printed. */ function safeFailureCode(value: string | undefined): string | undefined { - if (value !== undefined && /^[A-Z][A-Z0-9_]{0,63}$/u.test(value)) return value; + if (value !== undefined && (/^[A-Z][A-Z0-9_]{0,63}$/u.test(value) || isProviderFailureCategory(value))) { + return value; + } return undefined; } +/** Re-derive a printable line from the closed partial-reason projection instead of trusting the workflow-returned view directly, so a malformed reason prints nothing rather than something wrong. */ function safePartialReasonMessage(reason: PipelineState['partialReasons'][number]): string | undefined { if (reason.code === 'agentic_sast_reduced') return 'Agentic SAST completed with reduced coverage.'; const candidate = { @@ -577,6 +583,19 @@ async function waitForWorkflowResult( if (result.summary) { console.log(`Duration: ${Math.floor(result.summary.totalDurationMs / 1000)}s`); console.log(`Agents resolved: ${result.summary.agentCount}`); + // Agentic SAST is not an agent, so it is absent from the count above; name it so its spend in + // Run cost is accounted for. The failure detail, if any, already printed above. + if (result.agenticSast.status === 'succeeded') { + const sastGroup = summarizeOperationalMetrics(result.operationalMetrics).find( + (group) => group.key === 'agentic-sast', + ); + const cost = sastGroup === undefined || sastGroup.costUsd === null ? 'N/A' : `$${sastGroup.costUsd.toFixed(4)}`; + const duration = sastGroup === undefined ? '0s' : `${Math.floor(sastGroup.durationMs / 1000)}s`; + const coverage = result.agenticSast.coverage === 'reduced' ? ' — reduced coverage' : ''; + console.log(`Agentic SAST: completed (${duration}, ${cost})${coverage}`); + } else if (result.agenticSast.status === 'failed') { + console.log('Agentic SAST: failed'); + } console.log(`Total turns: ${result.summary.totalTurns}`); console.log(`Run cost: $${result.summary.totalCostUsd.toFixed(4)}`); if (result.summary.usageAccountingComplete === false) { diff --git a/apps/worker/src/temporal/workflows.ts b/apps/worker/src/temporal/workflows.ts index fc9dad13..c1371de6 100644 --- a/apps/worker/src/temporal/workflows.ts +++ b/apps/worker/src/temporal/workflows.ts @@ -42,6 +42,7 @@ import { type MiscellaneousOutcome, miscellaneousLaneIsSettled, type PartialReason, + partialReasonFromReduction, projectPartialReasons, renderSafeMessage, reportIsAuthored, @@ -313,6 +314,11 @@ function capellaMetrics(result: CapellaRunResult, model: string): OperationalMet }; } +/** + * A reconciliation stage's priced metrics. `StageMetrics` records spend only, so the duration + * stays zero rather than being invented here; the stage's real wall-clock is carried separately + * by `operationalStages` and is what terminal accounting reads. + */ function stageMetrics(metrics: StageMetrics): OperationalMetrics { return { durationMs: 0, @@ -426,6 +432,7 @@ export async function pentestPipeline(input: PipelineInput): Promise | null = null; + // What durable state records for the internal `miscellaneous` class, kept current by every + // durable summary this run reads or writes. The lane consults it before deciding admission, + // rather than re-deciding from a queue an earlier run already settled. + let miscellaneousOutcome: MiscellaneousOutcome | undefined; // Latched when a stopped run has recorded the terminal Capella state. The hard-failure path // does not wait for the child, so the child can still return while the terminal activity // yields; from that point the recorded state is final and no continuation may rewrite it. @@ -822,6 +832,7 @@ export async function pentestPipeline(input: PipelineInput): Promise { const key = 'miscellaneous-pipeline'; const label = 'Miscellaneous findings'; + // An earlier run already settled this class. Re-deciding admission would ask durable state to + // move backwards, which fails closed and would be recorded as a class failure that never + // happened; re-running the lane would also repeat work that run already paid for. if (miscellaneousLaneIsSettled(miscellaneousOutcome)) { if (miscellaneousOutcome === 'completed') markCompleted('miscellaneous-exploit'); skipOperation(key, label); @@ -1151,13 +1176,26 @@ export async function pentestPipeline(input: PipelineInput): Promise finalReportActs.finalizeReportOutputs(activityInput, true), ); } - if (finalized.sarifDisposition === 'render_failed') { - addPartialReason({ code: 'report_sarif_failed' }); - } state.reportProgress = await runOperation('report:terminal', 'Saving final report state', () => deterministicReportActs.persistFinalizedReportProgress( activityInput, diff --git a/apps/worker/templates/typst/report.typ b/apps/worker/templates/typst/report.typ index fa8e1575..d5e8a27f 100644 --- a/apps/worker/templates/typst/report.typ +++ b/apps/worker/templates/typst/report.typ @@ -12,6 +12,8 @@ // exploits → ExploitsReportData (exploit=true runs, full reproduction) // findings → FindingsReportData (exploit=false runs, analysis-only) #let mode = data.at("mode", default: "exploits") +#let coverage = data.at("coverage", default: (status: "complete", limitations: ())) +#let assessment-status = if coverage.status == "partial" { "Completed with limitations" } else { "Complete" } #let tester-override = sys.inputs.at("tester", default: "Shannon") #let brand = sys.inputs.at("brand", default: "Shannon | AI Pentester by Keygraph") @@ -133,7 +135,7 @@ "XSS", "Injection", "SSRF", - "Other", + "Miscellaneous", ) #let sev-chip(level) = chip(level, sev-color(level)) @@ -159,6 +161,15 @@ out } +// Preserve blank-line paragraph boundaries in model-authored report prose while +// retaining the inline-code treatment used by the rest of the template. +#let render-paragraphs(s) = { + if type(s) != str { return s } + for paragraph in s.split(regex("\\r?\\n\\s*\\r?\\n")) { + par(inline-code(paragraph)) + } +} + #let render-items(items) = { for item in items { if item.kind == "prose" [ @@ -291,12 +302,21 @@ (text(fill: muted, size: 10pt)[Application], text(size: 10.5pt)[#inline-code(data.meta.application)]) } else { () }), text(fill: muted, size: 10pt)[Tester], text(size: 10.5pt)[#tester-override], + text(fill: muted, size: 10pt)[Assessment Status], text(size: 10.5pt)[#assessment-status], ) +#render-paragraphs(data.executiveSummary) + == Scope #inline-code(data.scope) +#if coverage.status == "partial" and coverage.limitations.len() > 0 [ + == Limitations + + #list(..coverage.limitations.map(limitation => [#inline-code(limitation.message)])) +] + // ---------- BY TYPE --------------------------------------------------------- #let by-type-entries = if mode == "exploits" { data.exploitedByType } else { data.identifiedByType } #if mode == "exploits" [