From ed5659e2e2c622dbe3119d57094a546777826f20 Mon Sep 17 00:00:00 2001 From: ezl-keygraph Date: Wed, 26 Aug 2026 19:25:16 +0530 Subject: [PATCH] fix(report): emit SARIF by default for exploit runs (#431) * fix(report): emit SARIF by default for exploit runs, opt out with report.sarif: false * docs: describe SARIF as on-by-default for exploit runs --- CLAUDE.md | 2 +- README.md | 4 ++-- apps/worker/configs/config-schema.json | 2 +- apps/worker/configs/example-config.yaml | 5 +++-- apps/worker/src/config-parser.ts | 3 ++- apps/worker/src/paths.ts | 2 +- apps/worker/src/services/agent-git-paths.ts | 2 +- apps/worker/src/services/reporting.ts | 2 +- apps/worker/src/temporal/activities.ts | 16 +++++++++------- apps/worker/src/types/config.ts | 5 ++++- docs/configuration.md | 9 ++++----- llms-full.txt | 13 ++++++------- 12 files changed, 35 insertions(+), 30 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index f20d6f05..ce627364 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -152,7 +152,7 @@ Durable workflow orchestration with crash recovery, queryable progress, intellig 5. **Reporting** (`report`) — Executive-level security report ### Supporting Systems -- **Configuration** — YAML configs in `apps/worker/configs/` with JSON Schema validation (`config-schema.json`). Supports auth settings (MFA/TOTP), URL/code rule scoping (`rules.avoid`/`rules.focus`), run-scope steering (`vuln_classes`, `exploit`), free-form `rules_of_engagement`, and post-hoc `report` options (`min_severity`, `min_confidence`, `guidance`, and `sarif` to emit a SARIF 2.1.0 log via `apps/worker/src/services/sarif-renderer.ts`; exploit-only). `code_path` avoid rules are enforced via the `@gotgenes/pi-permission-system` extension: `apps/worker/src/temporal/activities.ts:syncCodePathDenyRules` writes a global `path` deny config once per workflow (`apps/worker/src/ai/pi/permission-system.ts:syncPermissionSystemConfig`), and the executor loads the extension when that config is present (`apps/worker/src/ai/pi/pi-executor.ts`), so denies fire across every tool and child `task` session. `vuln_classes`/`exploit` scope is locked into `session.json` on first run; resumes with a different scope fail fast (`persistOrValidateRunScope`). Credential resolution — local mode: env vars → `./.env`; npx mode: env vars → `~/.shannon/config.toml` (via `npx @keygraph/shannon setup`) +- **Configuration** — YAML configs in `apps/worker/configs/` with JSON Schema validation (`config-schema.json`). Supports auth settings (MFA/TOTP), URL/code rule scoping (`rules.avoid`/`rules.focus`), run-scope steering (`vuln_classes`, `exploit`), free-form `rules_of_engagement`, and post-hoc `report` options (`min_severity`, `min_confidence`, `guidance`, and `sarif` for a SARIF 2.1.0 log via `apps/worker/src/services/sarif-renderer.ts`, on by default for exploit runs and opt out with `report.sarif: false`). `code_path` avoid rules are enforced via the `@gotgenes/pi-permission-system` extension: `apps/worker/src/temporal/activities.ts:syncCodePathDenyRules` writes a global `path` deny config once per workflow (`apps/worker/src/ai/pi/permission-system.ts:syncPermissionSystemConfig`), and the executor loads the extension when that config is present (`apps/worker/src/ai/pi/pi-executor.ts`), so denies fire across every tool and child `task` session. `vuln_classes`/`exploit` scope is locked into `session.json` on first run; resumes with a different scope fail fast (`persistOrValidateRunScope`). Credential resolution — local mode: env vars → `./.env`; npx mode: env vars → `~/.shannon/config.toml` (via `npx @keygraph/shannon setup`) - **Prompts** — Per-phase templates in `apps/worker/prompts/` with variable substitution (`{{TARGET_URL}}`, `{{CONFIG_CONTEXT}}`). Shared partials in `apps/worker/prompts/shared/` via `apps/worker/src/services/prompt-manager.ts`, including `_code-path-rules.txt` (focus/avoid `[FILE]`/`[GLOB]` routing) and `_rules-of-engagement.txt` (free-text engagement rules). When `exploit: false`, `apps/worker/src/services/findings-renderer.ts` deterministically converts each `*_exploitation_queue.json` into a `*_findings.md` for report assembly — no LLM in the loop - **Agent Harness (pi)** — Uses the **pi harness** (`@earendil-works/pi-coding-agent`, requires Node ≥ 22.19) via `apps/worker/src/ai/pi/pi-executor.ts` (`runPiPrompt` → `createAgentSession`). Retry is split in `apps/worker/src/ai/pi/retry-settings.ts`: pi's agent-level loop is off so Temporal owns agent restarts, while `provider.maxRetries` stays on — pi reads the `provider` block independently of the `enabled` flag — so transport faults are absorbed in-session rather than costing a full agent re-run. `maxRetryDelayMs` is left at pi's 60s default. One model runs every phase, named by `SHANNON_AI_MODEL=:` (default `anthropic:claude-sonnet-4-6`). `apps/worker/src/ai/models.ts` parses the spec — splitting on the **first** colon only, so Bedrock IDs keep theirs — and resolves it through pi's `ModelRuntime`. pi ships the `CredentialStore` interface but no in-memory implementation (its own reads `auth.json` from disk), so `RuntimeCredentialStore` in that file supplies one: credentials arrive as env vars in an ephemeral container and must never touch disk. `createModelRuntime(providerId, apiKey)` builds the runtime; `allowModelNetwork` stays at its default `false` so a scan never blocks on a catalog refresh. `resolveModelSelection()` is **async** because `ModelRuntime.create()` is. Any pi-ai provider id is accepted — `parseModelSpec` no longer rejects against a hardcoded list, so pi's registry is the authority (an unknown provider/model surfaces as a clear "not found in pi registry" error at preflight, which points to the browsable catalogue at `pi.dev/models` — `PI_CATALOG_URL` in `apps/worker/src/ai/models.ts`, appended to the not-found errors and shown in the setup wizard's "Other provider" hint). Four providers are **curated** (`CURATED_PROVIDERS`: `anthropic`, `openai`, `xai`, `amazon-bedrock`) with their own credential variables, config sections, and setup flows; each provider's API key env var is declared once in `PROVIDER_API_KEY_ENV` — Shannon uses each vendor's own variable name (`OPENAI_API_KEY`, `XAI_API_KEY`, …), never an invented one; Bedrock's entry is `AWS_BEARER_TOKEN_BEDROCK`, paired with `AWS_REGION`, which preflight requires separately as provider config rather than a credential. Any other provider uses the **generic** credential path: `SHANNON_AI_API_KEY` (`GENERIC_API_KEY_ENV`) supplies the key for any provider whose credential is a plain API key. Curated providers' own variables take precedence over it, and it also works as a fallback for them — Bedrock is the sole exception (it authenticates through its AWS_ variables, so the generic key never stands in for it). The CLI forwards `SHANNON_AI_API_KEY` in `COMMON_FORWARD_VARS` (it is provider-neutral, binding to whatever `SHANNON_AI_MODEL` names, so the "only one provider configured" guard counts only named credentials), and stores it under a generic `[provider]` config.toml section (`provider.api_key`). `npx @keygraph/shannon setup` exposes this as the "Other provider" option: free-text provider id + model id + key (a curated provider id is rejected there, since it has its own option). `SHANNON_AI_BASE_URL` overrides the endpoint for any provider (proxies/gateways); the credential is unchanged. `pointAtGateway` (`apps/worker/src/ai/models.ts`) applies the one dialect change: behind a base URL, `openai` follows `SHANNON_AI_OPENAI_FORMAT` (`chat-completions` default, or `responses`). On `chat-completions` it switches the API to `openai-completions` and drops the catalogue's Responses-shaped `compat` block so pi's `detectCompat` derives completions settings; on `responses` the descriptor is unchanged but for the endpoint. `resolveGatewayFormat` rejects the variable when the provider is not `openai` or no base URL is set, since it cannot take effect there. All other providers keep their API. The CLI mirrors the accepted values in `apps/cli/src/model-spec.ts`, forwards the variable in `COMMON_FORWARD_VARS`, and maps it to `openai.format` in config.toml. `buildEnvFlags` forwards only the selected provider's credential into the worker container. The CLI mirrors the parse rule and the provider/credential tables in `apps/cli/src/model-spec.ts` (it cannot import from the worker package); the two must stay in sync. pi ships no JSON-schema output or `Task`/`TodoWrite` built-ins, so structured queues are captured via a `submit_exploitation_queue` custom tool (`apps/worker/src/ai/queue-schemas.ts`), and `task` (child sessions scoped to `read`, `grep`, `find`, `ls`, `write`, and `bash` — no nested `task` or collector tools; `CHILD_TOOLS` in `apps/worker/src/ai/pi/task-tool.ts`) + `todo_write` (`apps/worker/src/ai/pi/session-tools.ts`) are provided as custom tools; the per-phase collectors are pi custom tools (TypeBox `defineTool` in `apps/worker/src/collectors/`). Shannon sets no thinking configuration at all — no `thinkingLevel` is passed to any `createAgentSession` call, so pi's own default applies. There is no adaptive-thinking support and no `CLAUDE_ADAPTIVE_THINKING` / `core.adaptive_thinking` setting. Browser automation via `playwright-cli` with session isolation (`-s=`). TOTP generation via `generate-totp` CLI tool. Login flow template at `apps/worker/prompts/shared/login-instructions.txt` supports form, SSO, API, and basic auth. On authenticated whitebox scans, the `validate-authentication` preflight performs the single real login and saves the browser session to `auth-state.json` in the per-session audit directory (path from `authStateFile()` in `apps/worker/src/audit/utils.ts`, derived from `generateAuditPath()`). The validation activity (`apps/worker/src/services/validate-authentication.ts`) removes any stale file from a prior run before the agent runs and verifies the file parses and contains cookies or storage before the preflight is marked complete; `logWorkflowComplete` deletes it when the workflow ends so authenticated cookies don't sit on disk between scans. Agent prompts opt in to session reuse by `@include(shared/_shared-session.txt)` before their `` block — the partial restores the session and falls through to the full login flow if verification fails. `vuln-auth`/`exploit-auth` omit the include and own their own login - **Pi Credential Reuse** — `SHANNON_USE_PI_AUTH=1` opts into reusing the host's Pi login, including an `openai-codex` ChatGPT Plus/Pro subscription selected with `SHANNON_AI_MODEL=openai-codex:`. `apps/cli/src/env.ts` requires `~/.pi/agent/auth.json`; `start.ts` passes its path to `spawnWorker`, which mounts only that file read-write at `/tmp/.pi/agent/auth.json`. The flag itself is not forwarded: the worker detects the file with `piAuthPresent()` and passes its path to `ModelRuntime.create`. CLI and worker API-key presence checks are skipped on this path, but the normal preflight model probe still validates the credential. The image and UID-remapping entrypoint keep `/tmp/.pi/agent` owned by `pentest` so adjacent Pi/Shannon configuration remains writable. Refreshed OAuth state is persisted to the host for subsequent scans. diff --git a/README.md b/README.md index bd598c10..aae9eeef 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ For source builds, authenticated scans, provider-specific setup, and platform no - **Authenticated testing**: configuration files can describe login flows, test credentials, TOTP, email-based login flows, focus areas, and rules of engagement. - **OWASP-focused coverage**: Shannon targets exploitable Injection, XSS, SSRF, Broken Authentication, and Broken Authorization issues. - **Resumable workspaces**: Shannon can resume interrupted runs without re-running completed agents. -- **Machine-readable output**: Shannon emits findings as structured JSON, and as SARIF 2.1.0 when you enable it in configuration. SARIF is the OASIS standard for static analysis results, so findings flow into any code scanning service, vulnerability management platform, security dashboard, or CI/CD pipeline that reads it. +- **Machine-readable output**: Shannon emits findings as structured JSON, and as SARIF 2.1.0 by default on exploit-mode scans (opt out with `report.sarif: "false"`). SARIF is the OASIS standard for static analysis results, so findings flow into any code scanning service, vulnerability management platform, security dashboard, or CI/CD pipeline that reads it. - **Bring your own key, provider-agnostic**: Shannon runs on Anthropic, OpenAI, xAI, AWS Bedrock, and any endpoint speaking the Anthropic Messages API or the OpenAI Chat Completions or Responses API, including self-hosted models served through Ollama, vLLM, or LM Studio and gateways such as OpenRouter and LiteLLM. You supply the credentials, so source code and model traffic stay inside your infrastructure. Local and self-hosted models are technically supported but not recommended: they may not follow Shannon's instructions or tool-use constraints as reliably as frontier models, so take that path only if you know how your chosen model behaves. ## Editions @@ -267,7 +267,7 @@ Yes, always. You provide the LLM credentials Shannon uses to run a pentest, in e ### Does Shannon output SARIF? -Yes. Shannon emits SARIF 2.1.0, the OASIS standard format for static analysis results, alongside structured JSON. Any SARIF consumer reads it: code scanning services, vulnerability management platforms, security dashboards, and CI/CD pipelines. Set `report.sarif` to `"true"` in your configuration file to enable the SARIF log. +Yes. Shannon emits SARIF 2.1.0, the OASIS standard format for static analysis results, alongside structured JSON. Any SARIF consumer reads it: code scanning services, vulnerability management platforms, security dashboards, and CI/CD pipelines. It is written by default on exploit-mode scans; set `report.sarif` to `"false"` in your configuration file to opt out. ### Which AI providers does Shannon support? diff --git a/apps/worker/configs/config-schema.json b/apps/worker/configs/config-schema.json index bdd8cf8f..78e16dd2 100644 --- a/apps/worker/configs/config-schema.json +++ b/apps/worker/configs/config-schema.json @@ -164,7 +164,7 @@ "sarif": { "type": "string", "enum": ["true", "false"], - "description": "Emit a SARIF 2.1.0 log (report.sarif) beside the report. Requires exploit=true; ignored otherwise." + "description": "Emit a SARIF 2.1.0 log (report.sarif) beside the report. On by default for exploit runs; set \"false\" to opt out. Ignored when exploit=false." } }, "additionalProperties": false diff --git a/apps/worker/configs/example-config.yaml b/apps/worker/configs/example-config.yaml index 386d7200..ca4698d4 100644 --- a/apps/worker/configs/example-config.yaml +++ b/apps/worker/configs/example-config.yaml @@ -96,8 +96,9 @@ rules: # Report filters applied by the report agent when assembling the final report (optional). # Example below is illustrative; edit, remove, or add sections as needed. # report: -# # Emit a SARIF 2.1.0 log (report.sarif) beside the report. Requires exploit: "true". -# sarif: "true" +# # SARIF 2.1.0 log (report.sarif) beside the report. On by default for exploit runs; +# # set "false" to opt out. Ignored when exploit is "false". +# sarif: "false" # min_severity: low # min_confidence: low # guidance: | diff --git a/apps/worker/src/config-parser.ts b/apps/worker/src/config-parser.ts index defb3edf..fc3c62c5 100644 --- a/apps/worker/src/config-parser.ts +++ b/apps/worker/src/config-parser.ts @@ -679,7 +679,8 @@ export const distributeConfig = (config: Config | null): DistributedConfig => { const exploit = config?.exploit !== undefined ? config.exploit === 'true' : true; const report = { - sarif: config?.report?.sarif === 'true', + // Default on; only an explicit "false" opts out. + sarif: config?.report?.sarif !== 'false', ...(config?.report?.min_severity && { min_severity: config.report.min_severity }), ...(config?.report?.min_confidence && { min_confidence: config.report.min_confidence }), ...(config?.report?.guidance && { guidance: config.report.guidance.trim() }), diff --git a/apps/worker/src/paths.ts b/apps/worker/src/paths.ts index 595c76db..8b52b99a 100644 --- a/apps/worker/src/paths.ts +++ b/apps/worker/src/paths.ts @@ -43,7 +43,7 @@ export const FINAL_REPORT_MD_FILENAME = 'Security-Assessment-Report.md'; /** Structured findings the report agent emits; the markdown report is rendered from it. */ export const REPORT_JSON_FILENAME = 'report.json'; -/** SARIF 2.1.0 log, written only for exploit=true runs when report.sarif is enabled. */ +/** SARIF 2.1.0 log, written for exploit=true runs unless report.sarif is set to false. */ export const SARIF_FILENAME = 'report.sarif'; /** diff --git a/apps/worker/src/services/agent-git-paths.ts b/apps/worker/src/services/agent-git-paths.ts index 6a24900c..f190df55 100644 --- a/apps/worker/src/services/agent-git-paths.ts +++ b/apps/worker/src/services/agent-git-paths.ts @@ -29,7 +29,7 @@ export function getAgentGitPaths(agentName: AgentName): string[] { paths.push(queueFilename); } // The report agent also emits the structured findings the markdown is rendered from, and the - // SARIF log when enabled. Listing the log unconditionally is harmless when it was not written, + // SARIF log when produced. Listing the log unconditionally is harmless when it was not written, // and keeps a stale one from surviving the rollback of a failed attempt. if (agentName === 'report') { paths.push(REPORT_JSON_FILENAME); diff --git a/apps/worker/src/services/reporting.ts b/apps/worker/src/services/reporting.ts index fb1cfa3f..6b61e4d9 100644 --- a/apps/worker/src/services/reporting.ts +++ b/apps/worker/src/services/reporting.ts @@ -181,7 +181,7 @@ export async function injectModelIntoReport( * * The SARIF log is surfaced beside it when present, since a CI step consuming it needs a stable * path and cannot be expected to reach into the internals directory. It is absent whenever the - * run was analysis-only or `report.sarif` was not enabled. + * run was analysis-only or `report.sarif` was set to false. */ export async function copyReportToRunRoot( repoPath: string, diff --git a/apps/worker/src/temporal/activities.ts b/apps/worker/src/temporal/activities.ts index 837fbdcb..2b010b0c 100644 --- a/apps/worker/src/temporal/activities.ts +++ b/apps/worker/src/temporal/activities.ts @@ -450,13 +450,14 @@ export async function runAuthzExploitAgent(input: ActivityInput): Promise