feat(worker): deduplicate static and runtime findings before exploitation

Parse Agentic SAST SARIF into typed observations, enrich and route those observations, and reconcile them with pentest findings before exploitation.

Publish deterministic exploitation queues with stable lineage, exact-path Git commits, retry-safe manifests, named drop reasons, and confined task formation. Reject duplicate producer IDs before commit and adopt either legal provenance shape after a lost acknowledgement.
This commit is contained in:
ajmallesh
2026-08-26 19:37:20 -07:00
parent 980607c602
commit c33132b0ab
52 changed files with 7914 additions and 84 deletions
+5 -4
View File
@@ -8,7 +8,7 @@
* Deterministic exploit collector → markdown renderer.
*
* Single entry point renderExploitDeliverable(vulnClass, state, idToType)
* covers all 5 exploitation agents (injection, xss, auth, ssrf, authz). The
* covers all exploitation agents. The
* per-class deltas are limited to title and ID prefix; every section, label,
* and sort rule is class-agnostic. Section headers and bolded field labels
* mirror the prescribed-Markdown skeleton from the existing exploit-*.txt
@@ -30,18 +30,19 @@
* as `- {ID} ({vulnerability_type})`. Omitted when every queue ID was emitted.
*/
import type { AddExploitInput, VulnClass } from '../collectors/exploit-collector.js';
import type { AddExploitInput, ExploitClass } from '../collectors/exploit-collector.js';
// ============================================================================
// PER-CLASS CONSTANTS
// ============================================================================
const TITLES: Record<VulnClass, string> = {
const TITLES: Record<ExploitClass, string> = {
injection: 'Injection Exploitation Evidence',
xss: 'Cross-Site Scripting (XSS) Exploitation Evidence',
auth: 'Authentication Exploitation Evidence',
ssrf: 'SSRF Exploitation Evidence',
authz: 'Authorization Exploitation Evidence',
miscellaneous: 'Miscellaneous Exploitation Evidence',
};
// ============================================================================
@@ -203,7 +204,7 @@ function renderUnprocessedSection(missingIds: readonly string[], idToType: Reado
// ============================================================================
export function renderExploitDeliverable(
vulnClass: VulnClass,
vulnClass: ExploitClass,
state: readonly AddExploitInput[],
idToType: ReadonlyMap<string, string>,
): string {
+292 -7
View File
@@ -5,6 +5,7 @@
// as published by the Free Software Foundation.
import { AsyncLocalStorage } from 'node:async_hooks';
import { createHash } from 'node:crypto';
import { $ } from 'zx';
import type { ActivityLogger } from '../types/activity-logger.js';
import { ErrorCode } from '../types/errors.js';
@@ -233,9 +234,50 @@ export async function executeGitCommandWithRetry(
);
}
// Two-phase reset: hard reset (tracked files) + clean (untracked files).
// When paths is provided, the untracked clean is scoped to those paths so a
// failing agent's rollback can't delete a concurrent sibling agent's scratch.
// Filter paths to those present in the HEAD tree, so a subsequent
// `git restore --source=HEAD` won't abort on a pathspec the commit doesn't
// contain. Sourced from HEAD (not the index) to match what restore reads.
async function listPathsInHead(sourceDir: string, paths: readonly string[]): Promise<string[]> {
const result = await executeGitCommandWithRetry(
['git', 'ls-tree', '-r', '-z', '--name-only', 'HEAD', '--', ...paths],
sourceDir,
'list HEAD-tracked rollback paths',
);
return result.stdout.split('\0').filter((path) => path.length > 0);
}
async function restoreScopedPathsFromHead(
sourceDir: string,
paths: readonly string[],
description: string,
): Promise<void> {
const pathsInHead = await listPathsInHead(sourceDir, paths);
// Resetting the scoped index first also makes a newly staged path untracked,
// allowing the clean step to remove it when the path is absent from HEAD.
await executeGitCommandWithRetry(
['git', 'reset', 'HEAD', '--', ...paths],
sourceDir,
`resetting owned index paths for ${description}`,
);
if (pathsInHead.length > 0) {
await executeGitCommandWithRetry(
['git', 'restore', '--source=HEAD', '--worktree', '--', ...pathsInHead],
sourceDir,
`restoring owned worktree paths for ${description}`,
);
}
await executeGitCommandWithRetry(
['git', 'clean', '-fd', '--', ...paths],
sourceDir,
`cleaning untracked owned paths for ${description}`,
);
}
// Two-phase rollback to the last checkpoint: restore tracked files, then clean
// untracked files. When paths is provided, both phases are scoped to those
// paths so one agent's rollback cannot discard a concurrent sibling's work.
// Without paths, the existing whole-workspace reset remains available.
export async function rollbackGitWorkspace(
sourceDir: string,
reason: string = 'retry preparation',
@@ -250,11 +292,15 @@ export async function rollbackGitWorkspace(
logger.info(`Rolling back workspace for ${reason}`);
try {
const scoped = paths !== undefined && paths.length > 0;
const changes = await withGitRepoLock(async () => {
const pendingChanges = await getChangedFiles(sourceDir, 'status check for rollback');
await executeGitCommandWithRetry(['git', 'reset', '--hard', 'HEAD'], sourceDir, 'hard reset for rollback');
const cleanArgs = paths && paths.length > 0 ? ['git', 'clean', '-fd', '--', ...paths] : ['git', 'clean', '-fd'];
await executeGitCommandWithRetry(cleanArgs, sourceDir, 'cleaning untracked files for rollback');
const pendingChanges = await getChangedFiles(sourceDir, 'status check for rollback', paths);
if (scoped) {
await restoreScopedPathsFromHead(sourceDir, paths, 'rollback');
} else {
await executeGitCommandWithRetry(['git', 'reset', '--hard', 'HEAD'], sourceDir, 'hard reset for rollback');
await executeGitCommandWithRetry(['git', 'clean', '-fd'], sourceDir, 'cleaning untracked files for rollback');
}
return pendingChanges;
});
@@ -383,6 +429,95 @@ export async function commitGitSuccess(
}
}
/**
* Return the repo-relative paths changed by one commit.
*
* The result is NUL-delimited at the Git boundary so unusual path characters
* are not split or unquoted.
*/
export async function pathsChangedInCommit(sourceDir: string, commitHash: string): Promise<string[]> {
const result = await executeGitCommandWithRetry(
['git', 'diff-tree', '--root', '--no-commit-id', '--name-only', '-r', '-z', commitHash],
sourceDir,
'listing commit changed paths',
);
return result.stdout.split('\0').filter((path) => path.length > 0);
}
function samePathSet(first: readonly string[], second: readonly string[]): boolean {
return (
first.length === second.length &&
new Set(first).size === first.length &&
first.every((path) => second.includes(path))
);
}
async function stagedPaths(sourceDir: string, paths: readonly string[]): Promise<string[]> {
const result = await executeGitCommandWithRetry(
['git', 'diff', '--cached', '--name-only', '-z', '--', ...paths],
sourceDir,
'verifying exact staged paths',
);
return result.stdout.split('\0').filter((path) => path.length > 0);
}
/** Raised before commit when the staged delta differs from the caller's exact contract. */
export class ExactPathCommitMismatchError extends Error {
constructor() {
super('The staged Git path set differs from the exact publication contract');
this.name = 'ExactPathCommitMismatchError';
}
}
/**
* Commit only the supplied pathspecs, leaving unrelated staged and dirty paths untouched.
*
* There is no empty-path or empty-commit mode: either would weaken the exact-path
* contract. The caller owns cleanup after any failed write or commit.
*/
export async function commitExactPaths(
sourceDir: string,
paths: readonly string[],
description: string,
logger: ActivityLogger,
expectedChangedPaths?: readonly string[],
): Promise<{ commitHash: string; changedPaths: string[] }> {
if (paths.length === 0) {
throw new Error('commitExactPaths: refusing an empty pathspec set');
}
if (
expectedChangedPaths !== undefined &&
(new Set(expectedChangedPaths).size !== expectedChangedPaths.length ||
expectedChangedPaths.some((expectedPath) => !paths.includes(expectedPath)))
) {
throw new ExactPathCommitMismatchError();
}
return withGitRepoLock(async () => {
await executeGitCommandWithRetry(['git', 'add', '-A', '--', ...paths], sourceDir, 'staging exact paths');
if (expectedChangedPaths !== undefined) {
const prospectivePaths = await stagedPaths(sourceDir, paths);
if (!samePathSet(prospectivePaths, expectedChangedPaths)) {
throw new ExactPathCommitMismatchError();
}
}
await executeGitCommandWithRetry(
['git', 'commit', '-m', description, '--', ...paths],
sourceDir,
'creating path-limited commit',
);
const commitHash = await getGitCommitHash(sourceDir);
if (commitHash === null) {
throw new Error('commitExactPaths: HEAD is unreadable after commit');
}
const changedPaths = await pathsChangedInCommit(sourceDir, commitHash);
logger.info(`Path-limited commit ${commitHash.slice(0, 8)} changed ${changedPaths.length} path(s)`);
return { commitHash, changedPaths };
});
}
/**
* Get current git commit hash.
* Returns null if not a git repository.
@@ -398,3 +533,153 @@ export async function getGitCommitHash(sourceDir: string): Promise<string | null
return null;
}
}
/** Return whether one commit is an ancestor of or equal to another. */
export async function isAncestor(ancestor: string, descendant: string, sourceDir: string): Promise<boolean> {
return withGitRepoLock(async () => {
const result = await $`cd ${sourceDir} && git merge-base --is-ancestor ${ancestor} ${descendant}`.nothrow().quiet();
return result.exitCode === 0;
});
}
/** Read a file from `HEAD`, returning null only when the Git command cannot supply it. */
export async function readFileFromHead(sourceDir: string, relPath: string): Promise<string | null> {
return withGitRepoLock(async () => {
const result = await $`cd ${sourceDir} && git show ${`HEAD:${relPath}`}`.nothrow().quiet();
return result.exitCode === 0 ? result.stdout : null;
});
}
/**
* Classify a failed committed read.
*
* Transient markers are checked before corruption markers because Git can emit
* `bad object` after an earlier permission or I/O error. Unknown failures remain
* transient so callers retry instead of incorrectly treating them as absent.
*/
export function classifyHeadReadFailure(stderr: string): 'absent' | 'corrupt' | 'transient' {
const text = stderr.toLowerCase();
if (/does not exist in|exists on disk, but not in/.test(text)) {
return 'absent';
}
if (
/permission denied|resource temporarily unavailable|operation timed out|input\/output error|too many open files|no space left|unable to open|interrupted system call/.test(
text,
)
) {
return 'transient';
}
if (
/bad object|is corrupt|object file .* is empty|unable to unpack|inflate|did not match|hash mismatch|sha1 mismatch/.test(
text,
)
) {
return 'corrupt';
}
return 'transient';
}
/** The classifiable outcomes of reading one committed file from `HEAD`. */
export type CommittedReadResult =
| { readonly state: 'present'; readonly contents: string }
| { readonly state: 'absent' }
| { readonly state: 'corrupt' };
/** The classifiable outcomes of reading one committed blob identity from `HEAD`. */
export type CommittedBlobResult =
| { readonly state: 'present'; readonly sha: string }
| { readonly state: 'absent' }
| { readonly state: 'corrupt' };
function transientHeadReadError(operation: string): PentestError {
return new PentestError(
'A committed Git object could not be read because of a transient repository error',
'filesystem',
true,
{ operation },
ErrorCode.GIT_CHECKPOINT_FAILED,
);
}
/**
* Read a committed file while preserving absent, corrupt, and transient outcomes.
* Transient reads throw so the activity retry policy remains authoritative.
*/
export async function readCommittedFile(sourceDir: string, relPath: string): Promise<CommittedReadResult> {
return withGitRepoLock(async () => {
const result = await $`cd ${sourceDir} && git show ${`HEAD:${relPath}`}`.nothrow().quiet();
if (result.exitCode === 0) {
return { state: 'present', contents: result.stdout };
}
const failure = classifyHeadReadFailure(result.stderr);
if (failure === 'absent') {
return { state: 'absent' };
}
if (failure === 'corrupt') {
return { state: 'corrupt' };
}
throw transientHeadReadError('read-committed-file');
});
}
/** Read one Git blob identity from `HEAD` without collapsing transient failure into absence. */
export async function blobShaFromHead(sourceDir: string, relPath: string): Promise<CommittedBlobResult> {
return withGitRepoLock(async () => {
const result = await $`cd ${sourceDir} && git rev-parse ${`HEAD:${relPath}`}`.nothrow().quiet();
if (result.exitCode === 0) {
return { state: 'present', sha: result.stdout.trim() };
}
const failure = classifyHeadReadFailure(result.stderr);
if (failure === 'absent') {
return { state: 'absent' };
}
if (failure === 'corrupt') {
return { state: 'corrupt' };
}
throw transientHeadReadError('read-committed-blob-identity');
});
}
/**
* Compute the Git blob id for in-memory bytes without writing them, so a caller can compare
* intended contents against a committed blob id. Uses the repo's own object format (sha1 or
* sha256) so the id matches what this repository would store.
*/
export async function gitBlobShaForContents(sourceDir: string, contents: string): Promise<string> {
return withGitRepoLock(async () => {
const result = await executeGitCommandWithRetry(
['git', 'rev-parse', '--show-object-format'],
sourceDir,
'reading Git object format',
);
const objectFormat = result.stdout.trim();
if (objectFormat !== 'sha1' && objectFormat !== 'sha256') {
throw new Error('Unsupported Git object format');
}
const bytes = Buffer.from(contents, 'utf8');
return createHash(objectFormat).update(`blob ${bytes.length}\0`, 'utf8').update(bytes).digest('hex');
});
}
/** Return the newest reachable commit that changed one exact path. */
export async function lastCommitForPathAtHead(sourceDir: string, relPath: string): Promise<string | null> {
return withGitRepoLock(async () => {
const result = await executeGitCommandWithRetry(
['git', 'log', '-1', '--format=%H', 'HEAD', '--', relPath],
sourceDir,
'reading exact-path publication commit',
);
const commitHash = result.stdout.trim();
return commitHash.length > 0 ? commitHash : null;
});
}
/** Restore only the supplied paths in both the index and working tree from `HEAD`. */
export async function restorePathsFromHead(sourceDir: string, paths: readonly string[]): Promise<void> {
if (paths.length === 0) {
return;
}
await withGitRepoLock(() => restoreScopedPathsFromHead(sourceDir, paths, 'committed-state repair'));
}
+89 -35
View File
@@ -6,8 +6,9 @@
import { fs, path } from 'zx';
import type { ExploitationDecision, VulnType } from '../types/agents.js';
import type { ExploitationDecision } from '../types/agents.js';
import { ErrorCode } from '../types/errors.js';
import type { ReconciliationClass } from '../types/reconciliation.js';
import { err, ok, type Result } from '../types/result.js';
import { asyncPipe } from '../utils/functional.js';
import { PentestError } from './error-handling.js';
@@ -17,14 +18,15 @@ export type { ExploitationDecision, VulnType } from '../types/agents.js';
interface VulnTypeConfigItem {
deliverable: string;
queue: string;
deliverableRequired: boolean;
}
type VulnTypeConfig = Record<VulnType, VulnTypeConfigItem>;
type VulnTypeConfig = Record<ReconciliationClass, VulnTypeConfigItem>;
type ErrorMessageResolver = string | ((existence: FileExistence) => string);
type ErrorMessageResolver = string | ((context: ExistenceContext) => string);
interface ValidationRule {
predicate: (existence: FileExistence) => boolean;
predicate: (context: ExistenceContext) => boolean;
errorMessage: ErrorMessageResolver;
retryable: boolean;
}
@@ -34,8 +36,13 @@ interface FileExistence {
queueExists: boolean;
}
interface ExistenceContext {
existence: FileExistence;
deliverableRequired: boolean;
}
interface PathsBase {
vulnType: VulnType;
vulnType: ReconciliationClass;
deliverable: string;
queue: string;
sourceDir: string;
@@ -64,56 +71,87 @@ interface QueueValidationResult {
error: string | null;
}
function isRetryableQueueFileSystemError(error: unknown): boolean {
if (!(error instanceof Error)) return false;
const code = (error as NodeJS.ErrnoException).code;
return code !== undefined && !['EACCES', 'EINVAL', 'EISDIR', 'ENAMETOOLONG', 'ENOTDIR', 'EPERM'].includes(code);
}
/**
* Result type for safe validation - explicit error handling.
*/
export type SafeValidationResult = Result<ExploitationDecision, PentestError>;
export type ReconciliationExploitationDecision<T extends ReconciliationClass = ReconciliationClass> = Omit<
ExploitationDecision,
'vulnType'
> & {
vulnType: T;
};
export type SafeValidationResult<T extends ReconciliationClass = ReconciliationClass> = Result<
ReconciliationExploitationDecision<T>,
PentestError
>;
// Vulnerability type configuration as immutable data
const VULN_TYPE_CONFIG: VulnTypeConfig = Object.freeze({
injection: Object.freeze({
deliverable: 'injection_analysis_deliverable.md',
queue: 'injection_exploitation_queue.json',
deliverableRequired: true,
}),
xss: Object.freeze({
deliverable: 'xss_analysis_deliverable.md',
queue: 'xss_exploitation_queue.json',
deliverableRequired: true,
}),
auth: Object.freeze({
deliverable: 'auth_analysis_deliverable.md',
queue: 'auth_exploitation_queue.json',
deliverableRequired: true,
}),
ssrf: Object.freeze({
deliverable: 'ssrf_analysis_deliverable.md',
queue: 'ssrf_exploitation_queue.json',
deliverableRequired: true,
}),
authz: Object.freeze({
deliverable: 'authz_analysis_deliverable.md',
queue: 'authz_exploitation_queue.json',
deliverableRequired: true,
}),
miscellaneous: Object.freeze({
deliverable: 'miscellaneous_analysis_deliverable.md',
queue: 'miscellaneous_exploitation_queue.json',
deliverableRequired: false,
}),
}) as VulnTypeConfig;
// Pure function to create validation rule
function createValidationRule(
predicate: (existence: FileExistence) => boolean,
predicate: (context: ExistenceContext) => boolean,
errorMessage: ErrorMessageResolver,
retryable: boolean = true,
): ValidationRule {
return Object.freeze({ predicate, errorMessage, retryable });
}
// Symmetric deliverable rules: queue and deliverable must exist together (prevents partial analysis from triggering exploitation)
// A queue is always required. Analysis-backed classes also require their analysis deliverable;
// the analysis-less `miscellaneous` class deliberately has no such artifact.
const fileExistenceRules: readonly ValidationRule[] = Object.freeze([
createValidationRule(
({ deliverableExists, queueExists }) => deliverableExists && queueExists,
({ existence, deliverableRequired }) =>
existence.queueExists && (!deliverableRequired || existence.deliverableExists),
getExistenceErrorMessage,
),
]);
// Generate appropriate error message based on which files are missing
function getExistenceErrorMessage(existence: FileExistence): string {
function getExistenceErrorMessage({ existence, deliverableRequired }: ExistenceContext): string {
const { deliverableExists, queueExists } = existence;
if (!deliverableRequired) {
return 'Analysis failed: Queue file missing. A queue is required.';
}
if (!deliverableExists && !queueExists) {
return 'Analysis failed: Neither deliverable nor queue file exists. Both are required.';
}
@@ -124,7 +162,7 @@ function getExistenceErrorMessage(existence: FileExistence): string {
}
// Pure function to create file paths
const createPaths = (vulnType: VulnType, sourceDir: string): PathsBase | PathsWithError => {
const createPaths = (vulnType: ReconciliationClass, sourceDir: string): PathsBase | PathsWithError => {
const config = VULN_TYPE_CONFIG[vulnType];
if (!config) {
return {
@@ -144,10 +182,23 @@ const createPaths = (vulnType: VulnType, sourceDir: string): PathsBase | PathsWi
const checkFileExistence = async (paths: PathsBase | PathsWithError): Promise<PathsWithExistence | PathsWithError> => {
if ('error' in paths) return paths;
const [deliverableExists, queueExists] = await Promise.all([
fs.pathExists(paths.deliverable),
fs.pathExists(paths.queue),
]);
let deliverableExists: boolean;
let queueExists: boolean;
try {
[deliverableExists, queueExists] = await Promise.all([
fs.pathExists(paths.deliverable),
fs.pathExists(paths.queue),
]);
} catch (error) {
return {
error: new PentestError(
'Queue validation could not inspect the required files.',
'filesystem',
isRetryableQueueFileSystemError(error),
{ vulnType: paths.vulnType },
),
};
}
return Object.freeze({
...paths,
@@ -162,13 +213,15 @@ const validateExistenceRules = (
if ('error' in pathsWithExistence) return pathsWithExistence;
const { existence, vulnType } = pathsWithExistence;
const { deliverableRequired } = VULN_TYPE_CONFIG[vulnType];
const context: ExistenceContext = { existence, deliverableRequired };
// Find the first rule that fails
const failedRule = fileExistenceRules.find((rule) => !rule.predicate(existence));
const failedRule = fileExistenceRules.find((rule) => !rule.predicate(context));
if (failedRule) {
const message =
typeof failedRule.errorMessage === 'function' ? failedRule.errorMessage(existence) : failedRule.errorMessage;
typeof failedRule.errorMessage === 'function' ? failedRule.errorMessage(context) : failedRule.errorMessage;
return {
error: new PentestError(
@@ -177,8 +230,6 @@ const validateExistenceRules = (
failedRule.retryable,
{
vulnType,
deliverablePath: pathsWithExistence.deliverable,
queuePath: pathsWithExistence.queue,
existence,
},
ErrorCode.DELIVERABLE_NOT_FOUND,
@@ -204,11 +255,11 @@ const validateQueueStructure = (content: string): QueueValidationResult => {
data: isValid ? (parsed as QueueData) : null,
error: null,
});
} catch (parseError) {
} catch {
return Object.freeze({
valid: false,
data: null,
error: parseError instanceof Error ? parseError.message : String(parseError),
error: 'invalid_json',
});
}
};
@@ -234,9 +285,6 @@ const validateQueueContent = async (
true, // retryable
{
vulnType: pathsWithExistence.vulnType,
queuePath: pathsWithExistence.queue,
originalError: queueValidation.error,
queueStructure: queueValidation.data ? Object.keys(queueValidation.data) : [],
},
),
};
@@ -249,13 +297,11 @@ const validateQueueContent = async (
} catch (readError) {
return {
error: new PentestError(
`Failed to read queue file for ${pathsWithExistence.vulnType}: ${readError instanceof Error ? readError.message : String(readError)}`,
`Queue file for ${pathsWithExistence.vulnType} could not be read.`,
'filesystem',
false,
isRetryableQueueFileSystemError(readError),
{
vulnType: pathsWithExistence.vulnType,
queuePath: pathsWithExistence.queue,
originalError: readError instanceof Error ? readError.message : String(readError),
},
),
};
@@ -263,7 +309,9 @@ const validateQueueContent = async (
};
// Final decision: skip if queue says no vulns, proceed if vulns found, error otherwise
const determineExploitationDecision = (validatedData: PathsWithQueue | PathsWithError): ExploitationDecision => {
const determineExploitationDecision = (
validatedData: PathsWithQueue | PathsWithError,
): ReconciliationExploitationDecision => {
if ('error' in validatedData) {
throw validatedData.error;
}
@@ -281,11 +329,11 @@ const determineExploitationDecision = (validatedData: PathsWithQueue | PathsWith
};
// Main functional validation pipeline
export async function validateQueueAndDeliverable(
vulnType: VulnType,
export async function validateQueueAndDeliverable<T extends ReconciliationClass>(
vulnType: T,
sourceDir: string,
): Promise<ExploitationDecision> {
return asyncPipe<ExploitationDecision>(
): Promise<ReconciliationExploitationDecision<T>> {
return asyncPipe<ReconciliationExploitationDecision<T>>(
createPaths(vulnType, sourceDir),
checkFileExistence,
validateExistenceRules,
@@ -298,11 +346,17 @@ export async function validateQueueAndDeliverable(
* Safely validate queue and deliverable files.
* Returns Result<ExploitationDecision, PentestError> for explicit error handling.
*/
export async function validateQueueSafe(vulnType: VulnType, sourceDir: string): Promise<SafeValidationResult> {
export async function validateQueueSafe<T extends ReconciliationClass>(
vulnType: T,
sourceDir: string,
): Promise<SafeValidationResult<T>> {
try {
const result = await validateQueueAndDeliverable(vulnType, sourceDir);
return ok(result);
} catch (error) {
return err(error as PentestError);
if (error instanceof PentestError) return err(error);
return err(
new PentestError('Queue validation failed closed on an internal invariant.', 'unknown', false, { vulnType }),
);
}
}