mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-03 01:46:55 +02:00
garrytan
b1f5bc0032
Paid evals never retry (approved 2026-09-29): delete SHORT_CASE_RETRY_FILES and retriesWithinCaseCap, drop the retry fields from the registered wall rows (walls now cover one run plus reserve), make retriesForFiles return 0, pass --retry 0 explicitly, and drop --retry 1 from the package.json paid scripts. Add the eval:pass-rates alias. Tests that pinned the old retry allowance are updated as a policy change; review-finalization-budget now proves late-result recording under the production zero-retry arguments.
329 lines
21 KiB
TypeScript
329 lines
21 KiB
TypeScript
import { expect, test } from 'bun:test';
|
|
import * as fs from 'node:fs';
|
|
import * as os from 'node:os';
|
|
import * as path from 'node:path';
|
|
import { spawnSync } from 'node:child_process';
|
|
import type { Query, SDKMessage } from '@anthropic-ai/claude-agent-sdk';
|
|
import type { QueryProvider } from './helpers/agent-sdk-runner';
|
|
import { resolveClaudeBinary } from '../lib/claude-bin';
|
|
import type { EvalTestEntry } from './helpers/eval-store';
|
|
import { createBoundaryFixture, runBoundaryActor, type BoundaryCase } from './helpers/workflow-boundaries-fixture';
|
|
import { E2E_TIERS, E2E_TOUCHFILES } from './helpers/touchfiles-data';
|
|
import { PR_PROFILE_CASE_IDS, PR_PROFILE_FILES } from '../scripts/test-pr-profile';
|
|
import { applyHollowShardGuard, buildRunManifest, computePaidCaseSelection, DEFAULT_SHARD_TIMEOUT_MS, expectedPrCaseCount, prProfileTestNamePattern, resolvePaidShardBudget, retriesForFiles, verifySliceResults, type SliceResult } from '../scripts/test-paid-shards';
|
|
import { CAPTURE_MS } from './helpers/eval-budgets';
|
|
|
|
const cases: BoundaryCase[] = ['investigate-owned-completion', 'investigate-owned-abort', 'investigate-owned-ending-error'];
|
|
const files = ['test/skill-e2e-investigate-owned-completion.test.ts', 'test/skill-e2e-investigate-owned-termination.test.ts'];
|
|
|
|
for (const attack of ['read-state', 'cat-state', 'bash-edit', 'receipt-write']) test(`fixture denies undeclared interaction: ${attack}`, async () => {
|
|
const fixture = createBoundaryFixture('investigate-owned-completion');
|
|
try {
|
|
const tool = attack === 'read-state' ? 'Read' : 'Bash';
|
|
const input = attack === 'read-state' ? { file_path: fixture.boundary }
|
|
: { command: attack === 'cat-state' ? `cat '${fixture.boundary}'`
|
|
: attack === 'bash-edit' ? 'printf "arbitrary source" > src/value.js' : `printf FREEZE_RELEASED > '${fixture.receipts}'` };
|
|
const decision = await fixture.canUseTool(tool, input, { signal: new AbortController().signal, toolUseID: attack });
|
|
expect(decision.behavior).toBe('deny');
|
|
} finally { fs.rmSync(fixture.root, { recursive: true, force: true }); }
|
|
});
|
|
|
|
function protocolControl(id: BoundaryCase, faults: { skipCleanup?: boolean; duplicateQuestion?: boolean; failVerification?: boolean; rateLimit?: boolean; undeclared?: string; deniedBash?: string; includeAvailability?: boolean; misleadingFinal?: boolean; splitFinal?: boolean } = {}) {
|
|
let directory = '';
|
|
let calls = 0;
|
|
const provider: QueryProvider = input => {
|
|
calls++;
|
|
const options = input.options!;
|
|
directory = options.cwd!;
|
|
const env = options.env!;
|
|
expect(options.permissionMode).toBe('default');
|
|
expect(options.allowDangerouslySkipPermissions).toBe(false);
|
|
expect(options.settingSources).toEqual([]);
|
|
expect(options.maxTurns).toBe(12);
|
|
expect(options.allowedTools).toEqual([]);
|
|
expect(options.hooks?.PreToolUse).toHaveLength(1);
|
|
expect(options.pathToClaudeCodeExecutable).toBe(resolveClaudeBinary() ?? undefined);
|
|
expect(fs.realpathSync(env.GSTACK_HOME!)).toBe(path.join(path.dirname(directory), 'state'));
|
|
expect(env.HOME).toBe(path.join(path.dirname(directory), 'home'));
|
|
const executeTool = async (tool: string, input: Record<string, unknown>) => {
|
|
const decision = await options.hooks!.PreToolUse![0].hooks[0]({
|
|
hook_event_name: 'PreToolUse', session_id: 'fixture', transcript_path: path.join(directory, 'transcript'),
|
|
cwd: directory, tool_name: tool, tool_input: input, tool_use_id: `fixture-${calls}`,
|
|
}, `fixture-${calls}`, { signal: new AbortController().signal });
|
|
if ('async' in decision) throw new Error('fixture hook must be synchronous');
|
|
expect(decision.hookSpecificOutput?.hookEventName).toBe('PreToolUse');
|
|
const hook = decision.hookSpecificOutput as { permissionDecision?: string; updatedInput?: Record<string, unknown> };
|
|
if (hook.permissionDecision === 'deny') throw new Error(`native fixture guard denied ${tool}`);
|
|
expect(hook.permissionDecision).toBe(tool === 'AskUserQuestion' ? 'ask' : 'allow');
|
|
return hook.updatedInput ?? input;
|
|
};
|
|
const shell = async (command: string) => {
|
|
const approved = await executeTool('Bash', { command });
|
|
expect(approved.timeout).toBe(10000);
|
|
expect(approved.run_in_background).toBe(false);
|
|
return spawnSync('bash', ['-c', approved.command as string], { cwd: directory, env, encoding: 'utf8', timeout: 10000 });
|
|
};
|
|
const ask = async (question: string, labels: string[]) => {
|
|
const input = { questions: [{ header: 'Fixture', question, options: labels.map(label => ({ label, description: label })), multiSelect: false }] };
|
|
await executeTool('AskUserQuestion', input);
|
|
return options.canUseTool!('AskUserQuestion', input, { signal: new AbortController().signal, toolUseID: `fixture-${calls}` });
|
|
};
|
|
return {
|
|
async *[Symbol.asyncIterator]() {
|
|
yield { type: 'system', subtype: 'init', session_id: 'fixture-session' } as SDKMessage;
|
|
if (faults.misleadingFinal) yield { type: 'assistant', session_id: 'fixture-session', parent_tool_use_id: null,
|
|
message: { id: 'message-early', role: 'assistant', content: [{ type: 'text', text: 'I will ask whether to continue or abort, and stop if the verifier reports an error.' }] } } as SDKMessage;
|
|
if (faults.undeclared) {
|
|
const state = path.join(env.GSTACK_HOME!, 'freeze-dir.txt');
|
|
const receipt = path.join(path.dirname(directory), 'receipts');
|
|
await executeTool(faults.undeclared === 'read-state' ? 'Read' : 'Bash', faults.undeclared === 'read-state' ? { file_path: state } : {
|
|
command: faults.undeclared === 'cat-state' ? `cat '${state}'`
|
|
: faults.undeclared === 'bash-edit' ? 'printf "arbitrary source" > src/value.js' : `printf FREEZE_RELEASED > '${receipt}'`,
|
|
});
|
|
}
|
|
expect(fs.readFileSync(path.join(directory, 'workflow.md'), 'utf8')).toContain('Terminal cleanup');
|
|
expect(fs.existsSync(path.join(env.GSTACK_HOME!, 'freeze-dir.txt'))).toBe(false);
|
|
if (faults.deniedBash) {
|
|
await expect(executeTool('Bash', { command: faults.deniedBash })).rejects.toThrow('native fixture guard denied Bash');
|
|
const receipts = path.join(path.dirname(directory), 'receipts');
|
|
expect(fs.existsSync(receipts) ? fs.readFileSync(receipts, 'utf8') : '').toBe('');
|
|
expect(fs.existsSync(path.join(env.GSTACK_HOME!, 'freeze-dir.txt'))).toBe(false);
|
|
}
|
|
if (faults.includeAvailability) {
|
|
const workflow = fs.readFileSync(path.join(directory, 'workflow.md'), 'utf8');
|
|
const availability = workflow.match(/```bash\n([\s\S]*?)```/)![1].trim();
|
|
const checked = await shell(availability);
|
|
expect(checked.status).toBe(0);
|
|
expect(checked.stdout).toContain('FREEZE_AVAILABLE');
|
|
}
|
|
const acquisition = await shell('bash "$HOME/.claude/skills/gstack/freeze/bin/freeze-state.sh" acquire "src"');
|
|
expect(acquisition.status).toBe(0);
|
|
const owner = acquisition.stdout.match(/FREEZE_OWNER=([a-f0-9]{32})/)![1];
|
|
if (faults.rateLimit && calls === 1) {
|
|
yield { type: 'assistant', message: { content: [{ type: 'text', text: 'Attempt one acquired scope.' }] } } as SDKMessage;
|
|
throw Object.assign(new Error('rate limit'), { status: 429 });
|
|
}
|
|
const decision = await ask('Continue this investigation or abort?', ['Continue', 'Abort']);
|
|
expect(decision.behavior).toBe('allow');
|
|
if (decision.behavior !== 'allow') throw new Error('fixture refused its declared question');
|
|
if (faults.duplicateQuestion) await ask('Continue this investigation or abort?', ['Continue', 'Abort']);
|
|
const answer = (decision.updatedInput!.answers as Record<string, string>)['Continue this investigation or abort?'];
|
|
if (answer === 'Continue') {
|
|
await executeTool('Edit', { file_path: path.join(directory, 'src/value.js'), old_string: 'return 1', new_string: 'return 2' });
|
|
expect(fs.realpathSync(path.join(directory, 'src/value.js')).startsWith(directory + path.sep)).toBe(true);
|
|
fs.writeFileSync(path.join(directory, 'src/value.js'), 'export function value() { return 2; }\n');
|
|
if (faults.failVerification) {
|
|
const file = path.join(directory, 'verify.sh');
|
|
expect(fs.realpathSync(file).startsWith(directory + path.sep)).toBe(true);
|
|
fs.writeFileSync(file, fs.readFileSync(file, 'utf8').replace('value() !== 2', 'value() !== 3'));
|
|
}
|
|
const verified = await shell('bash ./verify.sh');
|
|
expect(verified.status).toBe(id === 'investigate-owned-ending-error' ? 69 : faults.failVerification ? 1 : 0);
|
|
} else expect(answer).toBe('Abort');
|
|
if (!faults.skipCleanup) expect((await shell(`bash "$HOME/.claude/skills/gstack/freeze/bin/freeze-state.sh" release "${owner}"`)).status).toBe(0);
|
|
const finalText = faults.misleadingFinal ? 'The fix is complete and verification succeeded.'
|
|
: id === 'investigate-owned-abort' ? 'Aborted at your request; no edit was made and the owned boundary was released.'
|
|
: id === 'investigate-owned-ending-error' ? 'The verifier is unavailable; the investigation ended with an error and the owned boundary was released.'
|
|
: 'The correction passed verification and the owned boundary was released.';
|
|
yield { type: 'assistant', session_id: 'fixture-session', parent_tool_use_id: null,
|
|
message: { id: 'message-final', role: 'assistant', content: [{ type: 'text', text: finalText }] } } as SDKMessage;
|
|
if (faults.splitFinal) yield { type: 'assistant', session_id: 'fixture-session', parent_tool_use_id: null,
|
|
message: { id: 'message-final', role: 'assistant', content: [{ type: 'text', text: 'No further action was taken.' }] } } as SDKMessage;
|
|
yield { type: 'result', subtype: 'success', num_turns: 1, total_cost_usd: 0 } as SDKMessage;
|
|
},
|
|
} as Query;
|
|
};
|
|
return { provider, directory: () => directory, calls: () => calls };
|
|
}
|
|
|
|
for (const id of cases) test(`fixture protocol control: ${id} persists receipts after cleanup`, async () => {
|
|
const driver = protocolControl(id);
|
|
const records: EvalTestEntry[] = [];
|
|
await runBoundaryActor(id, entry => records.push(entry), driver.provider);
|
|
expect(records).toHaveLength(1);
|
|
expect(records[0].passed).toBe(true);
|
|
expect(records[0].cost_usd).toBe(0);
|
|
expect(fs.existsSync(path.dirname(driver.directory()))).toBe(false);
|
|
const retained = JSON.parse(records[0].output!);
|
|
expect(retained.attempts).toHaveLength(1);
|
|
expect(retained.attempts[0].events).toHaveLength(3);
|
|
expect(retained.evidence.changedProtectedFiles).toEqual([]);
|
|
expect(retained.evidence.receipts).toContain('FREEZE_RELEASED');
|
|
expect(retained.evidence.interactions).toHaveLength(1);
|
|
});
|
|
|
|
test('captured commentless availability is denied, while the complete block is executable', async () => {
|
|
const command = `_FREEZE_SCRIPT="$HOME/.claude/skills/gstack/freeze/bin/check-freeze.sh"
|
|
[ -x "$_FREEZE_SCRIPT" ] && echo "FREEZE_AVAILABLE" || echo "FREEZE_UNAVAILABLE"`;
|
|
const driver = protocolControl('investigate-owned-completion', { deniedBash: command, includeAvailability: true });
|
|
const records: EvalTestEntry[] = [];
|
|
await expect(runBoundaryActor('investigate-owned-completion', entry => records.push(entry), driver.provider)).rejects.toThrow('undeclared interaction');
|
|
expect(records[0]).toMatchObject({ passed: false, exit_reason: 'assertion_failed' });
|
|
const evidence = JSON.parse(records[0].output!).evidence;
|
|
expect(evidence.executions.find((event: { input: { command?: string } }) => event.input.command === command).allowed).toBe(false);
|
|
expect(evidence.executions.some((event: { allowed: boolean; input: { command?: string } }) => event.allowed && event.input.command?.includes('FREEZE_AVAILABLE'))).toBe(true);
|
|
expect(evidence.source).toBe('export function value() { return 2; }\n');
|
|
expect(evidence.receipts).toContain('FREEZE_RELEASED');
|
|
expect(evidence.boundary).toBe('');
|
|
const clean = protocolControl('investigate-owned-completion', { includeAvailability: true });
|
|
const passing: EvalTestEntry[] = [];
|
|
await runBoundaryActor('investigate-owned-completion', entry => passing.push(entry), clean.provider);
|
|
expect(passing[0].passed).toBe(true);
|
|
expect(JSON.parse(passing[0].output!).evidence.executions.some((event: { allowed: boolean; input: { command?: string } }) => event.allowed && event.input.command?.includes('FREEZE_AVAILABLE'))).toBe(true);
|
|
});
|
|
|
|
for (const attack of ['read-state', 'cat-state', 'bash-edit', 'receipt-write']) test(`registered native hook rejects and retains denied ${attack}`, async () => {
|
|
const driver = protocolControl('investigate-owned-completion', { undeclared: attack });
|
|
const records: EvalTestEntry[] = [];
|
|
await expect(runBoundaryActor('investigate-owned-completion', entry => records.push(entry), driver.provider)).rejects.toThrow('native fixture guard denied');
|
|
expect(records[0].passed).toBe(false);
|
|
const evidence = JSON.parse(records[0].output!).evidence;
|
|
expect(evidence.executions).toHaveLength(1);
|
|
expect(evidence.executions[0].allowed).toBe(false);
|
|
expect(evidence.source).toBe('export function value() { return 1; }\n');
|
|
expect(evidence.receipts).toBe('');
|
|
expect(fs.existsSync(path.dirname(driver.directory()))).toBe(false);
|
|
});
|
|
|
|
test('duplicate owner questions are rejected by the registered native hook', async () => {
|
|
const driver = protocolControl('investigate-owned-completion', { duplicateQuestion: true });
|
|
const records: EvalTestEntry[] = [];
|
|
await expect(runBoundaryActor('investigate-owned-completion', entry => records.push(entry), driver.provider)).rejects.toThrow('native fixture guard denied AskUserQuestion');
|
|
const evidence = JSON.parse(records[0].output!).evidence;
|
|
expect(evidence.interactions.filter((event: { disposition: string }) => event.disposition === 'continue-investigation')).toHaveLength(1);
|
|
expect(evidence.interactions.at(-1).disposition).toBe('unsupported-tool');
|
|
expect(evidence.executions.at(-1).allowed).toBe(false);
|
|
});
|
|
|
|
test('fixture rejects a successful actor result when owned cleanup is missing', async () => {
|
|
const driver = protocolControl('investigate-owned-abort', { skipCleanup: true });
|
|
const records: EvalTestEntry[] = [];
|
|
await expect(runBoundaryActor('investigate-owned-abort', entry => records.push(entry), driver.provider)).rejects.toThrow('owned boundary remains');
|
|
expect(records[0].passed).toBe(false);
|
|
expect(records[0].exit_reason).toBe('assertion_failed');
|
|
expect(JSON.parse(records[0].output!).evidence.boundary).toContain('gstack-freeze-v1:');
|
|
expect(fs.existsSync(path.dirname(driver.directory()))).toBe(false);
|
|
});
|
|
|
|
test('fixture records failed verification rather than crediting successful cleanup', async () => {
|
|
const driver = protocolControl('investigate-owned-completion', { failVerification: true });
|
|
const records: EvalTestEntry[] = [];
|
|
await expect(runBoundaryActor('investigate-owned-completion', entry => records.push(entry), driver.provider)).rejects.toThrow('verification did not succeed');
|
|
expect(JSON.parse(records[0].output!).evidence.receipts).toContain('VERIFY_STATUS:1');
|
|
});
|
|
|
|
test('rate-limit retry restores the fixture and retains both native event streams', async () => {
|
|
const driver = protocolControl('investigate-owned-completion', { rateLimit: true });
|
|
const records: EvalTestEntry[] = [];
|
|
await runBoundaryActor('investigate-owned-completion', entry => records.push(entry), driver.provider);
|
|
expect(driver.calls()).toBe(2);
|
|
const retained = JSON.parse(records[0].output!);
|
|
expect(retained.attempts).toHaveLength(2);
|
|
expect(retained.attempts[0].evidence.boundary).toContain('gstack-freeze-v1:');
|
|
expect(retained.attempts[0].events.find((event: SDKMessage) => event.type === 'assistant').message.content[0].text).toContain('Attempt one');
|
|
expect(retained.attempts[1].evidence.boundary).toBe('');
|
|
});
|
|
|
|
for (const id of ['investigate-owned-abort', 'investigate-owned-ending-error'] as const) {
|
|
test(`${id}: an earlier acknowledgment cannot cover a misleading final response`, async () => {
|
|
const driver = protocolControl(id, { misleadingFinal: true });
|
|
const records: EvalTestEntry[] = [];
|
|
await expect(runBoundaryActor(id, entry => records.push(entry), driver.provider)).rejects.toThrow(/acknowledge/);
|
|
expect(records[0].passed).toBe(false);
|
|
const retained = JSON.parse(records[0].output!);
|
|
expect(retained.evidence.receipts).toContain('FREEZE_RELEASED');
|
|
expect(retained.evidence.boundary).toBe('');
|
|
expect(retained.assistant).toContain('whether to continue or abort');
|
|
expect(retained.assistant).toContain('The fix is complete and verification succeeded.');
|
|
expect(fs.existsSync(path.dirname(driver.directory()))).toBe(false);
|
|
});
|
|
test(`${id}: the actual final response can span multiple native fragments`, async () => {
|
|
const driver = protocolControl(id, { splitFinal: true });
|
|
const records: EvalTestEntry[] = [];
|
|
await runBoundaryActor(id, entry => records.push(entry), driver.provider);
|
|
expect(records[0].passed).toBe(true);
|
|
});
|
|
}
|
|
|
|
test('fixture binds the configured CLI executable rather than the SDK bundled version', async () => {
|
|
const previous = process.env.GSTACK_CLAUDE_BIN;
|
|
process.env.GSTACK_CLAUDE_BIN = process.execPath;
|
|
try {
|
|
const driver = protocolControl('investigate-owned-abort');
|
|
const records: EvalTestEntry[] = [];
|
|
await runBoundaryActor('investigate-owned-abort', entry => records.push(entry), driver.provider);
|
|
expect(records[0].passed).toBe(true);
|
|
} finally {
|
|
if (previous === undefined) delete process.env.GSTACK_CLAUDE_BIN;
|
|
else process.env.GSTACK_CLAUDE_BIN = previous;
|
|
}
|
|
});
|
|
|
|
test('fixture refuses a skill registration escaping the temporary root before writing', () => {
|
|
const root = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), 'gbound-link-'));
|
|
const outside = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), 'gbound-out-'));
|
|
try {
|
|
fs.symlinkSync(outside, path.join(root, 'home'));
|
|
expect(() => createBoundaryFixture('investigate-owned-completion', root)).toThrow('fixture write escapes');
|
|
expect(fs.readdirSync(outside)).toEqual([]);
|
|
} finally {
|
|
fs.rmSync(root, { recursive: true, force: true });
|
|
fs.rmSync(outside, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test('F9 actors are registered in the gate PR profile with exact file ownership', () => {
|
|
for (const id of cases) {
|
|
expect(E2E_TIERS[id], id).toBe('gate');
|
|
expect(PR_PROFILE_CASE_IDS as readonly string[]).toContain(id);
|
|
expect(E2E_TOUCHFILES[id]).toContain('test/helpers/workflow-boundaries-fixture.ts');
|
|
}
|
|
expect(PR_PROFILE_FILES[files[0]]).toEqual([cases[0]]);
|
|
expect(PR_PROFILE_FILES[files[1]]).toEqual(cases.slice(1));
|
|
});
|
|
|
|
test('F9 changed-input selection produces three cases with exact patterns and counts', () => {
|
|
const selected = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: ['freeze/bin/freeze-state.sh'] });
|
|
expect(selected.coverage?.mode).toBe('pr');
|
|
expect(selected.coverage?.unknownFiles).toEqual([]);
|
|
expect(selected.selection.e2e).toEqual([...cases].sort());
|
|
expect(files.map(file => expectedPrCaseCount(file, selected.selection))).toEqual([1, 2]);
|
|
expect(prProfileTestNamePattern(files[0], selected.selection)).toBe('(?:^|\\s)(?:investigate-owned-completion)$');
|
|
expect(prProfileTestNamePattern(files[1], selected.selection)).toBe('(?:^|\\s)(?:investigate-owned-abort|investigate-owned-ending-error)$');
|
|
});
|
|
|
|
test('both F9 files fit the existing wall with their one run and reserve', () => {
|
|
for (const file of files) {
|
|
const source = fs.readFileSync(path.join(import.meta.dir, '..', file), 'utf8');
|
|
const count = PR_PROFILE_FILES[file].length;
|
|
expect([...source.matchAll(/\}, CAPTURE_MS\);/g)]).toHaveLength(count);
|
|
expect(retriesForFiles([file])).toBe(0);
|
|
const budget = resolvePaidShardBudget([file]);
|
|
expect(budget).toEqual({ timeoutMs: DEFAULT_SHARD_TIMEOUT_MS, source: 'default', policyId: null });
|
|
expect(count * CAPTURE_MS * (retriesForFiles([file]) + 1) + 120000).toBeLessThanOrEqual(budget.timeoutMs);
|
|
}
|
|
});
|
|
|
|
test('F9 manifest rejects skipped, missing and hollow actor coverage', () => {
|
|
const manifest = buildRunManifest({ tier: 'gate', profile: 'pr', sliceCount: 1,
|
|
evalsAll: false, env: {}, changedFiles: ['freeze/bin/freeze-state.sh'], discovered: files });
|
|
expect(manifest.entries.filter(entry => entry.status === 'planned').map(entry => entry.file).sort()).toEqual(files);
|
|
const result: SliceResult = {
|
|
version: 1, tier: 'gate', profile: 'pr', selection: manifest.selection, sliceIndex: 1, sliceCount: 1,
|
|
outcomes: manifest.entries.map(entry => ({ files: [entry.file], status: 'passed', exitCode: 0, elapsedMs: 1,
|
|
executedTests: expectedPrCaseCount(entry.file, manifest.selection!), skippedTests: 0,
|
|
...(entry.budget ? { budget: entry.budget } : {}),
|
|
})),
|
|
};
|
|
expect(verifySliceResults(manifest, [result]).ok).toBe(true);
|
|
for (const mutation of ['skip', 'empty', 'missing']) {
|
|
const invalid = structuredClone(result);
|
|
if (mutation === 'skip') invalid.outcomes[1].skippedTests = 1;
|
|
if (mutation === 'empty') invalid.outcomes[1].executedTests = 0;
|
|
if (mutation === 'missing') invalid.outcomes.pop();
|
|
expect(verifySliceResults(manifest, [invalid]).ok).toBe(false);
|
|
}
|
|
const hollow = { ...result.outcomes[1], executedTests: 0, skippedTests: 2, shard: 1, groupPid: null };
|
|
expect(applyHollowShardGuard([hollow], { evalsAll: false, requireExecuted: true })[0].status).toBe('passed-empty');
|
|
});
|