mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-28 07:32:14 +02:00
v1.90.2.0 perf: halve local free-suite time and preserve coverage (#2972)
* perf: remove repeated test work and preserve AUQ execution budgets * fix: validate native evaluation fixture evidence at its actual boundaries * fix: clarify deployment approval and recovery state transitions * chore: document coverage and release v1.90.2.0 * test: preserve Windows scheduling and native no-change consent * test: recognize verified reads through fixture symlinks * test: isolate alias-name installation from runtime assets
This commit is contained in:
Garry Tan
@@ -738,6 +738,58 @@ describe('rate-limit detectors', () => {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('runAgentSdkTest — concurrency', () => {
|
||||
test('admission runs only once across a rate-limit retry', async () => {
|
||||
__resetSemaphoreForTests(1);
|
||||
let admissions = 0;
|
||||
const stub: StubConfig = {
|
||||
streams: [[systemInit(), resultRateLimit()], [systemInit(), resultSuccess()]], calls: [],
|
||||
};
|
||||
await runAgentSdkTest({ ...BASE_OPTS, queryProvider: makeStubProvider(stub), maxRetries: 1,
|
||||
onAdmission: () => { admissions++; } });
|
||||
expect(admissions).toBe(1);
|
||||
expect(stub.calls).toHaveLength(2);
|
||||
});
|
||||
|
||||
test.each(['throw', 'abort'] as const)('admission %s prevents transport creation and releases its slot', async mode => {
|
||||
__resetSemaphoreForTests(1);
|
||||
const controller = new AbortController(), failure = new Error(`admission ${mode}`);
|
||||
const stub: StubConfig = { streams: [[systemInit(), resultSuccess()]], calls: [] };
|
||||
const queryProvider = makeStubProvider(stub);
|
||||
const failed = runAgentSdkTest({ ...BASE_OPTS, queryProvider, signal: controller.signal,
|
||||
onAdmission: () => { if (mode === 'throw') throw failure; controller.abort(failure); } });
|
||||
const sibling = runAgentSdkTest({ ...BASE_OPTS, queryProvider });
|
||||
const results = await Promise.allSettled([failed, sibling]);
|
||||
expect(results[0]).toEqual({ status: 'rejected', reason: failure });
|
||||
expect(results[1].status).toBe('fulfilled');
|
||||
expect(stub.calls).toHaveLength(1);
|
||||
});
|
||||
|
||||
test('queued cancellation never calls admission or transport and leaves later siblings runnable', async () => {
|
||||
__resetSemaphoreForTests(1);
|
||||
const controller = new AbortController(), reason = new Error('queued cancellation');
|
||||
let unblock!: () => void, started!: () => void;
|
||||
const held = new Promise<void>(resolve => { unblock = resolve; });
|
||||
const ready = new Promise<void>(resolve => { started = resolve; });
|
||||
const provider: QueryProvider = () => (async function* () {
|
||||
started(); await held; yield resultSuccess();
|
||||
})() as unknown as Query;
|
||||
const occupying = runAgentSdkTest({ ...BASE_OPTS, queryProvider: provider });
|
||||
await ready;
|
||||
let admissions = 0;
|
||||
const stub: StubConfig = { streams: [[resultSuccess()]], calls: [] };
|
||||
const queryProvider = makeStubProvider(stub);
|
||||
const queued = runAgentSdkTest({ ...BASE_OPTS, queryProvider, signal: controller.signal,
|
||||
onAdmission: () => { admissions++; } });
|
||||
const sibling = runAgentSdkTest({ ...BASE_OPTS, queryProvider });
|
||||
controller.abort(reason);
|
||||
const cancelled = await Promise.allSettled([queued]);
|
||||
expect(cancelled[0]).toEqual({ status: 'rejected', reason });
|
||||
expect(admissions).toBe(0); expect(stub.calls).toHaveLength(0);
|
||||
unblock();
|
||||
await Promise.all([occupying, sibling]);
|
||||
expect(stub.calls).toHaveLength(1);
|
||||
});
|
||||
|
||||
test('process-level semaphore caps concurrent queries', async () => {
|
||||
__resetSemaphoreForTests(2);
|
||||
let inFlight = 0;
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
import { expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
type Spec = { suite: 'consistency' | 'ab' | 'direct'; runs?: number; reject?: number; rejectAll?: boolean;
|
||||
capacity?: number; queryMs?: number; outerMs?: number;
|
||||
setupReject?: number; cleanupReject?: number; judgeReject?: number; empty?: number; omit?: string; omitIndex?: number; scores?: number[] };
|
||||
|
||||
function exercise(spec: Spec) {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'auq-parallel-free-'));
|
||||
const worker = path.join(root, 'registration.test.ts');
|
||||
fs.writeFileSync(worker, `import ${JSON.stringify(path.join(import.meta.dir, 'helpers/auq-parallel-worker.ts'))};`);
|
||||
try {
|
||||
const child = Bun.spawnSync([process.execPath, 'test', worker], {
|
||||
cwd: ROOT, timeout: 15_000,
|
||||
env: { PATH: process.env.PATH ?? '', HOME: root, TMPDIR: root, TEMP: root, TMP: root,
|
||||
AUQ_PARALLEL_SPEC: JSON.stringify(spec), AUQ_PARALLEL_ROOT: root,
|
||||
AUQ_CONSISTENCY_RUNS: String(spec.runs ?? 3), EVALS_HERMETIC: '1', EVALS_RUN_ID: 'free',
|
||||
GSTACK_CLAUDE_BIN: '/fixture/claude', GSTACK_EVAL_DIR: path.join(root, 'artifacts'),
|
||||
ANTHROPIC_API_KEY: 'fixture-key', ANTHROPIC_BASE_URL: 'http://127.0.0.1:1',
|
||||
GSTACK_SDK_MAX_CONCURRENCY: String(spec.capacity ?? 3), GSTACK_EVAL_MODEL_CAPTURE: 'fixture-model',
|
||||
GIT_CONFIG_NOSYSTEM: '1', ...(process.env.SystemRoot ? { SystemRoot: process.env.SystemRoot } : {}),
|
||||
},
|
||||
});
|
||||
const output = child.stdout.toString() + child.stderr.toString();
|
||||
expect(child.signalCode ?? null, output).toBeNull();
|
||||
expect(fs.existsSync(path.join(root, 'facts.json')), output).toBe(true);
|
||||
const facts = JSON.parse(fs.readFileSync(path.join(root, 'facts.json'), 'utf8'));
|
||||
if (spec.suite === 'direct') return { code: child.exitCode, output, facts };
|
||||
const ids = spec.suite === 'consistency' ? Array.from({ length: spec.runs ?? 3 }, (_, i) => String(i)) : ['carved', 'verbose'];
|
||||
const started = ids.filter((_, i) => i !== spec.setupReject);
|
||||
for (const kind of ['capture-start', 'query-start', 'query-close', 'capture-settle', 'cleanup']) {
|
||||
expect(facts.events.filter((event: any) => event.kind === kind).map((event: any) => event.id).sort(), output)
|
||||
.toEqual([...started].sort());
|
||||
}
|
||||
expect(facts.active).toBe(0); expect(facts.judging).toBe(0); expect(facts.answers).toBe(0);
|
||||
expect(facts.leftovers).toEqual(spec.cleanupReject === undefined ? [] : [facts.fixtures[spec.cleanupReject]]);
|
||||
expect(facts.ownedStateLeftovers).toEqual([]);
|
||||
expect(new Set(facts.fixtures).size).toBe(started.length);
|
||||
expect(new Set(facts.inputs.map((input: any) => input.config)).size).toBe(started.length);
|
||||
expect(new Set(facts.inputs.map((input: any) => input.state)).size).toBe(started.length);
|
||||
expect(facts.peak).toBe(Math.min(spec.capacity ?? 3, started.length));
|
||||
expect(facts.judgePeak).toBeLessThanOrEqual(spec.suite === 'consistency' ? 1 : 2);
|
||||
const lastCapture = facts.events.findLastIndex((event: any) => event.kind === 'capture-settle');
|
||||
const firstCleanup = facts.events.findIndex((event: any) => event.kind === 'cleanup');
|
||||
expect(firstCleanup, output).toBeGreaterThan(lastCapture);
|
||||
if (spec.suite === 'ab') expect(firstCleanup).toBeGreaterThan(facts.events.findLastIndex((event: any) => event.kind === 'judge-settle'));
|
||||
expect(facts.judges.map((judge: any) => judge.id).sort()).toEqual(ids.filter((_, i) =>
|
||||
!spec.rejectAll && i !== spec.reject && i !== spec.setupReject && i !== spec.empty).sort());
|
||||
expect(facts.receipts).toHaveLength(started.length);
|
||||
for (const receipt of facts.receipts) {
|
||||
expect(receipt).toMatchObject({ source: 'can_use_tool', workflowCompleted: false, answered: false,
|
||||
maxTurns: 12, timeoutMs: 240_000 });
|
||||
}
|
||||
for (const { request } of facts.judgeRequests) {
|
||||
expect(request).toMatchObject({ model: 'claude-haiku-4-5-20251001', max_tokens: 8192 });
|
||||
}
|
||||
for (const input of facts.inputs) {
|
||||
expect(input).toMatchObject({ model: 'fixture-model', maxTurns: 12, tools: ['Read', 'Write', 'AskUserQuestion'],
|
||||
allowedTools: ['Read', 'Write', 'AskUserQuestion'], permissionMode: 'default', settingSources: [],
|
||||
systemPrompt: { type: 'preset', preset: 'claude_code' } });
|
||||
expect(input.skill).toBe(fs.readFileSync(path.join(ROOT, input.id === 'verbose'
|
||||
? 'test/fixtures/auq-pre-cut-plan-ceo-review-SKILL.md' : 'plan-ceo-review/SKILL.md'), 'utf8'));
|
||||
expect(input.sections.length > 0).toBe(input.id !== 'verbose');
|
||||
expect(input.prompt).toContain(path.join(input.cwd, 'plan-ceo-review/SKILL.md'));
|
||||
expect(input.prompt).toContain(path.join(input.cwd, 'plan.md'));
|
||||
}
|
||||
return { code: child.exitCode, output, facts };
|
||||
} finally { fs.rmSync(root, { recursive: true, force: true }); }
|
||||
}
|
||||
|
||||
test.each([
|
||||
{ suite: 'consistency', capacity: 1, runs: 3 },
|
||||
{ suite: 'consistency', capacity: 3, runs: 7 },
|
||||
{ suite: 'consistency', capacity: 3, runs: 3 },
|
||||
{ suite: 'ab', capacity: 1 },
|
||||
] as const)('slow native captures retain their admitted budget: %j', spec => {
|
||||
const result = exercise({ ...spec, queryMs: 150_000 });
|
||||
expect(result.code, result.output).toBe(0);
|
||||
expect(result.facts.pendingTimers).toBe(0);
|
||||
expect(result.facts.receipts.every((receipt: any) => receipt.outcome === 'question_captured')).toBe(true);
|
||||
const deadlines = result.facts.events.filter((event: any) => event.kind === 'capture-start').map((event: any) => event.caseDeadline);
|
||||
expect(new Set(deadlines).size).toBe(1);
|
||||
expect(deadlines[0]).toBe(spec.suite === 'consistency' ? spec.runs * 300_000 + 60_000 : 600_000);
|
||||
expect(result.facts.elapsed).toBe(Math.ceil((spec.suite === 'consistency' ? spec.runs : 2) / spec.capacity) * 150_000
|
||||
+ (spec.suite === 'consistency' ? spec.runs * 90 : 90));
|
||||
}, 20_000);
|
||||
|
||||
test.each([
|
||||
{ outerMs: 500_000, queryMs: 150_000, elapsed: 450_000, queries: 3, passed: 3 },
|
||||
{ outerMs: 200_000, queryMs: 150_000, elapsed: 200_000, queries: 2, passed: 1 },
|
||||
{ outerMs: -1, queryMs: 150_000, elapsed: 0, queries: 0, passed: 0 },
|
||||
{ outerMs: 960_000, queryMs: 250_000, elapsed: 720_000, queries: 3, passed: 0 },
|
||||
{ queryMs: 150_000, elapsed: 240_000, queries: 3, passed: 1 },
|
||||
{ queryMs: 250_000, elapsed: 240_000, queries: 3, passed: 0 },
|
||||
])('actual capture bounds queued and active work without changing legacy deadlines: %j', scenario => {
|
||||
const { code, output, facts } = exercise({ suite: 'direct', capacity: 1, runs: 3, ...scenario });
|
||||
expect(code, output).toBe(0);
|
||||
expect(facts.elapsed).toBe(scenario.elapsed);
|
||||
expect(facts.inputs).toHaveLength(scenario.queries);
|
||||
expect(facts.settlements).toHaveLength(3);
|
||||
expect(facts.settlements.filter((result: any) => result.status === 'fulfilled')).toHaveLength(scenario.passed);
|
||||
for (const result of facts.settlements.filter((result: any) => result.status === 'rejected')) {
|
||||
expect(result.reason).toContain('AUQ capture failed (timeout)');
|
||||
}
|
||||
expect(facts.receipts).toHaveLength(3);
|
||||
expect(facts.receipts.filter((receipt: any) => receipt.outcome === 'question_captured')).toHaveLength(scenario.passed);
|
||||
expect(facts.receipts.filter((receipt: any) => receipt.outcome === 'timeout')).toHaveLength(3 - scenario.passed);
|
||||
expect(facts.receipts.every((receipt: any) => receipt.timeoutMs === 240_000 && !receipt.answered)).toBe(true);
|
||||
expect(facts.active).toBe(0); expect(facts.answers).toBe(0); expect(facts.pendingTimers).toBe(0);
|
||||
expect(facts.leftovers).toEqual([]); expect(facts.ownedStateLeftovers).toEqual([]);
|
||||
expect(facts.events.filter((event: any) => event.kind === 'query-close')).toHaveLength(scenario.queries);
|
||||
expect(facts.events.filter((event: any) => event.kind === 'cleanup')).toHaveLength(3);
|
||||
}, 20_000);
|
||||
|
||||
test('consistency starts all three independent native captures and retains the passing score boundaries', () => {
|
||||
const result = exercise({ suite: 'consistency', scores: [3, 5, 4] });
|
||||
expect(result.code, result.output).toBe(0);
|
||||
expect(result.output).toContain('STABLE across 3 runs');
|
||||
}, 20_000);
|
||||
|
||||
test('consistency uses the existing three-query bound even for seven requested samples', () => {
|
||||
const result = exercise({ suite: 'consistency', runs: 7 });
|
||||
expect(result.code, result.output).toBe(0);
|
||||
expect(result.output).toContain('STABLE across 7 runs');
|
||||
}, 20_000);
|
||||
|
||||
test.each(['consistency', 'ab'] as const)('%s waits for successful siblings after a capture rejects', suite => {
|
||||
const result = exercise({ suite, reject: 0 });
|
||||
expect(result.code, result.output).toBe(1);
|
||||
expect(result.output).toContain('capture rejection');
|
||||
expect(result.output).toContain(suite === 'consistency' ? '[AUQ-consistency run 3/3]' : '[AUQ-AB VERBOSE]');
|
||||
}, 20_000);
|
||||
|
||||
test.each(['consistency', 'ab'] as const)('%s retains slow queued siblings and all-settled cleanup after rejection', suite => {
|
||||
const result = exercise({ suite, capacity: 1, queryMs: 150_000, reject: 0 });
|
||||
expect(result.code, result.output).toBe(1);
|
||||
expect(result.output).toContain('capture rejection');
|
||||
expect(result.facts.receipts.filter((receipt: any) => receipt.outcome === 'question_captured'))
|
||||
.toHaveLength(suite === 'consistency' ? 2 : 1);
|
||||
expect(result.facts.pendingTimers).toBe(0);
|
||||
}, 20_000);
|
||||
|
||||
test.each(['consistency', 'ab'] as const)('%s preserves every rejection after all children settle', suite => {
|
||||
const result = exercise({ suite, rejectAll: true });
|
||||
expect(result.code, result.output).toBe(1);
|
||||
for (const input of result.facts.inputs) expect(result.output).toContain(`capture rejection ${input.id}`);
|
||||
}, 20_000);
|
||||
|
||||
test.each(['consistency', 'ab'] as const)('%s cleans the owned sibling after fixture setup rejects', suite => {
|
||||
const result = exercise({ suite, setupReject: 1 });
|
||||
expect(result.code, result.output).toBe(1);
|
||||
expect(result.output).toContain('setup rejection');
|
||||
}, 20_000);
|
||||
|
||||
test.each(['consistency', 'ab'] as const)('%s retains capture errors and other results when cleanup also rejects', suite => {
|
||||
const result = exercise({ suite, reject: 0, cleanupReject: 0 });
|
||||
expect(result.code, result.output).toBe(1);
|
||||
expect(result.output).toContain('capture rejection');
|
||||
expect(result.output).toContain('cleanup rejection');
|
||||
expect(result.output).toContain(suite === 'consistency' ? '[AUQ-consistency run 3/3]' : '[AUQ-AB VERBOSE]');
|
||||
}, 20_000);
|
||||
|
||||
test.each(['ELI10:', 'Recommendation:', 'Pros / cons:', '✅', '❌', 'Net:', '(recommended)'])('consistency still rejects missing format element %s', omit => {
|
||||
const result = exercise({ suite: 'consistency', omit });
|
||||
expect(result.code, result.output).toBe(1);
|
||||
expect(result.output).toContain(`format element "${omit}" missing in run(s) 3`);
|
||||
}, 20_000);
|
||||
|
||||
test('consistency retains the substance floor and spread oracle', () => {
|
||||
const result = exercise({ suite: 'consistency', scores: [2, 5, 4] });
|
||||
expect(result.code, result.output).toBe(1);
|
||||
expect(result.output).toContain('min substance 2 < 3');
|
||||
expect(result.output).toContain('spread 3 > 2');
|
||||
}, 20_000);
|
||||
|
||||
test.each(['consistency', 'ab'] as const)('%s retains the empty-capture assertion', suite => {
|
||||
const result = exercise({ suite, empty: 0 });
|
||||
expect(result.code, result.output).toBe(1);
|
||||
expect(result.output).toContain(suite === 'consistency' ? 'produced no AUQ at all' : 'A/B inconclusive');
|
||||
}, 20_000);
|
||||
|
||||
test.each(['consistency', 'ab'] as const)('%s retains judge-unavailable scoring without abandoning siblings', suite => {
|
||||
const result = exercise({ suite, judgeReject: 0 });
|
||||
expect(result.code, result.output).toBe(1);
|
||||
expect(result.output).toContain(suite === 'consistency' ? 'min substance 0 < 3' : 'carved substance regressed >1 pt');
|
||||
}, 20_000);
|
||||
|
||||
test('A/B runs both distinct arms and preserves the one-point substance tolerance', () => {
|
||||
const result = exercise({ suite: 'ab', scores: [3, 4] });
|
||||
expect(result.code, result.output).toBe(0);
|
||||
expect(result.facts.judgePeak).toBe(2);
|
||||
expect(result.output).toContain('NO DEGRADATION');
|
||||
}, 20_000);
|
||||
|
||||
test('A/B still rejects any format regression', () => {
|
||||
const result = exercise({ suite: 'ab', omit: 'Net:', omitIndex: 0 });
|
||||
expect(result.code, result.output).toBe(1);
|
||||
expect(result.output).toContain('carved dropped: [Net:]');
|
||||
}, 20_000);
|
||||
|
||||
test('A/B still rejects a substance regression over one point', () => {
|
||||
const result = exercise({ suite: 'ab', scores: [3, 5] });
|
||||
expect(result.code, result.output).toBe(1);
|
||||
expect(result.output).toContain('carved substance regressed >1 pt');
|
||||
}, 20_000);
|
||||
@@ -1,4 +1,4 @@
|
||||
import { afterEach, describe, expect, test } from 'bun:test';
|
||||
import { afterEach, describe, expect, jest, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
@@ -28,6 +28,26 @@ async function withNativeProjectDirectory<T>(cwd: string | undefined, work: () =
|
||||
}
|
||||
}
|
||||
|
||||
async function withPublicationClock<T>(work: () => Promise<T>): Promise<T> {
|
||||
expect(jest.isFakeTimers()).toBe(false);
|
||||
const immediate = globalThis.setImmediate;
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
let settled = false;
|
||||
const pending = Promise.resolve().then(work);
|
||||
void pending.then(() => { settled = true; }, () => { settled = true; });
|
||||
for (let elapsed = 0; elapsed <= 2_000; elapsed += 50) {
|
||||
await new Promise<void>(resolve => immediate(resolve));
|
||||
if (settled) return await pending;
|
||||
if (elapsed < 2_000) jest.advanceTimersByTime(50);
|
||||
}
|
||||
throw new Error('Publication hook did not settle within its 2000ms polling budget.');
|
||||
} finally {
|
||||
jest.clearAllTimers();
|
||||
jest.useRealTimers();
|
||||
}
|
||||
}
|
||||
|
||||
function fixture(phase: Phase = 'ceo', next = 'design') {
|
||||
const cwd = fs.realpathSync(fs.mkdtempSync(path.join(tmpdir(), 'autoplan-publication-'))); dirs.push(cwd);
|
||||
const source = path.join(cwd, 'source.md'), active = path.join(cwd, 'active.md'), restore = path.join(cwd, 'restore.md');
|
||||
@@ -325,18 +345,18 @@ describe('Autoplan parent publication guard', () => {
|
||||
});
|
||||
|
||||
for (const project of ['absent', 'empty', 'relative', 'foreign', 'unnormalized'] as const) {
|
||||
test(`native project ownership rejects ${project} original-directory evidence after cd`, async () => {
|
||||
test.serial(`native project ownership rejects ${project} original-directory evidence after cd`, async () => {
|
||||
const { f } = changedDirectory();
|
||||
const value = project === 'absent' ? undefined : project === 'empty' ? '' : project === 'relative' ? 'relative' :
|
||||
project === 'foreign' ? path.join(f.cwd, 'foreign') : f.cwd + '/.';
|
||||
const output: any = await withNativeProjectDirectory(value, () => runPublicationHook(f.input, ROOT));
|
||||
const output: any = await withNativeProjectDirectory(value, () => withPublicationClock(() => runPublicationHook(f.input, ROOT)));
|
||||
expect(output.hookSpecificOutput.permissionDecision).toBe('deny');
|
||||
});
|
||||
}
|
||||
|
||||
for (const mutation of ['foreign-root', 'sidechain', 'dangling', 'competing-root', 'duplicate-root',
|
||||
'foreign-session', 'wrong-current-input', 'incomplete-current'] as const) {
|
||||
test(`native project ownership retains ${mutation} rejection after cd`, async () => {
|
||||
test.serial(`native project ownership retains ${mutation} rejection after cd`, async () => {
|
||||
const { f, rows, save } = changedDirectory();
|
||||
const current = rows.at(-1)!;
|
||||
if (mutation === 'foreign-root') rows[0]!.cwd = path.join(f.cwd, 'foreign');
|
||||
@@ -351,7 +371,7 @@ describe('Autoplan parent publication guard', () => {
|
||||
const bytes = fs.readFileSync(f.input.transcript_path, 'utf8');
|
||||
fs.writeFileSync(f.input.transcript_path, bytes.slice(0, -1));
|
||||
}
|
||||
const output: any = await withNativeProjectDirectory(f.cwd, () => runPublicationHook(f.input, ROOT));
|
||||
const output: any = await withNativeProjectDirectory(f.cwd, () => withPublicationClock(() => runPublicationHook(f.input, ROOT)));
|
||||
expect(output.hookSpecificOutput.permissionDecision).toBe('deny');
|
||||
});
|
||||
}
|
||||
@@ -618,6 +638,54 @@ describe('Autoplan parent publication guard', () => {
|
||||
f.current(); expect(f.evaluate()).toMatchObject({ allow: false, reason: expect.stringContaining('Publish the filled') });
|
||||
});
|
||||
|
||||
test.serial('unavailable journal polls every 50ms for the full 2000ms budget', async () => {
|
||||
const f = fixture();
|
||||
await withNativeProjectDirectory(f.cwd, () => withPublicationClock(async () => {
|
||||
const started = performance.now(), timer = jest.spyOn(globalThis, 'setTimeout');
|
||||
try {
|
||||
const output: any = await runPublicationHook(f.input, ROOT);
|
||||
expect(performance.now() - started).toBe(2_000);
|
||||
expect(timer.mock.calls.map(([, delay]) => delay)).toEqual(Array(40).fill(50));
|
||||
expect(output.hookSpecificOutput.permissionDecision).toBe('deny');
|
||||
expect(output.hookSpecificOutput.permissionDecisionReason).toContain('no missing-publication conclusion');
|
||||
} finally { timer.mockRestore(); }
|
||||
}));
|
||||
});
|
||||
test.serial('the publication clock restores native timers after success and failure', async () => {
|
||||
const timeout = globalThis.setTimeout, immediate = globalThis.setImmediate, now = performance.now;
|
||||
const failure = new Error('fixture failure');
|
||||
const project = process.env.CLAUDE_PROJECT_DIR;
|
||||
for (const outcome of ['success', 'throw', 'over-budget']) {
|
||||
const pending = withNativeProjectDirectory(undefined, () => withPublicationClock(async () => {
|
||||
await new Promise(resolve => setTimeout(resolve, outcome === 'over-budget' ? 2_050 : 50));
|
||||
if (outcome === 'throw') throw failure;
|
||||
return 'settled';
|
||||
}));
|
||||
const result = await pending.then(value => value, error => error);
|
||||
if (outcome === 'throw') expect(result).toBe(failure);
|
||||
else if (outcome === 'over-budget') {
|
||||
expect(result).toBeInstanceOf(Error);
|
||||
expect(result.message).toContain('did not settle within its 2000ms polling budget');
|
||||
} else expect(result).toBe('settled');
|
||||
expect(jest.isFakeTimers()).toBe(false);
|
||||
expect(globalThis.setTimeout).toBe(timeout);
|
||||
expect(globalThis.setImmediate).toBe(immediate);
|
||||
expect(performance.now).toBe(now);
|
||||
expect(process.env.CLAUDE_PROJECT_DIR).toBe(project);
|
||||
}
|
||||
});
|
||||
test.serial('the owned journal can arrive asynchronously on the last poll before the deadline', async () => {
|
||||
const f = fixture(); f.message(); f.current();
|
||||
await withNativeProjectDirectory(f.cwd, () => withPublicationClock(async () => {
|
||||
const started = performance.now();
|
||||
setTimeout(() => f.journal(), 1_950);
|
||||
expect(await runPublicationHook(f.input, ROOT)).toEqual({});
|
||||
expect(performance.now() - started).toBe(1_950);
|
||||
const decoded = readOwnedClaudePublicTranscript(f.input.transcript_path, f.cwd, f.sessionId);
|
||||
expect(decoded.transcript.status).toBe('ready');
|
||||
expect(decoded.events.some(e => e.kind === 'use' && e.toolUseId === 'next')).toBe(true);
|
||||
}));
|
||||
});
|
||||
test('the actual owned native reader and asynchronous hook admit a flushed same-response report', async () => {
|
||||
const f = fixture(); f.message(); f.current();
|
||||
setTimeout(() => f.journal(), 100);
|
||||
@@ -651,7 +719,7 @@ describe('Autoplan parent publication guard', () => {
|
||||
for (const kind of ['initial-entry', 'new-phase', 'agent', 'foreign-methodology', 'duplicate',
|
||||
'foreign-session', 'orphan-current-result', 'pending-prior-entry', 'unpublished-predecessor', 'rearmed-human',
|
||||
'forged-prior-range', 'malformed-journal', 'symlinked-journal'] as const)
|
||||
test(`an in-flight native Read does not bypass ${kind}`, async () => {
|
||||
test.serial(`an in-flight native Read does not bypass ${kind}`, async () => {
|
||||
const f = fixture(); f.input.tool_input = { file_path: f.method, offset: 1, limit: 1 };
|
||||
if (kind === 'initial-entry') f.events.splice(2);
|
||||
if (kind === 'new-phase') { f.message(); f.input.tool_input = { file_path: path.join(ROOT, 'autoplan/sections/design-phase.md') }; }
|
||||
@@ -679,7 +747,7 @@ describe('Autoplan parent publication guard', () => {
|
||||
fs.renameSync(f.input.transcript_path, target); fs.symlinkSync(target, f.input.transcript_path);
|
||||
}
|
||||
if (kind === 'foreign-session') f.input.session_id = randomUUID();
|
||||
const output: any = await withNativeProjectDirectory(f.cwd, () => runPublicationHook(f.input, ROOT));
|
||||
const output: any = await withNativeProjectDirectory(f.cwd, () => withPublicationClock(() => runPublicationHook(f.input, ROOT)));
|
||||
expect(output.hookSpecificOutput?.permissionDecision).toBe('deny');
|
||||
expect(readOwnedClaudePublicTranscript(f.input.transcript_path, f.cwd, f.sessionId).events
|
||||
.filter(e => e.kind === 'use' && e.toolUseId === f.input.tool_use_id)).toHaveLength(0);
|
||||
|
||||
@@ -61,7 +61,8 @@ describe('content-binding template drift', () => {
|
||||
const land = rendered('land-and-deploy/sections/readiness-gate.md');
|
||||
expect(land).toContain('wtree');
|
||||
expect(land).toContain('---WTREE---');
|
||||
expect(land).toMatch(/gstack-evidence check --label tests --expect-cmd '[^']+' --max-age 24/);
|
||||
expect(land).toContain('gstack-evidence check --label tests --expect-cmd "$TEST_COMMAND" --max-age 24');
|
||||
expect(land).toContain('gstack-evidence run --label tests -- "$TEST_COMMAND"');
|
||||
expect(land).toContain('UNKNOWN');
|
||||
});
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { afterAll, afterEach, describe, expect, spyOn, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';import * as os from 'node:os';import * as path from 'node:path';import { spawn, spawnSync } from 'node:child_process';
|
||||
import { assertSnapshot, capture } from '../lib/cso/snapshot';import { CsoError } from '../lib/cso/contracts';import { runProcess, sanitizeForJson, sanitizeHelperForJson } from '../lib/cso/process';import { discardAtomicNoReplaceTemp, finalizeReplayTemporary, loadReport, newRun, privateRoot, readJson, recoverAtomicNoReplaceJson, retention, saveReport, secureDirectory, stateRoot, withLock, writeHelperJson, writeJson } from '../lib/cso/state';
|
||||
import { assertSnapshot, capture } from '../lib/cso/snapshot';import { CsoError, snapshotPathHandle } from '../lib/cso/contracts';import { runProcess, sanitizeForJson, sanitizeHelperForJson } from '../lib/cso/process';import { discardAtomicNoReplaceTemp, finalizeReplayTemporary, loadReport, newRun, privateRoot, readJson, recoverAtomicNoReplaceJson, retention, saveReport, secureDirectory, stateRoot, withLock, writeHelperJson, writeJson } from '../lib/cso/state';
|
||||
const roots:string[]=[];const tmp=()=>{const p=fs.mkdtempSync(path.join(os.tmpdir(),'cso-snapshot-'));roots.push(p);return p;};afterEach(()=>{for(const p of roots.splice(0))fs.rmSync(p,{recursive:true,force:true});});
|
||||
const originalState=process.env.GSTACK_HOME,state=fs.mkdtempSync(path.join(os.tmpdir(),'cso-state-'));process.env.GSTACK_HOME=state;afterAll(()=>{if(originalState===undefined)delete process.env.GSTACK_HOME;else process.env.GSTACK_HOME=originalState;fs.rmSync(state,{recursive:true,force:true});});
|
||||
function git(repo:string,...args:string[]){const r=spawnSync('/usr/bin/git',['-C',repo,...args],{encoding:'utf8',env:{PATH:'/usr/bin:/bin',HOME:repo},timeout:30_000});if(r.status)throw new Error(r.stderr);return r.stdout;}
|
||||
@@ -24,7 +24,25 @@ describe('CSO dirty snapshot boundary',()=>{
|
||||
test('does not invoke configured clean filters, hooks, fsmonitor, or PATH shims',async()=>{const p=repo(),marker=path.join(p,'marker');fs.writeFileSync(path.join(p,'.gitattributes'),'*.txt filter=hostile\n');git(p,'config','filter.hostile.clean',`/bin/sh -c 'touch ${marker}; cat'`);git(p,'config','core.fsmonitor',`/bin/sh -c 'touch ${marker}'`);const fake=path.join(p,'bin');fs.mkdirSync(fake);fs.writeFileSync(path.join(fake,'git'),`#!/bin/sh\ntouch '${marker}'\nexit 99\n`,{mode:0o755});const old=process.env.PATH;process.env.PATH=`${fake}:${old}`;try{await capture(p,newRun(p).dir,'HEAD');}finally{process.env.PATH=old;}expect(fs.existsSync(marker)).toBe(false);});
|
||||
test('rejects symlinks and hard links as execution inputs',async()=>{const p=repo();fs.symlinkSync('/etc/passwd',path.join(p,'escape'));expect(capture(p,newRun(p).dir)).rejects.toThrow();fs.unlinkSync(path.join(p,'escape'));fs.linkSync(path.join(p,'tracked.txt'),path.join(p,'hard'));expect(capture(p,newRun(p).dir)).rejects.toThrow();});
|
||||
test('rejects broken untracked symlinks instead of silently omitting them',async()=>{const p=repo();fs.symlinkSync('missing-target',path.join(p,'broken'));expect(capture(p,newRun(p).dir)).rejects.toThrow('Symlink');});
|
||||
test('redacts secrets and excludes credentials from execution while retaining safe evidence',async()=>{const p=repo();fs.writeFileSync(path.join(p,'.env'),'TOKEN='+['ghp_','abcdefghijklmnopqrstuvwxyz1234567890'].join('')+'\n');const run=newRun(p),m=await capture(p,run.dir);expect(m.entries.find(e=>e.path==='.env')?.transformation).toContain('excluded');expect(fs.existsSync(path.join(run.dir,'snapshot','.env'))).toBe(false);expect(fs.readFileSync(path.join(run.dir,'readable','.env'),'utf8')).toBe('TOKEN=<REDACTED-env.kv+github.pat>\n');const evidence=JSON.parse(fs.readFileSync(path.join(run.dir,'sensitive-evidence.json'),'utf8'));expect(evidence[0].findings.map((x:any)=>x.id)).toContain('github.pat');expect(JSON.stringify(evidence)).not.toContain('ghp_');});
|
||||
for(const earlierSensitive of [false,true])test(earlierSensitive?'keeps credential evidence bound to .env when an earlier file is also sensitive':'redacts secrets and excludes credentials from execution while retaining safe evidence',async()=>{
|
||||
const p=repo();
|
||||
if(earlierSensitive)fs.writeFileSync(path.join(p,'.earlier.txt'),'Source: /home/fixture/project/source.ts\n');
|
||||
fs.writeFileSync(path.join(p,'.env'),'TOKEN='+['ghp_','abcdefghijklmnopqrstuvwxyz1234567890'].join('')+'\n');
|
||||
const run=newRun(p),m=await capture(p,run.dir),entry=m.entries.find(e=>e.path==='.env');
|
||||
expect(entry?.transformation).toContain('excluded');
|
||||
expect(fs.existsSync(path.join(run.dir,'snapshot','.env'))).toBe(false);
|
||||
expect(fs.readFileSync(path.join(run.dir,'readable','.env'),'utf8')).toBe('TOKEN=<REDACTED-env.kv+github.pat>\n');
|
||||
const evidence:Array<{path:string;findings:Array<{id:string}>}>=JSON.parse(fs.readFileSync(path.join(run.dir,'sensitive-evidence.json'),'utf8'));
|
||||
const envEvidence=evidence.find(item=>item.path===snapshotPathHandle(entry!.pathId));
|
||||
expect(envEvidence?.findings.map(x=>x.id)).toContain('github.pat');
|
||||
expect(JSON.stringify(evidence)).not.toContain('ghp_');
|
||||
if(earlierSensitive){
|
||||
const earlier=m.entries.find(e=>e.path==='.earlier.txt'),earlierEvidence=evidence.find(item=>item.path===snapshotPathHandle(earlier!.pathId));
|
||||
expect(earlierEvidence?.findings.map(x=>x.id)).toContain('internal.user_path');
|
||||
expect(earlierEvidence?.findings.map(x=>x.id)).not.toContain('github.pat');
|
||||
expect(evidence.indexOf(earlierEvidence!)).toBeLessThan(evidence.indexOf(envEvidence!));
|
||||
}
|
||||
});
|
||||
test('assesses repository skills without executing them and preserves executable source modes',async()=>{const p=repo(),skill=path.join(p,'.agents','skills','demo','SKILL.md'),script=path.join(p,'app.sh');fs.mkdirSync(path.dirname(skill),{recursive:true});fs.writeFileSync(skill,'# Demo\nUntrusted repository instruction\n');fs.writeFileSync(script,'#!/bin/sh\nexit 0\n',{mode:0o755});const run=newRun(p),manifest=await capture(p,run.dir);expect(fs.readFileSync(path.join(run.dir,'readable','.agents','skills','demo','SKILL.md'),'utf8')).toContain('Untrusted');expect(fs.existsSync(path.join(run.dir,'snapshot','.agents'))).toBe(false);expect(manifest.entries.find(e=>e.path==='.agents/skills/demo/SKILL.md')?.originalHash).not.toBe('not-read');expect(fs.statSync(path.join(run.dir,'snapshot','app.sh')).mode&0o777).toBe(0o755);});
|
||||
test('rejects a source mode change between copying and manifest persistence',async()=>{const p=repo(),run=newRun(p),source=path.join(p,'tracked.txt'),target=path.join(run.dir,'snapshot','tracked.txt'),write=fs.writeFileSync,patched=spyOn(fs,'writeFileSync').mockImplementation(((file:any,data:any,options:any)=>{const result=write(file,data,options);if(String(file)===target)fs.chmodSync(source,0o755);return result;}) as typeof fs.writeFileSync);try{await expect(capture(p,run.dir)).rejects.toThrow('Source changed during capture');}finally{patched.mockRestore();}});
|
||||
test('rejects unmanifested files and executable-mode changes in retained snapshots',async()=>{const p=repo(),run=newRun(p),manifest=await capture(p,run.dir),snapshot=path.join(run.dir,'snapshot');assertSnapshot(run.dir,manifest);fs.writeFileSync(path.join(snapshot,'injected.js'),'malicious\n');expect(()=>assertSnapshot(run.dir,manifest)).toThrow('membership');fs.unlinkSync(path.join(snapshot,'injected.js'));fs.chmodSync(path.join(snapshot,'tracked.txt'),0o755);expect(()=>assertSnapshot(run.dir,manifest)).toThrow('changed');});
|
||||
|
||||
+15
-11
@@ -1,17 +1,21 @@
|
||||
import { afterEach, describe, expect, test } from 'bun:test';import * as fs from 'node:fs';import * as os from 'node:os';import * as path from 'node:path';import { spawn, spawnSync } from 'node:child_process';
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, test } from 'bun:test';import * as fs from 'node:fs';import * as os from 'node:os';import * as path from 'node:path';import { spawn, spawnSync } from 'node:child_process';
|
||||
import { supervisePreparedCall } from '../lib/cso/preparation-docker';
|
||||
const dirs:string[]=[];const tmp=()=>{const p=fs.mkdtempSync(path.join(os.tmpdir(),'csowatchdog-'));dirs.push(p);return p;};afterEach(()=>{for(const p of dirs.splice(0))fs.rmSync(p,{recursive:true,force:true});});
|
||||
function compile(dir:string){const out=path.join(dir,'watchdog'),r=spawnSync('/usr/bin/cc',['-std=c11','-D_POSIX_C_SOURCE=200809L','-O2','-Wall','-Wextra',path.resolve(import.meta.dir,'../lib/cso/watchdog.c'),'-o',out],{encoding:'utf8',timeout:30_000});expect(r.status).toBe(0);expect(r.stderr).toBe('');return out;}
|
||||
function pinnedEndpoint(dir:string){const socket=path.join(dir,'daemon.sock'),made=spawnSync('/usr/bin/python3',['-c','import socket,sys; s=socket.socket(socket.AF_UNIX); s.bind(sys.argv[1]); s.close()',socket],{encoding:'utf8'});expect(made.status).toBe(0);const stat=fs.lstatSync(socket);expect(stat.isSocket()).toBe(true);return{uri:`unix://${socket}`,device:stat.dev,inode:stat.ino};}
|
||||
describe('detached CSO watchdog',()=>{
|
||||
test('prepared-call guard acknowledges normal exact cleanup',async()=>{const run=tmp(),watchdog=compile(run),call=path.join(run,'preparation-execution','offline-call'),control=path.join(run,'supervision','prepared-call');fs.mkdirSync(call,{recursive:true});fs.mkdirSync(control,{recursive:true});fs.writeFileSync(path.join(call,'prepared-source'),'copy');const guard=await supervisePreparedCall({watchdogPath:watchdog,ownerPid:process.pid,deadline:Date.now()+10_000,runRoot:run,callRoot:call,controlRoot:control});await guard.dispose();expect(fs.existsSync(call)).toBe(false);expect(fs.existsSync(control)).toBe(false);});
|
||||
test('prepared-call guard removes retained output after owner death and deadline',async()=>{for(const mode of ['owner','deadline'] as const){const run=tmp(),watchdog=compile(run),call=path.join(run,'preparation-execution',mode),control=path.join(run,'supervision',mode),event=path.join(control,'attempt.event');fs.mkdirSync(call,{recursive:true});fs.mkdirSync(control,{recursive:true});fs.writeFileSync(path.join(call,'prepared-source'),'copy');const owner=mode==='owner'?spawn('/bin/sleep',['30'],{stdio:'ignore'}):undefined,guard=await supervisePreparedCall({watchdogPath:watchdog,ownerPid:owner?.pid??process.pid,deadline:Date.now()+(mode==='owner'?10_000:50),runRoot:run,callRoot:call,controlRoot:control});if(owner)owner.kill('SIGKILL');for(let attempt=0;attempt<40&&(!fs.existsSync(event)||fs.existsSync(call));attempt++)await Bun.sleep(100);expect(fs.existsSync(call)).toBe(false);expect(fs.readFileSync(event,'utf8')).toContain(`${mode==='owner'?'supervisor-death':'deadline'} execution-copy cleanup complete`);await guard.dispose();expect(fs.existsSync(control)).toBe(false);}});
|
||||
test('survives supervisor death and cleans journaled plus label-race resources',async()=>{const dir=tmp(),watchdog=compile(dir),endpoint=pinnedEndpoint(dir),log=path.join(dir,'docker.log'),docker=path.join(dir,'docker'),id='a'.repeat(64),race='e'.repeat(64),lease=path.join(dir,'lease'),token='c'.repeat(32);fs.mkdirSync(lease);fs.writeFileSync(path.join(lease,'lease.json'),'{}');fs.writeFileSync(path.join(lease,'lease.token'),token+'\n');fs.writeFileSync(docker,`#!/bin/sh\nif [ "$3" = ps ]; then if [ "$8" = "id=${id}" ]; then printf '%s\\n' '${id}'; else printf '%s\\n' '${race}'; fi; exit 0; fi\nprintf '%s\\n' "$*" >> '${log}'\n`,{mode:0o755});fs.writeFileSync(path.join(dir,'resources.journal'),`container:${id}\n`,{mode:0o600});const owner=spawn('/bin/sleep',['1'],{stdio:'ignore'}),child=spawn(watchdog,['--owner',String(owner.pid),'--deadline',String(Math.ceil((Date.now()+10_000)/1000)),'--run-dir',dir,'--docker',docker,'--endpoint',endpoint.uri,'--socket-device',String(endpoint.device),'--socket-inode',String(endpoint.inode),'--run-label','test-label','--lease-path',lease,'--lease-token',token],{stdio:'ignore'}),closed=new Promise<number|null>(resolve=>child.on('close',resolve));expect(await closed).toBe(0);const calls=fs.readFileSync(log,'utf8');expect(calls).toContain(`rm --force --volumes ${id}`);expect(calls).toContain(`rm --force --volumes ${race}`);expect(fs.readFileSync(path.join(dir,'watchdog.event'),'utf8')).toContain('supervisor-death cleanup complete');expect(fs.existsSync(lease)).toBe(false);});
|
||||
test('quarantines an authenticated lease before a concurrent TS recovery claim can strand release',async()=>{const dir=tmp(),watchdog=compile(dir),endpoint=pinnedEndpoint(dir),docker=path.join(dir,'docker'),lease=path.join(dir,'lease'),token='1'.repeat(32),claim='2'.repeat(32);fs.mkdirSync(lease,{mode:0o700});fs.writeFileSync(path.join(lease,'lease.json'),JSON.stringify({token}),{mode:0o600});fs.writeFileSync(path.join(lease,'lease.token'),token+'\n',{mode:0o600});fs.writeFileSync(path.join(lease,'.recovery'),JSON.stringify({pid:process.pid,processIdentity:null,token:claim,createdAt:Date.now()})+'\n',{mode:0o600});fs.writeFileSync(docker,'#!/bin/sh\nif [ "$3" = ps ]; then exit 0; fi\nexit 0\n',{mode:0o755});const owner=spawn('/bin/sleep',['0.05'],{stdio:'ignore'}),child=spawn(watchdog,['--owner',String(owner.pid),'--deadline',String(Math.ceil((Date.now()+10_000)/1000)),'--run-dir',dir,'--docker',docker,'--endpoint',endpoint.uri,'--socket-device',String(endpoint.device),'--socket-inode',String(endpoint.inode),'--run-label','release-race','--lease-path',lease,'--lease-token',token],{stdio:'ignore'}),closed=new Promise<number|null>(resolve=>child.on('close',resolve));const outcome=await Promise.race([closed,Bun.sleep(5000).then(()=>-999)]);if(outcome===-999)child.kill('SIGKILL');expect(outcome).toBe(0);expect(fs.existsSync(lease)).toBe(false);expect(fs.readdirSync(dir).some(name=>name.includes('.watchdog-release-'))).toBe(false);expect(fs.readFileSync(path.join(dir,'watchdog.event'),'utf8')).toContain('cleanup complete');});
|
||||
test('attempt mode removes a patched execution copy after supervisor death',async()=>{const run=tmp(),watchdog=compile(run),control=path.join(run,'supervision','attempt'),work=path.join(run,'verification','attempt');fs.mkdirSync(control,{recursive:true});fs.mkdirSync(work,{recursive:true});fs.writeFileSync(path.join(work,'patched-source'),'sensitive execution copy');const owner=spawn('/bin/sleep',['1'],{stdio:'ignore'}),child=spawn(watchdog,['--attempt-owner',String(owner.pid),'--deadline',String(Math.ceil((Date.now()+10_000)/1000)),'--control-dir',control,'--work-root',work,'--run-root',run],{stdio:'ignore'}),closed=new Promise<number|null>(resolve=>child.on('close',resolve));expect(await closed).toBe(0);expect(fs.existsSync(work)).toBe(false);expect(fs.readFileSync(path.join(control,'attempt.event'),'utf8')).toContain('execution-copy cleanup complete');});
|
||||
test('attempt cleanup cannot remove the concurrent Docker watchdog journal',async()=>{const run=tmp(),watchdog=compile(run),endpoint=pinnedEndpoint(run),attemptControl=path.join(run,'supervision','attempt'),dockerControl=path.join(attemptControl,'docker-groups','before'),work=path.join(run,'verification','attempt'),lease=path.join(run,'lease'),log=path.join(dockerControl,'docker.log'),docker=path.join(dockerControl,'docker'),id='a'.repeat(64),race='e'.repeat(64),token='c'.repeat(32);for(const dir of [attemptControl,dockerControl,work,lease])fs.mkdirSync(dir,{recursive:true});fs.writeFileSync(path.join(work,'patched-source'),'sensitive execution copy');fs.writeFileSync(path.join(lease,'lease.json'),'{}');fs.writeFileSync(path.join(lease,'lease.token'),token+'\n');fs.writeFileSync(path.join(dockerControl,'resources.journal'),`container:${id}\n`,{mode:0o600});fs.writeFileSync(docker,`#!/bin/sh\nif [ "$3" = ps ]; then if [ "$8" = "id=${id}" ]; then printf '%s\\n' '${id}'; else printf '%s\\n' '${race}'; fi; exit 0; fi\nprintf '%s\\n' "$*" >> '${log}'\n`,{mode:0o755});const owner=spawn('/bin/sleep',['1'],{stdio:'ignore'}),attempt=spawn(watchdog,['--attempt-owner',String(owner.pid),'--deadline',String(Math.ceil((Date.now()+10_000)/1000)),'--control-dir',attemptControl,'--work-root',work,'--run-root',run],{stdio:'ignore'}),containers=spawn(watchdog,['--owner',String(owner.pid),'--deadline',String(Math.ceil((Date.now()+10_000)/1000)),'--run-dir',dockerControl,'--docker',docker,'--endpoint',endpoint.uri,'--socket-device',String(endpoint.device),'--socket-inode',String(endpoint.inode),'--run-label','test-label','--lease-path',lease,'--lease-token',token],{stdio:'ignore'});const closed=(child:any)=>new Promise<number|null>(resolve=>child.on('close',resolve));expect(await Promise.all([closed(attempt),closed(containers)])).toEqual([0,0]);expect(fs.existsSync(work)).toBe(false);expect(fs.readFileSync(log,'utf8')).toContain(`rm --force --volumes ${id}`);expect(fs.existsSync(lease)).toBe(false);expect(fs.readFileSync(path.join(dockerControl,'watchdog.event'),'utf8')).toContain('cleanup complete');});
|
||||
test('a terminal marker prevents any cleanup call',async()=>{const dir=tmp(),watchdog=compile(dir),endpoint=pinnedEndpoint(dir),log=path.join(dir,'docker.log'),docker=path.join(dir,'docker'),lease=path.join(dir,'lease'),token='d'.repeat(32);fs.mkdirSync(lease);fs.writeFileSync(path.join(lease,'lease.json'),'{}');fs.writeFileSync(path.join(lease,'lease.token'),token+'\n');fs.writeFileSync(docker,`#!/bin/sh\ntouch '${log}'\n`,{mode:0o755});fs.writeFileSync(path.join(dir,'resources.journal'),`container:${'b'.repeat(64)}\n`);fs.writeFileSync(path.join(dir,'watchdog.terminal'),'done\n');const r=spawnSync(watchdog,['--owner',String(process.pid),'--deadline',String(Math.ceil((Date.now()+10_000)/1000)),'--run-dir',dir,'--docker',docker,'--endpoint',endpoint.uri,'--socket-device',String(endpoint.device),'--socket-inode',String(endpoint.inode),'--run-label','test-label','--lease-path',lease,'--lease-token',token],{timeout:30_000});expect(r.status).toBe(0);expect(fs.existsSync(log)).toBe(false);});
|
||||
test('acknowledges terminal cleanup well inside the caller deadline',async()=>{const dir=tmp(),watchdog=compile(dir),endpoint=pinnedEndpoint(dir),docker=path.join(dir,'docker'),lease=path.join(dir,'lease'),token='9'.repeat(32),ready=path.join(dir,'watchdog.ready'),stopped=path.join(dir,'watchdog.stopped');fs.mkdirSync(lease);fs.writeFileSync(path.join(lease,'lease.json'),'{}');fs.writeFileSync(path.join(lease,'lease.token'),token+'\n');fs.writeFileSync(docker,'#!/bin/sh\nexit 0\n',{mode:0o755});const child=spawn(watchdog,['--owner',String(process.pid),'--deadline',String(Math.ceil((Date.now()+10_000)/1000)),'--run-dir',dir,'--docker',docker,'--endpoint',endpoint.uri,'--socket-device',String(endpoint.device),'--socket-inode',String(endpoint.inode),'--run-label','ack-latency','--lease-path',lease,'--lease-token',token],{stdio:'ignore'}),closed=new Promise<number|null>(resolve=>child.once('close',resolve));for(let i=0;i<100&&!fs.existsSync(ready);i++)await Bun.sleep(10);expect(fs.existsSync(ready)).toBe(true);await Bun.sleep(50);const started=Date.now();fs.writeFileSync(path.join(dir,'watchdog.terminal'),'done\n');for(let i=0;i<100&&!fs.existsSync(stopped);i++)await Bun.sleep(10);expect(fs.existsSync(stopped)).toBe(true);expect(Date.now()-started).toBeLessThan(500);expect(await closed).toBe(0);});
|
||||
test('a torn final journal row cannot retain the machine lease after exact label sweeps',async()=>{const dir=tmp(),watchdog=compile(dir),endpoint=pinnedEndpoint(dir),docker=path.join(dir,'docker'),lease=path.join(dir,'lease'),token='e'.repeat(32),id='a'.repeat(64);fs.mkdirSync(lease);fs.writeFileSync(path.join(lease,'lease.json'),'{}');fs.writeFileSync(path.join(lease,'lease.token'),token+'\n');fs.writeFileSync(docker,'#!/bin/sh\nif [ "$3" = ps ]; then exit 0; fi\nexit 0\n',{mode:0o755});fs.writeFileSync(path.join(dir,'resources.journal'),`container:${id}\ncontainer:abcd`,{mode:0o600});const owner=spawn('/bin/sleep',['1'],{stdio:'ignore'}),child=spawn(watchdog,['--owner',String(owner.pid),'--deadline',String(Math.ceil((Date.now()+10_000)/1000)),'--run-dir',dir,'--docker',docker,'--endpoint',endpoint.uri,'--socket-device',String(endpoint.device),'--socket-inode',String(endpoint.inode),'--run-label','test-label','--lease-path',lease,'--lease-token',token],{stdio:'ignore'});expect(await new Promise<number|null>(resolve=>child.on('close',resolve))).toBe(0);expect(fs.existsSync(lease)).toBe(false);expect(fs.readFileSync(path.join(dir,'watchdog.event'),'utf8')).toContain('malformed journal ignored after two exact label sweeps');});
|
||||
test('a replaced Docker socket blocks cleanup and lease release',async()=>{const dir=tmp(),watchdog=compile(dir),endpoint=pinnedEndpoint(dir),docker=path.join(dir,'docker'),lease=path.join(dir,'lease'),token='f'.repeat(32),log=path.join(dir,'docker.log');fs.mkdirSync(lease);fs.writeFileSync(path.join(lease,'lease.json'),'{}');fs.writeFileSync(path.join(lease,'lease.token'),token+'\n');fs.writeFileSync(docker,`#!/bin/sh\ntouch '${log}'\nexit 0\n`,{mode:0o755});const owner=spawn('/bin/sleep',['30'],{stdio:'ignore'}),child=spawn(watchdog,['--owner',String(owner.pid),'--deadline',String(Math.ceil((Date.now()+30_000)/1000)),'--run-dir',dir,'--docker',docker,'--endpoint',endpoint.uri,'--socket-device',String(endpoint.device),'--socket-inode',String(endpoint.inode),'--run-label','test-label','--lease-path',lease,'--lease-token',token],{stdio:'ignore'});for(let i=0;i<100&&!fs.existsSync(path.join(dir,'watchdog.ready'));i++)await Bun.sleep(10);expect(fs.existsSync(path.join(dir,'watchdog.ready'))).toBe(true);const socketPath=endpoint.uri.slice('unix://'.length);fs.renameSync(socketPath,`${socketPath}.old`);const replacement=pinnedEndpoint(dir);expect(replacement.inode).not.toBe(endpoint.inode);owner.kill('SIGKILL');for(let i=0;i<50&&!fs.existsSync(path.join(dir,'watchdog.event'));i++)await Bun.sleep(100);expect(fs.readFileSync(path.join(dir,'watchdog.event'),'utf8')).toContain('socket identity changed');expect(fs.existsSync(lease)).toBe(true);expect(fs.existsSync(log)).toBe(false);child.kill('SIGKILL');await new Promise(resolve=>child.once('close',resolve));});
|
||||
let seedDir:string;
|
||||
beforeAll(()=>{seedDir=fs.mkdtempSync(path.join(os.tmpdir(),'csowatchdog-seed-'));compile(seedDir);});
|
||||
afterAll(()=>{if(seedDir)fs.rmSync(seedDir,{recursive:true,force:true});});
|
||||
function copyWatchdog(dir:string){const out=path.join(dir,'watchdog');fs.copyFileSync(path.join(seedDir,'watchdog'),out);return out;}
|
||||
test('prepared-call guard acknowledges normal exact cleanup',async()=>{const run=tmp(),watchdog=copyWatchdog(run),call=path.join(run,'preparation-execution','offline-call'),control=path.join(run,'supervision','prepared-call');fs.mkdirSync(call,{recursive:true});fs.mkdirSync(control,{recursive:true});fs.writeFileSync(path.join(call,'prepared-source'),'copy');const guard=await supervisePreparedCall({watchdogPath:watchdog,ownerPid:process.pid,deadline:Date.now()+10_000,runRoot:run,callRoot:call,controlRoot:control});await guard.dispose();expect(fs.existsSync(call)).toBe(false);expect(fs.existsSync(control)).toBe(false);});
|
||||
test('prepared-call guard removes retained output after owner death and deadline',async()=>{for(const mode of ['owner','deadline'] as const){const run=tmp(),watchdog=copyWatchdog(run),call=path.join(run,'preparation-execution',mode),control=path.join(run,'supervision',mode),event=path.join(control,'attempt.event');fs.mkdirSync(call,{recursive:true});fs.mkdirSync(control,{recursive:true});fs.writeFileSync(path.join(call,'prepared-source'),'copy');const owner=mode==='owner'?spawn('/bin/sleep',['30'],{stdio:'ignore'}):undefined,guard=await supervisePreparedCall({watchdogPath:watchdog,ownerPid:owner?.pid??process.pid,deadline:Date.now()+(mode==='owner'?10_000:50),runRoot:run,callRoot:call,controlRoot:control});if(owner)owner.kill('SIGKILL');for(let attempt=0;attempt<40&&(!fs.existsSync(event)||fs.existsSync(call));attempt++)await Bun.sleep(100);expect(fs.existsSync(call)).toBe(false);expect(fs.readFileSync(event,'utf8')).toContain(`${mode==='owner'?'supervisor-death':'deadline'} execution-copy cleanup complete`);await guard.dispose();expect(fs.existsSync(control)).toBe(false);}});
|
||||
test('survives supervisor death and cleans journaled plus label-race resources',async()=>{const dir=tmp(),watchdog=copyWatchdog(dir),endpoint=pinnedEndpoint(dir),log=path.join(dir,'docker.log'),docker=path.join(dir,'docker'),id='a'.repeat(64),race='e'.repeat(64),lease=path.join(dir,'lease'),token='c'.repeat(32);fs.mkdirSync(lease);fs.writeFileSync(path.join(lease,'lease.json'),'{}');fs.writeFileSync(path.join(lease,'lease.token'),token+'\n');fs.writeFileSync(docker,`#!/bin/sh\nif [ "$3" = ps ]; then if [ "$8" = "id=${id}" ]; then printf '%s\\n' '${id}'; else printf '%s\\n' '${race}'; fi; exit 0; fi\nprintf '%s\\n' "$*" >> '${log}'\n`,{mode:0o755});fs.writeFileSync(path.join(dir,'resources.journal'),`container:${id}\n`,{mode:0o600});const owner=spawn('/bin/sleep',['1'],{stdio:'ignore'}),child=spawn(watchdog,['--owner',String(owner.pid),'--deadline',String(Math.ceil((Date.now()+10_000)/1000)),'--run-dir',dir,'--docker',docker,'--endpoint',endpoint.uri,'--socket-device',String(endpoint.device),'--socket-inode',String(endpoint.inode),'--run-label','test-label','--lease-path',lease,'--lease-token',token],{stdio:'ignore'}),closed=new Promise<number|null>(resolve=>child.on('close',resolve));expect(await closed).toBe(0);const calls=fs.readFileSync(log,'utf8');expect(calls).toContain(`rm --force --volumes ${id}`);expect(calls).toContain(`rm --force --volumes ${race}`);expect(fs.readFileSync(path.join(dir,'watchdog.event'),'utf8')).toContain('supervisor-death cleanup complete');expect(fs.existsSync(lease)).toBe(false);});
|
||||
test('quarantines an authenticated lease before a concurrent TS recovery claim can strand release',async()=>{const dir=tmp(),watchdog=copyWatchdog(dir),endpoint=pinnedEndpoint(dir),docker=path.join(dir,'docker'),lease=path.join(dir,'lease'),token='1'.repeat(32),claim='2'.repeat(32);fs.mkdirSync(lease,{mode:0o700});fs.writeFileSync(path.join(lease,'lease.json'),JSON.stringify({token}),{mode:0o600});fs.writeFileSync(path.join(lease,'lease.token'),token+'\n',{mode:0o600});fs.writeFileSync(path.join(lease,'.recovery'),JSON.stringify({pid:process.pid,processIdentity:null,token:claim,createdAt:Date.now()})+'\n',{mode:0o600});fs.writeFileSync(docker,'#!/bin/sh\nif [ "$3" = ps ]; then exit 0; fi\nexit 0\n',{mode:0o755});const owner=spawn('/bin/sleep',['0.05'],{stdio:'ignore'}),child=spawn(watchdog,['--owner',String(owner.pid),'--deadline',String(Math.ceil((Date.now()+10_000)/1000)),'--run-dir',dir,'--docker',docker,'--endpoint',endpoint.uri,'--socket-device',String(endpoint.device),'--socket-inode',String(endpoint.inode),'--run-label','release-race','--lease-path',lease,'--lease-token',token],{stdio:'ignore'}),closed=new Promise<number|null>(resolve=>child.on('close',resolve));const outcome=await Promise.race([closed,Bun.sleep(5000).then(()=>-999)]);if(outcome===-999)child.kill('SIGKILL');expect(outcome).toBe(0);expect(fs.existsSync(lease)).toBe(false);expect(fs.readdirSync(dir).some(name=>name.includes('.watchdog-release-'))).toBe(false);expect(fs.readFileSync(path.join(dir,'watchdog.event'),'utf8')).toContain('cleanup complete');});
|
||||
test('attempt mode removes a patched execution copy after supervisor death',async()=>{const run=tmp(),watchdog=copyWatchdog(run),control=path.join(run,'supervision','attempt'),work=path.join(run,'verification','attempt');fs.mkdirSync(control,{recursive:true});fs.mkdirSync(work,{recursive:true});fs.writeFileSync(path.join(work,'patched-source'),'sensitive execution copy');const owner=spawn('/bin/sleep',['1'],{stdio:'ignore'}),child=spawn(watchdog,['--attempt-owner',String(owner.pid),'--deadline',String(Math.ceil((Date.now()+10_000)/1000)),'--control-dir',control,'--work-root',work,'--run-root',run],{stdio:'ignore'}),closed=new Promise<number|null>(resolve=>child.on('close',resolve));expect(await closed).toBe(0);expect(fs.existsSync(work)).toBe(false);expect(fs.readFileSync(path.join(control,'attempt.event'),'utf8')).toContain('execution-copy cleanup complete');});
|
||||
test('attempt cleanup cannot remove the concurrent Docker watchdog journal',async()=>{const run=tmp(),watchdog=copyWatchdog(run),endpoint=pinnedEndpoint(run),attemptControl=path.join(run,'supervision','attempt'),dockerControl=path.join(attemptControl,'docker-groups','before'),work=path.join(run,'verification','attempt'),lease=path.join(run,'lease'),log=path.join(dockerControl,'docker.log'),docker=path.join(dockerControl,'docker'),id='a'.repeat(64),race='e'.repeat(64),token='c'.repeat(32);for(const dir of [attemptControl,dockerControl,work,lease])fs.mkdirSync(dir,{recursive:true});fs.writeFileSync(path.join(work,'patched-source'),'sensitive execution copy');fs.writeFileSync(path.join(lease,'lease.json'),'{}');fs.writeFileSync(path.join(lease,'lease.token'),token+'\n');fs.writeFileSync(path.join(dockerControl,'resources.journal'),`container:${id}\n`,{mode:0o600});fs.writeFileSync(docker,`#!/bin/sh\nif [ "$3" = ps ]; then if [ "$8" = "id=${id}" ]; then printf '%s\\n' '${id}'; else printf '%s\\n' '${race}'; fi; exit 0; fi\nprintf '%s\\n' "$*" >> '${log}'\n`,{mode:0o755});const owner=spawn('/bin/sleep',['1'],{stdio:'ignore'}),attempt=spawn(watchdog,['--attempt-owner',String(owner.pid),'--deadline',String(Math.ceil((Date.now()+10_000)/1000)),'--control-dir',attemptControl,'--work-root',work,'--run-root',run],{stdio:'ignore'}),containers=spawn(watchdog,['--owner',String(owner.pid),'--deadline',String(Math.ceil((Date.now()+10_000)/1000)),'--run-dir',dockerControl,'--docker',docker,'--endpoint',endpoint.uri,'--socket-device',String(endpoint.device),'--socket-inode',String(endpoint.inode),'--run-label','test-label','--lease-path',lease,'--lease-token',token],{stdio:'ignore'});const closed=(child:any)=>new Promise<number|null>(resolve=>child.on('close',resolve));expect(await Promise.all([closed(attempt),closed(containers)])).toEqual([0,0]);expect(fs.existsSync(work)).toBe(false);expect(fs.readFileSync(log,'utf8')).toContain(`rm --force --volumes ${id}`);expect(fs.existsSync(lease)).toBe(false);expect(fs.readFileSync(path.join(dockerControl,'watchdog.event'),'utf8')).toContain('cleanup complete');});
|
||||
test('a terminal marker prevents any cleanup call',async()=>{const dir=tmp(),watchdog=copyWatchdog(dir),endpoint=pinnedEndpoint(dir),log=path.join(dir,'docker.log'),docker=path.join(dir,'docker'),lease=path.join(dir,'lease'),token='d'.repeat(32);fs.mkdirSync(lease);fs.writeFileSync(path.join(lease,'lease.json'),'{}');fs.writeFileSync(path.join(lease,'lease.token'),token+'\n');fs.writeFileSync(docker,`#!/bin/sh\ntouch '${log}'\n`,{mode:0o755});fs.writeFileSync(path.join(dir,'resources.journal'),`container:${'b'.repeat(64)}\n`);fs.writeFileSync(path.join(dir,'watchdog.terminal'),'done\n');const r=spawnSync(watchdog,['--owner',String(process.pid),'--deadline',String(Math.ceil((Date.now()+10_000)/1000)),'--run-dir',dir,'--docker',docker,'--endpoint',endpoint.uri,'--socket-device',String(endpoint.device),'--socket-inode',String(endpoint.inode),'--run-label','test-label','--lease-path',lease,'--lease-token',token],{timeout:30_000});expect(r.status).toBe(0);expect(fs.existsSync(log)).toBe(false);});
|
||||
test('acknowledges terminal cleanup well inside the caller deadline',async()=>{const dir=tmp(),watchdog=copyWatchdog(dir),endpoint=pinnedEndpoint(dir),docker=path.join(dir,'docker'),lease=path.join(dir,'lease'),token='9'.repeat(32),ready=path.join(dir,'watchdog.ready'),stopped=path.join(dir,'watchdog.stopped');fs.mkdirSync(lease);fs.writeFileSync(path.join(lease,'lease.json'),'{}');fs.writeFileSync(path.join(lease,'lease.token'),token+'\n');fs.writeFileSync(docker,'#!/bin/sh\nexit 0\n',{mode:0o755});const child=spawn(watchdog,['--owner',String(process.pid),'--deadline',String(Math.ceil((Date.now()+10_000)/1000)),'--run-dir',dir,'--docker',docker,'--endpoint',endpoint.uri,'--socket-device',String(endpoint.device),'--socket-inode',String(endpoint.inode),'--run-label','ack-latency','--lease-path',lease,'--lease-token',token],{stdio:'ignore'}),closed=new Promise<number|null>(resolve=>child.once('close',resolve));for(let i=0;i<100&&!fs.existsSync(ready);i++)await Bun.sleep(10);expect(fs.existsSync(ready)).toBe(true);await Bun.sleep(50);const started=Date.now();fs.writeFileSync(path.join(dir,'watchdog.terminal'),'done\n');for(let i=0;i<100&&!fs.existsSync(stopped);i++)await Bun.sleep(10);expect(fs.existsSync(stopped)).toBe(true);expect(Date.now()-started).toBeLessThan(500);expect(await closed).toBe(0);});
|
||||
test('a torn final journal row cannot retain the machine lease after exact label sweeps',async()=>{const dir=tmp(),watchdog=copyWatchdog(dir),endpoint=pinnedEndpoint(dir),docker=path.join(dir,'docker'),lease=path.join(dir,'lease'),token='e'.repeat(32),id='a'.repeat(64);fs.mkdirSync(lease);fs.writeFileSync(path.join(lease,'lease.json'),'{}');fs.writeFileSync(path.join(lease,'lease.token'),token+'\n');fs.writeFileSync(docker,'#!/bin/sh\nif [ "$3" = ps ]; then exit 0; fi\nexit 0\n',{mode:0o755});fs.writeFileSync(path.join(dir,'resources.journal'),`container:${id}\ncontainer:abcd`,{mode:0o600});const owner=spawn('/bin/sleep',['1'],{stdio:'ignore'}),child=spawn(watchdog,['--owner',String(owner.pid),'--deadline',String(Math.ceil((Date.now()+10_000)/1000)),'--run-dir',dir,'--docker',docker,'--endpoint',endpoint.uri,'--socket-device',String(endpoint.device),'--socket-inode',String(endpoint.inode),'--run-label','test-label','--lease-path',lease,'--lease-token',token],{stdio:'ignore'});expect(await new Promise<number|null>(resolve=>child.on('close',resolve))).toBe(0);expect(fs.existsSync(lease)).toBe(false);expect(fs.readFileSync(path.join(dir,'watchdog.event'),'utf8')).toContain('malformed journal ignored after two exact label sweeps');});
|
||||
test('a replaced Docker socket blocks cleanup and lease release',async()=>{const dir=tmp(),watchdog=copyWatchdog(dir),endpoint=pinnedEndpoint(dir),docker=path.join(dir,'docker'),lease=path.join(dir,'lease'),token='f'.repeat(32),log=path.join(dir,'docker.log');fs.mkdirSync(lease);fs.writeFileSync(path.join(lease,'lease.json'),'{}');fs.writeFileSync(path.join(lease,'lease.token'),token+'\n');fs.writeFileSync(docker,`#!/bin/sh\ntouch '${log}'\nexit 0\n`,{mode:0o755});const owner=spawn('/bin/sleep',['30'],{stdio:'ignore'}),child=spawn(watchdog,['--owner',String(owner.pid),'--deadline',String(Math.ceil((Date.now()+30_000)/1000)),'--run-dir',dir,'--docker',docker,'--endpoint',endpoint.uri,'--socket-device',String(endpoint.device),'--socket-inode',String(endpoint.inode),'--run-label','test-label','--lease-path',lease,'--lease-token',token],{stdio:'ignore'});for(let i=0;i<100&&!fs.existsSync(path.join(dir,'watchdog.ready'));i++)await Bun.sleep(10);expect(fs.existsSync(path.join(dir,'watchdog.ready'))).toBe(true);const socketPath=endpoint.uri.slice('unix://'.length);fs.renameSync(socketPath,`${socketPath}.old`);const replacement=pinnedEndpoint(dir);expect(replacement.inode).not.toBe(endpoint.inode);owner.kill('SIGKILL');for(let i=0;i<50&&!fs.existsSync(path.join(dir,'watchdog.event'));i++)await Bun.sleep(100);expect(fs.readFileSync(path.join(dir,'watchdog.event'),'utf8')).toContain('socket identity changed');expect(fs.existsSync(lease)).toBe(true);expect(fs.existsSync(log)).toBe(false);child.kill('SIGKILL');await new Promise(resolve=>child.once('close',resolve));});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,251 @@
|
||||
import { expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { installFakeImpeccable, DETECT_SAMPLE } from './helpers/fake-impeccable';
|
||||
import { sliceBetween } from './helpers/skill-fixture';
|
||||
import { recordE2E } from './helpers/e2e-helpers';
|
||||
import { resolveEvalModel } from '../lib/eval-model';
|
||||
import captured from './fixtures/design-detector-source-public.json';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const source = fs.readFileSync(path.join(ROOT, 'test/skill-e2e-design.test.ts'), 'utf8');
|
||||
const id = 'design-review-detector-shim';
|
||||
|
||||
type Mode = 'captured' | 'literal' | 'other-variable' | 'wrong-base' | 'no-scan' | 'wrong-files'
|
||||
| 'extra-file' | 'no-probe' | 'no-report' | 'no-finding' | 'no-rule' | 'npx' | 'browser'
|
||||
| 'timeout' | 'browse-error' | 'runner-throw' | 'report-read-throw'
|
||||
| 'quoted-path' | 'relative-path' | 'symlink-path' | 'claimed-probe' | 'wrong-probe-host' | 'wrong-probe-cwd';
|
||||
|
||||
async function exercise(modes: Mode[], retention?: 'directory' | 'run-id' | 'both' | 'write-error') {
|
||||
const scratch = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'detect-free-')));
|
||||
const home = path.join(scratch, 'home');
|
||||
const bin = path.join(scratch, 'bin');
|
||||
const artifactRoot = path.join(scratch, 'eval-artifacts');
|
||||
if (retention === 'write-error') fs.writeFileSync(artifactRoot, 'blocked');
|
||||
fs.mkdirSync(home); fs.mkdirSync(bin);
|
||||
fs.writeFileSync(path.join(bin, 'gh'), '#!/bin/sh\nexit 1\n', { mode: 0o755 });
|
||||
const env = { PATH: `${bin}${path.delimiter}${process.env.PATH}`, HOME: home,
|
||||
GSTACK_HOME: path.join(home, '.gstack'), TMPDIR: scratch,
|
||||
GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: '/dev/null',
|
||||
EVALS_RUN_ID: retention === 'run-id' || retention === 'both' ? 'same-run/../probe' : '',
|
||||
GSTACK_EVAL_DIR: retention && retention !== 'run-id' ? artifactRoot : '' };
|
||||
const callbacks: Array<{ name: string; fn: () => Promise<void>; timeout: number }> = [];
|
||||
const before: Array<() => void> = [], after: Array<() => void> = [];
|
||||
const rows: any[] = [], errors: unknown[] = [], outputs: string[] = [];
|
||||
const retained: Array<{ file: string; text: string; calls: any[] }> = [], notices: string[] = [];
|
||||
let attempt = 0, repo = '';
|
||||
const controlledError = new Error('controlled detector fixture failure');
|
||||
const owned = (p: string) => {
|
||||
expect(path.resolve(p).startsWith(scratch + path.sep)).toBe(true);
|
||||
let existing = p;
|
||||
while (!fs.existsSync(existing)) existing = path.dirname(existing);
|
||||
const real = fs.realpathSync(existing);
|
||||
expect(real === scratch || real.startsWith(scratch + path.sep)).toBe(true);
|
||||
};
|
||||
const localFs = { ...fs,
|
||||
mkdtempSync(prefix: string) { owned(prefix); return fs.mkdtempSync(prefix); },
|
||||
mkdirSync(p: string, opts: any) { owned(p); return fs.mkdirSync(p, opts); },
|
||||
chmodSync(p: string, mode: number) { owned(p); fs.chmodSync(p, mode); },
|
||||
writeFileSync(p: string, data: any, opts?: any) { owned(p); fs.writeFileSync(p, data, opts); },
|
||||
rmSync(p: string, opts: any) { owned(p); fs.rmSync(p, opts); },
|
||||
readFileSync(p: any, opts?: any) {
|
||||
if (p === path.join(repo, 'detector-output.md')) expect(rows).toHaveLength(attempt);
|
||||
if (modes[attempt] === 'report-read-throw' && p === path.join(repo, 'detector-output.md')) throw controlledError;
|
||||
return fs.readFileSync(p, opts);
|
||||
},
|
||||
};
|
||||
const args = {
|
||||
ROOT, fs: localFs, os: { tmpdir: () => scratch }, path, expect, CAPTURE_MS, CAPTURE_LONG_MS,
|
||||
process: { ...process, env: { ...process.env, ...env } }, resolveEvalModel,
|
||||
getProjectEvalDir: () => artifactRoot,
|
||||
console: { ...console, log: (...args: any[]) => notices.push(args.join(' ')), error: (...args: any[]) => notices.push(args.join(' ')) },
|
||||
DETECT_SAMPLE, sliceBetween, runId: 'free-detector-replay', logCost: () => {}, recordE2E,
|
||||
evalCollector: { addTest(row: any) {
|
||||
expect(fs.readdirSync(scratch).some(name => name.startsWith('detector-source-evidence-'))).toBe(false);
|
||||
rows.push(row);
|
||||
} },
|
||||
installFakeImpeccable(prefix: string) {
|
||||
const result = installFakeImpeccable(path.join(path.relative(os.tmpdir(), scratch), prefix));
|
||||
owned(result.dir); return result;
|
||||
},
|
||||
beforeAll: (fn: () => void) => before.push(fn), afterAll: (fn: () => void) => after.push(fn),
|
||||
describeIfSelected: (_title: string, names: string[], fn: () => void) => { if (names.includes(id)) fn(); },
|
||||
testConcurrentIfSelected: (name: string, fn: () => Promise<void>, timeout: number) => callbacks.push({ name, fn, timeout }),
|
||||
spawnSync: (cmd: string, argv: string[], opts: any) => {
|
||||
owned(opts.cwd);
|
||||
const result = spawnSync(cmd, argv, { ...opts, env });
|
||||
expect(result.status, result.stderr?.toString()).toBe(0); return result;
|
||||
},
|
||||
runSkillTest: async (opts: any) => {
|
||||
const mode = modes[attempt];
|
||||
repo = opts.workingDirectory; owned(repo);
|
||||
expect(opts.testName).toBe(id); expect(opts.maxTurns).toBe(15); expect(opts.timeout).toBe(CAPTURE_MS);
|
||||
expect(opts.runId).toBe('free-detector-replay'); expect(opts).not.toHaveProperty('model');
|
||||
expect(opts.prompt.replaceAll(repo, '<repo>')).toBe(`You are in a git repo on branch feature/landing with changes against main (the base branch).
|
||||
Read design-review-detector.md: it is the Setup "Design detector" block and "Phase 0: mechanical scan" from /design-review.
|
||||
This is a diff-aware run with no URL, so it is SOURCE mode. Run the probe, then the Phase 0 source-mode scan with base main, exactly as written (use --host claude).
|
||||
Do not run any browser step, do not fix anything, do not run npx.
|
||||
Then write <repo>/detector-output.md: one FINDING-NNN row per rule in the DETECT_TOP block, each tagged with its [rule-id] and the printed impact, plus the first line the probe printed.`);
|
||||
expect(rows).toHaveLength(attempt);
|
||||
if (mode === 'runner-throw') throw controlledError;
|
||||
const toolCalls: any[] = [];
|
||||
const run = (command: string) => {
|
||||
const result = spawnSync('bash', ['-c', command], { cwd: repo, env: { ...env, ...opts.env }, encoding: 'utf8', timeout: 15_000 });
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
const output = result.stdout + result.stderr;
|
||||
toolCalls.push({ tool: 'Bash', input: { command }, output });
|
||||
outputs.push(output);
|
||||
};
|
||||
let wrapper = `${ROOT}/bin/gstack-design-detect.ts`;
|
||||
if (mode === 'relative-path') wrapper = path.relative(repo, wrapper);
|
||||
if (mode === 'symlink-path') {
|
||||
const alias = path.join(bin, 'detector-alias.ts');
|
||||
owned(alias); fs.symlinkSync(wrapper, alias); wrapper = alias;
|
||||
}
|
||||
if (mode === 'quoted-path') wrapper = `'${wrapper}'`;
|
||||
const probe = `${mode === 'wrong-probe-cwd' ? 'cd ..; ' : ''}bun --no-env-file run ${wrapper}${mode === 'quoted-path' ? ' ' : ' '}probe --host ${mode === 'wrong-probe-host' ? 'codex' : 'claude'}`;
|
||||
if (mode === 'claimed-probe') toolCalls.push({ tool: 'Bash', input: { command: probe }, output: 'IMPECCABLE_READY' });
|
||||
else if (mode !== 'no-probe') run(probe);
|
||||
let command = captured.scan.replaceAll('/workspace/gstack', ROOT);
|
||||
command = command.replace(`${ROOT}/bin/gstack-design-detect.ts`, wrapper);
|
||||
if (mode !== 'captured' && mode !== 'other-variable') command = command.replace('"$_BASE"', 'main');
|
||||
if (mode === 'other-variable') command = command.replaceAll('_BASE', 'REVIEW_REF');
|
||||
if (mode === 'wrong-base') {
|
||||
run('git branch other main');
|
||||
command = command.replace('scan --changed main', 'scan --changed other');
|
||||
}
|
||||
if (mode === 'wrong-files') command = command.replace('scan --changed main', 'scan index.html');
|
||||
if (mode === 'extra-file') {
|
||||
owned(path.join(repo, 'extra.css')); fs.writeFileSync(path.join(repo, 'extra.css'), 'body {}');
|
||||
}
|
||||
if (mode !== 'no-scan') run(command);
|
||||
else toolCalls.push({ tool: 'Bash', input: { command }, output: 'DETECT_TOP total=6 rules=4\nDETECT_SUMMARY: total=6\nDETECT_EXIT: 2' });
|
||||
if (mode === 'npx') toolCalls.push({ tool: 'Bash', input: { command: 'npx impeccable detect .' }, output: '' });
|
||||
if (mode === 'browser') toolCalls.push({ tool: 'Bash', input: { command: '$B goto http://localhost' }, output: '' });
|
||||
if (mode !== 'no-report') {
|
||||
let report = captured.report;
|
||||
if (mode === 'no-finding') report = report.replaceAll('FINDING-001', 'ROW-001');
|
||||
if (mode === 'no-rule') report = report.replaceAll('[ai-color-palette]', '[other-rule]');
|
||||
owned(path.join(repo, 'detector-output.md')); fs.writeFileSync(path.join(repo, 'detector-output.md'), report);
|
||||
}
|
||||
return { exitReason: mode === 'timeout' ? 'timeout' : 'success', output: 'Captured public report saved.',
|
||||
toolCalls, transcript: [{ type: 'public-replay' }], browseErrors: mode === 'browse-error' ? ['browser failed'] : [],
|
||||
duration: 42513, model: 'claude-fable-5-1', costEstimate: { estimatedCost: 0.2, turnsUsed: 5, estimatedTokens: 93719 } };
|
||||
},
|
||||
};
|
||||
const start = source.indexOf('function detectorSkillText(');
|
||||
const end = source.indexOf("describeIfSelected('Design HTML slop gate E2E'", start);
|
||||
expect(start).toBeGreaterThan(0); expect(end).toBeGreaterThan(start);
|
||||
try {
|
||||
new Function(...Object.keys(args), new Bun.Transpiler({ loader: 'ts' }).transformSync(source.slice(start, end)))(...Object.values(args));
|
||||
expect(callbacks.map(c => c.name)).toEqual([id, 'design-review-detector-shim-dom']);
|
||||
expect(callbacks[0]!.timeout).toBe(CAPTURE_MS);
|
||||
for (const fn of before) fn();
|
||||
for (; attempt < modes.length; attempt++) {
|
||||
let failure: unknown;
|
||||
try { await callbacks[0]!.fn(); } catch (error) { failure = error; }
|
||||
errors.push(failure);
|
||||
}
|
||||
for (const fn of after.splice(0)) fn();
|
||||
expect(fs.existsSync(repo)).toBe(false);
|
||||
expect(fs.readdirSync(scratch).some(name => name.startsWith('detector-source-evidence-'))).toBe(false);
|
||||
if (retention && retention !== 'write-error') {
|
||||
const walk = (dir: string) => {
|
||||
expect(fs.statSync(dir).mode & 0o777).toBe(0o700);
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
const file = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) walk(file);
|
||||
else {
|
||||
expect(entry.name).toBe('calls.jsonl');
|
||||
expect(fs.statSync(file).mode & 0o777).toBe(0o600);
|
||||
const text = fs.readFileSync(file, 'utf8');
|
||||
retained.push({ file, text, calls: text.trim() ? text.trim().split('\n').map(line => JSON.parse(line)) : [] });
|
||||
}
|
||||
}
|
||||
};
|
||||
walk(artifactRoot);
|
||||
} else if (!retention) expect(fs.existsSync(artifactRoot)).toBe(false);
|
||||
return { rows, errors, outputs, controlledError, retained, notices };
|
||||
} finally {
|
||||
for (const fn of after) fn();
|
||||
fs.rmSync(scratch, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
test.each(['captured', 'literal', 'other-variable'] as const)('source detector callback accepts executed main scope: %s', async mode => {
|
||||
const result = await exercise([mode]);
|
||||
expect(result.outputs.join('\n')).toContain('DETECT_TOP total=6 rules=4');
|
||||
expect(result.outputs.join('\n')).toContain('DETECT_EXIT: 2');
|
||||
expect(result.errors).toEqual([undefined]);
|
||||
expect(result.rows).toHaveLength(1);
|
||||
expect(result.rows[0]).toMatchObject({ passed: true, exit_reason: 'success', cost_usd: 0.2,
|
||||
duration_ms: 42513, turns_used: 5, tokens_used: 93719, model: 'claude-fable-5-1', transcript: [{ type: 'public-replay' }] });
|
||||
});
|
||||
|
||||
test.each(['wrong-base', 'no-scan', 'wrong-files', 'extra-file', 'no-probe', 'no-report', 'no-finding', 'no-rule', 'npx', 'browser', 'timeout', 'browse-error'] as const)(
|
||||
'source detector callback rejects and records invalid evidence once: %s', async mode => {
|
||||
const result = await exercise([mode]);
|
||||
expect(result.errors[0]).toBeDefined();
|
||||
expect(result.rows).toHaveLength(1);
|
||||
expect(result.rows[0]).toMatchObject({ passed: false, exit_reason: mode === 'timeout' ? 'timeout' : 'success', cost_usd: 0.2 });
|
||||
expect(result.rows[0].error).toBe((result.errors[0] as Error).message);
|
||||
},
|
||||
);
|
||||
|
||||
test.each(['runner-throw', 'report-read-throw'] as const)('source detector callback records and rethrows original exception: %s', async mode => {
|
||||
const result = await exercise([mode]);
|
||||
expect(result.errors[0]).toBe(result.controlledError);
|
||||
expect(result.rows).toHaveLength(1);
|
||||
expect(result.rows[0]).toMatchObject({ passed: false, exit_reason: mode === 'runner-throw' ? 'harness_error' : 'success' });
|
||||
expect(result.rows[0].error).toContain(result.controlledError.message);
|
||||
});
|
||||
|
||||
test('source detector retry cannot reuse prior scan evidence or report', async () => {
|
||||
const result = await exercise(['literal', 'no-scan', 'no-report', 'captured']);
|
||||
expect(result.errors.map(Boolean)).toEqual([false, true, true, false]);
|
||||
expect(result.rows.map(row => row.passed)).toEqual([true, false, false, true]);
|
||||
});
|
||||
|
||||
test.each(['quoted-path', 'relative-path', 'symlink-path'] as const)('source detector callback identifies executed canonical wrapper: %s', async mode => {
|
||||
const result = await exercise([mode]);
|
||||
expect(result.errors).toEqual([undefined]);
|
||||
expect(result.rows.map(row => row.passed)).toEqual([true]);
|
||||
});
|
||||
|
||||
test.each(['claimed-probe', 'wrong-probe-host', 'wrong-probe-cwd'] as const)('source detector callback requires an executed valid probe: %s', async mode => {
|
||||
const result = await exercise([mode]);
|
||||
expect(result.errors[0]).toBeDefined();
|
||||
expect(result.rows.map(row => row.passed)).toEqual([false]);
|
||||
});
|
||||
|
||||
test.each(['directory', 'run-id', 'both'] as const)('source detector receipts survive fixture cleanup privately with %s retention', async retention => {
|
||||
const result = await exercise(['wrong-base', 'claimed-probe', 'captured'], retention);
|
||||
expect(result.errors.map(Boolean)).toEqual([true, true, false]);
|
||||
expect(result.rows.map(row => row.passed)).toEqual([false, false, true]);
|
||||
expect(result.retained).toHaveLength(3);
|
||||
expect(new Set(result.retained.map(entry => path.dirname(entry.file))).size).toBe(3);
|
||||
const calls = result.retained.map(entry => entry.calls);
|
||||
expect(calls.filter(attempt => attempt.some(call => call.argv.includes('other')))).toHaveLength(1);
|
||||
expect(calls.filter(attempt => !attempt.some(call => call.argv[0] === 'probe'))).toHaveLength(1);
|
||||
expect(calls.filter(attempt => attempt.some(call => call.argv[0] === 'probe') && attempt.some(call => call.argv.includes('main')))).toHaveLength(1);
|
||||
for (const attempt of calls) {
|
||||
const scan = attempt.find(call => call.argv[0] === 'scan');
|
||||
expect(scan.exit).toBe(2);
|
||||
expect(scan.engine[0].argv.slice(0, 2)).toEqual(['detect', '--json']);
|
||||
expect(scan.engine[0].argv.slice(2).map((p: string) => path.basename(p))).toEqual(['index.html', 'styles.css']);
|
||||
}
|
||||
expect(result.notices.filter(note => note.includes('Detector artifacts:'))).toHaveLength(3);
|
||||
});
|
||||
|
||||
test.each(['directory', 'write-error'] as const)('source detector preserves its original exception with %s retention', async retention => {
|
||||
const result = await exercise(['report-read-throw'], retention);
|
||||
expect(result.errors[0]).toBe(result.controlledError);
|
||||
expect(result.rows).toHaveLength(1);
|
||||
expect(result.rows[0].passed).toBe(false);
|
||||
expect(result.rows[0].error).toContain(result.controlledError.message);
|
||||
if (retention === 'directory') expect(result.retained).toHaveLength(1);
|
||||
else expect(result.notices.some(note => note.includes('Detector artifact write failed:'))).toBe(true);
|
||||
});
|
||||
@@ -181,7 +181,7 @@ test.skipIf(process.platform === 'win32')('real count launcher scopes QA approva
|
||||
const resultFile = path.join(dir, 'result.json'), worker = path.join(dir, 'worker.ts');
|
||||
fs.writeFileSync(worker, `import fs from 'node:fs';\nimport {runPlanSkillCounting,resolveClaudeBinary} from ${JSON.stringify(runner)};\n` +
|
||||
`if(resolveClaudeBinary()!==${JSON.stringify(fake)})throw Error('fake CLI binding');\n` +
|
||||
`const result=await runPlanSkillCounting({skillName:'plan-eng-review',slashCommand:'/plan-eng-review',followUpPrompt:'Synthetic QA permission fixture',expectedPlanPath:${JSON.stringify(path.join(dir, 'report.md'))},approveEngTestPlanEdits:${variant !== 'default'},isLastStep0AUQ:()=>false,reviewCountCeiling:12,timeoutMs:18000,env:{QA_RESULT:${JSON.stringify(resultFile)},QA_MODE:${JSON.stringify(variant)},QA_FIXTURE:${JSON.stringify(path.join(import.meta.dir, 'fixtures/eng-test-plan-edit-dacc.json'))}}});\n` +
|
||||
`const result=await runPlanSkillCounting({skillName:'plan-eng-review',slashCommand:'/plan-eng-review',followUpPrompt:'Synthetic QA permission fixture',expectedPlanPath:${JSON.stringify(path.join(dir, 'report.md'))},approveEngTestPlanEdits:${variant !== 'default'},isLastStep0AUQ:()=>false,reviewCountCeiling:12,timeoutMs:18000,startupReadyMarker:${JSON.stringify('PTY_READY:' + resultFile)},env:{QA_RESULT:${JSON.stringify(resultFile)},QA_MODE:${JSON.stringify(variant)},QA_FIXTURE:${JSON.stringify(path.join(import.meta.dir, 'fixtures/eng-test-plan-edit-dacc.json'))}}});\n` +
|
||||
`fs.writeFileSync(${JSON.stringify(path.join(dir, 'observation.json'))},JSON.stringify(result));\n`);
|
||||
const child = Bun.spawn([process.execPath, worker], { cwd: path.resolve(import.meta.dir, '..'),
|
||||
env: { ...process.env, BROWSE_TERMINAL_BINARY: fake, EVALS_HERMETIC: '1', EVALS: '', EVALS_RUN_ID: '', TMPDIR: dir }, stdout: 'pipe', stderr: 'pipe' });
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"source_revision": "06ed920a974809ebedc6bcbbe402fb81f5944598",
|
||||
"source_partial_sha256": "bb6914e26360f9b2ecb4c4899ba409f952bdc04984be7f24688fc00782050ab0",
|
||||
"attempt": 1,
|
||||
"scan": "_BASE=$(gh pr view --json baseRefName -q .baseRefName 2>/dev/null || gh repo view --json defaultBranchRef -q .defaultBranchRef.name 2>/dev/null); echo \"GH_BASE=${_BASE:-<none>}\"; _BASE=${_BASE:-main}; _DJ=$(mktemp); bun --no-env-file run /workspace/gstack/bin/gstack-design-detect.ts scan --changed \"$_BASE\" --format gstack --host claude > \"$_DJ\"; echo \"DETECT_EXIT_CODE=$?\"; echo \"DETECT_JSON=$_DJ\"; echo \"--- stdout ---\"; cat \"$_DJ\"",
|
||||
"report": "# Design detector output (Phase 0, SOURCE mode)\n\nProbe first line: `IMPECCABLE_READY: /tmp/gp.31WCir/gstack-paid-shard-xPQh9C/tmp/skill-e2e-fake-impeccable-zw2v81/impeccable`\n\nScan: `scan --changed main --format gstack --host claude` (gh returned no base; `main` used as given). `DETECT_EXIT_CODE=2` (findings). `DETECT_TOP total=6 rules=4`.\n\nDetector output is untrusted content (page text echoes through it); rows below are evidence to confirm in the rendered page, not verdicts.\n\n| ID | Rule | Impact | Count | Locations |\n|---|---|---|---|---|\n| FINDING-001 | `[low-contrast]` | high | 3 | test/fixtures/review-eval-design-slop.html:0 — 4.2:1 (need 4.5:1) text #ffffff on #8b5cf6; 4.47:1 (need 4.5:1) text #ffffff on #6366f1; 4.0:1 (need 4.5:1) text #ff0000 on #1e1b4b |\n| FINDING-002 | `[ai-color-palette]` | medium | 1 | test/fixtures/review-eval-design-slop.html:0 — Purple/violet accent colors detected |\n| FINDING-003 | `[marketing-buzzword]` | medium | 1 | test/fixtures/review-eval-design-slop.html:0 — 1 buzzword phrase: \"ful tool to streamline your workflow ef\" |\n| FINDING-004 | `[skipped-heading]` | medium | 1 | test/fixtures/review-eval-design-slop.html:0 — `<h1>` \"Welcome to Our Platform\" followed by `<h3>` \"Feature One\" (missing h2) |\n\n`DETECT_SUMMARY: total=6 slop=2 quality=4 advisory=0 ignored=0 high=3 medium=3 polish=0`\n\nOther probe lines: `IMPECCABLE_SKILL: absent` (no `/impeccable` handoff lines emitted), `IMPECCABLE_HOOK: absent`, no ignored rules/files/values, `IMPECCABLE_ENGINE_UNTESTED: sha256:834a2acedcbb`.\n"
|
||||
}
|
||||
+1
@@ -49,3 +49,4 @@ process.stdin.on('data',async bytes=>{
|
||||
save();setTimeout(()=>process.exit(0),100);
|
||||
});
|
||||
process.on('SIGINT',()=>process.exit(0));
|
||||
process.stdout.write('PTY_READY:'+process.env.QA_RESULT+'\x1b[2J\x1b[H');
|
||||
|
||||
+76
-1
File diff suppressed because one or more lines are too long
+153
@@ -0,0 +1,153 @@
|
||||
{
|
||||
"source": "https://github.com/garrytan/gstack/actions/runs/36058320346/job/107831034133",
|
||||
"cases": [
|
||||
{
|
||||
"attempt": 1,
|
||||
"input": {
|
||||
"questions": [
|
||||
{
|
||||
"question": "Pre-Landing Review: 0 issues (0 critical, 0 informational). 1 [ADVISORY] needs your input:\n\n1. [ADVISORY] src/retry-worker.ts:2 — Duplicated `retrySeconds` parser (shared-libs, confidence 9/10, maintainability + core)\n Both src/retry-worker.ts:2-15 (this diff) and src/retry-route.ts:2-15 are verbatim copies of lib/retry-after.ts:2-15, which src/scheduler.ts already imports and test/retry-after.test.ts already covers (null/invalid/whitespace/ceiling/HTTP-date cases).\n Fix: replace each copy with `export { retrySeconds } from '../lib/retry-after';` (the worker's pre-diff state). Preserved differences: none — same signature, 3600 ceiling, fallback default, runtime (TS/Bun), no deployment boundary. Compatibility tests: existing test/retry-after.test.ts; module exports keep the same name. Shared-failure risk: a lib regression would hit scheduler+route+worker instead of scheduler only, mitigated by the existing contract test.\n Estimated implementation: −30 removed / +2 added / −28 saved. Total incl. tests: same (no new tests needed).\n\nNote: this was skipped in the prior review, but src/retry-route.ts has changed on disk (hidden by assume-unchanged), so the decision must be made fresh.\n\nRECOMMENDATION: A — the worker's own comment says both callers need the same contract, and the helper is proven and tested.",
|
||||
"header": "Advisory #1",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "A) Fix both callers (Recommended)",
|
||||
"description": "Restore the re-export in src/retry-worker.ts and replace the copy in src/retry-route.ts with the same re-export. −28 lines."
|
||||
},
|
||||
{
|
||||
"label": "B) Fix worker only",
|
||||
"description": "Restore the re-export in src/retry-worker.ts (files in this diff) and leave src/retry-route.ts as-is. −14 lines."
|
||||
},
|
||||
{
|
||||
"label": "C) Skip",
|
||||
"description": "Keep the inline copies; record the advisory as skipped."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"question": "src/retry-route.ts is marked assume-unchanged in your local index, so git status/diff hide its on-disk change (trailing comment on line 17) and any edit I make there would also be hidden. Clear the flag with `git update-index --no-assume-unchanged src/retry-route.ts`?",
|
||||
"header": "Index flag",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Yes, clear it (Recommended)",
|
||||
"description": "Makes the route's working-tree state visible to Git and to the review snapshot. Triggers a normal re-review pass with a new start token."
|
||||
},
|
||||
{
|
||||
"label": "No, leave it",
|
||||
"description": "Keep the local index flag. The route stays excluded from reusable review coverage."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"answers": {
|
||||
"Pre-Landing Review: 0 issues (0 critical, 0 informational). 1 [ADVISORY] needs your input:\n\n1. [ADVISORY] src/retry-worker.ts:2 — Duplicated `retrySeconds` parser (shared-libs, confidence 9/10, maintainability + core)\n Both src/retry-worker.ts:2-15 (this diff) and src/retry-route.ts:2-15 are verbatim copies of lib/retry-after.ts:2-15, which src/scheduler.ts already imports and test/retry-after.test.ts already covers (null/invalid/whitespace/ceiling/HTTP-date cases).\n Fix: replace each copy with `export { retrySeconds } from '../lib/retry-after';` (the worker's pre-diff state). Preserved differences: none — same signature, 3600 ceiling, fallback default, runtime (TS/Bun), no deployment boundary. Compatibility tests: existing test/retry-after.test.ts; module exports keep the same name. Shared-failure risk: a lib regression would hit scheduler+route+worker instead of scheduler only, mitigated by the existing contract test.\n Estimated implementation: −30 removed / +2 added / −28 saved. Total incl. tests: same (no new tests needed).\n\nNote: this was skipped in the prior review, but src/retry-route.ts has changed on disk (hidden by assume-unchanged), so the decision must be made fresh.\n\nRECOMMENDATION: A — the worker's own comment says both callers need the same contract, and the helper is proven and tested.": "C) Skip",
|
||||
"src/retry-route.ts is marked assume-unchanged in your local index, so git status/diff hide its on-disk change (trailing comment on line 17) and any edit I make there would also be hidden. Clear the flag with `git update-index --no-assume-unchanged src/retry-route.ts`?": "No, leave it"
|
||||
}
|
||||
},
|
||||
{
|
||||
"attempt": 1,
|
||||
"input": {
|
||||
"questions": [
|
||||
{
|
||||
"question": "[ADVISORY] src/retry-worker.ts:2 — the diff inlines a byte-identical copy of lib/retry-after.ts retrySeconds (tested in test/retry-after.test.ts, already used by src/scheduler.ts); src/retry-route.ts holds a third copy. Recommended: restore `export { retrySeconds } from '../lib/retry-after';` in both callers (~28–30 lines saved, no new tests needed). How do you want to proceed?",
|
||||
"header": "Shared helper",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Fix both callers (Recommended)",
|
||||
"description": "Re-export from lib/retry-after in src/retry-worker.ts and src/retry-route.ts. Removes all drift risk between the three identical copies."
|
||||
},
|
||||
{
|
||||
"label": "Fix worker only",
|
||||
"description": "Re-export in src/retry-worker.ts (the changed file) and leave src/retry-route.ts as-is for a separate change."
|
||||
},
|
||||
{
|
||||
"label": "Skip",
|
||||
"description": "Keep the inlined copy. Recorded as an explicit skip with verified snapshot coverage (retry-route.ts excluded from coverage due to skip-worktree)."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"question": "src/retry-route.ts is a skip-worktree index entry, so edits to it are invisible to git status/diff and won't be committed. If it gets edited, should I clear the flag (`git update-index --no-skip-worktree src/retry-route.ts`) so the change is committable?",
|
||||
"header": "skip-worktree",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Clear the flag (Recommended)",
|
||||
"description": "Run git update-index --no-skip-worktree on src/retry-route.ts so its change shows in the diff and can be committed."
|
||||
},
|
||||
{
|
||||
"label": "Leave the flag set",
|
||||
"description": "Edit the working copy only; you will handle the index flag yourself. The change stays hidden from git."
|
||||
},
|
||||
{
|
||||
"label": "Not applicable",
|
||||
"description": "Choose this if you are not editing src/retry-route.ts."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"answers": {
|
||||
"[ADVISORY] src/retry-worker.ts:2 — the diff inlines a byte-identical copy of lib/retry-after.ts retrySeconds (tested in test/retry-after.test.ts, already used by src/scheduler.ts); src/retry-route.ts holds a third copy. Recommended: restore `export { retrySeconds } from '../lib/retry-after';` in both callers (~28–30 lines saved, no new tests needed). How do you want to proceed?": "Skip",
|
||||
"src/retry-route.ts is a skip-worktree index entry, so edits to it are invisible to git status/diff and won't be committed. If it gets edited, should I clear the flag (`git update-index --no-skip-worktree src/retry-route.ts`) so the change is committable?": "Not applicable"
|
||||
}
|
||||
},
|
||||
{
|
||||
"attempt": 2,
|
||||
"input": {
|
||||
"questions": [
|
||||
{
|
||||
"question": "Pre-Landing Review: 0 issues (0 critical, 0 informational) + 1 advisory. The prior skip on this branch could not be reused (src/retry-route.ts is skip-worktree with raw bytes differing from its blob). How do you want to handle the advisory?\n\n1. [ADVISORY] (confidence 10/10, MULTI-SPECIALIST CONFIRMED: core + maintainability) src/retry-worker.ts:2 — The diff replaces `export { retrySeconds } from '../lib/retry-after'` with a byte-identical 15-line copy of the helper; src/retry-route.ts:2-15 carries the same copy. lib/retry-after.ts is the tested, established contract (used by src/scheduler.ts, covered by test/retry-after.test.ts).\n Fix: restore the one-line re-export in the worker and replace the route's copy with the same re-export. Estimated implementation: −30 removed / +2 added = −28 lines; total incl. tests ≈ −28 (existing tests already cover the contract). Shared-failure blast radius: a bug in lib/retry-after.ts would reach scheduler, worker, and route — scheduler is already exposed today.\n Note: src/retry-route.ts has the skip-worktree index bit, so an edit there will not appear in git status/diff until `git update-index --no-skip-worktree src/retry-route.ts` is run.\n\nRECOMMENDATION: Fix both — the change as written re-introduces duplication that HEAD had already removed.",
|
||||
"header": "Advisory #1",
|
||||
"multiSelect": false,
|
||||
"options": [
|
||||
{
|
||||
"label": "Fix both callers (Recommended)",
|
||||
"description": "Restore re-export in src/retry-worker.ts and replace src/retry-route.ts's copy with the same re-export (also clear its skip-worktree bit so the change is visible to Git)."
|
||||
},
|
||||
{
|
||||
"label": "Fix worker only",
|
||||
"description": "Restore the re-export in src/retry-worker.ts (the file in this diff); leave src/retry-route.ts untouched."
|
||||
},
|
||||
{
|
||||
"label": "Skip",
|
||||
"description": "Keep the inline copies. Recorded as an explicit skip; reuse coverage will exclude src/retry-route.ts since it is not snapshot-covered."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"answers": {
|
||||
"Pre-Landing Review: 0 issues (0 critical, 0 informational) + 1 advisory. The prior skip on this branch could not be reused (src/retry-route.ts is skip-worktree with raw bytes differing from its blob). How do you want to handle the advisory?\n\n1. [ADVISORY] (confidence 10/10, MULTI-SPECIALIST CONFIRMED: core + maintainability) src/retry-worker.ts:2 — The diff replaces `export { retrySeconds } from '../lib/retry-after'` with a byte-identical 15-line copy of the helper; src/retry-route.ts:2-15 carries the same copy. lib/retry-after.ts is the tested, established contract (used by src/scheduler.ts, covered by test/retry-after.test.ts).\n Fix: restore the one-line re-export in the worker and replace the route's copy with the same re-export. Estimated implementation: −30 removed / +2 added = −28 lines; total incl. tests ≈ −28 (existing tests already cover the contract). Shared-failure blast radius: a bug in lib/retry-after.ts would reach scheduler, worker, and route — scheduler is already exposed today.\n Note: src/retry-route.ts has the skip-worktree index bit, so an edit there will not appear in git status/diff until `git update-index --no-skip-worktree src/retry-route.ts` is run.\n\nRECOMMENDATION: Fix both — the change as written re-introduces duplication that HEAD had already removed.": "Skip"
|
||||
}
|
||||
},
|
||||
{
|
||||
"attempt": 2,
|
||||
"input": {
|
||||
"questions": [
|
||||
{
|
||||
"question": "Core review found 0 defects. 1 advisory needs your input:\n\n1. [ADVISORY] src/retry-worker.ts:2 — Duplicated `retrySeconds` (MULTI-SPECIALIST CONFIRMED: core + maintainability, confidence 10/10). The diff replaces `export { retrySeconds } from '../lib/retry-after'` with a byte-identical 15-line copy of lib/retry-after.ts; src/retry-route.ts:2-15 carries the same copy.\n Fix: Restore the re-export in src/retry-worker.ts and replace the body in src/retry-route.ts with `export { retrySeconds } from '../lib/retry-after';`. Helper: lib/retry-after.ts#retrySeconds (existing, tested by test/retry-after.test.ts, used by src/scheduler.ts). Implementation: -30/+2 = 28 lines saved; total same (no new tests needed). Shared-failure blast radius: scheduler, worker, route all share one parser (already true for scheduler).\n\nRECOMMENDATION: Fix — the helper is proven and identical; the copy adds drift risk for no benefit. Note the prior Skip couldn't be reused because src/retry-route.ts is assume-unchanged with hidden working-tree changes.\n\nHow do you want to handle it?",
|
||||
"header": "Advisory",
|
||||
"options": [
|
||||
{
|
||||
"label": "Fix (Recommended)",
|
||||
"description": "Re-export retrySeconds from lib/retry-after in both src/retry-worker.ts and src/retry-route.ts (removes 30 duplicated lines, adds 2)."
|
||||
},
|
||||
{
|
||||
"label": "Skip",
|
||||
"description": "Keep the duplicated copies; record the advisory as skipped for this branch."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"answers": {
|
||||
"Core review found 0 defects. 1 advisory needs your input:\n\n1. [ADVISORY] src/retry-worker.ts:2 — Duplicated `retrySeconds` (MULTI-SPECIALIST CONFIRMED: core + maintainability, confidence 10/10). The diff replaces `export { retrySeconds } from '../lib/retry-after'` with a byte-identical 15-line copy of lib/retry-after.ts; src/retry-route.ts:2-15 carries the same copy.\n Fix: Restore the re-export in src/retry-worker.ts and replace the body in src/retry-route.ts with `export { retrySeconds } from '../lib/retry-after';`. Helper: lib/retry-after.ts#retrySeconds (existing, tested by test/retry-after.test.ts, used by src/scheduler.ts). Implementation: -30/+2 = 28 lines saved; total same (no new tests needed). Shared-failure blast radius: scheduler, worker, route all share one parser (already true for scheduler).\n\nRECOMMENDATION: Fix — the helper is proven and identical; the copy adds drift risk for no benefit. Note the prior Skip couldn't be reused because src/retry-route.ts is assume-unchanged with hidden working-tree changes.\n\nHow do you want to handle it?": "Skip"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"source": "https://github.com/garrytan/gstack/actions/runs/36058320346/job/107831033952",
|
||||
"case": "tpa-broken",
|
||||
"responses": [
|
||||
{
|
||||
"attempt": 1,
|
||||
"text": "The contract requires the runtime Aside readiness probe before the consent question. Running it now (plus `uname -s`, which rule 1 needs to decide whether the download pitch applies).\n**Probe result:** `ASIDE_NOT_RUNNING` (the `aside` binary is present, but the readiness REPL check did not return `ASIDE_READY`). Platform: Linux, so no download pitch applies.\n\nPer rule 1, the contract wants one retry after you open Aside — and per rule 2, until a probe actually returns `READY`, the Aside drive option is omitted from the consent question. So here is the question as it stands right now:\n\n---\n\n**Third-party web action: create a test-mode API token in the Acme vendor dashboard at https://dashboard.acme.test.**\n\nHow would you like to proceed?\n\n- **A) Open Aside first, then re-probe.** Open the Aside app (sign in if it asks), tell me when it's up, and I'll re-run the readiness check exactly once. If it returns `READY`, I'll re-ask this question with the \"drive it in your Aside browser\" option included. If it still fails, I'll quote the probe output verbatim and treat Aside as not detected for this task.\n- **B) I drive it in gstack's own visible browser.** I open `https://dashboard.acme.test` in a headed browser and handle navigation; I hand off to you for sign-in and for copying the token (you paste it into an approved 0600 local file or your secret store — it never appears in chat), then I verify it with one non-mutating API call.\n- **C) Manual instructions.** I give you step-by-step instructions for the Acme dashboard and mark this step blocked on you.\n- **D) Defer.** Skip this for now; the step stays blocked on you.\n\nThis choice is consent for this task only — it won't be persisted or reused.\n\n---\n\nStopping here. No browsing has been done or will be done until you answer.",
|
||||
"sha256": "942905e9041f7e606bfa204ecf32f9d7e250570a64a5d29b41f5d50a9e2d4483"
|
||||
},
|
||||
{
|
||||
"attempt": 2,
|
||||
"text": "The contract requires the Aside readiness probe at runtime before the consent question. Running it now (and `uname -s`, which governs the NEEDS_ASIDE messaging).\n**Detection result:** the Aside readiness probe returned `ASIDE_NOT_RUNNING` (the `aside` binary is present but the `aside repl` readiness check did not return `ASIDE_READY`). Platform is `Linux`, so per the contract there is no Aside download pitch. Per rule 1, the recovery path is: you open the Aside app (and sign in if it asks), then I re-run the probe **once**; if it still fails I treat Aside as not detected for this task. Per rule 2, because no probe has returned `READY`, the Aside drive option is omitted from the consent question below.\n\n---\n\n**Consent question**\n\nThe next step needs an action on a third-party site you control: **create a test-mode API token in the Acme vendor dashboard at https://dashboard.acme.test**. Nothing has been browsed yet. How would you like to proceed?\n\n- **A)** Open the Aside app now (sign in if prompted) and tell me when it's up — I'll re-run the readiness probe once, and if it returns `READY` I'll re-ask with the Aside drive option included.\n- **B)** I drive it in gstack's own visible browser — you take over (`handoff`) for sign-in and any password/CAPTCHA/identity steps, then hand back (`resume`). I'd touch only dashboard.acme.test and only the token-creation flow; the captured token goes to a 0600 local file you approve (never into chat), and I'd verify it with one non-mutating API call before calling it done.\n- **C)** Manual instructions — I give you the step list and mark this step blocked on you.\n- **D)** Defer — skip this for now and mark it blocked.\n\nThis choice is per-task consent only; it won't be persisted or carried into a later task.\n\n**Stopping here** — no browsing or further action until you answer.",
|
||||
"sha256": "c25853670cd7033c16b1f6f95c2072b2bf620f946684af8c5853eddf7cbce382"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -108,6 +108,7 @@ export interface RunAgentSdkOptions {
|
||||
queryProvider?: QueryProvider;
|
||||
/** Cancel queueing, SDK work and retries under the caller's case deadline. */
|
||||
signal?: AbortSignal;
|
||||
onAdmission?: () => void;
|
||||
/** Max 429 retries per call. Default 3. */
|
||||
maxRetries?: number;
|
||||
/**
|
||||
@@ -330,6 +331,7 @@ export async function runAgentSdkTest(
|
||||
|
||||
let attempt = 0;
|
||||
let lastErr: unknown = null;
|
||||
let admitted = false;
|
||||
|
||||
while (attempt <= maxRetries) {
|
||||
await sem.acquire(opts.signal);
|
||||
@@ -376,6 +378,10 @@ export async function runAgentSdkTest(
|
||||
|
||||
try {
|
||||
opts.signal?.throwIfAborted();
|
||||
if (!admitted) {
|
||||
admitted = true;
|
||||
opts.onAdmission?.();
|
||||
}
|
||||
// When canUseTool is supplied, the SDK must route tool-use approval
|
||||
// decisions through the callback. bypassPermissions short-circuits
|
||||
// that. Flip to 'default' mode so canUseTool actually fires. Tests
|
||||
@@ -415,6 +421,7 @@ export async function runAgentSdkTest(
|
||||
sdkOpts.systemPrompt = opts.systemPrompt;
|
||||
}
|
||||
|
||||
opts.signal?.throwIfAborted();
|
||||
const q = queryImpl({
|
||||
prompt: opts.userPrompt,
|
||||
options: sdkOpts,
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
import { afterAll, describe, mock, spyOn, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { serializeNativeAuq } from './auq-native-capture';
|
||||
|
||||
const spec = JSON.parse(process.env.AUQ_PARALLEL_SPEC!);
|
||||
const root = process.env.AUQ_PARALLEL_ROOT!;
|
||||
const realSetTimeout = globalThis.setTimeout, realClearTimeout = globalThis.clearTimeout, realNow = Date.now;
|
||||
const timers = new Map<number, { at: number; callback: () => void }>();
|
||||
let now = 0, timerId = 0, pumping = false;
|
||||
if (spec.queryMs) {
|
||||
Date.now = () => now;
|
||||
globalThis.setTimeout = ((callback: (...args: any[]) => void, delay = 0, ...args: any[]) => {
|
||||
const id = ++timerId;
|
||||
timers.set(id, { at: now + Math.max(0, delay), callback: () => callback(...args) });
|
||||
if (!pumping) {
|
||||
pumping = true;
|
||||
realSetTimeout(function pump() {
|
||||
const next = [...timers].sort((a, b) => a[1].at - b[1].at)[0];
|
||||
if (!next) { pumping = false; return; }
|
||||
timers.delete(next[0]); now = next[1].at; next[1].callback();
|
||||
realSetTimeout(pump, 0);
|
||||
}, 0);
|
||||
}
|
||||
return id;
|
||||
}) as typeof setTimeout;
|
||||
globalThis.clearTimeout = ((id: number) => { timers.delete(id); }) as typeof clearTimeout;
|
||||
}
|
||||
const events: Array<{ kind: string; id: string; active?: number; at?: number; caseDeadline?: number }> = [];
|
||||
const inputs: any[] = [], judges: any[] = [], judgeRequests: any[] = [], fixtures: string[] = [];
|
||||
const ownedStates: string[] = [];
|
||||
let settlements: any[] = [];
|
||||
let active = 0, peak = 0, judging = 0, judgePeak = 0, answers = 0;
|
||||
const idFrom = (cwd: string) => path.basename(cwd).replace(/-owned$/, '').replace(/^auq-(?:consistency-|ab-)/, '');
|
||||
const indexFrom = (id: string) => id === 'carved' ? 0 : id === 'verbose' ? 1 : Number(id);
|
||||
const questionFor = (id: string) => {
|
||||
const question = {
|
||||
header: 'Mode',
|
||||
question: `Run ${id}\nELI10: Review this pricing plan.\nRecommendation: SELECTIVE EXPANSION because the untested premise needs a comparison.\nPros / cons:\nNet: Choose the review scope.`,
|
||||
options: ['SCOPE EXPANSION', 'SELECTIVE EXPANSION', 'HOLD SCOPE', 'SCOPE REDUCTION'].map(label => ({
|
||||
label: label + (label === 'SELECTIVE EXPANSION' ? ' (recommended)' : ''),
|
||||
description: '✅ Concrete benefit.\n❌ Honest tradeoff.',
|
||||
})),
|
||||
};
|
||||
if (spec.omit && indexFrom(id) === (spec.omitIndex ?? 2)) {
|
||||
question.question = question.question.replaceAll(spec.omit, '');
|
||||
for (const option of question.options) {
|
||||
option.label = option.label.replaceAll(spec.omit, '');
|
||||
option.description = option.description.replaceAll(spec.omit, '');
|
||||
}
|
||||
}
|
||||
return question;
|
||||
};
|
||||
|
||||
mock.module('@anthropic-ai/claude-agent-sdk', () => ({ query: ({ prompt, options }: any) => {
|
||||
const id = idFrom(options.cwd), index = indexFrom(id);
|
||||
active++; peak = Math.max(peak, active);
|
||||
events.push({ kind: 'query-start', id, active, at: Date.now() });
|
||||
inputs.push({ id, prompt, cwd: options.cwd, model: options.model, maxTurns: options.maxTurns,
|
||||
systemPrompt: options.systemPrompt, tools: options.tools, allowedTools: options.allowedTools,
|
||||
permissionMode: options.permissionMode, settingSources: options.settingSources,
|
||||
config: options.env.CLAUDE_CONFIG_DIR, state: options.env.GSTACK_HOME,
|
||||
skill: fs.readFileSync(path.join(options.cwd, 'plan-ceo-review/SKILL.md'), 'utf8'),
|
||||
plan: fs.readFileSync(path.join(options.cwd, 'plan.md'), 'utf8'),
|
||||
sections: fs.existsSync(path.join(options.cwd, 'plan-ceo-review/sections'))
|
||||
? fs.readdirSync(path.join(options.cwd, 'plan-ceo-review/sections')).sort().map(name => [name,
|
||||
fs.readFileSync(path.join(options.cwd, 'plan-ceo-review/sections', name), 'utf8')]) : [],
|
||||
});
|
||||
let closed = false;
|
||||
const close = () => {
|
||||
if (closed) return;
|
||||
closed = true; active--;
|
||||
events.push({ kind: 'query-close', id, active });
|
||||
};
|
||||
return {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
try {
|
||||
yield { type: 'system', subtype: 'init', claude_code_version: 'fixture' };
|
||||
await new Promise<void>(resolve => {
|
||||
const signal = options.abortController.signal;
|
||||
const abort = () => { clearTimeout(timer); resolve(); };
|
||||
const timer = setTimeout(() => { signal.removeEventListener('abort', abort); resolve(); },
|
||||
spec.queryMs ?? (index === 0 ? 10 : 60));
|
||||
signal.addEventListener('abort', abort, { once: true });
|
||||
});
|
||||
if (options.abortController.signal.aborted) return;
|
||||
if (spec.reject === index || spec.rejectAll) throw Error(`capture rejection ${id}`);
|
||||
void options.canUseTool('AskUserQuestion', { questions: [questionFor(id)] }, {
|
||||
toolUseID: `native-${id}`, signal: options.abortController.signal,
|
||||
}).then(() => answers++);
|
||||
yield { type: 'assistant', message: { content: [] } };
|
||||
} finally { close(); }
|
||||
},
|
||||
close,
|
||||
};
|
||||
} }));
|
||||
|
||||
mock.module('./e2e-gate', () => ({ describeE2ETier: (tier: string) => {
|
||||
if (tier !== 'periodic') throw Error(`unexpected tier ${tier}`);
|
||||
return describe;
|
||||
} }));
|
||||
mock.module('@anthropic-ai/sdk', () => ({ default: class {
|
||||
messages = { create: async (request: any) => {
|
||||
const id = request.messages[0].content.match(/Run (\w+)/)![1], index = indexFrom(id);
|
||||
judgeRequests.push({ id, request });
|
||||
await new Promise(resolve => setTimeout(resolve, 90));
|
||||
if (spec.judgeReject === index) throw Error(`judge rejection ${id}`);
|
||||
return { stop_reason: 'end_turn', content: [{ type: 'text', text: JSON.stringify({
|
||||
reason_substance: spec.scores?.[index] ?? 4, reasoning: 'controlled free verdict',
|
||||
}) }] };
|
||||
} };
|
||||
} }));
|
||||
const judgeHelper = await import('./llm-judge');
|
||||
const judgeRecommendation = judgeHelper.judgeRecommendation;
|
||||
mock.module('./llm-judge', () => ({ ...judgeHelper, judgeRecommendation: async (text: string) => {
|
||||
const id = text.match(/Run (\w+)/)![1];
|
||||
judges.push({ id, text });
|
||||
judging++; judgePeak = Math.max(judgePeak, judging);
|
||||
events.push({ kind: 'judge-start', id });
|
||||
try {
|
||||
return await judgeRecommendation(text);
|
||||
} finally {
|
||||
judging--;
|
||||
events.push({ kind: 'judge-settle', id });
|
||||
}
|
||||
} }));
|
||||
|
||||
const helper = await import('./auq-sdk-capture');
|
||||
const captureModeSelectionAuq = helper.captureModeSelectionAuq;
|
||||
mock.module('./auq-sdk-capture', () => ({ ...helper,
|
||||
captureModeSelectionAuq: async (opts: Parameters<typeof helper.captureModeSelectionAuq>[0]) => {
|
||||
const id = idFrom(opts.planDir);
|
||||
events.push({ kind: 'capture-start', id, at: Date.now(), caseDeadline: opts.caseDeadline });
|
||||
try {
|
||||
const text = await captureModeSelectionAuq(opts);
|
||||
if (text !== serializeNativeAuq(questionFor(id))) throw Error(`native text changed for ${id}`);
|
||||
return spec.empty === indexFrom(id) ? '' : text;
|
||||
} finally { events.push({ kind: 'capture-settle', id }); }
|
||||
},
|
||||
}));
|
||||
|
||||
const make = fs.mkdtempSync, remove = fs.rmSync;
|
||||
spyOn(fs, 'mkdtempSync').mockImplementation(((prefix: string, ...args: any[]) => {
|
||||
const name = path.basename(String(prefix));
|
||||
if (name === 'gstack-mode-auq-') {
|
||||
const dir = make(prefix, ...args);
|
||||
ownedStates.push(dir);
|
||||
return dir;
|
||||
}
|
||||
if (path.dirname(String(prefix)) !== root || !/^auq-(?:consistency-|ab-)/.test(name)) return make(prefix, ...args);
|
||||
const id = idFrom(name + 'owned');
|
||||
if (spec.setupReject === indexFrom(id)) throw Error(`setup rejection ${id}`);
|
||||
const dir = path.join(root, name + 'owned');
|
||||
fs.mkdirSync(dir);
|
||||
if (!fs.realpathSync(dir).startsWith(fs.realpathSync(root) + path.sep)) throw Error('fixture escaped owned root');
|
||||
fixtures.push(dir);
|
||||
return dir;
|
||||
}) as typeof fs.mkdtempSync);
|
||||
spyOn(fs, 'rmSync').mockImplementation((file, options) => {
|
||||
if (fixtures.includes(String(file))) {
|
||||
const id = idFrom(String(file));
|
||||
events.push({ kind: 'cleanup', id, active });
|
||||
if (spec.cleanupReject === indexFrom(id)) throw Error(`cleanup rejection ${id}`);
|
||||
}
|
||||
return remove(file, options);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
const receipts = fs.readdirSync(path.join(root, 'artifacts/native-auq'), { recursive: true })
|
||||
.filter(file => String(file).endsWith('capture.json'))
|
||||
.map(file => JSON.parse(fs.readFileSync(path.join(root, 'artifacts/native-auq', String(file)), 'utf8')));
|
||||
fs.writeFileSync(path.join(root, 'facts.json'), JSON.stringify({ events, inputs, judges, judgeRequests, receipts,
|
||||
fixtures, peak, judgePeak, active, judging, answers, settlements, elapsed: now, pendingTimers: timers.size,
|
||||
leftovers: fixtures.filter(dir => fs.existsSync(dir)),
|
||||
ownedStateLeftovers: [...ownedStates, ...inputs.flatMap(input => [input.config, input.state])].filter(dir => fs.existsSync(dir)),
|
||||
}));
|
||||
Date.now = realNow; globalThis.setTimeout = realSetTimeout; globalThis.clearTimeout = realClearTimeout;
|
||||
});
|
||||
|
||||
if (spec.suite === 'direct') {
|
||||
test('actual native capture admission lifecycle', async () => {
|
||||
const dirs: string[] = [];
|
||||
try {
|
||||
settlements = (await Promise.allSettled(Array.from({ length: spec.runs ?? 3 }, (_, i) => {
|
||||
const dir = helper.setupPlanCeoDir({ ...helper.carvedSkill(), tmpPrefix: `auq-consistency-${i}-` });
|
||||
dirs.push(dir);
|
||||
return captureModeSelectionAuq({ planDir: dir, testName: `direct-${i}`, runId: 'free',
|
||||
...(spec.outerMs === undefined ? {} : { caseDeadline: now + spec.outerMs }) });
|
||||
}))).map(result => result.status === 'fulfilled' ? result : { status: result.status, reason: String(result.reason) });
|
||||
} finally {
|
||||
for (const dir of dirs) fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
} else {
|
||||
await import(process.env.AUQ_PARALLEL_SOURCE ?? path.join(import.meta.dir, '..',
|
||||
spec.suite === 'consistency' ? 'skill-e2e-auq-consistency.test.ts' : 'skill-e2e-auq-verbose-vs-carved-ab.test.ts'));
|
||||
}
|
||||
@@ -418,8 +418,10 @@ export async function captureModeSelectionAuq(opts: {
|
||||
testName: string;
|
||||
runId?: string;
|
||||
model?: string;
|
||||
caseDeadline?: number;
|
||||
}): Promise<string> {
|
||||
const startedAt = Date.now(), deadline = startedAt + 240_000;
|
||||
const startedAt = Date.now();
|
||||
let deadline = opts.caseDeadline ?? startedAt + 240_000;
|
||||
const cwd = path.resolve(opts.planDir);
|
||||
const skillPath = path.join(cwd, 'plan-ceo-review', 'SKILL.md');
|
||||
const planPath = path.join(cwd, 'plan.md');
|
||||
@@ -448,7 +450,14 @@ Ask the user through the AskUserQuestion tool and wait for their answer.`;
|
||||
let outcome = 'error', diagnostic: string | undefined, actorFailure: Error | undefined;
|
||||
let captured: { toolUseId: string; input: Record<string, unknown>; question: NativePlanQuestion; text: string } | undefined;
|
||||
let terminal: { exitReason: string; turnsUsed: number; costUsd: number; sdkClaudeCodeVersion: string; errors?: string[] } | undefined;
|
||||
const timer = setTimeout(() => controller.abort(timeout), Math.max(0, deadline - Date.now()));
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
const armDeadline = () => {
|
||||
clearTimeout(timer);
|
||||
const remaining = deadline - Date.now();
|
||||
if (remaining <= 0) controller.abort(timeout);
|
||||
else timer = setTimeout(() => controller.abort(timeout), remaining);
|
||||
};
|
||||
armDeadline();
|
||||
const fail = (reason: string, detail?: string): never => {
|
||||
outcome = reason;
|
||||
throw new Error(`${opts.testName}: AUQ capture failed (${reason})${detail ? `: ${detail}` : ''}`);
|
||||
@@ -470,6 +479,10 @@ Ask the user through the AskUserQuestion tool and wait for their answer.`;
|
||||
workingDirectory: cwd, model, maxTurns: 12, maxRetries: 0,
|
||||
allowedTools: ['Read', 'Write', 'AskUserQuestion'], permissionMode: 'default', settingSources: [],
|
||||
pathToClaudeCodeExecutable: binary, signal: controller.signal,
|
||||
onAdmission: opts.caseDeadline === undefined ? undefined : () => {
|
||||
deadline = Math.min(opts.caseDeadline!, Date.now() + 240_000);
|
||||
armDeadline();
|
||||
},
|
||||
env: { CLAUDE_CONFIG_DIR: configDir, GSTACK_HOME: stateDir, GSTACK_HEADLESS: '' },
|
||||
testName: opts.testName, runId: opts.runId,
|
||||
canUseTool: async (name, input, options) => {
|
||||
|
||||
@@ -5480,7 +5480,8 @@ export interface PlanSkillFloorObservation {
|
||||
* matcher still authenticates the pending question and native menu. */
|
||||
export function planFloorDXPane(visible: string, call: NativePlanQuestionCall): string | null {
|
||||
if (call.answered || call.failed || call.questions.length !== 1) return null;
|
||||
const text = stripPtyResidue(visible).replace(/\r+\n?/g, '\n');
|
||||
const text = stripPtyResidue(visible).replace(/\r+\n?/g, '\n')
|
||||
.replace(/((?:^|\n)Enter\s+to\s+select\s*·\s*↑\/↓\s+to\s+navigate\s*·\s*)ctrl\+g\s+to\s+edit\s+in[ \t]+[^\s·\x00-\x1f\x7f][^·\x00-\x1f\x7f]*?\s*·\s*(Esc\s+to\s+cancel\s*)$/, '$1$2');
|
||||
const headers = [...text.matchAll(/(?:^|\n)[\t ]*[☐□][^\n]*\n/g)];
|
||||
const header = headers.at(-1);
|
||||
if (!header) return null;
|
||||
|
||||
@@ -445,7 +445,7 @@ export function isInternalClaudeGitRequest(request: SourceRequest, commands: str
|
||||
// Require direct process ancestry AND the exact observed host prefix AND no
|
||||
// matching model request. A shell/model-issued unguarded Git call still fails.
|
||||
return request.tool === 'git' && !!request.ppid &&
|
||||
/(?:^|[/\\])claude(?:$|[/\\])/.test(request.parentExecutable || '') &&
|
||||
/(?:^|[/\\])claude(?:\.exe)?$/.test(request.parentExecutable || '') &&
|
||||
JSON.stringify(request.args.slice(0, hostPrefix.length)) === JSON.stringify(hostPrefix) &&
|
||||
!commands.some(command => command.includes('core.safecrlf=false') || command.includes('protocol.ext.allow=never'));
|
||||
}
|
||||
@@ -836,7 +836,7 @@ function skippedReviewOption(question: any): any {
|
||||
option[field] !== undefined && typeof option[field] !== 'string')) return [];
|
||||
const label = option.label.replace(/[‘’]/g, "'").replace(/`/g, '').replace(/^\s*(?:[A-Z]|\d+)[.)]\s*/i, '')
|
||||
.replace(/\s*\(recommended\)\s*$/i, '').trim()
|
||||
.replace(/^no\s*[,.:!?]\s*(?=(?:skip|decline|keep|leave|do not|don't)\b)/i, '');
|
||||
.replace(/^no\b[\s,:;.!?-]*(?=(?:skip|decline|keep|leave|do not|don't)\b)/i, '');
|
||||
const referentialRetention = /^(?:keep|leave)\s+(?:it|this|that|them|these)$/i.test(label);
|
||||
const preservation = option.description?.trim().replace(/`/g, '').match(/^(?:keep|leave|retain|preserve)\s+([^,;.!?]+)/i);
|
||||
const preservedObject = preservation?.[1].split(/\b(?:and|but|while)\b/i)[0]
|
||||
@@ -847,13 +847,15 @@ function skippedReviewOption(question: any): any {
|
||||
|| qualifiedIndexState.test(preservedObject));
|
||||
const description = (option.description ?? '').replace(/[‘’]/g, "'").trim();
|
||||
const declinesChange = /^(?:do not|don't)\s+(?:apply|change|edit|fix|refactor|extract|modify|touch|clear|remove|update|replace|add|migrate|implement|reuse|import)\b/i;
|
||||
const inapplicable = /^not applicable$/i.test(label)
|
||||
&& /^(?:choose this(?: option)?\s+)?(?:if|when) you are not (?:editing|changing|modifying)\b/i.test(description);
|
||||
const labelObject = label.match(/^(?:keep|leave)\s+(?:the\s+)?(.+)$/i)?.[1]
|
||||
.replace(/\s+(?:as[- ]is|unchanged|untouched|set)$/i, '');
|
||||
const preservationRank = referentialRetention ? describedRetention || declinesChange.test(description)
|
||||
: /^(?:keep|leave)\b.*\b(?:current|existing|unchanged|untouched|as[- ]is|alone|set|copies|copy|implementation|code|source)\b/i.test(label)
|
||||
|| !!labelObject && qualifiedIndexState.test(labelObject);
|
||||
const rank = /^(?:skip|decline)(?=$|\s|[,.!])/i.test(label) ? 3
|
||||
: declinesChange.test(label) ? 2
|
||||
: inapplicable || declinesChange.test(label) ? 2
|
||||
: preservationRank ? 1 : 0;
|
||||
if (!rank) return [];
|
||||
// A leading decline names rejected work. Classify later commitments rather
|
||||
@@ -868,6 +870,7 @@ function skippedReviewOption(question: any): any {
|
||||
word.replace(/([a-z])\1(?:ed|ing)$/, '$1')].some(form => actions.has(form));
|
||||
const changes = commitment.toLowerCase().split(/[,;\n]|[.!?](?:\s|$)|\b(?:and|but|then|while)\b/).some(part => {
|
||||
const clause = part.replace(/^[^a-z]+/, '')
|
||||
.replace(/^(?:the\s+)?(?:review|reuse|snapshot)\s+coverage\s+(?=(?:will|would|should|must|can|may|does|do)\b)/, '')
|
||||
.replace(/^(?:(?:this|that|the|selected|chosen)\s+(?:option|choice|selection)|i|we|you|it|(?:the\s+)?(?:source|code|route|worker|helper|parser|index(?:\s+flag)?))\s+/, '')
|
||||
.replace(/^(?:will|would|should|must|can|may|does|do)\s+/, '')
|
||||
.replace(/^(?:(?:please|also|still|just|now|be)\s+)+/, '');
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import * as path from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
interface StartContext {
|
||||
repo: string;
|
||||
@@ -71,24 +72,29 @@ function substitutionEnd(source: string, start: number): number {
|
||||
|
||||
/** Bounded inspection syntax, not a shell executor: quoted arguments and comments
|
||||
* cannot introduce commands. Unknown inspection forms fail closed. */
|
||||
function commands(source: string): { words: string[]; before: string; after: string; substitutions: number[] }[] {
|
||||
function commands(source: string): { words: string[]; before: string; after: string; substitutions: number[]; quoted: number[] }[] {
|
||||
const executable = withoutHereDocBodies(source);
|
||||
if (executable === undefined) return [];
|
||||
source = executable;
|
||||
const result: { words: string[]; before: string; after: string; substitutions: number[] }[] = [];
|
||||
let words: string[] = [], substitutions: number[] = [];
|
||||
let word = '', quote = '', before = '', expanded = false;
|
||||
const result: ReturnType<typeof commands> = [];
|
||||
let words: string[] = [], substitutions: number[] = [], quoted: number[] = [];
|
||||
let word = '', quote = '', before = '', expanded = false, wasQuoted = false;
|
||||
const flush = () => {
|
||||
if (word) { if (expanded) substitutions.push(words.length); words.push(word); }
|
||||
word = ''; expanded = false;
|
||||
if (word) {
|
||||
if (expanded) substitutions.push(words.length);
|
||||
if (wasQuoted) quoted.push(words.length);
|
||||
words.push(word);
|
||||
}
|
||||
word = ''; expanded = false; wasQuoted = false;
|
||||
};
|
||||
const end = (separator: string) => {
|
||||
flush(); if (words.length) result.push({ words, before, after: separator, substitutions });
|
||||
words = []; substitutions = []; before = separator;
|
||||
flush(); if (words.length) result.push({ words, before, after: separator, substitutions, quoted });
|
||||
words = []; substitutions = []; quoted = []; before = separator;
|
||||
};
|
||||
for (let i = 0; i < source.length; i++) {
|
||||
const char = source[i];
|
||||
if (char === '\\' && quote !== "'") {
|
||||
wasQuoted = true;
|
||||
if (quote === '"' && !/[$`"\\\n]/.test(source[i + 1] ?? '')) word += char;
|
||||
else { const escaped = source[++i] ?? ''; word += escaped === '$' ? '\0$' : escaped; }
|
||||
continue;
|
||||
@@ -99,7 +105,7 @@ function commands(source: string): { words: string[]; before: string; after: str
|
||||
expanded = true; word += source.slice(i, end + 1); i = end; continue;
|
||||
}
|
||||
if (quote) { if (char === quote) quote = ''; else word += quote === "'" && char === '$' ? '\0$' : char; continue; }
|
||||
if (char === '"' || char === "'") { quote = char; continue; }
|
||||
if (char === '"' || char === "'") { quote = char; wasQuoted = true; continue; }
|
||||
if (char === '#' && !word) { while (i < source.length && source[i] !== '\n') i++; end(';'); }
|
||||
else if (';|&()\n'.includes(char)) {
|
||||
const separator = (char === '&' || char === '|') && source[i + 1] === char ? char + source[++i] : char;
|
||||
@@ -113,18 +119,122 @@ function commands(source: string): { words: string[]; before: string; after: str
|
||||
return result;
|
||||
}
|
||||
|
||||
function executedCommands(source: string): ReturnType<typeof commands> {
|
||||
const calls = commands(source);
|
||||
return calls.flatMap(call => [call, ...call.substitutions.flatMap(index => {
|
||||
const word = call.words[index], begin = word.indexOf('$('), end = substitutionEnd(word, begin);
|
||||
return end < 0 ? [] : executedCommands(word.slice(begin + 2, end));
|
||||
})]);
|
||||
}
|
||||
|
||||
const basename = (word = '') => word.split(/[\\/]/).at(-1);
|
||||
const sourcePaths = (file: string) => file.includes('\\') || /^[A-Za-z]:\//.test(file) ? path.win32 : path.posix;
|
||||
type SourcePaths = ReturnType<typeof sourcePaths>;
|
||||
|
||||
function topLevelCommands(calls: ReturnType<typeof commands>): ReturnType<typeof commands> {
|
||||
const scopes: string[] = [], result: ReturnType<typeof commands> = [];
|
||||
for (const call of calls) {
|
||||
if (call.before === '(') scopes.push(')');
|
||||
if (!scopes.length && ['', ';'].includes(call.before)) {
|
||||
if (['exit', 'return', 'exec'].includes(call.words[0])) break;
|
||||
result.push(call);
|
||||
}
|
||||
let index = 0;
|
||||
while (!call.quoted.includes(index) && ['then', 'else', 'do'].includes(call.words[index])) index++;
|
||||
const head = call.quoted.includes(index) ? '' : call.words[index];
|
||||
const close = ({ if: 'fi', for: 'done', while: 'done', until: 'done', case: 'esac', '{': '}' } as Record<string, string>)[head];
|
||||
if (close) scopes.push(close);
|
||||
else if (['fi', 'done', 'esac', '}'].includes(head) && scopes.pop() !== head) return [];
|
||||
if (call.after === ')' && scopes.pop() !== ')') return [];
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function reviewStart(words: string[]): boolean {
|
||||
return words.length === 3 && basename(words[0]) === 'gstack-review-log'
|
||||
&& words[1] === '--start' && words[2] === 'review';
|
||||
}
|
||||
|
||||
function startTokenOutput(source: string, variables: Map<string, string | undefined>, token: string): string | undefined {
|
||||
const calls = commands(source);
|
||||
if (calls.length === 1 && reviewStart(calls[0].words) && !calls[0].before && !calls[0].after) return token;
|
||||
const first = calls[0];
|
||||
if (!first || first.before || first.words[0] !== 'echo' || first.words.length !== 2
|
||||
|| expandVariables(first.words[1], variables) !== token) return undefined;
|
||||
for (let i = 1; i < calls.length; i++) {
|
||||
if (calls[i - 1].after !== '|') return undefined;
|
||||
const words = calls[i].words;
|
||||
if (['head', 'tail'].includes(words[0]) && (words.length === 2 && words[1] === '-1'
|
||||
|| words.length === 3 && words[1] === '-n' && words[2] === '1')) continue;
|
||||
if (words.length !== 3 || words[0] !== 'grep' || !['-oE', '-Eo'].includes(words[1])
|
||||
|| !/^\[[A-Za-z0-9_.-]+\]\+$/.test(words[2])) return undefined;
|
||||
try { if (new RegExp(words[2]).exec(token)?.[0] !== token) return undefined; } catch { return undefined; }
|
||||
}
|
||||
return calls.at(-1)?.after ? undefined : token;
|
||||
}
|
||||
|
||||
function sameCallStartRead(source: string, file: string, expected: StartContext, token: string): boolean {
|
||||
const paths = sourcePaths(file), variables = new Map<string, string | undefined>([
|
||||
['GSTACK_HOME', expected.state], ['SLUG', expected.slug],
|
||||
]);
|
||||
if (paths.normalize(expected.directory) !== paths.join(expected.state, 'projects', expected.slug, '.review-starts')) return false;
|
||||
let cwd: string | undefined = expected.repo, started = false;
|
||||
for (const call of commands(source)) {
|
||||
if (!['', ';'].includes(call.before) || !['', ';'].includes(call.after)
|
||||
|| /^(?:if|then|else|elif|fi|for|while|until|do|done|case|esac|function|exit|return|exec|eval|source|\.|[{}!])$/.test(call.words[0])) return false;
|
||||
if (started && reader(call.words, operand => {
|
||||
const value = expandVariables(operand, variables);
|
||||
return value !== undefined && literalPath(value, cwd, paths) === file;
|
||||
})) return true;
|
||||
if (reviewStart(call.words)) {
|
||||
if (cwd !== expected.repo || variables.get('GSTACK_HOME') !== expected.state) return false;
|
||||
started = true;
|
||||
} else if (call.words.every(word => /^[A-Za-z_]\w*=/.test(word))) {
|
||||
for (const word of call.words) {
|
||||
const equal = word.indexOf('='), expression = word.slice(equal + 1);
|
||||
const substitution = /^\$\(([\s\S]*)\)$/.exec(expression);
|
||||
const value = substitution ? startTokenOutput(substitution[1], variables, token) : expandVariables(expression, variables);
|
||||
if (substitution && value === token && commands(substitution[1]).some(child => reviewStart(child.words))) {
|
||||
if (cwd !== expected.repo || variables.get('GSTACK_HOME') !== expected.state) return false;
|
||||
started = true;
|
||||
}
|
||||
variables.set(word.slice(0, equal), value);
|
||||
}
|
||||
} else if (call.words[0] === 'cd') {
|
||||
const args = call.words.slice(call.words[1] === '--' ? 2 : 1);
|
||||
const value = args.length === 1 ? expandVariables(args[0], variables) : undefined;
|
||||
cwd = value ? literalPath(value, cwd, paths) : undefined;
|
||||
} else if (['export', 'local', 'declare', 'readonly', 'unset', 'read', 'pushd', 'popd'].includes(call.words[0])) return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function successfulFinish(source: string, text: string, token: string, expected: StartContext): boolean {
|
||||
const calls = commands(source), topLevel = topLevelCommands(calls);
|
||||
const located = withDirectories(calls, expected.repo, sourcePaths(expected.repo), new Map([
|
||||
['GSTACK_HOME', expected.state], ['SLUG', expected.slug],
|
||||
]));
|
||||
const finishes = (words: string[]) => basename(words[0]) === 'gstack-review-log'
|
||||
&& words.some((word, index) => word === '--finish' && (words[index + 1] === token
|
||||
|| /^\$(?:[A-Za-z_]\w*|\{[A-Za-z_]\w*\})$/.test(words[index + 1] ?? '')));
|
||||
for (const call of topLevel) {
|
||||
if (finishes(call.words)) return true;
|
||||
if (call.quoted.includes(0) || call.words[0] !== 'if' || call.after !== ';' || !finishes(call.words.slice(1))) continue;
|
||||
const index = calls.indexOf(call), success = calls[index + 1], failure = calls[index + 2], exit = calls[index + 3], end = calls[index + 4];
|
||||
const context = located[index], argument = call.words[call.words.indexOf('--finish') + 1];
|
||||
if (context.cwd !== expected.repo || context.variables.get('GSTACK_HOME') !== expected.state
|
||||
|| expandVariables(argument, context.variables) !== token) continue;
|
||||
if (success?.words[0] !== 'then' || success.quoted.includes(0) || success.words[1] !== 'echo' || success.words.length !== 3
|
||||
|| /[$`\0]/.test(success.words[2]) || !text.split('\n').includes(success.words[2])
|
||||
|| failure?.words[0] !== 'else' || failure.quoted.includes(0) || failure.words[1] !== 'echo'
|
||||
|| exit?.words[0] !== 'exit' || !/^[1-9]\d*$/.test(exit.words[1] ?? '') || exit.words.length !== 2
|
||||
|| end?.words[0] !== 'fi' || end.quoted.includes(0) || end.words.length !== 1
|
||||
|| ![success, failure, exit, end].every(part => part.before === ';' && part.after === ';')) continue;
|
||||
if (!topLevel.some(read => calls.indexOf(read) > index + 4 && basename(read.words[0]) === 'gstack-review-read')) continue;
|
||||
for (const line of text.split('\n')) {
|
||||
let row: any;
|
||||
try { row = JSON.parse(line); } catch { continue; }
|
||||
const binding = row?.review_binding;
|
||||
if (row?.skill === 'review' && row.completed === true && row.wtree === expected.wtree
|
||||
&& binding?.state === 'verified' && binding.start_wtree === expected.wtree && binding.end_wtree === expected.wtree
|
||||
&& binding.started_at === expected.startedAt && binding.branch_id === createHash('sha256').update(expected.branch).digest('hex')) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Resolve source-spelled paths, independent of the machine replaying the trace.
|
||||
* Shell expansion is handled only by the discovery forms below, never guessed. */
|
||||
function literalPath(value: string, cwd: string | undefined, paths: SourcePaths): string | undefined {
|
||||
@@ -335,8 +445,13 @@ export function hasTrustedReviewStartRead(events: unknown[], expected: StartCont
|
||||
|
||||
for (const start of pairs) {
|
||||
if (start.tool !== 'Bash' || typeof start.input?.command !== 'string'
|
||||
|| !executedCommands(start.input.command).some(call => basename(call.words[0]) === 'gstack-review-log'
|
||||
&& call.words[1] === '--start' && call.words[2] === 'review')) continue;
|
||||
|| !topLevelCommands(commands(start.input.command)).some(call => {
|
||||
if (reviewStart(call.words)) return true;
|
||||
if (call.words.length !== 1 || !call.substitutions.includes(0)) return false;
|
||||
const assignment = /^[A-Za-z_]\w*=\$\(([\s\S]*)\)$/.exec(call.words[0]);
|
||||
const children = assignment ? commands(assignment[1]) : [];
|
||||
return children.length === 1 && !children[0].before && !children[0].after && reviewStart(children[0].words);
|
||||
})) continue;
|
||||
for (const token of start.text.match(/\b[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}\b/g) ?? []) {
|
||||
// Archived public Linux paths keep their spelling when free tests run on Windows.
|
||||
const paths = sourcePaths(expected.directory);
|
||||
@@ -345,24 +460,25 @@ export function hasTrustedReviewStartRead(events: unknown[], expected: StartCont
|
||||
&& typeof pair.input?.command === 'string'
|
||||
// The documented shell variable is valid too: the trusted final row's
|
||||
// started_at below binds its resolved value to this observed capture.
|
||||
&& executedCommands(pair.input.command).some(call => basename(call.words[0]) === 'gstack-review-log'
|
||||
&& call.words.some((word, index) => word === '--finish' && (call.words[index + 1] === token
|
||||
|| /^\$(?:[A-Za-z_]\w*|\{[A-Za-z_]\w*\})$/.test(call.words[index + 1] ?? '')))));
|
||||
&& successfulFinish(pair.input.command, pair.text, token, expected));
|
||||
if (!finish) continue;
|
||||
for (const read of pairs) {
|
||||
if (read.at <= start.returnedAt || read.returnedAt >= finish.at) continue;
|
||||
const sameCall = read === start;
|
||||
if ((!sameCall && read.at <= start.returnedAt) || read.returnedAt >= finish.at) continue;
|
||||
const command = typeof read.input?.command === 'string' ? read.input.command : '';
|
||||
const directRead = read.tool === 'Read' && typeof read.input?.file_path === 'string'
|
||||
&& literalPath(read.input.file_path, expected.repo, paths) === file;
|
||||
// Discovery may return the path only in stdout; bind its cat invocation
|
||||
// to that discovery instead of accepting unrelated reader/find words.
|
||||
const shellRead = read.tool === 'Bash' && inspectsFile(command, file, containsPath(read.text, file), expected);
|
||||
const shellRead = read.tool === 'Bash' && (sameCall ? sameCallStartRead(command, file, expected, token)
|
||||
: inspectsFile(command, file, containsPath(read.text, file), expected));
|
||||
if (!directRead && !shellRead) continue;
|
||||
for (const line of read.text.split('\n')) {
|
||||
// Native Read can prefix the single-line JSON file with a line number.
|
||||
const json = line.replace(/^\s*\d+[\t →]+(?=\{)/, '').trim();
|
||||
let record: any;
|
||||
try { record = JSON.parse(json); } catch { continue; }
|
||||
if (sameCall && start.text.indexOf(token) >= start.text.indexOf(line)) continue;
|
||||
if (record?.skill === 'review' && record.repo === expected.repo && record.branch === expected.branch
|
||||
&& record.wtree === expected.wtree && typeof expected.startedAt === 'string'
|
||||
&& record.started_at === expected.startedAt) return true;
|
||||
|
||||
@@ -15,5 +15,11 @@ export function asideDriveOptions(text: string): string[] {
|
||||
}
|
||||
}
|
||||
if (current !== undefined) options.push(current);
|
||||
return options.filter(option => /\bAside\b/i.test(option) && /\b(?:drive|driving|browse|browsing|navigate|click)\b/i.test(option));
|
||||
return options.filter(option => {
|
||||
const currentOffer = option.replace(
|
||||
/\b(?:I|we)(?:['’]ll| will) (?:re[- ]?ask|ask again)(?: (?:you|this question))? with (?:the )?(?:"[^"]+"|“[^”]+”|[\w -]+?) option(?: included)?\b/gi,
|
||||
'',
|
||||
);
|
||||
return /\bAside\b/i.test(option) && /\b(?:drive|driving|browse|browsing|navigate|click)\b/i.test(currentOffer);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -21,9 +21,9 @@
|
||||
* Each test lists the file patterns that, if changed, require the test to run.
|
||||
*/
|
||||
export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'shared-libs-review-path-eligibility': ['review/**', 'scripts/resolvers/shared-libs.ts', 'scripts/resolvers/review.ts', 'scripts/resolvers/review-army.ts', 'lib/review-evidence.ts', 'bin/gstack-review-log', 'bin/gstack-review-read', 'bin/gstack-wtree', 'test/helpers/shared-libs-eval-fixture.ts', 'test/skill-e2e-shared-libs-paths.test.ts', 'test/helpers/shared-libs-path-fixture.ts', 'test/shared-libs-fixture.test.ts', 'test/helpers/e2e-gate.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/agent-sdk-runner.ts', 'lib/claude-bin.ts', 'lib/eval-model.ts', 'test/fixtures/shared-libs-index-flags-*.json', 'test/shared-libs-revalidation-prompt.test.ts'],
|
||||
'shared-libs-review-index-flags': ['review/**', 'scripts/resolvers/shared-libs.ts', 'scripts/resolvers/review.ts', 'scripts/resolvers/review-army.ts', 'lib/review-evidence.ts', 'bin/gstack-review-log', 'bin/gstack-review-read', 'bin/gstack-wtree', 'test/helpers/shared-libs-eval-fixture.ts', 'test/skill-e2e-shared-libs-paths.test.ts', 'test/helpers/shared-libs-path-fixture.ts', 'test/shared-libs-fixture.test.ts', 'test/helpers/e2e-gate.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/agent-sdk-runner.ts', 'lib/claude-bin.ts', 'lib/eval-model.ts', 'test/fixtures/shared-libs-index-flags-*.json', 'test/shared-libs-revalidation-prompt.test.ts', 'test/fixtures/shared-libs-paths-max-turns-public.json'],
|
||||
'shared-libs-review-prior-coverage': ['review/**', 'scripts/resolvers/shared-libs.ts', 'scripts/resolvers/review.ts', 'scripts/resolvers/review-army.ts', 'lib/review-evidence.ts', 'bin/gstack-review-log', 'bin/gstack-review-read', 'bin/gstack-wtree', 'test/helpers/shared-libs-eval-fixture.ts', 'test/skill-e2e-shared-libs-paths.test.ts', 'test/helpers/shared-libs-path-fixture.ts', 'test/shared-libs-fixture.test.ts', 'test/helpers/e2e-gate.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/agent-sdk-runner.ts', 'lib/claude-bin.ts', 'lib/eval-model.ts', 'test/fixtures/shared-libs-index-flags-*.json', 'test/shared-libs-revalidation-prompt.test.ts'],
|
||||
'shared-libs-review-path-eligibility': ['review/**', 'scripts/resolvers/shared-libs.ts', 'scripts/resolvers/review.ts', 'scripts/resolvers/review-army.ts', 'lib/review-evidence.ts', 'bin/gstack-review-log', 'bin/gstack-review-read', 'bin/gstack-wtree', 'test/helpers/shared-libs-eval-fixture.ts', 'test/skill-e2e-shared-libs-paths.test.ts', 'test/helpers/shared-libs-path-fixture.ts', 'test/shared-libs-fixture.test.ts', 'test/helpers/e2e-gate.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/agent-sdk-runner.ts', 'lib/claude-bin.ts', 'lib/eval-model.ts', 'test/fixtures/shared-libs-index-flags-*.json', 'test/shared-libs-revalidation-prompt.test.ts', 'test/shared-libs-source-reads.test.ts', 'test/fixtures/shared-libs-resolved-reads-public.json'],
|
||||
'shared-libs-review-index-flags': ['review/**', 'scripts/resolvers/shared-libs.ts', 'scripts/resolvers/review.ts', 'scripts/resolvers/review-army.ts', 'lib/review-evidence.ts', 'bin/gstack-review-log', 'bin/gstack-review-read', 'bin/gstack-wtree', 'test/helpers/shared-libs-eval-fixture.ts', 'test/skill-e2e-shared-libs-paths.test.ts', 'test/helpers/shared-libs-path-fixture.ts', 'test/shared-libs-fixture.test.ts', 'test/helpers/e2e-gate.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/agent-sdk-runner.ts', 'lib/claude-bin.ts', 'lib/eval-model.ts', 'test/fixtures/shared-libs-index-flags-*.json', 'test/shared-libs-revalidation-prompt.test.ts', 'test/fixtures/shared-libs-paths-max-turns-public.json', 'test/shared-libs-source-reads.test.ts', 'test/fixtures/shared-libs-resolved-reads-public.json'],
|
||||
'shared-libs-review-prior-coverage': ['review/**', 'scripts/resolvers/shared-libs.ts', 'scripts/resolvers/review.ts', 'scripts/resolvers/review-army.ts', 'lib/review-evidence.ts', 'bin/gstack-review-log', 'bin/gstack-review-read', 'bin/gstack-wtree', 'test/helpers/shared-libs-eval-fixture.ts', 'test/skill-e2e-shared-libs-paths.test.ts', 'test/helpers/shared-libs-path-fixture.ts', 'test/shared-libs-fixture.test.ts', 'test/helpers/e2e-gate.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/agent-sdk-runner.ts', 'lib/claude-bin.ts', 'lib/eval-model.ts', 'test/fixtures/shared-libs-index-flags-*.json', 'test/shared-libs-revalidation-prompt.test.ts', 'test/shared-libs-source-reads.test.ts', 'test/fixtures/shared-libs-resolved-reads-public.json'],
|
||||
'shared-libs-codex-read-only': ['deslop-shared-libs/**', 'scripts/resolvers/shared-libs.ts', 'scripts/resolvers/index.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/shared-libs-eval-fixture.ts', 'test/helpers/codex-session-runner.ts', 'test/helpers/skill-fixture.ts', 'test/helpers/hermetic-env.ts', 'test/helpers/eval-budgets.ts', 'test/codex-e2e-shared-libs.test.ts', 'test/shared-libs-fixture.test.ts', 'test/helpers/e2e-gate.ts', 'hosts/codex.ts', 'hosts/define-host.ts', 'scripts/resolvers/constants.ts', 'test/fixtures/shared-libs-readonly-substitution-ci16358.json'],
|
||||
// Shared-code audit and scoped review lifecycle
|
||||
'shared-libs-read-only': ['deslop-shared-libs/**', 'scripts/resolvers/shared-libs.ts', 'scripts/resolvers/index.ts', 'test/helpers/shared-libs-eval-fixture.ts', 'test/skill-e2e-shared-libs.test.ts', 'test/shared-libs-fixture.test.ts', 'test/helpers/e2e-gate.ts', 'scripts/gen-skill-docs.ts', 'lib/claude-bin.ts', 'lib/eval-model.ts', 'test/fixtures/shared-libs-readonly-substitution-ci16358.json'],
|
||||
|
||||
@@ -0,0 +1,255 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { tmpdir } from 'node:os';
|
||||
|
||||
const root = join(import.meta.dir, '..');
|
||||
const source = (name: string) => readFileSync(join(root, 'land-and-deploy', name), 'utf8');
|
||||
|
||||
describe('land-and-deploy decision ordering', () => {
|
||||
test('binds local evidence and merge approval to the selected PR head', () => {
|
||||
const main = source('SKILL.md.tmpl');
|
||||
const merge = source('sections/merge-and-deploy.md.tmpl');
|
||||
expect(main).toContain('headRefOid');
|
||||
expect(main).toContain('LOCAL_TARGET_MISMATCH');
|
||||
expect(main.indexOf('gstack-diff-scope')).toBeLessThan(main.indexOf('{{SECTION:readiness-gate}}'));
|
||||
expect(merge.match(/--match-head-commit "\$PR_HEAD"/g)).toHaveLength(2);
|
||||
expect(merge).toContain('invalidate the approval');
|
||||
});
|
||||
|
||||
test('readback precedes the only direct fallback and includes queue membership', () => {
|
||||
const merge = source('sections/merge-and-deploy.md.tmpl');
|
||||
expect(merge).toContain('mergeQueueEntry { id state }');
|
||||
expect(merge.indexOf('gh api graphql')).toBeLessThan(merge.indexOf('gh pr merge "$MERGE_FLAG" --delete-branch'));
|
||||
expect(merge).not.toContain('Both fall through');
|
||||
expect(merge).not.toContain('never call `gh pr merge` a second time');
|
||||
expect(merge).toMatch(/never\s+replay a merge after MERGED/);
|
||||
});
|
||||
|
||||
test('deploy and URL evidence take precedence over docs-only skipping', () => {
|
||||
const merge = source('sections/merge-and-deploy.md.tmpl');
|
||||
expect(merge).toContain('explicit verification URL or an actually triggered deployment takes precedence');
|
||||
expect(source('SKILL.md.tmpl')).not.toContain('| SCOPE_DOCS only | Already skipped');
|
||||
expect(merge).not.toContain('gstack-diff-scope');
|
||||
});
|
||||
|
||||
test('true staging-first stops before merge instead of pretending production is held', () => {
|
||||
const readiness = source('sections/readiness-gate.md.tmpl');
|
||||
const merge = source('sections/merge-and-deploy.md.tmpl');
|
||||
expect(readiness).toContain('true staging-first');
|
||||
expect(readiness).toContain('production hold');
|
||||
expect(readiness).toContain('STOP before merge');
|
||||
expect(merge).toContain('production may already be live');
|
||||
expect(merge).not.toContain('production is untouched');
|
||||
expect(merge).not.toContain('Now deploying to production');
|
||||
});
|
||||
|
||||
test('blockers and failed canary evidence cannot be approved into a pass', () => {
|
||||
expect(source('sections/readiness-gate.md.tmpl')).toContain('Do not offer A or C with blockers');
|
||||
const main = source('SKILL.md.tmpl');
|
||||
expect(main).not.toContain('Mark it as healthy');
|
||||
expect(main).toContain('DEGRADED');
|
||||
expect(main).not.toContain('Inline fix: <yes');
|
||||
expect(main).toContain('MERGED — NO DEPLOY NEEDED');
|
||||
expect(main).toContain('MERGED (UNVERIFIED)');
|
||||
});
|
||||
|
||||
test('revert distinguishes merge parents and refuses an unknown rebase range', () => {
|
||||
const main = source('SKILL.md.tmpl');
|
||||
expect(main).toContain('git revert -m 1 "$MERGE_SHA" --no-edit');
|
||||
expect(main).toContain('exact landed commit range');
|
||||
expect(main).toContain('ROLLBACK PENDING');
|
||||
expect(main).not.toContain('git revert <merge-commit-sha> --no-edit');
|
||||
});
|
||||
});
|
||||
|
||||
describe('land-and-deploy native readback dispatcher', () => {
|
||||
const shell = source('sections/merge-and-deploy.md.tmpl').match(/```bash\n(READBACK=[\s\S]*?)\n```/)?.[1];
|
||||
const head = 'a'.repeat(40);
|
||||
const pr = (extra = {}) => ({
|
||||
state: 'OPEN', headRefOid: head, baseRefName: 'main', mergeCommit: null,
|
||||
autoMergeRequest: null, mergeQueueEntry: null, ...extra,
|
||||
});
|
||||
const dispatch = (pullRequest: unknown, env: Record<string, string> = {}, errors?: unknown[]) => {
|
||||
expect(shell).toBeDefined();
|
||||
return spawnSync('bash', ['-c', `gh() { printf '%s\n' "$@" >&2; printf '%s' "$READBACK_FIXTURE"; return "${'${READBACK_EXIT:-0}'}"; }\n${shell}`], {
|
||||
encoding: 'utf8', timeout: 5000,
|
||||
env: {
|
||||
PATH: process.env.PATH, REPO: 'owner/project', PR_NUMBER: '42', PR_HEAD: head, BASE_BRANCH: 'main',
|
||||
MERGE_ATTEMPT: 'none', MERGE_EXIT: '0', MERGE_ERROR: '', WAITED: 'false',
|
||||
READBACK_FIXTURE: JSON.stringify({ data: { repository: { pullRequest } }, ...(errors ? { errors } : {}) }),
|
||||
...env,
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
test('confirmed open without either request starts once', () => {
|
||||
const result = dispatch(pr());
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout.trim()).toBe('START');
|
||||
expect(result.stderr).toContain('owner=owner');
|
||||
expect(result.stderr).toContain('name=project');
|
||||
expect(result.stderr).toContain('number=42');
|
||||
});
|
||||
|
||||
for (const error of [
|
||||
'Auto-merge is not allowed for this repository',
|
||||
'Pull request is in clean status',
|
||||
'Pull request is in unstable status',
|
||||
]) {
|
||||
test(`only an unsuccessful auto attempt permits direct fallback: ${error}`, () => {
|
||||
const env = { MERGE_ATTEMPT: 'auto', MERGE_EXIT: '1', MERGE_ERROR: error };
|
||||
expect(dispatch(pr(), env).stdout.trim()).toBe('DIRECT');
|
||||
expect(dispatch(pr(), { ...env, MERGE_ATTEMPT: 'direct' }).stdout.trim()).toBe('STOP');
|
||||
expect(dispatch(pr(), { ...env, MERGE_EXIT: '0' }).stdout.trim()).toBe('STOP');
|
||||
});
|
||||
}
|
||||
|
||||
test('merged wins over the original cleanup error, without replay', () => {
|
||||
const result = dispatch(pr({ state: 'MERGED', mergeCommit: { oid: 'b'.repeat(40) } }), {
|
||||
MERGE_ATTEMPT: 'auto', MERGE_EXIT: '1', MERGE_ERROR: 'Pull request is in clean status',
|
||||
});
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout.trim()).toBe('MERGED');
|
||||
});
|
||||
|
||||
test('either auto request or queue entry waits even when auto request is null', () => {
|
||||
for (const active of [
|
||||
{ autoMergeRequest: { enabledAt: '2026-09-24T00:00:00Z' } },
|
||||
{ mergeQueueEntry: { id: 'queue-entry', state: 'QUEUED' } },
|
||||
]) {
|
||||
expect(dispatch(pr(active), { MERGE_ATTEMPT: 'auto', MERGE_EXIT: '1', MERGE_ERROR: 'Pull request is in clean status' }).stdout.trim()).toBe('WAIT');
|
||||
}
|
||||
});
|
||||
|
||||
test('removal after waiting never falls back or restarts the merge', () => {
|
||||
for (const attempt of ['none', 'auto', 'direct']) {
|
||||
expect(dispatch(pr(), { WAITED: 'true', MERGE_ATTEMPT: attempt, MERGE_EXIT: '1', MERGE_ERROR: 'Pull request is in clean status' }).stdout.trim()).toBe('STOP');
|
||||
}
|
||||
});
|
||||
|
||||
test('a changed head invalidates approval even while queued', () => {
|
||||
expect(dispatch(pr({ headRefOid: 'c'.repeat(40), mergeQueueEntry: { id: 'q', state: 'QUEUED' } })).stdout.trim()).toBe('HEAD_CHANGED');
|
||||
});
|
||||
|
||||
test('retargeting with the same head invalidates destination approval', () => {
|
||||
expect(dispatch(pr({ baseRefName: 'production' })).stdout.trim()).toBe('BASE_CHANGED');
|
||||
});
|
||||
|
||||
test('unexpected externally merged revisions stop reconciliation without replay', () => {
|
||||
for (const changed of [{ headRefOid: 'c'.repeat(40) }, { baseRefName: 'production' }]) {
|
||||
expect(dispatch(pr({ state: 'MERGED', mergeCommit: { oid: 'b'.repeat(40) }, ...changed })).stdout.trim()).toBe('MERGED_CHANGED');
|
||||
}
|
||||
});
|
||||
|
||||
test('permission failures and closed PRs stop rather than taking direct fallback', () => {
|
||||
expect(dispatch(pr(), { MERGE_ATTEMPT: 'auto', MERGE_EXIT: '1', MERGE_ERROR: 'permission denied' }).stdout.trim()).toBe('STOP');
|
||||
expect(dispatch(pr({ state: 'CLOSED' })).stdout.trim()).toBe('STOP');
|
||||
});
|
||||
|
||||
test('missing queue fields, GraphQL errors and failed queries are unknown, not absence', () => {
|
||||
const missingQueue = pr();
|
||||
delete (missingQueue as Partial<typeof missingQueue>).mergeQueueEntry;
|
||||
for (const result of [
|
||||
dispatch(missingQueue), dispatch(null), dispatch(pr(), {}, [{ message: 'unsupported field' }]),
|
||||
dispatch(pr(), { READBACK_EXIT: '1' }), dispatch(pr(), { READBACK_FIXTURE: 'not JSON' }),
|
||||
]) {
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stdout.trim()).toBe('');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('land-and-deploy rollback command fixtures', () => {
|
||||
test('the authored mainline revert removes only the PR side of a real merge', () => {
|
||||
const cwd = mkdtempSync(join(tmpdir(), 'land-revert-'));
|
||||
const git = (...args: string[]) => {
|
||||
const result = spawnSync('git', args, { cwd, encoding: 'utf8', timeout: 5000 });
|
||||
expect({ args, status: result.status, stderr: result.status ? result.stderr : '' }).toEqual({ args, status: 0, stderr: '' });
|
||||
return result.stdout.trim();
|
||||
};
|
||||
try {
|
||||
git('init', '-b', 'main');
|
||||
writeFileSync(join(cwd, 'base.txt'), 'base\n');
|
||||
git('add', '.');
|
||||
git('commit', '-m', 'seed rollback fixture');
|
||||
git('switch', '-c', 'feature');
|
||||
writeFileSync(join(cwd, 'feature.txt'), 'feature\n');
|
||||
git('add', '.');
|
||||
git('commit', '-m', 'fixture feature');
|
||||
git('switch', 'main');
|
||||
writeFileSync(join(cwd, 'independent.txt'), 'base change\n');
|
||||
git('add', '.');
|
||||
git('commit', '-m', 'independent base change');
|
||||
const baseTree = git('rev-parse', 'HEAD^{tree}');
|
||||
git('merge', '--no-ff', 'feature', '-m', 'fixture merge');
|
||||
const mergeSha = git('rev-parse', 'HEAD');
|
||||
const wrong = spawnSync('git', ['revert', mergeSha, '--no-edit'], { cwd, encoding: 'utf8', timeout: 5000 });
|
||||
expect(wrong.status).not.toBe(0);
|
||||
expect(wrong.stderr).toContain('no -m option');
|
||||
const command = source('SKILL.md.tmpl').match(/`(git revert -m 1 "\$MERGE_SHA" --no-edit)`/)?.[1];
|
||||
expect(command).toBeDefined();
|
||||
const repaired = spawnSync('bash', ['-c', command!], { cwd, encoding: 'utf8', timeout: 5000, env: { ...process.env, MERGE_SHA: mergeSha } });
|
||||
expect(repaired.status).toBe(0);
|
||||
expect(git('rev-parse', 'HEAD^{tree}')).toBe(baseTree);
|
||||
expect(readFileSync(join(cwd, 'independent.txt'), 'utf8')).toBe('base change\n');
|
||||
} finally {
|
||||
rmSync(cwd, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('land-and-deploy selected target and CLI contracts', () => {
|
||||
test('evidence check and run consume the same exact project command', () => {
|
||||
const readiness = source('sections/readiness-gate.md.tmpl');
|
||||
expect(readiness).toContain('--expect-cmd "$TEST_COMMAND"');
|
||||
expect(readiness).toContain('gstack-evidence run --label tests -- "$TEST_COMMAND"');
|
||||
expect(readiness).not.toContain("run --label tests -- 'bun test");
|
||||
});
|
||||
|
||||
test('rollback binds fresh evidence to production and staging unavailability follows choice A', () => {
|
||||
const main = source('SKILL.md.tmpl');
|
||||
expect(main).toContain('`ROLLBACK=true`, `TARGET=production`');
|
||||
expect(main).toContain('`DEPLOY_SHA=REVERT_SHA`, and reset production deployment/health');
|
||||
expect(main).toContain('Staging choice A returns to its production route');
|
||||
expect(main).toContain('C goes to Step 9 without claiming');
|
||||
expect(source('sections/first-run-validation.md.tmpl')).not.toMatch(/still be able to deploy|still deploy through GitHub/);
|
||||
});
|
||||
test('every post-selection PR call targets the pinned PR and repository', () => {
|
||||
const main = source('SKILL.md.tmpl');
|
||||
const sources = [main.slice(main.indexOf('PR_JSON=')), ...['first-run-validation', 'readiness-gate', 'merge-and-deploy'].map(name => source(`sections/${name}.md.tmpl`))];
|
||||
const calls = sources.flatMap(text => text.split('\n').filter(line => /gh pr (view|checks|merge) /.test(line)));
|
||||
expect(calls.length).toBeGreaterThan(5);
|
||||
for (const call of calls.filter(line => !line.includes('does not expose'))) {
|
||||
expect(call).toContain('"$PR_NUMBER"');
|
||||
expect(call).toContain('--repo "$REPO"');
|
||||
}
|
||||
});
|
||||
|
||||
test('checks use native fields and missing queries do not count as no required checks', () => {
|
||||
const text = [source('SKILL.md.tmpl'), source('sections/first-run-validation.md.tmpl'), source('sections/readiness-gate.md.tmpl')].join('\n');
|
||||
for (const line of text.split('\n').filter(line => line.includes('gh pr checks') && line.includes('--json'))) {
|
||||
const fields = line.match(/--json ([a-zA-Z,]+)/)![1].split(',');
|
||||
expect(fields.every(field => ['bucket', 'completedAt', 'description', 'event', 'link', 'name', 'startedAt', 'state', 'workflow'].includes(field))).toBe(true);
|
||||
}
|
||||
expect(text).toContain('Auth/network/schema');
|
||||
expect(text).toContain('never "no required checks"');
|
||||
});
|
||||
|
||||
test('local head, branch, and worktree mismatches fail closed without checkout', () => {
|
||||
const shell = source('SKILL.md.tmpl').match(/```bash\n(LOCAL_HEAD=[\s\S]*?)\n```/)?.[1].split('git fetch')[0];
|
||||
expect(shell).toBeDefined();
|
||||
const fakeGit = `git() { case "$1" in rev-parse) printf '%s' "$TEST_HEAD";; branch) printf '%s' "$TEST_BRANCH";; status) printf '%s' "$TEST_STATUS";; *) return 99;; esac; }`;
|
||||
for (const [extra, expected] of [
|
||||
[{}, 0], [{ TEST_HEAD: 'other-sha' }, 1], [{ TEST_BRANCH: 'other-branch' }, 1], [{ TEST_STATUS: ' M app.ts' }, 1],
|
||||
] as const) {
|
||||
const result = spawnSync('bash', ['-c', `${fakeGit}\n${shell}`], {
|
||||
encoding: 'utf8', timeout: 5000,
|
||||
env: { PATH: process.env.PATH, TEST_HEAD: 'approved', PR_HEAD: 'approved', TEST_BRANCH: 'feature', HEAD_BRANCH: 'feature', TEST_STATUS: '', ...extra },
|
||||
});
|
||||
expect(result.status).toBe(expected);
|
||||
expect(result.stdout.trim()).toBe(expected ? 'LOCAL_TARGET_MISMATCH' : '');
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -7,8 +7,8 @@
|
||||
* on-demand section — prompt-token-load-reduction carve; the skeleton keeps
|
||||
* only the STOP-Read pointer). After ANY non-zero `gh pr merge`, the skill
|
||||
* must query authoritative PR state via
|
||||
* `gh pr view --json state,mergeCommit,mergedAt,mergedBy` and
|
||||
* branch on the result instead of retrying `gh pr merge` (cli/cli#3442,
|
||||
* GraphQL (including queue membership) and
|
||||
* branch on the result instead of blindly retrying `gh pr merge` (cli/cli#3442,
|
||||
* cli/cli#13380).
|
||||
*
|
||||
* Static invariants pin:
|
||||
@@ -20,7 +20,7 @@
|
||||
* - MERGED branch: continues to §4a CI watch
|
||||
* - OPEN branch: checks autoMergeRequest before treating as failure
|
||||
* - CLOSED branch: STOPs
|
||||
* - Hard rule: never retry `gh pr merge`
|
||||
* - Hard rule: no replay after MERGED; one guarded auto-to-direct fallback
|
||||
* - .tmpl edit propagated to generated SKILL.md (atomic per T-Codex-3)
|
||||
*/
|
||||
import { describe, expect, test } from "bun:test";
|
||||
@@ -60,9 +60,11 @@ describe("PR #1620 §4a-postfail in land-and-deploy template", () => {
|
||||
expect(body).toMatch(/cli\/cli#13380/);
|
||||
});
|
||||
|
||||
test("Authoritative state query uses gh pr view --json", () => {
|
||||
test("Authoritative state query includes auto request and queue membership", () => {
|
||||
const body = readTmpl();
|
||||
expect(body).toMatch(/gh pr view --json state,mergeCommit,mergedAt,mergedBy/);
|
||||
expect(body).toMatch(/gh api graphql/);
|
||||
expect(body).toContain('state headRefOid baseRefName mergedAt mergeCommit { oid }');
|
||||
expect(body).toContain('autoMergeRequest { enabledAt } mergeQueueEntry { id state }');
|
||||
});
|
||||
|
||||
test("All three state branches named: MERGED, OPEN, CLOSED", () => {
|
||||
@@ -74,7 +76,7 @@ describe("PR #1620 §4a-postfail in land-and-deploy template", () => {
|
||||
|
||||
test("MERGED branch captures merge SHA via mergeCommit.oid", () => {
|
||||
const body = readTmpl();
|
||||
expect(body).toMatch(/gh pr view --json mergeCommit -q \.mergeCommit\.oid/);
|
||||
expect(body).toMatch(/jq -er '\.data\.repository\.pullRequest\.mergeCommit\.oid'/);
|
||||
});
|
||||
|
||||
test("MERGED worktree cleanup is non-destructive (uncommitted-work guard)", () => {
|
||||
@@ -96,7 +98,7 @@ describe("PR #1620 §4a-postfail in land-and-deploy template", () => {
|
||||
// base checkout's origin, because fork branches do not exist in origin.
|
||||
test("MERGED branch reconciles the PR head repository (ls-remote, confirm-first delete)", () => {
|
||||
const body = readTmpl();
|
||||
expect(body).toMatch(/gh pr view --json headRepositoryOwner,headRepository,headRefName/);
|
||||
expect(body).toMatch(/gh pr view "\$PR_NUMBER" --repo "\$REPO" --json headRepositoryOwner,headRepository,headRefName/);
|
||||
// gh leaves .headRepository.nameWithOwner empty (verified live, gh 2.83) —
|
||||
// owner/name is composed from headRepositoryOwner.login + headRepository.name.
|
||||
expect(body).toMatch(/headRepositoryOwner\.login/);
|
||||
@@ -120,8 +122,8 @@ describe("PR #1620 §4a-postfail in land-and-deploy template", () => {
|
||||
|
||||
test("OPEN branch checks autoMergeRequest before treating as failure", () => {
|
||||
const body = readTmpl();
|
||||
expect(body).toMatch(/gh pr view --json autoMergeRequest/);
|
||||
expect(body).toMatch(/auto-merge is enabled or merge queue is in use/);
|
||||
expect(body).toMatch(/autoMergeRequest != null or \.mergeQueueEntry != null/);
|
||||
expect(body).toMatch(/auto-merge is enabled or\s+merge queue is in use/);
|
||||
});
|
||||
|
||||
test("CLOSED branch STOPs", () => {
|
||||
@@ -129,9 +131,12 @@ describe("PR #1620 §4a-postfail in land-and-deploy template", () => {
|
||||
expect(body).toMatch(/state == "CLOSED".*[\s\S]{0,200}STOP/);
|
||||
});
|
||||
|
||||
test("Hard rule: never retry gh pr merge after non-zero exit", () => {
|
||||
test("Hard rule: no replay after MERGED and only one guarded direct fallback", () => {
|
||||
const body = readTmpl();
|
||||
expect(body).toMatch(/never call `gh pr merge` a second time/);
|
||||
expect(body).toMatch(/never\s+replay a merge after MERGED/);
|
||||
expect(body).toContain('one direct fallback');
|
||||
expect(body).toContain('There is no fallback from a direct attempt');
|
||||
expect(body).toContain('readback has confirmed OPEN, no auto request and no queue entry');
|
||||
});
|
||||
|
||||
test("Generated merge-and-deploy.md carries the §4a-postfail section (atomic regen per T-Codex-3)", () => {
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { isBuiltin } from 'node:module';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { computePaidCaseSelection } from '../scripts/test-paid-shards';
|
||||
import { FREE_ONLY_PR_FILES, PR_PROFILE_CASE_IDS, selectPrProfile, type PrProfileMaps } from '../scripts/test-pr-profile';
|
||||
import { normalizeRelativePath } from '../scripts/test-strict-output';
|
||||
import { normalizeRelativePath as freeNormalize } from '../scripts/test-free-shards';
|
||||
import { PAID_TEST_GLOBS } from './helpers/paid-test-set';
|
||||
import { E2E_TIERS, LLM_JUDGE_TOUCHFILES } from './helpers/touchfiles-data';
|
||||
import { selectTests } from './helpers/test-selection';
|
||||
import { workflowJudgeDependencies } from './helpers/workflow-judge-cache';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
function runnerDependencies(root: string, entries: string[]): string[] {
|
||||
const seen = new Set<string>();
|
||||
const parser = new Bun.Transpiler({ loader: 'tsx' });
|
||||
const visit = (file: string) => {
|
||||
file = fs.realpathSync(file);
|
||||
const relative = path.relative(root, file).split(path.sep).join('/');
|
||||
if (relative.startsWith('../') || path.isAbsolute(relative)) throw new Error('Dependency outside checkout');
|
||||
if (seen.has(relative)) return;
|
||||
seen.add(relative);
|
||||
const source = fs.readFileSync(file, 'utf8').replace(/^#![^\n]*(?:\n|$)/, '\n');
|
||||
let audited = source;
|
||||
if (relative === 'test/helpers/test-selection.ts') {
|
||||
if (createHash('sha256').update(source).digest('hex') !== '4d2fbcb6249e8d22453d25bfe9b18ee0f4568bbec071918675c38a455d4e1e08') {
|
||||
throw new Error('Re-audit the historical touchfile map loader before excluding its computed import');
|
||||
}
|
||||
audited = source.replace('`const m = await import(${JSON.stringify(dataPath)});`,', "'',");
|
||||
}
|
||||
const compiled = parser.transformSync(audited);
|
||||
const imports = [...parser.scanImports(source), ...parser.scanImports(compiled)];
|
||||
const nonliteral = compiled.replace(/(?<![\w.])(?:import|require)\s*\(\s*(['"])[^'"\n]+\1\s*\)/g, '');
|
||||
if (/\bimport\s*\(|\brequire\s*[([;,.?)]|\bcreateRequire\b/.test(nonliteral)) {
|
||||
throw new Error(`Unresolved module loading in ${relative}`);
|
||||
}
|
||||
for (const entry of imports) {
|
||||
if (isBuiltin(entry.path) || entry.path.startsWith('bun:')) continue;
|
||||
visit(Bun.resolveSync(entry.path, path.dirname(file)));
|
||||
}
|
||||
};
|
||||
for (const entry of entries) visit(path.join(root, entry));
|
||||
return [...seen].sort();
|
||||
}
|
||||
|
||||
describe('paid/free dependency boundary', () => {
|
||||
test('shared normalization preserves the free export and path bytes', () => {
|
||||
expect(freeNormalize).toBe(normalizeRelativePath);
|
||||
for (const [input, expected] of [
|
||||
['', ''], ['test/example.test.ts', 'test/example.test.ts'],
|
||||
['test\\example.test.ts', 'test/example.test.ts'],
|
||||
['.\\test\\..\\test\\example.test.ts', './test/../test/example.test.ts'],
|
||||
['C:\\work\\test\\example.test.ts', 'C:/work/test/example.test.ts'],
|
||||
]) expect(normalizeRelativePath(input)).toBe(expected);
|
||||
});
|
||||
|
||||
test('the paid runner and all paid test static closures exclude every free-only exemption', () => {
|
||||
const runner = runnerDependencies(ROOT, ['scripts/test-paid-shards.ts', 'scripts/eval-select.ts']);
|
||||
const paid = [...new Set(PAID_TEST_GLOBS.flatMap(pattern => [...new Bun.Glob(pattern).scanSync(ROOT)]))];
|
||||
expect(paid.length).toBeGreaterThan(0);
|
||||
const all = workflowJudgeDependencies(ROOT, paid);
|
||||
for (const dependencies of [runner, all]) {
|
||||
expect(dependencies).toContain('scripts/test-strict-output.ts');
|
||||
expect(dependencies).toContain('test/helpers/test-selection.ts');
|
||||
expect(dependencies).not.toContain('scripts/eval-flake-rank.ts');
|
||||
for (const freeOnly of FREE_ONLY_PR_FILES) expect(dependencies).not.toContain(freeOnly);
|
||||
}
|
||||
expect(runnerDependencies(ROOT, ['scripts/eval-flake-rank.ts'])).toContain('scripts/test-free-shards.ts');
|
||||
});
|
||||
|
||||
test('the runner audit follows re-exports, literal imports and requires; unknown loading fails closed', () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'paid-imports-'));
|
||||
const write = (file: string, source: string) => fs.writeFileSync(path.join(root, file), source);
|
||||
try {
|
||||
write('entry.ts', "export * from './bridge';");
|
||||
write('bridge.ts', "export { value } from './free';");
|
||||
write('free.ts', 'export const value = 1;');
|
||||
expect(runnerDependencies(root, ['entry.ts'])).toContain('free.ts');
|
||||
for (const source of ["await import('./free');", "require('./free');", "module.require('./free');",
|
||||
"import './free';", "await import('./' + 'free');"]) {
|
||||
write('bridge.ts', source);
|
||||
expect(runnerDependencies(root, ['entry.ts'])).toContain('free.ts');
|
||||
}
|
||||
for (const source of [
|
||||
"const target = './free'; await import(target);",
|
||||
"const target = './free'; await import /* indirect */ (target);",
|
||||
"const target = 'free'; await import(`./${target}`);",
|
||||
"const target = './free'; require(target);",
|
||||
"const load = require; load('./free');",
|
||||
"const target = process.argv[2]; module.require(target);",
|
||||
"import { createRequire as loader } from 'node:module'; loader(import.meta.url)('./free');",
|
||||
"const target = process.argv[2]; await import('./' + target);",
|
||||
"import './missing';",
|
||||
'export {',
|
||||
]) {
|
||||
write('bridge.ts', source);
|
||||
expect(() => runnerDependencies(root, ['entry.ts']), source).toThrow();
|
||||
}
|
||||
fs.mkdirSync(path.join(root, 'test/helpers'), { recursive: true });
|
||||
write('test/helpers/test-selection.ts', fs.readFileSync(path.join(ROOT, 'test/helpers/test-selection.ts'), 'utf8') + '\n');
|
||||
expect(() => runnerDependencies(root, ['test/helpers/test-selection.ts'])).toThrow('Re-audit');
|
||||
} finally { fs.rmSync(root, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
test('only audited free-only dependencies avoid unknown-dependency fallback', () => {
|
||||
for (const file of [...FREE_ONLY_PR_FILES, 'scripts\\test-free-shards.ts']) {
|
||||
const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: [file] });
|
||||
expect(result.coverage?.mode).toBe('pr');
|
||||
expect(result.coverage?.unknownFiles).toEqual([]);
|
||||
expect(result.selection).toEqual({ e2e: [], judges: [] });
|
||||
}
|
||||
for (const file of [
|
||||
'scripts/new-helper.ts', 'scripts/free-test-durations.json', 'scripts/eval-flake-rank.ts',
|
||||
'lib/new-runtime.ts', 'test/helpers/new-helper.ts', 'test/fixtures/new-fixture.ts',
|
||||
'.github/workflows/new-free-tests.yml',
|
||||
]) {
|
||||
const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: [FREE_ONLY_PR_FILES[0], file] });
|
||||
expect(result.coverage?.mode, file).toBe('full-fallback');
|
||||
expect(result.coverage?.unknownFiles).toContain(file);
|
||||
expect(result.selection.e2e).toEqual(Object.keys(E2E_TIERS).filter(id => E2E_TIERS[id] === 'gate').sort());
|
||||
expect(result.selection.judges).toEqual(Object.keys(LLM_JUDGE_TOUCHFILES).sort());
|
||||
}
|
||||
const missingBase = computePaidCaseSelection({ profile: 'pr', env: { EVALS_BASE: 'missing-boundary-ref' },
|
||||
changedFiles: ['package.json'] });
|
||||
expect(missingBase.coverage?.mode).toBe('full-fallback');
|
||||
});
|
||||
|
||||
test('mapped shared infrastructure retains every fast PR case and judge, as at 06ed920', () => {
|
||||
for (const file of ['scripts/test-pr-profile.ts', 'scripts/test-strict-output.ts',
|
||||
'scripts/test-paid-shards.ts', 'test/helpers/eval-budgets.ts']) {
|
||||
const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles: [FREE_ONLY_PR_FILES[0], file] });
|
||||
expect(result.coverage?.mode, file).toBe('pr');
|
||||
expect(result.coverage?.unknownFiles).toEqual([]);
|
||||
expect(result.selection.e2e).toEqual([...PR_PROFILE_CASE_IDS].sort());
|
||||
expect(result.selection.judges).toEqual(Object.keys(LLM_JUDGE_TOUCHFILES).sort());
|
||||
}
|
||||
});
|
||||
|
||||
test('PR 2956 retains its mapped paid checks without the free runner restoring broad cases', () => {
|
||||
const changedFiles = [
|
||||
'plan-eng-review/SKILL.md', 'plan-eng-review/SKILL.md.tmpl',
|
||||
'plan-eng-review/sections/review-sections.md', 'plan-eng-review/sections/review-sections.md.tmpl',
|
||||
'scripts/test-free-shards.ts', 'scripts/test-strict-output.ts',
|
||||
'test/eng-scope-entry-ap.test.ts', 'test/helpers/plan-floor-review.ts',
|
||||
'test/plan-floor-permission.test.ts', 'test/plan-floor-review.test.ts',
|
||||
'test/plan-review-cases.test.ts', 'test/plan-scope-recovery-av.test.ts',
|
||||
'test/strict-output-formats.test.ts',
|
||||
];
|
||||
const result = computePaidCaseSelection({ profile: 'pr', env: {}, changedFiles });
|
||||
const withoutFree = computePaidCaseSelection({ profile: 'pr', env: {},
|
||||
changedFiles: changedFiles.filter(file => file !== 'scripts/test-free-shards.ts') });
|
||||
expect(result).toEqual(withoutFree);
|
||||
expect(result.coverage?.mode).toBe('pr');
|
||||
expect(result.coverage?.unknownFiles).toEqual([]);
|
||||
expect(result.selection.e2e).toEqual([...PR_PROFILE_CASE_IDS].sort());
|
||||
expect(result.selection.judges).toEqual(Object.keys(LLM_JUDGE_TOUCHFILES).sort());
|
||||
expect(result.coverage?.deferred.some(item => item.id === 'qa-only-no-fix')).toBe(true);
|
||||
expect(result.coverage?.needsFullValidation).toBe(false);
|
||||
});
|
||||
|
||||
test('mapped E2E, judge and global dependencies still win over the free-only exemption', () => {
|
||||
const file = FREE_ONLY_PR_FILES[0];
|
||||
for (const category of ['e2e', 'judge', 'global']) {
|
||||
const maps: PrProfileMaps = {
|
||||
e2eTouchfiles: { fast: category === 'e2e' ? [file] : ['skill/**'], broad: ['other/**'] },
|
||||
tiers: { fast: 'gate', broad: 'gate' },
|
||||
judgeTouchfiles: { quality: category === 'judge' ? [file] : ['skill/**'] },
|
||||
globalTouchfiles: category === 'global' ? [file] : [],
|
||||
};
|
||||
const e2e = selectTests([file], maps.e2eTouchfiles, maps.globalTouchfiles);
|
||||
const judges = selectTests([file], maps.judgeTouchfiles, maps.globalTouchfiles);
|
||||
const result = selectPrProfile({ changedFiles: [file], maps, profile: ['fast'],
|
||||
selectedE2E: e2e.selected, selectedJudges: judges.selected });
|
||||
expect(result.mode).toBe('pr');
|
||||
expect(result.e2e).toEqual(category === 'judge' ? [] : ['fast']);
|
||||
expect(result.judges).toEqual(category === 'e2e' ? [] : ['quality']);
|
||||
}
|
||||
const real = computePaidCaseSelection({ profile: 'pr', env: {},
|
||||
changedFiles: [file, 'plan-ceo-review/SKILL.md.tmpl'] });
|
||||
expect(real.coverage?.mode).toBe('pr');
|
||||
expect(real.selection.e2e).toContain('plan-ceo-review-benefits');
|
||||
expect(real.selection.judges).toContain('plan-ceo-review/SKILL.md modes');
|
||||
expect(real.selection.e2e?.every(id => PR_PROFILE_CASE_IDS.includes(id as typeof PR_PROFILE_CASE_IDS[number]))).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -47,7 +47,8 @@ const timeoutExpressions = (file: string) => [...read(file).matchAll(
|
||||
test('source allowances retain all captures, cases, and finalization grace', () => {
|
||||
const auq = read(AUQ_CONSISTENCY_RETRY_BUDGET.file);
|
||||
expect(auq).toContain("const N_RUNS = Number(process.env.AUQ_CONSISTENCY_RUNS ?? '3')");
|
||||
expect(auq).toContain('for (let i = 0; i < N_RUNS; i++)');
|
||||
expect(auq).toContain('Promise.allSettled(Array.from({ length: N_RUNS },');
|
||||
expect(auq).toContain('for (const [i, capture] of captures.entries())');
|
||||
expect(auq).toContain('N_RUNS * CAPTURE_MS + 60_000');
|
||||
expect(AUQ_CONSISTENCY_RETRY_BUDGET.testMs).toBe(960_000);
|
||||
expect(timeoutExpressions('test/codex-e2e-plan-format.test.ts')).toEqual(Array(4).fill('CAPTURE_LONG_MS'));
|
||||
|
||||
@@ -212,6 +212,7 @@ process.stdin.on('data',async data=>{
|
||||
process.stdout.write('\x1b[2J\x1b[HBOARD_ACKNOWLEDGED\r\n');
|
||||
});
|
||||
process.on('SIGINT',()=>process.exit(0));process.on('SIGTERM',()=>process.exit(0));
|
||||
process.stdout.write('PTY_READY:'+process.env.PROBE_EVENTS+'\x1b[2J\x1b[H');
|
||||
`, { mode: 0o755 });
|
||||
fs.writeFileSync(worker, `
|
||||
import * as fs from 'node:fs';
|
||||
@@ -220,6 +221,7 @@ if(resolveClaudeBinary()!==${JSON.stringify(fake)})throw Error('fake CLI binding
|
||||
let picks=0, originalError;
|
||||
try {
|
||||
const observation=await runPlanSkillCounting({skillName:'plan-design-review',slashCommand:'/plan-design-review',
|
||||
startupReadyMarker:${JSON.stringify('PTY_READY:' + events)},
|
||||
followUpPrompt:'# Owned board ordering control',observeSetupQuestions:true,
|
||||
bindDesignBoardState:${mode !== 'unbound'},
|
||||
env:${JSON.stringify({ PROBE_EVENTS: events, PROBE_HELP_FRAME: TOOL_HELP_FRAME, PROBE_STATE_MODULE: pathToFileURL(path.resolve(import.meta.dir, '../design/src/daemon-state.ts')).href, DESIGN_DAEMON_STATE_FILE: path.join(root, 'foreign', 'design.json') })},
|
||||
|
||||
@@ -53,11 +53,12 @@ process.stdin.on('data', data => {
|
||||
}, 30);
|
||||
});
|
||||
process.stdin.resume();
|
||||
process.stdout.write('PTY_READY:'+process.env.PROBE_INPUTS+'\x1b[2J\x1b[H');
|
||||
`);
|
||||
fs.chmodSync(fake, 0o755);
|
||||
fs.writeFileSync(worker, `import { runPlanSkillCounting, ceoStep0Boundary, ceoFirstReviewAUQ } from ${JSON.stringify(runner)};\n` +
|
||||
`if (process.env.BROWSE_TERMINAL_BINARY !== ${JSON.stringify(fake)}) throw new Error('fake CLI not selected');\n` +
|
||||
`const result = await runPlanSkillCounting({skillName:'plan-design-review',slashCommand:'/plan-design-review',followUpPrompt:'# Empty review fixture',expectedPlanPath:${JSON.stringify(report)},isLastStep0AUQ:ceoStep0Boundary,isFirstReviewAUQ:ceoFirstReviewAUQ,reviewCountCeiling:8,timeoutMs:33000,env:${JSON.stringify({PROBE_PLAN:report,PROBE_INPUTS:inputs,PROBE_PID:pidFile,PROBE_REPORT:REPORT,PROBE_SETUP_CALLS:JSON.stringify(questions === 'setup' ? setupCapture.calls : [])})}});\n` +
|
||||
`const result = await runPlanSkillCounting({skillName:'plan-design-review',slashCommand:'/plan-design-review',followUpPrompt:'# Empty review fixture',expectedPlanPath:${JSON.stringify(report)},isLastStep0AUQ:ceoStep0Boundary,isFirstReviewAUQ:ceoFirstReviewAUQ,reviewCountCeiling:8,timeoutMs:33000,startupReadyMarker:${JSON.stringify('PTY_READY:' + inputs)},env:${JSON.stringify({PROBE_PLAN:report,PROBE_INPUTS:inputs,PROBE_PID:pidFile,PROBE_REPORT:REPORT,PROBE_SETUP_CALLS:JSON.stringify(questions === 'setup' ? setupCapture.calls : [])})}});\n` +
|
||||
`await Bun.write(${JSON.stringify(resultFile)},JSON.stringify(result));\n`);
|
||||
const child = Bun.spawn([process.execPath, worker], {
|
||||
env: { ...process.env, EVALS_HERMETIC:'1', EVALS_RUN_ID:'', BROWSE_TERMINAL_BINARY:fake },
|
||||
|
||||
@@ -546,6 +546,7 @@ process.on('SIGINT', () => {
|
||||
process.exit(0);
|
||||
});
|
||||
process.stdin.resume();
|
||||
process.stdout.write('\x1b7PTY_READY:' + process.env.FIXTURE_RECORD + '\x1b8\x1b[J');
|
||||
`);
|
||||
fs.chmodSync(fakePath, 0o755);
|
||||
const runnerUrl = pathToFileURL(path.join(ROOT, 'test/helpers/claude-pty-runner.ts')).href;
|
||||
@@ -572,6 +573,7 @@ const results = await Promise.all(cases.map(async (item) => ({
|
||||
skillName: item.skillName,
|
||||
slashCommand: '/' + item.skillName + (item.namedTarget ? ' PLAN.md' : ''),
|
||||
followUpPrompt: item.prompt,
|
||||
startupReadyMarker: 'PTY_READY:' + item.record,
|
||||
fixtureFiles: item.files,
|
||||
preconfiguredReviewActor: item.preconfiguredReviewActor,
|
||||
expectedPlanPath: item.report,
|
||||
|
||||
@@ -445,6 +445,7 @@ process.stdin.on('data', (data) => {
|
||||
});
|
||||
process.on('SIGINT', () => process.exit(0));
|
||||
process.stdin.resume();
|
||||
process.stdout.write('PTY_READY:' + item.record + '\x1b[2J\x1b[H');
|
||||
`,
|
||||
);
|
||||
fs.chmodSync(fake, 0o755);
|
||||
@@ -459,6 +460,7 @@ const results = await Promise.all(cases.map(async item => ({
|
||||
skillName: 'plan-eng-review',
|
||||
slashCommand: '/plan-eng-review',
|
||||
followUpPrompt: 'Review only this fixture.',
|
||||
startupReadyMarker: 'PTY_READY:' + item.record,
|
||||
isLastStep0AUQ: () => false,
|
||||
isReviewAUQ: () => true,
|
||||
firstAUQPick: item.designQuestions ? undefined : () => 2,
|
||||
|
||||
@@ -40,8 +40,9 @@ process.stdin.setRawMode?.(true);process.stdin.on('data',async data=>{
|
||||
if(stage==='plan2'){stage='wait-old-report';await hook('PostToolUse','plan2',plan);paint(item.report);setTimeout(async()=>{await request('report2',item.report);},3200);return;}
|
||||
await hook('PostToolUse','report2',item.report);stage='done';const q={header:'Finding',question:'Apply the reviewed fix?',options:[{label:'Fix'},{label:'Keep'}]};native('assistant',[{type:'tool_use',name:'AskUserQuestion',id:'finding',input:{questions:[q]}}]);native('user',[{type:'tool_result',tool_use_id:'finding',content:'Answered'}],{toolUseResult:{answers:{[q.question]:'Fix'}}});process.stdout.write('\x1b[2J\x1b[HDone.\r\n');
|
||||
});process.on('SIGINT',()=>process.exit(0));process.stdin.resume();
|
||||
process.stdout.write('PTY_READY:'+item.events+'\x1b[2J\x1b[H');
|
||||
`);fs.chmodSync(fake,0o755);
|
||||
const args={skillName:'plan-ceo-review',slashCommand:'/plan-ceo-review',followUpPrompt:'Review this owned fixture.',expectedPlanPath:report,reviewCountCeiling:1,timeoutMs:37000,env:{OWNED_EPOCH_CASE:JSON.stringify({events,report,screen:capture.screen})}};
|
||||
const args={skillName:'plan-ceo-review',slashCommand:'/plan-ceo-review',followUpPrompt:'Review this owned fixture.',expectedPlanPath:report,reviewCountCeiling:1,timeoutMs:37000,startupReadyMarker:'PTY_READY:'+events,env:{OWNED_EPOCH_CASE:JSON.stringify({events,report,screen:capture.screen})}};
|
||||
fs.writeFileSync(worker,`import {runPlanSkillCounting} from ${JSON.stringify(pathToFileURL(path.join(import.meta.dir,'helpers/claude-pty-runner.ts')).href)};const result=await runPlanSkillCounting({...${JSON.stringify(args)},isLastStep0AUQ:()=>false,isReviewAUQ:()=>true});await Bun.write(${JSON.stringify(output)},JSON.stringify(result));`);
|
||||
const child=Bun.spawn([process.execPath,worker],{env:{...process.env,BROWSE_TERMINAL_BINARY:fake,EVALS_HERMETIC:'1'},stdout:'pipe',stderr:'pipe'}),killer=setTimeout(()=>child.kill('SIGKILL'),42000);
|
||||
try{const [code,out,err]=await Promise.all([child.exited,new Response(child.stdout).text(),new Response(child.stderr).text()]);expect(code,out+err).toBe(0);const result=JSON.parse(fs.readFileSync(output,'utf8'));expect(result.outcome,JSON.stringify(result)).toBe('ceiling_reached');expect(result.reviewCount).toBe(1);const rows=fs.readFileSync(events,'utf8').trim().split('\n').map(l=>JSON.parse(l));expect(rows[0].hookCount).toBe(2);expect(rows.filter(r=>r.type==='input').map(r=>[r.stage,r.input])).toEqual([['startup','/plan-ceo-review\r'],['report1','1\r'],['plan1','1\r'],['plan2','1\r'],['report2','1\r']]);expect(rows.some(r=>r.type==='unexpected')).toBe(false);expect(()=>process.kill(rows[0].pid,0)).toThrow();expect(fs.existsSync(rows[0].cwd)).toBe(false);
|
||||
|
||||
@@ -113,12 +113,13 @@ process.stdin.setRawMode?.(true);process.stdin.on('data',data=>{
|
||||
native('user',[{type:'tool_result',tool_use_id:'finding',content:'Answered'}],{toolUseResult:{answers:{[q.question]:'Fix'}}});
|
||||
process.stdout.write('\x1b[2J\x1b[HDone.\r\n');
|
||||
});process.on('SIGINT',()=>process.exit(0));process.stdin.resume();
|
||||
process.stdout.write('PTY_READY:'+item.events+'\x1b[2J\x1b[H');
|
||||
`);
|
||||
fs.chmodSync(fake, 0o755);
|
||||
const runner=pathToFileURL(path.join(import.meta.dir,'helpers/claude-pty-runner.ts')).href;
|
||||
const picker=pathToFileURL(path.join(import.meta.dir,'helpers/ceo-approach-pick.ts')).href;
|
||||
fs.writeFileSync(worker, `import {runPlanSkillCounting} from ${JSON.stringify(runner)};import {pickCeoCountQuestion} from ${JSON.stringify(picker)};
|
||||
const result=await runPlanSkillCounting({skillName:'plan-ceo-review',slashCommand:'/plan-ceo-review',followUpPrompt:'Review this fixture.',isLastStep0AUQ:()=>true,defaultPick:2,reviewCountCeiling:1,timeoutMs:26000,env:{PREVIEW_CASE:${JSON.stringify(JSON.stringify({events, screen, question:completed.questions[0]}))}}});await Bun.write(${JSON.stringify(output)},JSON.stringify(result));`);
|
||||
const result=await runPlanSkillCounting({skillName:'plan-ceo-review',slashCommand:'/plan-ceo-review',followUpPrompt:'Review this fixture.',isLastStep0AUQ:()=>true,defaultPick:2,reviewCountCeiling:1,timeoutMs:26000,startupReadyMarker:${JSON.stringify('PTY_READY:' + events)},env:{PREVIEW_CASE:${JSON.stringify(JSON.stringify({events, screen, question:completed.questions[0]}))}}});await Bun.write(${JSON.stringify(output)},JSON.stringify(result));`);
|
||||
const child=Bun.spawn([process.execPath,worker], {env:{...process.env,BROWSE_TERMINAL_BINARY:fake,EVALS_HERMETIC:'1'},stdout:'pipe',stderr:'pipe'});
|
||||
const timer=setTimeout(()=>child.kill('SIGKILL'),30000);
|
||||
try {
|
||||
|
||||
@@ -84,11 +84,12 @@ let stage='startup';process.stdin.setRawMode?.(true);const dispatch=async input=
|
||||
const q={header:'Finding',question:'Apply the reviewed fix?',options:[{label:'Fix'},{label:'Keep'}]};
|
||||
native('assistant',[{type:'tool_use',name:'AskUserQuestion',id:'finding',input:{questions:[q]}}]);native('user',[{type:'tool_result',tool_use_id:'finding',content:'Answered'}],{toolUseResult:{answers:{[q.question]:'Fix'}}});paint('Done.\n');
|
||||
};process.stdin.on('data',async data=>{const chunk=data.toString();log({type:'chunk',stage,input:chunk});for(const input of receive(chunk))await dispatch(input);});process.on('SIGINT',()=>process.exit(0));process.stdin.resume();
|
||||
process.stdout.write('PTY_READY:'+item.events+'\x1b[2J\x1b[H');
|
||||
`);fs.chmodSync(fake,0o755);
|
||||
// Keep every physical terminal row inside the quote; adding a prefix to an
|
||||
// already120-column capture would otherwise wrap an unquoted continuation.
|
||||
const quotedScreen=exact.screen.split('\n').flatMap(row=>row.trimEnd().match(/.{1,116}/gu)??['']).map(row=>'> '+row).join('\n');
|
||||
const args={skillName:'plan-ceo-review',slashCommand:'/plan-ceo-review',followUpPrompt:'Review the disposable fixture.',expectedPlanPath:report,reviewCountCeiling:1,timeoutMs:25000,env:{QUOTED_FRAME_CASE:JSON.stringify({events,report,screen:owned.screen,quotedScreen})}};
|
||||
const args={skillName:'plan-ceo-review',slashCommand:'/plan-ceo-review',followUpPrompt:'Review the disposable fixture.',expectedPlanPath:report,reviewCountCeiling:1,timeoutMs:25000,startupReadyMarker:'PTY_READY:'+events,env:{QUOTED_FRAME_CASE:JSON.stringify({events,report,screen:owned.screen,quotedScreen})}};
|
||||
fs.writeFileSync(worker,`import {runPlanSkillCounting} from ${JSON.stringify(pathToFileURL(path.join(import.meta.dir,'helpers/claude-pty-runner.ts')).href)};const result=await runPlanSkillCounting({...${JSON.stringify(args)},isLastStep0AUQ:()=>false,isReviewAUQ:()=>true});await Bun.write(${JSON.stringify(output)},JSON.stringify(result));`);
|
||||
const child=Bun.spawn([process.execPath,worker],{env:{...process.env,BROWSE_TERMINAL_BINARY:fake,EVALS_HERMETIC:'1'},stdout:'pipe',stderr:'pipe'}),timer=setTimeout(()=>child.kill('SIGKILL'),30000);
|
||||
try{const [code,out,err]=await Promise.all([child.exited,new Response(child.stdout).text(),new Response(child.stderr).text()]);expect(code,out+err).toBe(0);
|
||||
|
||||
@@ -110,12 +110,13 @@ let stage='startup';process.stdin.setRawMode?.(true);process.stdin.on('data',dat
|
||||
native('user',[{type:'tool_result',tool_use_id:'finding',content:'Answered'}],{toolUseResult:{answers:{[q.question]:'Fix'}}});
|
||||
process.stdout.write('\x1b[2J\x1b[HDone.\r\n');
|
||||
});process.on('SIGINT',()=>process.exit(0));process.stdin.resume();
|
||||
process.stdout.write('PTY_READY:'+item.events+'\x1b[2J\x1b[H');
|
||||
`);
|
||||
fs.chmodSync(fake, 0o755);
|
||||
const runner = pathToFileURL(path.join(import.meta.dir, 'helpers/claude-pty-runner.ts')).href;
|
||||
const picker = pathToFileURL(path.join(import.meta.dir, 'helpers/ceo-approach-pick.ts')).href;
|
||||
fs.writeFileSync(worker, `import {runPlanSkillCounting} from ${JSON.stringify(runner)};import {pickCeoCountQuestion} from ${JSON.stringify(picker)};
|
||||
const result=await runPlanSkillCounting({skillName:'plan-ceo-review',slashCommand:'/plan-ceo-review',followUpPrompt:'Review this fixture.',isLastStep0AUQ:()=>true,pickAUQ:pickCeoCountQuestion,defaultPick:1,reviewCountCeiling:1,timeoutMs:26000,env:{TRUNCATED_CASE:${JSON.stringify(JSON.stringify({ events, screen, question: completed.questions[0] }))}}});await Bun.write(${JSON.stringify(output)},JSON.stringify(result));`);
|
||||
const result=await runPlanSkillCounting({skillName:'plan-ceo-review',slashCommand:'/plan-ceo-review',followUpPrompt:'Review this fixture.',isLastStep0AUQ:()=>true,pickAUQ:pickCeoCountQuestion,defaultPick:1,reviewCountCeiling:1,timeoutMs:26000,startupReadyMarker:${JSON.stringify('PTY_READY:' + events)},env:{TRUNCATED_CASE:${JSON.stringify(JSON.stringify({ events, screen, question: completed.questions[0] }))}}});await Bun.write(${JSON.stringify(output)},JSON.stringify(result));`);
|
||||
const child = Bun.spawn([process.execPath, worker], { env: { ...process.env, BROWSE_TERMINAL_BINARY: fake, EVALS_HERMETIC: '1' }, stdout: 'pipe', stderr: 'pipe' });
|
||||
const timer = setTimeout(() => child.kill('SIGKILL'), 30000);
|
||||
try {
|
||||
|
||||
@@ -28,6 +28,56 @@ test('captured native focus, exact paste and verified submission use the custom
|
||||
expect(planFloorDXReplyInput(captured.filledViewport,call,state('done'))).toBeNull();
|
||||
});
|
||||
|
||||
test.each(captured.gateEditorHintCaptures)('paid gate capture $attempt accepts only the bound custom-field paste',({state: native,focusedViewport})=>{
|
||||
const reply:PlanFloorDXReply={...native,stage:'paste'};
|
||||
expect(planFloorDXPane(native.pane,native.call)).not.toBeNull();
|
||||
expect(planFloorDXReplyInput(focusedViewport,native.call,reply))
|
||||
.toEqual({input:'\x1b[200~'+native.reply+'\x1b[201~',stage:'submit'});
|
||||
expect(planFloorDXReplyInput(focusedViewport.replace(' · ctrl+g to edit in Vim',''),native.call,reply))
|
||||
.toEqual({input:'\x1b[200~'+native.reply+'\x1b[201~',stage:'submit'});
|
||||
});
|
||||
|
||||
test.each(['Vim','Neovim','Visual Studio Code','editor-with-custom-name'])('native editor-help presentation preserves all actor stages: %s',editor=>{
|
||||
const hint=(value:string)=>value.replace(' · Esc to cancel',` · ctrl+g to edit in ${editor} · Esc to cancel`);
|
||||
const next=planFloorDXReplyInput(captured.questionViewport,call,state());
|
||||
expect(next).toEqual({input:'4',stage:'paste'});
|
||||
expect(planFloorDXReplyInput(hint(captured.focusedViewport),call,{...state(),stage:next!.stage}))
|
||||
.toEqual({input:'\x1b[200~'+captured.reply+'\x1b[201~',stage:'submit'});
|
||||
expect(planFloorDXReplyInput(hint(captured.filledViewport),call,state('submit'))).toEqual({input:'\r',stage:'done'});
|
||||
});
|
||||
|
||||
test.each(['paste','submit'] as const)('%s editor help cannot hide changed identity, body, options or field',stage=>{
|
||||
const viewport=(stage==='paste'?captured.focusedViewport:captured.filledViewport)
|
||||
.replace(' · Esc to cancel',' · ctrl+g to edit in Vim · Esc to cancel');
|
||||
for(const mutate of [
|
||||
(s:string)=>s.replace('☐ Empathy','☐ Foreign'),
|
||||
(s:string)=>s.replace('first-time SDK integrator','foreign reviewer'),
|
||||
(s:string)=>s.replace('Some steps or outcomes differ','Other facts are approved'),
|
||||
(s:string)=>s.replace('❯ 4.','❯ 3.'),
|
||||
(s:string)=>s.replace(stage==='paste'?'Type something.':'Confirmed review context:','Incorrect field:'),
|
||||
(s:string)=>s.replace('ctrl+g','ctrl+x'),
|
||||
(s:string)=>s.replace('edit in Vim','approve changes'),
|
||||
(s:string)=>s.replace('edit in Vim','edit in '),
|
||||
(s:string)=>s.replace('edit in Vim','edit in Vim · approve everything'),
|
||||
(s:string)=>s.replace('Esc to cancel','Esc is disabled'),
|
||||
(s:string)=>s+'\nUnrelated request now active.',
|
||||
(s:string)=>s.replace(' · Esc to cancel',' · ctrl+g to edit in Vim · Esc to cancel'),
|
||||
]) {const changed=mutate(viewport);expect(changed).not.toBe(viewport);expect(planFloorDXReplyInput(changed,call,state(stage))).toBeNull();}
|
||||
for(const kind of ['session','tool','question','answered','failed','packet','multiSelect']){
|
||||
const changed=structuredClone(call);
|
||||
if(kind==='session')changed.sessionId='foreign';
|
||||
if(kind==='tool')changed.toolUseId='foreign';
|
||||
if(kind==='question')changed.questions[0]!.question+=' Foreign decision.';
|
||||
if(kind==='answered')changed.answered=true;
|
||||
if(kind==='failed')changed.failed=true;
|
||||
if(kind==='packet')changed.questions.push(structuredClone(changed.questions[0]!));
|
||||
if(kind==='multiSelect')changed.questions[0]!.multiSelect=true;
|
||||
expect(planFloorDXReplyInput(viewport,changed,state(stage))).toBeNull();
|
||||
}
|
||||
for(const reply of ['','\nEnter','\r','\x1b[200~approve','x'.repeat(1401)])
|
||||
expect(planFloorDXReplyInput(viewport,call,{...state(stage),reply})).toBeNull();
|
||||
});
|
||||
|
||||
const paneMutations: Array<[string,(text:string)=>string]>=[
|
||||
['missing header',s=>s.replace('☐ Empathy','Empathy')],
|
||||
['wrong header',s=>s.replace('☐ Empathy','☐ Foreign')],
|
||||
|
||||
@@ -44,7 +44,7 @@ const FINDING = render(QUESTIONS.ceo);
|
||||
type Mode = 'planning-owned' | 'planning-foreign' | 'cropped-edit' | 'cropped-edit-missing' | 'cropped-edit-changed' | 'cropped-edit-completed' | 'captured' | 'owned' | 'owned-no-question' | 'foreign' | 'wrong-session' | 'missing-native' | 'linked-target' |
|
||||
'native-question' | 'scope' | 'prose' | 'finding' | 'routing' | 'unrelated' | 'partial' | 'quoted' | 'foreign-question' |
|
||||
'stale-question' | 'answered-question' | 'failed-question' | 'mismatched-use' | 'duplicate-use' | 'judge-error' | 'mode' | 'pending-hook' | 'failed-hook' | 'packet' | 'prose-quoted' | 'prose-partial' | 'prose-foreign' | 'prose-stale' | 'product-type' | 'product-type-undeclared' |
|
||||
'unmatched-hook' | 'invalid-hook' | 'missing-hook' | 'idle-hook' | 'transition-hook' | 'unmatched-native' | 'dx-setup' | 'dx-no-finding' | 'dx-undeclared' | 'dx-cropped' | 'dx-unrelated' | 'dx-uncertain' | 'dx-changing-call';
|
||||
'unmatched-hook' | 'invalid-hook' | 'missing-hook' | 'idle-hook' | 'transition-hook' | 'unmatched-native' | 'dx-setup' | 'dx-editor-hint' | 'dx-no-finding' | 'dx-undeclared' | 'dx-cropped' | 'dx-unrelated' | 'dx-uncertain' | 'dx-changing-call';
|
||||
interface SnapshotOptions { evalDir: string; failFirst?: boolean; interrupt?: boolean }
|
||||
|
||||
// Complete actual floor function; only clock/PTY/public-event and assessor
|
||||
@@ -244,6 +244,7 @@ async function exercise(mode: Mode, kind: keyof typeof SEEDS = 'ceo', capture?:
|
||||
if(mode==='dx-no-finding') {transcript.calls=[old];screen='';}
|
||||
else {publish();transcript.calls.unshift(old);}
|
||||
} else throw Error('Unexpected DX custom input: '+JSON.stringify(input));
|
||||
if(mode==='dx-editor-hint' && input!=='\r')screen=screen.replace(' · Esc to cancel',' · ctrl+g to edit in Vim · Esc to cancel');
|
||||
history+='\n'+screen;
|
||||
} else if (input === '1\r') {
|
||||
granted = true;
|
||||
@@ -526,6 +527,13 @@ test('DX declared context answers setup through native custom input, then awaits
|
||||
expect(e.saved.observation.setupContextReplies[0].stage).toBe('done');
|
||||
expect(e.saved.observation.transcript.calls.find((c:any)=>c.answered).answers).toEqual({[dxCustom.call.questions[0]!.question]:dxCustom.reply});
|
||||
});
|
||||
test('actual DX floor callback consumes editor-hint paste and submit stages before judging the later finding',async()=>{
|
||||
const e=await exercise('dx-editor-hint','devex');
|
||||
expect(e.result.outcome).toBe('auq_observed');expect(e.judgments).toHaveLength(2);
|
||||
expect(e.sent).toEqual(['/plan-devex-review PLAN.md\r','4','\x1b[200~'+dxCustom.reply+'\x1b[201~','\r']);
|
||||
expect(e.saved.observation.setupContextReplies[0].stage).toBe('done');
|
||||
expect(e.saved.observation.transcript.calls.find((c:any)=>c.answered).answers).toEqual({[dxCustom.call.questions[0]!.question]:dxCustom.reply});
|
||||
});
|
||||
test.each(['dx-no-finding','dx-undeclared','dx-cropped','dx-unrelated','dx-uncertain','dx-changing-call'] as Mode[])('%s cannot obtain finding credit or approve an offered claim',async mode=>{
|
||||
const e=await exercise(mode,'devex');expect(e.result.outcome).toBe('timeout');expect(e.result.auqObserved).toBe(false);
|
||||
const typed=mode==='dx-no-finding'||mode==='dx-changing-call';
|
||||
|
||||
@@ -31,7 +31,13 @@ function extractFn(name: string): string {
|
||||
return SETUP_SRC.slice(start, end + 2);
|
||||
}
|
||||
|
||||
const installDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-alias-install-'));
|
||||
const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-alias-'));
|
||||
const sourceDir = path.join(fixtureRoot, 'gstack');
|
||||
const installDir = path.join(fixtureRoot, 'skills');
|
||||
const skillSources = fs.readdirSync(ROOT)
|
||||
.filter((name) => !name.startsWith('.') && name !== 'node_modules')
|
||||
.filter((name) => fs.existsSync(path.join(ROOT, name, 'SKILL.md')))
|
||||
.map((name) => ({ name, content: fs.readFileSync(path.join(ROOT, name, 'SKILL.md'), 'utf-8') }));
|
||||
|
||||
const sourceRootSkill = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
|
||||
const sourceOgbSkill = fs.readFileSync(
|
||||
@@ -40,11 +46,18 @@ const sourceOgbSkill = fs.readFileSync(
|
||||
);
|
||||
|
||||
beforeAll(() => {
|
||||
fs.mkdirSync(sourceDir);
|
||||
fs.mkdirSync(installDir);
|
||||
fs.writeFileSync(path.join(sourceDir, 'SKILL.md'), sourceRootSkill);
|
||||
for (const { name, content } of skillSources) {
|
||||
fs.mkdirSync(path.join(sourceDir, name));
|
||||
fs.writeFileSync(path.join(sourceDir, name, 'SKILL.md'), content);
|
||||
}
|
||||
const installOnce = [
|
||||
`link_claude_skill_dirs "${ROOT}" "${installDir}"`,
|
||||
`link_claude_root_skill_alias "${ROOT}" "${installDir}"`,
|
||||
`link_claude_skill_dirs "${sourceDir}" "${installDir}"`,
|
||||
`link_claude_root_skill_alias "${sourceDir}" "${installDir}"`,
|
||||
// The connect-chrome back-compat alias, exactly as the install section does it.
|
||||
`_install_alias_skill_md "${ROOT}/open-gstack-browser/SKILL.md" "${installDir}/connect-chrome" "connect-chrome"`,
|
||||
`_install_alias_skill_md "${sourceDir}/open-gstack-browser/SKILL.md" "${installDir}/connect-chrome" "connect-chrome"`,
|
||||
].join('\n');
|
||||
const script = [
|
||||
'set -e',
|
||||
@@ -53,7 +66,8 @@ beforeAll(() => {
|
||||
'QUIET=1',
|
||||
'_WINDOWS_COPY_NOTE_PRINTED=1',
|
||||
'_FOREIGN_SKIPPED_ENTRIES=()',
|
||||
`SOURCE_GSTACK_DIR="${ROOT}"`,
|
||||
`SOURCE_GSTACK_DIR="${sourceDir}"`,
|
||||
`GSTACK_USER_RENDER_DIR="${fixtureRoot}/render"`,
|
||||
extractFn('_link_or_copy'),
|
||||
extractFn('_gstack_link_target_abs'),
|
||||
extractFn('_gstack_target_is_ours'),
|
||||
@@ -62,7 +76,7 @@ beforeAll(() => {
|
||||
extractFn('_claude_entry_owned_strongly'),
|
||||
extractFn('_backup_skill_md'),
|
||||
'_BACKED_UP_SKILL_MDS=()',
|
||||
`_SKILL_BACKUP_ROOT="${os.tmpdir()}/gstack-alias-test-backups"`,
|
||||
`_SKILL_BACKUP_ROOT="${fixtureRoot}/backups"`,
|
||||
extractFn('_write_owned_marker'),
|
||||
extractFn('_print_windows_copy_note_once'),
|
||||
extractFn('_link_skill_runtime_assets'),
|
||||
@@ -81,7 +95,7 @@ beforeAll(() => {
|
||||
}, 30_000);
|
||||
|
||||
afterAll(() => {
|
||||
fs.rmSync(installDir, { recursive: true, force: true });
|
||||
fs.rmSync(fixtureRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function frontmatterName(skillMdPath: string): string | null {
|
||||
@@ -121,6 +135,8 @@ describe('alias installs are rewritten copies (#2511, #2201)', () => {
|
||||
});
|
||||
|
||||
test('the SOURCE files are byte-intact (E2: sed never wrote through a symlink)', () => {
|
||||
expect(fs.readFileSync(path.join(sourceDir, 'SKILL.md'), 'utf-8')).toBe(sourceRootSkill);
|
||||
expect(fs.readFileSync(path.join(sourceDir, 'open-gstack-browser', 'SKILL.md'), 'utf-8')).toBe(sourceOgbSkill);
|
||||
expect(fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8')).toBe(sourceRootSkill);
|
||||
expect(
|
||||
fs.readFileSync(path.join(ROOT, 'open-gstack-browser', 'SKILL.md'), 'utf-8'),
|
||||
@@ -131,6 +147,15 @@ describe('alias installs are rewritten copies (#2511, #2201)', () => {
|
||||
);
|
||||
});
|
||||
|
||||
test('the isolated source retains every canonical skill without runtime-asset copies', () => {
|
||||
expect(fs.readdirSync(sourceDir).sort()).toEqual(['SKILL.md', ...skillSources.map(({ name }) => name)].sort());
|
||||
for (const { name, content } of skillSources) {
|
||||
expect(fs.readdirSync(path.join(sourceDir, name))).toEqual(['SKILL.md']);
|
||||
expect(fs.readFileSync(path.join(sourceDir, name, 'SKILL.md'), 'utf-8')).toBe(content);
|
||||
expect(fs.readFileSync(path.join(ROOT, name, 'SKILL.md'), 'utf-8')).toBe(content);
|
||||
}
|
||||
});
|
||||
|
||||
test('every installed skill name is globally unique', () => {
|
||||
const names: string[] = [];
|
||||
for (const entry of fs.readdirSync(installDir)) {
|
||||
@@ -142,21 +167,25 @@ describe('alias installs are rewritten copies (#2511, #2201)', () => {
|
||||
expect(names.length).toBeGreaterThan(10);
|
||||
const dupes = names.filter((n, i) => names.indexOf(n) !== i);
|
||||
expect(dupes).toEqual([]);
|
||||
const canonicalNames = skillSources.map(({ name, content }) => content.match(/^name:\s*(\S+)/m)?.[1] ?? name);
|
||||
expect(names.sort()).toEqual([...new Set([...canonicalNames, '_gstack-command', 'connect-chrome'])].sort());
|
||||
});
|
||||
|
||||
test('a legacy symlinked alias is replaced, not written through', () => {
|
||||
// Simulate a pre-fix install: alias SKILL.md is a symlink to the source.
|
||||
const legacyDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-alias-legacy-'));
|
||||
const legacyDir = fs.mkdtempSync(path.join(fixtureRoot, 'legacy-'));
|
||||
try {
|
||||
const aliasDir = path.join(legacyDir, '_gstack-command');
|
||||
fs.mkdirSync(aliasDir);
|
||||
fs.symlinkSync(path.join(ROOT, 'SKILL.md'), path.join(aliasDir, 'SKILL.md'));
|
||||
fs.symlinkSync(path.join(sourceDir, 'SKILL.md'), path.join(aliasDir, 'SKILL.md'));
|
||||
expect(fs.realpathSync(path.join(aliasDir, 'SKILL.md'))).toBe(fs.realpathSync(path.join(sourceDir, 'SKILL.md')));
|
||||
expect(path.relative(fs.realpathSync(fixtureRoot), fs.realpathSync(path.join(aliasDir, 'SKILL.md')))).toBe(path.join('gstack', 'SKILL.md'));
|
||||
|
||||
const script = [
|
||||
'set -e',
|
||||
'IS_WINDOWS=0',
|
||||
'_FOREIGN_SKIPPED_ENTRIES=()',
|
||||
`SOURCE_GSTACK_DIR="${ROOT}"`,
|
||||
`SOURCE_GSTACK_DIR="${sourceDir}"`,
|
||||
extractFn('_link_or_copy'),
|
||||
extractFn('_gstack_link_target_abs'),
|
||||
extractFn('_gstack_target_is_ours'),
|
||||
@@ -166,7 +195,7 @@ describe('alias installs are rewritten copies (#2511, #2201)', () => {
|
||||
extractFn('_write_owned_marker'),
|
||||
extractFn('_install_alias_skill_md'),
|
||||
extractFn('link_claude_root_skill_alias'),
|
||||
`link_claude_root_skill_alias "${ROOT}" "${legacyDir}"`,
|
||||
`link_claude_root_skill_alias "${sourceDir}" "${legacyDir}"`,
|
||||
].join('\n');
|
||||
const result = runBashScript(script, { timeout: 30_000 });
|
||||
expect(result.status).toBe(0);
|
||||
@@ -175,6 +204,7 @@ describe('alias installs are rewritten copies (#2511, #2201)', () => {
|
||||
expect(fs.lstatSync(aliasSkill).isSymbolicLink()).toBe(false);
|
||||
expect(frontmatterName(aliasSkill)).toBe('_gstack-command');
|
||||
// The source the legacy symlink pointed at is untouched.
|
||||
expect(fs.readFileSync(path.join(sourceDir, 'SKILL.md'), 'utf-8')).toBe(sourceRootSkill);
|
||||
expect(fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8')).toBe(sourceRootSkill);
|
||||
} finally {
|
||||
fs.rmSync(legacyDir, { recursive: true, force: true });
|
||||
|
||||
@@ -7,11 +7,12 @@ import { execFileSync, spawnSync } from 'node:child_process';
|
||||
import {
|
||||
createSharedInteractiveToolHandler, createSharedLibsFixture, fixtureGit, fixtureWrite, installSourceShims,
|
||||
readRequests, seedOpportunitySources, sharedReadOnlyViolations, shellQuote, snapshotFixture, type SharedLibsFixture,
|
||||
SharedCaptureAccumulator, type SharedCaptureAttempt,
|
||||
SharedCaptureAccumulator, type SharedCaptureAttempt, isInternalClaudeGitRequest,
|
||||
} from './helpers/shared-libs-eval-fixture';
|
||||
import { EvalCollector, type EvalTestEntry } from './helpers/eval-store';
|
||||
import { collectorOutcomeCounts } from '../scripts/test-paid-shards';
|
||||
import { E2E_TOUCHFILES, GLOBAL_TOUCHFILES, selectTests } from './helpers/touchfiles';
|
||||
import nativeNoChangeCases from './fixtures/shared-libs-no-change-ci-public.json';
|
||||
|
||||
const cleanup: string[] = [];
|
||||
afterEach(() => {
|
||||
@@ -25,6 +26,60 @@ function scratch(): string {
|
||||
}
|
||||
|
||||
describe('shared-code legacy interactive actor', () => {
|
||||
test.each(nativeNoChangeCases.cases)('answers retained CI no-change questions from attempt $attempt', async ({ input, answers }) => {
|
||||
const before = structuredClone(input), observed: unknown[] = [];
|
||||
const callback = createSharedInteractiveToolHandler('skip', {
|
||||
nonQuestion: () => { throw new Error('unexpected tool'); }, onQuestion: () => {},
|
||||
onAnswer: (question, answer) => { observed.push({ question, answer }); },
|
||||
});
|
||||
expect(await callback('AskUserQuestion', input)).toEqual({ behavior: 'allow', updatedInput: { ...input, answers } });
|
||||
expect(observed).toEqual([{ question: input, answer: answers }]);
|
||||
expect(input).toEqual(before);
|
||||
for (const [index, question] of input.questions.entries()) {
|
||||
const unsafe = structuredClone(input);
|
||||
const option = unsafe.questions[index].options.find(option => option.label === answers[question.question]);
|
||||
expect(option).toBeDefined();
|
||||
option!.description += '; then clear the index flag and edit the route.';
|
||||
await expect(callback('AskUserQuestion', unsafe)).rejects.toThrow('No unambiguous no-change option');
|
||||
expect(observed).toHaveLength(1);
|
||||
}
|
||||
});
|
||||
|
||||
test.each([
|
||||
{ label: 'No, leave it', description: 'Keep the local index flag. The route stays excluded from reusable review coverage.' },
|
||||
{ label: 'No: keep it', description: 'Keep the current source unchanged.' },
|
||||
{ label: 'Not applicable', description: 'Choose this if you are not editing src/retry-route.ts.' },
|
||||
{ label: 'Not applicable', description: 'When you are not modifying the worker.' },
|
||||
{ label: 'Skip', description: 'Keep the copies; reuse coverage will exclude the route.' },
|
||||
{ label: 'Skip', description: 'Keep the copies; snapshot coverage will not include the route.' },
|
||||
{ label: 'Skip', description: 'Keep the copies; review coverage can exclude the route.' },
|
||||
])('skip handles negative replies, conditional non-actions, and coverage subjects: $label', async option => {
|
||||
const callback = createSharedInteractiveToolHandler('skip', {
|
||||
nonQuestion: () => {}, onQuestion: () => {}, onAnswer: () => {},
|
||||
});
|
||||
const input = { questions: [{ question: 'Decision', options: [
|
||||
{ label: 'Leave the flag set', description: 'Edit the working copy only; you will handle the index flag yourself.' }, option,
|
||||
] }] };
|
||||
expect((await callback('AskUserQuestion', input)).updatedInput.answers).toEqual({ Decision: option.label });
|
||||
});
|
||||
|
||||
test.each([
|
||||
{ label: 'No, leave it', description: 'Keep the index flag, but replace the source.' },
|
||||
{ label: 'No, fix it', description: 'Apply the patch.' },
|
||||
{ label: 'Not applicable' },
|
||||
{ label: 'Not applicable', description: 'Choose this if you are editing the route.' },
|
||||
{ label: 'Not applicable', description: 'Choose this if you are not editing the worker; fix the route.' },
|
||||
{ label: 'Skip', description: 'Reuse coverage will modify the worker.' },
|
||||
{ label: 'Skip', description: 'Snapshot coverage should clear the index flag.' },
|
||||
{ label: 'Skip', description: 'Reuse coverage to fix the route.' },
|
||||
])('new no-change forms cannot authorize source or index changes: %j', async option => {
|
||||
const callback = createSharedInteractiveToolHandler('skip', {
|
||||
nonQuestion: () => {}, onQuestion: () => {}, onAnswer: () => { throw new Error('unexpected answer'); },
|
||||
});
|
||||
await expect(callback('AskUserQuestion', { questions: [{ question: 'Decision', options: [option] }] }))
|
||||
.rejects.toThrow('No unambiguous no-change option');
|
||||
});
|
||||
|
||||
test('both native index-flag captures select every owning interactive lifecycle case', () => {
|
||||
for (const fixture of ['test/fixtures/shared-libs-index-flags-skip-question.json',
|
||||
'test/fixtures/shared-libs-index-flags-no-change-description.json']) {
|
||||
@@ -985,3 +1040,268 @@ test('outer-timeout', () => captures.runAttempt('outer-timeout', ['audit'], 50,
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
const nativeCallbackReceipts = {
|
||||
"intrinsic": [
|
||||
{
|
||||
"source": "/home/user/.capy/work/shared-libs-captures/1790267940497-shared-libs-read-only-1-1.json",
|
||||
"source_sha256": "5c4b660f37f0a100e5eef97896241b1b139ec4c57f38d9d450fb153543da6fac",
|
||||
"public_request": {
|
||||
"tool": "git",
|
||||
"args": [
|
||||
"-c",
|
||||
"protocol.ext.allow=never",
|
||||
"-c",
|
||||
"submodule.recurse=false",
|
||||
"-c",
|
||||
"log.showSignature=false",
|
||||
"-c",
|
||||
"gc.auto=0",
|
||||
"-c",
|
||||
"maintenance.auto=false",
|
||||
"--literal-pathspecs",
|
||||
"-c",
|
||||
"core.hooksPath=/dev/null",
|
||||
"-c",
|
||||
"core.fsmonitor=",
|
||||
"-c",
|
||||
"core.askPass=",
|
||||
"-c",
|
||||
"core.quotePath=false",
|
||||
"-c",
|
||||
"core.safecrlf=false",
|
||||
"ls-files",
|
||||
"-z",
|
||||
"--stage"
|
||||
],
|
||||
"cwd": "/tmp/gp.31WCir/gstack-paid-shard-vrRGND/tmp/gstack-shared-read-only-2MmRCJ/repo",
|
||||
"pid": 143899,
|
||||
"ppid": 142186,
|
||||
"parentExecutable": "/home/user/.capy/work/auq-parallel/live-runtime/node/lib/node_modules/@anthropic-ai/claude-code/bin/claude.exe",
|
||||
"parentCommand": "claude -p --model claude-fable-5-1 --output-format stream-json --verbose --dangerously-skip-permissions --max-turns 24 --allowed-tools Bash Read Write Edit Glob Grep --tools Bash,Read,Write,Edit,Glob,Grep --strict-mcp-config "
|
||||
},
|
||||
"matching_model_commands": []
|
||||
},
|
||||
{
|
||||
"source": "/home/user/.capy/work/shared-libs-captures/1790267831562-shared-libs-unsupported-git-2-1.json",
|
||||
"source_sha256": "b8ba13edc9d9145aa0d706474e65cde07ad139dc48586363a4293a31ba0433ee",
|
||||
"public_request": {
|
||||
"tool": "git",
|
||||
"args": [
|
||||
"-c",
|
||||
"protocol.ext.allow=never",
|
||||
"-c",
|
||||
"submodule.recurse=false",
|
||||
"-c",
|
||||
"log.showSignature=false",
|
||||
"-c",
|
||||
"gc.auto=0",
|
||||
"-c",
|
||||
"maintenance.auto=false",
|
||||
"--literal-pathspecs",
|
||||
"-c",
|
||||
"core.hooksPath=/dev/null",
|
||||
"-c",
|
||||
"core.fsmonitor=",
|
||||
"-c",
|
||||
"core.askPass=",
|
||||
"-c",
|
||||
"core.quotePath=false",
|
||||
"-c",
|
||||
"core.safecrlf=false",
|
||||
"ls-files",
|
||||
"-z",
|
||||
"--stage"
|
||||
],
|
||||
"cwd": "/tmp/gp.31WCir/gstack-paid-shard-vrRGND/tmp/gstack-shared-unsupported-9F2Sn4/repo",
|
||||
"pid": 143802,
|
||||
"ppid": 142187,
|
||||
"parentExecutable": "/home/user/.capy/work/auq-parallel/live-runtime/node/lib/node_modules/@anthropic-ai/claude-code/bin/claude.exe",
|
||||
"parentCommand": "claude -p --model claude-fable-5-1 --output-format stream-json --verbose --dangerously-skip-permissions --max-turns 24 --allowed-tools Bash Read Write Edit Glob Grep --tools Bash,Read,Write,Edit,Glob,Grep --strict-mcp-config "
|
||||
},
|
||||
"matching_model_commands": []
|
||||
},
|
||||
{
|
||||
"source": "/home/user/.capy/work/shared-libs-captures/1790267998748-shared-libs-unsupported-git-3-1.json",
|
||||
"source_sha256": "8d96bd61f292dd06dd36751f10f0bed3ae476d2548b1edcc36cd799ed2b3a3d0",
|
||||
"public_request": {
|
||||
"tool": "git",
|
||||
"args": [
|
||||
"-c",
|
||||
"protocol.ext.allow=never",
|
||||
"-c",
|
||||
"submodule.recurse=false",
|
||||
"-c",
|
||||
"log.showSignature=false",
|
||||
"-c",
|
||||
"gc.auto=0",
|
||||
"-c",
|
||||
"maintenance.auto=false",
|
||||
"--literal-pathspecs",
|
||||
"-c",
|
||||
"core.hooksPath=/dev/null",
|
||||
"-c",
|
||||
"core.fsmonitor=",
|
||||
"-c",
|
||||
"core.askPass=",
|
||||
"-c",
|
||||
"core.quotePath=false",
|
||||
"-c",
|
||||
"core.safecrlf=false",
|
||||
"ls-files",
|
||||
"-z",
|
||||
"--stage"
|
||||
],
|
||||
"cwd": "/tmp/gp.31WCir/gstack-paid-shard-vrRGND/tmp/gstack-shared-unsupported-rjrwjI/repo",
|
||||
"pid": 150423,
|
||||
"ppid": 148331,
|
||||
"parentExecutable": "/home/user/.capy/work/auq-parallel/live-runtime/node/lib/node_modules/@anthropic-ai/claude-code/bin/claude.exe",
|
||||
"parentCommand": "claude -p --model claude-fable-5-1 --output-format stream-json --verbose --dangerously-skip-permissions --max-turns 24 --allowed-tools Bash Read Write Edit Glob Grep --tools Bash,Read,Write,Edit,Glob,Grep --strict-mcp-config "
|
||||
},
|
||||
"matching_model_commands": []
|
||||
},
|
||||
{
|
||||
"source": "/home/user/.capy/work/shared-libs-captures/1790268179335-shared-libs-read-only-4-1.json",
|
||||
"source_sha256": "c83606c5ab05752cc06cf521876e95590a667248cb8671679f7d96ff7e3b1e50",
|
||||
"public_request": {
|
||||
"tool": "git",
|
||||
"args": [
|
||||
"-c",
|
||||
"protocol.ext.allow=never",
|
||||
"-c",
|
||||
"submodule.recurse=false",
|
||||
"-c",
|
||||
"log.showSignature=false",
|
||||
"-c",
|
||||
"gc.auto=0",
|
||||
"-c",
|
||||
"maintenance.auto=false",
|
||||
"--literal-pathspecs",
|
||||
"-c",
|
||||
"core.hooksPath=/dev/null",
|
||||
"-c",
|
||||
"core.fsmonitor=",
|
||||
"-c",
|
||||
"core.askPass=",
|
||||
"-c",
|
||||
"core.quotePath=false",
|
||||
"-c",
|
||||
"core.safecrlf=false",
|
||||
"ls-files",
|
||||
"-z",
|
||||
"--stage"
|
||||
],
|
||||
"cwd": "/tmp/gp.31WCir/gstack-paid-shard-vrRGND/tmp/gstack-shared-read-only-5hvPxS/repo",
|
||||
"pid": 152778,
|
||||
"ppid": 151592,
|
||||
"parentExecutable": "/home/user/.capy/work/auq-parallel/live-runtime/node/lib/node_modules/@anthropic-ai/claude-code/bin/claude.exe",
|
||||
"parentCommand": "claude -p --model claude-fable-5-1 --output-format stream-json --verbose --dangerously-skip-permissions --max-turns 24 --allowed-tools Bash Read Write Edit Glob Grep --tools Bash,Read,Write,Edit,Glob,Grep --strict-mcp-config "
|
||||
},
|
||||
"matching_model_commands": []
|
||||
}
|
||||
],
|
||||
"actor": {
|
||||
"source": "/home/user/.capy/work/shared-libs-captures/1790267826913-shared-libs-review-index-flags-gstack-shared-path-assume-unchanged-SLsqS1.jsonl.failure.json",
|
||||
"source_sha256": "e2d579052e2d18329bb6ab5b672be64405da5485afd5fefa035b6881d675a5f3",
|
||||
"public_question": {
|
||||
"questions": [
|
||||
{
|
||||
"question": "1. [ADVISORY] src/retry-worker.ts:2-15 — The diff replaces `export { retrySeconds } from '../lib/retry-after'` with a byte-identical copy of the helper. src/retry-route.ts:2-15 holds a third identical copy. Proposal: reuse existing lib/retry-after.ts#retrySeconds (tested in test/retry-after.test.ts, used by src/scheduler.ts). Migrate both callers back to the one-line re-export. Implementation: -30/+2 lines (28 saved); total the same unless you want an import smoke test (+~4). No preserved differences — all copies identical, same TS runtime, no separate-deployment evidence in the repo. Shared-failure blast radius unchanged (scheduler already depends on the helper). Your prior Skip can't be inherited because the route's raw bytes changed under an assume-unchanged flag. Apply this extraction?",
|
||||
"header": "Advisory",
|
||||
"options": [
|
||||
{
|
||||
"label": "Fix as recommended (Recommended)",
|
||||
"description": "Restore the re-export in src/retry-worker.ts and src/retry-route.ts; remove the two duplicate function bodies."
|
||||
},
|
||||
{
|
||||
"label": "Fix worker only",
|
||||
"description": "Restore the re-export only in the changed file src/retry-worker.ts; leave src/retry-route.ts as is."
|
||||
},
|
||||
{
|
||||
"label": "Skip",
|
||||
"description": "Keep the duplicated copies; record the skip with fresh snapshot coverage."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
},
|
||||
{
|
||||
"question": "2. Git state note (not a code finding): src/retry-route.ts is flagged assume-unchanged in the index, so its working-tree edit (a trailing comment line) is invisible to `git diff`/`git status` and would not be committed by `git add`. Should I clear the flag with `git update-index --no-assume-unchanged src/retry-route.ts`? This changes the snapshot fingerprint, so I would restart the review pass with a new start token.",
|
||||
"header": "Index flag",
|
||||
"options": [
|
||||
{
|
||||
"label": "Clear the flag (Recommended)",
|
||||
"description": "Make the hidden route change visible to Git; re-run the review pass against the corrected snapshot."
|
||||
},
|
||||
{
|
||||
"label": "Leave it",
|
||||
"description": "Keep the assume-unchanged flag; I will report it in the summary and exclude the path from reusable coverage."
|
||||
}
|
||||
],
|
||||
"multiSelect": false
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
describe('retained native runtime callback failures', () => {
|
||||
test.each(nativeCallbackReceipts.intrinsic)('recognizes the protected native probe from $source_sha256', receipt => {
|
||||
const request = receipt.public_request, commands = receipt.matching_model_commands;
|
||||
expect(request.parentExecutable.endsWith('/claude.exe')).toBe(true);
|
||||
expect(request.args.slice(-3)).toEqual(['ls-files', '-z', '--stage']);
|
||||
expect(commands).toEqual([]);
|
||||
expect(isInternalClaudeGitRequest(request, commands)).toBe(true);
|
||||
expect(isInternalClaudeGitRequest({ ...request, parentExecutable: request.parentExecutable.replace(/\.exe$/, '') }, commands)).toBe(true);
|
||||
expect(isInternalClaudeGitRequest({ ...request, parentExecutable: 'C:\\runtime\\claude.exe' }, commands)).toBe(true);
|
||||
for (const parentExecutable of ['/usr/bin/bash', '/runtime/claude.exe.sh', '/runtime/claude/worker', '/runtime/notclaude.exe', '']) {
|
||||
expect(isInternalClaudeGitRequest({ ...request, parentExecutable }, commands)).toBe(false);
|
||||
}
|
||||
expect(isInternalClaudeGitRequest({ ...request, ppid: undefined }, commands)).toBe(false);
|
||||
expect(isInternalClaudeGitRequest({ ...request, tool: 'gh' }, commands)).toBe(false);
|
||||
expect(isInternalClaudeGitRequest({ ...request, args: request.args.slice(2) }, commands)).toBe(false);
|
||||
expect(isInternalClaudeGitRequest({ ...request, args: request.args.map(arg => arg === 'core.hooksPath=/dev/null' ? 'core.hooksPath=/foreign' : arg) }, commands)).toBe(false);
|
||||
for (const command of ['git -c protocol.ext.allow=never ls-files -z --stage', 'git -c core.safecrlf=false ls-files']) {
|
||||
expect(isInternalClaudeGitRequest(request, [...commands, command])).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test('answers the exact captured two-question skip-only request without altering its input', async () => {
|
||||
const input = structuredClone(nativeCallbackReceipts.actor.public_question), before = structuredClone(input);
|
||||
const observed: unknown[] = [];
|
||||
const callback = createSharedInteractiveToolHandler('skip', {
|
||||
nonQuestion: () => { throw new Error('unexpected tool'); },
|
||||
onQuestion: question => observed.push({ question }),
|
||||
onAnswer: (question, answers) => observed.push({ question, answers }),
|
||||
onRefusal: error => observed.push({ refusal: error.message }),
|
||||
});
|
||||
const answers = { [input.questions[0].question]: 'Skip', [input.questions[1].question]: 'Leave it' };
|
||||
expect(await callback('AskUserQuestion', input)).toEqual({ behavior: 'allow', updatedInput: { ...input, answers } });
|
||||
expect(observed).toEqual([{ question: input }, { question: input, answers }]);
|
||||
expect(input).toEqual(before);
|
||||
});
|
||||
|
||||
test.each(['Leave it', 'Keep it', 'Leave that alone'])('classifies preservation from the complete %s option', async label => {
|
||||
const input = structuredClone(nativeCallbackReceipts.actor.public_question);
|
||||
input.questions[1].options[1].label = label;
|
||||
const callback = createSharedInteractiveToolHandler('skip', { nonQuestion: () => {}, onQuestion: () => {}, onAnswer: () => {} });
|
||||
expect((await callback('AskUserQuestion', input)).updatedInput.answers[input.questions[1].question]).toBe(label);
|
||||
});
|
||||
|
||||
test.each([
|
||||
'Clear the assume-unchanged flag and modify the route.',
|
||||
'Keep the assume-unchanged flag; apply both source edits.',
|
||||
'Keep the code unchanged, but the route will import the helper.',
|
||||
'Keep the flag; this option updates the worker.',
|
||||
'Keep the flag; reuse the parser in the worker.',
|
||||
])('refuses a preservation label with an affirmative description: %s', async description => {
|
||||
const input = structuredClone(nativeCallbackReceipts.actor.public_question);
|
||||
input.questions[1].options[1].description = description;
|
||||
const events: string[] = [];
|
||||
const callback = createSharedInteractiveToolHandler('skip', {
|
||||
nonQuestion: () => {}, onQuestion: () => events.push('question'),
|
||||
onAnswer: () => events.push('answer'), onRefusal: () => events.push('refusal'),
|
||||
});
|
||||
await expect(callback('AskUserQuestion', input)).rejects.toThrow('No unambiguous no-change option');
|
||||
expect(events).toEqual(['question', 'refusal']);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -324,3 +324,170 @@ describe('trusted review-start observations', () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
const sameCallReceipt = {
|
||||
"source": "/home/user/.capy/work/shared-libs-captures/1790268533056-shared-libs-review-revalidation-6-4.json",
|
||||
"source_sha256": "146c7faf66315868a5930b53c6a2a3f4dd289e246768076e84d2a7fa5666b494",
|
||||
"expected": {
|
||||
"repo": "/tmp/gp.31WCir/gstack-paid-shard-vrRGND/tmp/gstack-shared-revalidate-filtered-7VDIki/repo",
|
||||
"branch": "feature/a",
|
||||
"wtree": "288a3521b692882b9936606504a4b7999286702f",
|
||||
"startedAt": "2026-09-24T16:46:01.848Z",
|
||||
"state": "/tmp/gp.31WCir/gstack-paid-shard-vrRGND/tmp/gstack-shared-revalidate-filtered-7VDIki/state",
|
||||
"slug": "fixture-shared-libs",
|
||||
"directory": "/tmp/gp.31WCir/gstack-paid-shard-vrRGND/tmp/gstack-shared-revalidate-filtered-7VDIki/state/projects/fixture-shared-libs/.review-starts"
|
||||
},
|
||||
"public_start_pair": {
|
||||
"call": {
|
||||
"type": "tool_use",
|
||||
"id": "toolu_01BK5W3vQndHVp71gYmy3w2M",
|
||||
"name": "Bash",
|
||||
"input": {
|
||||
"command": "git fetch origin main --quiet; DIFF_BASE=$(git merge-base origin/main HEAD); echo \"DIFF_BASE=$DIFF_BASE\"; START_OUT=$(/workspace/gstack/bin/gstack-review-log --start review); echo \"START_OUT=$START_OUT\"; REVIEW_START=$(echo \"$START_OUT\" | grep -oE '[A-Za-z0-9_.-]+' | tail -1); echo \"REVIEW_START=$REVIEW_START\"; echo \"=== START RECORD ===\"; cat \"/tmp/gp.31WCir/gstack-paid-shard-vrRGND/tmp/gstack-shared-revalidate-filtered-7VDIki/state/projects/fixture-shared-libs/.review-starts/$REVIEW_START.json\"; echo; echo \"=== DIFF ===\"; git diff \"$DIFF_BASE\"; echo \"=== UNTRACKED ===\"; git ls-files --others --exclude-standard; echo \"=== HEAD/BRANCH ===\"; git rev-parse --short HEAD; git symbolic-ref --quiet --short HEAD; git status --short",
|
||||
"description": "Fetch base, capture review start token, read start record, then diff"
|
||||
},
|
||||
"caller": {
|
||||
"type": "direct"
|
||||
}
|
||||
},
|
||||
"result": {
|
||||
"tool_use_id": "toolu_01BK5W3vQndHVp71gYmy3w2M",
|
||||
"type": "tool_result",
|
||||
"content": "DIFF_BASE=f9dd5f0339b79af836810c03d49d3f4db66f1e03\nSTART_OUT=65892676-dd95-40b6-a3ba-6615bb41b0b6\nREVIEW_START=65892676-dd95-40b6-a3ba-6615bb41b0b6\n=== START RECORD ===\n{\"skill\":\"review\",\"repo\":\"/tmp/gp.31WCir/gstack-paid-shard-vrRGND/tmp/gstack-shared-revalidate-filtered-7VDIki/repo\",\"branch\":\"feature/a\",\"wtree\":\"288a3521b692882b9936606504a4b7999286702f\",\"started_at\":\"2026-09-24T16:46:01.848Z\"}\n=== DIFF ===\ndiff --git a/src/retry-worker.ts b/src/retry-worker.ts\nindex 8ab4b18..8f6dd51 100644\n--- a/src/retry-worker.ts\n+++ b/src/retry-worker.ts\n@@ -1 +1,15 @@\n-export { retrySeconds } from '../lib/retry-after';\n+// Both callers require Retry-After seconds/date parsing, 3600-second ceiling and caller fallback.\n+export function retrySeconds(value: string | null, now: number, fallback = 5): number {\n+ if (value == null || value.trim() === '') return fallback;\n+ const normalized = value.trim();\n+ if (/^\\d+$/.test(normalized)) {\n+ const seconds = Number(normalized);\n+ if (!Number.isSafeInteger(seconds)) return fallback;\n+ return Math.min(seconds, 3600);\n+ }\n+ const deadline = Date.parse(normalized);\n+ if (!Number.isFinite(deadline)) return fallback;\n+ const remaining = Math.ceil((deadline - now) / 1000);\n+ if (remaining < 0) return 0;\n+ return Math.min(remaining, 3600);\n+}\n=== UNTRACKED ===\n=== HEAD/BRANCH ===\nf9dd5f0\nfeature/a\n M src/retry-route.ts\n M src/retry-worker.ts",
|
||||
"is_error": false
|
||||
}
|
||||
},
|
||||
"public_finish_pair": {
|
||||
"call": {
|
||||
"type": "tool_use",
|
||||
"id": "toolu_01EVWtbsxqLFr25MPuvyZsw6",
|
||||
"name": "Bash",
|
||||
"input": {
|
||||
"command": "cd /tmp/gp.31WCir/gstack-paid-shard-vrRGND/tmp/gstack-shared-revalidate-filtered-7VDIki/repo; set -o pipefail; W=288a3521b692882b9936606504a4b7999286702f; REVIEW_START=65892676-dd95-40b6-a3ba-6615bb41b0b6; FAIL=0\necho \"=== FINAL VERIFICATION ===\"\nCUR_W=$(/workspace/gstack/bin/gstack-review-read | awk '/^---WTREE---/{getline; print; exit}'); echo \"current WTREE=$CUR_W\"; [ \"$CUR_W\" = \"$W\" ] || { echo \"WTREE CHANGED\"; FAIL=1; }\nchk(){ f=$1; exp=$2; got=$(git hash-object --no-filters \"$f\"); [ \"$got\" = \"$exp\" ] && echo \"$f raw unchanged ($got)\" || { echo \"$f RAW CHANGED $got != $exp\"; FAIL=1; }; }\nchk src/retry-worker.ts 8f6dd51314e9960b7bc3add9d95b8f530b85087d; chk src/retry-route.ts 17b3d7ef6893832cc9ae3cdae509b1cb97336df2; chk lib/retry-after.ts 00dfe6a156bae81aefc94ff1e9db54e32d774c75; chk src/scheduler.ts c362fe9878839740d82db85053758be5b95834cd; chk test/retry-after.test.ts a470f2c203c43e66c47991c64654dd524f9c5c29\n[ -z \"$(git ls-files --others --exclude-standard)\" ] && echo \"no untracked\" || { echo \"NEW UNTRACKED\"; FAIL=1; }\n[ \"$(git rev-parse --short HEAD)\" = \"f9dd5f0\" ] && [ \"$(git symbolic-ref --quiet --short HEAD)\" = \"feature/a\" ] && echo \"HEAD/branch unchanged\" || { echo \"HEAD/BRANCH CHANGED\"; FAIL=1; }\necho \"--- covered_paths recompute (regular, no attrs, byte-equal to snapshot blob) ---\"; COVERED=(); for f in src/retry-worker.ts src/retry-route.ts lib/retry-after.ts; do attrs=$(git check-attr filter working-tree-encoding ident text eol -- \"$f\" | grep -v ': unspecified$' || true); B=$(git ls-tree $W \"$f\" | awk '{print $3}'); if [ -z \"$attrs\" ] && [ -f \"$f\" ] && [ ! -L \"$f\" ] && git cat-file blob \"$B\" | cmp -s - \"$f\"; then echo \"COVERED $f\"; COVERED+=(\"$f\"); else echo \"EXCLUDED $f (attrs: ${attrs:-none}; blob match: $(git cat-file blob $B | cmp -s - $f && echo yes || echo no))\"; fi; done\nFP=$(bun -e 'const { sharedLibsFingerprint } = await import(process.argv[1]); const value = sharedLibsFingerprint(JSON.parse(await Bun.stdin.text())); if (!value) process.exit(1); console.log(value);' \"/workspace/gstack/lib/review-evidence.ts\" <<'GSTACK_SHARED_LIBS_JSON'\n{\"evidence_paths\":[\"src/retry-worker.ts\",\"src/retry-route.ts\",\"lib/retry-after.ts\"],\"helper_target\":{\"path\":\"lib/retry-after.ts\",\"symbol\":\"retrySeconds\"}}\nGSTACK_SHARED_LIBS_JSON\n); echo \"recomputed FP=$FP\"; [ \"$FP\" = \"shared-libs:af037ba20b8a3674855fb65b04f1855c7f0cb5581d3b31f1c4cf5ab648ee6d33\" ] || { echo \"FP MISMATCH\"; FAIL=1; }\n[ \"${COVERED[*]}\" = \"src/retry-worker.ts lib/retry-after.ts\" ] || { echo \"UNEXPECTED COVERAGE: ${COVERED[*]}\"; FAIL=1; }\nif [ \"$FAIL\" -ne 0 ]; then echo \"!!! VERIFICATION FAILED — NOT PERSISTING\"; exit 1; fi\necho \"=== PERSIST ===\"\nTS=$(date -u +%Y-%m-%dT%H:%M:%SZ)\nRECORD=$(cat <<EOF\n{\"skill\":\"review\",\"timestamp\":\"$TS\",\"status\":\"clean\",\"issues_found\":0,\"critical\":0,\"informational\":0,\"quality_score\":10.0,\"specialists\":{\"testing\":{\"dispatched\":false,\"reason\":\"scope\"},\"maintainability\":{\"dispatched\":true,\"findings\":1,\"critical\":0,\"informational\":1},\"security\":{\"dispatched\":false,\"reason\":\"scope\"},\"performance\":{\"dispatched\":false,\"reason\":\"scope\"},\"data-migration\":{\"dispatched\":false,\"reason\":\"scope\"},\"api-contract\":{\"dispatched\":false,\"reason\":\"scope\"},\"design\":{\"dispatched\":false,\"reason\":\"scope\"},\"simplification\":{\"dispatched\":false,\"reason\":\"scope\"}},\"findings\":[{\"fingerprint\":\"$FP\",\"severity\":\"INFORMATIONAL\",\"action\":\"skipped\",\"advisory\":true,\"confidence\":9,\"path\":\"src/retry-worker.ts\",\"line\":2,\"category\":\"shared-libs\",\"summary\":\"Reuse the tested parser\",\"evidence_paths\":[\"src/retry-worker.ts\",\"src/retry-route.ts\",\"lib/retry-after.ts\"],\"helper_target\":{\"path\":\"lib/retry-after.ts\",\"symbol\":\"retrySeconds\"},\"snapshot_covered_paths\":[\"src/retry-worker.ts\",\"lib/retry-after.ts\"]}],\"commit\":\"f9dd5f0\",\"completed\":true,\"converged\":true,\"cycles\":0}\nEOF\n)\necho \"$RECORD\" | bun -e 'JSON.parse(await Bun.stdin.text()); console.log(\"record JSON valid\")' || exit 1\nif /workspace/gstack/bin/gstack-review-log \"$RECORD\" --finish \"$REVIEW_START\"; then echo \"--finish OK\"; else echo \"!!! --finish FAILED (exit $?)\"; exit 1; fi\necho \"=== READ-BACK (full) ===\"; /workspace/gstack/bin/gstack-review-read",
|
||||
"description": "Final verification, recompute fingerprint and coverage, persist review with --finish, read back full record"
|
||||
},
|
||||
"caller": {
|
||||
"type": "direct"
|
||||
}
|
||||
},
|
||||
"result": {
|
||||
"tool_use_id": "toolu_01EVWtbsxqLFr25MPuvyZsw6",
|
||||
"type": "tool_result",
|
||||
"content": "=== FINAL VERIFICATION ===\ncurrent WTREE=288a3521b692882b9936606504a4b7999286702f\nsrc/retry-worker.ts raw unchanged (8f6dd51314e9960b7bc3add9d95b8f530b85087d)\nsrc/retry-route.ts raw unchanged (17b3d7ef6893832cc9ae3cdae509b1cb97336df2)\nlib/retry-after.ts raw unchanged (00dfe6a156bae81aefc94ff1e9db54e32d774c75)\nsrc/scheduler.ts raw unchanged (c362fe9878839740d82db85053758be5b95834cd)\ntest/retry-after.test.ts raw unchanged (a470f2c203c43e66c47991c64654dd524f9c5c29)\nno untracked\nHEAD/branch unchanged\n--- covered_paths recompute (regular, no attrs, byte-equal to snapshot blob) ---\nCOVERED src/retry-worker.ts\nEXCLUDED src/retry-route.ts (attrs: src/retry-route.ts: filter: normalize; blob match: no)\nCOVERED lib/retry-after.ts\nrecomputed FP=shared-libs:af037ba20b8a3674855fb65b04f1855c7f0cb5581d3b31f1c4cf5ab648ee6d33\n=== PERSIST ===\nrecord JSON valid\n--finish OK\n=== READ-BACK (full) ===\n{\"skill\":\"review\",\"timestamp\":\"2026-09-24T16:43:01.229Z\",\"status\":\"clean\",\"issues_found\":0,\"critical\":0,\"informational\":0,\"quality_score\":10,\"findings\":[{\"severity\":\"INFORMATIONAL\",\"confidence\":9,\"path\":\"src/retry-worker.ts\",\"line\":2,\"category\":\"shared-libs\",\"summary\":\"Reuse the tested parser\",\"advisory\":true,\"action\":\"skipped\",\"evidence_paths\":[\"src/retry-worker.ts\",\"src/retry-route.ts\",\"lib/retry-after.ts\"],\"helper_target\":{\"path\":\"lib/retry-after.ts\",\"symbol\":\"retrySeconds\"},\"fingerprint\":\"shared-libs:af037ba20b8a3674855fb65b04f1855c7f0cb5581d3b31f1c4cf5ab648ee6d33\",\"snapshot_covered_paths\":[]}],\"completed\":true,\"converged\":true,\"cycles\":0,\"commit_full\":\"f9dd5f0339b79af836810c03d49d3f4db66f1e03\",\"tree\":\"0d6f7d79e489259c97311903f75fe9570a21645d\",\"dirty\":true,\"review_binding\":{\"state\":\"verified\",\"start_wtree\":\"288a3521b692882b9936606504a4b7999286702f\",\"end_wtree\":\"288a3521b692882b9936606504a4b7999286702f\",\"started_at\":\"2026-09-24T16:43:01.226Z\",\"branch_id\":\"951d42dc02dc743167ac3dd9d8decc5eee71860498c4b1707e1d38816df1ed1d\"},\"wtree\":\"288a3521b692882b9936606504a4b7999286702f\",\"review_freshness\":{\"status\":\"CURRENT\",\"reason\":\"completed clean pass on unchanged content\"}}\n{\"skill\":\"review\",\"timestamp\":\"2026-09-24T16:48:32Z\",\"status\":\"clean\",\"issues_found\":0,\"critical\":0,\"informational\":0,\"quality_score\":10,\"specialists\":{\"testing\":{\"dispatched\":false,\"reason\":\"scope\"},\"maintainability\":{\"dispatched\":true,\"findings\":1,\"critical\":0,\"informational\":1},\"security\":{\"dispatched\":false,\"reason\":\"scope\"},\"performance\":{\"dispatched\":false,\"reason\":\"scope\"},\"data-migration\":{\"dispatched\":false,\"reason\":\"scope\"},\"api-contract\":{\"dispatched\":false,\"reason\":\"scope\"},\"design\":{\"dispatched\":false,\"reason\":\"scope\"},\"simplification\":{\"dispatched\":false,\"reason\":\"scope\"}},\"findings\":[{\"fingerprint\":\"shared-libs:af037ba20b8a3674855fb65b04f1855c7f0cb5581d3b31f1c4cf5ab648ee6d33\",\"severity\":\"INFORMATIONAL\",\"action\":\"skipped\",\"advisory\":true,\"confidence\":9,\"path\":\"src/retry-worker.ts\",\"line\":2,\"category\":\"shared-libs\",\"summary\":\"Reuse the tested parser\",\"evidence_paths\":[\"src/retry-worker.ts\",\"src/retry-route.ts\",\"lib/retry-after.ts\"],\"helper_target\":{\"path\":\"lib/retry-after.ts\",\"symbol\":\"retrySeconds\"},\"snapshot_covered_paths\":[\"src/retry-worker.ts\",\"lib/retry-after.ts\"]}],\"commit\":\"f9dd5f0\",\"completed\":true,\"converged\":true,\"cycles\":0,\"commit_full\":\"f9dd5f0339b79af836810c03d49d3f4db66f1e03\",\"tree\":\"0d6f7d79e489259c97311903f75fe9570a21645d\",\"dirty\":true,\"review_binding\":{\"state\":\"verified\",\"start_wtree\":\"288a3521b692882b9936606504a4b7999286702f\",\"end_wtree\":\"288a3521b692882b9936606504a4b7999286702f\",\"started_at\":\"2026-09-24T16:46:01.848Z\",\"branch_id\":\"951d42dc02dc743167ac3dd9d8decc5eee71860498c4b1707e1d38816df1ed1d\"},\"wtree\":\"288a3521b692882b9936606504a4b7999286702f\",\"review_freshness\":{\"status\":\"CURRENT\",\"reason\":\"completed clean pass on unchanged content\"}}\n---CONFIG---\nfalse---HEAD---\nf9dd5f0\n---WTREE---\n288a3521b692882b9936606504a4b7999286702f\n---TREE---\n0d6f7d79e489259c97311903f75fe9570a21645d\n---DIRTY---\ntrue",
|
||||
"is_error": false
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
function sameCallReplay() {
|
||||
const receipt = structuredClone(sameCallReceipt);
|
||||
const events = [receipt.public_start_pair, receipt.public_finish_pair].flatMap(pair => [
|
||||
{ type: 'assistant', message: { content: [pair.call] } },
|
||||
{ type: 'user', message: { content: [pair.result] } },
|
||||
]) as any[];
|
||||
return { events, expected: receipt.expected };
|
||||
}
|
||||
|
||||
describe('retained same-call start/read and conditional finish', () => {
|
||||
test.each(['renamed variables', 'direct token assignment', 'direct start and literal read', 'different success acknowledgment'])(
|
||||
'recognizes equivalent successful command structure: %s', form => {
|
||||
const run = sameCallReplay(), token = '65892676-dd95-40b6-a3ba-6615bb41b0b6';
|
||||
const start = run.events[0].message.content[0], finish = run.events[2].message.content[0];
|
||||
if (form === 'renamed variables') {
|
||||
for (const event of run.events) {
|
||||
const block = event.message.content[0];
|
||||
if (block.input?.command) block.input.command = block.input.command.replaceAll('START_OUT', 'OBSERVED').replaceAll('REVIEW_START', 'START_TOKEN');
|
||||
else block.content = block.content.replaceAll('START_OUT', 'OBSERVED').replaceAll('REVIEW_START', 'START_TOKEN');
|
||||
}
|
||||
}
|
||||
if (form === 'direct token assignment') start.input.command = `REVIEW_START=$(/workspace/gstack/bin/gstack-review-log --start review); echo "$REVIEW_START"; cat "${run.expected.directory}/$REVIEW_START.json"`;
|
||||
if (form === 'direct start and literal read') start.input.command = `/workspace/gstack/bin/gstack-review-log --start review; cat "${run.expected.directory}/${token}.json"`;
|
||||
if (form === 'different success acknowledgment') {
|
||||
finish.input.command = finish.input.command.replace('--finish OK', 'review saved');
|
||||
run.events[3].message.content[0].content = run.events[3].message.content[0].content.replace('--finish OK', 'review saved');
|
||||
}
|
||||
expect(hasTrustedReviewStartRead(run.events, run.expected)).toBe(true);
|
||||
});
|
||||
|
||||
test.each(['native', 'separate read', 'unconditional finish', 'both controls'])('recognizes successful public evidence: %s', form => {
|
||||
const run = sameCallReplay(), token = '65892676-dd95-40b6-a3ba-6615bb41b0b6';
|
||||
const finish = run.events[2].message.content[0];
|
||||
if (['unconditional finish', 'both controls'].includes(form)) {
|
||||
finish.input.command = finish.input.command.replace('if /workspace/gstack/bin/gstack-review-log', '/workspace/gstack/bin/gstack-review-log');
|
||||
}
|
||||
if (['separate read', 'both controls'].includes(form)) {
|
||||
const record = run.events[1].message.content[0].content.split('\n').find((line: string) => line.startsWith('{"skill":"review"'));
|
||||
run.events[0].message.content[0].input.command = '/workspace/gstack/bin/gstack-review-log --start review';
|
||||
run.events[1].message.content[0].content = token;
|
||||
run.events.splice(2, 0,
|
||||
{ type: 'assistant', message: { content: [{ type: 'tool_use', id: 'separate-read-control', name: 'Read', input: { file_path: `${run.expected.directory}/${token}.json` } }] } },
|
||||
{ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'separate-read-control', content: record, is_error: false }] } });
|
||||
}
|
||||
expect(hasTrustedReviewStartRead(run.events, run.expected)).toBe(true);
|
||||
});
|
||||
|
||||
test.each(['repo', 'branch', 'wtree', 'startedAt', 'state', 'slug', 'directory'])('rejects a mismatched trusted %s', field => {
|
||||
const run = sameCallReplay();
|
||||
(run.expected as any)[field] = 'foreign-context';
|
||||
expect(hasTrustedReviewStartRead(run.events, run.expected)).toBe(false);
|
||||
});
|
||||
|
||||
test.each([
|
||||
'read before start', 'conditional read', 'conditional start', 'quoted start', 'heredoc start', 'exit before start',
|
||||
'foreign start cwd', 'foreign start state', 'rebound token', 'foreign token filter', 'rebound read directory',
|
||||
'start error', 'start missing result', 'wrong start identity', 'record printed before token', 'finish before read',
|
||||
'finish error', 'finish missing result', 'conditional finish not executed', 'function finish not invoked',
|
||||
'quoted if', 'quoted finish', 'heredoc finish', 'exit before finish', 'wrong finish token', 'wrong finish state',
|
||||
'missing success acknowledgment', 'wrong success acknowledgment', 'failure arm succeeds', 'missing readback command',
|
||||
'missing persisted binding', 'stale persisted binding', 'wrong persisted branch', 'wrong persisted tree', 'unverified persistence',
|
||||
])('rejects %s despite retained success-looking output', kind => {
|
||||
const run = sameCallReplay();
|
||||
const start = run.events[0].message.content[0], started = run.events[1].message.content[0];
|
||||
const finish = run.events[2].message.content[0], finished = run.events[3].message.content[0];
|
||||
const token = '65892676-dd95-40b6-a3ba-6615bb41b0b6';
|
||||
const read = `cat "${run.expected.directory}/$REVIEW_START.json"`;
|
||||
if (kind === 'read before start') start.input.command = `${read}; ` + start.input.command.replace(read, 'true');
|
||||
if (kind === 'conditional read') start.input.command = start.input.command.replace(read, `false && ${read}`);
|
||||
if (kind === 'conditional start') start.input.command = `if false; then ${start.input.command}; fi`;
|
||||
if (kind === 'quoted start') start.input.command = `printf '%s' ${JSON.stringify(start.input.command)}`;
|
||||
if (kind === 'heredoc start') start.input.command = `cat <<'DATA'\n${start.input.command}\nDATA`;
|
||||
if (kind === 'exit before start') start.input.command = `exit 0; ${start.input.command}`;
|
||||
if (kind === 'foreign start cwd') start.input.command = `cd /foreign; ${start.input.command}`;
|
||||
if (kind === 'foreign start state') start.input.command = `GSTACK_HOME=/foreign; ${start.input.command}`;
|
||||
if (kind === 'rebound token') start.input.command = start.input.command.replace(read, `REVIEW_START=aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa; ${read}`);
|
||||
if (kind === 'foreign token filter') start.input.command = start.input.command.replace("grep -oE '[A-Za-z0-9_.-]+'", "cat /foreign/token");
|
||||
if (kind === 'rebound read directory') start.input.command = start.input.command.replace(read, 'cat /foreign/start.json');
|
||||
if (kind === 'start error') started.is_error = true;
|
||||
if (kind === 'start missing result') started.tool_use_id = 'unpaired';
|
||||
if (kind === 'wrong start identity') started.content = started.content.replace('"skill":"review"', '"skill":"ship"');
|
||||
if (kind === 'record printed before token') started.content = started.content.split('\n').find((line: string) => line.startsWith('{"skill"')) + `\n${token}`;
|
||||
if (kind === 'finish before read') run.events = [...run.events.slice(2), ...run.events.slice(0, 2)];
|
||||
if (kind === 'finish error') finished.is_error = true;
|
||||
if (kind === 'finish missing result') finished.tool_use_id = 'unpaired';
|
||||
if (kind === 'conditional finish not executed') finish.input.command = `if false; then\n${finish.input.command}\nfi`;
|
||||
if (kind === 'function finish not invoked') finish.input.command = `unused(){\n${finish.input.command}\n}`;
|
||||
if (kind === 'quoted if') finish.input.command = finish.input.command.replace('if /workspace/gstack/bin/gstack-review-log', '"if" /workspace/gstack/bin/gstack-review-log');
|
||||
if (kind === 'quoted finish') finish.input.command = `printf '%s' ${JSON.stringify(finish.input.command)}`;
|
||||
if (kind === 'heredoc finish') finish.input.command = `cat <<'DATA'\n${finish.input.command}\nDATA`;
|
||||
if (kind === 'exit before finish') finish.input.command = `exit 0; ${finish.input.command}`;
|
||||
if (kind === 'wrong finish token') finish.input.command = finish.input.command.replace(`REVIEW_START=${token}`, 'REVIEW_START=aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa');
|
||||
if (kind === 'wrong finish state') finish.input.command = `GSTACK_HOME=/foreign; ${finish.input.command}`;
|
||||
if (kind === 'missing success acknowledgment') finished.content = finished.content.replace('--finish OK\n', '');
|
||||
if (kind === 'wrong success acknowledgment') finished.content = finished.content.replace('--finish OK', '--finish FAILED');
|
||||
if (kind === 'failure arm succeeds') finish.input.command = finish.input.command.replace('exit 1; fi\necho "=== READ-BACK', 'exit 0; fi\necho "=== READ-BACK');
|
||||
if (kind === 'missing readback command') finish.input.command = finish.input.command.replace('/workspace/gstack/bin/gstack-review-read\n', 'true\n').replace(/\/workspace\/gstack\/bin\/gstack-review-read$/, 'true');
|
||||
if (kind.startsWith('missing persisted') || kind.startsWith('stale persisted') || kind.startsWith('wrong persisted') || kind === 'unverified persistence') {
|
||||
finished.content = finished.content.split('\n').map((line: string) => {
|
||||
let row: any;
|
||||
try { row = JSON.parse(line); } catch { return line; }
|
||||
if (kind === 'missing persisted binding') delete row.review_binding;
|
||||
if (kind === 'stale persisted binding' && row.review_binding) row.review_binding.started_at = '2000-01-01T00:00:00Z';
|
||||
if (kind === 'wrong persisted branch' && row.review_binding) row.review_binding.branch_id = 'wrong-branch';
|
||||
if (kind === 'wrong persisted tree' && row.review_binding) row.review_binding.end_wtree = 'wrong-tree';
|
||||
if (kind === 'unverified persistence' && row.review_binding) row.review_binding.state = 'unverified';
|
||||
return JSON.stringify(row);
|
||||
}).join('\n');
|
||||
}
|
||||
expect(hasTrustedReviewStartRead(run.events, run.expected)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
import { afterEach, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import native from './fixtures/shared-libs-resolved-reads-public.json';
|
||||
import { E2E_TOUCHFILES, GLOBAL_TOUCHFILES, selectTests } from './helpers/touchfiles';
|
||||
|
||||
const source = fs.readFileSync(path.join(import.meta.dir, 'skill-e2e-shared-libs-paths.test.ts'), 'utf8');
|
||||
const detectorStart = source.indexOf('function sourceReadTrace(');
|
||||
const callbackStart = source.indexOf('async function exerciseEligibility(');
|
||||
const callbackEnd = source.indexOf('\ndescribeE2E(', callbackStart);
|
||||
if (detectorStart < 0 || callbackStart <= detectorStart || callbackEnd <= callbackStart) throw new Error('Missing production detector or callback');
|
||||
const transpiler = new Bun.Transpiler({ loader: 'ts' });
|
||||
const detect = new Function('fs', 'path', `${transpiler.transformSync(source.slice(detectorStart, callbackStart))}; return sourceReadTrace;`)(fs, path);
|
||||
const roots: string[] = [];
|
||||
afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); });
|
||||
const readId = 'toolu_0165HzBNP4ydttNR7XzQBxnK';
|
||||
const nativeRead = native.matching_results.find(row => row.tool_use_id === readId)!;
|
||||
const aliases = ['src/retry-route.ts', 'src/retry-alias/retry.ts'];
|
||||
const targets = ['.fixture/first-party/direct-route.ts', '.fixture/first-party/routes/retry.ts'];
|
||||
|
||||
function fixture() {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-resolved-read-'));
|
||||
roots.push(root);
|
||||
const repo = path.join(root, 'repo');
|
||||
for (const file of ['src/retry-worker.ts', 'lib/retry-after.ts', ...targets]) {
|
||||
const body = nativeRead.content.split(`=== ${file} ===\n`)[1]?.split('\n=== ')[0];
|
||||
if (!body) throw new Error(`Missing native file contents: ${file}`);
|
||||
fs.mkdirSync(path.dirname(path.join(repo, file)), { recursive: true });
|
||||
fs.writeFileSync(path.join(repo, file), body);
|
||||
}
|
||||
fs.symlinkSync('../.fixture/first-party/direct-route.ts', path.join(repo, aliases[0]));
|
||||
fs.symlinkSync('../.fixture/first-party/routes', path.join(repo, 'src/retry-alias'));
|
||||
return { root, repo, state: path.join(root, 'state'), bin: path.join(root, 'bin') };
|
||||
}
|
||||
|
||||
function capture() {
|
||||
const tools = structuredClone(native.tools);
|
||||
const events = tools.flatMap(tool => {
|
||||
const output = native.matching_results.find(row => row.tool_use_id === tool.id);
|
||||
return [{ type: 'assistant', message: { content: [{ type: 'tool_use', ...tool }] } },
|
||||
...(output ? [{ type: 'user', message: { content: [structuredClone(output)] } }] : [])];
|
||||
});
|
||||
return { exitReason: 'success', output: '', events,
|
||||
toolCalls: tools.map(tool => ({ tool: tool.name, input: tool.input, output: '' })) };
|
||||
}
|
||||
|
||||
test('the retained native resolved-path read proves both current authored callers', () => {
|
||||
const f = fixture();
|
||||
const result = capture();
|
||||
const reads = detect(result, f, aliases);
|
||||
for (const alias of aliases) expect(reads).toContain(alias);
|
||||
for (const target of targets) expect(fs.readFileSync(path.join(f.repo, target), 'utf8')).toContain('changed after the prior decision');
|
||||
});
|
||||
|
||||
test.each(['missing-result', 'failed-result', 'wrong-result-id', 'metadata-only', 'stale-content', 'assistant-only'])('%s cannot prove resolved source reads', kind => {
|
||||
const f = fixture();
|
||||
const result: any = capture();
|
||||
const event = result.events.find((event: any) => event.message.content[0].tool_use_id === readId);
|
||||
const block = event.message.content[0];
|
||||
if (kind === 'missing-result') result.events = result.events.filter((candidate: any) => candidate !== event);
|
||||
if (kind === 'failed-result') block.is_error = true;
|
||||
if (kind === 'wrong-result-id') block.tool_use_id = 'unrelated';
|
||||
if (kind === 'metadata-only') block.content = 'Both target files exist and are 738 bytes.';
|
||||
if (kind === 'stale-content') block.content = block.content.replaceAll('// Authored caller changed after the prior decision (symlinks).', '');
|
||||
if (kind === 'assistant-only') event.type = 'assistant';
|
||||
const reads = detect(result, f, aliases);
|
||||
for (const alias of aliases) expect(reads).not.toContain(alias);
|
||||
});
|
||||
|
||||
test('a canonical Read result accepts native line prefixes but still requires current contents', () => {
|
||||
const f = fixture();
|
||||
const content = fs.readFileSync(path.join(f.repo, targets[0]), 'utf8');
|
||||
const input = { file_path: path.join(f.repo, targets[0]) };
|
||||
const block = { type: 'tool_result', tool_use_id: 'read-target', content: content.split('\n').map((line, index) => `${index + 1}→${line}`).join('\n') };
|
||||
const result = { toolCalls: [{ tool: 'Read', input }], events: [
|
||||
{ type: 'assistant', message: { content: [{ type: 'tool_use', id: 'read-target', name: 'Read', input }] } },
|
||||
{ type: 'user', message: { content: [block] } },
|
||||
] };
|
||||
expect(detect(result, f, aliases)).toContain(aliases[0]);
|
||||
expect(detect(result, f, aliases)).not.toContain(aliases[1]);
|
||||
block.content = '';
|
||||
expect(detect(result, f, aliases)).not.toContain(aliases[0]);
|
||||
});
|
||||
|
||||
test('resolved aliases cannot grant credit for targets outside the fixture repository', () => {
|
||||
const f = fixture();
|
||||
const outside = path.join(f.root, 'outside.ts');
|
||||
fs.writeFileSync(outside, fs.readFileSync(path.join(f.repo, targets[0])));
|
||||
fs.unlinkSync(path.join(f.repo, aliases[0]));
|
||||
fs.symlinkSync(outside, path.join(f.repo, aliases[0]));
|
||||
const result: any = capture();
|
||||
for (const tool of result.toolCalls) if (tool.tool === 'Bash') tool.input.command = tool.input.command.replaceAll(targets[0], outside);
|
||||
for (const event of result.events) for (const block of event.message.content) {
|
||||
if (block.type === 'tool_use' && block.name === 'Bash') block.input.command = block.input.command.replaceAll(targets[0], outside);
|
||||
}
|
||||
expect(detect(result, f, aliases)).not.toContain(aliases[0]);
|
||||
expect(detect(result, f, aliases)).toContain(aliases[1]);
|
||||
});
|
||||
|
||||
test.each(['.backup', '/foreign-root/'])('similarly named read targets cannot acquire alias credit: %s', spelling => {
|
||||
const f = fixture();
|
||||
const result: any = capture();
|
||||
const tool = result.events.flatMap((event: any) => event.message.content).find((block: any) => block.id === readId);
|
||||
for (const target of targets) tool.input.command = tool.input.command.replaceAll(target, spelling === '.backup' ? target + spelling : spelling + target);
|
||||
const reads = detect(result, f, aliases);
|
||||
for (const alias of aliases) expect(reads).not.toContain(alias);
|
||||
});
|
||||
|
||||
test('a Read result from another repository cannot prove an identically named target', () => {
|
||||
const f = fixture();
|
||||
const input = { file_path: path.join(f.root, 'another-repo', targets[0]) };
|
||||
const result = { toolCalls: [{ tool: 'Read', input }], events: [
|
||||
{ type: 'assistant', message: { content: [{ type: 'tool_use', id: 'foreign-read', name: 'Read', input }] } },
|
||||
{ type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: 'foreign-read', content: fs.readFileSync(path.join(f.repo, targets[0]), 'utf8') }] } },
|
||||
] };
|
||||
expect(detect(result, f, aliases)).not.toContain(aliases[0]);
|
||||
});
|
||||
|
||||
test.each(['Read', 'Bash'])('direct alias reads retain the existing %s contract', tool => {
|
||||
const f = fixture();
|
||||
const result = { toolCalls: [{ tool, input: tool === 'Read' ? { file_path: aliases[0] } : { command: `cat ${aliases[0]}` } }] };
|
||||
expect(detect(result, f, aliases)).toContain(aliases[0]);
|
||||
});
|
||||
|
||||
test.each(['test/shared-libs-source-reads.test.ts', 'test/fixtures/shared-libs-resolved-reads-public.json'])('%s selects every owning path callback without a global fallback', file => {
|
||||
expect(selectTests([file], E2E_TOUCHFILES, GLOBAL_TOUCHFILES).selected.sort()).toEqual([
|
||||
'shared-libs-review-index-flags', 'shared-libs-review-path-eligibility', 'shared-libs-review-prior-coverage',
|
||||
]);
|
||||
});
|
||||
|
||||
test.each([false, true])('the actual eligibility callback consumes the read detector result (missing output=%s)', async missingOutput => {
|
||||
const f = fixture();
|
||||
const result: any = capture();
|
||||
if (missingOutput) result.events = result.events.filter((event: any) => event.message.content[0].tool_use_id !== readId);
|
||||
const rows: any[] = [];
|
||||
let detectorCalls = 0;
|
||||
const exercise = new Function('deps', `const { captures, preparePathEligibilityFixture, fs, path,
|
||||
reviewLifecycleInstructions, reviewRevalidationPrompt, runSharedInteractive, readRequests,
|
||||
toolCommandTrace, sourceReadTrace, fixtureWorkingTree, reviewRecords, expect, CAPTURE_LONG_MS } = deps;
|
||||
${transpiler.transformSync(source.slice(callbackStart, callbackEnd))}; return exerciseEligibility;`)({
|
||||
captures: { runAttempt: (_name: string, _kinds: string[], _timeout: number, work: any) => work({ add: (_kind: string, row: any) => rows.push(row) }) },
|
||||
preparePathEligibilityFixture: () => ({ fixture: f, sourcePaths: aliases, beforeTree: 'tree', current: { evidence_paths: ['src/retry-worker.ts', 'lib/retry-after.ts', ...aliases] } }),
|
||||
fs, path, expect, CAPTURE_LONG_MS: 600_000,
|
||||
reviewLifecycleInstructions: () => 'read the authored sources', reviewRevalidationPrompt: () => 'revalidate',
|
||||
runSharedInteractive: async () => ({ result, questions: [{}] }), readRequests: () => [],
|
||||
toolCommandTrace: (value: any) => value.toolCalls.filter((call: any) => call.tool === 'Bash').map((call: any) => call.input.command),
|
||||
sourceReadTrace: (...args: any[]) => { detectorCalls++; return detect(...args); },
|
||||
fixtureWorkingTree: () => 'tree',
|
||||
reviewRecords: () => [{ skill: 'review' }, { skill: 'review', status: 'clean', issues_found: 0, completed: true, converged: true,
|
||||
review_binding: { state: 'verified' }, findings: [{ advisory: true, action: 'skipped', helper_target: { path: 'lib/retry-after.ts', symbol: 'retrySeconds' },
|
||||
fingerprint: `shared-libs:${'a'.repeat(64)}`, evidence_paths: ['src/retry-worker.ts', ...aliases], snapshot_covered_paths: ['src/retry-worker.ts', 'lib/retry-after.ts'] }] }],
|
||||
});
|
||||
if (missingOutput) await expect(exercise('shared-libs-review-path-eligibility', ['symlinks'])).rejects.toThrow();
|
||||
else await exercise('shared-libs-review-path-eligibility', ['symlinks']);
|
||||
expect(detectorCalls).toBe(1);
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0].passed).toBe(!missingOutput);
|
||||
expect(fs.existsSync(f.root)).toBe(false);
|
||||
});
|
||||
@@ -35,21 +35,33 @@ describeE2E('AUQ consistency across runs (periodic)', () => {
|
||||
test(
|
||||
`carved /plan-ceo-review AUQ format + substance stable across ${N_RUNS} runs`,
|
||||
async () => {
|
||||
const caseDeadline = Date.now() + N_RUNS * CAPTURE_MS + 60_000;
|
||||
const runs: Array<{ i: number; present: Set<string>; substance: number; empty: boolean }> = [];
|
||||
const problems: string[] = [];
|
||||
const dirs: string[] = [];
|
||||
let captures: PromiseSettledResult<string>[];
|
||||
|
||||
for (let i = 0; i < N_RUNS; i++) {
|
||||
const carved = carvedSkill();
|
||||
const dir = setupPlanCeoDir({
|
||||
skillMd: carved.skillMd,
|
||||
sectionsFrom: carved.sectionsFrom,
|
||||
tmpPrefix: `auq-consistency-${i}-`,
|
||||
});
|
||||
let text = '';
|
||||
try {
|
||||
text = await captureModeSelectionAuq({ planDir: dir, testName: `auq-consistency-${i}`, runId });
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
try {
|
||||
captures = await Promise.allSettled(Array.from({ length: N_RUNS }, async (_, i) => {
|
||||
const carved = carvedSkill();
|
||||
const dir = setupPlanCeoDir({
|
||||
skillMd: carved.skillMd,
|
||||
sectionsFrom: carved.sectionsFrom,
|
||||
tmpPrefix: `auq-consistency-${i}-`,
|
||||
});
|
||||
dirs.push(dir);
|
||||
return captureModeSelectionAuq({ planDir: dir, testName: `auq-consistency-${i}`, runId, caseDeadline });
|
||||
}));
|
||||
} finally {
|
||||
for (const dir of dirs) {
|
||||
try { fs.rmSync(dir, { recursive: true, force: true }); }
|
||||
catch (error) { problems.push(`fixture cleanup failed: ${error}`); }
|
||||
}
|
||||
}
|
||||
|
||||
for (const [i, capture] of captures.entries()) {
|
||||
if (capture.status === 'rejected') problems.push(`run ${i + 1} capture failed: ${capture.reason}`);
|
||||
const text = capture.status === 'fulfilled' ? capture.value : '';
|
||||
const present = new Set(AUQ_FORMAT_ELEMENTS.filter(e => e.re.test(text)).map(e => e.field));
|
||||
let substance = 0;
|
||||
if (text.trim()) {
|
||||
@@ -66,8 +78,6 @@ describeE2E('AUQ consistency across runs (periodic)', () => {
|
||||
);
|
||||
}
|
||||
|
||||
const problems: string[] = [];
|
||||
|
||||
const anyEmpty = runs.filter(r => r.empty).map(r => r.i + 1);
|
||||
if (anyEmpty.length > 0) problems.push(`run(s) produced no AUQ at all: ${anyEmpty.join(',')}`);
|
||||
|
||||
|
||||
@@ -38,8 +38,8 @@ import { judgeRecommendation } from './helpers/llm-judge';
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
const runId = `auq-ab-${process.env.EVALS_RUN_ID ?? 'local'}`;
|
||||
|
||||
async function grade(label: string, dir: string) {
|
||||
const text = await captureModeSelectionAuq({ planDir: dir, testName: `auq-ab-${label}`, runId });
|
||||
async function grade(label: string, dir: string, caseDeadline: number) {
|
||||
const text = await captureModeSelectionAuq({ planDir: dir, testName: `auq-ab-${label}`, runId, caseDeadline });
|
||||
const fmt = scoreAuqFormat(text);
|
||||
let substance = 0;
|
||||
let present = false;
|
||||
@@ -62,25 +62,34 @@ describeE2E('AUQ no-degradation: verbose vs carved (periodic)', () => {
|
||||
test(
|
||||
'carved plan-ceo-review AUQ is not worse than verbose on the same prompt',
|
||||
async () => {
|
||||
const carved = carvedSkill();
|
||||
const carvedDir = setupPlanCeoDir({
|
||||
skillMd: carved.skillMd,
|
||||
sectionsFrom: carved.sectionsFrom,
|
||||
tmpPrefix: 'auq-ab-carved-',
|
||||
});
|
||||
const verboseDir = setupPlanCeoDir({
|
||||
skillMd: verboseSkill(),
|
||||
tmpPrefix: 'auq-ab-verbose-',
|
||||
});
|
||||
|
||||
let c, v;
|
||||
const caseDeadline = Date.now() + CAPTURE_LONG_MS;
|
||||
const dirs: string[] = [];
|
||||
const cleanupErrors: unknown[] = [];
|
||||
let results: PromiseSettledResult<Awaited<ReturnType<typeof grade>>>[];
|
||||
try {
|
||||
c = await grade('CARVED', carvedDir);
|
||||
v = await grade('VERBOSE', verboseDir);
|
||||
results = await Promise.allSettled(['CARVED', 'VERBOSE'].map(async label => {
|
||||
const skill = label === 'CARVED' ? carvedSkill() : { skillMd: verboseSkill() };
|
||||
const dir = setupPlanCeoDir({
|
||||
...skill,
|
||||
tmpPrefix: `auq-ab-${label.toLowerCase()}-`,
|
||||
});
|
||||
dirs.push(dir);
|
||||
return grade(label, dir, caseDeadline);
|
||||
}));
|
||||
} finally {
|
||||
fs.rmSync(carvedDir, { recursive: true, force: true });
|
||||
fs.rmSync(verboseDir, { recursive: true, force: true });
|
||||
for (const dir of dirs) {
|
||||
try { fs.rmSync(dir, { recursive: true, force: true }); }
|
||||
catch (error) { cleanupErrors.push(error); }
|
||||
}
|
||||
}
|
||||
const [carvedResult, verboseResult] = results;
|
||||
if (carvedResult.status === 'rejected' || verboseResult.status === 'rejected' || cleanupErrors.length > 0) {
|
||||
throw new AggregateError(
|
||||
[...results.flatMap(result => result.status === 'rejected' ? [result.reason] : []), ...cleanupErrors],
|
||||
'AUQ A/B capture or cleanup failed',
|
||||
);
|
||||
}
|
||||
const c = carvedResult.value, v = verboseResult.value;
|
||||
|
||||
const summary = [
|
||||
`CARVED : format ${c.fmt.present}/${c.fmt.total}, substance ${c.substance}`,
|
||||
|
||||
+118
-26
@@ -3,6 +3,7 @@ import { CAPTURE_MS, CAPTURE_LONG_MS } from './helpers/eval-budgets';
|
||||
import { runSkillTest, type SkillTestResult } from './helpers/session-runner';
|
||||
import { OFFICE_HOURS_BUN_GRACE_MS, runRecordedOfficeHoursAttempt } from './helpers/office-hours-attempt';
|
||||
import { resolveEvalModel } from '../lib/eval-model';
|
||||
import { getProjectEvalDir } from './helpers/eval-store';
|
||||
import { callJudge } from './helpers/llm-judge';
|
||||
import {
|
||||
ROOT, runId, evalsEnabled, selectedTests,
|
||||
@@ -811,38 +812,129 @@ describeIfSelected('Design review detector shim E2E', ['design-review-detector-s
|
||||
});
|
||||
|
||||
testConcurrentIfSelected('design-review-detector-shim', async () => {
|
||||
const result = await runSkillTest({
|
||||
prompt: `You are in a git repo on branch feature/landing with changes against main (the base branch).
|
||||
const started = Date.now();
|
||||
let evidenceDir: string | undefined;
|
||||
let result: SkillTestResult | undefined;
|
||||
let passed = false;
|
||||
let failure: unknown;
|
||||
try {
|
||||
evidenceDir = fs.mkdtempSync(path.join(os.tmpdir(), 'detector-source-evidence-'));
|
||||
const receiptPath = path.join(evidenceDir, 'calls.jsonl');
|
||||
const outPath = path.join(repoDir, 'detector-output.md');
|
||||
fs.rmSync(outPath, { force: true });
|
||||
fs.writeFileSync(path.join(evidenceDir, 'bun'), `#!${process.execPath}
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
const argv = process.argv.slice(2);
|
||||
const wrapper = argv.findIndex(arg => {
|
||||
try { return fs.realpathSync(path.resolve(arg)) === ${JSON.stringify(fs.realpathSync(path.join(ROOT, 'bin', 'gstack-design-detect.ts')))}; }
|
||||
catch { return false; }
|
||||
});
|
||||
const dir = wrapper < 0 ? null : fs.mkdtempSync(${JSON.stringify(path.join(evidenceDir, 'call-'))});
|
||||
const log = dir && path.join(dir, 'engine.jsonl');
|
||||
const run = spawnSync(process.execPath, argv, {
|
||||
cwd: process.cwd(), stdio: 'inherit', timeout: ${CAPTURE_MS},
|
||||
env: { ...process.env, ...(log ? { IMPECCABLE_FAKE_LOG: log } : {}) },
|
||||
});
|
||||
if (dir) fs.appendFileSync(${JSON.stringify(receiptPath)}, JSON.stringify({
|
||||
argv: argv.slice(wrapper + 1), cwd: process.cwd(), exit: run.status,
|
||||
engine: log && fs.existsSync(log) ? fs.readFileSync(log, 'utf8').trim().split('\\n').map(line => JSON.parse(line)) : [],
|
||||
}) + '\\n', { mode: 0o600 });
|
||||
process.exitCode = run.status ?? 1;
|
||||
`, { mode: 0o755 });
|
||||
result = await runSkillTest({
|
||||
prompt: `You are in a git repo on branch feature/landing with changes against main (the base branch).
|
||||
Read design-review-detector.md: it is the Setup "Design detector" block and "Phase 0: mechanical scan" from /design-review.
|
||||
This is a diff-aware run with no URL, so it is SOURCE mode. Run the probe, then the Phase 0 source-mode scan with base main, exactly as written (use --host claude).
|
||||
Do not run any browser step, do not fix anything, do not run npx.
|
||||
Then write ${repoDir}/detector-output.md: one FINDING-NNN row per rule in the DETECT_TOP block, each tagged with its [rule-id] and the printed impact, plus the first line the probe printed.`,
|
||||
workingDirectory: repoDir,
|
||||
maxTurns: 15,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'design-review-detector-shim',
|
||||
runId,
|
||||
env: { IMPECCABLE_BIN: path.join(engineDir, 'impeccable'), IMPECCABLE_FAKE_OUTPUT: DETECT_SAMPLE },
|
||||
});
|
||||
workingDirectory: repoDir,
|
||||
maxTurns: 15,
|
||||
timeout: CAPTURE_MS,
|
||||
testName: 'design-review-detector-shim',
|
||||
runId,
|
||||
env: {
|
||||
IMPECCABLE_BIN: path.join(engineDir, 'impeccable'), IMPECCABLE_FAKE_OUTPUT: DETECT_SAMPLE,
|
||||
PATH: `${evidenceDir}${path.delimiter}${process.env.PATH ?? ''}`,
|
||||
},
|
||||
});
|
||||
|
||||
logCost('/design-review detector shim (source)', result);
|
||||
recordE2E(evalCollector, '/design-review detector shim', 'Design review detector shim E2E (source mode)', result);
|
||||
expect(result.exitReason).toBe('success');
|
||||
logCost('/design-review detector shim (source)', result);
|
||||
expect(result.exitReason).toBe('success');
|
||||
expect(result.browseErrors).toEqual([]);
|
||||
|
||||
const bash = result.toolCalls.filter(c => c.tool === 'Bash').map(c => String(c.input?.command ?? ''));
|
||||
expect(bash.some(c => c.includes('gstack-design-detect.ts probe'))).toBe(true);
|
||||
expect(bash.some(c => /gstack-design-detect\.ts scan --changed main/.test(c))).toBe(true);
|
||||
expect(bash.some(c => c.includes('npx impeccable'))).toBe(false);
|
||||
// The sentinel is evidence in the tool output and the report, not something the
|
||||
// agent must repeat in its closing message.
|
||||
const toolOutputs = result.toolCalls.map(c => String(c.output ?? '')).join('\n');
|
||||
const outPath = path.join(repoDir, 'detector-output.md');
|
||||
expect(fs.existsSync(outPath)).toBe(true);
|
||||
const out = fs.readFileSync(outPath, 'utf-8');
|
||||
expect(toolOutputs.includes('IMPECCABLE_READY') || out.includes('IMPECCABLE_READY')).toBe(true);
|
||||
expect(out).toContain('FINDING-001');
|
||||
expect(out).toContain('[ai-color-palette]');
|
||||
expect(out).toContain('[low-contrast]');
|
||||
const bash = result.toolCalls.filter(c => c.tool === 'Bash').map(c => String(c.input?.command ?? ''));
|
||||
expect(bash.some(c => c.includes('npx impeccable'))).toBe(false);
|
||||
expect(bash.some(c => /\$B\b|\bbrowse\s|\baside\s+repl\b|\bplaywright\b|\bpuppeteer\b/.test(c))).toBe(false);
|
||||
expect(result.toolCalls.some(c => /browser|browse|playwright|puppeteer/i.test(c.tool))).toBe(false);
|
||||
const calls: Array<{ argv: string[]; cwd: string; exit: number; engine: Array<{ argv: string[]; cwd: string }> }> =
|
||||
fs.readFileSync(receiptPath, 'utf8').trim().split('\n').map(line => JSON.parse(line));
|
||||
const probes = calls.filter(c => c.argv[0] === 'probe');
|
||||
expect(probes.length).toBeGreaterThan(0);
|
||||
for (const probe of probes) {
|
||||
expect(probe.argv).toEqual(['probe', '--host', 'claude']);
|
||||
expect(probe.cwd).toBe(fs.realpathSync(repoDir));
|
||||
expect(probe.exit).toBe(0);
|
||||
}
|
||||
const scans = calls.filter(c => c.argv[0] === 'scan');
|
||||
expect(scans.length).toBeGreaterThan(0);
|
||||
for (const scan of scans) {
|
||||
expect(scan.cwd).toBe(fs.realpathSync(repoDir));
|
||||
expect(scan.exit).toBe(2);
|
||||
expect(scan.argv[scan.argv.indexOf('--changed') + 1]).toBe('main');
|
||||
expect(scan.argv[scan.argv.indexOf('--host') + 1]).toBe('claude');
|
||||
expect(scan.argv[scan.argv.indexOf('--format') + 1]).toBe('gstack');
|
||||
expect(scan.argv.slice(1).sort()).toEqual(['--changed', 'main', '--format', 'gstack', '--host', 'claude'].sort());
|
||||
expect(scan.engine).toHaveLength(1);
|
||||
expect(scan.engine[0].cwd).toBe(fs.realpathSync(repoDir));
|
||||
expect(scan.engine[0].argv).toEqual(['detect', '--json',
|
||||
fs.realpathSync(path.join(repoDir, 'index.html')), fs.realpathSync(path.join(repoDir, 'styles.css'))]);
|
||||
}
|
||||
const toolOutputs = result.toolCalls.map(c => String(c.output ?? '')).join('\n');
|
||||
expect(fs.existsSync(outPath)).toBe(true);
|
||||
const out = fs.readFileSync(outPath, 'utf-8');
|
||||
expect(toolOutputs.includes('IMPECCABLE_READY') || out.includes('IMPECCABLE_READY')).toBe(true);
|
||||
expect(out).toContain('FINDING-001');
|
||||
expect(out).toContain('[ai-color-palette]');
|
||||
expect(out).toContain('[low-contrast]');
|
||||
passed = true;
|
||||
} catch (error) {
|
||||
failure = error;
|
||||
throw error;
|
||||
} finally {
|
||||
let artifactNote = '';
|
||||
if (evidenceDir && (process.env.EVALS_RUN_ID || process.env.GSTACK_EVAL_DIR)) {
|
||||
try {
|
||||
const segment = (process.env.EVALS_RUN_ID || 'local').replace(/[^a-zA-Z0-9_-]/g, '_').slice(0, 120);
|
||||
const root = path.resolve(process.env.GSTACK_EVAL_DIR || getProjectEvalDir(), 'design-detector', segment);
|
||||
fs.mkdirSync(root, { recursive: true, mode: 0o700 });
|
||||
const retained = fs.mkdtempSync(path.join(root, `design-review-detector-shim-${started}-`));
|
||||
fs.chmodSync(retained, 0o700);
|
||||
const receipt = path.join(evidenceDir, 'calls.jsonl');
|
||||
fs.writeFileSync(path.join(retained, 'calls.jsonl'), fs.existsSync(receipt) ? fs.readFileSync(receipt, 'utf8') : '', { mode: 0o600 });
|
||||
artifactNote = `Detector artifacts: ${retained}`;
|
||||
console.log(artifactNote);
|
||||
} catch (error) {
|
||||
artifactNote = `Detector artifact write failed: ${String(error)}`;
|
||||
console.error(artifactNote);
|
||||
}
|
||||
}
|
||||
if (evidenceDir) { try { fs.rmSync(evidenceDir, { recursive: true, force: true }); } catch {} }
|
||||
const error = (failure instanceof Error ? failure.message : String(failure)) + (artifactNote ? `\n${artifactNote}` : '');
|
||||
if (result) {
|
||||
recordE2E(evalCollector, '/design-review detector shim', 'Design review detector shim E2E (source mode)', result, {
|
||||
passed, ...(passed ? {} : { error }),
|
||||
});
|
||||
} else {
|
||||
evalCollector?.addTest({
|
||||
name: '/design-review detector shim', suite: 'Design review detector shim E2E (source mode)', tier: 'e2e',
|
||||
passed: false, duration_ms: Date.now() - started, cost_usd: 0,
|
||||
model: process.env.EVALS_MODEL ?? resolveEvalModel('capture'), exit_reason: 'harness_error',
|
||||
error: `${error}\nRunner returned no result; cost and usage unavailable.`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}, CAPTURE_MS);
|
||||
|
||||
// DOM mode needs a browser engine for the dump: gstack's own browse binary
|
||||
|
||||
@@ -16,13 +16,47 @@ const collector = e2eTierEnabled('gate') ? new EvalCollector('e2e') : null;
|
||||
const captures = new SharedCaptureAccumulator();
|
||||
afterAll(async () => { await captures.finalize(collector); });
|
||||
|
||||
function sourceReadTrace(result: any): string {
|
||||
return result.toolCalls.flatMap((call: any) => {
|
||||
if (call.tool === 'Read') return [String(call.input?.file_path || '')];
|
||||
const command = String(call.input?.command || '');
|
||||
if (call.tool === 'Bash' && /\b(?:cat|sed|head|tail|nl)\b|readFile|Bun\.file/.test(command)) return [command];
|
||||
return [];
|
||||
}).join('\n');
|
||||
function sourceReadTrace(result: any, fixture: SharedLibsFixture, sources: string[]): string {
|
||||
const readInput = (tool: string, input: any): string => {
|
||||
if (tool === 'Read') return String(input?.file_path || '');
|
||||
const command = String(input?.command || '');
|
||||
return tool === 'Bash' && /\b(?:cat|sed|head|tail|nl)\b|readFile|Bun\.file/.test(command) ? command : '';
|
||||
};
|
||||
const reads: string[] = result.toolCalls.map((call: any) => readInput(call.tool, call.input)).filter(Boolean);
|
||||
const calls = new Map<string, { tool: string; read: string }>();
|
||||
const returned: Array<{ tool: string; read: string; text: string }> = [];
|
||||
for (const event of result.events ?? []) {
|
||||
if (!Array.isArray(event.message?.content)) continue;
|
||||
for (const block of event.message.content) {
|
||||
if (event.type === 'assistant' && block.type === 'tool_use') {
|
||||
const read = readInput(block.name, block.input);
|
||||
if (read) calls.set(block.id, { tool: block.name, read });
|
||||
}
|
||||
if (event.type !== 'user' || block.type !== 'tool_result' || block.is_error === true) continue;
|
||||
const call = calls.get(block.tool_use_id);
|
||||
if (!call) continue;
|
||||
const text = typeof block.content === 'string' ? block.content
|
||||
: Array.isArray(block.content) ? block.content.filter((part: any) => part.type === 'text').map((part: any) => part.text).join('\n') : '';
|
||||
returned.push({ ...call, text: text.replace(/^\s*\d+→/gm, '') });
|
||||
}
|
||||
}
|
||||
if (!returned.length) return reads.join('\n');
|
||||
const repo = fs.realpathSync(fixture.repo);
|
||||
for (const source of sources) {
|
||||
let resolved: string;
|
||||
try { resolved = fs.realpathSync(path.resolve(repo, source)); }
|
||||
catch { continue; }
|
||||
const relative = path.relative(repo, resolved);
|
||||
if (!relative || relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) continue;
|
||||
if (resolved === path.resolve(repo, source)) continue;
|
||||
const contents = fs.readFileSync(resolved, 'utf8');
|
||||
if (!contents) continue;
|
||||
const spellings = [relative, resolved].map(value => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'));
|
||||
const namedPath = new RegExp(`(?:^|[\\s"'=;])(?:\\./)?(?:${spellings.join('|')})(?=$|[\\s"';|)])`);
|
||||
if (returned.some(({ tool, read, text }) => (tool === 'Read' ? path.resolve(repo, read) === resolved : namedPath.test(read))
|
||||
&& text.includes(contents))) reads.push(source);
|
||||
}
|
||||
return reads.join('\n');
|
||||
}
|
||||
|
||||
async function exerciseEligibility(testId: string, kinds: PathEligibilityCase[]) {
|
||||
@@ -56,7 +90,7 @@ async function exerciseEligibility(testId: string, kinds: PathEligibilityCase[])
|
||||
expect(trace).toContain('gstack-review-log');
|
||||
expect(trace).toContain('--start');
|
||||
expect(trace).toContain('--finish');
|
||||
const reads = sourceReadTrace(result);
|
||||
const reads = sourceReadTrace(result, f, prepared.sourcePaths);
|
||||
expect(reads).toContain('src/retry-worker.ts');
|
||||
expect(reads).toContain('lib/retry-after.ts');
|
||||
for (const source of prepared.sourcePaths) {
|
||||
|
||||
@@ -14,12 +14,12 @@
|
||||
* unattributed (retrying without knowing what to re-run is meaningless,
|
||||
* so main() must see [] and skip the retry).
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { describe, test, expect, spyOn } from 'bun:test';
|
||||
import * as os from 'os';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import {
|
||||
fullSuiteJobs,
|
||||
MAX_FULL_SUITE_JOBS,
|
||||
RESERVED_CPUS,
|
||||
runFreeShard,
|
||||
} from '../scripts/test-free-shards';
|
||||
|
||||
@@ -37,16 +37,85 @@ function withJobsEnv<T>(value: string | undefined, fn: () => T): T {
|
||||
}
|
||||
|
||||
describe('test-free-shards: fullSuiteJobs (GSTACK_FREE_JOBS override)', () => {
|
||||
test('unset and empty string both take the computed default — cpus minus reserve, capped, floor 1', () => {
|
||||
const expected = Math.max(1, Math.min(MAX_FULL_SUITE_JOBS, os.cpus().length - RESERVED_CPUS));
|
||||
test('unset and empty string both take the computed default — available CPUs, capped, floor 1', () => {
|
||||
const expected = Math.max(1, Math.min(MAX_FULL_SUITE_JOBS, os.availableParallelism?.() ?? os.cpus().length));
|
||||
expect(withJobsEnv(undefined, fullSuiteJobs)).toBe(expected);
|
||||
// A stray `export GSTACK_FREE_JOBS=` must not throw.
|
||||
expect(withJobsEnv('', fullSuiteJobs)).toBe(expected);
|
||||
});
|
||||
|
||||
test.each([[0, 1], [1, 1], [2, 2], [4, 4], [8, 6]])(
|
||||
'%i available CPUs default to %i serial shard processes regardless of host CPU count',
|
||||
(availableCpus, expected) => {
|
||||
const available = spyOn(os, 'availableParallelism').mockReturnValue(availableCpus);
|
||||
const cpus = spyOn(os, 'cpus').mockReturnValue(Array<os.CpuInfo>(16));
|
||||
try {
|
||||
expect(withJobsEnv(undefined, fullSuiteJobs)).toBe(expected);
|
||||
expect(withJobsEnv('', fullSuiteJobs)).toBe(expected);
|
||||
expect(cpus).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
available.mockRestore();
|
||||
cpus.mockRestore();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
test('runtimes without availableParallelism fall back to the bounded CPU count', () => {
|
||||
const result = Bun.spawnSync([process.execPath, '-e', `
|
||||
import { mock } from 'bun:test';
|
||||
import * as os from 'os';
|
||||
const originalOs = { ...os };
|
||||
let cpuCount = 0;
|
||||
mock.module('os', () => ({
|
||||
...originalOs,
|
||||
availableParallelism: undefined,
|
||||
cpus: () => Array(cpuCount),
|
||||
}));
|
||||
const { fullSuiteJobs } = await import(${JSON.stringify(import.meta.resolve('../scripts/test-free-shards'))});
|
||||
delete process.env.GSTACK_FREE_JOBS;
|
||||
console.log(JSON.stringify([0, 1, 2, 4, 8].map(count => {
|
||||
cpuCount = count;
|
||||
return fullSuiteJobs();
|
||||
})));
|
||||
`], { timeout: 10_000 });
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(result.stderr.toString()).toBe('');
|
||||
expect(JSON.parse(result.stdout.toString())).toEqual([1, 1, 2, 4, 6]);
|
||||
});
|
||||
|
||||
test('a positive integer override is honored exactly (the sandbox recipe sets 2)', () => {
|
||||
expect(withJobsEnv('2', fullSuiteJobs)).toBe(2);
|
||||
expect(withJobsEnv('1', fullSuiteJobs)).toBe(1);
|
||||
expect(withJobsEnv(' 2 ', fullSuiteJobs)).toBe(2);
|
||||
expect(withJobsEnv('02', fullSuiteJobs)).toBe(2);
|
||||
});
|
||||
|
||||
test('Windows CI pins its two-worker budget independently of local CPU defaults', () => {
|
||||
const workflow = Bun.YAML.parse(readFileSync(new URL('../.github/workflows/windows-free-tests.yml', import.meta.url), 'utf8')) as {
|
||||
jobs: Record<string, { steps: Array<{ run?: string; env?: Record<string, string> }> }>;
|
||||
};
|
||||
const step = workflow.jobs['windows-free-tests'].steps.find(step => step.run === 'bun run test:windows');
|
||||
expect(step).toBeDefined();
|
||||
expect(step?.env?.GSTACK_FREE_JOBS).toBe('2');
|
||||
expect(withJobsEnv(step?.env?.GSTACK_FREE_JOBS, fullSuiteJobs)).toBe(2);
|
||||
});
|
||||
|
||||
test('an explicit override does not probe the available CPUs', () => {
|
||||
const available = spyOn(os, 'availableParallelism').mockImplementation(() => {
|
||||
throw new Error('CPU detection must not run for an explicit override');
|
||||
});
|
||||
const cpus = spyOn(os, 'cpus').mockImplementation(() => {
|
||||
throw new Error('CPU detection must not run for an explicit override');
|
||||
});
|
||||
try {
|
||||
expect(withJobsEnv('2', fullSuiteJobs)).toBe(2);
|
||||
expect(withJobsEnv('12', fullSuiteJobs)).toBe(12);
|
||||
expect(available).not.toHaveBeenCalled();
|
||||
expect(cpus).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
available.mockRestore();
|
||||
cpus.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
test('override is deliberately NOT clamped by MAX_FULL_SUITE_JOBS (beefy boxes may raise it)', () => {
|
||||
@@ -57,7 +126,7 @@ describe('test-free-shards: fullSuiteJobs (GSTACK_FREE_JOBS override)', () => {
|
||||
test('zero, negative, and non-numeric values throw loudly instead of silently defaulting', () => {
|
||||
// '2abc' and '3.7' pin the strict digits-only check: parseInt would
|
||||
// silently truncate them to 2 and 3, defeating the loud-failure contract.
|
||||
for (const bad of ['0', '-2', 'abc', 'NaN', '2abc', '3.7']) {
|
||||
for (const bad of ['0', '-2', 'abc', 'NaN', '2abc', '3.7', ' ', '+2', '1e2', 'Infinity']) {
|
||||
expect(() => withJobsEnv(bad, fullSuiteJobs)).toThrow(/positive integer/);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -342,6 +342,7 @@ describe('test-free-shards: shard args', () => {
|
||||
expect(args).toContain(`--timeout=${FREE_TEST_TIMEOUT_MS}`);
|
||||
expect(args).toContain('--max-concurrency=1');
|
||||
expect(args).not.toContain('--parallel');
|
||||
expect(args).not.toContain('--concurrent');
|
||||
});
|
||||
|
||||
test('parallel mode swaps serial max-concurrency for --parallel', () => {
|
||||
|
||||
@@ -20,6 +20,7 @@ const root = ${JSON.stringify(ROOT)};
|
||||
const factsPath = ${JSON.stringify(facts)};
|
||||
const { CLAUDE_FRONTIER_EVAL_MODEL } = await import(path.join(root, 'lib/eval-model.ts'));
|
||||
const readFile = fs.readFileSync.bind(fs);
|
||||
const recoveryResponses = JSON.parse(readFile(path.join(root, 'test/fixtures/third-party-actions-recovery-public.json'), 'utf8')).responses;
|
||||
const writeFile = fs.writeFileSync.bind(fs);
|
||||
const remove = fs.rmSync.bind(fs);
|
||||
const source = readFile(path.join(root, 'test/helpers/e2e-helpers.ts'), 'utf8');
|
||||
@@ -151,6 +152,18 @@ test('actual paid assertion boundaries and all failure stages retain one accurat
|
||||
expect(narration.failed).toBe(false);
|
||||
expect(narration.entry.passed).toBe(true);
|
||||
expect(narration.recordCalls).toBe(1);
|
||||
for (const response of recoveryResponses) {
|
||||
for (const [output, passed] of [
|
||||
[response.text, true],
|
||||
[response.text.replace('option included.', 'option included; then I drive in your Aside browser.'), false],
|
||||
]) {
|
||||
const observed = await invoke('tpa-broken', 'pass', output);
|
||||
expect(observed.failed).toBe(!passed);
|
||||
expect(observed.entry.passed).toBe(passed);
|
||||
expect(observed.recordCalls).toBe(1);
|
||||
expect(observed.entry.transcript).toEqual(observed.result.transcript);
|
||||
}
|
||||
}
|
||||
// A runner/setup exception has no returned model. Mirror the same capture
|
||||
// resolution that the real session runner would have used, including overrides.
|
||||
for (const [overrides, expected] of [
|
||||
|
||||
@@ -27,6 +27,7 @@ import { generateAsideSetup } from "../scripts/resolvers/aside";
|
||||
import { HOST_PATHS } from "../scripts/resolvers/types";
|
||||
import { asideDriveOptions } from './helpers/third-party-actions';
|
||||
import { E2E_TOUCHFILES, selectTests } from './helpers/touchfiles';
|
||||
import recoveryFixture from './fixtures/third-party-actions-recovery-public.json';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, "..");
|
||||
|
||||
@@ -70,6 +71,27 @@ D) Defer
|
||||
expect(asideDriveOptions('A) Open the Aside app so I can re-run the probe.')).toEqual([]);
|
||||
expect(asideDriveOptions('A) Open the Aside app; if READY, I drive the dashboard.')).toHaveLength(1);
|
||||
});
|
||||
|
||||
test.each(recoveryFixture.responses)('native recovery response $attempt defers drive consent to a new question', ({ text }) => {
|
||||
expect(asideDriveOptions(text)).toEqual([]);
|
||||
expect(asideDriveOptions(text.replace('option included.', 'option included; then I drive in your Aside browser.'))).toHaveLength(1);
|
||||
});
|
||||
|
||||
test('a future question can name its option without offering the drive now', () => {
|
||||
for (const subject of ["I'll", 'I will', 'We’ll', 'we will']) {
|
||||
for (const question of ['re-ask', 'ask again', 're-ask this question']) {
|
||||
for (const label of ['the Aside drive', 'the "drive it in your Aside browser"', 'the “drive it in your Aside browser”']) {
|
||||
const reference = `${subject} ${question} with ${label} option included`;
|
||||
expect(asideDriveOptions(`A) Open Aside and re-probe; if READY, ${reference}.`)).toEqual([]);
|
||||
expect(asideDriveOptions(`A) I drive in Aside first; ${reference}.`)).toHaveLength(1);
|
||||
expect(asideDriveOptions(`A) Open Aside; ${reference}, then I drive the dashboard.`)).toHaveLength(1);
|
||||
expect(asideDriveOptions(`A) Open Aside; ${reference} and navigate the dashboard before that question.`)).toHaveLength(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(asideDriveOptions('A) I will re-ask after I drive in Aside with the drive option included.')).toHaveLength(1);
|
||||
expect(asideDriveOptions('A) Open Aside; I will re-ask with the Aside drive option, then I browse using that option.')).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
/** Generated skill markdown: every SKILL.md + carved sections at repo root. */
|
||||
|
||||
Reference in New Issue
Block a user