mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-28 07:32:14 +02:00
v1.90.2.0 perf: halve local free-suite time and preserve coverage (#2972)
* perf: remove repeated test work and preserve AUQ execution budgets * fix: validate native evaluation fixture evidence at its actual boundaries * fix: clarify deployment approval and recovery state transitions * chore: document coverage and release v1.90.2.0 * test: preserve Windows scheduling and native no-change consent * test: recognize verified reads through fixture symlinks * test: isolate alias-name installation from runtime assets
This commit is contained in:
Garry Tan
@@ -108,6 +108,7 @@ export interface RunAgentSdkOptions {
|
||||
queryProvider?: QueryProvider;
|
||||
/** Cancel queueing, SDK work and retries under the caller's case deadline. */
|
||||
signal?: AbortSignal;
|
||||
onAdmission?: () => void;
|
||||
/** Max 429 retries per call. Default 3. */
|
||||
maxRetries?: number;
|
||||
/**
|
||||
@@ -330,6 +331,7 @@ export async function runAgentSdkTest(
|
||||
|
||||
let attempt = 0;
|
||||
let lastErr: unknown = null;
|
||||
let admitted = false;
|
||||
|
||||
while (attempt <= maxRetries) {
|
||||
await sem.acquire(opts.signal);
|
||||
@@ -376,6 +378,10 @@ export async function runAgentSdkTest(
|
||||
|
||||
try {
|
||||
opts.signal?.throwIfAborted();
|
||||
if (!admitted) {
|
||||
admitted = true;
|
||||
opts.onAdmission?.();
|
||||
}
|
||||
// When canUseTool is supplied, the SDK must route tool-use approval
|
||||
// decisions through the callback. bypassPermissions short-circuits
|
||||
// that. Flip to 'default' mode so canUseTool actually fires. Tests
|
||||
@@ -415,6 +421,7 @@ export async function runAgentSdkTest(
|
||||
sdkOpts.systemPrompt = opts.systemPrompt;
|
||||
}
|
||||
|
||||
opts.signal?.throwIfAborted();
|
||||
const q = queryImpl({
|
||||
prompt: opts.userPrompt,
|
||||
options: sdkOpts,
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
import { afterAll, describe, mock, spyOn, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { serializeNativeAuq } from './auq-native-capture';
|
||||
|
||||
const spec = JSON.parse(process.env.AUQ_PARALLEL_SPEC!);
|
||||
const root = process.env.AUQ_PARALLEL_ROOT!;
|
||||
const realSetTimeout = globalThis.setTimeout, realClearTimeout = globalThis.clearTimeout, realNow = Date.now;
|
||||
const timers = new Map<number, { at: number; callback: () => void }>();
|
||||
let now = 0, timerId = 0, pumping = false;
|
||||
if (spec.queryMs) {
|
||||
Date.now = () => now;
|
||||
globalThis.setTimeout = ((callback: (...args: any[]) => void, delay = 0, ...args: any[]) => {
|
||||
const id = ++timerId;
|
||||
timers.set(id, { at: now + Math.max(0, delay), callback: () => callback(...args) });
|
||||
if (!pumping) {
|
||||
pumping = true;
|
||||
realSetTimeout(function pump() {
|
||||
const next = [...timers].sort((a, b) => a[1].at - b[1].at)[0];
|
||||
if (!next) { pumping = false; return; }
|
||||
timers.delete(next[0]); now = next[1].at; next[1].callback();
|
||||
realSetTimeout(pump, 0);
|
||||
}, 0);
|
||||
}
|
||||
return id;
|
||||
}) as typeof setTimeout;
|
||||
globalThis.clearTimeout = ((id: number) => { timers.delete(id); }) as typeof clearTimeout;
|
||||
}
|
||||
const events: Array<{ kind: string; id: string; active?: number; at?: number; caseDeadline?: number }> = [];
|
||||
const inputs: any[] = [], judges: any[] = [], judgeRequests: any[] = [], fixtures: string[] = [];
|
||||
const ownedStates: string[] = [];
|
||||
let settlements: any[] = [];
|
||||
let active = 0, peak = 0, judging = 0, judgePeak = 0, answers = 0;
|
||||
const idFrom = (cwd: string) => path.basename(cwd).replace(/-owned$/, '').replace(/^auq-(?:consistency-|ab-)/, '');
|
||||
const indexFrom = (id: string) => id === 'carved' ? 0 : id === 'verbose' ? 1 : Number(id);
|
||||
const questionFor = (id: string) => {
|
||||
const question = {
|
||||
header: 'Mode',
|
||||
question: `Run ${id}\nELI10: Review this pricing plan.\nRecommendation: SELECTIVE EXPANSION because the untested premise needs a comparison.\nPros / cons:\nNet: Choose the review scope.`,
|
||||
options: ['SCOPE EXPANSION', 'SELECTIVE EXPANSION', 'HOLD SCOPE', 'SCOPE REDUCTION'].map(label => ({
|
||||
label: label + (label === 'SELECTIVE EXPANSION' ? ' (recommended)' : ''),
|
||||
description: '✅ Concrete benefit.\n❌ Honest tradeoff.',
|
||||
})),
|
||||
};
|
||||
if (spec.omit && indexFrom(id) === (spec.omitIndex ?? 2)) {
|
||||
question.question = question.question.replaceAll(spec.omit, '');
|
||||
for (const option of question.options) {
|
||||
option.label = option.label.replaceAll(spec.omit, '');
|
||||
option.description = option.description.replaceAll(spec.omit, '');
|
||||
}
|
||||
}
|
||||
return question;
|
||||
};
|
||||
|
||||
mock.module('@anthropic-ai/claude-agent-sdk', () => ({ query: ({ prompt, options }: any) => {
|
||||
const id = idFrom(options.cwd), index = indexFrom(id);
|
||||
active++; peak = Math.max(peak, active);
|
||||
events.push({ kind: 'query-start', id, active, at: Date.now() });
|
||||
inputs.push({ id, prompt, cwd: options.cwd, model: options.model, maxTurns: options.maxTurns,
|
||||
systemPrompt: options.systemPrompt, tools: options.tools, allowedTools: options.allowedTools,
|
||||
permissionMode: options.permissionMode, settingSources: options.settingSources,
|
||||
config: options.env.CLAUDE_CONFIG_DIR, state: options.env.GSTACK_HOME,
|
||||
skill: fs.readFileSync(path.join(options.cwd, 'plan-ceo-review/SKILL.md'), 'utf8'),
|
||||
plan: fs.readFileSync(path.join(options.cwd, 'plan.md'), 'utf8'),
|
||||
sections: fs.existsSync(path.join(options.cwd, 'plan-ceo-review/sections'))
|
||||
? fs.readdirSync(path.join(options.cwd, 'plan-ceo-review/sections')).sort().map(name => [name,
|
||||
fs.readFileSync(path.join(options.cwd, 'plan-ceo-review/sections', name), 'utf8')]) : [],
|
||||
});
|
||||
let closed = false;
|
||||
const close = () => {
|
||||
if (closed) return;
|
||||
closed = true; active--;
|
||||
events.push({ kind: 'query-close', id, active });
|
||||
};
|
||||
return {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
try {
|
||||
yield { type: 'system', subtype: 'init', claude_code_version: 'fixture' };
|
||||
await new Promise<void>(resolve => {
|
||||
const signal = options.abortController.signal;
|
||||
const abort = () => { clearTimeout(timer); resolve(); };
|
||||
const timer = setTimeout(() => { signal.removeEventListener('abort', abort); resolve(); },
|
||||
spec.queryMs ?? (index === 0 ? 10 : 60));
|
||||
signal.addEventListener('abort', abort, { once: true });
|
||||
});
|
||||
if (options.abortController.signal.aborted) return;
|
||||
if (spec.reject === index || spec.rejectAll) throw Error(`capture rejection ${id}`);
|
||||
void options.canUseTool('AskUserQuestion', { questions: [questionFor(id)] }, {
|
||||
toolUseID: `native-${id}`, signal: options.abortController.signal,
|
||||
}).then(() => answers++);
|
||||
yield { type: 'assistant', message: { content: [] } };
|
||||
} finally { close(); }
|
||||
},
|
||||
close,
|
||||
};
|
||||
} }));
|
||||
|
||||
mock.module('./e2e-gate', () => ({ describeE2ETier: (tier: string) => {
|
||||
if (tier !== 'periodic') throw Error(`unexpected tier ${tier}`);
|
||||
return describe;
|
||||
} }));
|
||||
mock.module('@anthropic-ai/sdk', () => ({ default: class {
|
||||
messages = { create: async (request: any) => {
|
||||
const id = request.messages[0].content.match(/Run (\w+)/)![1], index = indexFrom(id);
|
||||
judgeRequests.push({ id, request });
|
||||
await new Promise(resolve => setTimeout(resolve, 90));
|
||||
if (spec.judgeReject === index) throw Error(`judge rejection ${id}`);
|
||||
return { stop_reason: 'end_turn', content: [{ type: 'text', text: JSON.stringify({
|
||||
reason_substance: spec.scores?.[index] ?? 4, reasoning: 'controlled free verdict',
|
||||
}) }] };
|
||||
} };
|
||||
} }));
|
||||
const judgeHelper = await import('./llm-judge');
|
||||
const judgeRecommendation = judgeHelper.judgeRecommendation;
|
||||
mock.module('./llm-judge', () => ({ ...judgeHelper, judgeRecommendation: async (text: string) => {
|
||||
const id = text.match(/Run (\w+)/)![1];
|
||||
judges.push({ id, text });
|
||||
judging++; judgePeak = Math.max(judgePeak, judging);
|
||||
events.push({ kind: 'judge-start', id });
|
||||
try {
|
||||
return await judgeRecommendation(text);
|
||||
} finally {
|
||||
judging--;
|
||||
events.push({ kind: 'judge-settle', id });
|
||||
}
|
||||
} }));
|
||||
|
||||
const helper = await import('./auq-sdk-capture');
|
||||
const captureModeSelectionAuq = helper.captureModeSelectionAuq;
|
||||
mock.module('./auq-sdk-capture', () => ({ ...helper,
|
||||
captureModeSelectionAuq: async (opts: Parameters<typeof helper.captureModeSelectionAuq>[0]) => {
|
||||
const id = idFrom(opts.planDir);
|
||||
events.push({ kind: 'capture-start', id, at: Date.now(), caseDeadline: opts.caseDeadline });
|
||||
try {
|
||||
const text = await captureModeSelectionAuq(opts);
|
||||
if (text !== serializeNativeAuq(questionFor(id))) throw Error(`native text changed for ${id}`);
|
||||
return spec.empty === indexFrom(id) ? '' : text;
|
||||
} finally { events.push({ kind: 'capture-settle', id }); }
|
||||
},
|
||||
}));
|
||||
|
||||
const make = fs.mkdtempSync, remove = fs.rmSync;
|
||||
spyOn(fs, 'mkdtempSync').mockImplementation(((prefix: string, ...args: any[]) => {
|
||||
const name = path.basename(String(prefix));
|
||||
if (name === 'gstack-mode-auq-') {
|
||||
const dir = make(prefix, ...args);
|
||||
ownedStates.push(dir);
|
||||
return dir;
|
||||
}
|
||||
if (path.dirname(String(prefix)) !== root || !/^auq-(?:consistency-|ab-)/.test(name)) return make(prefix, ...args);
|
||||
const id = idFrom(name + 'owned');
|
||||
if (spec.setupReject === indexFrom(id)) throw Error(`setup rejection ${id}`);
|
||||
const dir = path.join(root, name + 'owned');
|
||||
fs.mkdirSync(dir);
|
||||
if (!fs.realpathSync(dir).startsWith(fs.realpathSync(root) + path.sep)) throw Error('fixture escaped owned root');
|
||||
fixtures.push(dir);
|
||||
return dir;
|
||||
}) as typeof fs.mkdtempSync);
|
||||
spyOn(fs, 'rmSync').mockImplementation((file, options) => {
|
||||
if (fixtures.includes(String(file))) {
|
||||
const id = idFrom(String(file));
|
||||
events.push({ kind: 'cleanup', id, active });
|
||||
if (spec.cleanupReject === indexFrom(id)) throw Error(`cleanup rejection ${id}`);
|
||||
}
|
||||
return remove(file, options);
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
const receipts = fs.readdirSync(path.join(root, 'artifacts/native-auq'), { recursive: true })
|
||||
.filter(file => String(file).endsWith('capture.json'))
|
||||
.map(file => JSON.parse(fs.readFileSync(path.join(root, 'artifacts/native-auq', String(file)), 'utf8')));
|
||||
fs.writeFileSync(path.join(root, 'facts.json'), JSON.stringify({ events, inputs, judges, judgeRequests, receipts,
|
||||
fixtures, peak, judgePeak, active, judging, answers, settlements, elapsed: now, pendingTimers: timers.size,
|
||||
leftovers: fixtures.filter(dir => fs.existsSync(dir)),
|
||||
ownedStateLeftovers: [...ownedStates, ...inputs.flatMap(input => [input.config, input.state])].filter(dir => fs.existsSync(dir)),
|
||||
}));
|
||||
Date.now = realNow; globalThis.setTimeout = realSetTimeout; globalThis.clearTimeout = realClearTimeout;
|
||||
});
|
||||
|
||||
if (spec.suite === 'direct') {
|
||||
test('actual native capture admission lifecycle', async () => {
|
||||
const dirs: string[] = [];
|
||||
try {
|
||||
settlements = (await Promise.allSettled(Array.from({ length: spec.runs ?? 3 }, (_, i) => {
|
||||
const dir = helper.setupPlanCeoDir({ ...helper.carvedSkill(), tmpPrefix: `auq-consistency-${i}-` });
|
||||
dirs.push(dir);
|
||||
return captureModeSelectionAuq({ planDir: dir, testName: `direct-${i}`, runId: 'free',
|
||||
...(spec.outerMs === undefined ? {} : { caseDeadline: now + spec.outerMs }) });
|
||||
}))).map(result => result.status === 'fulfilled' ? result : { status: result.status, reason: String(result.reason) });
|
||||
} finally {
|
||||
for (const dir of dirs) fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
} else {
|
||||
await import(process.env.AUQ_PARALLEL_SOURCE ?? path.join(import.meta.dir, '..',
|
||||
spec.suite === 'consistency' ? 'skill-e2e-auq-consistency.test.ts' : 'skill-e2e-auq-verbose-vs-carved-ab.test.ts'));
|
||||
}
|
||||
@@ -418,8 +418,10 @@ export async function captureModeSelectionAuq(opts: {
|
||||
testName: string;
|
||||
runId?: string;
|
||||
model?: string;
|
||||
caseDeadline?: number;
|
||||
}): Promise<string> {
|
||||
const startedAt = Date.now(), deadline = startedAt + 240_000;
|
||||
const startedAt = Date.now();
|
||||
let deadline = opts.caseDeadline ?? startedAt + 240_000;
|
||||
const cwd = path.resolve(opts.planDir);
|
||||
const skillPath = path.join(cwd, 'plan-ceo-review', 'SKILL.md');
|
||||
const planPath = path.join(cwd, 'plan.md');
|
||||
@@ -448,7 +450,14 @@ Ask the user through the AskUserQuestion tool and wait for their answer.`;
|
||||
let outcome = 'error', diagnostic: string | undefined, actorFailure: Error | undefined;
|
||||
let captured: { toolUseId: string; input: Record<string, unknown>; question: NativePlanQuestion; text: string } | undefined;
|
||||
let terminal: { exitReason: string; turnsUsed: number; costUsd: number; sdkClaudeCodeVersion: string; errors?: string[] } | undefined;
|
||||
const timer = setTimeout(() => controller.abort(timeout), Math.max(0, deadline - Date.now()));
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
const armDeadline = () => {
|
||||
clearTimeout(timer);
|
||||
const remaining = deadline - Date.now();
|
||||
if (remaining <= 0) controller.abort(timeout);
|
||||
else timer = setTimeout(() => controller.abort(timeout), remaining);
|
||||
};
|
||||
armDeadline();
|
||||
const fail = (reason: string, detail?: string): never => {
|
||||
outcome = reason;
|
||||
throw new Error(`${opts.testName}: AUQ capture failed (${reason})${detail ? `: ${detail}` : ''}`);
|
||||
@@ -470,6 +479,10 @@ Ask the user through the AskUserQuestion tool and wait for their answer.`;
|
||||
workingDirectory: cwd, model, maxTurns: 12, maxRetries: 0,
|
||||
allowedTools: ['Read', 'Write', 'AskUserQuestion'], permissionMode: 'default', settingSources: [],
|
||||
pathToClaudeCodeExecutable: binary, signal: controller.signal,
|
||||
onAdmission: opts.caseDeadline === undefined ? undefined : () => {
|
||||
deadline = Math.min(opts.caseDeadline!, Date.now() + 240_000);
|
||||
armDeadline();
|
||||
},
|
||||
env: { CLAUDE_CONFIG_DIR: configDir, GSTACK_HOME: stateDir, GSTACK_HEADLESS: '' },
|
||||
testName: opts.testName, runId: opts.runId,
|
||||
canUseTool: async (name, input, options) => {
|
||||
|
||||
@@ -5480,7 +5480,8 @@ export interface PlanSkillFloorObservation {
|
||||
* matcher still authenticates the pending question and native menu. */
|
||||
export function planFloorDXPane(visible: string, call: NativePlanQuestionCall): string | null {
|
||||
if (call.answered || call.failed || call.questions.length !== 1) return null;
|
||||
const text = stripPtyResidue(visible).replace(/\r+\n?/g, '\n');
|
||||
const text = stripPtyResidue(visible).replace(/\r+\n?/g, '\n')
|
||||
.replace(/((?:^|\n)Enter\s+to\s+select\s*·\s*↑\/↓\s+to\s+navigate\s*·\s*)ctrl\+g\s+to\s+edit\s+in[ \t]+[^\s·\x00-\x1f\x7f][^·\x00-\x1f\x7f]*?\s*·\s*(Esc\s+to\s+cancel\s*)$/, '$1$2');
|
||||
const headers = [...text.matchAll(/(?:^|\n)[\t ]*[☐□][^\n]*\n/g)];
|
||||
const header = headers.at(-1);
|
||||
if (!header) return null;
|
||||
|
||||
@@ -445,7 +445,7 @@ export function isInternalClaudeGitRequest(request: SourceRequest, commands: str
|
||||
// Require direct process ancestry AND the exact observed host prefix AND no
|
||||
// matching model request. A shell/model-issued unguarded Git call still fails.
|
||||
return request.tool === 'git' && !!request.ppid &&
|
||||
/(?:^|[/\\])claude(?:$|[/\\])/.test(request.parentExecutable || '') &&
|
||||
/(?:^|[/\\])claude(?:\.exe)?$/.test(request.parentExecutable || '') &&
|
||||
JSON.stringify(request.args.slice(0, hostPrefix.length)) === JSON.stringify(hostPrefix) &&
|
||||
!commands.some(command => command.includes('core.safecrlf=false') || command.includes('protocol.ext.allow=never'));
|
||||
}
|
||||
@@ -836,7 +836,7 @@ function skippedReviewOption(question: any): any {
|
||||
option[field] !== undefined && typeof option[field] !== 'string')) return [];
|
||||
const label = option.label.replace(/[‘’]/g, "'").replace(/`/g, '').replace(/^\s*(?:[A-Z]|\d+)[.)]\s*/i, '')
|
||||
.replace(/\s*\(recommended\)\s*$/i, '').trim()
|
||||
.replace(/^no\s*[,.:!?]\s*(?=(?:skip|decline|keep|leave|do not|don't)\b)/i, '');
|
||||
.replace(/^no\b[\s,:;.!?-]*(?=(?:skip|decline|keep|leave|do not|don't)\b)/i, '');
|
||||
const referentialRetention = /^(?:keep|leave)\s+(?:it|this|that|them|these)$/i.test(label);
|
||||
const preservation = option.description?.trim().replace(/`/g, '').match(/^(?:keep|leave|retain|preserve)\s+([^,;.!?]+)/i);
|
||||
const preservedObject = preservation?.[1].split(/\b(?:and|but|while)\b/i)[0]
|
||||
@@ -847,13 +847,15 @@ function skippedReviewOption(question: any): any {
|
||||
|| qualifiedIndexState.test(preservedObject));
|
||||
const description = (option.description ?? '').replace(/[‘’]/g, "'").trim();
|
||||
const declinesChange = /^(?:do not|don't)\s+(?:apply|change|edit|fix|refactor|extract|modify|touch|clear|remove|update|replace|add|migrate|implement|reuse|import)\b/i;
|
||||
const inapplicable = /^not applicable$/i.test(label)
|
||||
&& /^(?:choose this(?: option)?\s+)?(?:if|when) you are not (?:editing|changing|modifying)\b/i.test(description);
|
||||
const labelObject = label.match(/^(?:keep|leave)\s+(?:the\s+)?(.+)$/i)?.[1]
|
||||
.replace(/\s+(?:as[- ]is|unchanged|untouched|set)$/i, '');
|
||||
const preservationRank = referentialRetention ? describedRetention || declinesChange.test(description)
|
||||
: /^(?:keep|leave)\b.*\b(?:current|existing|unchanged|untouched|as[- ]is|alone|set|copies|copy|implementation|code|source)\b/i.test(label)
|
||||
|| !!labelObject && qualifiedIndexState.test(labelObject);
|
||||
const rank = /^(?:skip|decline)(?=$|\s|[,.!])/i.test(label) ? 3
|
||||
: declinesChange.test(label) ? 2
|
||||
: inapplicable || declinesChange.test(label) ? 2
|
||||
: preservationRank ? 1 : 0;
|
||||
if (!rank) return [];
|
||||
// A leading decline names rejected work. Classify later commitments rather
|
||||
@@ -868,6 +870,7 @@ function skippedReviewOption(question: any): any {
|
||||
word.replace(/([a-z])\1(?:ed|ing)$/, '$1')].some(form => actions.has(form));
|
||||
const changes = commitment.toLowerCase().split(/[,;\n]|[.!?](?:\s|$)|\b(?:and|but|then|while)\b/).some(part => {
|
||||
const clause = part.replace(/^[^a-z]+/, '')
|
||||
.replace(/^(?:the\s+)?(?:review|reuse|snapshot)\s+coverage\s+(?=(?:will|would|should|must|can|may|does|do)\b)/, '')
|
||||
.replace(/^(?:(?:this|that|the|selected|chosen)\s+(?:option|choice|selection)|i|we|you|it|(?:the\s+)?(?:source|code|route|worker|helper|parser|index(?:\s+flag)?))\s+/, '')
|
||||
.replace(/^(?:will|would|should|must|can|may|does|do)\s+/, '')
|
||||
.replace(/^(?:(?:please|also|still|just|now|be)\s+)+/, '');
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import * as path from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
interface StartContext {
|
||||
repo: string;
|
||||
@@ -71,24 +72,29 @@ function substitutionEnd(source: string, start: number): number {
|
||||
|
||||
/** Bounded inspection syntax, not a shell executor: quoted arguments and comments
|
||||
* cannot introduce commands. Unknown inspection forms fail closed. */
|
||||
function commands(source: string): { words: string[]; before: string; after: string; substitutions: number[] }[] {
|
||||
function commands(source: string): { words: string[]; before: string; after: string; substitutions: number[]; quoted: number[] }[] {
|
||||
const executable = withoutHereDocBodies(source);
|
||||
if (executable === undefined) return [];
|
||||
source = executable;
|
||||
const result: { words: string[]; before: string; after: string; substitutions: number[] }[] = [];
|
||||
let words: string[] = [], substitutions: number[] = [];
|
||||
let word = '', quote = '', before = '', expanded = false;
|
||||
const result: ReturnType<typeof commands> = [];
|
||||
let words: string[] = [], substitutions: number[] = [], quoted: number[] = [];
|
||||
let word = '', quote = '', before = '', expanded = false, wasQuoted = false;
|
||||
const flush = () => {
|
||||
if (word) { if (expanded) substitutions.push(words.length); words.push(word); }
|
||||
word = ''; expanded = false;
|
||||
if (word) {
|
||||
if (expanded) substitutions.push(words.length);
|
||||
if (wasQuoted) quoted.push(words.length);
|
||||
words.push(word);
|
||||
}
|
||||
word = ''; expanded = false; wasQuoted = false;
|
||||
};
|
||||
const end = (separator: string) => {
|
||||
flush(); if (words.length) result.push({ words, before, after: separator, substitutions });
|
||||
words = []; substitutions = []; before = separator;
|
||||
flush(); if (words.length) result.push({ words, before, after: separator, substitutions, quoted });
|
||||
words = []; substitutions = []; quoted = []; before = separator;
|
||||
};
|
||||
for (let i = 0; i < source.length; i++) {
|
||||
const char = source[i];
|
||||
if (char === '\\' && quote !== "'") {
|
||||
wasQuoted = true;
|
||||
if (quote === '"' && !/[$`"\\\n]/.test(source[i + 1] ?? '')) word += char;
|
||||
else { const escaped = source[++i] ?? ''; word += escaped === '$' ? '\0$' : escaped; }
|
||||
continue;
|
||||
@@ -99,7 +105,7 @@ function commands(source: string): { words: string[]; before: string; after: str
|
||||
expanded = true; word += source.slice(i, end + 1); i = end; continue;
|
||||
}
|
||||
if (quote) { if (char === quote) quote = ''; else word += quote === "'" && char === '$' ? '\0$' : char; continue; }
|
||||
if (char === '"' || char === "'") { quote = char; continue; }
|
||||
if (char === '"' || char === "'") { quote = char; wasQuoted = true; continue; }
|
||||
if (char === '#' && !word) { while (i < source.length && source[i] !== '\n') i++; end(';'); }
|
||||
else if (';|&()\n'.includes(char)) {
|
||||
const separator = (char === '&' || char === '|') && source[i + 1] === char ? char + source[++i] : char;
|
||||
@@ -113,18 +119,122 @@ function commands(source: string): { words: string[]; before: string; after: str
|
||||
return result;
|
||||
}
|
||||
|
||||
function executedCommands(source: string): ReturnType<typeof commands> {
|
||||
const calls = commands(source);
|
||||
return calls.flatMap(call => [call, ...call.substitutions.flatMap(index => {
|
||||
const word = call.words[index], begin = word.indexOf('$('), end = substitutionEnd(word, begin);
|
||||
return end < 0 ? [] : executedCommands(word.slice(begin + 2, end));
|
||||
})]);
|
||||
}
|
||||
|
||||
const basename = (word = '') => word.split(/[\\/]/).at(-1);
|
||||
const sourcePaths = (file: string) => file.includes('\\') || /^[A-Za-z]:\//.test(file) ? path.win32 : path.posix;
|
||||
type SourcePaths = ReturnType<typeof sourcePaths>;
|
||||
|
||||
function topLevelCommands(calls: ReturnType<typeof commands>): ReturnType<typeof commands> {
|
||||
const scopes: string[] = [], result: ReturnType<typeof commands> = [];
|
||||
for (const call of calls) {
|
||||
if (call.before === '(') scopes.push(')');
|
||||
if (!scopes.length && ['', ';'].includes(call.before)) {
|
||||
if (['exit', 'return', 'exec'].includes(call.words[0])) break;
|
||||
result.push(call);
|
||||
}
|
||||
let index = 0;
|
||||
while (!call.quoted.includes(index) && ['then', 'else', 'do'].includes(call.words[index])) index++;
|
||||
const head = call.quoted.includes(index) ? '' : call.words[index];
|
||||
const close = ({ if: 'fi', for: 'done', while: 'done', until: 'done', case: 'esac', '{': '}' } as Record<string, string>)[head];
|
||||
if (close) scopes.push(close);
|
||||
else if (['fi', 'done', 'esac', '}'].includes(head) && scopes.pop() !== head) return [];
|
||||
if (call.after === ')' && scopes.pop() !== ')') return [];
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function reviewStart(words: string[]): boolean {
|
||||
return words.length === 3 && basename(words[0]) === 'gstack-review-log'
|
||||
&& words[1] === '--start' && words[2] === 'review';
|
||||
}
|
||||
|
||||
function startTokenOutput(source: string, variables: Map<string, string | undefined>, token: string): string | undefined {
|
||||
const calls = commands(source);
|
||||
if (calls.length === 1 && reviewStart(calls[0].words) && !calls[0].before && !calls[0].after) return token;
|
||||
const first = calls[0];
|
||||
if (!first || first.before || first.words[0] !== 'echo' || first.words.length !== 2
|
||||
|| expandVariables(first.words[1], variables) !== token) return undefined;
|
||||
for (let i = 1; i < calls.length; i++) {
|
||||
if (calls[i - 1].after !== '|') return undefined;
|
||||
const words = calls[i].words;
|
||||
if (['head', 'tail'].includes(words[0]) && (words.length === 2 && words[1] === '-1'
|
||||
|| words.length === 3 && words[1] === '-n' && words[2] === '1')) continue;
|
||||
if (words.length !== 3 || words[0] !== 'grep' || !['-oE', '-Eo'].includes(words[1])
|
||||
|| !/^\[[A-Za-z0-9_.-]+\]\+$/.test(words[2])) return undefined;
|
||||
try { if (new RegExp(words[2]).exec(token)?.[0] !== token) return undefined; } catch { return undefined; }
|
||||
}
|
||||
return calls.at(-1)?.after ? undefined : token;
|
||||
}
|
||||
|
||||
function sameCallStartRead(source: string, file: string, expected: StartContext, token: string): boolean {
|
||||
const paths = sourcePaths(file), variables = new Map<string, string | undefined>([
|
||||
['GSTACK_HOME', expected.state], ['SLUG', expected.slug],
|
||||
]);
|
||||
if (paths.normalize(expected.directory) !== paths.join(expected.state, 'projects', expected.slug, '.review-starts')) return false;
|
||||
let cwd: string | undefined = expected.repo, started = false;
|
||||
for (const call of commands(source)) {
|
||||
if (!['', ';'].includes(call.before) || !['', ';'].includes(call.after)
|
||||
|| /^(?:if|then|else|elif|fi|for|while|until|do|done|case|esac|function|exit|return|exec|eval|source|\.|[{}!])$/.test(call.words[0])) return false;
|
||||
if (started && reader(call.words, operand => {
|
||||
const value = expandVariables(operand, variables);
|
||||
return value !== undefined && literalPath(value, cwd, paths) === file;
|
||||
})) return true;
|
||||
if (reviewStart(call.words)) {
|
||||
if (cwd !== expected.repo || variables.get('GSTACK_HOME') !== expected.state) return false;
|
||||
started = true;
|
||||
} else if (call.words.every(word => /^[A-Za-z_]\w*=/.test(word))) {
|
||||
for (const word of call.words) {
|
||||
const equal = word.indexOf('='), expression = word.slice(equal + 1);
|
||||
const substitution = /^\$\(([\s\S]*)\)$/.exec(expression);
|
||||
const value = substitution ? startTokenOutput(substitution[1], variables, token) : expandVariables(expression, variables);
|
||||
if (substitution && value === token && commands(substitution[1]).some(child => reviewStart(child.words))) {
|
||||
if (cwd !== expected.repo || variables.get('GSTACK_HOME') !== expected.state) return false;
|
||||
started = true;
|
||||
}
|
||||
variables.set(word.slice(0, equal), value);
|
||||
}
|
||||
} else if (call.words[0] === 'cd') {
|
||||
const args = call.words.slice(call.words[1] === '--' ? 2 : 1);
|
||||
const value = args.length === 1 ? expandVariables(args[0], variables) : undefined;
|
||||
cwd = value ? literalPath(value, cwd, paths) : undefined;
|
||||
} else if (['export', 'local', 'declare', 'readonly', 'unset', 'read', 'pushd', 'popd'].includes(call.words[0])) return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function successfulFinish(source: string, text: string, token: string, expected: StartContext): boolean {
|
||||
const calls = commands(source), topLevel = topLevelCommands(calls);
|
||||
const located = withDirectories(calls, expected.repo, sourcePaths(expected.repo), new Map([
|
||||
['GSTACK_HOME', expected.state], ['SLUG', expected.slug],
|
||||
]));
|
||||
const finishes = (words: string[]) => basename(words[0]) === 'gstack-review-log'
|
||||
&& words.some((word, index) => word === '--finish' && (words[index + 1] === token
|
||||
|| /^\$(?:[A-Za-z_]\w*|\{[A-Za-z_]\w*\})$/.test(words[index + 1] ?? '')));
|
||||
for (const call of topLevel) {
|
||||
if (finishes(call.words)) return true;
|
||||
if (call.quoted.includes(0) || call.words[0] !== 'if' || call.after !== ';' || !finishes(call.words.slice(1))) continue;
|
||||
const index = calls.indexOf(call), success = calls[index + 1], failure = calls[index + 2], exit = calls[index + 3], end = calls[index + 4];
|
||||
const context = located[index], argument = call.words[call.words.indexOf('--finish') + 1];
|
||||
if (context.cwd !== expected.repo || context.variables.get('GSTACK_HOME') !== expected.state
|
||||
|| expandVariables(argument, context.variables) !== token) continue;
|
||||
if (success?.words[0] !== 'then' || success.quoted.includes(0) || success.words[1] !== 'echo' || success.words.length !== 3
|
||||
|| /[$`\0]/.test(success.words[2]) || !text.split('\n').includes(success.words[2])
|
||||
|| failure?.words[0] !== 'else' || failure.quoted.includes(0) || failure.words[1] !== 'echo'
|
||||
|| exit?.words[0] !== 'exit' || !/^[1-9]\d*$/.test(exit.words[1] ?? '') || exit.words.length !== 2
|
||||
|| end?.words[0] !== 'fi' || end.quoted.includes(0) || end.words.length !== 1
|
||||
|| ![success, failure, exit, end].every(part => part.before === ';' && part.after === ';')) continue;
|
||||
if (!topLevel.some(read => calls.indexOf(read) > index + 4 && basename(read.words[0]) === 'gstack-review-read')) continue;
|
||||
for (const line of text.split('\n')) {
|
||||
let row: any;
|
||||
try { row = JSON.parse(line); } catch { continue; }
|
||||
const binding = row?.review_binding;
|
||||
if (row?.skill === 'review' && row.completed === true && row.wtree === expected.wtree
|
||||
&& binding?.state === 'verified' && binding.start_wtree === expected.wtree && binding.end_wtree === expected.wtree
|
||||
&& binding.started_at === expected.startedAt && binding.branch_id === createHash('sha256').update(expected.branch).digest('hex')) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Resolve source-spelled paths, independent of the machine replaying the trace.
|
||||
* Shell expansion is handled only by the discovery forms below, never guessed. */
|
||||
function literalPath(value: string, cwd: string | undefined, paths: SourcePaths): string | undefined {
|
||||
@@ -335,8 +445,13 @@ export function hasTrustedReviewStartRead(events: unknown[], expected: StartCont
|
||||
|
||||
for (const start of pairs) {
|
||||
if (start.tool !== 'Bash' || typeof start.input?.command !== 'string'
|
||||
|| !executedCommands(start.input.command).some(call => basename(call.words[0]) === 'gstack-review-log'
|
||||
&& call.words[1] === '--start' && call.words[2] === 'review')) continue;
|
||||
|| !topLevelCommands(commands(start.input.command)).some(call => {
|
||||
if (reviewStart(call.words)) return true;
|
||||
if (call.words.length !== 1 || !call.substitutions.includes(0)) return false;
|
||||
const assignment = /^[A-Za-z_]\w*=\$\(([\s\S]*)\)$/.exec(call.words[0]);
|
||||
const children = assignment ? commands(assignment[1]) : [];
|
||||
return children.length === 1 && !children[0].before && !children[0].after && reviewStart(children[0].words);
|
||||
})) continue;
|
||||
for (const token of start.text.match(/\b[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}\b/g) ?? []) {
|
||||
// Archived public Linux paths keep their spelling when free tests run on Windows.
|
||||
const paths = sourcePaths(expected.directory);
|
||||
@@ -345,24 +460,25 @@ export function hasTrustedReviewStartRead(events: unknown[], expected: StartCont
|
||||
&& typeof pair.input?.command === 'string'
|
||||
// The documented shell variable is valid too: the trusted final row's
|
||||
// started_at below binds its resolved value to this observed capture.
|
||||
&& executedCommands(pair.input.command).some(call => basename(call.words[0]) === 'gstack-review-log'
|
||||
&& call.words.some((word, index) => word === '--finish' && (call.words[index + 1] === token
|
||||
|| /^\$(?:[A-Za-z_]\w*|\{[A-Za-z_]\w*\})$/.test(call.words[index + 1] ?? '')))));
|
||||
&& successfulFinish(pair.input.command, pair.text, token, expected));
|
||||
if (!finish) continue;
|
||||
for (const read of pairs) {
|
||||
if (read.at <= start.returnedAt || read.returnedAt >= finish.at) continue;
|
||||
const sameCall = read === start;
|
||||
if ((!sameCall && read.at <= start.returnedAt) || read.returnedAt >= finish.at) continue;
|
||||
const command = typeof read.input?.command === 'string' ? read.input.command : '';
|
||||
const directRead = read.tool === 'Read' && typeof read.input?.file_path === 'string'
|
||||
&& literalPath(read.input.file_path, expected.repo, paths) === file;
|
||||
// Discovery may return the path only in stdout; bind its cat invocation
|
||||
// to that discovery instead of accepting unrelated reader/find words.
|
||||
const shellRead = read.tool === 'Bash' && inspectsFile(command, file, containsPath(read.text, file), expected);
|
||||
const shellRead = read.tool === 'Bash' && (sameCall ? sameCallStartRead(command, file, expected, token)
|
||||
: inspectsFile(command, file, containsPath(read.text, file), expected));
|
||||
if (!directRead && !shellRead) continue;
|
||||
for (const line of read.text.split('\n')) {
|
||||
// Native Read can prefix the single-line JSON file with a line number.
|
||||
const json = line.replace(/^\s*\d+[\t →]+(?=\{)/, '').trim();
|
||||
let record: any;
|
||||
try { record = JSON.parse(json); } catch { continue; }
|
||||
if (sameCall && start.text.indexOf(token) >= start.text.indexOf(line)) continue;
|
||||
if (record?.skill === 'review' && record.repo === expected.repo && record.branch === expected.branch
|
||||
&& record.wtree === expected.wtree && typeof expected.startedAt === 'string'
|
||||
&& record.started_at === expected.startedAt) return true;
|
||||
|
||||
@@ -15,5 +15,11 @@ export function asideDriveOptions(text: string): string[] {
|
||||
}
|
||||
}
|
||||
if (current !== undefined) options.push(current);
|
||||
return options.filter(option => /\bAside\b/i.test(option) && /\b(?:drive|driving|browse|browsing|navigate|click)\b/i.test(option));
|
||||
return options.filter(option => {
|
||||
const currentOffer = option.replace(
|
||||
/\b(?:I|we)(?:['’]ll| will) (?:re[- ]?ask|ask again)(?: (?:you|this question))? with (?:the )?(?:"[^"]+"|“[^”]+”|[\w -]+?) option(?: included)?\b/gi,
|
||||
'',
|
||||
);
|
||||
return /\bAside\b/i.test(option) && /\b(?:drive|driving|browse|browsing|navigate|click)\b/i.test(currentOffer);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -21,9 +21,9 @@
|
||||
* Each test lists the file patterns that, if changed, require the test to run.
|
||||
*/
|
||||
export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'shared-libs-review-path-eligibility': ['review/**', 'scripts/resolvers/shared-libs.ts', 'scripts/resolvers/review.ts', 'scripts/resolvers/review-army.ts', 'lib/review-evidence.ts', 'bin/gstack-review-log', 'bin/gstack-review-read', 'bin/gstack-wtree', 'test/helpers/shared-libs-eval-fixture.ts', 'test/skill-e2e-shared-libs-paths.test.ts', 'test/helpers/shared-libs-path-fixture.ts', 'test/shared-libs-fixture.test.ts', 'test/helpers/e2e-gate.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/agent-sdk-runner.ts', 'lib/claude-bin.ts', 'lib/eval-model.ts', 'test/fixtures/shared-libs-index-flags-*.json', 'test/shared-libs-revalidation-prompt.test.ts'],
|
||||
'shared-libs-review-index-flags': ['review/**', 'scripts/resolvers/shared-libs.ts', 'scripts/resolvers/review.ts', 'scripts/resolvers/review-army.ts', 'lib/review-evidence.ts', 'bin/gstack-review-log', 'bin/gstack-review-read', 'bin/gstack-wtree', 'test/helpers/shared-libs-eval-fixture.ts', 'test/skill-e2e-shared-libs-paths.test.ts', 'test/helpers/shared-libs-path-fixture.ts', 'test/shared-libs-fixture.test.ts', 'test/helpers/e2e-gate.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/agent-sdk-runner.ts', 'lib/claude-bin.ts', 'lib/eval-model.ts', 'test/fixtures/shared-libs-index-flags-*.json', 'test/shared-libs-revalidation-prompt.test.ts', 'test/fixtures/shared-libs-paths-max-turns-public.json'],
|
||||
'shared-libs-review-prior-coverage': ['review/**', 'scripts/resolvers/shared-libs.ts', 'scripts/resolvers/review.ts', 'scripts/resolvers/review-army.ts', 'lib/review-evidence.ts', 'bin/gstack-review-log', 'bin/gstack-review-read', 'bin/gstack-wtree', 'test/helpers/shared-libs-eval-fixture.ts', 'test/skill-e2e-shared-libs-paths.test.ts', 'test/helpers/shared-libs-path-fixture.ts', 'test/shared-libs-fixture.test.ts', 'test/helpers/e2e-gate.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/agent-sdk-runner.ts', 'lib/claude-bin.ts', 'lib/eval-model.ts', 'test/fixtures/shared-libs-index-flags-*.json', 'test/shared-libs-revalidation-prompt.test.ts'],
|
||||
'shared-libs-review-path-eligibility': ['review/**', 'scripts/resolvers/shared-libs.ts', 'scripts/resolvers/review.ts', 'scripts/resolvers/review-army.ts', 'lib/review-evidence.ts', 'bin/gstack-review-log', 'bin/gstack-review-read', 'bin/gstack-wtree', 'test/helpers/shared-libs-eval-fixture.ts', 'test/skill-e2e-shared-libs-paths.test.ts', 'test/helpers/shared-libs-path-fixture.ts', 'test/shared-libs-fixture.test.ts', 'test/helpers/e2e-gate.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/agent-sdk-runner.ts', 'lib/claude-bin.ts', 'lib/eval-model.ts', 'test/fixtures/shared-libs-index-flags-*.json', 'test/shared-libs-revalidation-prompt.test.ts', 'test/shared-libs-source-reads.test.ts', 'test/fixtures/shared-libs-resolved-reads-public.json'],
|
||||
'shared-libs-review-index-flags': ['review/**', 'scripts/resolvers/shared-libs.ts', 'scripts/resolvers/review.ts', 'scripts/resolvers/review-army.ts', 'lib/review-evidence.ts', 'bin/gstack-review-log', 'bin/gstack-review-read', 'bin/gstack-wtree', 'test/helpers/shared-libs-eval-fixture.ts', 'test/skill-e2e-shared-libs-paths.test.ts', 'test/helpers/shared-libs-path-fixture.ts', 'test/shared-libs-fixture.test.ts', 'test/helpers/e2e-gate.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/agent-sdk-runner.ts', 'lib/claude-bin.ts', 'lib/eval-model.ts', 'test/fixtures/shared-libs-index-flags-*.json', 'test/shared-libs-revalidation-prompt.test.ts', 'test/fixtures/shared-libs-paths-max-turns-public.json', 'test/shared-libs-source-reads.test.ts', 'test/fixtures/shared-libs-resolved-reads-public.json'],
|
||||
'shared-libs-review-prior-coverage': ['review/**', 'scripts/resolvers/shared-libs.ts', 'scripts/resolvers/review.ts', 'scripts/resolvers/review-army.ts', 'lib/review-evidence.ts', 'bin/gstack-review-log', 'bin/gstack-review-read', 'bin/gstack-wtree', 'test/helpers/shared-libs-eval-fixture.ts', 'test/skill-e2e-shared-libs-paths.test.ts', 'test/helpers/shared-libs-path-fixture.ts', 'test/shared-libs-fixture.test.ts', 'test/helpers/e2e-gate.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/agent-sdk-runner.ts', 'lib/claude-bin.ts', 'lib/eval-model.ts', 'test/fixtures/shared-libs-index-flags-*.json', 'test/shared-libs-revalidation-prompt.test.ts', 'test/shared-libs-source-reads.test.ts', 'test/fixtures/shared-libs-resolved-reads-public.json'],
|
||||
'shared-libs-codex-read-only': ['deslop-shared-libs/**', 'scripts/resolvers/shared-libs.ts', 'scripts/resolvers/index.ts', 'scripts/gen-skill-docs.ts', 'test/helpers/shared-libs-eval-fixture.ts', 'test/helpers/codex-session-runner.ts', 'test/helpers/skill-fixture.ts', 'test/helpers/hermetic-env.ts', 'test/helpers/eval-budgets.ts', 'test/codex-e2e-shared-libs.test.ts', 'test/shared-libs-fixture.test.ts', 'test/helpers/e2e-gate.ts', 'hosts/codex.ts', 'hosts/define-host.ts', 'scripts/resolvers/constants.ts', 'test/fixtures/shared-libs-readonly-substitution-ci16358.json'],
|
||||
// Shared-code audit and scoped review lifecycle
|
||||
'shared-libs-read-only': ['deslop-shared-libs/**', 'scripts/resolvers/shared-libs.ts', 'scripts/resolvers/index.ts', 'test/helpers/shared-libs-eval-fixture.ts', 'test/skill-e2e-shared-libs.test.ts', 'test/shared-libs-fixture.test.ts', 'test/helpers/e2e-gate.ts', 'scripts/gen-skill-docs.ts', 'lib/claude-bin.ts', 'lib/eval-model.ts', 'test/fixtures/shared-libs-readonly-substitution-ci16358.json'],
|
||||
|
||||
Reference in New Issue
Block a user