mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-03 01:46:55 +02:00
feat(qa): helpers answer --help, and the QA eval interfaces declare it
Approved by Garry: asking gstack-qa-evidence or gstack-qa-deadline for usage
is read-only, so both helpers print usage and exit 0 on --help (the evidence
usage now names the annotation shape), and the functional and caller command
allowlists accept exactly 'bun <path>/bin/gstack-qa-{evidence,deadline} --help'.
Two CI runs failed only on that call.
This commit is contained in:
1 parent
4643cb8550
commit
ee929ff710
5 files changed
+37
-3
No files matched your search
+6
-1
@@ -7,6 +7,7 @@ import { initializeWindowsReviewJob } from './claude-code-windows-job';
|
|||||||
|
|
||||||
const MAX_MS = 2_147_483_647;
|
const MAX_MS = 2_147_483_647;
|
||||||
class QaDeadlineError extends Error {}
|
class QaDeadlineError extends Error {}
|
||||||
|
const QA_DEADLINE_USAGE = 'gstack-qa-deadline start FILE SECONDS [EARLIER_UTC] | status FILE | run FILE -- COMMAND ARGS...';
|
||||||
type QaCommandResult = { exitCode: number; signal: NodeJS.Signals | null; completed: boolean };
|
type QaCommandResult = { exitCode: number; signal: NodeJS.Signals | null; completed: boolean };
|
||||||
type Emit = (stream: 'stdout' | 'stderr', receipt: Record<string, unknown>, completion?: QaCommandResult) => void;
|
type Emit = (stream: 'stdout' | 'stderr', receipt: Record<string, unknown>, completion?: QaCommandResult) => void;
|
||||||
|
|
||||||
@@ -269,6 +270,10 @@ export async function qaDeadlineMain(args: string[], receiptWorker = false): Pro
|
|||||||
return withQaReceiptOutput(receiptWorker, 'qa-deadline-receipt', receipt => '\nQA_DEADLINE ' + JSON.stringify({ guard: 'qa-deadline', ...receipt }) + '\n', async emit => {
|
return withQaReceiptOutput(receiptWorker, 'qa-deadline-receipt', receipt => '\nQA_DEADLINE ' + JSON.stringify({ guard: 'qa-deadline', ...receipt }) + '\n', async emit => {
|
||||||
try {
|
try {
|
||||||
const [action, file, ...rest] = args;
|
const [action, file, ...rest] = args;
|
||||||
|
if (action === '--help' && args.length === 1) {
|
||||||
|
emit('stdout', { event: 'help', usage: QA_DEADLINE_USAGE });
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
if (action === 'start' && file && (rest.length === 1 || rest.length === 2)) {
|
if (action === 'start' && file && (rest.length === 1 || rest.length === 2)) {
|
||||||
const status = qaDeadlineStatus(startQaDeadline(file, rest[0], rest[1]));
|
const status = qaDeadlineStatus(startQaDeadline(file, rest[0], rest[1]));
|
||||||
emit('stdout', { event: 'start', ...status });
|
emit('stdout', { event: 'start', ...status });
|
||||||
@@ -283,7 +288,7 @@ export async function qaDeadlineMain(args: string[], receiptWorker = false): Pro
|
|||||||
if (process.platform === 'win32' && !receiptWorker) return await runWindowsWorker(args, emit);
|
if (process.platform === 'win32' && !receiptWorker) return await runWindowsWorker(args, emit);
|
||||||
return await runQaDeadlineCommand(file, rest[1], rest.slice(2), emit);
|
return await runQaDeadlineCommand(file, rest[1], rest.slice(2), emit);
|
||||||
}
|
}
|
||||||
throw new QaDeadlineError('Usage: gstack-qa-deadline start FILE SECONDS [EARLIER_UTC] | status FILE | run FILE -- COMMAND ARGS...');
|
throw new QaDeadlineError(`Usage: ${QA_DEADLINE_USAGE}`);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
emit('stderr', { event: 'error', message: error instanceof QaDeadlineError ? error.message : 'Deadline guard failed' });
|
emit('stderr', { event: 'error', message: error instanceof QaDeadlineError ? error.message : 'Deadline guard failed' });
|
||||||
return 2;
|
return 2;
|
||||||
|
|||||||
+7
-1
@@ -277,12 +277,18 @@ function materialize(root: string, source: string) {
|
|||||||
next: 'Include every reportLinks entry in the Markdown report.' };
|
next: 'Include every reportLinks entry in the Markdown report.' };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const QA_EVIDENCE_USAGE = 'capture ROOT ID [--public] --deadline FILE|--timeout-ms MS -- COMMAND ARGS | checkpoint ROOT ID CAPTURE OBSERVATION_COMMAND HYPOTHESIS NEXT_COMMAND | checkpoint ROOT ID INTENT_FILE | materialize ROOT ANNOTATIONS (annotations: {evidence: [{capture, command, contract, expected, classification}], limits: [..]}; revision, runtime, cwd and learning are filled in)';
|
||||||
|
|
||||||
export async function qaEvidenceMain(args: string[]): Promise<number> {
|
export async function qaEvidenceMain(args: string[]): Promise<number> {
|
||||||
return withQaReceiptOutput(false, 'qa-evidence-receipt', value => value.event === 'observation'
|
return withQaReceiptOutput(false, 'qa-evidence-receipt', value => value.event === 'observation'
|
||||||
? JSON.stringify(value.observed) + '\n' : value.event === 'diagnostic' ? String(value.stderr)
|
? JSON.stringify(value.observed) + '\n' : value.event === 'diagnostic' ? String(value.stderr)
|
||||||
: '\nQA_EVIDENCE ' + JSON.stringify({ producer: 'gstack-qa-evidence', version: 1, ...value }) + '\n', async emit => {
|
: '\nQA_EVIDENCE ' + JSON.stringify({ producer: 'gstack-qa-evidence', version: 1, ...value }) + '\n', async emit => {
|
||||||
try {
|
try {
|
||||||
const [action, reportRoot, ...rest] = args;
|
const [action, reportRoot, ...rest] = args;
|
||||||
|
if (action === '--help' && args.length === 1) {
|
||||||
|
emit('stdout', { action: 'help', status: 'complete', usage: QA_EVIDENCE_USAGE, exitCode: 0 });
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
const root = qaEvidenceRoot(reportRoot);
|
const root = qaEvidenceRoot(reportRoot);
|
||||||
let receipt: Record<string, any>;
|
let receipt: Record<string, any>;
|
||||||
const publicOutput = action === 'capture' && rest[1] === '--public';
|
const publicOutput = action === 'capture' && rest[1] === '--public';
|
||||||
@@ -297,7 +303,7 @@ export async function qaEvidenceMain(args: string[]): Promise<number> {
|
|||||||
} else if (action === 'checkpoint' && rest.length === 2) receipt = checkpoint(root, rest[0], rest[1]);
|
} else if (action === 'checkpoint' && rest.length === 2) receipt = checkpoint(root, rest[0], rest[1]);
|
||||||
else if (action === 'checkpoint' && rest.length === 5) receipt = checkpoint(root, rest[0], { capture: rest[1], observationCommand: rest[2], hypothesis: rest[3], nextCommand: rest[4] });
|
else if (action === 'checkpoint' && rest.length === 5) receipt = checkpoint(root, rest[0], { capture: rest[1], observationCommand: rest[2], hypothesis: rest[3], nextCommand: rest[4] });
|
||||||
else if (action === 'materialize' && rest.length === 1) receipt = materialize(root, rest[0]);
|
else if (action === 'materialize' && rest.length === 1) receipt = materialize(root, rest[0]);
|
||||||
else throw new QaEvidenceError('Usage: capture ROOT ID [--public] --deadline FILE|--timeout-ms MS -- COMMAND ARGS | checkpoint ROOT ID CAPTURE OBSERVATION_COMMAND HYPOTHESIS NEXT_COMMAND | checkpoint ROOT ID INTENT_FILE | materialize ROOT ANNOTATIONS');
|
else throw new QaEvidenceError(`Usage: ${QA_EVIDENCE_USAGE}`);
|
||||||
emit('stdout', receipt);
|
emit('stdout', receipt);
|
||||||
return receipt.status === 'complete' ? receipt.exitCode : receipt.status === 'incomplete' ? receipt.exitCode || 2 : 2;
|
return receipt.status === 'complete' ? receipt.exitCode : receipt.status === 'incomplete' ? receipt.exitCode || 2 : 2;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import { runSkillTest, SESSION_DRAIN_GRACE_MS } from './session-runner';
|
|||||||
import { CAPTURE_MS } from './eval-budgets';
|
import { CAPTURE_MS } from './eval-budgets';
|
||||||
import { refreshHermeticSkillRuntime } from './hermetic-skill-runtime';
|
import { refreshHermeticSkillRuntime } from './hermetic-skill-runtime';
|
||||||
import { seedHermeticGstackHome } from './hermetic-env';
|
import { seedHermeticGstackHome } from './hermetic-env';
|
||||||
import { observeQAWrites, type QAWriteObservation } from './qa-functional-observer';
|
import { observeQAWrites, qaHelperUsageCommand, type QAWriteObservation } from './qa-functional-observer';
|
||||||
import { nativeCalls, readQACheckpointFiles, validateQACheckpoints } from './qa-checkpoint-evidence';
|
import { nativeCalls, readQACheckpointFiles, validateQACheckpoints } from './qa-checkpoint-evidence';
|
||||||
import { ownedPath } from './qa-functional-fixture';
|
import { ownedPath } from './qa-functional-fixture';
|
||||||
import { qaEvidenceCommand, qaNativeCapture, qaProducerReceipt, type QaEvidenceContext } from './qa-evidence-producer';
|
import { qaEvidenceCommand, qaNativeCapture, qaProducerReceipt, type QaEvidenceContext } from './qa-evidence-producer';
|
||||||
@@ -214,6 +214,7 @@ function callerDeadlineCommand(command: string, context?: CallerDeadlineContext)
|
|||||||
|
|
||||||
export function qaCallerCommandAllowed(command: string, workflowCommands: string[] = [], deadline?: CallerDeadlineContext): boolean {
|
export function qaCallerCommandAllowed(command: string, workflowCommands: string[] = [], deadline?: CallerDeadlineContext): boolean {
|
||||||
const text = command.trim();
|
const text = command.trim();
|
||||||
|
if (qaHelperUsageCommand(text)) return true;
|
||||||
if (callerEvidenceCommand(text, deadline)) return true;
|
if (callerEvidenceCommand(text, deadline)) return true;
|
||||||
if (callerDeadlineCommand(text, deadline)) return true;
|
if (callerDeadlineCommand(text, deadline)) return true;
|
||||||
if (/\bgstack-qa-(?:deadline|evidence)\b/.test(text) && !literalCallerCLI.test(text)
|
if (/\bgstack-qa-(?:deadline|evidence)\b/.test(text) && !literalCallerCLI.test(text)
|
||||||
|
|||||||
@@ -281,7 +281,13 @@ export function qaWriteVerdict(observation: QAWriteObservation, mode: QAMode): s
|
|||||||
return failures;
|
return failures;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Asking an installed gstack QA helper for its own usage text is read-only and always declared. */
|
||||||
|
export function qaHelperUsageCommand(command: string): boolean {
|
||||||
|
return /^bun (?:[\w./-]+\/)?bin\/gstack-qa-(?:evidence|deadline) --help$/.test(command.trim());
|
||||||
|
}
|
||||||
|
|
||||||
export function qaCommandAllowed(command: string, root?: string): boolean {
|
export function qaCommandAllowed(command: string, root?: string): boolean {
|
||||||
|
if (qaHelperUsageCommand(command)) return true;
|
||||||
const producer = root ? qaEvidenceCommand(command, { cwd: root, reportRoot: path.join(root, 'qa-reports'), executable: path.join(root, 'bin/gstack-qa-evidence') }) : undefined;
|
const producer = root ? qaEvidenceCommand(command, { cwd: root, reportRoot: path.join(root, 'qa-reports'), executable: path.join(root, 'bin/gstack-qa-evidence') }) : undefined;
|
||||||
if (producer) {
|
if (producer) {
|
||||||
try { ownedPath(root!, 'bin/gstack-qa-evidence'); } catch { return false; }
|
try { ownedPath(root!, 'bin/gstack-qa-evidence'); } catch { return false; }
|
||||||
|
|||||||
@@ -5,6 +5,9 @@ import * as path from 'node:path';
|
|||||||
import { spawn, spawnSync } from 'node:child_process';
|
import { spawn, spawnSync } from 'node:child_process';
|
||||||
import { randomBytes } from 'node:crypto';
|
import { randomBytes } from 'node:crypto';
|
||||||
|
|
||||||
|
import { qaCommandAllowed } from './helpers/qa-functional-observer';
|
||||||
|
import { qaCallerCommandAllowed } from './helpers/qa-callers-fixture';
|
||||||
|
|
||||||
const CLI = path.resolve(import.meta.dir, '../bin/gstack-qa-evidence');
|
const CLI = path.resolve(import.meta.dir, '../bin/gstack-qa-evidence');
|
||||||
const ROOT = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), 'qa-evidence-'));
|
const ROOT = fs.mkdtempSync(path.join(fs.realpathSync(os.tmpdir()), 'qa-evidence-'));
|
||||||
afterAll(() => fs.rmSync(ROOT, { recursive: true, force: true }));
|
afterAll(() => fs.rmSync(ROOT, { recursive: true, force: true }));
|
||||||
@@ -294,3 +297,16 @@ test('materialize rejects placeholder metadata and same-probe learning with the
|
|||||||
expect(selected.status, selected.stderr).toBe(0);
|
expect(selected.status, selected.stderr).toBe(0);
|
||||||
expect(JSON.parse(fs.readFileSync(path.join(f.root, 'evidence.json'), 'utf8')).learning).toEqual([]);
|
expect(JSON.parse(fs.readFileSync(path.join(f.root, 'evidence.json'), 'utf8')).learning).toEqual([]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('both QA helpers answer --help with usage and exit 0, and the declared interfaces allow it', () => {
|
||||||
|
for (const [cli, needle] of [[CLI, 'materialize ROOT ANNOTATIONS'], [path.resolve(import.meta.dir, '../bin/gstack-qa-deadline'), 'status FILE']] as const) {
|
||||||
|
const result = spawnSync(process.execPath, [cli, '--help'], { encoding: 'utf8', timeout: 10_000 });
|
||||||
|
expect(result.status, result.stderr).toBe(0);
|
||||||
|
expect(result.stdout).toContain(needle);
|
||||||
|
}
|
||||||
|
expect(qaCommandAllowed('bun bin/gstack-qa-evidence --help')).toBe(true);
|
||||||
|
expect(qaCommandAllowed('bun /abs/runtime/bin/gstack-qa-deadline --help')).toBe(true);
|
||||||
|
expect(qaCommandAllowed('bun bin/gstack-qa-evidence --help; rm -rf x')).toBe(false);
|
||||||
|
expect(qaCommandAllowed('bun bin/gstack-qa-evidence --version')).toBe(false);
|
||||||
|
expect(qaCallerCommandAllowed('bun /abs/host/runtime/bin/gstack-qa-evidence --help')).toBe(true);
|
||||||
|
});
|
||||||
Reference in new issue
Block a user