mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-05 02:47:17 +02:00
v1.91.7.0 feat: add functional QA and pre-publication docs checks (#2983)
* feat: add surface-aware exploratory QA and ship documentation gates * test: preserve delegated QA setup authority after main integration * fix(qa): clarify exploration order and preserve report artifacts * test(qa): follow the shared setup reference directly * refactor(ship): make verification and recovery routes explicit * test(ship): align evidence and review guards with explicit routes * fix(workflows): clarify ship recovery and functional QA evidence * fix(workflows): clarify approval recovery and full QA coverage * refactor(workflows): order review transactions and clarify ship state * fix(ship): clarify final verification and fail closed at publication * fix(evals): attribute native atomic documentation writes * fix(ship): clarify recovery and documentation lifecycle guidance * fix(test): preserve observed native placeholder styling in CI * fix(codex): report watchdog timeouts without a process-exit race * Checkpoint functional QA implementation and workflow validation repairs * Fix documentation and shared-review fixture contracts * docs: clarify judge reuse and evaluation supervision * test: align review evidence and selected case contracts * test: verify append-only documentation checkpoints and recovery * fix: qualify QA workflows and CI validation repairs * fix: launch shared-libs fixture scripts on Windows * fix: qualify QA deadlines, fixture isolation, and shard cleanup * fix: preserve qualified QA and cancellation repairs * fix: enforce functional fixture authority and share strict event decoding * fix: retain free-test evidence and explain recovery * fix: reject malformed native evidence after decoder consolidation * test: use reliable capture for telemetry privacy filters * test: refresh measured quick coverage and document validation costs * Fix native fixture receipts and preserve VM validation evidence * Align negative judge controls with upstream clarity policy * Fix report-only QA preparation and public evidence handling * Clarify QA-only preparation and current-report preservation * Stream Ship quality judgments with an explicit 64k response contract * Validate compact judge reasoning locally with supported wire schema * Align functional QA fixture instructions with evidence acceptance * Bind native browser diagnostics to execution evidence and align review verdicts * Preserve native diagnostic line boundaries * Serialize functional QA evidence from native captures * Keep large QA evidence fixture payload out of Windows argv
This commit is contained in:
1 parent
65bfb0ce49
commit
dcaea52800
333 files changed
+41755
-7357
No files matched your search
@@ -2,10 +2,11 @@
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { sharedLibsFingerprint } from '../../lib/review-evidence';
|
||||
import {
|
||||
SHARED_LIBS_ROOT, commitFixture, createSharedLibsFixture, fixtureGit, fixtureWrite,
|
||||
fixtureWorkingTree, seedReviewSources, shellQuote, type SharedLibsFixture,
|
||||
fixtureWorkingTree, seedReviewSources, shellQuote, snapshotFixture, type SharedLibsFixture, type SharedReviewResume, type SharedReviewStageActor,
|
||||
} from './shared-libs-eval-fixture';
|
||||
|
||||
export type PathEligibilityCase = 'symlinks' | 'submodule' | 'ignored' | 'legacy' | 'assume-unchanged' | 'skip-worktree' | 'removed-filter';
|
||||
@@ -15,6 +16,158 @@ export interface PathEligibilityFixture {
|
||||
beforeTree: string;
|
||||
sourcePaths: string[];
|
||||
rawPaths: string[];
|
||||
resumed: SharedReviewResume;
|
||||
}
|
||||
|
||||
function pathReviewState(f: SharedLibsFixture) {
|
||||
const root = fs.realpathSync(f.root), repo = fs.realpathSync(f.repo), state = fs.realpathSync(f.state);
|
||||
if (repo !== path.join(root, 'repo') || state !== path.join(root, 'state')) throw new Error('Foreign path fixture state');
|
||||
const raw = Object.fromEntries(Object.entries(snapshotFixture(repo)).filter(([file, value]) =>
|
||||
!file.split(path.sep).includes('.git') || /(?:^|\/)(?:config|info\/(?:attributes|exclude))$/.test(file)
|
||||
|| path.basename(file) === '.git' && !value.startsWith('dir:')));
|
||||
return { root, repo, state, branch: fixtureGit(f, 'symbolic-ref', '--short', 'HEAD'),
|
||||
head: fixtureGit(f, 'rev-parse', 'HEAD'), base: fixtureGit(f, 'rev-parse', 'origin/main'),
|
||||
wtree: fixtureWorkingTree(f), index: fixtureGit(f, 'ls-files', '--stage', '-v'), raw };
|
||||
}
|
||||
|
||||
export function seedPathReviewPrerequisites(f: SharedLibsFixture): SharedReviewResume {
|
||||
const input = path.join(f.root, 'resumed-review-prerequisites.json');
|
||||
const changedLines = fixtureGit(f, 'diff', '--numstat', 'origin/main').split('\n')
|
||||
.reduce((sum, line) => sum + line.split('\t').slice(0, 2).reduce((n, value) => n + Number(value || 0), 0), 0);
|
||||
if (!Number.isFinite(changedLines) || changedLines >= 50) throw new Error('Resumed path fixture requires a tiny no-edit diff');
|
||||
const context = { kind: 'synthetic-path-review-prerequisites', synthetic: true, native_coverage: false,
|
||||
binding: pathReviewState(f),
|
||||
qa: { settled: true, required_probes: [{ id: 'retry-contract', status: 'passed',
|
||||
result: 'Synthetic fixture input: Retry-After seconds/date parsing, ceiling and fallback probes passed.' }], findings: [] },
|
||||
native_adversarial: { settled: true, status: 'completed', findings: [],
|
||||
result: 'Synthetic fixture input: native adversarial review returned no findings.' },
|
||||
structured_review: { required: false, reason: 'Tiny diff; no full-review, structured-review or P1 override requested.' } };
|
||||
fs.writeFileSync(input, JSON.stringify(context, null, 2) + '\n', { mode: 0o600 });
|
||||
return { input, checkCommand: `bun ${shellQuote(path.join(SHARED_LIBS_ROOT, 'test/helpers/shared-libs-path-fixture.ts'))} --check-review-prerequisites ${shellQuote(input)}` };
|
||||
}
|
||||
|
||||
export function checkPathReviewPrerequisites(f: SharedLibsFixture, input: string) {
|
||||
try {
|
||||
if (input !== path.join(f.root, 'resumed-review-prerequisites.json')) throw new Error('Foreign prerequisite file');
|
||||
const text = fs.readFileSync(input, 'utf8'), context = JSON.parse(text);
|
||||
const current = JSON.stringify(context.binding) === JSON.stringify(pathReviewState(f));
|
||||
const settled = current && context.kind === 'synthetic-path-review-prerequisites'
|
||||
&& context.synthetic === true && context.native_coverage === false
|
||||
&& context.qa?.settled === true && Array.isArray(context.qa.required_probes) && context.qa.required_probes.length === 1
|
||||
&& context.qa.required_probes.every((probe: any) => probe.id === 'retry-contract' && probe.status === 'passed'
|
||||
&& typeof probe.result === 'string' && probe.result.length > 0)
|
||||
&& Array.isArray(context.qa.findings) && context.qa.findings.length === 0
|
||||
&& context.native_adversarial?.settled === true && context.native_adversarial.status === 'completed'
|
||||
&& Array.isArray(context.native_adversarial.findings) && context.native_adversarial.findings.length === 0
|
||||
&& typeof context.native_adversarial.result === 'string' && context.native_adversarial.result.length > 0
|
||||
&& context.structured_review?.required === false;
|
||||
return { synthetic: true, native_coverage: false, settled, current,
|
||||
input_sha256: createHash('sha256').update(text).digest('hex'), context };
|
||||
} catch {
|
||||
return { synthetic: true, native_coverage: false, settled: false, current: false };
|
||||
}
|
||||
}
|
||||
|
||||
export function hasPathReviewPrerequisiteReceipt(events: any[], command: string, expected: ReturnType<typeof checkPathReviewPrerequisites>): boolean {
|
||||
if (!expected.settled) return false;
|
||||
const calls = new Set<string>();
|
||||
let verified = false;
|
||||
for (const event of events) for (const block of Array.isArray(event.message?.content) ? event.message.content : []) {
|
||||
if (event.type === 'assistant' && block.type === 'tool_use' && block.name === 'Bash') {
|
||||
if (block.input?.command === command) calls.add(block.id);
|
||||
if (String(block.input?.command).includes('--finish')) return verified;
|
||||
}
|
||||
if (event.type !== 'user' || block.type !== 'tool_result' || block.is_error === true || !calls.has(block.tool_use_id)) continue;
|
||||
const text = typeof block.content === 'string' ? block.content : Array.isArray(block.content)
|
||||
? block.content.filter((part: any) => part.type === 'text').map((part: any) => part.text).join('\n') : '';
|
||||
try { verified = JSON.stringify(JSON.parse(text)) === JSON.stringify(expected); } catch { verified = false; }
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export function createLifecyclePrerequisiteActor(f: SharedLibsFixture): SharedReviewStageActor {
|
||||
const directory = path.join(fs.realpathSync(f.root), 'synthetic-stage-receipts');
|
||||
fs.mkdirSync(directory, { mode: 0o700 });
|
||||
const output = path.join(directory, 'current.json');
|
||||
const actorCommand = `cat ${shellQuote(output)}`;
|
||||
let state = pathReviewState(f), generation = 0;
|
||||
const isolation = [state.root, state.repo, state.state];
|
||||
const issued = new Map<string, { text: string; file: string; generation: number; settled: boolean }>();
|
||||
const finishes = new Map<string, { command: string; generation: number }>();
|
||||
const observe = () => {
|
||||
const current = pathReviewState(f);
|
||||
if (JSON.stringify([current.root, current.repo, current.state]) !== JSON.stringify(isolation)) throw new Error('Rebound synthetic stage fixture');
|
||||
if (JSON.stringify(current) !== JSON.stringify(state)) { generation++; state = current; }
|
||||
return current;
|
||||
};
|
||||
const beforeTool: SharedReviewStageActor['hooks']['PreToolUse'][number]['hooks'][number] = async (input, toolUseID) => {
|
||||
if (input.hook_event_name !== 'PreToolUse') return {};
|
||||
const current = observe();
|
||||
const command = input.tool_name === 'Bash' ? (input.tool_input as any)?.command : undefined;
|
||||
if (command === actorCommand) {
|
||||
const id = input.tool_use_id;
|
||||
if (fs.realpathSync(input.cwd) !== current.repo || !id || toolUseID !== undefined && id !== toolUseID || issued.has(id)
|
||||
|| fs.realpathSync(directory) !== directory || fs.existsSync(output) && fs.lstatSync(output).isSymbolicLink()) {
|
||||
return { hookSpecificOutput: { hookEventName: 'PreToolUse', permissionDecision: 'deny', permissionDecisionReason: 'Invalid synthetic stage invocation' } };
|
||||
}
|
||||
const evidence_paths = ['src/retry-worker.ts', 'src/retry-route.ts', 'lib/retry-after.ts'];
|
||||
const fingerprint = sharedLibsFingerprint({ evidence_paths, helper_target: { path: 'lib/retry-after.ts', symbol: 'retrySeconds' } });
|
||||
const changedLines = fixtureGit(f, 'diff', '--numstat', 'origin/main').split('\n')
|
||||
.reduce((sum, line) => sum + line.split('\t').slice(0, 2).reduce((n, value) => n + Number(value || 0), 0), 0);
|
||||
const tinyDiff = Number.isFinite(changedLines) && changedLines < 50;
|
||||
const authoredEvidence = !!fingerprint && evidence_paths.every(source => {
|
||||
const file = path.join(current.repo, source);
|
||||
try { return fs.realpathSync(file) === file && fs.lstatSync(file).isFile() && fs.statSync(file).size > 0; }
|
||||
catch { return false; }
|
||||
});
|
||||
const supported = tinyDiff && authoredEvidence;
|
||||
const receipt = { kind: 'synthetic-lifecycle-stage-result', id: randomUUID(), tool_use_id: id,
|
||||
synthetic: true, native_coverage: false, generation, binding: current,
|
||||
deterministic_checks: { fixture_isolation: true, tiny_diff: tinyDiff, authored_evidence: authoredEvidence, fingerprint },
|
||||
qa: { settled: supported, required_probes: [{ id: 'retry-contract', status: supported ? 'passed' : 'blocked',
|
||||
result: 'Simulated fixture QA outcome, not execution of target tests.' }], findings: [] },
|
||||
native_adversarial: { settled: supported, status: supported ? 'completed' : 'blocked', findings: [],
|
||||
result: 'Simulated fixture adversarial outcome, not an actual native review.' },
|
||||
...(tinyDiff ? { structured_review: { required: false, reason: 'Tiny diff; no full-review override in this fixture.' } } : {}),
|
||||
settled: supported };
|
||||
const text = JSON.stringify(receipt), file = path.join(directory, `${receipt.id}.json`);
|
||||
fs.writeFileSync(file, text, { flag: 'wx', mode: 0o600 });
|
||||
fs.writeFileSync(output, text, { mode: 0o600 });
|
||||
issued.set(id, { text, file, generation, settled: supported });
|
||||
return { hookSpecificOutput: { hookEventName: 'PreToolUse', permissionDecision: 'allow' } };
|
||||
}
|
||||
if (typeof command === 'string' && command.includes('gstack-review-log') && command.includes('--finish')) {
|
||||
finishes.set(input.tool_use_id, { command, generation });
|
||||
}
|
||||
return {};
|
||||
};
|
||||
return { actorCommand, hooks: { PreToolUse: [{ hooks: [beforeTool] }] },
|
||||
history: () => [...issued.values()].map(receipt => JSON.parse(receipt.text)),
|
||||
verify(events) {
|
||||
try {
|
||||
observe();
|
||||
if (!issued.size || [...issued.values()].some(receipt => fs.readFileSync(receipt.file, 'utf8') !== receipt.text)) return false;
|
||||
const calls = new Map<string, string>();
|
||||
let consumed: string | undefined, final: string | undefined, finalReceipt: string | undefined, finished = false;
|
||||
for (const event of events) for (const block of Array.isArray(event.message?.content) ? event.message.content : []) {
|
||||
if (event.type === 'assistant' && block.type === 'tool_use' && block.name === 'Bash') {
|
||||
calls.set(block.id, block.input?.command);
|
||||
if (finishes.get(block.id)?.command === block.input?.command) { final = block.id; finalReceipt = consumed; finished = false; }
|
||||
}
|
||||
if (event.type !== 'user' || block.type !== 'tool_result') continue;
|
||||
if (block.tool_use_id === final) finished = block.is_error !== true;
|
||||
if (calls.get(block.tool_use_id) !== actorCommand) continue;
|
||||
const receipt = issued.get(block.tool_use_id);
|
||||
const text = typeof block.content === 'string' ? block.content : Array.isArray(block.content)
|
||||
? block.content.filter((part: any) => part.type === 'text').map((part: any) => part.text).join('\n') : '';
|
||||
consumed = receipt && receipt.settled && block.is_error !== true
|
||||
&& JSON.stringify(JSON.parse(text)) === receipt.text ? block.tool_use_id : undefined;
|
||||
}
|
||||
const receipt = finalReceipt ? issued.get(finalReceipt) : undefined;
|
||||
return !!receipt && finished && finalReceipt === [...issued.keys()].at(-1)
|
||||
&& final === [...finishes.keys()].at(-1) && receipt.generation === generation && finishes.get(final!)?.generation === generation;
|
||||
} catch { return false; }
|
||||
} };
|
||||
}
|
||||
|
||||
function seedBoundSkip(f: SharedLibsFixture, finding: Record<string, any>): void {
|
||||
@@ -150,9 +303,17 @@ export function preparePathEligibilityFixture(kind: PathEligibilityCase): PathEl
|
||||
+ `\n// Authored caller changed after the prior decision (${kind}).\n`);
|
||||
}
|
||||
if (fixtureWorkingTree(f) !== beforeTree) throw new Error(`${kind}: fixture must retain the parent Git tree`);
|
||||
return { fixture, current, beforeTree, sourcePaths, rawPaths };
|
||||
return { fixture, current, beforeTree, sourcePaths, rawPaths, resumed: seedPathReviewPrerequisites(f) };
|
||||
} catch (error) {
|
||||
fs.rmSync(f.root, { recursive: true, force: true });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
const [flag, input] = process.argv.slice(2);
|
||||
if (flag !== '--check-review-prerequisites' || !input || !process.env.GSTACK_HOME) process.exit(2);
|
||||
const f = { root: path.dirname(input), repo: process.cwd(), state: process.env.GSTACK_HOME,
|
||||
env: { GSTACK_HOME: process.env.GSTACK_HOME } } as SharedLibsFixture;
|
||||
console.log(JSON.stringify(checkPathReviewPrerequisites(f, input)));
|
||||
}
|
||||
Reference in new issue
Block a user