v1.91.7.0 feat: add functional QA and pre-publication docs checks (#2983)

* feat: add surface-aware exploratory QA and ship documentation gates

* test: preserve delegated QA setup authority after main integration

* fix(qa): clarify exploration order and preserve report artifacts

* test(qa): follow the shared setup reference directly

* refactor(ship): make verification and recovery routes explicit

* test(ship): align evidence and review guards with explicit routes

* fix(workflows): clarify ship recovery and functional QA evidence

* fix(workflows): clarify approval recovery and full QA coverage

* refactor(workflows): order review transactions and clarify ship state

* fix(ship): clarify final verification and fail closed at publication

* fix(evals): attribute native atomic documentation writes

* fix(ship): clarify recovery and documentation lifecycle guidance

* fix(test): preserve observed native placeholder styling in CI

* fix(codex): report watchdog timeouts without a process-exit race

* Checkpoint functional QA implementation and workflow validation repairs

* Fix documentation and shared-review fixture contracts

* docs: clarify judge reuse and evaluation supervision

* test: align review evidence and selected case contracts

* test: verify append-only documentation checkpoints and recovery

* fix: qualify QA workflows and CI validation repairs

* fix: launch shared-libs fixture scripts on Windows

* fix: qualify QA deadlines, fixture isolation, and shard cleanup

* fix: preserve qualified QA and cancellation repairs

* fix: enforce functional fixture authority and share strict event decoding

* fix: retain free-test evidence and explain recovery

* fix: reject malformed native evidence after decoder consolidation

* test: use reliable capture for telemetry privacy filters

* test: refresh measured quick coverage and document validation costs

* Fix native fixture receipts and preserve VM validation evidence

* Align negative judge controls with upstream clarity policy

* Fix report-only QA preparation and public evidence handling

* Clarify QA-only preparation and current-report preservation

* Stream Ship quality judgments with an explicit 64k response contract

* Validate compact judge reasoning locally with supported wire schema

* Align functional QA fixture instructions with evidence acceptance

* Bind native browser diagnostics to execution evidence and align review verdicts

* Preserve native diagnostic line boundaries

* Serialize functional QA evidence from native captures

* Keep large QA evidence fixture payload out of Windows argv
This commit is contained in:
Garry Tan authored and GitHub committed 2026-09-29 06:07:35 -07:00
1 parent 65bfb0ce49
commit dcaea52800
333 files changed
+41755 -7357

No files matched your search

+43 -4
View File
@@ -11,6 +11,8 @@ import { launchClaudePty, runPlanSkillObservation, isProseAUQVisible, isNumbered
const CLI = fs.readFileSync(path.join(import.meta.dir, 'fixtures', 'plan-seed-cli.ts'), 'utf8');
for (const scenario of ['success', 'completed-tool', 'status-updating', 'history-empty-box',
'native-paste', 'native-paste-block', 'native-paste-changed', 'native-paste-fused',
'native-paste-mismatched', 'native-paste-duplicate', 'native-paste-appended', 'native-paste-multiple-blocks',
'startup-placeholder', 'startup-placeholder-cursor', 'startup-placeholder-unicode',
'startup-typed-hint', 'startup-partial-dim', 'startup-prior-conversation', 'startup-missing-styles',
'startup-waiting', 'startup-prose-question', 'startup-permission', 'startup-fresh-waiting',
@@ -55,7 +57,7 @@ for (const scenario of ['success', 'completed-tool', 'status-updating', 'history
try { await submitPlanSeed(session, seed, { cwd: dir, launchedAt, deadlineAt,
isQuestionOrPermission: text => isProseAUQVisible(text) || isNumberedOptionListVisible(text) || isPermissionDialogVisible(text) }); }
catch (error) { failure = error; }
if (['success', 'completed-tool', 'status-updating', 'history-empty-box', 'startup-placeholder', 'startup-placeholder-cursor', 'startup-placeholder-unicode'].includes(scenario)) {
if (['success', 'completed-tool', 'status-updating', 'history-empty-box', 'native-paste', 'native-paste-block', 'startup-placeholder', 'startup-placeholder-cursor', 'startup-placeholder-unicode'].includes(scenario)) {
expect(failure).toBeUndefined();
session.send('/plan-eng-review\r');
await Bun.sleep(50);
@@ -69,7 +71,7 @@ for (const scenario of ['success', 'completed-tool', 'status-updating', 'history
'session-switch': 'native session changed', 'foreign-cwd': 'Foreign cwd',
question: 'requires an answer', 'prose-question': 'requires an answer', 'wrong-pid': 'does not match this launch',
'wrong-start': 'native process identity changed', 'wrong-domain': 'native process identity changed' } as Record<string, string>)[scenario]
?? 'existing case budget';
?? (scenario.startsWith('native-paste') ? 'fused, duplicated, or changed' : 'existing case budget');
expect((failure as Error).message).toContain(expected);
expect(sent.some(s => s === '/plan-eng-review\r')).toBe(false);
expect(sent.filter(s => s === '\r').length).toBeLessThanOrEqual(1);
@@ -85,7 +87,13 @@ for (const scenario of ['success', 'completed-tool', 'status-updating', 'history
}, 6000);
}
for (const inheritedTerm of ['dumb', '', 'xterm-256color']) test.skipIf(process.platform === 'win32')(`actual PTY launcher carries placeholder styling into owned seed submission: ${inheritedTerm || 'empty TERM'}`, async () => {
for (const entry of [
...['dumb', '', 'xterm-256color'].map(TERM => ({ name: TERM || 'empty TERM', env: { TERM } })),
{ name: 'CI', env: { CI: '1' } },
{ name: 'CI with explicit disabled color', env: { CI: '1', FORCE_COLOR: '0' } },
{ name: 'CI with explicit NO_COLOR', env: { CI: '1', NO_COLOR: '1' } },
{ name: 'CI with all explicit conflicting terminal knobs', env: { CI: '1', TERM: 'dumb', COLORTERM: '', FORCE_COLOR: '0', NO_COLOR: '1' } },
]) test.skipIf(process.platform === 'win32')(`actual PTY launcher carries placeholder styling into owned seed submission: ${entry.name}`, async () => {
const dir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'plan-seed-launcher-')));
const config = path.join(dir, '.claude'); fs.mkdirSync(config);
const script = path.join(dir, 'cli.ts'); fs.writeFileSync(script, `#!${process.execPath}\n${CLI}`, { mode: 0o700 });
@@ -93,7 +101,7 @@ for (const inheritedTerm of ['dumb', '', 'xterm-256color']) test.skipIf(process.
const launchedAt = Date.now(); let session: Awaited<ReturnType<typeof launchClaudePty>> | undefined;
try {
session = await launchClaudePty({ cwd: dir, observeScreen: true, permissionMode: 'plan', timeoutMs: 4000, model: 'fixture',
env: { CLAUDE_CONFIG_DIR: config, SEED_CASE: 'startup-terminal-placeholder-cursor', TERM: inheritedTerm } });
env: { CLAUDE_CONFIG_DIR: config, SEED_CASE: 'startup-terminal-placeholder-cursor', ...entry.env } });
const seed = '# Real launcher seed\nKeep this exact plan.';
await submitPlanSeed({...session, currentScreen: session.currentScreenFrame}, seed, { cwd: dir, launchedAt, deadlineAt: launchedAt + 2500,
isQuestionOrPermission: text => isProseAUQVisible(text) || isNumberedOptionListVisible(text) || isPermissionDialogVisible(text) });
@@ -101,6 +109,37 @@ for (const inheritedTerm of ['dumb', '', 'xterm-256color']) test.skipIf(process.
const events = fs.readFileSync(path.join(config, 'events.jsonl'), 'utf8').trim().split('\n').map(JSON.parse);
expect(events.map(e => e.kind)).toEqual(['paste', 'enter', 'end_turn', 'slash']);
expect(events.slice(0, 3).every(e => e.value === seed)).toBe(true);
const launch = JSON.parse(fs.readFileSync(path.join(config, 'launch.json'), 'utf8'));
expect(launch.terminalEnv.TERM).toBe('xterm-256color');
expect(launch.terminalEnv.FORCE_COLOR).toBe('1');
for (const key of ['CI', 'COLORTERM', 'NO_COLOR']) {
if (key in entry.env) expect(launch.terminalEnv[key]).toBe(entry.env[key]);
}
} finally {
try { await session?.close(); }
finally {
if (old === undefined) delete process.env.BROWSE_TERMINAL_BINARY; else process.env.BROWSE_TERMINAL_BINARY = old;
fs.rmSync(dir, { recursive: true, force: true });
}
}
}, 6000);
for (const terminalEnv of [
{ CI: '1', TERM: 'dumb', COLORTERM: '', FORCE_COLOR: '0', NO_COLOR: '1' },
{ CI: '1', TERM: 'xterm-256color', COLORTERM: 'truecolor', FORCE_COLOR: '3', NO_COLOR: '' },
]) test.skipIf(process.platform === 'win32')(`unobserved PTY preserves explicit terminal environment: ${terminalEnv.FORCE_COLOR}`, async () => {
const dir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'plan-seed-unobserved-')));
const config = path.join(dir, '.claude'); fs.mkdirSync(config);
const script = path.join(dir, 'cli.ts'); fs.writeFileSync(script, `#!${process.execPath}\n${CLI}`, { mode: 0o700 });
const old = process.env.BROWSE_TERMINAL_BINARY; process.env.BROWSE_TERMINAL_BINARY = script;
let session: Awaited<ReturnType<typeof launchClaudePty>> | undefined;
try {
session = await launchClaudePty({ cwd: dir, timeoutMs: 4000, model: 'fixture',
env: { CLAUDE_CONFIG_DIR: config, SEED_CASE: 'startup-terminal-placeholder-cursor', ...terminalEnv } });
await session.waitFor('❯', { timeoutMs: 2000 });
const launch = JSON.parse(fs.readFileSync(path.join(config, 'launch.json'), 'utf8'));
expect(launch.terminalEnv).toEqual(terminalEnv);
expect(fs.existsSync(path.join(config, 'events.jsonl'))).toBe(false);
} finally {
try { await session?.close(); }
finally {