v1.91.7.0 feat: add functional QA and pre-publication docs checks (#2983)

* feat: add surface-aware exploratory QA and ship documentation gates

* test: preserve delegated QA setup authority after main integration

* fix(qa): clarify exploration order and preserve report artifacts

* test(qa): follow the shared setup reference directly

* refactor(ship): make verification and recovery routes explicit

* test(ship): align evidence and review guards with explicit routes

* fix(workflows): clarify ship recovery and functional QA evidence

* fix(workflows): clarify approval recovery and full QA coverage

* refactor(workflows): order review transactions and clarify ship state

* fix(ship): clarify final verification and fail closed at publication

* fix(evals): attribute native atomic documentation writes

* fix(ship): clarify recovery and documentation lifecycle guidance

* fix(test): preserve observed native placeholder styling in CI

* fix(codex): report watchdog timeouts without a process-exit race

* Checkpoint functional QA implementation and workflow validation repairs

* Fix documentation and shared-review fixture contracts

* docs: clarify judge reuse and evaluation supervision

* test: align review evidence and selected case contracts

* test: verify append-only documentation checkpoints and recovery

* fix: qualify QA workflows and CI validation repairs

* fix: launch shared-libs fixture scripts on Windows

* fix: qualify QA deadlines, fixture isolation, and shard cleanup

* fix: preserve qualified QA and cancellation repairs

* fix: enforce functional fixture authority and share strict event decoding

* fix: retain free-test evidence and explain recovery

* fix: reject malformed native evidence after decoder consolidation

* test: use reliable capture for telemetry privacy filters

* test: refresh measured quick coverage and document validation costs

* Fix native fixture receipts and preserve VM validation evidence

* Align negative judge controls with upstream clarity policy

* Fix report-only QA preparation and public evidence handling

* Clarify QA-only preparation and current-report preservation

* Stream Ship quality judgments with an explicit 64k response contract

* Validate compact judge reasoning locally with supported wire schema

* Align functional QA fixture instructions with evidence acceptance

* Bind native browser diagnostics to execution evidence and align review verdicts

* Preserve native diagnostic line boundaries

* Serialize functional QA evidence from native captures

* Keep large QA evidence fixture payload out of Windows argv
This commit is contained in:
Garry Tan authored and GitHub committed 2026-09-29 06:07:35 -07:00
1 parent 65bfb0ce49
commit dcaea52800
333 files changed
+41755 -7357

No files matched your search

+41 -1
View File
@@ -5,7 +5,7 @@ import * as os from 'node:os';
import * as path from 'node:path';
import { spawnSync } from 'node:child_process';
import { outsideVoiceCommand, outsideVoicePreflight, outsideVoiceInvocation } from '../scripts/resolvers/outside-voice';
import { generateCodexDocReview, generateCodexPlanReview } from '../scripts/resolvers/review';
import { generateAdversarialStep, generateCodexDocReview, generateCodexPlanReview } from '../scripts/resolvers/review';
import { validateOutsideReview } from '../lib/outside-review-result';
import { HOST_PATHS, type TemplateContext } from '../scripts/resolvers/types';
import { ALL_HOST_CONFIGS } from '../hosts';
@@ -14,6 +14,46 @@ const ROOT = path.resolve(import.meta.dir, '..');
const TEMP = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-outside-preflight-'));
afterAll(() => fs.rmSync(TEMP, { recursive: true, force: true }));
test('adversarial outside failures retain the required native pass without duplicate dispatch', () => {
for (const host of ALL_HOST_CONFIGS) {
for (const skillName of ['ship', 'review']) {
const ctx: TemplateContext = { host: host.name, skillName, tmplPath: `${skillName}/SKILL.md.tmpl`, paths: HOST_PATHS[host.name] };
const preflight = outsideVoicePreflight(ctx, { disabledBehavior: 'codex-only' });
expect(preflight).toMatch(/(?:do not dispatch a duplicate|without duplicating it)/);
expect(preflight).not.toMatch(/fall(?:ing)? back to (?:a|the) .*subagent/i);
const output = generateAdversarialStep(ctx);
expect(output).toContain('adversarial subagent (always runs)');
expect(output).toContain('For non-ready modes, retain the native pass above; do not dispatch it again.');
expect(output.match(/Retain the required native pass without duplicating it; it cannot complete outside coverage\./g)).toHaveLength(2);
expect(output).not.toContain("Use the caller's fallback");
expect(output).toContain('Only this optional outside adversarial pass is non-blocking');
expect(output).toContain('GATE: MISSING COVERAGE');
expect(outsideVoiceInvocation(ctx)).toContain("Use the caller's fallback; missing coverage is never clean/PASS.");
const disabled = outsideVoicePreflight(ctx, { disabledBehavior: 'skip-all' });
expect(disabled).toMatch(/(?:do NOT fall back|Disabled ends this entire extra review step)/);
}
}
});
test('ship design availability is an existing automatic choice, not a new opt-in', () => {
for (const host of ALL_HOST_CONFIGS) {
const ctx: TemplateContext = { host: host.name, skillName: 'ship', tmplPath: 'ship/SKILL.md.tmpl', paths: HOST_PATHS[host.name] };
const output = outsideVoicePreflight(ctx, { disabledBehavior: 'opt-in' });
expect(output).toContain('Ship attempts this optional design check automatically when frontend review applies');
expect(output).toContain('No additional opt-in is needed');
expect(output).toContain('Step 11 keeps its separate outside-review switch');
expect(output).toContain('`CODEX_MODE` reports provider availability, not user consent');
expect(output).not.toContain('Honor this caller’s existing opt-in/skip choice');
expect(output).not.toContain('This caller has its own opt-in/skip control');
const other = outsideVoicePreflight({ ...ctx, skillName: 'review' }, { disabledBehavior: 'opt-in' });
expect(other).toContain('Honor this caller’s existing opt-in/skip choice');
expect(other).not.toContain('No additional opt-in is needed');
expect(other).toContain('_OUTSIDE_CFG=enabled # This caller has its own opt-in/skip control.');
expect(output.match(/```bash\n([\s\S]*?)\n```/)![1]).toBe(other.match(/```bash\n([\s\S]*?)\n```/)![1].replace(
'_OUTSIDE_CFG=enabled # This caller has its own opt-in/skip control.', '_OUTSIDE_CFG=enabled'));
}
});
test('CEO and Eng describe the actual disabled route and completion validator', () => {
for (const host of ALL_HOST_CONFIGS) {
for (const skillName of ['plan-ceo-review', 'plan-eng-review']) {