v1.91.7.0 feat: add functional QA and pre-publication docs checks (#2983)

* feat: add surface-aware exploratory QA and ship documentation gates

* test: preserve delegated QA setup authority after main integration

* fix(qa): clarify exploration order and preserve report artifacts

* test(qa): follow the shared setup reference directly

* refactor(ship): make verification and recovery routes explicit

* test(ship): align evidence and review guards with explicit routes

* fix(workflows): clarify ship recovery and functional QA evidence

* fix(workflows): clarify approval recovery and full QA coverage

* refactor(workflows): order review transactions and clarify ship state

* fix(ship): clarify final verification and fail closed at publication

* fix(evals): attribute native atomic documentation writes

* fix(ship): clarify recovery and documentation lifecycle guidance

* fix(test): preserve observed native placeholder styling in CI

* fix(codex): report watchdog timeouts without a process-exit race

* Checkpoint functional QA implementation and workflow validation repairs

* Fix documentation and shared-review fixture contracts

* docs: clarify judge reuse and evaluation supervision

* test: align review evidence and selected case contracts

* test: verify append-only documentation checkpoints and recovery

* fix: qualify QA workflows and CI validation repairs

* fix: launch shared-libs fixture scripts on Windows

* fix: qualify QA deadlines, fixture isolation, and shard cleanup

* fix: preserve qualified QA and cancellation repairs

* fix: enforce functional fixture authority and share strict event decoding

* fix: retain free-test evidence and explain recovery

* fix: reject malformed native evidence after decoder consolidation

* test: use reliable capture for telemetry privacy filters

* test: refresh measured quick coverage and document validation costs

* Fix native fixture receipts and preserve VM validation evidence

* Align negative judge controls with upstream clarity policy

* Fix report-only QA preparation and public evidence handling

* Clarify QA-only preparation and current-report preservation

* Stream Ship quality judgments with an explicit 64k response contract

* Validate compact judge reasoning locally with supported wire schema

* Align functional QA fixture instructions with evidence acceptance

* Bind native browser diagnostics to execution evidence and align review verdicts

* Preserve native diagnostic line boundaries

* Serialize functional QA evidence from native captures

* Keep large QA evidence fixture payload out of Windows argv
This commit is contained in:
Garry Tan authored and GitHub committed 2026-09-29 06:07:35 -07:00
1 parent 65bfb0ce49
commit dcaea52800
333 files changed
+41755 -7357

No files matched your search

+23 -4
View File
@@ -8,10 +8,18 @@ export const PR_PROFILE_CASE_IDS = [
'hermetic-canary', 'hermetic-sentinel',
'browse-basic', 'browse-snapshot', 'skillmd-setup-discovery',
'qa-bootstrap', 'review-sql-injection', 'review-coverage-audit',
'qa-functional-cli-report', 'qa-functional-webhook-report',
'qa-functional-cli-fix', 'qa-functional-webhook-fix',
'review-exploratory-small-cli', 'ship-exploratory-small-cli', 'ship-exploratory-unavailable',
'ship-exploratory-plan-checks', 'ship-exploratory-late-input',
'plan-ceo-review-benefits', 'plan-eng-coverage-audit', 'plan-review-report',
'auq-format-gate', 'plan-design-review-no-ui-scope', 'office-hours-spec-review',
'tpa-present', 'tpa-absent-linux',
'ship-local-workflow', 'ship-coverage-audit', 'docsync-spawned',
'ship-docsync', 'ship-docsync-completion', 'ship-docsync-current', 'ship-docsync-failure', 'ship-docsync-store',
'ship-docsync-missing-marker', 'ship-docsync-missing-asset', 'ship-docsync-launch-failure',
'ship-docsync-timeout-unsettled', 'ship-docsync-late-result', 'ship-docsync-stale-before',
'ship-docsync-stale-after', 'ship-docsync-recovery',
'ship-managed-hook-refresh', 'ship-unmanaged-hook-consent', 'ship-local-hook-preservation',
'setup-deploy-workflow', 'context-restore-loads-latest', 'plan-tune-inspect',
'skillify-provenance-refusal', 'diagram-triplet', 'learnings-show',
@@ -26,6 +34,9 @@ export const PR_PROFILE_FILES: Record<string, readonly string[]> = {
'test/skill-e2e-hermetic-canary.test.ts': ['hermetic-canary', 'hermetic-sentinel'],
'test/skill-e2e-bws.test.ts': ['browse-basic', 'browse-snapshot', 'skillmd-setup-discovery'],
'test/skill-e2e-qa-workflow.test.ts': ['qa-bootstrap'],
'test/skill-e2e-qa-functional.test.ts': ['qa-functional-cli-report', 'qa-functional-webhook-report'],
'test/skill-e2e-qa-functional-fix.test.ts': ['qa-functional-cli-fix', 'qa-functional-webhook-fix'],
'test/skill-e2e-qa-callers.test.ts': ['review-exploratory-small-cli', 'ship-exploratory-small-cli', 'ship-exploratory-unavailable', 'ship-exploratory-plan-checks', 'ship-exploratory-late-input'],
'test/skill-e2e-review.test.ts': ['review-sql-injection'],
'test/skill-e2e-coverage-audit.test.ts': ['review-coverage-audit', 'plan-eng-coverage-audit'],
'test/skill-e2e-plan.test.ts': ['plan-ceo-review-benefits', 'plan-review-report', 'office-hours-spec-review'],
@@ -36,6 +47,7 @@ export const PR_PROFILE_FILES: Record<string, readonly string[]> = {
'test/skill-e2e-ship-hook-refresh.test.ts': ['ship-managed-hook-refresh'],
'test/skill-e2e-ship-hook-consent.test.ts': ['ship-unmanaged-hook-consent', 'ship-local-hook-preservation'],
'test/skill-e2e-docsync-spawned.test.ts': ['docsync-spawned'],
'test/skill-e2e-ship-docsync.test.ts': ['ship-docsync', 'ship-docsync-completion', 'ship-docsync-current', 'ship-docsync-failure', 'ship-docsync-store', 'ship-docsync-missing-marker', 'ship-docsync-missing-asset', 'ship-docsync-launch-failure', 'ship-docsync-timeout-unsettled', 'ship-docsync-late-result', 'ship-docsync-stale-before', 'ship-docsync-stale-after', 'ship-docsync-recovery'],
'test/skill-e2e-deploy.test.ts': ['setup-deploy-workflow'],
'test/skill-e2e-session-intelligence.test.ts': ['context-restore-loads-latest'],
'test/skill-e2e-plan-tune.test.ts': ['plan-tune-inspect'],
@@ -103,6 +115,11 @@ export const FREE_ONLY_PR_FILES = [
'test/helpers/auq-parallel-worker.ts',
] as const;
const FULL_GATE_PR_FILES = [
'package.json', 'bun.lock', '.github/docker/Dockerfile.ci',
'scripts/host-config.ts', 'scripts/discover-skills.ts', 'hosts/index.ts',
] as const;
function knownNonBehaviorFile(file: string): boolean {
// A mapped dependency still wins over these exemptions. New helper/fixture,
// runtime, dependency, or workflow files are deliberately not exempted.
@@ -157,7 +174,8 @@ export function selectPrProfile(options: {
])];
const unknownFiles = files.filter(file => file !== TOUCHFILES_DATA_PATH
&& !depends(file, dependencyPatterns) && !knownNonBehaviorFile(file));
const fallback = unknownFiles.length > 0;
const sharedInputs = files.filter(file => depends(file, FULL_GATE_PR_FILES));
const fallback = unknownFiles.length > 0 || sharedInputs.length > 0;
const candidates = fallback ? Object.keys(maps.e2eTouchfiles).sort() : selectedE2E;
const e2e = candidates.filter(id => maps.tiers[id] === 'gate' && (fallback || profile.includes(id)));
const judges = fallback ? Object.keys(maps.judgeTouchfiles).sort() : selectedJudges;
@@ -177,9 +195,10 @@ export function selectPrProfile(options: {
const deferredPromptFiles = noQuickCoverage.filter(file => !hasQuickDependency(file)
&& Object.values(maps.e2eTouchfiles).some(patterns => depends(file, patterns)));
const missingCoverage = noQuickCoverage.filter(file => !deferredPromptFiles.includes(file));
const reasons = fallback
? [`Unknown dependencies restore every gate case and judge: ${unknownFiles.join(', ')}`]
: ['Changed-input selection intersected with the fast PR profile; selected judges retained'];
const reasons: string[] = [];
if (unknownFiles.length) reasons.push(`Unknown dependencies restore every gate case and judge: ${unknownFiles.join(', ')}`);
if (sharedInputs.length) reasons.push(`Shared runtime/build inputs restore every gate case and judge: ${sharedInputs.join(', ')}`);
if (!fallback) reasons.push('Changed-input selection intersected with the fast PR profile; selected judges retained');
if (deferred.length) reasons.push(`${deferred.length} selected behaviors remain scheduled/release coverage, not PR passes`);
if (deferredPromptFiles.length) reasons.push(`No quick live coverage; known broad prompt checks deferred: ${deferredPromptFiles.join(', ')}`);
if (missingCoverage.length) reasons.push(`Full validation required for prompts without a relevant PR check: ${missingCoverage.join(', ')}`);