CalvinBackup/gstack
1
0 Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-10-03 01:46:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

v1.91.7.0 feat: add functional QA and pre-publication docs checks (#2983)

Browse Source 
* feat: add surface-aware exploratory QA and ship documentation gates

* test: preserve delegated QA setup authority after main integration

* fix(qa): clarify exploration order and preserve report artifacts

* test(qa): follow the shared setup reference directly

* refactor(ship): make verification and recovery routes explicit

* test(ship): align evidence and review guards with explicit routes

* fix(workflows): clarify ship recovery and functional QA evidence

* fix(workflows): clarify approval recovery and full QA coverage

* refactor(workflows): order review transactions and clarify ship state

* fix(ship): clarify final verification and fail closed at publication

* fix(evals): attribute native atomic documentation writes

* fix(ship): clarify recovery and documentation lifecycle guidance

* fix(test): preserve observed native placeholder styling in CI

* fix(codex): report watchdog timeouts without a process-exit race

* Checkpoint functional QA implementation and workflow validation repairs

* Fix documentation and shared-review fixture contracts

* docs: clarify judge reuse and evaluation supervision

* test: align review evidence and selected case contracts

* test: verify append-only documentation checkpoints and recovery

* fix: qualify QA workflows and CI validation repairs

* fix: launch shared-libs fixture scripts on Windows

* fix: qualify QA deadlines, fixture isolation, and shard cleanup

* fix: preserve qualified QA and cancellation repairs

* fix: enforce functional fixture authority and share strict event decoding

* fix: retain free-test evidence and explain recovery

* fix: reject malformed native evidence after decoder consolidation

* test: use reliable capture for telemetry privacy filters

* test: refresh measured quick coverage and document validation costs

* Fix native fixture receipts and preserve VM validation evidence

* Align negative judge controls with upstream clarity policy

* Fix report-only QA preparation and public evidence handling

* Clarify QA-only preparation and current-report preservation

* Stream Ship quality judgments with an explicit 64k response contract

* Validate compact judge reasoning locally with supported wire schema

* Align functional QA fixture instructions with evidence acceptance

* Bind native browser diagnostics to execution evidence and align review verdicts

* Preserve native diagnostic line boundaries

* Serialize functional QA evidence from native captures

* Keep large QA evidence fixture payload out of Windows argv
This commit is contained in:
Garry Tan authored and GitHub committed 2026-09-29 06:07:35 -07:00
1 parent 65bfb0ce49
commit dcaea52800
333 files changed
+41755 -7357
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

test/skill-e2e-shared-libs.test.ts
+33 -9
View File
@@ -4,7 +4,7 @@ import * as fs from 'node:fs';
import * as path from 'node:path';
import { createHash } from 'node:crypto';
import { sharedLibsFingerprint } from '../lib/review-evidence';
import { hasTrustedReviewStartRead } from './helpers/shared-libs-review-start-evidence';
import { hasTrustedReviewStartRead, hasTrustedSharedLibsCheck } from './helpers/shared-libs-review-start-evidence';
import { CAPTURE_LONG_MS } from './helpers/eval-budgets';
import { describeE2ETier, e2eTierEnabled } from './helpers/e2e-gate';
import { EvalCollector } from './helpers/eval-store';
@@ -16,8 +16,11 @@ import {
reviewRecords, runSharedCapture, runSharedInteractive, seedOpportunitySources,
seedReviewSources, seedSkippedAdvisory, snapshotFixture, specialistFixture,
sharedReadOnlyViolations, standaloneInstructions, toolCommandTrace, type SharedLibsFixture,
SharedCaptureAccumulator, type SharedCaptureAttempt,
SharedCaptureAccumulator, type SharedCaptureAttempt, SHARED_LIBS_ROOT,
} from './helpers/shared-libs-eval-fixture';
import {
seedPathReviewPrerequisites, checkPathReviewPrerequisites, hasPathReviewPrerequisiteReceipt, createLifecyclePrerequisiteActor,
} from './helpers/shared-libs-path-fixture';
const describeE2E = describeE2ETier('gate');
const collector = e2eTierEnabled('gate') ? new EvalCollector('e2e') : null;
@@ -42,7 +45,9 @@ async function recordCapture(attempt: SharedCaptureAttempt, scenario: string, na
duration_ms: result?.duration ?? result?.durationMs ?? 0,
cost_usd: result?.costEstimate?.estimatedCost ?? result?.costUsd ?? 0,
model: result?.model, turns_used: result?.costEstimate?.turnsUsed ?? result?.turnsUsed ?? 0,
transcript: [...(result?.transcript ?? result?.events ?? []), { fixture_requests: result?.providerRequests ?? [] }], output: result?.output ?? '',
transcript: [...(result?.transcript ?? result?.events ?? []), { fixture_requests: result?.providerRequests ?? [],
...(result?.fixtureStageReceipts ? { prerequisite_receipts: result.fixtureStageReceipts } : {}),
...(result?.fixturePrerequisiteSource ? { prerequisite_source: result.fixturePrerequisiteSource, prerequisite_native_coverage: false } : {}) }], output: result?.output ?? '',
error: [failure ? String(failure) : '', result?.costKnown === false
? 'No terminal billing event; actual cost is unknown. Raw usage is retained in the transcript.' : ''].filter(Boolean).join('\n') || undefined,
exit_reason: result?.exitReason ?? 'capture_threw' });
@@ -104,7 +109,7 @@ describeE2E('Shared-code safety and review lifecycle (gate)', () => {
const instructions = standaloneInstructions(f);
const before = snapshotFixture(f.root);
await recordCapture(attempt, 'audit', 'shared-libs-read-only', () => runSharedCapture(f, 'shared-libs-read-only',
`Run /deslop-shared-libs for this repository using ${instructions}. Include relevant uncommitted source in your audit. Return the skill's report in conversation.`), result => {
`Run /deslop-shared-libs for this repository using ${instructions}. Include relevant uncommitted source in your audit. Return the skill's report in conversation.`, attempt), result => {
assertReadOnly(f, before, result);
expect(result.output).toMatch(/uncommitted|overlay|raw/i);
expect(result.output).toContain(f.tip.slice(0, 7));
@@ -127,7 +132,7 @@ describeE2E('Shared-code safety and review lifecycle (gate)', () => {
const instructions = standaloneInstructions(f);
const before = snapshotFixture(f.root);
await recordCapture(attempt, 'audit', 'shared-libs-unsupported-git', () => runSharedCapture(f, 'shared-libs-unsupported-git',
`Run /deslop-shared-libs for this repository using ${instructions}. Return the review report.`), result => {
`Run /deslop-shared-libs for this repository using ${instructions}. Return the review report.`, attempt), result => {
assertReadOnly(f, before, result);
expect(result.output).toMatch(/unavailable|unsupported|cannot|could not|coverage|limited/i);
const calls = readRequests(f).filter(row => row.tool === 'git' && !isInternalClaudeGitRequest(row, toolCommandTrace(result)));
@@ -150,12 +155,14 @@ describeE2E('Shared-code safety and review lifecycle (gate)', () => {
seedReviewSources(f);
const instructions = reviewLifecycleInstructions(f);
const input = specialistFixture(f);
const stageActor = createLifecyclePrerequisiteActor(f);
let questions: any[] = [];
await recordCapture(attempt, choose, 'shared-libs-review-lifecycle', async () => {
const capture = await runSharedInteractive(f, 'shared-libs-review-lifecycle', reviewPrompt(f, instructions, input), choose);
const capture = await runSharedInteractive(f, 'shared-libs-review-lifecycle', reviewPrompt(f, instructions, input, stageActor), choose, { stageActor, attempt });
questions = capture.questions;
return capture.result;
}, result => {
expect(stageActor.verify(result.events ?? []), 'consume a current invoked synthetic stage result before final persistence').toBe(true);
expect(questions.length).toBeGreaterThan(0);
const worker = fs.readFileSync(path.join(f.repo, 'src/retry-worker.ts'), 'utf8');
expect(worker).not.toContain('unusedRetryDiagnostic');
@@ -219,12 +226,17 @@ describeE2E('Shared-code safety and review lifecycle (gate)', () => {
const input = path.join(f.root, 'current-advisory.jsonl');
const { action: _priorAction, ...current } = prior;
fs.writeFileSync(input, JSON.stringify({ ...current, specialist: 'maintainability' }) + '\n');
const resumed = seedPathReviewPrerequisites(f);
const prerequisites = checkPathReviewPrerequisites(f, resumed.input);
expect(prerequisites.settled).toBe(true);
let questions: any[] = [];
await recordCapture(attempt, change, 'shared-libs-review-revalidation', async () => {
const capture = await runSharedInteractive(f, 'shared-libs-review-revalidation', reviewRevalidationPrompt(f, instructions, input), 'skip');
const capture = await runSharedInteractive(f, 'shared-libs-review-revalidation', reviewRevalidationPrompt(f, instructions, input, resumed), 'skip', { attempt, prerequisiteSource: 'synthetic-fixture-input' });
questions = capture.questions;
return capture.result;
}, result => {
expect(checkPathReviewPrerequisites(f, resumed.input)).toEqual(prerequisites);
expect(hasPathReviewPrerequisiteReceipt(result.events ?? [], resumed.checkCommand, prerequisites)).toBe(true);
if (change === 'unchanged') expect(questions.length).toBe(0);
else expect(questions.length).toBeGreaterThan(0);
const trace = toolCommandTrace(result).join('\n');
@@ -235,7 +247,19 @@ describeE2E('Shared-code safety and review lifecycle (gate)', () => {
const rows = reviewRecords(f).filter(row => row.skill === 'review');
expect(rows.length).toBeGreaterThanOrEqual(2);
const last = rows.at(-1);
if (change === 'unchanged' || change === 'filtered') {
if (trace.includes('--check-shared-libs')) {
expect(last).toMatchObject({ completed: true, converged: true, review_binding: { state: 'verified' } });
expect(hasTrustedSharedLibsCheck(result.events ?? result.transcript ?? [], {
helper: path.join(SHARED_LIBS_ROOT, 'bin/gstack-review-log'),
repo: f.repo, directory: path.join(f.state, 'projects/fixture-shared-libs/.review-starts'),
state: f.state, slug: 'fixture-shared-libs',
branch: fixtureGit(f, 'symbolic-ref', '--quiet', '--short', 'HEAD'), wtree: fixtureWorkingTree(f),
startedAt: last.review_binding.started_at, finding: current, reusable: change === 'unchanged',
coveredPaths: change === 'unchanged' ? current.evidence_paths
: last.findings.find((finding: any) => finding.advisory && finding.action === 'skipped'
&& finding.fingerprint === current.fingerprint)?.snapshot_covered_paths,
})).toBe(true);
} else if (change === 'unchanged' || change === 'filtered') {
// A true hash comparison cannot substitute for the trusted capture and path checks.
expect(hasTrustedReviewStartRead(result.events ?? result.transcript ?? [], {
repo: f.repo, directory: path.join(f.state, 'projects/fixture-shared-libs/.review-starts'),
@@ -245,8 +269,8 @@ describeE2E('Shared-code safety and review lifecycle (gate)', () => {
})).toBe(true);
expect(trace).toContain('check-attr');
expect(trace).toMatch(/ls-files[^\n]*(?:--stage|-s\b)|lstat|stat\s|test\s+-L|\[\s+-L/);
if (change === 'unchanged') expect(trace).toContain('canReuseSharedLibsAdvisory');
}
if (change === 'unchanged') expect(trace).toContain('canReuseSharedLibsAdvisory');
expect(last.review_binding.branch_id).toBe(createHash('sha256').update(change === 'branch' ? 'feature-a' : 'feature/a').digest('hex'));
if (change === 'unchanged') {
expect((last.findings || []).some((finding: any) => finding.advisory)).toBe(false);