mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-02 17:40:02 +02:00
* feat: add surface-aware exploratory QA and ship documentation gates * test: preserve delegated QA setup authority after main integration * fix(qa): clarify exploration order and preserve report artifacts * test(qa): follow the shared setup reference directly * refactor(ship): make verification and recovery routes explicit * test(ship): align evidence and review guards with explicit routes * fix(workflows): clarify ship recovery and functional QA evidence * fix(workflows): clarify approval recovery and full QA coverage * refactor(workflows): order review transactions and clarify ship state * fix(ship): clarify final verification and fail closed at publication * fix(evals): attribute native atomic documentation writes * fix(ship): clarify recovery and documentation lifecycle guidance * fix(test): preserve observed native placeholder styling in CI * fix(codex): report watchdog timeouts without a process-exit race * Checkpoint functional QA implementation and workflow validation repairs * Fix documentation and shared-review fixture contracts * docs: clarify judge reuse and evaluation supervision * test: align review evidence and selected case contracts * test: verify append-only documentation checkpoints and recovery * fix: qualify QA workflows and CI validation repairs * fix: launch shared-libs fixture scripts on Windows * fix: qualify QA deadlines, fixture isolation, and shard cleanup * fix: preserve qualified QA and cancellation repairs * fix: enforce functional fixture authority and share strict event decoding * fix: retain free-test evidence and explain recovery * fix: reject malformed native evidence after decoder consolidation * test: use reliable capture for telemetry privacy filters * test: refresh measured quick coverage and document validation costs * Fix native fixture receipts and preserve VM validation evidence * Align negative judge controls with upstream clarity policy * Fix report-only QA preparation and public evidence handling * Clarify QA-only preparation and current-report preservation * Stream Ship quality judgments with an explicit 64k response contract * Validate compact judge reasoning locally with supported wire schema * Align functional QA fixture instructions with evidence acceptance * Bind native browser diagnostics to execution evidence and align review verdicts * Preserve native diagnostic line boundaries * Serialize functional QA evidence from native captures * Keep large QA evidence fixture payload out of Windows argv
263 lines
12 KiB
TypeScript
263 lines
12 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
|
import { execFileSync, spawnSync } from 'node:child_process';
|
|
import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { join, resolve } from 'node:path';
|
|
import { sharedLibsSnapshotCoverage } from '../lib/review-evidence';
|
|
import { gitArgvIn } from './helpers/scratch-repo';
|
|
|
|
const root = resolve(import.meta.dir, '..');
|
|
let fixture: string;
|
|
let repo: string;
|
|
let state: string;
|
|
let env: NodeJS.ProcessEnv;
|
|
|
|
function git(...args: string[]) {
|
|
const result = gitArgvIn(repo, args, 10_000);
|
|
if (result.status !== 0 || result.error) throw new Error(result.error?.message ?? result.stderr.toString());
|
|
return result.stdout.toString().trim();
|
|
}
|
|
|
|
function cli(args: string[], input?: unknown) {
|
|
const result = spawnSync(join(root, 'bin/gstack-review-log'), args, {
|
|
cwd: repo, env, input: input === undefined ? undefined : JSON.stringify(input),
|
|
encoding: 'utf8', timeout: 15_000,
|
|
});
|
|
expect(result.status, result.stderr).toBe(0);
|
|
return result.stdout.trim();
|
|
}
|
|
|
|
function finding() {
|
|
return { advisory: true, severity: 'INFORMATIONAL', action: 'skipped', category: 'shared-libs',
|
|
evidence_paths: ['src/a.ts', 'src/b.ts'], helper_target: { path: 'src/shared.ts', symbol: 'parse' } };
|
|
}
|
|
|
|
function records() {
|
|
const output = execFileSync(join(root, 'bin/gstack-review-read'), [], {
|
|
cwd: repo, env, encoding: 'utf8', timeout: 15_000,
|
|
});
|
|
return output.split('---CONFIG---')[0].trim().split('\n').filter(line => line.startsWith('{')).map(line => JSON.parse(line));
|
|
}
|
|
|
|
function save(value = finding(), token = cli(['--start', 'review'])) {
|
|
cli([JSON.stringify({ skill: 'review', status: 'clean', completed: true, converged: true,
|
|
findings: [value] }), '--finish', token]);
|
|
return records().at(-1);
|
|
}
|
|
|
|
function check(value = finding(), token = cli(['--start', 'review'])) {
|
|
return JSON.parse(cli(['--check-shared-libs', token], value));
|
|
}
|
|
|
|
beforeEach(() => {
|
|
fixture = mkdtempSync(join(tmpdir(), 'reuse-'));
|
|
repo = join(fixture, 'repo');
|
|
state = join(fixture, 'state');
|
|
mkdirSync(join(repo, 'src'), { recursive: true });
|
|
env = { ...process.env, GSTACK_HOME: state, GIT_CONFIG_NOSYSTEM: '1' };
|
|
git('init', '-q', '-b', 'main');
|
|
git('config', 'commit.gpgsign', 'false');
|
|
git('config', 'core.autocrlf', 'false');
|
|
writeFileSync(join(repo, 'src/a.ts'), 'export const a = 1;\n');
|
|
writeFileSync(join(repo, 'src/b.ts'), 'export const b = 1;\n');
|
|
git('add', '.');
|
|
git('commit', '-qm', 'initial');
|
|
});
|
|
|
|
afterEach(() => { if (fixture) rmSync(fixture, { recursive: true, force: true }); });
|
|
|
|
describe('executable shared-code evidence checks', () => {
|
|
test('logger computes identity and coverage without caller-supplied proof', () => {
|
|
const row = save();
|
|
expect(row.findings[0].fingerprint).toMatch(/^shared-libs:[a-f0-9]{64}$/);
|
|
expect(row.findings[0].snapshot_covered_paths).toEqual(finding().evidence_paths);
|
|
expect(row.review_binding.state).toBe('verified');
|
|
});
|
|
|
|
test('checker reads real history and the unconsumed start receipt', () => {
|
|
save();
|
|
const token = cli(['--start', 'review']);
|
|
const result = check(finding(), token);
|
|
expect(result.reusable).toBe(true);
|
|
expect(result.review_start).toMatchObject({ skill: 'review', repo, branch: 'main', wtree: result.snapshot.wtree });
|
|
expect(result.snapshot.covered_paths).toEqual(finding().evidence_paths);
|
|
expect(save(finding(), token).review_binding.state).toBe('verified');
|
|
expect(check(finding(), token).reusable).toBe(false);
|
|
});
|
|
|
|
test('a supplied snapshot cannot invent a prior decision', () => {
|
|
expect(check({ ...finding(), currentSnapshot: { covered_paths: finding().evidence_paths },
|
|
priorReview: { completed: true, converged: true }, reusable: true } as any).reusable).toBe(false);
|
|
expect(check(finding(), '../not-a-token').reusable).toBe(false);
|
|
});
|
|
|
|
test('a changed secondary caller or a same-content different branch cannot reuse', () => {
|
|
save();
|
|
git('checkout', '-qb', 'feature/a');
|
|
save();
|
|
git('checkout', '-qb', 'feature-a');
|
|
expect(check().reusable).toBe(false);
|
|
git('checkout', '-q', 'feature/a');
|
|
const token = cli(['--start', 'review']);
|
|
writeFileSync(join(repo, 'src/b.ts'), 'export const b = 2;\n');
|
|
expect(check(finding(), token).reusable).toBe(false);
|
|
});
|
|
|
|
test('ordinary untracked authored paths are covered without modifying the real index', () => {
|
|
writeFileSync(join(repo, 'src/new.ts'), 'export const n = 3;\n');
|
|
const index = readFileSync(join(repo, '.git/index'));
|
|
const value = { ...finding(), evidence_paths: ['src/a.ts', 'src/new.ts'] };
|
|
expect(save(value).findings[0].snapshot_covered_paths).toEqual(value.evidence_paths);
|
|
expect(check(value).reusable).toBe(true);
|
|
expect(readFileSync(join(repo, '.git/index'))).toEqual(index);
|
|
});
|
|
|
|
for (const flag of ['--assume-unchanged', '--skip-worktree']) {
|
|
test(`${flag} cannot be forged into snapshot coverage`, () => {
|
|
save();
|
|
git('update-index', flag, 'src/b.ts');
|
|
const value = { ...finding(), snapshot_covered_paths: finding().evidence_paths };
|
|
expect(save(value).findings[0].snapshot_covered_paths).not.toContain('src/b.ts');
|
|
writeFileSync(join(repo, 'src/b.ts'), 'export const hidden = true;\n');
|
|
expect(check(value).reusable).toBe(false);
|
|
});
|
|
}
|
|
|
|
for (const attribute of ['filter=collapse', 'working-tree-encoding=UTF-8', 'ident', 'text', 'eol=lf', 'crlf', 'crlf=input']) {
|
|
test(`${attribute} is ineligible even when the normalized tree is unchanged`, () => {
|
|
save();
|
|
mkdirSync(join(repo, '.git/info'), { recursive: true });
|
|
writeFileSync(join(repo, '.git/info/attributes'), `src/b.ts ${attribute}\n`);
|
|
const value = { ...finding(), snapshot_covered_paths: finding().evidence_paths };
|
|
expect(save(value).findings[0].snapshot_covered_paths).not.toContain('src/b.ts');
|
|
expect(check(value).reusable).toBe(false);
|
|
});
|
|
}
|
|
|
|
test('configured line conversion fails closed, including supplied coverage', () => {
|
|
save();
|
|
git('config', 'core.autocrlf', 'input');
|
|
expect(check().reusable).toBe(false);
|
|
expect(save({ ...finding(), snapshot_covered_paths: finding().evidence_paths } as any)
|
|
.findings[0].snapshot_covered_paths).toEqual([]);
|
|
});
|
|
|
|
test('ignored tracked files, symlinks and missing paths cannot receive coverage', () => {
|
|
writeFileSync(join(repo, '.git/info/exclude'), 'src/b.ts\n');
|
|
expect(save().findings[0].snapshot_covered_paths).toEqual(['src/a.ts']);
|
|
writeFileSync(join(repo, '.git/info/exclude'), '');
|
|
rmSync(join(repo, 'src/b.ts'));
|
|
symlinkSync('a.ts', join(repo, 'src/b.ts'));
|
|
expect(save().findings[0].snapshot_covered_paths).toEqual(['src/a.ts']);
|
|
rmSync(join(repo, 'src/b.ts'));
|
|
expect(save().findings[0].snapshot_covered_paths).toEqual(['src/a.ts']);
|
|
});
|
|
|
|
test('a symlink ancestor does not read an external caller', () => {
|
|
mkdirSync(join(fixture, 'outside'));
|
|
writeFileSync(join(fixture, 'outside/caller.ts'), 'outside\n');
|
|
symlinkSync(join(fixture, 'outside'), join(repo, 'external'));
|
|
const value = { ...finding(), evidence_paths: ['src/a.ts', 'external/caller.ts'] };
|
|
expect(save(value).findings[0].snapshot_covered_paths).toEqual(['src/a.ts']);
|
|
expect(check(value).reusable).toBe(false);
|
|
});
|
|
|
|
test('literal spaces, Unicode and pathspec-looking names remain exact', () => {
|
|
const paths = ['src/a.ts', 'src/é file.ts', 'src/[x].ts', 'src/:special.ts'];
|
|
for (const path of paths.slice(1)) writeFileSync(join(repo, path), 'export {};\n');
|
|
const value = { ...finding(), evidence_paths: paths };
|
|
expect(save(value).findings[0].snapshot_covered_paths).toEqual(paths);
|
|
expect(check(value).reusable).toBe(true);
|
|
});
|
|
|
|
test('inspection cannot execute fsmonitor hooks or mutate the current start', () => {
|
|
save();
|
|
const marker = join(fixture, 'hook-ran');
|
|
const hook = join(fixture, 'monitor');
|
|
writeFileSync(hook, `#!/bin/sh\ntouch '${marker}'\n`);
|
|
chmodSync(hook, 0o700);
|
|
git('config', 'core.fsmonitor', hook);
|
|
const token = cli(['--start', 'review']);
|
|
expect(check(finding(), token).reusable).toBe(true);
|
|
expect(existsSync(marker)).toBe(false);
|
|
});
|
|
|
|
test('raw bytes must equal the captured blob, not merely an index entry', () => {
|
|
const tree = save().wtree;
|
|
writeFileSync(join(repo, 'src/b.ts'), 'export const changed = true;\n');
|
|
expect(sharedLibsSnapshotCoverage(repo, tree, finding().evidence_paths, env)).toEqual(['src/a.ts']);
|
|
});
|
|
|
|
test('removing a transform cannot manufacture prior coverage', () => {
|
|
writeFileSync(join(repo, '.git/info/attributes'), 'src/b.ts text\n');
|
|
const row = save();
|
|
expect(row.findings[0].snapshot_covered_paths).toEqual(['src/a.ts']);
|
|
writeFileSync(join(repo, '.git/info/attributes'), '');
|
|
const result = check();
|
|
expect(result.snapshot.wtree).toBe(row.wtree);
|
|
expect(result.snapshot.covered_paths).toEqual(finding().evidence_paths);
|
|
expect(result.reusable).toBe(false);
|
|
});
|
|
|
|
test('legacy history without recorded coverage is never enough', () => {
|
|
save();
|
|
const logs = [...new Bun.Glob('**/*-reviews.jsonl').scanSync({ cwd: state, absolute: true })];
|
|
expect(logs).toHaveLength(1);
|
|
const row = JSON.parse(readFileSync(logs[0], 'utf8'));
|
|
delete row.findings[0].snapshot_covered_paths;
|
|
writeFileSync(logs[0], JSON.stringify(row) + '\n');
|
|
expect(check().reusable).toBe(false);
|
|
});
|
|
|
|
test('legacy caller-supplied coverage cannot become logger-owned proof', () => {
|
|
save();
|
|
const file = [...new Bun.Glob('**/*-reviews.jsonl').scanSync({ cwd: state, absolute: true })][0];
|
|
const row = JSON.parse(readFileSync(file, 'utf8'));
|
|
expect(row.findings[0].snapshot_covered_paths).toEqual(finding().evidence_paths);
|
|
delete row.shared_libs_coverage_version;
|
|
writeFileSync(file, JSON.stringify(row) + '\n');
|
|
expect(check().reusable).toBe(false);
|
|
row.shared_libs_coverage_version = 999;
|
|
writeFileSync(file, JSON.stringify(row) + '\n');
|
|
expect(check().reusable).toBe(false);
|
|
});
|
|
|
|
test('the logger replaces supplied proof versions with its computed format', () => {
|
|
const token = cli(['--start', 'review']);
|
|
cli([JSON.stringify({ skill: 'review', status: 'clean', completed: true, converged: true,
|
|
shared_libs_coverage_version: 999, findings: [finding()] }), '--finish', token]);
|
|
expect(records().at(-1).shared_libs_coverage_version).toBe(1);
|
|
expect(check().reusable).toBe(true);
|
|
});
|
|
|
|
test('gitlinks do not cover a nested repository caller', () => {
|
|
const nested = join(repo, 'vendor');
|
|
mkdirSync(nested);
|
|
git('-C', nested, 'init', '-q');
|
|
writeFileSync(join(nested, 'caller.ts'), 'export {};\n');
|
|
git('-C', nested, 'add', '.');
|
|
git('-C', nested, '-c', 'commit.gpgsign=false', 'commit', '-qm', 'nested');
|
|
git('add', 'vendor');
|
|
const value = { ...finding(), evidence_paths: ['src/a.ts', 'vendor/caller.ts'] };
|
|
expect(save(value).findings[0].snapshot_covered_paths).toEqual(['src/a.ts']);
|
|
expect(check(value).reusable).toBe(false);
|
|
});
|
|
|
|
test('partial clones are ineligible without fetching missing objects', () => {
|
|
save();
|
|
git('config', 'remote.origin.promisor', 'true');
|
|
expect(check().reusable).toBe(false);
|
|
});
|
|
|
|
test('nonconverged passes and real defects cannot receive reusable advisory coverage', () => {
|
|
const token = cli(['--start', 'review']);
|
|
cli([JSON.stringify({ skill: 'review', status: 'issues_found', completed: true, converged: false,
|
|
findings: [{ ...finding(), snapshot_covered_paths: finding().evidence_paths }] }), '--finish', token]);
|
|
expect(records().at(-1).findings[0].snapshot_covered_paths).toEqual([]);
|
|
expect(check().reusable).toBe(false);
|
|
save();
|
|
expect(check({ ...finding(), advisory: false }).reusable).toBe(false);
|
|
expect(check({ ...finding(), severity: 'CRITICAL' }).reusable).toBe(false);
|
|
});
|
|
});
|