mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-28 07:32:14 +02:00
v1.90.0.0 feat: make browser cookie imports explicit and safe (#2964)
* fix(browse): prepare reliable cookie import wave for validation * ci: sequence quality and behavior for validation branch * fix(browse): isolate Windows qualification and preserve native diagnostics * test(browse): cover cookie workflow quality and isolate Windows user paths * test(browse): trace native member startup and initialize fresh folders * fix(browse): keep Windows member stdin alive through EOF * fix(browse): latch native timeouts and compare contained Edge startup * test(browse): verify native version metadata and actual Windows argv * test(browse): qualify Dia import on isolated macOS CI * fix(browse): require picker origin for session mutations * fix(browse): bound credential reads through stream completion * test(browse): inspect owned Windows process arguments natively * test(evals): preserve passing coverage during cookie repair reruns * test(browse): isolate Dia qualification in a fresh macOS account * test(browse): pass bounded integer timeouts to native Mac probes * test(browse): distinguish Windows profile initialization from containment * test(browse): await descendant pipe readiness before parent exit * test(browse): initialize and restore isolated macOS Keychain state * test(browse): initialize Windows fixture folders before qualification * test(ci): pin the same Node runtime across Windows checks * test(browse): distinguish native macOS browser preflight stages * test(browse): isolate Windows descendant console lifetime * test(browse): preserve native receipts and identify fixture lock holders * test(browse): prepare dependency resolution before native Mac worker startup * test(ci): include lock and close checks in native diagnostics * test(browse): preserve native owner probe stages and subprocess deadlines * fix(browse): classify Chromium profile-in-use exit precisely * test(browse): retain Mac qualification evidence through cleanup failures * test(browse): bound Mac fixture paths and retire its owned user domain * test(browse): accept vanished fixture entries without weakening cleanup * test(browse): identify probe-created macOS user domains safely * test(browse): observe Mac user domains without targeting them first * test(browse): use passive fresh-user ownership throughout Mac qualification * test(browse): distinguish profile and registered-home Keychain lookups * test(browse): qualify Dia under one registered account home * test(browse): identify Dia startup and owned process-group failures * test(browse): classify bounded Dia startup diagnostics without leaking output * fix(test): preserve native Mac sandboxing and reap owned browser children * fix(browse): preserve Chromium sandboxing for native profile imports * test(browse): inspect signed Mach-O architecture without launching Xcode tools * test(browse): sample pending Dia startup and reap on all cleanup paths * test(browse): compare protected Dia launches in fresh Bun and Node accounts * test(browse): inspect isolated Mac GUI readiness without browser access * v1.90.0.0 fix: bind cookie picker actions to their document * test: validate cookie guards and fit nested launch fixtures * ci: configure the bundled Chromium sandbox helper * fix(browse): classify Playwright authentication timeouts * test: retain bounded Windows lifecycle diagnostics * test(cso): reuse bounded NTFS precision candidates * test(review): handle explicit preservation choices safely * test(browse): remove owned fixture directories with explicit primitives * test(review): distinguish descriptive reuse from edit commitments * test: admit only the approved unscored cookie workflow refusal * test: keep the Office Hours judge mock export-complete * fix: keep dependency-free CI planners independent of the model SDK * test: observe the exact holder after a native fixture unlink failure * fix: start seeded PTY observations at owned readiness * test: acquire identity-bound Windows deletion admission before profile resets * test: preserve qualified Git index bits without authorizing mutations
This commit is contained in:
Garry Tan
@@ -5,6 +5,7 @@ import * as path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { buildRunManifest, collectPaidTestFiles, type PaidRunManifest, type SliceResult } from '../scripts/test-paid-shards';
|
||||
import { STRICT_RETRY_CASE_BUDGETS } from './helpers/eval-budgets';
|
||||
import { manualReviewFixture } from './helpers/manual-judge-review-fixture';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
type Step = { uses?: string; run?: string; if?: string; with?: Record<string, unknown> };
|
||||
@@ -23,6 +24,48 @@ const workflows = ['evals.yml', 'evals-periodic.yml'].map(name => ({
|
||||
}));
|
||||
|
||||
describe('paid CI coordination stays off the eval image', () => {
|
||||
test('the actual planner and reporter load from a checkout without installed packages', () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'paid-offline-'));
|
||||
const listed = spawnSync('git', ['ls-files', '-z'], { cwd: ROOT, encoding: 'utf8', timeout: 5_000 });
|
||||
expect(listed.status, listed.stderr).toBe(0);
|
||||
try {
|
||||
for (const relative of listed.stdout.split('\0').filter(Boolean)) {
|
||||
const source = path.join(ROOT, relative);
|
||||
const destination = path.join(directory, relative);
|
||||
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
||||
expect(fs.realpathSync(path.dirname(destination)).startsWith(fs.realpathSync(directory) + path.sep)
|
||||
|| fs.realpathSync(path.dirname(destination)) === fs.realpathSync(directory)).toBe(true);
|
||||
if (fs.lstatSync(source).isSymbolicLink()) fs.symlinkSync(fs.readlinkSync(source), destination);
|
||||
else fs.copyFileSync(source, destination);
|
||||
}
|
||||
const env = { ...process.env, ANTHROPIC_API_KEY: undefined, EVALS: undefined, EVALS_ALL: undefined,
|
||||
EVALS_PROFILE: 'pr', EVALS_TIER: 'gate' };
|
||||
for (const args of [['init', '-b', 'main'], ['add', '-A'], ['commit', '-m', 'Seed offline planner fixture'], ['checkout', '-b', 'fixture-head']]) {
|
||||
const git = spawnSync('git', args, { cwd: directory, env, encoding: 'utf8', timeout: 15_000 });
|
||||
expect(git.status, git.stderr).toBe(0);
|
||||
}
|
||||
fs.appendFileSync(path.join(directory, '.github/workflows/evals.yml'), '\n');
|
||||
const report = path.join(directory, 'report');
|
||||
const plan = spawnSync(process.execPath, ['--no-install', 'run', 'scripts/test-paid-shards.ts', '--tier', 'gate',
|
||||
'--emit-plan', path.join(report, 'manifest.json'), '--slices', '6'], { cwd: directory, env, encoding: 'utf8', timeout: 15_000 });
|
||||
expect(plan.status, plan.stdout + plan.stderr).toBe(0);
|
||||
const manifest = JSON.parse(fs.readFileSync(path.join(report, 'manifest.json'), 'utf8'));
|
||||
expect(manifest.sliceCount).toBe(6);
|
||||
expect(manifest.entries.some((entry: any) => entry.status === 'planned')).toBe(true);
|
||||
const reconcile = spawnSync(process.execPath, ['--no-install', 'run', 'scripts/test-paid-shards.ts', '--tier', 'gate',
|
||||
'--report', report], { cwd: directory, env, encoding: 'utf8', timeout: 15_000 });
|
||||
expect(reconcile.status, reconcile.stdout + reconcile.stderr).toBe(1);
|
||||
expect(reconcile.stdout).toContain('report: 0/6 slices');
|
||||
expect(reconcile.stderr.match(/slice \d\/6 reported NO result/g)).toHaveLength(6);
|
||||
expect(reconcile.stderr).not.toContain('Cannot find module');
|
||||
expect(fs.existsSync(path.join(directory, 'node_modules'))).toBe(false);
|
||||
const provider = spawnSync(process.execPath, ['--no-install', '-e', 'import "./test/helpers/llm-judge.ts"'],
|
||||
{ cwd: directory, env, encoding: 'utf8', timeout: 15_000 });
|
||||
expect(provider.status).toBe(1);
|
||||
expect(provider.stderr).toContain("Cannot find module '@anthropic-ai/sdk'");
|
||||
} finally { fs.rmSync(directory, { recursive: true, force: true }); }
|
||||
}, 60_000);
|
||||
|
||||
for (const { name, jobs } of workflows) {
|
||||
test(`${name}: planning is independent of image startup and has no dependency install`, () => {
|
||||
const planner = jobs['plan-slices'];
|
||||
@@ -88,8 +131,18 @@ describe('dependency-free CI planner and report execution', () => {
|
||||
|
||||
beforeAll(() => {
|
||||
fixture = fs.mkdtempSync(path.join(os.tmpdir(), 'ci-paid-coordination-'));
|
||||
fs.cpSync(path.join(ROOT, 'scripts'), path.join(fixture, 'scripts'), { recursive: true });
|
||||
fs.cpSync(path.join(ROOT, 'test/helpers'), path.join(fixture, 'test/helpers'), { recursive: true });
|
||||
const sourceOnly = { recursive: true, filter: (file: string) => path.basename(file) !== 'node_modules' };
|
||||
fs.cpSync(path.join(ROOT, 'scripts'), path.join(fixture, 'scripts'), sourceOnly);
|
||||
fs.cpSync(path.join(ROOT, 'test/helpers'), path.join(fixture, 'test/helpers'), sourceOnly);
|
||||
expect(fs.readFileSync(path.join(fixture, 'test/helpers/llm-judge.ts'), 'utf8'))
|
||||
.toBe(fs.readFileSync(path.join(ROOT, 'test/helpers/llm-judge.ts'), 'utf8'));
|
||||
fs.cpSync(path.join(ROOT, 'lib'), path.join(fixture, 'lib'), sourceOnly);
|
||||
expect(fs.existsSync(path.join(fixture, 'lib/diagram-render/node_modules'))).toBe(false);
|
||||
fs.mkdirSync(path.join(fixture, '.github'), { recursive: true });
|
||||
for (const file of ['.github/cookie-workflow-manual-review.json', 'setup-browser-cookies/SKILL.md', 'BROWSER.md']) {
|
||||
fs.mkdirSync(path.dirname(path.join(fixture, file)), { recursive: true });
|
||||
fs.copyFileSync(path.join(ROOT, file), path.join(fixture, file));
|
||||
}
|
||||
for (const file of collectPaidTestFiles()) {
|
||||
fs.copyFileSync(path.join(ROOT, file), path.join(fixture, file));
|
||||
}
|
||||
@@ -183,7 +236,7 @@ describe('dependency-free CI planner and report execution', () => {
|
||||
const red = run(['--report', reportDir], tier);
|
||||
expect(red.status).toBe(1);
|
||||
expect(red.stderr).toContain(`${failed.outcomes[0].files[0]}: failed`);
|
||||
expect(red.stdout).toContain('3 executed, 0 reused; 1 passed, 2 failed (6 attempt records from 1 collectors)');
|
||||
expect(red.stdout).toContain('3 executed, 0 reused; 1 passed, 2 failed, 0 manual accepted (unscored; no score-cache credit) (6 attempt records from 1 collectors)');
|
||||
expect(red.stdout).toContain('3 cases with multiple attempts this run:');
|
||||
expect(red.stdout).not.toMatch(/passed only on retry|not blocking/);
|
||||
|
||||
@@ -192,4 +245,116 @@ describe('dependency-free CI planner and report execution', () => {
|
||||
expect(corrupt.status).toBe(1);
|
||||
});
|
||||
}
|
||||
|
||||
test('report verifies every manual claim against current source, preserves attempts, and never masks a failed shard', () => {
|
||||
const reportDir = path.join(fixture, 'manual-report');
|
||||
const manifestPath = path.join(reportDir, 'manifest.json');
|
||||
const planned = run(['--emit-plan', manifestPath, '--slices', '1'], 'gate');
|
||||
expect(planned.status, planned.stderr).toBe(0);
|
||||
const manifest: PaidRunManifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
|
||||
const slice: SliceResult = {
|
||||
version: 1, tier: 'gate', sliceIndex: 1, sliceCount: 1,
|
||||
outcomes: manifest.entries.filter(entry => entry.status === 'planned').map(entry => ({
|
||||
files: [entry.file], status: 'passed', exitCode: 0, elapsedMs: 1,
|
||||
executedTests: STRICT_RETRY_CASE_BUDGETS.find(budget => budget.file === entry.file)?.cases ?? 1,
|
||||
skippedTests: 0, ...(entry.budget ? { budget: entry.budget } : {}),
|
||||
})),
|
||||
};
|
||||
const slicePath = path.join(reportDir, 'slice-1.json');
|
||||
const collectorPath = path.join(reportDir, 'judge-results.json');
|
||||
const summaryPath = path.join(reportDir, 'collector-outcomes.json');
|
||||
const receipt = manualReviewFixture(ROOT);
|
||||
const write = (tests: unknown[]) => fs.writeFileSync(collectorPath, JSON.stringify({
|
||||
total_tests: tests.length, tier: 'llm-judge', shard: 1, total_cost_usd: 0,
|
||||
tests, flaky_retries: [{ name: receipt.name, attempts: tests.length }],
|
||||
}));
|
||||
fs.writeFileSync(slicePath, JSON.stringify(slice));
|
||||
write([{ ...receipt, passed: true }, receipt]);
|
||||
const historical = run(['--report', reportDir], 'gate');
|
||||
expect(historical.status).toBe(1);
|
||||
expect(historical.stderr).toContain('attempt 1: Malformed manual-review claim');
|
||||
expect(fs.existsSync(summaryPath)).toBe(false);
|
||||
|
||||
write([{ ...receipt, manual_review: { ...receipt.manual_review, refusal: {
|
||||
...receipt.manual_review!.refusal, response_id: '',
|
||||
} } }, receipt]);
|
||||
const malformed = run(['--report', reportDir], 'gate');
|
||||
expect(malformed.status).toBe(1);
|
||||
expect(malformed.stderr).toContain('attempt 1: Malformed manual-review claim');
|
||||
|
||||
write([{ ...receipt, manual_review: undefined, passed: false }, receipt]);
|
||||
const forgedFirstAttempt = run(['--report', reportDir], 'gate');
|
||||
expect(forgedFirstAttempt.status).toBe(1);
|
||||
expect(forgedFirstAttempt.stderr).toContain('attempt 2: manual review is only valid on the first case attempt');
|
||||
expect(fs.existsSync(summaryPath)).toBe(false);
|
||||
write([receipt, { ...receipt, attempt: 2 }]);
|
||||
const retriedManual = run(['--report', reportDir], 'gate');
|
||||
expect(retriedManual.status).toBe(1);
|
||||
expect(retriedManual.stderr).toContain('attempt 2: Malformed manual-review claim');
|
||||
expect(fs.existsSync(summaryPath)).toBe(false);
|
||||
|
||||
write([receipt]);
|
||||
const secondCollector = path.join(reportDir, 'other-results.json');
|
||||
fs.writeFileSync(secondCollector, JSON.stringify({ total_tests: 1, tests: [receipt] }));
|
||||
const duplicateCollector = run(['--report', reportDir], 'gate');
|
||||
expect(duplicateCollector.status).toBe(1);
|
||||
expect(duplicateCollector.stderr).toContain('duplicate manual-review claim');
|
||||
expect(fs.existsSync(summaryPath)).toBe(false);
|
||||
fs.rmSync(secondCollector);
|
||||
|
||||
write([receipt, { name: 'automated', suite: 'other', passed: true, execution: 'reused' }]);
|
||||
const clean = run(['--report', reportDir], 'gate');
|
||||
expect(clean.status, clean.stderr).toBe(0);
|
||||
expect(clean.stdout).toContain('1 passed, 0 failed, 1 manual accepted (unscored; no score-cache credit) (2 attempt records');
|
||||
const summary = JSON.parse(fs.readFileSync(summaryPath, 'utf8'));
|
||||
expect(summary.totals).toEqual({ executed: 1, reused: 1, passed: 1, failed: 0,
|
||||
manual_accepted: 1, attempts: 2, total: 2, flaky: 1 });
|
||||
expect(summary.files[0]).toMatchObject({ file: 'judge-results.json', total: 2, manual_accepted: 1, passed: 1 });
|
||||
expect(JSON.parse(fs.readFileSync(collectorPath, 'utf8')).tests[0]).toEqual(receipt);
|
||||
|
||||
const browserPath = path.join(fixture, 'BROWSER.md');
|
||||
const browserSource = fs.readFileSync(browserPath, 'utf8');
|
||||
fs.writeFileSync(browserPath, browserSource.replace('#### Choosing a source and checking sign-in',
|
||||
'#### Choosing a source and checking sign-in\nchanged approved source'));
|
||||
const sourceDrift = run(['--report', reportDir], 'gate');
|
||||
expect(sourceDrift.status).toBe(1);
|
||||
expect(sourceDrift.stderr).toContain('does not match current source and approval');
|
||||
expect(fs.existsSync(summaryPath)).toBe(false);
|
||||
fs.writeFileSync(browserPath, browserSource);
|
||||
|
||||
write([receipt, { name: 'unapproved', passed: false, execution: 'executed' }]);
|
||||
const failedCollector = run(['--report', reportDir], 'gate');
|
||||
expect(failedCollector.status).toBe(1);
|
||||
expect(failedCollector.stderr).toContain('1 unapproved final collector failure(s)');
|
||||
expect(JSON.parse(fs.readFileSync(summaryPath, 'utf8')).totals).toMatchObject({ failed: 1, manual_accepted: 1 });
|
||||
|
||||
write([receipt, { passed: true }]);
|
||||
const missingName = run(['--report', reportDir], 'gate');
|
||||
expect(missingName.status).toBe(1);
|
||||
expect(missingName.stderr).toContain('attempt 2: malformed collector entry (name/passed required)');
|
||||
expect(fs.existsSync(summaryPath)).toBe(false);
|
||||
write([receipt, { name: 'malformed', passed: 'true' }]);
|
||||
const malformedPassed = run(['--report', reportDir], 'gate');
|
||||
expect(malformedPassed.status).toBe(1);
|
||||
expect(malformedPassed.stderr).toContain('attempt 2: malformed collector entry (name/passed required)');
|
||||
expect(fs.existsSync(summaryPath)).toBe(false);
|
||||
|
||||
write([receipt, { name: 'automated', suite: 'other', passed: true, execution: 'reused' }]);
|
||||
|
||||
slice.outcomes[0].status = 'failed';
|
||||
slice.outcomes[0].exitCode = 1;
|
||||
fs.writeFileSync(slicePath, JSON.stringify(slice));
|
||||
const failedShard = run(['--report', reportDir], 'gate');
|
||||
expect(failedShard.status).toBe(1);
|
||||
expect(failedShard.stderr).toContain(`${slice.outcomes[0].files[0]}: failed`);
|
||||
expect(JSON.parse(fs.readFileSync(summaryPath, 'utf8')).totals.manual_accepted).toBe(1);
|
||||
|
||||
const stale = structuredClone(receipt);
|
||||
stale.manual_review!.approval.prompt_sha256 = '0'.repeat(64);
|
||||
write([stale]);
|
||||
const mismatched = run(['--report', reportDir], 'gate');
|
||||
expect(mismatched.status).toBe(1);
|
||||
expect(mismatched.stderr).toContain('attempt 1: Malformed manual-review claim');
|
||||
expect(fs.existsSync(summaryPath)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user