mirror of
https://github.com/garrytan/gstack.git
synced 2026-10-31 14:06:50 +01:00
Garry Tanandcapy-ai[bot]
f592d67826
* fix(evals): three PR-gate background reds were a detector, an interface and a helper defect - docsync-fault-eval: a conditional mention (until/unless/once/when/before/if) of 'Documentation is current' is not a false current claim; replay corpus for ship-docsync-launch-failure from PR run 38336400566 with derived refusals. - gstack-skill-start/gstack-skill-end: --help prints usage and exits before any side effect (it used to start a session); the docsync interface admits the lifecycle helpers' own --help; captured run 38969924945 replays clean. - qa deadline policy: listing the owned qa-reports/screenshots directory is bookkeeping (qa-only-no-fix, run 38969924945). Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test(evals): the launch-failure corpus is a row in the docsync corpus test that owns docsync-fault-eval Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test(evals): register the launch-failure corpus and docsync-store capture in their contracts The B1 corpus contract wants its own detector-corpus-<case> replay test, so the launch-failure corpus keeps one, listed in the test-of-test baseline beside its sibling corpora; FREE_FIXTURES names both new fixtures' free consumers. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * fix(review-army): the scope block prints the SCOPE_* signals specialist selection reads The Detect stack and scope block sourced gstack-diff-scope silently, so an agent that needed SCOPE_FRONTEND to pick specialists appended its own echo and left the declared caller interface (ship-exploratory-plan-checks red in run 38981523623). The block now lists the sourced signals itself. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * v1.91.111.0 fix(evals): four PR-gate background reds were a detector, an interface, a helper and a workflow-block defect Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
50 lines
2.9 KiB
TypeScript
50 lines
2.9 KiB
TypeScript
import { describe, expect, test } from 'bun:test';
|
|
import * as fs from 'node:fs';
|
|
import * as os from 'node:os';
|
|
import * as path from 'node:path';
|
|
import { docsActorVerdict } from './helpers/docsync-fault-eval';
|
|
import type { DocsActorState } from './helpers/docsync-fault-actor';
|
|
|
|
const CASE = 'ship-docsync-launch-failure';
|
|
const DIR = path.join(import.meta.dir, 'fixtures/detector-corpora', CASE);
|
|
const ROOT = path.join(os.tmpdir(), 'docsync-launch-failure-corpus');
|
|
const names = fs.readdirSync(DIR).filter(name => name.endsWith('.json') && name !== 'inventory.json').sort();
|
|
const entries = names.map(name => ({ name, raw: fs.readFileSync(path.join(DIR, name), 'utf8') }))
|
|
.map(({ name, raw }) => ({ name, size: Buffer.byteLength(raw), entry: JSON.parse(raw.replaceAll('<ROOT>', JSON.stringify(ROOT).slice(1, -1))) }));
|
|
const inventory = JSON.parse(fs.readFileSync(path.join(DIR, 'inventory.json'), 'utf8'));
|
|
|
|
describe(`${CASE} detector corpus`, () => {
|
|
test('entries are well-formed, bounded, redacted and inventoried', () => {
|
|
expect(entries.some(({ entry }) => entry.expected === 'pass')).toBe(true);
|
|
expect(entries.some(({ entry }) => entry.expected === 'refuse')).toBe(true);
|
|
for (const { name, size, entry } of entries) {
|
|
expect(size, name).toBeLessThanOrEqual(64 * 1024);
|
|
expect(entry.schema).toBe('gstack-detector-corpus/v1');
|
|
expect(entry.case).toBe(CASE);
|
|
expect(entry.detector).toBe('test/helpers/docsync-fault-eval.ts#docsActorVerdict');
|
|
expect(entry.source.sha).toMatch(/^[0-9a-f]{40}$/);
|
|
expect(['passed', 'failed']).toContain(entry.census_outcome);
|
|
expect(['pass', 'refuse']).toContain(entry.expected);
|
|
expect(entry.justification.length).toBeGreaterThan(0);
|
|
expect(JSON.stringify(entry), name).not.toMatch(/\/home\/runner|\/Users\/|sk-[A-Za-z0-9]{8}|ghp_[A-Za-z0-9]{8}/);
|
|
if (entry.derived) {
|
|
const origin = entries.find(other => !other.entry.derived && other.entry.source.run === entry.source.run);
|
|
expect(origin?.entry.expected, `${name} derives from a capture the detector accepts`).toBe('pass');
|
|
} else {
|
|
expect(name).toBe(`${entry.source.run}-t1-${entry.census_outcome === 'passed' ? 'pass' : 'fail'}.json`);
|
|
expect(inventory.find((row: { entry: string | null }) => row.entry === name)?.outcome).toBe(entry.census_outcome);
|
|
}
|
|
}
|
|
for (const row of inventory.filter((row: { entry: string | null }) => row.entry)) expect(names).toContain(row.entry);
|
|
for (const row of inventory.filter((row: { entry: string | null }) => !row.entry)) expect(row.reason).toBeTruthy();
|
|
});
|
|
|
|
for (const { name, entry } of entries) {
|
|
test(`${name} → ${entry.expected}`, () => {
|
|
const failures = docsActorVerdict(entry.input.state as DocsActorState, entry.input.report, entry.input.published);
|
|
if (entry.expected === 'pass') expect(failures).toEqual([]);
|
|
else expect(failures.length).toBeGreaterThan(0);
|
|
});
|
|
}
|
|
});
|