mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-27 23:21:53 +02:00
Garry Tan
730a1017d1
* v1.89.1.0 fix: remove continuous checkpoint commits and repair validation blockers * fix: clarify shipping and engineering review recovery * fix: interpret native no-change review descriptions * test: separate descendant readiness from timeout delivery
282 lines
16 KiB
TypeScript
282 lines
16 KiB
TypeScript
/** Free regression coverage for the file bundle sent to workflow judges. */
|
|
import { afterEach, describe, expect, test } from 'bun:test';
|
|
import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { dirname, join, resolve } from 'node:path';
|
|
import { createHash } from 'node:crypto';
|
|
import { readWorkflowJudgeInput, buildWorkflowJudgePrompt } from './helpers/workflow-judge-input';
|
|
import { ENG_REVIEW_EXCERPT } from './helpers/workflow-excerpt';
|
|
|
|
const ROOT = resolve(import.meta.dir, '..');
|
|
const scratchRoots: string[] = [];
|
|
|
|
test('cache extraction preserves every byte of the original workflow request and rubric', () => {
|
|
// Captured from runWorkflowJudge's pre-cache template literal, not from the
|
|
// new builder: moving request construction must not change the paid metric.
|
|
const prompt = buildWorkflowJudgePrompt({ judgeContext: 'test workflow', judgeGoal: 'test goal' },
|
|
{ files: [], text: 'full prompt\nincluding lines' });
|
|
expect(createHash('sha256').update(prompt).digest('hex')).toBe('71cc9c777bf28ff0efd610259b411e3539852f83a0888fa0e92469331f8b9a43');
|
|
});
|
|
|
|
afterEach(() => {
|
|
for (const root of scratchRoots.splice(0)) rmSync(root, { recursive: true, force: true });
|
|
});
|
|
|
|
function fixture(files: Record<string, string>): string {
|
|
const root = mkdtempSync(join(tmpdir(), 'gstack-workflow-judge-'));
|
|
scratchRoots.push(root);
|
|
for (const [path, content] of Object.entries(files)) {
|
|
mkdirSync(dirname(join(root, path)), { recursive: true });
|
|
writeFileSync(join(root, path), content);
|
|
}
|
|
return root;
|
|
}
|
|
|
|
function occurrences(text: string, needle: string): number {
|
|
return text.split(needle).length - 1;
|
|
}
|
|
|
|
function sectionPaths(skill: string): string[] {
|
|
return readdirSync(join(ROOT, skill, 'sections'))
|
|
.filter(name => name.endsWith('.md'))
|
|
.sort()
|
|
.map(name => `${skill}/sections/${name}`);
|
|
}
|
|
|
|
describe('workflow judge file bundle', () => {
|
|
test('the actual Eng judge registration includes authorization and the referenced final gate', () => {
|
|
const source = readFileSync(join(ROOT, 'test/skill-llm-eval.test.ts'), 'utf8');
|
|
const registration = source.match(/testIfSelected\('plan-eng-review\/SKILL\.md sections',[\s\S]*?await runWorkflowJudge\(\{([\s\S]*?)\n \}\);/);
|
|
expect(registration).not.toBeNull();
|
|
const options = new Function('ENG_REVIEW_EXCERPT', `return ({${registration![1]}});`)(ENG_REVIEW_EXCERPT);
|
|
const input = readWorkflowJudgeInput({ root: ROOT, ...options });
|
|
const entry = input.files.find(file => file.kind === 'entrypoint')!;
|
|
const complete = readFileSync(join(ROOT, options.skillPath), 'utf8');
|
|
expect(entry.content).toBe(complete.slice(complete.indexOf('# Plan Review Mode')));
|
|
expect(entry.content).toContain('Do not build features, acceptance suites or benchmarks unless explicitly authorized');
|
|
expect(entry.content).toContain('## Scope gate');
|
|
expect(entry.content).toContain('## Section self-check');
|
|
expect(entry.content).toContain('## EXIT PLAN MODE GATE (BLOCKING)');
|
|
expect(entry.content.trimEnd()).toMatch(/After success telemetry and cache dispatch, call ExitPlanMode[^\n]*\.$/);
|
|
});
|
|
|
|
test('preserves the exact entrypoint excerpt and each complete section in sorted named files', () => {
|
|
const entrypoint = 'excluded preamble\n## Begin\nRead sections/z-last.md when directed.\n## End\nexcluded epilogue';
|
|
const sections = {
|
|
'example/sections/z-last.md': 'Z section prefix\nZ section suffix',
|
|
'example/sections/a-first.md': 'A section prefix\nA section suffix',
|
|
};
|
|
const root = fixture({
|
|
'example/SKILL.md': entrypoint,
|
|
...sections,
|
|
'example/sections/ignored.md.tmpl': 'DO NOT EVALUATE TEMPLATE',
|
|
'example/sections/manifest.json': '{"ignored": true}',
|
|
});
|
|
const input = readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Begin', endMarker: '## End' });
|
|
|
|
expect(input.files).toEqual([
|
|
{ path: 'example/SKILL.md', kind: 'entrypoint', content: '## Begin\nRead sections/z-last.md when directed.\n', startLine: 2, endLine: 3 },
|
|
{ path: 'example/sections/a-first.md', kind: 'section', content: sections['example/sections/a-first.md'], startLine: 1, endLine: 2 },
|
|
{ path: 'example/sections/z-last.md', kind: 'section', content: sections['example/sections/z-last.md'], startLine: 1, endLine: 2 },
|
|
]);
|
|
expect(input.text).not.toContain('excluded preamble');
|
|
expect(input.text).not.toContain('excluded epilogue');
|
|
expect(input.text).not.toContain('DO NOT EVALUATE TEMPLATE');
|
|
expect(input.text).not.toContain('manifest.json');
|
|
for (const file of input.files) {
|
|
expect(occurrences(input.text, file.content)).toBe(1);
|
|
const begin = input.text.split('\n').filter(line => line.includes('BEGIN FILE') && line.includes(file.path));
|
|
const end = input.text.split('\n').filter(line => line.includes('END FILE') && line.includes(file.path));
|
|
expect(begin).toHaveLength(1);
|
|
expect(end).toHaveLength(1);
|
|
expect(begin[0]).toContain(`${file.startLine}-${file.endLine}`);
|
|
}
|
|
});
|
|
|
|
test('describes lazy file loading without presenting bundle order as execution order', () => {
|
|
const root = fixture({
|
|
'example/SKILL.md': '## Begin\nSTOP and read sections/step.md.\n## End',
|
|
'example/sections/step.md': 'Run the separately loaded step.',
|
|
});
|
|
const { text } = readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Begin', endMarker: '## End' });
|
|
const header = text.slice(0, text.indexOf('BEGIN FILE'));
|
|
expect(header).toMatch(/bundle/i);
|
|
expect(header).toMatch(/file/i);
|
|
expect(header).toMatch(/separate|separately/i);
|
|
expect(header).toMatch(/lazy|on-demand/i);
|
|
expect(header).toMatch(/directives|instructions/i);
|
|
expect(header).toMatch(/(?:not|isn't|does not)[^.\n]*execution order/i);
|
|
expect(text).toContain('STOP and read sections/step.md.');
|
|
});
|
|
|
|
test('retains section prelude and suffix exactly once when both markers are inside a section', () => {
|
|
// Generated comments made the old 120-character prefix heuristic append
|
|
// this whole file after its partial slice, duplicating every review pass.
|
|
const section = [
|
|
'<!-- AUTO-GENERATED from review-sections.md.tmpl — do not edit directly -->',
|
|
'<!-- Regenerate: bun run gen:skill-docs -->',
|
|
'section context before the selected marker',
|
|
'## Review Sections',
|
|
'### Pass 1: Information Architecture',
|
|
'Evaluate the first pass.',
|
|
'## CRITICAL RULE',
|
|
'section context after the selected marker',
|
|
].join('\n');
|
|
const root = fixture({
|
|
'example/SKILL.md': 'Entrypoint preamble.\nSTOP and read sections/review-sections.md.',
|
|
'example/sections/review-sections.md': section,
|
|
});
|
|
const input = readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Review Sections', endMarker: '## CRITICAL RULE' });
|
|
|
|
expect(input.files).toEqual([
|
|
{ path: 'example/sections/review-sections.md', kind: 'section', content: section, startLine: 1, endLine: 8 },
|
|
]);
|
|
expect(occurrences(input.text, '### Pass 1: Information Architecture')).toBe(1);
|
|
expect(occurrences(input.text, 'section context before the selected marker')).toBe(1);
|
|
expect(occurrences(input.text, 'section context after the selected marker')).toBe(1);
|
|
expect(occurrences(input.text, section)).toBe(1);
|
|
expect(input.text).not.toContain('Entrypoint preamble.');
|
|
});
|
|
|
|
test('handles marker windows spanning files without losing section prefixes or suffixes', () => {
|
|
const root = fixture({
|
|
'example/SKILL.md': 'omitted\n## Begin\nEntrypoint tail',
|
|
'example/sections/a.md': 'First section prefix\nFirst section body\nFirst section suffix',
|
|
'example/sections/b.md': 'Second section prefix\n## End\nSecond section suffix',
|
|
});
|
|
const input = readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Begin', endMarker: '## End' });
|
|
|
|
expect(input.files[0]).toEqual({
|
|
path: 'example/SKILL.md', kind: 'entrypoint', content: '## Begin\nEntrypoint tail', startLine: 2, endLine: 3,
|
|
});
|
|
expect(input.files.slice(1).map(file => file.content)).toEqual([
|
|
'First section prefix\nFirst section body\nFirst section suffix',
|
|
'Second section prefix\n## End\nSecond section suffix',
|
|
]);
|
|
expect(occurrences(input.text, 'Second section prefix')).toBe(1);
|
|
expect(occurrences(input.text, 'Second section suffix')).toBe(1);
|
|
});
|
|
|
|
test('keeps separate files even when their generated preludes and contents match', () => {
|
|
const body = '<!-- AUTO-GENERATED -->\nShared instruction';
|
|
const root = fixture({
|
|
'example/SKILL.md': '## Begin\nRead both sections.\n## End',
|
|
'example/sections/a.md': body,
|
|
'example/sections/b.md': body,
|
|
});
|
|
const input = readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Begin', endMarker: '## End' });
|
|
expect(input.files.filter(file => file.kind === 'section').map(file => file.path)).toEqual([
|
|
'example/sections/a.md', 'example/sections/b.md',
|
|
]);
|
|
expect(occurrences(input.text, body)).toBe(2);
|
|
});
|
|
|
|
test('supports an uncarved workflow and an open-ended slice', () => {
|
|
const root = fixture({ 'example/SKILL.md': 'omitted\n## Begin\nRetain this final line' });
|
|
const input = readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Begin', endMarker: null });
|
|
expect(input.files).toEqual([
|
|
{ path: 'example/SKILL.md', kind: 'entrypoint', content: '## Begin\nRetain this final line', startLine: 2, endLine: 3 },
|
|
]);
|
|
});
|
|
|
|
test('rejects missing start markers, missing end markers, and end markers preceding the start', () => {
|
|
const root = fixture({ 'example/SKILL.md': '## Earlier\n## Begin\nBody' });
|
|
expect(() => readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Missing', endMarker: null })).toThrow(/Start marker not found/);
|
|
expect(() => readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Begin', endMarker: '## Missing' })).toThrow(/End marker not found/);
|
|
expect(() => readWorkflowJudgeInput({ root, skillPath: 'example/SKILL.md', startMarker: '## Begin', endMarker: '## Earlier' })).toThrow(/End marker not found/);
|
|
});
|
|
|
|
test('generated ship includes base-branch initialization and every lazy section once', () => {
|
|
const skillPath = 'ship/SKILL.md';
|
|
const source = readFileSync(join(ROOT, skillPath), 'utf8');
|
|
// Bind to the paid caller's actual slice and retain both the opening contract
|
|
// and initialization, regardless of their ordering in the authored workflow.
|
|
const caller = readFileSync(join(ROOT, 'test/skill-llm-eval.test.ts'), 'utf8');
|
|
const markers = caller.match(/skillPath: 'ship\/SKILL\.md',\s+startMarker: '([^']+)',\s+endMarker: '([^']+)'/);
|
|
expect(markers).not.toBeNull();
|
|
const [, startMarker, endMarker] = markers!;
|
|
expect(startMarker).toBe('# Ship:');
|
|
const input = readWorkflowJudgeInput({ root: ROOT, skillPath, startMarker, endMarker });
|
|
const entrypoint = input.files.find(file => file.kind === 'entrypoint');
|
|
expect(entrypoint?.content).toBe(source.slice(source.indexOf(startMarker), source.indexOf(endMarker, source.indexOf(startMarker))));
|
|
expect(entrypoint?.content).toContain('git remote get-url origin');
|
|
expect(entrypoint?.content).toContain('**Follow every STOP and AskUserQuestion gate**');
|
|
expect(entrypoint?.content).toContain('## Step 0: Detect platform and base branch');
|
|
expect(entrypoint?.content).toContain('gh pr view --json baseRefName');
|
|
expect(entrypoint?.content).toContain('Print the detected base branch name.');
|
|
expect(occurrences(input.text, startMarker)).toBe(1);
|
|
const sections = input.files.filter(file => file.kind === 'section');
|
|
expect(sections.map(file => file.path)).toEqual(sectionPaths('ship'));
|
|
for (const file of sections) {
|
|
const source = readFileSync(join(ROOT, file.path), 'utf8');
|
|
expect(file.content).toBe(source);
|
|
expect(occurrences(input.text, source)).toBe(1);
|
|
expect(file.startLine).toBe(1);
|
|
}
|
|
});
|
|
|
|
test('generated engineering review includes the scope choices and readiness probe its steps reference', () => {
|
|
const skillPath = 'plan-eng-review/SKILL.md';
|
|
const caller = readFileSync(join(ROOT, 'test/skill-llm-eval.test.ts'), 'utf8');
|
|
const registration = caller.match(/testIfSelected\('plan-eng-review\/SKILL\.md sections',[\s\S]*?await runWorkflowJudge\(\{([\s\S]*?)\n \}\);/);
|
|
expect(registration).not.toBeNull();
|
|
const options = new Function('ENG_REVIEW_EXCERPT', `return ({${registration![1]}});`)(ENG_REVIEW_EXCERPT);
|
|
expect(ENG_REVIEW_EXCERPT.skillPath).toBe(skillPath);
|
|
const input = readWorkflowJudgeInput({ root: ROOT, ...options });
|
|
const entrypoint = input.files.find(file => file.kind === 'entrypoint')!;
|
|
expect(entrypoint.content).toContain('B) A plan or design doc');
|
|
expect(entrypoint.content).toContain('## Scope gate');
|
|
expect(entrypoint.content.indexOf('## Scope gate')).toBeLessThan(entrypoint.content.indexOf('### Step 0: Scope Challenge'));
|
|
expect(entrypoint.content).toContain('## Web research runs in Aside');
|
|
expect(entrypoint.content).toContain('echo "READY: aside');
|
|
expect(input.text.indexOf('echo "READY: aside')).toBeLessThan(input.text.indexOf('- **Search check:**'));
|
|
expect(entrypoint.content).not.toContain('- **Search check:**');
|
|
expect(occurrences(input.text, '- **Search check:**')).toBe(1);
|
|
expect(occurrences(input.text, '## Scope gate')).toBe(1);
|
|
expect(occurrences(input.text, '### 1. Architecture review')).toBe(1);
|
|
const sections = input.files.filter(file => file.kind === 'section');
|
|
expect(sections.map(file => file.path)).toEqual(sectionPaths('plan-eng-review'));
|
|
for (const file of sections) {
|
|
const source = readFileSync(join(ROOT, file.path), 'utf8');
|
|
expect(file.content).toBe(source);
|
|
expect(occurrences(input.text, source)).toBe(1);
|
|
}
|
|
});
|
|
|
|
test('generated design consultation includes the prechecks its proposal and preview reference', () => {
|
|
const skillPath = 'design-consultation/SKILL.md';
|
|
const caller = readFileSync(join(ROOT, 'test/skill-llm-eval.test.ts'), 'utf8');
|
|
const markers = caller.match(/skillPath: 'design-consultation\/SKILL\.md',\s+startMarker: '([^']+)',\s+endMarker: '([^']+)'/);
|
|
expect(markers).not.toBeNull();
|
|
const [, startMarker, endMarker] = markers!;
|
|
const input = readWorkflowJudgeInput({ root: ROOT, skillPath, startMarker, endMarker });
|
|
const entrypoint = input.files.find(file => file.kind === 'entrypoint')!;
|
|
for (const prerequisite of ['## Phase 0: Pre-checks', 'DESIGN_MD_FORMAT:', 'DESIGN_READY', 'DESIGN_NOT_AVAILABLE']) {
|
|
expect(entrypoint.content).toContain(prerequisite);
|
|
}
|
|
expect(entrypoint.content.indexOf('## Phase 0:')).toBeLessThan(entrypoint.content.indexOf('## Phase 1:'));
|
|
expect(occurrences(input.text, '## Phase 0: Pre-checks')).toBe(1);
|
|
const sections = input.files.filter(file => file.kind === 'section');
|
|
expect(sections.map(file => file.path)).toEqual(sectionPaths('design-consultation'));
|
|
for (const file of sections) {
|
|
const source = readFileSync(join(ROOT, file.path), 'utf8');
|
|
expect(file.content).toBe(source);
|
|
expect(occurrences(input.text, source)).toBe(1);
|
|
}
|
|
});
|
|
|
|
test('generated plan-design passes retain their full section without duplicating Pass 1', () => {
|
|
const input = readWorkflowJudgeInput({
|
|
root: ROOT, skillPath: 'plan-design-review/SKILL.md', startMarker: '## Review Sections', endMarker: '## CRITICAL RULE',
|
|
});
|
|
expect(input.files.filter(file => file.kind === 'entrypoint')).toHaveLength(0);
|
|
expect(input.files.map(file => file.path)).toEqual(sectionPaths('plan-design-review'));
|
|
const section = input.files.find(file => file.path === 'plan-design-review/sections/review-sections.md');
|
|
expect(section?.content).toBe(readFileSync(join(ROOT, 'plan-design-review/sections/review-sections.md'), 'utf8'));
|
|
expect(section?.content).toStartWith('<!-- AUTO-GENERATED');
|
|
expect(section?.content).toContain('## CRITICAL RULE');
|
|
expect(section?.content).toContain('## Formatting Rules');
|
|
expect(occurrences(input.text, '### Pass 1: Information Architecture')).toBe(1);
|
|
});
|
|
});
|