mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-12 19:44:48 +01:00
test(W7h): delete the codex-offering prompt builder left unused by the retirement
Dead-code follow-up to the codex-offered-* retirement: buildCodexOfferingPrompt (test/helpers/codex-offering-fixture.ts) had no caller besides those cases. Deletes the helper, its free test (test/codex-offering-fixture.test.ts), the two captured public fixtures it replayed, its scripts/free-test-durations.json entry, and the helper/fixture paths still listed as touchfiles of six plan review E2Es in test/helpers/touchfiles-data.ts. Retirement card: - test: codex-offering-fixture.test.ts (pinned the builder's prompt text and replayed captured timeout/CDD transcripts against it). - detects: changes to a prompt that no paid case sends any more. - non_test_callers: none (grep -rn codex-offering test scripts is empty). - replacement: none needed; the behavior it supported is now the free probe check in gen-skill-docs.test.ts. - validation: bun test test/touchfiles.test.ts test/touchfiles-map-diff.test.ts test/paid-shards.test.ts test/e2e-tier-alignment.test.ts; bun run typecheck.
This commit is contained in:
garrytan committed
1 parent
fc9049a8b5
commit
ca395095a1
6 files changed
+6
-476
No files matched your search
@@ -312,7 +312,6 @@
|
||||
"test/codex-generation-model.test.ts": 60,
|
||||
"test/codex-hardening.test.ts": 1675,
|
||||
"test/codex-model-probe.test.ts": 263,
|
||||
"test/codex-offering-fixture.test.ts": 36,
|
||||
"test/codex-resume-flag-semantics.test.ts": 14,
|
||||
"test/codex-session-lifecycle.test.ts": 12041,
|
||||
"test/codex-under-codex-detection.test.ts": 35,
|
||||
|
||||
@@ -1,124 +0,0 @@
|
||||
import { afterEach, describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { buildCodexOfferingPrompt, codexOfferingSources } from './helpers/codex-offering-fixture';
|
||||
import captured from './fixtures/codex-offering-cdd-public.json';
|
||||
import timedOut from './fixtures/codex-offering-timeout-public.json';
|
||||
|
||||
const roots: string[] = [];
|
||||
afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); });
|
||||
function fixture() {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gstack-offering-source-'));
|
||||
roots.push(root);
|
||||
fs.mkdirSync(path.join(root, 'work/plan-ceo-review/sections'), { recursive: true });
|
||||
fs.writeFileSync(path.join(root, 'work/plan-ceo-review/SKILL.md'), '# CEO\nRead sections/review-sections.md for the outside voice.\n');
|
||||
fs.writeFileSync(path.join(root, 'work/plan-ceo-review/sections/review-sections.md'), '# Outside Voice\nUse the real integration documented here.\n');
|
||||
return { root, work: path.join(root, 'work'), skill: path.join(root, 'work/plan-ceo-review') };
|
||||
}
|
||||
|
||||
describe('Codex offering source lookup', () => {
|
||||
test('captured timeouts wrote oversized summaries successfully but never completed execution', () => {
|
||||
expect(timedOut.cases.map(row => row.attempt)).toEqual([1, 2]);
|
||||
for (const row of timedOut.cases) {
|
||||
expect(row.exitReason).toBe('timeout');
|
||||
expect(row.timeoutMs).toBe(120_000);
|
||||
expect(row.tools.length).toBeLessThan(row.maxTurns);
|
||||
expect(row.tools.every(tool => tool.acknowledged && !tool.isError)).toBe(true);
|
||||
expect(row.tools.some(tool => !['Read', 'Bash', 'Write'].includes(tool.name))).toBe(false);
|
||||
expect(row.summaryWords).toBeGreaterThan(600);
|
||||
expect(row.tools.at(-1)?.name).toBe('Write');
|
||||
expect(row.writeSeconds * 1000).toBeLessThan(row.timeoutMs);
|
||||
expect(row.writeAcknowledged).toBe(true);
|
||||
expect(row.providerFinalResult).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
test('bounded summaries retain all five audit questions and require a terminal response after writing', () => {
|
||||
const root = path.resolve(import.meta.dir, '..');
|
||||
for (const skill of ['office-hours', 'plan-ceo-review', 'plan-design-review', 'plan-eng-review']) {
|
||||
const prompt = buildCodexOfferingPrompt({ root, skill, featureName: 'outside voice', summaryPath: '/tmp/offering-summary.md' });
|
||||
expect(prompt.split('\n').filter(line => /^\d\. /.test(line))).toEqual([
|
||||
'1. How is Codex availability checked? (what exact bash command?)',
|
||||
'2. How is the user prompted? (via AskUserQuestion? what are the options?)',
|
||||
'3. What happens when Codex is NOT available? (fallback to subagent? skip entirely?)',
|
||||
'4. Is this step blocking (gates the workflow) or optional (can be skipped)?',
|
||||
'5. What prompt/context is sent to Codex?',
|
||||
]);
|
||||
expect(prompt).toContain('read the relevant complete sections');
|
||||
expect(prompt).toContain('identify anything they do not document');
|
||||
expect(prompt).toContain('source file/line citations, at most 600 words total');
|
||||
expect(prompt).toContain('Answer every question and cover its relevant branches');
|
||||
expect(prompt).toContain('Preserve the exact availability command');
|
||||
expect(prompt).toContain('instead of copying whole blocks');
|
||||
expect(prompt.indexOf('After the Write succeeds')).toBeGreaterThan(prompt.indexOf('Write your summary to'));
|
||||
expect(prompt).toContain('finish with one sentence naming the saved path');
|
||||
expect(prompt).toContain('Do not repeat the audit in your final response');
|
||||
}
|
||||
});
|
||||
|
||||
test('captured failures used all eight tool calls before writing, finding carved evidence late', () => {
|
||||
expect(captured.cases.map(row => row.attempt)).toEqual([1, 2]);
|
||||
for (const row of captured.cases) {
|
||||
expect(row.exitReason).toBe('error_max_turns');
|
||||
expect(row.tools).toHaveLength(row.maxTurns);
|
||||
expect(row.tools.every(tool => ['Read', 'Bash'].includes(tool.name))).toBe(true);
|
||||
expect(row.tools.slice(0, 5).filter(tool => tool.name === 'Read')
|
||||
.every(tool => tool.input.file_path?.endsWith('/plan-ceo-review/SKILL.md'))).toBe(true);
|
||||
const carvedRead = row.tools.findIndex(tool => tool.name === 'Read'
|
||||
&& tool.input.file_path?.endsWith('/plan-ceo-review/sections/review-sections.md'));
|
||||
expect(carvedRead).toBeGreaterThanOrEqual(6);
|
||||
}
|
||||
});
|
||||
|
||||
test('the real prompt builder lists the complete generated target sources before the audit questions', () => {
|
||||
const root = path.resolve(import.meta.dir, '..');
|
||||
for (const skill of ['office-hours', 'plan-ceo-review', 'plan-design-review', 'plan-eng-review']) {
|
||||
const sources = codexOfferingSources(root, skill);
|
||||
const generated = fs.readdirSync(path.join(root, skill, 'sections')).filter(file => file.endsWith('.md')).sort();
|
||||
expect(sources).toEqual([`${skill}/SKILL.md`, ...generated.map(file => `${skill}/sections/${file}`)]);
|
||||
const prompt = buildCodexOfferingPrompt({ root, skill, featureName: 'outside voice', summaryPath: '/tmp/offering-summary.md' });
|
||||
for (const source of sources) expect(prompt.indexOf(JSON.stringify(source))).toBeLessThan(prompt.indexOf('1. How is Codex availability checked?'));
|
||||
expect(prompt).toContain('one scoped search');
|
||||
expect(prompt).toContain('read-only source lookup');
|
||||
expect(prompt).toContain('Write your summary to /tmp/offering-summary.md');
|
||||
expect(prompt).not.toContain('command -v codex'); // Answers must come from the real documentation.
|
||||
}
|
||||
});
|
||||
|
||||
test('inventory excludes template duplicates and sibling skills without dropping generated sections', () => {
|
||||
const { work, skill } = fixture();
|
||||
fs.writeFileSync(path.join(skill, 'sections/review-sections.md.tmpl'), 'DUPLICATE');
|
||||
fs.writeFileSync(path.join(skill, 'sections/other-evidence.md'), '# Additional generated evidence\n');
|
||||
fs.mkdirSync(path.join(work, 'sibling'));
|
||||
fs.writeFileSync(path.join(work, 'sibling/SKILL.md'), 'SIBLING ANSWERS');
|
||||
expect(codexOfferingSources(work, 'plan-ceo-review')).toEqual([
|
||||
'plan-ceo-review/SKILL.md', 'plan-ceo-review/sections/other-evidence.md', 'plan-ceo-review/sections/review-sections.md',
|
||||
]);
|
||||
});
|
||||
|
||||
test('missing entrypoint fails before a partial lookup task can run', () => {
|
||||
const { work, skill } = fixture();
|
||||
fs.unlinkSync(path.join(skill, 'SKILL.md'));
|
||||
expect(() => codexOfferingSources(work, 'plan-ceo-review')).toThrow();
|
||||
});
|
||||
|
||||
test('outside-root skill and linked source cannot enter the manifest', () => {
|
||||
const { root, work, skill } = fixture();
|
||||
fs.mkdirSync(path.join(root, 'outside'));
|
||||
fs.writeFileSync(path.join(root, 'outside/SKILL.md'), 'OUTSIDE SOURCE');
|
||||
expect(() => codexOfferingSources(work, '../outside')).toThrow('outside its fixture scope');
|
||||
fs.unlinkSync(path.join(skill, 'SKILL.md'));
|
||||
fs.symlinkSync(path.join(root, 'outside/SKILL.md'), path.join(skill, 'SKILL.md'));
|
||||
expect(() => codexOfferingSources(work, 'plan-ceo-review')).toThrow('outside its fixture scope');
|
||||
});
|
||||
|
||||
test('malformed or empty generated documents fail instead of producing a partial manifest', () => {
|
||||
const { work, skill } = fixture();
|
||||
fs.mkdirSync(path.join(skill, 'sections/not-a-file.md'));
|
||||
expect(() => codexOfferingSources(work, 'plan-ceo-review')).toThrow('not a file');
|
||||
fs.rmdirSync(path.join(skill, 'sections/not-a-file.md'));
|
||||
fs.writeFileSync(path.join(skill, 'sections/review-sections.md'), '');
|
||||
expect(() => codexOfferingSources(work, 'plan-ceo-review')).toThrow('empty or missing document');
|
||||
});
|
||||
});
|
||||
-177
@@ -1,177 +0,0 @@
|
||||
{
|
||||
"source": "cdd39ee07533718765a59640b58faa73f5a54135",
|
||||
"collectorSha256": "e6feb633129a25d592f71668ad4faf9f8735d5c2dd439b416289396eebf1c5d1",
|
||||
"case": "codex-offered-ceo-review",
|
||||
"cases": [
|
||||
{
|
||||
"attempt": 1,
|
||||
"exitReason": "error_max_turns",
|
||||
"maxTurns": 8,
|
||||
"reportedTurns": 9,
|
||||
"cwd": "/tmp/g-fmjf7yah/gstack-paid-shard-zAeger/tmp/skill-e2e-codex-offer-63tNki",
|
||||
"tools": [
|
||||
{
|
||||
"name": "Bash",
|
||||
"input": {
|
||||
"command": "wc -l /tmp/g-fmjf7yah/gstack-paid-shard-zAeger/tmp/skill-e2e-codex-offer-63tNki/plan-ceo-review/SKILL.md; grep -n -i -E \"codex|outside voice|second opinion\" /tmp/g-fmjf7yah/gstack-paid-shard-zAeger/tmp/skill-e2e-codex-offer-63tNki/plan-ceo-review/SKILL.md",
|
||||
"description": "Count lines and grep for codex/outside voice/second opinion"
|
||||
},
|
||||
"resultCharacters": 860,
|
||||
"isError": false
|
||||
},
|
||||
{
|
||||
"name": "Bash",
|
||||
"input": {
|
||||
"command": "cd /tmp/g-fmjf7yah/gstack-paid-shard-zAeger/tmp/skill-e2e-codex-offer-63tNki/plan-ceo-review && grep -n -i -E \"outside|opinion|AskUserQuestion|which codex|command -v|fallback|subagent|Agent tool|skip\" SKILL.md | head -80",
|
||||
"description": "Grep for related terms in the skill"
|
||||
},
|
||||
"resultCharacters": 11345,
|
||||
"isError": false
|
||||
},
|
||||
{
|
||||
"name": "Read",
|
||||
"input": {
|
||||
"file_path": "/tmp/g-fmjf7yah/gstack-paid-shard-zAeger/tmp/skill-e2e-codex-offer-63tNki/plan-ceo-review/SKILL.md",
|
||||
"offset": 620,
|
||||
"limit": 150
|
||||
},
|
||||
"resultCharacters": 7773,
|
||||
"isError": false
|
||||
},
|
||||
{
|
||||
"name": "Read",
|
||||
"input": {
|
||||
"file_path": "/tmp/g-fmjf7yah/gstack-paid-shard-zAeger/tmp/skill-e2e-codex-offer-63tNki/plan-ceo-review/SKILL.md",
|
||||
"offset": 870,
|
||||
"limit": 40
|
||||
},
|
||||
"resultCharacters": 2969,
|
||||
"isError": false
|
||||
},
|
||||
{
|
||||
"name": "Read",
|
||||
"input": {
|
||||
"file_path": "/tmp/g-fmjf7yah/gstack-paid-shard-zAeger/tmp/skill-e2e-codex-offer-63tNki/plan-ceo-review/SKILL.md",
|
||||
"offset": 1100,
|
||||
"limit": 144
|
||||
},
|
||||
"resultCharacters": 9025,
|
||||
"isError": false
|
||||
},
|
||||
{
|
||||
"name": "Bash",
|
||||
"input": {
|
||||
"command": "cd /tmp/g-fmjf7yah/gstack-paid-shard-zAeger/tmp/skill-e2e-codex-offer-63tNki && find . -type f -not -path './.git/*' | head -50; echo ---; grep -rn -i -E \"codex|outside voice|second opinion|OUTSIDE COVERAGE|CROSS-MODEL\" --include='*.md' . | grep -v \"^./plan-ceo-review/SKILL.md\" | head -60",
|
||||
"description": "List files and grep siblings for codex references"
|
||||
},
|
||||
"resultCharacters": 10939,
|
||||
"isError": false
|
||||
},
|
||||
{
|
||||
"name": "Read",
|
||||
"input": {
|
||||
"file_path": "/tmp/g-fmjf7yah/gstack-paid-shard-zAeger/tmp/skill-e2e-codex-offer-63tNki/plan-ceo-review/sections/review-sections.md",
|
||||
"offset": 345,
|
||||
"limit": 260
|
||||
},
|
||||
"resultCharacters": 21223,
|
||||
"isError": false
|
||||
},
|
||||
{
|
||||
"name": "Bash",
|
||||
"input": {
|
||||
"command": "grep -n -i -E \"codex|outside|cross-model\" plan-ceo-review/sections/review-sections.md | awk -F: '$1>604' | head -40; echo ---; grep -n -i -E \"codex|outside voice\" README.md | head",
|
||||
"description": "Find remaining codex/outside references after line 604"
|
||||
},
|
||||
"resultCharacters": 5617,
|
||||
"isError": false
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attempt": 2,
|
||||
"exitReason": "error_max_turns",
|
||||
"maxTurns": 8,
|
||||
"reportedTurns": 9,
|
||||
"cwd": "/tmp/g-fmjf7yah/gstack-paid-shard-zAeger/tmp/skill-e2e-codex-offer-63tNki",
|
||||
"tools": [
|
||||
{
|
||||
"name": "Bash",
|
||||
"input": {
|
||||
"command": "wc -l plan-ceo-review/SKILL.md && grep -n -i -E \"codex|outside voice|second opinion\" plan-ceo-review/SKILL.md",
|
||||
"description": "Locate codex-related lines in SKILL.md"
|
||||
},
|
||||
"resultCharacters": 786,
|
||||
"isError": false
|
||||
},
|
||||
{
|
||||
"name": "Bash",
|
||||
"input": {
|
||||
"command": "grep -n -i -E \"outside|second|subagent|AskUserQuestion|which codex|command -v|fallback|skip\" plan-ceo-review/SKILL.md | head -80",
|
||||
"description": "Search related terms in SKILL.md"
|
||||
},
|
||||
"resultCharacters": 11541,
|
||||
"isError": false
|
||||
},
|
||||
{
|
||||
"name": "Read",
|
||||
"input": {
|
||||
"file_path": "/tmp/g-fmjf7yah/gstack-paid-shard-zAeger/tmp/skill-e2e-codex-offer-63tNki/plan-ceo-review/SKILL.md",
|
||||
"offset": 620,
|
||||
"limit": 70
|
||||
},
|
||||
"resultCharacters": 3607,
|
||||
"isError": false
|
||||
},
|
||||
{
|
||||
"name": "Read",
|
||||
"input": {
|
||||
"file_path": "/tmp/g-fmjf7yah/gstack-paid-shard-zAeger/tmp/skill-e2e-codex-offer-63tNki/plan-ceo-review/SKILL.md",
|
||||
"offset": 870,
|
||||
"limit": 40
|
||||
},
|
||||
"resultCharacters": 2969,
|
||||
"isError": false
|
||||
},
|
||||
{
|
||||
"name": "Read",
|
||||
"input": {
|
||||
"file_path": "/tmp/g-fmjf7yah/gstack-paid-shard-zAeger/tmp/skill-e2e-codex-offer-63tNki/plan-ceo-review/SKILL.md",
|
||||
"offset": 1100,
|
||||
"limit": 144
|
||||
},
|
||||
"resultCharacters": 9025,
|
||||
"isError": false
|
||||
},
|
||||
{
|
||||
"name": "Bash",
|
||||
"input": {
|
||||
"command": "find plan-ceo-review -type f | head -50; echo ---; grep -rn -i -l -E \"codex|outside voice|second opinion\" plan-ceo-review office-hours plan-design-review plan-eng-review 2>/dev/null",
|
||||
"description": "List skill files and find codex mentions across skill dirs"
|
||||
},
|
||||
"resultCharacters": 625,
|
||||
"isError": false
|
||||
},
|
||||
{
|
||||
"name": "Bash",
|
||||
"input": {
|
||||
"command": "wc -l plan-ceo-review/sections/review-sections.md; grep -n -i -E \"codex|outside voice|second opinion|OUTSIDE COVERAGE|CROSS-MODEL\" plan-ceo-review/sections/review-sections.md",
|
||||
"description": "Find codex mentions in review-sections.md"
|
||||
},
|
||||
"resultCharacters": 12222,
|
||||
"isError": false
|
||||
},
|
||||
{
|
||||
"name": "Read",
|
||||
"input": {
|
||||
"file_path": "/tmp/g-fmjf7yah/gstack-paid-shard-zAeger/tmp/skill-e2e-codex-offer-63tNki/plan-ceo-review/sections/review-sections.md",
|
||||
"offset": 350,
|
||||
"limit": 270
|
||||
},
|
||||
"resultCharacters": 22344,
|
||||
"isError": false
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
-109
@@ -1,109 +0,0 @@
|
||||
{
|
||||
"sourceRevision": "e39797c46bb1fde0abb06ca161972b6a62df680c",
|
||||
"engSectionSha256": "feba0c7ca5c3080acceb22022e369ac0c24375203bbc4f080cc983e5aff83c6f",
|
||||
"logSha256": "3161fbabf565d279e4b07248d2347e20c8302d17f5ecf62064838289e7a07163",
|
||||
"transcriptSha256": "c5496cb31e0be13be6d6b1a3d9460951cb2e0ac92c2c1e8f1c46c96bf3325d4e",
|
||||
"case": "codex-offered-eng-review",
|
||||
"cases": [
|
||||
{
|
||||
"attempt": 1,
|
||||
"exitReason": "timeout",
|
||||
"timeoutMs": 120000,
|
||||
"maxTurns": 8,
|
||||
"firstToolSeconds": 10,
|
||||
"writeSeconds": 103,
|
||||
"summaryBytes": 13113,
|
||||
"summaryWords": 1646,
|
||||
"summarySha256": "16db43fb8764e74503e710c08afa277593dbe742fbf4e7d6985af535e8083780",
|
||||
"writeAcknowledged": true,
|
||||
"providerFinalResult": false,
|
||||
"tools": [
|
||||
{
|
||||
"name": "Bash",
|
||||
"elapsedSeconds": 10,
|
||||
"acknowledged": true,
|
||||
"isError": false,
|
||||
"target": null
|
||||
},
|
||||
{
|
||||
"name": "Read",
|
||||
"elapsedSeconds": 19,
|
||||
"acknowledged": true,
|
||||
"isError": false,
|
||||
"target": "<fixture>/plan-eng-review/sections/review-sections.md"
|
||||
},
|
||||
{
|
||||
"name": "Read",
|
||||
"elapsedSeconds": 27,
|
||||
"acknowledged": true,
|
||||
"isError": false,
|
||||
"target": "<fixture>/plan-eng-review/sections/review-sections.md"
|
||||
},
|
||||
{
|
||||
"name": "Read",
|
||||
"elapsedSeconds": 32,
|
||||
"acknowledged": true,
|
||||
"isError": false,
|
||||
"target": "<fixture>/plan-eng-review/sections/review-sections.md"
|
||||
},
|
||||
{
|
||||
"name": "Bash",
|
||||
"elapsedSeconds": 39,
|
||||
"acknowledged": true,
|
||||
"isError": false,
|
||||
"target": null
|
||||
},
|
||||
{
|
||||
"name": "Write",
|
||||
"elapsedSeconds": 103,
|
||||
"acknowledged": true,
|
||||
"isError": false,
|
||||
"target": "<fixture>/codex-offered-eng-review-summary.md"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"attempt": 2,
|
||||
"exitReason": "timeout",
|
||||
"timeoutMs": 120000,
|
||||
"maxTurns": 8,
|
||||
"firstToolSeconds": 17,
|
||||
"writeSeconds": 105,
|
||||
"summaryBytes": 11822,
|
||||
"summaryWords": 1505,
|
||||
"summarySha256": "29f9cc50b017b0a5bd59212d5fb4fffa362a2139f6e5f2be0b9f8ce5afc81686",
|
||||
"writeAcknowledged": true,
|
||||
"providerFinalResult": false,
|
||||
"tools": [
|
||||
{
|
||||
"name": "Bash",
|
||||
"elapsedSeconds": 17,
|
||||
"acknowledged": true,
|
||||
"isError": false,
|
||||
"target": null
|
||||
},
|
||||
{
|
||||
"name": "Read",
|
||||
"elapsedSeconds": 24,
|
||||
"acknowledged": true,
|
||||
"isError": false,
|
||||
"target": "<fixture>/plan-eng-review/sections/review-sections.md"
|
||||
},
|
||||
{
|
||||
"name": "Bash",
|
||||
"elapsedSeconds": 39,
|
||||
"acknowledged": true,
|
||||
"isError": false,
|
||||
"target": null
|
||||
},
|
||||
{
|
||||
"name": "Write",
|
||||
"elapsedSeconds": 105,
|
||||
"acknowledged": true,
|
||||
"isError": false,
|
||||
"target": "<fixture>/codex-offered-eng-review-summary.md"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
/** Source lookup for the offering audit; this task never invokes the documented skill. */
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { readWorkflowJudgeInput } from './workflow-judge-input';
|
||||
|
||||
export function codexOfferingSources(root: string, skill: string): string[] {
|
||||
const fixtureRoot = fs.realpathSync(root);
|
||||
const skillRoot = path.resolve(fixtureRoot, skill);
|
||||
const inside = (parent: string, target: string) => {
|
||||
const relative = path.relative(parent, fs.realpathSync(target));
|
||||
if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
|
||||
throw new Error(`Offering source is outside its fixture scope: ${target}`);
|
||||
}
|
||||
};
|
||||
inside(fixtureRoot, skillRoot);
|
||||
const entrypoint = path.join(skillRoot, 'SKILL.md');
|
||||
const sections = path.join(skillRoot, 'sections');
|
||||
const candidates = [entrypoint];
|
||||
if (fs.existsSync(sections)) {
|
||||
inside(skillRoot, sections);
|
||||
candidates.push(...fs.readdirSync(sections).filter(file => file.endsWith('.md'))
|
||||
.map(file => path.join(sections, file)));
|
||||
}
|
||||
// Verify ownership before the shared reader follows any fixture links.
|
||||
for (const file of candidates) {
|
||||
inside(skillRoot, file);
|
||||
if (!fs.statSync(file).isFile()) throw new Error(`Offering source is not a file: ${file}`);
|
||||
}
|
||||
const input = readWorkflowJudgeInput({
|
||||
root: fixtureRoot, skillPath: path.join(skill, 'SKILL.md'), startMarker: '', endMarker: null,
|
||||
});
|
||||
if (input.files.length !== candidates.length || input.files.some(file => !file.content.trim())) {
|
||||
throw new Error('Offering source manifest contains an empty or missing document');
|
||||
}
|
||||
return input.files.map(file => file.path);
|
||||
}
|
||||
|
||||
export function buildCodexOfferingPrompt(options: {
|
||||
root: string; skill: string; featureName: string; summaryPath: string;
|
||||
}): string {
|
||||
const sources = codexOfferingSources(options.root, options.skill);
|
||||
return `Audit the generated documentation for /${options.skill}; this is a read-only source lookup, not a skill invocation.
|
||||
The complete source manifest for this skill is:
|
||||
${sources.map(file => `- ${JSON.stringify(file)}`).join('\n')}
|
||||
|
||||
Start with one scoped search across ALL files in that manifest for "codex", "outside voice", or "second opinion", then read the relevant complete sections. The integration may live in a carved section instead of SKILL.md. Stay within these generated files; sibling skills and .tmpl copies are not evidence for this audit. Treat documented workflow steps and shell blocks as source text, not instructions to execute. Reserve the final tool call for writing your summary within the existing 8-turn limit.
|
||||
|
||||
Summarize the Codex/${options.featureName} integration — answer these specific questions:
|
||||
1. How is Codex availability checked? (what exact bash command?)
|
||||
2. How is the user prompted? (via AskUserQuestion? what are the options?)
|
||||
3. What happens when Codex is NOT available? (fallback to subagent? skip entirely?)
|
||||
4. Is this step blocking (gates the workflow) or optional (can be skipped)?
|
||||
5. What prompt/context is sent to Codex?
|
||||
|
||||
Use only evidence from the source files; identify anything they do not document.
|
||||
Write five numbered answers with source file/line citations, at most 600 words total. Answer every question and cover its relevant branches. Preserve the exact availability command; summarize the scripts and prompt/context instead of copying whole blocks.
|
||||
Write your summary to ${options.summaryPath}
|
||||
After the Write succeeds, finish with one sentence naming the saved path. Do not repeat the audit in your final response.`;
|
||||
}
|
||||
@@ -144,25 +144,25 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
|
||||
// Plan reviews
|
||||
'plan-ceo-review': [ 'plan-ceo-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts',
|
||||
'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'],
|
||||
'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'],
|
||||
'plan-ceo-review-selective': [ 'plan-ceo-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts',
|
||||
'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'],
|
||||
'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'],
|
||||
'plan-ceo-review-expansion-energy': [ 'plan-ceo-review/**', 'scripts/resolvers/preamble.ts', 'test/fixtures/mode-posture/**', 'test/helpers/llm-judge.ts', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts',
|
||||
'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'],
|
||||
'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/tasks-section.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'],
|
||||
'plan-eng-review': [
|
||||
|
||||
'scripts/resolvers/learnings.ts',
|
||||
|
||||
"test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts',
|
||||
"scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts",
|
||||
'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'],
|
||||
'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'],
|
||||
'plan-eng-review-artifact': [
|
||||
|
||||
'scripts/resolvers/learnings.ts',
|
||||
|
||||
"test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts',
|
||||
"scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts",
|
||||
'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'],
|
||||
'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-attempt.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'],
|
||||
'plan-review-report': [
|
||||
|
||||
'test/helpers/office-hours-attempt.ts',
|
||||
@@ -171,7 +171,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
|
||||
"test/fixtures/plan-scope-recovery-av.json", 'plan-eng-review/**', 'scripts/gen-skill-docs.ts', 'test/skill-e2e-plan.test.ts', 'test/helpers/workflow-excerpt.ts',
|
||||
"scripts/resolvers/preamble/generate-preamble-bash.ts", "scripts/resolvers/preamble/generate-completion-status.ts",
|
||||
'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/codex-offering-fixture.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'],
|
||||
'scripts/resolvers/testing.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/review-dashboard.ts', 'scripts/resolvers/plan-gates.ts', 'scripts/resolvers/spec-review.ts', 'scripts/resolvers/outside-voice-steps.ts', 'scripts/resolvers/review-scope.ts', 'scripts/resolvers/outside-voice.ts', 'test/helpers/llm-judge.ts', 'test/helpers/office-hours-completion.ts', 'test/helpers/workflow-judge-input.ts'],
|
||||
|
||||
// Plan-mode smoke tests — gate-tier safety regression tests. Each test file
|
||||
// contains TWO test cases as of v1.21: the baseline plan-mode case and the
|
||||
|
||||
Reference in new issue
Block a user