fix: Codex E2E test now validates all skills load without warnings

- Install ALL skills to temp HOME (not just one) to catch missing SKILL.md
- Pre-flight asserts every .agents/ dir has both SKILL.md and openai.yaml
- Assert no "invalid SKILL.md" or "Skipped loading" in stderr
- Add existingHome option to runCodexSkill for pre-populated temp HOMEs
- Increase discover test timeout to 120s (all-skills load takes longer)
This commit is contained in:
Garry Tan
2026-03-23 09:14:17 -07:00
parent 308f5fc5ad
commit 95d9116003
2 changed files with 48 additions and 13 deletions
+36 -8
View File
@@ -120,15 +120,36 @@ afterAll(async () => {
describeCodex('Codex E2E', () => {
testIfSelected('codex-discover-skill', async () => {
// Install gstack-review skill to a temp HOME and ask Codex to list skills
const skillDir = path.join(ROOT, '.agents', 'skills', 'gstack-review');
// ── Pre-flight: verify .agents/ has both SKILL.md and openai.yaml ──
const agentsDir = path.join(ROOT, '.agents', 'skills');
const skillDirs = fs.existsSync(agentsDir)
? fs.readdirSync(agentsDir, { withFileTypes: true }).filter(d => d.isDirectory()).map(d => d.name)
: [];
expect(skillDirs.length).toBeGreaterThan(0); // .agents/ must be generated
const missingSkillMd: string[] = [];
const missingYaml: string[] = [];
for (const dir of skillDirs) {
if (!fs.existsSync(path.join(agentsDir, dir, 'SKILL.md'))) missingSkillMd.push(dir);
if (!fs.existsSync(path.join(agentsDir, dir, 'agents', 'openai.yaml'))) missingYaml.push(dir);
}
expect(missingSkillMd).toEqual([]); // Every skill dir must have SKILL.md
expect(missingYaml).toEqual([]); // Every skill dir must have agents/openai.yaml
// ── Install ALL skills to temp HOME (like a real install) ──
const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), 'codex-e2e-all-'));
try {
for (const dir of skillDirs) {
installSkillToTempHome(path.join(agentsDir, dir), dir, tempHome);
}
const result = await runCodexSkill({
skillDir,
prompt: 'List any skills or instructions you have available. Just list the names.',
timeoutMs: 60_000,
skillDir: path.join(agentsDir, 'gstack-review'),
prompt: 'List all skills or instructions you have available. List every skill name, one per line.',
timeoutMs: 120_000,
cwd: ROOT,
skillName: 'gstack-review',
existingHome: tempHome,
});
logCodexCost('codex-discover-skill', result);
@@ -139,14 +160,21 @@ describeCodex('Codex E2E', () => {
expect(result.exitCode).toBe(0);
expect(result.output.length).toBeGreaterThan(0);
// Skill loading errors mean our generated SKILL.md files are broken
expect(result.stderr).not.toContain('invalid');
// No skill loading warnings or errors in stderr
// (Codex CLI may emit its own PATH warnings in temp HOME — ignore those)
expect(result.stderr).not.toContain('invalid SKILL.md');
expect(result.stderr).not.toContain('Skipped loading');
// The output should reference the skill name in some form
expect(result.stderr).not.toContain('invalid skill');
// The output should reference skills by name
const outputLower = result.output.toLowerCase();
expect(
outputLower.includes('review') || outputLower.includes('gstack') || outputLower.includes('skill'),
).toBe(true);
} finally {
fs.rmSync(tempHome, { recursive: true, force: true });
}
}, 120_000);
// Validates that Codex can invoke the gstack-review skill, run a diff-based
+12 -5
View File
@@ -143,6 +143,7 @@ export async function runCodexSkill(opts: {
cwd?: string; // Working directory
skillName?: string; // Skill name for installation (default: dirname)
sandbox?: string; // Sandbox mode (default: 'read-only')
existingHome?: string; // Pre-populated temp HOME (skips installSkillToTempHome)
}): Promise<CodexResult> {
const {
skillDir,
@@ -151,6 +152,7 @@ export async function runCodexSkill(opts: {
cwd,
skillName,
sandbox = 'read-only',
existingHome,
} = opts;
const startTime = Date.now();
@@ -172,12 +174,15 @@ export async function runCodexSkill(opts: {
};
}
// Set up temp HOME with skill installed
const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), 'codex-e2e-'));
// Set up temp HOME with skill installed (or use pre-populated one)
const ownedHome = !existingHome;
const tempHome = existingHome || fs.mkdtempSync(path.join(os.tmpdir(), 'codex-e2e-'));
const realHome = os.homedir();
try {
installSkillToTempHome(skillDir, name, tempHome);
if (ownedHome) {
installSkillToTempHome(skillDir, name, tempHome);
}
// Symlink real Codex auth config so codex can authenticate from temp HOME.
// Codex stores auth in ~/.codex/ — we need the config but not the skills
@@ -287,7 +292,9 @@ export async function runCodexSkill(opts: {
stderr,
};
} finally {
// Clean up temp HOME
try { fs.rmSync(tempHome, { recursive: true, force: true }); } catch { /* non-fatal */ }
// Clean up temp HOME (only if we created it)
if (ownedHome) {
try { fs.rmSync(tempHome, { recursive: true, force: true }); } catch { /* non-fatal */ }
}
}
}