diff --git a/lib/skill-bill.ts b/lib/skill-bill.ts index f72e7d560..aeca2a739 100644 --- a/lib/skill-bill.ts +++ b/lib/skill-bill.ts @@ -43,7 +43,8 @@ export function checkEvalTasks(file: string): EvalTasks { const problems: string[] = []; const ids = new Set(); for (const t of tasks as Record[]) { - const where = String(t?.id ?? "?"); + if (!t || typeof t !== "object" || Array.isArray(t)) { problems.push(`?: task must be an object, got ${JSON.stringify(t)}`); continue; } + const where = String(t.id ?? "?"); for (const k of Object.keys(t ?? {})) if (!EVAL_TASK_KEYS.has(k)) problems.push(`${where}: unknown key ${k}`); if (typeof t?.id !== "string" || !/^[a-z0-9]+(-[a-z0-9]+)*$/.test(t.id)) problems.push(`${where}: id must be kebab-case`); else if (ids.has(t.id)) problems.push(`${where}: duplicate id`); @@ -198,7 +199,7 @@ export function buildSkillBill(dir: string): SkillBillReport { if (kind === "gstack-checkout") { const root = gstackRootOf(dir)!; const file = join(root, "test", "fixtures", "skill-evals", name, "tasks.json"); - evalTasks = existsSync(file) ? { ...checkEvalTasks(file), path: relative(root, file) } : null; + evalTasks = existsSync(file) ? { ...checkEvalTasks(file), path: `test/fixtures/skill-evals/${name}/tasks.json` } : null; if (evalTasks?.problems.length) warnings.push(`EVAL_TASKS_INVALID: ${evalTasks.problems[0]}${evalTasks.problems.length > 1 ? ` (+${evalTasks.problems.length - 1} more)` : ""}. fix: the task format in docs/designs/SKILL_EVAL_CONTRACT.md`); const changed = git(dir, ["status", "--porcelain", "--", "SKILL.md.tmpl", "SKILL.md"]).out.split("\n").filter(Boolean); if (changed.some((l) => l.endsWith("SKILL.md.tmpl")) && !changed.some((l) => l.endsWith(" SKILL.md") || l.endsWith("/SKILL.md"))) diff --git a/test/skill-eval-tasks.test.ts b/test/skill-eval-tasks.test.ts index 9e510c5bd..185b52d70 100644 --- a/test/skill-eval-tasks.test.ts +++ b/test/skill-eval-tasks.test.ts @@ -34,6 +34,8 @@ describe('skill-evals task files follow the shared eval contract', () => { { id: 'Bad Id', prompt: '' }, { id: 'dup', prompt: 'x', must: ['('], extra: 1 }, { id: 'dup', prompt: 'y', check: 'true', setup: 'sh "$SKILL_EVAL_FIXTURES/seed.sh"', safety: 'yes' }, + 1, + ['nested'], ])); expect(checkEvalTasks(file).problems).toEqual([ 'Bad Id: id must be kebab-case', @@ -44,6 +46,8 @@ describe('skill-evals task files follow the shared eval contract', () => { 'dup: duplicate id', 'dup: safety must be a boolean', 'dup: setup runs seed.sh but it is missing', + '?: task must be an object, got 1', + '?: task must be an object, got ["nested"]', ]); fs.writeFileSync(file, '{'); expect(checkEvalTasks(file).problems[0]).toStartWith('not valid JSON');