Merge origin/main (v1.91.15.0, #3017) into garrytan/parallel-waves; release metadata v1.91.16.0

Conflicts resolved by hand in templates, tests, CLAUDE.md, TODOS.md,
CHANGELOG.md and bin/gstack-config; generated SKILL.md/sections, the agents
digest and test/fixtures/context-budget.json regenerated. The eval codemod
re-applied to main's new gstack-paths/gstack-slug evals. Coverage-read rule:
bare echo separators after unknown output extend the unknown span (main's
pathspec CI shape). CLAUDE.md keeps the TREE_MUTATING, catalog-ceiling and
detach-timeout facts main's instruction-facts test pins.
This commit is contained in:
garrytan committed 2026-10-03 08:53:18 +00:00
commit 6fb8ea3d10
351 files changed
+7477 -8179

No files matched your search

+30 -37
View File
@@ -43,6 +43,8 @@ import { spawnSync } from 'child_process';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { readShippedSkillRouting } from './helpers/shipped-skill-routing';
import { hasNarrationLeak } from './helpers/skill-body-narration';
const evalCollector = createEvalCollector('e2e-skillify');
@@ -61,7 +63,7 @@ interface Workdir {
* - bin/ scripts referenced by the preamble
* - A scoped GSTACK_HOME under the workdir so on-disk artifacts are
* contained and assertable
* - A CLAUDE.md routing block instructing Skill-tool invocation
* - A CLAUDE.md with the ## Skill routing section gstack ships
*
* `installSkills` lets each test pick the minimum surface (e.g., the
* provenance-refusal scenario doesn't need /scrape).
@@ -110,16 +112,10 @@ function setupSkillifyWorkdir(suffix: string, installSkills: string[] = ['scrape
fs.writeFileSync(path.join(workDir, 'CLAUDE.md'), `# Project Instructions
## Skill routing
${readShippedSkillRouting().section}
When the user's request matches an available skill, ALWAYS invoke it via
the Skill tool as your FIRST action.
## Test environment
Key routing rules:
- /scrape, "scrape", "get data from", "extract from" → invoke scrape
- /skillify, "skillify", "codify this scrape" → invoke skillify
Environment:
- GSTACK_HOME="${gstackHome}" for all gstack bin scripts.
- bin scripts are at ./bin/ relative to this directory.
- Browse binary is at ${browseBin} — assign to $B (e.g., \`B=${browseBin}\`).
@@ -215,13 +211,8 @@ describeIfSelected('Browser-skills Phase 2a E2E (/scrape + /skillify)', [
installBundledHackernewsSkill(workDir);
const result = await runSkillTest({
prompt: `Run /scrape latest hacker news stories. Invoke /scrape via the Skill tool.
You MUST follow the skill's match-phase logic:
1. Run \`$B skill list\` to see what browser-skills are available
2. Recognize that "latest hacker news stories" matches the bundled
hackernews-frontpage skill's triggers
3. Run \`$B skill run hackernews-frontpage\` and emit the JSON
Do NOT enter the prototype phase. Do NOT use AskUserQuestion.`,
prompt: `Run /scrape latest hacker news stories.
This run is non-interactive; AskUserQuestion is unavailable.`,
workingDirectory: workDir,
env: { GSTACK_HOME: gstackHome },
maxTurns: 12,
@@ -233,18 +224,31 @@ Do NOT enter the prototype phase. Do NOT use AskUserQuestion.`,
logCost('scrape-match-path', result);
// Outcome: the matcher ran the bundled skill instead of prototyping, and
// nothing new was written. Listing first is one valid way to match.
const cmds = bashCommands(result);
const listedSkills = cmds.some(c => /\bskill\s+list\b/.test(c));
const ranBundledSkill = cmds.some(c => /\bskill\s+run\s+hackernews-frontpage\b/.test(c));
const prototyped = cmds.some(c => /\$B\s+goto\b|\/browse\s+goto\b/.test(c));
const newSkills = [workDir, gstackHome].flatMap((root) => {
const skillsRoot = path.join(root, '.gstack', 'browser-skills');
return fs.existsSync(skillsRoot) ? fs.readdirSync(skillsRoot).filter(d => !d.startsWith('.') && d !== 'hackernews-frontpage') : [];
});
const staged = [workDir, gstackHome].some((root) => {
const stagingRoot = path.join(root, '.gstack', '.tmp');
return fs.existsSync(stagingRoot) && fs.readdirSync(stagingRoot).some(d => d.startsWith('skillify-'));
});
console.log(`scrape-match-path: listed skills first = ${cmds.some(c => /\bskill\s+list\b/.test(c))}`);
const exitOk = ['success', 'error_max_turns'].includes(result.exitReason);
recordE2E(evalCollector, 'scrape match-path routes to bundled skill', 'Phase 2a E2E', result, {
passed: exitOk && listedSkills && ranBundledSkill,
passed: exitOk && ranBundledSkill && !prototyped && newSkills.length === 0 && !staged,
});
expect(exitOk).toBe(true);
expect(listedSkills).toBe(true);
expect(ranBundledSkill).toBe(true);
expect(prototyped, 'matched skill must run without prototyping').toBe(false);
expect(newSkills).toEqual([]);
expect(staged).toBe(false);
try { fs.rmSync(workDir, { recursive: true, force: true }); } catch {}
}, CAPTURE_MS);
@@ -259,13 +263,8 @@ Do NOT enter the prototype phase. Do NOT use AskUserQuestion.`,
const result = await runSkillTest({
prompt: `Run /scrape titles and scores from ${fileUrl}.
Invoke /scrape via the Skill tool. Follow the skill's prototype-phase logic:
1. \`$B skill list\` finds NO matching skill
2. Drive: \`$B goto ${fileUrl}\` then \`$B html\` (or \`$B text\`)
3. Parse the items (each has a title and a score)
4. Emit JSON of the form {"items": [{"title": "...", "score": N}, ...], "count": N}
5. Suggest /skillify in one line
Do NOT use AskUserQuestion.`,
Return JSON of the form {"items": [{"title": "...", "score": N}, ...], "count": N}.
This run is non-interactive; AskUserQuestion is unavailable.`,
workingDirectory: workDir,
env: { GSTACK_HOME: gstackHome },
maxTurns: 18,
@@ -371,16 +370,13 @@ Do NOT halt for clarification.`,
&& fs.existsSync(path.join(skillDir, '_lib', 'browse-client.ts'))
&& fs.existsSync(path.join(skillDir, 'fixtures'));
// D2 enforcement: the SKILL.md prose body MUST NOT contain conversation
// fragments. Cheap heuristic: it shouldn't have "I" or "Let me" or other
// first-person/agent-narration markers.
// D2 enforcement: the SKILL.md prose body must not contain conversation
// fragments (stored good/bad bodies: skill-fixture.test.ts).
let prosesClean = false;
if (hasAllFiles) {
const skillMd = fs.readFileSync(path.join(skillDir, 'SKILL.md'), 'utf-8');
const body = skillMd.split(/\n---\n/)[1] || '';
prosesClean = !/^I /m.test(body)
&& !/Let me /i.test(body)
&& !/^I'll /m.test(body);
prosesClean = !hasNarrationLeak(body);
}
const exitOk = ['success', 'error_max_turns'].includes(result.exitReason);
@@ -411,11 +407,8 @@ Do NOT halt for clarification.`,
const childEnv = { GSTACK_HOME: gstackHome, HOME: childHome };
const result = await runSkillTest({
prompt: `Run /skillify via the Skill tool. There has been NO prior /scrape
in this conversation. Follow the skill's Step 1 (D1 provenance guard) literally:
walk back through agent turns, find no /scrape result, refuse with the exact
message the skill specifies, and stop. Do NOT synthesize anything. Do NOT
write any files.`,
prompt: `Run /skillify.
This run is non-interactive; AskUserQuestion is unavailable.`,
workingDirectory: workDir,
env: childEnv,
maxTurns: 8,