mirror of
https://github.com/garrytan/gstack.git
synced 2026-05-06 21:46:40 +02:00
Merge remote-tracking branch 'origin/main' into garrytan/session-intelligence
# Conflicts: # SKILL.md # autoplan/SKILL.md # benchmark/SKILL.md # browse/SKILL.md # canary/SKILL.md # codex/SKILL.md # connect-chrome/SKILL.md # cso/SKILL.md # design-consultation/SKILL.md # design-html/SKILL.md # design-review/SKILL.md # design-shotgun/SKILL.md # document-release/SKILL.md # investigate/SKILL.md # land-and-deploy/SKILL.md # learn/SKILL.md # office-hours/SKILL.md # plan-ceo-review/SKILL.md # plan-design-review/SKILL.md # plan-eng-review/SKILL.md # qa-only/SKILL.md # qa/SKILL.md # retro/SKILL.md # review/SKILL.md # scripts/resolvers/preamble.ts # setup-browser-cookies/SKILL.md # setup-deploy/SKILL.md # ship/SKILL.md
This commit is contained in:
Garry Tan
@@ -213,11 +213,20 @@ describe('gen-skill-docs', () => {
|
||||
expect(browseTmpl).toContain('{{PREAMBLE}}');
|
||||
});
|
||||
|
||||
test('generated SKILL.md contains contributor mode check', () => {
|
||||
test('generated SKILL.md contains operational self-improvement (replaced contributor mode)', () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('Contributor Mode');
|
||||
expect(content).toContain('gstack_contributor');
|
||||
expect(content).toContain('contributor-logs');
|
||||
expect(content).not.toContain('Contributor Mode');
|
||||
expect(content).not.toContain('gstack_contributor');
|
||||
expect(content).not.toContain('contributor-logs');
|
||||
expect(content).toContain('Operational Self-Improvement');
|
||||
expect(content).toContain('gstack-learnings-log');
|
||||
expect(content).toContain('gstack-learnings-search --limit 3');
|
||||
});
|
||||
|
||||
test('generated SKILL.md with LEARNINGS_LOG contains operational type', () => {
|
||||
// Check a skill that has LEARNINGS_LOG (e.g., review)
|
||||
const content = fs.readFileSync(path.join(ROOT, 'review', 'SKILL.md'), 'utf-8');
|
||||
expect(content).toContain('operational');
|
||||
});
|
||||
|
||||
test('generated SKILL.md contains session awareness', () => {
|
||||
|
||||
@@ -41,8 +41,8 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
||||
'skillmd-no-local-binary': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'skillmd-outside-git': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
|
||||
'contributor-mode': ['SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'session-awareness': ['SKILL.md', 'SKILL.md.tmpl', 'scripts/gen-skill-docs.ts'],
|
||||
'operational-learning': ['scripts/resolvers/preamble.ts', 'bin/gstack-learnings-log'],
|
||||
|
||||
// QA (+ test-server dependency)
|
||||
'qa-quick': ['qa/**', 'browse/src/**', 'browse/test/test-server.ts'],
|
||||
@@ -193,8 +193,8 @@ export const E2E_TIERS: Record<string, 'gate' | 'periodic'> = {
|
||||
'skillmd-setup-discovery': 'gate',
|
||||
'skillmd-no-local-binary': 'gate',
|
||||
'skillmd-outside-git': 'gate',
|
||||
'contributor-mode': 'gate',
|
||||
'session-awareness': 'gate',
|
||||
'operational-learning': 'gate',
|
||||
|
||||
// QA — gate for functional, periodic for quality/benchmarks
|
||||
'qa-quick': 'gate',
|
||||
|
||||
+78
-30
@@ -20,6 +20,7 @@ let tmpDir: string;
|
||||
describeIfSelected('Skill E2E tests', [
|
||||
'browse-basic', 'browse-snapshot', 'skillmd-setup-discovery',
|
||||
'skillmd-no-local-binary', 'skillmd-outside-git', 'session-awareness',
|
||||
'operational-learning',
|
||||
], () => {
|
||||
beforeAll(() => {
|
||||
testServer = startTestServer();
|
||||
@@ -177,49 +178,96 @@ Report the exact output — either "READY: <path>" or "NEEDS_SETUP".`,
|
||||
try { fs.rmSync(nonGitDir, { recursive: true, force: true }); } catch {}
|
||||
}, 60_000);
|
||||
|
||||
testConcurrentIfSelected('contributor-mode', async () => {
|
||||
const contribDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-contrib-'));
|
||||
const logsDir = path.join(contribDir, 'contributor-logs');
|
||||
fs.mkdirSync(logsDir, { recursive: true });
|
||||
testConcurrentIfSelected('operational-learning', async () => {
|
||||
const opDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-oplearn-'));
|
||||
const gstackHome = path.join(opDir, '.gstack-home');
|
||||
|
||||
// Init git repo
|
||||
const run = (cmd: string, args: string[]) =>
|
||||
spawnSync(cmd, args, { cwd: opDir, stdio: 'pipe', timeout: 5000 });
|
||||
run('git', ['init', '-b', 'main']);
|
||||
run('git', ['config', 'user.email', 'test@test.com']);
|
||||
run('git', ['config', 'user.name', 'Test']);
|
||||
fs.writeFileSync(path.join(opDir, 'app.ts'), 'console.log("hello");\n');
|
||||
run('git', ['add', '.']);
|
||||
run('git', ['commit', '-m', 'initial']);
|
||||
|
||||
// Copy bin scripts
|
||||
const binDir = path.join(opDir, 'bin');
|
||||
fs.mkdirSync(binDir, { recursive: true });
|
||||
for (const script of ['gstack-learnings-log', 'gstack-slug']) {
|
||||
fs.copyFileSync(path.join(ROOT, 'bin', script), path.join(binDir, script));
|
||||
fs.chmodSync(path.join(binDir, script), 0o755);
|
||||
}
|
||||
|
||||
// gstack-learnings-log will create the project dir automatically via gstack-slug
|
||||
|
||||
const result = await runSkillTest({
|
||||
prompt: `You are in contributor mode (gstack_contributor=true). You just ran this browse command and it failed:
|
||||
prompt: `You just ran \`npm test\` in this project and it failed with this error:
|
||||
|
||||
$ /nonexistent/browse goto https://example.com
|
||||
/nonexistent/browse: No such file or directory
|
||||
Error: --experimental-vm-modules flag is required for ESM support in this project.
|
||||
Run: npm test --experimental-vm-modules
|
||||
|
||||
Per the contributor mode instructions, file a field report to ${logsDir}/browse-missing-binary.md using the Write tool. Include all required sections: title, what you tried, what happened, rating, repro steps, raw output, what would make it a 10, and the date/version footer.`,
|
||||
workingDirectory: contribDir,
|
||||
Per the Operational Self-Improvement instructions below, log an operational learning about this failure.
|
||||
|
||||
## Operational Self-Improvement
|
||||
|
||||
Before completing, reflect on this session:
|
||||
- Did any commands fail unexpectedly?
|
||||
|
||||
If yes, log an operational learning for future sessions:
|
||||
|
||||
\`\`\`bash
|
||||
GSTACK_HOME="${gstackHome}" ${binDir}/gstack-learnings-log '{"skill":"qa","type":"operational","key":"SHORT_KEY","insight":"DESCRIPTION","confidence":N,"source":"observed"}'
|
||||
\`\`\`
|
||||
|
||||
Replace SHORT_KEY with a kebab-case key like "esm-vm-modules-flag".
|
||||
Replace DESCRIPTION with a one-sentence description of what you learned.
|
||||
Replace N with a confidence score 1-10.
|
||||
|
||||
Log the operational learning now. Then say what you logged.`,
|
||||
workingDirectory: opDir,
|
||||
maxTurns: 5,
|
||||
timeout: 30_000,
|
||||
testName: 'contributor-mode',
|
||||
testName: 'operational-learning',
|
||||
runId,
|
||||
});
|
||||
|
||||
logCost('contributor mode', result);
|
||||
// Override passed: this test intentionally triggers a browse error (nonexistent binary)
|
||||
// so browseErrors will be non-empty — that's expected, not a failure
|
||||
recordE2E(evalCollector, 'contributor mode report', 'Skill E2E tests', result, {
|
||||
passed: result.exitReason === 'success',
|
||||
logCost('operational learning', result);
|
||||
|
||||
const exitOk = ['success', 'error_max_turns'].includes(result.exitReason);
|
||||
|
||||
// Check if learnings file was created with an operational entry
|
||||
// The slug is derived from the git repo (dirname), so search all project dirs
|
||||
let hasOperational = false;
|
||||
const projectsDir = path.join(gstackHome, 'projects');
|
||||
if (fs.existsSync(projectsDir)) {
|
||||
for (const slug of fs.readdirSync(projectsDir)) {
|
||||
const lPath = path.join(projectsDir, slug, 'learnings.jsonl');
|
||||
if (fs.existsSync(lPath)) {
|
||||
const jsonl = fs.readFileSync(lPath, 'utf-8').trim();
|
||||
if (jsonl) {
|
||||
const entries = jsonl.split('\n').map(l => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean);
|
||||
const opEntry = entries.find(e => e.type === 'operational');
|
||||
if (opEntry) {
|
||||
hasOperational = true;
|
||||
console.log(`Operational learning logged: key="${opEntry.key}" insight="${opEntry.insight}" (slug: ${slug})`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
recordE2E(evalCollector, 'operational learning', 'Skill E2E tests', result, {
|
||||
passed: exitOk && hasOperational,
|
||||
});
|
||||
|
||||
// Verify a contributor log was created with expected format
|
||||
const logFiles = fs.readdirSync(logsDir).filter(f => f.endsWith('.md'));
|
||||
expect(logFiles.length).toBeGreaterThan(0);
|
||||
|
||||
// Verify report has key structural sections (agent may phrase differently)
|
||||
const logContent = fs.readFileSync(path.join(logsDir, logFiles[0]), 'utf-8');
|
||||
// Must have a title (# heading)
|
||||
expect(logContent).toMatch(/^#\s/m);
|
||||
// Must mention the failed command or browse
|
||||
expect(logContent).toMatch(/browse|nonexistent|not found|no such file/i);
|
||||
// Must have some kind of rating
|
||||
expect(logContent).toMatch(/rating|\/10/i);
|
||||
// Must have steps or reproduction info
|
||||
expect(logContent).toMatch(/step|repro|reproduce/i);
|
||||
expect(exitOk).toBe(true);
|
||||
expect(hasOperational).toBe(true);
|
||||
|
||||
// Clean up
|
||||
try { fs.rmSync(contribDir, { recursive: true, force: true }); } catch {}
|
||||
try { fs.rmSync(opDir, { recursive: true, force: true }); } catch {}
|
||||
}, 90_000);
|
||||
|
||||
testConcurrentIfSelected('session-awareness', async () => {
|
||||
|
||||
@@ -44,8 +44,9 @@ describeIfSelected('Learnings E2E', ['learnings-show'], () => {
|
||||
fs.chmodSync(path.join(binDir, script), 0o755);
|
||||
}
|
||||
|
||||
// Seed learnings JSONL with 3 entries of different types
|
||||
const slug = 'test-project';
|
||||
// Seed learnings JSONL — slug must match what gstack-slug computes.
|
||||
// With no git remote, gstack-slug falls back to basename(workDir).
|
||||
const slug = path.basename(workDir).replace(/[^a-zA-Z0-9._-]/g, '');
|
||||
const projectDir = path.join(gstackHome, 'projects', slug);
|
||||
fs.mkdirSync(projectDir, { recursive: true });
|
||||
|
||||
@@ -67,6 +68,11 @@ describeIfSelected('Learnings E2E', ['learnings-show'], () => {
|
||||
insight: 'User wants rubocop to run before every commit, no exceptions.',
|
||||
confidence: 10, source: 'user-stated', ts: new Date().toISOString(),
|
||||
},
|
||||
{
|
||||
skill: 'qa', type: 'operational', key: 'test-timeout-flag',
|
||||
insight: 'bun test requires --timeout 30000 for E2E tests in this project.',
|
||||
confidence: 9, source: 'observed', ts: new Date().toISOString(),
|
||||
},
|
||||
];
|
||||
|
||||
fs.writeFileSync(
|
||||
|
||||
@@ -325,62 +325,6 @@ Report the exact output — either "READY: <path>" or "NEEDS_SETUP".`,
|
||||
try { fs.rmSync(nonGitDir, { recursive: true, force: true }); } catch {}
|
||||
}, 60_000);
|
||||
|
||||
testIfSelected('contributor-mode', async () => {
|
||||
const contribDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-contrib-'));
|
||||
const logsDir = path.join(contribDir, 'contributor-logs');
|
||||
fs.mkdirSync(logsDir, { recursive: true });
|
||||
|
||||
// Extract contributor mode instructions from generated SKILL.md
|
||||
const skillMd = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
|
||||
const contribStart = skillMd.indexOf('## Contributor Mode');
|
||||
const contribEnd = skillMd.indexOf('\n## ', contribStart + 1);
|
||||
const contribBlock = skillMd.slice(contribStart, contribEnd > 0 ? contribEnd : undefined);
|
||||
|
||||
const result = await runSkillTest({
|
||||
prompt: `You are in contributor mode (_CONTRIB=true).
|
||||
|
||||
${contribBlock}
|
||||
|
||||
OVERRIDE: Write contributor logs to ${logsDir}/ instead of ~/.gstack/contributor-logs/
|
||||
|
||||
Now try this browse command (it will fail — there is no binary at this path):
|
||||
/nonexistent/path/browse goto https://example.com
|
||||
|
||||
This is a gstack issue (the browse binary is missing/misconfigured).
|
||||
File a contributor report about this issue. Then tell me what you filed.`,
|
||||
workingDirectory: contribDir,
|
||||
maxTurns: 8,
|
||||
timeout: 60_000,
|
||||
testName: 'contributor-mode',
|
||||
runId,
|
||||
});
|
||||
|
||||
logCost('contributor mode', result);
|
||||
// Override passed: this test intentionally triggers a browse error (nonexistent binary)
|
||||
// so browseErrors will be non-empty — that's expected, not a failure
|
||||
recordE2E('contributor mode report', 'Skill E2E tests', result, {
|
||||
passed: result.exitReason === 'success',
|
||||
});
|
||||
|
||||
// Verify a contributor log was created with expected format
|
||||
const logFiles = fs.readdirSync(logsDir).filter(f => f.endsWith('.md'));
|
||||
expect(logFiles.length).toBeGreaterThan(0);
|
||||
|
||||
// Verify new reflection-based format
|
||||
const logContent = fs.readFileSync(path.join(logsDir, logFiles[0]), 'utf-8');
|
||||
expect(logContent).toContain('Hey gstack team');
|
||||
expect(logContent).toContain('What I was trying to do');
|
||||
expect(logContent).toContain('What happened instead');
|
||||
expect(logContent).toMatch(/rating/i);
|
||||
// Verify report has repro steps (agent may use "Steps to reproduce", "Repro Steps", etc.)
|
||||
expect(logContent).toMatch(/repro|steps to reproduce|how to reproduce/i);
|
||||
// Verify report has date/version footer (agent may format differently)
|
||||
expect(logContent).toMatch(/date.*2026|2026.*date/i);
|
||||
|
||||
// Clean up
|
||||
try { fs.rmSync(contribDir, { recursive: true, force: true }); } catch {}
|
||||
}, 90_000);
|
||||
|
||||
testIfSelected('session-awareness', async () => {
|
||||
const sessionDir = fs.mkdtempSync(path.join(os.tmpdir(), 'skill-e2e-session-'));
|
||||
|
||||
|
||||
@@ -735,45 +735,8 @@ describe('investigate skill structure', () => {
|
||||
}
|
||||
});
|
||||
|
||||
// --- Contributor mode preamble structure validation ---
|
||||
|
||||
describe('Contributor mode preamble structure', () => {
|
||||
const skillsWithPreamble = [
|
||||
'SKILL.md', 'browse/SKILL.md', 'qa/SKILL.md',
|
||||
'qa-only/SKILL.md',
|
||||
'setup-browser-cookies/SKILL.md',
|
||||
'ship/SKILL.md', 'review/SKILL.md',
|
||||
'plan-ceo-review/SKILL.md', 'plan-eng-review/SKILL.md',
|
||||
'retro/SKILL.md',
|
||||
'plan-design-review/SKILL.md',
|
||||
'design-review/SKILL.md',
|
||||
'design-consultation/SKILL.md',
|
||||
'document-release/SKILL.md',
|
||||
'canary/SKILL.md',
|
||||
'benchmark/SKILL.md',
|
||||
'land-and-deploy/SKILL.md',
|
||||
'setup-deploy/SKILL.md',
|
||||
];
|
||||
|
||||
for (const skill of skillsWithPreamble) {
|
||||
test(`${skill} has 0-10 rating in contributor mode`, () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, skill), 'utf-8');
|
||||
expect(content).toContain('0-10');
|
||||
expect(content).toContain('Rating');
|
||||
});
|
||||
|
||||
test(`${skill} has "what would make this a 10" field`, () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, skill), 'utf-8');
|
||||
expect(content).toContain('What would make this a 10');
|
||||
});
|
||||
|
||||
test(`${skill} uses periodic reflection (not per-command)`, () => {
|
||||
const content = fs.readFileSync(path.join(ROOT, skill), 'utf-8');
|
||||
expect(content).toContain('workflow step');
|
||||
expect(content).not.toContain('After you use gstack-provided CLIs');
|
||||
});
|
||||
}
|
||||
});
|
||||
// Contributor mode was removed in v0.13.10.0 — replaced by operational self-improvement.
|
||||
// Tests for contributor mode preamble structure are no longer applicable.
|
||||
|
||||
describe('Enum & Value Completeness in review checklist', () => {
|
||||
const checklist = fs.readFileSync(path.join(ROOT, 'review', 'checklist.md'), 'utf-8');
|
||||
|
||||
@@ -101,7 +101,7 @@ describe('selectTests', () => {
|
||||
expect(result.reason).toBe('diff');
|
||||
// Should include tests that depend on gen-skill-docs.ts
|
||||
expect(result.selected).toContain('skillmd-setup-discovery');
|
||||
expect(result.selected).toContain('contributor-mode');
|
||||
expect(result.selected).toContain('session-awareness');
|
||||
expect(result.selected).toContain('journey-ideation');
|
||||
// Should NOT include tests that don't depend on it
|
||||
expect(result.selected).not.toContain('retro');
|
||||
@@ -144,7 +144,7 @@ describe('selectTests', () => {
|
||||
const result = selectTests(['SKILL.md.tmpl'], E2E_TOUCHFILES);
|
||||
// Should select the 7 tests that depend on root SKILL.md
|
||||
expect(result.selected).toContain('skillmd-setup-discovery');
|
||||
expect(result.selected).toContain('contributor-mode');
|
||||
expect(result.selected).toContain('session-awareness');
|
||||
expect(result.selected).toContain('session-awareness');
|
||||
// Also selects journey routing tests (SKILL.md.tmpl in their touchfiles)
|
||||
expect(result.selected).toContain('journey-ideation');
|
||||
|
||||
Reference in New Issue
Block a user