CalvinBackup/gstack
1
0
Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-09-18 02:42:25 +02:00
Code Issues Packages Projects Releases Wiki Activity

test: cover first-run detection + repoint browse-content assertions to /browse

Browse Source 
- New unit tests for every detection bucket, the eval-safe enum contract, and the
  first-run gating (test/preamble-first-task-scaffold.test.ts); periodic E2E that runs
  the detector through the real harness (test/skill-e2e-first-task-scaffold.test.ts).
- Repoint browse-content assertions (gen-skill-docs, audit-compliance, skill-validation,
  LLM-judge eval) from the root skill to browse/SKILL.md following the router split;
  add a regression pinning that the router carries no browse body.
- Register first-task-scaffold touchfiles + periodic tier; bump parity/carve size caps
  ~1-2KB per skill for the shared first-run-guidance preamble section.
- Refresh ship golden fixtures for the preamble addition.
This commit is contained in:
Garry Tan
2026-06-21 07:16:24 -07:00
parent 938fa4a035
commit 866159861d
12 changed files with 438 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/skill-llm-eval.test.ts
+19 -17
View File
@@ -65,10 +65,10 @@ describeIfSelected('LLM-as-judge quality evals', [
], () => {
testIfSelected('command reference table', async () => {
const t0 = Date.now();
const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
const start = content.indexOf('## Command Reference');
const end = content.indexOf('## Tips');
const section = content.slice(start, end);
// P2 (v1.2.0): the command reference moved from the root router to browse/SKILL.md.
const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8');
const start = content.indexOf('## Full Command List');
const section = content.slice(start);
const scores = await judge('command reference table', section);
console.log('Command reference scores:', JSON.stringify(scores, null, 2));
@@ -94,9 +94,10 @@ describeIfSelected('LLM-as-judge quality evals', [
testIfSelected('snapshot flags reference', async () => {
const t0 = Date.now();
const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
const start = content.indexOf('## Snapshot System');
const end = content.indexOf('## Command Reference');
// P2 (v1.2.0): snapshot flags moved from the root router to browse/SKILL.md.
const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8');
const start = content.indexOf('## Snapshot Flags');
const end = content.indexOf('## CSS Inspector');
const section = content.slice(start, end);
const scores = await judge('snapshot flags reference', section);
@@ -145,9 +146,10 @@ describeIfSelected('LLM-as-judge quality evals', [
testIfSelected('setup block', async () => {
const t0 = Date.now();
const content = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
// P2 (v1.2.0): the browse setup block moved from the root router to browse/SKILL.md.
const content = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8');
const setupStart = content.indexOf('## SETUP');
const setupEnd = content.indexOf('## IMPORTANT');
const setupEnd = content.indexOf('## Core QA Patterns');
const section = content.slice(setupStart, setupEnd);
const scores = await judge('setup/binary discovery instructions', section);
@@ -172,10 +174,10 @@ describeIfSelected('LLM-as-judge quality evals', [
testIfSelected('regression vs baseline', async () => {
const t0 = Date.now();
const generated = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
const genStart = generated.indexOf('## Command Reference');
const genEnd = generated.indexOf('## Tips');
const genSection = generated.slice(genStart, genEnd);
// P2 (v1.2.0): the command reference moved from the root router to browse/SKILL.md.
const generated = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8');
const genStart = generated.indexOf('## Full Command List');
const genSection = generated.slice(genStart);
const baseline = `## Command Reference
@@ -480,10 +482,10 @@ describeIfSelected('Baseline score pinning', ['baseline score pinning'], () => {
const baselines = JSON.parse(fs.readFileSync(baselinesPath, 'utf-8'));
const regressions: string[] = [];
const skillContent = fs.readFileSync(path.join(ROOT, 'SKILL.md'), 'utf-8');
const cmdStart = skillContent.indexOf('## Command Reference');
const cmdEnd = skillContent.indexOf('## Tips');
const cmdSection = skillContent.slice(cmdStart, cmdEnd);
// P2 (v1.2.0): the command reference moved from the root router to browse/SKILL.md.
const skillContent = fs.readFileSync(path.join(ROOT, 'browse', 'SKILL.md'), 'utf-8');
const cmdStart = skillContent.indexOf('## Full Command List');
const cmdSection = skillContent.slice(cmdStart);
const cmdScores = await judge('command reference table', cmdSection);
for (const dim of ['clarity', 'completeness', 'actionability'] as const) {