CalvinBackup/gstack
1
0 Fork 0
mirror of https://github.com/garrytan/gstack.git synced 2026-10-03 01:46:55 +02:00
Code Issues Packages Projects Releases Wiki Activity

v1.91.5.0 feat: balanced free-suite shards, test:ubicloud, and faster PR eval lane (#2989)

Browse Source 
* v1.91.5.0 feat: balanced free-suite shards, 16-way Linux runs, and bun run test:ubicloud

* chore: regenerate agents digest for v1.91.5.0

* ci: serial flaky retry and flake ledger for the Windows free lane

* ci: cancel superseded eval runs; relax LLM-judge clarity bar to 3
This commit is contained in:
Garry Tan authored and GitHub committed 2026-09-28 16:00:12 -07:00
1 parent d2a0bbcf4c
commit 080a655791
30 files changed
+2474 -1784
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

test/skill-llm-eval.test.ts
+11 -11
View File
@@ -110,14 +110,14 @@ describeIfSelected('LLM-as-judge quality evals', [
name: 'command reference table',
suite: 'LLM-as-judge quality evals',
tier: 'llm-judge',
passed: scores.clarity >= 4 && scores.completeness >= 3 && scores.actionability >= 4,
passed: scores.clarity >= 3 && scores.completeness >= 3 && scores.actionability >= 4,
duration_ms: Date.now() - t0,
cost_usd: 0.02,
judge_scores: { clarity: scores.clarity, completeness: scores.completeness, actionability: scores.actionability },
judge_reasoning: scores.reasoning,
});
expect(scores.clarity).toBeGreaterThanOrEqual(4);
expect(scores.clarity).toBeGreaterThanOrEqual(3);
expect(scores.completeness).toBeGreaterThanOrEqual(3);
expect(scores.actionability).toBeGreaterThanOrEqual(4);
}, JUDGE_MS);
@@ -136,14 +136,14 @@ describeIfSelected('LLM-as-judge quality evals', [
name: 'snapshot flags reference',
suite: 'LLM-as-judge quality evals',
tier: 'llm-judge',
passed: scores.clarity >= 4 && scores.completeness >= 4 && scores.actionability >= 4,
passed: scores.clarity >= 3 && scores.completeness >= 4 && scores.actionability >= 4,
duration_ms: Date.now() - t0,
cost_usd: 0.02,
judge_scores: { clarity: scores.clarity, completeness: scores.completeness, actionability: scores.actionability },
judge_reasoning: scores.reasoning,
});
expect(scores.clarity).toBeGreaterThanOrEqual(4);
expect(scores.clarity).toBeGreaterThanOrEqual(3);
expect(scores.completeness).toBeGreaterThanOrEqual(4);
expect(scores.actionability).toBeGreaterThanOrEqual(4);
}, JUDGE_MS);
@@ -160,14 +160,14 @@ describeIfSelected('LLM-as-judge quality evals', [
name: 'browse/SKILL.md reference',
suite: 'LLM-as-judge quality evals',
tier: 'llm-judge',
passed: scores.clarity >= 4 && scores.completeness >= 4 && scores.actionability >= 4,
passed: scores.clarity >= 3 && scores.completeness >= 4 && scores.actionability >= 4,
duration_ms: Date.now() - t0,
cost_usd: 0.02,
judge_scores: { clarity: scores.clarity, completeness: scores.completeness, actionability: scores.actionability },
judge_reasoning: scores.reasoning,
});
expect(scores.clarity).toBeGreaterThanOrEqual(4);
expect(scores.clarity).toBeGreaterThanOrEqual(3);
expect(scores.completeness).toBeGreaterThanOrEqual(4);
expect(scores.actionability).toBeGreaterThanOrEqual(4);
}, JUDGE_MS);
@@ -349,14 +349,14 @@ ${section}`);
name: 'qa/SKILL.md workflow',
suite: 'QA skill quality evals',
tier: 'llm-judge',
passed: scores.clarity >= 4 && scores.completeness >= 3 && scores.actionability >= 4,
passed: scores.clarity >= 3 && scores.completeness >= 3 && scores.actionability >= 4,
duration_ms: Date.now() - t0,
cost_usd: 0.02,
judge_scores: { clarity: scores.clarity, completeness: scores.completeness, actionability: scores.actionability },
judge_reasoning: scores.reasoning,
});
expect(scores.clarity).toBeGreaterThanOrEqual(4);
expect(scores.clarity).toBeGreaterThanOrEqual(3);
// Completeness scores 3 when judge notes the health rubric is in a separate
// section (the eval only passes the Workflow section, not the full document).
expect(scores.completeness).toBeGreaterThanOrEqual(3);
@@ -391,14 +391,14 @@ ${section}`);
name: 'qa/SKILL.md health rubric',
suite: 'QA skill quality evals',
tier: 'llm-judge',
passed: scores.clarity >= 4 && scores.completeness >= 3 && scores.actionability >= 4,
passed: scores.clarity >= 3 && scores.completeness >= 3 && scores.actionability >= 4,
duration_ms: Date.now() - t0,
cost_usd: 0.02,
judge_scores: { clarity: scores.clarity, completeness: scores.completeness, actionability: scores.actionability },
judge_reasoning: scores.reasoning,
});
expect(scores.clarity).toBeGreaterThanOrEqual(4);
expect(scores.clarity).toBeGreaterThanOrEqual(3);
expect(scores.completeness).toBeGreaterThanOrEqual(3);
expect(scores.actionability).toBeGreaterThanOrEqual(4);
}, JUDGE_MS);
@@ -664,7 +664,7 @@ async function runWorkflowJudge(opts: {
// Timeout/retry finalizes once; late provider continuations cannot publish evidence.
const work = async () => {
checkActive();
const thresholds = { clarity: 4, completeness: 3, actionability: 4, ...opts.thresholds };
const thresholds = { clarity: 3, completeness: 3, actionability: 4, ...opts.thresholds };
const input = opts.readInput ? opts.readInput() : readWorkflowJudgeInput({ root: ROOT, skillPath: opts.skillPath,
startMarker: opts.startMarker, endMarker: opts.endMarker });
checkActive();