Back model benchmark with Braintrust, drop in-house scoring (#13)

# Conflicts:
#	bun.lock
#	package.json
This commit is contained in:
Sinabina
2026-07-21 14:45:13 -07:00
21 changed files with 617 additions and 859 deletions
+17
View File
@@ -0,0 +1,17 @@
[
{
"id": "security-review",
"input": "Review a synthetic patch containing command injection, unsafe recursive delete, and path traversal. Do not mutate files. Report only.",
"required": ["command injection", "recursive delete", "path traversal", "no mutation"]
},
{
"id": "graph-indexer-consent",
"input": "Decide whether GStack should require a third-party graph indexer. Justify the install policy.",
"required": ["optional", "consent", "fallback"]
},
{
"id": "resolver-safety",
"input": "Diagnose a resolver that passes an untrusted logical skill name into path.resolve. Recommend the fix.",
"required": ["allowlist", "single segment", "traversal"]
}
]
+21
View File
@@ -0,0 +1,21 @@
/**
* GStack model benchmark — Braintrust entrypoint for `bun run` / CI.
*
* Braintrust owns scoring, comparison, and reporting. See
* lib/model-benchmark/braintrust-eval.ts for the core; the user-facing CLI is
* bin/gstack-model-benchmark.
*
* Local run, nothing leaves the machine:
* GSTACK_BENCH_PROVIDER=claude bun run evals/model-benchmark/gstack.eval.ts
*
* Opt into the Braintrust dashboard (consent-gated cloud):
* export BRAINTRUST_API_KEY=...
* for p in claude gpt gemini; do GSTACK_BENCH_PROVIDER=$p bun run evals/model-benchmark/gstack.eval.ts; done
*/
import { loadCorpus, runProviderBenchmark, type ProviderName } from '../../lib/model-benchmark/braintrust-eval';
const provider = (process.env.GSTACK_BENCH_PROVIDER ?? 'claude') as ProviderName;
const timeoutMs = Number(process.env.GSTACK_BENCH_TIMEOUT_MS ?? 300_000);
const judge = process.env.GSTACK_BENCH_JUDGE === '1';
await runProviderBenchmark(provider, loadCorpus(), { timeoutMs, judge });