mirror of
https://github.com/garrytan/gstack.git
synced 2026-09-21 04:10:47 +02:00
Back model benchmark with Braintrust, drop in-house scoring (#13)
# Conflicts: # bun.lock # package.json
This commit is contained in:
@@ -0,0 +1,17 @@
|
||||
[
|
||||
{
|
||||
"id": "security-review",
|
||||
"input": "Review a synthetic patch containing command injection, unsafe recursive delete, and path traversal. Do not mutate files. Report only.",
|
||||
"required": ["command injection", "recursive delete", "path traversal", "no mutation"]
|
||||
},
|
||||
{
|
||||
"id": "graph-indexer-consent",
|
||||
"input": "Decide whether GStack should require a third-party graph indexer. Justify the install policy.",
|
||||
"required": ["optional", "consent", "fallback"]
|
||||
},
|
||||
{
|
||||
"id": "resolver-safety",
|
||||
"input": "Diagnose a resolver that passes an untrusted logical skill name into path.resolve. Recommend the fix.",
|
||||
"required": ["allowlist", "single segment", "traversal"]
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,21 @@
|
||||
/**
|
||||
* GStack model benchmark — Braintrust entrypoint for `bun run` / CI.
|
||||
*
|
||||
* Braintrust owns scoring, comparison, and reporting. See
|
||||
* lib/model-benchmark/braintrust-eval.ts for the core; the user-facing CLI is
|
||||
* bin/gstack-model-benchmark.
|
||||
*
|
||||
* Local run, nothing leaves the machine:
|
||||
* GSTACK_BENCH_PROVIDER=claude bun run evals/model-benchmark/gstack.eval.ts
|
||||
*
|
||||
* Opt into the Braintrust dashboard (consent-gated cloud):
|
||||
* export BRAINTRUST_API_KEY=...
|
||||
* for p in claude gpt gemini; do GSTACK_BENCH_PROVIDER=$p bun run evals/model-benchmark/gstack.eval.ts; done
|
||||
*/
|
||||
import { loadCorpus, runProviderBenchmark, type ProviderName } from '../../lib/model-benchmark/braintrust-eval';
|
||||
|
||||
const provider = (process.env.GSTACK_BENCH_PROVIDER ?? 'claude') as ProviderName;
|
||||
const timeoutMs = Number(process.env.GSTACK_BENCH_TIMEOUT_MS ?? 300_000);
|
||||
const judge = process.env.GSTACK_BENCH_JUDGE === '1';
|
||||
|
||||
await runProviderBenchmark(provider, loadCorpus(), { timeoutMs, judge });
|
||||
Reference in New Issue
Block a user