mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-18 22:24:17 +01:00
* Load the Capy crew skills only on Capy machines (GST-1833) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Add changelog entry for Capy-only crew skills (GST-1833) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Move the Capy-only crew tooling into crew/ and run its tests only when needed (GST-1833) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Test that the Capy launch file puts crew/bin on PATH Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Count any ~/.capy as a Capy machine, so a Mac running Capy keeps /captain Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Fix the tests and fixtures that broke when crew tooling moved into crew/ Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Plan every crew test when a pull request's merge-base can't be read Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Add a test that checks each crew-only module left the root tree for crew/ Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Run crew tests on pull requests that change bin/ commands or workflows they drive Review-Finding: p1/specialist-batched#1 reader=testing Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Keep running every crew test on every pull request; the skip comes in a later change Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Keep copied installs and older release bisects working after the crew/ move Setup now copies crew/lib into every host's runtime root beside bin and lib, so Windows installs (which copy instead of linking) can still start gstack-ship-receipt, gstack-pregate and the other commands that import it. The release bisect runs the train-gate script from wherever the checked-out commit has it, so a bisect over commits from before the move still gets a verdict. Review-Finding: p1/outside-voice#1 reader=outside-voice Review-Finding: p1/outside-voice#2 reader=outside-voice Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Say in the changelog that Windows installs keep working after the crew/ move Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Point the harness guide's merge-queue rows at crew/lib/ where those modules moved Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * Point the plain-title test at the wave module's new home in crew/ (GST-1833) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
24 lines
1.1 KiB
TypeScript
24 lines
1.1 KiB
TypeScript
/**
|
|
* W2 calibration request configurations, keyed by distinct request (prompt
|
|
* builder, model, max_tokens, stream, effort, thresholds), not by test case.
|
|
* Each config builds prompts through the same builder the eval sends; "old" is
|
|
* today's request and "new" adds only the JSON schema (comparison 1).
|
|
*/
|
|
import { ARM_JUDGE_MODEL, ARM_JUDGE_SCHEMA, buildArmJudgePrompt, parseArmJudgeResponse } from '../../test/helpers/llm-judge';
|
|
import type { CalibrationConfig } from '../../scripts/judge-calibration';
|
|
|
|
export const CALIBRATION_CONFIGS: CalibrationConfig[] = [
|
|
{
|
|
id: 'arm', priority: 1, cases: ['arm-benchmark (skill-e2e-arm-benchmark.test.ts)'], model: ARM_JUDGE_MODEL,
|
|
build: inputs => buildArmJudgePrompt(inputs.task!.trimEnd(), inputs.diff!),
|
|
oldOptions: {},
|
|
newOptions: { jsonSchema: ARM_JUDGE_SCHEMA },
|
|
validate: value => { parseArmJudgeResponse(value); },
|
|
verdict: samples => {
|
|
const scores = samples.map(sample => parseArmJudgeResponse(sample).over_engineering).sort((a, b) => a - b);
|
|
return scores[Math.floor(scores.length / 2)]! <= 1 ? 'pass' : 'fail';
|
|
},
|
|
estimatedOutputTokens: 300,
|
|
},
|
|
];
|