mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-08 01:38:10 +01:00
test: remove the AI scorer that grades skill prose for clarity (freeze repair, GST-1371) (#4009)
* test: remove the skill-prose clarity LLM judges (GST-1371) Removes the 21 eval cases where an LLM scored a skill's instruction text for clarity, completeness and actionability, with their registrations, input builders, response cache, calibration corpora, the cookie-workflow manual-review approval and scripts/judge-controls.ts. The three remaining skill-llm-eval judges keep their prompts, thresholds and samples; the file's supervision wall shrinks to fit them. Outcome judges are untouched. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test: pin the retired judge-control script and cookie manual-review record as deleted (GST-1371) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test: register the manual-review fixture as free (GST-1371) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test: keep the orphaned clarity-judge calibration corpora for a follow-up deletion (GST-1371) Deleting the four corpus directories alone puts the diff over the /ship review ceiling; they serve no configuration now and go in a follow-up. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * docs(todos): file the clarity-judge removal follow-ups (GST-1371) Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> * test: keep skill-bundle section paths POSIX so references dedupe on Windows (GST-1371) windows-scoped-shard (run 41918796523) failed 'cross-skill references retain complete bytes once': path.join gave the section entry a backslash path, so a reference to the same section was not recognized as already bundled. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --------- Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
Garry Tanandcapy-ai[bot] authored and GitHub committed
1 parent
622e8b1f76
commit
781f928ccf
55 files changed
+209
-3000
No files matched your search
@@ -1,6 +1,6 @@
|
||||
/**
|
||||
* Verified first-attempt reuse for PR-lane E2E shards, on the same receipts as
|
||||
* the workflow-judge reuse (scripts/eval-input-cache.ts).
|
||||
* Verified first-attempt reuse for PR-lane E2E shards, on the receipts of
|
||||
* scripts/eval-input-cache.ts.
|
||||
*
|
||||
* A PR-profile shard (a file, or one case of a case-sharded file) is reused
|
||||
* only when every consumed input is byte-identical to a fresh pass recorded in
|
||||
@@ -309,8 +309,8 @@ export function compareReceiptSources(a: { runId: string; completedAt: number },
|
||||
/**
|
||||
* Planner-side selection: copy `from` into `to`, dropping every pass or panel
|
||||
* receipt that has a same-or-newer negative receipt for its identity, and
|
||||
* every panel receipt that is not a whole PASS panel. Workflow-judge and
|
||||
* other receipts pass through for their own validation at lookup.
|
||||
* every panel receipt that is not a whole PASS panel. Other receipts pass
|
||||
* through for their own validation at lookup.
|
||||
*/
|
||||
export function selectPlanReceipts(from: string, to: string, now = Date.now()): { shipped: number; blocked: string[] } {
|
||||
fs.mkdirSync(to, { recursive: true });
|
||||
|
||||
@@ -302,8 +302,6 @@
|
||||
"test/context-budget-ratchet.test.ts": 586,
|
||||
"test/context-remaining-work-provenance.test.ts": 67,
|
||||
"test/context-restore-fragment-verdict.test.ts": 109,
|
||||
"test/cookie-workflow-judge-input.test.ts": 4466,
|
||||
"test/cookie-workflow-manual-review.test.ts": 256,
|
||||
"test/copilot-windows-bash.test.ts": 63,
|
||||
"test/cost-known.test.ts": 201,
|
||||
"test/coverage-audit.test.ts": 9537,
|
||||
@@ -819,7 +817,6 @@
|
||||
"test/workflow-concurrency.test.ts": 72,
|
||||
"test/workflow-excerpt.test.ts": 328,
|
||||
"test/workflow-gh-token.test.ts": 73,
|
||||
"test/workflow-judge-cache.test.ts": 17402,
|
||||
"test/workflow-required-steps.test.ts": 101,
|
||||
"test/workflow-shell-lint.test.ts": 107,
|
||||
"test/worktree-isolated-shapes.test.ts": 116,
|
||||
|
||||
@@ -441,8 +441,6 @@
|
||||
"test/context-restore-head.test.ts": 1363,
|
||||
"test/context-restore-transcripts.test.ts": 409,
|
||||
"test/context-save-hardening.test.ts": 410,
|
||||
"test/cookie-workflow-judge-input.test.ts": 1960,
|
||||
"test/cookie-workflow-manual-review.test.ts": 74,
|
||||
"test/copilot-windows-bash.test.ts": 12,
|
||||
"test/cost-known.test.ts": 62,
|
||||
"test/coverage-audit-evidence.test.ts": 193,
|
||||
@@ -1463,7 +1461,6 @@
|
||||
"test/workflow-concurrency.test.ts": 17,
|
||||
"test/workflow-excerpt.test.ts": 88,
|
||||
"test/workflow-gh-token.test.ts": 22,
|
||||
"test/workflow-judge-cache.test.ts": 1128,
|
||||
"test/workflow-judge-input.test.ts": 34,
|
||||
"test/workflow-required-steps.test.ts": 17,
|
||||
"test/workflow-shell-lint.test.ts": 26,
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"blob":"e0d18da4ea02ac443835a68992abeea2b47f22fc","since":6,"nonBehavioral":"the headline adds only streamed session cost; a billed session's cost is already in its collector total; cache writes without a 5m key fall back to the total minus 1h; cost fields only"}
|
||||
{"blob":"31355b5a97706943e5fccb2946adeba5fa17a08c","since":6,"nonBehavioral":"GST-1371 removes the 21 skill-prose clarity judges: llm-judge.ts drops judge() and two prompt builders no remaining case calls, paid-report.ts rejects manual-review claims that only the removed cookie judge made, and eval-budgets.ts shrinks the skill-llm-eval file wall to the three kept cases, whose own JUDGE_MS timeouts are unchanged; no kept case's prompt, threshold, samples or verdict can change"}
|
||||
@@ -1 +1 @@
|
||||
{"blob":"667d0e09b20fe5d7366f57b77b5fb9b04baa37ef","since":6}
|
||||
{"blob":"847a2225d7296280571377ea15ede19f24ac3a62","since":6,"nonBehavioral":"GST-1371 removes the 21 skill-prose clarity judges: llm-judge.ts drops judge() and two prompt builders no remaining case calls, paid-report.ts rejects manual-review claims that only the removed cookie judge made, and eval-budgets.ts shrinks the skill-llm-eval file wall to the three kept cases, whose own JUDGE_MS timeouts are unchanged; no kept case's prompt, threshold, samples or verdict can change"}
|
||||
@@ -1 +1 @@
|
||||
{"blob":"ff32d03f32e4af76c6c7314a79cd1b35263ed0dd","since":6,"nonBehavioral":"llm-judge.ts adds a pass-through byte counter (bytesReceived, lastByteMs) to the aborted-judge diagnostic; the request, response bytes, deadline and scoring are unchanged. GSTA-224 B2 (#3122): skill-fixture.ts follows review's renamed Step 2.5 heading so the review E2E cases extract the same section (now the skill's own reworded bytes); llm-judge.ts changes one bullet of the cross-skill consistency judge's intended architecture to name review-bot-history.md beside the older greptile-history.md name, so its calibration corpus (greptile-history inputs) stays valid. No other case's input or verdict can change; the cross-skill judge's judged files changed in this PR, so its owned inputs start its own series"}
|
||||
{"blob":"b804a4585e45b0e5c84bd90b1ede0191b66283cc","since":6,"nonBehavioral":"GST-1371 removes the 21 skill-prose clarity judges: llm-judge.ts drops judge() and two prompt builders no remaining case calls, paid-report.ts rejects manual-review claims that only the removed cookie judge made, and eval-budgets.ts shrinks the skill-llm-eval file wall to the three kept cases, whose own JUDGE_MS timeouts are unchanged; no kept case's prompt, threshold, samples or verdict can change"}
|
||||
@@ -6,14 +6,11 @@
|
||||
*/
|
||||
import {
|
||||
ARM_JUDGE_MODEL, ARM_JUDGE_SCHEMA, buildArmJudgePrompt, parseArmJudgeResponse,
|
||||
buildQaWorkflowJudgePrompt, buildQaHealthRubricJudgePrompt, buildQaAntiRefusalJudgePrompt,
|
||||
buildCrossSkillConsistencyJudgePrompt, buildVoiceDirectiveJudgePrompt,
|
||||
JUDGE_SCORE_SCHEMA, QA_ANTI_REFUSAL_JUDGE_SCHEMA, CROSS_SKILL_CONSISTENCY_JUDGE_SCHEMA,
|
||||
VOICE_DIRECTIVE_JUDGE_SCHEMA, VOICE_DIRECTIVE_DIMENSIONS, JUDGE_SCORE_DIMENSIONS, DEFAULT_JUDGE_MAX_TOKENS,
|
||||
buildQaAntiRefusalJudgePrompt, buildCrossSkillConsistencyJudgePrompt, buildVoiceDirectiveJudgePrompt,
|
||||
QA_ANTI_REFUSAL_JUDGE_SCHEMA, CROSS_SKILL_CONSISTENCY_JUDGE_SCHEMA,
|
||||
VOICE_DIRECTIVE_JUDGE_SCHEMA, VOICE_DIRECTIVE_DIMENSIONS,
|
||||
judgePanelMedian, judgePanelMajority,
|
||||
} from '../test/helpers/llm-judge';
|
||||
import { buildWorkflowJudgePrompt, WORKFLOW_JUDGE_RESPONSE_SCHEMA } from '../test/helpers/workflow-judge-input';
|
||||
import { COOKIE_WORKFLOW_JUDGE } from '../test/helpers/cookie-workflow-judge-input';
|
||||
import { CLAUDE_JUDGE_EVAL_MODEL } from '../lib/eval-model';
|
||||
import type { CalibrationConfig, Verdict } from './judge-calibration';
|
||||
|
||||
@@ -33,23 +30,6 @@ function requireFields(value: unknown, numbers: readonly string[], booleans: rea
|
||||
const meets = (median: Record<string, number>, floors: Record<string, number>): Verdict =>
|
||||
Object.entries(floors).every(([key, floor]) => median[key]! >= floor) ? 'pass' : 'fail';
|
||||
|
||||
const QA_FLOORS = { clarity: 3, completeness: 3, actionability: 4 };
|
||||
const scoreJudge = (floors: Record<string, number>) => ({
|
||||
validate: (value: unknown) => requireFields(value, JUDGE_SCORE_DIMENSIONS),
|
||||
verdict: (samples: unknown[]) => meets(judgePanelMedian<string>(samples as Sample[], JUDGE_SCORE_DIMENSIONS), floors),
|
||||
});
|
||||
|
||||
const workflow = (id: string, priority: number, cases: string[], extra: { agentCapability?: 'frontier'; model?: string; floors?: Record<string, number> } = {}): CalibrationConfig => ({
|
||||
id, priority, cases,
|
||||
model: extra.model ?? CLAUDE_JUDGE_EVAL_MODEL,
|
||||
build: inputs => buildWorkflowJudgePrompt({ judgeContext: inputs.judgeContext!, judgeGoal: inputs.judgeGoal!, agentCapability: extra.agentCapability },
|
||||
{ files: [], text: inputs.bundle! }),
|
||||
oldOptions: { max_tokens: DEFAULT_JUDGE_MAX_TOKENS },
|
||||
newOptions: { max_tokens: DEFAULT_JUDGE_MAX_TOKENS, jsonSchema: WORKFLOW_JUDGE_RESPONSE_SCHEMA },
|
||||
...scoreJudge(extra.floors ?? QA_FLOORS),
|
||||
estimatedOutputTokens: 1200,
|
||||
});
|
||||
|
||||
export const CALIBRATION_CONFIGS: CalibrationConfig[] = [
|
||||
{
|
||||
id: 'arm', priority: 1, cases: ['arm-benchmark (skill-e2e-arm-benchmark.test.ts)'], model: ARM_JUDGE_MODEL,
|
||||
@@ -63,18 +43,6 @@ export const CALIBRATION_CONFIGS: CalibrationConfig[] = [
|
||||
},
|
||||
estimatedOutputTokens: 300,
|
||||
},
|
||||
{
|
||||
id: 'qa-workflow', priority: 2, cases: ['qa/SKILL.md workflow'], model: CLAUDE_JUDGE_EVAL_MODEL,
|
||||
build: inputs => buildQaWorkflowJudgePrompt(inputs.section!),
|
||||
oldOptions: {}, newOptions: { jsonSchema: JUDGE_SCORE_SCHEMA },
|
||||
...scoreJudge(QA_FLOORS), estimatedOutputTokens: 1200,
|
||||
},
|
||||
{
|
||||
id: 'qa-health-rubric', priority: 3, cases: ['qa/SKILL.md health rubric'], model: CLAUDE_JUDGE_EVAL_MODEL,
|
||||
build: inputs => buildQaHealthRubricJudgePrompt(inputs.section!),
|
||||
oldOptions: {}, newOptions: { jsonSchema: JUDGE_SCORE_SCHEMA },
|
||||
...scoreJudge(QA_FLOORS), estimatedOutputTokens: 1200,
|
||||
},
|
||||
{
|
||||
id: 'qa-anti-refusal', priority: 4, cases: ['qa/SKILL.md anti-refusal'], model: CLAUDE_JUDGE_EVAL_MODEL,
|
||||
build: inputs => buildQaAntiRefusalJudgePrompt(inputs.diffAwareSection!, inputs.rulesSection!),
|
||||
@@ -102,16 +70,4 @@ export const CALIBRATION_CONFIGS: CalibrationConfig[] = [
|
||||
Object.fromEntries(VOICE_DIRECTIVE_DIMENSIONS.map(key => [key, 4]))),
|
||||
estimatedOutputTokens: 1200,
|
||||
},
|
||||
workflow('workflow-default', 7, ['document-release/SKILL.md workflow', 'plan-ceo-review/SKILL.md modes', 'plan-eng-review/SKILL.md sections',
|
||||
'plan-design-review/SKILL.md passes', 'design-review/SKILL.md fix loop', 'design-consultation/SKILL.md research',
|
||||
'land-and-deploy/SKILL.md workflow', 'canary/SKILL.md monitoring loop', 'benchmark/SKILL.md perf collection',
|
||||
'setup-deploy/SKILL.md platform setup', 'sync-gbrain/SKILL.md read-only readiness', 'retro/SKILL.md instructions',
|
||||
'qa-only/SKILL.md workflow', 'gstack-upgrade/SKILL.md upgrade flow']),
|
||||
{
|
||||
id: 'cookie-workflow', priority: 8, cases: ['setup-browser-cookies/SKILL.md workflow'], model: COOKIE_WORKFLOW_JUDGE.model,
|
||||
build: inputs => buildWorkflowJudgePrompt(COOKIE_WORKFLOW_JUDGE, { files: [], text: inputs.bundle! }),
|
||||
oldOptions: { max_tokens: DEFAULT_JUDGE_MAX_TOKENS },
|
||||
newOptions: { max_tokens: DEFAULT_JUDGE_MAX_TOKENS, jsonSchema: WORKFLOW_JUDGE_RESPONSE_SCHEMA },
|
||||
...scoreJudge(COOKIE_WORKFLOW_JUDGE.thresholds), estimatedOutputTokens: 1200,
|
||||
},
|
||||
];
|
||||
@@ -1,123 +0,0 @@
|
||||
#!/usr/bin/env -S bun --no-env-file --config=/dev/null
|
||||
/**
|
||||
* C1 workflow-judge control bundles (pre-registered in docs/evals/judge-controls-2026-10.md).
|
||||
*
|
||||
* For each judge that produced low samples, builds the production bundle with the
|
||||
* eval's own reader and prompt builder, derives a negative and a targeted-defect
|
||||
* bundle, and runs one 3-sample panel per bundle with the eval's request options.
|
||||
*
|
||||
* Usage:
|
||||
* bun run scripts/judge-controls.ts --list (builds every bundle, no API calls)
|
||||
* bun run scripts/judge-controls.ts --run (36 judge calls; writes docs/evals/judge-controls/results.json)
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { buildQaWorkflowJudgePrompt, callJudge, judgePanelMean, judgePanelMedian, DEFAULT_JUDGE_MAX_TOKENS, JUDGE_SCORE_DIMENSIONS, type CallJudgeOptions } from '../test/helpers/llm-judge';
|
||||
import { buildWorkflowJudgePrompt, readWorkflowJudgeInput, QA_DISCOVERY_REFERENCES, WORKFLOW_JUDGE_RESPONSE_SCHEMA } from '../test/helpers/workflow-judge-input';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const FLOORS = { clarity: 3, completeness: 3, actionability: 4 } as const;
|
||||
const SAMPLES = 3;
|
||||
|
||||
type Judge = {
|
||||
name: string;
|
||||
read: () => string;
|
||||
prompt: (text: string) => string;
|
||||
options: CallJudgeOptions;
|
||||
defect: (text: string) => string;
|
||||
};
|
||||
|
||||
function cut(text: string, start: string, end: string): string {
|
||||
const from = text.indexOf(start);
|
||||
const to = text.indexOf(end, from + start.length);
|
||||
if (from < 0 || to < 0) throw new Error(`defect anchor missing: ${start} … ${end}`);
|
||||
return text.slice(0, from) + text.slice(to);
|
||||
}
|
||||
|
||||
function insertAfter(text: string, anchor: string, line: string): string {
|
||||
const at = text.indexOf(anchor);
|
||||
if (at < 0) throw new Error(`defect anchor missing: ${anchor}`);
|
||||
return text.slice(0, at + anchor.length) + '\n\n' + line + text.slice(at + anchor.length);
|
||||
}
|
||||
|
||||
const workflow = (skillPath: string, startMarker: string, endMarker: string | null, references: readonly string[] | undefined) =>
|
||||
() => readWorkflowJudgeInput({ root: ROOT, skillPath, startMarker, endMarker, references }).text;
|
||||
|
||||
const JUDGES: Judge[] = [
|
||||
{
|
||||
name: 'qa/SKILL.md workflow',
|
||||
read: workflow('qa/SKILL.md', '# /qa: Test', null,
|
||||
['qa/templates/functional-report-template.md', 'qa/templates/qa-report-template.md', 'qa/references/issue-taxonomy.md']),
|
||||
prompt: buildQaWorkflowJudgePrompt,
|
||||
options: {},
|
||||
defect: text => cut(text, '## Phase 10: Report', '## Phase 11:'),
|
||||
},
|
||||
{
|
||||
name: 'qa-only/SKILL.md workflow',
|
||||
read: workflow('qa-only/SKILL.md', '# /qa-only:', null, QA_DISCOVERY_REFERENCES.filter(file => file !== 'qa/sections/exploratory.md')),
|
||||
prompt: text => buildWorkflowJudgePrompt({ judgeContext: 'a report-only QA testing workflow',
|
||||
judgeGoal: 'how to select browser/native functional/mixed targets, explore safely with repository tools, report exact contract evidence and coverage limits, conditionally load browser/DX instructions and never mutate product/tests/Git through any tool' }, { files: [], text }),
|
||||
options: { max_tokens: DEFAULT_JUDGE_MAX_TOKENS, jsonSchema: WORKFLOW_JUDGE_RESPONSE_SCHEMA },
|
||||
defect: text => insertAfter(text, '# /qa-only: Report-Only QA Testing', 'After reporting each bug, fix it in the source and commit the fix.'),
|
||||
},
|
||||
{
|
||||
name: 'review/SKILL.md workflow',
|
||||
read: workflow('review/SKILL.md', '## Step 0: Detect platform and base branch', null,
|
||||
[...QA_DISCOVERY_REFERENCES, 'review/checklist.md', 'review/specialists/testing.md']),
|
||||
prompt: text => buildWorkflowJudgePrompt({ judgeContext: 'a pre-landing review with bounded exploratory QA',
|
||||
judgeGoal: 'how to review and explore changed behavior even for small diffs without a plan or server, preserve report-only discovery and the test_stub ASK gate, handle incomplete probes honestly, and rerun affected evidence after approved repairs',
|
||||
agentCapability: 'frontier' }, { files: [], text }),
|
||||
options: { max_tokens: DEFAULT_JUDGE_MAX_TOKENS },
|
||||
defect: text => cut(text, '## Step 3: Get the diff', '## Step 3.4'),
|
||||
},
|
||||
{
|
||||
name: 'setup-deploy/SKILL.md platform setup',
|
||||
read: workflow('setup-deploy/SKILL.md', '### Step 2: Detect platform', '## Important Rules', undefined),
|
||||
prompt: text => buildWorkflowJudgePrompt({ judgeContext: 'a deployment configuration setup workflow that detects deploy platforms and writes config to CLAUDE.md',
|
||||
judgeGoal: 'how to detect deploy platforms (Fly.io, Render, Vercel, Netlify, Heroku, GitHub Actions, custom), gather platform-specific configuration (URLs, status commands, health checks, custom hooks), and persist everything to CLAUDE.md for future automated use' }, { files: [], text }),
|
||||
options: { max_tokens: DEFAULT_JUDGE_MAX_TOKENS, jsonSchema: WORKFLOW_JUDGE_RESPONSE_SCHEMA },
|
||||
defect: text => {
|
||||
const step = text.indexOf('### Step 2: Detect platform');
|
||||
const open = text.indexOf('```bash', step);
|
||||
const close = text.indexOf('```', open + 7);
|
||||
if (step < 0 || open < 0 || close < 0) throw new Error('defect anchor missing: Step 2 bash block');
|
||||
return text.slice(0, open) + text.slice(close + 3);
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
/** Keep only file boundaries and headings. */
|
||||
const negative = (text: string) => text.split('\n')
|
||||
.filter(line => /^--- (BEGIN|END) FILE /.test(line) || /^#{1,6} /.test(line)).join('\n');
|
||||
|
||||
function bundles(judge: Judge): Record<'positive' | 'negative' | 'defect', string> {
|
||||
const positive = judge.read();
|
||||
return { positive, negative: negative(positive), defect: judge.defect(positive) };
|
||||
}
|
||||
|
||||
const mode = process.argv[2];
|
||||
if (mode !== '--list' && mode !== '--run') {
|
||||
console.error('usage: bun run scripts/judge-controls.ts --list | --run');
|
||||
process.exit(2);
|
||||
}
|
||||
if (mode === '--list') {
|
||||
for (const judge of JUDGES) {
|
||||
const built = bundles(judge);
|
||||
console.log(judge.name, Object.fromEntries(Object.entries(built).map(([k, v]) => [k, v.length])));
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
type Sample = { clarity: number; completeness: number; actionability: number; reasoning: string };
|
||||
const results = await Promise.all(JUDGES.flatMap(judge => Object.entries(bundles(judge)).map(async ([bundle, text]) => {
|
||||
const prompt = judge.prompt(text);
|
||||
const samples = await Promise.all(Array.from({ length: SAMPLES }, () => callJudge<Sample>(prompt, undefined, judge.options)));
|
||||
const mean = judgePanelMean(samples, JUDGE_SCORE_DIMENSIONS);
|
||||
const median = judgePanelMedian(samples, JUDGE_SCORE_DIMENSIONS);
|
||||
const verdict = Object.entries(FLOORS).every(([key, floor]) => median[key as keyof typeof FLOORS] >= floor) ? 'pass' : 'fail';
|
||||
return { judge: judge.name, bundle, verdict, median, mean, samples };
|
||||
})));
|
||||
const out = path.join(ROOT, 'docs/evals/judge-controls/results.json');
|
||||
fs.mkdirSync(path.dirname(out), { recursive: true });
|
||||
fs.writeFileSync(out, JSON.stringify({ recorded_at: new Date().toISOString(), results }, null, 2) + '\n');
|
||||
for (const r of results) console.log(`${r.judge} | ${r.bundle} | ${r.verdict} | median ${JSON.stringify(r.median)} mean ${JSON.stringify(r.mean)}`);
|
||||
@@ -14,7 +14,6 @@ import {
|
||||
import { readSessionLedger, type SessionLedgerRow } from '../../test/helpers/session-ledger';
|
||||
import { publishedFence, sanitizeFixedFenceLine } from './published-text';
|
||||
import { E2E_KINDS } from '../../test/helpers/touchfiles-data';
|
||||
import { manualReviewProblem } from '../../test/helpers/cookie-workflow-manual-review';
|
||||
import { writeNegativeReceipt, writePanelReceipt } from '../e2e-shard-reuse';
|
||||
import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from '../../test/helpers/touchfiles';
|
||||
import { type EvalHost, hostSkipReason } from '../../test/helpers/e2e-hosts';
|
||||
@@ -617,7 +616,6 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
flaky: number; total: number; executed: number; reused: number; passed: number;
|
||||
failed: number; manual_accepted: number; attempts: number }> = [];
|
||||
const manualProblems: string[] = [];
|
||||
const manualClaims = new Map<string, string>();
|
||||
const recordsByShard = new Map<string, any[]>();
|
||||
let costUsd = 0;
|
||||
let flakyCollectorFailures = 0;
|
||||
@@ -638,26 +636,15 @@ export function runPaidReport(reportDir: string, options: { writeDurations?: boo
|
||||
if (shard) recordsByShard.set(shard.file, [...(recordsByShard.get(shard.file) ?? []), ...parsed.tests]);
|
||||
// Trial records are verdict input for panelVerdict(), never collector gates.
|
||||
if (shard?.trial) continue;
|
||||
const seen = new Map<string, number>();
|
||||
for (const [index, entry] of parsed.tests.entries()) {
|
||||
if (!entry || typeof entry !== 'object' || typeof entry.name !== 'string' || !entry.name
|
||||
|| typeof entry.passed !== 'boolean') {
|
||||
manualProblems.push(`${rel}: attempt ${index + 1}: malformed collector entry (name/passed required)`);
|
||||
continue;
|
||||
}
|
||||
const key = `${entry.suite ?? ''}\0${entry.name}`;
|
||||
const occurrence = (seen.get(key) ?? 0) + 1;
|
||||
seen.set(key, occurrence);
|
||||
if (Object.hasOwn(entry, 'manual_review') && occurrence !== 1) {
|
||||
manualProblems.push(`${rel}: attempt ${index + 1}: manual review is only valid on the first case attempt`);
|
||||
}
|
||||
if (Object.hasOwn(entry, 'manual_review')) {
|
||||
const previous = manualClaims.get(key);
|
||||
if (previous && previous !== rel) manualProblems.push(`${rel}: duplicate manual-review claim for ${entry.name} (also in ${previous})`);
|
||||
else manualClaims.set(key, rel);
|
||||
manualProblems.push(`${rel}: attempt ${index + 1}: manual-review claims were retired with the workflow judges`);
|
||||
}
|
||||
const problem = manualReviewProblem(entry, rootDir);
|
||||
if (problem) manualProblems.push(`${rel}: attempt ${index + 1}: ${problem}`);
|
||||
}
|
||||
collectors.push(parsed);
|
||||
const counts = collectorOutcomeCounts([parsed]);
|
||||
|
||||
@@ -120,7 +120,6 @@
|
||||
"test/claude-code-runner.test.ts\tTS7006\tParameter 'callback' implicitly has an 'any' type.": 1,
|
||||
"test/claude-code-runner.test.ts\tTS7006\tParameter 'delay' implicitly has an 'any' type.": 1,
|
||||
"test/claude-code-runner.test.ts\tTS7019\tRest parameter 'args' implicitly has an 'any[]' type.": 1,
|
||||
"test/cookie-workflow-manual-review.test.ts\tTS2769\tNo overload matches this call. The last overload gave the following error. Argument of type 'ManualJudgeReview | undefined' is not assignable to parameter of type 'ManualJudgeReview | null'. Type 'undefined' is not assignable to type 'ManualJudgeReview | null'.": 1,
|
||||
"test/cso-eval.test.ts\tTS2345\tArgument of type '(command: string, args: readonly string[] | undefined, options: any) => string' is not assignable to parameter of type '{ (file: string): NonSharedBuffer; (file: string, options: ExecFileSyncOptionsWithStringEncoding): string; (file: string, options: ExecFileSyncOptionsWithBufferEncoding): NonSharedBuffer; (file: string, options?: ExecFileSyncOptions | undefined): string | NonSharedBuffer; (file: string, args: readonly string[]): Non...'. Target signature provides too few arguments. Expected 3 or more, but got 1.": 1,
|
||||
"test/cso-eval.test.ts\tTS2790\tThe operand of a 'delete' operator must be optional.": 1,
|
||||
"test/cso-eval.test.ts\tTS7005\tVariable 'receipts' implicitly has an 'any[]' type.": 1,
|
||||
|
||||
Reference in new issue
Block a user