fix(evals): PR lane selects office-hours-auto-mode by its real test name

The PR paid lane filters audited files with -t built from
CASE_TEST_NAMES[id] ?? id. office-hours-auto-mode's Bun test is named
'AskUserQuestion surfaces when --disallowedTools AskUserQuestion is set',
so the filter matched 0 tests and the shard failed (#3077 eval-slices (6),
run 37741434771). Map the label, and check every audited PR-profile case
is addressable by name. Recorded as non-behavioral for HARNESS_VERSION.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-08 07:20:06 +00:00
1 parent 8c1715d081
commit 086a1339cc
3 files changed
+20 -2

No files matched your search

+1 -1
View File
@@ -10,7 +10,7 @@
"test/helpers/llm-judge.ts"
],
"files": {
"scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":2},
"scripts/lib/paid-cases.ts": {"blob":"c2c16290522e12df7bec88047737920636daad44","since":2,"nonBehavioral":"CASE_TEST_NAMES gains office-hours-auto-mode's real Bun test name: the PR lane's -t filter matched 0 tests and now selects the one test the full lane already runs; no case verdict rule changes"},
"scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":2},
"scripts/lib/paid-report.ts": {"blob":"3fda96a8b1f7f3d55797934847353f254cc4d054","since":2},
"scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":2},
+1
View File
@@ -76,6 +76,7 @@ export const CASE_TEST_NAMES: Record<string, string> = {
'plan-ceo-review-plan-mode': 'first terminal outcome is asked (Step 0 fires before any plan write)',
'plan-eng-review-artifact': 'an interactive review writes one QA test plan about the reviewed change',
'plan-eng-review-artifact-full': 'a fresh interactive review reaches Test review and writes one QA test plan',
'office-hours-auto-mode': 'AskUserQuestion surfaces when --disallowedTools AskUserQuestion is set',
};
export const CASE_KEY_SEPARATOR = '#';
+18 -1
View File
@@ -1,6 +1,9 @@
import { describe, expect, test } from 'bun:test';
import * as fs from 'fs';
import * as path from 'path';
import { CASE_TEST_NAMES, fileCaseRegistration } from '../scripts/lib/paid-cases';
import {
PR_PROFILE_CASE_IDS, PR_PROFILE_MAPS, caseNameAddressable, formatPrCoverageSummary, packageChangeOnlyVersion, selectPrProfile, validatePrProfileInventory,
PR_PROFILE_CASE_IDS, PR_PROFILE_FILES, PR_PROFILE_MAPS, caseNameAddressable, formatPrCoverageSummary, packageChangeOnlyVersion, selectPrProfile, validatePrProfileInventory,
type PrProfileMaps,
} from '../scripts/test-pr-profile';
@@ -161,6 +164,20 @@ describe('fast PR coverage policy', () => {
expect(() => select({ maps: marathon, profile: ['ceo-full'] })).toThrow('broad gate census');
});
test('every audited PR-profile case is addressable by its Bun test name', () => {
// The PR lane runs each audited file with -t built from CASE_TEST_NAMES[id] ?? id;
// office-hours-auto-mode's test is named differently and matched 0 tests (#3077 eval-slices run 37741434771).
const unaddressable = Object.entries(PR_PROFILE_FILES).flatMap(([file, ids]) => {
const source = fs.readFileSync(path.join(import.meta.dir, '..', file), 'utf8');
const computed = fileCaseRegistration(file, source).computed;
return ids.filter((id) => {
const name = CASE_TEST_NAMES[id] ?? id;
return !caseNameAddressable(name, source) && !(computed && source.includes(`'${name}'`));
}).map(id => `${file}: ${id}`);
});
expect(unaddressable).toEqual([]);
});
test('rejects stale profile IDs, incorrect tiers, missing cadence, and unknown selections', () => {
expect(() => select({ profile: ['missing'] })).toThrow('broad gate census');
expect(() => select({ profile: ['ceo-periodic'] })).toThrow('broad gate census');