mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-13 11:57:19 +01:00
fix(evals): PR lane selects office-hours-auto-mode by its real test name
The PR paid lane filters audited files with -t built from CASE_TEST_NAMES[id] ?? id. office-hours-auto-mode's Bun test is named 'AskUserQuestion surfaces when --disallowedTools AskUserQuestion is set', so the filter matched 0 tests and the shard failed (#3077 eval-slices (6), run 37741434771). Map the label, and check every audited PR-profile case is addressable by name. Recorded as non-behavioral for HARNESS_VERSION. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
8c1715d081
commit
086a1339cc
3 files changed
+20
-2
No files matched your search
@@ -10,7 +10,7 @@
|
||||
"test/helpers/llm-judge.ts"
|
||||
],
|
||||
"files": {
|
||||
"scripts/lib/paid-cases.ts": {"blob":"9bacd52446192a7ecf3dfaae901ea443cf9c9ae2","since":2},
|
||||
"scripts/lib/paid-cases.ts": {"blob":"c2c16290522e12df7bec88047737920636daad44","since":2,"nonBehavioral":"CASE_TEST_NAMES gains office-hours-auto-mode's real Bun test name: the PR lane's -t filter matched 0 tests and now selects the one test the full lane already runs; no case verdict rule changes"},
|
||||
"scripts/lib/paid-plan.ts": {"blob":"45721ce128a112666a0caed6c13c66637b74ca54","since":2},
|
||||
"scripts/lib/paid-report.ts": {"blob":"3fda96a8b1f7f3d55797934847353f254cc4d054","since":2},
|
||||
"scripts/lib/published-text.ts": {"blob":"3f2e4d616b5dd0204035710c66b64d49055c5714","since":2},
|
||||
|
||||
@@ -76,6 +76,7 @@ export const CASE_TEST_NAMES: Record<string, string> = {
|
||||
'plan-ceo-review-plan-mode': 'first terminal outcome is asked (Step 0 fires before any plan write)',
|
||||
'plan-eng-review-artifact': 'an interactive review writes one QA test plan about the reviewed change',
|
||||
'plan-eng-review-artifact-full': 'a fresh interactive review reaches Test review and writes one QA test plan',
|
||||
'office-hours-auto-mode': 'AskUserQuestion surfaces when --disallowedTools AskUserQuestion is set',
|
||||
};
|
||||
|
||||
export const CASE_KEY_SEPARATOR = '#';
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { CASE_TEST_NAMES, fileCaseRegistration } from '../scripts/lib/paid-cases';
|
||||
import {
|
||||
PR_PROFILE_CASE_IDS, PR_PROFILE_MAPS, caseNameAddressable, formatPrCoverageSummary, packageChangeOnlyVersion, selectPrProfile, validatePrProfileInventory,
|
||||
PR_PROFILE_CASE_IDS, PR_PROFILE_FILES, PR_PROFILE_MAPS, caseNameAddressable, formatPrCoverageSummary, packageChangeOnlyVersion, selectPrProfile, validatePrProfileInventory,
|
||||
type PrProfileMaps,
|
||||
} from '../scripts/test-pr-profile';
|
||||
|
||||
@@ -161,6 +164,20 @@ describe('fast PR coverage policy', () => {
|
||||
expect(() => select({ maps: marathon, profile: ['ceo-full'] })).toThrow('broad gate census');
|
||||
});
|
||||
|
||||
test('every audited PR-profile case is addressable by its Bun test name', () => {
|
||||
// The PR lane runs each audited file with -t built from CASE_TEST_NAMES[id] ?? id;
|
||||
// office-hours-auto-mode's test is named differently and matched 0 tests (#3077 eval-slices run 37741434771).
|
||||
const unaddressable = Object.entries(PR_PROFILE_FILES).flatMap(([file, ids]) => {
|
||||
const source = fs.readFileSync(path.join(import.meta.dir, '..', file), 'utf8');
|
||||
const computed = fileCaseRegistration(file, source).computed;
|
||||
return ids.filter((id) => {
|
||||
const name = CASE_TEST_NAMES[id] ?? id;
|
||||
return !caseNameAddressable(name, source) && !(computed && source.includes(`'${name}'`));
|
||||
}).map(id => `${file}: ${id}`);
|
||||
});
|
||||
expect(unaddressable).toEqual([]);
|
||||
});
|
||||
|
||||
test('rejects stale profile IDs, incorrect tiers, missing cadence, and unknown selections', () => {
|
||||
expect(() => select({ profile: ['missing'] })).toThrow('broad gate census');
|
||||
expect(() => select({ profile: ['ceo-periodic'] })).toThrow('broad gate census');
|
||||
|
||||
Reference in new issue
Block a user