test(evals): name the office-hours-auto-mode case so the PR lane can select it

The PR lane runs each short probe with a Bun name pattern ending in its case
id. The office-hours AskUserQuestion-blocked smoke had a descriptive test
name and no CASE_TEST_NAMES label, so whenever office-hours changed its shard
matched 0 tests and failed in 0 s (E2E Evals slice 6 on #3075). Name the
test after its id, keep the description on the describe, and check that
every literal-named PR probe has a test the pattern selects.

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-08 06:48:48 +00:00
1 parent b5604e91be
commit f555e9da54
2 files changed
+17 -2

No files matched your search

+14
View File
@@ -12,6 +12,7 @@ import {
type PaidCaseSelection, type SliceResult,
} from '../scripts/test-paid-shards';
import { PR_PROFILE_CASE_IDS, PR_PROFILE_FILES } from '../scripts/test-pr-profile';
import { fileCaseRegistration } from '../scripts/lib/paid-cases';
import { E2E_TOUCHFILES } from './helpers/touchfiles-data';
import { resolveModuleSelection } from './helpers/e2e-helpers';
@@ -64,6 +65,19 @@ describe('PR profile paid-runner integration', () => {
}
});
test('every short probe with literal test names has a test the PR lane name pattern selects', () => {
for (const [file, cases] of Object.entries(PR_PROFILE_FILES)) {
const source = fs.readFileSync(path.join(ROOT, file), 'utf8');
if (fileCaseRegistration(file, source).computed) continue;
const titles = [...source.matchAll(/\b(?:describe\w*|test\w*|it)(?:\.\w+)?\(\s*(['"`])((?:(?!\1)[^\\]|\\.)*)\1/g)].map(m => m[2]!);
const names = titles.flatMap(outer => [outer, ...titles.map(inner => `${outer} ${inner}`)]);
for (const id of cases) {
const pattern = new RegExp(prProfileTestNamePattern(file, { e2e: [id], judges: [] }));
expect(names.some(name => pattern.test(name)), `${file} has no test named for ${id}`).toBe(true);
}
}
});
test('planner binds E2E and judge IDs and plans the CEO plan-mode probe in the PR lane', () => {
const manifest = ceoManifest();
expect(manifest.profile).toBe('pr');
@@ -24,12 +24,13 @@ import { assertNoPlanFileDecisions } from './helpers/plan-mode-evidence';
const describeE2E = describeE2ETier('gate');
describeE2E('office-hours AskUserQuestion-blocked smoke (gate)', () => {
// The PR lane selects this case by a test name ending in its id.
describeE2E('office-hours AskUserQuestion-blocked smoke (gate): AskUserQuestion surfaces when --disallowedTools AskUserQuestion is set', () => {
// With AskUserQuestion blocked the mode question must still be asked (the
// prose fallback, observed as 'asked'); a plan-file ## Decisions section is
// not a substitute. Failure signals also include 'auto_decided' +
// silent_write/exited/timeout.
test('AskUserQuestion surfaces when --disallowedTools AskUserQuestion is set', async () => {
test('office-hours-auto-mode', async () => {
const obs = await runPlanSkillObservation({
skillName: 'office-hours',
inPlanMode: true,