mirror of
https://github.com/garrytan/gstack.git
synced 2026-11-05 16:34:00 +01:00
test(evals): name the office-hours-auto-mode case so the PR lane can select it
The PR lane runs each short probe with a Bun name pattern ending in its case id. The office-hours AskUserQuestion-blocked smoke had a descriptive test name and no CASE_TEST_NAMES label, so whenever office-hours changed its shard matched 0 tests and failed in 0 s (E2E Evals slice 6 on #3075). Name the test after its id, keep the description on the describe, and check that every literal-named PR probe has a test the pattern selects. Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
1 parent
b5604e91be
commit
f555e9da54
2 files changed
+17
-2
No files matched your search
@@ -12,6 +12,7 @@ import {
|
||||
type PaidCaseSelection, type SliceResult,
|
||||
} from '../scripts/test-paid-shards';
|
||||
import { PR_PROFILE_CASE_IDS, PR_PROFILE_FILES } from '../scripts/test-pr-profile';
|
||||
import { fileCaseRegistration } from '../scripts/lib/paid-cases';
|
||||
import { E2E_TOUCHFILES } from './helpers/touchfiles-data';
|
||||
import { resolveModuleSelection } from './helpers/e2e-helpers';
|
||||
|
||||
@@ -64,6 +65,19 @@ describe('PR profile paid-runner integration', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('every short probe with literal test names has a test the PR lane name pattern selects', () => {
|
||||
for (const [file, cases] of Object.entries(PR_PROFILE_FILES)) {
|
||||
const source = fs.readFileSync(path.join(ROOT, file), 'utf8');
|
||||
if (fileCaseRegistration(file, source).computed) continue;
|
||||
const titles = [...source.matchAll(/\b(?:describe\w*|test\w*|it)(?:\.\w+)?\(\s*(['"`])((?:(?!\1)[^\\]|\\.)*)\1/g)].map(m => m[2]!);
|
||||
const names = titles.flatMap(outer => [outer, ...titles.map(inner => `${outer} ${inner}`)]);
|
||||
for (const id of cases) {
|
||||
const pattern = new RegExp(prProfileTestNamePattern(file, { e2e: [id], judges: [] }));
|
||||
expect(names.some(name => pattern.test(name)), `${file} has no test named for ${id}`).toBe(true);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('planner binds E2E and judge IDs and plans the CEO plan-mode probe in the PR lane', () => {
|
||||
const manifest = ceoManifest();
|
||||
expect(manifest.profile).toBe('pr');
|
||||
|
||||
@@ -24,12 +24,13 @@ import { assertNoPlanFileDecisions } from './helpers/plan-mode-evidence';
|
||||
|
||||
const describeE2E = describeE2ETier('gate');
|
||||
|
||||
describeE2E('office-hours AskUserQuestion-blocked smoke (gate)', () => {
|
||||
// The PR lane selects this case by a test name ending in its id.
|
||||
describeE2E('office-hours AskUserQuestion-blocked smoke (gate): AskUserQuestion surfaces when --disallowedTools AskUserQuestion is set', () => {
|
||||
// With AskUserQuestion blocked the mode question must still be asked (the
|
||||
// prose fallback, observed as 'asked'); a plan-file ## Decisions section is
|
||||
// not a substitute. Failure signals also include 'auto_decided' +
|
||||
// silent_write/exited/timeout.
|
||||
test('AskUserQuestion surfaces when --disallowedTools AskUserQuestion is set', async () => {
|
||||
test('office-hours-auto-mode', async () => {
|
||||
const obs = await runPlanSkillObservation({
|
||||
skillName: 'office-hours',
|
||||
inPlanMode: true,
|
||||
|
||||
Reference in new issue
Block a user