fix(evals): keep a lone rerun-plan artifact off the plan manifest (run 39363705692)

Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com>
This commit is contained in:
garrytanandcapy-ai[bot] committed 2026-10-20 04:00:49 +00:00
1 parent a592321668
commit 3f91ff3caa
6 files changed
+62 -17

No files matched your search

+5 -2
View File
@@ -37,7 +37,7 @@ import { expectedPrCaseCount, shardSlug } from './paid-select';
import { ROOT } from './paid-types';
export const FLAKE_RECORDS_FILE = 'flake-records.jsonl';
/** Each attempt's rerun manifest lands in the report dir as `<RERUN_PLAN_ARTIFACT>-a<attempt>/manifest.json`. */
/** Each attempt's rerun manifest lands in the report dir as `<RERUN_PLAN_ARTIFACT>-a<attempt>/manifest.json` (one level deeper when several attempts' plans download). */
export const RERUN_PLAN_ARTIFACT = 'rerun-plan';
export type Phase = 'first' | 'rerun';
@@ -185,7 +185,10 @@ export function observationSets(reportDir: string, manifest: PaidRunManifest, fi
const reruns = loadSliceArtifacts(reportDir, { reruns: true });
for (const attempt of [...new Set(reruns.map(a => a.result.attempt ?? 1))].sort((a, b) => a - b)) {
let rerunManifest: PaidRunManifest;
try { rerunManifest = parseRunManifest(fs.readFileSync(path.join(reportDir, `${RERUN_PLAN_ARTIFACT}-a${attempt}`, 'manifest.json'), 'utf8')); } catch (error) {
// download-artifact unpacks a lone pattern match into the report dir and each of several into its own directory.
const dir = path.join(reportDir, `${RERUN_PLAN_ARTIFACT}-a${attempt}`);
const file = [path.join(dir, 'manifest.json'), path.join(dir, path.basename(dir), 'manifest.json')].find(candidate => fs.existsSync(candidate)) ?? path.join(dir, 'manifest.json');
try { rerunManifest = parseRunManifest(fs.readFileSync(file, 'utf8')); } catch (error) {
ignored.push(`attempt ${attempt}: rerun results without a readable rerun manifest (${error instanceof Error ? error.message : String(error)})`);
continue;
}