From a2607ca9c376e58303dbbe2d4ec16de116b115b8 Mon Sep 17 00:00:00 2001 From: Test Date: Sat, 29 Aug 2026 14:55:33 +0000 Subject: [PATCH] fix(test): context-restore assertion is evidence-based, not prose-matching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test failed twice per run in TWO CI cycles while passing locally 4/4: the prompt said 'present the content' and the check grepped the FINAL message for exact phrases — local runs quoted the file, CI runs paraphrased ('the most recent context is from branch-b...') and the substring check lost the coin flip. - prompt now demands machine-checkable output: the newest file's '## Working on:' heading VERBATIM + a literal 'RESTORED: ' marker (the mtime-scramble and cross-branch subject matter untouched) - assertion ordered strongest-first: RESTORED marker → legacy content phrases → tool-call corroboration (Read/Bash input naming the newer file, credited ONLY when the older file was never read — a both-files run must still present the right one) - the older-file negative got STRONGER: an explicit RESTORED marker naming the older file fails even if wintermute words appear elsewhere - sibling scan: context-recovery-artifacts got the additive prompt-side treatment only (quote the matched literals verbatim); its lenient 1-of-6 assertion deliberately unchanged 3/3 consecutive local green with all evidence classes firing (marker=true, content=true, toolNewer=true, toolOlder=false). Co-Authored-By: Claude Fable 5 --- test/skill-e2e-session-intelligence.test.ts | 40 ++++++++++++++++++--- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/test/skill-e2e-session-intelligence.test.ts b/test/skill-e2e-session-intelligence.test.ts index f9e525fb8..c79fceaab 100644 --- a/test/skill-e2e-session-intelligence.test.ts +++ b/test/skill-e2e-session-intelligence.test.ts @@ -163,7 +163,8 @@ IMPORTANT: Replace any references to ~/.claude/skills/gstack/bin/ with ./bin/ when running commands. - Do NOT use AskUserQuestion. - Just run the preamble bash block and report what you see. -- Look for "RECENT ARTIFACTS" and "LAST_SESSION" in the output.`, +- Look for "RECENT ARTIFACTS" and "LAST_SESSION" in the output. +- In your final message, quote VERBATIM (copy exactly, do not paraphrase) any output lines containing "RECENT ARTIFACTS" or "LAST_SESSION".`, workingDirectory: workDir, maxTurns: 10, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], @@ -337,7 +338,10 @@ IMPORTANT: - Look in ${checkpointDir} for saved context files. - Current branch is "main" — do NOT filter by current branch. Load across all branches. - The newest file by YYYYMMDD-HHMMSS prefix is the canonical "most recent". Filesystem mtime has been scrambled — do not use it. -- Do NOT use AskUserQuestion. Just present the content of the newest file.`, +- Do NOT use AskUserQuestion. Just present the content of the newest file. +- Your final message MUST end with these two lines (they are machine-checked — copy them exactly, do not paraphrase): + 1. The newest file's "## Working on:" heading line, VERBATIM as it appears in that file. + 2. A literal marker line: RESTORED: `, workingDirectory: workDir, maxTurns: 8, allowedTools: ['Bash', 'Read', 'Grep', 'Glob'], @@ -349,8 +353,34 @@ IMPORTANT: logCost('context-restore', result); const output = result.output ?? ''; - const loadedNewer = output.includes('newer wintermute work') || output.includes('wintermute integration'); - const loadedOlder = output.includes('old work') && !output.includes('newer'); + + // Evidence-based checks. CI receipts showed the agent finding + reading the + // RIGHT file, then paraphrasing the final message ("the most recent context + // is from branch-b...") — exact-substring checks over stochastic prose + // flaked. Three signal classes, strongest first: + // 1. Machine-checkable output contract (prompt demands the verbatim + // "## Working on:" heading + a "RESTORED: " marker line). + // 2. Lenient content echo (legacy): distinctive newer-file phrases. + // 3. Tool-call corroboration: a tool call whose INPUT names the newer + // file. Corroboration only — if the agent also read the OLDER file, + // tool evidence is void and the final output must present the newer. + const newerFileName = '20260202-130000-newer-wintermute-work'; + const olderFileName = '20260101-120000-old-work'; + const newerMarker = new RegExp(`RESTORED:.*${newerFileName}`, 'i').test(output); + const olderMarker = new RegExp(`RESTORED:.*${olderFileName}`, 'i').test(output); + const newerContent = output.includes('newer wintermute work') || output.includes('wintermute integration'); + const outputPresentsNewer = newerMarker || newerContent; + + const toolInputs = result.toolCalls.map(tc => JSON.stringify(tc.input ?? {})); + const toolReadNewer = toolInputs.some(input => input.includes(newerFileName)); + const toolReadOlder = toolInputs.some(input => input.includes(olderFileName)); + + // Presenting the OLDER file fails: an explicit RESTORED marker naming it, + // or older-file content with no newer-file presentation alongside. + const loadedOlder = olderMarker || (output.includes('old work') && !outputPresentsNewer); + // Tool evidence counts only when the older file was never read: a run that + // reads BOTH files must present the NEWER one in the final output to pass. + const loadedNewer = outputPresentsNewer || (toolReadNewer && !toolReadOlder); const exitOk = ['success', 'error_max_turns'].includes(result.exitReason); recordE2E(evalCollector, 'context-restore loads latest', 'Session Intelligence E2E', result, { @@ -361,6 +391,6 @@ IMPORTANT: expect(loadedNewer).toBe(true); expect(loadedOlder).toBe(false); - console.log(`context-restore: loadedNewer=${loadedNewer}, loadedOlder=${loadedOlder}`); + console.log(`context-restore: loadedNewer=${loadedNewer} (marker=${newerMarker}, content=${newerContent}, toolNewer=${toolReadNewer}, toolOlder=${toolReadOlder}), loadedOlder=${loadedOlder}`); }, CAPTURE_MS); });