diff --git a/test/skill-e2e-session-intelligence.test.ts b/test/skill-e2e-session-intelligence.test.ts index f9e525fb8..c79fceaab 100644 --- a/test/skill-e2e-session-intelligence.test.ts +++ b/test/skill-e2e-session-intelligence.test.ts @@ -163,7 +163,8 @@ IMPORTANT: Replace any references to ~/.claude/skills/gstack/bin/ with ./bin/ when running commands. - Do NOT use AskUserQuestion. - Just run the preamble bash block and report what you see. -- Look for "RECENT ARTIFACTS" and "LAST_SESSION" in the output.`, +- Look for "RECENT ARTIFACTS" and "LAST_SESSION" in the output. +- In your final message, quote VERBATIM (copy exactly, do not paraphrase) any output lines containing "RECENT ARTIFACTS" or "LAST_SESSION".`, workingDirectory: workDir, maxTurns: 10, allowedTools: ['Bash', 'Read', 'Write', 'Edit', 'Grep', 'Glob'], @@ -337,7 +338,10 @@ IMPORTANT: - Look in ${checkpointDir} for saved context files. - Current branch is "main" — do NOT filter by current branch. Load across all branches. - The newest file by YYYYMMDD-HHMMSS prefix is the canonical "most recent". Filesystem mtime has been scrambled — do not use it. -- Do NOT use AskUserQuestion. Just present the content of the newest file.`, +- Do NOT use AskUserQuestion. Just present the content of the newest file. +- Your final message MUST end with these two lines (they are machine-checked — copy them exactly, do not paraphrase): + 1. The newest file's "## Working on:" heading line, VERBATIM as it appears in that file. + 2. A literal marker line: RESTORED: `, workingDirectory: workDir, maxTurns: 8, allowedTools: ['Bash', 'Read', 'Grep', 'Glob'], @@ -349,8 +353,34 @@ IMPORTANT: logCost('context-restore', result); const output = result.output ?? ''; - const loadedNewer = output.includes('newer wintermute work') || output.includes('wintermute integration'); - const loadedOlder = output.includes('old work') && !output.includes('newer'); + + // Evidence-based checks. CI receipts showed the agent finding + reading the + // RIGHT file, then paraphrasing the final message ("the most recent context + // is from branch-b...") — exact-substring checks over stochastic prose + // flaked. Three signal classes, strongest first: + // 1. Machine-checkable output contract (prompt demands the verbatim + // "## Working on:" heading + a "RESTORED: " marker line). + // 2. Lenient content echo (legacy): distinctive newer-file phrases. + // 3. Tool-call corroboration: a tool call whose INPUT names the newer + // file. Corroboration only — if the agent also read the OLDER file, + // tool evidence is void and the final output must present the newer. + const newerFileName = '20260202-130000-newer-wintermute-work'; + const olderFileName = '20260101-120000-old-work'; + const newerMarker = new RegExp(`RESTORED:.*${newerFileName}`, 'i').test(output); + const olderMarker = new RegExp(`RESTORED:.*${olderFileName}`, 'i').test(output); + const newerContent = output.includes('newer wintermute work') || output.includes('wintermute integration'); + const outputPresentsNewer = newerMarker || newerContent; + + const toolInputs = result.toolCalls.map(tc => JSON.stringify(tc.input ?? {})); + const toolReadNewer = toolInputs.some(input => input.includes(newerFileName)); + const toolReadOlder = toolInputs.some(input => input.includes(olderFileName)); + + // Presenting the OLDER file fails: an explicit RESTORED marker naming it, + // or older-file content with no newer-file presentation alongside. + const loadedOlder = olderMarker || (output.includes('old work') && !outputPresentsNewer); + // Tool evidence counts only when the older file was never read: a run that + // reads BOTH files must present the NEWER one in the final output to pass. + const loadedNewer = outputPresentsNewer || (toolReadNewer && !toolReadOlder); const exitOk = ['success', 'error_max_turns'].includes(result.exitReason); recordE2E(evalCollector, 'context-restore loads latest', 'Session Intelligence E2E', result, { @@ -361,6 +391,6 @@ IMPORTANT: expect(loadedNewer).toBe(true); expect(loadedOlder).toBe(false); - console.log(`context-restore: loadedNewer=${loadedNewer}, loadedOlder=${loadedOlder}`); + console.log(`context-restore: loadedNewer=${loadedNewer} (marker=${newerMarker}, content=${newerContent}, toolNewer=${toolReadNewer}, toolOlder=${toolReadOlder}), loadedOlder=${loadedOlder}`); }, CAPTURE_MS); });