From 3aa358ebff2551c6ebd5eab53dce9aff60fb7e93 Mon Sep 17 00:00:00 2001
From: Jacqueline122 <89665099+Jacqueline122@users.noreply.github.com>
Date: Sun, 25 Oct 2026 14:49:55 -0700
Subject: [PATCH] fix(eval): check final response in failure attribution

---
 ag2/eval/scorers/attribution.py       |  5 ++--
 test/eval/scorers/test_attribution.py | 41 ++++++++++++++++++++++++++-
 2 files changed, 43 insertions(+), 3 deletions(-)

diff --git a/ag2/eval/scorers/attribution.py b/ag2/eval/scorers/attribution.py
index 43e1b04a1501..7a43285f3223 100644
--- a/ag2/eval/scorers/attribution.py
+++ b/ag2/eval/scorers/attribution.py
@@ -145,7 +145,7 @@ async def _attribute(
 
 
 def _detect_mechanical(trace: Trace, agent_name: str | None) -> Attribution | None:
-    """Attribute unambiguous mechanical failures; ``None`` to defer to the LLM."""
+    """Attribute mechanical failures using the final model response; ``None`` to defer to the LLM."""
     events = trace.events
 
     if trace.exception is not None:
@@ -157,7 +157,8 @@ def _detect_mechanical(trace: Trace, agent_name: str | None) -> Attribution | No
             f"run raised {type(trace.exception).__name__}: {trace.exception}",
         )
 
-    has_answer = any(r.content for r in trace.events_of(ModelResponse))
+    responses = trace.events_of(ModelResponse)
+    has_answer = bool(responses and responses[-1].content)
     tool_errors = trace.events_of(ToolErrorEvent)
     if tool_errors and not has_answer:
         return _mech(
diff --git a/test/eval/scorers/test_attribution.py b/test/eval/scorers/test_attribution.py
index 71906b2d5fbb..8c0494118629 100644
--- a/test/eval/scorers/test_attribution.py
+++ b/test/eval/scorers/test_attribution.py
@@ -5,12 +5,13 @@
 """Tests for failure attribution (deterministic detectors + LLM attributor)."""
 
 import pytest
+from dirty_equals import IsPartialDict
 
 from ag2.eval import InMemoryTraceSource, TraceRef, evaluate_traces
 from ag2.eval.dataset.task import Task
 from ag2.eval.scorers import failure_attribution
 from ag2.eval.trace import Trace
-from ag2.events import ModelMessage, ModelResponse, ToolCallEvent, ToolErrorEvent
+from ag2.events import ModelMessage, ModelResponse, ToolCallEvent, ToolErrorEvent, ToolResultEvent
 from ag2.testing import TestConfig
 
 
@@ -50,6 +51,44 @@ async def test_tool_failure_records_decisive_step() -> None:
     assert fb.detail["decisive_step"] == 1  # index of the error event
 
 
+@pytest.mark.asyncio()
+@pytest.mark.parametrize(
+    ("tool_failed", "final_answer", "expected_mode"),
+    [
+        (False, None, "no_answer"),
+        (True, None, "tool_failure"),
+        (False, "", "no_answer"),
+        (True, "", "tool_failure"),
+        (False, "The answer is 42.", "none"),
+        (True, "The answer is 42.", "none"),
+    ],
+)
+async def test_interim_text_does_not_hide_final_answer_state(
+    tool_failed: bool, final_answer: str | None, expected_mode: str
+) -> None:
+    call = ToolCallEvent("lookup", arguments="{}")
+    tool_result = (
+        ToolErrorEvent.from_call(call, RuntimeError("lookup failed"))
+        if tool_failed
+        else ToolResultEvent.from_call(call, result="42")
+    )
+    trace = _trace([
+        ModelResponse(message=ModelMessage("I will look that up.")),
+        call,
+        tool_result,
+        ModelResponse(message=ModelMessage(final_answer) if final_answer is not None else None),
+    ])
+
+    [fb] = await _attribute(failure_attribution(), trace, outputs={"body": final_answer})
+
+    assert fb.value == expected_mode
+    assert fb.detail == IsPartialDict({
+        "failed": not bool(final_answer),
+        "error_mode": expected_mode,
+        "decisive_step": 2 if expected_mode == "tool_failure" else None,
+    })
+
+
 @pytest.mark.asyncio()
 async def test_clean_run_without_llm_is_none() -> None:
     [fb] = await _attribute(