feat(evals): pin judge calls to the low inference class

livekit-agents/livekit/agents/evals/judge.py

@@ -13,6 +13,25 @@
 """The verdict of a judgment: pass, fail, or maybe (uncertain)."""
 
 
+def _judge_chat_kwargs(llm: LLM) -> dict[str, Any]:
+    """Extra ``chat()`` kwargs for a judgment call.
+
+    Judging is batch load: an eval suite fans out many judgments at once, they run
+    after the conversation they grade, and nobody is waiting on the verdict. So they
+    are pinned to the low inference class and must not compete with live traffic for
+    gateway capacity, even when the judge LLM was configured with another class.
+
+    Empty for a plugin LLM, which has no LiveKit Inference class to set.
+    """
+    from ..inference import LLM as InferenceLLM
+    from ..inference._utils import INFERENCE_CLASS_LOW
+
+    if isinstance(llm, InferenceLLM):
+        return {"inference_class": INFERENCE_CLASS_LOW}
+
+    return {}
+
+
 @dataclass
 class JudgmentResult:
     verdict: Verdict
@@ -131,6 +150,7 @@ async def submit_verdict(verdict: Verdict, reasoning: str) -> tuple[Verdict, str
         tool_choice="required",
         conn_options=_JUDGE_CONN_OPTIONS,
         extra_kwargs=extra_kwargs,
+        **_judge_chat_kwargs(llm),
     ).collect()
 
     if not response.tool_calls:

livekit-agents/livekit/agents/voice/run_result.py

@@ -1024,12 +1024,15 @@ async def check_intent(success: bool, reason: str) -> tuple[bool, str]:
         if not any(excluded_model in llm_v.model for excluded_model in excluded_models_temperature):
             extra_kwargs["temperature"] = 0.0
 
+        from ..evals.judge import _judge_chat_kwargs
+
         # TODO(theomonnom): LLMStream should provide utilities to make function calling easier.
         async for chunk in llm_v.chat(
             chat_ctx=chat_ctx,
             tools=[check_intent],
             tool_choice="required",
             extra_kwargs=extra_kwargs,
+            **_judge_chat_kwargs(llm_v),
         ):
             if chunk.usage is not None:
                 usage = chunk.usage