diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a7e8b9..45f79b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,16 @@ named rather than smoothed. if they cannot be proven; `--json` prints it instead. The bug-report issue template now asks for the bundle first. Ported idea from bielcarpi/hermes-live-voice's `diagnostics` (MIT) — idea only, no code. +- Added a deterministic, content-free endpointing trace evaluator that reports + nearest-rank endpoint/playback latency and classified cutoff, split, timeout, + and false-activation counts without retaining audio or transcripts. +- Provider-neutral semantic endpointing controls for the core realtime + contract: `RealtimeTurnDetection` with native/server/semantic modes and + `RealtimeSemanticEagerness` are mirrored through the setup and live-update + paths, Grok advertises native/server support and Gemini native-only with + fail-fast refusals on unsupported modes, and the semantic names are probed + as an optional capability so pre-semantic core heads keep the full lane + with provider-native turn detection instead of losing it. - hermes-talk's three realtime lanes now register on the Hermes core `RealtimeVoiceProvider` contract (`agent/realtime_voice_provider.py`, API v2 — NousResearch/hermes-agent#101808) as `hermes-talk/openai`, diff --git a/pyproject.toml b/pyproject.toml index 331d4f2..cde9105 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ py-modules = [ "talk_grok_realtime", "talk_gemini_realtime", "talk_core_realtime", + "talk_endpointing_benchmark", "talk_core_provider", "talk_core_session", "talk_discord", @@ -101,6 +102,7 @@ known-first-party = [ "talk_diagnostics", "talk_discord", "talk_doctor", + "talk_endpointing_benchmark", "talk_gemini_realtime", "talk_grok_auth", "talk_grok_realtime", diff --git a/talk_core_provider.py b/talk_core_provider.py index b722dd6..b46f283 100644 --- a/talk_core_provider.py +++ b/talk_core_provider.py @@ -108,6 +108,37 @@ # poison hermes-talk's own imports. Every released Hermes lands here. _CORE_IMPORT_ERROR = exc +_TURN_DETECTION_IMPORT_ERROR: BaseException | None = None +try: + from agent.realtime_voice_provider import ( + RealtimeSemanticEagerness, + RealtimeTurnDetection, + RealtimeTurnDetectionMode, + ) +except Exception as exc: # noqa: BLE001 - pre-semantic core heads lack these names + _TURN_DETECTION_IMPORT_ERROR = exc + RealtimeSemanticEagerness = None # type: ignore[assignment] + RealtimeTurnDetection = None # type: ignore[assignment] + RealtimeTurnDetectionMode = None # type: ignore[assignment] + + +def turn_detection_available() -> bool: + """True when the host contract carries the semantic turn-detection names. + + A pre-semantic core head (today's #101808) still gets the full core lane; + only turn-detection controls degrade to provider-native. Never raises. + """ + + return _CORE_IMPORT_ERROR is None and _TURN_DETECTION_IMPORT_ERROR is None + + +def _contract_turn_modes(*names: str) -> frozenset: + """Contract turn-detection modes by member name; empty on pre-semantic heads.""" + + if not turn_detection_available(): + return frozenset() + return frozenset(getattr(RealtimeTurnDetectionMode, name) for name in names) + def core_contract_available() -> bool: """True when this host exposes the exact core realtime contract we target.""" @@ -117,10 +148,10 @@ def core_contract_available() -> bool: def core_contract_diagnostic() -> dict[str, Any]: """Read-only receipt for ``talk doctor`` / ``talk_status``. Never raises.""" - available = core_contract_available() return { "contract_available": available, + "turn_detection_available": turn_detection_available(), "provider_names": list(PROVIDER_NAMES), "detail": "" if available else type(_CORE_IMPORT_ERROR).__name__, } @@ -355,6 +386,8 @@ class _TalkCoreProvider(RealtimeVoiceProvider): input_audio = PCM16_24K output_audio = PCM16_24K + supported_turn_detection_modes = _contract_turn_modes("PROVIDER_NATIVE") + def __init__( self, *, @@ -390,6 +423,34 @@ def get_setup_schema(self) -> Mapping[str, Any]: ), } + def _talk_turn_detection(self, turn_detection: Any) -> rt.RealtimeTurnDetection: + if turn_detection is None: + # Pre-semantic host: the setup carries no turn_detection at all. + return rt.RealtimeTurnDetection() + if turn_detection.mode not in self.supported_turn_detection_modes: + raise ValueError( + f"{self.display_name} does not support turn detection mode " + f"{turn_detection.mode.value}" + ) + modes = { + RealtimeTurnDetectionMode.PROVIDER_NATIVE: ( + rt.RealtimeTurnDetectionMode.PROVIDER_NATIVE + ), + RealtimeTurnDetectionMode.SERVER_VAD: (rt.RealtimeTurnDetectionMode.SERVER_VAD), + RealtimeTurnDetectionMode.SEMANTIC_VAD: (rt.RealtimeTurnDetectionMode.SEMANTIC_VAD), + } + eagerness = { + None: None, + RealtimeSemanticEagerness.AUTO: rt.RealtimeSemanticEagerness.AUTO, + RealtimeSemanticEagerness.LOW: rt.RealtimeSemanticEagerness.LOW, + RealtimeSemanticEagerness.MEDIUM: rt.RealtimeSemanticEagerness.MEDIUM, + RealtimeSemanticEagerness.HIGH: rt.RealtimeSemanticEagerness.HIGH, + } + return rt.RealtimeTurnDetection( + mode=modes[turn_detection.mode], + semantic_eagerness=eagerness[turn_detection.semantic_eagerness], + ) + def _talk_setup(self, setup: RealtimeVoiceSetup) -> Any: for label, requested, expected in ( ("input", setup.input_audio, self.input_audio), @@ -412,12 +473,13 @@ def _talk_setup(self, setup: RealtimeVoiceSetup) -> Any: ) for tool in setup.tools ), - automatic_response=setup.automatic_response, + turn_detection=self._talk_turn_detection(getattr(setup, "turn_detection", None)), ) async def open_session(self, setup: RealtimeVoiceSetup) -> RealtimeVoiceSession: # Shape first: an unusable setup is refused before any credential is # resolved and before a socket is opened. + self.validate_setup(setup) talk_setup = self._talk_setup(setup) resolver = self._auth_resolver or self._resolve_auth auth = resolver() @@ -466,6 +528,9 @@ class TalkOpenAICoreProvider(_TalkCoreProvider): provider_tag = "gpt-realtime speech-to-speech" env_key = "OPENAI_API_KEY" env_url = "https://platform.openai.com/api-keys" + supported_turn_detection_modes = _contract_turn_modes( + "PROVIDER_NATIVE", "SERVER_VAD", "SEMANTIC_VAD" + ) def default_model(self) -> str | None: return talk_config.talk_model() or talk_config.DEFAULT_TALK_MODEL @@ -527,6 +592,9 @@ class TalkGrokCoreProvider(_TalkCoreProvider): provider_tag = "grok-voice speech-to-speech" env_key = "XAI_API_KEY" env_url = "https://console.x.ai" + supported_turn_detection_modes = _contract_turn_modes( + "PROVIDER_NATIVE", "SERVER_VAD" + ) def default_model(self) -> str | None: return talk_config.talk_grok_model() or talk_config.DEFAULT_GROK_MODEL @@ -684,8 +752,8 @@ def build_providers() -> tuple[Any, ...]: "TalkGrokCoreProvider", "TalkOpenAICoreProvider", "build_providers", - "core_contract_available", "core_contract_diagnostic", "redact", "translate_event", + "turn_detection_available", ] diff --git a/talk_endpointing_benchmark.py b/talk_endpointing_benchmark.py new file mode 100644 index 0000000..baa4950 --- /dev/null +++ b/talk_endpointing_benchmark.py @@ -0,0 +1,174 @@ +"""Deterministic, content-free metrics for endpointing trace fixtures. + +Times are monotonic integer milliseconds chosen by the caller. This module has +no clock, provider, audio, or transcript dependency and deliberately stores no +media or text. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from dataclasses import dataclass +from math import ceil + + +@dataclass(frozen=True, slots=True) +class EndpointingTrace: + """Receipts for one annotated opportunity to activate. + + ``annotated_utterance_end_ms=None`` describes a period in which no utterance + was annotated; any provider or playback receipt then counts as one false + activation. A timeout is terminal but is not itself an activation. A + speech-stop before the annotation is a premature cutoff, and a turn-end + before it is a false split. ``timed_out_at_ms`` is an explicit terminal + receipt, not a timeout inferred from missing data. + """ + + trace_id: str + annotated_utterance_end_ms: int | None = None + provider_speech_stop_ms: int | None = None + provider_turn_end_ms: int | None = None + first_audio_ms: int | None = None + local_playback_ms: int | None = None + timed_out_at_ms: int | None = None + + def __post_init__(self) -> None: + if not isinstance(self.trace_id, str) or not self.trace_id.strip(): + raise ValueError("trace_id must be a non-empty string") + + names = ( + "annotated_utterance_end_ms", + "provider_speech_stop_ms", + "provider_turn_end_ms", + "first_audio_ms", + "local_playback_ms", + "timed_out_at_ms", + ) + for name in names: + value = getattr(self, name) + if value is not None and ( + isinstance(value, bool) or not isinstance(value, int) or value < 0 + ): + raise ValueError(f"{name} must be a non-negative integer or None") + + stop = self.provider_speech_stop_ms + turn = self.provider_turn_end_ms + audio = self.first_audio_ms + playback = self.local_playback_ms + timeout = self.timed_out_at_ms + annotated = self.annotated_utterance_end_ms + + if stop is not None and turn is not None and stop > turn: + raise ValueError("provider speech-stop cannot follow provider turn-end") + if audio is not None and turn is None: + raise ValueError("first-audio requires a provider turn-end receipt") + if turn is not None and audio is not None and turn > audio: + raise ValueError("provider turn-end cannot follow first-audio") + if playback is not None and audio is None: + raise ValueError("local-playback requires a first-audio receipt") + if audio is not None and playback is not None and audio > playback: + raise ValueError("first-audio cannot follow local-playback") + if stop is not None and timeout is not None and stop > timeout: + raise ValueError("provider speech-stop cannot follow timeout") + if timeout is not None and any(value is not None for value in (turn, audio, playback)): + raise ValueError("timeout and completion receipts are mutually exclusive") + if timeout is not None and annotated is not None and timeout < annotated: + raise ValueError("timeout cannot precede the annotated utterance end") + + +@dataclass(frozen=True, slots=True) +class LatencyDistribution: + """Nearest-rank latency summary. + + For ``n`` sorted samples, percentile ``p`` selects the 1-based item at + ``ceil(p * n)``. Empty distributions report ``None`` rather than inventing + zero latency. + """ + + count: int + p50_ms: int | None + p95_ms: int | None + max_ms: int | None + + +@dataclass(frozen=True, slots=True) +class EndpointingSummary: + trace_count: int + endpoint_latency: LatencyDistribution + playback_latency: LatencyDistribution + premature_cutoff_count: int + false_split_count: int + timeout_count: int + false_activation_count: int + + +def _distribution(samples: Iterable[int]) -> LatencyDistribution: + ordered = sorted(samples) + if not ordered: + return LatencyDistribution(count=0, p50_ms=None, p95_ms=None, max_ms=None) + + def nearest_rank(percentile: float) -> int: + return ordered[ceil(percentile * len(ordered)) - 1] + + return LatencyDistribution( + count=len(ordered), + p50_ms=nearest_rank(0.50), + p95_ms=nearest_rank(0.95), + max_ms=ordered[-1], + ) + + +def evaluate_endpointing_traces(traces: Iterable[EndpointingTrace]) -> EndpointingSummary: + """Evaluate immutable receipts without consulting a live provider. + + Endpoint latency runs from annotated utterance end to provider turn-end. + Premature (negative) turn-ends are classified as false splits and excluded + from latency. Playback latency runs from first-audio receipt to the local + playback receipt, isolating local delivery from model response time. + """ + + materialized = tuple(traces) + if any(not isinstance(trace, EndpointingTrace) for trace in materialized): + raise TypeError("traces must contain EndpointingTrace records") + ids = tuple(trace.trace_id for trace in materialized) + if len(ids) != len(set(ids)): + raise ValueError("trace_id values must be unique") + + endpoint_latencies: list[int] = [] + playback_latencies: list[int] = [] + premature_cutoffs = 0 + false_splits = 0 + timeouts = 0 + false_activations = 0 + + for trace in materialized: + annotated = trace.annotated_utterance_end_ms + activation_receipts = ( + trace.provider_speech_stop_ms, + trace.provider_turn_end_ms, + trace.first_audio_ms, + trace.local_playback_ms, + ) + if annotated is None: + false_activations += int(any(receipt is not None for receipt in activation_receipts)) + else: + stop = trace.provider_speech_stop_ms + turn = trace.provider_turn_end_ms + premature_cutoffs += int(stop is not None and stop < annotated) + false_splits += int(turn is not None and turn < annotated) + if turn is not None and turn >= annotated: + endpoint_latencies.append(turn - annotated) + + if trace.first_audio_ms is not None and trace.local_playback_ms is not None: + playback_latencies.append(trace.local_playback_ms - trace.first_audio_ms) + timeouts += int(trace.timed_out_at_ms is not None) + + return EndpointingSummary( + trace_count=len(materialized), + endpoint_latency=_distribution(endpoint_latencies), + playback_latency=_distribution(playback_latencies), + premature_cutoff_count=premature_cutoffs, + false_split_count=false_splits, + timeout_count=timeouts, + false_activation_count=false_activations, + ) diff --git a/talk_gemini_realtime.py b/talk_gemini_realtime.py index af0b35e..90351ea 100644 --- a/talk_gemini_realtime.py +++ b/talk_gemini_realtime.py @@ -212,6 +212,15 @@ def _tool_wire(tool: rt.ToolDefinition) -> dict[str, Any]: } +def _validate_turn_detection(setup: rt.SessionSetup) -> None: + turn_detection = setup.turn_detection + if turn_detection.mode is not rt.RealtimeTurnDetectionMode.PROVIDER_NATIVE: + raise rt.RealtimeSessionError( + "Gemini Live supports only provider-native turn detection; " + f"{turn_detection.mode.value} is refused rather than degraded" + ) + + def build_setup_message(setup: rt.SessionSetup) -> dict[str, Any]: """Map neutral setup to the Live ``setup`` message. @@ -221,6 +230,7 @@ def build_setup_message(setup: rt.SessionSetup) -> dict[str, Any]: deliberately absent because touching it can only narrow detection, never reproduce the OpenAI lane's ``create_response`` gating. """ + _validate_turn_detection(setup) payload: dict[str, Any] = { "model": _wire_model(setup.model), @@ -520,6 +530,11 @@ def __init__(self, *, auth_token: str, auth_source: str, aiohttp_module=None) -> async def connect(self, setup: rt.SessionSetup) -> None: if self.state is not rt.SessionState.NEW: raise rt.RealtimeSessionError("Realtime session connect may only run once") + try: + _validate_turn_detection(setup) + except rt.RealtimeSessionError: + self.state = rt.SessionState.FAILED + raise self.state = rt.SessionState.CONNECTING if not setup.automatic_response: # The gated-response flow (Discord's authorization ledger) needs diff --git a/talk_grok_realtime.py b/talk_grok_realtime.py index a259dfb..4c85441 100644 --- a/talk_grok_realtime.py +++ b/talk_grok_realtime.py @@ -92,6 +92,15 @@ def _tool_wire(tool: rt.ToolDefinition) -> dict[str, Any]: } +def _validate_turn_detection(setup: rt.SessionSetup) -> None: + turn_detection = setup.turn_detection + if turn_detection.mode is rt.RealtimeTurnDetectionMode.SEMANTIC_VAD: + raise rt.RealtimeSessionError( + "Grok realtime does not support semantic VAD; supported turn detection " + "modes are provider-native and server VAD" + ) + + def build_session_update(setup: rt.SessionSetup) -> dict[str, Any]: """Map neutral setup to the OpenAI-GA-shaped update xAI accepts. @@ -99,6 +108,7 @@ def build_session_update(setup: rt.SessionSetup) -> dict[str, Any]: ``?model=`` query, same split as the OpenAI lane. ``session.type`` stays: the GA shape requires it and the live endpoint accepted it. """ + _validate_turn_detection(setup) session: dict[str, Any] = { "type": "realtime", @@ -638,6 +648,11 @@ def __init__(self, *, auth_token: str, auth_source: str, aiohttp_module=None) -> async def connect(self, setup: rt.SessionSetup) -> None: if self.state is not rt.SessionState.NEW: raise rt.RealtimeSessionError("Realtime session connect may only run once") + try: + _validate_turn_detection(setup) + except rt.RealtimeSessionError: + self.state = rt.SessionState.FAILED + raise self.state = rt.SessionState.CONNECTING try: await self._wire.connect( diff --git a/talk_openai_realtime.py b/talk_openai_realtime.py index 0be8f5e..bec39f2 100644 --- a/talk_openai_realtime.py +++ b/talk_openai_realtime.py @@ -49,6 +49,7 @@ def build_session_update(setup: rt.SessionSetup) -> dict[str, Any]: instructions=setup.instructions, tools=[_tool_wire(tool) for tool in setup.tools] or None, automatic_response=setup.automatic_response, + turn_detection=setup.turn_detection, text_output=setup.text_output, ) return { @@ -348,6 +349,7 @@ def _mint(self, **configuration): instructions=configuration["instructions"], tools=configuration.get("tools"), automatic_response=configuration["automatic_response"], + turn_detection=configuration["turn_detection"], text_output=configuration.get("text_output", False), ) @@ -359,6 +361,7 @@ async def connect( instructions: str, tools: list[dict] | None, automatic_response: bool, + turn_detection: rt.RealtimeTurnDetection, session_update: dict[str, Any], text_output: bool = False, ) -> None: @@ -374,6 +377,7 @@ async def connect( instructions=instructions, tools=tools, automatic_response=automatic_response, + turn_detection=turn_detection, text_output=text_output, ) finally: @@ -549,6 +553,7 @@ async def connect(self, setup: rt.SessionSetup) -> None: instructions=setup.instructions, tools=tools, automatic_response=setup.automatic_response, + turn_detection=setup.turn_detection, session_update=build_session_update(setup), text_output=setup.text_output, ) diff --git a/talk_realtime.py b/talk_realtime.py index ca56f91..200dadf 100644 --- a/talk_realtime.py +++ b/talk_realtime.py @@ -43,6 +43,42 @@ class SessionState(StrEnum): FAILED = "failed" +class RealtimeTurnDetectionMode(StrEnum): + """Provider-neutral input-turn detection strategy.""" + + PROVIDER_NATIVE = "provider_native" + SERVER_VAD = "server_vad" + SEMANTIC_VAD = "semantic_vad" + + +class RealtimeSemanticEagerness(StrEnum): + """How readily semantic endpointing should close an input turn.""" + + AUTO = "auto" + LOW = "low" + MEDIUM = "medium" + HIGH = "high" + + +@dataclass(frozen=True, slots=True) +class RealtimeTurnDetection: + mode: RealtimeTurnDetectionMode = RealtimeTurnDetectionMode.PROVIDER_NATIVE + semantic_eagerness: RealtimeSemanticEagerness | None = None + + def __post_init__(self) -> None: + if not isinstance(self.mode, RealtimeTurnDetectionMode): + raise TypeError("mode must be RealtimeTurnDetectionMode") + if self.semantic_eagerness is not None and not isinstance( + self.semantic_eagerness, RealtimeSemanticEagerness + ): + raise TypeError("semantic_eagerness must be None or RealtimeSemanticEagerness") + if ( + self.semantic_eagerness is not None + and self.mode is not RealtimeTurnDetectionMode.SEMANTIC_VAD + ): + raise ValueError("semantic_eagerness is valid only for semantic_vad") + + class TranscriptRole(StrEnum): USER = "user" ASSISTANT = "assistant" @@ -82,11 +118,14 @@ class SessionSetup: #: an external TTS speaks; providers that cannot do text-only output #: refuse at their own boundary rather than silently speaking anyway. text_output: bool = False + turn_detection: RealtimeTurnDetection = RealtimeTurnDetection() def __post_init__(self) -> None: _identifier(self.model, "model") _identifier(self.voice, "voice") object.__setattr__(self, "tools", tuple(self.tools)) + if not isinstance(self.turn_detection, RealtimeTurnDetection): + raise TypeError("turn_detection must be RealtimeTurnDetection") class RealtimeEvent: @@ -330,8 +369,11 @@ async def close(self) -> None: ... "ProviderFailure", "RealtimeCommand", "RealtimeEvent", + "RealtimeSemanticEagerness", "RealtimeSession", "RealtimeSessionError", + "RealtimeTurnDetection", + "RealtimeTurnDetectionMode", "RemoveContext", "ResponseFinished", "ResponseStarted", diff --git a/talk_wire.py b/talk_wire.py index dc3e50a..4c886ad 100644 --- a/talk_wire.py +++ b/talk_wire.py @@ -20,8 +20,10 @@ import httpx try: + from . import talk_realtime as rt from .talk_config import DEFAULT_TALK_MODEL, DEFAULT_TALK_VOICE except ImportError: # pragma: no cover - flat-module fallback (Hermes file-path load) + import talk_realtime as rt from talk_config import DEFAULT_TALK_MODEL, DEFAULT_TALK_VOICE OPENAI_REALTIME_OFFER_URL = "https://api.openai.com/v1/realtime/calls" @@ -29,6 +31,7 @@ CLIENT_SECRETS_URL = "https://api.openai.com/v1/realtime/client_secrets" INPUT_TRANSCRIPTION_MODEL = "gpt-4o-mini-transcribe" MINT_TIMEOUT_S = 30.0 +_DEFAULT_TURN_DETECTION = rt.RealtimeTurnDetection() class TalkWireError(Exception): @@ -59,6 +62,40 @@ def to_wire(self) -> dict: } +def encode_turn_detection( + turn_detection: rt.RealtimeTurnDetection, + *, + automatic_response: bool, +) -> dict: + """Encode neutral endpointing as an OpenAI Realtime turn detector.""" + + mode = turn_detection.mode + eagerness = turn_detection.semantic_eagerness + if mode is rt.RealtimeTurnDetectionMode.SEMANTIC_VAD: + return { + "type": "semantic_vad", + "eagerness": ( + eagerness.value + if eagerness is not None + else rt.RealtimeSemanticEagerness.AUTO.value + ), + "create_response": automatic_response, + "interrupt_response": True, + } + if eagerness is not None: + raise ValueError("semantic eagerness is valid only for semantic turn detection") + if mode in { + rt.RealtimeTurnDetectionMode.PROVIDER_NATIVE, + rt.RealtimeTurnDetectionMode.SERVER_VAD, + }: + return { + "type": "server_vad", + "create_response": automatic_response, + "interrupt_response": True, + } + raise ValueError(f"unsupported OpenAI turn detection mode: {mode!r}") + + def build_session_payload( *, model: str = DEFAULT_TALK_MODEL, @@ -66,6 +103,7 @@ def build_session_payload( instructions: str, tools: list[dict] | None = None, automatic_response: bool = True, + turn_detection: rt.RealtimeTurnDetection = _DEFAULT_TURN_DETECTION, text_output: bool = False, ) -> dict: """OpenAI Realtime session config — server VAD, barge-in enabled. @@ -80,11 +118,10 @@ def build_session_payload( audio: dict = { "input": { "noise_reduction": {"type": "near_field"}, - "turn_detection": { - "type": "server_vad", - "create_response": automatic_response, - "interrupt_response": True, - }, + "turn_detection": encode_turn_detection( + turn_detection, + automatic_response=automatic_response, + ), "transcription": {"model": INPUT_TRANSCRIPTION_MODEL}, }, } @@ -154,6 +191,7 @@ def mint_ephemeral_session( instructions: str, tools: list[dict] | None = None, automatic_response: bool = True, + turn_detection: rt.RealtimeTurnDetection = _DEFAULT_TURN_DETECTION, text_output: bool = False, ) -> TalkSessionDescriptor: """Mint an ephemeral Realtime client secret for one client session. @@ -170,6 +208,7 @@ def mint_ephemeral_session( tools=tools, automatic_response=automatic_response, text_output=text_output, + turn_detection=turn_detection, ) payload = post_client_secret(auth_token, session) secret, expires_at_ms = parse_client_secret(payload) @@ -191,6 +230,7 @@ def mint_ephemeral_session( "TalkUpstreamError", "TalkWireError", "build_session_payload", + "encode_turn_detection", "mint_ephemeral_session", "parse_client_secret", "post_client_secret", diff --git a/tests/test_core_provider.py b/tests/test_core_provider.py index dcd1f9f..dc61029 100644 --- a/tests/test_core_provider.py +++ b/tests/test_core_provider.py @@ -47,6 +47,9 @@ "OutputAudio", "OutputTranscript", "RealtimeAudioFormat", + "RealtimeSemanticEagerness", + "RealtimeTurnDetection", + "RealtimeTurnDetectionMode", "RealtimeCapability", "RealtimeToolResult", "RealtimeVoiceProvider", @@ -861,6 +864,91 @@ async def run(): } +def test_provider_turn_detection_capability_matrix_is_exact(core): + mode = core.contract.RealtimeTurnDetectionMode + + assert core.TalkOpenAICoreProvider.supported_turn_detection_modes == frozenset(mode) + assert core.TalkGrokCoreProvider.supported_turn_detection_modes == frozenset( + {mode.PROVIDER_NATIVE, mode.SERVER_VAD} + ) + assert core.TalkGeminiCoreProvider.supported_turn_detection_modes == frozenset( + {mode.PROVIDER_NATIVE} + ) + + +@pytest.mark.parametrize( + ("core_mode_name", "talk_mode", "eagerness_name"), + [ + ("PROVIDER_NATIVE", rt.RealtimeTurnDetectionMode.PROVIDER_NATIVE, None), + ("SERVER_VAD", rt.RealtimeTurnDetectionMode.SERVER_VAD, None), + ( + "SEMANTIC_VAD", + rt.RealtimeTurnDetectionMode.SEMANTIC_VAD, + "HIGH", + ), + ], +) +def test_openai_turn_detection_bridge_is_exhaustive( + core, core_mode_name, talk_mode, eagerness_name +): + c = core.contract + session = FakeTalkSession() + provider = core.TalkOpenAICoreProvider( + auth_resolver=lambda: types.SimpleNamespace(token="t", source="test"), + session_factory=lambda auth: session, + ) + eagerness = ( + None if eagerness_name is None else getattr(c.RealtimeSemanticEagerness, eagerness_name) + ) + setup = c.RealtimeVoiceSetup( + instructions="hi", + turn_detection=c.RealtimeTurnDetection( + mode=getattr(c.RealtimeTurnDetectionMode, core_mode_name), + semantic_eagerness=eagerness, + ), + ) + + async def run(): + opened = await provider.open_session(setup) + await opened.close() + + asyncio.run(run()) + assert session.connected_with.turn_detection.mode is talk_mode + expected_eagerness = ( + None if eagerness_name is None else getattr(rt.RealtimeSemanticEagerness, eagerness_name) + ) + assert session.connected_with.turn_detection.semantic_eagerness is expected_eagerness + + +@pytest.mark.parametrize( + ("provider_name", "mode_name"), + [ + ("TalkGrokCoreProvider", "SEMANTIC_VAD"), + ("TalkGeminiCoreProvider", "SERVER_VAD"), + ("TalkGeminiCoreProvider", "SEMANTIC_VAD"), + ], +) +def test_unsupported_turn_detection_is_refused_before_auth_or_session_factory( + core, provider_name, mode_name +): + calls = [] + provider = getattr(core, provider_name)( + auth_resolver=lambda: calls.append("auth"), + session_factory=lambda auth: calls.append("session"), + ) + c = core.contract + setup = c.RealtimeVoiceSetup( + instructions="hi", + turn_detection=c.RealtimeTurnDetection( + mode=getattr(c.RealtimeTurnDetectionMode, mode_name) + ), + ) + + with pytest.raises(ValueError, match="unsupported turn detection mode"): + asyncio.run(provider.open_session(setup)) + assert calls == [] + + def test_an_unset_model_or_voice_falls_back_to_the_lanes_default(core): import talk_config @@ -1023,3 +1111,95 @@ def test_the_setup_schema_names_an_env_var_and_never_a_value(core): assert provider.default_model() assert provider.default_voice() assert provider.list_voices() + + +def _synthetic_old_head_contract(): + """Minimal #101808-shaped core: API v2 with every base name, no turn-detection names. + + Binds the exact head the maintainer reviewed against: the three semantic + turn-detection symbols are absent, so the adapter must degrade to native + instead of dropping the whole core lane. + """ + + module = types.ModuleType("agent.realtime_voice_provider") + module.REALTIME_VOICE_PROVIDER_API_VERSION = 2 + module.PCM16_24K = object() + capability_members = ( + "TOOL_CALLING", + "INPUT_TRANSCRIPTION", + "OUTPUT_TRANSCRIPTION", + "EXPLICIT_RESPONSE", + "RESPONSE_CANCELLATION", + "OUTPUT_TRUNCATION", + "DYNAMIC_CONTEXT", + "TOOL_CALL_CANCELLATION", + ) + module.RealtimeCapability = type( + "RealtimeCapability", (), {name: object() for name in capability_members} + ) + for name in ( + "InputAudioCommitted", + "InputSpeechStarted", + "InputSpeechStopped", + "InputTranscript", + "OutputAudio", + "OutputTranscript", + "RealtimeAudioFormat", + "RealtimeToolResult", + "RealtimeVoiceEvent", + "RealtimeVoiceProvider", + "RealtimeVoiceSession", + "RealtimeVoiceSetup", + "ResponseCompleted", + "ResponseStarted", + "SessionClosed", + "SessionFailure", + "SessionReady", + "ToolCall", + "ToolCallCancelled", + ): + setattr(module, name, type(name, (), {})) + assert not hasattr(module, "RealtimeTurnDetectionMode") + return module + + +def test_old_head_contract_keeps_core_lane_with_native_only_turn_detection(): + """The #101808 head must not take the whole core lane down with it.""" + + contract = _synthetic_old_head_contract() + saved = { + name: sys.modules.get(name) + for name in ("agent", "agent.realtime_voice_provider", "talk_core_provider") + } + package = types.ModuleType("agent") + package.__path__ = [] + package.realtime_voice_provider = contract + sys.modules["agent"] = package + sys.modules["agent.realtime_voice_provider"] = contract + sys.modules.pop("talk_core_provider", None) + try: + module = importlib.import_module("talk_core_provider") + assert module.core_contract_available() + assert not module.turn_detection_available() + providers = module.build_providers() + assert len(providers) == 3 + assert all(p.supported_turn_detection_modes == frozenset() for p in providers) + native = providers[0]._talk_turn_detection(None) + assert native == rt.RealtimeTurnDetection() + assert native.mode == rt.RealtimeTurnDetectionMode.PROVIDER_NATIVE + class _UnsupportedMode: + value = "semantic_vad" + + semantic = types.SimpleNamespace(mode=_UnsupportedMode(), semantic_eagerness=None) + with pytest.raises(ValueError, match="does not support turn detection mode"): + providers[0]._talk_turn_detection(semantic) + diagnostic = module.core_contract_diagnostic() + assert diagnostic["contract_available"] is True + assert diagnostic["turn_detection_available"] is False + finally: + for name, value in saved.items(): + if value is None: + sys.modules.pop(name, None) + else: + sys.modules[name] = value + importlib.import_module("talk_core_provider") diff --git a/tests/test_endpointing_benchmark.py b/tests/test_endpointing_benchmark.py new file mode 100644 index 0000000..3875dac --- /dev/null +++ b/tests/test_endpointing_benchmark.py @@ -0,0 +1,176 @@ +import tomllib +from dataclasses import FrozenInstanceError +from pathlib import Path + +import pytest + +from talk_endpointing_benchmark import ( + EndpointingTrace, + LatencyDistribution, + evaluate_endpointing_traces, +) + + +def test_benchmark_module_is_in_the_distribution_module_allowlist() -> None: + project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8")) + + assert "talk_endpointing_benchmark" in project["tool"]["setuptools"]["py-modules"] + + +def test_synthetic_traces_produce_deterministic_nearest_rank_summary() -> None: + traces = tuple( + EndpointingTrace( + trace_id=f"trace-{index}", + annotated_utterance_end_ms=1_000, + provider_speech_stop_ms=1_000, + provider_turn_end_ms=1_000 + endpoint, + first_audio_ms=2_000, + local_playback_ms=2_000 + playback, + ) + for index, (endpoint, playback) in enumerate( + zip(range(1, 21), range(101, 121), strict=True) + ) + ) + + forward = evaluate_endpointing_traces(traces) + reverse = evaluate_endpointing_traces(reversed(traces)) + + assert forward == reverse + assert forward.endpoint_latency == LatencyDistribution(20, 10, 19, 20) + assert forward.playback_latency == LatencyDistribution(20, 110, 119, 120) + assert forward.premature_cutoff_count == 0 + assert forward.false_split_count == 0 + assert forward.timeout_count == 0 + assert forward.false_activation_count == 0 + + +def test_counts_classified_outcomes_and_excludes_negative_endpoint_latency() -> None: + summary = evaluate_endpointing_traces( + ( + EndpointingTrace( + "premature", + annotated_utterance_end_ms=100, + provider_speech_stop_ms=90, + provider_turn_end_ms=95, + ), + EndpointingTrace("timeout", annotated_utterance_end_ms=100, timed_out_at_ms=500), + EndpointingTrace("false-activation", provider_speech_stop_ms=10), + EndpointingTrace("true-negative"), + EndpointingTrace( + "zero-boundary", + annotated_utterance_end_ms=100, + provider_speech_stop_ms=100, + provider_turn_end_ms=100, + first_audio_ms=100, + local_playback_ms=100, + ), + ) + ) + + assert summary.trace_count == 5 + assert summary.endpoint_latency == LatencyDistribution(1, 0, 0, 0) + assert summary.playback_latency == LatencyDistribution(1, 0, 0, 0) + assert summary.premature_cutoff_count == 1 + assert summary.false_split_count == 1 + assert summary.timeout_count == 1 + assert summary.false_activation_count == 1 + + +def test_silence_only_timeout_is_terminal_but_not_a_false_activation() -> None: + summary = evaluate_endpointing_traces( + (EndpointingTrace("silence-timeout", timed_out_at_ms=500),) + ) + + assert summary.timeout_count == 1 + assert summary.false_activation_count == 0 + + +def test_empty_input_reports_absent_percentiles_honestly() -> None: + summary = evaluate_endpointing_traces(()) + + empty = LatencyDistribution(0, None, None, None) + assert summary.trace_count == 0 + assert summary.endpoint_latency == empty + assert summary.playback_latency == empty + assert summary.premature_cutoff_count == 0 + assert summary.false_split_count == 0 + assert summary.timeout_count == 0 + assert summary.false_activation_count == 0 + + +def test_trace_records_are_immutable() -> None: + trace = EndpointingTrace("immutable", annotated_utterance_end_ms=1) + + with pytest.raises(FrozenInstanceError): + trace.annotated_utterance_end_ms = 2 # type: ignore[misc] + + +@pytest.mark.parametrize( + ("kwargs", "message"), + ( + ({"trace_id": ""}, "trace_id"), + ({"trace_id": "x", "annotated_utterance_end_ms": -1}, "non-negative"), + ( + { + "trace_id": "x", + "provider_speech_stop_ms": 2, + "provider_turn_end_ms": 1, + }, + "speech-stop", + ), + ({"trace_id": "x", "first_audio_ms": 1}, "requires"), + ({"trace_id": "x", "local_playback_ms": 1}, "requires"), + ( + { + "trace_id": "x", + "provider_turn_end_ms": 2, + "first_audio_ms": 1, + }, + "turn-end", + ), + ( + { + "trace_id": "x", + "provider_turn_end_ms": 1, + "first_audio_ms": 3, + "local_playback_ms": 2, + }, + "first-audio", + ), + ( + { + "trace_id": "x", + "provider_turn_end_ms": 1, + "timed_out_at_ms": 2, + }, + "mutually exclusive", + ), + ( + { + "trace_id": "x", + "annotated_utterance_end_ms": 2, + "timed_out_at_ms": 1, + }, + "cannot precede", + ), + ( + { + "trace_id": "x", + "provider_speech_stop_ms": 3, + "timed_out_at_ms": 2, + }, + "speech-stop cannot follow timeout", + ), + ), +) +def test_malformed_timelines_are_rejected(kwargs: dict[str, object], message: str) -> None: + with pytest.raises(ValueError, match=message): + EndpointingTrace(**kwargs) # type: ignore[arg-type] + + +def test_duplicate_trace_ids_and_untyped_inputs_are_rejected() -> None: + with pytest.raises(ValueError, match="unique"): + evaluate_endpointing_traces((EndpointingTrace("same"), EndpointingTrace("same"))) + + with pytest.raises(TypeError, match="EndpointingTrace"): + evaluate_endpointing_traces((object(),)) # type: ignore[arg-type] diff --git a/tests/test_gemini_realtime.py b/tests/test_gemini_realtime.py index 7f8bd20..7bf2d46 100644 --- a/tests/test_gemini_realtime.py +++ b/tests/test_gemini_realtime.py @@ -120,7 +120,10 @@ def ws_connect(self, *args, **kwargs): return _Context(self.socket, lambda: setattr(self.socket, "exited", True)) -def _setup(*, automatic_response=True): +def _setup(*, automatic_response=True, turn_detection=None): + kwargs = {} + if turn_detection is not None: + kwargs["turn_detection"] = turn_detection return rt.SessionSetup( model="gemini-live-test", voice="Puck", @@ -139,6 +142,7 @@ def _setup(*, automatic_response=True): ), ), automatic_response=automatic_response, + **kwargs, ) @@ -212,6 +216,42 @@ async def scenario(): assert adapter.state is rt.SessionState.CLOSED +def test_provider_native_turn_detection_keeps_realtime_input_config_omitted(): + message = gemini_rt.build_setup_message( + _setup( + turn_detection=rt.RealtimeTurnDetection( + mode=rt.RealtimeTurnDetectionMode.PROVIDER_NATIVE + ) + ) + ) + + assert "realtimeInputConfig" not in message["setup"] + + +@pytest.mark.parametrize( + "turn_detection", + [ + rt.RealtimeTurnDetection(mode=rt.RealtimeTurnDetectionMode.SERVER_VAD), + rt.RealtimeTurnDetection( + mode=rt.RealtimeTurnDetectionMode.SEMANTIC_VAD, + semantic_eagerness=rt.RealtimeSemanticEagerness.MEDIUM, + ), + ], +) +def test_explicit_non_native_turn_detection_is_refused_before_connection( + turn_detection, +): + socket = _Socket() + adapter, client = _adapter(socket) + + with pytest.raises(rt.RealtimeSessionError, match="supports only provider-native"): + asyncio.run(adapter.connect(_setup(turn_detection=turn_detection))) + + assert client.connect_args is None + assert socket.sent == [] + assert adapter.state is rt.SessionState.FAILED + + def test_wire_model_prefix_is_applied_exactly_once(): assert gemini_rt._wire_model("gemini-live-test") == "models/gemini-live-test" assert gemini_rt._wire_model("models/gemini-live-test") == "models/gemini-live-test" diff --git a/tests/test_grok_realtime.py b/tests/test_grok_realtime.py index 6d782b8..805dedd 100644 --- a/tests/test_grok_realtime.py +++ b/tests/test_grok_realtime.py @@ -107,7 +107,10 @@ def ws_connect(self, *args, **kwargs): return _Context(self.socket, lambda: setattr(self.socket, "exited", True)) -def _setup(*, automatic_response=True): +def _setup(*, automatic_response=True, turn_detection=None): + kwargs = {} + if turn_detection is not None: + kwargs["turn_detection"] = turn_detection return rt.SessionSetup( model="grok-voice-test", voice="ara", @@ -120,6 +123,7 @@ def _setup(*, automatic_response=True): ), ), automatic_response=automatic_response, + **kwargs, ) @@ -185,6 +189,48 @@ async def scenario(): assert adapter.state is rt.SessionState.CLOSED +@pytest.mark.parametrize( + "mode", + [ + rt.RealtimeTurnDetectionMode.PROVIDER_NATIVE, + rt.RealtimeTurnDetectionMode.SERVER_VAD, + ], +) +def test_supported_turn_detection_modes_keep_exact_ga_server_vad_payload(mode): + setup = _setup( + automatic_response=False, + turn_detection=rt.RealtimeTurnDetection(mode=mode), + ) + + turn_detection = grok_rt.build_session_update(setup)["session"]["audio"]["input"][ + "turn_detection" + ] + + assert turn_detection == { + "type": "server_vad", + "create_response": False, + "interrupt_response": True, + } + + +def test_semantic_vad_is_refused_before_websocket_connection(): + socket = _Socket() + adapter, client = _adapter(socket) + setup = _setup( + turn_detection=rt.RealtimeTurnDetection( + mode=rt.RealtimeTurnDetectionMode.SEMANTIC_VAD, + semantic_eagerness=rt.RealtimeSemanticEagerness.HIGH, + ) + ) + + with pytest.raises(rt.RealtimeSessionError, match="does not support semantic VAD"): + asyncio.run(adapter.connect(setup)) + + assert client.connect_args is None + assert socket.sent == [] + assert adapter.state is rt.SessionState.FAILED + + def test_wire_voice_prefix_is_applied_exactly_once(): assert grok_rt._wire_voice("ara") == "xai_ara" assert grok_rt._wire_voice("xai_eve") == "xai_eve" diff --git a/tests/test_openai_realtime.py b/tests/test_openai_realtime.py index a0b8e28..ff25645 100644 --- a/tests/test_openai_realtime.py +++ b/tests/test_openai_realtime.py @@ -94,6 +94,47 @@ def _setup(*, automatic_response=True): ) +def test_semantic_endpointing_mint_and_update_use_the_same_exact_wire_object(monkeypatch): + setup = _setup(automatic_response=False) + setup = rt.SessionSetup( + model=setup.model, + voice=setup.voice, + instructions=setup.instructions, + tools=setup.tools, + automatic_response=setup.automatic_response, + turn_detection=rt.RealtimeTurnDetection( + mode=rt.RealtimeTurnDetectionMode.SEMANTIC_VAD, + semantic_eagerness=rt.RealtimeSemanticEagerness.MEDIUM, + ), + ) + minted = {} + + def fake_post(_auth_token, session): + minted.update(session) + return {"value": "ephemeral"} + + monkeypatch.setattr(openai_rt.talk_wire, "post_client_secret", fake_post) + openai_rt.talk_wire.mint_ephemeral_session( + auth_token="secret", + model=setup.model, + voice=setup.voice, + instructions=setup.instructions, + tools=[openai_rt._tool_wire(tool) for tool in setup.tools], + automatic_response=setup.automatic_response, + turn_detection=setup.turn_detection, + ) + update = openai_rt.build_session_update(setup) + expected = { + "type": "semantic_vad", + "eagerness": "medium", + "create_response": False, + "interrupt_response": True, + } + assert minted["audio"]["input"]["turn_detection"] == expected + assert update["session"]["audio"]["input"]["turn_detection"] == expected + assert update["session"] == {key: value for key, value in minted.items() if key != "model"} + + def _adapter(socket): client = _Client(socket) aiohttp = types.SimpleNamespace( @@ -224,6 +265,7 @@ def test_wire_credentials_are_private_one_shot_and_cleared_on_every_terminal_pat "instructions": "Be brief.", "tools": None, "automatic_response": False, + "turn_detection": rt.RealtimeTurnDetection(), "session_update": {"type": "session.update", "session": {}}, } diff --git a/tests/test_realtime_contract.py b/tests/test_realtime_contract.py index b148363..ae353f8 100644 --- a/tests/test_realtime_contract.py +++ b/tests/test_realtime_contract.py @@ -111,6 +111,28 @@ def test_setup_events_and_commands_cover_one_ordinary_turn(): assert all(isinstance(command, rt.RealtimeCommand) for command in commands) +def test_turn_detection_defaults_preserve_provider_native_behavior(): + setup = rt.SessionSetup(model="provider-model", voice="provider-voice", instructions="") + + assert setup.turn_detection == rt.RealtimeTurnDetection() + assert setup.turn_detection.mode is rt.RealtimeTurnDetectionMode.PROVIDER_NATIVE + assert setup.turn_detection.semantic_eagerness is None + + +def test_semantic_eagerness_is_valid_only_for_semantic_endpointing(): + semantic = rt.RealtimeTurnDetection( + mode=rt.RealtimeTurnDetectionMode.SEMANTIC_VAD, + semantic_eagerness=rt.RealtimeSemanticEagerness.HIGH, + ) + assert semantic.semantic_eagerness is rt.RealtimeSemanticEagerness.HIGH + + with pytest.raises(ValueError, match="only for semantic_vad"): + rt.RealtimeTurnDetection( + mode=rt.RealtimeTurnDetectionMode.SERVER_VAD, + semantic_eagerness=rt.RealtimeSemanticEagerness.LOW, + ) + + @pytest.mark.parametrize( ("factory", "match"), [ diff --git a/tests/test_wire.py b/tests/test_wire.py index 6809711..6bbb7e8 100644 --- a/tests/test_wire.py +++ b/tests/test_wire.py @@ -3,9 +3,11 @@ from __future__ import annotations import json +import types import pytest +import talk_realtime as rt import talk_wire @@ -28,6 +30,73 @@ def test_session_payload_enables_server_vad_and_barge_in(): assert payload["audio"]["output"]["voice"] == "cedar" +def test_turn_detection_encoder_has_exact_native_server_and_semantic_shapes(): + expected_server = { + "type": "server_vad", + "create_response": False, + "interrupt_response": True, + } + assert ( + talk_wire.encode_turn_detection( + rt.RealtimeTurnDetection(), + automatic_response=False, + ) + == expected_server + ) + assert ( + talk_wire.encode_turn_detection( + rt.RealtimeTurnDetection(mode=rt.RealtimeTurnDetectionMode.SERVER_VAD), + automatic_response=False, + ) + == expected_server + ) + assert talk_wire.encode_turn_detection( + rt.RealtimeTurnDetection(mode=rt.RealtimeTurnDetectionMode.SEMANTIC_VAD), + automatic_response=False, + ) == { + "type": "semantic_vad", + "eagerness": "auto", + "create_response": False, + "interrupt_response": True, + } + assert talk_wire.encode_turn_detection( + rt.RealtimeTurnDetection( + mode=rt.RealtimeTurnDetectionMode.SEMANTIC_VAD, + semantic_eagerness=rt.RealtimeSemanticEagerness.HIGH, + ), + automatic_response=True, + ) == { + "type": "semantic_vad", + "eagerness": "high", + "create_response": True, + "interrupt_response": True, + } + + +def test_invalid_endpointing_is_refused_before_mint_network(monkeypatch): + called = False + + def fake_post(_auth_token, _session): + nonlocal called + called = True + return {"value": "must-not-be-used"} + + monkeypatch.setattr(talk_wire, "post_client_secret", fake_post) + invalid = types.SimpleNamespace( + mode=rt.RealtimeTurnDetectionMode.SERVER_VAD, + semantic_eagerness=rt.RealtimeSemanticEagerness.LOW, + ) + with pytest.raises(ValueError, match="only for semantic"): + talk_wire.mint_ephemeral_session( + auth_token="secret", + model="gpt-realtime-test", + voice="cedar", + instructions="brief", + turn_detection=invalid, + ) + assert called is False + + def test_input_only_payload_disables_automatic_response_at_mint_time(): payload = talk_wire.build_session_payload( model="gpt-realtime-2.1",