fix(tokenize): don't stall TTS streaming on angle-bracketed prose

livekit-agents/livekit/agents/tokenize/token_stream.py

@@ -13,8 +13,12 @@
 
 # the tag name must start with a letter so "<5>" / "<3 wins>" are not counted as
 # tags — this keeps the depth counter consistent with the letter-start tail check
-# in _has_unclosed_xml_tags (all TTS markup tags are letter-named)
-_XML_TAG_RE = re.compile(r"<(/?)([A-Za-z]\w*)[^>]*?(/?)\s*>")
+# in _has_unclosed_xml_tags (all TTS markup tags are letter-named).
+# The name must also be followed by whitespace, "/" or ">", as it is in real markup:
+# without that, angle-bracketed prose like <https://lk.io> or <bob@example.com>
+# parses as an open tag and holds every later sentence until flush. Hyphens are part
+# of the name so xAI's <higher-pitch>/<build-intensity> still match.
+_XML_TAG_RE = re.compile(r"<(/?)([A-Za-z][\w-]*)(?=[\s/>])[^>]*?(/?)\s*>")
 
 
 def _has_unclosed_xml_tags(text: str) -> bool:

tests/test_tokenizer_xml_markup.py

@@ -665,6 +665,48 @@ async def test_tag_shaped_text_streams_when_not_xml_aware(self) -> None:
         assert "bob@example.com" in ev.token
         stream.end_input()
 
+    def test_angle_bracketed_prose_is_not_a_tag(self) -> None:
+        # regression: an XML name is followed by whitespace, "/" or ">". Without that
+        # rule the depth counter read a markdown autolink or an angle-bracketed address
+        # as an unclosed open tag, so depth stayed > 0 and every later sentence was held
+        # until flush — the same end-of-turn batching the bare "<" fix removed
+        from livekit.agents.tokenize.token_stream import _has_unclosed_xml_tags
+
+        assert not _has_unclosed_xml_tags("Docs are at <https://docs.livekit.io> today.")
+        assert not _has_unclosed_xml_tags("Email me at <bob@example.com> please.")
+        assert not _has_unclosed_xml_tags("Press <ctrl+c> to quit.")
+        assert not _has_unclosed_xml_tags("See <http://example.com/a?b=1> for details.")
+
+    def test_hyphenated_tag_names_still_counted(self) -> None:
+        # the guard must not over-correct: xAI's prosody tags carry hyphens, and an
+        # unclosed one still has to hold the sentence
+        from livekit.agents.tokenize.token_stream import _has_unclosed_xml_tags
+
+        assert _has_unclosed_xml_tags("<higher-pitch>no way")
+        assert not _has_unclosed_xml_tags("<higher-pitch>no way</higher-pitch> done")
+        assert not _has_unclosed_xml_tags('<break time="500ms"/> done')
+
+    @pytest.mark.asyncio
+    async def test_autolink_streams_with_xml_aware(self) -> None:
+        # end-to-end: an expressive agent quoting a URL must keep streaming
+        tok = SentenceTokenizer(min_sentence_len=1, stream_context_len=5, xml_aware=True)
+        stream = tok.stream()
+        stream.push_text(
+            "Docs are at <https://docs.livekit.io> now. And a second sentence to split."
+        )
+        ev = await asyncio.wait_for(stream.__anext__(), timeout=1)
+        assert "docs.livekit.io" in ev.token
+        stream.end_input()
+
+    @pytest.mark.asyncio
+    async def test_email_streams_with_xml_aware(self) -> None:
+        tok = SentenceTokenizer(min_sentence_len=1, stream_context_len=5, xml_aware=True)
+        stream = tok.stream()
+        stream.push_text("Email me at <bob@example.com> please. Second sentence for the split.")
+        ev = await asyncio.wait_for(stream.__anext__(), timeout=1)
+        assert "bob@example.com" in ev.token
+        stream.end_input()
+
 
 # ===========================================================================
 # Universal transcript stripping (provider-agnostic, used by the transcript sinks)