` XML tags to `[...]` bracket format. */
+export function convertExpressionTags(text: string): string {
+ return text
+ .replace(EXPRESSION_RE, (_m, value: string) => `[${value}]`)
+ .replace(SOUND_RE, (_m, value: string) => `[${value}]`);
+}
+
+const VALUE_ATTR_RE = /\b[\w-]+\s*=\s*"([^"]*)"/;
+
+/**
+ * Horizontal whitespace immediately before a tag. Every removal pattern captures it as
+ * `pre` so {@link dedupRemovalSpace} can decide whether to keep it; newlines are excluded
+ * so paragraph breaks are never touched.
+ *
+ * @internal
+ */
+export const LEADING_WS = '(?[^\\S\\r\\n]*)';
+
+/**
+ * Replacement text for a stripped tag, minus the space its removal would double.
+ *
+ * The instructions place an expression marker before *every* sentence, so a turn is
+ * written with a marker delimited like a word — a space on each side — at every sentence
+ * boundary:
+ *
+ * ```
+ * Oh no, I'm sorry to hear
+ * that. I can certainly see what
+ * we have available for you.
+ * ```
+ *
+ * Both spaces are correct while the marker is there. Stripping it for the transcript
+ * collapses its width to zero and leaves both behind, so every sentence lands with two
+ * spaces after its punctuation (`"to hear that. I can certainly"`) — every sentence of
+ * every expressive turn, not an edge case.
+ *
+ * When nothing of the tag survives and whitespace follows the match, the whitespace
+ * captured *before* it (the `pre` group) is therefore dropped so a single separator
+ * remains — matching what `dropBracketCues` already does for bracket cues.
+ *
+ * Whitespace before a tag at the very end of *text* is kept: it may be the separator for
+ * words still streaming in, and the sinks dedup that seam themselves.
+ *
+ * @param pre - The whitespace the pattern captured before the tag.
+ * @param kept - The text that survives the removal (a wrapping tag's inner text or the
+ * native tag it lowers to), or `''` when the tag vanishes entirely.
+ * @param source - The full string being replaced.
+ * @param matchEnd - Index just past the end of the match in `source`.
+ *
+ * @internal
+ */
+export function dedupRemovalSpace(
+ pre: string,
+ kept: string,
+ source: string,
+ matchEnd: number,
+): string {
+ if (kept) return pre + kept;
+ if (!pre) return '';
+ const nxt = source.slice(matchEnd, matchEnd + 1);
+ return nxt !== '' && /\s/.test(nxt) ? '' : pre;
+}
+
+/**
+ * A replacer that receives the named groups, the match offset and the source string.
+ *
+ * `String.prototype.replace` passes `(match, ...groups, offset, string, groups?)`; the
+ * groups object is only appended when the pattern has named groups, so the positional
+ * arguments have to be read from the tail.
+ *
+ * @internal
+ */
+export function replaceWithGroups(
+ text: string,
+ pattern: RegExp,
+ fn: (args: {
+ match: string;
+ groups: Record;
+ offset: number;
+ source: string;
+ }) => string,
+): string {
+ return text.replace(pattern, (...args: unknown[]): string => {
+ const match = args[0] as string;
+ const groups = (args[args.length - 1] ?? {}) as Record;
+ const source = args[args.length - 2] as string;
+ const offset = args[args.length - 3] as number;
+ return fn({ match, groups, offset, source });
+ });
+}
+
+/** A markup tag stripped from text: the XML tag name and its payload. */
+export type StrippedTag = [tag: string, value: string];
+
+/**
+ * Strip XML markup tags and collect the stripped tags in a single pass.
+ *
+ * One regex scan both removes the markup and records each removed tag, so stripping and
+ * extraction can never disagree about what counts as a tag.
+ *
+ * Only XML-shaped markup is recognized. Square brackets are left alone: in LLM output
+ * they are prose (`[text](url)` links) that a strip would mangle, and provider-native
+ * ones are removed at their source by `dropBracketCues`.
+ *
+ * Returns `[cleanText, tags]` where `tags` is a list of `[type, value]` pairs in order of
+ * appearance:
+ *
+ * - `type` is the XML tag name.
+ * - `value` is a content tag's inner text (`A7X9` -> `"A7X9"`), else its
+ * first quoted attribute value (`` -> `"happy"`), falling back
+ * to `""`. Names in `attributeTags` invert that preference — see the parameter.
+ *
+ * Wrapping tags keep their inner content in `cleanText` (only the delimiters are
+ * removed); self-closing and lone tags are removed entirely.
+ *
+ * @param text - The text containing markup.
+ * @param xmlTags - XML tag names to handle (e.g. `['emotion', 'sound']`).
+ * @param attributeTags - Tag names whose payload is an attribute, never their content
+ * (`expression`, `emotion`, ...). These are self-closing by definition, but a model
+ * that writes `Hello there` would otherwise have
+ * the spoken sentence recorded as the delivery label and published as `lk.expression`.
+ * `normalizeMarkup` repairs that tag shape only on the audio path, so the transcript
+ * sinks see the raw form and have to handle it here.
+ */
+export function extractAndStrip(
+ text: string,
+ xmlTags: string[],
+ attributeTags: ReadonlySet = new Set(),
+): [string, StrippedTag[]] {
+ if (xmlTags.length === 0) {
+ return [text, []];
+ }
+
+ const tagPattern = xmlTags.map(escapeRegExp).join('|');
+ const pattern = new RegExp(
+ // leading space is part of the match so removing a tag can't double the separator
+ LEADING_WS +
+ '(?:' +
+ // self-closing ``, matched first and terminal: it has no span, so it must
+ // never consume a following `` — that would swallow whatever sat between the
+ // two and record it as this tag's value
+ `<(?${tagPattern})\\b(?[^>]*?)\\s*/\\s*>` +
+ // `` optionally followed by inner
+ `|<(?${tagPattern})\\b(?[^>]*?)\\s*>` +
+ '(?:(?.*?)\\s*>)?' +
+ // lone closing tag:
+ `|` +
+ ')',
+ 'gs',
+ );
+
+ const tags: StrippedTag[] = [];
+
+ const replacer = ({
+ groups,
+ match,
+ offset,
+ source,
+ }: {
+ match: string;
+ groups: Record;
+ offset: number;
+ source: string;
+ }): string => {
+ const pre = groups.pre ?? '';
+ const end = offset + match.length;
+
+ if (groups.selfTag !== undefined) {
+ const attrMatch = VALUE_ATTR_RE.exec(groups.selfAttrs ?? '');
+ tags.push([groups.selfTag, attrMatch ? attrMatch[1]! : '']);
+ return dedupRemovalSpace(pre, '', source, end); // self-closing tags vanish
+ }
+
+ const tag = groups.tag;
+ if (tag !== undefined) {
+ const inner = groups.inner;
+ const attrMatch = VALUE_ATTR_RE.exec(groups.attrs ?? '');
+ const attrValue = attrMatch ? attrMatch[1]! : '';
+ // an attribute-carrying tag's payload is the attribute even when the model wrapped
+ // text in it; everything else is a content tag, whose inner text wins
+ let value: string;
+ if (attributeTags.has(tag)) {
+ value = attrValue || (inner?.trim() ?? '');
+ } else if (inner !== undefined && inner.trim()) {
+ value = inner.trim();
+ } else {
+ value = attrValue;
+ }
+ tags.push([tag, value]);
+ // wrapping tags keep their inner content; lone open tags vanish
+ return dedupRemovalSpace(pre, inner || '', source, end);
+ }
+ return dedupRemovalSpace(pre, '', source, end); // lone closing tag
+ };
+
+ // iterate to a fixed point so nested wrapping tags are fully removed: a single pass
+ // strips only the outer tag (e.g. hi -> keeps the
+ // inner hi), so repeat until the text stops changing. Each pass removes
+ // at least the matched delimiters, so this always terminates.
+ let clean = text;
+ let prev: string | undefined;
+ while (clean !== prev) {
+ prev = clean;
+ clean = replaceWithGroups(clean, pattern, replacer);
+ }
+ return [clean, tags];
+}
+
+/** @internal */
+export function escapeRegExp(value: string): string {
+ return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
diff --git a/agents/src/tts/mood.ts b/agents/src/tts/mood.ts
new file mode 100644
index 000000000..7b522a201
--- /dev/null
+++ b/agents/src/tts/mood.ts
@@ -0,0 +1,99 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Normalize a free-form delivery label into a small, fixed set of moods.
+ *
+ * The label space is open-ended: Fish Audio emits single words from a closed set, Inworld
+ * emits free-form English ("soft, with genuine care"), and models drift outside whichever
+ * set they were given. Matching here is best-effort, and happens agent-side so no client
+ * SDK needs its own copy of the keyword table.
+ */
+import { MOOD_KEYWORDS } from './mood_data.js';
+
+export type AgentMood =
+ | 'excited'
+ | 'happy'
+ | 'playful'
+ | 'curious'
+ | 'surprised'
+ | 'hopeful'
+ | 'empathetic'
+ | 'sad'
+ | 'angry'
+ | 'anxious'
+ | 'calm';
+
+/**
+ * Tie-break order, most specific first. Two moods scoring equally on a compound label
+ * resolve to whichever appears earlier here.
+ */
+export const MOOD_PRIORITY: AgentMood[] = [
+ 'angry',
+ 'sad',
+ 'anxious',
+ 'surprised',
+ 'playful',
+ 'empathetic',
+ 'excited',
+ 'curious',
+ 'hopeful',
+ 'happy',
+ 'calm',
+];
+
+/**
+ * `calm` is the most recessive mood, so an unmatched label reads as "no strong signal"
+ * rather than asserting a feeling the agent never expressed.
+ */
+export const DEFAULT_MOOD: AgentMood = 'calm';
+
+const ALPHA_RE = /[a-z]/i;
+
+function matchesWord(text: string, keyword: string): boolean {
+ // word starts only: matching mid-word read "like a pirate" as `angry`, via the "irate" stem
+ let start = 0;
+ for (;;) {
+ const at = text.indexOf(keyword, start);
+ if (at === -1) return false;
+ if (at === 0 || !ALPHA_RE.test(text[at - 1]!)) return true;
+ start = at + 1;
+ }
+}
+
+/**
+ * Match a raw delivery label to a mood.
+ *
+ * Matching is keyword-based and deliberately lossy: the label space is open-ended, so an
+ * unrecognized label resolves to `fallback` rather than a wrong guess.
+ *
+ * @param label - The raw delivery label, as the provider wrote it.
+ * @param fallback - Mood to return when nothing matches. Pass `null` to handle the miss
+ * yourself. Defaults to {@link DEFAULT_MOOD}.
+ *
+ * @example
+ * ```ts
+ * matchMood('soft, with genuine care'); // 'empathetic'
+ * matchMood('like a pirate'); // 'calm'
+ * ```
+ */
+export function matchMood(label: string, fallback: AgentMood | null = DEFAULT_MOOD) {
+ const text = label.toLowerCase();
+ let best: AgentMood | null = null;
+ let bestScore = 0;
+
+ for (const mood of MOOD_PRIORITY) {
+ let score = 0;
+ for (const [keyword, weight] of Object.entries(MOOD_KEYWORDS[mood])) {
+ if (matchesWord(text, keyword)) score += weight;
+ }
+ // strictly greater, so MOOD_PRIORITY breaks ties
+ if (score > bestScore) {
+ best = mood;
+ bestScore = score;
+ }
+ }
+
+ return best !== null ? best : fallback;
+}
diff --git a/agents/src/tts/mood_data.ts b/agents/src/tts/mood_data.ts
new file mode 100644
index 000000000..87b1ea0b8
--- /dev/null
+++ b/agents/src/tts/mood_data.ts
@@ -0,0 +1,358 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { AgentMood } from './mood.js';
+
+/**
+ * Weighted keywords per mood: the emotion words from Parrott's hierarchical classification
+ * of emotions (2001) mapped onto the moods in `mood.ts`, plus the delivery descriptors
+ * providers use in place of naming a feeling ("bright", "hushed", "clipped").
+ *
+ * Weight 2 names a mood outright; weight 1 is a supporting descriptor that loses to an
+ * explicit naming elsewhere in the same label. That is what lets "gently curious,
+ * welcoming" resolve to `curious` rather than `empathetic`.
+ */
+export const MOOD_KEYWORDS: Record> = {
+ excited: {
+ excit: 2,
+ elat: 2,
+ thrill: 2,
+ exhilarat: 2,
+ ecstat: 2,
+ euphor: 2,
+ zeal: 2,
+ zest: 2,
+ enthusias: 2,
+ eager: 2,
+ giddy: 2,
+ hyped: 2,
+ pumped: 2,
+ buzzing: 2,
+ jubilant: 2,
+ jubilation: 2,
+ exuberant: 2,
+ rapture: 2,
+ enthrall: 2,
+ triumph: 2,
+ gleeful: 2,
+ glee: 2,
+ punchy: 2,
+ upbeat: 1,
+ bright: 1,
+ energetic: 1,
+ energy: 1,
+ lively: 1,
+ animated: 1,
+ vibrant: 1,
+ spirited: 1,
+ peppy: 1,
+ snappy: 1,
+ fast: 1,
+ loud: 1,
+ },
+ happy: {
+ happy: 2,
+ happiness: 2,
+ joy: 2,
+ joviality: 2,
+ jolli: 2,
+ cheer: 2,
+ glad: 2,
+ delight: 2,
+ pleas: 2,
+ content: 2,
+ bliss: 2,
+ gaiety: 2,
+ enjoy: 2,
+ satisf: 2,
+ relief: 2,
+ relieved: 2,
+ grateful: 2,
+ thankful: 2,
+ affection: 2,
+ fond: 2,
+ adore: 2,
+ proud: 2,
+ pride: 2,
+ smil: 2,
+ sunny: 2,
+ merry: 2,
+ amiable: 2,
+ warm: 1,
+ inviting: 1,
+ welcom: 1,
+ friendly: 1,
+ kind: 1,
+ pleasant: 1,
+ positive: 1,
+ easy: 1,
+ },
+ playful: {
+ playful: 2,
+ jok: 2,
+ comedic: 2,
+ comic: 2,
+ sarcas: 2,
+ teas: 2,
+ witty: 2,
+ silly: 2,
+ goofy: 2,
+ mischiev: 2,
+ amus: 2,
+ humor: 2,
+ humour: 2,
+ cheeky: 2,
+ sassy: 2,
+ ironic: 2,
+ irony: 2,
+ deadpan: 2,
+ banter: 2,
+ laugh: 2,
+ giggl: 2,
+ chuckl: 2,
+ grin: 2,
+ unimpressed: 2,
+ smirk: 2,
+ wry: 1,
+ sly: 1,
+ impish: 1,
+ },
+ curious: {
+ curious: 2,
+ curiosity: 2,
+ inquisitive: 2,
+ intrigu: 2,
+ wonder: 2,
+ quizzical: 2,
+ probing: 2,
+ interested: 2,
+ engaged: 2,
+ attentive: 2,
+ suspense: 2,
+ questioning: 1,
+ question: 1,
+ prompting: 1,
+ exploring: 1,
+ },
+ surprised: {
+ surpris: 2,
+ amaz: 2,
+ astonish: 2,
+ astound: 2,
+ awe: 2,
+ shock: 2,
+ startl: 2,
+ incredulous: 2,
+ stunned: 2,
+ bewilder: 2,
+ flabbergast: 2,
+ disbelie: 2,
+ unexpected: 2,
+ dumbfound: 2,
+ wow: 2,
+ whoa: 2,
+ gasp: 2,
+ },
+ hopeful: {
+ hopeful: 2,
+ hope: 2,
+ optimis: 2,
+ encourag: 2,
+ uplift: 2,
+ inspir: 2,
+ motivat: 2,
+ promising: 2,
+ determined: 2,
+ resolute: 2,
+ forward: 2,
+ buoyant: 2,
+ expectant: 2,
+ heartened: 2,
+ confident: 2,
+ assured: 2,
+ },
+ empathetic: {
+ empath: 2,
+ sympath: 2,
+ compassion: 2,
+ concern: 2,
+ care: 2,
+ tender: 2,
+ consol: 2,
+ sorry: 2,
+ apolog: 2,
+ understanding: 2,
+ supportive: 2,
+ comfort: 2,
+ soothing: 2,
+ nurtur: 2,
+ patient: 2,
+ earnest: 2,
+ heartfelt: 2,
+ sensitive: 2,
+ pity: 2,
+ condolence: 2,
+ sentimental: 2,
+ sincere: 1,
+ gentle: 1,
+ gently: 1,
+ soft: 1,
+ quiet: 1,
+ hushed: 1,
+ mild: 1,
+ reassur: 1,
+ },
+ sad: {
+ sad: 2,
+ sorrow: 2,
+ mourn: 2,
+ somber: 2,
+ sombre: 2,
+ melanchol: 2,
+ grief: 2,
+ griev: 2,
+ regret: 2,
+ remorse: 2,
+ deject: 2,
+ downcast: 2,
+ gloom: 2,
+ glum: 2,
+ forlorn: 2,
+ wistful: 2,
+ disappoint: 2,
+ dismay: 2,
+ crestfallen: 2,
+ despond: 2,
+ despair: 2,
+ depress: 2,
+ anguish: 2,
+ agony: 2,
+ woe: 2,
+ miser: 2,
+ unhappy: 2,
+ tearful: 2,
+ weep: 2,
+ heartbroken: 2,
+ heartbreak: 2,
+ lonely: 2,
+ loneliness: 2,
+ ashamed: 2,
+ shame: 2,
+ guilt: 2,
+ humiliat: 2,
+ mortified: 2,
+ longing: 2,
+ homesick: 2,
+ defeated: 2,
+ heavy: 1,
+ subdued: 1,
+ flat: 1,
+ weary: 1,
+ resigned: 1,
+ hollow: 1,
+ },
+ angry: {
+ angry: 2,
+ anger: 2,
+ furious: 2,
+ fury: 2,
+ frustrat: 2,
+ annoy: 2,
+ irritat: 2,
+ irate: 2,
+ indignant: 2,
+ outrag: 2,
+ exasperat: 2,
+ incensed: 2,
+ livid: 2,
+ wrath: 2,
+ hostil: 2,
+ resent: 2,
+ bitter: 2,
+ disgust: 2,
+ revulsion: 2,
+ contempt: 2,
+ loathing: 2,
+ scorn: 2,
+ spite: 2,
+ aggravat: 2,
+ agitat: 2,
+ grouchy: 2,
+ grumpy: 2,
+ seething: 2,
+ fuming: 2,
+ scold: 2,
+ stern: 2,
+ harsh: 2,
+ sharp: 1,
+ clipped: 1,
+ terse: 1,
+ cold: 1,
+ biting: 1,
+ curt: 1,
+ },
+ anxious: {
+ anxious: 2,
+ anxiety: 2,
+ afraid: 2,
+ fear: 2,
+ fright: 2,
+ scared: 2,
+ nervous: 2,
+ worri: 2,
+ uneasy: 2,
+ unease: 2,
+ tense: 2,
+ apprehensive: 2,
+ panic: 2,
+ alarm: 2,
+ dread: 2,
+ terror: 2,
+ horror: 2,
+ hesitant: 2,
+ unsure: 2,
+ uncertain: 2,
+ timid: 2,
+ jittery: 2,
+ urgent: 2,
+ stress: 2,
+ distress: 2,
+ insecure: 2,
+ flustered: 2,
+ cautious: 1,
+ wary: 1,
+ guarded: 1,
+ },
+ calm: {
+ calm: 2,
+ contemplative: 2,
+ thoughtful: 2,
+ measured: 2,
+ serene: 2,
+ easygoing: 2,
+ neutral: 2,
+ relax: 2,
+ composed: 2,
+ collected: 2,
+ tranquil: 2,
+ peaceful: 2,
+ restrained: 2,
+ reflective: 2,
+ pensive: 2,
+ matter: 2,
+ plain: 2,
+ professional: 2,
+ formal: 2,
+ informative: 2,
+ factual: 2,
+ deliberate: 2,
+ unhurried: 2,
+ placid: 2,
+ slow: 2,
+ steady: 1,
+ grounded: 1,
+ balanced: 1,
+ straightforward: 1,
+ even: 1,
+ },
+};
diff --git a/agents/src/tts/provider_format.test.ts b/agents/src/tts/provider_format.test.ts
new file mode 100644
index 000000000..2584de12b
--- /dev/null
+++ b/agents/src/tts/provider_format.test.ts
@@ -0,0 +1,759 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Tests for the LiveKit expression marker (expr) dialect.
+ *
+ * The LLM emits a single marker tag — `` (self-closing for
+ * expression/break/sound, wrapping for prosody/spell) — and the framework lowers it to
+ * each provider's native markup before synthesis while stripping it from transcripts. The
+ * syntax is shared, but the kinds and label vocabularies are per provider: each provider's
+ * instruction block advertises only what that provider supports.
+ */
+import { describe, expect, it } from 'vitest';
+import { ChatContext, ChatMessage } from '../llm/chat_context.js';
+import { type TimedString, createTimedString } from '../voice/io.js';
+import { matchMood } from './mood.js';
+import {
+ TranscriptMarkupStripper,
+ convertMarkup,
+ dropBracketCues,
+ expressionAttribute,
+ llmInstructions,
+ maxInputLen,
+ normalizeMarkup,
+ sentenceTokenizer,
+ splitAllMarkup,
+ steeringInstructions,
+ stripAllMarkup,
+ stripExprMarkup,
+ supportedNonverbals,
+} from './provider_format.js';
+
+// Inworld-flavored turn: free-form expression + sound + break
+const JOKE =
+ ' Why did the burger go to the gym? ' +
+ ' Because it wanted better buns! ' +
+ '';
+
+describe('convertMarkup: expr -> xAI', () => {
+ it('lowers sounds, breaks and wrapping prosody', () => {
+ const text =
+ 'So I walked in and there it was! ' +
+ ' ' +
+ 'It was a secret the whole time.';
+ expect(convertMarkup('xai', text)).toBe(
+ 'So I walked in and [pause] there it was! [laugh] ' +
+ 'It was a secret the whole time.',
+ );
+ });
+
+ it('maps break durations to the two native pause levels', () => {
+ expect(convertMarkup('xai', '')).toBe('[pause]');
+ expect(convertMarkup('xai', '')).toBe('[long-pause]');
+ });
+
+ it('aliases an Inworld-style sound label to the native cue', () => {
+ expect(convertMarkup('xai', '')).toBe('[breath]');
+ });
+
+ it('normalizes multi-word prosody labels to hyphenated tag names', () => {
+ const text = 'no way';
+ expect(convertMarkup('xai', text)).toBe('no way');
+ });
+
+ it('unwraps an unknown prosody label', () => {
+ const text = 'ahoy there';
+ expect(convertMarkup('xai', text)).toBe('ahoy there');
+ });
+
+ it('drops a hallucinated expression marker from the audio path', () => {
+ // xAI has no free-form delivery descriptions; the marker still surfaces in
+ // transcript tags
+ const text = ' Hello!';
+ expect(convertMarkup('xai', text)).toBe(' Hello!');
+ });
+});
+
+describe('convertMarkup: expr -> Inworld', () => {
+ it('lowers expression/sound to brackets and keeps break as native SSML', () => {
+ expect(convertMarkup('inworld', JOKE)).toBe(
+ '[say playfully] Why did the burger go to the gym? ' +
+ ' Because it wanted better buns! [laugh]',
+ );
+ });
+
+ it('salvages a stray prosody marker as a delivery hint', () => {
+ const text = 'keep it secret';
+ expect(convertMarkup('inworld', text)).toBe('[whisper]keep it secret');
+ });
+});
+
+describe('convertMarkup: expr -> Cartesia', () => {
+ it('lowers expression to , keeps break, drops sound', () => {
+ const text =
+ ' We won! ' +
+ ' Unbelievable.';
+ // without leaving the space the dropped marker sat between behind as a doubled
+ // separator
+ expect(convertMarkup('cartesia', text)).toBe(
+ ' We won! Unbelievable.',
+ );
+ });
+
+ it('lowers spell to the native tag', () => {
+ const text = 'Your code is A7X9.';
+ expect(convertMarkup('cartesia', text)).toBe('Your code is A7X9.');
+ });
+
+ it('unwraps spell for providers that lack it', () => {
+ const text = 'Your code is A7X9.';
+ expect(convertMarkup('xai', text)).toBe('Your code is A7X9.');
+ expect(convertMarkup('inworld', text)).toBe('Your code is A7X9.');
+ });
+
+ it('lowers prosody labels to native speed/volume point controls', () => {
+ expect(convertMarkup('cartesia', ' One moment.')).toBe(
+ ' One moment.',
+ );
+ expect(convertMarkup('cartesia', ' We won!')).toBe(
+ ' We won!',
+ );
+ // wrapping form applies the control before the span
+ expect(convertMarkup('cartesia', 'bad news')).toBe(
+ 'bad news',
+ );
+ });
+
+ it('unwraps an unknown prosody label', () => {
+ const text = 'keep it secret';
+ expect(convertMarkup('cartesia', text)).toBe('keep it secret');
+ });
+
+ it('does not let a self-closing prosody marker swallow a later span', () => {
+ // Cartesia's prosody markers are self-closing point controls. Read as an opening tag,
+ // one would run to the next and eat the spell marker with it — the code would
+ // reach the TTS as a bare word and be pronounced instead of spelled out.
+ const text =
+ ' One moment. ' +
+ 'Your code is A7X9.';
+ expect(convertMarkup('cartesia', text)).toBe(
+ ' One moment. Your code is A7X9.',
+ );
+ });
+
+ it('keeps each self-closing marker distinct across a sentence', () => {
+ const text =
+ ' Big news! Or not.';
+ expect(convertMarkup('cartesia', text)).toBe(
+ ' Big news! Or not.',
+ );
+ });
+});
+
+describe('convertMarkup: expr -> Fish Audio', () => {
+ it('lowers expression to an intensified bracket cue and sounds to their aliases', () => {
+ const text =
+ ' That\'s on us. ' +
+ '';
+ expect(convertMarkup('fishaudio', text)).toBe("[very regretful] That's on us. [sighing]");
+ });
+
+ it('lowers a tone wrapper to Fish’s prefix marker form', () => {
+ const text = 'don\'t tell anyone';
+ expect(convertMarkup('fishaudio', text)).toBe("[whispering] don't tell anyone");
+ });
+
+ it('lowers emphasis to the per-word stress marker', () => {
+ const text = 'Are you sure?';
+ expect(convertMarkup('fishaudio', text)).toBe('Are you [emphasis] sure?');
+ });
+
+ it('maps break durations to the two native pause levels', () => {
+ expect(convertMarkup('fishaudio', '')).toBe('[break]');
+ expect(convertMarkup('fishaudio', '')).toBe('[long-break]');
+ });
+});
+
+it('drops a stray expr tag rather than letting it reach the TTS', () => {
+ // an unpaired prosody open/close (e.g. split across stream chunks) is dropped, keeping
+ // the words
+ expect(convertMarkup('xai', 'hello there')).toBe('hello there');
+ expect(convertMarkup('xai', 'hello there')).toBe('hello there');
+});
+
+describe('transcript stripping (provider-agnostic)', () => {
+ it('strips expr and reports its tags', () => {
+ const [clean, tags] = splitAllMarkup(JOKE);
+ expect(clean.trim()).toBe('Why did the burger go to the gym? Because it wanted better buns!');
+ expect(tags).toEqual([
+ { type: 'expression', value: 'say playfully' },
+ { type: 'break', value: '500ms' },
+ { type: 'sound', value: 'laugh' },
+ ]);
+ });
+
+ it('keeps the inner text of a wrapping marker', () => {
+ const text =
+ 'She said keep it secret — ' +
+ 'code A7X9.';
+ const [clean, tags] = splitAllMarkup(text);
+ expect(clean).toBe('She said keep it secret — code A7X9.');
+ expect(tags).toEqual([
+ { type: 'prosody', value: 'whisper' },
+ { type: 'spell', value: '' },
+ ]);
+ });
+
+ it('strips mixed expr and provider-native tags', () => {
+ const text = ' Hello! ';
+ const [clean, tags] = splitAllMarkup(text);
+ expect(clean.trim()).toBe('Hello!');
+ expect(tags).toContainEqual({ type: 'expression', value: 'say playfully' });
+ expect(tags).toContainEqual({ type: 'sound', value: 'laugh' });
+ });
+
+ it('keeps square-bracket spans', () => {
+ // bracket spans are a TTS-only native form (convertMarkup emits them on the audio
+ // path), so the transcript strip must leave markdown links and prose brackets intact
+ const text =
+ 'Press [Enter], then read [the docs](https://docs.livekit.io). ';
+ const [clean, tags] = splitAllMarkup(text);
+ expect(clean).toBe('Press [Enter], then read [the docs](https://docs.livekit.io). ');
+ expect(tags).toEqual([{ type: 'sound', value: 'sigh' }]);
+ });
+
+ it('does not match the native tag with the expr regexes', () => {
+ // " there.');
+ });
+
+ it('holds a tag split across streamed chunks', () => {
+ const stripper = new TranscriptMarkupStripper();
+ let out = '';
+ // split mid-tag so the partial " Hello',
+ ' wor',
+ 'ld!',
+ ]) {
+ out += stripper.push(chunk);
+ }
+ out += stripper.flush();
+ // no leading space: the marker opened the segment, so the space it left behind is
+ // trimmed (Python leaves it, and its own tests .strip() around it)
+ expect(out).toBe('Hello world!');
+ expect(stripper.tags[0]).toEqual({ type: 'expression', value: 'say playfully' });
+ expect(stripper.tags).toContainEqual({ type: 'prosody', value: 'whisper' });
+ });
+
+ it('leaves a single space where a removed tag sat between two', () => {
+ // a marker between two spaces must not leave both behind, or punctuation ends up
+ // followed by a double space in the transcript
+ expect(stripAllMarkup('Right. Anyway.')).toBe(
+ 'Right. Anyway.',
+ );
+ expect(stripAllMarkup('Right. Anyway.')).toBe('Right. Anyway.');
+ // a wrapping marker keeps its inner text, so its spacing is untouched
+ expect(stripAllMarkup('a b c')).toBe('a b c');
+ // only the doubled separator goes: a marker with text on one side keeps the space
+ expect(stripAllMarkup('Right. Anyway.')).toBe(
+ 'Right. Anyway.',
+ );
+ expect(stripAllMarkup('Right. Anyway.')).toBe(
+ 'Right. Anyway.',
+ );
+ // newlines are structure, not a separator a strip may collapse
+ expect(stripAllMarkup('a\n\nb')).toBe('a\n\nb');
+ });
+
+ it('keeps the trailing space before a marker at the end of a chunk', () => {
+ // the space before a trailing marker is the separator for words still streaming in,
+ // so it survives the strip (the seam is deduped by TranscriptMarkupStripper)
+ expect(stripAllMarkup('Right. ')).toBe('Right. ');
+ });
+
+ it('dedups the space across chunk seams', () => {
+ // the space before the marker goes out with the previous chunk, so the in-text dedup
+ // can't see it — the stripper has to close that seam itself
+ for (const chunks of [
+ ['Right. ', '', ' Anyway.'],
+ ['Right. ', ' Anyway.'],
+ ['Right. ', '', ' Anyway.'],
+ ]) {
+ const stripper = new TranscriptMarkupStripper();
+ const out = chunks.map((c) => stripper.push(c)).join('') + stripper.flush();
+ expect(out, chunks.join('|')).toBe('Right. Anyway.');
+ }
+ });
+
+ it('leaves untagged whitespace alone', () => {
+ // without a stripped tag at the seam there is nothing to dedup: whitespace the LLM
+ // itself emitted is passed through untouched
+ let stripper = new TranscriptMarkupStripper();
+ let out = ['Right. ', ' Anyway.'].map((c) => stripper.push(c)).join('') + stripper.flush();
+ expect(out).toBe('Right. Anyway.');
+
+ // a tag stripped earlier in the chunk doesn't license collapsing the seam either:
+ // the whitespace here trails "hello", not the removed tag
+ stripper = new TranscriptMarkupStripper();
+ out =
+ ['hello ', ' world'].map((c) => stripper.push(c)).join('') +
+ stripper.flush();
+ expect(out).toBe('hello world');
+ });
+
+ it('builds the lk.expression attribute from the leading expression', () => {
+ const [, tags] = splitAllMarkup(JOKE);
+ const attr = expressionAttribute(tags);
+ expect(attr).toBeDefined();
+ expect(Object.values(attr!)[0]).toContain('"say playfully"');
+ });
+
+ it('has no lk.expression attribute without an expression tag', () => {
+ const [, tags] = splitAllMarkup('Hi there.');
+ expect(expressionAttribute(tags)).toBeUndefined();
+ });
+});
+
+describe('normalizeMarkup', () => {
+ it.each(['xai', 'inworld', 'cartesia', 'fishaudio'])(
+ 'closes an unclosed self-closing expr marker for %s',
+ (provider) => {
+ const text = ' Hello';
+ expect(normalizeMarkup(provider, text)).toBe(' Hello');
+ },
+ );
+
+ it('leaves wrapping and already-closed tags alone', () => {
+ const text =
+ 'hi ' +
+ 'A7X9';
+ expect(normalizeMarkup('xai', text)).toBe(text);
+ });
+});
+
+describe('llmInstructions', () => {
+ it.each(['xai', 'inworld', 'cartesia', 'fishaudio'])('uses expr syntax for %s', (provider) => {
+ const instructions = llmInstructions(provider);
+ expect(instructions).toBeDefined();
+ expect(instructions).toContain('');
+ expect(instructions).toContain('NOT free-form');
+ expect(instructions).toContain('');
+ // coarse self-closing prosody point controls
+ expect(instructions).toContain('');
+ // no non-verbal sounds
+ expect(instructions).not.toContain('type="sound"');
+ });
+
+ it('advertises Inworld’s kinds only', () => {
+ const instructions = llmInstructions('inworld')!;
+ // free-form delivery descriptions + Inworld's own sound list
+ expect(instructions).toContain('');
+ expect(instructions).toContain('free-form');
+ expect(instructions).toContain('clear throat');
+ // no wrapping prosody, no spell
+ expect(instructions).not.toContain('type="prosody"');
+ expect(instructions).not.toContain('type="spell"');
+ });
+
+ it('advertises xAI’s kinds only', () => {
+ const instructions = llmInstructions('xai')!;
+ // xAI's own sound cues + wrapping prosody vocabulary
+ expect(instructions).toContain('tongue-click');
+ expect(instructions).toContain('');
+ expect(instructions).toContain('sing-song');
+ // no free-form delivery descriptions, no spell
+ expect(instructions).not.toContain('type="expression"');
+ expect(instructions).not.toContain('type="spell"');
+ });
+
+ it('is undefined for a provider with no markup dialect', () => {
+ expect(llmInstructions('')).toBeUndefined();
+ expect(llmInstructions('openai')).toBeUndefined();
+ expect(llmInstructions('rime')).toBeUndefined();
+ });
+
+ it('omits a sound category that steering disables, and its examples', () => {
+ const instructions = llmInstructions('inworld', { nonverbalSounds: { laughing: false } })!;
+ expect(instructions).not.toContain('label="laugh"');
+ expect(instructions).toContain('clear throat');
+ });
+
+ it('omits the whole sounds section when every sound is disabled', () => {
+ const instructions = llmInstructions('inworld', { nonverbalSounds: false })!;
+ expect(instructions).not.toContain('type="sound"');
+ });
+
+ it('renders identically when steering explicitly enables everything', () => {
+ expect(llmInstructions('xai', { nonverbalSounds: true })).toBe(llmInstructions('xai'));
+ expect(llmInstructions('xai', {})).toBe(llmInstructions('xai'));
+ });
+});
+
+// assistant text mixing expr markers with content that must survive an expr-only strip:
+// provider-native tags, bracket spans, markdown links, and stray angle brackets
+const MIXED =
+ ' Press [Enter] to see bold, ' +
+ 'read [the docs](https://docs.livekit.io), then 1 < 2. ' +
+ 'keep it secret';
+const MIXED_CLEAN =
+ ' Press [Enter] to see bold, ' +
+ 'read [the docs](https://docs.livekit.io), then 1 < 2. ' +
+ 'keep it secret';
+
+describe('stripExprMarkup', () => {
+ it('only touches expr', () => {
+ expect(stripExprMarkup(MIXED)).toBe(MIXED_CLEAN);
+ });
+
+ it('is a no-op without expr', () => {
+ const text = 'plain text with [brackets] and ';
+ expect(stripExprMarkup(text)).toBe(text);
+ });
+
+ it('strips expr only from assistant textContent', () => {
+ const msg = ChatMessage.create({ role: 'assistant', content: [MIXED] });
+ expect(msg.textContent).toBe(MIXED_CLEAN);
+ expect(msg.rawTextContent).toBe(MIXED);
+ });
+
+ it.each(['user', 'system', 'developer'] as const)('leaves %s messages raw', (role) => {
+ // only assistant messages carry expressive markup; other roles are never stripped
+ const msg = ChatMessage.create({ role, content: [JOKE] });
+ expect(msg.textContent).toBe(JOKE);
+ expect(msg.rawTextContent).toBe(JOKE);
+ });
+
+ it('returns undefined without text content', () => {
+ const msg = ChatMessage.create({ role: 'assistant', content: [] });
+ expect(msg.textContent).toBeUndefined();
+ expect(msg.rawTextContent).toBeUndefined();
+ });
+
+ it('toJSON stripMarkup is expr-only and assistant-only', () => {
+ const chatCtx = ChatContext.empty();
+ chatCtx.addMessage({ role: 'user', content: [MIXED] });
+ chatCtx.addMessage({ role: 'assistant', content: [MIXED] });
+
+ let items = chatCtx.toJSON({ stripMarkup: true }).items as Array<{ content: string[] }>;
+ expect(items[0]!.content).toEqual([MIXED]); // user content untouched
+ expect(items[1]!.content).toEqual([MIXED_CLEAN]); // assistant loses only expr tags
+
+ // default keeps the raw content for persistence
+ items = chatCtx.toJSON().items as Array<{ content: string[] }>;
+ expect(items[1]!.content).toEqual([MIXED]);
+ });
+});
+
+describe('universal transcript stripping', () => {
+ // The transcript sinks strip downstream without knowing the provider, so they remove
+ // the union of every provider's XML tags — but never square brackets, which reach the
+ // transcript as markdown/prose.
+
+ it('strips every provider’s tags at once and leaves brackets', () => {
+ const [clean, tags] = splitAllMarkup(
+ 'Hi there ' +
+ '[pause] friend',
+ );
+ expect(clean).toBe('Hi there [pause] friend');
+ expect(tags).toContainEqual({ type: 'emotion', value: 'happy' });
+ expect(tags).toContainEqual({ type: 'expression', value: 'warm' });
+ expect(tags).toContainEqual({ type: 'sound', value: 'giggle' });
+ });
+
+ it('takes the attribute, not the wrapped words, as a delivery label', () => {
+ // the model writes `…` often enough that
+ // normalizeMarkup repairs it — but that runs on the audio path only, so the sinks see
+ // the raw shape. Recording the inner text here published the agent's own sentence as
+ // lk.expression, and matchMood then fell back to `calm`.
+ const [clean, tags] = splitAllMarkup('Hello there');
+ expect(clean).toBe('Hello there');
+ expect(tags).toEqual([{ type: 'expression', value: 'warm' }]);
+ expect(expressionAttribute(tags)).toEqual({
+ 'lk.expression': '{"expression":"warm","mood":"happy"}',
+ });
+ });
+
+ it('still reads a content tag from its inner text', () => {
+ // the inverse must keep working: spell/emphasis and xAI's wrapping emotion tags carry
+ // no attribute, so their content is the value
+ expect(splitAllMarkup('A7X9')[1]).toEqual([{ type: 'spell', value: 'A7X9' }]);
+ expect(splitAllMarkup('wow')[1]).toEqual([
+ { type: 'emphasis', value: 'wow' },
+ ]);
+ expect(splitAllMarkup('Great to hear!')[1]).toEqual([
+ { type: 'happy', value: 'Great to hear!' },
+ ]);
+ });
+
+ it('produces the documented lk.expression payload shape', () => {
+ let [, tags] = splitAllMarkup('oh no');
+ expect(expressionAttribute(tags)).toEqual({
+ 'lk.expression': '{"expression":"sad","mood":"sad"}',
+ });
+
+ // no expression/emotion tag -> no attribute
+ [, tags] = splitAllMarkup('hi');
+ expect(expressionAttribute(tags)).toBeUndefined();
+ });
+
+ it('holds partial tags while streaming', () => {
+ const s = new TranscriptMarkupStripper();
+ let out = s.push('Hi the');
+ out += s.push('re');
+ out += s.flush();
+ expect(out).not.toContain(' {
+ const s = new TranscriptMarkupStripper();
+ // an unclosed "[" must not stall the chunk (brackets aren't markup), and the link must
+ // arrive intact rather than collapsed to its (url) tail
+ const first = s.push('Read [the docs](https:');
+ expect(first).toBe('Read [the docs](https:');
+ const rest = s.push('//docs.livekit.io) now.') + s.flush();
+ expect(first + rest).toBe('Read [the docs](https://docs.livekit.io) now.');
+ });
+
+ it('does not stall on a bare "<"', () => {
+ const s = new TranscriptMarkupStripper();
+ const first = s.push('The value 3 < 5 ');
+ expect(first).toContain('3 < 5');
+ const rest = s.push('is true.') + s.flush();
+ expect((first + rest).replace(/ /g, '')).toBe('Thevalue3<5istrue.');
+ });
+
+ it('strips nested emotion + prosody cleanly', () => {
+ // combining emotion + prosody means nesting; the transcript must come out clean (no
+ // leaked inner markup) — this is what the fixed-point strip guarantees
+ const raw =
+ 'no way ' +
+ ' okay';
+ const [clean] = splitAllMarkup(raw);
+ expect(clean).not.toContain('<');
+ expect(clean).not.toContain('>');
+ expect(clean).toContain('no way');
+ expect(clean).toContain('okay');
+ });
+});
+
+describe('speech steering', () => {
+ it('renders nothing for the explicit all-on forms', () => {
+ // equivalent configurations must produce identical instructions: the explicit all-on
+ // forms add no sound guidance the default doesn't have
+ for (const provider of ['fishaudio', 'inworld', 'xai']) {
+ for (const steering of [{ nonverbalSounds: true } as const, { nonverbalSounds: {} }]) {
+ expect(steeringInstructions(provider, steering), provider).toBe('');
+ expect(llmInstructions(provider, steering), provider).toBe(llmInstructions(provider));
+ }
+ }
+ // all-off leaves nothing to guide; the vocabulary removal happens in llmInstructions
+ expect(steeringInstructions('fishaudio', { nonverbalSounds: false })).toBe('');
+ });
+
+ it('guides only about the sounds that survive an opt-out', () => {
+ const partial = steeringInstructions('fishaudio', { nonverbalSounds: { laughing: false } });
+ expect(partial).toContain('clear-throat');
+ expect(partial.toLowerCase()).not.toContain('laugh');
+ });
+
+ it('renders pace and disfluency guidelines', () => {
+ expect(steeringInstructions('inworld', { disfluencies: false })).toContain('No fillers');
+ expect(steeringInstructions('inworld', { pace: 'slow' })).toContain('slow overall speaking');
+ // "normal" is the default, so it adds nothing
+ expect(steeringInstructions('inworld', { pace: 'normal' })).toBe('');
+ });
+
+ it('never mentions an opted-out concept, not even prohibitively', () => {
+ const composed = llmInstructions('fishaudio', {
+ nonverbalSounds: false,
+ disfluencies: false,
+ })!;
+ expect(composed.toLowerCase()).not.toContain('laugh');
+ expect(composed.toLowerCase()).not.toContain('filler');
+ expect(composed).not.toContain('Um, uh');
+ });
+
+ it('few-shots fillers only while disfluencies are enabled', () => {
+ expect(llmInstructions('fishaudio', { disfluencies: true })).toContain('Um, uh');
+ expect(llmInstructions('fishaudio')).toContain('Um, uh'); // default is on
+ const off = llmInstructions('fishaudio', { disfluencies: false })!;
+ expect(off).not.toContain('Um, uh');
+ expect(off).not.toContain(', um,');
+ });
+
+ it('exposes the queryable non-verbal capability matrix', () => {
+ expect(supportedNonverbals('fishaudio')).toEqual({
+ laughing: ['laughing', 'chuckling'],
+ breathing: ['gasping'],
+ sighing: ['sighing'],
+ crying: ['sobbing'],
+ vocalizing: ['groaning'],
+ reflexSounds: ['clear throat', 'yawning'],
+ });
+ expect(supportedNonverbals('cartesia')).toEqual({});
+ });
+
+ it('treats a category object as a sparse opt-out', () => {
+ // omitted categories stay enabled, so { laughing: false } removes laughter and
+ // nothing else
+ const steering = { nonverbalSounds: { laughing: false } };
+ const inworld = llmInstructions('inworld', steering)!;
+ expect(inworld).not.toContain('label="laugh"');
+ for (const kept of ['sigh', 'breathe', 'clear throat', 'cough', 'yawn']) {
+ expect(inworld, kept).toContain(kept);
+ }
+ // xai's laugh-family prosody is governed by the same field
+ const xai = llmInstructions('xai', steering)!;
+ expect(xai).not.toContain('laugh-speak');
+ expect(xai).toContain('whisper');
+ });
+
+ it('carries the register rule in every provider’s block', () => {
+ // register inference is provider-neutral: every markup-capable provider's block
+ // carries the rule via the shared preamble
+ for (const provider of ['fishaudio', 'inworld', 'xai', 'cartesia']) {
+ expect(llmInstructions(provider), provider).toContain('REGISTER of the moment');
+ }
+ });
+});
+
+describe('expressive chunking', () => {
+ // a typical expressive reply: three short sentences with markers, ~270 chars — well
+ // under every provider's request cap
+ const REPLY =
+ ' Hey, good to hear from you! ' +
+ ' How did the interview go? ' +
+ ' I have been thinking about it all week.';
+
+ /** Feed the reply in LLM-sized chunks; report what the TTS would have received and when. */
+ async function synthesisRequests(provider: string, expressive: boolean) {
+ const stream = sentenceTokenizer(provider, { expressive }).stream();
+ const tokens: string[] = [];
+ const reader = (async () => {
+ for await (const ev of stream) tokens.push(ev.token);
+ })();
+
+ for (const chunk of REPLY.match(/.{1,12}/gs) ?? []) stream.pushText(chunk);
+ await new Promise((r) => setImmediate(r));
+ // everything emitted at this point went out while the LLM was still generating
+ const duringStream = [...tokens];
+
+ stream.endInput();
+ await reader;
+ return { duringStream, total: tokens.length, first: tokens[0] ?? '' };
+ }
+
+ it.each(['inworld', 'xai', 'cartesia'])(
+ 'leaves time-to-first-audio unchanged for %s',
+ async (provider) => {
+ const plain = await synthesisRequests(provider, false);
+ const expressive = await synthesisRequests(provider, true);
+
+ // regression: the batch target used to be the provider's request cap (400–1000
+ // chars). A typical reply never reaches it, so nothing was sent until generation
+ // finished and the whole turn was synthesized in one request — first audio waited
+ // for the full completion.
+ expect(expressive.duringStream.length).toBeGreaterThan(0);
+ expect(expressive.first).toBe(plain.first);
+
+ // ...while still batching the body of the turn into fewer requests than per-sentence
+ expect(expressive.total).toBeLessThan(plain.total);
+ },
+ );
+
+ it('keeps fishaudio per-sentence', async () => {
+ // its markers are sentence-scoped, so batching would cost first-audio and buy no
+ // steering — it is deliberately absent from the chunk-limit table
+ const plain = await synthesisRequests('fishaudio', false);
+ const expressive = await synthesisRequests('fishaudio', true);
+ expect(expressive.total).toBe(plain.total);
+ });
+
+ it('never exceeds the provider request cap', async () => {
+ const long = Array.from({ length: 40 }, (_, i) => `This is sentence number ${i}.`).join(' ');
+ const stream = sentenceTokenizer('cartesia', { expressive: true }).stream();
+ const tokens: string[] = [];
+ const reader = (async () => {
+ for await (const ev of stream) tokens.push(ev.token);
+ })();
+ stream.pushText(long);
+ stream.endInput();
+ await reader;
+
+ expect(tokens.length).toBeGreaterThan(1);
+ for (const t of tokens) expect(t.length).toBeLessThanOrEqual(maxInputLen('cartesia')!);
+ });
+});
+
+describe('mood matching', () => {
+ it('normalizes free-form delivery labels', () => {
+ expect(matchMood('soft, with genuine care')).toBe('empathetic');
+ expect(matchMood('gently curious, welcoming')).toBe('curious');
+ // word starts only: "like a pirate" must not match `angry` via the "irate" stem
+ expect(matchMood('like a pirate')).toBe('calm');
+ expect(matchMood('like a pirate', null)).toBeNull();
+ });
+
+ it('resolves every advertised Fish emotion to a real mood', () => {
+ // lk.expression consumers get a meaningful enum for the whole vocabulary
+ const instructions = llmInstructions('fishaudio')!;
+ for (const emotion of ['regretful', 'hopeful', 'delighted', 'determined', 'frustrated']) {
+ expect(instructions).toContain(emotion);
+ expect(matchMood(emotion, null), emotion).not.toBeNull();
+ }
+ });
+});
+
+describe('dropBracketCues', () => {
+ const timed = (text: string) => createTimedString({ text, startTime: 0, endTime: 1 });
+
+ it('drops a native cue and one of the spaces it sat between', () => {
+ const held: TimedString[] = [];
+ const out = dropBracketCues(['Right.', ' ', '[laughing]', ' ', 'Anyway.'].map(timed), held, {
+ final: true,
+ });
+ expect(out.map((t) => t.text).join('')).toBe('Right. Anyway.');
+ });
+
+ it('holds an unclosed span across messages until it closes', () => {
+ const held: TimedString[] = [];
+ let out = dropBracketCues(['Hello ', '[lau'].map(timed), held);
+ expect(out.map((t) => t.text).join('')).toBe('Hello ');
+ expect(held.length).toBeGreaterThan(0);
+
+ // the cue and the space after it are gone; "Hello " already went out above, so the
+ // seam still reads as a single separator
+ out = dropBracketCues(['ghing] there'].map(timed), held);
+ expect(out.map((t) => t.text).join('')).toBe('there');
+ });
+
+ it('releases an unresolved span at end of stream', () => {
+ const held: TimedString[] = [];
+ dropBracketCues([timed('Hello [lau')], held);
+ const out = dropBracketCues([], held, { final: true });
+ expect(out.map((t) => t.text).join('')).toBe('[lau');
+ });
+});
diff --git a/agents/src/tts/provider_format.ts b/agents/src/tts/provider_format.ts
new file mode 100644
index 000000000..a0b98a548
--- /dev/null
+++ b/agents/src/tts/provider_format.ts
@@ -0,0 +1,1490 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Shared provider-specific TTS formatting logic.
+ *
+ * Both TTS plugins and the inference gateway delegate to this module so there is a single
+ * source of truth for LLM instructions and markup stripping per provider.
+ *
+ * Provider docs:
+ * - Cartesia: https://docs.cartesia.ai/build-with-cartesia/sonic-3/ssml-tags
+ * - Cartesia: https://docs.cartesia.ai/build-with-cartesia/sonic-3/volume-speed-emotion
+ * - Inworld: https://docs.inworld.ai/tts/capabilities/steering
+ * - Inworld: https://docs.inworld.ai/tts/best-practices/prompting-for-tts-2
+ * - xAI: https://docs.x.ai/developers/model-capabilities/audio/text-to-speech
+ * - xAI: https://docs.x.ai/developers/model-capabilities/audio/voice
+ * - Fish Audio: https://docs.fish.audio/developer-guide/core-features/emotions
+ */
+import { ATTRIBUTE_TRANSCRIPTION_EXPRESSION } from '../constants.js';
+import { basic as tokenizeBasic } from '../tokenize/index.js';
+import type { SentenceTokenizer } from '../tokenize/tokenizer.js';
+import { type TimedString, createTimedString } from '../voice/io.js';
+import {
+ LEADING_WS,
+ convertExpressionTags,
+ dedupRemovalSpace,
+ escapeRegExp,
+ extractAndStrip,
+ replaceWithGroups,
+} from './markup_utils.js';
+import { matchMood } from './mood.js';
+
+/**
+ * An expressive markup tag stripped from a transcript, surfaced for the frontend.
+ *
+ * `type` is the markup tag name (`"emotion"`, `"expression"`, `"sound"`, ...) or the expr
+ * marker type. `value` is the spoken or semantic payload (the `value="..."` attribute, the
+ * expr `label`, or the tag's inner text).
+ */
+export interface ExpressiveTag {
+ type: string;
+ value: string;
+}
+
+/**
+ * Non-verbal vocalizations the TTS may produce (sounds that aren't words).
+ *
+ * A sparse opt-out: omitted keys default to ON, and a category set to `false` is never
+ * advertised to the LLM — `{ laughing: false }` removes laughter and nothing else.
+ * Together the keys cover every sound the providers offer.
+ */
+export interface NonverbalOptions {
+ /** laugh, chuckle, giggle — and laugh-speak delivery */
+ laughing?: boolean;
+ /** audible breath, inhale, exhale */
+ breathing?: boolean;
+ sighing?: boolean;
+ crying?: boolean;
+ /** non-lexical voiced sounds — humming a tune, sing-song or sung delivery */
+ vocalizing?: boolean;
+ /** tsk, tongue-click, lip-smack */
+ mouthSounds?: boolean;
+ /** cough, clearing the throat, yawn */
+ reflexSounds?: boolean;
+}
+
+/**
+ * Steers verbal delivery and non-verbal sounds in generated speech.
+ *
+ * Every key is a sparse override on the default (full sound vocabulary, light fillers):
+ * the expressive instructions already tell the LLM to match its delivery to the register
+ * of the moment, so most agents need no steering at all — set a key only to take an option
+ * away regardless of context.
+ */
+export interface SpeechSteeringOptions {
+ /**
+ * Filler words such as "um" / "uh". On by default
+ * ({@link DEFAULT_SPEECH_STEERING_OPTIONS}); set `false` to opt out.
+ */
+ disfluencies?: boolean;
+ /**
+ * Which non-verbal sounds the TTS may make. `true` (and omitting the key) keeps the
+ * provider's full vocabulary, `false` disables every sound, and a
+ * {@link NonverbalOptions} object toggles per category (omitted categories stay enabled).
+ */
+ nonverbalSounds?: boolean | NonverbalOptions;
+ pace?: 'slow' | 'normal' | 'fast';
+}
+
+/** The default steering: full sound vocabulary, light fillers. */
+export const DEFAULT_SPEECH_STEERING_OPTIONS: SpeechSteeringOptions = { disfluencies: true };
+
+/** What the expressive markup pipeline can do with a given voice. */
+export interface MarkupInfo {
+ /** {@link NonverbalOptions} field -> the labels it governs; an absent field is a no-op */
+ nonverbals: Partial>;
+}
+
+type NonverbalField = keyof NonverbalOptions;
+
+const CARTESIA_TAGS = ['emotion', 'speed', 'volume', 'break', 'spell'];
+
+const INWORLD_TAGS = ['expression', 'sound', 'break'];
+
+// xAI Grok TTS speech tags, from the xAI docs
+// (https://docs.x.ai/developers/rest-api-reference/inference/voice).
+//
+// The LLM is instructed in the expr dialect (below); these native tag names serve two
+// purposes: XAI_WRAPPING is the label vocabulary expr prosody markers lower to, and all
+// of them stay in XAI_TAGS so a hallucinated native tag is still stripped from the
+// transcript rather than leaking. The intermediate and
+// tags that expr lowering produces are rewritten to xAI's native
+// brackets by convertMarkup — -> [X] and -> [pause] or
+// [long-pause] by duration. Prosody is angle-bracketed (native).
+const XAI_INLINE = [
+ 'breath',
+ 'inhale',
+ 'exhale',
+ 'sigh',
+ 'laugh',
+ 'chuckle',
+ 'giggle',
+ 'cry',
+ 'tsk',
+ 'tongue-click',
+ 'lip-smack',
+ 'hum-tune',
+];
+const XAI_WRAPPING = [
+ 'emphasis', // stress the wrapped words
+ 'whisper', // quiet, intimate
+ 'soft', // lower volume
+ 'loud', // higher volume
+ 'build-intensity', // ramp energy up over the span
+ 'decrease-intensity', // ease energy off over the span
+ 'higher-pitch',
+ 'lower-pitch',
+ 'slow',
+ 'fast',
+ 'sing-song', // playful, musical lilt
+ 'singing', // actually sung
+ 'laugh-speak', // talk through a laugh
+];
+const XAI_EMOTIONS = [
+ 'happy',
+ 'sad',
+ 'angry',
+ 'excited',
+ 'calm',
+ 'surprised',
+ 'sympathetic',
+ 'curious',
+ 'sarcastic',
+ 'confident',
+ 'playful',
+ 'nervous',
+];
+// all tags are XML in the transcript, so all are stripped. inline sounds are the single
+// "sound" tag (, XAI_INLINE lists the NAMEs), and pauses use
+// "break" (), both modeled on Inworld.
+const XAI_TAGS = [...XAI_EMOTIONS, ...XAI_WRAPPING, 'sound', 'break'];
+
+// xAI has two pause levels ([pause], [long-pause]); map an Inworld-style
+// to the longer one past ~1s. This is the only per-provider bit convertMarkup needs.
+const XAI_BREAK_RE = //g;
+
+function parseDurationSeconds(raw: string): number {
+ const value = raw.trim().toLowerCase();
+ const parsed = value.endsWith('ms')
+ ? Number.parseFloat(value.slice(0, -2)) / 1000
+ : Number.parseFloat(value.replace(/s+$/, ''));
+ return Number.isNaN(parsed) ? 0 : parsed;
+}
+
+function xaiBreakToBracket(_match: string, time: string): string {
+ return parseDurationSeconds(time) >= 1 ? '[long-pause]' : '[pause]';
+}
+
+// Fish Audio (s2 family) speech markers, from the Fish docs
+// (https://docs.fish.audio/developer-guide/core-features/emotions).
+//
+// The LLM is instructed in the expr dialect (below); expr lowering produces the
+// framework-standard intermediates (, ,
+// , word) and convertMarkup rewrites them to
+// Fish's native square brackets: [very EMOTION], [SOUND], [break]/[long-break], and
+// [emphasis] word (a prefix marker stressing the word that follows). Tone wrapping
+// (...) lowers directly to Fish's prefix
+// form, [whispering] followed by the span. The tag names stay in FISHAUDIO_TAGS so
+// hallucinated native markup is still stripped from transcripts.
+//
+// Every label below is from Fish's documented vocabulary, and every emotion maps to a
+// non-fallback mood in mood.ts so lk.expression stays meaningful for clients.
+const FISHAUDIO_EMOTIONS = [
+ 'regretful',
+ 'hopeful',
+ 'happy',
+ 'excited',
+ 'curious',
+ 'surprised',
+ 'sad',
+ 'empathetic',
+ 'sarcastic',
+ 'calm',
+ 'angry',
+ 'worried',
+ 'nervous',
+ 'confident',
+ 'grateful',
+ 'delighted',
+ 'disappointed',
+ 'frustrated',
+ 'determined',
+];
+const FISHAUDIO_SOUNDS = [
+ 'laughing',
+ 'chuckling',
+ 'clear throat',
+ 'sighing',
+ 'gasping',
+ 'groaning',
+ 'yawning',
+ 'sobbing',
+];
+// Fish's tone controls: prefix markers steering the delivery of the words after them.
+// Neutral delivery styles, so they are never steering-filtered (same stance as xAI's
+// whisper/pitch wraps).
+const FISHAUDIO_TONES = ['whispering', 'soft', 'shouting', 'hurried'];
+const FISHAUDIO_TAGS = ['expression', 'sound', 'break', 'emphasis'];
+
+const FISHAUDIO_EXPRESSION_RE = /|>(?:.*?)<\/expression>)/g;
+const FISHAUDIO_BREAK_RE = //g;
+const FISHAUDIO_EMPHASIS_RE = /]*)?>([^<]*)<\/emphasis\s*>/gi;
+
+function fishaudioExpressionToBracket(_match: string, value: string): string {
+ // intensify with a leading "very" so the emotion lands harder in Fish's audio
+ // ([very regretful] steers more strongly than [regretful]); never doubled
+ let label = value.trim();
+ if (label && !label.toLowerCase().startsWith('very ')) {
+ label = `very ${label}`;
+ }
+ return `[${label}]`;
+}
+
+function fishaudioBreakToBracket(_match: string, time: string): string {
+ // Fish has two pause levels ([break], [long-break]); use the longer past ~1s
+ return parseDurationSeconds(time) >= 1 ? '[long-break]' : '[break]';
+}
+
+// --- LiveKit expression markers (expr) ---
+// The LLM emits a single marker tag, , instead of
+// provider-native tags. The *syntax* is shared, but each provider gets its own instruction
+// block advertising only the types and label vocabularies it actually supports — providers
+// offer different sound effects, some take only a discrete emotion vocabulary rather than
+// free-form delivery descriptions, and only some have wrapping prosody. Types (per
+// provider):
+// expression (self-closing) - delivery/emotion for what follows; free-form for
+// Inworld, Cartesia's discrete emotion vocabulary, absent
+// for xAI
+// break (self-closing) - pause, label is a duration ("500ms", "1s"); all providers
+// sound (self-closing) - non-verbal vocalization from the provider's own list
+// (Inworld: laugh/sigh/..., xAI: chuckle/tsk/...); absent
+// for Cartesia
+// prosody (wrapping) - words, labels
+// from xAI's wrapping-tag list; for Cartesia a self-closing
+// point control (slow/fast/soft/loud -> coarse speed/volume
+// ratios); absent for Inworld (folded into expression)
+// spell (wrapping) - A7X9 character-by-character
+// readout; Cartesia only
+// convertMarkup lowers expr to each provider's native syntax before synthesis (via the
+// existing framework-standard tags, so the per-provider conversions below still apply),
+// and the transcript strippers remove expr markers in a dedicated pre-pass so the
+// type/label pair surfaces correctly as an ExpressiveTag. This is the only dialect the
+// LLM is taught — llmInstructions() uses it; the provider-native tag tables remain solely
+// so hallucinated native markup is still stripped/converted instead of leaking.
+
+const EXPR_PREAMBLE = `You control speech delivery with a single XML marker tag: . Every marker has a \
+type attribute. Use only the marker types listed below, and where a type lists a label \
+vocabulary, only those labels. Use the markers often and diversify them so the voice \
+never sounds flat while ensuring the markers are appropriate for the moment. Write the \
+words themselves the way people talk: use contractions ("I'm", "you're", "don't") — \
+spelled-out forms like "I am" or "do not" sound stiff when spoken.
+
+Just as important is knowing when NOT to reach for a marker. Reserve surprise openers \
+like "oh" or "ah" for genuine surprise — an ordinary request isn't one. Don't stack markers \
+on short replies or decorate every sentence. If a reaction wouldn't happen in a real \
+conversation, skip it — there's always another genuine beat to lean into.
+
+Match your delivery to the REGISTER of the moment, and reassess every turn. When the \
+moment is professional, high-stakes, or emotionally heavy — bad news, an emergency, \
+real distress — keep delivery composed and restrained. When the moment is casual, \
+playful, or celebratory, let it loosen and brighten. A serious turn in an otherwise \
+casual conversation still gets a composed reply.`;
+
+const CARTESIA_EXPR_LLM_INSTRUCTIONS =
+ EXPR_PREAMBLE +
+ `
+
+1. Emotion - sets the emotional tone. Self-closing; place before EVERY sentence.
+
+ Labels are a fixed vocabulary, NOT free-form descriptions. Best results: neutral, \
+angry, excited, content, sad, scared.
+ Also available: happy, enthusiastic, elated, triumphant, amazed, surprised, \
+flirtatious, curious, peaceful, serene, calm, grateful, affectionate, sympathetic, \
+mysterious, frustrated, disgusted, sarcastic, ironic, dejected, melancholic, \
+disappointed, apologetic, hesitant, confused, anxious, panicked, proud, confident, \
+contemplative, determined, joking/comedic.
+
+2. Pauses - insert silence when appropriate. Self-closing.
+ - label is a duration in seconds or milliseconds.
+
+3. Prosody - adjusts pacing and loudness from that point on. Self-closing.
+ slower faster
+ quieter louder
+ Labels are a fixed vocabulary: slow, fast, soft, loud.
+
+4. Spell - wraps text read character by character (codes, IDs, or a spelled-out name).
+ A7X9
+ Keep punctuation out of a spell marker — a period inside is read as "dot"; add \
+spaces inside for grouped pauses (ABC 123).
+
+This voice has no non-verbal sounds and no free-form delivery descriptions — do not \
+invent other types or labels.
+
+Examples:
+ I can't wait to tell you! This is going to be great!
+ Really? Tell me more!
+ Your code is A7X9. Got it?`;
+
+const INWORLD_SOUNDS = ['laugh', 'sigh', 'breathe', 'clear throat', 'cough', 'yawn'];
+
+const INWORLD_EXAMPLES = [
+ ' Okay okay, why did the burger go to the gym? Because it wanted better buns! ',
+ ' Ah man, yeah that\'s on us. Lemme see what I can do.',
+ ' I know it\'s been a rough week.',
+ ' Welcome to the hotel. How can I help you today?',
+ ' That\'s all set. Your confirmation code is B 4 J 7.',
+ // persona carried into the tags: casual words, casual labels
+ ' Yeah, of course! Gimme one sec, pulling it up now.',
+];
+
+/** Drop example lines that demonstrate a *vocabulary* label not in `allowed`. */
+function soundExamples(examples: string[], allowed: string[], vocabulary: string[]): string[] {
+ const removed = vocabulary.filter((s) => !allowed.includes(s));
+ return examples.filter((ex) => !removed.some((s) => ex.includes(`label="${s}"`)));
+}
+
+function numberedSections(sections: string[]): string {
+ return sections.map((section, i) => `${i + 1}. ${section}`).join('\n\n');
+}
+
+function inworldExprLlmInstructions(sounds: string[]): string {
+ const sections = [
+ `Delivery - controls how a sentence sounds. Self-closing; place before EVERY sentence.
+
+ The label is free-form: describe vocal quality, pitch, volume, pace, and intonation \
+in plain English — "say really playfully", "slightly surprised, amiable", "sound a little \
+concerned", "drop to almost a whisper", "speak really slowly and clearly, patient and \
+reassuring".
+ Match the expression tag's energy to the sentence's punctuation. An exclamation \
+needs a bright or upbeat label (e.g. "bright, upbeat energy"); a calm or reassuring \
+label flattens the "!". Never lead an exclamatory sentence with a calm tag.
+ Put each question in its own sentence — don't comma-splice it onto a statement. \
+Write "Welcome to the hotel. How can I help you today?", not "Welcome to the hotel, \
+how can I help you today?", so the question carries its own delivery tag instead of \
+inheriting the statement's.
+ Never put "questioning" in a tag — describe the mood alone and let the question \
+mark carry the intonation.
+ Name a mood or speaking style, not a mechanical pitch contour. "gently upbeat, \
+amiable" steers far more reliably than "rising tone".
+ Use at most two adjectives per tag, and make sure they align — with the mood of \
+the sentence and with each other. Clashing descriptors ("calm, excited") cancel out \
+and muddy the delivery.
+ Put a degree modifier in EVERY tag — "a little", "almost", "slightly", "gently", \
+"really" — to set the exact strength of the feeling: "a little amused" or "almost a \
+whisper" lands truer than "amused" or "whisper", and "really excited" turns the \
+delivery up when the moment truly peaks. Most moments call for a shade, not the \
+extreme — default to the softeners and save "really" for true peaks.
+ Carry your persona into the tags — the labels should sound like the character, \
+not generic stage directions. An amiable, casual persona tags with "really relaxed \
+and amiable" or "casual, a little playful"; a formal concierge tags the same \
+sentence "gently courteous, composed". Delivery that contradicts who you are reads \
+as a different speaker.
+ Don't open a turn with a "slow" tag. The first expression colors the whole turn, \
+and a slow lead flattens questions and drags the energy down. Keep the pace neutral \
+by default and reserve slow, clearly-enunciated delivery for the specific line that \
+needs it (a total, date, address, or confirmation code).
+ Rotate expression labels — don't reuse the same one two turns in a row, and vary \
+the descriptor. A starting palette:
+ greeting / amiable open: "really amiable and welcoming" / "gently bright, \
+heartfelt" / "cheerful, really glad you called"
+ asking a question: "gently upbeat and amiable" / "really open and inquisitive" / \
+"gently inquisitive, attentive"
+ good news / exclamation: "really bright, upbeat energy" / "really delighted and \
+glad" / "gently pleased and bright"
+ reassuring / taking a request in stride: "really calm and confident" / \
+"gently easygoing and reassuring" / "really relaxed and grounded"
+ empathy / a problem or bad news: "really soft, with tender care" / \
+"gently concerned, caring" / "almost a murmur, gentle and steady"
+ reading back a total, date, or code: "slow and really clearly enunciated"`,
+ ];
+
+ if (sounds.length) {
+ const fits = ' (a clear-throat when shifting to a new step or topic, for example)';
+ let section = `Sounds - a non-verbal sound between sentences. Self-closing.
+
+ Labels are a fixed vocabulary: ${sounds.join(', ')}.
+ Use non-verbal sounds sparingly, and never the same one twice in a row — reach for \
+one only where it genuinely fits${sounds.includes('clear throat') ? fits : ''}. An enabled \
+sound gets over-used otherwise.`;
+ if (sounds.includes('breathe')) {
+ section += `
+ Use the "breathe" sound only for a real, gentle breath, never as filler — on this \
+model it easily reads as a weary or impatient sigh, which sounds wrong in a support \
+setting.`;
+ }
+ sections.push(section);
+ }
+
+ sections.push(`Pauses - insert silence when appropriate. Self-closing.
+ or (max 10s).
+ A period or an ellipsis (...) already creates a pause, so don't put a break marker \
+right next to one — pick one or the other.
+ After any , give the sentence that follows its own expression \
+tag — a fresh one, not necessarily the same as before (a break is often where the mood \
+shifts). A break resets delivery to neutral, so an untagged sentence after a break is \
+spoken flat.`);
+
+ const parts = [
+ EXPR_PREAMBLE,
+ numberedSections(sections),
+ 'There is no wrapping prosody marker for this voice — put pace, pitch, and volume in ' +
+ 'the expression label instead.',
+ `Write for the EAR, not the page: no em or en dashes anywhere in spoken text — \
+use a comma or a period for a short beat, or a break marker for a real pause. Avoid \
+semicolons, mid-sentence colons, and parenthetical asides; rewrite them as separate \
+sentences or commas.`,
+ `When the conversation is in another language, still write every marker label in \
+English — delivery descriptions and sound names steer the voice and are never \
+translated.`,
+ ];
+
+ if (sounds.includes('laugh')) {
+ parts.push(
+ 'Laughter belongs only in genuinely playful or celebratory beats, never at ' +
+ 'a serious moment.',
+ );
+ }
+
+ const examples = soundExamples(INWORLD_EXAMPLES, sounds, INWORLD_SOUNDS);
+ if (examples.length) {
+ parts.push('Examples:\n' + examples.map((ex) => ` ${ex}`).join('\n'));
+ }
+ return parts.join('\n\n');
+}
+
+const XAI_EXAMPLES = [
+ 'So I walked in and there it was! It was a secret the whole time.',
+ 'This is going to be so good. I can\'t wait!',
+ 'Hey. I know it\'s been a rough week. I\'m right here.',
+ 'You did not just say that okay, tell me everything.',
+ // sound-free, so at least one example survives any steering filter; the break lands
+ // mid-sentence before the key detail, never beside sentence punctuation
+ 'Everything is confirmed for Thursday the ninth. Is there anything else I can help you with?',
+];
+
+function xaiExprLlmInstructions(sounds: string[], prosody: string[]): string {
+ const sections: string[] = [];
+ if (sounds.length) {
+ sections.push(`Sounds - a non-verbal vocalization at the exact point where it happens. Self-closing.
+
+ Labels are a fixed vocabulary: ${sounds.join(', ')}.
+ Use non-verbal sounds sparingly, and never the same one twice in a row — reach for \
+one only where it genuinely fits. An enabled sound gets over-used otherwise.`);
+ }
+
+ sections.push(`Pauses - insert silence when appropriate. Self-closing.
+ a brief pause a longer, dramatic pause
+ NEVER place a break next to a period, question mark, exclamation point, or ellipsis \
+— sentence punctuation already pauses, and a break beside it double-pauses. Most \
+replies need no break markers at all; reserve them for a deliberate mid-sentence beat \
+before a key detail (a date, a name, a number).`);
+
+ const tones = prosody.filter((p) => p !== 'emphasis');
+ sections.push(`Prosody - wraps a span delivered in a distinct style, to shape HOW it's said.
+ the words it affects
+ Labels are a fixed vocabulary: ${tones.join(', ')}.
+ Use one only where the moment clearly calls for it — most sentences need none. \
+Never nest one prosody marker inside another, and always close it with .`);
+
+ sections.push(`Emphasis - stresses exactly the ONE word it wraps.
+ Are you sure you want to do this?
+ Wrap a single word, never a phrase, and never write it in all-caps — caps are read \
+out as individual letters. Never nest it, and always close it with .`);
+
+ const parts = [
+ EXPR_PREAMBLE,
+ numberedSections(sections),
+ 'This voice has no free-form delivery descriptions — shape delivery entirely through ' +
+ (sounds.length ? 'prosody markers, sounds, pauses' : 'prosody markers, pauses') +
+ ', punctuation, and word choice.',
+ `Write for the EAR, not the page: no em or en dashes anywhere in spoken text — \
+use a comma or a period for a short beat, or a break marker for a real pause. Avoid \
+semicolons, mid-sentence colons, and parenthetical asides; rewrite them as separate \
+sentences or commas.`,
+ `When the conversation is in another language, still write every marker label in \
+English — labels are a fixed vocabulary, never translated.`,
+ `Key details deserve care: stress the load-bearing word of a date, amount, or \
+name with the emphasis marker, and wrap a dense or easy-to-mishear span in \
+.... Read codes and reference numbers \
+character by character, spelled out with spaces, so each one lands.`,
+ ];
+
+ // Vocabulary-specific register guidance on top of the preamble's neutral rule,
+ // mentioning only concepts this steering leaves enabled (whisper/soft/loud are
+ // neutral delivery controls, never filtered).
+ const register = [
+ 'Whisper and soft belong to gentle or conspiratorial beats; loud only to ' +
+ 'genuinely high-energy ones.',
+ ];
+ if (['laugh', 'chuckle', 'giggle'].some((s) => sounds.includes(s))) {
+ register.push(
+ 'Laughter is RARE: a laugh, chuckle, or giggle belongs only where something ' +
+ 'is genuinely funny — friendliness, agreement, or mild amusement is not a ' +
+ 'reason, and never laugh at your own lines. Most replies have no laughter ' +
+ 'at all.',
+ );
+ }
+ parts.push(register.join(' '));
+
+ const examples = soundExamples(
+ XAI_EXAMPLES,
+ [...sounds, ...prosody],
+ [...XAI_INLINE, ...XAI_WRAPPING],
+ );
+ if (examples.length) {
+ parts.push('Examples:\n' + examples.map((ex) => ` ${ex}`).join('\n'));
+ }
+ return parts.join('\n\n');
+}
+
+// Examples carried over from the original Fish expressive block (PR #6232), rewritten
+// in the expr dialect. Breaks appear only mid-sentence, never beside a period/?/! —
+// an example pairing a break with sentence punctuation few-shots the LLM into
+// double-pausing every boundary.
+const FISHAUDIO_EXAMPLES = [
+ ' That\'s hilarious! You always lighten the mood.',
+ ' That sounds like a really difficult experience.',
+ ' Oh, my goodness that\'s a real shame.',
+ ' I\'ve been going in circles with this all morning. Okay. One more try.',
+ // sound-free, so at least one example survives any steering filter
+ ' You\'re all set for Thursday the ninth. Is there anything else I can help you with?',
+ // sound-free tone example: the wrap is scoped to the span, not the sentence
+ ' Okay, don\'t tell anyone yet but I think we actually pulled it off!',
+];
+
+// The original block baked light disfluencies into the few-shots — that's what made
+// fillers actually show up in generations. Appended only while steering has
+// disfluencies enabled, so the examples never contradict the "no fillers" guideline.
+const FISHAUDIO_DISFLUENT_EXAMPLES = [
+ ' Um, uh... really? Well, I\'m really sorry to hear that.',
+ ' I really wish I\'d, um, called sooner. But I\'m here now if, if you want to talk.',
+ ' What?! No way! I, I\'m flabbergasted! Fair play, I guess.',
+];
+
+function fishaudioExprLlmInstructions(sounds: string[], disfluencies = true): string {
+ const sections = [
+ `Emotion - sets how a sentence sounds. Self-closing; place at the START of a sentence.
+
+ Labels are a fixed vocabulary, NOT free-form descriptions: ${FISHAUDIO_EMOTIONS.join(', ')}.
+ Give every sentence its own emotion marker — repeat the same label to carry a \
+feeling across sentences, or switch labels when the feeling shifts.`,
+ ];
+
+ if (sounds.length) {
+ sections.push(`Sounds - a non-verbal sound between sentences. Self-closing.
+
+ Labels are a fixed vocabulary: ${sounds.join(', ')}.
+ Use non-verbal sounds sparingly, and never the same one twice in a row — reach for \
+one only where it genuinely fits. An enabled sound gets over-used otherwise.`);
+ }
+
+ sections.push(`Pauses - insert silence when appropriate. Self-closing.
+ or .
+ NEVER place a break next to a period, question mark, exclamation point, or ellipsis \
+— sentence punctuation already pauses, and a break beside it double-pauses. Most \
+replies need no break markers at all; reserve them for a deliberate mid-sentence beat \
+before a key detail (a date, a name, a number).`);
+
+ sections.push(`Tone - wraps a span delivered in a distinct style.
+ don't tell anyone yet.
+ Labels are a fixed vocabulary: ${FISHAUDIO_TONES.join(', ')}.
+ Use a tone only where the moment clearly calls for one — most sentences need \
+none. Never nest tone markers, and always close the tag with .`);
+
+ sections.push(`Emphasis - stresses exactly the ONE word it wraps.
+ Are you sure you want to do this?
+ Wrap a single word, never a phrase. Never nest it, and always close it with .`);
+
+ const parts = [
+ EXPR_PREAMBLE,
+ numberedSections(sections),
+ `Write for the EAR, not the page: no em or en dashes anywhere in spoken text — \
+use a comma or a period for a short beat, or a break marker for a real pause. Avoid \
+semicolons, mid-sentence colons, and parenthetical asides; rewrite them as separate \
+sentences or commas.`,
+ `When the conversation is in another language, still write every marker label in \
+English — labels are a fixed vocabulary, never translated.`,
+ ];
+
+ // Vocabulary-specific register guidance on top of the preamble's neutral rule.
+ // Each clause mentions only concepts this steering actually enables, so an
+ // opted-out option is never referenced (not even prohibitively).
+ const register = [
+ 'At heavy moments reach for empathetic, sad, regretful, or hopeful — never a ' +
+ 'bright label like "happy" or "excited" against hard news; bright labels belong ' +
+ 'to bright moments.',
+ 'Whispering and soft belong to gentle or conspiratorial beats; shouting only to ' +
+ 'genuinely high-energy ones.',
+ ];
+ if (['laughing', 'chuckling'].some((s) => sounds.includes(s))) {
+ register.push(
+ 'Laughter belongs only in genuinely playful or celebratory beats, never at ' +
+ 'a serious moment.',
+ );
+ }
+ if (disfluencies) {
+ register.push(
+ 'Save fillers for relaxed moments — never in an emergency or against grave news.',
+ );
+ }
+ parts.push(register.join(' '));
+
+ const pool = [...FISHAUDIO_EXAMPLES, ...(disfluencies ? FISHAUDIO_DISFLUENT_EXAMPLES : [])];
+ const examples = soundExamples(pool, sounds, FISHAUDIO_SOUNDS);
+ if (examples.length) {
+ parts.push('Examples:\n' + examples.map((ex) => ` ${ex}`).join('\n'));
+ }
+ return parts.join('\n\n');
+}
+
+// Every provider's full expr sound vocabulary (the advertised labels before any
+// speechSteering filtering). Providers absent here have no non-verbal sounds.
+const PROVIDER_SOUNDS: Record = {
+ inworld: INWORLD_SOUNDS,
+ xai: XAI_INLINE,
+ fishaudio: FISHAUDIO_SOUNDS,
+};
+
+type NonverbalTable = Record>>;
+
+/**
+ * Labels from a per-provider governance table that `steering` disables.
+ *
+ * `nonverbalSounds` accepts a boolean or a sparse per-category object: `true` (like
+ * omitting the key) keeps the full vocabulary, `false` disables every sound, and in an
+ * object an omitted category stays ENABLED — `{ laughing: false }` removes laughter and
+ * nothing else.
+ */
+function steeringRemoved(
+ table: NonverbalTable,
+ provider: string,
+ steering: SpeechSteeringOptions | undefined,
+): Set {
+ const nonverbals = steering?.nonverbalSounds;
+ const labels = table[provider];
+ if (nonverbals === undefined || nonverbals === true || labels === undefined) {
+ return new Set();
+ }
+ if (nonverbals === false) {
+ return new Set(Object.values(labels).flat());
+ }
+ const removed = new Set();
+ for (const [field, fieldLabels] of Object.entries(labels) as [NonverbalField, string[]][]) {
+ if (nonverbals[field] === false) {
+ for (const label of fieldLabels) removed.add(label);
+ }
+ }
+ return removed;
+}
+
+/**
+ * The provider's sound vocabulary minus labels steering disables.
+ *
+ * Every label is governed by a {@link NonverbalOptions} field, so passing
+ * `nonverbalSounds: false` returns an empty list — the instruction builders then omit the
+ * Sounds section entirely.
+ */
+function allowedSounds(provider: string, steering: SpeechSteeringOptions | undefined): string[] {
+ const removed = steeringRemoved(NONVERBAL_SOUND_LABELS, provider, steering);
+ return (PROVIDER_SOUNDS[provider] ?? []).filter((s) => !removed.has(s));
+}
+
+/**
+ * The provider's wrapping-prosody vocabulary minus labels steering disables.
+ *
+ * Unlike sounds, only the vocal-style labels (laugh-speak, singing, ...) are governed —
+ * neutral delivery controls (emphasis, whisper, pitch, pace) always survive, so the result
+ * is never empty.
+ */
+function allowedProsody(provider: string, steering: SpeechSteeringOptions | undefined): string[] {
+ const removed = steeringRemoved(NONVERBAL_PROSODY_LABELS, provider, steering);
+ return (PROVIDER_PROSODY[provider] ?? []).filter((p) => !removed.has(p));
+}
+
+// NonverbalOptions field -> the provider's expr sound labels it governs. A provider
+// absent here (cartesia) has no non-verbal sounds; an empty list means the provider
+// has no sound for that field (nothing to filter). allowedSounds uses this to remove
+// disabled labels from the advertised vocabulary, so a sound steering turns off is never
+// exposed to the LLM in the first place. Every label in PROVIDER_SOUNDS must be governed
+// by exactly one field, so a steering config controls the full vocabulary.
+const NONVERBAL_SOUND_LABELS: NonverbalTable = {
+ inworld: {
+ laughing: ['laugh'],
+ breathing: ['breathe'],
+ sighing: ['sigh'],
+ crying: [],
+ vocalizing: [],
+ mouthSounds: [],
+ reflexSounds: ['cough', 'clear throat', 'yawn'],
+ },
+ xai: {
+ laughing: ['laugh', 'chuckle', 'giggle'],
+ breathing: ['breath', 'inhale', 'exhale'],
+ sighing: ['sigh'],
+ crying: ['cry'],
+ vocalizing: ['hum-tune'], // non-lexical voiced sounds
+ mouthSounds: ['tsk', 'tongue-click', 'lip-smack'],
+ reflexSounds: [], // xAI has no cough/yawn sounds
+ },
+ fishaudio: {
+ laughing: ['laughing', 'chuckling'],
+ breathing: ['gasping'],
+ sighing: ['sighing'],
+ crying: ['sobbing'],
+ vocalizing: ['groaning'],
+ mouthSounds: [],
+ reflexSounds: ['clear throat', 'yawning'],
+ },
+};
+
+// NonverbalOptions field -> the provider's wrapping-prosody labels it governs.
+// Sparse on purpose: only vocal-style prosody (talking through a laugh, singing)
+// is steerable; neutral delivery controls are never filtered.
+const NONVERBAL_PROSODY_LABELS: NonverbalTable = {
+ xai: {
+ laughing: ['laugh-speak'],
+ vocalizing: ['sing-song', 'singing'],
+ },
+};
+
+// Every provider's full wrapping-prosody vocabulary (only xAI has one).
+const PROVIDER_PROSODY: Record = {
+ xai: XAI_WRAPPING,
+};
+
+/** {@link NonverbalOptions} field -> the sound/prosody labels it governs for `provider`. */
+export function supportedNonverbals(provider: string): Partial> {
+ const merged: Partial> = {};
+ for (const table of [NONVERBAL_SOUND_LABELS, NONVERBAL_PROSODY_LABELS]) {
+ for (const [field, labels] of Object.entries(table[provider] ?? {}) as [
+ NonverbalField,
+ string[],
+ ][]) {
+ if (labels.length) {
+ merged[field] = [...(merged[field] ?? []), ...labels];
+ }
+ }
+ }
+ return merged;
+}
+
+// Sound label -> when a real speaker would make it. The sounds guideline is composed
+// from the hints of whichever labels survived steering, so the LLM only ever reads
+// usage advice for sounds it's allowed to make. Labels sharing a hint (the laugh
+// family) collapse to one clause; labels without an entry fall back to the generic
+// sentence. Keyed by label, not NonverbalOptions field, so it's provider-agnostic.
+const SOUND_USAGE_HINTS: Record = {
+ laugh: 'a laugh at something obviously funny',
+ laughing: 'a laugh at something obviously funny',
+ chuckle: 'a chuckle at something subtly humorous',
+ chuckling: 'a chuckle at something subtly humorous',
+ giggle: 'a chuckle at something subtly humorous',
+ sigh: 'a sigh when commiserating',
+ sighing: 'a sigh when commiserating',
+ inhale: 'a sharp inhale before a big reveal',
+ gasping: 'a gasp at a sudden shock or reveal',
+ 'lip-smack': 'a lip-smack or tongue-click as a tiny beat of thought',
+ 'tongue-click': 'a lip-smack or tongue-click as a tiny beat of thought',
+ tsk: 'a tsk for mock-disapproval',
+ 'clear throat': 'a clear-throat when shifting to a new step or topic',
+ groaning: 'a groan at a groan-worthy pun or an unwelcome chore',
+ yawning: 'a yawn when tiredness itself is the topic',
+ sobbing: 'a sob reserved for real heartbreak',
+};
+
+/** The sparing-use guideline, illustrated only with the allowed sounds. */
+function soundGuidance(sounds: string[]): string {
+ const hints: string[] = [];
+ for (const sound of sounds) {
+ const hint = SOUND_USAGE_HINTS[sound];
+ if (hint && !hints.includes(hint)) hints.push(hint);
+ }
+ let line = 'Non-verbal sounds: use one only where the moment genuinely earns it';
+ if (hints.length) line += ' — ' + hints.join(', ');
+ return line + '. Most turns have none; never repeat the same sound twice in a row.';
+}
+
+/**
+ * Render a {@link SpeechSteeringOptions} into delivery guidelines for `provider`.
+ *
+ * Only fields that change the default produce output, so an empty object adds nothing on
+ * top of the base template. Disabled sounds never appear here: {@link llmInstructions}
+ * filters them out of the advertised vocabulary, so the only sound guidance left is how
+ * sparingly to use what remains.
+ */
+export function steeringInstructions(provider: string, steering: SpeechSteeringOptions): string {
+ const lines: string[] = [];
+
+ // sound guidance only when steering actually removes part of the vocabulary:
+ // the explicit all-on forms (true, an empty object) must render identically to
+ // omitting the key, and all-off leaves nothing to guide
+ if (steeringRemoved(NONVERBAL_SOUND_LABELS, provider, steering).size) {
+ const allowed = allowedSounds(provider, steering);
+ if (allowed.length) lines.push(soundGuidance(allowed));
+ }
+
+ if (steering.disfluencies !== undefined) {
+ lines.push(
+ steering.disfluencies
+ ? 'Sprinkle in natural fillers (um, uh) and openers (oh, well, so), ' +
+ 'zero to two per turn, never mechanical.'
+ : 'No fillers (um, uh). Sound composed and fluent.',
+ );
+ }
+
+ if (steering.pace !== undefined && steering.pace !== 'normal') {
+ lines.push(`Keep a ${steering.pace} overall speaking pace.`);
+ }
+
+ if (!lines.length) return '';
+ return 'Delivery guidelines:\n' + lines.map((line) => `- ${line}`).join('\n');
+}
+
+// Hard per-provider chunking defaults (characters). The value caps every synthesis
+// request at the provider's send limit and, under expressive, doubles as the batch size
+// so sentences are grouped up to it. Providers absent here are uncapped and always emit
+// per sentence.
+const MAX_INPUT_LEN: Record = {
+ inworld: 900,
+ cartesia: 400,
+ // well under xAI's 15,000-char request limit; sized as an expressive batch
+ // target (https://docs.x.ai/developers/model-capabilities/audio/text-to-speech)
+ xai: 1000,
+ // fishaudio is deliberately absent: its markers are sentence-scoped (every sentence
+ // carries its own [very EMOTION]), so per-sentence emission loses no steering and keeps
+ // time-to-first-audio low
+};
+
+/** The max text chunk length for a provider, or `undefined` if unlimited. */
+export function maxInputLen(provider: string): number | undefined {
+ return MAX_INPUT_LEN[provider];
+}
+
+/**
+ * How much text an expressive turn batches before emitting, in characters — roughly two
+ * sentences.
+ *
+ * Deliberately far below the providers' request caps. The cap is a transport limit
+ * (400–1000 chars); using it as the batch target means a typical reply never reaches it,
+ * so nothing is emitted while the LLM streams and the whole turn is synthesized in one
+ * request once generation ends — time-to-first-audio becomes "wait for the full
+ * completion". Batching a couple of sentences is all continuous prosody needs.
+ */
+const EXPRESSIVE_BATCH_LEN = 200;
+
+/**
+ * Minimum length of the first chunk of an expressive turn — the tokenizer's per-sentence
+ * default, so the opening sentence is sent the moment it is complete.
+ *
+ * Batching starts from the second chunk, which keeps prosody continuous over the body of
+ * the turn while leaving time-to-first-audio identical to a non-expressive turn.
+ */
+const EXPRESSIVE_FIRST_CHUNK_LEN = 20;
+
+/**
+ * Default sentence tokenizer for a provider's streamed TTS input.
+ *
+ * The provider's hard max chunk length caps every emitted token. When `expressive` is set,
+ * it also raises the *minimum* to {@link EXPRESSIVE_BATCH_LEN} so a couple of consecutive
+ * sentences ride one request, keeping prosody continuous across the turn; otherwise tokens
+ * emit per sentence (the unchanged default). Providers with no configured limit are
+ * uncapped and stay per-sentence even under expressive — Fish Audio's markers are
+ * sentence-scoped, so batching would cost time-to-first-audio and buy no steering.
+ */
+export function sentenceTokenizer(
+ provider: string,
+ options: { expressive: boolean },
+): SentenceTokenizer {
+ const maxLen = MAX_INPUT_LEN[provider];
+ const batching = options.expressive && maxLen !== undefined;
+ return new tokenizeBasic.SentenceTokenizer({
+ maxTokenLength: maxLen,
+ // the batch target is independent of the cap; clamped so it can never exceed it
+ minTokenLength: batching ? Math.min(EXPRESSIVE_BATCH_LEN, maxLen!) : undefined,
+ firstTokenLength: batching ? EXPRESSIVE_FIRST_CHUNK_LEN : undefined,
+ // markup only exists in the stream when expressive is active; xml-aware
+ // tokenization would otherwise hold streaming on a stray "<" in plain text
+ xmlAware: options.expressive,
+ });
+}
+
+const EXPR_ATTR_RE = /([\w-]+)\s*=\s*"([^"]*)"/g;
+// every marker pattern captures the space before it as "pre" so dedupRemovalSpace can
+// drop it when the marker vanishes from between two spaces
+// any or tag (open or self-closing)
+const EXPR_OPEN_RE = new RegExp(LEADING_WS + '[^>]*?)/?\\s*>', 'g');
+const EXPR_CLOSE_RE = new RegExp(LEADING_WS + '', 'g');
+// self-closing markers only (the trailing / is required)
+const EXPR_SELF_RE = new RegExp(LEADING_WS + '[^>]*?)/\\s*>', 'g');
+// a wrapping marker (prosody/spell) and its span; non-greedy, instructed not to nest.
+// The `(?` reads as an *opening*
+// tag whose span runs to the next `` — swallowing every marker in between. A
+// `` caught that way is discarded, and the confirmation code it
+// wrapped is spoken as a word instead of spelled out.
+const EXPR_WRAP_RE = new RegExp(
+ LEADING_WS +
+ ']*type="(?:prosody|spell)")(?[^>]*?)(?(?.*?)',
+ 'gs',
+);
+// a non-wrapping type the LLM forgot to self-close (normalizeMarkup fixes these)
+const EXPR_UNCLOSED_RE = /(]*type="(?:expression|break|sound)")[^>]*[^/>\s])\s*>/g;
+
+// expr sound labels that differ from xAI's native cue names
+const XAI_SOUND_ALIASES: Record = { breathe: 'breath' };
+
+// expr sound labels that differ from Fish's native marker names (other providers
+// advertise "laugh"/"chuckle", so a hallucinated one still lowers to a sound Fish renders)
+const FISHAUDIO_SOUND_ALIASES: Record = {
+ laugh: 'laughing',
+ chuckle: 'chuckling',
+ sigh: 'sighing',
+ gasp: 'gasping',
+ groan: 'groaning',
+ yawn: 'yawning',
+ sob: 'sobbing',
+ cry: 'sobbing',
+};
+
+// Cartesia prosody labels -> native point controls (coarse steps of the numeric ratios)
+const CARTESIA_PROSODY: Record = {
+ slow: '',
+ fast: '',
+ soft: '',
+ loud: '',
+};
+
+function exprAttrs(attrs: string): Record {
+ const out: Record = {};
+ for (const match of attrs.matchAll(EXPR_ATTR_RE)) {
+ out[match[1]!] = match[2]!;
+ }
+ return out;
+}
+
+/**
+ * Strip expr markers and collect (type, label) pairs, in document order.
+ *
+ * The generic {@link extractAndStrip} pass can't produce the right ExpressiveTag for expr
+ * (its type would be the literal tag name `expr` and its value the first quoted attribute,
+ * i.e. the marker type), so expr gets this dedicated pre-pass. A prosody wrapper's inner
+ * words stay in the clean text — only the delimiters are removed — which also keeps
+ * streaming safe when an open/close pair is split across chunks.
+ */
+function splitExpr(text: string): [string, ExpressiveTag[]] {
+ if (!text.includes(' {
+ const attrs = exprAttrs(groups.attrs ?? '');
+ tags.push({ type: attrs.type ?? '', value: attrs.label ?? '' });
+ return dedupRemovalSpace(groups.pre ?? '', '', source, offset + match.length);
+ });
+ clean = replaceWithGroups(clean, EXPR_CLOSE_RE, ({ groups, match, offset, source }) =>
+ dedupRemovalSpace(groups.pre ?? '', '', source, offset + match.length),
+ );
+ return [clean, tags];
+}
+
+/**
+ * Lower expr markers to the framework-standard / native tags for `provider`.
+ *
+ * The output still flows through the existing per-provider conversions in
+ * {@link convertMarkup} (e.g. `` -> `[X]` for Inworld/xAI), so this only
+ * has to translate expr into those intermediate tags. A type the provider doesn't support
+ * (its instructions never advertise it, so it's a hallucination) is dropped from the audio
+ * path — the words survive, the marker never leaks.
+ */
+function convertExpr(provider: string, text: string): string {
+ if (!text.includes(' {
+ const attrs = exprAttrs(attrsRaw);
+ const markerType = attrs.type ?? '';
+ const label = (attrs.label ?? '').trim().toLowerCase();
+ if (markerType === 'spell') {
+ return provider === 'cartesia' ? `${inner}` : inner;
+ }
+ // prosody: native wrapping tags exist only for xAI
+ if (provider === 'xai') {
+ const native = label.replace(/ /g, '-');
+ if (XAI_WRAPPING.includes(native)) {
+ return `<${native}>${inner}`;
+ }
+ return inner;
+ }
+ if (provider === 'inworld') {
+ // not advertised for Inworld; salvage a stray one as a delivery hint
+ return `${inner}`;
+ }
+ if (provider === 'cartesia') {
+ // wrapping form of the point controls: apply before the span
+ return (CARTESIA_PROSODY[label] ?? '') + inner;
+ }
+ if (provider === 'fishaudio') {
+ if (label === 'emphasis') {
+ return `${inner}`;
+ }
+ // tone controls are prefix markers: [whispering] steers the words after it
+ if (FISHAUDIO_TONES.includes(label)) {
+ return `[${label}] ${inner}`;
+ }
+ return inner;
+ }
+ return inner;
+ };
+
+ // a marker the provider doesn't support lowers to "" — dedupRemovalSpace keeps its
+ // removal from leaving two spaces behind (this text is the transcript when
+ // useTtsAlignedTranscript is on)
+ let out = replaceWithGroups(text, EXPR_WRAP_RE, ({ groups, match, offset, source }) =>
+ dedupRemovalSpace(
+ groups.pre ?? '',
+ wrap(groups.attrs ?? '', groups.inner ?? ''),
+ source,
+ offset + match.length,
+ ),
+ );
+
+ const self = (attrsRaw: string): string => {
+ const attrs = exprAttrs(attrsRaw);
+ const markerType = attrs.type ?? '';
+ let label = attrs.label ?? '';
+ if (markerType === 'expression') {
+ if (provider === 'cartesia') {
+ // Cartesia's discrete emotion vocabulary (instructions list it)
+ return ``;
+ }
+ if (provider === 'inworld' || provider === 'fishaudio') {
+ return ``;
+ }
+ return ''; // xAI has no free-form delivery descriptions
+ }
+ if (markerType === 'sound') {
+ if (provider === 'cartesia') {
+ return ''; // no non-verbal sound support
+ }
+ if (provider === 'xai') {
+ label = XAI_SOUND_ALIASES[label.toLowerCase()] ?? label;
+ }
+ if (provider === 'fishaudio') {
+ label = FISHAUDIO_SOUND_ALIASES[label.toLowerCase()] ?? label;
+ }
+ return ``;
+ }
+ if (markerType === 'break') {
+ return ``;
+ }
+ if (markerType === 'prosody' && provider === 'cartesia') {
+ // Cartesia prosody is a self-closing point control (speed/volume)
+ return CARTESIA_PROSODY[label.trim().toLowerCase()] ?? '';
+ }
+ if (markerType === 'prosody' && provider === 'fishaudio') {
+ // tones are taught as wrapping, but Fish's native form is a prefix marker anyway
+ // — salvage a self-closing one as-is
+ const tone = label.trim().toLowerCase();
+ return FISHAUDIO_TONES.includes(tone) ? `[${tone}]` : '';
+ }
+ return '';
+ };
+
+ out = replaceWithGroups(out, EXPR_SELF_RE, ({ groups, match, offset, source }) =>
+ dedupRemovalSpace(groups.pre ?? '', self(groups.attrs ?? ''), source, offset + match.length),
+ );
+ // a stray unpaired expr tag (e.g. a prosody wrapper split across stream chunks)
+ // must never reach the TTS as literal text — drop the delimiters, keep the words
+ out = replaceWithGroups(out, EXPR_OPEN_RE, ({ groups, match, offset, source }) =>
+ dedupRemovalSpace(groups.pre ?? '', '', source, offset + match.length),
+ );
+ out = replaceWithGroups(out, EXPR_CLOSE_RE, ({ groups, match, offset, source }) =>
+ dedupRemovalSpace(groups.pre ?? '', '', source, offset + match.length),
+ );
+ return out;
+}
+
+// Providers with an expr instruction block. Kept as a set so "does this voice speak
+// markup?" is answerable without rendering the block — the answer is needed on the
+// per-segment speech path, and the blocks run to several kilobytes.
+const MARKUP_DIALECTS = new Set(['cartesia', 'inworld', 'xai', 'fishaudio']);
+
+/**
+ * Whether `provider` has an expr instruction block, i.e. whether expressive can do
+ * anything for it. Allocation-free: prefer this over testing
+ * `llmInstructions(...) !== undefined`.
+ */
+export function hasMarkupDialect(provider: string): boolean {
+ return MARKUP_DIALECTS.has(provider);
+}
+
+/**
+ * LLM instruction text for a TTS provider, or `undefined` when it has no markup dialect.
+ *
+ * Each markup-capable provider gets its own expr instruction block — shared marker syntax,
+ * but only the types and label vocabularies that provider actually supports;
+ * {@link convertMarkup} lowers the markers to native syntax. Expr is the only dialect the
+ * LLM is ever taught. When `steering` disables a non-verbal sound, its labels (and any
+ * example demonstrating them) are omitted from the block entirely rather than advertised
+ * and then revoked.
+ */
+export function llmInstructions(
+ provider: string,
+ steering?: SpeechSteeringOptions,
+): string | undefined {
+ if (!hasMarkupDialect(provider)) {
+ return undefined;
+ }
+ if (provider === 'cartesia') {
+ return CARTESIA_EXPR_LLM_INSTRUCTIONS;
+ }
+ if (provider === 'inworld') {
+ return inworldExprLlmInstructions(allowedSounds(provider, steering));
+ }
+ if (provider === 'xai') {
+ return xaiExprLlmInstructions(
+ allowedSounds(provider, steering),
+ allowedProsody(provider, steering),
+ );
+ }
+ if (provider === 'fishaudio') {
+ return fishaudioExprLlmInstructions(
+ allowedSounds(provider, steering),
+ steering?.disfluencies ?? true,
+ );
+ }
+ return undefined;
+}
+
+// Per-provider native XML tag names. Membership also marks a provider as markup-capable
+// (see normalizeMarkup / convertMarkup); the LLM only ever writes expr markers, so these
+// names exist to lower expr onto and to catch hallucinated natives.
+const PROVIDER_MARKUP: Record = {
+ cartesia: CARTESIA_TAGS,
+ inworld: INWORLD_TAGS,
+ xai: XAI_TAGS,
+ // fish's native dialect is square brackets, produced only by convertMarkup for the TTS;
+ // these names exist to catch hallucinated XML natives in transcripts
+ fishaudio: FISHAUDIO_TAGS,
+};
+
+// Union of every provider's XML tag names — used by the transcript sinks to strip markup
+// without knowing which provider produced it (see TranscriptMarkupStripper).
+const ALL_MARKUP_TAGS: string[] = [...new Set(Object.values(PROVIDER_MARKUP).flat())].sort();
+
+// Tags whose payload lives in an attribute rather than in their content. They are
+// self-closing by definition, but models do write `words`
+// (which is exactly why `normalizeMarkup` repairs that shape) — and the transcript sinks
+// strip the raw text, before any repair. Without this, the wrapped sentence would be
+// recorded as the delivery label and published as `lk.expression`.
+const ATTRIBUTE_MARKUP_TAGS: ReadonlySet = new Set([
+ 'expression',
+ 'emotion',
+ 'sound',
+ 'break',
+ 'speed',
+ 'volume',
+]);
+
+/**
+ * Strip the union of every provider's expressive XML markup (provider-agnostic).
+ *
+ * The transcript sinks strip downstream, where the originating TTS/provider is no longer
+ * in scope, so they remove every provider's XML tags at once: expr markers (all the LLM is
+ * ever taught) plus every native tag name, so a hallucinated native tag is stripped rather
+ * than leaked.
+ *
+ * Square-bracket spans are *not* stripped: the LLM only writes expr, so brackets in its
+ * output are prose (a `[text](url)` link) that a strip would mangle. Provider-native
+ * brackets never arrive here — {@link dropBracketCues} removes them at their source.
+ */
+export function splitAllMarkup(text: string): [string, ExpressiveTag[]] {
+ // every markup shape is angle-bracketed, so text without "<" cannot contain any. The
+ // sinks call this per streamed chunk and expressive is off by default, making this the
+ // overwhelmingly common case — skip the tag-union scan entirely
+ if (!text.includes('<')) {
+ return [text, []];
+ }
+
+ const [withoutExpr, exprTags] = splitExpr(text);
+ const [clean, rawTags] = extractAndStrip(withoutExpr, ALL_MARKUP_TAGS, ATTRIBUTE_MARKUP_TAGS);
+ return [clean, [...exprTags, ...rawTags.map(([type, value]) => ({ type, value }))]];
+}
+
+/** {@link splitAllMarkup} returning only the clean text (tags discarded). */
+export function stripAllMarkup(text: string): string {
+ return splitAllMarkup(text)[0];
+}
+
+/**
+ * Strip only the `` dialect, leaving all other markup untouched.
+ *
+ * Unlike {@link stripAllMarkup}, provider-native tags survive (both leave square-bracket
+ * spans alone).
+ */
+export function stripExprMarkup(text: string): string {
+ return splitExpr(text)[0];
+}
+
+/**
+ * Build the `lk.expression` transcription attribute from stripped markup tags.
+ *
+ * Surfaces a segment's leading delivery/emotion (`expression` for Inworld/xAI, `emotion`
+ * for Cartesia) as `{"expression": ..., "mood": ...}`: the provider's own words, plus the
+ * mood they normalize to, so a client can drive UI off a fixed enum without
+ * reimplementing the matching. Returns `undefined` when no such tag was present.
+ */
+export function expressionAttribute(tags: ExpressiveTag[]): Record | undefined {
+ const expression = tags.find((t) => t.type === 'expression' || t.type === 'emotion')?.value;
+ if (expression === undefined) {
+ return undefined;
+ }
+ const payload = { expression, mood: matchMood(expression) };
+ return { [ATTRIBUTE_TRANSCRIPTION_EXPRESSION]: JSON.stringify(payload) };
+}
+
+/**
+ * Stateful, provider-agnostic markup stripper for one transcript segment.
+ *
+ * Fed text chunk-by-chunk, it returns the user-visible text and accumulates the stripped
+ * tags. A tag-shaped trailing fragment (a partial `<...` arriving split across chunks) is
+ * held back until it closes, so a tag straddling a chunk boundary is never emitted
+ * half-stripped. Shared by the transcript sinks (room output + transcript synchronizer) so
+ * stripping and expression extraction stay identical across them.
+ */
+export class TranscriptMarkupStripper {
+ #buf = '';
+ #tags: ExpressiveTag[] = [];
+ #seamAfterStrip = false;
+ #emittedVisible = false;
+
+ /**
+ * Strip `text`, record its tags, and keep a removed tag from doubling a space.
+ *
+ * {@link splitAllMarkup} drops one of the two spaces a removed tag sat between, but only
+ * when it can see both. Trailing whitespace is therefore held back rather than emitted,
+ * so a tag opening the *next* chunk is still stripped against the space before it;
+ * `final` releases the held whitespace at segment end.
+ */
+ #consume(text: string, final: boolean): string {
+ let input = text;
+ if (this.#seamAfterStrip && (input[0] === ' ' || input[0] === '\t')) {
+ // a tag was stripped right at the held whitespace: collapse that whitespace with the
+ // run following it, leaving the single separator the words need
+ input = input[0] + input.slice(1).replace(/^[ \t]+/, '');
+ }
+
+ const [clean, tags] = splitAllMarkup(input);
+ this.#tags.push(...tags);
+
+ const trimmed = trimEndSpaces(clean);
+ const held = final ? '' : clean.slice(trimmed.length);
+ this.#buf = held;
+ // the held whitespace only abuts a removal when this chunk *ended* on a tag; a tag
+ // stripped earlier in the chunk leaves whitespace the LLM itself wrote, which is
+ // passed through rather than collapsed
+ this.#seamAfterStrip = tags.length > 0 && held.length > 0 && trimEndSpaces(input).endsWith('>');
+
+ let emit = clean.slice(0, clean.length - held.length);
+ if (!this.#emittedVisible) {
+ // A marker opening the segment leaves the space that followed it behind: the dedup
+ // drops the whitespace *before* a removed tag, and at position 0 there is none. The
+ // instructions ask for a leading expression marker, so this is the common case —
+ // without this the transcript would open with a space on nearly every turn.
+ emit = emit.replace(/^\s+/, '');
+ }
+ if (emit) this.#emittedVisible = true;
+ return emit;
+ }
+
+ #hasOpenTag(): boolean {
+ // hold a tag-shaped trailing "<" (partial XML tag) so "3 < 5" isn't stalled. An
+ // unclosed "[" is not held: brackets aren't markup here, and stalling on one would
+ // delay every markdown link until its "]" arrives
+ const lastLt = this.#buf.lastIndexOf('<');
+ if (lastLt > this.#buf.lastIndexOf('>')) {
+ const nxt = this.#buf.slice(lastLt + 1, lastLt + 2);
+ if (nxt === '' || nxt === '/' || /[a-z]/i.test(nxt)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /** Feed a chunk; return the clean text ready to emit (may be empty). */
+ push(text: string): string {
+ this.#buf += text;
+ if (this.#hasOpenTag()) {
+ return '';
+ }
+ return this.#consume(this.#buf, false);
+ }
+
+ /** Drain any buffered text at segment end; return the remaining clean text. */
+ flush(): string {
+ if (!this.#buf) {
+ return '';
+ }
+ return this.#consume(this.#buf, true);
+ }
+
+ /** The markup tags stripped so far, in document order. */
+ get tags(): ExpressiveTag[] {
+ return this.#tags;
+ }
+
+ /** The `lk.expression` attribute for the tags stripped so far, if any. */
+ expressionAttribute(): Record | undefined {
+ return expressionAttribute(this.#tags);
+ }
+}
+
+function trimEndSpaces(value: string): string {
+ return value.replace(/[ \t]+$/, '');
+}
+
+const BRACKET_SPAN_RE = /\[[^\]]*\]/g;
+// cap on how long an unclosed "[" is held before it is released as plain text
+const MAX_HELD_CHARS = 256;
+
+/** A copy of `token` carrying `text`, keeping the alignment metadata. */
+function retext(token: TimedString, text: string): TimedString {
+ return createTimedString({
+ text,
+ startTime: token.startTime,
+ endTime: token.endTime,
+ confidence: token.confidence,
+ startTimeOffset: token.startTimeOffset,
+ speakerId: token.speakerId,
+ });
+}
+
+/**
+ * Remove bracket cues from TTS-aligned tokens, keeping the survivors' timings.
+ *
+ * `useTtsAlignedTranscript` makes the provider's alignment of the text it was sent the
+ * transcript, and that text is post-{@link convertMarkup}, so it carries native
+ * `[laugh]`/`[speak calmly]` cues as words the agent never spoke. Every bracket span goes:
+ * the provider reads them all as cues, so none is ever audio, and markdown links are
+ * already gone (`filterMarkdown` runs on TTS input by default).
+ *
+ * Alignment arrives in messages finer-grained than a cue — often one word at a time — so
+ * `held` carries the tail of an unclosed span across calls; pass the same array every time
+ * and call once more with `final: true` at end of stream to release it.
+ */
+export function dropBracketCues(
+ tokens: TimedString[],
+ held: TimedString[],
+ options: { final?: boolean } = {},
+): TimedString[] {
+ const all = [...held, ...tokens];
+ held.length = 0;
+ const text = all.map((t) => t.text).join('');
+ if (!text.includes('[')) {
+ return all;
+ }
+
+ const dropped = new Set();
+ for (const match of text.matchAll(BRACKET_SPAN_RE)) {
+ let start = match.index!;
+ let end = start + match[0].length;
+ // take one of the spaces the cue sat between, so it leaves a single separator
+ if (start > 0 && text[start - 1] === ' ' && (end === text.length || text[end] === ' ')) {
+ start -= 1;
+ } else if (start === 0 && end < text.length && text[end] === ' ') {
+ end += 1;
+ }
+ for (let i = start; i < end; i++) dropped.add(i);
+ }
+
+ // hold from an unclosed "[" so a cue straddling messages is still judged as a whole;
+ // past MAX_HELD_CHARS give up, since a lone bracket must not stall the transcript
+ let holdFrom = text.length;
+ if (!options.final) {
+ const openAt = text.lastIndexOf('[');
+ if (openAt > text.lastIndexOf(']') && text.length - openAt <= MAX_HELD_CHARS) {
+ holdFrom = openAt;
+ }
+ }
+
+ const out: TimedString[] = [];
+ let pos = 0;
+ for (const token of all) {
+ let emit = '';
+ let keep = '';
+ for (let i = 0; i < token.text.length; i++) {
+ const idx = pos + i;
+ const char = token.text[i]!;
+ if (idx >= holdFrom) {
+ keep += char;
+ } else if (!dropped.has(idx)) {
+ emit += char;
+ }
+ }
+ pos += token.text.length;
+ if (emit) out.push(emit === token.text ? token : retext(token, emit));
+ if (keep) held.push(keep === token.text ? token : retext(token, keep));
+ }
+ return out;
+}
+
+const SELF_CLOSING_TAGS: Record = {
+ cartesia: ['emotion', 'speed', 'volume', 'break'],
+ inworld: ['expression', 'sound', 'break'],
+ fishaudio: ['expression', 'sound', 'break'],
+};
+
+/**
+ * Fix common LLM markup mistakes for a provider.
+ *
+ * Closes opening tags that should be self-closing (e.g. the LLM writes
+ * `` instead of `` — or
+ * `` instead of ``).
+ */
+export function normalizeMarkup(provider: string, text: string): string {
+ let out = text;
+ if (provider in PROVIDER_MARKUP) {
+ out = out.replace(EXPR_UNCLOSED_RE, '$1/>');
+ }
+ const tags = SELF_CLOSING_TAGS[provider];
+ if (!tags) {
+ return out;
+ }
+ const pattern = new RegExp(`<(${tags.map(escapeRegExp).join('|')})\\b([^>]*[^/])\\s*>`, 'g');
+ return out.replace(pattern, '<$1$2/>');
+}
+
+/** Convert framework-standard markup to a provider's native syntax. */
+export function convertMarkup(provider: string, text: string): string {
+ let out = text;
+ if (provider in PROVIDER_MARKUP) {
+ // lower expr markers first; the per-provider conversions below then handle the
+ // intermediate framework-standard tags they produce
+ out = convertExpr(provider, out);
+ }
+ if (provider === 'inworld' || provider === 'xai') {
+ // -> [X] (and -> [X]); for xAI this turns
+ // inline sounds into its native brackets while emotion/prosody stay <..>
+ out = convertExpressionTags(out);
+ }
+ if (provider === 'xai') {
+ // xAI has no ; map it to its native [pause]/[long-pause]
+ out = out.replace(XAI_BREAK_RE, xaiBreakToBracket);
+ }
+ if (provider === 'fishaudio') {
+ // -> [very X] first (the intensified form steers harder),
+ // then the generic pass lowers the remaining -> [X]
+ out = out.replace(FISHAUDIO_EXPRESSION_RE, fishaudioExpressionToBracket);
+ out = convertExpressionTags(out);
+ out = out.replace(FISHAUDIO_BREAK_RE, fishaudioBreakToBracket);
+ // Fish's per-word stress marker: word -> [emphasis] word
+ out = out.replace(FISHAUDIO_EMPHASIS_RE, (_m, inner: string) => `[emphasis] ${inner.trim()}`);
+ }
+ // is otherwise passed through unchanged: Inworld accepts it as native SSML.
+ return out;
+}
diff --git a/agents/src/tts/tts.ts b/agents/src/tts/tts.ts
index 6851bc64d..5a3975191 100644
--- a/agents/src/tts/tts.ts
+++ b/agents/src/tts/tts.ts
@@ -20,6 +20,15 @@ import {
} from '../types.js';
import { AsyncIterableQueue, delay, mergeFrames, startSoon, toError } from '../utils.js';
import type { TimedString } from '../voice/io.js';
+import {
+ type MarkupInfo,
+ type SpeechSteeringOptions,
+ convertMarkup,
+ hasMarkupDialect,
+ llmInstructions,
+ normalizeMarkup,
+ supportedNonverbals,
+} from './provider_format.js';
/**
* SynthesizedAudio is a packet of speech synthesis as returned by the TTS.
@@ -54,6 +63,71 @@ export interface TTSCapabilities {
alignedTranscript?: boolean;
}
+/**
+ * Declares TTS markup capabilities for the expressive pipeline.
+ *
+ * Plugins opt in by overriding {@link TTS.markupProviderKey}: it selects which markup
+ * dialect the TTS speaks — what the LLM is taught to write, and how those markers are
+ * normalized and lowered to the provider's native syntax before synthesis. Stripping
+ * markup back out is not here — the transcript sinks do it provider-agnostically (see
+ * `splitAllMarkup`).
+ */
+export class TTSMarkup {
+ #providerKey: () => string;
+
+ /** @internal */
+ constructor(providerKey: () => string) {
+ this.#providerKey = providerKey;
+ }
+
+ /** Key into the shared `provider_format` markup tables, or `''` for none. */
+ get providerKey(): string {
+ return this.#providerKey();
+ }
+
+ /**
+ * Whether this voice speaks a markup dialect at all.
+ *
+ * Allocation-free, unlike testing {@link llmInstructions} for `undefined` — which the
+ * expressive gate does once per speech segment.
+ */
+ get supported(): boolean {
+ return hasMarkupDialect(this.providerKey);
+ }
+
+ /** The queryable markup matrix for this voice. */
+ get info(): MarkupInfo {
+ return { nonverbals: supportedNonverbals(this.providerKey) };
+ }
+
+ /**
+ * Instructions for the LLM describing available markup tags.
+ *
+ * The framework injects this into the LLM system prompt when expressive mode is active.
+ * Returns `undefined` if this TTS has no markup support. When `speechSteering` is given,
+ * sounds it disables are omitted from the advertised vocabulary.
+ */
+ llmInstructions(options: { speechSteering?: SpeechSteeringOptions } = {}): string | undefined {
+ return llmInstructions(this.providerKey, options.speechSteering);
+ }
+
+ /** Fix common LLM markup mistakes (e.g. unclosed self-closing tags). */
+ normalize(text: string): string {
+ return normalizeMarkup(this.providerKey, text);
+ }
+
+ /**
+ * Convert framework-standard markup to the provider's native format.
+ *
+ * Called before text is sent to the TTS; a no-op when the provider declares no markup.
+ * Plugins that use non-XML formats (e.g. square brackets) opt in via
+ * {@link TTS.markupProviderKey} so `` becomes native syntax.
+ */
+ convert(text: string): string {
+ return convertMarkup(this.providerKey, text);
+ }
+}
+
export interface TTSError {
type: 'tts_error';
timestamp: number;
@@ -90,6 +164,14 @@ export abstract class TTS extends (EventEmitter as new () => TypedEmitter TypedEmitter this.markupProviderKey());
+ }
+
+ /**
+ * Key into the shared markup tables, or `''` for none.
+ *
+ * Plugins override this to opt into markup support; the default (`''`) means no markup
+ * instructions, normalization, or conversion are applied. Every {@link TTS.markup} method
+ * delegates through this key, so a plugin only needs to override this one method.
+ */
+ protected markupProviderKey(): string {
+ return '';
+ }
+
+ /** Access TTS markup capabilities (instructions for the LLM, text conversion). */
+ get markup(): TTSMarkup {
+ return this.#markup;
+ }
+
+ /**
+ * Whether expressive is active for the current turn.
+ * @internal
+ */
+ get expressive(): boolean {
+ return this.#expressive;
+ }
+
+ /**
+ * Framework-internal: mark whether expressive is active for this turn.
+ *
+ * Called by the voice pipeline before each synthesis. TTS implementations widen their
+ * input chunking when enabled; a no-op for TTS that don't tokenize their own input.
+ *
+ * @internal
+ */
+ _setExpressive(enabled: boolean): void {
+ this.#expressive = enabled;
}
/** Returns this TTS's capabilities */
diff --git a/agents/src/voice/agent.test.ts b/agents/src/voice/agent.test.ts
index cbde76fa0..3b7bf7493 100644
--- a/agents/src/voice/agent.test.ts
+++ b/agents/src/voice/agent.test.ts
@@ -415,8 +415,10 @@ describe('Agent', () => {
capabilities: { streaming: true },
stream: () => ttsStream,
close: async () => {},
+ _setExpressive: () => {},
},
agentSession: { connOptions: { ttsConnOptions: {} } },
+ _resolveExpressiveOptions: () => undefined,
};
async function* textInput() {
diff --git a/agents/src/voice/agent.ts b/agents/src/voice/agent.ts
index 05536663b..b095d7d4f 100644
--- a/agents/src/voice/agent.ts
+++ b/agents/src/voice/agent.ts
@@ -522,12 +522,26 @@ export class Agent {
throw new Error('ttsNode called but no TTS node is available');
}
+ const expressiveActive = activity._resolveExpressiveOptions() !== undefined;
let wrappedTts = activity.tts;
if (!activity.tts.capabilities.streaming) {
- wrappedTts = new TTSStreamAdapter(wrappedTts, new BasicSentenceTokenizer());
+ wrappedTts = new TTSStreamAdapter(
+ wrappedTts,
+ // markup only exists in the stream when expressive is active. Python also
+ // passes retain_format here, but that predates expressive mode and is a
+ // separate gap — turning it on would change tokenization for every
+ // non-streaming TTS plugin, none of which can be expressive today.
+ new BasicSentenceTokenizer({ xmlAware: expressiveActive }),
+ );
}
+ // Mark whether expressive is active for this synthesis, synchronously just before
+ // stream() snapshots it. Doing it here (the single synthesis choke point for both
+ // generateReply and say()) scopes it to this turn rather than leaving stale state on
+ // the instance. The provider's chunk defaults then drive the TTS's input tokenizer.
+ activity.tts._setExpressive(expressiveActive);
+
const connOptions = activity.agentSession.connOptions.ttsConnOptions;
const stream = wrappedTts.stream({ connOptions });
stream.updateInputStream(input);
diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts
index 3ee2bce4c..1f67c6129 100644
--- a/agents/src/voice/agent_activity.ts
+++ b/agents/src/voice/agent_activity.ts
@@ -17,6 +17,7 @@ import {
import type { InterruptionDetectionError } from '../inference/interruption/errors.js';
import { AdaptiveInterruptionDetector } from '../inference/interruption/interruption_detector.js';
import type { OverlappingSpeechEvent } from '../inference/interruption/types.js';
+import { TTS as InferenceTTS } from '../inference/tts.js';
import {
AgentConfigUpdate,
type ChatContext,
@@ -93,7 +94,14 @@ import {
_setActivityTaskInfo,
speechHandleStorage,
} from './agent.js';
-import { type AgentSession, type TurnDetectionMode } from './agent_session.js';
+import {
+ type AgentSession,
+ DEFAULT_EXPRESSIVE_OPTIONS,
+ type ExpressiveOptions,
+ TTS_INSTRUCTIONS_PLACEHOLDER,
+ type TurnDetectionMode,
+ resolveExpressiveOptions,
+} from './agent_session.js';
import {
AudioRecognition,
type EndOfTurnInfo,
@@ -127,12 +135,16 @@ import {
_stripRunningToolCalls,
applyInstructionsModality,
forwardedTextFor,
+ hasExpressiveInstructions,
performAudioForwarding,
performLLMInference,
performTTSInference,
performTextForwarding,
performToolExecutions,
+ removeExpressiveInstructions,
removeInstructions,
+ stripAssistantMarkup,
+ updateExpressiveInstructions,
updateInstructions,
} from './generation.js';
import type { PlaybackFinishedEvent, TimedString } from './io.js';
@@ -972,6 +984,89 @@ export class AgentActivity implements RecognitionHooks {
}
}
+ /**
+ * Resolve the session's expressive setting. Returns `undefined` if disabled.
+ *
+ * Expressive mode requires three things, checked cheapest-first because this runs once
+ * per speech segment:
+ * - the session opted in.
+ * - the inference gateway TTS ({@link inference.TTS}): the markup normalization/conversion
+ * and expressive chunking run there, so direct provider plugins would receive
+ * unconverted markup.
+ * - a TTS that actually declares a markup dialect: gateway providers without one (e.g.
+ * `rime`, `deepgram`) get no markup instructions, so no tags can appear in the stream
+ * — leaving it "active" would enable xml-aware chunking with nothing to chunk and
+ * re-introduce the stray-`<` streaming stall. Asked via `markup.supported` rather than
+ * by rendering `llmInstructions()` and testing it for `undefined`: the blocks are
+ * several kilobytes, and every ordinary session would build and discard one per turn.
+ *
+ * @internal
+ */
+ _resolveExpressiveOptions(): ExpressiveOptions | undefined {
+ const expr = this.agentSession._expressive;
+ if (!expr && typeof expr !== 'object') {
+ return undefined;
+ }
+
+ if (!(this.tts instanceof InferenceTTS) || !this.tts.markup.supported) {
+ return undefined;
+ }
+ // speechSteering renders per-provider delivery guidelines on top of the
+ // provider-agnostic default; explicit templates override
+ return resolveExpressiveOptions(typeof expr === 'object' ? expr : {}, {
+ providerKey: this.tts.markup.providerKey,
+ defaults: DEFAULT_EXPRESSIVE_OPTIONS,
+ });
+ }
+
+ /** Inject the TTS markup guide into the chat context. */
+ private injectExpressiveInstructions(
+ chatCtx: ChatContext,
+ options: ExpressiveOptions,
+ speechHandle: SpeechHandle | undefined,
+ ): void {
+ const turnModality = speechHandle?.inputDetails.modality;
+
+ const ttsInstructions = this.tts?.markup.llmInstructions({
+ speechSteering: options.speechSteering,
+ });
+ if (!ttsInstructions) return;
+
+ const template = options.ttsInstructionsTemplate ?? '';
+ const raw = renderInstructions(template, turnModality);
+
+ if (
+ !raw.includes(TTS_INSTRUCTIONS_PLACEHOLDER) &&
+ !this.agentSession._warnedExpressiveTemplate
+ ) {
+ // The placeholder is the only channel the provider's markup vocabulary has. Without
+ // it the model is never taught the tags, yet the rest of the pipeline still runs as
+ // if it were: xml-aware chunking, markup conversion on the audio path, and markup
+ // stripping in the transcript sinks. Not an error — a template may legitimately
+ // hardcode the vocabulary — but silently shipping expressive with no guide is far
+ // more often a mistake.
+ this.agentSession._warnedExpressiveTemplate = true;
+ this.logger.warn(
+ { placeholder: TTS_INSTRUCTIONS_PLACEHOLDER },
+ 'expressive is enabled but the tts instructions template does not contain the markup ' +
+ "guide placeholder, so the LLM is never given the provider's markup vocabulary. " +
+ 'Include the placeholder in `expressive.ttsInstructionsTemplate`, use ' +
+ '`expressive.ttsInstructionsAppend` to add rules on top of the default template, ' +
+ 'or ignore this if the template spells out the vocabulary itself.',
+ );
+ }
+
+ const rendered = raw.replaceAll(TTS_INSTRUCTIONS_PLACEHOLDER, ttsInstructions);
+ if (rendered.trim()) {
+ // keyed message: re-injection replaces last turn's guide instead of stacking
+ // copies, and an expressive-off turn removes it again
+ updateExpressiveInstructions(chatCtx, { text: rendered });
+ // latch: a later expressive-off turn (or a handoff to a TTS without a markup
+ // dialect) needs to know markup may be sitting in history
+ this.agentSession._expressiveEverActive = true;
+ }
+ }
+
async updateTools(tools: ToolContextLike): Promise {
const oldToolCtx = this.agent._toolCtx;
const oldToolNames = new Set(Object.keys(oldToolCtx.functionTools));
@@ -2782,6 +2877,37 @@ export class AgentActivity implements RecognitionHooks {
// apply the correct variant of the instructions for the turn's input modality
applyInstructionsModality(chatCtx, { modality: speechHandle.inputDetails.modality });
+ // inject expressive instructions (TTS markup guide)
+ const expressiveOptions = this._resolveExpressiveOptions();
+ if (expressiveOptions !== undefined) {
+ this.injectExpressiveInstructions(chatCtx, expressiveOptions, speechHandle);
+ } else if (
+ // Only scrub when expressive was actually live at some point: this branch is the
+ // default path for every session that never enabled the feature, and the scrub
+ // mutates stored history using the union of every provider's tag names — so an
+ // agent that legitimately writes `` or `` would have it
+ // silently deleted. The stored flag covers a handoff to a TTS without a markup
+ // dialect (a fresh activity, same session); the message check covers history
+ // restored from an earlier run.
+ this.agentSession._expressiveEverActive ||
+ hasExpressiveInstructions(chatCtx) ||
+ hasExpressiveInstructions(this.agent._chatCtx)
+ ) {
+ // expressive is off for this turn (an agent override, or a handoff to a TTS without
+ // a markup dialect): remove the injected markup guide and scrub markup left in past
+ // assistant turns so the LLM isn't instructed or few-shotted into emitting tags
+ // nothing downstream converts or strips — an unsupported tag would reach the TTS as
+ // literal text and be spoken.
+ removeExpressiveInstructions(chatCtx);
+ stripAssistantMarkup(chatCtx);
+ if (chatCtx !== this.agent._chatCtx) {
+ // user turns run on a copy of the agent's history; clean the stored history too so
+ // stale markup doesn't survive into future snapshots
+ removeExpressiveInstructions(this.agent._chatCtx);
+ stripAssistantMarkup(this.agent._chatCtx);
+ }
+ }
+
const runningCalls = getRunningTasks(this.agentSession);
_injectRunningToolCalls(chatCtx, runningCalls);
const tasks: Array> = [];
diff --git a/agents/src/voice/agent_session.ts b/agents/src/voice/agent_session.ts
index bd0b5c47c..663c4bce3 100644
--- a/agents/src/voice/agent_session.ts
+++ b/agents/src/voice/agent_session.ts
@@ -32,6 +32,7 @@ import {
ChatContext,
ChatMessage,
type Instructions,
+ concatInstructions,
} from '../llm/chat_context.js';
import type {
LLM,
@@ -49,6 +50,11 @@ import { SimulationMode } from '../simulation.js';
import type { STT } from '../stt/index.js';
import type { STTError } from '../stt/stt.js';
import { traceTypes, tracer } from '../telemetry/index.js';
+import {
+ DEFAULT_SPEECH_STEERING_OPTIONS,
+ type SpeechSteeringOptions,
+ steeringInstructions,
+} from '../tts/provider_format.js';
import type { TTS, TTSError } from '../tts/tts.js';
import {
DEFAULT_API_CONNECT_OPTIONS,
@@ -331,8 +337,93 @@ export type AgentSessionOptions = {
* and `filter_emoji`; pass `null` to disable text transforms.
*/
ttsTextTransforms?: readonly TextTransform[] | null;
+
+ /**
+ * Let the LLM steer how the agent sounds.
+ *
+ * When enabled, the provider's markup guide is injected into the LLM prompt so it can
+ * emit inline delivery tags (emotion, pacing, non-verbal sounds), which are rendered by
+ * the TTS and stripped from the transcript. Pass an {@link ExpressiveOptions} object to
+ * steer or override the injected instructions. Requires an
+ * {@link inference.TTS | inference TTS} with a model that declares a markup dialect; it
+ * stays off otherwise.
+ *
+ * @defaultValue false
+ */
+ expressive?: boolean | ExpressiveOptions;
+};
+
+/**
+ * Configuration for the expressive pipeline, passed as `AgentSession({ expressive: ... })`.
+ *
+ * Controls how TTS markup instructions are injected into the LLM when expressive is
+ * enabled. All keys are optional; common shapes:
+ *
+ * - `{ speechSteering: {...} }` — steer delivery and non-verbal sounds on top of the
+ * provider-agnostic default instructions.
+ * - `{ ttsInstructionsTemplate: '...' }` — a fully custom prompt.
+ * - `{ ttsInstructionsAppend: '...' }` — your own rules appended to the template.
+ *
+ * Any explicit template overrides the default; unset parts fall back to the
+ * provider-agnostic default.
+ */
+export interface ExpressiveOptions {
+ speechSteering?: SpeechSteeringOptions;
+ ttsInstructionsTemplate?: Instructions | string;
+ ttsInstructionsAppend?: string;
+}
+
+/** The placeholder the expressive template substitutes the provider's markup guide into. */
+export const TTS_INSTRUCTIONS_PLACEHOLDER = '{tts.markup.llm_instructions}';
+
+export const DEFAULT_EXPRESSIVE_OPTIONS: ExpressiveOptions = {
+ ttsInstructionsTemplate:
+ 'You can control how you speak using the following formatting tags. ' +
+ 'Use them when appropriate to make your speech more expressive and natural:\n\n' +
+ TTS_INSTRUCTIONS_PLACEHOLDER,
+ speechSteering: DEFAULT_SPEECH_STEERING_OPTIONS,
};
+function appendInstructions(template: Instructions | string, extra: string): Instructions | string {
+ // concatenate the *raw* template text so any {placeholders} survive until render
+ return concatInstructions(template, '\n\n' + extra);
+}
+
+/**
+ * Resolve a user {@link ExpressiveOptions} to a concrete options object for a provider.
+ *
+ * Starts from `defaults`, renders `speechSteering` into per-provider delivery guidelines
+ * appended to the template, then applies any explicit `ttsInstructionsTemplate` override
+ * and `ttsInstructionsAppend` (last, so the user's free-form rules always win). Steering
+ * fields the user doesn't set fall back to `defaults`' `speechSteering`, so an explicit
+ * value always wins over a default. The returned object always has
+ * `ttsInstructionsTemplate` and `speechSteering` (never `ttsInstructionsAppend`);
+ * `speechSteering` passes through so injection can filter the advertised markup vocabulary
+ * (`TTSMarkup.llmInstructions`) with it.
+ */
+export function resolveExpressiveOptions(
+ expr: ExpressiveOptions,
+ options: { providerKey: string; defaults: ExpressiveOptions },
+): ExpressiveOptions {
+ const { providerKey, defaults } = options;
+ let ttsTemplate = expr.ttsInstructionsTemplate ?? defaults.ttsInstructionsTemplate!;
+
+ const steering: SpeechSteeringOptions = {
+ ...(defaults.speechSteering ?? {}),
+ ...(expr.speechSteering ?? {}),
+ };
+ const fragment = steeringInstructions(providerKey, steering);
+ if (fragment) {
+ ttsTemplate = appendInstructions(ttsTemplate, fragment);
+ }
+
+ if (expr.ttsInstructionsAppend) {
+ ttsTemplate = appendInstructions(ttsTemplate, expr.ttsInstructionsAppend);
+ }
+
+ return { ttsInstructionsTemplate: ttsTemplate, speechSteering: steering };
+}
+
export type AgentSessionUpdateOptions = {
/** Configuration updates for turn handling. */
turnHandling?: {
@@ -418,6 +509,34 @@ export class AgentSession<
private _aecWarmupTimer: NodeJS.Timeout | null = null;
+ /**
+ * The session's expressive setting, as the user passed it.
+ * @internal
+ */
+ _expressive: boolean | ExpressiveOptions = false;
+
+ /**
+ * Whether the markup guide has been injected at least once this session.
+ *
+ * Latches on: it is what licenses the history scrub on a later expressive-off turn
+ * (a handoff to a TTS without a markup dialect builds a fresh `AgentActivity`, so the
+ * flag has to outlive it). Sessions that never enabled expressive keep it `false` and
+ * are never scrubbed.
+ *
+ * @internal
+ */
+ _expressiveEverActive = false;
+
+ /**
+ * Whether the "template has no markup-guide placeholder" warning has been emitted.
+ *
+ * Session-scoped so a misconfigured template warns once rather than once per turn, and
+ * survives a handoff (which builds a fresh `AgentActivity`).
+ *
+ * @internal
+ */
+ _warnedExpressiveTemplate = false;
+
// Connection options for STT, LLM, and TTS
private _connOptions: ResolvedSessionConnectOptions;
@@ -522,8 +641,10 @@ export class AgentSession<
connOptions,
tools,
toolHandling,
+ expressive,
...resolvedSessionOptions
} = opts;
+ this._expressive = expressive ?? false;
// Merge user-provided connOptions with defaults
this._connOptions = {
sttConnOptions: { ...DEFAULT_API_CONNECT_OPTIONS, ...connOptions?.sttConnOptions },
diff --git a/agents/src/voice/expressive.test.ts b/agents/src/voice/expressive.test.ts
new file mode 100644
index 000000000..06fa35ae2
--- /dev/null
+++ b/agents/src/voice/expressive.test.ts
@@ -0,0 +1,241 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { describe, expect, it } from 'vitest';
+import { ChatContext } from '../llm/chat_context.js';
+import { splitWords } from '../tokenize/basic/word.js';
+import {
+ DEFAULT_SPEECH_STEERING_OPTIONS,
+ TranscriptMarkupStripper,
+} from '../tts/provider_format.js';
+import {
+ DEFAULT_EXPRESSIVE_OPTIONS,
+ type ExpressiveOptions,
+ TTS_INSTRUCTIONS_PLACEHOLDER,
+ resolveExpressiveOptions,
+} from './agent_session.js';
+import { AgentSession } from './agent_session.js';
+import {
+ EXPRESSIVE_INSTRUCTIONS_MESSAGE_ID,
+ hasExpressiveInstructions,
+ removeExpressiveInstructions,
+ stripAssistantMarkup,
+ updateExpressiveInstructions,
+} from './generation.js';
+
+// what an expressive turn actually leaves in history: the expr markers the LLM emitted,
+// plus (defensively) a hallucinated native tag. Square brackets are *not* markup here —
+// they reach history as prose or markdown links, so the scrub must leave them alone.
+const MARKED_UP =
+ ' Welcome back! ' +
+ 'Glad you called again. Docs: [the guide](https://docs.livekit.io).';
+
+describe('stripAssistantMarkup', () => {
+ it('scrubs assistant markup, keeps prose brackets and user content', () => {
+ const ctx = ChatContext.empty();
+ ctx.addMessage({ role: 'assistant', content: MARKED_UP });
+ ctx.addMessage({ role: 'user', content: 'I typed literally' });
+ const plain = ctx.addMessage({ role: 'assistant', content: 'No tags here.' });
+ const plainContent = plain.content;
+
+ stripAssistantMarkup(ctx);
+
+ const assistantTexts = ctx.items
+ .filter((item) => item.type === 'message' && item.role === 'assistant')
+ .map((item) => (item.type === 'message' ? item.textContent : undefined));
+
+ expect(assistantTexts[0]).not.toContain(' item.type === 'message' && item.role === 'user')!;
+ expect(userItem.type === 'message' && userItem.textContent).toContain(
+ '',
+ );
+
+ // tag-free assistant content is left as-is (fast path)
+ expect(plain.content).toBe(plainContent);
+ });
+});
+
+describe('expressive instruction message', () => {
+ it('replaces rather than stacks, and can be removed', () => {
+ const ctx = ChatContext.empty();
+ updateExpressiveInstructions(ctx, { text: 'markup guide v1' });
+ updateExpressiveInstructions(ctx, { text: 'markup guide v2' });
+
+ const guides = ctx.items.filter((item) => item.id === EXPRESSIVE_INSTRUCTIONS_MESSAGE_ID);
+ expect(guides, 're-injection must replace, not stack').toHaveLength(1);
+ expect(guides[0]!.type === 'message' && guides[0]!.textContent).toBe('markup guide v2');
+
+ removeExpressiveInstructions(ctx);
+ expect(ctx.items.every((item) => item.id !== EXPRESSIVE_INSTRUCTIONS_MESSAGE_ID)).toBe(true);
+ });
+});
+
+describe('history-scrub gate', () => {
+ // The scrub is destructive and uses the union of every provider's tag names, and its
+ // branch is the default path for every session that never enabled expressive — so both
+ // signals that license it must stay off until expressive was actually live.
+
+ it('is off for a session that never enabled expressive', () => {
+ expect(new AgentSession()._expressiveEverActive).toBe(false);
+ expect(new AgentSession({ expressive: true })._expressiveEverActive).toBe(
+ false,
+ // still false: the latch flips when the guide is injected, not when the flag is set,
+ // so a TTS with no markup dialect never licenses the scrub
+ );
+ });
+
+ it('detects a restored history carrying the guide', () => {
+ const ctx = ChatContext.empty();
+ expect(hasExpressiveInstructions(ctx)).toBe(false);
+
+ updateExpressiveInstructions(ctx, { text: 'markup guide' });
+ expect(hasExpressiveInstructions(ctx)).toBe(true);
+
+ removeExpressiveInstructions(ctx);
+ expect(hasExpressiveInstructions(ctx)).toBe(false);
+ });
+
+ it('leaves angle-bracketed assistant text alone when nothing licenses a scrub', () => {
+ // what the gate protects: an agent that legitimately writes provider-shaped tags in a
+ // session that never used expressive
+ const ctx = ChatContext.empty();
+ ctx.addMessage({ role: 'assistant', content: 'Hold on nearly there.' });
+
+ expect(hasExpressiveInstructions(ctx)).toBe(false);
+ // (the gate skips stripAssistantMarkup entirely; calling it would destroy this)
+ stripAssistantMarkup(ctx);
+ const scrubbed = ctx.items[0]!;
+ expect(scrubbed.type === 'message' && scrubbed.textContent).toBe('Hold on nearly there.');
+ });
+});
+
+describe('transcript pacing', () => {
+ it('recognizes a markup tag shredded across word tokens', () => {
+ // the word tokenizer emits whitespace-free runs, so the synchronizer must replay the
+ // original slices of pushedText — feeding it the bare word tokens reassembles
+ // ``, which matches nothing and gets
+ // paced as if it were spoken
+ const pushedText = ' Hello there';
+ const stripper = new TranscriptMarkupStripper();
+
+ let cursor = 0;
+ let paced = '';
+ for (const [word] of splitWords(pushedText, false)) {
+ let end = pushedText.indexOf(word, cursor) + word.length;
+ while (end < pushedText.length && !/\s/.test(pushedText[end]!)) end++;
+ paced += stripper.push(pushedText.slice(cursor, end));
+ cursor = end;
+ }
+ paced += stripper.flush();
+
+ expect(paced).not.toContain(' {
+ it('defaults to off and round-trips what was passed', () => {
+ expect(new AgentSession()._expressive).toBe(false);
+ expect(new AgentSession({ expressive: true })._expressive).toBe(true);
+
+ const opts: ExpressiveOptions = { ttsInstructionsAppend: 'Stay upbeat.' };
+ expect(new AgentSession({ expressive: opts })._expressive).toEqual(opts);
+ });
+});
+
+describe('custom template without the markup-guide placeholder', () => {
+ // The placeholder is the only channel the provider's vocabulary has. A template that
+ // omits it still injects a message and still turns on xml-aware chunking, markup
+ // conversion and transcript stripping — the model just never learns the tags.
+
+ const render = (expr: ExpressiveOptions) => {
+ const resolved = resolveExpressiveOptions(expr, {
+ providerKey: 'inworld',
+ defaults: DEFAULT_EXPRESSIVE_OPTIONS,
+ });
+ return String(resolved.ttsInstructionsTemplate);
+ };
+
+ it('drops the vocabulary when the placeholder is missing', () => {
+ const text = render({ ttsInstructionsTemplate: 'Be expressive, please.' });
+ expect(text).not.toContain(TTS_INSTRUCTIONS_PLACEHOLDER);
+ // ...yet it still renders to a non-empty guide message, so nothing downstream notices
+ expect(text.trim()).not.toBe('');
+ });
+
+ it('still renders non-empty for an empty template, via the steering fragment', () => {
+ // the appended delivery guidelines make `rendered.trim()` truthy even here
+ const text = render({ ttsInstructionsTemplate: '' });
+ expect(text).not.toContain(TTS_INSTRUCTIONS_PLACEHOLDER);
+ expect(text.trim()).not.toBe('');
+ expect(text).toContain('Delivery guidelines:');
+ });
+
+ it('keeps the placeholder when only appending', () => {
+ // the documented way to add your own rules keeps the guide intact
+ const text = render({ ttsInstructionsAppend: 'Stay upbeat.' });
+ expect(text).toContain(TTS_INSTRUCTIONS_PLACEHOLDER);
+ expect(text.endsWith('Stay upbeat.')).toBe(true);
+ });
+
+ it('warns at most once per session', () => {
+ expect(new AgentSession()._warnedExpressiveTemplate).toBe(false);
+ });
+});
+
+describe('resolveExpressiveOptions', () => {
+ const resolve = (expr: ExpressiveOptions, providerKey = 'inworld') =>
+ resolveExpressiveOptions(expr, { providerKey, defaults: DEFAULT_EXPRESSIVE_OPTIONS });
+
+ it('keeps the default template and steering when nothing is overridden', () => {
+ const resolved = resolve({});
+ const text = String(resolved.ttsInstructionsTemplate);
+ // the raw placeholder survives resolution — it is filled at injection time
+ expect(text.startsWith(String(DEFAULT_EXPRESSIVE_OPTIONS.ttsInstructionsTemplate))).toBe(true);
+ expect(text).toContain(TTS_INSTRUCTIONS_PLACEHOLDER);
+ // the default steering renders its one non-default field (fillers on)
+ expect(text).toContain('Sprinkle in natural fillers');
+ expect(resolved.speechSteering).toEqual(DEFAULT_SPEECH_STEERING_OPTIONS);
+ });
+
+ it('appends rendered steering guidelines to the template', () => {
+ const resolved = resolve({ speechSteering: { nonverbalSounds: { laughing: false } } });
+ const text = String(resolved.ttsInstructionsTemplate);
+ expect(text).toContain('Delivery guidelines:');
+ expect(text).toContain('Non-verbal sounds:');
+ // an explicit steering value wins over the default, unset ones fall back
+ expect(resolved.speechSteering).toEqual({
+ disfluencies: true,
+ nonverbalSounds: { laughing: false },
+ });
+ });
+
+ it('lets an explicit template override the default, with append applied last', () => {
+ const resolved = resolve({
+ ttsInstructionsTemplate: `Custom. ${TTS_INSTRUCTIONS_PLACEHOLDER}`,
+ ttsInstructionsAppend: 'Stay upbeat.',
+ speechSteering: { disfluencies: false },
+ });
+ const text = String(resolved.ttsInstructionsTemplate);
+ expect(text.startsWith('Custom. ')).toBe(true);
+ expect(text).toContain('No fillers');
+ // the user's free-form rules always win, so they come last
+ expect(text.endsWith('Stay upbeat.')).toBe(true);
+ });
+
+ it('adds nothing for a provider with no steerable vocabulary', () => {
+ const resolved = resolve({ speechSteering: {} }, 'cartesia');
+ expect(String(resolved.ttsInstructionsTemplate)).toBe(
+ String(DEFAULT_EXPRESSIVE_OPTIONS.ttsInstructionsTemplate) +
+ '\n\nDelivery guidelines:\n- Sprinkle in natural fillers (um, uh) and openers ' +
+ '(oh, well, so), zero to two per turn, never mechanical.',
+ );
+ });
+});
diff --git a/agents/src/voice/generation.ts b/agents/src/voice/generation.ts
index 5e951dd8c..76d62665d 100644
--- a/agents/src/voice/generation.ts
+++ b/agents/src/voice/generation.ts
@@ -31,6 +31,7 @@ import { isZodSchema, parseZodSchema } from '../llm/zod-utils.js';
import { log } from '../log.js';
import { IdentityTransform } from '../stream/identity_transform.js';
import { traceTypes, tracer } from '../telemetry/index.js';
+import { stripAllMarkup } from '../tts/provider_format.js';
import {
type FlushSentinel,
USERDATA_TIMED_TRANSCRIPT,
@@ -539,6 +540,79 @@ export function applyInstructionsModality(
});
}
+/**
+ * The ID of the expressive TTS markup-guide message in the chat context.
+ *
+ * The value must not change: it is what lets re-injection replace the previous guide.
+ */
+export const EXPRESSIVE_INSTRUCTIONS_MESSAGE_ID = 'lk.expressive.instructions';
+
+/**
+ * Insert or replace the expressive markup-guide system message.
+ *
+ * Keyed by {@link EXPRESSIVE_INSTRUCTIONS_MESSAGE_ID} so per-turn re-injection replaces the
+ * previous guide instead of accumulating one copy per turn, and a turn that runs with
+ * expressive off can remove it again ({@link removeExpressiveInstructions}).
+ */
+export function updateExpressiveInstructions(chatCtx: ChatContext, options: { text: string }) {
+ const idx = chatCtx.indexById(EXPRESSIVE_INSTRUCTIONS_MESSAGE_ID);
+ if (idx !== undefined) {
+ chatCtx.items[idx] = ChatMessage.create({
+ id: EXPRESSIVE_INSTRUCTIONS_MESSAGE_ID,
+ role: 'system',
+ content: [options.text],
+ createdAt: chatCtx.items[idx]!.createdAt,
+ });
+ } else {
+ chatCtx.addMessage({
+ role: 'system',
+ content: options.text,
+ id: EXPRESSIVE_INSTRUCTIONS_MESSAGE_ID,
+ });
+ }
+}
+
+/**
+ * Whether `chatCtx` carries the expressive markup-guide message.
+ *
+ * Cheap enough to call per turn (an id lookup, no content inspection), so it can gate the
+ * far more expensive — and destructive — {@link stripAssistantMarkup} scrub.
+ */
+export function hasExpressiveInstructions(chatCtx: ChatContext): boolean {
+ return chatCtx.indexById(EXPRESSIVE_INSTRUCTIONS_MESSAGE_ID) !== undefined;
+}
+
+/**
+ * Remove the expressive markup-guide message added by
+ * {@link updateExpressiveInstructions}, if present.
+ */
+export function removeExpressiveInstructions(chatCtx: ChatContext) {
+ for (;;) {
+ const idx = chatCtx.indexById(EXPRESSIVE_INSTRUCTIONS_MESSAGE_ID);
+ if (idx === undefined) break;
+ chatCtx.items.splice(idx, 1);
+ }
+}
+
+/**
+ * Remove expressive TTS markup from past assistant messages, in place.
+ *
+ * Called when a turn runs with expressive off (toggled off via an agent-level override, or
+ * a handoff to a TTS without a markup dialect): tags left in history would few-shot the
+ * LLM into emitting markup that nothing downstream converts or strips, so an unsupported
+ * tag would reach the TTS as literal text and be spoken. Mutates the stored history: once
+ * a turn runs with expressive off, prior turns' markup is gone even if expressive is
+ * re-enabled later (the re-injected instructions carry the style examples instead).
+ */
+export function stripAssistantMarkup(chatCtx: ChatContext) {
+ for (const item of chatCtx.items) {
+ if (item.type !== 'message' || item.role !== 'assistant') continue;
+ // markup is XML-only here: stripAllMarkup leaves square-bracket spans alone
+ if (!item.content.some((c) => typeof c === 'string' && c.includes('<'))) continue;
+ item.content = item.content.map((c) => (typeof c === 'string' ? stripAllMarkup(c) : c));
+ }
+}
+
export function performLLMInference(
node: LLMNode,
chatCtx: ChatContext,
diff --git a/agents/src/voice/index.ts b/agents/src/voice/index.ts
index 2e425cb47..1e7293ed1 100644
--- a/agents/src/voice/index.ts
+++ b/agents/src/voice/index.ts
@@ -19,8 +19,15 @@ export {
AgentSession,
type AgentSessionOptions,
type AgentSessionUsage,
+ type ExpressiveOptions,
type VoiceOptions,
+ DEFAULT_EXPRESSIVE_OPTIONS,
+ TTS_INSTRUCTIONS_PLACEHOLDER,
+ resolveExpressiveOptions,
} from './agent_session.js';
+// re-exported here (they are declared alongside the markup tables) so the expressive
+// option types all sit together on the session surface, as they do in Python
+export type { NonverbalOptions, SpeechSteeringOptions } from '../tts/provider_format.js';
export * from './avatar/index.js';
export * from './background_audio.js';
export { AgentsConsole, TcpAudioInput, TcpAudioOutput } from './console_io.js';
diff --git a/agents/src/voice/room_io/_output.test.ts b/agents/src/voice/room_io/_output.test.ts
index 49a74c119..ba8d202af 100644
--- a/agents/src/voice/room_io/_output.test.ts
+++ b/agents/src/voice/room_io/_output.test.ts
@@ -3,8 +3,13 @@
// SPDX-License-Identifier: Apache-2.0
import { LocalAudioTrack, TrackPublishOptions, TrackSource } from '@livekit/rtc-node';
import { describe, expect, it, vi } from 'vitest';
+import {
+ ATTRIBUTE_TRANSCRIPTION_EXPRESSION,
+ ATTRIBUTE_TRANSCRIPTION_FINAL,
+} from '../../constants.js';
+import { TranscriptMarkupStripper } from '../../tts/provider_format.js';
import { Future } from '../../utils.js';
-import { ParticipantAudioOutput } from './_output.js';
+import { ParticipantAudioOutput, ParticipantTranscriptionOutput } from './_output.js';
type CaptureFrameArg = Parameters[0];
@@ -259,3 +264,161 @@ describe('ParticipantAudioOutput publishTrack', () => {
expect(output.startedFuture.done).toBe(true);
});
});
+
+describe('ParticipantTranscriptionOutput markup stripping', () => {
+ const makeOutput = (expressive = true, isDeltaStream = true) => {
+ const writes: string[] = [];
+ const writers: Array<{ attributes: Record; closed: boolean }> = [];
+
+ const output = Object.create(
+ ParticipantTranscriptionOutput.prototype,
+ ) as ParticipantTranscriptionOutput & Record;
+
+ output.expressiveEnabled = () => expressive;
+ output.participantIdentity = 'agent';
+ output.isDeltaStream = isDeltaStream;
+ output.jsonFormat = false;
+ output.writer = null;
+ output.flushTask = null;
+ output.capturing = false;
+ output.latestText = '';
+ output.currentId = 'SG_test';
+ output.logger = { error: vi.fn(), warn: vi.fn() };
+ output.stripper = new TranscriptMarkupStripper();
+ output.segmentTags = [];
+ output.room = { isConnected: true };
+ output.createTextWriter = async (
+ attributes?: Record,
+ extra?: Record,
+ ) => {
+ const writer = { attributes: { ...attributes, ...extra }, closed: false };
+ writers.push(writer);
+ return {
+ write: async (text: string) => {
+ writes.push(text);
+ },
+ close: async () => {
+ writer.closed = true;
+ },
+ };
+ };
+
+ return { output, writes, writers };
+ };
+
+ it('publishes text held back by the stripper when the segment flushes', async () => {
+ // regression: a segment whose every chunk was held (a tag-shaped "<" that never
+ // closes) reached flush with no writer, and the whole transcript was dropped.
+ const { output, writes } = makeOutput();
+
+ await output.captureText('a {
+ const { output, writers } = makeOutput();
+
+ await output.captureText('a {
+ const { output, writes, writers } = makeOutput();
+
+ await output.captureText(' Hello there');
+ output.flush();
+ await output.flushTask.result;
+
+ // no leading space: the marker opened the segment, so the space it left is trimmed
+ expect(writes.join('')).toBe('Hello there');
+ expect(writers[0]!.attributes[ATTRIBUTE_TRANSCRIPTION_EXPRESSION]).toBe(
+ '{"expression":"happy","mood":"happy"}',
+ );
+ });
+
+ describe('a marker opening the segment', () => {
+ // the dedup drops the whitespace *before* a removed tag; at position 0 there is none,
+ // so the space that followed the marker survived and every turn opened with it
+ const TURN = ' Hey, good to hear from you!';
+
+ it('does not leave a leading space on the delta path', async () => {
+ const { output, writes } = makeOutput(true, true);
+
+ // chunked the way an LLM streams, so the marker and the text can split apart
+ for (const c of TURN.match(/.{1,14}/gs) ?? []) await output.captureText(c);
+ output.flush();
+ await output.flushTask.result;
+
+ expect(writes.join('')).toBe('Hey, good to hear from you!');
+ });
+
+ it('does not leave a leading space on the non-delta path', async () => {
+ const { output, writes } = makeOutput(true, false);
+
+ await output.captureText(TURN);
+ output.flush();
+ await output.flushTask?.result;
+
+ expect(writes[writes.length - 1]).toBe('Hey, good to hear from you!');
+ });
+
+ it('leaves leading whitespace alone when expressive is off', async () => {
+ // nothing was stripped, so the text is the agent's own and is published verbatim
+ const { output, writes } = makeOutput(false, true);
+
+ await output.captureText(' spaced out');
+ output.flush();
+ await output.flushTask.result;
+
+ expect(writes.join('')).toBe(' spaced out');
+ });
+
+ it('only trims the head, not later chunk boundaries', async () => {
+ const { output, writes } = makeOutput(true, true);
+
+ await output.captureText(' Hey there.');
+ await output.captureText(' And also this.');
+ output.flush();
+ await output.flushTask.result;
+
+ expect(writes.join('')).toBe('Hey there. And also this.');
+ });
+ });
+
+ describe('with expressive off', () => {
+ it('publishes tag-shaped text verbatim', async () => {
+ // the strip works off the union of every provider's tag names, so a session that
+ // never enabled expressive must not have `` removed, and must
+ // carry no expression attribute
+ const { output, writes, writers } = makeOutput(false);
+
+ await output.captureText('Hold on nearly there.');
+ output.flush();
+ await output.flushTask.result;
+
+ expect(writes.join('')).toBe('Hold on nearly there.');
+ expect(writers[0]!.attributes[ATTRIBUTE_TRANSCRIPTION_EXPRESSION]).toBeUndefined();
+ });
+
+ it('does not hold back a tag-shaped chunk', async () => {
+ const { output, writes } = makeOutput(false);
+
+ await output.captureText('3 <');
+ expect(writes, 'nothing is buffered without expressive').toEqual(['3 <']);
+
+ output.flush();
+ await output.flushTask.result;
+ expect(writes.join('')).toBe('3 <');
+ });
+ });
+});
diff --git a/agents/src/voice/room_io/_output.ts b/agents/src/voice/room_io/_output.ts
index a78caa9fe..c167c5747 100644
--- a/agents/src/voice/room_io/_output.ts
+++ b/agents/src/voice/room_io/_output.ts
@@ -23,10 +23,30 @@ import {
TOPIC_TRANSCRIPTION,
} from '../../constants.js';
import { log } from '../../log.js';
+import {
+ type ExpressiveTag,
+ TranscriptMarkupStripper,
+ expressionAttribute,
+ splitAllMarkup,
+ stripAllMarkup,
+} from '../../tts/provider_format.js';
import { Future, Task, shortuuid } from '../../utils.js';
import { AudioOutput, TextOutput, type TimedString, isTimedString } from '../io.js';
import { findMicrophoneTrackId } from '../transcription/index.js';
+export interface TranscriptionOutputOptions {
+ /**
+ * Whether expressive markup may be present in the text reaching this sink.
+ *
+ * Evaluated per chunk, because the session latches it on the first turn that injects
+ * the markup guide — after this sink is constructed. Defaults to "never", so a session
+ * that doesn't use expressive mode publishes its transcript untouched: the strip works
+ * off the union of every provider's tag names, and an agent that legitimately writes
+ * `` should not have it silently deleted.
+ */
+ expressiveEnabled?: () => boolean;
+}
+
abstract class BaseParticipantTranscriptionOutput extends TextOutput {
protected room: Room;
protected isDeltaStream: boolean;
@@ -36,11 +56,18 @@ abstract class BaseParticipantTranscriptionOutput extends TextOutput {
protected latestText: string = '';
protected currentId: string = this.generateCurrentId();
protected logger = log();
+ protected expressiveEnabled: () => boolean;
- constructor(room: Room, isDeltaStream: boolean, participant: Participant | string | null) {
+ constructor(
+ room: Room,
+ isDeltaStream: boolean,
+ participant: Participant | string | null,
+ options: TranscriptionOutputOptions = {},
+ ) {
super();
this.room = room;
this.isDeltaStream = isDeltaStream;
+ this.expressiveEnabled = options.expressiveEnabled ?? (() => false);
this.room.on(RoomEvent.TrackPublished, this.onTrackPublished);
this.room.on(RoomEvent.LocalTrackPublished, this.onLocalTrackPublished);
@@ -131,7 +158,7 @@ abstract class BaseParticipantTranscriptionOutput extends TextOutput {
protected abstract handleFlush(): void;
}
-export interface ParticipantTranscriptionOutputOptions {
+export interface ParticipantTranscriptionOutputOptions extends TranscriptionOutputOptions {
/** When true, each chunk sent on the `lk.transcription` datastream topic is serialized
* as a JSON object with `text`, and `start_time`/`end_time`/`confidence`/
* `start_time_offset` when the captured value is a TimedString. Each object is
@@ -143,6 +170,13 @@ export class ParticipantTranscriptionOutput extends BaseParticipantTranscription
private writer: TextStreamWriter | null = null;
private flushTask: Task | null = null;
private jsonFormat: boolean;
+ /**
+ * Per-segment markup stripping: delta streams strip incrementally (buffering a tag split
+ * across chunks); non-delta streams re-strip the full text each time and keep the latest
+ * tags in {@link segmentTags} for the expression attribute.
+ */
+ private stripper = new TranscriptMarkupStripper();
+ private segmentTags: ExpressiveTag[] = [];
constructor(
room: Room,
@@ -150,7 +184,7 @@ export class ParticipantTranscriptionOutput extends BaseParticipantTranscription
participant: Participant | string | null,
options: ParticipantTranscriptionOutputOptions = {},
) {
- super(room, isDeltaStream, participant);
+ super(room, isDeltaStream, participant, options);
this.jsonFormat = options.jsonFormat ?? false;
}
@@ -159,50 +193,87 @@ export class ParticipantTranscriptionOutput extends BaseParticipantTranscription
return;
}
+ if (this.flushTask && !this.flushTask.done) {
+ await this.flushTask.result;
+ }
+
+ if (!this.capturing) {
+ this.resetState();
+ this.capturing = true;
+ }
+
+ // the raw text (expressive markup intact) arrives here; publish only the visible text.
+ // Skip a chunk that strips to nothing (a partial tag still buffering, or a markup-only
+ // token) so the transcript cadence isn't disturbed. Without expressive there is no
+ // markup to remove, so the text is published exactly as it arrives.
+ const rawText = isTimedString(text) ? text.text : text;
+ let cleanText: string;
+ if (!this.expressiveEnabled()) {
+ cleanText = rawText;
+ } else if (this.isDeltaStream) {
+ cleanText = this.stripper.push(rawText);
+ } else {
+ [cleanText, this.segmentTags] = splitAllMarkup(rawText);
+ // a marker opening the segment leaves the space that followed it behind (see
+ // TranscriptMarkupStripper); this path re-strips the whole accumulation each time,
+ // so trimming the head is idempotent
+ cleanText = cleanText.replace(/^\s+/, '');
+ }
+ if (!cleanText) {
+ return;
+ }
+
// latestText must hold the encoded payload so non-delta flush (FINAL=true) republishes the
// same newline-delimited JSON format as the interim chunks.
- const payload = this.jsonFormat
- ? this.encodeJsonChunk(text)
- : isTimedString(text)
- ? text.text
- : text;
+ const payload = this.encode(cleanText, text);
this.latestText = payload;
- await this.handleCaptureText(payload);
+ await this.publish(payload);
}
- private encodeJsonChunk(text: string | TimedString): string {
- const isTimed = isTimedString(text);
+ private encode(cleanText: string, timingSrc?: string | TimedString): string {
+ if (!this.jsonFormat) {
+ return cleanText;
+ }
+ const isTimed = timingSrc !== undefined && isTimedString(timingSrc);
const message = new pb.TimedString({
- text: isTimed ? text.text : text,
- startTime: isTimed ? text.startTime : undefined,
- endTime: isTimed ? text.endTime : undefined,
- confidence: isTimed ? text.confidence : undefined,
- startTimeOffset: isTimed ? text.startTimeOffset : undefined,
+ text: cleanText,
+ startTime: isTimed ? timingSrc.startTime : undefined,
+ endTime: isTimed ? timingSrc.endTime : undefined,
+ confidence: isTimed ? timingSrc.confidence : undefined,
+ startTimeOffset: isTimed ? timingSrc.startTimeOffset : undefined,
});
return message.toJsonString({ useProtoFieldName: true }) + '\n';
}
- protected async handleCaptureText(text: string): Promise {
- if (this.flushTask && !this.flushTask.done) {
- await this.flushTask.result;
- }
-
- if (!this.capturing) {
- this.resetState();
- this.capturing = true;
- }
-
+ private async publish(payload: string): Promise {
try {
if (this.room.isConnected) {
if (this.isDeltaStream) {
// reuse the existing writer
if (this.writer === null) {
- this.writer = await this.createTextWriter();
+ // Whatever markup was stripped ahead of the first visible text goes on the
+ // opening header — a frontend can't colour the turn until the agent stops
+ // talking otherwise. The instructions ask for a leading expression marker, so
+ // this is normally already populated.
+ //
+ // If the model puts prose before its first expression marker, the tag arrives
+ // after this header and the segment carries no lk.expression: rtc-node's
+ // `TextStreamWriter.close()` takes no attributes, so unlike Python (which
+ // passes them to `aclose()`) there is no trailing header to fall back on. The
+ // same limitation is why the delta path can't send lk.transcription_final
+ // either. Audio and transcript text are unaffected — only the UI hint.
+ this.writer = await this.createTextWriter(
+ undefined,
+ expressionAttribute(this.stripper.tags),
+ );
}
- await this.writer.write(text);
+ await this.writer.write(payload);
} else {
- const tmpWriter = await this.createTextWriter();
- await tmpWriter.write(text);
+ const tmpWriter = await this.createTextWriter(
+ undefined,
+ expressionAttribute(this.segmentTags),
+ );
+ await tmpWriter.write(payload);
await tmpWriter.close();
}
}
@@ -211,13 +282,33 @@ export class ParticipantTranscriptionOutput extends BaseParticipantTranscription
}
}
+ protected async handleCaptureText(_text: string): Promise {
+ // captureText is overridden above; the base implementation is unused here.
+ }
+
protected handleFlush() {
const currWriter = this.writer;
this.writer = null;
- this.flushTask = Task.from((controller) => this.flushTaskImpl(currWriter, controller.signal));
+ const expressive = this.expressiveEnabled();
+ // visible text left in the strip buffer
+ const remaining = expressive && this.isDeltaStream ? this.stripper.flush() : '';
+ const tags = !expressive ? [] : this.isDeltaStream ? this.stripper.tags : this.segmentTags;
+ const pendingText = remaining ? this.encode(remaining) : '';
+ this.flushTask = Task.from((controller) =>
+ this.flushTaskImpl(currWriter, controller.signal, expressionAttribute(tags), pendingText),
+ );
+ }
+
+ protected override resetState() {
+ super.resetState();
+ this.stripper = new TranscriptMarkupStripper();
+ this.segmentTags = [];
}
- private async createTextWriter(attributes?: Record): Promise {
+ private async createTextWriter(
+ attributes?: Record,
+ extra?: Record,
+ ): Promise {
if (!this.participantIdentity) {
throw new Error('participantIdentity not found');
}
@@ -235,6 +326,11 @@ export class ParticipantTranscriptionOutput extends BaseParticipantTranscription
}
}
attributes[ATTRIBUTE_TRANSCRIPTION_SEGMENT_ID] = this.currentId;
+ // overlaid rather than replacing, so the caller can add a key without dropping the
+ // transcription attributes the protocol requires
+ if (extra) {
+ Object.assign(attributes, extra);
+ }
return await this.room.localParticipant.streamText({
topic: TOPIC_TRANSCRIPTION,
@@ -243,13 +339,21 @@ export class ParticipantTranscriptionOutput extends BaseParticipantTranscription
});
}
- private async flushTaskImpl(writer: TextStreamWriter | null, signal: AbortSignal): Promise {
+ private async flushTaskImpl(
+ writer: TextStreamWriter | null,
+ signal: AbortSignal,
+ extraAttributes?: Record,
+ pendingText = '',
+ ): Promise {
const attributes: Record = {
[ATTRIBUTE_TRANSCRIPTION_FINAL]: 'true',
};
if (this.trackId) {
attributes[ATTRIBUTE_TRANSCRIPTION_TRACK_ID] = this.trackId;
}
+ for (const [key, value] of Object.entries(extraAttributes ?? {})) {
+ attributes[key] ??= value;
+ }
const abortPromise = new Promise((resolve) => {
signal.addEventListener('abort', () => resolve());
@@ -258,8 +362,22 @@ export class ParticipantTranscriptionOutput extends BaseParticipantTranscription
try {
if (this.room.isConnected) {
if (this.isDeltaStream) {
- if (writer) {
- await Promise.race([writer.close(), abortPromise]);
+ // a segment whose every chunk was held back by the stripper (a tag-shaped "<"
+ // never resolves) reaches flush with text but no writer — open one here rather
+ // than dropping the transcript
+ let deltaWriter: TextStreamWriter | null = writer;
+ if (!deltaWriter && pendingText) {
+ const opened = await Promise.race([this.createTextWriter(attributes), abortPromise]);
+ if (signal.aborted || !opened) {
+ return;
+ }
+ deltaWriter = opened;
+ }
+ if (deltaWriter) {
+ if (pendingText) {
+ await Promise.race([deltaWriter.write(pendingText), abortPromise]);
+ }
+ await Promise.race([deltaWriter.close(), abortPromise]);
}
} else {
const tmpWriter = await Promise.race([this.createTextWriter(attributes), abortPromise]);
@@ -303,7 +421,18 @@ export class ParticipantLegacyTranscriptionOutput extends BaseParticipantTranscr
this.pushedText = text;
}
- await this.publishTranscription(this.currentId, this.pushedText, false);
+ // pushedText keeps the raw text (markup intact); publish the visible text only.
+ // Stripping the whole accumulation each time avoids partial-tag edge cases; the
+ // expression is dropped here — the deprecated rtc Transcription API has no attribute
+ // channel (the stream-based output carries lk.expression instead).
+ await this.publishTranscription(this.currentId, this.visibleText(), false);
+ }
+
+ /** The raw accumulation, with markup removed only when expressive could have written it. */
+ private visibleText(): string {
+ if (!this.expressiveEnabled()) return this.pushedText;
+ // trimStart: a marker opening the segment leaves the space that followed it behind
+ return stripAllMarkup(this.pushedText).replace(/^\s+/, '');
}
protected handleFlush() {
@@ -311,7 +440,7 @@ export class ParticipantLegacyTranscriptionOutput extends BaseParticipantTranscr
return;
}
- this.flushTask = this.publishTranscription(this.currentId, this.pushedText, true);
+ this.flushTask = this.publishTranscription(this.currentId, this.visibleText(), true);
this.resetState();
}
diff --git a/agents/src/voice/room_io/room_io.ts b/agents/src/voice/room_io/room_io.ts
index b83471ade..08b39416b 100644
--- a/agents/src/voice/room_io/room_io.ts
+++ b/agents/src/voice/room_io/room_io.ts
@@ -393,14 +393,19 @@ export class RoomIO {
isDeltaStream: boolean;
participant: Participant | string | null;
}) {
+ // markup only ever reaches these sinks once expressive has injected its guide; until
+ // then they publish the agent's text verbatim
+ const expressiveEnabled = () => this.agentSession._expressiveEverActive;
return new ParalellTextOutput([
new ParticipantLegacyTranscriptionOutput(
this.room,
options.isDeltaStream,
options.participant,
+ { expressiveEnabled },
),
new ParticipantTranscriptionOutput(this.room, options.isDeltaStream, options.participant, {
jsonFormat: this.outputOptions.jsonFormat,
+ expressiveEnabled,
}),
]);
}
@@ -556,7 +561,11 @@ export class RoomIO {
this.transcriptionSynchronizer = new TranscriptionSynchronizer(
audioOutput,
this.agentTranscriptOutput,
- { ...defaultTextSyncOptions, enabled: !nativeTranscriptSync },
+ {
+ ...defaultTextSyncOptions,
+ enabled: !nativeTranscriptSync,
+ expressiveEnabled: () => this.agentSession._expressiveEverActive,
+ },
);
}
}
diff --git a/agents/src/voice/transcription/synchronizer.ts b/agents/src/voice/transcription/synchronizer.ts
index 005b88727..c5ba9fd04 100644
--- a/agents/src/voice/transcription/synchronizer.ts
+++ b/agents/src/voice/transcription/synchronizer.ts
@@ -7,6 +7,7 @@ import { log } from '../../log.js';
import { IdentityTransform } from '../../stream/identity_transform.js';
import type { WordStream, WordTokenizer } from '../../tokenize/index.js';
import { basic } from '../../tokenize/index.js';
+import { TranscriptMarkupStripper } from '../../tts/provider_format.js';
import { Future, Task, delay } from '../../utils.js';
import {
AudioOutput,
@@ -25,6 +26,12 @@ interface TextSyncOptions {
splitWords: (words: string) => [string, number, number][];
wordTokenizer: WordTokenizer;
enabled: boolean;
+ /**
+ * Whether expressive markup may be present in the forwarded text, so pacing should
+ * discount it. Evaluated per word, because the session latches expressive on the first
+ * turn that injects the markup guide — after this synchronizer is constructed.
+ */
+ expressiveEnabled: () => boolean;
}
interface TextData {
@@ -158,6 +165,14 @@ class SegmentSynchronizerImpl {
private playbackCompleted: boolean = false;
private interrupted: boolean = false;
+ /**
+ * Paces against the visible text only; stateful because a markup tag with spaces in its
+ * attributes (e.g. ``) is shredded across
+ * word tokens and a per-token strip can't recognize the fragments — each would otherwise
+ * be paced as if it were spoken.
+ */
+ private pacingStripper = new TranscriptMarkupStripper();
+
private pausedWallTime?: number;
/** Accumulated paused time in milliseconds; subtracted from wall-clock elapsed. */
private pausedDuration: number = 0;
@@ -449,9 +464,22 @@ class SegmentSynchronizerImpl {
continue;
}
- const cleanWords = this.options.splitWords(word);
- const cleanWord = cleanWords.length > 0 ? cleanWords[0]![0] : word;
- const wordHyphens = this.options.hyphenateWord(cleanWord).length;
+ // forward the raw token (the room output strips markup and surfaces the expression
+ // downstream), but pace against the visible text only so markup adds no delay. The
+ // stripper holds back an unclosed tag across tokens and releases the clean text once
+ // it completes.
+ //
+ // `forwardedWord`, not `word`: the word tokenizer emits whitespace-free runs, so a
+ // tag with spaces in its attributes (``)
+ // would be reassembled without them and no longer match. The forwarded slices are
+ // contiguous over `pushedText`, so feeding those replays the original text exactly.
+ //
+ // Without expressive there is no markup to discount, and the stripper would hold a
+ // tag-shaped "<" in ordinary prose — so it is bypassed entirely.
+ const cleanWord = this.options.expressiveEnabled()
+ ? this.pacingStripper.push(forwardedWord)
+ : forwardedWord;
+ const wordHyphens = cleanWord.trim() ? this.calcHyphens(cleanWord).length : 0;
const elapsedSeconds = this.synchronizedElapsedSeconds()!;
let dHyphens = 0;
@@ -561,6 +589,8 @@ export interface TranscriptionSynchronizerOptions {
splitWords: (words: string) => [string, number, number][];
wordTokenizer: WordTokenizer;
enabled: boolean;
+ /** See {@link TextSyncOptions.expressiveEnabled}. Defaults to "never". */
+ expressiveEnabled?: () => boolean;
}
export const defaultTextSyncOptions: TranscriptionSynchronizerOptions = {
@@ -627,6 +657,7 @@ export class TranscriptionSynchronizer {
splitWords: options.splitWords,
wordTokenizer: options.wordTokenizer,
enabled: options.enabled,
+ expressiveEnabled: options.expressiveEnabled ?? (() => false),
};
// initial segment/first segment, recreated for each new segment
diff --git a/examples/src/expressive-agent/README.md b/examples/src/expressive-agent/README.md
new file mode 100644
index 000000000..13a09b1ea
--- /dev/null
+++ b/examples/src/expressive-agent/README.md
@@ -0,0 +1,54 @@
+# Expressive agent
+
+A free-form voice agent that demonstrates [Expressive Mode](https://docs.livekit.io/agents/build/expressive/).
+There is no task and no tool: you talk to it like a friend, and it matches your
+register. Tell it good news and it gets excited; tell it something went wrong
+and it drops the energy.
+
+Expressive Mode is the single `expressive: true` flag on `AgentSession`. With it
+enabled the framework injects the TTS provider's markup guide into the LLM
+prompt, so the model emits inline delivery tags (emotion, pacing, non-verbal
+sounds) that the TTS renders and the transcript never shows.
+
+## Architecture
+
+- `expressive_agent.ts` is the composition root: session setup and the server entrypoint.
+- `prompt.ts` holds the persona only. It steers _what_ the agent says, and
+ expressive mode owns _how_ it sounds, so the two never restate each other.
+
+The pipeline uses LiveKit Inference with Gemini 2.5 Flash, AssemblyAI Universal
+Streaming, Fish Audio S2.1 Pro, and the LiveKit turn detector.
+
+## Run locally
+
+Provide LiveKit Cloud credentials in the environment (`LIVEKIT_URL`,
+`LIVEKIT_API_KEY`, `LIVEKIT_API_SECRET`), then from the repository root:
+
+```bash
+pnpm build
+node ./examples/src/expressive-agent/expressive_agent.ts dev --log-level=debug
+```
+
+Use `console` instead of `dev` to talk to it in the terminal.
+
+## Trying it with and without expressive
+
+The comparison is the point of the demo. Run it once with `expressive: true` and
+once with `expressive: false`, and say the same thing to each. The words come out
+much the same; the delivery does not.
+
+Expressive Mode requires an `inference.TTS` model that declares a markup
+dialect. Fish Audio, Inworld TTS 2, Cartesia Sonic 3, and xAI qualify; providers
+without a dialect synthesize normally and the flag stays inert. To hear another
+one, swap the `tts` model in `expressive_agent.ts`:
+
+| Provider | Model | Voice |
+| ---------- | ----------------------- | -------------------------------------- |
+| Fish Audio | `fishaudio/s2.1-pro` | `51b44863613e405a896f7f4294c6e6d0` |
+| Inworld | `inworld/inworld-tts-2` | `Ashley` |
+| Cartesia | `cartesia/sonic-3` | `9626c31c-bec5-4cca-baa8-f8ba9e84c8bc` |
+| xAI | `xai/tts-1` | `eve` |
+
+Note that xAI steers delivery through prosody and sound tags but has no
+expression tag, so it publishes no `lk.expression`. Its speech is expressive;
+a frontend mood indicator just has nothing to read.
diff --git a/examples/src/expressive-agent/expressive_agent.ts b/examples/src/expressive-agent/expressive_agent.ts
new file mode 100644
index 000000000..be3d485ef
--- /dev/null
+++ b/examples/src/expressive-agent/expressive_agent.ts
@@ -0,0 +1,52 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import {
+ Agent,
+ AgentSession,
+ type JobContext,
+ ServerOptions,
+ cli,
+ defineAgent,
+ inference,
+} from '@livekit/agents';
+import { fileURLToPath } from 'node:url';
+import { GREETING, INSTRUCTIONS } from './prompt.js';
+
+class Friend extends Agent {
+ constructor() {
+ super({ instructions: INSTRUCTIONS });
+ }
+
+ async onEnter(): Promise {
+ this.session.generateReply({ instructions: GREETING });
+ }
+}
+
+export default defineAgent({
+ entry: async (ctx: JobContext) => {
+ const session = new AgentSession({
+ stt: new inference.STT({ model: 'assemblyai/universal-streaming', language: 'en' }),
+ llm: new inference.LLM({ model: 'google/gemini-2.5-flash' }),
+ tts: new inference.TTS({
+ model: 'fishaudio/s2.1-pro',
+ voice: '51b44863613e405a896f7f4294c6e6d0',
+ }),
+ turnHandling: {
+ turnDetection: new inference.TurnDetector(),
+ interruption: { mode: 'adaptive' },
+ preemptiveGeneration: { enabled: true },
+ },
+ // The single flag. With it enabled the framework injects the TTS provider's markup
+ // guide into the LLM prompt, so the model emits inline delivery tags (emotion,
+ // pacing, non-verbal sounds) that the TTS renders and the transcript never shows.
+ // Flip it to false and say the same things again — the words come out much the
+ // same; the delivery does not.
+ expressive: true,
+ });
+
+ await session.start({ agent: new Friend(), room: ctx.room });
+ },
+});
+
+cli.runApp(new ServerOptions({ agent: fileURLToPath(import.meta.url) }));
diff --git a/examples/src/expressive-agent/prompt.ts b/examples/src/expressive-agent/prompt.ts
new file mode 100644
index 000000000..e83b227f5
--- /dev/null
+++ b/examples/src/expressive-agent/prompt.ts
@@ -0,0 +1,51 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * The persona only. It steers *what* the agent says, and expressive mode owns *how* it
+ * sounds, so the two never restate each other.
+ */
+export const INSTRUCTIONS = `You are the user's closest friend, catching up over a call. There is no task
+here, no ticket to close, no form to fill. You are just talking.
+
+Expressive Mode injects the delivery guide separately, so this prompt covers
+only who you are and what you say. Tone and pacing rules don't belong here, but
+word choice does.
+
+# Output rules
+
+- One or two sentences. Three is already too many.
+- Plain prose. No markdown, lists, bullets, headers, or emojis.
+- Spell out numbers, money, and dates as you would say them out loud.
+- Use contractions. "It's", not "it is"; "you're", not "you are".
+- Never use input vocabulary like "enter" or "fill in". They are speaking, not typing.
+
+# How you talk
+
+- Speak naturally, not from a customer-service script. You are not assisting
+ anyone, you are talking with them.
+- Don't open two consecutive turns with the same word.
+- React before you respond. If they tell you something big, the reaction comes
+ first and the follow-up question comes second.
+- Ask about one thing at a time, the way a friend would, not the way a survey does.
+- Trust their memory. They heard what you said five seconds ago, so don't restate it.
+- If they interrupted you, don't restart the sentence. What they said is the
+ new subject.
+- When they are venting, stay on their side. Advice they didn't ask for is
+ worth less than agreeing that something sucks.
+- Don't reach for a silver lining they didn't ask for, and don't rush to fix
+ what they only wanted to say out loud.
+- Never explain or narrate your own tone.
+
+# Guardrails
+
+- You have no name unless they give you one, and you never introduce yourself
+ by one.
+- You are a friend, not a therapist or a doctor. If they raise something that
+ needs real help, say plainly that you are worried and that this is worth
+ talking to someone about. Don't lecture, and don't pretend to be qualified.`;
+
+export const GREETING =
+ "Open the call the way you'd answer the phone to someone you know well. " +
+ "Short and warm, and leave them room to say what's going on.";
From 210acaa093cef6f14aed654e7f566e0f9d9e5269 Mon Sep 17 00:00:00 2001
From: "rosetta-livekit-bot[bot]"
<282703043+rosetta-livekit-bot[bot]@users.noreply.github.com>
Date: Wed, 12 Aug 2026 16:15:29 +0000
Subject: [PATCH 2/2] examples: switch homepage agent to Fish Audio expressive
mode
---
examples/src/homepage/README.md | 3 ++-
examples/src/homepage/agent.ts | 21 +++++++------------
.../homepage/tests/unit/agent_config.test.ts | 3 ++-
3 files changed, 11 insertions(+), 16 deletions(-)
diff --git a/examples/src/homepage/README.md b/examples/src/homepage/README.md
index 959423a89..6e86e4210 100644
--- a/examples/src/homepage/README.md
+++ b/examples/src/homepage/README.md
@@ -16,7 +16,8 @@ knowledge is loaded on demand through a generated `lookup_product` tool.
- `tests/unit/` is deterministic; `tests/evals/` runs live behavioral evaluations.
The voice pipeline uses LiveKit Inference with Gemma 4 31B, Deepgram Nova-3,
-Inworld TTS, the LiveKit turn detector, and Krisp voice isolation.
+Fish Audio S2.1 Pro in expressive mode, the LiveKit turn detector, and Krisp
+voice isolation.
## Run locally
diff --git a/examples/src/homepage/agent.ts b/examples/src/homepage/agent.ts
index 94892f8a1..8f42a4d1b 100644
--- a/examples/src/homepage/agent.ts
+++ b/examples/src/homepage/agent.ts
@@ -5,19 +5,15 @@ import {
Agent,
AgentSession,
type JobContext,
- type ModelSettings,
ServerOptions,
cli,
defineAgent,
inference,
} from '@livekit/agents';
import * as krisp from '@livekit/agents-plugin-krisp';
-import type { AudioFrame } from '@livekit/rtc-node';
-import type { ReadableStream } from 'node:stream/web';
import { fileURLToPath } from 'node:url';
import { publishFrontendAttributes } from './behaviors/frontend_attributes.js';
import { checkInWhenUserAway } from './behaviors/user_away.js';
-import { pronounceLiveKit } from './filters/pronunciation.js';
import { KnowledgeBase } from './knowledge_base/index.js';
import { prompt } from './prompts/index.js';
@@ -28,14 +24,16 @@ export class AgentConfig {
readonly sttLanguage: string;
readonly ttsModel: string;
readonly ttsVoice: string;
+ readonly ttsVoiceLabel: string;
constructor({
name = 'homepage_agent_v3',
llmModel = 'google/gemma-4-31b-it',
sttModel = 'deepgram/nova-3',
sttLanguage = 'multi',
- ttsModel = 'inworld/inworld-tts-2',
- ttsVoice = 'Nate',
+ ttsModel = 'fishaudio/s2.1-pro',
+ ttsVoice = '51b44863613e405a896f7f4294c6e6d0',
+ ttsVoiceLabel = 'Marley',
}: Partial = {}) {
this.name = name;
this.llmModel = llmModel;
@@ -43,6 +41,7 @@ export class AgentConfig {
this.sttLanguage = sttLanguage;
this.ttsModel = ttsModel;
this.ttsVoice = ttsVoice;
+ this.ttsVoiceLabel = ttsVoiceLabel;
Object.freeze(this);
}
}
@@ -61,13 +60,6 @@ export class Assistant extends Agent {
});
}
- override async ttsNode(
- text: ReadableStream | AsyncIterable,
- modelSettings: ModelSettings,
- ): Promise | null> {
- return Agent.default.ttsNode(this, pronounceLiveKit(text), modelSettings);
- }
-
override async onEnter(): Promise {
await this.session.generateReply({ instructions: GREETING, allowInterruptions: true });
}
@@ -82,6 +74,7 @@ export default defineAgent({
turnDetection: new inference.TurnDetector(),
preemptiveGeneration: { enabled: true },
},
+ expressive: true,
});
checkInWhenUserAway(session);
@@ -91,7 +84,7 @@ export default defineAgent({
room: ctx.room,
inputOptions: { noiseCancellation: krisp.voiceIsolation() },
});
- await publishFrontendAttributes({ ttsVoice: CONFIG.ttsVoice });
+ await publishFrontendAttributes({ ttsVoice: CONFIG.ttsVoiceLabel });
},
});
diff --git a/examples/src/homepage/tests/unit/agent_config.test.ts b/examples/src/homepage/tests/unit/agent_config.test.ts
index 2f9c9a2b2..802db75c2 100644
--- a/examples/src/homepage/tests/unit/agent_config.test.ts
+++ b/examples/src/homepage/tests/unit/agent_config.test.ts
@@ -8,7 +8,8 @@ describe('agent config', () => {
it('is the single source of runtime identity', () => {
expect(new AgentConfig()).toEqual(CONFIG);
expect(CONFIG.name).toBe('homepage_agent_v3');
- expect(CONFIG.ttsVoice).toBe('Nate');
+ expect(CONFIG.ttsModel).toBe('fishaudio/s2.1-pro');
+ expect(CONFIG.ttsVoiceLabel).toBe('Marley');
expect(Object.isFrozen(CONFIG)).toBe(true);
expect(() => Object.assign(CONFIG, { ttsVoice: 'Alex' })).toThrow(TypeError);
});