feat: add optional speech-sdk TTS backend (14 cloud providers, BYOK)

.env.example

@@ -78,7 +78,7 @@ OLLAMA_BASE_URL=http://localhost:11434
 ########################################
 # Podcast / TTS
 ########################################
-TTS_PROVIDER=edge               # edge | google | eleven
+TTS_PROVIDER=edge               # edge | google | eleven | speechsdk
 FFMPEG_PATH=ffmpeg
 
 # Edge-tts voices
@@ -95,6 +95,16 @@ GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account.json
 TTS_VOICE_GOOGLE=en-US-Neural2-F
 TTS_VOICE_ALT_GOOGLE=en-US-Neural2-D
 
+# Speech SDK (one backend, 14 cloud TTS providers: openai, elevenlabs, cartesia,
+# hume, deepgram, google, minimax, fish-audio, murf, resemble, fal-ai, mistral,
+# xai, inworld). Uses the matching provider key from your env directly, e.g. the
+# OPENAI_API_KEY above. Optionally set SPEECHBASE_API_KEY (speechbase.ai) to
+# route every provider through that single key instead.
+SPEECH_SDK_MODEL=openai/gpt-4o-mini-tts
+SPEECH_SDK_VOICE_A=alloy
+SPEECH_SDK_VOICE_B=echo
+#SPEECHBASE_API_KEY=
+
 ########################################
 # Voice Transcription
 ########################################

README.md

@@ -95,7 +95,7 @@ The platform provides a modern interface for students, educators, and researcher
 | **Frontend**   | Vite, React, TailwindCSS                 |
 | **Database**   | JSON (default), optional vector DB       |
 | **AI/ML**      | Multiple LLM providers, embeddings       |
-| **Audio**      | Edge TTS, ElevenLabs, Google TTS         |
+| **Audio**      | Edge TTS, ElevenLabs, Google TTS, Speech SDK (OpenAI, Cartesia, Hume, MiniMax + 10 more) |
 | **Deployment** | Docker, Docker Compose                   |
 | **Docs**       | pdf-lib, mammoth, pdf-parse              |
 

backend/src/config/env.ts

@@ -44,6 +44,9 @@ export const config = {
   google_creds: process.env.GOOGLE_APPLICATION_CREDENTIALS || '',
   tts_voice_google: process.env.TTS_VOICE_GOOGLE || 'en-US-Neural2-F',
   tts_voice_alt_google: process.env.TTS_VOICE_ALT_GOOGLE || 'en-US-Neural2-D',
+  speech_sdk_model: process.env.SPEECH_SDK_MODEL || 'openai/gpt-4o-mini-tts',
+  speech_sdk_voice_a: process.env.SPEECH_SDK_VOICE_A || 'alloy',
+  speech_sdk_voice_b: process.env.SPEECH_SDK_VOICE_B || 'echo',
   transcription_provider: process.env.TRANSCRIPTION_PROVIDER || 'openai',
   assemblyai_api_key: process.env.ASSEMBLYAI_API_KEY || '',
   google_project_id: process.env.GOOGLE_CLOUD_PROJECT_ID || '',

backend/src/utils/tts/index.ts

@@ -7,6 +7,36 @@ import { config } from '../../config/env'
 export type TSeg = { text: string; voice?: string }
 export type TSay = (segs: TSeg[], dir: string, base: string, emit?: (m: any) => void) => Promise<string>
 
+// @speech-sdk/core is ESM-only and this backend compiles to CJS, so the import() must survive tsc transpilation
+const dynImport = new Function('s', 'return import(s)') as (s: string) => Promise<any>
+
+let sdkLoad: Promise<{ generateConversation: any; factories: Record<string, any> }> | null = null
+
+function sdk() {
+  if (!sdkLoad) {
+    sdkLoad = Promise.all([dynImport('@speech-sdk/core'), dynImport('@speech-sdk/core/providers')]).then(([core, prov]) => ({
+      generateConversation: core.generateConversation,
+      factories: {
+        cartesia: prov.createCartesia,
+        deepgram: prov.createDeepgram,
+        elevenlabs: prov.createElevenLabs,
+        'fal-ai': prov.createFal,
+        'fish-audio': prov.createFishAudio,
+        google: prov.createGoogle,
+        hume: prov.createHume,
+        inworld: prov.createInworld,
+        minimax: prov.createMiniMax,
+        mistral: prov.createMistral,
+        murf: prov.createMurf,
+        openai: prov.createOpenAI,
+        resemble: prov.createResemble,
+        xai: prov.createXai
+      }
+    }))
+  }
+  return sdkLoad
+}
+
 function ff(dir: string, parts: string[], out: string, emit?: (m: any) => void) {
   return new Promise<string>((res, rej) => {
     const list = path.join(dir, 'list.txt')
@@ -155,15 +185,44 @@ async function synth_google(segs: TSeg[], dir: string, base: string, emit?: (m:
   return await ff(dir, files, out, emit)
 }
 
+function sdk_model(m: string, factories: Record<string, any>) {
+  // with SPEECHBASE_API_KEY set, the bare string routes every provider through the hosted gateway; otherwise call the provider directly with its own env key
+  if (process.env.SPEECHBASE_API_KEY) return m
+  const i = m.indexOf('/')
+  const provider = i === -1 ? m : m.slice(0, i)
+  const modelId = i === -1 ? '' : m.slice(i + 1)
+  const factory = factories[provider]
+  if (!factory) throw new Error(`speechsdk_unknown_provider_${provider}`)
+  return factory()(modelId || undefined)
+}
+
+async function synth_speechsdk(segs: TSeg[], dir: string, base: string, emit?: (m: any) => void) {
+  const { generateConversation, factories } = await sdk()
+  const v0 = config.speech_sdk_voice_a || 'alloy'
+  const v1 = config.speech_sdk_voice_b || 'echo'
+  const model = sdk_model(config.speech_sdk_model || 'openai/gpt-4o-mini-tts', factories)
+  const turns = segs.map((s, i) => ({ text: s.text, voice: s.voice || (i % 2 ? v1 : v0) }))
+
+  // one call renders the whole dialogue: native multi-speaker models when the provider has one, otherwise per-turn synthesis stitched and loudness-normalized (-20 dBFS) by the SDK, so no ffmpeg pass is needed
+  const r = await generateConversation({ model, turns, output: { format: 'mp3' } })
+
+  const out = path.join(dir, `${base}.mp3`)
+  await fs.promises.writeFile(out, r.audio.uint8Array)
+  emit && emit({ type: 'audio_progress', i: segs.length - 1, len: segs.length })
+  return out
+}
+
 export const tts: TSay = async (segs, dir, base, emit) => {
   const p = config.tts_provider || 'edge'
-  
+
   if (p === 'edge') {
     return synth_edge(segs, dir, base, emit)
   } else if (p === 'eleven') {
     return synth_eleven(segs, dir, base, emit)
   } else if (p === 'google') {
     return synth_google(segs, dir, base, emit)
+  } else if (p === 'speechsdk') {
+    return synth_speechsdk(segs, dir, base, emit)
   } else {
     return synth_edge(segs, dir, base, emit)
   }

package.json

@@ -26,6 +26,7 @@
     "@langchain/ollama": "^1.2.7",
     "@langchain/openai": "^1.4.7",
     "@pdf-lib/fontkit": "^1.1.1",
+    "@speech-sdk/core": "^0.15.0",
     "busboy": "^1.6.0",
     "cors": "^2.8.5",
     "d3-force": "^3.0.0",