diff --git a/common/src/main/kotlin/io/homeassistant/companion/android/common/util/tts/TextToSpeechClient.kt b/common/src/main/kotlin/io/homeassistant/companion/android/common/util/tts/TextToSpeechClient.kt index 5c1ef70c22d..0a21ba45ef0 100644 --- a/common/src/main/kotlin/io/homeassistant/companion/android/common/util/tts/TextToSpeechClient.kt +++ b/common/src/main/kotlin/io/homeassistant/companion/android/common/util/tts/TextToSpeechClient.kt @@ -5,14 +5,21 @@ import android.media.AudioAttributes import android.media.AudioManager import android.widget.Toast import androidx.core.content.getSystemService +import androidx.media.AudioAttributesCompat +import androidx.media.AudioFocusRequestCompat +import androidx.media.AudioManagerCompat import io.homeassistant.companion.android.common.R import io.homeassistant.companion.android.common.notifications.NotificationData import java.util.UUID +import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job import kotlinx.coroutines.cancelChildren +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.first import kotlinx.coroutines.launch +import kotlinx.coroutines.withTimeoutOrNull import timber.log.Timber /** @@ -30,6 +37,38 @@ class TextToSpeechClient(private val applicationContext: Context, private val te private val mainScope: CoroutineScope = CoroutineScope(Dispatchers.Main + mainJob) private var isPlaying = false + private var playbackJob: Job? = null + private val hasFocus = MutableStateFlow(false) + private var isTransientLoss = false + private var currentUtterance: Utterance? = null + private var focusRequest: AudioFocusRequestCompat? = null + + private val focusListener = AudioManager.OnAudioFocusChangeListener { focusChange -> + mainScope.launch { + when (focusChange) { + AudioManager.AUDIOFOCUS_GAIN -> { + Timber.d("Audio focus gained") + hasFocus.value = true + if (utteranceQueue.isNotEmpty() && !isPlaying) { + startPlayback() + } + } + AudioManager.AUDIOFOCUS_LOSS -> { + Timber.d("Audio focus lost permanently") + hasFocus.value = false + stopTTS() + } + AudioManager.AUDIOFOCUS_LOSS_TRANSIENT -> { + Timber.d("Audio focus lost temporarily") + hasFocus.value = false + isTransientLoss = true + playbackJob?.cancel() + playbackJob = null + textToSpeechEngine.release() + } + } + } + } /** * Queues a text message to be played back if [data] with a [TextToSpeechData.TTS_TEXT] key is provided. @@ -57,7 +96,7 @@ class TextToSpeechClient(private val applicationContext: Context, private val te ), ) if (!isPlaying) { - play() + startPlayback() } } } @@ -67,18 +106,121 @@ class TextToSpeechClient(private val applicationContext: Context, private val te */ fun stopTTS() { Timber.d("stopped TTS") + playbackJob?.cancel() + playbackJob = null mainJob.cancelChildren() utteranceQueue.clear() textToSpeechEngine.release() + abandonAudioFocus() + hasFocus.value = false isPlaying = false } + private fun startPlayback() { + if (isPlaying) return + isPlaying = true + playbackJob = mainScope.launch { + try { + play() + } finally { + isPlaying = false + } + } + } + + private fun requestAudioFocus(): Boolean { + return try { + val audioManager = applicationContext.getSystemService() ?: return false + + val audioAttributes = utteranceQueue.firstOrNull()?.audioAttributes + val compatAttributes = if (audioAttributes != null) AudioAttributesCompat.wrap(audioAttributes) else null + val usage = if (audioAttributes != + null + ) { + compatAttributes?.usage ?: AudioAttributesCompat.USAGE_MEDIA + } else { + AudioAttributesCompat.USAGE_MEDIA + } + val contentType = if (audioAttributes != + null + ) { + compatAttributes?.contentType ?: AudioAttributesCompat.CONTENT_TYPE_SPEECH + } else { + AudioAttributesCompat.CONTENT_TYPE_SPEECH + } + + val audioAttributesCompat = AudioAttributesCompat.Builder() + .setUsage(usage) + .setContentType(contentType) + .build() + + val request = AudioFocusRequestCompat.Builder(AudioManagerCompat.AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK) + .setAudioAttributes(audioAttributesCompat) + .setOnAudioFocusChangeListener(focusListener) + .build() + + focusRequest = request + + val result = AudioManagerCompat.requestAudioFocus(audioManager, request) + if (result == AudioManager.AUDIOFOCUS_REQUEST_GRANTED) { + hasFocus.value = true + true + } else { + hasFocus.value = false + false + } + } catch (e: Throwable) { + Timber.w(e, "Failed to request audio focus") + hasFocus.value = false + false + } + } + + private fun abandonAudioFocus() { + val audioManager = applicationContext.getSystemService() ?: return + try { + val request = focusRequest + if (request != null) { + AudioManagerCompat.abandonAudioFocusRequest(audioManager, request) + } + } catch (e: Exception) { + Timber.w(e, "Failed to abandon audio focus") + } + focusRequest = null + } + /** * Plays each queued [Utterance] in sequence until [utteranceQueue] is empty. * There can be further additions to the queue while a message is playing which will be picked up in the running playback loop. */ private suspend fun play() { - isPlaying = true + isTransientLoss = false + val focusGranted = requestAudioFocus() + + if (!focusGranted && !hasFocus.value) { + Timber.e("Audio focus request denied") + handleError(applicationContext.getString(R.string.tts_error_focus_denied)) + utteranceQueue.clear() + abandonAudioFocus() + return + } + + if (!hasFocus.value) { + Timber.d("Audio focus not granted yet, waiting...") + val regained = withTimeoutOrNull(FOCUS_TIMEOUT_MS) { + hasFocus.first { it } + true + } ?: false + + if (!regained) { + Timber.e("Timed out waiting for initial audio focus") + handleError(applicationContext.getString(R.string.tts_error_focus_denied)) + utteranceQueue.clear() + abandonAudioFocus() + return + } + } + textToSpeechEngine.initialize().onFailure { throwable -> Timber.e( throwable, @@ -86,20 +228,64 @@ class TextToSpeechClient(private val applicationContext: Context, private val te ) handleError(applicationContext.getString(R.string.tts_error_init)) utteranceQueue.clear() + abandonAudioFocus() }.onSuccess { - while (utteranceQueue.isNotEmpty()) { - utteranceQueue.removeFirst().let { utterance -> + try { + // Over bluetooth connections, the first syllable or even word can be cut off. + // Adding an initial empty utterance seems to fix this. + // Testing shows this is more effective than utilizing the + // textToSpeech.playSilentUtterance method. + if (utteranceQueue.isNotEmpty()) { + utteranceQueue.addFirst( + Utterance( + id = UUID.randomUUID().toString(), + text = " ", + streamVolumeAdjustment = utteranceQueue.first().streamVolumeAdjustment, + audioAttributes = utteranceQueue.first().audioAttributes, + ), + ) + } + while (utteranceQueue.isNotEmpty()) { + if (!hasFocus.value) { + Timber.d("Audio focus lost before playing utterance, waiting...") + val regained = withTimeoutOrNull(FOCUS_TIMEOUT_MS) { + hasFocus.first { it } + true + } ?: false + + if (!regained) { + Timber.e("Timed out waiting to regain audio focus") + handleError(applicationContext.getString(R.string.tts_error_focus_denied)) + utteranceQueue.clear() + abandonAudioFocus() + return@onSuccess + } + } + + val utterance = utteranceQueue.removeFirst() + currentUtterance = utterance textToSpeechEngine.play(utterance).onFailure { throwable -> Timber.e(throwable, "Failed to play utterance '${utterance.id}'") handleError( applicationContext.getString(R.string.tts_error_utterance, utterance.text), ) } + currentUtterance = null + } + } catch (e: CancellationException) { + if (isTransientLoss) { + currentUtterance?.let { + utteranceQueue.addFirst(it) + } + } + throw e + } finally { + textToSpeechEngine.release() + if (utteranceQueue.isEmpty()) { + abandonAudioFocus() } } - textToSpeechEngine.release() } - isPlaying = false } private fun handleError(msg: String) { @@ -113,6 +299,8 @@ class TextToSpeechClient(private val applicationContext: Context, private val te } private companion object { + private const val FOCUS_TIMEOUT_MS = 10000L + private fun getStreamVolumeAdjustment(context: Context, data: Map): StreamVolumeAdjustment { val audioManager = context.getSystemService() return if ( @@ -136,7 +324,10 @@ class TextToSpeechClient(private val applicationContext: Context, private val te .setUsage(AudioAttributes.USAGE_ALARM) .build() } else { - AudioAttributes.Builder().build() + AudioAttributes.Builder() + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .setUsage(AudioAttributes.USAGE_MEDIA) + .build() } } } diff --git a/common/src/main/res/values/strings.xml b/common/src/main/res/values/strings.xml index db6f304c625..c9669efd37b 100644 --- a/common/src/main/res/values/strings.xml +++ b/common/src/main/res/values/strings.xml @@ -958,6 +958,7 @@ Allow this server to enable and manage sensors and send notification commands Unable to process notification \"%1$s\" as text to speech. Failed to initialize a text to speech engine. + Could not obtain audio focus for text to speech. Please set the text for text to speech to process Unknown address Update shortcut data diff --git a/common/src/test/kotlin/io/homeassistant/companion/android/common/util/tts/TextToSpeechClientTest.kt b/common/src/test/kotlin/io/homeassistant/companion/android/common/util/tts/TextToSpeechClientTest.kt new file mode 100644 index 00000000000..3eba84ff268 --- /dev/null +++ b/common/src/test/kotlin/io/homeassistant/companion/android/common/util/tts/TextToSpeechClientTest.kt @@ -0,0 +1,120 @@ +package io.homeassistant.companion.android.common.util.tts + +import android.content.Context +import android.media.AudioManager +import androidx.test.core.app.ApplicationProvider +import dagger.hilt.android.testing.HiltTestApplication +import io.homeassistant.companion.android.testing.unit.MainDispatcherJUnit4Rule +import io.mockk.coEvery +import io.mockk.coVerify +import io.mockk.mockk +import io.mockk.unmockkAll +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.test.runTest +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.Shadows.shadowOf +import org.robolectric.annotation.Config +import org.robolectric.shadows.ShadowToast + +@OptIn(ExperimentalCoroutinesApi::class) +@RunWith(RobolectricTestRunner::class) +@Config(application = HiltTestApplication::class) +class TextToSpeechClientTest { + + @get:Rule + val mainDispatcherRule = MainDispatcherJUnit4Rule() + + private lateinit var context: Context + private lateinit var audioManager: AudioManager + private lateinit var textToSpeechEngine: TextToSpeechEngine + private lateinit var client: TextToSpeechClient + + @Before + fun setUp() { + context = ApplicationProvider.getApplicationContext() + audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager + textToSpeechEngine = mockk(relaxed = true) + client = TextToSpeechClient(context, textToSpeechEngine) + } + + @After + fun tearDown() { + unmockkAll() + } + + @Test + @Config(sdk = [26]) + fun `Given focus granted when speakText on SDK 26 then plays utterance`() = runTest { + coEvery { textToSpeechEngine.initialize() } returns Result.success(Unit) + coEvery { textToSpeechEngine.play(any()) } returns Result.success(Unit) + + client.speakText(mapOf(TextToSpeechData.TTS_TEXT to "Hello World")) + mainDispatcherRule.testDispatcher.scheduler.advanceUntilIdle() + + // Verifies the engine played the text (plus the initial empty space utterance) + coVerify(exactly = 1) { textToSpeechEngine.initialize() } + coVerify(atLeast = 1) { textToSpeechEngine.play(any()) } + } + + @Test + @Config(sdk = [23]) + fun `Given focus granted when speakText on SDK 23 then plays utterance`() = runTest { + coEvery { textToSpeechEngine.initialize() } returns Result.success(Unit) + coEvery { textToSpeechEngine.play(any()) } returns Result.success(Unit) + + client.speakText(mapOf(TextToSpeechData.TTS_TEXT to "Hello World")) + mainDispatcherRule.testDispatcher.scheduler.advanceUntilIdle() + + coVerify(exactly = 1) { textToSpeechEngine.initialize() } + coVerify(atLeast = 1) { textToSpeechEngine.play(any()) } + } + + @Test + @Config(sdk = [26]) + fun `Given focus denied when speakText then shows error toast and aborts`() = runTest { + val shadowAudioManager = shadowOf(audioManager) + shadowAudioManager.setNextFocusRequestResponse(AudioManager.AUDIOFOCUS_REQUEST_FAILED) + + client.speakText(mapOf(TextToSpeechData.TTS_TEXT to "Hello World")) + mainDispatcherRule.testDispatcher.scheduler.advanceUntilIdle() + + coVerify(exactly = 0) { textToSpeechEngine.initialize() } + coVerify(exactly = 0) { textToSpeechEngine.play(any()) } + + val latestToastText = ShadowToast.getTextOfLatestToast() + assertEquals("Could not obtain audio focus for text to speech.", latestToastText) + } + + @Test + @Config(sdk = [26]) + fun `Given engine init fails when speakText then shows error toast`() = runTest { + coEvery { textToSpeechEngine.initialize() } returns Result.failure(Exception("Init error")) + + client.speakText(mapOf(TextToSpeechData.TTS_TEXT to "Hello World")) + mainDispatcherRule.testDispatcher.scheduler.advanceUntilIdle() + + coVerify(exactly = 1) { textToSpeechEngine.initialize() } + coVerify(exactly = 0) { textToSpeechEngine.play(any()) } + + val latestToastText = ShadowToast.getTextOfLatestToast() + assertEquals("Failed to initialize a text to speech engine.", latestToastText) + } + + @Test + @Config(sdk = [26]) + fun `Given playing when stopTTS called then stops and clears queue`() = runTest { + coEvery { textToSpeechEngine.initialize() } returns Result.success(Unit) + + client.speakText(mapOf(TextToSpeechData.TTS_TEXT to "Hello World")) + client.stopTTS() + mainDispatcherRule.testDispatcher.scheduler.advanceUntilIdle() + + coVerify(exactly = 1) { textToSpeechEngine.release() } + } +}