diff --git a/app/src/main/kotlin/io/homeassistant/companion/android/settings/developer/webrtc/WebRtcDebugFragment.kt b/app/src/main/kotlin/io/homeassistant/companion/android/settings/developer/webrtc/WebRtcDebugFragment.kt index 9f1099a34e0..0e5eb5fc6ad 100644 --- a/app/src/main/kotlin/io/homeassistant/companion/android/settings/developer/webrtc/WebRtcDebugFragment.kt +++ b/app/src/main/kotlin/io/homeassistant/companion/android/settings/developer/webrtc/WebRtcDebugFragment.kt @@ -25,12 +25,15 @@ class WebRtcDebugFragment : Fragment() { HomeAssistantAppTheme { val session by viewModel.session.collectAsStateWithLifecycle() val playerState by viewModel.playerState.collectAsStateWithLifecycle() + val micState by viewModel.micState.collectAsStateWithLifecycle() WebRtcDebugView( player = session, playerState = playerState, + micState = micState, eglContext = viewModel.eglContext, onStart = viewModel::startSession, onStop = viewModel::stopSession, + onMicEnabled = viewModel::setMicEnabled, ) } } diff --git a/app/src/main/kotlin/io/homeassistant/companion/android/settings/developer/webrtc/WebRtcDebugViewModel.kt b/app/src/main/kotlin/io/homeassistant/companion/android/settings/developer/webrtc/WebRtcDebugViewModel.kt index df661399a32..e57c7c8b20b 100644 --- a/app/src/main/kotlin/io/homeassistant/companion/android/settings/developer/webrtc/WebRtcDebugViewModel.kt +++ b/app/src/main/kotlin/io/homeassistant/companion/android/settings/developer/webrtc/WebRtcDebugViewModel.kt @@ -4,7 +4,9 @@ import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope import dagger.hilt.android.lifecycle.HiltViewModel import io.homeassistant.companion.android.common.data.servers.ServerManager +import io.homeassistant.companion.android.webrtc.core.MicState import io.homeassistant.companion.android.webrtc.core.PlayerState +import io.homeassistant.companion.android.webrtc.core.audio.AudioController import io.homeassistant.companion.android.webrtc.core.session.WebRtcSession import io.homeassistant.companion.android.webrtc.core.session.libwebrtc.LibWebRtcPeerConnectionControllerFactory import io.homeassistant.companion.android.webrtc.signaling.HaSignalingClient @@ -31,6 +33,7 @@ private val STATE_SHARING_TIMEOUT = 5.seconds class WebRtcDebugViewModel @Inject constructor( private val serverManager: ServerManager, private val controllerFactory: LibWebRtcPeerConnectionControllerFactory, + private val audioController: AudioController, ) : ViewModel() { private val _session = MutableStateFlow(null) @@ -45,6 +48,15 @@ class WebRtcDebugViewModel @Inject constructor( initialValue = PlayerState.Idle, ) + @OptIn(ExperimentalCoroutinesApi::class) + val micState: StateFlow = _session + .flatMapLatest { session -> session?.micState ?: flowOf(MicState.Off) } + .stateIn( + scope = viewModelScope, + started = SharingStarted.WhileSubscribed(STATE_SHARING_TIMEOUT.inWholeMilliseconds), + initialValue = MicState.Off, + ) + /** EGL context the renderer must share with the hardware decoder. */ val eglContext: EglBase.Context get() = controllerFactory.eglBase.eglBaseContext @@ -59,10 +71,16 @@ class WebRtcDebugViewModel @Inject constructor( entityId = trimmedEntityId, signalingClient = signalingClient, controllerFactory = controllerFactory, + audioController = audioController, ).also { it.start() } } } + /** The caller must hold the `RECORD_AUDIO` permission before enabling the microphone. */ + fun setMicEnabled(enabled: Boolean) { + _session.value?.setMicEnabled(enabled) + } + fun stopSession() { _session.value?.release() _session.value = null diff --git a/app/src/main/kotlin/io/homeassistant/companion/android/settings/developer/webrtc/views/WebRtcDebugView.kt b/app/src/main/kotlin/io/homeassistant/companion/android/settings/developer/webrtc/views/WebRtcDebugView.kt index 15d979c1840..9766cfee0e5 100644 --- a/app/src/main/kotlin/io/homeassistant/companion/android/settings/developer/webrtc/views/WebRtcDebugView.kt +++ b/app/src/main/kotlin/io/homeassistant/companion/android/settings/developer/webrtc/views/WebRtcDebugView.kt @@ -1,5 +1,11 @@ package io.homeassistant.companion.android.settings.developer.webrtc.views +import android.Manifest +import android.content.pm.PackageManager +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.result.contract.ActivityResultContracts +import androidx.compose.foundation.interaction.MutableInteractionSource +import androidx.compose.foundation.interaction.collectIsPressedAsState import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Row @@ -14,18 +20,23 @@ import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Text import androidx.compose.material3.TextField import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember import androidx.compose.runtime.saveable.rememberSaveable import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.graphics.Color +import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.res.stringResource import androidx.compose.ui.unit.dp +import androidx.core.content.ContextCompat import io.homeassistant.companion.android.common.R as commonR import io.homeassistant.companion.android.webrtc.compose.WebRtcVideo import io.homeassistant.companion.android.webrtc.core.CameraPlayer +import io.homeassistant.companion.android.webrtc.core.MicState import io.homeassistant.companion.android.webrtc.core.PlayerFailure import io.homeassistant.companion.android.webrtc.core.PlayerState import livekit.org.webrtc.EglBase @@ -40,9 +51,11 @@ private const val VIDEO_ASPECT_RATIO = 16f / 9f fun WebRtcDebugView( player: CameraPlayer?, playerState: PlayerState, + micState: MicState, eglContext: EglBase.Context?, onStart: (String) -> Unit, onStop: () -> Unit, + onMicEnabled: (Boolean) -> Unit, modifier: Modifier = Modifier, ) { var entityId by rememberSaveable { mutableStateOf("") } @@ -68,10 +81,14 @@ fun WebRtcDebugView( Button(onClick = onStop, enabled = player != null) { Text(stringResource(commonR.string.webrtc_debug_stop)) } + PushToTalkButton( + enabled = player != null, + onMicEnabled = onMicEnabled, + ) } // Raw technical state, on purpose not localized on this developer-only screen Text( - text = playerState.toDebugLabel(), + text = "${playerState.toDebugLabel()} | mic: ${micState.toDebugLabel()}", style = MaterialTheme.typography.bodyMedium, ) player?.let { @@ -94,6 +111,44 @@ fun WebRtcDebugView( } } +/** + * Push-to-talk: the microphone is live only while the button is pressed. The first press requests + * the `RECORD_AUDIO` permission instead of enabling the microphone. + */ +@Composable +private fun PushToTalkButton(enabled: Boolean, onMicEnabled: (Boolean) -> Unit) { + val context = LocalContext.current + var hasMicPermission by remember { + mutableStateOf( + ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == + PackageManager.PERMISSION_GRANTED, + ) + } + val permissionLauncher = rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { granted -> + hasMicPermission = granted + } + + val interactionSource = remember { MutableInteractionSource() } + val isPressed by interactionSource.collectIsPressedAsState() + LaunchedEffect(isPressed, hasMicPermission, enabled) { + onMicEnabled(isPressed && hasMicPermission && enabled) + } + + Button( + onClick = { + if (!hasMicPermission) { + permissionLauncher.launch(Manifest.permission.RECORD_AUDIO) + } + }, + enabled = enabled, + interactionSource = interactionSource, + ) { + Text(stringResource(commonR.string.webrtc_debug_talk)) + } +} + +private fun MicState.toDebugLabel(): String = this::class.simpleName.orEmpty() + private fun PlayerState.toDebugLabel(): String = when (this) { is PlayerState.Failed -> "${this::class.simpleName}: ${failure.toDebugLabel()}" else -> this::class.simpleName.orEmpty() diff --git a/app/src/main/kotlin/io/homeassistant/companion/android/webrtc/WebRtcModule.kt b/app/src/main/kotlin/io/homeassistant/companion/android/webrtc/WebRtcModule.kt index 873731f325e..8a01763abde 100644 --- a/app/src/main/kotlin/io/homeassistant/companion/android/webrtc/WebRtcModule.kt +++ b/app/src/main/kotlin/io/homeassistant/companion/android/webrtc/WebRtcModule.kt @@ -6,6 +6,8 @@ import dagger.Provides import dagger.hilt.InstallIn import dagger.hilt.android.qualifiers.ApplicationContext import dagger.hilt.components.SingletonComponent +import io.homeassistant.companion.android.webrtc.core.audio.AndroidAudioController +import io.homeassistant.companion.android.webrtc.core.audio.AudioController import io.homeassistant.companion.android.webrtc.core.session.PeerConnectionController import io.homeassistant.companion.android.webrtc.core.session.libwebrtc.LibWebRtcPeerConnectionControllerFactory import javax.inject.Singleton @@ -29,4 +31,12 @@ object WebRtcModule { fun providePeerConnectionControllerFactoryInterface( factory: LibWebRtcPeerConnectionControllerFactory, ): PeerConnectionController.Factory = factory + + /** + * A single controller for the whole process: it reference counts the communication audio mode + * across all sessions, so it must be shared to balance correctly. + */ + @Provides + @Singleton + fun provideAudioController(@ApplicationContext context: Context): AudioController = AndroidAudioController(context) } diff --git a/app/src/main/kotlin/io/homeassistant/companion/android/webview/WebViewActivity.kt b/app/src/main/kotlin/io/homeassistant/companion/android/webview/WebViewActivity.kt index 813a9d82a25..10399d29957 100644 --- a/app/src/main/kotlin/io/homeassistant/companion/android/webview/WebViewActivity.kt +++ b/app/src/main/kotlin/io/homeassistant/companion/android/webview/WebViewActivity.kt @@ -158,6 +158,7 @@ import io.homeassistant.companion.android.util.sensitive import io.homeassistant.companion.android.util.toRelativeUrl import io.homeassistant.companion.android.webrtc.core.PlayerFailure import io.homeassistant.companion.android.webrtc.core.PlayerState +import io.homeassistant.companion.android.webrtc.core.audio.AudioController import io.homeassistant.companion.android.webrtc.core.session.WebRtcSession import io.homeassistant.companion.android.webrtc.core.session.libwebrtc.LibWebRtcPeerConnectionControllerFactory import io.homeassistant.companion.android.webrtc.signaling.HaSignalingClient @@ -300,6 +301,9 @@ class WebViewActivity : @Inject lateinit var peerConnectionControllerFactory: LibWebRtcPeerConnectionControllerFactory + @Inject + lateinit var webRtcAudioController: AudioController + private lateinit var webView: WebView private var loadedUrl: Uri? = null private lateinit var decor: FrameLayout @@ -1574,6 +1578,7 @@ class WebViewActivity : entityId = entityId, signalingClient = signalingClient, controllerFactory = peerConnectionControllerFactory, + audioController = webRtcAudioController, ) session.setAudioEnabled(!muted) session.start() diff --git a/common/src/main/res/values/strings.xml b/common/src/main/res/values/strings.xml index 048a9ba166b..bfc5a71f1a4 100644 --- a/common/src/main/res/values/strings.xml +++ b/common/src/main/res/values/strings.xml @@ -981,6 +981,7 @@ Start Stop Test the native WebRTC player against a camera entity of the active server + Hold to talk Native WebRTC player (beta) Let dashboards play camera WebRTC streams with the native player instead of the WebView Connected to Home Assistant diff --git a/webrtc-compose/gradle.lockfile b/webrtc-compose/gradle.lockfile index 94198de98cd..aa2374667b0 100644 --- a/webrtc-compose/gradle.lockfile +++ b/webrtc-compose/gradle.lockfile @@ -100,6 +100,7 @@ androidx.lifecycle:lifecycle-viewmodel-savedstate:2.11.0=debugAndroidTestCompile androidx.lifecycle:lifecycle-viewmodel:2.11.0=debugAndroidTestCompileClasspath,debugAndroidTestLintChecksClasspath,debugAndroidTestRuntimeClasspath,debugCompileClasspath,debugLintChecksClasspath,debugRuntimeClasspath,debugScreenshotTestCompileClasspath,debugScreenshotTestLintChecksClasspath,debugScreenshotTestRuntimeClasspath,debugUnitTestCompileClasspath,debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath,releaseCompileClasspath,releaseLintChecksClasspath,releaseRuntimeClasspath,releaseScreenshotTestCompileClasspath,releaseScreenshotTestLintChecksClasspath,releaseScreenshotTestRuntimeClasspath androidx.loader:loader:1.0.0=debugAndroidTestCompileClasspath,debugAndroidTestLintChecksClasspath,debugAndroidTestRuntimeClasspath,debugCompileClasspath,debugLintChecksClasspath,debugRuntimeClasspath,debugScreenshotTestCompileClasspath,debugScreenshotTestLintChecksClasspath,debugScreenshotTestRuntimeClasspath,debugUnitTestCompileClasspath,debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath,releaseCompileClasspath,releaseLintChecksClasspath,releaseRuntimeClasspath,releaseScreenshotTestCompileClasspath,releaseScreenshotTestLintChecksClasspath,releaseScreenshotTestRuntimeClasspath androidx.localbroadcastmanager:localbroadcastmanager:1.0.0=debugAndroidTestLintChecksClasspath,debugAndroidTestRuntimeClasspath,debugLintChecksClasspath,debugRuntimeClasspath,debugScreenshotTestLintChecksClasspath,debugScreenshotTestRuntimeClasspath,debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath,releaseLintChecksClasspath,releaseRuntimeClasspath,releaseScreenshotTestLintChecksClasspath,releaseScreenshotTestRuntimeClasspath +androidx.media:media:1.8.0=debugAndroidTestLintChecksClasspath,debugAndroidTestRuntimeClasspath,debugLintChecksClasspath,debugRuntimeClasspath,debugScreenshotTestLintChecksClasspath,debugScreenshotTestRuntimeClasspath,debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath,releaseLintChecksClasspath,releaseRuntimeClasspath,releaseScreenshotTestLintChecksClasspath,releaseScreenshotTestRuntimeClasspath androidx.print:print:1.0.0=debugAndroidTestLintChecksClasspath,debugAndroidTestRuntimeClasspath,debugLintChecksClasspath,debugRuntimeClasspath,debugScreenshotTestLintChecksClasspath,debugScreenshotTestRuntimeClasspath,debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath,releaseLintChecksClasspath,releaseRuntimeClasspath,releaseScreenshotTestLintChecksClasspath,releaseScreenshotTestRuntimeClasspath androidx.profileinstaller:profileinstaller:1.4.0=debugAndroidTestLintChecksClasspath,debugAndroidTestRuntimeClasspath,debugLintChecksClasspath,debugRuntimeClasspath,debugScreenshotTestLintChecksClasspath,debugScreenshotTestRuntimeClasspath,debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath,releaseLintChecksClasspath,releaseRuntimeClasspath,releaseScreenshotTestLintChecksClasspath,releaseScreenshotTestRuntimeClasspath androidx.savedstate:savedstate-android:1.4.0=debugAndroidTestCompileClasspath,debugAndroidTestLintChecksClasspath,debugAndroidTestRuntimeClasspath,debugCompileClasspath,debugLintChecksClasspath,debugRuntimeClasspath,debugScreenshotTestCompileClasspath,debugScreenshotTestLintChecksClasspath,debugScreenshotTestRuntimeClasspath,debugUnitTestCompileClasspath,debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath,releaseCompileClasspath,releaseLintChecksClasspath,releaseRuntimeClasspath,releaseScreenshotTestCompileClasspath,releaseScreenshotTestLintChecksClasspath,releaseScreenshotTestRuntimeClasspath diff --git a/webrtc-core/build.gradle.kts b/webrtc-core/build.gradle.kts index 4e26ef60931..5aa34a7c91a 100644 --- a/webrtc-core/build.gradle.kts +++ b/webrtc-core/build.gradle.kts @@ -11,10 +11,14 @@ dependencies { implementation(libs.kotlin.stdlib) implementation(libs.kotlinx.coroutines.core) + // AudioFocusRequestCompat, the audio focus API compatible with the min SDK + implementation(libs.androidx.media) + // This module is the only one allowed to depend on libwebrtc directly so the artifact can be // swapped without touching consumers. `api` because VideoSink is part of the public player // interfaces used by renderers. api(libs.webrtc.sdk) testImplementation(libs.junit.jupiter.params) + testImplementation(libs.androidx.test.core) } diff --git a/webrtc-core/gradle.lockfile b/webrtc-core/gradle.lockfile index 8fd70fdd9d2..2f593c9a771 100644 --- a/webrtc-core/gradle.lockfile +++ b/webrtc-core/gradle.lockfile @@ -45,7 +45,9 @@ androidx.compose.ui:ui-util-android:1.11.4=debugUnitTestLintChecksClasspath,debu androidx.compose.ui:ui-util:1.11.4=debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath androidx.compose.ui:ui:1.11.4=debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath androidx.compose:compose-bom:2026.06.01=debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath -androidx.concurrent:concurrent-futures:1.1.0=debugAndroidTestLintChecksClasspath,debugAndroidTestRuntimeClasspath,debugLintChecksClasspath,debugRuntimeClasspath,debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath,releaseLintChecksClasspath,releaseRuntimeClasspath +androidx.concurrent:concurrent-futures-ktx:1.2.0=debugUnitTestCompileClasspath,debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath +androidx.concurrent:concurrent-futures:1.1.0=debugAndroidTestLintChecksClasspath,debugAndroidTestRuntimeClasspath,debugLintChecksClasspath,debugRuntimeClasspath,releaseLintChecksClasspath,releaseRuntimeClasspath +androidx.concurrent:concurrent-futures:1.2.0=debugUnitTestCompileClasspath,debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath androidx.core:core-ktx:1.19.0=debugAndroidTestCompileClasspath,debugAndroidTestLintChecksClasspath,debugAndroidTestRuntimeClasspath,debugCompileClasspath,debugLintChecksClasspath,debugRuntimeClasspath,debugUnitTestCompileClasspath,debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath,releaseCompileClasspath,releaseLintChecksClasspath,releaseRuntimeClasspath androidx.core:core-viewtree:1.0.0=debugAndroidTestCompileClasspath,debugAndroidTestLintChecksClasspath,debugAndroidTestRuntimeClasspath,debugCompileClasspath,debugLintChecksClasspath,debugRuntimeClasspath,debugUnitTestCompileClasspath,debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath,releaseCompileClasspath,releaseLintChecksClasspath,releaseRuntimeClasspath androidx.core:core:1.19.0=debugAndroidTestCompileClasspath,debugAndroidTestLintChecksClasspath,debugAndroidTestRuntimeClasspath,debugCompileClasspath,debugLintChecksClasspath,debugRuntimeClasspath,debugUnitTestCompileClasspath,debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath,releaseCompileClasspath,releaseLintChecksClasspath,releaseRuntimeClasspath @@ -83,6 +85,7 @@ androidx.lifecycle:lifecycle-viewmodel:2.6.2=debugAndroidTestCompileClasspath,de androidx.lifecycle:lifecycle-viewmodel:2.9.4=debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath androidx.loader:loader:1.0.0=debugAndroidTestCompileClasspath,debugAndroidTestLintChecksClasspath,debugAndroidTestRuntimeClasspath,debugCompileClasspath,debugLintChecksClasspath,debugRuntimeClasspath,debugUnitTestCompileClasspath,debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath,releaseCompileClasspath,releaseLintChecksClasspath,releaseRuntimeClasspath androidx.localbroadcastmanager:localbroadcastmanager:1.0.0=debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath +androidx.media:media:1.8.0=debugAndroidTestCompileClasspath,debugAndroidTestLintChecksClasspath,debugAndroidTestRuntimeClasspath,debugCompileClasspath,debugLintChecksClasspath,debugRuntimeClasspath,debugUnitTestCompileClasspath,debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath,releaseCompileClasspath,releaseLintChecksClasspath,releaseRuntimeClasspath androidx.print:print:1.0.0=debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath androidx.profileinstaller:profileinstaller:1.3.0=debugAndroidTestLintChecksClasspath,debugAndroidTestRuntimeClasspath,debugLintChecksClasspath,debugRuntimeClasspath,releaseLintChecksClasspath,releaseRuntimeClasspath androidx.profileinstaller:profileinstaller:1.4.0=debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath @@ -98,8 +101,7 @@ androidx.test.espresso:espresso-idling-resource:3.7.0=debugUnitTestLintChecksCla androidx.test.ext:junit:1.1.5=debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath androidx.test.services:storage:1.4.2=debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath androidx.test:annotation:1.0.1=debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath -androidx.test:core:1.4.0=debugUnitTestCompileClasspath -androidx.test:core:1.5.0=debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath +androidx.test:core:1.7.0=debugUnitTestCompileClasspath,debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath androidx.test:monitor:1.8.0=debugUnitTestCompileClasspath,debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath androidx.test:runner:1.5.0=debugUnitTestLintChecksClasspath,debugUnitTestRuntimeClasspath androidx.tracing:tracing:1.1.0=debugUnitTestCompileClasspath diff --git a/webrtc-core/src/main/kotlin/io/homeassistant/companion/android/webrtc/core/audio/AudioController.kt b/webrtc-core/src/main/kotlin/io/homeassistant/companion/android/webrtc/core/audio/AudioController.kt new file mode 100644 index 00000000000..b4382de6b08 --- /dev/null +++ b/webrtc-core/src/main/kotlin/io/homeassistant/companion/android/webrtc/core/audio/AudioController.kt @@ -0,0 +1,89 @@ +package io.homeassistant.companion.android.webrtc.core.audio + +import android.content.Context +import android.media.AudioManager +import androidx.media.AudioAttributesCompat +import androidx.media.AudioFocusRequestCompat +import androidx.media.AudioManagerCompat +import timber.log.Timber + +/** + * Puts the device audio stack in and out of communication mode while a microphone is live. + * + * [WebRtcSession][io.homeassistant.companion.android.webrtc.core.session.WebRtcSession] drives + * this structurally: it acquires when the microphone track starts sending and guarantees the + * release on every exit path (microphone off, stop, failure, disposal), so consumers cannot leak + * the communication audio mode. + */ +interface AudioController { + /** + * Enter communication mode. Balanced by [release]; implementations must support concurrent + * holders (reference counting). + */ + fun acquire() + + /** Leave communication mode once all holders released it, restoring the previous state. */ + fun release() + + /** + * Controller that leaves the audio stack untouched, for consumers that do not send + * microphone audio and for tests. + */ + object None : AudioController { + override fun acquire() {} + override fun release() {} + } +} + +/** + * [AudioController] backed by [AudioManager]: while held, the audio mode is + * [AudioManager.MODE_IN_COMMUNICATION] (enabling the platform echo cancellation path used for + * calls) and transient audio focus for voice communication is taken. On the last [release] the + * focus is abandoned and the previous audio mode restored. + */ +class AndroidAudioController(context: Context) : AudioController { + + private val audioManager = context.applicationContext.getSystemService(Context.AUDIO_SERVICE) as AudioManager + + private val lock = Any() + private var holders = 0 + private var previousAudioMode = AudioManager.MODE_NORMAL + private var focusRequest: AudioFocusRequestCompat? = null + + override fun acquire() { + synchronized(lock) { + holders++ + if (holders > 1) return + + previousAudioMode = audioManager.mode + audioManager.mode = AudioManager.MODE_IN_COMMUNICATION + val request = AudioFocusRequestCompat.Builder(AudioManagerCompat.AUDIOFOCUS_GAIN_TRANSIENT) + .setAudioAttributes( + AudioAttributesCompat.Builder() + .setUsage(AudioAttributesCompat.USAGE_VOICE_COMMUNICATION) + .setContentType(AudioAttributesCompat.CONTENT_TYPE_SPEECH) + .build(), + ) + .setOnAudioFocusChangeListener { + // The stream must keep running to not drop the camera session, there is + // nothing sensible to pause on focus loss + Timber.d("WebRTC audio focus changed: $it") + } + .build() + focusRequest = request + AudioManagerCompat.requestAudioFocus(audioManager, request) + } + } + + override fun release() { + synchronized(lock) { + if (holders == 0) return + holders-- + if (holders > 0) return + + focusRequest?.let { AudioManagerCompat.abandonAudioFocusRequest(audioManager, it) } + focusRequest = null + audioManager.mode = previousAudioMode + } + } +} diff --git a/webrtc-core/src/main/kotlin/io/homeassistant/companion/android/webrtc/core/session/WebRtcSession.kt b/webrtc-core/src/main/kotlin/io/homeassistant/companion/android/webrtc/core/session/WebRtcSession.kt index b7d114dcf16..08fc8bc4644 100644 --- a/webrtc-core/src/main/kotlin/io/homeassistant/companion/android/webrtc/core/session/WebRtcSession.kt +++ b/webrtc-core/src/main/kotlin/io/homeassistant/companion/android/webrtc/core/session/WebRtcSession.kt @@ -5,6 +5,7 @@ import io.homeassistant.companion.android.webrtc.core.MicState import io.homeassistant.companion.android.webrtc.core.PlayerFailure import io.homeassistant.companion.android.webrtc.core.PlayerState import io.homeassistant.companion.android.webrtc.core.TwoWayAudio +import io.homeassistant.companion.android.webrtc.core.audio.AudioController import io.homeassistant.companion.android.webrtc.core.signaling.IceCandidateInit import io.homeassistant.companion.android.webrtc.core.signaling.SignalingClient import io.homeassistant.companion.android.webrtc.core.signaling.SignalingEvent @@ -54,6 +55,8 @@ private val RECONNECT_BASE_DELAY = 2.seconds * @param entityId the camera entity to stream * @param signalingClient the signaling backend, scoped to the right server * @param controllerFactory creates one peer connection per negotiation + * @param audioController drives the device audio mode while the microphone is live; the session + * guarantees the release on every exit path * @param dispatcher dispatcher running the session state machine, it must be serial (the default * already is) */ @@ -61,6 +64,7 @@ class WebRtcSession( private val entityId: String, private val signalingClient: SignalingClient, private val controllerFactory: PeerConnectionController.Factory, + private val audioController: AudioController = AudioController.None, dispatcher: CoroutineDispatcher = Dispatchers.Default.limitedParallelism(1), ) : CameraPlayer, TwoWayAudio { @@ -81,6 +85,9 @@ class WebRtcSession( @Volatile private var micEnabled = false + @Volatile + private var audioAcquired = false + @Volatile private var audioEnabled = true @@ -105,6 +112,7 @@ class WebRtcSession( // The job reference is kept so a subsequent start() can await the cleanup sessionJob?.cancel() micEnabled = false + releaseAudio() _micState.value = MicState.Off _state.value = PlayerState.Idle } @@ -141,16 +149,32 @@ class WebRtcSession( val activeController = controller if (!enabled) { activeController?.setMicrophoneEnabled(false) + releaseAudio() _micState.value = MicState.Off } else if (activeController != null) { - activeController.setMicrophoneEnabled(true) - _micState.value = MicState.Live + enableMicrophone(activeController) } else { // Remembered and applied when the session (re)connects _micState.value = MicState.Unavailable } } + private fun enableMicrophone(controller: PeerConnectionController) { + controller.setMicrophoneEnabled(true) + if (!audioAcquired) { + audioAcquired = true + audioController.acquire() + } + _micState.value = MicState.Live + } + + private fun releaseAudio() { + if (audioAcquired) { + audioAcquired = false + audioController.release() + } + } + private suspend fun runSession() { var attempt = 0 while (true) { @@ -194,8 +218,7 @@ class WebRtcSession( controller.setRemoteAudioEnabled(audioEnabled) val offerSdp = controller.createOffer() if (micEnabled) { - controller.setMicrophoneEnabled(true) - _micState.value = MicState.Live + enableMicrophone(controller) } var sessionId: String? = null @@ -301,6 +324,7 @@ class WebRtcSession( private fun failWith(failure: PlayerFailure) { _state.value = PlayerState.Failed(failure) + releaseAudio() _micState.value = MicState.Off } diff --git a/webrtc-core/src/main/kotlin/io/homeassistant/companion/android/webrtc/core/session/libwebrtc/LibWebRtcPeerConnectionController.kt b/webrtc-core/src/main/kotlin/io/homeassistant/companion/android/webrtc/core/session/libwebrtc/LibWebRtcPeerConnectionController.kt index 94773578768..3ea8ef95316 100644 --- a/webrtc-core/src/main/kotlin/io/homeassistant/companion/android/webrtc/core/session/libwebrtc/LibWebRtcPeerConnectionController.kt +++ b/webrtc-core/src/main/kotlin/io/homeassistant/companion/android/webrtc/core/session/libwebrtc/LibWebRtcPeerConnectionController.kt @@ -13,6 +13,7 @@ import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.receiveAsFlow import kotlinx.coroutines.suspendCancellableCoroutine +import livekit.org.webrtc.AudioSource import livekit.org.webrtc.AudioTrack import livekit.org.webrtc.DataChannel import livekit.org.webrtc.IceCandidate @@ -35,10 +36,14 @@ private const val MICROPHONE_TRACK_ID = "ha_microphone" * [PeerConnectionController] implementation backed by libwebrtc. * * The peer connection is created with a receive-only video transceiver and a send-and-receive - * audio transceiver whose microphone track starts disabled, so talk-back can be toggled later - * without renegotiating. + * audio transceiver that is negotiated up front but has no track: the microphone capture is only + * created and attached (via `RtpSender.setTrack`, which does not renegotiate) the first time the + * microphone is enabled. Creating the capture eagerly would start the platform audio record + * pipeline for every session — libwebrtc supports only one capture at a time, so a session that + * never uses the microphone would break talk-back for every session after it, and the microphone + * privacy indicator would show without the microphone being used. */ -internal class LibWebRtcPeerConnectionController(factory: PeerConnectionFactory, config: RtcClientConfig) : +internal class LibWebRtcPeerConnectionController(private val factory: PeerConnectionFactory, config: RtcClientConfig) : PeerConnectionController { private val eventsChannel = Channel(Channel.UNLIMITED) @@ -106,11 +111,15 @@ internal class LibWebRtcPeerConnectionController(factory: PeerConnectionFactory, private val peerConnection = checkNotNull(factory.createPeerConnection(config.toRtcConfiguration(), observer)) { "PeerConnection could not be created" } - private val audioSource = factory.createAudioSource(MediaConstraints()) - private val microphoneTrack = factory.createAudioTrack(MICROPHONE_TRACK_ID, audioSource).apply { - setEnabled(false) - } + + @Volatile + private var audioSource: AudioSource? = null + + @Volatile + private var microphoneTrack: AudioTrack? = null + private var dataChannel: DataChannel? = null + private val audioTransceiver: RtpTransceiver init { // The data channel (when the provider uses one, like go2rtc) and the transceivers must @@ -122,8 +131,8 @@ internal class LibWebRtcPeerConnectionController(factory: PeerConnectionFactory, MediaStreamTrack.MediaType.MEDIA_TYPE_VIDEO, RtpTransceiver.RtpTransceiverInit(RtpTransceiver.RtpTransceiverDirection.RECV_ONLY), ) - peerConnection.addTransceiver( - microphoneTrack, + audioTransceiver = peerConnection.addTransceiver( + MediaStreamTrack.MediaType.MEDIA_TYPE_AUDIO, RtpTransceiver.RtpTransceiverInit(RtpTransceiver.RtpTransceiverDirection.SEND_RECV), ) } @@ -190,7 +199,22 @@ internal class LibWebRtcPeerConnectionController(factory: PeerConnectionFactory, ) override fun setMicrophoneEnabled(enabled: Boolean) { - microphoneTrack.setEnabled(enabled) + if (!enabled) { + microphoneTrack?.setEnabled(false) + return + } + if (disposed.get()) return + val track = microphoneTrack ?: run { + // First use: create the capture now and attach it to the negotiated transceiver. + // setTrack does not renegotiate, so the session stays untouched. + val source = factory.createAudioSource(MediaConstraints()) + val newTrack = factory.createAudioTrack(MICROPHONE_TRACK_ID, source) + audioSource = source + microphoneTrack = newTrack + audioTransceiver.sender.setTrack(newTrack, false) + newTrack + } + track.setEnabled(true) } override fun setRemoteAudioEnabled(enabled: Boolean) { @@ -223,11 +247,14 @@ internal class LibWebRtcPeerConnectionController(factory: PeerConnectionFactory, remoteAudioTrack = null dataChannel?.dispose() dataChannel = null - // Disposal order matters: peer connection (closes transports and its receivers/senders), - // then our local track wrapper, then its source + // Disposal order matters: peer connection (closes transports and its receivers/senders, + // which releases the platform audio capture), then our local track wrapper, then its + // source peerConnection.dispose() - microphoneTrack.dispose() - audioSource.dispose() + microphoneTrack?.dispose() + microphoneTrack = null + audioSource?.dispose() + audioSource = null eventsChannel.close() } } diff --git a/webrtc-core/src/test/kotlin/io/homeassistant/companion/android/webrtc/core/audio/AndroidAudioControllerTest.kt b/webrtc-core/src/test/kotlin/io/homeassistant/companion/android/webrtc/core/audio/AndroidAudioControllerTest.kt new file mode 100644 index 00000000000..8dbeee57ac5 --- /dev/null +++ b/webrtc-core/src/test/kotlin/io/homeassistant/companion/android/webrtc/core/audio/AndroidAudioControllerTest.kt @@ -0,0 +1,80 @@ +package io.homeassistant.companion.android.webrtc.core.audio + +import android.content.Context +import android.media.AudioManager +import androidx.test.core.app.ApplicationProvider +import org.junit.Before +import org.junit.Test +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.Shadows.shadowOf + +@RunWith(RobolectricTestRunner::class) +class AndroidAudioControllerTest { + + private lateinit var audioManager: AudioManager + private lateinit var controller: AndroidAudioController + + @Before + fun setUp() { + val context = ApplicationProvider.getApplicationContext() + audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager + controller = AndroidAudioController(context) + } + + @Test + fun `Given a normal audio mode When acquiring Then communication mode is set and focus requested`() { + controller.acquire() + + assertEquals(AudioManager.MODE_IN_COMMUNICATION, audioManager.mode) + assertNotNull(shadowOf(audioManager).lastAudioFocusRequest) + } + + @Test + fun `Given a held controller When releasing Then the previous mode is restored and focus abandoned`() { + audioManager.mode = AudioManager.MODE_RINGTONE + controller.acquire() + + controller.release() + + assertEquals(AudioManager.MODE_RINGTONE, audioManager.mode) + assertNotNull(shadowOf(audioManager).lastAbandonedAudioFocusRequest) + } + + @Test + fun `Given two holders When one releases Then communication mode is kept until the last release`() { + controller.acquire() + controller.acquire() + + controller.release() + assertEquals(AudioManager.MODE_IN_COMMUNICATION, audioManager.mode) + assertNull(shadowOf(audioManager).lastAbandonedAudioFocusRequest) + + controller.release() + assertEquals(AudioManager.MODE_NORMAL, audioManager.mode) + assertNotNull(shadowOf(audioManager).lastAbandonedAudioFocusRequest) + } + + @Test + fun `Given no holder When releasing Then nothing happens`() { + audioManager.mode = AudioManager.MODE_RINGTONE + + controller.release() + + assertEquals(AudioManager.MODE_RINGTONE, audioManager.mode) + assertNull(shadowOf(audioManager).lastAbandonedAudioFocusRequest) + } + + @Test + fun `Given a fully released controller When acquiring again Then communication mode is entered again`() { + controller.acquire() + controller.release() + + controller.acquire() + + assertEquals(AudioManager.MODE_IN_COMMUNICATION, audioManager.mode) + } +} diff --git a/webrtc-core/src/test/kotlin/io/homeassistant/companion/android/webrtc/core/session/Fakes.kt b/webrtc-core/src/test/kotlin/io/homeassistant/companion/android/webrtc/core/session/Fakes.kt index 1f3846b098d..c5461293c98 100644 --- a/webrtc-core/src/test/kotlin/io/homeassistant/companion/android/webrtc/core/session/Fakes.kt +++ b/webrtc-core/src/test/kotlin/io/homeassistant/companion/android/webrtc/core/session/Fakes.kt @@ -1,5 +1,6 @@ package io.homeassistant.companion.android.webrtc.core.session +import io.homeassistant.companion.android.webrtc.core.audio.AudioController import io.homeassistant.companion.android.webrtc.core.signaling.IceCandidateInit import io.homeassistant.companion.android.webrtc.core.signaling.RtcClientConfig import io.homeassistant.companion.android.webrtc.core.signaling.SignalingClient @@ -111,6 +112,24 @@ internal class FakePeerConnectionController(private val offerSdp: String) : Peer } } +internal class FakeAudioController : AudioController { + + var acquireCount = 0 + var releaseCount = 0 + + /** How many acquisitions are currently unbalanced by a release. */ + val activeHolds: Int + get() = acquireCount - releaseCount + + override fun acquire() { + acquireCount++ + } + + override fun release() { + releaseCount++ + } +} + internal class FakePeerConnectionControllerFactory : PeerConnectionController.Factory { val controllers = mutableListOf() diff --git a/webrtc-core/src/test/kotlin/io/homeassistant/companion/android/webrtc/core/session/WebRtcSessionTest.kt b/webrtc-core/src/test/kotlin/io/homeassistant/companion/android/webrtc/core/session/WebRtcSessionTest.kt index a52085f3093..42a47acc1e7 100644 --- a/webrtc-core/src/test/kotlin/io/homeassistant/companion/android/webrtc/core/session/WebRtcSessionTest.kt +++ b/webrtc-core/src/test/kotlin/io/homeassistant/companion/android/webrtc/core/session/WebRtcSessionTest.kt @@ -26,8 +26,15 @@ class WebRtcSessionTest { private val signaling = FakeSignalingClient() private val factory = FakePeerConnectionControllerFactory() + private val audio = FakeAudioController() - private fun TestScope.createSession() = WebRtcSession(ENTITY_ID, signaling, factory, StandardTestDispatcher(testScheduler)) + private fun TestScope.createSession() = WebRtcSession( + ENTITY_ID, + signaling, + factory, + audio, + StandardTestDispatcher(testScheduler), + ) private fun TestScope.startConnectedSession(session: WebRtcSession) { session.start() @@ -322,6 +329,104 @@ class WebRtcSessionTest { assertEquals(MicState.Off, session.micState.value) } + @Test + fun `Given a mic request When the session is not connected yet Then no audio mode is acquired`() = runTest { + val session = createSession() + + session.setMicEnabled(true) + + assertEquals(0, audio.acquireCount) + + startConnectedSession(session) + + assertEquals(1, audio.acquireCount) + } + + @Test + fun `Given a live mic When toggling it Then the audio mode is acquired and released exactly once`() = runTest { + val session = createSession() + startConnectedSession(session) + + session.setMicEnabled(true) + assertEquals(1, audio.acquireCount) + // Enabling again while live must not stack another hold + session.setMicEnabled(true) + assertEquals(1, audio.acquireCount) + + session.setMicEnabled(false) + assertEquals(0, audio.activeHolds) + // Disabling again must not over-release + session.setMicEnabled(false) + assertEquals(1, audio.releaseCount) + } + + @Test + fun `Given a live mic When the session reconnects Then the audio mode is held through the reconnection`() = runTest { + val session = createSession() + startConnectedSession(session) + session.setMicEnabled(true) + + factory.lastController.emit(PeerConnectionEvent.ConnectionStateChanged(RtcConnectionState.FAILED)) + advanceUntilIdle() + + assertEquals(2, factory.controllers.size) + assertEquals(1, audio.acquireCount) + assertEquals(1, audio.activeHolds) + } + + @Test + fun `Given a live mic When the session stops Then the audio mode is released`() = runTest { + val session = createSession() + startConnectedSession(session) + session.setMicEnabled(true) + + session.stop() + advanceUntilIdle() + + assertEquals(0, audio.activeHolds) + } + + @Test + fun `Given a live mic When the session is released Then the audio mode is released`() = runTest { + val session = createSession() + startConnectedSession(session) + session.setMicEnabled(true) + + session.release() + advanceUntilIdle() + + assertEquals(0, audio.activeHolds) + } + + @Test + fun `Given a live mic When signaling fails Then the audio mode is released`() = runTest { + val session = createSession() + startConnectedSession(session) + session.setMicEnabled(true) + + signaling.currentSession.trySend(SignalingEvent.Error(code = "webrtc_offer_failed", message = null)) + advanceUntilIdle() + + assertTrue(session.state.value is PlayerState.Failed) + assertEquals(MicState.Off, session.micState.value) + assertEquals(0, audio.activeHolds) + } + + @Test + fun `Given a live mic When the connection is lost for good Then the audio mode is released`() = runTest { + val session = createSession() + startConnectedSession(session) + session.setMicEnabled(true) + + repeat(4) { + factory.lastController.emit(PeerConnectionEvent.ConnectionStateChanged(RtcConnectionState.FAILED)) + advanceUntilIdle() + } + + assertEquals(PlayerState.Failed(PlayerFailure.ConnectionLost), session.state.value) + assertEquals(0, audio.activeHolds) + } + @Test fun `Given attached sinks and muted audio When the session connects Then they are applied`() = runTest { val session = createSession()