Add Compose WebRTC renderer and developer debug screen

.github/scripts/check_licenses.py

@@ -29,6 +29,9 @@
     "org.chromium.net:*",
     # https://github.com/Dimezis/BlurView/issues/259
     "com.github.Dimezis:BlurView",
+    # libwebrtc is BSD-3-Clause but the POM declares the non-SPDX name
+    # "The 3-Clause BSD License" which aboutlibraries cannot map
+    "io.github.webrtc-sdk:android-prefixed",
 ]
 
 

app/gradle.lockfile

@@ -493,6 +493,7 @@ io.github.detekt.sarif4k:sarif4k-jvm:0.6.0=ktlint,ktlintReporter
 io.github.detekt.sarif4k:sarif4k:0.6.0=ktlint,ktlintReporter
 io.github.oshai:kotlin-logging-jvm:7.0.3=ktlint,ktlintBaselineReporter,ktlintReporter,ktlintRuleset
 io.github.oshai:kotlin-logging:5.1.0=ktlint,ktlintBaselineReporter,ktlintReporter
+io.github.webrtc-sdk:android-prefixed:144.7559.09=fullDebugAndroidTestCompileClasspath,fullDebugAndroidTestLintChecksClasspath,fullDebugCompileClasspath,fullDebugLintChecksClasspath,fullDebugRuntimeClasspath,fullDebugScreenshotTestCompileClasspath,fullDebugScreenshotTestLintChecksClasspath,fullDebugScreenshotTestRuntimeClasspath,fullDebugUnitTestCompileClasspath,fullDebugUnitTestLintChecksClasspath,fullDebugUnitTestRuntimeClasspath,fullReleaseCompileClasspath,fullReleaseLintChecksClasspath,fullReleaseRuntimeClasspath,fullReleaseScreenshotTestCompileClasspath,fullReleaseScreenshotTestLintChecksClasspath,fullReleaseScreenshotTestRuntimeClasspath,minimalDebugAndroidTestCompileClasspath,minimalDebugAndroidTestLintChecksClasspath,minimalDebugCompileClasspath,minimalDebugLintChecksClasspath,minimalDebugRuntimeClasspath,minimalDebugScreenshotTestCompileClasspath,minimalDebugScreenshotTestLintChecksClasspath,minimalDebugScreenshotTestRuntimeClasspath,minimalDebugUnitTestCompileClasspath,minimalDebugUnitTestLintChecksClasspath,minimalDebugUnitTestRuntimeClasspath,minimalReleaseCompileClasspath,minimalReleaseLintChecksClasspath,minimalReleaseRuntimeClasspath,minimalReleaseScreenshotTestCompileClasspath,minimalReleaseScreenshotTestLintChecksClasspath,minimalReleaseScreenshotTestRuntimeClasspath
 io.grpc:grpc-api:1.57.2=unified-test-platform-core
 io.grpc:grpc-api:1.69.1=unified-test-platform-android-test-plugin-host-emulator-control
 io.grpc:grpc-context:1.57.2=unified-test-platform-core

app/src/main/kotlin/io/homeassistant/companion/android/settings/developer/DeveloperSettingsFragment.kt

@@ -17,6 +17,7 @@ import io.homeassistant.companion.android.R
 import io.homeassistant.companion.android.common.R as commonR
 import io.homeassistant.companion.android.common.data.servers.ServerManager
 import io.homeassistant.companion.android.settings.developer.location.LocationTrackingFragment
+import io.homeassistant.companion.android.settings.developer.webrtc.WebRtcDebugFragment
 import io.homeassistant.companion.android.settings.log.LogFragment
 import io.homeassistant.companion.android.settings.server.ServerChooserFragment
 import io.homeassistant.companion.android.util.applyBottomSafeDrawingInsets
@@ -93,6 +94,13 @@ class DeveloperSettingsFragment :
             }
         }
 
+        findPreference<Preference>("webrtc_debug")?.setOnPreferenceClickListener {
+            parentFragmentManager.commit {
+                replace(R.id.content, WebRtcDebugFragment::class.java, null)
+                addToBackStack(getString(commonR.string.webrtc_debug))
+            }
+            return@setOnPreferenceClickListener true
+        }
         findPreference<Preference>("webview_clear_cache")?.let {
             it.isVisible = presenter.webViewSupportsClearCache()
             it.setOnPreferenceClickListener {

app/src/main/kotlin/io/homeassistant/companion/android/settings/developer/webrtc/WebRtcDebugFragment.kt

@@ -0,0 +1,44 @@
+package io.homeassistant.companion.android.settings.developer.webrtc
+
+import android.os.Bundle
+import android.view.LayoutInflater
+import android.view.View
+import android.view.ViewGroup
+import androidx.compose.runtime.getValue
+import androidx.compose.ui.platform.ComposeView
+import androidx.fragment.app.Fragment
+import androidx.fragment.app.viewModels
+import androidx.lifecycle.compose.collectAsStateWithLifecycle
+import dagger.hilt.android.AndroidEntryPoint
+import io.homeassistant.companion.android.common.R as commonR
+import io.homeassistant.companion.android.settings.developer.webrtc.views.WebRtcDebugView
+import io.homeassistant.companion.android.util.compose.HomeAssistantAppTheme
+
+@AndroidEntryPoint
+class WebRtcDebugFragment : Fragment() {
+
+    private val viewModel: WebRtcDebugViewModel by viewModels()
+
+    override fun onCreateView(inflater: LayoutInflater, container: ViewGroup?, savedInstanceState: Bundle?): View {
+        return ComposeView(requireContext()).apply {
+            setContent {
+                HomeAssistantAppTheme {
+                    val session by viewModel.session.collectAsStateWithLifecycle()
+                    val playerState by viewModel.playerState.collectAsStateWithLifecycle()
+                    WebRtcDebugView(
+                        player = session,
+                        playerState = playerState,
+                        eglContext = viewModel.eglContext,
+                        onStart = viewModel::startSession,
+                        onStop = viewModel::stopSession,
+                    )
+                }
+            }
+        }
+    }
+
+    override fun onResume() {
+        super.onResume()
+        activity?.title = getString(commonR.string.webrtc_debug)
+    }
+}

app/src/main/kotlin/io/homeassistant/companion/android/settings/developer/webrtc/WebRtcDebugViewModel.kt

@@ -0,0 +1,74 @@
+package io.homeassistant.companion.android.settings.developer.webrtc
+
+import androidx.lifecycle.ViewModel
+import androidx.lifecycle.viewModelScope
+import dagger.hilt.android.lifecycle.HiltViewModel
+import io.homeassistant.companion.android.common.data.servers.ServerManager
+import io.homeassistant.companion.android.webrtc.core.PlayerState
+import io.homeassistant.companion.android.webrtc.core.session.WebRtcSession
+import io.homeassistant.companion.android.webrtc.core.session.libwebrtc.LibWebRtcPeerConnectionControllerFactory
+import io.homeassistant.companion.android.webrtc.signaling.HaSignalingClient
+import javax.inject.Inject
+import kotlin.time.Duration.Companion.seconds
+import kotlinx.coroutines.ExperimentalCoroutinesApi
+import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.SharingStarted
+import kotlinx.coroutines.flow.StateFlow
+import kotlinx.coroutines.flow.asStateFlow
+import kotlinx.coroutines.flow.flatMapLatest
+import kotlinx.coroutines.flow.flowOf
+import kotlinx.coroutines.flow.stateIn
+import kotlinx.coroutines.launch
+import livekit.org.webrtc.EglBase
+
+private val STATE_SHARING_TIMEOUT = 5.seconds
+
+/**
+ * Owns the WebRTC session of the developer debug screen, so the session survives configuration
+ * changes and is always released when the screen is destroyed.
+ */
+@HiltViewModel
+class WebRtcDebugViewModel @Inject constructor(
+    private val serverManager: ServerManager,
+    private val controllerFactory: LibWebRtcPeerConnectionControllerFactory,
+) : ViewModel() {
+
+    private val _session = MutableStateFlow<WebRtcSession?>(null)
+    val session: StateFlow<WebRtcSession?> = _session.asStateFlow()
+
+    @OptIn(ExperimentalCoroutinesApi::class)
+    val playerState: StateFlow<PlayerState> = _session
+        .flatMapLatest { session -> session?.state ?: flowOf(PlayerState.Idle) }
+        .stateIn(
+            scope = viewModelScope,
+            started = SharingStarted.WhileSubscribed(STATE_SHARING_TIMEOUT.inWholeMilliseconds),
+            initialValue = PlayerState.Idle,
+        )
+
+    /** EGL context the renderer must share with the hardware decoder. */
+    val eglContext: EglBase.Context
+        get() = controllerFactory.eglBase.eglBaseContext
+
+    fun startSession(entityId: String) {
+        val trimmedEntityId = entityId.trim()
+        if (trimmedEntityId.isEmpty()) return
+        viewModelScope.launch {
+            stopSession()
+            val signalingClient = HaSignalingClient(serverManager.webSocketRepository())
+            _session.value = WebRtcSession(
+                entityId = trimmedEntityId,
+                signalingClient = signalingClient,
+                controllerFactory = controllerFactory,
+            ).also { it.start() }
+        }
+    }
+
+    fun stopSession() {
+        _session.value?.release()
+        _session.value = null
+    }
+
+    override fun onCleared() {
+        stopSession()
+    }
+}

app/src/main/kotlin/io/homeassistant/companion/android/settings/developer/webrtc/views/WebRtcDebugView.kt

@@ -0,0 +1,106 @@
+package io.homeassistant.companion.android.settings.developer.webrtc.views
+
+import androidx.compose.foundation.layout.Arrangement
+import androidx.compose.foundation.layout.Column
+import androidx.compose.foundation.layout.Row
+import androidx.compose.foundation.layout.aspectRatio
+import androidx.compose.foundation.layout.fillMaxSize
+import androidx.compose.foundation.layout.fillMaxWidth
+import androidx.compose.foundation.layout.padding
+import androidx.compose.foundation.rememberScrollState
+import androidx.compose.foundation.verticalScroll
+import androidx.compose.material3.Button
+import androidx.compose.material3.MaterialTheme
+import androidx.compose.material3.Text
+import androidx.compose.material3.TextField
+import androidx.compose.runtime.Composable
+import androidx.compose.runtime.getValue
+import androidx.compose.runtime.mutableStateOf
+import androidx.compose.runtime.saveable.rememberSaveable
+import androidx.compose.runtime.setValue
+import androidx.compose.ui.Alignment
+import androidx.compose.ui.Modifier
+import androidx.compose.ui.graphics.Color
+import androidx.compose.ui.res.stringResource
+import androidx.compose.ui.unit.dp
+import io.homeassistant.companion.android.common.R as commonR
+import io.homeassistant.companion.android.webrtc.compose.WebRtcVideo
+import io.homeassistant.companion.android.webrtc.core.CameraPlayer
+import io.homeassistant.companion.android.webrtc.core.PlayerFailure
+import io.homeassistant.companion.android.webrtc.core.PlayerState
+import livekit.org.webrtc.EglBase
+
+private const val VIDEO_ASPECT_RATIO = 16f / 9f
+
+/**
+ * Developer screen to exercise the native WebRTC camera player against a camera entity, without
+ * any dashboard integration.
+ */
+@Composable
+fun WebRtcDebugView(
+    player: CameraPlayer?,
+    playerState: PlayerState,
+    eglContext: EglBase.Context?,
+    onStart: (String) -> Unit,
+    onStop: () -> Unit,
+    modifier: Modifier = Modifier,
+) {
+    var entityId by rememberSaveable { mutableStateOf("") }
+
+    Column(
+        modifier = modifier
+            .fillMaxSize()
+            .verticalScroll(rememberScrollState())
+            .padding(16.dp),
+        verticalArrangement = Arrangement.spacedBy(16.dp),
+    ) {
+        TextField(
+            value = entityId,
+            onValueChange = { entityId = it },
+            label = { Text(stringResource(commonR.string.webrtc_debug_entity_id)) },
+            singleLine = true,
+            modifier = Modifier.fillMaxWidth(),
+        )
+        Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) {
+            Button(onClick = { onStart(entityId) }, enabled = entityId.isNotBlank()) {
+                Text(stringResource(commonR.string.webrtc_debug_start))
+            }
+            Button(onClick = onStop, enabled = player != null) {
+                Text(stringResource(commonR.string.webrtc_debug_stop))
+            }
+        }
+        // Raw technical state, on purpose not localized on this developer-only screen
+        Text(
+            text = playerState.toDebugLabel(),
+            style = MaterialTheme.typography.bodyMedium,
+        )
+        player?.let {
+            WebRtcVideo(
+                player = it,
+                eglContext = eglContext,
+                modifier = Modifier
+                    .fillMaxWidth()
+                    .aspectRatio(VIDEO_ASPECT_RATIO),
+                failedContent = { failure ->
+                    Text(
+                        text = failure.toDebugLabel(),
+                        color = Color.White,
+                        style = MaterialTheme.typography.bodyMedium,
+                        modifier = Modifier.align(Alignment.Center),
+                    )
+                },
+            )
+        }
+    }
+}
+
+private fun PlayerState.toDebugLabel(): String = when (this) {
+    is PlayerState.Failed -> "${this::class.simpleName}: ${failure.toDebugLabel()}"
+    else -> this::class.simpleName.orEmpty()
+}
+
+private fun PlayerFailure.toDebugLabel(): String = when (this) {
+    is PlayerFailure.Signaling -> "Signaling(${code ?: "unknown"}) ${message.orEmpty()}"
+    is PlayerFailure.Internal -> "Internal: ${message.orEmpty()}"
+    PlayerFailure.ConnectionLost -> "ConnectionLost"
+}

app/src/main/kotlin/io/homeassistant/companion/android/webrtc/WebRtcModule.kt

@@ -0,0 +1,32 @@
+package io.homeassistant.companion.android.webrtc
+
+import android.content.Context
+import dagger.Module
+import dagger.Provides
+import dagger.hilt.InstallIn
+import dagger.hilt.android.qualifiers.ApplicationContext
+import dagger.hilt.components.SingletonComponent
+import io.homeassistant.companion.android.webrtc.core.session.PeerConnectionController
+import io.homeassistant.companion.android.webrtc.core.session.libwebrtc.LibWebRtcPeerConnectionControllerFactory
+import javax.inject.Singleton
+
+@Module
+@InstallIn(SingletonComponent::class)
+object WebRtcModule {
+
+    /**
+     * A single libwebrtc factory for the whole process: the underlying `PeerConnectionFactory`
+     * (native threads, audio device module, codec factories) is expensive and shared between all
+     * WebRTC sessions.
+     */
+    @Provides
+    @Singleton
+    fun providePeerConnectionControllerFactory(
+        @ApplicationContext context: Context,
+    ): LibWebRtcPeerConnectionControllerFactory = LibWebRtcPeerConnectionControllerFactory(context)
+
+    @Provides
+    fun providePeerConnectionControllerFactoryInterface(
+        factory: LibWebRtcPeerConnectionControllerFactory,
+    ): PeerConnectionController.Factory = factory
+}

app/src/main/res/xml/preferences_developer.xml

@@ -20,6 +20,11 @@
         android:icon="@drawable/ic_thread"
         android:title="@string/thread_debug"
         android:summary="@string/thread_debug_summary"/>
+    <Preference
+        android:key="webrtc_debug"
+        android:icon="@drawable/ic_play"
+        android:title="@string/webrtc_debug"
+        android:summary="@string/webrtc_debug_summary"/>
     <Preference
         android:key="tag_clear_allowed"
         android:icon="@drawable/ic_tag_remove"

automotive/gradle.lockfile

@@ -494,6 +494,7 @@ io.github.detekt.sarif4k:sarif4k-jvm:0.6.0=ktlint,ktlintReporter
 io.github.detekt.sarif4k:sarif4k:0.6.0=ktlint,ktlintReporter
 io.github.oshai:kotlin-logging-jvm:7.0.3=ktlint,ktlintBaselineReporter,ktlintReporter,ktlintRuleset
 io.github.oshai:kotlin-logging:5.1.0=ktlint,ktlintBaselineReporter,ktlintReporter
+io.github.webrtc-sdk:android-prefixed:144.7559.09=fullDebugAndroidTestCompileClasspath,fullDebugAndroidTestLintChecksClasspath,fullDebugCompileClasspath,fullDebugLintChecksClasspath,fullDebugRuntimeClasspath,fullDebugScreenshotTestCompileClasspath,fullDebugScreenshotTestLintChecksClasspath,fullDebugScreenshotTestRuntimeClasspath,fullDebugUnitTestCompileClasspath,fullDebugUnitTestLintChecksClasspath,fullDebugUnitTestRuntimeClasspath,fullReleaseCompileClasspath,fullReleaseLintChecksClasspath,fullReleaseRuntimeClasspath,fullReleaseScreenshotTestCompileClasspath,fullReleaseScreenshotTestLintChecksClasspath,fullReleaseScreenshotTestRuntimeClasspath,minimalDebugAndroidTestCompileClasspath,minimalDebugAndroidTestLintChecksClasspath,minimalDebugCompileClasspath,minimalDebugLintChecksClasspath,minimalDebugRuntimeClasspath,minimalDebugScreenshotTestCompileClasspath,minimalDebugScreenshotTestLintChecksClasspath,minimalDebugScreenshotTestRuntimeClasspath,minimalDebugUnitTestCompileClasspath,minimalDebugUnitTestLintChecksClasspath,minimalDebugUnitTestRuntimeClasspath,minimalReleaseCompileClasspath,minimalReleaseLintChecksClasspath,minimalReleaseRuntimeClasspath,minimalReleaseScreenshotTestCompileClasspath,minimalReleaseScreenshotTestLintChecksClasspath,minimalReleaseScreenshotTestRuntimeClasspath
 io.grpc:grpc-api:1.57.2=unified-test-platform-core
 io.grpc:grpc-api:1.69.1=unified-test-platform-android-test-plugin-host-emulator-control
 io.grpc:grpc-context:1.57.2=unified-test-platform-core

build-logic/convention/src/main/kotlin/AndroidApplicationDependenciesConventionPlugin.kt

@@ -27,6 +27,9 @@ class AndroidApplicationDependenciesConventionPlugin : Plugin<Project> {
                 dependencies {
                     "implementation"(project(":common"))
                     "implementation"(project(":microwakeword"))
+                    "implementation"(project(":webrtc-compose"))
+                    "implementation"(project(":webrtc-core"))
+                    "implementation"(project(":webrtc-signaling-ha"))
 
                     "implementation"(libs.blurView)
                     "implementation"(libs.haze)

common/src/main/res/values/strings.xml

@@ -976,6 +976,11 @@
     <string name="wear_settings">Wear device settings</string>
     <string name="wear_unsupported_title">Wear OS not supported</string>
     <string name="wear_unsupported_message">Wear OS onboarding requires the full version of the Home Assistant app with Google Play Services</string>
+    <string name="webrtc_debug">WebRTC camera player</string>
+    <string name="webrtc_debug_entity_id">Camera entity ID</string>
+    <string name="webrtc_debug_start">Start</string>
+    <string name="webrtc_debug_stop">Stop</string>
+    <string name="webrtc_debug_summary">Test the native WebRTC player against a camera entity of the active server</string>
     <string name="websocket_listening">Connected to Home Assistant</string>
     <string name="websocket_restricted_title">Unable to open connection to Home Assistant</string>
     <string name="websocket_restricted_fix">Update settings to fix or disable</string>

gradle.properties

@@ -6,7 +6,7 @@
 # http://www.gradle.org/docs/current/userguide/build_environment.html
 # Specifies the JVM arguments used for the daemon process.
 # The setting is particularly useful for tweaking memory settings.
-org.gradle.jvmargs=-Xmx4g -XX:MaxMetaspaceSize=1g -Dfile.encoding=UTF-8
+org.gradle.jvmargs=-Xmx5g -XX:MaxMetaspaceSize=1g -Dfile.encoding=UTF-8
 # When configured, Gradle will run in incubating parallel mode.
 # This option should only be used with decoupled projects. More details, visit
 # http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects

settings.gradle.kts

@@ -7,6 +7,7 @@ include(
     ":lint",
     ":microwakeword",
     ":provides-sensor-processor",
+    ":webrtc-compose",
     ":webrtc-core",
     ":webrtc-signaling-ha",
 )

webrtc-compose/build.gradle.kts

@@ -0,0 +1,18 @@
+plugins {
+    alias(libs.plugins.android.library)
+    alias(libs.plugins.homeassistant.android.common)
+    alias(libs.plugins.homeassistant.android.compose)
+}
+
+android {
+    namespace = "io.homeassistant.companion.android.webrtc.compose"
+}
+
+dependencies {
+    api(project(":webrtc-core"))
+
+    implementation(libs.kotlin.stdlib)
+    implementation(libs.kotlinx.coroutines.core)
+
+    testImplementation(libs.androidx.test.core)
+}