Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions Android/src/app/src/main/AndroidManifest.xml
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,9 @@
android:supportsRtl="true"
android:theme="@style/Theme.Gallery"
tools:targetApi="31">
<uses-native-library android:name="libedgetpu_util.so" android:required="false" />
<uses-native-library android:name="libedgetpu_tachyon.google.so" android:required="false" />
<uses-native-library android:name="libedgetpu_client.google.so" android:required="false" />
<!--
android:configChanges="uiMode" tells the system don't destroy and
recreate the activity when UI mode changes (e.g. setting dark mode).
Expand Down
14 changes: 14 additions & 0 deletions Android/src/app/src/main/LLM_CAMERA.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# LlmCamera TPU Configuration

To build the LlmCamera feature with TPU-accelerated VoiceLM TTS, you must pass the `voicelm_tpu` definition during your `blaze` build. This builds the **experimental** flavor of the AI Edge Gallery app (`ai_edge_gallery_app_experimental`), which is required because the LlmCamera feature is currently behind the experimental build target.

**Example Build Command:**
```bash
blaze mobile-install --start_app -c opt --config=android_arm64-v8a --define=libunwind=true --define=xnnpack_use_latest_ops=true --define=keep_litertlm_symbols=true --define=voicelm_tpu=true //third_party/ai_edge_gallery/Android/src/app/src/main:ai_edge_gallery_app_experimental
```

**Important:** Before testing the TPU flavor on a device, you must enable unlisted apps to use the Edge TPU service. Run the following ADB command at least once on the test device:

```bash
adb shell setprop vendor.edgetpu.service.allow_unlisted_app true
```
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package com.google.ai.edge.gallery.ui.common
import android.graphics.RuntimeShader
import android.os.Build
import androidx.compose.animation.core.Animatable
import androidx.compose.animation.core.LinearOutSlowInEasing
import androidx.compose.animation.core.tween
import androidx.compose.foundation.Canvas
import androidx.compose.foundation.layout.fillMaxSize
Expand All @@ -36,6 +37,15 @@ import androidx.compose.ui.graphics.ShaderBrush
import kotlin.math.pow
import kotlin.random.Random

private const val AMPLITUDE_VISUAL_SCALE_DOWN = 0.4f
// Maximum amplitude value from LlmCameraViewModel.convertRmsDbToAmplitude (0..65535 range).
// Used to normalize the amplitude input to 0..1 for the visual animation.
private const val MAX_16BIT_AMPLITUDE = 65535.0
// Normalized amplitude threshold for detecting high-to-low transitions that reset Perlin noise.
private const val AMPLITUDE_THRESHOLD = 0.2
// Multiplier for the random Perlin noise offset range.
private const val POFFSET_RANGE = 1000f

private const val SHADER =
"""
// The size of the render area.
Expand Down Expand Up @@ -89,20 +99,23 @@ half4 main(float2 fragCoord) {

// Add a wavy distortion to the y-coordinate of the uv.
//
// Control the amplitude of the wave
// Control the amplitude of the wave. Higher values create more pronounced vertical distortion.
float wave_strength = 0.036;
// Control the speed of the wave
float wave_speed = 1.2;
// Control the frequency of the wave
float wave_frequency = 4.0;
// Control the speed of the wave. Higher values make the idle wave animate faster.
float wave_speed = 0.8;
// Control the frequency of the wave. Higher values create more wave cycles across the screen.
float wave_frequency = 0.5;

// Idle.
if (amplitude == 0.) {
uv.y += sin(uv.x * wave_frequency + -iTime * wave_speed) * wave_strength;
}
// Visualizing amplitude by sampling the 1d perlin noise at the given offset.
// The 0.5 multiplier on uv.x reduces the noise sampling rate, creating wider, smoother
// wave patterns. Dividing amplitude by 2.0 scales the visual response to prevent
// excessive distortion at high audio amplitudes.
else {
uv.y -= perlin_noise_1d(pOffset + uv.x * 3.) * amplitude / 2.0;
uv.y -= perlin_noise_1d(pOffset + uv.x * 0.5) * amplitude / 2.0;
}

vec3 col = mix4(
Expand Down Expand Up @@ -142,8 +155,11 @@ fun AudioAnimation(bgColor: Color, amplitude: Int, modifier: Modifier = Modifier
var iTime by remember { mutableFloatStateOf(0f) }
var curPOffset by remember { mutableFloatStateOf(0f) }
var prevNormalizedAmplitude by remember { mutableDoubleStateOf(0.0) }
// Use pow(x, 0.5) to make low amplitude levels more significant.
val normalizedAmplitude = (amplitude / 32767.0).pow(0.5)
// Use pow(x, 0.5) (square root curve) to make low amplitude levels more significant.
// Scale down by AMPLITUDE_VISUAL_SCALE_DOWN (0.4) to reduce the visual intensity,
// preventing the animation from being too aggressive at normal speech volumes.
val normalizedAmplitude =
(amplitude / MAX_16BIT_AMPLITUDE).pow(0.5) * AMPLITUDE_VISUAL_SCALE_DOWN
var animatedAmplitude by remember { mutableFloatStateOf(normalizedAmplitude.toFloat()) }

// Animate the amplitude value whenever amplitude changes.
Expand All @@ -152,16 +168,18 @@ fun AudioAnimation(bgColor: Color, amplitude: Int, modifier: Modifier = Modifier
val animatable = Animatable(initialValue = animatedAmplitude)
animatable.animateTo(
targetValue = normalizedAmplitude.toFloat(),
animationSpec = tween(durationMillis = 100),
animationSpec = tween(durationMillis = 500, easing = LinearOutSlowInEasing),
) {
animatedAmplitude = this.value
}
}

// Updates the iTime uniform for the shader.
LaunchedEffect(Unit) {
while (true) {
withFrameMillis { frameTimeMs -> iTime = frameTimeMs / 1000f }
if ("robolectric" != Build.FINGERPRINT) {
while (true) {
withFrameMillis { frameTimeMs -> iTime = frameTimeMs / 1000f }
}
}
}

Expand All @@ -171,8 +189,10 @@ fun AudioAnimation(bgColor: Color, amplitude: Int, modifier: Modifier = Modifier
// level (0.2 or greater) to a low level (less than 0.2). This makes the noise-driven visual
// effect appear to "jump" or reset to a new, random state when the audio becomes quiet,
// preventing the visual from settling into a repetitive or static pattern.
if (normalizedAmplitude < 0.2 && prevNormalizedAmplitude >= 0.2) {
curPOffset = Random.nextFloat() * 1000f
if (
normalizedAmplitude < AMPLITUDE_THRESHOLD && prevNormalizedAmplitude >= AMPLITUDE_THRESHOLD
) {
curPOffset = Random.nextFloat() * POFFSET_RANGE
}
prevNormalizedAmplitude = normalizedAmplitude

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,11 @@ object LlmChatModelHelper : LlmModelHelper {

override fun stopResponse(model: Model) {
val instance = model.instance as? LlmModelInstance ?: return
instance.conversation.cancelProcess()
try {
instance.conversation.cancelProcess()
} catch (e: IllegalStateException) {
Log.w(TAG, "Conversation is not alive, cannot cancel process", e)
}
}

override fun runInference(
Expand Down
5 changes: 5 additions & 0 deletions Android/src/app/src/main/res/values/strings.xml
Original file line number Diff line number Diff line change
Expand Up @@ -125,9 +125,11 @@
<string name="listening" translatable="true">Listening...</string>
<string name="llm_camera_status_waiting_to_start" translatable="true" description="Status text shown when LlmCamera is initialized and waiting for user input.">Waiting to start…</string>
<string name="llm_camera_status_loading_model" translatable="true" description="Status text shown while loading the LlmCamera model.">Loading model…</string>
<string name="llm_camera_status_listening" translatable="true" description="Status text shown when LlmCamera is listening to user speech. [CHAR_LIMIT=30] [BACKUP_MESSAGE_ID: 4062899179021837624]">Listening…</string>
<string name="llm_camera_status_processing" translatable="true" description="Status text shown while LlmCamera is processing speech or image input.">Processing…</string>
<string name="llm_camera_status_cancelled" translatable="true" description="Status text shown when LlmCamera processing was cancelled.">Cancelled</string>
<string name="llm_camera_status_responding" translatable="true" description="Status text shown while LlmCamera model is generating a response.">Responding…</string>
<string name="llm_camera_status_done" translatable="true" description="Status text shown when LlmCamera finishes generating a response. [CHAR_LIMIT=15] [BACKUP_MESSAGE_ID: 5969525977325895045]">Done</string>
<string name="llm_camera_status_error" translatable="true" description="Status text template shown when an error occurs during LlmCamera operation.">Error: %1$s</string>
<string name="llm_camera_recognized_prefix" translatable="true" description="Prefix for recognized speech text in LlmCamera.">Recognized: %1$s</string>
<string name="llm_camera_response_prefix" translatable="true" description="Prefix for model response text in LlmCamera.">Response: %1$s</string>
Expand Down Expand Up @@ -641,4 +643,7 @@
<string name="config_label_font_size" translatable="true">Font size</string>
<!-- Label for the 'Max character count' option in the model configuration dialog. [CHAR_LIMIT=NONE] -->
<string name="config_label_max_char_count" translatable="true">Max character count</string>

<string name="llm_camera_title" translatable="true" description="Title of the LlmCamera screen. [CHAR_LIMIT=30]">LlmCamera</string>
<string name="llm_camera_cd_flip_camera" translatable="true" description="Content description for the flip button in LlmCamera. Camera is a generic term. [CHAR_LIMIT=30]">Flip Camera</string>
</resources>
Loading