test: implement AI-backed golden screenshot fallback

demo/src/androidTest/java/com/google/maps/android/utils/demo/BaseVisualTest.kt

@@ -19,11 +19,14 @@ import android.app.Instrumentation
 import android.content.Context
 import android.graphics.Bitmap
 import android.graphics.BitmapFactory
+import android.util.Log
 import androidx.test.platform.app.InstrumentationRegistry
 import androidx.test.uiautomator.UiDevice
 import com.google.maps.android.visualtesting.GeminiVisualTestHelper
 import org.junit.Assert.assertTrue
+import org.junit.Assert.fail
 import java.io.File
+import java.io.FileOutputStream
 
 abstract class BaseVisualTest {
     protected val instrumentation: Instrumentation = InstrumentationRegistry.getInstrumentation()
@@ -67,4 +70,57 @@ abstract class BaseVisualTest {
             e.printStackTrace()
         }
     }
+
+    /**
+     * Verifies the screenshot against a golden image.
+     * If the golden matches, the test passes immediately.
+     * If it fails (or no golden exists), it falls back to Gemini for visual verification.
+     * If Gemini passes, it saves/updates the golden image and logs a warning to review it.
+     */
+    protected suspend fun verifyScreenshotWithGoldenFallback(
+        testName: String,
+        screenshotBitmap: Bitmap,
+        prompt: String,
+        passCondition: (String) -> Boolean
+    ) {
+        val goldenFile = File(context.getExternalFilesDir(null), "goldens/$testName.png")
+
+        var isPixelMatch = false
+        if (goldenFile.exists()) {
+            val goldenBitmap = BitmapFactory.decodeFile(goldenFile.absolutePath)
+            isPixelMatch = compareBitmaps(screenshotBitmap, goldenBitmap)
+        }
+
+        if (isPixelMatch) {
+            Log.i("VisualTest", "Screenshot matched golden perfectly for '$testName'. Fast pass.")
+            return
+        }
+
+        Log.w("VisualTest", "Screenshot did not match golden for '$testName'. Falling back to Gemini.")
+        val geminiResponse = helper.analyzeImage(screenshotBitmap, prompt, geminiApiKey)
+
+        requireNotNull(geminiResponse) { "Gemini response was null for test '$testName'" }
+
+        val passed = passCondition(geminiResponse)
+
+        if (passed) {
+            // Update golden since Gemini approved it
+            goldenFile.parentFile?.mkdirs()
+            FileOutputStream(goldenFile).use { out ->
+                screenshotBitmap.compress(Bitmap.CompressFormat.PNG, 100, out)
+            }
+            Log.w("VisualTest", "Gemini approved the new UI state for '$testName'. Golden image updated at ${goldenFile.absolutePath}. Please review this image manually to prevent baking in hallucinations.")
+        } else {
+            fail("Visual verification failed for '$testName'. Gemini response: $geminiResponse")
+        }
+    }
+
+    private fun compareBitmaps(b1: Bitmap, b2: Bitmap): Boolean {
+        if (b1.width != b2.width || b1.height != b2.height) return false
+        val pixels1 = IntArray(b1.width * b1.height)
+        val pixels2 = IntArray(b2.width * b2.height)
+        b1.getPixels(pixels1, 0, b1.width, 0, 0, b1.width, b1.height)
+        b2.getPixels(pixels2, 0, b2.width, 0, 0, b2.width, b2.height)
+        return pixels1.contentEquals(pixels2)
+    }
 }

demo/src/androidTest/java/com/google/maps/android/utils/demo/ClusteringVisualTest.kt

@@ -21,7 +21,6 @@ import androidx.test.uiautomator.By
 import androidx.test.uiautomator.Until
 import kotlinx.coroutines.runBlocking
 import org.junit.Assert.assertNotNull
-import org.junit.Assert.assertTrue
 import org.junit.Before
 import org.junit.Test
 import org.junit.runner.RunWith
@@ -52,13 +51,13 @@ class ClusteringVisualTest : BaseVisualTest() {
             val screenshotBitmap = captureScreenshot("natural_lang_click_screenshot.png")
 
             // --- Perform a visual assertion on the new screen ---
-            val prompt = "Does this image show a map with several markers clustered together? Answer only YES or NO."
-            val geminiResponse = helper.analyzeImage(screenshotBitmap, prompt, geminiApiKey)
-
-            println("Gemini's analysis after natural language click: $geminiResponse")
-            assertTrue(
-                "Visual verification failed. Gemini did not confirm the presence of a map with clusters.",
-                geminiResponse?.contains("YES", ignoreCase = true) == true,
+            val prompt =
+                "Does this image show a map with several markers clustered together? Answer only YES or NO."
+            verifyScreenshotWithGoldenFallback(
+                testName = "Clustering_NaturalLanguageClick",
+                screenshotBitmap = screenshotBitmap,
+                prompt = prompt,
+                passCondition = { it.contains("YES", ignoreCase = true) },
             )
         }
 
@@ -72,15 +71,21 @@ class ClusteringVisualTest : BaseVisualTest() {
                 // Dump window hierarchy to logcat for debugging
                 val outputStream = ByteArrayOutputStream()
                 uiDevice.dumpWindowHierarchy(outputStream)
-                Log.e("ClusteringVisualTest", "Could not find clustering button. UI Hierarchy:\n${outputStream.toString("UTF-8")}")
+                Log.e(
+                    "ClusteringVisualTest",
+                    "Could not find clustering button. UI Hierarchy:\n${outputStream.toString("UTF-8")}",
+                )
 
                 // Take a screenshot for visual inspection
                 val screenshotFile = File(context.cacheDir, "test_failure_screenshot.png")
                 uiDevice.takeScreenshot(screenshotFile)
                 Log.e("ClusteringVisualTest", "Debug screenshot saved to device cache.")
             }
 
-            assertNotNull("Clustering button not found. Check logcat for UI hierarchy dump and debug screenshot.", clusteringButton)
+            assertNotNull(
+                "Clustering button not found. Check logcat for UI hierarchy dump and debug screenshot.",
+                clusteringButton,
+            )
             clusteringButton.click()
 
             // Wait for the clustering screen to load and map to render
@@ -99,15 +104,11 @@ class ClusteringVisualTest : BaseVisualTest() {
                 If all three elements are present and legible, just confirm that the visual test has PASSED. If any element is missing or incorrect, please detail the discrepancy.
                 """.trimIndent()
 
-            // --- STEP 3: Analyze the image using Gemini ---
-            val geminiResponse = helper.analyzeImage(screenshotBitmap, prompt, geminiApiKey)
-
-            // --- STEP 4: Assert on Gemini's response ---
-            println("Gemini's analysis: $geminiResponse")
-            // Example assertion: Check if Gemini confirms the presence of clusters
-            assertTrue(
-                "PASSED",
-                geminiResponse!!.contains("PASSED", ignoreCase = true),
+            verifyScreenshotWithGoldenFallback(
+                testName = "Clustering_ScreenContent",
+                screenshotBitmap = screenshotBitmap,
+                prompt = prompt,
+                passCondition = { it.contains("PASSED", ignoreCase = true) },
             )
         }
 }