diff --git a/demo/src/androidTest/java/com/google/maps/android/utils/demo/BaseVisualTest.kt b/demo/src/androidTest/java/com/google/maps/android/utils/demo/BaseVisualTest.kt index b3cad2dee..a92f36366 100644 --- a/demo/src/androidTest/java/com/google/maps/android/utils/demo/BaseVisualTest.kt +++ b/demo/src/androidTest/java/com/google/maps/android/utils/demo/BaseVisualTest.kt @@ -19,11 +19,14 @@ import android.app.Instrumentation import android.content.Context import android.graphics.Bitmap import android.graphics.BitmapFactory +import android.util.Log import androidx.test.platform.app.InstrumentationRegistry import androidx.test.uiautomator.UiDevice import com.google.maps.android.visualtesting.GeminiVisualTestHelper import org.junit.Assert.assertTrue +import org.junit.Assert.fail import java.io.File +import java.io.FileOutputStream abstract class BaseVisualTest { protected val instrumentation: Instrumentation = InstrumentationRegistry.getInstrumentation() @@ -67,4 +70,57 @@ abstract class BaseVisualTest { e.printStackTrace() } } + + /** + * Verifies the screenshot against a golden image. + * If the golden matches, the test passes immediately. + * If it fails (or no golden exists), it falls back to Gemini for visual verification. + * If Gemini passes, it saves/updates the golden image and logs a warning to review it. + */ + protected suspend fun verifyScreenshotWithGoldenFallback( + testName: String, + screenshotBitmap: Bitmap, + prompt: String, + passCondition: (String) -> Boolean + ) { + val goldenFile = File(context.getExternalFilesDir(null), "goldens/$testName.png") + + var isPixelMatch = false + if (goldenFile.exists()) { + val goldenBitmap = BitmapFactory.decodeFile(goldenFile.absolutePath) + isPixelMatch = compareBitmaps(screenshotBitmap, goldenBitmap) + } + + if (isPixelMatch) { + Log.i("VisualTest", "Screenshot matched golden perfectly for '$testName'. Fast pass.") + return + } + + Log.w("VisualTest", "Screenshot did not match golden for '$testName'. Falling back to Gemini.") + val geminiResponse = helper.analyzeImage(screenshotBitmap, prompt, geminiApiKey) + + requireNotNull(geminiResponse) { "Gemini response was null for test '$testName'" } + + val passed = passCondition(geminiResponse) + + if (passed) { + // Update golden since Gemini approved it + goldenFile.parentFile?.mkdirs() + FileOutputStream(goldenFile).use { out -> + screenshotBitmap.compress(Bitmap.CompressFormat.PNG, 100, out) + } + Log.w("VisualTest", "Gemini approved the new UI state for '$testName'. Golden image updated at ${goldenFile.absolutePath}. Please review this image manually to prevent baking in hallucinations.") + } else { + fail("Visual verification failed for '$testName'. Gemini response: $geminiResponse") + } + } + + private fun compareBitmaps(b1: Bitmap, b2: Bitmap): Boolean { + if (b1.width != b2.width || b1.height != b2.height) return false + val pixels1 = IntArray(b1.width * b1.height) + val pixels2 = IntArray(b2.width * b2.height) + b1.getPixels(pixels1, 0, b1.width, 0, 0, b1.width, b1.height) + b2.getPixels(pixels2, 0, b2.width, 0, 0, b2.width, b2.height) + return pixels1.contentEquals(pixels2) + } } diff --git a/demo/src/androidTest/java/com/google/maps/android/utils/demo/ClusteringVisualTest.kt b/demo/src/androidTest/java/com/google/maps/android/utils/demo/ClusteringVisualTest.kt index aa5827391..951d309d7 100644 --- a/demo/src/androidTest/java/com/google/maps/android/utils/demo/ClusteringVisualTest.kt +++ b/demo/src/androidTest/java/com/google/maps/android/utils/demo/ClusteringVisualTest.kt @@ -21,7 +21,6 @@ import androidx.test.uiautomator.By import androidx.test.uiautomator.Until import kotlinx.coroutines.runBlocking import org.junit.Assert.assertNotNull -import org.junit.Assert.assertTrue import org.junit.Before import org.junit.Test import org.junit.runner.RunWith @@ -52,13 +51,13 @@ class ClusteringVisualTest : BaseVisualTest() { val screenshotBitmap = captureScreenshot("natural_lang_click_screenshot.png") // --- Perform a visual assertion on the new screen --- - val prompt = "Does this image show a map with several markers clustered together? Answer only YES or NO." - val geminiResponse = helper.analyzeImage(screenshotBitmap, prompt, geminiApiKey) - - println("Gemini's analysis after natural language click: $geminiResponse") - assertTrue( - "Visual verification failed. Gemini did not confirm the presence of a map with clusters.", - geminiResponse?.contains("YES", ignoreCase = true) == true, + val prompt = + "Does this image show a map with several markers clustered together? Answer only YES or NO." + verifyScreenshotWithGoldenFallback( + testName = "Clustering_NaturalLanguageClick", + screenshotBitmap = screenshotBitmap, + prompt = prompt, + passCondition = { it.contains("YES", ignoreCase = true) }, ) } @@ -72,7 +71,10 @@ class ClusteringVisualTest : BaseVisualTest() { // Dump window hierarchy to logcat for debugging val outputStream = ByteArrayOutputStream() uiDevice.dumpWindowHierarchy(outputStream) - Log.e("ClusteringVisualTest", "Could not find clustering button. UI Hierarchy:\n${outputStream.toString("UTF-8")}") + Log.e( + "ClusteringVisualTest", + "Could not find clustering button. UI Hierarchy:\n${outputStream.toString("UTF-8")}", + ) // Take a screenshot for visual inspection val screenshotFile = File(context.cacheDir, "test_failure_screenshot.png") @@ -80,7 +82,10 @@ class ClusteringVisualTest : BaseVisualTest() { Log.e("ClusteringVisualTest", "Debug screenshot saved to device cache.") } - assertNotNull("Clustering button not found. Check logcat for UI hierarchy dump and debug screenshot.", clusteringButton) + assertNotNull( + "Clustering button not found. Check logcat for UI hierarchy dump and debug screenshot.", + clusteringButton, + ) clusteringButton.click() // Wait for the clustering screen to load and map to render @@ -99,15 +104,11 @@ class ClusteringVisualTest : BaseVisualTest() { If all three elements are present and legible, just confirm that the visual test has PASSED. If any element is missing or incorrect, please detail the discrepancy. """.trimIndent() - // --- STEP 3: Analyze the image using Gemini --- - val geminiResponse = helper.analyzeImage(screenshotBitmap, prompt, geminiApiKey) - - // --- STEP 4: Assert on Gemini's response --- - println("Gemini's analysis: $geminiResponse") - // Example assertion: Check if Gemini confirms the presence of clusters - assertTrue( - "PASSED", - geminiResponse!!.contains("PASSED", ignoreCase = true), + verifyScreenshotWithGoldenFallback( + testName = "Clustering_ScreenContent", + screenshotBitmap = screenshotBitmap, + prompt = prompt, + passCondition = { it.contains("PASSED", ignoreCase = true) }, ) } }