diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 1861d2608..72bd0a380 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -335,6 +335,17 @@ android { "-Dkotlinx.coroutines.scheduler.core.pool.size=$coroutinePoolFloor " + "-Dkotlinx.coroutines.scheduler.max.pool.size=${maxOf(32, coroutinePoolFloor * 2)}", ) + // Expose the test task's full runtime classpath so tests that + // fork child JVMs (MigrationExporterOomTest) can pass it + // explicitly; the worker's own java.class.path is not + // reliable. Set in doFirst because input fingerprinting + // cannot serialize a lazy classpath value. + it.doFirst { + it.systemProperty( + "columba.test.runtimeClasspath", + it.classpath.joinToString(File.pathSeparator), + ) + } // Enable JaCoCo coverage for Robolectric tests it.extensions.configure { isIncludeNoLocationClasses = true diff --git a/app/src/main/java/network/columba/app/migration/MigrationExporter.kt b/app/src/main/java/network/columba/app/migration/MigrationExporter.kt index 2e94ddb98..e8392cbde 100644 --- a/app/src/main/java/network/columba/app/migration/MigrationExporter.kt +++ b/app/src/main/java/network/columba/app/migration/MigrationExporter.kt @@ -14,8 +14,8 @@ import dagger.hilt.android.qualifiers.ApplicationContext import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.first import kotlinx.coroutines.withContext -import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json +import kotlinx.serialization.json.encodeToStream import java.io.File import java.io.FileOutputStream import java.text.SimpleDateFormat @@ -489,7 +489,9 @@ class MigrationExporter ZipOutputStream(FileOutputStream(exportFile)).use { zipOut -> zipOut.putNextEntry(ZipEntry(MANIFEST_FILENAME)) - zipOut.write(json.encodeToString(bundle).toByteArray()) + // Stream the manifest into the zip entry so the full JSON + // text is never buffered in memory (COLUMBA-DS). + json.encodeToStream(MigrationBundle.serializer(), bundle, zipOut) zipOut.closeEntry() onProgress(0.8f) diff --git a/app/src/test/java/network/columba/app/migration/MigrationDataTest.kt b/app/src/test/java/network/columba/app/migration/MigrationDataTest.kt index 70b766aef..01bc0594c 100644 --- a/app/src/test/java/network/columba/app/migration/MigrationDataTest.kt +++ b/app/src/test/java/network/columba/app/migration/MigrationDataTest.kt @@ -2,6 +2,8 @@ package network.columba.app.migration import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json +import kotlinx.serialization.json.encodeToStream +import java.io.ByteArrayOutputStream import org.junit.Assert.assertEquals import org.junit.Assert.assertNotNull import org.junit.Assert.assertNull @@ -18,6 +20,13 @@ class MigrationDataTest { encodeDefaults = true } + /** Same [Json] settings as [MigrationExporter]'s production manifest writer. */ + private val exporterJson = + Json { + prettyPrint = true + ignoreUnknownKeys = true + } + // region Test Data Helpers private fun createTestSettings() = @@ -253,6 +262,36 @@ class MigrationDataTest { assertEquals(bundle.settings, decoded.settings) } + @Test + fun `encodeToStream produces the same manifest bytes as encodeToString`() { + val bundle = + MigrationBundle( + identities = listOf(createTestIdentity()), + conversations = listOf(createTestConversation()), + messages = listOf(createTestMessage()), + contacts = listOf(createTestContact()), + announces = listOf(createTestAnnounce()), + interfaces = listOf(createTestInterface()), + customThemes = listOf(createTestCustomTheme()), + settings = createTestSettings(), + attachmentManifest = + listOf( + AttachmentRef( + messageId = "msg_001", + fieldKey = "attachment", + relativePath = "attachments/msg_001_attachment.png", + sizeBytes = 12345L, + ), + ), + ratchetFiles = listOf(createTestRatchetRef(type = "own", filename = "a1b2c3d4e5f6")), + ) + + val streamed = ByteArrayOutputStream() + exporterJson.encodeToStream(MigrationBundle.serializer(), bundle, streamed) + + assertEquals(exporterJson.encodeToString(bundle), String(streamed.toByteArray(), Charsets.UTF_8)) + } + @Test fun `MigrationBundle with empty optional lists serializes correctly`() { val bundle = diff --git a/app/src/test/java/network/columba/app/migration/MigrationExporterOomTest.kt b/app/src/test/java/network/columba/app/migration/MigrationExporterOomTest.kt new file mode 100644 index 000000000..e55fb6e3d --- /dev/null +++ b/app/src/test/java/network/columba/app/migration/MigrationExporterOomTest.kt @@ -0,0 +1,172 @@ +package network.columba.app.migration + +import android.content.Context +import io.mockk.every +import io.mockk.mockk +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.decodeFromStream +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test +import java.io.File +import java.util.zip.ZipFile + +/** + * Repro for COLUMBA-DS: the export manifest used to be serialized with + * `json.encodeToString(bundle)`, which buffers the full manifest as a + * char array, a String, and a byte array on top of the in-memory bundle. + * A large bundle OOMs the process once the manifest outgrows the heap headroom. + * + * [MigrationExporterOomTest] forks this [MigrationExportOomChild] main in a + * separate JVM capped at 256 MB (the production target footprint) and runs + * the real [MigrationExporter.createExportZip]. The bundle is sized so the + * whole-string encoding needs ~3x the manifest text (~300+ MB live) while + * the streaming encoder stays bounded to the bundle plus small buffers. + * Pre-fix the child dies with `OutOfMemoryError`; post-fix it completes and + * the manifest round-trips through the import-side `decodeFromStream`. + */ +@Suppress("NoRelaxedMocks") // createExportZip only touches context.cacheDir/filesDir; collaborators are unused +object MigrationExportOomChild { + private const val MESSAGE_COUNT = 200_000 + private const val CONTENT_LENGTH = 384 + private const val MANIFEST_FILENAME = "manifest.json" + + @JvmStatic + fun main(args: Array) { + val workDir = + File.createTempFile("columba-oom-repro", null).apply { + delete() + mkdirs() + } + try { + run(workDir) + } finally { + // Never leave export archives behind, even on OOM. + workDir.deleteRecursively() + } + } + + private fun run(workDir: File) { + val json = Json { ignoreUnknownKeys = true } + val context = + mockk().apply { + every { cacheDir } returns File(workDir, "cache").apply { mkdirs() } + every { filesDir } returns File(workDir, "files").apply { mkdirs() } + } + + val exporter = + MigrationExporter( + context, + mockk(relaxed = true), + mockk(relaxed = true), + mockk(relaxed = true), + mockk(relaxed = true), + mockk(relaxed = true), + ) + + val bundle = + MigrationBundle( + identities = emptyList(), + conversations = emptyList(), + messages = buildMessages(), + contacts = emptyList(), + settings = + SettingsExport( + notificationsEnabled = true, + notificationReceivedMessage = true, + notificationReceivedMessageFavorite = false, + notificationHeardAnnounce = true, + notificationBleConnected = false, + notificationBleDisconnected = false, + autoAnnounceEnabled = true, + autoAnnounceIntervalMinutes = 5, + themePreference = "preset:VIBRANT", + ), + ) + + // createExportZip is private; invoke the production method directly. + val method = + MigrationExporter::class.java.getDeclaredMethod( + "createExportZip", + MigrationBundle::class.java, + List::class.java, + Function1::class.java, + ).apply { isAccessible = true } + val onProgress: Function1 = { } + val exportFile = method.invoke(exporter, bundle, emptyList(), onProgress) as File + check(exportFile.exists()) { "export file missing: $exportFile" } + + // The streamed manifest must decode through the import-side path. + ZipFile(exportFile).use { zip -> + val entry = zip.getEntry(MANIFEST_FILENAME) ?: error("missing $MANIFEST_FILENAME") + zip.getInputStream(entry).use { stream -> + val decoded = json.decodeFromStream(MigrationBundle.serializer(), stream) + check(decoded.messages.size == MESSAGE_COUNT) { + "manifest message count mismatch: ${decoded.messages.size}" + } + } + } + println("OK messages=$MESSAGE_COUNT size=${exportFile.length()}") + } + + private fun buildMessages(): List { + val content = "x".repeat(CONTENT_LENGTH) + val status = "SENT" + val identityHash = "identity-1" + return List(MESSAGE_COUNT) { i -> + MessageExport( + id = "message-$i", + conversationHash = "conversation-${i % 1000}", + identityHash = identityHash, + content = content, + timestamp = i.toLong(), + isFromMe = i % 2 == 0, + status = status, + isRead = i % 3 == 0, + fieldsJson = null, + ) + } + } +} + +class MigrationExporterOomTest { + /** + * Runs the production export zip path under a 256 MB heap cap. Fails with + * the pre-fix `encodeToString` implementation (child OOMs) and passes with + * the streaming implementation. + */ + @Test + fun `export zip manifest stays bounded under a 256MB heap (COLUMBA-DS)`() { + val javaExe = File(System.getProperty("java.home"), "bin/java").absolutePath + // The Gradle test worker's own java.class.path is not reliable: test + // classes load through a separate class loader, so the build exposes + // the test task's runtime classpath explicitly. + val classpath = + System.getProperty("columba.test.runtimeClasspath") + ?.takeIf { it.isNotBlank() } + ?: System.getProperty("java.class.path") + val process = + ProcessBuilder( + javaExe, + "-Xmx256m", + // mockk self-attaches a ByteBuddy agent in the forked JVM. + "-Djdk.attach.allowAttachSelf=true", + "-cp", + classpath, + "network.columba.app.migration.MigrationExportOomChild", + ) + .redirectErrorStream(true) + .start() + val output = process.inputStream.bufferedReader().readText() + val exitCode = process.waitFor() + assertEquals( + "createExportZip must complete under a 256MB heap (COLUMBA-DS). Child output:\n$output", + 0, + exitCode, + ) + assertTrue( + "child must report a verified manifest. Output:\n$output", + output.contains("OK messages="), + ) + } +}