Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package org.jetbrains.exposed.v1.jdbc.statements

import org.jetbrains.exposed.v1.core.*
import org.jetbrains.exposed.v1.core.statements.BatchInsertStatement
import org.jetbrains.exposed.v1.core.statements.InsertStatement
import org.jetbrains.exposed.v1.core.vendors.PostgreSQLDialect
import org.jetbrains.exposed.v1.core.vendors.currentDialect
Expand All @@ -18,8 +19,18 @@ import java.sql.SQLException
open class InsertBlockingExecutable<Key : Any, S : InsertStatement<Key>>(
override val statement: S
) : BlockingExecutable<Int, S> {
/**
* Number of rows each argument set affected, in the order the sets were submitted, or `null` if the statement
* was not executed as a batch and the counts are therefore only known in total.
*/
private var affectedRowCounts: List<Int>? = null

protected open fun JdbcPreparedStatementApi.execInsertFunction(): Pair<Int, ResultSet?> {
val inserted = if (statement.arguments().count() > 1 || isAlwaysBatch) executeBatch().sum() else executeUpdate()
val inserted = if (statement.arguments().count() > 1 || isAlwaysBatch) {
executeBatch().also { affectedRowCounts = it }.sum()
} else {
executeUpdate()
}
// According to the `processResults()` method when supportsOnlyIdentifiersInGeneratedKeys is false
// all the columns could be taken from result set
val rs = if (columnsGeneratedOnDB().isNotEmpty() || !currentDialect.supportsOnlyIdentifiersInGeneratedKeys) {
Expand Down Expand Up @@ -73,7 +84,7 @@ open class InsertBlockingExecutable<Key : Any, S : InsertStatement<Key>>(
val allResultSetsValues = rs?.returnedValues(inserted)

@Suppress("UNCHECKED_CAST")
return statement.arguments!!
return statement.arguments!!.insertedOnly(inserted, affectedRowCounts)
// Join the values from ResultSet with arguments
.mapIndexed { index, columnValues ->
val resultSetValues = allResultSetsValues?.getOrNull(index) ?: hashMapOf()
Expand All @@ -87,6 +98,30 @@ open class InsertBlockingExecutable<Key : Any, S : InsertStatement<Key>>(
.map { ResultRow.createAndFillValues(it as Map<Expression<*>, Any?>) }
}

/**
* Drops the argument sets that the database skipped instead of inserting.
*
* Only an `INSERT IGNORE` style batch can skip a row while still succeeding, so anything else keeps all of its
* arguments. A single insert keeps them too: its caller is handed the statement itself rather than these rows,
* and reads [InsertStatement.insertedCount] to find out whether the row was inserted.
*
* Dropping the skipped sets also realigns the remaining ones with the returned values, which the database only
* sends for the rows it did insert.
*
* A batch that inserted nothing is recognisable from [inserted] alone. Telling apart which rows of a partly
* inserted batch were skipped needs [perArgumentSet], and a batch that was not executed as one reports no such
* counts, so it keeps all of its arguments.
*/
private fun List<List<Pair<Column<*>, Any?>>>.insertedOnly(
inserted: Int,
perArgumentSet: List<Int>?
): List<List<Pair<Column<*>, Any?>>> {
if (statement !is BatchInsertStatement || !statement.isIgnore) return this
if (inserted == 0) return emptyList()
val counts = perArgumentSet?.takeIf { it.size == size } ?: return this
return filterIndexed { index, _ -> counts[index] != 0 }
}

private fun defaultAndNullableValues(exceptColumns: Collection<Column<*>>): Map<Column<*>, Any?> {
return statement.table.columns
.filter { column -> !exceptColumns.contains(column) }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import org.jetbrains.exposed.v1.core.*
import org.jetbrains.exposed.v1.core.dao.id.EntityID
import org.jetbrains.exposed.v1.core.dao.id.IdTable
import org.jetbrains.exposed.v1.core.dao.id.IntIdTable
import org.jetbrains.exposed.v1.core.dao.id.java.UUIDTable
import org.jetbrains.exposed.v1.core.java.UUIDColumnType
import org.jetbrains.exposed.v1.core.java.javaUUID
import org.jetbrains.exposed.v1.core.statements.BatchInsertStatement
Expand All @@ -24,6 +25,7 @@ import org.jetbrains.exposed.v1.r2dbc.statements.toExecutable
import org.jetbrains.exposed.v1.r2dbc.tests.R2dbcDatabaseTestsBase
import org.jetbrains.exposed.v1.r2dbc.tests.TestDB
import org.jetbrains.exposed.v1.r2dbc.tests.currentTestDB
import org.jetbrains.exposed.v1.r2dbc.tests.shared.assertEqualLists
import org.jetbrains.exposed.v1.r2dbc.tests.shared.assertEquals
import org.jetbrains.exposed.v1.r2dbc.tests.shared.assertFailAndRollback
import org.jetbrains.exposed.v1.r2dbc.tests.shared.assertTrue
Expand Down Expand Up @@ -228,6 +230,52 @@ class InsertTests : R2dbcDatabaseTestsBase() {
}
}

@Test
fun batchInsertWithIgnoreDoesNotReturnSkippedRows() {
val tester = object : UUIDTable("batch_ignore_tester") {
val name = varchar("name", 32)
}

withTables(excludeSettings = insertIgnoreUnsupportedDB, tester) {
val existingId = JavaUUID.randomUUID()
tester.insert {
it[tester.id] = existingId
it[tester.name] = "original"
}

val inserted = tester.batchInsert(listOf(existingId), ignore = true) { conflictingId ->
this[tester.id] = conflictingId
this[tester.name] = "conflicting"
}

assertEqualLists(tester.selectAll().map { it[tester.name] }.toList(), listOf("original"))
assertEqualLists(inserted.map { it[tester.name] }, emptyList())
}
}

@Test
fun batchInsertWithIgnorePairsGeneratedValuesWithTheRowsTheyBelongTo() {
val tester = object : IntIdTable("partial_ignore_tester") {
val name = varchar("name", 32).uniqueIndex()
}

// telling apart which entries of a partly inserted batch were skipped needs the update count of each one,
// and the H2 and MariaDB drivers send none
val noUpdateCountsPerEntry = TestDB.ALL_H2_V2 + TestDB.MARIADB

withTables(excludeSettings = insertIgnoreUnsupportedDB + noUpdateCountsPerEntry, tester) {
tester.insert { it[tester.name] = "skipped" }

val inserted = tester.batchInsert(listOf("skipped", "added"), ignore = true) { name ->
this[tester.name] = name
}

val addedId = tester.selectAll().where { tester.name eq "added" }.single()[tester.id]
assertEqualLists(inserted.map { it[tester.name] }, listOf("added"))
assertEquals(addedId, inserted.single()[tester.id])
}
}

@Test
fun testRemoveOnlyBatch() {
val statement = BatchInsertStatement(DMLTestsData.Cities)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -302,6 +302,11 @@ fun <T : Table> T.insertReturning(
/**
* Represents the SQL statement that batch inserts new rows into a table.
*
* **Note:** On H2 and MariaDB, a batch with [ignore] enabled that inserts only some of its rows returns a row for
* every value in [data] rather than only the inserted ones, and pairs the values the database generated with the
* wrong rows. Their drivers report no update count per statement, leaving nothing to tell the skipped rows apart
* by. A batch that inserts no rows at all returns an empty list on every database.
*
* @param data Collection of values to use in the batch insert.
* @param ignore Whether to ignore errors or not.
* **Note** [ignore] is not supported by all vendors. Please check the documentation.
Expand All @@ -320,6 +325,11 @@ suspend fun <T : Table, E> T.batchInsert(
/**
* Represents the SQL statement that batch inserts new rows into a table.
*
* **Note:** On H2 and MariaDB, a batch with [ignore] enabled that inserts only some of its rows returns a row for
* every value in [data] rather than only the inserted ones, and pairs the values the database generated with the
* wrong rows. Their drivers report no update count per statement, leaving nothing to tell the skipped rows apart
* by. A batch that inserts no rows at all returns an empty list on every database.
*
* @param data Sequence of values to use in the batch insert.
* @param ignore Whether to ignore errors or not.
* **Note** [ignore] is not supported by all vendors. Please check the documentation.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import io.r2dbc.spi.RowMetadata
import kotlinx.coroutines.flow.flowOf
import kotlinx.coroutines.flow.reduce
import org.jetbrains.exposed.v1.core.*
import org.jetbrains.exposed.v1.core.statements.BatchInsertStatement
import org.jetbrains.exposed.v1.core.statements.BatchReplaceStatement
import org.jetbrains.exposed.v1.core.statements.InsertStatement
import org.jetbrains.exposed.v1.core.statements.ReplaceStatement
Expand Down Expand Up @@ -50,10 +51,10 @@ open class InsertSuspendExecutable<Key : Any, S : InsertStatement<Key>>(
override suspend fun R2dbcPreparedStatementApi.executeInternal(transaction: R2dbcTransaction): Int {
val (inserted, rs) = execInsertFunction()

val (processedCount, processedResults) = processResults(rs)
val returned = rs?.returnedValues()
val affectedRowCount = inserted ?: returned?.inserted ?: 0
@OptIn(InternalApi::class)
statement.resultedValues = processedResults
val affectedRowCount = inserted ?: processedCount
statement.resultedValues = processResults(returned, affectedRowCount)
statement.insertedCount = affectedRowCount
return affectedRowCount
}
Expand Down Expand Up @@ -98,11 +99,11 @@ open class InsertSuspendExecutable<Key : Any, S : InsertStatement<Key>>(
}
}

private suspend fun processResults(rs: R2dbcResult?): Pair<Int, List<ResultRow>> {
val (count, allResultSetsValues) = rs?.returnedValues() ?: (0 to null)
private fun processResults(returned: ReturnedValues?, affectedRowCount: Int): List<ResultRow> {
val allResultSetsValues = returned?.values

@Suppress("UNCHECKED_CAST")
val results = statement.arguments!!
return statement.arguments!!.insertedOnly(affectedRowCount, returned?.perArgumentSet)
.mapIndexed { index, columnValues ->
val resultSetValues = allResultSetsValues?.getOrNull(index) ?: hashMapOf()
val argumentValues = columnValues.toMap()
Expand All @@ -113,8 +114,36 @@ open class InsertSuspendExecutable<Key : Any, S : InsertStatement<Key>>(
}
.map { unwrapColumnValues(defaultAndNullableValues(exceptColumns = it.keys)) + it }
.map { ResultRow.createAndFillValues(it as Map<Expression<*>, Any?>) }
}

private class ReturnedValues(
val inserted: Int,
val perArgumentSet: List<Int>,
val values: ArrayList<MutableMap<Column<*>, Any?>>
)

return count to results
/**
* Drops the argument sets that the database skipped instead of inserting.
*
* Only an `INSERT IGNORE` style batch can skip a row while still succeeding, so anything else keeps all of its
* arguments. A single insert keeps them too: its caller is handed the statement itself rather than these rows,
* and reads [InsertStatement.insertedCount] to find out whether the row was inserted.
*
* Dropping the skipped sets also realigns the remaining ones with the returned values, which the database only
* sends for the rows it did insert.
*
* A batch that inserted nothing is recognisable from [inserted] alone. Telling apart which rows of a partly
* inserted batch were skipped needs [perArgumentSet], and not every driver reports it: H2 and MariaDB send no
* update counts at all, so such a batch keeps all of its arguments.
*/
private fun List<List<Pair<Column<*>, Any?>>>.insertedOnly(
inserted: Int,
perArgumentSet: List<Int>?
): List<List<Pair<Column<*>, Any?>>> {
if (statement !is BatchInsertStatement || !statement.isIgnore) return this
if (inserted == 0) return emptyList()
val counts = perArgumentSet?.takeIf { it.size == size } ?: return this
return filterIndexed { index, _ -> counts[index] != 0 }
}

private fun defaultAndNullableValues(exceptColumns: Collection<Column<*>>): Map<Column<*>, Any?> {
Expand All @@ -132,7 +161,7 @@ open class InsertSuspendExecutable<Key : Any, S : InsertStatement<Key>>(
}

@Suppress("NestedBlockDepth", "TooGenericExceptionCaught", "CyclomaticComplexMethod")
private suspend fun R2dbcResult.returnedValues(): Pair<Int, ArrayList<MutableMap<Column<*>, Any?>>> {
private suspend fun R2dbcResult.returnedValues(): ReturnedValues {
val resultSetsValues = arrayListOf<MutableMap<Column<*>, Any?>>()
val resultSetsCounts = mutableListOf<Int>()
var columnIndexesInResultSet: List<Pair<Column<*>, Int>>? = null
Expand Down Expand Up @@ -246,7 +275,7 @@ open class InsertSuspendExecutable<Key : Any, S : InsertStatement<Key>>(
}
}

return inserted to resultSetsValues
return ReturnedValues(inserted, resultSetsCounts, resultSetsValues)
}

private fun RowMetadata?.returnedColumns(): List<Pair<Column<*>, Int>> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import org.jetbrains.exposed.v1.core.*
import org.jetbrains.exposed.v1.core.dao.id.EntityID
import org.jetbrains.exposed.v1.core.dao.id.IdTable
import org.jetbrains.exposed.v1.core.dao.id.IntIdTable
import org.jetbrains.exposed.v1.core.dao.id.java.UUIDTable
import org.jetbrains.exposed.v1.core.java.UUIDColumnType
import org.jetbrains.exposed.v1.core.java.javaUUID
import org.jetbrains.exposed.v1.core.statements.BatchInsertStatement
Expand Down Expand Up @@ -224,6 +225,48 @@ class InsertTests : DatabaseTestsBase() {
}
}

@Test
fun batchInsertWithIgnoreDoesNotReturnSkippedRows() {
val tester = object : UUIDTable("batch_ignore_tester") {
val name = varchar("name", 32)
}

withTables(excludeSettings = insertIgnoreUnsupportedDB, tester) {
val existingId = JavaUUID.randomUUID()
tester.insert {
it[tester.id] = existingId
it[tester.name] = "original"
}

val inserted = tester.batchInsert(listOf(existingId), ignore = true) { conflictingId ->
this[tester.id] = conflictingId
this[tester.name] = "conflicting"
}

assertEqualLists(tester.selectAll().map { it[tester.name] }, listOf("original"))
assertEqualLists(inserted.map { it[tester.name] }, emptyList())
}
}

@Test
fun batchInsertWithIgnorePairsGeneratedValuesWithTheRowsTheyBelongTo() {
val tester = object : IntIdTable("partial_ignore_tester") {
val name = varchar("name", 32).uniqueIndex()
}

withTables(excludeSettings = insertIgnoreUnsupportedDB, tester) {
tester.insert { it[tester.name] = "skipped" }

val inserted = tester.batchInsert(listOf("skipped", "added"), ignore = true) { name ->
this[tester.name] = name
}

val addedId = tester.selectAll().where { tester.name eq "added" }.single()[tester.id]
assertEqualLists(inserted.map { it[tester.name] }, listOf("added"))
assertEquals(addedId, inserted.single()[tester.id])
}
}

@Test
fun testRemoveOnlyBatch() {
val statement = BatchInsertStatement(DMLTestsData.Cities)
Expand Down
Loading