From 35109391087441c45de1e6add3e5d8e83a0806ef Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Mon, 23 Mar 2026 17:47:07 +0100 Subject: [PATCH 01/62] feature: enable auto.evolve parameter --- CHANGELOG.md | 9 +- .../connect/sink/ClickHouseSinkConfig.java | 17 + .../connect/sink/db/ClickHouseWriter.java | 132 +++++++ .../db/helper/ClickHouseHelperClient.java | 37 ++ .../kafka/connect/sink/db/mapping/Column.java | 70 ++++ .../kafka/connect/sink/db/mapping/Table.java | 13 + .../ClickHouseSinkTaskWithSchemaTest.java | 358 ++++++++++++++++++ .../connect/sink/helper/SchemaTestData.java | 272 +++++++++++++ 8 files changed, 907 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e589b8f5..70a98d143 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +# 1.3.8 (unreleased) + +## New Features +* Added `auto.evolve` configuration option for automatic table schema evolution. When enabled, the connector detects new fields in incoming records and issues `ALTER TABLE ... ADD COLUMN IF NOT EXISTS` against ClickHouse. Disabled by default. (https://github.com/ClickHouse/clickhouse-kafka-connect/issues/277) + +## Bug Fixes +* Fixed RowBinary serialization for Map columns with Nullable value types. The nullable marker byte was missing when writing map values, causing `CANNOT_READ_ALL_DATA` errors for `Map(K, Nullable(V))` columns. + # 1.3.7, 2026-03-25 ## Security @@ -6,7 +14,6 @@ # Improvements * `Gson` replaced with `Jackson` for performance and better maintainability (https://github.com/ClickHouse/clickhouse-kafka-connect/pull/676). - # 1.3.6, 2026-03-18 ## New Features diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java b/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java index da6fafdb7..e4498fedc 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java @@ -57,6 +57,7 @@ public class ClickHouseSinkConfig { public static final String REPORT_INSERTED_OFFSETS = "reportInsertedOffsets"; public static final String ERROR_TOLERANCE_ALL = "all"; public static final String ERROR_TOLERANCE_NONE = "none"; + public static final String AUTO_EVOLVE = "auto.evolve"; public static final String CONNECTOR_RETRY_TIMEOUT = "errors.retry.timeout"; public static final long MINIMAL_RETRY_TIMEOUT_THR_WARN = TimeUnit.SECONDS.toMillis(10); @@ -110,6 +111,7 @@ public class ClickHouseSinkConfig { private final int bufferCount; private final long bufferFlushTime; private final boolean reportInsertedOffsets; + private final boolean autoEvolve; private final boolean binaryFormatWrtiteJsonAsString; private final String sslSocketSni; @@ -296,6 +298,8 @@ public ClickHouseSinkConfig(Map props) { LOGGER.info("Internal buffering enabled: bufferCount={}, bufferFlushTime={}ms", this.bufferCount, this.bufferFlushTime); } + this.autoEvolve = Boolean.parseBoolean(props.getOrDefault(AUTO_EVOLVE, "false")); + String jsonAsString = getClickhouseSettings().get("input_format_binary_read_json_as_string"); this.binaryFormatWrtiteJsonAsString = jsonAsString != null && (jsonAsString.equalsIgnoreCase("true") || jsonAsString.equals("1")); @@ -692,6 +696,19 @@ private static ConfigDef createConfigDef() { ConfigDef.Width.MEDIUM, "SSL Socket SNI" ); + + String ddlGroup = "DDL"; + int ddlOrderInGroup = 0; + configDef.define(AUTO_EVOLVE, + ConfigDef.Type.BOOLEAN, + false, + ConfigDef.Importance.MEDIUM, + "Whether to automatically add columns to the destination table when a record contains fields not present in the table. default: false", + ddlGroup, + ++ddlOrderInGroup, + ConfigDef.Width.SHORT, + "Auto evolve table schema." + ); return configDef; } } diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index 9e569240e..d9e28953f 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -33,6 +33,7 @@ import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.errors.DataException; +import org.apache.kafka.connect.sink.SinkRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -207,6 +208,52 @@ public void doInsert(List records, QueryIdentifier queryId, ErrorReporte String database = first.getDatabase(); Table table = getTable(database, topic); if (table == null) { return; }//We checked the error flag in getTable, so we don't need to check it again here + + if (csc.isAutoEvolve()) { + table = doInsertWithSchemaEvolution(records, table, queryId); + } else { + doInsertBatch(records, table, queryId); + } + } + + private Table doInsertWithSchemaEvolution(List records, Table table, QueryIdentifier queryId) throws IOException, ExecutionException, InterruptedException { + // Split records into sub batches at schema boundaries (like JDBC BufferedRecords.add pattern) + // When schema changes mid batch the current sub batch is flushed, the table is evolved, and the insertion continues. + Schema currentSchema = getValueSchema(records.get(0)); + int batchStart = 0; + + for (int i = 1; i <= records.size(); i++) { + Schema recordSchema = (i < records.size()) ? getValueSchema(records.get(i)) : null; + + // Flush sub batch when schema changes or the end of the records is reached + if (i == records.size() || !Objects.equals(currentSchema, recordSchema)) { + List subBatch = records.subList(batchStart, i); + Record subFirst = subBatch.get(0); + + // Evolve table for the sub batch schema + table = evolveTableSchema(table, subFirst); + + LOGGER.debug("Inserting sub-batch [{}-{}) of {} records with schema evolution (QueryId: [{}])", + batchStart, i, subBatch.size(), queryId.getQueryId()); + doInsertBatch(subBatch, table, queryId); + + if (i < records.size()) { + currentSchema = recordSchema; + batchStart = i; + } + } + } + + return table; + } + + private static Schema getValueSchema(Record record) { + SinkRecord sr = record.getSinkRecord(); + return sr != null ? sr.valueSchema() : null; + } + + private void doInsertBatch(List records, Table table, QueryIdentifier queryId) throws IOException, ExecutionException, InterruptedException { + Record first = records.get(0); LOGGER.debug("Trying to insert [{}] records to table name [{}] (QueryId: [{}])", records.size(), table.getName(), queryId.getQueryId()); switch (first.getSchemaType()) { case SCHEMA: @@ -494,6 +541,9 @@ protected void doWriteColValue(Column col, OutputStream stream, Data value, bool mapTmp.forEach((key, mapValue) -> { try { doWritePrimitive(col.getMapKeyType(), value.getMapKeySchema().type(), stream, key, col); + if (col.getMapValueType() != null && col.getMapValueType().isNullable() && mapValue != null) { + BinaryStreamUtils.writeNonNull(stream); + } doWriteColValue(col.getMapValueType(), stream, new Data(value.getNestedValueSchema(), mapValue), defaultsSupport); } catch (IOException e) { throw new RuntimeException(e); @@ -821,6 +871,88 @@ protected void doWriteCol(Data value, boolean fieldExists, Column col, OutputStr } } + protected Table evolveTableSchema(Table table, Record record) { + if (record.getFields() == null) { + LOGGER.warn("Cannot auto-evolve schema for records without a Connect schema (schemaless/string). Skipping schema evolution."); + return table; + } + + List fieldNames = record.getFields().stream().map(Field::name).collect(Collectors.toList()); + Set missingColumns = table.getMissingColumns(fieldNames); + + if (missingColumns.isEmpty()) { + return table; + } + + LOGGER.info("Detected {} new field(s) not present in table {}: {}", missingColumns.size(), table.getName(), missingColumns); + + Map schemaMap = record.getFields().stream().collect(Collectors.toMap(Field::name, Field::schema)); + List columnDefs = new java.util.ArrayList<>(); + + for (String fieldName : missingColumns) { + Schema fieldSchema = schemaMap.get(fieldName); + if (fieldSchema == null) { + continue; + } + + if (!fieldSchema.isOptional() && fieldSchema.defaultValue() == null) { + throw new RuntimeException(String.format( + "Cannot auto-evolve: field '%s' is not optional and has no default value. " + + "ClickHouse requires new columns to be either Nullable or have a DEFAULT.", fieldName)); + } + + String chType; + try { + chType = Column.connectTypeToClickHouseType(fieldSchema); + } catch (RuntimeException e) { + throw new RuntimeException(String.format( + "Cannot auto-evolve: field '%s' has unsupported type for auto-evolution. %s", fieldName, e.getMessage()), e); + } + + // ClickHouse does not allow Nullable wrapping for Array and Map types + if (fieldSchema.isOptional() + && fieldSchema.type() != Schema.Type.ARRAY + && fieldSchema.type() != Schema.Type.MAP) { + chType = "Nullable(" + chType + ")"; + } + + columnDefs.add(String.format("`%s` %s", fieldName, chType)); + } + + if (!columnDefs.isEmpty()) { + chc.alterTableAddColumns(table.getDatabase(), table.getCleanName(), columnDefs); + LOGGER.info("Schema evolution complete for table {}. Added columns: {}", table.getName(), columnDefs); + table = refreshTableAfterDDL(table, missingColumns); + } + + return table; + } + + private static final int DDL_REFRESH_MAX_RETRIES = 5; + private static final long DDL_REFRESH_BACKOFF_MS = 200; + + private Table refreshTableAfterDDL(Table table, Set expectedNewColumns) { + for (int attempt = 0; attempt < DDL_REFRESH_MAX_RETRIES; attempt++) { + Table refreshed = urgentTableUpdate(table); + Set stillMissing = refreshed.getMissingColumns(expectedNewColumns); + if (stillMissing.isEmpty()) { + return refreshed; + } + LOGGER.warn("DDL refresh attempt {}/{}: columns {} not yet visible, retrying in {}ms", + attempt + 1, DDL_REFRESH_MAX_RETRIES, stillMissing, DDL_REFRESH_BACKOFF_MS); + try { + Thread.sleep(DDL_REFRESH_BACKOFF_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted while waiting for DDL propagation", e); + } + } + // Final attempt, use whatever we have + LOGGER.error("DDL propagation timeout: some columns may not be visible yet after {} retries. Proceeding with latest table state.", + DDL_REFRESH_MAX_RETRIES); + return urgentTableUpdate(table); + } + protected void doInsertRawBinary(List records, Table table, QueryIdentifier queryId, boolean supportDefaults, boolean retry) throws IOException, ExecutionException, InterruptedException { try { if (chc.isUseClientV2()) { diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java index 9c8a201af..0498172cc 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java @@ -475,6 +475,43 @@ public Table describeTableV2(String database, String tableName) { return table; } + public void alterTableAddColumns(String database, String tableName, List columnDefs) { + for (String colDef : columnDefs) { + String sql = String.format("ALTER TABLE `%s`.`%s` ADD COLUMN IF NOT EXISTS %s", database, tableName, colDef); + LOGGER.info("Executing DDL: {}", sql); + if (useClientV2) { + alterTableAddColumnV2(sql); + } else { + alterTableAddColumnV1(sql); + } + } + } + + private void alterTableAddColumnV1(String sql) { + try (ClickHouseClient client = ClickHouseClient.builder() + .options(getDefaultClientOptions()) + .nodeSelector(ClickHouseNodeSelector.of(ClickHouseProtocol.HTTP)) + .build(); + ClickHouseResponse response = client.read(server) + .query(sql) + .set("alter_sync", "1") + .executeAndWait()) { + // DDL executed; alter_sync=1 waits for the local replica to apply + } catch (ClickHouseException e) { + throw new RuntimeException("Failed to execute ALTER TABLE: " + sql, e); + } + } + + private void alterTableAddColumnV2(String sql) { + try { + QuerySettings settings = new QuerySettings(); + settings.serverSetting("alter_sync", "1"); + client.query(sql, settings).get(); + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException("Failed to execute ALTER TABLE: " + sql, e); + } + } + public List extractTablesMapping(String database, Map cache) { List
tableList = new ArrayList<>(); for (Table table : showTables(database)) { diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java index dab474146..6f0fb6b1a 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java @@ -6,6 +6,11 @@ import lombok.Getter; import lombok.Setter; import lombok.experimental.Accessors; +import org.apache.kafka.connect.data.Date; +import org.apache.kafka.connect.data.Decimal; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.Time; +import org.apache.kafka.connect.data.Timestamp; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.clickhouse.kafka.connect.util.reactor.function.Tuple2; @@ -358,6 +363,71 @@ private static Map extractEnumValues(String valueType) { return data; } + public static String connectTypeToClickHouseType(Schema connectSchema) { + // Check logical types first (same pattern as JDBC connector) + if (connectSchema.name() != null) { + switch (connectSchema.name()) { + case Decimal.LOGICAL_NAME: + int precision = 38; // ClickHouse Decimal128 default + int scale = 0; + if (connectSchema.parameters() != null && connectSchema.parameters().containsKey("scale")) { + scale = Integer.parseInt(connectSchema.parameters().get("scale")); + } + return String.format("Decimal(%d, %d)", precision, scale); + case Date.LOGICAL_NAME: + return "Date32"; + case Time.LOGICAL_NAME: + return "Int64"; + case Timestamp.LOGICAL_NAME: + return "DateTime64(3)"; + } + } + + // Then check primitive types + switch (connectSchema.type()) { + case INT8: + return "Int8"; + case INT16: + return "Int16"; + case INT32: + return "Int32"; + case INT64: + return "Int64"; + case FLOAT32: + return "Float32"; + case FLOAT64: + return "Float64"; + case BOOLEAN: + return "Bool"; + case STRING: + return "String"; + case BYTES: + return "String"; + case ARRAY: + if (connectSchema.valueSchema() == null) { + return "Array(String)"; + } + String elementType = connectTypeToClickHouseType(connectSchema.valueSchema()); + if (connectSchema.valueSchema().isOptional()) { + elementType = "Nullable(" + elementType + ")"; + } + return "Array(" + elementType + ")"; + case MAP: + String keyType = connectTypeToClickHouseType(connectSchema.keySchema()); + String valType = connectTypeToClickHouseType(connectSchema.valueSchema()); + if (connectSchema.valueSchema().isOptional()) { + valType = "Nullable(" + valType + ")"; + } + return "Map(" + keyType + ", " + valType + ")"; + case STRUCT: + throw new RuntimeException( + "Cannot auto-evolve STRUCT fields to ClickHouse columns. " + + "STRUCT type requires manual mapping to Tuple, JSON, or Nested type."); + default: + throw new RuntimeException("Unsupported Connect type for auto-evolution: " + connectSchema.type()); + } + } + public Integer convertEnumValues(String value) { if ( this.enumValues != null ) { return enumValues.get(value); diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Table.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Table.java index 8bb1844f2..747fc14e7 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Table.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Table.java @@ -8,9 +8,12 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -78,6 +81,16 @@ public void addColumn(Column column) { } } + public Set getMissingColumns(Collection fieldNames) { + Set missing = new LinkedHashSet<>(); + for (String fieldName : fieldNames) { + if (!rootColumnsMap.containsKey(fieldName)) { + missing.add(fieldName); + } + } + return missing; + } + private void handleNonRoot(Column column) { String parentName = column.getName().substring(0, column.getName().lastIndexOf(".")); Column parent = allColumnsMap.getOrDefault(parentName, null); diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index 96401554f..39fe97b15 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -1802,4 +1802,362 @@ public void testAvroDateAndTimeTypes() throws Exception { assertEquals(event.getTime2().atDate(LocalDate.of(1970, 1, 1)).format(localFormatter), row.get("time2")); } } + + @Test + public void autoEvolveDisabledRejectsNewField() { + Map props = createProps(); + // auto.evolve defaults to false + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_disabled_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Insert V1 records first (should succeed) + Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV1); + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Insert V2 records with new field (should succeed because input_format_skip_unknown_fields=1) + // But the new column should NOT be added to the table + Collection srV2 = SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 10); + chst.put(srV2); + chst.stop(); + + // Rows inserted but new column should not exist + assertEquals(20, ClickHouseTestHelpers.countRows(chc, topic)); + } + + @Test + public void autoEvolveAddsNullableColumn() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_nullable_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Insert V1 records + Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV1); + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Insert V2 records with new nullable field -> should trigger ALTER TABLE ADD COLUMN + Collection srV2 = SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 10); + chst.put(srV2); + chst.stop(); + + assertEquals(20, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify the new column exists + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("new_string_field"), + "New column 'new_string_field' should have been added by auto.evolve"); + } + + @Test + public void autoEvolveRejectsNonNullableNoDefault() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_reject_non_nullable_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + Collection srV2 = SchemaTestData.createSchemaV2WithNewNonNullableField(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + + try { + chst.put(srV2); + // Should have thrown + assertTrue(false, "Expected exception for non-nullable field without default"); + } catch (RuntimeException e) { + // Walk the full cause chain. Utils.handleException wraps multiple times + Throwable t = e; + boolean found = false; + while (t != null) { + if (t.getMessage() != null && (t.getMessage().contains("not optional") || t.getMessage().contains("Cannot auto-evolve"))) { + found = true; + break; + } + t = t.getCause(); + } + assertTrue(found, "Expected descriptive error about non-nullable field in cause chain, got: " + e.getMessage()); + } finally { + chst.stop(); + } + } + + @Test + public void autoEvolveMultipleNewColumns() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_multi_cols_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Insert V1 first + Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV1); + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Insert V2 with multiple new nullable fields + Collection srV2 = SchemaTestData.createSchemaV2WithMultipleNewNullableFields(topic, 1, 10); + chst.put(srV2); + chst.stop(); + + assertEquals(20, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify all new columns exist + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("new_string_field"), + "Column 'new_string_field' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("new_int32_field"), + "Column 'new_int32_field' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("new_float64_field"), + "Column 'new_float64_field' should exist"); + } + + @Test + public void autoEvolveCachesSchemaAfterDDL() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_cache_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + + // First batch triggers DDL + Collection srV2batch1 = SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 10); + chst.put(srV2batch1); + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Second batch with same schema should not re-trigger DDL (just insert) + // Use partition 2 to avoid offset deduplication + Collection srV2batch2 = SchemaTestData.createSchemaV2WithNewNullableField(topic, 2, 10); + chst.put(srV2batch2); + chst.stop(); + + assertEquals(20, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify column exists (only one 'new_string_field' column) + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + long count = described.getRootColumnsList().stream() + .filter(c -> c.getName().equals("new_string_field")) + .count(); + assertEquals(1, count, "Should have exactly one 'new_string_field' column"); + } + + @Test + public void autoEvolveMixedSchemaInSingleBatch() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_mixed_batch_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Build a single batch with V1 records followed by V2 records (mixed schemas) + List mixedBatch = new ArrayList<>(); + mixedBatch.addAll(SchemaTestData.createSchemaV1(topic, 1, 5)); + mixedBatch.addAll(SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 5)); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(mixedBatch); + chst.stop(); + + // All 10 records should be inserted + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // The new column should exist (evolved from V2 records in same batch) + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("new_string_field"), + "Column 'new_string_field' should be added even when schema changes mid-batch"); + } + + @Test + public void autoEvolveLogicalTypes() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_logical_types_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Insert V1 first + Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV1); + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Insert V2 with logical type fields (Decimal, Date, Timestamp) + Collection srV2 = SchemaTestData.createSchemaV2WithLogicalTypes(topic, 1, 10); + chst.put(srV2); + chst.stop(); + + assertEquals(20, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify the logical type columns were created with correct ClickHouse types + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("new_decimal_field"), + "Column 'new_decimal_field' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("new_date_field"), + "Column 'new_date_field' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("new_timestamp_field"), + "Column 'new_timestamp_field' should exist"); + + // Verify types + com.clickhouse.kafka.connect.sink.db.mapping.Column decimalCol = described.getRootColumnsMap().get("new_decimal_field"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.Decimal, decimalCol.getType(), + "Decimal logical type should map to ClickHouse Decimal"); + + com.clickhouse.kafka.connect.sink.db.mapping.Column dateCol = described.getRootColumnsMap().get("new_date_field"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.Date32, dateCol.getType(), + "Date logical type should map to ClickHouse Date32"); + + com.clickhouse.kafka.connect.sink.db.mapping.Column tsCol = described.getRootColumnsMap().get("new_timestamp_field"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.DateTime64, tsCol.getType(), + "Timestamp logical type should map to ClickHouse DateTime64"); + } + + @Test + public void autoEvolveRejectsStructField() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_struct_reject_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + Collection srV2 = SchemaTestData.createSchemaV2WithStructField(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + + try { + chst.put(srV2); + assertTrue(false, "Expected exception for STRUCT field auto-evolution"); + } catch (RuntimeException e) { + Throwable t = e; + boolean found = false; + while (t != null) { + if (t.getMessage() != null && t.getMessage().contains("Cannot auto-evolve STRUCT")) { + found = true; + break; + } + t = t.getCause(); + } + assertTrue(found, "Should reject STRUCT field with appropriate message, got: " + e.getMessage()); + } finally { + chst.stop(); + } + } + + @Test + public void autoEvolveArrayAndMapFields() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_array_map_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Insert V1 first + Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV1); + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Insert V2 with Array and Map fields + Collection srV2 = SchemaTestData.createSchemaV2WithArrayAndMapFields(topic, 1, 10); + chst.put(srV2); + chst.stop(); + + assertEquals(20, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify array and map columns were created + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("new_array_field"), + "Column 'new_array_field' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("new_map_field"), + "Column 'new_map_field' should exist"); + } + + @Test + public void autoEvolveTripleSchemaInOneBatch() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_triple_schema_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Build a single batch with V1 + V2 + V3 records + List combined = new ArrayList<>(); + combined.addAll(SchemaTestData.createSchemaV1(topic, 1, 5)); + combined.addAll(SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 5)); + combined.addAll(SchemaTestData.createSchemaV3WithExtraField(topic, 1, 5)); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(combined); + chst.stop(); + + assertEquals(15, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify columns from both V2 and V3 exist + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("new_string_field"), + "V2 column 'new_string_field' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("v3_bool_field"), + "V3 column 'v3_bool_field' should exist"); + } + + @Test + public void autoEvolveSchemalessRecordsSkipped() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_schemaless_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Create schemaless (string) records. No valueSchema. + List schemaless = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + String json = String.format("{\"off16\": %d, \"p_int64\": %d}", i, (long) i); + schemaless.add(new SinkRecord( + topic, 1, null, null, null, json, + i, System.currentTimeMillis(), TimestampType.CREATE_TIME + )); + } + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(schemaless); + chst.stop(); + } } diff --git a/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java b/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java index 39bea1b57..3b861ebe7 100644 --- a/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java +++ b/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java @@ -1585,4 +1585,276 @@ public static List convertAvroToSinkRecord(String topic, ParsedSchem schemaAndValues.add(new SinkRecord(topic, 0, null, null, schemaAndValue.schema(), schemaAndValue.value(), schemaAndValues.size())), ArrayList::addAll); } + + public static Collection createSchemaV1(String topic, int partition) { + return createSchemaV1(topic, partition, DEFAULT_TOTAL_RECORDS); + } + + public static Collection createSchemaV1(String topic, int partition, int totalRecords) { + List array = new ArrayList<>(); + + Schema SCHEMA_V1 = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .build(); + + LongStream.range(0, totalRecords).forEachOrdered(n -> { + Struct value_struct = new Struct(SCHEMA_V1) + .put("off16", (short) n) + .put("p_int64", n); + + array.add(new SinkRecord( + topic, + partition, + null, + null, SCHEMA_V1, + value_struct, + n, + System.currentTimeMillis(), + TimestampType.CREATE_TIME + )); + }); + return array; + } + + public static Collection createSchemaV2WithNewNullableField(String topic, int partition) { + return createSchemaV2WithNewNullableField(topic, partition, DEFAULT_TOTAL_RECORDS); + } + + public static Collection createSchemaV2WithNewNullableField(String topic, int partition, int totalRecords) { + List array = new ArrayList<>(); + + Schema SCHEMA_V2 = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("new_string_field", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + + long offset = totalRecords; // continue offsets from V1 + for (long n = 0; n < totalRecords; n++) { + Struct value_struct = new Struct(SCHEMA_V2) + .put("off16", (short) n) + .put("p_int64", n) + .put("new_string_field", "value_" + n); + + array.add(new SinkRecord( + topic, + partition, + null, + null, SCHEMA_V2, + value_struct, + offset + n, + System.currentTimeMillis(), + TimestampType.CREATE_TIME + )); + } + return array; + } + + public static Collection createSchemaV2WithNewNonNullableField(String topic, int partition) { + return createSchemaV2WithNewNonNullableField(topic, partition, DEFAULT_TOTAL_RECORDS); + } + + public static Collection createSchemaV2WithNewNonNullableField(String topic, int partition, int totalRecords) { + List array = new ArrayList<>(); + + Schema SCHEMA_V2 = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("non_nullable_field", Schema.STRING_SCHEMA) + .build(); + + long offset = totalRecords; + for (long n = 0; n < totalRecords; n++) { + Struct value_struct = new Struct(SCHEMA_V2) + .put("off16", (short) n) + .put("p_int64", n) + .put("non_nullable_field", "required_" + n); + + array.add(new SinkRecord( + topic, + partition, + null, + null, SCHEMA_V2, + value_struct, + offset + n, + System.currentTimeMillis(), + TimestampType.CREATE_TIME + )); + } + return array; + } + + public static Collection createSchemaV2WithMultipleNewNullableFields(String topic, int partition) { + return createSchemaV2WithMultipleNewNullableFields(topic, partition, DEFAULT_TOTAL_RECORDS); + } + + public static Collection createSchemaV2WithMultipleNewNullableFields(String topic, int partition, int totalRecords) { + List array = new ArrayList<>(); + + Schema SCHEMA_V2 = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("new_string_field", Schema.OPTIONAL_STRING_SCHEMA) + .field("new_int32_field", SchemaBuilder.int32().optional().build()) + .field("new_float64_field", SchemaBuilder.float64().optional().build()) + .build(); + + long offset = totalRecords; + for (long n = 0; n < totalRecords; n++) { + Struct value_struct = new Struct(SCHEMA_V2) + .put("off16", (short) n) + .put("p_int64", n) + .put("new_string_field", "val_" + n) + .put("new_int32_field", (int) n) + .put("new_float64_field", n * 1.5); + + array.add(new SinkRecord( + topic, + partition, + null, + null, SCHEMA_V2, + value_struct, + offset + n, + System.currentTimeMillis(), + TimestampType.CREATE_TIME + )); + } + return array; + } + + public static Collection createSchemaV2WithStructField(String topic, int partition) { + return createSchemaV2WithStructField(topic, partition, DEFAULT_TOTAL_RECORDS); + } + + public static Collection createSchemaV2WithStructField(String topic, int partition, int totalRecords) { + List array = new ArrayList<>(); + + Schema INNER_SCHEMA = SchemaBuilder.struct() + .field("nested_str", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + + Schema SCHEMA_V2 = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("new_struct_field", SchemaBuilder.struct() + .field("nested_str", Schema.OPTIONAL_STRING_SCHEMA) + .optional() + .build()) + .build(); + + long offset = totalRecords; + for (long n = 0; n < totalRecords; n++) { + Struct nested = new Struct(SCHEMA_V2.field("new_struct_field").schema()) + .put("nested_str", "nested_" + n); + Struct value_struct = new Struct(SCHEMA_V2) + .put("off16", (short) n) + .put("p_int64", n) + .put("new_struct_field", nested); + + array.add(new SinkRecord( + topic, partition, null, null, SCHEMA_V2, value_struct, + offset + n, System.currentTimeMillis(), TimestampType.CREATE_TIME + )); + } + return array; + } + + public static Collection createSchemaV2WithArrayAndMapFields(String topic, int partition) { + return createSchemaV2WithArrayAndMapFields(topic, partition, DEFAULT_TOTAL_RECORDS); + } + + public static Collection createSchemaV2WithArrayAndMapFields(String topic, int partition, int totalRecords) { + List array = new ArrayList<>(); + + Schema SCHEMA_V2 = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("new_array_field", SchemaBuilder.array(Schema.OPTIONAL_STRING_SCHEMA).optional().build()) + .field("new_map_field", SchemaBuilder.map(Schema.STRING_SCHEMA, SchemaBuilder.int32().optional().build()).optional().build()) + .build(); + + long offset = totalRecords; + for (long n = 0; n < totalRecords; n++) { + Struct value_struct = new Struct(SCHEMA_V2) + .put("off16", (short) n) + .put("p_int64", n) + .put("new_array_field", Arrays.asList("a_" + n, "b_" + n)) + .put("new_map_field", Collections.singletonMap("key_" + n, (int) n)); + + array.add(new SinkRecord( + topic, partition, null, null, SCHEMA_V2, value_struct, + offset + n, System.currentTimeMillis(), TimestampType.CREATE_TIME + )); + } + return array; + } + + public static Collection createSchemaV3WithExtraField(String topic, int partition) { + return createSchemaV3WithExtraField(topic, partition, DEFAULT_TOTAL_RECORDS); + } + + public static Collection createSchemaV3WithExtraField(String topic, int partition, int totalRecords) { + List array = new ArrayList<>(); + + Schema SCHEMA_V3 = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("new_string_field", Schema.OPTIONAL_STRING_SCHEMA) + .field("v3_bool_field", SchemaBuilder.bool().optional().build()) + .build(); + + long offset = totalRecords * 2L; // after V1 and V2 offsets + for (long n = 0; n < totalRecords; n++) { + Struct value_struct = new Struct(SCHEMA_V3) + .put("off16", (short) n) + .put("p_int64", n) + .put("new_string_field", "v3_" + n) + .put("v3_bool_field", n % 2 == 0); + + array.add(new SinkRecord( + topic, partition, null, null, SCHEMA_V3, value_struct, + offset + n, System.currentTimeMillis(), TimestampType.CREATE_TIME + )); + } + return array; + } + + public static Collection createSchemaV2WithLogicalTypes(String topic, int partition) { + return createSchemaV2WithLogicalTypes(topic, partition, DEFAULT_TOTAL_RECORDS); + } + + public static Collection createSchemaV2WithLogicalTypes(String topic, int partition, int totalRecords) { + List array = new ArrayList<>(); + + Schema SCHEMA_V2 = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("new_decimal_field", Decimal.builder(2).optional().build()) + .field("new_date_field", org.apache.kafka.connect.data.Date.builder().optional().build()) + .field("new_timestamp_field", Timestamp.builder().optional().build()) + .build(); + + long offset = totalRecords; + for (long n = 0; n < totalRecords; n++) { + Struct value_struct = new Struct(SCHEMA_V2) + .put("off16", (short) n) + .put("p_int64", n) + .put("new_decimal_field", BigDecimal.valueOf(n * 100 + 50, 2)) + .put("new_date_field", Date.from(java.time.Instant.ofEpochMilli(n * 86400000L))) + .put("new_timestamp_field", Date.from(java.time.Instant.ofEpochMilli(System.currentTimeMillis()))); + + array.add(new SinkRecord( + topic, + partition, + null, + null, SCHEMA_V2, + value_struct, + offset + n, + System.currentTimeMillis(), + TimestampType.CREATE_TIME + )); + } + return array; + } } From 060a44d1a727b471dfe97f152de3f47f5c37ba59 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 10:57:00 +0100 Subject: [PATCH 02/62] fix: avoid query connection leak --- .../connect/sink/db/helper/ClickHouseHelperClient.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java index 0498172cc..29d51b9e4 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java @@ -503,10 +503,8 @@ private void alterTableAddColumnV1(String sql) { } private void alterTableAddColumnV2(String sql) { - try { - QuerySettings settings = new QuerySettings(); - settings.serverSetting("alter_sync", "1"); - client.query(sql, settings).get(); + try (QueryResponse response = client.query(sql, new QuerySettings().serverSetting("alter_sync", "1")).get()) { + // DDL executed; alter_sync=1 waits for the local replica to apply } catch (ExecutionException | InterruptedException e) { throw new RuntimeException("Failed to execute ALTER TABLE: " + sql, e); } From fe8c6beef2060fd636f590891437fd182fc0168e Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 11:05:09 +0100 Subject: [PATCH 03/62] chore: move value to static field to make it more clear --- .../com/clickhouse/kafka/connect/sink/db/mapping/Column.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java index 6f0fb6b1a..d0294d325 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java @@ -34,6 +34,7 @@ public class Column { private static final Pattern DECIMAL_TYPE_PATTERN = Pattern.compile("Decimal(?\\d{2,3})?\\s*(\\((?\\d{1,}\\s*)?,*\\s*(?\\d{1,})?\\))?"); private static final Pattern SIMPLE_AGGREGATE_FUNCTION_TYPE_PATTERN = Pattern.compile("^SimpleAggregateFunction\\s*\\([^,]+,\\s*(.+)\\)$"); + private static final int DECIMAL128_MAX_PRECISION = 38; private static final Logger LOGGER = LoggerFactory.getLogger(Column.class); private String name; @@ -368,7 +369,7 @@ public static String connectTypeToClickHouseType(Schema connectSchema) { if (connectSchema.name() != null) { switch (connectSchema.name()) { case Decimal.LOGICAL_NAME: - int precision = 38; // ClickHouse Decimal128 default + int precision = DECIMAL128_MAX_PRECISION; int scale = 0; if (connectSchema.parameters() != null && connectSchema.parameters().containsKey("scale")) { scale = Integer.parseInt(connectSchema.parameters().get("scale")); From 90807e7b45830e6a4d0c0d341a6aa3bfff7ad666 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 11:18:48 +0100 Subject: [PATCH 04/62] fix: create custom type inference exception --- .../clickhouse/kafka/connect/sink/db/mapping/Column.java | 4 ++-- .../sink/db/mapping/SchemaTypeInferenceException.java | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/SchemaTypeInferenceException.java diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java index d0294d325..b5af42806 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java @@ -421,11 +421,11 @@ public static String connectTypeToClickHouseType(Schema connectSchema) { } return "Map(" + keyType + ", " + valType + ")"; case STRUCT: - throw new RuntimeException( + throw new SchemaTypeInferenceException( "Cannot auto-evolve STRUCT fields to ClickHouse columns. " + "STRUCT type requires manual mapping to Tuple, JSON, or Nested type."); default: - throw new RuntimeException("Unsupported Connect type for auto-evolution: " + connectSchema.type()); + throw new SchemaTypeInferenceException("Unsupported Connect type for auto-evolution: " + connectSchema.type()); } } diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/SchemaTypeInferenceException.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/SchemaTypeInferenceException.java new file mode 100644 index 000000000..a718713aa --- /dev/null +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/SchemaTypeInferenceException.java @@ -0,0 +1,7 @@ +package com.clickhouse.kafka.connect.sink.db.mapping; + +public class SchemaTypeInferenceException extends RuntimeException { + public SchemaTypeInferenceException(String message) { + super(message); + } +} From c95ecb0baf37096d38bc557bfb89ce92df349fb4 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 11:27:48 +0100 Subject: [PATCH 05/62] fix: dont wrap connectTypeToClickHouseType in try-catch since method already throws exception --- .../kafka/connect/sink/db/ClickHouseWriter.java | 8 +------- .../connect/sink/db/helper/ClickHouseHelperClient.java | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index d9e28953f..78a823ec4 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -901,13 +901,7 @@ protected Table evolveTableSchema(Table table, Record record) { "ClickHouse requires new columns to be either Nullable or have a DEFAULT.", fieldName)); } - String chType; - try { - chType = Column.connectTypeToClickHouseType(fieldSchema); - } catch (RuntimeException e) { - throw new RuntimeException(String.format( - "Cannot auto-evolve: field '%s' has unsupported type for auto-evolution. %s", fieldName, e.getMessage()), e); - } + String chType = Column.connectTypeToClickHouseType(fieldSchema); // ClickHouse does not allow Nullable wrapping for Array and Map types if (fieldSchema.isOptional() diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java index 29d51b9e4..382ebc880 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java @@ -505,7 +505,7 @@ private void alterTableAddColumnV1(String sql) { private void alterTableAddColumnV2(String sql) { try (QueryResponse response = client.query(sql, new QuerySettings().serverSetting("alter_sync", "1")).get()) { // DDL executed; alter_sync=1 waits for the local replica to apply - } catch (ExecutionException | InterruptedException e) { + } catch (Exception e) { throw new RuntimeException("Failed to execute ALTER TABLE: " + sql, e); } } From 197cef2a0548630efc6cdbb714f6616fad677932 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 11:55:19 +0100 Subject: [PATCH 06/62] fix: move Nullable wrapping into connectTypeToClickHouseType and always create Nullable columns --- .../connect/sink/db/ClickHouseWriter.java | 14 -------- .../kafka/connect/sink/db/mapping/Column.java | 19 +++++++---- .../ClickHouseSinkTaskWithSchemaTest.java | 32 +++++++------------ 3 files changed, 23 insertions(+), 42 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index 78a823ec4..afce8f98c 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -895,21 +895,7 @@ protected Table evolveTableSchema(Table table, Record record) { continue; } - if (!fieldSchema.isOptional() && fieldSchema.defaultValue() == null) { - throw new RuntimeException(String.format( - "Cannot auto-evolve: field '%s' is not optional and has no default value. " + - "ClickHouse requires new columns to be either Nullable or have a DEFAULT.", fieldName)); - } - String chType = Column.connectTypeToClickHouseType(fieldSchema); - - // ClickHouse does not allow Nullable wrapping for Array and Map types - if (fieldSchema.isOptional() - && fieldSchema.type() != Schema.Type.ARRAY - && fieldSchema.type() != Schema.Type.MAP) { - chType = "Nullable(" + chType + ")"; - } - columnDefs.add(String.format("`%s` %s", fieldName, chType)); } diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java index b5af42806..854d5e2c5 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java @@ -365,6 +365,17 @@ private static Map extractEnumValues(String valueType) { } public static String connectTypeToClickHouseType(Schema connectSchema) { + String baseType = resolveBaseType(connectSchema); + + // ClickHouse forbids Nullable wrapping for Array and Map types + if (connectSchema.type() == Schema.Type.ARRAY || connectSchema.type() == Schema.Type.MAP) { + return baseType; + } + + return "Nullable(" + baseType + ")"; + } + + private static String resolveBaseType(Schema connectSchema) { // Check logical types first (same pattern as JDBC connector) if (connectSchema.name() != null) { switch (connectSchema.name()) { @@ -409,16 +420,10 @@ public static String connectTypeToClickHouseType(Schema connectSchema) { return "Array(String)"; } String elementType = connectTypeToClickHouseType(connectSchema.valueSchema()); - if (connectSchema.valueSchema().isOptional()) { - elementType = "Nullable(" + elementType + ")"; - } return "Array(" + elementType + ")"; case MAP: - String keyType = connectTypeToClickHouseType(connectSchema.keySchema()); + String keyType = resolveBaseType(connectSchema.keySchema()); String valType = connectTypeToClickHouseType(connectSchema.valueSchema()); - if (connectSchema.valueSchema().isOptional()) { - valType = "Nullable(" + valType + ")"; - } return "Map(" + keyType + ", " + valType + ")"; case STRUCT: throw new SchemaTypeInferenceException( diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index 39fe97b15..f791e968e 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -1861,38 +1861,28 @@ public void autoEvolveAddsNullableColumn() { } @Test - public void autoEvolveRejectsNonNullableNoDefault() { + public void autoEvolveAddsNonNullableFieldAsNullable() { Map props = createProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); ClickHouseHelperClient chc = createClient(props); - String topic = "auto_evolve_reject_non_nullable_test"; + String topic = "auto_evolve_non_nullable_as_nullable_test"; ClickHouseTestHelpers.dropTable(chc, topic); ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); Collection srV2 = SchemaTestData.createSchemaV2WithNewNonNullableField(topic, 1, 10); ClickHouseSinkTask chst = new ClickHouseSinkTask(); chst.start(props); + chst.put(srV2); + chst.stop(); - try { - chst.put(srV2); - // Should have thrown - assertTrue(false, "Expected exception for non-nullable field without default"); - } catch (RuntimeException e) { - // Walk the full cause chain. Utils.handleException wraps multiple times - Throwable t = e; - boolean found = false; - while (t != null) { - if (t.getMessage() != null && (t.getMessage().contains("not optional") || t.getMessage().contains("Cannot auto-evolve"))) { - found = true; - break; - } - t = t.getCause(); - } - assertTrue(found, "Expected descriptive error about non-nullable field in cause chain, got: " + e.getMessage()); - } finally { - chst.stop(); - } + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Non-nullable fields are created as Nullable columns — mandatory fields always have a value, + // and Nullable allows old records (without this field) to insert with NULL + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("non_nullable_field"), + "New column 'non_nullable_field' should have been added by auto.evolve"); } @Test From e62cba1e99ba509770241e7c11a592f9bd6414ed Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 13:06:50 +0100 Subject: [PATCH 07/62] chore: remove sub batching since is not needed --- .../connect/sink/db/ClickHouseWriter.java | 43 ++--------------- .../ClickHouseSinkTaskWithSchemaTest.java | 48 +++++++++++++++++++ 2 files changed, 53 insertions(+), 38 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index afce8f98c..c50c15baa 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -49,6 +49,7 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoField; +import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -210,46 +211,12 @@ public void doInsert(List records, QueryIdentifier queryId, ErrorReporte if (table == null) { return; }//We checked the error flag in getTable, so we don't need to check it again here if (csc.isAutoEvolve()) { - table = doInsertWithSchemaEvolution(records, table, queryId); - } else { - doInsertBatch(records, table, queryId); - } - } - - private Table doInsertWithSchemaEvolution(List records, Table table, QueryIdentifier queryId) throws IOException, ExecutionException, InterruptedException { - // Split records into sub batches at schema boundaries (like JDBC BufferedRecords.add pattern) - // When schema changes mid batch the current sub batch is flushed, the table is evolved, and the insertion continues. - Schema currentSchema = getValueSchema(records.get(0)); - int batchStart = 0; - - for (int i = 1; i <= records.size(); i++) { - Schema recordSchema = (i < records.size()) ? getValueSchema(records.get(i)) : null; - - // Flush sub batch when schema changes or the end of the records is reached - if (i == records.size() || !Objects.equals(currentSchema, recordSchema)) { - List subBatch = records.subList(batchStart, i); - Record subFirst = subBatch.get(0); - - // Evolve table for the sub batch schema - table = evolveTableSchema(table, subFirst); - - LOGGER.debug("Inserting sub-batch [{}-{}) of {} records with schema evolution (QueryId: [{}])", - batchStart, i, subBatch.size(), queryId.getQueryId()); - doInsertBatch(subBatch, table, queryId); - - if (i < records.size()) { - currentSchema = recordSchema; - batchStart = i; - } - } + // New columns are Nullable, so older records without the new fields insert with NULL. + Record last = records.get(records.size() - 1); + table = evolveTableSchema(table, last); } - return table; - } - - private static Schema getValueSchema(Record record) { - SinkRecord sr = record.getSinkRecord(); - return sr != null ? sr.valueSchema() : null; + doInsertBatch(records, table, queryId); } private void doInsertBatch(List records, Table table, QueryIdentifier queryId) throws IOException, ExecutionException, InterruptedException { diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index f791e968e..9c3955464 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -1,6 +1,7 @@ package com.clickhouse.kafka.connect.sink; import com.clickhouse.client.api.ClientConfigProperties; +import com.clickhouse.client.api.query.Records; import com.clickhouse.kafka.connect.avro.test.Event; import com.clickhouse.kafka.connect.avro.test.Image; import com.clickhouse.kafka.connect.sink.db.helper.ClickHouseHelperClient; @@ -1982,6 +1983,53 @@ public void autoEvolveMixedSchemaInSingleBatch() { "Column 'new_string_field' should be added even when schema changes mid-batch"); } + @Test + public void autoEvolveMixedSchemaOlderRecordsGetNull() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_older_records_null_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // V1 records (no new_string_field) followed by V2 records (has new_string_field) + // Schema is evolved using last record (V2), then entire batch is inserted. + // V1 records should get NULL for the new column. + List mixedBatch = new ArrayList<>(); + mixedBatch.addAll(SchemaTestData.createSchemaV1(topic, 1, 5)); + mixedBatch.addAll(SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 5)); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(mixedBatch); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // V1 records should have NULL for the new column + String nullCountQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `new_string_field` IS NULL SETTINGS select_sequential_consistency = 1", topic); + try { + Records nullRecords = chc.getClient().queryRecords(nullCountQuery).get(); + int nullCount = Integer.parseInt(nullRecords.iterator().next().getString(1)); + assertEquals(5, nullCount, "V1 records should have NULL for new_string_field"); + } catch (Exception e) { + throw new RuntimeException(e); + } + + // V2 records should have non-NULL values + String nonNullQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `new_string_field` IS NOT NULL SETTINGS select_sequential_consistency = 1", topic); + try { + Records nonNullRecords = chc.getClient().queryRecords(nonNullQuery).get(); + int nonNullCount = Integer.parseInt(nonNullRecords.iterator().next().getString(1)); + assertEquals(5, nonNullCount, "V2 records should have non-NULL values for new_string_field"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + @Test public void autoEvolveLogicalTypes() { Map props = createProps(); From 48767ffb56293689e48cdef920ee6baa64f595fe Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 14:57:15 +0100 Subject: [PATCH 08/62] feat: add auto evolve DDL refresh retries configuration to ClickHouseSinkConfig --- .../kafka/connect/sink/ClickHouseSinkConfig.java | 14 ++++++++++++++ .../kafka/connect/sink/db/ClickHouseWriter.java | 8 ++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java b/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java index e4498fedc..c548cf7ce 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java @@ -58,6 +58,7 @@ public class ClickHouseSinkConfig { public static final String ERROR_TOLERANCE_ALL = "all"; public static final String ERROR_TOLERANCE_NONE = "none"; public static final String AUTO_EVOLVE = "auto.evolve"; + public static final String AUTO_EVOLVE_DDL_REFRESH_RETRIES = "auto.evolve.ddl.refresh.retries"; public static final String CONNECTOR_RETRY_TIMEOUT = "errors.retry.timeout"; public static final long MINIMAL_RETRY_TIMEOUT_THR_WARN = TimeUnit.SECONDS.toMillis(10); @@ -112,6 +113,7 @@ public class ClickHouseSinkConfig { private final long bufferFlushTime; private final boolean reportInsertedOffsets; private final boolean autoEvolve; + private final int autoEvolveDdlRefreshRetries; private final boolean binaryFormatWrtiteJsonAsString; private final String sslSocketSni; @@ -299,6 +301,7 @@ public ClickHouseSinkConfig(Map props) { } this.autoEvolve = Boolean.parseBoolean(props.getOrDefault(AUTO_EVOLVE, "false")); + this.autoEvolveDdlRefreshRetries = Integer.parseInt(props.getOrDefault(AUTO_EVOLVE_DDL_REFRESH_RETRIES, "3")); String jsonAsString = getClickhouseSettings().get("input_format_binary_read_json_as_string"); this.binaryFormatWrtiteJsonAsString = jsonAsString != null && (jsonAsString.equalsIgnoreCase("true") || jsonAsString.equals("1")); @@ -709,6 +712,17 @@ private static ConfigDef createConfigDef() { ConfigDef.Width.SHORT, "Auto evolve table schema." ); + configDef.define(AUTO_EVOLVE_DDL_REFRESH_RETRIES, + ConfigDef.Type.INT, + 3, + ConfigDef.Range.atLeast(0), + ConfigDef.Importance.LOW, + "Number of retries when waiting for DDL changes to propagate after schema evolution. default: 3", + ddlGroup, + ++ddlOrderInGroup, + ConfigDef.Width.SHORT, + "DDL refresh retries" + ); return configDef; } } diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index c50c15baa..e6b00b759 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -875,18 +875,18 @@ protected Table evolveTableSchema(Table table, Record record) { return table; } - private static final int DDL_REFRESH_MAX_RETRIES = 5; private static final long DDL_REFRESH_BACKOFF_MS = 200; private Table refreshTableAfterDDL(Table table, Set expectedNewColumns) { - for (int attempt = 0; attempt < DDL_REFRESH_MAX_RETRIES; attempt++) { + int maxRetries = csc.getAutoEvolveDdlRefreshRetries(); + for (int attempt = 0; attempt < maxRetries; attempt++) { Table refreshed = urgentTableUpdate(table); Set stillMissing = refreshed.getMissingColumns(expectedNewColumns); if (stillMissing.isEmpty()) { return refreshed; } LOGGER.warn("DDL refresh attempt {}/{}: columns {} not yet visible, retrying in {}ms", - attempt + 1, DDL_REFRESH_MAX_RETRIES, stillMissing, DDL_REFRESH_BACKOFF_MS); + attempt + 1, maxRetries, stillMissing, DDL_REFRESH_BACKOFF_MS); try { Thread.sleep(DDL_REFRESH_BACKOFF_MS); } catch (InterruptedException e) { @@ -896,7 +896,7 @@ private Table refreshTableAfterDDL(Table table, Set expectedNewColumns) } // Final attempt, use whatever we have LOGGER.error("DDL propagation timeout: some columns may not be visible yet after {} retries. Proceeding with latest table state.", - DDL_REFRESH_MAX_RETRIES); + maxRetries); return urgentTableUpdate(table); } From 8d4d49f0490cefec5f83fcf508bb3ac883d87f69 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 15:01:07 +0100 Subject: [PATCH 09/62] fix: update DDL refresh logic to throw RetriableException on timeout --- .../connect/sink/db/ClickHouseWriter.java | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index e6b00b759..924788284 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -33,6 +33,7 @@ import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.errors.DataException; +import org.apache.kafka.connect.errors.RetriableException; import org.apache.kafka.connect.sink.SinkRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -838,7 +839,7 @@ protected void doWriteCol(Data value, boolean fieldExists, Column col, OutputStr } } - protected Table evolveTableSchema(Table table, Record record) { + protected Table evolveTableSchema(Table table, Record record) throws InterruptedException { if (record.getFields() == null) { LOGGER.warn("Cannot auto-evolve schema for records without a Connect schema (schemaless/string). Skipping schema evolution."); return table; @@ -877,7 +878,7 @@ protected Table evolveTableSchema(Table table, Record record) { private static final long DDL_REFRESH_BACKOFF_MS = 200; - private Table refreshTableAfterDDL(Table table, Set expectedNewColumns) { + private Table refreshTableAfterDDL(Table table, Set expectedNewColumns) throws InterruptedException { int maxRetries = csc.getAutoEvolveDdlRefreshRetries(); for (int attempt = 0; attempt < maxRetries; attempt++) { Table refreshed = urgentTableUpdate(table); @@ -887,17 +888,10 @@ private Table refreshTableAfterDDL(Table table, Set expectedNewColumns) } LOGGER.warn("DDL refresh attempt {}/{}: columns {} not yet visible, retrying in {}ms", attempt + 1, maxRetries, stillMissing, DDL_REFRESH_BACKOFF_MS); - try { - Thread.sleep(DDL_REFRESH_BACKOFF_MS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException("Interrupted while waiting for DDL propagation", e); - } + Thread.sleep(DDL_REFRESH_BACKOFF_MS); } - // Final attempt, use whatever we have - LOGGER.error("DDL propagation timeout: some columns may not be visible yet after {} retries. Proceeding with latest table state.", - maxRetries); - return urgentTableUpdate(table); + throw new RetriableException(String.format( + "DDL propagation timeout: columns not visible after %d retries", maxRetries)); } protected void doInsertRawBinary(List records, Table table, QueryIdentifier queryId, boolean supportDefaults, boolean retry) throws IOException, ExecutionException, InterruptedException { From e926ae166886eb2511577ffb3e374a60f6f84f4e Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 15:05:02 +0100 Subject: [PATCH 10/62] fix: schema requirement for auto evolve --- .../connect/sink/db/ClickHouseWriter.java | 5 +++-- .../ClickHouseSinkTaskWithSchemaTest.java | 22 ++++++++++++++++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index 924788284..edb8d2e16 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -841,8 +841,9 @@ protected void doWriteCol(Data value, boolean fieldExists, Column col, OutputStr protected Table evolveTableSchema(Table table, Record record) throws InterruptedException { if (record.getFields() == null) { - LOGGER.warn("Cannot auto-evolve schema for records without a Connect schema (schemaless/string). Skipping schema evolution."); - return table; + throw new RuntimeException( + "auto.evolve requires a Connect schema (Avro, Protobuf, or JSON Schema). " + + "Schemaless or string records are not supported with auto.evolve=true."); } List fieldNames = record.getFields().stream().map(Field::name).collect(Collectors.toList()); diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index 9c3955464..717b1c484 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -2174,7 +2174,7 @@ public void autoEvolveTripleSchemaInOneBatch() { } @Test - public void autoEvolveSchemalessRecordsSkipped() { + public void autoEvolveSchemalessRecordsThrowError() { Map props = createProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); ClickHouseHelperClient chc = createClient(props); @@ -2195,7 +2195,23 @@ public void autoEvolveSchemalessRecordsSkipped() { ClickHouseSinkTask chst = new ClickHouseSinkTask(); chst.start(props); - chst.put(schemaless); - chst.stop(); + + try { + chst.put(schemaless); + assertTrue(false, "Expected exception for schemaless records with auto.evolve=true"); + } catch (RuntimeException e) { + Throwable t = e; + boolean found = false; + while (t != null) { + if (t.getMessage() != null && t.getMessage().contains("auto.evolve requires a Connect schema")) { + found = true; + break; + } + t = t.getCause(); + } + assertTrue(found, "Expected error about schemaless records in cause chain, got: " + e.getMessage()); + } finally { + chst.stop(); + } } } From b458daf545f93b2cc95354def5adb89cea81f0de Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 15:37:22 +0100 Subject: [PATCH 11/62] refactor: optimize alterTableAddColumns method to use a single SQL statement for adding multiple columns --- .../sink/db/helper/ClickHouseHelperClient.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java index 382ebc880..98c553208 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java @@ -36,6 +36,7 @@ import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; public class ClickHouseHelperClient implements AutoCloseable { @@ -476,14 +477,15 @@ public Table describeTableV2(String database, String tableName) { } public void alterTableAddColumns(String database, String tableName, List columnDefs) { - for (String colDef : columnDefs) { - String sql = String.format("ALTER TABLE `%s`.`%s` ADD COLUMN IF NOT EXISTS %s", database, tableName, colDef); - LOGGER.info("Executing DDL: {}", sql); - if (useClientV2) { - alterTableAddColumnV2(sql); - } else { - alterTableAddColumnV1(sql); - } + String addClauses = columnDefs.stream() + .map(colDef -> "ADD COLUMN IF NOT EXISTS " + colDef) + .collect(Collectors.joining(", ")); + String sql = String.format("ALTER TABLE `%s`.`%s` %s", database, tableName, addClauses); + LOGGER.info("Executing DDL: {}", sql); + if (useClientV2) { + alterTableAddColumnV2(sql); + } else { + alterTableAddColumnV1(sql); } } From 3c755bb8c53673d14278eb90cfccfe7488858969 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 20:17:55 +0100 Subject: [PATCH 12/62] feat: add auto evolve struct to JSON configuration option to ClickHouseSinkConfig --- .../kafka/connect/sink/ClickHouseSinkConfig.java | 13 +++++++++++++ .../kafka/connect/sink/db/ClickHouseWriter.java | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java b/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java index c548cf7ce..22b1aff60 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java @@ -59,6 +59,7 @@ public class ClickHouseSinkConfig { public static final String ERROR_TOLERANCE_NONE = "none"; public static final String AUTO_EVOLVE = "auto.evolve"; public static final String AUTO_EVOLVE_DDL_REFRESH_RETRIES = "auto.evolve.ddl.refresh.retries"; + public static final String AUTO_EVOLVE_STRUCT_TO_JSON = "auto.evolve.struct.to.json"; public static final String CONNECTOR_RETRY_TIMEOUT = "errors.retry.timeout"; public static final long MINIMAL_RETRY_TIMEOUT_THR_WARN = TimeUnit.SECONDS.toMillis(10); @@ -114,6 +115,7 @@ public class ClickHouseSinkConfig { private final boolean reportInsertedOffsets; private final boolean autoEvolve; private final int autoEvolveDdlRefreshRetries; + private final boolean autoEvolveStructToJson; private final boolean binaryFormatWrtiteJsonAsString; private final String sslSocketSni; @@ -302,6 +304,7 @@ public ClickHouseSinkConfig(Map props) { this.autoEvolve = Boolean.parseBoolean(props.getOrDefault(AUTO_EVOLVE, "false")); this.autoEvolveDdlRefreshRetries = Integer.parseInt(props.getOrDefault(AUTO_EVOLVE_DDL_REFRESH_RETRIES, "3")); + this.autoEvolveStructToJson = Boolean.parseBoolean(props.getOrDefault(AUTO_EVOLVE_STRUCT_TO_JSON, "false")); String jsonAsString = getClickhouseSettings().get("input_format_binary_read_json_as_string"); this.binaryFormatWrtiteJsonAsString = jsonAsString != null && (jsonAsString.equalsIgnoreCase("true") || jsonAsString.equals("1")); @@ -723,6 +726,16 @@ private static ConfigDef createConfigDef() { ConfigDef.Width.SHORT, "DDL refresh retries" ); + configDef.define(AUTO_EVOLVE_STRUCT_TO_JSON, + ConfigDef.Type.BOOLEAN, + false, + ConfigDef.Importance.MEDIUM, + "Whether to map Connect STRUCT fields to ClickHouse JSON columns during schema evolution. default: false", + ddlGroup, + ++ddlOrderInGroup, + ConfigDef.Width.SHORT, + "Map STRUCT to JSON" + ); return configDef; } } diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index edb8d2e16..4d5a66282 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -864,7 +864,7 @@ protected Table evolveTableSchema(Table table, Record record) throws Interrupted continue; } - String chType = Column.connectTypeToClickHouseType(fieldSchema); + String chType = Column.connectTypeToClickHouseType(fieldSchema, csc.isAutoEvolveStructToJson()); columnDefs.add(String.format("`%s` %s", fieldName, chType)); } From 670eec53e6b81ed28a8eecb98bd75e4d8c7821cd Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 23:14:00 +0100 Subject: [PATCH 13/62] feat: enhance schema handling by adding union type detection and mapping for ClickHouse integration --- .../connect/sink/db/ClickHouseWriter.java | 3 + .../kafka/connect/sink/db/mapping/Column.java | 97 +++- .../ClickHouseSinkTaskWithSchemaTest.java | 419 +++++++++++++++++- .../connect/sink/db/mapping/ColumnTest.java | 177 ++++++++ .../connect/sink/helper/SchemaTestData.java | 164 +++++++ 5 files changed, 851 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index 4d5a66282..7196d4501 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -295,6 +295,9 @@ protected boolean validateDataSchema(Table table, Record record, boolean onlyFie if (colTypeName.equals("TUPLE") && dataTypeName.equals("STRUCT")) continue; + if (colTypeName.equals("VARIANT") && dataTypeName.equals("STRUCT")) + continue; + if (INT_TYPES.contains(colTypeName)) { continue; } diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java index 854d5e2c5..f8f16ee7f 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java @@ -8,6 +8,7 @@ import lombok.experimental.Accessors; import org.apache.kafka.connect.data.Date; import org.apache.kafka.connect.data.Decimal; +import org.apache.kafka.connect.data.Field; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.Time; import org.apache.kafka.connect.data.Timestamp; @@ -20,9 +21,11 @@ import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -37,6 +40,13 @@ public class Column { private static final int DECIMAL128_MAX_PRECISION = 38; private static final Logger LOGGER = LoggerFactory.getLogger(Column.class); + + // Confluent converter union schema markers + static final String AVRO_UNION_SCHEMA_NAME = "io.confluent.connect.avro.Union"; + static final String PROTOBUF_UNION_SCHEMA_PREFIX = "io.confluent.connect.protobuf.Union"; + static final String GENERALIZED_UNION_PREFIX = "connect_union_"; + static final String CONNECT_UNION_PARAMETER = "org.apache.kafka.connect.data.Union"; + private String name; private Type type; @@ -365,17 +375,21 @@ private static Map extractEnumValues(String valueType) { } public static String connectTypeToClickHouseType(Schema connectSchema) { - String baseType = resolveBaseType(connectSchema); + return connectTypeToClickHouseType(connectSchema, false); + } - // ClickHouse forbids Nullable wrapping for Array and Map types - if (connectSchema.type() == Schema.Type.ARRAY || connectSchema.type() == Schema.Type.MAP) { + public static String connectTypeToClickHouseType(Schema connectSchema, boolean structToJson) { + String baseType = resolveBaseType(connectSchema, structToJson); + + // ClickHouse forbids Nullable wrapping for Array, Map, and Variant types. + if (connectSchema.type() == Schema.Type.ARRAY || connectSchema.type() == Schema.Type.MAP || baseType.startsWith("Variant(")) { return baseType; } return "Nullable(" + baseType + ")"; } - private static String resolveBaseType(Schema connectSchema) { + private static String resolveBaseType(Schema connectSchema, boolean structToJson) { // Check logical types first (same pattern as JDBC connector) if (connectSchema.name() != null) { switch (connectSchema.name()) { @@ -419,21 +433,88 @@ private static String resolveBaseType(Schema connectSchema) { if (connectSchema.valueSchema() == null) { return "Array(String)"; } - String elementType = connectTypeToClickHouseType(connectSchema.valueSchema()); + String elementType = connectTypeToClickHouseType(connectSchema.valueSchema(), structToJson); return "Array(" + elementType + ")"; case MAP: - String keyType = resolveBaseType(connectSchema.keySchema()); - String valType = connectTypeToClickHouseType(connectSchema.valueSchema()); + String keyType = resolveBaseType(connectSchema.keySchema(), structToJson); + String valType = connectTypeToClickHouseType(connectSchema.valueSchema(), structToJson); return "Map(" + keyType + ", " + valType + ")"; case STRUCT: + if (isUnionSchema(connectSchema)) { + return resolveUnionType(connectSchema, structToJson); + } + if (structToJson) { + return "JSON"; + } throw new SchemaTypeInferenceException( "Cannot auto-evolve STRUCT fields to ClickHouse columns. " + - "STRUCT type requires manual mapping to Tuple, JSON, or Nested type."); + "Set auto.evolve.struct.to.json=true to map STRUCT to JSON, " + + "or manually create the column as Tuple, JSON, or Nested type."); default: throw new SchemaTypeInferenceException("Unsupported Connect type for auto-evolution: " + connectSchema.type()); } } + // Type groups that ClickHouse considers suspicious when mixed inside a Variant. + // See: https://clickhouse.com/docs/sql-reference/data-types/variant + private static final Set SUSPICIOUS_NUMERIC_TYPES = Set.of( + "Int8", "Int16", "Int32", "Int64", + "UInt8", "UInt16", "UInt32", "UInt64", + "Float32", "Float64" + ); + private static final Set SUSPICIOUS_DATE_TYPES = Set.of( + "Date32", "DateTime64(3)" + ); + + private static String resolveUnionType(Schema connectSchema, boolean structToJson) { + if (connectSchema.fields() == null || connectSchema.fields().isEmpty()) { + return "String"; + } + + LinkedHashSet chTypes = new LinkedHashSet<>(); + for (Field field : connectSchema.fields()) { + chTypes.add(resolveBaseType(field.schema(), structToJson)); + } + + // All branches resolve to the same ClickHouse type (e.g. union(string, bytes) → String) + if (chTypes.size() == 1) { + return chTypes.iterator().next(); + } + + // Check for suspicious similar types that ClickHouse rejects by default + if (hasSuspiciousSimilarTypes(chTypes)) { + return "String"; + } + + // Multiple distinct types map to Variant(T1, T2, ...) requires ClickHouse 24.1+. + return "Variant(" + String.join(", ", chTypes) + ")"; + } + + private static boolean hasSuspiciousSimilarTypes(Set chTypes) { + int numericCount = 0; + int dateCount = 0; + for (String t : chTypes) { + if (SUSPICIOUS_NUMERIC_TYPES.contains(t)) numericCount++; + if (SUSPICIOUS_DATE_TYPES.contains(t)) dateCount++; + } + return numericCount > 1 || dateCount > 1; + } + + static boolean isUnionSchema(Schema connectSchema) { + if (connectSchema.type() != Schema.Type.STRUCT) { + return false; + } + String name = connectSchema.name(); + if (name != null + && (name.equals(AVRO_UNION_SCHEMA_NAME) + || name.startsWith(PROTOBUF_UNION_SCHEMA_PREFIX) + || name.startsWith(GENERALIZED_UNION_PREFIX))) { + return true; + } + return connectSchema.parameters() != null + && connectSchema.parameters().containsKey(CONNECT_UNION_PARAMETER); + } + public Integer convertEnumValues(String value) { if ( this.enumValues != null ) { return enumValues.get(value); diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index 717b1c484..5cf081008 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -63,6 +63,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @ExtendWith(FromVersionConditionExtension.class) @@ -1879,7 +1880,7 @@ public void autoEvolveAddsNonNullableFieldAsNullable() { assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); - // Non-nullable fields are created as Nullable columns — mandatory fields always have a value, + // Non-nullable fields are created as Nullable columns - mandatory fields always have a value, // and Nullable allows old records (without this field) to insert with NULL com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); assertTrue(described.getRootColumnsMap().containsKey("non_nullable_field"), @@ -2110,6 +2111,105 @@ public void autoEvolveRejectsStructField() { } } + // STRUCT field auto-evolved as JSON column when auto.evolve.struct.to.json=true + @Test + public void autoEvolveStructToJsonCreatesJsonColumn() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE_STRUCT_TO_JSON, "true"); + props.put(ClickHouseSinkConfig.CLICKHOUSE_SETTINGS, "input_format_binary_read_json_as_string=1"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_struct_to_json_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + Collection srV2 = SchemaTestData.createSchemaV2WithStructField(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV2); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify the new column was created as JSON type + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("new_struct_field"), + "Column 'new_struct_field' should exist"); + com.clickhouse.kafka.connect.sink.db.mapping.Column col = described.getRootColumnsMap().get("new_struct_field"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.JSON, col.getType(), + "Column 'new_struct_field' should be JSON type"); + } + + // V1 records (no struct) inserted first, then V2 records (with struct) trigger JSON column creation. + @Test + public void autoEvolveStructToJsonMixedBatchOlderRecordsGetDefault() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE_STRUCT_TO_JSON, "true"); + props.put(ClickHouseSinkConfig.CLICKHOUSE_SETTINGS, "input_format_binary_read_json_as_string=1"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_struct_json_mixed_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Insert V1 records (no struct field) + Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 5); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV1); + assertEquals(5, ClickHouseTestHelpers.countRows(chc, topic)); + + // Insert V2 records (with struct field) - triggers JSON column creation + Collection srV2 = SchemaTestData.createSchemaV2WithStructField(topic, 1, 5); + chst.put(srV2); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify JSON column exists + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + com.clickhouse.kafka.connect.sink.db.mapping.Column col = described.getRootColumnsMap().get("new_struct_field"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.JSON, col.getType(), + "Column 'new_struct_field' should be JSON type"); + } + + // STRUCT field with auto.evolve.struct.to.json explicitly false rejects with helpful error message + @Test + public void autoEvolveStructToJsonExplicitlyFalseRejectsStruct() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE_STRUCT_TO_JSON, "false"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_struct_json_false_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + Collection srV2 = SchemaTestData.createSchemaV2WithStructField(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + + try { + chst.put(srV2); + assertTrue(false, "Expected exception for STRUCT field when struct.to.json is false"); + } catch (RuntimeException e) { + Throwable t = e; + boolean found = false; + while (t != null) { + if (t.getMessage() != null && t.getMessage().contains("auto.evolve.struct.to.json=true")) { + found = true; + break; + } + t = t.getCause(); + } + assertTrue(found, "Error message should suggest auto.evolve.struct.to.json=true, got: " + e.getMessage()); + } finally { + chst.stop(); + } + } + @Test public void autoEvolveArrayAndMapFields() { Map props = createProps(); @@ -2173,6 +2273,270 @@ public void autoEvolveTripleSchemaInOneBatch() { "V3 column 'v3_bool_field' should exist"); } + // auto-evolve adds columns for every supported primitive + logical type in a single batch + @Test + public void autoEvolveAllPrimitiveAndLogicalTypes() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_all_types_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Insert V1 first to ensure existing rows get NULL for new columns + Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 5); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV1); + assertEquals(5, ClickHouseTestHelpers.countRows(chc, topic)); + + // Insert V2 with all primitive + logical type fields + Collection srV2 = SchemaTestData.createSchemaV2WithAllPrimitiveTypes(topic, 1, 5); + chst.put(srV2); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify all columns were created with correct types + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + Map cols = described.getRootColumnsMap(); + + // Primitive types + assertTrue(cols.containsKey("new_int8"), "Column 'new_int8' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.INT8, cols.get("new_int8").getType()); + assertTrue(cols.containsKey("new_int16"), "Column 'new_int16' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.INT16, cols.get("new_int16").getType()); + assertTrue(cols.containsKey("new_int32"), "Column 'new_int32' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.INT32, cols.get("new_int32").getType()); + assertTrue(cols.containsKey("new_int64"), "Column 'new_int64' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.INT64, cols.get("new_int64").getType()); + assertTrue(cols.containsKey("new_float32"), "Column 'new_float32' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.FLOAT32, cols.get("new_float32").getType()); + assertTrue(cols.containsKey("new_float64"), "Column 'new_float64' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.FLOAT64, cols.get("new_float64").getType()); + assertTrue(cols.containsKey("new_bool"), "Column 'new_bool' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.BOOLEAN, cols.get("new_bool").getType()); + assertTrue(cols.containsKey("new_string"), "Column 'new_string' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.STRING, cols.get("new_string").getType()); + assertTrue(cols.containsKey("new_bytes"), "Column 'new_bytes' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.STRING, cols.get("new_bytes").getType()); + + // Logical types + assertTrue(cols.containsKey("new_decimal"), "Column 'new_decimal' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.Decimal, cols.get("new_decimal").getType()); + assertTrue(cols.containsKey("new_date"), "Column 'new_date' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.Date32, cols.get("new_date").getType()); + assertTrue(cols.containsKey("new_timestamp"), "Column 'new_timestamp' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.DateTime64, cols.get("new_timestamp").getType()); + } + + // auto-evolve creates Array columns with different element types (Int32, Float64, Bool, String) + @Test + public void autoEvolveTypedArrayColumns() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_typed_arrays_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + Collection srV2 = SchemaTestData.createSchemaV2WithTypedArrays(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV2); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify all array columns were created + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + Map cols = described.getRootColumnsMap(); + + assertTrue(cols.containsKey("arr_int32"), "Column 'arr_int32' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.ARRAY, cols.get("arr_int32").getType()); + assertTrue(cols.containsKey("arr_float64"), "Column 'arr_float64' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.ARRAY, cols.get("arr_float64").getType()); + assertTrue(cols.containsKey("arr_bool"), "Column 'arr_bool' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.ARRAY, cols.get("arr_bool").getType()); + assertTrue(cols.containsKey("arr_string"), "Column 'arr_string' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.ARRAY, cols.get("arr_string").getType()); + } + + // DDL refresh timeout - retries set to 0 so refresh loop never runs, throws RetriableException + @Test + public void autoEvolveDdlRefreshTimeoutThrowsRetriable() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE_DDL_REFRESH_RETRIES, "0"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_ddl_timeout_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // V2 schema with a new field - DDL will succeed but refresh will timeout with 0 retries + Collection srV2 = SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 5); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + + try { + chst.put(srV2); + assertTrue(false, "Expected RetriableException due to DDL refresh timeout with 0 retries"); + } catch (RuntimeException e) { + // Processing layer may wrap the RetriableException - walk the cause chain + Throwable t = e; + boolean found = false; + while (t != null) { + if (t instanceof org.apache.kafka.connect.errors.RetriableException + && t.getMessage() != null && t.getMessage().contains("DDL propagation timeout")) { + found = true; + break; + } + t = t.getCause(); + } + assertTrue(found, "Should contain RetriableException with DDL propagation timeout in cause chain, got: " + e); + } finally { + chst.stop(); + } + } + + // ALTER TABLE itself fails (table dropped externally after cache populated) + @Test + public void autoEvolveDdlExecutionFailureThrowsRuntimeException() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_ddl_exec_failure_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Insert V1 records to populate the connector's internal table mapping cache + Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 5); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV1); + assertEquals(5, ClickHouseTestHelpers.countRows(chc, topic)); + + // Drop the table externally - the connector still has it cached in memory + ClickHouseTestHelpers.dropTable(chc, topic); + + // V2 schema with a new field - ALTER TABLE will fail because the table no longer exists + Collection srV2 = SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 5); + try { + chst.put(srV2); + assertTrue(false, "Expected RuntimeException due to ALTER TABLE on dropped table"); + } catch (RuntimeException e) { + // Processing layer wraps exceptions - walk the cause chain for the DDL failure + Throwable t = e; + boolean found = false; + while (t != null) { + if (t.getMessage() != null && (t.getMessage().contains("ALTER TABLE") || t.getMessage().contains("UNKNOWN_TABLE"))) { + found = true; + break; + } + t = t.getCause(); + } + assertTrue(found, "Should indicate DDL failure in cause chain, got: " + e.getClass().getName() + ": " + e.getMessage()); + } finally { + chst.stop(); + } + } + + // STRUCT field without struct-to-json flag throws SchemaTypeInferenceException with helpful message + @Test + public void autoEvolveUnsupportedStructTypeThrowsError() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + // auto.evolve.struct.to.json is false by default + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_unsupported_struct_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + Collection srV2 = SchemaTestData.createSchemaV2WithStructField(topic, 1, 5); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + + try { + chst.put(srV2); + assertTrue(false, "Expected SchemaTypeInferenceException for unsupported STRUCT type"); + } catch (RuntimeException e) { + Throwable t = e; + boolean foundInference = false; + while (t != null) { + if (t instanceof com.clickhouse.kafka.connect.sink.db.mapping.SchemaTypeInferenceException) { + foundInference = true; + break; + } + t = t.getCause(); + } + assertTrue(foundInference, + "Should throw SchemaTypeInferenceException for unsupported STRUCT, got: " + e.getClass().getName() + ": " + e.getMessage()); + } finally { + chst.stop(); + } + } + + // Avro-style union(string, bytes) STRUCT collapses to Nullable(String), not JSON + @Test + public void autoEvolveStringBytesUnionCollapsesToString() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_union_string_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + Collection srV2 = SchemaTestData.createSchemaV2WithStringBytesUnionField(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV2); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify the union field was created as String (not JSON) + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + com.clickhouse.kafka.connect.sink.db.mapping.Column col = described.getRootColumnsMap().get("new_union_field"); + assertNotNull(col, "Column 'new_union_field' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.STRING, col.getType(), + "Union(string, bytes) should collapse to String, not JSON"); + } + + // Avro union(string, int) auto-evolved as Variant(String, Int32) column + @Test + @SinceClickHouseVersion("24.1") + public void autoEvolveMixedUnionCreatesVariantColumn() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + props.put(ClickHouseSinkConfig.CLICKHOUSE_SETTINGS, "allow_experimental_variant_type=1"); + ClickHouseHelperClient chc = createClient(props); + + String topic = createTopicName("auto_evolve_variant_test"); + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + Collection records = SchemaTestData.createSchemaV2WithMixedTypeUnionField(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(records); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify the union field was created as Variant + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + com.clickhouse.kafka.connect.sink.db.mapping.Column col = described.getRootColumnsMap().get("mixed_union"); + assertNotNull(col, "Column 'mixed_union' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.VARIANT, col.getType(), + "union(string, int) should map to Variant, not String or JSON"); + } + @Test public void autoEvolveSchemalessRecordsThrowError() { Map props = createProps(); @@ -2214,4 +2578,57 @@ public void autoEvolveSchemalessRecordsThrowError() { chst.stop(); } } + + // Avro union(string, bytes) fields auto-evolved as Nullable(String) columns + @Test + public void autoEvolveAvroUnionStringBytesCreatesStringColumn() throws Exception { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_avro_union_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + // Table starts with only "name" - union fields "content" and "description" will be auto-evolved + ClickHouseTestHelpers.createTable(chc, topic, + "CREATE TABLE `%s` (`name` String) Engine = MergeTree ORDER BY name"); + + Image image1 = Image.newBuilder() + .setName("image1") + .setContent("content1") + .build(); + Image image2 = Image.newBuilder() + .setName("image2") + .setContent(ByteBuffer.wrap("content2".getBytes())) + .setDescription("desc2") + .build(); + + List records = SchemaTestData.convertAvroToSinkRecord( + topic, new AvroSchema(Image.getClassSchema()), Arrays.asList(image1, image2)); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(records); + chst.stop(); + + assertEquals(2, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify that union columns were created as String (not JSON) + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + Map cols = described.getRootColumnsMap(); + + assertTrue(cols.containsKey("content"), "Column 'content' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.STRING, cols.get("content").getType(), + "union(string, bytes) should map to String, not JSON"); + + assertTrue(cols.containsKey("description"), "Column 'description' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.STRING, cols.get("description").getType(), + "union(null, string, bytes) should map to Nullable(String), not JSON"); + + // Verify data was inserted correctly + List rows = ClickHouseTestHelpers.getAllRowsAsJson(chc, topic); + if (rows.size() == 0) { + rows = ClickHouseTestHelpers.getAllRowsAsJson(chc, topic); + } + assertEquals(2, rows.size()); + } } diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java index 22296e8cb..ddd24a286 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java @@ -1,7 +1,10 @@ package com.clickhouse.kafka.connect.sink.db.mapping; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.SchemaBuilder; import org.junit.jupiter.api.Test; +import java.util.Collections; import java.util.List; import static com.clickhouse.kafka.connect.sink.helper.ClickHouseTestHelpers.col; @@ -177,5 +180,179 @@ public void extractEnumOfPrimitives() { assertTrue(col.getEnumValues().containsKey("a, valid")); assertTrue(col.getEnumValues().containsKey("b")); } + + // --- isUnionSchema detection tests --- + + @Test + public void isUnionSchema_avroUnion() { + Schema union = SchemaBuilder.struct() + .name(Column.AVRO_UNION_SCHEMA_NAME) + .field("string", Schema.OPTIONAL_STRING_SCHEMA) + .field("int", Schema.OPTIONAL_INT32_SCHEMA) + .optional() + .build(); + assertTrue(Column.isUnionSchema(union)); + } + + @Test + public void isUnionSchema_protobufOneof() { + Schema union = SchemaBuilder.struct() + .name("io.confluent.connect.protobuf.Union.content") + .field("user_info", Schema.OPTIONAL_STRING_SCHEMA) + .field("product_info", Schema.OPTIONAL_STRING_SCHEMA) + .optional() + .build(); + assertTrue(Column.isUnionSchema(union)); + } + + @Test + public void isUnionSchema_generalizedUnion() { + Schema union = SchemaBuilder.struct() + .name("connect_union_0") + .parameter(Column.CONNECT_UNION_PARAMETER, "connect_union_0") + .field("string", Schema.OPTIONAL_STRING_SCHEMA) + .field("int", Schema.OPTIONAL_INT32_SCHEMA) + .optional() + .build(); + assertTrue(Column.isUnionSchema(union)); + } + + @Test + public void isUnionSchema_connectUnionParameter() { + Schema union = SchemaBuilder.struct() + .parameter(Column.CONNECT_UNION_PARAMETER, "some_annotation") + .field("string", Schema.OPTIONAL_STRING_SCHEMA) + .field("int", Schema.OPTIONAL_INT32_SCHEMA) + .optional() + .build(); + assertTrue(Column.isUnionSchema(union)); + } + + @Test + public void isUnionSchema_realStruct_notDetected() { + Schema struct = SchemaBuilder.struct() + .field("name", Schema.STRING_SCHEMA) + .field("age", Schema.INT32_SCHEMA) + .build(); + assertFalse(Column.isUnionSchema(struct)); + } + + @Test + public void isUnionSchema_primitiveType_notDetected() { + assertFalse(Column.isUnionSchema(Schema.STRING_SCHEMA)); + } + + // --- connectTypeToClickHouseType union mapping tests --- + + @Test + public void unionStringBytes_collapsesToNullableString() { + Schema union = SchemaBuilder.struct() + .name(Column.AVRO_UNION_SCHEMA_NAME) + .field("string", Schema.OPTIONAL_STRING_SCHEMA) + .field("bytes", Schema.OPTIONAL_BYTES_SCHEMA) + .optional() + .build(); + assertEquals("Nullable(String)", Column.connectTypeToClickHouseType(union)); + } + + @Test + public void unionStringInt_mapsToVariant() { + Schema union = SchemaBuilder.struct() + .name(Column.AVRO_UNION_SCHEMA_NAME) + .field("string", Schema.OPTIONAL_STRING_SCHEMA) + .field("int", Schema.OPTIONAL_INT32_SCHEMA) + .optional() + .build(); + assertEquals("Variant(String, Int32)", Column.connectTypeToClickHouseType(union)); + } + + @Test + public void unionStringIntBoolean_mapsToVariant() { + Schema union = SchemaBuilder.struct() + .name(Column.AVRO_UNION_SCHEMA_NAME) + .field("string", Schema.OPTIONAL_STRING_SCHEMA) + .field("int", Schema.OPTIONAL_INT32_SCHEMA) + .field("boolean", Schema.OPTIONAL_BOOLEAN_SCHEMA) + .optional() + .build(); + assertEquals("Variant(String, Int32, Bool)", Column.connectTypeToClickHouseType(union)); + } + + @Test + public void unionSuspiciousNumericTypes_fallsBackToString() { + // Variant(Int32, Int64) is rejected by ClickHouse unless allow_suspicious_variant_types + Schema union = SchemaBuilder.struct() + .name(Column.AVRO_UNION_SCHEMA_NAME) + .field("int", Schema.OPTIONAL_INT32_SCHEMA) + .field("long", Schema.OPTIONAL_INT64_SCHEMA) + .optional() + .build(); + assertEquals("Nullable(String)", Column.connectTypeToClickHouseType(union)); + } + + @Test + public void unionSuspiciousNumericWithString_fallsBackToString() { + Schema union = SchemaBuilder.struct() + .name(Column.AVRO_UNION_SCHEMA_NAME) + .field("string", Schema.OPTIONAL_STRING_SCHEMA) + .field("int", Schema.OPTIONAL_INT32_SCHEMA) + .field("long", Schema.OPTIONAL_INT64_SCHEMA) + .optional() + .build(); + assertEquals("Nullable(String)", Column.connectTypeToClickHouseType(union)); + } + + @Test + public void protobufOneof_stringBytes_collapsesToNullableString() { + // Protobuf oneof { string url = 1; bytes raw = 2; } — field names differ from Avro + Schema union = SchemaBuilder.struct() + .name("io.confluent.connect.protobuf.Union.image") + .field("url", Schema.OPTIONAL_STRING_SCHEMA) + .field("raw", Schema.OPTIONAL_BYTES_SCHEMA) + .optional() + .build(); + assertEquals("Nullable(String)", Column.connectTypeToClickHouseType(union)); + } + + @Test + public void realStruct_withStructToJson_mapsToJSON() { + Schema struct = SchemaBuilder.struct() + .field("name", Schema.STRING_SCHEMA) + .field("age", Schema.INT32_SCHEMA) + .build(); + assertEquals("Nullable(JSON)", Column.connectTypeToClickHouseType(struct, true)); + } + + @Test + public void realStruct_withoutFlag_throws() { + Schema struct = SchemaBuilder.struct() + .field("name", Schema.STRING_SCHEMA) + .field("age", Schema.INT32_SCHEMA) + .build(); + assertThrows(SchemaTypeInferenceException.class, + () -> Column.connectTypeToClickHouseType(struct, false)); + } + + @Test + public void unionEmptyFields_fallsBackToNullableString() { + Schema union = SchemaBuilder.struct() + .name(Column.AVRO_UNION_SCHEMA_NAME) + .optional() + .build(); + assertEquals("Nullable(String)", Column.connectTypeToClickHouseType(union)); + } + + @Test + public void unionVariant_notWrappedInNullable() { + Schema union = SchemaBuilder.struct() + .name(Column.AVRO_UNION_SCHEMA_NAME) + .field("string", Schema.OPTIONAL_STRING_SCHEMA) + .field("boolean", Schema.OPTIONAL_BOOLEAN_SCHEMA) + .optional() + .build(); + String result = Column.connectTypeToClickHouseType(union); + assertEquals("Variant(String, Bool)", result); + assertFalse(result.startsWith("Nullable(")); + } } diff --git a/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java b/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java index 3b861ebe7..aacd7b122 100644 --- a/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java +++ b/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java @@ -1857,4 +1857,168 @@ public static Collection createSchemaV2WithLogicalTypes(String topic } return array; } + + // Schema with all supported primitive types + logical types as new columns + public static Collection createSchemaV2WithAllPrimitiveTypes(String topic, int partition) { + return createSchemaV2WithAllPrimitiveTypes(topic, partition, DEFAULT_TOTAL_RECORDS); + } + + public static Collection createSchemaV2WithAllPrimitiveTypes(String topic, int partition, int totalRecords) { + List array = new ArrayList<>(); + + Schema SCHEMA_V2 = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("new_int8", SchemaBuilder.int8().optional().build()) + .field("new_int16", SchemaBuilder.int16().optional().build()) + .field("new_int32", SchemaBuilder.int32().optional().build()) + .field("new_int64", SchemaBuilder.int64().optional().build()) + .field("new_float32", SchemaBuilder.float32().optional().build()) + .field("new_float64", SchemaBuilder.float64().optional().build()) + .field("new_bool", SchemaBuilder.bool().optional().build()) + .field("new_string", Schema.OPTIONAL_STRING_SCHEMA) + .field("new_bytes", Schema.OPTIONAL_BYTES_SCHEMA) + .field("new_decimal", Decimal.builder(4).optional().build()) + .field("new_date", org.apache.kafka.connect.data.Date.builder().optional().build()) + .field("new_timestamp", Timestamp.builder().optional().build()) + .build(); + + long offset = totalRecords; + for (long n = 0; n < totalRecords; n++) { + Struct value_struct = new Struct(SCHEMA_V2) + .put("off16", (short) n) + .put("p_int64", n) + .put("new_int8", (byte) n) + .put("new_int16", (short) n) + .put("new_int32", (int) n) + .put("new_int64", n * 100L) + .put("new_float32", (float) n * 1.1f) + .put("new_float64", n * 2.2) + .put("new_bool", n % 2 == 0) + .put("new_string", "str_" + n) + .put("new_bytes", ("bytes_" + n).getBytes()) + .put("new_decimal", java.math.BigDecimal.valueOf(n * 100 + 50, 4)) + .put("new_date", Date.from(java.time.Instant.ofEpochMilli(n * 86400000L))) + .put("new_timestamp", Date.from(java.time.Instant.ofEpochMilli(System.currentTimeMillis()))); + + array.add(new SinkRecord( + topic, partition, null, null, SCHEMA_V2, value_struct, + offset + n, System.currentTimeMillis(), TimestampType.CREATE_TIME + )); + } + return array; + } + + // Schema with arrays of different element types + public static Collection createSchemaV2WithTypedArrays(String topic, int partition) { + return createSchemaV2WithTypedArrays(topic, partition, DEFAULT_TOTAL_RECORDS); + } + + public static Collection createSchemaV2WithTypedArrays(String topic, int partition, int totalRecords) { + List array = new ArrayList<>(); + + Schema SCHEMA_V2 = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("arr_int32", SchemaBuilder.array(SchemaBuilder.int32().optional().build()).optional().build()) + .field("arr_float64", SchemaBuilder.array(SchemaBuilder.float64().optional().build()).optional().build()) + .field("arr_bool", SchemaBuilder.array(SchemaBuilder.bool().optional().build()).optional().build()) + .field("arr_string", SchemaBuilder.array(Schema.OPTIONAL_STRING_SCHEMA).optional().build()) + .build(); + + long offset = totalRecords; + for (long n = 0; n < totalRecords; n++) { + Struct value_struct = new Struct(SCHEMA_V2) + .put("off16", (short) n) + .put("p_int64", n) + .put("arr_int32", Arrays.asList((int) n, (int) n + 1)) + .put("arr_float64", Arrays.asList(n * 1.1, n * 2.2)) + .put("arr_bool", Arrays.asList(n % 2 == 0, n % 2 != 0)) + .put("arr_string", Arrays.asList("a_" + n, "b_" + n)); + + array.add(new SinkRecord( + topic, partition, null, null, SCHEMA_V2, value_struct, + offset + n, System.currentTimeMillis(), TimestampType.CREATE_TIME + )); + } + return array; + } + + // Schema with an Avro-style union STRUCT (all STRING/BYTES fields) that should collapse to String + public static Collection createSchemaV2WithStringBytesUnionField(String topic, int partition) { + return createSchemaV2WithStringBytesUnionField(topic, partition, DEFAULT_TOTAL_RECORDS); + } + + public static Collection createSchemaV2WithStringBytesUnionField(String topic, int partition, int totalRecords) { + List array = new ArrayList<>(); + + // Simulates how Confluent AvroConverter represents union(string, bytes) + // AvroConverter sets schema.name() to "io.confluent.connect.avro.Union" + Schema UNION_SCHEMA = SchemaBuilder.struct() + .name("io.confluent.connect.avro.Union") + .field("string", Schema.OPTIONAL_STRING_SCHEMA) + .field("bytes", Schema.OPTIONAL_BYTES_SCHEMA) + .optional() + .build(); + + Schema SCHEMA_V2 = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("new_union_field", UNION_SCHEMA) + .build(); + + long offset = totalRecords; + for (long n = 0; n < totalRecords; n++) { + Struct unionValue = new Struct(UNION_SCHEMA) + .put("string", "union_str_" + n); + Struct value_struct = new Struct(SCHEMA_V2) + .put("off16", (short) n) + .put("p_int64", n) + .put("new_union_field", unionValue); + + array.add(new SinkRecord( + topic, partition, null, null, SCHEMA_V2, value_struct, + offset + n, System.currentTimeMillis(), TimestampType.CREATE_TIME + )); + } + return array; + } + + // Schema with an Avro-style union STRUCT (string + int) that should map to Variant(String, Int32) + public static Collection createSchemaV2WithMixedTypeUnionField(String topic, int partition, int totalRecords) { + List array = new ArrayList<>(); + + Schema UNION_SCHEMA = SchemaBuilder.struct() + .name("io.confluent.connect.avro.Union") + .field("string", Schema.OPTIONAL_STRING_SCHEMA) + .field("int", Schema.OPTIONAL_INT32_SCHEMA) + .optional() + .build(); + + Schema SCHEMA_V2 = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("mixed_union", UNION_SCHEMA) + .build(); + + long offset = totalRecords; + for (long n = 0; n < totalRecords; n++) { + Struct unionValue; + if (n % 2 == 0) { + unionValue = new Struct(UNION_SCHEMA).put("string", "val_" + n); + } else { + unionValue = new Struct(UNION_SCHEMA).put("int", (int) n); + } + Struct value_struct = new Struct(SCHEMA_V2) + .put("off16", (short) n) + .put("p_int64", n) + .put("mixed_union", unionValue); + + array.add(new SinkRecord( + topic, partition, null, null, SCHEMA_V2, value_struct, + offset + n, System.currentTimeMillis(), TimestampType.CREATE_TIME + )); + } + return array; + } } From 80f0c4e93e61362c724126db5fcbf24fc7729d78 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Thu, 26 Mar 2026 12:22:13 +0100 Subject: [PATCH 14/62] feat: add tests for auto-evolving schemas in ClickHouseSinkTask --- .../ClickHouseSinkTaskWithSchemaTest.java | 128 ++++++++++++++++++ .../connect/sink/helper/SchemaTestData.java | 93 +++++++++++++ 2 files changed, 221 insertions(+) diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index 5cf081008..cad8d9a89 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -2273,6 +2273,134 @@ public void autoEvolveTripleSchemaInOneBatch() { "V3 column 'v3_bool_field' should exist"); } + @Test + public void autoEvolveThreeSeparateBatches() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_three_batches_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + + // Batch 1: Schema V1 (3 fields: off16, p_int64, name) + List batch1 = new ArrayList<>(SchemaTestData.createRichSchemaV1(topic, 1, 5, 0)); + chst.put(batch1); + + // Batch 2: Schema V2 (8 fields: off16, p_int64, name, email, age, score, active, city) + List batch2 = new ArrayList<>(SchemaTestData.createRichSchemaV2(topic, 1, 5, 5)); + chst.put(batch2); + + // Batch 3: Schema V3 (5 fields: off16, p_int64, name, email, country) + List batch3 = new ArrayList<>(SchemaTestData.createRichSchemaV3(topic, 1, 5, 10)); + chst.put(batch3); + + chst.stop(); + + assertEquals(15, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify all evolved columns exist + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("name"), "V1 column 'name' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("email"), "V2 column 'email' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("age"), "V2 column 'age' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("score"), "V2 column 'score' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("active"), "V2 column 'active' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("city"), "V2 column 'city' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("country"), "V3 column 'country' should exist"); + + // V1 records should have NULL for V2/V3 columns + String nullEmailQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `email` IS NULL SETTINGS select_sequential_consistency = 1", topic); + // V3 records don't include age/score/active/city — those should be NULL + String nullAgeQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `age` IS NULL SETTINGS select_sequential_consistency = 1", topic); + try { + Records emailNulls = chc.getClient().queryRecords(nullEmailQuery).get(); + int emailNullCount = Integer.parseInt(emailNulls.iterator().next().getString(1)); + assertEquals(5, emailNullCount, "V1 records should have NULL for email"); + + Records ageNulls = chc.getClient().queryRecords(nullAgeQuery).get(); + int ageNullCount = Integer.parseInt(ageNulls.iterator().next().getString(1)); + assertEquals(10, ageNullCount, "V1 + V3 records (10) should have NULL for age"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Test + public void autoEvolveMixedSchemasTenRecordsInOneBatch() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_mixed_ten_records_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Single batch with 10 records spanning 3 schema versions: + // Records 1–5: Schema V1 (3 fields: off16, p_int64, name) + // Records 6–7: Schema V2 (8 fields: off16, p_int64, name, email, age, score, active, city) + // Records 8–10: Schema V3 (5 fields: off16, p_int64, name, email, country) + List batch = new ArrayList<>(); + batch.addAll(SchemaTestData.createRichSchemaV1(topic, 1, 5, 0)); + batch.addAll(SchemaTestData.createRichSchemaV2(topic, 1, 2, 5)); + batch.addAll(SchemaTestData.createRichSchemaV3(topic, 1, 3, 7)); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(batch); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify all evolved columns from all versions exist + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("name"), "V1 column 'name' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("email"), "V2 column 'email' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("age"), "V2 column 'age' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("score"), "V2 column 'score' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("active"), "V2 column 'active' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("city"), "V2 column 'city' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("country"), "V3 column 'country' should exist"); + + // Verify NULL distribution: + // name: all 10 records have it - 0 NULLs + // email: V1 records (5) lack it - 5 NULLs + // age: only V2 records (2) have it - 8 NULLs + // country: only V3 records (3) have it - 7 NULLs + try { + String nameNullQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `name` IS NULL SETTINGS select_sequential_consistency = 1", topic); + Records nameNulls = chc.getClient().queryRecords(nameNullQuery).get(); + assertEquals(0, Integer.parseInt(nameNulls.iterator().next().getString(1)), + "All records have name, so 0 NULLs expected"); + + String emailNullQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `email` IS NULL SETTINGS select_sequential_consistency = 1", topic); + Records emailNulls = chc.getClient().queryRecords(emailNullQuery).get(); + assertEquals(5, Integer.parseInt(emailNulls.iterator().next().getString(1)), + "V1 records (5) should have NULL for email"); + + String ageNullQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `age` IS NULL SETTINGS select_sequential_consistency = 1", topic); + Records ageNulls = chc.getClient().queryRecords(ageNullQuery).get(); + assertEquals(8, Integer.parseInt(ageNulls.iterator().next().getString(1)), + "V1 (5) + V3 (3) records should have NULL for age"); + + String countryNullQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `country` IS NULL SETTINGS select_sequential_consistency = 1", topic); + Records countryNulls = chc.getClient().queryRecords(countryNullQuery).get(); + assertEquals(7, Integer.parseInt(countryNulls.iterator().next().getString(1)), + "V1 (5) + V2 (2) records should have NULL for country"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + // auto-evolve adds columns for every supported primitive + logical type in a single batch @Test public void autoEvolveAllPrimitiveAndLogicalTypes() { diff --git a/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java b/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java index aacd7b122..e0b88e3af 100644 --- a/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java +++ b/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java @@ -1820,6 +1820,99 @@ public static Collection createSchemaV3WithExtraField(String topic, return array; } + /** + * Rich V1 schema with 3 fields: off16, p_int64, name. + */ + public static Collection createRichSchemaV1(String topic, int partition, int totalRecords, long startOffset) { + List array = new ArrayList<>(); + + Schema SCHEMA = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("name", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + + for (long n = 0; n < totalRecords; n++) { + Struct value_struct = new Struct(SCHEMA) + .put("off16", (short) n) + .put("p_int64", n) + .put("name", "user_" + n); + + array.add(new SinkRecord( + topic, partition, null, null, SCHEMA, value_struct, + startOffset + n, System.currentTimeMillis(), TimestampType.CREATE_TIME + )); + } + return array; + } + + /** + * Rich V2 schema with 8 fields: off16, p_int64, name, email, age, score, active, city. + */ + public static Collection createRichSchemaV2(String topic, int partition, int totalRecords, long startOffset) { + List array = new ArrayList<>(); + + Schema SCHEMA = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("name", Schema.OPTIONAL_STRING_SCHEMA) + .field("email", Schema.OPTIONAL_STRING_SCHEMA) + .field("age", SchemaBuilder.int32().optional().build()) + .field("score", SchemaBuilder.float64().optional().build()) + .field("active", SchemaBuilder.bool().optional().build()) + .field("city", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + + for (long n = 0; n < totalRecords; n++) { + Struct value_struct = new Struct(SCHEMA) + .put("off16", (short) n) + .put("p_int64", n) + .put("name", "user_" + n) + .put("email", "user_" + n + "@example.com") + .put("age", 20 + (int) n) + .put("score", n * 1.5) + .put("active", n % 2 == 0) + .put("city", "city_" + n); + + array.add(new SinkRecord( + topic, partition, null, null, SCHEMA, value_struct, + startOffset + n, System.currentTimeMillis(), TimestampType.CREATE_TIME + )); + } + return array; + } + + /** + * Rich V3 schema with 5 fields: off16, p_int64, name, email, country. + * Drops some V2 fields (age, score, active, city) and adds country. + */ + public static Collection createRichSchemaV3(String topic, int partition, int totalRecords, long startOffset) { + List array = new ArrayList<>(); + + Schema SCHEMA = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("name", Schema.OPTIONAL_STRING_SCHEMA) + .field("email", Schema.OPTIONAL_STRING_SCHEMA) + .field("country", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + + for (long n = 0; n < totalRecords; n++) { + Struct value_struct = new Struct(SCHEMA) + .put("off16", (short) n) + .put("p_int64", n) + .put("name", "user_" + n) + .put("email", "user_" + n + "@example.com") + .put("country", "country_" + n); + + array.add(new SinkRecord( + topic, partition, null, null, SCHEMA, value_struct, + startOffset + n, System.currentTimeMillis(), TimestampType.CREATE_TIME + )); + } + return array; + } + public static Collection createSchemaV2WithLogicalTypes(String topic, int partition) { return createSchemaV2WithLogicalTypes(topic, partition, DEFAULT_TOTAL_RECORDS); } From f67be3599ca9fe396a6fded667f6822c22f06794 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Thu, 26 Mar 2026 12:46:21 +0100 Subject: [PATCH 15/62] feat: improve auto-evolve functionality to handle mixed schema versions --- .../connect/sink/db/ClickHouseWriter.java | 26 ++-- .../ClickHouseSinkTaskWithSchemaTest.java | 140 ++++++++++++++++++ .../connect/sink/helper/SchemaTestData.java | 32 ++++ 3 files changed, 188 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index 7196d4501..9ec38be73 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -53,6 +53,7 @@ import java.util.ArrayList; import java.util.Date; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -212,9 +213,16 @@ public void doInsert(List records, QueryIdentifier queryId, ErrorReporte if (table == null) { return; }//We checked the error flag in getTable, so we don't need to check it again here if (csc.isAutoEvolve()) { - // New columns are Nullable, so older records without the new fields insert with NULL. - Record last = records.get(records.size() - 1); - table = evolveTableSchema(table, last); + // Since auto-evolve only adds Nullable columns (never deletes), the superset is ok. + Map allFields = new LinkedHashMap<>(); + for (Record r : records) { + if (r.getFields() != null) { + for (Field f : r.getFields()) { + allFields.putIfAbsent(f.name(), f.schema()); + } + } + } + table = evolveTableSchema(table, allFields); } doInsertBatch(records, table, queryId); @@ -842,15 +850,14 @@ protected void doWriteCol(Data value, boolean fieldExists, Column col, OutputStr } } - protected Table evolveTableSchema(Table table, Record record) throws InterruptedException { - if (record.getFields() == null) { + protected Table evolveTableSchema(Table table, Map allFields) throws InterruptedException { + if (allFields.isEmpty()) { throw new RuntimeException( "auto.evolve requires a Connect schema (Avro, Protobuf, or JSON Schema). " + "Schemaless or string records are not supported with auto.evolve=true."); } - List fieldNames = record.getFields().stream().map(Field::name).collect(Collectors.toList()); - Set missingColumns = table.getMissingColumns(fieldNames); + Set missingColumns = table.getMissingColumns(allFields.keySet()); if (missingColumns.isEmpty()) { return table; @@ -858,11 +865,10 @@ protected Table evolveTableSchema(Table table, Record record) throws Interrupted LOGGER.info("Detected {} new field(s) not present in table {}: {}", missingColumns.size(), table.getName(), missingColumns); - Map schemaMap = record.getFields().stream().collect(Collectors.toMap(Field::name, Field::schema)); - List columnDefs = new java.util.ArrayList<>(); + List columnDefs = new ArrayList<>(); for (String fieldName : missingColumns) { - Schema fieldSchema = schemaMap.get(fieldName); + Schema fieldSchema = allFields.get(fieldName); if (fieldSchema == null) { continue; } diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index cad8d9a89..c7d53834b 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -2759,4 +2759,144 @@ public void autoEvolveAvroUnionStringBytesCreatesStringColumn() throws Exception } assertEquals(2, rows.size()); } + + @Test + public void autoEvolveMixedBatchLastRecordOlderSchema() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = createTopicName("auto_evolve_last_record_older_test"); + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Build batch where V2 records come first and V1 (older) records are last. + // Before the fix, only the last record was checked - V1 has no new fields, so ALTER TABLE was skipped. + List batch = new ArrayList<>(); + batch.addAll(SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 3)); + batch.addAll(SchemaTestData.createSchemaV1(topic, 1, 2)); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(batch); + chst.stop(); + + assertEquals(5, ClickHouseTestHelpers.countRows(chc, topic)); + + // The new column from V2 should exist even though V1 was the last record + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("new_string_field"), + "Column 'new_string_field' should be added even when last record is V1 (older schema)"); + + // V1 records should have NULL for the new column + String nullCountQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `new_string_field` IS NULL SETTINGS select_sequential_consistency = 1", topic); + try { + Records nullRecords = chc.getClient().queryRecords(nullCountQuery).get(); + int nullCount = Integer.parseInt(nullRecords.iterator().next().getString(1)); + assertEquals(2, nullCount, "V1 records should have NULL for new_string_field"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Test + public void autoEvolveMultiVersionUnionSemantics() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = createTopicName("auto_evolve_union_semantics_test"); + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Batch with V1, V2, V3, V4 - each version adds a different field. + // All new fields should be added in a single ALTER TABLE. + List batch = new ArrayList<>(); + batch.addAll(SchemaTestData.createSchemaV1(topic, 1, 2)); + batch.addAll(SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 2)); + batch.addAll(SchemaTestData.createSchemaV3WithExtraField(topic, 1, 2)); + batch.addAll(SchemaTestData.createSchemaV4WithUniqueField(topic, 1, 2)); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(batch); + chst.stop(); + + assertEquals(8, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify all fields from V2, V3, V4 exist + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("new_string_field"), + "V2 column 'new_string_field' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("v3_bool_field"), + "V3 column 'v3_bool_field' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("v4_float_field"), + "V4 column 'v4_float_field' should exist"); + } + + @Test + public void autoEvolveInterleavedSchemaVersions() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = createTopicName("auto_evolve_non_monotonic_test"); + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Interleaved schema versions [V1, V3, V2, V1, V3] + List batch = new ArrayList<>(); + batch.addAll(SchemaTestData.createSchemaV1(topic, 1, 1)); + batch.addAll(SchemaTestData.createSchemaV3WithExtraField(topic, 1, 1)); + batch.addAll(SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 1)); + batch.addAll(SchemaTestData.createSchemaV1(topic, 1, 1)); + batch.addAll(SchemaTestData.createSchemaV3WithExtraField(topic, 1, 1)); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(batch); + chst.stop(); + + assertEquals(5, ClickHouseTestHelpers.countRows(chc, topic)); + + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("new_string_field"), + "V2 column 'new_string_field' should exist despite non-monotonic order"); + assertTrue(described.getRootColumnsMap().containsKey("v3_bool_field"), + "V3 column 'v3_bool_field' should exist despite non-monotonic order"); + } + + @Test + public void autoEvolveCrossPartitionSchemaDrift() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + props.put(ClickHouseSinkConfig.IGNORE_PARTITIONS_WHEN_BATCHING, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = createTopicName("auto_evolve_cross_partition_test"); + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Records from different partitions with different schema versions. + // With ignorePartitionsWhenBatching=true, they are merged into a single batch. + // Partition 0: V2 records (has new_string_field) + // Partition 1: V1 records (no new_string_field) - these may end up last + List batch = new ArrayList<>(); + batch.addAll(SchemaTestData.createSchemaV2WithNewNullableField(topic, 0, 3)); + batch.addAll(SchemaTestData.createSchemaV1(topic, 1, 3)); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(batch); + chst.stop(); + + assertEquals(6, ClickHouseTestHelpers.countRows(chc, topic)); + + // The new column from V2 (partition 0) should exist even though + // V1 records from partition 1 may be last in the merged batch + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("new_string_field"), + "Column 'new_string_field' should be added with cross-partition schema drift"); + } } diff --git a/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java b/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java index e0b88e3af..192edba4e 100644 --- a/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java +++ b/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java @@ -1820,6 +1820,38 @@ public static Collection createSchemaV3WithExtraField(String topic, return array; } + /** + * Schema V4 with a unique field (v4_float_field) not in V2 or V3. + * Fields: off16, p_int64, v4_float_field. + */ + public static Collection createSchemaV4WithUniqueField(String topic, int partition) { + return createSchemaV4WithUniqueField(topic, partition, DEFAULT_TOTAL_RECORDS); + } + + public static Collection createSchemaV4WithUniqueField(String topic, int partition, int totalRecords) { + List array = new ArrayList<>(); + + Schema SCHEMA_V4 = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("v4_float_field", SchemaBuilder.float64().optional().build()) + .build(); + + long offset = totalRecords * 3L; // after V1, V2, V3 offsets + for (long n = 0; n < totalRecords; n++) { + Struct value_struct = new Struct(SCHEMA_V4) + .put("off16", (short) n) + .put("p_int64", n) + .put("v4_float_field", (double) n * 1.5); + + array.add(new SinkRecord( + topic, partition, null, null, SCHEMA_V4, value_struct, + offset + n, System.currentTimeMillis(), TimestampType.CREATE_TIME + )); + } + return array; + } + /** * Rich V1 schema with 3 fields: off16, p_int64, name. */ From e0d340a46360fe280150e2cb2917655acf930d90 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Fri, 27 Mar 2026 11:52:47 +0100 Subject: [PATCH 16/62] fix: remove unused import --- .../com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java index ddd24a286..0fba5cc60 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java @@ -4,7 +4,6 @@ import org.apache.kafka.connect.data.SchemaBuilder; import org.junit.jupiter.api.Test; -import java.util.Collections; import java.util.List; import static com.clickhouse.kafka.connect.sink.helper.ClickHouseTestHelpers.col; From 5b763eeb963f8f3dc85fdfe1f2273d3d45b27de9 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Mon, 23 Mar 2026 17:47:07 +0100 Subject: [PATCH 17/62] feature: enable auto.evolve parameter --- CHANGELOG.md | 3 ++- .../com/clickhouse/kafka/connect/sink/db/mapping/Column.java | 1 + .../kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java | 1 + .../clickhouse/kafka/connect/sink/helper/SchemaTestData.java | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70a98d143..c3584f173 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ ## Bug Fixes * Fixed RowBinary serialization for Map columns with Nullable value types. The nullable marker byte was missing when writing map values, causing `CANNOT_READ_ALL_DATA` errors for `Map(K, Nullable(V))` columns. -# 1.3.7, 2026-03-25 +# 1.3.7, 2026-03-25 ## Security * Upgraded `com.fasterxml.jackson.core` dependencies to version with fix for https://github.com/advisories/GHSA-72hv-8253-57qq (https://github.com/ClickHouse/clickhouse-kafka-connect/pull/690). @@ -14,6 +14,7 @@ # Improvements * `Gson` replaced with `Jackson` for performance and better maintainability (https://github.com/ClickHouse/clickhouse-kafka-connect/pull/676). + # 1.3.6, 2026-03-18 ## New Features diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java index f8f16ee7f..18558686e 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java @@ -515,6 +515,7 @@ static boolean isUnionSchema(Schema connectSchema) { && connectSchema.parameters().containsKey(CONNECT_UNION_PARAMETER); } + public Integer convertEnumValues(String value) { if ( this.enumValues != null ) { return enumValues.get(value); diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index c7d53834b..be7838aad 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -2210,6 +2210,7 @@ public void autoEvolveStructToJsonExplicitlyFalseRejectsStruct() { } } + @Test public void autoEvolveArrayAndMapFields() { Map props = createProps(); diff --git a/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java b/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java index 192edba4e..f159d046d 100644 --- a/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java +++ b/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java @@ -1945,6 +1945,7 @@ public static Collection createRichSchemaV3(String topic, int partit return array; } + public static Collection createSchemaV2WithLogicalTypes(String topic, int partition) { return createSchemaV2WithLogicalTypes(topic, partition, DEFAULT_TOTAL_RECORDS); } From 22092433899aa7924be641299f4ece89be48b4f4 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 20:17:55 +0100 Subject: [PATCH 18/62] feat: add auto evolve struct to JSON configuration option to ClickHouseSinkConfig --- .../kafka/connect/sink/ClickHouseSinkConfig.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java b/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java index 22b1aff60..23252609b 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java @@ -736,6 +736,17 @@ private static ConfigDef createConfigDef() { ConfigDef.Width.SHORT, "Map STRUCT to JSON" ); + configDef.define(SSL_SOCKET_SNI, + ConfigDef.Type.STRING, + "", + ConfigDef.Importance.LOW, + "Override the SNI hostname sent in the client handshake. When set, the client will explicitly include the specified name as the server_name extension in the TLS ClientHello." + + "This is useful to avoid handshake failure when routing TLS traffic through a proxy, where the proxy hostname and the server hostname may differ. Default: ''", + group, + ++orderInGroup, + ConfigDef.Width.MEDIUM, + "SSL Socket SNI" + ); return configDef; } } From b1e616aca59819c96fe74e6980524790729aa17c Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 23:14:00 +0100 Subject: [PATCH 19/62] feat: enhance schema handling by adding union type detection and mapping for ClickHouse integration --- .../ClickHouseSinkTaskWithSchemaTest.java | 264 ++++++++++++++++++ .../connect/sink/db/mapping/ColumnTest.java | 1 + 2 files changed, 265 insertions(+) diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index be7838aad..8ec3c9c35 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -2274,6 +2274,270 @@ public void autoEvolveTripleSchemaInOneBatch() { "V3 column 'v3_bool_field' should exist"); } + // auto-evolve adds columns for every supported primitive + logical type in a single batch + @Test + public void autoEvolveAllPrimitiveAndLogicalTypes() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_all_types_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Insert V1 first to ensure existing rows get NULL for new columns + Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 5); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV1); + assertEquals(5, ClickHouseTestHelpers.countRows(chc, topic)); + + // Insert V2 with all primitive + logical type fields + Collection srV2 = SchemaTestData.createSchemaV2WithAllPrimitiveTypes(topic, 1, 5); + chst.put(srV2); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify all columns were created with correct types + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + Map cols = described.getRootColumnsMap(); + + // Primitive types + assertTrue(cols.containsKey("new_int8"), "Column 'new_int8' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.INT8, cols.get("new_int8").getType()); + assertTrue(cols.containsKey("new_int16"), "Column 'new_int16' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.INT16, cols.get("new_int16").getType()); + assertTrue(cols.containsKey("new_int32"), "Column 'new_int32' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.INT32, cols.get("new_int32").getType()); + assertTrue(cols.containsKey("new_int64"), "Column 'new_int64' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.INT64, cols.get("new_int64").getType()); + assertTrue(cols.containsKey("new_float32"), "Column 'new_float32' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.FLOAT32, cols.get("new_float32").getType()); + assertTrue(cols.containsKey("new_float64"), "Column 'new_float64' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.FLOAT64, cols.get("new_float64").getType()); + assertTrue(cols.containsKey("new_bool"), "Column 'new_bool' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.BOOLEAN, cols.get("new_bool").getType()); + assertTrue(cols.containsKey("new_string"), "Column 'new_string' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.STRING, cols.get("new_string").getType()); + assertTrue(cols.containsKey("new_bytes"), "Column 'new_bytes' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.STRING, cols.get("new_bytes").getType()); + + // Logical types + assertTrue(cols.containsKey("new_decimal"), "Column 'new_decimal' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.Decimal, cols.get("new_decimal").getType()); + assertTrue(cols.containsKey("new_date"), "Column 'new_date' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.Date32, cols.get("new_date").getType()); + assertTrue(cols.containsKey("new_timestamp"), "Column 'new_timestamp' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.DateTime64, cols.get("new_timestamp").getType()); + } + + // auto-evolve creates Array columns with different element types (Int32, Float64, Bool, String) + @Test + public void autoEvolveTypedArrayColumns() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_typed_arrays_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + Collection srV2 = SchemaTestData.createSchemaV2WithTypedArrays(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV2); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify all array columns were created + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + Map cols = described.getRootColumnsMap(); + + assertTrue(cols.containsKey("arr_int32"), "Column 'arr_int32' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.ARRAY, cols.get("arr_int32").getType()); + assertTrue(cols.containsKey("arr_float64"), "Column 'arr_float64' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.ARRAY, cols.get("arr_float64").getType()); + assertTrue(cols.containsKey("arr_bool"), "Column 'arr_bool' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.ARRAY, cols.get("arr_bool").getType()); + assertTrue(cols.containsKey("arr_string"), "Column 'arr_string' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.ARRAY, cols.get("arr_string").getType()); + } + + // DDL refresh timeout - retries set to 0 so refresh loop never runs, throws RetriableException + @Test + public void autoEvolveDdlRefreshTimeoutThrowsRetriable() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE_DDL_REFRESH_RETRIES, "0"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_ddl_timeout_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // V2 schema with a new field - DDL will succeed but refresh will timeout with 0 retries + Collection srV2 = SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 5); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + + try { + chst.put(srV2); + assertTrue(false, "Expected RetriableException due to DDL refresh timeout with 0 retries"); + } catch (RuntimeException e) { + // Processing layer may wrap the RetriableException - walk the cause chain + Throwable t = e; + boolean found = false; + while (t != null) { + if (t instanceof org.apache.kafka.connect.errors.RetriableException + && t.getMessage() != null && t.getMessage().contains("DDL propagation timeout")) { + found = true; + break; + } + t = t.getCause(); + } + assertTrue(found, "Should contain RetriableException with DDL propagation timeout in cause chain, got: " + e); + } finally { + chst.stop(); + } + } + + // ALTER TABLE itself fails (table dropped externally after cache populated) + @Test + public void autoEvolveDdlExecutionFailureThrowsRuntimeException() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_ddl_exec_failure_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Insert V1 records to populate the connector's internal table mapping cache + Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 5); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV1); + assertEquals(5, ClickHouseTestHelpers.countRows(chc, topic)); + + // Drop the table externally - the connector still has it cached in memory + ClickHouseTestHelpers.dropTable(chc, topic); + + // V2 schema with a new field - ALTER TABLE will fail because the table no longer exists + Collection srV2 = SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 5); + try { + chst.put(srV2); + assertTrue(false, "Expected RuntimeException due to ALTER TABLE on dropped table"); + } catch (RuntimeException e) { + // Processing layer wraps exceptions - walk the cause chain for the DDL failure + Throwable t = e; + boolean found = false; + while (t != null) { + if (t.getMessage() != null && (t.getMessage().contains("ALTER TABLE") || t.getMessage().contains("UNKNOWN_TABLE"))) { + found = true; + break; + } + t = t.getCause(); + } + assertTrue(found, "Should indicate DDL failure in cause chain, got: " + e.getClass().getName() + ": " + e.getMessage()); + } finally { + chst.stop(); + } + } + + // STRUCT field without struct-to-json flag throws SchemaTypeInferenceException with helpful message + @Test + public void autoEvolveUnsupportedStructTypeThrowsError() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + // auto.evolve.struct.to.json is false by default + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_unsupported_struct_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + Collection srV2 = SchemaTestData.createSchemaV2WithStructField(topic, 1, 5); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + + try { + chst.put(srV2); + assertTrue(false, "Expected SchemaTypeInferenceException for unsupported STRUCT type"); + } catch (RuntimeException e) { + Throwable t = e; + boolean foundInference = false; + while (t != null) { + if (t instanceof com.clickhouse.kafka.connect.sink.db.mapping.SchemaTypeInferenceException) { + foundInference = true; + break; + } + t = t.getCause(); + } + assertTrue(foundInference, + "Should throw SchemaTypeInferenceException for unsupported STRUCT, got: " + e.getClass().getName() + ": " + e.getMessage()); + } finally { + chst.stop(); + } + } + + // Avro-style union(string, bytes) STRUCT collapses to Nullable(String), not JSON + @Test + public void autoEvolveStringBytesUnionCollapsesToString() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_union_string_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + Collection srV2 = SchemaTestData.createSchemaV2WithStringBytesUnionField(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV2); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify the union field was created as String (not JSON) + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + com.clickhouse.kafka.connect.sink.db.mapping.Column col = described.getRootColumnsMap().get("new_union_field"); + assertNotNull(col, "Column 'new_union_field' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.STRING, col.getType(), + "Union(string, bytes) should collapse to String, not JSON"); + } + + // Avro union(string, int) auto-evolved as Variant(String, Int32) column + @Test + @SinceClickHouseVersion("24.1") + public void autoEvolveMixedUnionCreatesVariantColumn() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + props.put(ClickHouseSinkConfig.CLICKHOUSE_SETTINGS, "allow_experimental_variant_type=1"); + ClickHouseHelperClient chc = createClient(props); + + String topic = createTopicName("auto_evolve_variant_test"); + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + Collection records = SchemaTestData.createSchemaV2WithMixedTypeUnionField(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(records); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify the union field was created as Variant + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + com.clickhouse.kafka.connect.sink.db.mapping.Column col = described.getRootColumnsMap().get("mixed_union"); + assertNotNull(col, "Column 'mixed_union' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.VARIANT, col.getType(), + "union(string, int) should map to Variant, not String or JSON"); + } + @Test public void autoEvolveThreeSeparateBatches() { Map props = createProps(); diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java index 0fba5cc60..ddd24a286 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java @@ -4,6 +4,7 @@ import org.apache.kafka.connect.data.SchemaBuilder; import org.junit.jupiter.api.Test; +import java.util.Collections; import java.util.List; import static com.clickhouse.kafka.connect.sink.helper.ClickHouseTestHelpers.col; From cd28e13a631d2bde1b4df611f518e8a562f99590 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Thu, 26 Mar 2026 12:22:13 +0100 Subject: [PATCH 20/62] feat: add tests for auto-evolving schemas in ClickHouseSinkTask --- .../ClickHouseSinkTaskWithSchemaTest.java | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index 8ec3c9c35..c60ec6ca0 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -2274,6 +2274,134 @@ public void autoEvolveTripleSchemaInOneBatch() { "V3 column 'v3_bool_field' should exist"); } + @Test + public void autoEvolveThreeSeparateBatches() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_three_batches_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + + // Batch 1: Schema V1 (3 fields: off16, p_int64, name) + List batch1 = new ArrayList<>(SchemaTestData.createRichSchemaV1(topic, 1, 5, 0)); + chst.put(batch1); + + // Batch 2: Schema V2 (8 fields: off16, p_int64, name, email, age, score, active, city) + List batch2 = new ArrayList<>(SchemaTestData.createRichSchemaV2(topic, 1, 5, 5)); + chst.put(batch2); + + // Batch 3: Schema V3 (5 fields: off16, p_int64, name, email, country) + List batch3 = new ArrayList<>(SchemaTestData.createRichSchemaV3(topic, 1, 5, 10)); + chst.put(batch3); + + chst.stop(); + + assertEquals(15, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify all evolved columns exist + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("name"), "V1 column 'name' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("email"), "V2 column 'email' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("age"), "V2 column 'age' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("score"), "V2 column 'score' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("active"), "V2 column 'active' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("city"), "V2 column 'city' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("country"), "V3 column 'country' should exist"); + + // V1 records should have NULL for V2/V3 columns + String nullEmailQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `email` IS NULL SETTINGS select_sequential_consistency = 1", topic); + // V3 records don't include age/score/active/city — those should be NULL + String nullAgeQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `age` IS NULL SETTINGS select_sequential_consistency = 1", topic); + try { + Records emailNulls = chc.getClient().queryRecords(nullEmailQuery).get(); + int emailNullCount = Integer.parseInt(emailNulls.iterator().next().getString(1)); + assertEquals(5, emailNullCount, "V1 records should have NULL for email"); + + Records ageNulls = chc.getClient().queryRecords(nullAgeQuery).get(); + int ageNullCount = Integer.parseInt(ageNulls.iterator().next().getString(1)); + assertEquals(10, ageNullCount, "V1 + V3 records (10) should have NULL for age"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Test + public void autoEvolveMixedSchemasTenRecordsInOneBatch() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_mixed_ten_records_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Single batch with 10 records spanning 3 schema versions: + // Records 1–5: Schema V1 (3 fields: off16, p_int64, name) + // Records 6–7: Schema V2 (8 fields: off16, p_int64, name, email, age, score, active, city) + // Records 8–10: Schema V3 (5 fields: off16, p_int64, name, email, country) + List batch = new ArrayList<>(); + batch.addAll(SchemaTestData.createRichSchemaV1(topic, 1, 5, 0)); + batch.addAll(SchemaTestData.createRichSchemaV2(topic, 1, 2, 5)); + batch.addAll(SchemaTestData.createRichSchemaV3(topic, 1, 3, 7)); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(batch); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify all evolved columns from all versions exist + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("name"), "V1 column 'name' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("email"), "V2 column 'email' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("age"), "V2 column 'age' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("score"), "V2 column 'score' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("active"), "V2 column 'active' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("city"), "V2 column 'city' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("country"), "V3 column 'country' should exist"); + + // Verify NULL distribution: + // name: all 10 records have it - 0 NULLs + // email: V1 records (5) lack it - 5 NULLs + // age: only V2 records (2) have it - 8 NULLs + // country: only V3 records (3) have it - 7 NULLs + try { + String nameNullQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `name` IS NULL SETTINGS select_sequential_consistency = 1", topic); + Records nameNulls = chc.getClient().queryRecords(nameNullQuery).get(); + assertEquals(0, Integer.parseInt(nameNulls.iterator().next().getString(1)), + "All records have name, so 0 NULLs expected"); + + String emailNullQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `email` IS NULL SETTINGS select_sequential_consistency = 1", topic); + Records emailNulls = chc.getClient().queryRecords(emailNullQuery).get(); + assertEquals(5, Integer.parseInt(emailNulls.iterator().next().getString(1)), + "V1 records (5) should have NULL for email"); + + String ageNullQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `age` IS NULL SETTINGS select_sequential_consistency = 1", topic); + Records ageNulls = chc.getClient().queryRecords(ageNullQuery).get(); + assertEquals(8, Integer.parseInt(ageNulls.iterator().next().getString(1)), + "V1 (5) + V3 (3) records should have NULL for age"); + + String countryNullQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `country` IS NULL SETTINGS select_sequential_consistency = 1", topic); + Records countryNulls = chc.getClient().queryRecords(countryNullQuery).get(); + assertEquals(7, Integer.parseInt(countryNulls.iterator().next().getString(1)), + "V1 (5) + V2 (2) records should have NULL for country"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + // auto-evolve adds columns for every supported primitive + logical type in a single batch @Test public void autoEvolveAllPrimitiveAndLogicalTypes() { From aec1935d300c0f66d0348b076ffdf4d31737f032 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Fri, 27 Mar 2026 11:52:47 +0100 Subject: [PATCH 21/62] fix: remove unused import --- .../com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java index ddd24a286..0fba5cc60 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java @@ -4,7 +4,6 @@ import org.apache.kafka.connect.data.SchemaBuilder; import org.junit.jupiter.api.Test; -import java.util.Collections; import java.util.List; import static com.clickhouse.kafka.connect.sink.helper.ClickHouseTestHelpers.col; From f8eacc5da99d4cc21a9ac97c74c9d09606c91c8c Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Fri, 27 Mar 2026 12:09:37 +0100 Subject: [PATCH 22/62] fix: escape column names in ClickHouseWriter to prevent SQL injection issues --- .../com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index 9ec38be73..67768a066 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -874,7 +874,7 @@ protected Table evolveTableSchema(Table table, Map allFields) th } String chType = Column.connectTypeToClickHouseType(fieldSchema, csc.isAutoEvolveStructToJson()); - columnDefs.add(String.format("`%s` %s", fieldName, chType)); + columnDefs.add(String.format("%s %s", Utils.escapeName(fieldName), chType)); } if (!columnDefs.isEmpty()) { From bd9ba2d0db9a7e8c7c03c389eabb35a716120984 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Fri, 27 Mar 2026 12:11:12 +0100 Subject: [PATCH 23/62] refactor: remove duplicated code from merge --- .../kafka/connect/sink/ClickHouseSinkConfig.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java b/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java index 23252609b..22b1aff60 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java @@ -736,17 +736,6 @@ private static ConfigDef createConfigDef() { ConfigDef.Width.SHORT, "Map STRUCT to JSON" ); - configDef.define(SSL_SOCKET_SNI, - ConfigDef.Type.STRING, - "", - ConfigDef.Importance.LOW, - "Override the SNI hostname sent in the client handshake. When set, the client will explicitly include the specified name as the server_name extension in the TLS ClientHello." + - "This is useful to avoid handshake failure when routing TLS traffic through a proxy, where the proxy hostname and the server hostname may differ. Default: ''", - group, - ++orderInGroup, - ConfigDef.Width.MEDIUM, - "SSL Socket SNI" - ); return configDef; } } From f2235e2479f14ebde6b47f5bf566f3fb3e70c4a1 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Fri, 27 Mar 2026 13:42:10 +0100 Subject: [PATCH 24/62] feat: add default expressions for Array and Map types --- .../connect/sink/db/ClickHouseWriter.java | 3 +- .../kafka/connect/sink/db/mapping/Column.java | 10 ++++++ .../ClickHouseSinkTaskWithSchemaTest.java | 34 +++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index 67768a066..4f9659cd7 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -874,7 +874,8 @@ protected Table evolveTableSchema(Table table, Map allFields) th } String chType = Column.connectTypeToClickHouseType(fieldSchema, csc.isAutoEvolveStructToJson()); - columnDefs.add(String.format("%s %s", Utils.escapeName(fieldName), chType)); + String defaultExpr = Column.defaultExpressionForType(chType); + columnDefs.add(String.format("%s %s%s", Utils.escapeName(fieldName), chType, defaultExpr)); } if (!columnDefs.isEmpty()) { diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java index 18558686e..fbf21e60d 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java @@ -549,4 +549,14 @@ public String toString() { String.format(", variantTypes=%s", variantTypes.stream().map(Tuple2::getT2).collect(Collectors.joining(", ", "[", "]"))) ) + "}"; } + + // Returns a DEFAULT expression for non-Nullable types (Array, Map) so RowBinaryWithDefaults can handle missing fields. + public static String defaultExpressionForType(String chType) { + if (chType.startsWith("Array(")) { + return " DEFAULT []"; + } else if (chType.startsWith("Map(")) { + return " DEFAULT map()"; + } + return ""; + } } diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index c60ec6ca0..87b8f190e 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -3292,4 +3292,38 @@ public void autoEvolveCrossPartitionSchemaDrift() { assertTrue(described.getRootColumnsMap().containsKey("new_string_field"), "Column 'new_string_field' should be added with cross-partition schema drift"); } + + // Mixed batch where older records lack auto-evolved Array/Map columns. + @Test + public void autoEvolveMixedBatchArrayMapFieldsMissingInOlderRecords() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = createTopicName("auto_evolve_mixed_array_map_test"); + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, + "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Build a single batch: V1 records (no array/map) followed by V2 records (with array/map) + List mixedBatch = new ArrayList<>(); + mixedBatch.addAll(SchemaTestData.createSchemaV1(topic, 1, 5)); + mixedBatch.addAll(SchemaTestData.createSchemaV2WithArrayAndMapFields(topic, 1, 5)); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(mixedBatch); + chst.stop(); + + // All 10 records should be inserted + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // The new array and map columns should exist + com.clickhouse.kafka.connect.sink.db.mapping.Table described = + chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("new_array_field"), + "Column 'new_array_field' should exist after auto-evolve"); + assertTrue(described.getRootColumnsMap().containsKey("new_map_field"), + "Column 'new_map_field' should exist after auto-evolve"); + } } From 16c0a4470148c445f9b1452d1134687d019954ae Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Fri, 27 Mar 2026 14:02:31 +0100 Subject: [PATCH 25/62] fix: handle Variant columns in mixed-schema batches with auto.evolve --- .../connect/sink/db/ClickHouseWriter.java | 9 ++++- .../ClickHouseSinkTaskWithSchemaTest.java | 34 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index 4f9659cd7..a059a2d86 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -257,7 +257,8 @@ protected boolean validateDataSchema(Table table, Record record, boolean onlyFie Type type = col.getType(); boolean isNullable = col.isNullable(); boolean hasDefault = col.hasDefault(); - if (!isNullable && !hasDefault) { + // Variant has a native NULL discriminator (255) so it can accept missing values without Nullable or DEFAULT. + if (!isNullable && !hasDefault && type != Type.VARIANT) { Map schemaMap = record.getFields().stream().collect(Collectors.toMap(Field::name, Field::schema)); var objSchema = schemaMap.get(colName); Data obj = record.getJsonMap().get(colName); @@ -842,6 +843,12 @@ protected void doWriteCol(Data value, boolean fieldExists, Column col, OutputStr BinaryStreamUtils.writeNonNull(stream); } BinaryStreamUtils.writeNull(stream); + } else if (col.getType() == Type.VARIANT) { + // Variant has a native NULL discriminator (255) — no Nullable/DEFAULT needed. + if (defaultsSupport) { + BinaryStreamUtils.writeNonNull(stream); + } + BinaryStreamUtils.writeUnsignedInt8(stream, 255); } else { // no filled and not nullable LOGGER.error("Column {} is not nullable and no value is provided", name); diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index 87b8f190e..e313118cb 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -3326,4 +3326,38 @@ public void autoEvolveMixedBatchArrayMapFieldsMissingInOlderRecords() { assertTrue(described.getRootColumnsMap().containsKey("new_map_field"), "Column 'new_map_field' should exist after auto-evolve"); } + + // Mixed batch where older records lack an auto-evolved Variant column. + @Test + public void autoEvolveMixedBatchVariantFieldMissingInOlderRecords() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + props.put(ClickHouseSinkConfig.CLICKHOUSE_SETTINGS, "allow_experimental_variant_type=1"); + ClickHouseHelperClient chc = createClient(props); + + String topic = createTopicName("auto_evolve_mixed_variant_test"); + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, + "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // V1 records (off16 + p_int64 only) followed by V2 records (off16 + p_int64 + mixed_union Variant) + List mixedBatch = new ArrayList<>(); + mixedBatch.addAll(SchemaTestData.createSchemaV1(topic, 1, 5)); + mixedBatch.addAll(SchemaTestData.createSchemaV2WithMixedTypeUnionField(topic, 1, 5)); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(mixedBatch); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify the Variant column was created + com.clickhouse.kafka.connect.sink.db.mapping.Table described = + chc.describeTable(chc.getDatabase(), topic); + com.clickhouse.kafka.connect.sink.db.mapping.Column col = described.getRootColumnsMap().get("mixed_union"); + assertNotNull(col, "Column 'mixed_union' should exist after auto-evolve"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.VARIANT, col.getType(), + "mixed_union should be Variant type"); + } } From 95e330a7e8e1a48ecb540b3961c45caff4a11703 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Fri, 27 Mar 2026 19:44:09 +0100 Subject: [PATCH 26/62] feat: optimize field extraction in auto-evolve by using IdentityHashMap for deduplication --- .../kafka/connect/sink/db/ClickHouseWriter.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index a059a2d86..1546f1e92 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -52,7 +52,9 @@ import java.time.temporal.ChronoField; import java.util.ArrayList; import java.util.Date; +import java.util.Collections; import java.util.HashMap; +import java.util.IdentityHashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -213,10 +215,12 @@ public void doInsert(List records, QueryIdentifier queryId, ErrorReporte if (table == null) { return; }//We checked the error flag in getTable, so we don't need to check it again here if (csc.isAutoEvolve()) { - // Since auto-evolve only adds Nullable columns (never deletes), the superset is ok. + // Collect the union of fields across all distinct schema versions in the batch. + // IdentityHashMap dedup ensures field extraction happens once per Schema object instance + Set seen = Collections.newSetFromMap(new IdentityHashMap<>()); Map allFields = new LinkedHashMap<>(); for (Record r : records) { - if (r.getFields() != null) { + if (r.getFields() != null && seen.add(r.getSinkRecord().valueSchema())) { for (Field f : r.getFields()) { allFields.putIfAbsent(f.name(), f.schema()); } From bcc0874628680bd91bfff0c0bb0e68d79c44ff4a Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Sun, 29 Mar 2026 20:02:58 +0200 Subject: [PATCH 27/62] feat: forward clickhouse settings to ALTER TABLE DDL in auto-evolve --- .../connect/sink/db/ClickHouseWriter.java | 2 +- .../db/helper/ClickHouseHelperClient.java | 31 ++++++++++++------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index 1546f1e92..ce457e973 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -890,7 +890,7 @@ protected Table evolveTableSchema(Table table, Map allFields) th } if (!columnDefs.isEmpty()) { - chc.alterTableAddColumns(table.getDatabase(), table.getCleanName(), columnDefs); + chc.alterTableAddColumns(table.getDatabase(), table.getCleanName(), columnDefs, csc.getClickhouseSettings()); LOGGER.info("Schema evolution complete for table {}. Added columns: {}", table.getName(), columnDefs); table = refreshTableAfterDDL(table, missingColumns); } diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java index 98c553208..cc513d2d8 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java @@ -5,6 +5,7 @@ import com.clickhouse.client.ClickHouseNode; import com.clickhouse.client.ClickHouseNodeSelector; import com.clickhouse.client.ClickHouseProtocol; +import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.ClickHouseResponse; import com.clickhouse.client.api.Client; import com.clickhouse.client.api.enums.ProxyType; @@ -476,36 +477,42 @@ public Table describeTableV2(String database, String tableName) { return table; } - public void alterTableAddColumns(String database, String tableName, List columnDefs) { + public void alterTableAddColumns(String database, String tableName, List columnDefs, Map clickhouseSettings) { String addClauses = columnDefs.stream() .map(colDef -> "ADD COLUMN IF NOT EXISTS " + colDef) .collect(Collectors.joining(", ")); String sql = String.format("ALTER TABLE `%s`.`%s` %s", database, tableName, addClauses); LOGGER.info("Executing DDL: {}", sql); if (useClientV2) { - alterTableAddColumnV2(sql); + alterTableAddColumnV2(sql, clickhouseSettings); } else { - alterTableAddColumnV1(sql); + alterTableAddColumnV1(sql, clickhouseSettings); } } - private void alterTableAddColumnV1(String sql) { + private void alterTableAddColumnV1(String sql, Map clickhouseSettings) { try (ClickHouseClient client = ClickHouseClient.builder() .options(getDefaultClientOptions()) .nodeSelector(ClickHouseNodeSelector.of(ClickHouseProtocol.HTTP)) - .build(); - ClickHouseResponse response = client.read(server) - .query(sql) - .set("alter_sync", "1") - .executeAndWait()) { - // DDL executed; alter_sync=1 waits for the local replica to apply + .build()) { + ClickHouseRequest request = client.read(server).query(sql).set("alter_sync", "1"); + for (Map.Entry entry : clickhouseSettings.entrySet()) { + request.set(entry.getKey(), entry.getValue()); + } + try (ClickHouseResponse response = request.executeAndWait()) { + // DDL executed; alter_sync=1 waits for the local replica to apply + } } catch (ClickHouseException e) { throw new RuntimeException("Failed to execute ALTER TABLE: " + sql, e); } } - private void alterTableAddColumnV2(String sql) { - try (QueryResponse response = client.query(sql, new QuerySettings().serverSetting("alter_sync", "1")).get()) { + private void alterTableAddColumnV2(String sql, Map clickhouseSettings) { + QuerySettings settings = new QuerySettings().serverSetting("alter_sync", "1"); + for (Map.Entry entry : clickhouseSettings.entrySet()) { + settings.serverSetting(entry.getKey(), entry.getValue()); + } + try (QueryResponse response = client.query(sql, settings).get()) { // DDL executed; alter_sync=1 waits for the local replica to apply } catch (Exception e) { throw new RuntimeException("Failed to execute ALTER TABLE: " + sql, e); From 598171b93776afbef77a9b59c6efb379be53e9a5 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Sun, 29 Mar 2026 20:03:18 +0200 Subject: [PATCH 28/62] fix: handle Variant and union null serialization in RowBinary writer --- .../kafka/connect/sink/db/ClickHouseWriter.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index ce457e973..784c1b434 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -761,7 +761,7 @@ protected void doWritePrimitive(Type columnType, Schema.Type dataType, OutputStr } else if (unionData.getObject() instanceof byte[]) { BinaryStreamUtils.writeString(stream, (byte[]) unionData.getObject()); } else { - throw new DataException("Not implemented conversion from " + unionData.getObject().getClass() + " to String"); + BinaryStreamUtils.writeString(stream, unionData.getObject().toString().getBytes(StandardCharsets.UTF_8)); } break; } @@ -820,6 +820,10 @@ protected void doWriteCol(Data value, boolean fieldExists, Column col, OutputStr return;//And we're done } else if (colType == Type.ARRAY) {//If the column is an array BinaryStreamUtils.writeNonNull(stream);//Then we send nonNull + } else if (colType == Type.VARIANT) { + BinaryStreamUtils.writeNonNull(stream); + BinaryStreamUtils.writeUnsignedInt8(stream, 255); + return; } else { throw new RuntimeException(String.format("An attempt to write null into not nullable column '%s'", name)); } @@ -832,7 +836,10 @@ protected void doWriteCol(Data value, boolean fieldExists, Column col, OutputStr if (!col.isNullable() && value.getObject() == null) { if (colType == Type.ARRAY) BinaryStreamUtils.writeNonNull(stream); - else + else if (colType == Type.VARIANT) { + BinaryStreamUtils.writeUnsignedInt8(stream, 255); + return; + } else throw new RuntimeException(String.format("An attempt to write null into not nullable column '%s'", name)); } } From d3a72a51dc843a2ba20838a6c25bbc8077f1b001 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Sun, 29 Mar 2026 20:03:22 +0200 Subject: [PATCH 29/62] refactor: simplify auto-evolve to check only last record schema --- .../connect/sink/db/ClickHouseWriter.java | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index 784c1b434..85c767196 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -52,9 +52,7 @@ import java.time.temporal.ChronoField; import java.util.ArrayList; import java.util.Date; -import java.util.Collections; import java.util.HashMap; -import java.util.IdentityHashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -215,18 +213,17 @@ public void doInsert(List records, QueryIdentifier queryId, ErrorReporte if (table == null) { return; }//We checked the error flag in getTable, so we don't need to check it again here if (csc.isAutoEvolve()) { - // Collect the union of fields across all distinct schema versions in the batch. - // IdentityHashMap dedup ensures field extraction happens once per Schema object instance - Set seen = Collections.newSetFromMap(new IdentityHashMap<>()); - Map allFields = new LinkedHashMap<>(); - for (Record r : records) { - if (r.getFields() != null && seen.add(r.getSinkRecord().valueSchema())) { - for (Field f : r.getFields()) { - allFields.putIfAbsent(f.name(), f.schema()); - } + // Check the last record's schema for new fields. + // Limitation: if a batch contains multiple schema versions, only the last one is checked. + // A full scan across all records can be implemented later if needed. + Record last = records.get(records.size() - 1); + Map lastFields = new LinkedHashMap<>(); + if (last.getFields() != null) { + for (Field f : last.getFields()) { + lastFields.put(f.name(), f.schema()); } } - table = evolveTableSchema(table, allFields); + table = evolveTableSchema(table, lastFields); } doInsertBatch(records, table, queryId); From 8646bc048e7e63b7db813ebc9e96c8b28acb49a7 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 1 Apr 2026 09:08:10 +0200 Subject: [PATCH 30/62] fix: replace createTable with runQuery to match main convention --- .../ClickHouseSinkTaskWithSchemaTest.java | 460 ++---------------- .../sink/helper/ClickHouseTestHelpers.java | 1 + 2 files changed, 35 insertions(+), 426 deletions(-) diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index e313118cb..cd3598876 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -1813,7 +1813,7 @@ public void autoEvolveDisabledRejectsNewField() { String topic = "auto_evolve_disabled_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Insert V1 records first (should succeed) Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 10); @@ -1840,7 +1840,7 @@ public void autoEvolveAddsNullableColumn() { String topic = "auto_evolve_nullable_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Insert V1 records Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 10); @@ -1870,7 +1870,7 @@ public void autoEvolveAddsNonNullableFieldAsNullable() { String topic = "auto_evolve_non_nullable_as_nullable_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); Collection srV2 = SchemaTestData.createSchemaV2WithNewNonNullableField(topic, 1, 10); ClickHouseSinkTask chst = new ClickHouseSinkTask(); @@ -1895,7 +1895,7 @@ public void autoEvolveMultipleNewColumns() { String topic = "auto_evolve_multi_cols_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Insert V1 first Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 10); @@ -1929,7 +1929,7 @@ public void autoEvolveCachesSchemaAfterDDL() { String topic = "auto_evolve_cache_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); ClickHouseSinkTask chst = new ClickHouseSinkTask(); chst.start(props); @@ -1963,7 +1963,7 @@ public void autoEvolveMixedSchemaInSingleBatch() { String topic = "auto_evolve_mixed_batch_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Build a single batch with V1 records followed by V2 records (mixed schemas) List mixedBatch = new ArrayList<>(); @@ -1992,7 +1992,7 @@ public void autoEvolveMixedSchemaOlderRecordsGetNull() { String topic = "auto_evolve_older_records_null_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // V1 records (no new_string_field) followed by V2 records (has new_string_field) // Schema is evolved using last record (V2), then entire batch is inserted. @@ -2039,7 +2039,7 @@ public void autoEvolveLogicalTypes() { String topic = "auto_evolve_logical_types_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Insert V1 first Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 10); @@ -2086,7 +2086,7 @@ public void autoEvolveRejectsStructField() { String topic = "auto_evolve_struct_reject_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); Collection srV2 = SchemaTestData.createSchemaV2WithStructField(topic, 1, 10); ClickHouseSinkTask chst = new ClickHouseSinkTask(); @@ -2122,7 +2122,7 @@ public void autoEvolveStructToJsonCreatesJsonColumn() { String topic = "auto_evolve_struct_to_json_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); Collection srV2 = SchemaTestData.createSchemaV2WithStructField(topic, 1, 10); ClickHouseSinkTask chst = new ClickHouseSinkTask(); @@ -2152,7 +2152,7 @@ public void autoEvolveStructToJsonMixedBatchOlderRecordsGetDefault() { String topic = "auto_evolve_struct_json_mixed_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Insert V1 records (no struct field) Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 5); @@ -2185,7 +2185,7 @@ public void autoEvolveStructToJsonExplicitlyFalseRejectsStruct() { String topic = "auto_evolve_struct_json_false_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); Collection srV2 = SchemaTestData.createSchemaV2WithStructField(topic, 1, 10); ClickHouseSinkTask chst = new ClickHouseSinkTask(); @@ -2219,7 +2219,7 @@ public void autoEvolveArrayAndMapFields() { String topic = "auto_evolve_array_map_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Insert V1 first Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 10); @@ -2251,7 +2251,7 @@ public void autoEvolveTripleSchemaInOneBatch() { String topic = "auto_evolve_triple_schema_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Build a single batch with V1 + V2 + V3 records List combined = new ArrayList<>(); @@ -2282,7 +2282,7 @@ public void autoEvolveThreeSeparateBatches() { String topic = "auto_evolve_three_batches_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); ClickHouseSinkTask chst = new ClickHouseSinkTask(); chst.start(props); @@ -2340,7 +2340,7 @@ public void autoEvolveMixedSchemasTenRecordsInOneBatch() { String topic = "auto_evolve_mixed_ten_records_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Single batch with 10 records spanning 3 schema versions: // Records 1–5: Schema V1 (3 fields: off16, p_int64, name) @@ -2411,7 +2411,7 @@ public void autoEvolveAllPrimitiveAndLogicalTypes() { String topic = "auto_evolve_all_types_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Insert V1 first to ensure existing rows get NULL for new columns Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 5); @@ -2469,7 +2469,7 @@ public void autoEvolveTypedArrayColumns() { String topic = "auto_evolve_typed_arrays_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); Collection srV2 = SchemaTestData.createSchemaV2WithTypedArrays(topic, 1, 10); ClickHouseSinkTask chst = new ClickHouseSinkTask(); @@ -2503,7 +2503,7 @@ public void autoEvolveDdlRefreshTimeoutThrowsRetriable() { String topic = "auto_evolve_ddl_timeout_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // V2 schema with a new field - DDL will succeed but refresh will timeout with 0 retries Collection srV2 = SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 5); @@ -2540,7 +2540,7 @@ public void autoEvolveDdlExecutionFailureThrowsRuntimeException() { String topic = "auto_evolve_ddl_exec_failure_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Insert V1 records to populate the connector's internal table mapping cache Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 5); @@ -2584,7 +2584,7 @@ public void autoEvolveUnsupportedStructTypeThrowsError() { String topic = "auto_evolve_unsupported_struct_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); Collection srV2 = SchemaTestData.createSchemaV2WithStructField(topic, 1, 5); ClickHouseSinkTask chst = new ClickHouseSinkTask(); @@ -2619,7 +2619,7 @@ public void autoEvolveStringBytesUnionCollapsesToString() { String topic = "auto_evolve_union_string_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); Collection srV2 = SchemaTestData.createSchemaV2WithStringBytesUnionField(topic, 1, 10); ClickHouseSinkTask chst = new ClickHouseSinkTask(); @@ -2648,399 +2648,7 @@ public void autoEvolveMixedUnionCreatesVariantColumn() { String topic = createTopicName("auto_evolve_variant_test"); ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); - - Collection records = SchemaTestData.createSchemaV2WithMixedTypeUnionField(topic, 1, 10); - ClickHouseSinkTask chst = new ClickHouseSinkTask(); - chst.start(props); - chst.put(records); - chst.stop(); - - assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); - - // Verify the union field was created as Variant - com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); - com.clickhouse.kafka.connect.sink.db.mapping.Column col = described.getRootColumnsMap().get("mixed_union"); - assertNotNull(col, "Column 'mixed_union' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.VARIANT, col.getType(), - "union(string, int) should map to Variant, not String or JSON"); - } - - @Test - public void autoEvolveThreeSeparateBatches() { - Map props = createProps(); - props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); - - String topic = "auto_evolve_three_batches_test"; - ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); - - ClickHouseSinkTask chst = new ClickHouseSinkTask(); - chst.start(props); - - // Batch 1: Schema V1 (3 fields: off16, p_int64, name) - List batch1 = new ArrayList<>(SchemaTestData.createRichSchemaV1(topic, 1, 5, 0)); - chst.put(batch1); - - // Batch 2: Schema V2 (8 fields: off16, p_int64, name, email, age, score, active, city) - List batch2 = new ArrayList<>(SchemaTestData.createRichSchemaV2(topic, 1, 5, 5)); - chst.put(batch2); - - // Batch 3: Schema V3 (5 fields: off16, p_int64, name, email, country) - List batch3 = new ArrayList<>(SchemaTestData.createRichSchemaV3(topic, 1, 5, 10)); - chst.put(batch3); - - chst.stop(); - - assertEquals(15, ClickHouseTestHelpers.countRows(chc, topic)); - - // Verify all evolved columns exist - com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); - assertTrue(described.getRootColumnsMap().containsKey("name"), "V1 column 'name' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("email"), "V2 column 'email' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("age"), "V2 column 'age' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("score"), "V2 column 'score' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("active"), "V2 column 'active' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("city"), "V2 column 'city' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("country"), "V3 column 'country' should exist"); - - // V1 records should have NULL for V2/V3 columns - String nullEmailQuery = String.format( - "SELECT COUNT(*) FROM `%s` WHERE `email` IS NULL SETTINGS select_sequential_consistency = 1", topic); - // V3 records don't include age/score/active/city — those should be NULL - String nullAgeQuery = String.format( - "SELECT COUNT(*) FROM `%s` WHERE `age` IS NULL SETTINGS select_sequential_consistency = 1", topic); - try { - Records emailNulls = chc.getClient().queryRecords(nullEmailQuery).get(); - int emailNullCount = Integer.parseInt(emailNulls.iterator().next().getString(1)); - assertEquals(5, emailNullCount, "V1 records should have NULL for email"); - - Records ageNulls = chc.getClient().queryRecords(nullAgeQuery).get(); - int ageNullCount = Integer.parseInt(ageNulls.iterator().next().getString(1)); - assertEquals(10, ageNullCount, "V1 + V3 records (10) should have NULL for age"); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Test - public void autoEvolveMixedSchemasTenRecordsInOneBatch() { - Map props = createProps(); - props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); - - String topic = "auto_evolve_mixed_ten_records_test"; - ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); - - // Single batch with 10 records spanning 3 schema versions: - // Records 1–5: Schema V1 (3 fields: off16, p_int64, name) - // Records 6–7: Schema V2 (8 fields: off16, p_int64, name, email, age, score, active, city) - // Records 8–10: Schema V3 (5 fields: off16, p_int64, name, email, country) - List batch = new ArrayList<>(); - batch.addAll(SchemaTestData.createRichSchemaV1(topic, 1, 5, 0)); - batch.addAll(SchemaTestData.createRichSchemaV2(topic, 1, 2, 5)); - batch.addAll(SchemaTestData.createRichSchemaV3(topic, 1, 3, 7)); - - ClickHouseSinkTask chst = new ClickHouseSinkTask(); - chst.start(props); - chst.put(batch); - chst.stop(); - - assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); - - // Verify all evolved columns from all versions exist - com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); - assertTrue(described.getRootColumnsMap().containsKey("name"), "V1 column 'name' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("email"), "V2 column 'email' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("age"), "V2 column 'age' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("score"), "V2 column 'score' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("active"), "V2 column 'active' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("city"), "V2 column 'city' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("country"), "V3 column 'country' should exist"); - - // Verify NULL distribution: - // name: all 10 records have it - 0 NULLs - // email: V1 records (5) lack it - 5 NULLs - // age: only V2 records (2) have it - 8 NULLs - // country: only V3 records (3) have it - 7 NULLs - try { - String nameNullQuery = String.format( - "SELECT COUNT(*) FROM `%s` WHERE `name` IS NULL SETTINGS select_sequential_consistency = 1", topic); - Records nameNulls = chc.getClient().queryRecords(nameNullQuery).get(); - assertEquals(0, Integer.parseInt(nameNulls.iterator().next().getString(1)), - "All records have name, so 0 NULLs expected"); - - String emailNullQuery = String.format( - "SELECT COUNT(*) FROM `%s` WHERE `email` IS NULL SETTINGS select_sequential_consistency = 1", topic); - Records emailNulls = chc.getClient().queryRecords(emailNullQuery).get(); - assertEquals(5, Integer.parseInt(emailNulls.iterator().next().getString(1)), - "V1 records (5) should have NULL for email"); - - String ageNullQuery = String.format( - "SELECT COUNT(*) FROM `%s` WHERE `age` IS NULL SETTINGS select_sequential_consistency = 1", topic); - Records ageNulls = chc.getClient().queryRecords(ageNullQuery).get(); - assertEquals(8, Integer.parseInt(ageNulls.iterator().next().getString(1)), - "V1 (5) + V3 (3) records should have NULL for age"); - - String countryNullQuery = String.format( - "SELECT COUNT(*) FROM `%s` WHERE `country` IS NULL SETTINGS select_sequential_consistency = 1", topic); - Records countryNulls = chc.getClient().queryRecords(countryNullQuery).get(); - assertEquals(7, Integer.parseInt(countryNulls.iterator().next().getString(1)), - "V1 (5) + V2 (2) records should have NULL for country"); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - // auto-evolve adds columns for every supported primitive + logical type in a single batch - @Test - public void autoEvolveAllPrimitiveAndLogicalTypes() { - Map props = createProps(); - props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); - - String topic = "auto_evolve_all_types_test"; - ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); - - // Insert V1 first to ensure existing rows get NULL for new columns - Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 5); - ClickHouseSinkTask chst = new ClickHouseSinkTask(); - chst.start(props); - chst.put(srV1); - assertEquals(5, ClickHouseTestHelpers.countRows(chc, topic)); - - // Insert V2 with all primitive + logical type fields - Collection srV2 = SchemaTestData.createSchemaV2WithAllPrimitiveTypes(topic, 1, 5); - chst.put(srV2); - chst.stop(); - - assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); - - // Verify all columns were created with correct types - com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); - Map cols = described.getRootColumnsMap(); - - // Primitive types - assertTrue(cols.containsKey("new_int8"), "Column 'new_int8' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.INT8, cols.get("new_int8").getType()); - assertTrue(cols.containsKey("new_int16"), "Column 'new_int16' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.INT16, cols.get("new_int16").getType()); - assertTrue(cols.containsKey("new_int32"), "Column 'new_int32' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.INT32, cols.get("new_int32").getType()); - assertTrue(cols.containsKey("new_int64"), "Column 'new_int64' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.INT64, cols.get("new_int64").getType()); - assertTrue(cols.containsKey("new_float32"), "Column 'new_float32' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.FLOAT32, cols.get("new_float32").getType()); - assertTrue(cols.containsKey("new_float64"), "Column 'new_float64' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.FLOAT64, cols.get("new_float64").getType()); - assertTrue(cols.containsKey("new_bool"), "Column 'new_bool' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.BOOLEAN, cols.get("new_bool").getType()); - assertTrue(cols.containsKey("new_string"), "Column 'new_string' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.STRING, cols.get("new_string").getType()); - assertTrue(cols.containsKey("new_bytes"), "Column 'new_bytes' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.STRING, cols.get("new_bytes").getType()); - - // Logical types - assertTrue(cols.containsKey("new_decimal"), "Column 'new_decimal' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.Decimal, cols.get("new_decimal").getType()); - assertTrue(cols.containsKey("new_date"), "Column 'new_date' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.Date32, cols.get("new_date").getType()); - assertTrue(cols.containsKey("new_timestamp"), "Column 'new_timestamp' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.DateTime64, cols.get("new_timestamp").getType()); - } - - // auto-evolve creates Array columns with different element types (Int32, Float64, Bool, String) - @Test - public void autoEvolveTypedArrayColumns() { - Map props = createProps(); - props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); - - String topic = "auto_evolve_typed_arrays_test"; - ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); - - Collection srV2 = SchemaTestData.createSchemaV2WithTypedArrays(topic, 1, 10); - ClickHouseSinkTask chst = new ClickHouseSinkTask(); - chst.start(props); - chst.put(srV2); - chst.stop(); - - assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); - - // Verify all array columns were created - com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); - Map cols = described.getRootColumnsMap(); - - assertTrue(cols.containsKey("arr_int32"), "Column 'arr_int32' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.ARRAY, cols.get("arr_int32").getType()); - assertTrue(cols.containsKey("arr_float64"), "Column 'arr_float64' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.ARRAY, cols.get("arr_float64").getType()); - assertTrue(cols.containsKey("arr_bool"), "Column 'arr_bool' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.ARRAY, cols.get("arr_bool").getType()); - assertTrue(cols.containsKey("arr_string"), "Column 'arr_string' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.ARRAY, cols.get("arr_string").getType()); - } - - // DDL refresh timeout - retries set to 0 so refresh loop never runs, throws RetriableException - @Test - public void autoEvolveDdlRefreshTimeoutThrowsRetriable() { - Map props = createProps(); - props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - props.put(ClickHouseSinkConfig.AUTO_EVOLVE_DDL_REFRESH_RETRIES, "0"); - ClickHouseHelperClient chc = createClient(props); - - String topic = "auto_evolve_ddl_timeout_test"; - ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); - - // V2 schema with a new field - DDL will succeed but refresh will timeout with 0 retries - Collection srV2 = SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 5); - ClickHouseSinkTask chst = new ClickHouseSinkTask(); - chst.start(props); - - try { - chst.put(srV2); - assertTrue(false, "Expected RetriableException due to DDL refresh timeout with 0 retries"); - } catch (RuntimeException e) { - // Processing layer may wrap the RetriableException - walk the cause chain - Throwable t = e; - boolean found = false; - while (t != null) { - if (t instanceof org.apache.kafka.connect.errors.RetriableException - && t.getMessage() != null && t.getMessage().contains("DDL propagation timeout")) { - found = true; - break; - } - t = t.getCause(); - } - assertTrue(found, "Should contain RetriableException with DDL propagation timeout in cause chain, got: " + e); - } finally { - chst.stop(); - } - } - - // ALTER TABLE itself fails (table dropped externally after cache populated) - @Test - public void autoEvolveDdlExecutionFailureThrowsRuntimeException() { - Map props = createProps(); - props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); - - String topic = "auto_evolve_ddl_exec_failure_test"; - ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); - - // Insert V1 records to populate the connector's internal table mapping cache - Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 5); - ClickHouseSinkTask chst = new ClickHouseSinkTask(); - chst.start(props); - chst.put(srV1); - assertEquals(5, ClickHouseTestHelpers.countRows(chc, topic)); - - // Drop the table externally - the connector still has it cached in memory - ClickHouseTestHelpers.dropTable(chc, topic); - - // V2 schema with a new field - ALTER TABLE will fail because the table no longer exists - Collection srV2 = SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 5); - try { - chst.put(srV2); - assertTrue(false, "Expected RuntimeException due to ALTER TABLE on dropped table"); - } catch (RuntimeException e) { - // Processing layer wraps exceptions - walk the cause chain for the DDL failure - Throwable t = e; - boolean found = false; - while (t != null) { - if (t.getMessage() != null && (t.getMessage().contains("ALTER TABLE") || t.getMessage().contains("UNKNOWN_TABLE"))) { - found = true; - break; - } - t = t.getCause(); - } - assertTrue(found, "Should indicate DDL failure in cause chain, got: " + e.getClass().getName() + ": " + e.getMessage()); - } finally { - chst.stop(); - } - } - - // STRUCT field without struct-to-json flag throws SchemaTypeInferenceException with helpful message - @Test - public void autoEvolveUnsupportedStructTypeThrowsError() { - Map props = createProps(); - props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - // auto.evolve.struct.to.json is false by default - ClickHouseHelperClient chc = createClient(props); - - String topic = "auto_evolve_unsupported_struct_test"; - ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); - - Collection srV2 = SchemaTestData.createSchemaV2WithStructField(topic, 1, 5); - ClickHouseSinkTask chst = new ClickHouseSinkTask(); - chst.start(props); - - try { - chst.put(srV2); - assertTrue(false, "Expected SchemaTypeInferenceException for unsupported STRUCT type"); - } catch (RuntimeException e) { - Throwable t = e; - boolean foundInference = false; - while (t != null) { - if (t instanceof com.clickhouse.kafka.connect.sink.db.mapping.SchemaTypeInferenceException) { - foundInference = true; - break; - } - t = t.getCause(); - } - assertTrue(foundInference, - "Should throw SchemaTypeInferenceException for unsupported STRUCT, got: " + e.getClass().getName() + ": " + e.getMessage()); - } finally { - chst.stop(); - } - } - - // Avro-style union(string, bytes) STRUCT collapses to Nullable(String), not JSON - @Test - public void autoEvolveStringBytesUnionCollapsesToString() { - Map props = createProps(); - props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); - - String topic = "auto_evolve_union_string_test"; - ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); - - Collection srV2 = SchemaTestData.createSchemaV2WithStringBytesUnionField(topic, 1, 10); - ClickHouseSinkTask chst = new ClickHouseSinkTask(); - chst.start(props); - chst.put(srV2); - chst.stop(); - - assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); - - // Verify the union field was created as String (not JSON) - com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); - com.clickhouse.kafka.connect.sink.db.mapping.Column col = described.getRootColumnsMap().get("new_union_field"); - assertNotNull(col, "Column 'new_union_field' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.STRING, col.getType(), - "Union(string, bytes) should collapse to String, not JSON"); - } - - // Avro union(string, int) auto-evolved as Variant(String, Int32) column - @Test - @SinceClickHouseVersion("24.1") - public void autoEvolveMixedUnionCreatesVariantColumn() { - Map props = createProps(); - props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - props.put(ClickHouseSinkConfig.CLICKHOUSE_SETTINGS, "allow_experimental_variant_type=1"); - ClickHouseHelperClient chc = createClient(props); - - String topic = createTopicName("auto_evolve_variant_test"); - ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); Collection records = SchemaTestData.createSchemaV2WithMixedTypeUnionField(topic, 1, 10); ClickHouseSinkTask chst = new ClickHouseSinkTask(); @@ -3066,7 +2674,7 @@ public void autoEvolveSchemalessRecordsThrowError() { String topic = "auto_evolve_schemaless_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Create schemaless (string) records. No valueSchema. List schemaless = new ArrayList<>(); @@ -3110,8 +2718,8 @@ public void autoEvolveAvroUnionStringBytesCreatesStringColumn() throws Exception String topic = "auto_evolve_avro_union_test"; ClickHouseTestHelpers.dropTable(chc, topic); // Table starts with only "name" - union fields "content" and "description" will be auto-evolved - ClickHouseTestHelpers.createTable(chc, topic, - "CREATE TABLE `%s` (`name` String) Engine = MergeTree ORDER BY name"); + ClickHouseTestHelpers.runQuery(chc, String.format( + "CREATE TABLE `%s` (`name` String) Engine = MergeTree ORDER BY name", topic)); Image image1 = Image.newBuilder() .setName("image1") @@ -3161,7 +2769,7 @@ public void autoEvolveMixedBatchLastRecordOlderSchema() { String topic = createTopicName("auto_evolve_last_record_older_test"); ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Build batch where V2 records come first and V1 (older) records are last. // Before the fix, only the last record was checked - V1 has no new fields, so ALTER TABLE was skipped. @@ -3201,7 +2809,7 @@ public void autoEvolveMultiVersionUnionSemantics() { String topic = createTopicName("auto_evolve_union_semantics_test"); ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Batch with V1, V2, V3, V4 - each version adds a different field. // All new fields should be added in a single ALTER TABLE. @@ -3236,7 +2844,7 @@ public void autoEvolveInterleavedSchemaVersions() { String topic = createTopicName("auto_evolve_non_monotonic_test"); ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Interleaved schema versions [V1, V3, V2, V1, V3] List batch = new ArrayList<>(); @@ -3269,7 +2877,7 @@ public void autoEvolveCrossPartitionSchemaDrift() { String topic = createTopicName("auto_evolve_cross_partition_test"); ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Records from different partitions with different schema versions. // With ignorePartitionsWhenBatching=true, they are merged into a single batch. @@ -3302,8 +2910,8 @@ public void autoEvolveMixedBatchArrayMapFieldsMissingInOlderRecords() { String topic = createTopicName("auto_evolve_mixed_array_map_test"); ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, - "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format( + "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Build a single batch: V1 records (no array/map) followed by V2 records (with array/map) List mixedBatch = new ArrayList<>(); @@ -3337,8 +2945,8 @@ public void autoEvolveMixedBatchVariantFieldMissingInOlderRecords() { String topic = createTopicName("auto_evolve_mixed_variant_test"); ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, - "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format( + "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // V1 records (off16 + p_int64 only) followed by V2 records (off16 + p_int64 + mixed_union Variant) List mixedBatch = new ArrayList<>(); diff --git a/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/ClickHouseTestHelpers.java b/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/ClickHouseTestHelpers.java index 25116fe0a..12f2c3bfe 100644 --- a/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/ClickHouseTestHelpers.java +++ b/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/ClickHouseTestHelpers.java @@ -395,4 +395,5 @@ public static void runQuery(ClickHouseHelperClient chc, String query) { throw new RuntimeException(e); } } + } From 564b1f9568414ad90c8ff21fc83888af03daf40a Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Mon, 23 Mar 2026 17:47:07 +0100 Subject: [PATCH 31/62] feature: enable auto.evolve parameter --- CHANGELOG.md | 4 +- .../connect/sink/ClickHouseSinkConfig.java | 17 + .../connect/sink/db/ClickHouseWriter.java | 132 +++++++ .../db/helper/ClickHouseHelperClient.java | 37 ++ .../kafka/connect/sink/db/mapping/Column.java | 70 ++++ .../kafka/connect/sink/db/mapping/Table.java | 13 + .../ClickHouseSinkTaskWithSchemaTest.java | 358 ++++++++++++++++++ .../connect/sink/helper/SchemaTestData.java | 272 +++++++++++++ 8 files changed, 902 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index afda21d80..3a98b1850 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Unreleased +## New Features +* Added `auto.evolve` configuration option for automatic table schema evolution. When enabled, the connector detects new fields in incoming records and issues `ALTER TABLE ... ADD COLUMN IF NOT EXISTS` against ClickHouse. Disabled by default. (https://github.com/ClickHouse/clickhouse-kafka-connect/issues/277) + ## Dependencies * Updated clickhouse-java version from `0.9.4` to `0.9.5` @@ -11,7 +14,6 @@ # Improvements * `Gson` replaced with `Jackson` for performance and better maintainability (https://github.com/ClickHouse/clickhouse-kafka-connect/pull/676). - # 1.3.6, 2026-03-18 ## New Features diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java b/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java index da6fafdb7..e4498fedc 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java @@ -57,6 +57,7 @@ public class ClickHouseSinkConfig { public static final String REPORT_INSERTED_OFFSETS = "reportInsertedOffsets"; public static final String ERROR_TOLERANCE_ALL = "all"; public static final String ERROR_TOLERANCE_NONE = "none"; + public static final String AUTO_EVOLVE = "auto.evolve"; public static final String CONNECTOR_RETRY_TIMEOUT = "errors.retry.timeout"; public static final long MINIMAL_RETRY_TIMEOUT_THR_WARN = TimeUnit.SECONDS.toMillis(10); @@ -110,6 +111,7 @@ public class ClickHouseSinkConfig { private final int bufferCount; private final long bufferFlushTime; private final boolean reportInsertedOffsets; + private final boolean autoEvolve; private final boolean binaryFormatWrtiteJsonAsString; private final String sslSocketSni; @@ -296,6 +298,8 @@ public ClickHouseSinkConfig(Map props) { LOGGER.info("Internal buffering enabled: bufferCount={}, bufferFlushTime={}ms", this.bufferCount, this.bufferFlushTime); } + this.autoEvolve = Boolean.parseBoolean(props.getOrDefault(AUTO_EVOLVE, "false")); + String jsonAsString = getClickhouseSettings().get("input_format_binary_read_json_as_string"); this.binaryFormatWrtiteJsonAsString = jsonAsString != null && (jsonAsString.equalsIgnoreCase("true") || jsonAsString.equals("1")); @@ -692,6 +696,19 @@ private static ConfigDef createConfigDef() { ConfigDef.Width.MEDIUM, "SSL Socket SNI" ); + + String ddlGroup = "DDL"; + int ddlOrderInGroup = 0; + configDef.define(AUTO_EVOLVE, + ConfigDef.Type.BOOLEAN, + false, + ConfigDef.Importance.MEDIUM, + "Whether to automatically add columns to the destination table when a record contains fields not present in the table. default: false", + ddlGroup, + ++ddlOrderInGroup, + ConfigDef.Width.SHORT, + "Auto evolve table schema." + ); return configDef; } } diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index 9e569240e..d9e28953f 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -33,6 +33,7 @@ import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.errors.DataException; +import org.apache.kafka.connect.sink.SinkRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -207,6 +208,52 @@ public void doInsert(List records, QueryIdentifier queryId, ErrorReporte String database = first.getDatabase(); Table table = getTable(database, topic); if (table == null) { return; }//We checked the error flag in getTable, so we don't need to check it again here + + if (csc.isAutoEvolve()) { + table = doInsertWithSchemaEvolution(records, table, queryId); + } else { + doInsertBatch(records, table, queryId); + } + } + + private Table doInsertWithSchemaEvolution(List records, Table table, QueryIdentifier queryId) throws IOException, ExecutionException, InterruptedException { + // Split records into sub batches at schema boundaries (like JDBC BufferedRecords.add pattern) + // When schema changes mid batch the current sub batch is flushed, the table is evolved, and the insertion continues. + Schema currentSchema = getValueSchema(records.get(0)); + int batchStart = 0; + + for (int i = 1; i <= records.size(); i++) { + Schema recordSchema = (i < records.size()) ? getValueSchema(records.get(i)) : null; + + // Flush sub batch when schema changes or the end of the records is reached + if (i == records.size() || !Objects.equals(currentSchema, recordSchema)) { + List subBatch = records.subList(batchStart, i); + Record subFirst = subBatch.get(0); + + // Evolve table for the sub batch schema + table = evolveTableSchema(table, subFirst); + + LOGGER.debug("Inserting sub-batch [{}-{}) of {} records with schema evolution (QueryId: [{}])", + batchStart, i, subBatch.size(), queryId.getQueryId()); + doInsertBatch(subBatch, table, queryId); + + if (i < records.size()) { + currentSchema = recordSchema; + batchStart = i; + } + } + } + + return table; + } + + private static Schema getValueSchema(Record record) { + SinkRecord sr = record.getSinkRecord(); + return sr != null ? sr.valueSchema() : null; + } + + private void doInsertBatch(List records, Table table, QueryIdentifier queryId) throws IOException, ExecutionException, InterruptedException { + Record first = records.get(0); LOGGER.debug("Trying to insert [{}] records to table name [{}] (QueryId: [{}])", records.size(), table.getName(), queryId.getQueryId()); switch (first.getSchemaType()) { case SCHEMA: @@ -494,6 +541,9 @@ protected void doWriteColValue(Column col, OutputStream stream, Data value, bool mapTmp.forEach((key, mapValue) -> { try { doWritePrimitive(col.getMapKeyType(), value.getMapKeySchema().type(), stream, key, col); + if (col.getMapValueType() != null && col.getMapValueType().isNullable() && mapValue != null) { + BinaryStreamUtils.writeNonNull(stream); + } doWriteColValue(col.getMapValueType(), stream, new Data(value.getNestedValueSchema(), mapValue), defaultsSupport); } catch (IOException e) { throw new RuntimeException(e); @@ -821,6 +871,88 @@ protected void doWriteCol(Data value, boolean fieldExists, Column col, OutputStr } } + protected Table evolveTableSchema(Table table, Record record) { + if (record.getFields() == null) { + LOGGER.warn("Cannot auto-evolve schema for records without a Connect schema (schemaless/string). Skipping schema evolution."); + return table; + } + + List fieldNames = record.getFields().stream().map(Field::name).collect(Collectors.toList()); + Set missingColumns = table.getMissingColumns(fieldNames); + + if (missingColumns.isEmpty()) { + return table; + } + + LOGGER.info("Detected {} new field(s) not present in table {}: {}", missingColumns.size(), table.getName(), missingColumns); + + Map schemaMap = record.getFields().stream().collect(Collectors.toMap(Field::name, Field::schema)); + List columnDefs = new java.util.ArrayList<>(); + + for (String fieldName : missingColumns) { + Schema fieldSchema = schemaMap.get(fieldName); + if (fieldSchema == null) { + continue; + } + + if (!fieldSchema.isOptional() && fieldSchema.defaultValue() == null) { + throw new RuntimeException(String.format( + "Cannot auto-evolve: field '%s' is not optional and has no default value. " + + "ClickHouse requires new columns to be either Nullable or have a DEFAULT.", fieldName)); + } + + String chType; + try { + chType = Column.connectTypeToClickHouseType(fieldSchema); + } catch (RuntimeException e) { + throw new RuntimeException(String.format( + "Cannot auto-evolve: field '%s' has unsupported type for auto-evolution. %s", fieldName, e.getMessage()), e); + } + + // ClickHouse does not allow Nullable wrapping for Array and Map types + if (fieldSchema.isOptional() + && fieldSchema.type() != Schema.Type.ARRAY + && fieldSchema.type() != Schema.Type.MAP) { + chType = "Nullable(" + chType + ")"; + } + + columnDefs.add(String.format("`%s` %s", fieldName, chType)); + } + + if (!columnDefs.isEmpty()) { + chc.alterTableAddColumns(table.getDatabase(), table.getCleanName(), columnDefs); + LOGGER.info("Schema evolution complete for table {}. Added columns: {}", table.getName(), columnDefs); + table = refreshTableAfterDDL(table, missingColumns); + } + + return table; + } + + private static final int DDL_REFRESH_MAX_RETRIES = 5; + private static final long DDL_REFRESH_BACKOFF_MS = 200; + + private Table refreshTableAfterDDL(Table table, Set expectedNewColumns) { + for (int attempt = 0; attempt < DDL_REFRESH_MAX_RETRIES; attempt++) { + Table refreshed = urgentTableUpdate(table); + Set stillMissing = refreshed.getMissingColumns(expectedNewColumns); + if (stillMissing.isEmpty()) { + return refreshed; + } + LOGGER.warn("DDL refresh attempt {}/{}: columns {} not yet visible, retrying in {}ms", + attempt + 1, DDL_REFRESH_MAX_RETRIES, stillMissing, DDL_REFRESH_BACKOFF_MS); + try { + Thread.sleep(DDL_REFRESH_BACKOFF_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted while waiting for DDL propagation", e); + } + } + // Final attempt, use whatever we have + LOGGER.error("DDL propagation timeout: some columns may not be visible yet after {} retries. Proceeding with latest table state.", + DDL_REFRESH_MAX_RETRIES); + return urgentTableUpdate(table); + } + protected void doInsertRawBinary(List records, Table table, QueryIdentifier queryId, boolean supportDefaults, boolean retry) throws IOException, ExecutionException, InterruptedException { try { if (chc.isUseClientV2()) { diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java index f89a44fe3..7a0014cbb 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java @@ -469,6 +469,43 @@ public Table describeTableV2(String database, String tableName) { return table; } + public void alterTableAddColumns(String database, String tableName, List columnDefs) { + for (String colDef : columnDefs) { + String sql = String.format("ALTER TABLE `%s`.`%s` ADD COLUMN IF NOT EXISTS %s", database, tableName, colDef); + LOGGER.info("Executing DDL: {}", sql); + if (useClientV2) { + alterTableAddColumnV2(sql); + } else { + alterTableAddColumnV1(sql); + } + } + } + + private void alterTableAddColumnV1(String sql) { + try (ClickHouseClient client = ClickHouseClient.builder() + .options(getDefaultClientOptions()) + .nodeSelector(ClickHouseNodeSelector.of(ClickHouseProtocol.HTTP)) + .build(); + ClickHouseResponse response = client.read(server) + .query(sql) + .set("alter_sync", "1") + .executeAndWait()) { + // DDL executed; alter_sync=1 waits for the local replica to apply + } catch (ClickHouseException e) { + throw new RuntimeException("Failed to execute ALTER TABLE: " + sql, e); + } + } + + private void alterTableAddColumnV2(String sql) { + try { + QuerySettings settings = new QuerySettings(); + settings.serverSetting("alter_sync", "1"); + client.query(sql, settings).get(); + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException("Failed to execute ALTER TABLE: " + sql, e); + } + } + public List
extractTablesMapping(String database, Map cache) { List
tableList = new ArrayList<>(); for (Table table : showTables(database)) { diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java index dab474146..6f0fb6b1a 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java @@ -6,6 +6,11 @@ import lombok.Getter; import lombok.Setter; import lombok.experimental.Accessors; +import org.apache.kafka.connect.data.Date; +import org.apache.kafka.connect.data.Decimal; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.Time; +import org.apache.kafka.connect.data.Timestamp; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.clickhouse.kafka.connect.util.reactor.function.Tuple2; @@ -358,6 +363,71 @@ private static Map extractEnumValues(String valueType) { return data; } + public static String connectTypeToClickHouseType(Schema connectSchema) { + // Check logical types first (same pattern as JDBC connector) + if (connectSchema.name() != null) { + switch (connectSchema.name()) { + case Decimal.LOGICAL_NAME: + int precision = 38; // ClickHouse Decimal128 default + int scale = 0; + if (connectSchema.parameters() != null && connectSchema.parameters().containsKey("scale")) { + scale = Integer.parseInt(connectSchema.parameters().get("scale")); + } + return String.format("Decimal(%d, %d)", precision, scale); + case Date.LOGICAL_NAME: + return "Date32"; + case Time.LOGICAL_NAME: + return "Int64"; + case Timestamp.LOGICAL_NAME: + return "DateTime64(3)"; + } + } + + // Then check primitive types + switch (connectSchema.type()) { + case INT8: + return "Int8"; + case INT16: + return "Int16"; + case INT32: + return "Int32"; + case INT64: + return "Int64"; + case FLOAT32: + return "Float32"; + case FLOAT64: + return "Float64"; + case BOOLEAN: + return "Bool"; + case STRING: + return "String"; + case BYTES: + return "String"; + case ARRAY: + if (connectSchema.valueSchema() == null) { + return "Array(String)"; + } + String elementType = connectTypeToClickHouseType(connectSchema.valueSchema()); + if (connectSchema.valueSchema().isOptional()) { + elementType = "Nullable(" + elementType + ")"; + } + return "Array(" + elementType + ")"; + case MAP: + String keyType = connectTypeToClickHouseType(connectSchema.keySchema()); + String valType = connectTypeToClickHouseType(connectSchema.valueSchema()); + if (connectSchema.valueSchema().isOptional()) { + valType = "Nullable(" + valType + ")"; + } + return "Map(" + keyType + ", " + valType + ")"; + case STRUCT: + throw new RuntimeException( + "Cannot auto-evolve STRUCT fields to ClickHouse columns. " + + "STRUCT type requires manual mapping to Tuple, JSON, or Nested type."); + default: + throw new RuntimeException("Unsupported Connect type for auto-evolution: " + connectSchema.type()); + } + } + public Integer convertEnumValues(String value) { if ( this.enumValues != null ) { return enumValues.get(value); diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Table.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Table.java index 8bb1844f2..747fc14e7 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Table.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Table.java @@ -8,9 +8,12 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -78,6 +81,16 @@ public void addColumn(Column column) { } } + public Set getMissingColumns(Collection fieldNames) { + Set missing = new LinkedHashSet<>(); + for (String fieldName : fieldNames) { + if (!rootColumnsMap.containsKey(fieldName)) { + missing.add(fieldName); + } + } + return missing; + } + private void handleNonRoot(Column column) { String parentName = column.getName().substring(0, column.getName().lastIndexOf(".")); Column parent = allColumnsMap.getOrDefault(parentName, null); diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index 5021d42b3..511f9ef84 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -1802,4 +1802,362 @@ public void testAvroDateAndTimeTypes() throws Exception { assertEquals(event.getTime2().atDate(LocalDate.of(1970, 1, 1)).format(localFormatter), row.get("time2")); } } + + @Test + public void autoEvolveDisabledRejectsNewField() { + Map props = createProps(); + // auto.evolve defaults to false + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_disabled_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Insert V1 records first (should succeed) + Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV1); + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Insert V2 records with new field (should succeed because input_format_skip_unknown_fields=1) + // But the new column should NOT be added to the table + Collection srV2 = SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 10); + chst.put(srV2); + chst.stop(); + + // Rows inserted but new column should not exist + assertEquals(20, ClickHouseTestHelpers.countRows(chc, topic)); + } + + @Test + public void autoEvolveAddsNullableColumn() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_nullable_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Insert V1 records + Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV1); + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Insert V2 records with new nullable field -> should trigger ALTER TABLE ADD COLUMN + Collection srV2 = SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 10); + chst.put(srV2); + chst.stop(); + + assertEquals(20, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify the new column exists + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("new_string_field"), + "New column 'new_string_field' should have been added by auto.evolve"); + } + + @Test + public void autoEvolveRejectsNonNullableNoDefault() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_reject_non_nullable_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + Collection srV2 = SchemaTestData.createSchemaV2WithNewNonNullableField(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + + try { + chst.put(srV2); + // Should have thrown + assertTrue(false, "Expected exception for non-nullable field without default"); + } catch (RuntimeException e) { + // Walk the full cause chain. Utils.handleException wraps multiple times + Throwable t = e; + boolean found = false; + while (t != null) { + if (t.getMessage() != null && (t.getMessage().contains("not optional") || t.getMessage().contains("Cannot auto-evolve"))) { + found = true; + break; + } + t = t.getCause(); + } + assertTrue(found, "Expected descriptive error about non-nullable field in cause chain, got: " + e.getMessage()); + } finally { + chst.stop(); + } + } + + @Test + public void autoEvolveMultipleNewColumns() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_multi_cols_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Insert V1 first + Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV1); + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Insert V2 with multiple new nullable fields + Collection srV2 = SchemaTestData.createSchemaV2WithMultipleNewNullableFields(topic, 1, 10); + chst.put(srV2); + chst.stop(); + + assertEquals(20, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify all new columns exist + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("new_string_field"), + "Column 'new_string_field' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("new_int32_field"), + "Column 'new_int32_field' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("new_float64_field"), + "Column 'new_float64_field' should exist"); + } + + @Test + public void autoEvolveCachesSchemaAfterDDL() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_cache_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + + // First batch triggers DDL + Collection srV2batch1 = SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 10); + chst.put(srV2batch1); + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Second batch with same schema should not re-trigger DDL (just insert) + // Use partition 2 to avoid offset deduplication + Collection srV2batch2 = SchemaTestData.createSchemaV2WithNewNullableField(topic, 2, 10); + chst.put(srV2batch2); + chst.stop(); + + assertEquals(20, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify column exists (only one 'new_string_field' column) + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + long count = described.getRootColumnsList().stream() + .filter(c -> c.getName().equals("new_string_field")) + .count(); + assertEquals(1, count, "Should have exactly one 'new_string_field' column"); + } + + @Test + public void autoEvolveMixedSchemaInSingleBatch() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_mixed_batch_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Build a single batch with V1 records followed by V2 records (mixed schemas) + List mixedBatch = new ArrayList<>(); + mixedBatch.addAll(SchemaTestData.createSchemaV1(topic, 1, 5)); + mixedBatch.addAll(SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 5)); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(mixedBatch); + chst.stop(); + + // All 10 records should be inserted + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // The new column should exist (evolved from V2 records in same batch) + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("new_string_field"), + "Column 'new_string_field' should be added even when schema changes mid-batch"); + } + + @Test + public void autoEvolveLogicalTypes() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_logical_types_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Insert V1 first + Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV1); + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Insert V2 with logical type fields (Decimal, Date, Timestamp) + Collection srV2 = SchemaTestData.createSchemaV2WithLogicalTypes(topic, 1, 10); + chst.put(srV2); + chst.stop(); + + assertEquals(20, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify the logical type columns were created with correct ClickHouse types + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("new_decimal_field"), + "Column 'new_decimal_field' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("new_date_field"), + "Column 'new_date_field' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("new_timestamp_field"), + "Column 'new_timestamp_field' should exist"); + + // Verify types + com.clickhouse.kafka.connect.sink.db.mapping.Column decimalCol = described.getRootColumnsMap().get("new_decimal_field"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.Decimal, decimalCol.getType(), + "Decimal logical type should map to ClickHouse Decimal"); + + com.clickhouse.kafka.connect.sink.db.mapping.Column dateCol = described.getRootColumnsMap().get("new_date_field"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.Date32, dateCol.getType(), + "Date logical type should map to ClickHouse Date32"); + + com.clickhouse.kafka.connect.sink.db.mapping.Column tsCol = described.getRootColumnsMap().get("new_timestamp_field"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.DateTime64, tsCol.getType(), + "Timestamp logical type should map to ClickHouse DateTime64"); + } + + @Test + public void autoEvolveRejectsStructField() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_struct_reject_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + Collection srV2 = SchemaTestData.createSchemaV2WithStructField(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + + try { + chst.put(srV2); + assertTrue(false, "Expected exception for STRUCT field auto-evolution"); + } catch (RuntimeException e) { + Throwable t = e; + boolean found = false; + while (t != null) { + if (t.getMessage() != null && t.getMessage().contains("Cannot auto-evolve STRUCT")) { + found = true; + break; + } + t = t.getCause(); + } + assertTrue(found, "Should reject STRUCT field with appropriate message, got: " + e.getMessage()); + } finally { + chst.stop(); + } + } + + @Test + public void autoEvolveArrayAndMapFields() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_array_map_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Insert V1 first + Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV1); + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Insert V2 with Array and Map fields + Collection srV2 = SchemaTestData.createSchemaV2WithArrayAndMapFields(topic, 1, 10); + chst.put(srV2); + chst.stop(); + + assertEquals(20, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify array and map columns were created + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("new_array_field"), + "Column 'new_array_field' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("new_map_field"), + "Column 'new_map_field' should exist"); + } + + @Test + public void autoEvolveTripleSchemaInOneBatch() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_triple_schema_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Build a single batch with V1 + V2 + V3 records + List combined = new ArrayList<>(); + combined.addAll(SchemaTestData.createSchemaV1(topic, 1, 5)); + combined.addAll(SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 5)); + combined.addAll(SchemaTestData.createSchemaV3WithExtraField(topic, 1, 5)); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(combined); + chst.stop(); + + assertEquals(15, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify columns from both V2 and V3 exist + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("new_string_field"), + "V2 column 'new_string_field' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("v3_bool_field"), + "V3 column 'v3_bool_field' should exist"); + } + + @Test + public void autoEvolveSchemalessRecordsSkipped() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_schemaless_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Create schemaless (string) records. No valueSchema. + List schemaless = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + String json = String.format("{\"off16\": %d, \"p_int64\": %d}", i, (long) i); + schemaless.add(new SinkRecord( + topic, 1, null, null, null, json, + i, System.currentTimeMillis(), TimestampType.CREATE_TIME + )); + } + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(schemaless); + chst.stop(); + } } diff --git a/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java b/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java index 39bea1b57..3b861ebe7 100644 --- a/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java +++ b/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java @@ -1585,4 +1585,276 @@ public static List convertAvroToSinkRecord(String topic, ParsedSchem schemaAndValues.add(new SinkRecord(topic, 0, null, null, schemaAndValue.schema(), schemaAndValue.value(), schemaAndValues.size())), ArrayList::addAll); } + + public static Collection createSchemaV1(String topic, int partition) { + return createSchemaV1(topic, partition, DEFAULT_TOTAL_RECORDS); + } + + public static Collection createSchemaV1(String topic, int partition, int totalRecords) { + List array = new ArrayList<>(); + + Schema SCHEMA_V1 = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .build(); + + LongStream.range(0, totalRecords).forEachOrdered(n -> { + Struct value_struct = new Struct(SCHEMA_V1) + .put("off16", (short) n) + .put("p_int64", n); + + array.add(new SinkRecord( + topic, + partition, + null, + null, SCHEMA_V1, + value_struct, + n, + System.currentTimeMillis(), + TimestampType.CREATE_TIME + )); + }); + return array; + } + + public static Collection createSchemaV2WithNewNullableField(String topic, int partition) { + return createSchemaV2WithNewNullableField(topic, partition, DEFAULT_TOTAL_RECORDS); + } + + public static Collection createSchemaV2WithNewNullableField(String topic, int partition, int totalRecords) { + List array = new ArrayList<>(); + + Schema SCHEMA_V2 = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("new_string_field", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + + long offset = totalRecords; // continue offsets from V1 + for (long n = 0; n < totalRecords; n++) { + Struct value_struct = new Struct(SCHEMA_V2) + .put("off16", (short) n) + .put("p_int64", n) + .put("new_string_field", "value_" + n); + + array.add(new SinkRecord( + topic, + partition, + null, + null, SCHEMA_V2, + value_struct, + offset + n, + System.currentTimeMillis(), + TimestampType.CREATE_TIME + )); + } + return array; + } + + public static Collection createSchemaV2WithNewNonNullableField(String topic, int partition) { + return createSchemaV2WithNewNonNullableField(topic, partition, DEFAULT_TOTAL_RECORDS); + } + + public static Collection createSchemaV2WithNewNonNullableField(String topic, int partition, int totalRecords) { + List array = new ArrayList<>(); + + Schema SCHEMA_V2 = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("non_nullable_field", Schema.STRING_SCHEMA) + .build(); + + long offset = totalRecords; + for (long n = 0; n < totalRecords; n++) { + Struct value_struct = new Struct(SCHEMA_V2) + .put("off16", (short) n) + .put("p_int64", n) + .put("non_nullable_field", "required_" + n); + + array.add(new SinkRecord( + topic, + partition, + null, + null, SCHEMA_V2, + value_struct, + offset + n, + System.currentTimeMillis(), + TimestampType.CREATE_TIME + )); + } + return array; + } + + public static Collection createSchemaV2WithMultipleNewNullableFields(String topic, int partition) { + return createSchemaV2WithMultipleNewNullableFields(topic, partition, DEFAULT_TOTAL_RECORDS); + } + + public static Collection createSchemaV2WithMultipleNewNullableFields(String topic, int partition, int totalRecords) { + List array = new ArrayList<>(); + + Schema SCHEMA_V2 = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("new_string_field", Schema.OPTIONAL_STRING_SCHEMA) + .field("new_int32_field", SchemaBuilder.int32().optional().build()) + .field("new_float64_field", SchemaBuilder.float64().optional().build()) + .build(); + + long offset = totalRecords; + for (long n = 0; n < totalRecords; n++) { + Struct value_struct = new Struct(SCHEMA_V2) + .put("off16", (short) n) + .put("p_int64", n) + .put("new_string_field", "val_" + n) + .put("new_int32_field", (int) n) + .put("new_float64_field", n * 1.5); + + array.add(new SinkRecord( + topic, + partition, + null, + null, SCHEMA_V2, + value_struct, + offset + n, + System.currentTimeMillis(), + TimestampType.CREATE_TIME + )); + } + return array; + } + + public static Collection createSchemaV2WithStructField(String topic, int partition) { + return createSchemaV2WithStructField(topic, partition, DEFAULT_TOTAL_RECORDS); + } + + public static Collection createSchemaV2WithStructField(String topic, int partition, int totalRecords) { + List array = new ArrayList<>(); + + Schema INNER_SCHEMA = SchemaBuilder.struct() + .field("nested_str", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + + Schema SCHEMA_V2 = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("new_struct_field", SchemaBuilder.struct() + .field("nested_str", Schema.OPTIONAL_STRING_SCHEMA) + .optional() + .build()) + .build(); + + long offset = totalRecords; + for (long n = 0; n < totalRecords; n++) { + Struct nested = new Struct(SCHEMA_V2.field("new_struct_field").schema()) + .put("nested_str", "nested_" + n); + Struct value_struct = new Struct(SCHEMA_V2) + .put("off16", (short) n) + .put("p_int64", n) + .put("new_struct_field", nested); + + array.add(new SinkRecord( + topic, partition, null, null, SCHEMA_V2, value_struct, + offset + n, System.currentTimeMillis(), TimestampType.CREATE_TIME + )); + } + return array; + } + + public static Collection createSchemaV2WithArrayAndMapFields(String topic, int partition) { + return createSchemaV2WithArrayAndMapFields(topic, partition, DEFAULT_TOTAL_RECORDS); + } + + public static Collection createSchemaV2WithArrayAndMapFields(String topic, int partition, int totalRecords) { + List array = new ArrayList<>(); + + Schema SCHEMA_V2 = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("new_array_field", SchemaBuilder.array(Schema.OPTIONAL_STRING_SCHEMA).optional().build()) + .field("new_map_field", SchemaBuilder.map(Schema.STRING_SCHEMA, SchemaBuilder.int32().optional().build()).optional().build()) + .build(); + + long offset = totalRecords; + for (long n = 0; n < totalRecords; n++) { + Struct value_struct = new Struct(SCHEMA_V2) + .put("off16", (short) n) + .put("p_int64", n) + .put("new_array_field", Arrays.asList("a_" + n, "b_" + n)) + .put("new_map_field", Collections.singletonMap("key_" + n, (int) n)); + + array.add(new SinkRecord( + topic, partition, null, null, SCHEMA_V2, value_struct, + offset + n, System.currentTimeMillis(), TimestampType.CREATE_TIME + )); + } + return array; + } + + public static Collection createSchemaV3WithExtraField(String topic, int partition) { + return createSchemaV3WithExtraField(topic, partition, DEFAULT_TOTAL_RECORDS); + } + + public static Collection createSchemaV3WithExtraField(String topic, int partition, int totalRecords) { + List array = new ArrayList<>(); + + Schema SCHEMA_V3 = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("new_string_field", Schema.OPTIONAL_STRING_SCHEMA) + .field("v3_bool_field", SchemaBuilder.bool().optional().build()) + .build(); + + long offset = totalRecords * 2L; // after V1 and V2 offsets + for (long n = 0; n < totalRecords; n++) { + Struct value_struct = new Struct(SCHEMA_V3) + .put("off16", (short) n) + .put("p_int64", n) + .put("new_string_field", "v3_" + n) + .put("v3_bool_field", n % 2 == 0); + + array.add(new SinkRecord( + topic, partition, null, null, SCHEMA_V3, value_struct, + offset + n, System.currentTimeMillis(), TimestampType.CREATE_TIME + )); + } + return array; + } + + public static Collection createSchemaV2WithLogicalTypes(String topic, int partition) { + return createSchemaV2WithLogicalTypes(topic, partition, DEFAULT_TOTAL_RECORDS); + } + + public static Collection createSchemaV2WithLogicalTypes(String topic, int partition, int totalRecords) { + List array = new ArrayList<>(); + + Schema SCHEMA_V2 = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("new_decimal_field", Decimal.builder(2).optional().build()) + .field("new_date_field", org.apache.kafka.connect.data.Date.builder().optional().build()) + .field("new_timestamp_field", Timestamp.builder().optional().build()) + .build(); + + long offset = totalRecords; + for (long n = 0; n < totalRecords; n++) { + Struct value_struct = new Struct(SCHEMA_V2) + .put("off16", (short) n) + .put("p_int64", n) + .put("new_decimal_field", BigDecimal.valueOf(n * 100 + 50, 2)) + .put("new_date_field", Date.from(java.time.Instant.ofEpochMilli(n * 86400000L))) + .put("new_timestamp_field", Date.from(java.time.Instant.ofEpochMilli(System.currentTimeMillis()))); + + array.add(new SinkRecord( + topic, + partition, + null, + null, SCHEMA_V2, + value_struct, + offset + n, + System.currentTimeMillis(), + TimestampType.CREATE_TIME + )); + } + return array; + } } From f1137459c58d010bbd66f6ca52e603fcdd38c2ef Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 10:57:00 +0100 Subject: [PATCH 32/62] fix: avoid query connection leak --- .../connect/sink/db/helper/ClickHouseHelperClient.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java index 7a0014cbb..7225d88d7 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java @@ -497,10 +497,8 @@ private void alterTableAddColumnV1(String sql) { } private void alterTableAddColumnV2(String sql) { - try { - QuerySettings settings = new QuerySettings(); - settings.serverSetting("alter_sync", "1"); - client.query(sql, settings).get(); + try (QueryResponse response = client.query(sql, new QuerySettings().serverSetting("alter_sync", "1")).get()) { + // DDL executed; alter_sync=1 waits for the local replica to apply } catch (ExecutionException | InterruptedException e) { throw new RuntimeException("Failed to execute ALTER TABLE: " + sql, e); } From 352b9f99053641b484995ac8d788860efe9f4915 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 11:05:09 +0100 Subject: [PATCH 33/62] chore: move value to static field to make it more clear --- .../com/clickhouse/kafka/connect/sink/db/mapping/Column.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java index 6f0fb6b1a..d0294d325 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java @@ -34,6 +34,7 @@ public class Column { private static final Pattern DECIMAL_TYPE_PATTERN = Pattern.compile("Decimal(?\\d{2,3})?\\s*(\\((?\\d{1,}\\s*)?,*\\s*(?\\d{1,})?\\))?"); private static final Pattern SIMPLE_AGGREGATE_FUNCTION_TYPE_PATTERN = Pattern.compile("^SimpleAggregateFunction\\s*\\([^,]+,\\s*(.+)\\)$"); + private static final int DECIMAL128_MAX_PRECISION = 38; private static final Logger LOGGER = LoggerFactory.getLogger(Column.class); private String name; @@ -368,7 +369,7 @@ public static String connectTypeToClickHouseType(Schema connectSchema) { if (connectSchema.name() != null) { switch (connectSchema.name()) { case Decimal.LOGICAL_NAME: - int precision = 38; // ClickHouse Decimal128 default + int precision = DECIMAL128_MAX_PRECISION; int scale = 0; if (connectSchema.parameters() != null && connectSchema.parameters().containsKey("scale")) { scale = Integer.parseInt(connectSchema.parameters().get("scale")); From 419bac5ab65e01385260a8d9e6ff92427533a16b Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 11:18:48 +0100 Subject: [PATCH 34/62] fix: create custom type inference exception --- .../clickhouse/kafka/connect/sink/db/mapping/Column.java | 4 ++-- .../sink/db/mapping/SchemaTypeInferenceException.java | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/SchemaTypeInferenceException.java diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java index d0294d325..b5af42806 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java @@ -421,11 +421,11 @@ public static String connectTypeToClickHouseType(Schema connectSchema) { } return "Map(" + keyType + ", " + valType + ")"; case STRUCT: - throw new RuntimeException( + throw new SchemaTypeInferenceException( "Cannot auto-evolve STRUCT fields to ClickHouse columns. " + "STRUCT type requires manual mapping to Tuple, JSON, or Nested type."); default: - throw new RuntimeException("Unsupported Connect type for auto-evolution: " + connectSchema.type()); + throw new SchemaTypeInferenceException("Unsupported Connect type for auto-evolution: " + connectSchema.type()); } } diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/SchemaTypeInferenceException.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/SchemaTypeInferenceException.java new file mode 100644 index 000000000..a718713aa --- /dev/null +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/SchemaTypeInferenceException.java @@ -0,0 +1,7 @@ +package com.clickhouse.kafka.connect.sink.db.mapping; + +public class SchemaTypeInferenceException extends RuntimeException { + public SchemaTypeInferenceException(String message) { + super(message); + } +} From 00d652d9c0fbf8f381dc633ffe9a357683ab882f Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 11:27:48 +0100 Subject: [PATCH 35/62] fix: dont wrap connectTypeToClickHouseType in try-catch since method already throws exception --- .../kafka/connect/sink/db/ClickHouseWriter.java | 8 +------- .../connect/sink/db/helper/ClickHouseHelperClient.java | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index d9e28953f..78a823ec4 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -901,13 +901,7 @@ protected Table evolveTableSchema(Table table, Record record) { "ClickHouse requires new columns to be either Nullable or have a DEFAULT.", fieldName)); } - String chType; - try { - chType = Column.connectTypeToClickHouseType(fieldSchema); - } catch (RuntimeException e) { - throw new RuntimeException(String.format( - "Cannot auto-evolve: field '%s' has unsupported type for auto-evolution. %s", fieldName, e.getMessage()), e); - } + String chType = Column.connectTypeToClickHouseType(fieldSchema); // ClickHouse does not allow Nullable wrapping for Array and Map types if (fieldSchema.isOptional() diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java index 7225d88d7..d5a78e700 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java @@ -499,7 +499,7 @@ private void alterTableAddColumnV1(String sql) { private void alterTableAddColumnV2(String sql) { try (QueryResponse response = client.query(sql, new QuerySettings().serverSetting("alter_sync", "1")).get()) { // DDL executed; alter_sync=1 waits for the local replica to apply - } catch (ExecutionException | InterruptedException e) { + } catch (Exception e) { throw new RuntimeException("Failed to execute ALTER TABLE: " + sql, e); } } From bff79bed77c13022a7c55a371a5801e54a9b40a6 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 11:55:19 +0100 Subject: [PATCH 36/62] fix: move Nullable wrapping into connectTypeToClickHouseType and always create Nullable columns --- .../connect/sink/db/ClickHouseWriter.java | 14 -------- .../kafka/connect/sink/db/mapping/Column.java | 19 +++++++---- .../ClickHouseSinkTaskWithSchemaTest.java | 32 +++++++------------ 3 files changed, 23 insertions(+), 42 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index 78a823ec4..afce8f98c 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -895,21 +895,7 @@ protected Table evolveTableSchema(Table table, Record record) { continue; } - if (!fieldSchema.isOptional() && fieldSchema.defaultValue() == null) { - throw new RuntimeException(String.format( - "Cannot auto-evolve: field '%s' is not optional and has no default value. " + - "ClickHouse requires new columns to be either Nullable or have a DEFAULT.", fieldName)); - } - String chType = Column.connectTypeToClickHouseType(fieldSchema); - - // ClickHouse does not allow Nullable wrapping for Array and Map types - if (fieldSchema.isOptional() - && fieldSchema.type() != Schema.Type.ARRAY - && fieldSchema.type() != Schema.Type.MAP) { - chType = "Nullable(" + chType + ")"; - } - columnDefs.add(String.format("`%s` %s", fieldName, chType)); } diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java index b5af42806..854d5e2c5 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java @@ -365,6 +365,17 @@ private static Map extractEnumValues(String valueType) { } public static String connectTypeToClickHouseType(Schema connectSchema) { + String baseType = resolveBaseType(connectSchema); + + // ClickHouse forbids Nullable wrapping for Array and Map types + if (connectSchema.type() == Schema.Type.ARRAY || connectSchema.type() == Schema.Type.MAP) { + return baseType; + } + + return "Nullable(" + baseType + ")"; + } + + private static String resolveBaseType(Schema connectSchema) { // Check logical types first (same pattern as JDBC connector) if (connectSchema.name() != null) { switch (connectSchema.name()) { @@ -409,16 +420,10 @@ public static String connectTypeToClickHouseType(Schema connectSchema) { return "Array(String)"; } String elementType = connectTypeToClickHouseType(connectSchema.valueSchema()); - if (connectSchema.valueSchema().isOptional()) { - elementType = "Nullable(" + elementType + ")"; - } return "Array(" + elementType + ")"; case MAP: - String keyType = connectTypeToClickHouseType(connectSchema.keySchema()); + String keyType = resolveBaseType(connectSchema.keySchema()); String valType = connectTypeToClickHouseType(connectSchema.valueSchema()); - if (connectSchema.valueSchema().isOptional()) { - valType = "Nullable(" + valType + ")"; - } return "Map(" + keyType + ", " + valType + ")"; case STRUCT: throw new SchemaTypeInferenceException( diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index 511f9ef84..8dde5bd3a 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -1861,38 +1861,28 @@ public void autoEvolveAddsNullableColumn() { } @Test - public void autoEvolveRejectsNonNullableNoDefault() { + public void autoEvolveAddsNonNullableFieldAsNullable() { Map props = createProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); ClickHouseHelperClient chc = createClient(props); - String topic = "auto_evolve_reject_non_nullable_test"; + String topic = "auto_evolve_non_nullable_as_nullable_test"; ClickHouseTestHelpers.dropTable(chc, topic); ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); Collection srV2 = SchemaTestData.createSchemaV2WithNewNonNullableField(topic, 1, 10); ClickHouseSinkTask chst = new ClickHouseSinkTask(); chst.start(props); + chst.put(srV2); + chst.stop(); - try { - chst.put(srV2); - // Should have thrown - assertTrue(false, "Expected exception for non-nullable field without default"); - } catch (RuntimeException e) { - // Walk the full cause chain. Utils.handleException wraps multiple times - Throwable t = e; - boolean found = false; - while (t != null) { - if (t.getMessage() != null && (t.getMessage().contains("not optional") || t.getMessage().contains("Cannot auto-evolve"))) { - found = true; - break; - } - t = t.getCause(); - } - assertTrue(found, "Expected descriptive error about non-nullable field in cause chain, got: " + e.getMessage()); - } finally { - chst.stop(); - } + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Non-nullable fields are created as Nullable columns — mandatory fields always have a value, + // and Nullable allows old records (without this field) to insert with NULL + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("non_nullable_field"), + "New column 'non_nullable_field' should have been added by auto.evolve"); } @Test From f45995c62edab7e8fc272d56340e47d8213f7b70 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 13:06:50 +0100 Subject: [PATCH 37/62] chore: remove sub batching since is not needed --- .../connect/sink/db/ClickHouseWriter.java | 43 ++--------------- .../ClickHouseSinkTaskWithSchemaTest.java | 48 +++++++++++++++++++ 2 files changed, 53 insertions(+), 38 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index afce8f98c..c50c15baa 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -49,6 +49,7 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoField; +import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -210,46 +211,12 @@ public void doInsert(List records, QueryIdentifier queryId, ErrorReporte if (table == null) { return; }//We checked the error flag in getTable, so we don't need to check it again here if (csc.isAutoEvolve()) { - table = doInsertWithSchemaEvolution(records, table, queryId); - } else { - doInsertBatch(records, table, queryId); - } - } - - private Table doInsertWithSchemaEvolution(List records, Table table, QueryIdentifier queryId) throws IOException, ExecutionException, InterruptedException { - // Split records into sub batches at schema boundaries (like JDBC BufferedRecords.add pattern) - // When schema changes mid batch the current sub batch is flushed, the table is evolved, and the insertion continues. - Schema currentSchema = getValueSchema(records.get(0)); - int batchStart = 0; - - for (int i = 1; i <= records.size(); i++) { - Schema recordSchema = (i < records.size()) ? getValueSchema(records.get(i)) : null; - - // Flush sub batch when schema changes or the end of the records is reached - if (i == records.size() || !Objects.equals(currentSchema, recordSchema)) { - List subBatch = records.subList(batchStart, i); - Record subFirst = subBatch.get(0); - - // Evolve table for the sub batch schema - table = evolveTableSchema(table, subFirst); - - LOGGER.debug("Inserting sub-batch [{}-{}) of {} records with schema evolution (QueryId: [{}])", - batchStart, i, subBatch.size(), queryId.getQueryId()); - doInsertBatch(subBatch, table, queryId); - - if (i < records.size()) { - currentSchema = recordSchema; - batchStart = i; - } - } + // New columns are Nullable, so older records without the new fields insert with NULL. + Record last = records.get(records.size() - 1); + table = evolveTableSchema(table, last); } - return table; - } - - private static Schema getValueSchema(Record record) { - SinkRecord sr = record.getSinkRecord(); - return sr != null ? sr.valueSchema() : null; + doInsertBatch(records, table, queryId); } private void doInsertBatch(List records, Table table, QueryIdentifier queryId) throws IOException, ExecutionException, InterruptedException { diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index 8dde5bd3a..5695e95d9 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -1,6 +1,7 @@ package com.clickhouse.kafka.connect.sink; import com.clickhouse.client.api.ClientConfigProperties; +import com.clickhouse.client.api.query.Records; import com.clickhouse.kafka.connect.avro.test.Event; import com.clickhouse.kafka.connect.avro.test.Image; import com.clickhouse.kafka.connect.sink.db.helper.ClickHouseHelperClient; @@ -1982,6 +1983,53 @@ public void autoEvolveMixedSchemaInSingleBatch() { "Column 'new_string_field' should be added even when schema changes mid-batch"); } + @Test + public void autoEvolveMixedSchemaOlderRecordsGetNull() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_older_records_null_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // V1 records (no new_string_field) followed by V2 records (has new_string_field) + // Schema is evolved using last record (V2), then entire batch is inserted. + // V1 records should get NULL for the new column. + List mixedBatch = new ArrayList<>(); + mixedBatch.addAll(SchemaTestData.createSchemaV1(topic, 1, 5)); + mixedBatch.addAll(SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 5)); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(mixedBatch); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // V1 records should have NULL for the new column + String nullCountQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `new_string_field` IS NULL SETTINGS select_sequential_consistency = 1", topic); + try { + Records nullRecords = chc.getClient().queryRecords(nullCountQuery).get(); + int nullCount = Integer.parseInt(nullRecords.iterator().next().getString(1)); + assertEquals(5, nullCount, "V1 records should have NULL for new_string_field"); + } catch (Exception e) { + throw new RuntimeException(e); + } + + // V2 records should have non-NULL values + String nonNullQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `new_string_field` IS NOT NULL SETTINGS select_sequential_consistency = 1", topic); + try { + Records nonNullRecords = chc.getClient().queryRecords(nonNullQuery).get(); + int nonNullCount = Integer.parseInt(nonNullRecords.iterator().next().getString(1)); + assertEquals(5, nonNullCount, "V2 records should have non-NULL values for new_string_field"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + @Test public void autoEvolveLogicalTypes() { Map props = createProps(); From b41ee1d486faab77eb8826027a6b17198cbaa7ef Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 14:57:15 +0100 Subject: [PATCH 38/62] feat: add auto evolve DDL refresh retries configuration to ClickHouseSinkConfig --- .../kafka/connect/sink/ClickHouseSinkConfig.java | 14 ++++++++++++++ .../kafka/connect/sink/db/ClickHouseWriter.java | 8 ++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java b/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java index e4498fedc..c548cf7ce 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java @@ -58,6 +58,7 @@ public class ClickHouseSinkConfig { public static final String ERROR_TOLERANCE_ALL = "all"; public static final String ERROR_TOLERANCE_NONE = "none"; public static final String AUTO_EVOLVE = "auto.evolve"; + public static final String AUTO_EVOLVE_DDL_REFRESH_RETRIES = "auto.evolve.ddl.refresh.retries"; public static final String CONNECTOR_RETRY_TIMEOUT = "errors.retry.timeout"; public static final long MINIMAL_RETRY_TIMEOUT_THR_WARN = TimeUnit.SECONDS.toMillis(10); @@ -112,6 +113,7 @@ public class ClickHouseSinkConfig { private final long bufferFlushTime; private final boolean reportInsertedOffsets; private final boolean autoEvolve; + private final int autoEvolveDdlRefreshRetries; private final boolean binaryFormatWrtiteJsonAsString; private final String sslSocketSni; @@ -299,6 +301,7 @@ public ClickHouseSinkConfig(Map props) { } this.autoEvolve = Boolean.parseBoolean(props.getOrDefault(AUTO_EVOLVE, "false")); + this.autoEvolveDdlRefreshRetries = Integer.parseInt(props.getOrDefault(AUTO_EVOLVE_DDL_REFRESH_RETRIES, "3")); String jsonAsString = getClickhouseSettings().get("input_format_binary_read_json_as_string"); this.binaryFormatWrtiteJsonAsString = jsonAsString != null && (jsonAsString.equalsIgnoreCase("true") || jsonAsString.equals("1")); @@ -709,6 +712,17 @@ private static ConfigDef createConfigDef() { ConfigDef.Width.SHORT, "Auto evolve table schema." ); + configDef.define(AUTO_EVOLVE_DDL_REFRESH_RETRIES, + ConfigDef.Type.INT, + 3, + ConfigDef.Range.atLeast(0), + ConfigDef.Importance.LOW, + "Number of retries when waiting for DDL changes to propagate after schema evolution. default: 3", + ddlGroup, + ++ddlOrderInGroup, + ConfigDef.Width.SHORT, + "DDL refresh retries" + ); return configDef; } } diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index c50c15baa..e6b00b759 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -875,18 +875,18 @@ protected Table evolveTableSchema(Table table, Record record) { return table; } - private static final int DDL_REFRESH_MAX_RETRIES = 5; private static final long DDL_REFRESH_BACKOFF_MS = 200; private Table refreshTableAfterDDL(Table table, Set expectedNewColumns) { - for (int attempt = 0; attempt < DDL_REFRESH_MAX_RETRIES; attempt++) { + int maxRetries = csc.getAutoEvolveDdlRefreshRetries(); + for (int attempt = 0; attempt < maxRetries; attempt++) { Table refreshed = urgentTableUpdate(table); Set stillMissing = refreshed.getMissingColumns(expectedNewColumns); if (stillMissing.isEmpty()) { return refreshed; } LOGGER.warn("DDL refresh attempt {}/{}: columns {} not yet visible, retrying in {}ms", - attempt + 1, DDL_REFRESH_MAX_RETRIES, stillMissing, DDL_REFRESH_BACKOFF_MS); + attempt + 1, maxRetries, stillMissing, DDL_REFRESH_BACKOFF_MS); try { Thread.sleep(DDL_REFRESH_BACKOFF_MS); } catch (InterruptedException e) { @@ -896,7 +896,7 @@ private Table refreshTableAfterDDL(Table table, Set expectedNewColumns) } // Final attempt, use whatever we have LOGGER.error("DDL propagation timeout: some columns may not be visible yet after {} retries. Proceeding with latest table state.", - DDL_REFRESH_MAX_RETRIES); + maxRetries); return urgentTableUpdate(table); } From b20608010efe7e5ce1300b242585d77bcb82e5b1 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 15:01:07 +0100 Subject: [PATCH 39/62] fix: update DDL refresh logic to throw RetriableException on timeout --- .../connect/sink/db/ClickHouseWriter.java | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index e6b00b759..924788284 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -33,6 +33,7 @@ import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.errors.DataException; +import org.apache.kafka.connect.errors.RetriableException; import org.apache.kafka.connect.sink.SinkRecord; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -838,7 +839,7 @@ protected void doWriteCol(Data value, boolean fieldExists, Column col, OutputStr } } - protected Table evolveTableSchema(Table table, Record record) { + protected Table evolveTableSchema(Table table, Record record) throws InterruptedException { if (record.getFields() == null) { LOGGER.warn("Cannot auto-evolve schema for records without a Connect schema (schemaless/string). Skipping schema evolution."); return table; @@ -877,7 +878,7 @@ protected Table evolveTableSchema(Table table, Record record) { private static final long DDL_REFRESH_BACKOFF_MS = 200; - private Table refreshTableAfterDDL(Table table, Set expectedNewColumns) { + private Table refreshTableAfterDDL(Table table, Set expectedNewColumns) throws InterruptedException { int maxRetries = csc.getAutoEvolveDdlRefreshRetries(); for (int attempt = 0; attempt < maxRetries; attempt++) { Table refreshed = urgentTableUpdate(table); @@ -887,17 +888,10 @@ private Table refreshTableAfterDDL(Table table, Set expectedNewColumns) } LOGGER.warn("DDL refresh attempt {}/{}: columns {} not yet visible, retrying in {}ms", attempt + 1, maxRetries, stillMissing, DDL_REFRESH_BACKOFF_MS); - try { - Thread.sleep(DDL_REFRESH_BACKOFF_MS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException("Interrupted while waiting for DDL propagation", e); - } + Thread.sleep(DDL_REFRESH_BACKOFF_MS); } - // Final attempt, use whatever we have - LOGGER.error("DDL propagation timeout: some columns may not be visible yet after {} retries. Proceeding with latest table state.", - maxRetries); - return urgentTableUpdate(table); + throw new RetriableException(String.format( + "DDL propagation timeout: columns not visible after %d retries", maxRetries)); } protected void doInsertRawBinary(List records, Table table, QueryIdentifier queryId, boolean supportDefaults, boolean retry) throws IOException, ExecutionException, InterruptedException { From 3923c867468daa19ff61cf6242c6dcede0b1c67a Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 15:05:02 +0100 Subject: [PATCH 40/62] fix: schema requirement for auto evolve --- .../connect/sink/db/ClickHouseWriter.java | 5 +++-- .../ClickHouseSinkTaskWithSchemaTest.java | 22 ++++++++++++++++--- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index 924788284..edb8d2e16 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -841,8 +841,9 @@ protected void doWriteCol(Data value, boolean fieldExists, Column col, OutputStr protected Table evolveTableSchema(Table table, Record record) throws InterruptedException { if (record.getFields() == null) { - LOGGER.warn("Cannot auto-evolve schema for records without a Connect schema (schemaless/string). Skipping schema evolution."); - return table; + throw new RuntimeException( + "auto.evolve requires a Connect schema (Avro, Protobuf, or JSON Schema). " + + "Schemaless or string records are not supported with auto.evolve=true."); } List fieldNames = record.getFields().stream().map(Field::name).collect(Collectors.toList()); diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index 5695e95d9..cc42d3abb 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -2174,7 +2174,7 @@ public void autoEvolveTripleSchemaInOneBatch() { } @Test - public void autoEvolveSchemalessRecordsSkipped() { + public void autoEvolveSchemalessRecordsThrowError() { Map props = createProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); ClickHouseHelperClient chc = createClient(props); @@ -2195,7 +2195,23 @@ public void autoEvolveSchemalessRecordsSkipped() { ClickHouseSinkTask chst = new ClickHouseSinkTask(); chst.start(props); - chst.put(schemaless); - chst.stop(); + + try { + chst.put(schemaless); + assertTrue(false, "Expected exception for schemaless records with auto.evolve=true"); + } catch (RuntimeException e) { + Throwable t = e; + boolean found = false; + while (t != null) { + if (t.getMessage() != null && t.getMessage().contains("auto.evolve requires a Connect schema")) { + found = true; + break; + } + t = t.getCause(); + } + assertTrue(found, "Expected error about schemaless records in cause chain, got: " + e.getMessage()); + } finally { + chst.stop(); + } } } From dde2b576253019d7636ca52bd79cff3cc8dd2f71 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 15:37:22 +0100 Subject: [PATCH 41/62] refactor: optimize alterTableAddColumns method to use a single SQL statement for adding multiple columns --- .../sink/db/helper/ClickHouseHelperClient.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java index d5a78e700..1039f0ed2 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java @@ -35,6 +35,7 @@ import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; public class ClickHouseHelperClient implements AutoCloseable { @@ -470,14 +471,15 @@ public Table describeTableV2(String database, String tableName) { } public void alterTableAddColumns(String database, String tableName, List columnDefs) { - for (String colDef : columnDefs) { - String sql = String.format("ALTER TABLE `%s`.`%s` ADD COLUMN IF NOT EXISTS %s", database, tableName, colDef); - LOGGER.info("Executing DDL: {}", sql); - if (useClientV2) { - alterTableAddColumnV2(sql); - } else { - alterTableAddColumnV1(sql); - } + String addClauses = columnDefs.stream() + .map(colDef -> "ADD COLUMN IF NOT EXISTS " + colDef) + .collect(Collectors.joining(", ")); + String sql = String.format("ALTER TABLE `%s`.`%s` %s", database, tableName, addClauses); + LOGGER.info("Executing DDL: {}", sql); + if (useClientV2) { + alterTableAddColumnV2(sql); + } else { + alterTableAddColumnV1(sql); } } From 5cf2e1cf56b81ae45039a3b30dc98a78b996629b Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 20:17:55 +0100 Subject: [PATCH 42/62] feat: add auto evolve struct to JSON configuration option to ClickHouseSinkConfig --- .../kafka/connect/sink/ClickHouseSinkConfig.java | 13 +++++++++++++ .../kafka/connect/sink/db/ClickHouseWriter.java | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java b/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java index c548cf7ce..22b1aff60 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java @@ -59,6 +59,7 @@ public class ClickHouseSinkConfig { public static final String ERROR_TOLERANCE_NONE = "none"; public static final String AUTO_EVOLVE = "auto.evolve"; public static final String AUTO_EVOLVE_DDL_REFRESH_RETRIES = "auto.evolve.ddl.refresh.retries"; + public static final String AUTO_EVOLVE_STRUCT_TO_JSON = "auto.evolve.struct.to.json"; public static final String CONNECTOR_RETRY_TIMEOUT = "errors.retry.timeout"; public static final long MINIMAL_RETRY_TIMEOUT_THR_WARN = TimeUnit.SECONDS.toMillis(10); @@ -114,6 +115,7 @@ public class ClickHouseSinkConfig { private final boolean reportInsertedOffsets; private final boolean autoEvolve; private final int autoEvolveDdlRefreshRetries; + private final boolean autoEvolveStructToJson; private final boolean binaryFormatWrtiteJsonAsString; private final String sslSocketSni; @@ -302,6 +304,7 @@ public ClickHouseSinkConfig(Map props) { this.autoEvolve = Boolean.parseBoolean(props.getOrDefault(AUTO_EVOLVE, "false")); this.autoEvolveDdlRefreshRetries = Integer.parseInt(props.getOrDefault(AUTO_EVOLVE_DDL_REFRESH_RETRIES, "3")); + this.autoEvolveStructToJson = Boolean.parseBoolean(props.getOrDefault(AUTO_EVOLVE_STRUCT_TO_JSON, "false")); String jsonAsString = getClickhouseSettings().get("input_format_binary_read_json_as_string"); this.binaryFormatWrtiteJsonAsString = jsonAsString != null && (jsonAsString.equalsIgnoreCase("true") || jsonAsString.equals("1")); @@ -723,6 +726,16 @@ private static ConfigDef createConfigDef() { ConfigDef.Width.SHORT, "DDL refresh retries" ); + configDef.define(AUTO_EVOLVE_STRUCT_TO_JSON, + ConfigDef.Type.BOOLEAN, + false, + ConfigDef.Importance.MEDIUM, + "Whether to map Connect STRUCT fields to ClickHouse JSON columns during schema evolution. default: false", + ddlGroup, + ++ddlOrderInGroup, + ConfigDef.Width.SHORT, + "Map STRUCT to JSON" + ); return configDef; } } diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index edb8d2e16..4d5a66282 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -864,7 +864,7 @@ protected Table evolveTableSchema(Table table, Record record) throws Interrupted continue; } - String chType = Column.connectTypeToClickHouseType(fieldSchema); + String chType = Column.connectTypeToClickHouseType(fieldSchema, csc.isAutoEvolveStructToJson()); columnDefs.add(String.format("`%s` %s", fieldName, chType)); } From 9ef332f0177a29be24502c74f333be2651fe54c0 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 23:14:00 +0100 Subject: [PATCH 43/62] feat: enhance schema handling by adding union type detection and mapping for ClickHouse integration --- .../connect/sink/db/ClickHouseWriter.java | 3 + .../kafka/connect/sink/db/mapping/Column.java | 97 +++- .../ClickHouseSinkTaskWithSchemaTest.java | 419 +++++++++++++++++- .../connect/sink/db/mapping/ColumnTest.java | 177 ++++++++ .../connect/sink/helper/SchemaTestData.java | 164 +++++++ 5 files changed, 851 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index 4d5a66282..7196d4501 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -295,6 +295,9 @@ protected boolean validateDataSchema(Table table, Record record, boolean onlyFie if (colTypeName.equals("TUPLE") && dataTypeName.equals("STRUCT")) continue; + if (colTypeName.equals("VARIANT") && dataTypeName.equals("STRUCT")) + continue; + if (INT_TYPES.contains(colTypeName)) { continue; } diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java index 854d5e2c5..f8f16ee7f 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java @@ -8,6 +8,7 @@ import lombok.experimental.Accessors; import org.apache.kafka.connect.data.Date; import org.apache.kafka.connect.data.Decimal; +import org.apache.kafka.connect.data.Field; import org.apache.kafka.connect.data.Schema; import org.apache.kafka.connect.data.Time; import org.apache.kafka.connect.data.Timestamp; @@ -20,9 +21,11 @@ import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -37,6 +40,13 @@ public class Column { private static final int DECIMAL128_MAX_PRECISION = 38; private static final Logger LOGGER = LoggerFactory.getLogger(Column.class); + + // Confluent converter union schema markers + static final String AVRO_UNION_SCHEMA_NAME = "io.confluent.connect.avro.Union"; + static final String PROTOBUF_UNION_SCHEMA_PREFIX = "io.confluent.connect.protobuf.Union"; + static final String GENERALIZED_UNION_PREFIX = "connect_union_"; + static final String CONNECT_UNION_PARAMETER = "org.apache.kafka.connect.data.Union"; + private String name; private Type type; @@ -365,17 +375,21 @@ private static Map extractEnumValues(String valueType) { } public static String connectTypeToClickHouseType(Schema connectSchema) { - String baseType = resolveBaseType(connectSchema); + return connectTypeToClickHouseType(connectSchema, false); + } - // ClickHouse forbids Nullable wrapping for Array and Map types - if (connectSchema.type() == Schema.Type.ARRAY || connectSchema.type() == Schema.Type.MAP) { + public static String connectTypeToClickHouseType(Schema connectSchema, boolean structToJson) { + String baseType = resolveBaseType(connectSchema, structToJson); + + // ClickHouse forbids Nullable wrapping for Array, Map, and Variant types. + if (connectSchema.type() == Schema.Type.ARRAY || connectSchema.type() == Schema.Type.MAP || baseType.startsWith("Variant(")) { return baseType; } return "Nullable(" + baseType + ")"; } - private static String resolveBaseType(Schema connectSchema) { + private static String resolveBaseType(Schema connectSchema, boolean structToJson) { // Check logical types first (same pattern as JDBC connector) if (connectSchema.name() != null) { switch (connectSchema.name()) { @@ -419,21 +433,88 @@ private static String resolveBaseType(Schema connectSchema) { if (connectSchema.valueSchema() == null) { return "Array(String)"; } - String elementType = connectTypeToClickHouseType(connectSchema.valueSchema()); + String elementType = connectTypeToClickHouseType(connectSchema.valueSchema(), structToJson); return "Array(" + elementType + ")"; case MAP: - String keyType = resolveBaseType(connectSchema.keySchema()); - String valType = connectTypeToClickHouseType(connectSchema.valueSchema()); + String keyType = resolveBaseType(connectSchema.keySchema(), structToJson); + String valType = connectTypeToClickHouseType(connectSchema.valueSchema(), structToJson); return "Map(" + keyType + ", " + valType + ")"; case STRUCT: + if (isUnionSchema(connectSchema)) { + return resolveUnionType(connectSchema, structToJson); + } + if (structToJson) { + return "JSON"; + } throw new SchemaTypeInferenceException( "Cannot auto-evolve STRUCT fields to ClickHouse columns. " + - "STRUCT type requires manual mapping to Tuple, JSON, or Nested type."); + "Set auto.evolve.struct.to.json=true to map STRUCT to JSON, " + + "or manually create the column as Tuple, JSON, or Nested type."); default: throw new SchemaTypeInferenceException("Unsupported Connect type for auto-evolution: " + connectSchema.type()); } } + // Type groups that ClickHouse considers suspicious when mixed inside a Variant. + // See: https://clickhouse.com/docs/sql-reference/data-types/variant + private static final Set SUSPICIOUS_NUMERIC_TYPES = Set.of( + "Int8", "Int16", "Int32", "Int64", + "UInt8", "UInt16", "UInt32", "UInt64", + "Float32", "Float64" + ); + private static final Set SUSPICIOUS_DATE_TYPES = Set.of( + "Date32", "DateTime64(3)" + ); + + private static String resolveUnionType(Schema connectSchema, boolean structToJson) { + if (connectSchema.fields() == null || connectSchema.fields().isEmpty()) { + return "String"; + } + + LinkedHashSet chTypes = new LinkedHashSet<>(); + for (Field field : connectSchema.fields()) { + chTypes.add(resolveBaseType(field.schema(), structToJson)); + } + + // All branches resolve to the same ClickHouse type (e.g. union(string, bytes) → String) + if (chTypes.size() == 1) { + return chTypes.iterator().next(); + } + + // Check for suspicious similar types that ClickHouse rejects by default + if (hasSuspiciousSimilarTypes(chTypes)) { + return "String"; + } + + // Multiple distinct types map to Variant(T1, T2, ...) requires ClickHouse 24.1+. + return "Variant(" + String.join(", ", chTypes) + ")"; + } + + private static boolean hasSuspiciousSimilarTypes(Set chTypes) { + int numericCount = 0; + int dateCount = 0; + for (String t : chTypes) { + if (SUSPICIOUS_NUMERIC_TYPES.contains(t)) numericCount++; + if (SUSPICIOUS_DATE_TYPES.contains(t)) dateCount++; + } + return numericCount > 1 || dateCount > 1; + } + + static boolean isUnionSchema(Schema connectSchema) { + if (connectSchema.type() != Schema.Type.STRUCT) { + return false; + } + String name = connectSchema.name(); + if (name != null + && (name.equals(AVRO_UNION_SCHEMA_NAME) + || name.startsWith(PROTOBUF_UNION_SCHEMA_PREFIX) + || name.startsWith(GENERALIZED_UNION_PREFIX))) { + return true; + } + return connectSchema.parameters() != null + && connectSchema.parameters().containsKey(CONNECT_UNION_PARAMETER); + } + public Integer convertEnumValues(String value) { if ( this.enumValues != null ) { return enumValues.get(value); diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index cc42d3abb..b9b41abf1 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -63,6 +63,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @ExtendWith(FromVersionConditionExtension.class) @@ -1879,7 +1880,7 @@ public void autoEvolveAddsNonNullableFieldAsNullable() { assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); - // Non-nullable fields are created as Nullable columns — mandatory fields always have a value, + // Non-nullable fields are created as Nullable columns - mandatory fields always have a value, // and Nullable allows old records (without this field) to insert with NULL com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); assertTrue(described.getRootColumnsMap().containsKey("non_nullable_field"), @@ -2110,6 +2111,105 @@ public void autoEvolveRejectsStructField() { } } + // STRUCT field auto-evolved as JSON column when auto.evolve.struct.to.json=true + @Test + public void autoEvolveStructToJsonCreatesJsonColumn() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE_STRUCT_TO_JSON, "true"); + props.put(ClickHouseSinkConfig.CLICKHOUSE_SETTINGS, "input_format_binary_read_json_as_string=1"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_struct_to_json_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + Collection srV2 = SchemaTestData.createSchemaV2WithStructField(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV2); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify the new column was created as JSON type + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("new_struct_field"), + "Column 'new_struct_field' should exist"); + com.clickhouse.kafka.connect.sink.db.mapping.Column col = described.getRootColumnsMap().get("new_struct_field"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.JSON, col.getType(), + "Column 'new_struct_field' should be JSON type"); + } + + // V1 records (no struct) inserted first, then V2 records (with struct) trigger JSON column creation. + @Test + public void autoEvolveStructToJsonMixedBatchOlderRecordsGetDefault() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE_STRUCT_TO_JSON, "true"); + props.put(ClickHouseSinkConfig.CLICKHOUSE_SETTINGS, "input_format_binary_read_json_as_string=1"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_struct_json_mixed_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Insert V1 records (no struct field) + Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 5); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV1); + assertEquals(5, ClickHouseTestHelpers.countRows(chc, topic)); + + // Insert V2 records (with struct field) - triggers JSON column creation + Collection srV2 = SchemaTestData.createSchemaV2WithStructField(topic, 1, 5); + chst.put(srV2); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify JSON column exists + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + com.clickhouse.kafka.connect.sink.db.mapping.Column col = described.getRootColumnsMap().get("new_struct_field"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.JSON, col.getType(), + "Column 'new_struct_field' should be JSON type"); + } + + // STRUCT field with auto.evolve.struct.to.json explicitly false rejects with helpful error message + @Test + public void autoEvolveStructToJsonExplicitlyFalseRejectsStruct() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE_STRUCT_TO_JSON, "false"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_struct_json_false_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + Collection srV2 = SchemaTestData.createSchemaV2WithStructField(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + + try { + chst.put(srV2); + assertTrue(false, "Expected exception for STRUCT field when struct.to.json is false"); + } catch (RuntimeException e) { + Throwable t = e; + boolean found = false; + while (t != null) { + if (t.getMessage() != null && t.getMessage().contains("auto.evolve.struct.to.json=true")) { + found = true; + break; + } + t = t.getCause(); + } + assertTrue(found, "Error message should suggest auto.evolve.struct.to.json=true, got: " + e.getMessage()); + } finally { + chst.stop(); + } + } + @Test public void autoEvolveArrayAndMapFields() { Map props = createProps(); @@ -2173,6 +2273,270 @@ public void autoEvolveTripleSchemaInOneBatch() { "V3 column 'v3_bool_field' should exist"); } + // auto-evolve adds columns for every supported primitive + logical type in a single batch + @Test + public void autoEvolveAllPrimitiveAndLogicalTypes() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_all_types_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Insert V1 first to ensure existing rows get NULL for new columns + Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 5); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV1); + assertEquals(5, ClickHouseTestHelpers.countRows(chc, topic)); + + // Insert V2 with all primitive + logical type fields + Collection srV2 = SchemaTestData.createSchemaV2WithAllPrimitiveTypes(topic, 1, 5); + chst.put(srV2); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify all columns were created with correct types + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + Map cols = described.getRootColumnsMap(); + + // Primitive types + assertTrue(cols.containsKey("new_int8"), "Column 'new_int8' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.INT8, cols.get("new_int8").getType()); + assertTrue(cols.containsKey("new_int16"), "Column 'new_int16' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.INT16, cols.get("new_int16").getType()); + assertTrue(cols.containsKey("new_int32"), "Column 'new_int32' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.INT32, cols.get("new_int32").getType()); + assertTrue(cols.containsKey("new_int64"), "Column 'new_int64' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.INT64, cols.get("new_int64").getType()); + assertTrue(cols.containsKey("new_float32"), "Column 'new_float32' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.FLOAT32, cols.get("new_float32").getType()); + assertTrue(cols.containsKey("new_float64"), "Column 'new_float64' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.FLOAT64, cols.get("new_float64").getType()); + assertTrue(cols.containsKey("new_bool"), "Column 'new_bool' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.BOOLEAN, cols.get("new_bool").getType()); + assertTrue(cols.containsKey("new_string"), "Column 'new_string' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.STRING, cols.get("new_string").getType()); + assertTrue(cols.containsKey("new_bytes"), "Column 'new_bytes' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.STRING, cols.get("new_bytes").getType()); + + // Logical types + assertTrue(cols.containsKey("new_decimal"), "Column 'new_decimal' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.Decimal, cols.get("new_decimal").getType()); + assertTrue(cols.containsKey("new_date"), "Column 'new_date' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.Date32, cols.get("new_date").getType()); + assertTrue(cols.containsKey("new_timestamp"), "Column 'new_timestamp' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.DateTime64, cols.get("new_timestamp").getType()); + } + + // auto-evolve creates Array columns with different element types (Int32, Float64, Bool, String) + @Test + public void autoEvolveTypedArrayColumns() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_typed_arrays_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + Collection srV2 = SchemaTestData.createSchemaV2WithTypedArrays(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV2); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify all array columns were created + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + Map cols = described.getRootColumnsMap(); + + assertTrue(cols.containsKey("arr_int32"), "Column 'arr_int32' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.ARRAY, cols.get("arr_int32").getType()); + assertTrue(cols.containsKey("arr_float64"), "Column 'arr_float64' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.ARRAY, cols.get("arr_float64").getType()); + assertTrue(cols.containsKey("arr_bool"), "Column 'arr_bool' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.ARRAY, cols.get("arr_bool").getType()); + assertTrue(cols.containsKey("arr_string"), "Column 'arr_string' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.ARRAY, cols.get("arr_string").getType()); + } + + // DDL refresh timeout - retries set to 0 so refresh loop never runs, throws RetriableException + @Test + public void autoEvolveDdlRefreshTimeoutThrowsRetriable() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE_DDL_REFRESH_RETRIES, "0"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_ddl_timeout_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // V2 schema with a new field - DDL will succeed but refresh will timeout with 0 retries + Collection srV2 = SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 5); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + + try { + chst.put(srV2); + assertTrue(false, "Expected RetriableException due to DDL refresh timeout with 0 retries"); + } catch (RuntimeException e) { + // Processing layer may wrap the RetriableException - walk the cause chain + Throwable t = e; + boolean found = false; + while (t != null) { + if (t instanceof org.apache.kafka.connect.errors.RetriableException + && t.getMessage() != null && t.getMessage().contains("DDL propagation timeout")) { + found = true; + break; + } + t = t.getCause(); + } + assertTrue(found, "Should contain RetriableException with DDL propagation timeout in cause chain, got: " + e); + } finally { + chst.stop(); + } + } + + // ALTER TABLE itself fails (table dropped externally after cache populated) + @Test + public void autoEvolveDdlExecutionFailureThrowsRuntimeException() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_ddl_exec_failure_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Insert V1 records to populate the connector's internal table mapping cache + Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 5); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV1); + assertEquals(5, ClickHouseTestHelpers.countRows(chc, topic)); + + // Drop the table externally - the connector still has it cached in memory + ClickHouseTestHelpers.dropTable(chc, topic); + + // V2 schema with a new field - ALTER TABLE will fail because the table no longer exists + Collection srV2 = SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 5); + try { + chst.put(srV2); + assertTrue(false, "Expected RuntimeException due to ALTER TABLE on dropped table"); + } catch (RuntimeException e) { + // Processing layer wraps exceptions - walk the cause chain for the DDL failure + Throwable t = e; + boolean found = false; + while (t != null) { + if (t.getMessage() != null && (t.getMessage().contains("ALTER TABLE") || t.getMessage().contains("UNKNOWN_TABLE"))) { + found = true; + break; + } + t = t.getCause(); + } + assertTrue(found, "Should indicate DDL failure in cause chain, got: " + e.getClass().getName() + ": " + e.getMessage()); + } finally { + chst.stop(); + } + } + + // STRUCT field without struct-to-json flag throws SchemaTypeInferenceException with helpful message + @Test + public void autoEvolveUnsupportedStructTypeThrowsError() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + // auto.evolve.struct.to.json is false by default + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_unsupported_struct_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + Collection srV2 = SchemaTestData.createSchemaV2WithStructField(topic, 1, 5); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + + try { + chst.put(srV2); + assertTrue(false, "Expected SchemaTypeInferenceException for unsupported STRUCT type"); + } catch (RuntimeException e) { + Throwable t = e; + boolean foundInference = false; + while (t != null) { + if (t instanceof com.clickhouse.kafka.connect.sink.db.mapping.SchemaTypeInferenceException) { + foundInference = true; + break; + } + t = t.getCause(); + } + assertTrue(foundInference, + "Should throw SchemaTypeInferenceException for unsupported STRUCT, got: " + e.getClass().getName() + ": " + e.getMessage()); + } finally { + chst.stop(); + } + } + + // Avro-style union(string, bytes) STRUCT collapses to Nullable(String), not JSON + @Test + public void autoEvolveStringBytesUnionCollapsesToString() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_union_string_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + Collection srV2 = SchemaTestData.createSchemaV2WithStringBytesUnionField(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV2); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify the union field was created as String (not JSON) + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + com.clickhouse.kafka.connect.sink.db.mapping.Column col = described.getRootColumnsMap().get("new_union_field"); + assertNotNull(col, "Column 'new_union_field' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.STRING, col.getType(), + "Union(string, bytes) should collapse to String, not JSON"); + } + + // Avro union(string, int) auto-evolved as Variant(String, Int32) column + @Test + @SinceClickHouseVersion("24.1") + public void autoEvolveMixedUnionCreatesVariantColumn() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + props.put(ClickHouseSinkConfig.CLICKHOUSE_SETTINGS, "allow_experimental_variant_type=1"); + ClickHouseHelperClient chc = createClient(props); + + String topic = createTopicName("auto_evolve_variant_test"); + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + Collection records = SchemaTestData.createSchemaV2WithMixedTypeUnionField(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(records); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify the union field was created as Variant + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + com.clickhouse.kafka.connect.sink.db.mapping.Column col = described.getRootColumnsMap().get("mixed_union"); + assertNotNull(col, "Column 'mixed_union' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.VARIANT, col.getType(), + "union(string, int) should map to Variant, not String or JSON"); + } + @Test public void autoEvolveSchemalessRecordsThrowError() { Map props = createProps(); @@ -2214,4 +2578,57 @@ public void autoEvolveSchemalessRecordsThrowError() { chst.stop(); } } + + // Avro union(string, bytes) fields auto-evolved as Nullable(String) columns + @Test + public void autoEvolveAvroUnionStringBytesCreatesStringColumn() throws Exception { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_avro_union_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + // Table starts with only "name" - union fields "content" and "description" will be auto-evolved + ClickHouseTestHelpers.createTable(chc, topic, + "CREATE TABLE `%s` (`name` String) Engine = MergeTree ORDER BY name"); + + Image image1 = Image.newBuilder() + .setName("image1") + .setContent("content1") + .build(); + Image image2 = Image.newBuilder() + .setName("image2") + .setContent(ByteBuffer.wrap("content2".getBytes())) + .setDescription("desc2") + .build(); + + List records = SchemaTestData.convertAvroToSinkRecord( + topic, new AvroSchema(Image.getClassSchema()), Arrays.asList(image1, image2)); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(records); + chst.stop(); + + assertEquals(2, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify that union columns were created as String (not JSON) + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + Map cols = described.getRootColumnsMap(); + + assertTrue(cols.containsKey("content"), "Column 'content' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.STRING, cols.get("content").getType(), + "union(string, bytes) should map to String, not JSON"); + + assertTrue(cols.containsKey("description"), "Column 'description' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.STRING, cols.get("description").getType(), + "union(null, string, bytes) should map to Nullable(String), not JSON"); + + // Verify data was inserted correctly + List rows = ClickHouseTestHelpers.getAllRowsAsJson(chc, topic); + if (rows.size() == 0) { + rows = ClickHouseTestHelpers.getAllRowsAsJson(chc, topic); + } + assertEquals(2, rows.size()); + } } diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java index 22296e8cb..ddd24a286 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java @@ -1,7 +1,10 @@ package com.clickhouse.kafka.connect.sink.db.mapping; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.data.SchemaBuilder; import org.junit.jupiter.api.Test; +import java.util.Collections; import java.util.List; import static com.clickhouse.kafka.connect.sink.helper.ClickHouseTestHelpers.col; @@ -177,5 +180,179 @@ public void extractEnumOfPrimitives() { assertTrue(col.getEnumValues().containsKey("a, valid")); assertTrue(col.getEnumValues().containsKey("b")); } + + // --- isUnionSchema detection tests --- + + @Test + public void isUnionSchema_avroUnion() { + Schema union = SchemaBuilder.struct() + .name(Column.AVRO_UNION_SCHEMA_NAME) + .field("string", Schema.OPTIONAL_STRING_SCHEMA) + .field("int", Schema.OPTIONAL_INT32_SCHEMA) + .optional() + .build(); + assertTrue(Column.isUnionSchema(union)); + } + + @Test + public void isUnionSchema_protobufOneof() { + Schema union = SchemaBuilder.struct() + .name("io.confluent.connect.protobuf.Union.content") + .field("user_info", Schema.OPTIONAL_STRING_SCHEMA) + .field("product_info", Schema.OPTIONAL_STRING_SCHEMA) + .optional() + .build(); + assertTrue(Column.isUnionSchema(union)); + } + + @Test + public void isUnionSchema_generalizedUnion() { + Schema union = SchemaBuilder.struct() + .name("connect_union_0") + .parameter(Column.CONNECT_UNION_PARAMETER, "connect_union_0") + .field("string", Schema.OPTIONAL_STRING_SCHEMA) + .field("int", Schema.OPTIONAL_INT32_SCHEMA) + .optional() + .build(); + assertTrue(Column.isUnionSchema(union)); + } + + @Test + public void isUnionSchema_connectUnionParameter() { + Schema union = SchemaBuilder.struct() + .parameter(Column.CONNECT_UNION_PARAMETER, "some_annotation") + .field("string", Schema.OPTIONAL_STRING_SCHEMA) + .field("int", Schema.OPTIONAL_INT32_SCHEMA) + .optional() + .build(); + assertTrue(Column.isUnionSchema(union)); + } + + @Test + public void isUnionSchema_realStruct_notDetected() { + Schema struct = SchemaBuilder.struct() + .field("name", Schema.STRING_SCHEMA) + .field("age", Schema.INT32_SCHEMA) + .build(); + assertFalse(Column.isUnionSchema(struct)); + } + + @Test + public void isUnionSchema_primitiveType_notDetected() { + assertFalse(Column.isUnionSchema(Schema.STRING_SCHEMA)); + } + + // --- connectTypeToClickHouseType union mapping tests --- + + @Test + public void unionStringBytes_collapsesToNullableString() { + Schema union = SchemaBuilder.struct() + .name(Column.AVRO_UNION_SCHEMA_NAME) + .field("string", Schema.OPTIONAL_STRING_SCHEMA) + .field("bytes", Schema.OPTIONAL_BYTES_SCHEMA) + .optional() + .build(); + assertEquals("Nullable(String)", Column.connectTypeToClickHouseType(union)); + } + + @Test + public void unionStringInt_mapsToVariant() { + Schema union = SchemaBuilder.struct() + .name(Column.AVRO_UNION_SCHEMA_NAME) + .field("string", Schema.OPTIONAL_STRING_SCHEMA) + .field("int", Schema.OPTIONAL_INT32_SCHEMA) + .optional() + .build(); + assertEquals("Variant(String, Int32)", Column.connectTypeToClickHouseType(union)); + } + + @Test + public void unionStringIntBoolean_mapsToVariant() { + Schema union = SchemaBuilder.struct() + .name(Column.AVRO_UNION_SCHEMA_NAME) + .field("string", Schema.OPTIONAL_STRING_SCHEMA) + .field("int", Schema.OPTIONAL_INT32_SCHEMA) + .field("boolean", Schema.OPTIONAL_BOOLEAN_SCHEMA) + .optional() + .build(); + assertEquals("Variant(String, Int32, Bool)", Column.connectTypeToClickHouseType(union)); + } + + @Test + public void unionSuspiciousNumericTypes_fallsBackToString() { + // Variant(Int32, Int64) is rejected by ClickHouse unless allow_suspicious_variant_types + Schema union = SchemaBuilder.struct() + .name(Column.AVRO_UNION_SCHEMA_NAME) + .field("int", Schema.OPTIONAL_INT32_SCHEMA) + .field("long", Schema.OPTIONAL_INT64_SCHEMA) + .optional() + .build(); + assertEquals("Nullable(String)", Column.connectTypeToClickHouseType(union)); + } + + @Test + public void unionSuspiciousNumericWithString_fallsBackToString() { + Schema union = SchemaBuilder.struct() + .name(Column.AVRO_UNION_SCHEMA_NAME) + .field("string", Schema.OPTIONAL_STRING_SCHEMA) + .field("int", Schema.OPTIONAL_INT32_SCHEMA) + .field("long", Schema.OPTIONAL_INT64_SCHEMA) + .optional() + .build(); + assertEquals("Nullable(String)", Column.connectTypeToClickHouseType(union)); + } + + @Test + public void protobufOneof_stringBytes_collapsesToNullableString() { + // Protobuf oneof { string url = 1; bytes raw = 2; } — field names differ from Avro + Schema union = SchemaBuilder.struct() + .name("io.confluent.connect.protobuf.Union.image") + .field("url", Schema.OPTIONAL_STRING_SCHEMA) + .field("raw", Schema.OPTIONAL_BYTES_SCHEMA) + .optional() + .build(); + assertEquals("Nullable(String)", Column.connectTypeToClickHouseType(union)); + } + + @Test + public void realStruct_withStructToJson_mapsToJSON() { + Schema struct = SchemaBuilder.struct() + .field("name", Schema.STRING_SCHEMA) + .field("age", Schema.INT32_SCHEMA) + .build(); + assertEquals("Nullable(JSON)", Column.connectTypeToClickHouseType(struct, true)); + } + + @Test + public void realStruct_withoutFlag_throws() { + Schema struct = SchemaBuilder.struct() + .field("name", Schema.STRING_SCHEMA) + .field("age", Schema.INT32_SCHEMA) + .build(); + assertThrows(SchemaTypeInferenceException.class, + () -> Column.connectTypeToClickHouseType(struct, false)); + } + + @Test + public void unionEmptyFields_fallsBackToNullableString() { + Schema union = SchemaBuilder.struct() + .name(Column.AVRO_UNION_SCHEMA_NAME) + .optional() + .build(); + assertEquals("Nullable(String)", Column.connectTypeToClickHouseType(union)); + } + + @Test + public void unionVariant_notWrappedInNullable() { + Schema union = SchemaBuilder.struct() + .name(Column.AVRO_UNION_SCHEMA_NAME) + .field("string", Schema.OPTIONAL_STRING_SCHEMA) + .field("boolean", Schema.OPTIONAL_BOOLEAN_SCHEMA) + .optional() + .build(); + String result = Column.connectTypeToClickHouseType(union); + assertEquals("Variant(String, Bool)", result); + assertFalse(result.startsWith("Nullable(")); + } } diff --git a/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java b/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java index 3b861ebe7..aacd7b122 100644 --- a/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java +++ b/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java @@ -1857,4 +1857,168 @@ public static Collection createSchemaV2WithLogicalTypes(String topic } return array; } + + // Schema with all supported primitive types + logical types as new columns + public static Collection createSchemaV2WithAllPrimitiveTypes(String topic, int partition) { + return createSchemaV2WithAllPrimitiveTypes(topic, partition, DEFAULT_TOTAL_RECORDS); + } + + public static Collection createSchemaV2WithAllPrimitiveTypes(String topic, int partition, int totalRecords) { + List array = new ArrayList<>(); + + Schema SCHEMA_V2 = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("new_int8", SchemaBuilder.int8().optional().build()) + .field("new_int16", SchemaBuilder.int16().optional().build()) + .field("new_int32", SchemaBuilder.int32().optional().build()) + .field("new_int64", SchemaBuilder.int64().optional().build()) + .field("new_float32", SchemaBuilder.float32().optional().build()) + .field("new_float64", SchemaBuilder.float64().optional().build()) + .field("new_bool", SchemaBuilder.bool().optional().build()) + .field("new_string", Schema.OPTIONAL_STRING_SCHEMA) + .field("new_bytes", Schema.OPTIONAL_BYTES_SCHEMA) + .field("new_decimal", Decimal.builder(4).optional().build()) + .field("new_date", org.apache.kafka.connect.data.Date.builder().optional().build()) + .field("new_timestamp", Timestamp.builder().optional().build()) + .build(); + + long offset = totalRecords; + for (long n = 0; n < totalRecords; n++) { + Struct value_struct = new Struct(SCHEMA_V2) + .put("off16", (short) n) + .put("p_int64", n) + .put("new_int8", (byte) n) + .put("new_int16", (short) n) + .put("new_int32", (int) n) + .put("new_int64", n * 100L) + .put("new_float32", (float) n * 1.1f) + .put("new_float64", n * 2.2) + .put("new_bool", n % 2 == 0) + .put("new_string", "str_" + n) + .put("new_bytes", ("bytes_" + n).getBytes()) + .put("new_decimal", java.math.BigDecimal.valueOf(n * 100 + 50, 4)) + .put("new_date", Date.from(java.time.Instant.ofEpochMilli(n * 86400000L))) + .put("new_timestamp", Date.from(java.time.Instant.ofEpochMilli(System.currentTimeMillis()))); + + array.add(new SinkRecord( + topic, partition, null, null, SCHEMA_V2, value_struct, + offset + n, System.currentTimeMillis(), TimestampType.CREATE_TIME + )); + } + return array; + } + + // Schema with arrays of different element types + public static Collection createSchemaV2WithTypedArrays(String topic, int partition) { + return createSchemaV2WithTypedArrays(topic, partition, DEFAULT_TOTAL_RECORDS); + } + + public static Collection createSchemaV2WithTypedArrays(String topic, int partition, int totalRecords) { + List array = new ArrayList<>(); + + Schema SCHEMA_V2 = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("arr_int32", SchemaBuilder.array(SchemaBuilder.int32().optional().build()).optional().build()) + .field("arr_float64", SchemaBuilder.array(SchemaBuilder.float64().optional().build()).optional().build()) + .field("arr_bool", SchemaBuilder.array(SchemaBuilder.bool().optional().build()).optional().build()) + .field("arr_string", SchemaBuilder.array(Schema.OPTIONAL_STRING_SCHEMA).optional().build()) + .build(); + + long offset = totalRecords; + for (long n = 0; n < totalRecords; n++) { + Struct value_struct = new Struct(SCHEMA_V2) + .put("off16", (short) n) + .put("p_int64", n) + .put("arr_int32", Arrays.asList((int) n, (int) n + 1)) + .put("arr_float64", Arrays.asList(n * 1.1, n * 2.2)) + .put("arr_bool", Arrays.asList(n % 2 == 0, n % 2 != 0)) + .put("arr_string", Arrays.asList("a_" + n, "b_" + n)); + + array.add(new SinkRecord( + topic, partition, null, null, SCHEMA_V2, value_struct, + offset + n, System.currentTimeMillis(), TimestampType.CREATE_TIME + )); + } + return array; + } + + // Schema with an Avro-style union STRUCT (all STRING/BYTES fields) that should collapse to String + public static Collection createSchemaV2WithStringBytesUnionField(String topic, int partition) { + return createSchemaV2WithStringBytesUnionField(topic, partition, DEFAULT_TOTAL_RECORDS); + } + + public static Collection createSchemaV2WithStringBytesUnionField(String topic, int partition, int totalRecords) { + List array = new ArrayList<>(); + + // Simulates how Confluent AvroConverter represents union(string, bytes) + // AvroConverter sets schema.name() to "io.confluent.connect.avro.Union" + Schema UNION_SCHEMA = SchemaBuilder.struct() + .name("io.confluent.connect.avro.Union") + .field("string", Schema.OPTIONAL_STRING_SCHEMA) + .field("bytes", Schema.OPTIONAL_BYTES_SCHEMA) + .optional() + .build(); + + Schema SCHEMA_V2 = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("new_union_field", UNION_SCHEMA) + .build(); + + long offset = totalRecords; + for (long n = 0; n < totalRecords; n++) { + Struct unionValue = new Struct(UNION_SCHEMA) + .put("string", "union_str_" + n); + Struct value_struct = new Struct(SCHEMA_V2) + .put("off16", (short) n) + .put("p_int64", n) + .put("new_union_field", unionValue); + + array.add(new SinkRecord( + topic, partition, null, null, SCHEMA_V2, value_struct, + offset + n, System.currentTimeMillis(), TimestampType.CREATE_TIME + )); + } + return array; + } + + // Schema with an Avro-style union STRUCT (string + int) that should map to Variant(String, Int32) + public static Collection createSchemaV2WithMixedTypeUnionField(String topic, int partition, int totalRecords) { + List array = new ArrayList<>(); + + Schema UNION_SCHEMA = SchemaBuilder.struct() + .name("io.confluent.connect.avro.Union") + .field("string", Schema.OPTIONAL_STRING_SCHEMA) + .field("int", Schema.OPTIONAL_INT32_SCHEMA) + .optional() + .build(); + + Schema SCHEMA_V2 = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("mixed_union", UNION_SCHEMA) + .build(); + + long offset = totalRecords; + for (long n = 0; n < totalRecords; n++) { + Struct unionValue; + if (n % 2 == 0) { + unionValue = new Struct(UNION_SCHEMA).put("string", "val_" + n); + } else { + unionValue = new Struct(UNION_SCHEMA).put("int", (int) n); + } + Struct value_struct = new Struct(SCHEMA_V2) + .put("off16", (short) n) + .put("p_int64", n) + .put("mixed_union", unionValue); + + array.add(new SinkRecord( + topic, partition, null, null, SCHEMA_V2, value_struct, + offset + n, System.currentTimeMillis(), TimestampType.CREATE_TIME + )); + } + return array; + } } From 79c91af082e2ae64bc4dea5b78ff929097525697 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Thu, 26 Mar 2026 12:22:13 +0100 Subject: [PATCH 44/62] feat: add tests for auto-evolving schemas in ClickHouseSinkTask --- .../ClickHouseSinkTaskWithSchemaTest.java | 128 ++++++++++++++++++ .../connect/sink/helper/SchemaTestData.java | 93 +++++++++++++ 2 files changed, 221 insertions(+) diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index b9b41abf1..9c22fa161 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -2273,6 +2273,134 @@ public void autoEvolveTripleSchemaInOneBatch() { "V3 column 'v3_bool_field' should exist"); } + @Test + public void autoEvolveThreeSeparateBatches() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_three_batches_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + + // Batch 1: Schema V1 (3 fields: off16, p_int64, name) + List batch1 = new ArrayList<>(SchemaTestData.createRichSchemaV1(topic, 1, 5, 0)); + chst.put(batch1); + + // Batch 2: Schema V2 (8 fields: off16, p_int64, name, email, age, score, active, city) + List batch2 = new ArrayList<>(SchemaTestData.createRichSchemaV2(topic, 1, 5, 5)); + chst.put(batch2); + + // Batch 3: Schema V3 (5 fields: off16, p_int64, name, email, country) + List batch3 = new ArrayList<>(SchemaTestData.createRichSchemaV3(topic, 1, 5, 10)); + chst.put(batch3); + + chst.stop(); + + assertEquals(15, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify all evolved columns exist + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("name"), "V1 column 'name' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("email"), "V2 column 'email' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("age"), "V2 column 'age' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("score"), "V2 column 'score' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("active"), "V2 column 'active' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("city"), "V2 column 'city' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("country"), "V3 column 'country' should exist"); + + // V1 records should have NULL for V2/V3 columns + String nullEmailQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `email` IS NULL SETTINGS select_sequential_consistency = 1", topic); + // V3 records don't include age/score/active/city — those should be NULL + String nullAgeQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `age` IS NULL SETTINGS select_sequential_consistency = 1", topic); + try { + Records emailNulls = chc.getClient().queryRecords(nullEmailQuery).get(); + int emailNullCount = Integer.parseInt(emailNulls.iterator().next().getString(1)); + assertEquals(5, emailNullCount, "V1 records should have NULL for email"); + + Records ageNulls = chc.getClient().queryRecords(nullAgeQuery).get(); + int ageNullCount = Integer.parseInt(ageNulls.iterator().next().getString(1)); + assertEquals(10, ageNullCount, "V1 + V3 records (10) should have NULL for age"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Test + public void autoEvolveMixedSchemasTenRecordsInOneBatch() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_mixed_ten_records_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Single batch with 10 records spanning 3 schema versions: + // Records 1–5: Schema V1 (3 fields: off16, p_int64, name) + // Records 6–7: Schema V2 (8 fields: off16, p_int64, name, email, age, score, active, city) + // Records 8–10: Schema V3 (5 fields: off16, p_int64, name, email, country) + List batch = new ArrayList<>(); + batch.addAll(SchemaTestData.createRichSchemaV1(topic, 1, 5, 0)); + batch.addAll(SchemaTestData.createRichSchemaV2(topic, 1, 2, 5)); + batch.addAll(SchemaTestData.createRichSchemaV3(topic, 1, 3, 7)); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(batch); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify all evolved columns from all versions exist + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("name"), "V1 column 'name' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("email"), "V2 column 'email' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("age"), "V2 column 'age' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("score"), "V2 column 'score' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("active"), "V2 column 'active' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("city"), "V2 column 'city' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("country"), "V3 column 'country' should exist"); + + // Verify NULL distribution: + // name: all 10 records have it - 0 NULLs + // email: V1 records (5) lack it - 5 NULLs + // age: only V2 records (2) have it - 8 NULLs + // country: only V3 records (3) have it - 7 NULLs + try { + String nameNullQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `name` IS NULL SETTINGS select_sequential_consistency = 1", topic); + Records nameNulls = chc.getClient().queryRecords(nameNullQuery).get(); + assertEquals(0, Integer.parseInt(nameNulls.iterator().next().getString(1)), + "All records have name, so 0 NULLs expected"); + + String emailNullQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `email` IS NULL SETTINGS select_sequential_consistency = 1", topic); + Records emailNulls = chc.getClient().queryRecords(emailNullQuery).get(); + assertEquals(5, Integer.parseInt(emailNulls.iterator().next().getString(1)), + "V1 records (5) should have NULL for email"); + + String ageNullQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `age` IS NULL SETTINGS select_sequential_consistency = 1", topic); + Records ageNulls = chc.getClient().queryRecords(ageNullQuery).get(); + assertEquals(8, Integer.parseInt(ageNulls.iterator().next().getString(1)), + "V1 (5) + V3 (3) records should have NULL for age"); + + String countryNullQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `country` IS NULL SETTINGS select_sequential_consistency = 1", topic); + Records countryNulls = chc.getClient().queryRecords(countryNullQuery).get(); + assertEquals(7, Integer.parseInt(countryNulls.iterator().next().getString(1)), + "V1 (5) + V2 (2) records should have NULL for country"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + // auto-evolve adds columns for every supported primitive + logical type in a single batch @Test public void autoEvolveAllPrimitiveAndLogicalTypes() { diff --git a/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java b/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java index aacd7b122..e0b88e3af 100644 --- a/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java +++ b/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java @@ -1820,6 +1820,99 @@ public static Collection createSchemaV3WithExtraField(String topic, return array; } + /** + * Rich V1 schema with 3 fields: off16, p_int64, name. + */ + public static Collection createRichSchemaV1(String topic, int partition, int totalRecords, long startOffset) { + List array = new ArrayList<>(); + + Schema SCHEMA = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("name", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + + for (long n = 0; n < totalRecords; n++) { + Struct value_struct = new Struct(SCHEMA) + .put("off16", (short) n) + .put("p_int64", n) + .put("name", "user_" + n); + + array.add(new SinkRecord( + topic, partition, null, null, SCHEMA, value_struct, + startOffset + n, System.currentTimeMillis(), TimestampType.CREATE_TIME + )); + } + return array; + } + + /** + * Rich V2 schema with 8 fields: off16, p_int64, name, email, age, score, active, city. + */ + public static Collection createRichSchemaV2(String topic, int partition, int totalRecords, long startOffset) { + List array = new ArrayList<>(); + + Schema SCHEMA = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("name", Schema.OPTIONAL_STRING_SCHEMA) + .field("email", Schema.OPTIONAL_STRING_SCHEMA) + .field("age", SchemaBuilder.int32().optional().build()) + .field("score", SchemaBuilder.float64().optional().build()) + .field("active", SchemaBuilder.bool().optional().build()) + .field("city", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + + for (long n = 0; n < totalRecords; n++) { + Struct value_struct = new Struct(SCHEMA) + .put("off16", (short) n) + .put("p_int64", n) + .put("name", "user_" + n) + .put("email", "user_" + n + "@example.com") + .put("age", 20 + (int) n) + .put("score", n * 1.5) + .put("active", n % 2 == 0) + .put("city", "city_" + n); + + array.add(new SinkRecord( + topic, partition, null, null, SCHEMA, value_struct, + startOffset + n, System.currentTimeMillis(), TimestampType.CREATE_TIME + )); + } + return array; + } + + /** + * Rich V3 schema with 5 fields: off16, p_int64, name, email, country. + * Drops some V2 fields (age, score, active, city) and adds country. + */ + public static Collection createRichSchemaV3(String topic, int partition, int totalRecords, long startOffset) { + List array = new ArrayList<>(); + + Schema SCHEMA = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("name", Schema.OPTIONAL_STRING_SCHEMA) + .field("email", Schema.OPTIONAL_STRING_SCHEMA) + .field("country", Schema.OPTIONAL_STRING_SCHEMA) + .build(); + + for (long n = 0; n < totalRecords; n++) { + Struct value_struct = new Struct(SCHEMA) + .put("off16", (short) n) + .put("p_int64", n) + .put("name", "user_" + n) + .put("email", "user_" + n + "@example.com") + .put("country", "country_" + n); + + array.add(new SinkRecord( + topic, partition, null, null, SCHEMA, value_struct, + startOffset + n, System.currentTimeMillis(), TimestampType.CREATE_TIME + )); + } + return array; + } + public static Collection createSchemaV2WithLogicalTypes(String topic, int partition) { return createSchemaV2WithLogicalTypes(topic, partition, DEFAULT_TOTAL_RECORDS); } From 3424a5e8789feba6b7cdb29d316956ba573dc5e3 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Thu, 26 Mar 2026 12:46:21 +0100 Subject: [PATCH 45/62] feat: improve auto-evolve functionality to handle mixed schema versions --- .../connect/sink/db/ClickHouseWriter.java | 26 ++-- .../ClickHouseSinkTaskWithSchemaTest.java | 140 ++++++++++++++++++ .../connect/sink/helper/SchemaTestData.java | 32 ++++ 3 files changed, 188 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index 7196d4501..9ec38be73 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -53,6 +53,7 @@ import java.util.ArrayList; import java.util.Date; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -212,9 +213,16 @@ public void doInsert(List records, QueryIdentifier queryId, ErrorReporte if (table == null) { return; }//We checked the error flag in getTable, so we don't need to check it again here if (csc.isAutoEvolve()) { - // New columns are Nullable, so older records without the new fields insert with NULL. - Record last = records.get(records.size() - 1); - table = evolveTableSchema(table, last); + // Since auto-evolve only adds Nullable columns (never deletes), the superset is ok. + Map allFields = new LinkedHashMap<>(); + for (Record r : records) { + if (r.getFields() != null) { + for (Field f : r.getFields()) { + allFields.putIfAbsent(f.name(), f.schema()); + } + } + } + table = evolveTableSchema(table, allFields); } doInsertBatch(records, table, queryId); @@ -842,15 +850,14 @@ protected void doWriteCol(Data value, boolean fieldExists, Column col, OutputStr } } - protected Table evolveTableSchema(Table table, Record record) throws InterruptedException { - if (record.getFields() == null) { + protected Table evolveTableSchema(Table table, Map allFields) throws InterruptedException { + if (allFields.isEmpty()) { throw new RuntimeException( "auto.evolve requires a Connect schema (Avro, Protobuf, or JSON Schema). " + "Schemaless or string records are not supported with auto.evolve=true."); } - List fieldNames = record.getFields().stream().map(Field::name).collect(Collectors.toList()); - Set missingColumns = table.getMissingColumns(fieldNames); + Set missingColumns = table.getMissingColumns(allFields.keySet()); if (missingColumns.isEmpty()) { return table; @@ -858,11 +865,10 @@ protected Table evolveTableSchema(Table table, Record record) throws Interrupted LOGGER.info("Detected {} new field(s) not present in table {}: {}", missingColumns.size(), table.getName(), missingColumns); - Map schemaMap = record.getFields().stream().collect(Collectors.toMap(Field::name, Field::schema)); - List columnDefs = new java.util.ArrayList<>(); + List columnDefs = new ArrayList<>(); for (String fieldName : missingColumns) { - Schema fieldSchema = schemaMap.get(fieldName); + Schema fieldSchema = allFields.get(fieldName); if (fieldSchema == null) { continue; } diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index 9c22fa161..35ca2c4c5 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -2759,4 +2759,144 @@ public void autoEvolveAvroUnionStringBytesCreatesStringColumn() throws Exception } assertEquals(2, rows.size()); } + + @Test + public void autoEvolveMixedBatchLastRecordOlderSchema() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = createTopicName("auto_evolve_last_record_older_test"); + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Build batch where V2 records come first and V1 (older) records are last. + // Before the fix, only the last record was checked - V1 has no new fields, so ALTER TABLE was skipped. + List batch = new ArrayList<>(); + batch.addAll(SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 3)); + batch.addAll(SchemaTestData.createSchemaV1(topic, 1, 2)); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(batch); + chst.stop(); + + assertEquals(5, ClickHouseTestHelpers.countRows(chc, topic)); + + // The new column from V2 should exist even though V1 was the last record + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("new_string_field"), + "Column 'new_string_field' should be added even when last record is V1 (older schema)"); + + // V1 records should have NULL for the new column + String nullCountQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `new_string_field` IS NULL SETTINGS select_sequential_consistency = 1", topic); + try { + Records nullRecords = chc.getClient().queryRecords(nullCountQuery).get(); + int nullCount = Integer.parseInt(nullRecords.iterator().next().getString(1)); + assertEquals(2, nullCount, "V1 records should have NULL for new_string_field"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Test + public void autoEvolveMultiVersionUnionSemantics() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = createTopicName("auto_evolve_union_semantics_test"); + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Batch with V1, V2, V3, V4 - each version adds a different field. + // All new fields should be added in a single ALTER TABLE. + List batch = new ArrayList<>(); + batch.addAll(SchemaTestData.createSchemaV1(topic, 1, 2)); + batch.addAll(SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 2)); + batch.addAll(SchemaTestData.createSchemaV3WithExtraField(topic, 1, 2)); + batch.addAll(SchemaTestData.createSchemaV4WithUniqueField(topic, 1, 2)); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(batch); + chst.stop(); + + assertEquals(8, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify all fields from V2, V3, V4 exist + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("new_string_field"), + "V2 column 'new_string_field' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("v3_bool_field"), + "V3 column 'v3_bool_field' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("v4_float_field"), + "V4 column 'v4_float_field' should exist"); + } + + @Test + public void autoEvolveInterleavedSchemaVersions() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = createTopicName("auto_evolve_non_monotonic_test"); + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Interleaved schema versions [V1, V3, V2, V1, V3] + List batch = new ArrayList<>(); + batch.addAll(SchemaTestData.createSchemaV1(topic, 1, 1)); + batch.addAll(SchemaTestData.createSchemaV3WithExtraField(topic, 1, 1)); + batch.addAll(SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 1)); + batch.addAll(SchemaTestData.createSchemaV1(topic, 1, 1)); + batch.addAll(SchemaTestData.createSchemaV3WithExtraField(topic, 1, 1)); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(batch); + chst.stop(); + + assertEquals(5, ClickHouseTestHelpers.countRows(chc, topic)); + + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("new_string_field"), + "V2 column 'new_string_field' should exist despite non-monotonic order"); + assertTrue(described.getRootColumnsMap().containsKey("v3_bool_field"), + "V3 column 'v3_bool_field' should exist despite non-monotonic order"); + } + + @Test + public void autoEvolveCrossPartitionSchemaDrift() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + props.put(ClickHouseSinkConfig.IGNORE_PARTITIONS_WHEN_BATCHING, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = createTopicName("auto_evolve_cross_partition_test"); + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Records from different partitions with different schema versions. + // With ignorePartitionsWhenBatching=true, they are merged into a single batch. + // Partition 0: V2 records (has new_string_field) + // Partition 1: V1 records (no new_string_field) - these may end up last + List batch = new ArrayList<>(); + batch.addAll(SchemaTestData.createSchemaV2WithNewNullableField(topic, 0, 3)); + batch.addAll(SchemaTestData.createSchemaV1(topic, 1, 3)); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(batch); + chst.stop(); + + assertEquals(6, ClickHouseTestHelpers.countRows(chc, topic)); + + // The new column from V2 (partition 0) should exist even though + // V1 records from partition 1 may be last in the merged batch + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("new_string_field"), + "Column 'new_string_field' should be added with cross-partition schema drift"); + } } diff --git a/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java b/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java index e0b88e3af..192edba4e 100644 --- a/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java +++ b/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java @@ -1820,6 +1820,38 @@ public static Collection createSchemaV3WithExtraField(String topic, return array; } + /** + * Schema V4 with a unique field (v4_float_field) not in V2 or V3. + * Fields: off16, p_int64, v4_float_field. + */ + public static Collection createSchemaV4WithUniqueField(String topic, int partition) { + return createSchemaV4WithUniqueField(topic, partition, DEFAULT_TOTAL_RECORDS); + } + + public static Collection createSchemaV4WithUniqueField(String topic, int partition, int totalRecords) { + List array = new ArrayList<>(); + + Schema SCHEMA_V4 = SchemaBuilder.struct() + .field("off16", Schema.INT16_SCHEMA) + .field("p_int64", Schema.INT64_SCHEMA) + .field("v4_float_field", SchemaBuilder.float64().optional().build()) + .build(); + + long offset = totalRecords * 3L; // after V1, V2, V3 offsets + for (long n = 0; n < totalRecords; n++) { + Struct value_struct = new Struct(SCHEMA_V4) + .put("off16", (short) n) + .put("p_int64", n) + .put("v4_float_field", (double) n * 1.5); + + array.add(new SinkRecord( + topic, partition, null, null, SCHEMA_V4, value_struct, + offset + n, System.currentTimeMillis(), TimestampType.CREATE_TIME + )); + } + return array; + } + /** * Rich V1 schema with 3 fields: off16, p_int64, name. */ From bfb5dad8c9f16825201048b2711dd4fe82b8ce5f Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Fri, 27 Mar 2026 11:52:47 +0100 Subject: [PATCH 46/62] fix: remove unused import --- .../com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java index ddd24a286..0fba5cc60 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java @@ -4,7 +4,6 @@ import org.apache.kafka.connect.data.SchemaBuilder; import org.junit.jupiter.api.Test; -import java.util.Collections; import java.util.List; import static com.clickhouse.kafka.connect.sink.helper.ClickHouseTestHelpers.col; From c3ee1a07a2a3944c104a8b9182f23ae97e8a07e3 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Mon, 23 Mar 2026 17:47:07 +0100 Subject: [PATCH 47/62] feature: enable auto.evolve parameter --- CHANGELOG.md | 3 ++- .../com/clickhouse/kafka/connect/sink/db/mapping/Column.java | 1 + .../kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java | 1 + .../clickhouse/kafka/connect/sink/helper/SchemaTestData.java | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a98b1850..de14fa485 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ ## Dependencies * Updated clickhouse-java version from `0.9.4` to `0.9.5` -# 1.3.7, 2026-03-25 +# 1.3.7, 2026-03-25 ## Security * Upgraded `com.fasterxml.jackson.core` dependencies to version with fix for https://github.com/advisories/GHSA-72hv-8253-57qq (https://github.com/ClickHouse/clickhouse-kafka-connect/pull/690). @@ -14,6 +14,7 @@ # Improvements * `Gson` replaced with `Jackson` for performance and better maintainability (https://github.com/ClickHouse/clickhouse-kafka-connect/pull/676). + # 1.3.6, 2026-03-18 ## New Features diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java index f8f16ee7f..18558686e 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java @@ -515,6 +515,7 @@ static boolean isUnionSchema(Schema connectSchema) { && connectSchema.parameters().containsKey(CONNECT_UNION_PARAMETER); } + public Integer convertEnumValues(String value) { if ( this.enumValues != null ) { return enumValues.get(value); diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index 35ca2c4c5..4f7d5015b 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -2210,6 +2210,7 @@ public void autoEvolveStructToJsonExplicitlyFalseRejectsStruct() { } } + @Test public void autoEvolveArrayAndMapFields() { Map props = createProps(); diff --git a/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java b/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java index 192edba4e..f159d046d 100644 --- a/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java +++ b/src/testFixtures/java/com/clickhouse/kafka/connect/sink/helper/SchemaTestData.java @@ -1945,6 +1945,7 @@ public static Collection createRichSchemaV3(String topic, int partit return array; } + public static Collection createSchemaV2WithLogicalTypes(String topic, int partition) { return createSchemaV2WithLogicalTypes(topic, partition, DEFAULT_TOTAL_RECORDS); } From 2b459286f603364516c4ae76c85546d94baeb717 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 20:17:55 +0100 Subject: [PATCH 48/62] feat: add auto evolve struct to JSON configuration option to ClickHouseSinkConfig --- .../kafka/connect/sink/ClickHouseSinkConfig.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java b/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java index 22b1aff60..23252609b 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java @@ -736,6 +736,17 @@ private static ConfigDef createConfigDef() { ConfigDef.Width.SHORT, "Map STRUCT to JSON" ); + configDef.define(SSL_SOCKET_SNI, + ConfigDef.Type.STRING, + "", + ConfigDef.Importance.LOW, + "Override the SNI hostname sent in the client handshake. When set, the client will explicitly include the specified name as the server_name extension in the TLS ClientHello." + + "This is useful to avoid handshake failure when routing TLS traffic through a proxy, where the proxy hostname and the server hostname may differ. Default: ''", + group, + ++orderInGroup, + ConfigDef.Width.MEDIUM, + "SSL Socket SNI" + ); return configDef; } } From 4d67ee9109db85ceea83d1ccc4f80d19d15f86d6 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 25 Mar 2026 23:14:00 +0100 Subject: [PATCH 49/62] feat: enhance schema handling by adding union type detection and mapping for ClickHouse integration --- .../ClickHouseSinkTaskWithSchemaTest.java | 264 ++++++++++++++++++ .../connect/sink/db/mapping/ColumnTest.java | 1 + 2 files changed, 265 insertions(+) diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index 4f7d5015b..85d5a88e6 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -2274,6 +2274,270 @@ public void autoEvolveTripleSchemaInOneBatch() { "V3 column 'v3_bool_field' should exist"); } + // auto-evolve adds columns for every supported primitive + logical type in a single batch + @Test + public void autoEvolveAllPrimitiveAndLogicalTypes() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_all_types_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Insert V1 first to ensure existing rows get NULL for new columns + Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 5); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV1); + assertEquals(5, ClickHouseTestHelpers.countRows(chc, topic)); + + // Insert V2 with all primitive + logical type fields + Collection srV2 = SchemaTestData.createSchemaV2WithAllPrimitiveTypes(topic, 1, 5); + chst.put(srV2); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify all columns were created with correct types + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + Map cols = described.getRootColumnsMap(); + + // Primitive types + assertTrue(cols.containsKey("new_int8"), "Column 'new_int8' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.INT8, cols.get("new_int8").getType()); + assertTrue(cols.containsKey("new_int16"), "Column 'new_int16' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.INT16, cols.get("new_int16").getType()); + assertTrue(cols.containsKey("new_int32"), "Column 'new_int32' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.INT32, cols.get("new_int32").getType()); + assertTrue(cols.containsKey("new_int64"), "Column 'new_int64' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.INT64, cols.get("new_int64").getType()); + assertTrue(cols.containsKey("new_float32"), "Column 'new_float32' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.FLOAT32, cols.get("new_float32").getType()); + assertTrue(cols.containsKey("new_float64"), "Column 'new_float64' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.FLOAT64, cols.get("new_float64").getType()); + assertTrue(cols.containsKey("new_bool"), "Column 'new_bool' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.BOOLEAN, cols.get("new_bool").getType()); + assertTrue(cols.containsKey("new_string"), "Column 'new_string' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.STRING, cols.get("new_string").getType()); + assertTrue(cols.containsKey("new_bytes"), "Column 'new_bytes' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.STRING, cols.get("new_bytes").getType()); + + // Logical types + assertTrue(cols.containsKey("new_decimal"), "Column 'new_decimal' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.Decimal, cols.get("new_decimal").getType()); + assertTrue(cols.containsKey("new_date"), "Column 'new_date' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.Date32, cols.get("new_date").getType()); + assertTrue(cols.containsKey("new_timestamp"), "Column 'new_timestamp' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.DateTime64, cols.get("new_timestamp").getType()); + } + + // auto-evolve creates Array columns with different element types (Int32, Float64, Bool, String) + @Test + public void autoEvolveTypedArrayColumns() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_typed_arrays_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + Collection srV2 = SchemaTestData.createSchemaV2WithTypedArrays(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV2); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify all array columns were created + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + Map cols = described.getRootColumnsMap(); + + assertTrue(cols.containsKey("arr_int32"), "Column 'arr_int32' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.ARRAY, cols.get("arr_int32").getType()); + assertTrue(cols.containsKey("arr_float64"), "Column 'arr_float64' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.ARRAY, cols.get("arr_float64").getType()); + assertTrue(cols.containsKey("arr_bool"), "Column 'arr_bool' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.ARRAY, cols.get("arr_bool").getType()); + assertTrue(cols.containsKey("arr_string"), "Column 'arr_string' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.ARRAY, cols.get("arr_string").getType()); + } + + // DDL refresh timeout - retries set to 0 so refresh loop never runs, throws RetriableException + @Test + public void autoEvolveDdlRefreshTimeoutThrowsRetriable() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE_DDL_REFRESH_RETRIES, "0"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_ddl_timeout_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // V2 schema with a new field - DDL will succeed but refresh will timeout with 0 retries + Collection srV2 = SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 5); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + + try { + chst.put(srV2); + assertTrue(false, "Expected RetriableException due to DDL refresh timeout with 0 retries"); + } catch (RuntimeException e) { + // Processing layer may wrap the RetriableException - walk the cause chain + Throwable t = e; + boolean found = false; + while (t != null) { + if (t instanceof org.apache.kafka.connect.errors.RetriableException + && t.getMessage() != null && t.getMessage().contains("DDL propagation timeout")) { + found = true; + break; + } + t = t.getCause(); + } + assertTrue(found, "Should contain RetriableException with DDL propagation timeout in cause chain, got: " + e); + } finally { + chst.stop(); + } + } + + // ALTER TABLE itself fails (table dropped externally after cache populated) + @Test + public void autoEvolveDdlExecutionFailureThrowsRuntimeException() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_ddl_exec_failure_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Insert V1 records to populate the connector's internal table mapping cache + Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 5); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV1); + assertEquals(5, ClickHouseTestHelpers.countRows(chc, topic)); + + // Drop the table externally - the connector still has it cached in memory + ClickHouseTestHelpers.dropTable(chc, topic); + + // V2 schema with a new field - ALTER TABLE will fail because the table no longer exists + Collection srV2 = SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 5); + try { + chst.put(srV2); + assertTrue(false, "Expected RuntimeException due to ALTER TABLE on dropped table"); + } catch (RuntimeException e) { + // Processing layer wraps exceptions - walk the cause chain for the DDL failure + Throwable t = e; + boolean found = false; + while (t != null) { + if (t.getMessage() != null && (t.getMessage().contains("ALTER TABLE") || t.getMessage().contains("UNKNOWN_TABLE"))) { + found = true; + break; + } + t = t.getCause(); + } + assertTrue(found, "Should indicate DDL failure in cause chain, got: " + e.getClass().getName() + ": " + e.getMessage()); + } finally { + chst.stop(); + } + } + + // STRUCT field without struct-to-json flag throws SchemaTypeInferenceException with helpful message + @Test + public void autoEvolveUnsupportedStructTypeThrowsError() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + // auto.evolve.struct.to.json is false by default + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_unsupported_struct_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + Collection srV2 = SchemaTestData.createSchemaV2WithStructField(topic, 1, 5); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + + try { + chst.put(srV2); + assertTrue(false, "Expected SchemaTypeInferenceException for unsupported STRUCT type"); + } catch (RuntimeException e) { + Throwable t = e; + boolean foundInference = false; + while (t != null) { + if (t instanceof com.clickhouse.kafka.connect.sink.db.mapping.SchemaTypeInferenceException) { + foundInference = true; + break; + } + t = t.getCause(); + } + assertTrue(foundInference, + "Should throw SchemaTypeInferenceException for unsupported STRUCT, got: " + e.getClass().getName() + ": " + e.getMessage()); + } finally { + chst.stop(); + } + } + + // Avro-style union(string, bytes) STRUCT collapses to Nullable(String), not JSON + @Test + public void autoEvolveStringBytesUnionCollapsesToString() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_union_string_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + Collection srV2 = SchemaTestData.createSchemaV2WithStringBytesUnionField(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(srV2); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify the union field was created as String (not JSON) + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + com.clickhouse.kafka.connect.sink.db.mapping.Column col = described.getRootColumnsMap().get("new_union_field"); + assertNotNull(col, "Column 'new_union_field' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.STRING, col.getType(), + "Union(string, bytes) should collapse to String, not JSON"); + } + + // Avro union(string, int) auto-evolved as Variant(String, Int32) column + @Test + @SinceClickHouseVersion("24.1") + public void autoEvolveMixedUnionCreatesVariantColumn() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + props.put(ClickHouseSinkConfig.CLICKHOUSE_SETTINGS, "allow_experimental_variant_type=1"); + ClickHouseHelperClient chc = createClient(props); + + String topic = createTopicName("auto_evolve_variant_test"); + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + Collection records = SchemaTestData.createSchemaV2WithMixedTypeUnionField(topic, 1, 10); + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(records); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify the union field was created as Variant + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + com.clickhouse.kafka.connect.sink.db.mapping.Column col = described.getRootColumnsMap().get("mixed_union"); + assertNotNull(col, "Column 'mixed_union' should exist"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.VARIANT, col.getType(), + "union(string, int) should map to Variant, not String or JSON"); + } + @Test public void autoEvolveThreeSeparateBatches() { Map props = createProps(); diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java index 0fba5cc60..ddd24a286 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java @@ -4,6 +4,7 @@ import org.apache.kafka.connect.data.SchemaBuilder; import org.junit.jupiter.api.Test; +import java.util.Collections; import java.util.List; import static com.clickhouse.kafka.connect.sink.helper.ClickHouseTestHelpers.col; From 7b8d2d776eaedf079c24ffac9308e8580bfed872 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Thu, 26 Mar 2026 12:22:13 +0100 Subject: [PATCH 50/62] feat: add tests for auto-evolving schemas in ClickHouseSinkTask --- .../ClickHouseSinkTaskWithSchemaTest.java | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index 85d5a88e6..c71f3cfdd 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -2274,6 +2274,134 @@ public void autoEvolveTripleSchemaInOneBatch() { "V3 column 'v3_bool_field' should exist"); } + @Test + public void autoEvolveThreeSeparateBatches() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_three_batches_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + + // Batch 1: Schema V1 (3 fields: off16, p_int64, name) + List batch1 = new ArrayList<>(SchemaTestData.createRichSchemaV1(topic, 1, 5, 0)); + chst.put(batch1); + + // Batch 2: Schema V2 (8 fields: off16, p_int64, name, email, age, score, active, city) + List batch2 = new ArrayList<>(SchemaTestData.createRichSchemaV2(topic, 1, 5, 5)); + chst.put(batch2); + + // Batch 3: Schema V3 (5 fields: off16, p_int64, name, email, country) + List batch3 = new ArrayList<>(SchemaTestData.createRichSchemaV3(topic, 1, 5, 10)); + chst.put(batch3); + + chst.stop(); + + assertEquals(15, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify all evolved columns exist + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("name"), "V1 column 'name' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("email"), "V2 column 'email' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("age"), "V2 column 'age' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("score"), "V2 column 'score' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("active"), "V2 column 'active' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("city"), "V2 column 'city' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("country"), "V3 column 'country' should exist"); + + // V1 records should have NULL for V2/V3 columns + String nullEmailQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `email` IS NULL SETTINGS select_sequential_consistency = 1", topic); + // V3 records don't include age/score/active/city — those should be NULL + String nullAgeQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `age` IS NULL SETTINGS select_sequential_consistency = 1", topic); + try { + Records emailNulls = chc.getClient().queryRecords(nullEmailQuery).get(); + int emailNullCount = Integer.parseInt(emailNulls.iterator().next().getString(1)); + assertEquals(5, emailNullCount, "V1 records should have NULL for email"); + + Records ageNulls = chc.getClient().queryRecords(nullAgeQuery).get(); + int ageNullCount = Integer.parseInt(ageNulls.iterator().next().getString(1)); + assertEquals(10, ageNullCount, "V1 + V3 records (10) should have NULL for age"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Test + public void autoEvolveMixedSchemasTenRecordsInOneBatch() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = "auto_evolve_mixed_ten_records_test"; + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Single batch with 10 records spanning 3 schema versions: + // Records 1–5: Schema V1 (3 fields: off16, p_int64, name) + // Records 6–7: Schema V2 (8 fields: off16, p_int64, name, email, age, score, active, city) + // Records 8–10: Schema V3 (5 fields: off16, p_int64, name, email, country) + List batch = new ArrayList<>(); + batch.addAll(SchemaTestData.createRichSchemaV1(topic, 1, 5, 0)); + batch.addAll(SchemaTestData.createRichSchemaV2(topic, 1, 2, 5)); + batch.addAll(SchemaTestData.createRichSchemaV3(topic, 1, 3, 7)); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(batch); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify all evolved columns from all versions exist + com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("name"), "V1 column 'name' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("email"), "V2 column 'email' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("age"), "V2 column 'age' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("score"), "V2 column 'score' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("active"), "V2 column 'active' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("city"), "V2 column 'city' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("country"), "V3 column 'country' should exist"); + + // Verify NULL distribution: + // name: all 10 records have it - 0 NULLs + // email: V1 records (5) lack it - 5 NULLs + // age: only V2 records (2) have it - 8 NULLs + // country: only V3 records (3) have it - 7 NULLs + try { + String nameNullQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `name` IS NULL SETTINGS select_sequential_consistency = 1", topic); + Records nameNulls = chc.getClient().queryRecords(nameNullQuery).get(); + assertEquals(0, Integer.parseInt(nameNulls.iterator().next().getString(1)), + "All records have name, so 0 NULLs expected"); + + String emailNullQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `email` IS NULL SETTINGS select_sequential_consistency = 1", topic); + Records emailNulls = chc.getClient().queryRecords(emailNullQuery).get(); + assertEquals(5, Integer.parseInt(emailNulls.iterator().next().getString(1)), + "V1 records (5) should have NULL for email"); + + String ageNullQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `age` IS NULL SETTINGS select_sequential_consistency = 1", topic); + Records ageNulls = chc.getClient().queryRecords(ageNullQuery).get(); + assertEquals(8, Integer.parseInt(ageNulls.iterator().next().getString(1)), + "V1 (5) + V3 (3) records should have NULL for age"); + + String countryNullQuery = String.format( + "SELECT COUNT(*) FROM `%s` WHERE `country` IS NULL SETTINGS select_sequential_consistency = 1", topic); + Records countryNulls = chc.getClient().queryRecords(countryNullQuery).get(); + assertEquals(7, Integer.parseInt(countryNulls.iterator().next().getString(1)), + "V1 (5) + V2 (2) records should have NULL for country"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + // auto-evolve adds columns for every supported primitive + logical type in a single batch @Test public void autoEvolveAllPrimitiveAndLogicalTypes() { From 4050a55d8c03dafaae710f7c09947cc75fc93504 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Fri, 27 Mar 2026 11:52:47 +0100 Subject: [PATCH 51/62] fix: remove unused import --- .../com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java index ddd24a286..0fba5cc60 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/db/mapping/ColumnTest.java @@ -4,7 +4,6 @@ import org.apache.kafka.connect.data.SchemaBuilder; import org.junit.jupiter.api.Test; -import java.util.Collections; import java.util.List; import static com.clickhouse.kafka.connect.sink.helper.ClickHouseTestHelpers.col; From c07421e243e994e0d53696ce89a6837370c1326e Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Fri, 27 Mar 2026 12:09:37 +0100 Subject: [PATCH 52/62] fix: escape column names in ClickHouseWriter to prevent SQL injection issues --- .../com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index 9ec38be73..67768a066 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -874,7 +874,7 @@ protected Table evolveTableSchema(Table table, Map allFields) th } String chType = Column.connectTypeToClickHouseType(fieldSchema, csc.isAutoEvolveStructToJson()); - columnDefs.add(String.format("`%s` %s", fieldName, chType)); + columnDefs.add(String.format("%s %s", Utils.escapeName(fieldName), chType)); } if (!columnDefs.isEmpty()) { From 34843af0d672a9f0a90b81b98b5d63275942941d Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Fri, 27 Mar 2026 12:11:12 +0100 Subject: [PATCH 53/62] refactor: remove duplicated code from merge --- .../kafka/connect/sink/ClickHouseSinkConfig.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java b/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java index 23252609b..22b1aff60 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkConfig.java @@ -736,17 +736,6 @@ private static ConfigDef createConfigDef() { ConfigDef.Width.SHORT, "Map STRUCT to JSON" ); - configDef.define(SSL_SOCKET_SNI, - ConfigDef.Type.STRING, - "", - ConfigDef.Importance.LOW, - "Override the SNI hostname sent in the client handshake. When set, the client will explicitly include the specified name as the server_name extension in the TLS ClientHello." + - "This is useful to avoid handshake failure when routing TLS traffic through a proxy, where the proxy hostname and the server hostname may differ. Default: ''", - group, - ++orderInGroup, - ConfigDef.Width.MEDIUM, - "SSL Socket SNI" - ); return configDef; } } From a24127b290db87caa863b1c2f7fa70b353917570 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Fri, 27 Mar 2026 13:42:10 +0100 Subject: [PATCH 54/62] feat: add default expressions for Array and Map types --- .../connect/sink/db/ClickHouseWriter.java | 3 +- .../kafka/connect/sink/db/mapping/Column.java | 10 ++++++ .../ClickHouseSinkTaskWithSchemaTest.java | 34 +++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index 67768a066..4f9659cd7 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -874,7 +874,8 @@ protected Table evolveTableSchema(Table table, Map allFields) th } String chType = Column.connectTypeToClickHouseType(fieldSchema, csc.isAutoEvolveStructToJson()); - columnDefs.add(String.format("%s %s", Utils.escapeName(fieldName), chType)); + String defaultExpr = Column.defaultExpressionForType(chType); + columnDefs.add(String.format("%s %s%s", Utils.escapeName(fieldName), chType, defaultExpr)); } if (!columnDefs.isEmpty()) { diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java index 18558686e..fbf21e60d 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/mapping/Column.java @@ -549,4 +549,14 @@ public String toString() { String.format(", variantTypes=%s", variantTypes.stream().map(Tuple2::getT2).collect(Collectors.joining(", ", "[", "]"))) ) + "}"; } + + // Returns a DEFAULT expression for non-Nullable types (Array, Map) so RowBinaryWithDefaults can handle missing fields. + public static String defaultExpressionForType(String chType) { + if (chType.startsWith("Array(")) { + return " DEFAULT []"; + } else if (chType.startsWith("Map(")) { + return " DEFAULT map()"; + } + return ""; + } } diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index c71f3cfdd..7c7d90c02 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -3292,4 +3292,38 @@ public void autoEvolveCrossPartitionSchemaDrift() { assertTrue(described.getRootColumnsMap().containsKey("new_string_field"), "Column 'new_string_field' should be added with cross-partition schema drift"); } + + // Mixed batch where older records lack auto-evolved Array/Map columns. + @Test + public void autoEvolveMixedBatchArrayMapFieldsMissingInOlderRecords() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + ClickHouseHelperClient chc = createClient(props); + + String topic = createTopicName("auto_evolve_mixed_array_map_test"); + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, + "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // Build a single batch: V1 records (no array/map) followed by V2 records (with array/map) + List mixedBatch = new ArrayList<>(); + mixedBatch.addAll(SchemaTestData.createSchemaV1(topic, 1, 5)); + mixedBatch.addAll(SchemaTestData.createSchemaV2WithArrayAndMapFields(topic, 1, 5)); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(mixedBatch); + chst.stop(); + + // All 10 records should be inserted + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // The new array and map columns should exist + com.clickhouse.kafka.connect.sink.db.mapping.Table described = + chc.describeTable(chc.getDatabase(), topic); + assertTrue(described.getRootColumnsMap().containsKey("new_array_field"), + "Column 'new_array_field' should exist after auto-evolve"); + assertTrue(described.getRootColumnsMap().containsKey("new_map_field"), + "Column 'new_map_field' should exist after auto-evolve"); + } } From 453833673a2de1613d3c5cfa2a3ff26c4e058b0e Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Fri, 27 Mar 2026 14:02:31 +0100 Subject: [PATCH 55/62] fix: handle Variant columns in mixed-schema batches with auto.evolve --- .../connect/sink/db/ClickHouseWriter.java | 9 ++++- .../ClickHouseSinkTaskWithSchemaTest.java | 34 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index 4f9659cd7..a059a2d86 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -257,7 +257,8 @@ protected boolean validateDataSchema(Table table, Record record, boolean onlyFie Type type = col.getType(); boolean isNullable = col.isNullable(); boolean hasDefault = col.hasDefault(); - if (!isNullable && !hasDefault) { + // Variant has a native NULL discriminator (255) so it can accept missing values without Nullable or DEFAULT. + if (!isNullable && !hasDefault && type != Type.VARIANT) { Map schemaMap = record.getFields().stream().collect(Collectors.toMap(Field::name, Field::schema)); var objSchema = schemaMap.get(colName); Data obj = record.getJsonMap().get(colName); @@ -842,6 +843,12 @@ protected void doWriteCol(Data value, boolean fieldExists, Column col, OutputStr BinaryStreamUtils.writeNonNull(stream); } BinaryStreamUtils.writeNull(stream); + } else if (col.getType() == Type.VARIANT) { + // Variant has a native NULL discriminator (255) — no Nullable/DEFAULT needed. + if (defaultsSupport) { + BinaryStreamUtils.writeNonNull(stream); + } + BinaryStreamUtils.writeUnsignedInt8(stream, 255); } else { // no filled and not nullable LOGGER.error("Column {} is not nullable and no value is provided", name); diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index 7c7d90c02..c80b0fc26 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -3326,4 +3326,38 @@ public void autoEvolveMixedBatchArrayMapFieldsMissingInOlderRecords() { assertTrue(described.getRootColumnsMap().containsKey("new_map_field"), "Column 'new_map_field' should exist after auto-evolve"); } + + // Mixed batch where older records lack an auto-evolved Variant column. + @Test + public void autoEvolveMixedBatchVariantFieldMissingInOlderRecords() { + Map props = createProps(); + props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); + props.put(ClickHouseSinkConfig.CLICKHOUSE_SETTINGS, "allow_experimental_variant_type=1"); + ClickHouseHelperClient chc = createClient(props); + + String topic = createTopicName("auto_evolve_mixed_variant_test"); + ClickHouseTestHelpers.dropTable(chc, topic); + ClickHouseTestHelpers.createTable(chc, topic, + "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + + // V1 records (off16 + p_int64 only) followed by V2 records (off16 + p_int64 + mixed_union Variant) + List mixedBatch = new ArrayList<>(); + mixedBatch.addAll(SchemaTestData.createSchemaV1(topic, 1, 5)); + mixedBatch.addAll(SchemaTestData.createSchemaV2WithMixedTypeUnionField(topic, 1, 5)); + + ClickHouseSinkTask chst = new ClickHouseSinkTask(); + chst.start(props); + chst.put(mixedBatch); + chst.stop(); + + assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); + + // Verify the Variant column was created + com.clickhouse.kafka.connect.sink.db.mapping.Table described = + chc.describeTable(chc.getDatabase(), topic); + com.clickhouse.kafka.connect.sink.db.mapping.Column col = described.getRootColumnsMap().get("mixed_union"); + assertNotNull(col, "Column 'mixed_union' should exist after auto-evolve"); + assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.VARIANT, col.getType(), + "mixed_union should be Variant type"); + } } From 401fe21511d1c698990df7128d1fd3a65d59ba7b Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Fri, 27 Mar 2026 19:44:09 +0100 Subject: [PATCH 56/62] feat: optimize field extraction in auto-evolve by using IdentityHashMap for deduplication --- .../kafka/connect/sink/db/ClickHouseWriter.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index a059a2d86..1546f1e92 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -52,7 +52,9 @@ import java.time.temporal.ChronoField; import java.util.ArrayList; import java.util.Date; +import java.util.Collections; import java.util.HashMap; +import java.util.IdentityHashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -213,10 +215,12 @@ public void doInsert(List records, QueryIdentifier queryId, ErrorReporte if (table == null) { return; }//We checked the error flag in getTable, so we don't need to check it again here if (csc.isAutoEvolve()) { - // Since auto-evolve only adds Nullable columns (never deletes), the superset is ok. + // Collect the union of fields across all distinct schema versions in the batch. + // IdentityHashMap dedup ensures field extraction happens once per Schema object instance + Set seen = Collections.newSetFromMap(new IdentityHashMap<>()); Map allFields = new LinkedHashMap<>(); for (Record r : records) { - if (r.getFields() != null) { + if (r.getFields() != null && seen.add(r.getSinkRecord().valueSchema())) { for (Field f : r.getFields()) { allFields.putIfAbsent(f.name(), f.schema()); } From 4571045590d3abef9670f5b8708643ef96443fb3 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Sun, 29 Mar 2026 20:02:58 +0200 Subject: [PATCH 57/62] feat: forward clickhouse settings to ALTER TABLE DDL in auto-evolve --- .../connect/sink/db/ClickHouseWriter.java | 2 +- .../db/helper/ClickHouseHelperClient.java | 31 ++++++++++++------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index 1546f1e92..ce457e973 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -890,7 +890,7 @@ protected Table evolveTableSchema(Table table, Map allFields) th } if (!columnDefs.isEmpty()) { - chc.alterTableAddColumns(table.getDatabase(), table.getCleanName(), columnDefs); + chc.alterTableAddColumns(table.getDatabase(), table.getCleanName(), columnDefs, csc.getClickhouseSettings()); LOGGER.info("Schema evolution complete for table {}. Added columns: {}", table.getName(), columnDefs); table = refreshTableAfterDDL(table, missingColumns); } diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java index 1039f0ed2..ea79d6a5a 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/helper/ClickHouseHelperClient.java @@ -5,6 +5,7 @@ import com.clickhouse.client.ClickHouseNode; import com.clickhouse.client.ClickHouseNodeSelector; import com.clickhouse.client.ClickHouseProtocol; +import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.ClickHouseResponse; import com.clickhouse.client.api.Client; import com.clickhouse.client.api.enums.ProxyType; @@ -470,36 +471,42 @@ public Table describeTableV2(String database, String tableName) { return table; } - public void alterTableAddColumns(String database, String tableName, List columnDefs) { + public void alterTableAddColumns(String database, String tableName, List columnDefs, Map clickhouseSettings) { String addClauses = columnDefs.stream() .map(colDef -> "ADD COLUMN IF NOT EXISTS " + colDef) .collect(Collectors.joining(", ")); String sql = String.format("ALTER TABLE `%s`.`%s` %s", database, tableName, addClauses); LOGGER.info("Executing DDL: {}", sql); if (useClientV2) { - alterTableAddColumnV2(sql); + alterTableAddColumnV2(sql, clickhouseSettings); } else { - alterTableAddColumnV1(sql); + alterTableAddColumnV1(sql, clickhouseSettings); } } - private void alterTableAddColumnV1(String sql) { + private void alterTableAddColumnV1(String sql, Map clickhouseSettings) { try (ClickHouseClient client = ClickHouseClient.builder() .options(getDefaultClientOptions()) .nodeSelector(ClickHouseNodeSelector.of(ClickHouseProtocol.HTTP)) - .build(); - ClickHouseResponse response = client.read(server) - .query(sql) - .set("alter_sync", "1") - .executeAndWait()) { - // DDL executed; alter_sync=1 waits for the local replica to apply + .build()) { + ClickHouseRequest request = client.read(server).query(sql).set("alter_sync", "1"); + for (Map.Entry entry : clickhouseSettings.entrySet()) { + request.set(entry.getKey(), entry.getValue()); + } + try (ClickHouseResponse response = request.executeAndWait()) { + // DDL executed; alter_sync=1 waits for the local replica to apply + } } catch (ClickHouseException e) { throw new RuntimeException("Failed to execute ALTER TABLE: " + sql, e); } } - private void alterTableAddColumnV2(String sql) { - try (QueryResponse response = client.query(sql, new QuerySettings().serverSetting("alter_sync", "1")).get()) { + private void alterTableAddColumnV2(String sql, Map clickhouseSettings) { + QuerySettings settings = new QuerySettings().serverSetting("alter_sync", "1"); + for (Map.Entry entry : clickhouseSettings.entrySet()) { + settings.serverSetting(entry.getKey(), entry.getValue()); + } + try (QueryResponse response = client.query(sql, settings).get()) { // DDL executed; alter_sync=1 waits for the local replica to apply } catch (Exception e) { throw new RuntimeException("Failed to execute ALTER TABLE: " + sql, e); From a0a71e104158239de18470e6439e6c4c90125bbb Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Sun, 29 Mar 2026 20:03:18 +0200 Subject: [PATCH 58/62] fix: handle Variant and union null serialization in RowBinary writer --- .../kafka/connect/sink/db/ClickHouseWriter.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index ce457e973..784c1b434 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -761,7 +761,7 @@ protected void doWritePrimitive(Type columnType, Schema.Type dataType, OutputStr } else if (unionData.getObject() instanceof byte[]) { BinaryStreamUtils.writeString(stream, (byte[]) unionData.getObject()); } else { - throw new DataException("Not implemented conversion from " + unionData.getObject().getClass() + " to String"); + BinaryStreamUtils.writeString(stream, unionData.getObject().toString().getBytes(StandardCharsets.UTF_8)); } break; } @@ -820,6 +820,10 @@ protected void doWriteCol(Data value, boolean fieldExists, Column col, OutputStr return;//And we're done } else if (colType == Type.ARRAY) {//If the column is an array BinaryStreamUtils.writeNonNull(stream);//Then we send nonNull + } else if (colType == Type.VARIANT) { + BinaryStreamUtils.writeNonNull(stream); + BinaryStreamUtils.writeUnsignedInt8(stream, 255); + return; } else { throw new RuntimeException(String.format("An attempt to write null into not nullable column '%s'", name)); } @@ -832,7 +836,10 @@ protected void doWriteCol(Data value, boolean fieldExists, Column col, OutputStr if (!col.isNullable() && value.getObject() == null) { if (colType == Type.ARRAY) BinaryStreamUtils.writeNonNull(stream); - else + else if (colType == Type.VARIANT) { + BinaryStreamUtils.writeUnsignedInt8(stream, 255); + return; + } else throw new RuntimeException(String.format("An attempt to write null into not nullable column '%s'", name)); } } From c44c2f422b8e38283ae74c2dbb52ffa8278f093d Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Sun, 29 Mar 2026 20:03:22 +0200 Subject: [PATCH 59/62] refactor: simplify auto-evolve to check only last record schema --- .../connect/sink/db/ClickHouseWriter.java | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java index 784c1b434..85c767196 100644 --- a/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java +++ b/src/main/java/com/clickhouse/kafka/connect/sink/db/ClickHouseWriter.java @@ -52,9 +52,7 @@ import java.time.temporal.ChronoField; import java.util.ArrayList; import java.util.Date; -import java.util.Collections; import java.util.HashMap; -import java.util.IdentityHashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -215,18 +213,17 @@ public void doInsert(List records, QueryIdentifier queryId, ErrorReporte if (table == null) { return; }//We checked the error flag in getTable, so we don't need to check it again here if (csc.isAutoEvolve()) { - // Collect the union of fields across all distinct schema versions in the batch. - // IdentityHashMap dedup ensures field extraction happens once per Schema object instance - Set seen = Collections.newSetFromMap(new IdentityHashMap<>()); - Map allFields = new LinkedHashMap<>(); - for (Record r : records) { - if (r.getFields() != null && seen.add(r.getSinkRecord().valueSchema())) { - for (Field f : r.getFields()) { - allFields.putIfAbsent(f.name(), f.schema()); - } + // Check the last record's schema for new fields. + // Limitation: if a batch contains multiple schema versions, only the last one is checked. + // A full scan across all records can be implemented later if needed. + Record last = records.get(records.size() - 1); + Map lastFields = new LinkedHashMap<>(); + if (last.getFields() != null) { + for (Field f : last.getFields()) { + lastFields.put(f.name(), f.schema()); } } - table = evolveTableSchema(table, allFields); + table = evolveTableSchema(table, lastFields); } doInsertBatch(records, table, queryId); From 65a802dfff86dd496dc83479002b6778ea742789 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 1 Apr 2026 09:08:10 +0200 Subject: [PATCH 60/62] fix: replace createTable with runQuery to match main convention --- .../ClickHouseSinkTaskWithSchemaTest.java | 460 ++---------------- 1 file changed, 34 insertions(+), 426 deletions(-) diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index c80b0fc26..3da6c269f 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -1813,7 +1813,7 @@ public void autoEvolveDisabledRejectsNewField() { String topic = "auto_evolve_disabled_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Insert V1 records first (should succeed) Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 10); @@ -1840,7 +1840,7 @@ public void autoEvolveAddsNullableColumn() { String topic = "auto_evolve_nullable_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Insert V1 records Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 10); @@ -1870,7 +1870,7 @@ public void autoEvolveAddsNonNullableFieldAsNullable() { String topic = "auto_evolve_non_nullable_as_nullable_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); Collection srV2 = SchemaTestData.createSchemaV2WithNewNonNullableField(topic, 1, 10); ClickHouseSinkTask chst = new ClickHouseSinkTask(); @@ -1895,7 +1895,7 @@ public void autoEvolveMultipleNewColumns() { String topic = "auto_evolve_multi_cols_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Insert V1 first Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 10); @@ -1929,7 +1929,7 @@ public void autoEvolveCachesSchemaAfterDDL() { String topic = "auto_evolve_cache_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); ClickHouseSinkTask chst = new ClickHouseSinkTask(); chst.start(props); @@ -1963,7 +1963,7 @@ public void autoEvolveMixedSchemaInSingleBatch() { String topic = "auto_evolve_mixed_batch_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Build a single batch with V1 records followed by V2 records (mixed schemas) List mixedBatch = new ArrayList<>(); @@ -1992,7 +1992,7 @@ public void autoEvolveMixedSchemaOlderRecordsGetNull() { String topic = "auto_evolve_older_records_null_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // V1 records (no new_string_field) followed by V2 records (has new_string_field) // Schema is evolved using last record (V2), then entire batch is inserted. @@ -2039,7 +2039,7 @@ public void autoEvolveLogicalTypes() { String topic = "auto_evolve_logical_types_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Insert V1 first Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 10); @@ -2086,7 +2086,7 @@ public void autoEvolveRejectsStructField() { String topic = "auto_evolve_struct_reject_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); Collection srV2 = SchemaTestData.createSchemaV2WithStructField(topic, 1, 10); ClickHouseSinkTask chst = new ClickHouseSinkTask(); @@ -2122,7 +2122,7 @@ public void autoEvolveStructToJsonCreatesJsonColumn() { String topic = "auto_evolve_struct_to_json_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); Collection srV2 = SchemaTestData.createSchemaV2WithStructField(topic, 1, 10); ClickHouseSinkTask chst = new ClickHouseSinkTask(); @@ -2152,7 +2152,7 @@ public void autoEvolveStructToJsonMixedBatchOlderRecordsGetDefault() { String topic = "auto_evolve_struct_json_mixed_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Insert V1 records (no struct field) Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 5); @@ -2185,7 +2185,7 @@ public void autoEvolveStructToJsonExplicitlyFalseRejectsStruct() { String topic = "auto_evolve_struct_json_false_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); Collection srV2 = SchemaTestData.createSchemaV2WithStructField(topic, 1, 10); ClickHouseSinkTask chst = new ClickHouseSinkTask(); @@ -2219,7 +2219,7 @@ public void autoEvolveArrayAndMapFields() { String topic = "auto_evolve_array_map_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Insert V1 first Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 10); @@ -2251,7 +2251,7 @@ public void autoEvolveTripleSchemaInOneBatch() { String topic = "auto_evolve_triple_schema_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Build a single batch with V1 + V2 + V3 records List combined = new ArrayList<>(); @@ -2282,7 +2282,7 @@ public void autoEvolveThreeSeparateBatches() { String topic = "auto_evolve_three_batches_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); ClickHouseSinkTask chst = new ClickHouseSinkTask(); chst.start(props); @@ -2340,7 +2340,7 @@ public void autoEvolveMixedSchemasTenRecordsInOneBatch() { String topic = "auto_evolve_mixed_ten_records_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Single batch with 10 records spanning 3 schema versions: // Records 1–5: Schema V1 (3 fields: off16, p_int64, name) @@ -2411,7 +2411,7 @@ public void autoEvolveAllPrimitiveAndLogicalTypes() { String topic = "auto_evolve_all_types_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Insert V1 first to ensure existing rows get NULL for new columns Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 5); @@ -2469,7 +2469,7 @@ public void autoEvolveTypedArrayColumns() { String topic = "auto_evolve_typed_arrays_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); Collection srV2 = SchemaTestData.createSchemaV2WithTypedArrays(topic, 1, 10); ClickHouseSinkTask chst = new ClickHouseSinkTask(); @@ -2503,7 +2503,7 @@ public void autoEvolveDdlRefreshTimeoutThrowsRetriable() { String topic = "auto_evolve_ddl_timeout_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // V2 schema with a new field - DDL will succeed but refresh will timeout with 0 retries Collection srV2 = SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 5); @@ -2540,7 +2540,7 @@ public void autoEvolveDdlExecutionFailureThrowsRuntimeException() { String topic = "auto_evolve_ddl_exec_failure_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Insert V1 records to populate the connector's internal table mapping cache Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 5); @@ -2584,7 +2584,7 @@ public void autoEvolveUnsupportedStructTypeThrowsError() { String topic = "auto_evolve_unsupported_struct_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); Collection srV2 = SchemaTestData.createSchemaV2WithStructField(topic, 1, 5); ClickHouseSinkTask chst = new ClickHouseSinkTask(); @@ -2619,7 +2619,7 @@ public void autoEvolveStringBytesUnionCollapsesToString() { String topic = "auto_evolve_union_string_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); Collection srV2 = SchemaTestData.createSchemaV2WithStringBytesUnionField(topic, 1, 10); ClickHouseSinkTask chst = new ClickHouseSinkTask(); @@ -2648,399 +2648,7 @@ public void autoEvolveMixedUnionCreatesVariantColumn() { String topic = createTopicName("auto_evolve_variant_test"); ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); - - Collection records = SchemaTestData.createSchemaV2WithMixedTypeUnionField(topic, 1, 10); - ClickHouseSinkTask chst = new ClickHouseSinkTask(); - chst.start(props); - chst.put(records); - chst.stop(); - - assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); - - // Verify the union field was created as Variant - com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); - com.clickhouse.kafka.connect.sink.db.mapping.Column col = described.getRootColumnsMap().get("mixed_union"); - assertNotNull(col, "Column 'mixed_union' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.VARIANT, col.getType(), - "union(string, int) should map to Variant, not String or JSON"); - } - - @Test - public void autoEvolveThreeSeparateBatches() { - Map props = createProps(); - props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); - - String topic = "auto_evolve_three_batches_test"; - ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); - - ClickHouseSinkTask chst = new ClickHouseSinkTask(); - chst.start(props); - - // Batch 1: Schema V1 (3 fields: off16, p_int64, name) - List batch1 = new ArrayList<>(SchemaTestData.createRichSchemaV1(topic, 1, 5, 0)); - chst.put(batch1); - - // Batch 2: Schema V2 (8 fields: off16, p_int64, name, email, age, score, active, city) - List batch2 = new ArrayList<>(SchemaTestData.createRichSchemaV2(topic, 1, 5, 5)); - chst.put(batch2); - - // Batch 3: Schema V3 (5 fields: off16, p_int64, name, email, country) - List batch3 = new ArrayList<>(SchemaTestData.createRichSchemaV3(topic, 1, 5, 10)); - chst.put(batch3); - - chst.stop(); - - assertEquals(15, ClickHouseTestHelpers.countRows(chc, topic)); - - // Verify all evolved columns exist - com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); - assertTrue(described.getRootColumnsMap().containsKey("name"), "V1 column 'name' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("email"), "V2 column 'email' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("age"), "V2 column 'age' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("score"), "V2 column 'score' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("active"), "V2 column 'active' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("city"), "V2 column 'city' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("country"), "V3 column 'country' should exist"); - - // V1 records should have NULL for V2/V3 columns - String nullEmailQuery = String.format( - "SELECT COUNT(*) FROM `%s` WHERE `email` IS NULL SETTINGS select_sequential_consistency = 1", topic); - // V3 records don't include age/score/active/city — those should be NULL - String nullAgeQuery = String.format( - "SELECT COUNT(*) FROM `%s` WHERE `age` IS NULL SETTINGS select_sequential_consistency = 1", topic); - try { - Records emailNulls = chc.getClient().queryRecords(nullEmailQuery).get(); - int emailNullCount = Integer.parseInt(emailNulls.iterator().next().getString(1)); - assertEquals(5, emailNullCount, "V1 records should have NULL for email"); - - Records ageNulls = chc.getClient().queryRecords(nullAgeQuery).get(); - int ageNullCount = Integer.parseInt(ageNulls.iterator().next().getString(1)); - assertEquals(10, ageNullCount, "V1 + V3 records (10) should have NULL for age"); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Test - public void autoEvolveMixedSchemasTenRecordsInOneBatch() { - Map props = createProps(); - props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); - - String topic = "auto_evolve_mixed_ten_records_test"; - ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); - - // Single batch with 10 records spanning 3 schema versions: - // Records 1–5: Schema V1 (3 fields: off16, p_int64, name) - // Records 6–7: Schema V2 (8 fields: off16, p_int64, name, email, age, score, active, city) - // Records 8–10: Schema V3 (5 fields: off16, p_int64, name, email, country) - List batch = new ArrayList<>(); - batch.addAll(SchemaTestData.createRichSchemaV1(topic, 1, 5, 0)); - batch.addAll(SchemaTestData.createRichSchemaV2(topic, 1, 2, 5)); - batch.addAll(SchemaTestData.createRichSchemaV3(topic, 1, 3, 7)); - - ClickHouseSinkTask chst = new ClickHouseSinkTask(); - chst.start(props); - chst.put(batch); - chst.stop(); - - assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); - - // Verify all evolved columns from all versions exist - com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); - assertTrue(described.getRootColumnsMap().containsKey("name"), "V1 column 'name' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("email"), "V2 column 'email' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("age"), "V2 column 'age' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("score"), "V2 column 'score' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("active"), "V2 column 'active' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("city"), "V2 column 'city' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("country"), "V3 column 'country' should exist"); - - // Verify NULL distribution: - // name: all 10 records have it - 0 NULLs - // email: V1 records (5) lack it - 5 NULLs - // age: only V2 records (2) have it - 8 NULLs - // country: only V3 records (3) have it - 7 NULLs - try { - String nameNullQuery = String.format( - "SELECT COUNT(*) FROM `%s` WHERE `name` IS NULL SETTINGS select_sequential_consistency = 1", topic); - Records nameNulls = chc.getClient().queryRecords(nameNullQuery).get(); - assertEquals(0, Integer.parseInt(nameNulls.iterator().next().getString(1)), - "All records have name, so 0 NULLs expected"); - - String emailNullQuery = String.format( - "SELECT COUNT(*) FROM `%s` WHERE `email` IS NULL SETTINGS select_sequential_consistency = 1", topic); - Records emailNulls = chc.getClient().queryRecords(emailNullQuery).get(); - assertEquals(5, Integer.parseInt(emailNulls.iterator().next().getString(1)), - "V1 records (5) should have NULL for email"); - - String ageNullQuery = String.format( - "SELECT COUNT(*) FROM `%s` WHERE `age` IS NULL SETTINGS select_sequential_consistency = 1", topic); - Records ageNulls = chc.getClient().queryRecords(ageNullQuery).get(); - assertEquals(8, Integer.parseInt(ageNulls.iterator().next().getString(1)), - "V1 (5) + V3 (3) records should have NULL for age"); - - String countryNullQuery = String.format( - "SELECT COUNT(*) FROM `%s` WHERE `country` IS NULL SETTINGS select_sequential_consistency = 1", topic); - Records countryNulls = chc.getClient().queryRecords(countryNullQuery).get(); - assertEquals(7, Integer.parseInt(countryNulls.iterator().next().getString(1)), - "V1 (5) + V2 (2) records should have NULL for country"); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - // auto-evolve adds columns for every supported primitive + logical type in a single batch - @Test - public void autoEvolveAllPrimitiveAndLogicalTypes() { - Map props = createProps(); - props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); - - String topic = "auto_evolve_all_types_test"; - ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); - - // Insert V1 first to ensure existing rows get NULL for new columns - Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 5); - ClickHouseSinkTask chst = new ClickHouseSinkTask(); - chst.start(props); - chst.put(srV1); - assertEquals(5, ClickHouseTestHelpers.countRows(chc, topic)); - - // Insert V2 with all primitive + logical type fields - Collection srV2 = SchemaTestData.createSchemaV2WithAllPrimitiveTypes(topic, 1, 5); - chst.put(srV2); - chst.stop(); - - assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); - - // Verify all columns were created with correct types - com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); - Map cols = described.getRootColumnsMap(); - - // Primitive types - assertTrue(cols.containsKey("new_int8"), "Column 'new_int8' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.INT8, cols.get("new_int8").getType()); - assertTrue(cols.containsKey("new_int16"), "Column 'new_int16' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.INT16, cols.get("new_int16").getType()); - assertTrue(cols.containsKey("new_int32"), "Column 'new_int32' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.INT32, cols.get("new_int32").getType()); - assertTrue(cols.containsKey("new_int64"), "Column 'new_int64' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.INT64, cols.get("new_int64").getType()); - assertTrue(cols.containsKey("new_float32"), "Column 'new_float32' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.FLOAT32, cols.get("new_float32").getType()); - assertTrue(cols.containsKey("new_float64"), "Column 'new_float64' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.FLOAT64, cols.get("new_float64").getType()); - assertTrue(cols.containsKey("new_bool"), "Column 'new_bool' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.BOOLEAN, cols.get("new_bool").getType()); - assertTrue(cols.containsKey("new_string"), "Column 'new_string' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.STRING, cols.get("new_string").getType()); - assertTrue(cols.containsKey("new_bytes"), "Column 'new_bytes' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.STRING, cols.get("new_bytes").getType()); - - // Logical types - assertTrue(cols.containsKey("new_decimal"), "Column 'new_decimal' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.Decimal, cols.get("new_decimal").getType()); - assertTrue(cols.containsKey("new_date"), "Column 'new_date' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.Date32, cols.get("new_date").getType()); - assertTrue(cols.containsKey("new_timestamp"), "Column 'new_timestamp' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.DateTime64, cols.get("new_timestamp").getType()); - } - - // auto-evolve creates Array columns with different element types (Int32, Float64, Bool, String) - @Test - public void autoEvolveTypedArrayColumns() { - Map props = createProps(); - props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); - - String topic = "auto_evolve_typed_arrays_test"; - ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); - - Collection srV2 = SchemaTestData.createSchemaV2WithTypedArrays(topic, 1, 10); - ClickHouseSinkTask chst = new ClickHouseSinkTask(); - chst.start(props); - chst.put(srV2); - chst.stop(); - - assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); - - // Verify all array columns were created - com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); - Map cols = described.getRootColumnsMap(); - - assertTrue(cols.containsKey("arr_int32"), "Column 'arr_int32' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.ARRAY, cols.get("arr_int32").getType()); - assertTrue(cols.containsKey("arr_float64"), "Column 'arr_float64' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.ARRAY, cols.get("arr_float64").getType()); - assertTrue(cols.containsKey("arr_bool"), "Column 'arr_bool' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.ARRAY, cols.get("arr_bool").getType()); - assertTrue(cols.containsKey("arr_string"), "Column 'arr_string' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.ARRAY, cols.get("arr_string").getType()); - } - - // DDL refresh timeout - retries set to 0 so refresh loop never runs, throws RetriableException - @Test - public void autoEvolveDdlRefreshTimeoutThrowsRetriable() { - Map props = createProps(); - props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - props.put(ClickHouseSinkConfig.AUTO_EVOLVE_DDL_REFRESH_RETRIES, "0"); - ClickHouseHelperClient chc = createClient(props); - - String topic = "auto_evolve_ddl_timeout_test"; - ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); - - // V2 schema with a new field - DDL will succeed but refresh will timeout with 0 retries - Collection srV2 = SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 5); - ClickHouseSinkTask chst = new ClickHouseSinkTask(); - chst.start(props); - - try { - chst.put(srV2); - assertTrue(false, "Expected RetriableException due to DDL refresh timeout with 0 retries"); - } catch (RuntimeException e) { - // Processing layer may wrap the RetriableException - walk the cause chain - Throwable t = e; - boolean found = false; - while (t != null) { - if (t instanceof org.apache.kafka.connect.errors.RetriableException - && t.getMessage() != null && t.getMessage().contains("DDL propagation timeout")) { - found = true; - break; - } - t = t.getCause(); - } - assertTrue(found, "Should contain RetriableException with DDL propagation timeout in cause chain, got: " + e); - } finally { - chst.stop(); - } - } - - // ALTER TABLE itself fails (table dropped externally after cache populated) - @Test - public void autoEvolveDdlExecutionFailureThrowsRuntimeException() { - Map props = createProps(); - props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); - - String topic = "auto_evolve_ddl_exec_failure_test"; - ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); - - // Insert V1 records to populate the connector's internal table mapping cache - Collection srV1 = SchemaTestData.createSchemaV1(topic, 1, 5); - ClickHouseSinkTask chst = new ClickHouseSinkTask(); - chst.start(props); - chst.put(srV1); - assertEquals(5, ClickHouseTestHelpers.countRows(chc, topic)); - - // Drop the table externally - the connector still has it cached in memory - ClickHouseTestHelpers.dropTable(chc, topic); - - // V2 schema with a new field - ALTER TABLE will fail because the table no longer exists - Collection srV2 = SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 5); - try { - chst.put(srV2); - assertTrue(false, "Expected RuntimeException due to ALTER TABLE on dropped table"); - } catch (RuntimeException e) { - // Processing layer wraps exceptions - walk the cause chain for the DDL failure - Throwable t = e; - boolean found = false; - while (t != null) { - if (t.getMessage() != null && (t.getMessage().contains("ALTER TABLE") || t.getMessage().contains("UNKNOWN_TABLE"))) { - found = true; - break; - } - t = t.getCause(); - } - assertTrue(found, "Should indicate DDL failure in cause chain, got: " + e.getClass().getName() + ": " + e.getMessage()); - } finally { - chst.stop(); - } - } - - // STRUCT field without struct-to-json flag throws SchemaTypeInferenceException with helpful message - @Test - public void autoEvolveUnsupportedStructTypeThrowsError() { - Map props = createProps(); - props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - // auto.evolve.struct.to.json is false by default - ClickHouseHelperClient chc = createClient(props); - - String topic = "auto_evolve_unsupported_struct_test"; - ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); - - Collection srV2 = SchemaTestData.createSchemaV2WithStructField(topic, 1, 5); - ClickHouseSinkTask chst = new ClickHouseSinkTask(); - chst.start(props); - - try { - chst.put(srV2); - assertTrue(false, "Expected SchemaTypeInferenceException for unsupported STRUCT type"); - } catch (RuntimeException e) { - Throwable t = e; - boolean foundInference = false; - while (t != null) { - if (t instanceof com.clickhouse.kafka.connect.sink.db.mapping.SchemaTypeInferenceException) { - foundInference = true; - break; - } - t = t.getCause(); - } - assertTrue(foundInference, - "Should throw SchemaTypeInferenceException for unsupported STRUCT, got: " + e.getClass().getName() + ": " + e.getMessage()); - } finally { - chst.stop(); - } - } - - // Avro-style union(string, bytes) STRUCT collapses to Nullable(String), not JSON - @Test - public void autoEvolveStringBytesUnionCollapsesToString() { - Map props = createProps(); - props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); - - String topic = "auto_evolve_union_string_test"; - ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); - - Collection srV2 = SchemaTestData.createSchemaV2WithStringBytesUnionField(topic, 1, 10); - ClickHouseSinkTask chst = new ClickHouseSinkTask(); - chst.start(props); - chst.put(srV2); - chst.stop(); - - assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); - - // Verify the union field was created as String (not JSON) - com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); - com.clickhouse.kafka.connect.sink.db.mapping.Column col = described.getRootColumnsMap().get("new_union_field"); - assertNotNull(col, "Column 'new_union_field' should exist"); - assertEquals(com.clickhouse.kafka.connect.sink.db.mapping.Type.STRING, col.getType(), - "Union(string, bytes) should collapse to String, not JSON"); - } - - // Avro union(string, int) auto-evolved as Variant(String, Int32) column - @Test - @SinceClickHouseVersion("24.1") - public void autoEvolveMixedUnionCreatesVariantColumn() { - Map props = createProps(); - props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - props.put(ClickHouseSinkConfig.CLICKHOUSE_SETTINGS, "allow_experimental_variant_type=1"); - ClickHouseHelperClient chc = createClient(props); - - String topic = createTopicName("auto_evolve_variant_test"); - ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); Collection records = SchemaTestData.createSchemaV2WithMixedTypeUnionField(topic, 1, 10); ClickHouseSinkTask chst = new ClickHouseSinkTask(); @@ -3066,7 +2674,7 @@ public void autoEvolveSchemalessRecordsThrowError() { String topic = "auto_evolve_schemaless_test"; ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Create schemaless (string) records. No valueSchema. List schemaless = new ArrayList<>(); @@ -3110,8 +2718,8 @@ public void autoEvolveAvroUnionStringBytesCreatesStringColumn() throws Exception String topic = "auto_evolve_avro_union_test"; ClickHouseTestHelpers.dropTable(chc, topic); // Table starts with only "name" - union fields "content" and "description" will be auto-evolved - ClickHouseTestHelpers.createTable(chc, topic, - "CREATE TABLE `%s` (`name` String) Engine = MergeTree ORDER BY name"); + ClickHouseTestHelpers.runQuery(chc, String.format( + "CREATE TABLE `%s` (`name` String) Engine = MergeTree ORDER BY name", topic)); Image image1 = Image.newBuilder() .setName("image1") @@ -3161,7 +2769,7 @@ public void autoEvolveMixedBatchLastRecordOlderSchema() { String topic = createTopicName("auto_evolve_last_record_older_test"); ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Build batch where V2 records come first and V1 (older) records are last. // Before the fix, only the last record was checked - V1 has no new fields, so ALTER TABLE was skipped. @@ -3201,7 +2809,7 @@ public void autoEvolveMultiVersionUnionSemantics() { String topic = createTopicName("auto_evolve_union_semantics_test"); ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Batch with V1, V2, V3, V4 - each version adds a different field. // All new fields should be added in a single ALTER TABLE. @@ -3236,7 +2844,7 @@ public void autoEvolveInterleavedSchemaVersions() { String topic = createTopicName("auto_evolve_non_monotonic_test"); ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Interleaved schema versions [V1, V3, V2, V1, V3] List batch = new ArrayList<>(); @@ -3269,7 +2877,7 @@ public void autoEvolveCrossPartitionSchemaDrift() { String topic = createTopicName("auto_evolve_cross_partition_test"); ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Records from different partitions with different schema versions. // With ignorePartitionsWhenBatching=true, they are merged into a single batch. @@ -3302,8 +2910,8 @@ public void autoEvolveMixedBatchArrayMapFieldsMissingInOlderRecords() { String topic = createTopicName("auto_evolve_mixed_array_map_test"); ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, - "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format( + "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // Build a single batch: V1 records (no array/map) followed by V2 records (with array/map) List mixedBatch = new ArrayList<>(); @@ -3337,8 +2945,8 @@ public void autoEvolveMixedBatchVariantFieldMissingInOlderRecords() { String topic = createTopicName("auto_evolve_mixed_variant_test"); ClickHouseTestHelpers.dropTable(chc, topic); - ClickHouseTestHelpers.createTable(chc, topic, - "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16"); + ClickHouseTestHelpers.runQuery(chc, String.format( + "CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); // V1 records (off16 + p_int64 only) followed by V2 records (off16 + p_int64 + mixed_union Variant) List mixedBatch = new ArrayList<>(); From 2e56b81b3a539404e14e6b130a957cd00b5b7b12 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 29 Apr 2026 23:27:00 +0200 Subject: [PATCH 61/62] fix: use existing test helpers for auto-evolve tests After merging main, the new auto-evolve tests called createProps() and createClient(props) which don't exist on the test class. Replace with the existing helpers getBaseProps() and ClickHouseTestHelpers.createClient(props) to fix the compileTestJava failures. --- .../ClickHouseSinkTaskWithSchemaTest.java | 124 +++++++++--------- 1 file changed, 62 insertions(+), 62 deletions(-) diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index 3da6c269f..f84ab2a7b 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -1807,9 +1807,9 @@ public void testAvroDateAndTimeTypes() throws Exception { @Test public void autoEvolveDisabledRejectsNewField() { - Map props = createProps(); + Map props = getBaseProps(); // auto.evolve defaults to false - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = "auto_evolve_disabled_test"; ClickHouseTestHelpers.dropTable(chc, topic); @@ -1834,9 +1834,9 @@ public void autoEvolveDisabledRejectsNewField() { @Test public void autoEvolveAddsNullableColumn() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = "auto_evolve_nullable_test"; ClickHouseTestHelpers.dropTable(chc, topic); @@ -1864,9 +1864,9 @@ public void autoEvolveAddsNullableColumn() { @Test public void autoEvolveAddsNonNullableFieldAsNullable() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = "auto_evolve_non_nullable_as_nullable_test"; ClickHouseTestHelpers.dropTable(chc, topic); @@ -1889,9 +1889,9 @@ public void autoEvolveAddsNonNullableFieldAsNullable() { @Test public void autoEvolveMultipleNewColumns() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = "auto_evolve_multi_cols_test"; ClickHouseTestHelpers.dropTable(chc, topic); @@ -1923,9 +1923,9 @@ public void autoEvolveMultipleNewColumns() { @Test public void autoEvolveCachesSchemaAfterDDL() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = "auto_evolve_cache_test"; ClickHouseTestHelpers.dropTable(chc, topic); @@ -1957,9 +1957,9 @@ public void autoEvolveCachesSchemaAfterDDL() { @Test public void autoEvolveMixedSchemaInSingleBatch() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = "auto_evolve_mixed_batch_test"; ClickHouseTestHelpers.dropTable(chc, topic); @@ -1986,9 +1986,9 @@ public void autoEvolveMixedSchemaInSingleBatch() { @Test public void autoEvolveMixedSchemaOlderRecordsGetNull() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = "auto_evolve_older_records_null_test"; ClickHouseTestHelpers.dropTable(chc, topic); @@ -2033,9 +2033,9 @@ public void autoEvolveMixedSchemaOlderRecordsGetNull() { @Test public void autoEvolveLogicalTypes() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = "auto_evolve_logical_types_test"; ClickHouseTestHelpers.dropTable(chc, topic); @@ -2080,9 +2080,9 @@ public void autoEvolveLogicalTypes() { @Test public void autoEvolveRejectsStructField() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = "auto_evolve_struct_reject_test"; ClickHouseTestHelpers.dropTable(chc, topic); @@ -2114,11 +2114,11 @@ public void autoEvolveRejectsStructField() { // STRUCT field auto-evolved as JSON column when auto.evolve.struct.to.json=true @Test public void autoEvolveStructToJsonCreatesJsonColumn() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); props.put(ClickHouseSinkConfig.AUTO_EVOLVE_STRUCT_TO_JSON, "true"); props.put(ClickHouseSinkConfig.CLICKHOUSE_SETTINGS, "input_format_binary_read_json_as_string=1"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = "auto_evolve_struct_to_json_test"; ClickHouseTestHelpers.dropTable(chc, topic); @@ -2144,11 +2144,11 @@ public void autoEvolveStructToJsonCreatesJsonColumn() { // V1 records (no struct) inserted first, then V2 records (with struct) trigger JSON column creation. @Test public void autoEvolveStructToJsonMixedBatchOlderRecordsGetDefault() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); props.put(ClickHouseSinkConfig.AUTO_EVOLVE_STRUCT_TO_JSON, "true"); props.put(ClickHouseSinkConfig.CLICKHOUSE_SETTINGS, "input_format_binary_read_json_as_string=1"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = "auto_evolve_struct_json_mixed_test"; ClickHouseTestHelpers.dropTable(chc, topic); @@ -2178,10 +2178,10 @@ public void autoEvolveStructToJsonMixedBatchOlderRecordsGetDefault() { // STRUCT field with auto.evolve.struct.to.json explicitly false rejects with helpful error message @Test public void autoEvolveStructToJsonExplicitlyFalseRejectsStruct() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); props.put(ClickHouseSinkConfig.AUTO_EVOLVE_STRUCT_TO_JSON, "false"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = "auto_evolve_struct_json_false_test"; ClickHouseTestHelpers.dropTable(chc, topic); @@ -2213,9 +2213,9 @@ public void autoEvolveStructToJsonExplicitlyFalseRejectsStruct() { @Test public void autoEvolveArrayAndMapFields() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = "auto_evolve_array_map_test"; ClickHouseTestHelpers.dropTable(chc, topic); @@ -2245,9 +2245,9 @@ public void autoEvolveArrayAndMapFields() { @Test public void autoEvolveTripleSchemaInOneBatch() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = "auto_evolve_triple_schema_test"; ClickHouseTestHelpers.dropTable(chc, topic); @@ -2276,9 +2276,9 @@ public void autoEvolveTripleSchemaInOneBatch() { @Test public void autoEvolveThreeSeparateBatches() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = "auto_evolve_three_batches_test"; ClickHouseTestHelpers.dropTable(chc, topic); @@ -2334,9 +2334,9 @@ public void autoEvolveThreeSeparateBatches() { @Test public void autoEvolveMixedSchemasTenRecordsInOneBatch() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = "auto_evolve_mixed_ten_records_test"; ClickHouseTestHelpers.dropTable(chc, topic); @@ -2405,9 +2405,9 @@ public void autoEvolveMixedSchemasTenRecordsInOneBatch() { // auto-evolve adds columns for every supported primitive + logical type in a single batch @Test public void autoEvolveAllPrimitiveAndLogicalTypes() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = "auto_evolve_all_types_test"; ClickHouseTestHelpers.dropTable(chc, topic); @@ -2463,9 +2463,9 @@ public void autoEvolveAllPrimitiveAndLogicalTypes() { // auto-evolve creates Array columns with different element types (Int32, Float64, Bool, String) @Test public void autoEvolveTypedArrayColumns() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = "auto_evolve_typed_arrays_test"; ClickHouseTestHelpers.dropTable(chc, topic); @@ -2496,10 +2496,10 @@ public void autoEvolveTypedArrayColumns() { // DDL refresh timeout - retries set to 0 so refresh loop never runs, throws RetriableException @Test public void autoEvolveDdlRefreshTimeoutThrowsRetriable() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); props.put(ClickHouseSinkConfig.AUTO_EVOLVE_DDL_REFRESH_RETRIES, "0"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = "auto_evolve_ddl_timeout_test"; ClickHouseTestHelpers.dropTable(chc, topic); @@ -2534,9 +2534,9 @@ public void autoEvolveDdlRefreshTimeoutThrowsRetriable() { // ALTER TABLE itself fails (table dropped externally after cache populated) @Test public void autoEvolveDdlExecutionFailureThrowsRuntimeException() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = "auto_evolve_ddl_exec_failure_test"; ClickHouseTestHelpers.dropTable(chc, topic); @@ -2577,10 +2577,10 @@ public void autoEvolveDdlExecutionFailureThrowsRuntimeException() { // STRUCT field without struct-to-json flag throws SchemaTypeInferenceException with helpful message @Test public void autoEvolveUnsupportedStructTypeThrowsError() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); // auto.evolve.struct.to.json is false by default - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = "auto_evolve_unsupported_struct_test"; ClickHouseTestHelpers.dropTable(chc, topic); @@ -2613,9 +2613,9 @@ public void autoEvolveUnsupportedStructTypeThrowsError() { // Avro-style union(string, bytes) STRUCT collapses to Nullable(String), not JSON @Test public void autoEvolveStringBytesUnionCollapsesToString() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = "auto_evolve_union_string_test"; ClickHouseTestHelpers.dropTable(chc, topic); @@ -2641,10 +2641,10 @@ public void autoEvolveStringBytesUnionCollapsesToString() { @Test @SinceClickHouseVersion("24.1") public void autoEvolveMixedUnionCreatesVariantColumn() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); props.put(ClickHouseSinkConfig.CLICKHOUSE_SETTINGS, "allow_experimental_variant_type=1"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = createTopicName("auto_evolve_variant_test"); ClickHouseTestHelpers.dropTable(chc, topic); @@ -2668,9 +2668,9 @@ public void autoEvolveMixedUnionCreatesVariantColumn() { @Test public void autoEvolveSchemalessRecordsThrowError() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = "auto_evolve_schemaless_test"; ClickHouseTestHelpers.dropTable(chc, topic); @@ -2711,9 +2711,9 @@ public void autoEvolveSchemalessRecordsThrowError() { // Avro union(string, bytes) fields auto-evolved as Nullable(String) columns @Test public void autoEvolveAvroUnionStringBytesCreatesStringColumn() throws Exception { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = "auto_evolve_avro_union_test"; ClickHouseTestHelpers.dropTable(chc, topic); @@ -2763,9 +2763,9 @@ public void autoEvolveAvroUnionStringBytesCreatesStringColumn() throws Exception @Test public void autoEvolveMixedBatchLastRecordOlderSchema() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = createTopicName("auto_evolve_last_record_older_test"); ClickHouseTestHelpers.dropTable(chc, topic); @@ -2803,9 +2803,9 @@ public void autoEvolveMixedBatchLastRecordOlderSchema() { @Test public void autoEvolveMultiVersionUnionSemantics() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = createTopicName("auto_evolve_union_semantics_test"); ClickHouseTestHelpers.dropTable(chc, topic); @@ -2838,9 +2838,9 @@ public void autoEvolveMultiVersionUnionSemantics() { @Test public void autoEvolveInterleavedSchemaVersions() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = createTopicName("auto_evolve_non_monotonic_test"); ClickHouseTestHelpers.dropTable(chc, topic); @@ -2870,10 +2870,10 @@ public void autoEvolveInterleavedSchemaVersions() { @Test public void autoEvolveCrossPartitionSchemaDrift() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); props.put(ClickHouseSinkConfig.IGNORE_PARTITIONS_WHEN_BATCHING, "true"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = createTopicName("auto_evolve_cross_partition_test"); ClickHouseTestHelpers.dropTable(chc, topic); @@ -2904,9 +2904,9 @@ public void autoEvolveCrossPartitionSchemaDrift() { // Mixed batch where older records lack auto-evolved Array/Map columns. @Test public void autoEvolveMixedBatchArrayMapFieldsMissingInOlderRecords() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = createTopicName("auto_evolve_mixed_array_map_test"); ClickHouseTestHelpers.dropTable(chc, topic); @@ -2938,10 +2938,10 @@ public void autoEvolveMixedBatchArrayMapFieldsMissingInOlderRecords() { // Mixed batch where older records lack an auto-evolved Variant column. @Test public void autoEvolveMixedBatchVariantFieldMissingInOlderRecords() { - Map props = createProps(); + Map props = getBaseProps(); props.put(ClickHouseSinkConfig.AUTO_EVOLVE, "true"); props.put(ClickHouseSinkConfig.CLICKHOUSE_SETTINGS, "allow_experimental_variant_type=1"); - ClickHouseHelperClient chc = createClient(props); + ClickHouseHelperClient chc = ClickHouseTestHelpers.createClient(props); String topic = createTopicName("auto_evolve_mixed_variant_test"); ClickHouseTestHelpers.dropTable(chc, topic); From 714435fc103f633a9fd4266874fe3a47f74f0244 Mon Sep 17 00:00:00 2001 From: Guillermo Ovejero Date: Wed, 29 Apr 2026 23:39:23 +0200 Subject: [PATCH 62/62] test: align auto-evolve tests with last-record-only contract The auto-evolve detection was simplified in c44c2f4 to inspect only the last record's schema in a batch. Update the four mixed-schema tests that still asserted the previous full-batch-scan behavior: - autoEvolveMixedSchemasTenRecordsInOneBatch: keep V3 last and assert only V3 columns are added; explicitly assert V2-only columns are NOT. - autoEvolveMultiVersionUnionSemantics: assert only V4's unique field is added; V2/V3 unique fields are NOT. - autoEvolveMixedBatchLastRecordOlderSchema: invert to document that no ALTER is issued when the last record is the older schema. - autoEvolveCrossPartitionSchemaDrift: invert similarly for the cross-partition case. --- .../ClickHouseSinkTaskWithSchemaTest.java | 105 ++++++++++-------- 1 file changed, 56 insertions(+), 49 deletions(-) diff --git a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java index f84ab2a7b..874af162d 100644 --- a/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java +++ b/src/test/java/com/clickhouse/kafka/connect/sink/ClickHouseSinkTaskWithSchemaTest.java @@ -62,6 +62,7 @@ import java.util.stream.LongStream; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -2342,10 +2343,13 @@ public void autoEvolveMixedSchemasTenRecordsInOneBatch() { ClickHouseTestHelpers.dropTable(chc, topic); ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); - // Single batch with 10 records spanning 3 schema versions: + // Single batch with 10 records spanning 3 schema versions, ordered so the + // newest schema (V3) is LAST. Auto-evolve only inspects the last record's + // schema, so only fields present in V3 will be added (name, email, country). + // V2-only fields (age, score, active, city) will NOT be evolved. // Records 1–5: Schema V1 (3 fields: off16, p_int64, name) // Records 6–7: Schema V2 (8 fields: off16, p_int64, name, email, age, score, active, city) - // Records 8–10: Schema V3 (5 fields: off16, p_int64, name, email, country) + // Records 8–10: Schema V3 (5 fields: off16, p_int64, name, email, country) <-- LAST List batch = new ArrayList<>(); batch.addAll(SchemaTestData.createRichSchemaV1(topic, 1, 5, 0)); batch.addAll(SchemaTestData.createRichSchemaV2(topic, 1, 2, 5)); @@ -2358,21 +2362,27 @@ public void autoEvolveMixedSchemasTenRecordsInOneBatch() { assertEquals(10, ClickHouseTestHelpers.countRows(chc, topic)); - // Verify all evolved columns from all versions exist + // Only columns present in the LAST record's schema (V3) should be added. com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); - assertTrue(described.getRootColumnsMap().containsKey("name"), "V1 column 'name' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("email"), "V2 column 'email' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("age"), "V2 column 'age' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("score"), "V2 column 'score' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("active"), "V2 column 'active' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("city"), "V2 column 'city' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("name"), "V3 column 'name' should exist"); + assertTrue(described.getRootColumnsMap().containsKey("email"), "V3 column 'email' should exist"); assertTrue(described.getRootColumnsMap().containsKey("country"), "V3 column 'country' should exist"); - // Verify NULL distribution: - // name: all 10 records have it - 0 NULLs - // email: V1 records (5) lack it - 5 NULLs - // age: only V2 records (2) have it - 8 NULLs - // country: only V3 records (3) have it - 7 NULLs + // V2-only fields are NOT added because the last record (V3) does not include them. + // This documents the trade-off of last-record-only schema detection. + assertFalse(described.getRootColumnsMap().containsKey("age"), + "V2-only column 'age' should NOT be added (last record is V3)"); + assertFalse(described.getRootColumnsMap().containsKey("score"), + "V2-only column 'score' should NOT be added (last record is V3)"); + assertFalse(described.getRootColumnsMap().containsKey("active"), + "V2-only column 'active' should NOT be added (last record is V3)"); + assertFalse(described.getRootColumnsMap().containsKey("city"), + "V2-only column 'city' should NOT be added (last record is V3)"); + + // Verify NULL distribution for the columns that DO exist: + // name: all 10 records have it -> 0 NULLs + // email: V1 records (5) lack it -> 5 NULLs + // country: only V3 records (3) have it -> 7 NULLs try { String nameNullQuery = String.format( "SELECT COUNT(*) FROM `%s` WHERE `name` IS NULL SETTINGS select_sequential_consistency = 1", topic); @@ -2386,12 +2396,6 @@ public void autoEvolveMixedSchemasTenRecordsInOneBatch() { assertEquals(5, Integer.parseInt(emailNulls.iterator().next().getString(1)), "V1 records (5) should have NULL for email"); - String ageNullQuery = String.format( - "SELECT COUNT(*) FROM `%s` WHERE `age` IS NULL SETTINGS select_sequential_consistency = 1", topic); - Records ageNulls = chc.getClient().queryRecords(ageNullQuery).get(); - assertEquals(8, Integer.parseInt(ageNulls.iterator().next().getString(1)), - "V1 (5) + V3 (3) records should have NULL for age"); - String countryNullQuery = String.format( "SELECT COUNT(*) FROM `%s` WHERE `country` IS NULL SETTINGS select_sequential_consistency = 1", topic); Records countryNulls = chc.getClient().queryRecords(countryNullQuery).get(); @@ -2771,8 +2775,11 @@ public void autoEvolveMixedBatchLastRecordOlderSchema() { ClickHouseTestHelpers.dropTable(chc, topic); ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); - // Build batch where V2 records come first and V1 (older) records are last. - // Before the fix, only the last record was checked - V1 has no new fields, so ALTER TABLE was skipped. + // Build batch where V2 records come first and V1 (older) records are LAST. + // Auto-evolve only inspects the last record's schema, so V1 (which has no + // new fields) means no ALTER TABLE is issued. This test documents that + // trade-off: producers SHOULD ensure the newest schema is the last record + // in a batch when relying on auto-evolve. List batch = new ArrayList<>(); batch.addAll(SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 3)); batch.addAll(SchemaTestData.createSchemaV1(topic, 1, 2)); @@ -2782,23 +2789,14 @@ public void autoEvolveMixedBatchLastRecordOlderSchema() { chst.put(batch); chst.stop(); + // All 5 rows are still inserted (V2 extra field is dropped because the + // table doesn't have the column and input_format_skip_unknown_fields=1). assertEquals(5, ClickHouseTestHelpers.countRows(chc, topic)); - // The new column from V2 should exist even though V1 was the last record + // The new column is NOT added because the LAST record (V1) does not include it. com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); - assertTrue(described.getRootColumnsMap().containsKey("new_string_field"), - "Column 'new_string_field' should be added even when last record is V1 (older schema)"); - - // V1 records should have NULL for the new column - String nullCountQuery = String.format( - "SELECT COUNT(*) FROM `%s` WHERE `new_string_field` IS NULL SETTINGS select_sequential_consistency = 1", topic); - try { - Records nullRecords = chc.getClient().queryRecords(nullCountQuery).get(); - int nullCount = Integer.parseInt(nullRecords.iterator().next().getString(1)); - assertEquals(2, nullCount, "V1 records should have NULL for new_string_field"); - } catch (Exception e) { - throw new RuntimeException(e); - } + assertFalse(described.getRootColumnsMap().containsKey("new_string_field"), + "Column 'new_string_field' should NOT be added when the last record is V1 (older schema)"); } @Test @@ -2811,8 +2809,10 @@ public void autoEvolveMultiVersionUnionSemantics() { ClickHouseTestHelpers.dropTable(chc, topic); ClickHouseTestHelpers.runQuery(chc, String.format("CREATE TABLE %s ( `off16` Int16, `p_int64` Int64 ) Engine = MergeTree ORDER BY off16", topic)); - // Batch with V1, V2, V3, V4 - each version adds a different field. - // All new fields should be added in a single ALTER TABLE. + // Batch with V1, V2, V3, V4 in order. Auto-evolve only inspects the LAST + // record's schema (V4), which only adds `v4_float_field`. The unique + // fields from V2 (`new_string_field`) and V3 (`v3_bool_field`) are NOT + // added because they are absent from V4. List batch = new ArrayList<>(); batch.addAll(SchemaTestData.createSchemaV1(topic, 1, 2)); batch.addAll(SchemaTestData.createSchemaV2WithNewNullableField(topic, 1, 2)); @@ -2826,14 +2826,17 @@ public void autoEvolveMultiVersionUnionSemantics() { assertEquals(8, ClickHouseTestHelpers.countRows(chc, topic)); - // Verify all fields from V2, V3, V4 exist + // Only V4's unique field is added (V4 is the last record's schema). com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); - assertTrue(described.getRootColumnsMap().containsKey("new_string_field"), - "V2 column 'new_string_field' should exist"); - assertTrue(described.getRootColumnsMap().containsKey("v3_bool_field"), - "V3 column 'v3_bool_field' should exist"); assertTrue(described.getRootColumnsMap().containsKey("v4_float_field"), - "V4 column 'v4_float_field' should exist"); + "V4 column 'v4_float_field' should exist (V4 is last record)"); + + // V2 and V3 unique fields are NOT added because the last record (V4) does not include them. + // This documents the trade-off of last-record-only schema detection. + assertFalse(described.getRootColumnsMap().containsKey("new_string_field"), + "V2-only column 'new_string_field' should NOT be added (last record is V4)"); + assertFalse(described.getRootColumnsMap().containsKey("v3_bool_field"), + "V3-only column 'v3_bool_field' should NOT be added (last record is V4)"); } @Test @@ -2882,7 +2885,11 @@ public void autoEvolveCrossPartitionSchemaDrift() { // Records from different partitions with different schema versions. // With ignorePartitionsWhenBatching=true, they are merged into a single batch. // Partition 0: V2 records (has new_string_field) - // Partition 1: V1 records (no new_string_field) - these may end up last + // Partition 1: V1 records (no new_string_field) - these are last in the merged batch + // Auto-evolve only inspects the last record's schema (V1), so the new + // column is NOT added. This documents the trade-off when schema drifts + // across partitions: the producer of the last partition's records + // determines which schema is used for evolution. List batch = new ArrayList<>(); batch.addAll(SchemaTestData.createSchemaV2WithNewNullableField(topic, 0, 3)); batch.addAll(SchemaTestData.createSchemaV1(topic, 1, 3)); @@ -2894,11 +2901,11 @@ public void autoEvolveCrossPartitionSchemaDrift() { assertEquals(6, ClickHouseTestHelpers.countRows(chc, topic)); - // The new column from V2 (partition 0) should exist even though - // V1 records from partition 1 may be last in the merged batch + // The new column from V2 is NOT added because the last record in the + // merged cross-partition batch is V1 (older schema). com.clickhouse.kafka.connect.sink.db.mapping.Table described = chc.describeTable(chc.getDatabase(), topic); - assertTrue(described.getRootColumnsMap().containsKey("new_string_field"), - "Column 'new_string_field' should be added with cross-partition schema drift"); + assertFalse(described.getRootColumnsMap().containsKey("new_string_field"), + "Column 'new_string_field' should NOT be added when last record across partitions is V1"); } // Mixed batch where older records lack auto-evolved Array/Map columns.