> partitionTransforms,
+ int formatVersion) {
return IcebergUtil.loadIcebergTable(icebergCatalog, tableId).orElseGet(() -> {
try {
// no need to check if the table already exists, because the table is created by the thread that calls the get_or_create_table method
- return IcebergUtil.createIcebergTable(icebergCatalog, tableId, schema, "parquet", partitionTransforms);
+ return IcebergUtil.createIcebergTable(icebergCatalog, tableId, schema, "parquet", partitionTransforms, formatVersion);
} catch (Exception e) {
String errorMessage = String.format("Failed to create table from debezium event schema: %s Error: %s",
tableId, e.getMessage());
diff --git a/destination/iceberg/olake-iceberg-java-writer/src/main/java/io/debezium/server/iceberg/rpc/RecordIngest.java b/destination/iceberg/olake-iceberg-java-writer/src/main/java/io/debezium/server/iceberg/rpc/RecordIngest.java
index 54eaf98a0..3788eca7b 100644
--- a/destination/iceberg/olake-iceberg-java-writer/src/main/java/io/debezium/server/iceberg/rpc/RecordIngest.java
+++ b/destination/iceberg/olake-iceberg-java-writer/src/main/java/io/debezium/server/iceberg/rpc/RecordIngest.java
@@ -92,11 +92,6 @@ protected java.lang.Object newInstance(
return new IcebergPayload();
}
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return io.debezium.server.iceberg.rpc.RecordIngest.internal_static_io_debezium_server_iceberg_rpc_IcebergPayload_descriptor;
@@ -405,6 +400,30 @@ io.debezium.server.iceberg.rpc.RecordIngest.IcebergPayload.SchemaFieldOrBuilder
io.debezium.server.iceberg.rpc.RecordIngest.IcebergPayload.PartitionFieldOrBuilder getPartitionFieldsOrBuilder(
int index);
+ /**
+ *
+ * Delete representation the writer should use: "eq" (equality deletes),
+ * "pos" (positional delete files) or "dv" (v3 deletion vectors). Empty falls
+ * back to use_positional_deletes so older callers keep working.
+ *
+ *
+ * string delete_mode = 12;
+ * @return The deleteMode.
+ */
+ java.lang.String getDeleteMode();
+ /**
+ *
+ * Delete representation the writer should use: "eq" (equality deletes),
+ * "pos" (positional delete files) or "dv" (v3 deletion vectors). Empty falls
+ * back to use_positional_deletes so older callers keep working.
+ *
+ *
+ * string delete_mode = 12;
+ * @return The bytes for deleteMode.
+ */
+ com.google.protobuf.ByteString
+ getDeleteModeBytes();
+
/**
*
* COMMIT: snapshot the caller's row index is checkpointed at. The server
@@ -448,6 +467,7 @@ private Metadata() {
payload_ = "";
namespace_ = "";
partitionFields_ = java.util.Collections.emptyList();
+ deleteMode_ = "";
}
@java.lang.Override
@@ -457,11 +477,6 @@ protected java.lang.Object newInstance(
return new Metadata();
}
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return io.debezium.server.iceberg.rpc.RecordIngest.internal_static_io_debezium_server_iceberg_rpc_IcebergPayload_Metadata_descriptor;
@@ -796,6 +811,57 @@ public io.debezium.server.iceberg.rpc.RecordIngest.IcebergPayload.PartitionField
return partitionFields_.get(index);
}
+ public static final int DELETE_MODE_FIELD_NUMBER = 12;
+ @SuppressWarnings("serial")
+ private volatile java.lang.Object deleteMode_ = "";
+ /**
+ *
+ * Delete representation the writer should use: "eq" (equality deletes),
+ * "pos" (positional delete files) or "dv" (v3 deletion vectors). Empty falls
+ * back to use_positional_deletes so older callers keep working.
+ *
+ *
+ * string delete_mode = 12;
+ * @return The deleteMode.
+ */
+ @java.lang.Override
+ public java.lang.String getDeleteMode() {
+ java.lang.Object ref = deleteMode_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ deleteMode_ = s;
+ return s;
+ }
+ }
+ /**
+ *
+ * Delete representation the writer should use: "eq" (equality deletes),
+ * "pos" (positional delete files) or "dv" (v3 deletion vectors). Empty falls
+ * back to use_positional_deletes so older callers keep working.
+ *
+ *
+ * string delete_mode = 12;
+ * @return The bytes for deleteMode.
+ */
+ @java.lang.Override
+ public com.google.protobuf.ByteString
+ getDeleteModeBytes() {
+ java.lang.Object ref = deleteMode_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ deleteMode_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
public static final int BASE_SNAPSHOT_ID_FIELD_NUMBER = 11;
private long baseSnapshotId_ = 0L;
/**
@@ -871,6 +937,9 @@ public void writeTo(com.google.protobuf.CodedOutputStream output)
if (((bitField0_ & 0x00000002) != 0)) {
output.writeInt64(11, baseSnapshotId_);
}
+ if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(deleteMode_)) {
+ com.google.protobuf.GeneratedMessageV3.writeString(output, 12, deleteMode_);
+ }
getUnknownFields().writeTo(output);
}
@@ -915,6 +984,9 @@ public int getSerializedSize() {
size += com.google.protobuf.CodedOutputStream
.computeInt64Size(11, baseSnapshotId_);
}
+ if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(deleteMode_)) {
+ size += com.google.protobuf.GeneratedMessageV3.computeStringSize(12, deleteMode_);
+ }
size += getUnknownFields().getSerializedSize();
memoizedSize = size;
return size;
@@ -951,6 +1023,8 @@ public boolean equals(final java.lang.Object obj) {
!= other.getUsePositionalDeletes()) return false;
if (!getPartitionFieldsList()
.equals(other.getPartitionFieldsList())) return false;
+ if (!getDeleteMode()
+ .equals(other.getDeleteMode())) return false;
if (hasBaseSnapshotId() != other.hasBaseSnapshotId()) return false;
if (hasBaseSnapshotId()) {
if (getBaseSnapshotId()
@@ -993,6 +1067,8 @@ public int hashCode() {
hash = (37 * hash) + PARTITION_FIELDS_FIELD_NUMBER;
hash = (53 * hash) + getPartitionFieldsList().hashCode();
}
+ hash = (37 * hash) + DELETE_MODE_FIELD_NUMBER;
+ hash = (53 * hash) + getDeleteMode().hashCode();
if (hasBaseSnapshotId()) {
hash = (37 * hash) + BASE_SNAPSHOT_ID_FIELD_NUMBER;
hash = (53 * hash) + com.google.protobuf.Internal.hashLong(
@@ -1047,11 +1123,13 @@ public static io.debezium.server.iceberg.rpc.RecordIngest.IcebergPayload.Metadat
return com.google.protobuf.GeneratedMessageV3
.parseWithIOException(PARSER, input, extensionRegistry);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.IcebergPayload.Metadata parseDelimitedFrom(java.io.InputStream input)
throws java.io.IOException {
return com.google.protobuf.GeneratedMessageV3
.parseDelimitedWithIOException(PARSER, input);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.IcebergPayload.Metadata parseDelimitedFrom(
java.io.InputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -1148,6 +1226,7 @@ public Builder clear() {
partitionFieldsBuilder_.clear();
}
bitField0_ = (bitField0_ & ~0x00000100);
+ deleteMode_ = "";
baseSnapshotId_ = 0L;
return this;
}
@@ -1228,6 +1307,9 @@ private void buildPartial0(io.debezium.server.iceberg.rpc.RecordIngest.IcebergPa
result.usePositionalDeletes_ = usePositionalDeletes_;
}
if (((from_bitField0_ & 0x00000200) != 0)) {
+ result.deleteMode_ = deleteMode_;
+ }
+ if (((from_bitField0_ & 0x00000400) != 0)) {
result.baseSnapshotId_ = baseSnapshotId_;
to_bitField0_ |= 0x00000002;
}
@@ -1361,6 +1443,11 @@ public Builder mergeFrom(io.debezium.server.iceberg.rpc.RecordIngest.IcebergPayl
}
}
}
+ if (!other.getDeleteMode().isEmpty()) {
+ deleteMode_ = other.deleteMode_;
+ bitField0_ |= 0x00000200;
+ onChanged();
+ }
if (other.hasBaseSnapshotId()) {
setBaseSnapshotId(other.getBaseSnapshotId());
}
@@ -1453,9 +1540,14 @@ public Builder mergeFrom(
} // case 82
case 88: {
baseSnapshotId_ = input.readInt64();
- bitField0_ |= 0x00000200;
+ bitField0_ |= 0x00000400;
break;
} // case 88
+ case 98: {
+ deleteMode_ = input.readStringRequireUtf8();
+ bitField0_ |= 0x00000200;
+ break;
+ } // case 98
default: {
if (!super.parseUnknownField(input, extensionRegistry, tag)) {
done = true; // was an endgroup tag
@@ -2115,7 +2207,7 @@ public boolean getUpsert() {
* @return This builder for chaining.
*/
public Builder setUpsert(boolean value) {
-
+
upsert_ = value;
bitField0_ |= 0x00000040;
onChanged();
@@ -2157,7 +2249,7 @@ public boolean getUsePositionalDeletes() {
* @return This builder for chaining.
*/
public Builder setUsePositionalDeletes(boolean value) {
-
+
usePositionalDeletes_ = value;
bitField0_ |= 0x00000080;
onChanged();
@@ -2419,6 +2511,108 @@ public io.debezium.server.iceberg.rpc.RecordIngest.IcebergPayload.PartitionField
return partitionFieldsBuilder_;
}
+ private java.lang.Object deleteMode_ = "";
+ /**
+ *
+ * Delete representation the writer should use: "eq" (equality deletes),
+ * "pos" (positional delete files) or "dv" (v3 deletion vectors). Empty falls
+ * back to use_positional_deletes so older callers keep working.
+ *
+ *
+ * string delete_mode = 12;
+ * @return The deleteMode.
+ */
+ public java.lang.String getDeleteMode() {
+ java.lang.Object ref = deleteMode_;
+ if (!(ref instanceof java.lang.String)) {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ deleteMode_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ *
+ * Delete representation the writer should use: "eq" (equality deletes),
+ * "pos" (positional delete files) or "dv" (v3 deletion vectors). Empty falls
+ * back to use_positional_deletes so older callers keep working.
+ *
+ *
+ * string delete_mode = 12;
+ * @return The bytes for deleteMode.
+ */
+ public com.google.protobuf.ByteString
+ getDeleteModeBytes() {
+ java.lang.Object ref = deleteMode_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ deleteMode_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ *
+ * Delete representation the writer should use: "eq" (equality deletes),
+ * "pos" (positional delete files) or "dv" (v3 deletion vectors). Empty falls
+ * back to use_positional_deletes so older callers keep working.
+ *
+ *
+ * string delete_mode = 12;
+ * @param value The deleteMode to set.
+ * @return This builder for chaining.
+ */
+ public Builder setDeleteMode(
+ java.lang.String value) {
+ if (value == null) { throw new NullPointerException(); }
+ deleteMode_ = value;
+ bitField0_ |= 0x00000200;
+ onChanged();
+ return this;
+ }
+ /**
+ *
+ * Delete representation the writer should use: "eq" (equality deletes),
+ * "pos" (positional delete files) or "dv" (v3 deletion vectors). Empty falls
+ * back to use_positional_deletes so older callers keep working.
+ *
+ *
+ * string delete_mode = 12;
+ * @return This builder for chaining.
+ */
+ public Builder clearDeleteMode() {
+ deleteMode_ = getDefaultInstance().getDeleteMode();
+ bitField0_ = (bitField0_ & ~0x00000200);
+ onChanged();
+ return this;
+ }
+ /**
+ *
+ * Delete representation the writer should use: "eq" (equality deletes),
+ * "pos" (positional delete files) or "dv" (v3 deletion vectors). Empty falls
+ * back to use_positional_deletes so older callers keep working.
+ *
+ *
+ * string delete_mode = 12;
+ * @param value The bytes for deleteMode to set.
+ * @return This builder for chaining.
+ */
+ public Builder setDeleteModeBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) { throw new NullPointerException(); }
+ checkByteStringIsUtf8(value);
+ deleteMode_ = value;
+ bitField0_ |= 0x00000200;
+ onChanged();
+ return this;
+ }
+
private long baseSnapshotId_ ;
/**
*
@@ -2432,7 +2626,7 @@ public io.debezium.server.iceberg.rpc.RecordIngest.IcebergPayload.PartitionField
*/
@java.lang.Override
public boolean hasBaseSnapshotId() {
- return ((bitField0_ & 0x00000200) != 0);
+ return ((bitField0_ & 0x00000400) != 0);
}
/**
*
@@ -2460,9 +2654,9 @@ public long getBaseSnapshotId() {
* @return This builder for chaining.
*/
public Builder setBaseSnapshotId(long value) {
-
+
baseSnapshotId_ = value;
- bitField0_ |= 0x00000200;
+ bitField0_ |= 0x00000400;
onChanged();
return this;
}
@@ -2477,7 +2671,7 @@ public Builder setBaseSnapshotId(long value) {
* @return This builder for chaining.
*/
public Builder clearBaseSnapshotId() {
- bitField0_ = (bitField0_ & ~0x00000200);
+ bitField0_ = (bitField0_ & ~0x00000400);
baseSnapshotId_ = 0L;
onChanged();
return this;
@@ -2598,11 +2792,6 @@ protected java.lang.Object newInstance(
return new SchemaField();
}
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return io.debezium.server.iceberg.rpc.RecordIngest.internal_static_io_debezium_server_iceberg_rpc_IcebergPayload_SchemaField_descriptor;
@@ -2812,11 +3001,13 @@ public static io.debezium.server.iceberg.rpc.RecordIngest.IcebergPayload.SchemaF
return com.google.protobuf.GeneratedMessageV3
.parseWithIOException(PARSER, input, extensionRegistry);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.IcebergPayload.SchemaField parseDelimitedFrom(java.io.InputStream input)
throws java.io.IOException {
return com.google.protobuf.GeneratedMessageV3
.parseDelimitedWithIOException(PARSER, input);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.IcebergPayload.SchemaField parseDelimitedFrom(
java.io.InputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -3301,11 +3492,6 @@ protected java.lang.Object newInstance(
return new PartitionField();
}
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return io.debezium.server.iceberg.rpc.RecordIngest.internal_static_io_debezium_server_iceberg_rpc_IcebergPayload_PartitionField_descriptor;
@@ -3515,11 +3701,13 @@ public static io.debezium.server.iceberg.rpc.RecordIngest.IcebergPayload.Partiti
return com.google.protobuf.GeneratedMessageV3
.parseWithIOException(PARSER, input, extensionRegistry);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.IcebergPayload.PartitionField parseDelimitedFrom(java.io.InputStream input)
throws java.io.IOException {
return com.google.protobuf.GeneratedMessageV3
.parseDelimitedWithIOException(PARSER, input);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.IcebergPayload.PartitionField parseDelimitedFrom(
java.io.InputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -4075,11 +4263,6 @@ protected java.lang.Object newInstance(
return new IceRecord();
}
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return io.debezium.server.iceberg.rpc.RecordIngest.internal_static_io_debezium_server_iceberg_rpc_IcebergPayload_IceRecord_descriptor;
@@ -4180,7 +4363,7 @@ public interface FieldValueOrBuilder extends
*/
com.google.protobuf.ByteString getBytesValue();
- public io.debezium.server.iceberg.rpc.RecordIngest.IcebergPayload.IceRecord.FieldValue.ValueCase getValueCase();
+ io.debezium.server.iceberg.rpc.RecordIngest.IcebergPayload.IceRecord.FieldValue.ValueCase getValueCase();
}
/**
*
@@ -4208,11 +4391,6 @@ protected java.lang.Object newInstance(
return new FieldValue();
}
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return io.debezium.server.iceberg.rpc.RecordIngest.internal_static_io_debezium_server_iceberg_rpc_IcebergPayload_IceRecord_FieldValue_descriptor;
@@ -4227,6 +4405,7 @@ protected java.lang.Object newInstance(
}
private int valueCase_ = 0;
+ @SuppressWarnings("serial")
private java.lang.Object value_;
public enum ValueCase
implements com.google.protobuf.Internal.EnumLite,
@@ -4684,11 +4863,13 @@ public static io.debezium.server.iceberg.rpc.RecordIngest.IcebergPayload.IceReco
return com.google.protobuf.GeneratedMessageV3
.parseWithIOException(PARSER, input, extensionRegistry);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.IcebergPayload.IceRecord.FieldValue parseDelimitedFrom(java.io.InputStream input)
throws java.io.IOException {
return com.google.protobuf.GeneratedMessageV3
.parseDelimitedWithIOException(PARSER, input);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.IcebergPayload.IceRecord.FieldValue parseDelimitedFrom(
java.io.InputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -5100,7 +5281,7 @@ public int getIntValue() {
* @return This builder for chaining.
*/
public Builder setIntValue(int value) {
-
+
valueCase_ = 2;
value_ = value;
onChanged();
@@ -5142,7 +5323,7 @@ public long getLongValue() {
* @return This builder for chaining.
*/
public Builder setLongValue(long value) {
-
+
valueCase_ = 3;
value_ = value;
onChanged();
@@ -5184,7 +5365,7 @@ public float getFloatValue() {
* @return This builder for chaining.
*/
public Builder setFloatValue(float value) {
-
+
valueCase_ = 4;
value_ = value;
onChanged();
@@ -5226,7 +5407,7 @@ public double getDoubleValue() {
* @return This builder for chaining.
*/
public Builder setDoubleValue(double value) {
-
+
valueCase_ = 5;
value_ = value;
onChanged();
@@ -5268,7 +5449,7 @@ public boolean getBoolValue() {
* @return This builder for chaining.
*/
public Builder setBoolValue(boolean value) {
-
+
valueCase_ = 6;
value_ = value;
onChanged();
@@ -5718,11 +5899,13 @@ public static io.debezium.server.iceberg.rpc.RecordIngest.IcebergPayload.IceReco
return com.google.protobuf.GeneratedMessageV3
.parseWithIOException(PARSER, input, extensionRegistry);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.IcebergPayload.IceRecord parseDelimitedFrom(java.io.InputStream input)
throws java.io.IOException {
return com.google.protobuf.GeneratedMessageV3
.parseDelimitedWithIOException(PARSER, input);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.IcebergPayload.IceRecord parseDelimitedFrom(
java.io.InputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -6497,7 +6680,7 @@ public long getDeletePosition() {
* @return This builder for chaining.
*/
public Builder setDeletePosition(long value) {
-
+
deletePosition_ = value;
bitField0_ |= 0x00000008;
onChanged();
@@ -6799,11 +6982,13 @@ public static io.debezium.server.iceberg.rpc.RecordIngest.IcebergPayload parseFr
return com.google.protobuf.GeneratedMessageV3
.parseWithIOException(PARSER, input, extensionRegistry);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.IcebergPayload parseDelimitedFrom(java.io.InputStream input)
throws java.io.IOException {
return com.google.protobuf.GeneratedMessageV3
.parseDelimitedWithIOException(PARSER, input);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.IcebergPayload parseDelimitedFrom(
java.io.InputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -7710,11 +7895,6 @@ protected java.lang.Object newInstance(
return new RecordIngestResponse();
}
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return io.debezium.server.iceberg.rpc.RecordIngest.internal_static_io_debezium_server_iceberg_rpc_RecordIngestResponse_descriptor;
@@ -8095,11 +8275,13 @@ public static io.debezium.server.iceberg.rpc.RecordIngest.RecordIngestResponse p
return com.google.protobuf.GeneratedMessageV3
.parseWithIOException(PARSER, input, extensionRegistry);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.RecordIngestResponse parseDelimitedFrom(java.io.InputStream input)
throws java.io.IOException {
return com.google.protobuf.GeneratedMessageV3
.parseDelimitedWithIOException(PARSER, input);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.RecordIngestResponse parseDelimitedFrom(
java.io.InputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -8515,7 +8697,7 @@ public boolean getSuccess() {
* @return This builder for chaining.
*/
public Builder setSuccess(boolean value) {
-
+
success_ = value;
bitField0_ |= 0x00000002;
onChanged();
@@ -8655,7 +8837,7 @@ public long getSnapshotId() {
* @return This builder for chaining.
*/
public Builder setSnapshotId(long value) {
-
+
snapshotId_ = value;
bitField0_ |= 0x00000008;
onChanged();
@@ -8703,7 +8885,7 @@ public boolean getHasEqualityDeletes() {
* @return This builder for chaining.
*/
public Builder setHasEqualityDeletes(boolean value) {
-
+
hasEqualityDeletes_ = value;
bitField0_ |= 0x00000010;
onChanged();
@@ -9181,11 +9363,6 @@ protected java.lang.Object newInstance(
return new WriteRun();
}
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return io.debezium.server.iceberg.rpc.RecordIngest.internal_static_io_debezium_server_iceberg_rpc_WriteRun_descriptor;
@@ -9413,11 +9590,13 @@ public static io.debezium.server.iceberg.rpc.RecordIngest.WriteRun parseFrom(
return com.google.protobuf.GeneratedMessageV3
.parseWithIOException(PARSER, input, extensionRegistry);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.WriteRun parseDelimitedFrom(java.io.InputStream input)
throws java.io.IOException {
return com.google.protobuf.GeneratedMessageV3
.parseDelimitedWithIOException(PARSER, input);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.WriteRun parseDelimitedFrom(
java.io.InputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -9758,7 +9937,7 @@ public int getBatchStartIdx() {
* @return This builder for chaining.
*/
public Builder setBatchStartIdx(int value) {
-
+
batchStartIdx_ = value;
bitField0_ |= 0x00000002;
onChanged();
@@ -9790,7 +9969,7 @@ public long getStartPosition() {
* @return This builder for chaining.
*/
public Builder setStartPosition(long value) {
-
+
startPosition_ = value;
bitField0_ |= 0x00000004;
onChanged();
@@ -9822,7 +10001,7 @@ public int getCount() {
* @return This builder for chaining.
*/
public Builder setCount(int value) {
-
+
count_ = value;
bitField0_ |= 0x00000008;
onChanged();
@@ -9955,11 +10134,6 @@ protected java.lang.Object newInstance(
return new ArrowPayload();
}
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return io.debezium.server.iceberg.rpc.RecordIngest.internal_static_io_debezium_server_iceberg_rpc_ArrowPayload_descriptor;
@@ -10182,11 +10356,6 @@ protected java.lang.Object newInstance(
return new FileMetadata();
}
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return io.debezium.server.iceberg.rpc.RecordIngest.internal_static_io_debezium_server_iceberg_rpc_ArrowPayload_FileMetadata_descriptor;
@@ -10276,7 +10445,7 @@ public interface PartitionValueOrBuilder extends
*/
boolean getBoolValue();
- public io.debezium.server.iceberg.rpc.RecordIngest.ArrowPayload.FileMetadata.PartitionValue.ValueCase getValueCase();
+ io.debezium.server.iceberg.rpc.RecordIngest.ArrowPayload.FileMetadata.PartitionValue.ValueCase getValueCase();
}
/**
* Protobuf type {@code io.debezium.server.iceberg.rpc.ArrowPayload.FileMetadata.PartitionValue}
@@ -10300,11 +10469,6 @@ protected java.lang.Object newInstance(
return new PartitionValue();
}
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return io.debezium.server.iceberg.rpc.RecordIngest.internal_static_io_debezium_server_iceberg_rpc_ArrowPayload_FileMetadata_PartitionValue_descriptor;
@@ -10319,6 +10483,7 @@ protected java.lang.Object newInstance(
}
private int valueCase_ = 0;
+ @SuppressWarnings("serial")
private java.lang.Object value_;
public enum ValueCase
implements com.google.protobuf.Internal.EnumLite,
@@ -10736,11 +10901,13 @@ public static io.debezium.server.iceberg.rpc.RecordIngest.ArrowPayload.FileMetad
return com.google.protobuf.GeneratedMessageV3
.parseWithIOException(PARSER, input, extensionRegistry);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.ArrowPayload.FileMetadata.PartitionValue parseDelimitedFrom(java.io.InputStream input)
throws java.io.IOException {
return com.google.protobuf.GeneratedMessageV3
.parseDelimitedWithIOException(PARSER, input);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.ArrowPayload.FileMetadata.PartitionValue parseDelimitedFrom(
java.io.InputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -11046,7 +11213,7 @@ public int getIntValue() {
* @return This builder for chaining.
*/
public Builder setIntValue(int value) {
-
+
valueCase_ = 1;
value_ = value;
onChanged();
@@ -11088,7 +11255,7 @@ public long getLongValue() {
* @return This builder for chaining.
*/
public Builder setLongValue(long value) {
-
+
valueCase_ = 2;
value_ = value;
onChanged();
@@ -11223,7 +11390,7 @@ public float getFloatValue() {
* @return This builder for chaining.
*/
public Builder setFloatValue(float value) {
-
+
valueCase_ = 4;
value_ = value;
onChanged();
@@ -11265,7 +11432,7 @@ public double getDoubleValue() {
* @return This builder for chaining.
*/
public Builder setDoubleValue(double value) {
-
+
valueCase_ = 5;
value_ = value;
onChanged();
@@ -11307,7 +11474,7 @@ public boolean getBoolValue() {
* @return This builder for chaining.
*/
public Builder setBoolValue(boolean value) {
-
+
valueCase_ = 6;
value_ = value;
onChanged();
@@ -11662,11 +11829,13 @@ public static io.debezium.server.iceberg.rpc.RecordIngest.ArrowPayload.FileMetad
return com.google.protobuf.GeneratedMessageV3
.parseWithIOException(PARSER, input, extensionRegistry);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.ArrowPayload.FileMetadata parseDelimitedFrom(java.io.InputStream input)
throws java.io.IOException {
return com.google.protobuf.GeneratedMessageV3
.parseDelimitedWithIOException(PARSER, input);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.ArrowPayload.FileMetadata parseDelimitedFrom(
java.io.InputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -12122,7 +12291,7 @@ public long getRecordCount() {
* @return This builder for chaining.
*/
public Builder setRecordCount(long value) {
-
+
recordCount_ = value;
bitField0_ |= 0x00000004;
onChanged();
@@ -12488,11 +12657,6 @@ protected java.lang.Object newInstance(
return new FileUploadRequest();
}
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return io.debezium.server.iceberg.rpc.RecordIngest.internal_static_io_debezium_server_iceberg_rpc_ArrowPayload_FileUploadRequest_descriptor;
@@ -12675,11 +12839,13 @@ public static io.debezium.server.iceberg.rpc.RecordIngest.ArrowPayload.FileUploa
return com.google.protobuf.GeneratedMessageV3
.parseWithIOException(PARSER, input, extensionRegistry);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.ArrowPayload.FileUploadRequest parseDelimitedFrom(java.io.InputStream input)
throws java.io.IOException {
return com.google.protobuf.GeneratedMessageV3
.parseDelimitedWithIOException(PARSER, input);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.ArrowPayload.FileUploadRequest parseDelimitedFrom(
java.io.InputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -13196,11 +13362,6 @@ protected java.lang.Object newInstance(
return new Metadata();
}
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return io.debezium.server.iceberg.rpc.RecordIngest.internal_static_io_debezium_server_iceberg_rpc_ArrowPayload_Metadata_descriptor;
@@ -13602,11 +13763,13 @@ public static io.debezium.server.iceberg.rpc.RecordIngest.ArrowPayload.Metadata
return com.google.protobuf.GeneratedMessageV3
.parseWithIOException(PARSER, input, extensionRegistry);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.ArrowPayload.Metadata parseDelimitedFrom(java.io.InputStream input)
throws java.io.IOException {
return com.google.protobuf.GeneratedMessageV3
.parseDelimitedWithIOException(PARSER, input);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.ArrowPayload.Metadata parseDelimitedFrom(
java.io.InputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -14561,7 +14724,7 @@ public long getBaseSnapshotId() {
* @return This builder for chaining.
*/
public Builder setBaseSnapshotId(long value) {
-
+
baseSnapshotId_ = value;
bitField0_ |= 0x00000020;
onChanged();
@@ -14814,11 +14977,13 @@ public static io.debezium.server.iceberg.rpc.RecordIngest.ArrowPayload parseFrom
return com.google.protobuf.GeneratedMessageV3
.parseWithIOException(PARSER, input, extensionRegistry);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.ArrowPayload parseDelimitedFrom(java.io.InputStream input)
throws java.io.IOException {
return com.google.protobuf.GeneratedMessageV3
.parseDelimitedWithIOException(PARSER, input);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.ArrowPayload parseDelimitedFrom(
java.io.InputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -15377,11 +15542,6 @@ protected java.lang.Object newInstance(
return new ArrowIngestResponse();
}
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return io.debezium.server.iceberg.rpc.RecordIngest.internal_static_io_debezium_server_iceberg_rpc_ArrowIngestResponse_descriptor;
@@ -15702,11 +15862,13 @@ public static io.debezium.server.iceberg.rpc.RecordIngest.ArrowIngestResponse pa
return com.google.protobuf.GeneratedMessageV3
.parseWithIOException(PARSER, input, extensionRegistry);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.ArrowIngestResponse parseDelimitedFrom(java.io.InputStream input)
throws java.io.IOException {
return com.google.protobuf.GeneratedMessageV3
.parseDelimitedWithIOException(PARSER, input);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.ArrowIngestResponse parseDelimitedFrom(
java.io.InputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -16209,7 +16371,7 @@ public long getSnapshotId() {
* @return This builder for chaining.
*/
public Builder setSnapshotId(long value) {
-
+
snapshotId_ = value;
bitField0_ |= 0x00000004;
onChanged();
@@ -16358,11 +16520,6 @@ protected java.lang.Object newInstance(
return new RowIndexScanRequest();
}
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return io.debezium.server.iceberg.rpc.RecordIngest.internal_static_io_debezium_server_iceberg_rpc_RowIndexScanRequest_descriptor;
@@ -16574,11 +16731,13 @@ public static io.debezium.server.iceberg.rpc.RecordIngest.RowIndexScanRequest pa
return com.google.protobuf.GeneratedMessageV3
.parseWithIOException(PARSER, input, extensionRegistry);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.RowIndexScanRequest parseDelimitedFrom(java.io.InputStream input)
throws java.io.IOException {
return com.google.protobuf.GeneratedMessageV3
.parseDelimitedWithIOException(PARSER, input);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.RowIndexScanRequest parseDelimitedFrom(
java.io.InputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -16921,7 +17080,7 @@ public long getFromSnapshotId() {
* @return This builder for chaining.
*/
public Builder setFromSnapshotId(long value) {
-
+
fromSnapshotId_ = value;
bitField0_ |= 0x00000002;
onChanged();
@@ -17081,11 +17240,6 @@ protected java.lang.Object newInstance(
return new RowIndexScanBatch();
}
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return io.debezium.server.iceberg.rpc.RecordIngest.internal_static_io_debezium_server_iceberg_rpc_RowIndexScanBatch_descriptor;
@@ -17163,11 +17317,6 @@ protected java.lang.Object newInstance(
return new Entry();
}
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return io.debezium.server.iceberg.rpc.RecordIngest.internal_static_io_debezium_server_iceberg_rpc_RowIndexScanBatch_Entry_descriptor;
@@ -17423,11 +17572,13 @@ public static io.debezium.server.iceberg.rpc.RecordIngest.RowIndexScanBatch.Entr
return com.google.protobuf.GeneratedMessageV3
.parseWithIOException(PARSER, input, extensionRegistry);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.RowIndexScanBatch.Entry parseDelimitedFrom(java.io.InputStream input)
throws java.io.IOException {
return com.google.protobuf.GeneratedMessageV3
.parseDelimitedWithIOException(PARSER, input);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.RowIndexScanBatch.Entry parseDelimitedFrom(
java.io.InputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -17836,7 +17987,7 @@ public long getPosition() {
* @return This builder for chaining.
*/
public Builder setPosition(long value) {
-
+
position_ = value;
bitField0_ |= 0x00000004;
onChanged();
@@ -17868,7 +18019,7 @@ public boolean getDeleted() {
* @return This builder for chaining.
*/
public Builder setDeleted(boolean value) {
-
+
deleted_ = value;
bitField0_ |= 0x00000008;
onChanged();
@@ -18156,11 +18307,13 @@ public static io.debezium.server.iceberg.rpc.RecordIngest.RowIndexScanBatch pars
return com.google.protobuf.GeneratedMessageV3
.parseWithIOException(PARSER, input, extensionRegistry);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.RowIndexScanBatch parseDelimitedFrom(java.io.InputStream input)
throws java.io.IOException {
return com.google.protobuf.GeneratedMessageV3
.parseDelimitedWithIOException(PARSER, input);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.RowIndexScanBatch parseDelimitedFrom(
java.io.InputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -18706,7 +18859,7 @@ public long getSnapshotId() {
* @return This builder for chaining.
*/
public Builder setSnapshotId(long value) {
-
+
snapshotId_ = value;
bitField0_ |= 0x00000002;
onChanged();
@@ -18753,7 +18906,7 @@ public boolean getRequiresFullScan() {
* @return This builder for chaining.
*/
public Builder setRequiresFullScan(boolean value) {
-
+
requiresFullScan_ = value;
bitField0_ |= 0x00000004;
onChanged();
@@ -18853,6 +19006,28 @@ public interface MigrateEqualityDeletesRequestOrBuilder extends
*/
com.google.protobuf.ByteString
getThreadIdBytes();
+
+ /**
+ *
+ * Representation the equality deletes should be rewritten into: "pos" or "dv".
+ * Empty means "pos", which is what callers predating deletion vectors expect.
+ *
+ *
+ * string target_mode = 2;
+ * @return The targetMode.
+ */
+ java.lang.String getTargetMode();
+ /**
+ *
+ * Representation the equality deletes should be rewritten into: "pos" or "dv".
+ * Empty means "pos", which is what callers predating deletion vectors expect.
+ *
+ *
+ * string target_mode = 2;
+ * @return The bytes for targetMode.
+ */
+ com.google.protobuf.ByteString
+ getTargetModeBytes();
}
/**
* Protobuf type {@code io.debezium.server.iceberg.rpc.MigrateEqualityDeletesRequest}
@@ -18868,6 +19043,7 @@ private MigrateEqualityDeletesRequest(com.google.protobuf.GeneratedMessageV3.Bui
}
private MigrateEqualityDeletesRequest() {
threadId_ = "";
+ targetMode_ = "";
}
@java.lang.Override
@@ -18877,11 +19053,6 @@ protected java.lang.Object newInstance(
return new MigrateEqualityDeletesRequest();
}
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return io.debezium.server.iceberg.rpc.RecordIngest.internal_static_io_debezium_server_iceberg_rpc_MigrateEqualityDeletesRequest_descriptor;
@@ -18934,6 +19105,55 @@ public java.lang.String getThreadId() {
}
}
+ public static final int TARGET_MODE_FIELD_NUMBER = 2;
+ @SuppressWarnings("serial")
+ private volatile java.lang.Object targetMode_ = "";
+ /**
+ *
+ * Representation the equality deletes should be rewritten into: "pos" or "dv".
+ * Empty means "pos", which is what callers predating deletion vectors expect.
+ *
+ *
+ * string target_mode = 2;
+ * @return The targetMode.
+ */
+ @java.lang.Override
+ public java.lang.String getTargetMode() {
+ java.lang.Object ref = targetMode_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ targetMode_ = s;
+ return s;
+ }
+ }
+ /**
+ *
+ * Representation the equality deletes should be rewritten into: "pos" or "dv".
+ * Empty means "pos", which is what callers predating deletion vectors expect.
+ *
+ *
+ * string target_mode = 2;
+ * @return The bytes for targetMode.
+ */
+ @java.lang.Override
+ public com.google.protobuf.ByteString
+ getTargetModeBytes() {
+ java.lang.Object ref = targetMode_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ targetMode_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
private byte memoizedIsInitialized = -1;
@java.lang.Override
public final boolean isInitialized() {
@@ -18951,6 +19171,9 @@ public void writeTo(com.google.protobuf.CodedOutputStream output)
if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(threadId_)) {
com.google.protobuf.GeneratedMessageV3.writeString(output, 1, threadId_);
}
+ if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(targetMode_)) {
+ com.google.protobuf.GeneratedMessageV3.writeString(output, 2, targetMode_);
+ }
getUnknownFields().writeTo(output);
}
@@ -18963,6 +19186,9 @@ public int getSerializedSize() {
if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(threadId_)) {
size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, threadId_);
}
+ if (!com.google.protobuf.GeneratedMessageV3.isStringEmpty(targetMode_)) {
+ size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, targetMode_);
+ }
size += getUnknownFields().getSerializedSize();
memoizedSize = size;
return size;
@@ -18980,6 +19206,8 @@ public boolean equals(final java.lang.Object obj) {
if (!getThreadId()
.equals(other.getThreadId())) return false;
+ if (!getTargetMode()
+ .equals(other.getTargetMode())) return false;
if (!getUnknownFields().equals(other.getUnknownFields())) return false;
return true;
}
@@ -18993,6 +19221,8 @@ public int hashCode() {
hash = (19 * hash) + getDescriptor().hashCode();
hash = (37 * hash) + THREAD_ID_FIELD_NUMBER;
hash = (53 * hash) + getThreadId().hashCode();
+ hash = (37 * hash) + TARGET_MODE_FIELD_NUMBER;
+ hash = (53 * hash) + getTargetMode().hashCode();
hash = (29 * hash) + getUnknownFields().hashCode();
memoizedHashCode = hash;
return hash;
@@ -19042,11 +19272,13 @@ public static io.debezium.server.iceberg.rpc.RecordIngest.MigrateEqualityDeletes
return com.google.protobuf.GeneratedMessageV3
.parseWithIOException(PARSER, input, extensionRegistry);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.MigrateEqualityDeletesRequest parseDelimitedFrom(java.io.InputStream input)
throws java.io.IOException {
return com.google.protobuf.GeneratedMessageV3
.parseDelimitedWithIOException(PARSER, input);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.MigrateEqualityDeletesRequest parseDelimitedFrom(
java.io.InputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -19123,6 +19355,7 @@ public Builder clear() {
super.clear();
bitField0_ = 0;
threadId_ = "";
+ targetMode_ = "";
return this;
}
@@ -19159,6 +19392,9 @@ private void buildPartial0(io.debezium.server.iceberg.rpc.RecordIngest.MigrateEq
if (((from_bitField0_ & 0x00000001) != 0)) {
result.threadId_ = threadId_;
}
+ if (((from_bitField0_ & 0x00000002) != 0)) {
+ result.targetMode_ = targetMode_;
+ }
}
@java.lang.Override
@@ -19210,6 +19446,11 @@ public Builder mergeFrom(io.debezium.server.iceberg.rpc.RecordIngest.MigrateEqua
bitField0_ |= 0x00000001;
onChanged();
}
+ if (!other.getTargetMode().isEmpty()) {
+ targetMode_ = other.targetMode_;
+ bitField0_ |= 0x00000002;
+ onChanged();
+ }
this.mergeUnknownFields(other.getUnknownFields());
onChanged();
return this;
@@ -19241,6 +19482,11 @@ public Builder mergeFrom(
bitField0_ |= 0x00000001;
break;
} // case 10
+ case 18: {
+ targetMode_ = input.readStringRequireUtf8();
+ bitField0_ |= 0x00000002;
+ break;
+ } // case 18
default: {
if (!super.parseUnknownField(input, extensionRegistry, tag)) {
done = true; // was an endgroup tag
@@ -19329,6 +19575,103 @@ public Builder setThreadIdBytes(
onChanged();
return this;
}
+
+ private java.lang.Object targetMode_ = "";
+ /**
+ *
+ * Representation the equality deletes should be rewritten into: "pos" or "dv".
+ * Empty means "pos", which is what callers predating deletion vectors expect.
+ *
+ *
+ * string target_mode = 2;
+ * @return The targetMode.
+ */
+ public java.lang.String getTargetMode() {
+ java.lang.Object ref = targetMode_;
+ if (!(ref instanceof java.lang.String)) {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ targetMode_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ *
+ * Representation the equality deletes should be rewritten into: "pos" or "dv".
+ * Empty means "pos", which is what callers predating deletion vectors expect.
+ *
+ *
+ * string target_mode = 2;
+ * @return The bytes for targetMode.
+ */
+ public com.google.protobuf.ByteString
+ getTargetModeBytes() {
+ java.lang.Object ref = targetMode_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ targetMode_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ *
+ * Representation the equality deletes should be rewritten into: "pos" or "dv".
+ * Empty means "pos", which is what callers predating deletion vectors expect.
+ *
+ *
+ * string target_mode = 2;
+ * @param value The targetMode to set.
+ * @return This builder for chaining.
+ */
+ public Builder setTargetMode(
+ java.lang.String value) {
+ if (value == null) { throw new NullPointerException(); }
+ targetMode_ = value;
+ bitField0_ |= 0x00000002;
+ onChanged();
+ return this;
+ }
+ /**
+ *
+ * Representation the equality deletes should be rewritten into: "pos" or "dv".
+ * Empty means "pos", which is what callers predating deletion vectors expect.
+ *
+ *
+ * string target_mode = 2;
+ * @return This builder for chaining.
+ */
+ public Builder clearTargetMode() {
+ targetMode_ = getDefaultInstance().getTargetMode();
+ bitField0_ = (bitField0_ & ~0x00000002);
+ onChanged();
+ return this;
+ }
+ /**
+ *
+ * Representation the equality deletes should be rewritten into: "pos" or "dv".
+ * Empty means "pos", which is what callers predating deletion vectors expect.
+ *
+ *
+ * string target_mode = 2;
+ * @param value The bytes for targetMode to set.
+ * @return This builder for chaining.
+ */
+ public Builder setTargetModeBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) { throw new NullPointerException(); }
+ checkByteStringIsUtf8(value);
+ targetMode_ = value;
+ bitField0_ |= 0x00000002;
+ onChanged();
+ return this;
+ }
@java.lang.Override
public final Builder setUnknownFields(
final com.google.protobuf.UnknownFieldSet unknownFields) {
@@ -19442,11 +19785,6 @@ protected java.lang.Object newInstance(
return new MigrateEqualityDeletesResponse();
}
- @java.lang.Override
- public final com.google.protobuf.UnknownFieldSet
- getUnknownFields() {
- return this.unknownFields;
- }
public static final com.google.protobuf.Descriptors.Descriptor
getDescriptor() {
return io.debezium.server.iceberg.rpc.RecordIngest.internal_static_io_debezium_server_iceberg_rpc_MigrateEqualityDeletesResponse_descriptor;
@@ -19632,11 +19970,13 @@ public static io.debezium.server.iceberg.rpc.RecordIngest.MigrateEqualityDeletes
return com.google.protobuf.GeneratedMessageV3
.parseWithIOException(PARSER, input, extensionRegistry);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.MigrateEqualityDeletesResponse parseDelimitedFrom(java.io.InputStream input)
throws java.io.IOException {
return com.google.protobuf.GeneratedMessageV3
.parseDelimitedWithIOException(PARSER, input);
}
+
public static io.debezium.server.iceberg.rpc.RecordIngest.MigrateEqualityDeletesResponse parseDelimitedFrom(
java.io.InputStream input,
com.google.protobuf.ExtensionRegistryLite extensionRegistry)
@@ -19895,7 +20235,7 @@ public long getSnapshotId() {
* @return This builder for chaining.
*/
public Builder setSnapshotId(long value) {
-
+
snapshotId_ = value;
bitField0_ |= 0x00000001;
onChanged();
@@ -19932,7 +20272,7 @@ public long getRewrittenDeleteFiles() {
* @return This builder for chaining.
*/
public Builder setRewrittenDeleteFiles(long value) {
-
+
rewrittenDeleteFiles_ = value;
bitField0_ |= 0x00000002;
onChanged();
@@ -19964,7 +20304,7 @@ public long getPositionalDeletesWritten() {
* @return This builder for chaining.
*/
public Builder setPositionalDeletesWritten(long value) {
-
+
positionalDeletesWritten_ = value;
bitField0_ |= 0x00000004;
onChanged();
@@ -20154,13 +20494,13 @@ public io.debezium.server.iceberg.rpc.RecordIngest.MigrateEqualityDeletesRespons
static {
java.lang.String[] descriptorData = {
"\n\023record_ingest.proto\022\036io.debezium.serve" +
- "r.iceberg.rpc\"\223\n\n\016IcebergPayload\022H\n\004type" +
+ "r.iceberg.rpc\"\250\n\n\016IcebergPayload\022H\n\004type" +
"\030\001 \001(\0162:.io.debezium.server.iceberg.rpc." +
"IcebergPayload.PayloadType\022I\n\010metadata\030\002" +
" \001(\01327.io.debezium.server.iceberg.rpc.Ic" +
"ebergPayload.Metadata\022I\n\007records\030\003 \003(\01328" +
".io.debezium.server.iceberg.rpc.IcebergP" +
- "ayload.IceRecord\032\227\003\n\010Metadata\022\027\n\017dest_ta" +
+ "ayload.IceRecord\032\254\003\n\010Metadata\022\027\n\017dest_ta" +
"ble_name\030\001 \001(\t\022\021\n\tthread_id\030\002 \001(\t\022\035\n\020ide" +
"ntifier_field\030\003 \001(\tH\000\210\001\001\022J\n\006schema\030\004 \003(\013" +
"2:.io.debezium.server.iceberg.rpc.Iceber" +
@@ -20168,91 +20508,92 @@ public io.debezium.server.iceberg.rpc.RecordIngest.MigrateEqualityDeletesRespons
"\tnamespace\030\007 \001(\t\022\016\n\006upsert\030\010 \001(\010\022\036\n\026use_" +
"positional_deletes\030\t \001(\010\022W\n\020partition_fi" +
"elds\030\n \003(\0132=.io.debezium.server.iceberg." +
- "rpc.IcebergPayload.PartitionField\022\035\n\020bas" +
- "e_snapshot_id\030\013 \001(\003H\001\210\001\001B\023\n\021_identifier_" +
- "fieldB\023\n\021_base_snapshot_id\032,\n\013SchemaFiel" +
- "d\022\020\n\010ice_type\030\001 \001(\t\022\013\n\003key\030\002 \001(\t\0322\n\016Part" +
- "itionField\022\r\n\005field\030\001 \001(\t\022\021\n\ttransform\030\002" +
- " \001(\t\032\222\003\n\tIceRecord\022S\n\006fields\030\001 \003(\0132C.io." +
- "debezium.server.iceberg.rpc.IcebergPaylo" +
- "ad.IceRecord.FieldValue\022\023\n\013record_type\030\002" +
- " \001(\t\022\035\n\020delete_file_path\030\003 \001(\tH\000\210\001\001\022\034\n\017d" +
- "elete_position\030\004 \001(\003H\001\210\001\001\032\264\001\n\nFieldValue" +
- "\022\026\n\014string_value\030\001 \001(\tH\000\022\023\n\tint_value\030\002 " +
- "\001(\005H\000\022\024\n\nlong_value\030\003 \001(\003H\000\022\025\n\013float_val" +
- "ue\030\004 \001(\002H\000\022\026\n\014double_value\030\005 \001(\001H\000\022\024\n\nbo" +
- "ol_value\030\006 \001(\010H\000\022\025\n\013bytes_value\030\007 \001(\014H\000B" +
- "\007\n\005valueB\023\n\021_delete_file_pathB\022\n\020_delete" +
- "_position\"\217\001\n\013PayloadType\022\013\n\007RECORDS\020\000\022\n" +
- "\n\006COMMIT\020\001\022\021\n\rEVOLVE_SCHEMA\020\002\022\016\n\nDROP_TA" +
- "BLE\020\003\022\027\n\023GET_OR_CREATE_TABLE\020\004\022\030\n\024REFRES" +
- "H_TABLE_SCHEMA\020\005\022\021\n\rCLOSE_SESSION\020\006\"\301\001\n\024" +
- "RecordIngestResponse\022\016\n\006result\030\001 \001(\t\022\017\n\007" +
- "success\030\002 \001(\010\022\027\n\017olake_2pc_state\030\003 \001(\t\022\023" +
- "\n\013snapshot_id\030\004 \001(\003\022\034\n\024has_equality_dele" +
- "tes\030\005 \001(\010\022<\n\nwrite_runs\030\006 \003(\0132(.io.debez" +
- "ium.server.iceberg.rpc.WriteRun\"]\n\010Write" +
- "Run\022\021\n\tfile_path\030\001 \001(\t\022\027\n\017batch_start_id" +
- "x\030\002 \001(\005\022\026\n\016start_position\030\003 \001(\003\022\r\n\005count" +
- "\030\004 \001(\005\"\300\007\n\014ArrowPayload\022F\n\004type\030\001 \001(\01628." +
- "io.debezium.server.iceberg.rpc.ArrowPayl" +
- "oad.PayloadType\022G\n\010metadata\030\002 \001(\01325.io.d" +
- "ebezium.server.iceberg.rpc.ArrowPayload." +
- "Metadata\032\322\002\n\014FileMetadata\022\021\n\tfile_type\030\001" +
- " \001(\t\022\021\n\tfile_path\030\002 \001(\t\022\024\n\014record_count\030" +
- "\003 \001(\003\022b\n\020partition_values\030\005 \003(\0132H.io.deb" +
- "ezium.server.iceberg.rpc.ArrowPayload.Fi" +
- "leMetadata.PartitionValue\032\241\001\n\016PartitionV" +
- "alue\022\023\n\tint_value\030\001 \001(\005H\000\022\024\n\nlong_value\030" +
- "\002 \001(\003H\000\022\026\n\014string_value\030\003 \001(\tH\000\022\025\n\013float" +
- "_value\030\004 \001(\002H\000\022\026\n\014double_value\030\005 \001(\001H\000\022\024" +
- "\n\nbool_value\030\006 \001(\010H\000B\007\n\005value\0329\n\021FileUpl" +
- "oadRequest\022\021\n\tfile_data\030\001 \001(\014\022\021\n\tfile_pa" +
- "th\030\002 \001(\t\032\267\002\n\010Metadata\022\027\n\017dest_table_name" +
- "\030\001 \001(\t\022\021\n\tthread_id\030\002 \001(\t\022P\n\rfile_metada" +
- "ta\030\003 \003(\01329.io.debezium.server.iceberg.rp" +
- "c.ArrowPayload.FileMetadata\022X\n\013file_uplo" +
- "ad\030\004 \001(\0132>.io.debezium.server.iceberg.rp" +
- "c.ArrowPayload.FileUploadRequestH\000\210\001\001\022\017\n" +
- "\007payload\030\006 \001(\t\022\035\n\020base_snapshot_id\030\007 \001(\003" +
- "H\001\210\001\001B\016\n\014_file_uploadB\023\n\021_base_snapshot_" +
- "id\"U\n\013PayloadType\022\017\n\013UPLOAD_FILE\020\000\022\027\n\023RE" +
- "GISTER_AND_COMMIT\020\001\022\016\n\nJSONSCHEMA\020\002\022\014\n\010F" +
- "ILEPATH\020\003\"\347\001\n\023ArrowIngestResponse\022\016\n\006res" +
- "ult\030\001 \001(\t\022_\n\016icebergSchemas\030\002 \003(\0132G.io.d" +
- "ebezium.server.iceberg.rpc.ArrowIngestRe" +
- "sponse.IcebergSchemasEntry\022\030\n\013snapshot_i" +
- "d\030\003 \001(\003H\000\210\001\001\0325\n\023IcebergSchemasEntry\022\013\n\003k" +
- "ey\030\001 \001(\t\022\r\n\005value\030\002 \001(\t:\0028\001B\016\n\014_snapshot" +
- "_id\"\\\n\023RowIndexScanRequest\022\021\n\tthread_id\030" +
- "\001 \001(\t\022\035\n\020from_snapshot_id\030\002 \001(\003H\000\210\001\001B\023\n\021" +
- "_from_snapshot_id\"\366\001\n\021RowIndexScanBatch\022" +
- "H\n\007entries\030\001 \003(\01327.io.debezium.server.ic" +
- "eberg.rpc.RowIndexScanBatch.Entry\022\023\n\013sna" +
- "pshot_id\030\002 \001(\003\022\032\n\022requires_full_scan\030\003 \001" +
- "(\010\032f\n\005Entry\022\020\n\010olake_id\030\001 \001(\t\022\021\n\tfile_pa" +
- "th\030\002 \001(\t\022\020\n\010position\030\003 \001(\003\022\017\n\007deleted\030\004 " +
- "\001(\010J\004\010\005\020\006R\017sequence_number\"2\n\035MigrateEqu" +
- "alityDeletesRequest\022\021\n\tthread_id\030\001 \001(\t\"y" +
- "\n\036MigrateEqualityDeletesResponse\022\023\n\013snap" +
- "shot_id\030\001 \001(\003\022\036\n\026rewritten_delete_files\030" +
- "\002 \001(\003\022\"\n\032positional_deletes_written\030\003 \001(" +
- "\0032\212\001\n\023RecordIngestService\022s\n\013SendRecords" +
- "\022..io.debezium.server.iceberg.rpc.Iceber" +
- "gPayload\0324.io.debezium.server.iceberg.rp" +
- "c.RecordIngestResponse2\205\001\n\022ArrowIngestSe" +
- "rvice\022o\n\nIcebergAPI\022,.io.debezium.server" +
- ".iceberg.rpc.ArrowPayload\0323.io.debezium." +
- "server.iceberg.rpc.ArrowIngestResponse2\245" +
- "\002\n\017RowIndexService\022x\n\014ScanRowIndex\0223.io." +
- "debezium.server.iceberg.rpc.RowIndexScan" +
- "Request\0321.io.debezium.server.iceberg.rpc" +
- ".RowIndexScanBatch0\001\022\227\001\n\026MigrateEquality" +
- "Deletes\022=.io.debezium.server.iceberg.rpc" +
- ".MigrateEqualityDeletesRequest\032>.io.debe" +
- "zium.server.iceberg.rpc.MigrateEqualityD" +
- "eletesResponseB\035B\014RecordIngestZ\riceberg/" +
- "protob\006proto3"
+ "rpc.IcebergPayload.PartitionField\022\023\n\013del" +
+ "ete_mode\030\014 \001(\t\022\035\n\020base_snapshot_id\030\013 \001(\003" +
+ "H\001\210\001\001B\023\n\021_identifier_fieldB\023\n\021_base_snap" +
+ "shot_id\032,\n\013SchemaField\022\020\n\010ice_type\030\001 \001(\t" +
+ "\022\013\n\003key\030\002 \001(\t\0322\n\016PartitionField\022\r\n\005field" +
+ "\030\001 \001(\t\022\021\n\ttransform\030\002 \001(\t\032\222\003\n\tIceRecord\022" +
+ "S\n\006fields\030\001 \003(\0132C.io.debezium.server.ice" +
+ "berg.rpc.IcebergPayload.IceRecord.FieldV" +
+ "alue\022\023\n\013record_type\030\002 \001(\t\022\035\n\020delete_file" +
+ "_path\030\003 \001(\tH\000\210\001\001\022\034\n\017delete_position\030\004 \001(" +
+ "\003H\001\210\001\001\032\264\001\n\nFieldValue\022\026\n\014string_value\030\001 " +
+ "\001(\tH\000\022\023\n\tint_value\030\002 \001(\005H\000\022\024\n\nlong_value" +
+ "\030\003 \001(\003H\000\022\025\n\013float_value\030\004 \001(\002H\000\022\026\n\014doubl" +
+ "e_value\030\005 \001(\001H\000\022\024\n\nbool_value\030\006 \001(\010H\000\022\025\n" +
+ "\013bytes_value\030\007 \001(\014H\000B\007\n\005valueB\023\n\021_delete" +
+ "_file_pathB\022\n\020_delete_position\"\217\001\n\013Paylo" +
+ "adType\022\013\n\007RECORDS\020\000\022\n\n\006COMMIT\020\001\022\021\n\rEVOLV" +
+ "E_SCHEMA\020\002\022\016\n\nDROP_TABLE\020\003\022\027\n\023GET_OR_CRE" +
+ "ATE_TABLE\020\004\022\030\n\024REFRESH_TABLE_SCHEMA\020\005\022\021\n" +
+ "\rCLOSE_SESSION\020\006\"\301\001\n\024RecordIngestRespons" +
+ "e\022\016\n\006result\030\001 \001(\t\022\017\n\007success\030\002 \001(\010\022\027\n\017ol" +
+ "ake_2pc_state\030\003 \001(\t\022\023\n\013snapshot_id\030\004 \001(\003" +
+ "\022\034\n\024has_equality_deletes\030\005 \001(\010\022<\n\nwrite_" +
+ "runs\030\006 \003(\0132(.io.debezium.server.iceberg." +
+ "rpc.WriteRun\"]\n\010WriteRun\022\021\n\tfile_path\030\001 " +
+ "\001(\t\022\027\n\017batch_start_idx\030\002 \001(\005\022\026\n\016start_po" +
+ "sition\030\003 \001(\003\022\r\n\005count\030\004 \001(\005\"\300\007\n\014ArrowPay" +
+ "load\022F\n\004type\030\001 \001(\01628.io.debezium.server." +
+ "iceberg.rpc.ArrowPayload.PayloadType\022G\n\010" +
+ "metadata\030\002 \001(\01325.io.debezium.server.iceb" +
+ "erg.rpc.ArrowPayload.Metadata\032\322\002\n\014FileMe" +
+ "tadata\022\021\n\tfile_type\030\001 \001(\t\022\021\n\tfile_path\030\002" +
+ " \001(\t\022\024\n\014record_count\030\003 \001(\003\022b\n\020partition_" +
+ "values\030\005 \003(\0132H.io.debezium.server.iceber" +
+ "g.rpc.ArrowPayload.FileMetadata.Partitio" +
+ "nValue\032\241\001\n\016PartitionValue\022\023\n\tint_value\030\001" +
+ " \001(\005H\000\022\024\n\nlong_value\030\002 \001(\003H\000\022\026\n\014string_v" +
+ "alue\030\003 \001(\tH\000\022\025\n\013float_value\030\004 \001(\002H\000\022\026\n\014d" +
+ "ouble_value\030\005 \001(\001H\000\022\024\n\nbool_value\030\006 \001(\010H" +
+ "\000B\007\n\005value\0329\n\021FileUploadRequest\022\021\n\tfile_" +
+ "data\030\001 \001(\014\022\021\n\tfile_path\030\002 \001(\t\032\267\002\n\010Metada" +
+ "ta\022\027\n\017dest_table_name\030\001 \001(\t\022\021\n\tthread_id" +
+ "\030\002 \001(\t\022P\n\rfile_metadata\030\003 \003(\01329.io.debez" +
+ "ium.server.iceberg.rpc.ArrowPayload.File" +
+ "Metadata\022X\n\013file_upload\030\004 \001(\0132>.io.debez" +
+ "ium.server.iceberg.rpc.ArrowPayload.File" +
+ "UploadRequestH\000\210\001\001\022\017\n\007payload\030\006 \001(\t\022\035\n\020b" +
+ "ase_snapshot_id\030\007 \001(\003H\001\210\001\001B\016\n\014_file_uplo" +
+ "adB\023\n\021_base_snapshot_id\"U\n\013PayloadType\022\017" +
+ "\n\013UPLOAD_FILE\020\000\022\027\n\023REGISTER_AND_COMMIT\020\001" +
+ "\022\016\n\nJSONSCHEMA\020\002\022\014\n\010FILEPATH\020\003\"\347\001\n\023Arrow" +
+ "IngestResponse\022\016\n\006result\030\001 \001(\t\022_\n\016iceber" +
+ "gSchemas\030\002 \003(\0132G.io.debezium.server.iceb" +
+ "erg.rpc.ArrowIngestResponse.IcebergSchem" +
+ "asEntry\022\030\n\013snapshot_id\030\003 \001(\003H\000\210\001\001\0325\n\023Ice" +
+ "bergSchemasEntry\022\013\n\003key\030\001 \001(\t\022\r\n\005value\030\002" +
+ " \001(\t:\0028\001B\016\n\014_snapshot_id\"\\\n\023RowIndexScan" +
+ "Request\022\021\n\tthread_id\030\001 \001(\t\022\035\n\020from_snaps" +
+ "hot_id\030\002 \001(\003H\000\210\001\001B\023\n\021_from_snapshot_id\"\366" +
+ "\001\n\021RowIndexScanBatch\022H\n\007entries\030\001 \003(\01327." +
+ "io.debezium.server.iceberg.rpc.RowIndexS" +
+ "canBatch.Entry\022\023\n\013snapshot_id\030\002 \001(\003\022\032\n\022r" +
+ "equires_full_scan\030\003 \001(\010\032f\n\005Entry\022\020\n\010olak" +
+ "e_id\030\001 \001(\t\022\021\n\tfile_path\030\002 \001(\t\022\020\n\010positio" +
+ "n\030\003 \001(\003\022\017\n\007deleted\030\004 \001(\010J\004\010\005\020\006R\017sequence" +
+ "_number\"G\n\035MigrateEqualityDeletesRequest" +
+ "\022\021\n\tthread_id\030\001 \001(\t\022\023\n\013target_mode\030\002 \001(\t" +
+ "\"y\n\036MigrateEqualityDeletesResponse\022\023\n\013sn" +
+ "apshot_id\030\001 \001(\003\022\036\n\026rewritten_delete_file" +
+ "s\030\002 \001(\003\022\"\n\032positional_deletes_written\030\003 " +
+ "\001(\0032\212\001\n\023RecordIngestService\022s\n\013SendRecor" +
+ "ds\022..io.debezium.server.iceberg.rpc.Iceb" +
+ "ergPayload\0324.io.debezium.server.iceberg." +
+ "rpc.RecordIngestResponse2\205\001\n\022ArrowIngest" +
+ "Service\022o\n\nIcebergAPI\022,.io.debezium.serv" +
+ "er.iceberg.rpc.ArrowPayload\0323.io.debeziu" +
+ "m.server.iceberg.rpc.ArrowIngestResponse" +
+ "2\245\002\n\017RowIndexService\022x\n\014ScanRowIndex\0223.i" +
+ "o.debezium.server.iceberg.rpc.RowIndexSc" +
+ "anRequest\0321.io.debezium.server.iceberg.r" +
+ "pc.RowIndexScanBatch0\001\022\227\001\n\026MigrateEquali" +
+ "tyDeletes\022=.io.debezium.server.iceberg.r" +
+ "pc.MigrateEqualityDeletesRequest\032>.io.de" +
+ "bezium.server.iceberg.rpc.MigrateEqualit" +
+ "yDeletesResponseB\035B\014RecordIngestZ\riceber" +
+ "g/protob\006proto3"
};
descriptor = com.google.protobuf.Descriptors.FileDescriptor
.internalBuildGeneratedFileFrom(descriptorData,
@@ -20269,7 +20610,7 @@ public io.debezium.server.iceberg.rpc.RecordIngest.MigrateEqualityDeletesRespons
internal_static_io_debezium_server_iceberg_rpc_IcebergPayload_Metadata_fieldAccessorTable = new
com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
internal_static_io_debezium_server_iceberg_rpc_IcebergPayload_Metadata_descriptor,
- new java.lang.String[] { "DestTableName", "ThreadId", "IdentifierField", "Schema", "Payload", "Namespace", "Upsert", "UsePositionalDeletes", "PartitionFields", "BaseSnapshotId", "IdentifierField", "BaseSnapshotId", });
+ new java.lang.String[] { "DestTableName", "ThreadId", "IdentifierField", "Schema", "Payload", "Namespace", "Upsert", "UsePositionalDeletes", "PartitionFields", "DeleteMode", "BaseSnapshotId", "IdentifierField", "BaseSnapshotId", });
internal_static_io_debezium_server_iceberg_rpc_IcebergPayload_SchemaField_descriptor =
internal_static_io_debezium_server_iceberg_rpc_IcebergPayload_descriptor.getNestedTypes().get(1);
internal_static_io_debezium_server_iceberg_rpc_IcebergPayload_SchemaField_fieldAccessorTable = new
@@ -20371,7 +20712,7 @@ public io.debezium.server.iceberg.rpc.RecordIngest.MigrateEqualityDeletesRespons
internal_static_io_debezium_server_iceberg_rpc_MigrateEqualityDeletesRequest_fieldAccessorTable = new
com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
internal_static_io_debezium_server_iceberg_rpc_MigrateEqualityDeletesRequest_descriptor,
- new java.lang.String[] { "ThreadId", });
+ new java.lang.String[] { "ThreadId", "TargetMode", });
internal_static_io_debezium_server_iceberg_rpc_MigrateEqualityDeletesResponse_descriptor =
getDescriptor().getMessageTypes().get(8);
internal_static_io_debezium_server_iceberg_rpc_MigrateEqualityDeletesResponse_fieldAccessorTable = new
diff --git a/destination/iceberg/olake-iceberg-java-writer/src/main/java/io/debezium/server/iceberg/tableoperator/DeleteMode.java b/destination/iceberg/olake-iceberg-java-writer/src/main/java/io/debezium/server/iceberg/tableoperator/DeleteMode.java
new file mode 100644
index 000000000..e08ad5ff1
--- /dev/null
+++ b/destination/iceberg/olake-iceberg-java-writer/src/main/java/io/debezium/server/iceberg/tableoperator/DeleteMode.java
@@ -0,0 +1,48 @@
+package io.debezium.server.iceberg.tableoperator;
+
+/** How a writer represents the removal of a row that a later version supersedes. */
+public enum DeleteMode {
+ /** Equality delete files keyed on the table's identifier fields. */
+ EQUALITY("eq"),
+ /** Positional delete files addressing (data file, row offset). */
+ POSITION("pos"),
+ /** Format v3 deletion vectors: one Puffin bitmap per data file. */
+ DELETION_VECTOR("dv");
+
+ private final String wireName;
+
+ DeleteMode(String wireName) {
+ this.wireName = wireName;
+ }
+
+ public String wireName() {
+ return wireName;
+ }
+
+ /** True when the writer addresses rows by position, which needs the caller's row index. */
+ public boolean addressesPositions() {
+ return this == POSITION || this == DELETION_VECTOR;
+ }
+
+ /** Deletion vectors are a v3 construct; everything else works on v2. */
+ public int minimumFormatVersion() {
+ return this == DELETION_VECTOR ? 3 : 2;
+ }
+
+ /**
+ * Resolves the mode a request asked for. {@code deleteMode} wins when present;
+ * otherwise the older boolean is honoured so callers predating this field keep the
+ * behaviour they had.
+ */
+ public static DeleteMode resolve(String deleteMode, boolean usePositionalDeletes) {
+ if (deleteMode != null && !deleteMode.isBlank()) {
+ for (DeleteMode mode : values()) {
+ if (mode.wireName.equalsIgnoreCase(deleteMode.trim())) {
+ return mode;
+ }
+ }
+ throw new IllegalArgumentException("unknown delete mode: " + deleteMode);
+ }
+ return usePositionalDeletes ? POSITION : EQUALITY;
+ }
+}
diff --git a/destination/iceberg/olake-iceberg-java-writer/src/main/java/io/debezium/server/iceberg/tableoperator/IcebergTableOperator.java b/destination/iceberg/olake-iceberg-java-writer/src/main/java/io/debezium/server/iceberg/tableoperator/IcebergTableOperator.java
index 9acf8fa53..72f7f5318 100644
--- a/destination/iceberg/olake-iceberg-java-writer/src/main/java/io/debezium/server/iceberg/tableoperator/IcebergTableOperator.java
+++ b/destination/iceberg/olake-iceberg-java-writer/src/main/java/io/debezium/server/iceberg/tableoperator/IcebergTableOperator.java
@@ -68,19 +68,31 @@ public class IcebergTableOperator {
* only if it is told which files the deletes depend on.
*/
final CharSequenceSet referencedDataFiles = CharSequenceSet.empty();
+ /**
+ * Delete files this commit supersedes. Only deletion vectors produce these: a data
+ * file may carry one vector, so publishing a new one for a file that already had it
+ * must retire the old, or the table ends up with two vectors for the same file.
+ */
+ final ArrayList rewrittenDeleteFiles = new ArrayList<>();
public IcebergTableOperator(boolean upsert_records) {
this(upsert_records, false);
}
public IcebergTableOperator(boolean upsert_records, boolean usePositionalDeletes) {
+ this(upsert_records, usePositionalDeletes ? DeleteMode.POSITION : DeleteMode.EQUALITY);
+ }
+
+ public IcebergTableOperator(boolean upsert_records, DeleteMode deleteMode) {
writerFactory2 = new IcebergTableWriterFactory();
writerFactory2.keepDeletes = true;
writerFactory2.upsert = upsert_records;
- writerFactory2.usePositionalDeletes = usePositionalDeletes;
+ writerFactory2.deleteMode = deleteMode;
+ writerFactory2.usePositionalDeletes = deleteMode.addressesPositions();
this.allowFieldAddition = true;
this.upsert = upsert_records;
- this.usePositionalDeletes = usePositionalDeletes;
+ this.deleteMode = deleteMode;
+ this.usePositionalDeletes = deleteMode.addressesPositions();
this.cdcOpField = "_op_type";
this.cdcSourceTsMsField = "_cdc_timestamp";
}
@@ -105,6 +117,7 @@ public IcebergTableOperator(boolean upsert_records, boolean usePositionalDeletes
boolean allowFieldAddition;
boolean upsert;
boolean usePositionalDeletes;
+ DeleteMode deleteMode = DeleteMode.EQUALITY;
/**
* If given schema contains new fields compared to target table schema then it
* adds new fields to target iceberg
@@ -183,6 +196,7 @@ public long commitThread(String threadId, String payload, Table table, Long base
LOGGER.info("No files to commit for thread: {}", threadId);
filesToCommit.clear();
referencedDataFiles.clear();
+ rewrittenDeleteFiles.clear();
if (table.currentSnapshot() != null) {
return table.currentSnapshot().snapshotId();
}
@@ -231,6 +245,10 @@ public long commitThread(String threadId, String payload, Table table, Long base
}
}
+ for (DeleteFile replaced : rewrittenDeleteFiles) {
+ rowDelta.removeDeletes(replaced);
+ }
+
applyRowIndexValidations(rowDelta, baseSnapshotId);
rowDelta.commit();
}
@@ -251,6 +269,7 @@ public long commitThread(String threadId, String payload, Table table, Long base
filesToCommit.clear();
referencedDataFiles.clear();
+ rewrittenDeleteFiles.clear();
return snapshotId;
@@ -315,6 +334,7 @@ public void completeWriter() {
ArrayList dataFiles = new ArrayList<>(Arrays.asList(writerResult.dataFiles()));
filesToCommit.add(filesToCommit.size(), Pair.of(deleteFiles, dataFiles));
referencedDataFiles.addAll(Arrays.asList(writerResult.referencedDataFiles()));
+ rewrittenDeleteFiles.addAll(Arrays.asList(writerResult.rewrittenDeleteFiles()));
} catch (IOException e) {
LOGGER.error("Failed to complete writer", e);
throw new RuntimeException("Failed to complete writer", e);
diff --git a/destination/iceberg/olake-iceberg-java-writer/src/main/java/io/debezium/server/iceberg/tableoperator/IcebergTableWriterFactory.java b/destination/iceberg/olake-iceberg-java-writer/src/main/java/io/debezium/server/iceberg/tableoperator/IcebergTableWriterFactory.java
index ce4663170..127bc14a4 100644
--- a/destination/iceberg/olake-iceberg-java-writer/src/main/java/io/debezium/server/iceberg/tableoperator/IcebergTableWriterFactory.java
+++ b/destination/iceberg/olake-iceberg-java-writer/src/main/java/io/debezium/server/iceberg/tableoperator/IcebergTableWriterFactory.java
@@ -29,10 +29,12 @@ public class IcebergTableWriterFactory {
public boolean upsert = true;
public boolean keepDeletes = true;
public boolean usePositionalDeletes = false;
+ public DeleteMode deleteMode = DeleteMode.EQUALITY;
- // One positional delete file per referenced data file. Matches the granularity the
- // equality path has always used. PARTITION trades reader-side skipping for far fewer
- // delete files, which matters once deletes can reference arbitrary historical files.
+ // One positional delete file per referenced data file. Keeping deletes file-scoped is
+ // what lets Iceberg match them to data files by path instead of by partition, so a
+ // row that moves partitions is still superseded. PARTITION granularity would trade
+ // that away for fewer delete files.
private static final DeleteGranularity DELETE_GRANULARITY = DeleteGranularity.FILE;
public BaseTaskWriter create(Table icebergTable) {
@@ -76,14 +78,27 @@ private BaseTaskWriter appendWriter(Table icebergTable, FileFormat forma
}
}
+ private PositionalDeleteSink deleteSink(Table icebergTable, FileFormat format,
+ GenericAppenderFactory appenderFactory, OutputFileFactory fileFactory) {
+ if (deleteMode == DeleteMode.DELETION_VECTOR) {
+ // A vector replaces the data file's previous one, so it has to be seeded with the
+ // positions already deleted or this commit would resurrect them.
+ return new PositionalDeleteSink.DeletionVectors(
+ fileFactory, new PreviousDeleteLoader(icebergTable));
+ }
+ return new PositionalDeleteSink.PositionalFiles(
+ format, appenderFactory, fileFactory, DELETE_GRANULARITY);
+ }
+
private BaseTaskWriter deltaWriter(Table icebergTable, FileFormat format, GenericAppenderFactory appenderFactory, OutputFileFactory fileFactory, long targetFileSize) {
- if (usePositionalDeletes) {
+ if (deleteMode.addressesPositions()) {
// One writer for both layouts: an unpartitioned table is a single entry keyed
// on the empty partition struct, so there is no partitioned/unpartitioned split.
return new PositionalDeltaWriter(icebergTable.spec(), format, appenderFactory, fileFactory,
icebergTable.io(),
- targetFileSize, icebergTable.schema(), keepDeletes, DELETE_GRANULARITY);
+ targetFileSize, icebergTable.schema(), keepDeletes,
+ deleteSink(icebergTable, format, appenderFactory, fileFactory));
}
Set identifierFieldIds = icebergTable.schema().identifierFieldIds();
diff --git a/destination/iceberg/olake-iceberg-java-writer/src/main/java/io/debezium/server/iceberg/tableoperator/PositionalDeleteSink.java b/destination/iceberg/olake-iceberg-java-writer/src/main/java/io/debezium/server/iceberg/tableoperator/PositionalDeleteSink.java
new file mode 100644
index 000000000..4819af18b
--- /dev/null
+++ b/destination/iceberg/olake-iceberg-java-writer/src/main/java/io/debezium/server/iceberg/tableoperator/PositionalDeleteSink.java
@@ -0,0 +1,177 @@
+package io.debezium.server.iceberg.tableoperator;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.data.Record;
+import org.apache.iceberg.deletes.BaseDVFileWriter;
+import org.apache.iceberg.deletes.DVFileWriter;
+import org.apache.iceberg.deletes.DeleteGranularity;
+import org.apache.iceberg.deletes.PositionDelete;
+import org.apache.iceberg.deletes.PositionDeleteIndex;
+import org.apache.iceberg.deletes.SortingPositionOnlyDeleteWriter;
+import org.apache.iceberg.io.DeleteWriteResult;
+import org.apache.iceberg.io.FileAppenderFactory;
+import org.apache.iceberg.io.OutputFileFactory;
+import org.apache.iceberg.io.WriteResult;
+import org.apache.iceberg.util.CharSequenceSet;
+
+import com.google.common.collect.Maps;
+
+/**
+ * Where a writer sends "the row at this offset of this file is superseded".
+ *
+ * The two representations differ in more than encoding. Positional delete files are
+ * append-only: every commit adds another file addressing whatever it supersedes, and a
+ * data file can accumulate many of them. A deletion vector is one bitmap per data file,
+ * so a commit that deletes more rows from a file must publish a vector holding the
+ * union of the old and new positions and retire the old one. That is why the deletion
+ * vector implementation needs a loader for the file's existing deletes and reports
+ * rewritten files, while the positional one does neither.
+ */
+interface PositionalDeleteSink extends Closeable {
+
+ /** Marks one row superseded. Partition is null on an unpartitioned spec. */
+ void delete(String path, long position, PartitionSpec spec, StructLike partition) throws IOException;
+
+ /** Delete files produced, plus the data files they reference and any files they replace. */
+ DeleteWriteResult result();
+
+ /** Adds this sink's output to a write result under construction. */
+ default void addTo(WriteResult.Builder builder) {
+ DeleteWriteResult result = result();
+ builder.addDeleteFiles(result.deleteFiles());
+ builder.addReferencedDataFiles(result.referencedDataFiles());
+ builder.addRewrittenDeleteFiles(result.rewrittenDeleteFiles());
+ }
+
+ /** Stand-in for a sink that never wrote anything. */
+ DeleteWriteResult EMPTY = new DeleteWriteResult(List.of(), CharSequenceSet.empty(), List.of());
+
+ /** Map key for a partition, since an unpartitioned spec routes on a null partition. */
+ Object UNPARTITIONED = new Object();
+
+ static Object partitionKey(StructLike partition) {
+ return partition == null ? UNPARTITIONED : partition;
+ }
+
+ /**
+ * Writes positional delete files, one per data file they reference.
+ *
+ *
Iceberg wants a delete file's positions sorted by path then offset, which CDC
+ * order does not give us, so each partition gets a writer that buffers and sorts on
+ * close. Keeping one delete file per referenced data file is what lets Iceberg treat
+ * them as file-scoped and match them to data files by path rather than by partition.
+ */
+ final class PositionalFiles implements PositionalDeleteSink {
+ private final FileFormat format;
+ private final FileAppenderFactory appenderFactory;
+ private final OutputFileFactory fileFactory;
+ private final DeleteGranularity granularity;
+ private final Map> writers = Maps.newHashMap();
+ private final PositionDelete positionDelete = PositionDelete.create();
+ private DeleteWriteResult aggregated;
+
+ PositionalFiles(FileFormat format,
+ FileAppenderFactory appenderFactory,
+ OutputFileFactory fileFactory,
+ DeleteGranularity granularity) {
+ this.format = format;
+ this.appenderFactory = appenderFactory;
+ this.fileFactory = fileFactory;
+ this.granularity = granularity;
+ }
+
+ @Override
+ public void delete(String path, long position, PartitionSpec spec, StructLike partition) {
+ writerFor(spec, partition).write(positionDelete.set(path, position, null));
+ }
+
+ private SortingPositionOnlyDeleteWriter writerFor(PartitionSpec spec, StructLike partition) {
+ return writers.computeIfAbsent(
+ partitionKey(partition),
+ ignored -> new SortingPositionOnlyDeleteWriter<>(
+ () -> appenderFactory.newPosDeleteWriter(
+ partition == null
+ ? fileFactory.newOutputFile()
+ : fileFactory.newOutputFile(spec, partition),
+ format, partition),
+ granularity));
+ }
+
+ @Override
+ public void close() throws IOException {
+ IOException failure = null;
+ for (SortingPositionOnlyDeleteWriter writer : writers.values()) {
+ try {
+ writer.close();
+ } catch (IOException e) {
+ if (failure == null) {
+ failure = e;
+ } else {
+ failure.addSuppressed(e);
+ }
+ }
+ }
+ if (failure != null) {
+ throw failure;
+ }
+
+ List files = new ArrayList<>();
+ CharSequenceSet referenced = CharSequenceSet.empty();
+ for (SortingPositionOnlyDeleteWriter writer : writers.values()) {
+ DeleteWriteResult result = writer.result();
+ files.addAll(result.deleteFiles());
+ referenced.addAll(result.referencedDataFiles());
+ }
+ aggregated = new DeleteWriteResult(files, referenced, List.of());
+ }
+
+ @Override
+ public DeleteWriteResult result() {
+ return aggregated == null ? EMPTY : aggregated;
+ }
+ }
+
+ /**
+ * Writes v3 deletion vectors, one Puffin blob per data file.
+ *
+ * Iceberg permits a single vector per data file, so a file that already has one
+ * must be republished with the union of old and new positions and the old vector
+ * retired. {@code previousDeletes} supplies those existing positions; the writer
+ * reports the retired files through {@code rewrittenDeleteFiles}, which the commit
+ * has to remove or the table ends up with two vectors for one data file.
+ */
+ final class DeletionVectors implements PositionalDeleteSink {
+ private final DVFileWriter writer;
+ private DeleteWriteResult result;
+
+ DeletionVectors(OutputFileFactory fileFactory, Function previousDeletes) {
+ this.writer = new BaseDVFileWriter(fileFactory, previousDeletes);
+ }
+
+ @Override
+ public void delete(String path, long position, PartitionSpec spec, StructLike partition) {
+ writer.delete(path, position, spec, partition);
+ }
+
+ @Override
+ public void close() throws IOException {
+ writer.close();
+ result = writer.result();
+ }
+
+ @Override
+ public DeleteWriteResult result() {
+ return result == null ? EMPTY : result;
+ }
+ }
+}
diff --git a/destination/iceberg/olake-iceberg-java-writer/src/main/java/io/debezium/server/iceberg/tableoperator/PositionalDeltaWriter.java b/destination/iceberg/olake-iceberg-java-writer/src/main/java/io/debezium/server/iceberg/tableoperator/PositionalDeltaWriter.java
index d0de7d8fa..4dc73bb09 100644
--- a/destination/iceberg/olake-iceberg-java-writer/src/main/java/io/debezium/server/iceberg/tableoperator/PositionalDeltaWriter.java
+++ b/destination/iceberg/olake-iceberg-java-writer/src/main/java/io/debezium/server/iceberg/tableoperator/PositionalDeltaWriter.java
@@ -1,24 +1,19 @@
package io.debezium.server.iceberg.tableoperator;
import java.io.Closeable;
+import org.apache.iceberg.FileFormat;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import org.apache.iceberg.FileFormat;
import org.apache.iceberg.PartitionKey;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Schema;
import org.apache.iceberg.StructLike;
import org.apache.iceberg.data.InternalRecordWrapper;
import org.apache.iceberg.data.Record;
-import org.apache.iceberg.deletes.DeleteGranularity;
-import org.apache.iceberg.deletes.PositionDelete;
-import org.apache.iceberg.deletes.SortingPositionOnlyDeleteWriter;
-import org.apache.iceberg.encryption.EncryptedOutputFile;
import org.apache.iceberg.io.BaseTaskWriter;
-import org.apache.iceberg.io.DeleteWriteResult;
import org.apache.iceberg.io.FileAppenderFactory;
import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.io.OutputFileFactory;
@@ -46,19 +41,14 @@
public class PositionalDeltaWriter extends BaseTaskWriter implements PositionTrackableWriter {
private final PartitionSpec spec;
- private final FileFormat format;
- private final FileAppenderFactory appenderFactory;
- private final OutputFileFactory fileFactory;
private final boolean keepDeletes;
- private final DeleteGranularity deleteGranularity;
+ private final PositionalDeleteSink deleteSink;
private final PartitionKey partitionKeyTemplate;
private final InternalRecordWrapper wrapper;
private final List identifierFieldNames;
- private final Map dataWriters = Maps.newHashMap();
- private final Map> deleteWriters = Maps.newHashMap();
- private final PositionDelete positionDelete = PositionDelete.create();
+ private final Map partitions = Maps.newHashMap();
/**
* Where each identifier written by this writer currently lives.
@@ -83,7 +73,7 @@ public class PositionalDeltaWriter extends BaseTaskWriter implements Pos
// write(), so the same record is routed three times in a row. Caching on
// reference identity collapses that back to one partition-key evaluation.
private Record lastRouted;
- private RollingFileWriter lastDataWriter;
+ private PartitionWriter lastPartitionWriter;
PositionalDeltaWriter(PartitionSpec spec,
FileFormat format,
@@ -93,14 +83,11 @@ public class PositionalDeltaWriter extends BaseTaskWriter implements Pos
long targetFileSize,
Schema schema,
boolean keepDeletes,
- DeleteGranularity deleteGranularity) {
+ PositionalDeleteSink deleteSink) {
super(spec, format, appenderFactory, fileFactory, io, targetFileSize);
this.spec = spec;
- this.format = format;
- this.appenderFactory = appenderFactory;
- this.fileFactory = fileFactory;
this.keepDeletes = keepDeletes;
- this.deleteGranularity = deleteGranularity;
+ this.deleteSink = deleteSink;
this.partitionKeyTemplate = new PartitionKey(spec, schema);
this.wrapper = new InternalRecordWrapper(schema.asStruct());
@@ -116,7 +103,7 @@ public class PositionalDeltaWriter extends BaseTaskWriter implements Pos
@Override
public void write(Record row) throws IOException {
RecordWrapper wrapped = (RecordWrapper) row;
- PartitionKey key = routeKey(row);
+ PartitionWriter target = partitionWriter(row);
if (wrapped.hasPositionalDelete()) {
// NOTE: the delete is routed to the partition of the NEW record, because that is
@@ -124,8 +111,7 @@ public void write(Record row) throws IOException {
// superseded row lives in a different partition and Iceberg will not apply this
// delete to it. Fixing that needs the old row's partition to travel with the row
// index entry; the routing here is already per-partition, so only the key changes.
- deleteWriter(key).write(
- positionDelete.set(wrapped.deleteFilePath(), wrapped.deletePosition(), null));
+ deleteSink.delete(wrapped.deleteFilePath(), wrapped.deletePosition(), spec, target.partition);
}
Object identifier = identifierOf(row);
@@ -137,23 +123,22 @@ public void write(Record row) throws IOException {
return;
}
- RollingFileWriter dataWriter = dataWriter(row);
+ RollingFileWriter dataWriter = target.data;
if (identifier == null) {
dataWriter.write(row);
return;
}
- // Resolving the partition's delete writer now, rather than copying a PartitionKey
- // into every entry, keeps this to one object per row written.
- PathOffset landing = new PathOffset(
- deleteWriter(key), dataWriter.currentPath().toString(), dataWriter.currentRows());
+ // Holding the partition rather than copying a PartitionKey keeps this to one small
+ // object per row written.
+ PathOffset landing = new PathOffset(target, dataWriter.currentPath().toString(), dataWriter.currentRows());
dataWriter.write(row);
PathOffset previous = insertedRows.put(identifier, landing);
if (previous != null) {
// Same key written twice by this writer: the earlier row is superseded.
- previous.deleteWriter.write(positionDelete.set(previous.path, previous.position, null));
+ deleteSink.delete(previous.path, previous.position, spec, previous.partition.partition);
}
}
@@ -179,12 +164,12 @@ public void batchCompleted() {
@Override
public CharSequence currentPath(Record record) {
- return dataWriter(record).currentPath();
+ return partitionWriter(record).data.currentPath();
}
@Override
public long currentRows(Record record) {
- return dataWriter(record).currentRows();
+ return partitionWriter(record).data.currentRows();
}
private void supersedePrevious(Object identifier) throws IOException {
@@ -193,7 +178,7 @@ private void supersedePrevious(Object identifier) throws IOException {
}
PathOffset previous = insertedRows.remove(identifier);
if (previous != null) {
- previous.deleteWriter.write(positionDelete.set(previous.path, previous.position, null));
+ deleteSink.delete(previous.path, previous.position, spec, previous.partition.partition);
}
}
@@ -216,50 +201,30 @@ private PartitionKey routeKey(Record row) {
return partitionKeyTemplate;
}
- private RollingFileWriter dataWriter(Record row) {
+ private PartitionWriter partitionWriter(Record row) {
if (row == lastRouted) {
- return lastDataWriter;
+ return lastPartitionWriter;
}
- RollingFileWriter writer = dataWriter(routeKey(row));
+ PartitionWriter writer = partitionWriter(routeKey(row));
lastRouted = row;
- lastDataWriter = writer;
+ lastPartitionWriter = writer;
return writer;
}
- private RollingFileWriter dataWriter(PartitionKey key) {
- RollingFileWriter writer = dataWriters.get(key);
+ private PartitionWriter partitionWriter(PartitionKey key) {
+ PartitionWriter writer = partitions.get(key);
if (writer == null) {
// the template is mutated on every route, so the map must own a copy
PartitionKey copiedKey = key.copy();
- writer = new RollingFileWriter(partitionOrNull(copiedKey));
- dataWriters.put(copiedKey, writer);
- }
- return writer;
- }
-
- private SortingPositionOnlyDeleteWriter deleteWriter(PartitionKey key) {
- SortingPositionOnlyDeleteWriter writer = deleteWriters.get(key);
- if (writer == null) {
- PartitionKey copiedKey = key.copy();
- StructLike partition = partitionOrNull(copiedKey);
- writer = new SortingPositionOnlyDeleteWriter<>(
- () -> appenderFactory.newPosDeleteWriter(newDeleteOutputFile(partition), format, partition),
- deleteGranularity);
- deleteWriters.put(copiedKey, writer);
+ // Iceberg rejects a non-null partition on an unpartitioned spec, and vice versa.
+ StructLike partition = spec.isUnpartitioned() ? null : copiedKey;
+ writer = new PartitionWriter(partition, new RollingFileWriter(partition));
+ partitions.put(copiedKey, writer);
}
return writer;
}
- private EncryptedOutputFile newDeleteOutputFile(StructLike partition) {
- return partition == null ? fileFactory.newOutputFile() : fileFactory.newOutputFile(spec, partition);
- }
-
- /** Iceberg rejects a non-null partition on an unpartitioned spec, and vice versa. */
- private StructLike partitionOrNull(PartitionKey key) {
- return spec.isUnpartitioned() ? null : key;
- }
-
@Override
public WriteResult complete() throws IOException {
// super.complete() closes this writer, which flushes every delete writer, so the
@@ -270,12 +235,7 @@ public WriteResult complete() throws IOException {
.addDataFiles(dataResult.dataFiles())
.addDeleteFiles(dataResult.deleteFiles())
.addReferencedDataFiles(dataResult.referencedDataFiles());
-
- for (SortingPositionOnlyDeleteWriter writer : deleteWriters.values()) {
- DeleteWriteResult result = writer.result();
- builder.addDeleteFiles(result.deleteFiles());
- builder.addReferencedDataFiles(result.referencedDataFiles());
- }
+ deleteSink.addTo(builder);
return builder.build();
}
@@ -283,21 +243,19 @@ public WriteResult complete() throws IOException {
@Override
public void close() throws IOException {
lastRouted = null;
- lastDataWriter = null;
+ lastPartitionWriter = null;
insertedRows.clear();
// RollingFileWriter inherits close() from a package-private base, so a method
// reference to it cannot be linked from here; close them with a plain loop.
IOException failure = null;
- for (RollingFileWriter writer : dataWriters.values()) {
- failure = closeQuietly(writer, failure);
+ for (PartitionWriter writer : partitions.values()) {
+ failure = closeQuietly(writer.data, failure);
}
- dataWriters.clear();
+ partitions.clear();
- // Deliberately not cleared: complete() reads each writer's result after close.
- for (SortingPositionOnlyDeleteWriter writer : deleteWriters.values()) {
- failure = closeQuietly(writer, failure);
- }
+ // Closed last, and not discarded: complete() reads its result afterwards.
+ failure = closeQuietly(deleteSink, failure);
if (failure != null) {
throw failure;
@@ -318,14 +276,25 @@ private static IOException closeQuietly(Closeable writer, IOException failure) {
}
}
+ /** A partition's data writer plus the partition value Iceberg wants alongside it. */
+ private final class PartitionWriter {
+ private final StructLike partition;
+ private final RollingFileWriter data;
+
+ private PartitionWriter(StructLike partition, RollingFileWriter data) {
+ this.partition = partition;
+ this.data = data;
+ }
+ }
+
/** Where a row this writer produced landed. */
private static final class PathOffset {
- private final SortingPositionOnlyDeleteWriter deleteWriter;
+ private final PartitionWriter partition;
private final String path;
private final long position;
- private PathOffset(SortingPositionOnlyDeleteWriter deleteWriter, String path, long position) {
- this.deleteWriter = deleteWriter;
+ private PathOffset(PartitionWriter partition, String path, long position) {
+ this.partition = partition;
this.path = path;
this.position = position;
}
diff --git a/destination/iceberg/olake-iceberg-java-writer/src/main/java/io/debezium/server/iceberg/tableoperator/PreviousDeleteLoader.java b/destination/iceberg/olake-iceberg-java-writer/src/main/java/io/debezium/server/iceberg/tableoperator/PreviousDeleteLoader.java
new file mode 100644
index 000000000..f54aa41cf
--- /dev/null
+++ b/destination/iceberg/olake-iceberg-java-writer/src/main/java/io/debezium/server/iceberg/tableoperator/PreviousDeleteLoader.java
@@ -0,0 +1,73 @@
+package io.debezium.server.iceberg.tableoperator;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.data.BaseDeleteLoader;
+import org.apache.iceberg.deletes.PositionDeleteIndex;
+import org.apache.iceberg.io.CloseableIterable;
+
+import com.google.common.collect.Maps;
+
+/**
+ * Supplies the positions already deleted from a data file, so a new deletion vector can
+ * be published as the union of what it supersedes and what was superseded before.
+ *
+ * Iceberg allows one vector per data file. Writing a fresh vector holding only this
+ * commit's positions would resurrect every row an earlier commit had deleted, so the
+ * writer has to read the old vector first. Only files this writer actually deletes from
+ * are loaded, and the table is planned once on first use rather than per lookup.
+ *
+ *
The plan is deliberately taken once and reused: it describes the snapshot the
+ * caller's row index was built against, which is the same snapshot the commit refuses
+ * to move past.
+ */
+final class PreviousDeleteLoader implements Function {
+
+ private final Table table;
+ private Map> deletesByDataFile;
+ private BaseDeleteLoader loader;
+
+ PreviousDeleteLoader(Table table) {
+ this.table = table;
+ }
+
+ @Override
+ public PositionDeleteIndex apply(String dataFilePath) {
+ if (deletesByDataFile == null) {
+ deletesByDataFile = planDeletes();
+ loader = new BaseDeleteLoader(file -> table.io().newInputFile(file.location()));
+ }
+
+ List existing = deletesByDataFile.get(dataFilePath);
+ if (existing == null || existing.isEmpty()) {
+ return null;
+ }
+ // Handles both Puffin vectors and positional delete files, so a table part way
+ // through a migration still reports every position already deleted.
+ return loader.loadPositionDeletes(existing, dataFilePath);
+ }
+
+ private Map> planDeletes() {
+ Map> byPath = Maps.newHashMap();
+ try (CloseableIterable tasks = table.newScan().planFiles()) {
+ for (FileScanTask task : tasks) {
+ if (task.deletes().isEmpty()) {
+ continue;
+ }
+ // planFiles can split one file across several tasks; merge their delete lists.
+ byPath.computeIfAbsent(task.file().location(), path -> new java.util.ArrayList<>())
+ .addAll(task.deletes());
+ }
+ } catch (IOException e) {
+ throw new UncheckedIOException("failed to plan existing deletes of " + table.name(), e);
+ }
+ return byPath;
+ }
+}
diff --git a/destination/iceberg/olake-iceberg-java-writer/src/main/resources/record_ingest.proto b/destination/iceberg/olake-iceberg-java-writer/src/main/resources/record_ingest.proto
index f947dd44d..61b83d43b 100644
--- a/destination/iceberg/olake-iceberg-java-writer/src/main/resources/record_ingest.proto
+++ b/destination/iceberg/olake-iceberg-java-writer/src/main/resources/record_ingest.proto
@@ -35,6 +35,10 @@ message IcebergPayload {
// (fed by the caller's row index) instead of equality deleteKey.
bool use_positional_deletes = 9;
repeated PartitionField partition_fields = 10;
+ // Delete representation the writer should use: "eq" (equality deletes),
+ // "pos" (positional delete files) or "dv" (v3 deletion vectors). Empty falls
+ // back to use_positional_deletes so older callers keep working.
+ string delete_mode = 12;
// COMMIT: snapshot the caller's row index is checkpointed at. The server
// refreshes the table and refuses the commit when the tip has moved, so
// positional deletes built from a stale index cannot be published.
@@ -209,6 +213,9 @@ message RowIndexScanBatch {
message MigrateEqualityDeletesRequest {
string thread_id = 1;
+ // Representation the equality deletes should be rewritten into: "pos" or "dv".
+ // Empty means "pos", which is what callers predating deletion vectors expect.
+ string target_mode = 2;
}
message MigrateEqualityDeletesResponse {
diff --git a/destination/iceberg/olake-iceberg-java-writer/src/test/java/io/debezium/server/iceberg/tableoperator/PositionalDeltaWriterTest.java b/destination/iceberg/olake-iceberg-java-writer/src/test/java/io/debezium/server/iceberg/tableoperator/PositionalDeltaWriterTest.java
new file mode 100644
index 000000000..a323ecd02
--- /dev/null
+++ b/destination/iceberg/olake-iceberg-java-writer/src/test/java/io/debezium/server/iceberg/tableoperator/PositionalDeltaWriterTest.java
@@ -0,0 +1,855 @@
+package io.debezium.server.iceberg.tableoperator;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileContent;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.RowDelta;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.ValidationException;
+import org.apache.iceberg.util.ContentFileUtil;
+import io.debezium.server.iceberg.IcebergUtil;
+import org.apache.iceberg.data.GenericAppenderFactory;
+import org.apache.iceberg.data.GenericRecord;
+import org.apache.iceberg.data.IcebergGenerics;
+import org.apache.iceberg.data.Record;
+import org.apache.iceberg.deletes.DeleteGranularity;
+import org.apache.iceberg.hadoop.HadoopCatalog;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.OutputFileFactory;
+import org.apache.iceberg.io.WriteResult;
+import org.apache.iceberg.types.Types;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * End-to-end coverage for {@link PositionalDeltaWriter} against a real Iceberg table on
+ * a local Hadoop catalog: rows are written, committed through {@link RowDelta}, and read
+ * back through a normal scan, so Iceberg itself decides whether each positional delete
+ * applied.
+ */
+class PositionalDeltaWriterTest {
+
+ private static final Schema SCHEMA = new Schema(
+ List.of(
+ Types.NestedField.required(1, "_olake_id", Types.StringType.get()),
+ Types.NestedField.optional(2, "name", Types.StringType.get()),
+ Types.NestedField.optional(3, "region", Types.StringType.get()),
+ Types.NestedField.optional(4, "_op_type", Types.StringType.get())),
+ Set.of(1));
+
+ @TempDir
+ Path warehouse;
+
+ private HadoopCatalog catalog;
+ private final AtomicInteger partitionId = new AtomicInteger();
+
+ @BeforeEach
+ void setUp() {
+ catalog = new HadoopCatalog(new Configuration(), warehouse.toAbsolutePath().toString());
+ }
+
+ @AfterEach
+ void tearDown() throws IOException {
+ catalog.close();
+ }
+
+ // ---------------------------------------------------------------- scenarios
+
+ @ParameterizedTest
+ @EnumSource(names = {"POSITION", "DELETION_VECTOR"})
+ void unpartitionedInsertsLandAsPlainDataFiles(DeleteMode mode) throws Exception {
+ Table table = createTable("unpartitioned_inserts", PartitionSpec.unpartitioned(), mode);
+
+ WriteResult result = write(table, List.of(
+ insert("a", "Alice", "in"),
+ insert("b", "Bob", "in"),
+ insert("c", "Carol", "eu")), mode);
+
+ assertEquals(1, result.dataFiles().length, "one data file for an unpartitioned table");
+ assertEquals(0, result.deleteFiles().length, "inserts alone must not produce delete files");
+
+ commit(table, result);
+ assertEquals(Set.of("a", "b", "c"), liveIds(table));
+ }
+
+ @ParameterizedTest
+ @EnumSource(names = {"POSITION", "DELETION_VECTOR"})
+ void positionalDeleteSupersedesACommittedRow(DeleteMode mode) throws Exception {
+ Table table = createTable("supersede_committed", PartitionSpec.unpartitioned(), mode);
+
+ // sync 1: three rows land, and we note where "b" went
+ WriteResult first = write(table, List.of(
+ insert("a", "Alice", "in"),
+ insert("b", "Bob", "in"),
+ insert("c", "Carol", "eu")), mode);
+ commit(table, first);
+
+ String dataFile = first.dataFiles()[0].location();
+
+ // sync 2: "b" is updated, superseding position 1 of that file — exactly what the
+ // row index feeds the writer
+ WriteResult second = write(table, List.of(
+ update("b", "Bobby", "in", dataFile, 1L)), mode);
+ assertEquals(1, second.deleteFiles().length, "one positional delete file");
+ assertEquals(FileContent.POSITION_DELETES, second.deleteFiles()[0].content());
+ commit(table, second);
+
+ assertEquals(Set.of("a", "b", "c"), liveIds(table), "no row lost, none duplicated");
+ assertEquals(1, countById(table, "b"), "the superseded version of b must be gone");
+ assertEquals("Bobby", nameOf(table, "b"));
+ }
+
+ @ParameterizedTest
+ @EnumSource(names = {"POSITION", "DELETION_VECTOR"})
+ void sameKeyUpdatedTwiceInOneBatchLeavesOneLiveRow(DeleteMode mode) throws Exception {
+ Table table = createTable("same_batch_updates", PartitionSpec.unpartitioned(), mode);
+
+ WriteResult first = write(table, List.of(insert("k", "v1", "in")), mode);
+ commit(table, first);
+ String dataFile = first.dataFiles()[0].location();
+
+ // Both updates carry the SAME superseded location, because the caller's index is
+ // only refreshed once the batch is answered. The writer has to notice that it
+ // wrote "k" itself a moment ago and supersede that row too.
+ WriteResult second = write(table, List.of(
+ update("k", "v2", "in", dataFile, 0L),
+ update("k", "v3", "in", dataFile, 0L)), mode);
+ commit(table, second);
+
+ assertEquals(1, countById(table, "k"), "only the newest version of k may survive");
+ assertEquals("v3", nameOf(table, "k"));
+ }
+
+ @ParameterizedTest
+ @EnumSource(names = {"POSITION", "DELETION_VECTOR"})
+ void sameKeyUpdatedThreeTimesInOneBatchLeavesOneLiveRow(DeleteMode mode) throws Exception {
+ Table table = createTable("same_batch_updates_thrice", PartitionSpec.unpartitioned(), mode);
+
+ WriteResult first = write(table, List.of(insert("k", "v1", "in"), insert("other", "x", "in")), mode);
+ commit(table, first);
+ String dataFile = first.dataFiles()[0].location();
+
+ WriteResult second = write(table, List.of(
+ update("k", "v2", "in", dataFile, 0L),
+ update("k", "v3", "in", dataFile, 0L),
+ update("k", "v4", "in", dataFile, 0L)), mode);
+ commit(table, second);
+
+ assertEquals(1, countById(table, "k"));
+ assertEquals("v4", nameOf(table, "k"));
+ assertEquals(1, countById(table, "other"), "an untouched row must be unaffected");
+ }
+
+ @ParameterizedTest
+ @EnumSource(names = {"POSITION", "DELETION_VECTOR"})
+ void sameKeyUpdatedAcrossBatchesLeavesOneLiveRow(DeleteMode mode) throws Exception {
+ Table table = createTable("cross_batch_updates", PartitionSpec.unpartitioned(), mode);
+
+ WriteResult first = write(table, List.of(insert("k", "v1", "in")), mode);
+ commit(table, first);
+
+ // batch 2 supersedes the committed row; batch 3 supersedes batch 2's row, which is
+ // where the caller's index now points
+ WriteResult second = write(table, List.of(
+ update("k", "v2", "in", first.dataFiles()[0].location(), 0L)), mode);
+ commit(table, second);
+
+ WriteResult third = write(table, List.of(
+ update("k", "v3", "in", second.dataFiles()[0].location(), 0L)), mode);
+ commit(table, third);
+
+ assertEquals(1, countById(table, "k"));
+ assertEquals("v3", nameOf(table, "k"));
+ }
+
+ @ParameterizedTest
+ @EnumSource(names = {"POSITION", "DELETION_VECTOR"})
+ void partitionedFanoutRoutesInterleavedRecords(DeleteMode mode) throws Exception {
+ Table table = createTable("partitioned_fanout",
+ PartitionSpec.builderFor(SCHEMA).identity("region").build(), mode);
+
+ // deliberately interleaved so consecutive records hit different partition writers
+ WriteResult result = write(table, List.of(
+ insert("a", "Alice", "in"),
+ insert("b", "Bob", "eu"),
+ insert("c", "Carol", "in"),
+ insert("d", "Dan", "us"),
+ insert("e", "Eve", "eu")), mode);
+
+ assertEquals(3, result.dataFiles().length, "one data file per partition");
+ assertEquals(0, result.deleteFiles().length);
+
+ commit(table, result);
+ assertEquals(Set.of("a", "b", "c", "d", "e"), liveIds(table));
+
+ Map perPartition = new HashMap<>();
+ for (DataFile file : result.dataFiles()) {
+ perPartition.put(file.partition().get(0, String.class), file.recordCount());
+ }
+ assertEquals(Map.of("in", 2L, "eu", 2L, "us", 1L), perPartition);
+ }
+
+ @ParameterizedTest
+ @EnumSource(names = {"POSITION", "DELETION_VECTOR"})
+ void partitionedPositionalDeleteSupersedesWithinTheSamePartition(DeleteMode mode) throws Exception {
+ Table table = createTable("partitioned_supersede",
+ PartitionSpec.builderFor(SCHEMA).identity("region").build(), mode);
+
+ WriteResult first = write(table, List.of(
+ insert("a", "Alice", "in"),
+ insert("b", "Bob", "eu"),
+ insert("c", "Carol", "in")), mode);
+ commit(table, first);
+
+ Map byRegion = filesByRegion(first);
+ // "c" is the second row written into the "in" partition
+ DataFile inFile = byRegion.get("in");
+ assertEquals(2L, inFile.recordCount());
+
+ WriteResult second = write(table, List.of(
+ update("c", "Caroline", "in", inFile.location(), 1L)), mode);
+ assertEquals(1, second.deleteFiles().length);
+ assertEquals("in", second.deleteFiles()[0].partition().get(0, String.class),
+ "the delete file must carry the partition of the data file it targets");
+ commit(table, second);
+
+ assertEquals(Set.of("a", "b", "c"), liveIds(table));
+ assertEquals(1, countById(table, "c"));
+ assertEquals("Caroline", nameOf(table, "c"));
+ }
+
+ @ParameterizedTest
+ @EnumSource(names = {"POSITION", "DELETION_VECTOR"})
+ void partitionedSameKeyUpdatedTwiceInOneBatch(DeleteMode mode) throws Exception {
+ Table table = createTable("partitioned_same_batch",
+ PartitionSpec.builderFor(SCHEMA).identity("region").build(), mode);
+
+ WriteResult first = write(table, List.of(insert("k", "v1", "eu"), insert("j", "w1", "in")), mode);
+ commit(table, first);
+ DataFile euFile = filesByRegion(first).get("eu");
+
+ WriteResult second = write(table, List.of(
+ update("k", "v2", "eu", euFile.location(), 0L),
+ update("j", "w2", "in", filesByRegion(first).get("in").location(), 0L),
+ update("k", "v3", "eu", euFile.location(), 0L)), mode);
+ commit(table, second);
+
+ assertEquals(1, countById(table, "k"));
+ assertEquals("v3", nameOf(table, "k"));
+ assertEquals(1, countById(table, "j"));
+ assertEquals("w2", nameOf(table, "j"));
+ }
+
+ @ParameterizedTest
+ @EnumSource(names = {"POSITION", "DELETION_VECTOR"})
+ void tenThousandRowBatchWithRepeatedKeys(DeleteMode mode) throws Exception {
+ Table table = createTable("large_batch", PartitionSpec.unpartitioned(), mode);
+
+ // seed 1000 distinct keys
+ List seed = new ArrayList<>();
+ for (int i = 0; i < 1000; i++) {
+ seed.add(insert("k" + i, "v0", i % 4 == 0 ? "in" : "eu"));
+ }
+ WriteResult first = write(table, seed, mode);
+ commit(table, first);
+ assertEquals(1000, liveIds(table).size());
+
+ String dataFile = first.dataFiles()[0].location();
+
+ // 10k updates over the same 1000 keys: every key is superseded ten times, nine of
+ // those against a row this very batch produced
+ List batch = new ArrayList<>();
+ for (int round = 0; round < 10; round++) {
+ for (int i = 0; i < 1000; i++) {
+ batch.add(update("k" + i, "v" + round, i % 4 == 0 ? "in" : "eu", dataFile, (long) i));
+ }
+ }
+ assertEquals(10_000, batch.size());
+
+ WriteResult second = write(table, batch, mode);
+ commit(table, second);
+
+ Set live = liveIds(table);
+ assertEquals(1000, live.size(), "10k updates over 1000 keys must leave 1000 live rows");
+ assertEquals(1000, countRows(table), "no duplicates may survive");
+ assertEquals("v9", nameOf(table, "k7"), "the last write of each key wins");
+ assertEquals("v9", nameOf(table, "k0"));
+ }
+
+ @Test
+ void writeRunsDescribeWhereEachRowLanded() throws Exception {
+ Table table = createTable("write_runs", PartitionSpec.unpartitioned());
+
+ IcebergTableOperator operator = new IcebergTableOperator(true, true);
+ List events = List.of(
+ insert("a", "Alice", "in"),
+ insert("b", "Bob", "in"),
+ insert("c", "Carol", "in"));
+
+ List runs =
+ operator.addToTablePerSchema("t1", table, events);
+
+ assertEquals(1, runs.size(), "contiguous writes into one file collapse to one run");
+ assertEquals(0, runs.get(0).getBatchStartIdx());
+ assertEquals(0L, runs.get(0).getStartPosition());
+ assertEquals(3, runs.get(0).getCount());
+
+ operator.completeWriter();
+ assertTrue(runs.get(0).getFilePath().endsWith(".parquet"));
+ }
+
+ @Test
+ void writeRunsBreakPerPartitionOnInterleavedInput() throws Exception {
+ Table table = createTable("write_runs_partitioned",
+ PartitionSpec.builderFor(SCHEMA).identity("region").build());
+
+ IcebergTableOperator operator = new IcebergTableOperator(true, true);
+ List events = List.of(
+ insert("a", "Alice", "in"),
+ insert("b", "Bob", "eu"),
+ insert("c", "Carol", "in"));
+
+ List runs =
+ operator.addToTablePerSchema("t1", table, events);
+ operator.completeWriter();
+
+ // every record maps to exactly one (path, position) pair, whichever way the runs
+ // were cut
+ Map pathByIdx = new LinkedHashMap<>();
+ Map posByIdx = new LinkedHashMap<>();
+ for (io.debezium.server.iceberg.rpc.RecordIngest.WriteRun run : runs) {
+ for (int i = 0; i < run.getCount(); i++) {
+ pathByIdx.put(run.getBatchStartIdx() + i, run.getFilePath());
+ posByIdx.put(run.getBatchStartIdx() + i, run.getStartPosition() + i);
+ }
+ }
+
+ assertEquals(3, pathByIdx.size(), "every record must be covered exactly once");
+ assertEquals(pathByIdx.get(0), pathByIdx.get(2), "both 'in' rows share a file");
+ assertFalse(pathByIdx.get(0).equals(pathByIdx.get(1)), "'eu' lands elsewhere");
+ assertEquals(0L, posByIdx.get(0));
+ assertEquals(0L, posByIdx.get(1));
+ assertEquals(1L, posByIdx.get(2), "the second 'in' row is at offset 1 of its file");
+ }
+
+ @Test
+ void equalityModeStillUsesEqualityDeletes() throws Exception {
+ Table table = createTable("equality_mode", PartitionSpec.unpartitioned());
+
+ IcebergTableWriterFactory factory = new IcebergTableWriterFactory();
+ factory.upsert = true;
+ factory.keepDeletes = true;
+ factory.usePositionalDeletes = false;
+
+ var writer = factory.create(table);
+ assertFalse(writer instanceof PositionalDeltaWriter, "equality mode keeps the old writer");
+
+ // "u" takes the equality-delete branch
+ writer.write(update("a", "Alice", "in", null, null));
+ WriteResult result = writer.complete();
+
+ assertEquals(1, result.deleteFiles().length);
+ assertEquals(FileContent.EQUALITY_DELETES, result.deleteFiles()[0].content());
+ }
+
+ @ParameterizedTest
+ @EnumSource(names = {"POSITION", "DELETION_VECTOR"})
+ void rollingToANewFileKeepsPositionsConsistent(DeleteMode mode) throws Exception {
+ Table table = createTable("rolling_files", PartitionSpec.unpartitioned(), mode);
+
+ // a tiny target size forces the data writer to roll mid-batch
+ List rows = new ArrayList<>();
+ for (int i = 0; i < 5000; i++) {
+ rows.add(insert("k" + i, "value-" + i, "in"));
+ }
+
+ WriteResult result = write(table, rows, 4096L, mode);
+ assertTrue(result.dataFiles().length > 1, "expected the writer to roll to new files");
+
+ long total = 0;
+ for (DataFile file : result.dataFiles()) {
+ total += file.recordCount();
+ }
+ assertEquals(5000, total, "every row must land exactly once across the rolled files");
+
+ commit(table, result);
+ assertEquals(5000, countRows(table));
+ assertEquals(5000, liveIds(table).size());
+ }
+
+ @ParameterizedTest
+ @EnumSource(names = {"POSITION", "DELETION_VECTOR"})
+ void updatesSpanningARollAreStillSuperseded(DeleteMode mode) throws Exception {
+ Table table = createTable("rolling_updates", PartitionSpec.unpartitioned(), mode);
+
+ List seed = new ArrayList<>();
+ for (int i = 0; i < 2000; i++) {
+ seed.add(insert("k" + i, "v0", "in"));
+ }
+ WriteResult first = write(table, seed, 4096L, mode);
+ commit(table, first);
+ assertTrue(first.dataFiles().length > 1, "seed should span several files");
+
+ // Supersede every seeded row using the file/offset it actually landed at, which is
+ // what the row index would hold. Walk the files in write order.
+ List updates = new ArrayList<>();
+ int seen = 0;
+ for (DataFile file : first.dataFiles()) {
+ for (long pos = 0; pos < file.recordCount(); pos++) {
+ updates.add(update("k" + seen, "v1", "in", file.location(), pos));
+ seen++;
+ }
+ }
+ assertEquals(2000, seen);
+
+ WriteResult second = write(table, updates, 4096L, mode);
+ commit(table, second);
+
+ assertEquals(2000, countRows(table), "no superseded row may survive a roll");
+ assertEquals("v1", nameOf(table, "k1999"));
+ }
+
+ @ParameterizedTest
+ @EnumSource(names = {"POSITION", "DELETION_VECTOR"})
+ void partitionChangingUpdateSupersedesTheOldRow(DeleteMode mode) throws Exception {
+ // The delete is written into the NEW record's partition, but FILE granularity plus
+ // full file_path bounds make it file-scoped, so Iceberg matches it to the old row's
+ // data file by path and the partition it was filed under does not matter.
+ Table table = createTable("partition_change",
+ PartitionSpec.builderFor(SCHEMA).identity("region").build(), mode);
+
+ WriteResult first = write(table, List.of(insert("m", "Mo", "in"), insert("n", "Ned", "in")), mode);
+ commit(table, first);
+ DataFile inFile = filesByRegion(first).get("in");
+
+ // the row moves from region=in to region=eu
+ WriteResult second = write(table, List.of(
+ update("m", "Mo2", "eu", inFile.location(), 0L)), mode);
+ assertEquals("eu", second.deleteFiles()[0].partition().get(0, String.class),
+ "the delete is filed under the new record's partition");
+ assertEquals(inFile.location(), ContentFileUtil.referencedDataFile(second.deleteFiles()[0]).toString(),
+ "and is file-scoped, so it is matched by path rather than by partition");
+ commit(table, second);
+
+ assertEquals(1, countById(table, "m"), "the superseded row must not survive the move");
+ assertEquals("Mo2", nameOf(table, "m"));
+ assertEquals("eu", regionOf(table, "m"));
+ assertEquals(2, countRows(table), "the untouched row in region=in stays");
+ }
+
+ @ParameterizedTest
+ @EnumSource(names = {"POSITION", "DELETION_VECTOR"})
+ void writeResultReportsTheDataFilesItsDeletesDependOn(DeleteMode mode) throws Exception {
+ // The commit path needs these to ask Iceberg for validateDataFilesExist, which is
+ // what turns a concurrent compaction into a refused commit rather than positional
+ // deletes that silently resolve to nothing.
+ Table table = createTable("referenced_files", PartitionSpec.unpartitioned(), mode);
+
+ WriteResult first = write(table, List.of(insert("a", "A", "in"), insert("b", "B", "in")), mode);
+ commit(table, first);
+ String dataFile = first.dataFiles()[0].location();
+
+ WriteResult second = write(table, List.of(update("a", "A2", "in", dataFile, 0L)), mode);
+
+ assertEquals(1, second.referencedDataFiles().length,
+ "the delta must report the file its positional delete points into");
+ assertEquals(dataFile, second.referencedDataFiles()[0].toString());
+ }
+
+ @ParameterizedTest
+ @EnumSource(names = {"POSITION", "DELETION_VECTOR"})
+ void referencedFilesCoverEveryPartitionTouchedByDeletes(DeleteMode mode) throws Exception {
+ Table table = createTable("referenced_files_partitioned",
+ PartitionSpec.builderFor(SCHEMA).identity("region").build(), mode);
+
+ WriteResult first = write(table, List.of(insert("a", "A", "in"), insert("b", "B", "eu")), mode);
+ commit(table, first);
+ Map byRegion = filesByRegion(first);
+
+ WriteResult second = write(table, List.of(
+ update("a", "A2", "in", byRegion.get("in").location(), 0L),
+ update("b", "B2", "eu", byRegion.get("eu").location(), 0L)), mode);
+
+ Set referenced = new HashSet<>();
+ for (CharSequence path : second.referencedDataFiles()) {
+ referenced.add(path.toString());
+ }
+ assertEquals(Set.of(byRegion.get("in").location(), byRegion.get("eu").location()), referenced);
+ }
+
+ @Test
+ void validationRefusesDeletesWhoseDataFileWasRewritten() throws Exception {
+ // The scenario assertRowIndexCurrent cannot cover: a concurrent rewrite lands after
+ // the pre-check but before the catalog commit. Without these validations the commit
+ // succeeds and the positional deletes resolve to nothing.
+ Table table = createTable("stale_reference", PartitionSpec.unpartitioned());
+
+ WriteResult first = write(table, List.of(insert("a", "A", "in"), insert("b", "B", "in")));
+ commit(table, first);
+ long baseSnapshot = table.currentSnapshot().snapshotId();
+ DataFile original = first.dataFiles()[0];
+
+ // a concurrent compaction rewrites the file our positions point into
+ table.newDelete().deleteFile(original).commit();
+ table.refresh();
+
+ WriteResult second = write(table, List.of(
+ update("a", "A2", "in", original.location(), 0L)));
+
+ RowDelta rowDelta = table.newRowDelta();
+ for (DataFile file : second.dataFiles()) {
+ rowDelta.addRows(file);
+ }
+ for (DeleteFile file : second.deleteFiles()) {
+ rowDelta.addDeletes(file);
+ }
+ rowDelta.validateFromSnapshot(baseSnapshot);
+ rowDelta.validateDeletedFiles();
+ rowDelta.validateDataFilesExist(Arrays.asList(second.referencedDataFiles()));
+
+ assertThrows(ValidationException.class, rowDelta::commit,
+ "committing deletes against a rewritten data file must be refused");
+ }
+
+ @Test
+ void supersedeStateIsScopedToOneBatchButStillCorrectAcrossThem() throws Exception {
+ // Exercises the real batch loop: per-batch state is released once the write runs
+ // are handed back, and the next batch resolves the same key through the delete
+ // path the caller supplies. Both batches share one uncommitted writer session.
+ Table table = createTable("batch_scoped_supersede", PartitionSpec.unpartitioned());
+ IcebergTableOperator operator = new IcebergTableOperator(true, true);
+
+ List runs1 =
+ operator.addToTablePerSchema("t1", table, List.of(insert("k", "v1", "in")));
+ String path1 = runs1.get(0).getFilePath();
+ long pos1 = runs1.get(0).getStartPosition();
+
+ // Second batch: the caller now knows where v1 landed, so it addresses that row.
+ // Both updates carry the same location, as the legacy path always does.
+ List runs2 =
+ operator.addToTablePerSchema("t1", table, List.of(
+ update("k", "v2", "in", path1, pos1),
+ update("k", "v3", "in", path1, pos1)));
+ assertFalse(runs2.isEmpty());
+
+ operator.commitThread("t1", null, table, null);
+ table.refresh();
+
+ assertEquals(1, countById(table, "k"), "only the newest version may survive");
+ assertEquals("v3", nameOf(table, "k"));
+ assertEquals(1, countRows(table));
+ }
+
+ @Test
+ void deletionVectorMergesWithTheOneAlreadyOnTheDataFile() throws Exception {
+ // The case that makes or breaks vectors: a data file carries one vector, so a
+ // later commit deleting another row from it must publish the union and retire the
+ // old vector. Getting this wrong resurrects everything deleted earlier.
+ Table table = createTable("dv_merge", PartitionSpec.unpartitioned(), DeleteMode.DELETION_VECTOR);
+
+ WriteResult seed = write(table, List.of(
+ insert("a", "A", "in"), insert("b", "B", "in"), insert("c", "C", "in")),
+ DeleteMode.DELETION_VECTOR);
+ commit(table, seed);
+ String dataFile = seed.dataFiles()[0].location();
+
+ WriteResult first = write(table, List.of(update("a", "A2", "in", dataFile, 0L)),
+ DeleteMode.DELETION_VECTOR);
+ assertEquals(0, first.rewrittenDeleteFiles().length, "no prior vector to retire yet");
+ commit(table, first);
+ assertEquals(3, countRows(table));
+
+ // second sync deletes a different row of the same data file
+ WriteResult second = write(table, List.of(update("b", "B2", "in", dataFile, 1L)),
+ DeleteMode.DELETION_VECTOR);
+ assertEquals(1, second.rewrittenDeleteFiles().length,
+ "the data file's previous vector must be retired");
+ commit(table, second);
+
+ assertEquals(3, countRows(table), "a must not come back when b is deleted");
+ assertEquals("A2", nameOf(table, "a"));
+ assertEquals("B2", nameOf(table, "b"));
+ assertEquals("C", nameOf(table, "c"));
+ }
+
+ @Test
+ void aDataFileEndsUpWithExactlyOneDeletionVector() throws Exception {
+ // Three commits each delete a different row of the same seed file, so each one has
+ // to merge into the vector the previous commit left behind rather than add another.
+ Table table = createTable("dv_single", PartitionSpec.unpartitioned(), DeleteMode.DELETION_VECTOR);
+
+ WriteResult seed = write(table, List.of(
+ insert("a", "A", "in"), insert("b", "B", "in"), insert("c", "C", "in")),
+ DeleteMode.DELETION_VECTOR);
+ commit(table, seed);
+ String dataFile = seed.dataFiles()[0].location();
+
+ String[] keys = {"a", "b", "c"};
+ for (int round = 0; round < 3; round++) {
+ WriteResult next = write(table, List.of(
+ update(keys[round], keys[round] + "-v2", "in", dataFile, (long) round)),
+ DeleteMode.DELETION_VECTOR);
+ assertEquals(round == 0 ? 0 : 1, next.rewrittenDeleteFiles().length,
+ "every commit after the first must retire the file's previous vector");
+ commit(table, next);
+ }
+
+ List live = new ArrayList<>();
+ try (CloseableIterable tasks = table.newScan().planFiles()) {
+ for (FileScanTask task : tasks) {
+ live.addAll(task.deletes());
+ }
+ }
+ assertEquals(1, live.size(), "exactly one vector may remain for the data file");
+ assertTrue(ContentFileUtil.isDV(live.get(0)));
+
+ assertEquals(3, countRows(table), "three rows superseded, three rewritten");
+ assertEquals("a-v2", nameOf(table, "a"));
+ assertEquals("b-v2", nameOf(table, "b"));
+ assertEquals("c-v2", nameOf(table, "c"));
+ }
+
+ @Test
+ void equalityDeletesMigrateIntoDeletionVectors() throws Exception {
+ // Equality deletes are legal on v3, so a table can be created for vectors and
+ // still arrive carrying them from an earlier sync.
+ Table table = createTable("eq_to_dv", PartitionSpec.unpartitioned(), DeleteMode.DELETION_VECTOR);
+
+ IcebergTableWriterFactory equality = new IcebergTableWriterFactory();
+ equality.upsert = true;
+ equality.keepDeletes = true;
+ equality.deleteMode = DeleteMode.EQUALITY;
+
+ // Equality deletes only apply to data files from earlier snapshots, so the rows
+ // have to be committed before the delete that supersedes them.
+ var seedWriter = equality.create(table);
+ seedWriter.write(insert("a", "A", "in"));
+ seedWriter.write(insert("b", "B", "in"));
+ seedWriter.write(insert("c", "C0", "in"));
+ commit(table, seedWriter.complete());
+
+ var writer = equality.create(table);
+ writer.write(update("c", "C", "in", null, null)); // equality-deletes then writes
+ commit(table, writer.complete());
+
+ boolean hadEqualityDeletes = false;
+ try (CloseableIterable tasks = table.newScan().planFiles()) {
+ for (FileScanTask task : tasks) {
+ for (DeleteFile file : task.deletes()) {
+ hadEqualityDeletes |= file.content() == FileContent.EQUALITY_DELETES;
+ }
+ }
+ }
+ assertTrue(hadEqualityDeletes, "the table must start with equality deletes");
+
+ OutputFileFactory fileFactory =
+ OutputFileFactory.builderFor(table, partitionId.incrementAndGet(), 1L)
+ .format(FileFormat.PARQUET).build();
+ io.debezium.server.iceberg.rowindex.EqualityDeleteMigrator.Result result =
+ io.debezium.server.iceberg.rowindex.EqualityDeleteMigrator.migrate(
+ table, "_olake_id", fileFactory, DeleteMode.DELETION_VECTOR);
+ table.refresh();
+
+ assertTrue(result.rewrittenDeleteFiles > 0, "the equality deletes must be rewritten");
+
+ try (CloseableIterable tasks = table.newScan().planFiles()) {
+ for (FileScanTask task : tasks) {
+ for (DeleteFile file : task.deletes()) {
+ assertNotEquals(FileContent.EQUALITY_DELETES, file.content(),
+ "no equality delete may survive the migration");
+ assertTrue(ContentFileUtil.isDV(file), "deletes must now be vectors");
+ }
+ }
+ }
+ assertEquals(Set.of("a", "b", "c"), liveIds(table));
+ assertEquals(3, countRows(table));
+ }
+
+ // ---------------------------------------------------------------- helpers
+
+ /** Deletion vectors are a v3 construct; positional deletes stay on v2. */
+ private Table createTable(String name, PartitionSpec spec) {
+ return createTable(name, spec, DeleteMode.POSITION);
+ }
+
+ private Table createTable(String name, PartitionSpec spec, DeleteMode mode) {
+ return catalog.buildTable(TableIdentifier.of("test", name + "_" + mode.wireName()), SCHEMA)
+ .withPartitionSpec(spec)
+ .withProperty(TableProperties.FORMAT_VERSION, String.valueOf(mode.minimumFormatVersion()))
+ .create();
+ }
+
+ private WriteResult write(Table table, List records) throws IOException {
+ return write(table, records, 128 * 1024 * 1024L, DeleteMode.POSITION);
+ }
+
+ private WriteResult write(Table table, List records, DeleteMode mode) throws IOException {
+ return write(table, records, 128 * 1024 * 1024L, mode);
+ }
+
+ private WriteResult write(Table table, List records, long targetFileSize)
+ throws IOException {
+ return write(table, records, targetFileSize, DeleteMode.POSITION);
+ }
+
+ private WriteResult write(Table table, List records, long targetFileSize, DeleteMode mode)
+ throws IOException {
+ // The production factory sets write.metadata.metrics.column.file_path=full, which
+ // is what makes FILE-granularity delete files file-scoped. Using a bare
+ // GenericAppenderFactory here would truncate the bounds and change how Iceberg
+ // matches deletes to data files.
+ GenericAppenderFactory appenderFactory = IcebergUtil.getTableAppender(table);
+ OutputFileFactory fileFactory =
+ OutputFileFactory.builderFor(table, partitionId.incrementAndGet(), 1L)
+ .format(FileFormat.PARQUET)
+ .build();
+
+ PositionalDeleteSink sink = mode == DeleteMode.DELETION_VECTOR
+ ? new PositionalDeleteSink.DeletionVectors(fileFactory, new PreviousDeleteLoader(table))
+ : new PositionalDeleteSink.PositionalFiles(
+ FileFormat.PARQUET, appenderFactory, fileFactory, DeleteGranularity.FILE);
+
+ PositionalDeltaWriter writer = new PositionalDeltaWriter(
+ table.spec(), FileFormat.PARQUET, appenderFactory, fileFactory, table.io(),
+ targetFileSize, table.schema(), true, sink);
+
+ for (RecordWrapper record : records) {
+ writer.write(record);
+ }
+ return writer.complete();
+ }
+
+ private void commit(Table table, WriteResult result) {
+ RowDelta rowDelta = table.newRowDelta();
+ if (table.currentSnapshot() != null) {
+ // Mirrors the production commit: conflicts are judged from the snapshot this
+ // write was planned against. Without it Iceberg scans from the first snapshot
+ // and reads the previous commit's own vector as a concurrent addition.
+ rowDelta.validateFromSnapshot(table.currentSnapshot().snapshotId());
+ }
+ for (DataFile file : result.dataFiles()) {
+ rowDelta.addRows(file);
+ }
+ for (DeleteFile file : result.deleteFiles()) {
+ rowDelta.addDeletes(file);
+ }
+ // A data file may carry only one vector, so a replaced one has to be retired.
+ for (DeleteFile file : result.rewrittenDeleteFiles()) {
+ rowDelta.removeDeletes(file);
+ }
+ rowDelta.commit();
+ table.refresh();
+ }
+
+ private Map filesByRegion(WriteResult result) {
+ Map byRegion = new HashMap<>();
+ for (DataFile file : result.dataFiles()) {
+ byRegion.put(file.partition().get(0, String.class), file);
+ }
+ return byRegion;
+ }
+
+ private Set liveIds(Table table) throws IOException {
+ Set ids = new HashSet<>();
+ try (CloseableIterable rows = IcebergGenerics.read(table).build()) {
+ for (Record row : rows) {
+ ids.add((String) row.getField("_olake_id"));
+ }
+ }
+ return ids;
+ }
+
+ private int countRows(Table table) throws IOException {
+ int count = 0;
+ try (CloseableIterable rows = IcebergGenerics.read(table).build()) {
+ for (Record ignored : rows) {
+ count++;
+ }
+ }
+ return count;
+ }
+
+ private int countById(Table table, String id) throws IOException {
+ int count = 0;
+ try (CloseableIterable rows = IcebergGenerics.read(table).build()) {
+ for (Record row : rows) {
+ if (id.equals(row.getField("_olake_id"))) {
+ count++;
+ }
+ }
+ }
+ return count;
+ }
+
+ private String regionOf(Table table, String id) throws IOException {
+ try (CloseableIterable rows = IcebergGenerics.read(table).build()) {
+ for (Record row : rows) {
+ if (id.equals(row.getField("_olake_id"))) {
+ return (String) row.getField("region");
+ }
+ }
+ }
+ return null;
+ }
+
+ private String nameOf(Table table, String id) throws IOException {
+ try (CloseableIterable rows = IcebergGenerics.read(table).build()) {
+ for (Record row : rows) {
+ if (id.equals(row.getField("_olake_id"))) {
+ return (String) row.getField("name");
+ }
+ }
+ }
+ return null;
+ }
+
+ private RecordWrapper insert(String id, String name, String region) {
+ return wrap(id, name, region, "c", Operation.CREATE, null, null);
+ }
+
+ private RecordWrapper update(String id, String name, String region, String deletePath, Long deletePos) {
+ return wrap(id, name, region, "u", Operation.UPDATE, deletePath, deletePos);
+ }
+
+ private RecordWrapper wrap(String id, String name, String region, String opType,
+ Operation op, String deletePath, Long deletePos) {
+ Record record = GenericRecord.create(SCHEMA);
+ record.setField("_olake_id", id);
+ record.setField("name", name);
+ record.setField("region", region);
+ record.setField("_op_type", opType);
+ RecordWrapper wrapped = new RecordWrapper(record, op, deletePath, deletePos);
+ assertNotNull(wrapped.getField("_olake_id"));
+ return wrapped;
+ }
+}
diff --git a/destination/iceberg/proto/records_ingest.pb.go b/destination/iceberg/proto/records_ingest.pb.go
index ab34df7aa..e937f1d82 100644
--- a/destination/iceberg/proto/records_ingest.pb.go
+++ b/destination/iceberg/proto/records_ingest.pb.go
@@ -1,18 +1,17 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
-// protoc-gen-go v1.36.5
-// protoc v3.21.12
+// protoc-gen-go v1.36.11
+// protoc v7.35.1
// source: records_ingest.proto
package proto
import (
+ protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+ protoimpl "google.golang.org/protobuf/runtime/protoimpl"
reflect "reflect"
sync "sync"
unsafe "unsafe"
-
- protoreflect "google.golang.org/protobuf/reflect/protoreflect"
- protoimpl "google.golang.org/protobuf/runtime/protoimpl"
)
const (
@@ -289,6 +288,22 @@ func (x *RecordIngestResponse) GetWriteRuns() []*WriteRun {
// WriteRun describes a contiguous block of rows written to a single data file.
// Because Java writes records sequentially, a batch of records typically maps
// to just one or two runs (if a file rolled).
+// example:
+//
+// WriteRuns: []*proto.WriteRun{
+// {
+// FilePath: "s3a://warehouse/olake_test/test_table/data/00000-0-1111.parquet",
+// BatchStartIdx: 0, // maps to sentRecords[0], sentRecords[1], sentRecords[2]
+// StartPosition: 100, // row offset 100 in File 1
+// Count: 3,
+// },
+// {
+// FilePath: "s3a://warehouse/olake_test/test_table/data/00001-0-2222.parquet",
+// BatchStartIdx: 3, // maps to sentRecords[3], sentRecords[4]
+// StartPosition: 0, // row offset 0 in File 2
+// Count: 2,
+// },
+// },
type WriteRun struct {
state protoimpl.MessageState `protogen:"open.v1"`
FilePath string `protobuf:"bytes,1,opt,name=file_path,json=filePath,proto3" json:"file_path,omitempty"`
@@ -592,8 +607,11 @@ func (x *RowIndexScanBatch) GetRequiresFullScan() bool {
}
type MigrateEqualityDeletesRequest struct {
- state protoimpl.MessageState `protogen:"open.v1"`
- ThreadId string `protobuf:"bytes,1,opt,name=thread_id,json=threadId,proto3" json:"thread_id,omitempty"`
+ state protoimpl.MessageState `protogen:"open.v1"`
+ ThreadId string `protobuf:"bytes,1,opt,name=thread_id,json=threadId,proto3" json:"thread_id,omitempty"`
+ // Representation the equality deletes should be rewritten into: "pos" or "dv".
+ // Empty means "pos", which is what callers predating deletion vectors expect.
+ TargetMode string `protobuf:"bytes,2,opt,name=target_mode,json=targetMode,proto3" json:"target_mode,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -635,6 +653,13 @@ func (x *MigrateEqualityDeletesRequest) GetThreadId() string {
return ""
}
+func (x *MigrateEqualityDeletesRequest) GetTargetMode() string {
+ if x != nil {
+ return x.TargetMode
+ }
+ return ""
+}
+
type MigrateEqualityDeletesResponse struct {
state protoimpl.MessageState `protogen:"open.v1"`
// Snapshot created by the rewrite, or the unchanged current snapshot when the
@@ -711,6 +736,10 @@ type IcebergPayload_Metadata struct {
// (fed by the caller's row index) instead of equality deleteKey.
UsePositionalDeletes bool `protobuf:"varint,9,opt,name=use_positional_deletes,json=usePositionalDeletes,proto3" json:"use_positional_deletes,omitempty"`
PartitionFields []*IcebergPayload_PartitionField `protobuf:"bytes,10,rep,name=partition_fields,json=partitionFields,proto3" json:"partition_fields,omitempty"`
+ // Delete representation the writer should use: "eq" (equality deletes),
+ // "pos" (positional delete files) or "dv" (v3 deletion vectors). Empty falls
+ // back to use_positional_deletes so older callers keep working.
+ DeleteMode string `protobuf:"bytes,12,opt,name=delete_mode,json=deleteMode,proto3" json:"delete_mode,omitempty"`
// COMMIT: snapshot the caller's row index is checkpointed at. The server
// refreshes the table and refuses the commit when the tip has moved, so
// positional deletes built from a stale index cannot be published.
@@ -812,6 +841,13 @@ func (x *IcebergPayload_Metadata) GetPartitionFields() []*IcebergPayload_Partiti
return nil
}
+func (x *IcebergPayload_Metadata) GetDeleteMode() string {
+ if x != nil {
+ return x.DeleteMode
+ }
+ return ""
+}
+
func (x *IcebergPayload_Metadata) GetBaseSnapshotId() int64 {
if x != nil && x.BaseSnapshotId != nil {
return *x.BaseSnapshotId
@@ -1590,313 +1626,159 @@ func (x *RowIndexScanBatch_Entry) GetDeleted() bool {
var File_records_ingest_proto protoreflect.FileDescriptor
-var file_records_ingest_proto_rawDesc = string([]byte{
- 0x0a, 0x14, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x5f, 0x69, 0x6e, 0x67, 0x65, 0x73, 0x74,
- 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x1e, 0x69, 0x6f, 0x2e, 0x64, 0x65, 0x62, 0x65, 0x7a,
- 0x69, 0x75, 0x6d, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x69, 0x63, 0x65, 0x62, 0x65,
- 0x72, 0x67, 0x2e, 0x72, 0x70, 0x63, 0x22, 0xd7, 0x0c, 0x0a, 0x0e, 0x49, 0x63, 0x65, 0x62, 0x65,
- 0x72, 0x67, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x4e, 0x0a, 0x04, 0x74, 0x79, 0x70,
- 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x3a, 0x2e, 0x69, 0x6f, 0x2e, 0x64, 0x65, 0x62,
- 0x65, 0x7a, 0x69, 0x75, 0x6d, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x69, 0x63, 0x65,
- 0x62, 0x65, 0x72, 0x67, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x63, 0x65, 0x62, 0x65, 0x72, 0x67,
- 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54,
- 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x53, 0x0a, 0x08, 0x6d, 0x65, 0x74,
- 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x69, 0x6f,
- 0x2e, 0x64, 0x65, 0x62, 0x65, 0x7a, 0x69, 0x75, 0x6d, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72,
- 0x2e, 0x69, 0x63, 0x65, 0x62, 0x65, 0x72, 0x67, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x63, 0x65,
- 0x62, 0x65, 0x72, 0x67, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x4d, 0x65, 0x74, 0x61,
- 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x52,
- 0x0a, 0x07, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32,
- 0x38, 0x2e, 0x69, 0x6f, 0x2e, 0x64, 0x65, 0x62, 0x65, 0x7a, 0x69, 0x75, 0x6d, 0x2e, 0x73, 0x65,
- 0x72, 0x76, 0x65, 0x72, 0x2e, 0x69, 0x63, 0x65, 0x62, 0x65, 0x72, 0x67, 0x2e, 0x72, 0x70, 0x63,
- 0x2e, 0x49, 0x63, 0x65, 0x62, 0x65, 0x72, 0x67, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e,
- 0x49, 0x63, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x52, 0x07, 0x72, 0x65, 0x63, 0x6f, 0x72,
- 0x64, 0x73, 0x1a, 0x9c, 0x04, 0x0a, 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12,
- 0x26, 0x0a, 0x0f, 0x64, 0x65, 0x73, 0x74, 0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6e, 0x61,
- 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x64, 0x65, 0x73, 0x74, 0x54, 0x61,
- 0x62, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x68, 0x72, 0x65, 0x61,
- 0x64, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x68, 0x72, 0x65,
- 0x61, 0x64, 0x49, 0x64, 0x12, 0x2e, 0x0a, 0x10, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69,
- 0x65, 0x72, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00,
- 0x52, 0x0f, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x46, 0x69, 0x65, 0x6c,
- 0x64, 0x88, 0x01, 0x01, 0x12, 0x52, 0x0a, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x18, 0x04,
- 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x69, 0x6f, 0x2e, 0x64, 0x65, 0x62, 0x65, 0x7a, 0x69,
- 0x75, 0x6d, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x69, 0x63, 0x65, 0x62, 0x65, 0x72,
- 0x67, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x63, 0x65, 0x62, 0x65, 0x72, 0x67, 0x50, 0x61, 0x79,
- 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x46, 0x69, 0x65, 0x6c, 0x64,
- 0x52, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c,
- 0x6f, 0x61, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f,
- 0x61, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18,
- 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65,
- 0x12, 0x16, 0x0a, 0x06, 0x75, 0x70, 0x73, 0x65, 0x72, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08,
- 0x52, 0x06, 0x75, 0x70, 0x73, 0x65, 0x72, 0x74, 0x12, 0x34, 0x0a, 0x16, 0x75, 0x73, 0x65, 0x5f,
- 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x5f, 0x64, 0x65, 0x6c, 0x65, 0x74,
- 0x65, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x75, 0x73, 0x65, 0x50, 0x6f, 0x73,
- 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x73, 0x12, 0x68,
- 0x0a, 0x10, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x66, 0x69, 0x65, 0x6c,
- 0x64, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x69, 0x6f, 0x2e, 0x64, 0x65,
- 0x62, 0x65, 0x7a, 0x69, 0x75, 0x6d, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x69, 0x63,
- 0x65, 0x62, 0x65, 0x72, 0x67, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x49, 0x63, 0x65, 0x62, 0x65, 0x72,
- 0x67, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69,
- 0x6f, 0x6e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x0f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69,
- 0x6f, 0x6e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x2d, 0x0a, 0x10, 0x62, 0x61, 0x73, 0x65,
- 0x5f, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x0b, 0x20, 0x01,
- 0x28, 0x03, 0x48, 0x01, 0x52, 0x0e, 0x62, 0x61, 0x73, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68,
- 0x6f, 0x74, 0x49, 0x64, 0x88, 0x01, 0x01, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x69, 0x64, 0x65, 0x6e,
- 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x42, 0x13, 0x0a, 0x11,
- 0x5f, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x69,
- 0x64, 0x1a, 0x3a, 0x0a, 0x0b, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x46, 0x69, 0x65, 0x6c, 0x64,
- 0x12, 0x19, 0x0a, 0x08, 0x69, 0x63, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01,
- 0x28, 0x09, 0x52, 0x07, 0x69, 0x63, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6b,
- 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x1a, 0x44, 0x0a,
- 0x0e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x12,
- 0x14, 0x0a, 0x05, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05,
- 0x66, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f,
- 0x72, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66,
- 0x6f, 0x72, 0x6d, 0x1a, 0x98, 0x04, 0x0a, 0x09, 0x49, 0x63, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72,
- 0x64, 0x12, 0x5b, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28,
- 0x0b, 0x32, 0x43, 0x2e, 0x69, 0x6f, 0x2e, 0x64, 0x65, 0x62, 0x65, 0x7a, 0x69, 0x75, 0x6d, 0x2e,
- 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x69, 0x63, 0x65, 0x62, 0x65, 0x72, 0x67, 0x2e, 0x72,
- 0x70, 0x63, 0x2e, 0x49, 0x63, 0x65, 0x62, 0x65, 0x72, 0x67, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61,
- 0x64, 0x2e, 0x49, 0x63, 0x65, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x2e, 0x46, 0x69, 0x65, 0x6c,
- 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x1f,
- 0x0a, 0x0b, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20,
- 0x01, 0x28, 0x09, 0x52, 0x0a, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12,
- 0x2d, 0x0a, 0x10, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x70,
- 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0e, 0x64, 0x65, 0x6c,
- 0x65, 0x74, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x50, 0x61, 0x74, 0x68, 0x88, 0x01, 0x01, 0x12, 0x2c,
- 0x0a, 0x0f, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x5f, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f,
- 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x48, 0x01, 0x52, 0x0e, 0x64, 0x65, 0x6c, 0x65, 0x74,
- 0x65, 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x1a, 0x86, 0x02, 0x0a,
- 0x0a, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x23, 0x0a, 0x0c, 0x73,
- 0x74, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
- 0x09, 0x48, 0x00, 0x52, 0x0b, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65,
- 0x12, 0x1d, 0x0a, 0x09, 0x69, 0x6e, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20,
- 0x01, 0x28, 0x05, 0x48, 0x00, 0x52, 0x08, 0x69, 0x6e, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12,
- 0x1f, 0x0a, 0x0a, 0x6c, 0x6f, 0x6e, 0x67, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20,
- 0x01, 0x28, 0x03, 0x48, 0x00, 0x52, 0x09, 0x6c, 0x6f, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65,
- 0x12, 0x21, 0x0a, 0x0b, 0x66, 0x6c, 0x6f, 0x61, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18,
- 0x04, 0x20, 0x01, 0x28, 0x02, 0x48, 0x00, 0x52, 0x0a, 0x66, 0x6c, 0x6f, 0x61, 0x74, 0x56, 0x61,
- 0x6c, 0x75, 0x65, 0x12, 0x23, 0x0a, 0x0c, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x5f, 0x76, 0x61,
- 0x6c, 0x75, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, 0x0b, 0x64, 0x6f, 0x75,
- 0x62, 0x6c, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1f, 0x0a, 0x0a, 0x62, 0x6f, 0x6f, 0x6c,
- 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x09,
- 0x62, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x21, 0x0a, 0x0b, 0x62, 0x79, 0x74,
- 0x65, 0x73, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00,
- 0x52, 0x0a, 0x62, 0x79, 0x74, 0x65, 0x73, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x07, 0x0a, 0x05,
- 0x76, 0x61, 0x6c, 0x75, 0x65, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65,
- 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x42, 0x12, 0x0a, 0x10, 0x5f, 0x64,
- 0x65, 0x6c, 0x65, 0x74, 0x65, 0x5f, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x8f,
- 0x01, 0x0a, 0x0b, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b,
- 0x0a, 0x07, 0x52, 0x45, 0x43, 0x4f, 0x52, 0x44, 0x53, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x43,
- 0x4f, 0x4d, 0x4d, 0x49, 0x54, 0x10, 0x01, 0x12, 0x11, 0x0a, 0x0d, 0x45, 0x56, 0x4f, 0x4c, 0x56,
- 0x45, 0x5f, 0x53, 0x43, 0x48, 0x45, 0x4d, 0x41, 0x10, 0x02, 0x12, 0x0e, 0x0a, 0x0a, 0x44, 0x52,
- 0x4f, 0x50, 0x5f, 0x54, 0x41, 0x42, 0x4c, 0x45, 0x10, 0x03, 0x12, 0x17, 0x0a, 0x13, 0x47, 0x45,
- 0x54, 0x5f, 0x4f, 0x52, 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x41, 0x42, 0x4c,
- 0x45, 0x10, 0x04, 0x12, 0x18, 0x0a, 0x14, 0x52, 0x45, 0x46, 0x52, 0x45, 0x53, 0x48, 0x5f, 0x54,
- 0x41, 0x42, 0x4c, 0x45, 0x5f, 0x53, 0x43, 0x48, 0x45, 0x4d, 0x41, 0x10, 0x05, 0x12, 0x11, 0x0a,
- 0x0d, 0x43, 0x4c, 0x4f, 0x53, 0x45, 0x5f, 0x53, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x10, 0x06,
- 0x22, 0x8c, 0x02, 0x0a, 0x14, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x49, 0x6e, 0x67, 0x65, 0x73,
- 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x65, 0x73,
- 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c,
- 0x74, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01,
- 0x28, 0x08, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x26, 0x0a, 0x0f, 0x6f,
- 0x6c, 0x61, 0x6b, 0x65, 0x5f, 0x32, 0x70, 0x63, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03,
- 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6f, 0x6c, 0x61, 0x6b, 0x65, 0x32, 0x70, 0x63, 0x53, 0x74,
- 0x61, 0x74, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f,
- 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68,
- 0x6f, 0x74, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x14, 0x68, 0x61, 0x73, 0x5f, 0x65, 0x71, 0x75, 0x61,
- 0x6c, 0x69, 0x74, 0x79, 0x5f, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01,
- 0x28, 0x08, 0x52, 0x12, 0x68, 0x61, 0x73, 0x45, 0x71, 0x75, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x44,
- 0x65, 0x6c, 0x65, 0x74, 0x65, 0x73, 0x12, 0x47, 0x0a, 0x0a, 0x77, 0x72, 0x69, 0x74, 0x65, 0x5f,
- 0x72, 0x75, 0x6e, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x69, 0x6f, 0x2e,
- 0x64, 0x65, 0x62, 0x65, 0x7a, 0x69, 0x75, 0x6d, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e,
- 0x69, 0x63, 0x65, 0x62, 0x65, 0x72, 0x67, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x57, 0x72, 0x69, 0x74,
- 0x65, 0x52, 0x75, 0x6e, 0x52, 0x09, 0x77, 0x72, 0x69, 0x74, 0x65, 0x52, 0x75, 0x6e, 0x73, 0x22,
- 0x8c, 0x01, 0x0a, 0x08, 0x57, 0x72, 0x69, 0x74, 0x65, 0x52, 0x75, 0x6e, 0x12, 0x1b, 0x0a, 0x09,
- 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
- 0x08, 0x66, 0x69, 0x6c, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x26, 0x0a, 0x0f, 0x62, 0x61, 0x74,
- 0x63, 0x68, 0x5f, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x69, 0x64, 0x78, 0x18, 0x02, 0x20, 0x01,
- 0x28, 0x05, 0x52, 0x0d, 0x62, 0x61, 0x74, 0x63, 0x68, 0x53, 0x74, 0x61, 0x72, 0x74, 0x49, 0x64,
- 0x78, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x70, 0x6f, 0x73, 0x69, 0x74,
- 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x73, 0x74, 0x61, 0x72, 0x74,
- 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e,
- 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xa8,
- 0x09, 0x0a, 0x0c, 0x41, 0x72, 0x72, 0x6f, 0x77, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12,
- 0x4c, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x38, 0x2e,
- 0x69, 0x6f, 0x2e, 0x64, 0x65, 0x62, 0x65, 0x7a, 0x69, 0x75, 0x6d, 0x2e, 0x73, 0x65, 0x72, 0x76,
- 0x65, 0x72, 0x2e, 0x69, 0x63, 0x65, 0x62, 0x65, 0x72, 0x67, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x41,
- 0x72, 0x72, 0x6f, 0x77, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x50, 0x61, 0x79, 0x6c,
- 0x6f, 0x61, 0x64, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x51, 0x0a,
- 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32,
- 0x35, 0x2e, 0x69, 0x6f, 0x2e, 0x64, 0x65, 0x62, 0x65, 0x7a, 0x69, 0x75, 0x6d, 0x2e, 0x73, 0x65,
- 0x72, 0x76, 0x65, 0x72, 0x2e, 0x69, 0x63, 0x65, 0x62, 0x65, 0x72, 0x67, 0x2e, 0x72, 0x70, 0x63,
- 0x2e, 0x41, 0x72, 0x72, 0x6f, 0x77, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x4d, 0x65,
- 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
- 0x1a, 0xca, 0x03, 0x0a, 0x0c, 0x46, 0x69, 0x6c, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74,
- 0x61, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01,
- 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1b,
- 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28,
- 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x21, 0x0a, 0x0c, 0x72,
- 0x65, 0x63, 0x6f, 0x72, 0x64, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28,
- 0x03, 0x52, 0x0b, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x73,
- 0x0a, 0x10, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x76, 0x61, 0x6c, 0x75,
- 0x65, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x48, 0x2e, 0x69, 0x6f, 0x2e, 0x64, 0x65,
- 0x62, 0x65, 0x7a, 0x69, 0x75, 0x6d, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x69, 0x63,
- 0x65, 0x62, 0x65, 0x72, 0x67, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x41, 0x72, 0x72, 0x6f, 0x77, 0x50,
- 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64,
- 0x61, 0x74, 0x61, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x56, 0x61, 0x6c,
- 0x75, 0x65, 0x52, 0x0f, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x56, 0x61, 0x6c,
- 0x75, 0x65, 0x73, 0x1a, 0xe7, 0x01, 0x0a, 0x0e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f,
- 0x6e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1d, 0x0a, 0x09, 0x69, 0x6e, 0x74, 0x5f, 0x76, 0x61,
- 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x48, 0x00, 0x52, 0x08, 0x69, 0x6e, 0x74,
- 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1f, 0x0a, 0x0a, 0x6c, 0x6f, 0x6e, 0x67, 0x5f, 0x76, 0x61,
- 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x48, 0x00, 0x52, 0x09, 0x6c, 0x6f, 0x6e,
- 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x23, 0x0a, 0x0c, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67,
- 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0b,
- 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x21, 0x0a, 0x0b, 0x66,
- 0x6c, 0x6f, 0x61, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x02,
- 0x48, 0x00, 0x52, 0x0a, 0x66, 0x6c, 0x6f, 0x61, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x23,
- 0x0a, 0x0c, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x05,
- 0x20, 0x01, 0x28, 0x01, 0x48, 0x00, 0x52, 0x0b, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x56, 0x61,
- 0x6c, 0x75, 0x65, 0x12, 0x1f, 0x0a, 0x0a, 0x62, 0x6f, 0x6f, 0x6c, 0x5f, 0x76, 0x61, 0x6c, 0x75,
- 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x09, 0x62, 0x6f, 0x6f, 0x6c, 0x56,
- 0x61, 0x6c, 0x75, 0x65, 0x42, 0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x4d, 0x0a,
- 0x11, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65,
- 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18,
- 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x44, 0x61, 0x74, 0x61, 0x12,
- 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01,
- 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x50, 0x61, 0x74, 0x68, 0x1a, 0x83, 0x03, 0x0a,
- 0x08, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x26, 0x0a, 0x0f, 0x64, 0x65, 0x73,
- 0x74, 0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01,
- 0x28, 0x09, 0x52, 0x0d, 0x64, 0x65, 0x73, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x4e, 0x61, 0x6d,
- 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x02,
- 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x49, 0x64, 0x12, 0x5e,
- 0x0a, 0x0d, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18,
- 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x69, 0x6f, 0x2e, 0x64, 0x65, 0x62, 0x65, 0x7a,
- 0x69, 0x75, 0x6d, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x69, 0x63, 0x65, 0x62, 0x65,
- 0x72, 0x67, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x41, 0x72, 0x72, 0x6f, 0x77, 0x50, 0x61, 0x79, 0x6c,
- 0x6f, 0x61, 0x64, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61,
- 0x52, 0x0c, 0x66, 0x69, 0x6c, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x64,
- 0x0a, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x75, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x04, 0x20,
- 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x69, 0x6f, 0x2e, 0x64, 0x65, 0x62, 0x65, 0x7a, 0x69, 0x75,
- 0x6d, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x69, 0x63, 0x65, 0x62, 0x65, 0x72, 0x67,
- 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x41, 0x72, 0x72, 0x6f, 0x77, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61,
- 0x64, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75,
- 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x55, 0x70, 0x6c, 0x6f, 0x61,
- 0x64, 0x88, 0x01, 0x01, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18,
- 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x2d,
- 0x0a, 0x10, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f,
- 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x48, 0x01, 0x52, 0x0e, 0x62, 0x61, 0x73, 0x65,
- 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x49, 0x64, 0x88, 0x01, 0x01, 0x42, 0x0e, 0x0a,
- 0x0c, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x75, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x42, 0x13, 0x0a,
- 0x11, 0x5f, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f,
- 0x69, 0x64, 0x22, 0x55, 0x0a, 0x0b, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x54, 0x79, 0x70,
- 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x55, 0x50, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x46, 0x49, 0x4c, 0x45,
- 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x52, 0x45, 0x47, 0x49, 0x53, 0x54, 0x45, 0x52, 0x5f, 0x41,
- 0x4e, 0x44, 0x5f, 0x43, 0x4f, 0x4d, 0x4d, 0x49, 0x54, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x4a,
- 0x53, 0x4f, 0x4e, 0x53, 0x43, 0x48, 0x45, 0x4d, 0x41, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x46,
- 0x49, 0x4c, 0x45, 0x50, 0x41, 0x54, 0x48, 0x10, 0x03, 0x22, 0x97, 0x02, 0x0a, 0x13, 0x41, 0x72,
- 0x72, 0x6f, 0x77, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
- 0x65, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28,
- 0x09, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x6f, 0x0a, 0x0e, 0x69, 0x63, 0x65,
- 0x62, 0x65, 0x72, 0x67, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28,
- 0x0b, 0x32, 0x47, 0x2e, 0x69, 0x6f, 0x2e, 0x64, 0x65, 0x62, 0x65, 0x7a, 0x69, 0x75, 0x6d, 0x2e,
- 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x69, 0x63, 0x65, 0x62, 0x65, 0x72, 0x67, 0x2e, 0x72,
- 0x70, 0x63, 0x2e, 0x41, 0x72, 0x72, 0x6f, 0x77, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x52, 0x65,
- 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x49, 0x63, 0x65, 0x62, 0x65, 0x72, 0x67, 0x53, 0x63,
- 0x68, 0x65, 0x6d, 0x61, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0e, 0x69, 0x63, 0x65, 0x62,
- 0x65, 0x72, 0x67, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x73, 0x12, 0x24, 0x0a, 0x0b, 0x73, 0x6e,
- 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x48,
- 0x00, 0x52, 0x0a, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x49, 0x64, 0x88, 0x01, 0x01,
- 0x1a, 0x41, 0x0a, 0x13, 0x49, 0x63, 0x65, 0x62, 0x65, 0x72, 0x67, 0x53, 0x63, 0x68, 0x65, 0x6d,
- 0x61, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01,
- 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c,
- 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a,
- 0x02, 0x38, 0x01, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74,
- 0x5f, 0x69, 0x64, 0x22, 0x76, 0x0a, 0x13, 0x52, 0x6f, 0x77, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53,
- 0x63, 0x61, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x68,
- 0x72, 0x65, 0x61, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74,
- 0x68, 0x72, 0x65, 0x61, 0x64, 0x49, 0x64, 0x12, 0x2d, 0x0a, 0x10, 0x66, 0x72, 0x6f, 0x6d, 0x5f,
- 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28,
- 0x03, 0x48, 0x00, 0x52, 0x0e, 0x66, 0x72, 0x6f, 0x6d, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f,
- 0x74, 0x49, 0x64, 0x88, 0x01, 0x01, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x66, 0x72, 0x6f, 0x6d, 0x5f,
- 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f, 0x69, 0x64, 0x22, 0xc4, 0x02, 0x0a, 0x11,
- 0x52, 0x6f, 0x77, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x63, 0x61, 0x6e, 0x42, 0x61, 0x74, 0x63,
- 0x68, 0x12, 0x51, 0x0a, 0x07, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03,
- 0x28, 0x0b, 0x32, 0x37, 0x2e, 0x69, 0x6f, 0x2e, 0x64, 0x65, 0x62, 0x65, 0x7a, 0x69, 0x75, 0x6d,
- 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x69, 0x63, 0x65, 0x62, 0x65, 0x72, 0x67, 0x2e,
- 0x72, 0x70, 0x63, 0x2e, 0x52, 0x6f, 0x77, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x63, 0x61, 0x6e,
- 0x42, 0x61, 0x74, 0x63, 0x68, 0x2e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x65, 0x6e, 0x74,
- 0x72, 0x69, 0x65, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74,
- 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x73, 0x6e, 0x61, 0x70, 0x73,
- 0x68, 0x6f, 0x74, 0x49, 0x64, 0x12, 0x2c, 0x0a, 0x12, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65,
- 0x73, 0x5f, 0x66, 0x75, 0x6c, 0x6c, 0x5f, 0x73, 0x63, 0x61, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28,
- 0x08, 0x52, 0x10, 0x72, 0x65, 0x71, 0x75, 0x69, 0x72, 0x65, 0x73, 0x46, 0x75, 0x6c, 0x6c, 0x53,
- 0x63, 0x61, 0x6e, 0x1a, 0x8c, 0x01, 0x0a, 0x05, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x19, 0x0a,
- 0x08, 0x6f, 0x6c, 0x61, 0x6b, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
- 0x07, 0x6f, 0x6c, 0x61, 0x6b, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65,
- 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c,
- 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f,
- 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f,
- 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x04, 0x20, 0x01,
- 0x28, 0x08, 0x52, 0x07, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x4a, 0x04, 0x08, 0x05, 0x10,
- 0x06, 0x52, 0x0f, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, 0x5f, 0x6e, 0x75, 0x6d, 0x62,
- 0x65, 0x72, 0x22, 0x3c, 0x0a, 0x1d, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x45, 0x71, 0x75,
- 0x61, 0x6c, 0x69, 0x74, 0x79, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75,
- 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x69, 0x64,
- 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x49, 0x64,
- 0x22, 0xb5, 0x01, 0x0a, 0x1e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x45, 0x71, 0x75, 0x61,
- 0x6c, 0x69, 0x74, 0x79, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f,
- 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x5f,
- 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68,
- 0x6f, 0x74, 0x49, 0x64, 0x12, 0x34, 0x0a, 0x16, 0x72, 0x65, 0x77, 0x72, 0x69, 0x74, 0x74, 0x65,
- 0x6e, 0x5f, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x02,
- 0x20, 0x01, 0x28, 0x03, 0x52, 0x14, 0x72, 0x65, 0x77, 0x72, 0x69, 0x74, 0x74, 0x65, 0x6e, 0x44,
- 0x65, 0x6c, 0x65, 0x74, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x3c, 0x0a, 0x1a, 0x70, 0x6f,
- 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x5f, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x73,
- 0x5f, 0x77, 0x72, 0x69, 0x74, 0x74, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x18,
- 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65,
- 0x73, 0x57, 0x72, 0x69, 0x74, 0x74, 0x65, 0x6e, 0x32, 0x8a, 0x01, 0x0a, 0x13, 0x52, 0x65, 0x63,
- 0x6f, 0x72, 0x64, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65,
- 0x12, 0x73, 0x0a, 0x0b, 0x53, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x12,
- 0x2e, 0x2e, 0x69, 0x6f, 0x2e, 0x64, 0x65, 0x62, 0x65, 0x7a, 0x69, 0x75, 0x6d, 0x2e, 0x73, 0x65,
- 0x72, 0x76, 0x65, 0x72, 0x2e, 0x69, 0x63, 0x65, 0x62, 0x65, 0x72, 0x67, 0x2e, 0x72, 0x70, 0x63,
- 0x2e, 0x49, 0x63, 0x65, 0x62, 0x65, 0x72, 0x67, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x1a,
- 0x34, 0x2e, 0x69, 0x6f, 0x2e, 0x64, 0x65, 0x62, 0x65, 0x7a, 0x69, 0x75, 0x6d, 0x2e, 0x73, 0x65,
- 0x72, 0x76, 0x65, 0x72, 0x2e, 0x69, 0x63, 0x65, 0x62, 0x65, 0x72, 0x67, 0x2e, 0x72, 0x70, 0x63,
- 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x52, 0x65, 0x73,
- 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0x85, 0x01, 0x0a, 0x12, 0x41, 0x72, 0x72, 0x6f, 0x77, 0x49,
- 0x6e, 0x67, 0x65, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x6f, 0x0a, 0x0a,
- 0x49, 0x63, 0x65, 0x62, 0x65, 0x72, 0x67, 0x41, 0x50, 0x49, 0x12, 0x2c, 0x2e, 0x69, 0x6f, 0x2e,
- 0x64, 0x65, 0x62, 0x65, 0x7a, 0x69, 0x75, 0x6d, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e,
- 0x69, 0x63, 0x65, 0x62, 0x65, 0x72, 0x67, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x41, 0x72, 0x72, 0x6f,
- 0x77, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x1a, 0x33, 0x2e, 0x69, 0x6f, 0x2e, 0x64, 0x65,
- 0x62, 0x65, 0x7a, 0x69, 0x75, 0x6d, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x69, 0x63,
- 0x65, 0x62, 0x65, 0x72, 0x67, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x41, 0x72, 0x72, 0x6f, 0x77, 0x49,
- 0x6e, 0x67, 0x65, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xa5, 0x02,
- 0x0a, 0x0f, 0x52, 0x6f, 0x77, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63,
- 0x65, 0x12, 0x78, 0x0a, 0x0c, 0x53, 0x63, 0x61, 0x6e, 0x52, 0x6f, 0x77, 0x49, 0x6e, 0x64, 0x65,
- 0x78, 0x12, 0x33, 0x2e, 0x69, 0x6f, 0x2e, 0x64, 0x65, 0x62, 0x65, 0x7a, 0x69, 0x75, 0x6d, 0x2e,
- 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x69, 0x63, 0x65, 0x62, 0x65, 0x72, 0x67, 0x2e, 0x72,
- 0x70, 0x63, 0x2e, 0x52, 0x6f, 0x77, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x53, 0x63, 0x61, 0x6e, 0x52,
- 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, 0x69, 0x6f, 0x2e, 0x64, 0x65, 0x62, 0x65,
- 0x7a, 0x69, 0x75, 0x6d, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x69, 0x63, 0x65, 0x62,
- 0x65, 0x72, 0x67, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x6f, 0x77, 0x49, 0x6e, 0x64, 0x65, 0x78,
- 0x53, 0x63, 0x61, 0x6e, 0x42, 0x61, 0x74, 0x63, 0x68, 0x30, 0x01, 0x12, 0x97, 0x01, 0x0a, 0x16,
- 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x45, 0x71, 0x75, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x44,
- 0x65, 0x6c, 0x65, 0x74, 0x65, 0x73, 0x12, 0x3d, 0x2e, 0x69, 0x6f, 0x2e, 0x64, 0x65, 0x62, 0x65,
- 0x7a, 0x69, 0x75, 0x6d, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x69, 0x63, 0x65, 0x62,
- 0x65, 0x72, 0x67, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x45,
- 0x71, 0x75, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x73, 0x52, 0x65,
- 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x3e, 0x2e, 0x69, 0x6f, 0x2e, 0x64, 0x65, 0x62, 0x65, 0x7a,
- 0x69, 0x75, 0x6d, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x2e, 0x69, 0x63, 0x65, 0x62, 0x65,
- 0x72, 0x67, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x45, 0x71,
- 0x75, 0x61, 0x6c, 0x69, 0x74, 0x79, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x73, 0x52, 0x65, 0x73,
- 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x1d, 0x42, 0x0c, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x49,
- 0x6e, 0x67, 0x65, 0x73, 0x74, 0x5a, 0x0d, 0x69, 0x63, 0x65, 0x62, 0x65, 0x72, 0x67, 0x2f, 0x70,
- 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
-})
+const file_records_ingest_proto_rawDesc = "" +
+ "\n" +
+ "\x14records_ingest.proto\x12\x1eio.debezium.server.iceberg.rpc\"\xf8\f\n" +
+ "\x0eIcebergPayload\x12N\n" +
+ "\x04type\x18\x01 \x01(\x0e2:.io.debezium.server.iceberg.rpc.IcebergPayload.PayloadTypeR\x04type\x12S\n" +
+ "\bmetadata\x18\x02 \x01(\v27.io.debezium.server.iceberg.rpc.IcebergPayload.MetadataR\bmetadata\x12R\n" +
+ "\arecords\x18\x03 \x03(\v28.io.debezium.server.iceberg.rpc.IcebergPayload.IceRecordR\arecords\x1a\xbd\x04\n" +
+ "\bMetadata\x12&\n" +
+ "\x0fdest_table_name\x18\x01 \x01(\tR\rdestTableName\x12\x1b\n" +
+ "\tthread_id\x18\x02 \x01(\tR\bthreadId\x12.\n" +
+ "\x10identifier_field\x18\x03 \x01(\tH\x00R\x0fidentifierField\x88\x01\x01\x12R\n" +
+ "\x06schema\x18\x04 \x03(\v2:.io.debezium.server.iceberg.rpc.IcebergPayload.SchemaFieldR\x06schema\x12\x18\n" +
+ "\apayload\x18\x06 \x01(\tR\apayload\x12\x1c\n" +
+ "\tnamespace\x18\a \x01(\tR\tnamespace\x12\x16\n" +
+ "\x06upsert\x18\b \x01(\bR\x06upsert\x124\n" +
+ "\x16use_positional_deletes\x18\t \x01(\bR\x14usePositionalDeletes\x12h\n" +
+ "\x10partition_fields\x18\n" +
+ " \x03(\v2=.io.debezium.server.iceberg.rpc.IcebergPayload.PartitionFieldR\x0fpartitionFields\x12\x1f\n" +
+ "\vdelete_mode\x18\f \x01(\tR\n" +
+ "deleteMode\x12-\n" +
+ "\x10base_snapshot_id\x18\v \x01(\x03H\x01R\x0ebaseSnapshotId\x88\x01\x01B\x13\n" +
+ "\x11_identifier_fieldB\x13\n" +
+ "\x11_base_snapshot_id\x1a:\n" +
+ "\vSchemaField\x12\x19\n" +
+ "\bice_type\x18\x01 \x01(\tR\aiceType\x12\x10\n" +
+ "\x03key\x18\x02 \x01(\tR\x03key\x1aD\n" +
+ "\x0ePartitionField\x12\x14\n" +
+ "\x05field\x18\x01 \x01(\tR\x05field\x12\x1c\n" +
+ "\ttransform\x18\x02 \x01(\tR\ttransform\x1a\x98\x04\n" +
+ "\tIceRecord\x12[\n" +
+ "\x06fields\x18\x01 \x03(\v2C.io.debezium.server.iceberg.rpc.IcebergPayload.IceRecord.FieldValueR\x06fields\x12\x1f\n" +
+ "\vrecord_type\x18\x02 \x01(\tR\n" +
+ "recordType\x12-\n" +
+ "\x10delete_file_path\x18\x03 \x01(\tH\x00R\x0edeleteFilePath\x88\x01\x01\x12,\n" +
+ "\x0fdelete_position\x18\x04 \x01(\x03H\x01R\x0edeletePosition\x88\x01\x01\x1a\x86\x02\n" +
+ "\n" +
+ "FieldValue\x12#\n" +
+ "\fstring_value\x18\x01 \x01(\tH\x00R\vstringValue\x12\x1d\n" +
+ "\tint_value\x18\x02 \x01(\x05H\x00R\bintValue\x12\x1f\n" +
+ "\n" +
+ "long_value\x18\x03 \x01(\x03H\x00R\tlongValue\x12!\n" +
+ "\vfloat_value\x18\x04 \x01(\x02H\x00R\n" +
+ "floatValue\x12#\n" +
+ "\fdouble_value\x18\x05 \x01(\x01H\x00R\vdoubleValue\x12\x1f\n" +
+ "\n" +
+ "bool_value\x18\x06 \x01(\bH\x00R\tboolValue\x12!\n" +
+ "\vbytes_value\x18\a \x01(\fH\x00R\n" +
+ "bytesValueB\a\n" +
+ "\x05valueB\x13\n" +
+ "\x11_delete_file_pathB\x12\n" +
+ "\x10_delete_position\"\x8f\x01\n" +
+ "\vPayloadType\x12\v\n" +
+ "\aRECORDS\x10\x00\x12\n" +
+ "\n" +
+ "\x06COMMIT\x10\x01\x12\x11\n" +
+ "\rEVOLVE_SCHEMA\x10\x02\x12\x0e\n" +
+ "\n" +
+ "DROP_TABLE\x10\x03\x12\x17\n" +
+ "\x13GET_OR_CREATE_TABLE\x10\x04\x12\x18\n" +
+ "\x14REFRESH_TABLE_SCHEMA\x10\x05\x12\x11\n" +
+ "\rCLOSE_SESSION\x10\x06\"\x8c\x02\n" +
+ "\x14RecordIngestResponse\x12\x16\n" +
+ "\x06result\x18\x01 \x01(\tR\x06result\x12\x18\n" +
+ "\asuccess\x18\x02 \x01(\bR\asuccess\x12&\n" +
+ "\x0folake_2pc_state\x18\x03 \x01(\tR\rolake2pcState\x12\x1f\n" +
+ "\vsnapshot_id\x18\x04 \x01(\x03R\n" +
+ "snapshotId\x120\n" +
+ "\x14has_equality_deletes\x18\x05 \x01(\bR\x12hasEqualityDeletes\x12G\n" +
+ "\n" +
+ "write_runs\x18\x06 \x03(\v2(.io.debezium.server.iceberg.rpc.WriteRunR\twriteRuns\"\x8c\x01\n" +
+ "\bWriteRun\x12\x1b\n" +
+ "\tfile_path\x18\x01 \x01(\tR\bfilePath\x12&\n" +
+ "\x0fbatch_start_idx\x18\x02 \x01(\x05R\rbatchStartIdx\x12%\n" +
+ "\x0estart_position\x18\x03 \x01(\x03R\rstartPosition\x12\x14\n" +
+ "\x05count\x18\x04 \x01(\x05R\x05count\"\xa8\t\n" +
+ "\fArrowPayload\x12L\n" +
+ "\x04type\x18\x01 \x01(\x0e28.io.debezium.server.iceberg.rpc.ArrowPayload.PayloadTypeR\x04type\x12Q\n" +
+ "\bmetadata\x18\x02 \x01(\v25.io.debezium.server.iceberg.rpc.ArrowPayload.MetadataR\bmetadata\x1a\xca\x03\n" +
+ "\fFileMetadata\x12\x1b\n" +
+ "\tfile_type\x18\x01 \x01(\tR\bfileType\x12\x1b\n" +
+ "\tfile_path\x18\x02 \x01(\tR\bfilePath\x12!\n" +
+ "\frecord_count\x18\x03 \x01(\x03R\vrecordCount\x12s\n" +
+ "\x10partition_values\x18\x05 \x03(\v2H.io.debezium.server.iceberg.rpc.ArrowPayload.FileMetadata.PartitionValueR\x0fpartitionValues\x1a\xe7\x01\n" +
+ "\x0ePartitionValue\x12\x1d\n" +
+ "\tint_value\x18\x01 \x01(\x05H\x00R\bintValue\x12\x1f\n" +
+ "\n" +
+ "long_value\x18\x02 \x01(\x03H\x00R\tlongValue\x12#\n" +
+ "\fstring_value\x18\x03 \x01(\tH\x00R\vstringValue\x12!\n" +
+ "\vfloat_value\x18\x04 \x01(\x02H\x00R\n" +
+ "floatValue\x12#\n" +
+ "\fdouble_value\x18\x05 \x01(\x01H\x00R\vdoubleValue\x12\x1f\n" +
+ "\n" +
+ "bool_value\x18\x06 \x01(\bH\x00R\tboolValueB\a\n" +
+ "\x05value\x1aM\n" +
+ "\x11FileUploadRequest\x12\x1b\n" +
+ "\tfile_data\x18\x01 \x01(\fR\bfileData\x12\x1b\n" +
+ "\tfile_path\x18\x02 \x01(\tR\bfilePath\x1a\x83\x03\n" +
+ "\bMetadata\x12&\n" +
+ "\x0fdest_table_name\x18\x01 \x01(\tR\rdestTableName\x12\x1b\n" +
+ "\tthread_id\x18\x02 \x01(\tR\bthreadId\x12^\n" +
+ "\rfile_metadata\x18\x03 \x03(\v29.io.debezium.server.iceberg.rpc.ArrowPayload.FileMetadataR\ffileMetadata\x12d\n" +
+ "\vfile_upload\x18\x04 \x01(\v2>.io.debezium.server.iceberg.rpc.ArrowPayload.FileUploadRequestH\x00R\n" +
+ "fileUpload\x88\x01\x01\x12\x18\n" +
+ "\apayload\x18\x06 \x01(\tR\apayload\x12-\n" +
+ "\x10base_snapshot_id\x18\a \x01(\x03H\x01R\x0ebaseSnapshotId\x88\x01\x01B\x0e\n" +
+ "\f_file_uploadB\x13\n" +
+ "\x11_base_snapshot_id\"U\n" +
+ "\vPayloadType\x12\x0f\n" +
+ "\vUPLOAD_FILE\x10\x00\x12\x17\n" +
+ "\x13REGISTER_AND_COMMIT\x10\x01\x12\x0e\n" +
+ "\n" +
+ "JSONSCHEMA\x10\x02\x12\f\n" +
+ "\bFILEPATH\x10\x03\"\x97\x02\n" +
+ "\x13ArrowIngestResponse\x12\x16\n" +
+ "\x06result\x18\x01 \x01(\tR\x06result\x12o\n" +
+ "\x0eicebergSchemas\x18\x02 \x03(\v2G.io.debezium.server.iceberg.rpc.ArrowIngestResponse.IcebergSchemasEntryR\x0eicebergSchemas\x12$\n" +
+ "\vsnapshot_id\x18\x03 \x01(\x03H\x00R\n" +
+ "snapshotId\x88\x01\x01\x1aA\n" +
+ "\x13IcebergSchemasEntry\x12\x10\n" +
+ "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
+ "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01B\x0e\n" +
+ "\f_snapshot_id\"v\n" +
+ "\x13RowIndexScanRequest\x12\x1b\n" +
+ "\tthread_id\x18\x01 \x01(\tR\bthreadId\x12-\n" +
+ "\x10from_snapshot_id\x18\x02 \x01(\x03H\x00R\x0efromSnapshotId\x88\x01\x01B\x13\n" +
+ "\x11_from_snapshot_id\"\xc4\x02\n" +
+ "\x11RowIndexScanBatch\x12Q\n" +
+ "\aentries\x18\x01 \x03(\v27.io.debezium.server.iceberg.rpc.RowIndexScanBatch.EntryR\aentries\x12\x1f\n" +
+ "\vsnapshot_id\x18\x02 \x01(\x03R\n" +
+ "snapshotId\x12,\n" +
+ "\x12requires_full_scan\x18\x03 \x01(\bR\x10requiresFullScan\x1a\x8c\x01\n" +
+ "\x05Entry\x12\x19\n" +
+ "\bolake_id\x18\x01 \x01(\tR\aolakeId\x12\x1b\n" +
+ "\tfile_path\x18\x02 \x01(\tR\bfilePath\x12\x1a\n" +
+ "\bposition\x18\x03 \x01(\x03R\bposition\x12\x18\n" +
+ "\adeleted\x18\x04 \x01(\bR\adeletedJ\x04\b\x05\x10\x06R\x0fsequence_number\"]\n" +
+ "\x1dMigrateEqualityDeletesRequest\x12\x1b\n" +
+ "\tthread_id\x18\x01 \x01(\tR\bthreadId\x12\x1f\n" +
+ "\vtarget_mode\x18\x02 \x01(\tR\n" +
+ "targetMode\"\xb5\x01\n" +
+ "\x1eMigrateEqualityDeletesResponse\x12\x1f\n" +
+ "\vsnapshot_id\x18\x01 \x01(\x03R\n" +
+ "snapshotId\x124\n" +
+ "\x16rewritten_delete_files\x18\x02 \x01(\x03R\x14rewrittenDeleteFiles\x12<\n" +
+ "\x1apositional_deletes_written\x18\x03 \x01(\x03R\x18positionalDeletesWritten2\x8a\x01\n" +
+ "\x13RecordIngestService\x12s\n" +
+ "\vSendRecords\x12..io.debezium.server.iceberg.rpc.IcebergPayload\x1a4.io.debezium.server.iceberg.rpc.RecordIngestResponse2\x85\x01\n" +
+ "\x12ArrowIngestService\x12o\n" +
+ "\n" +
+ "IcebergAPI\x12,.io.debezium.server.iceberg.rpc.ArrowPayload\x1a3.io.debezium.server.iceberg.rpc.ArrowIngestResponse2\xa5\x02\n" +
+ "\x0fRowIndexService\x12x\n" +
+ "\fScanRowIndex\x123.io.debezium.server.iceberg.rpc.RowIndexScanRequest\x1a1.io.debezium.server.iceberg.rpc.RowIndexScanBatch0\x01\x12\x97\x01\n" +
+ "\x16MigrateEqualityDeletes\x12=.io.debezium.server.iceberg.rpc.MigrateEqualityDeletesRequest\x1a>.io.debezium.server.iceberg.rpc.MigrateEqualityDeletesResponseB\x1dB\fRecordIngestZ\riceberg/protob\x06proto3"
var (
file_records_ingest_proto_rawDescOnce sync.Once
diff --git a/destination/iceberg/proto/records_ingest.proto b/destination/iceberg/proto/records_ingest.proto
index da6521b53..9518d051a 100644
--- a/destination/iceberg/proto/records_ingest.proto
+++ b/destination/iceberg/proto/records_ingest.proto
@@ -35,6 +35,10 @@ message IcebergPayload {
// (fed by the caller's row index) instead of equality deleteKey.
bool use_positional_deletes = 9;
repeated PartitionField partition_fields = 10;
+ // Delete representation the writer should use: "eq" (equality deletes),
+ // "pos" (positional delete files) or "dv" (v3 deletion vectors). Empty falls
+ // back to use_positional_deletes so older callers keep working.
+ string delete_mode = 12;
// COMMIT: snapshot the caller's row index is checkpointed at. The server
// refreshes the table and refuses the commit when the tip has moved, so
// positional deletes built from a stale index cannot be published.
@@ -224,6 +228,9 @@ message RowIndexScanBatch {
message MigrateEqualityDeletesRequest {
string thread_id = 1;
+ // Representation the equality deletes should be rewritten into: "pos" or "dv".
+ // Empty means "pos", which is what callers predating deletion vectors expect.
+ string target_mode = 2;
}
message MigrateEqualityDeletesResponse {
diff --git a/destination/iceberg/proto/records_ingest_grpc.pb.go b/destination/iceberg/proto/records_ingest_grpc.pb.go
index e9f8685a8..272c4f634 100644
--- a/destination/iceberg/proto/records_ingest_grpc.pb.go
+++ b/destination/iceberg/proto/records_ingest_grpc.pb.go
@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions:
-// - protoc-gen-go-grpc v1.5.1
-// - protoc v3.21.12
+// - protoc-gen-go-grpc v1.6.2
+// - protoc v7.35.1
// source: records_ingest.proto
package proto
@@ -63,7 +63,7 @@ type RecordIngestServiceServer interface {
type UnimplementedRecordIngestServiceServer struct{}
func (UnimplementedRecordIngestServiceServer) SendRecords(context.Context, *IcebergPayload) (*RecordIngestResponse, error) {
- return nil, status.Errorf(codes.Unimplemented, "method SendRecords not implemented")
+ return nil, status.Error(codes.Unimplemented, "method SendRecords not implemented")
}
func (UnimplementedRecordIngestServiceServer) mustEmbedUnimplementedRecordIngestServiceServer() {}
func (UnimplementedRecordIngestServiceServer) testEmbeddedByValue() {}
@@ -76,7 +76,7 @@ type UnsafeRecordIngestServiceServer interface {
}
func RegisterRecordIngestServiceServer(s grpc.ServiceRegistrar, srv RecordIngestServiceServer) {
- // If the following call pancis, it indicates UnimplementedRecordIngestServiceServer was
+ // If the following call panics, it indicates UnimplementedRecordIngestServiceServer was
// embedded by pointer and is nil. This will cause panics if an
// unimplemented method is ever invoked, so we test this at initialization
// time to prevent it from happening at runtime later due to I/O.
@@ -165,7 +165,7 @@ type ArrowIngestServiceServer interface {
type UnimplementedArrowIngestServiceServer struct{}
func (UnimplementedArrowIngestServiceServer) IcebergAPI(context.Context, *ArrowPayload) (*ArrowIngestResponse, error) {
- return nil, status.Errorf(codes.Unimplemented, "method IcebergAPI not implemented")
+ return nil, status.Error(codes.Unimplemented, "method IcebergAPI not implemented")
}
func (UnimplementedArrowIngestServiceServer) mustEmbedUnimplementedArrowIngestServiceServer() {}
func (UnimplementedArrowIngestServiceServer) testEmbeddedByValue() {}
@@ -178,7 +178,7 @@ type UnsafeArrowIngestServiceServer interface {
}
func RegisterArrowIngestServiceServer(s grpc.ServiceRegistrar, srv ArrowIngestServiceServer) {
- // If the following call pancis, it indicates UnimplementedArrowIngestServiceServer was
+ // If the following call panics, it indicates UnimplementedArrowIngestServiceServer was
// embedded by pointer and is nil. This will cause panics if an
// unimplemented method is ever invoked, so we test this at initialization
// time to prevent it from happening at runtime later due to I/O.
@@ -303,10 +303,10 @@ type RowIndexServiceServer interface {
type UnimplementedRowIndexServiceServer struct{}
func (UnimplementedRowIndexServiceServer) ScanRowIndex(*RowIndexScanRequest, grpc.ServerStreamingServer[RowIndexScanBatch]) error {
- return status.Errorf(codes.Unimplemented, "method ScanRowIndex not implemented")
+ return status.Error(codes.Unimplemented, "method ScanRowIndex not implemented")
}
func (UnimplementedRowIndexServiceServer) MigrateEqualityDeletes(context.Context, *MigrateEqualityDeletesRequest) (*MigrateEqualityDeletesResponse, error) {
- return nil, status.Errorf(codes.Unimplemented, "method MigrateEqualityDeletes not implemented")
+ return nil, status.Error(codes.Unimplemented, "method MigrateEqualityDeletes not implemented")
}
func (UnimplementedRowIndexServiceServer) mustEmbedUnimplementedRowIndexServiceServer() {}
func (UnimplementedRowIndexServiceServer) testEmbeddedByValue() {}
@@ -319,7 +319,7 @@ type UnsafeRowIndexServiceServer interface {
}
func RegisterRowIndexServiceServer(s grpc.ServiceRegistrar, srv RowIndexServiceServer) {
- // If the following call pancis, it indicates UnimplementedRowIndexServiceServer was
+ // If the following call panics, it indicates UnimplementedRowIndexServiceServer was
// embedded by pointer and is nil. This will cause panics if an
// unimplemented method is ever invoked, so we test this at initialization
// time to prevent it from happening at runtime later due to I/O.
diff --git a/destination/iceberg/rowindex.go b/destination/iceberg/rowindex.go
index 412da605b..e082d1aa3 100644
--- a/destination/iceberg/rowindex.go
+++ b/destination/iceberg/rowindex.go
@@ -26,6 +26,9 @@ func (i *Iceberg) reconcileRowIndex(ctx context.Context, index types.TableIndex,
// first check for equality deletes and migrate them to positional deletes
migrated, err := i.server.rowIndexClient.MigrateEqualityDeletes(ctx, &proto.MigrateEqualityDeletesRequest{
ThreadId: i.options.ThreadID,
+ // Rewrite straight into the mode this sync writes, so a table switching
+ // from equality deletes lands on its target representation in one commit.
+ TargetMode: string(i.options.DeleteMode),
})
if err != nil {
return fmt.Errorf("failed to migrate equality deletes of table[%s]: %s", table, err)
diff --git a/destination/writers.go b/destination/writers.go
index 0bdf5ac4a..6efbd0981 100644
--- a/destination/writers.go
+++ b/destination/writers.go
@@ -19,6 +19,8 @@ type (
Backfill bool
ThreadID string
ApplyFilter bool
+ // DeleteMode tells the destination how to represent a superseded row.
+ DeleteMode types.DeleteMode
// RowIndex maps _olake_id to the row's location in the destination table.
// Set only when the destination's delete mode cannot be served without it.
RowIndex types.TableIndex
@@ -182,6 +184,7 @@ func (w *WriterPool) NewWriter(ctx context.Context, stream types.StreamInterface
// Threads of one stream share the stream's index; it is nil in equality mode.
// TODO: can we pass things through contexts like streamContext ?
opts.RowIndex = streamArtifact.rowIndex
+ opts.DeleteMode = w.deleteMode
writerThread, prevStreamState, err := func() (Writer, *types.MetadataState, error) {
// init writer and point it at the config parsed once at pool creation,
diff --git a/protocol/root.go b/protocol/root.go
index 57760debb..db2889729 100644
--- a/protocol/root.go
+++ b/protocol/root.go
@@ -146,7 +146,7 @@ func init() {
RootCmd.PersistentFlags().StringVarP(&destinationDatabasePrefix, "destination-database-prefix", "", "", "(Optional) Destination database prefix is used as prefix for destination database name")
RootCmd.PersistentFlags().Int64VarP(&timeout, "timeout", "", -1, "(Optional) Timeout to override default timeouts (in seconds)")
RootCmd.PersistentFlags().StringVarP(&differencePath, "difference", "", "", "new streams.json file path to be compared. Generates a difference_streams.json file.")
- RootCmd.PersistentFlags().StringVarP(&deleteType, "delete-type", "", "eq", "iceberg delete type that sync job should write to iceberg table supported: eq, pos)")
+ RootCmd.PersistentFlags().StringVarP(&deleteType, "delete-type", "", "eq", "iceberg delete type that sync job should write to iceberg table supported: eq, pos, dv")
// Disable Cobra CLI's built-in usage and error handling
RootCmd.SilenceUsage = true
RootCmd.SilenceErrors = true
diff --git a/types/delete_mode.go b/types/delete_mode.go
index 53d3f6360..4add20d01 100644
--- a/types/delete_mode.go
+++ b/types/delete_mode.go
@@ -13,13 +13,15 @@ const (
// row as (data file, ordinal). Producing them requires a durable
// identifier -> RowLocation index of every live row in the table.
DeleteModePosition DeleteMode = "pos"
- // DeleteModeDeletionVector writes Iceberg v3 deletion vectors.
- // TODO: implement dv writing in Olake (Difficulty: Medium)
+ // DeleteModeDeletionVector writes Iceberg v3 deletion vectors: one Puffin
+ // bitmap per data file rather than a delete file per commit. Addresses rows
+ // the same way positional deletes do, so it needs the same row index, and
+ // requires the destination table to be format version 3.
DeleteModeDeletionVector DeleteMode = "dv"
)
// NeedsRowIndex reports whether the mode can only be served by maintaining a
// TableIndex alongside the destination table.
func (m DeleteMode) NeedsRowIndex(destinationType DestinationType) bool {
- return destinationType == Iceberg && m == DeleteModePosition
+ return destinationType == Iceberg && (m == DeleteModePosition || m == DeleteModeDeletionVector)
}