From e37912a3760c9f0bc24eff2475ebb9c64d032e8c Mon Sep 17 00:00:00 2001
From: Matt Books <matt.books@gusto.com>
Date: Mon, 14 Jul 2025 15:43:12 -0700
Subject: [PATCH 1/2] Fix conversion bug when string data comes from binlog

Fixes https://github.com/github/gh-ost/issues/1568

When using gh-ost to migrate a table from latin1 to utf8mb3 character encoding, the initial data copy works correctly, but new data with special characters inserted during the migration via binlog replication fails with "Incorrect string value" errors.

The reason for this is that the data is a binary byte array when
converted from the binlog, so the character set conversion is not
applied.

This fix updates the character set conversion logic to apply to both
string and []uint8 types when the column has a character set conversion.

I added a new test for latin1 input to this method and confirmed that
the reproduction from the linked issue is fixed with this change.
---
 go/sql/types.go      | 14 +++++++++++---
 go/sql/types_test.go | 16 ++++++++++++++++
 2 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/go/sql/types.go b/go/sql/types.go
index 6ecf75881..aac52bc32 100644
--- a/go/sql/types.go
+++ b/go/sql/types.go
@@ -53,14 +53,22 @@ type Column struct {
 }
 
 func (this *Column) convertArg(arg interface{}, isUniqueKeyColumn bool) interface{} {
+	var arg2Bytes []byte
 	if s, ok := arg.(string); ok {
-		arg2Bytes := []byte(s)
-		// convert to bytes if character string without charsetConversion.
+		arg2Bytes = []byte(s)
+	} else if b, ok := arg.([]uint8); ok {
+		arg2Bytes = b
+	} else {
+		arg2Bytes = nil
+	}
+
+	if arg2Bytes != nil {
 		if this.Charset != "" && this.charsetConversion == nil {
 			arg = arg2Bytes
 		} else {
 			if encoding, ok := charsetEncodingMap[this.Charset]; ok {
-				arg, _ = encoding.NewDecoder().String(s)
+				decodedBytes, _ := encoding.NewDecoder().Bytes(arg2Bytes)
+				arg = string(decodedBytes)
 			}
 		}
 
diff --git a/go/sql/types_test.go b/go/sql/types_test.go
index 5d71070c2..baa84120b 100644
--- a/go/sql/types_test.go
+++ b/go/sql/types_test.go
@@ -49,3 +49,19 @@ func TestBinaryToString(t *testing.T) {
 
 	require.Equal(t, "1b99", cv.StringColumn(0))
 }
+
+func TestConvertArgCharsetDecoding(t *testing.T) {
+	latin1Bytes := []uint8{0x47, 0x61, 0x72, 0xe7, 0x6f, 0x6e, 0x20, 0x21}
+
+	col := Column{
+		Charset: "latin1",
+		charsetConversion: &CharacterSetConversion{
+			FromCharset: "latin1",
+			ToCharset:   "utf8mb4",
+		},
+	}
+
+	// Should decode []uint8
+	str := col.convertArg(latin1Bytes, false)
+	require.Equal(t, "Garçon !", str)
+}
\ No newline at end of file

From 21da86eb7c56f35848738680d3af52a9f5924b3c Mon Sep 17 00:00:00 2001
From: Matt Books <matt.books@gusto.com>
Date: Wed, 23 Jul 2025 11:42:54 -0700
Subject: [PATCH 2/2] Update whitespace formatting

---
 go/sql/types_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/go/sql/types_test.go b/go/sql/types_test.go
index baa84120b..7b808e64f 100644
--- a/go/sql/types_test.go
+++ b/go/sql/types_test.go
@@ -64,4 +64,4 @@ func TestConvertArgCharsetDecoding(t *testing.T) {
 	// Should decode []uint8
 	str := col.convertArg(latin1Bytes, false)
 	require.Equal(t, "Garçon !", str)
-}
\ No newline at end of file
+}