From a3a98be8a2c056572f5e9c973f0fc012d630b219 Mon Sep 17 00:00:00 2001 From: Som Samantray Date: Thu, 20 Aug 2026 23:51:06 +0530 Subject: [PATCH 1/4] test: add regression test for truncated UTF-8 in UTF8ToUTF32 Adds UnicharTest.TruncatedUtf8 covering 2/3/4-byte truncated prefixes, a mid-string truncated prefix, and an illegal leading continuation byte. The test triggers the out-of-bounds read in UNICHAR::UTF8ToUTF32 on unfixed code (issue #4495). --- unittest/unichar_test.cc | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/unittest/unichar_test.cc b/unittest/unichar_test.cc index e03dad1718..97650c6130 100644 --- a/unittest/unichar_test.cc +++ b/unittest/unichar_test.cc @@ -40,4 +40,21 @@ TEST(UnicharTest, InvalidText) { EXPECT_TRUE(utf8.empty()); } +TEST(UnicharTest, TruncatedUtf8) { + // This test verifies that UTF8ToUTF32 does not read past the end of a + // string that ends with a truncated multibyte prefix (issue #4495). + // A truncated multibyte prefix is invalid UTF-8, so the conversion + // must return an empty vector instead of reading past the NUL. + const char *kTruncated2 = "\xC2\0"; + const char *kTruncated3 = "\xE8\0"; + const char *kTruncated4 = "\xF0\0"; + const char *kTruncatedMid = "ab\xE8\0"; + const char *kIllegalLeading = "\x80\0"; + EXPECT_TRUE(UNICHAR::UTF8ToUTF32(kTruncated2).empty()); + EXPECT_TRUE(UNICHAR::UTF8ToUTF32(kTruncated3).empty()); + EXPECT_TRUE(UNICHAR::UTF8ToUTF32(kTruncated4).empty()); + EXPECT_TRUE(UNICHAR::UTF8ToUTF32(kTruncatedMid).empty()); + EXPECT_TRUE(UNICHAR::UTF8ToUTF32(kIllegalLeading).empty()); +} + } // namespace tesseract From 46f63b7026357b32f1d50bbc3a8000103341a079 Mon Sep 17 00:00:00 2001 From: Som Samantray Date: Thu, 20 Aug 2026 23:51:16 +0530 Subject: [PATCH 2/4] fix: prevent out-of-bounds read in UNICHAR::UTF8ToUTF32 utf8_step() reports the full multibyte width from the leading byte alone, so a truncated prefix (e.g. "\xE8\0") made the iterator read one byte past the end of the string. Clamp the per-character step to the remaining bytes and return an empty vector for truncated input. Fixes #4495. --- src/ccutil/unichar.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/ccutil/unichar.cpp b/src/ccutil/unichar.cpp index ae6d61339f..3d86514864 100644 --- a/src/ccutil/unichar.cpp +++ b/src/ccutil/unichar.cpp @@ -223,12 +223,17 @@ std::vector UNICHAR::UTF8ToUTF32(const char *utf8_str) { unicodes.reserve(utf8_length); const_iterator end_it(end(utf8_str, utf8_length)); for (const_iterator it(begin(utf8_str, utf8_length)); it != end_it; ++it) { - if (it.is_legal()) { - unicodes.push_back(*it); - } else { + // Reject a truncated multibyte prefix (issue #4495): utf8_step() reports + // the full width from the leading byte alone, but the string may end + // mid-sequence. Clamp the step to the remaining bytes so the iterator + // never reads past the end of the string. + const int remaining = end_it.utf8_data() - it.utf8_data(); + const int step = utf8_step(it.utf8_data()); + if (step <= 0 || step > remaining) { unicodes.clear(); return unicodes; } + unicodes.push_back(*it); } return unicodes; } From 6314b421576f800b319911ad497a1fc8ff9df7f1 Mon Sep 17 00:00:00 2001 From: Som Samantray Date: Fri, 21 Aug 2026 00:27:14 +0530 Subject: [PATCH 3/4] refactor: clarify comment in UTF8ToUTF32 truncation guard Reword the comment to describe the rejection behavior accurately (reject a truncated trailing sequence) rather than "clamping", per code review feedback. --- src/ccutil/unichar.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/ccutil/unichar.cpp b/src/ccutil/unichar.cpp index 3d86514864..f7a98dbc8b 100644 --- a/src/ccutil/unichar.cpp +++ b/src/ccutil/unichar.cpp @@ -223,10 +223,9 @@ std::vector UNICHAR::UTF8ToUTF32(const char *utf8_str) { unicodes.reserve(utf8_length); const_iterator end_it(end(utf8_str, utf8_length)); for (const_iterator it(begin(utf8_str, utf8_length)); it != end_it; ++it) { - // Reject a truncated multibyte prefix (issue #4495): utf8_step() reports - // the full width from the leading byte alone, but the string may end - // mid-sequence. Clamp the step to the remaining bytes so the iterator - // never reads past the end of the string. + // utf8_step() reports the width from the leading byte alone; reject a + // truncated trailing sequence rather than let the iterator run past the + // end of the string (issue #4495). const int remaining = end_it.utf8_data() - it.utf8_data(); const int step = utf8_step(it.utf8_data()); if (step <= 0 || step > remaining) { From 4f3590e8a244861ef92612555f018146912dcfe3 Mon Sep 17 00:00:00 2001 From: Som Samantray Date: Wed, 26 Aug 2026 07:41:16 +0530 Subject: [PATCH 4/4] test: explain truncated UTF-8 terminators (AI-assisted) --- unittest/unichar_test.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unittest/unichar_test.cc b/unittest/unichar_test.cc index 97650c6130..028be7dc18 100644 --- a/unittest/unichar_test.cc +++ b/unittest/unichar_test.cc @@ -45,6 +45,8 @@ TEST(UnicharTest, TruncatedUtf8) { // string that ends with a truncated multibyte prefix (issue #4495). // A truncated multibyte prefix is invalid UTF-8, so the conversion // must return an empty vector instead of reading past the NUL. + // Keep the explicit NULs to make the truncation boundary visible in each + // fixture; without them, the literal terminator is implicit. const char *kTruncated2 = "\xC2\0"; const char *kTruncated3 = "\xE8\0"; const char *kTruncated4 = "\xF0\0";