Commit e07bbb8f for tesseract

commit e07bbb8f75a256515e39fcb4de10241adb552421
Author: Stefan Weil <sw@weilnetz.de>
Date:   Tue Aug 25 21:56:03 2026 +0200

    Stop UTF-8 span scans at truncated sequences

    SpanUTF8Whitespace and SpanUTF8NotWhitespace iterated with
    UNICHAR::const_iterator, which derives the character width from the
    leading byte alone. A multibyte sequence truncated at the end of the
    string (for example "ab\xE8") made *it read past the NUL terminator
    and ++it run past the end of the string: an out-of-bounds read that
    only stopped at a crash.

    Both functions now step the bytes manually: a truncated trailing
    sequence ends the whitespace span and is counted as non-whitespace
    bytes (never more than the bytes actually present), and an illegal
    leading byte keeps the previous iterator semantics (one whitespace
    byte, and a boundary for the non-whitespace span). Valid UTF-8 input
    is handled exactly as before.

    Key changes:
    - normstrngs.cpp: rewrite both span scans with explicit width checks
      against the remaining bytes.
    - unittest: add SpanUTF8TruncatedPrefix covering 2/3/4-byte truncated
      prefixes after leading spaces and illegal leading bytes. On unpatched
      code the test dies on an ASan global-buffer-overflow read in
      UNICHAR::utf8_step via const_iterator::operator*.

    Assisted-by: OpenCode / qwen3.8-27b-thinking (Alibaba Cloud)
    Signed-off-by: Stefan Weil <sw@weilnetz.de>

diff --git a/src/training/unicharset/normstrngs.cpp b/src/training/unicharset/normstrngs.cpp
index c172bec6..e0bb4c56 100644
--- a/src/training/unicharset/normstrngs.cpp
+++ b/src/training/unicharset/normstrngs.cpp
@@ -243,24 +243,54 @@ bool IsUTF8Whitespace(const char *text) {

 unsigned int SpanUTF8Whitespace(const char *text) {
   int n_white = 0;
-  for (UNICHAR::const_iterator it = UNICHAR::begin(text, strlen(text));
-       it != UNICHAR::end(text, strlen(text)); ++it) {
-    if (!IsWhitespace(*it)) {
+  const char *p = text;
+  const char *const end = p + strlen(text);
+  while (p < end) {
+    const int step = UNICHAR::utf8_step(p);
+    if (step <= 0) {
+      // The iterator maps an illegal leading byte to a space (one byte).
+      ++n_white;
+      ++p;
+      continue;
+    }
+    if (step > end - p) {
+      // A multibyte sequence truncated at the end of the string is not a
+      // whitespace character. Stopping here also avoids reading past the NUL
+      // terminator.
+      break;
+    }
+    if (!IsWhitespace(UNICHAR(p, step).first_uni())) {
       break;
     }
-    n_white += it.utf8_len();
+    n_white += step;
+    p += step;
   }
   return n_white;
 }

 unsigned int SpanUTF8NotWhitespace(const char *text) {
   int n_notwhite = 0;
-  for (UNICHAR::const_iterator it = UNICHAR::begin(text, strlen(text));
-       it != UNICHAR::end(text, strlen(text)); ++it) {
-    if (IsWhitespace(*it)) {
+  const char *p = text;
+  const char *const end = p + strlen(text);
+  while (p < end) {
+    const int step = UNICHAR::utf8_step(p);
+    if (step <= 0) {
+      // The iterator maps an illegal leading byte to a space, which ends the
+      // span of non-whitespace.
+      break;
+    }
+    if (step > end - p) {
+      // A multibyte sequence truncated at the end of the string is not
+      // whitespace. Count the remaining bytes without reading past the NUL
+      // terminator.
+      n_notwhite += static_cast<int>(end - p);
+      break;
+    }
+    if (IsWhitespace(UNICHAR(p, step).first_uni())) {
       break;
     }
-    n_notwhite += it.utf8_len();
+    n_notwhite += step;
+    p += step;
   }
   return n_notwhite;
 }
diff --git a/unittest/normstrngs_test.cc b/unittest/normstrngs_test.cc
index 26b050f6..336f11dd 100644
--- a/unittest/normstrngs_test.cc
+++ b/unittest/normstrngs_test.cc
@@ -348,6 +348,27 @@ TEST(NormstrngsTest, SpanUTF8NotWhitespace) {
   EXPECT_EQ(12, SpanUTF8NotWhitespace(kMixedText));
 }

+TEST(NormstrngsTest, SpanUTF8TruncatedPrefix) {
+  // A multibyte sequence truncated at the end of the string must not be
+  // read past the NUL terminator (issue #4495).
+  // 2-, 3- and 4-byte prefixes truncated at the terminator.
+  EXPECT_EQ(0, SpanUTF8Whitespace("\xC2"));
+  EXPECT_EQ(0, SpanUTF8Whitespace("\xE8"));
+  EXPECT_EQ(0, SpanUTF8Whitespace("\xF0"));
+  // A truncated prefix is not whitespace, even after leading spaces.
+  EXPECT_EQ(2, SpanUTF8Whitespace("  \xE8"));
+  EXPECT_FALSE(IsUTF8Whitespace(" \xE8"));
+  // A truncated prefix is counted as non-whitespace.
+  EXPECT_EQ(3, SpanUTF8NotWhitespace("ab\xE8"));
+  EXPECT_EQ(1, SpanUTF8NotWhitespace("\xC2"));
+  // An illegal leading byte keeps the previous semantics: one whitespace
+  // byte for the whitespace span, and a boundary for the other.
+  EXPECT_EQ(1, SpanUTF8Whitespace("\x80"));
+  EXPECT_EQ(2, SpanUTF8Whitespace("\x80 "));
+  EXPECT_EQ(0, SpanUTF8NotWhitespace("\x80" "abc"));
+  EXPECT_EQ(3, SpanUTF8NotWhitespace("abc\x80"));
+}
+
 // Test that the method clones the util/utf8/unilib definition of
 // interchange validity.
 TEST(NormstrngsTest, IsInterchangeValid) {