Commit 2f4d2f4b for tesseract

commit 2f4d2f4bf45c363785d7bf1da29b6628f8939a72
Author: Stefan Weil <sw@weilnetz.de>
Date:   Thu Jul 23 18:05:39 2026 +0200

    Fix integer overflow in LSTM Convolve and Reconfig deserialization (#4588)

    Add range and overflow validation in Convolve::DeSerialize and
    Reconfig::DeSerialize to prevent a crafted .traineddata file from
    triggering a heap out-of-bounds write via unchecked signed integer
    multiplication when computing the output-channel count.

    Validate ni/no/num_weights in Network::CreateFromFile.

    Add defense-in-depth bounds assertions in NetworkIO::Randomize and
    NetworkIO::CopyTimeStepGeneral.

    Reported-by: Eunho Kim <eunhok98@gmail.com>
    Signed-off-by: Stefan Weil <sw@weilnetz.de>
    Assisted-by: OpenCode / big-pickle (opencode)
    Tested-by: Eunho Kim <eunhok98@gmail.com>

diff --git a/src/lstm/convolve.cpp b/src/lstm/convolve.cpp
index 23c59313..66c126d0 100644
--- a/src/lstm/convolve.cpp
+++ b/src/lstm/convolve.cpp
@@ -23,8 +23,11 @@

 #include "convolve.h"

+#include <cstdint>
+
 #include "networkscratch.h"
 #include "serialis.h"
+#include "tprintf.h"

 namespace tesseract {

@@ -46,7 +49,26 @@ bool Convolve::DeSerialize(TFile *fp) {
   if (!fp->DeSerialize(&half_y_)) {
     return false;
   }
-  no_ = ni_ * (2 * half_x_ + 1) * (2 * half_y_ + 1);
+  if (half_x_ < 0 || half_y_ < 0 || ni_ <= 0) {
+    tprintf("Error: invalid Convolve parameters: ni=%d half_x=%d half_y=%d\n", ni_, half_x_,
+            half_y_);
+    return false;
+  }
+  int64_t kx = 2LL * half_x_ + 1;
+  int64_t ky = 2LL * half_y_ + 1;
+  // Stepwise overflow check: ni_ * kx * ky must fit in int.
+  if (kx > INT_MAX / ky) {
+    tprintf("Error: Convolve output-channel count overflows: ni=%d half_x=%d half_y=%d\n", ni_,
+            half_x_, half_y_);
+    return false;
+  }
+  int64_t kxky = kx * ky;
+  if (static_cast<int64_t>(ni_) > INT_MAX / kxky) {
+    tprintf("Error: Convolve output-channel count overflows: ni=%d half_x=%d half_y=%d\n", ni_,
+            half_x_, half_y_);
+    return false;
+  }
+  no_ = static_cast<int>(static_cast<int64_t>(ni_) * kxky);
   return true;
 }

diff --git a/src/lstm/network.cpp b/src/lstm/network.cpp
index 2b9c4d0c..7a3b7973 100644
--- a/src/lstm/network.cpp
+++ b/src/lstm/network.cpp
@@ -247,6 +247,12 @@ Network *Network::CreateFromFile(TFile *fp) {
     return nullptr;
   }

+  if (ni < 0 || no < 0 || num_weights < 0) {
+    tprintf("Error: invalid network layer parameters: type=%d ni=%d no=%d num_weights=%d\n", type,
+            ni, no, num_weights);
+    return nullptr;
+  }
+
   switch (type) {
     case NT_CONVOLVE:
       network = new Convolve(name, ni, 0, 0);
diff --git a/src/lstm/networkio.cpp b/src/lstm/networkio.cpp
index f8718f27..8e1c6679 100644
--- a/src/lstm/networkio.cpp
+++ b/src/lstm/networkio.cpp
@@ -405,6 +405,7 @@ void NetworkIO::CopyTimeStepFrom(int dest_t, const NetworkIO &src, int src_t) {
 void NetworkIO::CopyTimeStepGeneral(int dest_t, int dest_offset, int num_features,
                                     const NetworkIO &src, int src_t, int src_offset) {
   ASSERT_HOST(int_mode_ == src.int_mode_);
+  ASSERT_HOST(dest_offset + num_features <= NumFeatures());
   if (int_mode_) {
     memcpy(i_[dest_t] + dest_offset, src.i_[src_t] + src_offset, num_features * sizeof(i_[0][0]));
   } else {
@@ -414,6 +415,7 @@ void NetworkIO::CopyTimeStepGeneral(int dest_t, int dest_offset, int num_feature

 // Sets the given range to random values.
 void NetworkIO::Randomize(int t, int offset, int num_features, TRand *randomizer) {
+  ASSERT_HOST(offset + num_features <= NumFeatures());
   if (int_mode_) {
     int8_t *line = i_[t] + offset;
     for (int i = 0; i < num_features; ++i) {
diff --git a/src/lstm/reconfig.cpp b/src/lstm/reconfig.cpp
index a4bda0e5..348ad60a 100644
--- a/src/lstm/reconfig.cpp
+++ b/src/lstm/reconfig.cpp
@@ -18,6 +18,10 @@

 #include "reconfig.h"

+#include <cstdint>
+
+#include "tprintf.h"
+
 namespace tesseract {

 Reconfig::Reconfig(const std::string &name, int ni, int x_scale, int y_scale)
@@ -60,7 +64,21 @@ bool Reconfig::DeSerialize(TFile *fp) {
   if (!fp->DeSerialize(&y_scale_)) {
     return false;
   }
-  no_ = ni_ * x_scale_ * y_scale_;
+  if (x_scale_ <= 0 || y_scale_ <= 0 || ni_ <= 0) {
+    tprintf("Error: invalid Reconfig parameters: ni=%d x_scale=%d y_scale=%d\n", ni_, x_scale_,
+            y_scale_);
+    return false;
+  }
+  int64_t xs = x_scale_;
+  int64_t ys = y_scale_;
+  // Stepwise overflow check: ni_ * x_scale_ * y_scale_ must fit in int.
+  int64_t xsys = xs * ys;
+  if (static_cast<int64_t>(ni_) > INT_MAX / xsys) {
+    tprintf("Error: Reconfig output-channel count overflows: ni=%d x_scale=%d y_scale=%d\n", ni_,
+            x_scale_, y_scale_);
+    return false;
+  }
+  no_ = static_cast<int>(static_cast<int64_t>(ni_) * xsys);
   return true;
 }