Commit 24ff6e94 for openh264

commit 24ff6e94c5cea94bb99ff4f30b5967b0422c4bee
Author: BenzhengZhang <140143892+BenzhengZhang@users.noreply.github.com>
Date:   Mon Aug 10 16:13:59 2026 +0800

    encoder: validate geometry area using widened arithmetic (#3976)

    * encoder: validate geometry area using widened arithmetic

    * test: skip PartialInitLeakUAF when thread limits are not enforceable

    ---------

    Co-authored-by: benzzhan <benzzhan@cisco.com>

diff --git a/codec/encoder/core/src/encoder_ext.cpp b/codec/encoder/core/src/encoder_ext.cpp
index e7bbf7e2..a3d9d8b7 100644
--- a/codec/encoder/core/src/encoder_ext.cpp
+++ b/codec/encoder/core/src/encoder_ext.cpp
@@ -496,6 +496,7 @@ int32_t ParamValidationExt (SLogContext* pLogCtx, SWelsSvcCodingParam* pCodingPa
     SSpatialLayerConfig* pSpatialLayer = &pCodingParam->sSpatialLayers[i];
     int32_t kiPicWidth = pSpatialLayer->iVideoWidth;
     int32_t kiPicHeight = pSpatialLayer->iVideoHeight;
+    const int64_t kiMaxPixelsPerFrame = (static_cast<int64_t> (MAX_MBS_PER_FRAME) << 8);
     uint32_t iMbWidth           = 0;
     uint32_t iMbHeight          = 0;
     int32_t iMbNumInFrame       = 0;
@@ -512,7 +513,8 @@ int32_t ParamValidationExt (SLogContext* pLogCtx, SWelsSvcCodingParam* pCodingPa
                pSpatialLayer->iVideoWidth, pSpatialLayer->iVideoHeight);
     }

-    if ((kiPicWidth <= 0) || (kiPicHeight <= 0) || (kiPicWidth * kiPicHeight > (MAX_MBS_PER_FRAME << 8))) {
+    if ((kiPicWidth <= 0) || (kiPicHeight <= 0)
+        || (static_cast<int64_t> (kiPicWidth) * kiPicHeight > kiMaxPixelsPerFrame)) {
       WelsLog (pLogCtx, WELS_LOG_ERROR,
                "ParamValidationExt(), width > 0, height > 0, width * height <= %d, invalid %d x %d in dependency layer settings!",
                (MAX_MBS_PER_FRAME << 8), kiPicWidth, kiPicHeight);
diff --git a/test/common/WelsThreadPoolTest.cpp b/test/common/WelsThreadPoolTest.cpp
index f3e39286..c4052a85 100644
--- a/test/common/WelsThreadPoolTest.cpp
+++ b/test/common/WelsThreadPoolTest.cpp
@@ -134,6 +134,10 @@ TEST_F(CThreadPoolTestFixture, PartialInitLeakUAF) {
   // triggering a controlled partial initialization failure.
   CWelsThreadPool::SetThreadNum(2);
   CWelsThreadPool* pPool = CWelsThreadPool::AddReference();
+  if (pPool != NULL) {
+    pPool->RemoveInstance();
+    GTEST_SKIP() << "Thread limit did not force partial initialization failure in this environment.";
+  }
   EXPECT_EQ(NULL, pPool);

   // Restore old rlimit immediately so we can create threads/signals normally.
diff --git a/test/encoder/EncUT_EncoderExt.cpp b/test/encoder/EncUT_EncoderExt.cpp
index 4168a07b..69293fde 100644
--- a/test/encoder/EncUT_EncoderExt.cpp
+++ b/test/encoder/EncUT_EncoderExt.cpp
@@ -2,11 +2,20 @@
 #include <stdlib.h>
 #include "codec_api.h"
 #include "codec_app_def.h"
+#include "extern.h"
+#include "param_svc.h"
 #include "svc_enc_slice_segment.h"
 #include "test_stdint.h"
 #include "utils/FileInputStream.h"
 //TODO: consider using BaseEncoderTest class from #include "../BaseEncoderTest.h"

+static void NoopLogCallback (void* pCtx, const int32_t iLevel, const char* kpFmt, va_list argv) {
+  (void)pCtx;
+  (void)iLevel;
+  (void)kpFmt;
+  (void)argv;
+}
+
 class EncoderInterfaceTest : public ::testing::Test {
 #define MB_SIZE (16)
 #define MAX_WIDTH (3840)
@@ -612,6 +621,31 @@ TEST_F (EncoderInterfaceTest, BasicInitializeTestFalse) {
   pPtrEnc->SetOption (ENCODER_OPTION_TRACE_LEVEL, &uiTraceLevel);
 }

+TEST_F (EncoderInterfaceTest, InitializeExtRejectsGeometryOverflow) {
+  WelsEnc::SWelsSvcCodingParam sCodingParam;
+  sCodingParam.FillDefault();
+
+  // 46368 * 46368 overflows signed int32_t, so this must be validated using
+  // widened arithmetic in ParamValidationExt().
+  sCodingParam.iPicWidth = 46368;
+  sCodingParam.iPicHeight = 46368;
+  sCodingParam.iSpatialLayerNum = 1;
+  sCodingParam.iTemporalLayerNum = 1;
+  sCodingParam.iTargetBitrate = 500000;
+  sCodingParam.fMaxFrameRate = 30.0f;
+  sCodingParam.sSpatialLayers[0].iVideoWidth = sCodingParam.iPicWidth;
+  sCodingParam.sSpatialLayers[0].iVideoHeight = sCodingParam.iPicHeight;
+  sCodingParam.sSpatialLayers[0].iSpatialBitrate = sCodingParam.iTargetBitrate;
+  sCodingParam.sSpatialLayers[0].fFrameRate = sCodingParam.fMaxFrameRate;
+
+  SLogContext sLogCtx;
+  memset (&sLogCtx, 0, sizeof (sLogCtx));
+  sLogCtx.pfLog = NoopLogCallback;
+
+  int iResult = WelsEnc::ParamValidationExt (&sLogCtx, &sCodingParam);
+  EXPECT_EQ (iResult, static_cast<int> (ENC_RETURN_UNSUPPORTED_PARA));
+}
+
 TEST_F (EncoderInterfaceTest, BasicInitializeTestAutoAdjustment) {
   SEncParamBase sEncParamBase;