Commit a8e04adb for openh264

commit a8e04adb69c79757da014007d4694684a64c7b74
Author: Erik Språng <sprang@google.com>
Date:   Tue Jun 30 10:00:56 2026 +0200

    Make sure we allocate enough space for all slices (#3956)

    * Make sure we allocate enough memory for all slices

    * fix compare sign warning

    ---------

    Co-authored-by: Erik Språng <sprang@chromium.org>
    Co-authored-by: Erik Språng <sprang@webrtc.org>

diff --git a/codec/encoder/core/src/encoder_ext.cpp b/codec/encoder/core/src/encoder_ext.cpp
index 71ec0d1a..c67a6ad2 100644
--- a/codec/encoder/core/src/encoder_ext.cpp
+++ b/codec/encoder/core/src/encoder_ext.cpp
@@ -1591,8 +1591,7 @@ int32_t RequestMemorySvc (sWelsEncCtx** ppCtx, SExistingParasetList* pExistingPa
     iLayerBsSize = WELS_ROUND (((3 * fDlp->iVideoWidth * fDlp->iVideoHeight) >> 1) * fCompressRatioThr) +
                    MAX_MACROBLOCK_SIZE_IN_BYTE_x2;
     iLayerBsSize = WELS_ALIGN (iLayerBsSize, 4); // 4 bytes alinged
-    iVclLayersBsSizeCount += iLayerBsSize;
-
+    int32_t iMaxLayerBsSize = 0;
     SSliceArgument* pSliceArgument = & (fDlp->sSliceArgument);
     if (pSliceArgument->uiSliceMode == SM_SIZELIMITED_SLICE) {
       bDynamicSlice = true;
@@ -1601,15 +1600,19 @@ int32_t RequestMemorySvc (sWelsEncCtx** ppCtx, SExistingParasetList* pExistingPa
       (*ppCtx)->iMaxSliceCount = WELS_MAX ((*ppCtx)->iMaxSliceCount, (int) uiMaxSliceNumEstimation);
       iSliceBufferSize = (WELS_MAX (pSliceArgument->uiSliceSizeConstraint,
                                     iLayerBsSize / uiMaxSliceNumEstimation) << 1) + MAX_MACROBLOCK_SIZE_IN_BYTE_x2;
+      iMaxLayerBsSize = iSliceBufferSize * uiMaxSliceNumEstimation;
     } else {
       (*ppCtx)->iMaxSliceCount = WELS_MAX ((*ppCtx)->iMaxSliceCount, (int) pSliceArgument->uiSliceNum);
       iSliceBufferSize = ((iLayerBsSize / pSliceArgument->uiSliceNum) << 1) + MAX_MACROBLOCK_SIZE_IN_BYTE_x2;
+      iMaxLayerBsSize = iSliceBufferSize * pSliceArgument->uiSliceNum;
     }
+    iMaxLayerBsSize = WELS_MAX (iMaxLayerBsSize, iLayerBsSize);
+    iVclLayersBsSizeCount += iMaxLayerBsSize;
     iMaxSliceBufferSize                = WELS_MAX (iMaxSliceBufferSize, iSliceBufferSize);
     (*ppCtx)->iSliceBufferSize[iIndex] = iSliceBufferSize;
     ++ iIndex;
   }
-  iTargetSpatialBsSize = iLayerBsSize;
+  iTargetSpatialBsSize = iVclLayersBsSizeCount;
   iCountBsLen = iNonVclLayersBsSizeCount + iVclLayersBsSizeCount;

   iMaxSliceBufferSize = WELS_MIN (iMaxSliceBufferSize, iTargetSpatialBsSize);
@@ -3738,6 +3741,7 @@ int32_t WelsEncoderEncodeExt (sWelsEncCtx* pCtx, SFrameBSInfo* pFbi, const SSour
         }

         iLayerSize = AppendSliceToFrameBs (pCtx, pLayerBsInfo, iSliceCount);
+        WELS_VERIFY_RETURN_IFNEQ (pCtx->iEncoderError, ENC_RETURN_SUCCESS)
       }
       // THREAD_FULLY_FIRE_MODE && SM_SIZELIMITED_SLICE
       else if ((SM_SIZELIMITED_SLICE == pParam->sSliceArgument.uiSliceMode) && (pSvcParam->iMultipleThreadIdc > 1)) {
@@ -3787,6 +3791,7 @@ int32_t WelsEncoderEncodeExt (sWelsEncCtx* pCtx, SFrameBSInfo* pFbi, const SSour

         iSliceCount = GetCurrentSliceNum (pCtx->pCurDqLayer);
         iLayerSize  = AppendSliceToFrameBs (pCtx, pLayerBsInfo, iSliceCount);
+        WELS_VERIFY_RETURN_IFNEQ (pCtx->iEncoderError, ENC_RETURN_SUCCESS)
       } else { // for non-dynamic-slicing mode single threading branch..
         const bool bNeedPrefix = pCtx->bNeedPrefixNalFlag;
         int32_t iSliceIdx    = 0;
diff --git a/codec/encoder/core/src/slice_multi_threading.cpp b/codec/encoder/core/src/slice_multi_threading.cpp
index da4fe3d6..09ea2c08 100644
--- a/codec/encoder/core/src/slice_multi_threading.cpp
+++ b/codec/encoder/core/src/slice_multi_threading.cpp
@@ -444,6 +444,15 @@ int32_t AppendSliceToFrameBs (sWelsEncCtx* pCtx, SLayerBSInfo* pLbi, const int32
       assert (pSliceBs->bSliceCodedFlag);
 #endif//MT_DEBUG_BS_WR

+      if (static_cast<uint64_t>(pCtx->iPosBsBuffer) + pSliceBs->uiBsPos > static_cast<uint64_t>(pCtx->iFrameBsSize)) {
+        WelsLog (&pCtx->sLogCtx, WELS_LOG_ERROR,
+                 "AppendSliceToFrameBs(), insufficient memory for the allocation! "
+                 "iPosBsBuffer:%d, uiBsPos:%d, iFrameBsSize:%d",
+                 pCtx->iPosBsBuffer, pSliceBs->uiBsPos, pCtx->iFrameBsSize);
+        pCtx->iEncoderError |= ENC_RETURN_MEMALLOCERR;
+        return 0;
+      }
+
       memmove (pCtx->pFrameBs + pCtx->iPosBsBuffer, pSliceBs->pBs, pSliceBs->uiBsPos); // confirmed_safe_unsafe_usage
       pCtx->iPosBsBuffer += pSliceBs->uiBsPos;

diff --git a/test/api/encoder_test.cpp b/test/api/encoder_test.cpp
index 6f0c4657..764d0257 100644
--- a/test/api/encoder_test.cpp
+++ b/test/api/encoder_test.cpp
@@ -1,7 +1,9 @@
 #include <gtest/gtest.h>
 #include "utils/HashFunctions.h"
 #include "BaseEncoderTest.h"
+#include "utils/BufferedData.h"
 #include <string>
+#include <cstdlib>

 static void UpdateHashFromFrame (const SFrameBSInfo& info, SHA1Context* ctx) {
   for (int i = 0; i < info.iLayerNum; ++i) {
@@ -181,3 +183,79 @@ static const EncodeFileParam kFileParamArray[] = {

 INSTANTIATE_TEST_CASE_P (EncodeFile, EncoderOutputTest,
                          ::testing::ValuesIn (kFileParamArray));
+
+class RandomInputStream : public InputStream {
+ public:
+  RandomInputStream(int max_frames) : max_frames_(max_frames), count_(0) {
+    srand(12345);
+  }
+  virtual int read (void* ptr, size_t len) {
+    if (count_ >= max_frames_) {
+      return 0;
+    }
+    unsigned char* p = (unsigned char*)ptr;
+    for (size_t i = 0; i < len; ++i) {
+      p[i] = rand() % 256;
+    }
+    count_++;
+    return static_cast<int>(len);
+  }
+ private:
+  int max_frames_;
+  int count_;
+};
+
+// This test uses random noise input and a very low QP to intentionally create
+// poor compression (less than 1:1 ratio). This produces very large slices
+// and tests the encoder's ability to handle inflated buffers correctly without
+// insufficient memory errors.
+TEST_F(EncoderInitTest, VeryLargeSlices) {
+  SEncParamExt param;
+  encoder_->GetDefaultParams(&param);
+
+  param.iUsageType = CAMERA_VIDEO_REAL_TIME;
+  param.iPicWidth = 1280;
+  param.iPicHeight = 720;
+  param.fMaxFrameRate = 30.0f;
+  param.iSpatialLayerNum = 1;
+  param.iMultipleThreadIdc = 4;
+  param.iRCMode = RC_OFF_MODE;
+
+  param.sSpatialLayers[0].iVideoWidth = param.iPicWidth;
+  param.sSpatialLayers[0].iVideoHeight = param.iPicHeight;
+  param.sSpatialLayers[0].fFrameRate = param.fMaxFrameRate;
+  param.sSpatialLayers[0].sSliceArgument.uiSliceMode = SM_FIXEDSLCNUM_SLICE;
+  param.sSpatialLayers[0].sSliceArgument.uiSliceNum = 4;
+  param.sSpatialLayers[0].iDLayerQp = 12;
+  param.iMinQp = 0;
+  param.iMaxQp = 51;
+
+  int rv = encoder_->InitializeExt(&param);
+  ASSERT_EQ(0, rv);
+
+  RandomInputStream stream(1);
+
+  int frameSize = param.iPicWidth * param.iPicHeight * 3 / 2;
+  BufferedData buf;
+  buf.SetLength(frameSize);
+  ASSERT_EQ(buf.Length(), (size_t)frameSize);
+
+  SFrameBSInfo info;
+  memset(&info, 0, sizeof(SFrameBSInfo));
+
+  SSourcePicture pic;
+  memset(&pic, 0, sizeof(SSourcePicture));
+  pic.iPicWidth = param.iPicWidth;
+  pic.iPicHeight = param.iPicHeight;
+  pic.iColorFormat = videoFormatI420;
+  pic.iStride[0] = pic.iPicWidth;
+  pic.iStride[1] = pic.iStride[2] = pic.iPicWidth >> 1;
+  pic.pData[0] = buf.data();
+  pic.pData[1] = pic.pData[0] + param.iPicWidth * param.iPicHeight;
+  pic.pData[2] = pic.pData[1] + (param.iPicWidth * param.iPicHeight >> 2);
+
+  while (stream.read(buf.data(), frameSize) == frameSize) {
+    rv = encoder_->EncodeFrame(&pic, &info);
+    ASSERT_EQ(0, rv);
+  }
+}