Commit 9547f96e for openh264
commit 9547f96ea5c47f4d465d97b07264b997ecc4b4b6
Author: Erik Språng <sprang@google.com>
Date: Thu Aug 27 11:14:52 2026 +0200
rate control: prevent signed integer overflow in target bit allocation (#3982)
When encoding IDR frames with high bitrates (large iBitsPerFrame) and
non-zero iIdrNum, RcDecideTargetBits calculates:
pWelsSvcRc->iBitsPerFrame * pEncCtx->pSvcParam->iIdrBitrateRatio / 100
Since both iBitsPerFrame and iIdrBitrateRatio (default 400) are 32-bit
signed integers, their product can overflow INT32_MAX (e.g. 7064092 * 400 =
2825636800 > 2147483647), leading to undefined behavior and UBSan crashes.
This was introduced in #3514 when configurable IDR bitrate ratios were added.
Similarly, intermediate multiplications when calculating iQStep
(iTargetBits * INT_MULTIPLY) and kiGopBits in ratectl.cpp can overflow 32-bit
integers with high bitrates.
This change widens intermediate calculations to 64-bit integers and adds a
unit test (HighBitrateIdrTargetBits) covering high bitrate IDR encoding.
Co-authored-by: Erik Språng <sprang@webrtc.org>
diff --git a/codec/encoder/core/src/ratectl.cpp b/codec/encoder/core/src/ratectl.cpp
index 40af74d9..e9c2f3d3 100644
--- a/codec/encoder/core/src/ratectl.cpp
+++ b/codec/encoder/core/src/ratectl.cpp
@@ -220,7 +220,7 @@ void RcUpdateBitrateFps (sWelsEncCtx* pEncCtx) {
const int32_t kiHighestTid = pDLayerParamInternal->iHighestTemporalId;
const int32_t input_iBitsPerFrame = WELS_DIV_ROUND (pDLayerParam->iSpatialBitrate,
pDLayerParamInternal->fOutputFrameRate);
- const int64_t kiGopBits = input_iBitsPerFrame * kiGopSize;
+ const int64_t kiGopBits = static_cast<int64_t> (input_iBitsPerFrame) * kiGopSize;
int32_t i;
pWelsSvcRc->iBitRate = pDLayerParam->iSpatialBitrate;
@@ -452,15 +452,15 @@ void RcCalculateIdrQp (sWelsEncCtx* pEncCtx) {
//obtain the idr qp using previous idr complexity
if (pWelsSvcRc->iNumberMbFrame != pWelsSvcRc->iIntraMbCount) {
- pWelsSvcRc->iIntraComplexity = pWelsSvcRc->iIntraComplexity * pWelsSvcRc->iNumberMbFrame /
- pWelsSvcRc->iIntraMbCount;
+ pWelsSvcRc->iIntraComplexity = static_cast<int32_t> (static_cast<int64_t> (pWelsSvcRc->iIntraComplexity) * pWelsSvcRc->iNumberMbFrame /
+ pWelsSvcRc->iIntraMbCount);
}
int64_t iCmplxRatio = WELS_DIV_ROUND64 (iFrameComplexity * INT_MULTIPLY,
pWelsSvcRc->iIntraComplxMean);
iCmplxRatio = WELS_CLIP3 (iCmplxRatio, INT_MULTIPLY - FRAME_CMPLX_RATIO_RANGE, INT_MULTIPLY + FRAME_CMPLX_RATIO_RANGE);
- pWelsSvcRc->iQStep = WELS_DIV_ROUND ((pWelsSvcRc->iIntraComplexity * iCmplxRatio),
- (pWelsSvcRc->iTargetBits * INT_MULTIPLY));
+ pWelsSvcRc->iQStep = static_cast<int32_t> (WELS_DIV_ROUND64 (static_cast<int64_t> (pWelsSvcRc->iIntraComplexity) * iCmplxRatio,
+ static_cast<int64_t> (pWelsSvcRc->iTargetBits) * INT_MULTIPLY));
pWelsSvcRc->iInitialQp = RcConvertQStep2Qp (pWelsSvcRc->iQStep);
}
@@ -504,7 +504,8 @@ void RcCalculatePictureQp (sWelsEncCtx* pEncCtx) {
pTOverRc->iFrameCmplxMean);
iCmplxRatio = WELS_CLIP3 (iCmplxRatio, INT_MULTIPLY - FRAME_CMPLX_RATIO_RANGE, INT_MULTIPLY + FRAME_CMPLX_RATIO_RANGE);
- pWelsSvcRc->iQStep = WELS_DIV_ROUND ((pTOverRc->iLinearCmplx * iCmplxRatio), (pWelsSvcRc->iTargetBits * INT_MULTIPLY));
+ pWelsSvcRc->iQStep = static_cast<int32_t> (WELS_DIV_ROUND64 (static_cast<int64_t> (pTOverRc->iLinearCmplx) * iCmplxRatio,
+ static_cast<int64_t> (pWelsSvcRc->iTargetBits) * INT_MULTIPLY));
iLumaQp = RcConvertQStep2Qp (pWelsSvcRc->iQStep);
WelsLog (& (pEncCtx->sLogCtx), WELS_LOG_DEBUG,
"iCmplxRatio = %d,frameComplexity = %" PRId64 ",iFrameCmplxMean = %" PRId64 ",iQStep = %d,iLumaQp = %d", (int)iCmplxRatio,
@@ -574,10 +575,10 @@ void RcDecideTargetBits (sWelsEncCtx* pEncCtx) {
const bool fix_rc_overshoot = pEncCtx->pSvcParam->bFixRCOverShoot;
//allocate bits
if (pEncCtx->eSliceType == I_SLICE) {
- if( pWelsSvcRc->iIdrNum != 0 ){
- pWelsSvcRc->iTargetBits = pWelsSvcRc->iBitsPerFrame * pEncCtx->pSvcParam->iIdrBitrateRatio / 100;
+ if (pWelsSvcRc->iIdrNum != 0) {
+ pWelsSvcRc->iTargetBits = static_cast<int32_t> (static_cast<int64_t> (pWelsSvcRc->iBitsPerFrame) * pEncCtx->pSvcParam->iIdrBitrateRatio / 100);
} else {
- pWelsSvcRc->iTargetBits = pWelsSvcRc->iBitsPerFrame * IDR_BITRATE_RATIO;
+ pWelsSvcRc->iTargetBits = static_cast<int32_t> (static_cast<int64_t> (pWelsSvcRc->iBitsPerFrame) * IDR_BITRATE_RATIO);
}
} else {
if (pWelsSvcRc->iRemainingWeights > pTOverRc->iTlayerWeight ||
diff --git a/test/encoder/EncUT_EncoderExt.cpp b/test/encoder/EncUT_EncoderExt.cpp
index eb89660a..46017a68 100644
--- a/test/encoder/EncUT_EncoderExt.cpp
+++ b/test/encoder/EncUT_EncoderExt.cpp
@@ -45,7 +45,8 @@ class EncoderInterfaceTest : public ::testing::Test {
pParamExt = new SEncParamExt();
ASSERT_TRUE (pParamExt != NULL);
- pSrcPic = new SSourcePicture;
+ pSrcPic = new SSourcePicture();
+ memset (pSrcPic, 0, sizeof (SSourcePicture));
ASSERT_TRUE (pSrcPic != NULL);
memset (pSrcPic, 0, sizeof (SSourcePicture));
@@ -647,6 +648,39 @@ TEST_F (EncoderInterfaceTest, InitializeExtRejectsGeometryOverflow) {
EXPECT_EQ (iResult, static_cast<int> (ENC_RETURN_UNSUPPORTED_PARA));
}
+TEST_F (EncoderInterfaceTest, HighBitrateIdrTargetBits) {
+ SEncParamExt sEncParamExt;
+ pPtrEnc->GetDefaultParams (&sEncParamExt);
+ sEncParamExt.iUsageType = CAMERA_VIDEO_REAL_TIME;
+ sEncParamExt.iPicWidth = 1280;
+ sEncParamExt.iPicHeight = 720;
+ sEncParamExt.iTargetBitrate = 200000000;
+ sEncParamExt.fMaxFrameRate = 30.0f;
+ sEncParamExt.iRCMode = RC_BITRATE_MODE;
+ sEncParamExt.iSpatialLayerNum = 1;
+ sEncParamExt.sSpatialLayers[0].iVideoWidth = sEncParamExt.iPicWidth;
+ sEncParamExt.sSpatialLayers[0].iVideoHeight = sEncParamExt.iPicHeight;
+ sEncParamExt.sSpatialLayers[0].iSpatialBitrate = sEncParamExt.iTargetBitrate;
+ sEncParamExt.sSpatialLayers[0].fFrameRate = sEncParamExt.fMaxFrameRate;
+
+ int iResult = pPtrEnc->InitializeExt (&sEncParamExt);
+ EXPECT_EQ (iResult, static_cast<int> (cmResultSuccess));
+
+ PrepareOneSrcFrame();
+
+ iResult = pPtrEnc->EncodeFrame (pSrcPic, &sFbi);
+ EXPECT_EQ (iResult, static_cast<int> (cmResultSuccess));
+
+ // Force subsequent IDR frame so that iIdrNum != 0 and RcDecideTargetBits uses iIdrBitrateRatio
+ bool bIDR = true;
+ pPtrEnc->ForceIntraFrame (bIDR);
+ pSrcPic->uiTimeStamp += 33;
+ iResult = pPtrEnc->EncodeFrame (pSrcPic, &sFbi);
+ EXPECT_EQ (iResult, static_cast<int> (cmResultSuccess));
+
+ pPtrEnc->Uninitialize();
+}
+
TEST_F (EncoderInterfaceTest, BasicInitializeTestAutoAdjustment) {
SEncParamBase sEncParamBase;