Commit fca40fc1 for openh264
commit fca40fc19f5fb854609d297ae4bde71df72787c9
Author: Erik Språng <sprang@google.com>
Date: Tue Jul 21 11:34:50 2026 +0200
Fix slice buffer size validation and allocation during dynamic load balancing (#3966)
* Fix dynamic slice adjustment for frames with extreme aspect ratios
When rate control is disabled (RC_OFF_MODE), dynamic slice adjustment
in DynamicAdjustSlicing() defaults the minimum macroblock requirement per
slice (iMinimalMbNum) to the frame's macroblock width (1 row per slice).
For frames with extreme aspect ratios (such as very wide resolutions),
requiring 1 macroblock row per slice across multiple slices can exceed the
total number of macroblocks in the frame.
This change ensures robust macroblock distribution and load balancing by:
- Falling back iMinimalMbNum to 1 macroblock per slice when row-based
heuristics exceed frame capacity in RC_OFF_MODE.
- Skipping adjustment and logging a warning when the requested number of
slices equals or exceeds the total macroblocks in the frame.
- Adding a capacity guard in GomValidCheckSliceMbNum() for rate-controlled
encoding modes.
- Adding a unit test (DynamicAdjustSlicingExtremeAspectRatio) to verify
dynamic slice adjustment and load balancing on wide aspect ratio frames.
* Reduce frame size in dynamic slicing test to prevent buffer overflow
* Allow slices to expand to full thread buffer size under dynamic load balancing
* Fix dynamic slicing crash on wide frames with ASan fix
---------
Co-authored-by: Erik Språng <sprang@webrtc.org>
diff --git a/codec/encoder/core/inc/nal_encap.h b/codec/encoder/core/inc/nal_encap.h
index 70a9cad2..80268341 100644
--- a/codec/encoder/core/inc/nal_encap.h
+++ b/codec/encoder/core/inc/nal_encap.h
@@ -85,6 +85,7 @@ int32_t iLayerBsIndex; // layer index of bit stream for SFrame
typedef struct TagWelsSliceBs {
uint8_t* pBs; // output bitstream, pBitStringAux not needed for slice 0 due to no dependency of pFrameBs available
+uint32_t uiBsSize; // size of pBs buffer
uint32_t uiBsPos; // position of output bitstream
uint8_t* pBsBuffer; // overall bitstream pBuffer allocation for a coded slice, recycling use intend.
uint32_t uiSize; // size of allocation pBuffer above
diff --git a/codec/encoder/core/src/encoder_ext.cpp b/codec/encoder/core/src/encoder_ext.cpp
index 228a67dc..e7bbf7e2 100644
--- a/codec/encoder/core/src/encoder_ext.cpp
+++ b/codec/encoder/core/src/encoder_ext.cpp
@@ -1603,7 +1603,11 @@ int32_t RequestMemorySvc (sWelsEncCtx** ppCtx, SExistingParasetList* pExistingPa
iMaxLayerBsSize = iSliceBufferSize * uiMaxSliceNumEstimation;
} else {
(*ppCtx)->iMaxSliceCount = WELS_MAX ((*ppCtx)->iMaxSliceCount, (int) pSliceArgument->uiSliceNum);
- iSliceBufferSize = ((iLayerBsSize / pSliceArgument->uiSliceNum) << 1) + MAX_MACROBLOCK_SIZE_IN_BYTE_x2;
+ if (pParam->bUseLoadBalancing) {
+ iSliceBufferSize = iLayerBsSize + MAX_MACROBLOCK_SIZE_IN_BYTE_x2;
+ } else {
+ iSliceBufferSize = ((iLayerBsSize / pSliceArgument->uiSliceNum) << 1) + MAX_MACROBLOCK_SIZE_IN_BYTE_x2;
+ }
iMaxLayerBsSize = iSliceBufferSize * pSliceArgument->uiSliceNum;
}
iMaxLayerBsSize = WELS_MAX (iMaxLayerBsSize, iLayerBsSize);
diff --git a/codec/encoder/core/src/slice_multi_threading.cpp b/codec/encoder/core/src/slice_multi_threading.cpp
index 09ea2c08..d505ac58 100644
--- a/codec/encoder/core/src/slice_multi_threading.cpp
+++ b/codec/encoder/core/src/slice_multi_threading.cpp
@@ -197,6 +197,24 @@ void DynamicAdjustSlicing (sWelsEncCtx* pCtx,
return;
}
iMinimalMbNum = iNumMbInEachGom;
+ } else {
+ // If the number of requested slices equals or exceeds the total macroblocks
+ // in the frame, each slice cannot be assigned even 1 macroblock. In this
+ // case, do not adjust slice sizes.
+ if (kiCountSliceNum >= kiCountNumMb) {
+ WelsLog(&(pCtx->sLogCtx), WELS_LOG_WARNING,
+ "[MT] DynamicAdjustSlicing(), requested slice number (%d) equals "
+ "or exceeds total macroblocks (%d), do not adjust",
+ kiCountSliceNum, kiCountNumMb);
+ return;
+ } else if (iMinimalMbNum * kiCountSliceNum >= kiCountNumMb) {
+ // When rate control is off, iMinimalMbNum defaults to 1 macroblock row
+ // (iMbWidth). For extreme aspect ratios (such as very wide resolutions),
+ // requiring 1 row per slice may exceed total frame capacity. Fall back to
+ // 1 macroblock per slice to ensure valid upper bounds and allow proper
+ // load balancing across slices.
+ iMinimalMbNum = 1;
+ }
}
if (kiCountSliceNum < 2 || (kiCountSliceNum & 0x01)) // we need suppose uiSliceNum is even for multiple threading
@@ -476,7 +494,7 @@ int32_t WriteSliceBs (sWelsEncCtx* pCtx, SWelsSliceBs* pSliceBs, const int32_t i
int32_t iNalIdx = 0;
int32_t iNalSize = 0;
int32_t iReturn = ENC_RETURN_SUCCESS;
- int32_t iTotalLeftLength = pSliceBs->uiSize - pSliceBs->uiBsPos;
+ int32_t iTotalLeftLength = pSliceBs->uiBsSize - pSliceBs->uiBsPos;
SNalUnitHeaderExt* pNalHdrExt = &pCtx->pCurDqLayer->sLayerInfo.sNalHeaderExt;
uint8_t* pDst = pSliceBs->pBs;
diff --git a/codec/encoder/core/src/svc_enc_slice_segment.cpp b/codec/encoder/core/src/svc_enc_slice_segment.cpp
index 5a9bf739..a920de7f 100644
--- a/codec/encoder/core/src/svc_enc_slice_segment.cpp
+++ b/codec/encoder/core/src/svc_enc_slice_segment.cpp
@@ -281,6 +281,12 @@ bool GomValidCheckSliceMbNum (const int32_t kiMbWidth, const int32_t kiMbHeight,
int32_t iCurNumMbAssigning = 0;
iMinimalMbNum = iGomSize;
+ // Ensure that the minimum macroblock requirement across all slices does not
+ // exceed total frame capacity, preventing negative calculations for remaining
+ // macroblocks.
+ if (iMinimalMbNum * static_cast<int32_t>(kuiSliceNum) > kiMbNumInFrame) {
+ return false;
+ }
while (uiSliceIdx + 1 < kuiSliceNum) {
iMaximalMbNum = iNumMbLeft - (kuiSliceNum - uiSliceIdx - 1) * iMinimalMbNum; // get maximal num_mb in left parts
// make sure one GOM at least in each slice for safe
diff --git a/codec/encoder/core/src/svc_encode_slice.cpp b/codec/encoder/core/src/svc_encode_slice.cpp
index 196f7d09..1fd7d46e 100644
--- a/codec/encoder/core/src/svc_encode_slice.cpp
+++ b/codec/encoder/core/src/svc_encode_slice.cpp
@@ -914,11 +914,14 @@ int32_t InitSliceBsBuffer (SSlice* pSlice,
pSlice->pSliceBsa = &pSlice->sSliceBs.sBsWrite;
pSlice->sSliceBs.pBs = (uint8_t*)pMa->WelsMallocz (iMaxSliceBufferSize, "sSliceBs.pBs");
if (NULL == pSlice->sSliceBs.pBs) {
+ pSlice->sSliceBs.uiBsSize = 0;
return ENC_RETURN_MEMALLOCERR;
}
+ pSlice->sSliceBs.uiBsSize = iMaxSliceBufferSize;
} else {
pSlice->pSliceBsa = pBsWrite;
pSlice->sSliceBs.pBs = NULL;
+ pSlice->sSliceBs.uiBsSize = 0;
}
return ENC_RETURN_SUCCESS;
}
@@ -1026,6 +1029,7 @@ int32_t InitOneSliceInThread (sWelsEncCtx* pCtx,
pSlice->sSliceBs.uiBsPos = 0;
pSlice->sSliceBs.iNalIndex = 0;
pSlice->sSliceBs.pBsBuffer = pCtx->pSliceThreading->pThreadBsBuffer[kiSlcBuffIdx];
+ pSlice->sSliceBs.uiSize = pCtx->iFrameBsSize;
return ENC_RETURN_SUCCESS;
}
diff --git a/test/api/encoder_test.cpp b/test/api/encoder_test.cpp
index 69e877b5..4d2e4f5a 100644
--- a/test/api/encoder_test.cpp
+++ b/test/api/encoder_test.cpp
@@ -387,6 +387,81 @@ TEST_F(EncoderInitTest, ScreenContentScrollMotionVectorBounds) {
ASSERT_EQ(0, rv);
}
+// This test verifies dynamic slice adjustment when encoding frames with extreme
+// aspect ratios (very wide resolution) in RC_OFF_MODE using multiple
+// slices/threads. It ensures that macroblocks are correctly distributed across
+// slices without producing negative slice limits or invalid memory access when
+// slice encoding speeds vary.
+TEST_F(EncoderInitTest, DynamicAdjustSlicingExtremeAspectRatio) {
+ SEncParamExt param;
+ encoder_->GetDefaultParams(¶m);
+
+ param.iUsageType = CAMERA_VIDEO_REAL_TIME;
+ // Extreme wide aspect ratio: frame height is only 2 macroblock rows (32
+ // pixels), which tests slicing behavior when row-based heuristics exceed
+ // frame dimensions.
+ param.iPicWidth = 32752;
+ param.iPicHeight = 32;
+ param.fMaxFrameRate = 30.0f;
+ param.iSpatialLayerNum = 1;
+ param.iMultipleThreadIdc = 4;
+ param.iRCMode = RC_OFF_MODE;
+ // Enable load balancing and high complexity mode to trigger dynamic slice
+ // adjustments.
+ param.bUseLoadBalancing = true;
+ param.iComplexityMode = HIGH_COMPLEXITY;
+ param.iMinQp = 0;
+ param.iMaxQp = 51;
+
+ param.sSpatialLayers[0].iVideoWidth = param.iPicWidth;
+ param.sSpatialLayers[0].iVideoHeight = param.iPicHeight;
+ param.sSpatialLayers[0].fFrameRate = param.fMaxFrameRate;
+ param.sSpatialLayers[0].sSliceArgument.uiSliceMode = SM_FIXEDSLCNUM_SLICE;
+ param.sSpatialLayers[0].sSliceArgument.uiSliceNum = 4;
+ param.sSpatialLayers[0].iDLayerQp = 24;
+
+ int rv = encoder_->InitializeExt(¶m);
+ ASSERT_EQ(0, rv);
+
+ int frameSize = param.iPicWidth * param.iPicHeight * 3 / 2;
+ BufferedData buf;
+ buf.SetLength(frameSize);
+ ASSERT_EQ(buf.Length(), (size_t)frameSize);
+
+ SFrameBSInfo info;
+ memset(&info, 0, sizeof(SFrameBSInfo));
+
+ SSourcePicture pic;
+ memset(&pic, 0, sizeof(SSourcePicture));
+ pic.iPicWidth = param.iPicWidth;
+ pic.iPicHeight = param.iPicHeight;
+ pic.iColorFormat = videoFormatI420;
+ pic.iStride[0] = pic.iPicWidth;
+ pic.iStride[1] = pic.iStride[2] = pic.iPicWidth >> 1;
+ pic.pData[0] = buf.data();
+ pic.pData[1] = pic.pData[0] + param.iPicWidth * param.iPicHeight;
+ pic.pData[2] = pic.pData[1] + (param.iPicWidth * param.iPicHeight >> 2);
+
+ for (int i = 0; i < 10; i++) {
+ // Generate varying random noise in Slices 1..3 while keeping Slice 0
+ // completely flat (zeros). This creates a significant encoding speed
+ // differential, causing the encoder to assign a larger proportion of
+ // macroblocks to Slice 0 during dynamic load balancing.
+ for (int idx = 0; idx < frameSize; idx++) {
+ buf.data()[idx] = rand() % 256;
+ }
+ for (int y = 0; y < 16; y++) {
+ memset(pic.pData[0] + y * pic.iStride[0], 0, 16368);
+ }
+ for (int y = 0; y < 8; y++) {
+ memset(pic.pData[1] + y * pic.iStride[1], 0, 8184);
+ memset(pic.pData[2] + y * pic.iStride[2], 0, 8184);
+ }
+ rv = encoder_->EncodeFrame(&pic, &info);
+ ASSERT_EQ(0, rv);
+ }
+}
+
// This test verifies that the encoder correctly handles frames with distinct,
// asymmetric strides for the U and V chroma planes (e.g., when U stride is much
// larger than V stride, or vice versa). It ensures that memory copy routines