Commit 129522fa62 for aom
commit 129522fa623771be2eeb03c1deaaf82bb04d3253
Author: Jerome Jiang <jianj@google.com>
Date: Thu Nov 12 10:49:35 2026 -0500
Prevent int overflows in var_based_part.c
- Clamp VPartVar.variance to INT_MAX in get_variance() so 12-bit
inter-frame variance cannot wrap to negative.
- Widen sub-block variance differences and minvar_16x16 << 2 shift
to int64_t.
- Add EncodeAPI.VarBasedPart12BitVarianceOverflow unit test.
Bug: b:584264300
Change-Id: Ic41ed0390b736d18d9cc0cde12958b8b58292f17
diff --git a/av1/encoder/var_based_part.c b/av1/encoder/var_based_part.c
index a25d6ce03a..dc92fdf0c3 100644
--- a/av1/encoder/var_based_part.c
+++ b/av1/encoder/var_based_part.c
@@ -111,7 +111,7 @@ static inline void get_variance(VPartVar *v) {
(uint32_t)(((int64_t)v->sum_error * v->sum_error) >> v->log2_count);
const int64_t variance =
(int64_t)256 * (v->sum_square_error - square_sum_error) >> v->log2_count;
- v->variance = (int)variance;
+ v->variance = (variance > INT_MAX) ? INT_MAX : (int)variance;
}
static inline void sum_2_variances(const VPartVar *a, const VPartVar *b,
@@ -1829,8 +1829,9 @@ int av1_choose_var_based_partitioning(AV1_COMP *cpi, const TileInfo *const tile,
vt->split[blk64_idx].split[lvl1_idx].part_variances.none.variance;
max_var_32x32[blk64_idx] = AOMMAX(var_32x32, max_var_32x32[blk64_idx]);
min_var_32x32[blk64_idx] = AOMMIN(var_32x32, min_var_32x32[blk64_idx]);
- const int max_min_var_16X16_diff = (maxvar_16x16[blk64_idx][lvl1_idx] -
- minvar_16x16[blk64_idx][lvl1_idx]);
+ const int64_t max_min_var_16X16_diff =
+ (int64_t)maxvar_16x16[blk64_idx][lvl1_idx] -
+ minvar_16x16[blk64_idx][lvl1_idx];
if (var_32x32 > thresholds[2] ||
(!is_key_frame && var_32x32 > (thresholds[2] >> 1) &&
@@ -1846,7 +1847,7 @@ int av1_choose_var_based_partitioning(AV1_COMP *cpi, const TileInfo *const tile,
cpi->rc.frame_source_sad < frame_sad_thresh &&
maxvar_16x16[blk64_idx][lvl1_idx] > (thresholds[2] >> 4) &&
maxvar_16x16[blk64_idx][lvl1_idx] >
- (minvar_16x16[blk64_idx][lvl1_idx] << 2)))) {
+ ((int64_t)minvar_16x16[blk64_idx][lvl1_idx] << 2)))) {
force_split[5 + blk64_scale_idx + lvl1_idx] = PART_EVAL_ONLY_SPLIT;
force_split[blk64_idx + 1] = PART_EVAL_ONLY_SPLIT;
force_split[0] = PART_EVAL_ONLY_SPLIT;
@@ -1863,8 +1864,8 @@ int av1_choose_var_based_partitioning(AV1_COMP *cpi, const TileInfo *const tile,
// variance of a sub-block is above some threshold of then force this
// block to split. Only checking this for noise level >= medium, if
// encoder is in SVC or if we already forced large blocks.
- const int max_min_var_32x32_diff =
- max_var_32x32[blk64_idx] - min_var_32x32[blk64_idx];
+ const int64_t max_min_var_32x32_diff =
+ (int64_t)max_var_32x32[blk64_idx] - min_var_32x32[blk64_idx];
const int check_max_var = max_var_32x32[blk64_idx] > thresholds[1] >> 1;
const bool check_noise_lvl = noise_level >= kMedium ||
cpi->ppi->use_svc ||
@@ -1889,7 +1890,7 @@ int av1_choose_var_based_partitioning(AV1_COMP *cpi, const TileInfo *const tile,
force_split[0] = PART_EVAL_ONLY_SPLIT;
if (!is_key_frame &&
- (max_var_64x64 - min_var_64x64) > 3 * (thresholds[0] >> 3) &&
+ ((int64_t)max_var_64x64 - min_var_64x64) > 3 * (thresholds[0] >> 3) &&
max_var_64x64 > thresholds[0] >> 1)
force_split[0] = PART_EVAL_ONLY_SPLIT;
}
diff --git a/test/encode_api_test.cc b/test/encode_api_test.cc
index 3051b46b82..ea42365a7a 100644
--- a/test/encode_api_test.cc
+++ b/test/encode_api_test.cc
@@ -5387,6 +5387,90 @@ TEST(EncodeAPI, Avg64x64IntegerOverflow) {
ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK);
}
+
+// Reproduces signed integer overflow in av1_choose_var_based_partitioning()
+// when 12-bit inter-frame variance in get_variance() exceeds INT_MAX and wraps
+// negative, causing max_var_64x64 - min_var_64x64 to overflow int.
+TEST(EncodeAPI, VarBasedPart12BitVarianceOverflow) {
+ aom_codec_iface_t *const iface = aom_codec_av1_cx();
+ aom_codec_enc_cfg_t cfg;
+ ASSERT_EQ(aom_codec_enc_config_default(iface, &cfg, AOM_USAGE_REALTIME),
+ AOM_CODEC_OK);
+ cfg.g_profile = 2;
+ cfg.g_w = 128;
+ cfg.g_h = 128;
+ cfg.g_bit_depth = AOM_BITS_12;
+ cfg.g_input_bit_depth = 12;
+ cfg.g_pass = AOM_RC_ONE_PASS;
+ cfg.g_lag_in_frames = 0;
+ cfg.rc_end_usage = AOM_Q;
+ cfg.rc_min_quantizer = 2;
+ cfg.rc_max_quantizer = 2;
+
+ aom_codec_ctx_t enc;
+ ASSERT_EQ(aom_codec_enc_init(&enc, iface, &cfg, AOM_CODEC_USE_HIGHBITDEPTH),
+ AOM_CODEC_OK);
+ ASSERT_EQ(aom_codec_control(&enc, AOME_SET_CPUUSED, 10), AOM_CODEC_OK);
+ ASSERT_EQ(aom_codec_control(&enc, AOME_SET_CQ_LEVEL, 2), AOM_CODEC_OK);
+ ASSERT_EQ(aom_codec_control(&enc, AV1E_SET_SUPERBLOCK_SIZE,
+ AOM_SUPERBLOCK_SIZE_128X128),
+ AOM_CODEC_OK);
+
+ aom_image_t *const img0 =
+ aom_img_alloc(nullptr, AOM_IMG_FMT_I42016, cfg.g_w, cfg.g_h, 1);
+ aom_image_t *const img1 =
+ aom_img_alloc(nullptr, AOM_IMG_FMT_I42016, cfg.g_w, cfg.g_h, 1);
+ ASSERT_NE(img0, nullptr);
+ ASSERT_NE(img1, nullptr);
+
+ // Fill chroma planes with mid-gray.
+ for (int plane = 1; plane < 3; ++plane) {
+ const unsigned int w = aom_img_plane_width(img0, plane);
+ const unsigned int h = aom_img_plane_height(img0, plane);
+ for (unsigned int y = 0; y < h; ++y) {
+ uint16_t *const row0 = reinterpret_cast<uint16_t *>(
+ img0->planes[plane] + y * img0->stride[plane]);
+ uint16_t *const row1 = reinterpret_cast<uint16_t *>(
+ img1->planes[plane] + y * img1->stride[plane]);
+ for (unsigned int x = 0; x < w; ++x) {
+ row0[x] = 2048;
+ row1[x] = 2048;
+ }
+ }
+ }
+
+ // Construct a 32x32 checkerboard in the top two 64x64 blocks so that every
+ // 16x16 and 32x32 sub-block has zero variance (keeping force_split[0] ==
+ // PART_EVAL_ALL), while the 64x64 inter-frame residual is +/-2890 in the
+ // left block (variance ~ 2.138e9 < INT_MAX) and +/-2910 in the right block
+ // (variance ~ 2.168e9 > INT_MAX, wrapping to ~ -2.127e9 without clamping).
+ for (unsigned int y = 0; y < cfg.g_h; ++y) {
+ uint16_t *const row0 = reinterpret_cast<uint16_t *>(
+ img0->planes[AOM_PLANE_Y] + y * img0->stride[AOM_PLANE_Y]);
+ uint16_t *const row1 = reinterpret_cast<uint16_t *>(
+ img1->planes[AOM_PLANE_Y] + y * img1->stride[AOM_PLANE_Y]);
+ for (unsigned int x = 0; x < cfg.g_w; ++x) {
+ if (y < 64) {
+ const bool high = (((x >> 5) ^ (y >> 5)) & 1) != 0;
+ const uint16_t delta = (x < 64) ? 2890 : 2910;
+ row0[x] = high ? 4095 : 0;
+ row1[x] = high ? static_cast<uint16_t>(4095 - delta) : delta;
+ } else {
+ row0[x] = 0;
+ row1[x] = 0;
+ }
+ }
+ }
+
+ ASSERT_EQ(aom_codec_encode(&enc, img0, 0, 1, AOM_EFLAG_FORCE_KF),
+ AOM_CODEC_OK);
+ ASSERT_EQ(aom_codec_encode(&enc, img1, 1, 1, 0), AOM_CODEC_OK);
+ ASSERT_EQ(aom_codec_encode(&enc, nullptr, 0, 0, 0), AOM_CODEC_OK);
+
+ aom_img_free(img1);
+ aom_img_free(img0);
+ ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK);
+}
#endif // CONFIG_AV1_HIGHBITDEPTH
#if CONFIG_AV1_DECODER