Commit ead0096eea for aom
commit ead0096eea66f0b30618abc83e209fad20900042
Author: Yuan Tong <tongyuan200097@gmail.com>
Date: Tue Oct 20 21:30:52 2026 +0800
Fix integer overflow on large block variances when picking mode
The variance of 64x64 and 128x128 blocks can reach 2^28 and 2^30, so
combined_motion_search() and calculate_tx_size() could overflow when
multiplying it by a constant and then take the wrong decision.
Tested: test_libaom --gtest_filter=EncodeAPI.NonrdPickMode*
Note: Build with -fsanitize=integer using Clang to detect these overflow.
Change-Id: If835c3ee7dcff3647c775c83120d3edc9fc8b2ba
diff --git a/av1/encoder/nonrd_pickmode.c b/av1/encoder/nonrd_pickmode.c
index 7c4f32e7e4..edc99d0807 100644
--- a/av1/encoder/nonrd_pickmode.c
+++ b/av1/encoder/nonrd_pickmode.c
@@ -235,7 +235,7 @@ static int combined_motion_search(AV1_COMP *cpi, MACROBLOCK *x,
start_mv, src_search_sites, search_method,
/*fine_search_interval=*/0);
- const unsigned int full_var_rd = av1_full_pixel_search(
+ const int full_var_rd = av1_full_pixel_search(
start_mv, &full_ms_params, step_param, cond_cost_list(cpi, cost_list),
&tmp_mv->as_fullmv, &best_mv_stats, NULL);
@@ -253,9 +253,9 @@ static int combined_motion_search(AV1_COMP *cpi, MACROBLOCK *x,
av1_make_default_subpel_ms_params(&ms_params, cpi, x, bsize, &ref_mv,
cost_list);
const bool fullpel_performed_well =
- (bsize == BLOCK_64X64 && full_var_rd * 40 < 62267 * 7) ||
- (bsize == BLOCK_32X32 && full_var_rd * 8 < 42380) ||
- (bsize == BLOCK_16X16 && full_var_rd * 8 < 10127);
+ (bsize == BLOCK_64X64 && (int64_t)full_var_rd * 40 < 62267 * 7) ||
+ (bsize == BLOCK_32X32 && (int64_t)full_var_rd * 8 < 42380) ||
+ (bsize == BLOCK_16X16 && (int64_t)full_var_rd * 8 < 10127);
if (sf->rt_sf.reduce_mv_pel_precision_highmotion ||
sf->rt_sf.reduce_mv_pel_precision_lowcomplex)
ms_params.forced_stop = subpel_select(cpi, x, bsize, tmp_mv, ref_mv,
@@ -483,7 +483,7 @@ static TX_SIZE calculate_tx_size(const AV1_COMP *const cpi, BLOCK_SIZE bsize,
}
// Choose larger transform size for blocks where dc component is dominant or
// the ac component is low.
- if (sse > ((var * multiplier) >> 2) || (var < var_thresh))
+ if (sse > (((uint64_t)var * multiplier) >> 2) || (var < var_thresh))
tx_size =
AOMMIN(max_txsize_lookup[bsize],
tx_mode_to_biggest_tx_size[txfm_params->tx_mode_search_type]);
diff --git a/test/encode_api_test.cc b/test/encode_api_test.cc
index 76b36e7961..73c7d7679b 100644
--- a/test/encode_api_test.cc
+++ b/test/encode_api_test.cc
@@ -697,6 +697,87 @@ TEST(EncodeAPI, LowBDEncoderHighBDImage) {
ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK);
}
+// Encodes three frames with the realtime encoder. The first frame is random
+// binary noise of 0 and the maximum value, and each following frame is the
+// inverse of the previous one, so the variance of the inter prediction residue
+// of large blocks is close to its maximum.
+void EncodeInvertedBinaryNoise(aom_bit_depth_t bit_depth, unsigned int size,
+ aom_superblock_size_t sb_size) {
+ aom_codec_iface_t *iface = aom_codec_av1_cx();
+ aom_codec_enc_cfg_t cfg;
+ ASSERT_EQ(aom_codec_enc_config_default(iface, &cfg, AOM_USAGE_REALTIME),
+ AOM_CODEC_OK);
+ cfg.g_w = size;
+ cfg.g_h = size;
+ cfg.g_bit_depth = bit_depth;
+ cfg.g_input_bit_depth = bit_depth;
+ cfg.g_lag_in_frames = 0;
+ cfg.rc_end_usage = AOM_CBR;
+
+ const bool high_bit_depth = bit_depth > AOM_BITS_8;
+ aom_codec_ctx_t enc;
+ ASSERT_EQ(aom_codec_enc_init(&enc, iface, &cfg,
+ high_bit_depth ? AOM_CODEC_USE_HIGHBITDEPTH : 0),
+ AOM_CODEC_OK);
+ ASSERT_EQ(aom_codec_control(&enc, AOME_SET_CPUUSED, 9), AOM_CODEC_OK);
+ ASSERT_EQ(aom_codec_control(&enc, AV1E_SET_SUPERBLOCK_SIZE, sb_size),
+ AOM_CODEC_OK);
+
+ aom_image_t *image = aom_img_alloc(
+ nullptr, high_bit_depth ? AOM_IMG_FMT_I42016 : AOM_IMG_FMT_I420, size,
+ size, 1);
+ ASSERT_NE(image, nullptr);
+ const int max_value = (1 << bit_depth) - 1;
+ const unsigned int uv_size = (size + 1) / 2;
+ for (int plane = 1; plane < 3; ++plane) {
+ for (unsigned int i = 0; i < uv_size; ++i) {
+ uint8_t *row = image->planes[plane] + i * image->stride[plane];
+ if (high_bit_depth) {
+ Memset16(row, (max_value + 1) / 2, uv_size);
+ } else {
+ memset(row, (max_value + 1) / 2, uv_size);
+ }
+ }
+ }
+
+ ::libaom_test::ACMRandom rnd;
+ rnd.Reset(::libaom_test::ACMRandom::DeterministicSeed());
+ for (int frame = 0; frame < 3; ++frame) {
+ for (unsigned int i = 0; i < size; ++i) {
+ uint8_t *row = image->planes[0] + i * image->stride[0];
+ for (unsigned int j = 0; j < size; ++j) {
+ if (high_bit_depth) {
+ uint16_t *row16 = reinterpret_cast<uint16_t *>(row);
+ row16[j] =
+ frame == 0 ? (rnd.Rand8() & 1) * max_value : max_value - row16[j];
+ } else {
+ row[j] =
+ frame == 0 ? (rnd.Rand8() & 1) * max_value : max_value - row[j];
+ }
+ }
+ }
+ ASSERT_EQ(aom_codec_encode(&enc, image, frame, 1, 0), AOM_CODEC_OK);
+ }
+ ASSERT_EQ(aom_codec_encode(&enc, nullptr, 0, 0, 0), AOM_CODEC_OK);
+
+ aom_img_free(image);
+ ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK);
+}
+
+// The 64x64 full pixel motion search variance in combined_motion_search() used
+// to overflow when multiplied by 40.
+TEST(EncodeAPI, NonrdPickModeLargeFullPixelVariance) {
+ EncodeInvertedBinaryNoise(AOM_BITS_8, 64, AOM_SUPERBLOCK_SIZE_64X64);
+}
+
+#if CONFIG_AV1_HIGHBITDEPTH
+// The 128x128 variance in calculate_tx_size() used to overflow when multiplied
+// by the transform size multiplier.
+TEST(EncodeAPI, NonrdPickModeLargeTxSizeVariance) {
+ EncodeInvertedBinaryNoise(AOM_BITS_10, 128, AOM_SUPERBLOCK_SIZE_128X128);
+}
+#endif // CONFIG_AV1_HIGHBITDEPTH
+
aom_image_t *CreateGrayImage(aom_img_fmt_t fmt, unsigned int w,
unsigned int h) {
aom_image_t *const image = aom_img_alloc(nullptr, fmt, w, h, 1);