Commit eeb9dcca98 for aom
commit eeb9dcca9835bfb5d17846a5e2f905413969013f
Author: Marco Paniconi <marpan@google.com>
Date: Tue Oct 6 04:05:28 2026 +0000
Fix rate overflow in av1_block_yrd()
In av1_block_yrd() and av1_block_yrd_idtx(), the unscaled coefficient
rate accumulated across transform blocks can exceed INT_MAX >> 11 on
large partitions in lossless or low-quantizer realtime mode, causing a
signed 32-bit integer overflow when shifted left by
2 + AV1_PROB_COST_SHIFT.
Compute the shifted rate in int64_t and clamp to INT_MAX / 2 in
av1_block_yrd(), av1_block_yrd_idtx(), and av1_estimate_block_intra().
Unittest added.
Bug: 568565869
Change-Id: I8ef2e6cfcd6efed37f13f5ad7f343effacde9065
diff --git a/av1/encoder/nonrd_opt.c b/av1/encoder/nonrd_opt.c
index e6d0d94a6d..cd79dbb3f7 100644
--- a/av1/encoder/nonrd_opt.c
+++ b/av1/encoder/nonrd_opt.c
@@ -320,8 +320,12 @@ void av1_block_yrd(MACROBLOCK *x, RD_STATS *this_rdc, int *skippable,
}
// If skippable is set, rate gets clobbered later.
- this_rdc->rate <<= (2 + AV1_PROB_COST_SHIFT);
- this_rdc->rate += (eob_cost << AV1_PROB_COST_SHIFT);
+ // Clamp to INT_MAX / 2 to avoid the INT_MAX invalid-rate sentinel and leave
+ // headroom for mode, MV, and chroma rate costs added by callers.
+ const int64_t block_rate =
+ ((int64_t)this_rdc->rate << (2 + AV1_PROB_COST_SHIFT)) +
+ ((int64_t)eob_cost << AV1_PROB_COST_SHIFT);
+ this_rdc->rate = (int)AOMMIN(block_rate, INT_MAX / 2);
}
// Explicitly enumerate the cases so the compiler can generate SIMD for the
@@ -455,8 +459,10 @@ void av1_block_yrd_idtx(MACROBLOCK *x, const uint8_t *const pred_buf,
}
}
// If skippable is set, rate gets clobbered later.
- this_rdc->rate <<= (2 + AV1_PROB_COST_SHIFT);
- this_rdc->rate += (eob_cost << AV1_PROB_COST_SHIFT);
+ const int64_t block_rate =
+ ((int64_t)this_rdc->rate << (2 + AV1_PROB_COST_SHIFT)) +
+ ((int64_t)eob_cost << AV1_PROB_COST_SHIFT);
+ this_rdc->rate = (int)AOMMIN(block_rate, INT_MAX / 2);
}
int64_t av1_model_rd_for_sb_uv(AV1_COMP *cpi, BLOCK_SIZE plane_bsize,
@@ -664,7 +670,8 @@ void av1_estimate_block_intra(int plane, int block, int row, int col,
p->src.buf = src_buf_base;
pd->dst.buf = dst_buf_base;
assert(args->rdc->rate != INT_MAX && args->rdc->dist != INT64_MAX);
- args->rdc->rate += this_rdc.rate;
+ args->rdc->rate =
+ (int)AOMMIN((int64_t)args->rdc->rate + this_rdc.rate, INT_MAX / 2);
args->rdc->dist += this_rdc.dist;
}
diff --git a/test/encode_api_test.cc b/test/encode_api_test.cc
index eb1ce6f914..65a59e1002 100644
--- a/test/encode_api_test.cc
+++ b/test/encode_api_test.cc
@@ -3594,4 +3594,50 @@ TEST(EncodeAPI, Buganizer565488030) {
}
#endif // !CONFIG_REALTIME_ONLY
+// Regression test for b/568565869: Undefined left-shift overflow of
+// this_rdc->rate in av1_block_yrd() when encoding a large partition in
+// realtime lossless mode.
+TEST(EncodeAPI, NonrdBlockYrdRateOverflow) {
+ aom_codec_iface_t *const iface = aom_codec_av1_cx();
+ aom_codec_enc_cfg_t cfg;
+ ASSERT_EQ(aom_codec_enc_config_default(iface, &cfg, AOM_USAGE_REALTIME),
+ AOM_CODEC_OK);
+
+ cfg.g_w = 128;
+ cfg.g_h = 128;
+ cfg.g_lag_in_frames = 0;
+
+ aom_codec_ctx_t enc;
+ ASSERT_EQ(aom_codec_enc_init(&enc, iface, &cfg, 0), AOM_CODEC_OK);
+ ASSERT_EQ(aom_codec_control(&enc, AOME_SET_CPUUSED, 10), AOM_CODEC_OK);
+ ASSERT_EQ(aom_codec_control(&enc, AV1E_SET_LOSSLESS, 1), AOM_CODEC_OK);
+ ASSERT_EQ(aom_codec_control(&enc, AV1E_SET_SUPERBLOCK_SIZE,
+ AOM_SUPERBLOCK_SIZE_128X128),
+ AOM_CODEC_OK);
+
+ aom_image_t *img = aom_img_alloc(nullptr, AOM_IMG_FMT_I420, 128, 128, 16);
+ ASSERT_NE(img, nullptr);
+ FillImage(img, 0);
+ EncodeOne(&enc, img, 0);
+
+ // Use a 253/255 checkerboard in Frame 1 against all-zero Frame 0:
+ // - The residual variance around the mean (254) is tiny (1), so
+ // variance-based partitioning does not split and evaluates a large
+ // partition.
+ // - The large DC residual (~254) combined with non-zero AC (ncoeffs > 1)
+ // causes aom_satd_lp(low_qcoeff) in av1_block_yrd() to accumulate a rate
+ // above INT_MAX >> 11, which would overflow 32-bit int when shifted left
+ // by 2 + AV1_PROB_COST_SHIFT (11).
+ for (int r = 0; r < 128; ++r) {
+ for (int c = 0; c < 128; ++c) {
+ img->planes[AOM_PLANE_Y][r * img->stride[AOM_PLANE_Y] + c] =
+ ((r ^ c) & 1) ? 255 : 253;
+ }
+ }
+ EncodeOne(&enc, img, 1);
+
+ aom_img_free(img);
+ ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK);
+}
+
} // namespace