Commit 46bd978472 for aom
commit 46bd978472725e7f0b55f5a8809848c45c012f4f
Author: Cheng Chen <chengchen@google.com>
Date: Fri Sep 11 11:54:53 2026 -0700
Fix integer overflow and bounds checks in allintra
1. Remove narrowing (int) casts in get_satd() and get_sse() to avoid
signed integer overflow when accumulating high-bitdepth or large
superblock SATD/SSE values.
2. Guard against negative row/col indices and zero mb_count in
get_satd(), get_sse(), get_max_scale(), get_window_wiener_var(),
and get_rate_guided_quantizer().
Bug: b/558417547
Bug: b/558463892
Bug: b/558516408
Bug: b/558589747
TAG=agy
Change-Id: I29fad3393f1c4383aa2a6c203674375a90be5fda
diff --git a/av1/encoder/allintra_vis.c b/av1/encoder/allintra_vis.c
index be21bc9765..91b51471e1 100644
--- a/av1/encoder/allintra_vis.c
+++ b/av1/encoder/allintra_vis.c
@@ -110,7 +110,8 @@ static int64_t get_satd(AV1_COMP *const cpi, BLOCK_SIZE bsize, int mi_row,
for (int row = mi_row; row < mi_row + mi_high; row += mi_step) {
for (int col = mi_col; col < mi_col + mi_wide; col += mi_step) {
- if (row >= cm->mi_params.mi_rows || col >= cm->mi_params.mi_cols)
+ if (row < 0 || col < 0 || row >= cm->mi_params.mi_rows ||
+ col >= cm->mi_params.mi_cols)
continue;
satd += cpi->mb_weber_stats[(row / mi_step) * mb_stride + (col / mi_step)]
@@ -119,10 +120,10 @@ static int64_t get_satd(AV1_COMP *const cpi, BLOCK_SIZE bsize, int mi_row,
}
}
- if (mb_count) satd = (int)(satd / mb_count);
+ if (mb_count) satd = satd / mb_count;
satd = AOMMAX(1, satd);
- return (int)satd;
+ return satd;
}
static int64_t get_sse(AV1_COMP *const cpi, BLOCK_SIZE bsize, int mi_row,
@@ -138,7 +139,8 @@ static int64_t get_sse(AV1_COMP *const cpi, BLOCK_SIZE bsize, int mi_row,
for (int row = mi_row; row < mi_row + mi_high; row += mi_step) {
for (int col = mi_col; col < mi_col + mi_wide; col += mi_step) {
- if (row >= cm->mi_params.mi_rows || col >= cm->mi_params.mi_cols)
+ if (row < 0 || col < 0 || row >= cm->mi_params.mi_rows ||
+ col >= cm->mi_params.mi_cols)
continue;
distortion +=
@@ -148,10 +150,10 @@ static int64_t get_sse(AV1_COMP *const cpi, BLOCK_SIZE bsize, int mi_row,
}
}
- if (mb_count) distortion = (int)(distortion / mb_count);
+ if (mb_count) distortion = distortion / mb_count;
distortion = AOMMAX(1, distortion);
- return (int)distortion;
+ return distortion;
}
static double get_max_scale(const AV1_COMP *const cpi, BLOCK_SIZE bsize,
@@ -165,7 +167,8 @@ static double get_max_scale(const AV1_COMP *const cpi, BLOCK_SIZE bsize,
for (int row = mi_row; row < mi_row + mi_high; row += mi_step) {
for (int col = mi_col; col < mi_col + mi_wide; col += mi_step) {
- if (row >= cm->mi_params.mi_rows || col >= cm->mi_params.mi_cols)
+ if (row < 0 || col < 0 || row >= cm->mi_params.mi_rows ||
+ col >= cm->mi_params.mi_cols)
continue;
const WeberStats *weber_stats =
&cpi->mb_weber_stats[(row / mi_step) * mb_stride + (col / mi_step)];
@@ -193,7 +196,8 @@ static int get_window_wiener_var(const AV1_COMP *const cpi, BLOCK_SIZE bsize,
for (int row = mi_row; row < mi_row + mi_high; row += mi_step) {
for (int col = mi_col; col < mi_col + mi_wide; col += mi_step) {
- if (row >= cm->mi_params.mi_rows || col >= cm->mi_params.mi_cols)
+ if (row < 0 || col < 0 || row >= cm->mi_params.mi_rows ||
+ col >= cm->mi_params.mi_cols)
continue;
const WeberStats *weber_stats =
@@ -213,6 +217,7 @@ static int get_window_wiener_var(const AV1_COMP *const cpi, BLOCK_SIZE bsize,
}
}
+ if (mb_count == 0) return 1;
sb_wiener_var =
(int)(((base_num + base_reg) / (base_den + base_reg)) / mb_count);
sb_wiener_var = AOMMAX(1, sb_wiener_var);
@@ -697,16 +702,20 @@ void av1_set_mb_wiener_variance(AV1_COMP *cpi) {
static int get_rate_guided_quantizer(const AV1_COMP *const cpi,
BLOCK_SIZE bsize, int mi_row, int mi_col) {
+ const AV1_COMMON *const cm = &cpi->common;
// Calculation uses 8x8.
const int mb_step = mi_size_wide[cpi->weber_bsize];
// Accumulate to 16x16
const int block_step = mi_size_wide[BLOCK_16X16];
double sb_rate_hific = 0.0;
double sb_rate_uniform = 0.0;
- for (int row = mi_row; row < mi_row + mi_size_wide[bsize];
+ for (int row = mi_row; row < mi_row + mi_size_high[bsize];
row += block_step) {
- for (int col = mi_col; col < mi_col + mi_size_high[bsize];
+ for (int col = mi_col; col < mi_col + mi_size_wide[bsize];
col += block_step) {
+ if (row < 0 || col < 0 || row >= cm->mi_params.mi_rows ||
+ col >= cm->mi_params.mi_cols)
+ continue;
sb_rate_hific +=
cpi->ext_rate_distribution[(row / mb_step) * cpi->frame_info.mi_cols +
(col / mb_step)];
@@ -715,6 +724,10 @@ static int get_rate_guided_quantizer(const AV1_COMP *const cpi,
for (int c = 0; c < block_step; c += mb_step) {
const int this_row = row + r;
const int this_col = col + c;
+ if (this_row < 0 || this_col < 0 ||
+ this_row >= cm->mi_params.mi_rows ||
+ this_col >= cm->mi_params.mi_cols)
+ continue;
sb_rate_uniform +=
cpi->prep_rate_estimates[(this_row / mb_step) *
cpi->frame_info.mi_cols +
@@ -734,7 +747,6 @@ static int get_rate_guided_quantizer(const AV1_COMP *const cpi,
double min_max_scale = AOMMAX(1.0, get_max_scale(cpi, bsize, mi_row, mi_col));
scale = 1.0 / AOMMIN(1.0 / scale, min_max_scale);
- const AV1_COMMON *const cm = &cpi->common;
const int base_qindex = cm->quant_params.base_qindex;
int offset =
av1_get_deltaq_offset(cm->seq_params->bit_depth, base_qindex, scale);
diff --git a/test/encode_api_test.cc b/test/encode_api_test.cc
index 4cd53e0f98..68eaaf44b8 100644
--- a/test/encode_api_test.cc
+++ b/test/encode_api_test.cc
@@ -31,7 +31,9 @@
#include "av1/common/blockd.h"
#include "av1/common/reconintra.h"
+extern "C" {
#include "av1/encoder/allintra_vis.h"
+}
#include "test/codec_factory.h"
#include "test/encode_test_driver.h"
#include "test/util.h"
@@ -3002,4 +3004,107 @@ TEST(EncodeAPI, Buganizer558446054) {
}
#endif // !CONFIG_REALTIME_ONLY
+#if !CONFIG_REALTIME_ONLY
+TEST(EncodeAPI, Buganizer558463892_558589747) {
+#if !CONFIG_SHARED
+ std::unique_ptr<AV1_COMP> cpi_test(new AV1_COMP());
+ struct aom_internal_error_info error = {};
+ if (setjmp(error.jmp)) FAIL();
+ error.setjmp = 1;
+ cpi_test->common.error = &error;
+ cpi_test->frame_info.mi_rows = 16;
+ cpi_test->frame_info.mi_cols = 16;
+ av1_init_mb_wiener_var_buffer(cpi_test.get());
+ ASSERT_NE(cpi_test->mb_weber_stats, nullptr);
+ cpi_test->mb_weber_stats[0].satd = 12345;
+
+ // Increase dimensions to 64x64 MI units: mb_weber_stats must be reallocated.
+ cpi_test->frame_info.mi_rows = 64;
+ cpi_test->frame_info.mi_cols = 64;
+ av1_init_mb_wiener_var_buffer(cpi_test.get());
+ EXPECT_EQ(cpi_test->mb_weber_stats[0].satd, 0);
+ aom_free(cpi_test->mb_weber_stats);
+#endif // !CONFIG_SHARED
+
+ aom_codec_iface_t *iface = aom_codec_av1_cx();
+ aom_codec_enc_cfg_t cfg;
+ ASSERT_EQ(aom_codec_enc_config_default(iface, &cfg, AOM_USAGE_ALL_INTRA),
+ AOM_CODEC_OK);
+
+ cfg.g_w = 64;
+ cfg.g_h = 64;
+ cfg.g_forced_max_frame_width = 1024;
+ cfg.g_forced_max_frame_height = 1024;
+ cfg.g_threads = 0;
+ cfg.g_lag_in_frames = 0;
+ cfg.g_pass = AOM_RC_ONE_PASS;
+ cfg.rc_end_usage = AOM_Q;
+
+ aom_codec_ctx_t codec;
+ ASSERT_EQ(aom_codec_enc_init(&codec, iface, &cfg, 0), AOM_CODEC_OK);
+ ASSERT_EQ(aom_codec_control(&codec, AV1E_SET_DELTAQ_MODE, 3), AOM_CODEC_OK);
+ ASSERT_EQ(aom_codec_control(&codec, AOME_SET_CPUUSED, 0), AOM_CODEC_OK);
+
+ // Encode frame 0 at 64x64
+ aom_image_t raw;
+ ASSERT_NE(aom_img_alloc(&raw, AOM_IMG_FMT_I420, 64, 64, 1), nullptr);
+ FillImageRandom(&raw);
+ ASSERT_EQ(aom_codec_encode(&codec, &raw, 0, 1, 0), AOM_CODEC_OK);
+ aom_img_free(&raw);
+
+ // Change resolution to 256x256 mid-stream
+ cfg.g_w = 256;
+ cfg.g_h = 256;
+ ASSERT_EQ(aom_codec_enc_config_set(&codec, &cfg), AOM_CODEC_OK);
+
+ // Encode frame 1 at 256x256
+ ASSERT_NE(aom_img_alloc(&raw, AOM_IMG_FMT_I420, 256, 256, 1), nullptr);
+ FillImageRandom(&raw);
+ ASSERT_EQ(aom_codec_encode(&codec, &raw, 1, 1, 0), AOM_CODEC_OK);
+ aom_img_free(&raw);
+
+ ASSERT_EQ(aom_codec_destroy(&codec), AOM_CODEC_OK);
+}
+#endif // !CONFIG_REALTIME_ONLY
+
+#if !CONFIG_REALTIME_ONLY
+TEST(EncodeAPI, Buganizer558417547) {
+#if !CONFIG_SHARED
+ std::unique_ptr<AV1_COMP> cpi_test(new AV1_COMP());
+ SequenceHeader seq_params = {};
+ seq_params.bit_depth = AOM_BITS_8;
+ seq_params.sb_size = BLOCK_64X64;
+ cpi_test->common.seq_params = &seq_params;
+ cpi_test->common.mi_params.mi_rows = 16;
+ cpi_test->common.mi_params.mi_cols = 16;
+ cpi_test->common.quant_params.base_qindex = 128;
+ cpi_test->common.delta_q_info.delta_q_res = 4;
+ cpi_test->frame_info.mi_rows = 16;
+ cpi_test->frame_info.mi_cols = 16;
+ cpi_test->norm_wiener_variance = 100;
+ struct aom_internal_error_info error = {};
+ if (setjmp(error.jmp)) FAIL();
+ error.setjmp = 1;
+ cpi_test->common.error = &error;
+ av1_init_mb_wiener_var_buffer(cpi_test.get());
+ ASSERT_NE(cpi_test->mb_weber_stats, nullptr);
+
+ // Set large SATD and distortion values exceeding INT32_MAX.
+ for (int i = 0; i < 16 * 16; ++i) {
+ cpi_test->mb_weber_stats[i].satd = 3000000000LL;
+ cpi_test->mb_weber_stats[i].distortion = 3000000000LL;
+ cpi_test->mb_weber_stats[i].rec_pix_max = 255;
+ }
+
+ // Test out-of-bounds / negative mi_row and mi_col without crashing or SIGFPE.
+ int q_neg = av1_get_sbq_perceptual_ai(cpi_test.get(), BLOCK_64X64, -16, -16);
+ EXPECT_GE(q_neg, 0);
+ int q_oob = av1_get_sbq_perceptual_ai(cpi_test.get(), BLOCK_64X64, 100, 100);
+ EXPECT_GE(q_oob, 0);
+
+ aom_free(cpi_test->mb_weber_stats);
+#endif // !CONFIG_SHARED
+}
+#endif // !CONFIG_REALTIME_ONLY
+
} // namespace