Commit 4096df61f9 for aom
commit 4096df61f9f86fbe3ab57f750e8b19ce3c4ba627
Author: Alejandro Pacheco <alepacheco99@gmail.com>
Date: Tue Oct 20 00:22:58 2026 +0200
Use the allocated stride for the coefficient buffer
av1_alloc_txb_buf() sizes coeff_buffer_base for the forced maximum
frame width at allocation time, but av1_get_cb_coeff_buffer() computes
the stride from the current oxcf forced_max_frame_width. Raising
g_forced_max_frame_width with aom_codec_enc_config_set() without
changing the frame size does not reallocate the buffer, so the stride
grows and the offsets for later superblock rows point past the end of
the buffer (heap buffer overflow read in
av1_update_and_record_txb_context()).
Store the stride when the buffer is allocated, reset it when the buffer
is freed, and use it in av1_get_cb_coeff_buffer().
The new tests raise the forced maximum frame size after the first frame
in realtime and good quality mode. ASan reports a heap buffer overflow
without the fix.
Bug: 573358005
Change-Id: I8c1784a059848b973a8e2430311428ccaac61fc5
diff --git a/av1/encoder/encoder.h b/av1/encoder/encoder.h
index 1b13fc9136..db6e16367d 100644
--- a/av1/encoder/encoder.h
+++ b/av1/encoder/encoder.h
@@ -2938,6 +2938,11 @@ typedef struct AV1_COMP {
*/
CB_COEFF_BUFFER *coeff_buffer_base;
+ /*!
+ * Number of superblock columns per row of coeff_buffer_base, as allocated.
+ */
+ int coeff_buffer_stride;
+
/*!
* Structure holding pointers to frame level memory allocated for transform
* block related information.
diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c
index 6066f9e010..50cf0e4f6b 100644
--- a/av1/encoder/encodetxb.c
+++ b/av1/encoder/encodetxb.c
@@ -62,6 +62,7 @@ void av1_alloc_txb_buf(AV1_COMP *cpi) {
// TODO(jingning): This should be further reduced.
CHECK_MEM_ERROR(cm, cpi->coeff_buffer_base,
aom_malloc(sizeof(*cpi->coeff_buffer_base) * size));
+ cpi->coeff_buffer_stride = num_sb_cols;
if (sizeof(*coeff_buf_pool->tcoeff) > SIZE_MAX / num_tcoeffs) {
aom_internal_error(cm->error, AOM_CODEC_ERROR,
"A multiplication would overflow size_t");
@@ -105,6 +106,7 @@ void av1_free_txb_buf(AV1_COMP *cpi) {
CoeffBufferPool *coeff_buf_pool = &cpi->coeff_buffer_pool;
aom_free(cpi->coeff_buffer_base);
cpi->coeff_buffer_base = NULL;
+ cpi->coeff_buffer_stride = 0;
aom_free(coeff_buf_pool->tcoeff);
coeff_buf_pool->tcoeff = NULL;
aom_free(coeff_buf_pool->eobs);
@@ -908,12 +910,7 @@ CB_COEFF_BUFFER *av1_get_cb_coeff_buffer(const struct AV1_COMP *cpi, int mi_row,
int mi_col) {
const AV1_COMMON *const cm = &cpi->common;
const int mib_size_log2 = cm->seq_params->mib_size_log2;
- int mi_cols = cm->mi_params.mi_cols;
- if (cpi->oxcf.frm_dim_cfg.forced_max_frame_width) {
- mi_cols = size_in_mi(cpi->oxcf.frm_dim_cfg.forced_max_frame_width);
- }
- const int stride = CEIL_POWER_OF_TWO(mi_cols, cm->seq_params->mib_size_log2);
- const int offset =
- (mi_row >> mib_size_log2) * stride + (mi_col >> mib_size_log2);
+ const int offset = (mi_row >> mib_size_log2) * cpi->coeff_buffer_stride +
+ (mi_col >> mib_size_log2);
return cpi->coeff_buffer_base + offset;
}
diff --git a/test/forced_max_frame_width_height_test.cc b/test/forced_max_frame_width_height_test.cc
index 762de80114..4ae6701eab 100644
--- a/test/forced_max_frame_width_height_test.cc
+++ b/test/forced_max_frame_width_height_test.cc
@@ -199,7 +199,6 @@ TEST(EncodeForcedMaxFrameWidthHeight, DimensionDecreasing) {
EXPECT_EQ(AOM_CODEC_OK, aom_codec_destroy(&enc));
}
-
#endif // !CONFIG_REALTIME_ONLY
TEST(EncodeForcedMaxFrameWidthHeight, RealtimeLag0TunePSNR) {
@@ -569,4 +568,69 @@ TEST(EncodeForcedMaxFrameWidthHeight,
EXPECT_EQ(AOM_CODEC_OK, aom_codec_destroy(&enc));
}
+// Fills an 8-bit image with a non-flat pattern, different for each value of
+// |seed|, so that the encoder codes coefficients in every block.
+void FillImagePattern(aom_image_t *image, int seed) {
+ for (int plane = 0; plane < 3; plane++) {
+ const int plane_width = aom_img_plane_width(image, plane);
+ const int plane_height = aom_img_plane_height(image, plane);
+ for (int y = 0; y < plane_height; ++y) {
+ unsigned char *row = image->planes[plane] + y * image->stride[plane];
+ for (int x = 0; x < plane_width; ++x) {
+ row[x] =
+ static_cast<unsigned char>((x * 7 + y * 13 + seed * 37) ^ (x * y));
+ }
+ }
+ }
+}
+
+// Raising g_forced_max_frame_width/height with aom_codec_enc_config_set()
+// after a frame has been encoded, without changing the frame size, does not
+// reallocate the coefficient buffer. It must still be indexed with the layout
+// it was allocated with, not one computed from the new forced maximum.
+void RunRaiseForcedMaxTest(unsigned int usage) {
+ constexpr unsigned int kWidth = 256;
+ constexpr unsigned int kHeight = 256;
+ constexpr unsigned int kMaxWidth = 1024;
+ constexpr unsigned int kMaxHeight = 1024;
+ aom_codec_iface_t *iface = aom_codec_av1_cx();
+ aom_codec_enc_cfg_t cfg;
+ ASSERT_EQ(AOM_CODEC_OK, aom_codec_enc_config_default(iface, &cfg, usage));
+ cfg.g_w = kWidth;
+ cfg.g_h = kHeight;
+ cfg.g_lag_in_frames = 0;
+ aom_codec_ctx_t enc;
+ ASSERT_EQ(AOM_CODEC_OK, aom_codec_enc_init(&enc, iface, &cfg, 0));
+ ASSERT_EQ(AOM_CODEC_OK,
+ aom_codec_control(&enc, AOME_SET_CPUUSED,
+ usage == AOM_USAGE_REALTIME ? 9 : 6));
+ ASSERT_EQ(AOM_CODEC_OK, aom_codec_set_option(&enc, "tune", "ssim"));
+
+ aom_image_t *image =
+ aom_img_alloc(nullptr, AOM_IMG_FMT_I420, kWidth, kHeight, 1);
+ ASSERT_NE(image, nullptr);
+ FillImagePattern(image, 0);
+ ASSERT_EQ(AOM_CODEC_OK, aom_codec_encode(&enc, image, 0, 1, 0));
+
+ cfg.g_forced_max_frame_width = kMaxWidth;
+ cfg.g_forced_max_frame_height = kMaxHeight;
+ ASSERT_EQ(AOM_CODEC_OK, aom_codec_enc_config_set(&enc, &cfg));
+ FillImagePattern(image, 1);
+ ASSERT_EQ(AOM_CODEC_OK, aom_codec_encode(&enc, image, 1, 1, 0));
+
+ ASSERT_EQ(AOM_CODEC_OK, aom_codec_encode(&enc, nullptr, 0, 0, 0));
+ aom_img_free(image);
+ ASSERT_EQ(AOM_CODEC_OK, aom_codec_destroy(&enc));
+}
+
+TEST(EncodeForcedMaxFrameWidthHeight, RaiseForcedMaxRealtime) {
+ RunRaiseForcedMaxTest(AOM_USAGE_REALTIME);
+}
+
+#if !CONFIG_REALTIME_ONLY
+TEST(EncodeForcedMaxFrameWidthHeight, RaiseForcedMaxGoodQuality) {
+ RunRaiseForcedMaxTest(AOM_USAGE_GOOD_QUALITY);
+}
+#endif // !CONFIG_REALTIME_ONLY
+
} // namespace