Commit 60ce6c8920 for aom
commit 60ce6c8920646d6c1e236e386dee9b65c7ae8e12
Author: Alejandro Pacheco <alepacheco99@gmail.com>
Date: Tue Oct 20 00:22:58 2026 +0200
Reallocate rdmult scaling factors when the frame grows
ssim_rdmult_scaling_factors and tpl_rdmult_scaling_factors are allocated
once in av1_create_compressor() for the initial frame size (or the
forced maximum frame size at that time). av1_change_config() and
set_size_literal() reallocate the other frame size dependent buffers
when the frame grows, but not these. They are written for every 16x16
block of the current frame, so after a larger frame size is set with
aom_codec_enc_config_set(), av1_set_mb_ssim_rdmult_scaling() writes
past their end (heap buffer overflow with tune=ssim or tune=iq). The
VMAF, Butteraugli and saliency map buffers have the same problem in
their build configurations; the VMAF one was sized for the initial
size even with a forced maximum.
Move all of them into alloc_rdmult_scaling_factors(), sized like the
segmentation maps, and call it from av1_create_compressor() and from
both reallocation paths.
The new tests grow the frame size before the first frame is encoded,
with tune=ssim and tune=iq. ASan reports a heap buffer overflow in
av1_set_mb_ssim_rdmult_scaling() without the fix.
Bug: 573358005
Change-Id: Ic3d5a4fad1effeb3166b5c3c3ad3c05b83db74e9
diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index dea9aa6d65..d9953523f3 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -1056,6 +1056,7 @@ void av1_change_config(struct AV1_COMP *cpi, const AV1EncoderConfig *oxcf,
cpi->td.firstpass_ctx = NULL;
alloc_compressor_data(cpi);
realloc_segmentation_maps(cpi);
+ alloc_rdmult_scaling_factors(cpi);
cpi->data_alloc_width = oxcf->frm_dim_cfg.forced_max_frame_width
? oxcf->frm_dim_cfg.forced_max_frame_width
: cm->width;
@@ -1644,30 +1645,10 @@ AV1_COMP *av1_create_compressor(AV1_PRIMARY *ppi, const AV1EncoderConfig *oxcf,
cpi->palette_pixel_num = 0;
cpi->scaled_last_source_available = 0;
- {
- const BLOCK_SIZE bsize = BLOCK_16X16;
- const int w = mi_size_wide[bsize];
- const int h = mi_size_high[bsize];
- const int num_cols = (max_mi_cols + w - 1) / w;
- const int num_rows = (max_mi_rows + h - 1) / h;
- CHECK_MEM_ERROR(cm, cpi->ssim_rdmult_scaling_factors,
- aom_calloc(num_rows * num_cols,
- sizeof(*cpi->ssim_rdmult_scaling_factors)));
- CHECK_MEM_ERROR(cm, cpi->tpl_rdmult_scaling_factors,
- aom_calloc(num_rows * num_cols,
- sizeof(*cpi->tpl_rdmult_scaling_factors)));
- }
+ alloc_rdmult_scaling_factors(cpi);
#if CONFIG_TUNE_VMAF
{
- const BLOCK_SIZE bsize = BLOCK_64X64;
- const int w = mi_size_wide[bsize];
- const int h = mi_size_high[bsize];
- const int num_cols = (mi_params->mi_cols + w - 1) / w;
- const int num_rows = (mi_params->mi_rows + h - 1) / h;
- CHECK_MEM_ERROR(cm, cpi->vmaf_info.rdmult_scaling_factors,
- aom_calloc(num_rows * num_cols,
- sizeof(*cpi->vmaf_info.rdmult_scaling_factors)));
for (int i = 0; i < MAX_ARF_LAYERS; i++) {
cpi->vmaf_info.last_frame_unsharp_amount[i] = -1.0;
cpi->vmaf_info.last_frame_ysse[i] = -1.0;
@@ -1679,41 +1660,11 @@ AV1_COMP *av1_create_compressor(AV1_PRIMARY *ppi, const AV1EncoderConfig *oxcf,
#endif
#if CONFIG_TUNE_BUTTERAUGLI
- {
- const int w = mi_size_wide[butteraugli_rdo_bsize];
- const int h = mi_size_high[butteraugli_rdo_bsize];
- const int num_cols = (mi_params->mi_cols + w - 1) / w;
- const int num_rows = (mi_params->mi_rows + h - 1) / h;
- CHECK_MEM_ERROR(
- cm, cpi->butteraugli_info.rdmult_scaling_factors,
- aom_malloc(num_rows * num_cols *
- sizeof(*cpi->butteraugli_info.rdmult_scaling_factors)));
- memset(&cpi->butteraugli_info.source, 0,
- sizeof(cpi->butteraugli_info.source));
- memset(&cpi->butteraugli_info.resized_source, 0,
- sizeof(cpi->butteraugli_info.resized_source));
- cpi->butteraugli_info.recon_set = false;
- }
-#endif
-
-#if CONFIG_SALIENCY_MAP
- {
- CHECK_MEM_ERROR(cm, cpi->saliency_map,
- (uint8_t *)aom_calloc(cm->height * cm->width,
- sizeof(*cpi->saliency_map)));
- // Buffer initialization based on MIN_MIB_SIZE_LOG2 to ensure that
- // cpi->sm_scaling_factor buffer is allocated big enough, since we have no
- // idea of the actual superblock size we are going to use yet.
- const int min_mi_w_sb = (1 << MIN_MIB_SIZE_LOG2);
- const int min_mi_h_sb = (1 << MIN_MIB_SIZE_LOG2);
- const int max_sb_cols =
- (cm->mi_params.mi_cols + min_mi_w_sb - 1) / min_mi_w_sb;
- const int max_sb_rows =
- (cm->mi_params.mi_rows + min_mi_h_sb - 1) / min_mi_h_sb;
- CHECK_MEM_ERROR(cm, cpi->sm_scaling_factor,
- (double *)aom_calloc(max_sb_rows * max_sb_cols,
- sizeof(*cpi->sm_scaling_factor)));
- }
+ memset(&cpi->butteraugli_info.source, 0,
+ sizeof(cpi->butteraugli_info.source));
+ memset(&cpi->butteraugli_info.resized_source, 0,
+ sizeof(cpi->butteraugli_info.resized_source));
+ cpi->butteraugli_info.recon_set = false;
#endif
#if CONFIG_COLLECT_PARTITION_STATS
@@ -2660,6 +2611,7 @@ static int set_size_literal(AV1_COMP *cpi, int width, int height) {
cpi->td.firstpass_ctx = NULL;
alloc_compressor_data(cpi);
realloc_segmentation_maps(cpi);
+ alloc_rdmult_scaling_factors(cpi);
cpi->data_alloc_width = cpi->oxcf.frm_dim_cfg.forced_max_frame_width
? cpi->oxcf.frm_dim_cfg.forced_max_frame_width
: cm->width;
diff --git a/av1/encoder/encoder_alloc.h b/av1/encoder/encoder_alloc.h
index ac4a873cc3..fc3143a3d9 100644
--- a/av1/encoder/encoder_alloc.h
+++ b/av1/encoder/encoder_alloc.h
@@ -131,6 +131,98 @@ static inline void realloc_segmentation_maps(AV1_COMP *cpi) {
aom_calloc(max_mi_rows * max_mi_cols, 1));
}
+// Allocates the per-block RD multiplier scaling factors (and, with
+// CONFIG_SALIENCY_MAP, the saliency map they are derived from) for the same
+// maximum frame size as the other frame size dependent buffers. They are
+// written for every block of the current frame, so they must be reallocated
+// whenever that maximum grows.
+static inline void alloc_rdmult_scaling_factors(AV1_COMP *cpi) {
+ AV1_COMMON *const cm = &cpi->common;
+ const CommonModeInfoParams *const mi_params = &cm->mi_params;
+ int max_mi_cols = mi_params->mi_cols;
+ int max_mi_rows = mi_params->mi_rows;
+ if (cpi->oxcf.frm_dim_cfg.forced_max_frame_width) {
+ max_mi_cols = size_in_mi(cpi->oxcf.frm_dim_cfg.forced_max_frame_width);
+ }
+ if (cpi->oxcf.frm_dim_cfg.forced_max_frame_height) {
+ max_mi_rows = size_in_mi(cpi->oxcf.frm_dim_cfg.forced_max_frame_height);
+ }
+ const BLOCK_SIZE bsize = BLOCK_16X16;
+ const int w = mi_size_wide[bsize];
+ const int h = mi_size_high[bsize];
+ const int num_cols = (max_mi_cols + w - 1) / w;
+ const int num_rows = (max_mi_rows + h - 1) / h;
+
+ aom_free(cpi->ssim_rdmult_scaling_factors);
+ cpi->ssim_rdmult_scaling_factors = NULL;
+ aom_free(cpi->tpl_rdmult_scaling_factors);
+ cpi->tpl_rdmult_scaling_factors = NULL;
+ CHECK_MEM_ERROR(cm, cpi->ssim_rdmult_scaling_factors,
+ aom_calloc(num_rows * num_cols,
+ sizeof(*cpi->ssim_rdmult_scaling_factors)));
+ CHECK_MEM_ERROR(cm, cpi->tpl_rdmult_scaling_factors,
+ aom_calloc(num_rows * num_cols,
+ sizeof(*cpi->tpl_rdmult_scaling_factors)));
+
+#if CONFIG_TUNE_VMAF
+ {
+ const BLOCK_SIZE vmaf_bsize = BLOCK_64X64;
+ const int vmaf_w = mi_size_wide[vmaf_bsize];
+ const int vmaf_h = mi_size_high[vmaf_bsize];
+ const int vmaf_num_cols = (max_mi_cols + vmaf_w - 1) / vmaf_w;
+ const int vmaf_num_rows = (max_mi_rows + vmaf_h - 1) / vmaf_h;
+ aom_free(cpi->vmaf_info.rdmult_scaling_factors);
+ cpi->vmaf_info.rdmult_scaling_factors = NULL;
+ CHECK_MEM_ERROR(cm, cpi->vmaf_info.rdmult_scaling_factors,
+ aom_calloc(vmaf_num_rows * vmaf_num_cols,
+ sizeof(*cpi->vmaf_info.rdmult_scaling_factors)));
+ }
+#endif // CONFIG_TUNE_VMAF
+
+#if CONFIG_TUNE_BUTTERAUGLI
+ {
+ const int ba_w = mi_size_wide[butteraugli_rdo_bsize];
+ const int ba_h = mi_size_high[butteraugli_rdo_bsize];
+ const int ba_num_cols = (max_mi_cols + ba_w - 1) / ba_w;
+ const int ba_num_rows = (max_mi_rows + ba_h - 1) / ba_h;
+ aom_free(cpi->butteraugli_info.rdmult_scaling_factors);
+ cpi->butteraugli_info.rdmult_scaling_factors = NULL;
+ CHECK_MEM_ERROR(
+ cm, cpi->butteraugli_info.rdmult_scaling_factors,
+ aom_malloc(ba_num_rows * ba_num_cols *
+ sizeof(*cpi->butteraugli_info.rdmult_scaling_factors)));
+ }
+#endif // CONFIG_TUNE_BUTTERAUGLI
+
+#if CONFIG_SALIENCY_MAP
+ {
+ const int max_width = cpi->oxcf.frm_dim_cfg.forced_max_frame_width
+ ? cpi->oxcf.frm_dim_cfg.forced_max_frame_width
+ : cm->width;
+ const int max_height = cpi->oxcf.frm_dim_cfg.forced_max_frame_height
+ ? cpi->oxcf.frm_dim_cfg.forced_max_frame_height
+ : cm->height;
+ aom_free(cpi->saliency_map);
+ cpi->saliency_map = NULL;
+ CHECK_MEM_ERROR(cm, cpi->saliency_map,
+ (uint8_t *)aom_calloc(max_height * max_width,
+ sizeof(*cpi->saliency_map)));
+ // Buffer initialization based on MIN_MIB_SIZE_LOG2 to ensure that
+ // cpi->sm_scaling_factor buffer is allocated big enough, since we have no
+ // idea of the actual superblock size we are going to use yet.
+ const int min_mi_w_sb = (1 << MIN_MIB_SIZE_LOG2);
+ const int min_mi_h_sb = (1 << MIN_MIB_SIZE_LOG2);
+ const int max_sb_cols = (max_mi_cols + min_mi_w_sb - 1) / min_mi_w_sb;
+ const int max_sb_rows = (max_mi_rows + min_mi_h_sb - 1) / min_mi_h_sb;
+ aom_free(cpi->sm_scaling_factor);
+ cpi->sm_scaling_factor = NULL;
+ CHECK_MEM_ERROR(cm, cpi->sm_scaling_factor,
+ (double *)aom_calloc(max_sb_rows * max_sb_cols,
+ sizeof(*cpi->sm_scaling_factor)));
+ }
+#endif // CONFIG_SALIENCY_MAP
+}
+
static inline void alloc_obmc_buffers(OBMCBuffer *obmc_buffer,
struct aom_internal_error_info *error) {
AOM_CHECK_MEM_ERROR(
diff --git a/test/forced_max_frame_width_height_test.cc b/test/forced_max_frame_width_height_test.cc
index 4ae6701eab..3f4b153146 100644
--- a/test/forced_max_frame_width_height_test.cc
+++ b/test/forced_max_frame_width_height_test.cc
@@ -631,6 +631,55 @@ TEST(EncodeForcedMaxFrameWidthHeight, RaiseForcedMaxRealtime) {
TEST(EncodeForcedMaxFrameWidthHeight, RaiseForcedMaxGoodQuality) {
RunRaiseForcedMaxTest(AOM_USAGE_GOOD_QUALITY);
}
+
+// Without a forced maximum frame size, growing the frame size with
+// aom_codec_enc_config_set() before the first frame is encoded (while the
+// sequence header can still change) must also grow the SSIM/TPL rdmult
+// scaling factor buffers.
+void RunUpscaleWithoutForcedMaxTest(unsigned int usage, const char *tune) {
+ constexpr unsigned int kWidth = 64;
+ constexpr unsigned int kHeight = 64;
+ constexpr unsigned int kNewWidth = 640;
+ constexpr unsigned int kNewHeight = 480;
+ aom_codec_iface_t *iface = aom_codec_av1_cx();
+ aom_codec_enc_cfg_t cfg;
+ ASSERT_EQ(AOM_CODEC_OK, aom_codec_enc_config_default(iface, &cfg, usage));
+ cfg.g_w = kWidth;
+ cfg.g_h = kHeight;
+ cfg.g_lag_in_frames = 0;
+ aom_codec_ctx_t enc;
+ ASSERT_EQ(AOM_CODEC_OK, aom_codec_enc_init(&enc, iface, &cfg, 0));
+ ASSERT_EQ(AOM_CODEC_OK, aom_codec_control(&enc, AOME_SET_CPUUSED, 6));
+ ASSERT_EQ(AOM_CODEC_OK, aom_codec_set_option(&enc, "tune", tune));
+
+ cfg.g_w = kNewWidth;
+ cfg.g_h = kNewHeight;
+ ASSERT_EQ(AOM_CODEC_OK, aom_codec_enc_config_set(&enc, &cfg));
+
+ aom_image_t *image =
+ aom_img_alloc(nullptr, AOM_IMG_FMT_I420, kNewWidth, kNewHeight, 1);
+ ASSERT_NE(image, nullptr);
+ FillImagePattern(image, 0);
+ ASSERT_EQ(AOM_CODEC_OK, aom_codec_encode(&enc, image, 0, 1, 0));
+ FillImagePattern(image, 1);
+ ASSERT_EQ(AOM_CODEC_OK, aom_codec_encode(&enc, image, 1, 1, 0));
+
+ ASSERT_EQ(AOM_CODEC_OK, aom_codec_encode(&enc, nullptr, 0, 0, 0));
+ aom_img_free(image);
+ ASSERT_EQ(AOM_CODEC_OK, aom_codec_destroy(&enc));
+}
+
+TEST(EncodeForcedMaxFrameWidthHeight, UpscaleWithoutForcedMaxAllIntraSSIM) {
+ RunUpscaleWithoutForcedMaxTest(AOM_USAGE_ALL_INTRA, "ssim");
+}
+
+TEST(EncodeForcedMaxFrameWidthHeight, UpscaleWithoutForcedMaxAllIntraIQ) {
+ RunUpscaleWithoutForcedMaxTest(AOM_USAGE_ALL_INTRA, "iq");
+}
+
+TEST(EncodeForcedMaxFrameWidthHeight, UpscaleWithoutForcedMaxGoodSSIM) {
+ RunUpscaleWithoutForcedMaxTest(AOM_USAGE_GOOD_QUALITY, "ssim");
+}
#endif // !CONFIG_REALTIME_ONLY
} // namespace