Commit e8b13a0936 for aom
commit e8b13a0936be81657c466f6a99f9a86e0365ef99
Author: James Zern <jzern@google.com>
Date: Thu Mar 5 19:16:02 2026 -0800
Fix int16_t overflow in CDEF search for frames > 32768 pixels
get_filt_error() used FULLPEL_MV (int16_t row/col) to store pixel
positions within the frame and compute buffer offsets via
get_offset_from_fullmv(). For frames with height or width exceeding
32768 pixels, the pixel coordinate overflows int16_t, wrapping to a
negative value. This causes get_offset_from_fullmv() to compute a
negative buffer offset, resulting in an out-of-bounds read.
FULLPEL_MV was designed for motion vectors (bounded by search range),
not for absolute pixel coordinates in large frames. The AV1 spec
allows frames up to 65536x65536, so these dimensions are within valid
encoder input range.
Replace FULLPEL_MV + get_offset_from_fullmv() / get_buf_from_fullmv()
with direct int arithmetic at all four usage sites in get_filt_error().
Remove the now-unused mcomp.h include.
Based on original set of patches from Chun-Min Chang.
Bug: 489473886
Change-Id: I01fb3c981a5a7750fc029b6c98fc73f60614849c
diff --git a/av1/encoder/pickcdef.c b/av1/encoder/pickcdef.c
index 918f6fd3a6..0b14bf870f 100644
--- a/av1/encoder/pickcdef.c
+++ b/av1/encoder/pickcdef.c
@@ -11,6 +11,7 @@
#include <math.h>
#include <stdbool.h>
+#include <stddef.h>
#include <string.h>
#include "config/aom_dsp_rtcd.h"
@@ -22,7 +23,6 @@
#include "av1/encoder/encoder.h"
#include "av1/encoder/ethread.h"
#include "av1/encoder/pickcdef.h"
-#include "av1/encoder/mcomp.h"
// Get primary and secondary filter strength for the given strength index and
// search method
@@ -418,15 +418,14 @@ static inline uint64_t get_filt_error(
(block_size_wide[plane_bsize] * block_size_high[plane_bsize]) >>
(bw_log2 + bh_log2);
if (cdef_count == tot_blk_count) {
- // Calculate the offset in the buffer based on block position
- const FULLPEL_MV this_mv = { row, col };
- const int buf_offset = get_offset_from_fullmv(&this_mv, ref_stride);
+ const ptrdiff_t buf_offset = (ptrdiff_t)row * ref_stride + col;
+ const ptrdiff_t dst_offset = (ptrdiff_t)row * pd->dst.stride + col;
if (pri_strength == 0 && sec_strength == 0) {
// When CDEF strength is zero, filtering is not applied. Hence
// error is calculated between source and unfiltered pixels
curr_sse =
aom_sse(&ref_buffer[buf_offset], ref_stride,
- get_buf_from_fullmv(&pd->dst, &this_mv), pd->dst.stride,
+ &pd->dst.buf[dst_offset], pd->dst.stride,
block_size_wide[plane_bsize], block_size_high[plane_bsize]);
} else {
DECLARE_ALIGNED(32, uint8_t, tmp_dst8[1 << (MAX_SB_SIZE_LOG2 * 2)]);
@@ -451,17 +450,18 @@ static inline uint64_t get_filt_error(
for (int bi = 0; bi < cdef_count; bi = bi + num_error_calc_filt_units) {
const uint8_t by = dlist[bi].by;
const uint8_t bx = dlist[bi].bx;
- const int16_t by_pos = (by << bh_log2);
- const int16_t bx_pos = (bx << bw_log2);
- // Calculate the offset in the buffer based on block position
- const FULLPEL_MV this_mv = { row + by_pos, col + bx_pos };
- const int buf_offset = get_offset_from_fullmv(&this_mv, ref_stride);
+ const int by_pos = by << bh_log2;
+ const int bx_pos = bx << bw_log2;
+ const ptrdiff_t buf_offset =
+ (ptrdiff_t)(row + by_pos) * ref_stride + (col + bx_pos);
+ const ptrdiff_t dst_offset =
+ (ptrdiff_t)(row + by_pos) * pd->dst.stride + (col + bx_pos);
num_error_calc_filt_units = get_error_calc_width_in_filt_units(
dlist, cdef_count, bi, pd->subsampling_x, pd->subsampling_y);
- curr_sse += aom_sse(
- &ref_buffer[buf_offset], ref_stride,
- get_buf_from_fullmv(&pd->dst, &this_mv), pd->dst.stride,
- num_error_calc_filt_units * (1 << bw_log2), (1 << bh_log2));
+ curr_sse +=
+ aom_sse(&ref_buffer[buf_offset], ref_stride,
+ &pd->dst.buf[dst_offset], pd->dst.stride,
+ num_error_calc_filt_units * (1 << bw_log2), 1 << bh_log2);
}
} else {
DECLARE_ALIGNED(32, uint8_t, tmp_dst8[1 << (MAX_SB_SIZE_LOG2 * 2)]);
@@ -475,14 +475,12 @@ static inline uint64_t get_filt_error(
for (int bi = 0; bi < cdef_count; bi = bi + num_error_calc_filt_units) {
const uint8_t by = dlist[bi].by;
const uint8_t bx = dlist[bi].bx;
- const int16_t by_pos = (by << bh_log2);
- const int16_t bx_pos = (bx << bw_log2);
- // Calculate the offset in the buffer based on block position
- const FULLPEL_MV this_mv = { row + by_pos, col + bx_pos };
- const FULLPEL_MV tmp_buf_pos = { by_pos, bx_pos };
- const int buf_offset = get_offset_from_fullmv(&this_mv, ref_stride);
- const int tmp_buf_offset =
- get_offset_from_fullmv(&tmp_buf_pos, (1 << MAX_SB_SIZE_LOG2));
+ const int by_pos = by << bh_log2;
+ const int bx_pos = bx << bw_log2;
+ const ptrdiff_t buf_offset =
+ (ptrdiff_t)(row + by_pos) * ref_stride + (col + bx_pos);
+ const ptrdiff_t tmp_buf_offset =
+ by_pos * (1 << MAX_SB_SIZE_LOG2) + bx_pos;
num_error_calc_filt_units = get_error_calc_width_in_filt_units(
dlist, cdef_count, bi, pd->subsampling_x, pd->subsampling_y);
curr_sse += aom_sse(
diff --git a/test/encode_large_width_height_test.cc b/test/encode_large_width_height_test.cc
new file mode 100644
index 0000000000..dfce8b3b60
--- /dev/null
+++ b/test/encode_large_width_height_test.cc
@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2026, Alliance for Open Media. All rights reserved.
+ *
+ * This source code is subject to the terms of the BSD 2 Clause License and
+ * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+ * was not distributed with this source code in the LICENSE file, you can
+ * obtain it at www.aomedia.org/license/software. If the Alliance for Open
+ * Media Patent License 1.0 was not distributed with this source code in the
+ * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+ */
+
+// Test for encoding frames with dimensions > 32768 pixels.
+//
+// pickcdef.c used FULLPEL_MV (int16_t row/col) to store pixel positions in
+// get_filt_error(). For frames taller than 32768 pixels, the pixel row
+// overflows int16_t, wrapping to a negative value and causing an out-of-bounds
+// read. aom_codec_encode() should not crash or read out of bounds. See bug
+// 489473886.
+
+#include <memory>
+
+#include "gtest/gtest.h"
+
+#include "aom/aom_encoder.h"
+#include "aom/aom_image.h"
+#include "aom/aomcx.h"
+#include "config/aom_config.h"
+#include "test/video_source.h"
+
+namespace {
+
+// Encode a single frame with the given dimensions and flush.
+void EncodeSingleFrame(unsigned int width, unsigned int height,
+ unsigned int usage, int cpu_used) {
+ aom_codec_iface_t *iface = aom_codec_av1_cx();
+ aom_codec_enc_cfg_t cfg;
+ ASSERT_EQ(aom_codec_enc_config_default(iface, &cfg, usage), AOM_CODEC_OK);
+ cfg.g_w = width;
+ cfg.g_h = height;
+ cfg.g_lag_in_frames = 0;
+ cfg.g_pass = AOM_RC_ONE_PASS;
+ // Use VBR with moderate QP to ensure non-skip blocks, which is necessary
+ // to exercise the CDEF search path. AOM_Q with low cq_level triggers an
+ // early return in adaptive CDEF for ALL_INTRA mode.
+ cfg.rc_end_usage = AOM_VBR;
+ cfg.rc_target_bitrate = 1000;
+
+ aom_codec_ctx_t ctx;
+ ASSERT_EQ(aom_codec_enc_init(&ctx, iface, &cfg, 0), AOM_CODEC_OK);
+ std::unique_ptr<aom_codec_ctx_t, decltype(&aom_codec_destroy)> enc(
+ &ctx, &aom_codec_destroy);
+ ASSERT_EQ(aom_codec_control(enc.get(), AOME_SET_CPUUSED, cpu_used),
+ AOM_CODEC_OK);
+
+ libaom_test::RandomVideoSource video;
+ video.SetSize(width, height);
+ video.SetImageFormat(AOM_IMG_FMT_I420);
+ video.Begin();
+
+ ASSERT_EQ(aom_codec_encode(enc.get(), video.img(), video.pts(),
+ /*duration=*/1, /*flags=*/0),
+ AOM_CODEC_OK)
+ << aom_codec_error_detail(enc.get());
+ ASSERT_EQ(aom_codec_encode(enc.get(), nullptr, 0, 0, 0), AOM_CODEC_OK)
+ << aom_codec_error_detail(enc.get());
+}
+
+class EncodeBigDimension
+ : public testing::TestWithParam<unsigned int /*usage*/> {};
+
+// Height > 32768: triggers the CDEF int16_t overflow at superblock row 512.
+// 32832 = 513 * 64, giving 513 superblock rows (fbr 0..512).
+// Use ALL_INTRA at speed 6 so the full CDEF search runs (not CDEF_PICK_FROM_Q)
+// while keeping encode time reasonable.
+TEST_P(EncodeBigDimension, TallFrame) {
+ EncodeSingleFrame(/*width=*/64, /*height=*/32832, /*usage=*/GetParam(),
+ /*cpu_used=*/5);
+}
+
+// Width > 32768: same int16_t overflow but in the column direction.
+TEST_P(EncodeBigDimension, WideFrame) {
+ EncodeSingleFrame(/*width=*/32832, /*height=*/64, /*usage=*/GetParam(),
+ /*cpu_used=*/5);
+}
+
+constexpr unsigned int kUsages[] = {
+ AOM_USAGE_REALTIME,
+#if !CONFIG_REALTIME_ONLY
+ AOM_USAGE_GOOD_QUALITY,
+ AOM_USAGE_ALL_INTRA,
+#endif
+};
+
+INSTANTIATE_TEST_SUITE_P(All, EncodeBigDimension, testing::ValuesIn(kUsages));
+
+} // namespace
diff --git a/test/test.cmake b/test/test.cmake
index c50f84a2de..ddef74db20 100644
--- a/test/test.cmake
+++ b/test/test.cmake
@@ -73,6 +73,7 @@ list(APPEND AOM_UNIT_TEST_ENCODER_SOURCES
"${AOM_ROOT}/test/dropframe_encode_test.cc"
"${AOM_ROOT}/test/svc_datarate_test.cc"
"${AOM_ROOT}/test/encode_api_test.cc"
+ "${AOM_ROOT}/test/encode_large_width_height_test.cc"
"${AOM_ROOT}/test/encode_small_width_height_test.cc"
"${AOM_ROOT}/test/encode_test_driver.cc"
"${AOM_ROOT}/test/encode_test_driver.h"