Commit 1aea66455a for aom
commit 1aea66455a5e7f29619cb118cd2643715971a366
Author: Cheng Chen <chengchen@google.com>
Date: Wed Sep 9 12:14:08 2026 -0700
Clamp intra predictor neighbor pixel counts
When predicting subblocks near or beyond frame boundaries (where
mb_to_bottom_edge or mb_to_right_edge is negative), xr + txwpx,
yd + txhpx, xr, and yd can evaluate to negative numbers. Using
AOMMIN(txhpx, yd + txhpx) yields a negative n_left_px, triggering
assert(n_left_px >= 0) in build_intra_predictors_common().
Use clamp(..., 0, ...) to ensure n_top_px, n_left_px, n_topright_px,
and n_bottomleft_px are always non-negative.
Bug: b/558446054
TAG=agy
Change-Id: I746e612a44d8e9e0f83c1d3c3b47870cd459a80f
diff --git a/av1/common/reconintra.c b/av1/common/reconintra.c
index 33016a2c24..4e94c00913 100644
--- a/av1/common/reconintra.c
+++ b/av1/common/reconintra.c
@@ -1750,8 +1750,8 @@ void av1_predict_intra_block(const MACROBLOCKD *xd, BLOCK_SIZE sb_size,
// separate function build_non_directional_intra_predictors() is introduced
// for these modes to avoid redundant computations while generating pred data.
- const int n_top_px = have_top ? AOMMIN(txwpx, xr + txwpx) : 0;
- const int n_left_px = have_left ? AOMMIN(txhpx, yd + txhpx) : 0;
+ const int n_top_px = have_top ? clamp(xr + txwpx, 0, txwpx) : 0;
+ const int n_left_px = have_left ? clamp(yd + txhpx, 0, txhpx) : 0;
if (!use_filter_intra && !is_dr_mode) {
#if CONFIG_AV1_HIGHBITDEPTH
if (is_hbd) {
@@ -1815,9 +1815,9 @@ void av1_predict_intra_block(const MACROBLOCKD *xd, BLOCK_SIZE sb_size,
const int disable_edge_filter = !enable_intra_edge_filter;
const int intra_edge_filter_type = get_intra_edge_filter_type(xd, plane);
const int n_topright_px =
- have_top_right > 0 ? AOMMIN(txwpx, xr) : have_top_right;
+ have_top_right > 0 ? clamp(xr, 0, txwpx) : have_top_right;
const int n_bottomleft_px =
- have_bottom_left > 0 ? AOMMIN(txhpx, yd) : have_bottom_left;
+ have_bottom_left > 0 ? clamp(yd, 0, txhpx) : have_bottom_left;
#if CONFIG_AV1_HIGHBITDEPTH
if (is_hbd) {
highbd_build_directional_and_filter_intra_predictors(
diff --git a/test/encode_api_test.cc b/test/encode_api_test.cc
index b95b3c22f0..baf8b10c68 100644
--- a/test/encode_api_test.cc
+++ b/test/encode_api_test.cc
@@ -29,6 +29,9 @@
#include "aom/aom_image.h"
#include "aom_mem/aom_mem.h"
+#include "av1/common/blockd.h"
+#include "av1/common/reconintra.h"
+#include "av1/encoder/allintra_vis.h"
#include "test/codec_factory.h"
#include "test/encode_test_driver.h"
#include "test/util.h"
@@ -2889,4 +2892,74 @@ TEST(EncodeAPI, Issue559225640) {
/*init_threads=*/1, /*max_threads=*/4);
}
+#if !CONFIG_REALTIME_ONLY
+TEST(EncodeAPI, Buganizer558446054) {
+#if !CONFIG_SHARED
+ av1_init_intra_predictors();
+
+ MACROBLOCKD xd = {};
+ YV12_BUFFER_CONFIG cur_buf = {};
+ xd.cur_buf = &cur_buf;
+ MB_MODE_INFO mbmi = {};
+ mbmi.bsize = BLOCK_8X8;
+ mbmi.partition = PARTITION_NONE;
+ mbmi.mode = D45_PRED;
+ MB_MODE_INFO *mbmi_ptr = &mbmi;
+ xd.mi = &mbmi_ptr;
+ xd.bd = 8;
+ xd.left_available = 1;
+ xd.up_available = 1;
+ xd.tile.mi_row_end = 100;
+ xd.tile.mi_col_end = 100;
+ // Set mb_to_bottom_edge and mb_to_right_edge such that
+ // yd + txhpx < 0 and xr < 0:
+ // yd + txhpx = (mb_to_bottom_edge >> 3) + hpx - y = -16 + 8 - 0 = -8 < 0
+ // xr = (mb_to_right_edge >> 3) + wpx - x - txwpx = -5 + 8 - 0 - 4 = -1 < 0
+ // xr + txwpx = 3 > 0 (so n_top_px = 3).
+ xd.mb_to_bottom_edge = -16 * 8;
+ xd.mb_to_right_edge = -5 * 8;
+
+ uint8_t ref_buf[64 * 64];
+ memset(ref_buf, 200, sizeof(ref_buf));
+ uint8_t dst_buf[64 * 64] = { 0 };
+ av1_predict_intra_block(&xd, BLOCK_64X64, /*enable_intra_edge_filter=*/0,
+ /*wpx=*/8, /*hpx=*/8, TX_4X4, D45_PRED,
+ /*angle_delta=*/0, /*use_palette=*/0,
+ FILTER_INTRA_MODES, ref_buf + 64 * 8 + 8, 64, dst_buf,
+ 64, /*col_off=*/0, /*row_off=*/0, /*plane=*/0);
+ // In Debug builds (-UNDEBUG), reverting clamp() triggers:
+ // Assertion `n_left_px >= 0' failed.
+ // In Release builds (-DNDEBUG), reverting clamp() causes n_topright_px < 0
+ // (-4 < 0), skipping top-right extension of above_row[3]=200 into
+ // above_row[4..7] and leaving dst_buf[3] == 127 instead of 200.
+ ASSERT_EQ(dst_buf[3], 200);
+#endif // !CONFIG_SHARED
+
+ aom_codec_iface_t *iface = aom_codec_av1_cx();
+ aom_codec_enc_cfg_t cfg;
+ ASSERT_EQ(aom_codec_enc_config_default(iface, &cfg, AOM_USAGE_ALL_INTRA),
+ AOM_CODEC_OK);
+
+ cfg.g_w = 65;
+ cfg.g_h = 33;
+ cfg.g_threads = 0;
+ cfg.g_lag_in_frames = 0;
+ cfg.g_pass = AOM_RC_ONE_PASS;
+ cfg.rc_end_usage = AOM_Q;
+
+ aom_codec_ctx_t codec;
+ ASSERT_EQ(aom_codec_enc_init(&codec, iface, &cfg, 0), AOM_CODEC_OK);
+ ASSERT_EQ(aom_codec_control(&codec, AOME_SET_CPUUSED, 1), AOM_CODEC_OK);
+
+ aom_image_t raw;
+ ASSERT_NE(aom_img_alloc(&raw, AOM_IMG_FMT_I420, 65, 33, 1), nullptr);
+ FillImageRandom(&raw);
+
+ ASSERT_EQ(aom_codec_encode(&codec, &raw, 0, 1, 0), AOM_CODEC_OK);
+
+ aom_img_free(&raw);
+ ASSERT_EQ(aom_codec_destroy(&codec), AOM_CODEC_OK);
+}
+#endif // !CONFIG_REALTIME_ONLY
+
} // namespace