Commit 84877ce808 for aom

commit 84877ce80853dba082fc90ecd16380a7428cbae4
Author: Ihor Buhaiov <freedbrt@gmail.com>
Date:   Sat Oct 17 16:42:25 2026 +0200

    Grow reference borders when the border grows

    The frame border comes from av1_get_enc_border_size() and is
    recomputed for every frame in av1_set_frame_size(). It is 288 pixels
    when the encoder may resize and the superblock size plus 32 otherwise.
    Frames coded before resizing was turned on keep the small border.
    av1_scale_references() only gives a reference the new border when it
    has to scale it, so a reference with the same size as the new frame
    keeps the old one.

    The screen motion search in variance based partitioning reads up to
    border_in_pixels around the reference, so it reads past the end of
    such a buffer. Setting AOME_SET_SCALEMODE to ONETWO before the first
    frame and again before the second one does it. So does turning on
    rc_resize_mode through aom_codec_enc_config_set() in the middle of a
    stream. ASan reports the read in the Neon, AVX2, SSE2 and C versions
    of aom_int_pro_col().

    Give such references the larger border in av1_set_frame_size(), with
    the helper av1_scale_references() already uses. This costs one copy
    per reference, and only when the border grows.

    The new EncodeAPI.ScaleModeBeforeFirstFrameScreen test fails under
    ASan without this change.

    Change-Id: I09908d8e8e35cba715aa2f36b30bb7bf200ab085

diff --git a/av1/encoder/encoder.c b/av1/encoder/encoder.c
index f3ded02c2b..fa9f81fca3 100644
--- a/av1/encoder/encoder.c
+++ b/av1/encoder/encoder.c
@@ -2738,6 +2738,16 @@ void av1_set_frame_size(AV1_COMP *cpi, int width, int height) {
   for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
     RefCntBuffer *const buf = get_ref_frame_buf(cm, ref_frame);
     if (buf != NULL) {
+      // Motion search may read up to border_in_pixels outside a reference.
+      // The border grows when resizing is turned on, so give references
+      // coded before that the same border.
+      if (buf->buf.border < cpi->oxcf.border_in_pixels &&
+          aom_yv12_realloc_with_new_border(
+              &buf->buf, cpi->oxcf.border_in_pixels,
+              cm->features.byte_alignment, cpi->alloc_pyramid, num_planes)) {
+        aom_internal_error(cm->error, AOM_CODEC_MEM_ERROR,
+                           "Failed to allocate frame buffer");
+      }
       struct scale_factors *sf = get_ref_scale_factors(cm, ref_frame);
       av1_setup_scale_factors_for_frame(sf, buf->buf.y_crop_width,
                                         buf->buf.y_crop_height, cm->width,
diff --git a/test/encode_api_test.cc b/test/encode_api_test.cc
index 1a2e43c06d..7a9e280b87 100644
--- a/test/encode_api_test.cc
+++ b/test/encode_api_test.cc
@@ -1776,6 +1776,44 @@ TEST(EncodeAPI, IntraBCFrameSizeDecreaseRealtime) {
   ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK);
 }

+// Sets the scale mode before the first frame and again before the second one
+// in realtime screen mode. The first frame is coded with the frame border of
+// an encoder that doesn't resize, and the second call makes the encoder use
+// the larger resize border. The screen motion search in variance based
+// partitioning reads up to that border around the reference, so the
+// reference must get the larger border too.
+TEST(EncodeAPI, ScaleModeBeforeFirstFrameScreen) {
+  aom_codec_iface_t *iface = aom_codec_av1_cx();
+  aom_codec_enc_cfg_t cfg;
+  ASSERT_EQ(aom_codec_enc_config_default(iface, &cfg, AOM_USAGE_REALTIME),
+            AOM_CODEC_OK);
+  cfg.g_w = 640;
+  cfg.g_h = 360;
+  cfg.g_lag_in_frames = 0;
+  cfg.g_threads = 1;
+
+  aom_codec_ctx_t enc;
+  ASSERT_EQ(aom_codec_enc_init(&enc, iface, &cfg, 0), AOM_CODEC_OK);
+  ASSERT_EQ(aom_codec_control(&enc, AOME_SET_CPUUSED, 10), AOM_CODEC_OK);
+  ASSERT_EQ(aom_codec_control(&enc, AV1E_SET_TUNE_CONTENT, AOM_CONTENT_SCREEN),
+            AOM_CODEC_OK);
+
+  aom_image_t *img = aom_img_alloc(nullptr, AOM_IMG_FMT_I420, 640, 360, 1);
+  ASSERT_NE(img, nullptr);
+  aom_scaling_mode_t mode = { AOME_ONETWO, AOME_ONETWO };
+
+  ASSERT_EQ(aom_codec_control(&enc, AOME_SET_SCALEMODE, &mode), AOM_CODEC_OK);
+  FillImage(img, 128);
+  EncodeOne(&enc, img, 0);
+
+  ASSERT_EQ(aom_codec_control(&enc, AOME_SET_SCALEMODE, &mode), AOM_CODEC_OK);
+  FillImageRandom(img);
+  EncodeOne(&enc, img, 1);
+
+  aom_img_free(img);
+  ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK);
+}
+
 #if CONFIG_AV1_DECODER
 // Draws a text like screen texture moved left by offset_x pixels.
 void FillPannedScreen(aom_image_t *img, int offset_x) {