Commit 70fb50a413 for aom

commit 70fb50a413950046bc173a582fec0debd08c9313
Author: Jingning Han <jingning@google.com>
Date:   Mon Nov 9 14:03:12 2026 -0800

    Fix integer overflow when computing set_avg_64x64

    Fix the following UBSan error by declaring avg_64x64 as int64_t
    instead of int, and computing set_avg_64x64 in int64_t:

    av1/encoder/var_based_part.c:1887:34: runtime error:
    signed integer overflow: 9 * 471859200 cannot be represented in type
    'int'

    On key frames, the 4x4 averages are compared against a fixed dst_avg
    of 128 even for high bit depth input, and the samples outside the
    frame boundary contribute zero. This makes the variance of the 64x64
    blocks that straddle the frame edge very large for 12-bit input, and
    the sum of up to four such variances times 9 overflows int.

    Tested: test_libaom --gtest_filter=EncodeAPI.Avg64x64IntegerOverflow

    Bug: b:580524406
    Change-Id: Id537dc1e107e711c15b0e5af1f7f6a5bb92344ad

diff --git a/av1/encoder/var_based_part.c b/av1/encoder/var_based_part.c
index 24b367ff8f..a25d6ce03a 100644
--- a/av1/encoder/var_based_part.c
+++ b/av1/encoder/var_based_part.c
@@ -1611,7 +1611,7 @@ int av1_choose_var_based_partitioning(AV1_COMP *cpi, const TileInfo *const tile,
   MACROBLOCKD *xd = &x->e_mbd;
   const int64_t *const vbp_thresholds = cpi->vbp_info.thresholds;
   PART_EVAL_STATUS force_split[85];
-  int avg_64x64;
+  int64_t avg_64x64;
   int max_var_32x32[4];
   int min_var_32x32[4];
   int var_32x32;
@@ -1884,7 +1884,7 @@ int av1_choose_var_based_partitioning(AV1_COMP *cpi, const TileInfo *const tile,
   if (force_split[0] == PART_EVAL_ALL) {
     fill_variance_tree(vt, BLOCK_128X128);
     get_variance(&vt->part_variances.none);
-    const int set_avg_64x64 = (9 * avg_64x64) >> 5;
+    const int64_t set_avg_64x64 = (9 * avg_64x64) >> 5;
     if (!is_key_frame && vt->part_variances.none.variance > set_avg_64x64)
       force_split[0] = PART_EVAL_ONLY_SPLIT;

diff --git a/test/encode_api_test.cc b/test/encode_api_test.cc
index 651e4bcdfc..8332b14072 100644
--- a/test/encode_api_test.cc
+++ b/test/encode_api_test.cc
@@ -4848,6 +4848,73 @@ TEST(EncodeAPI, HighbdMisalignedPlanesAndStrides) {

   ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK);
 }
+
+// A test that reproduces b/580524406: signed integer overflow when computing
+// (9 * avg_64x64) in av1_choose_var_based_partitioning().
+TEST(EncodeAPI, Avg64x64IntegerOverflow) {
+  aom_codec_iface_t *const iface = aom_codec_av1_cx();
+  aom_codec_enc_cfg_t cfg;
+  ASSERT_EQ(aom_codec_enc_config_default(iface, &cfg, AOM_USAGE_REALTIME),
+            AOM_CODEC_OK);
+  cfg.g_threads = 16;
+  cfg.g_profile = 2;
+  cfg.g_w = 729;
+  cfg.g_h = 806;
+  cfg.g_forced_max_frame_width = cfg.g_w;
+  cfg.g_forced_max_frame_height = cfg.g_h;
+  cfg.g_bit_depth = AOM_BITS_12;
+  cfg.g_input_bit_depth = 12;
+  cfg.g_timebase.num = 1;
+  cfg.g_timebase.den = 1000 * 1000;  // microseconds
+  cfg.g_pass = AOM_RC_ONE_PASS;
+  cfg.g_lag_in_frames = 0;
+  cfg.rc_end_usage = AOM_CBR;
+  cfg.rc_min_quantizer = 2;
+  cfg.rc_max_quantizer = 58;
+
+  aom_codec_ctx_t enc;
+  ASSERT_EQ(aom_codec_enc_init(&enc, iface, &cfg, AOM_CODEC_USE_HIGHBITDEPTH),
+            AOM_CODEC_OK);
+  ASSERT_EQ(aom_codec_control(&enc, AOME_SET_CPUUSED, 7), AOM_CODEC_OK);
+
+  // Encode a gray 12-bit 4:2:0 key frame.
+  aom_image_t *const image =
+      aom_img_alloc(nullptr, AOM_IMG_FMT_I42016, cfg.g_w, cfg.g_h, 1);
+  ASSERT_NE(image, nullptr);
+  const uint16_t kGray = 1 << 11;
+  for (int plane = 0; plane < 3; ++plane) {
+    const unsigned int w = aom_img_plane_width(image, plane);
+    const unsigned int h = aom_img_plane_height(image, plane);
+    for (unsigned int i = 0; i < h; ++i) {
+      uint16_t *const row = reinterpret_cast<uint16_t *>(
+          image->planes[plane] + i * image->stride[plane]);
+      for (unsigned int j = 0; j < w; ++j) row[j] = kGray;
+    }
+  }
+  ASSERT_EQ(aom_codec_encode(&enc, image, 0, 1, AOM_EFLAG_FORCE_KF),
+            AOM_CODEC_OK);
+  aom_codec_iter_t iter = nullptr;
+  const aom_codec_cx_pkt_t *pkt;
+  while ((pkt = aom_codec_get_cx_data(&enc, &iter)) != nullptr) {
+    ASSERT_EQ(pkt->kind, AOM_CODEC_CX_FRAME_PKT);
+    ASSERT_EQ(pkt->data.frame.flags & AOM_FRAME_IS_KEY, AOM_FRAME_IS_KEY);
+  }
+  aom_img_free(image);
+
+  // Flush the encoder.
+  bool got_data;
+  do {
+    ASSERT_EQ(aom_codec_encode(&enc, nullptr, 0, 0, 0), AOM_CODEC_OK);
+    got_data = false;
+    iter = nullptr;
+    while ((pkt = aom_codec_get_cx_data(&enc, &iter)) != nullptr) {
+      ASSERT_EQ(pkt->kind, AOM_CODEC_CX_FRAME_PKT);
+      got_data = true;
+    }
+  } while (got_data);
+
+  ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK);
+}
 #endif  // CONFIG_AV1_HIGHBITDEPTH

 // A key frame whose luma is a block of noise repeated over the frame. IntraBC