Commit 129522fa62 for aom

commit 129522fa623771be2eeb03c1deaaf82bb04d3253
Author: Jerome Jiang <jianj@google.com>
Date:   Thu Nov 12 10:49:35 2026 -0500

    Prevent int overflows in var_based_part.c

    - Clamp VPartVar.variance to INT_MAX in get_variance() so 12-bit
      inter-frame variance cannot wrap to negative.
    - Widen sub-block variance differences and minvar_16x16 << 2 shift
      to int64_t.
    - Add EncodeAPI.VarBasedPart12BitVarianceOverflow unit test.

    Bug: b:584264300
    Change-Id: Ic41ed0390b736d18d9cc0cde12958b8b58292f17

diff --git a/av1/encoder/var_based_part.c b/av1/encoder/var_based_part.c
index a25d6ce03a..dc92fdf0c3 100644
--- a/av1/encoder/var_based_part.c
+++ b/av1/encoder/var_based_part.c
@@ -111,7 +111,7 @@ static inline void get_variance(VPartVar *v) {
       (uint32_t)(((int64_t)v->sum_error * v->sum_error) >> v->log2_count);
   const int64_t variance =
       (int64_t)256 * (v->sum_square_error - square_sum_error) >> v->log2_count;
-  v->variance = (int)variance;
+  v->variance = (variance > INT_MAX) ? INT_MAX : (int)variance;
 }

 static inline void sum_2_variances(const VPartVar *a, const VPartVar *b,
@@ -1829,8 +1829,9 @@ int av1_choose_var_based_partitioning(AV1_COMP *cpi, const TileInfo *const tile,
             vt->split[blk64_idx].split[lvl1_idx].part_variances.none.variance;
         max_var_32x32[blk64_idx] = AOMMAX(var_32x32, max_var_32x32[blk64_idx]);
         min_var_32x32[blk64_idx] = AOMMIN(var_32x32, min_var_32x32[blk64_idx]);
-        const int max_min_var_16X16_diff = (maxvar_16x16[blk64_idx][lvl1_idx] -
-                                            minvar_16x16[blk64_idx][lvl1_idx]);
+        const int64_t max_min_var_16X16_diff =
+            (int64_t)maxvar_16x16[blk64_idx][lvl1_idx] -
+            minvar_16x16[blk64_idx][lvl1_idx];

         if (var_32x32 > thresholds[2] ||
             (!is_key_frame && var_32x32 > (thresholds[2] >> 1) &&
@@ -1846,7 +1847,7 @@ int av1_choose_var_based_partitioning(AV1_COMP *cpi, const TileInfo *const tile,
                      cpi->rc.frame_source_sad < frame_sad_thresh &&
                      maxvar_16x16[blk64_idx][lvl1_idx] > (thresholds[2] >> 4) &&
                      maxvar_16x16[blk64_idx][lvl1_idx] >
-                         (minvar_16x16[blk64_idx][lvl1_idx] << 2)))) {
+                         ((int64_t)minvar_16x16[blk64_idx][lvl1_idx] << 2)))) {
           force_split[5 + blk64_scale_idx + lvl1_idx] = PART_EVAL_ONLY_SPLIT;
           force_split[blk64_idx + 1] = PART_EVAL_ONLY_SPLIT;
           force_split[0] = PART_EVAL_ONLY_SPLIT;
@@ -1863,8 +1864,8 @@ int av1_choose_var_based_partitioning(AV1_COMP *cpi, const TileInfo *const tile,
       // variance of a sub-block is above some threshold of then force this
       // block to split. Only checking this for noise level >= medium, if
       // encoder is in SVC or if we already forced large blocks.
-      const int max_min_var_32x32_diff =
-          max_var_32x32[blk64_idx] - min_var_32x32[blk64_idx];
+      const int64_t max_min_var_32x32_diff =
+          (int64_t)max_var_32x32[blk64_idx] - min_var_32x32[blk64_idx];
       const int check_max_var = max_var_32x32[blk64_idx] > thresholds[1] >> 1;
       const bool check_noise_lvl = noise_level >= kMedium ||
                                    cpi->ppi->use_svc ||
@@ -1889,7 +1890,7 @@ int av1_choose_var_based_partitioning(AV1_COMP *cpi, const TileInfo *const tile,
       force_split[0] = PART_EVAL_ONLY_SPLIT;

     if (!is_key_frame &&
-        (max_var_64x64 - min_var_64x64) > 3 * (thresholds[0] >> 3) &&
+        ((int64_t)max_var_64x64 - min_var_64x64) > 3 * (thresholds[0] >> 3) &&
         max_var_64x64 > thresholds[0] >> 1)
       force_split[0] = PART_EVAL_ONLY_SPLIT;
   }
diff --git a/test/encode_api_test.cc b/test/encode_api_test.cc
index 3051b46b82..ea42365a7a 100644
--- a/test/encode_api_test.cc
+++ b/test/encode_api_test.cc
@@ -5387,6 +5387,90 @@ TEST(EncodeAPI, Avg64x64IntegerOverflow) {

   ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK);
 }
+
+// Reproduces signed integer overflow in av1_choose_var_based_partitioning()
+// when 12-bit inter-frame variance in get_variance() exceeds INT_MAX and wraps
+// negative, causing max_var_64x64 - min_var_64x64 to overflow int.
+TEST(EncodeAPI, VarBasedPart12BitVarianceOverflow) {
+  aom_codec_iface_t *const iface = aom_codec_av1_cx();
+  aom_codec_enc_cfg_t cfg;
+  ASSERT_EQ(aom_codec_enc_config_default(iface, &cfg, AOM_USAGE_REALTIME),
+            AOM_CODEC_OK);
+  cfg.g_profile = 2;
+  cfg.g_w = 128;
+  cfg.g_h = 128;
+  cfg.g_bit_depth = AOM_BITS_12;
+  cfg.g_input_bit_depth = 12;
+  cfg.g_pass = AOM_RC_ONE_PASS;
+  cfg.g_lag_in_frames = 0;
+  cfg.rc_end_usage = AOM_Q;
+  cfg.rc_min_quantizer = 2;
+  cfg.rc_max_quantizer = 2;
+
+  aom_codec_ctx_t enc;
+  ASSERT_EQ(aom_codec_enc_init(&enc, iface, &cfg, AOM_CODEC_USE_HIGHBITDEPTH),
+            AOM_CODEC_OK);
+  ASSERT_EQ(aom_codec_control(&enc, AOME_SET_CPUUSED, 10), AOM_CODEC_OK);
+  ASSERT_EQ(aom_codec_control(&enc, AOME_SET_CQ_LEVEL, 2), AOM_CODEC_OK);
+  ASSERT_EQ(aom_codec_control(&enc, AV1E_SET_SUPERBLOCK_SIZE,
+                              AOM_SUPERBLOCK_SIZE_128X128),
+            AOM_CODEC_OK);
+
+  aom_image_t *const img0 =
+      aom_img_alloc(nullptr, AOM_IMG_FMT_I42016, cfg.g_w, cfg.g_h, 1);
+  aom_image_t *const img1 =
+      aom_img_alloc(nullptr, AOM_IMG_FMT_I42016, cfg.g_w, cfg.g_h, 1);
+  ASSERT_NE(img0, nullptr);
+  ASSERT_NE(img1, nullptr);
+
+  // Fill chroma planes with mid-gray.
+  for (int plane = 1; plane < 3; ++plane) {
+    const unsigned int w = aom_img_plane_width(img0, plane);
+    const unsigned int h = aom_img_plane_height(img0, plane);
+    for (unsigned int y = 0; y < h; ++y) {
+      uint16_t *const row0 = reinterpret_cast<uint16_t *>(
+          img0->planes[plane] + y * img0->stride[plane]);
+      uint16_t *const row1 = reinterpret_cast<uint16_t *>(
+          img1->planes[plane] + y * img1->stride[plane]);
+      for (unsigned int x = 0; x < w; ++x) {
+        row0[x] = 2048;
+        row1[x] = 2048;
+      }
+    }
+  }
+
+  // Construct a 32x32 checkerboard in the top two 64x64 blocks so that every
+  // 16x16 and 32x32 sub-block has zero variance (keeping force_split[0] ==
+  // PART_EVAL_ALL), while the 64x64 inter-frame residual is +/-2890 in the
+  // left block (variance ~ 2.138e9 < INT_MAX) and +/-2910 in the right block
+  // (variance ~ 2.168e9 > INT_MAX, wrapping to ~ -2.127e9 without clamping).
+  for (unsigned int y = 0; y < cfg.g_h; ++y) {
+    uint16_t *const row0 = reinterpret_cast<uint16_t *>(
+        img0->planes[AOM_PLANE_Y] + y * img0->stride[AOM_PLANE_Y]);
+    uint16_t *const row1 = reinterpret_cast<uint16_t *>(
+        img1->planes[AOM_PLANE_Y] + y * img1->stride[AOM_PLANE_Y]);
+    for (unsigned int x = 0; x < cfg.g_w; ++x) {
+      if (y < 64) {
+        const bool high = (((x >> 5) ^ (y >> 5)) & 1) != 0;
+        const uint16_t delta = (x < 64) ? 2890 : 2910;
+        row0[x] = high ? 4095 : 0;
+        row1[x] = high ? static_cast<uint16_t>(4095 - delta) : delta;
+      } else {
+        row0[x] = 0;
+        row1[x] = 0;
+      }
+    }
+  }
+
+  ASSERT_EQ(aom_codec_encode(&enc, img0, 0, 1, AOM_EFLAG_FORCE_KF),
+            AOM_CODEC_OK);
+  ASSERT_EQ(aom_codec_encode(&enc, img1, 1, 1, 0), AOM_CODEC_OK);
+  ASSERT_EQ(aom_codec_encode(&enc, nullptr, 0, 0, 0), AOM_CODEC_OK);
+
+  aom_img_free(img1);
+  aom_img_free(img0);
+  ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK);
+}
 #endif  // CONFIG_AV1_HIGHBITDEPTH

 #if CONFIG_AV1_DECODER