Commit 037ca0e886 for aom

commit 037ca0e8863b171cc284390c7f1a845a6b2d7f4f
Author: Lin Zheng <linzhen@google.com>
Date:   Mon Sep 14 21:08:57 2026 +0000

    Fix integer overflow in av1_caq_select_segment

    In av1_caq_select_segment(), sb64_target_rate * xmis * ymis was
    evaluated using 32-bit signed integer arithmetic before being cast
    to int64_t. When sb64_target_rate is large, this causes signed
    integer overflow.

    Cast sb64_target_rate to int64_t before multiplying with xmis and ymis,
    and declare target_rate as int64_t to prevent overflow when num / denom
    exceeds INT_MAX.

    Bug: 559079132
    Change-Id: Ifde1ae6acef26f2320d23eab13a47222fb287619

diff --git a/av1/encoder/aq_complexity.c b/av1/encoder/aq_complexity.c
index 090463bf81..a49752308e 100644
--- a/av1/encoder/aq_complexity.c
+++ b/av1/encoder/aq_complexity.c
@@ -141,10 +141,10 @@ void av1_caq_select_segment(const AV1_COMP *cpi, MACROBLOCK *mb, BLOCK_SIZE bs,

   // Rate depends on fraction of a SB64 in frame (xmis * ymis / bw * bh).
   // It is converted to bits << AV1_PROB_COST_SHIFT units.
-  const int64_t num = (int64_t)(cpi->rc.sb64_target_rate * xmis * ymis)
+  const int64_t num = ((int64_t)cpi->rc.sb64_target_rate * xmis * ymis)
                       << AV1_PROB_COST_SHIFT;
   const int denom = cm->seq_params->mib_size * cm->seq_params->mib_size;
-  const int target_rate = (int)(num / denom);
+  const int64_t target_rate = num / denom;
   double logvar;
   double low_var_thresh;
   const int aq_strength = get_aq_c_strength(cm->quant_params.base_qindex,
diff --git a/test/encode_api_test.cc b/test/encode_api_test.cc
index baf8b10c68..4cd53e0f98 100644
--- a/test/encode_api_test.cc
+++ b/test/encode_api_test.cc
@@ -2795,6 +2795,46 @@ TEST(EncodeAPI, PerceptualAIDynamicResolutionChange) {
   aom_img_free(img_large);
   ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK);
 }
+
+// Test for OSS-Fuzz Issue 559079132: Integer-overflow in
+// av1_caq_select_segment. When COMPLEXITY_AQ (aq_mode 2) is enabled with a high
+// target bitrate, sb64_target_rate is large, causing signed integer overflow
+// when multiplying sb64_target_rate * xmis * ymis in 32-bit arithmetic and when
+// casting target_rate to int.
+TEST(EncodeAPI, Issue559079132) {
+  aom_codec_iface_t *const iface = aom_codec_av1_cx();
+  aom_codec_ctx_t enc;
+  aom_codec_enc_cfg_t cfg;
+
+  ASSERT_EQ(aom_codec_enc_config_default(iface, &cfg, AOM_USAGE_GOOD_QUALITY),
+            AOM_CODEC_OK);
+
+  cfg.g_w = 64;
+  cfg.g_h = 64;
+  cfg.g_timebase.num = 1;
+  cfg.g_timebase.den = 1;
+  cfg.rc_target_bitrate = 2000000;
+  cfg.rc_end_usage = AOM_CBR;
+  cfg.g_lag_in_frames = 0;
+
+  ASSERT_EQ(aom_codec_enc_init(&enc, iface, &cfg, 0), AOM_CODEC_OK);
+  ASSERT_EQ(aom_codec_control(&enc, AV1E_SET_AQ_MODE, 2), AOM_CODEC_OK);
+
+  aom_image_t *img = aom_img_alloc(nullptr, AOM_IMG_FMT_I420, 64, 64, 1);
+  ASSERT_NE(img, nullptr);
+  FillImageRandom(img);
+
+  EncodeOne(&enc, img, 0);
+
+  // Flush encoder.
+  ASSERT_EQ(aom_codec_encode(&enc, nullptr, 0, 0, 0), AOM_CODEC_OK);
+  aom_codec_iter_t iter = nullptr;
+  while (aom_codec_get_cx_data(&enc, &iter) != nullptr) {
+  }
+
+  aom_img_free(img);
+  ASSERT_EQ(aom_codec_destroy(&enc), AOM_CODEC_OK);
+}
 #endif  // !CONFIG_REALTIME_ONLY

 // Tests for OSS-Fuzz Issues 558463888, 559075253, 559225640: