Commit 71494451cc for aom

commit 71494451cc11db63276410e0adf7dfc0378baea1
Author: Jerome Jiang <jianj@google.com>
Date:   Tue Feb 24 16:28:41 2026 -0500

    Fix integer overflow in av1_iadst4

    Unscaled intermediate values in 4-point ADST transforms
    (av1_iadst4 and av1_fadst4) could exceed INT32_MAX on
    extreme inputs, causing a signed integer overflow.

    This commit promotes the intermediate state variables to
    int64_t to safely handle the accumulation before the
    final downscale. A regression test is also included.

    Bug: 471847850
    Change-Id: I1178da96f6d0007c50bd3c7a3d95b974dfd8c2da

diff --git a/av1/common/av1_inv_txfm1d.c b/av1/common/av1_inv_txfm1d.c
index 37c2091490..49380e84f7 100644
--- a/av1/common/av1_inv_txfm1d.c
+++ b/av1/common/av1_inv_txfm1d.c
@@ -657,12 +657,12 @@ void av1_iadst4(const int32_t *input, int32_t *output, int8_t cos_bit,
                 const int8_t *stage_range) {
   int bit = cos_bit;
   const int32_t *sinpi = sinpi_arr(bit);
-  int32_t s0, s1, s2, s3, s4, s5, s6, s7;
+  int64_t s0, s1, s2, s3, s4, s5, s6, s7;

-  int32_t x0 = input[0];
-  int32_t x1 = input[1];
-  int32_t x2 = input[2];
-  int32_t x3 = input[3];
+  int64_t x0 = input[0];
+  int64_t x1 = input[1];
+  int64_t x2 = input[2];
+  int64_t x3 = input[3];

   if (!(x0 | x1 | x2 | x3)) {
     output[0] = output[1] = output[2] = output[3] = 0;
@@ -672,37 +672,37 @@ void av1_iadst4(const int32_t *input, int32_t *output, int8_t cos_bit,
   assert(sinpi[1] + sinpi[2] == sinpi[4]);

   // stage 1
-  s0 = range_check_value(sinpi[1] * x0, stage_range[1] + bit);
-  s1 = range_check_value(sinpi[2] * x0, stage_range[1] + bit);
-  s2 = range_check_value(sinpi[3] * x1, stage_range[1] + bit);
-  s3 = range_check_value(sinpi[4] * x2, stage_range[1] + bit);
-  s4 = range_check_value(sinpi[1] * x2, stage_range[1] + bit);
-  s5 = range_check_value(sinpi[2] * x3, stage_range[1] + bit);
-  s6 = range_check_value(sinpi[4] * x3, stage_range[1] + bit);
+  s0 = range_check_value64(sinpi[1] * x0, stage_range[1] + bit);
+  s1 = range_check_value64(sinpi[2] * x0, stage_range[1] + bit);
+  s2 = range_check_value64(sinpi[3] * x1, stage_range[1] + bit);
+  s3 = range_check_value64(sinpi[4] * x2, stage_range[1] + bit);
+  s4 = range_check_value64(sinpi[1] * x2, stage_range[1] + bit);
+  s5 = range_check_value64(sinpi[2] * x3, stage_range[1] + bit);
+  s6 = range_check_value64(sinpi[4] * x3, stage_range[1] + bit);

   // stage 2
   // NOTICE: (x0 - x2) here may use one extra bit compared to the
   // opt_range_row/col specified in av1_gen_inv_stage_range()
-  s7 = range_check_value((x0 - x2) + x3, stage_range[2]);
+  s7 = range_check_value64((x0 - x2) + x3, stage_range[2]);

   // stage 3
-  s0 = range_check_value(s0 + s3, stage_range[3] + bit);
-  s1 = range_check_value(s1 - s4, stage_range[3] + bit);
-  s3 = range_check_value(s2, stage_range[3] + bit);
-  s2 = range_check_value(sinpi[3] * s7, stage_range[3] + bit);
+  s0 = range_check_value64(s0 + s3, stage_range[3] + bit);
+  s1 = range_check_value64(s1 - s4, stage_range[3] + bit);
+  s3 = range_check_value64(s2, stage_range[3] + bit);
+  s2 = range_check_value64(sinpi[3] * s7, stage_range[3] + bit);

   // stage 4
-  s0 = range_check_value(s0 + s5, stage_range[4] + bit);
-  s1 = range_check_value(s1 - s6, stage_range[4] + bit);
+  s0 = range_check_value64(s0 + s5, stage_range[4] + bit);
+  s1 = range_check_value64(s1 - s6, stage_range[4] + bit);

   // stage 5
-  x0 = range_check_value(s0 + s3, stage_range[5] + bit);
-  x1 = range_check_value(s1 + s3, stage_range[5] + bit);
-  x2 = range_check_value(s2, stage_range[5] + bit);
-  x3 = range_check_value(s0 + s1, stage_range[5] + bit);
+  x0 = range_check_value64(s0 + s3, stage_range[5] + bit);
+  x1 = range_check_value64(s1 + s3, stage_range[5] + bit);
+  x2 = range_check_value64(s2, stage_range[5] + bit);
+  x3 = range_check_value64(s0 + s1, stage_range[5] + bit);

   // stage 6
-  x3 = range_check_value(x3 - s3, stage_range[6] + bit);
+  x3 = range_check_value64(x3 - s3, stage_range[6] + bit);

   output[0] = round_shift(x0, bit);
   output[1] = round_shift(x1, bit);
diff --git a/av1/common/av1_txfm.h b/av1/common/av1_txfm.h
index c3c3123497..8603c3dd0b 100644
--- a/av1/common/av1_txfm.h
+++ b/av1/common/av1_txfm.h
@@ -13,6 +13,7 @@
 #define AOM_AV1_COMMON_AV1_TXFM_H_

 #include <assert.h>
+#include <inttypes.h>
 #include <math.h>
 #include <stdio.h>

@@ -94,6 +95,26 @@ static inline int32_t range_check_value(int32_t value, int8_t bit) {
   return value;
 }

+static inline int64_t range_check_value64(int64_t value, int8_t bit) {
+#if CONFIG_COEFFICIENT_RANGE_CHECKING
+  const int64_t max_value = (1LL << (bit - 1)) - 1;
+  const int64_t min_value = -(1LL << (bit - 1));
+  if (value < min_value || value > max_value) {
+    fprintf(stderr, "coeff out of bit range, value: %" PRId64 " bit %d\n",
+            value, bit);
+#if !CONFIG_AV1_ENCODER
+    assert(0);
+#endif
+  }
+#endif  // CONFIG_COEFFICIENT_RANGE_CHECKING
+#if DO_RANGE_CHECK_CLAMP
+  bit = AOMMIN(bit, 63);
+  return clamp64(value, -(1LL << (bit - 1)), (1LL << (bit - 1)) - 1);
+#endif  // DO_RANGE_CHECK_CLAMP
+  (void)bit;
+  return value;
+}
+
 static inline int32_t round_shift(int64_t value, int bit) {
   assert(bit >= 1);
   return (int32_t)((value + (1ll << (bit - 1))) >> bit);
diff --git a/test/av1_inv_txfm1d_test.cc b/test/av1_inv_txfm1d_test.cc
index 4220ad8d67..30b9a14914 100644
--- a/test/av1_inv_txfm1d_test.cc
+++ b/test/av1_inv_txfm1d_test.cc
@@ -152,4 +152,15 @@ TEST(av1_inv_txfm1d, round_trip) {
   }
 }

+TEST(av1_inv_txfm1d, iadst4_overflow_bug471847850) {
+  // Test case to trigger integer overflow in av1_iadst4
+  int32_t input[4] = { 300000, 0, 300000, 300000 };
+  int32_t output[4];
+
+  av1_iadst4(input, output, 12, range_bit);
+
+  // Verify that the transform completes and the output is greater than zero.
+  EXPECT_GT(output[0], 0);
+}
+
 }  // namespace