Commit 397cbaba75 for aom

commit 397cbaba7561cd9ec6b71522a58d18fa9739ce61
Author: Yuan Tong <tongyuan200097@gmail.com>
Date:   Tue Oct 20 21:30:52 2026 +0800

    Fix integer overflow in AVX2 highbd subpel variance

    When the sum and the sse are rounded, the variance of a nearly constant
    residue can be negative. aom_highbd_var_filter_block2d_bil_avx2()
    computed it in unsigned arithmetic so it can overflow.

    The reference functions in variance_test.cc had the same overflow and
    did not clamp the result. Clamp it to 0 as well and add a test with a
    nearly constant residue.

    Tested: test_libaom --gtest_filter=*AvxHBDSubpelVarianceTest.NearlyConstantDiff*
    Note: Build with -fsanitize=integer using Clang to detect these overflow.
    Change-Id: I7db4a96a0a2ef2118c229fbe79e6c8e06798f5e5

diff --git a/aom_dsp/x86/highbd_variance_avx2.c b/aom_dsp/x86/highbd_variance_avx2.c
index 0606916910..78c4e53851 100644
--- a/aom_dsp/x86/highbd_variance_avx2.c
+++ b/aom_dsp/x86/highbd_variance_avx2.c
@@ -613,9 +613,9 @@ static uint32_t aom_highbd_var_filter_block2d_bil_avx2(
   *sse = (uint32_t)ROUND_POWER_OF_TWO(sse_long, 4);
   int sum = (int)ROUND_POWER_OF_TWO(sum_long, 2);

-  int32_t var = *sse - (uint32_t)(((int64_t)sum * sum) >> rshift);
+  const int64_t var = (int64_t)*sse - (((int64_t)sum * sum) >> rshift);

-  return (var > 0) ? var : 0;
+  return (var > 0) ? (uint32_t)var : 0;
 }

 static void highbd_calc8x8var_avx2(const uint16_t *src, int src_stride,
diff --git a/test/variance_test.cc b/test/variance_test.cc
index 13a3868fe1..781f57446e 100644
--- a/test/variance_test.cc
+++ b/test/variance_test.cc
@@ -86,6 +86,14 @@ static void RoundHighBitDepth(int bit_depth, int64_t *se, uint64_t *sse) {
   }
 }

+// Returns the variance of 2^log2_count errors, given the sum of their squares
+// and their sum. The rounding of high bit depth sums can make the variance
+// negative, in which case 0 is returned.
+static uint32_t VarianceFromSums(uint64_t sse, int64_t se, int log2_count) {
+  const int64_t var = static_cast<int64_t>(sse) - ((se * se) >> log2_count);
+  return var > 0 ? static_cast<uint32_t>(var) : 0;
+}
+
 /* Note:
  *  Our codebase calculates the "diff" value in the variance algorithm by
  *  (src - ref).
@@ -115,7 +123,7 @@ static uint32_t variance_ref(const uint8_t *src, const uint8_t *ref, int l2w,
   }
   RoundHighBitDepth(bit_depth, &se, &sse);
   *sse_ptr = static_cast<uint32_t>(sse);
-  return static_cast<uint32_t>(sse - ((se * se) >> (l2w + l2h)));
+  return VarianceFromSums(sse, se, l2w + l2h);
 }

 /* The subpel reference functions differ from the codec version in one aspect:
@@ -167,7 +175,7 @@ static uint32_t subpel_variance_ref(const uint8_t *ref, const uint8_t *src,
   }
   RoundHighBitDepth(bit_depth, &se, &sse);
   *sse_ptr = static_cast<uint32_t>(sse);
-  return static_cast<uint32_t>(sse - ((se * se) >> (l2w + l2h)));
+  return VarianceFromSums(sse, se, l2w + l2h);
 }

 static uint32_t subpel_avg_variance_ref(const uint8_t *ref, const uint8_t *src,
@@ -218,7 +226,7 @@ static uint32_t subpel_avg_variance_ref(const uint8_t *ref, const uint8_t *src,
   }
   RoundHighBitDepth(bit_depth, &se, &sse);
   *sse_ptr = static_cast<uint32_t>(sse);
-  return static_cast<uint32_t>(sse - ((se * se) >> (l2w + l2h)));
+  return VarianceFromSums(sse, se, l2w + l2h);
 }

 #if !CONFIG_REALTIME_ONLY
@@ -269,7 +277,7 @@ static uint32_t obmc_subpel_variance_ref(const uint8_t *pre, int l2w, int l2h,
   }
   RoundHighBitDepth(bit_depth, &se, &sse);
   *sse_ptr = static_cast<uint32_t>(sse);
-  return static_cast<uint32_t>(sse - ((se * se) >> (l2w + l2h)));
+  return VarianceFromSums(sse, se, l2w + l2h);
 }
 #endif

@@ -1264,6 +1272,7 @@ class SubpelVarianceTest
  protected:
   void RefTest();
   void ExtremeRefTest();
+  void NearlyConstantDiffTest();
   void SpeedTest();

   ACMRandom rnd_;
@@ -1347,6 +1356,38 @@ void SubpelVarianceTest<SubpelVarianceFunctionType>::ExtremeRefTest() {
   }
 }

+template <typename SubpelVarianceFunctionType>
+void SubpelVarianceTest<SubpelVarianceFunctionType>::NearlyConstantDiffTest() {
+  // Compare against reference.
+  // Ref: Set all values to half the maximum, so that filtering keeps them.
+  // Src: Set all values to 0, except the first one, which is set to 1.
+  // The variance is close to 0, so rounding the sum and the sse of the high
+  // bitdepth differences can make it negative.
+  for (int x = 0; x < 8; ++x) {
+    for (int y = 0; y < 8; ++y) {
+      const int ref_size = block_size() + width() + height() + 1;
+      if (!use_high_bit_depth()) {
+        memset(src_, 0, block_size());
+        src_[0] = 1;
+        memset(ref_, 128, ref_size);
+      } else {
+        aom_memset16(CONVERT_TO_SHORTPTR(src_), 0, block_size());
+        CONVERT_TO_SHORTPTR(src_)[0] = 1;
+        aom_memset16(CONVERT_TO_SHORTPTR(ref_), mask() >> 1, ref_size);
+      }
+      unsigned int sse1, sse2;
+      unsigned int var1;
+      API_REGISTER_STATE_CHECK(
+          var1 = params_.func(ref_, width() + 1, x, y, src_, width(), &sse1));
+      const unsigned int var2 = subpel_variance_ref(
+          ref_, src_, params_.log2width, params_.log2height, x, y, &sse2,
+          use_high_bit_depth(), params_.bit_depth);
+      EXPECT_EQ(sse1, sse2) << "for xoffset " << x << " and yoffset " << y;
+      EXPECT_EQ(var1, var2) << "for xoffset " << x << " and yoffset " << y;
+    }
+  }
+}
+
 template <typename SubpelVarianceFunctionType>
 void SubpelVarianceTest<SubpelVarianceFunctionType>::SpeedTest() {
   if (!use_high_bit_depth()) {
@@ -1933,6 +1974,9 @@ TEST_P(AvxHBDVarianceTest, OneQuarter) { OneQuarterTest(); }
 TEST_P(AvxHBDVarianceTest, DISABLED_Speed) { SpeedTest(); }
 TEST_P(AvxHBDSubpelVarianceTest, Ref) { RefTest(); }
 TEST_P(AvxHBDSubpelVarianceTest, ExtremeRef) { ExtremeRefTest(); }
+TEST_P(AvxHBDSubpelVarianceTest, NearlyConstantDiff) {
+  NearlyConstantDiffTest();
+}
 TEST_P(AvxHBDSubpelVarianceTest, DISABLED_Speed) { SpeedTest(); }
 TEST_P(AvxHBDSubpelAvgVarianceTest, Ref) { RefTest(); }
 #if !CONFIG_REALTIME_ONLY