Commit 0a70e28d36 for aom
commit 0a70e28d36698105936a7e23f5f1108f2c66bc83
Author: Jingning Han <jingning@google.com>
Date: Thu Jul 16 16:33:51 2026 -0700
Add AVX2 functions to variance_stats
Tested on 1080 videos, this reduces the total encoding time
by 30% in the svod settings.
Change-Id: I2b0f1b34ba8a47fef916ba6d96f8acda0e5442d9
diff --git a/aom_dsp/aom_dsp_rtcd_defs.pl b/aom_dsp/aom_dsp_rtcd_defs.pl
index e5fb5dcb66..346282f02c 100755
--- a/aom_dsp/aom_dsp_rtcd_defs.pl
+++ b/aom_dsp/aom_dsp_rtcd_defs.pl
@@ -1385,6 +1385,9 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
add_proto qw/uint64_t/, "aom_mse_wxh_16bit", "uint8_t *dst, int dstride,uint16_t *src, int sstride, int w, int h";
specialize qw/aom_mse_wxh_16bit sse2 avx2 neon/;
+ add_proto qw/int64_t/, "aom_calc_variance_stat", "const uint8_t *src, int stride, int bw, int bh";
+ specialize qw/aom_calc_variance_stat avx2/;
+
add_proto qw/uint64_t/, "aom_mse_16xh_16bit", "uint8_t *dst, int dstride,uint16_t *src, int w, int h";
specialize qw/aom_mse_16xh_16bit sse2 avx2 neon/;
diff --git a/aom_dsp/variance.c b/aom_dsp/variance.c
index 2d632a7125..639e83200c 100644
--- a/aom_dsp/variance.c
+++ b/aom_dsp/variance.c
@@ -1098,6 +1098,53 @@ uint64_t aom_mse_16xh_16bit_c(uint8_t *dst, int dstride, uint16_t *src, int w,
return sum;
}
+int64_t aom_calc_variance_stat_c(const uint8_t *src, int stride, int bw,
+ int bh) {
+ DECLARE_ALIGNED(16, uint8_t, dclevel[(MAX_SB_SIZE + 2) * (MAX_SB_SIZE + 2)]);
+ int pstride = bw + 2;
+ uint8_t *pred_ptr = &dclevel[pstride + 1];
+
+ static const int gau_filter[3][3] = {
+ { 1, 2, 1 },
+ { 2, 4, 2 },
+ { 1, 2, 1 },
+ };
+
+ for (int idy = -1; idy < bh + 1; ++idy) {
+ for (int idx = -1; idx < bw + 1; ++idx) {
+ int offset_idy = idy;
+ int offset_idx = idx;
+ if (idy == -1) offset_idy = 0;
+ if (idy == bh) offset_idy = bh - 1;
+ if (idx == -1) offset_idx = 0;
+ if (idx == bw) offset_idx = bw - 1;
+
+ int offset = offset_idy * stride + offset_idx;
+ pred_ptr[idy * pstride + idx] = src[offset];
+ }
+ }
+
+ int64_t var_stats = 0;
+
+ for (int idy = 0; idy < bh; ++idy) {
+ for (int idx = 0; idx < bw; ++idx) {
+ int sum = 0;
+ for (int iy = 0; iy < 3; ++iy)
+ for (int ix = 0; ix < 3; ++ix)
+ sum += pred_ptr[(idy + iy - 1) * pstride + (idx + ix - 1)] *
+ gau_filter[iy][ix];
+
+ sum = sum >> 4;
+
+ int64_t diff = pred_ptr[idy * pstride + idx] - sum;
+ var_stats += diff * diff;
+ }
+ }
+ var_stats <<= 4;
+
+ return var_stats;
+}
+
#if CONFIG_AV1_HIGHBITDEPTH
uint64_t aom_mse_wxh_16bit_highbd_c(uint16_t *dst, int dstride, uint16_t *src,
int sstride, int w, int h) {
diff --git a/aom_dsp/x86/variance_avx2.c b/aom_dsp/x86/variance_avx2.c
index 59d170c7bd..d431c850e5 100644
--- a/aom_dsp/x86/variance_avx2.c
+++ b/aom_dsp/x86/variance_avx2.c
@@ -946,6 +946,191 @@ static inline void get_var_sse_sum_16x16_dual_avx2(
_mm_storel_epi64((__m128i *)var16x16, variance_16x16);
}
+static inline int32_t yy_hsum_epi32_si32(__m256i v) {
+ __m128i v128 =
+ _mm_add_epi32(_mm256_castsi256_si128(v), _mm256_extracti128_si256(v, 1));
+ v128 = _mm_hadd_epi32(v128, v128);
+ v128 = _mm_hadd_epi32(v128, v128);
+ return _mm_cvtsi128_si32(v128);
+}
+
+static inline int32_t xx_hsum_epi32_si32(__m128i v) {
+ v = _mm_hadd_epi32(v, v);
+ v = _mm_hadd_epi32(v, v);
+ return _mm_cvtsi128_si32(v);
+}
+
+int64_t aom_calc_variance_stat_avx2(const uint8_t *src, int stride, int bw,
+ int bh) {
+ // Temporary buffer to store horizontal filter results H[y][x]
+ // Max block size in AV1 is 128x128
+ DECLARE_ALIGNED(32, uint16_t, H_buf[128 * 128]);
+
+ // Step 1: Compute Horizontal 1D Filter H[y][x] = P(y, x-1) + 2*P(y, x) + P(y,
+ // x + 1)
+ for (int y = 0; y < bh; ++y) {
+ const uint8_t *src_row = src + y * stride;
+ uint16_t *H_row = H_buf + y * bw;
+
+ if (bw >= 16) {
+ for (int x = 0; x < bw; x += 16) {
+ __m128i v_curr = _mm_loadu_si128((const __m128i *)(src_row + x));
+ __m128i v_left, v_right;
+
+ if (x == 0) {
+ v_left =
+ _mm_insert_epi8(_mm_slli_si128(v_curr, 1), (int8_t)src_row[0], 0);
+ } else {
+ v_left = _mm_loadu_si128((const __m128i *)(src_row + x - 1));
+ }
+
+ if (x + 16 < bw) {
+ v_right = _mm_loadu_si128((const __m128i *)(src_row + x + 1));
+ } else {
+ v_right = _mm_insert_epi8(_mm_srli_si128(v_curr, 1),
+ (int8_t)src_row[bw - 1], 15);
+ }
+
+ __m256i u16_left = _mm256_cvtepu8_epi16(v_left);
+ __m256i u16_curr = _mm256_cvtepu8_epi16(v_curr);
+ __m256i u16_right = _mm256_cvtepu8_epi16(v_right);
+
+ __m256i u16_H = _mm256_add_epi16(_mm256_add_epi16(u16_left, u16_right),
+ _mm256_slli_epi16(u16_curr, 1));
+
+ _mm256_storeu_si256((__m256i *)(H_row + x), u16_H);
+ }
+ } else if (bw == 8) {
+ __m128i v_curr = _mm_loadl_epi64((const __m128i *)src_row);
+ __m128i v_left =
+ _mm_insert_epi8(_mm_slli_si128(v_curr, 1), (int8_t)src_row[0], 0);
+ __m128i v_right =
+ _mm_insert_epi8(_mm_srli_si128(v_curr, 1), (int8_t)src_row[7], 7);
+
+ __m128i u16_left = _mm_cvtepu8_epi16(v_left);
+ __m128i u16_curr = _mm_cvtepu8_epi16(v_curr);
+ __m128i u16_right = _mm_cvtepu8_epi16(v_right);
+
+ __m128i u16_H = _mm_add_epi16(_mm_add_epi16(u16_left, u16_right),
+ _mm_slli_epi16(u16_curr, 1));
+
+ _mm_storeu_si128((__m128i *)H_row, u16_H);
+ } else { // bw == 4
+ __m128i v_curr = _mm_cvtsi32_si128(*(const int32_t *)src_row);
+ __m128i v_left =
+ _mm_insert_epi8(_mm_slli_si128(v_curr, 1), (int8_t)src_row[0], 0);
+ __m128i v_right =
+ _mm_insert_epi8(_mm_srli_si128(v_curr, 1), (int8_t)src_row[3], 3);
+
+ __m128i u16_left = _mm_cvtepu8_epi16(v_left);
+ __m128i u16_curr = _mm_cvtepu8_epi16(v_curr);
+ __m128i u16_right = _mm_cvtepu8_epi16(v_right);
+
+ __m128i u16_H = _mm_add_epi16(_mm_add_epi16(u16_left, u16_right),
+ _mm_slli_epi16(u16_curr, 1));
+
+ _mm_storel_epi64((__m128i *)H_row, u16_H);
+ }
+ }
+
+ // Step 2: Compute Vertical Filter V[y][x] = H(y-1, x) + 2*H(y, x) + H(y + 1,
+ // x), smooth = V >> 4, diff = P - smooth, and accum (diff^2)
+ int64_t total_var = 0;
+
+ if (bw >= 16) {
+ __m256i acc_var_256 = _mm256_setzero_si256();
+
+ for (int y = 0; y < bh; ++y) {
+ const uint8_t *src_row = src + y * stride;
+ const uint16_t *H_curr_row = H_buf + y * bw;
+ const uint16_t *H_top_row = (y == 0) ? H_curr_row : H_buf + (y - 1) * bw;
+ const uint16_t *H_bot_row =
+ (y == bh - 1) ? H_curr_row : H_buf + (y + 1) * bw;
+
+ for (int x = 0; x < bw; x += 16) {
+ __m256i H_top = _mm256_loadu_si256((const __m256i *)(H_top_row + x));
+ __m256i H_curr = _mm256_loadu_si256((const __m256i *)(H_curr_row + x));
+ __m256i H_bot = _mm256_loadu_si256((const __m256i *)(H_bot_row + x));
+
+ __m256i u16_V = _mm256_add_epi16(_mm256_add_epi16(H_top, H_bot),
+ _mm256_slli_epi16(H_curr, 1));
+
+ __m256i u16_sum = _mm256_srli_epi16(u16_V, 4);
+
+ __m128i v_p_curr = _mm_loadu_si128((const __m128i *)(src_row + x));
+ __m256i u16_p_curr = _mm256_cvtepu8_epi16(v_p_curr);
+
+ __m256i diff = _mm256_sub_epi16(u16_p_curr, u16_sum);
+ __m256i diff_sq = _mm256_madd_epi16(diff, diff);
+
+ acc_var_256 = _mm256_add_epi32(acc_var_256, diff_sq);
+ }
+ }
+
+ total_var = (int64_t)yy_hsum_epi32_si32(acc_var_256);
+ } else if (bw == 8) {
+ __m128i acc_var_128 = _mm_setzero_si128();
+
+ for (int y = 0; y < bh; ++y) {
+ const uint8_t *src_row = src + y * stride;
+ const uint16_t *H_curr_row = H_buf + y * 8;
+ const uint16_t *H_top_row = (y == 0) ? H_curr_row : H_buf + (y - 1) * 8;
+ const uint16_t *H_bot_row =
+ (y == bh - 1) ? H_curr_row : H_buf + (y + 1) * 8;
+
+ __m128i H_top = _mm_loadu_si128((const __m128i *)H_top_row);
+ __m128i H_curr = _mm_loadu_si128((const __m128i *)H_curr_row);
+ __m128i H_bot = _mm_loadu_si128((const __m128i *)H_bot_row);
+
+ __m128i u16_V =
+ _mm_add_epi16(_mm_add_epi16(H_top, H_bot), _mm_slli_epi16(H_curr, 1));
+
+ __m128i u16_sum = _mm_srli_epi16(u16_V, 4);
+
+ __m128i v_p_curr = _mm_loadl_epi64((const __m128i *)src_row);
+ __m128i u16_p_curr = _mm_cvtepu8_epi16(v_p_curr);
+
+ __m128i diff = _mm_sub_epi16(u16_p_curr, u16_sum);
+ __m128i diff_sq = _mm_madd_epi16(diff, diff);
+
+ acc_var_128 = _mm_add_epi32(acc_var_128, diff_sq);
+ }
+
+ total_var = (int64_t)xx_hsum_epi32_si32(acc_var_128);
+ } else { // bw == 4
+ __m128i acc_var_128 = _mm_setzero_si128();
+
+ for (int y = 0; y < bh; ++y) {
+ const uint8_t *src_row = src + y * stride;
+ const uint16_t *H_curr_row = H_buf + y * 4;
+ const uint16_t *H_top_row = (y == 0) ? H_curr_row : H_buf + (y - 1) * 4;
+ const uint16_t *H_bot_row =
+ (y == bh - 1) ? H_curr_row : H_buf + (y + 1) * 4;
+
+ __m128i H_top = _mm_loadl_epi64((const __m128i *)H_top_row);
+ __m128i H_curr = _mm_loadl_epi64((const __m128i *)H_curr_row);
+ __m128i H_bot = _mm_loadl_epi64((const __m128i *)H_bot_row);
+
+ __m128i u16_V =
+ _mm_add_epi16(_mm_add_epi16(H_top, H_bot), _mm_slli_epi16(H_curr, 1));
+
+ __m128i u16_sum = _mm_srli_epi16(u16_V, 4);
+
+ __m128i v_p_curr = _mm_cvtsi32_si128(*(const int32_t *)src_row);
+ __m128i u16_p_curr = _mm_cvtepu8_epi16(v_p_curr);
+
+ __m128i diff = _mm_sub_epi16(u16_p_curr, u16_sum);
+ __m128i diff_sq = _mm_madd_epi16(diff, diff);
+
+ acc_var_128 = _mm_add_epi32(acc_var_128, diff_sq);
+ }
+
+ total_var = (int64_t)xx_hsum_epi32_si32(acc_var_128);
+ }
+
+ return total_var << 4;
+}
+
void aom_get_var_sse_sum_8x8_quad_avx2(const uint8_t *src_ptr,
int source_stride,
const uint8_t *ref_ptr, int ref_stride,
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index e805c2c3a4..3b75c340cf 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -718,79 +718,8 @@ static void get_variance_stats(const MACROBLOCK *x, int64_t *src_var,
int bw = block_size_wide[bsize];
int bh = block_size_high[bsize];
- static const int gau_filter[3][3] = {
- { 1, 2, 1 },
- { 2, 4, 2 },
- { 1, 2, 1 },
- };
-
- DECLARE_ALIGNED(16, uint8_t, dclevel[(MAX_SB_SIZE + 2) * (MAX_SB_SIZE + 2)]);
-
- uint8_t *pred_ptr = &dclevel[bw + 1];
- int pred_stride = xd->plane[0].dst.stride;
-
- for (int idy = -1; idy < bh + 1; ++idy) {
- for (int idx = -1; idx < bw + 1; ++idx) {
- int offset_idy = idy;
- int offset_idx = idx;
- if (idy == -1) offset_idy = 0;
- if (idy == bh) offset_idy = bh - 1;
- if (idx == -1) offset_idx = 0;
- if (idx == bw) offset_idx = bw - 1;
-
- int offset = offset_idy * pred_stride + offset_idx;
- pred_ptr[idy * bw + idx] = pd->dst.buf[offset];
- }
- }
-
- *rec_var = 0;
- for (int idy = 0; idy < bh; ++idy) {
- for (int idx = 0; idx < bw; ++idx) {
- int sum = 0;
- for (int iy = 0; iy < 3; ++iy)
- for (int ix = 0; ix < 3; ++ix)
- sum += pred_ptr[(idy + iy - 1) * bw + (idx + ix - 1)] *
- gau_filter[iy][ix];
-
- sum = sum >> 4;
-
- int64_t diff = pred_ptr[idy * bw + idx] - sum;
- *rec_var += diff * diff;
- }
- }
- *rec_var <<= 4;
-
- int src_stride = p->src.stride;
- for (int idy = -1; idy < bh + 1; ++idy) {
- for (int idx = -1; idx < bw + 1; ++idx) {
- int offset_idy = idy;
- int offset_idx = idx;
- if (idy == -1) offset_idy = 0;
- if (idy == bh) offset_idy = bh - 1;
- if (idx == -1) offset_idx = 0;
- if (idx == bw) offset_idx = bw - 1;
-
- int offset = offset_idy * src_stride + offset_idx;
- pred_ptr[idy * bw + idx] = p->src.buf[offset];
- }
- }
-
- *src_var = 0;
- for (int idy = 0; idy < bh; ++idy) {
- for (int idx = 0; idx < bw; ++idx) {
- int sum = 0;
- for (int iy = 0; iy < 3; ++iy)
- for (int ix = 0; ix < 3; ++ix)
- sum += pred_ptr[(idy + iy - 1) * bw + (idx + ix - 1)] *
- gau_filter[iy][ix];
-
- sum = sum >> 4;
-
- int64_t diff = pred_ptr[idy * bw + idx] - sum;
- *src_var += diff * diff;
- }
- }
- *src_var <<= 4;
+ *rec_var = aom_calc_variance_stat(pd->dst.buf, pd->dst.stride, bw, bh);
+ *src_var = aom_calc_variance_stat(p->src.buf, p->src.stride, bw, bh);
}
static void adjust_rdcost(const AV1_COMP *cpi, const MACROBLOCK *x,
diff --git a/test/variance_test.cc b/test/variance_test.cc
index ff5caeb35e..55b1b99eb5 100644
--- a/test/variance_test.cc
+++ b/test/variance_test.cc
@@ -3812,4 +3812,46 @@ INSTANTIATE_TEST_SUITE_P(SVE, AvxHBDVarianceTest,
#endif // CONFIG_AV1_HIGHBITDEPTH
#endif // HAVE_SVE
+#if HAVE_AVX2
+using CalcVarianceStatFunc = int64_t (*)(const uint8_t *src, int stride, int bw,
+ int bh);
+
+class CalcVarianceStatTest
+ : public ::testing::TestWithParam<CalcVarianceStatFunc> {
+ protected:
+ void SetUp() override {
+ target_func_ = GetParam();
+ rnd_.Reset(ACMRandom::DeterministicSeed());
+ }
+
+ CalcVarianceStatFunc target_func_;
+ ACMRandom rnd_;
+};
+
+TEST_P(CalcVarianceStatTest, CompareWithC) {
+ static const int kSizes[] = { 4, 8, 16, 32, 64, 128 };
+ DECLARE_ALIGNED(32, uint8_t, src[128 * 128]);
+
+ for (int w : kSizes) {
+ for (int h : kSizes) {
+ SCOPED_TRACE(::testing::Message() << "bw=" << w << " bh=" << h);
+ int stride = 128;
+ for (int iter = 0; iter < 500; ++iter) {
+ for (int r = 0; r < h; ++r) {
+ for (int c = 0; c < w; ++c) {
+ src[r * stride + c] = rnd_.Rand8();
+ }
+ }
+ int64_t res_c = aom_calc_variance_stat_c(src, stride, w, h);
+ int64_t res_target = target_func_(src, stride, w, h);
+ EXPECT_EQ(res_c, res_target) << "iter=" << iter;
+ }
+ }
+ }
+}
+
+INSTANTIATE_TEST_SUITE_P(AVX2, CalcVarianceStatTest,
+ ::testing::Values(&aom_calc_variance_stat_avx2));
+#endif // HAVE_AVX2
+
} // namespace