Commit 32406f5615 for aom
commit 32406f561508d84379b1fe70764d7ff7a4a21d03
Author: Frank Galligan <fgalligan@google.com>
Date: Fri Oct 30 12:06:49 2026 -0700
av1: Add AVX2 vectorization for av1_block_error_qm
Vectorize transform-domain weighted block error calculation
(av1_block_error_qm) using AVX2 SIMD (av1_block_error_qm_avx2).
Optimization details:
- Processes 8 coefficients per iteration in 64-bit precision lanes.
- Adds a zero-dqcoeff fast path via _mm256_testz_si256 that skips
redundant error squaring when all 8 dequantized coefficients are
zero, and skips the chunk entirely when coeff is also zero.
- Preserves 100% bit-exact output parity.
Benchmarking results (incremental vs prev CL / cumulative vs baseline):
- Stock avifenc still-image (Default 1T): -3.20% / -14.77% encode time
- Stock avifenc still-image (Default 8T): -1.93% / -11.95% encode time
- 48-image multi-rate corpus (Full Suite 1T): -2.07% / -14.86% time
- Bit-exact output (1T and 8T)
TAG=agy
CONV=b929a630-a6cd-45a0-979c-1767c30178ee
Change-Id: I009490d33dd4f16b381e6a409a138d09cc8c5301
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl
index 3981c57579..b71c347764 100644
--- a/av1/common/av1_rtcd_defs.pl
+++ b/av1/common/av1_rtcd_defs.pl
@@ -297,6 +297,9 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
add_proto qw/int64_t av1_block_error/, "const tran_low_t *coeff, const tran_low_t *dqcoeff, intptr_t block_size, int64_t *ssz";
specialize qw/av1_block_error avx2 avx512 neon sve/, "$sse2_asm";
+ add_proto qw/int64_t av1_block_error_qm/, "const tran_low_t *coeff, const tran_low_t *dqcoeff, intptr_t block_size, const qm_val_t *qmatrix, const int16_t *scan, int64_t *ssz, int bd";
+ specialize qw/av1_block_error_qm avx2/;
+
add_proto qw/int64_t av1_block_error_lp/, "const int16_t *coeff, const int16_t *dqcoeff, intptr_t block_size";
specialize qw/av1_block_error_lp sse2 avx2 avx512 neon sve/;
diff --git a/av1/encoder/tx_search.c b/av1/encoder/tx_search.c
index 3f31343f4c..05aef376cd 100644
--- a/av1/encoder/tx_search.c
+++ b/av1/encoder/tx_search.c
@@ -1091,9 +1091,9 @@ static inline void sort_rd(int64_t rds[], int txk[], int len) {
}
}
-static inline int64_t av1_block_error_qm(
- const tran_low_t *coeff, const tran_low_t *dqcoeff, intptr_t block_size,
- const qm_val_t *qmatrix, const int16_t *scan, int64_t *ssz, int bd) {
+int64_t av1_block_error_qm_c(const tran_low_t *coeff, const tran_low_t *dqcoeff,
+ intptr_t block_size, const qm_val_t *qmatrix,
+ const int16_t *scan, int64_t *ssz, int bd) {
int i;
int64_t error = 0, sqcoeff = 0;
int shift = 2 * (bd - 8);
diff --git a/av1/encoder/x86/error_intrin_avx2.c b/av1/encoder/x86/error_intrin_avx2.c
index a389fb87f2..6b514718d0 100644
--- a/av1/encoder/x86/error_intrin_avx2.c
+++ b/av1/encoder/x86/error_intrin_avx2.c
@@ -14,6 +14,7 @@
#include "config/av1_rtcd.h"
#include "aom/aom_integer.h"
+#include "aom_dsp/aom_dsp_common.h"
static inline void read_coeff(const tran_low_t *coeff, intptr_t offset,
__m256i *c) {
@@ -208,3 +209,97 @@ int64_t av1_block_error_avx2(const tran_low_t *coeff, const tran_low_t *dqcoeff,
_mm256_zeroupper();
return sse;
}
+
+int64_t av1_block_error_qm_avx2(const tran_low_t *coeff,
+ const tran_low_t *dqcoeff, intptr_t block_size,
+ const qm_val_t *qmatrix, const int16_t *scan,
+ int64_t *ssz, int bd) {
+ assert(block_size % 8 == 0);
+ assert(bd == 8 || bd == 10 || bd == 12);
+ __m256i sse_reg = _mm256_setzero_si256();
+ __m256i ssz_reg = _mm256_setzero_si256();
+ const __m256i round_qm = _mm256_set1_epi64x(1 << (2 * AOM_QM_BITS - 1));
+
+ for (intptr_t i = 0; i < block_size; i += 8) {
+ // Load 8 transform coefficients and dequantized coefficients.
+ const __m256i coeff_reg = _mm256_loadu_si256((const __m256i *)(coeff + i));
+ const __m256i dqcoeff_reg =
+ _mm256_loadu_si256((const __m256i *)(dqcoeff + i));
+
+ // When all 8 dequantized coefficients are zero, sse and ssz accumulate the
+ // same weighted squared coefficient energy.
+ if (_mm256_testz_si256(dqcoeff_reg, dqcoeff_reg)) {
+ if (_mm256_testz_si256(coeff_reg, coeff_reg)) continue;
+
+ const __m256i wt_reg = _mm256_setr_epi32(
+ qmatrix[scan[i + 0]], qmatrix[scan[i + 1]], qmatrix[scan[i + 2]],
+ qmatrix[scan[i + 3]], qmatrix[scan[i + 4]], qmatrix[scan[i + 5]],
+ qmatrix[scan[i + 6]], qmatrix[scan[i + 7]]);
+ const __m256i cc = _mm256_mullo_epi32(coeff_reg, wt_reg);
+ const __m256i cc_hi = _mm256_srli_epi64(cc, 32);
+ const __m256i cc_lo_sq = _mm256_mul_epi32(cc, cc);
+ const __m256i cc_hi_sq = _mm256_mul_epi32(cc_hi, cc_hi);
+
+ const __m256i sq_lo = _mm256_srli_epi64(
+ _mm256_add_epi64(cc_lo_sq, round_qm), 2 * AOM_QM_BITS);
+ const __m256i sq_hi = _mm256_srli_epi64(
+ _mm256_add_epi64(cc_hi_sq, round_qm), 2 * AOM_QM_BITS);
+ const __m256i sq = _mm256_add_epi64(sq_lo, sq_hi);
+
+ sse_reg = _mm256_add_epi64(sse_reg, sq);
+ ssz_reg = _mm256_add_epi64(ssz_reg, sq);
+ } else {
+ const __m256i wt_reg = _mm256_setr_epi32(
+ qmatrix[scan[i + 0]], qmatrix[scan[i + 1]], qmatrix[scan[i + 2]],
+ qmatrix[scan[i + 3]], qmatrix[scan[i + 4]], qmatrix[scan[i + 5]],
+ qmatrix[scan[i + 6]], qmatrix[scan[i + 7]]);
+ const __m256i diff = _mm256_sub_epi32(coeff_reg, dqcoeff_reg);
+ const __m256i dd = _mm256_mullo_epi32(diff, wt_reg);
+ const __m256i cc = _mm256_mullo_epi32(coeff_reg, wt_reg);
+
+ // Compute weighted squared error (coeff - dqcoeff)^2 in 64-bit lanes.
+ const __m256i dd_hi = _mm256_srli_epi64(dd, 32);
+ const __m256i dd_lo_sq = _mm256_mul_epi32(dd, dd);
+ const __m256i dd_hi_sq = _mm256_mul_epi32(dd_hi, dd_hi);
+ const __m256i err_lo = _mm256_srli_epi64(
+ _mm256_add_epi64(dd_lo_sq, round_qm), 2 * AOM_QM_BITS);
+ const __m256i err_hi = _mm256_srli_epi64(
+ _mm256_add_epi64(dd_hi_sq, round_qm), 2 * AOM_QM_BITS);
+ sse_reg = _mm256_add_epi64(sse_reg, _mm256_add_epi64(err_lo, err_hi));
+
+ // Compute weighted squared coefficient energy coeff^2 in 64-bit lanes.
+ const __m256i cc_hi = _mm256_srli_epi64(cc, 32);
+ const __m256i cc_lo_sq = _mm256_mul_epi32(cc, cc);
+ const __m256i cc_hi_sq = _mm256_mul_epi32(cc_hi, cc_hi);
+ const __m256i sq_lo = _mm256_srli_epi64(
+ _mm256_add_epi64(cc_lo_sq, round_qm), 2 * AOM_QM_BITS);
+ const __m256i sq_hi = _mm256_srli_epi64(
+ _mm256_add_epi64(cc_hi_sq, round_qm), 2 * AOM_QM_BITS);
+ ssz_reg = _mm256_add_epi64(ssz_reg, _mm256_add_epi64(sq_lo, sq_hi));
+ }
+ }
+
+ // Save the higher 64 bit of each 128 bit lane and add to the low 64 bit.
+ const __m256i sse_reg_64hi = _mm256_srli_si256(sse_reg, 8);
+ const __m256i ssz_reg_64hi = _mm256_srli_si256(ssz_reg, 8);
+ sse_reg = _mm256_add_epi64(sse_reg, sse_reg_64hi);
+ ssz_reg = _mm256_add_epi64(ssz_reg, ssz_reg_64hi);
+
+ // Add each 64 bit from each of the 128 bit lanes of the 256 bit register.
+ const __m128i sse_reg128 = _mm_add_epi64(
+ _mm256_castsi256_si128(sse_reg), _mm256_extracti128_si256(sse_reg, 1));
+ const __m128i ssz_reg128 = _mm_add_epi64(
+ _mm256_castsi256_si128(ssz_reg), _mm256_extracti128_si256(ssz_reg, 1));
+
+ // Scale down for high bit-depth and store the results.
+ int64_t sse, sqcoeff;
+ _mm_storel_epi64((__m128i *)&sse, sse_reg128);
+ _mm_storel_epi64((__m128i *)&sqcoeff, ssz_reg128);
+ const int shift = 2 * (bd - 8);
+ const int rounding = (1 << shift) >> 1;
+ sse = (sse + rounding) >> shift;
+ sqcoeff = (sqcoeff + rounding) >> shift;
+
+ *ssz = sqcoeff;
+ return sse;
+}
diff --git a/test/error_block_test.cc b/test/error_block_test.cc
index 794558ac83..62856603a7 100644
--- a/test/error_block_test.cc
+++ b/test/error_block_test.cc
@@ -43,6 +43,12 @@ using ErrorBlockLpFunc = int64_t (*)(const int16_t *coeff,
const int16_t *dqcoeff,
intptr_t block_size);
+using ErrorBlockQmFunc = int64_t (*)(const tran_low_t *coeff,
+ const tran_low_t *dqcoeff,
+ intptr_t block_size,
+ const qm_val_t *qmatrix,
+ const int16_t *scan, int64_t *ssz, int bd);
+
using ErrorBlockParam =
std::tuple<ErrorBlockFunc, ErrorBlockFunc, aom_bit_depth_t>;
@@ -63,6 +69,24 @@ int64_t BlockErrorLpWrapper(const tran_low_t *coeff, const tran_low_t *dqcoeff,
reinterpret_cast<const int16_t *>(dqcoeff), block_size);
}
+template <ErrorBlockQmFunc fn>
+int64_t BlockErrorQmWrapper(const tran_low_t *coeff, const tran_low_t *dqcoeff,
+ intptr_t block_size, int64_t *ssz, int bps) {
+ static qm_val_t qmatrix[4096];
+ static int16_t scan[4096];
+ static const bool initialized = [&]() {
+ for (int i = 0; i < 4096; ++i) {
+ qmatrix[i] = static_cast<qm_val_t>(16 + ((i * 37) & 0x7f));
+ // Permute scan within each 16-element chunk to exercise non-identity
+ // gathered qmatrix indexing for all valid block sizes >= 16.
+ scan[i] = static_cast<int16_t>((i & ~15) | ((i * 5 + 3) & 15));
+ }
+ return true;
+ }();
+ (void)initialized;
+ return fn(coeff, dqcoeff, block_size, qmatrix, scan, ssz, bps);
+}
+
class ErrorBlockTest : public ::testing::TestWithParam<ErrorBlockParam> {
public:
~ErrorBlockTest() override = default;
@@ -107,6 +131,19 @@ TEST_P(ErrorBlockTest, OperationCheck) {
dqcoeff[j] = -rnd(1 << msb);
}
}
+ // Zero out some 8-element chunks of dqcoeff (and occasionally coeff) to
+ // exercise the zero-chunk fast paths on mixed random inputs.
+ for (int j = 0; j < block_size; j += 8) {
+ const int mode = rnd(4);
+ if (mode == 0) {
+ for (int k = 0; k < 8; ++k) dqcoeff[j + k] = 0;
+ } else if (mode == 1) {
+ for (int k = 0; k < 8; ++k) {
+ coeff[j + k] = 0;
+ dqcoeff[j + k] = 0;
+ }
+ }
+ }
ref_ret =
ref_error_block_op_(coeff, dqcoeff, block_size, &ref_ssz, bit_depth_);
API_REGISTER_STATE_CHECK(
@@ -280,7 +317,13 @@ const ErrorBlockParam kErrorBlockTestParamsAvx2[] = {
make_tuple(&BlockError8BitWrapper<av1_block_error_avx2>,
&BlockError8BitWrapper<av1_block_error_c>, AOM_BITS_8),
make_tuple(&BlockErrorLpWrapper<av1_block_error_lp_avx2>,
- &BlockErrorLpWrapper<av1_block_error_lp_c>, AOM_BITS_8)
+ &BlockErrorLpWrapper<av1_block_error_lp_c>, AOM_BITS_8),
+ make_tuple(&BlockErrorQmWrapper<av1_block_error_qm_avx2>,
+ &BlockErrorQmWrapper<av1_block_error_qm_c>, AOM_BITS_8),
+ make_tuple(&BlockErrorQmWrapper<av1_block_error_qm_avx2>,
+ &BlockErrorQmWrapper<av1_block_error_qm_c>, AOM_BITS_10),
+ make_tuple(&BlockErrorQmWrapper<av1_block_error_qm_avx2>,
+ &BlockErrorQmWrapper<av1_block_error_qm_c>, AOM_BITS_12)
};
INSTANTIATE_TEST_SUITE_P(AVX2, ErrorBlockTest,