Commit b67a8004fd for aom
commit b67a8004fd89d204c7b34ad5d2ee41d09a1ef646
Author: Frank Galligan <fgalligan@google.com>
Date: Thu Oct 29 22:24:08 2026 -0700
av1: Add AVX2 vectorization for quantize_fp_qm
Vectorize forward quantization with quantization matrices (qm_ptr !=
NULL) using AVX2 SIMD for both low-bit-depth and high-bit-depth paths
(av1_quantize_fp_qm_avx2 and av1_highbd_quantize_fp_qm_avx2).
Previously, when quantization matrices were enabled (the default for
AVIF encoding via AOM_TUNE_IQ and when --enable-qm=1 or --tune=iq is
specified for video), the encoder fell back to the scalar C loop in
quantize_fp_helper_c, causing quantization to dominate CPU cycles.
Optimization details:
- Processes 8 coefficients per iteration using 256-bit AVX2 vectors.
- Employs vpmaddwd / vpabsd / vpsrad with fast zero-chunk skipping.
- Vectorizes dequantization and EOB reduction using parallel tree
reduction over inverted scan indices.
- Preserves 100% bit-exact output parity.
Benchmarking results (with Quantization Matrices enabled, enable_qm=1):
- Stock avifenc still-image (Default 1T): -8.71% encode time
- Stock avifenc still-image (Default 8T): -7.26% encode time
- 48-image multi-rate corpus (Full Suite 1T): -9.31% time
- Bit-exact output (1T and 8T)
Note: When quantization matrices are disabled (enable_qm=0), the encoder
uses the existing av1_quantize_fp_avx2 path without change.
TAG=agy
CONV=b929a630-a6cd-45a0-979c-1767c30178ee
Change-Id: I6aaafbc3997e1c6407d035480c5a4b21dc0de95c
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl
index 006017c8fd..46af4fb91d 100644
--- a/av1/common/av1_rtcd_defs.pl
+++ b/av1/common/av1_rtcd_defs.pl
@@ -312,6 +312,9 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
add_proto qw/void av1_quantize_fp_64x64/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan";
specialize qw/av1_quantize_fp_64x64 neon avx2/;
+ add_proto qw/void av1_quantize_fp_qm/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, int log_scale";
+ specialize qw/av1_quantize_fp_qm avx2/;
+
add_proto qw/void aom_quantize_b_helper/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, const int log_scale";
specialize qw/aom_quantize_b_helper neon/;
@@ -433,6 +436,9 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
if (aom_config("CONFIG_AV1_HIGHBITDEPTH") eq "yes") {
add_proto qw/void av1_highbd_quantize_fp/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan, int log_scale";
specialize qw/av1_highbd_quantize_fp sse4_1 avx2 neon/;
+
+ add_proto qw/void av1_highbd_quantize_fp_qm/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, int log_scale";
+ specialize qw/av1_highbd_quantize_fp_qm avx2/;
}
# End av1_high encoder functions
diff --git a/av1/encoder/av1_quantize.c b/av1/encoder/av1_quantize.c
index 49acf3332b..9a3ccd5de0 100644
--- a/av1/encoder/av1_quantize.c
+++ b/av1/encoder/av1_quantize.c
@@ -265,6 +265,18 @@ void av1_quantize_fp_64x64_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
eob_ptr, scan, iscan, NULL, NULL, 2);
}
+void av1_quantize_fp_qm_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
+ const int16_t *round_ptr, const int16_t *quant_ptr,
+ tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
+ const int16_t *dequant_ptr, uint16_t *eob_ptr,
+ const int16_t *scan, const int16_t *iscan,
+ const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr,
+ int log_scale) {
+ quantize_fp_helper_c(coeff_ptr, n_coeffs, NULL, round_ptr, quant_ptr, NULL,
+ qcoeff_ptr, dqcoeff_ptr, dequant_ptr, eob_ptr, scan,
+ iscan, qm_ptr, iqm_ptr, log_scale);
+}
+
void av1_quantize_fp_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const MACROBLOCK_PLANE *p, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
@@ -272,10 +284,9 @@ void av1_quantize_fp_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const qm_val_t *qm_ptr = qparam->qmatrix;
const qm_val_t *iqm_ptr = qparam->iqmatrix;
if (qm_ptr != NULL && iqm_ptr != NULL) {
- quantize_fp_helper_c(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_fp_QTX,
- p->quant_fp_QTX, p->quant_shift_QTX, qcoeff_ptr,
- dqcoeff_ptr, p->dequant_QTX, eob_ptr, sc->scan,
- sc->iscan, qm_ptr, iqm_ptr, qparam->log_scale);
+ av1_quantize_fp_qm(coeff_ptr, n_coeffs, p->round_fp_QTX, p->quant_fp_QTX,
+ qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, eob_ptr,
+ sc->scan, sc->iscan, qm_ptr, iqm_ptr, qparam->log_scale);
} else {
switch (qparam->log_scale) {
case 0:
@@ -431,10 +442,10 @@ void av1_highbd_quantize_fp_facade(const tran_low_t *coeff_ptr,
const qm_val_t *qm_ptr = qparam->qmatrix;
const qm_val_t *iqm_ptr = qparam->iqmatrix;
if (qm_ptr != NULL && iqm_ptr != NULL) {
- highbd_quantize_fp_helper_c(
- coeff_ptr, n_coeffs, p->zbin_QTX, p->round_fp_QTX, p->quant_fp_QTX,
- p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, eob_ptr,
- sc->scan, sc->iscan, qm_ptr, iqm_ptr, qparam->log_scale);
+ av1_highbd_quantize_fp_qm(coeff_ptr, n_coeffs, p->round_fp_QTX,
+ p->quant_fp_QTX, qcoeff_ptr, dqcoeff_ptr,
+ p->dequant_QTX, eob_ptr, sc->scan, sc->iscan,
+ qm_ptr, iqm_ptr, qparam->log_scale);
} else {
av1_highbd_quantize_fp(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_fp_QTX,
p->quant_fp_QTX, p->quant_shift_QTX, qcoeff_ptr,
@@ -577,6 +588,17 @@ void av1_highbd_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t count,
dequant_ptr, eob_ptr, scan, iscan, NULL, NULL,
log_scale);
}
+
+void av1_highbd_quantize_fp_qm_c(
+ const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr,
+ const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
+ const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan,
+ const int16_t *iscan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr,
+ int log_scale) {
+ highbd_quantize_fp_helper_c(coeff_ptr, n_coeffs, NULL, round_ptr, quant_ptr,
+ NULL, qcoeff_ptr, dqcoeff_ptr, dequant_ptr,
+ eob_ptr, scan, iscan, qm_ptr, iqm_ptr, log_scale);
+}
#endif // CONFIG_AV1_HIGHBITDEPTH
static void invert_quant(int16_t *quant, int16_t *shift, int d) {
diff --git a/av1/encoder/x86/av1_quantize_avx2.c b/av1/encoder/x86/av1_quantize_avx2.c
index d890a3541f..fb2eeb9807 100644
--- a/av1/encoder/x86/av1_quantize_avx2.c
+++ b/av1/encoder/x86/av1_quantize_avx2.c
@@ -412,3 +412,180 @@ void av1_quantize_fp_64x64_avx2(
}
*eob_ptr = quant_gather_eob(eob);
}
+
+static inline void init_qp_qm(const int16_t *round_ptr,
+ const int16_t *quant_ptr,
+ const int16_t *dequant_ptr, int log_scale,
+ __m256i *thr, __m256i *qp) {
+ const __m128i round = _mm_loadu_si128((const __m128i *)round_ptr);
+ const __m128i quant = _mm_loadu_si128((const __m128i *)quant_ptr);
+ const __m128i dequant = _mm_loadu_si128((const __m128i *)dequant_ptr);
+
+ // Sign-extend 16-bit quantizer parameters (lane 0: DC, lanes 1..7: AC) to
+ // 32-bit lanes.
+ qp[0] = _mm256_cvtepi16_epi32(round);
+ qp[1] = _mm256_cvtepi16_epi32(quant);
+ qp[2] = _mm256_cvtepi16_epi32(dequant);
+
+ if (log_scale > 0) {
+ const __m256i rnd = _mm256_set1_epi32(1 << (log_scale - 1));
+ qp[0] = _mm256_add_epi32(qp[0], rnd);
+ qp[0] = _mm256_srai_epi32(qp[0], log_scale);
+ }
+
+ *thr = _mm256_slli_epi32(qp[2], AOM_QM_BITS - (1 + log_scale));
+ // Subtracting 1 here eliminates a _mm256_cmpeq_epi32() instruction when
+ // calculating the zbin/threshold mask.
+ *thr = _mm256_sub_epi32(*thr, _mm256_set1_epi32(1));
+}
+
+static AOM_FORCE_INLINE __m256i get_max_lane_eob_qm(const int16_t *iscan_ptr,
+ __m256i v_eobmax,
+ __m256i v_mask) {
+ const __m256i v_iscan =
+ _mm256_cvtepi16_epi32(_mm_loadu_si128((const __m128i *)iscan_ptr));
+ // Add 1 to non-zero scan indices to convert to 1-based EOB counts.
+ const __m256i v_iscan_plus1 = _mm256_sub_epi32(v_iscan, v_mask);
+ const __m256i v_nz_iscan = _mm256_and_si256(v_iscan_plus1, v_mask);
+ return _mm256_max_epi32(v_eobmax, v_nz_iscan);
+}
+
+static AOM_FORCE_INLINE void quantize_fp_qm_8(
+ const __m256i *thr, const __m256i *qp, const tran_low_t *coeff_ptr,
+ const int16_t *iscan_ptr, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr,
+ tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, __m128i shift,
+ __m128i log_scale_vec, int is_highbd, __m256i *eob) {
+ // Load 8 coefficients and 8 quantization matrix weights, then check if any
+ // weighted coefficient exceeds the quantization threshold.
+ const __m256i coeff = _mm256_loadu_si256((const __m256i *)coeff_ptr);
+ const __m256i abs_coeff = _mm256_abs_epi32(coeff);
+ const __m256i wt =
+ _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)qm_ptr));
+ const __m256i abs_wt =
+ is_highbd
+ ? _mm256_mullo_epi32(abs_coeff, wt)
+ : _mm256_madd_epi16(
+ _mm256_min_epi32(abs_coeff, _mm256_set1_epi32(INT16_MAX)), wt);
+ const __m256i mask = _mm256_cmpgt_epi32(abs_wt, *thr);
+ const int nzflag = _mm256_movemask_epi8(mask);
+
+ if (nzflag) {
+ // Round and apply forward quantization matrix weights. For low bit-depth,
+ // clamp the rounded coefficient to INT16_MAX to match the C reference.
+ __m256i tmp_rnd = _mm256_add_epi32(abs_coeff, qp[0]);
+ if (!is_highbd) {
+ tmp_rnd = _mm256_min_epi32(tmp_rnd, _mm256_set1_epi32(INT16_MAX));
+ }
+ const __m256i tmp_wt = is_highbd ? _mm256_mullo_epi32(tmp_rnd, wt)
+ : _mm256_madd_epi16(tmp_rnd, wt);
+
+ // Multiply by quantizer (qp[1]) in 64-bit precision for even and odd
+ // 32-bit lanes, shift right by (16 - log_scale + AOM_QM_BITS), and
+ // recombine.
+ const __m256i prod_lo = _mm256_mul_epu32(tmp_wt, qp[1]);
+ const __m256i res_lo = _mm256_srl_epi64(prod_lo, shift);
+ const __m256i prod_hi = _mm256_mul_epu32(_mm256_srli_epi64(tmp_wt, 32),
+ _mm256_srli_epi64(qp[1], 32));
+ const __m256i res_hi =
+ _mm256_slli_epi64(_mm256_srl_epi64(prod_hi, shift), 32);
+ const __m256i abs_q =
+ _mm256_and_si256(_mm256_or_si256(res_lo, res_hi), mask);
+
+ // Dequantize using the inverse quantization matrix weights (iqm_ptr).
+ const __m256i iwt =
+ _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)iqm_ptr));
+ const __m256i dequant = _mm256_srai_epi32(
+ _mm256_add_epi32(_mm256_madd_epi16(qp[2], iwt),
+ _mm256_set1_epi32(1 << (AOM_QM_BITS - 1))),
+ AOM_QM_BITS);
+ const __m256i abs_dq =
+ _mm256_sra_epi32(_mm256_mullo_epi32(abs_q, dequant), log_scale_vec);
+
+ // Restore original signs and store quantized and dequantized coefficients.
+ const __m256i q = _mm256_sign_epi32(abs_q, coeff);
+ const __m256i dq = _mm256_sign_epi32(abs_dq, coeff);
+ const __m256i nz_mask = _mm256_cmpgt_epi32(abs_q, _mm256_setzero_si256());
+
+ _mm256_storeu_si256((__m256i *)qcoeff_ptr, q);
+ _mm256_storeu_si256((__m256i *)dqcoeff_ptr, dq);
+
+ *eob = get_max_lane_eob_qm(iscan_ptr, *eob, nz_mask);
+ } else {
+ const __m256i zero = _mm256_setzero_si256();
+ _mm256_storeu_si256((__m256i *)qcoeff_ptr, zero);
+ _mm256_storeu_si256((__m256i *)dqcoeff_ptr, zero);
+ }
+}
+
+static AOM_FORCE_INLINE void quantize_fp_qm_helper_avx2(
+ const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr,
+ const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
+ const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *iscan_ptr,
+ const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, int log_scale,
+ int is_highbd) {
+ const int step = 8;
+ const __m128i shift = _mm_cvtsi32_si128(16 - log_scale + AOM_QM_BITS);
+ const __m128i log_scale_vec = _mm_cvtsi32_si128(log_scale);
+ __m256i qp[3], thr;
+ __m256i eob = _mm256_setzero_si256();
+
+ init_qp_qm(round_ptr, quant_ptr, dequant_ptr, log_scale, &thr, qp);
+
+ // Process DC and the first 7 AC coeffs.
+ quantize_fp_qm_8(&thr, qp, coeff_ptr, iscan_ptr, qm_ptr, iqm_ptr, qcoeff_ptr,
+ dqcoeff_ptr, shift, log_scale_vec, is_highbd, &eob);
+
+ coeff_ptr += step;
+ qcoeff_ptr += step;
+ dqcoeff_ptr += step;
+ iscan_ptr += step;
+ qm_ptr += step;
+ iqm_ptr += step;
+ n_coeffs -= step;
+
+ // Overwrite the DC constants with AC constants.
+ update_qp(&thr, qp);
+
+ // AC only loop.
+ while (n_coeffs > 0) {
+ quantize_fp_qm_8(&thr, qp, coeff_ptr, iscan_ptr, qm_ptr, iqm_ptr,
+ qcoeff_ptr, dqcoeff_ptr, shift, log_scale_vec, is_highbd,
+ &eob);
+
+ coeff_ptr += step;
+ qcoeff_ptr += step;
+ dqcoeff_ptr += step;
+ iscan_ptr += step;
+ qm_ptr += step;
+ iqm_ptr += step;
+ n_coeffs -= step;
+ }
+ *eob_ptr = quant_gather_eob(eob);
+}
+
+void av1_quantize_fp_qm_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
+ const int16_t *round_ptr, const int16_t *quant_ptr,
+ tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
+ const int16_t *dequant_ptr, uint16_t *eob_ptr,
+ const int16_t *scan_ptr, const int16_t *iscan_ptr,
+ const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr,
+ int log_scale) {
+ (void)scan_ptr;
+ quantize_fp_qm_helper_avx2(coeff_ptr, n_coeffs, round_ptr, quant_ptr,
+ qcoeff_ptr, dqcoeff_ptr, dequant_ptr, eob_ptr,
+ iscan_ptr, qm_ptr, iqm_ptr, log_scale, 0);
+}
+
+#if CONFIG_AV1_HIGHBITDEPTH
+void av1_highbd_quantize_fp_qm_avx2(
+ const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr,
+ const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
+ const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan_ptr,
+ const int16_t *iscan_ptr, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr,
+ int log_scale) {
+ (void)scan_ptr;
+ quantize_fp_qm_helper_avx2(coeff_ptr, n_coeffs, round_ptr, quant_ptr,
+ qcoeff_ptr, dqcoeff_ptr, dequant_ptr, eob_ptr,
+ iscan_ptr, qm_ptr, iqm_ptr, log_scale, 1);
+}
+#endif
diff --git a/test/quantize_func_test.cc b/test/quantize_func_test.cc
index a36752964f..0c7cc123b7 100644
--- a/test/quantize_func_test.cc
+++ b/test/quantize_func_test.cc
@@ -22,6 +22,8 @@
#include "aom_dsp/txfm_common.h"
#include "aom_ports/aom_timer.h"
#include "av1/encoder/encoder.h"
+#include "av1/common/idct.h"
+#include "av1/common/quant_common.h"
#include "av1/common/scan.h"
#include "test/acm_random.h"
#include "test/register_state_check.h"
@@ -476,6 +478,36 @@ TEST_P(LowPrecisionQuantizeTest, DISABLED_Speed) {
using std::make_tuple;
#if HAVE_AVX2
+#if CONFIG_QUANT_MATRIX || CONFIG_AV1_DECODER
+using QuantizeFpQmFunc = void (*)(
+ const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr,
+ const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
+ const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan,
+ const int16_t *iscan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr,
+ int log_scale);
+
+const CommonQuantParams *GetQmParams() {
+ static CommonQuantParams qm_params;
+ static const bool initialized = [&]() {
+ av1_qm_init(&qm_params, 1);
+ return true;
+ }();
+ (void)initialized;
+ return &qm_params;
+}
+
+template <QuantizeFpQmFunc fn, TX_SIZE tx_size, int qm_level = 5>
+void quan_fp_qm_wrapper(QUAN_PARAM_LIST) {
+ (void)zbin_ptr;
+ (void)quant_shift_ptr;
+ const CommonQuantParams *qm_params = GetQmParams();
+ const qm_val_t *qm_ptr = qm_params->gqmatrix[qm_level][0][tx_size];
+ const qm_val_t *iqm_ptr = qm_params->giqmatrix[qm_level][0][tx_size];
+ const int log_scale = av1_get_tx_scale(tx_size);
+ fn(coeff_ptr, n_coeffs, round_ptr, quant_ptr, qcoeff_ptr, dqcoeff_ptr,
+ dequant_ptr, eob_ptr, scan, iscan, qm_ptr, iqm_ptr, log_scale);
+}
+#endif // CONFIG_QUANT_MATRIX || CONFIG_AV1_DECODER
const QuantizeParam<LPQuantizeFunc> kLPQParamArrayAvx2[] = {
make_tuple(&av1_quantize_lp_c, &av1_quantize_lp_avx2,
@@ -508,6 +540,17 @@ const QuantizeParam<QuantizeFunc> kQParamArrayAvx2[] = {
static_cast<TX_SIZE>(TX_64X16), TYPE_FP, AOM_BITS_8),
make_tuple(&av1_quantize_fp_64x64_c, &av1_quantize_fp_64x64_avx2,
static_cast<TX_SIZE>(TX_64X64), TYPE_FP, AOM_BITS_8),
+#if CONFIG_QUANT_MATRIX || CONFIG_AV1_DECODER
+ make_tuple(&quan_fp_qm_wrapper<av1_quantize_fp_qm_c, TX_16X16>,
+ &quan_fp_qm_wrapper<av1_quantize_fp_qm_avx2, TX_16X16>,
+ static_cast<TX_SIZE>(TX_16X16), TYPE_FP, AOM_BITS_8),
+ make_tuple(&quan_fp_qm_wrapper<av1_quantize_fp_qm_c, TX_32X32>,
+ &quan_fp_qm_wrapper<av1_quantize_fp_qm_avx2, TX_32X32>,
+ static_cast<TX_SIZE>(TX_32X32), TYPE_FP, AOM_BITS_8),
+ make_tuple(&quan_fp_qm_wrapper<av1_quantize_fp_qm_c, TX_64X64>,
+ &quan_fp_qm_wrapper<av1_quantize_fp_qm_avx2, TX_64X64>,
+ static_cast<TX_SIZE>(TX_64X64), TYPE_FP, AOM_BITS_8),
+#endif // CONFIG_QUANT_MATRIX || CONFIG_AV1_DECODER
#if CONFIG_AV1_HIGHBITDEPTH
make_tuple(&highbd_quan16x16_wrapper<av1_highbd_quantize_fp_c>,
&highbd_quan16x16_wrapper<av1_highbd_quantize_fp_avx2>,
@@ -536,6 +579,35 @@ const QuantizeParam<QuantizeFunc> kQParamArrayAvx2[] = {
make_tuple(&highbd_quan64x64_wrapper<av1_highbd_quantize_fp_c>,
&highbd_quan64x64_wrapper<av1_highbd_quantize_fp_avx2>,
static_cast<TX_SIZE>(TX_64X64), TYPE_FP, AOM_BITS_12),
+#if CONFIG_QUANT_MATRIX || CONFIG_AV1_DECODER
+ make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_16X16>,
+ &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_avx2, TX_16X16>,
+ static_cast<TX_SIZE>(TX_16X16), TYPE_FP, AOM_BITS_8),
+ make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_16X16>,
+ &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_avx2, TX_16X16>,
+ static_cast<TX_SIZE>(TX_16X16), TYPE_FP, AOM_BITS_10),
+ make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_16X16>,
+ &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_avx2, TX_16X16>,
+ static_cast<TX_SIZE>(TX_16X16), TYPE_FP, AOM_BITS_12),
+ make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_32X32>,
+ &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_avx2, TX_32X32>,
+ static_cast<TX_SIZE>(TX_32X32), TYPE_FP, AOM_BITS_8),
+ make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_32X32>,
+ &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_avx2, TX_32X32>,
+ static_cast<TX_SIZE>(TX_32X32), TYPE_FP, AOM_BITS_10),
+ make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_32X32>,
+ &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_avx2, TX_32X32>,
+ static_cast<TX_SIZE>(TX_32X32), TYPE_FP, AOM_BITS_12),
+ make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_64X64>,
+ &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_avx2, TX_64X64>,
+ static_cast<TX_SIZE>(TX_64X64), TYPE_FP, AOM_BITS_8),
+ make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_64X64>,
+ &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_avx2, TX_64X64>,
+ static_cast<TX_SIZE>(TX_64X64), TYPE_FP, AOM_BITS_10),
+ make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_64X64>,
+ &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_avx2, TX_64X64>,
+ static_cast<TX_SIZE>(TX_64X64), TYPE_FP, AOM_BITS_12),
+#endif // CONFIG_QUANT_MATRIX || CONFIG_AV1_DECODER
make_tuple(&aom_highbd_quantize_b_c, &aom_highbd_quantize_b_avx2,
static_cast<TX_SIZE>(TX_16X16), TYPE_B, AOM_BITS_8),
make_tuple(&aom_highbd_quantize_b_c, &aom_highbd_quantize_b_avx2,