Commit b0904c4c10 for aom
commit b0904c4c10f4d3db936cd3f9787b8c5d39bdfdf0
Author: Frank Galligan <fgalligan@google.com>
Date: Sat Nov 14 11:52:39 2026 -0800
av1,neon: Add LBD/HBD quantize_{fp,b}_qm_neon
Add NEON implementations for av1_quantize_fp_qm, av1_quantize_b_qm,
av1_highbd_quantize_fp_qm, and av1_highbd_quantize_b_qm, optimize
zero-skip pre-scans in quantize_fp and highbd_quantize_fp, and
register the new kernels in QuantizeTest.
Speedup: 3.64x-6.02x vs C (quantize_{fp,b}_qm_neon) in test_libaom on
Neoverse N3; -11.99% (1T) / -12.41% (4T) 8b on Neoverse N3, -12.26%
(1T) / -12.48% (4T) 10b on Neoverse N2, and -12.52% (1T Cortex-X4) /
-12.45% (1T Cortex-A720) on Pixel 9 avifenc (bit-exact).
TAG=agy
CONV=490488d4-7005-402a-8cc4-c245227f80fe
Change-Id: I435e150f71313c08856ab25175aafeca0295170b
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl
index 5b1aa3e1ca..4cd3e13958 100644
--- a/av1/common/av1_rtcd_defs.pl
+++ b/av1/common/av1_rtcd_defs.pl
@@ -320,10 +320,10 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
specialize qw/av1_quantize_fp_64x64 neon avx2/;
add_proto qw/void av1_quantize_fp_qm/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, int log_scale";
- specialize qw/av1_quantize_fp_qm avx2/;
+ specialize qw/av1_quantize_fp_qm avx2 neon/;
add_proto qw/void av1_quantize_b_qm/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, int log_scale";
- specialize qw/av1_quantize_b_qm avx2/;
+ specialize qw/av1_quantize_b_qm avx2 neon/;
add_proto qw/void aom_quantize_b_helper/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, const int log_scale";
specialize qw/aom_quantize_b_helper neon/;
@@ -456,10 +456,10 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
specialize qw/av1_highbd_quantize_fp sse4_1 avx2 neon/;
add_proto qw/void av1_highbd_quantize_fp_qm/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, int log_scale";
- specialize qw/av1_highbd_quantize_fp_qm avx2/;
+ specialize qw/av1_highbd_quantize_fp_qm avx2 neon/;
add_proto qw/void av1_highbd_quantize_b_qm/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, int log_scale";
- specialize qw/av1_highbd_quantize_b_qm avx2/;
+ specialize qw/av1_highbd_quantize_b_qm avx2 neon/;
}
# End av1_high encoder functions
diff --git a/av1/encoder/arm/av1_highbd_quantize_neon.c b/av1/encoder/arm/av1_highbd_quantize_neon.c
index fd4749d286..e70a04840f 100644
--- a/av1/encoder/arm/av1_highbd_quantize_neon.c
+++ b/av1/encoder/arm/av1_highbd_quantize_neon.c
@@ -10,42 +10,140 @@
*/
#include <arm_neon.h>
+#include <assert.h>
#include "config/aom_config.h"
+#include "config/av1_rtcd.h"
#include "aom_dsp/arm/mem_neon.h"
#include "av1/common/quant_common.h"
#include "av1/encoder/av1_quantize.h"
-static inline uint16x4_t quantize_4(const tran_low_t *coeff_ptr,
- tran_low_t *qcoeff_ptr,
- tran_low_t *dqcoeff_ptr,
- int32x4_t v_quant_s32,
- int32x4_t v_dequant_s32,
- int32x4_t v_round_s32, int log_scale) {
- const int32x4_t v_coeff = vld1q_s32(coeff_ptr);
+static AOM_FORCE_INLINE uint16x4_t quantize_4_from_abs_ls0(
+ int32x4_t v_coeff, int32x4_t v_abs_coeff, tran_low_t *qcoeff_ptr,
+ tran_low_t *dqcoeff_ptr, int32x4_t v_quant_s32, int32x4_t v_dequant_s32,
+ int32x4_t v_round_s32) {
+ const int32x4_t v_zero = vdupq_n_s32(0);
const int32x4_t v_coeff_sign =
- vreinterpretq_s32_u32(vcltq_s32(v_coeff, vdupq_n_s32(0)));
- const int32x4_t v_log_scale = vdupq_n_s32(log_scale);
- const int32x4_t v_abs_coeff = vabsq_s32(v_coeff);
+ vreinterpretq_s32_u32(vcltq_s32(v_coeff, v_zero));
+ const int32x4_t v_abs_coeff_scaled = vshlq_n_s32(v_abs_coeff, 1);
+ const uint32x4_t v_mask = vcgeq_s32(v_abs_coeff_scaled, v_dequant_s32);
+ const int32x4_t v_tmp = vandq_s32(vaddq_s32(v_abs_coeff, v_round_s32),
+ vreinterpretq_s32_u32(v_mask));
+ const int32x4_t v_abs_qcoeff = vqdmulhq_s32(v_tmp, v_quant_s32);
+ const int32x4_t v_qcoeff =
+ vsubq_s32(veorq_s32(v_abs_qcoeff, v_coeff_sign), v_coeff_sign);
+ const int32x4_t v_abs_dqcoeff = vmulq_s32(v_abs_qcoeff, v_dequant_s32);
+ const int32x4_t v_dqcoeff =
+ vsubq_s32(veorq_s32(v_abs_dqcoeff, v_coeff_sign), v_coeff_sign);
+ vst1q_s32(qcoeff_ptr, v_qcoeff);
+ vst1q_s32(dqcoeff_ptr, v_dqcoeff);
+ const uint32x4_t nz_qcoeff_mask = vcgtq_s32(v_abs_qcoeff, v_zero);
+ return vmovn_u32(nz_qcoeff_mask);
+}
+
+static AOM_FORCE_INLINE uint16x4_t quantize_4_from_abs_ls1(
+ int32x4_t v_coeff, int32x4_t v_abs_coeff, tran_low_t *qcoeff_ptr,
+ tran_low_t *dqcoeff_ptr, int32x4_t v_quant_s32, int32x4_t v_dequant_s32,
+ int32x4_t v_round_s32) {
+ const int32x4_t v_zero = vdupq_n_s32(0);
+ const int32x4_t v_coeff_sign =
+ vreinterpretq_s32_u32(vcltq_s32(v_coeff, v_zero));
+ const int32x4_t v_abs_coeff_scaled = vshlq_n_s32(v_abs_coeff, 2);
+ const uint32x4_t v_mask = vcgeq_s32(v_abs_coeff_scaled, v_dequant_s32);
+ const int32x4_t v_tmp = vandq_s32(vaddq_s32(v_abs_coeff, v_round_s32),
+ vreinterpretq_s32_u32(v_mask));
+ const int32x4_t v_abs_qcoeff =
+ vqdmulhq_s32(vshlq_n_s32(v_tmp, 1), v_quant_s32);
+ const int32x4_t v_qcoeff =
+ vsubq_s32(veorq_s32(v_abs_qcoeff, v_coeff_sign), v_coeff_sign);
+ const int32x4_t v_abs_dqcoeff =
+ vshrq_n_s32(vmulq_s32(v_abs_qcoeff, v_dequant_s32), 1);
+ const int32x4_t v_dqcoeff =
+ vsubq_s32(veorq_s32(v_abs_dqcoeff, v_coeff_sign), v_coeff_sign);
+ vst1q_s32(qcoeff_ptr, v_qcoeff);
+ vst1q_s32(dqcoeff_ptr, v_dqcoeff);
+ const uint32x4_t nz_qcoeff_mask = vcgtq_s32(v_abs_qcoeff, v_zero);
+ return vmovn_u32(nz_qcoeff_mask);
+}
+
+static AOM_FORCE_INLINE uint16x4_t quantize_4_from_abs_ls2(
+ int32x4_t v_coeff, int32x4_t v_abs_coeff, tran_low_t *qcoeff_ptr,
+ tran_low_t *dqcoeff_ptr, int32x4_t v_quant_s32, int32x4_t v_dequant_s32,
+ int32x4_t v_round_s32) {
+ const int32x4_t v_zero = vdupq_n_s32(0);
+ const int32x4_t v_coeff_sign =
+ vreinterpretq_s32_u32(vcltq_s32(v_coeff, v_zero));
+ const int32x4_t v_abs_coeff_scaled = vshlq_n_s32(v_abs_coeff, 3);
+ const uint32x4_t v_mask = vcgeq_s32(v_abs_coeff_scaled, v_dequant_s32);
+ const int32x4_t v_tmp = vandq_s32(vaddq_s32(v_abs_coeff, v_round_s32),
+ vreinterpretq_s32_u32(v_mask));
+ const int32x4_t v_abs_qcoeff =
+ vqdmulhq_s32(vshlq_n_s32(v_tmp, 2), v_quant_s32);
+ const int32x4_t v_qcoeff =
+ vsubq_s32(veorq_s32(v_abs_qcoeff, v_coeff_sign), v_coeff_sign);
+ const int32x4_t v_abs_dqcoeff =
+ vshrq_n_s32(vmulq_s32(v_abs_qcoeff, v_dequant_s32), 2);
+ const int32x4_t v_dqcoeff =
+ vsubq_s32(veorq_s32(v_abs_dqcoeff, v_coeff_sign), v_coeff_sign);
+ vst1q_s32(qcoeff_ptr, v_qcoeff);
+ vst1q_s32(dqcoeff_ptr, v_dqcoeff);
+ const uint32x4_t nz_qcoeff_mask = vcgtq_s32(v_abs_qcoeff, v_zero);
+ return vmovn_u32(nz_qcoeff_mask);
+}
+
+static AOM_FORCE_INLINE int hbd_zero_skip_u32(int32x4_t v_abs0,
+ int32x4_t v_abs1,
+ uint32_t ac_thresh,
+ tran_low_t *qcoeff_ptr,
+ tran_low_t *dqcoeff_ptr) {
+#if AOM_ARCH_AARCH64
+ const int32x4_t v_zero = vdupq_n_s32(0);
+ const uint32x4_t v_abs_max =
+ vmaxq_u32(vreinterpretq_u32_s32(v_abs0), vreinterpretq_u32_s32(v_abs1));
+ if (vmaxvq_u32(v_abs_max) < ac_thresh) {
+ vst1q_s32(qcoeff_ptr, v_zero);
+ vst1q_s32(qcoeff_ptr + 4, v_zero);
+ vst1q_s32(dqcoeff_ptr, v_zero);
+ vst1q_s32(dqcoeff_ptr + 4, v_zero);
+ return 1;
+ }
+#else
+ (void)v_abs0;
+ (void)v_abs1;
+ (void)ac_thresh;
+ (void)qcoeff_ptr;
+ (void)dqcoeff_ptr;
+#endif
+ return 0;
+}
+
+static inline uint16x4_t quantize_4_from_abs(
+ int32x4_t v_coeff, int32x4_t v_abs_coeff, tran_low_t *qcoeff_ptr,
+ tran_low_t *dqcoeff_ptr, int32x4_t v_quant_s32, int32x4_t v_dequant_s32,
+ int32x4_t v_round_s32, int32x4_t v_log_scale, int32x4_t v_neg_log_scale,
+ int32x4_t v_one_plus_log_scale) {
+ const int32x4_t v_zero = vdupq_n_s32(0);
+ const int32x4_t v_coeff_sign =
+ vreinterpretq_s32_u32(vcltq_s32(v_coeff, v_zero));
// ((abs_coeff << (1 + log_scale)) >= dequant_ptr[rc01])
const int32x4_t v_abs_coeff_scaled =
- vshlq_s32(v_abs_coeff, vdupq_n_s32(1 + log_scale));
+ vshlq_s32(v_abs_coeff, v_one_plus_log_scale);
const uint32x4_t v_mask = vcgeq_s32(v_abs_coeff_scaled, v_dequant_s32);
// const int64_t tmp = vmask ? (int64_t)abs_coeff + log_scaled_round : 0
const int32x4_t v_tmp = vandq_s32(vaddq_s32(v_abs_coeff, v_round_s32),
vreinterpretq_s32_u32(v_mask));
- // const int abs_qcoeff = (int)((tmp * quant) >> (16 - log_scale));
+ // const int abs_qcoeff = (int)((tmp * quant) >> (16 - log_scale));
const int32x4_t v_abs_qcoeff =
vqdmulhq_s32(vshlq_s32(v_tmp, v_log_scale), v_quant_s32);
- // qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
+ // qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
const int32x4_t v_qcoeff =
vsubq_s32(veorq_s32(v_abs_qcoeff, v_coeff_sign), v_coeff_sign);
// vshlq_s32 will shift right if shift value is negative.
const int32x4_t v_abs_dqcoeff =
- vshlq_s32(vmulq_s32(v_abs_qcoeff, v_dequant_s32), vnegq_s32(v_log_scale));
- // dqcoeff_ptr[rc] = (tran_low_t)((abs_dqcoeff ^ coeff_sign) - coeff_sign);
+ vshlq_s32(vmulq_s32(v_abs_qcoeff, v_dequant_s32), v_neg_log_scale);
+ // dqcoeff_ptr[rc] = (tran_low_t)((abs_dqcoeff ^ coeff_sign) - coeff_sign);
const int32x4_t v_dqcoeff =
vsubq_s32(veorq_s32(v_abs_dqcoeff, v_coeff_sign), v_coeff_sign);
@@ -53,7 +151,7 @@ static inline uint16x4_t quantize_4(const tran_low_t *coeff_ptr,
vst1q_s32(dqcoeff_ptr, v_dqcoeff);
// Used to find eob.
- const uint32x4_t nz_qcoeff_mask = vcgtq_s32(v_abs_qcoeff, vdupq_n_s32(0));
+ const uint32x4_t nz_qcoeff_mask = vcgtq_s32(v_abs_qcoeff, v_zero);
return vmovn_u32(nz_qcoeff_mask);
}
@@ -84,6 +182,50 @@ static inline uint16_t get_max_eob(int16x8_t v_eobmax) {
#endif
}
+#define HBD_QUANTIZE_BODY(LS, V_ROUND_INIT) \
+ do { \
+ const int16x4_t v_round = (V_ROUND_INIT); \
+ int32x4_t v_round_s32 = vaddl_s16(v_round, v_zero); \
+ int32x4_t v_quant_s32 = vshlq_n_s32(vaddl_s16(v_quant, v_zero), 15); \
+ int32x4_t v_dequant_s32 = vaddl_s16(v_dequant, v_zero); \
+ const int32x4_t v_dc_coeff = vld1q_s32(coeff_ptr); \
+ v_mask_lo = quantize_4_from_abs_ls##LS( \
+ v_dc_coeff, vabsq_s32(v_dc_coeff), qcoeff_ptr, dqcoeff_ptr, \
+ v_quant_s32, v_dequant_s32, v_round_s32); \
+ v_round_s32 = vdupq_lane_s32(vget_low_s32(v_round_s32), 1); \
+ v_quant_s32 = vdupq_lane_s32(vget_low_s32(v_quant_s32), 1); \
+ v_dequant_s32 = vdupq_lane_s32(vget_low_s32(v_dequant_s32), 1); \
+ const int32x4_t v_ac_coeff = vld1q_s32(coeff_ptr + 4); \
+ v_mask_hi = quantize_4_from_abs_ls##LS( \
+ v_ac_coeff, vabsq_s32(v_ac_coeff), qcoeff_ptr + 4, dqcoeff_ptr + 4, \
+ v_quant_s32, v_dequant_s32, v_round_s32); \
+ v_eobmax = \
+ get_max_lane_eob(iscan, v_eobmax, vcombine_u16(v_mask_lo, v_mask_hi)); \
+ count -= 8; \
+ do { \
+ coeff_ptr += 8; \
+ qcoeff_ptr += 8; \
+ dqcoeff_ptr += 8; \
+ iscan += 8; \
+ const int32x4_t v_c0 = vld1q_s32(coeff_ptr); \
+ const int32x4_t v_c1 = vld1q_s32(coeff_ptr + 4); \
+ const int32x4_t v_abs0 = vabsq_s32(v_c0); \
+ const int32x4_t v_abs1 = vabsq_s32(v_c1); \
+ if (!hbd_zero_skip_u32(v_abs0, v_abs1, ac_thresh, qcoeff_ptr, \
+ dqcoeff_ptr)) { \
+ v_mask_lo = quantize_4_from_abs_ls##LS(v_c0, v_abs0, qcoeff_ptr, \
+ dqcoeff_ptr, v_quant_s32, \
+ v_dequant_s32, v_round_s32); \
+ v_mask_hi = quantize_4_from_abs_ls##LS(v_c1, v_abs1, qcoeff_ptr + 4, \
+ dqcoeff_ptr + 4, v_quant_s32, \
+ v_dequant_s32, v_round_s32); \
+ v_eobmax = get_max_lane_eob(iscan, v_eobmax, \
+ vcombine_u16(v_mask_lo, v_mask_hi)); \
+ } \
+ count -= 8; \
+ } while (count != 0); \
+ } while (0)
+
void av1_highbd_quantize_fp_neon(
const tran_low_t *coeff_ptr, intptr_t count, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
@@ -93,54 +235,361 @@ void av1_highbd_quantize_fp_neon(
(void)scan;
(void)zbin_ptr;
(void)quant_shift_ptr;
+ assert(count >= 16 && (count & 7) == 0);
const int16x4_t v_quant = vld1_s16(quant_ptr);
const int16x4_t v_dequant = vld1_s16(dequant_ptr);
const int16x4_t v_zero = vdup_n_s16(0);
- const uint16x4_t v_round_select = vcgt_s16(vdup_n_s16(log_scale), v_zero);
- const int16x4_t v_round_no_scale = vld1_s16(round_ptr);
- const int16x4_t v_round_log_scale =
- vqrdmulh_n_s16(v_round_no_scale, (int16_t)(1 << (15 - log_scale)));
- const int16x4_t v_round =
- vbsl_s16(v_round_select, v_round_log_scale, v_round_no_scale);
- int32x4_t v_round_s32 = vaddl_s16(v_round, v_zero);
- int32x4_t v_quant_s32 = vshlq_n_s32(vaddl_s16(v_quant, v_zero), 15);
- int32x4_t v_dequant_s32 = vaddl_s16(v_dequant, v_zero);
uint16x4_t v_mask_lo, v_mask_hi;
int16x8_t v_eobmax = vdupq_n_s16(-1);
+ const int ac_dequant = dequant_ptr[1];
+ const uint32_t ac_thresh =
+ ac_dequant > 0 ? (((uint32_t)ac_dequant + (1u << (1 + log_scale)) - 1u) >>
+ (1 + log_scale))
+ : 0u;
- // DC and first 3 AC
- v_mask_lo = quantize_4(coeff_ptr, qcoeff_ptr, dqcoeff_ptr, v_quant_s32,
- v_dequant_s32, v_round_s32, log_scale);
+ if (log_scale == 0) {
+ HBD_QUANTIZE_BODY(0, vld1_s16(round_ptr));
+ } else if (log_scale == 1) {
+ HBD_QUANTIZE_BODY(1,
+ vqrdmulh_n_s16(vld1_s16(round_ptr), (int16_t)(1 << 14)));
+ } else if (log_scale == 2) {
+ HBD_QUANTIZE_BODY(2,
+ vqrdmulh_n_s16(vld1_s16(round_ptr), (int16_t)(1 << 13)));
+ } else {
+ const uint16x4_t v_round_select = vcgt_s16(vdup_n_s16(log_scale), v_zero);
+ const int16x4_t v_round_no_scale = vld1_s16(round_ptr);
+ const int16x4_t v_round_log_scale =
+ vqrdmulh_n_s16(v_round_no_scale, (int16_t)(1 << (15 - log_scale)));
+ const int16x4_t v_round =
+ vbsl_s16(v_round_select, v_round_log_scale, v_round_no_scale);
+ int32x4_t v_round_s32 = vaddl_s16(v_round, v_zero);
+ int32x4_t v_quant_s32 = vshlq_n_s32(vaddl_s16(v_quant, v_zero), 15);
+ int32x4_t v_dequant_s32 = vaddl_s16(v_dequant, v_zero);
+ const int32x4_t v_log_scale = vdupq_n_s32(log_scale);
+ const int32x4_t v_neg_log_scale = vnegq_s32(v_log_scale);
+ const int32x4_t v_one_plus_log_scale = vdupq_n_s32(1 + log_scale);
- // overwrite the DC constants with AC constants
- v_round_s32 = vdupq_lane_s32(vget_low_s32(v_round_s32), 1);
- v_quant_s32 = vdupq_lane_s32(vget_low_s32(v_quant_s32), 1);
- v_dequant_s32 = vdupq_lane_s32(vget_low_s32(v_dequant_s32), 1);
+ // DC and first 3 AC
+ const int32x4_t v_dc_coeff = vld1q_s32(coeff_ptr);
+ v_mask_lo = quantize_4_from_abs(v_dc_coeff, vabsq_s32(v_dc_coeff),
+ qcoeff_ptr, dqcoeff_ptr, v_quant_s32,
+ v_dequant_s32, v_round_s32, v_log_scale,
+ v_neg_log_scale, v_one_plus_log_scale);
- // 4 more AC
- v_mask_hi = quantize_4(coeff_ptr + 4, qcoeff_ptr + 4, dqcoeff_ptr + 4,
- v_quant_s32, v_dequant_s32, v_round_s32, log_scale);
+ // overwrite the DC constants with AC constants
+ v_round_s32 = vdupq_lane_s32(vget_low_s32(v_round_s32), 1);
+ v_quant_s32 = vdupq_lane_s32(vget_low_s32(v_quant_s32), 1);
+ v_dequant_s32 = vdupq_lane_s32(vget_low_s32(v_dequant_s32), 1);
- // Find the max lane eob for the first 8 coeffs.
- v_eobmax =
- get_max_lane_eob(iscan, v_eobmax, vcombine_u16(v_mask_lo, v_mask_hi));
+ // 4 more AC
+ const int32x4_t v_ac_coeff = vld1q_s32(coeff_ptr + 4);
+ v_mask_hi = quantize_4_from_abs(
+ v_ac_coeff, vabsq_s32(v_ac_coeff), qcoeff_ptr + 4, dqcoeff_ptr + 4,
+ v_quant_s32, v_dequant_s32, v_round_s32, v_log_scale, v_neg_log_scale,
+ v_one_plus_log_scale);
- count -= 8;
- do {
- coeff_ptr += 8;
- qcoeff_ptr += 8;
- dqcoeff_ptr += 8;
- iscan += 8;
- v_mask_lo = quantize_4(coeff_ptr, qcoeff_ptr, dqcoeff_ptr, v_quant_s32,
- v_dequant_s32, v_round_s32, log_scale);
- v_mask_hi = quantize_4(coeff_ptr + 4, qcoeff_ptr + 4, dqcoeff_ptr + 4,
- v_quant_s32, v_dequant_s32, v_round_s32, log_scale);
- // Find the max lane eob for 8 coeffs.
+ // Find the max lane eob for the first 8 coeffs.
v_eobmax =
get_max_lane_eob(iscan, v_eobmax, vcombine_u16(v_mask_lo, v_mask_hi));
+
count -= 8;
- } while (count);
+ do {
+ coeff_ptr += 8;
+ qcoeff_ptr += 8;
+ dqcoeff_ptr += 8;
+ iscan += 8;
+ const int32x4_t v_c0 = vld1q_s32(coeff_ptr);
+ const int32x4_t v_c1 = vld1q_s32(coeff_ptr + 4);
+ const int32x4_t v_abs0 = vabsq_s32(v_c0);
+ const int32x4_t v_abs1 = vabsq_s32(v_c1);
+ if (!hbd_zero_skip_u32(v_abs0, v_abs1, ac_thresh, qcoeff_ptr,
+ dqcoeff_ptr)) {
+ v_mask_lo = quantize_4_from_abs(
+ v_c0, v_abs0, qcoeff_ptr, dqcoeff_ptr, v_quant_s32, v_dequant_s32,
+ v_round_s32, v_log_scale, v_neg_log_scale, v_one_plus_log_scale);
+ v_mask_hi = quantize_4_from_abs(v_c1, v_abs1, qcoeff_ptr + 4,
+ dqcoeff_ptr + 4, v_quant_s32,
+ v_dequant_s32, v_round_s32, v_log_scale,
+ v_neg_log_scale, v_one_plus_log_scale);
+ v_eobmax = get_max_lane_eob(iscan, v_eobmax,
+ vcombine_u16(v_mask_lo, v_mask_hi));
+ }
+ count -= 8;
+ } while (count != 0);
+ }
+
+ *eob_ptr = get_max_eob(v_eobmax);
+}
+#undef HBD_QUANTIZE_BODY
+
+#if CONFIG_AV1_HIGHBITDEPTH
+static AOM_FORCE_INLINE int mask_any_nonzero_u32(uint32x4_t m0, uint32x4_t m1) {
+ const uint32x4_t any = vorrq_u32(m0, m1);
+#if AOM_ARCH_AARCH64
+ return vmaxvq_u32(any) != 0;
+#else
+ const uint64x2_t u64 = vreinterpretq_u64_u32(any);
+ return (vgetq_lane_u64(u64, 0) | vgetq_lane_u64(u64, 1)) != 0;
+#endif
+}
+static AOM_FORCE_INLINE int16x8_t update_qm_eob(const int16_t *iscan_ptr,
+ int16x8_t v_eobmax,
+ uint32x4_t nz_mask0,
+ uint32x4_t nz_mask1) {
+#if AOM_ARCH_AARCH64
+ const uint16x8_t v_nz_mask = vuzp1q_u16(vreinterpretq_u16_u32(nz_mask0),
+ vreinterpretq_u16_u32(nz_mask1));
+#else
+ const uint16x8_t v_nz_mask =
+ vcombine_u16(vmovn_u32(nz_mask0), vmovn_u32(nz_mask1));
+#endif
+ return get_max_lane_eob(iscan_ptr, v_eobmax, v_nz_mask);
+}
+
+static AOM_FORCE_INLINE int16x8_t highbd_quantize_fp_qm_8_neon(
+ const tran_low_t *coeff_ptr, const int16_t *iscan_ptr,
+ const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, tran_low_t *qcoeff_ptr,
+ tran_low_t *dqcoeff_ptr, int32x4_t v_thr0, int32x4_t v_thr1,
+ int32x4_t v_round0, int32x4_t v_round1, int32x4_t v_quant0,
+ int32x4_t v_quant1, int32x4_t v_dequant0, int32x4_t v_dequant1,
+ int32x4_t v_log_scale, int32x4_t v_neg_log_scale, int16x8_t v_eobmax) {
+ const int32x4_t v_coeff0 = vld1q_s32(coeff_ptr);
+ const int32x4_t v_coeff1 = vld1q_s32(coeff_ptr + 4);
+ const int32x4_t v_abs0 = vabsq_s32(v_coeff0);
+ const int32x4_t v_abs1 = vabsq_s32(v_coeff1);
+ const uint16x8_t v_wt_u16 = vmovl_u8(vld1_u8(qm_ptr));
+ const int32x4_t v_wt0 =
+ vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(v_wt_u16)));
+ const int32x4_t v_wt1 =
+ vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(v_wt_u16)));
+ const uint32x4_t v_mask0 = vcgeq_s32(vmulq_s32(v_abs0, v_wt0), v_thr0);
+ const uint32x4_t v_mask1 = vcgeq_s32(vmulq_s32(v_abs1, v_wt1), v_thr1);
+
+ if (mask_any_nonzero_u32(v_mask0, v_mask1)) {
+ const int32x4_t v_zero = vdupq_n_s32(0);
+ const int32x4_t v_sign0 = vshrq_n_s32(v_coeff0, 31);
+ const int32x4_t v_sign1 = vshrq_n_s32(v_coeff1, 31);
+ const int32x4_t v_tmp0 =
+ vandq_s32(vaddq_s32(v_abs0, v_round0), vreinterpretq_s32_u32(v_mask0));
+ const int32x4_t v_tmp1 =
+ vandq_s32(vaddq_s32(v_abs1, v_round1), vreinterpretq_s32_u32(v_mask1));
+ // Forward QM wt < 256 (AOM_QM_BITS = 5) and log_scale <= 2 fit in 32 bits.
+ const int32x4_t v_tmp_wt0 = vmulq_s32(v_tmp0, v_wt0);
+ const int32x4_t v_tmp_wt1 = vmulq_s32(v_tmp1, v_wt1);
+ const int32x4_t v_abs_q0 = vshrq_n_s32(
+ vqdmulhq_s32(vshlq_s32(v_tmp_wt0, v_log_scale), v_quant0), AOM_QM_BITS);
+ const int32x4_t v_abs_q1 = vshrq_n_s32(
+ vqdmulhq_s32(vshlq_s32(v_tmp_wt1, v_log_scale), v_quant1), AOM_QM_BITS);
+
+ const uint16x8_t v_iwt_u16 = vmovl_u8(vld1_u8(iqm_ptr));
+ const int32x4_t v_iwt0 =
+ vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(v_iwt_u16)));
+ const int32x4_t v_iwt1 =
+ vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(v_iwt_u16)));
+ const int32x4_t v_dq_rnd = vdupq_n_s32(1 << (AOM_QM_BITS - 1));
+ const int32x4_t v_dq0 =
+ vshrq_n_s32(vmlaq_s32(v_dq_rnd, v_dequant0, v_iwt0), AOM_QM_BITS);
+ const int32x4_t v_dq1 =
+ vshrq_n_s32(vmlaq_s32(v_dq_rnd, v_dequant1, v_iwt1), AOM_QM_BITS);
+ const int32x4_t v_abs_dq0 =
+ vshlq_s32(vmulq_s32(v_abs_q0, v_dq0), v_neg_log_scale);
+ const int32x4_t v_abs_dq1 =
+ vshlq_s32(vmulq_s32(v_abs_q1, v_dq1), v_neg_log_scale);
+
+ const int32x4_t v_q0 = vsubq_s32(veorq_s32(v_abs_q0, v_sign0), v_sign0);
+ const int32x4_t v_q1 = vsubq_s32(veorq_s32(v_abs_q1, v_sign1), v_sign1);
+ const int32x4_t v_dq_out0 =
+ vsubq_s32(veorq_s32(v_abs_dq0, v_sign0), v_sign0);
+ const int32x4_t v_dq_out1 =
+ vsubq_s32(veorq_s32(v_abs_dq1, v_sign1), v_sign1);
+ vst1q_s32(qcoeff_ptr, v_q0);
+ vst1q_s32(qcoeff_ptr + 4, v_q1);
+ vst1q_s32(dqcoeff_ptr, v_dq_out0);
+ vst1q_s32(dqcoeff_ptr + 4, v_dq_out1);
+
+ const uint32x4_t v_nz0 = vcgtq_s32(v_abs_q0, v_zero);
+ const uint32x4_t v_nz1 = vcgtq_s32(v_abs_q1, v_zero);
+ return update_qm_eob(iscan_ptr, v_eobmax, v_nz0, v_nz1);
+ }
+ const int32x4_t v_zero = vdupq_n_s32(0);
+ vst1q_s32(qcoeff_ptr, v_zero);
+ vst1q_s32(qcoeff_ptr + 4, v_zero);
+ vst1q_s32(dqcoeff_ptr, v_zero);
+ vst1q_s32(dqcoeff_ptr + 4, v_zero);
+ return v_eobmax;
+}
+
+void av1_highbd_quantize_fp_qm_neon(
+ const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr,
+ const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
+ const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan,
+ const int16_t *iscan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr,
+ int log_scale) {
+ (void)scan;
+ assert(qm_ptr != NULL && iqm_ptr != NULL);
+ assert(n_coeffs >= 16 && (n_coeffs & 7) == 0);
+ const int thr_shift = AOM_QM_BITS - (1 + log_scale);
+ const int32x4_t v_thr_ac = vdupq_n_s32((int32_t)dequant_ptr[1] << thr_shift);
+ const int32x4_t v_thr0 =
+ vsetq_lane_s32((int32_t)dequant_ptr[0] << thr_shift, v_thr_ac, 0);
+ const int32x4_t v_round_ac =
+ vdupq_n_s32(ROUND_POWER_OF_TWO(round_ptr[1], log_scale));
+ const int32x4_t v_round0 = vsetq_lane_s32(
+ ROUND_POWER_OF_TWO(round_ptr[0], log_scale), v_round_ac, 0);
+ const int32x4_t v_quant_ac =
+ vdupq_n_s32((int32_t)((uint32_t)quant_ptr[1] << 15));
+ const int32x4_t v_quant0 =
+ vsetq_lane_s32((int32_t)((uint32_t)quant_ptr[0] << 15), v_quant_ac, 0);
+ const int32x4_t v_dequant_ac = vdupq_n_s32(dequant_ptr[1]);
+ const int32x4_t v_dequant0 = vsetq_lane_s32(dequant_ptr[0], v_dequant_ac, 0);
+ const int32x4_t v_log_scale = vdupq_n_s32(log_scale);
+ const int32x4_t v_neg_log_scale = vdupq_n_s32(-log_scale);
+ int16x8_t v_eobmax = vdupq_n_s16(-1);
+
+ v_eobmax = highbd_quantize_fp_qm_8_neon(
+ coeff_ptr, iscan, qm_ptr, iqm_ptr, qcoeff_ptr, dqcoeff_ptr, v_thr0,
+ v_thr_ac, v_round0, v_round_ac, v_quant0, v_quant_ac, v_dequant0,
+ v_dequant_ac, v_log_scale, v_neg_log_scale, v_eobmax);
+
+ intptr_t i = 8;
+ do {
+ v_eobmax = highbd_quantize_fp_qm_8_neon(
+ coeff_ptr + i, iscan + i, qm_ptr + i, iqm_ptr + i, qcoeff_ptr + i,
+ dqcoeff_ptr + i, v_thr_ac, v_thr_ac, v_round_ac, v_round_ac, v_quant_ac,
+ v_quant_ac, v_dequant_ac, v_dequant_ac, v_log_scale, v_neg_log_scale,
+ v_eobmax);
+ i += 8;
+ } while (i != n_coeffs);
+ *eob_ptr = get_max_eob(v_eobmax);
+}
+
+static AOM_FORCE_INLINE int16x8_t highbd_quantize_b_qm_8_neon(
+ const tran_low_t *coeff_ptr, const int16_t *iscan_ptr,
+ const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, tran_low_t *qcoeff_ptr,
+ tran_low_t *dqcoeff_ptr, int32x4_t v_zbin0, int32x4_t v_zbin1,
+ int32x4_t v_round0, int32x4_t v_round1, int32x4_t v_quant0,
+ int32x4_t v_quant1, int32x4_t v_qshift0, int32x4_t v_qshift1,
+ int32x4_t v_dequant0, int32x4_t v_dequant1, int32x4_t v_log_scale,
+ int32x4_t v_neg_log_scale, int16x8_t v_eobmax) {
+ const int32x4_t v_coeff0 = vld1q_s32(coeff_ptr);
+ const int32x4_t v_coeff1 = vld1q_s32(coeff_ptr + 4);
+ const int32x4_t v_abs0 = vabsq_s32(v_coeff0);
+ const int32x4_t v_abs1 = vabsq_s32(v_coeff1);
+ const uint16x8_t v_wt_u16 = vmovl_u8(vld1_u8(qm_ptr));
+ const int32x4_t v_wt0 =
+ vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(v_wt_u16)));
+ const int32x4_t v_wt1 =
+ vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(v_wt_u16)));
+ const uint32x4_t v_mask0 = vcgeq_s32(vmulq_s32(v_abs0, v_wt0), v_zbin0);
+ const uint32x4_t v_mask1 = vcgeq_s32(vmulq_s32(v_abs1, v_wt1), v_zbin1);
+
+ if (mask_any_nonzero_u32(v_mask0, v_mask1)) {
+ const int32x4_t v_zero = vdupq_n_s32(0);
+ const int32x4_t v_sign0 = vshrq_n_s32(v_coeff0, 31);
+ const int32x4_t v_sign1 = vshrq_n_s32(v_coeff1, 31);
+ const int32x4_t v_tmp0 =
+ vandq_s32(vaddq_s32(v_abs0, v_round0), vreinterpretq_s32_u32(v_mask0));
+ const int32x4_t v_tmp1 =
+ vandq_s32(vaddq_s32(v_abs1, v_round1), vreinterpretq_s32_u32(v_mask1));
+ // Forward QM wt < 256 (AOM_QM_BITS = 5) and log_scale <= 2 fit in 32 bits.
+ const int32x4_t v_tmp_wt0 = vmulq_s32(v_tmp0, v_wt0);
+ const int32x4_t v_tmp_wt1 = vmulq_s32(v_tmp1, v_wt1);
+ const int32x4_t v_tmp2_0 =
+ vaddq_s32(vqdmulhq_s32(v_tmp_wt0, v_quant0), v_tmp_wt0);
+ const int32x4_t v_tmp2_1 =
+ vaddq_s32(vqdmulhq_s32(v_tmp_wt1, v_quant1), v_tmp_wt1);
+ const int32x4_t v_abs_q0 = vshrq_n_s32(
+ vqdmulhq_s32(vshlq_s32(v_tmp2_0, v_log_scale), v_qshift0), AOM_QM_BITS);
+ const int32x4_t v_abs_q1 = vshrq_n_s32(
+ vqdmulhq_s32(vshlq_s32(v_tmp2_1, v_log_scale), v_qshift1), AOM_QM_BITS);
+
+ const uint16x8_t v_iwt_u16 = vmovl_u8(vld1_u8(iqm_ptr));
+ const int32x4_t v_iwt0 =
+ vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(v_iwt_u16)));
+ const int32x4_t v_iwt1 =
+ vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(v_iwt_u16)));
+ const int32x4_t v_dq_rnd = vdupq_n_s32(1 << (AOM_QM_BITS - 1));
+ const int32x4_t v_dq0 =
+ vshrq_n_s32(vmlaq_s32(v_dq_rnd, v_dequant0, v_iwt0), AOM_QM_BITS);
+ const int32x4_t v_dq1 =
+ vshrq_n_s32(vmlaq_s32(v_dq_rnd, v_dequant1, v_iwt1), AOM_QM_BITS);
+ const int32x4_t v_abs_dq0 =
+ vshlq_s32(vmulq_s32(v_abs_q0, v_dq0), v_neg_log_scale);
+ const int32x4_t v_abs_dq1 =
+ vshlq_s32(vmulq_s32(v_abs_q1, v_dq1), v_neg_log_scale);
+
+ const int32x4_t v_q0 = vsubq_s32(veorq_s32(v_abs_q0, v_sign0), v_sign0);
+ const int32x4_t v_q1 = vsubq_s32(veorq_s32(v_abs_q1, v_sign1), v_sign1);
+ const int32x4_t v_dq_out0 =
+ vsubq_s32(veorq_s32(v_abs_dq0, v_sign0), v_sign0);
+ const int32x4_t v_dq_out1 =
+ vsubq_s32(veorq_s32(v_abs_dq1, v_sign1), v_sign1);
+ vst1q_s32(qcoeff_ptr, v_q0);
+ vst1q_s32(qcoeff_ptr + 4, v_q1);
+ vst1q_s32(dqcoeff_ptr, v_dq_out0);
+ vst1q_s32(dqcoeff_ptr + 4, v_dq_out1);
+
+ const uint32x4_t v_nz0 = vcgtq_s32(v_abs_q0, v_zero);
+ const uint32x4_t v_nz1 = vcgtq_s32(v_abs_q1, v_zero);
+ return update_qm_eob(iscan_ptr, v_eobmax, v_nz0, v_nz1);
+ }
+ const int32x4_t v_zero = vdupq_n_s32(0);
+ vst1q_s32(qcoeff_ptr, v_zero);
+ vst1q_s32(qcoeff_ptr + 4, v_zero);
+ vst1q_s32(dqcoeff_ptr, v_zero);
+ vst1q_s32(dqcoeff_ptr + 4, v_zero);
+ return v_eobmax;
+}
+
+void av1_highbd_quantize_b_qm_neon(
+ const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
+ const int16_t *round_ptr, const int16_t *quant_ptr,
+ const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
+ tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
+ const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr,
+ const qm_val_t *iqm_ptr, int log_scale) {
+ (void)scan;
+ assert(qm_ptr != NULL && iqm_ptr != NULL);
+ assert(n_coeffs >= 16 && (n_coeffs & 7) == 0);
+ const int32x4_t v_zbin_ac =
+ vdupq_n_s32(ROUND_POWER_OF_TWO(zbin_ptr[1], log_scale) << AOM_QM_BITS);
+ const int32x4_t v_zbin0 = vsetq_lane_s32(
+ ROUND_POWER_OF_TWO(zbin_ptr[0], log_scale) << AOM_QM_BITS, v_zbin_ac, 0);
+ const int32x4_t v_round_ac =
+ vdupq_n_s32(ROUND_POWER_OF_TWO(round_ptr[1], log_scale));
+ const int32x4_t v_round0 = vsetq_lane_s32(
+ ROUND_POWER_OF_TWO(round_ptr[0], log_scale), v_round_ac, 0);
+ const int32x4_t v_quant_ac =
+ vdupq_n_s32((int32_t)((uint32_t)quant_ptr[1] << 15));
+ const int32x4_t v_quant0 =
+ vsetq_lane_s32((int32_t)((uint32_t)quant_ptr[0] << 15), v_quant_ac, 0);
+ const int32x4_t v_qshift_ac =
+ vdupq_n_s32((int32_t)((uint32_t)quant_shift_ptr[1] << 15));
+ const int32x4_t v_qshift0 = vsetq_lane_s32(
+ (int32_t)((uint32_t)quant_shift_ptr[0] << 15), v_qshift_ac, 0);
+ const int32x4_t v_dequant_ac = vdupq_n_s32(dequant_ptr[1]);
+ const int32x4_t v_dequant0 = vsetq_lane_s32(dequant_ptr[0], v_dequant_ac, 0);
+ const int32x4_t v_log_scale = vdupq_n_s32(log_scale);
+ const int32x4_t v_neg_log_scale = vdupq_n_s32(-log_scale);
+ int16x8_t v_eobmax = vdupq_n_s16(-1);
+
+ v_eobmax = highbd_quantize_b_qm_8_neon(
+ coeff_ptr, iscan, qm_ptr, iqm_ptr, qcoeff_ptr, dqcoeff_ptr, v_zbin0,
+ v_zbin_ac, v_round0, v_round_ac, v_quant0, v_quant_ac, v_qshift0,
+ v_qshift_ac, v_dequant0, v_dequant_ac, v_log_scale, v_neg_log_scale,
+ v_eobmax);
+
+ intptr_t i = 8;
+ do {
+ v_eobmax = highbd_quantize_b_qm_8_neon(
+ coeff_ptr + i, iscan + i, qm_ptr + i, iqm_ptr + i, qcoeff_ptr + i,
+ dqcoeff_ptr + i, v_zbin_ac, v_zbin_ac, v_round_ac, v_round_ac,
+ v_quant_ac, v_quant_ac, v_qshift_ac, v_qshift_ac, v_dequant_ac,
+ v_dequant_ac, v_log_scale, v_neg_log_scale, v_eobmax);
+ i += 8;
+ } while (i != n_coeffs);
*eob_ptr = get_max_eob(v_eobmax);
}
+#endif // CONFIG_AV1_HIGHBITDEPTH
diff --git a/av1/encoder/arm/quantize_neon.c b/av1/encoder/arm/quantize_neon.c
index e9c0ca7619..e3a0f7566a 100644
--- a/av1/encoder/arm/quantize_neon.c
+++ b/av1/encoder/arm/quantize_neon.c
@@ -15,6 +15,7 @@
#include <math.h>
#include "config/aom_config.h"
+#include "config/av1_rtcd.h"
#include "aom_dsp/arm/mem_neon.h"
#include "aom_dsp/arm/sum_neon.h"
@@ -27,6 +28,20 @@
#include "av1/encoder/encoder.h"
#include "av1/encoder/rd.h"
+static AOM_FORCE_INLINE uint64_t mask_nonzero_u16(uint16x8_t v_mask) {
+#if AOM_ARCH_AARCH64
+ return vmaxvq_u32(vreinterpretq_u32_u16(v_mask));
+#else
+ return vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_mask)), 0);
+#endif
+}
+
+static AOM_FORCE_INLINE void store_zero_8_tran_low(tran_low_t *ptr) {
+ const int32x4_t zero = vdupq_n_s32(0);
+ vst1q_s32(ptr, zero);
+ vst1q_s32(ptr + 4, zero);
+}
+
static inline uint16_t get_max_eob(int16x8_t v_eobmax) {
#if AOM_ARCH_AARCH64
int16_t max_val = vmaxvq_s16(v_eobmax);
@@ -54,23 +69,29 @@ static inline int16x8_t get_max_lane_eob(const int16_t *iscan,
return vmaxq_s16(v_eobmax, v_nz_iscan);
}
-static inline uint16x8_t quantize_fp_8(const tran_low_t *coeff_ptr,
- tran_low_t *qcoeff_ptr,
- tran_low_t *dqcoeff_ptr,
- int16x8_t v_quant, int16x8_t v_dequant,
- int16x8_t v_round, int16x8_t v_zero) {
+static AOM_FORCE_INLINE int16x8_t
+quantize_fp_8(const tran_low_t *coeff_ptr, tran_low_t *qcoeff_ptr,
+ tran_low_t *dqcoeff_ptr, const int16_t *iscan, int16x8_t v_quant,
+ int16x8_t v_dequant, int16x8_t v_round, uint16x8_t v_thresh,
+ int16x8_t v_eobmax) {
const int16x8_t v_coeff = load_tran_low_to_s16q(&coeff_ptr[0]);
- const int16x8_t v_coeff_sign = vshrq_n_s16(v_coeff, 15);
const int16x8_t v_abs = vabsq_s16(v_coeff);
- const int16x8_t v_tmp = vqaddq_s16(v_abs, v_round);
- const int16x8_t v_tmp2 = vshrq_n_s16(vqdmulhq_s16(v_tmp, v_quant), 1);
- const uint16x8_t v_nz_mask = vcgtq_s16(v_tmp2, v_zero);
- const int16x8_t v_qcoeff_a = veorq_s16(v_tmp2, v_coeff_sign);
- const int16x8_t v_qcoeff = vsubq_s16(v_qcoeff_a, v_coeff_sign);
- const int16x8_t v_dqcoeff = vmulq_s16(v_qcoeff, v_dequant);
- store_s16q_to_tran_low(&qcoeff_ptr[0], v_qcoeff);
- store_s16q_to_tran_low(&dqcoeff_ptr[0], v_dqcoeff);
- return v_nz_mask;
+ const uint16x8_t v_mask = vcgeq_u16(vreinterpretq_u16_s16(v_abs), v_thresh);
+ if (mask_nonzero_u16(v_mask)) {
+ const int16x8_t v_coeff_sign = vshrq_n_s16(v_coeff, 15);
+ const int16x8_t v_tmp = vqaddq_s16(v_abs, v_round);
+ const int16x8_t v_tmp2 = vshrq_n_s16(vqdmulhq_s16(v_tmp, v_quant), 1);
+ const uint16x8_t v_nz_mask = vcgtq_s16(v_tmp2, vdupq_n_s16(0));
+ const int16x8_t v_qcoeff_a = veorq_s16(v_tmp2, v_coeff_sign);
+ const int16x8_t v_qcoeff = vsubq_s16(v_qcoeff_a, v_coeff_sign);
+ const int16x8_t v_dqcoeff = vmulq_s16(v_qcoeff, v_dequant);
+ store_s16q_to_tran_low(&qcoeff_ptr[0], v_qcoeff);
+ store_s16q_to_tran_low(&dqcoeff_ptr[0], v_dqcoeff);
+ return get_max_lane_eob(iscan, v_eobmax, v_nz_mask);
+ }
+ store_zero_8_tran_low(&qcoeff_ptr[0]);
+ store_zero_8_tran_low(&dqcoeff_ptr[0]);
+ return v_eobmax;
}
void av1_quantize_fp_neon(const tran_low_t *coeff_ptr, intptr_t count,
@@ -85,23 +106,24 @@ void av1_quantize_fp_neon(const tran_low_t *coeff_ptr, intptr_t count,
(void)zbin_ptr;
(void)quant_shift_ptr;
(void)scan;
+ assert(count >= 16 && (count & 7) == 0);
- // Quantization pass: All coefficients with index >= zero_flag are
- // skippable. Note: zero_flag can be zero.
- const int16x8_t v_zero = vdupq_n_s16(0);
int16x8_t v_quant = vld1q_s16(quant_ptr);
int16x8_t v_dequant = vld1q_s16(dequant_ptr);
int16x8_t v_round = vld1q_s16(round_ptr);
+ uint16x8_t v_thresh = vreinterpretq_u16_s16(vshrq_n_s16(v_dequant, 1));
int16x8_t v_eobmax_76543210 = vdupq_n_s16(-1);
- uint16x8_t v_nz_mask;
+
// process dc and the first seven ac coeffs
- v_nz_mask = quantize_fp_8(coeff_ptr, qcoeff_ptr, dqcoeff_ptr, v_quant,
- v_dequant, v_round, v_zero);
- v_eobmax_76543210 = get_max_lane_eob(&iscan[0], v_eobmax_76543210, v_nz_mask);
+ v_eobmax_76543210 =
+ quantize_fp_8(coeff_ptr, qcoeff_ptr, dqcoeff_ptr, iscan, v_quant,
+ v_dequant, v_round, v_thresh, v_eobmax_76543210);
+
// overwrite the dc constants with ac constants
v_quant = vdupq_lane_s16(vget_low_s16(v_quant), 1);
v_dequant = vdupq_lane_s16(vget_low_s16(v_dequant), 1);
v_round = vdupq_lane_s16(vget_low_s16(v_round), 1);
+ v_thresh = vdupq_lane_u16(vget_low_u16(v_thresh), 1);
count -= 8;
// now process the rest of the ac coeffs
@@ -110,11 +132,11 @@ void av1_quantize_fp_neon(const tran_low_t *coeff_ptr, intptr_t count,
qcoeff_ptr += 8;
dqcoeff_ptr += 8;
iscan += 8;
- v_nz_mask = quantize_fp_8(coeff_ptr, qcoeff_ptr, dqcoeff_ptr, v_quant,
- v_dequant, v_round, v_zero);
- v_eobmax_76543210 = get_max_lane_eob(iscan, v_eobmax_76543210, v_nz_mask);
+ v_eobmax_76543210 =
+ quantize_fp_8(coeff_ptr, qcoeff_ptr, dqcoeff_ptr, iscan, v_quant,
+ v_dequant, v_round, v_thresh, v_eobmax_76543210);
count -= 8;
- } while (count > 0);
+ } while (count != 0);
*eob_ptr = get_max_eob(v_eobmax_76543210);
}
@@ -177,69 +199,78 @@ void av1_quantize_lp_neon(const int16_t *coeff_ptr, intptr_t n_coeffs,
*eob_ptr = get_max_eob(v_eobmax_76543210);
}
-static AOM_FORCE_INLINE uint16x8_t quantize_fp_logscale_8(
+static AOM_FORCE_INLINE int16x8_t quantize_fp_logscale_8(
const tran_low_t *coeff_ptr, tran_low_t *qcoeff_ptr,
- tran_low_t *dqcoeff_ptr, int16x8_t v_quant, int16x8_t v_dequant,
- int16x8_t v_round, int16x8_t v_zero, int log_scale) {
- const int16x8_t v_log_scale_minus_1 = vdupq_n_s16(log_scale - 1);
- const int16x8_t v_neg_log_scale_plus_1 = vdupq_n_s16(-(1 + log_scale));
+ tran_low_t *dqcoeff_ptr, const int16_t *iscan, int16x8_t v_quant,
+ int16x8_t v_dequant, int16x8_t v_round, uint16x8_t v_thresh,
+ int16x8_t v_eobmax, int log_scale) {
const int16x8_t v_coeff = load_tran_low_to_s16q(coeff_ptr);
- const int16x8_t v_coeff_sign = vshrq_n_s16(v_coeff, 15);
const int16x8_t v_abs_coeff = vabsq_s16(v_coeff);
const uint16x8_t v_mask =
- vcgeq_s16(v_abs_coeff, vshlq_s16(v_dequant, v_neg_log_scale_plus_1));
- // const int64_t tmp = vmask ? (int64_t)abs_coeff + log_scaled_round : 0
- const int16x8_t v_tmp = vandq_s16(vqaddq_s16(v_abs_coeff, v_round),
- vreinterpretq_s16_u16(v_mask));
- const int16x8_t v_tmp2 =
- vqdmulhq_s16(vshlq_s16(v_tmp, v_log_scale_minus_1), v_quant);
- const uint16x8_t v_nz_mask = vcgtq_s16(v_tmp2, v_zero);
- const int16x8_t v_qcoeff =
- vsubq_s16(veorq_s16(v_tmp2, v_coeff_sign), v_coeff_sign);
- // Multiplying by dequant here will use all 16 bits. Cast to unsigned before
- // shifting right. (vshlq_s16 will shift right if shift value is negative)
- const uint16x8_t v_abs_dqcoeff =
- vshlq_u16(vreinterpretq_u16_s16(vmulq_s16(v_tmp2, v_dequant)),
- vdupq_n_s16(-log_scale));
- const int16x8_t v_dqcoeff =
- vsubq_s16(veorq_s16(vreinterpretq_s16_u16(v_abs_dqcoeff), v_coeff_sign),
- v_coeff_sign);
- store_s16q_to_tran_low(qcoeff_ptr, v_qcoeff);
- store_s16q_to_tran_low(dqcoeff_ptr, v_dqcoeff);
- return v_nz_mask;
+ vcgeq_u16(vreinterpretq_u16_s16(v_abs_coeff), v_thresh);
+ if (mask_nonzero_u16(v_mask)) {
+ const int16x8_t v_coeff_sign = vshrq_n_s16(v_coeff, 15);
+ // const int64_t tmp = vmask ? (int64_t)abs_coeff + log_scaled_round : 0
+ const int16x8_t v_tmp = vandq_s16(vqaddq_s16(v_abs_coeff, v_round),
+ vreinterpretq_s16_u16(v_mask));
+ const int16x8_t v_tmp2 =
+ vqdmulhq_s16(vshlq_s16(v_tmp, vdupq_n_s16(log_scale - 1)), v_quant);
+ const uint16x8_t v_nz_mask = vcgtq_s16(v_tmp2, vdupq_n_s16(0));
+ const int16x8_t v_qcoeff =
+ vsubq_s16(veorq_s16(v_tmp2, v_coeff_sign), v_coeff_sign);
+ // Multiplying by dequant here will use all 16 bits. Cast to unsigned
+ // before shifting right.
+ const uint16x8_t v_abs_dqcoeff =
+ vshlq_u16(vreinterpretq_u16_s16(vmulq_s16(v_tmp2, v_dequant)),
+ vdupq_n_s16(-log_scale));
+ const int16x8_t v_dqcoeff =
+ vsubq_s16(veorq_s16(vreinterpretq_s16_u16(v_abs_dqcoeff), v_coeff_sign),
+ v_coeff_sign);
+ store_s16q_to_tran_low(qcoeff_ptr, v_qcoeff);
+ store_s16q_to_tran_low(dqcoeff_ptr, v_dqcoeff);
+ return get_max_lane_eob(iscan, v_eobmax, v_nz_mask);
+ }
+ store_zero_8_tran_low(qcoeff_ptr);
+ store_zero_8_tran_low(dqcoeff_ptr);
+ return v_eobmax;
}
-static AOM_FORCE_INLINE uint16x8_t quantize_fp_logscale2_8(
+static AOM_FORCE_INLINE int16x8_t quantize_fp_logscale2_8(
const tran_low_t *coeff_ptr, tran_low_t *qcoeff_ptr,
- tran_low_t *dqcoeff_ptr, int16x8_t v_quant, int16x8_t v_dequant,
- int16x8_t v_round, int16x8_t v_zero) {
+ tran_low_t *dqcoeff_ptr, const int16_t *iscan, int16x8_t v_quant,
+ int16x8_t v_dequant, int16x8_t v_round, uint16x8_t v_thresh,
+ int16x8_t v_eobmax) {
const int16x8_t v_coeff = load_tran_low_to_s16q(coeff_ptr);
- const int16x8_t v_coeff_sign = vshrq_n_s16(v_coeff, 15);
const int16x8_t v_abs_coeff = vabsq_s16(v_coeff);
const uint16x8_t v_mask =
- vcgeq_u16(vshlq_n_u16(vreinterpretq_u16_s16(v_abs_coeff), 1),
- vshrq_n_u16(vreinterpretq_u16_s16(v_dequant), 2));
- // abs_coeff = vmask ? (int64_t)abs_coeff + log_scaled_round : 0
- const int16x8_t v_tmp = vandq_s16(vqaddq_s16(v_abs_coeff, v_round),
- vreinterpretq_s16_u16(v_mask));
- // tmp32 = (int)((abs_coeff * quant_ptr[rc != 0]) >> (16 - log_scale));
- const int16x8_t v_tmp2 =
- vorrq_s16(vshlq_n_s16(vqdmulhq_s16(v_tmp, v_quant), 1),
- vreinterpretq_s16_u16(vshrq_n_u16(
- vreinterpretq_u16_s16(vmulq_s16(v_tmp, v_quant)), 14)));
- const uint16x8_t v_nz_mask = vcgtq_s16(v_tmp2, v_zero);
- const int16x8_t v_qcoeff =
- vsubq_s16(veorq_s16(v_tmp2, v_coeff_sign), v_coeff_sign);
- // const tran_low_t abs_dqcoeff = (tmp32 * dequant_ptr[rc != 0]) >> log_scale;
- const int16x8_t v_abs_dqcoeff =
- vorrq_s16(vshlq_n_s16(vqdmulhq_s16(v_tmp2, v_dequant), 13),
- vreinterpretq_s16_u16(vshrq_n_u16(
- vreinterpretq_u16_s16(vmulq_s16(v_tmp2, v_dequant)), 2)));
- const int16x8_t v_dqcoeff =
- vsubq_s16(veorq_s16(v_abs_dqcoeff, v_coeff_sign), v_coeff_sign);
- store_s16q_to_tran_low(qcoeff_ptr, v_qcoeff);
- store_s16q_to_tran_low(dqcoeff_ptr, v_dqcoeff);
- return v_nz_mask;
+ vcgeq_u16(vreinterpretq_u16_s16(v_abs_coeff), v_thresh);
+ if (mask_nonzero_u16(v_mask)) {
+ const int16x8_t v_coeff_sign = vshrq_n_s16(v_coeff, 15);
+ // abs_coeff = vmask ? (int64_t)abs_coeff + log_scaled_round : 0
+ const int16x8_t v_tmp = vandq_s16(vqaddq_s16(v_abs_coeff, v_round),
+ vreinterpretq_s16_u16(v_mask));
+ // tmp32 = (int)((abs_coeff * quant_ptr[rc != 0]) >> (16 - log_scale));
+ const int16x8_t v_tmp2 =
+ vorrq_s16(vshlq_n_s16(vqdmulhq_s16(v_tmp, v_quant), 1),
+ vreinterpretq_s16_u16(vshrq_n_u16(
+ vreinterpretq_u16_s16(vmulq_s16(v_tmp, v_quant)), 14)));
+ const uint16x8_t v_nz_mask = vcgtq_s16(v_tmp2, vdupq_n_s16(0));
+ const int16x8_t v_qcoeff =
+ vsubq_s16(veorq_s16(v_tmp2, v_coeff_sign), v_coeff_sign);
+ // abs_dqcoeff = (tmp32 * dequant_ptr[rc != 0]) >> log_scale;
+ const int16x8_t v_abs_dqcoeff =
+ vorrq_s16(vshlq_n_s16(vqdmulhq_s16(v_tmp2, v_dequant), 13),
+ vreinterpretq_s16_u16(vshrq_n_u16(
+ vreinterpretq_u16_s16(vmulq_s16(v_tmp2, v_dequant)), 2)));
+ const int16x8_t v_dqcoeff =
+ vsubq_s16(veorq_s16(v_abs_dqcoeff, v_coeff_sign), v_coeff_sign);
+ store_s16q_to_tran_low(qcoeff_ptr, v_qcoeff);
+ store_s16q_to_tran_low(dqcoeff_ptr, v_dqcoeff);
+ return get_max_lane_eob(iscan, v_eobmax, v_nz_mask);
+ }
+ store_zero_8_tran_low(qcoeff_ptr);
+ store_zero_8_tran_low(dqcoeff_ptr);
+ return v_eobmax;
}
static AOM_FORCE_INLINE void quantize_fp_no_qmatrix_neon(
@@ -247,7 +278,7 @@ static AOM_FORCE_INLINE void quantize_fp_no_qmatrix_neon(
const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *iscan,
int log_scale) {
- const int16x8_t v_zero = vdupq_n_s16(0);
+ assert(n_coeffs > 16 && (n_coeffs & 15) == 0);
int16x8_t v_quant = vld1q_s16(quant_ptr);
int16x8_t v_dequant = vld1q_s16(dequant_ptr);
const int16x8_t v_round_no_scale = vld1q_s16(round_ptr);
@@ -256,66 +287,64 @@ static AOM_FORCE_INLINE void quantize_fp_no_qmatrix_neon(
int16x8_t v_eobmax_76543210 = vdupq_n_s16(-1);
intptr_t non_zero_count = n_coeffs;
- assert(n_coeffs > 16);
- // Pre-scan pass
- const int16x8_t v_dequant_scaled =
- vshlq_s16(v_dequant, vdupq_n_s16(-(1 + log_scale)));
- const int16x8_t v_zbin_s16 =
- vdupq_lane_s16(vget_low_s16(v_dequant_scaled), 1);
- intptr_t i = n_coeffs;
+ // Pre-scan pass: compute exact ceil(dequant / 2^(1 + log_scale)) threshold.
+ const uint16x8_t v_bias = vdupq_n_u16((uint16_t)((1 << (1 + log_scale)) - 1));
+ uint16x8_t v_thresh =
+ vshlq_u16(vaddq_u16(vreinterpretq_u16_s16(v_dequant), v_bias),
+ vdupq_n_s16(-(1 + log_scale)));
+ const uint16x8_t v_zbin_u16 = vdupq_lane_u16(vget_low_u16(v_thresh), 1);
do {
- const int16x8_t v_coeff_a = load_tran_low_to_s16q(coeff_ptr + i - 8);
- const int16x8_t v_coeff_b = load_tran_low_to_s16q(coeff_ptr + i - 16);
- const int16x8_t v_abs_coeff_a = vabsq_s16(v_coeff_a);
- const int16x8_t v_abs_coeff_b = vabsq_s16(v_coeff_b);
- const uint16x8_t v_mask_a = vcgeq_s16(v_abs_coeff_a, v_zbin_s16);
- const uint16x8_t v_mask_b = vcgeq_s16(v_abs_coeff_b, v_zbin_s16);
- // If the coefficient is in the base ZBIN range, then discard.
- if (horizontal_long_add_u16x8(v_mask_a, v_mask_b) == 0) {
- non_zero_count -= 16;
- } else {
+ const int16x8_t v_coeff_a =
+ load_tran_low_to_s16q(coeff_ptr + non_zero_count - 8);
+ const int16x8_t v_coeff_b =
+ load_tran_low_to_s16q(coeff_ptr + non_zero_count - 16);
+ const uint16x8_t v_abs_a = vreinterpretq_u16_s16(vabsq_s16(v_coeff_a));
+ const uint16x8_t v_abs_b = vreinterpretq_u16_s16(vabsq_s16(v_coeff_b));
+ const uint16x8_t v_mask_a = vcgeq_u16(v_abs_a, v_zbin_u16);
+ const uint16x8_t v_mask_b = vcgeq_u16(v_abs_b, v_zbin_u16);
+ if (mask_nonzero_u16(vorrq_u16(v_mask_a, v_mask_b))) {
break;
}
- i -= 16;
- } while (i > 0);
-
- const intptr_t remaining_zcoeffs = n_coeffs - non_zero_count;
- memset(qcoeff_ptr + non_zero_count, 0,
- remaining_zcoeffs * sizeof(*qcoeff_ptr));
- memset(dqcoeff_ptr + non_zero_count, 0,
- remaining_zcoeffs * sizeof(*dqcoeff_ptr));
+ non_zero_count -= 16;
+ store_zero_8_tran_low(qcoeff_ptr + non_zero_count);
+ store_zero_8_tran_low(qcoeff_ptr + non_zero_count + 8);
+ store_zero_8_tran_low(dqcoeff_ptr + non_zero_count);
+ store_zero_8_tran_low(dqcoeff_ptr + non_zero_count + 8);
+ } while (non_zero_count != 16);
// process dc and the first seven ac coeffs
- uint16x8_t v_nz_mask;
if (log_scale == 2) {
- v_nz_mask = quantize_fp_logscale2_8(coeff_ptr, qcoeff_ptr, dqcoeff_ptr,
- v_quant, v_dequant, v_round, v_zero);
+ v_eobmax_76543210 = quantize_fp_logscale2_8(
+ coeff_ptr, qcoeff_ptr, dqcoeff_ptr, iscan, v_quant, v_dequant, v_round,
+ v_thresh, v_eobmax_76543210);
} else {
- v_nz_mask =
- quantize_fp_logscale_8(coeff_ptr, qcoeff_ptr, dqcoeff_ptr, v_quant,
- v_dequant, v_round, v_zero, log_scale);
+ v_eobmax_76543210 = quantize_fp_logscale_8(
+ coeff_ptr, qcoeff_ptr, dqcoeff_ptr, iscan, v_quant, v_dequant, v_round,
+ v_thresh, v_eobmax_76543210, log_scale);
}
- v_eobmax_76543210 = get_max_lane_eob(iscan, v_eobmax_76543210, v_nz_mask);
// overwrite the dc constants with ac constants
v_quant = vdupq_lane_s16(vget_low_s16(v_quant), 1);
v_dequant = vdupq_lane_s16(vget_low_s16(v_dequant), 1);
v_round = vdupq_lane_s16(vget_low_s16(v_round), 1);
+ v_thresh = v_zbin_u16;
- for (intptr_t count = non_zero_count - 8; count > 0; count -= 8) {
+ intptr_t count = non_zero_count - 8;
+ do {
coeff_ptr += 8;
qcoeff_ptr += 8;
dqcoeff_ptr += 8;
iscan += 8;
if (log_scale == 2) {
- v_nz_mask = quantize_fp_logscale2_8(coeff_ptr, qcoeff_ptr, dqcoeff_ptr,
- v_quant, v_dequant, v_round, v_zero);
+ v_eobmax_76543210 = quantize_fp_logscale2_8(
+ coeff_ptr, qcoeff_ptr, dqcoeff_ptr, iscan, v_quant, v_dequant,
+ v_round, v_thresh, v_eobmax_76543210);
} else {
- v_nz_mask =
- quantize_fp_logscale_8(coeff_ptr, qcoeff_ptr, dqcoeff_ptr, v_quant,
- v_dequant, v_round, v_zero, log_scale);
+ v_eobmax_76543210 = quantize_fp_logscale_8(
+ coeff_ptr, qcoeff_ptr, dqcoeff_ptr, iscan, v_quant, v_dequant,
+ v_round, v_thresh, v_eobmax_76543210, log_scale);
}
- v_eobmax_76543210 = get_max_lane_eob(iscan, v_eobmax_76543210, v_nz_mask);
- }
+ count -= 8;
+ } while (count != 0);
*eob_ptr = get_max_eob(v_eobmax_76543210);
}
@@ -385,12 +414,11 @@ void aom_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr,
uint16_t *eob_ptr, const int16_t *scan,
const int16_t *iscan) {
- (void)quant_shift_ptr;
(void)scan;
int16x8_t v_eobmax_76543210 = vdupq_n_s16(-1);
- int16x8_t v_zbins = vdupq_n_s16(zbin_ptr[1]);
+ const int16x8_t v_zbins = vdupq_n_s16(zbin_ptr[1]);
int16x8_t v_round = vdupq_n_s16(round_ptr[1]);
int16x8_t v_dequant = vdupq_n_s16(dequant_ptr[1]);
int16x8_t v_quant = vdupq_n_s16(quant_ptr[1]);
@@ -402,34 +430,37 @@ void aom_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
int16x8_t v_abs = vabsq_s16(v_coeff);
uint16x8_t v_cond = vcgeq_s16(v_abs, v_zbins0);
- uint64_t nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_cond)), 0);
- if (nz_check) {
- int16x8_t v_round0 = vsetq_lane_s16(round_ptr[0], v_round, 0);
- int16x8_t v_quant0 = vsetq_lane_s16(quant_ptr[0], v_quant, 0);
- int16x8_t v_dequant0 = vsetq_lane_s16(dequant_ptr[0], v_dequant, 0);
+ if (mask_nonzero_u16(v_cond)) {
+ v_round = vsetq_lane_s16(round_ptr[0], v_round, 0);
+ v_quant = vsetq_lane_s16(quant_ptr[0], v_quant, 0);
+ v_dequant = vsetq_lane_s16(dequant_ptr[0], v_dequant, 0);
// Shift by 1 in order to save one shift in the kernel function.
- int16x8_t v_quant_shift0 =
- vsetq_lane_s16(quant_shift_ptr[0] >> 1, v_quant_shift, 0);
+ v_quant_shift = vsetq_lane_s16(quant_shift_ptr[0] >> 1, v_quant_shift, 0);
- const uint16x8_t v_nz_mask = quantize_b_logscale0_8(
- v_coeff, v_abs, v_cond, v_round0, v_dequant0, v_quant0, v_quant_shift0,
- qcoeff_ptr, dqcoeff_ptr);
+ const uint16x8_t v_nz_mask =
+ quantize_b_logscale0_8(v_coeff, v_abs, v_cond, v_round, v_dequant,
+ v_quant, v_quant_shift, qcoeff_ptr, dqcoeff_ptr);
+
+ v_round = vdupq_lane_s16(vget_low_s16(v_round), 1);
+ v_quant = vdupq_lane_s16(vget_low_s16(v_quant), 1);
+ v_dequant = vdupq_lane_s16(vget_low_s16(v_dequant), 1);
+ v_quant_shift = vdupq_lane_s16(vget_low_s16(v_quant_shift), 1);
int16x8_t v_iscan = vld1q_s16(iscan);
int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210);
v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210);
} else {
- store_s16q_to_tran_low(qcoeff_ptr, vdupq_n_s16(0));
- store_s16q_to_tran_low(dqcoeff_ptr, vdupq_n_s16(0));
+ store_zero_8_tran_low(qcoeff_ptr);
+ store_zero_8_tran_low(dqcoeff_ptr);
}
- for (int i = 8; i < n_coeffs; i += 8) {
+ intptr_t i = 8;
+ do {
v_coeff = load_tran_low_to_s16q(coeff_ptr + i);
v_abs = vabsq_s16(v_coeff);
v_cond = vcgeq_s16(v_abs, v_zbins);
- nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_cond)), 0);
- if (nz_check) {
+ if (mask_nonzero_u16(v_cond)) {
const uint16x8_t v_nz_mask = quantize_b_logscale0_8(
v_coeff, v_abs, v_cond, v_round, v_dequant, v_quant, v_quant_shift,
qcoeff_ptr + i, dqcoeff_ptr + i);
@@ -438,10 +469,11 @@ void aom_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210);
v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210);
} else {
- store_s16q_to_tran_low(qcoeff_ptr + i, vdupq_n_s16(0));
- store_s16q_to_tran_low(dqcoeff_ptr + i, vdupq_n_s16(0));
+ store_zero_8_tran_low(qcoeff_ptr + i);
+ store_zero_8_tran_low(dqcoeff_ptr + i);
}
- }
+ i += 8;
+ } while (i != n_coeffs);
*eob_ptr = get_max_eob(v_eobmax_76543210);
}
@@ -943,44 +975,66 @@ void aom_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
int16x8_t v_eobmax_76543210 = vdupq_n_s16(-1);
- int16x8_t v_zbins = vdupq_n_s16(zbins[1]);
+ const int16x8_t v_zbins = vdupq_n_s16(zbins[1]);
int16x8_t v_round = vdupq_n_s16(rounds[1]);
int16x8_t v_dequant = vdupq_n_s16(dequant_ptr[1]);
int16x8_t v_quant = vdupq_n_s16(quant_ptr[1]);
int16x8_t v_quant_shift = vdupq_n_s16(quant_shift_ptr[1]);
+ assert(n_coeffs > 16 && (n_coeffs & 15) == 0);
+ intptr_t non_zero_count = n_coeffs;
+ do {
+ const int16x8_t v_coeff_a =
+ load_tran_low_to_s16q(coeff_ptr + non_zero_count - 8);
+ const int16x8_t v_coeff_b =
+ load_tran_low_to_s16q(coeff_ptr + non_zero_count - 16);
+ const uint16x8_t v_mask_a = vcgeq_s16(vabsq_s16(v_coeff_a), v_zbins);
+ const uint16x8_t v_mask_b = vcgeq_s16(vabsq_s16(v_coeff_b), v_zbins);
+ if (mask_nonzero_u16(vorrq_u16(v_mask_a, v_mask_b))) {
+ break;
+ }
+ non_zero_count -= 16;
+ store_zero_8_tran_low(qcoeff_ptr + non_zero_count);
+ store_zero_8_tran_low(qcoeff_ptr + non_zero_count + 8);
+ store_zero_8_tran_low(dqcoeff_ptr + non_zero_count);
+ store_zero_8_tran_low(dqcoeff_ptr + non_zero_count + 8);
+ } while (non_zero_count != 16);
+
int16x8_t v_zbins0 = vsetq_lane_s16(zbins[0], v_zbins, 0);
int16x8_t v_coeff = load_tran_low_to_s16q(coeff_ptr);
int16x8_t v_abs = vabsq_s16(v_coeff);
uint16x8_t v_cond = vcgeq_s16(v_abs, v_zbins0);
- uint64_t nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_cond)), 0);
- if (nz_check) {
- int16x8_t v_round0 = vsetq_lane_s16(rounds[0], v_round, 0);
- int16x8_t v_quant0 = vsetq_lane_s16(quant_ptr[0], v_quant, 0);
- int16x8_t v_dequant0 = vsetq_lane_s16(dequant_ptr[0], v_dequant, 0);
- int16x8_t v_quant_shift0 =
- vsetq_lane_s16(quant_shift_ptr[0], v_quant_shift, 0);
+ if (mask_nonzero_u16(v_cond)) {
+ v_round = vsetq_lane_s16(rounds[0], v_round, 0);
+ v_quant = vsetq_lane_s16(quant_ptr[0], v_quant, 0);
+ v_dequant = vsetq_lane_s16(dequant_ptr[0], v_dequant, 0);
+ v_quant_shift = vsetq_lane_s16(quant_shift_ptr[0], v_quant_shift, 0);
+
+ const uint16x8_t v_nz_mask =
+ quantize_b_logscale1_8(v_coeff, v_abs, v_cond, v_round, v_dequant,
+ v_quant, v_quant_shift, qcoeff_ptr, dqcoeff_ptr);
- const uint16x8_t v_nz_mask = quantize_b_logscale1_8(
- v_coeff, v_abs, v_cond, v_round0, v_dequant0, v_quant0, v_quant_shift0,
- qcoeff_ptr, dqcoeff_ptr);
+ v_round = vdupq_lane_s16(vget_low_s16(v_round), 1);
+ v_quant = vdupq_lane_s16(vget_low_s16(v_quant), 1);
+ v_dequant = vdupq_lane_s16(vget_low_s16(v_dequant), 1);
+ v_quant_shift = vdupq_lane_s16(vget_low_s16(v_quant_shift), 1);
int16x8_t v_iscan = vld1q_s16(iscan);
int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210);
v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210);
} else {
- store_s16q_to_tran_low(qcoeff_ptr, vdupq_n_s16(0));
- store_s16q_to_tran_low(dqcoeff_ptr, vdupq_n_s16(0));
+ store_zero_8_tran_low(qcoeff_ptr);
+ store_zero_8_tran_low(dqcoeff_ptr);
}
- for (int i = 8; i < n_coeffs; i += 8) {
+ intptr_t i = 8;
+ do {
v_coeff = load_tran_low_to_s16q(coeff_ptr + i);
v_abs = vabsq_s16(v_coeff);
v_cond = vcgeq_s16(v_abs, v_zbins);
- nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_cond)), 0);
- if (nz_check) {
+ if (mask_nonzero_u16(v_cond)) {
const uint16x8_t v_nz_mask = quantize_b_logscale1_8(
v_coeff, v_abs, v_cond, v_round, v_dequant, v_quant, v_quant_shift,
qcoeff_ptr + i, dqcoeff_ptr + i);
@@ -989,10 +1043,11 @@ void aom_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210);
v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210);
} else {
- store_s16q_to_tran_low(qcoeff_ptr + i, vdupq_n_s16(0));
- store_s16q_to_tran_low(dqcoeff_ptr + i, vdupq_n_s16(0));
+ store_zero_8_tran_low(qcoeff_ptr + i);
+ store_zero_8_tran_low(dqcoeff_ptr + i);
}
- }
+ i += 8;
+ } while (i != non_zero_count);
*eob_ptr = get_max_eob(v_eobmax_76543210);
}
@@ -1047,44 +1102,66 @@ void aom_quantize_b_64x64_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
int16x8_t v_eobmax_76543210 = vdupq_n_s16(-1);
- int16x8_t v_zbins = vdupq_n_s16(zbins[1]);
+ const int16x8_t v_zbins = vdupq_n_s16(zbins[1]);
int16x8_t v_round = vdupq_n_s16(rounds[1]);
int16x8_t v_dequant = vdupq_n_s16(dequant_ptr[1]);
int16x8_t v_quant = vdupq_n_s16(quant_ptr[1]);
int16x8_t v_quant_shift = vdupq_n_s16(quant_shift_ptr[1]);
+ assert(n_coeffs > 16 && (n_coeffs & 15) == 0);
+ intptr_t non_zero_count = n_coeffs;
+ do {
+ const int16x8_t v_coeff_a =
+ load_tran_low_to_s16q(coeff_ptr + non_zero_count - 8);
+ const int16x8_t v_coeff_b =
+ load_tran_low_to_s16q(coeff_ptr + non_zero_count - 16);
+ const uint16x8_t v_mask_a = vcgeq_s16(vabsq_s16(v_coeff_a), v_zbins);
+ const uint16x8_t v_mask_b = vcgeq_s16(vabsq_s16(v_coeff_b), v_zbins);
+ if (mask_nonzero_u16(vorrq_u16(v_mask_a, v_mask_b))) {
+ break;
+ }
+ non_zero_count -= 16;
+ store_zero_8_tran_low(qcoeff_ptr + non_zero_count);
+ store_zero_8_tran_low(qcoeff_ptr + non_zero_count + 8);
+ store_zero_8_tran_low(dqcoeff_ptr + non_zero_count);
+ store_zero_8_tran_low(dqcoeff_ptr + non_zero_count + 8);
+ } while (non_zero_count != 16);
+
int16x8_t v_zbins0 = vsetq_lane_s16(zbins[0], v_zbins, 0);
int16x8_t v_coeff = load_tran_low_to_s16q(coeff_ptr);
int16x8_t v_abs = vabsq_s16(v_coeff);
uint16x8_t v_cond = vcgeq_s16(v_abs, v_zbins0);
- uint64_t nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_cond)), 0);
- if (nz_check) {
- int16x8_t v_round0 = vsetq_lane_s16(rounds[0], v_round, 0);
- int16x8_t v_quant0 = vsetq_lane_s16(quant_ptr[0], v_quant, 0);
- int16x8_t v_dequant0 = vsetq_lane_s16(dequant_ptr[0], v_dequant, 0);
- int16x8_t v_quant_shift0 =
- vsetq_lane_s16(quant_shift_ptr[0], v_quant_shift, 0);
+ if (mask_nonzero_u16(v_cond)) {
+ v_round = vsetq_lane_s16(rounds[0], v_round, 0);
+ v_quant = vsetq_lane_s16(quant_ptr[0], v_quant, 0);
+ v_dequant = vsetq_lane_s16(dequant_ptr[0], v_dequant, 0);
+ v_quant_shift = vsetq_lane_s16(quant_shift_ptr[0], v_quant_shift, 0);
+
+ const uint16x8_t v_nz_mask =
+ quantize_b_logscale2_8(v_coeff, v_abs, v_cond, v_round, v_dequant,
+ v_quant, v_quant_shift, qcoeff_ptr, dqcoeff_ptr);
- const uint16x8_t v_nz_mask = quantize_b_logscale2_8(
- v_coeff, v_abs, v_cond, v_round0, v_dequant0, v_quant0, v_quant_shift0,
- qcoeff_ptr, dqcoeff_ptr);
+ v_round = vdupq_lane_s16(vget_low_s16(v_round), 1);
+ v_quant = vdupq_lane_s16(vget_low_s16(v_quant), 1);
+ v_dequant = vdupq_lane_s16(vget_low_s16(v_dequant), 1);
+ v_quant_shift = vdupq_lane_s16(vget_low_s16(v_quant_shift), 1);
int16x8_t v_iscan = vld1q_s16(iscan);
int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210);
v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210);
} else {
- store_s16q_to_tran_low(qcoeff_ptr, vdupq_n_s16(0));
- store_s16q_to_tran_low(dqcoeff_ptr, vdupq_n_s16(0));
+ store_zero_8_tran_low(qcoeff_ptr);
+ store_zero_8_tran_low(dqcoeff_ptr);
}
- for (int i = 8; i < n_coeffs; i += 8) {
+ intptr_t i = 8;
+ do {
v_coeff = load_tran_low_to_s16q(coeff_ptr + i);
v_abs = vabsq_s16(v_coeff);
v_cond = vcgeq_s16(v_abs, v_zbins);
- nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_cond)), 0);
- if (nz_check) {
+ if (mask_nonzero_u16(v_cond)) {
const uint16x8_t v_nz_mask = quantize_b_logscale2_8(
v_coeff, v_abs, v_cond, v_round, v_dequant, v_quant, v_quant_shift,
qcoeff_ptr + i, dqcoeff_ptr + i);
@@ -1093,9 +1170,286 @@ void aom_quantize_b_64x64_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210);
v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210);
} else {
- store_s16q_to_tran_low(qcoeff_ptr + i, vdupq_n_s16(0));
- store_s16q_to_tran_low(dqcoeff_ptr + i, vdupq_n_s16(0));
+ store_zero_8_tran_low(qcoeff_ptr + i);
+ store_zero_8_tran_low(dqcoeff_ptr + i);
}
- }
+ i += 8;
+ } while (i != non_zero_count);
*eob_ptr = get_max_eob(v_eobmax_76543210);
}
+
+static AOM_FORCE_INLINE int mask_any_nonzero_u32(uint32x4_t m0, uint32x4_t m1) {
+ const uint32x4_t any = vorrq_u32(m0, m1);
+#if AOM_ARCH_AARCH64
+ return vmaxvq_u32(any) != 0;
+#else
+ const uint64x2_t u64 = vreinterpretq_u64_u32(any);
+ return (vgetq_lane_u64(u64, 0) | vgetq_lane_u64(u64, 1)) != 0;
+#endif
+}
+
+static AOM_FORCE_INLINE int16x8_t update_qm_eob(const int16_t *iscan_ptr,
+ int16x8_t v_eobmax,
+ uint32x4_t nz_mask0,
+ uint32x4_t nz_mask1) {
+#if AOM_ARCH_AARCH64
+ const uint16x8_t v_nz_mask = vuzp1q_u16(vreinterpretq_u16_u32(nz_mask0),
+ vreinterpretq_u16_u32(nz_mask1));
+#else
+ const uint16x8_t v_nz_mask =
+ vcombine_u16(vmovn_u32(nz_mask0), vmovn_u32(nz_mask1));
+#endif
+ return get_max_lane_eob(iscan_ptr, v_eobmax, v_nz_mask);
+}
+
+static AOM_FORCE_INLINE int16x8_t quantize_fp_qm_8_neon(
+ const tran_low_t *coeff_ptr, const int16_t *iscan_ptr,
+ const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, tran_low_t *qcoeff_ptr,
+ tran_low_t *dqcoeff_ptr, int32x4_t v_thr0, int32x4_t v_thr1,
+ int32x4_t v_round0, int32x4_t v_round1, int32x4_t v_quant0,
+ int32x4_t v_quant1, int32x4_t v_dequant0, int32x4_t v_dequant1,
+ int32x4_t v_log_scale, int32x4_t v_neg_log_scale, int16x8_t v_eobmax) {
+ const int32x4_t v_coeff0 = vld1q_s32(coeff_ptr);
+ const int32x4_t v_coeff1 = vld1q_s32(coeff_ptr + 4);
+ const int32x4_t v_abs0 = vabsq_s32(v_coeff0);
+ const int32x4_t v_abs1 = vabsq_s32(v_coeff1);
+ const uint16x8_t v_wt_u16 = vmovl_u8(vld1_u8(qm_ptr));
+ const int32x4_t v_wt0 =
+ vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(v_wt_u16)));
+ const int32x4_t v_wt1 =
+ vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(v_wt_u16)));
+ const uint32x4_t v_mask0 = vcgeq_s32(vmulq_s32(v_abs0, v_wt0), v_thr0);
+ const uint32x4_t v_mask1 = vcgeq_s32(vmulq_s32(v_abs1, v_wt1), v_thr1);
+
+ if (mask_any_nonzero_u32(v_mask0, v_mask1)) {
+ const int32x4_t v_zero = vdupq_n_s32(0);
+ const int32x4_t v_max_s16 = vdupq_n_s32(INT16_MAX);
+ const int32x4_t v_sign0 = vshrq_n_s32(v_coeff0, 31);
+ const int32x4_t v_sign1 = vshrq_n_s32(v_coeff1, 31);
+ const int32x4_t v_tmp0 =
+ vandq_s32(vminq_s32(vaddq_s32(v_abs0, v_round0), v_max_s16),
+ vreinterpretq_s32_u32(v_mask0));
+ const int32x4_t v_tmp1 =
+ vandq_s32(vminq_s32(vaddq_s32(v_abs1, v_round1), v_max_s16),
+ vreinterpretq_s32_u32(v_mask1));
+ const int32x4_t v_tmp_wt0 = vmulq_s32(v_tmp0, v_wt0);
+ const int32x4_t v_tmp_wt1 = vmulq_s32(v_tmp1, v_wt1);
+ const int32x4_t v_abs_q0 = vshrq_n_s32(
+ vqdmulhq_s32(vshlq_s32(v_tmp_wt0, v_log_scale), v_quant0), AOM_QM_BITS);
+ const int32x4_t v_abs_q1 = vshrq_n_s32(
+ vqdmulhq_s32(vshlq_s32(v_tmp_wt1, v_log_scale), v_quant1), AOM_QM_BITS);
+
+ const uint16x8_t v_iwt_u16 = vmovl_u8(vld1_u8(iqm_ptr));
+ const int32x4_t v_iwt0 =
+ vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(v_iwt_u16)));
+ const int32x4_t v_iwt1 =
+ vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(v_iwt_u16)));
+ const int32x4_t v_dq_rnd = vdupq_n_s32(1 << (AOM_QM_BITS - 1));
+ const int32x4_t v_dq0 =
+ vshrq_n_s32(vmlaq_s32(v_dq_rnd, v_dequant0, v_iwt0), AOM_QM_BITS);
+ const int32x4_t v_dq1 =
+ vshrq_n_s32(vmlaq_s32(v_dq_rnd, v_dequant1, v_iwt1), AOM_QM_BITS);
+ const int32x4_t v_abs_dq0 =
+ vshlq_s32(vmulq_s32(v_abs_q0, v_dq0), v_neg_log_scale);
+ const int32x4_t v_abs_dq1 =
+ vshlq_s32(vmulq_s32(v_abs_q1, v_dq1), v_neg_log_scale);
+
+ const int32x4_t v_q0 = vsubq_s32(veorq_s32(v_abs_q0, v_sign0), v_sign0);
+ const int32x4_t v_q1 = vsubq_s32(veorq_s32(v_abs_q1, v_sign1), v_sign1);
+ const int32x4_t v_dq_out0 =
+ vsubq_s32(veorq_s32(v_abs_dq0, v_sign0), v_sign0);
+ const int32x4_t v_dq_out1 =
+ vsubq_s32(veorq_s32(v_abs_dq1, v_sign1), v_sign1);
+ vst1q_s32(qcoeff_ptr, v_q0);
+ vst1q_s32(qcoeff_ptr + 4, v_q1);
+ vst1q_s32(dqcoeff_ptr, v_dq_out0);
+ vst1q_s32(dqcoeff_ptr + 4, v_dq_out1);
+
+ const uint32x4_t v_nz0 = vcgtq_s32(v_abs_q0, v_zero);
+ const uint32x4_t v_nz1 = vcgtq_s32(v_abs_q1, v_zero);
+ return update_qm_eob(iscan_ptr, v_eobmax, v_nz0, v_nz1);
+ }
+ const int32x4_t v_zero = vdupq_n_s32(0);
+ vst1q_s32(qcoeff_ptr, v_zero);
+ vst1q_s32(qcoeff_ptr + 4, v_zero);
+ vst1q_s32(dqcoeff_ptr, v_zero);
+ vst1q_s32(dqcoeff_ptr + 4, v_zero);
+ return v_eobmax;
+}
+
+void av1_quantize_fp_qm_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
+ const int16_t *round_ptr, const int16_t *quant_ptr,
+ tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
+ const int16_t *dequant_ptr, uint16_t *eob_ptr,
+ const int16_t *scan, const int16_t *iscan,
+ const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr,
+ int log_scale) {
+ (void)scan;
+ assert(qm_ptr != NULL && iqm_ptr != NULL);
+ assert(n_coeffs >= 16 && (n_coeffs & 7) == 0);
+ const int thr_shift = AOM_QM_BITS - (1 + log_scale);
+ const int32x4_t v_thr_ac = vdupq_n_s32((int32_t)dequant_ptr[1] << thr_shift);
+ const int32x4_t v_thr0 =
+ vsetq_lane_s32((int32_t)dequant_ptr[0] << thr_shift, v_thr_ac, 0);
+ const int32x4_t v_round_ac =
+ vdupq_n_s32(ROUND_POWER_OF_TWO(round_ptr[1], log_scale));
+ const int32x4_t v_round0 = vsetq_lane_s32(
+ ROUND_POWER_OF_TWO(round_ptr[0], log_scale), v_round_ac, 0);
+ const int32x4_t v_quant_ac =
+ vdupq_n_s32((int32_t)((uint32_t)quant_ptr[1] << 15));
+ const int32x4_t v_quant0 =
+ vsetq_lane_s32((int32_t)((uint32_t)quant_ptr[0] << 15), v_quant_ac, 0);
+ const int32x4_t v_dequant_ac = vdupq_n_s32(dequant_ptr[1]);
+ const int32x4_t v_dequant0 = vsetq_lane_s32(dequant_ptr[0], v_dequant_ac, 0);
+ const int32x4_t v_log_scale = vdupq_n_s32(log_scale);
+ const int32x4_t v_neg_log_scale = vdupq_n_s32(-log_scale);
+ int16x8_t v_eobmax = vdupq_n_s16(-1);
+
+ v_eobmax = quantize_fp_qm_8_neon(
+ coeff_ptr, iscan, qm_ptr, iqm_ptr, qcoeff_ptr, dqcoeff_ptr, v_thr0,
+ v_thr_ac, v_round0, v_round_ac, v_quant0, v_quant_ac, v_dequant0,
+ v_dequant_ac, v_log_scale, v_neg_log_scale, v_eobmax);
+
+ intptr_t i = 8;
+ do {
+ v_eobmax = quantize_fp_qm_8_neon(
+ coeff_ptr + i, iscan + i, qm_ptr + i, iqm_ptr + i, qcoeff_ptr + i,
+ dqcoeff_ptr + i, v_thr_ac, v_thr_ac, v_round_ac, v_round_ac, v_quant_ac,
+ v_quant_ac, v_dequant_ac, v_dequant_ac, v_log_scale, v_neg_log_scale,
+ v_eobmax);
+ i += 8;
+ } while (i != n_coeffs);
+ *eob_ptr = get_max_eob(v_eobmax);
+}
+
+static AOM_FORCE_INLINE int16x8_t quantize_b_qm_8_neon(
+ const tran_low_t *coeff_ptr, const int16_t *iscan_ptr,
+ const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, tran_low_t *qcoeff_ptr,
+ tran_low_t *dqcoeff_ptr, int32x4_t v_zbin0, int32x4_t v_zbin1,
+ int32x4_t v_round0, int32x4_t v_round1, int32x4_t v_quant0,
+ int32x4_t v_quant1, int32x4_t v_qshift0, int32x4_t v_qshift1,
+ int32x4_t v_dequant0, int32x4_t v_dequant1, int32x4_t v_log_scale,
+ int32x4_t v_neg_log_scale, int16x8_t v_eobmax) {
+ const int32x4_t v_coeff0 = vld1q_s32(coeff_ptr);
+ const int32x4_t v_coeff1 = vld1q_s32(coeff_ptr + 4);
+ const int32x4_t v_abs0 = vabsq_s32(v_coeff0);
+ const int32x4_t v_abs1 = vabsq_s32(v_coeff1);
+ const uint16x8_t v_wt_u16 = vmovl_u8(vld1_u8(qm_ptr));
+ const int32x4_t v_wt0 =
+ vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(v_wt_u16)));
+ const int32x4_t v_wt1 =
+ vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(v_wt_u16)));
+ const uint32x4_t v_mask0 = vcgeq_s32(vmulq_s32(v_abs0, v_wt0), v_zbin0);
+ const uint32x4_t v_mask1 = vcgeq_s32(vmulq_s32(v_abs1, v_wt1), v_zbin1);
+
+ if (mask_any_nonzero_u32(v_mask0, v_mask1)) {
+ const int32x4_t v_zero = vdupq_n_s32(0);
+ const int32x4_t v_max_s16 = vdupq_n_s32(INT16_MAX);
+ const int32x4_t v_sign0 = vshrq_n_s32(v_coeff0, 31);
+ const int32x4_t v_sign1 = vshrq_n_s32(v_coeff1, 31);
+ const int32x4_t v_tmp0 =
+ vandq_s32(vminq_s32(vaddq_s32(v_abs0, v_round0), v_max_s16),
+ vreinterpretq_s32_u32(v_mask0));
+ const int32x4_t v_tmp1 =
+ vandq_s32(vminq_s32(vaddq_s32(v_abs1, v_round1), v_max_s16),
+ vreinterpretq_s32_u32(v_mask1));
+ const int32x4_t v_tmp_wt0 = vmulq_s32(v_tmp0, v_wt0);
+ const int32x4_t v_tmp_wt1 = vmulq_s32(v_tmp1, v_wt1);
+ const int32x4_t v_tmp2_0 =
+ vaddq_s32(vqdmulhq_s32(v_tmp_wt0, v_quant0), v_tmp_wt0);
+ const int32x4_t v_tmp2_1 =
+ vaddq_s32(vqdmulhq_s32(v_tmp_wt1, v_quant1), v_tmp_wt1);
+ const int32x4_t v_abs_q0 = vshrq_n_s32(
+ vqdmulhq_s32(vshlq_s32(v_tmp2_0, v_log_scale), v_qshift0), AOM_QM_BITS);
+ const int32x4_t v_abs_q1 = vshrq_n_s32(
+ vqdmulhq_s32(vshlq_s32(v_tmp2_1, v_log_scale), v_qshift1), AOM_QM_BITS);
+
+ const uint16x8_t v_iwt_u16 = vmovl_u8(vld1_u8(iqm_ptr));
+ const int32x4_t v_iwt0 =
+ vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(v_iwt_u16)));
+ const int32x4_t v_iwt1 =
+ vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(v_iwt_u16)));
+ const int32x4_t v_dq_rnd = vdupq_n_s32(1 << (AOM_QM_BITS - 1));
+ const int32x4_t v_dq0 =
+ vshrq_n_s32(vmlaq_s32(v_dq_rnd, v_dequant0, v_iwt0), AOM_QM_BITS);
+ const int32x4_t v_dq1 =
+ vshrq_n_s32(vmlaq_s32(v_dq_rnd, v_dequant1, v_iwt1), AOM_QM_BITS);
+ const int32x4_t v_abs_dq0 =
+ vshlq_s32(vmulq_s32(v_abs_q0, v_dq0), v_neg_log_scale);
+ const int32x4_t v_abs_dq1 =
+ vshlq_s32(vmulq_s32(v_abs_q1, v_dq1), v_neg_log_scale);
+
+ const int32x4_t v_q0 = vsubq_s32(veorq_s32(v_abs_q0, v_sign0), v_sign0);
+ const int32x4_t v_q1 = vsubq_s32(veorq_s32(v_abs_q1, v_sign1), v_sign1);
+ const int32x4_t v_dq_out0 =
+ vsubq_s32(veorq_s32(v_abs_dq0, v_sign0), v_sign0);
+ const int32x4_t v_dq_out1 =
+ vsubq_s32(veorq_s32(v_abs_dq1, v_sign1), v_sign1);
+ vst1q_s32(qcoeff_ptr, v_q0);
+ vst1q_s32(qcoeff_ptr + 4, v_q1);
+ vst1q_s32(dqcoeff_ptr, v_dq_out0);
+ vst1q_s32(dqcoeff_ptr + 4, v_dq_out1);
+
+ const uint32x4_t v_nz0 = vcgtq_s32(v_abs_q0, v_zero);
+ const uint32x4_t v_nz1 = vcgtq_s32(v_abs_q1, v_zero);
+ return update_qm_eob(iscan_ptr, v_eobmax, v_nz0, v_nz1);
+ }
+ const int32x4_t v_zero = vdupq_n_s32(0);
+ vst1q_s32(qcoeff_ptr, v_zero);
+ vst1q_s32(qcoeff_ptr + 4, v_zero);
+ vst1q_s32(dqcoeff_ptr, v_zero);
+ vst1q_s32(dqcoeff_ptr + 4, v_zero);
+ return v_eobmax;
+}
+
+void av1_quantize_b_qm_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
+ const int16_t *zbin_ptr, const int16_t *round_ptr,
+ const int16_t *quant_ptr,
+ const int16_t *quant_shift_ptr,
+ tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
+ const int16_t *dequant_ptr, uint16_t *eob_ptr,
+ const int16_t *scan, const int16_t *iscan,
+ const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr,
+ int log_scale) {
+ (void)scan;
+ assert(qm_ptr != NULL && iqm_ptr != NULL);
+ assert(n_coeffs >= 16 && (n_coeffs & 7) == 0);
+ const int32x4_t v_zbin_ac =
+ vdupq_n_s32(ROUND_POWER_OF_TWO(zbin_ptr[1], log_scale) << AOM_QM_BITS);
+ const int32x4_t v_zbin0 = vsetq_lane_s32(
+ ROUND_POWER_OF_TWO(zbin_ptr[0], log_scale) << AOM_QM_BITS, v_zbin_ac, 0);
+ const int32x4_t v_round_ac =
+ vdupq_n_s32(ROUND_POWER_OF_TWO(round_ptr[1], log_scale));
+ const int32x4_t v_round0 = vsetq_lane_s32(
+ ROUND_POWER_OF_TWO(round_ptr[0], log_scale), v_round_ac, 0);
+ const int32x4_t v_quant_ac =
+ vdupq_n_s32((int32_t)((uint32_t)quant_ptr[1] << 15));
+ const int32x4_t v_quant0 =
+ vsetq_lane_s32((int32_t)((uint32_t)quant_ptr[0] << 15), v_quant_ac, 0);
+ const int32x4_t v_qshift_ac =
+ vdupq_n_s32((int32_t)((uint32_t)quant_shift_ptr[1] << 15));
+ const int32x4_t v_qshift0 = vsetq_lane_s32(
+ (int32_t)((uint32_t)quant_shift_ptr[0] << 15), v_qshift_ac, 0);
+ const int32x4_t v_dequant_ac = vdupq_n_s32(dequant_ptr[1]);
+ const int32x4_t v_dequant0 = vsetq_lane_s32(dequant_ptr[0], v_dequant_ac, 0);
+ const int32x4_t v_log_scale = vdupq_n_s32(log_scale);
+ const int32x4_t v_neg_log_scale = vdupq_n_s32(-log_scale);
+ int16x8_t v_eobmax = vdupq_n_s16(-1);
+
+ v_eobmax = quantize_b_qm_8_neon(coeff_ptr, iscan, qm_ptr, iqm_ptr, qcoeff_ptr,
+ dqcoeff_ptr, v_zbin0, v_zbin_ac, v_round0,
+ v_round_ac, v_quant0, v_quant_ac, v_qshift0,
+ v_qshift_ac, v_dequant0, v_dequant_ac,
+ v_log_scale, v_neg_log_scale, v_eobmax);
+
+ intptr_t i = 8;
+ do {
+ v_eobmax = quantize_b_qm_8_neon(
+ coeff_ptr + i, iscan + i, qm_ptr + i, iqm_ptr + i, qcoeff_ptr + i,
+ dqcoeff_ptr + i, v_zbin_ac, v_zbin_ac, v_round_ac, v_round_ac,
+ v_quant_ac, v_quant_ac, v_qshift_ac, v_qshift_ac, v_dequant_ac,
+ v_dequant_ac, v_log_scale, v_neg_log_scale, v_eobmax);
+ i += 8;
+ } while (i != n_coeffs);
+ *eob_ptr = get_max_eob(v_eobmax);
+}
diff --git a/test/quantize_func_test.cc b/test/quantize_func_test.cc
index 3e35c8dc50..a91fa58f83 100644
--- a/test/quantize_func_test.cc
+++ b/test/quantize_func_test.cc
@@ -477,8 +477,7 @@ TEST_P(LowPrecisionQuantizeTest, DISABLED_Speed) {
using std::make_tuple;
-#if HAVE_AVX2
-#if CONFIG_QUANT_MATRIX || CONFIG_AV1_DECODER
+#if (HAVE_AVX2 || HAVE_NEON) && (CONFIG_QUANT_MATRIX || CONFIG_AV1_DECODER)
using QuantizeFpQmFunc = void (*)(
const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr,
const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
@@ -520,7 +519,10 @@ void quan_b_qm_wrapper(QUAN_PARAM_LIST) {
qcoeff_ptr, dqcoeff_ptr, dequant_ptr, eob_ptr, scan, iscan, qm_ptr,
iqm_ptr, log_scale);
}
-#endif // CONFIG_QUANT_MATRIX || CONFIG_AV1_DECODER
+#endif // (HAVE_AVX2 || HAVE_NEON) && (CONFIG_QUANT_MATRIX ||
+ // CONFIG_AV1_DECODER)
+
+#if HAVE_AVX2
const QuantizeParam<LPQuantizeFunc> kLPQParamArrayAvx2[] = {
make_tuple(&av1_quantize_lp_c, &av1_quantize_lp_avx2,
@@ -905,17 +907,159 @@ const QuantizeParam<QuantizeFunc> kQParamArrayNEON[] = {
static_cast<TX_SIZE>(TX_32X32), TYPE_B, AOM_BITS_8),
make_tuple(&aom_quantize_b_64x64_c, &aom_quantize_b_64x64_neon,
static_cast<TX_SIZE>(TX_64X64), TYPE_B, AOM_BITS_8),
+#if CONFIG_QUANT_MATRIX || CONFIG_AV1_DECODER
+ make_tuple(&quan_fp_qm_wrapper<av1_quantize_fp_qm_c, TX_4X4>,
+ &quan_fp_qm_wrapper<av1_quantize_fp_qm_neon, TX_4X4>,
+ static_cast<TX_SIZE>(TX_4X4), TYPE_FP, AOM_BITS_8),
+ make_tuple(&quan_fp_qm_wrapper<av1_quantize_fp_qm_c, TX_8X8>,
+ &quan_fp_qm_wrapper<av1_quantize_fp_qm_neon, TX_8X8>,
+ static_cast<TX_SIZE>(TX_8X8), TYPE_FP, AOM_BITS_8),
+ make_tuple(&quan_fp_qm_wrapper<av1_quantize_fp_qm_c, TX_16X16>,
+ &quan_fp_qm_wrapper<av1_quantize_fp_qm_neon, TX_16X16>,
+ static_cast<TX_SIZE>(TX_16X16), TYPE_FP, AOM_BITS_8),
+ make_tuple(&quan_fp_qm_wrapper<av1_quantize_fp_qm_c, TX_32X32>,
+ &quan_fp_qm_wrapper<av1_quantize_fp_qm_neon, TX_32X32>,
+ static_cast<TX_SIZE>(TX_32X32), TYPE_FP, AOM_BITS_8),
+ make_tuple(&quan_fp_qm_wrapper<av1_quantize_fp_qm_c, TX_64X64>,
+ &quan_fp_qm_wrapper<av1_quantize_fp_qm_neon, TX_64X64>,
+ static_cast<TX_SIZE>(TX_64X64), TYPE_FP, AOM_BITS_8),
+ make_tuple(&quan_b_qm_wrapper<av1_quantize_b_qm_c, TX_4X4>,
+ &quan_b_qm_wrapper<av1_quantize_b_qm_neon, TX_4X4>,
+ static_cast<TX_SIZE>(TX_4X4), TYPE_B, AOM_BITS_8),
+ make_tuple(&quan_b_qm_wrapper<av1_quantize_b_qm_c, TX_8X8>,
+ &quan_b_qm_wrapper<av1_quantize_b_qm_neon, TX_8X8>,
+ static_cast<TX_SIZE>(TX_8X8), TYPE_B, AOM_BITS_8),
+ make_tuple(&quan_b_qm_wrapper<av1_quantize_b_qm_c, TX_16X16>,
+ &quan_b_qm_wrapper<av1_quantize_b_qm_neon, TX_16X16>,
+ static_cast<TX_SIZE>(TX_16X16), TYPE_B, AOM_BITS_8),
+ make_tuple(&quan_b_qm_wrapper<av1_quantize_b_qm_c, TX_32X32>,
+ &quan_b_qm_wrapper<av1_quantize_b_qm_neon, TX_32X32>,
+ static_cast<TX_SIZE>(TX_32X32), TYPE_B, AOM_BITS_8),
+ make_tuple(&quan_b_qm_wrapper<av1_quantize_b_qm_c, TX_64X64>,
+ &quan_b_qm_wrapper<av1_quantize_b_qm_neon, TX_64X64>,
+ static_cast<TX_SIZE>(TX_64X64), TYPE_B, AOM_BITS_8),
+#endif // CONFIG_QUANT_MATRIX || CONFIG_AV1_DECODER
#if CONFIG_AV1_HIGHBITDEPTH
+ make_tuple(&highbd_quan16x16_wrapper<av1_highbd_quantize_fp_c>,
+ &highbd_quan16x16_wrapper<av1_highbd_quantize_fp_neon>,
+ static_cast<TX_SIZE>(TX_16X16), TYPE_FP, AOM_BITS_8),
+ make_tuple(&highbd_quan16x16_wrapper<av1_highbd_quantize_fp_c>,
+ &highbd_quan16x16_wrapper<av1_highbd_quantize_fp_neon>,
+ static_cast<TX_SIZE>(TX_16X16), TYPE_FP, AOM_BITS_10),
make_tuple(&highbd_quan16x16_wrapper<av1_highbd_quantize_fp_c>,
&highbd_quan16x16_wrapper<av1_highbd_quantize_fp_neon>,
static_cast<TX_SIZE>(TX_16X16), TYPE_FP, AOM_BITS_12),
+ make_tuple(&highbd_quan32x32_wrapper<av1_highbd_quantize_fp_c>,
+ &highbd_quan32x32_wrapper<av1_highbd_quantize_fp_neon>,
+ static_cast<TX_SIZE>(TX_32X32), TYPE_FP, AOM_BITS_8),
+ make_tuple(&highbd_quan32x32_wrapper<av1_highbd_quantize_fp_c>,
+ &highbd_quan32x32_wrapper<av1_highbd_quantize_fp_neon>,
+ static_cast<TX_SIZE>(TX_32X32), TYPE_FP, AOM_BITS_10),
make_tuple(&highbd_quan32x32_wrapper<av1_highbd_quantize_fp_c>,
&highbd_quan32x32_wrapper<av1_highbd_quantize_fp_neon>,
static_cast<TX_SIZE>(TX_32X32), TYPE_FP, AOM_BITS_12),
+ make_tuple(&highbd_quan64x64_wrapper<av1_highbd_quantize_fp_c>,
+ &highbd_quan64x64_wrapper<av1_highbd_quantize_fp_neon>,
+ static_cast<TX_SIZE>(TX_64X64), TYPE_FP, AOM_BITS_8),
+ make_tuple(&highbd_quan64x64_wrapper<av1_highbd_quantize_fp_c>,
+ &highbd_quan64x64_wrapper<av1_highbd_quantize_fp_neon>,
+ static_cast<TX_SIZE>(TX_64X64), TYPE_FP, AOM_BITS_10),
make_tuple(&highbd_quan64x64_wrapper<av1_highbd_quantize_fp_c>,
&highbd_quan64x64_wrapper<av1_highbd_quantize_fp_neon>,
static_cast<TX_SIZE>(TX_64X64), TYPE_FP, AOM_BITS_12),
+#if CONFIG_QUANT_MATRIX || CONFIG_AV1_DECODER
+ make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_4X4>,
+ &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_4X4>,
+ static_cast<TX_SIZE>(TX_4X4), TYPE_FP, AOM_BITS_8),
+ make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_4X4>,
+ &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_4X4>,
+ static_cast<TX_SIZE>(TX_4X4), TYPE_FP, AOM_BITS_10),
+ make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_4X4>,
+ &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_4X4>,
+ static_cast<TX_SIZE>(TX_4X4), TYPE_FP, AOM_BITS_12),
+ make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_8X8>,
+ &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_8X8>,
+ static_cast<TX_SIZE>(TX_8X8), TYPE_FP, AOM_BITS_8),
+ make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_8X8>,
+ &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_8X8>,
+ static_cast<TX_SIZE>(TX_8X8), TYPE_FP, AOM_BITS_10),
+ make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_8X8>,
+ &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_8X8>,
+ static_cast<TX_SIZE>(TX_8X8), TYPE_FP, AOM_BITS_12),
+ make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_16X16>,
+ &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_16X16>,
+ static_cast<TX_SIZE>(TX_16X16), TYPE_FP, AOM_BITS_8),
+ make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_16X16>,
+ &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_16X16>,
+ static_cast<TX_SIZE>(TX_16X16), TYPE_FP, AOM_BITS_10),
+ make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_16X16>,
+ &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_16X16>,
+ static_cast<TX_SIZE>(TX_16X16), TYPE_FP, AOM_BITS_12),
+ make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_32X32>,
+ &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_32X32>,
+ static_cast<TX_SIZE>(TX_32X32), TYPE_FP, AOM_BITS_8),
+ make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_32X32>,
+ &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_32X32>,
+ static_cast<TX_SIZE>(TX_32X32), TYPE_FP, AOM_BITS_10),
+ make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_32X32>,
+ &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_32X32>,
+ static_cast<TX_SIZE>(TX_32X32), TYPE_FP, AOM_BITS_12),
+ make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_64X64>,
+ &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_64X64>,
+ static_cast<TX_SIZE>(TX_64X64), TYPE_FP, AOM_BITS_8),
+ make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_64X64>,
+ &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_64X64>,
+ static_cast<TX_SIZE>(TX_64X64), TYPE_FP, AOM_BITS_10),
+ make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_64X64>,
+ &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_64X64>,
+ static_cast<TX_SIZE>(TX_64X64), TYPE_FP, AOM_BITS_12),
+ make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_4X4>,
+ &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_4X4>,
+ static_cast<TX_SIZE>(TX_4X4), TYPE_B, AOM_BITS_8),
+ make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_4X4>,
+ &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_4X4>,
+ static_cast<TX_SIZE>(TX_4X4), TYPE_B, AOM_BITS_10),
+ make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_4X4>,
+ &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_4X4>,
+ static_cast<TX_SIZE>(TX_4X4), TYPE_B, AOM_BITS_12),
+ make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_8X8>,
+ &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_8X8>,
+ static_cast<TX_SIZE>(TX_8X8), TYPE_B, AOM_BITS_8),
+ make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_8X8>,
+ &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_8X8>,
+ static_cast<TX_SIZE>(TX_8X8), TYPE_B, AOM_BITS_10),
+ make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_8X8>,
+ &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_8X8>,
+ static_cast<TX_SIZE>(TX_8X8), TYPE_B, AOM_BITS_12),
+ make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_16X16>,
+ &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_16X16>,
+ static_cast<TX_SIZE>(TX_16X16), TYPE_B, AOM_BITS_8),
+ make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_16X16>,
+ &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_16X16>,
+ static_cast<TX_SIZE>(TX_16X16), TYPE_B, AOM_BITS_10),
+ make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_16X16>,
+ &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_16X16>,
+ static_cast<TX_SIZE>(TX_16X16), TYPE_B, AOM_BITS_12),
+ make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_32X32>,
+ &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_32X32>,
+ static_cast<TX_SIZE>(TX_32X32), TYPE_B, AOM_BITS_8),
+ make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_32X32>,
+ &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_32X32>,
+ static_cast<TX_SIZE>(TX_32X32), TYPE_B, AOM_BITS_10),
+ make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_32X32>,
+ &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_32X32>,
+ static_cast<TX_SIZE>(TX_32X32), TYPE_B, AOM_BITS_12),
+ make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_64X64>,
+ &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_64X64>,
+ static_cast<TX_SIZE>(TX_64X64), TYPE_B, AOM_BITS_8),
+ make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_64X64>,
+ &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_64X64>,
+ static_cast<TX_SIZE>(TX_64X64), TYPE_B, AOM_BITS_10),
+ make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_64X64>,
+ &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_64X64>,
+ static_cast<TX_SIZE>(TX_64X64), TYPE_B, AOM_BITS_12),
+#endif // CONFIG_QUANT_MATRIX || CONFIG_AV1_DECODER
make_tuple(&aom_highbd_quantize_b_c, &aom_highbd_quantize_b_neon,
static_cast<TX_SIZE>(TX_16X16), TYPE_B, AOM_BITS_12),
make_tuple(&aom_highbd_quantize_b_32x32_c, &aom_highbd_quantize_b_32x32_neon,