Commit b0904c4c10 for aom

commit b0904c4c10f4d3db936cd3f9787b8c5d39bdfdf0
Author: Frank Galligan <fgalligan@google.com>
Date:   Sat Nov 14 11:52:39 2026 -0800

    av1,neon: Add LBD/HBD quantize_{fp,b}_qm_neon

    Add NEON implementations for av1_quantize_fp_qm, av1_quantize_b_qm,
    av1_highbd_quantize_fp_qm, and av1_highbd_quantize_b_qm, optimize
    zero-skip pre-scans in quantize_fp and highbd_quantize_fp, and
    register the new kernels in QuantizeTest.

    Speedup: 3.64x-6.02x vs C (quantize_{fp,b}_qm_neon) in test_libaom on
    Neoverse N3; -11.99% (1T) / -12.41% (4T) 8b on Neoverse N3, -12.26%
    (1T) / -12.48% (4T) 10b on Neoverse N2, and -12.52% (1T Cortex-X4) /
    -12.45% (1T Cortex-A720) on Pixel 9 avifenc (bit-exact).

    TAG=agy
    CONV=490488d4-7005-402a-8cc4-c245227f80fe

    Change-Id: I435e150f71313c08856ab25175aafeca0295170b

diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl
index 5b1aa3e1ca..4cd3e13958 100644
--- a/av1/common/av1_rtcd_defs.pl
+++ b/av1/common/av1_rtcd_defs.pl
@@ -320,10 +320,10 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
   specialize qw/av1_quantize_fp_64x64 neon avx2/;

   add_proto qw/void av1_quantize_fp_qm/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, int log_scale";
-  specialize qw/av1_quantize_fp_qm avx2/;
+  specialize qw/av1_quantize_fp_qm avx2 neon/;

   add_proto qw/void av1_quantize_b_qm/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, int log_scale";
-  specialize qw/av1_quantize_b_qm avx2/;
+  specialize qw/av1_quantize_b_qm avx2 neon/;

   add_proto qw/void aom_quantize_b_helper/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, const int log_scale";
   specialize qw/aom_quantize_b_helper neon/;
@@ -456,10 +456,10 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
     specialize qw/av1_highbd_quantize_fp sse4_1 avx2 neon/;

     add_proto qw/void av1_highbd_quantize_fp_qm/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, int log_scale";
-    specialize qw/av1_highbd_quantize_fp_qm avx2/;
+    specialize qw/av1_highbd_quantize_fp_qm avx2 neon/;

     add_proto qw/void av1_highbd_quantize_b_qm/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, int log_scale";
-    specialize qw/av1_highbd_quantize_b_qm avx2/;
+    specialize qw/av1_highbd_quantize_b_qm avx2 neon/;
   }

   # End av1_high encoder functions
diff --git a/av1/encoder/arm/av1_highbd_quantize_neon.c b/av1/encoder/arm/av1_highbd_quantize_neon.c
index fd4749d286..e70a04840f 100644
--- a/av1/encoder/arm/av1_highbd_quantize_neon.c
+++ b/av1/encoder/arm/av1_highbd_quantize_neon.c
@@ -10,42 +10,140 @@
  */

 #include <arm_neon.h>
+#include <assert.h>

 #include "config/aom_config.h"
+#include "config/av1_rtcd.h"

 #include "aom_dsp/arm/mem_neon.h"

 #include "av1/common/quant_common.h"
 #include "av1/encoder/av1_quantize.h"

-static inline uint16x4_t quantize_4(const tran_low_t *coeff_ptr,
-                                    tran_low_t *qcoeff_ptr,
-                                    tran_low_t *dqcoeff_ptr,
-                                    int32x4_t v_quant_s32,
-                                    int32x4_t v_dequant_s32,
-                                    int32x4_t v_round_s32, int log_scale) {
-  const int32x4_t v_coeff = vld1q_s32(coeff_ptr);
+static AOM_FORCE_INLINE uint16x4_t quantize_4_from_abs_ls0(
+    int32x4_t v_coeff, int32x4_t v_abs_coeff, tran_low_t *qcoeff_ptr,
+    tran_low_t *dqcoeff_ptr, int32x4_t v_quant_s32, int32x4_t v_dequant_s32,
+    int32x4_t v_round_s32) {
+  const int32x4_t v_zero = vdupq_n_s32(0);
   const int32x4_t v_coeff_sign =
-      vreinterpretq_s32_u32(vcltq_s32(v_coeff, vdupq_n_s32(0)));
-  const int32x4_t v_log_scale = vdupq_n_s32(log_scale);
-  const int32x4_t v_abs_coeff = vabsq_s32(v_coeff);
+      vreinterpretq_s32_u32(vcltq_s32(v_coeff, v_zero));
+  const int32x4_t v_abs_coeff_scaled = vshlq_n_s32(v_abs_coeff, 1);
+  const uint32x4_t v_mask = vcgeq_s32(v_abs_coeff_scaled, v_dequant_s32);
+  const int32x4_t v_tmp = vandq_s32(vaddq_s32(v_abs_coeff, v_round_s32),
+                                    vreinterpretq_s32_u32(v_mask));
+  const int32x4_t v_abs_qcoeff = vqdmulhq_s32(v_tmp, v_quant_s32);
+  const int32x4_t v_qcoeff =
+      vsubq_s32(veorq_s32(v_abs_qcoeff, v_coeff_sign), v_coeff_sign);
+  const int32x4_t v_abs_dqcoeff = vmulq_s32(v_abs_qcoeff, v_dequant_s32);
+  const int32x4_t v_dqcoeff =
+      vsubq_s32(veorq_s32(v_abs_dqcoeff, v_coeff_sign), v_coeff_sign);
+  vst1q_s32(qcoeff_ptr, v_qcoeff);
+  vst1q_s32(dqcoeff_ptr, v_dqcoeff);
+  const uint32x4_t nz_qcoeff_mask = vcgtq_s32(v_abs_qcoeff, v_zero);
+  return vmovn_u32(nz_qcoeff_mask);
+}
+
+static AOM_FORCE_INLINE uint16x4_t quantize_4_from_abs_ls1(
+    int32x4_t v_coeff, int32x4_t v_abs_coeff, tran_low_t *qcoeff_ptr,
+    tran_low_t *dqcoeff_ptr, int32x4_t v_quant_s32, int32x4_t v_dequant_s32,
+    int32x4_t v_round_s32) {
+  const int32x4_t v_zero = vdupq_n_s32(0);
+  const int32x4_t v_coeff_sign =
+      vreinterpretq_s32_u32(vcltq_s32(v_coeff, v_zero));
+  const int32x4_t v_abs_coeff_scaled = vshlq_n_s32(v_abs_coeff, 2);
+  const uint32x4_t v_mask = vcgeq_s32(v_abs_coeff_scaled, v_dequant_s32);
+  const int32x4_t v_tmp = vandq_s32(vaddq_s32(v_abs_coeff, v_round_s32),
+                                    vreinterpretq_s32_u32(v_mask));
+  const int32x4_t v_abs_qcoeff =
+      vqdmulhq_s32(vshlq_n_s32(v_tmp, 1), v_quant_s32);
+  const int32x4_t v_qcoeff =
+      vsubq_s32(veorq_s32(v_abs_qcoeff, v_coeff_sign), v_coeff_sign);
+  const int32x4_t v_abs_dqcoeff =
+      vshrq_n_s32(vmulq_s32(v_abs_qcoeff, v_dequant_s32), 1);
+  const int32x4_t v_dqcoeff =
+      vsubq_s32(veorq_s32(v_abs_dqcoeff, v_coeff_sign), v_coeff_sign);
+  vst1q_s32(qcoeff_ptr, v_qcoeff);
+  vst1q_s32(dqcoeff_ptr, v_dqcoeff);
+  const uint32x4_t nz_qcoeff_mask = vcgtq_s32(v_abs_qcoeff, v_zero);
+  return vmovn_u32(nz_qcoeff_mask);
+}
+
+static AOM_FORCE_INLINE uint16x4_t quantize_4_from_abs_ls2(
+    int32x4_t v_coeff, int32x4_t v_abs_coeff, tran_low_t *qcoeff_ptr,
+    tran_low_t *dqcoeff_ptr, int32x4_t v_quant_s32, int32x4_t v_dequant_s32,
+    int32x4_t v_round_s32) {
+  const int32x4_t v_zero = vdupq_n_s32(0);
+  const int32x4_t v_coeff_sign =
+      vreinterpretq_s32_u32(vcltq_s32(v_coeff, v_zero));
+  const int32x4_t v_abs_coeff_scaled = vshlq_n_s32(v_abs_coeff, 3);
+  const uint32x4_t v_mask = vcgeq_s32(v_abs_coeff_scaled, v_dequant_s32);
+  const int32x4_t v_tmp = vandq_s32(vaddq_s32(v_abs_coeff, v_round_s32),
+                                    vreinterpretq_s32_u32(v_mask));
+  const int32x4_t v_abs_qcoeff =
+      vqdmulhq_s32(vshlq_n_s32(v_tmp, 2), v_quant_s32);
+  const int32x4_t v_qcoeff =
+      vsubq_s32(veorq_s32(v_abs_qcoeff, v_coeff_sign), v_coeff_sign);
+  const int32x4_t v_abs_dqcoeff =
+      vshrq_n_s32(vmulq_s32(v_abs_qcoeff, v_dequant_s32), 2);
+  const int32x4_t v_dqcoeff =
+      vsubq_s32(veorq_s32(v_abs_dqcoeff, v_coeff_sign), v_coeff_sign);
+  vst1q_s32(qcoeff_ptr, v_qcoeff);
+  vst1q_s32(dqcoeff_ptr, v_dqcoeff);
+  const uint32x4_t nz_qcoeff_mask = vcgtq_s32(v_abs_qcoeff, v_zero);
+  return vmovn_u32(nz_qcoeff_mask);
+}
+
+static AOM_FORCE_INLINE int hbd_zero_skip_u32(int32x4_t v_abs0,
+                                              int32x4_t v_abs1,
+                                              uint32_t ac_thresh,
+                                              tran_low_t *qcoeff_ptr,
+                                              tran_low_t *dqcoeff_ptr) {
+#if AOM_ARCH_AARCH64
+  const int32x4_t v_zero = vdupq_n_s32(0);
+  const uint32x4_t v_abs_max =
+      vmaxq_u32(vreinterpretq_u32_s32(v_abs0), vreinterpretq_u32_s32(v_abs1));
+  if (vmaxvq_u32(v_abs_max) < ac_thresh) {
+    vst1q_s32(qcoeff_ptr, v_zero);
+    vst1q_s32(qcoeff_ptr + 4, v_zero);
+    vst1q_s32(dqcoeff_ptr, v_zero);
+    vst1q_s32(dqcoeff_ptr + 4, v_zero);
+    return 1;
+  }
+#else
+  (void)v_abs0;
+  (void)v_abs1;
+  (void)ac_thresh;
+  (void)qcoeff_ptr;
+  (void)dqcoeff_ptr;
+#endif
+  return 0;
+}
+
+static inline uint16x4_t quantize_4_from_abs(
+    int32x4_t v_coeff, int32x4_t v_abs_coeff, tran_low_t *qcoeff_ptr,
+    tran_low_t *dqcoeff_ptr, int32x4_t v_quant_s32, int32x4_t v_dequant_s32,
+    int32x4_t v_round_s32, int32x4_t v_log_scale, int32x4_t v_neg_log_scale,
+    int32x4_t v_one_plus_log_scale) {
+  const int32x4_t v_zero = vdupq_n_s32(0);
+  const int32x4_t v_coeff_sign =
+      vreinterpretq_s32_u32(vcltq_s32(v_coeff, v_zero));
   // ((abs_coeff << (1 + log_scale)) >= dequant_ptr[rc01])
   const int32x4_t v_abs_coeff_scaled =
-      vshlq_s32(v_abs_coeff, vdupq_n_s32(1 + log_scale));
+      vshlq_s32(v_abs_coeff, v_one_plus_log_scale);
   const uint32x4_t v_mask = vcgeq_s32(v_abs_coeff_scaled, v_dequant_s32);
   // const int64_t tmp = vmask ? (int64_t)abs_coeff + log_scaled_round : 0
   const int32x4_t v_tmp = vandq_s32(vaddq_s32(v_abs_coeff, v_round_s32),
                                     vreinterpretq_s32_u32(v_mask));
-  //  const int abs_qcoeff = (int)((tmp * quant) >> (16 - log_scale));
+  // const int abs_qcoeff = (int)((tmp * quant) >> (16 - log_scale));
   const int32x4_t v_abs_qcoeff =
       vqdmulhq_s32(vshlq_s32(v_tmp, v_log_scale), v_quant_s32);
-  //  qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
+  // qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
   const int32x4_t v_qcoeff =
       vsubq_s32(veorq_s32(v_abs_qcoeff, v_coeff_sign), v_coeff_sign);
   // vshlq_s32 will shift right if shift value is negative.
   const int32x4_t v_abs_dqcoeff =
-      vshlq_s32(vmulq_s32(v_abs_qcoeff, v_dequant_s32), vnegq_s32(v_log_scale));
-  //  dqcoeff_ptr[rc] = (tran_low_t)((abs_dqcoeff ^ coeff_sign) - coeff_sign);
+      vshlq_s32(vmulq_s32(v_abs_qcoeff, v_dequant_s32), v_neg_log_scale);
+  // dqcoeff_ptr[rc] = (tran_low_t)((abs_dqcoeff ^ coeff_sign) - coeff_sign);
   const int32x4_t v_dqcoeff =
       vsubq_s32(veorq_s32(v_abs_dqcoeff, v_coeff_sign), v_coeff_sign);

@@ -53,7 +151,7 @@ static inline uint16x4_t quantize_4(const tran_low_t *coeff_ptr,
   vst1q_s32(dqcoeff_ptr, v_dqcoeff);

   // Used to find eob.
-  const uint32x4_t nz_qcoeff_mask = vcgtq_s32(v_abs_qcoeff, vdupq_n_s32(0));
+  const uint32x4_t nz_qcoeff_mask = vcgtq_s32(v_abs_qcoeff, v_zero);
   return vmovn_u32(nz_qcoeff_mask);
 }

@@ -84,6 +182,50 @@ static inline uint16_t get_max_eob(int16x8_t v_eobmax) {
 #endif
 }

+#define HBD_QUANTIZE_BODY(LS, V_ROUND_INIT)                                    \
+  do {                                                                         \
+    const int16x4_t v_round = (V_ROUND_INIT);                                  \
+    int32x4_t v_round_s32 = vaddl_s16(v_round, v_zero);                        \
+    int32x4_t v_quant_s32 = vshlq_n_s32(vaddl_s16(v_quant, v_zero), 15);       \
+    int32x4_t v_dequant_s32 = vaddl_s16(v_dequant, v_zero);                    \
+    const int32x4_t v_dc_coeff = vld1q_s32(coeff_ptr);                         \
+    v_mask_lo = quantize_4_from_abs_ls##LS(                                    \
+        v_dc_coeff, vabsq_s32(v_dc_coeff), qcoeff_ptr, dqcoeff_ptr,            \
+        v_quant_s32, v_dequant_s32, v_round_s32);                              \
+    v_round_s32 = vdupq_lane_s32(vget_low_s32(v_round_s32), 1);                \
+    v_quant_s32 = vdupq_lane_s32(vget_low_s32(v_quant_s32), 1);                \
+    v_dequant_s32 = vdupq_lane_s32(vget_low_s32(v_dequant_s32), 1);            \
+    const int32x4_t v_ac_coeff = vld1q_s32(coeff_ptr + 4);                     \
+    v_mask_hi = quantize_4_from_abs_ls##LS(                                    \
+        v_ac_coeff, vabsq_s32(v_ac_coeff), qcoeff_ptr + 4, dqcoeff_ptr + 4,    \
+        v_quant_s32, v_dequant_s32, v_round_s32);                              \
+    v_eobmax =                                                                 \
+        get_max_lane_eob(iscan, v_eobmax, vcombine_u16(v_mask_lo, v_mask_hi)); \
+    count -= 8;                                                                \
+    do {                                                                       \
+      coeff_ptr += 8;                                                          \
+      qcoeff_ptr += 8;                                                         \
+      dqcoeff_ptr += 8;                                                        \
+      iscan += 8;                                                              \
+      const int32x4_t v_c0 = vld1q_s32(coeff_ptr);                             \
+      const int32x4_t v_c1 = vld1q_s32(coeff_ptr + 4);                         \
+      const int32x4_t v_abs0 = vabsq_s32(v_c0);                                \
+      const int32x4_t v_abs1 = vabsq_s32(v_c1);                                \
+      if (!hbd_zero_skip_u32(v_abs0, v_abs1, ac_thresh, qcoeff_ptr,            \
+                             dqcoeff_ptr)) {                                   \
+        v_mask_lo = quantize_4_from_abs_ls##LS(v_c0, v_abs0, qcoeff_ptr,       \
+                                               dqcoeff_ptr, v_quant_s32,       \
+                                               v_dequant_s32, v_round_s32);    \
+        v_mask_hi = quantize_4_from_abs_ls##LS(v_c1, v_abs1, qcoeff_ptr + 4,   \
+                                               dqcoeff_ptr + 4, v_quant_s32,   \
+                                               v_dequant_s32, v_round_s32);    \
+        v_eobmax = get_max_lane_eob(iscan, v_eobmax,                           \
+                                    vcombine_u16(v_mask_lo, v_mask_hi));       \
+      }                                                                        \
+      count -= 8;                                                              \
+    } while (count != 0);                                                      \
+  } while (0)
+
 void av1_highbd_quantize_fp_neon(
     const tran_low_t *coeff_ptr, intptr_t count, const int16_t *zbin_ptr,
     const int16_t *round_ptr, const int16_t *quant_ptr,
@@ -93,54 +235,361 @@ void av1_highbd_quantize_fp_neon(
   (void)scan;
   (void)zbin_ptr;
   (void)quant_shift_ptr;
+  assert(count >= 16 && (count & 7) == 0);

   const int16x4_t v_quant = vld1_s16(quant_ptr);
   const int16x4_t v_dequant = vld1_s16(dequant_ptr);
   const int16x4_t v_zero = vdup_n_s16(0);
-  const uint16x4_t v_round_select = vcgt_s16(vdup_n_s16(log_scale), v_zero);
-  const int16x4_t v_round_no_scale = vld1_s16(round_ptr);
-  const int16x4_t v_round_log_scale =
-      vqrdmulh_n_s16(v_round_no_scale, (int16_t)(1 << (15 - log_scale)));
-  const int16x4_t v_round =
-      vbsl_s16(v_round_select, v_round_log_scale, v_round_no_scale);
-  int32x4_t v_round_s32 = vaddl_s16(v_round, v_zero);
-  int32x4_t v_quant_s32 = vshlq_n_s32(vaddl_s16(v_quant, v_zero), 15);
-  int32x4_t v_dequant_s32 = vaddl_s16(v_dequant, v_zero);
   uint16x4_t v_mask_lo, v_mask_hi;
   int16x8_t v_eobmax = vdupq_n_s16(-1);
+  const int ac_dequant = dequant_ptr[1];
+  const uint32_t ac_thresh =
+      ac_dequant > 0 ? (((uint32_t)ac_dequant + (1u << (1 + log_scale)) - 1u) >>
+                        (1 + log_scale))
+                     : 0u;

-  // DC and first 3 AC
-  v_mask_lo = quantize_4(coeff_ptr, qcoeff_ptr, dqcoeff_ptr, v_quant_s32,
-                         v_dequant_s32, v_round_s32, log_scale);
+  if (log_scale == 0) {
+    HBD_QUANTIZE_BODY(0, vld1_s16(round_ptr));
+  } else if (log_scale == 1) {
+    HBD_QUANTIZE_BODY(1,
+                      vqrdmulh_n_s16(vld1_s16(round_ptr), (int16_t)(1 << 14)));
+  } else if (log_scale == 2) {
+    HBD_QUANTIZE_BODY(2,
+                      vqrdmulh_n_s16(vld1_s16(round_ptr), (int16_t)(1 << 13)));
+  } else {
+    const uint16x4_t v_round_select = vcgt_s16(vdup_n_s16(log_scale), v_zero);
+    const int16x4_t v_round_no_scale = vld1_s16(round_ptr);
+    const int16x4_t v_round_log_scale =
+        vqrdmulh_n_s16(v_round_no_scale, (int16_t)(1 << (15 - log_scale)));
+    const int16x4_t v_round =
+        vbsl_s16(v_round_select, v_round_log_scale, v_round_no_scale);
+    int32x4_t v_round_s32 = vaddl_s16(v_round, v_zero);
+    int32x4_t v_quant_s32 = vshlq_n_s32(vaddl_s16(v_quant, v_zero), 15);
+    int32x4_t v_dequant_s32 = vaddl_s16(v_dequant, v_zero);
+    const int32x4_t v_log_scale = vdupq_n_s32(log_scale);
+    const int32x4_t v_neg_log_scale = vnegq_s32(v_log_scale);
+    const int32x4_t v_one_plus_log_scale = vdupq_n_s32(1 + log_scale);

-  // overwrite the DC constants with AC constants
-  v_round_s32 = vdupq_lane_s32(vget_low_s32(v_round_s32), 1);
-  v_quant_s32 = vdupq_lane_s32(vget_low_s32(v_quant_s32), 1);
-  v_dequant_s32 = vdupq_lane_s32(vget_low_s32(v_dequant_s32), 1);
+    // DC and first 3 AC
+    const int32x4_t v_dc_coeff = vld1q_s32(coeff_ptr);
+    v_mask_lo = quantize_4_from_abs(v_dc_coeff, vabsq_s32(v_dc_coeff),
+                                    qcoeff_ptr, dqcoeff_ptr, v_quant_s32,
+                                    v_dequant_s32, v_round_s32, v_log_scale,
+                                    v_neg_log_scale, v_one_plus_log_scale);

-  // 4 more AC
-  v_mask_hi = quantize_4(coeff_ptr + 4, qcoeff_ptr + 4, dqcoeff_ptr + 4,
-                         v_quant_s32, v_dequant_s32, v_round_s32, log_scale);
+    // overwrite the DC constants with AC constants
+    v_round_s32 = vdupq_lane_s32(vget_low_s32(v_round_s32), 1);
+    v_quant_s32 = vdupq_lane_s32(vget_low_s32(v_quant_s32), 1);
+    v_dequant_s32 = vdupq_lane_s32(vget_low_s32(v_dequant_s32), 1);

-  // Find the max lane eob for the first 8 coeffs.
-  v_eobmax =
-      get_max_lane_eob(iscan, v_eobmax, vcombine_u16(v_mask_lo, v_mask_hi));
+    // 4 more AC
+    const int32x4_t v_ac_coeff = vld1q_s32(coeff_ptr + 4);
+    v_mask_hi = quantize_4_from_abs(
+        v_ac_coeff, vabsq_s32(v_ac_coeff), qcoeff_ptr + 4, dqcoeff_ptr + 4,
+        v_quant_s32, v_dequant_s32, v_round_s32, v_log_scale, v_neg_log_scale,
+        v_one_plus_log_scale);

-  count -= 8;
-  do {
-    coeff_ptr += 8;
-    qcoeff_ptr += 8;
-    dqcoeff_ptr += 8;
-    iscan += 8;
-    v_mask_lo = quantize_4(coeff_ptr, qcoeff_ptr, dqcoeff_ptr, v_quant_s32,
-                           v_dequant_s32, v_round_s32, log_scale);
-    v_mask_hi = quantize_4(coeff_ptr + 4, qcoeff_ptr + 4, dqcoeff_ptr + 4,
-                           v_quant_s32, v_dequant_s32, v_round_s32, log_scale);
-    // Find the max lane eob for 8 coeffs.
+    // Find the max lane eob for the first 8 coeffs.
     v_eobmax =
         get_max_lane_eob(iscan, v_eobmax, vcombine_u16(v_mask_lo, v_mask_hi));
+
     count -= 8;
-  } while (count);
+    do {
+      coeff_ptr += 8;
+      qcoeff_ptr += 8;
+      dqcoeff_ptr += 8;
+      iscan += 8;
+      const int32x4_t v_c0 = vld1q_s32(coeff_ptr);
+      const int32x4_t v_c1 = vld1q_s32(coeff_ptr + 4);
+      const int32x4_t v_abs0 = vabsq_s32(v_c0);
+      const int32x4_t v_abs1 = vabsq_s32(v_c1);
+      if (!hbd_zero_skip_u32(v_abs0, v_abs1, ac_thresh, qcoeff_ptr,
+                             dqcoeff_ptr)) {
+        v_mask_lo = quantize_4_from_abs(
+            v_c0, v_abs0, qcoeff_ptr, dqcoeff_ptr, v_quant_s32, v_dequant_s32,
+            v_round_s32, v_log_scale, v_neg_log_scale, v_one_plus_log_scale);
+        v_mask_hi = quantize_4_from_abs(v_c1, v_abs1, qcoeff_ptr + 4,
+                                        dqcoeff_ptr + 4, v_quant_s32,
+                                        v_dequant_s32, v_round_s32, v_log_scale,
+                                        v_neg_log_scale, v_one_plus_log_scale);
+        v_eobmax = get_max_lane_eob(iscan, v_eobmax,
+                                    vcombine_u16(v_mask_lo, v_mask_hi));
+      }
+      count -= 8;
+    } while (count != 0);
+  }
+
+  *eob_ptr = get_max_eob(v_eobmax);
+}
+#undef HBD_QUANTIZE_BODY
+
+#if CONFIG_AV1_HIGHBITDEPTH
+static AOM_FORCE_INLINE int mask_any_nonzero_u32(uint32x4_t m0, uint32x4_t m1) {
+  const uint32x4_t any = vorrq_u32(m0, m1);
+#if AOM_ARCH_AARCH64
+  return vmaxvq_u32(any) != 0;
+#else
+  const uint64x2_t u64 = vreinterpretq_u64_u32(any);
+  return (vgetq_lane_u64(u64, 0) | vgetq_lane_u64(u64, 1)) != 0;
+#endif
+}

+static AOM_FORCE_INLINE int16x8_t update_qm_eob(const int16_t *iscan_ptr,
+                                                int16x8_t v_eobmax,
+                                                uint32x4_t nz_mask0,
+                                                uint32x4_t nz_mask1) {
+#if AOM_ARCH_AARCH64
+  const uint16x8_t v_nz_mask = vuzp1q_u16(vreinterpretq_u16_u32(nz_mask0),
+                                          vreinterpretq_u16_u32(nz_mask1));
+#else
+  const uint16x8_t v_nz_mask =
+      vcombine_u16(vmovn_u32(nz_mask0), vmovn_u32(nz_mask1));
+#endif
+  return get_max_lane_eob(iscan_ptr, v_eobmax, v_nz_mask);
+}
+
+static AOM_FORCE_INLINE int16x8_t highbd_quantize_fp_qm_8_neon(
+    const tran_low_t *coeff_ptr, const int16_t *iscan_ptr,
+    const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, tran_low_t *qcoeff_ptr,
+    tran_low_t *dqcoeff_ptr, int32x4_t v_thr0, int32x4_t v_thr1,
+    int32x4_t v_round0, int32x4_t v_round1, int32x4_t v_quant0,
+    int32x4_t v_quant1, int32x4_t v_dequant0, int32x4_t v_dequant1,
+    int32x4_t v_log_scale, int32x4_t v_neg_log_scale, int16x8_t v_eobmax) {
+  const int32x4_t v_coeff0 = vld1q_s32(coeff_ptr);
+  const int32x4_t v_coeff1 = vld1q_s32(coeff_ptr + 4);
+  const int32x4_t v_abs0 = vabsq_s32(v_coeff0);
+  const int32x4_t v_abs1 = vabsq_s32(v_coeff1);
+  const uint16x8_t v_wt_u16 = vmovl_u8(vld1_u8(qm_ptr));
+  const int32x4_t v_wt0 =
+      vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(v_wt_u16)));
+  const int32x4_t v_wt1 =
+      vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(v_wt_u16)));
+  const uint32x4_t v_mask0 = vcgeq_s32(vmulq_s32(v_abs0, v_wt0), v_thr0);
+  const uint32x4_t v_mask1 = vcgeq_s32(vmulq_s32(v_abs1, v_wt1), v_thr1);
+
+  if (mask_any_nonzero_u32(v_mask0, v_mask1)) {
+    const int32x4_t v_zero = vdupq_n_s32(0);
+    const int32x4_t v_sign0 = vshrq_n_s32(v_coeff0, 31);
+    const int32x4_t v_sign1 = vshrq_n_s32(v_coeff1, 31);
+    const int32x4_t v_tmp0 =
+        vandq_s32(vaddq_s32(v_abs0, v_round0), vreinterpretq_s32_u32(v_mask0));
+    const int32x4_t v_tmp1 =
+        vandq_s32(vaddq_s32(v_abs1, v_round1), vreinterpretq_s32_u32(v_mask1));
+    // Forward QM wt < 256 (AOM_QM_BITS = 5) and log_scale <= 2 fit in 32 bits.
+    const int32x4_t v_tmp_wt0 = vmulq_s32(v_tmp0, v_wt0);
+    const int32x4_t v_tmp_wt1 = vmulq_s32(v_tmp1, v_wt1);
+    const int32x4_t v_abs_q0 = vshrq_n_s32(
+        vqdmulhq_s32(vshlq_s32(v_tmp_wt0, v_log_scale), v_quant0), AOM_QM_BITS);
+    const int32x4_t v_abs_q1 = vshrq_n_s32(
+        vqdmulhq_s32(vshlq_s32(v_tmp_wt1, v_log_scale), v_quant1), AOM_QM_BITS);
+
+    const uint16x8_t v_iwt_u16 = vmovl_u8(vld1_u8(iqm_ptr));
+    const int32x4_t v_iwt0 =
+        vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(v_iwt_u16)));
+    const int32x4_t v_iwt1 =
+        vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(v_iwt_u16)));
+    const int32x4_t v_dq_rnd = vdupq_n_s32(1 << (AOM_QM_BITS - 1));
+    const int32x4_t v_dq0 =
+        vshrq_n_s32(vmlaq_s32(v_dq_rnd, v_dequant0, v_iwt0), AOM_QM_BITS);
+    const int32x4_t v_dq1 =
+        vshrq_n_s32(vmlaq_s32(v_dq_rnd, v_dequant1, v_iwt1), AOM_QM_BITS);
+    const int32x4_t v_abs_dq0 =
+        vshlq_s32(vmulq_s32(v_abs_q0, v_dq0), v_neg_log_scale);
+    const int32x4_t v_abs_dq1 =
+        vshlq_s32(vmulq_s32(v_abs_q1, v_dq1), v_neg_log_scale);
+
+    const int32x4_t v_q0 = vsubq_s32(veorq_s32(v_abs_q0, v_sign0), v_sign0);
+    const int32x4_t v_q1 = vsubq_s32(veorq_s32(v_abs_q1, v_sign1), v_sign1);
+    const int32x4_t v_dq_out0 =
+        vsubq_s32(veorq_s32(v_abs_dq0, v_sign0), v_sign0);
+    const int32x4_t v_dq_out1 =
+        vsubq_s32(veorq_s32(v_abs_dq1, v_sign1), v_sign1);
+    vst1q_s32(qcoeff_ptr, v_q0);
+    vst1q_s32(qcoeff_ptr + 4, v_q1);
+    vst1q_s32(dqcoeff_ptr, v_dq_out0);
+    vst1q_s32(dqcoeff_ptr + 4, v_dq_out1);
+
+    const uint32x4_t v_nz0 = vcgtq_s32(v_abs_q0, v_zero);
+    const uint32x4_t v_nz1 = vcgtq_s32(v_abs_q1, v_zero);
+    return update_qm_eob(iscan_ptr, v_eobmax, v_nz0, v_nz1);
+  }
+  const int32x4_t v_zero = vdupq_n_s32(0);
+  vst1q_s32(qcoeff_ptr, v_zero);
+  vst1q_s32(qcoeff_ptr + 4, v_zero);
+  vst1q_s32(dqcoeff_ptr, v_zero);
+  vst1q_s32(dqcoeff_ptr + 4, v_zero);
+  return v_eobmax;
+}
+
+void av1_highbd_quantize_fp_qm_neon(
+    const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr,
+    const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
+    const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan,
+    const int16_t *iscan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr,
+    int log_scale) {
+  (void)scan;
+  assert(qm_ptr != NULL && iqm_ptr != NULL);
+  assert(n_coeffs >= 16 && (n_coeffs & 7) == 0);
+  const int thr_shift = AOM_QM_BITS - (1 + log_scale);
+  const int32x4_t v_thr_ac = vdupq_n_s32((int32_t)dequant_ptr[1] << thr_shift);
+  const int32x4_t v_thr0 =
+      vsetq_lane_s32((int32_t)dequant_ptr[0] << thr_shift, v_thr_ac, 0);
+  const int32x4_t v_round_ac =
+      vdupq_n_s32(ROUND_POWER_OF_TWO(round_ptr[1], log_scale));
+  const int32x4_t v_round0 = vsetq_lane_s32(
+      ROUND_POWER_OF_TWO(round_ptr[0], log_scale), v_round_ac, 0);
+  const int32x4_t v_quant_ac =
+      vdupq_n_s32((int32_t)((uint32_t)quant_ptr[1] << 15));
+  const int32x4_t v_quant0 =
+      vsetq_lane_s32((int32_t)((uint32_t)quant_ptr[0] << 15), v_quant_ac, 0);
+  const int32x4_t v_dequant_ac = vdupq_n_s32(dequant_ptr[1]);
+  const int32x4_t v_dequant0 = vsetq_lane_s32(dequant_ptr[0], v_dequant_ac, 0);
+  const int32x4_t v_log_scale = vdupq_n_s32(log_scale);
+  const int32x4_t v_neg_log_scale = vdupq_n_s32(-log_scale);
+  int16x8_t v_eobmax = vdupq_n_s16(-1);
+
+  v_eobmax = highbd_quantize_fp_qm_8_neon(
+      coeff_ptr, iscan, qm_ptr, iqm_ptr, qcoeff_ptr, dqcoeff_ptr, v_thr0,
+      v_thr_ac, v_round0, v_round_ac, v_quant0, v_quant_ac, v_dequant0,
+      v_dequant_ac, v_log_scale, v_neg_log_scale, v_eobmax);
+
+  intptr_t i = 8;
+  do {
+    v_eobmax = highbd_quantize_fp_qm_8_neon(
+        coeff_ptr + i, iscan + i, qm_ptr + i, iqm_ptr + i, qcoeff_ptr + i,
+        dqcoeff_ptr + i, v_thr_ac, v_thr_ac, v_round_ac, v_round_ac, v_quant_ac,
+        v_quant_ac, v_dequant_ac, v_dequant_ac, v_log_scale, v_neg_log_scale,
+        v_eobmax);
+    i += 8;
+  } while (i != n_coeffs);
+  *eob_ptr = get_max_eob(v_eobmax);
+}
+
+static AOM_FORCE_INLINE int16x8_t highbd_quantize_b_qm_8_neon(
+    const tran_low_t *coeff_ptr, const int16_t *iscan_ptr,
+    const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, tran_low_t *qcoeff_ptr,
+    tran_low_t *dqcoeff_ptr, int32x4_t v_zbin0, int32x4_t v_zbin1,
+    int32x4_t v_round0, int32x4_t v_round1, int32x4_t v_quant0,
+    int32x4_t v_quant1, int32x4_t v_qshift0, int32x4_t v_qshift1,
+    int32x4_t v_dequant0, int32x4_t v_dequant1, int32x4_t v_log_scale,
+    int32x4_t v_neg_log_scale, int16x8_t v_eobmax) {
+  const int32x4_t v_coeff0 = vld1q_s32(coeff_ptr);
+  const int32x4_t v_coeff1 = vld1q_s32(coeff_ptr + 4);
+  const int32x4_t v_abs0 = vabsq_s32(v_coeff0);
+  const int32x4_t v_abs1 = vabsq_s32(v_coeff1);
+  const uint16x8_t v_wt_u16 = vmovl_u8(vld1_u8(qm_ptr));
+  const int32x4_t v_wt0 =
+      vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(v_wt_u16)));
+  const int32x4_t v_wt1 =
+      vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(v_wt_u16)));
+  const uint32x4_t v_mask0 = vcgeq_s32(vmulq_s32(v_abs0, v_wt0), v_zbin0);
+  const uint32x4_t v_mask1 = vcgeq_s32(vmulq_s32(v_abs1, v_wt1), v_zbin1);
+
+  if (mask_any_nonzero_u32(v_mask0, v_mask1)) {
+    const int32x4_t v_zero = vdupq_n_s32(0);
+    const int32x4_t v_sign0 = vshrq_n_s32(v_coeff0, 31);
+    const int32x4_t v_sign1 = vshrq_n_s32(v_coeff1, 31);
+    const int32x4_t v_tmp0 =
+        vandq_s32(vaddq_s32(v_abs0, v_round0), vreinterpretq_s32_u32(v_mask0));
+    const int32x4_t v_tmp1 =
+        vandq_s32(vaddq_s32(v_abs1, v_round1), vreinterpretq_s32_u32(v_mask1));
+    // Forward QM wt < 256 (AOM_QM_BITS = 5) and log_scale <= 2 fit in 32 bits.
+    const int32x4_t v_tmp_wt0 = vmulq_s32(v_tmp0, v_wt0);
+    const int32x4_t v_tmp_wt1 = vmulq_s32(v_tmp1, v_wt1);
+    const int32x4_t v_tmp2_0 =
+        vaddq_s32(vqdmulhq_s32(v_tmp_wt0, v_quant0), v_tmp_wt0);
+    const int32x4_t v_tmp2_1 =
+        vaddq_s32(vqdmulhq_s32(v_tmp_wt1, v_quant1), v_tmp_wt1);
+    const int32x4_t v_abs_q0 = vshrq_n_s32(
+        vqdmulhq_s32(vshlq_s32(v_tmp2_0, v_log_scale), v_qshift0), AOM_QM_BITS);
+    const int32x4_t v_abs_q1 = vshrq_n_s32(
+        vqdmulhq_s32(vshlq_s32(v_tmp2_1, v_log_scale), v_qshift1), AOM_QM_BITS);
+
+    const uint16x8_t v_iwt_u16 = vmovl_u8(vld1_u8(iqm_ptr));
+    const int32x4_t v_iwt0 =
+        vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(v_iwt_u16)));
+    const int32x4_t v_iwt1 =
+        vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(v_iwt_u16)));
+    const int32x4_t v_dq_rnd = vdupq_n_s32(1 << (AOM_QM_BITS - 1));
+    const int32x4_t v_dq0 =
+        vshrq_n_s32(vmlaq_s32(v_dq_rnd, v_dequant0, v_iwt0), AOM_QM_BITS);
+    const int32x4_t v_dq1 =
+        vshrq_n_s32(vmlaq_s32(v_dq_rnd, v_dequant1, v_iwt1), AOM_QM_BITS);
+    const int32x4_t v_abs_dq0 =
+        vshlq_s32(vmulq_s32(v_abs_q0, v_dq0), v_neg_log_scale);
+    const int32x4_t v_abs_dq1 =
+        vshlq_s32(vmulq_s32(v_abs_q1, v_dq1), v_neg_log_scale);
+
+    const int32x4_t v_q0 = vsubq_s32(veorq_s32(v_abs_q0, v_sign0), v_sign0);
+    const int32x4_t v_q1 = vsubq_s32(veorq_s32(v_abs_q1, v_sign1), v_sign1);
+    const int32x4_t v_dq_out0 =
+        vsubq_s32(veorq_s32(v_abs_dq0, v_sign0), v_sign0);
+    const int32x4_t v_dq_out1 =
+        vsubq_s32(veorq_s32(v_abs_dq1, v_sign1), v_sign1);
+    vst1q_s32(qcoeff_ptr, v_q0);
+    vst1q_s32(qcoeff_ptr + 4, v_q1);
+    vst1q_s32(dqcoeff_ptr, v_dq_out0);
+    vst1q_s32(dqcoeff_ptr + 4, v_dq_out1);
+
+    const uint32x4_t v_nz0 = vcgtq_s32(v_abs_q0, v_zero);
+    const uint32x4_t v_nz1 = vcgtq_s32(v_abs_q1, v_zero);
+    return update_qm_eob(iscan_ptr, v_eobmax, v_nz0, v_nz1);
+  }
+  const int32x4_t v_zero = vdupq_n_s32(0);
+  vst1q_s32(qcoeff_ptr, v_zero);
+  vst1q_s32(qcoeff_ptr + 4, v_zero);
+  vst1q_s32(dqcoeff_ptr, v_zero);
+  vst1q_s32(dqcoeff_ptr + 4, v_zero);
+  return v_eobmax;
+}
+
+void av1_highbd_quantize_b_qm_neon(
+    const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
+    const int16_t *round_ptr, const int16_t *quant_ptr,
+    const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
+    tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
+    const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr,
+    const qm_val_t *iqm_ptr, int log_scale) {
+  (void)scan;
+  assert(qm_ptr != NULL && iqm_ptr != NULL);
+  assert(n_coeffs >= 16 && (n_coeffs & 7) == 0);
+  const int32x4_t v_zbin_ac =
+      vdupq_n_s32(ROUND_POWER_OF_TWO(zbin_ptr[1], log_scale) << AOM_QM_BITS);
+  const int32x4_t v_zbin0 = vsetq_lane_s32(
+      ROUND_POWER_OF_TWO(zbin_ptr[0], log_scale) << AOM_QM_BITS, v_zbin_ac, 0);
+  const int32x4_t v_round_ac =
+      vdupq_n_s32(ROUND_POWER_OF_TWO(round_ptr[1], log_scale));
+  const int32x4_t v_round0 = vsetq_lane_s32(
+      ROUND_POWER_OF_TWO(round_ptr[0], log_scale), v_round_ac, 0);
+  const int32x4_t v_quant_ac =
+      vdupq_n_s32((int32_t)((uint32_t)quant_ptr[1] << 15));
+  const int32x4_t v_quant0 =
+      vsetq_lane_s32((int32_t)((uint32_t)quant_ptr[0] << 15), v_quant_ac, 0);
+  const int32x4_t v_qshift_ac =
+      vdupq_n_s32((int32_t)((uint32_t)quant_shift_ptr[1] << 15));
+  const int32x4_t v_qshift0 = vsetq_lane_s32(
+      (int32_t)((uint32_t)quant_shift_ptr[0] << 15), v_qshift_ac, 0);
+  const int32x4_t v_dequant_ac = vdupq_n_s32(dequant_ptr[1]);
+  const int32x4_t v_dequant0 = vsetq_lane_s32(dequant_ptr[0], v_dequant_ac, 0);
+  const int32x4_t v_log_scale = vdupq_n_s32(log_scale);
+  const int32x4_t v_neg_log_scale = vdupq_n_s32(-log_scale);
+  int16x8_t v_eobmax = vdupq_n_s16(-1);
+
+  v_eobmax = highbd_quantize_b_qm_8_neon(
+      coeff_ptr, iscan, qm_ptr, iqm_ptr, qcoeff_ptr, dqcoeff_ptr, v_zbin0,
+      v_zbin_ac, v_round0, v_round_ac, v_quant0, v_quant_ac, v_qshift0,
+      v_qshift_ac, v_dequant0, v_dequant_ac, v_log_scale, v_neg_log_scale,
+      v_eobmax);
+
+  intptr_t i = 8;
+  do {
+    v_eobmax = highbd_quantize_b_qm_8_neon(
+        coeff_ptr + i, iscan + i, qm_ptr + i, iqm_ptr + i, qcoeff_ptr + i,
+        dqcoeff_ptr + i, v_zbin_ac, v_zbin_ac, v_round_ac, v_round_ac,
+        v_quant_ac, v_quant_ac, v_qshift_ac, v_qshift_ac, v_dequant_ac,
+        v_dequant_ac, v_log_scale, v_neg_log_scale, v_eobmax);
+    i += 8;
+  } while (i != n_coeffs);
   *eob_ptr = get_max_eob(v_eobmax);
 }
+#endif  // CONFIG_AV1_HIGHBITDEPTH
diff --git a/av1/encoder/arm/quantize_neon.c b/av1/encoder/arm/quantize_neon.c
index e9c0ca7619..e3a0f7566a 100644
--- a/av1/encoder/arm/quantize_neon.c
+++ b/av1/encoder/arm/quantize_neon.c
@@ -15,6 +15,7 @@
 #include <math.h>

 #include "config/aom_config.h"
+#include "config/av1_rtcd.h"

 #include "aom_dsp/arm/mem_neon.h"
 #include "aom_dsp/arm/sum_neon.h"
@@ -27,6 +28,20 @@
 #include "av1/encoder/encoder.h"
 #include "av1/encoder/rd.h"

+static AOM_FORCE_INLINE uint64_t mask_nonzero_u16(uint16x8_t v_mask) {
+#if AOM_ARCH_AARCH64
+  return vmaxvq_u32(vreinterpretq_u32_u16(v_mask));
+#else
+  return vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_mask)), 0);
+#endif
+}
+
+static AOM_FORCE_INLINE void store_zero_8_tran_low(tran_low_t *ptr) {
+  const int32x4_t zero = vdupq_n_s32(0);
+  vst1q_s32(ptr, zero);
+  vst1q_s32(ptr + 4, zero);
+}
+
 static inline uint16_t get_max_eob(int16x8_t v_eobmax) {
 #if AOM_ARCH_AARCH64
   int16_t max_val = vmaxvq_s16(v_eobmax);
@@ -54,23 +69,29 @@ static inline int16x8_t get_max_lane_eob(const int16_t *iscan,
   return vmaxq_s16(v_eobmax, v_nz_iscan);
 }

-static inline uint16x8_t quantize_fp_8(const tran_low_t *coeff_ptr,
-                                       tran_low_t *qcoeff_ptr,
-                                       tran_low_t *dqcoeff_ptr,
-                                       int16x8_t v_quant, int16x8_t v_dequant,
-                                       int16x8_t v_round, int16x8_t v_zero) {
+static AOM_FORCE_INLINE int16x8_t
+quantize_fp_8(const tran_low_t *coeff_ptr, tran_low_t *qcoeff_ptr,
+              tran_low_t *dqcoeff_ptr, const int16_t *iscan, int16x8_t v_quant,
+              int16x8_t v_dequant, int16x8_t v_round, uint16x8_t v_thresh,
+              int16x8_t v_eobmax) {
   const int16x8_t v_coeff = load_tran_low_to_s16q(&coeff_ptr[0]);
-  const int16x8_t v_coeff_sign = vshrq_n_s16(v_coeff, 15);
   const int16x8_t v_abs = vabsq_s16(v_coeff);
-  const int16x8_t v_tmp = vqaddq_s16(v_abs, v_round);
-  const int16x8_t v_tmp2 = vshrq_n_s16(vqdmulhq_s16(v_tmp, v_quant), 1);
-  const uint16x8_t v_nz_mask = vcgtq_s16(v_tmp2, v_zero);
-  const int16x8_t v_qcoeff_a = veorq_s16(v_tmp2, v_coeff_sign);
-  const int16x8_t v_qcoeff = vsubq_s16(v_qcoeff_a, v_coeff_sign);
-  const int16x8_t v_dqcoeff = vmulq_s16(v_qcoeff, v_dequant);
-  store_s16q_to_tran_low(&qcoeff_ptr[0], v_qcoeff);
-  store_s16q_to_tran_low(&dqcoeff_ptr[0], v_dqcoeff);
-  return v_nz_mask;
+  const uint16x8_t v_mask = vcgeq_u16(vreinterpretq_u16_s16(v_abs), v_thresh);
+  if (mask_nonzero_u16(v_mask)) {
+    const int16x8_t v_coeff_sign = vshrq_n_s16(v_coeff, 15);
+    const int16x8_t v_tmp = vqaddq_s16(v_abs, v_round);
+    const int16x8_t v_tmp2 = vshrq_n_s16(vqdmulhq_s16(v_tmp, v_quant), 1);
+    const uint16x8_t v_nz_mask = vcgtq_s16(v_tmp2, vdupq_n_s16(0));
+    const int16x8_t v_qcoeff_a = veorq_s16(v_tmp2, v_coeff_sign);
+    const int16x8_t v_qcoeff = vsubq_s16(v_qcoeff_a, v_coeff_sign);
+    const int16x8_t v_dqcoeff = vmulq_s16(v_qcoeff, v_dequant);
+    store_s16q_to_tran_low(&qcoeff_ptr[0], v_qcoeff);
+    store_s16q_to_tran_low(&dqcoeff_ptr[0], v_dqcoeff);
+    return get_max_lane_eob(iscan, v_eobmax, v_nz_mask);
+  }
+  store_zero_8_tran_low(&qcoeff_ptr[0]);
+  store_zero_8_tran_low(&dqcoeff_ptr[0]);
+  return v_eobmax;
 }

 void av1_quantize_fp_neon(const tran_low_t *coeff_ptr, intptr_t count,
@@ -85,23 +106,24 @@ void av1_quantize_fp_neon(const tran_low_t *coeff_ptr, intptr_t count,
   (void)zbin_ptr;
   (void)quant_shift_ptr;
   (void)scan;
+  assert(count >= 16 && (count & 7) == 0);

-  // Quantization pass: All coefficients with index >= zero_flag are
-  // skippable. Note: zero_flag can be zero.
-  const int16x8_t v_zero = vdupq_n_s16(0);
   int16x8_t v_quant = vld1q_s16(quant_ptr);
   int16x8_t v_dequant = vld1q_s16(dequant_ptr);
   int16x8_t v_round = vld1q_s16(round_ptr);
+  uint16x8_t v_thresh = vreinterpretq_u16_s16(vshrq_n_s16(v_dequant, 1));
   int16x8_t v_eobmax_76543210 = vdupq_n_s16(-1);
-  uint16x8_t v_nz_mask;
+
   // process dc and the first seven ac coeffs
-  v_nz_mask = quantize_fp_8(coeff_ptr, qcoeff_ptr, dqcoeff_ptr, v_quant,
-                            v_dequant, v_round, v_zero);
-  v_eobmax_76543210 = get_max_lane_eob(&iscan[0], v_eobmax_76543210, v_nz_mask);
+  v_eobmax_76543210 =
+      quantize_fp_8(coeff_ptr, qcoeff_ptr, dqcoeff_ptr, iscan, v_quant,
+                    v_dequant, v_round, v_thresh, v_eobmax_76543210);
+
   // overwrite the dc constants with ac constants
   v_quant = vdupq_lane_s16(vget_low_s16(v_quant), 1);
   v_dequant = vdupq_lane_s16(vget_low_s16(v_dequant), 1);
   v_round = vdupq_lane_s16(vget_low_s16(v_round), 1);
+  v_thresh = vdupq_lane_u16(vget_low_u16(v_thresh), 1);

   count -= 8;
   // now process the rest of the ac coeffs
@@ -110,11 +132,11 @@ void av1_quantize_fp_neon(const tran_low_t *coeff_ptr, intptr_t count,
     qcoeff_ptr += 8;
     dqcoeff_ptr += 8;
     iscan += 8;
-    v_nz_mask = quantize_fp_8(coeff_ptr, qcoeff_ptr, dqcoeff_ptr, v_quant,
-                              v_dequant, v_round, v_zero);
-    v_eobmax_76543210 = get_max_lane_eob(iscan, v_eobmax_76543210, v_nz_mask);
+    v_eobmax_76543210 =
+        quantize_fp_8(coeff_ptr, qcoeff_ptr, dqcoeff_ptr, iscan, v_quant,
+                      v_dequant, v_round, v_thresh, v_eobmax_76543210);
     count -= 8;
-  } while (count > 0);
+  } while (count != 0);
   *eob_ptr = get_max_eob(v_eobmax_76543210);
 }

@@ -177,69 +199,78 @@ void av1_quantize_lp_neon(const int16_t *coeff_ptr, intptr_t n_coeffs,
   *eob_ptr = get_max_eob(v_eobmax_76543210);
 }

-static AOM_FORCE_INLINE uint16x8_t quantize_fp_logscale_8(
+static AOM_FORCE_INLINE int16x8_t quantize_fp_logscale_8(
     const tran_low_t *coeff_ptr, tran_low_t *qcoeff_ptr,
-    tran_low_t *dqcoeff_ptr, int16x8_t v_quant, int16x8_t v_dequant,
-    int16x8_t v_round, int16x8_t v_zero, int log_scale) {
-  const int16x8_t v_log_scale_minus_1 = vdupq_n_s16(log_scale - 1);
-  const int16x8_t v_neg_log_scale_plus_1 = vdupq_n_s16(-(1 + log_scale));
+    tran_low_t *dqcoeff_ptr, const int16_t *iscan, int16x8_t v_quant,
+    int16x8_t v_dequant, int16x8_t v_round, uint16x8_t v_thresh,
+    int16x8_t v_eobmax, int log_scale) {
   const int16x8_t v_coeff = load_tran_low_to_s16q(coeff_ptr);
-  const int16x8_t v_coeff_sign = vshrq_n_s16(v_coeff, 15);
   const int16x8_t v_abs_coeff = vabsq_s16(v_coeff);
   const uint16x8_t v_mask =
-      vcgeq_s16(v_abs_coeff, vshlq_s16(v_dequant, v_neg_log_scale_plus_1));
-  // const int64_t tmp = vmask ? (int64_t)abs_coeff + log_scaled_round : 0
-  const int16x8_t v_tmp = vandq_s16(vqaddq_s16(v_abs_coeff, v_round),
-                                    vreinterpretq_s16_u16(v_mask));
-  const int16x8_t v_tmp2 =
-      vqdmulhq_s16(vshlq_s16(v_tmp, v_log_scale_minus_1), v_quant);
-  const uint16x8_t v_nz_mask = vcgtq_s16(v_tmp2, v_zero);
-  const int16x8_t v_qcoeff =
-      vsubq_s16(veorq_s16(v_tmp2, v_coeff_sign), v_coeff_sign);
-  // Multiplying by dequant here will use all 16 bits. Cast to unsigned before
-  // shifting right. (vshlq_s16 will shift right if shift value is negative)
-  const uint16x8_t v_abs_dqcoeff =
-      vshlq_u16(vreinterpretq_u16_s16(vmulq_s16(v_tmp2, v_dequant)),
-                vdupq_n_s16(-log_scale));
-  const int16x8_t v_dqcoeff =
-      vsubq_s16(veorq_s16(vreinterpretq_s16_u16(v_abs_dqcoeff), v_coeff_sign),
-                v_coeff_sign);
-  store_s16q_to_tran_low(qcoeff_ptr, v_qcoeff);
-  store_s16q_to_tran_low(dqcoeff_ptr, v_dqcoeff);
-  return v_nz_mask;
+      vcgeq_u16(vreinterpretq_u16_s16(v_abs_coeff), v_thresh);
+  if (mask_nonzero_u16(v_mask)) {
+    const int16x8_t v_coeff_sign = vshrq_n_s16(v_coeff, 15);
+    // const int64_t tmp = vmask ? (int64_t)abs_coeff + log_scaled_round : 0
+    const int16x8_t v_tmp = vandq_s16(vqaddq_s16(v_abs_coeff, v_round),
+                                      vreinterpretq_s16_u16(v_mask));
+    const int16x8_t v_tmp2 =
+        vqdmulhq_s16(vshlq_s16(v_tmp, vdupq_n_s16(log_scale - 1)), v_quant);
+    const uint16x8_t v_nz_mask = vcgtq_s16(v_tmp2, vdupq_n_s16(0));
+    const int16x8_t v_qcoeff =
+        vsubq_s16(veorq_s16(v_tmp2, v_coeff_sign), v_coeff_sign);
+    // Multiplying by dequant here will use all 16 bits. Cast to unsigned
+    // before shifting right.
+    const uint16x8_t v_abs_dqcoeff =
+        vshlq_u16(vreinterpretq_u16_s16(vmulq_s16(v_tmp2, v_dequant)),
+                  vdupq_n_s16(-log_scale));
+    const int16x8_t v_dqcoeff =
+        vsubq_s16(veorq_s16(vreinterpretq_s16_u16(v_abs_dqcoeff), v_coeff_sign),
+                  v_coeff_sign);
+    store_s16q_to_tran_low(qcoeff_ptr, v_qcoeff);
+    store_s16q_to_tran_low(dqcoeff_ptr, v_dqcoeff);
+    return get_max_lane_eob(iscan, v_eobmax, v_nz_mask);
+  }
+  store_zero_8_tran_low(qcoeff_ptr);
+  store_zero_8_tran_low(dqcoeff_ptr);
+  return v_eobmax;
 }

-static AOM_FORCE_INLINE uint16x8_t quantize_fp_logscale2_8(
+static AOM_FORCE_INLINE int16x8_t quantize_fp_logscale2_8(
     const tran_low_t *coeff_ptr, tran_low_t *qcoeff_ptr,
-    tran_low_t *dqcoeff_ptr, int16x8_t v_quant, int16x8_t v_dequant,
-    int16x8_t v_round, int16x8_t v_zero) {
+    tran_low_t *dqcoeff_ptr, const int16_t *iscan, int16x8_t v_quant,
+    int16x8_t v_dequant, int16x8_t v_round, uint16x8_t v_thresh,
+    int16x8_t v_eobmax) {
   const int16x8_t v_coeff = load_tran_low_to_s16q(coeff_ptr);
-  const int16x8_t v_coeff_sign = vshrq_n_s16(v_coeff, 15);
   const int16x8_t v_abs_coeff = vabsq_s16(v_coeff);
   const uint16x8_t v_mask =
-      vcgeq_u16(vshlq_n_u16(vreinterpretq_u16_s16(v_abs_coeff), 1),
-                vshrq_n_u16(vreinterpretq_u16_s16(v_dequant), 2));
-  // abs_coeff = vmask ? (int64_t)abs_coeff + log_scaled_round : 0
-  const int16x8_t v_tmp = vandq_s16(vqaddq_s16(v_abs_coeff, v_round),
-                                    vreinterpretq_s16_u16(v_mask));
-  // tmp32 = (int)((abs_coeff * quant_ptr[rc != 0]) >> (16 - log_scale));
-  const int16x8_t v_tmp2 =
-      vorrq_s16(vshlq_n_s16(vqdmulhq_s16(v_tmp, v_quant), 1),
-                vreinterpretq_s16_u16(vshrq_n_u16(
-                    vreinterpretq_u16_s16(vmulq_s16(v_tmp, v_quant)), 14)));
-  const uint16x8_t v_nz_mask = vcgtq_s16(v_tmp2, v_zero);
-  const int16x8_t v_qcoeff =
-      vsubq_s16(veorq_s16(v_tmp2, v_coeff_sign), v_coeff_sign);
-  // const tran_low_t abs_dqcoeff = (tmp32 * dequant_ptr[rc != 0]) >> log_scale;
-  const int16x8_t v_abs_dqcoeff =
-      vorrq_s16(vshlq_n_s16(vqdmulhq_s16(v_tmp2, v_dequant), 13),
-                vreinterpretq_s16_u16(vshrq_n_u16(
-                    vreinterpretq_u16_s16(vmulq_s16(v_tmp2, v_dequant)), 2)));
-  const int16x8_t v_dqcoeff =
-      vsubq_s16(veorq_s16(v_abs_dqcoeff, v_coeff_sign), v_coeff_sign);
-  store_s16q_to_tran_low(qcoeff_ptr, v_qcoeff);
-  store_s16q_to_tran_low(dqcoeff_ptr, v_dqcoeff);
-  return v_nz_mask;
+      vcgeq_u16(vreinterpretq_u16_s16(v_abs_coeff), v_thresh);
+  if (mask_nonzero_u16(v_mask)) {
+    const int16x8_t v_coeff_sign = vshrq_n_s16(v_coeff, 15);
+    // abs_coeff = vmask ? (int64_t)abs_coeff + log_scaled_round : 0
+    const int16x8_t v_tmp = vandq_s16(vqaddq_s16(v_abs_coeff, v_round),
+                                      vreinterpretq_s16_u16(v_mask));
+    // tmp32 = (int)((abs_coeff * quant_ptr[rc != 0]) >> (16 - log_scale));
+    const int16x8_t v_tmp2 =
+        vorrq_s16(vshlq_n_s16(vqdmulhq_s16(v_tmp, v_quant), 1),
+                  vreinterpretq_s16_u16(vshrq_n_u16(
+                      vreinterpretq_u16_s16(vmulq_s16(v_tmp, v_quant)), 14)));
+    const uint16x8_t v_nz_mask = vcgtq_s16(v_tmp2, vdupq_n_s16(0));
+    const int16x8_t v_qcoeff =
+        vsubq_s16(veorq_s16(v_tmp2, v_coeff_sign), v_coeff_sign);
+    // abs_dqcoeff = (tmp32 * dequant_ptr[rc != 0]) >> log_scale;
+    const int16x8_t v_abs_dqcoeff =
+        vorrq_s16(vshlq_n_s16(vqdmulhq_s16(v_tmp2, v_dequant), 13),
+                  vreinterpretq_s16_u16(vshrq_n_u16(
+                      vreinterpretq_u16_s16(vmulq_s16(v_tmp2, v_dequant)), 2)));
+    const int16x8_t v_dqcoeff =
+        vsubq_s16(veorq_s16(v_abs_dqcoeff, v_coeff_sign), v_coeff_sign);
+    store_s16q_to_tran_low(qcoeff_ptr, v_qcoeff);
+    store_s16q_to_tran_low(dqcoeff_ptr, v_dqcoeff);
+    return get_max_lane_eob(iscan, v_eobmax, v_nz_mask);
+  }
+  store_zero_8_tran_low(qcoeff_ptr);
+  store_zero_8_tran_low(dqcoeff_ptr);
+  return v_eobmax;
 }

 static AOM_FORCE_INLINE void quantize_fp_no_qmatrix_neon(
@@ -247,7 +278,7 @@ static AOM_FORCE_INLINE void quantize_fp_no_qmatrix_neon(
     const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
     const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *iscan,
     int log_scale) {
-  const int16x8_t v_zero = vdupq_n_s16(0);
+  assert(n_coeffs > 16 && (n_coeffs & 15) == 0);
   int16x8_t v_quant = vld1q_s16(quant_ptr);
   int16x8_t v_dequant = vld1q_s16(dequant_ptr);
   const int16x8_t v_round_no_scale = vld1q_s16(round_ptr);
@@ -256,66 +287,64 @@ static AOM_FORCE_INLINE void quantize_fp_no_qmatrix_neon(
   int16x8_t v_eobmax_76543210 = vdupq_n_s16(-1);
   intptr_t non_zero_count = n_coeffs;

-  assert(n_coeffs > 16);
-  // Pre-scan pass
-  const int16x8_t v_dequant_scaled =
-      vshlq_s16(v_dequant, vdupq_n_s16(-(1 + log_scale)));
-  const int16x8_t v_zbin_s16 =
-      vdupq_lane_s16(vget_low_s16(v_dequant_scaled), 1);
-  intptr_t i = n_coeffs;
+  // Pre-scan pass: compute exact ceil(dequant / 2^(1 + log_scale)) threshold.
+  const uint16x8_t v_bias = vdupq_n_u16((uint16_t)((1 << (1 + log_scale)) - 1));
+  uint16x8_t v_thresh =
+      vshlq_u16(vaddq_u16(vreinterpretq_u16_s16(v_dequant), v_bias),
+                vdupq_n_s16(-(1 + log_scale)));
+  const uint16x8_t v_zbin_u16 = vdupq_lane_u16(vget_low_u16(v_thresh), 1);
   do {
-    const int16x8_t v_coeff_a = load_tran_low_to_s16q(coeff_ptr + i - 8);
-    const int16x8_t v_coeff_b = load_tran_low_to_s16q(coeff_ptr + i - 16);
-    const int16x8_t v_abs_coeff_a = vabsq_s16(v_coeff_a);
-    const int16x8_t v_abs_coeff_b = vabsq_s16(v_coeff_b);
-    const uint16x8_t v_mask_a = vcgeq_s16(v_abs_coeff_a, v_zbin_s16);
-    const uint16x8_t v_mask_b = vcgeq_s16(v_abs_coeff_b, v_zbin_s16);
-    // If the coefficient is in the base ZBIN range, then discard.
-    if (horizontal_long_add_u16x8(v_mask_a, v_mask_b) == 0) {
-      non_zero_count -= 16;
-    } else {
+    const int16x8_t v_coeff_a =
+        load_tran_low_to_s16q(coeff_ptr + non_zero_count - 8);
+    const int16x8_t v_coeff_b =
+        load_tran_low_to_s16q(coeff_ptr + non_zero_count - 16);
+    const uint16x8_t v_abs_a = vreinterpretq_u16_s16(vabsq_s16(v_coeff_a));
+    const uint16x8_t v_abs_b = vreinterpretq_u16_s16(vabsq_s16(v_coeff_b));
+    const uint16x8_t v_mask_a = vcgeq_u16(v_abs_a, v_zbin_u16);
+    const uint16x8_t v_mask_b = vcgeq_u16(v_abs_b, v_zbin_u16);
+    if (mask_nonzero_u16(vorrq_u16(v_mask_a, v_mask_b))) {
       break;
     }
-    i -= 16;
-  } while (i > 0);
-
-  const intptr_t remaining_zcoeffs = n_coeffs - non_zero_count;
-  memset(qcoeff_ptr + non_zero_count, 0,
-         remaining_zcoeffs * sizeof(*qcoeff_ptr));
-  memset(dqcoeff_ptr + non_zero_count, 0,
-         remaining_zcoeffs * sizeof(*dqcoeff_ptr));
+    non_zero_count -= 16;
+    store_zero_8_tran_low(qcoeff_ptr + non_zero_count);
+    store_zero_8_tran_low(qcoeff_ptr + non_zero_count + 8);
+    store_zero_8_tran_low(dqcoeff_ptr + non_zero_count);
+    store_zero_8_tran_low(dqcoeff_ptr + non_zero_count + 8);
+  } while (non_zero_count != 16);

   // process dc and the first seven ac coeffs
-  uint16x8_t v_nz_mask;
   if (log_scale == 2) {
-    v_nz_mask = quantize_fp_logscale2_8(coeff_ptr, qcoeff_ptr, dqcoeff_ptr,
-                                        v_quant, v_dequant, v_round, v_zero);
+    v_eobmax_76543210 = quantize_fp_logscale2_8(
+        coeff_ptr, qcoeff_ptr, dqcoeff_ptr, iscan, v_quant, v_dequant, v_round,
+        v_thresh, v_eobmax_76543210);
   } else {
-    v_nz_mask =
-        quantize_fp_logscale_8(coeff_ptr, qcoeff_ptr, dqcoeff_ptr, v_quant,
-                               v_dequant, v_round, v_zero, log_scale);
+    v_eobmax_76543210 = quantize_fp_logscale_8(
+        coeff_ptr, qcoeff_ptr, dqcoeff_ptr, iscan, v_quant, v_dequant, v_round,
+        v_thresh, v_eobmax_76543210, log_scale);
   }
-  v_eobmax_76543210 = get_max_lane_eob(iscan, v_eobmax_76543210, v_nz_mask);
   // overwrite the dc constants with ac constants
   v_quant = vdupq_lane_s16(vget_low_s16(v_quant), 1);
   v_dequant = vdupq_lane_s16(vget_low_s16(v_dequant), 1);
   v_round = vdupq_lane_s16(vget_low_s16(v_round), 1);
+  v_thresh = v_zbin_u16;

-  for (intptr_t count = non_zero_count - 8; count > 0; count -= 8) {
+  intptr_t count = non_zero_count - 8;
+  do {
     coeff_ptr += 8;
     qcoeff_ptr += 8;
     dqcoeff_ptr += 8;
     iscan += 8;
     if (log_scale == 2) {
-      v_nz_mask = quantize_fp_logscale2_8(coeff_ptr, qcoeff_ptr, dqcoeff_ptr,
-                                          v_quant, v_dequant, v_round, v_zero);
+      v_eobmax_76543210 = quantize_fp_logscale2_8(
+          coeff_ptr, qcoeff_ptr, dqcoeff_ptr, iscan, v_quant, v_dequant,
+          v_round, v_thresh, v_eobmax_76543210);
     } else {
-      v_nz_mask =
-          quantize_fp_logscale_8(coeff_ptr, qcoeff_ptr, dqcoeff_ptr, v_quant,
-                                 v_dequant, v_round, v_zero, log_scale);
+      v_eobmax_76543210 = quantize_fp_logscale_8(
+          coeff_ptr, qcoeff_ptr, dqcoeff_ptr, iscan, v_quant, v_dequant,
+          v_round, v_thresh, v_eobmax_76543210, log_scale);
     }
-    v_eobmax_76543210 = get_max_lane_eob(iscan, v_eobmax_76543210, v_nz_mask);
-  }
+    count -= 8;
+  } while (count != 0);
   *eob_ptr = get_max_eob(v_eobmax_76543210);
 }

@@ -385,12 +414,11 @@ void aom_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
                          tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr,
                          uint16_t *eob_ptr, const int16_t *scan,
                          const int16_t *iscan) {
-  (void)quant_shift_ptr;
   (void)scan;

   int16x8_t v_eobmax_76543210 = vdupq_n_s16(-1);

-  int16x8_t v_zbins = vdupq_n_s16(zbin_ptr[1]);
+  const int16x8_t v_zbins = vdupq_n_s16(zbin_ptr[1]);
   int16x8_t v_round = vdupq_n_s16(round_ptr[1]);
   int16x8_t v_dequant = vdupq_n_s16(dequant_ptr[1]);
   int16x8_t v_quant = vdupq_n_s16(quant_ptr[1]);
@@ -402,34 +430,37 @@ void aom_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
   int16x8_t v_abs = vabsq_s16(v_coeff);
   uint16x8_t v_cond = vcgeq_s16(v_abs, v_zbins0);

-  uint64_t nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_cond)), 0);
-  if (nz_check) {
-    int16x8_t v_round0 = vsetq_lane_s16(round_ptr[0], v_round, 0);
-    int16x8_t v_quant0 = vsetq_lane_s16(quant_ptr[0], v_quant, 0);
-    int16x8_t v_dequant0 = vsetq_lane_s16(dequant_ptr[0], v_dequant, 0);
+  if (mask_nonzero_u16(v_cond)) {
+    v_round = vsetq_lane_s16(round_ptr[0], v_round, 0);
+    v_quant = vsetq_lane_s16(quant_ptr[0], v_quant, 0);
+    v_dequant = vsetq_lane_s16(dequant_ptr[0], v_dequant, 0);
     // Shift by 1 in order to save one shift in the kernel function.
-    int16x8_t v_quant_shift0 =
-        vsetq_lane_s16(quant_shift_ptr[0] >> 1, v_quant_shift, 0);
+    v_quant_shift = vsetq_lane_s16(quant_shift_ptr[0] >> 1, v_quant_shift, 0);

-    const uint16x8_t v_nz_mask = quantize_b_logscale0_8(
-        v_coeff, v_abs, v_cond, v_round0, v_dequant0, v_quant0, v_quant_shift0,
-        qcoeff_ptr, dqcoeff_ptr);
+    const uint16x8_t v_nz_mask =
+        quantize_b_logscale0_8(v_coeff, v_abs, v_cond, v_round, v_dequant,
+                               v_quant, v_quant_shift, qcoeff_ptr, dqcoeff_ptr);
+
+    v_round = vdupq_lane_s16(vget_low_s16(v_round), 1);
+    v_quant = vdupq_lane_s16(vget_low_s16(v_quant), 1);
+    v_dequant = vdupq_lane_s16(vget_low_s16(v_dequant), 1);
+    v_quant_shift = vdupq_lane_s16(vget_low_s16(v_quant_shift), 1);

     int16x8_t v_iscan = vld1q_s16(iscan);
     int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210);
     v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210);
   } else {
-    store_s16q_to_tran_low(qcoeff_ptr, vdupq_n_s16(0));
-    store_s16q_to_tran_low(dqcoeff_ptr, vdupq_n_s16(0));
+    store_zero_8_tran_low(qcoeff_ptr);
+    store_zero_8_tran_low(dqcoeff_ptr);
   }

-  for (int i = 8; i < n_coeffs; i += 8) {
+  intptr_t i = 8;
+  do {
     v_coeff = load_tran_low_to_s16q(coeff_ptr + i);
     v_abs = vabsq_s16(v_coeff);
     v_cond = vcgeq_s16(v_abs, v_zbins);

-    nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_cond)), 0);
-    if (nz_check) {
+    if (mask_nonzero_u16(v_cond)) {
       const uint16x8_t v_nz_mask = quantize_b_logscale0_8(
           v_coeff, v_abs, v_cond, v_round, v_dequant, v_quant, v_quant_shift,
           qcoeff_ptr + i, dqcoeff_ptr + i);
@@ -438,10 +469,11 @@ void aom_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
       int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210);
       v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210);
     } else {
-      store_s16q_to_tran_low(qcoeff_ptr + i, vdupq_n_s16(0));
-      store_s16q_to_tran_low(dqcoeff_ptr + i, vdupq_n_s16(0));
+      store_zero_8_tran_low(qcoeff_ptr + i);
+      store_zero_8_tran_low(dqcoeff_ptr + i);
     }
-  }
+    i += 8;
+  } while (i != n_coeffs);
   *eob_ptr = get_max_eob(v_eobmax_76543210);
 }

@@ -943,44 +975,66 @@ void aom_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,

   int16x8_t v_eobmax_76543210 = vdupq_n_s16(-1);

-  int16x8_t v_zbins = vdupq_n_s16(zbins[1]);
+  const int16x8_t v_zbins = vdupq_n_s16(zbins[1]);
   int16x8_t v_round = vdupq_n_s16(rounds[1]);
   int16x8_t v_dequant = vdupq_n_s16(dequant_ptr[1]);
   int16x8_t v_quant = vdupq_n_s16(quant_ptr[1]);
   int16x8_t v_quant_shift = vdupq_n_s16(quant_shift_ptr[1]);

+  assert(n_coeffs > 16 && (n_coeffs & 15) == 0);
+  intptr_t non_zero_count = n_coeffs;
+  do {
+    const int16x8_t v_coeff_a =
+        load_tran_low_to_s16q(coeff_ptr + non_zero_count - 8);
+    const int16x8_t v_coeff_b =
+        load_tran_low_to_s16q(coeff_ptr + non_zero_count - 16);
+    const uint16x8_t v_mask_a = vcgeq_s16(vabsq_s16(v_coeff_a), v_zbins);
+    const uint16x8_t v_mask_b = vcgeq_s16(vabsq_s16(v_coeff_b), v_zbins);
+    if (mask_nonzero_u16(vorrq_u16(v_mask_a, v_mask_b))) {
+      break;
+    }
+    non_zero_count -= 16;
+    store_zero_8_tran_low(qcoeff_ptr + non_zero_count);
+    store_zero_8_tran_low(qcoeff_ptr + non_zero_count + 8);
+    store_zero_8_tran_low(dqcoeff_ptr + non_zero_count);
+    store_zero_8_tran_low(dqcoeff_ptr + non_zero_count + 8);
+  } while (non_zero_count != 16);
+
   int16x8_t v_zbins0 = vsetq_lane_s16(zbins[0], v_zbins, 0);
   int16x8_t v_coeff = load_tran_low_to_s16q(coeff_ptr);
   int16x8_t v_abs = vabsq_s16(v_coeff);
   uint16x8_t v_cond = vcgeq_s16(v_abs, v_zbins0);

-  uint64_t nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_cond)), 0);
-  if (nz_check) {
-    int16x8_t v_round0 = vsetq_lane_s16(rounds[0], v_round, 0);
-    int16x8_t v_quant0 = vsetq_lane_s16(quant_ptr[0], v_quant, 0);
-    int16x8_t v_dequant0 = vsetq_lane_s16(dequant_ptr[0], v_dequant, 0);
-    int16x8_t v_quant_shift0 =
-        vsetq_lane_s16(quant_shift_ptr[0], v_quant_shift, 0);
+  if (mask_nonzero_u16(v_cond)) {
+    v_round = vsetq_lane_s16(rounds[0], v_round, 0);
+    v_quant = vsetq_lane_s16(quant_ptr[0], v_quant, 0);
+    v_dequant = vsetq_lane_s16(dequant_ptr[0], v_dequant, 0);
+    v_quant_shift = vsetq_lane_s16(quant_shift_ptr[0], v_quant_shift, 0);
+
+    const uint16x8_t v_nz_mask =
+        quantize_b_logscale1_8(v_coeff, v_abs, v_cond, v_round, v_dequant,
+                               v_quant, v_quant_shift, qcoeff_ptr, dqcoeff_ptr);

-    const uint16x8_t v_nz_mask = quantize_b_logscale1_8(
-        v_coeff, v_abs, v_cond, v_round0, v_dequant0, v_quant0, v_quant_shift0,
-        qcoeff_ptr, dqcoeff_ptr);
+    v_round = vdupq_lane_s16(vget_low_s16(v_round), 1);
+    v_quant = vdupq_lane_s16(vget_low_s16(v_quant), 1);
+    v_dequant = vdupq_lane_s16(vget_low_s16(v_dequant), 1);
+    v_quant_shift = vdupq_lane_s16(vget_low_s16(v_quant_shift), 1);

     int16x8_t v_iscan = vld1q_s16(iscan);
     int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210);
     v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210);
   } else {
-    store_s16q_to_tran_low(qcoeff_ptr, vdupq_n_s16(0));
-    store_s16q_to_tran_low(dqcoeff_ptr, vdupq_n_s16(0));
+    store_zero_8_tran_low(qcoeff_ptr);
+    store_zero_8_tran_low(dqcoeff_ptr);
   }

-  for (int i = 8; i < n_coeffs; i += 8) {
+  intptr_t i = 8;
+  do {
     v_coeff = load_tran_low_to_s16q(coeff_ptr + i);
     v_abs = vabsq_s16(v_coeff);
     v_cond = vcgeq_s16(v_abs, v_zbins);

-    nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_cond)), 0);
-    if (nz_check) {
+    if (mask_nonzero_u16(v_cond)) {
       const uint16x8_t v_nz_mask = quantize_b_logscale1_8(
           v_coeff, v_abs, v_cond, v_round, v_dequant, v_quant, v_quant_shift,
           qcoeff_ptr + i, dqcoeff_ptr + i);
@@ -989,10 +1043,11 @@ void aom_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
       int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210);
       v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210);
     } else {
-      store_s16q_to_tran_low(qcoeff_ptr + i, vdupq_n_s16(0));
-      store_s16q_to_tran_low(dqcoeff_ptr + i, vdupq_n_s16(0));
+      store_zero_8_tran_low(qcoeff_ptr + i);
+      store_zero_8_tran_low(dqcoeff_ptr + i);
     }
-  }
+    i += 8;
+  } while (i != non_zero_count);
   *eob_ptr = get_max_eob(v_eobmax_76543210);
 }

@@ -1047,44 +1102,66 @@ void aom_quantize_b_64x64_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,

   int16x8_t v_eobmax_76543210 = vdupq_n_s16(-1);

-  int16x8_t v_zbins = vdupq_n_s16(zbins[1]);
+  const int16x8_t v_zbins = vdupq_n_s16(zbins[1]);
   int16x8_t v_round = vdupq_n_s16(rounds[1]);
   int16x8_t v_dequant = vdupq_n_s16(dequant_ptr[1]);
   int16x8_t v_quant = vdupq_n_s16(quant_ptr[1]);
   int16x8_t v_quant_shift = vdupq_n_s16(quant_shift_ptr[1]);

+  assert(n_coeffs > 16 && (n_coeffs & 15) == 0);
+  intptr_t non_zero_count = n_coeffs;
+  do {
+    const int16x8_t v_coeff_a =
+        load_tran_low_to_s16q(coeff_ptr + non_zero_count - 8);
+    const int16x8_t v_coeff_b =
+        load_tran_low_to_s16q(coeff_ptr + non_zero_count - 16);
+    const uint16x8_t v_mask_a = vcgeq_s16(vabsq_s16(v_coeff_a), v_zbins);
+    const uint16x8_t v_mask_b = vcgeq_s16(vabsq_s16(v_coeff_b), v_zbins);
+    if (mask_nonzero_u16(vorrq_u16(v_mask_a, v_mask_b))) {
+      break;
+    }
+    non_zero_count -= 16;
+    store_zero_8_tran_low(qcoeff_ptr + non_zero_count);
+    store_zero_8_tran_low(qcoeff_ptr + non_zero_count + 8);
+    store_zero_8_tran_low(dqcoeff_ptr + non_zero_count);
+    store_zero_8_tran_low(dqcoeff_ptr + non_zero_count + 8);
+  } while (non_zero_count != 16);
+
   int16x8_t v_zbins0 = vsetq_lane_s16(zbins[0], v_zbins, 0);
   int16x8_t v_coeff = load_tran_low_to_s16q(coeff_ptr);
   int16x8_t v_abs = vabsq_s16(v_coeff);
   uint16x8_t v_cond = vcgeq_s16(v_abs, v_zbins0);

-  uint64_t nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_cond)), 0);
-  if (nz_check) {
-    int16x8_t v_round0 = vsetq_lane_s16(rounds[0], v_round, 0);
-    int16x8_t v_quant0 = vsetq_lane_s16(quant_ptr[0], v_quant, 0);
-    int16x8_t v_dequant0 = vsetq_lane_s16(dequant_ptr[0], v_dequant, 0);
-    int16x8_t v_quant_shift0 =
-        vsetq_lane_s16(quant_shift_ptr[0], v_quant_shift, 0);
+  if (mask_nonzero_u16(v_cond)) {
+    v_round = vsetq_lane_s16(rounds[0], v_round, 0);
+    v_quant = vsetq_lane_s16(quant_ptr[0], v_quant, 0);
+    v_dequant = vsetq_lane_s16(dequant_ptr[0], v_dequant, 0);
+    v_quant_shift = vsetq_lane_s16(quant_shift_ptr[0], v_quant_shift, 0);
+
+    const uint16x8_t v_nz_mask =
+        quantize_b_logscale2_8(v_coeff, v_abs, v_cond, v_round, v_dequant,
+                               v_quant, v_quant_shift, qcoeff_ptr, dqcoeff_ptr);

-    const uint16x8_t v_nz_mask = quantize_b_logscale2_8(
-        v_coeff, v_abs, v_cond, v_round0, v_dequant0, v_quant0, v_quant_shift0,
-        qcoeff_ptr, dqcoeff_ptr);
+    v_round = vdupq_lane_s16(vget_low_s16(v_round), 1);
+    v_quant = vdupq_lane_s16(vget_low_s16(v_quant), 1);
+    v_dequant = vdupq_lane_s16(vget_low_s16(v_dequant), 1);
+    v_quant_shift = vdupq_lane_s16(vget_low_s16(v_quant_shift), 1);

     int16x8_t v_iscan = vld1q_s16(iscan);
     int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210);
     v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210);
   } else {
-    store_s16q_to_tran_low(qcoeff_ptr, vdupq_n_s16(0));
-    store_s16q_to_tran_low(dqcoeff_ptr, vdupq_n_s16(0));
+    store_zero_8_tran_low(qcoeff_ptr);
+    store_zero_8_tran_low(dqcoeff_ptr);
   }

-  for (int i = 8; i < n_coeffs; i += 8) {
+  intptr_t i = 8;
+  do {
     v_coeff = load_tran_low_to_s16q(coeff_ptr + i);
     v_abs = vabsq_s16(v_coeff);
     v_cond = vcgeq_s16(v_abs, v_zbins);

-    nz_check = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_cond)), 0);
-    if (nz_check) {
+    if (mask_nonzero_u16(v_cond)) {
       const uint16x8_t v_nz_mask = quantize_b_logscale2_8(
           v_coeff, v_abs, v_cond, v_round, v_dequant, v_quant, v_quant_shift,
           qcoeff_ptr + i, dqcoeff_ptr + i);
@@ -1093,9 +1170,286 @@ void aom_quantize_b_64x64_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
       int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210);
       v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210);
     } else {
-      store_s16q_to_tran_low(qcoeff_ptr + i, vdupq_n_s16(0));
-      store_s16q_to_tran_low(dqcoeff_ptr + i, vdupq_n_s16(0));
+      store_zero_8_tran_low(qcoeff_ptr + i);
+      store_zero_8_tran_low(dqcoeff_ptr + i);
     }
-  }
+    i += 8;
+  } while (i != non_zero_count);
   *eob_ptr = get_max_eob(v_eobmax_76543210);
 }
+
+static AOM_FORCE_INLINE int mask_any_nonzero_u32(uint32x4_t m0, uint32x4_t m1) {
+  const uint32x4_t any = vorrq_u32(m0, m1);
+#if AOM_ARCH_AARCH64
+  return vmaxvq_u32(any) != 0;
+#else
+  const uint64x2_t u64 = vreinterpretq_u64_u32(any);
+  return (vgetq_lane_u64(u64, 0) | vgetq_lane_u64(u64, 1)) != 0;
+#endif
+}
+
+static AOM_FORCE_INLINE int16x8_t update_qm_eob(const int16_t *iscan_ptr,
+                                                int16x8_t v_eobmax,
+                                                uint32x4_t nz_mask0,
+                                                uint32x4_t nz_mask1) {
+#if AOM_ARCH_AARCH64
+  const uint16x8_t v_nz_mask = vuzp1q_u16(vreinterpretq_u16_u32(nz_mask0),
+                                          vreinterpretq_u16_u32(nz_mask1));
+#else
+  const uint16x8_t v_nz_mask =
+      vcombine_u16(vmovn_u32(nz_mask0), vmovn_u32(nz_mask1));
+#endif
+  return get_max_lane_eob(iscan_ptr, v_eobmax, v_nz_mask);
+}
+
+static AOM_FORCE_INLINE int16x8_t quantize_fp_qm_8_neon(
+    const tran_low_t *coeff_ptr, const int16_t *iscan_ptr,
+    const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, tran_low_t *qcoeff_ptr,
+    tran_low_t *dqcoeff_ptr, int32x4_t v_thr0, int32x4_t v_thr1,
+    int32x4_t v_round0, int32x4_t v_round1, int32x4_t v_quant0,
+    int32x4_t v_quant1, int32x4_t v_dequant0, int32x4_t v_dequant1,
+    int32x4_t v_log_scale, int32x4_t v_neg_log_scale, int16x8_t v_eobmax) {
+  const int32x4_t v_coeff0 = vld1q_s32(coeff_ptr);
+  const int32x4_t v_coeff1 = vld1q_s32(coeff_ptr + 4);
+  const int32x4_t v_abs0 = vabsq_s32(v_coeff0);
+  const int32x4_t v_abs1 = vabsq_s32(v_coeff1);
+  const uint16x8_t v_wt_u16 = vmovl_u8(vld1_u8(qm_ptr));
+  const int32x4_t v_wt0 =
+      vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(v_wt_u16)));
+  const int32x4_t v_wt1 =
+      vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(v_wt_u16)));
+  const uint32x4_t v_mask0 = vcgeq_s32(vmulq_s32(v_abs0, v_wt0), v_thr0);
+  const uint32x4_t v_mask1 = vcgeq_s32(vmulq_s32(v_abs1, v_wt1), v_thr1);
+
+  if (mask_any_nonzero_u32(v_mask0, v_mask1)) {
+    const int32x4_t v_zero = vdupq_n_s32(0);
+    const int32x4_t v_max_s16 = vdupq_n_s32(INT16_MAX);
+    const int32x4_t v_sign0 = vshrq_n_s32(v_coeff0, 31);
+    const int32x4_t v_sign1 = vshrq_n_s32(v_coeff1, 31);
+    const int32x4_t v_tmp0 =
+        vandq_s32(vminq_s32(vaddq_s32(v_abs0, v_round0), v_max_s16),
+                  vreinterpretq_s32_u32(v_mask0));
+    const int32x4_t v_tmp1 =
+        vandq_s32(vminq_s32(vaddq_s32(v_abs1, v_round1), v_max_s16),
+                  vreinterpretq_s32_u32(v_mask1));
+    const int32x4_t v_tmp_wt0 = vmulq_s32(v_tmp0, v_wt0);
+    const int32x4_t v_tmp_wt1 = vmulq_s32(v_tmp1, v_wt1);
+    const int32x4_t v_abs_q0 = vshrq_n_s32(
+        vqdmulhq_s32(vshlq_s32(v_tmp_wt0, v_log_scale), v_quant0), AOM_QM_BITS);
+    const int32x4_t v_abs_q1 = vshrq_n_s32(
+        vqdmulhq_s32(vshlq_s32(v_tmp_wt1, v_log_scale), v_quant1), AOM_QM_BITS);
+
+    const uint16x8_t v_iwt_u16 = vmovl_u8(vld1_u8(iqm_ptr));
+    const int32x4_t v_iwt0 =
+        vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(v_iwt_u16)));
+    const int32x4_t v_iwt1 =
+        vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(v_iwt_u16)));
+    const int32x4_t v_dq_rnd = vdupq_n_s32(1 << (AOM_QM_BITS - 1));
+    const int32x4_t v_dq0 =
+        vshrq_n_s32(vmlaq_s32(v_dq_rnd, v_dequant0, v_iwt0), AOM_QM_BITS);
+    const int32x4_t v_dq1 =
+        vshrq_n_s32(vmlaq_s32(v_dq_rnd, v_dequant1, v_iwt1), AOM_QM_BITS);
+    const int32x4_t v_abs_dq0 =
+        vshlq_s32(vmulq_s32(v_abs_q0, v_dq0), v_neg_log_scale);
+    const int32x4_t v_abs_dq1 =
+        vshlq_s32(vmulq_s32(v_abs_q1, v_dq1), v_neg_log_scale);
+
+    const int32x4_t v_q0 = vsubq_s32(veorq_s32(v_abs_q0, v_sign0), v_sign0);
+    const int32x4_t v_q1 = vsubq_s32(veorq_s32(v_abs_q1, v_sign1), v_sign1);
+    const int32x4_t v_dq_out0 =
+        vsubq_s32(veorq_s32(v_abs_dq0, v_sign0), v_sign0);
+    const int32x4_t v_dq_out1 =
+        vsubq_s32(veorq_s32(v_abs_dq1, v_sign1), v_sign1);
+    vst1q_s32(qcoeff_ptr, v_q0);
+    vst1q_s32(qcoeff_ptr + 4, v_q1);
+    vst1q_s32(dqcoeff_ptr, v_dq_out0);
+    vst1q_s32(dqcoeff_ptr + 4, v_dq_out1);
+
+    const uint32x4_t v_nz0 = vcgtq_s32(v_abs_q0, v_zero);
+    const uint32x4_t v_nz1 = vcgtq_s32(v_abs_q1, v_zero);
+    return update_qm_eob(iscan_ptr, v_eobmax, v_nz0, v_nz1);
+  }
+  const int32x4_t v_zero = vdupq_n_s32(0);
+  vst1q_s32(qcoeff_ptr, v_zero);
+  vst1q_s32(qcoeff_ptr + 4, v_zero);
+  vst1q_s32(dqcoeff_ptr, v_zero);
+  vst1q_s32(dqcoeff_ptr + 4, v_zero);
+  return v_eobmax;
+}
+
+void av1_quantize_fp_qm_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
+                             const int16_t *round_ptr, const int16_t *quant_ptr,
+                             tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
+                             const int16_t *dequant_ptr, uint16_t *eob_ptr,
+                             const int16_t *scan, const int16_t *iscan,
+                             const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr,
+                             int log_scale) {
+  (void)scan;
+  assert(qm_ptr != NULL && iqm_ptr != NULL);
+  assert(n_coeffs >= 16 && (n_coeffs & 7) == 0);
+  const int thr_shift = AOM_QM_BITS - (1 + log_scale);
+  const int32x4_t v_thr_ac = vdupq_n_s32((int32_t)dequant_ptr[1] << thr_shift);
+  const int32x4_t v_thr0 =
+      vsetq_lane_s32((int32_t)dequant_ptr[0] << thr_shift, v_thr_ac, 0);
+  const int32x4_t v_round_ac =
+      vdupq_n_s32(ROUND_POWER_OF_TWO(round_ptr[1], log_scale));
+  const int32x4_t v_round0 = vsetq_lane_s32(
+      ROUND_POWER_OF_TWO(round_ptr[0], log_scale), v_round_ac, 0);
+  const int32x4_t v_quant_ac =
+      vdupq_n_s32((int32_t)((uint32_t)quant_ptr[1] << 15));
+  const int32x4_t v_quant0 =
+      vsetq_lane_s32((int32_t)((uint32_t)quant_ptr[0] << 15), v_quant_ac, 0);
+  const int32x4_t v_dequant_ac = vdupq_n_s32(dequant_ptr[1]);
+  const int32x4_t v_dequant0 = vsetq_lane_s32(dequant_ptr[0], v_dequant_ac, 0);
+  const int32x4_t v_log_scale = vdupq_n_s32(log_scale);
+  const int32x4_t v_neg_log_scale = vdupq_n_s32(-log_scale);
+  int16x8_t v_eobmax = vdupq_n_s16(-1);
+
+  v_eobmax = quantize_fp_qm_8_neon(
+      coeff_ptr, iscan, qm_ptr, iqm_ptr, qcoeff_ptr, dqcoeff_ptr, v_thr0,
+      v_thr_ac, v_round0, v_round_ac, v_quant0, v_quant_ac, v_dequant0,
+      v_dequant_ac, v_log_scale, v_neg_log_scale, v_eobmax);
+
+  intptr_t i = 8;
+  do {
+    v_eobmax = quantize_fp_qm_8_neon(
+        coeff_ptr + i, iscan + i, qm_ptr + i, iqm_ptr + i, qcoeff_ptr + i,
+        dqcoeff_ptr + i, v_thr_ac, v_thr_ac, v_round_ac, v_round_ac, v_quant_ac,
+        v_quant_ac, v_dequant_ac, v_dequant_ac, v_log_scale, v_neg_log_scale,
+        v_eobmax);
+    i += 8;
+  } while (i != n_coeffs);
+  *eob_ptr = get_max_eob(v_eobmax);
+}
+
+static AOM_FORCE_INLINE int16x8_t quantize_b_qm_8_neon(
+    const tran_low_t *coeff_ptr, const int16_t *iscan_ptr,
+    const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, tran_low_t *qcoeff_ptr,
+    tran_low_t *dqcoeff_ptr, int32x4_t v_zbin0, int32x4_t v_zbin1,
+    int32x4_t v_round0, int32x4_t v_round1, int32x4_t v_quant0,
+    int32x4_t v_quant1, int32x4_t v_qshift0, int32x4_t v_qshift1,
+    int32x4_t v_dequant0, int32x4_t v_dequant1, int32x4_t v_log_scale,
+    int32x4_t v_neg_log_scale, int16x8_t v_eobmax) {
+  const int32x4_t v_coeff0 = vld1q_s32(coeff_ptr);
+  const int32x4_t v_coeff1 = vld1q_s32(coeff_ptr + 4);
+  const int32x4_t v_abs0 = vabsq_s32(v_coeff0);
+  const int32x4_t v_abs1 = vabsq_s32(v_coeff1);
+  const uint16x8_t v_wt_u16 = vmovl_u8(vld1_u8(qm_ptr));
+  const int32x4_t v_wt0 =
+      vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(v_wt_u16)));
+  const int32x4_t v_wt1 =
+      vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(v_wt_u16)));
+  const uint32x4_t v_mask0 = vcgeq_s32(vmulq_s32(v_abs0, v_wt0), v_zbin0);
+  const uint32x4_t v_mask1 = vcgeq_s32(vmulq_s32(v_abs1, v_wt1), v_zbin1);
+
+  if (mask_any_nonzero_u32(v_mask0, v_mask1)) {
+    const int32x4_t v_zero = vdupq_n_s32(0);
+    const int32x4_t v_max_s16 = vdupq_n_s32(INT16_MAX);
+    const int32x4_t v_sign0 = vshrq_n_s32(v_coeff0, 31);
+    const int32x4_t v_sign1 = vshrq_n_s32(v_coeff1, 31);
+    const int32x4_t v_tmp0 =
+        vandq_s32(vminq_s32(vaddq_s32(v_abs0, v_round0), v_max_s16),
+                  vreinterpretq_s32_u32(v_mask0));
+    const int32x4_t v_tmp1 =
+        vandq_s32(vminq_s32(vaddq_s32(v_abs1, v_round1), v_max_s16),
+                  vreinterpretq_s32_u32(v_mask1));
+    const int32x4_t v_tmp_wt0 = vmulq_s32(v_tmp0, v_wt0);
+    const int32x4_t v_tmp_wt1 = vmulq_s32(v_tmp1, v_wt1);
+    const int32x4_t v_tmp2_0 =
+        vaddq_s32(vqdmulhq_s32(v_tmp_wt0, v_quant0), v_tmp_wt0);
+    const int32x4_t v_tmp2_1 =
+        vaddq_s32(vqdmulhq_s32(v_tmp_wt1, v_quant1), v_tmp_wt1);
+    const int32x4_t v_abs_q0 = vshrq_n_s32(
+        vqdmulhq_s32(vshlq_s32(v_tmp2_0, v_log_scale), v_qshift0), AOM_QM_BITS);
+    const int32x4_t v_abs_q1 = vshrq_n_s32(
+        vqdmulhq_s32(vshlq_s32(v_tmp2_1, v_log_scale), v_qshift1), AOM_QM_BITS);
+
+    const uint16x8_t v_iwt_u16 = vmovl_u8(vld1_u8(iqm_ptr));
+    const int32x4_t v_iwt0 =
+        vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(v_iwt_u16)));
+    const int32x4_t v_iwt1 =
+        vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(v_iwt_u16)));
+    const int32x4_t v_dq_rnd = vdupq_n_s32(1 << (AOM_QM_BITS - 1));
+    const int32x4_t v_dq0 =
+        vshrq_n_s32(vmlaq_s32(v_dq_rnd, v_dequant0, v_iwt0), AOM_QM_BITS);
+    const int32x4_t v_dq1 =
+        vshrq_n_s32(vmlaq_s32(v_dq_rnd, v_dequant1, v_iwt1), AOM_QM_BITS);
+    const int32x4_t v_abs_dq0 =
+        vshlq_s32(vmulq_s32(v_abs_q0, v_dq0), v_neg_log_scale);
+    const int32x4_t v_abs_dq1 =
+        vshlq_s32(vmulq_s32(v_abs_q1, v_dq1), v_neg_log_scale);
+
+    const int32x4_t v_q0 = vsubq_s32(veorq_s32(v_abs_q0, v_sign0), v_sign0);
+    const int32x4_t v_q1 = vsubq_s32(veorq_s32(v_abs_q1, v_sign1), v_sign1);
+    const int32x4_t v_dq_out0 =
+        vsubq_s32(veorq_s32(v_abs_dq0, v_sign0), v_sign0);
+    const int32x4_t v_dq_out1 =
+        vsubq_s32(veorq_s32(v_abs_dq1, v_sign1), v_sign1);
+    vst1q_s32(qcoeff_ptr, v_q0);
+    vst1q_s32(qcoeff_ptr + 4, v_q1);
+    vst1q_s32(dqcoeff_ptr, v_dq_out0);
+    vst1q_s32(dqcoeff_ptr + 4, v_dq_out1);
+
+    const uint32x4_t v_nz0 = vcgtq_s32(v_abs_q0, v_zero);
+    const uint32x4_t v_nz1 = vcgtq_s32(v_abs_q1, v_zero);
+    return update_qm_eob(iscan_ptr, v_eobmax, v_nz0, v_nz1);
+  }
+  const int32x4_t v_zero = vdupq_n_s32(0);
+  vst1q_s32(qcoeff_ptr, v_zero);
+  vst1q_s32(qcoeff_ptr + 4, v_zero);
+  vst1q_s32(dqcoeff_ptr, v_zero);
+  vst1q_s32(dqcoeff_ptr + 4, v_zero);
+  return v_eobmax;
+}
+
+void av1_quantize_b_qm_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
+                            const int16_t *zbin_ptr, const int16_t *round_ptr,
+                            const int16_t *quant_ptr,
+                            const int16_t *quant_shift_ptr,
+                            tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
+                            const int16_t *dequant_ptr, uint16_t *eob_ptr,
+                            const int16_t *scan, const int16_t *iscan,
+                            const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr,
+                            int log_scale) {
+  (void)scan;
+  assert(qm_ptr != NULL && iqm_ptr != NULL);
+  assert(n_coeffs >= 16 && (n_coeffs & 7) == 0);
+  const int32x4_t v_zbin_ac =
+      vdupq_n_s32(ROUND_POWER_OF_TWO(zbin_ptr[1], log_scale) << AOM_QM_BITS);
+  const int32x4_t v_zbin0 = vsetq_lane_s32(
+      ROUND_POWER_OF_TWO(zbin_ptr[0], log_scale) << AOM_QM_BITS, v_zbin_ac, 0);
+  const int32x4_t v_round_ac =
+      vdupq_n_s32(ROUND_POWER_OF_TWO(round_ptr[1], log_scale));
+  const int32x4_t v_round0 = vsetq_lane_s32(
+      ROUND_POWER_OF_TWO(round_ptr[0], log_scale), v_round_ac, 0);
+  const int32x4_t v_quant_ac =
+      vdupq_n_s32((int32_t)((uint32_t)quant_ptr[1] << 15));
+  const int32x4_t v_quant0 =
+      vsetq_lane_s32((int32_t)((uint32_t)quant_ptr[0] << 15), v_quant_ac, 0);
+  const int32x4_t v_qshift_ac =
+      vdupq_n_s32((int32_t)((uint32_t)quant_shift_ptr[1] << 15));
+  const int32x4_t v_qshift0 = vsetq_lane_s32(
+      (int32_t)((uint32_t)quant_shift_ptr[0] << 15), v_qshift_ac, 0);
+  const int32x4_t v_dequant_ac = vdupq_n_s32(dequant_ptr[1]);
+  const int32x4_t v_dequant0 = vsetq_lane_s32(dequant_ptr[0], v_dequant_ac, 0);
+  const int32x4_t v_log_scale = vdupq_n_s32(log_scale);
+  const int32x4_t v_neg_log_scale = vdupq_n_s32(-log_scale);
+  int16x8_t v_eobmax = vdupq_n_s16(-1);
+
+  v_eobmax = quantize_b_qm_8_neon(coeff_ptr, iscan, qm_ptr, iqm_ptr, qcoeff_ptr,
+                                  dqcoeff_ptr, v_zbin0, v_zbin_ac, v_round0,
+                                  v_round_ac, v_quant0, v_quant_ac, v_qshift0,
+                                  v_qshift_ac, v_dequant0, v_dequant_ac,
+                                  v_log_scale, v_neg_log_scale, v_eobmax);
+
+  intptr_t i = 8;
+  do {
+    v_eobmax = quantize_b_qm_8_neon(
+        coeff_ptr + i, iscan + i, qm_ptr + i, iqm_ptr + i, qcoeff_ptr + i,
+        dqcoeff_ptr + i, v_zbin_ac, v_zbin_ac, v_round_ac, v_round_ac,
+        v_quant_ac, v_quant_ac, v_qshift_ac, v_qshift_ac, v_dequant_ac,
+        v_dequant_ac, v_log_scale, v_neg_log_scale, v_eobmax);
+    i += 8;
+  } while (i != n_coeffs);
+  *eob_ptr = get_max_eob(v_eobmax);
+}
diff --git a/test/quantize_func_test.cc b/test/quantize_func_test.cc
index 3e35c8dc50..a91fa58f83 100644
--- a/test/quantize_func_test.cc
+++ b/test/quantize_func_test.cc
@@ -477,8 +477,7 @@ TEST_P(LowPrecisionQuantizeTest, DISABLED_Speed) {

 using std::make_tuple;

-#if HAVE_AVX2
-#if CONFIG_QUANT_MATRIX || CONFIG_AV1_DECODER
+#if (HAVE_AVX2 || HAVE_NEON) && (CONFIG_QUANT_MATRIX || CONFIG_AV1_DECODER)
 using QuantizeFpQmFunc = void (*)(
     const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr,
     const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
@@ -520,7 +519,10 @@ void quan_b_qm_wrapper(QUAN_PARAM_LIST) {
      qcoeff_ptr, dqcoeff_ptr, dequant_ptr, eob_ptr, scan, iscan, qm_ptr,
      iqm_ptr, log_scale);
 }
-#endif  // CONFIG_QUANT_MATRIX || CONFIG_AV1_DECODER
+#endif  // (HAVE_AVX2 || HAVE_NEON) && (CONFIG_QUANT_MATRIX ||
+        // CONFIG_AV1_DECODER)
+
+#if HAVE_AVX2

 const QuantizeParam<LPQuantizeFunc> kLPQParamArrayAvx2[] = {
   make_tuple(&av1_quantize_lp_c, &av1_quantize_lp_avx2,
@@ -905,17 +907,159 @@ const QuantizeParam<QuantizeFunc> kQParamArrayNEON[] = {
              static_cast<TX_SIZE>(TX_32X32), TYPE_B, AOM_BITS_8),
   make_tuple(&aom_quantize_b_64x64_c, &aom_quantize_b_64x64_neon,
              static_cast<TX_SIZE>(TX_64X64), TYPE_B, AOM_BITS_8),
+#if CONFIG_QUANT_MATRIX || CONFIG_AV1_DECODER
+  make_tuple(&quan_fp_qm_wrapper<av1_quantize_fp_qm_c, TX_4X4>,
+             &quan_fp_qm_wrapper<av1_quantize_fp_qm_neon, TX_4X4>,
+             static_cast<TX_SIZE>(TX_4X4), TYPE_FP, AOM_BITS_8),
+  make_tuple(&quan_fp_qm_wrapper<av1_quantize_fp_qm_c, TX_8X8>,
+             &quan_fp_qm_wrapper<av1_quantize_fp_qm_neon, TX_8X8>,
+             static_cast<TX_SIZE>(TX_8X8), TYPE_FP, AOM_BITS_8),
+  make_tuple(&quan_fp_qm_wrapper<av1_quantize_fp_qm_c, TX_16X16>,
+             &quan_fp_qm_wrapper<av1_quantize_fp_qm_neon, TX_16X16>,
+             static_cast<TX_SIZE>(TX_16X16), TYPE_FP, AOM_BITS_8),
+  make_tuple(&quan_fp_qm_wrapper<av1_quantize_fp_qm_c, TX_32X32>,
+             &quan_fp_qm_wrapper<av1_quantize_fp_qm_neon, TX_32X32>,
+             static_cast<TX_SIZE>(TX_32X32), TYPE_FP, AOM_BITS_8),
+  make_tuple(&quan_fp_qm_wrapper<av1_quantize_fp_qm_c, TX_64X64>,
+             &quan_fp_qm_wrapper<av1_quantize_fp_qm_neon, TX_64X64>,
+             static_cast<TX_SIZE>(TX_64X64), TYPE_FP, AOM_BITS_8),
+  make_tuple(&quan_b_qm_wrapper<av1_quantize_b_qm_c, TX_4X4>,
+             &quan_b_qm_wrapper<av1_quantize_b_qm_neon, TX_4X4>,
+             static_cast<TX_SIZE>(TX_4X4), TYPE_B, AOM_BITS_8),
+  make_tuple(&quan_b_qm_wrapper<av1_quantize_b_qm_c, TX_8X8>,
+             &quan_b_qm_wrapper<av1_quantize_b_qm_neon, TX_8X8>,
+             static_cast<TX_SIZE>(TX_8X8), TYPE_B, AOM_BITS_8),
+  make_tuple(&quan_b_qm_wrapper<av1_quantize_b_qm_c, TX_16X16>,
+             &quan_b_qm_wrapper<av1_quantize_b_qm_neon, TX_16X16>,
+             static_cast<TX_SIZE>(TX_16X16), TYPE_B, AOM_BITS_8),
+  make_tuple(&quan_b_qm_wrapper<av1_quantize_b_qm_c, TX_32X32>,
+             &quan_b_qm_wrapper<av1_quantize_b_qm_neon, TX_32X32>,
+             static_cast<TX_SIZE>(TX_32X32), TYPE_B, AOM_BITS_8),
+  make_tuple(&quan_b_qm_wrapper<av1_quantize_b_qm_c, TX_64X64>,
+             &quan_b_qm_wrapper<av1_quantize_b_qm_neon, TX_64X64>,
+             static_cast<TX_SIZE>(TX_64X64), TYPE_B, AOM_BITS_8),
+#endif  // CONFIG_QUANT_MATRIX || CONFIG_AV1_DECODER

 #if CONFIG_AV1_HIGHBITDEPTH
+  make_tuple(&highbd_quan16x16_wrapper<av1_highbd_quantize_fp_c>,
+             &highbd_quan16x16_wrapper<av1_highbd_quantize_fp_neon>,
+             static_cast<TX_SIZE>(TX_16X16), TYPE_FP, AOM_BITS_8),
+  make_tuple(&highbd_quan16x16_wrapper<av1_highbd_quantize_fp_c>,
+             &highbd_quan16x16_wrapper<av1_highbd_quantize_fp_neon>,
+             static_cast<TX_SIZE>(TX_16X16), TYPE_FP, AOM_BITS_10),
   make_tuple(&highbd_quan16x16_wrapper<av1_highbd_quantize_fp_c>,
              &highbd_quan16x16_wrapper<av1_highbd_quantize_fp_neon>,
              static_cast<TX_SIZE>(TX_16X16), TYPE_FP, AOM_BITS_12),
+  make_tuple(&highbd_quan32x32_wrapper<av1_highbd_quantize_fp_c>,
+             &highbd_quan32x32_wrapper<av1_highbd_quantize_fp_neon>,
+             static_cast<TX_SIZE>(TX_32X32), TYPE_FP, AOM_BITS_8),
+  make_tuple(&highbd_quan32x32_wrapper<av1_highbd_quantize_fp_c>,
+             &highbd_quan32x32_wrapper<av1_highbd_quantize_fp_neon>,
+             static_cast<TX_SIZE>(TX_32X32), TYPE_FP, AOM_BITS_10),
   make_tuple(&highbd_quan32x32_wrapper<av1_highbd_quantize_fp_c>,
              &highbd_quan32x32_wrapper<av1_highbd_quantize_fp_neon>,
              static_cast<TX_SIZE>(TX_32X32), TYPE_FP, AOM_BITS_12),
+  make_tuple(&highbd_quan64x64_wrapper<av1_highbd_quantize_fp_c>,
+             &highbd_quan64x64_wrapper<av1_highbd_quantize_fp_neon>,
+             static_cast<TX_SIZE>(TX_64X64), TYPE_FP, AOM_BITS_8),
+  make_tuple(&highbd_quan64x64_wrapper<av1_highbd_quantize_fp_c>,
+             &highbd_quan64x64_wrapper<av1_highbd_quantize_fp_neon>,
+             static_cast<TX_SIZE>(TX_64X64), TYPE_FP, AOM_BITS_10),
   make_tuple(&highbd_quan64x64_wrapper<av1_highbd_quantize_fp_c>,
              &highbd_quan64x64_wrapper<av1_highbd_quantize_fp_neon>,
              static_cast<TX_SIZE>(TX_64X64), TYPE_FP, AOM_BITS_12),
+#if CONFIG_QUANT_MATRIX || CONFIG_AV1_DECODER
+  make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_4X4>,
+             &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_4X4>,
+             static_cast<TX_SIZE>(TX_4X4), TYPE_FP, AOM_BITS_8),
+  make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_4X4>,
+             &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_4X4>,
+             static_cast<TX_SIZE>(TX_4X4), TYPE_FP, AOM_BITS_10),
+  make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_4X4>,
+             &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_4X4>,
+             static_cast<TX_SIZE>(TX_4X4), TYPE_FP, AOM_BITS_12),
+  make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_8X8>,
+             &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_8X8>,
+             static_cast<TX_SIZE>(TX_8X8), TYPE_FP, AOM_BITS_8),
+  make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_8X8>,
+             &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_8X8>,
+             static_cast<TX_SIZE>(TX_8X8), TYPE_FP, AOM_BITS_10),
+  make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_8X8>,
+             &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_8X8>,
+             static_cast<TX_SIZE>(TX_8X8), TYPE_FP, AOM_BITS_12),
+  make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_16X16>,
+             &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_16X16>,
+             static_cast<TX_SIZE>(TX_16X16), TYPE_FP, AOM_BITS_8),
+  make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_16X16>,
+             &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_16X16>,
+             static_cast<TX_SIZE>(TX_16X16), TYPE_FP, AOM_BITS_10),
+  make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_16X16>,
+             &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_16X16>,
+             static_cast<TX_SIZE>(TX_16X16), TYPE_FP, AOM_BITS_12),
+  make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_32X32>,
+             &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_32X32>,
+             static_cast<TX_SIZE>(TX_32X32), TYPE_FP, AOM_BITS_8),
+  make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_32X32>,
+             &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_32X32>,
+             static_cast<TX_SIZE>(TX_32X32), TYPE_FP, AOM_BITS_10),
+  make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_32X32>,
+             &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_32X32>,
+             static_cast<TX_SIZE>(TX_32X32), TYPE_FP, AOM_BITS_12),
+  make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_64X64>,
+             &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_64X64>,
+             static_cast<TX_SIZE>(TX_64X64), TYPE_FP, AOM_BITS_8),
+  make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_64X64>,
+             &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_64X64>,
+             static_cast<TX_SIZE>(TX_64X64), TYPE_FP, AOM_BITS_10),
+  make_tuple(&quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_c, TX_64X64>,
+             &quan_fp_qm_wrapper<av1_highbd_quantize_fp_qm_neon, TX_64X64>,
+             static_cast<TX_SIZE>(TX_64X64), TYPE_FP, AOM_BITS_12),
+  make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_4X4>,
+             &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_4X4>,
+             static_cast<TX_SIZE>(TX_4X4), TYPE_B, AOM_BITS_8),
+  make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_4X4>,
+             &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_4X4>,
+             static_cast<TX_SIZE>(TX_4X4), TYPE_B, AOM_BITS_10),
+  make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_4X4>,
+             &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_4X4>,
+             static_cast<TX_SIZE>(TX_4X4), TYPE_B, AOM_BITS_12),
+  make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_8X8>,
+             &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_8X8>,
+             static_cast<TX_SIZE>(TX_8X8), TYPE_B, AOM_BITS_8),
+  make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_8X8>,
+             &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_8X8>,
+             static_cast<TX_SIZE>(TX_8X8), TYPE_B, AOM_BITS_10),
+  make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_8X8>,
+             &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_8X8>,
+             static_cast<TX_SIZE>(TX_8X8), TYPE_B, AOM_BITS_12),
+  make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_16X16>,
+             &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_16X16>,
+             static_cast<TX_SIZE>(TX_16X16), TYPE_B, AOM_BITS_8),
+  make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_16X16>,
+             &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_16X16>,
+             static_cast<TX_SIZE>(TX_16X16), TYPE_B, AOM_BITS_10),
+  make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_16X16>,
+             &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_16X16>,
+             static_cast<TX_SIZE>(TX_16X16), TYPE_B, AOM_BITS_12),
+  make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_32X32>,
+             &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_32X32>,
+             static_cast<TX_SIZE>(TX_32X32), TYPE_B, AOM_BITS_8),
+  make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_32X32>,
+             &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_32X32>,
+             static_cast<TX_SIZE>(TX_32X32), TYPE_B, AOM_BITS_10),
+  make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_32X32>,
+             &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_32X32>,
+             static_cast<TX_SIZE>(TX_32X32), TYPE_B, AOM_BITS_12),
+  make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_64X64>,
+             &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_64X64>,
+             static_cast<TX_SIZE>(TX_64X64), TYPE_B, AOM_BITS_8),
+  make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_64X64>,
+             &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_64X64>,
+             static_cast<TX_SIZE>(TX_64X64), TYPE_B, AOM_BITS_10),
+  make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_64X64>,
+             &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_neon, TX_64X64>,
+             static_cast<TX_SIZE>(TX_64X64), TYPE_B, AOM_BITS_12),
+#endif  // CONFIG_QUANT_MATRIX || CONFIG_AV1_DECODER
   make_tuple(&aom_highbd_quantize_b_c, &aom_highbd_quantize_b_neon,
              static_cast<TX_SIZE>(TX_16X16), TYPE_B, AOM_BITS_12),
   make_tuple(&aom_highbd_quantize_b_32x32_c, &aom_highbd_quantize_b_32x32_neon,