Commit b64bd5b786 for aom

commit b64bd5b78635440b68dc7e62c26957be0be6b7cf
Author: Frank Galligan <fgalligan@google.com>
Date:   Tue Nov 17 06:28:34 2026 -0800

    av1,neon: Optimize quantize_b & quantize_fp

    Add early zero-block skipping via mask_nonzero_u16() and
    mask_nonzero_pair_u16() with compact unrolled loops and 16-coeff
    alignment asserts in av1_quantize_fp_neon and
    aom_quantize_b_{neon,32x32_neon,64x64_neon}. Share non-qmatrix
    32x32/64x64 quantization via quantize_b_logscale_8() and
    quantize_b_no_qmatrix_neon() using vrshrq_n_s16() and uniform
    zero-mask polarity, eliminating stack spills (stack=0B).

    Also pre-fill SIMD output buffers with a non-zero pattern and add
    TX_4X4, TX_4X8, TX_8X8, and TX_16X32 NEON entries in
    test/quantize_func_test.cc.

    Speedup (Pixel 9 Tensor G4 Cortex-A720, 1T, bit-exact on 72 tier-1
    configs):
    test_libaom NEON/FullPrecisionQuantizeTest.DISABLED_Speed (16x16, A720):
    - av1_quantize_fp_neon: cnt=0 285679->255339us (-10.62%),
      cnt=16 915803->851099us (-7.07%)
    - aom_quantize_b_neon: cnt=0 278030->256507us (-7.74%),
      cnt=16 1360652->1014131us (-25.47%)

    TAG=agy
    CONV=bc5228f2-7fa6-4522-aad6-34935d739e28
    Change-Id: I5a4124eea5aede65f9849baa7ae9f63ba205df54

diff --git a/av1/encoder/arm/quantize_neon.c b/av1/encoder/arm/quantize_neon.c
index e3a0f7566a..56df4f4abb 100644
--- a/av1/encoder/arm/quantize_neon.c
+++ b/av1/encoder/arm/quantize_neon.c
@@ -29,10 +29,21 @@
 #include "av1/encoder/rd.h"

 static AOM_FORCE_INLINE uint64_t mask_nonzero_u16(uint16x8_t v_mask) {
+  return vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_mask)), 0);
+}
+
+// Requires each 16-bit lane of v_m0 and v_m1 to be 0x0000 or 0xFFFF.
+static AOM_FORCE_INLINE void mask_nonzero_pair_u16(uint16x8_t v_m0,
+                                                   uint16x8_t v_m1,
+                                                   uint64_t *m0, uint64_t *m1) {
 #if AOM_ARCH_AARCH64
-  return vmaxvq_u32(vreinterpretq_u32_u16(v_mask));
+  const uint8x16_t packed =
+      vuzp1q_u8(vreinterpretq_u8_u16(v_m0), vreinterpretq_u8_u16(v_m1));
+  *m0 = vgetq_lane_u64(vreinterpretq_u64_u8(packed), 0);
+  *m1 = vgetq_lane_u64(vreinterpretq_u64_u8(packed), 1);
 #else
-  return vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(v_mask)), 0);
+  *m0 = mask_nonzero_u16(v_m0);
+  *m1 = mask_nonzero_u16(v_m1);
 #endif
 }

@@ -61,6 +72,15 @@ static inline uint16_t get_max_eob(int16x8_t v_eobmax) {
 #endif
 }

+static inline int16x8_t get_max_lane_eob_zmask(const int16_t *iscan,
+                                               int16x8_t v_eobmax,
+                                               uint16x8_t v_zero_mask) {
+  const int16x8_t v_iscan = vld1q_s16(iscan);
+  const int16x8_t v_nz_iscan =
+      vorrq_s16(v_iscan, vreinterpretq_s16_u16(v_zero_mask));
+  return vmaxq_s16(v_eobmax, v_nz_iscan);
+}
+
 static inline int16x8_t get_max_lane_eob(const int16_t *iscan,
                                          int16x8_t v_eobmax,
                                          uint16x8_t v_mask) {
@@ -69,29 +89,20 @@ static inline int16x8_t get_max_lane_eob(const int16_t *iscan,
   return vmaxq_s16(v_eobmax, v_nz_iscan);
 }

-static AOM_FORCE_INLINE int16x8_t
-quantize_fp_8(const tran_low_t *coeff_ptr, tran_low_t *qcoeff_ptr,
-              tran_low_t *dqcoeff_ptr, const int16_t *iscan, int16x8_t v_quant,
-              int16x8_t v_dequant, int16x8_t v_round, uint16x8_t v_thresh,
-              int16x8_t v_eobmax) {
-  const int16x8_t v_coeff = load_tran_low_to_s16q(&coeff_ptr[0]);
-  const int16x8_t v_abs = vabsq_s16(v_coeff);
-  const uint16x8_t v_mask = vcgeq_u16(vreinterpretq_u16_s16(v_abs), v_thresh);
-  if (mask_nonzero_u16(v_mask)) {
-    const int16x8_t v_coeff_sign = vshrq_n_s16(v_coeff, 15);
-    const int16x8_t v_tmp = vqaddq_s16(v_abs, v_round);
-    const int16x8_t v_tmp2 = vshrq_n_s16(vqdmulhq_s16(v_tmp, v_quant), 1);
-    const uint16x8_t v_nz_mask = vcgtq_s16(v_tmp2, vdupq_n_s16(0));
-    const int16x8_t v_qcoeff_a = veorq_s16(v_tmp2, v_coeff_sign);
-    const int16x8_t v_qcoeff = vsubq_s16(v_qcoeff_a, v_coeff_sign);
-    const int16x8_t v_dqcoeff = vmulq_s16(v_qcoeff, v_dequant);
-    store_s16q_to_tran_low(&qcoeff_ptr[0], v_qcoeff);
-    store_s16q_to_tran_low(&dqcoeff_ptr[0], v_dqcoeff);
-    return get_max_lane_eob(iscan, v_eobmax, v_nz_mask);
-  }
-  store_zero_8_tran_low(&qcoeff_ptr[0]);
-  store_zero_8_tran_low(&dqcoeff_ptr[0]);
-  return v_eobmax;
+// Returns a zero-mask (0xFFFF for quantized zero lanes, 0x0000 otherwise).
+static AOM_FORCE_INLINE uint16x8_t
+quantize_fp_nz_8(int16x8_t v_coeff, int16x8_t v_abs, tran_low_t *qcoeff_ptr,
+                 tran_low_t *dqcoeff_ptr, int16x8_t v_quant,
+                 int16x8_t v_dequant, int16x8_t v_round) {
+  const int16x8_t v_coeff_sign = vshrq_n_s16(v_coeff, 15);
+  const int16x8_t v_tmp = vqaddq_s16(v_abs, v_round);
+  const int16x8_t v_tmp2 = vshrq_n_s16(vqdmulhq_s16(v_tmp, v_quant), 1);
+  const int16x8_t v_qcoeff_a = veorq_s16(v_tmp2, v_coeff_sign);
+  const int16x8_t v_qcoeff = vsubq_s16(v_qcoeff_a, v_coeff_sign);
+  const int16x8_t v_dqcoeff = vmulq_s16(v_qcoeff, v_dequant);
+  store_s16q_to_tran_low(&qcoeff_ptr[0], v_qcoeff);
+  store_s16q_to_tran_low(&dqcoeff_ptr[0], v_dqcoeff);
+  return vcleq_s16(v_tmp2, vdupq_n_s16(0));
 }

 void av1_quantize_fp_neon(const tran_low_t *coeff_ptr, intptr_t count,
@@ -106,37 +117,140 @@ void av1_quantize_fp_neon(const tran_low_t *coeff_ptr, intptr_t count,
   (void)zbin_ptr;
   (void)quant_shift_ptr;
   (void)scan;
-  assert(count >= 16 && (count & 7) == 0);
+  assert(count >= 16 && (count & 15) == 0);

-  int16x8_t v_quant = vld1q_s16(quant_ptr);
-  int16x8_t v_dequant = vld1q_s16(dequant_ptr);
-  int16x8_t v_round = vld1q_s16(round_ptr);
-  uint16x8_t v_thresh = vreinterpretq_u16_s16(vshrq_n_s16(v_dequant, 1));
-  int16x8_t v_eobmax_76543210 = vdupq_n_s16(-1);
+  const int16x8_t v_dequant0 = vld1q_s16(dequant_ptr);
+  const uint16x8_t v_thresh0 =
+      vreinterpretq_u16_s16(vshrq_n_s16(v_dequant0, 1));
+  const uint16x8_t v_thresh = vdupq_lane_u16(vget_low_u16(v_thresh0), 1);

-  // process dc and the first seven ac coeffs
-  v_eobmax_76543210 =
-      quantize_fp_8(coeff_ptr, qcoeff_ptr, dqcoeff_ptr, iscan, v_quant,
-                    v_dequant, v_round, v_thresh, v_eobmax_76543210);
+  const int16x8_t v_coeff0 = load_tran_low_to_s16q(coeff_ptr);
+  const int16x8_t v_coeff1 = load_tran_low_to_s16q(coeff_ptr + 8);
+  const int16x8_t v_abs0 = vabsq_s16(v_coeff0);
+  const int16x8_t v_abs1 = vabsq_s16(v_coeff1);
+  const uint16x8_t v_mask0 =
+      vcgeq_u16(vreinterpretq_u16_s16(v_abs0), v_thresh0);
+  const uint16x8_t v_mask1 = vcgeq_u16(vreinterpretq_u16_s16(v_abs1), v_thresh);

-  // overwrite the dc constants with ac constants
-  v_quant = vdupq_lane_s16(vget_low_s16(v_quant), 1);
-  v_dequant = vdupq_lane_s16(vget_low_s16(v_dequant), 1);
-  v_round = vdupq_lane_s16(vget_low_s16(v_round), 1);
-  v_thresh = vdupq_lane_u16(vget_low_u16(v_thresh), 1);
+  uint64_t m0, m1;
+  mask_nonzero_pair_u16(v_mask0, v_mask1, &m0, &m1);
+
+  if ((m0 | m1) == 0 && count == 16) {
+    store_zero_8_tran_low(qcoeff_ptr);
+    store_zero_8_tran_low(qcoeff_ptr + 8);
+    store_zero_8_tran_low(dqcoeff_ptr);
+    store_zero_8_tran_low(dqcoeff_ptr + 8);
+    *eob_ptr = 0;
+    return;
+  }
+
+  const int16x8_t v_quant0 = vld1q_s16(quant_ptr);
+  const int16x8_t v_round0 = vld1q_s16(round_ptr);
+  const int16x8_t v_quant = vdupq_lane_s16(vget_low_s16(v_quant0), 1);
+  const int16x8_t v_dequant = vdupq_lane_s16(vget_low_s16(v_dequant0), 1);
+  const int16x8_t v_round = vdupq_lane_s16(vget_low_s16(v_round0), 1);
+
+  int16x8_t v_eobmax_76543210;
+  const tran_low_t *const coeff_end = coeff_ptr + count;
+
+  if ((m0 | m1) != 0) {
+    if (m0 != 0) {
+      const uint16x8_t v_zero_mask0 =
+          quantize_fp_nz_8(v_coeff0, v_abs0, qcoeff_ptr, dqcoeff_ptr, v_quant0,
+                           v_dequant0, v_round0);
+      if (m1 != 0) {
+        const uint16x8_t v_zero_mask1 =
+            quantize_fp_nz_8(v_coeff1, v_abs1, qcoeff_ptr + 8, dqcoeff_ptr + 8,
+                             v_quant, v_dequant, v_round);
+        const int16x8_t v_nz_iscan0 =
+            vorrq_s16(vld1q_s16(iscan), vreinterpretq_s16_u16(v_zero_mask0));
+        const int16x8_t v_nz_iscan1 = vorrq_s16(
+            vld1q_s16(iscan + 8), vreinterpretq_s16_u16(v_zero_mask1));
+        v_eobmax_76543210 = vmaxq_s16(v_nz_iscan0, v_nz_iscan1);
+      } else {
+        v_eobmax_76543210 =
+            vorrq_s16(vld1q_s16(iscan), vreinterpretq_s16_u16(v_zero_mask0));
+        store_zero_8_tran_low(qcoeff_ptr + 8);
+        store_zero_8_tran_low(dqcoeff_ptr + 8);
+      }
+    } else {
+      store_zero_8_tran_low(qcoeff_ptr);
+      store_zero_8_tran_low(dqcoeff_ptr);
+      const uint16x8_t v_zero_mask1 =
+          quantize_fp_nz_8(v_coeff1, v_abs1, qcoeff_ptr + 8, dqcoeff_ptr + 8,
+                           v_quant, v_dequant, v_round);
+      v_eobmax_76543210 =
+          vorrq_s16(vld1q_s16(iscan + 8), vreinterpretq_s16_u16(v_zero_mask1));
+    }
+    if (count == 16) {
+      *eob_ptr = get_max_eob(v_eobmax_76543210);
+      return;
+    }
+  } else {
+    store_zero_8_tran_low(qcoeff_ptr);
+    store_zero_8_tran_low(qcoeff_ptr + 8);
+    store_zero_8_tran_low(dqcoeff_ptr);
+    store_zero_8_tran_low(dqcoeff_ptr + 8);
+    v_eobmax_76543210 = vdupq_n_s16(-1);
+  }
+
+  coeff_ptr += 16;
+  qcoeff_ptr += 16;
+  dqcoeff_ptr += 16;
+  iscan += 16;

-  count -= 8;
-  // now process the rest of the ac coeffs
   do {
-    coeff_ptr += 8;
-    qcoeff_ptr += 8;
-    dqcoeff_ptr += 8;
-    iscan += 8;
-    v_eobmax_76543210 =
-        quantize_fp_8(coeff_ptr, qcoeff_ptr, dqcoeff_ptr, iscan, v_quant,
-                      v_dequant, v_round, v_thresh, v_eobmax_76543210);
-    count -= 8;
-  } while (count != 0);
+    const int16x8_t v_c0 = load_tran_low_to_s16q(coeff_ptr);
+    const int16x8_t v_c1 = load_tran_low_to_s16q(coeff_ptr + 8);
+    const int16x8_t v_a0 = vabsq_s16(v_c0);
+    const int16x8_t v_a1 = vabsq_s16(v_c1);
+    const uint16x8_t v_m0 = vcgeq_u16(vreinterpretq_u16_s16(v_a0), v_thresh);
+    const uint16x8_t v_m1 = vcgeq_u16(vreinterpretq_u16_s16(v_a1), v_thresh);
+    uint64_t l_m0, l_m1;
+    mask_nonzero_pair_u16(v_m0, v_m1, &l_m0, &l_m1);
+
+    if ((l_m0 | l_m1) != 0) {
+      if (l_m0 != 0) {
+        const uint16x8_t v_zero_mask0 = quantize_fp_nz_8(
+            v_c0, v_a0, qcoeff_ptr, dqcoeff_ptr, v_quant, v_dequant, v_round);
+        if (l_m1 != 0) {
+          const uint16x8_t v_zero_mask1 =
+              quantize_fp_nz_8(v_c1, v_a1, qcoeff_ptr + 8, dqcoeff_ptr + 8,
+                               v_quant, v_dequant, v_round);
+          const int16x8_t v_nz_iscan0 =
+              vorrq_s16(vld1q_s16(iscan), vreinterpretq_s16_u16(v_zero_mask0));
+          const int16x8_t v_nz_iscan1 = vorrq_s16(
+              vld1q_s16(iscan + 8), vreinterpretq_s16_u16(v_zero_mask1));
+          v_eobmax_76543210 =
+              vmaxq_s16(v_eobmax_76543210, vmaxq_s16(v_nz_iscan0, v_nz_iscan1));
+        } else {
+          v_eobmax_76543210 =
+              get_max_lane_eob_zmask(iscan, v_eobmax_76543210, v_zero_mask0);
+          store_zero_8_tran_low(qcoeff_ptr + 8);
+          store_zero_8_tran_low(dqcoeff_ptr + 8);
+        }
+      } else {
+        store_zero_8_tran_low(qcoeff_ptr);
+        store_zero_8_tran_low(dqcoeff_ptr);
+        const uint16x8_t v_zero_mask1 =
+            quantize_fp_nz_8(v_c1, v_a1, qcoeff_ptr + 8, dqcoeff_ptr + 8,
+                             v_quant, v_dequant, v_round);
+        v_eobmax_76543210 =
+            get_max_lane_eob_zmask(iscan + 8, v_eobmax_76543210, v_zero_mask1);
+      }
+    } else {
+      store_zero_8_tran_low(qcoeff_ptr);
+      store_zero_8_tran_low(qcoeff_ptr + 8);
+      store_zero_8_tran_low(dqcoeff_ptr);
+      store_zero_8_tran_low(dqcoeff_ptr + 8);
+    }
+
+    coeff_ptr += 16;
+    qcoeff_ptr += 16;
+    dqcoeff_ptr += 16;
+    iscan += 16;
+  } while (coeff_ptr != coeff_end);
+
   *eob_ptr = get_max_eob(v_eobmax_76543210);
 }

@@ -380,6 +494,7 @@ void av1_quantize_fp_64x64_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
                               iscan, 2);
 }

+// Returns a zero-mask (0xFFFF for quantized zero lanes, 0x0000 otherwise).
 static inline uint16x8_t quantize_b_logscale0_8(
     int16x8_t coeff, int16x8_t abs, uint16x8_t cond, int16x8_t round,
     int16x8_t dequant, int16x8_t quant, int16x8_t quant_shift,
@@ -391,20 +506,15 @@ static inline uint16x8_t quantize_b_logscale0_8(
   int16x8_t tmp = vqaddq_s16(abs, round);
   tmp = vsraq_n_s16(tmp, vqdmulhq_s16(tmp, quant), 1);
   tmp = vqdmulhq_s16(tmp, quant_shift);
+  tmp = vandq_s16(tmp, vreinterpretq_s16_u16(cond));

-  int16x8_t qcoeff = vsubq_s16(veorq_s16(tmp, coeff_sign), coeff_sign);
-  qcoeff = vbslq_s16(cond, qcoeff, zero);
+  const int16x8_t qcoeff = vsubq_s16(veorq_s16(tmp, coeff_sign), coeff_sign);
   store_s16q_to_tran_low(qcoeff_ptr, qcoeff);

-  int16x8_t dqcoeff = vmulq_s16(tmp, dequant);
-  dqcoeff = vsubq_s16(veorq_s16(dqcoeff, coeff_sign), coeff_sign);
-  dqcoeff = vbslq_s16(cond, dqcoeff, zero);
+  const int16x8_t dqcoeff = vmulq_s16(qcoeff, dequant);
   store_s16q_to_tran_low(dqcoeff_ptr, dqcoeff);

-  uint16x8_t tmp_mask = vcgtq_s16(tmp, zero);
-  uint16x8_t nz_mask = vandq_u16(tmp_mask, cond);
-
-  return nz_mask;
+  return vcleq_s16(tmp, zero);
 }

 void aom_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
@@ -415,65 +525,140 @@ void aom_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
                          uint16_t *eob_ptr, const int16_t *scan,
                          const int16_t *iscan) {
   (void)scan;
+  assert(n_coeffs >= 16 && (n_coeffs & 15) == 0);

-  int16x8_t v_eobmax_76543210 = vdupq_n_s16(-1);
+  const int16x8_t v_zbins0 = vld1q_s16(zbin_ptr);
+  const int16x8_t v_zbins = vdupq_lane_s16(vget_low_s16(v_zbins0), 1);
+
+  const int16x8_t v_coeff0 = load_tran_low_to_s16q(coeff_ptr);
+  const int16x8_t v_coeff1 = load_tran_low_to_s16q(coeff_ptr + 8);
+  const int16x8_t v_abs0 = vabsq_s16(v_coeff0);
+  const int16x8_t v_abs1 = vabsq_s16(v_coeff1);
+  const uint16x8_t v_cond0 = vcgeq_s16(v_abs0, v_zbins0);
+  const uint16x8_t v_cond1 = vcgeq_s16(v_abs1, v_zbins);
+
+  uint64_t m0, m1;
+  mask_nonzero_pair_u16(v_cond0, v_cond1, &m0, &m1);

-  const int16x8_t v_zbins = vdupq_n_s16(zbin_ptr[1]);
-  int16x8_t v_round = vdupq_n_s16(round_ptr[1]);
-  int16x8_t v_dequant = vdupq_n_s16(dequant_ptr[1]);
-  int16x8_t v_quant = vdupq_n_s16(quant_ptr[1]);
+  if ((m0 | m1) == 0 && n_coeffs == 16) {
+    store_zero_8_tran_low(qcoeff_ptr);
+    store_zero_8_tran_low(qcoeff_ptr + 8);
+    store_zero_8_tran_low(dqcoeff_ptr);
+    store_zero_8_tran_low(dqcoeff_ptr + 8);
+    *eob_ptr = 0;
+    return;
+  }
+
+  const int16x8_t v_round0 = vld1q_s16(round_ptr);
+  const int16x8_t v_dequant0 = vld1q_s16(dequant_ptr);
+  const int16x8_t v_quant0 = vld1q_s16(quant_ptr);
+  const int16x8_t v_quant_shift0 = vshrq_n_s16(vld1q_s16(quant_shift_ptr), 1);
+  const int16x8_t v_round = vdupq_lane_s16(vget_low_s16(v_round0), 1);
+  const int16x8_t v_dequant = vdupq_lane_s16(vget_low_s16(v_dequant0), 1);
+  const int16x8_t v_quant = vdupq_lane_s16(vget_low_s16(v_quant0), 1);
   // Shift by 1 in order to save one shift in the kernel function.
-  int16x8_t v_quant_shift = vdupq_n_s16(quant_shift_ptr[1] >> 1);
+  const int16x8_t v_quant_shift =
+      vdupq_lane_s16(vget_low_s16(v_quant_shift0), 1);
+
+  int16x8_t v_eobmax_76543210;
+  const tran_low_t *const coeff_end = coeff_ptr + n_coeffs;
+
+  if ((m0 | m1) != 0) {
+    if (m0 != 0) {
+      const uint16x8_t v_zero_mask0 = quantize_b_logscale0_8(
+          v_coeff0, v_abs0, v_cond0, v_round0, v_dequant0, v_quant0,
+          v_quant_shift0, qcoeff_ptr, dqcoeff_ptr);
+      v_eobmax_76543210 =
+          vorrq_s16(vld1q_s16(iscan), vreinterpretq_s16_u16(v_zero_mask0));
+
+      if (m1 != 0) {
+        const uint16x8_t v_zero_mask1 = quantize_b_logscale0_8(
+            v_coeff1, v_abs1, v_cond1, v_round, v_dequant, v_quant,
+            v_quant_shift, qcoeff_ptr + 8, dqcoeff_ptr + 8);
+        v_eobmax_76543210 =
+            get_max_lane_eob_zmask(iscan + 8, v_eobmax_76543210, v_zero_mask1);
+      } else {
+        store_zero_8_tran_low(qcoeff_ptr + 8);
+        store_zero_8_tran_low(dqcoeff_ptr + 8);
+      }
+    } else {
+      store_zero_8_tran_low(qcoeff_ptr);
+      store_zero_8_tran_low(dqcoeff_ptr);
+      const uint16x8_t v_zero_mask1 = quantize_b_logscale0_8(
+          v_coeff1, v_abs1, v_cond1, v_round, v_dequant, v_quant, v_quant_shift,
+          qcoeff_ptr + 8, dqcoeff_ptr + 8);
+      v_eobmax_76543210 =
+          vorrq_s16(vld1q_s16(iscan + 8), vreinterpretq_s16_u16(v_zero_mask1));
+    }

-  int16x8_t v_zbins0 = vsetq_lane_s16(zbin_ptr[0], v_zbins, 0);
-  int16x8_t v_coeff = load_tran_low_to_s16q(coeff_ptr);
-  int16x8_t v_abs = vabsq_s16(v_coeff);
-  uint16x8_t v_cond = vcgeq_s16(v_abs, v_zbins0);
-
-  if (mask_nonzero_u16(v_cond)) {
-    v_round = vsetq_lane_s16(round_ptr[0], v_round, 0);
-    v_quant = vsetq_lane_s16(quant_ptr[0], v_quant, 0);
-    v_dequant = vsetq_lane_s16(dequant_ptr[0], v_dequant, 0);
-    // Shift by 1 in order to save one shift in the kernel function.
-    v_quant_shift = vsetq_lane_s16(quant_shift_ptr[0] >> 1, v_quant_shift, 0);
-
-    const uint16x8_t v_nz_mask =
-        quantize_b_logscale0_8(v_coeff, v_abs, v_cond, v_round, v_dequant,
-                               v_quant, v_quant_shift, qcoeff_ptr, dqcoeff_ptr);
-
-    v_round = vdupq_lane_s16(vget_low_s16(v_round), 1);
-    v_quant = vdupq_lane_s16(vget_low_s16(v_quant), 1);
-    v_dequant = vdupq_lane_s16(vget_low_s16(v_dequant), 1);
-    v_quant_shift = vdupq_lane_s16(vget_low_s16(v_quant_shift), 1);
-
-    int16x8_t v_iscan = vld1q_s16(iscan);
-    int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210);
-    v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210);
+    coeff_ptr += 16;
+    if (coeff_ptr == coeff_end) {
+      *eob_ptr = get_max_eob(v_eobmax_76543210);
+      return;
+    }
   } else {
     store_zero_8_tran_low(qcoeff_ptr);
+    store_zero_8_tran_low(qcoeff_ptr + 8);
     store_zero_8_tran_low(dqcoeff_ptr);
+    store_zero_8_tran_low(dqcoeff_ptr + 8);
+    v_eobmax_76543210 = vdupq_n_s16(-1);
+    coeff_ptr += 16;
   }

-  intptr_t i = 8;
+  qcoeff_ptr += 16;
+  dqcoeff_ptr += 16;
+  iscan += 16;
   do {
-    v_coeff = load_tran_low_to_s16q(coeff_ptr + i);
-    v_abs = vabsq_s16(v_coeff);
-    v_cond = vcgeq_s16(v_abs, v_zbins);
-
-    if (mask_nonzero_u16(v_cond)) {
-      const uint16x8_t v_nz_mask = quantize_b_logscale0_8(
-          v_coeff, v_abs, v_cond, v_round, v_dequant, v_quant, v_quant_shift,
-          qcoeff_ptr + i, dqcoeff_ptr + i);
-
-      int16x8_t v_iscan = vld1q_s16(iscan + i);
-      int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210);
-      v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210);
+    const int16x8_t v_c0 = load_tran_low_to_s16q(coeff_ptr);
+    const int16x8_t v_c1 = load_tran_low_to_s16q(coeff_ptr + 8);
+    const int16x8_t v_a0 = vabsq_s16(v_c0);
+    const int16x8_t v_a1 = vabsq_s16(v_c1);
+    const uint16x8_t v_m0 = vcgeq_s16(v_a0, v_zbins);
+    const uint16x8_t v_m1 = vcgeq_s16(v_a1, v_zbins);
+    uint64_t l_m0, l_m1;
+    mask_nonzero_pair_u16(v_m0, v_m1, &l_m0, &l_m1);
+
+    if ((l_m0 | l_m1) != 0) {
+      if (l_m0 != 0) {
+        const uint16x8_t v_zero_mask0 = quantize_b_logscale0_8(
+            v_c0, v_a0, v_m0, v_round, v_dequant, v_quant, v_quant_shift,
+            qcoeff_ptr, dqcoeff_ptr);
+        if (l_m1 != 0) {
+          const uint16x8_t v_zero_mask1 = quantize_b_logscale0_8(
+              v_c1, v_a1, v_m1, v_round, v_dequant, v_quant, v_quant_shift,
+              qcoeff_ptr + 8, dqcoeff_ptr + 8);
+          const int16x8_t v_nz_iscan0 =
+              vorrq_s16(vld1q_s16(iscan), vreinterpretq_s16_u16(v_zero_mask0));
+          const int16x8_t v_nz_iscan1 = vorrq_s16(
+              vld1q_s16(iscan + 8), vreinterpretq_s16_u16(v_zero_mask1));
+          v_eobmax_76543210 =
+              vmaxq_s16(v_eobmax_76543210, vmaxq_s16(v_nz_iscan0, v_nz_iscan1));
+        } else {
+          v_eobmax_76543210 =
+              get_max_lane_eob_zmask(iscan, v_eobmax_76543210, v_zero_mask0);
+          store_zero_8_tran_low(qcoeff_ptr + 8);
+          store_zero_8_tran_low(dqcoeff_ptr + 8);
+        }
+      } else {
+        store_zero_8_tran_low(qcoeff_ptr);
+        store_zero_8_tran_low(dqcoeff_ptr);
+        const uint16x8_t v_zero_mask1 = quantize_b_logscale0_8(
+            v_c1, v_a1, v_m1, v_round, v_dequant, v_quant, v_quant_shift,
+            qcoeff_ptr + 8, dqcoeff_ptr + 8);
+        v_eobmax_76543210 =
+            get_max_lane_eob_zmask(iscan + 8, v_eobmax_76543210, v_zero_mask1);
+      }
     } else {
-      store_zero_8_tran_low(qcoeff_ptr + i);
-      store_zero_8_tran_low(dqcoeff_ptr + i);
+      store_zero_8_tran_low(qcoeff_ptr);
+      store_zero_8_tran_low(qcoeff_ptr + 8);
+      store_zero_8_tran_low(dqcoeff_ptr);
+      store_zero_8_tran_low(dqcoeff_ptr + 8);
     }
-    i += 8;
-  } while (i != n_coeffs);
+    coeff_ptr += 16;
+    qcoeff_ptr += 16;
+    dqcoeff_ptr += 16;
+    iscan += 16;
+  } while (coeff_ptr != coeff_end);
   *eob_ptr = get_max_eob(v_eobmax_76543210);
 }

@@ -928,6 +1113,7 @@ void aom_quantize_b_helper_neon(
   }
 }

+// Returns a zero-mask (0xFFFF for quantized zero lanes, 0x0000 otherwise).
 static inline uint16x8_t quantize_b_logscale1_8(
     int16x8_t coeff, int16x8_t abs, uint16x8_t cond, int16x8_t round,
     int16x8_t dequant, int16x8_t quant, int16x8_t quant_shift,
@@ -939,118 +1125,22 @@ static inline uint16x8_t quantize_b_logscale1_8(
   int16x8_t tmp = vqaddq_s16(abs, round);
   tmp = vsraq_n_s16(tmp, vqdmulhq_s16(tmp, quant), 1);
   tmp = vqdmulhq_s16(tmp, quant_shift);
+  tmp = vandq_s16(tmp, vreinterpretq_s16_u16(cond));

-  int16x8_t qcoeff = vsubq_s16(veorq_s16(tmp, coeff_sign), coeff_sign);
-  qcoeff = vbslq_s16(cond, qcoeff, zero);
+  const int16x8_t qcoeff = vsubq_s16(veorq_s16(tmp, coeff_sign), coeff_sign);
   store_s16q_to_tran_low(qcoeff_ptr, qcoeff);

   // Shift by log_scale = 1.
-  int16x8_t dqcoeff = vreinterpretq_s16_u16(vhaddq_u16(
+  const int16x8_t abs_dqcoeff = vreinterpretq_s16_u16(vhaddq_u16(
       vreinterpretq_u16_s16(vmulq_s16(tmp, dequant)), vdupq_n_u16(0)));
-  dqcoeff = vsubq_s16(veorq_s16(dqcoeff, coeff_sign), coeff_sign);
-  dqcoeff = vbslq_s16(cond, dqcoeff, zero);
+  const int16x8_t dqcoeff =
+      vsubq_s16(veorq_s16(abs_dqcoeff, coeff_sign), coeff_sign);
   store_s16q_to_tran_low(dqcoeff_ptr, dqcoeff);

-  uint16x8_t tmp_mask = vcgtq_s16(tmp, zero);
-  const uint16x8_t nz_mask = vandq_u16(tmp_mask, cond);
-
-  return nz_mask;
-}
-
-void aom_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
-                               const int16_t *zbin_ptr,
-                               const int16_t *round_ptr,
-                               const int16_t *quant_ptr,
-                               const int16_t *quant_shift_ptr,
-                               tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
-                               const int16_t *dequant_ptr, uint16_t *eob_ptr,
-                               const int16_t *scan, const int16_t *iscan) {
-  (void)scan;
-
-  const int log_scale = 1;
-  const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0], log_scale),
-                         ROUND_POWER_OF_TWO(zbin_ptr[1], log_scale) };
-  const int rounds[2] = { ROUND_POWER_OF_TWO(round_ptr[0], log_scale),
-                          ROUND_POWER_OF_TWO(round_ptr[1], log_scale) };
-
-  int16x8_t v_eobmax_76543210 = vdupq_n_s16(-1);
-
-  const int16x8_t v_zbins = vdupq_n_s16(zbins[1]);
-  int16x8_t v_round = vdupq_n_s16(rounds[1]);
-  int16x8_t v_dequant = vdupq_n_s16(dequant_ptr[1]);
-  int16x8_t v_quant = vdupq_n_s16(quant_ptr[1]);
-  int16x8_t v_quant_shift = vdupq_n_s16(quant_shift_ptr[1]);
-
-  assert(n_coeffs > 16 && (n_coeffs & 15) == 0);
-  intptr_t non_zero_count = n_coeffs;
-  do {
-    const int16x8_t v_coeff_a =
-        load_tran_low_to_s16q(coeff_ptr + non_zero_count - 8);
-    const int16x8_t v_coeff_b =
-        load_tran_low_to_s16q(coeff_ptr + non_zero_count - 16);
-    const uint16x8_t v_mask_a = vcgeq_s16(vabsq_s16(v_coeff_a), v_zbins);
-    const uint16x8_t v_mask_b = vcgeq_s16(vabsq_s16(v_coeff_b), v_zbins);
-    if (mask_nonzero_u16(vorrq_u16(v_mask_a, v_mask_b))) {
-      break;
-    }
-    non_zero_count -= 16;
-    store_zero_8_tran_low(qcoeff_ptr + non_zero_count);
-    store_zero_8_tran_low(qcoeff_ptr + non_zero_count + 8);
-    store_zero_8_tran_low(dqcoeff_ptr + non_zero_count);
-    store_zero_8_tran_low(dqcoeff_ptr + non_zero_count + 8);
-  } while (non_zero_count != 16);
-
-  int16x8_t v_zbins0 = vsetq_lane_s16(zbins[0], v_zbins, 0);
-  int16x8_t v_coeff = load_tran_low_to_s16q(coeff_ptr);
-  int16x8_t v_abs = vabsq_s16(v_coeff);
-  uint16x8_t v_cond = vcgeq_s16(v_abs, v_zbins0);
-
-  if (mask_nonzero_u16(v_cond)) {
-    v_round = vsetq_lane_s16(rounds[0], v_round, 0);
-    v_quant = vsetq_lane_s16(quant_ptr[0], v_quant, 0);
-    v_dequant = vsetq_lane_s16(dequant_ptr[0], v_dequant, 0);
-    v_quant_shift = vsetq_lane_s16(quant_shift_ptr[0], v_quant_shift, 0);
-
-    const uint16x8_t v_nz_mask =
-        quantize_b_logscale1_8(v_coeff, v_abs, v_cond, v_round, v_dequant,
-                               v_quant, v_quant_shift, qcoeff_ptr, dqcoeff_ptr);
-
-    v_round = vdupq_lane_s16(vget_low_s16(v_round), 1);
-    v_quant = vdupq_lane_s16(vget_low_s16(v_quant), 1);
-    v_dequant = vdupq_lane_s16(vget_low_s16(v_dequant), 1);
-    v_quant_shift = vdupq_lane_s16(vget_low_s16(v_quant_shift), 1);
-
-    int16x8_t v_iscan = vld1q_s16(iscan);
-    int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210);
-    v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210);
-  } else {
-    store_zero_8_tran_low(qcoeff_ptr);
-    store_zero_8_tran_low(dqcoeff_ptr);
-  }
-
-  intptr_t i = 8;
-  do {
-    v_coeff = load_tran_low_to_s16q(coeff_ptr + i);
-    v_abs = vabsq_s16(v_coeff);
-    v_cond = vcgeq_s16(v_abs, v_zbins);
-
-    if (mask_nonzero_u16(v_cond)) {
-      const uint16x8_t v_nz_mask = quantize_b_logscale1_8(
-          v_coeff, v_abs, v_cond, v_round, v_dequant, v_quant, v_quant_shift,
-          qcoeff_ptr + i, dqcoeff_ptr + i);
-
-      int16x8_t v_iscan = vld1q_s16(iscan + i);
-      int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210);
-      v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210);
-    } else {
-      store_zero_8_tran_low(qcoeff_ptr + i);
-      store_zero_8_tran_low(dqcoeff_ptr + i);
-    }
-    i += 8;
-  } while (i != non_zero_count);
-  *eob_ptr = get_max_eob(v_eobmax_76543210);
+  return vcleq_s16(tmp, zero);
 }

+// Returns a zero-mask (0xFFFF for quantized zero lanes, 0x0000 otherwise).
 static inline uint16x8_t quantize_b_logscale2_8(
     int16x8_t coeff, int16x8_t abs, uint16x8_t cond, int16x8_t round,
     int16x8_t dequant, int16x8_t quant, int16x8_t quant_shift,
@@ -1065,50 +1155,50 @@ static inline uint16x8_t quantize_b_logscale2_8(
   int16x8_t ones = vandq_s16(vshrq_n_s16(vmulq_s16(tmp, quant_shift), 14), one);
   tmp = vqdmulhq_s16(tmp, quant_shift);
   tmp = vaddq_s16(vshlq_s16(tmp, one), ones);
+  tmp = vandq_s16(tmp, vreinterpretq_s16_u16(cond));

-  int16x8_t qcoeff = vsubq_s16(veorq_s16(tmp, coeff_sign), coeff_sign);
-  qcoeff = vbslq_s16(cond, qcoeff, zero);
+  const int16x8_t qcoeff = vsubq_s16(veorq_s16(tmp, coeff_sign), coeff_sign);
   store_s16q_to_tran_low(qcoeff_ptr, qcoeff);

   // Shift right by log_scale = 2.
-  int16x8_t dqcoeff = vreinterpretq_s16_u16(
+  const int16x8_t dqcoeff_low = vreinterpretq_s16_u16(
       vshrq_n_u16(vreinterpretq_u16_s16(vmulq_s16(tmp, dequant)), 2));
-  dqcoeff = vorrq_s16(vshlq_n_s16(vqdmulhq_s16(tmp, dequant), 13), dqcoeff);
-  dqcoeff = vsubq_s16(veorq_s16(dqcoeff, coeff_sign), coeff_sign);
-  dqcoeff = vbslq_s16(cond, dqcoeff, zero);
+  const int16x8_t abs_dqcoeff =
+      vorrq_s16(vshlq_n_s16(vqdmulhq_s16(tmp, dequant), 13), dqcoeff_low);
+  const int16x8_t dqcoeff =
+      vsubq_s16(veorq_s16(abs_dqcoeff, coeff_sign), coeff_sign);
   store_s16q_to_tran_low(dqcoeff_ptr, dqcoeff);

-  uint16x8_t tmp_mask = vcgtq_s16(tmp, zero);
-  const uint16x8_t nz_mask = vandq_u16(tmp_mask, cond);
-
-  return nz_mask;
+  return vcleq_s16(tmp, zero);
 }

-void aom_quantize_b_64x64_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
-                               const int16_t *zbin_ptr,
-                               const int16_t *round_ptr,
-                               const int16_t *quant_ptr,
-                               const int16_t *quant_shift_ptr,
-                               tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
-                               const int16_t *dequant_ptr, uint16_t *eob_ptr,
-                               const int16_t *scan, const int16_t *iscan) {
-  (void)scan;
-
-  const int log_scale = 2;
-  const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0], log_scale),
-                         ROUND_POWER_OF_TWO(zbin_ptr[1], log_scale) };
-  const int rounds[2] = { ROUND_POWER_OF_TWO(round_ptr[0], log_scale),
-                          ROUND_POWER_OF_TWO(round_ptr[1], log_scale) };
+static AOM_FORCE_INLINE uint16x8_t quantize_b_logscale_8(
+    int16x8_t coeff, int16x8_t abs, uint16x8_t cond, int16x8_t round,
+    int16x8_t dequant, int16x8_t quant, int16x8_t quant_shift,
+    tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, int log_scale) {
+  if (log_scale == 1) {
+    return quantize_b_logscale1_8(coeff, abs, cond, round, dequant, quant,
+                                  quant_shift, qcoeff_ptr, dqcoeff_ptr);
+  }
+  assert(log_scale == 2);
+  return quantize_b_logscale2_8(coeff, abs, cond, round, dequant, quant,
+                                quant_shift, qcoeff_ptr, dqcoeff_ptr);
+}

-  int16x8_t v_eobmax_76543210 = vdupq_n_s16(-1);
+static AOM_FORCE_INLINE void quantize_b_no_qmatrix_neon(
+    const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
+    const int16_t *round_ptr, const int16_t *quant_ptr,
+    const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
+    tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
+    const int16_t *iscan, int log_scale) {
+  assert(n_coeffs > 16 && (n_coeffs & 15) == 0);
+  assert(log_scale == 1 || log_scale == 2);

-  const int16x8_t v_zbins = vdupq_n_s16(zbins[1]);
-  int16x8_t v_round = vdupq_n_s16(rounds[1]);
-  int16x8_t v_dequant = vdupq_n_s16(dequant_ptr[1]);
-  int16x8_t v_quant = vdupq_n_s16(quant_ptr[1]);
-  int16x8_t v_quant_shift = vdupq_n_s16(quant_shift_ptr[1]);
+  const int16x8_t v_zbins_raw = vld1q_s16(zbin_ptr);
+  const int16x8_t v_zbins0 = (log_scale == 1) ? vrshrq_n_s16(v_zbins_raw, 1)
+                                              : vrshrq_n_s16(v_zbins_raw, 2);
+  const int16x8_t v_zbins = vdupq_lane_s16(vget_low_s16(v_zbins0), 1);

-  assert(n_coeffs > 16 && (n_coeffs & 15) == 0);
   intptr_t non_zero_count = n_coeffs;
   do {
     const int16x8_t v_coeff_a =
@@ -1127,57 +1217,162 @@ void aom_quantize_b_64x64_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
     store_zero_8_tran_low(dqcoeff_ptr + non_zero_count + 8);
   } while (non_zero_count != 16);

-  int16x8_t v_zbins0 = vsetq_lane_s16(zbins[0], v_zbins, 0);
-  int16x8_t v_coeff = load_tran_low_to_s16q(coeff_ptr);
-  int16x8_t v_abs = vabsq_s16(v_coeff);
-  uint16x8_t v_cond = vcgeq_s16(v_abs, v_zbins0);
-
-  if (mask_nonzero_u16(v_cond)) {
-    v_round = vsetq_lane_s16(rounds[0], v_round, 0);
-    v_quant = vsetq_lane_s16(quant_ptr[0], v_quant, 0);
-    v_dequant = vsetq_lane_s16(dequant_ptr[0], v_dequant, 0);
-    v_quant_shift = vsetq_lane_s16(quant_shift_ptr[0], v_quant_shift, 0);
-
-    const uint16x8_t v_nz_mask =
-        quantize_b_logscale2_8(v_coeff, v_abs, v_cond, v_round, v_dequant,
-                               v_quant, v_quant_shift, qcoeff_ptr, dqcoeff_ptr);
-
-    v_round = vdupq_lane_s16(vget_low_s16(v_round), 1);
-    v_quant = vdupq_lane_s16(vget_low_s16(v_quant), 1);
-    v_dequant = vdupq_lane_s16(vget_low_s16(v_dequant), 1);
-    v_quant_shift = vdupq_lane_s16(vget_low_s16(v_quant_shift), 1);
-
-    int16x8_t v_iscan = vld1q_s16(iscan);
-    int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210);
-    v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210);
+  const int16x8_t v_coeff0 = load_tran_low_to_s16q(coeff_ptr);
+  const int16x8_t v_coeff1 = load_tran_low_to_s16q(coeff_ptr + 8);
+  const int16x8_t v_abs0 = vabsq_s16(v_coeff0);
+  const int16x8_t v_abs1 = vabsq_s16(v_coeff1);
+  const uint16x8_t v_cond0 = vcgeq_s16(v_abs0, v_zbins0);
+  const uint16x8_t v_cond1 = vcgeq_s16(v_abs1, v_zbins);
+
+  uint64_t m0, m1;
+  mask_nonzero_pair_u16(v_cond0, v_cond1, &m0, &m1);
+  if ((m0 | m1) == 0 && non_zero_count == 16) {
+    store_zero_8_tran_low(qcoeff_ptr);
+    store_zero_8_tran_low(qcoeff_ptr + 8);
+    store_zero_8_tran_low(dqcoeff_ptr);
+    store_zero_8_tran_low(dqcoeff_ptr + 8);
+    *eob_ptr = 0;
+    return;
+  }
+
+  const int16x8_t v_quant0 = vld1q_s16(quant_ptr);
+  const int16x8_t v_dequant0 = vld1q_s16(dequant_ptr);
+  const int16x8_t v_quant_shift0 = vld1q_s16(quant_shift_ptr);
+  const int16x8_t v_round_raw = vld1q_s16(round_ptr);
+  const int16x8_t v_round0 = (log_scale == 1) ? vrshrq_n_s16(v_round_raw, 1)
+                                              : vrshrq_n_s16(v_round_raw, 2);
+  const int16x8_t v_quant = vdupq_lane_s16(vget_low_s16(v_quant0), 1);
+  const int16x8_t v_dequant = vdupq_lane_s16(vget_low_s16(v_dequant0), 1);
+  const int16x8_t v_quant_shift =
+      vdupq_lane_s16(vget_low_s16(v_quant_shift0), 1);
+  const int16x8_t v_round = vdupq_lane_s16(vget_low_s16(v_round0), 1);
+
+  int16x8_t v_eobmax_76543210;
+  if ((m0 | m1) != 0) {
+    if (m0 != 0) {
+      const uint16x8_t v_zero_mask0 = quantize_b_logscale_8(
+          v_coeff0, v_abs0, v_cond0, v_round0, v_dequant0, v_quant0,
+          v_quant_shift0, qcoeff_ptr, dqcoeff_ptr, log_scale);
+      v_eobmax_76543210 =
+          vorrq_s16(vld1q_s16(iscan), vreinterpretq_s16_u16(v_zero_mask0));
+      if (m1 != 0) {
+        const uint16x8_t v_zero_mask1 = quantize_b_logscale_8(
+            v_coeff1, v_abs1, v_cond1, v_round, v_dequant, v_quant,
+            v_quant_shift, qcoeff_ptr + 8, dqcoeff_ptr + 8, log_scale);
+        v_eobmax_76543210 =
+            get_max_lane_eob_zmask(iscan + 8, v_eobmax_76543210, v_zero_mask1);
+      } else {
+        store_zero_8_tran_low(qcoeff_ptr + 8);
+        store_zero_8_tran_low(dqcoeff_ptr + 8);
+      }
+    } else {
+      store_zero_8_tran_low(qcoeff_ptr);
+      store_zero_8_tran_low(dqcoeff_ptr);
+      const uint16x8_t v_zero_mask1 = quantize_b_logscale_8(
+          v_coeff1, v_abs1, v_cond1, v_round, v_dequant, v_quant, v_quant_shift,
+          qcoeff_ptr + 8, dqcoeff_ptr + 8, log_scale);
+      v_eobmax_76543210 =
+          vorrq_s16(vld1q_s16(iscan + 8), vreinterpretq_s16_u16(v_zero_mask1));
+    }
+    if (non_zero_count == 16) {
+      *eob_ptr = get_max_eob(v_eobmax_76543210);
+      return;
+    }
   } else {
     store_zero_8_tran_low(qcoeff_ptr);
+    store_zero_8_tran_low(qcoeff_ptr + 8);
     store_zero_8_tran_low(dqcoeff_ptr);
+    store_zero_8_tran_low(dqcoeff_ptr + 8);
+    v_eobmax_76543210 = vdupq_n_s16(-1);
   }

-  intptr_t i = 8;
+  const tran_low_t *const coeff_end = coeff_ptr + non_zero_count;
+  coeff_ptr += 16;
+  qcoeff_ptr += 16;
+  dqcoeff_ptr += 16;
+  iscan += 16;
   do {
-    v_coeff = load_tran_low_to_s16q(coeff_ptr + i);
-    v_abs = vabsq_s16(v_coeff);
-    v_cond = vcgeq_s16(v_abs, v_zbins);
-
-    if (mask_nonzero_u16(v_cond)) {
-      const uint16x8_t v_nz_mask = quantize_b_logscale2_8(
-          v_coeff, v_abs, v_cond, v_round, v_dequant, v_quant, v_quant_shift,
-          qcoeff_ptr + i, dqcoeff_ptr + i);
-
-      int16x8_t v_iscan = vld1q_s16(iscan + i);
-      int16x8_t v_eobmax = vmaxq_s16(v_iscan, v_eobmax_76543210);
-      v_eobmax_76543210 = vbslq_s16(v_nz_mask, v_eobmax, v_eobmax_76543210);
+    const int16x8_t v_c0 = load_tran_low_to_s16q(coeff_ptr);
+    const int16x8_t v_c1 = load_tran_low_to_s16q(coeff_ptr + 8);
+    const int16x8_t v_a0 = vabsq_s16(v_c0);
+    const int16x8_t v_a1 = vabsq_s16(v_c1);
+    const uint16x8_t v_m0 = vcgeq_s16(v_a0, v_zbins);
+    const uint16x8_t v_m1 = vcgeq_s16(v_a1, v_zbins);
+    uint64_t l_m0, l_m1;
+    mask_nonzero_pair_u16(v_m0, v_m1, &l_m0, &l_m1);
+
+    if ((l_m0 | l_m1) != 0) {
+      if (l_m0 != 0) {
+        const uint16x8_t v_zero_mask0 = quantize_b_logscale_8(
+            v_c0, v_a0, v_m0, v_round, v_dequant, v_quant, v_quant_shift,
+            qcoeff_ptr, dqcoeff_ptr, log_scale);
+        if (l_m1 != 0) {
+          const uint16x8_t v_zero_mask1 = quantize_b_logscale_8(
+              v_c1, v_a1, v_m1, v_round, v_dequant, v_quant, v_quant_shift,
+              qcoeff_ptr + 8, dqcoeff_ptr + 8, log_scale);
+          const int16x8_t v_nz_iscan0 =
+              vorrq_s16(vld1q_s16(iscan), vreinterpretq_s16_u16(v_zero_mask0));
+          const int16x8_t v_nz_iscan1 = vorrq_s16(
+              vld1q_s16(iscan + 8), vreinterpretq_s16_u16(v_zero_mask1));
+          v_eobmax_76543210 =
+              vmaxq_s16(v_eobmax_76543210, vmaxq_s16(v_nz_iscan0, v_nz_iscan1));
+        } else {
+          v_eobmax_76543210 =
+              get_max_lane_eob_zmask(iscan, v_eobmax_76543210, v_zero_mask0);
+          store_zero_8_tran_low(qcoeff_ptr + 8);
+          store_zero_8_tran_low(dqcoeff_ptr + 8);
+        }
+      } else {
+        store_zero_8_tran_low(qcoeff_ptr);
+        store_zero_8_tran_low(dqcoeff_ptr);
+        const uint16x8_t v_zero_mask1 = quantize_b_logscale_8(
+            v_c1, v_a1, v_m1, v_round, v_dequant, v_quant, v_quant_shift,
+            qcoeff_ptr + 8, dqcoeff_ptr + 8, log_scale);
+        v_eobmax_76543210 =
+            get_max_lane_eob_zmask(iscan + 8, v_eobmax_76543210, v_zero_mask1);
+      }
     } else {
-      store_zero_8_tran_low(qcoeff_ptr + i);
-      store_zero_8_tran_low(dqcoeff_ptr + i);
+      store_zero_8_tran_low(qcoeff_ptr);
+      store_zero_8_tran_low(qcoeff_ptr + 8);
+      store_zero_8_tran_low(dqcoeff_ptr);
+      store_zero_8_tran_low(dqcoeff_ptr + 8);
     }
-    i += 8;
-  } while (i != non_zero_count);
+    coeff_ptr += 16;
+    qcoeff_ptr += 16;
+    dqcoeff_ptr += 16;
+    iscan += 16;
+  } while (coeff_ptr != coeff_end);
   *eob_ptr = get_max_eob(v_eobmax_76543210);
 }

+void aom_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
+                               const int16_t *zbin_ptr,
+                               const int16_t *round_ptr,
+                               const int16_t *quant_ptr,
+                               const int16_t *quant_shift_ptr,
+                               tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
+                               const int16_t *dequant_ptr, uint16_t *eob_ptr,
+                               const int16_t *scan, const int16_t *iscan) {
+  (void)scan;
+  quantize_b_no_qmatrix_neon(coeff_ptr, n_coeffs, zbin_ptr, round_ptr,
+                             quant_ptr, quant_shift_ptr, qcoeff_ptr,
+                             dqcoeff_ptr, dequant_ptr, eob_ptr, iscan, 1);
+}
+
+void aom_quantize_b_64x64_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
+                               const int16_t *zbin_ptr,
+                               const int16_t *round_ptr,
+                               const int16_t *quant_ptr,
+                               const int16_t *quant_shift_ptr,
+                               tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
+                               const int16_t *dequant_ptr, uint16_t *eob_ptr,
+                               const int16_t *scan, const int16_t *iscan) {
+  (void)scan;
+  quantize_b_no_qmatrix_neon(coeff_ptr, n_coeffs, zbin_ptr, round_ptr,
+                             quant_ptr, quant_shift_ptr, qcoeff_ptr,
+                             dqcoeff_ptr, dequant_ptr, eob_ptr, iscan, 2);
+}
+
 static AOM_FORCE_INLINE int mask_any_nonzero_u32(uint32x4_t m0, uint32x4_t m1) {
   const uint32x4_t any = vorrq_u32(m0, m1);
 #if AOM_ARCH_AARCH64
diff --git a/test/quantize_func_test.cc b/test/quantize_func_test.cc
index 7a1bfad3ed..9daf1a25bb 100644
--- a/test/quantize_func_test.cc
+++ b/test/quantize_func_test.cc
@@ -166,7 +166,10 @@ class QuantizeTestBase
     for (int i = 0; i < test_num; ++i) {
       if (is_loop) FillCoeffRandom();

-      memset(qcoeff_ref, 0, 5 * n_coeffs * sizeof(*qcoeff_ref));
+      memset(qcoeff_ref, 0, 2 * n_coeffs * sizeof(*qcoeff_ref));
+      memset(qcoeff, 0x5a, 2 * n_coeffs * sizeof(*qcoeff));
+      eob[0] = 0;
+      eob[1] = 0x5a5a;

       RunQuantizeFunc(coeff_ptr, n_coeffs, zbin, round, quant, quant_shift,
                       qcoeff, qcoeff_ref, dqcoeff, dqcoeff_ref, dequant,
@@ -947,6 +950,12 @@ INSTANTIATE_TEST_SUITE_P(NEON, LowPrecisionQuantizeTest,
                          ::testing::ValuesIn(kLPQParamArrayNEON));

 const QuantizeParam<QuantizeFunc> kQParamArrayNEON[] = {
+  make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_neon,
+             static_cast<TX_SIZE>(TX_4X4), TYPE_FP, AOM_BITS_8),
+  make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_neon,
+             static_cast<TX_SIZE>(TX_4X8), TYPE_FP, AOM_BITS_8),
+  make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_neon,
+             static_cast<TX_SIZE>(TX_8X8), TYPE_FP, AOM_BITS_8),
   make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_neon,
              static_cast<TX_SIZE>(TX_16X16), TYPE_FP, AOM_BITS_8),
   make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_neon,
@@ -957,12 +966,22 @@ const QuantizeParam<QuantizeFunc> kQParamArrayNEON[] = {
              static_cast<TX_SIZE>(TX_8X32), TYPE_FP, AOM_BITS_8),
   make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_neon,
              static_cast<TX_SIZE>(TX_32X8), TYPE_FP, AOM_BITS_8),
+  make_tuple(&av1_quantize_fp_32x32_c, &av1_quantize_fp_32x32_neon,
+             static_cast<TX_SIZE>(TX_16X32), TYPE_FP, AOM_BITS_8),
   make_tuple(&av1_quantize_fp_32x32_c, &av1_quantize_fp_32x32_neon,
              static_cast<TX_SIZE>(TX_32X32), TYPE_FP, AOM_BITS_8),
   make_tuple(&av1_quantize_fp_64x64_c, &av1_quantize_fp_64x64_neon,
              static_cast<TX_SIZE>(TX_64X64), TYPE_FP, AOM_BITS_8),
+  make_tuple(&aom_quantize_b_c, &aom_quantize_b_neon,
+             static_cast<TX_SIZE>(TX_4X4), TYPE_B, AOM_BITS_8),
+  make_tuple(&aom_quantize_b_c, &aom_quantize_b_neon,
+             static_cast<TX_SIZE>(TX_4X8), TYPE_B, AOM_BITS_8),
+  make_tuple(&aom_quantize_b_c, &aom_quantize_b_neon,
+             static_cast<TX_SIZE>(TX_8X8), TYPE_B, AOM_BITS_8),
   make_tuple(&aom_quantize_b_c, &aom_quantize_b_neon,
              static_cast<TX_SIZE>(TX_16X16), TYPE_B, AOM_BITS_8),
+  make_tuple(&aom_quantize_b_32x32_c, &aom_quantize_b_32x32_neon,
+             static_cast<TX_SIZE>(TX_16X32), TYPE_B, AOM_BITS_8),
   make_tuple(&aom_quantize_b_32x32_c, &aom_quantize_b_32x32_neon,
              static_cast<TX_SIZE>(TX_32X32), TYPE_B, AOM_BITS_8),
   make_tuple(&aom_quantize_b_64x64_c, &aom_quantize_b_64x64_neon,