Commit df063ff8bb for aom

commit df063ff8bbd62460dfd87b7b52de3a58d9108dfe
Author: Frank Galligan <fgalligan@google.com>
Date:   Sat Nov 14 11:52:29 2026 -0800

    av1,neon: Add av1_block_error_qm_neon

    Add NEON implementation of av1_block_error_qm_neon (8, 10, and 12-bit)
    for quantization-matrix weighted block error calculation, using dual
    64-bit accumulators and skipping zeroed 8-coefficient slices.

    Speedup: 2.00x-2.11x (Neoverse N2), 1.78x-1.95x (Cortex-X4), and
    1.41x-1.64x (Cortex-A520) vs C in ErrorBlockTest; -0.77% (1T) / -1.75%
    (4T) on Neoverse N3 and -0.41% (1T X4) / -2.65% (1T A720) / -1.88% (4T)
    on Pixel 9 avifenc (speed 6, bit-exact).

    TAG=agy
    CONV=490488d4-7005-402a-8cc4-c245227f80fe
    Change-Id: Ibf6e03909b166666666666666666666666666666

diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl
index e0e23cbbe7..5b1aa3e1ca 100644
--- a/av1/common/av1_rtcd_defs.pl
+++ b/av1/common/av1_rtcd_defs.pl
@@ -302,7 +302,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
   specialize qw/av1_block_error avx2 neon sve/, "$sse2_asm";

   add_proto qw/int64_t av1_block_error_qm/, "const tran_low_t *coeff, const tran_low_t *dqcoeff, intptr_t block_size, const qm_val_t *qmatrix, const int16_t *scan, int64_t *ssz, int bd";
-  specialize qw/av1_block_error_qm avx2/;
+  specialize qw/av1_block_error_qm avx2 neon/;

   add_proto qw/int64_t av1_block_error_lp/, "const int16_t *coeff, const int16_t *dqcoeff, intptr_t block_size";
   specialize qw/av1_block_error_lp sse2 avx2 neon sve/;
diff --git a/av1/encoder/arm/av1_error_neon.c b/av1/encoder/arm/av1_error_neon.c
index fe44f20f57..38c5e2a10c 100644
--- a/av1/encoder/arm/av1_error_neon.c
+++ b/av1/encoder/arm/av1_error_neon.c
@@ -11,6 +11,7 @@

 #include <arm_neon.h>
 #include <assert.h>
+#include <stdint.h>

 #include "config/aom_config.h"
 #include "config/av1_rtcd.h"
@@ -18,6 +19,7 @@
 #include "aom_dsp/aom_dsp_common.h"
 #include "aom_dsp/arm/mem_neon.h"
 #include "aom_dsp/arm/sum_neon.h"
+#include "aom_ports/mem.h"

 int64_t av1_block_error_neon(const tran_low_t *coeff, const tran_low_t *dqcoeff,
                              intptr_t block_size, int64_t *ssz) {
@@ -94,3 +96,88 @@ int64_t av1_block_error_lp_neon(const int16_t *coeff, const int16_t *dqcoeff,

   return (int64_t)horizontal_add_u64x2(err_u64);
 }
+
+static AOM_FORCE_INLINE int any_nz_s32(int32x4_t v) {
+#if AOM_ARCH_AARCH64
+  return vmaxvq_u32(vreinterpretq_u32_s32(v)) != 0;
+#else
+  const uint64x2_t u64 = vreinterpretq_u64_s32(v);
+  return (vgetq_lane_u64(u64, 0) | vgetq_lane_u64(u64, 1)) != 0;
+#endif
+}
+
+static AOM_FORCE_INLINE uint64x2_t sum_sq_qm_4_u64(int32x4_t v, int32x4_t wt) {
+  const int32x4_t prod = vmulq_s32(v, wt);
+  const uint64x2_t sq_lo =
+      vreinterpretq_u64_s64(vmull_s32(vget_low_s32(prod), vget_low_s32(prod)));
+  const uint64x2_t sq_hi = vreinterpretq_u64_s64(
+      vmull_s32(vget_high_s32(prod), vget_high_s32(prod)));
+  return vrsraq_n_u64(vrshrq_n_u64(sq_lo, 2 * AOM_QM_BITS), sq_hi,
+                      2 * AOM_QM_BITS);
+}
+
+int64_t av1_block_error_qm_neon(const tran_low_t *coeff,
+                                const tran_low_t *dqcoeff, intptr_t block_size,
+                                const qm_val_t *qmatrix, const int16_t *scan,
+                                int64_t *ssz, int bd) {
+  assert(coeff != NULL);
+  assert(dqcoeff != NULL);
+  assert(qmatrix != NULL);
+  assert(scan != NULL);
+  assert(ssz != NULL);
+  assert(block_size >= 16);
+  assert((block_size % 16) == 0);
+  assert(bd == 8 || bd == 10 || bd == 12);
+
+  uint64x2_t sse0 = vdupq_n_u64(0);
+  uint64x2_t sse1 = vdupq_n_u64(0);
+  uint64x2_t ssz0 = vdupq_n_u64(0);
+  uint64x2_t ssz1 = vdupq_n_u64(0);
+
+  do {
+    const int32x4_t c0 = vld1q_s32(coeff);
+    const int32x4_t c1 = vld1q_s32(coeff + 4);
+    const int32x4_t d0 = vld1q_s32(dqcoeff);
+    const int32x4_t d1 = vld1q_s32(dqcoeff + 4);
+    const int32x4_t c_or = vorrq_s32(c0, c1);
+    const int32x4_t d_or = vorrq_s32(d0, d1);
+    if (any_nz_s32(vorrq_s32(c_or, d_or))) {
+      uint8x8_t wt_u8 = vld1_dup_u8(&qmatrix[scan[0]]);
+      wt_u8 = vld1_lane_u8(&qmatrix[scan[1]], wt_u8, 1);
+      wt_u8 = vld1_lane_u8(&qmatrix[scan[2]], wt_u8, 2);
+      wt_u8 = vld1_lane_u8(&qmatrix[scan[3]], wt_u8, 3);
+      wt_u8 = vld1_lane_u8(&qmatrix[scan[4]], wt_u8, 4);
+      wt_u8 = vld1_lane_u8(&qmatrix[scan[5]], wt_u8, 5);
+      wt_u8 = vld1_lane_u8(&qmatrix[scan[6]], wt_u8, 6);
+      wt_u8 = vld1_lane_u8(&qmatrix[scan[7]], wt_u8, 7);
+      const uint16x8_t wt_u16 = vmovl_u8(wt_u8);
+      const int32x4_t wt0 =
+          vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(wt_u16)));
+      const int32x4_t wt1 =
+          vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(wt_u16)));
+      const uint64x2_t sq0 = sum_sq_qm_4_u64(c0, wt0);
+      const uint64x2_t sq1 = sum_sq_qm_4_u64(c1, wt1);
+      ssz0 = vaddq_u64(ssz0, sq0);
+      ssz1 = vaddq_u64(ssz1, sq1);
+
+      if (!any_nz_s32(d_or)) {
+        sse0 = vaddq_u64(sse0, sq0);
+        sse1 = vaddq_u64(sse1, sq1);
+      } else {
+        sse0 = vaddq_u64(sse0, sum_sq_qm_4_u64(vsubq_s32(c0, d0), wt0));
+        sse1 = vaddq_u64(sse1, sum_sq_qm_4_u64(vsubq_s32(c1, d1), wt1));
+      }
+    }
+    coeff += 8;
+    dqcoeff += 8;
+    scan += 8;
+    block_size -= 8;
+  } while (block_size != 0);
+
+  const int shift = 2 * (bd - 8);
+  const int64_t sse = ROUND_POWER_OF_TWO_64(
+      (int64_t)horizontal_add_u64x2(vaddq_u64(sse0, sse1)), shift);
+  *ssz = ROUND_POWER_OF_TWO_64(
+      (int64_t)horizontal_add_u64x2(vaddq_u64(ssz0, ssz1)), shift);
+  return sse;
+}
diff --git a/test/error_block_test.cc b/test/error_block_test.cc
index 49cb2af327..c9b5906dd1 100644
--- a/test/error_block_test.cc
+++ b/test/error_block_test.cc
@@ -344,7 +344,13 @@ const ErrorBlockParam kErrorBlockTestParamsNeon[] = {
   make_tuple(&BlockError8BitWrapper<av1_block_error_neon>,
              &BlockError8BitWrapper<av1_block_error_c>, AOM_BITS_8),
   make_tuple(&BlockErrorLpWrapper<av1_block_error_lp_neon>,
-             &BlockErrorLpWrapper<av1_block_error_lp_c>, AOM_BITS_8)
+             &BlockErrorLpWrapper<av1_block_error_lp_c>, AOM_BITS_8),
+  make_tuple(&BlockErrorQmWrapper<av1_block_error_qm_neon>,
+             &BlockErrorQmWrapper<av1_block_error_qm_c>, AOM_BITS_8),
+  make_tuple(&BlockErrorQmWrapper<av1_block_error_qm_neon>,
+             &BlockErrorQmWrapper<av1_block_error_qm_c>, AOM_BITS_10),
+  make_tuple(&BlockErrorQmWrapper<av1_block_error_qm_neon>,
+             &BlockErrorQmWrapper<av1_block_error_qm_c>, AOM_BITS_12)
 };

 INSTANTIATE_TEST_SUITE_P(NEON, ErrorBlockTest,