Commit df063ff8bb for aom
commit df063ff8bbd62460dfd87b7b52de3a58d9108dfe
Author: Frank Galligan <fgalligan@google.com>
Date: Sat Nov 14 11:52:29 2026 -0800
av1,neon: Add av1_block_error_qm_neon
Add NEON implementation of av1_block_error_qm_neon (8, 10, and 12-bit)
for quantization-matrix weighted block error calculation, using dual
64-bit accumulators and skipping zeroed 8-coefficient slices.
Speedup: 2.00x-2.11x (Neoverse N2), 1.78x-1.95x (Cortex-X4), and
1.41x-1.64x (Cortex-A520) vs C in ErrorBlockTest; -0.77% (1T) / -1.75%
(4T) on Neoverse N3 and -0.41% (1T X4) / -2.65% (1T A720) / -1.88% (4T)
on Pixel 9 avifenc (speed 6, bit-exact).
TAG=agy
CONV=490488d4-7005-402a-8cc4-c245227f80fe
Change-Id: Ibf6e03909b166666666666666666666666666666
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl
index e0e23cbbe7..5b1aa3e1ca 100644
--- a/av1/common/av1_rtcd_defs.pl
+++ b/av1/common/av1_rtcd_defs.pl
@@ -302,7 +302,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
specialize qw/av1_block_error avx2 neon sve/, "$sse2_asm";
add_proto qw/int64_t av1_block_error_qm/, "const tran_low_t *coeff, const tran_low_t *dqcoeff, intptr_t block_size, const qm_val_t *qmatrix, const int16_t *scan, int64_t *ssz, int bd";
- specialize qw/av1_block_error_qm avx2/;
+ specialize qw/av1_block_error_qm avx2 neon/;
add_proto qw/int64_t av1_block_error_lp/, "const int16_t *coeff, const int16_t *dqcoeff, intptr_t block_size";
specialize qw/av1_block_error_lp sse2 avx2 neon sve/;
diff --git a/av1/encoder/arm/av1_error_neon.c b/av1/encoder/arm/av1_error_neon.c
index fe44f20f57..38c5e2a10c 100644
--- a/av1/encoder/arm/av1_error_neon.c
+++ b/av1/encoder/arm/av1_error_neon.c
@@ -11,6 +11,7 @@
#include <arm_neon.h>
#include <assert.h>
+#include <stdint.h>
#include "config/aom_config.h"
#include "config/av1_rtcd.h"
@@ -18,6 +19,7 @@
#include "aom_dsp/aom_dsp_common.h"
#include "aom_dsp/arm/mem_neon.h"
#include "aom_dsp/arm/sum_neon.h"
+#include "aom_ports/mem.h"
int64_t av1_block_error_neon(const tran_low_t *coeff, const tran_low_t *dqcoeff,
intptr_t block_size, int64_t *ssz) {
@@ -94,3 +96,88 @@ int64_t av1_block_error_lp_neon(const int16_t *coeff, const int16_t *dqcoeff,
return (int64_t)horizontal_add_u64x2(err_u64);
}
+
+static AOM_FORCE_INLINE int any_nz_s32(int32x4_t v) {
+#if AOM_ARCH_AARCH64
+ return vmaxvq_u32(vreinterpretq_u32_s32(v)) != 0;
+#else
+ const uint64x2_t u64 = vreinterpretq_u64_s32(v);
+ return (vgetq_lane_u64(u64, 0) | vgetq_lane_u64(u64, 1)) != 0;
+#endif
+}
+
+static AOM_FORCE_INLINE uint64x2_t sum_sq_qm_4_u64(int32x4_t v, int32x4_t wt) {
+ const int32x4_t prod = vmulq_s32(v, wt);
+ const uint64x2_t sq_lo =
+ vreinterpretq_u64_s64(vmull_s32(vget_low_s32(prod), vget_low_s32(prod)));
+ const uint64x2_t sq_hi = vreinterpretq_u64_s64(
+ vmull_s32(vget_high_s32(prod), vget_high_s32(prod)));
+ return vrsraq_n_u64(vrshrq_n_u64(sq_lo, 2 * AOM_QM_BITS), sq_hi,
+ 2 * AOM_QM_BITS);
+}
+
+int64_t av1_block_error_qm_neon(const tran_low_t *coeff,
+ const tran_low_t *dqcoeff, intptr_t block_size,
+ const qm_val_t *qmatrix, const int16_t *scan,
+ int64_t *ssz, int bd) {
+ assert(coeff != NULL);
+ assert(dqcoeff != NULL);
+ assert(qmatrix != NULL);
+ assert(scan != NULL);
+ assert(ssz != NULL);
+ assert(block_size >= 16);
+ assert((block_size % 16) == 0);
+ assert(bd == 8 || bd == 10 || bd == 12);
+
+ uint64x2_t sse0 = vdupq_n_u64(0);
+ uint64x2_t sse1 = vdupq_n_u64(0);
+ uint64x2_t ssz0 = vdupq_n_u64(0);
+ uint64x2_t ssz1 = vdupq_n_u64(0);
+
+ do {
+ const int32x4_t c0 = vld1q_s32(coeff);
+ const int32x4_t c1 = vld1q_s32(coeff + 4);
+ const int32x4_t d0 = vld1q_s32(dqcoeff);
+ const int32x4_t d1 = vld1q_s32(dqcoeff + 4);
+ const int32x4_t c_or = vorrq_s32(c0, c1);
+ const int32x4_t d_or = vorrq_s32(d0, d1);
+ if (any_nz_s32(vorrq_s32(c_or, d_or))) {
+ uint8x8_t wt_u8 = vld1_dup_u8(&qmatrix[scan[0]]);
+ wt_u8 = vld1_lane_u8(&qmatrix[scan[1]], wt_u8, 1);
+ wt_u8 = vld1_lane_u8(&qmatrix[scan[2]], wt_u8, 2);
+ wt_u8 = vld1_lane_u8(&qmatrix[scan[3]], wt_u8, 3);
+ wt_u8 = vld1_lane_u8(&qmatrix[scan[4]], wt_u8, 4);
+ wt_u8 = vld1_lane_u8(&qmatrix[scan[5]], wt_u8, 5);
+ wt_u8 = vld1_lane_u8(&qmatrix[scan[6]], wt_u8, 6);
+ wt_u8 = vld1_lane_u8(&qmatrix[scan[7]], wt_u8, 7);
+ const uint16x8_t wt_u16 = vmovl_u8(wt_u8);
+ const int32x4_t wt0 =
+ vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(wt_u16)));
+ const int32x4_t wt1 =
+ vreinterpretq_s32_u32(vmovl_u16(vget_high_u16(wt_u16)));
+ const uint64x2_t sq0 = sum_sq_qm_4_u64(c0, wt0);
+ const uint64x2_t sq1 = sum_sq_qm_4_u64(c1, wt1);
+ ssz0 = vaddq_u64(ssz0, sq0);
+ ssz1 = vaddq_u64(ssz1, sq1);
+
+ if (!any_nz_s32(d_or)) {
+ sse0 = vaddq_u64(sse0, sq0);
+ sse1 = vaddq_u64(sse1, sq1);
+ } else {
+ sse0 = vaddq_u64(sse0, sum_sq_qm_4_u64(vsubq_s32(c0, d0), wt0));
+ sse1 = vaddq_u64(sse1, sum_sq_qm_4_u64(vsubq_s32(c1, d1), wt1));
+ }
+ }
+ coeff += 8;
+ dqcoeff += 8;
+ scan += 8;
+ block_size -= 8;
+ } while (block_size != 0);
+
+ const int shift = 2 * (bd - 8);
+ const int64_t sse = ROUND_POWER_OF_TWO_64(
+ (int64_t)horizontal_add_u64x2(vaddq_u64(sse0, sse1)), shift);
+ *ssz = ROUND_POWER_OF_TWO_64(
+ (int64_t)horizontal_add_u64x2(vaddq_u64(ssz0, ssz1)), shift);
+ return sse;
+}
diff --git a/test/error_block_test.cc b/test/error_block_test.cc
index 49cb2af327..c9b5906dd1 100644
--- a/test/error_block_test.cc
+++ b/test/error_block_test.cc
@@ -344,7 +344,13 @@ const ErrorBlockParam kErrorBlockTestParamsNeon[] = {
make_tuple(&BlockError8BitWrapper<av1_block_error_neon>,
&BlockError8BitWrapper<av1_block_error_c>, AOM_BITS_8),
make_tuple(&BlockErrorLpWrapper<av1_block_error_lp_neon>,
- &BlockErrorLpWrapper<av1_block_error_lp_c>, AOM_BITS_8)
+ &BlockErrorLpWrapper<av1_block_error_lp_c>, AOM_BITS_8),
+ make_tuple(&BlockErrorQmWrapper<av1_block_error_qm_neon>,
+ &BlockErrorQmWrapper<av1_block_error_qm_c>, AOM_BITS_8),
+ make_tuple(&BlockErrorQmWrapper<av1_block_error_qm_neon>,
+ &BlockErrorQmWrapper<av1_block_error_qm_c>, AOM_BITS_10),
+ make_tuple(&BlockErrorQmWrapper<av1_block_error_qm_neon>,
+ &BlockErrorQmWrapper<av1_block_error_qm_c>, AOM_BITS_12)
};
INSTANTIATE_TEST_SUITE_P(NEON, ErrorBlockTest,