Commit bbbb65c2ae for aom
commit bbbb65c2ae46dfcbc1a634f3773db84586c7bf1f
Author: Frank Galligan <fgalligan@google.com>
Date: Sat Nov 14 11:54:13 2026 -0800
av1,neon: Opt horver_corr & add energy_dist_finer
Optimize av1_get_horver_correlation_full_neon with dedicated width=4,
width=8, and width>=16 single-pass row/column accumulators, and add a
NEON implementation of av1_get_energy_distribution_finer_neon.
Speedup: 1.92x-2.15x vs NEON in HorverTest and 2.75x-3.02x vs C in
EnergyDistributionTest on Arm Neoverse N3; min -0.13% (1T) on Arm
Neoverse N3 and -0.31% (1T Cortex-X4) / -1.68% (4T) on Pixel 9 Tensor
G4 avifenc (speed 6, bit-exact).
TAG=agy
CONV=490488d4-7005-402a-8cc4-c245227f80fe
Change-Id: Ibf6e03909b188888888888888888888888888888
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl
index 4cd3e13958..d1cb4427a4 100644
--- a/av1/common/av1_rtcd_defs.pl
+++ b/av1/common/av1_rtcd_defs.pl
@@ -505,7 +505,7 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
specialize qw/av1_get_horver_correlation_full sse4_1 avx2 neon/;
add_proto qw/void av1_get_energy_distribution_finer/, "const int16_t *diff, int stride, int bw, int bh, float *hordist, float *verdist";
- specialize qw/av1_get_energy_distribution_finer avx2/;
+ specialize qw/av1_get_energy_distribution_finer avx2 neon/;
add_proto qw/void av1_interp_cubic_rate_dist/, "const double *p1, const double *p2, double x, double rate_dist_f[2]";
specialize qw/av1_interp_cubic_rate_dist sse2/;
diff --git a/av1/encoder/arm/rdopt_neon.c b/av1/encoder/arm/rdopt_neon.c
index 237878b091..3a9d197eeb 100644
--- a/av1/encoder/arm/rdopt_neon.c
+++ b/av1/encoder/arm/rdopt_neon.c
@@ -13,414 +13,264 @@
#include <arm_neon.h>
+#include "aom_dsp/arm/sum_neon.h"
#include "av1/encoder/rdopt.h"
#include "config/aom_config.h"
#include "config/av1_rtcd.h"
-// Process horizontal and vertical correlations in a 4x4 block of pixels.
-// We actually use the 4x4 pixels to calculate correlations corresponding to
-// the top-left 3x3 pixels, so this function must be called with 1x1 overlap,
-// moving the window along/down by 3 pixels at a time.
-static inline void horver_correlation_4x4(const int16_t *diff, int stride,
- int32x4_t *xy_sum_32,
- int32x4_t *xz_sum_32,
- int32x4_t *x_sum_32,
- int32x4_t *x2_sum_32) {
- // Pixels in this 4x4 [ a b c d ]
- // are referred to as: [ e f g h ]
- // [ i j k l ]
- // [ m n o p ]
-
- const int16x4_t pixelsa_2_lo = vld1_s16(diff + (0 * stride));
- const int16x4_t pixelsa_2_sli =
- vreinterpret_s16_s64(vshl_n_s64(vreinterpret_s64_s16(pixelsa_2_lo), 16));
- const int16x4_t pixelsb_2_lo = vld1_s16(diff + (1 * stride));
- const int16x4_t pixelsb_2_sli =
- vreinterpret_s16_s64(vshl_n_s64(vreinterpret_s64_s16(pixelsb_2_lo), 16));
- const int16x4_t pixelsa_1_lo = vld1_s16(diff + (2 * stride));
- const int16x4_t pixelsa_1_sli =
- vreinterpret_s16_s64(vshl_n_s64(vreinterpret_s64_s16(pixelsa_1_lo), 16));
- const int16x4_t pixelsb_1_lo = vld1_s16(diff + (3 * stride));
- const int16x4_t pixelsb_1_sli =
- vreinterpret_s16_s64(vshl_n_s64(vreinterpret_s64_s16(pixelsb_1_lo), 16));
-
- const int16x8_t slli_a = vcombine_s16(pixelsa_1_sli, pixelsa_2_sli);
-
- *xy_sum_32 = vmlal_s16(*xy_sum_32, pixelsa_1_lo, pixelsa_1_sli);
- *xy_sum_32 = vmlal_s16(*xy_sum_32, pixelsa_2_lo, pixelsa_2_sli);
- *xy_sum_32 = vmlal_s16(*xy_sum_32, pixelsb_2_lo, pixelsb_2_sli);
-
- *xz_sum_32 = vmlal_s16(*xz_sum_32, pixelsa_1_sli, pixelsb_1_sli);
- *xz_sum_32 = vmlal_s16(*xz_sum_32, pixelsa_2_sli, pixelsb_2_sli);
- *xz_sum_32 = vmlal_s16(*xz_sum_32, pixelsa_1_sli, pixelsb_2_sli);
-
- // Now calculate the straight sums, x_sum += a+b+c+e+f+g+i+j+k
- // (sum up every element in slli_a and swap_b)
- *x_sum_32 = vpadalq_s16(*x_sum_32, slli_a);
- *x_sum_32 = vaddw_s16(*x_sum_32, pixelsb_2_sli);
-
- // Also sum their squares
- *x2_sum_32 = vmlal_s16(*x2_sum_32, pixelsa_1_sli, pixelsa_1_sli);
- *x2_sum_32 = vmlal_s16(*x2_sum_32, pixelsa_2_sli, pixelsa_2_sli);
- *x2_sum_32 = vmlal_s16(*x2_sum_32, pixelsb_2_sli, pixelsb_2_sli);
-}
-
void av1_get_horver_correlation_full_neon(const int16_t *diff, int stride,
int width, int height, float *hcorr,
float *vcorr) {
- // The following notation is used:
- // x - current pixel
- // y - right neighbour pixel
- // z - below neighbour pixel
- // w - down-right neighbour pixel
+ assert(width >= 4 && height >= 4);
+ assert(width == 4 || (width & 7) == 0);
+
int64_t xy_sum = 0, xz_sum = 0;
int64_t x_sum = 0, x2_sum = 0;
- int32x4_t zero = vdupq_n_s32(0);
- int64x2_t v_x_sum = vreinterpretq_s64_s32(zero);
- int64x2_t v_xy_sum = vreinterpretq_s64_s32(zero);
- int64x2_t v_xz_sum = vreinterpretq_s64_s32(zero);
- int64x2_t v_x2_sum = vreinterpretq_s64_s32(zero);
- // Process horizontal and vertical correlations through the body in 4x4
- // blocks. This excludes the final row and column and possibly one extra
- // column depending how 3 divides into width and height
-
- for (int i = 0; i <= height - 4; i += 3) {
- int32x4_t xy_sum_32 = zero;
- int32x4_t xz_sum_32 = zero;
- int32x4_t x_sum_32 = zero;
- int32x4_t x2_sum_32 = zero;
- for (int j = 0; j <= width - 4; j += 3) {
- horver_correlation_4x4(&diff[i * stride + j], stride, &xy_sum_32,
- &xz_sum_32, &x_sum_32, &x2_sum_32);
- }
- v_xy_sum = vpadalq_s32(v_xy_sum, xy_sum_32);
- v_xz_sum = vpadalq_s32(v_xz_sum, xz_sum_32);
- v_x_sum = vpadalq_s32(v_x_sum, x_sum_32);
- v_x2_sum = vpadalq_s32(v_x2_sum, x2_sum_32);
- }
-#if AOM_ARCH_AARCH64
- xy_sum = vaddvq_s64(v_xy_sum);
- xz_sum = vaddvq_s64(v_xz_sum);
- x2_sum = vaddvq_s64(v_x2_sum);
- x_sum = vaddvq_s64(v_x_sum);
-#else
- xy_sum = vget_lane_s64(
- vadd_s64(vget_low_s64(v_xy_sum), vget_high_s64(v_xy_sum)), 0);
- xz_sum = vget_lane_s64(
- vadd_s64(vget_low_s64(v_xz_sum), vget_high_s64(v_xz_sum)), 0);
- x2_sum = vget_lane_s64(
- vadd_s64(vget_low_s64(v_x2_sum), vget_high_s64(v_x2_sum)), 0);
- x_sum =
- vget_lane_s64(vadd_s64(vget_low_s64(v_x_sum), vget_high_s64(v_x_sum)), 0);
-#endif
- // x_sum now covers every pixel except the final 1-2 rows and 1-2 cols
- int64_t x_finalrow = 0, x_finalcol = 0, x2_finalrow = 0, x2_finalcol = 0;
-
- // Do we have 2 rows remaining or just the one? Note that width and height
- // are powers of 2, so each modulo 3 must be 1 or 2.
- if (height % 3 == 1) { // Just horiz corrs on the final row
- const int16_t x0 = diff[(height - 1) * stride];
- x_sum += x0;
- x_finalrow += x0;
- x2_sum += x0 * x0;
- x2_finalrow += x0 * x0;
- if (width >= 8) {
- int32x4_t v_y_sum = zero;
- int32x4_t v_y2_sum = zero;
- int32x4_t v_xy_sum_a = zero;
- int k = width - 1;
- int j = 0;
- while ((k - 8) > 0) {
- const int16x8_t v_x = vld1q_s16(&diff[(height - 1) * stride + j]);
- const int16x8_t v_y = vld1q_s16(&diff[(height - 1) * stride + j + 1]);
- const int16x4_t v_x_lo = vget_low_s16(v_x);
- const int16x4_t v_x_hi = vget_high_s16(v_x);
- const int16x4_t v_y_lo = vget_low_s16(v_y);
- const int16x4_t v_y_hi = vget_high_s16(v_y);
- v_xy_sum_a = vmlal_s16(v_xy_sum_a, v_x_lo, v_y_lo);
- v_xy_sum_a = vmlal_s16(v_xy_sum_a, v_x_hi, v_y_hi);
- v_y2_sum = vmlal_s16(v_y2_sum, v_y_lo, v_y_lo);
- v_y2_sum = vmlal_s16(v_y2_sum, v_y_hi, v_y_hi);
- v_y_sum = vpadalq_s16(v_y_sum, v_y);
- k -= 8;
- j += 8;
- }
-
- const int16x8_t v_l = vld1q_s16(&diff[(height - 1) * stride] + j);
- const int16x8_t v_x =
- vextq_s16(vextq_s16(vreinterpretq_s16_s32(zero), v_l, 7),
- vreinterpretq_s16_s32(zero), 1);
- const int16x8_t v_y = vextq_s16(v_l, vreinterpretq_s16_s32(zero), 1);
- const int16x4_t v_x_lo = vget_low_s16(v_x);
- const int16x4_t v_x_hi = vget_high_s16(v_x);
- const int16x4_t v_y_lo = vget_low_s16(v_y);
- const int16x4_t v_y_hi = vget_high_s16(v_y);
- v_xy_sum_a = vmlal_s16(v_xy_sum_a, v_x_lo, v_y_lo);
- v_xy_sum_a = vmlal_s16(v_xy_sum_a, v_x_hi, v_y_hi);
- v_y2_sum = vmlal_s16(v_y2_sum, v_y_lo, v_y_lo);
- v_y2_sum = vmlal_s16(v_y2_sum, v_y_hi, v_y_hi);
- const int32x4_t v_y_sum_a = vpadalq_s16(v_y_sum, v_y);
- const int64x2_t v_xy_sum2 = vpaddlq_s32(v_xy_sum_a);
-#if AOM_ARCH_AARCH64
- const int64x2_t v_y2_sum_a = vpaddlq_s32(v_y2_sum);
- xy_sum += vaddvq_s64(v_xy_sum2);
- const int32_t y = vaddvq_s32(v_y_sum_a);
- const int64_t y2 = vaddvq_s64(v_y2_sum_a);
-#else
- xy_sum += vget_lane_s64(
- vadd_s64(vget_low_s64(v_xy_sum2), vget_high_s64(v_xy_sum2)), 0);
- const int64x2_t v_y_a = vpaddlq_s32(v_y_sum_a);
- const int64_t y =
- vget_lane_s64(vadd_s64(vget_low_s64(v_y_a), vget_high_s64(v_y_a)), 0);
- const int64x2_t v_y2_sum_b = vpaddlq_s32(v_y2_sum);
- int64_t y2 = vget_lane_s64(
- vadd_s64(vget_low_s64(v_y2_sum_b), vget_high_s64(v_y2_sum_b)), 0);
-#endif
- x_sum += y;
- x2_sum += y2;
- x_finalrow += y;
- x2_finalrow += y2;
- } else {
- for (int j = 0; j < width - 1; ++j) {
- const int16_t x = diff[(height - 1) * stride + j];
- const int16_t y = diff[(height - 1) * stride + j + 1];
- xy_sum += x * y;
- x_sum += y;
- x2_sum += y * y;
- x_finalrow += y;
- x2_finalrow += y * y;
- }
- }
- } else { // Two rows remaining to do
- const int16_t x0 = diff[(height - 2) * stride];
- const int16_t z0 = diff[(height - 1) * stride];
- x_sum += x0 + z0;
- x2_sum += x0 * x0 + z0 * z0;
- x_finalrow += z0;
- x2_finalrow += z0 * z0;
- if (width >= 8) {
- int32x4_t v_y2_sum = zero;
- int32x4_t v_w2_sum = zero;
- int32x4_t v_xy_sum_a = zero;
- int32x4_t v_xz_sum_a = zero;
- int32x4_t v_x_sum_a = zero;
- int32x4_t v_w_sum = zero;
- int k = width - 1;
- int j = 0;
- while ((k - 8) > 0) {
- const int16x8_t v_x = vld1q_s16(&diff[(height - 2) * stride + j]);
- const int16x8_t v_y = vld1q_s16(&diff[(height - 2) * stride + j + 1]);
- const int16x8_t v_z = vld1q_s16(&diff[(height - 1) * stride + j]);
- const int16x8_t v_w = vld1q_s16(&diff[(height - 1) * stride + j + 1]);
-
- const int16x4_t v_x_lo = vget_low_s16(v_x);
- const int16x4_t v_y_lo = vget_low_s16(v_y);
- const int16x4_t v_z_lo = vget_low_s16(v_z);
- const int16x4_t v_w_lo = vget_low_s16(v_w);
- const int16x4_t v_x_hi = vget_high_s16(v_x);
- const int16x4_t v_y_hi = vget_high_s16(v_y);
- const int16x4_t v_z_hi = vget_high_s16(v_z);
- const int16x4_t v_w_hi = vget_high_s16(v_w);
-
- v_xy_sum_a = vmlal_s16(v_xy_sum_a, v_x_lo, v_y_lo);
- v_xy_sum_a = vmlal_s16(v_xy_sum_a, v_x_hi, v_y_hi);
- v_xy_sum_a = vmlal_s16(v_xy_sum_a, v_z_lo, v_w_lo);
- v_xy_sum_a = vmlal_s16(v_xy_sum_a, v_z_hi, v_w_hi);
-
- v_xz_sum_a = vmlal_s16(v_xz_sum_a, v_x_lo, v_z_lo);
- v_xz_sum_a = vmlal_s16(v_xz_sum_a, v_x_hi, v_z_hi);
-
- v_w2_sum = vmlal_s16(v_w2_sum, v_w_lo, v_w_lo);
- v_w2_sum = vmlal_s16(v_w2_sum, v_w_hi, v_w_hi);
- v_y2_sum = vmlal_s16(v_y2_sum, v_y_lo, v_y_lo);
- v_y2_sum = vmlal_s16(v_y2_sum, v_y_hi, v_y_hi);
-
- v_w_sum = vpadalq_s16(v_w_sum, v_w);
- v_x_sum_a = vpadalq_s16(v_x_sum_a, v_y);
- v_x_sum_a = vpadalq_s16(v_x_sum_a, v_w);
-
- k -= 8;
+ int64_t x_firstrow = 0, x_finalrow = 0;
+ int64_t x2_firstrow = 0, x2_finalrow = 0;
+ int64_t x_firstcol = 0, x_finalcol = 0;
+ int64_t x2_firstcol = 0, x2_finalcol = 0;
+
+ if (width == 4) {
+ const int16x4_t zero4 = vdup_n_s16(0);
+ int16x4_t prev = vld1_s16(diff);
+ const int16x4_t prev_sli = vext_s16(zero4, prev, 3);
+ int32x4_t col_x = vmovl_s16(prev);
+ int32x4_t col_x2 = vmull_s16(prev, prev);
+ int32x4_t xy_v = vmull_s16(prev, prev_sli);
+ int32x4_t xz_v = vdupq_n_s32(0);
+
+ x_firstrow = horizontal_add_s32x4(col_x);
+ x2_firstrow = horizontal_long_add_s32x4(col_x2);
+
+ int64x2_t v_col_x2_lo = vdupq_n_s64(0);
+ int64x2_t v_col_x2_hi = vdupq_n_s64(0);
+ int64x2_t v_xy_sum = vdupq_n_s64(0);
+ int64x2_t v_xz_sum = vdupq_n_s64(0);
+
+ const int16_t *p = diff + stride;
+ int i = 1;
+ do {
+ const int strip_end = (i + 32 < height) ? (i + 32) : height;
+ do {
+ const int16x4_t cur = vld1_s16(p);
+ const int16x4_t cur_sli = vext_s16(zero4, cur, 3);
+ col_x = vaddw_s16(col_x, cur);
+ col_x2 = vmlal_s16(col_x2, cur, cur);
+ xy_v = vmlal_s16(xy_v, cur, cur_sli);
+ xz_v = vmlal_s16(xz_v, prev, cur);
+ prev = cur;
+ p += stride;
+ ++i;
+ } while (i != strip_end);
+ v_col_x2_lo = vaddw_s32(v_col_x2_lo, vget_low_s32(col_x2));
+ v_col_x2_hi = vaddw_s32(v_col_x2_hi, vget_high_s32(col_x2));
+ v_xy_sum = vpadalq_s32(v_xy_sum, xy_v);
+ v_xz_sum = vpadalq_s32(v_xz_sum, xz_v);
+ col_x2 = vdupq_n_s32(0);
+ xy_v = vdupq_n_s32(0);
+ xz_v = vdupq_n_s32(0);
+ } while (i != height);
+
+ const int32x4_t last_x2 = vmull_s16(prev, prev);
+ x_finalrow = horizontal_add_s32x4(vmovl_s16(prev));
+ x2_finalrow = horizontal_long_add_s32x4(last_x2);
+
+ x_sum = horizontal_long_add_s32x4(col_x);
+ x_firstcol = vgetq_lane_s32(col_x, 0);
+ x_finalcol = vgetq_lane_s32(col_x, 3);
+ x2_sum = horizontal_add_s64x2(vaddq_s64(v_col_x2_lo, v_col_x2_hi));
+ x2_firstcol = vgetq_lane_s64(v_col_x2_lo, 0);
+ x2_finalcol = vgetq_lane_s64(v_col_x2_hi, 1);
+ xy_sum = horizontal_add_s64x2(v_xy_sum);
+ xz_sum = horizontal_add_s64x2(v_xz_sum);
+ } else if (width == 8) {
+ const int16x8_t zero8 = vdupq_n_s16(0);
+ int16x8_t prev = vld1q_s16(diff);
+ const int16x8_t prev_sli = vextq_s16(zero8, prev, 7);
+ int32x4_t x_v = vpaddlq_s16(prev);
+ int32x4_t x2_v = vmull_s16(vget_low_s16(prev), vget_low_s16(prev));
+ x2_v = vmlal_s16(x2_v, vget_high_s16(prev), vget_high_s16(prev));
+ int32x4_t xy_v = vmull_s16(vget_low_s16(prev), vget_low_s16(prev_sli));
+ xy_v = vmlal_s16(xy_v, vget_high_s16(prev), vget_high_s16(prev_sli));
+ int32x4_t xz_v = vdupq_n_s32(0);
+
+ x_firstrow = horizontal_add_s32x4(x_v);
+ x2_firstrow = horizontal_long_add_s32x4(x2_v);
+ x_firstcol = diff[0];
+ x_finalcol = diff[7];
+ x2_firstcol = x_firstcol * x_firstcol;
+ x2_finalcol = x_finalcol * x_finalcol;
+
+ int64x2_t v_x_sum = vdupq_n_s64(0);
+ int64x2_t v_x2_sum = vdupq_n_s64(0);
+ int64x2_t v_xy_sum = vdupq_n_s64(0);
+ int64x2_t v_xz_sum = vdupq_n_s64(0);
+
+ const int16_t *p = diff + stride;
+ int i = 1;
+ do {
+ const int strip_end = (i + 16 < height) ? (i + 16) : height;
+ do {
+ const int16x8_t cur = vld1q_s16(p);
+ const int16x8_t cur_sli = vextq_s16(zero8, cur, 7);
+ const int16x4_t cur_lo = vget_low_s16(cur);
+ const int16x4_t cur_hi = vget_high_s16(cur);
+ const int16x4_t sli_lo = vget_low_s16(cur_sli);
+ const int16x4_t sli_hi = vget_high_s16(cur_sli);
+ const int16x4_t prev_lo = vget_low_s16(prev);
+ const int16x4_t prev_hi = vget_high_s16(prev);
+
+ x_v = vpadalq_s16(x_v, cur);
+ x2_v = vmlal_s16(x2_v, cur_lo, cur_lo);
+ x2_v = vmlal_s16(x2_v, cur_hi, cur_hi);
+ xy_v = vmlal_s16(xy_v, cur_lo, sli_lo);
+ xy_v = vmlal_s16(xy_v, cur_hi, sli_hi);
+ xz_v = vmlal_s16(xz_v, prev_lo, cur_lo);
+ xz_v = vmlal_s16(xz_v, prev_hi, cur_hi);
+
+ const int64_t fc = p[0];
+ const int64_t lc = p[7];
+ x_firstcol += fc;
+ x_finalcol += lc;
+ x2_firstcol += fc * fc;
+ x2_finalcol += lc * lc;
+ prev = cur;
+ p += stride;
+ ++i;
+ } while (i != strip_end);
+ v_x_sum = vpadalq_s32(v_x_sum, x_v);
+ v_x2_sum = vpadalq_s32(v_x2_sum, x2_v);
+ v_xy_sum = vpadalq_s32(v_xy_sum, xy_v);
+ v_xz_sum = vpadalq_s32(v_xz_sum, xz_v);
+ x_v = vdupq_n_s32(0);
+ x2_v = vdupq_n_s32(0);
+ xy_v = vdupq_n_s32(0);
+ xz_v = vdupq_n_s32(0);
+ } while (i != height);
+
+ int32x4_t last_x2 = vmull_s16(vget_low_s16(prev), vget_low_s16(prev));
+ last_x2 = vmlal_s16(last_x2, vget_high_s16(prev), vget_high_s16(prev));
+ x_finalrow = horizontal_add_s16x8(prev);
+ x2_finalrow = horizontal_long_add_s32x4(last_x2);
+
+ x_sum = horizontal_add_s64x2(v_x_sum);
+ x2_sum = horizontal_add_s64x2(v_x2_sum);
+ xy_sum = horizontal_add_s64x2(v_xy_sum);
+ xz_sum = horizontal_add_s64x2(v_xz_sum);
+ } else {
+ // Strip heights (32 for w==4, 16 for w==8) and per-row s32 accumulators
+ // (up to 128 / 4 = 32 squares per lane) stay < INT32_MAX for 12-bit diffs.
+ const int16x8_t zero8 = vdupq_n_s16(0);
+ int64x2_t v_x_sum = vdupq_n_s64(0);
+ int64x2_t v_x2_sum = vdupq_n_s64(0);
+ int64x2_t v_xy_sum = vdupq_n_s64(0);
+ int64x2_t v_xz_sum = vdupq_n_s64(0);
+
+ // Row 0 (peel j = 0 to avoid zero-initializing row accumulators)
+ {
+ const int16x8_t cur0 = vld1q_s16(diff);
+ const int16x8_t cur0_sli = vextq_s16(zero8, cur0, 7);
+ const int16x4_t cur0_lo = vget_low_s16(cur0);
+ const int16x4_t cur0_hi = vget_high_s16(cur0);
+ int16x8_t prev_vec = cur0;
+ int32x4_t x_32 = vpaddlq_s16(cur0);
+ int32x4_t x2_32 =
+ vmlal_s16(vmull_s16(cur0_lo, cur0_lo), cur0_hi, cur0_hi);
+ int32x4_t xy_32 = vmlal_s16(vmull_s16(cur0_lo, vget_low_s16(cur0_sli)),
+ cur0_hi, vget_high_s16(cur0_sli));
+ int j = 8;
+ do {
+ const int16x8_t cur = vld1q_s16(diff + j);
+ const int16x8_t cur_sli = vextq_s16(prev_vec, cur, 7);
+ const int16x4_t cur_lo = vget_low_s16(cur);
+ const int16x4_t cur_hi = vget_high_s16(cur);
+ prev_vec = cur;
+ x_32 = vpadalq_s16(x_32, cur);
+ x2_32 = vmlal_s16(x2_32, cur_lo, cur_lo);
+ x2_32 = vmlal_s16(x2_32, cur_hi, cur_hi);
+ xy_32 = vmlal_s16(xy_32, cur_lo, vget_low_s16(cur_sli));
+ xy_32 = vmlal_s16(xy_32, cur_hi, vget_high_s16(cur_sli));
j += 8;
- }
- const int16x8_t v_l = vld1q_s16(&diff[(height - 2) * stride] + j);
- const int16x8_t v_x =
- vextq_s16(vextq_s16(vreinterpretq_s16_s32(zero), v_l, 7),
- vreinterpretq_s16_s32(zero), 1);
- const int16x8_t v_y = vextq_s16(v_l, vreinterpretq_s16_s32(zero), 1);
- const int16x8_t v_l_2 = vld1q_s16(&diff[(height - 1) * stride] + j);
- const int16x8_t v_z =
- vextq_s16(vextq_s16(vreinterpretq_s16_s32(zero), v_l_2, 7),
- vreinterpretq_s16_s32(zero), 1);
- const int16x8_t v_w = vextq_s16(v_l_2, vreinterpretq_s16_s32(zero), 1);
-
- const int16x4_t v_x_lo = vget_low_s16(v_x);
- const int16x4_t v_y_lo = vget_low_s16(v_y);
- const int16x4_t v_z_lo = vget_low_s16(v_z);
- const int16x4_t v_w_lo = vget_low_s16(v_w);
- const int16x4_t v_x_hi = vget_high_s16(v_x);
- const int16x4_t v_y_hi = vget_high_s16(v_y);
- const int16x4_t v_z_hi = vget_high_s16(v_z);
- const int16x4_t v_w_hi = vget_high_s16(v_w);
-
- v_xy_sum_a = vmlal_s16(v_xy_sum_a, v_x_lo, v_y_lo);
- v_xy_sum_a = vmlal_s16(v_xy_sum_a, v_x_hi, v_y_hi);
- v_xy_sum_a = vmlal_s16(v_xy_sum_a, v_z_lo, v_w_lo);
- v_xy_sum_a = vmlal_s16(v_xy_sum_a, v_z_hi, v_w_hi);
-
- v_xz_sum_a = vmlal_s16(v_xz_sum_a, v_x_lo, v_z_lo);
- v_xz_sum_a = vmlal_s16(v_xz_sum_a, v_x_hi, v_z_hi);
-
- v_w2_sum = vmlal_s16(v_w2_sum, v_w_lo, v_w_lo);
- v_w2_sum = vmlal_s16(v_w2_sum, v_w_hi, v_w_hi);
- v_y2_sum = vmlal_s16(v_y2_sum, v_y_lo, v_y_lo);
- v_y2_sum = vmlal_s16(v_y2_sum, v_y_hi, v_y_hi);
-
- v_w_sum = vpadalq_s16(v_w_sum, v_w);
- v_x_sum_a = vpadalq_s16(v_x_sum_a, v_y);
- v_x_sum_a = vpadalq_s16(v_x_sum_a, v_w);
-
-#if AOM_ARCH_AARCH64
- xy_sum += vaddvq_s64(vpaddlq_s32(v_xy_sum_a));
- xz_sum += vaddvq_s64(vpaddlq_s32(v_xz_sum_a));
- x_sum += vaddvq_s32(v_x_sum_a);
- x_finalrow += vaddvq_s32(v_w_sum);
- int64_t y2 = vaddvq_s64(vpaddlq_s32(v_y2_sum));
- int64_t w2 = vaddvq_s64(vpaddlq_s32(v_w2_sum));
-#else
- const int64x2_t v_xy_sum2 = vpaddlq_s32(v_xy_sum_a);
- xy_sum += vget_lane_s64(
- vadd_s64(vget_low_s64(v_xy_sum2), vget_high_s64(v_xy_sum2)), 0);
- const int64x2_t v_xz_sum2 = vpaddlq_s32(v_xz_sum_a);
- xz_sum += vget_lane_s64(
- vadd_s64(vget_low_s64(v_xz_sum2), vget_high_s64(v_xz_sum2)), 0);
- const int64x2_t v_x_sum2 = vpaddlq_s32(v_x_sum_a);
- x_sum += vget_lane_s64(
- vadd_s64(vget_low_s64(v_x_sum2), vget_high_s64(v_x_sum2)), 0);
- const int64x2_t v_w_sum_a = vpaddlq_s32(v_w_sum);
- x_finalrow += vget_lane_s64(
- vadd_s64(vget_low_s64(v_w_sum_a), vget_high_s64(v_w_sum_a)), 0);
- const int64x2_t v_y2_sum_a = vpaddlq_s32(v_y2_sum);
- int64_t y2 = vget_lane_s64(
- vadd_s64(vget_low_s64(v_y2_sum_a), vget_high_s64(v_y2_sum_a)), 0);
- const int64x2_t v_w2_sum_a = vpaddlq_s32(v_w2_sum);
- int64_t w2 = vget_lane_s64(
- vadd_s64(vget_low_s64(v_w2_sum_a), vget_high_s64(v_w2_sum_a)), 0);
-#endif
- x2_sum += y2 + w2;
- x2_finalrow += w2;
- } else {
- for (int j = 0; j < width - 1; ++j) {
- const int16_t x = diff[(height - 2) * stride + j];
- const int16_t y = diff[(height - 2) * stride + j + 1];
- const int16_t z = diff[(height - 1) * stride + j];
- const int16_t w = diff[(height - 1) * stride + j + 1];
-
- // Horizontal and vertical correlations for the penultimate row:
- xy_sum += x * y;
- xz_sum += x * z;
-
- // Now just horizontal correlations for the final row:
- xy_sum += z * w;
-
- x_sum += y + w;
- x2_sum += y * y + w * w;
- x_finalrow += w;
- x2_finalrow += w * w;
- }
- }
- }
-
- // Do we have 2 columns remaining or just the one?
- if (width % 3 == 1) { // Just vert corrs on the final col
- const int16_t x0 = diff[width - 1];
- x_sum += x0;
- x_finalcol += x0;
- x2_sum += x0 * x0;
- x2_finalcol += x0 * x0;
- for (int i = 0; i < height - 1; ++i) {
- const int16_t x = diff[i * stride + width - 1];
- const int16_t z = diff[(i + 1) * stride + width - 1];
- xz_sum += x * z;
- x_finalcol += z;
- x2_finalcol += z * z;
- // So the bottom-right elements don't get counted twice:
- if (i < height - (height % 3 == 1 ? 2 : 3)) {
- x_sum += z;
- x2_sum += z * z;
- }
+ } while (j != width);
+ x_firstrow = horizontal_add_s32x4(x_32);
+ x2_firstrow = horizontal_long_add_s32x4(x2_32);
+ xy_sum += horizontal_long_add_s32x4(xy_32);
+ x_sum += x_firstrow;
+ x2_sum += x2_firstrow;
+ const int64_t fc = diff[0];
+ const int64_t lc = diff[width - 1];
+ x_firstcol = fc;
+ x_finalcol = lc;
+ x2_firstcol = fc * fc;
+ x2_finalcol = lc * lc;
}
- } else { // Two cols remaining
- const int16_t x0 = diff[width - 2];
- const int16_t y0 = diff[width - 1];
- x_sum += x0 + y0;
- x2_sum += x0 * x0 + y0 * y0;
- x_finalcol += y0;
- x2_finalcol += y0 * y0;
- for (int i = 0; i < height - 1; ++i) {
- const int16_t x = diff[i * stride + width - 2];
- const int16_t y = diff[i * stride + width - 1];
- const int16_t z = diff[(i + 1) * stride + width - 2];
- const int16_t w = diff[(i + 1) * stride + width - 1];
-
- // Horizontal and vertical correlations for the penultimate col:
- // Skip these on the last iteration of this loop if we also had two
- // rows remaining, otherwise the final horizontal and vertical correlation
- // get erroneously processed twice
- if (i < height - 2 || height % 3 == 1) {
- xy_sum += x * y;
- xz_sum += x * z;
- }
- x_finalcol += w;
- x2_finalcol += w * w;
- // So the bottom-right elements don't get counted twice:
- if (i < height - (height % 3 == 1 ? 2 : 3)) {
- x_sum += z + w;
- x2_sum += z * z + w * w;
+ // Rows 1 .. height - 1
+ const int16_t *row_prev = diff;
+ const int16_t *row_cur = diff + stride;
+ int i = 1;
+ do {
+ const int16x8_t cur0 = vld1q_s16(row_cur);
+ const int16x8_t above0 = vld1q_s16(row_prev);
+ const int16x8_t cur0_sli = vextq_s16(zero8, cur0, 7);
+ const int16x4_t cur0_lo = vget_low_s16(cur0);
+ const int16x4_t cur0_hi = vget_high_s16(cur0);
+ int16x8_t prev_vec = cur0;
+ int32x4_t x_32 = vpaddlq_s16(cur0);
+ int32x4_t x2_32 =
+ vmlal_s16(vmull_s16(cur0_lo, cur0_lo), cur0_hi, cur0_hi);
+ int32x4_t xy_32 = vmlal_s16(vmull_s16(cur0_lo, vget_low_s16(cur0_sli)),
+ cur0_hi, vget_high_s16(cur0_sli));
+ int32x4_t xz_32 = vmlal_s16(vmull_s16(vget_low_s16(above0), cur0_lo),
+ vget_high_s16(above0), cur0_hi);
+
+ int j = 8;
+ do {
+ const int16x8_t cur = vld1q_s16(row_cur + j);
+ const int16x8_t above = vld1q_s16(row_prev + j);
+ const int16x8_t cur_sli = vextq_s16(prev_vec, cur, 7);
+ const int16x4_t cur_lo = vget_low_s16(cur);
+ const int16x4_t cur_hi = vget_high_s16(cur);
+ prev_vec = cur;
+
+ x_32 = vpadalq_s16(x_32, cur);
+ x2_32 = vmlal_s16(x2_32, cur_lo, cur_lo);
+ x2_32 = vmlal_s16(x2_32, cur_hi, cur_hi);
+ xy_32 = vmlal_s16(xy_32, cur_lo, vget_low_s16(cur_sli));
+ xy_32 = vmlal_s16(xy_32, cur_hi, vget_high_s16(cur_sli));
+ xz_32 = vmlal_s16(xz_32, vget_low_s16(above), cur_lo);
+ xz_32 = vmlal_s16(xz_32, vget_high_s16(above), cur_hi);
+ j += 8;
+ } while (j != width);
+
+ if (i == height - 1) {
+ x_finalrow = horizontal_add_s32x4(x_32);
+ x2_finalrow = horizontal_long_add_s32x4(x2_32);
+ x_sum += x_finalrow;
+ x2_sum += x2_finalrow;
+ } else {
+ v_x_sum = vpadalq_s32(v_x_sum, x_32);
+ v_x2_sum = vpadalq_s32(v_x2_sum, x2_32);
}
-
- // Now just vertical correlations for the final column:
- xz_sum += y * w;
- }
- }
-
- // Calculate the simple sums and squared-sums
- int64_t x_firstrow = 0, x_firstcol = 0;
- int64_t x2_firstrow = 0, x2_firstcol = 0;
-
- if (width >= 8) {
- int32x4_t v_x_firstrow = zero;
- int32x4_t v_x2_firstrow = zero;
- for (int j = 0; j < width; j += 8) {
- const int16x8_t v_diff = vld1q_s16(diff + j);
- const int16x4_t v_diff_lo = vget_low_s16(v_diff);
- const int16x4_t v_diff_hi = vget_high_s16(v_diff);
- v_x_firstrow = vpadalq_s16(v_x_firstrow, v_diff);
- v_x2_firstrow = vmlal_s16(v_x2_firstrow, v_diff_lo, v_diff_lo);
- v_x2_firstrow = vmlal_s16(v_x2_firstrow, v_diff_hi, v_diff_hi);
- }
-#if AOM_ARCH_AARCH64
- x_firstrow += vaddvq_s32(v_x_firstrow);
- x2_firstrow += vaddvq_s32(v_x2_firstrow);
-#else
- const int64x2_t v_x_firstrow_64 = vpaddlq_s32(v_x_firstrow);
- x_firstrow += vget_lane_s64(
- vadd_s64(vget_low_s64(v_x_firstrow_64), vget_high_s64(v_x_firstrow_64)),
- 0);
- const int64x2_t v_x2_firstrow_64 = vpaddlq_s32(v_x2_firstrow);
- x2_firstrow += vget_lane_s64(vadd_s64(vget_low_s64(v_x2_firstrow_64),
- vget_high_s64(v_x2_firstrow_64)),
- 0);
-#endif
- } else {
- for (int j = 0; j < width; ++j) {
- x_firstrow += diff[j];
- x2_firstrow += diff[j] * diff[j];
- }
- }
- for (int i = 0; i < height; ++i) {
- x_firstcol += diff[i * stride];
- x2_firstcol += diff[i * stride] * diff[i * stride];
+ v_xy_sum = vpadalq_s32(v_xy_sum, xy_32);
+ v_xz_sum = vpadalq_s32(v_xz_sum, xz_32);
+
+ const int64_t fc = row_cur[0];
+ const int64_t lc = row_cur[width - 1];
+ x_firstcol += fc;
+ x_finalcol += lc;
+ x2_firstcol += fc * fc;
+ x2_finalcol += lc * lc;
+ row_prev = row_cur;
+ row_cur += stride;
+ ++i;
+ } while (i != height);
+
+ x_sum += horizontal_add_s64x2(v_x_sum);
+ x2_sum += horizontal_add_s64x2(v_x2_sum);
+ xy_sum += horizontal_add_s64x2(v_xy_sum);
+ xz_sum += horizontal_add_s64x2(v_xz_sum);
}
int64_t xhor_sum = x_sum - x_finalcol;
@@ -457,3 +307,255 @@ void av1_get_horver_correlation_full_neon(const int16_t *diff, int stride,
*vcorr = 1.0;
}
}
+
+static inline uint32x4_t energy_sq_s16x4(int16x4_t a) {
+ return vreinterpretq_u32_s32(vmull_s16(a, a));
+}
+
+static inline uint32x4_t energy_mla_sq_s16x4(uint32x4_t acc, int16x4_t a) {
+ return vreinterpretq_u32_s32(vmlal_s16(vreinterpretq_s32_u32(acc), a, a));
+}
+
+static inline uint32x4_t energy_pair8_1row(const int16_t *r) {
+ const int16x8_t d = vld1q_s16(r);
+ const uint32x4_t sq_lo = energy_sq_s16x4(vget_low_s16(d));
+ const uint32x4_t sq_hi = energy_sq_s16x4(vget_high_s16(d));
+ return horizontal_add_2d_u32(sq_lo, sq_hi);
+}
+
+static inline uint32x4_t energy_pair8_2row(const int16_t *r0,
+ const int16_t *r1) {
+ const int16x8_t d0 = vld1q_s16(r0);
+ const int16x8_t d1 = vld1q_s16(r1);
+ uint32x4_t sq_lo = energy_sq_s16x4(vget_low_s16(d0));
+ uint32x4_t sq_hi = energy_sq_s16x4(vget_high_s16(d0));
+ sq_lo = energy_mla_sq_s16x4(sq_lo, vget_low_s16(d1));
+ sq_hi = energy_mla_sq_s16x4(sq_hi, vget_high_s16(d1));
+ return horizontal_add_2d_u32(sq_lo, sq_hi);
+}
+
+static inline uint64_t energy_dist_w4(const int16_t *diff, int stride,
+ int esq_h, int h_shift, uint32_t *col_sum,
+ uint32_t *row_sum) {
+ uint32x4_t v_col;
+ if (h_shift == 0) {
+ const uint32x4_t sq0 = energy_sq_s16x4(vld1_s16(diff));
+ const uint32x4_t sq1 = energy_sq_s16x4(vld1_s16(diff + stride));
+ const uint32x4_t sq2 = energy_sq_s16x4(vld1_s16(diff + 2 * stride));
+ const uint32x4_t sq3 = energy_sq_s16x4(vld1_s16(diff + 3 * stride));
+ v_col = vaddq_u32(vaddq_u32(sq0, sq1), vaddq_u32(sq2, sq3));
+ const uint32x4_t sum4[4] = { sq0, sq1, sq2, sq3 };
+ vst1q_u32(row_sum, horizontal_add_4d_u32x4(sum4));
+ for (esq_h -= 4; esq_h != 0; esq_h -= 4) {
+ diff += 4 * stride;
+ row_sum += 4;
+ const uint32x4_t s0 = energy_sq_s16x4(vld1_s16(diff));
+ const uint32x4_t s1 = energy_sq_s16x4(vld1_s16(diff + stride));
+ const uint32x4_t s2 = energy_sq_s16x4(vld1_s16(diff + 2 * stride));
+ const uint32x4_t s3 = energy_sq_s16x4(vld1_s16(diff + 3 * stride));
+ v_col = vaddq_u32(v_col, vaddq_u32(vaddq_u32(s0, s1), vaddq_u32(s2, s3)));
+ const uint32x4_t s4[4] = { s0, s1, s2, s3 };
+ vst1q_u32(row_sum, horizontal_add_4d_u32x4(s4));
+ }
+ } else {
+ uint32x4_t sum4[4];
+ for (int k = 0; k < 4; ++k) {
+ sum4[k] = energy_mla_sq_s16x4(energy_sq_s16x4(vld1_s16(diff)),
+ vld1_s16(diff + stride));
+ diff += 2 * stride;
+ }
+ v_col = vaddq_u32(vaddq_u32(sum4[0], sum4[1]), vaddq_u32(sum4[2], sum4[3]));
+ vst1q_u32(row_sum, horizontal_add_4d_u32x4(sum4));
+ for (esq_h -= 4; esq_h != 0; esq_h -= 4) {
+ row_sum += 4;
+ for (int k = 0; k < 4; ++k) {
+ const uint32x4_t sq = energy_mla_sq_s16x4(
+ energy_sq_s16x4(vld1_s16(diff)), vld1_s16(diff + stride));
+ v_col = vaddq_u32(v_col, sq);
+ sum4[k] = sq;
+ diff += 2 * stride;
+ }
+ vst1q_u32(row_sum, horizontal_add_4d_u32x4(sum4));
+ }
+ }
+ vst1q_u32(col_sum, v_col);
+ return horizontal_long_add_u32x4(v_col);
+}
+
+static inline uint64_t energy_dist_w8(const int16_t *diff, int stride,
+ int esq_h, int h_shift, uint32_t *col_sum,
+ uint32_t *row_sum) {
+ uint32x4_t v_col0, v_col1;
+ int i = 0;
+ if (h_shift == 0) {
+ do {
+ uint32x4_t sum4[4];
+ const int16x8_t d0 = vld1q_s16(diff + i * stride);
+ const uint32x4_t sq_lo0 = energy_sq_s16x4(vget_low_s16(d0));
+ const uint32x4_t sq_hi0 = energy_sq_s16x4(vget_high_s16(d0));
+ v_col0 = (i == 0) ? sq_lo0 : vaddq_u32(v_col0, sq_lo0);
+ v_col1 = (i == 0) ? sq_hi0 : vaddq_u32(v_col1, sq_hi0);
+ sum4[0] = vaddq_u32(sq_lo0, sq_hi0);
+ for (int k = 1; k < 4; ++k) {
+ const int16x8_t d = vld1q_s16(diff + (i + k) * stride);
+ const uint32x4_t sq_lo = energy_sq_s16x4(vget_low_s16(d));
+ const uint32x4_t sq_hi = energy_sq_s16x4(vget_high_s16(d));
+ v_col0 = vaddq_u32(v_col0, sq_lo);
+ v_col1 = vaddq_u32(v_col1, sq_hi);
+ sum4[k] = vaddq_u32(sq_lo, sq_hi);
+ }
+ vst1q_u32(row_sum + i, horizontal_add_4d_u32x4(sum4));
+ i += 4;
+ } while (i != esq_h);
+ } else {
+ do {
+ uint32x4_t sum4[4];
+ for (int k = 0; k < 4; ++k) {
+ const int16_t *r = diff + 2 * (i + k) * stride;
+ const int16x8_t d0 = vld1q_s16(r);
+ const int16x8_t d1 = vld1q_s16(r + stride);
+ uint32x4_t sq_lo = energy_sq_s16x4(vget_low_s16(d0));
+ uint32x4_t sq_hi = energy_sq_s16x4(vget_high_s16(d0));
+ sq_lo = energy_mla_sq_s16x4(sq_lo, vget_low_s16(d1));
+ sq_hi = energy_mla_sq_s16x4(sq_hi, vget_high_s16(d1));
+ v_col0 = (i == 0 && k == 0) ? sq_lo : vaddq_u32(v_col0, sq_lo);
+ v_col1 = (i == 0 && k == 0) ? sq_hi : vaddq_u32(v_col1, sq_hi);
+ sum4[k] = vaddq_u32(sq_lo, sq_hi);
+ }
+ vst1q_u32(row_sum + i, horizontal_add_4d_u32x4(sum4));
+ i += 4;
+ } while (i != esq_h);
+ }
+ vst1q_u32(col_sum, v_col0);
+ vst1q_u32(col_sum + 4, v_col1);
+ const uint32x4_t v_cols[2] = { v_col0, v_col1 };
+ return horizontal_long_add_u32x4_x2(v_cols);
+}
+
+static inline uint64_t energy_dist_w16(const int16_t *diff, int stride,
+ int esq_h, int h_shift,
+ uint32_t *col_sum, uint32_t *row_sum) {
+ uint32x4_t v_col0, v_col1;
+ int i = 0;
+ if (h_shift == 0) {
+ do {
+ uint32x4_t sum4[4];
+ for (int k = 0; k < 4; ++k) {
+ const int16_t *r = diff + (i + k) * stride;
+ const uint32x4_t p0 = energy_pair8_1row(r);
+ const uint32x4_t p1 = energy_pair8_1row(r + 8);
+ v_col0 = (i == 0 && k == 0) ? p0 : vaddq_u32(v_col0, p0);
+ v_col1 = (i == 0 && k == 0) ? p1 : vaddq_u32(v_col1, p1);
+ sum4[k] = vaddq_u32(p0, p1);
+ }
+ vst1q_u32(row_sum + i, horizontal_add_4d_u32x4(sum4));
+ i += 4;
+ } while (i != esq_h);
+ } else {
+ do {
+ uint32x4_t sum4[4];
+ for (int k = 0; k < 4; ++k) {
+ const int16_t *r0 = diff + 2 * (i + k) * stride;
+ const int16_t *r1 = r0 + stride;
+ const uint32x4_t p0 = energy_pair8_2row(r0, r1);
+ const uint32x4_t p1 = energy_pair8_2row(r0 + 8, r1 + 8);
+ v_col0 = (i == 0 && k == 0) ? p0 : vaddq_u32(v_col0, p0);
+ v_col1 = (i == 0 && k == 0) ? p1 : vaddq_u32(v_col1, p1);
+ sum4[k] = vaddq_u32(p0, p1);
+ }
+ vst1q_u32(row_sum + i, horizontal_add_4d_u32x4(sum4));
+ i += 4;
+ } while (i != esq_h);
+ }
+ vst1q_u32(col_sum, v_col0);
+ vst1q_u32(col_sum + 4, v_col1);
+ const uint32x4_t v_cols[2] = { v_col0, v_col1 };
+ return horizontal_long_add_u32x4_x2(v_cols);
+}
+
+static inline uint64_t energy_dist_w32(const int16_t *diff, int stride,
+ int esq_h, int h_shift,
+ uint32_t *col_sum, uint32_t *row_sum) {
+ uint32x4_t v_col[4];
+ int i = 0;
+ if (h_shift == 0) {
+ do {
+ uint32x4_t sum4[4];
+ for (int k = 0; k < 4; ++k) {
+ const int16_t *r = diff + (i + k) * stride;
+ const uint32x4_t p0 = energy_pair8_1row(r);
+ const uint32x4_t p1 = energy_pair8_1row(r + 8);
+ const uint32x4_t p2 = energy_pair8_1row(r + 16);
+ const uint32x4_t p3 = energy_pair8_1row(r + 24);
+ v_col[0] = (i == 0 && k == 0) ? p0 : vaddq_u32(v_col[0], p0);
+ v_col[1] = (i == 0 && k == 0) ? p1 : vaddq_u32(v_col[1], p1);
+ v_col[2] = (i == 0 && k == 0) ? p2 : vaddq_u32(v_col[2], p2);
+ v_col[3] = (i == 0 && k == 0) ? p3 : vaddq_u32(v_col[3], p3);
+ sum4[k] = vaddq_u32(vaddq_u32(p0, p1), vaddq_u32(p2, p3));
+ }
+ vst1q_u32(row_sum + i, horizontal_add_4d_u32x4(sum4));
+ i += 4;
+ } while (i != esq_h);
+ } else {
+ do {
+ uint32x4_t sum4[4];
+ for (int k = 0; k < 4; ++k) {
+ const int16_t *r0 = diff + 2 * (i + k) * stride;
+ const int16_t *r1 = r0 + stride;
+ const uint32x4_t p0 = energy_pair8_2row(r0, r1);
+ const uint32x4_t p1 = energy_pair8_2row(r0 + 8, r1 + 8);
+ const uint32x4_t p2 = energy_pair8_2row(r0 + 16, r1 + 16);
+ const uint32x4_t p3 = energy_pair8_2row(r0 + 24, r1 + 24);
+ v_col[0] = (i == 0 && k == 0) ? p0 : vaddq_u32(v_col[0], p0);
+ v_col[1] = (i == 0 && k == 0) ? p1 : vaddq_u32(v_col[1], p1);
+ v_col[2] = (i == 0 && k == 0) ? p2 : vaddq_u32(v_col[2], p2);
+ v_col[3] = (i == 0 && k == 0) ? p3 : vaddq_u32(v_col[3], p3);
+ sum4[k] = vaddq_u32(vaddq_u32(p0, p1), vaddq_u32(p2, p3));
+ }
+ vst1q_u32(row_sum + i, horizontal_add_4d_u32x4(sum4));
+ i += 4;
+ } while (i != esq_h);
+ }
+ vst1q_u32(col_sum, v_col[0]);
+ vst1q_u32(col_sum + 4, v_col[1]);
+ vst1q_u32(col_sum + 8, v_col[2]);
+ vst1q_u32(col_sum + 12, v_col[3]);
+ return horizontal_long_add_u32x4_x4(v_col);
+}
+
+void av1_get_energy_distribution_finer_neon(const int16_t *diff, int stride,
+ int bw, int bh, float *hordist,
+ float *verdist) {
+ const int w_shift = bw <= 8 ? 0 : 1;
+ const int h_shift = bh <= 8 ? 0 : 1;
+ const int esq_w = bw >> w_shift;
+ const int esq_h = bh >> h_shift;
+ assert(esq_w >= 4 && esq_w <= 16 && (esq_w & 3) == 0);
+ assert(esq_h >= 4 && esq_h <= 16 && (esq_h & 3) == 0);
+ uint32_t col_sum[16];
+ uint32_t row_sum[16];
+ uint64_t total;
+
+ if (bw == 4) {
+ total = energy_dist_w4(diff, stride, esq_h, h_shift, col_sum, row_sum);
+ } else if (bw == 8) {
+ total = energy_dist_w8(diff, stride, esq_h, h_shift, col_sum, row_sum);
+ } else if (bw == 16) {
+ total = energy_dist_w16(diff, stride, esq_h, h_shift, col_sum, row_sum);
+ } else {
+ assert(bw == 32);
+ total = energy_dist_w32(diff, stride, esq_h, h_shift, col_sum, row_sum);
+ }
+
+ if (total == 0) {
+ const float hor_val = 1.0f / esq_w;
+ for (int j = 0; j < esq_w - 1; ++j) hordist[j] = hor_val;
+ const float ver_val = 1.0f / esq_h;
+ for (int i = 0; i < esq_h - 1; ++i) verdist[i] = ver_val;
+ return;
+ }
+
+ const float e_recip = 1.0f / (float)total;
+ for (int j = 0; j < esq_w - 1; ++j) hordist[j] = (float)col_sum[j] * e_recip;
+ for (int i = 0; i < esq_h - 1; ++i) verdist[i] = (float)row_sum[i] * e_recip;
+}
diff --git a/test/horver_correlation_test.cc b/test/horver_correlation_test.cc
index 476d2e00e4..9b149f6804 100644
--- a/test/horver_correlation_test.cc
+++ b/test/horver_correlation_test.cc
@@ -263,6 +263,12 @@ TEST_P(EnergyDistributionTest, ZeroValues) { RunZeroTest(); }
TEST_P(EnergyDistributionTest, ExtremeValues) { RunExtremeValuesTest(); }
+#if HAVE_NEON
+INSTANTIATE_TEST_SUITE_P(
+ NEON, EnergyDistributionTest,
+ ::testing::Values(av1_get_energy_distribution_finer_neon));
+#endif // HAVE_NEON
+
#if HAVE_AVX2
INSTANTIATE_TEST_SUITE_P(
AVX2, EnergyDistributionTest,