Commit f4976dd2b8 for aom

commit f4976dd2b8e2cd1a8d3fe24a39c2b5bf409e621b
Author: Frank Galligan <fgalligan@google.com>
Date:   Sat Nov 14 11:16:06 2026 -0800

    av1,neon: Add eob fast paths in highbd_inv_txfm

    Add eob == 1 DC-only fast paths and eob-bounded 2D inverse transform
    loops in highbd_inv_txfm2d_add_*_neon, and fuse rounding into vmlaq/
    vmlsq butterfly stages.

    Speedup: 2.85x-97.0x (av1_highbd_inv_txfm_add eob=1, 4x4..32x32) in
    test_libaom on Neoverse N3; -0.80% (4T Neoverse N2) and -0.54% (1T
    Cortex-X4) 10b avifenc on Pixel 9 (bit-exact).

    TAG=agy
    CONV=490488d4-7005-402a-8cc4-c245227f80fe

    Change-Id: Iefe729446f50280bab57554e88a4c9bcc08dcd90

diff --git a/av1/common/arm/highbd_inv_txfm_neon.c b/av1/common/arm/highbd_inv_txfm_neon.c
index f3914da364..2a55cd337f 100644
--- a/av1/common/arm/highbd_inv_txfm_neon.c
+++ b/av1/common/arm/highbd_inv_txfm_neon.c
@@ -1504,8 +1504,8 @@ static void idct8x8_low1_neon(int32x4_t *in, int32x4_t *out, int bit,
   const int32x4_t v_bit = vdupq_n_s32(-bit);
   const int32x4_t rnding = vdupq_n_s32(1 << (bit - 1));
   // stage 0-1-2-3
-  x = vmulq_n_s32(in[0], cospi[32]);
-  x = vaddq_s32(vshlq_s32(x, v_bit), rnding);
+  x = vmlaq_n_s32(rnding, in[0], cospi[32]);
+  x = vshlq_s32(x, v_bit);

   // stage 4-5
   if (!do_cols) {
@@ -1627,8 +1627,8 @@ static void iadst8x8_low1_neon(int32x4_t *in, int32x4_t *out, int bit,
   u[0] = vmlaq_n_s32(rnding, in[0], cospi[60]);
   u[0] = vshlq_s32(u[0], v_bit);

-  u[1] = vmlaq_n_s32(rnding, in[0], cospi[4]);
-  u[1] = vshlq_s32(vnegq_s32(u[1]), v_bit);
+  u[1] = vmlsq_n_s32(rnding, in[0], cospi[4]);
+  u[1] = vshlq_s32(u[1], v_bit);

   // stage 3-4
   int32x4_t temp1, temp2;
@@ -2256,7 +2256,7 @@ static void iadst16x16_low8_neon(int32x4_t *in, int32x4_t *out, int bit,
   u[7] = vshlq_s32(u[7], v_bit);

   y = vmlaq_n_s32(rnding, u[12], cospi[48]);
-  u[12] = vmulq_n_s32(u[12], cospi[16]);
+  u[12] = vmlaq_n_s32(rnding, u[12], cospi[16]);
   u[12] = vmlaq_n_s32(u[12], u[13], cospi[48]);
   u[12] = vshlq_s32(u[12], v_bit);

@@ -3463,8 +3463,8 @@ static void idct64x64_low16_neon(int32x4_t *in, int32x4_t *out, int bit,
                                   &v_bit, &rnding);
     tmp2 = half_btf_neon_mode11_r(&cospi[44], &u[42], &cospi[20], &u[53],
                                   &v_bit, &rnding);
-    tmp3 = half_btf_neon_r(&cospi[52], &u[45], &cospi[12], &u[50], &v_bit,
-                           &rnding);
+    tmp3 = half_btf_neon_mode10_r(&cospi[52], &u[45], &cospi[12], &u[50],
+                                  &v_bit, &rnding);
     tmp4 = half_btf_neon_mode11_r(&cospi[12], &u[46], &cospi[52], &u[49],
                                   &v_bit, &rnding);
     u[49] = half_btf_neon_mode10_r(&cospi[52], &u[46], &cospi[12], &u[49],
@@ -3534,7 +3534,7 @@ static void idct64x64_low16_neon(int32x4_t *in, int32x4_t *out, int bit,
     u[14] =
         half_btf_neon_r(&cospi[48], &u[9], &cospi[16], &u[14], &v_bit, &rnding);
     u[9] = tmp1;
-    tmp2 = half_btf_neon_mode01_r(&cospi[48], &u[10], &cospi[16], &u[13],
+    tmp2 = half_btf_neon_mode11_r(&cospi[48], &u[10], &cospi[16], &u[13],
                                   &v_bit, &rnding);
     u[13] = half_btf_neon_mode10_r(&cospi[16], &u[10], &cospi[48], &u[13],
                                    &v_bit, &rnding);
@@ -5398,10 +5398,10 @@ static void highbd_inv_txfm2d_add_no_identity_neon(const int32_t *input,
   const int txh_idx = get_txh_idx(tx_size);
   const int txfm_size_col = tx_size_wide[tx_size];
   const int txfm_size_row = tx_size_high[tx_size];
-  const int buf_size_w_div8 = txfm_size_col >> 2;
-  const int buf_size_nonzero_w_div8 = (eobx + 8) >> 3;
+  const int buf_size_w_div4 = txfm_size_col >> 2;
+  const int buf_size_nonzero_w = (eobx + 8) >> 3 << 3;
   const int buf_size_nonzero_h_div8 = (eoby + 8) >> 3;
-  const int input_stride = AOMMIN(32, txfm_size_col);
+  const int input_stride = AOMMIN(32, txfm_size_row);
   const int rect_type = get_rect_tx_log_ratio(txfm_size_col, txfm_size_row);

   const int fun_idx_x = lowbd_txfm_all_1d_zeros_idx[eobx];
@@ -5415,35 +5415,29 @@ static void highbd_inv_txfm2d_add_no_identity_neon(const int32_t *input,
   assert(row_txfm != NULL);
   int ud_flip, lr_flip;
   get_flip_cfg(tx_type, &ud_flip, &lr_flip);
-  // 1st stage: column transform
+  // 1st stage: row transform
   for (int i = 0; i < buf_size_nonzero_h_div8 << 1; i++) {
     int32x4_t buf0[64];
-    const int32_t *input_row = input + i * input_stride * 4;
-    for (int j = 0; j < buf_size_nonzero_w_div8 << 1; ++j) {
-      int32x4_t *buf0_cur = &buf0[j * 4];
-      load_buffer_32bit_input(input_row + j * 4, input_stride, buf0_cur, 4);
-
-      TRANSPOSE_4X4(buf0_cur[0], buf0_cur[1], buf0_cur[2], buf0_cur[3],
-                    buf0_cur[0], buf0_cur[1], buf0_cur[2], buf0_cur[3]);
-    }
+    load_buffer_32bit_input(input + i * 4, input_stride, buf0,
+                            buf_size_nonzero_w);
     if (rect_type == 1 || rect_type == -1) {
-      round_shift_rect_array_32_neon(buf0, buf0, buf_size_nonzero_w_div8 << 3);
+      round_shift_rect_array_32_neon(buf0, buf0, buf_size_nonzero_w);
     }
     row_txfm(buf0, buf0, INV_COS_BIT, 0, bd, -shift[0]);

     int32x4_t *_buf1 = &buf1[i * 4];

     if (lr_flip) {
-      for (int j = 0; j < buf_size_w_div8; ++j) {
+      for (int j = 0; j < buf_size_w_div4; ++j) {
         TRANSPOSE_4X4(buf0[4 * j + 3], buf0[4 * j + 2], buf0[4 * j + 1],
                       buf0[4 * j],
-                      _buf1[txfm_size_row * (buf_size_w_div8 - 1 - j) + 0],
-                      _buf1[txfm_size_row * (buf_size_w_div8 - 1 - j) + 1],
-                      _buf1[txfm_size_row * (buf_size_w_div8 - 1 - j) + 2],
-                      _buf1[txfm_size_row * (buf_size_w_div8 - 1 - j) + 3]);
+                      _buf1[txfm_size_row * (buf_size_w_div4 - 1 - j) + 0],
+                      _buf1[txfm_size_row * (buf_size_w_div4 - 1 - j) + 1],
+                      _buf1[txfm_size_row * (buf_size_w_div4 - 1 - j) + 2],
+                      _buf1[txfm_size_row * (buf_size_w_div4 - 1 - j) + 3]);
       }
     } else {
-      for (int j = 0; j < buf_size_w_div8; ++j) {
+      for (int j = 0; j < buf_size_w_div4; ++j) {
         TRANSPOSE_4X4(
             buf0[j * 4 + 0], buf0[j * 4 + 1], buf0[j * 4 + 2], buf0[j * 4 + 3],
             _buf1[j * txfm_size_row + 0], _buf1[j * txfm_size_row + 1],
@@ -5452,7 +5446,7 @@ static void highbd_inv_txfm2d_add_no_identity_neon(const int32_t *input,
     }
   }
   // 2nd stage: column transform
-  for (int i = 0; i < buf_size_w_div8; i++) {
+  for (int i = 0; i < buf_size_w_div4; i++) {
     col_txfm(buf1 + i * txfm_size_row, buf1 + i * txfm_size_row, INV_COS_BIT, 1,
              bd, 0);

@@ -5647,13 +5641,6 @@ void av1_inv_txfm2d_add_16x64_neon(const tran_low_t *input, uint16_t *dest,
                                TX_16X64, bd);
 }

-static void av1_inv_txfm2d_add_16x16_neon(const tran_low_t *input,
-                                          uint16_t *dest, int stride,
-                                          TX_TYPE tx_type, const int bd) {
-  inv_txfm2d_add_universe_neon(input, (uint8_t *)dest, stride, tx_type,
-                               TX_16X16, bd);
-}
-
 void av1_inv_txfm2d_add_32x8_neon(const tran_low_t *input, uint16_t *dest,
                                   int stride, TX_TYPE tx_type, const int bd) {
   inv_txfm2d_add_universe_neon(input, (uint8_t *)dest, stride, tx_type, TX_32X8,
@@ -5666,12 +5653,134 @@ void av1_inv_txfm2d_add_8x32_neon(const tran_low_t *input, uint16_t *dest,
                                bd);
 }

+static inline void highbd_add_dc_row_w8_neon(uint16_t *output,
+                                             const int16x8_t v_dc,
+                                             const int16x8_t v_zero,
+                                             const int16x8_t v_max) {
+  int16x8_t p = vreinterpretq_s16_u16(vld1q_u16(output));
+  p = vaddq_s16(p, v_dc);
+  p = vmaxq_s16(p, v_zero);
+  p = vminq_s16(p, v_max);
+  vst1q_u16(output, vreinterpretq_u16_s16(p));
+}
+
+static inline void highbd_inv_txfm_dc_only_neon(const tran_low_t *input,
+                                                uint16_t *output, int stride,
+                                                TX_SIZE tx_size, int bd) {
+  const int w = tx_size_wide[tx_size];
+  int h = tx_size_high[tx_size];
+  assert(h >= 4 && (h & 3) == 0);
+  const int rect_type = get_rect_tx_log_ratio(w, h);
+  const int8_t *shift = av1_inv_txfm_shift_ls[tx_size];
+  const int32_t cospi32 = cospi_arr(INV_COS_BIT)[32];
+  int32_t dc = input[0];
+
+  if (rect_type == 1 || rect_type == -1) {
+    dc = (int32_t)(((int64_t)dc * NewInvSqrt2 + (1 << (NewSqrt2Bits - 1))) >>
+                   NewSqrt2Bits);
+  }
+  dc = clamp_value(dc, bd + 8);
+  dc = (int32_t)(((int64_t)dc * cospi32 + (1 << (INV_COS_BIT - 1))) >>
+                 INV_COS_BIT);
+  dc = clamp_value(dc, get_log_range(bd, 0));
+  if (shift[0] < 0) {
+    const int rshift0 = -shift[0];
+    dc = (int32_t)((dc + (1 << (rshift0 - 1))) >> rshift0);
+  }
+  dc = clamp_value(dc, get_log_range_out(bd));
+  dc = (int32_t)(((int64_t)dc * cospi32 + (1 << (INV_COS_BIT - 1))) >>
+                 INV_COS_BIT);
+  dc = clamp_value(dc, get_log_range(bd, 1));
+  if (shift[1] < 0) {
+    const int rshift1 = -shift[1];
+    dc = (int32_t)((dc + (1 << (rshift1 - 1))) >> rshift1);
+  }
+  if (dc == 0) return;
+
+  if (w == 4) {
+    const int16x4_t d_dc = vdup_n_s16((int16_t)dc);
+    const int16x4_t d_zero = vdup_n_s16(0);
+    const int16x4_t d_max = vdup_n_s16((int16_t)((1 << bd) - 1));
+    do {
+      uint16_t *row0 = output;
+      uint16_t *row1 = row0 + stride;
+      uint16_t *row2 = row1 + stride;
+      uint16_t *row3 = row2 + stride;
+      int16x4_t p0 = vadd_s16(vreinterpret_s16_u16(vld1_u16(row0)), d_dc);
+      int16x4_t p1 = vadd_s16(vreinterpret_s16_u16(vld1_u16(row1)), d_dc);
+      int16x4_t p2 = vadd_s16(vreinterpret_s16_u16(vld1_u16(row2)), d_dc);
+      int16x4_t p3 = vadd_s16(vreinterpret_s16_u16(vld1_u16(row3)), d_dc);
+      p0 = vmin_s16(vmax_s16(p0, d_zero), d_max);
+      p1 = vmin_s16(vmax_s16(p1, d_zero), d_max);
+      p2 = vmin_s16(vmax_s16(p2, d_zero), d_max);
+      p3 = vmin_s16(vmax_s16(p3, d_zero), d_max);
+      vst1_u16(row0, vreinterpret_u16_s16(p0));
+      vst1_u16(row1, vreinterpret_u16_s16(p1));
+      vst1_u16(row2, vreinterpret_u16_s16(p2));
+      vst1_u16(row3, vreinterpret_u16_s16(p3));
+      output = row3 + stride;
+      h -= 4;
+    } while (h != 0);
+    return;
+  }
+
+  const int16x8_t v_dc = vdupq_n_s16((int16_t)dc);
+  const int16x8_t v_zero = vdupq_n_s16(0);
+  const int16x8_t v_max = vdupq_n_s16((int16_t)((1 << bd) - 1));
+  if (w == 8) {
+    do {
+      uint16_t *row0 = output;
+      uint16_t *row1 = row0 + stride;
+      uint16_t *row2 = row1 + stride;
+      uint16_t *row3 = row2 + stride;
+      highbd_add_dc_row_w8_neon(row0, v_dc, v_zero, v_max);
+      highbd_add_dc_row_w8_neon(row1, v_dc, v_zero, v_max);
+      highbd_add_dc_row_w8_neon(row2, v_dc, v_zero, v_max);
+      highbd_add_dc_row_w8_neon(row3, v_dc, v_zero, v_max);
+      output = row3 + stride;
+      h -= 4;
+    } while (h != 0);
+  } else if (w == 16) {
+    do {
+      highbd_add_dc_row_w8_neon(output, v_dc, v_zero, v_max);
+      highbd_add_dc_row_w8_neon(output + 8, v_dc, v_zero, v_max);
+      output += stride;
+    } while (--h != 0);
+  } else if (w == 32) {
+    do {
+      highbd_add_dc_row_w8_neon(output + 0, v_dc, v_zero, v_max);
+      highbd_add_dc_row_w8_neon(output + 8, v_dc, v_zero, v_max);
+      highbd_add_dc_row_w8_neon(output + 16, v_dc, v_zero, v_max);
+      highbd_add_dc_row_w8_neon(output + 24, v_dc, v_zero, v_max);
+      output += stride;
+    } while (--h != 0);
+  } else {
+    assert(w >= 64 && (w & 31) == 0);
+    do {
+      int c = 0;
+      do {
+        highbd_add_dc_row_w8_neon(output + c + 0, v_dc, v_zero, v_max);
+        highbd_add_dc_row_w8_neon(output + c + 8, v_dc, v_zero, v_max);
+        highbd_add_dc_row_w8_neon(output + c + 16, v_dc, v_zero, v_max);
+        highbd_add_dc_row_w8_neon(output + c + 24, v_dc, v_zero, v_max);
+        c += 32;
+      } while (c != w);
+      output += stride;
+    } while (--h != 0);
+  }
+}
+
 void av1_highbd_inv_txfm_add_neon(const tran_low_t *input, uint8_t *dest,
                                   int stride, const TxfmParam *txfm_param) {
   const TX_SIZE tx_size = txfm_param->tx_size;

   TX_TYPE tx_type = txfm_param->tx_type;
   int bd = txfm_param->bd;
+  if (!txfm_param->lossless && txfm_param->eob == 1 && tx_type == DCT_DCT) {
+    highbd_inv_txfm_dc_only_neon(input, CONVERT_TO_SHORTPTR(dest), stride,
+                                 tx_size, bd);
+    return;
+  }
   switch (tx_size) {
     case TX_8X8:
       highbd_inv_txfm_add_8x8_neon(input, dest, stride, txfm_param);
@@ -5695,57 +5804,9 @@ void av1_highbd_inv_txfm_add_neon(const tran_low_t *input, uint8_t *dest,
       av1_inv_txfm2d_add_4x16_neon(input, CONVERT_TO_SHORTPTR(dest), stride,
                                    txfm_param->tx_type, txfm_param->bd);
       break;
-    case TX_8X16:
-      av1_inv_txfm2d_add_8x16_neon(input, (uint16_t *)dest, stride, tx_type,
-                                   bd);
-      break;
-    case TX_16X8:
-      av1_inv_txfm2d_add_16x8_neon(input, (uint16_t *)dest, stride, tx_type,
-                                   bd);
-      break;
-    case TX_16X32:
-      av1_inv_txfm2d_add_16x32_neon(input, (uint16_t *)dest, stride, tx_type,
-                                    bd);
-      break;
-    case TX_32X16:
-      av1_inv_txfm2d_add_32x16_neon(input, (uint16_t *)dest, stride, tx_type,
-                                    bd);
-      break;
-    case TX_16X16:
-      av1_inv_txfm2d_add_16x16_neon(input, (uint16_t *)dest, stride, tx_type,
-                                    bd);
-      break;
-    case TX_32X32:
-      av1_inv_txfm2d_add_32x32_neon(input, (uint16_t *)dest, stride, tx_type,
-                                    bd);
-      break;
-    case TX_64X64:
-      av1_inv_txfm2d_add_64x64_neon(input, (uint16_t *)dest, stride, tx_type,
-                                    bd);
-      break;
-    case TX_32X64:
-      av1_inv_txfm2d_add_32x64_neon(input, (uint16_t *)dest, stride, tx_type,
-                                    bd);
-      break;
-    case TX_64X32:
-      av1_inv_txfm2d_add_64x32_neon(input, (uint16_t *)dest, stride, tx_type,
-                                    bd);
-      break;
-    case TX_16X64:
-      av1_inv_txfm2d_add_16x64_neon(input, (uint16_t *)dest, stride, tx_type,
-                                    bd);
-      break;
-    case TX_64X16:
-      av1_inv_txfm2d_add_64x16_neon(input, (uint16_t *)dest, stride, tx_type,
-                                    bd);
-      break;
-    case TX_32X8:
-      av1_inv_txfm2d_add_32x8_neon(input, (uint16_t *)dest, stride, tx_type,
-                                   bd);
-      break;
-    case TX_8X32:
-      av1_inv_txfm2d_add_8x32_neon(input, (uint16_t *)dest, stride, tx_type,
-                                   bd);
+    default:
+      highbd_inv_txfm2d_add_universe_neon(input, dest, stride, tx_type, tx_size,
+                                          txfm_param->eob, bd);
       break;
   }
 }