Commit 7fa1a3381f for aom

commit 7fa1a3381f8d879af0cb5b6dd68af096511073ac
Author: Frank Galligan <fgalligan@google.com>
Date:   Sat Nov 14 18:29:58 2026 -0800

    av1,neon: Optimize inv_txfm 4x8/8x4 & eob==1

    Vectorize lowbd_inv_txfm2d_add_{4x8,8x4,4x16,16x4}_neon in 16-bit NEON
    and add eob == 1 DC-only fast paths and DCT_DCT direct dispatch in
    av1_inv_txfm2d_add_{8x8..64x64}_neon to avoid zeroing 16 KB stack
    buffers on DC-only blocks.

    Speedup: 18.91x (8x8 eob=1), 11.18x (16x16 eob=1), and 5.55x-6.28x
    (4x8/8x4 ADST_DCT) in test_libaom on Neoverse N3; -0.61% (1T) / -2.23%
    (4T) on AmpereOne and -0.68% (1T Cortex-X4) / -1.84% (1T Cortex-A720)
    on Pixel 9 avifenc (bit-exact).

    TAG=agy
    CONV=490488d4-7005-402a-8cc4-c245227f80fe

    Change-Id: I06add844f575744c020146a0dae858cd4a5aae72

diff --git a/av1/common/arm/av1_inv_txfm_neon.c b/av1/common/arm/av1_inv_txfm_neon.c
index d7a83680d3..7b188a9aa7 100644
--- a/av1/common/arm/av1_inv_txfm_neon.c
+++ b/av1/common/arm/av1_inv_txfm_neon.c
@@ -12,11 +12,13 @@
 #include "av1/common/arm/av1_inv_txfm_neon.h"

 #include <arm_neon.h>
+#include <assert.h>

 #include "config/aom_config.h"
 #include "config/aom_dsp_rtcd.h"
 #include "config/av1_rtcd.h"

+#include "aom_dsp/arm/mem_neon.h"
 #include "aom_dsp/arm/transpose_neon.h"
 #include "av1/common/av1_inv_txfm1d.h"
 #include "av1/common/av1_inv_txfm1d_cfg.h"
@@ -3815,266 +3817,354 @@ static inline void lowbd_inv_txfm2d_add_4x4_neon(const int32_t *input,
   }
 }

-static void lowbd_inv_txfm2d_add_4x8_neon(const int32_t *input, uint8_t *output,
-                                          int stride, TX_TYPE tx_type,
-                                          int eob) {
-  (void)eob;
-  TX_SIZE tx_size = TX_4X8;
-  DECLARE_ALIGNED(32, int, txfm_buf[4 * 8 + 8 + 8]);
-  int32_t *temp_in = txfm_buf;
+static inline void lowbd_add_flip_buffer_4xn_neon(const int16x8_t *in,
+                                                  uint8_t *output, int stride,
+                                                  int flipud, int height) {
+  assert(height >= 2 && (height & 1) == 0);
+  int j = flipud ? (height - 1) : 0;
+  const int step = flipud ? -1 : 1;
+  do {
+    const uint8x8_t pred = load_u8_4x2(output, stride);
+    const int16x8_t res =
+        vcombine_s16(vget_low_s16(in[j]), vget_low_s16(in[j + step]));
+    const int16x8_t sum =
+        vqaddq_s16(vreinterpretq_s16_u16(vmovl_u8(pred)), res);
+    store_u8x4_strided_x2(output, stride, vqmovun_s16(sum));
+    output += 2 * stride;
+    j += 2 * step;
+    height -= 2;
+  } while (height != 0);
+}

-  const int8_t *shift = av1_inv_txfm_shift_ls[tx_size];
-  const int txw_idx = get_txw_idx(tx_size);
-  const int txh_idx = get_txh_idx(tx_size);
-  const int txfm_size_col = tx_size_wide[tx_size];
-  const int txfm_size_row = tx_size_high[tx_size];
-  const int buf_offset = AOMMAX(txfm_size_row, txfm_size_col);
-  int32_t *temp_out = temp_in + buf_offset;
-  int32_t *buf = temp_out + buf_offset;
-  int32_t *buf_ptr = buf;
-  const int8_t stage_range[MAX_TXFM_STAGE_NUM] = { 16, 16, 16, 16,
-                                                   16, 16, 16, 16 };
-  int r;
-  const transform_1d_neon row_txfm =
-      lowbd_txfm_all_1d_arr[txw_idx][hitx_1d_tab[tx_type]];
-  const transform_1d_neon col_txfm =
-      lowbd_txfm_all_1d_arr[txh_idx][vitx_1d_tab[tx_type]];
+static inline void lowbd_inv_txfm_dc_only_4xn_neon(uint8_t *output, int stride,
+                                                   int32_t dc, int height) {
+  assert(height >= 2 && (height & 1) == 0);
+  const int16_t clamped_dc = (int16_t)clamp(dc, -255, 255);
+  const int16x8_t dc_v = vdupq_n_s16(clamped_dc);
+  do {
+    const uint8x8_t pred = load_u8_4x2(output, stride);
+    const int16x8_t sum =
+        vaddq_s16(vreinterpretq_s16_u16(vmovl_u8(pred)), dc_v);
+    store_u8x4_strided_x2(output, stride, vqmovun_s16(sum));
+    output += 2 * stride;
+    height -= 2;
+  } while (height != 0);
+}

-  int ud_flip, lr_flip;
-  get_flip_cfg(tx_type, &ud_flip, &lr_flip);
+static AOM_FORCE_INLINE int16x8_t narrow_inv_txfm4_out_neon(int32x4_t lo,
+                                                            int32x4_t hi,
+                                                            int out_rshift) {
+  if (out_rshift == 4) {
+    return vcombine_s16(vqrshrn_n_s32(lo, 4), vqrshrn_n_s32(hi, 4));
+  }
+  if (out_rshift == 1) {
+    return vcombine_s16(vqrshrn_n_s32(lo, 1), vqrshrn_n_s32(hi, 1));
+  }
+  assert(out_rshift == 0);
+  return vcombine_s16(vqmovn_s32(lo), vqmovn_s32(hi));
+}

-  for (int i = 0; i < txfm_size_row; i++) {
-    for (int c = 0; c < txfm_size_col; c++)
-      temp_in[c] = round_shift((int64_t)input[c * txfm_size_row] * NewInvSqrt2,
-                               NewSqrt2Bits);
+static AOM_FORCE_INLINE void idct4_s16x8_neon(const int16x8_t *in,
+                                              int16x8_t *out, int out_rshift) {
+  const int32_t *cospi = cospi_arr(INV_COS_BIT);
+  const int32x4_t x0_lo = vmovl_s16(vget_low_s16(in[0]));
+  const int32x4_t x0_hi = vmovl_s16(vget_high_s16(in[0]));
+  const int32x4_t x2_lo = vmovl_s16(vget_low_s16(in[2]));
+  const int32x4_t x2_hi = vmovl_s16(vget_high_s16(in[2]));
+  const int16x4_t x1_lo = vget_low_s16(in[1]);
+  const int16x4_t x1_hi = vget_high_s16(in[1]);
+  const int16x4_t x3_lo = vget_low_s16(in[3]);
+  const int16x4_t x3_hi = vget_high_s16(in[3]);
+
+  const int32x4_t s0_lo = vrshrq_n_s32(
+      vmulq_n_s32(vaddq_s32(x0_lo, x2_lo), cospi[32]), INV_COS_BIT);
+  const int32x4_t s0_hi = vrshrq_n_s32(
+      vmulq_n_s32(vaddq_s32(x0_hi, x2_hi), cospi[32]), INV_COS_BIT);
+  const int32x4_t s1_lo = vrshrq_n_s32(
+      vmulq_n_s32(vsubq_s32(x0_lo, x2_lo), cospi[32]), INV_COS_BIT);
+  const int32x4_t s1_hi = vrshrq_n_s32(
+      vmulq_n_s32(vsubq_s32(x0_hi, x2_hi), cospi[32]), INV_COS_BIT);
+
+  const int32x4_t s2_lo =
+      vrshrq_n_s32(vmlsl_n_s16(vmull_n_s16(x1_lo, (int16_t)cospi[48]), x3_lo,
+                               (int16_t)cospi[16]),
+                   INV_COS_BIT);
+  const int32x4_t s2_hi =
+      vrshrq_n_s32(vmlsl_n_s16(vmull_n_s16(x1_hi, (int16_t)cospi[48]), x3_hi,
+                               (int16_t)cospi[16]),
+                   INV_COS_BIT);
+  const int32x4_t s3_lo =
+      vrshrq_n_s32(vmlal_n_s16(vmull_n_s16(x1_lo, (int16_t)cospi[16]), x3_lo,
+                               (int16_t)cospi[48]),
+                   INV_COS_BIT);
+  const int32x4_t s3_hi =
+      vrshrq_n_s32(vmlal_n_s16(vmull_n_s16(x1_hi, (int16_t)cospi[16]), x3_hi,
+                               (int16_t)cospi[48]),
+                   INV_COS_BIT);
+
+  out[0] = narrow_inv_txfm4_out_neon(vaddq_s32(s0_lo, s3_lo),
+                                     vaddq_s32(s0_hi, s3_hi), out_rshift);
+  out[1] = narrow_inv_txfm4_out_neon(vaddq_s32(s1_lo, s2_lo),
+                                     vaddq_s32(s1_hi, s2_hi), out_rshift);
+  out[2] = narrow_inv_txfm4_out_neon(vsubq_s32(s1_lo, s2_lo),
+                                     vsubq_s32(s1_hi, s2_hi), out_rshift);
+  out[3] = narrow_inv_txfm4_out_neon(vsubq_s32(s0_lo, s3_lo),
+                                     vsubq_s32(s0_hi, s3_hi), out_rshift);
+}

-    row_txfm(temp_in, buf_ptr, INV_COS_BIT, stage_range);
-    input++;
-    buf_ptr += txfm_size_col;
+static AOM_FORCE_INLINE void iadst4_s16x8_neon(const int16x8_t *in,
+                                               int16x8_t *out, int out_rshift) {
+  const int32_t *sinpi = sinpi_arr(INV_COS_BIT);
+  const int16_t sin1 = (int16_t)sinpi[1];
+  const int16_t sin2 = (int16_t)sinpi[2];
+  const int16_t sin3 = (int16_t)sinpi[3];
+  const int16_t sin4 = (int16_t)sinpi[4];
+
+  const int16x4_t x0_lo = vget_low_s16(in[0]);
+  const int16x4_t x0_hi = vget_high_s16(in[0]);
+  const int16x4_t x1_lo = vget_low_s16(in[1]);
+  const int16x4_t x1_hi = vget_high_s16(in[1]);
+  const int16x4_t x2_lo = vget_low_s16(in[2]);
+  const int16x4_t x2_hi = vget_high_s16(in[2]);
+  const int16x4_t x3_lo = vget_low_s16(in[3]);
+  const int16x4_t x3_hi = vget_high_s16(in[3]);
+
+  int32x4_t s0_lo = vmull_n_s16(x0_lo, sin1);
+  int32x4_t s0_hi = vmull_n_s16(x0_hi, sin1);
+  s0_lo = vmlal_n_s16(s0_lo, x2_lo, sin4);
+  s0_hi = vmlal_n_s16(s0_hi, x2_hi, sin4);
+  s0_lo = vmlal_n_s16(s0_lo, x3_lo, sin2);
+  s0_hi = vmlal_n_s16(s0_hi, x3_hi, sin2);
+
+  int32x4_t s1_lo = vmull_n_s16(x0_lo, sin2);
+  int32x4_t s1_hi = vmull_n_s16(x0_hi, sin2);
+  s1_lo = vmlsl_n_s16(s1_lo, x2_lo, sin1);
+  s1_hi = vmlsl_n_s16(s1_hi, x2_hi, sin1);
+  s1_lo = vmlsl_n_s16(s1_lo, x3_lo, sin4);
+  s1_hi = vmlsl_n_s16(s1_hi, x3_hi, sin4);
+
+  const int32x4_t s3_lo = vmull_n_s16(x1_lo, sin3);
+  const int32x4_t s3_hi = vmull_n_s16(x1_hi, sin3);
+  const int32x4_t s7_lo = vaddw_s16(vsubl_s16(x0_lo, x2_lo), x3_lo);
+  const int32x4_t s7_hi = vaddw_s16(vsubl_s16(x0_hi, x2_hi), x3_hi);
+  const int32x4_t s2_lo = vmulq_n_s32(s7_lo, sinpi[3]);
+  const int32x4_t s2_hi = vmulq_n_s32(s7_hi, sinpi[3]);
+
+  const int32x4_t y0_lo = vrshrq_n_s32(vaddq_s32(s0_lo, s3_lo), INV_COS_BIT);
+  const int32x4_t y0_hi = vrshrq_n_s32(vaddq_s32(s0_hi, s3_hi), INV_COS_BIT);
+  const int32x4_t y1_lo = vrshrq_n_s32(vaddq_s32(s1_lo, s3_lo), INV_COS_BIT);
+  const int32x4_t y1_hi = vrshrq_n_s32(vaddq_s32(s1_hi, s3_hi), INV_COS_BIT);
+  const int32x4_t y2_lo = vrshrq_n_s32(s2_lo, INV_COS_BIT);
+  const int32x4_t y2_hi = vrshrq_n_s32(s2_hi, INV_COS_BIT);
+  const int32x4_t y3_lo =
+      vrshrq_n_s32(vsubq_s32(vaddq_s32(s0_lo, s1_lo), s3_lo), INV_COS_BIT);
+  const int32x4_t y3_hi =
+      vrshrq_n_s32(vsubq_s32(vaddq_s32(s0_hi, s1_hi), s3_hi), INV_COS_BIT);
+
+  out[0] = narrow_inv_txfm4_out_neon(y0_lo, y0_hi, out_rshift);
+  out[1] = narrow_inv_txfm4_out_neon(y1_lo, y1_hi, out_rshift);
+  out[2] = narrow_inv_txfm4_out_neon(y2_lo, y2_hi, out_rshift);
+  out[3] = narrow_inv_txfm4_out_neon(y3_lo, y3_hi, out_rshift);
+}
+
+static AOM_FORCE_INLINE void apply_txfm4_s16x8_neon(int16x8_t *a,
+                                                    ITX_TYPE_1D type,
+                                                    int out_rshift) {
+  switch (type) {
+    case IDCT_1D: idct4_s16x8_neon(a, a, out_rshift); break;
+    case IADST_1D: iadst4_s16x8_neon(a, a, out_rshift); break;
+    default: identity_txfm_round_neon(a, a, 0, 4, out_rshift); break;
   }
+}

-  for (int c = 0; c < txfm_size_col; ++c) {
-    if (lr_flip == 0) {
-      for (r = 0; r < txfm_size_row; ++r)
-        temp_in[r] = buf[r * txfm_size_col + c];
-    } else {
-      // flip left right
-      for (r = 0; r < txfm_size_row; ++r)
-        temp_in[r] = buf[r * txfm_size_col + (txfm_size_col - c - 1)];
-    }
-    clamp_buf(temp_in, txfm_size_row, 16);
-    col_txfm(temp_in, temp_out, INV_COS_BIT, stage_range);
-    av1_round_shift_array(temp_out, txfm_size_row, -shift[1]);
+static AOM_FORCE_INLINE void apply_txfm8_s16x8_neon(int16x8_t *b,
+                                                    ITX_TYPE_1D type,
+                                                    int out_rshift) {
+  switch (type) {
+    case IDCT_1D:
+      idct8_neon(b, b, INV_COS_BIT);
+      round_shift_array_16_neon(b, 8, out_rshift);
+      break;
+    case IADST_1D:
+      iadst8_neon(b, b, INV_COS_BIT);
+      round_shift_array_16_neon(b, 8, out_rshift);
+      break;
+    default: identity_txfm_round_neon(b, b, 1, 8, out_rshift); break;
+  }
+}

-    if (ud_flip == 0) {
-      for (r = 0; r < txfm_size_row; ++r) {
-        output[r * stride + c] =
-            clip_pixel(output[r * stride + c] + temp_out[r]);
-      }
-    } else {
-      // flip upside down
-      for (r = 0; r < txfm_size_row; ++r) {
-        output[r * stride + c] = clip_pixel(output[r * stride + c] +
-                                            temp_out[txfm_size_row - r - 1]);
-      }
-    }
+static AOM_FORCE_INLINE void apply_txfm16_s16x8_neon(int16x8_t *b,
+                                                     ITX_TYPE_1D type,
+                                                     int out_rshift) {
+  switch (type) {
+    case IDCT_1D:
+      idct16_neon(b, b, INV_COS_BIT);
+      round_shift_array_16_neon(b, 16, out_rshift);
+      break;
+    case IADST_1D:
+      iadst16_neon(b, b, INV_COS_BIT);
+      round_shift_array_16_neon(b, 16, out_rshift);
+      break;
+    default: identity_txfm_round_neon(b, b, 2, 16, out_rshift); break;
   }
 }

-static void lowbd_inv_txfm2d_add_8x4_neon(const int32_t *input, uint8_t *output,
+static void lowbd_inv_txfm2d_add_4x8_neon(const int32_t *input, uint8_t *output,
                                           int stride, TX_TYPE tx_type,
                                           int eob) {
-  (void)eob;
-  TX_SIZE tx_size = TX_8X4;
-  DECLARE_ALIGNED(32, int, txfm_buf[8 * 4 + 8 + 8]);
-  int32_t *temp_in = txfm_buf;
-
-  const int8_t *shift = av1_inv_txfm_shift_ls[tx_size];
-  const int txw_idx = get_txw_idx(tx_size);
-  const int txh_idx = get_txh_idx(tx_size);
-  const int txfm_size_col = tx_size_wide[tx_size];
-  const int txfm_size_row = tx_size_high[tx_size];
-  const int buf_offset = AOMMAX(txfm_size_row, txfm_size_col);
-  int32_t *temp_out = temp_in + buf_offset;
-  int32_t *buf = temp_out + buf_offset;
-  int32_t *buf_ptr = buf;
-  const int8_t stage_range[MAX_TXFM_STAGE_NUM] = { 16, 16, 16, 16,
-                                                   16, 16, 16, 16 };
-  int r;
-  const transform_1d_neon row_txfm =
-      lowbd_txfm_all_1d_arr[txw_idx][hitx_1d_tab[tx_type]];
-  const transform_1d_neon col_txfm =
-      lowbd_txfm_all_1d_arr[txh_idx][vitx_1d_tab[tx_type]];
+  if (eob == 1 && tx_type == DCT_DCT) {
+    const int32_t *cospi = cospi_arr(INV_COS_BIT);
+    int32_t dc = round_shift((int64_t)input[0] * NewInvSqrt2, NewSqrt2Bits);
+    dc = round_shift((int64_t)dc * cospi[32], INV_COS_BIT);
+    dc = round_shift((int64_t)dc * cospi[32], INV_COS_BIT);
+    dc = round_shift((int64_t)dc, 4);
+    lowbd_inv_txfm_dc_only_4xn_neon(output, stride, dc, 8);
+    return;
+  }

+  int16x8_t a[4];
+  int16x4_t t[8];
+  int16x8_t b[8];
   int ud_flip, lr_flip;
   get_flip_cfg(tx_type, &ud_flip, &lr_flip);

-  for (int i = 0; i < txfm_size_row; i++) {
-    for (int c = 0; c < txfm_size_col; c++)
-      temp_in[c] = round_shift((int64_t)input[c * txfm_size_row] * NewInvSqrt2,
-                               NewSqrt2Bits);
-
-    row_txfm(temp_in, buf_ptr, INV_COS_BIT, stage_range);
-    input++;
-    buf_ptr += txfm_size_col;
+  load_buffer_32bit_to_16bit_neon(input, 8, a, 4);
+  round_shift_for_rect(a, a, 4);
+  apply_txfm4_s16x8_neon(a, hitx_1d_tab[tx_type], 0);
+  if (lr_flip) {
+    flip_buf_ud_neon(a, 4);
+  }
+  transpose_arrays_s16_8x4(a, t);
+  for (int i = 0; i < 8; ++i) {
+    b[i] = vcombine_s16(t[i], vdup_n_s16(0));
   }
+  apply_txfm8_s16x8_neon(b, vitx_1d_tab[tx_type], 4);
+  lowbd_add_flip_buffer_4xn_neon(b, output, stride, ud_flip, 8);
+}

-  for (int c = 0; c < txfm_size_col; ++c) {
-    if (lr_flip == 0) {
-      for (r = 0; r < txfm_size_row; ++r)
-        temp_in[r] = buf[r * txfm_size_col + c];
-    } else {
-      // flip left right
-      for (r = 0; r < txfm_size_row; ++r)
-        temp_in[r] = buf[r * txfm_size_col + (txfm_size_col - c - 1)];
+static void lowbd_inv_txfm2d_add_8x4_neon(const int32_t *input, uint8_t *output,
+                                          int stride, TX_TYPE tx_type,
+                                          int eob) {
+  if (eob == 1 && tx_type == DCT_DCT) {
+    const int32_t *cospi = cospi_arr(INV_COS_BIT);
+    int32_t dc = round_shift((int64_t)input[0] * NewInvSqrt2, NewSqrt2Bits);
+    dc = round_shift((int64_t)dc * cospi[32], INV_COS_BIT);
+    dc = round_shift((int64_t)dc * cospi[32], INV_COS_BIT);
+    dc = round_shift((int64_t)dc, 4);
+    const int16x8_t dc_v = vdupq_n_s16((int16_t)clamp(dc, -255, 255));
+    for (int i = 0; i < 4; ++i) {
+      const int16x8_t pred = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(output)));
+      vst1_u8(output, vqmovun_s16(vaddq_s16(pred, dc_v)));
+      output += stride;
     }
-    clamp_buf(temp_in, txfm_size_row, 16);
-    col_txfm(temp_in, temp_out, INV_COS_BIT, stage_range);
-    av1_round_shift_array(temp_out, txfm_size_row, -shift[1]);
+    return;
+  }

-    if (ud_flip == 0) {
-      for (r = 0; r < txfm_size_row; ++r) {
-        output[r * stride + c] =
-            clip_pixel(output[r * stride + c] + temp_out[r]);
-      }
-    } else {
-      // flip upside down
-      for (r = 0; r < txfm_size_row; ++r) {
-        output[r * stride + c] = clip_pixel(output[r * stride + c] +
-                                            temp_out[txfm_size_row - r - 1]);
-      }
+  int16x8_t a[8];
+  int16x4_t row_in[8];
+  int16x8_t b[4];
+  int ud_flip, lr_flip;
+  get_flip_cfg(tx_type, &ud_flip, &lr_flip);
+
+  for (int c = 0; c < 8; ++c) {
+    a[c] = vcombine_s16(vmovn_s32(vld1q_s32(input + c * 4)), vdup_n_s16(0));
+  }
+  round_shift_for_rect(a, a, 8);
+  apply_txfm8_s16x8_neon(a, hitx_1d_tab[tx_type], 0);
+  if (lr_flip) {
+    for (int c = 0; c < 8; ++c) {
+      row_in[c] = vget_low_s16(a[7 - c]);
+    }
+  } else {
+    for (int c = 0; c < 8; ++c) {
+      row_in[c] = vget_low_s16(a[c]);
     }
   }
+  transpose_arrays_s16_4x8(row_in, b);
+  apply_txfm4_s16x8_neon(b, vitx_1d_tab[tx_type], 4);
+  lowbd_add_flip_buffer_8xn_neon(b, output, stride, ud_flip, 4);
 }

 static void lowbd_inv_txfm2d_add_4x16_neon(const int32_t *input,
                                            uint8_t *output, int stride,
                                            TX_TYPE tx_type, int eob) {
-  (void)eob;
-  TX_SIZE tx_size = TX_4X16;
-  DECLARE_ALIGNED(32, int, txfm_buf[4 * 16 + 16 + 16]);
-  int32_t *temp_in = txfm_buf;
-
-  const int8_t *shift = av1_inv_txfm_shift_ls[tx_size];
-  const int txw_idx = get_txw_idx(tx_size);
-  const int txh_idx = get_txh_idx(tx_size);
-  const int txfm_size_col = tx_size_wide[tx_size];
-  const int txfm_size_row = tx_size_high[tx_size];
-  const int buf_offset = AOMMAX(txfm_size_row, txfm_size_col);
-  int32_t *temp_out = temp_in + buf_offset;
-  int32_t *buf = temp_out + buf_offset;
-  int32_t *buf_ptr = buf;
-  const int8_t stage_range[MAX_TXFM_STAGE_NUM] = { 16, 16, 16, 16, 16,
-                                                   16, 16, 16, 16, 16 };
-  int r;
-  const transform_1d_neon row_txfm =
-      lowbd_txfm_all_1d_arr[txw_idx][hitx_1d_tab[tx_type]];
-  const transform_1d_neon col_txfm =
-      lowbd_txfm_all_1d_arr[txh_idx][vitx_1d_tab[tx_type]];
+  if (eob == 1 && tx_type == DCT_DCT) {
+    const int32_t *cospi = cospi_arr(INV_COS_BIT);
+    int32_t dc = round_shift((int64_t)input[0] * cospi[32], INV_COS_BIT);
+    dc = round_shift((int64_t)dc, 1);
+    dc = round_shift((int64_t)dc * cospi[32], INV_COS_BIT);
+    dc = round_shift((int64_t)dc, 4);
+    lowbd_inv_txfm_dc_only_4xn_neon(output, stride, dc, 16);
+    return;
+  }

+  int16x8_t a0[4], a1[4];
+  int16x4_t t[16];
+  int16x8_t b[16];
   int ud_flip, lr_flip;
   get_flip_cfg(tx_type, &ud_flip, &lr_flip);

-  for (int i = 0; i < txfm_size_row; i++) {
-    for (int c = 0; c < txfm_size_col; c++)
-      temp_in[c] = input[c * txfm_size_row];
-    row_txfm(temp_in, buf_ptr, INV_COS_BIT, stage_range);
-    av1_round_shift_array(buf_ptr, txfm_size_col, -shift[0]);
-    input++;
-    buf_ptr += txfm_size_col;
+  load_buffer_32bit_to_16bit_neon(input, 16, a0, 4);
+  load_buffer_32bit_to_16bit_neon(input + 8, 16, a1, 4);
+  apply_txfm4_s16x8_neon(a0, hitx_1d_tab[tx_type], 1);
+  apply_txfm4_s16x8_neon(a1, hitx_1d_tab[tx_type], 1);
+  if (lr_flip) {
+    flip_buf_ud_neon(a0, 4);
+    flip_buf_ud_neon(a1, 4);
   }
-
-  for (int c = 0; c < txfm_size_col; ++c) {
-    if (lr_flip == 0) {
-      for (r = 0; r < txfm_size_row; ++r)
-        temp_in[r] = buf[r * txfm_size_col + c];
-    } else {
-      // flip left right
-      for (r = 0; r < txfm_size_row; ++r)
-        temp_in[r] = buf[r * txfm_size_col + (txfm_size_col - c - 1)];
-    }
-    clamp_buf(temp_in, txfm_size_row, 16);
-    col_txfm(temp_in, temp_out, INV_COS_BIT, stage_range);
-    av1_round_shift_array(temp_out, txfm_size_row, -shift[1]);
-
-    if (ud_flip == 0) {
-      for (r = 0; r < txfm_size_row; ++r) {
-        output[r * stride + c] =
-            clip_pixel(output[r * stride + c] + temp_out[r]);
-      }
-    } else {
-      // flip upside down
-      for (r = 0; r < txfm_size_row; ++r) {
-        output[r * stride + c] = clip_pixel(output[r * stride + c] +
-                                            temp_out[txfm_size_row - r - 1]);
-      }
-    }
+  transpose_arrays_s16_8x4(a0, t);
+  transpose_arrays_s16_8x4(a1, t + 8);
+  for (int i = 0; i < 16; ++i) {
+    b[i] = vcombine_s16(t[i], vdup_n_s16(0));
   }
+  apply_txfm16_s16x8_neon(b, vitx_1d_tab[tx_type], 4);
+  lowbd_add_flip_buffer_4xn_neon(b, output, stride, ud_flip, 16);
 }

 static void lowbd_inv_txfm2d_add_16x4_neon(const int32_t *input,
                                            uint8_t *output, int stride,
                                            TX_TYPE tx_type, int eob) {
-  (void)eob;
-  TX_SIZE tx_size = TX_16X4;
-  DECLARE_ALIGNED(32, int, txfm_buf[16 * 4 + 16 + 16]);
-  int32_t *temp_in = txfm_buf;
-
-  const int8_t *shift = av1_inv_txfm_shift_ls[tx_size];
-  const int txw_idx = get_txw_idx(tx_size);
-  const int txh_idx = get_txh_idx(tx_size);
-  const int txfm_size_col = tx_size_wide[tx_size];
-  const int txfm_size_row = tx_size_high[tx_size];
-  const int buf_offset = AOMMAX(txfm_size_row, txfm_size_col);
-  int32_t *temp_out = temp_in + buf_offset;
-  int32_t *buf = temp_out + buf_offset;
-  int32_t *buf_ptr = buf;
-  const int8_t stage_range[MAX_TXFM_STAGE_NUM] = { 16, 16, 16, 16, 16,
-                                                   16, 16, 16, 16, 16 };
-  int r;
-  const transform_1d_neon row_txfm =
-      lowbd_txfm_all_1d_arr[txw_idx][hitx_1d_tab[tx_type]];
-  const transform_1d_neon col_txfm =
-      lowbd_txfm_all_1d_arr[txh_idx][vitx_1d_tab[tx_type]];
+  if (eob == 1 && tx_type == DCT_DCT) {
+    const int32_t *cospi = cospi_arr(INV_COS_BIT);
+    int32_t dc = round_shift((int64_t)input[0] * cospi[32], INV_COS_BIT);
+    dc = round_shift((int64_t)dc, 1);
+    dc = round_shift((int64_t)dc * cospi[32], INV_COS_BIT);
+    dc = round_shift((int64_t)dc, 4);
+    const int16x8_t dc_v = vdupq_n_s16((int16_t)clamp(dc, -255, 255));
+    for (int i = 0; i < 4; ++i) {
+      const uint8x16_t pred = vld1q_u8(output);
+      vst1q_u8(output, lowbd_get_recon_16x16_neon(pred, dc_v, dc_v));
+      output += stride;
+    }
+    return;
+  }

+  int16x8_t a[16];
+  int16x4_t row_lo[8], row_hi[8];
+  int16x8_t b[8];
   int ud_flip, lr_flip;
   get_flip_cfg(tx_type, &ud_flip, &lr_flip);

-  for (int i = 0; i < txfm_size_row; i++) {
-    for (int c = 0; c < txfm_size_col; c++)
-      temp_in[c] = input[c * txfm_size_row];
-    row_txfm(temp_in, buf_ptr, INV_COS_BIT, stage_range);
-    av1_round_shift_array(buf_ptr, txfm_size_col, -shift[0]);
-    input++;
-    buf_ptr += txfm_size_col;
+  for (int c = 0; c < 16; ++c) {
+    a[c] = vcombine_s16(vmovn_s32(vld1q_s32(input + c * 4)), vdup_n_s16(0));
   }
-
-  for (int c = 0; c < txfm_size_col; ++c) {
-    if (lr_flip == 0) {
-      for (r = 0; r < txfm_size_row; ++r)
-        temp_in[r] = buf[r * txfm_size_col + c];
-    } else {
-      // flip left right
-      for (r = 0; r < txfm_size_row; ++r)
-        temp_in[r] = buf[r * txfm_size_col + (txfm_size_col - c - 1)];
+  apply_txfm16_s16x8_neon(a, hitx_1d_tab[tx_type], 1);
+  if (lr_flip) {
+    for (int c = 0; c < 8; ++c) {
+      row_lo[c] = vget_low_s16(a[15 - c]);
+      row_hi[c] = vget_low_s16(a[7 - c]);
     }
-    clamp_buf(temp_in, txfm_size_row, 16);
-    col_txfm(temp_in, temp_out, INV_COS_BIT, stage_range);
-    av1_round_shift_array(temp_out, txfm_size_row, -shift[1]);
-
-    if (ud_flip == 0) {
-      for (r = 0; r < txfm_size_row; ++r) {
-        output[r * stride + c] =
-            clip_pixel(output[r * stride + c] + temp_out[r]);
-      }
-    } else {
-      // flip upside down
-      for (r = 0; r < txfm_size_row; ++r) {
-        output[r * stride + c] = clip_pixel(output[r * stride + c] +
-                                            temp_out[txfm_size_row - r - 1]);
-      }
+  } else {
+    for (int c = 0; c < 8; ++c) {
+      row_lo[c] = vget_low_s16(a[c]);
+      row_hi[c] = vget_low_s16(a[c + 8]);
     }
   }
+  transpose_arrays_s16_4x8(row_lo, b);
+  transpose_arrays_s16_4x8(row_hi, b + 4);
+  apply_txfm4_s16x8_neon(b, vitx_1d_tab[tx_type], 4);
+  apply_txfm4_s16x8_neon(b + 4, vitx_1d_tab[tx_type], 4);
+  lowbd_add_flip_buffer_16xn_neon(b, output, stride, ud_flip, 4);
 }

 static inline void lowbd_inv_txfm2d_add_no_identity_neon(
@@ -4098,6 +4188,45 @@ static inline void lowbd_inv_txfm2d_add_no_identity_neon(
   const int fun_idx_y = lowbd_txfm_all_1d_zeros_idx[eoby];
   int temp_b = 0;

+  if (tx_type == DCT_DCT) {
+    const transform_neon row_txfm =
+        lowbd_txfm_all_1d_zeros_w_arr[txw_idx][IDCT_1D][fun_idx_x];
+    const transform_neon col_txfm =
+        lowbd_txfm_all_1d_zeros_w_arr[txh_idx][IDCT_1D][fun_idx_y];
+
+    for (int i = 0; i < buf_size_nonzero_h_div8; i++) {
+      int16x8_t *cur_a = &a[i * txfm_size_col];
+      load_buffer_32bit_to_16bit_neon(input, input_stride, cur_a,
+                                      buf_size_nonzero_w);
+      input += 8;
+      if (abs(rect_type) == 1) {
+        round_shift_for_rect(cur_a, cur_a, buf_size_nonzero_w);
+      }
+      row_txfm(cur_a, cur_a, INV_COS_BIT);
+      round_shift_array_16_neon(cur_a, txfm_size_col, -shift[0]);
+      for (int j = 0; j < buf_size_w_div8; ++j) {
+        transpose_arrays_s16_8x8(&cur_a[j * 8], &b[temp_b + txfm_size_row * j]);
+      }
+      temp_b += 8;
+    }
+    for (int j = 0; j < buf_size_w_div8; ++j) {
+      col_txfm(&b[j * txfm_size_row], &b[j * txfm_size_row], INV_COS_BIT);
+      round_shift_array_16_neon(&b[j * txfm_size_row], txfm_size_row,
+                                -shift[1]);
+    }
+
+    if (txfm_size_col >= 16) {
+      for (int i = 0; i < (txfm_size_col >> 4); i++) {
+        lowbd_add_flip_buffer_16xn_neon(&b[i * txfm_size_row * 2],
+                                        output + 16 * i, stride, 0,
+                                        txfm_size_row);
+      }
+    } else if (txfm_size_col == 8) {
+      lowbd_add_flip_buffer_8xn_neon(b, output, stride, 0, txfm_size_row);
+    }
+    return;
+  }
+
   const transform_neon row_txfm =
       lowbd_txfm_all_1d_zeros_w_arr[txw_idx][hitx_1d_tab[tx_type]][fun_idx_x];
   const transform_neon col_txfm =
@@ -4149,7 +4278,60 @@ static inline void lowbd_inv_txfm2d_add_no_identity_neon(
   }
 }

-static inline void lowbd_inv_txfm2d_add_universe_neon(
+static inline int16_t lowbd_inv_txfm_dc_coeff_neon(int32_t dc,
+                                                   TX_SIZE tx_size) {
+  const int8_t *shift = av1_inv_txfm_shift_ls[tx_size];
+  const int txfm_size_col = tx_size_wide[tx_size];
+  const int txfm_size_row = tx_size_high[tx_size];
+  const int rect_type = get_rect_tx_log_ratio(txfm_size_col, txfm_size_row);
+  const int16_t cospi32 = (int16_t)cospi_arr(INV_COS_BIT)[32];
+  int16x4_t v_dc = vdup_n_s16((int16_t)dc);
+  if (abs(rect_type) == 1) {
+    v_dc = vqrshrn_n_s32(vmull_n_s16(v_dc, (int16_t)NewInvSqrt2),
+                         (int32_t)NewSqrt2Bits);
+  }
+  v_dc = vrshrn_n_s32(vmull_n_s16(v_dc, cospi32), INV_COS_BIT);
+  int16x8_t v_dc8 = vcombine_s16(v_dc, v_dc);
+  if (shift[0]) {
+    v_dc8 = vrshlq_s16(v_dc8, vdupq_n_s16(shift[0]));
+  }
+  v_dc = vrshrn_n_s32(vmull_n_s16(vget_low_s16(v_dc8), cospi32), INV_COS_BIT);
+  v_dc8 = vcombine_s16(v_dc, v_dc);
+  if (shift[1]) {
+    v_dc8 = vrshlq_s16(v_dc8, vdupq_n_s16(shift[1]));
+  }
+  return vgetq_lane_s16(v_dc8, 0);
+}
+
+static void lowbd_inv_txfm2d_add_dc_only_neon(int32_t dc, uint8_t *output,
+                                              int stride, TX_SIZE tx_size) {
+  const int16x8_t v_dc = vdupq_n_s16(lowbd_inv_txfm_dc_coeff_neon(dc, tx_size));
+  const int width = tx_size_wide[tx_size];
+  int height = tx_size_high[tx_size];
+  assert(height >= 8 && (height & 7) == 0);
+  if (width == 8) {
+    do {
+      const int16x8_t p = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(output)));
+      vst1_u8(output, vqmovun_s16(vaddq_s16(p, v_dc)));
+      output += stride;
+    } while (--height != 0);
+  } else {
+    assert(width >= 16 && (width & 15) == 0);
+    do {
+      int c = 0;
+      do {
+        const uint8x16_t p = vld1q_u8(output + c);
+        vst1q_u8(output + c, lowbd_get_recon_16x16_neon(p, v_dc, v_dc));
+        c += 16;
+      } while (c != width);
+      output += stride;
+    } while (--height != 0);
+  }
+}
+
+// Keep universe dispatcher out of the common fast path (small blocks & DC-only)
+// to prevent code bloat and preserve small-block invocation latency.
+static AOM_NOINLINE void lowbd_inv_txfm2d_add_universe_neon(
     const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type,
     TX_SIZE tx_size, int eob) {
   switch (tx_type) {
@@ -4204,8 +4386,12 @@ void av1_lowbd_inv_txfm2d_add_neon(const int32_t *input, uint8_t *output,
       break;

     default:
-      lowbd_inv_txfm2d_add_universe_neon(input, output, stride, tx_type,
-                                         tx_size, eob);
+      if (eob == 1 && tx_type == DCT_DCT) {
+        lowbd_inv_txfm2d_add_dc_only_neon(input[0], output, stride, tx_size);
+      } else {
+        lowbd_inv_txfm2d_add_universe_neon(input, output, stride, tx_type,
+                                           tx_size, eob);
+      }
       break;
   }
 }