Commit ce4f50cd22 for aom

commit ce4f50cd22ac81b0c11e189cdbac89a9bd177aea
Author: Frank Galligan <fgalligan@google.com>
Date:   Sun Nov 15 23:22:36 2026 -0800

    av1,neon: Use vtx_tab/htx_tab in highbd_fwd_txfm

    Replace indirect function pointer calls in av1_fwd_txfm2d_4x8_neon,
    av1_fwd_txfm2d_8x4_neon, av1_fwd_txfm2d_4x16_neon, and
    av1_fwd_txfm2d_16x4_neon with vtx_tab/htx_tab switches, hoisted input
    loads, in-place transposes, and AOM_FORCE_INLINE on
    TRANSFORM_ROW_RECT_ONE (inlining 4- and 8-point 1D transforms and
    eliminating 304B stack frames and vector spills in 4x8 and 8x4).

    Speedup (DTT4 mix 25% DCT_DCT / 75% ADST, bit-exact across 5 ARM CPUs):
    - 4x8 / 8x4: -19.0% / -18.9% (Neoverse N3), -27.1% / -27.6% (Apple M2
      Ultra), -22.2% / -26.7% (Cortex-X4), -19.5% / -23.3% (A720), and
      -32.6% / -22.6% (A520).
    - 4x16 / 16x4: -4.2% / -2.1% (N3), -12.8% / -1.6% (M2 Ultra), -6.5% /
      -9.8% (X4), -34.1% / -8.0% (A520); -1.08% (1T A720) / -2.10% (4T
      Pixel 9) 10-bit avifenc speed 6.

    TAG=agy
    CONV=490488d4-7005-402a-8cc4-c245227f80fe
    Change-Id: Ibf6e03909b133333333333333333333333333333

diff --git a/av1/encoder/arm/highbd_fwd_txfm_neon.c b/av1/encoder/arm/highbd_fwd_txfm_neon.c
index f1b060510d..766a1e3583 100644
--- a/av1/encoder/arm/highbd_fwd_txfm_neon.c
+++ b/av1/encoder/arm/highbd_fwd_txfm_neon.c
@@ -1186,15 +1186,11 @@ void av1_fwd_txfm2d_16x16_neon(const int16_t *input, int32_t *coeff, int stride,
   }
 }

-typedef void (*fwd_transform_1d_col_neon)(const int16_t *in, int32x4_t *out,
-                                          int stride, int bit, int lr_flip);
 typedef void (*fwd_transform_1d_col_many_neon)(const int16_t *in,
                                                int32x4_t *out, int stride,
                                                int bit, int lr_flip,
                                                int howmany, int hm_stride);

-typedef void (*fwd_transform_1d_row_neon)(const int32x4_t *in, int32_t *out,
-                                          int bit, int stride);
 typedef void (*fwd_transform_1d_row_many_neon)(const int32x4_t *in,
                                                int32_t *out, int bit,
                                                int howmany, int hm_stride,
@@ -1203,18 +1199,6 @@ typedef void (*fwd_transform_1d_row_many_neon)(const int32x4_t *in,
 // Construct component kernels that include the load_buffer and store_buffer
 // stages to avoid the need to spill loaded data to the stack between these and
 // the txfm kernel calls.
-// The TRANSFORM_*_ONE cases are only ever called in situations where the
-// howmany parameter would be one, so no need for the loop at all in these
-// cases.
-
-#define TRANSFORM_COL_ONE(name, n)                                    \
-  static void highbd_##name##_col_neon(const int16_t *input,          \
-                                       int32x4_t *output, int stride, \
-                                       int cos_bit, int lr_flip) {    \
-    int32x4_t buf0[n];                                                \
-    load_buffer_4x##n(input, buf0, stride, lr_flip);                  \
-    highbd_##name##_x4_neon(buf0, output, cos_bit);                   \
-  }

 #define TRANSFORM_COL_MANY(name, n)                                     \
   static void highbd_##name##_col_many_neon(                            \
@@ -1228,16 +1212,8 @@ typedef void (*fwd_transform_1d_row_many_neon)(const int32x4_t *in,
     } while (++i < howmany);                                            \
   }

-#define TRANSFORM_ROW_ONE(name, n)                                        \
-  static void highbd_##name##_row_neon(                                   \
-      const int32x4_t *input, int32_t *output, int cos_bit, int stride) { \
-    int32x4_t buf0[n];                                                    \
-    highbd_##name##_x4_neon(input, buf0, cos_bit);                        \
-    store_buffer_##n##x4(buf0, output, stride);                           \
-  }
-
 #define TRANSFORM_ROW_RECT_ONE(name, n)                                   \
-  static void highbd_##name##_row_rect_neon(                              \
+  static AOM_FORCE_INLINE void highbd_##name##_row_rect_neon(             \
       const int32x4_t *input, int32_t *output, int cos_bit, int stride) { \
     int32x4_t buf0[n];                                                    \
     highbd_##name##_x4_neon(input, buf0, cos_bit);                        \
@@ -1270,44 +1246,27 @@ typedef void (*fwd_transform_1d_row_many_neon)(const int32x4_t *in,
     } while (++i < howmany);                                             \
   }

-TRANSFORM_COL_ONE(fdct8, 8)
-TRANSFORM_COL_ONE(fadst8, 8)
-TRANSFORM_COL_ONE(fidentity8, 8)
-
-TRANSFORM_COL_MANY(fdct4, 4)
 TRANSFORM_COL_MANY(fdct8, 8)
 TRANSFORM_COL_MANY(fdct16, 16)
-TRANSFORM_COL_MANY(fadst4, 4)
 TRANSFORM_COL_MANY(fadst8, 8)
 TRANSFORM_COL_MANY(fadst16, 16)
-TRANSFORM_COL_MANY(fidentity4, 4)
 TRANSFORM_COL_MANY(fidentity8, 8)
 TRANSFORM_COL_MANY(fidentity16, 16)

-TRANSFORM_ROW_ONE(fdct16, 16)
-TRANSFORM_ROW_ONE(fadst16, 16)
-TRANSFORM_ROW_ONE(fidentity16, 16)
-
 TRANSFORM_ROW_RECT_ONE(fdct8, 8)
 TRANSFORM_ROW_RECT_ONE(fadst8, 8)
 TRANSFORM_ROW_RECT_ONE(fidentity8, 8)

 #if !CONFIG_REALTIME_ONLY
-TRANSFORM_ROW_MANY(fdct4, 4)
 TRANSFORM_ROW_MANY(fdct8, 8)
-TRANSFORM_ROW_MANY(fadst4, 4)
 TRANSFORM_ROW_MANY(fadst8, 8)
-TRANSFORM_ROW_MANY(fidentity4, 4)
 TRANSFORM_ROW_MANY(fidentity8, 8)
 #endif

-TRANSFORM_ROW_RECT_MANY(fdct4, 4)
 TRANSFORM_ROW_RECT_MANY(fdct8, 8)
 TRANSFORM_ROW_RECT_MANY(fdct16, 16)
-TRANSFORM_ROW_RECT_MANY(fadst4, 4)
 TRANSFORM_ROW_RECT_MANY(fadst8, 8)
 TRANSFORM_ROW_RECT_MANY(fadst16, 16)
-TRANSFORM_ROW_RECT_MANY(fidentity4, 4)
 TRANSFORM_ROW_RECT_MANY(fidentity8, 8)
 TRANSFORM_ROW_RECT_MANY(fidentity16, 16)

@@ -1331,25 +1290,6 @@ static const fwd_transform_1d_col_many_neon
       highbd_fidentity8_col_many_neon   // H_FLIPADST
     };

-static const fwd_transform_1d_col_neon col_highbd_txfm8_x4_arr[TX_TYPES] = {
-  highbd_fdct8_col_neon,       // DCT_DCT
-  highbd_fadst8_col_neon,      // ADST_DCT
-  highbd_fdct8_col_neon,       // DCT_ADST
-  highbd_fadst8_col_neon,      // ADST_ADST
-  highbd_fadst8_col_neon,      // FLIPADST_DCT
-  highbd_fdct8_col_neon,       // DCT_FLIPADST
-  highbd_fadst8_col_neon,      // FLIPADST_FLIPADST
-  highbd_fadst8_col_neon,      // ADST_FLIPADST
-  highbd_fadst8_col_neon,      // FLIPADST_ADST
-  highbd_fidentity8_col_neon,  // IDTX
-  highbd_fdct8_col_neon,       // V_DCT
-  highbd_fidentity8_col_neon,  // H_DCT
-  highbd_fadst8_col_neon,      // V_ADST
-  highbd_fidentity8_col_neon,  // H_ADST
-  highbd_fadst8_col_neon,      // V_FLIPADST
-  highbd_fidentity8_col_neon   // H_FLIPADST
-};
-
 static const fwd_transform_1d_col_many_neon
     col_highbd_txfm16_xn_arr[TX_TYPES] = {
       highbd_fdct16_col_many_neon,       // DCT_DCT
@@ -1370,45 +1310,6 @@ static const fwd_transform_1d_col_many_neon
       highbd_fidentity16_col_many_neon   // H_FLIPADST
     };

-static const fwd_transform_1d_col_many_neon
-    col_highbd_txfm4_xn_arr[TX_TYPES] = {
-      highbd_fdct4_col_many_neon,       // DCT_DCT
-      highbd_fadst4_col_many_neon,      // ADST_DCT
-      highbd_fdct4_col_many_neon,       // DCT_ADST
-      highbd_fadst4_col_many_neon,      // ADST_ADST
-      highbd_fadst4_col_many_neon,      // FLIPADST_DCT
-      highbd_fdct4_col_many_neon,       // DCT_FLIPADST
-      highbd_fadst4_col_many_neon,      // FLIPADST_FLIPADST
-      highbd_fadst4_col_many_neon,      // ADST_FLIPADST
-      highbd_fadst4_col_many_neon,      // FLIPADST_ADST
-      highbd_fidentity4_col_many_neon,  // IDTX
-      highbd_fdct4_col_many_neon,       // V_DCT
-      highbd_fidentity4_col_many_neon,  // H_DCT
-      highbd_fadst4_col_many_neon,      // V_ADST
-      highbd_fidentity4_col_many_neon,  // H_ADST
-      highbd_fadst4_col_many_neon,      // V_FLIPADST
-      highbd_fidentity4_col_many_neon   // H_FLIPADST
-    };
-
-static const fwd_transform_1d_row_neon row_highbd_txfm16_xn_arr[TX_TYPES] = {
-  highbd_fdct16_row_neon,       // DCT_DCT
-  highbd_fdct16_row_neon,       // ADST_DCT
-  highbd_fadst16_row_neon,      // DCT_ADST
-  highbd_fadst16_row_neon,      // ADST_ADST
-  highbd_fdct16_row_neon,       // FLIPADST_DCT
-  highbd_fadst16_row_neon,      // DCT_FLIPADST
-  highbd_fadst16_row_neon,      // FLIPADST_FLIPADST
-  highbd_fadst16_row_neon,      // ADST_FLIPADST
-  highbd_fadst16_row_neon,      // FLIPADST_ADST
-  highbd_fidentity16_row_neon,  // IDTX
-  highbd_fidentity16_row_neon,  // V_DCT
-  highbd_fdct16_row_neon,       // H_DCT
-  highbd_fidentity16_row_neon,  // V_ADST
-  highbd_fadst16_row_neon,      // H_ADST
-  highbd_fidentity16_row_neon,  // V_FLIPADST
-  highbd_fadst16_row_neon       // H_FLIPADST
-};
-
 static const fwd_transform_1d_row_many_neon
     row_rect_highbd_txfm16_xn_arr[TX_TYPES] = {
       highbd_fdct16_row_rect_many_neon,       // DCT_DCT
@@ -1471,67 +1372,6 @@ static const fwd_transform_1d_row_many_neon
       highbd_fadst8_row_rect_many_neon       // H_FLIPADST
     };

-static const fwd_transform_1d_row_neon row_highbd_txfm8_x4_arr[TX_TYPES] = {
-  highbd_fdct8_row_rect_neon,       // DCT_DCT
-  highbd_fdct8_row_rect_neon,       // ADST_DCT
-  highbd_fadst8_row_rect_neon,      // DCT_ADST
-  highbd_fadst8_row_rect_neon,      // ADST_ADST
-  highbd_fdct8_row_rect_neon,       // FLIPADST_DCT
-  highbd_fadst8_row_rect_neon,      // DCT_FLIPADST
-  highbd_fadst8_row_rect_neon,      // FLIPADST_FLIPADST
-  highbd_fadst8_row_rect_neon,      // ADST_FLIPADST
-  highbd_fadst8_row_rect_neon,      // FLIPADST_ADST
-  highbd_fidentity8_row_rect_neon,  // IDTX
-  highbd_fidentity8_row_rect_neon,  // V_DCT
-  highbd_fdct8_row_rect_neon,       // H_DCT
-  highbd_fidentity8_row_rect_neon,  // V_ADST
-  highbd_fadst8_row_rect_neon,      // H_ADST
-  highbd_fidentity8_row_rect_neon,  // V_FLIPADST
-  highbd_fadst8_row_rect_neon       // H_FLIPADST
-};
-
-#if !CONFIG_REALTIME_ONLY
-static const fwd_transform_1d_row_many_neon
-    row_highbd_txfm4_xn_arr[TX_TYPES] = {
-      highbd_fdct4_row_many_neon,       // DCT_DCT
-      highbd_fdct4_row_many_neon,       // ADST_DCT
-      highbd_fadst4_row_many_neon,      // DCT_ADST
-      highbd_fadst4_row_many_neon,      // ADST_ADST
-      highbd_fdct4_row_many_neon,       // FLIPADST_DCT
-      highbd_fadst4_row_many_neon,      // DCT_FLIPADST
-      highbd_fadst4_row_many_neon,      // FLIPADST_FLIPADST
-      highbd_fadst4_row_many_neon,      // ADST_FLIPADST
-      highbd_fadst4_row_many_neon,      // FLIPADST_ADST
-      highbd_fidentity4_row_many_neon,  // IDTX
-      highbd_fidentity4_row_many_neon,  // V_DCT
-      highbd_fdct4_row_many_neon,       // H_DCT
-      highbd_fidentity4_row_many_neon,  // V_ADST
-      highbd_fadst4_row_many_neon,      // H_ADST
-      highbd_fidentity4_row_many_neon,  // V_FLIPADST
-      highbd_fadst4_row_many_neon       // H_FLIPADST
-    };
-#endif
-
-static const fwd_transform_1d_row_many_neon
-    row_rect_highbd_txfm4_xn_arr[TX_TYPES] = {
-      highbd_fdct4_row_rect_many_neon,       // DCT_DCT
-      highbd_fdct4_row_rect_many_neon,       // ADST_DCT
-      highbd_fadst4_row_rect_many_neon,      // DCT_ADST
-      highbd_fadst4_row_rect_many_neon,      // ADST_ADST
-      highbd_fdct4_row_rect_many_neon,       // FLIPADST_DCT
-      highbd_fadst4_row_rect_many_neon,      // DCT_FLIPADST
-      highbd_fadst4_row_rect_many_neon,      // FLIPADST_FLIPADST
-      highbd_fadst4_row_rect_many_neon,      // ADST_FLIPADST
-      highbd_fadst4_row_rect_many_neon,      // FLIPADST_ADST
-      highbd_fidentity4_row_rect_many_neon,  // IDTX
-      highbd_fidentity4_row_rect_many_neon,  // V_DCT
-      highbd_fdct4_row_rect_many_neon,       // H_DCT
-      highbd_fidentity4_row_rect_many_neon,  // V_ADST
-      highbd_fadst4_row_rect_many_neon,      // H_ADST
-      highbd_fidentity4_row_rect_many_neon,  // V_FLIPADST
-      highbd_fadst4_row_rect_many_neon       // H_FLIPADST
-    };
-
 static void highbd_fdct32_x4_neon(const int32x4_t *input, int32x4_t *output,
                                   int cos_bit) {
   const int32_t *const cospi = cospi_arr_s32(cos_bit);
@@ -2254,12 +2094,8 @@ void av1_fwd_txfm2d_8x16_neon(const int16_t *input, int32_t *coeff, int stride,
 void av1_fwd_txfm2d_4x16_neon(const int16_t *input, int32_t *coeff, int stride,
                               TX_TYPE tx_type, int bd) {
   (void)bd;
-  int bitcol = av1_fwd_cos_bit_col[0][2];
-  int bitrow = av1_fwd_cos_bit_row[0][2];
-  const fwd_transform_1d_col_many_neon col_txfm =
-      col_highbd_txfm16_xn_arr[tx_type];
-  const fwd_transform_1d_row_many_neon row_txfm =
-      row_highbd_txfm4_xn_arr[tx_type];
+  const int bitcol = av1_fwd_cos_bit_col[0][2];
+  const int bitrow = av1_fwd_cos_bit_row[0][2];

   int ud_flip, lr_flip;
   get_flip_cfg(tx_type, &ud_flip, &lr_flip);
@@ -2267,31 +2103,59 @@ void av1_fwd_txfm2d_4x16_neon(const int16_t *input, int32_t *coeff, int stride,

   // Column-wise transform.
   int32x4_t buf0[16];
-  if (lr_flip) {
-    col_txfm(input, buf0, stride, bitcol, /*lr_flip=*/1, /*howmany=*/1,
-             /*hm_stride=*/0);
-  } else {
-    col_txfm(input, buf0, stride, bitcol, /*lr_flip=*/0, /*howmany=*/1,
-             /*hm_stride=*/0);
+  load_buffer_4x16(input, buf0, stride, lr_flip);
+  switch (vtx_tab[tx_type]) {
+    case DCT_1D: highbd_fdct16_x4_neon(buf0, buf0, bitcol); break;
+    case ADST_1D:
+    case FLIPADST_1D: highbd_fadst16_x4_neon(buf0, buf0, bitcol); break;
+    default:
+      assert(vtx_tab[tx_type] == IDTX_1D);
+      highbd_fidentity16_x4_neon(buf0, buf0, bitcol);
+      break;
   }
   shift_right_1_round_s32_x4(buf0, buf0, 16);
-
-  int32x4_t buf1[16];
-  transpose_arrays_s32_4x16(buf0, buf1);
+  transpose_arrays_s32_4x16(buf0, buf0);

   // Row-wise transform.
-  row_txfm(buf1, coeff, bitrow, /*howmany=*/4, /*hm_stride=*/4, /*stride=*/16);
+  switch (htx_tab[tx_type]) {
+    case DCT_1D: {
+      int i = 0;
+      do {
+        int32x4_t tmp[4];
+        highbd_fdct4_x4_neon(buf0 + 4 * i, tmp, bitrow);
+        store_buffer_4x4(tmp, coeff + 4 * i, /*stride=*/16);
+      } while (++i != 4);
+      break;
+    }
+    case ADST_1D:
+    case FLIPADST_1D: {
+      int i = 0;
+      do {
+        int32x4_t tmp[4];
+        highbd_fadst4_x4_neon(buf0 + 4 * i, tmp, bitrow);
+        store_buffer_4x4(tmp, coeff + 4 * i, /*stride=*/16);
+      } while (++i != 4);
+      break;
+    }
+    default: {
+      assert(htx_tab[tx_type] == IDTX_1D);
+      int i = 0;
+      do {
+        int32x4_t tmp[4];
+        highbd_fidentity4_x4_neon(buf0 + 4 * i, tmp, bitrow);
+        store_buffer_4x4(tmp, coeff + 4 * i, /*stride=*/16);
+      } while (++i != 4);
+      break;
+    }
+  }
 }
 #endif

 void av1_fwd_txfm2d_16x4_neon(const int16_t *input, int32_t *coeff, int stride,
                               TX_TYPE tx_type, int bd) {
   (void)bd;
-  int bitcol = av1_fwd_cos_bit_col[2][0];
-  int bitrow = av1_fwd_cos_bit_row[2][0];
-  const fwd_transform_1d_col_many_neon col_txfm =
-      col_highbd_txfm4_xn_arr[tx_type];
-  const fwd_transform_1d_row_neon row_txfm = row_highbd_txfm16_xn_arr[tx_type];
+  const int bitcol = av1_fwd_cos_bit_col[2][0];
+  const int bitrow = av1_fwd_cos_bit_row[2][0];

   int ud_flip, lr_flip;
   get_flip_cfg(tx_type, &ud_flip, &lr_flip);
@@ -2299,19 +2163,53 @@ void av1_fwd_txfm2d_16x4_neon(const int16_t *input, int32_t *coeff, int stride,

   // Column-wise transform.
   int32x4_t buf0[16];
-  if (lr_flip) {
-    col_txfm(input, buf0 + 3 * 4, stride, bitcol, /*lr_flip=*/1, /*howmany=*/4,
-             /*hm_stride=*/-4);
-  } else {
-    col_txfm(input, buf0, stride, bitcol, /*lr_flip=*/0, /*howmany=*/4,
-             /*hm_stride=*/4);
+  const int16_t *in_ptr = lr_flip ? input + 12 : input;
+  const int in_step = lr_flip ? -4 : 4;
+  int i = 0;
+  do {
+    load_buffer_4x4(in_ptr + i * in_step, buf0 + 4 * i, stride, lr_flip);
+  } while (++i != 4);
+
+  switch (vtx_tab[tx_type]) {
+    case DCT_1D: {
+      int j = 0;
+      do {
+        highbd_fdct4_x4_neon(buf0 + 4 * j, buf0 + 4 * j, bitcol);
+      } while (++j != 4);
+      break;
+    }
+    case ADST_1D:
+    case FLIPADST_1D: {
+      int j = 0;
+      do {
+        highbd_fadst4_x4_neon(buf0 + 4 * j, buf0 + 4 * j, bitcol);
+      } while (++j != 4);
+      break;
+    }
+    default: {
+      assert(vtx_tab[tx_type] == IDTX_1D);
+      int j = 0;
+      do {
+        highbd_fidentity4_x4_neon(buf0 + 4 * j, buf0 + 4 * j, bitcol);
+      } while (++j != 4);
+      break;
+    }
   }

   shift_right_1_round_s32_x4(buf0, buf0, 16);
   transpose_arrays_s32_4x16(buf0, buf0);

   // Row-wise transform.
-  row_txfm(buf0, coeff, bitrow, /*stride=*/4);
+  switch (htx_tab[tx_type]) {
+    case DCT_1D: highbd_fdct16_x4_neon(buf0, buf0, bitrow); break;
+    case ADST_1D:
+    case FLIPADST_1D: highbd_fadst16_x4_neon(buf0, buf0, bitrow); break;
+    default:
+      assert(htx_tab[tx_type] == IDTX_1D);
+      highbd_fidentity16_x4_neon(buf0, buf0, bitrow);
+      break;
+  }
+  store_buffer_16x4(buf0, coeff, /*stride=*/4);
 }

 void av1_fwd_txfm2d_16x32_neon(const int16_t *input, int32_t *coeff, int stride,
@@ -2463,11 +2361,8 @@ void av1_fwd_txfm2d_32x8_neon(const int16_t *input, int32_t *coeff, int stride,
 void av1_fwd_txfm2d_4x8_neon(const int16_t *input, int32_t *coeff, int stride,
                              TX_TYPE tx_type, int bd) {
   (void)bd;
-  int bitcol = av1_fwd_cos_bit_col[0][1];
-  int bitrow = av1_fwd_cos_bit_row[0][1];
-  const fwd_transform_1d_col_neon col_txfm = col_highbd_txfm8_x4_arr[tx_type];
-  const fwd_transform_1d_row_many_neon row_txfm =
-      row_rect_highbd_txfm4_xn_arr[tx_type];
+  const int bitcol = av1_fwd_cos_bit_col[0][1];
+  const int bitrow = av1_fwd_cos_bit_row[0][1];

   int ud_flip, lr_flip;
   get_flip_cfg(tx_type, &ud_flip, &lr_flip);
@@ -2475,14 +2370,54 @@ void av1_fwd_txfm2d_4x8_neon(const int16_t *input, int32_t *coeff, int stride,

   // Column-wise transform.
   int32x4_t buf0[8];
-  col_txfm(input, buf0, stride, bitcol, lr_flip);
+  load_buffer_4x8(input, buf0, stride, lr_flip);
+  switch (vtx_tab[tx_type]) {
+    case DCT_1D: highbd_fdct8_x4_neon(buf0, buf0, bitcol); break;
+    case ADST_1D:
+    case FLIPADST_1D: highbd_fadst8_x4_neon(buf0, buf0, bitcol); break;
+    default:
+      assert(vtx_tab[tx_type] == IDTX_1D);
+      highbd_fidentity8_x4_neon(buf0, buf0, bitcol);
+      break;
+  }
   shift_right_1_round_s32_x4(buf0, buf0, 8);
-
-  int32x4_t buf1[8];
-  transpose_arrays_s32_4x8(buf0, buf1);
+  transpose_arrays_s32_4x8(buf0, buf0);

   // Row-wise transform.
-  row_txfm(buf1, coeff, bitrow, /*howmany=*/2, /*hm_stride=*/4, /*stride=*/8);
+  switch (htx_tab[tx_type]) {
+    case DCT_1D: {
+      int i = 0;
+      do {
+        int32x4_t tmp[4];
+        highbd_fdct4_x4_neon(buf0 + 4 * i, tmp, bitrow);
+        round_rect_array_s32_neon(tmp, tmp, 4);
+        store_buffer_4x4(tmp, coeff + 4 * i, /*stride=*/8);
+      } while (++i != 2);
+      break;
+    }
+    case ADST_1D:
+    case FLIPADST_1D: {
+      int i = 0;
+      do {
+        int32x4_t tmp[4];
+        highbd_fadst4_x4_neon(buf0 + 4 * i, tmp, bitrow);
+        round_rect_array_s32_neon(tmp, tmp, 4);
+        store_buffer_4x4(tmp, coeff + 4 * i, /*stride=*/8);
+      } while (++i != 2);
+      break;
+    }
+    default: {
+      assert(htx_tab[tx_type] == IDTX_1D);
+      int i = 0;
+      do {
+        int32x4_t tmp[4];
+        highbd_fidentity4_x4_neon(buf0 + 4 * i, tmp, bitrow);
+        round_rect_array_s32_neon(tmp, tmp, 4);
+        store_buffer_4x4(tmp, coeff + 4 * i, /*stride=*/8);
+      } while (++i != 2);
+      break;
+    }
+  }
 }

 void av1_fwd_txfm2d_8x4_neon(const int16_t *input, int32_t *coeff, int stride,
@@ -2490,9 +2425,6 @@ void av1_fwd_txfm2d_8x4_neon(const int16_t *input, int32_t *coeff, int stride,
   (void)bd;
   const int bitcol = av1_fwd_cos_bit_col[1][0];
   const int bitrow = av1_fwd_cos_bit_row[1][0];
-  const fwd_transform_1d_col_many_neon col_txfm =
-      col_highbd_txfm4_xn_arr[tx_type];
-  const fwd_transform_1d_row_neon row_txfm = row_highbd_txfm8_x4_arr[tx_type];

   int ud_flip, lr_flip;
   get_flip_cfg(tx_type, &ud_flip, &lr_flip);
@@ -2500,21 +2432,42 @@ void av1_fwd_txfm2d_8x4_neon(const int16_t *input, int32_t *coeff, int stride,

   // Column-wise transform.
   int32x4_t buf0[8];
-  if (lr_flip) {
-    col_txfm(input, buf0 + 4, stride, bitcol, /*lr_flip=*/1, /*howmany=*/2,
-             /*hm_stride=*/-4);
-  } else {
-    col_txfm(input, buf0, stride, bitcol, /*lr_flip=*/0, /*howmany=*/2,
-             /*hm_stride=*/4);
+  load_buffer_4x4(lr_flip ? input + 4 : input, buf0, stride, lr_flip);
+  load_buffer_4x4(lr_flip ? input : input + 4, buf0 + 4, stride, lr_flip);
+  switch (vtx_tab[tx_type]) {
+    case DCT_1D:
+      highbd_fdct4_x4_neon(buf0, buf0, bitcol);
+      highbd_fdct4_x4_neon(buf0 + 4, buf0 + 4, bitcol);
+      break;
+    case ADST_1D:
+    case FLIPADST_1D:
+      highbd_fadst4_x4_neon(buf0, buf0, bitcol);
+      highbd_fadst4_x4_neon(buf0 + 4, buf0 + 4, bitcol);
+      break;
+    default:
+      assert(vtx_tab[tx_type] == IDTX_1D);
+      highbd_fidentity4_x4_neon(buf0, buf0, bitcol);
+      highbd_fidentity4_x4_neon(buf0 + 4, buf0 + 4, bitcol);
+      break;
   }

   shift_right_1_round_s32_x4(buf0, buf0, 8);
-
-  int32x4_t buf1[8];
-  transpose_arrays_s32_8x4(buf0, buf1);
+  transpose_arrays_s32_8x4(buf0, buf0);

   // Row-wise transform.
-  row_txfm(buf1, coeff, bitrow, /*stride=*/4);
+  switch (htx_tab[tx_type]) {
+    case DCT_1D:
+      highbd_fdct8_row_rect_neon(buf0, coeff, bitrow, /*stride=*/4);
+      break;
+    case ADST_1D:
+    case FLIPADST_1D:
+      highbd_fadst8_row_rect_neon(buf0, coeff, bitrow, /*stride=*/4);
+      break;
+    default:
+      assert(htx_tab[tx_type] == IDTX_1D);
+      highbd_fidentity8_row_rect_neon(buf0, coeff, bitrow, /*stride=*/4);
+      break;
+  }
 }

 #if !CONFIG_REALTIME_ONLY