Commit 9a3d085ec1 for aom
commit 9a3d085ec17928aaf436c049ae5ea7eb4d970751
Author: Frank Galligan <fgalligan@google.com>
Date: Sun Nov 15 23:22:08 2026 -0800
av1,neon: Use vtx_tab/htx_tab in fwd_txfm2d_neon
Dispatch 8x8 and rectangular forward transforms (4x8, 8x4, 4x16, 16x4,
8x16, 16x8, 32x8) via hoisted vtx_tab/htx_tab switches with inlined 1D
transforms and in-place lr_flip, and call 64-point 1D DCTs directly
while trimming unused upper-half shifts and stack buffers.
Speedup (DTT4 mix 25% DCT_DCT / 75% ADST, bit-exact across 5 ARM CPUs):
- 8x8: -14.0% (Neoverse N3), -9.7% (Apple M2 Ultra), -10.7% / -9.1% /
-23.0% (Pixel 9 Cortex-X4 / A720 / A520).
- 4x8/8x4/4x16/16x4: -6.7% to -18.5% (N3), -8.5% to -22.5% (M2 Ultra),
-9.6% to -18.2% (X4), -13.5% to -20.9% (A520); -0.47% (1T N3) and
-1.58% (4T Pixel 9) avifenc speed 6.
TAG=agy
CONV=490488d4-7005-402a-8cc4-c245227f80fe
Change-Id: Ibf6e03909b122222222222222222222222222222
diff --git a/av1/encoder/arm/av1_fwd_txfm2d_neon.c b/av1/encoder/arm/av1_fwd_txfm2d_neon.c
index ded7387601..6dd84300b6 100644
--- a/av1/encoder/arm/av1_fwd_txfm2d_neon.c
+++ b/av1/encoder/arm/av1_fwd_txfm2d_neon.c
@@ -178,20 +178,42 @@ static AOM_FORCE_INLINE void butterfly_s16_s32_x8_1223_neon(
butterfly_s16_s32_x8_neon(w0101, 1, 2, 2, 3, in0, in1, out0, out1);
}
-static AOM_FORCE_INLINE void flip_buf_4_neon(int16x4_t *in, int16x4_t *out,
- int size) {
+static AOM_FORCE_INLINE void flip_buf_4_neon(const int16x4_t *in,
+ int16x4_t *out, int size) {
for (int i = 0; i < size; ++i) {
out[size - i - 1] = in[i];
}
}
-static AOM_FORCE_INLINE void flip_buf_8_neon(int16x8_t *in, int16x8_t *out,
- int size) {
+static AOM_FORCE_INLINE void flip_buf_8_neon(const int16x8_t *in,
+ int16x8_t *out, int size) {
for (int i = 0; i < size; ++i) {
out[size - i - 1] = in[i];
}
}
+static AOM_FORCE_INLINE void flip_buf_4_inplace_neon(int16x4_t *buf, int size) {
+ assert(size >= 2 && (size & 1) == 0);
+ const int half = size >> 1;
+ int i = 0;
+ do {
+ const int16x4_t tmp = buf[i];
+ buf[i] = buf[size - 1 - i];
+ buf[size - 1 - i] = tmp;
+ } while (++i != half);
+}
+
+static AOM_FORCE_INLINE void flip_buf_8_inplace_neon(int16x8_t *buf, int size) {
+ assert(size >= 2 && (size & 1) == 0);
+ const int half = size >> 1;
+ int i = 0;
+ do {
+ const int16x8_t tmp = buf[i];
+ buf[i] = buf[size - 1 - i];
+ buf[size - 1 - i] = tmp;
+ } while (++i != half);
+}
+
static AOM_FORCE_INLINE void store_buffer_interleaved_s32_x8(
int32_t *const out, const int32x4_t *const in1, const int32x4_t *const in2,
const int stride, const int out_size) {
@@ -1500,218 +1522,114 @@ static AOM_FORCE_INLINE void fidentity8x32_neon(const int16x8_t *input,
shift_left_2_s16_x8(input, output, 32);
}
+#define TRANSFORM_COL_IMPL(name, tw, n) \
+ static AOM_FORCE_INLINE void name##_col_impl_neon( \
+ const int16_t *input, int16x##tw##_t *output, int stride, int cos_bit) { \
+ int16x##tw##_t buf0[n]; \
+ load_buffer_s16_x##tw(input, stride, buf0, n); \
+ shift_left_2_s16_x##tw(buf0, buf0, n); \
+ name##_neon(buf0, output, cos_bit); \
+ }
+
#define TRANSFORM_COL(name, tw, n) \
+ TRANSFORM_COL_IMPL(name, tw, n) \
static void name##_col_neon(const int16_t *input, int16x##tw##_t *output, \
int stride, int cos_bit) { \
- int16x##tw##_t buf0[n]; \
- load_buffer_s16_x##tw(input, stride, buf0, n); \
- shift_left_2_s16_x##tw(buf0, buf0, n); \
- name##_neon(buf0, output, cos_bit); \
+ name##_col_impl_neon(input, output, stride, cos_bit); \
}
TRANSFORM_COL(fadst4x4, 4, 4)
-TRANSFORM_COL(fadst4x8, 4, 8)
-TRANSFORM_COL(fadst4x16, 4, 16)
-TRANSFORM_COL(fadst8x4, 8, 4)
-TRANSFORM_COL(fadst8x8, 8, 8)
+TRANSFORM_COL_IMPL(fadst4x8, 4, 8)
+TRANSFORM_COL_IMPL(fadst4x16, 4, 16)
+TRANSFORM_COL_IMPL(fadst8x4, 8, 4)
+TRANSFORM_COL_IMPL(fadst8x8, 8, 8)
TRANSFORM_COL(fadst8x16, 8, 16)
TRANSFORM_COL(fdct4x4, 4, 4)
-TRANSFORM_COL(fdct4x8, 4, 8)
-TRANSFORM_COL(fdct4x16, 4, 16)
-TRANSFORM_COL(fdct8x4, 8, 4)
-TRANSFORM_COL(fdct8x8, 8, 8)
+TRANSFORM_COL_IMPL(fdct4x8, 4, 8)
+TRANSFORM_COL_IMPL(fdct4x16, 4, 16)
+TRANSFORM_COL_IMPL(fdct8x4, 8, 4)
+TRANSFORM_COL_IMPL(fdct8x8, 8, 8)
TRANSFORM_COL(fdct8x16, 8, 16)
TRANSFORM_COL(fdct8x32, 8, 32)
TRANSFORM_COL(fidentity4x4, 4, 4)
-TRANSFORM_COL(fidentity4x8, 4, 8)
-TRANSFORM_COL(fidentity4x16, 4, 16)
-TRANSFORM_COL(fidentity8x4, 8, 4)
-TRANSFORM_COL(fidentity8x8, 8, 8)
+TRANSFORM_COL_IMPL(fidentity4x8, 4, 8)
+TRANSFORM_COL_IMPL(fidentity4x16, 4, 16)
+TRANSFORM_COL_IMPL(fidentity8x4, 8, 4)
+TRANSFORM_COL_IMPL(fidentity8x8, 8, 8)
TRANSFORM_COL(fidentity8x16, 8, 16)
TRANSFORM_COL(fidentity8x32, 8, 32)
+#define TRANSFORM_ROW_IMPL(name, tw, n) \
+ static AOM_FORCE_INLINE void name##_row_impl_neon( \
+ const int16x##tw##_t *input, int32_t *output, int stride, int cos_bit) { \
+ int16x##tw##_t buf0[n]; \
+ name##_neon(input, buf0, cos_bit); \
+ store_buffer_s16_x##tw(buf0, output, stride, n); \
+ }
+
#define TRANSFORM_ROW(name, tw, n) \
+ TRANSFORM_ROW_IMPL(name, tw, n) \
static void name##_row_neon(const int16x##tw##_t *input, int32_t *output, \
int stride, int cos_bit) { \
- int16x##tw##_t buf0[n]; \
- name##_neon(input, buf0, cos_bit); \
- store_buffer_s16_x##tw(buf0, output, stride, n); \
+ name##_row_impl_neon(input, output, stride, cos_bit); \
}
-#define TRANSFORM_ROW_RECT(name, tw, n) \
- static void name##_row_rect_neon(const int16x##tw##_t *input, \
- int32_t *output, int stride, int cos_bit) { \
+#define TRANSFORM_ROW_RECT_IMPL(name, tw, n) \
+ static AOM_FORCE_INLINE void name##_row_rect_impl_neon( \
+ const int16x##tw##_t *input, int32_t *output, int stride, int cos_bit) { \
int16x##tw##_t buf0[n]; \
name##_neon(input, buf0, cos_bit); \
store_rect_buffer_s16_x##tw(buf0, output, stride, n); \
}
+#define TRANSFORM_ROW_RECT(name, tw, n) \
+ TRANSFORM_ROW_RECT_IMPL(name, tw, n) \
+ static void name##_row_rect_neon(const int16x##tw##_t *input, \
+ int32_t *output, int stride, int cos_bit) { \
+ name##_row_rect_impl_neon(input, output, stride, cos_bit); \
+ }
+
TRANSFORM_ROW(fadst4x4, 4, 4)
-TRANSFORM_ROW(fadst4x16, 4, 16)
-TRANSFORM_ROW(fadst8x4, 8, 4)
+TRANSFORM_ROW_IMPL(fadst4x16, 4, 16)
+TRANSFORM_ROW_IMPL(fadst8x4, 8, 4)
TRANSFORM_ROW(fadst8x8, 8, 8)
TRANSFORM_ROW(fadst8x16, 8, 16)
TRANSFORM_ROW(fdct4x4, 4, 4)
-TRANSFORM_ROW(fdct4x16, 4, 16)
-TRANSFORM_ROW(fdct8x4, 8, 4)
+TRANSFORM_ROW_IMPL(fdct4x16, 4, 16)
+TRANSFORM_ROW_IMPL(fdct8x4, 8, 4)
TRANSFORM_ROW(fdct8x8, 8, 8)
TRANSFORM_ROW(fdct8x16, 8, 16)
TRANSFORM_ROW(fdct8x32, 8, 32)
TRANSFORM_ROW(fidentity4x4, 4, 4)
-TRANSFORM_ROW(fidentity4x16, 4, 16)
-TRANSFORM_ROW(fidentity8x4, 8, 4)
+TRANSFORM_ROW_IMPL(fidentity4x16, 4, 16)
+TRANSFORM_ROW_IMPL(fidentity8x4, 8, 4)
TRANSFORM_ROW(fidentity8x8, 8, 8)
TRANSFORM_ROW(fidentity8x16, 8, 16)
TRANSFORM_ROW(fidentity8x32, 8, 32)
-TRANSFORM_ROW_RECT(fadst4x8, 4, 8)
-TRANSFORM_ROW_RECT(fadst8x4, 8, 4)
-TRANSFORM_ROW_RECT(fadst8x8, 8, 8)
+TRANSFORM_ROW_RECT_IMPL(fadst4x8, 4, 8)
+TRANSFORM_ROW_RECT_IMPL(fadst8x4, 8, 4)
+TRANSFORM_ROW_RECT_IMPL(fadst8x8, 8, 8)
TRANSFORM_ROW_RECT(fadst8x16, 8, 16)
-TRANSFORM_ROW_RECT(fdct4x8, 4, 8)
-TRANSFORM_ROW_RECT(fdct8x4, 8, 4)
-TRANSFORM_ROW_RECT(fdct8x8, 8, 8)
+TRANSFORM_ROW_RECT_IMPL(fdct4x8, 4, 8)
+TRANSFORM_ROW_RECT_IMPL(fdct8x4, 8, 4)
+TRANSFORM_ROW_RECT_IMPL(fdct8x8, 8, 8)
TRANSFORM_ROW_RECT(fdct8x16, 8, 16)
TRANSFORM_ROW_RECT(fdct8x32, 8, 32)
-TRANSFORM_ROW_RECT(fidentity4x8, 4, 8)
-TRANSFORM_ROW_RECT(fidentity8x4, 8, 4)
-TRANSFORM_ROW_RECT(fidentity8x8, 8, 8)
+TRANSFORM_ROW_RECT_IMPL(fidentity4x8, 4, 8)
+TRANSFORM_ROW_RECT_IMPL(fidentity8x4, 8, 4)
+TRANSFORM_ROW_RECT_IMPL(fidentity8x8, 8, 8)
TRANSFORM_ROW_RECT(fidentity8x16, 8, 16)
TRANSFORM_ROW_RECT(fidentity8x32, 8, 32)
-typedef void (*transform_1d_lbd_4_neon)(const int16x4_t *input,
- int16x4_t *output, int cos_bit);
-typedef void (*transform_1d_lbd_8_neon)(const int16x8_t *input,
- int16x8_t *output, int cos_bit);
-
-typedef void (*col_transform_1d_lbd_4_neon)(const int16_t *input,
- int16x4_t *output, int stride,
- int cos_bit);
typedef void (*col_transform_1d_lbd_8_neon)(const int16_t *input,
int16x8_t *output, int stride,
int cos_bit);
-typedef void (*row_transform_1d_lbd_4_neon)(const int16x4_t *input,
- int32_t *output, int stride,
- int cos_bit);
typedef void (*row_transform_1d_lbd_8_neon)(const int16x8_t *input,
int32_t *output, int stride,
int cos_bit);
-static const col_transform_1d_lbd_4_neon col_txfm4x8_arr[TX_TYPES] = {
- fdct4x8_col_neon, // DCT_DCT
- fadst4x8_col_neon, // ADST_DCT
- fdct4x8_col_neon, // DCT_ADST
- fadst4x8_col_neon, // ADST_ADST
- fadst4x8_col_neon, // FLIPADST_DCT
- fdct4x8_col_neon, // DCT_FLIPADST
- fadst4x8_col_neon, // FLIPADST_FLIPADST
- fadst4x8_col_neon, // ADST_FLIPADST
- fadst4x8_col_neon, // FLIPADST_ADST
- fidentity4x8_col_neon, // IDTX
- fdct4x8_col_neon, // V_DCT
- fidentity4x8_col_neon, // H_DCT
- fadst4x8_col_neon, // V_ADST
- fidentity4x8_col_neon, // H_ADST
- fadst4x8_col_neon, // V_FLIPADST
- fidentity4x8_col_neon // H_FLIPADST
-};
-
-static const row_transform_1d_lbd_8_neon row_txfm8x4_arr[TX_TYPES] = {
- fdct8x4_row_neon, // DCT_DCT
- fdct8x4_row_neon, // ADST_DCT
- fadst8x4_row_neon, // DCT_ADST
- fadst8x4_row_neon, // ADST_ADST
- fdct8x4_row_neon, // FLIPADST_DCT
- fadst8x4_row_neon, // DCT_FLIPADST
- fadst8x4_row_neon, // FLIPADST_FLIPADST
- fadst8x4_row_neon, // ADST_FLIPADST
- fadst8x4_row_neon, // FLIPADST_ADST
- fidentity8x4_row_neon, // IDTX
- fidentity8x4_row_neon, // V_DCT
- fdct8x4_row_neon, // H_DCT
- fidentity8x4_row_neon, // V_ADST
- fadst8x4_row_neon, // H_ADST
- fidentity8x4_row_neon, // V_FLIPADST
- fadst8x4_row_neon // H_FLIPADST
-};
-
-static const row_transform_1d_lbd_8_neon row_rect_txfm8x4_arr[TX_TYPES] = {
- fdct8x4_row_rect_neon, // DCT_DCT
- fdct8x4_row_rect_neon, // ADST_DCT
- fadst8x4_row_rect_neon, // DCT_ADST
- fadst8x4_row_rect_neon, // ADST_ADST
- fdct8x4_row_rect_neon, // FLIPADST_DCT
- fadst8x4_row_rect_neon, // DCT_FLIPADST
- fadst8x4_row_rect_neon, // FLIPADST_FLIPADST
- fadst8x4_row_rect_neon, // ADST_FLIPADST
- fadst8x4_row_rect_neon, // FLIPADST_ADST
- fidentity8x4_row_rect_neon, // IDTX
- fidentity8x4_row_rect_neon, // V_DCT
- fdct8x4_row_rect_neon, // H_DCT
- fidentity8x4_row_rect_neon, // V_ADST
- fadst8x4_row_rect_neon, // H_ADST
- fidentity8x4_row_rect_neon, // V_FLIPADST
- fadst8x4_row_rect_neon // H_FLIPADST
-};
-
-static const col_transform_1d_lbd_8_neon col_txfm8x4_arr[TX_TYPES] = {
- fdct8x4_col_neon, // DCT_DCT
- fadst8x4_col_neon, // ADST_DCT
- fdct8x4_col_neon, // DCT_ADST
- fadst8x4_col_neon, // ADST_ADST
- fadst8x4_col_neon, // FLIPADST_DCT
- fdct8x4_col_neon, // DCT_FLIPADST
- fadst8x4_col_neon, // FLIPADST_FLIPADST
- fadst8x4_col_neon, // ADST_FLIPADST
- fadst8x4_col_neon, // FLIPADST_ADST
- fidentity8x4_col_neon, // IDTX
- fdct8x4_col_neon, // V_DCT
- fidentity8x4_col_neon, // H_DCT
- fadst8x4_col_neon, // V_ADST
- fidentity8x4_col_neon, // H_ADST
- fadst8x4_col_neon, // V_FLIPADST
- fidentity8x4_col_neon // H_FLIPADST
-};
-
-static const row_transform_1d_lbd_4_neon row_rect_txfm4x8_arr[TX_TYPES] = {
- fdct4x8_row_rect_neon, // DCT_DCT
- fdct4x8_row_rect_neon, // ADST_DCT
- fadst4x8_row_rect_neon, // DCT_ADST
- fadst4x8_row_rect_neon, // ADST_ADST
- fdct4x8_row_rect_neon, // FLIPADST_DCT
- fadst4x8_row_rect_neon, // DCT_FLIPADST
- fadst4x8_row_rect_neon, // FLIPADST_FLIPADST
- fadst4x8_row_rect_neon, // ADST_FLIPADST
- fadst4x8_row_rect_neon, // FLIPADST_ADST
- fidentity4x8_row_rect_neon, // IDTX
- fidentity4x8_row_rect_neon, // V_DCT
- fdct4x8_row_rect_neon, // H_DCT
- fidentity4x8_row_rect_neon, // V_ADST
- fadst4x8_row_rect_neon, // H_ADST
- fidentity4x8_row_rect_neon, // V_FLIPADST
- fadst4x8_row_rect_neon // H_FLIPADST
-};
-
-static const col_transform_1d_lbd_8_neon col_txfm8x8_arr[TX_TYPES] = {
- fdct8x8_col_neon, // DCT_DCT
- fadst8x8_col_neon, // ADST_DCT
- fdct8x8_col_neon, // DCT_ADST
- fadst8x8_col_neon, // ADST_ADST
- fadst8x8_col_neon, // FLIPADST_DCT
- fdct8x8_col_neon, // DCT_FLIPADST
- fadst8x8_col_neon, // FLIPADST_FLIPADST
- fadst8x8_col_neon, // ADST_FLIPADST
- fadst8x8_col_neon, // FLIPADST_ADST
- fidentity8x8_col_neon, // IDTX
- fdct8x8_col_neon, // V_DCT
- fidentity8x8_col_neon, // H_DCT
- fadst8x8_col_neon, // V_ADST
- fidentity8x8_col_neon, // H_ADST
- fadst8x8_col_neon, // V_FLIPADST
- fidentity8x8_col_neon, // H_FLIPADST
-};
-
static const row_transform_1d_lbd_8_neon row_txfm8x8_arr[TX_TYPES] = {
fdct8x8_row_neon, // DCT_DCT
fdct8x8_row_neon, // ADST_DCT
@@ -1731,63 +1649,6 @@ static const row_transform_1d_lbd_8_neon row_txfm8x8_arr[TX_TYPES] = {
fadst8x8_row_neon // H_FLIPADST
};
-static const row_transform_1d_lbd_8_neon row_rect_txfm8x8_arr[TX_TYPES] = {
- fdct8x8_row_rect_neon, // DCT_DCT
- fdct8x8_row_rect_neon, // ADST_DCT
- fadst8x8_row_rect_neon, // DCT_ADST
- fadst8x8_row_rect_neon, // ADST_ADST
- fdct8x8_row_rect_neon, // FLIPADST_DCT
- fadst8x8_row_rect_neon, // DCT_FLIPADST
- fadst8x8_row_rect_neon, // FLIPADST_FLIPADST
- fadst8x8_row_rect_neon, // ADST_FLIPADST
- fadst8x8_row_rect_neon, // FLIPADST_ADST
- fidentity8x8_row_rect_neon, // IDTX
- fidentity8x8_row_rect_neon, // V_DCT
- fdct8x8_row_rect_neon, // H_DCT
- fidentity8x8_row_rect_neon, // V_ADST
- fadst8x8_row_rect_neon, // H_ADST
- fidentity8x8_row_rect_neon, // V_FLIPADST
- fadst8x8_row_rect_neon // H_FLIPADST
-};
-
-static const col_transform_1d_lbd_4_neon col_txfm4x16_arr[TX_TYPES] = {
- fdct4x16_col_neon, // DCT_DCT
- fadst4x16_col_neon, // ADST_DCT
- fdct4x16_col_neon, // DCT_ADST
- fadst4x16_col_neon, // ADST_ADST
- fadst4x16_col_neon, // FLIPADST_DCT
- fdct4x16_col_neon, // DCT_FLIPADST
- fadst4x16_col_neon, // FLIPADST_FLIPADST
- fadst4x16_col_neon, // ADST_FLIPADST
- fadst4x16_col_neon, // FLIPADST_ADST
- fidentity4x16_col_neon, // IDTX
- fdct4x16_col_neon, // V_DCT
- fidentity4x16_col_neon, // H_DCT
- fadst4x16_col_neon, // V_ADST
- fidentity4x16_col_neon, // H_ADST
- fadst4x16_col_neon, // V_FLIPADST
- fidentity4x16_col_neon // H_FLIPADST
-};
-
-static const row_transform_1d_lbd_4_neon row_txfm4x16_arr[TX_TYPES] = {
- fdct4x16_row_neon, // DCT_DCT
- fdct4x16_row_neon, // ADST_DCT
- fadst4x16_row_neon, // DCT_ADST
- fadst4x16_row_neon, // ADST_ADST
- fdct4x16_row_neon, // FLIPADST_DCT
- fadst4x16_row_neon, // DCT_FLIPADST
- fadst4x16_row_neon, // FLIPADST_FLIPADST
- fadst4x16_row_neon, // ADST_FLIPADST
- fadst4x16_row_neon, // FLIPADST_ADST
- fidentity4x16_row_neon, // IDTX
- fidentity4x16_row_neon, // V_DCT
- fdct4x16_row_neon, // H_DCT
- fidentity4x16_row_neon, // V_ADST
- fadst4x16_row_neon, // H_ADST
- fidentity4x16_row_neon, // V_FLIPADST
- fadst4x16_row_neon // H_FLIPADST
-};
-
static const col_transform_1d_lbd_8_neon col_txfm8x16_arr[TX_TYPES] = {
fdct8x16_col_neon, // DCT_DCT
fadst8x16_col_neon, // ADST_DCT
@@ -2002,23 +1863,34 @@ static void lowbd_fwd_txfm2d_4x8_neon(const int16_t *input, int32_t *output,
int stride, TX_TYPE tx_type, int bd) {
(void)bd;
int16x4_t buf0[8];
- int16x8_t buf1[8];
- const col_transform_1d_lbd_4_neon col_txfm = col_txfm4x8_arr[tx_type];
- const row_transform_1d_lbd_8_neon row_txfm = row_rect_txfm8x4_arr[tx_type];
-
+ int16x8_t buf1[4];
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
ud_adjust_input_and_stride(ud_flip, &input, &stride, 8);
- col_txfm(input, buf0, stride, 13);
+
+ switch (vtx_tab[tx_type]) {
+ case DCT_1D: fdct4x8_col_impl_neon(input, buf0, stride, 13); break;
+ case ADST_1D:
+ case FLIPADST_1D: fadst4x8_col_impl_neon(input, buf0, stride, 13); break;
+ default:
+ assert(vtx_tab[tx_type] == IDTX_1D);
+ fidentity4x8_col_impl_neon(input, buf0, stride, 13);
+ break;
+ }
shift_right_1_round_s16_x4(buf0, buf0, 8);
transpose_arrays_s16_4x8(buf0, buf1);
if (lr_flip) {
- int16x8_t buf2[8];
- flip_buf_8_neon(buf1, buf2, 4);
- row_txfm(buf2, output, 8, 13);
- } else {
- row_txfm(buf1, output, 8, 13);
+ flip_buf_8_inplace_neon(buf1, 4);
+ }
+ switch (htx_tab[tx_type]) {
+ case DCT_1D: fdct8x4_row_rect_impl_neon(buf1, output, 8, 13); break;
+ case ADST_1D:
+ case FLIPADST_1D: fadst8x4_row_rect_impl_neon(buf1, output, 8, 13); break;
+ default:
+ assert(htx_tab[tx_type] == IDTX_1D);
+ fidentity8x4_row_rect_impl_neon(buf1, output, 8, 13);
+ break;
}
}
@@ -2026,26 +1898,51 @@ static void lowbd_fwd_txfm2d_4x16_neon(const int16_t *input, int32_t *output,
int stride, TX_TYPE tx_type, int bd) {
(void)bd;
int16x4_t buf0[16];
- int16x8_t buf1[16];
- const col_transform_1d_lbd_4_neon col_txfm = col_txfm4x16_arr[tx_type];
- const row_transform_1d_lbd_8_neon row_txfm = row_txfm8x4_arr[tx_type];
+ int16x8_t buf1[8];
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
ud_adjust_input_and_stride(ud_flip, &input, &stride, 16);
- col_txfm(input, buf0, stride, 13);
+ switch (vtx_tab[tx_type]) {
+ case DCT_1D: fdct4x16_col_impl_neon(input, buf0, stride, 13); break;
+ case ADST_1D:
+ case FLIPADST_1D: fadst4x16_col_impl_neon(input, buf0, stride, 13); break;
+ default:
+ assert(vtx_tab[tx_type] == IDTX_1D);
+ fidentity4x16_col_impl_neon(input, buf0, stride, 13);
+ break;
+ }
shift_right_1_round_s16_x4(buf0, buf0, 16);
transpose_arrays_s16_4x8(buf0, buf1);
- transpose_arrays_s16_4x8(buf0 + 8, buf1 + 8);
+ transpose_arrays_s16_4x8(buf0 + 8, buf1 + 4);
- for (int i = 0; i < 2; i++) {
- if (lr_flip) {
- int16x8_t buf2[16];
- flip_buf_8_neon(buf1 + 8 * i, buf2, 4);
- row_txfm(buf2, output + 8 * i, 16, 12);
- } else {
- int16x8_t *buf = buf1 + 8 * i;
- row_txfm(buf, output + 8 * i, 16, 12);
+ if (lr_flip) {
+ flip_buf_8_inplace_neon(buf1, 4);
+ flip_buf_8_inplace_neon(buf1 + 4, 4);
+ }
+ switch (htx_tab[tx_type]) {
+ case DCT_1D: {
+ int i = 0;
+ do {
+ fdct8x4_row_impl_neon(buf1 + 4 * i, output + 8 * i, 16, 12);
+ } while (++i != 2);
+ break;
+ }
+ case ADST_1D:
+ case FLIPADST_1D: {
+ int i = 0;
+ do {
+ fadst8x4_row_impl_neon(buf1 + 4 * i, output + 8 * i, 16, 12);
+ } while (++i != 2);
+ break;
+ }
+ default: {
+ assert(htx_tab[tx_type] == IDTX_1D);
+ int i = 0;
+ do {
+ fidentity8x4_row_impl_neon(buf1 + 4 * i, output + 8 * i, 16, 12);
+ } while (++i != 2);
+ break;
}
}
}
@@ -2053,136 +1950,68 @@ static void lowbd_fwd_txfm2d_4x16_neon(const int16_t *input, int32_t *output,
static void lowbd_fwd_txfm2d_8x4_neon(const int16_t *input, int32_t *output,
int stride, TX_TYPE tx_type, int bd) {
(void)bd;
- int16x8_t buf0[8];
+ int16x8_t buf0[4];
int16x4_t buf1[8];
- const col_transform_1d_lbd_8_neon col_txfm = col_txfm8x4_arr[tx_type];
- const row_transform_1d_lbd_4_neon row_txfm = row_rect_txfm4x8_arr[tx_type];
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
ud_adjust_input_and_stride(ud_flip, &input, &stride, 4);
- col_txfm(input, buf0, stride, 13);
+ switch (vtx_tab[tx_type]) {
+ case DCT_1D: fdct8x4_col_impl_neon(input, buf0, stride, 13); break;
+ case ADST_1D:
+ case FLIPADST_1D: fadst8x4_col_impl_neon(input, buf0, stride, 13); break;
+ default:
+ assert(vtx_tab[tx_type] == IDTX_1D);
+ fidentity8x4_col_impl_neon(input, buf0, stride, 13);
+ break;
+ }
shift_right_1_round_s16_x8(buf0, buf0, 4);
transpose_arrays_s16_8x4(buf0, buf1);
if (lr_flip) {
- int16x4_t buf2[8];
- flip_buf_4_neon(buf1, buf2, 8);
- row_txfm(buf2, output, 4, 13);
- } else {
- row_txfm(buf1, output, 4, 13);
+ flip_buf_4_inplace_neon(buf1, 8);
+ }
+ switch (htx_tab[tx_type]) {
+ case DCT_1D: fdct4x8_row_rect_impl_neon(buf1, output, 4, 13); break;
+ case ADST_1D:
+ case FLIPADST_1D: fadst4x8_row_rect_impl_neon(buf1, output, 4, 13); break;
+ default:
+ assert(htx_tab[tx_type] == IDTX_1D);
+ fidentity4x8_row_rect_impl_neon(buf1, output, 4, 13);
+ break;
}
}
static void lowbd_fwd_txfm2d_8x8_neon(const int16_t *input, int32_t *output,
int stride, TX_TYPE tx_type, int bd) {
(void)bd;
+ int16x8_t buf0[8], buf1[8];
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
ud_adjust_input_and_stride(ud_flip, &input, &stride, 8);
- int16x8_t buf0[8], buf1[8];
-
- switch (tx_type) {
- case DCT_DCT:
- fdct8x8_col_neon(input, buf0, stride, 13);
- shift_right_1_round_s16_x8(buf0, buf0, 8);
- transpose_arrays_s16_8x8(buf0, buf1);
- fdct8x8_row_neon(buf1, output, 8, 13);
- break;
- case ADST_DCT:
- fadst8x8_col_neon(input, buf0, stride, 13);
- shift_right_1_round_s16_x8(buf0, buf0, 8);
- transpose_arrays_s16_8x8(buf0, buf1);
- fdct8x8_row_neon(buf1, output, 8, 13);
- break;
- case DCT_ADST:
- fdct8x8_col_neon(input, buf0, stride, 13);
- shift_right_1_round_s16_x8(buf0, buf0, 8);
- transpose_arrays_s16_8x8(buf0, buf1);
- fadst8x8_row_neon(buf1, output, 8, 13);
- break;
- case ADST_ADST:
- fadst8x8_col_neon(input, buf0, stride, 13);
- shift_right_1_round_s16_x8(buf0, buf0, 8);
- transpose_arrays_s16_8x8(buf0, buf1);
- fadst8x8_row_neon(buf1, output, 8, 13);
- break;
- case FLIPADST_DCT:
- fadst8x8_col_neon(input, buf0, stride, 13);
- shift_right_1_round_s16_x8(buf0, buf0, 8);
- transpose_arrays_s16_8x8(buf0, buf1);
- fdct8x8_row_neon(buf1, output, 8, 13);
- break;
- case DCT_FLIPADST:
- fdct8x8_col_neon(input, buf0, stride, 13);
- shift_right_1_round_s16_x8(buf0, buf0, 8);
- transpose_arrays_s16_8x8(buf0, buf1);
- flip_buf_8_neon(buf1, buf0, 8);
- fadst8x8_row_neon(buf0, output, 8, 13);
- break;
- case FLIPADST_FLIPADST:
- fadst8x8_col_neon(input, buf0, stride, 13);
- shift_right_1_round_s16_x8(buf0, buf0, 8);
- transpose_arrays_s16_8x8(buf0, buf1);
- flip_buf_8_neon(buf1, buf0, 8);
- fadst8x8_row_neon(buf0, output, 8, 13);
- break;
- case ADST_FLIPADST:
- fadst8x8_col_neon(input, buf0, stride, 13);
- shift_right_1_round_s16_x8(buf0, buf0, 8);
- transpose_arrays_s16_8x8(buf0, buf1);
- flip_buf_8_neon(buf1, buf0, 8);
- fadst8x8_row_neon(buf0, output, 8, 13);
- break;
- case FLIPADST_ADST:
- fadst8x8_col_neon(input, buf0, stride, 13);
- shift_right_1_round_s16_x8(buf0, buf0, 8);
- transpose_arrays_s16_8x8(buf0, buf1);
- fadst8x8_row_neon(buf1, output, 8, 13);
- break;
- case IDTX:
- fidentity8x8_col_neon(input, buf0, stride, 13);
- shift_right_1_round_s16_x8(buf0, buf0, 8);
- transpose_arrays_s16_8x8(buf0, buf1);
- fidentity8x8_row_neon(buf1, output, 8, 13);
- break;
- case V_DCT:
- fdct8x8_col_neon(input, buf0, stride, 13);
- shift_right_1_round_s16_x8(buf0, buf0, 8);
- transpose_arrays_s16_8x8(buf0, buf1);
- fidentity8x8_row_neon(buf1, output, 8, 13);
- break;
- case H_DCT:
- fidentity8x8_col_neon(input, buf0, stride, 13);
- shift_right_1_round_s16_x8(buf0, buf0, 8);
- transpose_arrays_s16_8x8(buf0, buf1);
- fdct8x8_row_neon(buf1, output, 8, 13);
- break;
- case V_ADST:
- fadst8x8_col_neon(input, buf0, stride, 13);
- shift_right_1_round_s16_x8(buf0, buf0, 8);
- transpose_arrays_s16_8x8(buf0, buf1);
- fidentity8x8_row_neon(buf1, output, 8, 13);
+ switch (vtx_tab[tx_type]) {
+ case DCT_1D: fdct8x8_col_impl_neon(input, buf0, stride, 13); break;
+ case ADST_1D:
+ case FLIPADST_1D: fadst8x8_col_impl_neon(input, buf0, stride, 13); break;
+ default:
+ assert(vtx_tab[tx_type] == IDTX_1D);
+ fidentity8x8_col_impl_neon(input, buf0, stride, 13);
break;
- case H_ADST:
- fidentity8x8_col_neon(input, buf0, stride, 13);
- shift_right_1_round_s16_x8(buf0, buf0, 8);
- transpose_arrays_s16_8x8(buf0, buf1);
- fadst8x8_row_neon(buf1, output, 8, 13);
- break;
- case V_FLIPADST:
- fadst8x8_col_neon(input, buf0, stride, 13);
- shift_right_1_round_s16_x8(buf0, buf0, 8);
- transpose_arrays_s16_8x8(buf0, buf1);
- fidentity8x8_row_neon(buf1, output, 8, 13);
- break;
- case H_FLIPADST:
- fidentity8x8_col_neon(input, buf0, stride, 13);
- shift_right_1_round_s16_x8(buf0, buf0, 8);
- transpose_arrays_s16_8x8(buf0, buf1);
- flip_buf_8_neon(buf1, buf0, 8);
- fadst8x8_row_neon(buf0, output, 8, 13);
+ }
+ shift_right_1_round_s16_x8(buf0, buf0, 8);
+ transpose_arrays_s16_8x8(buf0, buf1);
+
+ if (lr_flip) {
+ flip_buf_8_inplace_neon(buf1, 8);
+ }
+ switch (htx_tab[tx_type]) {
+ case DCT_1D: fdct8x8_row_impl_neon(buf1, output, 8, 13); break;
+ case ADST_1D:
+ case FLIPADST_1D: fadst8x8_row_impl_neon(buf1, output, 8, 13); break;
+ default:
+ assert(htx_tab[tx_type] == IDTX_1D);
+ fidentity8x8_row_impl_neon(buf1, output, 8, 13);
break;
}
}
@@ -2191,24 +2020,50 @@ static void lowbd_fwd_txfm2d_8x16_neon(const int16_t *input, int32_t *output,
int stride, TX_TYPE tx_type, int bd) {
(void)bd;
int16x8_t buf0[16], buf1[16];
- const col_transform_1d_lbd_8_neon col_txfm = col_txfm8x16_arr[tx_type];
- const row_transform_1d_lbd_8_neon row_txfm = row_rect_txfm8x8_arr[tx_type];
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
ud_adjust_input_and_stride(ud_flip, &input, &stride, 16);
- col_txfm(input, buf0, stride, 13);
+ switch (vtx_tab[tx_type]) {
+ case DCT_1D: fdct8x16_col_impl_neon(input, buf0, stride, 13); break;
+ case ADST_1D:
+ case FLIPADST_1D: fadst8x16_col_impl_neon(input, buf0, stride, 13); break;
+ default:
+ assert(vtx_tab[tx_type] == IDTX_1D);
+ fidentity8x16_col_impl_neon(input, buf0, stride, 13);
+ break;
+ }
shift_right_2_round_s16_x8(buf0, buf0, 16);
transpose_arrays_s16_8x8(buf0, buf1);
transpose_arrays_s16_8x8(buf0 + 8, buf1 + 8);
- for (int i = 0; i < 2; i++) {
- if (lr_flip) {
- flip_buf_8_neon(buf1 + 8 * i, buf0, 8);
- row_txfm(buf0, output + 8 * i, 16, 13);
- } else {
- int16x8_t *buf = buf1 + 8 * i;
- row_txfm(buf, output + 8 * i, 16, 13);
+ if (lr_flip) {
+ flip_buf_8_inplace_neon(buf1, 8);
+ flip_buf_8_inplace_neon(buf1 + 8, 8);
+ }
+ switch (htx_tab[tx_type]) {
+ case DCT_1D: {
+ int i = 0;
+ do {
+ fdct8x8_row_rect_impl_neon(buf1 + 8 * i, output + 8 * i, 16, 13);
+ } while (++i != 2);
+ break;
+ }
+ case ADST_1D:
+ case FLIPADST_1D: {
+ int i = 0;
+ do {
+ fadst8x8_row_rect_impl_neon(buf1 + 8 * i, output + 8 * i, 16, 13);
+ } while (++i != 2);
+ break;
+ }
+ default: {
+ assert(htx_tab[tx_type] == IDTX_1D);
+ int i = 0;
+ do {
+ fidentity8x8_row_rect_impl_neon(buf1 + 8 * i, output + 8 * i, 16, 13);
+ } while (++i != 2);
+ break;
}
}
}
@@ -2244,50 +2099,109 @@ static void lowbd_fwd_txfm2d_8x32_neon(const int16_t *input, int32_t *output,
static void lowbd_fwd_txfm2d_16x4_neon(const int16_t *input, int32_t *output,
int stride, TX_TYPE tx_type, int bd) {
(void)bd;
- int16x8_t buf0[16];
+ int16x8_t buf0[4];
int16x4_t buf1[16];
- int16x4_t buf2[16];
- const col_transform_1d_lbd_8_neon col_txfm = col_txfm8x4_arr[tx_type];
- const row_transform_1d_lbd_4_neon row_txfm = row_txfm4x16_arr[tx_type];
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
ud_adjust_input_and_stride(ud_flip, &input, &stride, 4);
- for (int i = 0; i < 2; i++) {
- col_txfm(input + 8 * i, buf0, stride, 13);
- shift_right_1_round_s16_x8(buf0, buf0, 4);
- transpose_arrays_s16_8x4(buf0, buf1 + 8 * i);
+ switch (vtx_tab[tx_type]) {
+ case DCT_1D: {
+ int i = 0;
+ do {
+ fdct8x4_col_impl_neon(input + 8 * i, buf0, stride, 13);
+ shift_right_1_round_s16_x8(buf0, buf0, 4);
+ transpose_arrays_s16_8x4(buf0, buf1 + 8 * i);
+ } while (++i != 2);
+ break;
+ }
+ case ADST_1D:
+ case FLIPADST_1D: {
+ int i = 0;
+ do {
+ fadst8x4_col_impl_neon(input + 8 * i, buf0, stride, 13);
+ shift_right_1_round_s16_x8(buf0, buf0, 4);
+ transpose_arrays_s16_8x4(buf0, buf1 + 8 * i);
+ } while (++i != 2);
+ break;
+ }
+ default: {
+ assert(vtx_tab[tx_type] == IDTX_1D);
+ int i = 0;
+ do {
+ fidentity8x4_col_impl_neon(input + 8 * i, buf0, stride, 13);
+ shift_right_1_round_s16_x8(buf0, buf0, 4);
+ transpose_arrays_s16_8x4(buf0, buf1 + 8 * i);
+ } while (++i != 2);
+ break;
+ }
}
if (lr_flip) {
- flip_buf_4_neon(buf1, buf2, 16);
- row_txfm(buf2, output, 4, 13);
- } else {
- row_txfm(buf1, output, 4, 13);
+ flip_buf_4_inplace_neon(buf1, 16);
+ }
+ switch (htx_tab[tx_type]) {
+ case DCT_1D: fdct4x16_row_impl_neon(buf1, output, 4, 13); break;
+ case ADST_1D:
+ case FLIPADST_1D: fadst4x16_row_impl_neon(buf1, output, 4, 13); break;
+ default:
+ assert(htx_tab[tx_type] == IDTX_1D);
+ fidentity4x16_row_impl_neon(buf1, output, 4, 13);
+ break;
}
}
static void lowbd_fwd_txfm2d_16x8_neon(const int16_t *input, int32_t *output,
int stride, TX_TYPE tx_type, int bd) {
(void)bd;
- int16x8_t buf0[16], buf1[16];
- const col_transform_1d_lbd_8_neon col_txfm = col_txfm8x8_arr[tx_type];
- const row_transform_1d_lbd_8_neon row_txfm = row_rect_txfm8x16_arr[tx_type];
+ int16x8_t buf0[8], buf1[16];
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
ud_adjust_input_and_stride(ud_flip, &input, &stride, 8);
- for (int i = 0; i < 2; i++) {
- col_txfm(input + 8 * i, buf0, stride, 13);
- shift_right_2_round_s16_x8(buf0, buf0, 8);
- transpose_arrays_s16_8x8(buf0, buf1 + 8 * i);
+ switch (vtx_tab[tx_type]) {
+ case DCT_1D: {
+ int i = 0;
+ do {
+ fdct8x8_col_impl_neon(input + 8 * i, buf0, stride, 13);
+ shift_right_2_round_s16_x8(buf0, buf0, 8);
+ transpose_arrays_s16_8x8(buf0, buf1 + 8 * i);
+ } while (++i != 2);
+ break;
+ }
+ case ADST_1D:
+ case FLIPADST_1D: {
+ int i = 0;
+ do {
+ fadst8x8_col_impl_neon(input + 8 * i, buf0, stride, 13);
+ shift_right_2_round_s16_x8(buf0, buf0, 8);
+ transpose_arrays_s16_8x8(buf0, buf1 + 8 * i);
+ } while (++i != 2);
+ break;
+ }
+ default: {
+ assert(vtx_tab[tx_type] == IDTX_1D);
+ int i = 0;
+ do {
+ fidentity8x8_col_impl_neon(input + 8 * i, buf0, stride, 13);
+ shift_right_2_round_s16_x8(buf0, buf0, 8);
+ transpose_arrays_s16_8x8(buf0, buf1 + 8 * i);
+ } while (++i != 2);
+ break;
+ }
}
if (lr_flip) {
- flip_buf_8_neon(buf1, buf0, 16);
- row_txfm(buf0, output, 8, 13);
- } else {
- row_txfm(buf1, output, 8, 13);
+ flip_buf_8_inplace_neon(buf1, 16);
+ }
+ switch (htx_tab[tx_type]) {
+ case DCT_1D: fdct8x16_row_rect_impl_neon(buf1, output, 8, 13); break;
+ case ADST_1D:
+ case FLIPADST_1D: fadst8x16_row_rect_impl_neon(buf1, output, 8, 13); break;
+ default:
+ assert(htx_tab[tx_type] == IDTX_1D);
+ fidentity8x16_row_rect_impl_neon(buf1, output, 8, 13);
+ break;
}
}
@@ -2357,30 +2271,39 @@ static void lowbd_fwd_txfm2d_16x32_neon(const int16_t *input, int32_t *output,
static void lowbd_fwd_txfm2d_32x8_neon(const int16_t *input, int32_t *output,
int stride, TX_TYPE tx_type, int bd) {
(void)bd;
- int16x8_t buf0[32], buf1[32];
- const col_transform_1d_lbd_8_neon col_txfm = col_txfm8x8_arr[tx_type];
+ int16x8_t buf0[8], buf1[32];
const row_transform_1d_lbd_8_neon row_txfm = row_txfm8x32_arr[tx_type];
-
- if (col_txfm == NULL || row_txfm == NULL) {
- av1_fwd_txfm2d_32x16_c(input, output, stride, tx_type, bd);
- return;
- }
+ assert(row_txfm != NULL);
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
ud_adjust_input_and_stride(ud_flip, &input, &stride, 8);
- for (int i = 0; i < 4; i++) {
- col_txfm(input + 8 * i, buf0, stride, 13);
- shift_right_2_round_s16_x8(buf0, buf0, 8);
- transpose_arrays_s16_8x8(buf0, buf1 + 0 * 32 + 8 * i);
+ switch (vtx_tab[tx_type]) {
+ case DCT_1D: {
+ int i = 0;
+ do {
+ fdct8x8_col_impl_neon(input + 8 * i, buf0, stride, 13);
+ shift_right_2_round_s16_x8(buf0, buf0, 8);
+ transpose_arrays_s16_8x8(buf0, buf1 + 8 * i);
+ } while (++i != 4);
+ break;
+ }
+ default: {
+ assert(vtx_tab[tx_type] == IDTX_1D);
+ int i = 0;
+ do {
+ fidentity8x8_col_impl_neon(input + 8 * i, buf0, stride, 13);
+ shift_right_2_round_s16_x8(buf0, buf0, 8);
+ transpose_arrays_s16_8x8(buf0, buf1 + 8 * i);
+ } while (++i != 4);
+ break;
+ }
}
if (lr_flip) {
- flip_buf_8_neon(buf1, buf0, 32);
- row_txfm(buf0, output, 8, 12);
- } else {
- row_txfm(buf1, output, 8, 12);
+ flip_buf_8_inplace_neon(buf1, 32);
}
+ row_txfm(buf1, output, 8, 12);
}
static void lowbd_fwd_txfm2d_32x16_neon(const int16_t *input, int32_t *output,
@@ -2456,14 +2379,12 @@ static void lowbd_fwd_txfm2d_64x16_neon(const int16_t *input, int32_t *output,
(void)bd;
(void)tx_type;
assert(tx_type == DCT_DCT);
- int16x8_t buf0[64], buf1[128];
- const transform_1d_lbd_8_neon col_txfm = fdct8x16_neon;
- const transform_1d_lbd_8_neon row_txfm = fdct8x64_neon;
+ int16x8_t buf0[16], buf1[128];
for (int i = 0; i < 8; i++) {
load_buffer_s16_x8(input + 8 * i, stride, buf0, 16);
shift_left_2_s16_x8(buf0, buf0, 16);
- col_txfm(buf0, buf0, 13);
+ fdct8x16_neon(buf0, buf0, 13);
shift_right_4_round_s16_x8(buf0, buf0, 16);
for (int j = 0; j < 2; ++j) {
transpose_arrays_s16_8x8(buf0 + j * 8, buf1 + j * 64 + 8 * i);
@@ -2472,7 +2393,7 @@ static void lowbd_fwd_txfm2d_64x16_neon(const int16_t *input, int32_t *output,
for (int i = 0; i < 2; i++) {
int16x8_t *buf = buf1 + 64 * i;
- row_txfm(buf, buf, 12);
+ fdct8x64_neon(buf, buf, 12);
store_buffer_s16_x8(buf, output + 8 * i, 16, 32);
}
// Zero out the bottom 16x32 area.
@@ -2484,22 +2405,20 @@ static void lowbd_fwd_txfm2d_16x64_neon(const int16_t *input, int32_t *output,
(void)bd;
(void)tx_type;
assert(tx_type == DCT_DCT);
- int16x8_t buf0[64], buf1[128];
- const transform_1d_lbd_8_neon col_txfm = fdct8x64_neon;
- const transform_1d_lbd_8_neon row_txfm = fdct8x16_neon;
+ int16x8_t buf0[64], buf1[64];
for (int i = 0; i < 2; i++) {
load_buffer_s16_x8(input + 8 * i, stride, buf0, 64);
- col_txfm(buf0, buf0, 13);
- shift_right_2_round_s16_x8(buf0, buf0, 64);
- for (int j = 0; j < 8; ++j) {
+ fdct8x64_neon(buf0, buf0, 13);
+ shift_right_2_round_s16_x8(buf0, buf0, 32);
+ for (int j = 0; j < 4; ++j) {
transpose_arrays_s16_8x8(buf0 + j * 8, buf1 + j * 16 + 8 * i);
}
}
for (int i = 0; i < 4; i++) {
int16x8_t *buf = buf1 + 16 * i;
- row_txfm(buf, buf, 12);
+ fdct8x16_neon(buf, buf, 12);
store_buffer_s16_x8(buf, output + 8 * i, 32, 16);
}
}
@@ -2962,13 +2881,12 @@ static void lowbd_fwd_txfm2d_64x64_neon(const int16_t *input, int32_t *output,
(void)bd;
(void)tx_type;
assert(tx_type == DCT_DCT);
- int16x8_t buf0[64], buf1[512];
- const transform_1d_lbd_8_neon col_txfm = fdct8x64_neon;
+ int16x8_t buf0[64], buf1[256];
for (int i = 0; i < 8; i++) {
load_buffer_s16_x8(input + 8 * i, stride, buf0, 64);
- col_txfm(buf0, buf0, 13);
- shift_right_2_round_s16_x8(buf0, buf0, 64);
+ fdct8x64_neon(buf0, buf0, 13);
+ shift_right_2_round_s16_x8(buf0, buf0, 32);
for (int j = 0; j < 4; ++j) {
transpose_arrays_s16_8x8(buf0 + j * 8, buf1 + j * 64 + 8 * i);
}
@@ -2992,17 +2910,17 @@ static void lowbd_fwd_txfm2d_64x64_neon(const int16_t *input, int32_t *output,
static void lowbd_fwd_txfm2d_64x32_neon(const int16_t *input, int32_t *output,
int stride, TX_TYPE tx_type, int bd) {
(void)bd;
- int16x8_t buf0[64], buf1[256];
- const col_transform_1d_lbd_8_neon col_txfm = col_txfm8x32_arr[tx_type];
+ (void)tx_type;
+ assert(tx_type == DCT_DCT);
+ int16x8_t buf0[32], buf1[256];
for (int i = 0; i < 8; i++) {
- col_txfm(input + 8 * i, buf0, stride, 12);
+ fdct8x32_col_neon(input + 8 * i, buf0, stride, 12);
shift_right_4_round_s16_x8(buf0, buf0, 32);
for (int j = 0; j < 4; ++j) {
transpose_arrays_s16_8x8(buf0 + j * 8, buf1 + j * 64 + 8 * i);
}
}
- assert(tx_type == DCT_DCT);
for (int i = 0; i < 4; i++) {
int32x4_t bufA[64];
int32x4_t bufB[64];
@@ -3026,13 +2944,12 @@ static void lowbd_fwd_txfm2d_32x64_neon(const int16_t *input, int32_t *output,
(void)bd;
(void)tx_type;
assert(tx_type == DCT_DCT);
- int16x8_t buf0[64], buf1[256];
- const transform_1d_lbd_8_neon col_txfm = fdct8x64_neon;
+ int16x8_t buf0[64], buf1[128];
for (int i = 0; i < 4; i++) {
load_buffer_s16_x8(input + 8 * i, stride, buf0, 64);
- col_txfm(buf0, buf0, 13);
- shift_right_2_round_s16_x8(buf0, buf0, 64);
+ fdct8x64_neon(buf0, buf0, 13);
+ shift_right_2_round_s16_x8(buf0, buf0, 32);
for (int j = 0; j < 4; ++j) {
transpose_arrays_s16_8x8(buf0 + j * 8, buf1 + j * 32 + 8 * i);
}