Commit 6db771baca for aom
commit 6db771baca2db5522076d8a20b1e6171cb09dd2d
Author: Frank Galligan <fgalligan@google.com>
Date: Tue Nov 17 06:28:32 2026 -0800
av1,neon: Optimize lowbd inv_txfm 4x4 & 8x8
Vectorize 4x4 lowbd inverse transforms (IDCT, IADST, identity) in NEON
over int16x4_t lanes with direct 4x4 residual add and a peeled DCT_DCT
path, eliminating the scalar 4x4 path and unused lowbd_txfm_all_1d_arr
table. Add compact TX_8X8 DCT and non-DCT NEON paths with force-inlined
1D helpers (zero vector spills), specialized identity shifts, and
eob == 1 low1 dispatch (skipping unused a[1..7] zeroing for non-id
horiz and vertical apply_txfm8_low1 when vert is IIDENTITY_1D), while
routing TX_8X8 DCT eob == 1 to DC-only. Hoist rectangular
4x8/8x4/4x16/16x4 DCT eob == 1 DC-only paths to the dispatcher and keep
non-4x4 arms AOM_NOINLINE to prevent 4x4 spills. Remove dead
transform_1d_neon and av1_eob_to_eobxy_8x8_default header decls and
unused av1_inv_txfm1d.h includes.
Speedup (Pixel 9 Tensor G4 Cortex-A720, 1T, tier1 ABBA, bit-exact):
-8.91% paired median encode time (-10.95% speed 6, -6.00% speed 7).
NEON/AV1LbdInvTxfm2d.DISABLED_Speed/0 (eob=1; skips TX_4X4; C/NEON):
8x8 DCT 92.25x (2128 ns), ADST/FLIPADST 28.97x..38.42x (5513..7638 ns),
IDTX 77.35x (1156 ns), H_DCT 96.76x (1511 ns), H_ADST/H_FLIPADST
55.51x..56.15x (2753..2817 ns), V_* 31.45x..48.34x (3037..4962 ns);
4x8/8x4/4x16/16x4 DCT 80.56x/112.17x/90.83x/177.77x.
Standalone Pixel 9 A720 TX_4X4 NEON microbench (eob=16 / eob=4):
DCT_DCT 16.22 / 16.23 ns (eob=1 4.19 ns), ADST_ADST 21.57 / 21.50 ns,
IDTX 14.25 / 14.25 ns, 50/25/25 DCT/ADST/IDTX mix 17.23 / 17.23 ns
(-8.92% / -8.79% with DCT_DCT peel), 16-type mix 18.44 / 18.47 ns
(-1.72% / -1.57%). Standalone TX_8X8 eob=1: IDTX 6.53 ns (-48.4% via
vert identity fast exit), H_DCT 8.29 ns (-53.2%).
TAG=agy
CONV=bc5228f2-7fa6-4522-aad6-34935d739e28
Change-Id: Ic9a5cabb81558144bbfb6fca01ea7487a8051b8e
diff --git a/av1/common/arm/av1_inv_txfm_neon.c b/av1/common/arm/av1_inv_txfm_neon.c
index 7b188a9aa7..2ad8c6ec62 100644
--- a/av1/common/arm/av1_inv_txfm_neon.c
+++ b/av1/common/arm/av1_inv_txfm_neon.c
@@ -18,9 +18,9 @@
#include "config/aom_dsp_rtcd.h"
#include "config/av1_rtcd.h"
+#include "aom_dsp/aom_dsp_common.h"
#include "aom_dsp/arm/mem_neon.h"
#include "aom_dsp/arm/transpose_neon.h"
-#include "av1/common/av1_inv_txfm1d.h"
#include "av1/common/av1_inv_txfm1d_cfg.h"
#include "av1/common/av1_inv_txfm2d.h"
#include "av1/common/av1_txfm.h"
@@ -50,15 +50,6 @@ static const ITX_TYPE_1D hitx_1d_tab[TX_TYPES] = {
IIDENTITY_1D, IADST_1D, IIDENTITY_1D, IFLIPADST_1D,
};
-// 1D functions
-static const transform_1d_neon lowbd_txfm_all_1d_arr[TX_SIZES][ITX_TYPES_1D] = {
- { av1_idct4, av1_iadst4, av1_iidentity4_c },
- { av1_idct8, av1_iadst8, av1_iidentity8_c },
- { av1_idct16, av1_iadst16, av1_iidentity16_c },
- { av1_idct32, NULL, NULL },
- { av1_idct64, NULL, NULL },
-};
-
static inline void lowbd_add_flip_buffer_8xn_neon(int16x8_t *in,
uint8_t *output, int stride,
int flipud,
@@ -261,8 +252,8 @@ static inline int16x4_t set_s16x4_neon(const int16_t c0, const int16_t c1,
return val;
}
-static inline void iadst8_neon(int16x8_t *const in, int16x8_t *out,
- int8_t cos_bit) {
+static AOM_FORCE_INLINE void iadst8_neon_impl(int16x8_t *const in,
+ int16x8_t *out, int8_t cos_bit) {
const int32_t *cospi = cospi_arr(cos_bit);
const int16x4_t c0 = set_s16x4_neon((int16_t)cospi[4], (int16_t)cospi[60],
@@ -334,8 +325,14 @@ static inline void iadst8_neon(int16x8_t *const in, int16x8_t *out,
out[7] = vqnegq_s16(x[1]);
}
-static inline void iadst8_low1_neon(int16x8_t *const in, int16x8_t *out,
- int8_t cos_bit) {
+static inline void iadst8_neon(int16x8_t *const in, int16x8_t *out,
+ int8_t cos_bit) {
+ iadst8_neon_impl(in, out, cos_bit);
+}
+
+static AOM_FORCE_INLINE void iadst8_low1_neon_impl(int16x8_t *const in,
+ int16x8_t *out,
+ int8_t cos_bit) {
const int32_t *cospi = cospi_arr(cos_bit);
const int16x4_t c2 = set_s16x4_neon((int16_t)cospi[32], (int16_t)cospi[32],
(int16_t)cospi[16], (int16_t)cospi[48]);
@@ -386,7 +383,13 @@ static inline void iadst8_low1_neon(int16x8_t *const in, int16x8_t *out,
out[7] = vqnegq_s16(x[1]);
}
-static inline void idct8_neon(int16x8_t *in, int16x8_t *out, int8_t cos_bit) {
+static inline void iadst8_low1_neon(int16x8_t *const in, int16x8_t *out,
+ int8_t cos_bit) {
+ iadst8_low1_neon_impl(in, out, cos_bit);
+}
+
+static AOM_FORCE_INLINE void idct8_neon_impl(int16x8_t *in, int16x8_t *out,
+ int8_t cos_bit) {
const int32_t *cospi = cospi_arr(cos_bit);
int16x8_t step1[8], step2[8];
const int16x4_t c0 = set_s16x4_neon((int16_t)cospi[8], (int16_t)cospi[56],
@@ -424,8 +427,12 @@ static inline void idct8_neon(int16x8_t *in, int16x8_t *out, int8_t cos_bit) {
out[7] = vqsubq_s16(step1[0], step2[7]);
}
-static inline void idct8_low1_neon(int16x8_t *in, int16x8_t *out,
- int8_t cos_bit) {
+static inline void idct8_neon(int16x8_t *in, int16x8_t *out, int8_t cos_bit) {
+ idct8_neon_impl(in, out, cos_bit);
+}
+
+static AOM_FORCE_INLINE void idct8_low1_neon_impl(int16x8_t *in, int16x8_t *out,
+ int8_t cos_bit) {
const int32_t *cospi = cospi_arr(cos_bit);
int16x8_t step1;
int32x4_t t32[2];
@@ -451,6 +458,11 @@ static inline void idct8_low1_neon(int16x8_t *in, int16x8_t *out,
out[7] = step1;
}
+static inline void idct8_low1_neon(int16x8_t *in, int16x8_t *out,
+ int8_t cos_bit) {
+ idct8_low1_neon_impl(in, out, cos_bit);
+}
+
static void round_shift_array_16_neon(int16x8_t *arr, int size, int bit) {
assert(!(size % 4));
if (!bit) return;
@@ -3753,70 +3765,6 @@ static inline void lowbd_inv_txfm2d_add_h_identity_neon(
}
}
-static inline void lowbd_inv_txfm2d_add_4x4_neon(const int32_t *input,
- uint8_t *output, int stride,
- TX_TYPE tx_type, int eob) {
- (void)eob;
- TX_SIZE tx_size = TX_4X4;
- DECLARE_ALIGNED(32, int, txfm_buf[4 * 4 + 8 + 8]);
- int32_t *temp_in = txfm_buf;
-
- const int8_t *shift = av1_inv_txfm_shift_ls[tx_size];
- const int txw_idx = get_txw_idx(tx_size);
- const int txh_idx = get_txh_idx(tx_size);
- const int txfm_size_col = tx_size_wide[tx_size];
- const int txfm_size_row = tx_size_high[tx_size];
- const int buf_offset = AOMMAX(txfm_size_row, txfm_size_col);
- int32_t *temp_out = temp_in + buf_offset;
- int32_t *buf = temp_out + buf_offset;
- int32_t *buf_ptr = buf;
- const int8_t stage_range[MAX_TXFM_STAGE_NUM] = { 16, 16, 16, 16, 16, 16, 16 };
- int r;
- const transform_1d_neon row_txfm =
- lowbd_txfm_all_1d_arr[txw_idx][hitx_1d_tab[tx_type]];
- const transform_1d_neon col_txfm =
- lowbd_txfm_all_1d_arr[txh_idx][vitx_1d_tab[tx_type]];
-
- int ud_flip, lr_flip;
- get_flip_cfg(tx_type, &ud_flip, &lr_flip);
-
- for (int i = 0; i < txfm_size_row; i++) {
- for (int c = 0; c < txfm_size_col; ++c)
- temp_in[c] = input[c * txfm_size_row];
- row_txfm(temp_in, buf_ptr, INV_COS_BIT, stage_range);
-
- input++;
- buf_ptr += txfm_size_col;
- }
-
- for (int c = 0; c < txfm_size_col; ++c) {
- if (lr_flip == 0) {
- for (r = 0; r < txfm_size_row; ++r)
- temp_in[r] = buf[r * txfm_size_col + c];
- } else {
- // flip left right
- for (r = 0; r < txfm_size_row; ++r)
- temp_in[r] = buf[r * txfm_size_col + (txfm_size_col - c - 1)];
- }
- clamp_buf(temp_in, txfm_size_row, 16);
- col_txfm(temp_in, temp_out, INV_COS_BIT, stage_range);
- av1_round_shift_array(temp_out, txfm_size_row, -shift[1]);
-
- if (ud_flip == 0) {
- for (r = 0; r < txfm_size_row; ++r) {
- output[r * stride + c] =
- clip_pixel(output[r * stride + c] + temp_out[r]);
- }
- } else {
- // flip upside down
- for (r = 0; r < txfm_size_row; ++r) {
- output[r * stride + c] = clip_pixel(output[r * stride + c] +
- temp_out[txfm_size_row - r - 1]);
- }
- }
- }
-}
-
static inline void lowbd_add_flip_buffer_4xn_neon(const int16x8_t *in,
uint8_t *output, int stride,
int flipud, int height) {
@@ -3977,19 +3925,226 @@ static AOM_FORCE_INLINE void apply_txfm4_s16x8_neon(int16x8_t *a,
}
}
+static AOM_FORCE_INLINE int16x4_t narrow_inv_txfm4_s16x4_neon(int32x4_t v,
+ int out_rshift) {
+ if (out_rshift == 4) {
+ return vqrshrn_n_s32(v, 4);
+ }
+ assert(out_rshift == 0);
+ return vqmovn_s32(v);
+}
+
+static AOM_FORCE_INLINE void idct4_s16x4_neon(const int16x4_t *in,
+ int16x4_t *out, int out_rshift) {
+ const int32_t *cospi = cospi_arr(INV_COS_BIT);
+ const int16_t cospi32 = (int16_t)cospi[32];
+ const int32x4_t p = vmull_n_s16(in[0], cospi32);
+
+ const int32x4_t s0 =
+ vrshrq_n_s32(vmlal_n_s16(p, in[2], cospi32), INV_COS_BIT);
+ const int32x4_t s1 =
+ vrshrq_n_s32(vmlsl_n_s16(p, in[2], cospi32), INV_COS_BIT);
+ const int32x4_t s2 =
+ vrshrq_n_s32(vmlsl_n_s16(vmull_n_s16(in[1], (int16_t)cospi[48]), in[3],
+ (int16_t)cospi[16]),
+ INV_COS_BIT);
+ const int32x4_t s3 =
+ vrshrq_n_s32(vmlal_n_s16(vmull_n_s16(in[1], (int16_t)cospi[16]), in[3],
+ (int16_t)cospi[48]),
+ INV_COS_BIT);
+
+ out[0] = narrow_inv_txfm4_s16x4_neon(vaddq_s32(s0, s3), out_rshift);
+ out[1] = narrow_inv_txfm4_s16x4_neon(vaddq_s32(s1, s2), out_rshift);
+ out[2] = narrow_inv_txfm4_s16x4_neon(vsubq_s32(s1, s2), out_rshift);
+ out[3] = narrow_inv_txfm4_s16x4_neon(vsubq_s32(s0, s3), out_rshift);
+}
+
+static AOM_FORCE_INLINE void iadst4_s16x4_neon(const int16x4_t *in,
+ int16x4_t *out, int out_rshift) {
+ const int32_t *sinpi = sinpi_arr(INV_COS_BIT);
+ const int16_t sin1 = (int16_t)sinpi[1];
+ const int16_t sin2 = (int16_t)sinpi[2];
+ const int16_t sin3 = (int16_t)sinpi[3];
+ const int16_t sin4 = (int16_t)sinpi[4];
+
+ int32x4_t s0 = vmull_n_s16(in[0], sin1);
+ s0 = vmlal_n_s16(s0, in[2], sin4);
+ s0 = vmlal_n_s16(s0, in[3], sin2);
+
+ int32x4_t s1 = vmull_n_s16(in[0], sin2);
+ s1 = vmlsl_n_s16(s1, in[2], sin1);
+ s1 = vmlsl_n_s16(s1, in[3], sin4);
+
+ const int32x4_t s3 = vmull_n_s16(in[1], sin3);
+ const int32x4_t s7 = vaddw_s16(vsubl_s16(in[0], in[2]), in[3]);
+ const int32x4_t s2 = vmulq_n_s32(s7, sinpi[3]);
+
+ const int32x4_t y0 = vrshrq_n_s32(vaddq_s32(s0, s3), INV_COS_BIT);
+ const int32x4_t y1 = vrshrq_n_s32(vaddq_s32(s1, s3), INV_COS_BIT);
+ const int32x4_t y2 = vrshrq_n_s32(s2, INV_COS_BIT);
+ const int32x4_t y3 =
+ vrshrq_n_s32(vsubq_s32(vaddq_s32(s0, s1), s3), INV_COS_BIT);
+
+ out[0] = narrow_inv_txfm4_s16x4_neon(y0, out_rshift);
+ out[1] = narrow_inv_txfm4_s16x4_neon(y1, out_rshift);
+ out[2] = narrow_inv_txfm4_s16x4_neon(y2, out_rshift);
+ out[3] = narrow_inv_txfm4_s16x4_neon(y3, out_rshift);
+}
+
+static AOM_FORCE_INLINE void identity_txfm4_s16x4_neon(const int16x4_t *in,
+ int16x4_t *out,
+ int out_rshift) {
+ const int16_t scale = (int16_t)NewSqrt2;
+ int i = 0;
+ if (out_rshift == 4) {
+ do {
+ out[i] = vqrshrn_n_s32(vrshrq_n_s32(vmull_n_s16(in[i], scale), 12), 4);
+ } while (++i != 4);
+ } else {
+ assert(out_rshift == 0);
+ do {
+ out[i] = vqrshrn_n_s32(vmull_n_s16(in[i], scale), 12);
+ } while (++i != 4);
+ }
+}
+
+static AOM_FORCE_INLINE void apply_txfm4_s16x4_neon(int16x4_t *a,
+ ITX_TYPE_1D type,
+ int out_rshift) {
+ switch (type) {
+ case IDCT_1D: idct4_s16x4_neon(a, a, out_rshift); break;
+ case IADST_1D: iadst4_s16x4_neon(a, a, out_rshift); break;
+ default:
+ assert(type == IIDENTITY_1D);
+ identity_txfm4_s16x4_neon(a, a, out_rshift);
+ break;
+ }
+}
+
+static AOM_FORCE_INLINE void flip_buf_4_neon(int16x4_t *a) {
+ const int16x4_t t0 = a[0];
+ const int16x4_t t1 = a[1];
+ a[0] = a[3];
+ a[1] = a[2];
+ a[2] = t1;
+ a[3] = t0;
+}
+
+static AOM_FORCE_INLINE void lowbd_add_buffer_4x4_neon(const int16x4_t *in,
+ uint8_t *output,
+ int stride) {
+ const uint8x8_t pred01 = load_u8_4x2(output, stride);
+ const int16x8_t res01 = vcombine_s16(in[0], in[1]);
+ const int16x8_t sum01 =
+ vaddq_s16(vreinterpretq_s16_u16(vmovl_u8(pred01)), res01);
+ store_u8x4_strided_x2(output, stride, vqmovun_s16(sum01));
+
+ output += 2 * stride;
+ const uint8x8_t pred23 = load_u8_4x2(output, stride);
+ const int16x8_t res23 = vcombine_s16(in[2], in[3]);
+ const int16x8_t sum23 =
+ vaddq_s16(vreinterpretq_s16_u16(vmovl_u8(pred23)), res23);
+ store_u8x4_strided_x2(output, stride, vqmovun_s16(sum23));
+}
+
+static inline void lowbd_inv_txfm2d_add_4x4_neon(const int32_t *input,
+ uint8_t *output, int stride,
+ TX_TYPE tx_type, int eob) {
+ if (eob == 1 && tx_type == DCT_DCT) {
+ const int32_t *cospi = cospi_arr(INV_COS_BIT);
+ int32_t dc = round_shift((int64_t)input[0] * cospi[32], INV_COS_BIT);
+ dc = round_shift((int64_t)dc * cospi[32], INV_COS_BIT);
+ dc = round_shift((int64_t)dc, 4);
+ lowbd_inv_txfm_dc_only_4xn_neon(output, stride, dc, 4);
+ return;
+ }
+
+ int16x4_t a[4];
+ int i = 0;
+ do {
+ a[i] = vqmovn_s32(vld1q_s32(input + i * 4));
+ } while (++i != 4);
+
+ // Avoid get_flip_cfg and 1D transform switch dispatch for common 4x4 DCT.
+ if (tx_type == DCT_DCT) {
+ idct4_s16x4_neon(a, a, 0);
+ transpose_elems_inplace_s16_4x4(&a[0], &a[1], &a[2], &a[3]);
+ idct4_s16x4_neon(a, a, 4);
+ lowbd_add_buffer_4x4_neon(a, output, stride);
+ return;
+ }
+
+ int ud_flip, lr_flip;
+ get_flip_cfg(tx_type, &ud_flip, &lr_flip);
+
+ apply_txfm4_s16x4_neon(a, hitx_1d_tab[tx_type], 0);
+ if (lr_flip) {
+ flip_buf_4_neon(a);
+ }
+ transpose_elems_inplace_s16_4x4(&a[0], &a[1], &a[2], &a[3]);
+ apply_txfm4_s16x4_neon(a, vitx_1d_tab[tx_type], 4);
+ if (ud_flip) {
+ flip_buf_4_neon(a);
+ }
+ lowbd_add_buffer_4x4_neon(a, output, stride);
+}
+
+static AOM_FORCE_INLINE void identity_txfm8_s16x8_neon(int16x8_t *b,
+ int out_rshift) {
+ if (out_rshift == 4) {
+ int i = 0;
+ do {
+ b[i] = vrshrq_n_s16(b[i], 3);
+ } while (++i != 8);
+ } else if (out_rshift == 0) {
+ int i = 0;
+ do {
+ b[i] = vqaddq_s16(b[i], b[i]);
+ } while (++i != 8);
+ } else {
+ assert(out_rshift == 1);
+ }
+}
+
static AOM_FORCE_INLINE void apply_txfm8_s16x8_neon(int16x8_t *b,
ITX_TYPE_1D type,
int out_rshift) {
switch (type) {
case IDCT_1D:
- idct8_neon(b, b, INV_COS_BIT);
+ idct8_neon_impl(b, b, INV_COS_BIT);
round_shift_array_16_neon(b, 8, out_rshift);
break;
case IADST_1D:
- iadst8_neon(b, b, INV_COS_BIT);
+ iadst8_neon_impl(b, b, INV_COS_BIT);
round_shift_array_16_neon(b, 8, out_rshift);
break;
- default: identity_txfm_round_neon(b, b, 1, 8, out_rshift); break;
+ default:
+ assert(type == IIDENTITY_1D);
+ identity_txfm8_s16x8_neon(b, out_rshift);
+ break;
+ }
+}
+
+static AOM_FORCE_INLINE void apply_txfm8_low1_s16x8_neon(int16x8_t *b,
+ ITX_TYPE_1D type,
+ int out_rshift) {
+ switch (type) {
+ case IDCT_1D:
+ idct8_low1_neon_impl(b, b, INV_COS_BIT);
+ round_shift_array_16_neon(b, 8, out_rshift);
+ break;
+ case IADST_1D:
+ iadst8_low1_neon_impl(b, b, INV_COS_BIT);
+ round_shift_array_16_neon(b, 8, out_rshift);
+ break;
+ default:
+ assert(type == IIDENTITY_1D);
+ if (out_rshift == 4) {
+ b[0] = vrshrq_n_s16(b[0], 3);
+ } else {
+ assert(out_rshift == 1);
+ }
+ break;
}
}
@@ -4009,19 +4164,52 @@ static AOM_FORCE_INLINE void apply_txfm16_s16x8_neon(int16x8_t *b,
}
}
-static void lowbd_inv_txfm2d_add_4x8_neon(const int32_t *input, uint8_t *output,
- int stride, TX_TYPE tx_type,
- int eob) {
- if (eob == 1 && tx_type == DCT_DCT) {
- const int32_t *cospi = cospi_arr(INV_COS_BIT);
- int32_t dc = round_shift((int64_t)input[0] * NewInvSqrt2, NewSqrt2Bits);
- dc = round_shift((int64_t)dc * cospi[32], INV_COS_BIT);
- dc = round_shift((int64_t)dc * cospi[32], INV_COS_BIT);
- dc = round_shift((int64_t)dc, 4);
- lowbd_inv_txfm_dc_only_4xn_neon(output, stride, dc, 8);
- return;
- }
+static AOM_FORCE_INLINE int32_t lowbd_dc_only_rect1_val(int32_t dc_in) {
+ const int32_t *cospi = cospi_arr(INV_COS_BIT);
+ int32_t dc = round_shift((int64_t)dc_in * NewInvSqrt2, NewSqrt2Bits);
+ dc = round_shift((int64_t)dc * cospi[32], INV_COS_BIT);
+ dc = round_shift((int64_t)dc * cospi[32], INV_COS_BIT);
+ return round_shift((int64_t)dc, 4);
+}
+
+static AOM_FORCE_INLINE int32_t lowbd_dc_only_rect2_val(int32_t dc_in) {
+ const int32_t *cospi = cospi_arr(INV_COS_BIT);
+ int32_t dc = round_shift((int64_t)dc_in * cospi[32], INV_COS_BIT);
+ dc = round_shift((int64_t)dc, 1);
+ dc = round_shift((int64_t)dc * cospi[32], INV_COS_BIT);
+ return round_shift((int64_t)dc, 4);
+}
+
+static AOM_FORCE_INLINE void lowbd_inv_txfm_dc_only_8x4_neon(uint8_t *output,
+ int stride,
+ int32_t dc) {
+ const int16x8_t dc_v = vdupq_n_s16((int16_t)clamp(dc, -255, 255));
+ int h = 4;
+ do {
+ const int16x8_t pred = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(output)));
+ vst1_u8(output, vqmovun_s16(vaddq_s16(pred, dc_v)));
+ output += stride;
+ } while (--h != 0);
+}
+static AOM_FORCE_INLINE void lowbd_inv_txfm_dc_only_16x4_neon(uint8_t *output,
+ int stride,
+ int32_t dc) {
+ const int16x8_t dc_v = vdupq_n_s16((int16_t)clamp(dc, -255, 255));
+ int h = 4;
+ do {
+ const uint8x16_t pred = vld1q_u8(output);
+ vst1q_u8(output, lowbd_get_recon_16x16_neon(pred, dc_v, dc_v));
+ output += stride;
+ } while (--h != 0);
+}
+
+// Keep out-of-line so large stack buffers do not force spills in the
+// dispatcher / TX_4X4 fast path.
+static AOM_NOINLINE void lowbd_inv_txfm2d_add_4x8_neon(const int32_t *input,
+ uint8_t *output,
+ int stride,
+ TX_TYPE tx_type) {
int16x8_t a[4];
int16x4_t t[8];
int16x8_t b[8];
@@ -4042,24 +4230,12 @@ static void lowbd_inv_txfm2d_add_4x8_neon(const int32_t *input, uint8_t *output,
lowbd_add_flip_buffer_4xn_neon(b, output, stride, ud_flip, 8);
}
-static void lowbd_inv_txfm2d_add_8x4_neon(const int32_t *input, uint8_t *output,
- int stride, TX_TYPE tx_type,
- int eob) {
- if (eob == 1 && tx_type == DCT_DCT) {
- const int32_t *cospi = cospi_arr(INV_COS_BIT);
- int32_t dc = round_shift((int64_t)input[0] * NewInvSqrt2, NewSqrt2Bits);
- dc = round_shift((int64_t)dc * cospi[32], INV_COS_BIT);
- dc = round_shift((int64_t)dc * cospi[32], INV_COS_BIT);
- dc = round_shift((int64_t)dc, 4);
- const int16x8_t dc_v = vdupq_n_s16((int16_t)clamp(dc, -255, 255));
- for (int i = 0; i < 4; ++i) {
- const int16x8_t pred = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(output)));
- vst1_u8(output, vqmovun_s16(vaddq_s16(pred, dc_v)));
- output += stride;
- }
- return;
- }
-
+// Keep out-of-line so large stack buffers do not force spills in the
+// dispatcher / TX_4X4 fast path.
+static AOM_NOINLINE void lowbd_inv_txfm2d_add_8x4_neon(const int32_t *input,
+ uint8_t *output,
+ int stride,
+ TX_TYPE tx_type) {
int16x8_t a[8];
int16x4_t row_in[8];
int16x8_t b[4];
@@ -4085,19 +4261,12 @@ static void lowbd_inv_txfm2d_add_8x4_neon(const int32_t *input, uint8_t *output,
lowbd_add_flip_buffer_8xn_neon(b, output, stride, ud_flip, 4);
}
-static void lowbd_inv_txfm2d_add_4x16_neon(const int32_t *input,
- uint8_t *output, int stride,
- TX_TYPE tx_type, int eob) {
- if (eob == 1 && tx_type == DCT_DCT) {
- const int32_t *cospi = cospi_arr(INV_COS_BIT);
- int32_t dc = round_shift((int64_t)input[0] * cospi[32], INV_COS_BIT);
- dc = round_shift((int64_t)dc, 1);
- dc = round_shift((int64_t)dc * cospi[32], INV_COS_BIT);
- dc = round_shift((int64_t)dc, 4);
- lowbd_inv_txfm_dc_only_4xn_neon(output, stride, dc, 16);
- return;
- }
-
+// Keep out-of-line so large stack buffers do not force spills in the
+// dispatcher / TX_4X4 fast path.
+static AOM_NOINLINE void lowbd_inv_txfm2d_add_4x16_neon(const int32_t *input,
+ uint8_t *output,
+ int stride,
+ TX_TYPE tx_type) {
int16x8_t a0[4], a1[4];
int16x4_t t[16];
int16x8_t b[16];
@@ -4121,24 +4290,12 @@ static void lowbd_inv_txfm2d_add_4x16_neon(const int32_t *input,
lowbd_add_flip_buffer_4xn_neon(b, output, stride, ud_flip, 16);
}
-static void lowbd_inv_txfm2d_add_16x4_neon(const int32_t *input,
- uint8_t *output, int stride,
- TX_TYPE tx_type, int eob) {
- if (eob == 1 && tx_type == DCT_DCT) {
- const int32_t *cospi = cospi_arr(INV_COS_BIT);
- int32_t dc = round_shift((int64_t)input[0] * cospi[32], INV_COS_BIT);
- dc = round_shift((int64_t)dc, 1);
- dc = round_shift((int64_t)dc * cospi[32], INV_COS_BIT);
- dc = round_shift((int64_t)dc, 4);
- const int16x8_t dc_v = vdupq_n_s16((int16_t)clamp(dc, -255, 255));
- for (int i = 0; i < 4; ++i) {
- const uint8x16_t pred = vld1q_u8(output);
- vst1q_u8(output, lowbd_get_recon_16x16_neon(pred, dc_v, dc_v));
- output += stride;
- }
- return;
- }
-
+// Keep out-of-line so large stack buffers do not force spills in the
+// dispatcher / TX_4X4 fast path.
+static AOM_NOINLINE void lowbd_inv_txfm2d_add_16x4_neon(const int32_t *input,
+ uint8_t *output,
+ int stride,
+ TX_TYPE tx_type) {
int16x8_t a[16];
int16x4_t row_lo[8], row_hi[8];
int16x8_t b[8];
@@ -4167,6 +4324,68 @@ static void lowbd_inv_txfm2d_add_16x4_neon(const int32_t *input,
lowbd_add_flip_buffer_16xn_neon(b, output, stride, ud_flip, 4);
}
+// Keep out-of-line to isolate TX_8X8 register/stack footprint from TX_4X4.
+static AOM_NOINLINE void lowbd_inv_txfm2d_add_8x8_dct_neon(const int32_t *input,
+ uint8_t *output,
+ int stride) {
+ int16x8_t a[8], b[8];
+ load_buffer_32bit_to_16bit_neon(input, 8, a, 8);
+ idct8_neon_impl(a, a, INV_COS_BIT);
+ round_shift_array_16_neon(a, 8, 1);
+ transpose_arrays_s16_8x8(a, b);
+ idct8_neon_impl(b, b, INV_COS_BIT);
+ round_shift_array_16_neon(b, 8, 4);
+ lowbd_add_flip_buffer_8xn_neon(b, output, stride, 0, 8);
+}
+
+// Keep out-of-line to isolate non-DCT TX_8X8 paths from the common path.
+static AOM_NOINLINE void lowbd_inv_txfm2d_add_8x8_non_dct_neon(
+ const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type,
+ int eob) {
+ int16x8_t a[8], b[8];
+ int ud_flip, lr_flip;
+ get_flip_cfg(tx_type, &ud_flip, &lr_flip);
+
+ if (eob == 1) {
+ a[0] = vsetq_lane_s16((int16_t)input[0], vdupq_n_s16(0), 0);
+ if (hitx_1d_tab[tx_type] == IIDENTITY_1D) {
+ int i = 1;
+ do {
+ a[i] = vdupq_n_s16(0);
+ } while (++i != 8);
+ }
+ apply_txfm8_low1_s16x8_neon(a, hitx_1d_tab[tx_type], 1);
+ if (lr_flip) {
+ flip_buf_ud_neon(a, 8);
+ }
+ transpose_arrays_s16_8x8(a, b);
+ if (vitx_1d_tab[tx_type] == IIDENTITY_1D) {
+ assert(!ud_flip);
+ b[0] = vrshrq_n_s16(b[0], 3);
+ lowbd_add_flip_buffer_8xn_neon(b, output, stride, 0, 1);
+ return;
+ }
+ apply_txfm8_low1_s16x8_neon(b, vitx_1d_tab[tx_type], 4);
+ if (ud_flip) {
+ flip_buf_ud_neon(b, 8);
+ }
+ lowbd_add_flip_buffer_8xn_neon(b, output, stride, 0, 8);
+ return;
+ }
+
+ load_buffer_32bit_to_16bit_neon(input, 8, a, 8);
+ apply_txfm8_s16x8_neon(a, hitx_1d_tab[tx_type], 1);
+ if (lr_flip) {
+ flip_buf_ud_neon(a, 8);
+ }
+ transpose_arrays_s16_8x8(a, b);
+ apply_txfm8_s16x8_neon(b, vitx_1d_tab[tx_type], 4);
+ if (ud_flip) {
+ flip_buf_ud_neon(b, 8);
+ }
+ lowbd_add_flip_buffer_8xn_neon(b, output, stride, 0, 8);
+}
+
static inline void lowbd_inv_txfm2d_add_no_identity_neon(
const int32_t *input, uint8_t *output, int stride, TX_TYPE tx_type,
TX_SIZE tx_size, int eob) {
@@ -4370,19 +4589,50 @@ void av1_lowbd_inv_txfm2d_add_neon(const int32_t *input, uint8_t *output,
break;
case TX_4X8:
- lowbd_inv_txfm2d_add_4x8_neon(input, output, stride, tx_type, eob);
+ if (eob == 1 && tx_type == DCT_DCT) {
+ lowbd_inv_txfm_dc_only_4xn_neon(output, stride,
+ lowbd_dc_only_rect1_val(input[0]), 8);
+ } else {
+ lowbd_inv_txfm2d_add_4x8_neon(input, output, stride, tx_type);
+ }
break;
case TX_8X4:
- lowbd_inv_txfm2d_add_8x4_neon(input, output, stride, tx_type, eob);
+ if (eob == 1 && tx_type == DCT_DCT) {
+ lowbd_inv_txfm_dc_only_8x4_neon(output, stride,
+ lowbd_dc_only_rect1_val(input[0]));
+ } else {
+ lowbd_inv_txfm2d_add_8x4_neon(input, output, stride, tx_type);
+ }
break;
case TX_4X16:
- lowbd_inv_txfm2d_add_4x16_neon(input, output, stride, tx_type, eob);
+ if (eob == 1 && tx_type == DCT_DCT) {
+ lowbd_inv_txfm_dc_only_4xn_neon(output, stride,
+ lowbd_dc_only_rect2_val(input[0]), 16);
+ } else {
+ lowbd_inv_txfm2d_add_4x16_neon(input, output, stride, tx_type);
+ }
break;
case TX_16X4:
- lowbd_inv_txfm2d_add_16x4_neon(input, output, stride, tx_type, eob);
+ if (eob == 1 && tx_type == DCT_DCT) {
+ lowbd_inv_txfm_dc_only_16x4_neon(output, stride,
+ lowbd_dc_only_rect2_val(input[0]));
+ } else {
+ lowbd_inv_txfm2d_add_16x4_neon(input, output, stride, tx_type);
+ }
+ break;
+
+ case TX_8X8:
+ if (tx_type != DCT_DCT) {
+ lowbd_inv_txfm2d_add_8x8_non_dct_neon(input, output, stride, tx_type,
+ eob);
+ } else if (eob == 1) {
+ lowbd_inv_txfm2d_add_dc_only_neon(input[0], output, stride, tx_size);
+ } else {
+ lowbd_inv_txfm2d_add_8x8_dct_neon(input, output, stride);
+ }
break;
default:
diff --git a/av1/common/arm/av1_inv_txfm_neon.h b/av1/common/arm/av1_inv_txfm_neon.h
index 87d48dd576..da5704e457 100644
--- a/av1/common/arm/av1_inv_txfm_neon.h
+++ b/av1/common/arm/av1_inv_txfm_neon.h
@@ -12,13 +12,13 @@
#define AOM_AV1_COMMON_ARM_AV1_INV_TXFM_NEON_H_
#include <arm_neon.h>
+#include <assert.h>
#include "config/aom_config.h"
#include "config/av1_rtcd.h"
#include "aom/aom_integer.h"
#include "av1/common/enums.h"
-#include "av1/common/av1_inv_txfm1d.h"
#include "av1/common/av1_inv_txfm1d_cfg.h"
#include "av1/common/av1_txfm.h"
@@ -26,16 +26,9 @@
extern "C" {
#endif
-typedef void (*transform_1d_neon)(const int32_t *input, int32_t *output,
- const int8_t cos_bit,
- const int8_t *stage_ptr);
typedef void (*transform_neon)(int16x8_t *input, int16x8_t *output,
int8_t cos_bit);
-DECLARE_ALIGNED(16, static const int16_t, av1_eob_to_eobxy_8x8_default[8]) = {
- 0x0707, 0x0707, 0x0707, 0x0707, 0x0707, 0x0707, 0x0707, 0x0707,
-};
-
DECLARE_ALIGNED(16, static const int16_t,
av1_eob_to_eobxy_16x16_default[16]) = {
0x0707, 0x0707, 0x0f0f, 0x0f0f, 0x0f0f, 0x0f0f, 0x0f0f, 0x0f0f,
@@ -87,7 +80,7 @@ DECLARE_ALIGNED(16, static const int16_t, av1_eob_to_eobxy_32x8_default[8]) = {
DECLARE_ALIGNED(16, static const int16_t *,
av1_eob_to_eobxy_default[TX_SIZES_ALL]) = {
NULL,
- av1_eob_to_eobxy_8x8_default,
+ NULL,
av1_eob_to_eobxy_16x16_default,
av1_eob_to_eobxy_32x32_default,
av1_eob_to_eobxy_32x32_default,
@@ -132,6 +125,7 @@ static inline void get_eobx_eoby_scan_default(int *eobx, int *eoby,
const int tx_w_log2 = tx_size_wide_log2_eob[tx_size];
const int eob_row = (eob - 1) >> tx_w_log2;
+ assert(av1_eob_to_eobxy_default[tx_size] != NULL);
const int eobxy = av1_eob_to_eobxy_default[tx_size][eob_row];
*eobx = eobxy & 0xFF;
*eoby = eobxy >> 8;