Commit a497ab4d07 for ffmpeg
commit a497ab4d071ba64be947cb99e4e2cfae3930a439
Author: Zuxy Meng <zuxy.meng@gmail.com>
Date: Sat Oct 17 19:36:32 2026 -0700
avcodec/x86/h264: convert IDCT MMX to SSE2
Convert the last MMX (mmx/mmxext) IDCT functions, idct8_dc_add_8,
idct_add8_422_8 (plus its plane and DC helpers) and
idct_dc_add_10, to SSE2. The DC_ADD macros are renamed to
DC_ADD_SSE2 (pshuflw, movq traffic) and shared with the
pre-existing sse2/avx IDCT_XMM path. The mmxext DC helper
tail-called from SSE2 functions is now XMM, which also removes
the MMX-state handoff without emms. Dispatch moves from
EXTERNAL_MMX/MMXEXT to EXTERNAL_SSE2.
checkasm h264dsp passes. identical performances.
diff --git a/libavcodec/x86/h264_idct.asm b/libavcodec/x86/h264_idct.asm
index 1d4c9729fa..d9fadabefc 100644
--- a/libavcodec/x86/h264_idct.asm
+++ b/libavcodec/x86/h264_idct.asm
@@ -68,6 +68,22 @@ SECTION .text
%endmacro
; %1=uint8_t *dst, %2=int16_t *block, %3=ptrdiff_t stride
+; STORE_DIFFx2 not suitable here: it uses movh
+%macro STORE_DIFFx4 8 ; add1, add2, reg1, reg2, zero, shift, source, stride
+ movd %3, [%7]
+ movd %4, [%7+%8]
+ psraw %1, %6
+ psraw %2, %6
+ punpcklbw %3, %5
+ punpcklbw %4, %5
+ paddw %3, %1
+ paddw %4, %2
+ packuswb %3, %5
+ packuswb %4, %5
+ movd [%7], %3
+ movd [%7+%8], %4
+%endmacro
+
%macro IDCT4_ADD 3
; Load dct coeffs
movq m0, [%2]
@@ -99,9 +115,9 @@ SECTION .text
movq [%2+24], m7
%endif
- STORE_DIFFx2 m0, m1, m4, m5, m7, 6, %1, %3
+ STORE_DIFFx4 m0, m1, m4, m5, m7, 6, %1, %3
lea %1, [%1+%3*2]
- STORE_DIFFx2 m2, m3, m4, m5, m7, 6, %1, %3
+ STORE_DIFFx4 m2, m3, m4, m5, m7, 6, %1, %3
%endmacro
%macro IDCT8_1D 2
@@ -230,19 +246,21 @@ cglobal h264_idct8_add_8, 3, 4, 10
IDCT8_ADD_SSE r0, r1, r2, r3
RET
-%macro DC_ADD_MMXEXT_INIT 2
+%macro DC_ADD_SSE2_INIT 2
add %1, 32
sar %1, 6
movd m0, %1d
lea %1, [%2*3]
- pshufw m0, m0, 0
+ pshuflw m0, m0, 0
pxor m1, m1
psubw m1, m0
packuswb m0, m0
packuswb m1, m1
+ pshufd m0, m0, 0
+ pshufd m1, m1, 0
%endmacro
-%macro DC_ADD_MMXEXT_OP 4
+%macro DC_ADD_SSE2_OP 4
%1 m2, [%2 ]
%1 m3, [%2+%3 ]
%1 m4, [%2+%3*2]
@@ -261,27 +279,27 @@ cglobal h264_idct8_add_8, 3, 4, 10
%1 [%2+%4 ], m5
%endmacro
-INIT_MMX mmxext
+INIT_XMM sse2
%if ARCH_X86_64
-; void ff_h264_idct8_dc_add_8_mmxext(uint8_t *dst, int16_t *block, ptrdiff_t stride)
-cglobal h264_idct8_dc_add_8, 3, 4, 0
+; void ff_h264_idct8_dc_add_8_sse2(uint8_t *dst, int16_t *block, ptrdiff_t stride)
+cglobal h264_idct8_dc_add_8, 3, 4, 6
movsx r3, word [r1]
mov dword [r1], 0
- DC_ADD_MMXEXT_INIT r3, r2
- DC_ADD_MMXEXT_OP mova, r0, r2, r3
+ DC_ADD_SSE2_INIT r3, r2
+ DC_ADD_SSE2_OP movq, r0, r2, r3
lea r0, [r0+r2*4]
- DC_ADD_MMXEXT_OP mova, r0, r2, r3
+ DC_ADD_SSE2_OP movq, r0, r2, r3
RET
%else
-; void ff_h264_idct8_dc_add_8_mmxext(uint8_t *dst, int16_t *block, ptrdiff_t stride)
-cglobal h264_idct8_dc_add_8, 2, 3, 0
+; void ff_h264_idct8_dc_add_8_sse2(uint8_t *dst, int16_t *block, ptrdiff_t stride)
+cglobal h264_idct8_dc_add_8, 2, 3, 6
movsx r2, word [r1]
mov dword [r1], 0
mov r1, r2m
- DC_ADD_MMXEXT_INIT r2, r1
- DC_ADD_MMXEXT_OP mova, r0, r1, r2
+ DC_ADD_SSE2_INIT r2, r1
+ DC_ADD_SSE2_OP movq, r0, r1, r2
lea r0, [r0+r1*4]
- DC_ADD_MMXEXT_OP mova, r0, r1, r2
+ DC_ADD_SSE2_OP movq, r0, r1, r2
RET
%endif
@@ -304,18 +322,17 @@ cglobal h264_idct8_add4_8, 5, 8 + npicregs, 10, dst1, block_offset, block, strid
movsx r6, word [r2]
test r6, r6
jz .no_dc
-INIT_MMX cpuname
mov word [r2], 0
- DC_ADD_MMXEXT_INIT r6, r3
+ DC_ADD_SSE2_INIT r6, r3
%if ARCH_X86_64 == 0
%define dst2q r1
%define dst2d r1d
%endif
mov dst2d, dword [r1+r5*4]
add dst2q, r0
- DC_ADD_MMXEXT_OP mova, dst2q, r3, r6
+ DC_ADD_SSE2_OP movq, dst2q, r3, r6
lea dst2q, [dst2q+r3*4]
- DC_ADD_MMXEXT_OP mova, dst2q, r3, r6
+ DC_ADD_SSE2_OP movq, dst2q, r3, r6
%if ARCH_X86_64 == 0
mov r1, r1m
%endif
@@ -325,7 +342,6 @@ INIT_MMX cpuname
jl .nextblock
RET
.no_dc:
-INIT_XMM cpuname
mov dst2d, dword [r1+r5*4]
add dst2q, r0
IDCT8_ADD_SSE dst2q, r2, r3, r6
@@ -339,8 +355,8 @@ INIT_XMM cpuname
jl .nextblock
RET
-INIT_MMX mmx
-cglobal h264_idct_add8_422_8, 5, 8 + npicregs, 0, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg
+INIT_XMM sse2
+cglobal h264_idct_add8_422_8, 5, 8 + npicregs, 8, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg
; dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg
%if PIC
lea picregq, [scan8_mem]
@@ -352,9 +368,9 @@ cglobal h264_idct_add8_422_8, 5, 8 + npicregs, 0, dst1, block_offset, block, str
mov r5, 16 ; i
add r2, 512 ; i * 16 * sizeof(dctcoef) ; #define dctcoef int16_t
- call h264_idct_add8_mmx_plane
+ call h264_idct_add8_sse2_plane
add r5, 4
- call h264_idct_add8_mmx_plane
+ call h264_idct_add8_sse2_plane
%if ARCH_X86_64
add dst2q, gprsize ; dest[1]
@@ -365,11 +381,11 @@ cglobal h264_idct_add8_422_8, 5, 8 + npicregs, 0, dst1, block_offset, block, str
add r5, 4 ; set to 32
add r2, 256 ; set to i * 16 * sizeof(dctcoef)
- call h264_idct_add8_mmx_plane
+ call h264_idct_add8_sse2_plane
add r5, 4
- TAIL_CALL h264_idct_add8_mmx_plane, 0
+ TAIL_CALL h264_idct_add8_sse2_plane, 0
-h264_idct_add8_mmx_plane:
+h264_idct_add8_sse2_plane:
.nextblock:
movzx r6d, byte [scan8+r5]
movzx r6d, byte [r4+r6]
@@ -394,7 +410,7 @@ h264_idct_add8_mmx_plane:
; r0 = uint8_t *dst, r2 = int16_t *block, r3 = ptrdiff_t stride, r6=clobbered
-h264_idct_dc_add8_mmxext:
+h264_idct_dc_add8_sse2:
movd m0, [r2 ] ; 0 0 X D
mov word [r2+ 0], 0
punpcklwd m0, [r2+32] ; x X d D
@@ -404,11 +420,12 @@ h264_idct_dc_add8_mmxext:
punpcklwd m0, m0 ; d d D D
pxor m1, m1 ; 0 0 0 0
psubw m1, m0 ; -d-d-D-D
- packuswb m0, m1 ; -d-d-D-D d d D D
- pshufw m1, m0, 0xFA ; -d-d-d-d-D-D-D-D
+ packuswb m0, m1 ; d d D D 0.. -d-d-D-D 0..
+ pshufd m1, m0, q3232 ; -d-d-D-D ..
+ punpcklwd m1, m1 ; -d-d-d-d-D-D-D-D
punpcklwd m0, m0 ; d d d d D D D D
lea r6, [r3*3]
- DC_ADD_MMXEXT_OP movq, r0, r3, r6
+ DC_ADD_SSE2_OP movq, r0, r3, r6
ret
ALIGN 16
@@ -494,7 +511,7 @@ RET
%else
add r0, r0m
%endif
- TAIL_CALL_IF_LAST h264_idct_dc_add8_mmxext, , %1, 7
+ TAIL_CALL_IF_LAST h264_idct_dc_add8_sse2, , %1, 7
.cycle%1end:
%if %1 < 7
add r2, 64
@@ -543,7 +560,7 @@ RET
mov r0, [r0]
add r0, dword [r1+(%1&1)*8+64*(1+(%1>>1))]
%endif
- TAIL_CALL_IF_LAST h264_idct_dc_add8_mmxext, , %1, 3
+ TAIL_CALL_IF_LAST h264_idct_dc_add8_sse2, , %1, 3
.cycle%1end:
%if %1 == 1
add r2, 384+64
@@ -698,26 +715,6 @@ cglobal h264_luma_dc_dequant_idct, 3, 4, 6
DEQUANT_STORE m6
RET
-%ifdef __NASM_VER__
-%if __NASM_MAJOR__ > 2 || (__NASM_MAJOR__ == 2 && __NASM_MINOR__ >= 4)
-%unmacro STORE_DIFFx2 8 ; remove macro from x86util.asm but yasm doesn't have this yet
-%endif
-%endif
-%macro STORE_DIFFx2 8 ; add1, add2, reg1, reg2, zero, shift, source, stride
- movd %3, [%7]
- movd %4, [%7+%8]
- psraw %1, %6
- psraw %2, %6
- punpcklbw %3, %5
- punpcklbw %4, %5
- paddw %3, %1
- paddw %4, %2
- packuswb %3, %5
- packuswb %4, %5
- movd [%7], %3
- movd [%7+%8], %4
-%endmacro
-
%macro DC_ADD_INIT 1
add %1d, 32
sar %1d, 6
@@ -742,7 +739,7 @@ cglobal h264_idct_dc_add_8, 3, 4, 6, dst_, block_, stride_
movsx r3d, word [block_q]
mov dword [block_q], 0
DC_ADD_INIT r3
- DC_ADD_MMXEXT_OP movd, dst_q, stride_q, r3
+ DC_ADD_SSE2_OP movd, dst_q, stride_q, r3
RET
%endmacro
diff --git a/libavcodec/x86/h264_idct_10bit.asm b/libavcodec/x86/h264_idct_10bit.asm
index 741b79d044..1c3e7c8012 100644
--- a/libavcodec/x86/h264_idct_10bit.asm
+++ b/libavcodec/x86/h264_idct_10bit.asm
@@ -166,7 +166,9 @@ IDCT_ADD16_10
;-----------------------------------------------------------------------------
; void ff_h264_idct_dc_add_10(pixel *dst, int16_t *block, ptrdiff_t stride)
;-----------------------------------------------------------------------------
-%macro IDCT_DC_ADD_OP_10 3
+; %4 selects the dst traffic: mova for 16-byte (8-wide) rows, movq
+; for 8-byte (4-wide) rows whose alignment is only guaranteed to 8.
+%macro IDCT_DC_ADD_OP_10 3-4 mova
pxor m5, m5
%if avx_enabled
paddw m1, m0, [%1+0 ]
@@ -174,10 +176,10 @@ IDCT_ADD16_10
paddw m3, m0, [%1+%2*2]
paddw m4, m0, [%1+%3 ]
%else
- mova m1, [%1+0 ]
- mova m2, [%1+%2 ]
- mova m3, [%1+%2*2]
- mova m4, [%1+%3 ]
+ %4 m1, [%1+0 ]
+ %4 m2, [%1+%2 ]
+ %4 m3, [%1+%2*2]
+ %4 m4, [%1+%3 ]
paddw m1, m0
paddw m2, m0
paddw m3, m0
@@ -187,22 +189,22 @@ IDCT_ADD16_10
CLIPW m2, m5, m6
CLIPW m3, m5, m6
CLIPW m4, m5, m6
- mova [%1+0 ], m1
- mova [%1+%2 ], m2
- mova [%1+%2*2], m3
- mova [%1+%3 ], m4
+ %4 [%1+0 ], m1
+ %4 [%1+%2 ], m2
+ %4 [%1+%2*2], m3
+ %4 [%1+%3 ], m4
%endmacro
-INIT_MMX mmxext
-cglobal h264_idct_dc_add_10,3,3
+INIT_XMM sse2
+cglobal h264_idct_dc_add_10,3,3,7
movd m0, [r1]
mov dword [r1], 0
paddd m0, [pd_32]
psrad m0, 6
lea r1, [r2*3]
- pshufw m0, m0, 0
+ pshuflw m0, m0, 0
mova m6, [pw_pixel_max]
- IDCT_DC_ADD_OP_10 r0, r2, r1
+ IDCT_DC_ADD_OP_10 r0, r2, r1, movq
RET
;-----------------------------------------------------------------------------
diff --git a/libavcodec/x86/h264dsp_init.c b/libavcodec/x86/h264dsp_init.c
index 654316c2af..d103fe041f 100644
--- a/libavcodec/x86/h264dsp_init.c
+++ b/libavcodec/x86/h264dsp_init.c
@@ -35,8 +35,8 @@ IDCT_ADD_FUNC(, 8, avx)
IDCT_ADD_FUNC(, 10, sse2)
IDCT_ADD_FUNC(_dc, 8, sse2)
IDCT_ADD_FUNC(_dc, 8, avx)
-IDCT_ADD_FUNC(_dc, 10, mmxext)
-IDCT_ADD_FUNC(8_dc, 8, mmxext)
+IDCT_ADD_FUNC(_dc, 10, sse2)
+IDCT_ADD_FUNC(8_dc, 8, sse2)
IDCT_ADD_FUNC(8_dc, 10, sse2)
IDCT_ADD_FUNC(8, 8, sse2)
IDCT_ADD_FUNC(8, 10, sse2)
@@ -70,7 +70,7 @@ IDCT_ADD_REP_FUNC2(, 8, 8, sse2)
IDCT_ADD_REP_FUNC2(, 8, 10, sse2)
IDCT_ADD_REP_FUNC2(, 8, 10, avx)
-IDCT_ADD_REP_FUNC2(, 8_422, 8, mmx)
+IDCT_ADD_REP_FUNC2(, 8_422, 8, sse2)
IDCT_ADD_REP_FUNC2(, 8_422, 10, sse2)
IDCT_ADD_REP_FUNC2(, 8_422, 10, avx)
@@ -186,22 +186,15 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,
c->loop_filter_strength = ff_h264_loop_filter_strength_mmxext;
if (bit_depth == 8) {
- if (EXTERNAL_MMX(cpu_flags)) {
- if (chroma_format_idc <= 1) {
- } else {
- c->idct_add8 = ff_h264_idct_add8_422_8_mmx;
- }
- }
- if (EXTERNAL_MMXEXT(cpu_flags)) {
- c->idct8_dc_add = ff_h264_idct8_dc_add_8_mmxext;
- }
if (EXTERNAL_SSE2(cpu_flags)) {
c->idct8_add = ff_h264_idct8_add_8_sse2;
-
+ c->idct8_dc_add = ff_h264_idct8_dc_add_8_sse2;
c->idct_add16 = ff_h264_idct_add16_8_sse2;
c->idct8_add4 = ff_h264_idct8_add4_8_sse2;
if (chroma_format_idc <= 1)
c->idct_add8 = ff_h264_idct_add8_8_sse2;
+ else
+ c->idct_add8 = ff_h264_idct_add8_422_8_sse2;
c->idct_add16intra = ff_h264_idct_add16intra_8_sse2;
c->luma_dc_dequant_idct = ff_h264_luma_dc_dequant_idct_sse2;
@@ -263,10 +256,8 @@ av_cold void ff_h264dsp_init_x86(H264DSPContext *c, const int bit_depth,
c->idct_dc_add = ff_h264_idct_dc_add_8_avx;
}
} else if (bit_depth == 10) {
- if (EXTERNAL_MMXEXT(cpu_flags)) {
- c->idct_dc_add = ff_h264_idct_dc_add_10_mmxext;
- }
if (EXTERNAL_SSE2(cpu_flags)) {
+ c->idct_dc_add = ff_h264_idct_dc_add_10_sse2;
c->idct_add = ff_h264_idct_add_10_sse2;
c->idct8_dc_add = ff_h264_idct8_dc_add_10_sse2;
diff --git a/tests/checkasm/h264dsp.c b/tests/checkasm/h264dsp.c
index 3bd0dec2c3..6e9792a0f6 100644
--- a/tests/checkasm/h264dsp.c
+++ b/tests/checkasm/h264dsp.c
@@ -184,7 +184,7 @@ static void check_idct(void)
LOCAL_ALIGNED_16(int16_t, subcoef1, [8 * 8 * 2]);
H264DSPContext h;
int bit_depth, sz, align, dc, i;
- declare_func_emms(AV_CPU_FLAG_MMX, void, uint8_t *dst, int16_t *block, ptrdiff_t stride);
+ declare_func(void, uint8_t *dst, int16_t *block, ptrdiff_t stride);
for (i = 0; i < FF_ARRAY_ELEMS(depths); i++) {
bit_depth = depths[i];
@@ -250,7 +250,7 @@ static void check_idct_multiple(void)
LOCAL_ALIGNED_16(uint8_t, nnzc, [15 * 8]);
H264DSPContext h;
int bit_depth, i, y, func;
- declare_func_emms(AV_CPU_FLAG_MMX, void, uint8_t *dst, const int *block_offset, int16_t *block, ptrdiff_t stride, const uint8_t nnzc[15*8]);
+ declare_func(void, uint8_t *dst, const int *block_offset, int16_t *block, ptrdiff_t stride, const uint8_t nnzc[15*8]);
for (bit_depth = 8; bit_depth <= 10; bit_depth++) {
ff_h264dsp_init(&h, bit_depth, 1);
@@ -325,6 +325,85 @@ static void check_idct_multiple(void)
}
}
+static void check_idct_add8(void)
+{
+ LOCAL_ALIGNED_16(uint8_t, dst_full, [16 * 32 * 2]);
+ LOCAL_ALIGNED_16(int16_t, coef_full, [48 * 16 * 2]);
+ LOCAL_ALIGNED_16(uint8_t, dst0, [16 * 32 * 2]);
+ LOCAL_ALIGNED_16(uint8_t, dst1, [16 * 32 * 2]);
+ LOCAL_ALIGNED_16(int16_t, coef0, [48 * 16 * 2]);
+ LOCAL_ALIGNED_16(int16_t, coef1, [48 * 16 * 2]);
+ LOCAL_ALIGNED_16(uint8_t, nnzc, [15 * 8]);
+ H264DSPContext h;
+ int bit_depth, format, j, k;
+ int sz = 4;
+ int block_offset[48] = { 0 };
+ declare_func(void, uint8_t **dst, const int *block_offset,
+ int16_t *block, ptrdiff_t stride, const uint8_t nnzc[15*8]);
+
+ for (bit_depth = 8; bit_depth <= 10; bit_depth += 2) {
+ for (format = 1; format <= 2; format++) {
+ void (*idct)(uint8_t **, const int *, int16_t *, ptrdiff_t, const uint8_t[]) = NULL;
+ const char *name = format == 1 ? "idct_add8" : "idct_add8_422";
+ uint8_t *dstp0[2], *dstp1[2];
+
+ ff_h264dsp_init(&h, bit_depth, format);
+ idct = h.idct_add8;
+
+ memset(nnzc, 0, 15 * 8);
+ memset(coef_full, 0, 48 * 16 * SIZEOF_COEF);
+ for (j = 1; j < 3; j++) {
+ for (k = 0; k < (format == 1 ? 4 : 8); k++) {
+ uint8_t src[8 * 8 * 2];
+ uint8_t dst[8 * 8 * 2];
+ int16_t coef[16 * 2];
+ int second = k >= 4;
+ int i = j * 16 + (k % 4) + (second ? 4 : 0);
+ int bo = second ? i + 4 : i;
+ int row = (second ? 8 : 0);
+ int col = (k % 4) * 4;
+ int offset = (row * 16 + col) * SIZEOF_PIXEL +
+ (j - 1) * 16 * 16 * SIZEOF_PIXEL;
+ int nnz = rnd() % 3;
+ int16_t *cblk = &coef_full[i * 16 * SIZEOF_COEF / 2];
+
+ randomize_buffers(bit_depth - 8);
+ dct4x4(coef, bit_depth);
+
+ if (nnz > 1)
+ nnz = 16;
+ memcpy(cblk, coef, nnz * SIZEOF_COEF);
+
+ for (int y = 0; y < 4; y++)
+ memcpy(&dst_full[offset + y * 16 * SIZEOF_PIXEL],
+ &dst[y * 4 * SIZEOF_PIXEL], 4 * SIZEOF_PIXEL);
+
+ nnzc[scan8[second ? i + 4 : i]] = nnz;
+ block_offset[bo] = offset - (j - 1) * 16 * 16 * SIZEOF_PIXEL;
+ }
+ }
+
+ dstp0[0] = dst0;
+ dstp0[1] = dst0 + 16 * 16 * SIZEOF_PIXEL;
+ dstp1[0] = dst1;
+ dstp1[1] = dst1 + 16 * 16 * SIZEOF_PIXEL;
+
+ if (check_func(idct, "%s_%dbpp", name, bit_depth)) {
+ memcpy(coef0, coef_full, 48 * 16 * SIZEOF_COEF);
+ memcpy(coef1, coef_full, 48 * 16 * SIZEOF_COEF);
+ memcpy(dst0, dst_full, 16 * 32 * 2);
+ memcpy(dst1, dst_full, 16 * 32 * 2);
+ call_ref(dstp0, block_offset, coef0, 16 * SIZEOF_PIXEL, nnzc);
+ call_new(dstp1, block_offset, coef1, 16 * SIZEOF_PIXEL, nnzc);
+ if (memcmp(dst0, dst1, 16 * 32 * 2) ||
+ memcmp(coef0, coef1, 48 * 16 * SIZEOF_COEF))
+ fail();
+ bench_new(dstp1, block_offset, coef1, 16 * SIZEOF_PIXEL, nnzc);
+ }
+ }
+ }
+}
+
static void check_idct_dequant(void)
{
static const int depths[5] = { 8, 9, 10, 12, 14 };
@@ -617,6 +696,7 @@ void checkasm_check_h264dsp(void)
{
check_idct();
check_idct_multiple();
+ check_idct_add8();
check_idct_dequant();
report("idct");