Commit 93e5b180a7 for ffmpeg
commit 93e5b180a79e8c28561b0651c5c5064f802499ff
Author: Rainbaby <r@bgr.jp>
Date: Tue Nov 10 08:26:16 2026 +0900
avcodec/x86/cavsidct: fix overflow on large coefficients
The SSE2 code worked in 16 bit lanes, so sums past int16_t wrapped.
Sum in 32 bits with pmaddwd and let packssdw clip like the C code.
The function now adds to the picture itself and takes the block in
FF_IDCT_PERM_LIBMPEG2 order, which pairs the coefficients for pmaddwd.
Signed-off-by: Rainbaby <r@bgr.jp>
diff --git a/libavcodec/x86/cavsdsp.c b/libavcodec/x86/cavsdsp.c
index a26fc9661c..55056cdd14 100644
--- a/libavcodec/x86/cavsdsp.c
+++ b/libavcodec/x86/cavsdsp.c
@@ -24,25 +24,16 @@
#include "libavutil/attributes.h"
#include "libavutil/cpu.h"
-#include "libavutil/mem_internal.h"
#include "libavutil/x86/cpu.h"
#include "libavcodec/cavsdsp.h"
#include "libavcodec/idctdsp.h"
#include "fpel.h"
-#include "idctdsp.h"
#include "config.h"
#if HAVE_SSE2_EXTERNAL
-void ff_cavs_idct8_sse2(int16_t *out, const int16_t *in);
-
-static void cavs_idct8_add_sse2(uint8_t *dst, int16_t *block, ptrdiff_t stride)
-{
- LOCAL_ALIGNED(16, int16_t, b2, [64]);
- ff_cavs_idct8_sse2(b2, block);
- ff_add_pixels_clamped_sse2(b2, dst, stride);
-}
+void ff_cavs_idct8_add_sse2(uint8_t *dst, int16_t *block, ptrdiff_t stride);
#define DEF_QPEL(OPNAME) \
void ff_ ## OPNAME ## _cavs_qpel8_mc20_sse2(uint8_t *dst, const uint8_t *src, ptrdiff_t stride); \
@@ -112,8 +103,8 @@ av_cold void ff_cavsdsp_init_x86(CAVSDSPContext *c)
c->avg_cavs_qpel_pixels_tab[1][ 8] = ff_avg_cavs_qpel8_mc02_sse2;
c->avg_cavs_qpel_pixels_tab[1][12] = ff_avg_cavs_qpel8_mc03_sse2;
- c->cavs_idct8_add = cavs_idct8_add_sse2;
- c->idct_perm = FF_IDCT_PERM_TRANSPOSE;
+ c->cavs_idct8_add = ff_cavs_idct8_add_sse2;
+ c->idct_perm = FF_IDCT_PERM_LIBMPEG2;
}
#endif
}
diff --git a/libavcodec/x86/cavsidct.asm b/libavcodec/x86/cavsidct.asm
index a0af002a78..091dad2ea7 100644
--- a/libavcodec/x86/cavsidct.asm
+++ b/libavcodec/x86/cavsidct.asm
@@ -24,141 +24,145 @@
%include "libavutil/x86/x86util.asm"
-cextern pw_4
-cextern pw_64
+SECTION_RODATA
-SECTION .text
-
-%macro CAVS_IDCT8_1D 2-3 1 ; source, round, init_load
-%if %3 == 1
- mova m4, [%1+7*16] ; m4 = src7
- mova m5, [%1+1*16] ; m5 = src1
- mova m2, [%1+5*16] ; m2 = src5
- mova m7, [%1+3*16] ; m7 = src3
-%else
- SWAP 1, 7
- SWAP 4, 6
-%endif
- mova m0, m4
- mova m3, m5
- mova m6, m2
- mova m1, m7
+; Horizontal pass. A line is stored as src0 src2 src4 src6 src1 src3 src5 src7
+; (FF_IDCT_PERM_LIBMPEG2). row_xy_01 holds the weights of srcx and srcy in
+; out0 (low half) and out1 (high half), doubled for two lines at once.
+row_02_01: dw 8, 10, 8, 10, 8, 4, 8, 4
+row_02_23: dw 8, -4, 8, -4, 8, -10, 8, -10
+row_46_01: dw 8, 4, 8, 4, -8, -10, -8, -10
+row_46_23: dw -8, 10, -8, 10, 8, -4, 8, -4
+row_13_01: dw 10, 9, 10, 9, 9, -2, 9, -2
+row_13_23: dw 6, -10, 6, -10, 2, -6, 2, -6
+row_57_01: dw 6, 2, 6, 2, -10, -6, -10, -6
+row_57_23: dw 2, 9, 2, 9, 9, -10, 9, -10
- paddw m4, m4 ; m4 = 2*src7
- paddw m3, m3 ; m3 = 2*src1
- paddw m6, m6 ; m6 = 2*src5
- paddw m1, m1 ; m1 = 2*src3
- paddw m0, m4 ; m0 = 3*src7
- paddw m5, m3 ; m5 = 3*src1
- paddw m2, m6 ; m2 = 3*src5
- paddw m7, m1 ; m7 = 3*src3
- psubw m5, m4 ; m5 = 3*src1 - 2*src7 = a0
- paddw m7, m6 ; m7 = 3*src3 - 2*src5 = a1
- psubw m1, m2 ; m1 = 2*src3 - 3*src5 = a2
- paddw m3, m0 ; m3 = 2*src1 - 3*src7 = a3
+; Vertical pass, on the interleaved words of lines n and n+4.
+pw_8_8: times 4 dw 8, 8
+pw_8_m8: times 4 dw 8, -8
+pw_10_4: times 4 dw 10, 4
+pw_4_m10: times 4 dw 4, -10
+pw_10_6: times 4 dw 10, 6
+pw_9_2: times 4 dw 9, 2
+pw_9_m10: times 4 dw 9, -10
+pw_m2_m6: times 4 dw -2, -6
+pw_6_2: times 4 dw 6, 2
+pw_m10_9: times 4 dw -10, 9
+pw_2_9: times 4 dw 2, 9
+pw_m6_m10: times 4 dw -6, -10
- mova m4, m5
- mova m6, m7
- mova m0, m3
- mova m2, m1
- paddw m7, m5 ; m7 = a0 + a1
- psubw m5, m6 ; m5 = a0 - a1
- paddw m7, m3 ; m7 = a0 + a1 + a3
- paddw m5, m1 ; m5 = a0 - a1 + a2
- paddw m7, m7
- paddw m5, m5
- paddw m7, m6 ; m7 = b4
- paddw m5, m4 ; m5 = b5
+pw_8_0: times 4 dw 8, 0
+pd_4: times 4 dd 4
- paddw m1, m3 ; m1 = a3 + a2
- psubw m3, m2 ; m3 = a3 - a2
- psubw m4, m1 ; m4 = a0 - a2 - a3
- mova m1, m4 ; m1 = a0 - a2 - a3
- psubw m3, m6 ; m3 = a3 - a2 - a1
- paddw m1, m1
- paddw m3, m3
- psubw m1, m2 ; m1 = b7
- paddw m3, m0 ; m3 = b6
+cextern pb_0
- mova m2, [%1+2*16] ; m2 = src2
- mova m6, [%1+6*16] ; m6 = src6
- mova m4, m2
- mova m0, m6
- psllw m4, 2 ; m4 = 4*src2
- psllw m6, 2 ; m6 = 4*src6
- paddw m2, m4 ; m2 = 5*src2
- paddw m0, m6 ; m0 = 5*src6
- paddw m2, m2
- paddw m0, m0
- psubw m4, m0 ; m4 = 4*src2 - 10*src6 = a7
- paddw m6, m2 ; m6 = 4*src6 + 10*src2 = a6
+SECTION .text
- mova m2, [%1+0*16] ; m2 = src0
- mova m0, [%1+4*16] ; m0 = src4
-%if ARCH_X86_64
- SUMSUB_BA w, 0, 2, 8 ; m0 = src0 + src4, m2 = src0 - src4
-%else
- SUMSUB_BA w, 0, 2 ; m0 = src0 + src4, m2 = src0 - src4
+; Horizontal pass of lines %1 and %1+4. Line %1 gets the two lines' words
+; of columns 0-3 interleaved, line %1+4 those of columns 4-7.
+%macro CAVS_IDCT8_ROWS 1
+ mova m1, [blockq+(%1)*16]
+ punpckldq m0, m1, [blockq+(%1)*16+64] ; src0 src2 x2, src4 src6 x2
+ punpckhdq m1, [blockq+(%1)*16+64] ; src1 src3 x2, src5 src7 x2
+ pshufd m2, m0, q1010
+ pshufd m0, m0, q3232
+ pmaddwd m3, m2, [row_02_23]
+ pmaddwd m2, [row_02_01]
+ pmaddwd m4, m0, [row_46_23]
+ pmaddwd m0, [row_46_01]
+ paddd m2, m0 ; even part of out0 and out1
+ paddd m3, m4 ; even part of out2 and out3
+ pshufd m0, m1, q1010
+ pshufd m1, m1, q3232
+ pmaddwd m4, m0, [row_13_23]
+ pmaddwd m0, [row_13_01]
+ pmaddwd m5, m1, [row_57_23]
+ pmaddwd m1, [row_57_01]
+ paddd m0, m1 ; odd part of out0 and out1
+ paddd m4, m5 ; odd part of out2 and out3
+ paddd m2, [pd_4]
+ paddd m3, [pd_4]
+ psubd m1, m2, m0 ; out7 out6
+ psubd m5, m3, m4 ; out5 out4
+ paddd m2, m0 ; out0 out1
+ paddd m3, m4 ; out2 out3
+ packssdw m2, m3 ; the clip to 16 bits
+ packssdw m5, m1
+ pshufd m5, m5, q2301 ; out4 out5 out6 out7
+ psraw m2, 3
+ psraw m5, 3
+%if %1 == 0
+ paddw m2, [pw_8_0] ; rounding of the vertical pass,
+ paddw m5, [pw_8_0] ; which multiplies line 0 by 8
%endif
- psllw m0, 3
- psllw m2, 3
- paddw m0, %2 ; add rounding bias
- paddw m2, %2 ; add rounding bias
+ mova [blockq+(%1)*16], m2
+ mova [blockq+(%1)*16+64], m5
+%endmacro
-%if ARCH_X86_64
- SUMSUB_BA w, 6, 0, 8 ; m6 = a4 + a6, m0 = a4 - a6
- SUMSUB_BA w, 4, 2, 8 ; m4 = a5 + a7, m2 = a5 - a7
- SUMSUB_BA w, 7, 6, 8 ; m7 = dst0, m6 = dst7
- SUMSUB_BA w, 5, 4, 8 ; m5 = dst1, m4 = dst6
- SUMSUB_BA w, 3, 2, 8 ; m3 = dst2, m2 = dst5
- SUMSUB_BA w, 1, 0, 8 ; m1 = dst3, m0 = dst4
-%else
- SUMSUB_BA w, 6, 0 ; m6 = a4 + a6, m0 = a4 - a6
- SUMSUB_BA w, 4, 2 ; m4 = a5 + a7, m2 = a5 - a7
- SUMSUB_BA w, 7, 6 ; m7 = dst0, m6 = dst7
- SUMSUB_BA w, 5, 4 ; m5 = dst1, m4 = dst6
- SUMSUB_BA w, 3, 2 ; m3 = dst2, m2 = dst5
- SUMSUB_BA w, 1, 0 ; m1 = dst3, m0 = dst4
-%endif
+; Even part of the vertical pass: m0 and m4 get %1 + %2 and %1 - %2 of
+; columns 0-3, m2 and m5 those of columns 4-7.
+%macro CAVS_IDCT8_EVEN 2
+ mova m0, [blockq+0*16]
+ mova m1, [blockq+2*16]
+ mova m2, [blockq+0*16+64]
+ mova m3, [blockq+2*16+64]
+ pmaddwd m0, [%1]
+ pmaddwd m1, [%2]
+ pmaddwd m2, [%1]
+ pmaddwd m3, [%2]
+ psubd m4, m0, m1
+ psubd m5, m2, m3
+ paddd m0, m1
+ paddd m2, m3
%endmacro
-INIT_XMM sse2
-cglobal cavs_idct8, 2, 2, 8 + ARCH_X86_64, 0 - 8 * 16, out, in
- CAVS_IDCT8_1D inq, [pw_4]
- psraw m7, 3
- psraw m6, 3
- psraw m5, 3
- psraw m4, 3
- psraw m3, 3
- psraw m2, 3
- psraw m1, 3
- psraw m0, 3
-%if ARCH_X86_64
- TRANSPOSE8x8W 7, 5, 3, 1, 0, 2, 4, 6, 8
- mova [rsp+4*16], m0
-%else
- mova [rsp+0*16], m4
- TRANSPOSE8x8W 7, 5, 3, 1, 0, 2, 4, 6, [rsp+0*16], [rsp+4*16], 1
-%endif
- mova [rsp+0*16], m7
- mova [rsp+2*16], m3
- mova [rsp+6*16], m4
- CAVS_IDCT8_1D rsp, [pw_64], 0
- psraw m7, 7
- psraw m6, 7
- psraw m5, 7
- psraw m4, 7
+; Add lines n and 7-n to the picture: even part in m%1 (columns 0-3) and
+; m%2 (columns 4-7), odd part weights %3 and %4, line addresses %5 and %6.
+%macro CAVS_IDCT8_OUT 6
+ mova m1, [blockq+1*16]
+ mova m3, [blockq+3*16]
+ pmaddwd m1, [%3]
+ pmaddwd m3, [%4]
+ paddd m1, m3
+ psubd m3, m%1, m1
+ paddd m%1, m1
+ mova m1, [blockq+1*16+64]
+ mova m6, [blockq+3*16+64]
+ pmaddwd m1, [%3]
+ pmaddwd m6, [%4]
+ paddd m1, m6
+ psubd m6, m%2, m1
+ paddd m%2, m1
+ packssdw m%1, m%2 ; the clip to 16 bits
+ packssdw m3, m6
+ psraw m%1, 7
psraw m3, 7
- psraw m2, 7
- psraw m1, 7
- psraw m0, 7
+ movq m1, [%5]
+ movq m6, [%6]
+ punpcklbw m1, [pb_0]
+ punpcklbw m6, [pb_0]
+ paddw m%1, m1
+ paddw m3, m6
+ packuswb m%1, m3
+ movq [%5], m%1
+ movhps [%6], m%1
+%endmacro
+
+; void ff_cavs_idct8_add(uint8_t *dst, int16_t *block, ptrdiff_t stride)
+INIT_XMM sse2
+cglobal cavs_idct8_add, 3, 5, 7, dst, block, stride
+ CAVS_IDCT8_ROWS 0
+ CAVS_IDCT8_ROWS 1
+ CAVS_IDCT8_ROWS 2
+ CAVS_IDCT8_ROWS 3
- mova [outq+0*16], m7
- mova [outq+1*16], m5
- mova [outq+2*16], m3
- mova [outq+3*16], m1
- mova [outq+4*16], m0
- mova [outq+5*16], m2
- mova [outq+6*16], m4
- mova [outq+7*16], m6
+ lea r3, [strideq*3]
+ lea r4, [dstq+strideq*4]
+ CAVS_IDCT8_EVEN pw_8_8, pw_10_4
+ CAVS_IDCT8_OUT 0, 2, pw_10_6, pw_9_2, dstq, r4+r3
+ CAVS_IDCT8_OUT 4, 5, pw_2_9, pw_m6_m10, dstq+r3, r4
+ CAVS_IDCT8_EVEN pw_8_m8, pw_4_m10
+ CAVS_IDCT8_OUT 0, 2, pw_9_m10, pw_m2_m6, dstq+strideq, r4+strideq*2
+ CAVS_IDCT8_OUT 4, 5, pw_6_2, pw_m10_9, dstq+strideq*2, r4+strideq
RET