Commit 658b215474 for ffmpeg
commit 658b21547452f3c3d2c0b5fceb9037f3e7db4dbd
Author: Ramiro Polla <ramiro.polla@gmail.com>
Date: Tue Nov 10 15:34:38 2026 +0100
avcodec/x86/cabac: use tzcnt for the get_cabac refill shift unconditionally
The refill shift is the number of trailing zeros in low minus 16. It was
computed with tzcnt only when the compiler targets a CPU with lzcnt, and
with a table lookup otherwise.
tzcnt is encoded as rep bsf, so CPUs without BMI1 execute it as bsf. bsf
gives the same result as tzcnt when its input is not zero, and low is
never zero here: its lowest set bit marks the position of the bits that
still have to be refilled, which is at least bit 16 when the refill is
needed. The C version relies on this too, since it uses ff_ctz(low).
The flags set by tzcnt and bsf differ, but they aren't used.
So use tzcnt on all CPUs.
Speedup of single-threaded H.264 decoding on an i9-13900K:
P-core E-core
1080p 3M +0.1% -0.2%
1080p 15M +0.3% +0.1%
1080p 30M +0.4% +0.3%
4K 150M +0.4% +0.4%
Geometric Mean +0.3% +0.1%
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Ramiro Polla <ramiro.polla@gmail.com>
diff --git a/libavcodec/x86/cabac.h b/libavcodec/x86/cabac.h
index 36a534cb78..810a1b0525 100644
--- a/libavcodec/x86/cabac.h
+++ b/libavcodec/x86/cabac.h
@@ -54,8 +54,11 @@
#define CABAC_TMP_DECL int cabac_t1, cabac_t2;
#define CABAC_TMP_OPERANDS , [cabac_t1]"=&r"(cabac_t1), [cabac_t2]"=&r"(cabac_t2)
-#ifdef __LZCNT__
-/* refill shift = tzcnt(low) - 16 */
+/*
+ * refill shift = tzcnt(low) - 16
+ * low is never 0 here, so this also works on CPUs without BMI1, which
+ * execute tzcnt as bsf.
+ */
#define BRANCHLESS_GET_CABAC_REFILL(low, tmp, norm_off, tables) \
"movzwl (%%"FF_REG_c") , "tmp" \n\t"\
"tzcnt "low" , %%ecx \n\t"\
@@ -65,21 +68,6 @@
"sub $0xFFFF , "tmp" \n\t"\
"shl %%cl , "tmp" \n\t"\
"add "tmp" , "low" \n\t"
-#else /* __LZCNT__ */
-#define BRANCHLESS_GET_CABAC_REFILL(low, tmp, norm_off, tables) \
- "movzwl (%%"FF_REG_c") , "tmp" \n\t"\
- "lea -1("low") , %%ecx \n\t"\
- "xor "low" , %%ecx \n\t"\
- "shr $15 , %%ecx \n\t"\
- "bswap "tmp" \n\t"\
- "shr $15 , "tmp" \n\t"\
- "movzbl "norm_off"("tables", %%rcx), %%ecx \n\t"\
- "sub $0xFFFF , "tmp" \n\t"\
- "neg %%ecx \n\t"\
- "add $7 , %%ecx \n\t"\
- "shl %%cl , "tmp" \n\t"\
- "add "tmp" , "low" \n\t"
-#endif /* __LZCNT__ */
#define BRANCHLESS_GET_CABAC(ret, retq, statep, low, lowword, range, rangeq, tmp, tmpbyte, t1, t1q, t2, byte, end, norm_off, lps_off, mlps_off, tables) \
"movzbl "statep" , "ret" \n\t"\
@@ -150,32 +138,19 @@
#define BRANCHLESS_GET_CABAC_RENORM(range, rangeq, norm_off, tables) \
"lzcnt "range" , %%ecx \n\t"\
"sub $23 , %%ecx \n\t"
-#define BRANCHLESS_GET_CABAC_REFILL(low, tmp, norm_off, tables) \
- "movzwl (%%"FF_REG_c") , "tmp" \n\t"\
- "tzcnt "low" , %%ecx \n\t"\
- "bswap "tmp" \n\t"\
- "shr $15 , "tmp" \n\t"\
- "sub $16 , %%ecx \n\t"\
- "sub $0xFFFF , "tmp" \n\t"\
- "shl %%cl , "tmp" \n\t"\
- "add "tmp" , "low" \n\t"
#else /* __LZCNT__ */
#define BRANCHLESS_GET_CABAC_RENORM(range, rangeq, norm_off, tables) \
"movzbl "MANGLE(ff_h264_cabac_tables)"+"norm_off"("range"), %%ecx \n\t"
+#endif /* __LZCNT__ */
#define BRANCHLESS_GET_CABAC_REFILL(low, tmp, norm_off, tables) \
"movzwl (%%"FF_REG_c") , "tmp" \n\t"\
- "lea -1("low") , %%ecx \n\t"\
- "xor "low" , %%ecx \n\t"\
- "shr $15 , %%ecx \n\t"\
+ "tzcnt "low" , %%ecx \n\t"\
"bswap "tmp" \n\t"\
"shr $15 , "tmp" \n\t"\
- "movzbl "MANGLE(ff_h264_cabac_tables)"+"norm_off"(%%ecx), %%ecx \n\t"\
+ "sub $16 , %%ecx \n\t"\
"sub $0xFFFF , "tmp" \n\t"\
- "neg %%ecx \n\t"\
- "add $7 , %%ecx \n\t"\
"shl %%cl , "tmp" \n\t"\
"add "tmp" , "low" \n\t"
-#endif /* __LZCNT__ */
#define BRANCHLESS_GET_CABAC(ret, retq, statep, low, lowword, range, rangeq, tmp, tmpbyte, t1, t1q, t2, byte, end, norm_off, lps_off, mlps_off, tables) \
"movzbl "statep" , "ret" \n\t"\