Commit cd8d538fe7 for ffmpeg
commit cd8d538fe7af4450e129d87f71f25b730d53de41
Author: Ramiro Polla <ramiro.polla@gmail.com>
Date: Mon Nov 2 01:14:12 2026 +0100
avcodec/x86/cabac: improve ILP of the get_cabac renormalisation shift
Each decision computes its range from the range of the previous one, so
the instructions that go from one range to the next are the critical
path of get_cabac, and their latency limits decoding speed.
The renormalisation shift in the x86-64 PIC version was looked up from
the new range, so the lookup had to wait for the MPS/LPS decision.
Compute the shift for both outcomes in parallel instead, and select it
together with the new range. The LPS shift is looked up from RangeLPS.
The MPS part of the range is never below 128, so its shift is either 0
or 1, and two simple instructions compute it without a lookup:
before: shift = norm_shift[new_range]
after: lps_shift = norm_shift[RangeLPS]
mps_shift = (range - RangeLPS) < 256
shift = is_lps ? lps_shift : mps_shift
This needs another temporary register in the inline asm. The lzcnt
version of the renormalisation is removed, since the new code is faster
with lzcnt too.
Speedup of single-threaded H.264 decoding on an i9-13900K:
P-core E-core
1080p 3M +0.1% -0.1%
1080p 15M +1.1% +0.4%
1080p 30M +1.2% +0.3%
4K 150M +1.3% +0.2%
Geometric Mean +0.9% +0.2%
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Ramiro Polla <ramiro.polla@gmail.com>
diff --git a/libavcodec/x86/cabac.h b/libavcodec/x86/cabac.h
index 8407afb19c..3ddd876f5c 100644
--- a/libavcodec/x86/cabac.h
+++ b/libavcodec/x86/cabac.h
@@ -51,14 +51,11 @@
#ifdef BROKEN_RELOCATIONS
#define TABLES_ARG , [tables]"r"(tables)
-#define CABAC_TMP_DECL int cabac_t1;
-#define CABAC_TMP_OPERANDS , [cabac_t1]"=&r"(cabac_t1)
+#define CABAC_TMP_DECL int cabac_t1, cabac_t2;
+#define CABAC_TMP_OPERANDS , [cabac_t1]"=&r"(cabac_t1), [cabac_t2]"=&r"(cabac_t2)
#ifdef __LZCNT__
-/* renormalisation shift = lzcnt(range) - 23, refill shift = tzcnt(low) - 16 */
-#define BRANCHLESS_GET_CABAC_RENORM(range, rangeq, norm_off, tables) \
- "lzcnt "range" , %%ecx \n\t"\
- "sub $23 , %%ecx \n\t"
+/* refill shift = tzcnt(low) - 16 */
#define BRANCHLESS_GET_CABAC_REFILL(low, tmp, norm_off, tables) \
"movzwl (%%"FF_REG_c") , "tmp" \n\t"\
"tzcnt "low" , %%ecx \n\t"\
@@ -69,8 +66,6 @@
"shl %%cl , "tmp" \n\t"\
"add "tmp" , "low" \n\t"
#else /* __LZCNT__ */
-#define BRANCHLESS_GET_CABAC_RENORM(range, rangeq, norm_off, tables) \
- "movzbl "norm_off"("tables", "rangeq"), %%ecx \n\t"
#define BRANCHLESS_GET_CABAC_REFILL(low, tmp, norm_off, tables) \
"movzwl (%%"FF_REG_c") , "tmp" \n\t"\
"lea -1("low") , %%ecx \n\t"\
@@ -86,7 +81,7 @@
"add "tmp" , "low" \n\t"
#endif /* __LZCNT__ */
-#define BRANCHLESS_GET_CABAC(ret, retq, statep, low, lowword, range, rangeq, tmp, tmpbyte, t1, t1q, byte, end, norm_off, lps_off, mlps_off, tables) \
+#define BRANCHLESS_GET_CABAC(ret, retq, statep, low, lowword, range, rangeq, tmp, tmpbyte, t1, t1q, t2, byte, end, norm_off, lps_off, mlps_off, tables) \
"movzbl "statep" , "ret" \n\t"\
"mov "low" , "t1" \n\t"\
"lea ("tables", "retq"), %%rcx \n\t"\
@@ -96,14 +91,18 @@
"add $1 , "t1" \n\t"\
"movzbl "lps_off"(%%rcx, "rangeq", 2), "range" \n\t"\
"sub "range" , "tmp" \n\t"\
+ "movzbl "norm_off"("tables", "rangeq"), %%ecx \n\t"\
+ "mov "tmp" , "t2" \n\t"\
+ "shr $8 , "t2" \n\t"\
+ "xor $1 , "t2" \n\t"\
"cmp "t1" , "tmp" \n\t"\
"cmovae "tmp" , "range" \n\t"\
+ "cmovae "t2" , %%ecx \n\t"\
"sbb "t1q" , "t1q" \n\t"\
"shl $17 , "tmp" \n\t"\
"and "t1" , "tmp" \n\t"\
"xor "t1q" , "retq" \n\t"\
"sub "tmp" , "low" \n\t"\
- BRANCHLESS_GET_CABAC_RENORM(range, rangeq, norm_off, tables) \
"shl %%cl , "range" \n\t"\
"movzbl "mlps_off"+128("tables", "retq"), "tmp" \n\t"\
"shl %%cl , "low" \n\t"\
@@ -178,7 +177,7 @@
"add "tmp" , "low" \n\t"
#endif /* __LZCNT__ */
-#define BRANCHLESS_GET_CABAC(ret, retq, statep, low, lowword, range, rangeq, tmp, tmpbyte, t1, t1q, byte, end, norm_off, lps_off, mlps_off, tables) \
+#define BRANCHLESS_GET_CABAC(ret, retq, statep, low, lowword, range, rangeq, tmp, tmpbyte, t1, t1q, t2, byte, end, norm_off, lps_off, mlps_off, tables) \
"movzbl "statep" , "ret" \n\t"\
"mov "range" , "tmp" \n\t"\
"and $0xC0 , "range" \n\t"\
@@ -226,7 +225,7 @@ int get_cabac_inline_x86(CABACContext *c, uint8_t *const state)
__asm__ volatile(
BRANCHLESS_GET_CABAC("%[bit]", "%q[bit]", "(%[state])", "%[low]", "%w[low]",
"%[range]", "%q[range]", "%[tmp]", "%b[tmp]",
- "%[cabac_t1]", "%q[cabac_t1]",
+ "%[cabac_t1]", "%q[cabac_t1]", "%[cabac_t2]",
"%c[bs](%[c])", "%c[bsend](%[c])",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
diff --git a/libavcodec/x86/h264_cabac.c b/libavcodec/x86/h264_cabac.c
index 92f38ebe59..51d098cd7a 100644
--- a/libavcodec/x86/h264_cabac.c
+++ b/libavcodec/x86/h264_cabac.c
@@ -68,7 +68,7 @@ static int decode_significance_x86(CABACContext *c, int max_coeff,
BRANCHLESS_GET_CABAC("%[bit]", "%q[bit]", "(%[ctx])", "%[low]", "%w[low]",
"%[range]", "%q[range]", "%k[cnt]", "%b[cnt]",
- "%[cabac_t1]", "%q[cabac_t1]",
+ "%[cabac_t1]", "%q[cabac_t1]", "%[cabac_t2]",
"%c[bs](%[c])", "%c[bsend](%[c])",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
@@ -81,7 +81,7 @@ static int decode_significance_x86(CABACContext *c, int max_coeff,
BRANCHLESS_GET_CABAC("%[bit]", "%q[bit]", "(%[ctx])", "%[low]", "%w[low]",
"%[range]", "%q[range]", "%k[cnt]", "%b[cnt]",
- "%[cabac_t1]", "%q[cabac_t1]",
+ "%[cabac_t1]", "%q[cabac_t1]", "%[cabac_t2]",
"%c[bs](%[c])", "%c[bsend](%[c])",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
@@ -153,7 +153,7 @@ static int decode_significance_8x8_x86(CABACContext *c,
BRANCHLESS_GET_CABAC("%[bit]", "%q[bit]", "(%[state])", "%[low]", "%w[low]",
"%[range]", "%q[range]", "%k[cnt]", "%b[cnt]",
- "%[cabac_t1]", "%q[cabac_t1]",
+ "%[cabac_t1]", "%q[cabac_t1]", "%[cabac_t2]",
"%c[bs](%[c])", "%c[bsend](%[c])",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
@@ -173,7 +173,7 @@ static int decode_significance_8x8_x86(CABACContext *c,
BRANCHLESS_GET_CABAC("%[bit]", "%q[bit]", "(%[state])", "%[low]", "%w[low]",
"%[range]", "%q[range]", "%k[cnt]", "%b[cnt]",
- "%[cabac_t1]", "%q[cabac_t1]",
+ "%[cabac_t1]", "%q[cabac_t1]", "%[cabac_t2]",
"%c[bs](%[c])", "%c[bsend](%[c])",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),