Commit c8020cfc0d for ffmpeg
commit c8020cfc0da385e14ebddd398aa39f3c908bbcd7
Author: Ramiro Polla <ramiro.polla@gmail.com>
Date: Mon Nov 2 01:14:12 2026 +0100
avcodec/x86/cabac: improve ILP of the get_cabac MPS/LPS decision
Each decision computes its range from the range of the previous one, so
the instructions that go from one range to the next are the critical
path of get_cabac, and their latency limits decoding speed.
The MPS/LPS decision in the x86-64 PIC version compared low against the
MPS part of the range shifted left by 17, so the shift had to wait for
the range. Shift low right instead, which doesn't depend on the range,
and takes the shift off the critical path. The decision for MPS becomes:
before: low < (range - RangeLPS) << 17
after: (low >> 17) + 1 <= range - RangeLPS
The two only differ when low is exactly equal to the shifted MPS part,
which cannot happen because the 16 low bits of low are never all zero
(they hold a marker bit that every refill sets).
This needs one more temporary register in the inline asm.
Speedup of single-threaded H.264 decoding on an i9-13900K:
P-core E-core
1080p 3M +0.8% +0.8%
1080p 15M +1.9% +1.5%
1080p 30M +2.4% +2.2%
4K 150M +2.4% +1.7%
Geometric Mean +1.9% +1.6%
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Ramiro Polla <ramiro.polla@gmail.com>
diff --git a/libavcodec/x86/cabac.h b/libavcodec/x86/cabac.h
index 308c99a52d..8407afb19c 100644
--- a/libavcodec/x86/cabac.h
+++ b/libavcodec/x86/cabac.h
@@ -51,6 +51,8 @@
#ifdef BROKEN_RELOCATIONS
#define TABLES_ARG , [tables]"r"(tables)
+#define CABAC_TMP_DECL int cabac_t1;
+#define CABAC_TMP_OPERANDS , [cabac_t1]"=&r"(cabac_t1)
#ifdef __LZCNT__
/* renormalisation shift = lzcnt(range) - 23, refill shift = tzcnt(low) - 16 */
@@ -84,20 +86,22 @@
"add "tmp" , "low" \n\t"
#endif /* __LZCNT__ */
-#define BRANCHLESS_GET_CABAC(ret, retq, statep, low, lowword, range, rangeq, tmp, tmpbyte, byte, end, norm_off, lps_off, mlps_off, tables) \
+#define BRANCHLESS_GET_CABAC(ret, retq, statep, low, lowword, range, rangeq, tmp, tmpbyte, t1, t1q, byte, end, norm_off, lps_off, mlps_off, tables) \
"movzbl "statep" , "ret" \n\t"\
+ "mov "low" , "t1" \n\t"\
"lea ("tables", "retq"), %%rcx \n\t"\
+ "shr $17 , "t1" \n\t"\
"mov "range" , "tmp" \n\t"\
"and $0xC0 , "range" \n\t"\
+ "add $1 , "t1" \n\t"\
"movzbl "lps_off"(%%rcx, "rangeq", 2), "range" \n\t"\
"sub "range" , "tmp" \n\t"\
- "mov "tmp" , %%ecx \n\t"\
+ "cmp "t1" , "tmp" \n\t"\
+ "cmovae "tmp" , "range" \n\t"\
+ "sbb "t1q" , "t1q" \n\t"\
"shl $17 , "tmp" \n\t"\
- "cmp "low" , "tmp" \n\t"\
- "cmova %%ecx , "range" \n\t"\
- "sbb %%rcx , %%rcx \n\t"\
- "and %%ecx , "tmp" \n\t"\
- "xor %%rcx , "retq" \n\t"\
+ "and "t1" , "tmp" \n\t"\
+ "xor "t1q" , "retq" \n\t"\
"sub "tmp" , "low" \n\t"\
BRANCHLESS_GET_CABAC_RENORM(range, rangeq, norm_off, tables) \
"shl %%cl , "range" \n\t"\
@@ -114,6 +118,8 @@
"2: \n\t"
#else /* BROKEN_RELOCATIONS */
+#define CABAC_TMP_DECL
+#define CABAC_TMP_OPERANDS
#define TABLES_ARG NAMED_CONSTRAINTS_ARRAY_ADD(ff_h264_cabac_tables)
#if HAVE_FAST_CMOV
@@ -172,7 +178,7 @@
"add "tmp" , "low" \n\t"
#endif /* __LZCNT__ */
-#define BRANCHLESS_GET_CABAC(ret, retq, statep, low, lowword, range, rangeq, tmp, tmpbyte, byte, end, norm_off, lps_off, mlps_off, tables) \
+#define BRANCHLESS_GET_CABAC(ret, retq, statep, low, lowword, range, rangeq, tmp, tmpbyte, t1, t1q, byte, end, norm_off, lps_off, mlps_off, tables) \
"movzbl "statep" , "ret" \n\t"\
"mov "range" , "tmp" \n\t"\
"and $0xC0 , "range" \n\t"\
@@ -206,6 +212,7 @@ av_always_inline
int get_cabac_inline_x86(CABACContext *c, uint8_t *const state)
{
int bit, tmp;
+ CABAC_TMP_DECL
#ifdef BROKEN_RELOCATIONS
void *tables;
@@ -219,12 +226,14 @@ int get_cabac_inline_x86(CABACContext *c, uint8_t *const state)
__asm__ volatile(
BRANCHLESS_GET_CABAC("%[bit]", "%q[bit]", "(%[state])", "%[low]", "%w[low]",
"%[range]", "%q[range]", "%[tmp]", "%b[tmp]",
+ "%[cabac_t1]", "%q[cabac_t1]",
"%c[bs](%[c])", "%c[bsend](%[c])",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
AV_STRINGIFY(H264_MLPS_STATE_OFFSET),
"%[tables]")
: [bit]"=&r"(bit), [low]"+&r"(c->low), [range]"+&r"(c->range), [tmp]"=&q"(tmp)
+ CABAC_TMP_OPERANDS
: [state]"r"(state), [c]"r"(c),
[bs]"i"(offsetof(CABACContext, bytestream)),
[bsend]"i"(offsetof(CABACContext, bytestream_end))
diff --git a/libavcodec/x86/h264_cabac.c b/libavcodec/x86/h264_cabac.c
index 4f595ec05a..92f38ebe59 100644
--- a/libavcodec/x86/h264_cabac.c
+++ b/libavcodec/x86/h264_cabac.c
@@ -51,6 +51,7 @@ static int decode_significance_x86(CABACContext *c, int max_coeff,
int minusindex= 4-(intptr_t)index;
int bit;
x86_reg coeff_count;
+ CABAC_TMP_DECL
#ifdef BROKEN_RELOCATIONS
void *tables;
@@ -67,6 +68,7 @@ static int decode_significance_x86(CABACContext *c, int max_coeff,
BRANCHLESS_GET_CABAC("%[bit]", "%q[bit]", "(%[ctx])", "%[low]", "%w[low]",
"%[range]", "%q[range]", "%k[cnt]", "%b[cnt]",
+ "%[cabac_t1]", "%q[cabac_t1]",
"%c[bs](%[c])", "%c[bsend](%[c])",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
@@ -79,6 +81,7 @@ static int decode_significance_x86(CABACContext *c, int max_coeff,
BRANCHLESS_GET_CABAC("%[bit]", "%q[bit]", "(%[ctx])", "%[low]", "%w[low]",
"%[range]", "%q[range]", "%k[cnt]", "%b[cnt]",
+ "%[cabac_t1]", "%q[cabac_t1]",
"%c[bs](%[c])", "%c[bsend](%[c])",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
@@ -109,6 +112,7 @@ static int decode_significance_x86(CABACContext *c, int max_coeff,
"shr $2, %k[cnt] \n\t"
: [cnt]"=&q"(coeff_count), [ctx]"+r"(significant_coeff_ctx_base), [index]"+m"(index),
[low]"+&r"(c->low), [bit]"=&r"(bit), [range]"+&r"(c->range)
+ CABAC_TMP_OPERANDS
: [c]"r"(c), [minusstart]"m"(minusstart), [end]"m"(end), [minusindex]"m"(minusindex), [last_off]"m"(last_off),
[bs]"i"(offsetof(CABACContext, bytestream)),
[bsend]"i"(offsetof(CABACContext, bytestream_end))
@@ -127,6 +131,7 @@ static int decode_significance_8x8_x86(CABACContext *c,
x86_reg coeff_count;
x86_reg last=0;
x86_reg state;
+ CABAC_TMP_DECL
#ifdef BROKEN_RELOCATIONS
void *tables;
@@ -148,6 +153,7 @@ static int decode_significance_8x8_x86(CABACContext *c,
BRANCHLESS_GET_CABAC("%[bit]", "%q[bit]", "(%[state])", "%[low]", "%w[low]",
"%[range]", "%q[range]", "%k[cnt]", "%b[cnt]",
+ "%[cabac_t1]", "%q[cabac_t1]",
"%c[bs](%[c])", "%c[bsend](%[c])",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
@@ -167,6 +173,7 @@ static int decode_significance_8x8_x86(CABACContext *c,
BRANCHLESS_GET_CABAC("%[bit]", "%q[bit]", "(%[state])", "%[low]", "%w[low]",
"%[range]", "%q[range]", "%k[cnt]", "%b[cnt]",
+ "%[cabac_t1]", "%q[cabac_t1]",
"%c[bs](%[c])", "%c[bsend](%[c])",
AV_STRINGIFY(H264_NORM_SHIFT_OFFSET),
AV_STRINGIFY(H264_LPS_RANGE_OFFSET),
@@ -194,6 +201,7 @@ static int decode_significance_8x8_x86(CABACContext *c,
"shr $2, %k[cnt] \n\t"
: [cnt]"=&q"(coeff_count), [last]"+"REG64(last), [index]"+"REG64(index), [low]"+&r"(c->low),
[bit]"=&r"(bit), [range]"+&r"(c->range), [state]"=&r"(state)
+ CABAC_TMP_OPERANDS
: [c]"r"(c), [minusindex]"m"(minusindex), [sig_base]"m"(significant_coeff_ctx_base),
[sig_off]REG64(sig_off), [last_base]REG64(last_coeff_ctx_base),
[bs]"i"(offsetof(CABACContext, bytestream)),