Commit 6120b70d66 for ffmpeg
commit 6120b70d660f73dfcd5884d3b29bf65fe6b2005d
Author: Lynne <dev@lynne.ee>
Date: Tue Sep 29 00:19:37 2026 +0900
ffv1_vulkan: range code with subgroups smaller than 32 invocations
Range coded slices were decoded and encoded by a subgroup of exactly 32
invocations, one for each of the states of a context, and devices which
could not provide one were refused. The number of invocations now
follows the subgroup size: 32 if the subgroup size can be set to it,
and the largest subgroup size below it otherwise. Each invocation holds
32 divided by that number of the states of a context and of the bytes
of the decoder's window, and lines are coded in chunks of as many
samples. The encoder keeps its queue of 64 output bytes, and computes
the slice CRC over as many segments. Devices whose subgroups are all
wider than 32 invocations run the 32 invocations as part of one, and
are no longer refused either. The ballot quantizers need 32
invocations, and are only used with them.
With 32 invocations, the instructions executed per sample and the speed
of both the decoder and the encoder are unchanged. Decoding and encoding
were checked with 1 to 16 invocations on an RX 6900 XT, running the
workgroups as partial subgroups, and decoding with the subgroups of 8
invocations of lavapipe.
diff --git a/libavcodec/ffv1_vulkan.c b/libavcodec/ffv1_vulkan.c
index 9d61f35946..fb67cf0d07 100644
--- a/libavcodec/ffv1_vulkan.c
+++ b/libavcodec/ffv1_vulkan.c
@@ -82,6 +82,25 @@ static void set_rc_state_tab(FFV1Context *f, uint8_t *buf)
}
}
+uint32_t ff_ffv1_vk_rc_lanes(const FFVulkanContext *s, uint32_t *required)
+{
+ const VkPhysicalDeviceSubgroupSizeControlProperties *p = &s->subgroup_props;
+ uint32_t min = p->minSubgroupSize ? p->minSubgroupSize : s->props_11.subgroupSize;
+ uint32_t max = p->maxSubgroupSize ? p->maxSubgroupSize : s->props_11.subgroupSize;
+
+ /* A workgroup of the required size is a single, full subgroup */
+ if ((p->requiredSubgroupSizeStages & VK_SHADER_STAGE_COMPUTE_BIT) &&
+ min <= CONTEXT_SIZE) {
+ *required = FFMIN(max, CONTEXT_SIZE);
+ return *required;
+ }
+
+ /* Otherwise, a workgroup no larger than the smallest subgroup is part of
+ * a single one */
+ *required = 0;
+ return FFMIN(min, CONTEXT_SIZE);
+}
+
int ff_ffv1_vk_quant_ballot(const FFV1Context *f, FFv1QuantBallot *qb)
{
int ok = 1;
diff --git a/libavcodec/ffv1_vulkan.h b/libavcodec/ffv1_vulkan.h
index f4b6f167be..0428f44814 100644
--- a/libavcodec/ffv1_vulkan.h
+++ b/libavcodec/ffv1_vulkan.h
@@ -39,6 +39,15 @@ typedef struct FFv1QuantBallot {
int32_t scale_off[MAX_QUANT_TABLES][2];
} FFv1QuantBallot;
+/**
+ * Number of invocations that range code a slice together, as a single
+ * subgroup. Each of them holds CONTEXT_SIZE/lanes of the states of a
+ * context. This is CONTEXT_SIZE if the subgroup size can be set to it, or
+ * the largest subgroup size below it otherwise. required is set to the
+ * subgroup size the shader must be created with, or 0.
+ */
+uint32_t ff_ffv1_vk_rc_lanes(const FFVulkanContext *s, uint32_t *required);
+
/**
* Fill in the ballot quantizers of all quant tables.
* Returns 1 if all of them can be evaluated with a ballot.
diff --git a/libavcodec/ffv1enc_vulkan.c b/libavcodec/ffv1enc_vulkan.c
index 89bb4a50f7..699df37b6d 100644
--- a/libavcodec/ffv1enc_vulkan.c
+++ b/libavcodec/ffv1enc_vulkan.c
@@ -1108,10 +1108,11 @@ static int init_encode_shader(AVCodecContext *avctx, VkSpecializationInfo *sl)
FFV1Context *f = &fv->ctx;
FFVulkanShader *shd = &fv->enc;
- uint32_t wg_x = fv->ctx.ac != AC_GOLOMB_RICE ? CONTEXT_SIZE : 1;
+ uint32_t subgroup_size;
+ uint32_t lanes = ff_ffv1_vk_rc_lanes(&fv->s, &subgroup_size);
ff_vk_shader_load(shd, VK_SHADER_STAGE_COMPUTE_BIT, sl,
- (uint32_t []) { wg_x, 1, 1 },
- fv->ctx.ac != AC_GOLOMB_RICE ? CONTEXT_SIZE : 0);
+ (uint32_t []) { fv->ctx.ac != AC_GOLOMB_RICE ? lanes : 1, 1, 1 },
+ fv->ctx.ac != AC_GOLOMB_RICE ? subgroup_size : 0);
ff_vk_shader_add_push_const(shd, 0, sizeof(FFv1ShaderParams),
VK_SHADER_STAGE_COMPUTE_BIT);
@@ -1320,17 +1321,6 @@ static av_cold int vulkan_encode_ffv1_init(AVCodecContext *avctx)
return err;
}
- /* Range coded slices are encoded by one subgroup of CONTEXT_SIZE
- * invocations, each holding one of the states of a context */
- if (f->ac != AC_GOLOMB_RICE &&
- !(fv->s.subgroup_props.minSubgroupSize <= CONTEXT_SIZE &&
- fv->s.subgroup_props.maxSubgroupSize >= CONTEXT_SIZE &&
- (fv->s.subgroup_props.requiredSubgroupSizeStages & VK_SHADER_STAGE_COMPUTE_BIT))) {
- av_log(avctx, AV_LOG_ERROR, "Range coded FFv1 encoding needs subgroups "
- "of %i invocations\n", CONTEXT_SIZE);
- return AVERROR(ENOTSUP);
- }
-
maxsize = ffv1_vk_buffer_size(avctx);
if (maxsize > fv->s.props_11.maxMemoryAllocationSize) {
av_log(avctx, AV_LOG_WARNING, "Encoding buffer size (%zu) larger "
diff --git a/libavcodec/vulkan/ffv1_dec.comp.glsl b/libavcodec/vulkan/ffv1_dec.comp.glsl
index fbc587ce9d..c0c459cba0 100644
--- a/libavcodec/vulkan/ffv1_dec.comp.glsl
+++ b/libavcodec/vulkan/ffv1_dec.comp.glsl
@@ -98,8 +98,12 @@ void decode_line(ivec2 sp, int w,
uint ctx = abs(c);
uint sbase = state_off + gl_LocalInvocationID.x;
uint soff = sbase + CONTEXT_SIZE*ctx;
- uint ld = slice_rc_state[soff];
- uint8_t adapted = uint8_t(0);
+ uint ld[RC_K];
+ uint8_t adapted[RC_K];
+ [[unroll]] for (uint k = 0; k < RC_K; k++) {
+ ld[k] = slice_rc_state[soff + k*RC_LANES];
+ adapted[k] = uint8_t(0);
+ }
bool same = false;
uint row = 0;
ivec2 qthr = quant_ballot ? quant_thresh[quant_table_idx][gl_LocalInvocationID.x] : ivec2(0);
@@ -113,7 +117,7 @@ void decode_line(ivec2 sp, int w,
ivec4 tr = get_top(dec[p], sp, ivec2(min(1 + int(gl_LocalInvocationID.x), w - 1), y),
0, w, ext);
- for (int x = 0; x < w; x += 32) {
+ for (int x = 0; x < w; x += int(RC_LANES)) {
ivec3 tn;
if (slot >= 0) {
int tb = quant_top[slot][0][(tr[0] - tr[1]) & MAX_QUANT_TABLE_MASK] +
@@ -125,15 +129,18 @@ void decode_line(ivec2 sp, int w,
tn = get_pred_top_quant(tr, quant_table_idx, ext);
}
tn.z += qso.y;
- tr = get_top(dec[p], sp, ivec2(min(x + 33 + int(gl_LocalInvocationID.x), w - 1), y),
+ tr = get_top(dec[p], sp,
+ ivec2(min(x + int(RC_LANES) + 1 + int(gl_LocalInvocationID.x), w - 1), y),
0, w, ext);
int gmin = min(tn.y - tn.x, 0);
int gmax = max(tn.y - tn.x, 0);
- int n = min(w - x, 32);
+ int n = min(w - x, int(RC_LANES));
int j = 0;
do {
- uint st = same ? uint(adapted) : ld;
+ RCStates st;
+ [[unroll]] for (uint k = 0; k < RC_K; k++)
+ st.v[k] = same ? uint(adapted[k]) : ld[k];
int base = subgroupBroadcast(tn.z, j);
int t = subgroupBroadcast(tn.y, j);
@@ -159,12 +166,16 @@ void decode_line(ivec2 sp, int w,
soff = sbase + CONTEXT_SIZE*ctx;
same = ctx == ctx_prev;
if (!same)
- ld = slice_rc_state[soff];
+ [[unroll]] for (uint k = 0; k < RC_K; k++)
+ ld[k] = slice_rc_state[soff + k*RC_LANES];
rac_renorm();
- uint nst = zero_one_state[st + (subgroupInverseBallot(uvec4(used_bits, 0, 0, 0)) ? 256 : 0)];
-
- adapted = uint8_t(subgroupInverseBallot(uvec4(used, 0, 0, 0)) ? nst : st);
- slice_rc_state[soff_prev] = adapted;
+ [[unroll]] for (uint k = 0; k < RC_K; k++) {
+ bool one = subgroupInverseBallot(uvec4(used_bits >> (k*RC_LANES), 0, 0, 0));
+ uint nst = zero_one_state[st.v[k] + (one ? 256 : 0)];
+ bool read = subgroupInverseBallot(uvec4(used >> (k*RC_LANES), 0, 0, 0));
+ adapted[k] = uint8_t(read ? nst : st.v[k]);
+ slice_rc_state[soff_prev + k*RC_LANES] = adapted[k];
+ }
sgn = c < 0 ? -1 : 1;
int vm = int(TYPE(vz));
diff --git a/libavcodec/vulkan/ffv1_enc.comp.glsl b/libavcodec/vulkan/ffv1_enc.comp.glsl
index 177a9f6fe8..d45c32980b 100644
--- a/libavcodec/vulkan/ffv1_enc.comp.glsl
+++ b/libavcodec/vulkan/ffv1_enc.comp.glsl
@@ -110,7 +110,9 @@ void encode_line(in SliceContext sc, readonly uimage2D img, uint state_off,
int lane = int(gl_SubgroupInvocationID);
uint last_off = ~0u;
- uint adapted = 0;
+ uint adapted[RC_K];
+ [[unroll]] for (uint k = 0; k < RC_K; k++)
+ adapted[k] = 0;
ivec2 pc = ivec2(min(lane, w - 1), y);
uvec4 rcur, rT, rTR, rL, rTL, rLL = uvec4(0), rTT = uvec4(0);
@@ -139,7 +141,7 @@ void encode_line(in SliceContext sc, readonly uimage2D img, uint state_off,
}
#endif
- for (int x = 0; x < w; x += 32) {
+ for (int x = 0; x < w; x += int(RC_LANES)) {
int px = min(x + lane, w - 1);
int cur = int(TYPE(rcur[comp]));
int T = int(TYPE(rT[comp]));
@@ -182,16 +184,19 @@ void encode_line(in SliceContext sc, readonly uimage2D img, uint state_off,
uint ones = d == 0 ? 1u : ((2u << ec) - 2u) | ((ad & ((1u << ec) - 1u)) << 22) |
(uint(d < 0) << es);
- int n = min(w - x, 32);
+ int n = min(w - x, int(RC_LANES));
uint so_n = subgroupBroadcast(soff, 0);
int v_n = subgroupBroadcast(d, 0);
uint used_n = subgroupBroadcast(used, 0);
uint ones_n = subgroupBroadcast(ones, 0);
- uint st = uint(slice_rc_state[so_n + lane]);
- st = so_n == last_off ? adapted : st;
+ RCStates st;
+ [[unroll]] for (uint k = 0; k < RC_K; k++) {
+ st.v[k] = uint(slice_rc_state[so_n + lane + k*RC_LANES]);
+ st.v[k] = so_n == last_off ? adapted[k] : st.v[k];
+ }
- if (x + 32 < w) {
- pc = ivec2(min(x + 32 + lane, w - 1), y);
+ if (x + int(RC_LANES) < w) {
+ pc = ivec2(min(x + int(RC_LANES) + lane, w - 1), y);
#ifdef RGB
rcur = imageLoad(img, sp + LADDR(pc));
rT = imageLoad(img, sp + LADDR(pc + ivec2(0, -1)));
@@ -199,7 +204,8 @@ void encode_line(in SliceContext sc, readonly uimage2D img, uint state_off,
rL = imageLoad(img, sp + LADDR(pc + ivec2(-1, 0)));
rTL = imageLoad(img, sp + LADDR(pc + ivec2(-1, -1)));
if (enc_ext) {
- rLL = imageLoad(img, sp + LADDR(pc + ivec2(-2, 0)));
+ rLL = imageLoad(img, sp + LADDR(RC_LANES > 1 || pc.x > 1 ? pc + ivec2(-2, 0) :
+ ivec2(0, y - 1)));
rTT = imageLoad(img, sp + LADDR(pc + ivec2(0, -2)));
}
#else
@@ -209,7 +215,8 @@ void encode_line(in SliceContext sc, readonly uimage2D img, uint state_off,
rL = imageLoad(img, sp + pc + ivec2(-1, 0));
rTL = imageLoad(img, sp + ivec2(pc.x - 1, y1));
if (enc_ext) {
- rLL = imageLoad(img, sp + pc + ivec2(-2, 0));
+ rLL = RC_LANES > 1 || pc.x > 1 ? imageLoad(img, sp + pc + ivec2(-2, 0)) :
+ y > 0 ? imageLoad(img, sp + ivec2(0, y1)) : uvec4(0);
rTT = y > 1 ? imageLoad(img, sp + ivec2(pc.x, y2)) : uvec4(0);
}
#endif
@@ -222,25 +229,34 @@ void encode_line(in SliceContext sc, readonly uimage2D img, uint state_off,
int v = v_n;
uint used_j = used_n;
uint ones_j = ones_n;
- uint nst = zero_one_state[(bitfieldExtract(ones_j, lane, 1) << 8) + st];
+ uint nst[RC_K];
+ [[unroll]] for (uint k = 0; k < RC_K; k++) {
+ uint one = bitfieldExtract(ones_j, lane + int(k*RC_LANES), 1);
+ nst[k] = zero_one_state[(one << 8) + st.v[k]];
+ }
int jn = min(j + 1, n - 1);
so_n = subgroupBroadcast(soff, jn);
v_n = subgroupBroadcast(d, jn);
used_n = subgroupBroadcast(used, jn);
ones_n = subgroupBroadcast(ones, jn);
- uint ld = uint(slice_rc_state[so_n + lane]);
+ uint ld[RC_K];
+ [[unroll]] for (uint k = 0; k < RC_K; k++)
+ ld[k] = uint(slice_rc_state[so_n + lane + k*RC_LANES]);
uint s10, s31;
put_isymbol(st, v, s10, s31);
- adapted = bitfieldExtract(used_j, lane, 1) != 0 ? nst : st;
- if (abs(v) >= 1024)
- adapted = lane == 10 ? s10 : lane == 31 ? s31 : adapted;
+ [[unroll]] for (uint k = 0; k < RC_K; k++) {
+ int l = lane + int(k*RC_LANES);
+ adapted[k] = bitfieldExtract(used_j, l, 1) != 0 ? nst[k] : st.v[k];
+ if (abs(v) >= 1024)
+ adapted[k] = l == 10 ? s10 : l == 31 ? s31 : adapted[k];
- slice_rc_state[so + lane] = uint8_t(adapted);
+ slice_rc_state[so + uint(l)] = uint8_t(adapted[k]);
+ st.v[k] = so_n == so ? adapted[k] : ld[k];
+ }
last_off = so;
- st = so_n == so ? adapted : ld;
} while (++j < n && rc_nev <= 42);
if (rc_nev > 42)
rac_emit();
@@ -601,20 +617,22 @@ void finalize_slice(in uint slice_idx)
uint lane = gl_SubgroupInvocationID;
u8buf bs = u8buf(slice_data + rc.bs_start);
- if (lane < 3 + uint(has_crc))
- bs[enc_len + lane].v = uint8_t(lane < 3 ? enc_len >> (16 - 8*lane) : 0);
+ for (uint i = lane; i < 3 + uint(has_crc); i += RC_LANES)
+ bs[enc_len + i].v = uint8_t(i < 3 ? enc_len >> (16 - 8*i) : 0);
enc_len += 3 + uint(has_crc);
if (has_crc) {
controlBarrier(gl_ScopeWorkgroup, gl_ScopeWorkgroup,
gl_StorageSemanticsBuffer, gl_SemanticsAcquireRelease);
- uint seg = enc_len >> 5;
- uint len0 = enc_len - 31*seg;
+ uint seg = enc_len / RC_LANES;
+ uint len0 = enc_len - (RC_LANES - 1)*seg;
uint start = lane == 0 ? 0 : len0 + (lane - 1)*seg;
uint len = lane == 0 ? len0 : seg;
uint crc = lane == 0 ? crcref : 0;
- uint z = 1u << lane;
+ uint z[RC_K];
+ [[unroll]] for (uint c = 0; c < RC_K; c++)
+ z[c] = 1u << (lane + c*RC_LANES);
for (uint i = 0; i < len0; i += 8) {
uint b[8];
[[unroll]] for (uint k = 0; k < 8; k++)
@@ -623,19 +641,23 @@ void finalize_slice(in uint slice_idx)
if (i + k < len)
crc = crc_tab[(crc ^ b[k]) & 0xFF] ^ (crc >> 8);
if (i + k < seg)
- z = crc_tab[z & 0xFF] ^ (z >> 8);
+ [[unroll]] for (uint c = 0; c < RC_K; c++)
+ z[c] = crc_tab[z[c] & 0xFF] ^ (z[c] >> 8);
}
}
uint acc = subgroupBroadcast(crc, 0);
- for (uint i = 1; i < 32; i++)
- acc = subgroupXor(bitfieldExtract(acc, int(lane), 1) != 0 ? z : 0) ^
- subgroupBroadcast(crc, i);
+ for (uint i = 1; i < RC_LANES; i++) {
+ uint m = 0;
+ [[unroll]] for (uint c = 0; c < RC_K; c++)
+ m ^= bitfieldExtract(acc, int(lane + c*RC_LANES), 1) != 0 ? z[c] : 0;
+ acc = subgroupXor(m) ^ subgroupBroadcast(crc, i);
+ }
if (crcref != 0x00000000)
acc ^= 0x8CD88196;
- if (lane < 4)
- bs[enc_len + lane].v = uint8_t(acc >> (8*lane));
+ for (uint i = lane; i < 4; i += RC_LANES)
+ bs[enc_len + i].v = uint8_t(acc >> (8*i));
enc_len += 4;
}
diff --git a/libavcodec/vulkan/rangecoder_subgroup.glsl b/libavcodec/vulkan/rangecoder_subgroup.glsl
index f8ae202495..81db662a1b 100644
--- a/libavcodec/vulkan/rangecoder_subgroup.glsl
+++ b/libavcodec/vulkan/rangecoder_subgroup.glsl
@@ -31,6 +31,13 @@
#define CONTEXT_SIZE 32
#define MAX_OVERREAD 2
+#define RC_LANES gl_WorkGroupSize.x
+#define RC_K (CONTEXT_SIZE/RC_LANES)
+
+struct RCStates {
+ uint v[RC_K];
+};
+
#ifdef DECODE
#define RC_BTYPE readonly buffer
#else
@@ -67,12 +74,11 @@ uint rc_top;
uint rc_oc;
int rc_ob;
uint rc_nev;
-uint rc_ev;
-uint rc_ev2;
+uint rc_ev[64/RC_LANES];
void rac_init_enc(in RangeCoder c)
{
- for (uint i = gl_SubgroupInvocationID; i < 512; i += gl_SubgroupSize)
+ for (uint i = gl_SubgroupInvocationID; i < 512; i += RC_LANES)
zero_one_state[i] = rangecoder_state[i];
barrier();
@@ -81,8 +87,8 @@ void rac_init_enc(in RangeCoder c)
rc_oc = uint(rc.outstanding_count);
rc_ob = int(rc.outstanding_byte);
rc_nev = 0;
- rc_ev = 0;
- rc_ev2 = 0;
+ [[unroll]] for (uint k = 0; k < 64/RC_LANES; k++)
+ rc_ev[k] = 0;
}
void rac_emit_block(uint low, uint cnt)
@@ -108,7 +114,7 @@ void rac_emit_block(uint low, uint cnt)
uint carry = l >> 16;
if (lane == 0)
slice_data[rc.bs_off].v = uint8_t(uint(rc_ob) + carry);
- for (uint k = 0; k < rc_oc; k += 32)
+ for (uint k = 0; k < rc_oc; k += RC_LANES)
if (lane < rc_oc - k)
slice_data[rc.bs_off + 1 + k + lane].v = uint8_t(carry - 1u);
rc.bs_off += 1 + rc_oc;
@@ -121,11 +127,9 @@ void rac_emit_block(uint low, uint cnt)
void rac_emit(void)
{
- if (rc_nev == 0)
- return;
- rac_emit_block(rc_ev, min(rc_nev, 32u));
- if (rc_nev > 32)
- rac_emit_block(rc_ev2, rc_nev - 32);
+ [[unroll]] for (uint k = 0; k < 64/RC_LANES; k++)
+ if (rc_nev > k*RC_LANES)
+ rac_emit_block(rc_ev[k], min(rc_nev - k*RC_LANES, RC_LANES));
rc_nev = 0;
}
@@ -134,8 +138,8 @@ void rac_renorm_enc(void)
uint low = rc_top - rc.range;
rc.range <<= 8;
rc_top = ((low & 0xFFu) << 8) + rc.range;
- rc_ev = gl_SubgroupInvocationID == rc_nev ? low : rc_ev;
- rc_ev2 = gl_SubgroupInvocationID + 32 == rc_nev ? low : rc_ev2;
+ [[unroll]] for (uint k = 0; k < 64/RC_LANES; k++)
+ rc_ev[k] = gl_SubgroupInvocationID + k*RC_LANES == rc_nev ? low : rc_ev[k];
rc_nev++;
}
@@ -181,16 +185,16 @@ uint rac_terminate(void)
return rc.bs_off - rc.bs_start;
}
-void put_isymbol_tail(int e, uint st24, uint a, bool neg)
+void put_isymbol_tail(int e, RCStates st24, uint a, bool neg)
{
uint s[21];
- [[unroll]] for (int i = 0; i < 11; i++)
+ [[unroll]] for (uint i = 0; i < 11; i++)
if (i <= e + 1)
- s[i] = subgroupBroadcast(st24, i);
- [[unroll]] for (int i = 0; i < 9; i++)
+ s[i] = subgroupBroadcast(st24.v[i/RC_LANES], i % RC_LANES);
+ [[unroll]] for (uint i = 0; i < 9; i++)
if (i < e)
- s[11 + i] = subgroupBroadcast(st24, 22 + i);
- s[20] = subgroupBroadcast(st24, 11 + e);
+ s[11 + i] = subgroupBroadcast(st24.v[(22 + i)/RC_LANES], (22 + i) % RC_LANES);
+ s[20] = subgroupBroadcast(st24.v[(11 + e)/RC_LANES], (11 + e) % RC_LANES);
uint range = rc.range;
uint top = rc_top;
@@ -218,14 +222,14 @@ void put_isymbol_tail(int e, uint st24, uint a, bool neg)
put_rac(s[20], neg);
}
-void put_isymbol_esc(int e, uint st, uint a, bool neg, out uint s10, out uint s31)
+void put_isymbol_esc(int e, RCStates st, RCStates st24, uint a, bool neg,
+ out uint s10, out uint s31)
{
- uint st24 = st << 24;
uint s[10];
- [[unroll]] for (int i = 0; i < 10; i++)
- s[i] = subgroupBroadcast(st24, i);
- s10 = subgroupBroadcast(st, 10);
- s31 = subgroupBroadcast(st, 31);
+ [[unroll]] for (uint i = 0; i < 10; i++)
+ s[i] = subgroupBroadcast(st24.v[i/RC_LANES], i % RC_LANES);
+ s10 = subgroupBroadcast(st.v[10/RC_LANES], 10 % RC_LANES);
+ s31 = subgroupBroadcast(st.v[31/RC_LANES], 31 % RC_LANES);
put_rac(s[0], false);
[[unroll]] for (int i = 1; i < 10; i++)
put_rac(s[i], true);
@@ -256,16 +260,18 @@ void put_isymbol_esc(int e, uint st, uint a, bool neg, out uint s10, out uint s3
rac_emit();
[[unroll]] for (int i = 8; i >= 0; i--)
- s[i] = subgroupBroadcast(st24, 22 + i);
- uint ss = subgroupBroadcast(st24, 21);
+ s[i] = subgroupBroadcast(st24.v[(22 + i)/RC_LANES], (22 + i) % RC_LANES);
+ uint ss = subgroupBroadcast(st24.v[21/RC_LANES], 21 % RC_LANES);
[[unroll]] for (int i = 8; i >= 0; i--)
put_rac(s[i], bitfieldExtract(a, i, 1) != 0);
put_rac(ss, neg);
}
-void put_isymbol(uint st, int v, out uint s10, out uint s31)
+void put_isymbol(RCStates st, int v, out uint s10, out uint s31)
{
- uint st24 = st << 24;
+ RCStates st24;
+ [[unroll]] for (uint k = 0; k < RC_K; k++)
+ st24.v[k] = st.v[k] << 24;
uint a = abs(v);
int e = findMSB(a);
bool neg = v < 0;
@@ -299,16 +305,17 @@ void put_isymbol(uint st, int v, out uint s10, out uint s31)
else
put_isymbol_tail(9, st24, a, neg);
} else {
- put_isymbol_esc(e, st, a, neg, s10, s31);
+ put_isymbol_esc(e, st, st24, a, neg, s10, s31);
}
} else {
- put_rac(subgroupBroadcast(st24, 0), true);
+ put_rac(subgroupBroadcast(st24.v[0], 0), true);
}
}
#endif
#ifdef DECODE
uint rc_win;
+uint rc_winx[RC_K];
uint rc_dist;
uint rc_next;
uint rc_pos;
@@ -319,11 +326,15 @@ void rac_load_window(void)
rc_pos &= ~31u;
uint o = rc.bs_off + gl_SubgroupInvocationID;
rc_win = uint(~(o < rc.bs_end ? u8buf(uint64_t(slice_data) + o).v : uint8_t(0)));
+ [[unroll]] for (uint k = 1; k < RC_K; k++) {
+ o += RC_LANES;
+ rc_winx[k] = uint(~(o < rc.bs_end ? u8buf(uint64_t(slice_data) + o).v : uint8_t(0)));
+ }
}
void rac_init_dec(in RangeCoder c)
{
- for (uint i = gl_SubgroupInvocationID; i < 512; i += gl_SubgroupSize)
+ for (uint i = gl_SubgroupInvocationID; i < 512; i += RC_LANES)
zero_one_state[i] = rangecoder_state[i];
barrier();
@@ -344,7 +355,12 @@ void refill(void)
{
rc.range <<= 8;
rc_dist = (rc_dist << 8) | rc_next;
- rc_next = subgroupBroadcast(rc_win, ++rc_pos);
+ if (RC_K == 1)
+ rc_next = subgroupBroadcast(rc_win, ++rc_pos);
+ else if (++rc_pos < RC_LANES)
+ rc_next = subgroupBroadcast(rc_win, rc_pos);
+ else
+ rc_next = subgroupBroadcast(rc_winx[rc_pos/RC_LANES], rc_pos % RC_LANES);
}
void rac_renorm(void)
@@ -378,14 +394,14 @@ bool get_rac_equi(void)
return bit;
}
-int get_isymbol_tail(int e, uint range, uint range1, uint sx, int pred, int sgn,
+int get_isymbol_tail(int e, uint range, uint range1, RCStates sx, int pred, int sgn,
out uint read, out uint bits)
{
uint m[9];
[[unroll]] for (int k = 8; k >= 0; k--)
if (k + 1 < e)
- m[k] = subgroupBroadcast(sx, 22 + k);
- uint ss = subgroupBroadcast(sx, 10 + e);
+ m[k] = subgroupBroadcast(sx.v[(22 + k)/RC_LANES], (22 + k) % RC_LANES);
+ uint ss = subgroupBroadcast(sx.v[(10 + e)/RC_LANES], (10 + e) % RC_LANES);
rc.range = range - range1;
rc_dist -= range1;
@@ -412,12 +428,13 @@ int get_isymbol_tail(int e, uint range, uint range1, uint sx, int pred, int sgn,
const int AVERROR_INVALIDDATA = -0x41444E49;
-int get_isymbol_esc(inout uint st, uint sx, int pred, int sgn, out uint read, out uint bits)
+int get_isymbol_esc(inout RCStates st, RCStates sx, int pred, int sgn,
+ out uint read, out uint bits)
{
bool esc = c_bits > 10;
int n = 11;
if (esc) {
- uint s10 = subgroupBroadcast(st, 10);
+ uint s10 = subgroupBroadcast(st.v[10/RC_LANES], 10 % RC_LANES);
bool one;
[[dont_unroll]] do {
s10 = rangecoder_state[s10 + 256];
@@ -425,7 +442,7 @@ int get_isymbol_esc(inout uint st, uint sx, int pred, int sgn, out uint read, ou
rac_check_window();
one = get_rac(s10 << 24);
} while (one && n < 33);
- st = gl_SubgroupInvocationID == 10 ? s10 : st;
+ st.v[10/RC_LANES] = gl_SubgroupInvocationID == 10 % RC_LANES ? s10 : st.v[10/RC_LANES];
if (one) {
read = 0x7FFu;
@@ -434,7 +451,7 @@ int get_isymbol_esc(inout uint st, uint sx, int pred, int sgn, out uint read, ou
}
}
- uint s31 = subgroupBroadcast(st, 31);
+ uint s31 = subgroupBroadcast(st.v[31/RC_LANES], 31 % RC_LANES);
rac_check_window();
bool b = get_rac(s31 << 24);
uint a = b ? 0x3 : 0x2;
@@ -444,25 +461,30 @@ int get_isymbol_esc(inout uint st, uint sx, int pred, int sgn, out uint read, ou
b = get_rac(s31 << 24);
a = (a << 1) | uint(b);
}
- st = gl_SubgroupInvocationID == 31 ? s31 : st;
+ st.v[31/RC_LANES] = gl_SubgroupInvocationID == 31 % RC_LANES ? s31 : st.v[31/RC_LANES];
rac_check_window();
- [[unroll]] for (int k = 8; k >= 0; k--)
- a = (a << 1) | uint(get_rac(subgroupBroadcast(sx, 22 + k)));
+ [[unroll]] for (int k = 8; k >= 0; k--) {
+ uint sk = subgroupBroadcast(sx.v[(22 + k)/RC_LANES], (22 + k) % RC_LANES);
+ a = (a << 1) | uint(get_rac(sk));
+ }
- bool neg = get_rac_internal(rac_range1(rc.range, subgroupBroadcast(sx, 21)));
+ uint ss = subgroupBroadcast(sx.v[21/RC_LANES], 21 % RC_LANES);
+ bool neg = get_rac_internal(rac_range1(rc.range, ss));
int sa = int(a)*sgn;
read = 0xFFE007FFu;
bits = ((esc ? 0x1FFu : 0x3FFu) << 1) | ((a & 0x3FFu) << 22) | (uint(neg) << 21);
return neg ? pred - sa : pred + sa;
}
-int get_isymbol(inout uint st, int pred, int sgn, out uint read, out uint bits)
+int get_isymbol(inout RCStates st, int pred, int sgn, out uint read, out uint bits)
{
- uint st24 = st << 24;
+ RCStates sx;
+ [[unroll]] for (uint k = 0; k < RC_K; k++)
+ sx.v[k] = st.v[k] << 24;
uint s[11];
- [[unroll]] for (int i = 0; i < 11; i++)
- s[i] = subgroupBroadcast(st24, i);
+ [[unroll]] for (uint i = 0; i < 11; i++)
+ s[i] = subgroupBroadcast(sx.v[i/RC_LANES], i % RC_LANES);
read = 1u;
bits = 1u;
@@ -478,7 +500,6 @@ int get_isymbol(inout uint st, int pred, int sgn, out uint read, out uint bits)
uint lim = max(rc_dist, 0xffu);
int v;
- uint sx = st24;
while (true) {
uint skip;
[[unroll]] for (int i = 2; i < 6; i++)
@@ -584,9 +605,11 @@ int get_isymbol(inout uint st, int pred, int sgn, out uint read, out uint bits)
r[0] = rc.range;
r[1] = skip > 0 ? rc.range + skip - 1 : rac_range1(rc.range, s[1]);
range0 = 0;
- sx = subgroupInverseBallot(uvec4((2u << skip) - 2u, 0, 0, 0)) ? ~0u : sx;
- [[unroll]] for (int i = 2; i < 11; i++)
- s[i] = subgroupBroadcast(sx, i);
+ uint skipped = (2u << skip) - 2u;
+ [[unroll]] for (uint k = 0; k < RC_K; k++)
+ sx.v[k] = subgroupInverseBallot(uvec4(skipped >> (k*RC_LANES), 0, 0, 0)) ? ~0u : sx.v[k];
+ [[unroll]] for (uint i = 2; i < 11; i++)
+ s[i] = subgroupBroadcast(sx.v[i/RC_LANES], i % RC_LANES);
}
return v;
diff --git a/libavcodec/vulkan_ffv1.c b/libavcodec/vulkan_ffv1.c
index f9e5ce7449..f8f3050e19 100644
--- a/libavcodec/vulkan_ffv1.c
+++ b/libavcodec/vulkan_ffv1.c
@@ -708,14 +708,13 @@ static int init_decode_shader(FFV1Context *f, FFVulkanContext *s,
AVHWFramesContext *dec_frames_ctx,
AVHWFramesContext *out_frames_ctx,
VkSpecializationInfo *sl, int ac, int rgb,
- int bayer)
+ int bayer, uint32_t lanes, uint32_t subgroup_size)
{
int err;
- uint32_t wg_x = ac != AC_GOLOMB_RICE ? CONTEXT_SIZE : 1;
ff_vk_shader_load(shd, VK_SHADER_STAGE_COMPUTE_BIT, sl,
- (uint32_t []) { wg_x, 1, 1 },
- ac != AC_GOLOMB_RICE ? CONTEXT_SIZE : 0);
+ (uint32_t []) { ac != AC_GOLOMB_RICE ? lanes : 1, 1, 1 },
+ ac != AC_GOLOMB_RICE ? subgroup_size : 0);
ff_vk_shader_add_push_const(shd, 0, sizeof(FFv1ShaderParams),
VK_SHADER_STAGE_COMPUTE_BIT);
@@ -898,18 +897,6 @@ static int vk_decode_ffv1_init(AVCodecContext *avctx)
return err;
ctx = dec->shared_ctx;
- /* Range coded slices are decoded by one subgroup of CONTEXT_SIZE
- * invocations, each holding one of the states of a context */
- if (f->ac != AC_GOLOMB_RICE &&
- !(ctx->s.subgroup_props.minSubgroupSize <= CONTEXT_SIZE &&
- ctx->s.subgroup_props.maxSubgroupSize >= CONTEXT_SIZE &&
- (ctx->s.subgroup_props.requiredSubgroupSizeStages & VK_SHADER_STAGE_COMPUTE_BIT))) {
- av_log(avctx, AV_LOG_ERROR, "Range coded FFv1 decoding needs subgroups "
- "of %i invocations\n", CONTEXT_SIZE);
- err = AVERROR(ENOTSUP);
- goto fail;
- }
-
fv = ctx->sd_ctx = av_mallocz(sizeof(*fv));
if (!fv) {
err = AVERROR(ENOMEM);
@@ -940,7 +927,12 @@ static int vk_decode_ffv1_init(AVCodecContext *avctx)
if (f->ec && !!(avctx->err_recognition & AV_EF_CRCCHECK))
SPEC_LIST_ADD(sl, 1, 32, 1);
- if (f->ac != AC_GOLOMB_RICE) {
+ uint32_t subgroup_size;
+ uint32_t lanes = ff_ffv1_vk_rc_lanes(&ctx->s, &subgroup_size);
+
+ /* The ballot quantizers have a threshold for each of CONTEXT_SIZE
+ * invocations */
+ if (f->ac != AC_GOLOMB_RICE && lanes == CONTEXT_SIZE) {
FFv1QuantBallot qb;
if (ff_ffv1_vk_quant_ballot(f, &qb))
SPEC_LIST_ADD(sl, 20, 32, 1);
@@ -954,7 +946,8 @@ static int vk_decode_ffv1_init(AVCodecContext *avctx)
/* Decode shaders */
RET(init_decode_shader(f, &ctx->s, &ctx->exec_pool, &fv->decode,
- dctx, hwfc, sl, f->ac, is_rgb, f->bayer));
+ dctx, hwfc, sl, f->ac, is_rgb, f->bayer,
+ lanes, subgroup_size));
/* Init static data */
RET(ff_ffv1_vk_init_consts(&ctx->s, &fv->consts_buf, f));