Commit c145db272 for llama.cpp
commit c145db2728887880dfd706de56b740f1436bfd45
Author: Takamichi Ohara <tokinasin@tutanota.com>
Date: Tue Nov 17 17:40:23 2026 +0900
webgpu : use f32 while dequantizing to avoid FTZ (#32140)
diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl
index 457df07ff..978f2012b 100644
--- a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl
+++ b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_staging.tmpl
@@ -49,7 +49,7 @@ fn load_k_tile_block(local_x: u32, kv_count: u32, kv_tile: u32, k_head_offset: u
let row_offset = k_row * HEAD_DIM_QK;
let global_block_idx = k_head_offset + global_k_row * params.stride_k1 + block_k;
let block_byte_base = global_block_idx * K_BLOCK_SIZE_BYTES;
- let d = f16_from_u16(load_k_u16_at(block_byte_base));
+ let d = unpack2x16float(load_k_u16_at(block_byte_base)).x;
let thread_byte_offset = block_offset * K_BYTES_PER_THREAD;
let shmem_idx = row_offset + block_k * BLOCK_SIZE + thread_byte_offset;
for (var j = 0u; j < K_BYTES_PER_THREAD / K_BYTES_PER_INNER_LOOP; j += 1u) {
@@ -98,7 +98,7 @@ fn load_v_tile_block(local_x: u32, kv_count: u32, kv_tile: u32, v_head_offset: u
let row_offset = v_row * HEAD_DIM_V;
let global_block_idx = v_head_offset + global_v_row * params.stride_v1 + block_k;
let block_byte_base = global_block_idx * V_BLOCK_SIZE_BYTES;
- let d = f16_from_u16(load_v_u16_at(block_byte_base));
+ let d = unpack2x16float(load_v_u16_at(block_byte_base)).x;
let thread_byte_offset = block_offset * V_BYTES_PER_THREAD;
let shmem_idx = row_offset + block_k * BLOCK_SIZE + thread_byte_offset;
for (var j = 0u; j < V_BYTES_PER_THREAD / V_BYTES_PER_INNER_LOOP; j += 1u) {
diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl b/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl
index 1af6286ac..3e331b761 100644
--- a/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl
+++ b/ggml/src/ggml-webgpu/wgsl-shaders/mul_mat_decls.tmpl
@@ -98,13 +98,13 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3
let byte_in_block = (global_k_start % BLOCK_SIZE) / 8u;
let src0_idx = batch_offset + global_m * params.stride_01 + block_k;
let block_byte_base = src0_idx * BLOCK_SIZE_BYTES;
- let d = load_f16_at_src0(block_byte_base);
+ let d = load_f16_as_f32_at_src0(block_byte_base);
let q_byte = load_u32_at_src0(block_byte_base + 2u + byte_in_block) & 0xFFu;
for (var bit = 0u; bit < NQ; bit++) {
let global_k = global_k_start + bit;
if (global_k < params.k) {
- shmem[i + bit] = select(-d, d, ((q_byte >> bit) & 1u) != 0u);
+ shmem[i + bit] = f16(select(-d, d, ((q_byte >> bit) & 1u) != 0u));
}
}
}
@@ -123,11 +123,11 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3
if (global_m < params.m && global_k < params.k) {
let block_idx = batch_offset + global_m * params.stride_01 + global_k / BLOCK_SIZE;
let block_byte_base = block_idx * BLOCK_SIZE_BYTES;
- let d = load_f16_at_src0(block_byte_base);
+ let d = load_f16_as_f32_at_src0(block_byte_base);
let q_packed = load_u32_at_src0(block_byte_base + 2u + (global_k % BLOCK_SIZE) / 4u);
for (var k = 0u; k < NQ; k++) {
let q = (q_packed >> (2u * k)) & 3u;
- shmem[i + k] = (f16(q) - 1.0) * d;
+ shmem[i + k] = f16((f32(q) - 1.0) * d);
}
}
}
@@ -163,7 +163,7 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3
#if defined(INIT_SRC0_SHMEM_Q4_0)
let block_byte_base = src0_idx * 18u; // BLOCK_SIZE_BYTES = 18u;
- let d = load_f16_at_src0(block_byte_base);
+ let d = load_f16_as_f32_at_src0(block_byte_base);
// load NQ(16) weights
for (var j = 0u; j < BYTES_PER_THREAD / BYTES_PER_INNER_LOOP; j += 1) {
@@ -176,8 +176,8 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3
#if defined(INIT_SRC0_SHMEM_Q4_1)
let block_byte_base = src0_idx * 20u; // BLOCK_SIZE_BYTES = 20u;
let dm = unpack2x16float(load_u32_at_src0_aligned(block_byte_base));
- let d = f16(dm[0]);
- let m = f16(dm[1]);
+ let d = dm[0];
+ let m = dm[1];
// load NQ(16) weights
for (var j = 0u; j < BYTES_PER_THREAD / BYTES_PER_INNER_LOOP; j += 1) {
@@ -186,10 +186,10 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3
for (var k = 0u; k < BYTES_PER_INNER_LOOP; k++) {
let q_byte = get_byte(q_packed, k);
- let q_lo = f16(q_byte & 0xF) * d + m;
- let q_hi = f16((q_byte >> 4) & 0xF) * d + m;
- shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k] = q_lo;
- shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k + 16u] = q_hi;
+ let q_lo = f32(q_byte & 0xF) * d + m;
+ let q_hi = f32((q_byte >> 4) & 0xF) * d + m;
+ shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k] = f16(q_lo);
+ shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k + 16u] = f16(q_hi);
}
}
#endif // INIT_SRC0_SHMEM_Q4_1
@@ -197,7 +197,7 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3
#if defined(INIT_SRC0_SHMEM_Q5_0)
let block_byte_base = src0_idx * 22u; // BLOCK_SIZE_BYTES = 22u;
- let d = load_f16_at_src0(block_byte_base);
+ let d = load_f16_as_f32_at_src0(block_byte_base);
let qh_packed = load_u32_at_src0(block_byte_base + 2u);
// load NQ(16) weights
@@ -210,11 +210,11 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3
let byte_idx = block_offset * BYTES_PER_THREAD + j * BYTES_PER_INNER_LOOP + k;
let qh_hi = (qh_packed >> (byte_idx + 12u)) & 0x10;
- let q_hi = (f16(((q_byte >> 4) & 0xF) | qh_hi) - 16.0) * d;
+ let q_hi = (f32(((q_byte >> 4) & 0xF) | qh_hi) - 16.0) * d;
let qh_lo = ((qh_packed >> byte_idx) << 4) & 0x10;
- let q_lo = (f16((q_byte & 0xF) | qh_lo) - 16.0) * d;
- shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k] = q_lo;
- shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k + 16u] = q_hi;
+ let q_lo = (f32((q_byte & 0xF) | qh_lo) - 16.0) * d;
+ shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k] = f16(q_lo);
+ shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k + 16u] = f16(q_hi);
}
}
#endif // INIT_SRC0_SHMEM_Q5_0
@@ -223,8 +223,8 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3
let block_byte_base = src0_idx * 24u; // BLOCK_SIZE_BYTES = 24u;
let dm = unpack2x16float(load_u32_at_src0_aligned(block_byte_base));
- let d = f16(dm[0]);
- let m = f16(dm[1]);
+ let d = dm[0];
+ let m = dm[1];
let qh_packed = load_u32_at_src0_aligned(block_byte_base + 4u);
// load NQ(16) weights
@@ -237,18 +237,18 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3
let byte_idx = block_offset * BYTES_PER_THREAD + j * BYTES_PER_INNER_LOOP + k;
let qh_hi = (qh_packed >> (byte_idx + 12u)) & 0x10;
- let q_hi = f16(((q_byte >> 4) & 0xF) | qh_hi) * d + m;
+ let q_hi = f32(((q_byte >> 4) & 0xF) | qh_hi) * d + m;
let qh_lo = ((qh_packed >> byte_idx) << 4) & 0x10;
- let q_lo = f16((q_byte & 0xF) | qh_lo) * d + m;
- shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k] = q_lo;
- shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k + 16u] = q_hi;
+ let q_lo = f32((q_byte & 0xF) | qh_lo) * d + m;
+ shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k] = f16(q_lo);
+ shmem[shmem_idx + j * BYTES_PER_INNER_LOOP + k + 16u] = f16(q_hi);
}
}
#endif // INIT_SRC0_SHMEM_Q5_1
#if defined(INIT_SRC0_SHMEM_Q8_0)
let block_byte_base = src0_idx * 34u; // BLOCK_SIZE_BYTES = 34u;
- let d = load_f16_at_src0(block_byte_base);
+ let d = load_f16_as_f32_at_src0(block_byte_base);
// load NQ(16) weights
for (var j = 0u; j < BYTES_PER_THREAD / BYTES_PER_INNER_LOOP; j += 1) {
@@ -381,8 +381,8 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3
let dm_byte_base = block_byte_base + 80u;
let d_packed = unpack2x16float(load_u32_at_src0_aligned(dm_byte_base));
- let d = f16(d_packed[0]);
- let dmin = f16(d_packed[1]);
+ let d = d_packed[0];
+ let dmin = d_packed[1];
let chunk = k_in_block / 128u;
let pos_in_chunk = k_in_block % 32u;
@@ -391,19 +391,19 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3
// whole 2 bits (4 elems)
let qs_word = load_u32_at_src0_aligned(qs_byte_base + 32u * chunk + 1u * pos_in_chunk);
- let qs_vec4 = vec4<f16>(
- f16((qs_word >> (2u * shift_phase + 0u)) & 0x3u),
- f16((qs_word >> (2u * shift_phase + 8u)) & 0x3u),
- f16((qs_word >> (2u * shift_phase + 16u)) & 0x3u),
- f16((qs_word >> (2u * shift_phase + 24u)) & 0x3u),
+ let qs_vec4 = vec4<f32>(
+ f32((qs_word >> (2u * shift_phase + 0u)) & 0x3u),
+ f32((qs_word >> (2u * shift_phase + 8u)) & 0x3u),
+ f32((qs_word >> (2u * shift_phase + 16u)) & 0x3u),
+ f32((qs_word >> (2u * shift_phase + 24u)) & 0x3u),
);
let scale = load_byte_at_src0_aligned(scales_byte_base + sub_block);
- let dl = d * f16(scale & 0xFu);
- let ml = dmin * f16(scale >> 4u);
+ let dl = d * f32(scale & 0xFu);
+ let ml = dmin * f32(scale >> 4u);
- store_shmem_kquants(qs_vec4 * dl - ml, elem_idx);
+ store_shmem_kquants(vec4<f16>(qs_vec4 * dl - ml), elem_idx);
#endif // INIT_SRC0_SHMEM_Q2_K
#if defined(INIT_SRC0_SHMEM_Q3_K)
@@ -412,7 +412,7 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3
let qs_byte_base = block_byte_base + 32u;
let scales_byte_base = block_byte_base + 96u;
- let d_all = load_f16_at_src0(block_byte_base + 108u);
+ let d_all = load_f16_as_f32_at_src0(block_byte_base + 108u);
let chunk = k_in_block / 128u;
let pos_in_chunk = k_in_block % 32u;
@@ -424,29 +424,29 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3
// low 2 bits (4 elems)
let q_lo2_word = load_u32_at_src0(qs_byte_base + 32u * chunk + 1u * hmask_block);
- let q_lo2_vec4 = vec4<f16>(
- f16((q_lo2_word >> (2u * shift_phase + 0u)) & 3u),
- f16((q_lo2_word >> (2u * shift_phase + 8u)) & 3u),
- f16((q_lo2_word >> (2u * shift_phase + 16u)) & 3u),
- f16((q_lo2_word >> (2u * shift_phase + 24u)) & 3u)
+ let q_lo2_vec4 = vec4<f32>(
+ f32((q_lo2_word >> (2u * shift_phase + 0u)) & 3u),
+ f32((q_lo2_word >> (2u * shift_phase + 8u)) & 3u),
+ f32((q_lo2_word >> (2u * shift_phase + 16u)) & 3u),
+ f32((q_lo2_word >> (2u * shift_phase + 24u)) & 3u)
);
// high 1 bit (4 elems)
let q_hi1_word = load_u32_at_src0(hmask_byte_base + pos_in_chunk);
- let q_hi1_vec4 = vec4<f16>(
- f16(select(4.0, 0.0, ((q_hi1_word >> (1u * hmask_shift_phase + 0u)) & 1u) == 1u)),
- f16(select(4.0, 0.0, ((q_hi1_word >> (1u * hmask_shift_phase + 8u)) & 1u) == 1u)),
- f16(select(4.0, 0.0, ((q_hi1_word >> (1u * hmask_shift_phase + 16u)) & 1u) == 1u)),
- f16(select(4.0, 0.0, ((q_hi1_word >> (1u * hmask_shift_phase + 24u)) & 1u) == 1u))
+ let q_hi1_vec4 = vec4<f32>(
+ select(4.0, 0.0, ((q_hi1_word >> (1u * hmask_shift_phase + 0u)) & 1u) == 1u),
+ select(4.0, 0.0, ((q_hi1_word >> (1u * hmask_shift_phase + 8u)) & 1u) == 1u),
+ select(4.0, 0.0, ((q_hi1_word >> (1u * hmask_shift_phase + 16u)) & 1u) == 1u),
+ select(4.0, 0.0, ((q_hi1_word >> (1u * hmask_shift_phase + 24u)) & 1u) == 1u)
);
let q_vec4 = q_lo2_vec4 - q_hi1_vec4;
let scale_low4 = (load_byte_at_src0_aligned(scales_byte_base + (sub_block % 8u)) >> (4u * (sub_block / 8u))) & 0xFu;
let scale_hi2 = (load_byte_at_src0_aligned(scales_byte_base + 8u + (sub_block % 4u)) >> (2u * (sub_block / 4u))) & 3u;
- let dl = d_all * (f16((scale_hi2 << 4u) | scale_low4) - 32.0);
+ let dl = d_all * (f32((scale_hi2 << 4u) | scale_low4) - 32.0);
- store_shmem_kquants(dl * q_vec4, elem_idx);
+ store_shmem_kquants(vec4<f16>(dl * q_vec4), elem_idx);
#endif // INIT_SRC0_SHMEM_Q3_K
#if defined(INIT_SRC0_SHMEM_Q4_K)
@@ -456,8 +456,8 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3
let qs_byte_base = block_byte_base + 16u;
let dm = unpack2x16float(load_u32_at_src0_aligned(dm_byte_base));
- let d = f16(dm[0]);
- let dmin = f16(dm[1]);
+ let d = dm[0];
+ let dmin = dm[1];
let chunk = k_in_block / 64u;
let pos_in_chunk = (k_in_block % 64u) % 32u;
@@ -466,11 +466,11 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3
// whole 4 bits (4 elems)
let qs_word = load_u32_at_src0_aligned(qs_byte_base + 32u * chunk + 1u * pos_in_chunk);
- let qs_vec4 = vec4<f16>(
- f16((qs_word >> (4u * shift_phase + 0u)) & 0xFu),
- f16((qs_word >> (4u * shift_phase + 8u)) & 0xFu),
- f16((qs_word >> (4u * shift_phase + 16u)) & 0xFu),
- f16((qs_word >> (4u * shift_phase + 24u)) & 0xFu)
+ let qs_vec4 = vec4<f32>(
+ f32((qs_word >> (4u * shift_phase + 0u)) & 0xFu),
+ f32((qs_word >> (4u * shift_phase + 8u)) & 0xFu),
+ f32((qs_word >> (4u * shift_phase + 16u)) & 0xFu),
+ f32((qs_word >> (4u * shift_phase + 24u)) & 0xFu)
);
var sc: u32;
@@ -489,10 +489,10 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3
mn = (sc_min_lo >> 4u) | ((min_hi >> 6u) << 4u);
}
- let dl = d * f16(sc);
- let ml = dmin * f16(mn);
+ let dl = d * f32(sc);
+ let ml = dmin * f32(mn);
- store_shmem_kquants(dl * qs_vec4 - vec4(ml, ml, ml, ml), elem_idx);
+ store_shmem_kquants(vec4<f16>(dl * qs_vec4 - vec4(ml, ml, ml, ml)), elem_idx);
#endif // INIT_SRC0_SHMEM_Q4_K
#if defined(INIT_SRC0_SHMEM_Q5_K)
@@ -503,8 +503,8 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3
let qs_byte_base = block_byte_base + 48u;
let dm = unpack2x16float(load_u32_at_src0_aligned(dm_byte_base));
- let d = f16(dm[0]);
- let dmin = f16(dm[1]);
+ let d = dm[0];
+ let dmin = dm[1];
let chunk = k_in_block / 64u;
let pos_in_chunk = (k_in_block % 64u) % 32u;
@@ -516,20 +516,20 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3
// low 4 bits (4 elems)
let qs_word = load_u32_at_src0_aligned(qs_byte_base + 32u * chunk + 1u * pos_in_chunk);
- let qs_lo4_vec4 = vec4<f16>(
- f16((qs_word >> (4u * shift_phase + 0u)) & 0xFu),
- f16((qs_word >> (4u * shift_phase + 8u)) & 0xFu),
- f16((qs_word >> (4u * shift_phase + 16u)) & 0xFu),
- f16((qs_word >> (4u * shift_phase + 24u)) & 0xFu)
+ let qs_lo4_vec4 = vec4<f32>(
+ f32((qs_word >> (4u * shift_phase + 0u)) & 0xFu),
+ f32((qs_word >> (4u * shift_phase + 8u)) & 0xFu),
+ f32((qs_word >> (4u * shift_phase + 16u)) & 0xFu),
+ f32((qs_word >> (4u * shift_phase + 24u)) & 0xFu)
);
// high 1 bit (4 elems)
let qh_word = load_u32_at_src0_aligned(qh_byte_base + qh_block);
- let qh_vec4 = vec4<f16>(
- f16(select(0.0, 16.0, ((qh_word >> (1u * qh_shift_phase + 0u)) & 1u) == 1u)),
- f16(select(0.0, 16.0, ((qh_word >> (1u * qh_shift_phase + 8u)) & 1u) == 1u)),
- f16(select(0.0, 16.0, ((qh_word >> (1u * qh_shift_phase + 16u)) & 1u) == 1u)),
- f16(select(0.0, 16.0, ((qh_word >> (1u * qh_shift_phase + 24u)) & 1u) == 1u))
+ let qh_vec4 = vec4<f32>(
+ select(0.0, 16.0, ((qh_word >> (1u * qh_shift_phase + 0u)) & 1u) == 1u),
+ select(0.0, 16.0, ((qh_word >> (1u * qh_shift_phase + 8u)) & 1u) == 1u),
+ select(0.0, 16.0, ((qh_word >> (1u * qh_shift_phase + 16u)) & 1u) == 1u),
+ select(0.0, 16.0, ((qh_word >> (1u * qh_shift_phase + 24u)) & 1u) == 1u)
);
var sc: u32;
@@ -548,10 +548,10 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3
mn = (sc_min_lo >> 4u) | ((min_hi >> 6u) << 4u);
}
- let dl = d * f16(sc);
- let ml = dmin * f16(mn);
+ let dl = d * f32(sc);
+ let ml = dmin * f32(mn);
- store_shmem_kquants((qh_vec4 + qs_lo4_vec4) * dl - vec4<f16>(ml, ml, ml, ml), elem_idx);
+ store_shmem_kquants(vec4<f16>((qh_vec4 + qs_lo4_vec4) * dl - vec4<f32>(ml, ml, ml, ml)), elem_idx);
#endif // INIT_SRC0_SHMEM_Q5_K
#if defined(INIT_SRC0_SHMEM_Q6_K)
@@ -561,7 +561,7 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3
let scales_byte_base = block_byte_base + 192u;
let d_byte_base = block_byte_base + 208u;
- let d = load_f16_at_src0(d_byte_base);
+ let d = load_f16_as_f32_at_src0(d_byte_base);
let chunk = k_in_block / 128u;
let ql_pos_in_chunk = (k_in_block % 128u) % 64u;
@@ -588,13 +588,13 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3
((qh_word >> (2u * qh_shift_phase + 24u)) & 0x3u) << 4u,
);
- let q_vec4 = vec4<f16>(qh_hi2_vec4 | ql_lo4_vec4) - vec4<f16>(32.0, 32.0, 32.0, 32.0);
+ let q_vec4 = vec4<f32>(qh_hi2_vec4 | ql_lo4_vec4) - vec4<f32>(32.0, 32.0, 32.0, 32.0);
let scale_byte = scales_byte_base + 1u * sub_block;
let scale_word = load_u32_at_src0_aligned(scale_byte);
let scale = get_byte_i32(scale_word, scale_byte & 3u);
- store_shmem_kquants(d * q_vec4 * f16(scale), elem_idx);
+ store_shmem_kquants(vec4<f16>(d * q_vec4 * f32(scale)), elem_idx);
#endif // INIT_SRC0_SHMEM_Q6_K
}
}
@@ -626,17 +626,17 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3
let d_byte_base = block_byte_base + 0u;
let qs_byte_base = block_byte_base + 2u;
- let d = load_f16_at_src0(d_byte_base);
+ let d = load_f16_as_f32_at_src0(d_byte_base);
let id_qtr = (k_in_block % 16u) / 4u;
let shift_phase = k_in_block / 16u;
let qs_u32 = load_u32_at_src0(qs_byte_base + 4u * id_qtr);
- shmem[elem_idx + 0u] = d * f16(kvalues_iq4nl[(qs_u32 >> ( 0u + 4u * shift_phase)) & 0xFu]);
- shmem[elem_idx + 1u] = d * f16(kvalues_iq4nl[(qs_u32 >> ( 8u + 4u * shift_phase)) & 0xFu]);
- shmem[elem_idx + 2u] = d * f16(kvalues_iq4nl[(qs_u32 >> (16u + 4u * shift_phase)) & 0xFu]);
- shmem[elem_idx + 3u] = d * f16(kvalues_iq4nl[(qs_u32 >> (24u + 4u * shift_phase)) & 0xFu]);
+ shmem[elem_idx + 0u] = f16(d * f32(kvalues_iq4nl[(qs_u32 >> ( 0u + 4u * shift_phase)) & 0xFu]));
+ shmem[elem_idx + 1u] = f16(d * f32(kvalues_iq4nl[(qs_u32 >> ( 8u + 4u * shift_phase)) & 0xFu]));
+ shmem[elem_idx + 2u] = f16(d * f32(kvalues_iq4nl[(qs_u32 >> (16u + 4u * shift_phase)) & 0xFu]));
+ shmem[elem_idx + 3u] = f16(d * f32(kvalues_iq4nl[(qs_u32 >> (24u + 4u * shift_phase)) & 0xFu]));
}
}
#endif // INIT_SRC0_SHMEM_IQ4_NL
@@ -670,12 +670,12 @@ fn create_iq_gw4(dl: f32, gw: u32, shift_base: u32, delta: f32) -> vec4<f16> {
#endif
#if defined(INIT_SRC0_SHMEM_IQ4_XS)
-fn create_iq_gw4(dl: f16, qs_u32: u32, shift_phase: u32) -> vec4<f16> {
+fn create_iq_gw4(dl: f32, qs_u32: u32, shift_phase: u32) -> vec4<f16> {
return vec4<f16>(
- dl * f16(kvalues_iq4nl[(qs_u32 >> (4 * shift_phase + 0u)) & 0xFu]),
- dl * f16(kvalues_iq4nl[(qs_u32 >> (4 * shift_phase + 8u)) & 0xFu]),
- dl * f16(kvalues_iq4nl[(qs_u32 >> (4 * shift_phase + 16u)) & 0xFu]),
- dl * f16(kvalues_iq4nl[(qs_u32 >> (4 * shift_phase + 24u)) & 0xFu]),
+ f16(dl * f32(kvalues_iq4nl[(qs_u32 >> (4 * shift_phase + 0u)) & 0xFu])),
+ f16(dl * f32(kvalues_iq4nl[(qs_u32 >> (4 * shift_phase + 8u)) & 0xFu])),
+ f16(dl * f32(kvalues_iq4nl[(qs_u32 >> (4 * shift_phase + 16u)) & 0xFu])),
+ f16(dl * f32(kvalues_iq4nl[(qs_u32 >> (4 * shift_phase + 24u)) & 0xFu])),
);
}
#endif
@@ -775,7 +775,7 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3
let qs_byte_base = block_byte_base + 8u;
let d_scales_h = load_u32_at_src0_aligned(d_byte_base);
- let d = bitcast<vec2<f16>>(d_scales_h).x;
+ let d = unpack2x16float(d_scales_h).x;
let scales_h = d_scales_h >> 16u;
let sub_block = k_in_block / 32u;
@@ -784,7 +784,7 @@ fn init_shmem_src0(thread_id: u32, batch_offset: u32, offset_m: u32, k_outer: u3
let scales_l_u32 = load_u32_at_src0_aligned(scales_l_byte_base);
let ls_lo = (get_byte(scales_l_u32, sub_block / 2u) >> (4u * (sub_block % 2u))) & 0xFu;
let ls_hi = ((scales_h >> (2u * sub_block)) & 3u) << 4u;
- let dl = d * f16(i32(ls_lo | ls_hi) - 32);
+ let dl = d * f32(i32(ls_lo | ls_hi) - 32);
let qs_0_3_u32 = load_u32_at_src0_aligned(qs_byte_base + 16u * sub_block + 0u);
let qs_4_7_u32 = load_u32_at_src0_aligned(qs_byte_base + 16u * sub_block + 4u);
diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl b/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl
index d1da46084..008fa1f27 100644
--- a/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl
+++ b/ggml/src/ggml-webgpu/wgsl-shaders/quant_inner_loops.tmpl
@@ -1,21 +1,19 @@
#ifdef U32_DEQUANT_HELPERS
-fn dequant_q4_0_packed_to_shmem(q_packed: u32, d: f16, dst_idx: u32) {
- let scale = QUANT_OUT_TYPE(d);
+fn dequant_q4_0_packed_to_shmem(q_packed: u32, d: f32, dst_idx: u32) {
for (var k = 0u; k < 4u; k++) {
let q_byte = get_byte(q_packed, k);
- let q_hi = (QUANT_OUT_TYPE((q_byte >> 4) & 0xFu) - QUANT_OUT_TYPE(8.0)) * scale;
- let q_lo = (QUANT_OUT_TYPE(q_byte & 0xFu) - QUANT_OUT_TYPE(8.0)) * scale;
- QUANT_SHMEM[dst_idx + k] = q_lo;
- QUANT_SHMEM[dst_idx + k + 16u] = q_hi;
+ let q_hi = (f32((q_byte >> 4) & 0xFu) - 8.0) * d;
+ let q_lo = (f32(q_byte & 0xFu) - 8.0) * d;
+ QUANT_SHMEM[dst_idx + k] = QUANT_OUT_TYPE(q_lo);
+ QUANT_SHMEM[dst_idx + k + 16u] = QUANT_OUT_TYPE(q_hi);
}
}
-fn dequant_q8_0_packed_to_shmem(q_packed: u32, d: f16, dst_idx: u32) {
- let scale = QUANT_OUT_TYPE(d);
+fn dequant_q8_0_packed_to_shmem(q_packed: u32, d: f32, dst_idx: u32) {
for (var k = 0u; k < 4u; k++) {
let q_byte = get_byte_i32(q_packed, k);
- let q_val = QUANT_OUT_TYPE(q_byte) * scale;
- QUANT_SHMEM[dst_idx + k] = q_val;
+ let q_val = f32(q_byte) * d;
+ QUANT_SHMEM[dst_idx + k] = QUANT_OUT_TYPE(q_val);
}
}
#endif