Commit 7a4117eef for llama.cpp
commit 7a4117eef97ed6419241da2384b2b1c011ecd763
Author: Birk Skyum <74932975+birkskyum@users.noreply.github.com>
Date: Thu Nov 19 07:15:09 2026 +0100
vulkan : enable MMQ with mixed signedness integer dots (#31045)
* vulkan : enable MMQ with mixed signedness integer dots
* vulkan: specialize MMQ shaders for mixed-signedness dot products
* vulkan: track integer dot product support with signed, mixed and unsigned flags
* vulkan: move signed dot check to the MMVQ call sites
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-types.h b/ggml/src/ggml-vulkan/ggml-vulkan-types.h
index f976c5795..6cc76b323 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan-types.h
+++ b/ggml/src/ggml-vulkan/ggml-vulkan-types.h
@@ -747,6 +747,10 @@ struct vk_device_struct {
bool shader_64b_indexing;
bool integer_dot_product;
+ // Which 4x8 bit packed integer dot products are accelerated
+ bool integer_dot_signed {};
+ bool integer_dot_mixed {};
+ bool integer_dot_unsigned {};
// 0: default, 1: force mmvq, -1: disable mmvq
int32_t mmvq_mode;
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index e279693cb..3674579cc 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -1726,6 +1726,11 @@ static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vec
return supported;
}
+// MMQ uses the mixed signedness dot product (MMQ_SUDOT) when the signed one is not accelerated
+static bool ggml_vk_mmq_use_sudot(const vk_device& device) {
+ return device->integer_dot_mixed && !device->integer_dot_signed;
+}
+
static bool ggml_vk_matmul_int_shmem_support(const vk_device& device, const std::vector<uint32_t>& warptile, bool mul_mat_id, ggml_type src0_type) {
// FLOAT_TYPE in the shader is float16_t with fp16 support, otherwise float.
@@ -1782,7 +1787,12 @@ static bool ggml_vk_matmul_int_shmem_support(const vk_device& device, const std:
const uint32_t buf_b_size = BN * BK_STEP * block_b_size;
const uint32_t mmid_row_ids = mul_mat_id ? (BN * 2u * (uint32_t)sizeof(uint16_t)) : 0u;
- const uint32_t total_size = buf_a_size + buf_b_size + mmid_row_ids + lut_size;
+ // Signed quants declare buf_b_qsum: 2 int32_t per B block with MMQ_SUDOT, a single int32_t without
+ const bool b_sums = src0_type == GGML_TYPE_Q8_0 || src0_type == GGML_TYPE_MXFP4 || src0_type == GGML_TYPE_IQ4_XS ||
+ src0_type == GGML_TYPE_IQ3_S || src0_type == GGML_TYPE_Q3_K || src0_type == GGML_TYPE_Q6_K;
+ const uint32_t qsum_size = !b_sums ? 0u : ggml_vk_mmq_use_sudot(device) ? BN * BK_STEP * 2u * 4u : 4u;
+
+ const uint32_t total_size = buf_a_size + buf_b_size + mmid_row_ids + lut_size + qsum_size;
const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize;
VK_LOG_DEBUG("ggml_vk_matmul_int_shmem_support(warptile=(" << warptile[0] << "," << warptile[1] << "," << warptile[2] << "), "
@@ -1906,7 +1916,7 @@ static bool ggml_vk_fa_type_needs_shmem(ggml_type type) {
static bool ggml_vk_fa_scalar_uses_mmq(const vk_device& device, ggml_type k_type, ggml_type v_type) {
#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT)
- return device->integer_dot_product && device->subgroup_clustered &&
+ return device->integer_dot_product && device->integer_dot_signed && device->subgroup_clustered &&
!ggml_vk_fa_type_needs_shmem(v_type) &&
(k_type == GGML_TYPE_Q4_0 || k_type == GGML_TYPE_Q4_1 ||
k_type == GGML_TYPE_Q5_0 || k_type == GGML_TYPE_Q5_1 ||
@@ -2952,8 +2962,13 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
uint32_t rsgs = 0) {
auto tc = filter_tc(tc_base, key.type_a, key.mul_mat_id, true);
if (!tc.empty()) {
- spec_fn_t identity = [](const std::vector<uint32_t>& wt, bool) { return wt; };
- create_mm_pipelines(key, tc, name, len, data, pc_size, pc, identity, false, rsgs > 0, rsgs, false);
+ // constant_id 11 is MMQ_SUDOT
+ spec_fn_t mmq_spec = [&](const std::vector<uint32_t>& wt, bool) {
+ std::vector<uint32_t> spec = wt;
+ spec.push_back(ggml_vk_mmq_use_sudot(device) ? 1u : 0u);
+ return spec;
+ };
+ create_mm_pipelines(key, tc, name, len, data, pc_size, pc, mmq_spec, false, rsgs > 0, rsgs, false);
}
};
#endif
@@ -4744,7 +4759,10 @@ vk_device ggml_vk_get_device(size_t idx) {
device->coopmat_support = false;
}
- device->integer_dot_product = device->integer_dot_product && shader_integer_dot_product_props.integerDotProduct4x8BitPackedSignedAccelerated;
+ device->integer_dot_signed = shader_integer_dot_product_props.integerDotProduct4x8BitPackedSignedAccelerated;
+ device->integer_dot_mixed = shader_integer_dot_product_props.integerDotProduct4x8BitPackedMixedSignednessAccelerated;
+ device->integer_dot_unsigned = shader_integer_dot_product_props.integerDotProduct4x8BitPackedUnsignedAccelerated;
+ device->integer_dot_product = device->integer_dot_product && (device->integer_dot_signed || device->integer_dot_mixed);
device->min_imported_host_pointer_alignment = external_memory_host_props.minImportedHostPointerAlignment;
@@ -5623,7 +5641,8 @@ static void ggml_vk_print_gpu_info(size_t idx) {
const bool rebar = ggml_vk_has_rebar(physical_device);
integer_dot_product = integer_dot_product
- && shader_integer_dot_product_props.integerDotProduct4x8BitPackedSignedAccelerated
+ && (shader_integer_dot_product_props.integerDotProduct4x8BitPackedSignedAccelerated ||
+ shader_integer_dot_product_props.integerDotProduct4x8BitPackedMixedSignednessAccelerated)
&& shader_integer_dot_product_features.shaderIntegerDotProduct;
coopmat_support = coopmat_support
@@ -7395,7 +7414,7 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context&
const ggml_type y_kernel_type = widen_bf16 ? GGML_TYPE_F32 : src1->type;
const bool f16_f32_kernel = y_kernel_type == GGML_TYPE_F32;
- bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && (swap_inputs || ggml_vk_src1_prec_allows_q8(dst)) && ggml_vk_should_use_mmvq(ctx->device, ne01, ne11, ne10, src0->type);
+ bool quantize_y = ctx->device->integer_dot_signed && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && (swap_inputs || ggml_vk_src1_prec_allows_q8(dst)) && ggml_vk_should_use_mmvq(ctx->device, ne01, ne11, ne10, src0->type);
vk_pipeline to_fp16_vk_0 = nullptr;
vk_pipeline to_fp16_vk_1 = nullptr;
@@ -8463,7 +8482,7 @@ static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_conte
const bool y_non_contig = !ggml_vk_dim01_contiguous(src1);
const bool f16_f32_kernel = src1->type == GGML_TYPE_F32;
- bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && ggml_vk_src1_prec_allows_q8(dst) && ggml_vk_should_use_mmvq(ctx->device, ne01, ne12, ne10, src0->type);
+ bool quantize_y = ctx->device->integer_dot_signed && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && ggml_vk_src1_prec_allows_q8(dst) && ggml_vk_should_use_mmvq(ctx->device, ne01, ne12, ne10, src0->type);
vk_pipeline to_fp16_vk_0 = nullptr;
vk_pipeline to_fp16_vk_1 = nullptr;
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp
index 1866386a7..f831fd788 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp
@@ -82,6 +82,8 @@ layout (constant_id = 7) const uint TM = 4;
layout (constant_id = 8) const uint TN = 2;
layout (constant_id = 9) const uint TK = 1; // Only needed for coopmat
layout (constant_id = 10) const uint WARP = 32;
+// Use the mixed signedness dot product, for devices that accelerate only that one
+layout (constant_id = 11) const bool MMQ_SUDOT = false;
#define BK 32
@@ -102,6 +104,13 @@ shared block_b_cache buf_b[BN * BK_STEP];
block_a_cache cache_a[WMITER * TM];
block_b_cache cache_b;
+#if defined(DATA_A_Q8_0) || defined(DATA_A_MXFP4) || defined(DATA_A_IQ4_XS) || defined(DATA_A_IQ3_S) || defined(DATA_A_Q3_K) || defined(DATA_A_Q6_K)
+// These quants are signed. MMQ_SUDOT gives them a bias and uses the sum of each half of the q8_1 block to remove it.
+#define MMQ_B_SUMS
+shared int32_t buf_b_qsum[MMQ_SUDOT ? BN * BK_STEP * 2 : 1];
+int32_t cache_b_qsum[2];
+#endif
+
#define LOAD_VEC_A (4 * QUANT_R_MMQ)
#define LOAD_VEC_B 16
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl
index d9f2f5748..9d94fd2b7 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl
@@ -4,6 +4,19 @@
#include "types.glsl"
+int32_t mmq_dot(const int32_t a, const int32_t b) {
+ // With MMQ_SUDOT the A quants must be unsigned
+ if (MMQ_SUDOT) {
+ return dotPacked4x8EXT(uint32_t(a), b);
+ }
+ return dotPacked4x8EXT(a, b);
+}
+
+// With MMQ_SUDOT, flip the sign bit of each byte. This adds a bias of 128 to signed quants.
+int32_t mmq_a_store(const int32_t v) {
+ return MMQ_SUDOT ? v ^ int32_t(0x80808080) : v;
+}
+
// Each iqs value maps to a 32-bit integer
#if defined(DATA_A_Q2_0)
@@ -41,7 +54,7 @@ void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
ACC_TYPE mmq_dot_product(const uint ib_a) {
int32_t q_sum = 0;
[[unroll]] for (uint iqs = 0; iqs < 8; ++iqs) {
- q_sum += dotPacked4x8EXT(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]);
+ q_sum += mmq_dot(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]);
}
return ACC_TYPE(float(cache_a[ib_a].dm) * (float(q_sum) * float(cache_b.ds.x) - float(cache_b.ds.y)));
@@ -102,8 +115,8 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
const int32_t qs_b0 = cache_b.qs[iqs];
const int32_t qs_b1 = cache_b.qs[iqs + 4];
- q_sum += dotPacked4x8EXT(qs_a.x, qs_b0);
- q_sum += dotPacked4x8EXT(qs_a.y, qs_b1);
+ q_sum += mmq_dot(qs_a.x, qs_b0);
+ q_sum += mmq_dot(qs_a.y, qs_b1);
}
#ifdef DATA_A_Q4_0
@@ -176,8 +189,8 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
const int32_t qs_b0 = cache_b.qs[iqs];
const int32_t qs_b1 = cache_b.qs[iqs + 4];
- q_sum += dotPacked4x8EXT(qs_a0, qs_b0);
- q_sum += dotPacked4x8EXT(qs_a1, qs_b1);
+ q_sum += mmq_dot(qs_a0, qs_b0);
+ q_sum += mmq_dot(qs_a1, qs_b1);
}
#ifdef DATA_A_Q5_0
@@ -192,8 +205,8 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
// 2-byte loads for Q8_0 blocks (34 bytes)
void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bool is_in_bounds) {
if (is_in_bounds) {
- buf_a[buf_ib].qs[iqs] = pack32(i16vec2(data_a_packed16[ib].qs[iqs * 2],
- data_a_packed16[ib].qs[iqs * 2 + 1]));
+ buf_a[buf_ib].qs[iqs] = mmq_a_store(pack32(i16vec2(data_a_packed16[ib].qs[iqs * 2],
+ data_a_packed16[ib].qs[iqs * 2 + 1])));
if (iqs == 0) {
buf_a[buf_ib].dm = FLOAT_TYPE(data_a_packed16[ib].d);
@@ -221,7 +234,10 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
const int32_t qs_a = cache_a[ib_a].qs[iqs];
const int32_t qs_b = cache_b.qs[iqs];
- q_sum += dotPacked4x8EXT(qs_a, qs_b);
+ q_sum += mmq_dot(qs_a, qs_b);
+ }
+ if (MMQ_SUDOT) {
+ q_sum -= 128 * (cache_b_qsum[0] + cache_b_qsum[1]);
}
return ACC_TYPE(float(q_sum) * float(cache_a[ib_a].dm) * float(cache_b.ds.x));
@@ -240,8 +256,8 @@ void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bo
const u8vec4 i_a0 = unpack8( qs & 0x0F0F0F0F);
const u8vec4 i_a1 = unpack8((qs >> 4) & 0x0F0F0F0F);
- buf_a[buf_ib].qs[iqs ] = pack32(i8vec4(kvalues_mxfp4[i_a0.x], kvalues_mxfp4[i_a0.y], kvalues_mxfp4[i_a0.z], kvalues_mxfp4[i_a0.w]));
- buf_a[buf_ib].qs[iqs + 4] = pack32(i8vec4(kvalues_mxfp4[i_a1.x], kvalues_mxfp4[i_a1.y], kvalues_mxfp4[i_a1.z], kvalues_mxfp4[i_a1.w]));
+ buf_a[buf_ib].qs[iqs ] = mmq_a_store(pack32(i8vec4(kvalues_mxfp4[i_a0.x], kvalues_mxfp4[i_a0.y], kvalues_mxfp4[i_a0.z], kvalues_mxfp4[i_a0.w])));
+ buf_a[buf_ib].qs[iqs + 4] = mmq_a_store(pack32(i8vec4(kvalues_mxfp4[i_a1.x], kvalues_mxfp4[i_a1.y], kvalues_mxfp4[i_a1.z], kvalues_mxfp4[i_a1.w])));
if (iqs == 0) {
buf_a[buf_ib].d = FLOAT_TYPE(e8m0_to_fp32(data_a[ib].e) * 0.5);
@@ -269,7 +285,10 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
[[unroll]] for (uint iqs = 0; iqs < 8; iqs++) {
const int32_t qs_a = cache_a[ib_a].qs[iqs];
- q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
+ q_sum += mmq_dot(qs_a, cache_b.qs[iqs]);
+ }
+ if (MMQ_SUDOT) {
+ q_sum -= 128 * (cache_b_qsum[0] + cache_b_qsum[1]);
}
return ACC_TYPE(float(cache_a[ib_a].d) * float(cache_b.ds.x) * float(q_sum));
@@ -284,8 +303,8 @@ void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bo
const uint32_t vui = data_a_packed32[ib_k].qs[4 * ib32 + iqs];
const i32vec2 qs = iq4nl_to_i8x8(vui);
- buf_a[buf_ib].qs[iqs ] = qs.x;
- buf_a[buf_ib].qs[iqs + 4] = qs.y;
+ buf_a[buf_ib].qs[iqs ] = mmq_a_store(qs.x);
+ buf_a[buf_ib].qs[iqs + 4] = mmq_a_store(qs.y);
if (iqs == 0) {
const uint sl = (data_a_packed32[ib_k].scales_l >> (4 * ib32)) & 0xF;
@@ -313,7 +332,10 @@ void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
ACC_TYPE mmq_dot_product(const uint ib_a) {
int32_t q_sum = 0;
[[unroll]] for (uint iqs = 0; iqs < 8; iqs++) {
- q_sum += dotPacked4x8EXT(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]);
+ q_sum += mmq_dot(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]);
+ }
+ if (MMQ_SUDOT) {
+ q_sum -= 128 * (cache_b_qsum[0] + cache_b_qsum[1]);
}
return ACC_TYPE(float(cache_a[ib_a].d) * float(cache_b.ds.x) * float(q_sum));
@@ -372,8 +394,8 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
const int32_t scale_m = int32_t(scale >> 4) * 0x01010101; // Duplicate 8-bit value across 32-bits.
const int32_t qs_a = int32_t((cache_a[ib_a].qs[iqs / 4] >> ((iqs % 4) * 2)) & 0x03030303);
- sum_d += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]) * (scale & 0xF);
- sum_m += dotPacked4x8EXT(scale_m, cache_b.qs[iqs]);
+ sum_d += mmq_dot(qs_a, cache_b.qs[iqs]) * (scale & 0xF);
+ sum_m += mmq_dot(scale_m, cache_b.qs[iqs]);
}
return ACC_TYPE(float(cache_b.ds.x) * (float(cache_a[ib_a].dm.x) * float(sum_d) - float(cache_a[ib_a].dm.y) * float(sum_m)));
@@ -435,18 +457,26 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
int32_t q_sum = 0;
[[unroll]] for (uint iqs = 0; iqs < 4; iqs++) {
- // Subtract 4 from the quants to correct the 3rd bit offset
- const int32_t qs_a = pack32(unpack8(int32_t((cache_a[ib_a].qs[iqs / 2] >> ((iqs % 2) * 4)) & 0x0F0F0F0F)) - int8_t(4));
+ // Subtract 4 from the quants to correct the 3rd bit offset. MMQ_SUDOT keeps them unsigned and subtracts 4 times the B sum.
+ const int32_t qs_u = int32_t((cache_a[ib_a].qs[iqs / 2] >> ((iqs % 2) * 4)) & 0x0F0F0F0F);
+ const int32_t qs_a = MMQ_SUDOT ? qs_u : pack32(unpack8(qs_u) - int8_t(4));
- q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
+ q_sum += mmq_dot(qs_a, cache_b.qs[iqs]);
+ }
+ if (MMQ_SUDOT) {
+ q_sum -= 4 * cache_b_qsum[0];
}
result += float(cache_a[ib_a].d_scales[0]) * float(q_sum);
q_sum = 0;
[[unroll]] for (uint iqs = 4; iqs < 8; iqs++) {
- const int32_t qs_a = pack32(unpack8(int32_t((cache_a[ib_a].qs[iqs / 2] >> ((iqs % 2) * 4)) & 0x0F0F0F0F)) - int8_t(4));
+ const int32_t qs_u = int32_t((cache_a[ib_a].qs[iqs / 2] >> ((iqs % 2) * 4)) & 0x0F0F0F0F);
+ const int32_t qs_a = MMQ_SUDOT ? qs_u : pack32(unpack8(qs_u) - int8_t(4));
- q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
+ q_sum += mmq_dot(qs_a, cache_b.qs[iqs]);
+ }
+ if (MMQ_SUDOT) {
+ q_sum -= 4 * cache_b_qsum[1];
}
result += float(cache_a[ib_a].d_scales[1]) * float(q_sum);
@@ -518,7 +548,7 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
const int32_t qs_a = cache_a[ib_a].qs[iqs];
#endif
- q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
+ q_sum += mmq_dot(qs_a, cache_b.qs[iqs]);
}
return ACC_TYPE(float(cache_b.ds.x) * float(cache_a[ib_a].dm.x) * float(q_sum) - float(cache_a[ib_a].dm.y) * float(cache_b.ds.y));
@@ -539,10 +569,11 @@ void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bo
const uint qh_shift = ((iqs_k % 32) / 8) * 2;
const i8vec2 vals00 = (unpack8(int32_t((data_a_packed16[ib_k].ql[ql_idx * 2 ] >> ql_shift) & uint16_t(0x0F0F))).xy |
- unpack8(int32_t(((data_a_packed16[ib_k].qh[qh_idx * 2 ] >> qh_shift) & uint16_t(0x0303)) << 4)).xy) - int8_t(32);
+ unpack8(int32_t(((data_a_packed16[ib_k].qh[qh_idx * 2 ] >> qh_shift) & uint16_t(0x0303)) << 4)).xy);
const i8vec2 vals01 = (unpack8(int32_t((data_a_packed16[ib_k].ql[ql_idx * 2 + 1] >> ql_shift) & uint16_t(0x0F0F))).xy |
- unpack8(int32_t(((data_a_packed16[ib_k].qh[qh_idx * 2 + 1] >> qh_shift) & uint16_t(0x0303)) << 4)).xy) - int8_t(32);
- buf_a[buf_ib].qs[iqs] = pack32(i8vec4(vals00.x, vals00.y, vals01.x, vals01.y));
+ unpack8(int32_t(((data_a_packed16[ib_k].qh[qh_idx * 2 + 1] >> qh_shift) & uint16_t(0x0303)) << 4)).xy);
+ // MMQ_SUDOT keeps the quants unsigned and subtracts 32 times the B sum
+ buf_a[buf_ib].qs[iqs] = MMQ_SUDOT ? pack32(i8vec4(vals00, vals01)) : pack32(i8vec4(vals00 - int8_t(32), vals01 - int8_t(32)));
if (iqs == 0) {
const uint is = iqs_k / 4;
@@ -574,7 +605,10 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
[[unroll]] for (uint iqs = 0; iqs < 4; iqs++) {
const int32_t qs_a = cache_a[ib_a].qs[iqs];
- q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
+ q_sum += mmq_dot(qs_a, cache_b.qs[iqs]);
+ }
+ if (MMQ_SUDOT) {
+ q_sum -= 32 * cache_b_qsum[0];
}
result += float(cache_a[ib_a].d_scales[0]) * float(q_sum);
q_sum = 0;
@@ -582,7 +616,10 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
[[unroll]] for (uint iqs = 4; iqs < 8; iqs++) {
const int32_t qs_a = cache_a[ib_a].qs[iqs];
- q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
+ q_sum += mmq_dot(qs_a, cache_b.qs[iqs]);
+ }
+ if (MMQ_SUDOT) {
+ q_sum -= 32 * cache_b_qsum[1];
}
result += float(cache_a[ib_a].d_scales[1]) * float(q_sum);
@@ -612,8 +649,8 @@ void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bo
const ivec4 m0 = -(ivec4(signs, signs >> 1, signs >> 2, signs >> 3) & 1);
const ivec4 m1 = -(ivec4(signs >> 4, signs >> 5, signs >> 6, signs >> 7) & 1);
- buf_a[buf_ib].qs[2 * iqs ] = pack32(i8vec4((vals0 ^ m0) - m0));
- buf_a[buf_ib].qs[2 * iqs + 1] = pack32(i8vec4((vals1 ^ m1) - m1));
+ buf_a[buf_ib].qs[2 * iqs ] = mmq_a_store(pack32(i8vec4((vals0 ^ m0) - m0)));
+ buf_a[buf_ib].qs[2 * iqs + 1] = mmq_a_store(pack32(i8vec4((vals1 ^ m1) - m1)));
if (iqs == 0) {
const uint scale = (uint(data_a_packed16[ib_k].scales[ib32 / 4]) >> ((ib32 & 3) * 4)) & 0xF;
@@ -641,7 +678,10 @@ void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
ACC_TYPE mmq_dot_product(const uint ib_a) {
int32_t q_sum = 0;
[[unroll]] for (uint iqs = 0; iqs < 8; iqs++) {
- q_sum += dotPacked4x8EXT(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]);
+ q_sum += mmq_dot(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]);
+ }
+ if (MMQ_SUDOT) {
+ q_sum -= 128 * (cache_b_qsum[0] + cache_b_qsum[1]);
}
return ACC_TYPE(float(cache_a[ib_a].d) * float(cache_b.ds.x) * float(q_sum));
@@ -662,6 +702,12 @@ void block_b_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bo
buf_b[buf_ib].qs[iqs * 4 + 1] = values.y;
buf_b[buf_ib].qs[iqs * 4 + 2] = values.z;
buf_b[buf_ib].qs[iqs * 4 + 3] = values.w;
+#ifdef MMQ_B_SUMS
+ if (MMQ_SUDOT) {
+ buf_b_qsum[buf_ib * 2 + iqs] = dotPacked4x8EXT(values.x, 0x01010101u) + dotPacked4x8EXT(values.y, 0x01010101u) +
+ dotPacked4x8EXT(values.z, 0x01010101u) + dotPacked4x8EXT(values.w, 0x01010101u);
+ }
+#endif
} else {
if (iqs == 0) {
buf_b[buf_ib].ds = FLOAT_TYPEV2(0.0f);
@@ -671,6 +717,11 @@ void block_b_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bo
buf_b[buf_ib].qs[iqs * 4 + 1] = 0;
buf_b[buf_ib].qs[iqs * 4 + 2] = 0;
buf_b[buf_ib].qs[iqs * 4 + 3] = 0;
+#ifdef MMQ_B_SUMS
+ if (MMQ_SUDOT) {
+ buf_b_qsum[buf_ib * 2 + iqs] = 0;
+ }
+#endif
}
}
@@ -679,4 +730,10 @@ void block_b_to_registers(const uint ib) {
[[unroll]] for (uint iqs = 0; iqs < BK / 4; iqs++) {
cache_b.qs[iqs] = buf_b[ib].qs[iqs];
}
+#ifdef MMQ_B_SUMS
+ if (MMQ_SUDOT) {
+ cache_b_qsum[0] = buf_b_qsum[ib * 2];
+ cache_b_qsum[1] = buf_b_qsum[ib * 2 + 1];
+ }
+#endif
}