Commit 7a4117eef for llama.cpp

commit 7a4117eef97ed6419241da2384b2b1c011ecd763
Author: Birk Skyum <74932975+birkskyum@users.noreply.github.com>
Date:   Thu Nov 19 07:15:09 2026 +0100

    vulkan : enable MMQ with mixed signedness integer dots (#31045)

    * vulkan : enable MMQ with mixed signedness integer dots

    * vulkan: specialize MMQ shaders for mixed-signedness dot products

    * vulkan: track integer dot product support with signed, mixed and unsigned flags

    * vulkan: move signed dot check to the MMVQ call sites

diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-types.h b/ggml/src/ggml-vulkan/ggml-vulkan-types.h
index f976c5795..6cc76b323 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan-types.h
+++ b/ggml/src/ggml-vulkan/ggml-vulkan-types.h
@@ -747,6 +747,10 @@ struct vk_device_struct {
     bool shader_64b_indexing;

     bool integer_dot_product;
+    // Which 4x8 bit packed integer dot products are accelerated
+    bool integer_dot_signed {};
+    bool integer_dot_mixed {};
+    bool integer_dot_unsigned {};
     // 0: default, 1: force mmvq, -1: disable mmvq
     int32_t mmvq_mode;

diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index e279693cb..3674579cc 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -1726,6 +1726,11 @@ static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vec
     return supported;
 }

+// MMQ uses the mixed signedness dot product (MMQ_SUDOT) when the signed one is not accelerated
+static bool ggml_vk_mmq_use_sudot(const vk_device& device) {
+    return device->integer_dot_mixed && !device->integer_dot_signed;
+}
+
 static bool ggml_vk_matmul_int_shmem_support(const vk_device& device, const std::vector<uint32_t>& warptile, bool mul_mat_id, ggml_type src0_type) {

     // FLOAT_TYPE in the shader is float16_t with fp16 support, otherwise float.
@@ -1782,7 +1787,12 @@ static bool ggml_vk_matmul_int_shmem_support(const vk_device& device, const std:
     const uint32_t buf_b_size = BN * BK_STEP * block_b_size;
     const uint32_t mmid_row_ids = mul_mat_id ? (BN * 2u * (uint32_t)sizeof(uint16_t)) : 0u;

-    const uint32_t total_size = buf_a_size + buf_b_size + mmid_row_ids + lut_size;
+    // Signed quants declare buf_b_qsum: 2 int32_t per B block with MMQ_SUDOT, a single int32_t without
+    const bool b_sums = src0_type == GGML_TYPE_Q8_0 || src0_type == GGML_TYPE_MXFP4 || src0_type == GGML_TYPE_IQ4_XS ||
+                        src0_type == GGML_TYPE_IQ3_S || src0_type == GGML_TYPE_Q3_K || src0_type == GGML_TYPE_Q6_K;
+    const uint32_t qsum_size = !b_sums ? 0u : ggml_vk_mmq_use_sudot(device) ? BN * BK_STEP * 2u * 4u : 4u;
+
+    const uint32_t total_size = buf_a_size + buf_b_size + mmid_row_ids + lut_size + qsum_size;
     const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize;

     VK_LOG_DEBUG("ggml_vk_matmul_int_shmem_support(warptile=(" << warptile[0] << "," << warptile[1] << "," << warptile[2] << "), "
@@ -1906,7 +1916,7 @@ static bool ggml_vk_fa_type_needs_shmem(ggml_type type) {

 static bool ggml_vk_fa_scalar_uses_mmq(const vk_device& device, ggml_type k_type, ggml_type v_type) {
 #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT)
-    return device->integer_dot_product && device->subgroup_clustered &&
+    return device->integer_dot_product && device->integer_dot_signed && device->subgroup_clustered &&
            !ggml_vk_fa_type_needs_shmem(v_type) &&
            (k_type == GGML_TYPE_Q4_0 || k_type == GGML_TYPE_Q4_1 ||
             k_type == GGML_TYPE_Q5_0 || k_type == GGML_TYPE_Q5_1 ||
@@ -2952,8 +2962,13 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
                                  uint32_t rsgs = 0) {
             auto tc = filter_tc(tc_base, key.type_a, key.mul_mat_id, true);
             if (!tc.empty()) {
-                spec_fn_t identity = [](const std::vector<uint32_t>& wt, bool) { return wt; };
-                create_mm_pipelines(key, tc, name, len, data, pc_size, pc, identity, false, rsgs > 0, rsgs, false);
+                // constant_id 11 is MMQ_SUDOT
+                spec_fn_t mmq_spec = [&](const std::vector<uint32_t>& wt, bool) {
+                    std::vector<uint32_t> spec = wt;
+                    spec.push_back(ggml_vk_mmq_use_sudot(device) ? 1u : 0u);
+                    return spec;
+                };
+                create_mm_pipelines(key, tc, name, len, data, pc_size, pc, mmq_spec, false, rsgs > 0, rsgs, false);
             }
         };
 #endif
@@ -4744,7 +4759,10 @@ vk_device ggml_vk_get_device(size_t idx) {
             device->coopmat_support = false;
         }

-        device->integer_dot_product = device->integer_dot_product && shader_integer_dot_product_props.integerDotProduct4x8BitPackedSignedAccelerated;
+        device->integer_dot_signed = shader_integer_dot_product_props.integerDotProduct4x8BitPackedSignedAccelerated;
+        device->integer_dot_mixed = shader_integer_dot_product_props.integerDotProduct4x8BitPackedMixedSignednessAccelerated;
+        device->integer_dot_unsigned = shader_integer_dot_product_props.integerDotProduct4x8BitPackedUnsignedAccelerated;
+        device->integer_dot_product = device->integer_dot_product && (device->integer_dot_signed || device->integer_dot_mixed);

         device->min_imported_host_pointer_alignment = external_memory_host_props.minImportedHostPointerAlignment;

@@ -5623,7 +5641,8 @@ static void ggml_vk_print_gpu_info(size_t idx) {
     const bool rebar = ggml_vk_has_rebar(physical_device);

     integer_dot_product = integer_dot_product
-                       && shader_integer_dot_product_props.integerDotProduct4x8BitPackedSignedAccelerated
+                       && (shader_integer_dot_product_props.integerDotProduct4x8BitPackedSignedAccelerated ||
+                           shader_integer_dot_product_props.integerDotProduct4x8BitPackedMixedSignednessAccelerated)
                        && shader_integer_dot_product_features.shaderIntegerDotProduct;

     coopmat_support = coopmat_support
@@ -7395,7 +7414,7 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context&

     const ggml_type y_kernel_type = widen_bf16 ? GGML_TYPE_F32 : src1->type;
     const bool f16_f32_kernel = y_kernel_type == GGML_TYPE_F32;
-    bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && (swap_inputs || ggml_vk_src1_prec_allows_q8(dst)) && ggml_vk_should_use_mmvq(ctx->device, ne01, ne11, ne10, src0->type);
+    bool quantize_y = ctx->device->integer_dot_signed && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && (swap_inputs || ggml_vk_src1_prec_allows_q8(dst)) && ggml_vk_should_use_mmvq(ctx->device, ne01, ne11, ne10, src0->type);

     vk_pipeline to_fp16_vk_0 = nullptr;
     vk_pipeline to_fp16_vk_1 = nullptr;
@@ -8463,7 +8482,7 @@ static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_conte
     const bool y_non_contig = !ggml_vk_dim01_contiguous(src1);

     const bool f16_f32_kernel = src1->type == GGML_TYPE_F32;
-    bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && ggml_vk_src1_prec_allows_q8(dst) && ggml_vk_should_use_mmvq(ctx->device, ne01, ne12, ne10, src0->type);
+    bool quantize_y = ctx->device->integer_dot_signed && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && ggml_vk_src1_prec_allows_q8(dst) && ggml_vk_should_use_mmvq(ctx->device, ne01, ne12, ne10, src0->type);

     vk_pipeline to_fp16_vk_0 = nullptr;
     vk_pipeline to_fp16_vk_1 = nullptr;
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp
index 1866386a7..f831fd788 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp
@@ -82,6 +82,8 @@ layout (constant_id = 7) const uint TM = 4;
 layout (constant_id = 8) const uint TN = 2;
 layout (constant_id = 9) const uint TK = 1;  // Only needed for coopmat
 layout (constant_id = 10) const uint WARP = 32;
+// Use the mixed signedness dot product, for devices that accelerate only that one
+layout (constant_id = 11) const bool MMQ_SUDOT = false;

 #define BK 32

@@ -102,6 +104,13 @@ shared block_b_cache buf_b[BN * BK_STEP];
 block_a_cache cache_a[WMITER * TM];
 block_b_cache cache_b;

+#if defined(DATA_A_Q8_0) || defined(DATA_A_MXFP4) || defined(DATA_A_IQ4_XS) || defined(DATA_A_IQ3_S) || defined(DATA_A_Q3_K) || defined(DATA_A_Q6_K)
+// These quants are signed. MMQ_SUDOT gives them a bias and uses the sum of each half of the q8_1 block to remove it.
+#define MMQ_B_SUMS
+shared int32_t buf_b_qsum[MMQ_SUDOT ? BN * BK_STEP * 2 : 1];
+int32_t cache_b_qsum[2];
+#endif
+
 #define LOAD_VEC_A (4 * QUANT_R_MMQ)
 #define LOAD_VEC_B 16

diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl
index d9f2f5748..9d94fd2b7 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl
@@ -4,6 +4,19 @@

 #include "types.glsl"

+int32_t mmq_dot(const int32_t a, const int32_t b) {
+    // With MMQ_SUDOT the A quants must be unsigned
+    if (MMQ_SUDOT) {
+        return dotPacked4x8EXT(uint32_t(a), b);
+    }
+    return dotPacked4x8EXT(a, b);
+}
+
+// With MMQ_SUDOT, flip the sign bit of each byte. This adds a bias of 128 to signed quants.
+int32_t mmq_a_store(const int32_t v) {
+    return MMQ_SUDOT ? v ^ int32_t(0x80808080) : v;
+}
+
 // Each iqs value maps to a 32-bit integer

 #if defined(DATA_A_Q2_0)
@@ -41,7 +54,7 @@ void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
 ACC_TYPE mmq_dot_product(const uint ib_a) {
     int32_t q_sum = 0;
     [[unroll]] for (uint iqs = 0; iqs < 8; ++iqs) {
-        q_sum += dotPacked4x8EXT(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]);
+        q_sum += mmq_dot(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]);
     }

     return ACC_TYPE(float(cache_a[ib_a].dm) * (float(q_sum) * float(cache_b.ds.x) - float(cache_b.ds.y)));
@@ -102,8 +115,8 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
         const int32_t qs_b0 = cache_b.qs[iqs];
         const int32_t qs_b1 = cache_b.qs[iqs + 4];

-        q_sum += dotPacked4x8EXT(qs_a.x, qs_b0);
-        q_sum += dotPacked4x8EXT(qs_a.y, qs_b1);
+        q_sum += mmq_dot(qs_a.x, qs_b0);
+        q_sum += mmq_dot(qs_a.y, qs_b1);
     }

 #ifdef DATA_A_Q4_0
@@ -176,8 +189,8 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
         const int32_t qs_b0 = cache_b.qs[iqs];
         const int32_t qs_b1 = cache_b.qs[iqs + 4];

-        q_sum += dotPacked4x8EXT(qs_a0, qs_b0);
-        q_sum += dotPacked4x8EXT(qs_a1, qs_b1);
+        q_sum += mmq_dot(qs_a0, qs_b0);
+        q_sum += mmq_dot(qs_a1, qs_b1);
     }

 #ifdef DATA_A_Q5_0
@@ -192,8 +205,8 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
 // 2-byte loads for Q8_0 blocks (34 bytes)
 void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bool is_in_bounds) {
     if (is_in_bounds) {
-        buf_a[buf_ib].qs[iqs] = pack32(i16vec2(data_a_packed16[ib].qs[iqs * 2],
-                                               data_a_packed16[ib].qs[iqs * 2 + 1]));
+        buf_a[buf_ib].qs[iqs] = mmq_a_store(pack32(i16vec2(data_a_packed16[ib].qs[iqs * 2],
+                                                           data_a_packed16[ib].qs[iqs * 2 + 1])));

         if (iqs == 0) {
             buf_a[buf_ib].dm = FLOAT_TYPE(data_a_packed16[ib].d);
@@ -221,7 +234,10 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
         const int32_t qs_a = cache_a[ib_a].qs[iqs];
         const int32_t qs_b = cache_b.qs[iqs];

-        q_sum += dotPacked4x8EXT(qs_a, qs_b);
+        q_sum += mmq_dot(qs_a, qs_b);
+    }
+    if (MMQ_SUDOT) {
+        q_sum -= 128 * (cache_b_qsum[0] + cache_b_qsum[1]);
     }

     return ACC_TYPE(float(q_sum) * float(cache_a[ib_a].dm) * float(cache_b.ds.x));
@@ -240,8 +256,8 @@ void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bo
         const u8vec4 i_a0 = unpack8( qs       & 0x0F0F0F0F);
         const u8vec4 i_a1 = unpack8((qs >> 4) & 0x0F0F0F0F);

-        buf_a[buf_ib].qs[iqs    ] = pack32(i8vec4(kvalues_mxfp4[i_a0.x], kvalues_mxfp4[i_a0.y], kvalues_mxfp4[i_a0.z], kvalues_mxfp4[i_a0.w]));
-        buf_a[buf_ib].qs[iqs + 4] = pack32(i8vec4(kvalues_mxfp4[i_a1.x], kvalues_mxfp4[i_a1.y], kvalues_mxfp4[i_a1.z], kvalues_mxfp4[i_a1.w]));
+        buf_a[buf_ib].qs[iqs    ] = mmq_a_store(pack32(i8vec4(kvalues_mxfp4[i_a0.x], kvalues_mxfp4[i_a0.y], kvalues_mxfp4[i_a0.z], kvalues_mxfp4[i_a0.w])));
+        buf_a[buf_ib].qs[iqs + 4] = mmq_a_store(pack32(i8vec4(kvalues_mxfp4[i_a1.x], kvalues_mxfp4[i_a1.y], kvalues_mxfp4[i_a1.z], kvalues_mxfp4[i_a1.w])));

         if (iqs == 0) {
             buf_a[buf_ib].d = FLOAT_TYPE(e8m0_to_fp32(data_a[ib].e) * 0.5);
@@ -269,7 +285,10 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
     [[unroll]] for (uint iqs = 0; iqs < 8; iqs++) {
         const int32_t qs_a = cache_a[ib_a].qs[iqs];

-        q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
+        q_sum += mmq_dot(qs_a, cache_b.qs[iqs]);
+    }
+    if (MMQ_SUDOT) {
+        q_sum -= 128 * (cache_b_qsum[0] + cache_b_qsum[1]);
     }

     return ACC_TYPE(float(cache_a[ib_a].d) * float(cache_b.ds.x) * float(q_sum));
@@ -284,8 +303,8 @@ void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bo
         const uint32_t vui = data_a_packed32[ib_k].qs[4 * ib32 + iqs];
         const i32vec2 qs = iq4nl_to_i8x8(vui);

-        buf_a[buf_ib].qs[iqs    ] = qs.x;
-        buf_a[buf_ib].qs[iqs + 4] = qs.y;
+        buf_a[buf_ib].qs[iqs    ] = mmq_a_store(qs.x);
+        buf_a[buf_ib].qs[iqs + 4] = mmq_a_store(qs.y);

         if (iqs == 0) {
             const uint sl = (data_a_packed32[ib_k].scales_l >> (4 * ib32)) & 0xF;
@@ -313,7 +332,10 @@ void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
 ACC_TYPE mmq_dot_product(const uint ib_a) {
     int32_t q_sum = 0;
     [[unroll]] for (uint iqs = 0; iqs < 8; iqs++) {
-        q_sum += dotPacked4x8EXT(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]);
+        q_sum += mmq_dot(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]);
+    }
+    if (MMQ_SUDOT) {
+        q_sum -= 128 * (cache_b_qsum[0] + cache_b_qsum[1]);
     }

     return ACC_TYPE(float(cache_a[ib_a].d) * float(cache_b.ds.x) * float(q_sum));
@@ -372,8 +394,8 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
         const int32_t scale_m = int32_t(scale >> 4) * 0x01010101; // Duplicate 8-bit value across 32-bits.
         const int32_t qs_a = int32_t((cache_a[ib_a].qs[iqs / 4] >> ((iqs % 4) * 2)) & 0x03030303);

-        sum_d += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]) * (scale & 0xF);
-        sum_m += dotPacked4x8EXT(scale_m, cache_b.qs[iqs]);
+        sum_d += mmq_dot(qs_a, cache_b.qs[iqs]) * (scale & 0xF);
+        sum_m += mmq_dot(scale_m, cache_b.qs[iqs]);
     }

     return ACC_TYPE(float(cache_b.ds.x) * (float(cache_a[ib_a].dm.x) * float(sum_d) - float(cache_a[ib_a].dm.y) * float(sum_m)));
@@ -435,18 +457,26 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
     int32_t q_sum = 0;

     [[unroll]] for (uint iqs = 0; iqs < 4; iqs++) {
-        // Subtract 4 from the quants to correct the 3rd bit offset
-        const int32_t qs_a = pack32(unpack8(int32_t((cache_a[ib_a].qs[iqs / 2] >> ((iqs % 2) * 4)) & 0x0F0F0F0F)) - int8_t(4));
+        // Subtract 4 from the quants to correct the 3rd bit offset. MMQ_SUDOT keeps them unsigned and subtracts 4 times the B sum.
+        const int32_t qs_u = int32_t((cache_a[ib_a].qs[iqs / 2] >> ((iqs % 2) * 4)) & 0x0F0F0F0F);
+        const int32_t qs_a = MMQ_SUDOT ? qs_u : pack32(unpack8(qs_u) - int8_t(4));

-        q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
+        q_sum += mmq_dot(qs_a, cache_b.qs[iqs]);
+    }
+    if (MMQ_SUDOT) {
+        q_sum -= 4 * cache_b_qsum[0];
     }
     result += float(cache_a[ib_a].d_scales[0]) * float(q_sum);
     q_sum = 0;

     [[unroll]] for (uint iqs = 4; iqs < 8; iqs++) {
-        const int32_t qs_a = pack32(unpack8(int32_t((cache_a[ib_a].qs[iqs / 2] >> ((iqs % 2) * 4)) & 0x0F0F0F0F)) - int8_t(4));
+        const int32_t qs_u = int32_t((cache_a[ib_a].qs[iqs / 2] >> ((iqs % 2) * 4)) & 0x0F0F0F0F);
+        const int32_t qs_a = MMQ_SUDOT ? qs_u : pack32(unpack8(qs_u) - int8_t(4));

-        q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
+        q_sum += mmq_dot(qs_a, cache_b.qs[iqs]);
+    }
+    if (MMQ_SUDOT) {
+        q_sum -= 4 * cache_b_qsum[1];
     }
     result += float(cache_a[ib_a].d_scales[1]) * float(q_sum);

@@ -518,7 +548,7 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
         const int32_t qs_a = cache_a[ib_a].qs[iqs];
 #endif

-        q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
+        q_sum += mmq_dot(qs_a, cache_b.qs[iqs]);
     }

     return ACC_TYPE(float(cache_b.ds.x) * float(cache_a[ib_a].dm.x) * float(q_sum) - float(cache_a[ib_a].dm.y) * float(cache_b.ds.y));
@@ -539,10 +569,11 @@ void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bo
         const uint qh_shift = ((iqs_k % 32) / 8) * 2;

         const i8vec2 vals00 = (unpack8(int32_t((data_a_packed16[ib_k].ql[ql_idx * 2    ] >> ql_shift) & uint16_t(0x0F0F))).xy |
-                              unpack8(int32_t(((data_a_packed16[ib_k].qh[qh_idx * 2    ] >> qh_shift) & uint16_t(0x0303)) << 4)).xy) - int8_t(32);
+                              unpack8(int32_t(((data_a_packed16[ib_k].qh[qh_idx * 2    ] >> qh_shift) & uint16_t(0x0303)) << 4)).xy);
         const i8vec2 vals01 = (unpack8(int32_t((data_a_packed16[ib_k].ql[ql_idx * 2 + 1] >> ql_shift) & uint16_t(0x0F0F))).xy |
-                              unpack8(int32_t(((data_a_packed16[ib_k].qh[qh_idx * 2 + 1] >> qh_shift) & uint16_t(0x0303)) << 4)).xy) - int8_t(32);
-        buf_a[buf_ib].qs[iqs] = pack32(i8vec4(vals00.x, vals00.y, vals01.x, vals01.y));
+                              unpack8(int32_t(((data_a_packed16[ib_k].qh[qh_idx * 2 + 1] >> qh_shift) & uint16_t(0x0303)) << 4)).xy);
+        // MMQ_SUDOT keeps the quants unsigned and subtracts 32 times the B sum
+        buf_a[buf_ib].qs[iqs] = MMQ_SUDOT ? pack32(i8vec4(vals00, vals01)) : pack32(i8vec4(vals00 - int8_t(32), vals01 - int8_t(32)));

         if (iqs == 0) {
             const uint is = iqs_k / 4;
@@ -574,7 +605,10 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
     [[unroll]] for (uint iqs = 0; iqs < 4; iqs++) {
         const int32_t qs_a = cache_a[ib_a].qs[iqs];

-        q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
+        q_sum += mmq_dot(qs_a, cache_b.qs[iqs]);
+    }
+    if (MMQ_SUDOT) {
+        q_sum -= 32 * cache_b_qsum[0];
     }
     result += float(cache_a[ib_a].d_scales[0]) * float(q_sum);
     q_sum = 0;
@@ -582,7 +616,10 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
     [[unroll]] for (uint iqs = 4; iqs < 8; iqs++) {
         const int32_t qs_a = cache_a[ib_a].qs[iqs];

-        q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
+        q_sum += mmq_dot(qs_a, cache_b.qs[iqs]);
+    }
+    if (MMQ_SUDOT) {
+        q_sum -= 32 * cache_b_qsum[1];
     }
     result += float(cache_a[ib_a].d_scales[1]) * float(q_sum);

@@ -612,8 +649,8 @@ void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bo
         const ivec4 m0 = -(ivec4(signs,      signs >> 1, signs >> 2, signs >> 3) & 1);
         const ivec4 m1 = -(ivec4(signs >> 4, signs >> 5, signs >> 6, signs >> 7) & 1);

-        buf_a[buf_ib].qs[2 * iqs    ] = pack32(i8vec4((vals0 ^ m0) - m0));
-        buf_a[buf_ib].qs[2 * iqs + 1] = pack32(i8vec4((vals1 ^ m1) - m1));
+        buf_a[buf_ib].qs[2 * iqs    ] = mmq_a_store(pack32(i8vec4((vals0 ^ m0) - m0)));
+        buf_a[buf_ib].qs[2 * iqs + 1] = mmq_a_store(pack32(i8vec4((vals1 ^ m1) - m1)));

         if (iqs == 0) {
             const uint scale = (uint(data_a_packed16[ib_k].scales[ib32 / 4]) >> ((ib32 & 3) * 4)) & 0xF;
@@ -641,7 +678,10 @@ void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
 ACC_TYPE mmq_dot_product(const uint ib_a) {
     int32_t q_sum = 0;
     [[unroll]] for (uint iqs = 0; iqs < 8; iqs++) {
-        q_sum += dotPacked4x8EXT(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]);
+        q_sum += mmq_dot(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]);
+    }
+    if (MMQ_SUDOT) {
+        q_sum -= 128 * (cache_b_qsum[0] + cache_b_qsum[1]);
     }

     return ACC_TYPE(float(cache_a[ib_a].d) * float(cache_b.ds.x) * float(q_sum));
@@ -662,6 +702,12 @@ void block_b_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bo
         buf_b[buf_ib].qs[iqs * 4 + 1] = values.y;
         buf_b[buf_ib].qs[iqs * 4 + 2] = values.z;
         buf_b[buf_ib].qs[iqs * 4 + 3] = values.w;
+#ifdef MMQ_B_SUMS
+        if (MMQ_SUDOT) {
+            buf_b_qsum[buf_ib * 2 + iqs] = dotPacked4x8EXT(values.x, 0x01010101u) + dotPacked4x8EXT(values.y, 0x01010101u) +
+                                           dotPacked4x8EXT(values.z, 0x01010101u) + dotPacked4x8EXT(values.w, 0x01010101u);
+        }
+#endif
     } else {
         if (iqs == 0) {
             buf_b[buf_ib].ds = FLOAT_TYPEV2(0.0f);
@@ -671,6 +717,11 @@ void block_b_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bo
         buf_b[buf_ib].qs[iqs * 4 + 1] = 0;
         buf_b[buf_ib].qs[iqs * 4 + 2] = 0;
         buf_b[buf_ib].qs[iqs * 4 + 3] = 0;
+#ifdef MMQ_B_SUMS
+        if (MMQ_SUDOT) {
+            buf_b_qsum[buf_ib * 2 + iqs] = 0;
+        }
+#endif
     }
 }

@@ -679,4 +730,10 @@ void block_b_to_registers(const uint ib) {
     [[unroll]] for (uint iqs = 0; iqs < BK / 4; iqs++) {
         cache_b.qs[iqs] = buf_b[ib].qs[iqs];
     }
+#ifdef MMQ_B_SUMS
+    if (MMQ_SUDOT) {
+        cache_b_qsum[0] = buf_b_qsum[ib * 2];
+        cache_b_qsum[1] = buf_b_qsum[ib * 2 + 1];
+    }
+#endif
 }