Commit bed8ec2f8 for llama.cpp
commit bed8ec2f8d484520d4822c582a94424a03e172e7
Author: Aaron Teo <aaron.teo1@ibm.com>
Date: Fri Nov 20 15:47:41 2026 +0800
llamafile(s390x): add QK data type support (#32600)
* llamafile: add QK support for s390x
Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>
* llamafile: clean up for s390x
Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>
---------
Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>
diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp
index 68105f029..5d6609113 100644
--- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp
+++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp
@@ -1474,6 +1474,282 @@ class tinyBLAS_Q0_VXE {
const int32_t ith;
const int32_t nth;
};
+
+template <typename TA>
+class tinyBLAS_QK_VXE {
+ public:
+ tinyBLAS_QK_VXE(int64_t k,
+ const TA * A, int64_t lda,
+ const block_q8_K * B, int64_t ldb,
+ float * C, int64_t ldc,
+ int32_t ith, int32_t nth)
+ : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
+ }
+
+ void matmul(int64_t m, int64_t n) {
+ const int64_t n4 = n / 4 * 4;
+ const int64_t n2 = n / 2 * 2;
+ gemm<4>(m, 0, n4);
+ gemm<2>(m, n4, n2);
+ gemm<1>(m, n2, n);
+ }
+
+ private:
+ // one block of A is loaded once and used for RN columns of B
+ template <int32_t RN>
+ void gemm(int64_t m, int64_t n0, int64_t n) {
+ const int64_t tiles = m * ((n - n0) / RN);
+ const int64_t duty = (tiles + nth - 1) / nth;
+ const int64_t start = duty * ith;
+ const int64_t end = MIN(start + duty, tiles);
+
+ for (int64_t job = start; job < end; ++job) {
+ const int64_t ii = job % m;
+ const int64_t jj = n0 + RN * (job / m);
+ float32x4_t v_sum[RN] = {};
+
+ for (int64_t l = 0; l < k; ++l) {
+ const TA & x = A[lda * ii + l];
+
+ int8x16_t v_x[n_group];
+ load(x, v_x);
+
+ // scale and minimum of the 16 groups, 8 groups per vector
+ int16x8_t v_scalel, v_scaleh;
+ int16x8_t v_minl, v_minh;
+ load_scale(x, v_scalel, v_scaleh, v_minl, v_minh);
+
+ int16x8_t v_scale[n_group];
+ for (int32_t g = 0; g < n_group; ++g) {
+ v_scale[g] = vec_splats((g < n_group / 2 ? v_scalel : v_scaleh)[g % (n_group / 2)]);
+ }
+
+ const int8_t * y_qs[RN];
+ for (int32_t j = 0; j < RN; ++j) {
+ y_qs[j] = B[ldb * (jj + j) + l].qs;
+ }
+
+ // the columns are in the inner loop, so that their sums do not wait for each other
+ int32x4_t v_dot[RN] = {};
+ for (int32_t g = 0; g < n_group; ++g) {
+ for (int32_t j = 0; j < RN; ++j) {
+ const int8x16_t v_y = vec_xl(0, y_qs[j] + 16 * g);
+ const int16x8_t v_xy = vec_meadd(v_x[g], v_y, vec_mulo(v_x[g], v_y));
+ v_dot[j] = vec_meadd(v_xy, v_scale[g], vec_moadd(v_xy, v_scale[g], v_dot[j]));
+ }
+ }
+
+ const float x_d = unhalf(x.d);
+ for (int32_t j = 0; j < RN; ++j) {
+ const block_q8_K & y = B[ldb * (jj + j) + l];
+ v_sum[j] = vec_madd(vec_float(v_dot[j]), vec_splats(x_d * y.d), v_sum[j]);
+ }
+
+ if (has_min) {
+ const float x_dmin = dmin(x);
+
+ for (int32_t j = 0; j < RN; ++j) {
+ const block_q8_K & y = B[ldb * (jj + j) + l];
+
+ const int16_t * y_bsums = y.bsums;
+ const int16x8_t v_bsumsl = vec_xl(0, y_bsums);
+ const int16x8_t v_bsumsh = vec_xl(0, y_bsums + n_group / 2);
+
+ const int32x4_t v_minsl = vec_meadd(v_bsumsl, v_minl, vec_mulo(v_bsumsl, v_minl));
+ const int32x4_t v_minsh = vec_meadd(v_bsumsh, v_minh, vec_mulo(v_bsumsh, v_minh));
+ v_sum[j] = vec_madd(vec_float(v_minsl + v_minsh), vec_splats(-x_dmin * y.d), v_sum[j]);
+ }
+ }
+ }
+
+ for (int32_t j = 0; j < RN; ++j) {
+ C[ldc * (jj + j) + ii] = hsum(v_sum[j]);
+ }
+ }
+ }
+
+ // a block has 16 groups of 16 weights, Q4_K and Q5_K share one scale and minimum per 2 groups
+ static constexpr int32_t n_group = QK_K / 16;
+
+ // Q3_K and Q6_K have no minimum, their weights are loaded as signed values.
+ static constexpr bool has_min = !std::is_same<TA, block_q3_K>::value && !std::is_same<TA, block_q6_K>::value;
+
+ static float dmin(const block_q2_K & x) { return unhalf(x.dmin); }
+ static float dmin(const block_q4_K & x) { return unhalf(x.dmin); }
+ static float dmin(const block_q5_K & x) { return unhalf(x.dmin); }
+ template <typename T> static float dmin(const T &) { return 0.0f; }
+
+ static void load(const block_q2_K & x, int8x16_t * v_x) {
+ const uint8x16_t v_mask = vec_splats((uint8_t)0x03);
+ const uint8_t * x_qs = x.qs;
+
+ for (int32_t g = 0; g < n_group; ++g) {
+ const int32_t qs_offset = 16 * (2 * (g / 8) + g % 2);
+ const int32_t qs_shift = 2 * (g % 8 / 2);
+
+ const uint8x16_t v_qs = vec_xl(0, x_qs + qs_offset);
+ v_x[g] = (int8x16_t)vec_and(vec_sr(v_qs, qs_shift), v_mask);
+ }
+ }
+
+ static void load(const block_q3_K & x, int8x16_t * v_x) {
+ const uint8x16_t v_mask = vec_splats((uint8_t)0x03);
+ const uint8x16_t v_one = vec_splats((uint8_t)0x01);
+
+ const uint8_t * x_qs = x.qs;
+ const uint8_t * x_hmask = x.hmask;
+
+ for (int32_t g = 0; g < n_group; ++g) {
+ const int32_t qs_offset = 16 * (2 * (g / 8) + g % 2);
+ const int32_t qs_shift = 2 * (g % 8 / 2);
+ const int32_t hmask_offset = 16 * (g % 2);
+ const int32_t hmask_shift = g / 2;
+
+ const uint8x16_t v_qs = vec_xl(0, x_qs + qs_offset);
+ const uint8x16_t v_hmask = vec_xl(0, x_hmask + hmask_offset);
+ const uint8x16_t v_lo = vec_and(vec_sr(v_qs, qs_shift), v_mask);
+ const uint8x16_t v_hi = vec_sl(vec_and(vec_sr(v_hmask, hmask_shift), v_one), 2);
+ v_x[g] = (int8x16_t)vec_or(v_lo, v_hi) - vec_splats((int8_t)4);
+ }
+ }
+
+ static void load(const block_q4_K & x, int8x16_t * v_x) {
+ const uint8x16_t v_mask = vec_splats((uint8_t)0x0F);
+ const uint8_t * x_qs = x.qs;
+
+ for (int32_t g = 0; g < n_group; ++g) {
+ const int32_t qs_offset = 16 * (2 * (g / 4) + g % 2);
+ const bool qs_low = g % 4 < 2;
+
+ const uint8x16_t v_qs = vec_xl(0, x_qs + qs_offset);
+ v_x[g] = (int8x16_t)(qs_low ? vec_and(v_qs, v_mask) : vec_sr(v_qs, 4));
+ }
+ }
+
+ static void load(const block_q5_K & x, int8x16_t * v_x) {
+ const uint8x16_t v_mask = vec_splats((uint8_t)0x0F);
+ const uint8x16_t v_one = vec_splats((uint8_t)0x01);
+ const uint8_t * x_qs = x.qs;
+ const uint8_t * x_qh = x.qh;
+
+ for (int32_t g = 0; g < n_group; ++g) {
+ const int32_t qs_offset = 16 * (2 * (g / 4) + g % 2);
+ const bool qs_low = g % 4 < 2;
+ const int32_t qh_offset = 16 * (g % 2);
+ const int32_t qh_shift = g / 2;
+
+ const uint8x16_t v_qs = vec_xl(0, x_qs + qs_offset);
+ const uint8x16_t v_qh = vec_xl(0, x_qh + qh_offset);
+ const uint8x16_t v_lo = qs_low ? vec_and(v_qs, v_mask) : vec_sr(v_qs, 4);
+ const uint8x16_t v_hi = vec_sl(vec_and(vec_sr(v_qh, qh_shift), v_one), 4);
+ v_x[g] = (int8x16_t)vec_or(v_lo, v_hi);
+ }
+ }
+
+ static void load(const block_q6_K & x, int8x16_t * v_x) {
+ const uint8x16_t v_mask4 = vec_splats((uint8_t)0x0F);
+ const uint8x16_t v_mask2 = vec_splats((uint8_t)0x03);
+ const uint8_t * x_ql = x.ql;
+ const uint8_t * x_qh = x.qh;
+
+ for (int32_t g = 0; g < n_group; ++g) {
+ const int32_t ql_offset = 16 * (4 * (g / 8) + 2 * (g % 4 / 2) + g % 2);
+ const bool ql_low = g % 8 < 4;
+ const int32_t qh_offset = 16 * (2 * (g / 8) + g % 2);
+ const int32_t qh_shift = 2 * (g % 8 / 2);
+
+ const uint8x16_t v_ql = vec_xl(0, x_ql + ql_offset);
+ const uint8x16_t v_qh = vec_xl(0, x_qh + qh_offset);
+ const uint8x16_t v_lo = ql_low ? vec_and(v_ql, v_mask4) : vec_sr(v_ql, 4);
+ const uint8x16_t v_hi = vec_sl(vec_and(vec_sr(v_qh, qh_shift), v_mask2), 4);
+ v_x[g] = (int8x16_t)vec_or(v_lo, v_hi) - vec_splats((int8_t)32);
+ }
+ }
+
+ static void load_scale(const block_q2_K & x, int16x8_t & v_scalel, int16x8_t & v_scaleh, int16x8_t & v_minl, int16x8_t & v_minh) {
+ const uint8_t * x_scales = x.scales;
+ const uint8x16_t v_s = vec_xl(0, x_scales);
+ const uint8x16_t v_scale = vec_and(v_s, vec_splats((uint8_t)0x0F));
+ const uint8x16_t v_min = vec_sr(v_s, 4);
+
+ v_scalel = (int16x8_t)vec_unpackh(v_scale);
+ v_scaleh = (int16x8_t)vec_unpackl(v_scale);
+ v_minl = (int16x8_t)vec_unpackh(v_min);
+ v_minh = (int16x8_t)vec_unpackl(v_min);
+ }
+
+ static void load_scale(const block_q3_K & x, int16x8_t & v_scalel, int16x8_t & v_scaleh, int16x8_t & v_minl, int16x8_t & v_minh) {
+ // 6-bit scales: low 4 bits in scales[0..7], high 2 bits in scales[8..11]
+ const uint8x16_t v_lo_byte = { 4, 5, 6, 7, 8, 9, 10, 11, 4, 5, 6, 7, 8, 9, 10, 11 };
+ const uint8x16_t v_lo_shift = { 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4 };
+ const uint8x16_t v_hi_byte = { 12, 13, 14, 15, 12, 13, 14, 15, 12, 13, 14, 15, 12, 13, 14, 15 };
+ const uint8x16_t v_hi_shift = { 0, 0, 0, 0, 2, 2, 2, 2, 4, 4, 4, 4, 6, 6, 6, 6 };
+
+ // the load starts 4 bytes before scales, so that it stays inside the block
+ const uint8_t * x_qs = x.qs;
+ const uint8x16_t v_s = vec_xl(0, x_qs + sizeof(x.qs) - 4);
+ const uint8x16_t v_lo = vec_and(vec_sr(vec_perm(v_s, v_s, v_lo_byte), v_lo_shift), vec_splats((uint8_t)0x0F));
+ const uint8x16_t v_hi = vec_sl(vec_and(vec_sr(vec_perm(v_s, v_s, v_hi_byte), v_hi_shift), vec_splats((uint8_t)0x03)), 4);
+ const int8x16_t v_scale = (int8x16_t)vec_or(v_lo, v_hi) - vec_splats((int8_t)32);
+
+ v_scalel = vec_unpackh(v_scale);
+ v_scaleh = vec_unpackl(v_scale);
+ v_minl = vec_splats((int16_t)0);
+ v_minh = vec_splats((int16_t)0);
+ }
+
+ static void load_scale(const block_q4_K & x, int16x8_t & v_scalel, int16x8_t & v_scaleh, int16x8_t & v_minl, int16x8_t & v_minh) {
+ load_scale_min_k4(x.scales, v_scalel, v_scaleh, v_minl, v_minh);
+ }
+
+ static void load_scale(const block_q5_K & x, int16x8_t & v_scalel, int16x8_t & v_scaleh, int16x8_t & v_minl, int16x8_t & v_minh) {
+ load_scale_min_k4(x.scales, v_scalel, v_scaleh, v_minl, v_minh);
+ }
+
+ static void load_scale(const block_q6_K & x, int16x8_t & v_scalel, int16x8_t & v_scaleh, int16x8_t & v_minl, int16x8_t & v_minh) {
+ const int8_t * x_scales = x.scales;
+ const int8x16_t v_scale = vec_xl(0, x_scales);
+
+ v_scalel = vec_unpackh(v_scale);
+ v_scaleh = vec_unpackl(v_scale);
+ v_minl = vec_splats((int16_t)0);
+ v_minh = vec_splats((int16_t)0);
+ }
+
+ // 8 scales and 8 minimums of 6 bits, same packing as get_scale_min_k4 in ggml-quants.c
+ static void load_scale_min_k4(const uint8_t * q, int16x8_t & v_scalel, int16x8_t & v_scaleh, int16x8_t & v_minl, int16x8_t & v_minh) {
+ // bytes 0..3 and 4..7 have the high 2 bits of scale 4..7 and minimum 4..7, move them to bytes 8..11
+ const uint8x16_t v_hi_scale = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 0, 0, 0, 0 };
+ const uint8x16_t v_hi_min = { 0, 0, 0, 0, 0, 0, 0, 0, 4, 5, 6, 7, 0, 0, 0, 0 };
+ // each scale and minimum is for 2 groups
+ const uint8x16_t v_group_scale = { 0, 0, 1, 1, 2, 2, 3, 3, 24, 24, 25, 25, 26, 26, 27, 27 };
+ const uint8x16_t v_group_min = { 4, 4, 5, 5, 6, 6, 7, 7, 24, 24, 25, 25, 26, 26, 27, 27 };
+
+ // the 4 bytes after the 12 scale bytes are in the same block
+ const uint8x16_t v_s = vec_xl(0, q);
+ const uint8x16_t v_lo = vec_and(v_s, vec_splats((uint8_t)0x3F));
+ const uint8x16_t v_hi = vec_sl(vec_sr(v_s, 6), 4);
+ const uint8x16_t v_scale_47 = vec_or(vec_and(v_s, vec_splats((uint8_t)0x0F)), vec_perm(v_hi, v_hi, v_hi_scale));
+ const uint8x16_t v_min_47 = vec_or(vec_sr(v_s, 4), vec_perm(v_hi, v_hi, v_hi_min));
+ const uint8x16_t v_scale = vec_perm(v_lo, v_scale_47, v_group_scale);
+ const uint8x16_t v_min = vec_perm(v_lo, v_min_47, v_group_min);
+
+ v_scalel = (int16x8_t)vec_unpackh(v_scale);
+ v_scaleh = (int16x8_t)vec_unpackl(v_scale);
+ v_minl = (int16x8_t)vec_unpackh(v_min);
+ v_minh = (int16x8_t)vec_unpackl(v_min);
+ }
+
+ const TA * const A;
+ const block_q8_K * const B;
+ float * const C;
+ const int64_t k;
+ const int64_t lda;
+ const int64_t ldb;
+ const int64_t ldc;
+ const int32_t ith;
+ const int32_t nth;
+};
#endif
#if defined(__ARM_FEATURE_DOTPROD)
@@ -4488,6 +4764,91 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64
#endif
}
+ case GGML_TYPE_Q2_K: {
+ if (Btype != GGML_TYPE_Q8_K) {
+ return false;
+ }
+#if defined(__VXE__) || defined(__VXE2__)
+ tinyBLAS_QK_VXE<block_q2_K> tb{
+ k, (const block_q2_K *)A, lda,
+ (const block_q8_K *)B, ldb,
+ (float *)C, ldc,
+ params->ith, params->nth};
+ tb.matmul(m, n);
+ return true;
+#else
+ return false;
+#endif
+ }
+
+ case GGML_TYPE_Q3_K: {
+ if (Btype != GGML_TYPE_Q8_K) {
+ return false;
+ }
+#if defined(__VXE__) || defined(__VXE2__)
+ tinyBLAS_QK_VXE<block_q3_K> tb{
+ k, (const block_q3_K *)A, lda,
+ (const block_q8_K *)B, ldb,
+ (float *)C, ldc,
+ params->ith, params->nth};
+ tb.matmul(m, n);
+ return true;
+#else
+ return false;
+#endif
+ }
+
+ case GGML_TYPE_Q4_K: {
+ if (Btype != GGML_TYPE_Q8_K) {
+ return false;
+ }
+#if defined(__VXE__) || defined(__VXE2__)
+ tinyBLAS_QK_VXE<block_q4_K> tb{
+ k, (const block_q4_K *)A, lda,
+ (const block_q8_K *)B, ldb,
+ (float *)C, ldc,
+ params->ith, params->nth};
+ tb.matmul(m, n);
+ return true;
+#else
+ return false;
+#endif
+ }
+
+ case GGML_TYPE_Q5_K: {
+ if (Btype != GGML_TYPE_Q8_K) {
+ return false;
+ }
+#if defined(__VXE__) || defined(__VXE2__)
+ tinyBLAS_QK_VXE<block_q5_K> tb{
+ k, (const block_q5_K *)A, lda,
+ (const block_q8_K *)B, ldb,
+ (float *)C, ldc,
+ params->ith, params->nth};
+ tb.matmul(m, n);
+ return true;
+#else
+ return false;
+#endif
+ }
+
+ case GGML_TYPE_Q6_K: {
+ if (Btype != GGML_TYPE_Q8_K) {
+ return false;
+ }
+#if defined(__VXE__) || defined(__VXE2__)
+ tinyBLAS_QK_VXE<block_q6_K> tb{
+ k, (const block_q6_K *)A, lda,
+ (const block_q8_K *)B, ldb,
+ (float *)C, ldc,
+ params->ith, params->nth};
+ tb.matmul(m, n);
+ return true;
+#else
+ return false;
+#endif
+ }
+
case GGML_TYPE_IQ4_NL: {
if (Btype != GGML_TYPE_Q8_0)
return false;