Commit a3c7cf08f for llama.cpp
commit a3c7cf08f98fd769f246fcb72be8872d54ff3c35
Author: Aaron Teo <aaron.teo1@ibm.com>
Date: Sun Nov 8 22:44:25 2026 +0800
llamafile: impl bf16 and q8_0 vxe (#31866)
Signed-off-by: Aaron Teo <aaron.teo1@ibm.com>
diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp
index 6d39cd6e7..384f7dbe2 100644
--- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp
+++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp
@@ -326,6 +326,12 @@ template <> inline float32x4_t load(const ggml_fp16_t * p) {
return vec_xl(0, (const float *)(tmp));
}
+template <> inline float32x4_t load(const ggml_bf16_t * p) {
+ uint64_t bits;
+ memcpy(&bits, p, sizeof(bits));
+ const uint64x2_t v_bits = { bits, bits };
+ return GGML_BF16_TO_F32_HI((GGML_BF16_VEC) v_bits);
+}
template <> inline float32x4_t load(const float * p) {
return vec_xl(0, p);
}
@@ -1302,6 +1308,88 @@ class tinyBLAS_RVV {
//////////////////////////////////////////////////////////////////////////////////////////
// QUANT ZERO MATRIX MULTIPLICATION
+#if defined(__VXE__) || defined(__VXE2__)
+template <typename TA>
+class tinyBLAS_Q0_VXE {
+ static_assert(sizeof(TA::qs) == QK8_0, "VXE weight loads require 32 quantized bytes per block");
+
+ public:
+ tinyBLAS_Q0_VXE(int64_t k,
+ const TA * A, int64_t lda,
+ const block_q8_0 * B, int64_t ldb,
+ float * C, int64_t ldc,
+ int32_t ith, int32_t nth)
+ : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
+ }
+
+ void matmul(int64_t m, int64_t n) {
+ constexpr int32_t tile_size = 2;
+ const int64_t row_tiles = m / tile_size;
+ const int64_t tiles = row_tiles * (n / tile_size);
+ const int64_t duty = (tiles + nth - 1) / nth;
+ const int64_t start = duty * ith;
+ const int64_t end = MIN(start + duty, tiles);
+
+ for (int64_t job = start; job < end; ++job) {
+ const int64_t ii = tile_size * (job % row_tiles);
+ const int64_t jj = tile_size * (job / row_tiles);
+ float32x4_t v_sum[tile_size][tile_size] = {};
+
+ for (int64_t l = 0; l < k; ++l) {
+ int8x16_t v_xl[tile_size], v_xh[tile_size], v_yl[tile_size], v_yh[tile_size];
+ float x_scale[tile_size], y_scale[tile_size];
+
+ for (int32_t i = 0; i < tile_size; ++i) {
+ const TA & x = A[lda * (ii + i) + l];
+ const block_q8_0 & y = B[ldb * (jj + i) + l];
+
+ const int8_t * x_qs = x.qs;
+ const int8_t * y_qs = y.qs;
+ v_xl[i] = vec_xl(0, x_qs);
+ v_xh[i] = vec_xl(0, x_qs + QK8_0 / 2);
+ v_yl[i] = vec_xl(0, y_qs);
+ v_yh[i] = vec_xl(0, y_qs + QK8_0 / 2);
+ x_scale[i] = unhalf(x.d);
+ y_scale[i] = unhalf(y.d);
+ }
+
+ for (int32_t j = 0; j < tile_size; ++j) {
+ for (int32_t i = 0; i < tile_size; ++i) {
+ const int32x4_t v_dot = dot_s8_wide(v_xl[i], v_yl[j]) + dot_s8_wide(v_xh[i], v_yh[j]);
+ const float32x4_t v_scale = vec_splats(x_scale[i] * y_scale[j]);
+ v_sum[j][i] = vec_madd(vec_float(v_dot), v_scale, v_sum[j][i]);
+ }
+ }
+ }
+
+ for (int32_t j = 0; j < tile_size; ++j) {
+ for (int32_t i = 0; i < tile_size; ++i) {
+ C[ldc * (jj + j) + ii + i] = hsum(v_sum[j][i]);
+ }
+ }
+ }
+ }
+
+ private:
+ static int32x4_t dot_s8_wide(int8x16_t v_x, int8x16_t v_y) {
+ const int16x8_t v_even = vec_mule(v_x, v_y);
+ const int16x8_t v_odd = vec_mulo(v_x, v_y);
+ // Two (-128) * (-128) products can overflow a signed 16-bit sum.
+ return vec_unpackh(v_even) + vec_unpackl(v_even) + vec_unpackh(v_odd) + vec_unpackl(v_odd);
+ }
+
+ const TA * const A;
+ const block_q8_0 * const B;
+ float * const C;
+ const int64_t k;
+ const int64_t lda;
+ const int64_t ldb;
+ const int64_t ldc;
+ const int32_t ith;
+ const int32_t nth;
+};
+#endif
+
#if defined(__ARM_FEATURE_DOTPROD)
template <typename TA>
class tinyBLAS_Q0_ARM {
@@ -4006,6 +4094,14 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64
(float *)C, ldc};
return tb.matmul(m, n);
}
+#elif defined(__VXE__) || defined(__VXE2__)
+ if (Btype == GGML_TYPE_BF16) {
+ tinyBLAS<4, float32x4_t, float32x4_t, ggml_bf16_t, ggml_bf16_t, float> tb{ params, k,
+ (const ggml_bf16_t *)A, lda,
+ (const ggml_bf16_t *)B, ldb,
+ (float *)C, ldc};
+ return tb.matmul(m, n);
+ }
#elif defined(__MMA__)
if (k % 8) {
return false;
@@ -4162,6 +4258,17 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64
params->ith, params->nth};
tb.matmul(m, n);
return true;
+#elif defined(__VXE__) || defined(__VXE2__)
+ if (m % 2 || n % 2) {
+ return false;
+ }
+ tinyBLAS_Q0_VXE<block_q8_0> tb{
+ k, (const block_q8_0 *)A, lda,
+ (const block_q8_0 *)B, ldb,
+ (float *)C, ldc,
+ params->ith, params->nth};
+ tb.matmul(m, n);
+ return true;
#elif defined(__MMA__)
//TO-DO: Remove this condition once gemv forwarding is enabled.
if (n < 8 && n != 4)