Commit 660be5506 for llama.cpp
commit 660be5506b15eae512f0df3ae0cc168712555319
Author: Kazutoshi, Akao <73567414+kazutoshi-ak@users.noreply.github.com>
Date: Sat Nov 21 23:16:40 2026 +0900
ggml : improvements to the memory access pattern in `ggml_compute_forward_mul_mat_one_chunk` (#30846)
diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
index 4d9760bad..33193a5af 100644
--- a/ggml/src/ggml-cpu/ggml-cpu.c
+++ b/ggml/src/ggml-cpu/ggml-cpu.c
@@ -1169,6 +1169,28 @@ void ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2,
////////////////////////////////////////////////////////////////////////////////
+static inline void ggml_deinterleave_f32(
+ float * GGML_RESTRICT dst0,
+ float * GGML_RESTRICT dst1,
+ const float * GGML_RESTRICT src,
+ int64_t n,
+ size_t bs,
+ int nrc) {
+ GGML_ASSERT(nrc >= 1 && nrc <= 2);
+
+ if (nrc == 1) {
+ memcpy(dst0, src, n * sizeof(float));
+ return;
+ }
+
+ for (int64_t i = 0; i < n; ++i) {
+ dst0[i ] = src[2*i + 0 ];
+ dst1[i ] = src[2*i + 1 ];
+ dst0[bs + i] = src[2*i + 0 + 16];
+ dst1[bs + i] = src[2*i + 1 + 16];
+ }
+}
+
// ggml_compute_forward_mul_mat
static void ggml_compute_forward_mul_mat_one_chunk(
@@ -1218,8 +1240,12 @@ static void ggml_compute_forward_mul_mat_one_chunk(
// 16 * 2, accounting for mmla kernels
float tmp[32];
+ const int64_t nr0_split = (ir0_end - ir0_start) / num_rows_per_vec_dot; // rows per stream
+ const int64_t ir0_end_split = ir0_start + nr0_split;
+ const int64_t blck_0_split = blck_0 / num_rows_per_vec_dot;
+
for (int64_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) {
- for (int64_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) {
+ for (int64_t iir0 = ir0_start; iir0 < ir0_end_split; iir0 += blck_0_split) {
for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir1_end; ir1 += num_rows_per_vec_dot) {
const int64_t i13 = (ir1 / (ne12 * ne1));
const int64_t i12 = (ir1 - i13 * ne12 * ne1) / ne1;
@@ -1249,13 +1275,13 @@ static void ggml_compute_forward_mul_mat_one_chunk(
// vec_dot(ne00, &dst_col[ir0], src0_row + ir0*nb01, src1_col);
//}
- for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ir0 += num_rows_per_vec_dot) {
- vec_dot(ne00, &tmp[ir0 - iir0], (num_rows_per_vec_dot > 1 ? 16 : 0), src0_row + ir0 * nb01, (num_rows_per_vec_dot > 1 ? nb01 : 0), src1_col, (num_rows_per_vec_dot > 1 ? src1_col_stride : 0), num_rows_per_vec_dot);
+ for (int64_t ir0 = iir0; ir0 < iir0 + blck_0_split && ir0 < ir0_end_split; ++ir0) {
+ vec_dot(ne00, &tmp[(ir0 - iir0) * num_rows_per_vec_dot], (num_rows_per_vec_dot > 1 ? 16 : 0), src0_row + ir0 * nb01, (num_rows_per_vec_dot > 1 ? nb01 * nr0_split : 0), src1_col, (num_rows_per_vec_dot > 1 ? src1_col_stride : 0), num_rows_per_vec_dot);
}
- for (int cn = 0; cn < num_rows_per_vec_dot; ++cn) {
- memcpy(&dst_col[iir0 + cn * nb1 / nb0], tmp + (cn * 16), (MIN(iir0 + blck_0, ir0_end) - iir0) * sizeof(float));
- }
+ const int64_t n = MIN(iir0 + blck_0_split, ir0_end_split) - iir0;
+ float * dst0 = dst_col + iir0;
+ ggml_deinterleave_f32(dst0, dst0 + nr0_split, tmp, n, nb1 / nb0, num_rows_per_vec_dot);
}
}
}