Commit 660be5506 for llama.cpp

commit 660be5506b15eae512f0df3ae0cc168712555319
Author: Kazutoshi, Akao <73567414+kazutoshi-ak@users.noreply.github.com>
Date:   Sat Nov 21 23:16:40 2026 +0900

    ggml : improvements to the memory access pattern in `ggml_compute_forward_mul_mat_one_chunk` (#30846)

diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
index 4d9760bad..33193a5af 100644
--- a/ggml/src/ggml-cpu/ggml-cpu.c
+++ b/ggml/src/ggml-cpu/ggml-cpu.c
@@ -1169,6 +1169,28 @@ void ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2,

 ////////////////////////////////////////////////////////////////////////////////

+static inline void ggml_deinterleave_f32(
+        float * GGML_RESTRICT dst0,
+        float * GGML_RESTRICT dst1,
+        const float * GGML_RESTRICT src,
+        int64_t n,
+        size_t bs,
+        int nrc) {
+    GGML_ASSERT(nrc >= 1 && nrc <= 2);
+
+    if (nrc == 1) {
+        memcpy(dst0, src, n * sizeof(float));
+        return;
+    }
+
+    for (int64_t i = 0; i < n; ++i) {
+        dst0[i     ] = src[2*i + 0     ];
+        dst1[i     ] = src[2*i + 1     ];
+        dst0[bs + i] = src[2*i + 0 + 16];
+        dst1[bs + i] = src[2*i + 1 + 16];
+    }
+}
+
 // ggml_compute_forward_mul_mat

 static void ggml_compute_forward_mul_mat_one_chunk(
@@ -1218,8 +1240,12 @@ static void ggml_compute_forward_mul_mat_one_chunk(
     // 16 * 2, accounting for mmla kernels
     float tmp[32];

+    const int64_t nr0_split = (ir0_end - ir0_start) / num_rows_per_vec_dot; // rows per stream
+    const int64_t ir0_end_split = ir0_start + nr0_split;
+    const int64_t blck_0_split = blck_0 / num_rows_per_vec_dot;
+
     for (int64_t iir1 = ir1_start; iir1 < ir1_end; iir1 += blck_1) {
-        for (int64_t iir0 = ir0_start; iir0 < ir0_end; iir0 += blck_0) {
+        for (int64_t iir0 = ir0_start; iir0 < ir0_end_split; iir0 += blck_0_split) {
             for (int64_t ir1 = iir1; ir1 < iir1 + blck_1 && ir1 < ir1_end; ir1 += num_rows_per_vec_dot) {
                 const int64_t i13 = (ir1 / (ne12 * ne1));
                 const int64_t i12 = (ir1 - i13 * ne12 * ne1) / ne1;
@@ -1249,13 +1275,13 @@ static void ggml_compute_forward_mul_mat_one_chunk(
                 //    vec_dot(ne00, &dst_col[ir0], src0_row + ir0*nb01, src1_col);
                 //}

-                for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ir0 += num_rows_per_vec_dot) {
-                    vec_dot(ne00, &tmp[ir0 - iir0], (num_rows_per_vec_dot > 1 ? 16 : 0), src0_row + ir0 * nb01, (num_rows_per_vec_dot > 1 ? nb01 : 0), src1_col, (num_rows_per_vec_dot > 1 ? src1_col_stride : 0), num_rows_per_vec_dot);
+                for (int64_t ir0 = iir0; ir0 < iir0 + blck_0_split && ir0 < ir0_end_split; ++ir0) {
+                    vec_dot(ne00, &tmp[(ir0 - iir0) * num_rows_per_vec_dot], (num_rows_per_vec_dot > 1 ? 16 : 0), src0_row + ir0 * nb01, (num_rows_per_vec_dot > 1 ? nb01 * nr0_split : 0), src1_col, (num_rows_per_vec_dot > 1 ? src1_col_stride : 0), num_rows_per_vec_dot);
                 }

-                for (int cn = 0; cn < num_rows_per_vec_dot; ++cn) {
-                    memcpy(&dst_col[iir0 + cn * nb1 / nb0], tmp + (cn * 16), (MIN(iir0 + blck_0, ir0_end) - iir0) * sizeof(float));
-                }
+                const int64_t n = MIN(iir0 + blck_0_split, ir0_end_split) - iir0;
+                float * dst0 = dst_col + iir0;
+                ggml_deinterleave_f32(dst0, dst0 + nr0_split, tmp, n, nb1 / nb0, num_rows_per_vec_dot);
             }
         }
     }