Commit e0a9118fe for llama.cpp

commit e0a9118fe9ed9229b715f4bd83fd9767963659c6
Author: Souleyman <79574807+Sou-ly@users.noreply.github.com>
Date:   Tue Nov 17 17:46:24 2026 +0900

    metal : add implicit GEMM kernel for CONV_2D (#28069)

diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp
index 0d18a5f01..3fa6856e1 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-device.cpp
@@ -2540,6 +2540,37 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d(ggml_met
     return res;
 }

+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm(ggml_metal_library_t lib, const ggml_tensor * op) {
+    assert(op->op == GGML_OP_CONV_2D);
+
+    GGML_ASSERT(ggml_is_contiguous(op->src[0]));
+    GGML_ASSERT(op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32);
+    GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32);
+    GGML_ASSERT(op->type         == GGML_TYPE_F32);
+
+    char base[256];
+    char name[256];
+
+    const int nr0 = op->src[0]->ne[3] <= 32 ? 32 : 64;
+    const int nr1 = 2048/nr0;
+
+    snprintf(base, 256, "kernel_conv_2d_mm_%s_%s_%dx%d", ggml_type_name(op->src[0]->type), ggml_type_name(op->src[1]->type), nr0, nr1);
+    snprintf(name, 256, "%s", base);
+
+    ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
+    if (!res.pipeline) {
+        res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);
+    }
+
+    res.nr0 = nr0;
+    res.nr1 = nr1;
+    res.nsg = 4;
+
+    res.smem = nr0*nr1*sizeof(float);
+
+    return res;
+}
+
 ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_dw(ggml_metal_library_t lib, const ggml_tensor * op, bool tiled) {
     assert(op->op == GGML_OP_CONV_2D_DW);

diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h
index 4053a624e..df22635e0 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.h
+++ b/ggml/src/ggml-metal/ggml-metal-device.h
@@ -169,6 +169,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_tran
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_col2im_1d         (ggml_metal_library_t lib, const struct ggml_tensor * op);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_snake             (ggml_metal_library_t lib, enum ggml_type type);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d           (ggml_metal_library_t lib, const struct ggml_tensor * op);
+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm        (ggml_metal_library_t lib, const struct ggml_tensor * op);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_dw        (ggml_metal_library_t lib, const struct ggml_tensor * op, bool tiled);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_3d           (ggml_metal_library_t lib, const struct ggml_tensor * op);
 struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_upscale           (ggml_metal_library_t lib, const struct ggml_tensor * op);
diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp
index 5c93bc4f0..65a81b81e 100644
--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp
@@ -5014,6 +5014,31 @@ int ggml_metal_op_conv_2d(ggml_metal_op_t ctx, int idx) {
         /*.d1   =*/ d1,
     };

+    const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx->dev);
+
+    if (props_dev->has_simdgroup_mm) {
+        auto pipeline = ggml_metal_library_get_pipeline_conv_2d_mm(lib, op);
+
+        const int nr0 = pipeline.nr0;
+        const int nr1 = pipeline.nr1;
+        const int nsg = pipeline.nsg;
+
+        const int64_t NP = ne3*ne1*ne0;
+        const int64_t OC = ne03;
+
+        ggml_metal_encoder_set_pipeline(enc, pipeline);
+        ggml_metal_encoder_set_bytes   (enc, &args, sizeof(args), 0);
+        ggml_metal_encoder_set_buffer  (enc, ggml_metal_get_buffer_id(op->src[0]), 1);
+        ggml_metal_encoder_set_buffer  (enc, ggml_metal_get_buffer_id(op->src[1]), 2);
+        ggml_metal_encoder_set_buffer  (enc, ggml_metal_get_buffer_id(op),         3);
+
+        ggml_metal_encoder_set_threadgroup_memory_size(enc, pipeline.smem, 0);
+
+        ggml_metal_encoder_dispatch_threadgroups(enc, (NP + nr1 - 1)/nr1, (OC + nr0 - 1)/nr0, 1, 32, nsg, 1);
+
+        return 1;
+    }
+
     auto pipeline = ggml_metal_library_get_pipeline_conv_2d(lib, op);

     int nth = ggml_metal_pipeline_max_theads_per_threadgroup(pipeline);
diff --git a/ggml/src/ggml-metal/kernels/conv.metal b/ggml/src/ggml-metal/kernels/conv.metal
index a5d5aa9d9..fdc32756a 100644
--- a/ggml/src/ggml-metal/kernels/conv.metal
+++ b/ggml/src/ggml-metal/kernels/conv.metal
@@ -274,6 +274,187 @@ kernel void kernel_conv_2d<half>(
         uint3   tpitg[[thread_position_in_threadgroup]],
         uint3     ntg[[threads_per_threadgroup]]);

+template <typename TK, short NR0, short NR1>
+kernel void kernel_conv_2d_mm(
+        constant ggml_metal_kargs_conv_2d & args,
+        device const char * weights,
+        device const char * src,
+        device       char * dst,
+        threadgroup  char * shmem [[threadgroup(0)]],
+        uint3   tgpig[[threadgroup_position_in_grid]],
+        ushort  tiitg[[thread_index_in_threadgroup]],
+        ushort  tiisg[[thread_index_in_simdgroup]],
+        ushort  sgitg[[simdgroup_index_in_threadgroup]]) {
+    constexpr short NSG = 4;
+    constexpr short NTH = 32*NSG;
+    constexpr short NK  = 32;
+    constexpr short NKB = NK/8;
+
+    constexpr short NB = 72;
+
+    constexpr short SGY = NR0/32;
+
+    threadgroup half * sa = (threadgroup half *) shmem;
+    threadgroup half * sb = (threadgroup half *) shmem + (NR0/8)*NKB*NB;
+
+    const int K  = args.IC*args.KH*args.KW;
+    const int NP = args.N*args.OH*args.OW;
+
+    const int r0 = tgpig.y*NR0;
+    const int r1 = tgpig.x*NR1;
+
+    constexpr short NPL = NR1/32;
+
+    const short lb_n = 8*sgitg + tiisg%8;
+    const short lb_k = 8*(tiisg/8);
+
+    int  ix0[NPL];
+    int  iy0[NPL];
+    bool ip_ok[NPL];
+
+    device const char * srcb[NPL];
+
+    FOR_UNROLL (short t = 0; t < NPL; t++) {
+        const int ip = r1 + 32*t + lb_n;
+
+        const int iow = ip % args.OW;
+        const int ioh = (ip / args.OW) % args.OH;
+        const int in  = ip / (args.OW*args.OH);
+
+        ix0[t] = iow*args.s0 - args.p0;
+        iy0[t] = ioh*args.s1 - args.p1;
+
+        ip_ok[t] = ip < NP;
+
+        srcb[t] = src + in*args.nb13;
+    }
+
+    simdgroup_half8x8  ma[4];
+    simdgroup_half8x8  mb[2];
+    simdgroup_float8x8 mc[8];
+
+    for (short i = 0; i < 8; i++) {
+        mc[i] = make_filled_simdgroup_matrix<float, 8>(0.f);
+    }
+
+    for (int k0 = 0; k0 < K; k0 += NK) {
+        threadgroup_barrier(mem_flags::mem_threadgroup);
+
+        {
+            const short k  = tiisg;
+            const int   kk = k0 + k;
+
+            FOR_UNROLL (short j = 0; j < NR0/NSG; j++) {
+                const short row = (NR0/NSG)*sgitg + j;
+                const int   oc  = r0 + row;
+
+                half v = 0;
+                if (kk < K && oc < args.OC) {
+                    v = *((device const TK *)(weights + oc*args.nb03) + kk);
+                }
+
+                const short ib = NKB*(row/8) + k/8;
+
+                sa[NB*ib + 8*(row%8) + k%8] = v;
+            }
+        }
+
+        {
+            int kk = k0 + lb_k;
+
+            int kw = kk % args.KW;
+            int kh = (kk / args.KW) % args.KH;
+            int ic = kk / (args.KW*args.KH);
+
+            FOR_UNROLL (short i = 0; i < 8; i++) {
+                const short k = lb_k + i;
+
+                FOR_UNROLL (short t = 0; t < NPL; t++) {
+                    const short n = 32*t + lb_n;
+
+                    const int ix = ix0[t] + kw*args.d0;
+                    const int iy = iy0[t] + kh*args.d1;
+
+                    half v = 0;
+                    if (ip_ok[t] && kk < K && ix >= 0 && ix < args.IW && iy >= 0 && iy < args.IH) {
+                        v = *(device const float *)(srcb[t] + ic*args.nb12 + iy*args.nb11 + ix*args.nb10);
+                    }
+
+                    const short ib = NKB*(n/8) + k/8;
+
+                    sb[NB*ib + 8*(k%8) + n%8] = v;
+                }
+
+                kk++;
+                if (++kw == args.KW) {
+                    kw = 0;
+                    if (++kh == args.KH) {
+                        kh = 0;
+                        ic++;
+                    }
+                }
+            }
+        }
+
+        threadgroup_barrier(mem_flags::mem_threadgroup);
+
+        const short nkb = min((K - k0 + 7)/8, (int) NKB);
+
+        for (short ik = 0; ik < nkb; ik++) {
+            FOR_UNROLL (short i = 0; i < 4; i++) {
+                simdgroup_load(ma[i], sa + NB*(NKB*(4*(sgitg%SGY) + i) + ik), 8, 0, false);
+            }
+
+            FOR_UNROLL (short j = 0; j < 2; j++) {
+                simdgroup_load(mb[j], sb + NB*(NKB*(2*(sgitg/SGY) + j) + ik), 8, 0, false);
+            }
+
+            FOR_UNROLL (short i = 0; i < 8; i++) {
+                simdgroup_multiply_accumulate(mc[i], ma[i%4], mb[i/4], mc[i]);
+            }
+        }
+    }
+
+    threadgroup_barrier(mem_flags::mem_threadgroup);
+
+    threadgroup float * sc = (threadgroup float *) shmem;
+
+    for (short i = 0; i < 8; i++) {
+        simdgroup_store(mc[i], sc + NR1*(32*(sgitg%SGY) + 8*(i%4)) + 16*(sgitg/SGY) + 8*(i/4), NR1, 0, false);
+    }
+
+    threadgroup_barrier(mem_flags::mem_threadgroup);
+
+    const short cn = tiitg%NR1;
+    const int   p  = r1 + cn;
+
+    if (p >= NP) {
+        return;
+    }
+
+    const int ow = p % args.OW;
+    const int oh = (p / args.OW) % args.OH;
+    const int n  = p / (args.OW*args.OH);
+
+    device char * dstp = dst + n*args.nb3 + oh*args.nb1 + ow*args.nb0;
+
+    for (short j = 0; j < NR0*NR1/NTH; j++) {
+        const short row = tiitg/NR1 + (NTH/NR1)*j;
+        const int   oc  = r0 + row;
+
+        if (oc < args.OC) {
+            *(device float *)(dstp + oc*args.nb2) = sc[NR1*row + cn];
+        }
+    }
+}
+
+typedef decltype(kernel_conv_2d_mm<float, 64, 32>) kernel_conv_2d_mm_t;
+
+template [[host_name("kernel_conv_2d_mm_f32_f32_64x32")]] kernel kernel_conv_2d_mm_t kernel_conv_2d_mm<float, 64, 32>;
+template [[host_name("kernel_conv_2d_mm_f16_f32_64x32")]] kernel kernel_conv_2d_mm_t kernel_conv_2d_mm<half,  64, 32>;
+template [[host_name("kernel_conv_2d_mm_f32_f32_32x64")]] kernel kernel_conv_2d_mm_t kernel_conv_2d_mm<float, 32, 64>;
+template [[host_name("kernel_conv_2d_mm_f16_f32_32x64")]] kernel kernel_conv_2d_mm_t kernel_conv_2d_mm<half,  32, 64>;
+
 typedef void (conv_transpose_1d_t)(
         constant ggml_metal_kargs_conv_transpose_1d & args,
         device const float * src0,