Commit e0a9118fe for llama.cpp
commit e0a9118fe9ed9229b715f4bd83fd9767963659c6
Author: Souleyman <79574807+Sou-ly@users.noreply.github.com>
Date: Tue Nov 17 17:46:24 2026 +0900
metal : add implicit GEMM kernel for CONV_2D (#28069)
diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp
index 0d18a5f01..3fa6856e1 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-device.cpp
@@ -2540,6 +2540,37 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d(ggml_met
return res;
}
+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm(ggml_metal_library_t lib, const ggml_tensor * op) {
+ assert(op->op == GGML_OP_CONV_2D);
+
+ GGML_ASSERT(ggml_is_contiguous(op->src[0]));
+ GGML_ASSERT(op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_F32);
+ GGML_ASSERT(op->src[1]->type == GGML_TYPE_F32);
+ GGML_ASSERT(op->type == GGML_TYPE_F32);
+
+ char base[256];
+ char name[256];
+
+ const int nr0 = op->src[0]->ne[3] <= 32 ? 32 : 64;
+ const int nr1 = 2048/nr0;
+
+ snprintf(base, 256, "kernel_conv_2d_mm_%s_%s_%dx%d", ggml_type_name(op->src[0]->type), ggml_type_name(op->src[1]->type), nr0, nr1);
+ snprintf(name, 256, "%s", base);
+
+ ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
+ if (!res.pipeline) {
+ res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);
+ }
+
+ res.nr0 = nr0;
+ res.nr1 = nr1;
+ res.nsg = 4;
+
+ res.smem = nr0*nr1*sizeof(float);
+
+ return res;
+}
+
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_dw(ggml_metal_library_t lib, const ggml_tensor * op, bool tiled) {
assert(op->op == GGML_OP_CONV_2D_DW);
diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h
index 4053a624e..df22635e0 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.h
+++ b/ggml/src/ggml-metal/ggml-metal-device.h
@@ -169,6 +169,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_tran
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_col2im_1d (ggml_metal_library_t lib, const struct ggml_tensor * op);
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_snake (ggml_metal_library_t lib, enum ggml_type type);
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d (ggml_metal_library_t lib, const struct ggml_tensor * op);
+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm (ggml_metal_library_t lib, const struct ggml_tensor * op);
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_dw (ggml_metal_library_t lib, const struct ggml_tensor * op, bool tiled);
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_3d (ggml_metal_library_t lib, const struct ggml_tensor * op);
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_upscale (ggml_metal_library_t lib, const struct ggml_tensor * op);
diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp
index 5c93bc4f0..65a81b81e 100644
--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp
@@ -5014,6 +5014,31 @@ int ggml_metal_op_conv_2d(ggml_metal_op_t ctx, int idx) {
/*.d1 =*/ d1,
};
+ const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx->dev);
+
+ if (props_dev->has_simdgroup_mm) {
+ auto pipeline = ggml_metal_library_get_pipeline_conv_2d_mm(lib, op);
+
+ const int nr0 = pipeline.nr0;
+ const int nr1 = pipeline.nr1;
+ const int nsg = pipeline.nsg;
+
+ const int64_t NP = ne3*ne1*ne0;
+ const int64_t OC = ne03;
+
+ ggml_metal_encoder_set_pipeline(enc, pipeline);
+ ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);
+ ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);
+
+ ggml_metal_encoder_set_threadgroup_memory_size(enc, pipeline.smem, 0);
+
+ ggml_metal_encoder_dispatch_threadgroups(enc, (NP + nr1 - 1)/nr1, (OC + nr0 - 1)/nr0, 1, 32, nsg, 1);
+
+ return 1;
+ }
+
auto pipeline = ggml_metal_library_get_pipeline_conv_2d(lib, op);
int nth = ggml_metal_pipeline_max_theads_per_threadgroup(pipeline);
diff --git a/ggml/src/ggml-metal/kernels/conv.metal b/ggml/src/ggml-metal/kernels/conv.metal
index a5d5aa9d9..fdc32756a 100644
--- a/ggml/src/ggml-metal/kernels/conv.metal
+++ b/ggml/src/ggml-metal/kernels/conv.metal
@@ -274,6 +274,187 @@ kernel void kernel_conv_2d<half>(
uint3 tpitg[[thread_position_in_threadgroup]],
uint3 ntg[[threads_per_threadgroup]]);
+template <typename TK, short NR0, short NR1>
+kernel void kernel_conv_2d_mm(
+ constant ggml_metal_kargs_conv_2d & args,
+ device const char * weights,
+ device const char * src,
+ device char * dst,
+ threadgroup char * shmem [[threadgroup(0)]],
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ ushort tiitg[[thread_index_in_threadgroup]],
+ ushort tiisg[[thread_index_in_simdgroup]],
+ ushort sgitg[[simdgroup_index_in_threadgroup]]) {
+ constexpr short NSG = 4;
+ constexpr short NTH = 32*NSG;
+ constexpr short NK = 32;
+ constexpr short NKB = NK/8;
+
+ constexpr short NB = 72;
+
+ constexpr short SGY = NR0/32;
+
+ threadgroup half * sa = (threadgroup half *) shmem;
+ threadgroup half * sb = (threadgroup half *) shmem + (NR0/8)*NKB*NB;
+
+ const int K = args.IC*args.KH*args.KW;
+ const int NP = args.N*args.OH*args.OW;
+
+ const int r0 = tgpig.y*NR0;
+ const int r1 = tgpig.x*NR1;
+
+ constexpr short NPL = NR1/32;
+
+ const short lb_n = 8*sgitg + tiisg%8;
+ const short lb_k = 8*(tiisg/8);
+
+ int ix0[NPL];
+ int iy0[NPL];
+ bool ip_ok[NPL];
+
+ device const char * srcb[NPL];
+
+ FOR_UNROLL (short t = 0; t < NPL; t++) {
+ const int ip = r1 + 32*t + lb_n;
+
+ const int iow = ip % args.OW;
+ const int ioh = (ip / args.OW) % args.OH;
+ const int in = ip / (args.OW*args.OH);
+
+ ix0[t] = iow*args.s0 - args.p0;
+ iy0[t] = ioh*args.s1 - args.p1;
+
+ ip_ok[t] = ip < NP;
+
+ srcb[t] = src + in*args.nb13;
+ }
+
+ simdgroup_half8x8 ma[4];
+ simdgroup_half8x8 mb[2];
+ simdgroup_float8x8 mc[8];
+
+ for (short i = 0; i < 8; i++) {
+ mc[i] = make_filled_simdgroup_matrix<float, 8>(0.f);
+ }
+
+ for (int k0 = 0; k0 < K; k0 += NK) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ {
+ const short k = tiisg;
+ const int kk = k0 + k;
+
+ FOR_UNROLL (short j = 0; j < NR0/NSG; j++) {
+ const short row = (NR0/NSG)*sgitg + j;
+ const int oc = r0 + row;
+
+ half v = 0;
+ if (kk < K && oc < args.OC) {
+ v = *((device const TK *)(weights + oc*args.nb03) + kk);
+ }
+
+ const short ib = NKB*(row/8) + k/8;
+
+ sa[NB*ib + 8*(row%8) + k%8] = v;
+ }
+ }
+
+ {
+ int kk = k0 + lb_k;
+
+ int kw = kk % args.KW;
+ int kh = (kk / args.KW) % args.KH;
+ int ic = kk / (args.KW*args.KH);
+
+ FOR_UNROLL (short i = 0; i < 8; i++) {
+ const short k = lb_k + i;
+
+ FOR_UNROLL (short t = 0; t < NPL; t++) {
+ const short n = 32*t + lb_n;
+
+ const int ix = ix0[t] + kw*args.d0;
+ const int iy = iy0[t] + kh*args.d1;
+
+ half v = 0;
+ if (ip_ok[t] && kk < K && ix >= 0 && ix < args.IW && iy >= 0 && iy < args.IH) {
+ v = *(device const float *)(srcb[t] + ic*args.nb12 + iy*args.nb11 + ix*args.nb10);
+ }
+
+ const short ib = NKB*(n/8) + k/8;
+
+ sb[NB*ib + 8*(k%8) + n%8] = v;
+ }
+
+ kk++;
+ if (++kw == args.KW) {
+ kw = 0;
+ if (++kh == args.KH) {
+ kh = 0;
+ ic++;
+ }
+ }
+ }
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ const short nkb = min((K - k0 + 7)/8, (int) NKB);
+
+ for (short ik = 0; ik < nkb; ik++) {
+ FOR_UNROLL (short i = 0; i < 4; i++) {
+ simdgroup_load(ma[i], sa + NB*(NKB*(4*(sgitg%SGY) + i) + ik), 8, 0, false);
+ }
+
+ FOR_UNROLL (short j = 0; j < 2; j++) {
+ simdgroup_load(mb[j], sb + NB*(NKB*(2*(sgitg/SGY) + j) + ik), 8, 0, false);
+ }
+
+ FOR_UNROLL (short i = 0; i < 8; i++) {
+ simdgroup_multiply_accumulate(mc[i], ma[i%4], mb[i/4], mc[i]);
+ }
+ }
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ threadgroup float * sc = (threadgroup float *) shmem;
+
+ for (short i = 0; i < 8; i++) {
+ simdgroup_store(mc[i], sc + NR1*(32*(sgitg%SGY) + 8*(i%4)) + 16*(sgitg/SGY) + 8*(i/4), NR1, 0, false);
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ const short cn = tiitg%NR1;
+ const int p = r1 + cn;
+
+ if (p >= NP) {
+ return;
+ }
+
+ const int ow = p % args.OW;
+ const int oh = (p / args.OW) % args.OH;
+ const int n = p / (args.OW*args.OH);
+
+ device char * dstp = dst + n*args.nb3 + oh*args.nb1 + ow*args.nb0;
+
+ for (short j = 0; j < NR0*NR1/NTH; j++) {
+ const short row = tiitg/NR1 + (NTH/NR1)*j;
+ const int oc = r0 + row;
+
+ if (oc < args.OC) {
+ *(device float *)(dstp + oc*args.nb2) = sc[NR1*row + cn];
+ }
+ }
+}
+
+typedef decltype(kernel_conv_2d_mm<float, 64, 32>) kernel_conv_2d_mm_t;
+
+template [[host_name("kernel_conv_2d_mm_f32_f32_64x32")]] kernel kernel_conv_2d_mm_t kernel_conv_2d_mm<float, 64, 32>;
+template [[host_name("kernel_conv_2d_mm_f16_f32_64x32")]] kernel kernel_conv_2d_mm_t kernel_conv_2d_mm<half, 64, 32>;
+template [[host_name("kernel_conv_2d_mm_f32_f32_32x64")]] kernel kernel_conv_2d_mm_t kernel_conv_2d_mm<float, 32, 64>;
+template [[host_name("kernel_conv_2d_mm_f16_f32_32x64")]] kernel kernel_conv_2d_mm_t kernel_conv_2d_mm<half, 32, 64>;
+
typedef void (conv_transpose_1d_t)(
constant ggml_metal_kargs_conv_transpose_1d & args,
device const float * src0,