Commit 41d8297fe for llama.cpp
commit 41d8297fe4c26b9ff304805178199dc14aa9ab52
Author: Pascal <admin@serveurperso.com>
Date: Thu Nov 19 10:48:44 2026 +0100
metal : CONV_2D tensor API for M5, split-K for all chips (#32542)
* metal: add a tensor API conv_2d kernel and split long reductions
The conv_2d matmul kernel runs on simdgroup matrices only, and gives
each threadgroup the whole reduction of its output tile. On M5 this
leaves the tensor cores unused, and convolutions with few output
positions and a long reduction (IC * KW * KH in the thousands) run a
handful of threadgroups on an otherwise idle GPU.
Add kernel_conv_2d_mm_tensor for devices with the tensor API: like
kernel_mul_mm, it gathers the patches of 64 output positions into
threadgroup memory as the left operand and reads the kernel rows
from device memory as the right operand. Devices without the tensor
API keep the simdgroup kernel.
Both kernels can now split the reduction until there are enough
threadgroups: each split writes its partial sums to a buffer placed
after the output, as flash attention does, and a small kernel adds
them in a fixed order, so the result stays deterministic. CONV_2D is
marked as an op whose allocation may exceed its output size.
test-backend-ops -o CONV_2D passes, with and without
GGML_METAL_TENSOR_DISABLE.
CONV_2D, F16 kernel, us/run on an M5 Max:
[19,19,256,16] k [4,4,256,4096] 19008 -> 5235
[16,16,128,8] k [3,3,128,512] 333 -> 124
[1536,1536,32,1] k [3,3,32,64] 12086 -> 6512
[33,1,1280,1] k [3,1,1280,1280] s2 659 -> 33
[2054,1,768,1] k [7,1,768,768] 3161 -> 699
* metal: walk the conv_2d patches incrementally, size splits by output
The patch gather of both conv_2d matmul kernels spent most of its
time on address math: 64-bit multiplies, divisions and four bound
checks per value. Walk the reduction index with 32-bit byte offsets
inside the image, updated by additions, with one unsigned compare per
bound; the host checks that an image fits. The tensor kernel no
longer gathers the reduction indices past the end of its split.
The number of splits now targets about 1M partial sums, with at least
3 reduction steps per split, instead of counting the tiles of one
kernel.
CONV_2D, F16 kernel, us/run on an M5 Max:
[19,19,256,16] k [4,4,256,4096] 4998 -> 4512
[1536,1536,32,1] k [3,3,32,64] 6721 -> 5354
[4096,1,1024,1] k [7,1,1024,1024] 2701 -> 2236
[2054,1,768,1] k [7,1,768,768] 1050 -> 871
with GGML_METAL_TENSOR_DISABLE:
[4096,1,1024,1] k [7,1,1024,1024] 9047 -> 8283
* nit: share the CONV_2D simdgroup count between host and kernel
kernel_conv_2d_mm and its pipeline both hardcoded 4 simdgroups per
threadgroup. N_SG_CONV_2D in ggml-metal-impl.h now feeds the kernel
constant and res.nsg, so the dispatch and the kernel cannot drift apart.
* metal: fall back to the generic conv_2d kernel for images past 4 GiB
The matmul conv_2d kernels walk the patches with 32-bit byte offsets
inside an image. Instead of asserting on nb13, compute the largest
offset actually read from ne/nb, which also covers strided views of
the input, and route images that do not fit to the generic kernel.
* cont : comments
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
---------
Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp
index 2e2f9c036..d598ca3d8 100644
--- a/ggml/src/ggml-backend.cpp
+++ b/ggml/src/ggml-backend.cpp
@@ -2584,6 +2584,7 @@ bool ggml_op_alloc_size_may_expand(enum ggml_op op) {
case GGML_OP_ARGSORT:
case GGML_OP_TOP_K:
case GGML_OP_ARGMAX:
+ case GGML_OP_CONV_2D:
return true;
default:
return false;
diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp
index 3fa6856e1..cd9e07850 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-device.cpp
@@ -2551,6 +2551,26 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm(ggml_
char base[256];
char name[256];
+ const bool has_tensor = ggml_metal_device_get_props(ggml_metal_library_get_device(lib))->has_tensor;
+
+ if (has_tensor) {
+ // nr0 output positions per threadgroup, nr1 output channels, same tiles as kernel_mul_mm
+ snprintf(base, 256, "kernel_conv_2d_mm_tensor_%s_%s", ggml_type_name(op->src[0]->type), ggml_type_name(op->src[1]->type));
+ snprintf(name, 256, "%s", base);
+
+ ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
+ if (!res.pipeline) {
+ res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);
+ }
+
+ res.nr0 = SZ_SIMDGROUP * N_MM_BLOCK_Y * N_MM_SIMD_GROUP_Y;
+ res.nr1 = SZ_SIMDGROUP * N_MM_BLOCK_X * N_MM_SIMD_GROUP_X;
+ res.nsg = N_MM_SIMD_GROUP_X * N_MM_SIMD_GROUP_Y;
+ res.smem = res.nr0 * N_MM_NK_TOTAL * sizeof(ggml_fp16_t);
+
+ return res;
+ }
+
const int nr0 = op->src[0]->ne[3] <= 32 ? 32 : 64;
const int nr1 = 2048/nr0;
@@ -2564,13 +2584,27 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm(ggml_
res.nr0 = nr0;
res.nr1 = nr1;
- res.nsg = 4;
+ res.nsg = N_SG_CONV_2D;
res.smem = nr0*nr1*sizeof(float);
return res;
}
+ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm_reduce(ggml_metal_library_t lib, const ggml_tensor * op) {
+ assert(op->op == GGML_OP_CONV_2D);
+ GGML_UNUSED(op);
+
+ const char * name = "kernel_conv_2d_mm_reduce";
+
+ ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
+ if (!res.pipeline) {
+ res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr);
+ }
+
+ return res;
+}
+
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_dw(ggml_metal_library_t lib, const ggml_tensor * op, bool tiled) {
assert(op->op == GGML_OP_CONV_2D_DW);
diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h
index df22635e0..c0466d62f 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.h
+++ b/ggml/src/ggml-metal/ggml-metal-device.h
@@ -170,6 +170,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_col2im_1d
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_snake (ggml_metal_library_t lib, enum ggml_type type);
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d (ggml_metal_library_t lib, const struct ggml_tensor * op);
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm (ggml_metal_library_t lib, const struct ggml_tensor * op);
+struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm_reduce (ggml_metal_library_t lib, const struct ggml_tensor * op);
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_dw (ggml_metal_library_t lib, const struct ggml_tensor * op, bool tiled);
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_3d (ggml_metal_library_t lib, const struct ggml_tensor * op);
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_upscale (ggml_metal_library_t lib, const struct ggml_tensor * op);
diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h
index 7fd9d97cf..aad014d83 100644
--- a/ggml/src/ggml-metal/ggml-metal-impl.h
+++ b/ggml/src/ggml-metal/ggml-metal-impl.h
@@ -17,6 +17,8 @@
#define N_ARGMAX_NPART_MAX 256
#define N_ARGMAX_CHUNK 4096
+#define N_SG_CONV_2D 4
+
// kernel parameters for mat-vec threadgroups
//
// N_R0: number of src0 rows to process per simdgroup
@@ -752,8 +754,14 @@ typedef struct {
int32_t p1;
int32_t d0;
int32_t d1;
+ int32_t nk; // reduction length per split
} ggml_metal_kargs_conv_2d;
+typedef struct {
+ int32_t ne; // output elements
+ int32_t nsplit;
+} ggml_metal_kargs_conv_2d_reduce;
+
typedef struct {
uint64_t nb00; // kernel strides
uint64_t nb01;
diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp
index 65a81b81e..1ae7f3d5d 100644
--- a/ggml/src/ggml-metal/ggml-metal-ops.cpp
+++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp
@@ -4959,6 +4959,33 @@ int ggml_metal_op_im2col(ggml_metal_op_t ctx, int idx) {
return 1;
}
+// number of reduction splits of the conv_2d matmul kernels
+// A small output with a long reduction leaves the GPU idle, so the reduction is split until there are about 1M partial sums.
+static int ggml_metal_op_conv_2d_nsplit(const ggml_tensor * op, int32_t * nk) {
+ const int64_t K = op->src[0]->ne[0]*op->src[0]->ne[1]*op->src[0]->ne[2];
+ const int64_t nsteps = (K + 31)/32;
+ const int64_t ne = ggml_nelements(op);
+
+ // at least 3 reduction steps of 32 per split, at most 32 splits
+ const int64_t nsplit = std::max<int64_t>(1, std::min<int64_t>({ (1024*1024 + ne - 1)/ne, nsteps/3, 32 }));
+ const int64_t steps = (nsteps + nsplit - 1)/nsplit;
+
+ if (nk) {
+ *nk = (int32_t) (steps*32);
+ }
+
+ return (int) ((nsteps + steps - 1)/steps);
+}
+
+size_t ggml_metal_op_conv_2d_extra_tmp(const ggml_tensor * op) {
+ assert(op->op == GGML_OP_CONV_2D);
+
+ const int nsplit = ggml_metal_op_conv_2d_nsplit(op, nullptr);
+
+ // partial sums of the reduction splits
+ return nsplit > 1 ? nsplit*ggml_nelements(op)*sizeof(float) : 0;
+}
+
int ggml_metal_op_conv_2d(ggml_metal_op_t ctx, int idx) {
ggml_tensor * op = ctx->node(idx);
@@ -5012,11 +5039,17 @@ int ggml_metal_op_conv_2d(ggml_metal_op_t ctx, int idx) {
/*.p1 =*/ p1,
/*.d0 =*/ d0,
/*.d1 =*/ d1,
+ /*.nk =*/ 0,
};
const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx->dev);
- if (props_dev->has_simdgroup_mm) {
+ // the matmul kernels walk the patches with 32-bit byte offsets inside an image
+ // larger images, possibly a strided view, take the generic kernel
+ const uint64_t max_off = (ne12 - 1)*nb12 + (ne11 - 1)*nb11 + (ne10 - 1)*nb10 + sizeof(float);
+ const bool fits_32 = max_off <= UINT32_MAX;
+
+ if (props_dev->has_simdgroup_mm && fits_32) {
auto pipeline = ggml_metal_library_get_pipeline_conv_2d_mm(lib, op);
const int nr0 = pipeline.nr0;
@@ -5026,15 +5059,48 @@ int ggml_metal_op_conv_2d(ggml_metal_op_t ctx, int idx) {
const int64_t NP = ne3*ne1*ne0;
const int64_t OC = ne03;
+ const int nsplit = ggml_metal_op_conv_2d_nsplit(op, &args.nk);
+
+ ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op);
+ ggml_metal_buffer_id bid_tmp = bid_dst;
+ bid_tmp.offs += ggml_nbytes(op);
+
ggml_metal_encoder_set_pipeline(enc, pipeline);
ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);
- ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);
+ ggml_metal_encoder_set_buffer (enc, nsplit > 1 ? bid_tmp : bid_dst, 3);
ggml_metal_encoder_set_threadgroup_memory_size(enc, pipeline.smem, 0);
- ggml_metal_encoder_dispatch_threadgroups(enc, (NP + nr1 - 1)/nr1, (OC + nr0 - 1)/nr0, 1, 32, nsg, 1);
+ if (props_dev->has_tensor) {
+ // per image: nr0 positions times nr1 channels per threadgroup
+ const int64_t OHW = ne1*ne0;
+ ggml_metal_encoder_dispatch_threadgroups(enc, (OC + nr1 - 1)/nr1, (OHW + nr0 - 1)/nr0, ne3*nsplit, 32, nsg, 1);
+ } else {
+ ggml_metal_encoder_dispatch_threadgroups(enc, (NP + nr1 - 1)/nr1, (OC + nr0 - 1)/nr0, nsplit, 32, nsg, 1);
+ }
+
+ if (nsplit > 1) {
+ GGML_ASSERT(ggml_is_contiguous(op));
+
+ // sync the 2 kernels
+ ggml_metal_op_concurrency_reset(ctx);
+
+ ggml_metal_kargs_conv_2d_reduce args0 = {
+ /*.ne =*/ (int32_t) ggml_nelements(op),
+ /*.nsplit =*/ nsplit,
+ };
+
+ auto pipeline0 = ggml_metal_library_get_pipeline_conv_2d_mm_reduce(lib, op);
+
+ ggml_metal_encoder_set_pipeline(enc, pipeline0);
+ ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0);
+ ggml_metal_encoder_set_buffer (enc, bid_tmp, 1);
+ ggml_metal_encoder_set_buffer (enc, bid_dst, 2);
+
+ ggml_metal_encoder_dispatch_threadgroups(enc, (args0.ne + 255)/256, 1, 1, 256, 1, 1);
+ }
return 1;
}
diff --git a/ggml/src/ggml-metal/ggml-metal-ops.h b/ggml/src/ggml-metal/ggml-metal-ops.h
index 47889fa84..7dae178d5 100644
--- a/ggml/src/ggml-metal/ggml-metal-ops.h
+++ b/ggml/src/ggml-metal/ggml-metal-ops.h
@@ -49,6 +49,8 @@ size_t ggml_metal_op_flash_attn_ext_extra_rows(const struct ggml_tensor * op);
size_t ggml_metal_op_argmax_extra(const struct ggml_tensor * op);
+size_t ggml_metal_op_conv_2d_extra_tmp(const struct ggml_tensor * op);
+
int ggml_metal_op_concat (ggml_metal_op_t ctx, int idx);
int ggml_metal_op_repeat (ggml_metal_op_t ctx, int idx);
int ggml_metal_op_acc (ggml_metal_op_t ctx, int idx);
diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp
index 9bed6421b..9a1bd2c77 100644
--- a/ggml/src/ggml-metal/ggml-metal.cpp
+++ b/ggml/src/ggml-metal/ggml-metal.cpp
@@ -311,6 +311,10 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_
{
res = 2*sizeof(int32_t)*ggml_nelements(tensor->src[0]);
} break;
+ case GGML_OP_CONV_2D:
+ {
+ res += ggml_metal_op_conv_2d_extra_tmp(tensor);
+ } break;
case GGML_OP_ARGMAX:
{
res += ggml_metal_op_argmax_extra(tensor);
diff --git a/ggml/src/ggml-metal/kernels/conv.metal b/ggml/src/ggml-metal/kernels/conv.metal
index fdc32756a..9f9b6edfc 100644
--- a/ggml/src/ggml-metal/kernels/conv.metal
+++ b/ggml/src/ggml-metal/kernels/conv.metal
@@ -285,7 +285,7 @@ kernel void kernel_conv_2d_mm(
ushort tiitg[[thread_index_in_threadgroup]],
ushort tiisg[[thread_index_in_simdgroup]],
ushort sgitg[[simdgroup_index_in_threadgroup]]) {
- constexpr short NSG = 4;
+ constexpr short NSG = N_SG_CONV_2D;
constexpr short NTH = 32*NSG;
constexpr short NK = 32;
constexpr short NKB = NK/8;
@@ -300,6 +300,10 @@ kernel void kernel_conv_2d_mm(
const int K = args.IC*args.KH*args.KW;
const int NP = args.N*args.OH*args.OW;
+ // this threadgroup reduces [k_begin, k_end), the splits are summed by kernel_conv_2d_mm_reduce
+ const int k_begin = tgpig.z*args.nk;
+ const int k_end = min(K, k_begin + args.nk);
+
const int r0 = tgpig.y*NR0;
const int r1 = tgpig.x*NR1;
@@ -308,8 +312,14 @@ kernel void kernel_conv_2d_mm(
const short lb_n = 8*sgitg + tiisg%8;
const short lb_k = 8*(tiisg/8);
+ // the patches are walked with 32-bit byte offsets inside the image, the host checks that an image fits
+ const uint nb10 = args.nb10;
+ const uint nb11 = args.nb11;
+ const uint nb12 = args.nb12;
+
int ix0[NPL];
int iy0[NPL];
+ uint off0[NPL];
bool ip_ok[NPL];
device const char * srcb[NPL];
@@ -324,6 +334,8 @@ kernel void kernel_conv_2d_mm(
ix0[t] = iow*args.s0 - args.p0;
iy0[t] = ioh*args.s1 - args.p1;
+ off0[t] = iy0[t]*nb11 + ix0[t]*nb10;
+
ip_ok[t] = ip < NP;
srcb[t] = src + in*args.nb13;
@@ -337,7 +349,7 @@ kernel void kernel_conv_2d_mm(
mc[i] = make_filled_simdgroup_matrix<float, 8>(0.f);
}
- for (int k0 = 0; k0 < K; k0 += NK) {
+ for (int k0 = k_begin; k0 < k_end; k0 += NK) {
threadgroup_barrier(mem_flags::mem_threadgroup);
{
@@ -349,7 +361,7 @@ kernel void kernel_conv_2d_mm(
const int oc = r0 + row;
half v = 0;
- if (kk < K && oc < args.OC) {
+ if (kk < k_end && oc < args.OC) {
v = *((device const TK *)(weights + oc*args.nb03) + kk);
}
@@ -364,7 +376,11 @@ kernel void kernel_conv_2d_mm(
int kw = kk % args.KW;
int kh = (kk / args.KW) % args.KH;
- int ic = kk / (args.KW*args.KH);
+ int dx = kw*args.d0;
+ int dy = kh*args.d1;
+
+ // byte offset of the reduction index kk, relative to the patch origin
+ uint offk = (kk / (args.KW*args.KH))*nb12 + dy*nb11 + dx*nb10;
FOR_UNROLL (short i = 0; i < 8; i++) {
const short k = lb_k + i;
@@ -372,12 +388,12 @@ kernel void kernel_conv_2d_mm(
FOR_UNROLL (short t = 0; t < NPL; t++) {
const short n = 32*t + lb_n;
- const int ix = ix0[t] + kw*args.d0;
- const int iy = iy0[t] + kh*args.d1;
+ const int ix = ix0[t] + dx;
+ const int iy = iy0[t] + dy;
half v = 0;
- if (ip_ok[t] && kk < K && ix >= 0 && ix < args.IW && iy >= 0 && iy < args.IH) {
- v = *(device const float *)(srcb[t] + ic*args.nb12 + iy*args.nb11 + ix*args.nb10);
+ if (ip_ok[t] && kk < k_end && (uint) ix < (uint) args.IW && (uint) iy < (uint) args.IH) {
+ v = *(device const float *)(srcb[t] + (off0[t] + offk));
}
const short ib = NKB*(n/8) + k/8;
@@ -386,11 +402,17 @@ kernel void kernel_conv_2d_mm(
}
kk++;
+ dx += args.d0;
+ offk += args.d0*nb10;
if (++kw == args.KW) {
- kw = 0;
+ kw = 0;
+ dx = 0;
+ dy += args.d1;
+ offk += args.d1*nb11 - args.KW*args.d0*nb10;
if (++kh == args.KH) {
- kh = 0;
- ic++;
+ kh = 0;
+ dy = 0;
+ offk += nb12 - args.KH*args.d1*nb11;
}
}
}
@@ -398,7 +420,7 @@ kernel void kernel_conv_2d_mm(
threadgroup_barrier(mem_flags::mem_threadgroup);
- const short nkb = min((K - k0 + 7)/8, (int) NKB);
+ const short nkb = min((k_end - k0 + 7)/8, (int) NKB);
for (short ik = 0; ik < nkb; ik++) {
FOR_UNROLL (short i = 0; i < 4; i++) {
@@ -436,7 +458,8 @@ kernel void kernel_conv_2d_mm(
const int oh = (p / args.OW) % args.OH;
const int n = p / (args.OW*args.OH);
- device char * dstp = dst + n*args.nb3 + oh*args.nb1 + ow*args.nb0;
+ // with several splits, dst is the partial sum buffer, one contiguous output per split
+ device char * dstp = dst + (uint64_t) tgpig.z*args.N*args.nb3 + n*args.nb3 + oh*args.nb1 + ow*args.nb0;
for (short j = 0; j < NR0*NR1/NTH; j++) {
const short row = tiitg/NR1 + (NTH/NR1)*j;
@@ -448,6 +471,158 @@ kernel void kernel_conv_2d_mm(
}
}
+#ifdef GGML_METAL_HAS_TENSOR
+// conv_2d as a matmul on the tensor cores, like kernel_mul_mm
+// The patches of NRA output positions are gathered into threadgroup memory, the kernel rows are read from device memory.
+// One threadgroup computes NRA positions of one image times NRB output channels, over the reduction range of its split.
+template <typename TK>
+kernel void kernel_conv_2d_mm_tensor(
+ constant ggml_metal_kargs_conv_2d & args,
+ device const char * weights,
+ device const char * src,
+ device char * dst,
+ threadgroup char * shmem [[threadgroup(0)]],
+ uint3 tgpig[[threadgroup_position_in_grid]],
+ uint3 tgpg[[threadgroups_per_grid]],
+ ushort tiitg[[thread_index_in_threadgroup]]) {
+ constexpr int NRB = SZ_SIMDGROUP * N_MM_BLOCK_X * N_MM_SIMD_GROUP_X; // output channels
+ constexpr int NRA = SZ_SIMDGROUP * N_MM_BLOCK_Y * N_MM_SIMD_GROUP_Y; // output positions
+ constexpr int NK = N_MM_NK_TOTAL;
+ constexpr int NTH = N_SIMDWIDTH * N_MM_SIMD_GROUP_X * N_MM_SIMD_GROUP_Y;
+
+ // each thread gathers NK/(NTH/NRA) consecutive reduction indices of one position
+ constexpr int KPT = NK/(NTH/NRA);
+
+ const int K = args.IC*args.KH*args.KW;
+ const int OHW = args.OH*args.OW;
+
+ const int nsplit = tgpg.z/args.N;
+ const int img = tgpig.z/nsplit;
+ const int split = tgpig.z%nsplit;
+
+ const int k_begin = split*args.nk;
+ const int k_end = min(K, k_begin + args.nk);
+
+ const int ra = tgpig.y*NRA;
+ const int rb = tgpig.x*NRB;
+
+ threadgroup half * sa = (threadgroup half *) shmem;
+
+ // the position this thread gathers
+ const int row = tiitg%NRA;
+ const int kc = (tiitg/NRA)*KPT;
+ const int p = ra + row;
+ const bool pok = p < OHW;
+ const int ix0 = (p%args.OW)*args.s0 - args.p0;
+ const int iy0 = (p/args.OW)*args.s1 - args.p1;
+
+ device const char * srcb = src + img*args.nb13;
+
+ // the patch is walked with 32-bit byte offsets inside the image, the host checks that an image fits
+ const uint nb10 = args.nb10;
+ const uint nb11 = args.nb11;
+ const uint nb12 = args.nb12;
+
+ const uint off0 = iy0*nb11 + ix0*nb10;
+
+ device TK * ptrB = (device TK *) weights;
+ const int strideB = args.nb03/sizeof(TK);
+
+ mpp::tensor_ops::matmul2d<
+ mpp::tensor_ops::matmul2d_descriptor(
+ NRB, NRA, static_cast<int>(dynamic_extent), false, true, true,
+ mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate),
+ execution_simdgroups<N_MM_SIMD_GROUP_X * N_MM_SIMD_GROUP_Y>> mm;
+
+ auto tB0 = tensor(ptrB, dextents<int32_t, 2>(K, args.OC), array<int, 2>({1, strideB}));
+ auto tA0 = tensor(sa, dextents<int32_t, 2>(NK, NRA));
+ auto cT = mm.get_destination_cooperative_tensor<decltype(tB0), decltype(tA0), float>();
+
+ for (int k0 = k_begin; k0 < k_end; k0 += NK) {
+ // the reduction indices past k_end are beyond the extent given to the matmul, they are not gathered
+ if (k0 + kc < k_end) {
+ // gather KPT patch values of this position, walking the reduction index incrementally
+ int kk = k0 + kc;
+ int kw = kk % args.KW;
+ int kh = (kk / args.KW) % args.KH;
+ int dx = kw*args.d0;
+ int dy = kh*args.d1;
+
+ // byte offset of the reduction index kk, relative to the patch origin
+ uint offk = (kk / (args.KW*args.KH))*nb12 + dy*nb11 + dx*nb10;
+
+ FOR_UNROLL (short i = 0; i < KPT; i++) {
+ const int ix = ix0 + dx;
+ const int iy = iy0 + dy;
+
+ half v = 0;
+ if (pok && kk < k_end && (uint) ix < (uint) args.IW && (uint) iy < (uint) args.IH) {
+ v = *(device const float *)(srcb + (off0 + offk));
+ }
+ sa[row*NK + kc + i] = v;
+
+ kk++;
+ dx += args.d0;
+ offk += args.d0*nb10;
+ if (++kw == args.KW) {
+ kw = 0;
+ dx = 0;
+ dy += args.d1;
+ offk += args.d1*nb11 - args.KW*args.d0*nb10;
+ if (++kh == args.KH) {
+ kh = 0;
+ dy = 0;
+ offk += nb12 - args.KH*args.d1*nb11;
+ }
+ }
+ }
+ }
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ // clamp the reduction extent so neither operand is read past the end of the split
+ const int kExt = min(NK, k_end - k0);
+
+ auto tAv = tensor(sa, dextents<int32_t, 2>(kExt, NRA), array<int, 2>({1, NK}));
+ auto tBv = tensor(ptrB + k0 + rb*strideB, dextents<int32_t, 2>(kExt, args.OC - rb), array<int, 2>({1, strideB}));
+
+ mm.run(tBv, tAv, cT);
+
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+
+ // dst holds [OHW, OC] per image, with several splits it is the partial sum buffer, one contiguous output per split
+ const int tileM = min(NRA, OHW - ra);
+ const int tileN = min(NRB, args.OC - rb);
+
+ device float * dstTile = (device float *) (dst + (uint64_t) split*args.N*args.nb3 + img*args.nb3) + (uint64_t) rb*OHW + ra;
+
+ auto tD = tensor(dstTile, dextents<int32_t, 2>(tileM, tileN), array<int, 2>({1, OHW}));
+ cT.store(tD);
+}
+
+typedef decltype(kernel_conv_2d_mm_tensor<float>) kernel_conv_2d_mm_tensor_t;
+
+template [[host_name("kernel_conv_2d_mm_tensor_f32_f32")]] kernel kernel_conv_2d_mm_tensor_t kernel_conv_2d_mm_tensor<float>;
+template [[host_name("kernel_conv_2d_mm_tensor_f16_f32")]] kernel kernel_conv_2d_mm_tensor_t kernel_conv_2d_mm_tensor<half>;
+#endif
+
+kernel void kernel_conv_2d_mm_reduce(
+ constant ggml_metal_kargs_conv_2d_reduce & args,
+ device const float * tmp,
+ device float * dst,
+ uint tpig[[thread_position_in_grid]]) {
+ if ((int) tpig >= args.ne) {
+ return;
+ }
+
+ float sum = 0.0f;
+ for (int s = 0; s < args.nsplit; ++s) {
+ sum += tmp[(uint64_t) s*args.ne + tpig];
+ }
+ dst[tpig] = sum;
+}
+
typedef decltype(kernel_conv_2d_mm<float, 64, 32>) kernel_conv_2d_mm_t;
template [[host_name("kernel_conv_2d_mm_f32_f32_64x32")]] kernel kernel_conv_2d_mm_t kernel_conv_2d_mm<float, 64, 32>;