Commit 0878f7c30 for llama.cpp
commit 0878f7c304cc13915c831ae17694534c94940176
Author: Andrea Rossetti <rossettiandrea@proton.me>
Date: Sun Oct 18 18:57:36 2026 +0200
ggml: support non-contiguous SUM (CPU, CUDA, Vulkan, WebGPU) (#28851)
* ggml: support non-contiguous SUM (CPU, CUDA)
* tests: add non-contiguous SUM cases
* tests: SUM over non-contiguous views
* vulkan: support non-contiguous SUM
* webgpu: support non-contiguous SUM
diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
index 7a73e138e..7dff1d11b 100644
--- a/ggml/src/ggml-cpu/ops.cpp
+++ b/ggml/src/ggml-cpu/ops.cpp
@@ -1292,24 +1292,37 @@ static void ggml_compute_forward_sum_f32(
}
assert(ggml_is_scalar(dst));
- assert(src0->nb[0] == sizeof(float));
GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne)
GGML_TENSOR_LOCALS(size_t, nb0, src0, nb)
- ggml_float sum = 0;
- ggml_float row_sum = 0;
+ ggml_float sum = 0;
- for (int64_t i03 = 0; i03 < ne03; i03++) {
- for (int64_t i02 = 0; i02 < ne02; i02++) {
- for (int64_t i01 = 0; i01 < ne01; i01++) {
- ggml_vec_sum_f32_ggf(ne00,
- &row_sum,
- (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03));
- sum += row_sum;
+ if (nb00 == sizeof(float)) {
+ ggml_float row_sum = 0;
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ for (int64_t i01 = 0; i01 < ne01; i01++) {
+ ggml_vec_sum_f32_ggf(ne00,
+ &row_sum,
+ (float *) ((char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03));
+ sum += row_sum;
+ }
+ }
+ }
+ } else {
+ for (int64_t i03 = 0; i03 < ne03; i03++) {
+ for (int64_t i02 = 0; i02 < ne02; i02++) {
+ for (int64_t i01 = 0; i01 < ne01; i01++) {
+ const char * row = (const char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03;
+ for (int64_t i00 = 0; i00 < ne00; i00++) {
+ sum += (ggml_float) *(const float *) (row + i00*nb00);
+ }
+ }
}
}
}
+
((float *) dst->data)[0] = sum;
}
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index cb85321f5..dc30492fa 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -5897,7 +5897,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
//return ggml_is_contiguous_rows(op->src[0]) && ggml_is_contiguous_rows(op->src[1]);
return ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]);
case GGML_OP_SUM:
- return ggml_is_contiguous_rows(op->src[0]);
+ return ggml_is_contiguously_allocated(op->src[0]);
case GGML_OP_TOP_K:
return op->src[0]->ne[0] <= INT_MAX;
case GGML_OP_ARGSORT:
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index fa038bb5b..e603cbcb1 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -15872,6 +15872,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32
&& ggml_is_contiguous(op->src[1]) && op->src[1]->type == GGML_TYPE_F32;
case GGML_OP_SUM:
+ return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguously_allocated(op->src[0]);
case GGML_OP_SUM_ROWS:
case GGML_OP_MEAN:
return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[0]);
diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cpp
index 15e6c784b..77fbb4f47 100644
--- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp
+++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp
@@ -4781,6 +4781,8 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const
supports_op = op->type == GGML_TYPE_F32 && src0->type == op->type;
break;
case GGML_OP_SUM:
+ supports_op = op->type == GGML_TYPE_F32 && src0->type == op->type && ggml_is_contiguously_allocated(src0);
+ break;
case GGML_OP_SUM_ROWS:
supports_op = op->type == GGML_TYPE_F32 && src0->type == op->type && ggml_is_contiguous_rows(src0);
break;
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 011f93865..8fc90e037 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -7804,24 +7804,43 @@ struct test_sum : public test_case {
const std::array<int64_t, 4> ne;
const std::array<int64_t, 4> permute;
bool _use_permute;
+ const bool slice;
std::string vars() override {
std::string v = VARS_TO_STR2(type, ne);
if (_use_permute) v += "," + VAR_TO_STR(permute);
+ if (slice) v += "," + VAR_TO_STR(slice);
return v;
}
test_sum(ggml_type type = GGML_TYPE_F32,
std::array<int64_t, 4> ne = {10, 5, 4, 3},
- std::array<int64_t, 4> permute = {0, 0, 0, 0})
+ std::array<int64_t, 4> permute = {0, 0, 0, 0},
+ bool slice = false)
: type(type), ne(ne), permute(permute),
- _use_permute(permute[0] + permute[1] + permute[2] + permute[3] > 0) {}
+ _use_permute(permute[0] + permute[1] + permute[2] + permute[3] > 0),
+ slice(slice) {}
ggml_tensor * build_graph(ggml_context * ctx) override {
ggml_tensor * a = ggml_new_tensor(ctx, type, 4, ne.data());
- ggml_set_param(a);
+
+ // the backward pass of a sliced view scatters the gradient via GGML_OP_ACC,
+ // whose CPU implementation requires contiguous rows; a row-breaking permute
+ // (permute[0] != 0) on top of a slice violates that, so skip grad there
+ const bool grad_supported = !(slice && _use_permute && permute[0] != 0);
+ if (grad_supported) {
+ ggml_set_param(a);
+ }
+
ggml_set_name(a, "a");
+ if (slice) {
+ a = ggml_view_4d(ctx, a,
+ ne[0], ne[1], ne[2] / 2, ne[3] - 1,
+ a->nb[1], a->nb[2] * 2, a->nb[3], /*offset=*/a->nb[3]);
+ ggml_set_name(a, "a_sliced");
+ }
+
if (_use_permute) {
a = ggml_permute(ctx, a, permute[0], permute[1], permute[2], permute[3]);
ggml_set_name(a, "a_permuted");
@@ -11540,6 +11559,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 1024, 1, 1 }));
test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 256, 1, 1 }));
test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 33, 256, 1, 1 }, { 1, 0, 2, 3 })); // sum dst not-contiguous
+ test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 10, 5, 4, 3 }, { 1, 0, 2, 3 })); // non-contiguous rows, 4D
+ test_cases.emplace_back(new test_sum(GGML_TYPE_F32, { 11, 5, 6, 3 }, { 2, 0, 1, 3 })); // rotated dims, nb[0] != sizeof(float)
+ test_cases.emplace_back(new test_sum(GGML_TYPE_F32, {11, 5, 6, 3}, {0, 0, 0, 0}, true)); // sliced view (gaps)
+ test_cases.emplace_back(new test_sum(GGML_TYPE_F32, {11, 5, 6, 3}, {1, 0, 2, 3}, true)); // sliced + non-contiguous rows
test_cases.emplace_back(new test_sum_rows());
test_cases.emplace_back(new test_sum_rows(GGML_TYPE_F32, { 11, 5, 6, 3 }, true, false));
test_cases.emplace_back(new test_sum_rows(GGML_TYPE_F32, { 11, 5, 6, 3 }, false, true));