Commit 12ab59798 for llama.cpp
commit 12ab5979869aecdf3593de260a99ca91263bf5a1
Author: Georgi Gerganov <ggerganov@gmail.com>
Date: Sun Nov 8 11:44:00 2026 +0200
ggml : switch to F32 src1 in the CPU backend (#31854)
* ggml : switch to F32 src1 in the CPU backend
Change the default computation for GGML_TYPE_F16 to use F32 for src1
data in matrix multiplications: vec_dot_type(F16) is now F32 and the dot
product goes through the new ggml_vec_dot_f16_f32 (x = F16 src0, y = F32
src1), so src1 is no longer quantized to F16 and worst-case inputs that
only fit in the F32 FP range are handled correctly.
- conv_2d/conv_3d: store im2col patches (the inner mul_mat src1) as F32
and pass separate src0/src1 types to the inner mul_mat; the previous
F16 patches made the inner mul_mat widen src1 into the conv work
buffer, corrupting the patches and crashing on large inputs
- conv_transpose_1d/2d (F16 kernel): keep src1 in F32 in the work buffer
and use ggml_vec_dot_f16_f32; update work buffer planning accordingly
- supports_op: restore MUL_MAT support for F16 src1, which is now
widened to F32
- ggml_vec_dot_f16_f32: add GGML_SIMD support by converting chunks of x
to F32 and reusing the F32 SIMD dot
- mul_mat/mul_mat_id: add TODOs for a more efficient F16 x F16 path that
can be used if ggml_prec_get_src(dst, 1) allows converting src1 to F16
MUL_MAT f16 x f32 perf on M5 Ultra (GFLOPS):
- m=2048, n=512, k=16 : 120.9 -> 101.7
- m=2048, n=512, k=64 : 150.5 -> 344.2
- m=4096, k=14336 GEMV: 63 -> 343-415
- m=4096, n=512, k=14336: 65.6 -> 770.2
Assisted-by: dsh:llama.cpp/ggml-org/GLM-5.3-Flash-GGUF (Mac Studio, M5 Ultra, 256GB)
* cont : add some F16 x F32 vectorized paths
* cont : enable the fused F16 x F32 dot on AVX without FMA
The 256-bit fused path of ggml_vec_dot_f16_f32 required AVX2 and FMA,
so CPUs with AVX and F16C only fell back to the chunked conversion and
F16 decode was 4x slower than before. Enable it on __AVX__ && __F16C__
and accumulate with GGML_F32x8_FMA, which does mul + add without FMA.
---------
Co-authored-by: Pascal <admin@serveurperso.com>
diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
index c2826fccc..40e93ccab 100644
--- a/ggml/src/ggml-cpu/ggml-cpu.c
+++ b/ggml/src/ggml-cpu/ggml-cpu.c
@@ -224,8 +224,8 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = {
},
[GGML_TYPE_F16] = {
.from_float = (ggml_from_float_t) ggml_cpu_fp32_to_fp16,
- .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_f16,
- .vec_dot_type = GGML_TYPE_F16,
+ .vec_dot = (ggml_vec_dot_t) ggml_vec_dot_f16_f32,
+ .vec_dot_type = GGML_TYPE_F32,
.nrows = 1,
},
[GGML_TYPE_Q1_0] = {
@@ -1284,9 +1284,12 @@ void ggml_compute_forward_mul_mat(
const int ith = params->ith;
const int nth = params->nth;
- enum ggml_type const vec_dot_type = type_traits_cpu[src0->type].vec_dot_type;
- ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float;
- int64_t const vec_dot_num_rows = type_traits_cpu[src0->type].nrows;
+ enum ggml_type const vec_dot_type = type_traits_cpu[src0->type].vec_dot_type;
+ ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float;
+ int64_t const vec_dot_num_rows = type_traits_cpu[src0->type].nrows;
+
+ // TODO: a more efficient F16 x F16 path can be used here if ggml_prec_get_src(dst, 1) allows converting src1 to F16
+ // the code needs to be refactored to enable this (src1 is currently converted based on vec_dot_type alone)
GGML_ASSERT(ne0 == ne01);
GGML_ASSERT(ne1 == ne11);
@@ -1509,8 +1512,8 @@ static void ggml_compute_forward_mul_mat_id_one_chunk(
const enum ggml_type type = src0->type;
- ggml_vec_dot_t const vec_dot = type_traits_cpu[type].vec_dot;
- enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type;
+ ggml_vec_dot_t const vec_dot = type_traits_cpu[type].vec_dot;
+ enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type;
const int64_t blck_0 = 16;
const int64_t blck_1 = 16;
@@ -1577,8 +1580,11 @@ static void ggml_compute_forward_mul_mat_id(
const bool src1_cont = ggml_is_contiguous(src1);
- enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type;
- ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float;
+ enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type;
+ ggml_from_float_t const from_float = type_traits_cpu[vec_dot_type].from_float;
+
+ // TODO: a more efficient F16 x F16 path can be used here if ggml_prec_get_src(dst, 1) allows converting src1 to F16
+ // the code needs to be refactored to enable this (src1 is currently converted based on vec_dot_type alone)
// we don't support permuted src0 or src1
GGML_ASSERT(nb00 == ggml_type_size(type));
@@ -2965,8 +2971,8 @@ struct ggml_cplan ggml_graph_plan(
if ((node->src[0]->type == GGML_TYPE_F16 ||
node->src[0]->type == GGML_TYPE_BF16) &&
node->src[1]->type == GGML_TYPE_F32) {
+ cur += sizeof(float)*ne10*ne11;
cur += sizeof(ggml_fp16_t)*ne00*ne01*ne02;
- cur += sizeof(ggml_fp16_t)*ne10*ne11;
} else if (node->src[0]->type == GGML_TYPE_F32 &&
node->src[1]->type == GGML_TYPE_F32) {
cur += sizeof(float)*ne00*ne01*ne02;
@@ -2995,8 +3001,8 @@ struct ggml_cplan ggml_graph_plan(
GGML_ASSERT(node->src[0]->type == GGML_TYPE_F16 || node->src[0]->type == GGML_TYPE_F32);
GGML_ASSERT(node->src[1]->type == GGML_TYPE_F32);
+ cur += sizeof(float) * ne10 * ne11 * ne12 * ne13;
cur += ggml_type_size(node->src[0]->type) * ne00 * ne01 * ne02 * ne03;
- cur += ggml_type_size(node->src[0]->type) * ne10 * ne11 * ne12 * ne13;
} break;
case GGML_OP_TOP_K:
diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp
index 81ff5d79f..297834001 100644
--- a/ggml/src/ggml-cpu/ggml-cpu.cpp
+++ b/ggml/src/ggml-cpu/ggml-cpu.cpp
@@ -455,10 +455,11 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st
src0->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32) {
return src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16;
}
- // BF16 in src1 is widened into the F32 work buffer
+ // F16 or BF16 in src1 is widened into the F32 work buffer
return src1->type == GGML_TYPE_F32 ||
src1->type == ggml_get_type_traits_cpu(src0->type)->vec_dot_type ||
- (src1->type == GGML_TYPE_BF16 && ggml_get_type_traits_cpu(src0->type)->vec_dot_type == GGML_TYPE_F32);
+ ((src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_BF16) &&
+ ggml_get_type_traits_cpu(src0->type)->vec_dot_type == GGML_TYPE_F32);
case GGML_OP_SOFT_MAX_BACK: {
if (op->src[0]->type != GGML_TYPE_F32 || op->src[1]->type != GGML_TYPE_F32) {
return false;
diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp
index b406f375a..6d39cd6e7 100644
--- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp
+++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp
@@ -4054,6 +4054,13 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64
(float *)C, ldc};
return tb.matmul(m, n);
}
+ if (Btype == GGML_TYPE_F32) {
+ tinyBLAS<16, __m512, __m512, ggml_fp16_t, float, float> tb{ params, k,
+ (const ggml_fp16_t *)A, lda,
+ (const float *)B, ldb,
+ (float *)C, ldc};
+ return tb.matmul(m, n);
+ }
#elif (defined(__AVX__) || defined(__AVX2__)) && defined(__F16C__)
if (Btype == GGML_TYPE_F16) {
tinyBLAS<8, __m256, __m256, ggml_fp16_t, ggml_fp16_t, float> tb{ params, k,
@@ -4062,6 +4069,13 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64
(float *)C, ldc};
return tb.matmul(m, n);
}
+ if (Btype == GGML_TYPE_F32) {
+ tinyBLAS<8, __m256, __m256, ggml_fp16_t, float, float> tb{ params, k,
+ (const ggml_fp16_t *)A, lda,
+ (const float *)B, ldb,
+ (float *)C, ldc};
+ return tb.matmul(m, n);
+ }
#elif defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && !defined(_MSC_VER)
if (n < 8)
return false;
diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
index 750367c76..1884600e6 100644
--- a/ggml/src/ggml-cpu/ops.cpp
+++ b/ggml/src/ggml-cpu/ops.cpp
@@ -6514,17 +6514,28 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32(
const int ith = params->ith;
const int nth = params->nth;
- const int nk = ne00*ne01*ne02;
-
GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));
GGML_ASSERT(nb10 == sizeof(float));
if (ith == 0) {
memset(params->wdata, 0, params->wsize);
+ // permute source data (src1) from (L x Cin) to (Cin x L)
+ {
+ float * const wdata = (float *) params->wdata + 0;
+ float * dst_data = wdata;
+
+ for (int64_t i11 = 0; i11 < ne11; i11++) {
+ const float * const src = (float *)((char *) src1->data + i11*nb11);
+ for (int64_t i10 = 0; i10 < ne10; i10++) {
+ dst_data[i10*ne11 + i11] = src[i10];
+ }
+ }
+ }
+
// permute kernel data (src0) from (K x Cout x Cin) to (Cin x K x Cout)
{
- ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0;
+ ggml_fp16_t * const wdata = (ggml_fp16_t *)((char *) params->wdata + ne10*ne11*sizeof(float));
for (int64_t i02 = 0; i02 < ne02; i02++) {
for (int64_t i01 = 0; i01 < ne01; i01++) {
@@ -6537,19 +6548,6 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32(
}
}
- // permute source data (src1) from (L x Cin) to (Cin x L)
- {
- ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + nk;
- ggml_fp16_t * dst_data = wdata;
-
- for (int64_t i11 = 0; i11 < ne11; i11++) {
- const float * const src = (float *)((char *) src1->data + i11*nb11);
- for (int64_t i10 = 0; i10 < ne10; i10++) {
- dst_data[i10*ne11 + i11] = GGML_CPU_FP32_TO_FP16(src[i10]);
- }
- }
- }
-
// need to zero dst since we are accumulating into it
memset(dst->data, 0, ggml_nbytes(dst));
}
@@ -6567,19 +6565,19 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32(
const int ir0 = dr*ith;
const int ir1 = MIN(ir0 + dr, nr);
- ggml_fp16_t * const wdata = (ggml_fp16_t *) params->wdata + 0;
- ggml_fp16_t * const wdata_src = wdata + nk;
+ float * const wdata_src = (float *) params->wdata + 0;
+ ggml_fp16_t * const wdata_kernel = (ggml_fp16_t *)((char *) params->wdata + ne10*ne11*sizeof(float));
for (int i1 = ir0; i1 < ir1; i1++) {
float * dst_data = (float *)((char *) dst->data + i1*nb1);
- ggml_fp16_t * wdata_kernel = wdata + i1*ne02*ne00;
+ ggml_fp16_t * wdata_kernel_row = wdata_kernel + i1*ne02*ne00;
for (int i10 = 0; i10 < ne10; i10++) {
const int i1n = i10*ne11;
for (int i00 = 0; i00 < ne00; i00++) {
float v = 0;
- ggml_vec_dot_f16(ne02, &v, 0,
- (ggml_fp16_t *) wdata_src + i1n, 0,
- (ggml_fp16_t *) wdata_kernel + i00*ne02, 0, 1);
+ ggml_vec_dot_f16_f32(ne02, &v, 0,
+ wdata_kernel_row + i00*ne02, 0,
+ wdata_src + i1n, 0, 1);
dst_data[i10*s0 + i00] += v;
}
}
@@ -7172,29 +7170,30 @@ void ggml_compute_forward_im2col_3d(
}
}
-static void ggml_call_mul_mat(ggml_type type, const ggml_compute_params * params, int64_t m, int64_t n, int64_t k,
+static void ggml_call_mul_mat(ggml_type src0_type, ggml_type src1_type, const ggml_compute_params * params, int64_t m, int64_t n, int64_t k,
void * a, void * b, float * c) {
- const ggml_type_traits * traits = ggml_get_type_traits(type);
+ const ggml_type_traits * traits0 = ggml_get_type_traits(src0_type);
+ const ggml_type_traits * traits1 = ggml_get_type_traits(src1_type);
struct ggml_tensor src1 = {};
- src1.type = type;
+ src1.type = src1_type;
src1.ne[0] = k;
src1.ne[1] = m;
src1.ne[2] = 1;
src1.ne[3] = 1;
- src1.nb[0] = traits->type_size;
- src1.nb[1] = k * traits->type_size;
+ src1.nb[0] = traits1->type_size;
+ src1.nb[1] = k * traits1->type_size;
src1.nb[2] = src1.nb[1];
src1.nb[3] = src1.nb[2];
src1.data = a;
struct ggml_tensor src0 = {};
- src0.type = type;
+ src0.type = src0_type;
src0.ne[0] = k;
src0.ne[1] = n;
src0.ne[2] = 1;
src0.ne[3] = 1;
- src0.nb[0] = traits->type_size;
- src0.nb[1] = k * traits->type_size;
+ src0.nb[0] = traits0->type_size;
+ src0.nb[1] = k * traits0->type_size;
src0.nb[2] = src0.nb[1];
src0.nb[3] = src0.nb[2];
src0.data = b;
@@ -7304,8 +7303,6 @@ static void ggml_compute_forward_conv_2d_impl(const ggml_compute_params * params
GGML_ASSERT(kernel_type == GGML_TYPE_F16 || kernel_type == GGML_TYPE_F32);
GGML_ASSERT(kernel->type == kernel_type);
- const ggml_type_traits * traits = ggml_get_type_traits(kernel_type);
-
const int32_t stride_x = dst->op_params[0];
const int32_t stride_y = dst->op_params[1];
const int32_t pad_x = dst->op_params[2];
@@ -7331,7 +7328,7 @@ static void ggml_compute_forward_conv_2d_impl(const ggml_compute_params * params
const int64_t knl_n = knl_w * knl_h * c_in;
const int64_t patch_total = dst->ne[3] * dst_w * dst_h;
- const int64_t space_per_patch = knl_n * traits->type_size + c_out * sizeof(float);
+ const int64_t space_per_patch = knl_n * sizeof(float) + c_out * sizeof(float);
const int64_t batch_size = params->wsize / space_per_patch;
const int64_t patches_per_batch = batch_size > 8 ? (batch_size / 8) * 8 : batch_size;
const int64_t batch_n = (patch_total + patches_per_batch - 1) / patches_per_batch;
@@ -7358,7 +7355,7 @@ static void ggml_compute_forward_conv_2d_impl(const ggml_compute_params * params
const int64_t src_y = p % dst_w;
const float * src_base = (const float *)((const char *)src_data + batch_n * src->nb[3]);
- char * dst_row = (char *) tmp + (p % patches_per_batch) * knl_n * traits->type_size;
+ float * dst_row = (float *) tmp + (p % patches_per_batch) * knl_n;
for (int64_t ic = 0; ic < c_in; ++ic) {
for (int64_t ky = 0; ky < knl_h; ++ky) {
@@ -7376,12 +7373,7 @@ static void ggml_compute_forward_conv_2d_impl(const ggml_compute_params * params
src_val = *src_ptr;
}
- char * element_ptr = dst_row + dst_idx * traits->type_size;
- if (kernel_type == GGML_TYPE_F32) {
- *(float *) element_ptr = src_val;
- } else if (kernel_type == GGML_TYPE_F16) {
- *(ggml_fp16_t *) element_ptr = GGML_CPU_FP32_TO_FP16(src_val);
- }
+ dst_row[dst_idx] = src_val;
}
}
}
@@ -7389,12 +7381,12 @@ static void ggml_compute_forward_conv_2d_impl(const ggml_compute_params * params
ggml_barrier(params->threadpool);
- float * gemm_output = (float *) ((char *) tmp + patches_per_batch * knl_n * traits->type_size);
+ float * gemm_output = (float *) ((char *) tmp + patches_per_batch * knl_n * sizeof(float));
GGML_ASSERT(gemm_output + patch_n * c_out <= (float*)tmp + params->wsize);
// GEMM: patches[patch_n, knl_n] × kernel[knl_n, c_out] = output[patch_n, c_out]
- ggml_call_mul_mat(kernel_type, params, patch_n, c_out, knl_n, tmp, knl_data, gemm_output);
+ ggml_call_mul_mat(kernel_type, GGML_TYPE_F32, params, patch_n, c_out, knl_n, tmp, knl_data, gemm_output);
ggml_barrier(params->threadpool);
@@ -7441,8 +7433,6 @@ static void ggml_compute_forward_conv_3d_impl(const ggml_compute_params * params
GGML_ASSERT(kernel_type == GGML_TYPE_F16 || kernel_type == GGML_TYPE_F32);
GGML_ASSERT(kernel->type == kernel_type);
- const ggml_type_traits * traits = ggml_get_type_traits(kernel_type);
-
const int32_t s0 = dst->op_params[0];
const int32_t s1 = dst->op_params[1];
const int32_t s2 = dst->op_params[2];
@@ -7474,7 +7464,7 @@ static void ggml_compute_forward_conv_3d_impl(const ggml_compute_params * params
const int64_t knl_n_total = knl_n_per_channel * c;
const int64_t patch_total = n * dst_w * dst_h * dst_d;
- const int64_t space_per_patch = knl_n_total * traits->type_size + oc * sizeof(float);
+ const int64_t space_per_patch = knl_n_total * sizeof(float) + oc * sizeof(float);
const int64_t batch_size = params->wsize / space_per_patch;
const int64_t patches_per_batch = batch_size > 8 ? (batch_size / 8) * 8 : batch_size;
const int64_t batch_n = (patch_total + patches_per_batch - 1) / patches_per_batch;
@@ -7500,7 +7490,7 @@ static void ggml_compute_forward_conv_3d_impl(const ggml_compute_params * params
const int64_t dst_y = p_in_depth / dst_w;
const int64_t dst_x = p_in_depth % dst_w;
- char * dst_row = (char *) tmp + (p % patches_per_batch) * knl_n_total * traits->type_size;
+ float * dst_row = (float *) tmp + (p % patches_per_batch) * knl_n_total;
for (int64_t ic = 0; ic < c; ++ic) {
for (int64_t kz = 0; kz < knl_d; ++kz) {
@@ -7521,12 +7511,7 @@ static void ggml_compute_forward_conv_3d_impl(const ggml_compute_params * params
src_val = *src_ptr;
}
- char * element_ptr = dst_row + dst_idx * traits->type_size;
- if (kernel_type == GGML_TYPE_F32) {
- *(float *)element_ptr = src_val;
- } else if (kernel_type == GGML_TYPE_F16) {
- *(ggml_fp16_t *)element_ptr = GGML_CPU_FP32_TO_FP16(src_val);
- }
+ dst_row[dst_idx] = src_val;
}
}
}
@@ -7535,8 +7520,8 @@ static void ggml_compute_forward_conv_3d_impl(const ggml_compute_params * params
ggml_barrier(params->threadpool);
- float * gemm_output = (float *) ((char *) tmp + patches_per_batch * knl_n_total * traits->type_size);
- ggml_call_mul_mat(kernel_type, params, patch_n_in_batch, oc, knl_n_total, tmp, knl_data, gemm_output);
+ float * gemm_output = (float *) ((char *) tmp + patches_per_batch * knl_n_total * sizeof(float));
+ ggml_call_mul_mat(kernel_type, GGML_TYPE_F32, params, patch_n_in_batch, oc, knl_n_total, tmp, knl_data, gemm_output);
ggml_barrier(params->threadpool);
@@ -7588,7 +7573,7 @@ static void ggml_compute_forward_conv_transpose_2d_impl(
const int ith = params->ith;
const int nth = params->nth;
- const int nk = ne00*ne01*ne02*ne03;
+ const int nsrc = ne10*ne11*ne12*ne13;
GGML_ASSERT(nb00 == ggml_type_size(src0->type));
GGML_ASSERT(nb10 == sizeof(float));
@@ -7596,9 +7581,26 @@ static void ggml_compute_forward_conv_transpose_2d_impl(
if (ith == 0) {
memset(params->wdata, 0, params->wsize);
+ // permute source data (src1) from (Sw x Sh x Cin) to (Cin x Sw x Sh), for all batches
+ {
+ float * const wdata = (float *) params->wdata + 0;
+ for (int i13 = 0; i13 < ne13; i13++) {
+ float * const wdata_b = wdata + i13*ne10*ne11*ne12;
+ for (int i12 = 0; i12 < ne12; i12++) {
+ for (int i11 = 0; i11 < ne11; i11++) {
+ const float * const src = (float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11);
+ float * dst_data = wdata_b + i11*ne10*ne12;
+ for (int i10 = 0; i10 < ne10; i10++) {
+ dst_data[i10*ne12 + i12] = src[i10];
+ }
+ }
+ }
+ }
+ }
+
// permute kernel data (src0) from (Kw x Kh x Cout x Cin) to (Cin x Kw x Kh x Cout)
{
- kernel_t * const wdata = (kernel_t *) params->wdata + 0;
+ kernel_t * const wdata = (kernel_t *)((char *) params->wdata + nsrc*sizeof(float));
for (int64_t i03 = 0; i03 < ne03; i03++) {
for (int64_t i02 = 0; i02 < ne02; i02++) {
@@ -7613,27 +7615,6 @@ static void ggml_compute_forward_conv_transpose_2d_impl(
}
}
- // permute source data (src1) from (Sw x Sh x Cin) to (Cin x Sw x Sh), for all batches
- {
- kernel_t * const wdata = (kernel_t *) params->wdata + nk;
- for (int i13 = 0; i13 < ne13; i13++) {
- kernel_t * const wdata_b = wdata + i13*ne10*ne11*ne12;
- for (int i12 = 0; i12 < ne12; i12++) {
- for (int i11 = 0; i11 < ne11; i11++) {
- const float * const src = (float *)((char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11);
- kernel_t * dst_data = wdata_b + i11*ne10*ne12;
- for (int i10 = 0; i10 < ne10; i10++) {
- if constexpr (std::is_same_v<kernel_t, ggml_fp16_t>) {
- dst_data[i10*ne12 + i12] = GGML_CPU_FP32_TO_FP16(src[i10]);
- } else {
- dst_data[i10*ne12 + i12] = src[i10];
- }
- }
- }
- }
- }
- }
-
memset(dst->data, 0, ggml_nbytes(dst));
}
ggml_barrier(params->threadpool);
@@ -7650,14 +7631,14 @@ static void ggml_compute_forward_conv_transpose_2d_impl(
const int ip0 = dp*ith;
const int ip1 = MIN(ip0 + dp, np);
- kernel_t * const wdata = (kernel_t *) params->wdata + 0;
- kernel_t * const wdata_src = wdata + nk;
+ float * const wdata_src = (float *) params->wdata + 0;
+ kernel_t * const wdata_kernel = (kernel_t *)((char *) params->wdata + nsrc*sizeof(float));
for (int i2 = ip0; i2 < ip1; i2++) { // Cout
- kernel_t * wdata_kernel = wdata + i2*ne01*ne00*ne03;
+ kernel_t * wdata_kernel_c = wdata_kernel + i2*ne01*ne00*ne03;
for (int i3 = 0; i3 < ne3; i3++) { // batch
float * dst_data = (float *)((char *) dst->data + i3*nb3 + i2*nb2);
- kernel_t * wdata_src_b = wdata_src + i3*ne10*ne11*ne12;
+ float * wdata_src_b = wdata_src + i3*ne10*ne11*ne12;
for (int i11 = 0; i11 < ne11; i11++) {
for (int i10 = 0; i10 < ne10; i10++) {
const int i1n = i11*ne10*ne12 + i10*ne12;
@@ -7665,13 +7646,13 @@ static void ggml_compute_forward_conv_transpose_2d_impl(
for (int i00 = 0; i00 < ne00; i00++) {
float v = 0;
if constexpr (std::is_same_v<kernel_t, ggml_fp16_t>) {
- ggml_vec_dot_f16(ne03, &v, 0,
- wdata_src_b + i1n, 0,
- wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1);
+ ggml_vec_dot_f16_f32(ne03, &v, 0,
+ wdata_kernel_c + i01*ne00*ne03 + i00*ne03, 0,
+ wdata_src_b + i1n, 0, 1);
} else {
ggml_vec_dot_f32(ne03, &v, 0,
- wdata_src_b + i1n, 0,
- wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1);
+ wdata_kernel_c + i01*ne00*ne03 + i00*ne03, 0,
+ wdata_src_b + i1n, 0, 1);
}
dst_data[(i11*stride + i01)*ne0 + i10*stride + i00] += v;
}
diff --git a/ggml/src/ggml-cpu/vec.cpp b/ggml/src/ggml-cpu/vec.cpp
index ed604299a..9ff7335b8 100644
--- a/ggml/src/ggml-cpu/vec.cpp
+++ b/ggml/src/ggml-cpu/vec.cpp
@@ -377,6 +377,85 @@ void ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * G
*s = sumf;
}
+void ggml_vec_dot_f16_f32(int n, float * GGML_RESTRICT s, size_t bs, const ggml_fp16_t * GGML_RESTRICT x, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc) {
+ assert(nrc == 1);
+ GGML_UNUSED(nrc);
+ GGML_UNUSED(bx);
+ GGML_UNUSED(by);
+ GGML_UNUSED(bs);
+
+ ggml_float sumf = 0.0;
+ int i = 0;
+
+ // on AVX512 and AVX with F16C, x is widened to F32 in-register and accumulated in F32 over 4 independent sums
+#if defined(__AVX512F__)
+ __m512 acc0 = _mm512_setzero_ps();
+ __m512 acc1 = _mm512_setzero_ps();
+ __m512 acc2 = _mm512_setzero_ps();
+ __m512 acc3 = _mm512_setzero_ps();
+ for (; i + 64 <= n; i += 64) {
+ acc0 = _mm512_fmadd_ps(_mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x + i + 0))), _mm512_loadu_ps(y + i + 0), acc0);
+ acc1 = _mm512_fmadd_ps(_mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x + i + 16))), _mm512_loadu_ps(y + i + 16), acc1);
+ acc2 = _mm512_fmadd_ps(_mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x + i + 32))), _mm512_loadu_ps(y + i + 32), acc2);
+ acc3 = _mm512_fmadd_ps(_mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x + i + 48))), _mm512_loadu_ps(y + i + 48), acc3);
+ }
+ for (; i + 16 <= n; i += 16) {
+ acc0 = _mm512_fmadd_ps(_mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x + i))), _mm512_loadu_ps(y + i), acc0);
+ }
+ sumf = _mm512_reduce_add_ps(_mm512_add_ps(_mm512_add_ps(acc0, acc1), _mm512_add_ps(acc2, acc3)));
+#elif defined(__AVX__) && defined(__F16C__)
+ __m256 acc0 = _mm256_setzero_ps();
+ __m256 acc1 = _mm256_setzero_ps();
+ __m256 acc2 = _mm256_setzero_ps();
+ __m256 acc3 = _mm256_setzero_ps();
+ for (; i + 32 <= n; i += 32) {
+ acc0 = GGML_F32x8_FMA(acc0, _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x + i + 0))), _mm256_loadu_ps(y + i + 0));
+ acc1 = GGML_F32x8_FMA(acc1, _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x + i + 8))), _mm256_loadu_ps(y + i + 8));
+ acc2 = GGML_F32x8_FMA(acc2, _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x + i + 16))), _mm256_loadu_ps(y + i + 16));
+ acc3 = GGML_F32x8_FMA(acc3, _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x + i + 24))), _mm256_loadu_ps(y + i + 24));
+ }
+ for (; i + 8 <= n; i += 8) {
+ acc0 = GGML_F32x8_FMA(acc0, _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x + i))), _mm256_loadu_ps(y + i));
+ }
+ const __m256 acc = _mm256_add_ps(_mm256_add_ps(acc0, acc1), _mm256_add_ps(acc2, acc3));
+ __m128 r = _mm_add_ps(_mm256_castps256_ps128(acc), _mm256_extractf128_ps(acc, 1));
+ r = _mm_add_ps(r, _mm_movehl_ps(r, r));
+ r = _mm_add_ss(r, _mm_movehdup_ps(r));
+ sumf = _mm_cvtss_f32(r);
+#elif defined(GGML_SIMD)
+ // convert chunks of x to F32 and reuse the F32 SIMD dot
+ // TODO: convert F16 to F32 in-register and fuse the dot (needs a new SIMD macro); also avoid the per-chunk call overhead for tiny k
+ float tx[64];
+ for (; i + 64 <= n; i += 64) {
+ const ggml_fp16_t * GGML_RESTRICT xr = x + i;
+ float * GGML_RESTRICT tr = tx;
+ for (int j = 0; j < 64; ++j) {
+ tr[j] = GGML_CPU_FP16_TO_FP32(xr[j]);
+ }
+ float v;
+ ggml_vec_dot_f32(64, &v, 0, tx, 0, y + i, 0, 1);
+ sumf += v;
+ }
+ if (i < n) {
+ const int nr = n - i;
+ const ggml_fp16_t * GGML_RESTRICT xr = x + i;
+ float * GGML_RESTRICT tr = tx;
+ for (int j = 0; j < nr; ++j) {
+ tr[j] = GGML_CPU_FP16_TO_FP32(xr[j]);
+ }
+ float v;
+ ggml_vec_dot_f32(nr, &v, 0, tx, 0, y + i, 0, 1);
+ sumf += v;
+ i = n;
+ }
+#endif
+ for (; i < n; ++i) {
+ sumf += (ggml_float)(GGML_CPU_FP16_TO_FP32(x[i])*y[i]);
+ }
+
+ *s = sumf;
+}
+
void ggml_vec_silu_f32(const int n, float * y, const float * x) {
int i = 0;
#if defined(__AVX512F__) && defined(__AVX512DQ__)
diff --git a/ggml/src/ggml-cpu/vec.h b/ggml/src/ggml-cpu/vec.h
index aa9b5c80e..6598016d3 100644
--- a/ggml/src/ggml-cpu/vec.h
+++ b/ggml/src/ggml-cpu/vec.h
@@ -70,6 +70,7 @@ extern ggml_fp16_t ggml_table_gelu_quick_f16[1 << 16];
void ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * GGML_RESTRICT x, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc);
void ggml_vec_dot_bf16(int n, float * GGML_RESTRICT s, size_t bs, ggml_bf16_t * GGML_RESTRICT x, size_t bx, ggml_bf16_t * GGML_RESTRICT y, size_t by, int nrc);
void ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * GGML_RESTRICT x, size_t bx, ggml_fp16_t * GGML_RESTRICT y, size_t by, int nrc);
+void ggml_vec_dot_f16_f32(int n, float * GGML_RESTRICT s, size_t bs, const ggml_fp16_t * GGML_RESTRICT x, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc);
void ggml_vec_silu_f32(const int n, float * y, const float * x);
ggml_float ggml_vec_cvar_f32(const int n, float * y, const float * x, const float mean); //it will also center y ( y = y - mean )