Commit 43bd7a2a4 for llama.cpp
commit 43bd7a2a42210726fa77ae1c97a33c9151d7fbb3
Author: Ruben Ortlam <rortlam@redhat.com>
Date: Wed Nov 18 11:26:03 2026 +0100
vulkan: add bfloat16 support to various ops (#32202)
* vulkan: expand bf16 support for unary and binary ops and rms_norm
* cpu: add bf16 support for scale, clamp, leaky_relu, swiglu_oai, swiglu_clamp, tri and diag
* fix sycl fill supports_op condition
* restore CUDA op order
diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
index 7c2542741..72efbc6e9 100644
--- a/ggml/src/ggml-cpu/ops.cpp
+++ b/ggml/src/ggml-cpu/ops.cpp
@@ -2240,7 +2240,8 @@ void ggml_compute_forward_fill(const ggml_compute_params * params, ggml_tensor *
// ggml_compute_tri
-static void ggml_compute_forward_tri_f32(const ggml_compute_params * params, ggml_tensor * dst) {
+template <typename T>
+static void ggml_compute_forward_tri_t(const ggml_compute_params * params, ggml_tensor * dst) {
const ggml_tensor * src0 = dst->src[0];
const ggml_tri_type ttype = (ggml_tri_type) ggml_get_op_params_i32(dst, 0);
@@ -2266,11 +2267,11 @@ static void ggml_compute_forward_tri_f32(const ggml_compute_params * params, ggm
const int64_t i02 = (ir - i03*ne02*ne01)/ne01;
const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);
- const float * src_ptr = (const float *) ((const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);
- float * dst_ptr = ( float *) (( char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1);
+ const T * src_ptr = (const T *) ((const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);
+ T * dst_ptr = ( T *) (( char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1);
for (int i0 = 0; i0 < ne0; ++i0) {
- dst_ptr[i0] = bipred(i0, i01) ? src_ptr[i0] : 0.0f;
+ dst_ptr[i0] = bipred(i0, i01) ? src_ptr[i0] : T(0);
}
}
}
@@ -2281,7 +2282,12 @@ void ggml_compute_forward_tri(const ggml_compute_params * params, ggml_tensor *
switch (src0->type) {
case GGML_TYPE_F32:
{
- ggml_compute_forward_tri_f32(params, dst);
+ ggml_compute_forward_tri_t<uint32_t>(params, dst);
+ } break;
+ case GGML_TYPE_F16:
+ case GGML_TYPE_BF16:
+ {
+ ggml_compute_forward_tri_t<uint16_t>(params, dst);
} break;
default:
{
@@ -2708,7 +2714,8 @@ static void ggml_compute_forward_leaky_relu_f32(
}
}
-static void ggml_compute_forward_leaky_relu_f16(
+template <typename T>
+static void ggml_compute_forward_leaky_relu_half(
const ggml_compute_params * params,
ggml_tensor * dst) {
@@ -2728,13 +2735,16 @@ static void ggml_compute_forward_leaky_relu_f16(
float negative_slope;
memcpy(&negative_slope, dst->op_params, sizeof(float));
- assert(dst->nb[0] == sizeof(ggml_fp16_t));
- assert(src0->nb[0] == sizeof(ggml_fp16_t));
+ assert(dst->nb[0] == sizeof(T));
+ assert(src0->nb[0] == sizeof(T));
for (int i = 0; i < n; i++) {
- ggml_vec_leaky_relu_f16(nc,
- (ggml_fp16_t *) ((char *) dst->data + i*( dst->nb[1])),
- (ggml_fp16_t *) ((char *) src0->data + i*(src0->nb[1])), negative_slope);
+ T * y = (T *) ((char *) dst->data + i*( dst->nb[1]));
+ const T * x = (const T *) ((const char *) src0->data + i*(src0->nb[1]));
+ for (int k = 0; k < nc; k++) {
+ const float v = type_conversion_table<T>::to_f32(x[k]);
+ y[k] = type_conversion_table<T>::from_f32(((v > 0.f) ? v : 0.f) + negative_slope * ((v < 0.0f) ? v : 0.f));
+ }
}
}
@@ -2751,7 +2761,11 @@ void ggml_compute_forward_leaky_relu(
} break;
case GGML_TYPE_F16:
{
- ggml_compute_forward_leaky_relu_f16(params, dst);
+ ggml_compute_forward_leaky_relu_half<ggml_fp16_t>(params, dst);
+ } break;
+ case GGML_TYPE_BF16:
+ {
+ ggml_compute_forward_leaky_relu_half<ggml_bf16_t>(params, dst);
} break;
default:
{
@@ -3452,7 +3466,8 @@ static void ggml_compute_forward_swiglu_oai_f32(
}
}
-static void ggml_compute_forward_swiglu_oai_f16(
+template <typename T>
+static void ggml_compute_forward_swiglu_oai_half(
const ggml_compute_params * params,
ggml_tensor * dst) {
@@ -3492,9 +3507,9 @@ static void ggml_compute_forward_swiglu_oai_f16(
const int ir1 = MIN(ir0 + dr, nr);
for (int i1 = ir0; i1 < ir1; i1++) {
- ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o);
- ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o);
- ggml_fp16_t * dst_p = (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1]));
+ T * src0_p = (T *) (src0_d + i1*src0_o);
+ T * src1_p = (T *) (src1_d + i1*src1_o);
+ T * dst_p = (T *) ((char *) dst->data + i1*(dst->nb[1]));
if (!src1) {
src0_p += swapped ? nc : 0;
@@ -3502,15 +3517,15 @@ static void ggml_compute_forward_swiglu_oai_f16(
}
for (int k = 0; k < nc; k++) {
- const float x = std::min(GGML_CPU_FP16_TO_FP32(src0_p[k]), limit);
- const float y = std::clamp(GGML_CPU_FP16_TO_FP32(src1_p[k]), -limit, limit);
+ const float x = std::min(type_conversion_table<T>::to_f32(src0_p[k]), limit);
+ const float y = std::clamp(type_conversion_table<T>::to_f32(src1_p[k]), -limit, limit);
const float out_glu = x / (1.f + expf(alpha * (-x)));
- dst_p[k] = GGML_CPU_FP32_TO_FP16(out_glu * (y + 1.f));
+ dst_p[k] = type_conversion_table<T>::from_f32(out_glu * (y + 1.f));
}
#ifndef NDEBUG
for (int k = 0; k < nc; k++) {
- const float x = GGML_CPU_FP16_TO_FP32(dst_p[k]);
+ const float x = type_conversion_table<T>::to_f32(dst_p[k]);
GGML_UNUSED(x);
assert(!isnan(x));
assert(!isinf(x));
@@ -3532,7 +3547,11 @@ static void ggml_compute_forward_swiglu_oai(
} break;
case GGML_TYPE_F16:
{
- ggml_compute_forward_swiglu_oai_f16(params, dst);
+ ggml_compute_forward_swiglu_oai_half<ggml_fp16_t>(params, dst);
+ } break;
+ case GGML_TYPE_BF16:
+ {
+ ggml_compute_forward_swiglu_oai_half<ggml_bf16_t>(params, dst);
} break;
default:
{
@@ -3602,7 +3621,8 @@ static void ggml_compute_forward_swiglu_clamp_f32(const ggml_compute_params * pa
}
}
-static void ggml_compute_forward_swiglu_clamp_f16(const ggml_compute_params * params, ggml_tensor * dst) {
+template <typename T>
+static void ggml_compute_forward_swiglu_clamp_half(const ggml_compute_params * params, ggml_tensor * dst) {
const ggml_tensor * src0 = dst->src[0];
const ggml_tensor * src1 = dst->src[1];
char * src0_d = (char *) src0->data;
@@ -3635,9 +3655,9 @@ static void ggml_compute_forward_swiglu_clamp_f16(const ggml_compute_params * pa
const int ir1 = MIN(ir0 + dr, nr);
for (int i1 = ir0; i1 < ir1; i1++) {
- ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1 * src0_o);
- ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1 * src1_o);
- ggml_fp16_t * dst_p = (ggml_fp16_t *) ((char *) dst->data + i1 * (dst->nb[1]));
+ T * src0_p = (T *) (src0_d + i1 * src0_o);
+ T * src1_p = (T *) (src1_d + i1 * src1_o);
+ T * dst_p = (T *) ((char *) dst->data + i1 * (dst->nb[1]));
if (!src1) {
src0_p += swapped ? nc : 0;
@@ -3645,14 +3665,14 @@ static void ggml_compute_forward_swiglu_clamp_f16(const ggml_compute_params * pa
}
for (int k = 0; k < nc; k++) {
- const float gate = std::min(GGML_FP16_TO_FP32(src0_p[k]), limit);
- const float up = std::clamp(GGML_FP16_TO_FP32(src1_p[k]), -limit, limit);
- dst_p[k] = GGML_FP32_TO_FP16(gate / (1.f + expf(-gate)) * up);
+ const float gate = std::min(type_conversion_table<T>::to_f32(src0_p[k]), limit);
+ const float up = std::clamp(type_conversion_table<T>::to_f32(src1_p[k]), -limit, limit);
+ dst_p[k] = type_conversion_table<T>::from_f32(gate / (1.f + expf(-gate)) * up);
}
#ifndef NDEBUG
for (int k = 0; k < nc; k++) {
- const float x = GGML_FP16_TO_FP32(dst_p[k]);
+ const float x = type_conversion_table<T>::to_f32(dst_p[k]);
GGML_UNUSED(x);
assert(!isnan(x));
assert(!isinf(x));
@@ -3667,7 +3687,10 @@ static void ggml_compute_forward_swiglu_clamp(const ggml_compute_params * params
ggml_compute_forward_swiglu_clamp_f32(params, dst);
break;
case GGML_TYPE_F16:
- ggml_compute_forward_swiglu_clamp_f16(params, dst);
+ ggml_compute_forward_swiglu_clamp_half<ggml_fp16_t>(params, dst);
+ break;
+ case GGML_TYPE_BF16:
+ ggml_compute_forward_swiglu_clamp_half<ggml_bf16_t>(params, dst);
break;
default:
GGML_ABORT("fatal error");
@@ -4953,7 +4976,8 @@ static void ggml_compute_forward_scale_f32(
}
}
-static void ggml_compute_forward_scale_bf16(
+template <typename T>
+static void ggml_compute_forward_scale_half(
const ggml_compute_params * params,
ggml_tensor * dst) {
@@ -4989,10 +5013,10 @@ static void ggml_compute_forward_scale_bf16(
const int64_t i2 = (i / ne01) % ne02;
const int64_t i3 = i / (ne01*ne02);
- const ggml_bf16_t * x = (const ggml_bf16_t *) ((const char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);
- ggml_bf16_t * y = (ggml_bf16_t *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3);
+ const T * x = (const T *) ((const char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);
+ T * y = (T *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3);
for (int k = 0; k < nc; k++) {
- y[k] = GGML_FP32_TO_BF16(s*GGML_BF16_TO_FP32(x[k]) + b);
+ y[k] = type_conversion_table<T>::from_f32(s*type_conversion_table<T>::to_f32(x[k]) + b);
}
}
}
@@ -5008,9 +5032,13 @@ void ggml_compute_forward_scale(
{
ggml_compute_forward_scale_f32(params, dst);
} break;
+ case GGML_TYPE_F16:
+ {
+ ggml_compute_forward_scale_half<ggml_fp16_t>(params, dst);
+ } break;
case GGML_TYPE_BF16:
{
- ggml_compute_forward_scale_bf16(params, dst);
+ ggml_compute_forward_scale_half<ggml_bf16_t>(params, dst);
} break;
default:
{
@@ -5694,7 +5722,8 @@ void ggml_compute_forward_get_rows_back(
// ggml_compute_forward_diag
-static void ggml_compute_forward_diag_f32(
+template <typename T>
+static void ggml_compute_forward_diag_t(
const ggml_compute_params * params,
ggml_tensor * dst) {
@@ -5714,14 +5743,14 @@ static void ggml_compute_forward_diag_f32(
GGML_ASSERT(ne02 == ne2);
GGML_ASSERT(ne03 == ne3);
- GGML_ASSERT(nb00 == sizeof(float));
- GGML_ASSERT(nb0 == sizeof(float));
+ GGML_ASSERT(nb00 == sizeof(T));
+ GGML_ASSERT(nb0 == sizeof(T));
for (int i3 = 0; i3 < ne3; i3++) {
for (int i2 = 0; i2 < ne2; i2++) {
for (int i1 = 0; i1 < ne1; i1++) {
- float * d = (float *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);
- float * s = (float *)((char *) src0->data + i3*nb03 + i2*nb02);
+ T * d = (T *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);
+ T * s = (T *)((char *) src0->data + i3*nb03 + i2*nb02);
for (int i0 = 0; i0 < i1; i0++) {
d[i0] = 0;
}
@@ -5743,7 +5772,12 @@ void ggml_compute_forward_diag(
switch (src0->type) {
case GGML_TYPE_F32:
{
- ggml_compute_forward_diag_f32(params, dst);
+ ggml_compute_forward_diag_t<uint32_t>(params, dst);
+ } break;
+ case GGML_TYPE_F16:
+ case GGML_TYPE_BF16:
+ {
+ ggml_compute_forward_diag_t<uint16_t>(params, dst);
} break;
default:
{
@@ -6026,7 +6060,8 @@ static void ggml_compute_forward_clamp_f32(
}
}
-static void ggml_compute_forward_clamp_f16(
+template <typename T>
+static void ggml_compute_forward_clamp_half(
const ggml_compute_params * params,
ggml_tensor * dst) {
@@ -6045,20 +6080,20 @@ static void ggml_compute_forward_clamp_f16(
GGML_TENSOR_UNARY_OP_LOCALS
- GGML_ASSERT( nb0 == sizeof(ggml_fp16_t));
- GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));
+ GGML_ASSERT( nb0 == sizeof(T));
+ GGML_ASSERT(nb00 == sizeof(T));
for (int j = ith; j < n; j += nth) {
const int64_t i1 = j % ne01;
const int64_t i2 = (j / ne01) % ne02;
const int64_t i3 = j / (ne01*ne02);
- ggml_fp16_t * dst_ptr = (ggml_fp16_t *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3);
- ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);
+ T * dst_ptr = (T *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3);
+ T * src0_ptr = (T *) ((char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);
for (int i = 0; i < nc; i++) {
- float v = GGML_CPU_FP16_TO_FP32(src0_ptr[i]);
- dst_ptr[i] = GGML_CPU_FP32_TO_FP16(MAX(MIN(v, max), min));
+ float v = type_conversion_table<T>::to_f32(src0_ptr[i]);
+ dst_ptr[i] = type_conversion_table<T>::from_f32(MAX(MIN(v, max), min));
}
}
}
@@ -6076,9 +6111,12 @@ void ggml_compute_forward_clamp(
} break;
case GGML_TYPE_F16:
{
- ggml_compute_forward_clamp_f16(params, dst);
+ ggml_compute_forward_clamp_half<ggml_fp16_t>(params, dst);
} break;
case GGML_TYPE_BF16:
+ {
+ ggml_compute_forward_clamp_half<ggml_bf16_t>(params, dst);
+ } break;
case GGML_TYPE_Q1_0:
case GGML_TYPE_Q2_0:
case GGML_TYPE_Q4_0:
diff --git a/ggml/src/ggml-cpu/vec.h b/ggml/src/ggml-cpu/vec.h
index c54ed3f07..4aac5e966 100644
--- a/ggml/src/ggml-cpu/vec.h
+++ b/ggml/src/ggml-cpu/vec.h
@@ -962,12 +962,6 @@ inline static void ggml_vec_relu_f16 (const int n, ggml_fp16_t * y, const ggml_f
}
}
inline static void ggml_vec_leaky_relu_f32 (const int n, float * y, const float * x, const float ns) { for (int i = 0; i < n; ++i) y[i] = ((x[i] > 0.f) ? x[i] : 0.f) + ns * ((x[i] < 0.0f) ? x[i] : 0.f); }
-inline static void ggml_vec_leaky_relu_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float ns) {
- for (int i = 0; i < n; ++i) {
- float v = GGML_CPU_FP16_TO_FP32(x[i]);
- y[i] = GGML_CPU_FP32_TO_FP16(((v > 0.f) ? v : 0.f) + ns * ((v < 0.0f) ? v : 0.f));
- }
-}
inline static void ggml_vec_sigmoid_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = 1.f / (1.f + expf(-x[i])); }
inline static void ggml_vec_sigmoid_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {
for (int i = 0; i < n; ++i) {
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index cc3ce73d0..b7842d289 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -6564,7 +6564,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16) && op->type == op->src[0]->type &&
ggml_is_contiguous_rows(op->src[0]);
case GGML_OP_CLAMP:
- return ggml_is_contiguous_rows(op->src[0]);
+ return ggml_is_contiguous_rows(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16);
case GGML_OP_ADD:
case GGML_OP_SUB:
case GGML_OP_MUL:
@@ -6664,7 +6664,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
case GGML_OP_PAD_REFLECT_1D:
case GGML_OP_ARANGE:
case GGML_OP_TIMESTEP_EMBEDDING:
+ return true;
case GGML_OP_LEAKY_RELU:
+ return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;
case GGML_OP_RWKV_WKV6:
case GGML_OP_GATED_LINEAR_ATTN:
case GGML_OP_RWKV_WKV7:
@@ -6690,7 +6692,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
case GGML_OP_FILL:
case GGML_OP_CUMSUM:
case GGML_OP_TRI:
+ return true;
case GGML_OP_DIAG:
+ return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;
case GGML_OP_SOLVE_TRI:
return true;
case GGML_OP_LIGHTNING_INDEXER:
diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m
index 0ea4a821a..f4c4991e9 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.m
+++ b/ggml/src/ggml-metal/ggml-metal-device.m
@@ -2019,7 +2019,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
};
}
case GGML_OP_DIAG:
- return true;
+ return op->src[0]->type == GGML_TYPE_F32;
case GGML_OP_OPT_STEP_ADAMW:
case GGML_OP_OPT_STEP_SGD:
return has_simdgroup_reduction;
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index 8cd38f382..3a15efabf 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -12283,7 +12283,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
case GGML_OP_TRANSPOSE:
return true;
case GGML_OP_DIAG:
- return true;
+ return op->src[0]->type == GGML_TYPE_F32;
case GGML_OP_ROPE: {
const int mode = ((const int32_t *) op->op_params)[2];
if (!ggml_rope_get_truncate(op)) {
diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp
index 3bcbad6f8..4fc90e543 100644
--- a/ggml/src/ggml-sycl/ggml-sycl.cpp
+++ b/ggml/src/ggml-sycl/ggml-sycl.cpp
@@ -7700,8 +7700,10 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
return false;
}
case GGML_OP_FILL:
+ return op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16;
case GGML_OP_CUMSUM:
case GGML_OP_DIAG:
+ return op->src[0]->type == GGML_TYPE_F32;
case GGML_OP_CROSS_ENTROPY_LOSS:
case GGML_OP_CROSS_ENTROPY_LOSS_BACK:
return true;
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan-types.h b/ggml/src/ggml-vulkan/ggml-vulkan-types.h
index e19e2d029..54c7a3690 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan-types.h
+++ b/ggml/src/ggml-vulkan/ggml-vulkan-types.h
@@ -838,15 +838,16 @@ struct vk_device_struct {
vk_pipeline pipeline_acc_f32;
vk_pipeline pipeline_set_f32;
+ // [src0][src1][dst], 0=fp32,1=fp16,2=bf16, null where unsupported
+ vk_pipeline pipeline_add[3][3][3];
+ vk_pipeline pipeline_add_norepeat[3][3][3];
+ vk_pipeline pipeline_sub[3][3][3];
+ vk_pipeline pipeline_sub_norepeat[3][3][3];
+ vk_pipeline pipeline_mul[3][3][3];
+ vk_pipeline pipeline_mul_norepeat[3][3][3];
+ vk_pipeline pipeline_div[3][3][3];
+ vk_pipeline pipeline_div_norepeat[3][3][3];
// [src0 0=fp32,1=fp16][src1 0=fp32,1=fp16][dst 0=fp32,1=fp16]
- vk_pipeline pipeline_add[2][2][2];
- vk_pipeline pipeline_add_norepeat[2][2][2];
- vk_pipeline pipeline_sub[2][2][2];
- vk_pipeline pipeline_sub_norepeat[2][2][2];
- vk_pipeline pipeline_mul[2][2][2];
- vk_pipeline pipeline_mul_norepeat[2][2][2];
- vk_pipeline pipeline_div[2][2][2];
- vk_pipeline pipeline_div_norepeat[2][2][2];
vk_pipeline pipeline_add_rms[2][2][2];
vk_pipeline pipeline_add_rms_norepeat[2][2][2];
@@ -859,11 +860,17 @@ struct vk_device_struct {
vk_pipeline pipeline_concat_i8, pipeline_concat_i16, pipeline_concat_i32, pipeline_concat_i64;
vk_pipeline pipeline_concat_transpose_i8, pipeline_concat_transpose_i16, pipeline_concat_transpose_i32;
vk_pipeline pipeline_upscale_nearest_f32, pipeline_upscale_bilinear_f32, pipeline_upscale_bicubic_f32, pipeline_upscale_bilinear_antialias_f32;
- vk_pipeline pipeline_scale_f32;
- vk_pipeline pipeline_log[2];
- vk_pipeline pipeline_tri[2];
- vk_pipeline pipeline_diag[2];
- vk_pipeline pipeline_clamp[2];
+ // [src/dst 0=fp32,1=fp16,2=bf16]
+ vk_pipeline pipeline_scale[3];
+ vk_pipeline pipeline_log[3];
+ vk_pipeline pipeline_tri[3];
+ vk_pipeline pipeline_diag[3];
+ vk_pipeline pipeline_clamp[3];
+ vk_pipeline pipeline_sqr[3];
+ vk_pipeline pipeline_sqrt[3];
+ vk_pipeline pipeline_sin[3];
+ vk_pipeline pipeline_cos[3];
+ vk_pipeline pipeline_leaky_relu[3];
vk_pipeline pipeline_pad_f32;
vk_pipeline pipeline_pad_reflect_1d_f32;
vk_pipeline pipeline_roll_f32;
@@ -880,10 +887,10 @@ struct vk_device_struct {
vk_pipeline pipeline_set_rows_i64[2][GGML_TYPE_COUNT];
vk_pipeline pipeline_norm_f32;
vk_pipeline pipeline_group_norm_f32;
- std::array<vk_pipeline, 3> pipeline_rms_norm_f32;
- std::array<vk_pipeline, 3> pipeline_rms_norm_mul_f32;
- std::array<vk_pipeline, 3> pipeline_rms_norm_mul_add_f32;
- std::array<vk_pipeline, 3> pipeline_rms_norm_mul_add_mul_f32;
+ // [src/dst 0=fp32,1=bf16][mul]
+ std::array<vk_pipeline, 3> pipeline_rms_norm[2][2];
+ // [src/dst 0=fp32,1=bf16][residual 0=fp32,1=bf16][post mul]
+ std::array<vk_pipeline, 3> pipeline_rms_norm_mul_add[2][2][2];
vk_pipeline pipeline_rms_norm_mul_add_partials_f32;
vk_pipeline pipeline_rms_norm_mul_add_mul_partials_f32;
vk_pipeline pipeline_rms_norm_set_rows_f32_f32;
@@ -895,36 +902,11 @@ struct vk_device_struct {
vk_pipeline pipeline_rms_norm_back_f32;
vk_pipeline pipeline_l2_norm_f32;
- // [src/dst 0=fp32,1=fp16]
- vk_pipeline pipeline_exp[2];
- vk_pipeline pipeline_expm1[2];
- vk_pipeline pipeline_elu[2];
- vk_pipeline pipeline_gelu[2];
- vk_pipeline pipeline_gelu_erf[2];
- vk_pipeline pipeline_gelu_quick[2];
- vk_pipeline pipeline_silu[2];
- vk_pipeline pipeline_relu[2];
- vk_pipeline pipeline_sqr[2];
- vk_pipeline pipeline_sqrt[2];
- vk_pipeline pipeline_sin[2];
- vk_pipeline pipeline_cos[2];
- vk_pipeline pipeline_xielu[2];
- vk_pipeline pipeline_neg[2];
- vk_pipeline pipeline_tanh[2];
- vk_pipeline pipeline_sigmoid[2];
- vk_pipeline pipeline_hardsigmoid[2];
- vk_pipeline pipeline_hardswish[2];
- vk_pipeline pipeline_abs[2];
- vk_pipeline pipeline_softplus[2];
- vk_pipeline pipeline_step[2];
- vk_pipeline pipeline_round[2];
- vk_pipeline pipeline_ceil[2];
- vk_pipeline pipeline_floor[2];
- vk_pipeline pipeline_trunc[2];
- vk_pipeline pipeline_sgn[2];
-
- // fused UNARY+MUL pipelines: [op][f16][norepeat][op_on_b]
- vk_pipeline pipeline_unary_mul[4][2][2][2];
+ // [src/dst 0=fp32,1=fp16,2=bf16]
+ vk_pipeline pipeline_unary[GGML_UNARY_OP_COUNT][3];
+
+ // fused UNARY+MUL pipelines: [op][0=fp32,1=fp16,2=bf16][norepeat][op_on_b]
+ vk_pipeline pipeline_unary_mul[4][3][2][2];
vk_pipeline pipeline_add1_f16_f16;
vk_pipeline pipeline_add1_f16_f32;
@@ -935,15 +917,9 @@ struct vk_device_struct {
vk_pipeline pipeline_fill_f32;
vk_pipeline pipeline_fill_f16;
- vk_pipeline pipeline_geglu[2];
- vk_pipeline pipeline_reglu[2];
- vk_pipeline pipeline_swiglu[2];
- vk_pipeline pipeline_swiglu_oai[2];
- vk_pipeline pipeline_swiglu_clamp[2];
- vk_pipeline pipeline_geglu_erf[2];
- vk_pipeline pipeline_geglu_quick[2];
+ // [src/dst 0=fp32,1=fp16,2=bf16]
+ vk_pipeline pipeline_glu[GGML_GLU_OP_COUNT][3];
- vk_pipeline pipeline_leaky_relu[2];
vk_pipeline pipeline_silu_back_f32;
vk_pipeline pipeline_soft_max_f32, pipeline_soft_max_f32_f16;
vk_pipeline pipeline_soft_max_f32_wg512, pipeline_soft_max_f32_f16_wg512;
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index 45bfe52f4..616a17c4c 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -3600,10 +3600,16 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
for (uint32_t i = 0; i < 3; ++i) {
const uint32_t block_size = 128u << i;
const std::string suffix = "_" + std::to_string(block_size);
- ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_f32[i], "rms_norm_f32" + suffix, RMS_SHADER(rms_norm_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0, 0, block_size, rms_unroll}, 1, true, rms_subgroups);
- ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_f32[i], "rms_norm_mul_f32" + suffix, RMS_SHADER(rms_norm_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0, block_size, rms_unroll}, 1, true, rms_subgroups);
- ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_add_f32[i], "rms_norm_mul_add_f32" + suffix, RMS_SHADER(rms_norm_mul_add_f32), "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0, block_size, rms_unroll}, 1, true, rms_subgroups);
- ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_add_mul_f32[i], "rms_norm_mul_add_mul_f32" + suffix, RMS_SHADER(rms_norm_mul_add_f32), "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 1, block_size, rms_unroll}, 1, true, rms_subgroups);
+ ggml_vk_create_pipeline2(device, device->pipeline_rms_norm[0][0][i], "rms_norm_f32" + suffix, RMS_SHADER(rms_norm_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0, 0, block_size, rms_unroll}, 1, true, rms_subgroups);
+ ggml_vk_create_pipeline2(device, device->pipeline_rms_norm[0][1][i], "rms_norm_mul_f32" + suffix, RMS_SHADER(rms_norm_f32), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0, block_size, rms_unroll}, 1, true, rms_subgroups);
+ ggml_vk_create_pipeline2(device, device->pipeline_rms_norm[1][0][i], "rms_norm_bf16" + suffix, RMS_SHADER(rms_norm_bf16), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 0, 0, block_size, rms_unroll}, 1, true, rms_subgroups);
+ ggml_vk_create_pipeline2(device, device->pipeline_rms_norm[1][1][i], "rms_norm_mul_bf16" + suffix, RMS_SHADER(rms_norm_bf16), "main", 4, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0, block_size, rms_unroll}, 1, true, rms_subgroups);
+ for (uint32_t pm : {0, 1}) {
+ const std::string name = pm ? "rms_norm_mul_add_mul" : "rms_norm_mul_add";
+ ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_add[0][0][pm][i], name + "_f32" + suffix, RMS_SHADER(rms_norm_mul_add_f32), "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, pm, block_size, rms_unroll}, 1, true, rms_subgroups);
+ ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_add[1][0][pm][i], name + "_bf16_f32" + suffix, RMS_SHADER(rms_norm_mul_add_bf16_f32), "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, pm, block_size, rms_unroll}, 1, true, rms_subgroups);
+ ggml_vk_create_pipeline2(device, device->pipeline_rms_norm_mul_add[1][1][pm][i], name + "_bf16_bf16" + suffix, RMS_SHADER(rms_norm_mul_add_bf16_bf16), "main", 5, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, pm, block_size, rms_unroll}, 1, true, rms_subgroups);
+ }
}
ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_partials_f32, "rms_norm_mul_add_partials_f32", rms_norm_mul_add_partials_f32_len, rms_norm_mul_add_partials_f32_data, "main", 6, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 0}, 1, true);
ggml_vk_create_pipeline(device, device->pipeline_rms_norm_mul_add_mul_partials_f32, "rms_norm_mul_add_mul_partials_f32", rms_norm_mul_add_partials_f32_len, rms_norm_mul_add_partials_f32_data, "main", 6, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {0, 1, 1}, 1, true);
@@ -3714,6 +3720,22 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
CREATE_BINARY(add_rms, _norepeat, {1}, 4)
#undef CREATE_BINARY
+#define CREATE_BINARY_BF16(name, namemod, spec, bindings) \
+ ggml_vk_create_pipeline2(device, device->pipeline_ ## name ## namemod[2][0][2], #name "_bf16_f32_bf16" #namemod, name ## _bf16_f32_bf16_len, name ## _bf16_f32_bf16_data, \
+ "main", (bindings), sizeof(vk_op_binary_push_constants), {512, 1, 1}, spec, 1); \
+ ggml_vk_create_pipeline2(device, device->pipeline_ ## name ## namemod[2][2][2], #name "_bf16_bf16_bf16" #namemod, name ## _bf16_bf16_bf16_len, name ## _bf16_bf16_bf16_data, \
+ "main", (bindings), sizeof(vk_op_binary_push_constants), {512, 1, 1}, spec, 1);
+
+ CREATE_BINARY_BF16(add, , {0}, 4)
+ CREATE_BINARY_BF16(add, _norepeat, {1}, 4)
+ CREATE_BINARY_BF16(sub, , {0}, 3)
+ CREATE_BINARY_BF16(sub, _norepeat, {1}, 3)
+ CREATE_BINARY_BF16(mul, , {0}, 3)
+ CREATE_BINARY_BF16(mul, _norepeat, {1}, 3)
+ CREATE_BINARY_BF16(div, , {0}, 3)
+ CREATE_BINARY_BF16(div, _norepeat, {1}, 3)
+#undef CREATE_BINARY_BF16
+
if (device->multi_add) {
for (uint32_t i = 0; i < MAX_FUSED_ADDS; ++i) {
ggml_vk_create_pipeline2(device, device->pipeline_multi_add[i], "multi_add_f32_" + std::to_string(i+1), multi_add_f32_len, multi_add_f32_data, "main", MAX_PARAMETER_COUNT, sizeof(vk_op_multi_add_push_constants), {512, 1, 1}, {i+2}, 1);
@@ -3739,17 +3761,6 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
ggml_vk_create_pipeline(device, device->pipeline_upscale_bicubic_f32, "upscale_f32", upscale_f32_len, upscale_f32_data, "main", 2, sizeof(vk_op_upscale_push_constants), {512, 1, 1}, {GGML_SCALE_MODE_BICUBIC}, 1);
ggml_vk_create_pipeline(device, device->pipeline_upscale_bilinear_antialias_f32, "upscale_f32", upscale_f32_len, upscale_f32_data, "main", 2, sizeof(vk_op_upscale_push_constants), {512, 1, 1}, {GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ANTIALIAS}, 1);
- ggml_vk_create_pipeline(device, device->pipeline_scale_f32, "scale_f32", scale_f32_len, scale_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
-
- ggml_vk_create_pipeline(device, device->pipeline_log[0], "log_f32", log_f32_len, log_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
- ggml_vk_create_pipeline(device, device->pipeline_log[1], "log_f16", log_f16_len, log_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
-
- ggml_vk_create_pipeline(device, device->pipeline_tri[0], "tri_f32", tri_f32_len, tri_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
- ggml_vk_create_pipeline(device, device->pipeline_tri[1], "tri_f16", tri_f16_len, tri_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
-
- ggml_vk_create_pipeline(device, device->pipeline_diag[0], "diag_f32", diag_f32_len, diag_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
- ggml_vk_create_pipeline(device, device->pipeline_diag[1], "diag_f16", diag_f16_len, diag_f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
-
ggml_vk_create_pipeline(device, device->pipeline_pad_f32, "pad_f32", pad_f32_len, pad_f32_data, "main", 2, sizeof(vk_op_pad_push_constants), {512, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_pad_reflect_1d_f32, "pad_reflect_1d_f32", pad_reflect_1d_f32_len, pad_reflect_1d_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
@@ -3762,44 +3773,63 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
#define CREATE_UNARY(name) \
ggml_vk_create_pipeline(device, device->pipeline_ ## name [0], #name "_f32", name ## _f32_len, name ## _f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); \
- ggml_vk_create_pipeline(device, device->pipeline_ ## name [1], #name "_f16", name ## _f16_len, name ## _f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
-
- CREATE_UNARY(elu)
- CREATE_UNARY(gelu)
- CREATE_UNARY(gelu_erf)
- CREATE_UNARY(gelu_quick)
- CREATE_UNARY(silu)
- CREATE_UNARY(relu)
+ ggml_vk_create_pipeline(device, device->pipeline_ ## name [1], #name "_f16", name ## _f16_len, name ## _f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); \
+ ggml_vk_create_pipeline(device, device->pipeline_ ## name [2], #name "_bf16", name ## _bf16_len, name ## _bf16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
+
+ CREATE_UNARY(scale)
+ CREATE_UNARY(log)
CREATE_UNARY(sqr)
CREATE_UNARY(sqrt)
CREATE_UNARY(sin)
CREATE_UNARY(cos)
CREATE_UNARY(clamp)
CREATE_UNARY(leaky_relu)
- CREATE_UNARY(xielu)
- CREATE_UNARY(neg)
- CREATE_UNARY(tanh)
- CREATE_UNARY(sigmoid)
- CREATE_UNARY(hardsigmoid)
- CREATE_UNARY(hardswish)
- CREATE_UNARY(abs)
- CREATE_UNARY(softplus)
- CREATE_UNARY(step)
- CREATE_UNARY(round)
- CREATE_UNARY(ceil)
- CREATE_UNARY(floor)
- CREATE_UNARY(trunc)
- CREATE_UNARY(sgn)
- CREATE_UNARY(exp)
- CREATE_UNARY(expm1)
#undef CREATE_UNARY
+#define CREATE_COPY_OP(name) \
+ ggml_vk_create_pipeline(device, device->pipeline_ ## name [0], #name "_32", name ## _32_len, name ## _32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); \
+ ggml_vk_create_pipeline(device, device->pipeline_ ## name [1], #name "_16", name ## _16_len, name ## _16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); \
+ device->pipeline_ ## name [2] = device->pipeline_ ## name [1];
+
+ CREATE_COPY_OP(tri)
+ CREATE_COPY_OP(diag)
+#undef CREATE_COPY_OP
+
+#define CREATE_UNARY_OP(name, op) \
+ ggml_vk_create_pipeline(device, device->pipeline_unary[GGML_UNARY_OP_ ## op][0], #name "_f32", name ## _f32_len, name ## _f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); \
+ ggml_vk_create_pipeline(device, device->pipeline_unary[GGML_UNARY_OP_ ## op][1], #name "_f16", name ## _f16_len, name ## _f16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1); \
+ ggml_vk_create_pipeline(device, device->pipeline_unary[GGML_UNARY_OP_ ## op][2], #name "_bf16", name ## _bf16_len, name ## _bf16_data, "main", 2, sizeof(vk_op_unary_push_constants), {512, 1, 1}, {}, 1);
+
+ CREATE_UNARY_OP(elu, ELU)
+ CREATE_UNARY_OP(gelu, GELU)
+ CREATE_UNARY_OP(gelu_erf, GELU_ERF)
+ CREATE_UNARY_OP(gelu_quick, GELU_QUICK)
+ CREATE_UNARY_OP(silu, SILU)
+ CREATE_UNARY_OP(relu, RELU)
+ CREATE_UNARY_OP(xielu, XIELU)
+ CREATE_UNARY_OP(neg, NEG)
+ CREATE_UNARY_OP(tanh, TANH)
+ CREATE_UNARY_OP(sigmoid, SIGMOID)
+ CREATE_UNARY_OP(hardsigmoid, HARDSIGMOID)
+ CREATE_UNARY_OP(hardswish, HARDSWISH)
+ CREATE_UNARY_OP(abs, ABS)
+ CREATE_UNARY_OP(softplus, SOFTPLUS)
+ CREATE_UNARY_OP(step, STEP)
+ CREATE_UNARY_OP(round, ROUND)
+ CREATE_UNARY_OP(ceil, CEIL)
+ CREATE_UNARY_OP(floor, FLOOR)
+ CREATE_UNARY_OP(trunc, TRUNC)
+ CREATE_UNARY_OP(sgn, SGN)
+ CREATE_UNARY_OP(exp, EXP)
+ CREATE_UNARY_OP(expm1, EXPM1)
+#undef CREATE_UNARY_OP
+
// spec constants: {norepeat, op_on_b}
#define CREATE_UNARY_MUL(name, idx) \
- for (int dt = 0; dt < 2; ++dt) { \
- const size_t len_ = dt ? name ## _mul_f16_len : name ## _mul_f32_len; \
- const unsigned char * data_ = dt ? name ## _mul_f16_data : name ## _mul_f32_data; \
- const std::string dts_ = dt ? "f16" : "f32"; \
+ for (int dt = 0; dt < 3; ++dt) { \
+ const size_t len_ = dt == 2 ? name ## _mul_bf16_len : dt ? name ## _mul_f16_len : name ## _mul_f32_len; \
+ const unsigned char * data_ = dt == 2 ? name ## _mul_bf16_data : dt ? name ## _mul_f16_data : name ## _mul_f32_data; \
+ const std::string dts_ = dt == 2 ? "bf16" : dt ? "f16" : "f32"; \
for (int ob = 0; ob < 2; ++ob) \
for (int nr = 0; nr < 2; ++nr) \
ggml_vk_create_pipeline(device, device->pipeline_unary_mul[(idx)][dt][nr][ob], \
@@ -3819,17 +3849,18 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
ggml_vk_create_pipeline(device, device->pipeline_fill_f32, "fill_f32", fill_f32_len, fill_f32_data, "main", 1, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1);
ggml_vk_create_pipeline(device, device->pipeline_fill_f16, "fill_f16", fill_f16_len, fill_f16_data, "main", 1, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1);
-#define CREATE_GLU(name) \
- ggml_vk_create_pipeline(device, device->pipeline_ ## name [0], #name "_f32", name ## _f32_len, name ## _f32_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true); \
- ggml_vk_create_pipeline(device, device->pipeline_ ## name [1], #name "_f16", name ## _f16_len, name ## _f16_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true);
-
- CREATE_GLU(geglu)
- CREATE_GLU(reglu)
- CREATE_GLU(swiglu)
- CREATE_GLU(swiglu_oai)
- CREATE_GLU(swiglu_clamp)
- CREATE_GLU(geglu_erf)
- CREATE_GLU(geglu_quick)
+#define CREATE_GLU(name, op) \
+ ggml_vk_create_pipeline(device, device->pipeline_glu[GGML_GLU_OP_ ## op][0], #name "_f32", name ## _f32_len, name ## _f32_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true); \
+ ggml_vk_create_pipeline(device, device->pipeline_glu[GGML_GLU_OP_ ## op][1], #name "_f16", name ## _f16_len, name ## _f16_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true); \
+ ggml_vk_create_pipeline(device, device->pipeline_glu[GGML_GLU_OP_ ## op][2], #name "_bf16", name ## _bf16_len, name ## _bf16_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true);
+
+ CREATE_GLU(geglu, GEGLU)
+ CREATE_GLU(reglu, REGLU)
+ CREATE_GLU(swiglu, SWIGLU)
+ CREATE_GLU(swiglu_oai, SWIGLU_OAI)
+ CREATE_GLU(swiglu_clamp, SWIGLU_CLAMP)
+ CREATE_GLU(geglu_erf, GEGLU_ERF)
+ CREATE_GLU(geglu_quick, GEGLU_QUICK)
#undef CREATE_GLU
ggml_vk_create_pipeline(device, device->pipeline_silu_back_f32, "silu_back_f32", silu_back_f32_len, silu_back_f32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1);
@@ -9374,6 +9405,33 @@ static bool ggml_vk_concat_is_transposed(const vk_device & device, const ggml_te
return src0->ne[1] == src1->ne[1] && dst->ne[1] == src1->ne[1];
}
+static bool ggml_vk_is_float_type(ggml_type type) {
+ return type == GGML_TYPE_F32 || type == GGML_TYPE_F16 || type == GGML_TYPE_BF16;
+}
+
+static uint32_t ggml_vk_float_type_idx(ggml_type type) {
+ return type == GGML_TYPE_BF16 ? 2 : type == GGML_TYPE_F16 ? 1 : 0;
+}
+
+static bool ggml_vk_is_float_elementwise(const ggml_tensor * op) {
+ return ggml_vk_is_float_type(op->type) && op->src[0]->type == op->type;
+}
+
+// bf16: bf16 src0/dst, f32 or bf16 src1
+static bool ggml_vk_binary_types_supported(ggml_type src0, ggml_type src1, ggml_type dst) {
+ if (src0 == GGML_TYPE_BF16 || src1 == GGML_TYPE_BF16 || dst == GGML_TYPE_BF16) {
+ return src0 == GGML_TYPE_BF16 && dst == GGML_TYPE_BF16 && (src1 == GGML_TYPE_BF16 || src1 == GGML_TYPE_F32);
+ }
+ return ggml_vk_is_float_type(src0) && ggml_vk_is_float_type(src1) && ggml_vk_is_float_type(dst);
+}
+
+static vk_pipeline ggml_vk_float_pipeline(const vk_pipeline (&pipelines)[3], const ggml_tensor * src0, const ggml_tensor * dst) {
+ if (src0->type != dst->type || !ggml_vk_is_float_type(dst->type)) {
+ return nullptr;
+ }
+ return pipelines[ggml_vk_float_type_idx(dst->type)];
+}
+
static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * dst, ggml_op op) {
switch (op) {
case GGML_OP_GET_ROWS:
@@ -9410,14 +9468,19 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
case GGML_OP_SUB:
case GGML_OP_MUL:
case GGML_OP_DIV:
- if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) ||
- (src1->type != GGML_TYPE_F32 && src1->type != GGML_TYPE_F16) ||
- (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16)) {
+ {
+ if (!ggml_vk_binary_types_supported(src0->type, src1->type, dst->type)) {
return nullptr;
}
+ const uint32_t s0 = ggml_vk_float_type_idx(src0->type);
+ const uint32_t s1 = ggml_vk_float_type_idx(src1->type);
+ const uint32_t d = ggml_vk_float_type_idx(dst->type);
switch (op) {
case GGML_OP_ADD:
{
+ if ((ctx->num_additional_fused_ops > 0 || ctx->do_add_rms_partials) && (s0 == 2 || s1 == 2 || d == 2)) {
+ return nullptr;
+ }
if (ctx->num_additional_fused_ops > 0) {
if (ctx->do_add_rms_partials) {
return ctx->device->pipeline_multi_add_rms[ctx->num_additional_fused_ops];
@@ -9430,28 +9493,29 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
return pipelines[src0->type == GGML_TYPE_F16][src1->type == GGML_TYPE_F16][dst->type == GGML_TYPE_F16];
} else {
auto pipelines = ggml_are_same_shape(src0, src1) ? ctx->device->pipeline_add_norepeat : ctx->device->pipeline_add;
- return pipelines[src0->type == GGML_TYPE_F16][src1->type == GGML_TYPE_F16][dst->type == GGML_TYPE_F16];
+ return pipelines[s0][s1][d];
}
}
case GGML_OP_SUB:
{
auto pipelines = ggml_are_same_shape(src0, src1) ? ctx->device->pipeline_sub_norepeat : ctx->device->pipeline_sub;
- return pipelines[src0->type == GGML_TYPE_F16][src1->type == GGML_TYPE_F16][dst->type == GGML_TYPE_F16];
+ return pipelines[s0][s1][d];
}
case GGML_OP_MUL:
{
auto pipelines = ggml_are_same_shape(src0, src1) ? ctx->device->pipeline_mul_norepeat : ctx->device->pipeline_mul;
- return pipelines[src0->type == GGML_TYPE_F16][src1->type == GGML_TYPE_F16][dst->type == GGML_TYPE_F16];
+ return pipelines[s0][s1][d];
}
case GGML_OP_DIV:
{
auto pipelines = ggml_are_same_shape(src0, src1) ? ctx->device->pipeline_div_norepeat : ctx->device->pipeline_div;
- return pipelines[src0->type == GGML_TYPE_F16][src1->type == GGML_TYPE_F16][dst->type == GGML_TYPE_F16];
+ return pipelines[s0][s1][d];
}
default:
break;
}
return nullptr;
+ }
case GGML_OP_ADD_ID:
if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && src2->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_F32) {
return ctx->device->pipeline_add_id_f32;
@@ -9509,58 +9573,23 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
}
return nullptr;
case GGML_OP_SCALE:
- if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
- return ctx->device->pipeline_scale_f32;
- }
- return nullptr;
+ return ggml_vk_float_pipeline(ctx->device->pipeline_scale, src0, dst);
case GGML_OP_SQR:
- if (src0->type == dst->type &&
- (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {
- return ctx->device->pipeline_sqr[dst->type == GGML_TYPE_F16];
- }
- return nullptr;
+ return ggml_vk_float_pipeline(ctx->device->pipeline_sqr, src0, dst);
case GGML_OP_SQRT:
- if (src0->type == dst->type &&
- (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {
- return ctx->device->pipeline_sqrt[dst->type == GGML_TYPE_F16];
- }
- return nullptr;
+ return ggml_vk_float_pipeline(ctx->device->pipeline_sqrt, src0, dst);
case GGML_OP_SIN:
- if (src0->type == dst->type &&
- (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {
- return ctx->device->pipeline_sin[dst->type == GGML_TYPE_F16];
- }
- return nullptr;
+ return ggml_vk_float_pipeline(ctx->device->pipeline_sin, src0, dst);
case GGML_OP_COS:
- if (src0->type == dst->type &&
- (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {
- return ctx->device->pipeline_cos[dst->type == GGML_TYPE_F16];
- }
- return nullptr;
+ return ggml_vk_float_pipeline(ctx->device->pipeline_cos, src0, dst);
case GGML_OP_LOG:
- if (src0->type == dst->type &&
- (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {
- return ctx->device->pipeline_log[dst->type == GGML_TYPE_F16];
- }
- return nullptr;
+ return ggml_vk_float_pipeline(ctx->device->pipeline_log, src0, dst);
case GGML_OP_TRI:
- if (src0->type == dst->type &&
- (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {
- return ctx->device->pipeline_tri[dst->type == GGML_TYPE_F16];
- }
- return nullptr;
+ return ggml_vk_float_pipeline(ctx->device->pipeline_tri, src0, dst);
case GGML_OP_DIAG:
- if (src0->type == dst->type &&
- (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {
- return ctx->device->pipeline_diag[dst->type == GGML_TYPE_F16];
- }
- return nullptr;
+ return ggml_vk_float_pipeline(ctx->device->pipeline_diag, src0, dst);
case GGML_OP_CLAMP:
- if (src0->type == dst->type &&
- (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {
- return ctx->device->pipeline_clamp[dst->type == GGML_TYPE_F16];
- }
- return nullptr;
+ return ggml_vk_float_pipeline(ctx->device->pipeline_clamp, src0, dst);
case GGML_OP_PAD:
if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
return ctx->device->pipeline_pad_f32;
@@ -9626,7 +9655,10 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
if (ctx->do_add_rms_partials) {
return ctx->fused_rms_norm_mode == RMS_NORM_MUL ? ctx->device->pipeline_rms_norm_mul_partials_f32 : ctx->device->pipeline_rms_norm_partials_f32;
}
- return ggml_vk_rms_norm_pipeline(ctx->fused_rms_norm_mode == RMS_NORM_MUL ? ctx->device->pipeline_rms_norm_mul_f32 : ctx->device->pipeline_rms_norm_f32, src0->ne[0]);
+ return ggml_vk_rms_norm_pipeline(ctx->device->pipeline_rms_norm[0][ctx->fused_rms_norm_mode == RMS_NORM_MUL], src0->ne[0]);
+ }
+ if (src0->type == GGML_TYPE_BF16 && dst->type == GGML_TYPE_BF16 && !ctx->do_add_rms_partials) {
+ return ggml_vk_rms_norm_pipeline(ctx->device->pipeline_rms_norm[1][ctx->fused_rms_norm_mode == RMS_NORM_MUL], src0->ne[0]);
}
return nullptr;
case GGML_OP_RMS_NORM_BACK:
@@ -9640,87 +9672,9 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
}
return nullptr;
case GGML_OP_UNARY:
- if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) ||
- (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) ||
- (src0->type != dst->type)) {
- return nullptr;
- }
-
- switch (ggml_get_unary_op(dst)) {
- case GGML_UNARY_OP_EXP:
- return ctx->device->pipeline_exp[dst->type == GGML_TYPE_F16];
- case GGML_UNARY_OP_EXPM1:
- return ctx->device->pipeline_expm1[dst->type == GGML_TYPE_F16];
- case GGML_UNARY_OP_ELU:
- return ctx->device->pipeline_elu[dst->type == GGML_TYPE_F16];
- case GGML_UNARY_OP_SILU:
- return ctx->device->pipeline_silu[dst->type == GGML_TYPE_F16];
- case GGML_UNARY_OP_GELU:
- return ctx->device->pipeline_gelu[dst->type == GGML_TYPE_F16];
- case GGML_UNARY_OP_GELU_ERF:
- return ctx->device->pipeline_gelu_erf[dst->type == GGML_TYPE_F16];
- case GGML_UNARY_OP_GELU_QUICK:
- return ctx->device->pipeline_gelu_quick[dst->type == GGML_TYPE_F16];
- case GGML_UNARY_OP_RELU:
- return ctx->device->pipeline_relu[dst->type == GGML_TYPE_F16];
- case GGML_UNARY_OP_XIELU:
- return ctx->device->pipeline_xielu[dst->type == GGML_TYPE_F16];
- case GGML_UNARY_OP_NEG:
- return ctx->device->pipeline_neg[dst->type == GGML_TYPE_F16];
- case GGML_UNARY_OP_TANH:
- return ctx->device->pipeline_tanh[dst->type == GGML_TYPE_F16];
- case GGML_UNARY_OP_SIGMOID:
- return ctx->device->pipeline_sigmoid[dst->type == GGML_TYPE_F16];
- case GGML_UNARY_OP_HARDSIGMOID:
- return ctx->device->pipeline_hardsigmoid[dst->type == GGML_TYPE_F16];
- case GGML_UNARY_OP_HARDSWISH:
- return ctx->device->pipeline_hardswish[dst->type == GGML_TYPE_F16];
- case GGML_UNARY_OP_ABS:
- return ctx->device->pipeline_abs[dst->type == GGML_TYPE_F16];
- case GGML_UNARY_OP_SOFTPLUS:
- return ctx->device->pipeline_softplus[dst->type == GGML_TYPE_F16];
- case GGML_UNARY_OP_STEP:
- return ctx->device->pipeline_step[dst->type == GGML_TYPE_F16];
- case GGML_UNARY_OP_ROUND:
- return ctx->device->pipeline_round[dst->type == GGML_TYPE_F16];
- case GGML_UNARY_OP_CEIL:
- return ctx->device->pipeline_ceil[dst->type == GGML_TYPE_F16];
- case GGML_UNARY_OP_FLOOR:
- return ctx->device->pipeline_floor[dst->type == GGML_TYPE_F16];
- case GGML_UNARY_OP_TRUNC:
- return ctx->device->pipeline_trunc[dst->type == GGML_TYPE_F16];
- case GGML_UNARY_OP_SGN:
- return ctx->device->pipeline_sgn[dst->type == GGML_TYPE_F16];
- default:
- break;
- }
- return nullptr;
+ return ggml_vk_float_pipeline(ctx->device->pipeline_unary[ggml_get_unary_op(dst)], src0, dst);
case GGML_OP_GLU:
- if ((src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) ||
- (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16) ||
- (src0->type != dst->type)) {
- return nullptr;
- }
-
- switch (ggml_get_glu_op(dst)) {
- case GGML_GLU_OP_GEGLU:
- return ctx->device->pipeline_geglu[dst->type == GGML_TYPE_F16];
- case GGML_GLU_OP_REGLU:
- return ctx->device->pipeline_reglu[dst->type == GGML_TYPE_F16];
- case GGML_GLU_OP_SWIGLU:
- return ctx->device->pipeline_swiglu[dst->type == GGML_TYPE_F16];
- case GGML_GLU_OP_SWIGLU_OAI:
- return ctx->device->pipeline_swiglu_oai[dst->type == GGML_TYPE_F16];
- case GGML_GLU_OP_SWIGLU_CLAMP:
- return ctx->device->pipeline_swiglu_clamp[dst->type == GGML_TYPE_F16];
- case GGML_GLU_OP_GEGLU_ERF:
- return ctx->device->pipeline_geglu_erf[dst->type == GGML_TYPE_F16];
- case GGML_GLU_OP_GEGLU_QUICK:
- return ctx->device->pipeline_geglu_quick[dst->type == GGML_TYPE_F16];
- default:
- break;
- }
- return nullptr;
+ return ggml_vk_float_pipeline(ctx->device->pipeline_glu[ggml_get_glu_op(dst)], src0, dst);
case GGML_OP_SOFT_MAX:
GGML_ASSERT(!src1 || src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_F16);
GGML_ASSERT(!src2 || src2->type == GGML_TYPE_F32);
@@ -9965,11 +9919,7 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
}
return nullptr;
case GGML_OP_LEAKY_RELU:
- if (src0->type == dst->type &&
- (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16)) {
- return ctx->device->pipeline_leaky_relu[dst->type == GGML_TYPE_F16];
- }
- return nullptr;
+ return ggml_vk_float_pipeline(ctx->device->pipeline_leaky_relu, src0, dst);
case GGML_OP_CONV_2D:
case GGML_OP_CONV_TRANSPOSE_2D:
if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
@@ -10805,13 +10755,12 @@ void ggml_vk_unary_mul(ggml_backend_vk_context * ctx, vk_context& subctx, const
const ggml_tensor * src1 = op_on_b ? unary->src[0] :
((mul->src[0] == unary) ? mul->src[1] : mul->src[0]);
- const bool f16 = src0->type == GGML_TYPE_F16;
const bool norepeat = ggml_are_same_shape(src0, src1);
const int oi = ggml_vk_unary_mul_op_index(ggml_get_unary_op(unary));
if (oi < 0) {
GGML_ABORT("fatal error");
}
- vk_pipeline pipeline = ctx->device->pipeline_unary_mul[oi][f16][norepeat][op_on_b];
+ vk_pipeline pipeline = ctx->device->pipeline_unary_mul[oi][ggml_vk_float_type_idx(src0->type)][norepeat][op_on_b];
const uint32_t src0_type_size = ggml_type_size(src0->type);
const uint32_t src1_type_size = ggml_type_size(src1->type);
@@ -11592,11 +11541,13 @@ void ggml_vk_rms_norm(ggml_backend_vk_context * ctx, vk_context& subctx, const s
vk_pipeline pipeline;
if (ctx->do_add_rms_partials) {
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
pipeline = do_post_multiply ?
ctx->device->pipeline_rms_norm_mul_add_mul_partials_f32 : ctx->device->pipeline_rms_norm_mul_add_partials_f32;
} else {
- pipeline = ggml_vk_rms_norm_pipeline(do_post_multiply ?
- ctx->device->pipeline_rms_norm_mul_add_mul_f32 : ctx->device->pipeline_rms_norm_mul_add_f32, src0->ne[0]);
+ const bool src0_bf16 = src0->type == GGML_TYPE_BF16;
+ const bool residual_bf16 = residual->type == GGML_TYPE_BF16;
+ pipeline = ggml_vk_rms_norm_pipeline(ctx->device->pipeline_rms_norm_mul_add[src0_bf16][residual_bf16][do_post_multiply], src0->ne[0]);
}
ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1);
if (ctx->do_add_rms_partials) {
@@ -12994,6 +12945,8 @@ bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, in
if (next_node_idx < cgraph->n_nodes &&
cgraph->nodes[next_node_idx]->op == GGML_OP_RMS_NORM &&
cgraph->nodes[next_node_idx]->src[0] == cgraph->nodes[next_node_idx - 1] &&
+ cgraph->nodes[next_node_idx]->type == GGML_TYPE_F32 &&
+ node->type == GGML_TYPE_F32 &&
ggml_nrows(cgraph->nodes[next_node_idx]) == 1 &&
ctx->device->add_rms_fusion && !ctx->device->disable_fusion) {
uint32_t size = ggml_vk_rms_partials_size(ctx, cgraph->nodes[node_idx]);
@@ -14292,7 +14245,7 @@ static bool ggml_vk_can_fuse_unary_mul(const struct ggml_cgraph * cgraph, int un
if (ggml_vk_unary_mul_op_index(ggml_get_unary_op(unary)) < 0) {
return false;
}
- if (unary->type != GGML_TYPE_F32 && unary->type != GGML_TYPE_F16) {
+ if (unary->type != GGML_TYPE_F32 && unary->type != GGML_TYPE_F16 && unary->type != GGML_TYPE_BF16) {
return false;
}
if (unary->type != mul->type) {
@@ -14350,12 +14303,24 @@ bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgr
const ggml_tensor *rms_norm = cgraph->nodes[node_idx];
const ggml_tensor *mul = cgraph->nodes[node_idx + 1];
- GGML_ASSERT(rms_norm->src[0]->type == GGML_TYPE_F32);
- GGML_ASSERT(rms_norm->type == GGML_TYPE_F32);
- // rms_norm only supports f32
- if (mul->src[0]->type != GGML_TYPE_F32 ||
- mul->src[1]->type != GGML_TYPE_F32 ||
- mul->type != GGML_TYPE_F32) {
+ const ggml_type t = rms_norm->type;
+ if (t == GGML_TYPE_F32) {
+ if (mul->src[0]->type != GGML_TYPE_F32 ||
+ mul->src[1]->type != GGML_TYPE_F32 ||
+ mul->type != GGML_TYPE_F32) {
+ return false;
+ }
+ } else if (t == GGML_TYPE_BF16) {
+ // bf16 activations, f32 weights
+ if (mul->src[0] != rms_norm ||
+ mul->src[1]->type != GGML_TYPE_F32 ||
+ mul->type != GGML_TYPE_BF16) {
+ return false;
+ }
+ if (ops.size() >= 3 && ops.begin()[2] != GGML_OP_ADD) {
+ return false;
+ }
+ } else {
return false;
}
// if rms_norm is the B operand, then we don't handle broadcast
@@ -14374,7 +14339,7 @@ bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgr
if (add->src[0] != mul && add->src[1] != mul) {
return false;
}
- if (residual->type != GGML_TYPE_F32 || add->type != GGML_TYPE_F32 ||
+ if ((residual->type != GGML_TYPE_F32 && residual->type != t) || add->type != t ||
!ggml_are_same_shape(add, residual) || !ggml_is_contiguous(residual) ||
!ggml_is_contiguous(add) || get_misalign_bytes(ctx, residual) != 0) {
return false;
@@ -14392,7 +14357,7 @@ bool ggml_vk_can_fuse(const ggml_backend_vk_context * ctx, const struct ggml_cgr
return false;
}
// The shader reads data_e[0], so the final multiply must use a scalar.
- if (scale->type != GGML_TYPE_F32 || post_mul->type != GGML_TYPE_F32 ||
+ if (scale->type != GGML_TYPE_F32 || post_mul->type != t ||
ggml_nelements(scale) != 1 || !ggml_is_contiguous(post_mul) ||
get_misalign_bytes(ctx, scale) != 0) {
return false;
@@ -14933,6 +14898,10 @@ bool ggml_vk_can_fuse_rms_norm_mul_rope(ggml_backend_vk_context * ctx, const str
const int mode = ((const int32_t *) rope->op_params)[2];
+ if (rms->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32) {
+ return false;
+ }
+
// noncontig tensors aren't tested, and don't seem common in practice
if (!ggml_is_contiguous(rms) ||
!ggml_is_contiguous(mul) ||
@@ -16257,9 +16226,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
case GGML_UNARY_OP_FLOOR:
case GGML_UNARY_OP_TRUNC:
case GGML_UNARY_OP_SGN:
- return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&
- (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) &&
- (op->src[0]->type == op->type);
+ return ggml_vk_is_float_elementwise(op);
default:
return false;
}
@@ -16272,10 +16239,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
case GGML_GLU_OP_GEGLU_ERF:
case GGML_GLU_OP_GEGLU_QUICK:
case GGML_GLU_OP_SWIGLU_CLAMP:
- return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&
- (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) &&
- (op->src[0]->type == op->type) &&
- (!op->src[1] || op->src[1]->type == op->src[0]->type);
+ return ggml_vk_is_float_elementwise(op) && (!op->src[1] || op->src[1]->type == op->type);
default:
return false;
}
@@ -16548,16 +16512,15 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
case GGML_OP_GROUP_NORM:
return ggml_is_contiguous(op->src[0]);
case GGML_OP_NORM:
- case GGML_OP_RMS_NORM:
case GGML_OP_L2_NORM:
return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32;
+ case GGML_OP_RMS_NORM:
+ return ggml_vk_is_float_elementwise(op) && op->type != GGML_TYPE_F16;
case GGML_OP_ADD:
case GGML_OP_SUB:
case GGML_OP_MUL:
case GGML_OP_DIV:
- return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&
- (op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_F16) &&
- (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16);
+ return ggml_vk_binary_types_supported(op->src[0]->type, op->src[1]->type, op->type);
case GGML_OP_ADD_ID:
return op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->src[2]->type == GGML_TYPE_I32 &&
op->type == GGML_TYPE_F32;
@@ -16568,10 +16531,12 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
case GGML_OP_SQRT:
case GGML_OP_SIN:
case GGML_OP_COS:
+ case GGML_OP_LOG:
case GGML_OP_CLAMP:
case GGML_OP_LEAKY_RELU:
- return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&
- op->type == op->src[0]->type;
+ case GGML_OP_TRI:
+ case GGML_OP_DIAG:
+ return ggml_vk_is_float_elementwise(op);
case GGML_OP_OPT_STEP_ADAMW:
case GGML_OP_OPT_STEP_SGD:
return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32;
@@ -16579,11 +16544,6 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32
&& ggml_is_contiguous(op->src[1]) && op->src[1]->type == GGML_TYPE_F32
&& op->type == GGML_TYPE_F32;
- case GGML_OP_LOG:
- case GGML_OP_TRI:
- case GGML_OP_DIAG:
- return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&
- op->type == op->src[0]->type;
case GGML_OP_ARGSORT:
{
if (!ggml_is_contiguous(op) || !ggml_is_contiguous(op->src[0])) {
@@ -16629,7 +16589,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
case GGML_OP_FILL:
return op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16;
case GGML_OP_SCALE:
- return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32;
+ return ggml_is_contiguous(op->src[0]) && ggml_vk_is_float_elementwise(op);
case GGML_OP_PAD:
case GGML_OP_PAD_REFLECT_1D:
case GGML_OP_ROLL:
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/add.comp b/ggml/src/ggml-vulkan/vulkan-shaders/add.comp
index 3bcfe6908..2d93792ae 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/add.comp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/add.comp
@@ -36,10 +36,10 @@ void main() {
uint i00, i01, i02, i03;
get_indices(idx, i00, i01, i02, i03);
- FLOAT_TYPE sum = FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) + FLOAT_TYPE(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]);
+ FLOAT_TYPE sum = A_TO_FLOAT(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) + B_TO_FLOAT(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]);
sum_sq += sum*sum;
- data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = D_TYPE(sum);
+ data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = FLOAT_TO_D(sum);
idx += num_threads;
}
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp b/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp
index 249e6b16e..790a5421d 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/diag.comp
@@ -20,8 +20,7 @@ void main() {
const uint i10 = idx - i13_offset - i12_offset - i11*p.ne10;
if (i10 == i11) {
- const float val = float(data_a[get_aoffset() + i13*p.nb03 + i12*p.nb02 + 0*p.nb01 + i10*p.nb00]);
- data_d[get_doffset() + dst_idx(idx)] = D_TYPE(val);
+ data_d[get_doffset() + dst_idx(idx)] = D_TYPE(data_a[get_aoffset() + i13*p.nb03 + i12*p.nb02 + 0*p.nb01 + i10*p.nb00]);
} else {
data_d[get_doffset() + dst_idx(idx)] = D_TYPE(0);
}
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/div.comp b/ggml/src/ggml-vulkan/vulkan-shaders/div.comp
index 572472f8a..781f2004d 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/div.comp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/div.comp
@@ -20,7 +20,7 @@ void main() {
uint i00, i01, i02, i03;
get_indices(idx, i00, i01, i02, i03);
- data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) / FLOAT_TYPE(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]));
+ data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = FLOAT_TO_D(A_TO_FLOAT(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) / B_TO_FLOAT(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]));
idx += num_threads;
}
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl
index fc2951ec2..aed5a65a6 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/glu_head.glsl
@@ -1,5 +1,6 @@
#extension GL_EXT_shader_16bit_storage : require
+#include "types.glsl"
#include "utils.glsl"
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl
index 14c5e7a54..327191d51 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/glu_main.glsl
@@ -21,15 +21,15 @@ void main() {
const uint offset = (p.ne00 / 2) * p.nb00;
const uint idx = src_idx_a;
- data_d[dst_idx] = D_TYPE(op(float(data_a[idx]), float(data_a[idx + offset])));
+ data_d[dst_idx] = FLOAT_TO_D(op(A_TO_FLOAT(data_a[idx]), A_TO_FLOAT(data_a[idx + offset])));
} else if (p.mode == 1) {
// Swapped
const uint offset = (p.ne00 / 2) * p.nb00;
const uint idx = src_idx_a;
- data_d[dst_idx] = D_TYPE(op(float(data_a[idx + offset]), float(data_a[idx])));
+ data_d[dst_idx] = FLOAT_TO_D(op(A_TO_FLOAT(data_a[idx + offset]), A_TO_FLOAT(data_a[idx])));
} else {
// Split
- data_d[dst_idx] = D_TYPE(op(float(data_a[src_idx_a]), float(data_b[src_idx_b])));
+ data_d[dst_idx] = FLOAT_TO_D(op(A_TO_FLOAT(data_a[src_idx_a]), A_TO_FLOAT(data_b[src_idx_b])));
}
}
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/log.comp b/ggml/src/ggml-vulkan/vulkan-shaders/log.comp
index 3cda6a63c..7eb1e7fd8 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/log.comp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/log.comp
@@ -12,6 +12,6 @@ void main() {
return;
}
- const float val = float(data_a[get_aoffset() + src0_idx(idx)]);
- data_d[get_doffset() + dst_idx(idx)] = D_TYPE(log(val));
+ const float val = A_TO_FLOAT(data_a[get_aoffset() + src0_idx(idx)]);
+ data_d[get_doffset() + dst_idx(idx)] = FLOAT_TO_D(log(val));
}
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp
index 02ef1eace..eca78e33e 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp
@@ -20,7 +20,7 @@ void main() {
uint i00, i01, i02, i03;
get_indices(idx, i00, i01, i02, i03);
- data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) * FLOAT_TYPE(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]));
+ data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = FLOAT_TO_D(A_TO_FLOAT(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) * B_TO_FLOAT(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]));
idx += num_threads;
}
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp
index 31e4483b2..b9f0a1841 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp
@@ -41,7 +41,7 @@ layout (binding = 6) readonly buffer R_I {uvec2 rope_data_i[];}; // indices for
#elif RMS_NORM_ADD_FUSION
-layout (binding = 3) readonly buffer C {float data_c[];};
+layout (binding = 3) readonly buffer C {C_TYPE data_c[];};
layout (binding = 4) readonly buffer E {float data_e[];};
#elif RMS_NORM_SET_ROWS_FUSION
@@ -90,7 +90,7 @@ void rms_norm(uint num_iters) {
[[unroll]] for (uint col = tid, idx = 0; idx < num_iters; col += BLOCK_SIZE, ++idx) {
FLOAT_TYPE xi = FLOAT_TYPE(0);
if (col < ncols) {
- xi = FLOAT_TYPE(data_a[a_offset + col]);
+ xi = FLOAT_TYPE(A_TO_FLOAT(data_a[a_offset + col]));
}
sum += xi * xi;
}
@@ -137,28 +137,28 @@ void rms_norm(uint num_iters) {
if (col >= ncols) {
continue;
}
- FLOAT_TYPE value = scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + fastmod(col, p.ne10)]);
+ FLOAT_TYPE value = scale * FLOAT_TYPE(A_TO_FLOAT(data_a[a_offset + col])) * FLOAT_TYPE(B_TO_FLOAT(data_b[b_offset + fastmod(col, p.ne10)]));
#if RMS_NORM_ADD_FUSION
- value += FLOAT_TYPE(data_c[d_offset + col]);
+ value += FLOAT_TYPE(C_TO_FLOAT(data_c[d_offset + col]));
if (do_post_multiply) {
value *= FLOAT_TYPE(data_e[0]);
}
#endif
- data_d[d_offset + col] = D_TYPE(value);
+ data_d[d_offset + col] = FLOAT_TO_D(value);
}
} else {
[[unroll]] for (uint col = tid, idx = 0; idx < num_iters; col += BLOCK_SIZE, ++idx) {
if (col >= ncols) {
continue;
}
- FLOAT_TYPE value = scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + col]);
+ FLOAT_TYPE value = scale * FLOAT_TYPE(A_TO_FLOAT(data_a[a_offset + col])) * FLOAT_TYPE(B_TO_FLOAT(data_b[b_offset + col]));
#if RMS_NORM_ADD_FUSION
- value += FLOAT_TYPE(data_c[d_offset + col]);
+ value += FLOAT_TYPE(C_TO_FLOAT(data_c[d_offset + col]));
if (do_post_multiply) {
value *= FLOAT_TYPE(data_e[0]);
}
#endif
- data_d[d_offset + col] = D_TYPE(value);
+ data_d[d_offset + col] = FLOAT_TO_D(value);
}
}
} else {
@@ -166,7 +166,7 @@ void rms_norm(uint num_iters) {
if (col >= ncols) {
continue;
}
- data_d[d_offset + col] = D_TYPE(scale * FLOAT_TYPE(data_a[a_offset + col]));
+ data_d[d_offset + col] = FLOAT_TO_D(scale * FLOAT_TYPE(A_TO_FLOAT(data_a[a_offset + col])));
}
}
#if RMS_NORM_ROPE_FUSION
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp b/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp
index 35ec726a0..83ee31a72 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/scale.comp
@@ -18,7 +18,7 @@ void main() {
continue;
}
- data_d[get_doffset() + idx] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + idx]) * FLOAT_TYPE(p.param1) + FLOAT_TYPE(p.param2));
+ data_d[get_doffset() + idx] = FLOAT_TO_D(A_TO_FLOAT(data_a[get_aoffset() + idx]) * FLOAT_TYPE(p.param1) + FLOAT_TYPE(p.param2));
idx += num_threads;
}
}
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp b/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp
index bc924b520..ebecb2c0f 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/sub.comp
@@ -22,7 +22,7 @@ void main() {
uint i00, i01, i02, i03;
get_indices(idx, i00, i01, i02, i03);
- data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) - FLOAT_TYPE(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]));
+ data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = FLOAT_TO_D(A_TO_FLOAT(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) - B_TO_FLOAT(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]));
idx += num_threads;
}
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp b/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp
index 9def5dbc9..2585540af 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/tri.comp
@@ -34,8 +34,7 @@ void main() {
}
if (pass) {
- const float val = float(data_a[get_aoffset() + src0_idx(idx)]);
- data_d[get_doffset() + dst_idx(idx)] = D_TYPE(val);
+ data_d[get_doffset() + dst_idx(idx)] = D_TYPE(data_a[get_aoffset() + src0_idx(idx)]);
} else {
data_d[get_doffset() + dst_idx(idx)] = D_TYPE(0);
}
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl
index 99ce1458a..b7e5a31aa 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl
@@ -1976,6 +1976,27 @@ vec4 bf16_to_fp32(uvec4 u)
return vec4(bf16_to_fp32(u.x), bf16_to_fp32(u.y), bf16_to_fp32(u.z), bf16_to_fp32(u.w));
}
+#if defined(DATA_A_BF16)
+#define A_TO_FLOAT(x) bf16_to_fp32(uint32_t(x))
+#else
+#define A_TO_FLOAT(x) float(x)
+#endif
+#if defined(DATA_B_BF16)
+#define B_TO_FLOAT(x) bf16_to_fp32(uint32_t(x))
+#else
+#define B_TO_FLOAT(x) float(x)
+#endif
+#if defined(DATA_C_BF16)
+#define C_TO_FLOAT(x) bf16_to_fp32(uint32_t(x))
+#else
+#define C_TO_FLOAT(x) float(x)
+#endif
+#if defined(DATA_D_BF16)
+#define FLOAT_TO_D(x) D_TYPE(fp32_to_bf16(x))
+#else
+#define FLOAT_TO_D(x) D_TYPE(x)
+#endif
+
float e8m0_to_fp32(uint8_t x) {
uint32_t bits;
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp b/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp
index 9ee7769ba..04a3756bd 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/unary.comp
@@ -184,10 +184,10 @@ void main() {
if (op_on_b) {
data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] =
- D_TYPE(FLOAT_TYPE(OP(float(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]))) * FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]));
+ FLOAT_TO_D(OP(B_TO_FLOAT(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)])) * A_TO_FLOAT(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]));
} else {
data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] =
- D_TYPE(FLOAT_TYPE(OP(float(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]))) * FLOAT_TYPE(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]));
+ FLOAT_TO_D(OP(A_TO_FLOAT(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)])) * B_TO_FLOAT(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]));
}
idx += num_threads;
@@ -200,6 +200,6 @@ void main() {
const uint a_idx = get_aoffset() + src0_idx(idx);
const uint d_idx = get_doffset() + dst_idx(idx);
- data_d[d_idx] = D_TYPE(OP(float(data_a[a_idx])));
+ data_d[d_idx] = FLOAT_TO_D(OP(A_TO_FLOAT(data_a[a_idx])));
#endif
}
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
index 3952030fb..86b69d15f 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp
@@ -863,9 +863,13 @@ void process_shaders() {
string_to_spv("group_norm_f32", "group_norm.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}}));
for (bool use_subgroups : {false, true}) {
const std::string suffix = use_subgroups ? "_subgroup" : "";
- const auto rms_dict = merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"USE_SUBGROUP_ADD", use_subgroups ? "1" : "0"}});
+ const auto rms_dict = merge_maps(base_dict, {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"C_TYPE", "float"}, {"D_TYPE", "float"}, {"USE_SUBGROUP_ADD", use_subgroups ? "1" : "0"}});
+ const auto rms_bf16_dict = merge_maps(base_dict, {{"A_TYPE", "uint16_t"}, {"B_TYPE", "float"}, {"C_TYPE", "float"}, {"D_TYPE", "uint16_t"}, {"DATA_A_BF16", "1"}, {"DATA_D_BF16", "1"}, {"USE_SUBGROUP_ADD", use_subgroups ? "1" : "0"}});
string_to_spv("rms_norm_f32" + suffix, "rms_norm.comp", rms_dict);
+ string_to_spv("rms_norm_bf16" + suffix, "rms_norm.comp", rms_bf16_dict);
string_to_spv("rms_norm_mul_add_f32" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"RMS_NORM_ADD_FUSION", "1"}}));
+ string_to_spv("rms_norm_mul_add_bf16_f32" + suffix, "rms_norm.comp", merge_maps(rms_bf16_dict, {{"RMS_NORM_ADD_FUSION", "1"}}));
+ string_to_spv("rms_norm_mul_add_bf16_bf16" + suffix, "rms_norm.comp", merge_maps({{"C_TYPE", "uint16_t"}, {"DATA_C_BF16", "1"}, {"RMS_NORM_ADD_FUSION", "1"}}, rms_bf16_dict));
string_to_spv("rms_norm_set_rows_f32_f32" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"RMS_NORM_SET_ROWS_FUSION", "1"}}));
string_to_spv("rms_norm_set_rows_f32_f16" + suffix, "rms_norm.comp", merge_maps({{"D_TYPE", "float16_t"}, {"RMS_NORM_SET_ROWS_FUSION", "1"}}, rms_dict)); // keep rms_dict last so 'merge_maps' overwrites the value for key 'D_TYPE'
string_to_spv("rms_norm_mul_rope_f32_f32" + suffix, "rms_norm.comp", merge_maps(rms_dict, {{"ROPE_D_TYPE", "float"}, {"RMS_NORM_ROPE_FUSION", "1"}}));
@@ -934,6 +938,10 @@ void process_shaders() {
}
}
}
+ for (std::string op : {"add", "sub", "mul", "div"}) {
+ string_to_spv(op + "_bf16_f32_bf16", op + ".comp", {{"A_TYPE", "uint16_t"}, {"B_TYPE", "float"}, {"D_TYPE", "uint16_t"}, {"DATA_A_BF16", "1"}, {"DATA_D_BF16", "1"}, {"FLOAT_TYPE", "float"}, {"ADD_RMS" , "0"}});
+ string_to_spv(op + "_bf16_bf16_bf16", op + ".comp", {{"A_TYPE", "uint16_t"}, {"B_TYPE", "uint16_t"}, {"D_TYPE", "uint16_t"}, {"DATA_A_BF16", "1"}, {"DATA_B_BF16", "1"}, {"DATA_D_BF16", "1"}, {"FLOAT_TYPE", "float"}, {"ADD_RMS" , "0"}});
+ }
string_to_spv("sub_f32", "sub.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
@@ -969,8 +977,6 @@ void process_shaders() {
string_to_spv("repeat_i16", "repeat.comp", {{"A_TYPE", "int16_t"}, {"D_TYPE", "int16_t"}});
- string_to_spv("scale_f32", "scale.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
-
string_to_spv("pad_f32", "pad.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("pad_reflect_1d_f32", "pad_reflect_1d.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
@@ -984,70 +990,36 @@ void process_shaders() {
string_to_spv("upscale_f32", "upscale.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}});
- string_to_spv("exp_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_exp"}});
- string_to_spv("exp_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_exp"}});
- string_to_spv("expm1_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_expm1"}});
- string_to_spv("expm1_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_expm1"}});
-
- string_to_spv("log_f16", "log.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
- string_to_spv("log_f32", "log.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
- string_to_spv("gelu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_gelu"}});
- string_to_spv("gelu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_gelu"}});
- string_to_spv("gelu_erf_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_gelu_erf"}});
- string_to_spv("gelu_erf_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_gelu_erf"}});
- string_to_spv("gelu_quick_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_gelu_quick"}});
- string_to_spv("gelu_quick_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_gelu_quick"}});
- string_to_spv("silu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_silu"}});
- string_to_spv("silu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_silu"}});
- string_to_spv("relu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_relu"}});
- string_to_spv("relu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_relu"}});
- string_to_spv("sqr_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_sqr"}});
- string_to_spv("sqr_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_sqr"}});
- string_to_spv("sqrt_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_sqrt"}});
- string_to_spv("sqrt_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_sqrt"}});
- string_to_spv("sin_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_sin"}});
- string_to_spv("sin_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_sin"}});
- string_to_spv("cos_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_cos"}});
- string_to_spv("cos_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_cos"}});
- string_to_spv("clamp_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_clamp"}});
- string_to_spv("clamp_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_clamp"}});
- string_to_spv("leaky_relu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_leaky_relu"}});
- string_to_spv("leaky_relu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_leaky_relu"}});
- string_to_spv("neg_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_neg"}});
- string_to_spv("neg_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_neg"}});
- string_to_spv("tanh_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_tanh"}});
- string_to_spv("tanh_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_tanh"}});
- string_to_spv("sigmoid_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_sigmoid"}});
- string_to_spv("sigmoid_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_sigmoid"}});
- string_to_spv("hardsigmoid_f16","unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_hardsigmoid"}});
- string_to_spv("hardsigmoid_f32","unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_hardsigmoid"}});
- string_to_spv("hardswish_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_hardswish"}});
- string_to_spv("hardswish_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_hardswish"}});
- string_to_spv("abs_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_abs"}});
- string_to_spv("abs_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_abs"}});
- string_to_spv("elu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_elu"}});
- string_to_spv("elu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_elu"}});
- string_to_spv("xielu_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_xielu"}});
- string_to_spv("xielu_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_xielu"}});
- string_to_spv("sgn_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_sgn"}});
- string_to_spv("sgn_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_sgn"}});
-
- string_to_spv("tri_f16", "tri.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
- string_to_spv("tri_f32", "tri.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
- string_to_spv("diag_f16", "diag.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
- string_to_spv("diag_f32", "diag.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
-
- string_to_spv("softplus_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_softplus"}});
- string_to_spv("softplus_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_softplus"}});
-
- string_to_spv("gelu_mul_f32", "unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_gelu"}, {"UNARY_MUL_FUSION", "1"}});
- string_to_spv("gelu_mul_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_gelu"}, {"UNARY_MUL_FUSION", "1"}});
- string_to_spv("sigmoid_mul_f32", "unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_sigmoid"}, {"UNARY_MUL_FUSION", "1"}});
- string_to_spv("sigmoid_mul_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_sigmoid"}, {"UNARY_MUL_FUSION", "1"}});
- string_to_spv("silu_mul_f32", "unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_silu"}, {"UNARY_MUL_FUSION", "1"}});
- string_to_spv("silu_mul_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_silu"}, {"UNARY_MUL_FUSION", "1"}});
- string_to_spv("softplus_mul_f32","unary.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}, {"OP", "op_softplus"}, {"UNARY_MUL_FUSION", "1"}});
- string_to_spv("softplus_mul_f16","unary.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}, {"OP", "op_softplus"}, {"UNARY_MUL_FUSION", "1"}});
+ const std::vector<std::pair<std::string, std::map<std::string, std::string>>> float_types = {
+ {"f32", {{"A_TYPE", "float"}, {"D_TYPE", "float"}}},
+ {"f16", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}}},
+ {"bf16", {{"A_TYPE", "uint16_t"}, {"D_TYPE", "uint16_t"}, {"DATA_A_BF16", "1"}, {"DATA_D_BF16", "1"}}},
+ };
+ for (const auto & [t, dict] : float_types) {
+ for (const std::string op : {"exp", "expm1", "gelu", "gelu_erf", "gelu_quick", "silu", "relu", "sqr", "sqrt", "sin",
+ "cos", "clamp", "leaky_relu", "neg", "tanh", "sigmoid", "hardsigmoid", "hardswish", "abs",
+ "elu", "xielu", "sgn", "softplus", "step", "round", "ceil", "floor", "trunc"}) {
+ string_to_spv(op + "_" + t, "unary.comp", merge_maps(dict, {{"OP", "op_" + op}}));
+ }
+ for (const std::string op : {"geglu", "reglu", "swiglu", "swiglu_oai", "swiglu_clamp", "geglu_erf", "geglu_quick"}) {
+ string_to_spv(op + "_" + t, op + ".comp", dict);
+ }
+ string_to_spv("log_" + t, "log.comp", dict);
+ string_to_spv("scale_" + t, "scale.comp", merge_maps(dict, {{"FLOAT_TYPE", "float"}}));
+
+ auto mul_dict = merge_maps(dict, {{"B_TYPE", dict.at("A_TYPE")}, {"FLOAT_TYPE", "float"}, {"UNARY_MUL_FUSION", "1"}});
+ if (t == "bf16") {
+ mul_dict["DATA_B_BF16"] = "1";
+ }
+ for (const std::string op : {"gelu", "sigmoid", "silu", "softplus"}) {
+ string_to_spv(op + "_mul_" + t, "unary.comp", merge_maps(mul_dict, {{"OP", "op_" + op}}));
+ }
+ }
+
+ string_to_spv("tri_16", "tri.comp", {{"A_TYPE", "uint16_t"}, {"D_TYPE", "uint16_t"}});
+ string_to_spv("tri_32", "tri.comp", {{"A_TYPE", "uint"}, {"D_TYPE", "uint"}});
+ string_to_spv("diag_16", "diag.comp", {{"A_TYPE", "uint16_t"}, {"D_TYPE", "uint16_t"}});
+ string_to_spv("diag_32", "diag.comp", {{"A_TYPE", "uint"}, {"D_TYPE", "uint"}});
string_to_spv("add1_f16_f16", "add1.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}});
string_to_spv("add1_f16_f32", "add1.comp", {{"A_TYPE", "float16_t"}, {"B_TYPE", "float"}, {"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}});
@@ -1055,31 +1027,6 @@ void process_shaders() {
string_to_spv("arange_f32", "arange.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
string_to_spv("fill_f32", "fill.comp", {{"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
string_to_spv("fill_f16", "fill.comp", {{"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}});
- string_to_spv("step_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_step"}});
- string_to_spv("step_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_step"}});
- string_to_spv("round_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_round"}});
- string_to_spv("round_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_round"}});
- string_to_spv("ceil_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_ceil"}});
- string_to_spv("ceil_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_ceil"}});
- string_to_spv("floor_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_floor"}});
- string_to_spv("floor_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_floor"}});
- string_to_spv("trunc_f16", "unary.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}, {"OP", "op_trunc"}});
- string_to_spv("trunc_f32", "unary.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"OP", "op_trunc"}});
-
- string_to_spv("geglu_f16", "geglu.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
- string_to_spv("geglu_f32", "geglu.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
- string_to_spv("reglu_f16", "reglu.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
- string_to_spv("reglu_f32", "reglu.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
- string_to_spv("swiglu_f16", "swiglu.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
- string_to_spv("swiglu_f32", "swiglu.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
- string_to_spv("swiglu_oai_f16", "swiglu_oai.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
- string_to_spv("swiglu_oai_f32", "swiglu_oai.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
- string_to_spv("swiglu_clamp_f16", "swiglu_clamp.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
- string_to_spv("swiglu_clamp_f32", "swiglu_clamp.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
- string_to_spv("geglu_erf_f16", "geglu_erf.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
- string_to_spv("geglu_erf_f32", "geglu_erf.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
- string_to_spv("geglu_quick_f16","geglu_quick.comp", {{"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}});
- string_to_spv("geglu_quick_f32","geglu_quick.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}});
string_to_spv("silu_back_f32", "silu_back.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}});
diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c
index 1fca258a5..7132abc51 100644
--- a/ggml/src/ggml.c
+++ b/ggml/src/ggml.c
@@ -5280,7 +5280,7 @@ struct ggml_tensor * ggml_tri(
struct ggml_context * ctx,
struct ggml_tensor * a,
enum ggml_tri_type type) {
- GGML_ASSERT(a->type == GGML_TYPE_F32);
+ GGML_ASSERT(a->type == GGML_TYPE_F32 || a->type == GGML_TYPE_F16 || a->type == GGML_TYPE_BF16);
GGML_ASSERT(ggml_is_contiguous(a));
GGML_ASSERT(a->ne[0] == a->ne[1]);
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index f2ede4290..cddc2dbaa 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -10770,7 +10770,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
}
}
- for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) {
+ for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16}) {
for (int v : {0, 1}) {
for (float alpha : {.5f, 1.702f}) {
for (float limit : {2.0f, 7.0f}) {
@@ -10780,7 +10780,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
}
}
- for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) {
+ for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32, GGML_TYPE_BF16}) {
for (int v : {0, 1}) {
for (float limit : {2.0f, 10.0f}) {
test_cases.emplace_back(new test_swiglu_clamp(type, { 128, 2, 2, 2 }, v, limit));
@@ -11574,6 +11574,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_scale(GGML_TYPE_BF16, {10, 10, 10, 10}, 2.0f, 1.0f));
test_cases.emplace_back(new test_scale(GGML_TYPE_BF16, {10, 10, 10, 10}, 2.0f, 1.0f, true)); // inplace test
test_cases.emplace_back(new test_scale(GGML_TYPE_BF16, {100, 10, 10, 10}, 0.5f, 0.0f));
+ test_cases.emplace_back(new test_scale(GGML_TYPE_F16, {10, 10, 10, 10}, 2.0f, 1.0f));
+ test_cases.emplace_back(new test_scale(GGML_TYPE_F16, {10, 10, 10, 10}, 2.0f, 1.0f, true)); // inplace test
+ test_cases.emplace_back(new test_scale(GGML_TYPE_F16, {100, 10, 10, 10}, 0.5f, 0.0f));
test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 0.0f, false, 1)); // view test
test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 1.0f, false, 1)); // view test
test_cases.emplace_back(new test_scale(GGML_TYPE_F32, {10, 10, 10, 10}, 2.0f, 1.0f, true, 1)); // view test, inplace
@@ -11684,6 +11687,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, { 1536, 1, 1, 1 }, 1e-6f, false, false, false, true));
test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_F32, { 256, 4, 1, 1 }, 1e-6f, false, false, false, true));
test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_BF16, { 1536, 1, 1, 1 }, 1e-6f, false, false, true, false, false, GGML_TYPE_F32));
+ test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_BF16, { 1536, 1, 1, 1 }, 1e-6f, false, false, true, false, false, GGML_TYPE_F32, GGML_TYPE_BF16));
// the ADD operand is the BF16 input itself, added in place
test_cases.emplace_back(new test_rms_norm_mul_add(GGML_TYPE_BF16, { 1536, 1, 1, 1 }, 1e-6f, false, false, false, true, false, GGML_TYPE_F32));
@@ -12471,7 +12475,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
}
}
- for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) {
+ for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_F32}) {
test_cases.emplace_back(new test_sqr (type));
test_cases.emplace_back(new test_sqrt (type));
test_cases.emplace_back(new test_log (type));
@@ -12946,10 +12950,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_xielu(GGML_TYPE_F16, { 512, 16, 1, 1 }));
test_cases.emplace_back(new test_xielu(GGML_TYPE_BF16, { 512, 16, 1, 1 }));
- test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_LOWER));
- test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_LOWER_DIAG));
- test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_UPPER));
- test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_UPPER_DIAG));
+ for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16}) {
+ test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_LOWER, type));
+ test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_LOWER_DIAG, type));
+ test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_UPPER, type));
+ test_cases.emplace_back(new test_tri(GGML_TRI_TYPE_UPPER_DIAG, type));
+ }
test_cases.emplace_back(new test_tri_special(GGML_TRI_TYPE_LOWER));
test_cases.emplace_back(new test_tri_special(GGML_TRI_TYPE_LOWER_DIAG));
test_cases.emplace_back(new test_tri_special(GGML_TRI_TYPE_UPPER));
@@ -12963,9 +12969,11 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_fill(-152.0f, GGML_TYPE_F16, { 800, 600, 4, 4 }));
test_cases.emplace_back(new test_fill(3.5f, GGML_TYPE_F16, { 2048, 512, 2, 2 }));
- test_cases.emplace_back(new test_diag());
- test_cases.emplace_back(new test_diag(GGML_TYPE_F32, { 79, 1, 19, 13 }));
- test_cases.emplace_back(new test_diag(GGML_TYPE_F32, { 256, 1, 8, 16 }));
+ for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16}) {
+ test_cases.emplace_back(new test_diag(type));
+ test_cases.emplace_back(new test_diag(type, { 79, 1, 19, 13 }));
+ test_cases.emplace_back(new test_diag(type, { 256, 1, 8, 16 }));
+ }
test_cases.emplace_back(new test_solve_tri());
test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 11, 11, 1, 1 }, { 5, 11, 1, 1 }));