Commit 7fce5439f for llama.cpp
commit 7fce5439fe451cd48340b112ec37d2befe02c776
Author: anujj <ajalota@nvidia.com>
Date: Thu Nov 19 23:43:55 2026 +0530
ggml : add BF16 support to ROPE and SET_ROWS (CPU, CUDA) (#32387)
* ggml : add BF16 support to ROPE and SET_ROWS (CPU, CUDA)
* cuda : SET_ROWS from BF16 into quantized types and F8_E4M3
* cuda : fuse BF16 rope into a BF16 cache
* tests : trim the BF16 rope cases to one per code path
diff --git a/ggml/src/ggml-cann/ggml-cann.cpp b/ggml/src/ggml-cann/ggml-cann.cpp
index 7e692f1b5..0adc8680f 100644
--- a/ggml/src/ggml-cann/ggml-cann.cpp
+++ b/ggml/src/ggml-cann/ggml-cann.cpp
@@ -2501,6 +2501,9 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten
break;
case GGML_OP_SET_ROWS:
{
+ if (op->src[0]->type == GGML_TYPE_BF16) {
+ return false;
+ }
switch (op->type) {
case GGML_TYPE_F32:
case GGML_TYPE_F16:
@@ -2553,6 +2556,9 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten
if (((const int32_t *) op->op_params)[15] != 0) {
return false; // FIXME: support ggml_rope_set_offset
}
+ if (op->src[0]->type != GGML_TYPE_F32 && op->src[0]->type != GGML_TYPE_F16) {
+ return false;
+ }
if (op->src[0]->ne[0] > 896) {
return false;
}
diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
index 8b4c7a4ef..4d9760bad 100644
--- a/ggml/src/ggml-cpu/ggml-cpu.c
+++ b/ggml/src/ggml-cpu/ggml-cpu.c
@@ -2933,7 +2933,8 @@ struct ggml_cplan ggml_graph_plan(
} break;
case GGML_OP_SET_ROWS:
{
- if (node->src[0]->type == GGML_TYPE_F16 && node->type != GGML_TYPE_F16) {
+ if ((node->src[0]->type == GGML_TYPE_F16 && node->type != GGML_TYPE_F16) ||
+ (node->src[0]->type == GGML_TYPE_BF16 && node->type != GGML_TYPE_BF16)) {
cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks;
}
} break;
diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
index 0b540c5a7..cd8134740 100644
--- a/ggml/src/ggml-cpu/ops.cpp
+++ b/ggml/src/ggml-cpu/ops.cpp
@@ -5531,7 +5531,7 @@ static void ggml_compute_forward_set_rows_impl(
assert(ne0 == nc);
assert(ne2 == ne02);
assert(ne3 == ne03);
- GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);
+ GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16);
assert(ne02 % ne11 == 0);
assert(ne03 % ne12 == 0);
@@ -5578,6 +5578,20 @@ static void ggml_compute_forward_set_rows_impl(
from_float(wdata,
((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), nc);
}
+ } else if constexpr (std::is_same_v<src_t, ggml_bf16_t>) {
+ if (dst->type == GGML_TYPE_BF16) {
+ memcpy(
+ ((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3),
+ ((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03),
+ rs);
+ } else {
+ float * wdata = (float *) params->wdata + (nc + CACHE_LINE_SIZE_F32) * ith;
+ ggml_bf16_to_fp32_row(
+ (const ggml_bf16_t *) ((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03),
+ wdata, nc);
+ from_float(wdata,
+ ((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), nc);
+ }
} else {
GGML_ABORT("src0->type = %d (%s) not supported", src0->type, ggml_type_name(src0->type));
}
@@ -5614,6 +5628,16 @@ void ggml_compute_forward_set_rows(
GGML_ABORT("src1->type = %d (%s) not supported", src1->type, ggml_type_name(src1->type));
}
} break;
+ case GGML_TYPE_BF16:
+ {
+ if (src1->type == GGML_TYPE_I64) {
+ ggml_compute_forward_set_rows_impl<ggml_bf16_t, int64_t>(params, dst);
+ } else if (src1->type == GGML_TYPE_I32) {
+ ggml_compute_forward_set_rows_impl<ggml_bf16_t, int32_t>(params, dst);
+ } else {
+ GGML_ABORT("src1->type = %d (%s) not supported", src1->type, ggml_type_name(src1->type));
+ }
+ } break;
default:
{
GGML_ABORT("src0->type = %d (%s) not supported", src0->type, ggml_type_name(src0->type));
@@ -6302,7 +6326,7 @@ static void rotate_pairs(const int64_t n, const int64_t n_offset, const float *
}
}
-template<typename T> //float or ggml_fp16_t
+template<typename T> //float, ggml_fp16_t or ggml_bf16_t
static void ggml_compute_forward_rope_flt(
const ggml_compute_params * params,
ggml_tensor * dst,
@@ -6312,7 +6336,7 @@ static void ggml_compute_forward_rope_flt(
const ggml_tensor * src1 = dst->src[1];
const ggml_tensor * src2 = dst->src[2];
- GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);
+ GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16);
GGML_ASSERT(src1->type == GGML_TYPE_I32);
float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;
@@ -6472,6 +6496,10 @@ void ggml_compute_forward_rope(
{
ggml_compute_forward_rope_flt<ggml_fp16_t>(params, dst, true);
} break;
+ case GGML_TYPE_BF16:
+ {
+ ggml_compute_forward_rope_flt<ggml_bf16_t>(params, dst, true);
+ } break;
case GGML_TYPE_F32:
{
ggml_compute_forward_rope_flt<float>(params, dst, true);
@@ -6496,6 +6524,10 @@ void ggml_compute_forward_rope_back(
{
ggml_compute_forward_rope_flt<ggml_fp16_t>(params, dst, false);
} break;
+ case GGML_TYPE_BF16:
+ {
+ ggml_compute_forward_rope_flt<ggml_bf16_t>(params, dst, false);
+ } break;
case GGML_TYPE_F32:
{
ggml_compute_forward_rope_flt<float>(params, dst, false);
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index 2ce3d0b80..a58f020c2 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -3504,7 +3504,12 @@ static bool ggml_cuda_should_fuse_rope_set_rows(const ggml_tensor * rope,
return false;
}
- if (set_rows->type != GGML_TYPE_F32 && set_rows->type != GGML_TYPE_F16) {
+ if (set_rows->type != GGML_TYPE_F32 && set_rows->type != GGML_TYPE_F16 && set_rows->type != GGML_TYPE_BF16) {
+ return false;
+ }
+
+ // BF16 rope fuses into an F16 or BF16 cache, other ropes not into a BF16 cache
+ if (rope->type == GGML_TYPE_BF16 ? set_rows->type == GGML_TYPE_F32 : set_rows->type == GGML_TYPE_BF16) {
return false;
}
@@ -6510,7 +6515,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
(op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_BF16 || op->type == GGML_TYPE_F8_E4M3 ||
op->type == GGML_TYPE_Q4_0 || op->type == GGML_TYPE_Q4_1 || op->type == GGML_TYPE_Q5_0 ||
op->type == GGML_TYPE_Q5_1 || op->type == GGML_TYPE_Q8_0 || op->type == GGML_TYPE_IQ4_NL) &&
- op->src[0]->type == GGML_TYPE_F32
+ (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16)
) || (
op->type == GGML_TYPE_F16 && op->src[0]->type == GGML_TYPE_F16
)
@@ -6731,7 +6736,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
return false;
case GGML_OP_ROPE:
case GGML_OP_ROPE_BACK: {
- return op->src[0]->nb[0] == ggml_type_size(op->src[0]->type) && ggml_is_contiguous_2(op->src[0]);
+ return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16) &&
+ op->type == op->src[0]->type &&
+ op->src[0]->nb[0] == ggml_type_size(op->src[0]->type) && ggml_is_contiguous_2(op->src[0]);
}
case GGML_OP_IM2COL:
case GGML_OP_IM2COL_3D:
diff --git a/ggml/src/ggml-cuda/rope.cu b/ggml/src/ggml-cuda/rope.cu
index 2604d014f..657b58a4d 100644
--- a/ggml/src/ggml-cuda/rope.cu
+++ b/ggml/src/ggml-cuda/rope.cu
@@ -92,13 +92,16 @@ static __global__ void rope_norm(const T * x,
} else if constexpr (std::is_same_v<half, D>) {
half2 v = make_half2(x0, x1);
ggml_cuda_memcpy_1<4>(dst + idst, &v);
+ } else if constexpr (std::is_same_v<nv_bfloat16, D>) {
+ const __align__(4) nv_bfloat16 v[2] = { ggml_cuda_cast<nv_bfloat16>(x0), ggml_cuda_cast<nv_bfloat16>(x1) };
+ ggml_cuda_memcpy_1<sizeof(v)>(dst + idst, v);
}
};
if (i0 < n_offs || i0 >= n_offs + n_dims) {
if (inplace) {
return;
}
- store_coaelsced(x[ix + 0], x[ix + 1]);
+ store_coaelsced(ggml_cuda_cast<float>(x[ix + 0]), ggml_cuda_cast<float>(x[ix + 1]));
return;
}
@@ -113,8 +116,8 @@ static __global__ void rope_norm(const T * x,
rope_yarn<forward>(theta_base/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor, cos_theta, sin_theta);
- const float x0 = x[ix + 0];
- const float x1 = x[ix + 1];
+ const float x0 = ggml_cuda_cast<float>(x[ix + 0]);
+ const float x1 = ggml_cuda_cast<float>(x[ix + 1]);
store_coaelsced(x0 * cos_theta - x1 * sin_theta, x0 * sin_theta + x1 * cos_theta);
}
@@ -189,8 +192,8 @@ static __global__ void rope_neox(const T * x,
rope_yarn<forward>(theta_base/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor, cos_theta, sin_theta);
// idst/ix point at channel i0/2; the first channel of the rotated pair is n_offs + iw/2 = i0/2 + n_offs/2
- const float x0 = x[ix + n_offs/2 + 0];
- const float x1 = x[ix + n_offs/2 + n_dims/2];
+ const float x0 = ggml_cuda_cast<float>(x[ix + n_offs/2 + 0]);
+ const float x1 = ggml_cuda_cast<float>(x[ix + n_offs/2 + n_dims/2]);
dst[idst + n_offs/2 + 0] = ggml_cuda_cast<D>(x0 * cos_theta - x1 * sin_theta);
dst[idst + n_offs/2 + n_dims / 2] = ggml_cuda_cast<D>(x0 * sin_theta + x1 * cos_theta);
@@ -293,8 +296,8 @@ static __global__ void rope_multi(const T * x,
rope_yarn<forward>(theta_base/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor, cos_theta, sin_theta);
// idst/ix point at channel i0/2; the first channel of the rotated pair is n_offs + iw/2 = i0/2 + n_offs/2
- const float x0 = x[ix + n_offs/2 + 0];
- const float x1 = x[ix + n_offs/2 + n_dims/2];
+ const float x0 = ggml_cuda_cast<float>(x[ix + n_offs/2 + 0]);
+ const float x1 = ggml_cuda_cast<float>(x[ix + n_offs/2 + n_dims/2]);
dst[idst + n_offs/2 + 0] = ggml_cuda_cast<D>(x0*cos_theta - x1*sin_theta);
dst[idst + n_offs/2 + n_dims/2] = ggml_cuda_cast<D>(x0*sin_theta + x1*cos_theta);
@@ -357,11 +360,11 @@ static __global__ void rope_vision(const T * x,
rope_yarn<forward>(theta_base/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor, cos_theta, sin_theta);
- const float x0 = x[ix + 0];
- const float x1 = x[ix + n_dims];
+ const float x0 = ggml_cuda_cast<float>(x[ix + 0]);
+ const float x1 = ggml_cuda_cast<float>(x[ix + n_dims]);
- dst[idst + 0] = x0*cos_theta - x1*sin_theta;
- dst[idst + n_dims] = x0*sin_theta + x1*cos_theta;
+ dst[idst + 0] = ggml_cuda_cast<T>(x0*cos_theta - x1*sin_theta);
+ dst[idst + n_dims] = ggml_cuda_cast<T>(x0*sin_theta + x1*cos_theta);
}
template <bool forward, typename T, typename D>
@@ -569,8 +572,8 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx,
}
cudaStream_t stream = ctx.stream();
- GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);
- GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
+ GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16);
+ GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_BF16);
// When not fused, src0 and dst types must match
// When fused (ROPE+VIEW+SET_ROWS), src0 may be F32 and dst may be F16
GGML_ASSERT(src0->type == dst->type || (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16));
@@ -655,6 +658,16 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx,
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
set_rows_stride, inplace, stream);
+ } else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_BF16) {
+ rope_neox_cuda<forward, nv_bfloat16, nv_bfloat16>((const nv_bfloat16 *) src0_d, (nv_bfloat16 *) dst_d, ne00, ne01, ne02, s01, s02,
+ s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
+ ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
+ set_rows_stride, inplace, stream);
+ } else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_F16) {
+ rope_neox_cuda<forward, nv_bfloat16, half>((const nv_bfloat16 *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02,
+ s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
+ ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
+ set_rows_stride, inplace, stream);
} else {
GGML_ABORT("fatal error");
}
@@ -674,6 +687,16 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx,
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
ext_factor, attn_factor, corr_dims, freq_factors, sections, is_imrope,
row_indices, set_rows_stride, inplace, stream);
+ } else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_BF16) {
+ rope_multi_cuda<forward, nv_bfloat16, nv_bfloat16>((const nv_bfloat16 *) src0_d, (nv_bfloat16 *) dst_d, ne00, ne01, ne02, s01, s02,
+ s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
+ ext_factor, attn_factor, corr_dims, freq_factors, sections, is_imrope,
+ row_indices, set_rows_stride, inplace, stream);
+ } else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_F16) {
+ rope_multi_cuda<forward, nv_bfloat16, half>((const nv_bfloat16 *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02,
+ s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
+ ext_factor, attn_factor, corr_dims, freq_factors, sections, is_imrope,
+ row_indices, set_rows_stride, inplace, stream);
} else {
GGML_ABORT("fatal error");
}
@@ -686,6 +709,10 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx,
rope_vision_cuda<forward>((const half *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02, s03, s1,
s2, s3, n_dims, nr, pos, freq_scale, freq_base, ext_factor, attn_factor,
corr_dims, freq_factors, sections, stream);
+ } else if (src0->type == GGML_TYPE_BF16) {
+ rope_vision_cuda<forward>((const nv_bfloat16 *) src0_d, (nv_bfloat16 *) dst_d, ne00, ne01, ne02, s01, s02, s03, s1,
+ s2, s3, n_dims, nr, pos, freq_scale, freq_base, ext_factor, attn_factor,
+ corr_dims, freq_factors, sections, stream);
} else {
GGML_ABORT("fatal error");
}
@@ -705,6 +732,16 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx,
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
set_rows_stride, inplace, stream);
+ } else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_BF16) {
+ rope_norm_cuda<forward, nv_bfloat16, nv_bfloat16>((const nv_bfloat16 *) src0_d, (nv_bfloat16 *) dst_d, ne00, ne01, ne02, s01, s02,
+ s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
+ ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
+ set_rows_stride, inplace, stream);
+ } else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_F16) {
+ rope_norm_cuda<forward, nv_bfloat16, half>((const nv_bfloat16 *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02,
+ s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
+ ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
+ set_rows_stride, inplace, stream);
} else {
GGML_ABORT("fatal error");
}
diff --git a/ggml/src/ggml-cuda/set-rows.cu b/ggml/src/ggml-cuda/set-rows.cu
index 4be3e649b..e4fb51619 100644
--- a/ggml/src/ggml-cuda/set-rows.cu
+++ b/ggml/src/ggml-cuda/set-rows.cu
@@ -4,8 +4,8 @@
typedef void (*set_rows_kernel_t)(const char * src, char * dst);
// Generic quantized set_rows kernel template
-template <typename idx_t, typename block_type, int qk, void (*quantize_func)(const float *, block_type *)>
-static __global__ void k_set_rows_quant(const float * __restrict__ src0,
+template <typename src_t, typename idx_t, typename block_type, int qk, void (*quantize_func)(const float *, block_type *)>
+static __global__ void k_set_rows_quant(const src_t * __restrict__ src0,
const idx_t * __restrict__ src1,
block_type * __restrict__ dst,
const int64_t ne_total,
@@ -56,13 +56,23 @@ static __global__ void k_set_rows_quant(const float * __restrict__ src0,
ggml_cuda_pdl_sync();
const int64_t dst_row = *(src1 + i10*s10 + i11*s11 + i12*s12);
- const float * src0_row = src0 + i01*s01 + i02*s02 + i03*s03;
+ const src_t * src0_row = src0 + i01*s01 + i02*s02 + i03*s03;
block_type * dst_row_ptr = dst + (dst_row*s1 + i02*s2 + i03*s3) / sizeof(block_type);
- const float * src_block = src0_row + i00;
+ const src_t * src_block = src0_row + i00;
block_type * dst_block = dst_row_ptr + i00 / qk;
- quantize_func(src_block, dst_block);
+ if constexpr (std::is_same_v<src_t, float>) {
+ quantize_func(src_block, dst_block);
+ } else {
+ // the quantize functions take F32 input: widen the block first
+ float src_block_f32[qk];
+#pragma unroll
+ for (int j = 0; j < qk; ++j) {
+ src_block_f32[j] = ggml_cuda_cast<float>(src_block[j]);
+ }
+ quantize_func(src_block_f32, dst_block);
+ }
GGML_UNUSED(ne10);
GGML_UNUSED(ne11);
@@ -71,9 +81,9 @@ static __global__ void k_set_rows_quant(const float * __restrict__ src0,
}
// Template dispatch function for quantized set_rows
-template<typename idx_t, typename block_type, int qk, void (*quantize_func)(const float*, block_type*)>
+template<typename src_t, typename idx_t, typename block_type, int qk, void (*quantize_func)(const float*, block_type*)>
static void set_rows_cuda_quant(
- const float * src0_d, const idx_t * src1_d, block_type * dst_d,
+ const src_t * src0_d, const idx_t * src1_d, block_type * dst_d,
const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,
const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13,
const size_t nb01, const size_t nb02, const size_t nb03,
@@ -87,9 +97,9 @@ static void set_rows_cuda_quant(
const dim3 block_size(CUDA_SET_ROWS_BLOCK_SIZE);
const dim3 grid_size(num_blocks);
- const int64_t s01 = nb01/sizeof(float);
- const int64_t s02 = nb02/sizeof(float);
- const int64_t s03 = nb03/sizeof(float);
+ const int64_t s01 = nb01/sizeof(src_t);
+ const int64_t s02 = nb02/sizeof(src_t);
+ const int64_t s03 = nb03/sizeof(src_t);
const int64_t s10 = nb10/sizeof(idx_t);
const int64_t s11 = nb11/sizeof(idx_t);
const int64_t s12 = nb12/sizeof(idx_t);
@@ -104,7 +114,7 @@ static void set_rows_cuda_quant(
const uint3 ne11_fd = init_fastdiv_values((uint32_t) ne11);
const uint3 ne12_fd = init_fastdiv_values((uint32_t) ne12);
- k_set_rows_quant<idx_t, block_type, qk, quantize_func><<<grid_size, block_size, 0, stream>>>(
+ k_set_rows_quant<src_t, idx_t, block_type, qk, quantize_func><<<grid_size, block_size, 0, stream>>>(
src0_d, src1_d, dst_d, ne_total, ne10, ne11, ne12, ne13, s01, s02, s03, s10, s11, s12, s1, s2, s3, ne00_fd,
ne01_fd, ne02_fd, ne11_fd, ne12_fd);
}
@@ -268,7 +278,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s
stream
);
} else if (dst->type == GGML_TYPE_Q4_0) {
- set_rows_cuda_quant<idx_t, block_q4_0, QK4_0, quantize_f32_q4_0_block>(
+ set_rows_cuda_quant<src_t, idx_t, block_q4_0, QK4_0, quantize_f32_q4_0_block>(
src0_d, src1_d, (block_q4_0*)dst->data,
ne00, ne01, ne02, ne03,
ne10, ne11, ne12, ne13,
@@ -278,7 +288,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s
stream
);
} else if (dst->type == GGML_TYPE_Q4_1) {
- set_rows_cuda_quant<idx_t, block_q4_1, QK4_1, quantize_f32_q4_1_block>(
+ set_rows_cuda_quant<src_t, idx_t, block_q4_1, QK4_1, quantize_f32_q4_1_block>(
src0_d, src1_d, (block_q4_1*)dst->data,
ne00, ne01, ne02, ne03,
ne10, ne11, ne12, ne13,
@@ -288,7 +298,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s
stream
);
} else if (dst->type == GGML_TYPE_Q5_0) {
- set_rows_cuda_quant<idx_t, block_q5_0, QK5_0, quantize_f32_q5_0_block>(
+ set_rows_cuda_quant<src_t, idx_t, block_q5_0, QK5_0, quantize_f32_q5_0_block>(
src0_d, src1_d, (block_q5_0*)dst->data,
ne00, ne01, ne02, ne03,
ne10, ne11, ne12, ne13,
@@ -298,7 +308,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s
stream
);
} else if (dst->type == GGML_TYPE_Q5_1) {
- set_rows_cuda_quant<idx_t, block_q5_1, QK5_1, quantize_f32_q5_1_block>(
+ set_rows_cuda_quant<src_t, idx_t, block_q5_1, QK5_1, quantize_f32_q5_1_block>(
src0_d, src1_d, (block_q5_1*)dst->data,
ne00, ne01, ne02, ne03,
ne10, ne11, ne12, ne13,
@@ -308,7 +318,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s
stream
);
} else if (dst->type == GGML_TYPE_Q8_0) {
- set_rows_cuda_quant<idx_t, block_q8_0, QK8_0, quantize_f32_q8_0_block>(
+ set_rows_cuda_quant<src_t, idx_t, block_q8_0, QK8_0, quantize_f32_q8_0_block>(
src0_d, src1_d, (block_q8_0*)dst->data,
ne00, ne01, ne02, ne03,
ne10, ne11, ne12, ne13,
@@ -318,7 +328,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s
stream
);
} else if (dst->type == GGML_TYPE_IQ4_NL) {
- set_rows_cuda_quant<idx_t, block_iq4_nl, QK4_NL, quantize_f32_iq4_nl_block>(
+ set_rows_cuda_quant<src_t, idx_t, block_iq4_nl, QK4_NL, quantize_f32_iq4_nl_block>(
src0_d, src1_d, (block_iq4_nl*)dst->data,
ne00, ne01, ne02, ne03,
ne10, ne11, ne12, ne13,
@@ -382,12 +392,11 @@ void set_rows_cuda<half, int64_t>(ggml_backend_cuda_context & ctx, const ggml_te
}
}
-
void ggml_cuda_op_set_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
const ggml_tensor * src0 = dst->src[0];
const ggml_tensor * src1 = dst->src[1];
- GGML_ASSERT(src0->type == GGML_TYPE_F32 || (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16));
+ GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16 || (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16));
GGML_ASSERT(src1->type == GGML_TYPE_I64 || src1->type == GGML_TYPE_I32);
if (src0->type == GGML_TYPE_F32) {
@@ -402,6 +411,12 @@ void ggml_cuda_op_set_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
} else {
set_rows_cuda<half, int32_t>(ctx, src0, src1, dst);
}
+ } else if (src0->type == GGML_TYPE_BF16) {
+ if (src1->type == GGML_TYPE_I64) {
+ set_rows_cuda<nv_bfloat16, int64_t>(ctx, src0, src1, dst);
+ } else {
+ set_rows_cuda<nv_bfloat16, int32_t>(ctx, src0, src1, dst);
+ }
} else {
GGML_ABORT("unsupported type %s", ggml_type_name(src0->type));
}
diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m
index 26746d0b3..2e085453a 100644
--- a/ggml/src/ggml-metal/ggml-metal-device.m
+++ b/ggml/src/ggml-metal/ggml-metal-device.m
@@ -1713,7 +1713,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
return has_simdgroup_reduction && op->src[0]->type == GGML_TYPE_F32 && (ggml_is_contiguous_rows(op->src[0]));
case GGML_OP_ROPE:
case GGML_OP_ROPE_BACK:
- return ggml_rope_get_truncate(op);
+ return ggml_rope_get_truncate(op) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16);
case GGML_OP_IM2COL:
return ggml_is_contiguous(op->src[1]) && op->src[1]->type == GGML_TYPE_F32 && (op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32);
case GGML_OP_CONV_2D:
diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp
index 0d9212232..8162ace2e 100644
--- a/ggml/src/ggml-opencl/ggml-opencl.cpp
+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp
@@ -12305,7 +12305,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
}
return false;
}
- return true;
+ return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;
}
case GGML_OP_SOLVE_TRI:
return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]);
diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp
index 98f22cfd7..bd49560d8 100644
--- a/ggml/src/ggml-openvino/ggml-openvino.cpp
+++ b/ggml/src/ggml-openvino/ggml-openvino.cpp
@@ -1766,6 +1766,9 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) {
!GgmlOvDecoder::is_conv_state_snapshot_writeback(op)) {
return {false, "GET_ROWS/SET_ROWS with ne[3] != 1 (ne[3]=" + std::to_string(op->ne[3]) + ") is not supported"};
}
+ if (op->op == GGML_OP_SET_ROWS && op->src[0]->type == GGML_TYPE_BF16) {
+ return {false, "SET_ROWS with BF16 src0 is not supported"};
+ }
if (op->ne[0] == 256 && (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K ||
op->src[0]->type == GGML_TYPE_Q4_1 || op->src[0]->type == GGML_TYPE_Q5_1)) {
// These are all f16-arithmetic dequant rounding errors that intermittently exceed the
@@ -2035,6 +2038,9 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) {
if (op->view_src != nullptr && !ggml_is_contiguous(op->src[0])) {
return {false, "ROPE on VIEW / non-contiguous input is not supported"};
}
+ if (op->type == GGML_TYPE_BF16) {
+ return {false, "ROPE with BF16 type is not supported"};
+ }
break;
}
case GGML_OP_TRANSPOSE: {
diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp
index 4fc90e543..9da436ced 100644
--- a/ggml/src/ggml-sycl/ggml-sycl.cpp
+++ b/ggml/src/ggml-sycl/ggml-sycl.cpp
@@ -7434,8 +7434,7 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
if (op->type == GGML_TYPE_TQ2_0 || op->type == GGML_TYPE_TQ1_0) {
return false;
}
- auto res = (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 ||
- op->src[0]->type == GGML_TYPE_BF16) &&
+ auto res = (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&
(op->src[1]->type == GGML_TYPE_I64 || op->src[1]->type == GGML_TYPE_I32);
return res;
}
@@ -7624,6 +7623,7 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
}
case GGML_OP_ROPE:
case GGML_OP_ROPE_BACK:
+ return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;
case GGML_OP_IM2COL:
case GGML_OP_IM2COL_3D:
case GGML_OP_UPSCALE:
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index 6955bb406..ecd46d828 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -16729,8 +16729,9 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
case GGML_OP_REPEAT_BACK:
return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32;
case GGML_OP_ROPE:
- return ggml_is_contiguous_rows(op) && ggml_is_contiguous_rows(op->src[0]);
+ return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && ggml_is_contiguous_rows(op) && ggml_is_contiguous_rows(op->src[0]);
case GGML_OP_ROPE_BACK:
+ return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;
case GGML_OP_NONE:
case GGML_OP_RESHAPE:
case GGML_OP_VIEW:
diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c
index 68e3a011a..ecd667753 100644
--- a/ggml/src/ggml.c
+++ b/ggml/src/ggml.c
@@ -4019,7 +4019,7 @@ struct ggml_tensor * ggml_set_rows(
GGML_ASSERT(b->ne[2] % c->ne[1] == 0);
GGML_ASSERT(b->ne[3] % c->ne[2] == 0);
GGML_ASSERT(c->ne[3] == 1);
- GGML_ASSERT(b->type == GGML_TYPE_F32 || b->type == GGML_TYPE_F16);
+ GGML_ASSERT(b->type == GGML_TYPE_F32 || b->type == GGML_TYPE_F16 || b->type == GGML_TYPE_BF16);
GGML_ASSERT(c->type == GGML_TYPE_I64 || c->type == GGML_TYPE_I32);
GGML_ASSERT(ggml_is_contiguous_rows(a));
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 01ea4c115..d6b849944 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -2830,6 +2830,12 @@ struct test_set_rows : public test_case {
if (type_src == GGML_TYPE_F16) {
err_estimate *= 16.0f;
}
+ if (type_src == GGML_TYPE_BF16) {
+ // BF16 values have 8 significant bits, so many of them land exactly on a rounding tie,
+ // and backends break ties differently (e.g. the CPU q8_0 rounds half to even, CUDA half away from zero).
+ // Allow 1 in 32 values to be off by one instead of one value in total.
+ return err_estimate / (0.25f*32.0f);
+ }
err_estimate /= 0.25f*float(ne[0] * r * ne[2]*nr23[0] * ne[3]*nr23[1]);
return err_estimate;
}
@@ -2860,10 +2866,15 @@ struct test_rope_set_rows : public test_case {
const int n_dims;
const bool yarn;
const bool truncate;
+ const ggml_type type_a; // type of the rope input
const int n_ctx{512};
std::string vars() override {
- return VARS_TO_STR7(type, type_idx, ne_a, mode, n_dims, yarn, truncate);
+ std::string s = VARS_TO_STR7(type, type_idx, ne_a, mode, n_dims, yarn, truncate);
+ if (type_a != GGML_TYPE_F32) {
+ s += "," + VAR_TO_STR(type_a);
+ }
+ return s;
}
std::string op_desc(ggml_tensor * t) override {
@@ -2871,17 +2882,27 @@ struct test_rope_set_rows : public test_case {
return "ROPE_SET_ROWS";
}
+ double max_nmse_err() override {
+ if (type_a != GGML_TYPE_BF16) {
+ return 1e-7;
+ }
+ // into F16 the reference rounds the BF16 rope result to BF16 first, the fused rope + set_rows does not
+ // otherwise the limit of a BF16 test_rope: the angle error n_ctx*FLT_EPSILON can flip a BF16 rounding
+ return type == GGML_TYPE_F16 ? 1e-5 : n_ctx * FLT_EPSILON / 128.0;
+ }
+
bool run_whole_graph() override { return true; }
test_rope_set_rows(ggml_type type,
ggml_type type_idx,
std::array<int64_t, 4> ne_a,
int mode,
- int n_dims = 128, bool yarn = false, bool truncate = true)
- : type(type), type_idx(type_idx), ne_a(ne_a), mode(mode), n_dims(n_dims), yarn(yarn), truncate(truncate) {}
+ int n_dims = 128, bool yarn = false, bool truncate = true,
+ ggml_type type_a = GGML_TYPE_F32)
+ : type(type), type_idx(type_idx), ne_a(ne_a), mode(mode), n_dims(n_dims), yarn(yarn), truncate(truncate), type_a(type_a) {}
ggml_tensor * build_graph(ggml_context * ctx) override {
- ggml_tensor * a = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne_a[0], ne_a[1], ne_a[2], 1);
+ ggml_tensor * a = ggml_new_tensor_4d(ctx, type_a, ne_a[0], ne_a[1], ne_a[2], 1);
ggml_set_name(a, "a");
const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE;
@@ -6793,6 +6814,10 @@ struct test_rope : public test_case {
// the f32 rotation angle of a position below n_ctx carries a rounding error of up to n_ctx*FLT_EPSILON radians
// backends compute the angle with different rounding sequences, so their agreement is bounded by that error
const double eps_theta = n_ctx * FLT_EPSILON;
+ if (type == GGML_TYPE_BF16) {
+ // with BF16 output that error can also change the rounding: a step of 2^-7 with a chance of about eps_theta/2^-7
+ return std::max(test_case::max_nmse_err(), eps_theta / 128.0);
+ }
return std::max(test_case::max_nmse_err(), eps_theta * eps_theta);
}
@@ -10895,6 +10920,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I32, { 1, 8, 1, 3 }, { 1, 1 }, 2, false));
test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I64, { 1, 8, 1, 3 }, { 1, 1 }, 2, true));
test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I32, { 1, 8, 1, 3 }, { 1, 1 }, 2, true));
+ // BF16 source: one case per type covers batch dims, broadcast indices and gaps between rows
+ for (ggml_type type : all_types) {
+ test_cases.emplace_back(new test_set_rows(GGML_TYPE_BF16, type, GGML_TYPE_I64, { 256, 11, 1, 1 }, { 2, 3, }, 7, true));
+ }
+ test_cases.emplace_back(new test_set_rows(GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_I32, { 256, 5, 1, 3 }, { 1, 1, }, 1, false));
+ test_cases.emplace_back(new test_set_rows(GGML_TYPE_BF16, GGML_TYPE_F16, GGML_TYPE_I64, { 1, 8, 1, 3 }, { 1, 1 }, 2, false));
for (int mode : { GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_VISION, GGML_ROPE_TYPE_IMROPE }) {
for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) {
@@ -10905,11 +10936,19 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
}
}
test_cases.emplace_back(new test_rope_set_rows(GGML_TYPE_F32, GGML_TYPE_I32, { 128, 32, 8, 1 }, GGML_ROPE_TYPE_IMROPE));
+ for (int mode : { GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_VISION, GGML_ROPE_TYPE_IMROPE }) {
+ for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_BF16}) {
+ for (int ne2 : {1, 8, 512}) {
+ test_cases.emplace_back(new test_rope_set_rows(type, GGML_TYPE_I64, { 128, 32, ne2, 1 }, mode, 128, false, true, GGML_TYPE_BF16));
+ }
+ }
+ }
// partial rotation
for (int mode : { GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_IMROPE }) {
for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) {
test_cases.emplace_back(new test_rope_set_rows(type, GGML_TYPE_I64, { 256, 4, 8, 1 }, mode, 64));
}
+ test_cases.emplace_back(new test_rope_set_rows(GGML_TYPE_F16, GGML_TYPE_I64, { 256, 4, 8, 1 }, mode, 64, false, true, GGML_TYPE_BF16));
}
for (int mode : {GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX}) {
@@ -12716,8 +12755,17 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
}
}
+ // BF16: each mode forward on a strided view and backward on an offset view, with and without freq factors, and one in place
+ for (int mode : {GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_IMROPE, GGML_ROPE_TYPE_VISION}) {
+ for (bool ff : {false, true}) {
+ test_cases.emplace_back(new test_rope(GGML_TYPE_BF16, {128, 32, 2, 1}, 128, mode, 512, 1.4245f, 0.7465f, 1.4245f, ff, 1, true));
+ test_cases.emplace_back(new test_rope(GGML_TYPE_BF16, {128, 32, 2, 1}, 128, mode, 512, 1.4245f, 0.7465f, 1.4245f, ff, 2, false));
+ }
+ test_cases.emplace_back(new test_rope(GGML_TYPE_BF16, {128, 32, 2, 1}, 128, mode, 512, 1.4245f, 0.7465f, 1.4245f, false, 0, true, true));
+ }
+
// rotated dims window at an offset (ggml_rope_set_offset), not supported for vision mode
- for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) {
+ for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16}) {
for (bool fw : {true, false}) { // fw == forward
for (bool ff : {false, true}) {
test_cases.emplace_back(new test_rope(type, {128, 32, 2, 1}, 32, GGML_ROPE_TYPE_NORMAL, 512, 1.4245f, 0.7465f, 1.4245f, ff, 0, fw, false, 32));
@@ -14108,7 +14156,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
}
for (bool fw : {true, false}) { // fw == forward
- for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) {
+ for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16}) {
for (bool ff : {false, true}) { // freq_factors
for (float v : { 0, 1 }) {
test_cases.emplace_back(new test_rope(type, {128, 32, 512, 1}, 128, GGML_ROPE_TYPE_NORMAL, 512, 1.0f, 0.0f, 1.0f, ff, v, fw)); // llama 7B