Commit 9e9619d03 for llama.cpp
commit 9e9619d030bbe6b584d826a0a6cf06e38251f52b
Author: Oliver Simons <osimons@nvidia.com>
Date: Wed Nov 11 09:44:30 2026 +0100
CUDA: Add OCP FP8 support (#28413)
* Initial CUDA WIP version
* Use fp8 intrinsics
* Do W8A16 for FP8 in GEMV case
(base) osimons@ub-osimons:~/lcpp_fp8.git$ ./build-x64-linux-gcc-reldbg/bin/llama-bench -m /mnt/share/gguf/nvidia/Qwen3.6-27B-NVFP4/Qwen3.6-27B-NVFP4_fp8.gguf -p 8196 -ub 512,2048 -d 0,32000 -n 256 --load-mode dio
ggml_cuda_init: found 1 CUDA devices (Total VRAM: 32165 MiB):
Device 0: NVIDIA RTX PRO 4500 Blackwell, compute capability 12.0, VMM: yes, VRAM: 32165 MiB
| model | size | params | backend | ngl | n_ubatch | lm | test | t/s |
| ------------------------------ | ---------: | ---------: | ---------- | --: | -------: | ---------: | --------------: | -------------------: |
| qwen35 27B NVFP4 | 19.56 GiB | 27.32 B | CUDA | -1 | 512 | dio | pp8196 | 2781.53 ± 8.48 |
| qwen35 27B NVFP4 | 19.56 GiB | 27.32 B | CUDA | -1 | 512 | dio | tg256 | 37.56 ± 0.02 |
| qwen35 27B NVFP4 | 19.56 GiB | 27.32 B | CUDA | -1 | 512 | dio | pp8196 @ d32000 | 1616.31 ± 2.61 |
| qwen35 27B NVFP4 | 19.56 GiB | 27.32 B | CUDA | -1 | 512 | dio | tg256 @ d32000 | 34.11 ± 0.03 |
| qwen35 27B NVFP4 | 19.56 GiB | 27.32 B | CUDA | -1 | 2048 | dio | pp8196 | 2542.00 ± 0.51 |
| qwen35 27B NVFP4 | 19.56 GiB | 27.32 B | CUDA | -1 | 2048 | dio | tg256 | 37.36 ± 0.02 |
| qwen35 27B NVFP4 | 19.56 GiB | 27.32 B | CUDA | -1 | 2048 | dio | pp8196 @ d32000 | 1762.86 ± 1.33 |
| qwen35 27B NVFP4 | 19.56 GiB | 27.32 B | CUDA | -1 | 2048 | dio | tg256 @ d32000 | 34.09 ± 0.02 |
* Optimize GEMV kernel for FP8
* Guard __nv_cvt_fp8x2_to_bf162raw on CTK 13.2
Fixes https://github.com/ggml-org/llama.cpp/pull/28413#discussion_r4044500099
* Remove trailing whitespace
* Add supported for strided copies to/from FP8 in CUDA
* Prepare Q_reg for FP8 FlashAttention in fattn-vec.cuh
* Fix odd-rows for convert/getrows, possible with FP8
* Fix FP8 fallback in cuBlas
* Gate mmvf_f8x2_e4m3_to_bf162 on FP8 availability, i.e. CTK >= 11.8
* Restrict (fused) MMVF to 2*ts alignment, and cublas FP8 fo 16-byte
* Add todo about accumulator/precision honoring
This is beyond the scope of the stacked PR
* Avoid races in fp8_fallback
* Allow Hopper and other BW SKUs for FP8
* Use intrinsic over explicit manual conversions
* Resolve TODO
As per spec, we don't have to preserve signs
* Add commend on cublaslt workspaces
* Use convert_unary_cont_cuda over block dequant in convert.cu
* Use get_rows_cuda_float for fp8 kv cache
* Make FP8 copies work on all IHV's
* More cleanup
* Simplify by falling back to shared cuBLAS helper
* Fuse weight_scale multiplication for FP8 also
* More code-cleanup
* Guard __nv_cvt_fp8x2_to_bf162raw on CTK 13.4
Only cvt.rn.bf16x2.e4m3x2 is exposed from 13.2 onwards, so
93e98b9203dfacb85f9ac665f5f0482a8c6b4192 was incorrect fix
* Use F32 activations in fallback mmvf path
* Explicitly link against cublaslt
* Move FP8 conversion intrinsics into ggml_cuda_cast
Cleaner this way
* WIP src1/prec for fp8 in MMVF
* Remove comment
* Keep only f8 x f8 KV cache for now
* Just do FP8 x FP32 x FP32 in MMVF as its fast enough
* Update ggml/src/ggml-cuda/ggml-cuda.cu
Co-authored-by: Aman Gupta <amangupta052@gmail.com>
* Add batched FP8 GEMMS in cublas as suggested by code review
https://github.com/ggml-org/llama.cpp/pull/28413#issuecomment-6676015644
* Extract fp8x2 -> float2 load into helper
* Searapte copy checks to/from fp8
* Compiler massaging to avoid repeated address offsets
---------
Co-authored-by: Aman Gupta <amangupta052@gmail.com>
diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt
index 03fc428a8..9b6b8b6ba 100644
--- a/ggml/CMakeLists.txt
+++ b/ggml/CMakeLists.txt
@@ -205,7 +205,7 @@ option(GGML_CUDA_NO_PEER_COPY "ggml: do not use peer to peer copie
option(GGML_CUDA_NO_VMM "ggml: do not try to use CUDA VMM" OFF)
option(GGML_CUDA_FA "ggml: compile ggml FlashAttention CUDA kernels" ON)
option(GGML_CUDA_FA_ALL_QUANTS "ggml: compile all quants for FlashAttention" OFF)
-set (GGML_CUDA_FA_QUANTS "q4_0-q4_0;q8_0-q8_0;f16-f16;bf16-bf16" CACHE STRING
+set (GGML_CUDA_FA_QUANTS "q4_0-q4_0;q8_0-q8_0;f16-f16;bf16-bf16;f8_e4m3-f8_e4m3" CACHE STRING
"ggml: FlashAttention K-V type combinations to compile, \"all\" or a list such as \"q8_0-q8_0;q8_0-q4_0\"")
option(GGML_CUDA_GRAPHS "ggml: use CUDA graphs (llama.cpp only)" ${GGML_CUDA_GRAPHS_DEFAULT})
option(GGML_CUDA_NCCL "ggml: use NVIDIA Collective Comm. Library" ON)
diff --git a/ggml/cmake/common.cmake b/ggml/cmake/common.cmake
index 42b65a580..c8d232294 100644
--- a/ggml/cmake/common.cmake
+++ b/ggml/cmake/common.cmake
@@ -58,7 +58,7 @@ endfunction()
# Determines which FlashAttention vector kernel template instances to compile, returns them in OUT_SRCS.
function(ggml_cuda_fattn_vec_instances DIR OUT_SRCS)
- set(FA_TYPES q4_0 q4_1 q5_0 q5_1 q8_0 bf16 f16)
+ set(FA_TYPES q4_0 q4_1 q5_0 q5_1 q8_0 bf16 f16 f8_e4m3)
string(TOLOWER "${GGML_CUDA_FA_QUANTS}" FA_QUANTS)
string(STRIP "${FA_QUANTS}" FA_QUANTS)
diff --git a/ggml/src/ggml-cuda/CMakeLists.txt b/ggml/src/ggml-cuda/CMakeLists.txt
index 297d9ffc9..dce8827ae 100644
--- a/ggml/src/ggml-cuda/CMakeLists.txt
+++ b/ggml/src/ggml-cuda/CMakeLists.txt
@@ -163,7 +163,7 @@ if (CUDAToolkit_FOUND)
if (GGML_STATIC)
if (WIN32)
# As of 12.3.1 CUDA Toolkit for Windows does not offer a static cublas library
- target_link_libraries(ggml-cuda PRIVATE CUDA::cudart_static CUDA::cublas)
+ target_link_libraries(ggml-cuda PRIVATE CUDA::cudart_static CUDA::cublas CUDA::cublasLt)
else ()
if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL "10.1")
target_link_libraries(ggml-cuda PRIVATE CUDA::cudart_static CUDA::cublas_static CUDA::cublasLt_static)
@@ -172,7 +172,7 @@ if (CUDAToolkit_FOUND)
endif()
endif()
else()
- target_link_libraries(ggml-cuda PRIVATE CUDA::cudart CUDA::cublas)
+ target_link_libraries(ggml-cuda PRIVATE CUDA::cudart CUDA::cublas CUDA::cublasLt)
endif()
if (GGML_CUDA_NO_VMM)
diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh
index aedf9bca1..185ed98d4 100644
--- a/ggml/src/ggml-cuda/common.cuh
+++ b/ggml/src/ggml-cuda/common.cuh
@@ -388,6 +388,10 @@ static bool blackwell_mma_available(const int cc) {
ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_RUBIN;
}
+static bool fp8_mma_hardware_available(const int cc) {
+ return GGML_CUDA_CC_IS_NVIDIA(cc) && cc >= GGML_CUDA_CC_ADA_LOVELACE && cc < GGML_CUDA_CC_RUBIN;
+}
+
// Checks whether the tensor's base data pointer and higher-dimensional strides are byte-aligned to `alignment` bytes.
static bool ggml_cuda_is_aligned(const ggml_tensor * tensor, const size_t alignment) {
GGML_ASSERT(tensor != nullptr);
@@ -883,6 +887,67 @@ static __device__ __forceinline__ float ggml_cuda_ue4m3_to_fp32(uint8_t x) {
#endif // defined(GGML_USE_HIP) && defined(CDNA3) && defined(FP8_AVAILABLE) && HIP_VERSION >= 60200000
}
+static __host__ __device__ __forceinline__ float ggml_cuda_f8_e4m3_to_fp32(uint8_t x) {
+#if defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
+ __nv_fp8_e4m3 xf;
+ xf.__x = x;
+ return static_cast<float>(xf);
+#else
+ const uint8_t magnitude = x & 0x7F;
+ if (magnitude == 0x7F) {
+ return NAN;
+ }
+
+ const int exp = (magnitude >> 3) & 0x0F;
+ const int man = magnitude & 0x07;
+ float value;
+ if (exp == 0) {
+ value = ldexpf((float) man, -9);
+ } else {
+ value = ldexpf(1.0f + (float) man / 8.0f, exp - 7);
+ }
+ return x & 0x80 ? -value : value;
+#endif // defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
+}
+
+static __host__ __device__ __forceinline__ uint8_t ggml_cuda_fp32_to_f8_e4m3(float x) {
+#if defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
+ return __nv_cvt_float_to_fp8(x, __NV_SATFINITE, __NV_E4M3);
+#elif !defined(__CUDA_ARCH__) && !defined(__HIP_DEVICE_COMPILE__) && !defined(__MUSA_ARCH__)
+ // ggml_cuda_cast also runs on the host, where __float2int_rn is not available.
+ return ggml_fp32_to_f8_e4m3(x);
+#else
+ const uint8_t sign = signbit(x) ? 0x80 : 0;
+ x = fabsf(x);
+
+ if (isnan(x)) {
+ return sign | 0x7F;
+ }
+ if (x == 0.0f) {
+ return sign;
+ }
+ if (isinf(x) || x >= 448.0f) {
+ return sign | 0x7E;
+ }
+ if (x < 0.015625f) {
+ return sign | (uint8_t) __float2int_rn(x * 512.0f);
+ }
+
+ int exp;
+ const float mantissa = frexpf(x, &exp) * 2.0f;
+ int encoded_exp = exp + 6;
+ int encoded_man = __float2int_rn((mantissa - 1.0f) * 8.0f);
+ if (encoded_man == 8) {
+ encoded_man = 0;
+ encoded_exp++;
+ }
+ if (encoded_exp > 15 || (encoded_exp == 15 && encoded_man > 6)) {
+ return sign | 0x7E;
+ }
+ return sign | (uint8_t) (encoded_exp << 3) | (uint8_t) encoded_man;
+#endif // defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
+}
+
static __device__ __forceinline__ uint8_t ggml_cuda_fp32_to_ue4m3(float x) {
#if defined(BLACKWELL_MMA_AVAILABLE) // This is used for NVFP4 subblock scale quantizations only
if (!(x > 0.0f)) {
@@ -1283,6 +1348,9 @@ struct ggml_backend_cuda_context {
cublasHandle_t cublas_handles[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {};
void * cublas_workspaces[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS] = {};
size_t cublas_workspace_sizes[GGML_CUDA_MAX_DEVICES] = {};
+#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11080
+ cublasLtHandle_t cublaslt_handles[GGML_CUDA_MAX_DEVICES] = {};
+#endif
int curr_stream_no = 0;
@@ -1341,6 +1409,16 @@ struct ggml_backend_cuda_context {
return cublas_handles[device][curr_stream_no];
}
+#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11080
+ cublasLtHandle_t cublaslt_handle() {
+ if (cublaslt_handles[device] == nullptr) {
+ ggml_cuda_set_device(device);
+ CUBLAS_CHECK(cublasLtCreate(&cublaslt_handles[device]));
+ }
+ return cublaslt_handles[device];
+ }
+#endif
+
// pool
std::unique_ptr<ggml_cuda_pool> pools[GGML_CUDA_MAX_DEVICES][GGML_CUDA_MAX_STREAMS];
diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu
index 77fd28894..a8f5047a2 100644
--- a/ggml/src/ggml-cuda/convert.cu
+++ b/ggml/src/ggml-cuda/convert.cu
@@ -541,6 +541,8 @@ to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) {
return dequantize_row_mxfp4_cuda;
case GGML_TYPE_NVFP4:
return dequantize_row_nvfp4_cuda;
+ case GGML_TYPE_F8_E4M3:
+ return convert_unary_cont_cuda<ggml_fp8_e4m3_t>;
case GGML_TYPE_F32:
return convert_unary_cont_cuda<float>;
case GGML_TYPE_F16:
@@ -601,6 +603,8 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {
return dequantize_row_mxfp4_cuda;
case GGML_TYPE_NVFP4:
return dequantize_row_nvfp4_cuda;
+ case GGML_TYPE_F8_E4M3:
+ return convert_unary_cont_cuda<ggml_fp8_e4m3_t>;
case GGML_TYPE_F32:
return convert_unary_cont_cuda<float>;
case GGML_TYPE_BF16:
@@ -658,6 +662,8 @@ to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) {
return dequantize_row_mxfp4_cuda;
case GGML_TYPE_NVFP4:
return dequantize_row_nvfp4_cuda;
+ case GGML_TYPE_F8_E4M3:
+ return convert_unary_cont_cuda<ggml_fp8_e4m3_t>;
case GGML_TYPE_F16:
return convert_unary_cont_cuda<half>;
case GGML_TYPE_BF16:
@@ -685,6 +691,8 @@ to_fp16_nc_cuda_t ggml_get_to_fp16_nc_cuda(ggml_type type) {
return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;
case GGML_TYPE_Q8_0:
return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;
+ case GGML_TYPE_F8_E4M3:
+ return convert_unary_cuda<ggml_fp8_e4m3_t>;
case GGML_TYPE_BF16:
return convert_unary_cuda<nv_bfloat16>;
default:
@@ -710,6 +718,8 @@ to_bf16_nc_cuda_t ggml_get_to_bf16_nc_cuda(ggml_type type) {
return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;
case GGML_TYPE_Q8_0:
return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;
+ case GGML_TYPE_F8_E4M3:
+ return convert_unary_cuda<ggml_fp8_e4m3_t>;
case GGML_TYPE_F16:
return convert_unary_cuda<half, nv_bfloat16>;
default:
@@ -735,6 +745,8 @@ to_fp32_nc_cuda_t ggml_get_to_fp32_nc_cuda(ggml_type type) {
return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;
case GGML_TYPE_Q8_0:
return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;
+ case GGML_TYPE_F8_E4M3:
+ return convert_unary_cuda<ggml_fp8_e4m3_t>;
case GGML_TYPE_BF16:
return convert_unary_cuda<nv_bfloat16, float>;
default:
diff --git a/ggml/src/ggml-cuda/convert.cuh b/ggml/src/ggml-cuda/convert.cuh
index 91255ccf3..fb2a1b3bc 100644
--- a/ggml/src/ggml-cuda/convert.cuh
+++ b/ggml/src/ggml-cuda/convert.cuh
@@ -35,6 +35,36 @@ template<typename dst_t, typename src_t>
__host__ __device__ inline dst_t ggml_cuda_cast(src_t x) {
if constexpr (std::is_same_v<dst_t, src_t>) {
return x;
+ } else if constexpr (std::is_same_v<dst_t, ggml_fp8_e4m3_t>) {
+ return { ggml_cuda_fp32_to_f8_e4m3(ggml_cuda_cast<float>(x)) };
+ } else if constexpr (std::is_same_v<src_t, ggml_fp8_e4m3_t>) {
+#if defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
+#if CUDART_VERSION >= 13040
+ if constexpr (std::is_same_v<dst_t, nv_bfloat16>) {
+ return static_cast<nv_bfloat16>(__nv_cvt_fp8_to_bf16raw(x.bits, __NV_E4M3));
+ }
+#endif
+ if constexpr (std::is_same_v<dst_t, half>) {
+ return static_cast<half>(__nv_cvt_fp8_to_halfraw(x.bits, __NV_E4M3));
+ }
+#endif
+ return ggml_cuda_cast<dst_t>(ggml_cuda_f8_e4m3_to_fp32(x.bits));
+#if defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
+ } else if constexpr (std::is_same_v<src_t, __nv_fp8x2_e4m3> && std::is_same_v<dst_t, nv_bfloat162>) {
+#if CUDART_VERSION >= 13040
+ return static_cast<nv_bfloat162>(__nv_cvt_fp8x2_to_bf162raw(x.__x, __NV_E4M3));
+#else
+ return make_bfloat162(
+ ggml_cuda_cast<nv_bfloat16>(ggml_cuda_f8_e4m3_to_fp32(x.__x & 0xFF)),
+ ggml_cuda_cast<nv_bfloat16>(ggml_cuda_f8_e4m3_to_fp32(x.__x >> 8)));
+#endif
+ } else if constexpr (std::is_same_v<src_t, __nv_fp8x2_e4m3> && std::is_same_v<dst_t, float2>) {
+#if CUDART_VERSION >= 13040
+ return ggml_cuda_cast<float2>(ggml_cuda_cast<nv_bfloat162>(x));
+#else
+ return __half22float2(static_cast<half2>(__nv_cvt_fp8x2_to_halfraw2(x.__x, __NV_E4M3)));
+#endif
+#endif
} else if constexpr(std::is_same_v<dst_t, nv_bfloat16>) {
return __float2bfloat16(float(x));
} else if constexpr(std::is_same_v<src_t, nv_bfloat16>) {
diff --git a/ggml/src/ggml-cuda/cpy.cu b/ggml/src/ggml-cuda/cpy.cu
index 90a06b97b..f1f1bf2da 100644
--- a/ggml/src/ggml-cuda/cpy.cu
+++ b/ggml/src/ggml-cuda/cpy.cu
@@ -477,6 +477,41 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg
} else if (ggml_cuda_cpy_as_memcpy_2d(src0, src1, mc_width, mc_height, mc_spitch, mc_dpitch)) {
CUDA_CHECK(cudaMemcpy2DAsync(src1_ddc, mc_dpitch, src0_ddc, mc_spitch,
mc_width, mc_height, cudaMemcpyDeviceToDevice, main_stream));
+ } else if (src0->type == GGML_TYPE_F8_E4M3 && src1->type == GGML_TYPE_F8_E4M3) {
+ ggml_cpy_scalar_cuda<uint8_t, uint8_t>
+ (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);
+ } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F8_E4M3) {
+ if (contiguous_srcs) {
+ ggml_cpy_scalar_contiguous_cuda<float, ggml_fp8_e4m3_t>
+ (src0_ddc, src1_ddc, ne, main_stream);
+ } else {
+ ggml_cpy_scalar_cuda<float, ggml_fp8_e4m3_t>
+ (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);
+ }
+ } else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F8_E4M3) {
+ if (contiguous_srcs) {
+ ggml_cpy_scalar_contiguous_cuda<half, ggml_fp8_e4m3_t>
+ (src0_ddc, src1_ddc, ne, main_stream);
+ } else {
+ ggml_cpy_scalar_cuda<half, ggml_fp8_e4m3_t>
+ (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);
+ }
+ } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F8_E4M3) {
+ if (contiguous_srcs) {
+ ggml_cpy_scalar_contiguous_cuda<nv_bfloat16, ggml_fp8_e4m3_t>
+ (src0_ddc, src1_ddc, ne, main_stream);
+ } else {
+ ggml_cpy_scalar_cuda<nv_bfloat16, ggml_fp8_e4m3_t>
+ (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);
+ }
+ } else if (src0->type == GGML_TYPE_F8_E4M3 && src1->type == GGML_TYPE_F32) {
+ if (contiguous_srcs) {
+ ggml_cpy_scalar_contiguous_cuda<ggml_fp8_e4m3_t, float>
+ (src0_ddc, src1_ddc, ne, main_stream);
+ } else {
+ ggml_cpy_scalar_cuda<ggml_fp8_e4m3_t, float>
+ (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream);
+ }
} else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32) {
if (can_be_transposed) {
ggml_cpy_scalar_cuda<float, float, true>
diff --git a/ggml/src/ggml-cuda/fattn-common.cuh b/ggml/src/ggml-cuda/fattn-common.cuh
index 26a432d89..6667ca400 100644
--- a/ggml/src/ggml-cuda/fattn-common.cuh
+++ b/ggml/src/ggml-cuda/fattn-common.cuh
@@ -146,6 +146,34 @@ static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_bf16(
return sum;
}
+template <int D, int nthreads>
+static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_f8_e4m3(
+ const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) {
+
+ const uint8_t * K_f8 = (const uint8_t *) K_c;
+ GGML_UNUSED(Q_q8);
+ GGML_UNUSED(Q_ds_v);
+
+ float sum = 0.0f;
+
+#pragma unroll
+ for (int k_KQ_0 = 0; k_KQ_0 < D; k_KQ_0 += nthreads*4) {
+ uint8_t tmp[4];
+ ggml_cuda_memcpy_1<sizeof(tmp)>(tmp, K_f8 + k_KQ_0 + (threadIdx.x % nthreads)*4);
+#pragma unroll
+ for (int k_KQ_1 = 0; k_KQ_1 < 4; ++k_KQ_1) {
+ const float k = ggml_cuda_f8_e4m3_to_fp32(tmp[k_KQ_1]);
+#ifdef V_DOT2_F32_F16_AVAILABLE
+ sum += k * __half2float(((const half *) Q_v)[k_KQ_0/nthreads + k_KQ_1]);
+#else
+ sum += k * ((const float *) Q_v)[k_KQ_0/nthreads + k_KQ_1];
+#endif // V_DOT2_F32_F16_AVAILABLE
+ }
+ }
+
+ return sum;
+}
+
template<int D, int nthreads>
static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_q4_0(
const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) {
@@ -406,6 +434,19 @@ static __device__ __forceinline__ void dequantize_V_bf16(const void * __restrict
}
}
+template <typename T, int ne>
+static __device__ __forceinline__ void dequantize_V_f8_e4m3(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) {
+ static_assert(ne == 2 || ne == 4, "bad ne");
+ static_assert(std::is_same_v<T, half> || std::is_same_v<T, float>, "unsupported type");
+ ggml_fp8_e4m3_t tmp[ne];
+ ggml_cuda_memcpy_1<sizeof(tmp)>(tmp, (const ggml_fp8_e4m3_t *) vx + i0);
+
+#pragma unroll
+ for (int l = 0; l < ne; ++l) {
+ ((T *) dst)[l] = ggml_cuda_cast<T>(tmp[l]);
+ }
+}
+
template <typename T, int ne>
static __device__ __forceinline__ void dequantize_V_q4_0(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) {
const block_q4_0 * x = (const block_q4_0 *) vx;
@@ -644,6 +685,8 @@ constexpr __device__ vec_dot_KQ_t get_vec_dot_KQ() {
return vec_dot_fattn_vec_KQ_q8_0<D, nthreads>;
} else if constexpr (type_K == GGML_TYPE_BF16) {
return vec_dot_fattn_vec_KQ_bf16<D, nthreads>;
+ } else if constexpr (type_K == GGML_TYPE_F8_E4M3) {
+ return vec_dot_fattn_vec_KQ_f8_e4m3<D, nthreads>;
} else {
static_assert(type_K == -1, "bad type");
return nullptr;
@@ -666,6 +709,8 @@ constexpr __device__ dequantize_V_t get_dequantize_V() {
return dequantize_V_q8_0<T, ne>;
} else if constexpr (type_V == GGML_TYPE_BF16) {
return dequantize_V_bf16<float, ne>;
+ } else if constexpr (type_V == GGML_TYPE_F8_E4M3) {
+ return dequantize_V_f8_e4m3<T, ne>;
} else {
static_assert(type_V == -1, "bad type");
return nullptr;
diff --git a/ggml/src/ggml-cuda/fattn-vec.cuh b/ggml/src/ggml-cuda/fattn-vec.cuh
index 9df15128f..02c3c327f 100644
--- a/ggml/src/ggml-cuda/fattn-vec.cuh
+++ b/ggml/src/ggml-cuda/fattn-vec.cuh
@@ -96,7 +96,7 @@ static __global__ void flash_attn_ext_vec(
constexpr int V_cols_per_iter = WARP_SIZE / nthreads_V;
constexpr vec_dot_KQ_t vec_dot_KQ = get_vec_dot_KQ<type_K, D, nthreads_KQ>();
- constexpr bool Q_q8_1 = type_K != GGML_TYPE_F16 && type_K != GGML_TYPE_BF16;
+ constexpr bool Q_q8_1 = type_K != GGML_TYPE_F16 && type_K != GGML_TYPE_BF16 && type_K != GGML_TYPE_F8_E4M3;
#ifdef V_DOT2_F32_F16_AVAILABLE
constexpr dequantize_V_t dequantize_V = get_dequantize_V<type_V, half, V_rows_per_thread>();
#else
@@ -203,6 +203,27 @@ static __global__ void flash_attn_ext_vec(
}
__syncthreads();
+ } else if constexpr (type_K == GGML_TYPE_F8_E4M3) {
+#pragma unroll
+ for (int j = 0; j < ncols; ++j) {
+ const float * Q_j = (const float *) (Q + j*nb01);
+#pragma unroll
+ for (int i0 = 0; i0 < D; i0 += nthreads_KQ*4) {
+ const int i = i0 + (threadIdx.x % nthreads_KQ)*4;
+ float4 tmp = make_float4(0.0f, 0.0f, 0.0f, 0.0f);
+ if (ncols == 1 || ic0 + j < int(ne01.z)) {
+ ggml_cuda_memcpy_1<sizeof(tmp)>(&tmp, Q_j + i);
+ }
+#ifdef V_DOT2_F32_F16_AVAILABLE
+ const half2 scale_h2 = make_half2(scale, scale);
+ Q_reg[j][i0/(2*nthreads_KQ) + 0] = __hmul2(make_half2(tmp.x, tmp.y), scale_h2);
+ Q_reg[j][i0/(2*nthreads_KQ) + 1] = __hmul2(make_half2(tmp.z, tmp.w), scale_h2);
+#else
+ Q_reg[j][i0/(2*nthreads_KQ) + 0] = make_float2(tmp.x*scale, tmp.y*scale);
+ Q_reg[j][i0/(2*nthreads_KQ) + 1] = make_float2(tmp.z*scale, tmp.w*scale);
+#endif
+ }
+ }
} else {
#ifdef V_DOT2_F32_F16_AVAILABLE
const half2 scale_h2 = make_half2(scale, scale);
@@ -587,6 +608,7 @@ void ggml_cuda_flash_attn_ext_vec_case(ggml_backend_cuda_context & ctx, ggml_ten
extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q5_1); \
extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_Q8_0); \
extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_BF16); \
+ extern DECL_FATTN_VEC_CASE(D, type_K, GGML_TYPE_F8_E4M3); \
EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_F16)
EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q4_0)
@@ -595,6 +617,7 @@ EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q5_0)
EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q5_1)
EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_Q8_0)
EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_BF16)
+EXTERN_DECL_FATTN_VEC_CASES( 64, GGML_TYPE_F8_E4M3)
EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_F16)
EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q4_0)
@@ -603,6 +626,7 @@ EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q5_0)
EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q5_1)
EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_Q8_0)
EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_BF16)
+EXTERN_DECL_FATTN_VEC_CASES(128, GGML_TYPE_F8_E4M3)
EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_F16)
EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q4_0)
@@ -611,3 +635,4 @@ EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q5_0)
EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q5_1)
EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q8_0)
EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_BF16)
+EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_F8_E4M3)
diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu
index 5d7ebf966..49dfef14c 100644
--- a/ggml/src/ggml-cuda/fattn.cu
+++ b/ggml/src/ggml-cuda/fattn.cu
@@ -462,6 +462,7 @@ static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, co
FATTN_VEC_CASES_ALL_D(Q5_1, F16)
FATTN_VEC_CASES_ALL_D(Q8_0, F16)
FATTN_VEC_CASES_ALL_D(BF16, F16)
+ FATTN_VEC_CASES_ALL_D(F8_E4M3, F16)
FATTN_VEC_CASES_ALL_D(F16, Q4_0)
FATTN_VEC_CASES_ALL_D(Q4_0, Q4_0)
@@ -470,6 +471,7 @@ static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, co
FATTN_VEC_CASES_ALL_D(Q5_1, Q4_0)
FATTN_VEC_CASES_ALL_D(Q8_0, Q4_0)
FATTN_VEC_CASES_ALL_D(BF16, Q4_0)
+ FATTN_VEC_CASES_ALL_D(F8_E4M3, Q4_0)
FATTN_VEC_CASES_ALL_D(F16, Q4_1)
FATTN_VEC_CASES_ALL_D(Q4_0, Q4_1)
@@ -478,6 +480,7 @@ static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, co
FATTN_VEC_CASES_ALL_D(Q5_1, Q4_1)
FATTN_VEC_CASES_ALL_D(Q8_0, Q4_1)
FATTN_VEC_CASES_ALL_D(BF16, Q4_1)
+ FATTN_VEC_CASES_ALL_D(F8_E4M3, Q4_1)
FATTN_VEC_CASES_ALL_D(F16, Q5_0)
FATTN_VEC_CASES_ALL_D(Q4_0, Q5_0)
@@ -486,6 +489,7 @@ static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, co
FATTN_VEC_CASES_ALL_D(Q5_1, Q5_0)
FATTN_VEC_CASES_ALL_D(Q8_0, Q5_0)
FATTN_VEC_CASES_ALL_D(BF16, Q5_0)
+ FATTN_VEC_CASES_ALL_D(F8_E4M3, Q5_0)
FATTN_VEC_CASES_ALL_D(F16, Q5_1)
FATTN_VEC_CASES_ALL_D(Q4_0, Q5_1)
@@ -494,6 +498,7 @@ static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, co
FATTN_VEC_CASES_ALL_D(Q5_1, Q5_1)
FATTN_VEC_CASES_ALL_D(Q8_0, Q5_1)
FATTN_VEC_CASES_ALL_D(BF16, Q5_1)
+ FATTN_VEC_CASES_ALL_D(F8_E4M3, Q5_1)
FATTN_VEC_CASES_ALL_D(F16, Q8_0)
FATTN_VEC_CASES_ALL_D(Q4_0, Q8_0)
@@ -502,6 +507,7 @@ static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, co
FATTN_VEC_CASES_ALL_D(Q5_1, Q8_0)
FATTN_VEC_CASES_ALL_D(Q8_0, Q8_0)
FATTN_VEC_CASES_ALL_D(BF16, Q8_0)
+ FATTN_VEC_CASES_ALL_D(F8_E4M3, Q8_0)
FATTN_VEC_CASES_ALL_D(F16, BF16)
FATTN_VEC_CASES_ALL_D(Q4_0, BF16)
@@ -510,6 +516,16 @@ static fattn_vec_case_t ggml_cuda_get_fattn_vec_case(const int64_t head_size, co
FATTN_VEC_CASES_ALL_D(Q5_1, BF16)
FATTN_VEC_CASES_ALL_D(Q8_0, BF16)
FATTN_VEC_CASES_ALL_D(BF16, BF16)
+ FATTN_VEC_CASES_ALL_D(F8_E4M3, BF16)
+
+ FATTN_VEC_CASES_ALL_D(F16, F8_E4M3)
+ FATTN_VEC_CASES_ALL_D(Q4_0, F8_E4M3)
+ FATTN_VEC_CASES_ALL_D(Q4_1, F8_E4M3)
+ FATTN_VEC_CASES_ALL_D(Q5_0, F8_E4M3)
+ FATTN_VEC_CASES_ALL_D(Q5_1, F8_E4M3)
+ FATTN_VEC_CASES_ALL_D(Q8_0, F8_E4M3)
+ FATTN_VEC_CASES_ALL_D(BF16, F8_E4M3)
+ FATTN_VEC_CASES_ALL_D(F8_E4M3, F8_E4M3)
return nullptr;
}
@@ -553,6 +569,7 @@ static bool ggml_cuda_fattn_kv_type_supported(const ggml_type type) {
case GGML_TYPE_Q5_0:
case GGML_TYPE_Q5_1:
case GGML_TYPE_Q8_0:
+ case GGML_TYPE_F8_E4M3:
return true;
default:
return false;
diff --git a/ggml/src/ggml-cuda/fp8.cu b/ggml/src/ggml-cuda/fp8.cu
new file mode 100644
index 000000000..5fdec4220
--- /dev/null
+++ b/ggml/src/ggml-cuda/fp8.cu
@@ -0,0 +1,206 @@
+#include "fp8.cuh"
+
+#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11080
+
+struct fp8_abs_src {
+ const float * x;
+ int64_t ne0;
+ int64_t ne1;
+ int64_t ne2;
+ int64_t s1;
+ int64_t s2;
+ int64_t s3;
+
+ __device__ float operator()(int64_t i) const {
+ const int64_t i0 = i % ne0;
+ i /= ne0;
+ const int64_t i1 = i % ne1;
+ i /= ne1;
+ const int64_t i2 = i % ne2;
+ const int64_t i3 = i / ne2;
+ const float value = fabsf(x[i0 + i1*s1 + i2*s2 + i3*s3]);
+ return isfinite(value) ? value : 448.0f;
+ }
+};
+
+static __global__ void fp8_amax_partials(fp8_abs_src src, int64_t ne, float * partials) {
+ float amax = 0.0f;
+ for (int64_t i = (int64_t) blockIdx.x*blockDim.x + threadIdx.x; i < ne; i += (int64_t) blockDim.x*gridDim.x) {
+ amax = fmaxf(amax, src(i));
+ }
+
+ __shared__ float shared[WARP_SIZE];
+ amax = block_reduce<block_reduce_method::MAX, 256>(amax, shared);
+ if (threadIdx.x == 0) {
+ partials[blockIdx.x] = amax;
+ }
+}
+
+static __global__ void fp8_amax_final(const float * partials, int n, float * amax) {
+ float value = 0.0f;
+ for (int i = threadIdx.x; i < n; i += blockDim.x) {
+ value = fmaxf(value, partials[i]);
+ }
+
+ __shared__ float shared[WARP_SIZE];
+ value = block_reduce<block_reduce_method::MAX, 256>(value, shared);
+ if (threadIdx.x == 0) {
+ *amax = value;
+ }
+}
+
+static __global__ void quantize_fp8_e4m3(
+ const float * __restrict__ x, uint8_t * __restrict__ y, const float * __restrict__ amax,
+ float * __restrict__ scale, int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne, int64_t s1, int64_t s2, int64_t s3) {
+ const int64_t i = (int64_t) blockIdx.x*blockDim.x + threadIdx.x;
+ if (i >= ne) {
+ return;
+ }
+
+ int64_t tmp = i / ne0;
+ const int64_t i0 = i - tmp*ne0;
+ const int64_t i1 = tmp % ne1;
+ tmp /= ne1;
+ const int64_t i2 = tmp % ne2;
+ const int64_t i3 = tmp / ne2;
+
+ const float d = *amax > 0.0f ? *amax / 448.0f : 1.0f;
+ y[i] = ggml_cuda_fp32_to_f8_e4m3(x[i0 + i1*s1 + i2*s2 + i3*s3] / d);
+ if (i == 0) {
+ *scale = d;
+ }
+}
+
+static void fp8_set_batch(cublasLtMatrixLayout_t desc, int32_t count, int64_t stride) {
+ CUBLAS_CHECK(cublasLtMatrixLayoutSetAttribute(desc, CUBLASLT_MATRIX_LAYOUT_BATCH_COUNT, &count, sizeof(count)));
+ CUBLAS_CHECK(cublasLtMatrixLayoutSetAttribute(desc, CUBLASLT_MATRIX_LAYOUT_STRIDED_BATCH_OFFSET, &stride, sizeof(stride)));
+}
+
+static void fp8_destroy_matmul(
+ cublasLtMatmulDesc_t op_desc, cublasLtMatrixLayout_t a_desc, cublasLtMatrixLayout_t b_desc,
+ cublasLtMatrixLayout_t d_desc) {
+ CUBLAS_CHECK(cublasLtMatrixLayoutDestroy(d_desc));
+ CUBLAS_CHECK(cublasLtMatrixLayoutDestroy(b_desc));
+ CUBLAS_CHECK(cublasLtMatrixLayoutDestroy(a_desc));
+ CUBLAS_CHECK(cublasLtMatmulDescDestroy(op_desc));
+}
+
+bool ggml_cuda_mul_mat_fp8(
+ ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,
+ const ggml_tensor * weight_scale) {
+ const int cc = ggml_cuda_info().devices[ctx.device].cc;
+ if (!fp8_mma_hardware_available(cc) || src0->type != GGML_TYPE_F8_E4M3 || src1->type != GGML_TYPE_F32 ||
+ dst->type != GGML_TYPE_F32 || !ggml_is_contiguous(dst) || src0->ne[0] % 16 != 0 || src0->ne[1] % 16 != 0 ||
+ src0->nb[0] != sizeof(uint8_t) || src0->nb[1] != (size_t) src0->ne[0] || src1->nb[0] != sizeof(float) ||
+ !ggml_cuda_is_aligned(src0, 16) || !ggml_cuda_is_aligned(dst, 16)) {
+ return false;
+ }
+
+ GGML_ASSERT(!weight_scale || (weight_scale->type == GGML_TYPE_F32 && ggml_is_contiguous(weight_scale) && ggml_nelements(weight_scale) == 1));
+
+ GGML_TENSOR_BINARY_OP_LOCALS
+ GGML_ASSERT(ne10 == ne00);
+ GGML_ASSERT(ne0 == ne01);
+ GGML_ASSERT(ne12 % ne02 == 0);
+ GGML_ASSERT(ne13 % ne03 == 0);
+
+ cudaStream_t stream = ctx.stream();
+ const int64_t ne_src1 = ggml_nelements(src1);
+ ggml_cuda_pool_alloc<uint8_t> src1_fp8(ctx.pool(), ne_src1);
+ ggml_cuda_pool_alloc<float> src1_scale(ctx.pool(), 1);
+ ggml_cuda_pool_alloc<float> src1_amax(ctx.pool(), 1);
+
+ const fp8_abs_src abs_src = {
+ (const float *) src1->data, ne10, ne11, ne12,
+ (int64_t) (nb11 / sizeof(float)), (int64_t) (nb12 / sizeof(float)), (int64_t) (nb13 / sizeof(float))
+ };
+ const int reduce_blocks = std::min<int64_t>((ne_src1 + 255)/256, 1024);
+ ggml_cuda_pool_alloc<float> reduce_tmp(ctx.pool(), reduce_blocks);
+ fp8_amax_partials<<<reduce_blocks, 256, 0, stream>>>(abs_src, ne_src1, reduce_tmp.ptr);
+ fp8_amax_final<<<1, 256, 0, stream>>>(reduce_tmp.ptr, reduce_blocks, src1_amax.ptr);
+
+ quantize_fp8_e4m3<<<(ne_src1 + 255)/256, 256, 0, stream>>>(
+ (const float *) src1->data, src1_fp8.ptr, src1_amax.ptr, src1_scale.ptr, ne10, ne11, ne12, ne_src1,
+ nb11 / sizeof(float), nb12 / sizeof(float), nb13 / sizeof(float));
+ CUDA_CHECK(cudaGetLastError());
+
+ cublasLtMatmulDesc_t op_desc;
+ cublasLtMatrixLayout_t a_desc;
+ cublasLtMatrixLayout_t b_desc;
+ cublasLtMatrixLayout_t d_desc;
+ CUBLAS_CHECK(cublasLtMatmulDescCreate(&op_desc, CUBLAS_COMPUTE_32F, CUDA_R_32F));
+ const cublasOperation_t trans_a = CUBLAS_OP_T;
+ CUBLAS_CHECK(cublasLtMatmulDescSetAttribute(op_desc, CUBLASLT_MATMUL_DESC_TRANSA, &trans_a, sizeof(trans_a)));
+ CUBLAS_CHECK(cublasLtMatmulDescSetAttribute(
+ op_desc, CUBLASLT_MATMUL_DESC_B_SCALE_POINTER, &src1_scale.ptr, sizeof(src1_scale.ptr)));
+ if (weight_scale) {
+ const float * scale_ptr = (const float *) weight_scale->data;
+ CUBLAS_CHECK(cublasLtMatmulDescSetAttribute(
+ op_desc, CUBLASLT_MATMUL_DESC_A_SCALE_POINTER, &scale_ptr, sizeof(scale_ptr)));
+ }
+ const int64_t r2 = ne12 / ne02;
+ const int64_t r3 = ne13 / ne03;
+
+ // src1 batches that share a src0 matrix are adjacent in the compact src1 copy and in dst, fold them into N
+ const int64_t n = ne11*(ne02 == 1 ? r2*r3 : r2);
+
+ // batch the remaining matmuls with a fixed src0 stride, or issue one batched call per i3 if there is none
+ int64_t batch_count = ne02;
+ int64_t stride_a = nb02;
+ int64_t n_calls = ne13;
+ if (ne02 == 1) {
+ batch_count = ne03;
+ stride_a = nb03;
+ n_calls = 1;
+ } else if (r3 == 1 && (ne03 == 1 || nb03 == ne02*nb02)) {
+ batch_count = ne02*ne03;
+ n_calls = 1;
+ }
+
+ CUBLAS_CHECK(cublasLtMatrixLayoutCreate(&a_desc, CUDA_R_8F_E4M3, ne00, ne01, ne00));
+ CUBLAS_CHECK(cublasLtMatrixLayoutCreate(&b_desc, CUDA_R_8F_E4M3, ne10, n, ne10));
+ CUBLAS_CHECK(cublasLtMatrixLayoutCreate(&d_desc, CUDA_R_32F, ne0, n, ne0));
+ if (batch_count > 1) {
+ fp8_set_batch(a_desc, batch_count, stride_a);
+ fp8_set_batch(b_desc, batch_count, ne10*n);
+ fp8_set_batch(d_desc, batch_count, ne0*n);
+ }
+
+ // TODO: Allow workspace in the heuristic search for potential performance gains.
+ // cuBLASLt takes caller-owned workspace per matmul call.
+ cublasLtMatmulPreference_t preference;
+ CUBLAS_CHECK(cublasLtMatmulPreferenceCreate(&preference));
+ cublasLtMatmulHeuristicResult_t heuristic;
+ int returned = 0;
+ const cublasStatus_t status = cublasLtMatmulAlgoGetHeuristic(
+ ctx.cublaslt_handle(), op_desc, a_desc, b_desc, d_desc, d_desc, preference, 1, &heuristic, &returned);
+ CUBLAS_CHECK(cublasLtMatmulPreferenceDestroy(preference));
+ if (status != CUBLAS_STATUS_SUCCESS || returned == 0) {
+ fp8_destroy_matmul(op_desc, a_desc, b_desc, d_desc);
+ return false;
+ }
+
+ const float alpha = 1.0f;
+ const float beta = 0.0f;
+ for (int64_t i3 = 0; i3 < n_calls; ++i3) {
+ const char * a = (const char *) src0->data + (i3/r3)*nb03;
+ const uint8_t * b = src1_fp8.ptr + i3*ne12*ne11*ne10;
+ float * d = (float *) ((char *) dst->data + i3*dst->nb[3]);
+ CUBLAS_CHECK(cublasLtMatmul(ctx.cublaslt_handle(), op_desc, &alpha, a, a_desc, b, b_desc,
+ &beta, d, d_desc, d, d_desc, &heuristic.algo, nullptr, 0, stream));
+ }
+
+ fp8_destroy_matmul(op_desc, a_desc, b_desc, d_desc);
+ return true;
+}
+
+#else
+
+bool ggml_cuda_mul_mat_fp8(
+ ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,
+ const ggml_tensor * weight_scale) {
+ GGML_UNUSED_VARS(ctx, src0, src1, dst, weight_scale);
+ return false;
+}
+
+#endif
diff --git a/ggml/src/ggml-cuda/fp8.cuh b/ggml/src/ggml-cuda/fp8.cuh
new file mode 100644
index 000000000..069349ed9
--- /dev/null
+++ b/ggml/src/ggml-cuda/fp8.cuh
@@ -0,0 +1,7 @@
+#pragma once
+
+#include "common.cuh"
+
+bool ggml_cuda_mul_mat_fp8(
+ ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,
+ const ggml_tensor * weight_scale = nullptr);
diff --git a/ggml/src/ggml-cuda/getrows.cu b/ggml/src/ggml-cuda/getrows.cu
index d97e8fdb0..09b7f02b8 100644
--- a/ggml/src/ggml-cuda/getrows.cu
+++ b/ggml/src/ggml-cuda/getrows.cu
@@ -316,6 +316,10 @@ static void ggml_cuda_get_rows_switch_src0_type(
get_rows_cuda_float((const nv_bfloat16 *) src0_d, src1_d, dst_d,
ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
break;
+ case GGML_TYPE_F8_E4M3:
+ get_rows_cuda_float((const ggml_fp8_e4m3_t *) src0_d, src1_d, dst_d,
+ ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
+ break;
case GGML_TYPE_Q1_0:
get_rows_cuda_q<QK1_0, QR1_0, dequantize_q1_0>(src0_d, src1_d, dst_d,
ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream);
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index 848565460..17661c19f 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -28,6 +28,7 @@
#include "ggml-cuda/diagmask.cuh"
#include "ggml-cuda/diag.cuh"
#include "ggml-cuda/fattn.cuh"
+#include "ggml-cuda/fp8.cuh"
#include "ggml-cuda/fwht.cuh"
#include "ggml-cuda/getrows.cuh"
#include "ggml-cuda/im2col.cuh"
@@ -890,6 +891,11 @@ ggml_backend_cuda_context::~ggml_backend_cuda_context() {
CUDA_CHECK(cudaFree(cublas_workspaces[i][j]));
}
}
+#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && CUDART_VERSION >= 11080
+ if (cublaslt_handles[i] != nullptr) {
+ CUBLAS_CHECK(cublasLtDestroy(cublaslt_handles[i]));
+ }
+#endif
}
}
@@ -2000,7 +2006,7 @@ static void ggml_cuda_mul_mat_cublas(ggml_backend_cuda_context & ctx, const ggml
const int cc = ggml_cuda_info().devices[ctx.device].cc;
const ggml_prec prec = ggml_prec_get_acc(dst);
ggml_type compute_type = src0->type;
- if (ggml_is_quantized(compute_type)) {
+ if (ggml_is_quantized(compute_type) || compute_type == GGML_TYPE_F8_E4M3) {
compute_type = fast_fp16_hardware_available(cc) ? GGML_TYPE_F16 : GGML_TYPE_F32;
} else if (compute_type == GGML_TYPE_F16 && !fast_fp16_hardware_available(cc)) {
compute_type = GGML_TYPE_F32;
@@ -2171,7 +2177,7 @@ static bool ggml_cuda_should_fuse_mul_mat(const ggml_tensor * ffn_up,
return true;
}
-static bool ggml_cuda_should_fuse_mul_mat_vec_f(const ggml_tensor * tensor) {
+static bool ggml_cuda_should_fuse_mul_mat_vec_f(const ggml_tensor * tensor, const ggml_tensor * gate = nullptr) {
ggml_tensor * src0 = tensor->src[0];
ggml_tensor * src1 = tensor->src[1];
const ggml_tensor * dst = tensor;
@@ -2179,12 +2185,17 @@ static bool ggml_cuda_should_fuse_mul_mat_vec_f(const ggml_tensor * tensor) {
const bool is_mul_mat_id = tensor->op == GGML_OP_MUL_MAT_ID;
bool use_mul_mat_vec_f =
- (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16) &&
+ (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16 ||
+ src0->type == GGML_TYPE_F8_E4M3) &&
src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32;
const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size;
- use_mul_mat_vec_f = use_mul_mat_vec_f && ggml_cuda_should_use_mmvf(src0->type, cc, warp_size, src0->ne, src0->nb, is_mul_mat_id ? src1->ne[2] : src1->ne[1]);
+ use_mul_mat_vec_f = use_mul_mat_vec_f && ggml_cuda_should_use_mmvf(src0, cc, warp_size, is_mul_mat_id ? src1->ne[2] : src1->ne[1]);
+
+ if (gate && !ggml_cuda_should_use_mmvf(gate->src[0], cc, warp_size, is_mul_mat_id ? src1->ne[2] : src1->ne[1])) {
+ return false;
+ }
//we only support fusion for ncols_dst = 1
if (tensor->op == GGML_OP_MUL_MAT && dst->ne[1] != 1) {
@@ -2208,11 +2219,11 @@ static bool ggml_cuda_should_fuse_mul_mat_vec_q(const ggml_tensor * tensor) {
ggml_nbytes(src0) != ggml_backend_buffer_get_alloc_size(src0->buffer, src0) &&
src0->view_src;
- bool use_mul_mat_vec_q = ggml_is_quantized(src0->type) && !bad_padding_clear && src1->type == GGML_TYPE_F32 &&
- dst->type == GGML_TYPE_F32 && src1->ne[1] <= MMVQ_MAX_BATCH_SIZE;
+ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
+ bool use_mul_mat_vec_q = ggml_cuda_should_use_mmvq(src0->type, cc, src1->ne[1]) && !bad_padding_clear &&
+ src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32;
// fusion is not universally faster on Pascal
- const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
if (cc <= GGML_CUDA_CC_PASCAL) {
return false;
}
@@ -2280,7 +2291,7 @@ static bool ggml_cuda_match_shared_expert(const ggml_cgraph * graph, int routed_
return true;
}
-// The NVFP4 weight scale that scale_node (MUL) applies to the output of mm_node (MUL_MAT), or nullptr.
+// The weight scale that scale_node (MUL) applies to the output of mm_node (MUL_MAT), or nullptr.
// Shared by ggml_cuda_try_fuse and graph_optimize so that both accept the same patterns.
static const ggml_tensor * ggml_cuda_get_mul_mat_scale(const ggml_tensor * scale_node, const ggml_tensor * mm_node) {
const bool scale_lhs_mm = scale_node->src[0] == mm_node;
@@ -2290,7 +2301,7 @@ static const ggml_tensor * ggml_cuda_get_mul_mat_scale(const ggml_tensor * scale
}
const ggml_tensor * scale = scale_lhs_mm ? scale_node->src[1] : scale_node->src[0];
- if (mm_node->src[0]->type != GGML_TYPE_NVFP4 || scale_node->type != GGML_TYPE_F32 ||
+ if ((mm_node->src[0]->type != GGML_TYPE_NVFP4 && mm_node->src[0]->type != GGML_TYPE_F8_E4M3) || scale_node->type != GGML_TYPE_F32 ||
scale->type != GGML_TYPE_F32 || !ggml_is_contiguous(scale) || ggml_nelements(scale) != 1 ||
!ggml_are_same_shape(scale_node, mm_node)) {
return nullptr;
@@ -2311,7 +2322,7 @@ static const ggml_tensor * ggml_cuda_get_mul_mat_id_scale(const ggml_tensor * re
}
const ggml_tensor * scale = reshape->src[0];
- if (mm_node->src[0]->type != GGML_TYPE_NVFP4 || scale_node->type != GGML_TYPE_F32 ||
+ if ((mm_node->src[0]->type != GGML_TYPE_NVFP4 && mm_node->src[0]->type != GGML_TYPE_F8_E4M3) || scale_node->type != GGML_TYPE_F32 ||
scale->type != GGML_TYPE_F32 || !ggml_is_contiguous(scale) || ggml_nelements(scale) != mm_node->src[0]->ne[2] ||
!ggml_are_same_shape(scale_node, mm_node)) {
return nullptr;
@@ -2409,17 +2420,17 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor
const int cc = ggml_cuda_info().devices[ctx.device].cc;
const int warp_size = ggml_cuda_info().devices[ctx.device].warp_size;
- if (ggml_cuda_should_use_mmvf(src0->type, cc, warp_size, src0->ne, src0->nb, ne11)) {
- // The custom F16 vector kernel can be used over batched cuBLAS GEMM.
+ if (ggml_cuda_should_use_mmvf(src0, cc, warp_size, ne11)) {
+ // The custom vector kernel can be used over batched cuBLAS GEMM.
// But this is only faster for GPUs without tensor cores or with a thin src0 matrix (particularly KQV in attention)
ggml_cuda_mul_mat_vec_f(ctx, src0, src1, nullptr, dst);
return;
}
- // A transposed vector can still use MMVQ (i.e. ne01 == 1)
+ // A transposed vector can still use MMVF (i.e. ne01 == 1)
if (ne01 == 1 && ne11 > MMVF_MAX_BATCH_SIZE && ne2 == 1 && ne3 == 1
&& src0->type == GGML_TYPE_F32
&& ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst)
- && ggml_cuda_should_use_mmvf(src1->type, cc, warp_size, src1->ne, src1->nb, /*ne11 =*/ 1)) {
+ && ggml_cuda_should_use_mmvf(src1, cc, warp_size, /*ne11 =*/ 1)) {
ggml_tensor dst_vec = *dst;
dst_vec.ne[0] = ne11;
dst_vec.ne[1] = 1;
@@ -2441,6 +2452,9 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor
ggml_cuda_mul_mat_q(ctx, src0, src1, nullptr, dst);
return;
}
+ if (src0->type == GGML_TYPE_F8_E4M3 && ggml_cuda_mul_mat_fp8(ctx, src0, src1, dst)) {
+ return;
+ }
ggml_cuda_mul_mat_cublas(ctx, src0, src1, dst);
}
@@ -2449,7 +2463,6 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor
static bool ggml_cuda_mul_mat_id_needs_sync(const ggml_tensor * dst, const int cc) {
const ggml_tensor * src0 = dst->src[0];
const ggml_tensor * src1 = dst->src[1];
-
const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size;
if (src1->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) {
@@ -2461,7 +2474,10 @@ static bool ggml_cuda_mul_mat_id_needs_sync(const ggml_tensor * dst, const int c
if (dst->ne[2] <= get_mmvq_mmid_max_batch(src0->type, cc)) {
return false;
}
- } else if (GGML_CUDA_CC_IS_AMD(cc)) {
+ } else if (src0->type == GGML_TYPE_F8_E4M3 &&
+ ggml_cuda_should_use_mmvf(src0, cc, warp_size, dst->ne[2])) {
+ return false;
+ } else if (src0->type != GGML_TYPE_F8_E4M3 && GGML_CUDA_CC_IS_AMD(cc)) {
return false;
}
}
@@ -2487,7 +2503,7 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *
GGML_TENSOR_BINARY_OP_LOCALS
- const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
+ const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size;
// [TAG_MUL_MAT_ID_CUDA_GRAPHS]
@@ -2500,8 +2516,12 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *
ggml_cuda_mul_mat_vec_q(ctx, src0, src1, ids, dst);
return;
}
+ } else if (src0->type == GGML_TYPE_F8_E4M3 &&
+ ggml_cuda_should_use_mmvf(src0, cc, warp_size, ne2)) {
+ ggml_cuda_mul_mat_vec_f(ctx, src0, src1, ids, dst);
+ return;
} else {
- if (GGML_CUDA_CC_IS_AMD(cc)) {
+ if (src0->type != GGML_TYPE_F8_E4M3 && GGML_CUDA_CC_IS_AMD(cc)) {
ggml_cuda_mul_mat_vec_f(ctx, src0, src1, ids, dst);
return;
}
@@ -2527,7 +2547,7 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *
GGML_ASSERT(nb2 % nb1 == 0);
const ggml_type type_src1_sorted = (src0->type == GGML_TYPE_F16 && !fast_fp16_hardware_available(cc))
- || ggml_is_quantized(src0->type) ? GGML_TYPE_F32 : src0->type;
+ || ggml_is_quantized(src0->type) || src0->type == GGML_TYPE_F8_E4M3 ? GGML_TYPE_F32 : src0->type;
const ggml_type type_dst_sorted = GGML_TYPE_F32;
const size_t ts_src1_sorted = ggml_type_size(type_src1_sorted);
const size_t ts_dst_sorted = ggml_type_size(type_dst_sorted);
@@ -4553,6 +4573,13 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
fusion_data.glu_op = ggml_get_glu_op(glu);
fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3);
+ if (ggml_cuda_should_fuse_mul_mat_vec_f(up_n, gate_n)) {
+ ggml_cuda_mul_mat_vec_f(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data);
+ fused_mul_mat_vec = true;
+ fused_node_count = n_ops;
+ break;
+ }
+
if (ggml_cuda_should_fuse_mul_mat_vec_q(up_n)) {
ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data);
fused_mul_mat_vec = true;
@@ -4655,6 +4682,13 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
fusion_data.glu_op = ggml_get_glu_op(glu);
fusion_data.glu_limit = ggml_get_op_params_f32(glu, 3);
+ if (ggml_cuda_should_fuse_mul_mat_vec_f(up_n, gate_n)) {
+ ggml_cuda_mul_mat_vec_f(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data);
+ fused_mul_mat_vec = true;
+ fused_node_count = n_ops;
+ break;
+ }
+
if (ggml_cuda_should_fuse_mul_mat_vec_q(up_n)) {
ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, cgraph->nodes[glu_idx], &fusion_data);
fused_mul_mat_vec = true;
@@ -4712,7 +4746,7 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
const ggml_tensor * src1 = up_n->src[1];
const ggml_tensor * ids = up_n->src[2];
- if (ggml_cuda_should_fuse_mul_mat_vec_f(up_n)) {
+ if (ggml_cuda_should_fuse_mul_mat_vec_f(up_n, gate_n)) {
ggml_cuda_mm_fusion_args_host fusion_data{};
fusion_data.gate = gate_n->src[0];
fusion_data.x_bias = up_bias_tensor;
@@ -4755,7 +4789,7 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
const ggml_tensor * src1 = up->src[1];
const ggml_tensor * ids = up->src[2];
- if (ggml_cuda_should_fuse_mul_mat_vec_f(up)) {
+ if (ggml_cuda_should_fuse_mul_mat_vec_f(up, gate)) {
ggml_cuda_mm_fusion_args_host fusion_data{};
fusion_data.gate = gate->src[0];
fusion_data.glu_op = ggml_get_glu_op(glu);
@@ -4873,6 +4907,19 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
fusion_data.x_bias = bias;
fusion_data.x_scale = scale;
+ if (ggml_cuda_should_fuse_mul_mat_vec_f(mm_node)) {
+ ggml_cuda_mul_mat_vec_f(*cuda_ctx, src0, src1, ids, out_node, &fusion_data);
+ fused_mul_mat_vec = true;
+ fused_node_count = n_ops;
+ break;
+ }
+
+ if (op == GGML_OP_MUL_MAT && !with_bias && src0->type == GGML_TYPE_F8_E4M3 &&
+ ggml_get_op_params_i32(mm_node, 3) != GGML_PREC_F32 &&
+ ggml_cuda_mul_mat_fp8(*cuda_ctx, src0, src1, scale_node, scale)) {
+ return 1;
+ }
+
if (ggml_cuda_should_fuse_mul_mat_vec_q(mm_node)) {
ggml_cuda_mul_mat_vec_q(*cuda_ctx, src0, src1, ids, out_node, &fusion_data);
fused_mul_mat_vec = true;
@@ -6112,7 +6159,10 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
const int cc_dev = ggml_cuda_info().devices[dev_ctx->device].cc;
const bool f32_src1_ok = GGML_CUDA_CC_IS_NVIDIA(cc_dev) && cc_dev < GGML_CUDA_CC_AMPERE;
const enum ggml_prec prec_src1 = ggml_prec_get_src(op, 1);
- if (prec_src1 == GGML_PREC_F32 && (op->op == GGML_OP_MUL_MAT_ID || !f32_src1_ok || a->type != GGML_TYPE_F32)) {
+ const bool fp8_id_mmvf = op->op == GGML_OP_MUL_MAT_ID &&
+ a->type == GGML_TYPE_F8_E4M3 && b->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 &&
+ ggml_cuda_should_use_mmvf(a, cc_dev, ggml_cuda_info().devices[dev_ctx->device].warp_size, b->ne[2]);
+ if (prec_src1 == GGML_PREC_F32 && !fp8_id_mmvf && (op->op == GGML_OP_MUL_MAT_ID || !f32_src1_ok || a->type != GGML_TYPE_F32)) {
return false;
}
if (prec_src1 == GGML_PREC_BF16 && (op->op == GGML_OP_MUL_MAT_ID || a->type != GGML_TYPE_F32)) {
@@ -6143,6 +6193,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
case GGML_TYPE_Q8_0:
case GGML_TYPE_MXFP4:
case GGML_TYPE_NVFP4:
+ case GGML_TYPE_F8_E4M3:
case GGML_TYPE_Q2_K:
case GGML_TYPE_Q3_K:
case GGML_TYPE_Q4_K:
@@ -6180,6 +6231,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
case GGML_TYPE_Q5_0:
case GGML_TYPE_Q5_1:
case GGML_TYPE_Q8_0:
+ case GGML_TYPE_F8_E4M3:
case GGML_TYPE_Q2_K:
case GGML_TYPE_Q3_K:
case GGML_TYPE_Q4_K:
@@ -6209,7 +6261,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
{
return (
(
- (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_BF16 ||
+ (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_BF16 || op->type == GGML_TYPE_F8_E4M3 ||
op->type == GGML_TYPE_Q4_0 || op->type == GGML_TYPE_Q4_1 || op->type == GGML_TYPE_Q5_0 ||
op->type == GGML_TYPE_Q5_1 || op->type == GGML_TYPE_Q8_0 || op->type == GGML_TYPE_IQ4_NL) &&
op->src[0]->type == GGML_TYPE_F32
@@ -6230,6 +6282,16 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
{
ggml_type src0_type = op->src[0]->type;
ggml_type src1_type = op->src[1]->type;
+ if (src0_type == GGML_TYPE_F8_E4M3 && src1_type == GGML_TYPE_F8_E4M3) {
+ return true;
+ }
+ if ((src0_type == GGML_TYPE_F32 || src0_type == GGML_TYPE_F16 || src0_type == GGML_TYPE_BF16) &&
+ src1_type == GGML_TYPE_F8_E4M3) {
+ return true;
+ }
+ if (src0_type == GGML_TYPE_F8_E4M3 && src1_type == GGML_TYPE_F32) {
+ return true;
+ }
if ((src0_type == GGML_TYPE_F32 || src0_type == GGML_TYPE_BF16 || src0_type == GGML_TYPE_F16) &&
(src1_type == GGML_TYPE_F32 || src1_type == GGML_TYPE_BF16 || src1_type == GGML_TYPE_F16)
) {
@@ -6647,12 +6709,21 @@ static ggml_backend_feature * ggml_backend_cuda_get_features(ggml_backend_reg_t
{
const auto & info = ggml_cuda_info();
+ bool has_fp8 = false;
+ bool has_fp4 = false;
for (int id = 0; id < info.device_count; ++id) {
- if (blackwell_mma_available(info.devices[id].cc)) {
- features.push_back({ "BLACKWELL_NATIVE_FP4", "1"});
+ has_fp8 |= fp8_mma_hardware_available(info.devices[id].cc);
+ has_fp4 |= blackwell_mma_available(info.devices[id].cc);
+ if (has_fp8 && has_fp4) {
break;
}
}
+ if (has_fp8) {
+ features.push_back({ "NATIVE_FP8", "1"});
+ }
+ if (has_fp4) {
+ features.push_back({ "BLACKWELL_NATIVE_FP4", "1"});
+ }
}
#undef _STRINGIFY
diff --git a/ggml/src/ggml-cuda/mmvf.cu b/ggml/src/ggml-cuda/mmvf.cu
index 6008305ef..d96dbd04d 100644
--- a/ggml/src/ggml-cuda/mmvf.cu
+++ b/ggml/src/ggml-cuda/mmvf.cu
@@ -5,6 +5,14 @@
#include "mmf.cuh"
#include "convert.cuh"
+static __device__ __forceinline__ float2 mmvf_load_fp8x2(const ggml_fp8_e4m3_t * x, const int col2) {
+#if defined(FP8_AVAILABLE) && !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
+ return ggml_cuda_cast<float2>(reinterpret_cast<const __nv_fp8x2_e4m3 *>(x)[col2]);
+#else
+ return make_float2(ggml_cuda_cast<float>(x[2*col2]), ggml_cuda_cast<float>(x[2*col2 + 1]));
+#endif
+}
+
template <typename T, typename type_acc, int ncols_dst, int block_size, bool has_fusion = false, bool is_multi_token_id = false>
static __global__ void mul_mat_vec_f(
const T * x_ptr, const float * y_ptr, const int32_t * ids_ptr, const ggml_cuda_mm_fusion_args_device fusion, float * dst_ptr,
@@ -56,16 +64,24 @@ static __global__ void mul_mat_vec_f(
bool use_gate = false;
bool use_bias = false;
bool use_gate_bias = false;
+ bool use_scale = false;
+ bool use_gate_scale = false;
ggml_glu_op glu_op = ggml_glu_op::GGML_GLU_OP_SWIGLU;
float glu_limit = 0.0f;
const T * gate_x = nullptr;
const float * x_bias = nullptr;
const float * gate_bias = nullptr;
+ const float * x_scale = nullptr;
+ const float * gate_scale = nullptr;
if constexpr (has_fusion) {
use_gate = fusion.gate != nullptr;
use_bias = fusion.x_bias != nullptr;
use_gate_bias = fusion.gate_bias != nullptr;
+ if constexpr (std::is_same_v<T, ggml_fp8_e4m3_t>) {
+ use_scale = fusion.x_scale != nullptr;
+ use_gate_scale = fusion.gate_scale != nullptr && use_gate;
+ }
glu_op = fusion.glu_op;
glu_limit = fusion.glu_limit;
@@ -81,6 +97,12 @@ static __global__ void mul_mat_vec_f(
} else {
use_gate_bias = false;
}
+ if (use_scale) {
+ x_scale = static_cast<const float *>(fusion.x_scale);
+ }
+ if (use_gate_scale) {
+ gate_scale = static_cast<const float *>(fusion.gate_scale);
+ }
}
if (use_gate) {
@@ -97,8 +119,6 @@ static __global__ void mul_mat_vec_f(
}
}
- const float2 * y2 = (const float2 *) y;
-
extern __shared__ char data_mmv[];
float * buf_iw = (float *) data_mmv;
[[maybe_unused]] float * buf_iw_gate = nullptr;
@@ -128,6 +148,7 @@ static __global__ void mul_mat_vec_f(
}
if constexpr (std::is_same_v<T, float>) {
+ const float2 * y2 = (const float2 *) y;
const float2 * x2 = (const float2 *) x;
[[maybe_unused]] const float2 * gate_x2 = nullptr;
if constexpr (has_fusion) {
@@ -160,6 +181,7 @@ static __global__ void mul_mat_vec_f(
}
}
} else if constexpr (std::is_same_v<T, half>) {
+ const float2 * y2 = (const float2 *) y;
const half2 * x2 = (const half2 *) x;
[[maybe_unused]] const half2 * gate_x2 = nullptr;
if constexpr (has_fusion) {
@@ -235,6 +257,7 @@ static __global__ void mul_mat_vec_f(
#endif // FP16_AVAILABLE
}
} else if constexpr (std::is_same_v<T, nv_bfloat16>) {
+ const float2 * y2 = (const float2 *) y;
//TODO: add support for ggml_cuda_mad for hip_bfloat162
#if defined(GGML_USE_HIP)
const int * x2 = (const int *) x;
@@ -301,6 +324,28 @@ static __global__ void mul_mat_vec_f(
}
}
#endif
+ } else if constexpr (std::is_same_v<T, ggml_fp8_e4m3_t>) {
+ const float2 * y2 = (const float2 *) y;
+ for (int col2 = tid; col2 < ncols2; col2 += block_size) {
+ const float2 tmpx = mmvf_load_fp8x2(x, col2);
+ float2 tmpx_gate = {};
+ if constexpr (has_fusion) {
+ if (use_gate) {
+ tmpx_gate = mmvf_load_fp8x2(gate_x, col2);
+ }
+ }
+#pragma unroll
+ for (int j = 0; j < ncols_dst; ++j) {
+ const float2 tmpy = y2[j*stride_col_y2 + col2];
+ ggml_cuda_mad(sumf[j], tmpx, tmpy);
+
+ if constexpr (has_fusion) {
+ if (use_gate) {
+ ggml_cuda_mad(sumf_gate[j], tmpx_gate, tmpy);
+ }
+ }
+ }
+ }
} else {
static_assert(std::is_same_v<T, void>, "unsupported type");
}
@@ -348,12 +393,18 @@ static __global__ void mul_mat_vec_f(
float value = sumf[tid];
if constexpr (has_fusion) {
+ if (use_scale) {
+ value *= x_scale[ids ? channel_x : 0];
+ }
if (use_bias) {
value += x_bias[tid*stride_col_dst + row];
}
if (use_gate) {
float gate_value = sumf_gate[tid];
+ if (use_gate_scale) {
+ gate_value *= gate_scale[ids ? channel_x : 0];
+ }
if (use_gate_bias) {
gate_value += gate_bias[tid*stride_col_dst + row];
}
@@ -380,7 +431,11 @@ static __global__ void mul_mat_vec_f(
dst[tid*stride_col_dst + row] = value;
if constexpr (!has_fusion) {
- GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, glu_op, glu_limit, gate_x, x_bias, gate_bias, sumf_gate);
+ GGML_UNUSED_VARS(use_gate, use_bias, use_gate_bias, use_scale, use_gate_scale, glu_op, glu_limit, gate_x, x_bias, gate_bias,
+ x_scale, gate_scale, sumf_gate);
+ }
+ if constexpr (!std::is_same_v<T, ggml_fp8_e4m3_t>) {
+ GGML_UNUSED_VARS(use_scale, use_gate_scale, x_scale, gate_scale);
}
}
@@ -395,7 +450,8 @@ static void mul_mat_vec_f_switch_fusion(
const ggml_cuda_kernel_launch_params launch_params = {block_nums, block_dims, nbytes_shared, stream};
- const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr;
+ const bool has_fusion = fusion.gate != nullptr || fusion.x_bias != nullptr || fusion.gate_bias != nullptr ||
+ fusion.x_scale != nullptr || fusion.gate_scale != nullptr;
if constexpr (ncols_dst == 1) {
if (has_fusion) {
ggml_cuda_kernel_launch(mul_mat_vec_f<T, type_acc, ncols_dst, block_size, true, is_multi_token_id>, launch_params,
@@ -649,6 +705,7 @@ void ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensor
GGML_ASSERT( nb10 == ts_src1);
GGML_ASSERT(!ids || ids->nb[0] == ggml_type_size(ids->type));
GGML_ASSERT( nb0 == ts_dst);
+ GGML_ASSERT(src0->type != GGML_TYPE_F8_E4M3 || (uintptr_t) src0->data % alignof(uint16_t) == 0);
const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
// the fused GLU path gets the GLU node as dst, which carries no matmul precision, so it accumulates in F32
@@ -663,6 +720,7 @@ void ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensor
if (fusion) {
GGML_ASSERT( !ids || dst->ne[2] == 1);
GGML_ASSERT( ids || dst->ne[1] == 1);
+ GGML_ASSERT((fusion->x_scale == nullptr && fusion->gate_scale == nullptr) || src0->type == GGML_TYPE_F8_E4M3);
if (fusion->x_bias) {
GGML_ASSERT(fusion->x_bias->type == GGML_TYPE_F32);
GGML_ASSERT(fusion->x_bias->ne[0] == dst->ne[0]);
@@ -671,6 +729,7 @@ void ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensor
}
if (fusion->gate) {
GGML_ASSERT(fusion->gate->type == src0->type && ggml_are_same_stride(fusion->gate, src0));
+ GGML_ASSERT(src0->type != GGML_TYPE_F8_E4M3 || (uintptr_t) fusion->gate->data % alignof(uint16_t) == 0);
fusion_local.gate = fusion->gate->data;
}
if (fusion->gate_bias) {
@@ -679,6 +738,18 @@ void ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensor
GGML_ASSERT(!ids || fusion->gate_bias->ne[1] == src0->ne[2]);
fusion_local.gate_bias = fusion->gate_bias->data;
}
+ if (fusion->x_scale) {
+ GGML_ASSERT(fusion->x_scale->type == GGML_TYPE_F32);
+ GGML_ASSERT(ggml_is_contiguous(fusion->x_scale));
+ GGML_ASSERT(ggml_nelements(fusion->x_scale) == (ids ? src0->ne[2] : 1));
+ fusion_local.x_scale = fusion->x_scale->data;
+ }
+ if (fusion->gate_scale) {
+ GGML_ASSERT(fusion->gate_scale->type == GGML_TYPE_F32);
+ GGML_ASSERT(ggml_is_contiguous(fusion->gate_scale));
+ GGML_ASSERT(ggml_nelements(fusion->gate_scale) == (ids ? src0->ne[2] : 1));
+ fusion_local.gate_scale = fusion->gate_scale->data;
+ }
fusion_local.glu_op = fusion->glu_op;
fusion_local.glu_limit = fusion->glu_limit;
}
@@ -723,6 +794,12 @@ void ggml_cuda_mul_mat_vec_f(ggml_backend_cuda_context & ctx, const ggml_tensor
ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst,
ne03, ne3, s03, s13, s3, ids_stride, prec, ctx.stream());
} break;
+ case GGML_TYPE_F8_E4M3: {
+ const ggml_fp8_e4m3_t * src0_d = (const ggml_fp8_e4m3_t *) src0->data;
+ mul_mat_vec_f_cuda(src0_d, src1_d, ids_d, fusion_local, dst_d, ne00, ne01, ncols_dst, s01, stride_col_y, stride_col_dst,
+ ne02, nchannels_y, nchannels_dst, s02, stride_channel_y, stride_channel_dst,
+ ne03, ne3, s03, s13, s3, ids_stride, prec, ctx.stream());
+ } break;
default:
GGML_ABORT("unsupported type: %s", ggml_type_name(src0->type));
}
@@ -789,7 +866,11 @@ void ggml_cuda_op_mul_mat_vec_f(
GGML_UNUSED_VARS(ctx, src1, dst, src1_ddq_i, src1_ncols, src1_padded_row_size);
}
-bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, int warp_size, const int64_t * src0_ne, const size_t * src0_nb, int64_t ne11) {
+bool ggml_cuda_should_use_mmvf(const ggml_tensor * src0, int cc, int warp_size, int64_t ne11) {
+ const ggml_type type = src0->type;
+ const int64_t * src0_ne = src0->ne;
+ const size_t * src0_nb = src0->nb;
+
if (src0_ne[0] % 2 != 0) {
return false;
}
@@ -799,11 +880,8 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, int warp_size, const
return false;
}
- // Pointers not aligned to the size of half2/nv_bfloat162/float2 would result in a crash:
- for (size_t i = 1; i < GGML_MAX_DIMS; ++i) {
- if (src0_nb[i] % (2*ts) != 0) {
- return false;
- }
+ if (!ggml_cuda_is_aligned(src0, 2*ts)) {
+ return false;
}
switch (type) {
@@ -883,6 +961,8 @@ bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, int warp_size, const
return ne11 <= MMVF_MAX_BATCH_SIZE;
}
return ne11 <= MMVF_MAX_BATCH_SIZE;
+ case GGML_TYPE_F8_E4M3:
+ return ne11 <= MMVF_MAX_BATCH_SIZE;
default:
return false;
}
diff --git a/ggml/src/ggml-cuda/mmvf.cuh b/ggml/src/ggml-cuda/mmvf.cuh
index 350b5cc06..5a2477b07 100644
--- a/ggml/src/ggml-cuda/mmvf.cuh
+++ b/ggml/src/ggml-cuda/mmvf.cuh
@@ -11,4 +11,4 @@ void ggml_cuda_op_mul_mat_vec_f(
const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,
const int64_t src1_padded_row_size, cudaStream_t stream);
-bool ggml_cuda_should_use_mmvf(enum ggml_type type, int cc, int warp_size, const int64_t * src0_ne, const size_t * src0_nb, int64_t ne11);
+bool ggml_cuda_should_use_mmvf(const ggml_tensor * src0, int cc, int warp_size, int64_t ne11);
diff --git a/ggml/src/ggml-cuda/set-rows.cu b/ggml/src/ggml-cuda/set-rows.cu
index 465997065..4be3e649b 100644
--- a/ggml/src/ggml-cuda/set-rows.cu
+++ b/ggml/src/ggml-cuda/set-rows.cu
@@ -257,6 +257,16 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s
nb1, nb2, nb3,
stream
);
+ } else if (dst->type == GGML_TYPE_F8_E4M3) {
+ set_rows_cuda(
+ src0_d, src1_d, (ggml_fp8_e4m3_t *) dst->data,
+ ne00, ne01, ne02, ne03,
+ ne10, ne11, ne12, ne13,
+ nb01, nb02, nb03,
+ nb10, nb11, nb12,
+ nb1, nb2, nb3,
+ stream
+ );
} else if (dst->type == GGML_TYPE_Q4_0) {
set_rows_cuda_quant<idx_t, block_q4_0, QK4_0, quantize_f32_q4_0_block>(
src0_d, src1_d, (block_q4_0*)dst->data,
diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f8_e4m3.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f8_e4m3.cu
new file mode 100644
index 000000000..ae9672a9e
--- /dev/null
+++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-f8_e4m3.cu
@@ -0,0 +1,7 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec.cuh"
+
+DECL_FATTN_VEC_CASE( 64, GGML_TYPE_BF16, GGML_TYPE_F8_E4M3);
+DECL_FATTN_VEC_CASE(128, GGML_TYPE_BF16, GGML_TYPE_F8_E4M3);
+DECL_FATTN_VEC_CASE(256, GGML_TYPE_BF16, GGML_TYPE_F8_E4M3);
diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f8_e4m3.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f8_e4m3.cu
new file mode 100644
index 000000000..2afaefa83
--- /dev/null
+++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f16-f8_e4m3.cu
@@ -0,0 +1,7 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec.cuh"
+
+DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F16, GGML_TYPE_F8_E4M3);
+DECL_FATTN_VEC_CASE(128, GGML_TYPE_F16, GGML_TYPE_F8_E4M3);
+DECL_FATTN_VEC_CASE(256, GGML_TYPE_F16, GGML_TYPE_F8_E4M3);
diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-bf16.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-bf16.cu
new file mode 100644
index 000000000..9dfb9c45e
--- /dev/null
+++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-bf16.cu
@@ -0,0 +1,7 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec.cuh"
+
+DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F8_E4M3, GGML_TYPE_BF16);
+DECL_FATTN_VEC_CASE(128, GGML_TYPE_F8_E4M3, GGML_TYPE_BF16);
+DECL_FATTN_VEC_CASE(256, GGML_TYPE_F8_E4M3, GGML_TYPE_BF16);
diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-f16.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-f16.cu
new file mode 100644
index 000000000..9e7570d36
--- /dev/null
+++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-f16.cu
@@ -0,0 +1,7 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec.cuh"
+
+DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F8_E4M3, GGML_TYPE_F16);
+DECL_FATTN_VEC_CASE(128, GGML_TYPE_F8_E4M3, GGML_TYPE_F16);
+DECL_FATTN_VEC_CASE(256, GGML_TYPE_F8_E4M3, GGML_TYPE_F16);
diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-f8_e4m3.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-f8_e4m3.cu
new file mode 100644
index 000000000..6c380e54c
--- /dev/null
+++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-f8_e4m3.cu
@@ -0,0 +1,7 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec.cuh"
+
+DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F8_E4M3, GGML_TYPE_F8_E4M3);
+DECL_FATTN_VEC_CASE(128, GGML_TYPE_F8_E4M3, GGML_TYPE_F8_E4M3);
+DECL_FATTN_VEC_CASE(256, GGML_TYPE_F8_E4M3, GGML_TYPE_F8_E4M3);
diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q4_0.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q4_0.cu
new file mode 100644
index 000000000..fe22bb964
--- /dev/null
+++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q4_0.cu
@@ -0,0 +1,7 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec.cuh"
+
+DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F8_E4M3, GGML_TYPE_Q4_0);
+DECL_FATTN_VEC_CASE(128, GGML_TYPE_F8_E4M3, GGML_TYPE_Q4_0);
+DECL_FATTN_VEC_CASE(256, GGML_TYPE_F8_E4M3, GGML_TYPE_Q4_0);
diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q4_1.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q4_1.cu
new file mode 100644
index 000000000..317273afc
--- /dev/null
+++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q4_1.cu
@@ -0,0 +1,7 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec.cuh"
+
+DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F8_E4M3, GGML_TYPE_Q4_1);
+DECL_FATTN_VEC_CASE(128, GGML_TYPE_F8_E4M3, GGML_TYPE_Q4_1);
+DECL_FATTN_VEC_CASE(256, GGML_TYPE_F8_E4M3, GGML_TYPE_Q4_1);
diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q5_0.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q5_0.cu
new file mode 100644
index 000000000..e05b1ee38
--- /dev/null
+++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q5_0.cu
@@ -0,0 +1,7 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec.cuh"
+
+DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F8_E4M3, GGML_TYPE_Q5_0);
+DECL_FATTN_VEC_CASE(128, GGML_TYPE_F8_E4M3, GGML_TYPE_Q5_0);
+DECL_FATTN_VEC_CASE(256, GGML_TYPE_F8_E4M3, GGML_TYPE_Q5_0);
diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q5_1.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q5_1.cu
new file mode 100644
index 000000000..6cc638687
--- /dev/null
+++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q5_1.cu
@@ -0,0 +1,7 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec.cuh"
+
+DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F8_E4M3, GGML_TYPE_Q5_1);
+DECL_FATTN_VEC_CASE(128, GGML_TYPE_F8_E4M3, GGML_TYPE_Q5_1);
+DECL_FATTN_VEC_CASE(256, GGML_TYPE_F8_E4M3, GGML_TYPE_Q5_1);
diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q8_0.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q8_0.cu
new file mode 100644
index 000000000..0a88e240b
--- /dev/null
+++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-f8_e4m3-q8_0.cu
@@ -0,0 +1,7 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec.cuh"
+
+DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F8_E4M3, GGML_TYPE_Q8_0);
+DECL_FATTN_VEC_CASE(128, GGML_TYPE_F8_E4M3, GGML_TYPE_Q8_0);
+DECL_FATTN_VEC_CASE(256, GGML_TYPE_F8_E4M3, GGML_TYPE_Q8_0);
diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f8_e4m3.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f8_e4m3.cu
new file mode 100644
index 000000000..44b4d4fc9
--- /dev/null
+++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_0-f8_e4m3.cu
@@ -0,0 +1,7 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec.cuh"
+
+DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_0, GGML_TYPE_F8_E4M3);
+DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_F8_E4M3);
+DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_0, GGML_TYPE_F8_E4M3);
diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f8_e4m3.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f8_e4m3.cu
new file mode 100644
index 000000000..f16490953
--- /dev/null
+++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q4_1-f8_e4m3.cu
@@ -0,0 +1,7 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec.cuh"
+
+DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_1, GGML_TYPE_F8_E4M3);
+DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_F8_E4M3);
+DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_1, GGML_TYPE_F8_E4M3);
diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f8_e4m3.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f8_e4m3.cu
new file mode 100644
index 000000000..6dab7b408
--- /dev/null
+++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_0-f8_e4m3.cu
@@ -0,0 +1,7 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec.cuh"
+
+DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q5_0, GGML_TYPE_F8_E4M3);
+DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_F8_E4M3);
+DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q5_0, GGML_TYPE_F8_E4M3);
diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f8_e4m3.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f8_e4m3.cu
new file mode 100644
index 000000000..2cdaa289f
--- /dev/null
+++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q5_1-f8_e4m3.cu
@@ -0,0 +1,7 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec.cuh"
+
+DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q5_1, GGML_TYPE_F8_E4M3);
+DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q5_1, GGML_TYPE_F8_E4M3);
+DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q5_1, GGML_TYPE_F8_E4M3);
diff --git a/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f8_e4m3.cu b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f8_e4m3.cu
new file mode 100644
index 000000000..2556c8c82
--- /dev/null
+++ b/ggml/src/ggml-cuda/template-instances/fattn-vec-instance-q8_0-f8_e4m3.cu
@@ -0,0 +1,7 @@
+// This file has been autogenerated by generate_cu_files.py, do not edit manually.
+
+#include "../fattn-vec.cuh"
+
+DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_F8_E4M3);
+DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_F8_E4M3);
+DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_F8_E4M3);
diff --git a/ggml/src/ggml-cuda/template-instances/generate_cu_files.py b/ggml/src/ggml-cuda/template-instances/generate_cu_files.py
index 7be409bd3..427db00e0 100755
--- a/ggml/src/ggml-cuda/template-instances/generate_cu_files.py
+++ b/ggml/src/ggml-cuda/template-instances/generate_cu_files.py
@@ -8,7 +8,7 @@ HEAD_SIZES_KQ = [40, 64, 72, 80, 96, 112, 128, 192, 256, 320, 512, 576]
# DKQ -> DV override for asymmetric head dims.
HEAD_SIZES_V_OVERRIDE = {576: 512, 320: 256, 192: 128}
-TYPES_KV = ["GGML_TYPE_F16", "GGML_TYPE_Q4_0", "GGML_TYPE_Q4_1", "GGML_TYPE_Q5_0", "GGML_TYPE_Q5_1", "GGML_TYPE_Q8_0", "GGML_TYPE_BF16"]
+TYPES_KV = ["GGML_TYPE_F16", "GGML_TYPE_Q4_0", "GGML_TYPE_Q4_1", "GGML_TYPE_Q5_0", "GGML_TYPE_Q5_1", "GGML_TYPE_Q8_0", "GGML_TYPE_BF16", "GGML_TYPE_F8_E4M3"]
SOURCE_FATTN_TILE = """// This file has been autogenerated by generate_cu_files.py, do not edit manually.
diff --git a/ggml/src/ggml-cuda/vendors/cuda.h b/ggml/src/ggml-cuda/vendors/cuda.h
index 323c98019..86c1ed9ce 100644
--- a/ggml/src/ggml-cuda/vendors/cuda.h
+++ b/ggml/src/ggml-cuda/vendors/cuda.h
@@ -3,6 +3,7 @@
#include <cuda_runtime.h>
#include <cuda.h>
#include <cublas_v2.h>
+#include <cublasLt.h>
#include <cuda_bf16.h>
#include <cuda_fp16.h>
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index e7314fee4..35b0e4eaa 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -5472,9 +5472,10 @@ struct test_mul_mat : public test_case {
const bool src_overlap; // a and b are overlapping views of the same tensor
const int64_t m_v; // rows of a in memory, the batches of a are strided for m_v > m, no view for m_v == 0
const int64_t pad; // bytes after the m_v rows of each batch of a, so nb[2] of a is not a multiple of nb[1]
+ const size_t offset_a; // byte offset of the a view when k_v > k or m_v > m
std::string vars() override {
- return VARS_TO_STR13(type_a, type_b, m, n, k, bs, nr, per, k_v, o, src_overlap, m_v, pad);
+ return VARS_TO_STR14(type_a, type_b, m, n, k, bs, nr, per, k_v, o, src_overlap, m_v, pad, offset_a);
}
double max_nmse_err() override {
@@ -5508,8 +5509,8 @@ struct test_mul_mat : public test_case {
std::array<int64_t, 2> bs = {10, 10},
std::array<int64_t, 2> nr = {2, 2},
std::array<int64_t, 4> per = {0, 1, 2, 3},
- int64_t k_v = 0, uint32_t o = 1, bool src_overlap = false, int64_t m_v = 0, int64_t pad = 0)
- : type_a(type_a), type_b(type_b), m(m), n(n), k(k), bs(bs), nr(nr), per(per), k_v(k_v), o(o), src_overlap(src_overlap), m_v(m_v), pad(pad) {}
+ int64_t k_v = 0, uint32_t o = 1, bool src_overlap = false, int64_t m_v = 0, int64_t pad = 0, size_t offset_a = 0)
+ : type_a(type_a), type_b(type_b), m(m), n(n), k(k), bs(bs), nr(nr), per(per), k_v(k_v), o(o), src_overlap(src_overlap), m_v(m_v), pad(pad), offset_a(offset_a) {}
ggml_tensor * build_graph(ggml_context * ctx) override {
// C^T = A * B^T: (k, m) * (k, n) => (m, n)
@@ -5571,13 +5572,15 @@ struct test_mul_mat : public test_case {
if (k_v != 0) {
GGML_ASSERT(k_v > k);
- a = ggml_view_4d(ctx, a, k, m, bs[0], bs[1], a->nb[1], a->nb[2], a->nb[3], 0);
+ GGML_ASSERT(offset_a <= ggml_row_size(type_a, k_v) - ggml_row_size(type_a, k));
+ a = ggml_view_4d(ctx, a, k, m, bs[0], bs[1], a->nb[1], a->nb[2], a->nb[3], offset_a);
b = ggml_view_4d(ctx, b, k, n, bs[0]*nr[0], bs[1]*nr[1], b->nb[1], b->nb[2], b->nb[3], 0);
}
if (m_v != 0) {
GGML_ASSERT(m_v > m);
GGML_ASSERT(pad < (int64_t) a->nb[1]);
- a = ggml_view_4d(ctx, a, k, m, bs[0], bs[1], a->nb[1], m_v*a->nb[1] + pad, a->nb[3], 0);
+ GGML_ASSERT(k_v != 0 || offset_a <= (m_v - m)*a->nb[1]);
+ a = ggml_view_4d(ctx, a, k, m, bs[0], bs[1], a->nb[1], m_v*a->nb[1] + pad, a->nb[3], k_v == 0 ? offset_a : 0);
}
ggml_set_name(a, "a");
ggml_set_name(b, "b");
@@ -10716,6 +10719,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
for (ggml_type type : {GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S}) {
test_cases.emplace_back(new test_mul_mat_reorder_then_big_batch(type));
}
+ for (int n : {1, 3, 257}) {
+ test_cases.emplace_back(new test_get_rows(GGML_TYPE_F8_E4M3, n, 5, 3, 1, 1, false));
+ }
for (ggml_type type : all_types) {
for (int b : {1, 7}) {
for (bool v : {false, true}) {
@@ -11840,6 +11846,17 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 1, 512, 2048, {1, 1}, {1, 1}));
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 1, 509, 2051, {1, 1}, {1, 1}));
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 1, 509, 2051, {1, 1}, {1, 1}));
+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F8_E4M3, GGML_TYPE_BF16, 16, 4, 3, {1, 1}, {1, 1}));
+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F8_E4M3, GGML_TYPE_BF16, 16, 4, 256, {1, 1}, {1, 1}));
+ // FP8 views with an odd byte offset must use the cuBLAS fallback instead of MMVF.
+ for (int n : {1, 2}) {
+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F8_E4M3, GGML_TYPE_F32, 2, n, 64, {1, 1}, {1, 1}, {0, 1, 2, 3}, 66, 1, false, 0, 0, 1));
+ }
+ // Native FP8 cuBLASLt requires 16-byte alignment, including each batch pointer.
+ for (size_t offset_a : {1, 2, 4, 16}) {
+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F8_E4M3, GGML_TYPE_F32, 16, 16, 64, {1, 1}, {1, 1}, {0, 1, 2, 3}, 0, 1, false, 17, 0, offset_a));
+ }
+ test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F8_E4M3, GGML_TYPE_F32, 16, 16, 64, {2, 1}, {1, 1}, {0, 1, 2, 3}, 0, 1, false, 17, 4));
// testing very large N
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F32, 16, 2105344, 32, {1, 1}, {1, 1}));
@@ -12918,9 +12935,17 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_flash_attn_ext(64, 128, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q2_0));
test_cases.emplace_back(new test_flash_attn_ext(128, 64, 4, {1, 1}, 64, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q2_0, GGML_TYPE_F16));
test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F8_E4M3, GGML_TYPE_F8_E4M3));
+ test_cases.emplace_back(new test_flash_attn_ext(128, 128, 4, {1, 1}, 128, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F8_E4M3, GGML_TYPE_F8_E4M3));
+ test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {1, 1}, 128, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F8_E4M3, GGML_TYPE_F8_E4M3));
test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F8_E4M3, GGML_TYPE_F16));
test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F8_E4M3));
+ for (int hs : {64, 128, 256}) {
+ for (ggml_type type_K : {GGML_TYPE_F16, GGML_TYPE_F8_E4M3}) {
+ test_cases.emplace_back(new test_flash_attn_ext(hs, hs, 4, {1, 1}, 256, 1, true, false, 0, 0, GGML_PREC_F32, type_K, GGML_TYPE_F8_E4M3));
+ }
+ }
+
// q8_0 KV cases: decode and prompt batches, KV pad, permuted KV, feature flags, and long context
test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 113, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));
test_cases.emplace_back(new test_flash_attn_ext(256, 256, 2, {16, 1}, 1024, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q8_0));
@@ -13063,6 +13088,15 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_opt_step_adamw(GGML_TYPE_F32, {10, 5, 4, 3}));
test_cases.emplace_back(new test_opt_step_sgd(GGML_TYPE_F32, {10, 5, 4, 3}));
+ for (bool with_bias : {false, true}) {
+ test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_F8_E4M3, GGML_GLU_OP_SWIGLU, 16, 32, 256,
+ false, 1, 1, false, with_bias, false, true, {1, 1}));
+ }
+ test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_F8_E4M3, GGML_GLU_OP_SWIGLU, 16, 32, 256,
+ false, 1, 1, false, false, false, true, {2, 1}));
+ test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_F8_E4M3, GGML_GLU_OP_SWIGLU, 16, 31, 257,
+ false, 1, 1, false, false, false, true, {1, 1}));
+
for (ggml_type type : base_types) {
for (bool with_gate : {false, true}) {
for (bool use_id : {false, true}) {