Commit dfeea5200 for llama.cpp
commit dfeea520058802ef3528d7a262d3fc4a91268cf9
Author: Charles Xu <charles.xu@arm.com>
Date: Thu Oct 22 08:56:21 2026 +0200
kleidiai : add Q8_0 SVE dotprod GEMV and SVE i8mm GEMM kernels (#30808)
diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt
index d347757e1..8327c52e6 100644
--- a/ggml/src/ggml-cpu/CMakeLists.txt
+++ b/ggml/src/ggml-cpu/CMakeLists.txt
@@ -650,9 +650,9 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
# Fetch KleidiAI sources:
include(FetchContent)
- set(KLEIDIAI_COMMIT_TAG "v1.31.0")
+ set(KLEIDIAI_COMMIT_TAG "v1.32.0")
set(KLEIDIAI_DOWNLOAD_URL "https://github.com/ARM-software/kleidiai/releases/download/${KLEIDIAI_COMMIT_TAG}/kleidiai-${KLEIDIAI_COMMIT_TAG}-src.tar.gz")
- set(KLEIDIAI_RELEASE_ARCHIVE_MD5 "930495f3aeb16b25dc18a2bde45924f4")
+ set(KLEIDIAI_RELEASE_ARCHIVE_MD5 "05f1953428ffe8e8c7f585dc6d508887")
set(KLEIDIAI_FETCH_ARGS
URL ${KLEIDIAI_DOWNLOAD_URL}
diff --git a/ggml/src/ggml-cpu/kleidiai/kernels.cpp b/ggml/src/ggml-cpu/kleidiai/kernels.cpp
index 89e95968c..24aa28244 100644
--- a/ggml/src/ggml-cpu/kleidiai/kernels.cpp
+++ b/ggml/src/ggml-cpu/kleidiai/kernels.cpp
@@ -201,85 +201,117 @@ static struct kai_matmul_uker_config q4_k_matmul_config() {
return config;
}
-template<struct kai_matmul_uker_api (*GetApi)(void)>
-static size_t q4_k_get_m_step() {
- const struct kai_matmul_uker_api api = GetApi();
- const struct kai_matmul_uker_config config = q4_k_matmul_config();
- return api.get_step(&config).m;
+static struct kai_matmul_uker_config default_matmul_config() {
+ return {};
}
-template<struct kai_matmul_uker_api (*GetApi)(void)>
-static size_t q4_k_get_n_step() {
- const struct kai_matmul_uker_api api = GetApi();
- const struct kai_matmul_uker_config config = q4_k_matmul_config();
- return api.get_step(&config).n;
-}
+template<
+ struct kai_matmul_uker_api (*GetApi)(void),
+ struct kai_matmul_uker_config (*GetConfig)(void),
+ size_t MR, size_t NR, size_t KR, size_t SR>
+struct matmul_uker_adapter {
+ static size_t get_m_step() {
+ const struct kai_matmul_uker_api api = GetApi();
+ const struct kai_matmul_uker_config config = GetConfig();
+ return api.get_step(&config).m;
+ }
-template<size_t Value>
-static size_t q4_k_get_constant() {
- return Value;
-}
+ static size_t get_n_step() {
+ const struct kai_matmul_uker_api api = GetApi();
+ const struct kai_matmul_uker_config config = GetConfig();
+ return api.get_step(&config).n;
+ }
-template<struct kai_matmul_uker_api (*GetApi)(void)>
-static size_t q4_k_get_dst_offset(size_t m_idx, size_t n_idx, size_t dst_stride) {
- const struct kai_matmul_uker_api api = GetApi();
- const struct kai_matmul_uker_config config = q4_k_matmul_config();
- const struct kai_matmul_uker_dst_dim_args index = { m_idx, n_idx };
- const struct kai_matmul_uker_dst_stride_args stride = { dst_stride };
- return api.get_dst_offset(&config, &index, &stride);
-}
+ static size_t get_mr() {
+ if constexpr (MR == 0) {
+ return get_m_step();
+ }
+ return MR;
+ }
-template<struct kai_matmul_uker_api (*GetApi)(void)>
-static size_t q4_k_get_lhs_packed_offset(size_t m_idx, size_t k, size_t bl) {
- GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);
+ static size_t get_nr() {
+ if constexpr (NR == 0) {
+ return get_n_step();
+ }
+ return NR;
+ }
- const struct kai_matmul_uker_api api = GetApi();
- const struct kai_matmul_uker_config config = q4_k_matmul_config();
- const struct kai_matmul_uker_lhs_dim_args shape = { m_idx, k };
- const struct kai_matmul_uker_lhs_dim_args index = { m_idx, 0 };
- const struct kai_matmul_uker_lhs_stride_args stride = api.get_lhs_stride(&config, &shape);
- return api.get_lhs_offset(&config, &index, &stride);
-}
+ static size_t get_kr() {
+ return KR;
+ }
-template<struct kai_matmul_uker_api (*GetApi)(void)>
-static size_t q4_k_get_rhs_packed_offset(size_t n_idx, size_t k, size_t bl) {
- GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);
+ static size_t get_sr() {
+ return SR;
+ }
- const struct kai_matmul_uker_api api = GetApi();
- const struct kai_matmul_uker_config config = q4_k_matmul_config();
- const struct kai_matmul_uker_rhs_dim_args shape = { n_idx, k };
- const struct kai_matmul_uker_rhs_dim_args index = { n_idx, 0 };
- const struct kai_matmul_uker_rhs_stride_args stride = api.get_rhs_stride(&config, &shape);
- return api.get_rhs_offset(&config, &index, &stride);
-}
+ static size_t get_dst_offset(size_t m_idx, size_t n_idx, size_t dst_stride) {
+ const struct kai_matmul_uker_api api = GetApi();
+ const struct kai_matmul_uker_config config = GetConfig();
+ const struct kai_matmul_uker_dst_dim_args index = { m_idx, n_idx };
+ const struct kai_matmul_uker_dst_stride_args stride = { dst_stride };
+ return api.get_dst_offset(&config, &index, &stride);
+ }
-template<struct kai_matmul_uker_api (*GetApi)(void)>
-static void q4_k_run_matmul(
- size_t m, size_t n, size_t k, size_t bl,
- const void * lhs_packed, const void * rhs_packed,
- void * dst, size_t dst_stride_row, size_t dst_stride_col,
- float clamp_min, float clamp_max) {
- GGML_ASSERT(bl == KLEIDIAI_Q4_K_BLOCK_LENGTH);
- GGML_ASSERT(dst_stride_col == sizeof(float));
-
- const struct kai_matmul_uker_api api = GetApi();
- const struct kai_matmul_uker_config config = q4_k_matmul_config();
- const struct kai_matmul_uker_lhs_dim_args lhs_shape = { m, k };
- const struct kai_matmul_uker_rhs_dim_args rhs_shape = { n, k };
-
- struct kai_matmul_uker_args args = {};
- args.flags = KAI_MATMUL_UKER_FLAGS_ARGS_CLAMP;
- args.shape = { m, n, k };
- args.operand.lhs.ptr = lhs_packed;
- args.operand.lhs.stride = api.get_lhs_stride(&config, &lhs_shape);
- args.operand.rhs.ptr = rhs_packed;
- args.operand.rhs.stride = api.get_rhs_stride(&config, &rhs_shape);
- args.operand.dst.ptr = dst;
- args.operand.dst.stride.m = dst_stride_row;
- args.activation.clamp.min_ptr = &clamp_min;
- args.activation.clamp.max_ptr = &clamp_max;
- api.run(&config, &args);
-}
+ static size_t get_lhs_packed_offset(size_t m_idx, size_t k, size_t bl) {
+ const struct kai_matmul_uker_api api = GetApi();
+ const struct kai_matmul_uker_config config = GetConfig();
+ GGML_ASSERT(bl == config.format.bl);
+ const struct kai_matmul_uker_lhs_dim_args shape = { m_idx, k };
+ const struct kai_matmul_uker_lhs_dim_args index = { m_idx, 0 };
+ const struct kai_matmul_uker_lhs_stride_args stride = api.get_lhs_stride(&config, &shape);
+ return api.get_lhs_offset(&config, &index, &stride);
+ }
+
+ static size_t get_rhs_packed_offset(size_t n_idx, size_t k, size_t bl) {
+ const struct kai_matmul_uker_api api = GetApi();
+ const struct kai_matmul_uker_config config = GetConfig();
+ GGML_ASSERT(bl == config.format.bl);
+ const struct kai_matmul_uker_rhs_dim_args shape = { n_idx, k };
+ const struct kai_matmul_uker_rhs_dim_args index = { n_idx, 0 };
+ const struct kai_matmul_uker_rhs_stride_args stride = api.get_rhs_stride(&config, &shape);
+ return api.get_rhs_offset(&config, &index, &stride);
+ }
+
+ static void run(
+ size_t m, size_t n, size_t k, size_t bl,
+ const void * lhs_packed, const void * rhs_packed,
+ void * dst, size_t dst_stride_row, size_t dst_stride_col,
+ float clamp_min, float clamp_max) {
+ const struct kai_matmul_uker_api api = GetApi();
+ const struct kai_matmul_uker_config config = GetConfig();
+ GGML_ASSERT(bl == config.format.bl);
+ GGML_ASSERT(dst_stride_col == sizeof(float));
+ const struct kai_matmul_uker_lhs_dim_args lhs_shape = { m, k };
+ const struct kai_matmul_uker_rhs_dim_args rhs_shape = { n, k };
+
+ struct kai_matmul_uker_args args = {};
+ args.flags = KAI_MATMUL_UKER_FLAGS_ARGS_CLAMP;
+ args.shape = { m, n, k };
+ args.operand.lhs.ptr = lhs_packed;
+ args.operand.lhs.stride = api.get_lhs_stride(&config, &lhs_shape);
+ args.operand.rhs.ptr = rhs_packed;
+ args.operand.rhs.stride = api.get_rhs_stride(&config, &rhs_shape);
+ args.operand.dst.ptr = dst;
+ args.operand.dst.stride.m = dst_stride_row;
+ args.activation.clamp.min_ptr = &clamp_min;
+ args.activation.clamp.max_ptr = &clamp_max;
+ api.run(&config, &args);
+ }
+};
+
+using q4_k_gemm_adapter = matmul_uker_adapter<
+ kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa,
+ q4_k_matmul_config, 0, 0, 4, 2>;
+using q4_k_gemv_adapter = matmul_uker_adapter<
+ kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot,
+ q4_k_matmul_config, 0, 0, 4, 2>;
+// GEMM and GEMV share the RHS layout only at SVE256 (NR=8, KR=8, SR=1).
+using sve_q8_gemm_adapter = matmul_uker_adapter<
+ kai_matmul_clamp_f32_qai8dxp4x8sf32_qsi8cxp4vsx8sf32bf32_16x4vs_sve_i8mm,
+ default_matmul_config, 4, 0, 8, 1>;
+using sve_q8_gemv_adapter = matmul_uker_adapter<
+ kai_matmul_clamp_f32_qai8dxp1x8_qsi8cxp8x8_1x8_sve_dot,
+ default_matmul_config, 1, 8, 8, 1>;
static struct kai_matmul_pack_lhs_uker_config q4_k_lhs_pack_config() {
struct kai_matmul_pack_lhs_uker_config config = {};
@@ -628,7 +660,7 @@ static ggml_kleidiai_kernels gemm_gemv_kernels[] = {
/* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
/* .pack_func_ex = */ &rhs_pack_fn12<kai_run_rhs_pack_nxk_qsi4c32pscalef16_qsu4c32s16s0>,
},
- /* .required_cpu = */ CPU_FEATURE_SVE | CPU_FEATURE_I8MM | CPU_FEATURE_DOTPROD,
+ /* .required_cpu = */ CPU_FEATURE_SVE | CPU_FEATURE_SVEI8MM | CPU_FEATURE_DOTPROD,
/* .lhs_type = */ GGML_TYPE_F32,
/* .rhs_type = */ GGML_TYPE_Q4_0,
/* .op_type = */ GGML_TYPE_F32,
@@ -737,16 +769,16 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q4_k[] = {
{
/* SME2 GEMM */
{
- /* .get_m_step = */ &q4_k_get_m_step<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
- /* .get_n_step = */ &q4_k_get_n_step<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
- /* .get_mr = */ &q4_k_get_m_step<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
- /* .get_nr = */ &q4_k_get_n_step<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
- /* .get_kr = */ &q4_k_get_constant<4>,
- /* .get_sr = */ &q4_k_get_constant<2>,
- /* .get_dst_offset = */ &q4_k_get_dst_offset<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
- /* .get_lhs_offset_ex = */ &q4_k_get_lhs_packed_offset<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
- /* .get_rhs_packed_offset_ex = */ &q4_k_get_rhs_packed_offset<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
- /* .run_kernel_ex = */ &q4_k_run_matmul<kai_matmul_clamp_f32_f16p4vsx2_qai4c32p16vsx4s1s0sf16_4vsx16vs_sme2_mopa>,
+ /* .get_m_step = */ q4_k_gemm_adapter::get_m_step,
+ /* .get_n_step = */ q4_k_gemm_adapter::get_n_step,
+ /* .get_mr = */ q4_k_gemm_adapter::get_mr,
+ /* .get_nr = */ q4_k_gemm_adapter::get_nr,
+ /* .get_kr = */ q4_k_gemm_adapter::get_kr,
+ /* .get_sr = */ q4_k_gemm_adapter::get_sr,
+ /* .get_dst_offset = */ q4_k_gemm_adapter::get_dst_offset,
+ /* .get_lhs_offset_ex = */ q4_k_gemm_adapter::get_lhs_packed_offset,
+ /* .get_rhs_packed_offset_ex = */ q4_k_gemm_adapter::get_rhs_packed_offset,
+ /* .run_kernel_ex = */ q4_k_gemm_adapter::run,
},
{
/* .get_offset = */ kai_get_lhs_offset_lhs_pack_f16pmrx2_f32_neon,
@@ -756,16 +788,16 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q4_k[] = {
},
/* SME2 GEMV */
{
- /* .get_m_step = */ &q4_k_get_m_step<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
- /* .get_n_step = */ &q4_k_get_n_step<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
- /* .get_mr = */ &q4_k_get_m_step<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
- /* .get_nr = */ &q4_k_get_n_step<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
- /* .get_kr = */ &q4_k_get_constant<4>,
- /* .get_sr = */ &q4_k_get_constant<2>,
- /* .get_dst_offset = */ &q4_k_get_dst_offset<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
- /* .get_lhs_offset_ex = */ &q4_k_get_lhs_packed_offset<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
- /* .get_rhs_packed_offset_ex = */ &q4_k_get_rhs_packed_offset<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
- /* .run_kernel_ex = */ &q4_k_run_matmul<kai_matmul_clamp_f32_qsi8d32p1x4_qai4c32p16vsx4s1s0sf16_1x16vs_sme2_dot>,
+ /* .get_m_step = */ q4_k_gemv_adapter::get_m_step,
+ /* .get_n_step = */ q4_k_gemv_adapter::get_n_step,
+ /* .get_mr = */ q4_k_gemv_adapter::get_mr,
+ /* .get_nr = */ q4_k_gemv_adapter::get_nr,
+ /* .get_kr = */ q4_k_gemv_adapter::get_kr,
+ /* .get_sr = */ q4_k_gemv_adapter::get_sr,
+ /* .get_dst_offset = */ q4_k_gemv_adapter::get_dst_offset,
+ /* .get_lhs_offset_ex = */ q4_k_gemv_adapter::get_lhs_packed_offset,
+ /* .get_rhs_packed_offset_ex = */ q4_k_gemv_adapter::get_rhs_packed_offset,
+ /* .run_kernel_ex = */ q4_k_gemv_adapter::run,
},
{
/* .get_offset = */ q4_k_lhs_get_offset,
@@ -883,6 +915,54 @@ static ggml_kleidiai_kernels gemm_gemv_kernels_q8[] = {
/* .rhs_type = */ GGML_TYPE_Q8_0,
/* .op_type = */ GGML_TYPE_F32,
},
+ {
+ /* SVE I8MM GEMM */
+ {
+ /* .get_m_step = */ sve_q8_gemm_adapter::get_m_step,
+ /* .get_n_step = */ sve_q8_gemm_adapter::get_n_step,
+ /* .get_mr = */ sve_q8_gemm_adapter::get_mr,
+ /* .get_nr = */ sve_q8_gemm_adapter::get_nr,
+ /* .get_kr = */ sve_q8_gemm_adapter::get_kr,
+ /* .get_sr = */ sve_q8_gemm_adapter::get_sr,
+ /* .get_dst_offset = */ sve_q8_gemm_adapter::get_dst_offset,
+ /* .get_lhs_offset_ex = */ sve_q8_gemm_adapter::get_lhs_packed_offset,
+ /* .get_rhs_packed_offset_ex = */ sve_q8_gemm_adapter::get_rhs_packed_offset,
+ /* .run_kernel_ex = */ sve_q8_gemm_adapter::run,
+ },
+ /* .gemm_lhs_info = */ {
+ /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f32,
+ /* .get_packed_offset_ex = */ &lhs_offs_fn5<kai_get_lhs_packed_offset_lhs_quant_pack_qai8dxp_f32>,
+ /* .packed_size_ex = */ &lhs_ps_fn5<kai_get_lhs_packed_size_lhs_quant_pack_qai8dxp_f32>,
+ /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>,
+ },
+ /* SVE DOT GEMV */
+ {
+ /* .get_m_step = */ sve_q8_gemv_adapter::get_m_step,
+ /* .get_n_step = */ sve_q8_gemv_adapter::get_n_step,
+ /* .get_mr = */ sve_q8_gemv_adapter::get_mr,
+ /* .get_nr = */ sve_q8_gemv_adapter::get_nr,
+ /* .get_kr = */ sve_q8_gemv_adapter::get_kr,
+ /* .get_sr = */ sve_q8_gemv_adapter::get_sr,
+ /* .get_dst_offset = */ sve_q8_gemv_adapter::get_dst_offset,
+ /* .get_lhs_offset_ex = */ sve_q8_gemv_adapter::get_lhs_packed_offset,
+ /* .get_rhs_packed_offset_ex = */ sve_q8_gemv_adapter::get_rhs_packed_offset,
+ /* .run_kernel_ex = */ sve_q8_gemv_adapter::run,
+ },
+ /* .gemv_lhs_info = */ {
+ /* .get_offset = */ kai_get_lhs_offset_lhs_quant_pack_qai8dxp_f32,
+ /* .get_packed_offset_ex = */ &lhs_offs_fn5<kai_get_lhs_packed_offset_lhs_quant_pack_qai8dxp_f32>,
+ /* .packed_size_ex = */ &lhs_ps_fn5<kai_get_lhs_packed_size_lhs_quant_pack_qai8dxp_f32>,
+ /* .pack_func_ex = */ &lhs_pack_float_fn9_no_bl<kai_run_lhs_quant_pack_qai8dxp_f32>,
+ },
+ /* .rhs_info = */ {
+ /* .packed_size_ex = */ &rhs_ps_fn5<kai_get_rhs_packed_size_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
+ /* .pack_func_ex = */ &rhs_pack_scale_fn12<kai_run_rhs_pack_nxk_qsi8cxp_qsi8cx_neon>,
+ },
+ /* .required_cpu = */ CPU_FEATURE_SVE | CPU_FEATURE_SVEI8MM | CPU_FEATURE_DOTPROD,
+ /* .lhs_type = */ GGML_TYPE_F32,
+ /* .rhs_type = */ GGML_TYPE_Q8_0,
+ /* .op_type = */ GGML_TYPE_F32,
+ },
{
/* I8MM GEMM */
{
diff --git a/ggml/src/ggml-cpu/kleidiai/kernels.h b/ggml/src/ggml-cpu/kleidiai/kernels.h
index 308ea9451..ac5d5d302 100644
--- a/ggml/src/ggml-cpu/kleidiai/kernels.h
+++ b/ggml/src/ggml-cpu/kleidiai/kernels.h
@@ -13,7 +13,8 @@ enum cpu_feature {
CPU_FEATURE_SVE = 4,
CPU_FEATURE_SME = 8,
CPU_FEATURE_SME2 = 16,
- CPU_FEATURE_FP16 = 32
+ CPU_FEATURE_FP16 = 32,
+ CPU_FEATURE_SVEI8MM = 64
};
inline cpu_feature& operator|=(cpu_feature& lhs, cpu_feature rhs) {
diff --git a/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp b/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp
index dd9bcbf63..57a4a5662 100644
--- a/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp
+++ b/ggml/src/ggml-cpu/kleidiai/kleidiai.cpp
@@ -319,8 +319,10 @@ static void init_kleidiai_context(void) {
size_t detected_smcus = 0;
+ // The fixed-width SVE kernels require 32-byte vectors, including the Q8 shared RHS layout.
ctx.features = (runtime_feat.has_dotprod ? CPU_FEATURE_DOTPROD : CPU_FEATURE_NONE) |
(runtime_feat.has_i8mm ? CPU_FEATURE_I8MM : CPU_FEATURE_NONE) |
+ (runtime_feat.has_svei8mm ? CPU_FEATURE_SVEI8MM : CPU_FEATURE_NONE) |
(runtime_feat.has_fp16 ? CPU_FEATURE_FP16 : CPU_FEATURE_NONE) |
(runtime_feat.sve_cnt == QK8_0 ? CPU_FEATURE_SVE : CPU_FEATURE_NONE);
@@ -1616,6 +1618,11 @@ public:
const size_t kr = kernel->get_kr();
const size_t sr = kernel->get_sr();
const ggml_type rhs_type = kernels->rhs_type;
+ if (rhs_type == GGML_TYPE_Q8_0 && (kernels->required_cpu & CPU_FEATURE_SVE)) {
+ GGML_ASSERT(nr == kernels->gemv.get_nr());
+ GGML_ASSERT(kr == kernels->gemv.get_kr());
+ GGML_ASSERT(sr == kernels->gemv.get_sr());
+ }
const size_t block_len = rhs_type == GGML_TYPE_Q8_0 ? QK8_0 :
rhs_type == GGML_TYPE_Q4_0 ? QK4_0 :
rhs_type == GGML_TYPE_Q4_K ? GGML_KLEIDIAI_Q4_K_BL :
diff --git a/ggml/src/ggml-feats.h b/ggml/src/ggml-feats.h
index 79a0afd87..c45e4ad08 100644
--- a/ggml/src/ggml-feats.h
+++ b/ggml/src/ggml-feats.h
@@ -22,6 +22,10 @@
#define HWCAP2_I8MM (1ULL << 13)
#endif
+#if !defined(HWCAP2_SVEI8MM)
+#define HWCAP2_SVEI8MM (1ULL << 9)
+#endif
+
#if !defined(HWCAP_ASIMDDP)
#define HWCAP_ASIMDDP (1 << 20)
#endif
@@ -63,6 +67,10 @@
#define PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE 47
#endif
+#if !defined(PF_ARM_SVE_I8MM_INSTRUCTIONS_AVAILABLE)
+#define PF_ARM_SVE_I8MM_INSTRUCTIONS_AVAILABLE 57
+#endif
+
#if !defined(PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE)
#define PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE 66
#endif
@@ -87,6 +95,7 @@ typedef struct ggml_feats_arch64_runtime {
bool has_sve;
bool has_sve2;
bool has_i8mm;
+ bool has_svei8mm;
bool has_sme;
bool has_sme2;
int sve_cnt;
@@ -104,6 +113,7 @@ static inline ggml_feats_arch64_runtime_t ggml_feats_get_arch64_runtime(void) {
runtime_feat.has_sve = !!(hwcap & HWCAP_SVE);
runtime_feat.has_sve2 = !!(hwcap2 & HWCAP2_SVE2);
runtime_feat.has_i8mm = !!(hwcap2 & HWCAP2_I8MM);
+ runtime_feat.has_svei8mm = !!(hwcap2 & HWCAP2_SVEI8MM);
runtime_feat.has_sme = !!(hwcap2 & HWCAP2_SME);
runtime_feat.has_sme2 = !!(hwcap2 & HWCAP2_SME2);
@@ -153,6 +163,7 @@ static inline ggml_feats_arch64_runtime_t ggml_feats_get_arch64_runtime(void) {
runtime_feat.has_sve = IsProcessorFeaturePresent(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE) != 0;
runtime_feat.has_sve2 = IsProcessorFeaturePresent(PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE) != 0;
runtime_feat.has_i8mm = IsProcessorFeaturePresent(PF_ARM_V82_I8MM_INSTRUCTIONS_AVAILABLE) != 0;
+ runtime_feat.has_svei8mm = IsProcessorFeaturePresent(PF_ARM_SVE_I8MM_INSTRUCTIONS_AVAILABLE) != 0;
runtime_feat.has_sme = IsProcessorFeaturePresent(PF_ARM_SME_INSTRUCTIONS_AVAILABLE) != 0;
runtime_feat.has_sme2 = IsProcessorFeaturePresent(PF_ARM_SME2_INSTRUCTIONS_AVAILABLE) != 0;