Commit ea84d7090 for llama.cpp
commit ea84d7090302e8cb9fa1e49d34c08fde5e5c2c55
Author: Teng Mei <73174080+meitmt@users.noreply.github.com>
Date: Fri Nov 20 16:44:04 2026 +0900
CUDA: optimize Orin single-column MMVQ (#31268)
* CUDA: optimize Orin single-column MMVQ
* Create a separate Orin branch in `calc_rows_per_block`
diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu
index 612d3e313..091fc3071 100644
--- a/ggml/src/ggml-cuda/mmvq.cu
+++ b/ggml/src/ggml-cuda/mmvq.cu
@@ -96,7 +96,8 @@ enum mmvq_parameter_table_id {
MMVQ_PARAMETERS_RDNA2,
MMVQ_PARAMETERS_RDNA3_0,
MMVQ_PARAMETERS_RDNA4,
- MMVQ_PARAMETERS_GB10
+ MMVQ_PARAMETERS_GB10,
+ MMVQ_PARAMETERS_ORIN
};
static constexpr __device__ mmvq_parameter_table_id get_device_table_id() {
@@ -112,6 +113,8 @@ static constexpr __device__ mmvq_parameter_table_id get_device_table_id() {
return MMVQ_PARAMETERS_TURING;
#elif __CUDA_ARCH__ == GGML_CUDA_CC_GB10
return MMVQ_PARAMETERS_GB10;
+#elif __CUDA_ARCH__ == GGML_CUDA_CC_ORIN
+ return MMVQ_PARAMETERS_ORIN;
#else
return MMVQ_PARAMETERS_GENERIC;
#endif
@@ -136,6 +139,9 @@ static __host__ mmvq_parameter_table_id get_device_table_id(int cc) {
if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_GB10) {
return MMVQ_PARAMETERS_GB10;
}
+ if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_ORIN) {
+ return MMVQ_PARAMETERS_ORIN;
+ }
return MMVQ_PARAMETERS_GENERIC;
}
@@ -580,6 +586,9 @@ static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_d
}
return generic;
}
+ if (table_id == MMVQ_PARAMETERS_ORIN) {
+ return calc_nwarps(type, ncols_dst, MMVQ_PARAMETERS_GENERIC);
+ }
return 1;
}
@@ -620,6 +629,41 @@ static constexpr __host__ __device__ int calc_rows_per_block(ggml_type type, int
return 1;
}
}
+ if (table_id == MMVQ_PARAMETERS_ORIN) {
+ switch (ncols_dst) {
+ case 1:
+ if (small_k) {
+ return nwarps;
+ }
+ // One row per warp (more rows per block, more loads in flight) for these types, measured on Jetson Orin Nano.
+ // Other types are faster in isolated mat-vec too, but gained at most 2 % end to end.
+ switch (type) {
+ case GGML_TYPE_Q4_0:
+ case GGML_TYPE_Q4_1:
+ case GGML_TYPE_Q5_0:
+ case GGML_TYPE_Q5_1:
+ case GGML_TYPE_Q8_0:
+ case GGML_TYPE_Q4_K:
+ case GGML_TYPE_Q5_K:
+ case GGML_TYPE_Q6_K:
+ case GGML_TYPE_IQ4_NL:
+ return nwarps;
+ default:
+ break;
+ }
+ return mmvq_two_rows_ncols1(type) ? 2 : 1;
+ case 2:
+ case 3:
+ case 4:
+ case 5:
+ case 6:
+ case 7:
+ case 8:
+ return 2;
+ default:
+ return 1;
+ }
+ }
return 1;
}