Commit ea84d7090 for llama.cpp

commit ea84d7090302e8cb9fa1e49d34c08fde5e5c2c55
Author: Teng Mei <73174080+meitmt@users.noreply.github.com>
Date:   Fri Nov 20 16:44:04 2026 +0900

    CUDA: optimize Orin single-column MMVQ (#31268)

    * CUDA: optimize Orin single-column MMVQ

    * Create a separate Orin branch in `calc_rows_per_block`

diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu
index 612d3e313..091fc3071 100644
--- a/ggml/src/ggml-cuda/mmvq.cu
+++ b/ggml/src/ggml-cuda/mmvq.cu
@@ -96,7 +96,8 @@ enum mmvq_parameter_table_id {
     MMVQ_PARAMETERS_RDNA2,
     MMVQ_PARAMETERS_RDNA3_0,
     MMVQ_PARAMETERS_RDNA4,
-    MMVQ_PARAMETERS_GB10
+    MMVQ_PARAMETERS_GB10,
+    MMVQ_PARAMETERS_ORIN
 };

 static constexpr __device__ mmvq_parameter_table_id get_device_table_id() {
@@ -112,6 +113,8 @@ static constexpr __device__ mmvq_parameter_table_id get_device_table_id() {
     return MMVQ_PARAMETERS_TURING;
 #elif __CUDA_ARCH__ == GGML_CUDA_CC_GB10
     return MMVQ_PARAMETERS_GB10;
+#elif __CUDA_ARCH__ == GGML_CUDA_CC_ORIN
+    return MMVQ_PARAMETERS_ORIN;
 #else
     return MMVQ_PARAMETERS_GENERIC;
 #endif
@@ -136,6 +139,9 @@ static __host__ mmvq_parameter_table_id get_device_table_id(int cc) {
     if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_GB10) {
         return MMVQ_PARAMETERS_GB10;
     }
+    if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_ORIN) {
+        return MMVQ_PARAMETERS_ORIN;
+    }
     return MMVQ_PARAMETERS_GENERIC;
 }

@@ -580,6 +586,9 @@ static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_d
         }
         return generic;
     }
+    if (table_id == MMVQ_PARAMETERS_ORIN) {
+        return calc_nwarps(type, ncols_dst, MMVQ_PARAMETERS_GENERIC);
+    }
     return 1;
 }

@@ -620,6 +629,41 @@ static constexpr __host__ __device__ int calc_rows_per_block(ggml_type type, int
                 return 1;
         }
     }
+    if (table_id == MMVQ_PARAMETERS_ORIN) {
+        switch (ncols_dst) {
+            case 1:
+                if (small_k) {
+                    return nwarps;
+                }
+                // One row per warp (more rows per block, more loads in flight) for these types, measured on Jetson Orin Nano.
+                // Other types are faster in isolated mat-vec too, but gained at most 2 % end to end.
+                switch (type) {
+                    case GGML_TYPE_Q4_0:
+                    case GGML_TYPE_Q4_1:
+                    case GGML_TYPE_Q5_0:
+                    case GGML_TYPE_Q5_1:
+                    case GGML_TYPE_Q8_0:
+                    case GGML_TYPE_Q4_K:
+                    case GGML_TYPE_Q5_K:
+                    case GGML_TYPE_Q6_K:
+                    case GGML_TYPE_IQ4_NL:
+                        return nwarps;
+                    default:
+                        break;
+                }
+                return mmvq_two_rows_ncols1(type) ? 2 : 1;
+            case 2:
+            case 3:
+            case 4:
+            case 5:
+            case 6:
+            case 7:
+            case 8:
+                return 2;
+            default:
+                return 1;
+        }
+    }
     return 1;
 }