From 68d9053afd4f4d0752ced6187585f862355a40be Mon Sep 17 00:00:00 2001 From: Yangyu Chen Date: Mon, 21 Sep 2026 15:45:31 +0800 Subject: [PATCH] cuda : tune MMVQ to MMQ crossover for SM70 (Volta) (#28912) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * tune MMVQ to MMQ crossover for SM70 (Volta) Signed-off-by: Yangyu Chen * Apply suggestion from @JohannesGaessler * Apply suggestion from @JohannesGaessler * Apply suggestion from @JohannesGaessler --------- Signed-off-by: Yangyu Chen Co-authored-by: Johannes Gäßler --- ggml/src/ggml-cuda/mmvq.cu | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index 6305230b1f..a85155360a 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -365,6 +365,22 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) { return ne11 <= MMVQ_MAX_BATCH_SIZE; } } + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_VOLTA) { + switch (type) { + case GGML_TYPE_Q2_K: + return ne11 <= 4; + case GGML_TYPE_Q3_K: + return ne11 <= 6; + case GGML_TYPE_Q4_K: + return ne11 <= 5; + case GGML_TYPE_Q5_K: + return ne11 <= 6; + case GGML_TYPE_Q6_K: + return ne11 <= 7; + default: + return ne11 <= MMVQ_MAX_BATCH_SIZE; + } + } if (GGML_CUDA_CC_IS_CDNA(cc)) { if (GGML_CUDA_CC_IS_CDNA1(cc)) { switch (type) {