Superminaren/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-04 02:37:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

AVX2: Speed up large batch size prompt processing of IQ models (#27402)

Browse Source 
* Batched gemm for grid IQ quants

Style updates and a bit more performance

Clean up comments

Move code around

Vectorize IQ panel decode, lower threshold for speedup

IQ panel: single-source gather layout, gate bias, vectorize interleave

Add ggml_gemm_iqp_8x8_q8_K_p4 kernel, remove gather buffer

Move IQ panel code out of repack into iqp.cpp, clean up comments

Another comment sweep

* Add myself as iqp.* codeownder

* Remove ggml_cpu_iqp_scratch_offset and ggml_cpu_iqp_src1_conv_size

* Renaming and moving

* The other half of renaming and moving

* Move macros and ggml_cpu_iqp_mul_mat_id_min_batch definition

* Update ggml/src/ggml-cpu/iqp.h

Co-authored-by: Georgi Gerganov <[email protected]>

* Add iqp_rows work buffer

* Revert "Add iqp_rows work buffer"

This reverts commit 425542991e.

* Add NUMA fallback

* Add 10 row batch tests for IQP coverage on all grid IQ types

* Swap assert for return false in support check

* Move IQP mul_mat_id test

---------

Co-authored-by: Georgi Gerganov <[email protected]>
This commit is contained in:
Bartowski
2026-08-31 14:33:50 -04:00
committed by GitHub
co-authored by Georgi Gerganov
parent 2a74817f93
commit 85c55223ca
7 changed files with 1342 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CODEOWNERS
+1
View File
@@ -57,6 +57,7 @@
/ggml/src/ggml-cann/ @ggml-org/ggml-cann
/ggml/src/ggml-common.h @ggerganov
/ggml/src/ggml-cpu/ @ggerganov
/ggml/src/ggml-cpu/iqp.* @bartowski1182
/ggml/src/ggml-cpu/spacemit/ @alex-spacemit
/ggml/src/ggml-cuda/ @ggml-org/ggml-cuda
/ggml/src/ggml-cuda/vendors/hip.h @IMbackK
ggml/src/ggml-common.h
+1 -1
View File
@@ -1131,7 +1131,7 @@ GGML_TABLE_END()
#define NGRID_IQ1S 2048
#define IQ1S_DELTA 0.125f
#define IQ1M_DELTA 0.125f
#if defined(GGML_COMMON_IMPL_C)
#if defined(GGML_COMMON_IMPL_C) || defined(GGML_COMMON_IMPL_CPP)
GGML_TABLE_BEGIN(uint64_t, iq1s_grid, NGRID_IQ1S)
0xffffffffffffffff, 0xffffffffffffff01, 0xffffffffffff0000, 0xffffffffffff01ff,
0xffffffffffff0101, 0xffffffffff00ff00, 0xffffffffff000000, 0xffffffffff01ffff,
ggml/src/ggml-cpu/CMakeLists.txt
+2
View File
@@ -31,6 +31,8 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
ggml-cpu/ggml-cpu.cpp
ggml-cpu/repack.cpp
ggml-cpu/repack.h
ggml-cpu/iqp.cpp
ggml-cpu/iqp.h
ggml-cpu/hbm.cpp
ggml-cpu/hbm.h
ggml-cpu/quants.c
ggml/src/ggml-cpu/ggml-cpu.c
+34
View File
@@ -4,6 +4,7 @@
#include "ggml-backend-impl.h"
#include "ggml-backend.h"
#include "traits.h"
#include "iqp.h"
#include "ggml-cpu-impl.h"
#include "ggml-impl.h"
#include "quants.h"
@@ -1363,6 +1364,13 @@ UseGgmlGemm1:;
ggml_barrier(params->threadpool);
// IQ panel gemm (see iqp.h) - must come after the barrier above, it consumes the q8_K rows
// of src1 from the work buffer
if (ggml_cpu_iqp_supports_mul_mat(dst) && !params->use_ref) {
ggml_compute_forward_mul_mat_iqp(params, dst);
return;
}
#if GGML_USE_LLAMAFILE
if (src1->type != vec_dot_type) {
const void* wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;
@@ -1580,6 +1588,16 @@ static void ggml_compute_forward_mul_mat_id(
char (*atomic_current_chunk)[CACHE_LINE_SIZE] = // [n_as]
incr_ptr_aligned(&wdata_cur, CACHE_LINE_SIZE * n_as, CACHE_LINE_SIZE);
// IQ panel gemm (see iqp.h); per expert eligibility is decided below, but the work buffer is
// reserved for the whole node (ggml_graph_plan sizes it without params, use_ref only skips the dispatch)
const bool iqp = ggml_cpu_iqp_supports_mul_mat_id(dst) && !params->use_ref;
char * iqp_panels = NULL;
if (iqp) {
iqp_panels = incr_ptr_aligned(&wdata_cur, nth * ggml_cpu_iqp_scratch_size(dst), 64);
}
GGML_ASSERT(params->wsize >= (size_t)((char *) wdata_cur - (char *) params->wdata));
if (src1->type != vec_dot_type) {
@@ -1651,6 +1669,13 @@ static void ggml_compute_forward_mul_mat_id(
continue;
}
if (iqp && ggml_cpu_iqp_mul_mat_id_min_batch(cne1)) {
ggml_compute_forward_mul_mat_id_iqp(params, dst, cur_a, cne1, (const int32_t *) &MMID_MATRIX_ROW(cur_a, 0),
iqp_panels);
continue;
}
const char * src0_cur = (const char *) src0->data + cur_a * nb02;
const void * wdata = (src1->type == vec_dot_type) ? src1->data : params->wdata;
const size_t row_size = ggml_row_size(vec_dot_type, ne10);
@@ -2858,6 +2883,11 @@ struct ggml_cplan ggml_graph_plan(
if (node->src[1]->type != vec_dot_type) {
cur = ggml_row_size(vec_dot_type, ggml_nelements(node->src[1]));
}
// the IQ panel path needs one scratch panel per thread past the q8_K rows
if (ggml_cpu_iqp_supports_mul_mat(node)) {
cur = GGML_PAD(cur, 64) + n_tasks * ggml_cpu_iqp_scratch_size(node);
}
} break;
case GGML_OP_MUL_MAT_ID:
{
@@ -2877,6 +2907,10 @@ struct ggml_cplan ggml_graph_plan(
cur += n_as*ids->ne[0]*ids->ne[1]*sizeof(struct mmid_row_mapping) + sizeof(int64_t);
// atomic_current_chunk
cur += CACHE_LINE_SIZE*n_as + CACHE_LINE_SIZE;
// the IQ panel path needs one scratch panel per thread on top of that
if (ggml_cpu_iqp_supports_mul_mat_id(node)) {
cur += n_tasks * ggml_cpu_iqp_scratch_size(node) + 64;
}
} break;
case GGML_OP_OUT_PROD:
{
ggml/src/ggml-cpu/iqp.cpp
+1253
View File
File diff suppressed because it is too large Load Diff
ggml/src/ggml-cpu/iqp.h
+39
View File
@@ -0,0 +1,39 @@
#pragma once
#include "ggml-cpu-impl.h"
#include "ggml.h"
// GGML internal header
// batched mul_mat path for the grid based IQ types: decode 8 src0 rows at a time into per thread scratch
// (block_iqp_x8, see iqp.cpp) and run an integer gemm over them against all src1 columns
#ifdef __cplusplus
extern "C" {
#endif
// whether cne1 rows of src1 are enough for the decode to pay for itself, per expert, for MUL_MAT_ID
bool ggml_cpu_iqp_mul_mat_id_min_batch(int64_t cne1);
bool ggml_cpu_iqp_supports_mul_mat(const struct ggml_tensor * dst);
// node level test only - per expert eligibility is decided with ggml_cpu_iqp_mul_mat_id_min_batch
bool ggml_cpu_iqp_supports_mul_mat_id(const struct ggml_tensor * dst);
// per thread panel scratch bytes, padded
size_t ggml_cpu_iqp_scratch_size(const struct ggml_tensor * dst);
// must be called after src1 has been converted to q8_K into params->wdata and the threads have synchronized on it
void ggml_compute_forward_mul_mat_iqp(const struct ggml_compute_params * params, struct ggml_tensor * dst);
// one expert: expert_rows points at its row of the matrix_rows table of (i1, i2) int32 pairs, panels at the base of the per thread panel scratches
void ggml_compute_forward_mul_mat_id_iqp(const struct ggml_compute_params * params,
struct ggml_tensor * dst,
int64_t cur_a,
int64_t cne1,
const int32_t * expert_rows,
void * panels);
#ifdef __cplusplus
}
#endif
tests/test-backend-ops.cpp
+12
View File
@@ -9420,6 +9420,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, 256, {1, 1}, {1, 1}));
}
}
// Test IQP panel path for all grid IQ types
for (ggml_type type_a : {GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ2_S, GGML_TYPE_IQ3_XXS,
GGML_TYPE_IQ3_S, GGML_TYPE_IQ1_S, GGML_TYPE_IQ1_M, GGML_TYPE_IQ4_XS}) {
test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 16, 10, 256, {1, 1}, {1, 1}));
}
#else
// m = a rows
// n = b rows
@@ -9531,6 +9537,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 4, 2, false, 64, 16, 3*ggml_blck_size(type_a)));
}
// Test IQP panel path for all grid IQ types
for (ggml_type type_a : {GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ2_S, GGML_TYPE_IQ3_XXS,
GGML_TYPE_IQ3_S, GGML_TYPE_IQ1_S, GGML_TYPE_IQ1_M, GGML_TYPE_IQ4_XS}) {
test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 4, 4, false, 16, 10, 256));
}
for (ggml_type type_a : base_types) {
for (ggml_type type_b : {GGML_TYPE_F32 /*, GGML_TYPE_F16 */}) {
for (int n_mats : {4, 8}) {