diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cpp index 220663d5ac..933bc77d2e 100644 --- a/ggml/src/ggml-sycl/mmvq.cpp +++ b/ggml/src/ggml-sycl/mmvq.cpp @@ -6,6 +6,24 @@ #include "quants.hpp" #include "vecdotq.hpp" +// Minimum weight-row count at which the Q4_K multi-column MMVQ kernel handles two output rows per +// subgroup (rows_per_sg == 2) instead of one, when ncols_dst == 2. +// +// Pairing rows lets a subgroup load each activation block once and apply it to two rows, at the cost +// of halving the number of subgroups in the launch. With only two destination columns there is too +// little work per row to hide that loss of parallelism, so pairing only pays off once there are +// enough rows to keep the device occupied. This is a measured performance crossover, not a +// correctness or hardware limit - both variants compute the same result for any nrows. +// +// Derived on Intel Arc Pro B70 with `test-backend-ops perf -o MUL_MAT` (Q4_K, ncols_dst == 2), +// sweeping nrows over 5120..6912 at ncols 17408 and 19968: one row per subgroup was up to 9% faster +// below the crossover, two rows per subgroup 8-15% faster above it, and the crossover fell inside +// (6144, 6272] for both ncols with no measurable ncols dependence. A later 32-row granularity sweep +// narrowed it to (6144, 6176], so 6272 is a conservative gate rather than the exact crossover. +// ncols_dst >= 3 amortizes the activation loads over more columns and is faster with two rows at +// every row count, so it does not consult this threshold. +static constexpr int Q4_K_MMVQ_ROW_PAIR_MIN_NROWS = 6272; + template static void mul_mat_vec_q_reorder(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, const int ncols, const int nrows, const sycl::nd_item<3> & nd_item) { @@ -59,7 +77,7 @@ static void mul_mat_vec_q_reorder(const void * __restrict__ vx, const void * __r // With has_fusion, `vgate` is a second weight matrix sharing vx's shape, stride and reorder // layout: one pass computes both row dot products and the epilogue writes glu(gate, up). -template +template static void mul_mat_vec_q_reorder_ncols(const void * __restrict__ vx, const void * __restrict__ vgate, const void * __restrict__ vy, float * __restrict__ dst, const int ncols, const int nrows, const int stride_col_y_bytes, const int stride_col_dst, @@ -71,14 +89,17 @@ static void mul_mat_vec_q_reorder_ncols(const void * __restrict__ vx, const void const int sg_range = sg.get_group_linear_range(); const int workgroup_id = nd_item.get_group_linear_id(); const int sg_id = sg.get_group_linear_id(); - const int row = workgroup_id * sg_range + sg_id; + const int row0 = (workgroup_id * sg_range + sg_id) * rows_per_sg; // row is sub-group uniform, so this retires whole sub-groups and the collectives below // stay convergent - if (row >= nrows) { + if (row0 >= nrows) { return; } + static_assert(rows_per_sg == 1 || + reorder_vec_dot_shared_activations::value); + const int blocks_per_row = ncols / block_traits::qk; constexpr int blocks_per_subgroup = ceil_div(block_traits::vdr_mmvq * WARP_SIZE, block_traits::qi); constexpr int block_elements_per_subgroup = block_traits::qi / block_traits::vdr_mmvq; @@ -87,34 +108,96 @@ static void mul_mat_vec_q_reorder_ncols(const void * __restrict__ vx, const void static_assert(blocks_per_subgroup > 0); static_assert(block_elements_per_subgroup > 0); - float partial_sum[ncols_dst] = { 0.0f }; + float partial_sum[ncols_dst][rows_per_sg] = {}; // sized 1 rather than 0 when unused: zero-length arrays are not standard C++, and the // array is dead and eliminated in that case - [[maybe_unused]] float partial_gate[has_fusion ? ncols_dst : 1] = { 0.0f }; + [[maybe_unused]] float partial_gate[has_fusion ? ncols_dst : 1][has_fusion ? rows_per_sg : 1] = {}; for (int i = sg.get_local_linear_id() / block_elements_per_subgroup; i < blocks_per_row; i += blocks_per_subgroup) { - const int ibx = row * blocks_per_row + i; - - // the offsets depend only on the block index and the matrix shape, never on the base - // pointer, which is what lets vgate reuse them - const auto bx_offset = block_type::get_block_offset(ibx, nblocks); - const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx); const int iby = i * block_type::block_to_q8_1_ratio(); #pragma unroll for (int elem = 0; elem < block_elements_per_subgroup; elem += WARP_SIZE) { const int iqs = elem + block_traits::vdr_mmvq * (sg.get_local_linear_id() % block_elements_per_subgroup); + if constexpr (rows_per_sg > 1) { + typename reorder_vec_dot_q_sycl::weights wx[rows_per_sg]; + [[maybe_unused]] typename reorder_vec_dot_q_sycl::weights wg[rows_per_sg]; #pragma unroll - for (int j = 0; j < ncols_dst; ++j) { - const char * vy_j = (const char *) vy + j * stride_col_y_bytes; - const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1; - const sycl::half2 * q8_1_ds_ptr = (const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2)); - - partial_sum[j] += reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs); - + for (int r = 0; r < rows_per_sg; ++r) { + const int row = sycl::min(row0 + r, nrows - 1); + const int ibx = row * blocks_per_row + i; + const auto bx_offset = block_type::get_block_offset(ibx, nblocks); + const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx); + wx[r] = reorder_vec_dot_q_sycl::load(vx, bx_offset, d_offset, iqs); + if constexpr (has_fusion) { + wg[r] = reorder_vec_dot_q_sycl::load(vgate, bx_offset, d_offset, iqs); + } + } +#pragma unroll + for (int j = 0; j < ncols_dst; ++j) { + const char * vy_j = (const char *) vy + j * stride_col_y_bytes; + const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1; + const sycl::half2 * q8_1_ds_ptr = + (const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2)); + const auto a = reorder_vec_dot_q_sycl::load_activations(q8_1_quant_ptr, q8_1_ds_ptr, iqs); +#pragma unroll + for (int r = 0; r < rows_per_sg; ++r) { + partial_sum[j][r] += reorder_vec_dot_q_sycl::apply(wx[r], a); + if constexpr (has_fusion) { + partial_gate[j][r] += reorder_vec_dot_q_sycl::apply(wg[r], a); + } + } + } + } else if constexpr (reorder_vec_dot_shared_weights::value) { + const int ibx = row0 * blocks_per_row + i; + const auto bx_offset = block_type::get_block_offset(ibx, nblocks); + const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx); + const auto wx = reorder_vec_dot_q_sycl::load(vx, bx_offset, d_offset, iqs); if constexpr (has_fusion) { - partial_gate[j] += - reorder_vec_dot_q_sycl()(vgate, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs); + const auto wg = reorder_vec_dot_q_sycl::load(vgate, bx_offset, d_offset, iqs); + +#pragma unroll + for (int j = 0; j < ncols_dst; ++j) { + const char * vy_j = (const char *) vy + j * stride_col_y_bytes; + const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1; + const sycl::half2 * q8_1_ds_ptr = + (const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2)); + + // up and gate share the activation, so load it once and apply it twice + const auto a = reorder_vec_dot_q_sycl::load_activations(q8_1_quant_ptr, q8_1_ds_ptr, iqs); + + partial_sum[j][0] += reorder_vec_dot_q_sycl::apply(wx, a); + partial_gate[j][0] += reorder_vec_dot_q_sycl::apply(wg, a); + } + } else { +#pragma unroll + for (int j = 0; j < ncols_dst; ++j) { + const char * vy_j = (const char *) vy + j * stride_col_y_bytes; + const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1; + const sycl::half2 * q8_1_ds_ptr = + (const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2)); + + partial_sum[j][0] += reorder_vec_dot_q_sycl::dot(wx, q8_1_quant_ptr, q8_1_ds_ptr, iqs); + } + } + } else { + const int ibx = row0 * blocks_per_row + i; + const auto bx_offset = block_type::get_block_offset(ibx, nblocks); + const auto d_offset = block_type::get_d_offset(nrows, ncols, ibx); +#pragma unroll + for (int j = 0; j < ncols_dst; ++j) { + const char * vy_j = (const char *) vy + j * stride_col_y_bytes; + const int8_t * q8_1_quant_ptr = (const int8_t *) vy_j + iby * QK8_1; + const sycl::half2 * q8_1_ds_ptr = + (const sycl::half2 *) (vy_j + ncols + iby * sizeof(sycl::half2)); + + partial_sum[j][0] += + reorder_vec_dot_q_sycl()(vx, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs); + + if constexpr (has_fusion) { + partial_gate[j][0] += + reorder_vec_dot_q_sycl()(vgate, bx_offset, d_offset, q8_1_quant_ptr, q8_1_ds_ptr, iqs); + } } } } @@ -122,17 +205,20 @@ static void mul_mat_vec_q_reorder_ncols(const void * __restrict__ vx, const void #pragma unroll for (int j = 0; j < ncols_dst; ++j) { - float sum = sycl::reduce_over_group(nd_item.get_sub_group(), partial_sum[j], std::plus<>()); +#pragma unroll + for (int r = 0; r < rows_per_sg; ++r) { + float sum = sycl::reduce_over_group(nd_item.get_sub_group(), partial_sum[j][r], std::plus<>()); - if constexpr (has_fusion) { - const float gate = sycl::reduce_over_group(nd_item.get_sub_group(), partial_gate[j], std::plus<>()); + if constexpr (has_fusion) { + const float gate = sycl::reduce_over_group(nd_item.get_sub_group(), partial_gate[j][r], std::plus<>()); - // uniform across the launch; the launcher only instantiates SWIGLU and GEGLU - sum *= glu_op == GGML_GLU_OP_SWIGLU ? op_silu(gate) : op_gelu(gate); - } + // uniform across the launch; the launcher only instantiates SWIGLU and GEGLU + sum *= glu_op == GGML_GLU_OP_SWIGLU ? op_silu(gate) : op_gelu(gate); + } - if (sg.leader()) { - dst[j * stride_col_dst + row] = sum; + if (sg.leader() && row0 + r < nrows) { + dst[j * stride_col_dst + row0 + r] = sum; + } } } } @@ -1671,8 +1757,8 @@ static void reorder_mul_mat_vec_q4_k_q8_1_sycl(const void * vx, const void * vy, }); } -template -static void reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols( +template +static void reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols_impl( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int stride_col_y_bytes, const int stride_col_dst, @@ -1680,20 +1766,31 @@ static void reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols( GGML_ASSERT(ncols % QK_K == 0); constexpr size_t num_subgroups = WARP_SIZE; - const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups * rows_per_sg); const sycl::range<3> block_nums(1, 1, block_num_y); const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); stream->submit([&](sycl::handler & cgh) { cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_reorder_ncols, ncols_dst>( + mul_mat_vec_q_reorder_ncols, ncols_dst, + /*has_fusion=*/ false, rows_per_sg>( vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item); }); }); } +template +static void reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + constexpr int rows_per_sg = ncols_dst >= 3 && ncols_dst <= 4 ? 2 : 1; + reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols_impl(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); +} + static void reorder_mul_mat_vec_q4_k_q8_1_sycl_switch_ncols( const void * vx, const void * vy, float * dst, const int ncols, const int nrows, const int ncols_dst, @@ -1701,7 +1798,13 @@ static void reorder_mul_mat_vec_q4_k_q8_1_sycl_switch_ncols( dpct::queue_ptr stream) { switch (ncols_dst) { case 1: reorder_mul_mat_vec_q4_k_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 2: + if (nrows >= Q4_K_MMVQ_ROW_PAIR_MIN_NROWS) { + reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols_impl<2, 2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); + } else { + reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols_impl<2, 1>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); + } + break; case 3: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; case 4: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; case 5: reorder_mul_mat_vec_q4_k_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; @@ -2839,8 +2942,8 @@ bool ggml_sycl_mul_mat_vec_q_id_reorder( } } -template -static void launch_mul_mat_vec_q_reorder_glu(const void * vx, const void * vgate, const void * vy, float * dst, +template +static void launch_mul_mat_vec_q_reorder_glu_impl(const void * vx, const void * vgate, const void * vy, float * dst, const int ncols, const int nrows, const int stride_col_y_bytes, const int stride_col_dst, const ggml_glu_op glu_op, dpct::queue_ptr stream) { @@ -2848,20 +2951,33 @@ static void launch_mul_mat_vec_q_reorder_glu(const void * vx, const void * vgate constexpr size_t num_subgroups = WARP_SIZE; - const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups * rows_per_sg); const sycl::range<3> block_nums(1, 1, block_num_y); const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); stream->submit([&](sycl::handler & cgh) { cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_reorder_ncols( + mul_mat_vec_q_reorder_ncols( vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, nd_item); }); }); } +template +static void launch_mul_mat_vec_q_reorder_glu(const void * vx, const void * vgate, const void * vy, float * dst, + const int ncols, const int nrows, const int stride_col_y_bytes, + const int stride_col_dst, const ggml_glu_op glu_op, + dpct::queue_ptr stream) { + constexpr int rows_per_sg = + reorder_vec_dot_shared_activations::value && ncols_dst >= 3 && ncols_dst <= 4 + ? 2 + : 1; + launch_mul_mat_vec_q_reorder_glu_impl(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, stream); +} + bool ggml_sycl_mul_mat_vec_q_glu_reorder(enum ggml_type src0_type, enum ggml_glu_op glu_op, const void * vx, const void * vgate, const void * vy, float * dst, int ncols, int nrows, int ncols_dst, int stride_col_y_bytes, int stride_col_dst, @@ -2881,8 +2997,11 @@ bool ggml_sycl_mul_mat_vec_q_glu_reorder(enum ggml_type src0_type, enum ggml_glu stride_col_dst, glu_op, stream); return true; case 2: - launch_mul_mat_vec_q_reorder_glu(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, - stride_col_dst, glu_op, stream); + if (nrows >= Q4_K_MMVQ_ROW_PAIR_MIN_NROWS) { + launch_mul_mat_vec_q_reorder_glu_impl(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, stream); + } else { + launch_mul_mat_vec_q_reorder_glu_impl(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, stream); + } return true; case 3: launch_mul_mat_vec_q_reorder_glu(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, diff --git a/ggml/src/ggml-sycl/vecdotq.hpp b/ggml/src/ggml-sycl/vecdotq.hpp index 3ad4cee93a..ed5fd7de89 100644 --- a/ggml/src/ggml-sycl/vecdotq.hpp +++ b/ggml/src/ggml-sycl/vecdotq.hpp @@ -351,6 +351,25 @@ template struct reorder_vec_dot_q_sycl { static_assert(T != T, "ggml_type for reorder vecdot not implemented"); }; +// For some types the weight side of the dot product does not depend on the destination column, so a +// multi-column mul_mat_vec can unpack it once per block instead of once per column. Such a type adds +// load() and dot() next to operator() and opts in here. See reorder_vec_dot_q_sycl. +template struct reorder_vec_dot_shared_weights { + static constexpr bool value = false; +}; + +template <> struct reorder_vec_dot_shared_weights { + static constexpr bool value = true; +}; + +template struct reorder_vec_dot_shared_activations { + static constexpr bool value = false; +}; + +template <> struct reorder_vec_dot_shared_activations { + static constexpr bool value = true; +}; + template <> struct reorder_vec_dot_q_sycl { static constexpr ggml_type gtype = GGML_TYPE_Q4_0; @@ -540,50 +559,84 @@ template <> struct reorder_vec_dot_q_sycl { using q4_k_block = ggml_sycl_reordered::block_q_t; using q4_k_traits = typename q4_k_block::traits; + struct weights { + int v[2]; + uint16_t aux[2]; + ggml_half2 dm; + int bq8_offset; + }; + + struct activations { + int u[2 * QR4_K]; + float d8[QR4_K]; + }; + + __dpct_inline__ static weights load(const void * __restrict__ vbq, const std::pair ibx_offset, + const std::pair d_offset, const int & iqs) { + const uint8_t * base = static_cast(vbq); + const uint8_t * qs = base + ibx_offset.first; + const uint8_t * scs = base + d_offset.first; + const ggml_half2 * dms = reinterpret_cast(base + d_offset.second); + + weights w; + w.bq8_offset = QR4_K * ((iqs / 2) / (QI8_1 / 2)); + + const int * q4 = (const int *) (qs + 16 * w.bq8_offset + 4 * ((iqs / 2) % 4)); + const uint16_t * scales = (const uint16_t *) scs; + + w.v[0] = q4[0]; + w.v[1] = q4[4]; + + const int j = (QR4_K * ((iqs / 2) / (QI8_1 / 2))) / 2; + if (j < 2) { + w.aux[0] = scales[j + 0] & 0x3f3f; + w.aux[1] = scales[j + 2] & 0x3f3f; + } else { + w.aux[0] = ((scales[j + 2] >> 0) & 0x0f0f) | ((scales[j - 2] & 0xc0c0) >> 2); + w.aux[1] = ((scales[j + 2] >> 4) & 0x0f0f) | ((scales[j - 0] & 0xc0c0) >> 2); + } + + w.dm = *dms; + + return w; + } + + __dpct_inline__ static activations load_activations(const int8_t * q8_1_quant_ptr, + const sycl::half2 * q8_1_ds, const int & iqs) { + activations a; + const int bq8_offset = QR4_K * ((iqs / 2) / (QI8_1 / 2)); + for (int i = 0; i < QR4_K; ++i) { + const int8_t * quant_base_ptr = q8_1_quant_ptr + (bq8_offset + i) * QK8_1; + sycl::half2 ds_values = *(q8_1_ds + bq8_offset + i); + + a.d8[i] = ds_values[0]; + + const int * q8 = (const int *) quant_base_ptr + ((iqs / 2) % 4); + a.u[2 * i + 0] = q8[0]; + a.u[2 * i + 1] = q8[4]; + } + + return a; + } + + __dpct_inline__ static float apply(const weights & w, const activations & a) { + const uint8_t * sc = (const uint8_t *) w.aux; + const uint8_t * m = sc + 2; + + return vec_dot_q4_K_q8_1_impl_vmmq(w.v, a.u, sc, m, w.dm, a.d8); + } + + __dpct_inline__ static float dot(const weights & w, const int8_t * q8_1_quant_ptr, + const sycl::half2 * q8_1_ds, const int & iqs) { + const auto a = load_activations(q8_1_quant_ptr, q8_1_ds, iqs); + + return apply(w, a); + } + __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair ibx_offset, const std::pair d_offset, const int8_t * q8_1_quant_ptr, const sycl::half2 * q8_1_ds, const int & iqs) { - const uint8_t * base = static_cast(vbq); - const uint8_t * qs = base + ibx_offset.first; - const uint8_t * scs = base + d_offset.first; - const ggml_half2 * dms = reinterpret_cast(base + d_offset.second); - - const int bq8_offset = QR4_K * ((iqs / 2) / (QI8_1 / 2)); - const int * q4 = (const int *) (qs + 16 * bq8_offset + 4 * ((iqs / 2) % 4)); - const uint16_t * scales = (const uint16_t *) scs; - - int v[2]; - int u[2 * QR4_K]; - float d8[QR4_K]; - - v[0] = q4[0]; - v[1] = q4[4]; - - uint16_t aux[2]; - const int j = (QR4_K * ((iqs / 2) / (QI8_1 / 2))) / 2; - if (j < 2) { - aux[0] = scales[j + 0] & 0x3f3f; - aux[1] = scales[j + 2] & 0x3f3f; - } else { - aux[0] = ((scales[j + 2] >> 0) & 0x0f0f) | ((scales[j - 2] & 0xc0c0) >> 2); - aux[1] = ((scales[j + 2] >> 4) & 0x0f0f) | ((scales[j - 0] & 0xc0c0) >> 2); - } - - const uint8_t * sc = (const uint8_t *) aux; - const uint8_t * m = sc + 2; - - for (int i = 0; i < QR4_K; ++i) { - const int8_t* quant_base_ptr = q8_1_quant_ptr + (bq8_offset + i) * QK8_1; - sycl::half2 ds_values = *(q8_1_ds + bq8_offset + i); - - d8[i] = ds_values[0]; - - const int * q8 = (const int *) quant_base_ptr + ((iqs / 2) % 4); - u[2 * i + 0] = q8[0]; - u[2 * i + 1] = q8[4]; - } - - return vec_dot_q4_K_q8_1_impl_vmmq(v, u, sc, m, *dms, d8); + return dot(load(vbq, ibx_offset, d_offset, iqs), q8_1_quant_ptr, q8_1_ds, iqs); } }; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 1b0eaca8f0..154146dda4 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -9435,6 +9435,21 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 16, 8, 16*256, { 1, 1}, {1, 1})); } + // Multi-column MMVQ coverage for the Q4_K weight-reuse path and a Q5_K control. + for (ggml_type type_a : { GGML_TYPE_Q4_K, GGML_TYPE_Q5_K }) { + for (int n = 1; n <= 8; ++n) { + test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 4096, n, 1024, { 1, 1 }, { 1, 1 })); + test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 1023, n, 4096, { 1, 1 }, { 1, 1 })); + } + } + + // The SYCL backend picks between one and two output rows per subgroup by row count when there + // are two destination columns (Q4_K_MMVQ_ROW_PAIR_MIN_NROWS in ggml-sycl/mmvq.cpp). Cover both + // sides of that boundary, including an odd row count above it for the row-pair tail. + for (int64_t m : {6271, 6272, 6273}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, m, 2, 1024, { 1, 1 }, { 1, 1 })); + } + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1})); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1})); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_MXFP4, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1})); @@ -10364,6 +10379,22 @@ static std::vector> make_test_cases_eval() { true, 16, 8, b, false, true, false)); } + // Fused row-pair coverage: minimum rows, an even pair, and an odd tail. + for (ggml_glu_op glu_op : { GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU }) { + for (int64_t m_batch : { 2, 3, 4 }) { + for (int64_t rows : { 1, 2, 3 }) { + test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q4_K, glu_op, m_batch, rows, 256, + false, 16, 8, false, false, true, false, { 1, 1 })); + } + } + } + + // Both sides of the same row-count boundary as above, on the fused path. + for (int64_t rows : {6271, 6272, 6273}) { + test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q4_K, GGML_GLU_OP_SWIGLU, 2, rows, 256, + false, 16, 8, false, false, true, false, { 1, 1 })); + } + for (auto gate : {GATING_FUNC_SOFTMAX, GATING_FUNC_SIGMOID, GATING_FUNC_SOFTMAX_WEIGHT, GATING_FUNC_SQRT_SOFTPLUS}) { for (bool with_norm : {false, true}) { for (bool bias_probs : {false, true}) { @@ -10651,6 +10682,16 @@ static std::vector> make_test_cases_perf() { } } + // Q4_K multi-column mat-vec, at ffn_up/ffn_gate geometry (k = n_embd, m = n_ff): n sweeps the + // per-column specializations used for short prompts and speculative/MTP verify, and m brackets + // the row count at which the SYCL backend switches to two output rows per subgroup + // (Q4_K_MMVQ_ROW_PAIR_MIN_NROWS in ggml-sycl/mmvq.cpp), so both sides of it can be measured. + for (int64_t m : {4096, 6144, 6272, 14336}) { + for (int bs : {1, 2, 3, 4, 8}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, m, bs, 4096, {1, 1}, {1, 1})); + } + } + // qwen3-30b-a3b for (int bs : {1, 4, 8, 32, 64, 128, 256, 512}) { for (ggml_type type_a : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, GGML_TYPE_IQ2_XS}) {