Superminaren/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-04 02:37:27 +02:00
Code Issues Packages Projects Releases Wiki Activity

revert iq shmem table renames

Browse Source
This commit is contained in:
Ruben Ortlam
2026-09-03 13:13:38 +02:00
parent 842771698a
commit 36be26cc08
14 changed files with 58 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl
+10 -10
View File
@@ -244,7 +244,7 @@ vec2 dequantize(uint ib, uint iqs, uint a_offset) {
// Add parity bit
const uint sign8 = sign7 | (bitCount(sign7) << 7);
const uint sign = sign8 >> (iqs % 8);
const u8vec4 grid = unpack8(iq2_grid[qs][(iqs % 8) / 4] >> (8 * (iqs % 4)));
const u8vec4 grid = unpack8(iq2xxs_grid[qs][(iqs % 8) / 4] >> (8 * (iqs % 4)));
bool sign0 = (sign & 1) != 0;
bool sign1 = (sign & 2) != 0;
return db * vec2(
@@ -264,7 +264,7 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) {
// Add parity bit
const uint sign8 = sign7 | (bitCount(sign7) << 7);
const uint sign = sign8 >> (iqs % 8);
const u8vec4 grid = unpack8(iq2_grid[qs][(iqs % 8) / 4] >> (8 * (iqs % 4)));
const u8vec4 grid = unpack8(iq2xxs_grid[qs][(iqs % 8) / 4] >> (8 * (iqs % 4)));
bool sign0 = (sign & 1) != 0;
bool sign1 = (sign & 2) != 0;
bool sign2 = (sign & 4) != 0;
@@ -287,7 +287,7 @@ vec2 dequantize(uint ib, uint iqs, uint a_offset) {
// Add parity bit
const uint sign8 = sign7 | (bitCount(sign7) << 7);
const uint sign = sign8 >> (iqs % 8);
const u8vec4 grid = unpack8(iq2_grid[qs & 511][(iqs % 8) / 4] >> (8 * (iqs % 4)));
const u8vec4 grid = unpack8(iq2xs_grid[qs & 511][(iqs % 8) / 4] >> (8 * (iqs % 4)));
bool sign0 = (sign & 1) != 0;
bool sign1 = (sign & 2) != 0;
return db * vec2(
@@ -303,7 +303,7 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) {
// Add parity bit
const uint sign8 = sign7 | (bitCount(sign7) << 7);
const uint sign = sign8 >> (iqs % 8);
const u8vec4 grid = unpack8(iq2_grid[qs & 511][(iqs % 8) / 4] >> (8 * (iqs % 4)));
const u8vec4 grid = unpack8(iq2xs_grid[qs & 511][(iqs % 8) / 4] >> (8 * (iqs % 4)));
bool sign0 = (sign & 1) != 0;
bool sign1 = (sign & 2) != 0;
bool sign2 = (sign & 4) != 0;
@@ -329,7 +329,7 @@ vec2 dequantize(uint ib, uint iqs, uint a_offset) {
const uint sign = data_a[a_offset + ib].qs[QUANT_K / 8 + ib8] >> (iqs % 8);
const float db = 0.25 * (0.5 + scale);
const u8vec4 grid = unpack8(iq2_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(iqs % 8) / 4]);
const u8vec4 grid = unpack8(iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(iqs % 8) / 4]);
bool sign0 = (sign & 1) != 0;
bool sign1 = (sign & 2) != 0;
return db * vec2(
@@ -348,7 +348,7 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) {
const uint sign = data_a[a_offset + ib].qs[QUANT_K / 8 + ib8] >> (iqs % 8);
const float db = 0.25 * (0.5 + scale);
const u8vec4 grid = unpack8(iq2_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(iqs % 8) / 4]);
const u8vec4 grid = unpack8(iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(iqs % 8) / 4]);
bool sign0 = (sign & 1) != 0;
bool sign1 = (sign & 2) != 0;
bool sign2 = (sign & 4) != 0;
@@ -376,7 +376,7 @@ vec2 dequantize(uint ib, uint iqs, uint a_offset) {
// Add parity bit
const uint sign8 = sign7 | (bitCount(sign7) << 7);
const uint sign = sign8 >> (iqs % 8);
const u8vec4 grid = unpack8(iq3_grid[qs] >> (8 * (iqs % 4)));
const u8vec4 grid = unpack8(iq3xxs_grid[qs] >> (8 * (iqs % 4)));
bool sign0 = (sign & 1) != 0;
bool sign1 = (sign & 2) != 0;
return db * vec2(
@@ -396,7 +396,7 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) {
// Add parity bit
const uint sign8 = sign7 | (bitCount(sign7) << 7);
const uint sign = sign8 >> (iqs % 8);
const u8vec4 grid = unpack8(iq3_grid[qs]);
const u8vec4 grid = unpack8(iq3xxs_grid[qs]);
bool sign0 = (sign & 1) != 0;
bool sign1 = (sign & 2) != 0;
bool sign2 = (sign & 4) != 0;
@@ -419,7 +419,7 @@ vec2 dequantize(uint ib, uint iqs, uint a_offset) {
bool sign0 = (sign & 1) != 0;
bool sign1 = (sign & 2) != 0;
const float db = 1 + 2 * ((scale >> (4 * ((iqs / 32) & 1))) & 0xf);
const uint32_t grid = iq3_grid[qs | ((qh << (8 - ((iqs / 4) % 8))) & 256)] >> (8 * (iqs % 4));
const uint32_t grid = iq3s_grid[qs | ((qh << (8 - ((iqs / 4) % 8))) & 256)] >> (8 * (iqs % 4));
return db * vec2(
int(grid & 0xFF) * (sign0 ? -1.0 : 1.0),
int((grid >> 8) & 0xFF) * (sign1 ? -1.0 : 1.0)
@@ -437,7 +437,7 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) {
bool sign2 = (sign & 4) != 0;
bool sign3 = (sign & 8) != 0;
const float db = 1 + 2 * ((scale >> (4 * (ib32 & 1))) & 0xf);
const uint32_t grid = iq3_grid[qs | ((qh << (8 - ib4 % 8)) & 256)] >> (8 * (iqs % 4));
const uint32_t grid = iq3s_grid[qs | ((qh << (8 - ib4 % 8)) & 256)] >> (8 * (iqs % 4));
return db * vec4(
int(grid & 0xFF) * (sign0 ? -1.0 : 1.0),
int((grid >> 8) & 0xFF) * (sign1 ? -1.0 : 1.0),
ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl
+10 -10
View File
@@ -936,7 +936,7 @@ float16_t dequantFuncIQ2_XXS(const in decodeBufIQ2_XXS bl, const in uint blockCo
uint sign = bitfieldExtract(signscale, 7 * int(ib8), 7);
sign |= bitCount(sign) << 7;
uint g2 = iq2_grid[qs][(idx & 4) >> 2];
uint g2 = iq2xxs_grid[qs][(idx & 4) >> 2];
g2 >>= (idx & 2) * 8;
const vec2 g = vec2(unpack8(g2));
@@ -961,7 +961,7 @@ f16vec4 dequantFuncIQ2_XXS_v(const in decodeBufIQ2_XXS bl, const in uint blockCo
sign |= bitCount(sign) << 7;
const uint sb = sign >> (idx & 7u);
const uint g2 = iq2_grid[qs][(idx & 4) >> 2];
const uint g2 = iq2xxs_grid[qs][(idx & 4) >> 2];
const u8vec4 g = unpack8(g2);
return f16vec4(
@@ -991,7 +991,7 @@ float16_t dequantFuncIQ2_XS(const in decodeBufIQ2_XS bl, const in uint blockCoor
uint sign = uint(qs >> 9);
sign |= bitCount(sign) << 7;
uint g2 = iq2_grid[qs & 0x1FF][(idx & 4) >> 2];
uint g2 = iq2xs_grid[qs & 0x1FF][(idx & 4) >> 2];
g2 >>= (idx & 2) * 8;
const vec2 g = vec2(unpack8(g2));
@@ -1014,7 +1014,7 @@ f16vec4 dequantFuncIQ2_XS_v(const in decodeBufIQ2_XS bl, const in uint blockCoor
sign |= bitCount(sign) << 7;
const uint sb = sign >> (idx & 7u);
const uint g2 = iq2_grid[qs & 0x1FF][(idx & 4) >> 2];
const uint g2 = iq2xs_grid[qs & 0x1FF][(idx & 4) >> 2];
const u8vec4 g = unpack8(g2);
return f16vec4(
@@ -1046,7 +1046,7 @@ float16_t dequantFuncIQ2_S(const in decodeBufIQ2_S bl, const in uint blockCoords
const float d = float(bl.block.d);
const float db = d * 0.25 * (0.5 + scale);
const ivec2 sign01 = 1 - (2 & ivec2(sign << 1, sign));
uint g2 = iq2_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(idx & 4) >> 2];
uint g2 = iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(idx & 4) >> 2];
g2 >>= (idx & 2) * 8;
const vec2 v = db * vec2(sign01) * vec2(unpack8(g2));
return float16_t(v[idx & 1]);
@@ -1068,7 +1068,7 @@ f16vec4 dequantFuncIQ2_S_v(const in decodeBufIQ2_S bl, const in uint blockCoords
const float d = float(bl.block.d);
const float db = d * 0.25 * (0.5 + scale);
const uint g2 = iq2_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(idx & 4) >> 2];
const uint g2 = iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(idx & 4) >> 2];
const u8vec4 g = unpack8(g2);
return f16vec4(
@@ -1106,7 +1106,7 @@ float16_t dequantFuncIQ3_XXS(const in decodeBufIQ3_XXS bl, const in uint blockCo
const uint32_t sign7 = bitfieldExtract(signs, 7 * (int(iqs / 2) % 4), 7);
const uint sign = (sign7 | (bitCount(sign7) << 7)) >> (idx & 0x6);
const ivec2 sign01 = ivec2(1 - (2 & ivec2(sign << 1, sign)));
const uint grid = iq3_grid[qs] >> (16 * ((idx & 2) >> 1));
const uint grid = iq3xxs_grid[qs] >> (16 * ((idx & 2) >> 1));
const vec2 v = db * vec2(sign01) * vec2(unpack8(grid).xy);
return float16_t(v[idx & 1]);
}
@@ -1127,7 +1127,7 @@ f16vec4 dequantFuncIQ3_XXS_v(const in decodeBufIQ3_XXS bl, const in uint blockCo
const uint sign7 = bitfieldExtract(signs, 7 * (int(iqs / 2) % 4), 7);
const uint sb = (sign7 | (bitCount(sign7) << 7)) >> (idx & 0x6u);
const uint grid = iq3_grid[qs];
const uint grid = iq3xxs_grid[qs];
const u8vec4 g = unpack8(grid);
return f16vec4(
@@ -1157,7 +1157,7 @@ float16_t dequantFuncIQ3_S(const in decodeBufIQ3_S bl, const in uint blockCoords
const uint scale = bl.block.scales[iqs / 16];
const ivec2 sign01 = ivec2(1 - (2 & ivec2(sign << 1, sign)));
const float db = d * (1 + 2 * ((scale >> (4 * (iqh & 1))) & 0xf));
const uint32_t grid = iq3_grid[qs | ((qh << (8 - (iqs % 8))) & 256)] >> ((idx & 2) << 3);
const uint32_t grid = iq3s_grid[qs | ((qh << (8 - (iqs % 8))) & 256)] >> ((idx & 2) << 3);
const vec2 v = db * vec2(sign01) * vec2(unpack8(grid).xy);
return float16_t(v[idx & 1]);
@@ -1177,7 +1177,7 @@ f16vec4 dequantFuncIQ3_S_v(const in decodeBufIQ3_S bl, const in uint blockCoords
const uint scale = bl.block.scales[iqs / 16];
const float db = d * (1 + 2 * ((scale >> (4 * (iqh & 1))) & 0xf));
const uint grid = iq3_grid[qs | ((qh << (8 - (iqs % 8))) & 256)];
const uint grid = iq3s_grid[qs | ((qh << (8 - (iqs % 8))) & 256)];
const u8vec4 g = unpack8(grid);
return f16vec4(
ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp
+1 -1
View File
@@ -29,7 +29,7 @@ void main() {
uint qs = data_a[ib].qs[4 * ib32 + l];
const uint8_t sign = data_a[ib].qs[QUANT_K / 8 + 4 * ib32 + l];
qs |= (qh << (8 - 2 * l)) & 0x300;
const uvec2 grid = iq2_grid[qs];
const uvec2 grid = iq2s_grid[qs];
const u8vec4 grid0 = unpack8(grid.x);
const u8vec4 grid1 = unpack8(grid.y);
data_b[b_idx + 8 * l + 0] = D_TYPE(db[l/2] * grid0.x * ((sign & 1) != 0 ? -1.0 : 1.0));
ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp
+1 -1
View File
@@ -28,7 +28,7 @@ void main() {
uint16_t qs = data_a[ib].qs[4 * ib32 + l];
const uint sign7 = qs >> 9;
const uint sign8 = sign7 | (bitCount(sign7) << 7); // parity bit
const uvec2 grid = iq2_grid[qs & 511];
const uvec2 grid = iq2xs_grid[qs & 511];
const u8vec4 grid0 = unpack8(grid.x);
const u8vec4 grid1 = unpack8(grid.y);
data_b[b_idx + 8 * l + 0] = D_TYPE(db[l/2] * grid0.x * ((sign8 & 1) != 0 ? -1.0 : 1.0));
ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp
+1 -1
View File
@@ -34,7 +34,7 @@ void main() {
const uint sign7 = bitfieldExtract(signscale, 7 * int(l), 7);
const uint sign8 = sign7 | (bitCount(sign7) << 7); // parity bit
const uint qs = data_a[ib].qs[8 * is + l];
const uvec2 grid = iq2_grid[qs];
const uvec2 grid = iq2xxs_grid[qs];
const u8vec4 grid0 = unpack8(grid.x);
const u8vec4 grid1 = unpack8(grid.y);
data_b[b_idx + 8 * l + 0] = D_TYPE(db * grid0.x * ((sign8 & 1) != 0 ? -1.0 : 1.0));
ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp
+1 -1
View File
@@ -31,7 +31,7 @@ void main() {
const uint qs = data_a[ib].qs[iqs];
const uint gidx = qs | ((qh << (8 - l)) & 256);
const uint8_t signs = data_a[ib].signs[iqs / 2] >> (4 * (l & 1));
const u8vec4 grid = unpack8(iq3_grid[gidx]);
const u8vec4 grid = unpack8(iq3s_grid[gidx]);
data_b[b_idx + 4 * l + 0] = D_TYPE(db * grid.x * ((signs & 1) != 0 ? -1.0 : 1.0));
data_b[b_idx + 4 * l + 1] = D_TYPE(db * grid.y * ((signs & 2) != 0 ? -1.0 : 1.0));
data_b[b_idx + 4 * l + 2] = D_TYPE(db * grid.z * ((signs & 4) != 0 ? -1.0 : 1.0));
ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp
+2 -2
View File
@@ -37,8 +37,8 @@ void main() {
const uint sign8 = sign7 | (bitCount(sign7) << 7);
const uint qs0 = data_a[ib].qs[8 * is + 2 * l];
const uint qs1 = data_a[ib].qs[8 * is + 2 * l + 1];
const u8vec4 grid0 = unpack8(iq3_grid[qs0]);
const u8vec4 grid1 = unpack8(iq3_grid[qs1]);
const u8vec4 grid0 = unpack8(iq3xxs_grid[qs0]);
const u8vec4 grid1 = unpack8(iq3xxs_grid[qs1]);
data_b[b_idx + 8 * l + 0] = D_TYPE(db * grid0.x * ((sign8 & 1) != 0 ? -1.0 : 1.0));
data_b[b_idx + 8 * l + 1] = D_TYPE(db * grid0.y * ((sign8 & 2) != 0 ? -1.0 : 1.0));
data_b[b_idx + 8 * l + 2] = D_TYPE(db * grid0.z * ((sign8 & 4) != 0 ? -1.0 : 1.0));
ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp
+1 -1
View File
@@ -24,7 +24,7 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid,
[[unroll]] for (uint l = 0; l < 2; ++l) {
const uint8_t sign = sign16[l];
const uint qs = qs16[l] | ((qh << (8 - nibble_shift - 2 * l)) & 0x300);
const uvec2 grid = iq2_grid[qs];
const uvec2 grid = iq2s_grid[qs];
const vec4 grid0 = vec4(unpack8(grid.x));
const vec4 grid1 = vec4(unpack8(grid.y));
ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp
+1 -1
View File
@@ -31,7 +31,7 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid,
const uint qs = data_a[ibi].qs[2 * itid + l];
sign_vals[l] = qs >> 9;
sign7_vals[l] = bitCount(sign_vals[l]);
const uvec2 grid_data = iq2_grid[qs & 511];
const uvec2 grid_data = iq2xs_grid[qs & 511];
grid0_vals[l] = vec4(unpack8(grid_data.x));
grid1_vals[l] = vec4(unpack8(grid_data.y));
}
ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp
+2 -2
View File
@@ -22,8 +22,8 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid,
const uint qs = data_a[ibi].qs[8 * ib32 + 2 * (itid & 1) + l];
const uint sign = bitfieldExtract(signscale, 7 * int(2 * (itid & 1) + l), 7);
const uint sign7 = bitCount(sign);
const vec4 grid0 = vec4(unpack8(iq2_grid[qs].x));
const vec4 grid1 = vec4(unpack8(iq2_grid[qs].y));
const vec4 grid0 = vec4(unpack8(iq2xxs_grid[qs].x));
const vec4 grid1 = vec4(unpack8(iq2xxs_grid[qs].y));
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
const vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]);
ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp
+2 -2
View File
@@ -27,8 +27,8 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid,
const uint l = l0 + ll;
const u8vec2 qs = unpack8(uint32_t(data_a_packed16[ibi].qs[4 * ib32 + l])).xy; // vec4 used due to #12147
const uint sign = data_a[ibi].signs[4 * ib32 + l];
const vec4 grid0 = vec4(unpack8(iq3_grid[qs.x | ((qh << (8 - 2*l)) & 0x100)]));
const vec4 grid1 = vec4(unpack8(iq3_grid[qs.y | ((qh << (7 - 2*l)) & 0x100)]));
const vec4 grid0 = vec4(unpack8(iq3s_grid[qs.x | ((qh << (8 - 2*l)) & 0x100)]));
const vec4 grid1 = vec4(unpack8(iq3s_grid[qs.y | ((qh << (7 - 2*l)) & 0x100)]));
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
const vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]);
ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp
+2 -2
View File
@@ -23,8 +23,8 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid,
const uint qs1 = data_a[ibi].qs[8 * ib32 + 4 * (itid & 1) + 2 * l + 1];
const uint sign = bitfieldExtract(signscale, 7 * int(2 * (itid & 1) + l), 7);
const uint sign7 = bitCount(sign);
const vec4 grid0 = vec4(unpack8(iq3_grid[qs0]));
const vec4 grid1 = vec4(unpack8(iq3_grid[qs1]));
const vec4 grid0 = vec4(unpack8(iq3xxs_grid[qs0]));
const vec4 grid1 = vec4(unpack8(iq3xxs_grid[qs1]));
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
const vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]);
ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl
+5 -5
View File
@@ -131,7 +131,7 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin
const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + (signs >> 28)));
const uint32_t sign7 = bitfieldExtract(signs, 7 * int(ib8), 7);
const uint sign = sign7 | (bitCount(sign7) << 7);
const uvec2 grid = iq2_grid[qs];
const uvec2 grid = iq2xxs_grid[qs];
const vec4 grid0 = vec4(unpack8(grid.x));
const vec4 grid1 = vec4(unpack8(grid.y));
@@ -161,7 +161,7 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin
const uint qs = data_a[ib].qs[4 * ib32 + ib8];
const uint sign7 = qs >> 9;
const uint sign = sign7 | (bitCount(sign7) << 7);
const uvec2 grid = iq2_grid[qs & 511];
const uvec2 grid = iq2xs_grid[qs & 511];
const vec4 grid0 = vec4(unpack8(grid.x));
const vec4 grid1 = vec4(unpack8(grid.y));
@@ -193,7 +193,7 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin
const float d = float(data_a[ib].d);
const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale));
const uvec2 grid = iq2_grid[qs | ((qh << (8 - qhshift)) & 0x300)];
const uvec2 grid = iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)];
const vec4 grid0 = vec4(unpack8(grid.x));
const vec4 grid1 = vec4(unpack8(grid.y));
@@ -226,7 +226,7 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin
const float db = d * 0.5 * (0.5 + (signs >> 28));
const uint32_t sign7 = bitfieldExtract(signs, 7 * (int(iqs / 2) % 4), 7);
const uint sign = (sign7 | (bitCount(sign7) << 7)) >> (4 * (idx % 2));
const uint grid = iq3_grid[qs];
const uint grid = iq3xxs_grid[qs];
const vec4 v = db * vec4(unpack8(grid));
store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x,
@@ -250,7 +250,7 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin
const uint scale = data_a[ib].scales[iqs / 16];
const i8vec2 sign01 = i8vec2(1 - (2 & i8vec2(sign << 1, sign)));
const float db = d * (1 + 2 * ((scale >> (4 * (iqh & 1))) & 0xf));
const uint32_t grid = iq3_grid[qs | ((qh << (8 - (iqs % 8))) & 256)];
const uint32_t grid = iq3s_grid[qs | ((qh << (8 - (iqs % 8))) & 256)];
const vec4 v = db * vec4(unpack8(grid));
store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x,
ggml/src/ggml-vulkan/vulkan-shaders/types.glsl
+19 -19
View File
@@ -948,11 +948,11 @@ void init_iq_shmem(uvec3 wgsize)
#if defined(DATA_A_IQ2_XXS) || defined(DATA_A_IQ2_XS) || defined(DATA_A_IQ2_S)
#if defined(DATA_A_IQ2_S)
shared uvec2 iq2_grid[1024];
shared uvec2 iq2s_grid[1024];
#elif defined(DATA_A_IQ2_XS)
shared uvec2 iq2_grid[512];
shared uvec2 iq2xs_grid[512];
#else
shared uvec2 iq2_grid[256];
shared uvec2 iq2xxs_grid[256];
#endif
#endif
@@ -1045,9 +1045,9 @@ const uvec2[256] iq2xxs_grid_const = {
void init_iq_shmem(uvec3 wgsize)
{
// copy the table into shared memory and sync
[[unroll]] for (uint i = 0; i < iq2_grid.length(); i += wgsize.x) {
[[unroll]] for (uint i = 0; i < iq2xxs_grid.length(); i += wgsize.x) {
if (iq2xxs_grid_const.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2xxs_grid_const.length()) {
iq2_grid[i + gl_LocalInvocationIndex.x] = iq2xxs_grid_const[i + gl_LocalInvocationIndex.x];
iq2xxs_grid[i + gl_LocalInvocationIndex.x] = iq2xxs_grid_const[i + gl_LocalInvocationIndex.x];
}
}
barrier();
@@ -1217,9 +1217,9 @@ const uvec2 iq2xs_grid_const[512] = {
void init_iq_shmem(uvec3 wgsize)
{
// copy the table into shared memory and sync
[[unroll]] for (uint i = 0; i < iq2_grid.length(); i += wgsize.x) {
if (iq2_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2xs_grid_const.length()) {
iq2_grid[i + gl_LocalInvocationIndex.x] = iq2xs_grid_const[i + gl_LocalInvocationIndex.x];
[[unroll]] for (uint i = 0; i < iq2xs_grid.length(); i += wgsize.x) {
if (iq2xs_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2xs_grid_const.length()) {
iq2xs_grid[i + gl_LocalInvocationIndex.x] = iq2xs_grid_const[i + gl_LocalInvocationIndex.x];
}
}
barrier();
@@ -1519,9 +1519,9 @@ const uvec2 iq2s_grid_const[1024] = {
void init_iq_shmem(uvec3 wgsize)
{
// copy the table into shared memory and sync
[[unroll]] for (uint i = 0; i < iq2_grid.length(); i += wgsize.x) {
if (iq2_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2s_grid_const.length()) {
iq2_grid[i + gl_LocalInvocationIndex.x] = iq2s_grid_const[i + gl_LocalInvocationIndex.x];
[[unroll]] for (uint i = 0; i < iq2s_grid.length(); i += wgsize.x) {
if (iq2s_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2s_grid_const.length()) {
iq2s_grid[i + gl_LocalInvocationIndex.x] = iq2s_grid_const[i + gl_LocalInvocationIndex.x];
}
}
barrier();
@@ -1538,9 +1538,9 @@ void init_iq_shmem(uvec3 wgsize)
#if defined(DATA_A_IQ3_XXS) || defined(DATA_A_IQ3_S)
#if defined(DATA_A_IQ3_S)
shared uint32_t iq3_grid[512];
shared uint32_t iq3s_grid[512];
#else
shared uint32_t iq3_grid[256];
shared uint32_t iq3xxs_grid[256];
#endif
#endif
@@ -1601,9 +1601,9 @@ const uint32_t iq3xxs_grid_const[256] = {
void init_iq_shmem(uvec3 wgsize)
{
// copy the table into shared memory and sync
[[unroll]] for (uint i = 0; i < iq3_grid.length(); i += wgsize.x) {
if (iq3_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq3_grid.length()) {
iq3_grid[i + gl_LocalInvocationIndex.x] = iq3xxs_grid_const[i + gl_LocalInvocationIndex.x];
[[unroll]] for (uint i = 0; i < iq3xxs_grid.length(); i += wgsize.x) {
if (iq3xxs_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq3xxs_grid.length()) {
iq3xxs_grid[i + gl_LocalInvocationIndex.x] = iq3xxs_grid_const[i + gl_LocalInvocationIndex.x];
}
}
barrier();
@@ -1713,9 +1713,9 @@ const uint32_t iq3s_grid_const[512] = {
void init_iq_shmem(uvec3 wgsize)
{
// copy the table into shared memory and sync
[[unroll]] for (uint i = 0; i < iq3_grid.length(); i += wgsize.x) {
if (iq3_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq3_grid.length()) {
iq3_grid[i + gl_LocalInvocationIndex.x] = iq3s_grid_const[i + gl_LocalInvocationIndex.x];
[[unroll]] for (uint i = 0; i < iq3s_grid.length(); i += wgsize.x) {
if (iq3s_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq3s_grid.length()) {
iq3s_grid[i + gl_LocalInvocationIndex.x] = iq3s_grid_const[i + gl_LocalInvocationIndex.x];
}
}
barrier();