From 36be26cc08ec540c236bf8f56aa7d3d73e004aec Mon Sep 17 00:00:00 2001 From: Ruben Ortlam Date: Mon, 31 Aug 2026 07:32:55 +0200 Subject: [PATCH] revert iq shmem table renames --- .../vulkan-shaders/dequant_funcs.glsl | 20 +++++----- .../vulkan-shaders/dequant_funcs_cm2.glsl | 20 +++++----- .../vulkan-shaders/dequant_iq2_s.comp | 2 +- .../vulkan-shaders/dequant_iq2_xs.comp | 2 +- .../vulkan-shaders/dequant_iq2_xxs.comp | 2 +- .../vulkan-shaders/dequant_iq3_s.comp | 2 +- .../vulkan-shaders/dequant_iq3_xxs.comp | 4 +- .../vulkan-shaders/mul_mat_vec_iq2_s.comp | 2 +- .../vulkan-shaders/mul_mat_vec_iq2_xs.comp | 2 +- .../vulkan-shaders/mul_mat_vec_iq2_xxs.comp | 4 +- .../vulkan-shaders/mul_mat_vec_iq3_s.comp | 4 +- .../vulkan-shaders/mul_mat_vec_iq3_xxs.comp | 4 +- .../vulkan-shaders/mul_mm_funcs.glsl | 10 ++--- .../src/ggml-vulkan/vulkan-shaders/types.glsl | 38 +++++++++---------- 14 files changed, 58 insertions(+), 58 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl index 6c852fb426..627932bd35 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl @@ -244,7 +244,7 @@ vec2 dequantize(uint ib, uint iqs, uint a_offset) { // Add parity bit const uint sign8 = sign7 | (bitCount(sign7) << 7); const uint sign = sign8 >> (iqs % 8); - const u8vec4 grid = unpack8(iq2_grid[qs][(iqs % 8) / 4] >> (8 * (iqs % 4))); + const u8vec4 grid = unpack8(iq2xxs_grid[qs][(iqs % 8) / 4] >> (8 * (iqs % 4))); bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; return db * vec2( @@ -264,7 +264,7 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) { // Add parity bit const uint sign8 = sign7 | (bitCount(sign7) << 7); const uint sign = sign8 >> (iqs % 8); - const u8vec4 grid = unpack8(iq2_grid[qs][(iqs % 8) / 4] >> (8 * (iqs % 4))); + const u8vec4 grid = unpack8(iq2xxs_grid[qs][(iqs % 8) / 4] >> (8 * (iqs % 4))); bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; bool sign2 = (sign & 4) != 0; @@ -287,7 +287,7 @@ vec2 dequantize(uint ib, uint iqs, uint a_offset) { // Add parity bit const uint sign8 = sign7 | (bitCount(sign7) << 7); const uint sign = sign8 >> (iqs % 8); - const u8vec4 grid = unpack8(iq2_grid[qs & 511][(iqs % 8) / 4] >> (8 * (iqs % 4))); + const u8vec4 grid = unpack8(iq2xs_grid[qs & 511][(iqs % 8) / 4] >> (8 * (iqs % 4))); bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; return db * vec2( @@ -303,7 +303,7 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) { // Add parity bit const uint sign8 = sign7 | (bitCount(sign7) << 7); const uint sign = sign8 >> (iqs % 8); - const u8vec4 grid = unpack8(iq2_grid[qs & 511][(iqs % 8) / 4] >> (8 * (iqs % 4))); + const u8vec4 grid = unpack8(iq2xs_grid[qs & 511][(iqs % 8) / 4] >> (8 * (iqs % 4))); bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; bool sign2 = (sign & 4) != 0; @@ -329,7 +329,7 @@ vec2 dequantize(uint ib, uint iqs, uint a_offset) { const uint sign = data_a[a_offset + ib].qs[QUANT_K / 8 + ib8] >> (iqs % 8); const float db = 0.25 * (0.5 + scale); - const u8vec4 grid = unpack8(iq2_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(iqs % 8) / 4]); + const u8vec4 grid = unpack8(iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(iqs % 8) / 4]); bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; return db * vec2( @@ -348,7 +348,7 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) { const uint sign = data_a[a_offset + ib].qs[QUANT_K / 8 + ib8] >> (iqs % 8); const float db = 0.25 * (0.5 + scale); - const u8vec4 grid = unpack8(iq2_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(iqs % 8) / 4]); + const u8vec4 grid = unpack8(iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(iqs % 8) / 4]); bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; bool sign2 = (sign & 4) != 0; @@ -376,7 +376,7 @@ vec2 dequantize(uint ib, uint iqs, uint a_offset) { // Add parity bit const uint sign8 = sign7 | (bitCount(sign7) << 7); const uint sign = sign8 >> (iqs % 8); - const u8vec4 grid = unpack8(iq3_grid[qs] >> (8 * (iqs % 4))); + const u8vec4 grid = unpack8(iq3xxs_grid[qs] >> (8 * (iqs % 4))); bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; return db * vec2( @@ -396,7 +396,7 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) { // Add parity bit const uint sign8 = sign7 | (bitCount(sign7) << 7); const uint sign = sign8 >> (iqs % 8); - const u8vec4 grid = unpack8(iq3_grid[qs]); + const u8vec4 grid = unpack8(iq3xxs_grid[qs]); bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; bool sign2 = (sign & 4) != 0; @@ -419,7 +419,7 @@ vec2 dequantize(uint ib, uint iqs, uint a_offset) { bool sign0 = (sign & 1) != 0; bool sign1 = (sign & 2) != 0; const float db = 1 + 2 * ((scale >> (4 * ((iqs / 32) & 1))) & 0xf); - const uint32_t grid = iq3_grid[qs | ((qh << (8 - ((iqs / 4) % 8))) & 256)] >> (8 * (iqs % 4)); + const uint32_t grid = iq3s_grid[qs | ((qh << (8 - ((iqs / 4) % 8))) & 256)] >> (8 * (iqs % 4)); return db * vec2( int(grid & 0xFF) * (sign0 ? -1.0 : 1.0), int((grid >> 8) & 0xFF) * (sign1 ? -1.0 : 1.0) @@ -437,7 +437,7 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) { bool sign2 = (sign & 4) != 0; bool sign3 = (sign & 8) != 0; const float db = 1 + 2 * ((scale >> (4 * (ib32 & 1))) & 0xf); - const uint32_t grid = iq3_grid[qs | ((qh << (8 - ib4 % 8)) & 256)] >> (8 * (iqs % 4)); + const uint32_t grid = iq3s_grid[qs | ((qh << (8 - ib4 % 8)) & 256)] >> (8 * (iqs % 4)); return db * vec4( int(grid & 0xFF) * (sign0 ? -1.0 : 1.0), int((grid >> 8) & 0xFF) * (sign1 ? -1.0 : 1.0), diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl index d32a30c86f..4d4bb35379 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl @@ -936,7 +936,7 @@ float16_t dequantFuncIQ2_XXS(const in decodeBufIQ2_XXS bl, const in uint blockCo uint sign = bitfieldExtract(signscale, 7 * int(ib8), 7); sign |= bitCount(sign) << 7; - uint g2 = iq2_grid[qs][(idx & 4) >> 2]; + uint g2 = iq2xxs_grid[qs][(idx & 4) >> 2]; g2 >>= (idx & 2) * 8; const vec2 g = vec2(unpack8(g2)); @@ -961,7 +961,7 @@ f16vec4 dequantFuncIQ2_XXS_v(const in decodeBufIQ2_XXS bl, const in uint blockCo sign |= bitCount(sign) << 7; const uint sb = sign >> (idx & 7u); - const uint g2 = iq2_grid[qs][(idx & 4) >> 2]; + const uint g2 = iq2xxs_grid[qs][(idx & 4) >> 2]; const u8vec4 g = unpack8(g2); return f16vec4( @@ -991,7 +991,7 @@ float16_t dequantFuncIQ2_XS(const in decodeBufIQ2_XS bl, const in uint blockCoor uint sign = uint(qs >> 9); sign |= bitCount(sign) << 7; - uint g2 = iq2_grid[qs & 0x1FF][(idx & 4) >> 2]; + uint g2 = iq2xs_grid[qs & 0x1FF][(idx & 4) >> 2]; g2 >>= (idx & 2) * 8; const vec2 g = vec2(unpack8(g2)); @@ -1014,7 +1014,7 @@ f16vec4 dequantFuncIQ2_XS_v(const in decodeBufIQ2_XS bl, const in uint blockCoor sign |= bitCount(sign) << 7; const uint sb = sign >> (idx & 7u); - const uint g2 = iq2_grid[qs & 0x1FF][(idx & 4) >> 2]; + const uint g2 = iq2xs_grid[qs & 0x1FF][(idx & 4) >> 2]; const u8vec4 g = unpack8(g2); return f16vec4( @@ -1046,7 +1046,7 @@ float16_t dequantFuncIQ2_S(const in decodeBufIQ2_S bl, const in uint blockCoords const float d = float(bl.block.d); const float db = d * 0.25 * (0.5 + scale); const ivec2 sign01 = 1 - (2 & ivec2(sign << 1, sign)); - uint g2 = iq2_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(idx & 4) >> 2]; + uint g2 = iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(idx & 4) >> 2]; g2 >>= (idx & 2) * 8; const vec2 v = db * vec2(sign01) * vec2(unpack8(g2)); return float16_t(v[idx & 1]); @@ -1068,7 +1068,7 @@ f16vec4 dequantFuncIQ2_S_v(const in decodeBufIQ2_S bl, const in uint blockCoords const float d = float(bl.block.d); const float db = d * 0.25 * (0.5 + scale); - const uint g2 = iq2_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(idx & 4) >> 2]; + const uint g2 = iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)][(idx & 4) >> 2]; const u8vec4 g = unpack8(g2); return f16vec4( @@ -1106,7 +1106,7 @@ float16_t dequantFuncIQ3_XXS(const in decodeBufIQ3_XXS bl, const in uint blockCo const uint32_t sign7 = bitfieldExtract(signs, 7 * (int(iqs / 2) % 4), 7); const uint sign = (sign7 | (bitCount(sign7) << 7)) >> (idx & 0x6); const ivec2 sign01 = ivec2(1 - (2 & ivec2(sign << 1, sign))); - const uint grid = iq3_grid[qs] >> (16 * ((idx & 2) >> 1)); + const uint grid = iq3xxs_grid[qs] >> (16 * ((idx & 2) >> 1)); const vec2 v = db * vec2(sign01) * vec2(unpack8(grid).xy); return float16_t(v[idx & 1]); } @@ -1127,7 +1127,7 @@ f16vec4 dequantFuncIQ3_XXS_v(const in decodeBufIQ3_XXS bl, const in uint blockCo const uint sign7 = bitfieldExtract(signs, 7 * (int(iqs / 2) % 4), 7); const uint sb = (sign7 | (bitCount(sign7) << 7)) >> (idx & 0x6u); - const uint grid = iq3_grid[qs]; + const uint grid = iq3xxs_grid[qs]; const u8vec4 g = unpack8(grid); return f16vec4( @@ -1157,7 +1157,7 @@ float16_t dequantFuncIQ3_S(const in decodeBufIQ3_S bl, const in uint blockCoords const uint scale = bl.block.scales[iqs / 16]; const ivec2 sign01 = ivec2(1 - (2 & ivec2(sign << 1, sign))); const float db = d * (1 + 2 * ((scale >> (4 * (iqh & 1))) & 0xf)); - const uint32_t grid = iq3_grid[qs | ((qh << (8 - (iqs % 8))) & 256)] >> ((idx & 2) << 3); + const uint32_t grid = iq3s_grid[qs | ((qh << (8 - (iqs % 8))) & 256)] >> ((idx & 2) << 3); const vec2 v = db * vec2(sign01) * vec2(unpack8(grid).xy); return float16_t(v[idx & 1]); @@ -1177,7 +1177,7 @@ f16vec4 dequantFuncIQ3_S_v(const in decodeBufIQ3_S bl, const in uint blockCoords const uint scale = bl.block.scales[iqs / 16]; const float db = d * (1 + 2 * ((scale >> (4 * (iqh & 1))) & 0xf)); - const uint grid = iq3_grid[qs | ((qh << (8 - (iqs % 8))) & 256)]; + const uint grid = iq3s_grid[qs | ((qh << (8 - (iqs % 8))) & 256)]; const u8vec4 g = unpack8(grid); return f16vec4( diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp index f834b78562..78490162cd 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_s.comp @@ -29,7 +29,7 @@ void main() { uint qs = data_a[ib].qs[4 * ib32 + l]; const uint8_t sign = data_a[ib].qs[QUANT_K / 8 + 4 * ib32 + l]; qs |= (qh << (8 - 2 * l)) & 0x300; - const uvec2 grid = iq2_grid[qs]; + const uvec2 grid = iq2s_grid[qs]; const u8vec4 grid0 = unpack8(grid.x); const u8vec4 grid1 = unpack8(grid.y); data_b[b_idx + 8 * l + 0] = D_TYPE(db[l/2] * grid0.x * ((sign & 1) != 0 ? -1.0 : 1.0)); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp index f5bba538f3..9b8ce0a7f8 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xs.comp @@ -28,7 +28,7 @@ void main() { uint16_t qs = data_a[ib].qs[4 * ib32 + l]; const uint sign7 = qs >> 9; const uint sign8 = sign7 | (bitCount(sign7) << 7); // parity bit - const uvec2 grid = iq2_grid[qs & 511]; + const uvec2 grid = iq2xs_grid[qs & 511]; const u8vec4 grid0 = unpack8(grid.x); const u8vec4 grid1 = unpack8(grid.y); data_b[b_idx + 8 * l + 0] = D_TYPE(db[l/2] * grid0.x * ((sign8 & 1) != 0 ? -1.0 : 1.0)); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp index dca5b84c6a..aacf07d0f8 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq2_xxs.comp @@ -34,7 +34,7 @@ void main() { const uint sign7 = bitfieldExtract(signscale, 7 * int(l), 7); const uint sign8 = sign7 | (bitCount(sign7) << 7); // parity bit const uint qs = data_a[ib].qs[8 * is + l]; - const uvec2 grid = iq2_grid[qs]; + const uvec2 grid = iq2xxs_grid[qs]; const u8vec4 grid0 = unpack8(grid.x); const u8vec4 grid1 = unpack8(grid.y); data_b[b_idx + 8 * l + 0] = D_TYPE(db * grid0.x * ((sign8 & 1) != 0 ? -1.0 : 1.0)); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp index b18a9d0cd1..f2c20b1d2c 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_s.comp @@ -31,7 +31,7 @@ void main() { const uint qs = data_a[ib].qs[iqs]; const uint gidx = qs | ((qh << (8 - l)) & 256); const uint8_t signs = data_a[ib].signs[iqs / 2] >> (4 * (l & 1)); - const u8vec4 grid = unpack8(iq3_grid[gidx]); + const u8vec4 grid = unpack8(iq3s_grid[gidx]); data_b[b_idx + 4 * l + 0] = D_TYPE(db * grid.x * ((signs & 1) != 0 ? -1.0 : 1.0)); data_b[b_idx + 4 * l + 1] = D_TYPE(db * grid.y * ((signs & 2) != 0 ? -1.0 : 1.0)); data_b[b_idx + 4 * l + 2] = D_TYPE(db * grid.z * ((signs & 4) != 0 ? -1.0 : 1.0)); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp index 60c5318e3b..671c1f4a0d 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_iq3_xxs.comp @@ -37,8 +37,8 @@ void main() { const uint sign8 = sign7 | (bitCount(sign7) << 7); const uint qs0 = data_a[ib].qs[8 * is + 2 * l]; const uint qs1 = data_a[ib].qs[8 * is + 2 * l + 1]; - const u8vec4 grid0 = unpack8(iq3_grid[qs0]); - const u8vec4 grid1 = unpack8(iq3_grid[qs1]); + const u8vec4 grid0 = unpack8(iq3xxs_grid[qs0]); + const u8vec4 grid1 = unpack8(iq3xxs_grid[qs1]); data_b[b_idx + 8 * l + 0] = D_TYPE(db * grid0.x * ((sign8 & 1) != 0 ? -1.0 : 1.0)); data_b[b_idx + 8 * l + 1] = D_TYPE(db * grid0.y * ((sign8 & 2) != 0 ? -1.0 : 1.0)); data_b[b_idx + 8 * l + 2] = D_TYPE(db * grid0.z * ((sign8 & 4) != 0 ? -1.0 : 1.0)); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp index 27c461d60d..b4f6d1d6b6 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp @@ -24,7 +24,7 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, [[unroll]] for (uint l = 0; l < 2; ++l) { const uint8_t sign = sign16[l]; const uint qs = qs16[l] | ((qh << (8 - nibble_shift - 2 * l)) & 0x300); - const uvec2 grid = iq2_grid[qs]; + const uvec2 grid = iq2s_grid[qs]; const vec4 grid0 = vec4(unpack8(grid.x)); const vec4 grid1 = vec4(unpack8(grid.y)); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp index 1a202e1297..d8dafe5f70 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp @@ -31,7 +31,7 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint qs = data_a[ibi].qs[2 * itid + l]; sign_vals[l] = qs >> 9; sign7_vals[l] = bitCount(sign_vals[l]); - const uvec2 grid_data = iq2_grid[qs & 511]; + const uvec2 grid_data = iq2xs_grid[qs & 511]; grid0_vals[l] = vec4(unpack8(grid_data.x)); grid1_vals[l] = vec4(unpack8(grid_data.y)); } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp index f55ee10e8a..f75dcf8331 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp @@ -22,8 +22,8 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint qs = data_a[ibi].qs[8 * ib32 + 2 * (itid & 1) + l]; const uint sign = bitfieldExtract(signscale, 7 * int(2 * (itid & 1) + l), 7); const uint sign7 = bitCount(sign); - const vec4 grid0 = vec4(unpack8(iq2_grid[qs].x)); - const vec4 grid1 = vec4(unpack8(iq2_grid[qs].y)); + const vec4 grid0 = vec4(unpack8(iq2xxs_grid[qs].x)); + const vec4 grid1 = vec4(unpack8(iq2xxs_grid[qs].y)); [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { const vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp index e41538997f..42f52b4a12 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp @@ -27,8 +27,8 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint l = l0 + ll; const u8vec2 qs = unpack8(uint32_t(data_a_packed16[ibi].qs[4 * ib32 + l])).xy; // vec4 used due to #12147 const uint sign = data_a[ibi].signs[4 * ib32 + l]; - const vec4 grid0 = vec4(unpack8(iq3_grid[qs.x | ((qh << (8 - 2*l)) & 0x100)])); - const vec4 grid1 = vec4(unpack8(iq3_grid[qs.y | ((qh << (7 - 2*l)) & 0x100)])); + const vec4 grid0 = vec4(unpack8(iq3s_grid[qs.x | ((qh << (8 - 2*l)) & 0x100)])); + const vec4 grid1 = vec4(unpack8(iq3s_grid[qs.y | ((qh << (7 - 2*l)) & 0x100)])); [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { const vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp index 5f7a89bc1c..a88898109a 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp @@ -23,8 +23,8 @@ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint qs1 = data_a[ibi].qs[8 * ib32 + 4 * (itid & 1) + 2 * l + 1]; const uint sign = bitfieldExtract(signscale, 7 * int(2 * (itid & 1) + l), 7); const uint sign7 = bitCount(sign); - const vec4 grid0 = vec4(unpack8(iq3_grid[qs0])); - const vec4 grid1 = vec4(unpack8(iq3_grid[qs1])); + const vec4 grid0 = vec4(unpack8(iq3xxs_grid[qs0])); + const vec4 grid1 = vec4(unpack8(iq3xxs_grid[qs1])); [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { const vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl index 2c5d69c1fd..2d829741fc 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl @@ -131,7 +131,7 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + (signs >> 28))); const uint32_t sign7 = bitfieldExtract(signs, 7 * int(ib8), 7); const uint sign = sign7 | (bitCount(sign7) << 7); - const uvec2 grid = iq2_grid[qs]; + const uvec2 grid = iq2xxs_grid[qs]; const vec4 grid0 = vec4(unpack8(grid.x)); const vec4 grid1 = vec4(unpack8(grid.y)); @@ -161,7 +161,7 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin const uint qs = data_a[ib].qs[4 * ib32 + ib8]; const uint sign7 = qs >> 9; const uint sign = sign7 | (bitCount(sign7) << 7); - const uvec2 grid = iq2_grid[qs & 511]; + const uvec2 grid = iq2xs_grid[qs & 511]; const vec4 grid0 = vec4(unpack8(grid.x)); const vec4 grid1 = vec4(unpack8(grid.y)); @@ -193,7 +193,7 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin const float d = float(data_a[ib].d); const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale)); - const uvec2 grid = iq2_grid[qs | ((qh << (8 - qhshift)) & 0x300)]; + const uvec2 grid = iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)]; const vec4 grid0 = vec4(unpack8(grid.x)); const vec4 grid1 = vec4(unpack8(grid.y)); @@ -226,7 +226,7 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin const float db = d * 0.5 * (0.5 + (signs >> 28)); const uint32_t sign7 = bitfieldExtract(signs, 7 * (int(iqs / 2) % 4), 7); const uint sign = (sign7 | (bitCount(sign7) << 7)) >> (4 * (idx % 2)); - const uint grid = iq3_grid[qs]; + const uint grid = iq3xxs_grid[qs]; const vec4 v = db * vec4(unpack8(grid)); store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x, @@ -250,7 +250,7 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin const uint scale = data_a[ib].scales[iqs / 16]; const i8vec2 sign01 = i8vec2(1 - (2 & i8vec2(sign << 1, sign))); const float db = d * (1 + 2 * ((scale >> (4 * (iqh & 1))) & 0xf)); - const uint32_t grid = iq3_grid[qs | ((qh << (8 - (iqs % 8))) & 256)]; + const uint32_t grid = iq3s_grid[qs | ((qh << (8 - (iqs % 8))) & 256)]; const vec4 v = db * vec4(unpack8(grid)); store_a(col, k_pair, FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x, diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl index 0b9ac710b5..2ea1c6fb1a 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl @@ -948,11 +948,11 @@ void init_iq_shmem(uvec3 wgsize) #if defined(DATA_A_IQ2_XXS) || defined(DATA_A_IQ2_XS) || defined(DATA_A_IQ2_S) #if defined(DATA_A_IQ2_S) -shared uvec2 iq2_grid[1024]; +shared uvec2 iq2s_grid[1024]; #elif defined(DATA_A_IQ2_XS) -shared uvec2 iq2_grid[512]; +shared uvec2 iq2xs_grid[512]; #else -shared uvec2 iq2_grid[256]; +shared uvec2 iq2xxs_grid[256]; #endif #endif @@ -1045,9 +1045,9 @@ const uvec2[256] iq2xxs_grid_const = { void init_iq_shmem(uvec3 wgsize) { // copy the table into shared memory and sync - [[unroll]] for (uint i = 0; i < iq2_grid.length(); i += wgsize.x) { + [[unroll]] for (uint i = 0; i < iq2xxs_grid.length(); i += wgsize.x) { if (iq2xxs_grid_const.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2xxs_grid_const.length()) { - iq2_grid[i + gl_LocalInvocationIndex.x] = iq2xxs_grid_const[i + gl_LocalInvocationIndex.x]; + iq2xxs_grid[i + gl_LocalInvocationIndex.x] = iq2xxs_grid_const[i + gl_LocalInvocationIndex.x]; } } barrier(); @@ -1217,9 +1217,9 @@ const uvec2 iq2xs_grid_const[512] = { void init_iq_shmem(uvec3 wgsize) { // copy the table into shared memory and sync - [[unroll]] for (uint i = 0; i < iq2_grid.length(); i += wgsize.x) { - if (iq2_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2xs_grid_const.length()) { - iq2_grid[i + gl_LocalInvocationIndex.x] = iq2xs_grid_const[i + gl_LocalInvocationIndex.x]; + [[unroll]] for (uint i = 0; i < iq2xs_grid.length(); i += wgsize.x) { + if (iq2xs_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2xs_grid_const.length()) { + iq2xs_grid[i + gl_LocalInvocationIndex.x] = iq2xs_grid_const[i + gl_LocalInvocationIndex.x]; } } barrier(); @@ -1519,9 +1519,9 @@ const uvec2 iq2s_grid_const[1024] = { void init_iq_shmem(uvec3 wgsize) { // copy the table into shared memory and sync - [[unroll]] for (uint i = 0; i < iq2_grid.length(); i += wgsize.x) { - if (iq2_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2s_grid_const.length()) { - iq2_grid[i + gl_LocalInvocationIndex.x] = iq2s_grid_const[i + gl_LocalInvocationIndex.x]; + [[unroll]] for (uint i = 0; i < iq2s_grid.length(); i += wgsize.x) { + if (iq2s_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq2s_grid_const.length()) { + iq2s_grid[i + gl_LocalInvocationIndex.x] = iq2s_grid_const[i + gl_LocalInvocationIndex.x]; } } barrier(); @@ -1538,9 +1538,9 @@ void init_iq_shmem(uvec3 wgsize) #if defined(DATA_A_IQ3_XXS) || defined(DATA_A_IQ3_S) #if defined(DATA_A_IQ3_S) -shared uint32_t iq3_grid[512]; +shared uint32_t iq3s_grid[512]; #else -shared uint32_t iq3_grid[256]; +shared uint32_t iq3xxs_grid[256]; #endif #endif @@ -1601,9 +1601,9 @@ const uint32_t iq3xxs_grid_const[256] = { void init_iq_shmem(uvec3 wgsize) { // copy the table into shared memory and sync - [[unroll]] for (uint i = 0; i < iq3_grid.length(); i += wgsize.x) { - if (iq3_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq3_grid.length()) { - iq3_grid[i + gl_LocalInvocationIndex.x] = iq3xxs_grid_const[i + gl_LocalInvocationIndex.x]; + [[unroll]] for (uint i = 0; i < iq3xxs_grid.length(); i += wgsize.x) { + if (iq3xxs_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq3xxs_grid.length()) { + iq3xxs_grid[i + gl_LocalInvocationIndex.x] = iq3xxs_grid_const[i + gl_LocalInvocationIndex.x]; } } barrier(); @@ -1713,9 +1713,9 @@ const uint32_t iq3s_grid_const[512] = { void init_iq_shmem(uvec3 wgsize) { // copy the table into shared memory and sync - [[unroll]] for (uint i = 0; i < iq3_grid.length(); i += wgsize.x) { - if (iq3_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq3_grid.length()) { - iq3_grid[i + gl_LocalInvocationIndex.x] = iq3s_grid_const[i + gl_LocalInvocationIndex.x]; + [[unroll]] for (uint i = 0; i < iq3s_grid.length(); i += wgsize.x) { + if (iq3s_grid.length() % wgsize.x == 0 || i + gl_LocalInvocationIndex.x < iq3s_grid.length()) { + iq3s_grid[i + gl_LocalInvocationIndex.x] = iq3s_grid_const[i + gl_LocalInvocationIndex.x]; } } barrier();