Superminaren/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-24 13:37:01 +02:00
Code Issues Packages Projects Releases Wiki Activity

speaker encoder loading ok

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-07-29 01:32:01 +02:00
parent de0ac58c1f
commit 39b9c2d5a0
7 changed files with 174 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conversion/qwen3tts.py
+11 -1
View File
@@ -51,7 +51,8 @@ class Qwen3TTSTalkerModel(TextModel):
def set_gguf_parameters(self):
super().set_gguf_parameters()
logger.warning("Qwen3-TTS: only the talker backbone is converted; code_predictor and speaker_encoder are skipped")
# TODO: figure out the template
self.gguf_writer.add_chat_template("{% for m in messages %}{{m['content']}}{% endfor %}")
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
@@ -114,6 +115,15 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
self.gguf_writer.add_clip_has_audio_encoder(True)
self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.QWEN3TTS_SPKENC)
self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
# mel_spectrogram() front-end: sr=24000, n_fft=1024, hop=256, n_mels=128, fmin=0, fmax=12000 (=sr/2, the clip.cpp default)
self.gguf_writer.add_audio_num_mel_bins(128)
# the 3 SE-Res2Net stages (blocks 1-3); the stem conv, mfa, asp and fc are singletons, not part of this count
self.gguf_writer.add_audio_block_count(3)
# ECAPA-TDNN has no attention/FFN, these are dummy to allow clip.cpp to load it
self.gguf_writer.add_audio_embedding_length(1536)
self.gguf_writer.add_audio_head_count(1)
self.gguf_writer.add_audio_feed_forward_length(1536)
self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
@classmethod
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
tools/mtmd/clip-impl.h
+9
View File
@@ -199,6 +199,13 @@
#define TN_MM_A_LOCAL_LN2 "mm.a.local_blk.%d.ln2.%s"
#define TN_MM_A_LOCAL_NORM "mm.a.local_norm.%s"
// qwen3tts speaker encoder (ECAPA-TDNN)
#define TN_A_SE_CONV1 "a.blk.%d.se_conv1.%s"
#define TN_A_SE_CONV2 "a.blk.%d.se_conv2.%s"
#define TN_A_CONV_RES2 "a.blk.%d.res2.%d.%s"
#define TN_A_ASP_ATTN "a.asp_attn.%s"
#define TN_A_ASP_TDNN "a.asp_tdnn.%s"
// cogvlm
#define TN_MM_POST_FC_NORM "mm.post_fc_norm.%s"
#define TN_MM_H_TO_4H "mm.up.%s"
@@ -399,6 +406,7 @@ enum projector_type {
PROJECTOR_TYPE_MINIMAX_M3,
PROJECTOR_TYPE_GRANITE4_VISION,
PROJECTOR_TYPE_MIMO_AUDIO,
PROJECTOR_TYPE_QWEN3TTS_SPKENC,
PROJECTOR_TYPE_UNKNOWN,
};
@@ -455,6 +463,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_MINIMAX_M3, "minimax_m3"},
{ PROJECTOR_TYPE_GRANITE4_VISION, "granite4_vision"},
{ PROJECTOR_TYPE_MIMO_AUDIO, "mimo_audio"},
{ PROJECTOR_TYPE_QWEN3TTS_SPKENC, "qwen3tts_spkenc"},
};
static projector_type clip_projector_type_from_string(const std::string & str) {
tools/mtmd/clip-model.h
+17
View File
@@ -276,6 +276,15 @@ struct clip_layer {
ggml_tensor * cross_attn_norm_w = nullptr;
ggml_tensor * cross_attn_norm_b = nullptr;
// qwen3tts speaker encoder: SE-Res2Net block (conv_pw1_w/b and conv_pw2_w/b
// above are reused for this block's tdnn1/tdnn2 bottleneck convs)
ggml_tensor * se_conv1_w = nullptr;
ggml_tensor * se_conv1_b = nullptr;
ggml_tensor * se_conv2_w = nullptr;
ggml_tensor * se_conv2_b = nullptr;
std::vector<ggml_tensor *> res2_conv_w; // Res2Net hierarchical branches
std::vector<ggml_tensor *> res2_conv_b;
bool has_deepstack() const {
return deepstack_fc1_w != nullptr;
}
@@ -567,6 +576,14 @@ struct clip_model {
ggml_tensor * conv2d_3_w = nullptr;
ggml_tensor * conv2d_3_b = nullptr;
// qwen3tts speaker encoder (ECAPA-TDNN): the stem conv (block 0) reuses
// conv1d_1_w/b, the multi-layer feature aggregation conv reuses conv_out_w/b,
// and the final speaker embedding projection reuses mm_fc_w/b
ggml_tensor * spk_asp_attn_w = nullptr;
ggml_tensor * spk_asp_attn_b = nullptr;
ggml_tensor * spk_asp_tdnn_w = nullptr;
ggml_tensor * spk_asp_tdnn_b = nullptr;
// cogvlm
ggml_tensor * mm_post_fc_norm_w = nullptr;
ggml_tensor * mm_post_fc_norm_b = nullptr;
tools/mtmd/clip.cpp
+60 -1
View File
@@ -1627,6 +1627,15 @@ struct clip_model_loader {
"%s: mimo_audio: %s must be > 0\n", __func__, KEY_A_LOCAL_GROUP_SIZE));
}
} break;
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
{
// ECAPA-TDNN speaker/voice encoder; mel_spectrogram() front-end
// matches the Slaney mel default (fmin=0, fmax=sample_rate/2)
hparams.audio_sample_rate = 24000;
hparams.audio_n_fft = 1024;
hparams.audio_window_len = 1024;
hparams.audio_hop_len = 256;
} break;
case PROJECTOR_TYPE_PADDLEOCR:
{
hparams.n_merge = 2;
@@ -1923,7 +1932,8 @@ struct clip_model_loader {
model.position_embeddings = get_tensor(string_format(TN_POS_EMBD, prefix), false);
const bool has_standard_layers = (
model.proj_type != PROJECTOR_TYPE_GEMMA3NV);
model.proj_type != PROJECTOR_TYPE_GEMMA3NV &&
model.proj_type != PROJECTOR_TYPE_QWEN3TTS_SPKENC);
// layers
const int n_layers_to_load = has_standard_layers ? hparams.n_layer : 0;
@@ -2544,6 +2554,47 @@ struct clip_model_loader {
model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"));
model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight"));
} break;
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
{
// stem TDNN (block 0)
model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 0, "weight"));
model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 0, "bias"));
// SE-Res2Net blocks (GGUF bid 1..3, one per hparams.n_layer)
model.layers.resize(hparams.n_layer);
for (int il = 0; il < hparams.n_layer; il++) {
auto & layer = model.layers[il];
int bid = il + 1;
layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, bid, "weight"));
layer.conv_pw1_b = get_tensor(string_format(TN_CONV_PW1, prefix, bid, "bias"));
layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, bid, "weight"));
layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, bid, "bias"));
layer.se_conv1_w = get_tensor(string_format(TN_A_SE_CONV1, bid, "weight"));
layer.se_conv1_b = get_tensor(string_format(TN_A_SE_CONV1, bid, "bias"));
layer.se_conv2_w = get_tensor(string_format(TN_A_SE_CONV2, bid, "weight"));
layer.se_conv2_b = get_tensor(string_format(TN_A_SE_CONV2, bid, "bias"));
layer.res2_conv_w.resize(7);
layer.res2_conv_b.resize(7);
for (int xid = 0; xid < 7; xid++) {
layer.res2_conv_w[xid] = get_tensor(string_format(TN_A_CONV_RES2, bid, xid, "weight"));
layer.res2_conv_b[xid] = get_tensor(string_format(TN_A_CONV_RES2, bid, xid, "bias"));
}
}
// multi-layer feature aggregation
model.conv_out_w = get_tensor(string_format(TN_CONV_OUT, "weight"));
model.conv_out_b = get_tensor(string_format(TN_CONV_OUT, "bias"));
// attentive statistics pooling
model.spk_asp_attn_w = get_tensor(string_format(TN_A_ASP_ATTN, "weight"));
model.spk_asp_attn_b = get_tensor(string_format(TN_A_ASP_ATTN, "bias"));
model.spk_asp_tdnn_w = get_tensor(string_format(TN_A_ASP_TDNN, "weight"));
model.spk_asp_tdnn_b = get_tensor(string_format(TN_A_ASP_TDNN, "bias"));
// final speaker embedding projection
model.mm_fc_w = get_tensor(string_format(TN_MM_AUDIO_FC, "weight"));
model.mm_fc_b = get_tensor(string_format(TN_MM_AUDIO_FC, "bias"));
} break;
case PROJECTOR_TYPE_VOXTRAL:
{
model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight"));
@@ -3664,6 +3715,12 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
const int ds = ctx->model.hparams.audio_proj_downsample_rate;
n_patches = ((img->nx() + ws - 1) / ws) * (ws / ds);
} break;
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
{
// attentive statistics pooling collapses the whole clip into
// a single speaker embedding vector, regardless of its length
n_patches = 1;
} break;
case PROJECTOR_TYPE_GRANITE4_VISION:
{
// Per-tile output token count: each projector block outputs
@@ -4841,6 +4898,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
return ctx->model.mm_ffn_down_w->ne[1];
case PROJECTOR_TYPE_MIMO_AUDIO:
return ctx->model.mm_2_w->ne[1];
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
return ctx->model.mm_fc_w->ne[1];
default:
GGML_ABORT("Unknown projector type");
}
tools/mtmd/mtmd-audio.cpp
+64
View File
@@ -791,6 +791,70 @@ bool mtmd_audio_preprocessor_mimo_audio::preprocess(const float *
return true;
}
//
// mtmd_audio_preprocessor_qwen3tts_spk
//
// Mirrors qwen_tts.core.models.modeling_qwen3_tts.mel_spectrogram():
// pad reflect by (n_fft - hop) / 2, STFT (n_fft, hop, win=n_fft, hann
// periodic, center=False), mel = slaney_mel_basis @ |STFT|, log(max(mel, 1e-5)).
// Unlike Whisper-style encoders the whole clip is consumed in a single
// forward pass by the ECAPA-TDNN body, so there's no 30s/3000-frame chunking
// or Whisper (max-8)/4 normalization here.
//
void mtmd_audio_preprocessor_qwen3tts_spk::initialize() {
cache.fill_sin_cos_table(hparams.audio_n_fft);
cache.fill_hann_window(hparams.audio_window_len, true);
cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate);
}
bool mtmd_audio_preprocessor_qwen3tts_spk::preprocess(const float * samples,
size_t n_samples,
std::vector<mtmd_audio_mel> & output) {
if (n_samples == 0) {
return false;
}
GGML_ASSERT(!cache.sin_vals.empty());
GGML_ASSERT(!cache.cos_vals.empty());
GGML_ASSERT(!cache.filters.data.empty());
// reflect pad by (n_fft - hop) / 2 = 384, matching center=False STFT framing
const int pad = (hparams.audio_n_fft - hparams.audio_hop_len) / 2;
if ((int) n_samples < pad + 1) {
return false;
}
std::vector<float> padded(n_samples + 2 * pad, 0.0f);
for (int i = 0; i < pad; i++) {
padded[i] = samples[pad - i];
}
std::copy(samples, samples + n_samples, padded.begin() + pad);
for (int i = 0; i < pad; i++) {
padded[n_samples + pad + i] = samples[n_samples - 2 - i];
}
filter_params params;
params.n_mel = hparams.n_mel_bins;
params.n_fft_bins = 1 + (hparams.audio_n_fft / 2);
params.hann_window_size = hparams.audio_window_len;
params.hop_length = hparams.audio_hop_len;
params.sample_rate = hparams.audio_sample_rate;
params.no_padding = true; // reflect padding already applied above
params.use_natural_log = true;
params.use_magnitude = true;
params.mel_floor = 1e-5f;
mtmd_audio_mel out;
bool ok = log_mel_spectrogram(padded.data(), (int) padded.size(), 4, params, cache, out);
if (!ok) {
return false;
}
output.push_back(std::move(out));
return true;
}
//
// mtmd_audio_preprocessor_conformer
//
tools/mtmd/mtmd-audio.h
+9
View File
@@ -120,6 +120,15 @@ struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor {
mtmd_audio_cache cache;
};
struct mtmd_audio_preprocessor_qwen3tts_spk : mtmd_audio_preprocessor {
mtmd_audio_preprocessor_qwen3tts_spk(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
void initialize() override;
bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) override;
private:
mtmd_audio_cache cache;
};
//
// streaming ISTFT - converts spectrogram frames back to audio one frame at a time
//
tools/mtmd/mtmd.cpp
+4
View File
@@ -736,6 +736,10 @@ struct mtmd_context {
aud_end = "<|mimo_audio_end|>";
audio_preproc = std::make_unique<mtmd_audio_preprocessor_mimo_audio>(ctx_a);
} break;
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
{
audio_preproc = std::make_unique<mtmd_audio_preprocessor_qwen3tts_spk>(ctx_a);
} break;
default:
throw std::runtime_error(string_format("%s: unexpected audio projector type %d\n", __func__, proj));
}