From 39b9c2d5a0c0fb30e2f395ef261ea70e89191903 Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Wed, 29 Jul 2026 01:32:01 +0200 Subject: [PATCH] speaker encoder loading ok --- conversion/qwen3tts.py | 12 +++++++- tools/mtmd/clip-impl.h | 9 ++++++ tools/mtmd/clip-model.h | 17 +++++++++++ tools/mtmd/clip.cpp | 61 ++++++++++++++++++++++++++++++++++++- tools/mtmd/mtmd-audio.cpp | 64 +++++++++++++++++++++++++++++++++++++++ tools/mtmd/mtmd-audio.h | 9 ++++++ tools/mtmd/mtmd.cpp | 4 +++ 7 files changed, 174 insertions(+), 2 deletions(-) diff --git a/conversion/qwen3tts.py b/conversion/qwen3tts.py index de52f01e1a..d73383eb75 100644 --- a/conversion/qwen3tts.py +++ b/conversion/qwen3tts.py @@ -51,7 +51,8 @@ class Qwen3TTSTalkerModel(TextModel): def set_gguf_parameters(self): super().set_gguf_parameters() - logger.warning("Qwen3-TTS: only the talker backbone is converted; code_predictor and speaker_encoder are skipped") + # TODO: figure out the template + self.gguf_writer.add_chat_template("{% for m in messages %}{{m['content']}}{% endfor %}") @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: @@ -114,6 +115,15 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel): self.gguf_writer.add_clip_has_audio_encoder(True) self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.QWEN3TTS_SPKENC) self.gguf_writer.add_audio_projection_dim(self.n_embd_text) + # mel_spectrogram() front-end: sr=24000, n_fft=1024, hop=256, n_mels=128, fmin=0, fmax=12000 (=sr/2, the clip.cpp default) + self.gguf_writer.add_audio_num_mel_bins(128) + # the 3 SE-Res2Net stages (blocks 1-3); the stem conv, mfa, asp and fc are singletons, not part of this count + self.gguf_writer.add_audio_block_count(3) + # ECAPA-TDNN has no attention/FFN, these are dummy to allow clip.cpp to load it + self.gguf_writer.add_audio_embedding_length(1536) + self.gguf_writer.add_audio_head_count(1) + self.gguf_writer.add_audio_feed_forward_length(1536) + self.gguf_writer.add_audio_attention_layernorm_eps(1e-5) @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 0920411380..3981ff5bb6 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -199,6 +199,13 @@ #define TN_MM_A_LOCAL_LN2 "mm.a.local_blk.%d.ln2.%s" #define TN_MM_A_LOCAL_NORM "mm.a.local_norm.%s" +// qwen3tts speaker encoder (ECAPA-TDNN) +#define TN_A_SE_CONV1 "a.blk.%d.se_conv1.%s" +#define TN_A_SE_CONV2 "a.blk.%d.se_conv2.%s" +#define TN_A_CONV_RES2 "a.blk.%d.res2.%d.%s" +#define TN_A_ASP_ATTN "a.asp_attn.%s" +#define TN_A_ASP_TDNN "a.asp_tdnn.%s" + // cogvlm #define TN_MM_POST_FC_NORM "mm.post_fc_norm.%s" #define TN_MM_H_TO_4H "mm.up.%s" @@ -399,6 +406,7 @@ enum projector_type { PROJECTOR_TYPE_MINIMAX_M3, PROJECTOR_TYPE_GRANITE4_VISION, PROJECTOR_TYPE_MIMO_AUDIO, + PROJECTOR_TYPE_QWEN3TTS_SPKENC, PROJECTOR_TYPE_UNKNOWN, }; @@ -455,6 +463,7 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_MINIMAX_M3, "minimax_m3"}, { PROJECTOR_TYPE_GRANITE4_VISION, "granite4_vision"}, { PROJECTOR_TYPE_MIMO_AUDIO, "mimo_audio"}, + { PROJECTOR_TYPE_QWEN3TTS_SPKENC, "qwen3tts_spkenc"}, }; static projector_type clip_projector_type_from_string(const std::string & str) { diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index 8dc8754976..472487351c 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -276,6 +276,15 @@ struct clip_layer { ggml_tensor * cross_attn_norm_w = nullptr; ggml_tensor * cross_attn_norm_b = nullptr; + // qwen3tts speaker encoder: SE-Res2Net block (conv_pw1_w/b and conv_pw2_w/b + // above are reused for this block's tdnn1/tdnn2 bottleneck convs) + ggml_tensor * se_conv1_w = nullptr; + ggml_tensor * se_conv1_b = nullptr; + ggml_tensor * se_conv2_w = nullptr; + ggml_tensor * se_conv2_b = nullptr; + std::vector res2_conv_w; // Res2Net hierarchical branches + std::vector res2_conv_b; + bool has_deepstack() const { return deepstack_fc1_w != nullptr; } @@ -567,6 +576,14 @@ struct clip_model { ggml_tensor * conv2d_3_w = nullptr; ggml_tensor * conv2d_3_b = nullptr; + // qwen3tts speaker encoder (ECAPA-TDNN): the stem conv (block 0) reuses + // conv1d_1_w/b, the multi-layer feature aggregation conv reuses conv_out_w/b, + // and the final speaker embedding projection reuses mm_fc_w/b + ggml_tensor * spk_asp_attn_w = nullptr; + ggml_tensor * spk_asp_attn_b = nullptr; + ggml_tensor * spk_asp_tdnn_w = nullptr; + ggml_tensor * spk_asp_tdnn_b = nullptr; + // cogvlm ggml_tensor * mm_post_fc_norm_w = nullptr; ggml_tensor * mm_post_fc_norm_b = nullptr; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 04614b93bd..0f7bd1596c 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1627,6 +1627,15 @@ struct clip_model_loader { "%s: mimo_audio: %s must be > 0\n", __func__, KEY_A_LOCAL_GROUP_SIZE)); } } break; + case PROJECTOR_TYPE_QWEN3TTS_SPKENC: + { + // ECAPA-TDNN speaker/voice encoder; mel_spectrogram() front-end + // matches the Slaney mel default (fmin=0, fmax=sample_rate/2) + hparams.audio_sample_rate = 24000; + hparams.audio_n_fft = 1024; + hparams.audio_window_len = 1024; + hparams.audio_hop_len = 256; + } break; case PROJECTOR_TYPE_PADDLEOCR: { hparams.n_merge = 2; @@ -1923,7 +1932,8 @@ struct clip_model_loader { model.position_embeddings = get_tensor(string_format(TN_POS_EMBD, prefix), false); const bool has_standard_layers = ( - model.proj_type != PROJECTOR_TYPE_GEMMA3NV); + model.proj_type != PROJECTOR_TYPE_GEMMA3NV && + model.proj_type != PROJECTOR_TYPE_QWEN3TTS_SPKENC); // layers const int n_layers_to_load = has_standard_layers ? hparams.n_layer : 0; @@ -2544,6 +2554,47 @@ struct clip_model_loader { model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight")); model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight")); } break; + case PROJECTOR_TYPE_QWEN3TTS_SPKENC: + { + // stem TDNN (block 0) + model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 0, "weight")); + model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 0, "bias")); + + // SE-Res2Net blocks (GGUF bid 1..3, one per hparams.n_layer) + model.layers.resize(hparams.n_layer); + for (int il = 0; il < hparams.n_layer; il++) { + auto & layer = model.layers[il]; + int bid = il + 1; + layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, bid, "weight")); + layer.conv_pw1_b = get_tensor(string_format(TN_CONV_PW1, prefix, bid, "bias")); + layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, bid, "weight")); + layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, bid, "bias")); + layer.se_conv1_w = get_tensor(string_format(TN_A_SE_CONV1, bid, "weight")); + layer.se_conv1_b = get_tensor(string_format(TN_A_SE_CONV1, bid, "bias")); + layer.se_conv2_w = get_tensor(string_format(TN_A_SE_CONV2, bid, "weight")); + layer.se_conv2_b = get_tensor(string_format(TN_A_SE_CONV2, bid, "bias")); + layer.res2_conv_w.resize(7); + layer.res2_conv_b.resize(7); + for (int xid = 0; xid < 7; xid++) { + layer.res2_conv_w[xid] = get_tensor(string_format(TN_A_CONV_RES2, bid, xid, "weight")); + layer.res2_conv_b[xid] = get_tensor(string_format(TN_A_CONV_RES2, bid, xid, "bias")); + } + } + + // multi-layer feature aggregation + model.conv_out_w = get_tensor(string_format(TN_CONV_OUT, "weight")); + model.conv_out_b = get_tensor(string_format(TN_CONV_OUT, "bias")); + + // attentive statistics pooling + model.spk_asp_attn_w = get_tensor(string_format(TN_A_ASP_ATTN, "weight")); + model.spk_asp_attn_b = get_tensor(string_format(TN_A_ASP_ATTN, "bias")); + model.spk_asp_tdnn_w = get_tensor(string_format(TN_A_ASP_TDNN, "weight")); + model.spk_asp_tdnn_b = get_tensor(string_format(TN_A_ASP_TDNN, "bias")); + + // final speaker embedding projection + model.mm_fc_w = get_tensor(string_format(TN_MM_AUDIO_FC, "weight")); + model.mm_fc_b = get_tensor(string_format(TN_MM_AUDIO_FC, "bias")); + } break; case PROJECTOR_TYPE_VOXTRAL: { model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight")); @@ -3664,6 +3715,12 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { const int ds = ctx->model.hparams.audio_proj_downsample_rate; n_patches = ((img->nx() + ws - 1) / ws) * (ws / ds); } break; + case PROJECTOR_TYPE_QWEN3TTS_SPKENC: + { + // attentive statistics pooling collapses the whole clip into + // a single speaker embedding vector, regardless of its length + n_patches = 1; + } break; case PROJECTOR_TYPE_GRANITE4_VISION: { // Per-tile output token count: each projector block outputs @@ -4841,6 +4898,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { return ctx->model.mm_ffn_down_w->ne[1]; case PROJECTOR_TYPE_MIMO_AUDIO: return ctx->model.mm_2_w->ne[1]; + case PROJECTOR_TYPE_QWEN3TTS_SPKENC: + return ctx->model.mm_fc_w->ne[1]; default: GGML_ABORT("Unknown projector type"); } diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp index ed68951c01..ed48329fca 100644 --- a/tools/mtmd/mtmd-audio.cpp +++ b/tools/mtmd/mtmd-audio.cpp @@ -791,6 +791,70 @@ bool mtmd_audio_preprocessor_mimo_audio::preprocess(const float * return true; } +// +// mtmd_audio_preprocessor_qwen3tts_spk +// +// Mirrors qwen_tts.core.models.modeling_qwen3_tts.mel_spectrogram(): +// pad reflect by (n_fft - hop) / 2, STFT (n_fft, hop, win=n_fft, hann +// periodic, center=False), mel = slaney_mel_basis @ |STFT|, log(max(mel, 1e-5)). +// Unlike Whisper-style encoders the whole clip is consumed in a single +// forward pass by the ECAPA-TDNN body, so there's no 30s/3000-frame chunking +// or Whisper (max-8)/4 normalization here. +// + +void mtmd_audio_preprocessor_qwen3tts_spk::initialize() { + cache.fill_sin_cos_table(hparams.audio_n_fft); + cache.fill_hann_window(hparams.audio_window_len, true); + cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate); +} + +bool mtmd_audio_preprocessor_qwen3tts_spk::preprocess(const float * samples, + size_t n_samples, + std::vector & output) { + if (n_samples == 0) { + return false; + } + + GGML_ASSERT(!cache.sin_vals.empty()); + GGML_ASSERT(!cache.cos_vals.empty()); + GGML_ASSERT(!cache.filters.data.empty()); + + // reflect pad by (n_fft - hop) / 2 = 384, matching center=False STFT framing + const int pad = (hparams.audio_n_fft - hparams.audio_hop_len) / 2; + if ((int) n_samples < pad + 1) { + return false; + } + + std::vector padded(n_samples + 2 * pad, 0.0f); + for (int i = 0; i < pad; i++) { + padded[i] = samples[pad - i]; + } + std::copy(samples, samples + n_samples, padded.begin() + pad); + for (int i = 0; i < pad; i++) { + padded[n_samples + pad + i] = samples[n_samples - 2 - i]; + } + + filter_params params; + params.n_mel = hparams.n_mel_bins; + params.n_fft_bins = 1 + (hparams.audio_n_fft / 2); + params.hann_window_size = hparams.audio_window_len; + params.hop_length = hparams.audio_hop_len; + params.sample_rate = hparams.audio_sample_rate; + params.no_padding = true; // reflect padding already applied above + params.use_natural_log = true; + params.use_magnitude = true; + params.mel_floor = 1e-5f; + + mtmd_audio_mel out; + bool ok = log_mel_spectrogram(padded.data(), (int) padded.size(), 4, params, cache, out); + if (!ok) { + return false; + } + + output.push_back(std::move(out)); + return true; +} + // // mtmd_audio_preprocessor_conformer // diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.h index d8ec72b9d5..2011d95517 100644 --- a/tools/mtmd/mtmd-audio.h +++ b/tools/mtmd/mtmd-audio.h @@ -120,6 +120,15 @@ struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor { mtmd_audio_cache cache; }; +struct mtmd_audio_preprocessor_qwen3tts_spk : mtmd_audio_preprocessor { + mtmd_audio_preprocessor_qwen3tts_spk(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + void initialize() override; + bool preprocess(const float * samples, size_t n_samples, std::vector & output) override; + + private: + mtmd_audio_cache cache; +}; + // // streaming ISTFT - converts spectrogram frames back to audio one frame at a time // diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 6e61cf3e52..991c177b60 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -736,6 +736,10 @@ struct mtmd_context { aud_end = "<|mimo_audio_end|>"; audio_preproc = std::make_unique(ctx_a); } break; + case PROJECTOR_TYPE_QWEN3TTS_SPKENC: + { + audio_preproc = std::make_unique(ctx_a); + } break; default: throw std::runtime_error(string_format("%s: unexpected audio projector type %d\n", __func__, proj)); }