From d827cdedcf141fb003512f0e359f0fbdb124a3ee Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Mon, 3 Aug 2026 00:13:25 +0200 Subject: [PATCH] rename CODE2WAV --> GEN_WAV, update docs --- conversion/qwen3tts.py | 9 ++++++++- tools/mtmd/README-dev.md | 3 +++ tools/mtmd/mtmd-helper-gen.cpp | 12 ++++++------ tools/mtmd/mtmd.cpp | 7 ++++--- tools/mtmd/mtmd.h | 7 ++++--- 5 files changed, 25 insertions(+), 13 deletions(-) diff --git a/conversion/qwen3tts.py b/conversion/qwen3tts.py index 3b46db11fb..ae977148f2 100644 --- a/conversion/qwen3tts.py +++ b/conversion/qwen3tts.py @@ -22,6 +22,12 @@ from .base import ModelBase, MmprojModel, TextModel, gguf # - output tensor codec_head is smaller than vocab, so logits will be padded at inference time # - suppress_tokens is used to limit the backbone to only sample either semantic or EOS (stop) token +# pipeline stage mapping: +# speaker reference encoder --> mapped to normal mtmd audio encoder +# backbone --> mapped to normal libllama text model (autoregressive) +# code_predictor --> MTMD_GEN_PROCESS_TYPE_GEN_CODE +# code2wav --> MTMD_GEN_PROCESS_TYPE_GEN_WAV + # torch activation functions used by Qwen3TTSTalkerResizeMLP (config's hidden_act) _ACT2FN = { "silu": F.silu, @@ -251,7 +257,7 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel): self.gguf_writer.add_gen_audio_attention_layernorm_eps(code_predictor_config["rms_norm_eps"]) # note: code2wav hparams are hardcoded on the mtmd/clip.cpp side for now, not written here - def _wav_decoder_config(self) -> dict[str, Any]: + def _wav_decoder_config(self) -> dict[str, Any] | None: # code2wav (RVQ codes -> raw PCM) lives in its own checkpoint dir, sibling to # the main safetensors, with its own config.json if self._wav_config_cache is None: @@ -415,6 +421,7 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel): "mlp.down_proj.weight": T.A_GEN_WAV_TFM_FFN_DOWN, "mlp_layer_scale.scale": T.A_GEN_WAV_TFM_FFN_SCALE, } + assert wav_config is not None for bid in range(wav_config["num_hidden_layers"]): for key, tensor_id in tfm_layer_map.items(): yield (self.format_tensor_name(tensor_id, bid), get(f"decoder.pre_transformer.layers.{bid}.{key}")) diff --git a/tools/mtmd/README-dev.md b/tools/mtmd/README-dev.md index eed9f242e6..3cddd085ec 100644 --- a/tools/mtmd/README-dev.md +++ b/tools/mtmd/README-dev.md @@ -67,6 +67,9 @@ Due to wide variety of audio generation pipelines, the `mtmd_gen_audio` system i - Sidecar models (code2wav, bigvgan, etc) must live inside the mmproj GGUF (but can be in different `clip_context` if necessary) - Note: it should use `ggml_build_forward_select` to select graphs if multiple graphs living in the same context - Reuse existing GGUF metadata key name and tensor name whenever possible; think twice before adding extensive changes to GGUF writer. For example, Qwen3-TTS hard-code part of the hparams to `clip.cpp` as they won't likely to change. + - For tensor naming: + - Prefixed with `a.*` for tensors used by speaker encoder pipeline + - Prefixed with `a.gen.*` for generation stages (code / mel-spectrogram / PCM generation) 3. Make sure most of the changes happen inside `mtmd-helper-gen.cpp`. A good PR looks like this: - 10-20% changes is to add new backbone (text) model and conversion - 60% changes inside `mtmd-helper-gen.cpp` diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp index 145d2ee827..b35b56cb20 100644 --- a/tools/mtmd/mtmd-helper-gen.cpp +++ b/tools/mtmd/mtmd-helper-gen.cpp @@ -234,7 +234,7 @@ public: codes_buf.insert(codes_buf.end(), out.codes, out.codes + out.n_codes); if (out.n_codes > 0 && codes_buf.size() / out.n_codes >= window_frames) { - if (!flush_c2w()) { + if (!flush_gen_wav()) { return 1; } } @@ -264,7 +264,7 @@ public: } int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override { - if (!flush_c2w()) { + if (!flush_gen_wav()) { return 1; } @@ -360,21 +360,21 @@ private: return ok; } - // runs one CODE2WAV process() call on whatever is currently buffered, carrying + // runs one GEN_WAV process() call on whatever is currently buffered, carrying // the persisted state (KV cache + conv left-context) across batches - bool flush_c2w() { + bool flush_gen_wav() { if (codes_buf.empty()) { return true; } mtmd_gen_inp inp{}; - inp.type = MTMD_GEN_PROCESS_TYPE_CODE2WAV; + inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; inp.codes = codes_buf.data(); inp.n_codes = codes_buf.size(); inp.state_data = c2w_state.empty() ? nullptr : (const char *) c2w_state.data(); inp.state_size = c2w_state.size(); mtmd_gen_out out{}; if (mtmd_gen_audio_process(mctx, &inp, &out) != 0) { - LOG_ERR("mtmd_helper_gen_audio: code2wav process failed\n"); + LOG_ERR("mtmd_helper_gen_audio: gen_wav process failed\n"); return false; } audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples); diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 361af6dfb5..39e97467a8 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -1642,9 +1642,9 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, const mtmd_gen_in return 0; } - // MTMD_GEN_PROCESS_TYPE_CODE2WAV + // MTMD_GEN_PROCESS_TYPE_GEN_WAV if (!inp->codes || inp->n_codes == 0) { - LOG_ERR("%s: codes required for code2wav\n", __func__); + LOG_ERR("%s: codes required for gen_wav\n", __func__); return 1; } std::vector in_codes(inp->codes, inp->codes + inp->n_codes); @@ -1653,7 +1653,8 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, const mtmd_gen_in in_state.assign(inp->state_data, inp->state_data + inp->state_size); } - // code2wav has no hidden-state input, the batch entry is an unused placeholder + // gen_wav has no hidden-state input, the batch entry is an unused placeholder + // TODO @ngxson : some models in the future may require hidden-state input, need to update this code later clip_image_f32 dummy; dummy.set_size({1, 1}, false, true); dummy.cpy_buf(std::vector(1, 0.0f)); diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index f7d1fc65b7..7f686ce83a 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -343,7 +343,8 @@ MTMD_API struct mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * enum mtmd_gen_process_type { MTMD_GEN_PROCESS_TYPE_GEN_CODE, // h_state to codes - MTMD_GEN_PROCESS_TYPE_CODE2WAV, // codes to raw PCM audio + MTMD_GEN_PROCESS_TYPE_GEN_WAV, // convert internal representation (codes, mel-spectrogram, etc.) to PCM audio + // for qwen3tts, this is code2wav }; struct mtmd_gen_inp { enum mtmd_gen_process_type type; @@ -354,7 +355,7 @@ struct mtmd_gen_inp { int32_t top_k; float top_p; - // for MTMD_GEN_PROCESS_TYPE_CODE2WAV + // for MTMD_GEN_PROCESS_TYPE_GEN_WAV int32_t * codes; size_t n_codes; const char * state_data; @@ -369,7 +370,7 @@ struct mtmd_gen_out { const float * embd; // the generated hidden state, to be fed back to backbone // it must have n_text_embd elements - // for MTMD_GEN_PROCESS_TYPE_CODE2WAV + // for MTMD_GEN_PROCESS_TYPE_GEN_WAV const float * audio; size_t n_samples; const char * state_data;