rename CODE2WAV --> GEN_WAV, update docs

conversion/qwen3tts.py

@@ -22,6 +22,12 @@ from .base import ModelBase, MmprojModel, TextModel, gguf
 # - output tensor codec_head is smaller than vocab, so logits will be padded at inference time
 # - suppress_tokens is used to limit the backbone to only sample either semantic or EOS (stop) token
 
+# pipeline stage mapping:
+#   speaker reference encoder --> mapped to normal mtmd audio encoder
+#   backbone --> mapped to normal libllama text model (autoregressive)
+#   code_predictor --> MTMD_GEN_PROCESS_TYPE_GEN_CODE
+#   code2wav --> MTMD_GEN_PROCESS_TYPE_GEN_WAV
+
 # torch activation functions used by Qwen3TTSTalkerResizeMLP (config's hidden_act)
 _ACT2FN = {
     "silu": F.silu,
@@ -251,7 +257,7 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
         self.gguf_writer.add_gen_audio_attention_layernorm_eps(code_predictor_config["rms_norm_eps"])
         # note: code2wav hparams are hardcoded on the mtmd/clip.cpp side for now, not written here
 
-    def _wav_decoder_config(self) -> dict[str, Any]:
+    def _wav_decoder_config(self) -> dict[str, Any] | None:
         # code2wav (RVQ codes -> raw PCM) lives in its own checkpoint dir, sibling to
         # the main safetensors, with its own config.json
         if self._wav_config_cache is None:
@@ -415,6 +421,7 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
             "mlp.down_proj.weight":           T.A_GEN_WAV_TFM_FFN_DOWN,
             "mlp_layer_scale.scale":          T.A_GEN_WAV_TFM_FFN_SCALE,
         }
+        assert wav_config is not None
         for bid in range(wav_config["num_hidden_layers"]):
             for key, tensor_id in tfm_layer_map.items():
                 yield (self.format_tensor_name(tensor_id, bid), get(f"decoder.pre_transformer.layers.{bid}.{key}"))

tools/mtmd/README-dev.md

@@ -67,6 +67,9 @@ Due to wide variety of audio generation pipelines, the `mtmd_gen_audio` system i
     - Sidecar models (code2wav, bigvgan, etc) must live inside the mmproj GGUF (but can be in different `clip_context` if necessary)
         - Note: it should use `ggml_build_forward_select` to select graphs if multiple graphs living in the same context
     - Reuse existing GGUF metadata key name and tensor name whenever possible; think twice before adding extensive changes to GGUF writer. For example, Qwen3-TTS hard-code part of the hparams to `clip.cpp` as they won't likely to change.
+    - For tensor naming:
+        - Prefixed with `a.*` for tensors used by speaker encoder pipeline
+        - Prefixed with `a.gen.*` for generation stages (code / mel-spectrogram / PCM generation)
 3. Make sure most of the changes happen inside `mtmd-helper-gen.cpp`. A good PR looks like this:
     - 10-20% changes is to add new backbone (text) model and conversion
     - 60% changes inside `mtmd-helper-gen.cpp`

tools/mtmd/mtmd-helper-gen.cpp

@@ -234,7 +234,7 @@ public:
 
         codes_buf.insert(codes_buf.end(), out.codes, out.codes + out.n_codes);
         if (out.n_codes > 0 && codes_buf.size() / out.n_codes >= window_frames) {
-            if (!flush_c2w()) {
+            if (!flush_gen_wav()) {
                 return 1;
             }
         }
@@ -264,7 +264,7 @@ public:
     }
 
     int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
-        if (!flush_c2w()) {
+        if (!flush_gen_wav()) {
             return 1;
         }
 
@@ -360,21 +360,21 @@ private:
         return ok;
     }
 
-    // runs one CODE2WAV process() call on whatever is currently buffered, carrying
+    // runs one GEN_WAV process() call on whatever is currently buffered, carrying
     // the persisted state (KV cache + conv left-context) across batches
-    bool flush_c2w() {
+    bool flush_gen_wav() {
         if (codes_buf.empty()) {
             return true;
         }
         mtmd_gen_inp inp{};
-        inp.type       = MTMD_GEN_PROCESS_TYPE_CODE2WAV;
+        inp.type       = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
         inp.codes      = codes_buf.data();
         inp.n_codes    = codes_buf.size();
         inp.state_data = c2w_state.empty() ? nullptr : (const char *) c2w_state.data();
         inp.state_size = c2w_state.size();
         mtmd_gen_out out{};
         if (mtmd_gen_audio_process(mctx, &inp, &out) != 0) {
-            LOG_ERR("mtmd_helper_gen_audio: code2wav process failed\n");
+            LOG_ERR("mtmd_helper_gen_audio: gen_wav process failed\n");
             return false;
         }
         audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples);

tools/mtmd/mtmd.cpp

@@ -1642,9 +1642,9 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, const mtmd_gen_in
         return 0;
     }
 
-    // MTMD_GEN_PROCESS_TYPE_CODE2WAV
+    // MTMD_GEN_PROCESS_TYPE_GEN_WAV
     if (!inp->codes || inp->n_codes == 0) {
-        LOG_ERR("%s: codes required for code2wav\n", __func__);
+        LOG_ERR("%s: codes required for gen_wav\n", __func__);
         return 1;
     }
     std::vector<int32_t> in_codes(inp->codes, inp->codes + inp->n_codes);
@@ -1653,7 +1653,8 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, const mtmd_gen_in
         in_state.assign(inp->state_data, inp->state_data + inp->state_size);
     }
 
-    // code2wav has no hidden-state input, the batch entry is an unused placeholder
+    // gen_wav has no hidden-state input, the batch entry is an unused placeholder
+    // TODO @ngxson : some models in the future may require hidden-state input, need to update this code later
     clip_image_f32 dummy;
     dummy.set_size({1, 1}, false, true);
     dummy.cpy_buf(std::vector<float>(1, 0.0f));

tools/mtmd/mtmd.h

@@ -343,7 +343,8 @@ MTMD_API struct mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context *
 
 enum mtmd_gen_process_type {
     MTMD_GEN_PROCESS_TYPE_GEN_CODE, // h_state to codes
-    MTMD_GEN_PROCESS_TYPE_CODE2WAV, // codes to raw PCM audio
+    MTMD_GEN_PROCESS_TYPE_GEN_WAV,  // convert internal representation (codes, mel-spectrogram, etc.) to PCM audio
+                                    // for qwen3tts, this is code2wav
 };
 struct mtmd_gen_inp {
     enum mtmd_gen_process_type type;
@@ -354,7 +355,7 @@ struct mtmd_gen_inp {
     int32_t top_k;
     float   top_p;
 
-    // for MTMD_GEN_PROCESS_TYPE_CODE2WAV
+    // for MTMD_GEN_PROCESS_TYPE_GEN_WAV
     int32_t * codes;
     size_t    n_codes;
     const char * state_data;
@@ -369,7 +370,7 @@ struct mtmd_gen_out {
     const float * embd; // the generated hidden state, to be fed back to backbone
                         // it must have n_text_embd elements
 
-    // for MTMD_GEN_PROCESS_TYPE_CODE2WAV
+    // for MTMD_GEN_PROCESS_TYPE_GEN_WAV
     const float * audio;
     size_t        n_samples;
     const char * state_data;