clean up tts.cpp

conversion/qwen3tts.py

@@ -36,10 +36,6 @@ _ACT2FN = {
 }
 
 
-# TODO: figure out the correct template
-DEFAULT_TEMPLATE = """{% for m in messages %}{{m['content']}}{% endfor %}"""
-
-
 @ModelBase.register("Qwen3TTSForConditionalGeneration")
 class Qwen3TTSTalkerModel(TextModel):
     model_arch = gguf.MODEL_ARCH.QWEN3TTS
@@ -120,7 +116,6 @@ class Qwen3TTSTalkerModel(TextModel):
 
     def set_gguf_parameters(self):
         super().set_gguf_parameters()
-        self.gguf_writer.add_chat_template(DEFAULT_TEMPLATE)
 
         # note: final vocab layout is [text_vocab | codec_vocab], with text_vocab is actually padded with -inf in cgraph
         # for codec_vocab, only first 2048 rows can be sampled for semantic code

tools/mtmd/mtmd-helper-gen.cpp

@@ -21,9 +21,6 @@
 
 // maps the --tts-lang codes (see tools/tts/README.md) to the language
 // names used by the codec_language special tokens
-//
-// primary keys are ISO 639-1 codes; the old non-standard codes ("cn", "ge",
-// "po", "sp") are kept as aliases for backward compatibility
 static const std::unordered_map<std::string, std::string> tts_lang_codes = {
     { "zh", "chinese"    },
     { "en", "english"    },

tools/mtmd/mtmd.h

@@ -342,8 +342,8 @@ struct mtmd_gen_audio_info {
 MTMD_API struct mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx);
 
 enum mtmd_gen_process_type {
-    MTMD_GEN_PROCESS_TYPE_GEN_CODE, // h_state to codes
-    MTMD_GEN_PROCESS_TYPE_GEN_WAV,  // convert internal representation (codes, mel-spectrogram, etc.) to PCM audio
+    MTMD_GEN_PROCESS_TYPE_GEN_CODE, // h_state to semantic (codes, mel-spectrogram, etc.)
+    MTMD_GEN_PROCESS_TYPE_GEN_WAV,  // convert semantic to PCM audio
                                     // for qwen3tts, this is code2wav
 };
 struct mtmd_gen_inp {

tools/tts/tts.cpp

@@ -62,12 +62,19 @@ int main(int argc, char ** argv) {
         return 1;
     }
 
+    // important: keep this file as generic as possible
+    //            model-specific logic should be in mtmd-helper-gen or mtmd API
+
     // always enable embd, so that we can pass hidden states to the audio generation helper
     params.embedding = true;
 
     llama_backend_init();
     llama_numa_init(params.numa);
 
+    //
+    // load backbone model and mmproj
+    //
+
     auto llama_init = common_init_from_params(params);
     llama_model    * model = llama_init->model();
     llama_context  * lctx  = llama_init->context();
@@ -89,6 +96,10 @@ int main(int argc, char ** argv) {
         return 1;
     }
 
+    //
+    // stage 0: process speaker reference file, if any
+    //
+
     mtmd::bitmap_ptr speaker_bitmap;
     if (!params.tts_speaker_file.empty()) {
         auto wrapper = mtmd_helper_bitmap_init_from_file(mctx.get(), params.tts_speaker_file.c_str(), false);
@@ -109,24 +120,21 @@ int main(int argc, char ** argv) {
     inp.top_p       = params.sampling.top_p;
     inp.out_type    = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
     const int64_t t_prompt_start_us = ggml_time_us();
+
+    //
+    // stage 1: process prompt via backbone model, generate semantic representation
+    //
+
     if (gen.set_input(&inp) != 0) {
         LOG_ERR("set_input failed\n");
         return 1;
     }
 
-    // codec_0 (backbone) EOS token: ordinary LLM sampling concern, kept out of the
-    // model-agnostic audio-generation helper
-    const llama_vocab * vocab = llama_model_get_vocab(model);
-    llama_token codec_eos_tok = LLAMA_TOKEN_NULL;
-    for (llama_token t = 0; t < llama_vocab_n_tokens(vocab); t++) {
-        if (!strcmp(llama_vocab_get_text(vocab, t), "<|codec_eos_token|>")) { codec_eos_tok = t; break; }
-    }
-    if (codec_eos_tok == LLAMA_TOKEN_NULL) {
-        LOG_ERR("missing codec eos token in vocab\n");
-        return 1;
-    }
+    // TODO: explicit prompt processing step here
 
-    auto sample_codec0 = [&]() -> llama_token {
+    const llama_vocab * vocab = llama_model_get_vocab(model);
+
+    auto sample_semantic_code = [&]() -> llama_token {
         llama_token t = common_sampler_sample(smpl, lctx, -1);
         common_sampler_accept(smpl, t, true);
         return t;
@@ -134,20 +142,24 @@ int main(int argc, char ** argv) {
 
     const int max_new = params.n_predict > 0 ? params.n_predict : 512;
     int n_frames = 0;
-    llama_token sampled = sample_codec0();
+    llama_token sampled = sample_semantic_code();
     const float * h_state = llama_get_embeddings_ith(lctx, -1);
 
     tts_timings timings;
     const int64_t t_gen_start_us = ggml_time_us();
 
-    for (; n_frames < max_new && sampled != codec_eos_tok; n_frames++) {
+    for (; n_frames < max_new && !llama_vocab_is_eog(vocab, sampled); n_frames++) {
         const float * h_next = nullptr;
+
+        // stage 2+3: semantic --> acoustic details --> audio waveform
+        //            step() runs both stages and returns new h_state for next step
         if (gen.step(sampled, h_state, &h_next) != 0) {
             LOG_ERR("step failed at frame %d\n", n_frames);
             return 1;
         }
+
         h_state = h_next;
-        sampled = sample_codec0();
+        sampled = sample_semantic_code();
         timings.report(n_frames + 1);
     }
     const double t_gen_s = (ggml_time_us() - t_gen_start_us) / 1e6;