From d14bb68e5f60ffba4b4bf9541924d8a783fab30e Mon Sep 17 00:00:00 2001 From: Pascal Date: Thu, 6 Aug 2026 13:50:09 +0200 Subject: [PATCH] mtmd: carry the remaining pocket-tts per-pack settings The language packs also tune the end-of-speech padding and the padding of short prompts, next to the temperature already carried in the mmproj: french_24l asks for 8 tail frames instead of the guessed 3, english_2026-01 asks for short prompts to be padded with spaces. Write both in the mmproj as clip.gen.audio.frames_after_eos and clip.gen.audio.pad_short_text, keyed on the pack in the conversion script like the temperature. The loader keeps them optional, so a mmproj without them behaves as before. Map semicolons to commas for every pack instead, the reference only asks for it on three of them and it costs nothing elsewhere. Existing mmproj files must be converted again to carry the two keys. On a long french text the port now lands within 2% of the reference: 22.96s against 23.44s, with the same peak level and the same amount of silence. --- conversion/pockettts.py | 16 ++++++++++++++-- gguf-py/gguf/constants.py | 2 ++ gguf-py/gguf/gguf_writer.py | 6 ++++++ tools/mtmd/clip-impl.h | 2 ++ tools/mtmd/clip-model.h | 2 ++ tools/mtmd/clip.cpp | 4 +++- tools/mtmd/mtmd-helper-gen.cpp | 15 +++++++++++---- tools/mtmd/mtmd.cpp | 10 +++++++--- tools/mtmd/mtmd.h | 2 ++ 9 files changed, 49 insertions(+), 10 deletions(-) diff --git a/conversion/pockettts.py b/conversion/pockettts.py index b6608cf884..05934acab4 100644 --- a/conversion/pockettts.py +++ b/conversion/pockettts.py @@ -41,13 +41,21 @@ _SAMPLE_RATE = 24000 # It lives only in the pip package's pocket_tts/config/.yaml, so it is keyed on the # model directory name here. 0.7 is the reference default (Config.default_temperature). # -# The packs also tune pad_with_spaces_for_short_inputs and remove_semicolons, which only -# affect text normalization for one or two packs each. Those are not carried over. +# model_recommended_frames_after_eos and pad_with_spaces_for_short_inputs come from the same +# per-pack yaml. remove_semicolons does too, but it maps ";" to "," and is applied to every +# pack on the cpp side instead of being carried here. _DEFAULT_TEMP = 0.7 _PACK_TEMP = { "english": 0.3, "english_2026-04": 0.3, } +# 0 leaves the tail length to the caller, which guesses it from the text +_PACK_FRAMES_AFTER_EOS = { + "french_24l": 8, +} +_PACK_PAD_SHORT_TEXT = { + "english_2026-01": True, +} def _pack_temp(name: str) -> float: @@ -203,6 +211,10 @@ class PocketTTSMmprojModel(MmprojModel): # the flow decoder draws its noise at this scale, see lsd_decode() in the reference self.gguf_writer.add_gen_audio_flow_temperature(_pack_temp(self.dir_model.name)) + self.gguf_writer.add_gen_audio_frames_after_eos( + _PACK_FRAMES_AFTER_EOS.get(self.dir_model.name, 0)) + self.gguf_writer.add_gen_audio_pad_short_text( + _PACK_PAD_SHORT_TEXT.get(self.dir_model.name, False)) def tensor_force_quant(self, name, new_name, bid, n_dims): del name, bid, n_dims diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 14deedf51c..3e6a9e17a8 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -402,6 +402,8 @@ class Keys: PROJECTOR_TYPE = "clip.gen.audio.projector_type" # for mixed modality models # noise scale of the flow decoder, differs between pocket-tts language packs FLOW_TEMPERATURE = "clip.gen.audio.flow_temperature" + FRAMES_AFTER_EOS = "clip.gen.audio.frames_after_eos" + PAD_SHORT_TEXT = "clip.gen.audio.pad_short_text" EMBEDDING_LENGTH = "clip.gen.audio.embedding_length" FEED_FORWARD_LENGTH = "clip.gen.audio.feed_forward_length" BLOCK_COUNT = "clip.gen.audio.block_count" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 9eacb94650..e67e2b6fef 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1441,6 +1441,12 @@ class GGUFWriter: def add_gen_audio_flow_temperature(self, value: float) -> None: self.add_float32(Keys.ClipGenAudio.FLOW_TEMPERATURE, value) + def add_gen_audio_frames_after_eos(self, value: int) -> None: + self.add_uint32(Keys.ClipGenAudio.FRAMES_AFTER_EOS, value) + + def add_gen_audio_pad_short_text(self, value: bool) -> None: + self.add_bool(Keys.ClipGenAudio.PAD_SHORT_TEXT, value) + def add_xielu_alpha_p(self, values: Sequence[float]): self.add_array(Keys.xIELU.ALPHA_P, values) diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index c3c35d6c3e..f6657e0bd5 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -94,6 +94,8 @@ #define KEY_GEN_AUDIO_PROJ_TYPE "clip.gen.audio.projector_type" // for models with mixed modalities // noise scale of the flow decoder, differs between pocket-tts language packs #define KEY_GEN_AUDIO_FLOW_TEMP "clip.gen.audio.flow_temperature" +#define KEY_GEN_AUDIO_FRAMES_EOS "clip.gen.audio.frames_after_eos" +#define KEY_GEN_AUDIO_PAD_SHORT "clip.gen.audio.pad_short_text" #define KEY_AUDIO_SUBSMPL_FACTOR "clip.audio.subsampling_factor" // diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index 1d6bea3952..9302f65a41 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -146,6 +146,8 @@ struct clip_hparams { int32_t mimi_tfm_context = 0; // attention window of the mimi transformers, in frames int32_t flow_n_step = 1; // lsd_decode steps float flow_temp = 0.0f; // noise std is sqrt(temp), differs per language pack + int32_t gen_frames_after_eos = 0; // tail the pack asks for, 0 leaves the guess to the caller + bool gen_pad_short_text = false; // qwen3tts code2wav int32_t wav_tfm_n_layer = 0; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 520e921bf0..bf5e386a51 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1760,7 +1760,9 @@ struct clip_model_loader { // differs per language pack, the converter writes it out. // the fallback is the reference's own default hparams.flow_temp = 0.7f; - get_f32(KEY_GEN_AUDIO_FLOW_TEMP, hparams.flow_temp, false); + get_f32 (KEY_GEN_AUDIO_FLOW_TEMP, hparams.flow_temp, false); + get_u32 (KEY_GEN_AUDIO_FRAMES_EOS, hparams.gen_frames_after_eos, false); + get_bool(KEY_GEN_AUDIO_PAD_SHORT, hparams.gen_pad_short_text, false); } break; case PROJECTOR_TYPE_PADDLEOCR: { diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp index 884b11680e..f2045bffd2 100644 --- a/tools/mtmd/mtmd-helper-gen.cpp +++ b/tools/mtmd/mtmd-helper-gen.cpp @@ -504,7 +504,8 @@ public: } } - const std::string text = prepare_text(std::string(inp->prompt, inp->prompt_len)); + const std::string text = prepare_text(std::string(inp->prompt, inp->prompt_len), + info.pad_short_text); if (text.empty()) { LOG_ERR("mtmd_helper_gen_audio: empty prompt\n"); return 1; @@ -775,8 +776,9 @@ private: void arm_chunk_budget(size_t idx) { const int n_tok = (int) chunks[idx].size(); chunk_budget = (int) std::ceil((n_tok / 3.0 + 2.0) * frame_rate); - // the reference guesses the tail from the word count, approximated here by tokens - frames_after_eos = n_tok <= 6 ? 5 : 3; + // the pack may pin the tail, otherwise the reference guesses it from the word count, + // approximated here by tokens + frames_after_eos = info.frames_after_eos > 0 ? info.frames_after_eos : (n_tok <= 6 ? 5 : 3); step_idx = 0; eos_step = -1; } @@ -832,12 +834,14 @@ private: } // same normalization as prepare_text_prompt() in the reference, it affects quality - static std::string prepare_text(const std::string & in) { + static std::string prepare_text(const std::string & in, bool pad_short) { std::string s; s.reserve(in.size() + 1); for (char c : in) { if (c == '\n' || c == '\r') { s += ' '; + } else if (c == ';') { + s += ','; } else { s += c; } @@ -855,6 +859,9 @@ private: if (std::isalnum(last)) { s += '.'; } + if (pad_short && count_words(s) < 5) { + s = std::string(8, ' ') + s; + } return s; } diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index efa66a972b..a3d72314cb 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -1596,9 +1596,13 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) { info.sample_rate = 24000; break; case PROJECTOR_TYPE_POCKETTTS_GEN: - info.type = MTMD_GEN_AUDIO_TYPE_POCKETTTS; - info.sample_rate = 24000; - break; + { + const clip_hparams * hp = clip_get_hparams(ctx->ctx_gen_a); + info.type = MTMD_GEN_AUDIO_TYPE_POCKETTTS; + info.sample_rate = 24000; + info.frames_after_eos = hp->gen_frames_after_eos; + info.pad_short_text = hp->gen_pad_short_text; + } break; default: info.type = MTMD_GEN_AUDIO_TYPE_NONE; break; diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index 6502d73def..e977ecd8a3 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -339,6 +339,8 @@ enum mtmd_gen_audio_type { struct mtmd_gen_audio_info { enum mtmd_gen_audio_type type; int32_t sample_rate; // in Hz, for example 24000 for qwen3tts + int32_t frames_after_eos; // tail the model asks for, 0 to guess it from the text + bool pad_short_text; // the model wants short prompts padded with spaces }; MTMD_API struct mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx);