diff --git a/conversion/pockettts.py b/conversion/pockettts.py index b6608cf884..05934acab4 100644 --- a/conversion/pockettts.py +++ b/conversion/pockettts.py @@ -41,13 +41,21 @@ _SAMPLE_RATE = 24000 # It lives only in the pip package's pocket_tts/config/.yaml, so it is keyed on the # model directory name here. 0.7 is the reference default (Config.default_temperature). # -# The packs also tune pad_with_spaces_for_short_inputs and remove_semicolons, which only -# affect text normalization for one or two packs each. Those are not carried over. +# model_recommended_frames_after_eos and pad_with_spaces_for_short_inputs come from the same +# per-pack yaml. remove_semicolons does too, but it maps ";" to "," and is applied to every +# pack on the cpp side instead of being carried here. _DEFAULT_TEMP = 0.7 _PACK_TEMP = { "english": 0.3, "english_2026-04": 0.3, } +# 0 leaves the tail length to the caller, which guesses it from the text +_PACK_FRAMES_AFTER_EOS = { + "french_24l": 8, +} +_PACK_PAD_SHORT_TEXT = { + "english_2026-01": True, +} def _pack_temp(name: str) -> float: @@ -203,6 +211,10 @@ class PocketTTSMmprojModel(MmprojModel): # the flow decoder draws its noise at this scale, see lsd_decode() in the reference self.gguf_writer.add_gen_audio_flow_temperature(_pack_temp(self.dir_model.name)) + self.gguf_writer.add_gen_audio_frames_after_eos( + _PACK_FRAMES_AFTER_EOS.get(self.dir_model.name, 0)) + self.gguf_writer.add_gen_audio_pad_short_text( + _PACK_PAD_SHORT_TEXT.get(self.dir_model.name, False)) def tensor_force_quant(self, name, new_name, bid, n_dims): del name, bid, n_dims diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 14deedf51c..3e6a9e17a8 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -402,6 +402,8 @@ class Keys: PROJECTOR_TYPE = "clip.gen.audio.projector_type" # for mixed modality models # noise scale of the flow decoder, differs between pocket-tts language packs FLOW_TEMPERATURE = "clip.gen.audio.flow_temperature" + FRAMES_AFTER_EOS = "clip.gen.audio.frames_after_eos" + PAD_SHORT_TEXT = "clip.gen.audio.pad_short_text" EMBEDDING_LENGTH = "clip.gen.audio.embedding_length" FEED_FORWARD_LENGTH = "clip.gen.audio.feed_forward_length" BLOCK_COUNT = "clip.gen.audio.block_count" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 9eacb94650..e67e2b6fef 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1441,6 +1441,12 @@ class GGUFWriter: def add_gen_audio_flow_temperature(self, value: float) -> None: self.add_float32(Keys.ClipGenAudio.FLOW_TEMPERATURE, value) + def add_gen_audio_frames_after_eos(self, value: int) -> None: + self.add_uint32(Keys.ClipGenAudio.FRAMES_AFTER_EOS, value) + + def add_gen_audio_pad_short_text(self, value: bool) -> None: + self.add_bool(Keys.ClipGenAudio.PAD_SHORT_TEXT, value) + def add_xielu_alpha_p(self, values: Sequence[float]): self.add_array(Keys.xIELU.ALPHA_P, values) diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index c3c35d6c3e..f6657e0bd5 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -94,6 +94,8 @@ #define KEY_GEN_AUDIO_PROJ_TYPE "clip.gen.audio.projector_type" // for models with mixed modalities // noise scale of the flow decoder, differs between pocket-tts language packs #define KEY_GEN_AUDIO_FLOW_TEMP "clip.gen.audio.flow_temperature" +#define KEY_GEN_AUDIO_FRAMES_EOS "clip.gen.audio.frames_after_eos" +#define KEY_GEN_AUDIO_PAD_SHORT "clip.gen.audio.pad_short_text" #define KEY_AUDIO_SUBSMPL_FACTOR "clip.audio.subsampling_factor" // diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index 1d6bea3952..9302f65a41 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -146,6 +146,8 @@ struct clip_hparams { int32_t mimi_tfm_context = 0; // attention window of the mimi transformers, in frames int32_t flow_n_step = 1; // lsd_decode steps float flow_temp = 0.0f; // noise std is sqrt(temp), differs per language pack + int32_t gen_frames_after_eos = 0; // tail the pack asks for, 0 leaves the guess to the caller + bool gen_pad_short_text = false; // qwen3tts code2wav int32_t wav_tfm_n_layer = 0; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 520e921bf0..bf5e386a51 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1760,7 +1760,9 @@ struct clip_model_loader { // differs per language pack, the converter writes it out. // the fallback is the reference's own default hparams.flow_temp = 0.7f; - get_f32(KEY_GEN_AUDIO_FLOW_TEMP, hparams.flow_temp, false); + get_f32 (KEY_GEN_AUDIO_FLOW_TEMP, hparams.flow_temp, false); + get_u32 (KEY_GEN_AUDIO_FRAMES_EOS, hparams.gen_frames_after_eos, false); + get_bool(KEY_GEN_AUDIO_PAD_SHORT, hparams.gen_pad_short_text, false); } break; case PROJECTOR_TYPE_PADDLEOCR: { diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp index 884b11680e..f2045bffd2 100644 --- a/tools/mtmd/mtmd-helper-gen.cpp +++ b/tools/mtmd/mtmd-helper-gen.cpp @@ -504,7 +504,8 @@ public: } } - const std::string text = prepare_text(std::string(inp->prompt, inp->prompt_len)); + const std::string text = prepare_text(std::string(inp->prompt, inp->prompt_len), + info.pad_short_text); if (text.empty()) { LOG_ERR("mtmd_helper_gen_audio: empty prompt\n"); return 1; @@ -775,8 +776,9 @@ private: void arm_chunk_budget(size_t idx) { const int n_tok = (int) chunks[idx].size(); chunk_budget = (int) std::ceil((n_tok / 3.0 + 2.0) * frame_rate); - // the reference guesses the tail from the word count, approximated here by tokens - frames_after_eos = n_tok <= 6 ? 5 : 3; + // the pack may pin the tail, otherwise the reference guesses it from the word count, + // approximated here by tokens + frames_after_eos = info.frames_after_eos > 0 ? info.frames_after_eos : (n_tok <= 6 ? 5 : 3); step_idx = 0; eos_step = -1; } @@ -832,12 +834,14 @@ private: } // same normalization as prepare_text_prompt() in the reference, it affects quality - static std::string prepare_text(const std::string & in) { + static std::string prepare_text(const std::string & in, bool pad_short) { std::string s; s.reserve(in.size() + 1); for (char c : in) { if (c == '\n' || c == '\r') { s += ' '; + } else if (c == ';') { + s += ','; } else { s += c; } @@ -855,6 +859,9 @@ private: if (std::isalnum(last)) { s += '.'; } + if (pad_short && count_words(s) < 5) { + s = std::string(8, ' ') + s; + } return s; } diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index efa66a972b..a3d72314cb 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -1596,9 +1596,13 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) { info.sample_rate = 24000; break; case PROJECTOR_TYPE_POCKETTTS_GEN: - info.type = MTMD_GEN_AUDIO_TYPE_POCKETTTS; - info.sample_rate = 24000; - break; + { + const clip_hparams * hp = clip_get_hparams(ctx->ctx_gen_a); + info.type = MTMD_GEN_AUDIO_TYPE_POCKETTTS; + info.sample_rate = 24000; + info.frames_after_eos = hp->gen_frames_after_eos; + info.pad_short_text = hp->gen_pad_short_text; + } break; default: info.type = MTMD_GEN_AUDIO_TYPE_NONE; break; diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index 6502d73def..e977ecd8a3 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -339,6 +339,8 @@ enum mtmd_gen_audio_type { struct mtmd_gen_audio_info { enum mtmd_gen_audio_type type; int32_t sample_rate; // in Hz, for example 24000 for qwen3tts + int32_t frames_after_eos; // tail the model asks for, 0 to guess it from the text + bool pad_short_text; // the model wants short prompts padded with spaces }; MTMD_API struct mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx);