From 1d591cc2b7efbfd155676ef75e7d89bd9fcab92f Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Tue, 11 Aug 2026 00:51:10 +0200 Subject: [PATCH] add mtmd_gen_inp_default --- tools/mtmd/clip-model.h | 3 +++ tools/mtmd/clip.cpp | 6 +++--- tools/mtmd/clip.h | 3 +-- tools/mtmd/mtmd-helper-gen.cpp | 37 ++++++++++++++++++---------------- tools/mtmd/mtmd.cpp | 28 +++++++++++++++++++++++-- tools/mtmd/mtmd.h | 14 ++++++++++--- 6 files changed, 64 insertions(+), 27 deletions(-) diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index ad25c008e7..4465d9e0ef 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -144,6 +144,9 @@ struct clip_hparams { // threshold for the "out_eos_score" graph output float gen_eos_threshold = 0.0f; + // default noise scale of a flow-matching decoder, see mtmd_gen_inp_default() + float gen_flow_temp = 0.0f; + // name of the weight variant, some pipelines tune themselves on it std::string gen_model_variant; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index a0c1c169ba..469da5f4aa 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1070,7 +1070,7 @@ static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const case PROJECTOR_TYPE_POCKETTTS_GEN: { const auto gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE; - const int n_step = params && params->n_steps > 0 ? params->n_steps : ctx->model.hparams.flow_n_step; + const int n_step = ctx->model.hparams.flow_n_step; const int64_t n_latent = ctx->model.gen_input_lin_w->ne[0]; GGML_ASSERT(n_step > 0); GGML_ASSERT(n_latent > 0); @@ -1784,6 +1784,7 @@ struct clip_model_loader { // flow_lm defaults, see pocket_tts/default_parameters.py hparams.flow_n_step = 1; hparams.gen_eos_threshold = -4.0f; + hparams.gen_flow_temp = 0.7f; // Config.default_temperature } break; case PROJECTOR_TYPE_PADDLEOCR: { @@ -4963,8 +4964,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { } else { // flow matching starts from gaussian noise, std = sqrt(temp) ggml_tensor * t = get_inp_tensor("inp_noise"); - // Config.default_temperature, a caller that knows its variant overrides it - const float temp = params->flow_temp > 0.0f ? params->flow_temp : 0.7f; + const float temp = params->temp > 0.0f ? params->temp : hparams.gen_flow_temp; std::normal_distribution dist(0.0f, std::sqrt(temp)); std::vector noise(ggml_nelements(t)); for (auto & v : noise) { diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h index a237a9466a..a5b7137752 100644 --- a/tools/mtmd/clip.h +++ b/tools/mtmd/clip.h @@ -106,8 +106,7 @@ struct clip_encode_params { std::vector * out_codes = nullptr; // this frame's 16 sampled codes std::vector * out_feats = nullptr; // continuous counterpart of out_codes uint32_t seed = UINT32_MAX; // UINT32_MAX for random - int32_t n_steps = -1; // integration steps, for flow-matching decoders - float flow_temp = 0.0f; // noise scale of the flow decoder, 0 for default + float temp = 0.0f; // sampling temperature, noise scale for flow-matching decoders bool * out_is_eos = nullptr; // GEN_WAV diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp index 72106aefb2..1c58d3ae19 100644 --- a/tools/mtmd/mtmd-helper-gen.cpp +++ b/tools/mtmd/mtmd-helper-gen.cpp @@ -203,8 +203,9 @@ public: prompt_pos = 0; pos = 0; - top_k = inp->top_k > 0 ? inp->top_k : 50; - top_p = inp->top_p > 0 ? inp->top_p : 1.0f; + const mtmd_gen_inp def = mtmd_gen_inp_default(mctx); + top_k = inp->top_k > 0 ? inp->top_k : def.top_k; + top_p = inp->top_p > 0 ? inp->top_p : def.top_p; seed = inp->seed; out_type = inp->out_type; @@ -258,7 +259,7 @@ public: return 0; } - mtmd_gen_inp inp{}; + mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE; inp.code0 = sampled - codec_0; inp.embd = const_cast(h_state_in); @@ -401,10 +402,11 @@ private: if (codes_buf.empty()) { return true; } - mtmd_gen_inp inp{}; + mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; inp.codes = codes_buf.data(); inp.n_codes = codes_buf.size(); + inp.seed = seed; // same seed as gen_code, else clip reseeds mid-generation inp.state_data = c2w_state.empty() ? nullptr : (const char *) c2w_state.data(); inp.state_size = c2w_state.size(); mtmd_gen_out out{}; @@ -458,9 +460,10 @@ private: // settings that only live in the reference's per-pack yaml, not in the checkpoint // the english packs share the same shapes and tokenizer, but disagree on these +// all three are 0 / false when the pack does not tune them, the model default is then used struct pockettts_pack_settings { - float temp = 0.7f; // Config.default_temperature - int frames_after_eos = 0; // 0 leaves the tail length to the caller + float temp = 0.0f; + int frames_after_eos = 0; bool pad_short_text = false; }; @@ -473,10 +476,9 @@ static pockettts_pack_settings pockettts_pack(const char * variant) { }; auto it = packs.find(variant ? variant : ""); if (it == packs.end()) { - pockettts_pack_settings def; - LOG_WRN("mtmd_helper_gen_audio: no tuned settings for pocket-tts variant \"%s\", " - "using temperature %.1f\n", variant ? variant : "", def.temp); - return def; + LOG_WRN("mtmd_helper_gen_audio: no tuned settings for pocket-tts variant \"%s\"\n", + variant ? variant : ""); + return {}; } return it->second; } @@ -609,13 +611,14 @@ public: int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override { (void) sampled; // the backbone output is continuous, there is no token to consume - mtmd_gen_inp inp{}; - inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE; - inp.embd = const_cast(h_state_in); + mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); + inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE; + inp.embd = const_cast(h_state_in); // clip only reseeds when the seed changes, so pass the same one on every step - inp.seed = seed; - inp.n_steps = -1; - inp.flow_temp = pack.temp; + inp.seed = seed; + if (pack.temp > 0.0f) { + inp.temp = pack.temp; + } mtmd_gen_out out{}; if (mtmd_gen_audio_process(mctx, &inp, &out) != 0) { LOG_ERR("mtmd_helper_gen_audio: flow decode failed\n"); @@ -938,7 +941,7 @@ private: if (feats_buf.empty()) { return true; } - mtmd_gen_inp inp{}; + mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; inp.feats = feats_buf.data(); inp.n_feats = feats_buf.size(); diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 1249775545..1cf297829a 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -1826,6 +1826,31 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) { return info; } +mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) { + mtmd_gen_inp inp{}; + inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE; + inp.seed = UINT32_MAX; + if (!ctx->ctx_gen_a) { + return inp; + } + + const clip_hparams * hparams = clip_get_hparams(ctx->ctx_gen_a); + switch (clip_get_projector_type(ctx->ctx_gen_a)) { + case PROJECTOR_TYPE_QWEN3TTS_GEN: + // https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-Base/blob/main/generation_config.json + inp.top_k = 50; + inp.top_p = 1.0f; + inp.temp = 0.9f; // TODO: handle this on graph + break; + case PROJECTOR_TYPE_POCKETTTS_GEN: + inp.temp = hparams->gen_flow_temp; + break; + default: + break; + } + return inp; +} + static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, const mtmd_gen_inp * inp, mtmd_gen_out * out) { clip_ctx * ctx_clip = ctx->ctx_gen_a; if (!ctx_clip) { @@ -1862,8 +1887,7 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, const mtmd_gen_in params.top_k = inp->top_k; params.top_p = inp->top_p; params.seed = inp->seed; - params.n_steps = inp->n_steps; - params.flow_temp = inp->flow_temp; + params.temp = inp->temp; params.out_is_eos = &is_eos; if (!clip_encode(ctx_clip, ¶ms)) { diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index 5e8c9a35e3..c1a5921db2 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -346,19 +346,23 @@ enum mtmd_gen_audio_type { MTMD_GEN_AUDIO_TYPE_QWEN3TTS, MTMD_GEN_AUDIO_TYPE_POCKETTTS, }; + struct mtmd_gen_audio_info { enum mtmd_gen_audio_type type; int32_t sample_rate; // in Hz, for example 24000 for qwen3tts const char * model_variant; // name of the weight variant, can be nullptr if not applicable }; + MTMD_API struct mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx); + enum mtmd_gen_process_type { MTMD_GEN_PROCESS_TYPE_GEN_CODE, // h_state to semantic (codes, mel-spectrogram, etc.) MTMD_GEN_PROCESS_TYPE_GEN_WAV, // convert semantic to PCM audio // for qwen3tts, this is code2wav // for pocket-tts, this is mimi decoder }; + struct mtmd_gen_inp { enum mtmd_gen_process_type type; @@ -367,9 +371,8 @@ struct mtmd_gen_inp { float * embd; // the hidden state from backbone, must have n_text_embd elements int32_t top_k; float top_p; - uint32_t seed; // UINT32_MAX for random - int32_t n_steps; // integration steps, for flow-matching decoders (-1 for default) - float flow_temp; // noise scale, for flow-matching decoders (0 for default) + uint32_t seed; // UINT32_MAX for random + float temp; // sampling temperature, or noise scale for flow-matching decoders // for MTMD_GEN_PROCESS_TYPE_GEN_WAV // pass either codes (discrete) or feats (continuous), depending on the pipeline @@ -380,6 +383,7 @@ struct mtmd_gen_inp { const char * state_data; size_t state_size; }; + struct mtmd_gen_out { // note: output memory is allocated by the context, valid until next process() call @@ -398,6 +402,10 @@ struct mtmd_gen_out { const char * state_data; size_t state_size; }; + +// defaults tuned for the loaded pipeline, callers override only what they care about +MTMD_API struct mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx); + // note: this API is stateless, caller must handle state management and audio frame accumulation MTMD_API int32_t mtmd_gen_audio_process(mtmd_context * ctx, const struct mtmd_gen_inp * inp,