Superminaren/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-07 20:47:30 +02:00
Code Issues Packages Projects Releases Wiki Activity

add mtmd_gen_inp_default

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-08-11 00:51:10 +02:00
parent 3e47aaec49
commit 1d591cc2b7
6 changed files with 64 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/clip-model.h
+3
View File
@@ -144,6 +144,9 @@ struct clip_hparams {
// threshold for the "out_eos_score" graph output
float gen_eos_threshold = 0.0f;
// default noise scale of a flow-matching decoder, see mtmd_gen_inp_default()
float gen_flow_temp = 0.0f;
// name of the weight variant, some pipelines tune themselves on it
std::string gen_model_variant;
tools/mtmd/clip.cpp
+3 -3
View File
@@ -1070,7 +1070,7 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
case PROJECTOR_TYPE_POCKETTTS_GEN:
{
const auto gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE;
const int n_step = params && params->n_steps > 0 ? params->n_steps : ctx->model.hparams.flow_n_step;
const int n_step = ctx->model.hparams.flow_n_step;
const int64_t n_latent = ctx->model.gen_input_lin_w->ne[0];
GGML_ASSERT(n_step > 0);
GGML_ASSERT(n_latent > 0);
@@ -1784,6 +1784,7 @@ struct clip_model_loader {
// flow_lm defaults, see pocket_tts/default_parameters.py
hparams.flow_n_step = 1;
hparams.gen_eos_threshold = -4.0f;
hparams.gen_flow_temp = 0.7f; // Config.default_temperature
} break;
case PROJECTOR_TYPE_PADDLEOCR:
{
@@ -4963,8 +4964,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
} else {
// flow matching starts from gaussian noise, std = sqrt(temp)
ggml_tensor * t = get_inp_tensor("inp_noise");
// Config.default_temperature, a caller that knows its variant overrides it
const float temp = params->flow_temp > 0.0f ? params->flow_temp : 0.7f;
const float temp = params->temp > 0.0f ? params->temp : hparams.gen_flow_temp;
std::normal_distribution<float> dist(0.0f, std::sqrt(temp));
std::vector<float> noise(ggml_nelements(t));
for (auto & v : noise) {
tools/mtmd/clip.h
+1 -2
View File
@@ -106,8 +106,7 @@ struct clip_encode_params {
std::vector<int32_t> * out_codes = nullptr; // this frame's 16 sampled codes
std::vector<float> * out_feats = nullptr; // continuous counterpart of out_codes
uint32_t seed = UINT32_MAX; // UINT32_MAX for random
int32_t n_steps = -1; // integration steps, for flow-matching decoders
float flow_temp = 0.0f; // noise scale of the flow decoder, 0 for default
float temp = 0.0f; // sampling temperature, noise scale for flow-matching decoders
bool * out_is_eos = nullptr;
// GEN_WAV
tools/mtmd/mtmd-helper-gen.cpp
+20 -17
View File
@@ -203,8 +203,9 @@ public:
prompt_pos = 0;
pos = 0;
top_k = inp->top_k > 0 ? inp->top_k : 50;
top_p = inp->top_p > 0 ? inp->top_p : 1.0f;
const mtmd_gen_inp def = mtmd_gen_inp_default(mctx);
top_k = inp->top_k > 0 ? inp->top_k : def.top_k;
top_p = inp->top_p > 0 ? inp->top_p : def.top_p;
seed = inp->seed;
out_type = inp->out_type;
@@ -258,7 +259,7 @@ public:
return 0;
}
mtmd_gen_inp inp{};
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
inp.code0 = sampled - codec_0;
inp.embd = const_cast<float *>(h_state_in);
@@ -401,10 +402,11 @@ private:
if (codes_buf.empty()) {
return true;
}
mtmd_gen_inp inp{};
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
inp.codes = codes_buf.data();
inp.n_codes = codes_buf.size();
inp.seed = seed; // same seed as gen_code, else clip reseeds mid-generation
inp.state_data = c2w_state.empty() ? nullptr : (const char *) c2w_state.data();
inp.state_size = c2w_state.size();
mtmd_gen_out out{};
@@ -458,9 +460,10 @@ private:
// settings that only live in the reference's per-pack yaml, not in the checkpoint
// the english packs share the same shapes and tokenizer, but disagree on these
// all three are 0 / false when the pack does not tune them, the model default is then used
struct pockettts_pack_settings {
float temp = 0.7f; // Config.default_temperature
int frames_after_eos = 0; // 0 leaves the tail length to the caller
float temp = 0.0f;
int frames_after_eos = 0;
bool pad_short_text = false;
};
@@ -473,10 +476,9 @@ static pockettts_pack_settings pockettts_pack(const char * variant) {
};
auto it = packs.find(variant ? variant : "");
if (it == packs.end()) {
pockettts_pack_settings def;
LOG_WRN("mtmd_helper_gen_audio: no tuned settings for pocket-tts variant \"%s\", "
"using temperature %.1f\n", variant ? variant : "", def.temp);
return def;
LOG_WRN("mtmd_helper_gen_audio: no tuned settings for pocket-tts variant \"%s\"\n",
variant ? variant : "");
return {};
}
return it->second;
}
@@ -609,13 +611,14 @@ public:
int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
(void) sampled; // the backbone output is continuous, there is no token to consume
mtmd_gen_inp inp{};
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
inp.embd = const_cast<float *>(h_state_in);
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
inp.embd = const_cast<float *>(h_state_in);
// clip only reseeds when the seed changes, so pass the same one on every step
inp.seed = seed;
inp.n_steps = -1;
inp.flow_temp = pack.temp;
inp.seed = seed;
if (pack.temp > 0.0f) {
inp.temp = pack.temp;
}
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, &inp, &out) != 0) {
LOG_ERR("mtmd_helper_gen_audio: flow decode failed\n");
@@ -938,7 +941,7 @@ private:
if (feats_buf.empty()) {
return true;
}
mtmd_gen_inp inp{};
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
inp.feats = feats_buf.data();
inp.n_feats = feats_buf.size();
tools/mtmd/mtmd.cpp
+26 -2
View File
@@ -1826,6 +1826,31 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) {
return info;
}
mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) {
mtmd_gen_inp inp{};
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
inp.seed = UINT32_MAX;
if (!ctx->ctx_gen_a) {
return inp;
}
const clip_hparams * hparams = clip_get_hparams(ctx->ctx_gen_a);
switch (clip_get_projector_type(ctx->ctx_gen_a)) {
case PROJECTOR_TYPE_QWEN3TTS_GEN:
// https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-Base/blob/main/generation_config.json
inp.top_k = 50;
inp.top_p = 1.0f;
inp.temp = 0.9f; // TODO: handle this on graph
break;
case PROJECTOR_TYPE_POCKETTTS_GEN:
inp.temp = hparams->gen_flow_temp;
break;
default:
break;
}
return inp;
}
static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, const mtmd_gen_inp * inp, mtmd_gen_out * out) {
clip_ctx * ctx_clip = ctx->ctx_gen_a;
if (!ctx_clip) {
@@ -1862,8 +1887,7 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, const mtmd_gen_in
params.top_k = inp->top_k;
params.top_p = inp->top_p;
params.seed = inp->seed;
params.n_steps = inp->n_steps;
params.flow_temp = inp->flow_temp;
params.temp = inp->temp;
params.out_is_eos = &is_eos;
if (!clip_encode(ctx_clip, &params)) {
tools/mtmd/mtmd.h
+11 -3
View File
@@ -346,19 +346,23 @@ enum mtmd_gen_audio_type {
MTMD_GEN_AUDIO_TYPE_QWEN3TTS,
MTMD_GEN_AUDIO_TYPE_POCKETTTS,
};
struct mtmd_gen_audio_info {
enum mtmd_gen_audio_type type;
int32_t sample_rate; // in Hz, for example 24000 for qwen3tts
const char * model_variant; // name of the weight variant, can be nullptr if not applicable
};
MTMD_API struct mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx);
enum mtmd_gen_process_type {
MTMD_GEN_PROCESS_TYPE_GEN_CODE, // h_state to semantic (codes, mel-spectrogram, etc.)
MTMD_GEN_PROCESS_TYPE_GEN_WAV, // convert semantic to PCM audio
// for qwen3tts, this is code2wav
// for pocket-tts, this is mimi decoder
};
struct mtmd_gen_inp {
enum mtmd_gen_process_type type;
@@ -367,9 +371,8 @@ struct mtmd_gen_inp {
float * embd; // the hidden state from backbone, must have n_text_embd elements
int32_t top_k;
float top_p;
uint32_t seed; // UINT32_MAX for random
int32_t n_steps; // integration steps, for flow-matching decoders (-1 for default)
float flow_temp; // noise scale, for flow-matching decoders (0 for default)
uint32_t seed; // UINT32_MAX for random
float temp; // sampling temperature, or noise scale for flow-matching decoders
// for MTMD_GEN_PROCESS_TYPE_GEN_WAV
// pass either codes (discrete) or feats (continuous), depending on the pipeline
@@ -380,6 +383,7 @@ struct mtmd_gen_inp {
const char * state_data;
size_t state_size;
};
struct mtmd_gen_out {
// note: output memory is allocated by the context, valid until next process() call
@@ -398,6 +402,10 @@ struct mtmd_gen_out {
const char * state_data;
size_t state_size;
};
// defaults tuned for the loaded pipeline, callers override only what they care about
MTMD_API struct mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx);
// note: this API is stateless, caller must handle state management and audio frame accumulation
MTMD_API int32_t mtmd_gen_audio_process(mtmd_context * ctx,
const struct mtmd_gen_inp * inp,