diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index 67ab22c3fc..6946b607f5 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -53,6 +53,7 @@ add_library(mtmd models/qwen3a.cpp models/mimo-audio.cpp models/qwen3tts-spkenc.cpp + models/qwen3tts-gen.cpp models/step3vl.cpp models/siglip.cpp models/whisper-enc.cpp diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 5274638941..978750724a 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1029,6 +1029,10 @@ static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const { builder = std::make_unique(ctx, img); } break; + case PROJECTOR_TYPE_QWEN3TTS_GEN: + { + builder = std::make_unique(ctx, img); + } break; case PROJECTOR_TYPE_YOUTUVL: { builder = std::make_unique(ctx, img); @@ -3543,6 +3547,8 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params ctx_gen_audio = new clip_ctx(ctx_params); loader.load_hparams(ctx_gen_audio->model, CLIP_MODALITY_GEN_AUDIO); loader.load_tensors(*ctx_gen_audio); + // TODO: fix warmup + ctx_gen_audio->buf_compute_meta.resize(ctx_gen_audio->max_nodes * ggml_tensor_overhead() + ggml_graph_overhead()); } } catch (const std::exception & e) { @@ -3878,6 +3884,11 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { // a single speaker embedding vector, regardless of its length n_patches = 1; } break; + case PROJECTOR_TYPE_QWEN3TTS_GEN: + { + // one hidden-state vector fed back to the talker per call + n_patches = 1; + } break; case PROJECTOR_TYPE_GRANITE4_VISION: { // Per-tile output token count: each projector block outputs @@ -4578,6 +4589,11 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { { // do nothing } break; + case PROJECTOR_TYPE_QWEN3TTS_GEN: + { + std::vector code0 = { params->code0 }; + set_input_i32("inp_code0", code0); + } break; case PROJECTOR_TYPE_HUNYUANVL: { // Compute the HunyuanVL 2D position embedding on CPU (with the diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index ca57a5f677..bab2cffdd5 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -226,6 +226,11 @@ struct clip_graph_qwen3tts_spkenc : clip_graph { ggml_tensor * attentive_stats_pool(ggml_tensor * x) const; }; +struct clip_graph_qwen3tts_gen : clip_graph { + clip_graph_qwen3tts_gen(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} + ggml_cgraph * build() override; +}; + struct clip_graph_kimik25 : clip_graph { clip_graph_kimik25(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} ggml_cgraph * build() override; diff --git a/tools/mtmd/models/qwen3tts-gen.cpp b/tools/mtmd/models/qwen3tts-gen.cpp new file mode 100644 index 0000000000..2fe485f0d8 --- /dev/null +++ b/tools/mtmd/models/qwen3tts-gen.cpp @@ -0,0 +1,29 @@ +#include "models.h" + +ggml_cgraph * clip_graph_qwen3tts_gen::build() { + ggml_tensor * h_state = build_inp_raw(1); + h_state = ggml_reshape_1d(ctx0, h_state, h_state->ne[0]); + cb(h_state, "inp_h_state", -1); + + // inp_code0: sampled codebook-0 id from the talker's codec_head + ggml_tensor * code0_embd; // [n_embd, 1] + { + ggml_tensor * code0 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1); + ggml_set_name(code0, "inp_code0"); + ggml_set_input(code0); + + ggml_tensor * code0_embd = ggml_get_rows(ctx0, model.gen_code_out_embd_w, code0); + code0_embd = ggml_reshape_1d(ctx0, code0_embd, code0_embd->ne[0]); + cb(code0_embd, "code0_embd", -1); + } + + // TODO: code_predictor transformer block (5 layers) + per-codebook + // sampling loop; placeholder combination so the graph has a single + // well-defined output tensor until that lands + ggml_tensor * cur = ggml_add(ctx0, h_state, code0_embd); + cur = ggml_reshape_2d(ctx0, cur, cur->ne[0], 1); + cb(cur, "gen_audio_out", -1); + + ggml_build_forward_expand(gf, cur); + return gf; +} diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 5e83bf039a..4508863e2d 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -1562,6 +1562,73 @@ float * mtmd_get_output_embd(mtmd_context * ctx) { return ctx->out_embd.data(); } +// +// audio generation +// + +mtmd_gen_audio_type mtmd_gen_audio_get_type(const mtmd_context * ctx) { + if (!ctx->ctx_gen_a) { + return MTMD_GEN_AUDIO_TYPE_NONE; + } + switch (clip_get_projector_type(ctx->ctx_gen_a)) { + case PROJECTOR_TYPE_QWEN3TTS_GEN: + return MTMD_GEN_AUDIO_TYPE_MTP; + default: + return MTMD_GEN_AUDIO_TYPE_NONE; + } +} + +static int32_t mtmd_gen_audio_impl(mtmd_context * ctx, const mtmd_gen_inp * inp, mtmd_gen_out * out) { + clip_ctx * ctx_clip = ctx->ctx_gen_a; + if (!ctx_clip) { + LOG_ERR("%s: model does not support audio generation\n", __func__); + return 1; + } + + const size_t n_embd = (size_t) clip_n_mmproj_embd(ctx_clip); + if (inp->n_embd != n_embd) { + LOG_ERR("%s: n_embd mismatch: model expects %zu, got %zu\n", __func__, n_embd, inp->n_embd); + return 1; + } + + clip_image_f32 hidden_state; + hidden_state.set_size({(int) n_embd, 1}, false, true); + hidden_state.cpy_buf(std::vector(inp->embd, inp->embd + n_embd)); + + clip_image_f32_batch batch; + batch.is_audio = true; + batch.entries.push_back(std::move(hidden_state)); + + std::vector out_embd(n_embd); + clip_encode_params params; + params.imgs = &batch; + params.n_threads = ctx->n_threads; + params.out_embd = &out_embd; + params.code0 = inp->code0; + + if (!clip_encode(ctx_clip, ¶ms)) { + LOG_ERR("%s: clip_encode failed\n", __func__); + return 1; + } + + if (!out->embd || out->n_embd != out_embd.size()) { + LOG_ERR("%s: output buffer size mismatch: expected %zu, got %zu\n", __func__, out_embd.size(), out->n_embd); + return 1; + } + std::copy(out_embd.begin(), out_embd.end(), out->embd); + + return 0; +} + +int32_t mtmd_gen_audio(mtmd_context * ctx, const struct mtmd_gen_inp * inp, struct mtmd_gen_out * out) { + try { + return mtmd_gen_audio_impl(ctx, inp, out); + } catch (const std::exception & e) { + LOG_ERR("%s: error: %s\n", __func__, e.what()); + return 1; + } +} + mtmd_batch * mtmd_batch_init(mtmd_context * ctx) { return new mtmd_batch(ctx); }