Superminaren/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-11 04:56:56 +02:00
Code Issues Packages Projects Releases Wiki Activity

wire up

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-07-30 00:12:08 +02:00
parent 4712dce74f
commit 2e9e8bbe34
5 changed files with 118 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/CMakeLists.txt
+1
View File
@@ -53,6 +53,7 @@ add_library(mtmd
models/qwen3a.cpp
models/mimo-audio.cpp
models/qwen3tts-spkenc.cpp
models/qwen3tts-gen.cpp
models/step3vl.cpp
models/siglip.cpp
models/whisper-enc.cpp
tools/mtmd/clip.cpp
+16
View File
@@ -1029,6 +1029,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
{
builder = std::make_unique<clip_graph_qwen3tts_spkenc>(ctx, img);
} break;
case PROJECTOR_TYPE_QWEN3TTS_GEN:
{
builder = std::make_unique<clip_graph_qwen3tts_gen>(ctx, img);
} break;
case PROJECTOR_TYPE_YOUTUVL:
{
builder = std::make_unique<clip_graph_youtuvl>(ctx, img);
@@ -3543,6 +3547,8 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params
ctx_gen_audio = new clip_ctx(ctx_params);
loader.load_hparams(ctx_gen_audio->model, CLIP_MODALITY_GEN_AUDIO);
loader.load_tensors(*ctx_gen_audio);
// TODO: fix warmup
ctx_gen_audio->buf_compute_meta.resize(ctx_gen_audio->max_nodes * ggml_tensor_overhead() + ggml_graph_overhead());
}
} catch (const std::exception & e) {
@@ -3878,6 +3884,11 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
// a single speaker embedding vector, regardless of its length
n_patches = 1;
} break;
case PROJECTOR_TYPE_QWEN3TTS_GEN:
{
// one hidden-state vector fed back to the talker per call
n_patches = 1;
} break;
case PROJECTOR_TYPE_GRANITE4_VISION:
{
// Per-tile output token count: each projector block outputs
@@ -4578,6 +4589,11 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
{
// do nothing
} break;
case PROJECTOR_TYPE_QWEN3TTS_GEN:
{
std::vector<int32_t> code0 = { params->code0 };
set_input_i32("inp_code0", code0);
} break;
case PROJECTOR_TYPE_HUNYUANVL:
{
// Compute the HunyuanVL 2D position embedding on CPU (with the
tools/mtmd/models/models.h
+5
View File
@@ -226,6 +226,11 @@ struct clip_graph_qwen3tts_spkenc : clip_graph {
ggml_tensor * attentive_stats_pool(ggml_tensor * x) const;
};
struct clip_graph_qwen3tts_gen : clip_graph {
clip_graph_qwen3tts_gen(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
};
struct clip_graph_kimik25 : clip_graph {
clip_graph_kimik25(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
ggml_cgraph * build() override;
tools/mtmd/models/qwen3tts-gen.cpp
+29
View File
@@ -0,0 +1,29 @@
#include "models.h"
ggml_cgraph * clip_graph_qwen3tts_gen::build() {
ggml_tensor * h_state = build_inp_raw(1);
h_state = ggml_reshape_1d(ctx0, h_state, h_state->ne[0]);
cb(h_state, "inp_h_state", -1);
// inp_code0: sampled codebook-0 id from the talker's codec_head
ggml_tensor * code0_embd; // [n_embd, 1]
{
ggml_tensor * code0 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1);
ggml_set_name(code0, "inp_code0");
ggml_set_input(code0);
ggml_tensor * code0_embd = ggml_get_rows(ctx0, model.gen_code_out_embd_w, code0);
code0_embd = ggml_reshape_1d(ctx0, code0_embd, code0_embd->ne[0]);
cb(code0_embd, "code0_embd", -1);
}
// TODO: code_predictor transformer block (5 layers) + per-codebook
// sampling loop; placeholder combination so the graph has a single
// well-defined output tensor until that lands
ggml_tensor * cur = ggml_add(ctx0, h_state, code0_embd);
cur = ggml_reshape_2d(ctx0, cur, cur->ne[0], 1);
cb(cur, "gen_audio_out", -1);
ggml_build_forward_expand(gf, cur);
return gf;
}
tools/mtmd/mtmd.cpp
+67
View File
@@ -1562,6 +1562,73 @@ float * mtmd_get_output_embd(mtmd_context * ctx) {
return ctx->out_embd.data();
}
//
// audio generation
//
mtmd_gen_audio_type mtmd_gen_audio_get_type(const mtmd_context * ctx) {
if (!ctx->ctx_gen_a) {
return MTMD_GEN_AUDIO_TYPE_NONE;
}
switch (clip_get_projector_type(ctx->ctx_gen_a)) {
case PROJECTOR_TYPE_QWEN3TTS_GEN:
return MTMD_GEN_AUDIO_TYPE_MTP;
default:
return MTMD_GEN_AUDIO_TYPE_NONE;
}
}
static int32_t mtmd_gen_audio_impl(mtmd_context * ctx, const mtmd_gen_inp * inp, mtmd_gen_out * out) {
clip_ctx * ctx_clip = ctx->ctx_gen_a;
if (!ctx_clip) {
LOG_ERR("%s: model does not support audio generation\n", __func__);
return 1;
}
const size_t n_embd = (size_t) clip_n_mmproj_embd(ctx_clip);
if (inp->n_embd != n_embd) {
LOG_ERR("%s: n_embd mismatch: model expects %zu, got %zu\n", __func__, n_embd, inp->n_embd);
return 1;
}
clip_image_f32 hidden_state;
hidden_state.set_size({(int) n_embd, 1}, false, true);
hidden_state.cpy_buf(std::vector<float>(inp->embd, inp->embd + n_embd));
clip_image_f32_batch batch;
batch.is_audio = true;
batch.entries.push_back(std::move(hidden_state));
std::vector<float> out_embd(n_embd);
clip_encode_params params;
params.imgs = &batch;
params.n_threads = ctx->n_threads;
params.out_embd = &out_embd;
params.code0 = inp->code0;
if (!clip_encode(ctx_clip, &params)) {
LOG_ERR("%s: clip_encode failed\n", __func__);
return 1;
}
if (!out->embd || out->n_embd != out_embd.size()) {
LOG_ERR("%s: output buffer size mismatch: expected %zu, got %zu\n", __func__, out_embd.size(), out->n_embd);
return 1;
}
std::copy(out_embd.begin(), out_embd.end(), out->embd);
return 0;
}
int32_t mtmd_gen_audio(mtmd_context * ctx, const struct mtmd_gen_inp * inp, struct mtmd_gen_out * out) {
try {
return mtmd_gen_audio_impl(ctx, inp, out);
} catch (const std::exception & e) {
LOG_ERR("%s: error: %s\n", __func__, e.what());
return 1;
}
}
mtmd_batch * mtmd_batch_init(mtmd_context * ctx) {
return new mtmd_batch(ctx);
}