diff --git a/tools/server/README.md b/tools/server/README.md index 9f4835eecc..16579ae8a2 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -796,7 +796,7 @@ Returns raw audio bytes (`audio/wav` by default) rather than JSON. For more info `lang`: Language code for the utterance (model-dependent, e.g. `en`, `zh`). Optional. -`speaker_ref_b64`: Base64-encoded reference audio to clone the speaker's voice. Optional. Alternatively, upload the reference audio as a `speaker_ref` file field via `multipart/form-data` (see example below) — if both are provided, the uploaded file takes precedence. +`speaker_ref_b64`: Base64-encoded reference audio to clone the speaker's voice. Optional. Alternatively, upload the reference audio as a `speaker_ref` file field via `multipart/form-data` (see example below) - if both are provided, the uploaded file takes precedence. `top_k`, `top_p`: Sampling params for the acoustic code predictor. Optional, model-dependent defaults apply. diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 7a2f222c6b..881d767b96 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -5246,8 +5246,8 @@ void server_routes::init_routes() { json body = json::parse(req.body); - // multipart form fields arrive as strings, correct the types of the typed params - for (const char * k : { "top_k", "n_predict" }) { + // multipart form fields arrive as strings, correct the types of the numeric params + for (const char * k : { "top_k", "n_predict", "max_tokens" }) { if (body.contains(k) && body[k].is_string()) { body[k] = std::stoi(body[k].get()); } @@ -5257,12 +5257,6 @@ void server_routes::init_routes() { body[k] = std::stof(body[k].get()); } } - if (body.contains("seed") && body["seed"].is_string()) { - body["seed"] = (int64_t) std::stoll(body["seed"].get()); - } - if (body.contains("stream") && body["stream"].is_string()) { - body["stream"] = body["stream"] == "true"; - } std::string prompt = json_value(body, "input", json_value(body, "prompt", std::string())); if (prompt.empty()) { @@ -5271,25 +5265,19 @@ void server_routes::init_routes() { } const std::string response_format = json_value(body, "response_format", std::string("wav")); - const bool stream = json_value(body, "stream", false); server_task task(SERVER_TASK_TYPE_TTS); + server_schema::eval_tts_schema(params, task, body); + const bool stream = task.params.stream; task.tts_inp.set_prompt(prompt); task.tts_inp.set_lang(json_value(body, "lang", std::string())); - task.tts_inp.data.top_k = json_value(body, "top_k", 0); - task.tts_inp.data.top_p = json_value(body, "top_p", 0.0f); task.tts_inp.data.stream = stream; task.tts_inp.data.out_type = response_format == "pcm" ? MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM : MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; - task.params.stream = stream; - task.params.n_predict = json_value(body, "n_predict", -1); - task.params.sampling = params.sampling; // baseline defaults, then apply overrides below - task.params.sampling.penalty_repeat = json_value(body, "repeat_penalty", 1.05f); // note: -1 is clamped to 0 by the sampler, it will disable the penalty - task.params.sampling.penalty_last_n = task.params.n_predict > 0 ? task.params.n_predict : 512; - task.params.sampling.seed = json_value(body, "seed", params.sampling.seed); - task.tts_inp.data.seed = task.params.sampling.seed; // same seed for the codec/vocoder RNG + task.params.sampling.penalty_last_n = task.params.n_predict > 0 ? task.params.n_predict : 512; + task.params.sampling.seed = task.tts_inp.data.seed; // same seed for the codec/vocoder RNG if (task.tts_inp.data.top_k > 0) { task.params.sampling.top_k = task.tts_inp.data.top_k; } diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cpp index 5d7fa6ae6e..2b3965b5c5 100644 --- a/tools/server/server-schema.cpp +++ b/tools/server/server-schema.cpp @@ -566,6 +566,58 @@ task_params eval_llama_cmpl_schema( return params; } +// +// TTS schema +// + +std::vector> make_tts_schema(server_task & task) { + std::vector> fields; + auto add = [&](field * f) { + fields.emplace_back(f); + }; + + add((new field_num("top_k", task.tts_inp.data.top_k)) + ->set_limits(0, INT32_MAX) + ->set_desc("Top-k for the acoustic code predictor, 0 to use the model default")); + + add((new field_num("top_p", task.tts_inp.data.top_p)) + ->set_limits(0.0f, 1.0f) + ->set_desc("Top-p for the acoustic code predictor, 0.0 to use the model default")); + + add((new field_num("seed", task.tts_inp.data.seed)) + ->set_desc("RNG seed for the backbone sampler and the codec/vocoder (-1 = random)")); + + add((new field_num("n_predict", task.params.n_predict)) + ->set_hard_limits(-1, INT32_MAX) + ->add_alias("max_tokens") + ->set_desc("Max number of audio frames to generate")); + + add((new field_num("repeat_penalty", task.params.sampling.penalty_repeat)) + ->set_desc("Repetition penalty applied to the backbone sampler over the whole generation")); + + add((new field_bool("stream", task.params.stream)) + ->set_desc("Stream the audio as it becomes available instead of waiting for the full generation")); + + return fields; +} + +void eval_tts_schema(const common_params & params_base, server_task & task, const json & data) { + // baseline defaults, individual requests can override them + // TODO @ngxson : change the default values based on model + task.params.sampling = params_base.sampling; + task.params.sampling.penalty_repeat = 1.05f; + task.params.n_predict = -1; + task.tts_inp.data.top_k = 0; + task.tts_inp.data.top_p = 0.0f; + task.tts_inp.data.seed = params_base.sampling.seed; + + field_eval_context ctx(task.params); + + for (const auto & f : make_tts_schema(task)) { + f->eval(ctx, data); + } +} + // // eval() implementations // diff --git a/tools/server/server-schema.h b/tools/server/server-schema.h index d0a81431bc..542136a975 100644 --- a/tools/server/server-schema.h +++ b/tools/server/server-schema.h @@ -101,4 +101,12 @@ task_params eval_llama_cmpl_schema( const std::vector & logit_bias_eog, const json & data); +std::vector> make_tts_schema(server_task & task); + +// evaluates the /tts request params into task.params and task.tts_inp.data +void eval_tts_schema( + const common_params & params_base, + server_task & task, + const json & data); + } // namespace server_schema