Commit 2d99adb10 for llama.cpp
commit 2d99adb1083670b5676f01614689453b6d378498
Author: Marco Colombo <ing.marco.colombo@gmail.com>
Date: Mon Nov 9 11:20:45 2026 +0100
mtmd: Pocket TTS - add generic support for temperature set via API (#31655)
* mtmd, server : add temperature to /tts, default Pocket TTS to 0.3
Assisted-by: Claude Opus 5.5
* feat: add temp parameter in llama-tts
* temp: fix default hardcoded values
* fix: remove too narrow test
* fix: addressing PR comments
diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp
index acdee5d12..5e77eccd9 100644
--- a/tools/mtmd/mtmd-helper-gen.cpp
+++ b/tools/mtmd/mtmd-helper-gen.cpp
@@ -521,19 +521,19 @@ private:
// settings that only live in the reference's per-pack yaml, not in the checkpoint
// the english packs share the same shapes and tokenizer, but disagree on these
-// all three are 0 / false when the pack does not tune them, the model default is then used
+// both are 0 / false when the pack does not tune them, the model default is then used
struct pockettts_pack_settings {
- float temp = 0.0f;
- int frames_after_eos = 0;
- bool pad_short_text = false;
+ int frames_after_eos = 0;
+ bool pad_short_text = false;
};
static pockettts_pack_settings pockettts_pack(const char * variant) {
static const std::unordered_map<std::string, pockettts_pack_settings> packs = {
- { "english", { 0.3f, 0, false } },
- { "english_2026-01", { 0.7f, 0, true } },
- { "english_2026-04", { 0.3f, 0, false } },
- { "french_24l", { 0.7f, 8, false } },
+ { "english", { 0, false } },
+ { "english_2026-01", { 0, true } },
+ { "english_2026-04", { 0, false } },
+ { "french_24l", { 8, false } },
+ { "italian", { 0, false } },
};
auto it = packs.find(variant ? variant : "");
if (it == packs.end()) {
@@ -638,6 +638,7 @@ public:
prompt_batch->set_position_normal(0, seq_id);
prompt_pos = 0;
+ temp = inp->temp;
out_type = inp->out_type;
stream = inp->stream;
if (!init_gen_state(inp->seed)) {
@@ -680,8 +681,8 @@ public:
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
inp.embd = const_cast<float *>(h_state_in);
- if (pack.temp > 0.0f) {
- inp.temp = pack.temp;
+ if (temp >= 0.0f) {
+ inp.temp = temp;
}
mtmd_gen_out out{};
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
@@ -1016,6 +1017,7 @@ private:
}
pockettts_pack_settings pack;
+ float temp = -1.0f; // the caller's flow noise scale, < 0 for the model default
bool specials_ok = false;
llama_token bos_before_voice = LLAMA_TOKEN_NULL;
llama_token audio_bos = LLAMA_TOKEN_NULL;
@@ -1306,9 +1308,9 @@ public:
return 1;
}
- // the sampling defaults of the model, the caller's top-k/top-p are not used
+ // the sampling defaults of the model, the caller can only override the temperature
const mtmd_gen_inp def = mtmd_gen_inp_default(mctx);
- temp = def.temp;
+ temp = inp->temp >= 0.0f ? inp->temp : def.temp;
top_k = def.top_k;
rep_penalty = def.penalty_repeat;
rep_window = def.penalty_last_n;
@@ -2459,6 +2461,7 @@ struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void) {
mtmd_helper_gen_audio_inp inp{};
inp.top_k = 50;
inp.top_p = 1.0f;
+ inp.temp = -1.0f; // model default
inp.seed = UINT32_MAX; // random
inp.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
return inp;
diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h
index a9ffdf71e..8a1d3aae1 100644
--- a/tools/mtmd/mtmd-helper.h
+++ b/tools/mtmd/mtmd-helper.h
@@ -248,6 +248,7 @@ struct mtmd_helper_gen_audio_inp {
int32_t top_k;
float top_p;
+ float temp; // audio frame temperature if the model supports it, < 0 = model default, 0 = greedy
uint32_t seed; // UINT32_MAX for random (default: random)
enum mtmd_helper_gen_audio_outtype out_type;
diff --git a/tools/mtmd/mtmd-subcontext.cpp b/tools/mtmd/mtmd-subcontext.cpp
index fa33d5887..7f30d3949 100644
--- a/tools/mtmd/mtmd-subcontext.cpp
+++ b/tools/mtmd/mtmd-subcontext.cpp
@@ -1483,14 +1483,14 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
// positions and mask are derived in-graph from the persisted counter
set_gen_state_in();
} else {
- // flow matching starts from gaussian noise, std = sqrt(temp)
+ // flow matching starts from gaussian noise, std = sqrt(temp), no noise at temp 0
ggml_tensor * t = get_inp_tensor("inp_noise");
- // Config.default_temperature, for a caller that does not set one
- const float temp = params->temp > 0.0f ? params->temp : 0.7f;
- std::normal_distribution<float> dist(0.0f, std::sqrt(temp));
- std::vector<float> noise(ggml_nelements(t));
- for (auto & v : noise) {
- v = dist(params->gen_state->rng);
+ std::vector<float> noise(ggml_nelements(t), 0.0f);
+ if (params->temp > 0.0f) {
+ std::normal_distribution<float> dist(0.0f, std::sqrt(params->temp));
+ for (auto & v : noise) {
+ v = dist(params->gen_state->rng);
+ }
}
set_input_f32("inp_noise", noise);
}
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 6b4fc4d0a..7c6bcc7be 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -2802,7 +2802,7 @@ mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) {
// https://github.com/kyutai-labs/pocket-tts/blob/main/pocket_tts/default_parameters.py
inp.top_k = 50;
inp.top_p = 1.0f;
- inp.temp = 0.7f;
+ inp.temp = 0.3f;
break;
case PROJECTOR_TYPE_SOPRANO:
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h
index 27763563c..e023f2c31 100644
--- a/tools/mtmd/mtmd.h
+++ b/tools/mtmd/mtmd.h
@@ -505,7 +505,7 @@ struct mtmd_gen_inp {
float * embd; // the hidden state from backbone, must have n_text_embd elements
int32_t top_k;
float top_p;
- float temp; // sampling temperature, or noise scale for flow-matching decoders
+ float temp; // sampling temperature, or noise scale for flow-matching decoders (0 = no noise)
float penalty_repeat; // repetition penalty of the sampled codes, <= 1 is off
int32_t penalty_last_n; // frames the repetition penalty looks back on
diff --git a/tools/server/README.md b/tools/server/README.md
index c249e05d4..562e8dd9d 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -862,7 +862,9 @@ Returns raw audio bytes (`audio/wav` by default) rather than JSON. For more info
`speaker_ref_b64`: Base64-encoded reference audio to clone the speaker's voice. Optional. Alternatively, upload the reference audio as a `speaker_ref` file field via `multipart/form-data` (see example below) - if both are provided, the uploaded file takes precedence. Rejected when the model has no voice encoder (e.g. Pocket TTS without voice cloning).
-`top_k`, `top_p`: Sampling params for the acoustic code predictor. Optional, model-dependent defaults apply.
+`top_k`, `top_p`: Sampling params for both the backbone sampler and the acoustic code predictor. Optional, `0` or absent keeps the server `--top-k`/`--top-p` for the backbone and the model defaults for the code predictor.
+
+`temperature`: Temperature for the backbone sampler and the audio frames, for models that support it. `0` is greedy sampling. Optional, absent keeps the server `--temp` for the backbone and the model defaults for the audio.
`repeat_penalty`: Repetition penalty applied to the backbone sampler over the whole generation (default `1.05`). Without this, the backbone can loop and re-generate the same utterance.
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 78c8ef149..407672d5e 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -6485,7 +6485,7 @@ void server_routes::init_routes() {
body[k] = std::stoll(body[k].get<std::string>());
}
}
- for (const char * k : { "top_p", "repeat_penalty" }) {
+ for (const char * k : { "top_p", "temperature", "repeat_penalty" }) {
if (body.contains(k) && body[k].is_string()) {
body[k] = std::stof(body[k].get<std::string>());
}
@@ -6533,6 +6533,9 @@ void server_routes::init_routes() {
if (task.tts_inp.data.top_p > 0) {
task.params.sampling.top_p = task.tts_inp.data.top_p;
}
+ if (task.tts_inp.data.temp >= 0) {
+ task.params.sampling.temp = task.tts_inp.data.temp;
+ }
// speaker reference: either an uploaded form file ("speaker_ref") or a base64 JSON field ("speaker_ref_b64")
const unsigned char * speaker_ref_data = nullptr;
diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cpp
index 0a14a9620..76d634d58 100644
--- a/tools/server/server-schema.cpp
+++ b/tools/server/server-schema.cpp
@@ -542,11 +542,15 @@ std::vector<std::unique_ptr<field>> make_tts_schema(server_task & task) {
add((new field_num("top_k", task.tts_inp.data.top_k))
->set_limits(0, INT32_MAX)
- ->set_desc("Top-k for the acoustic code predictor, 0 to use the model default"));
+ ->set_desc("Top-k for the backbone sampler and the acoustic code predictor, 0 to use the defaults"));
add((new field_num("top_p", task.tts_inp.data.top_p))
->set_limits(0.0f, 1.0f)
- ->set_desc("Top-p for the acoustic code predictor, 0.0 to use the model default"));
+ ->set_desc("Top-p for the backbone sampler and the acoustic code predictor, 0.0 to use the defaults"));
+
+ add((new field_num("temperature", task.tts_inp.data.temp))
+ ->set_limits(0.0f, std::numeric_limits<float>::infinity())
+ ->set_desc("Temperature for the backbone sampler and the audio frames, for models that support it, 0.0 = greedy"));
add((new field_num("seed", task.tts_inp.data.seed))
->set_desc("RNG seed for the backbone sampler and the codec/vocoder (-1 = random)"));
@@ -573,6 +577,7 @@ void eval_tts_schema(const common_params & params_base, server_task & task, cons
task.params.n_predict = -1;
task.tts_inp.data.top_k = 0;
task.tts_inp.data.top_p = 0.0f;
+ task.tts_inp.data.temp = -1.0f; // not given: --temp for the backbone, model default for the audio
task.tts_inp.data.seed = params_base.sampling.seed;
field_eval_context ctx(task.params);
diff --git a/tools/tts/README.md b/tools/tts/README.md
index 66c7af496..39d773d85 100644
--- a/tools/tts/README.md
+++ b/tools/tts/README.md
@@ -15,7 +15,7 @@ llama-tts -hf ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF -p "Hello world" --output o
Note: it's highly recommended to always provide a speaker reference voice (via `--tts-speaker-file`); otherwise, the model's performance may be degraded.
Common params:
-- Sampling params such as `--top-k`, `--top-p`, `--temp`, etc.
+- Sampling params such as `--top-k`, `--top-p`, `--temp`, etc. `--temp` also sets the temperature of the audio frames for models that support it, without it they use the model default
- `-n <number_of_frames>` limits the output length, e.g. `-n 500`. Note that how many milliseconds each frame represents varies by model
- Core inference params such as `-ngl`, `-b`, `-ub`, etc.
diff --git a/tools/tts/tts.cpp b/tools/tts/tts.cpp
index 8578f8869..5f5c2b973 100644
--- a/tools/tts/tts.cpp
+++ b/tools/tts/tts.cpp
@@ -151,6 +151,10 @@ int main(int argc, char ** argv) {
inp.top_k = params.sampling.top_k;
inp.top_p = params.sampling.top_p;
inp.seed = params.sampling.seed;
+ // without --temp the audio frames keep the model default, not the CLI default
+ if (params.sampling.user_sampling_config & common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_TEMP) {
+ inp.temp = params.sampling.temp;
+ }
//
// stage 1: process prompt via backbone model, generate semantic representation