Commit bafa0420c for llama.cpp
commit bafa0420c1e57edf04d71587d32fe57d45df7b95
Author: Tarek Dakhran <tarek@liquid.ai>
Date: Wed Nov 18 20:15:43 2026 +0100
server : enable packing for d1-2B and d1-Omni-2B models (#32553)
* llama-bench : size the batch to the prompt for models without memory
Assisted-by: Claude Opus 5.5
* server : read up to 16 lfm3-d1 questions in one pass
Assisted-by: Claude Opus 5.5
diff --git a/common/common.cpp b/common/common.cpp
index 657dcdbf5..b009d460d 100644
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -1313,6 +1313,11 @@ common_init_result::common_init_result(common_params & params, bool model_only)
LOG_INF("%s", "decision model reads the embeddings output, enabling embedding mode\n");
}
+ // 0 is n_batch
+ if ((decision_type == COMMON_DECISION_TYPE_LFM3_D1 || decision_type == COMMON_DECISION_TYPE_LFM3_D1_OMNI) && cparams.n_outputs_max != 0) {
+ cparams.n_outputs_max = std::max(cparams.n_outputs_max, (uint32_t) COMMON_DECISION_LFM3_D1_SET_SIZE);
+ }
+
if (llama_model_classifier_free_guidance(model)) {
params.kv_unified = true;
cparams.kv_unified = true;
@@ -2334,6 +2339,9 @@ llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) {
if (t.decision_order != 0) {
llama_batch_ext_set_decision_order(res, idx, (llama_decision_order) t.decision_order);
}
+ if (t.decision_group != 0 && !llama_batch_ext_set_decision_group(res, idx, t.decision_group)) {
+ GGML_ABORT("%s: failed to set the decision group of the entry at index %d\n", __func__, i);
+ }
}
return res;
diff --git a/common/common.h b/common/common.h
index 9b9e892d8..e16bee7c1 100644
--- a/common/common.h
+++ b/common/common.h
@@ -1002,6 +1002,9 @@ enum common_decision_type {
COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported
};
+// lfm3-d1: max questions packed in one prompt
+#define COMMON_DECISION_LFM3_D1_SET_SIZE 16
+
common_decision_type common_get_decision_type(const struct llama_model * model);
// metadata of a GGUF file, read without loading the model
@@ -1117,6 +1120,7 @@ struct common_batch {
llama_embd state; // non-owning view of the data passed to set_embd_state(), data == NULL if none
std::vector<llama_seq_id> seq_ids_extra; // see add_seq()
int32_t decision_order = 0; // see llama_batch_ext_set_decision_order()
+ int32_t decision_group = 0; // see llama_batch_ext_set_decision_group()
int32_t embd_row = -1; // see add_embd_row(), -1 if none
int32_t state_row = -1; // see set_embd_state_row(), -1 if none
};
diff --git a/src/llama-batch.cpp b/src/llama-batch.cpp
index 7fce6624f..ca3e308f4 100644
--- a/src/llama-batch.cpp
+++ b/src/llama-batch.cpp
@@ -259,6 +259,10 @@ bool llama_batch_allocr::init(
decision_order.resize(n_tok, 0);
decision_order[i] = batch_inp.tokens[i].decision_order;
}
+ if (batch_inp.tokens[i].decision_group != 0) {
+ decision_group.resize(n_tok, 0);
+ decision_group[i] = batch_inp.tokens[i].decision_group;
+ }
}
//
@@ -353,6 +357,7 @@ bool llama_batch_allocr::init(
/*.output =*/ batch.logits,
/*.type =*/ is_embd_vec.empty() ? nullptr : is_embd_vec.data(),
/*.decision_order =*/ decision_order.empty() ? nullptr : decision_order.data(),
+ /*.decision_group =*/ decision_group.empty() ? nullptr : decision_group.data(),
/*.data =*/ {},
};
@@ -579,6 +584,7 @@ llama_ubatch llama_batch_allocr::ubatch_reserve(uint32_t n_seq_tokens, uint32_t
/*.output =*/ udata->output.data(),
/*.type =*/ nullptr,
/*.decision_order =*/ nullptr,
+ /*.decision_group =*/ nullptr,
/*.data =*/ std::move(udata),
};
@@ -887,6 +893,7 @@ void llama_batch_allocr::clear() {
seq_id_unq .clear();
output .clear();
decision_order.clear();
+ decision_group.clear();
for (auto & cur : seq_pos) {
cur.clear();
@@ -943,6 +950,7 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
udata->output .resize(n_tokens);
udata->type .resize(mixed ? n_tokens : 0);
udata->decision_order.resize(decision_order.empty() ? 0 : n_tokens);
+ udata->decision_group.resize(decision_group.empty() ? 0 : n_tokens);
udata->batch_idxs = idxs;
udata->seq_id_data.reserve(n_tokens);
@@ -984,6 +992,9 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
if (!decision_order.empty()) {
udata->decision_order[i] = decision_order[idxs[i]];
}
+ if (!decision_group.empty()) {
+ udata->decision_group[i] = decision_group[idxs[i]];
+ }
for (int s = 0; s < udata->n_seq_id[i]; ++s) {
const llama_seq_id seq_id = batch.seq_id[idxs[i]][s];
@@ -1030,6 +1041,7 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
/*.output =*/ udata->output.data(),
/*.type =*/ mixed ? udata->type.data() : nullptr,
/*.decision_order =*/ udata->decision_order.empty() ? nullptr : udata->decision_order.data(),
+ /*.decision_group =*/ udata->decision_group.empty() ? nullptr : udata->decision_group.data(),
/*.data =*/ std::move(udata),
};
@@ -1430,6 +1442,14 @@ bool llama_batch_ext::set_decision_order(int32_t idx, int32_t order) {
return true;
}
+bool llama_batch_ext::set_decision_group(int32_t idx, int32_t group) {
+ if (idx < 0 || idx >= (int32_t) tokens.size() || group < 0) {
+ return false;
+ }
+ tokens[idx].decision_group = group;
+ return true;
+}
+
// llama_batch_ext C API
llama_batch_ext * llama_batch_ext_init(llama_context * ctx) {
@@ -1513,6 +1533,10 @@ bool llama_batch_ext_set_decision_order(llama_batch_ext * batch, int32_t idx, ll
return batch->set_decision_order(idx, order);
}
+bool llama_batch_ext_set_decision_group(llama_batch_ext * batch, int32_t idx, int32_t group) {
+ return batch->set_decision_group(idx, group);
+}
+
// llama_batch_compat
void llama_batch_compat::init(llama_batch_ext & dst, const llama_batch & batch_inp, size_t n_embd_row) {
diff --git a/src/llama-batch.h b/src/llama-batch.h
index d16662898..e6f4be6a2 100644
--- a/src/llama-batch.h
+++ b/src/llama-batch.h
@@ -61,6 +61,7 @@ struct llama_ubatch {
int8_t * output; // [n_tokens] | i | -
int8_t * type; // [n_tokens] | i | - (mixed ubatch only, 0 - token, 1 - embd)
int32_t * decision_order; // [n_tokens], NULL if no entry has one, see llama_batch_ext_set_decision_order()
+ int32_t * decision_group; // [n_tokens], NULL if no entry has one, see llama_batch_ext_set_decision_group()
struct data_t {
std::vector<llama_token> token;
@@ -77,6 +78,7 @@ struct llama_ubatch {
std::vector<int8_t> type;
std::vector<int32_t> batch_idxs; // original batch index for each token
std::vector<int32_t> decision_order;
+ std::vector<int32_t> decision_group;
std::vector<llama_seq_id> seq_id_data;
};
@@ -121,6 +123,7 @@ struct llama_batch_ext {
int32_t state_row = -1; // row of ctx_state_src's h_nextn device buffer used as state, -1 if none
bool output = false; // TODO: have dedicated output flags
int32_t decision_order = 0; // see llama_batch_ext_set_decision_order()
+ int32_t decision_group = 0; // see llama_batch_ext_set_decision_group()
std::unordered_set<llama_seq_id> seq_ids;
std::array<llama_pos, GGML_MROPE_SECTIONS> pos = {0, 0, 0, 0};
};
@@ -155,6 +158,7 @@ struct llama_batch_ext {
bool set_token_pos(int32_t idx, const llama_pos * pos_in);
bool set_output(int32_t idx, bool output_last);
bool set_decision_order(int32_t idx, int32_t order);
+ bool set_decision_group(int32_t idx, int32_t group);
};
// a helper for sanitizing, fulfilling and splitting a batch
@@ -241,6 +245,7 @@ private:
std::vector<int32_t> seq_idx;
std::vector<int8_t> output;
std::vector<int32_t> decision_order; // empty if no entry has one
+ std::vector<int32_t> decision_group; // empty if no entry has one
using pos_set_t = std::set<llama_pos>;
using seq_cpl_t = std::vector<bool>;
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index 1d8f66d0e..6cd39ac7d 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -1934,7 +1934,11 @@ int llama_context::encode(const llama_batch_ext & batch_inp) {
const llama_ubatch ubatch = balloc->split_simple(n_tokens);
// micro-batching is not possible for non-causal encoding, so we process the batch in a single shot
- GGML_ASSERT(cparams.n_ubatch >= n_tokens && "encoder requires n_ubatch >= n_tokens");
+ if (n_tokens > cparams.n_ubatch) {
+ LLAMA_LOG_ERROR("%s: the encoder reads the batch in one pass, n_tokens = %u > n_ubatch = %u, increase n_ubatch (-ub)\n",
+ __func__, n_tokens, cparams.n_ubatch);
+ return -1;
+ }
// TODO: this clear of the buffer can easily be forgotten - need something better
// sync first so any in-flight async copies into embd_seq complete before it is freed
diff --git a/src/llama-ext.h b/src/llama-ext.h
index 3ac617b0f..2b2256fb6 100644
--- a/src/llama-ext.h
+++ b/src/llama-ext.h
@@ -137,6 +137,10 @@ enum llama_decision_order {
// The embeddings output has one value per entry: row i is the score of option i
LLAMA_API bool llama_batch_ext_set_decision_order(struct llama_batch_ext * batch, int32_t idx, enum llama_decision_order order);
+// Questions packed after a shared state in one sequence, contexts without memory only
+// 0 (default) is the state, k > 0 the k-th question; full attention layers hide other questions from a question
+LLAMA_API bool llama_batch_ext_set_decision_group(struct llama_batch_ext * batch, int32_t idx, int32_t group);
+
// Label of option i of a decision model ("<arch>.decision.labels"), nullptr past the last one
LLAMA_API const char * llama_model_decision_label(const struct llama_model * model, uint32_t i);
diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp
index ab604034e..7b01af1b9 100644
--- a/src/llama-graph.cpp
+++ b/src/llama-graph.cpp
@@ -608,25 +608,32 @@ void llm_graph_input_attn_no_cache::set_input(const llama_ubatch * ubatch) {
const int64_t n_kv = self_attn_mask.raw->ne[0];
const int64_t n_tokens = ubatch->n_tokens;
- const auto fill_mask = [&](auto * data, int64_t ne, int n_swa, llama_swa_type swa_type, bool causal) {
+ const auto fill_mask = [&](auto * data, int64_t ne, int n_swa, llama_swa_type swa_type, bool causal, const int32_t * group) {
using T = std::remove_reference_t<decltype(*data)>;
std::fill(data, data + ne, llama_cast<T>(-INFINITY));
for (int i1 = 0; i1 < n_tokens; ++i1) {
const llama_seq_id s1 = ubatch->seq_id[i1][0];
const llama_pos p1 = ubatch->pos[i1];
+ const int32_t g1 = group ? group[i1] : 0;
const uint64_t idst = i1*n_kv;
for (int i0 = 0; i0 < n_tokens; ++i0) {
const llama_seq_id s0 = ubatch->seq_id[i0][0];
const llama_pos p0 = ubatch->pos[i0];
+ const int32_t g0 = group ? group[i0] : 0;
// mask different sequences
if (s0 != s1) {
continue;
}
+ // mask other questions, see llama_batch_ext_set_decision_group()
+ if (g0 != 0 && g1 != 0 && g0 != g1) {
+ continue;
+ }
+
// mask future tokens
if (causal && p0 > p1) {
continue;
@@ -653,18 +660,18 @@ void llm_graph_input_attn_no_cache::set_input(const llama_ubatch * ubatch) {
GGML_ASSERT(self_attn_mask.raw);
GGML_ASSERT(ggml_backend_buffer_is_host(self_attn_mask.raw->buffer));
if (self_attn_mask.raw->type == GGML_TYPE_F16) {
- fill_mask((ggml_fp16_t *) self_attn_mask.raw->data, ggml_nelements(self_attn_mask.raw), 0, LLAMA_SWA_TYPE_NONE, causal);
+ fill_mask((ggml_fp16_t *) self_attn_mask.raw->data, ggml_nelements(self_attn_mask.raw), 0, LLAMA_SWA_TYPE_NONE, causal, ubatch->decision_group);
} else {
- fill_mask((float *) self_attn_mask.raw->data, ggml_nelements(self_attn_mask.raw), 0, LLAMA_SWA_TYPE_NONE, causal);
+ fill_mask((float *) self_attn_mask.raw->data, ggml_nelements(self_attn_mask.raw), 0, LLAMA_SWA_TYPE_NONE, causal, ubatch->decision_group);
}
if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) {
GGML_ASSERT(self_attn_mask_swa.raw);
GGML_ASSERT(ggml_backend_buffer_is_host(self_attn_mask_swa.raw->buffer));
if (self_attn_mask_swa.raw->type == GGML_TYPE_F16) {
- fill_mask((ggml_fp16_t *) self_attn_mask_swa.raw->data, ggml_nelements(self_attn_mask_swa.raw), hparams.n_swa, hparams.swa_type, causal_swa);
+ fill_mask((ggml_fp16_t *) self_attn_mask_swa.raw->data, ggml_nelements(self_attn_mask_swa.raw), hparams.n_swa, hparams.swa_type, causal_swa, nullptr);
} else {
- fill_mask((float *) self_attn_mask_swa.raw->data, ggml_nelements(self_attn_mask_swa.raw), hparams.n_swa, hparams.swa_type, causal_swa);
+ fill_mask((float *) self_attn_mask_swa.raw->data, ggml_nelements(self_attn_mask_swa.raw), hparams.n_swa, hparams.swa_type, causal_swa, nullptr);
}
}
}
diff --git a/src/llama-kv-cache-dsv4.cpp b/src/llama-kv-cache-dsv4.cpp
index ef1be94a2..9c0a9b22d 100644
--- a/src/llama-kv-cache-dsv4.cpp
+++ b/src/llama-kv-cache-dsv4.cpp
@@ -169,6 +169,7 @@ static llama_ubatch dsv4_build_raw_write_ubatch(const llama_ubatch & ubatch) {
/*.output =*/ data->output.data(),
/*.type =*/ nullptr,
/*.decision_order =*/ nullptr,
+ /*.decision_group =*/ nullptr,
/*.data =*/ data,
};
diff --git a/tools/llama-bench/llama-bench.cpp b/tools/llama-bench/llama-bench.cpp
index 5cbdda14c..1ddd73ce8 100644
--- a/tools/llama-bench/llama-bench.cpp
+++ b/tools/llama-bench/llama-bench.cpp
@@ -2379,6 +2379,18 @@ int llama_bench(int argc, char ** argv) {
}
llama_context * ctx = llama_init_from_model(lmodel, cparams);
+
+ // a model without memory reads a prompt in one ubatch
+ const int n_tokens_max = std::max(inst.n_prompt, inst.n_depth);
+ const bool resize = ctx != NULL && !llama_get_memory(ctx) && (int) llama_n_ubatch(ctx) < n_tokens_max;
+ if (resize) {
+ fprintf(stderr, "%s: the model has no memory, setting n_batch = n_ubatch = %d\n", __func__, n_tokens_max);
+ llama_free(ctx);
+ cparams.n_batch = n_tokens_max;
+ cparams.n_ubatch = n_tokens_max;
+ ctx = llama_init_from_model(lmodel, cparams);
+ }
+
if (ctx == NULL) {
fprintf(stderr, "%s: error: failed to create context with model '%s'\n", __func__, inst.model.c_str());
llama_model_free(lmodel);
@@ -2386,6 +2398,10 @@ int llama_bench(int argc, char ** argv) {
}
test t(inst, lmodel, ctx);
+ if (resize) {
+ t.n_batch = n_tokens_max;
+ t.n_ubatch = n_tokens_max;
+ }
llama_memory_clear(llama_get_memory(ctx), false);
diff --git a/tools/server/README.md b/tools/server/README.md
index 292fe598e..278518dce 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -1931,11 +1931,11 @@ Follows the [TypeSafe API](https://docs.typesafe.ai/api), streaming is not suppo
- `score`: An array of 2 to 10 level descriptions, lowest level first.
- `noul`: Optional. An object with the descriptions of `true` and `false`.
-The questions of a request are answered independently, an answer does not depend on the other questions. The exception is clef: it reads all the questions in one prompt and decides them jointly.
+The questions of a request are answered independently, an answer does not depend on the other questions. The exceptions are clef, which reads all the questions in one prompt and decides them jointly, and lfm3-d1 and lfm3-d1-omni, which read up to 16 questions after the state in one pass, as the models were trained (requests with a spoken question read one question per pass).
The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for granite-switch, laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1, lfm2-d1-omni and h2o-lightning, 2 for openbmb-minicpm-ocr-jev. For laya, long questions and options are truncated to the token budget the model was trained with.
-For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. An h2o-lightning state longer than 32000 tokens keeps its first and last 16000 tokens. A server that runs clef only serves this endpoint, text generation is not available. A decision model that is also a text model, such as granite-switch, serves both and reports `["text", "decisions"]`.
+For laya, clef, vllm-d3, lfm2-d1-omni, lfm3-d1 and lfm3-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. An h2o-lightning state longer than 32000 tokens keeps its first and last 16000 tokens. A server that runs clef only serves this endpoint, text generation is not available. A decision model that is also a text model, such as granite-switch, serves both and reports `["text", "decisions"]`.
openbmb-minicpm-ocr-jev answers one fixed question: is an OCR text, rendered as an image, consistent with the original image? `state` is the OCR text, as a string, special tokens written in it are escaped. `files` (or `images`) has exactly 2 images: the original, then the rendered one, both are required. The request has one question and no `instructions`: the task is part of the prompt of the model, so a request that sends one is rejected. It is either a `choice` with the `criteria` `{"0": null, "1": null}`, where `1` means consistent, or a `noul` without `criteria`, which returns the probability of `1`. The model has no LM head, so text generation is not available.
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 0cdc02284..0b479fc69 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -159,6 +159,7 @@ struct server_batch {
bool output;
bool is_prompt; // for stats tracking
int32_t decision_order = 0;
+ int32_t decision_group = 0;
int32_t i_embd = -1; // row in embd, -1 if this is a token
};
std::vector<token> tokens;
@@ -239,6 +240,11 @@ struct server_batch {
tokens[idx].decision_order = order;
}
+ void set_decision_group(int32_t idx, int32_t group) {
+ GGML_ASSERT(idx >= 0 && idx < (int32_t)tokens.size());
+ tokens[idx].decision_group = group;
+ }
+
// render the sub-batch [off, off + n_tokens) into view, index i in view is index off + i here
void render(int32_t off, int32_t n_tokens) {
GGML_ASSERT(off >= 0 && off < size());
@@ -253,6 +259,7 @@ struct server_batch {
view.add(t.token, t.pos[0], t.id_slot, t.output);
}
view.tokens.back().decision_order = t.decision_order;
+ view.tokens.back().decision_group = t.decision_group;
}
}
};
@@ -2469,25 +2476,26 @@ private:
}
res->scores.assign(embd, embd + llama_model_n_embd_out(model_tgt));
} else if (!decision.labels.empty()) {
- const float * logits = llama_get_logits_ith(slot.ctx_tgt, i_batch);
- if (logits == nullptr) {
- send_error(slot, "failed to get logits", ERROR_TYPE_SERVER);
- return;
- }
const int32_t n_vocab = llama_vocab_n_tokens(vocab);
- for (const llama_token label : decision.labels) {
- GGML_ASSERT(label >= 0 && label < n_vocab);
- res->scores.push_back(logits[label]);
- }
- if (!decision.label_groups.empty()) {
- std::vector<float> scores;
- size_t i = 0;
- for (const int32_t n : decision.label_groups) {
- GGML_ASSERT(n > 0 && i + n <= res->scores.size());
- scores.push_back(*std::max_element(res->scores.begin() + i, res->scores.begin() + i + n));
- i += n;
+ const size_t n_out = decision.label_groups.empty() ? decision.labels.size() : decision.label_groups.size();
+ GGML_ASSERT(decision.label_pos.empty() || decision.label_pos.size() == n_out);
+ size_t i = 0;
+ for (size_t o = 0; o < n_out; o++) {
+ const int32_t i_out = decision.label_pos.empty() ? i_batch : i_batch - (slot.task->n_tokens() - 1 - decision.label_pos[o]);
+ const float * logits = llama_get_logits_ith(slot.ctx_tgt, i_out);
+ if (logits == nullptr) {
+ send_error(slot, "failed to get logits", ERROR_TYPE_SERVER);
+ return;
}
- res->scores = std::move(scores);
+ const size_t n = decision.label_groups.empty() ? 1 : decision.label_groups[o];
+ GGML_ASSERT(n > 0 && i + n <= decision.labels.size());
+ float score = -INFINITY;
+ for (size_t j = i; j < i + n; j++) {
+ GGML_ASSERT(decision.labels[j] >= 0 && decision.labels[j] < n_vocab);
+ score = std::max(score, logits[decision.labels[j]]);
+ }
+ res->scores.push_back(score);
+ i += n;
}
} else {
// the outputs of this slot in this batch are the last tokens of the prompt
@@ -4430,6 +4438,9 @@ private:
if (!slot.task->decision.order.empty()) {
batch.set_decision_order(batch.size() - 1, slot.task->decision.order[slot.prompt.n_tokens()]);
}
+ if (!slot.task->decision.group.empty()) {
+ batch.set_decision_group(batch.size() - 1, slot.task->decision.group[slot.prompt.n_tokens()]);
+ }
slot.prompt.tokens.push_back(cur_tok);
// break at the last user message, or at user messages at least min step past the last checkpoint
@@ -4483,6 +4494,11 @@ private:
// extract the logits only for the last token
batch.set_output(batch.size() - 1, true);
+ // and at the answer of each packed question
+ for (const int32_t pos : slot.task->decision.label_pos) {
+ batch.set_output(batch.size() - slot.task->n_tokens() + pos, true);
+ }
+
slot.stats.n_gen = 0;
slot.i_batch = batch.size() - 1;
@@ -4555,6 +4571,9 @@ private:
if (!slot.task->decision.order.empty()) {
batch.set_decision_order(batch.size() - 1, slot.task->decision.order[cur_token_idx]);
}
+ if (!slot.task->decision.group.empty()) {
+ batch.set_decision_group(batch.size() - 1, slot.task->decision.group[cur_token_idx]);
+ }
slot.prompt.tokens.push_back(cur_tok);
continue;
}
@@ -6464,6 +6483,7 @@ void server_routes::init_routes() {
const json body = json::parse(req.body);
const auto questions = decision.parse_questions(body);
+ const size_t set_size = decision.set_size(questions);
std::vector<raw_buffer> files;
const json state = decision.prepare_state(decision.parse_state(body, files));
@@ -6472,7 +6492,8 @@ void server_routes::init_routes() {
return res;
}
- // one task per variant of each question, or one task for all the questions
+ // one task per variant of each question, or one task for all the questions, or per packed set
+ const bool is_packed = decision.is_joint() || set_size > 0;
auto & rd = res->rd;
{
std::vector<server_task> tasks;
@@ -6481,6 +6502,14 @@ void server_routes::init_routes() {
task.id = rd.get_new_id();
decision.fill_task_joint(state, questions, files, ctx_server.mctx, ctx_server.init_opt, task);
tasks.push_back(std::move(task));
+ } else if (set_size > 0) {
+ for (size_t i = 0; i < questions.size(); i += set_size) {
+ const std::vector<server_decision_question> set(questions.begin() + i, questions.begin() + std::min(questions.size(), i + set_size));
+ server_task task = server_task(SERVER_TASK_TYPE_DECISION);
+ task.id = rd.get_new_id();
+ decision.fill_task_set(state, set, files, ctx_server.mctx, ctx_server.init_opt, task);
+ tasks.push_back(std::move(task));
+ }
} else {
for (const auto & question : questions) {
for (size_t variant = 0; variant < decision.n_variants(question); variant++) {
@@ -6512,15 +6541,20 @@ void server_routes::init_routes() {
size_t i_score = 0;
for (const auto & question : questions) {
std::vector<std::vector<float>> scores;
- if (decision.is_joint()) {
- // one result with the scores of all the questions, in order
- auto * result = dynamic_cast<server_task_result_decision *>(all_results.results[0].get());
+ if (is_packed) {
+ auto * result = dynamic_cast<server_task_result_decision *>(all_results.results[i_result].get());
GGML_ASSERT(result != nullptr && i_score + question.options.size() <= result->scores.size());
scores.emplace_back(result->scores.begin() + i_score, result->scores.begin() + i_score + question.options.size());
+ if (i_score == 0) {
+ n_tokens += result->n_tokens;
+ }
i_score += question.options.size();
- n_tokens = result->n_tokens;
+ if (i_score == result->scores.size()) {
+ i_result++;
+ i_score = 0;
+ }
}
- for (size_t variant = 0; !decision.is_joint() && variant < decision.n_variants(question); variant++) {
+ for (size_t variant = 0; !is_packed && variant < decision.n_variants(question); variant++) {
auto * result = dynamic_cast<server_task_result_decision *>(all_results.results[i_result++].get());
GGML_ASSERT(result != nullptr);
scores.push_back(result->scores);
diff --git a/tools/server/server-decision.cpp b/tools/server/server-decision.cpp
index a48786c80..c4414aa16 100644
--- a/tools/server/server-decision.cpp
+++ b/tools/server/server-decision.cpp
@@ -1451,6 +1451,72 @@ void server_decision_context::fill_task_d1omni(
//
// given to the template: text between the pieces of the prompt, and at the start of the span of a question or of an option
+// lfm3-d1 packs the questions of a state as d1 attention.py pack_set does
+static const std::string D1_HEAD = "<|startoftext|><|im_start|>user\n";
+static const std::string D1_HEADER = "QUESTION:\n";
+
+size_t server_decision_context::set_size(const std::vector<server_decision_question> & questions) const {
+ if (!is_lfm3_d1()) {
+ return 0;
+ }
+ // TODO: pack spoken questions
+ for (const auto & question : questions) {
+ if (decision_is_spoken(question.instructions)) {
+ return 0;
+ }
+ }
+ return COMMON_DECISION_LFM3_D1_SET_SIZE;
+}
+
+void server_decision_context::fill_task_set(
+ const json & state,
+ const std::vector<server_decision_question> & questions,
+ const std::vector<raw_buffer> & files,
+ mtmd_context * mctx,
+ const mtmd_helper_init_opt & init_opt,
+ server_task & task) const {
+ GGML_ASSERT(!questions.empty());
+
+ std::string head = D1_HEAD;
+ for (size_t i = 0; i < files.size(); i++) {
+ head += get_media_marker();
+ }
+ auto suffix = [&](const server_decision_question & question) {
+ const std::string lone = render(json(), questions, question, 0, files.size());
+ if (!string_starts_with(lone, head)) {
+ throw std::runtime_error("unexpected layout of the decision prompt");
+ }
+ return lone.substr(head.size());
+ };
+
+ // the first question is its lone prompt
+ fill_task(state, questions, questions[0], 0, files, mctx, init_opt, task);
+ const size_t n_first = common_tokenize(vocab, suffix(questions[0]), false, true).size();
+ if (n_first >= task.tokens.size()) {
+ throw std::runtime_error("unexpected layout of the decision prompt");
+ }
+ task.decision.group.assign(task.tokens.size(), 0);
+ std::fill(task.decision.group.end() - n_first, task.decision.group.end(), 1);
+ task.decision.label_pos.assign(task.decision.label_groups.size(), task.tokens.size() - 1);
+
+ for (size_t k = 1; k < questions.size(); k++) {
+ const std::string text = (state.is_null() ? "" : D1_HEADER) + suffix(questions[k]);
+ for (const llama_token token : common_tokenize(vocab, text, false, true)) {
+ task.tokens.push_back(token);
+ task.decision.group.push_back(k + 1);
+ }
+
+ std::vector<std::string> texts;
+ std::vector<llama_tokens> groups;
+ d1_labels(questions[k], texts, groups);
+ for (const auto & group : groups) {
+ task.decision.labels.insert(task.decision.labels.end(), group.begin(), group.end());
+ task.decision.label_groups.push_back(group.size());
+ task.decision.label_pos.push_back(task.tokens.size() - 1);
+ }
+ }
+}
+
static const std::string CLEF_MARKER = "<<clef:";
static const std::string CLEF_SEP = "<<clef:sep>>";
static const std::string CLEF_MARK_QUESTION = "<<clef:question>>";
diff --git a/tools/server/server-decision.h b/tools/server/server-decision.h
index 82af98ca1..8d5f709fc 100644
--- a/tools/server/server-decision.h
+++ b/tools/server/server-decision.h
@@ -124,6 +124,18 @@ struct server_decision_context {
const mtmd_helper_init_opt & init_opt,
server_task & task) const;
+ // lfm3-d1: max questions per prompt, 0 if each question has its own prompt
+ size_t set_size(const std::vector<server_decision_question> & questions) const;
+
+ // set the prompt of these questions packed after their state, the result has the scores of all their options
+ void fill_task_set(
+ const json & state,
+ const std::vector<server_decision_question> & questions,
+ const std::vector<raw_buffer> & files,
+ mtmd_context * mctx,
+ const mtmd_helper_init_opt & init_opt,
+ server_task & task) const;
+
// scores: the raw model outputs of each variant
json format_answer(const server_decision_question & question, const std::vector<std::vector<float>> & scores, bool has_media = false) const;
diff --git a/tools/server/server-task.h b/tools/server/server-task.h
index aa6b779ed..234cd36e1 100644
--- a/tools/server/server-task.h
+++ b/tools/server/server-task.h
@@ -209,6 +209,7 @@ struct server_task {
struct decision {
std::vector<llama_token> labels; // logits of these tokens, at the last prompt token
std::vector<int32_t> label_groups; // if set, number of labels per output, the output is their max
+ std::vector<int32_t> label_pos; // if set, prompt position of each output, else the last token
std::vector<int32_t> markers; // embeddings[column] at these prompt positions
int32_t column = 0;
// if set, embeddings is [q | k], and the output is instead the scaled dot product of q[pointer] and k[marker]
@@ -237,6 +238,9 @@ struct server_task {
// the scores are the first n_scores rows of the embeddings
std::vector<int32_t> order;
int32_t n_scores = 0;
+
+ // one value per prompt token, see llama_batch_ext_set_decision_group()
+ std::vector<int32_t> group;
};
decision decision;