Commit 097cf33d9 for llama.cpp
commit 097cf33d9499b1eade67d51e16d7c75427e7b267
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Mon Oct 12 13:39:55 2026 +0200
model: support vllm decision 2 model (#30378)
* models: support pplx-decider
* vllm d2
* update
* clarify naming rule
* stricter validation
* fix ci
* rm redundant filter_tensors
diff --git a/common/common.cpp b/common/common.cpp
index 9a71a33ff..9d2578424 100644
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -1170,8 +1170,10 @@ static const std::map<common_decision_type, std::string> COMMON_DECISION_TYPE_NA
{ COMMON_DECISION_TYPE_LAYA, "laya" },
{ COMMON_DECISION_TYPE_CLEF, "clef" },
{ COMMON_DECISION_TYPE_PPLX_DECIDER, "pplx-decider" },
+ { COMMON_DECISION_TYPE_VLLM_DECISION2, "vllm-decision2" },
{ COMMON_DECISION_TYPE_LFM2_D1, "lfm2-d1" },
{ COMMON_DECISION_TYPE_LFM2_D1_OMNI, "lfm2-d1-omni" },
+ // for future models, ALWAYS add org name prefix to avoid name collisions, e.g. "orgname-modelname"
};
static common_decision_type common_decision_type_from_string(const std::string & str) {
@@ -1289,7 +1291,7 @@ common_init_result::common_init_result(common_params & params, bool model_only)
// TODO: maybe improve this in the future
const auto decision_type = common_get_decision_type(model);
if (decision_type == COMMON_DECISION_TYPE_LAYA || decision_type == COMMON_DECISION_TYPE_KEV || decision_type == COMMON_DECISION_TYPE_CLEF ||
- decision_type == COMMON_DECISION_TYPE_LFM2_D1_OMNI) {
+ decision_type == COMMON_DECISION_TYPE_LFM2_D1_OMNI || decision_type == COMMON_DECISION_TYPE_VLLM_DECISION2) {
params.embedding = true;
params.pooling_type = LLAMA_POOLING_TYPE_NONE;
diff --git a/common/common.h b/common/common.h
index d89f803af..9513a04fa 100644
--- a/common/common.h
+++ b/common/common.h
@@ -963,6 +963,7 @@ enum common_decision_type {
COMMON_DECISION_TYPE_LAYA, // score of one marker token per option, read from the embeddings output
COMMON_DECISION_TYPE_CLEF, // all questions in one prompt, score of option i read from the embeddings output at row i
COMMON_DECISION_TYPE_PPLX_DECIDER, // same as openjev, label codes of 1 or 2 letters
+ COMMON_DECISION_TYPE_VLLM_DECISION2, // score of each option read from the embeddings output, the options of a question are marked in the batch
COMMON_DECISION_TYPE_LFM2_D1, // same as openjev, the labels depend on the question type
COMMON_DECISION_TYPE_LFM2_D1_OMNI, // same as laya, other prompt layout
COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported
diff --git a/conversion/__init__.py b/conversion/__init__.py
index 2a62dbac9..c2a8fb92c 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -291,6 +291,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"UltravoxModel": "ultravox",
"UnlimitedOCRForCausalLM": "deepseek",
"VLlama3ForCausalLM": "llama",
+ "VllmDecision2Model": "vllm_decision2",
"VoxtralForConditionalGeneration": "llama",
"WavTokenizerDec": "wavtokenizer",
"XLMRobertaForSequenceClassification": "bert",
diff --git a/conversion/vllm_decision2.py b/conversion/vllm_decision2.py
new file mode 100644
index 000000000..3e7085f14
--- /dev/null
+++ b/conversion/vllm_decision2.py
@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import json
+
+from pathlib import Path
+from typing import Any, Callable, Iterator, TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from torch import Tensor
+
+from .base import ModelBase, gguf, logger
+from .qwen import Qwen3Model
+
+
+def _is_vllm_decision2_checkpoint(dir_model: Path) -> bool:
+ if not (dir_model / "config.json").is_file():
+ return False
+ with open(dir_model / "config.json", encoding="utf-8") as f:
+ return json.load(f).get("decision_format") == "vllm-sr-decision"
+
+
+@ModelBase.register_hparams_loader(_is_vllm_decision2_checkpoint)
+def _load_vllm_decision2_hparams(dir_model: Path) -> dict[str, Any]:
+ logger.info("gguf: detected vllm-sr Decision 2.0 checkpoint")
+ with open(dir_model / "config.json", encoding="utf-8") as f:
+ package = json.load(f)
+ hparams = ModelBase.load_hparams((dir_model / package["backbone"]["config"]).parent, False, guess=False)
+ hparams["architectures"] = ["VllmDecision2Model"]
+ with open(dir_model / package["model_config"], encoding="utf-8") as f:
+ hparams["decision"] = json.load(f)
+ with open(dir_model / "score_bias.json", encoding="utf-8") as f:
+ hparams["decision"]["score_bias"] = json.load(f)["offsets"]
+ return hparams
+
+
+@ModelBase.register("VllmDecision2Model")
+@ModelBase.example("vllm-sr/Decision-2.0-Kai-0.6B")
+class VllmDecision2Model(Qwen3Model):
+ model_arch = gguf.MODEL_ARCH.VLLM_DECISION2
+
+ # torch.nn.LayerNorm default, used by the head
+ _HEAD_NORM_EPS = 1e-5
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ head = self.hparams["decision"]
+ if head["architecture"] != "qwen3-text-endpoints-global-query-shared-bilinear-mlp" or head["head_variant"] != "shared":
+ raise ValueError(f"unsupported decision head: {head['architecture']}")
+
+ def index_tensors(self, *args, **kwargs) -> dict[str, Callable[[], Tensor]]:
+ # the backbone weights are in their own directory
+ dir_model, self.dir_model = self.dir_model, self.dir_model / "backbone"
+ try:
+ return super().index_tensors(*args, **kwargs)
+ finally:
+ self.dir_model = dir_model
+
+ def get_tensors(self) -> Iterator[tuple[str, Tensor]]:
+ yield from super().get_tensors()
+ from safetensors.torch import load_file
+ for name, data in load_file(self.dir_model / "decision_head.safetensors").items():
+ yield "decision_head." + name, data
+
+ def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
+ # the head is computed in F32
+ if new_name.startswith("decision."):
+ return gguf.GGMLQuantizationType.F32
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+ def set_vocab(self):
+ super().set_vocab()
+ self.gguf_writer.add_chat_template([{"name": "systemone", "template": self._systemone_template()}])
+
+ @classmethod
+ def _systemone_template(cls) -> str:
+ # prompt follows decision2/_vendor/dev2model/decision_model.py (segments) of the model repo
+ # the pieces of the prompt are tokenized one by one, the server gives the text that separates them (sep)
+ # and the text that starts the span of a question or of an option (mark_question, mark_option)
+ # the head reads the last token of each option and the last token of the question
+ # the start of an option is a piece of its own, so that the spans of two options are not joined
+ # the keys of JSON objects are given in sorted order
+ def text(value: str) -> str:
+ return "{{ " + json.dumps(value) + " }}"
+
+ def render(name: str) -> str:
+ return "{{ " + name + " if " + name + " is string else " + name + " | tojson(separators=[',', ':']) }}"
+
+ option = (
+ "{% set d = o.description %}"
+ "{% if q.type == 'noul' and d is none %}{% set d = 'Yes' if o.key == 'true' else 'No' %}{% endif %}"
+ "{{ {'description': d, 'key': o.key} | tojson(separators=[',', ':']) }}"
+ )
+ return (
+ "{% for q in questions %}"
+ + text("Context:\n") + render("state")
+ + text("\n\nTask type: ") + "{{ q.type }}" + text("\nQuestion:\n") + render("q.instructions") + text("\nOptions:")
+ + "{% for o in q.options %}{{ sep }}" + text("\n<option>\n") + "{{ sep }}{{ mark_option }}" + option + text("\n</option>") + "{% endfor %}"
+ + "{{ sep }}{{ mark_question }}" + text("\n\nSelect the single option best supported by the context and instructions.\nDecision:")
+ + "{% endfor %}"
+ )
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ self.gguf_writer.add_decision_type(gguf.DecisionType.VLLM_DECISION2)
+ self.gguf_writer.add_layer_norm_eps(self._HEAD_NORM_EPS)
+ for offsets in self.hparams["decision"]["score_bias"].values():
+ self.gguf_writer.add_decision_score_bias(offsets)
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 5adb40205..a569afc96 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -332,6 +332,7 @@ class Keys:
HEAD_COUNT = "{arch}.decision.head_count"
MAX_HEAD_TOKENS = "{arch}.decision.max_head_tokens"
TEMPERATURE = "{arch}.decision.temperature.{name}" # name: "<type>" or "<type>.<n_opt bucket>"
+ SCORE_BIAS = "{arch}.decision.score_bias.{n}.{i}" # added to the logit of level i of a score question with n levels
class Tokenizer:
MODEL = "tokenizer.ggml.model"
@@ -559,6 +560,7 @@ class MODEL_ARCH(IntEnum):
QWEN35 = auto()
QWEN35MOE = auto()
CLEF = auto()
+ VLLM_DECISION2 = auto()
QWEN4EXP = auto()
PHI2 = auto()
PHI3 = auto()
@@ -923,6 +925,11 @@ class MODEL_TENSOR(IntEnum):
DECISION_SCALES = auto()
DECISION_SCORER = auto()
DECISION_SCORER_OUT = auto()
+ DECISION_QUERY_NORM = auto()
+ DECISION_KEY = auto()
+ DECISION_QUERY = auto()
+ DECISION_OPTION_MLP = auto()
+ DECISION_QUERY_MLP = auto()
CONV1D = auto()
CONVNEXT_DW = auto()
CONVNEXT_NORM = auto()
@@ -1343,6 +1350,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
MODEL_ARCH.QWEN35: "qwen35",
MODEL_ARCH.QWEN35MOE: "qwen35moe",
MODEL_ARCH.CLEF: "clef",
+ MODEL_ARCH.VLLM_DECISION2: "vllm-decision2",
MODEL_ARCH.QWEN4EXP: "qwen4exp",
MODEL_ARCH.PHI2: "phi2",
MODEL_ARCH.PHI3: "phi3",
@@ -1706,6 +1714,11 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.DECISION_SCALES: "decision.scales",
MODEL_TENSOR.DECISION_SCORER: "decision.scorer",
MODEL_TENSOR.DECISION_SCORER_OUT: "decision.scorer_out",
+ MODEL_TENSOR.DECISION_QUERY_NORM: "decision.query_norm",
+ MODEL_TENSOR.DECISION_KEY: "decision.key",
+ MODEL_TENSOR.DECISION_QUERY: "decision.query",
+ MODEL_TENSOR.DECISION_OPTION_MLP: "decision.option_mlp",
+ MODEL_TENSOR.DECISION_QUERY_MLP: "decision.query_mlp",
MODEL_TENSOR.CONV1D: "conv1d",
MODEL_TENSOR.CONVNEXT_DW: "convnext.{bid}.dw",
MODEL_TENSOR.CONVNEXT_NORM: "convnext.{bid}.norm",
@@ -3034,6 +3047,29 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.DECISION_SCORER,
MODEL_TENSOR.DECISION_SCORER_OUT,
],
+ MODEL_ARCH.VLLM_DECISION2: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ # decision head
+ MODEL_TENSOR.DECISION_OPTION_NORM,
+ MODEL_TENSOR.DECISION_QUERY_NORM,
+ MODEL_TENSOR.DECISION_KEY,
+ MODEL_TENSOR.DECISION_QUERY,
+ MODEL_TENSOR.DECISION_OPTION_MLP,
+ MODEL_TENSOR.DECISION_QUERY_MLP,
+ MODEL_TENSOR.DECISION_SCORER_OUT,
+ ],
MODEL_ARCH.QWEN35MOE: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT_NORM,
@@ -6100,6 +6136,7 @@ class DecisionType:
NIMBLE = "nimble" # same as openjev, the prompt lists all the questions of the request
CLEF = "clef" # joint head over all questions, one score per option
PPLX_DECIDER = "pplx-decider" # same as openjev, label codes of 1 or 2 letters
+ VLLM_DECISION2 = "vllm-decision2" # head over the end token of each option and the last token, one score per option
LFM2_D1 = "lfm2-d1" # same as openjev, the labels depend on the question type
LFM2_D1_OMNI = "lfm2-d1-omni" # same head as laya on a bidirectional LFM2 trunk, other prompt layout
diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py
index 86d4d6e2f..cda59a628 100644
--- a/gguf-py/gguf/gguf_writer.py
+++ b/gguf-py/gguf/gguf_writer.py
@@ -1365,6 +1365,10 @@ class GGUFWriter:
def add_decision_temperature(self, name: str, value: float) -> None:
self.add_float32(Keys.Decision.TEMPERATURE.format(arch=self.arch, name=name), value)
+ def add_decision_score_bias(self, values: Sequence[float]) -> None:
+ for i, value in enumerate(values):
+ self.add_float32(Keys.Decision.SCORE_BIAS.format(arch=self.arch, n=len(values), i=i), value)
+
# for vision models
def add_clip_has_vision_encoder(self, value: bool) -> None:
diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py
index 566b7348a..6445784a3 100644
--- a/gguf-py/gguf/tensor_mapping.py
+++ b/gguf-py/gguf/tensor_mapping.py
@@ -1328,7 +1328,8 @@ class TensorNameMap:
),
MODEL_TENSOR.DECISION_OPTION_NORM: (
- "joint_head.option_norm", # clef
+ "joint_head.option_norm", # clef
+ "decision_head.candidate_norm", # vllm-decision2
),
MODEL_TENSOR.DECISION_SCORER: (
@@ -1337,6 +1338,27 @@ class TensorNameMap:
MODEL_TENSOR.DECISION_SCORER_OUT: (
"joint_head.residual_scorer.3", # clef
+ "decision_head.scalar", # vllm-decision2
+ ),
+
+ MODEL_TENSOR.DECISION_QUERY_NORM: (
+ "decision_head.query_norm", # vllm-decision2
+ ),
+
+ MODEL_TENSOR.DECISION_KEY: (
+ "decision_head.key", # vllm-decision2
+ ),
+
+ MODEL_TENSOR.DECISION_QUERY: (
+ "decision_head.query", # vllm-decision2
+ ),
+
+ MODEL_TENSOR.DECISION_OPTION_MLP: (
+ "decision_head.candidate_mlp", # vllm-decision2
+ ),
+
+ MODEL_TENSOR.DECISION_QUERY_MLP: (
+ "decision_head.query_mlp", # vllm-decision2
),
MODEL_TENSOR.ENC_ATTN_NORM: (
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index 88eab7b4d..0dc4cc961 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -41,6 +41,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
{ LLM_ARCH_QWEN35, "qwen35" },
{ LLM_ARCH_QWEN35MOE, "qwen35moe" },
{ LLM_ARCH_CLEF, "clef" },
+ { LLM_ARCH_VLLM_DECISION2, "vllm-decision2" },
{ LLM_ARCH_QWEN4EXP, "qwen4exp" },
{ LLM_ARCH_PHI2, "phi2" },
{ LLM_ARCH_PHI3, "phi3" },
@@ -524,6 +525,11 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
{ LLM_TENSOR_DECISION_SCALES, "decision.scales" },
{ LLM_TENSOR_DECISION_SCORER, "decision.scorer" },
{ LLM_TENSOR_DECISION_SCORER_OUT, "decision.scorer_out" },
+ { LLM_TENSOR_DECISION_QUERY_NORM, "decision.query_norm" },
+ { LLM_TENSOR_DECISION_KEY, "decision.key" },
+ { LLM_TENSOR_DECISION_QUERY, "decision.query" },
+ { LLM_TENSOR_DECISION_OPTION_MLP, "decision.option_mlp" },
+ { LLM_TENSOR_DECISION_QUERY_MLP, "decision.query_mlp" },
{ LLM_TENSOR_ENC_OUTPUT_NORM, "enc.output_norm" },
{ LLM_TENSOR_FFN_GATE_INP_SHEXP, "blk.%d.ffn_gate_inp_shexp" },
{ LLM_TENSOR_SSM_A_NOSCAN, "blk.%d.ssm_a" },
@@ -789,6 +795,11 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
{LLM_TENSOR_DECISION_SCALES, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
{LLM_TENSOR_DECISION_SCORER, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_DECISION_SCORER_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_DECISION_QUERY_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
+ {LLM_TENSOR_DECISION_KEY, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_DECISION_QUERY, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_DECISION_OPTION_MLP, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_DECISION_QUERY_MLP, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_DENSE_2_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, // Dense layer output
{LLM_TENSOR_DENSE_3_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, // Dense layer output
{LLM_TENSOR_OUTPUT_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
diff --git a/src/llama-arch.h b/src/llama-arch.h
index 2d8ad6b07..8f8cc27ed 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -46,6 +46,7 @@ enum llm_arch {
LLM_ARCH_QWEN35,
LLM_ARCH_QWEN35MOE,
LLM_ARCH_CLEF,
+ LLM_ARCH_VLLM_DECISION2,
LLM_ARCH_QWEN4EXP,
LLM_ARCH_PHI2,
LLM_ARCH_PHI3,
@@ -695,6 +696,11 @@ enum llm_tensor {
LLM_TENSOR_DECISION_SCALES,
LLM_TENSOR_DECISION_SCORER,
LLM_TENSOR_DECISION_SCORER_OUT,
+ LLM_TENSOR_DECISION_QUERY_NORM,
+ LLM_TENSOR_DECISION_KEY,
+ LLM_TENSOR_DECISION_QUERY,
+ LLM_TENSOR_DECISION_OPTION_MLP,
+ LLM_TENSOR_DECISION_QUERY_MLP,
LLM_TENSOR_CONV1D,
LLM_TENSOR_CONVNEXT_DW,
LLM_TENSOR_CONVNEXT_NORM,
diff --git a/src/llama-ext.h b/src/llama-ext.h
index fcc78f47e..faf4beb69 100644
--- a/src/llama-ext.h
+++ b/src/llama-ext.h
@@ -100,7 +100,7 @@ LLAMA_API void llama_set_embeddings_nextn(struct llama_context * ctx, bool value
// chain multiple trained NextN heads. Default 0 (first head).
LLAMA_API void llama_set_nextn_layer_offset(struct llama_context * ctx, int32_t offset);
-// Marks the entries that a joint decision head (clef) reads, the default is 0
+// Marks the entries that a decision head (clef, vllm-decision2) reads, the default is 0
// See https://github.com/ggml-org/llama.cpp/pull/29831 for details
// A run of entries with the same value is one span, spans must be separated by entries with value 0
// An option belongs to the last question before it
diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp
index 1c7cbd900..47e36d374 100644
--- a/src/llama-model-saver.cpp
+++ b/src/llama-model-saver.cpp
@@ -21,6 +21,7 @@ bool llama_model_saver_supports_arch(llm_arch arch) {
case LLM_ARCH_APERTUS:
case LLM_ARCH_STEP35:
case LLM_ARCH_CLEF: // the head tensors are not saved
+ case LLM_ARCH_VLLM_DECISION2:
return false;
default:
return true;
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index b3e88322b..6872589fa 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -330,6 +330,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_qwen35moe(params);
case LLM_ARCH_CLEF:
return new llama_model_clef(params);
+ case LLM_ARCH_VLLM_DECISION2:
+ return new llama_model_vllm_decision2(params);
case LLM_ARCH_QWEN4EXP:
return new llama_model_qwen4exp(params);
case LLM_ARCH_MISTRAL3:
@@ -3534,6 +3536,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_DREAM:
case LLM_ARCH_QWEN2MOE:
case LLM_ARCH_QWEN3:
+ case LLM_ARCH_VLLM_DECISION2:
case LLM_ARCH_QWEN3MOE:
case LLM_ARCH_LLADA_MOE:
case LLM_ARCH_RND1:
diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp
index 821b4798c..9c6066796 100644
--- a/src/llama-quant.cpp
+++ b/src/llama-quant.cpp
@@ -346,6 +346,11 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param
quantize &= name.find("attn_v_b.weight") == std::string::npos;
}
+ // the decision head of vllm-decision2 is small and computed in F32
+ if (arch == LLM_ARCH_VLLM_DECISION2) {
+ quantize &= name.compare(0, 9, "decision.") != 0;
+ }
+
// do not quantize RWKV's small yet 2D weights
quantize &= name.find("time_mix_first.weight") == std::string::npos;
quantize &= name.find("time_mix_w0.weight") == std::string::npos;
diff --git a/src/models/models.h b/src/models/models.h
index d6ddf9d16..a55983cfb 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -583,6 +583,44 @@ struct llama_model_qwen3 : public llama_model_base {
struct graph : public llm_graph_context {
graph(const llama_model & model, const llm_graph_params & params);
+ protected:
+ // the helpers alone, a derived graph builds its own body
+ struct no_build {};
+ graph(const llm_graph_params & params, no_build) : llm_graph_context(params) {}
+
+ // the layers and the final norm: [n_embd, n_outputs]
+ ggml_tensor * build_backbone(const llama_model & model);
+ };
+
+ std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
+// Qwen3 with the decision head of vllm-sr Decision 2.0: one score per option
+struct llama_model_vllm_decision2 : public llama_model_qwen3 {
+ llama_model_vllm_decision2(const struct llama_model_params & params) : llama_model_qwen3(params) {}
+ void load_arch_hparams(llama_model_loader & ml) override;
+ void load_arch_tensors(llama_model_loader & ml) override;
+
+ ggml_tensor * option_norm = nullptr;
+ ggml_tensor * option_norm_b = nullptr;
+ ggml_tensor * query_norm = nullptr;
+ ggml_tensor * query_norm_b = nullptr;
+ ggml_tensor * key = nullptr;
+ ggml_tensor * query = nullptr;
+ ggml_tensor * option_mlp = nullptr;
+ ggml_tensor * option_mlp_b = nullptr;
+ ggml_tensor * query_mlp = nullptr;
+ ggml_tensor * scorer_out = nullptr;
+
+ class input_decision;
+
+ struct graph : public llama_model_qwen3::graph {
+ graph(const llama_model & model, const llm_graph_params & params);
+ private:
+ // hidden: [n_embd, n_outputs], returns one score per option: [1, n_options]
+ ggml_tensor * build_head(ggml_tensor * hidden, input_decision * inp);
+
+ const llama_model_vllm_decision2 & model;
};
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
diff --git a/src/models/qwen3.cpp b/src/models/qwen3.cpp
index f4b2a2aeb..a565e3e68 100644
--- a/src/models/qwen3.cpp
+++ b/src/models/qwen3.cpp
@@ -51,6 +51,18 @@ std::unique_ptr<llm_graph_context> llama_model_qwen3::build_arch_graph(const llm
}
llama_model_qwen3::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
+ ggml_tensor * cur = build_backbone(model);
+
+ // lm_head
+ cur = build_lora_mm(model.output, cur, model.output_s);
+
+ cb(cur, "result_output", -1);
+ res->t_logits = cur;
+
+ ggml_build_forward_expand(gf, cur);
+}
+
+ggml_tensor * llama_model_qwen3::graph::build_backbone(const llama_model & model) {
const int64_t n_embd_head = hparams.n_embd_head_v();
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
@@ -149,11 +161,5 @@ llama_model_qwen3::graph::graph(const llama_model & model, const llm_graph_param
cb(cur, "result_norm", -1);
res->t_embd = cur;
- // lm_head
- cur = build_lora_mm(model.output, cur, model.output_s);
-
- cb(cur, "result_output", -1);
- res->t_logits = cur;
-
- ggml_build_forward_expand(gf, cur);
+ return cur;
}
diff --git a/src/models/vllm-decision2.cpp b/src/models/vllm-decision2.cpp
new file mode 100644
index 000000000..bbac808de
--- /dev/null
+++ b/src/models/vllm-decision2.cpp
@@ -0,0 +1,179 @@
+#include "models.h"
+
+#include "llama-ext.h"
+
+#include <cmath>
+
+void llama_model_vllm_decision2::load_arch_hparams(llama_model_loader & ml) {
+ llama_model_qwen3::load_arch_hparams(ml);
+
+ // used by the head
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
+
+ // the output is one score per token, see llama_batch_ext_set_decision_order()
+ hparams.n_embd_out_impl = 1;
+}
+
+void llama_model_vllm_decision2::load_arch_tensors(llama_model_loader & ml) {
+ llama_model_qwen3::load_arch_tensors(ml);
+
+ LLAMA_LOAD_LOCALS;
+
+ const auto * w_key = ml.get_weight(tn(LLM_TENSOR_DECISION_KEY, "weight").str().c_str());
+ if (w_key == nullptr) {
+ throw std::runtime_error("the decision head is missing");
+ }
+ const int64_t n_embd_h = w_key->tensor->ne[1];
+
+ option_norm = create_tensor(tn(LLM_TENSOR_DECISION_OPTION_NORM, "weight"), {n_embd}, 0);
+ option_norm_b = create_tensor(tn(LLM_TENSOR_DECISION_OPTION_NORM, "bias"), {n_embd}, 0);
+ query_norm = create_tensor(tn(LLM_TENSOR_DECISION_QUERY_NORM, "weight"), {n_embd}, 0);
+ query_norm_b = create_tensor(tn(LLM_TENSOR_DECISION_QUERY_NORM, "bias"), {n_embd}, 0);
+
+ key = create_tensor(tn(LLM_TENSOR_DECISION_KEY, "weight"), {n_embd, n_embd_h}, 0);
+ query = create_tensor(tn(LLM_TENSOR_DECISION_QUERY, "weight"), {n_embd, n_embd_h}, 0);
+ option_mlp = create_tensor(tn(LLM_TENSOR_DECISION_OPTION_MLP, "weight"), {n_embd, n_embd_h}, 0);
+ option_mlp_b = create_tensor(tn(LLM_TENSOR_DECISION_OPTION_MLP, "bias"), {n_embd_h}, 0);
+ query_mlp = create_tensor(tn(LLM_TENSOR_DECISION_QUERY_MLP, "weight"), {n_embd, n_embd_h}, 0);
+ scorer_out = create_tensor(tn(LLM_TENSOR_DECISION_SCORER_OUT, "weight"), {n_embd_h, 1}, 0);
+}
+
+std::unique_ptr<llm_graph_context> llama_model_vllm_decision2::build_arch_graph(const llm_graph_params & params) const {
+ return std::make_unique<graph>(*this, params);
+}
+
+// output rows read by the head: the last token of each option span and the last token of the question span
+struct decision2_rows {
+ std::vector<int32_t> options;
+ int32_t query = 0;
+ bool valid = false;
+};
+
+// see llama_batch_ext_set_decision_order()
+// if the batch has no usable order, returns one option at row 0
+static decision2_rows decision2_get_rows(const llama_ubatch & ubatch) {
+ decision2_rows res;
+
+ const int32_t n_tokens = ubatch.n_tokens;
+ bool ok = ubatch.decision_order != nullptr && ubatch.n_seqs_unq == 1;
+
+ // output row of each token, -1 if the token has no output
+ std::vector<int32_t> rows(n_tokens, -1);
+ for (int32_t i = 0, n = 0; i < n_tokens; i++) {
+ if (ubatch.output[i]) {
+ rows[i] = n++;
+ }
+ }
+
+ int32_t n_questions = 0;
+ for (int32_t i = 0; ok && i < n_tokens;) {
+ const int32_t order = ubatch.decision_order[i];
+ int32_t end = i + 1;
+ while (end < n_tokens && ubatch.decision_order[end] == order) {
+ end++;
+ }
+ switch (order) {
+ case LLAMA_DECISION_ORDER_NONE:
+ break;
+ case LLAMA_DECISION_ORDER_QUESTION_NOUL:
+ case LLAMA_DECISION_ORDER_QUESTION_CHOICE:
+ case LLAMA_DECISION_ORDER_QUESTION_SCORE:
+ res.query = rows[end - 1];
+ n_questions++;
+ break;
+ case LLAMA_DECISION_ORDER_OPTION:
+ res.options.push_back(rows[end - 1]);
+ break;
+ default:
+ ok = false;
+ }
+ ok = ok && (order == LLAMA_DECISION_ORDER_NONE || rows[end - 1] >= 0);
+ i = end;
+ }
+
+ res.valid = ok && n_questions == 1 && !res.options.empty();
+ if (!res.valid) {
+ res.options.assign(1, 0);
+ res.query = 0;
+ }
+ return res;
+}
+
+class llama_model_vllm_decision2::input_decision : public llm_graph_input_i {
+public:
+ input_decision(const llama_ubatch & ubatch) : n_options(decision2_get_rows(ubatch).options.size()) {}
+
+ void set_input(const llama_ubatch * ubatch) override {
+ const auto rows = decision2_get_rows(*ubatch);
+ GGML_ASSERT(rows.options.size() == n_options);
+
+ // the scores are NaN if the batch has a decision order that cannot be used
+ const float status_data = rows.valid || ubatch->decision_order == nullptr ? 0.0f : NAN;
+
+ ggml_backend_tensor_set(options, rows.options.data(), 0, ggml_nbytes(options));
+ ggml_backend_tensor_set(query, &rows.query, 0, ggml_nbytes(query));
+ ggml_backend_tensor_set(status, &status_data, 0, ggml_nbytes(status));
+ }
+
+ bool can_reuse(const llm_graph_params & params) override {
+ // the values are computed again in set_input(), only the shapes must match
+ return decision2_get_rows(params.ubatch).options.size() == n_options;
+ }
+
+ ggml_tensor * options = nullptr; // I32 [n_options]
+ ggml_tensor * query = nullptr; // I32 [1]
+ ggml_tensor * status = nullptr; // F32 [1], added to the scores: 0, or NaN on invalid input
+
+ const size_t n_options;
+};
+
+llama_model_vllm_decision2::graph::graph(const llama_model & model_base, const llm_graph_params & params) :
+ llama_model_qwen3::graph(params, no_build()), model(static_cast<const llama_model_vllm_decision2 &>(model_base)) {
+ auto inp_decision_ptr = std::make_unique<input_decision>(ubatch);
+ auto * inp_decision = inp_decision_ptr.get();
+ res->add_input(std::move(inp_decision_ptr));
+
+ ggml_tensor * cur = build_backbone(model);
+
+ // the head is always evaluated, so that the graph has the same nodes for every batch
+ cur = build_head(cur, inp_decision);
+
+ // row i of the output is the score of option i
+ cur = ggml_pad(ctx0, cur, 0, n_outputs - cur->ne[1], 0, 0);
+ cur = ggml_add(ctx0, cur, inp_decision->status);
+ cb(cur, "result_decision", -1);
+
+ res->t_embd = cur;
+
+ ggml_build_forward_expand(gf, cur);
+}
+
+ggml_tensor * llama_model_vllm_decision2::graph::build_head(ggml_tensor * hidden, input_decision * inp) {
+ inp->options = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, inp->n_options);
+ inp->query = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1);
+ inp->status = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1);
+ for (ggml_tensor * t : { inp->options, inp->query, inp->status }) {
+ ggml_set_input(t);
+ }
+
+ ggml_tensor * options = ggml_get_rows(ctx0, hidden, inp->options); // [n_embd, n_options]
+ ggml_tensor * query = ggml_get_rows(ctx0, hidden, inp->query); // [n_embd, 1]
+
+ options = build_norm(options, model.option_norm, model.option_norm_b, LLM_NORM, -1);
+ query = build_norm(query, model.query_norm, model.query_norm_b, LLM_NORM, -1);
+
+ // scaled dot product of the option key and the query
+ ggml_tensor * k = ggml_mul_mat(ctx0, model.key, options); // [n_embd_h, n_options]
+ ggml_tensor * q = ggml_mul_mat(ctx0, model.query, query); // [n_embd_h, 1]
+ ggml_tensor * bilinear = ggml_sum_rows(ctx0, ggml_mul(ctx0, k, q));
+ bilinear = ggml_scale(ctx0, bilinear, 1.0f / sqrtf((float) k->ne[0])); // [1, n_options]
+
+ ggml_tensor * mlp = ggml_add(ctx0, ggml_mul_mat(ctx0, model.option_mlp, options), model.option_mlp_b);
+ mlp = ggml_add(ctx0, mlp, ggml_mul_mat(ctx0, model.query_mlp, query));
+ mlp = ggml_gelu_erf(ctx0, mlp);
+ ggml_tensor * nonlinear = ggml_mul_mat(ctx0, model.scorer_out, mlp); // [1, n_options]
+
+ ggml_tensor * scores = ggml_add(ctx0, bilinear, nonlinear);
+ cb(scores, "decision_scores", -1);
+ return scores;
+}
diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp
index 78128ebf0..1e8e915b1 100644
--- a/tests/test-llama-archs.cpp
+++ b/tests/test-llama-archs.cpp
@@ -757,7 +757,7 @@ static bool arch_supported(const llm_arch arch) {
if (arch == LLM_ARCH_PLM) {
return false; // TODO tensor shapes
}
- if (arch == LLM_ARCH_CLEF) {
+ if (arch == LLM_ARCH_CLEF || arch == LLM_ARCH_VLLM_DECISION2) {
return false; // TODO decision head tensors
}
if (arch == LLM_ARCH_DEEPSEEK2OCR) {
diff --git a/tools/server/README.md b/tools/server/README.md
index 0c6650412..c7cf50d5a 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -1718,9 +1718,9 @@ Follows the [TypeSafe API](https://docs.typesafe.ai/api), streaming is not suppo
The questions of a request are answered independently, an answer does not depend on the other questions. The exception is clef: it reads all the questions in one prompt and decides them jointly.
-The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for laya, clef, pplx-decider, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with.
+The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for laya, clef, pplx-decider, vllm-decision2, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with.
-For laya, clef and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. A server that runs clef only serves this endpoint, text generation is not available.
+For laya, clef and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. A server that runs clef only serves this endpoint, text generation is not available.
*Image input:*
diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp
index 5f77d43d0..9f43e7a2c 100644
--- a/tools/server/server-common.cpp
+++ b/tools/server/server-common.cpp
@@ -156,6 +156,7 @@ std::vector<std::string> server_model_output_modalities(common_decision_type dec
case COMMON_DECISION_TYPE_LAYA:
case COMMON_DECISION_TYPE_CLEF:
case COMMON_DECISION_TYPE_PPLX_DECIDER:
+ case COMMON_DECISION_TYPE_VLLM_DECISION2:
case COMMON_DECISION_TYPE_LFM2_D1:
case COMMON_DECISION_TYPE_LFM2_D1_OMNI:
return {"decisions"};
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 2a25b75d8..3014f52cf 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -3604,6 +3604,11 @@ private:
n_past = std::min(n_past, slot.task->n_tokens_shared);
}
+ // the outputs of a decision are read from the batch, so its first position must be evaluated
+ if (n_decision_first >= 0) {
+ n_past = std::min(n_past, n_decision_first);
+ }
+
// if there is an alora invoked, don't cache after the invocation start
if (slot.alora_invocation_start > 0) {
SLT_DBG(slot, "only caching to alora invocation start (n_past = %d, alora_invocation_start = %d)\n", n_past, slot.alora_invocation_start);
diff --git a/tools/server/server-decision.cpp b/tools/server/server-decision.cpp
index 11ad66a19..6b7d2c16e 100644
--- a/tools/server/server-decision.cpp
+++ b/tools/server/server-decision.cpp
@@ -66,6 +66,28 @@ void server_decision_context::init(const llama_model * model) {
temperatures[key + prefix_temp.size()] = temp;
}
+ // "<n_levels>.<level>" -> offset
+ const std::string prefix_bias = prefix + "score_bias.";
+ for (int32_t i = 0; i < llama_model_meta_count(model); i++) {
+ char key[256];
+ char val[64];
+ if (llama_model_meta_key_by_index(model, i, key, sizeof(key)) < 0 || !string_starts_with(key, prefix_bias)) {
+ continue;
+ }
+ if (llama_model_meta_val_str_by_index(model, i, val, sizeof(val)) < 0) {
+ continue;
+ }
+ const auto parts = string_split<std::string>(key + prefix_bias.size(), '.');
+ const size_t n = parts.size() == 2 ? std::strtoul(parts[0].c_str(), nullptr, 10) : 0;
+ const size_t level = parts.size() == 2 ? std::strtoul(parts[1].c_str(), nullptr, 10) : 0;
+ if (n < 2 || n > 255 || level >= n) {
+ throw std::runtime_error(string_format("invalid decision score bias: %s", key));
+ }
+ auto & bias = score_bias[n];
+ bias.resize(n, 0.0f);
+ bias[level] = std::strtof(val, nullptr);
+ }
+
if (model_type == COMMON_DECISION_TYPE_OPENJEV) {
// one letter per option, each must be a single token
const std::string letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
@@ -123,6 +145,10 @@ void server_decision_context::init(const llama_model * model) {
n_options_max = 255;
noul_true_first = true;
choice_sorted = true;
+ } else if (model_type == COMMON_DECISION_TYPE_VLLM_DECISION2) {
+ n_options_max = 255;
+ noul_criteria_order = true;
+ strict_payload = true;
} else if (model_type == COMMON_DECISION_TYPE_LFM2_D1) {
n_options_max = 255;
noul_true_first = true;
@@ -144,11 +170,18 @@ void server_decision_context::init(const llama_model * model) {
// request parsing
//
+static bool decision_is_payload(const json & val) {
+ return val.is_string() || val.is_object() || val.is_array();
+}
+
std::vector<server_decision_question> server_decision_context::parse_questions(const json & body) const {
// lfm2-d1 and lfm2-d1-omni accept a null state, for example to ask about images only
if (!body.contains("state") || (body.at("state").is_null() && type != COMMON_DECISION_TYPE_LFM2_D1 && type != COMMON_DECISION_TYPE_LFM2_D1_OMNI)) {
throw std::invalid_argument("\"state\" must be provided");
}
+ if (strict_payload && !decision_is_payload(body.at("state"))) {
+ throw std::invalid_argument("\"state\" must be text, an object or an array");
+ }
if (!body.contains("questions") || !body.at("questions").is_object() || body.at("questions").empty()) {
throw std::invalid_argument("\"questions\" must be a non-empty object");
}
@@ -164,6 +197,9 @@ std::vector<server_decision_question> server_decision_context::parse_questions(c
if (!q.contains("instructions") || q.at("instructions").is_null()) {
throw err("\"instructions\" must be provided");
}
+ if (strict_payload && (!decision_is_payload(q.at("instructions")) || q.at("instructions") == "")) {
+ throw err("\"instructions\" must be non-empty text, an object or an array");
+ }
server_decision_question question;
question.id = id;
@@ -178,8 +214,14 @@ std::vector<server_decision_question> server_decision_context::parse_questions(c
throw err("\"criteria\" must be a non-empty object");
}
for (const auto & [key, description] : criteria.items()) {
+ if (strict_payload && (key.empty() || !(description.is_null() || decision_is_payload(description)))) {
+ throw err("option keys must be non-empty, descriptions must be null, text, an object or an array");
+ }
question.options.push_back({key, description});
}
+ if (strict_payload && question.options.size() < 2) {
+ throw err("\"criteria\" must have at least 2 options");
+ }
if (choice_sorted) {
std::sort(question.options.begin(), question.options.end(), [](const auto & a, const auto & b) {
return a.key < b.key;
@@ -191,6 +233,9 @@ std::vector<server_decision_question> server_decision_context::parse_questions(c
throw err("\"criteria\" must be an array of 2 to 10 levels");
}
for (size_t i = 0; i < criteria.size(); i++) {
+ if (strict_payload && !decision_is_payload(criteria.at(i))) {
+ throw err("level descriptions must be text, an object or an array");
+ }
question.options.push_back({std::to_string(i), criteria.at(i)});
}
} else if (type_name == "noul") {
@@ -198,6 +243,13 @@ std::vector<server_decision_question> server_decision_context::parse_questions(c
if (!criteria.is_null() && !criteria.is_object()) {
throw err("\"criteria\" must be an object");
}
+ if (strict_payload && criteria.is_object()) {
+ for (const auto & [key, description] : criteria.items()) {
+ if ((key != "false" && key != "true") || !decision_is_payload(description)) {
+ throw err("\"criteria\" may only describe \"false\" and \"true\" with text, an object or an array");
+ }
+ }
+ }
for (const char * key : {"false", "true"}) {
json description;
if (criteria.is_object() && criteria.contains(key)) {
@@ -209,7 +261,7 @@ std::vector<server_decision_question> server_decision_context::parse_questions(c
}
question.options.push_back({key, description});
}
- if (noul_true_first) {
+ if (noul_true_first || (noul_criteria_order && criteria.is_object() && criteria.size() == 2 && criteria.begin().key() == "true")) {
std::swap(question.options[0], question.options[1]);
}
} else {
@@ -658,6 +710,11 @@ void server_decision_context::fill_task(
mtmd_context * mctx,
const mtmd_helper_init_opt & init_opt,
server_task & task) const {
+ // each question has its own prompt, with the spans that the head reads
+ if (type == COMMON_DECISION_TYPE_VLLM_DECISION2) {
+ fill_task_joint(state, { question }, files, mctx, init_opt, task);
+ return;
+ }
if (type == COMMON_DECISION_TYPE_LFM2_D1_OMNI) {
fill_task_d1omni(state, questions, question, variant, files, mctx, init_opt, task);
return;
@@ -1056,6 +1113,32 @@ static double decision_confidence_score(const std::vector<double> & probs) {
return std::max(0.0, 1.0 - dist / dist_uniform);
}
+// VLLM_DECISION2: normalized entropy, as in the reference implementation
+static double decision_confidence_entropy(const std::vector<double> & probs) {
+ if (probs.size() < 2) {
+ return 1.0;
+ }
+ double entropy = 0.0;
+ for (const double p : probs) {
+ if (p > 0.0) {
+ entropy -= p * std::log(p);
+ }
+ }
+ return std::max(0.0, std::min(1.0, 1.0 - entropy / std::log((double) probs.size())));
+}
+
+double server_decision_context::get_decision_confidence(server_decision_question_type question_type, const std::vector<double> & probs) const {
+ if (type == COMMON_DECISION_TYPE_VLLM_DECISION2) {
+ return decision_confidence_entropy(probs);
+ }
+ switch (question_type) {
+ case SERVER_DECISION_QUESTION_CHOICE: return decision_confidence_choice(probs);
+ case SERVER_DECISION_QUESTION_SCORE: return decision_confidence_score(probs);
+ case SERVER_DECISION_QUESTION_NOUL: break;
+ }
+ throw std::runtime_error("no confidence for this question type");
+}
+
json server_decision_context::format_answer(const server_decision_question & question, const std::vector<std::vector<float>> & scores, bool has_media) const {
const size_t n = n_outputs(question);
if (scores.size() != n_variants(question)) {
@@ -1075,11 +1158,17 @@ json server_decision_context::format_answer(const server_decision_question & que
if (std::any_of(s.begin(), s.end(), [](float v) { return std::isnan(v); })) {
throw std::runtime_error("the model could not evaluate the decision");
}
- const float score_max = *std::max_element(s.begin(), s.end());
+ std::vector<double> logits(s.begin(), s.end());
+ if (question.type == SERVER_DECISION_QUESTION_SCORE && score_bias.count(n) > 0) {
+ for (size_t i = 0; i < n; i++) {
+ logits[i] += score_bias.at(n)[i];
+ }
+ }
+ const double logit_max = *std::max_element(logits.begin(), logits.end());
std::vector<double> p(n);
double sum = 0.0;
for (size_t i = 0; i < n; i++) {
- p[i] = std::exp((double) (s[i] - score_max) / temperature);
+ p[i] = std::exp((logits[i] - logit_max) / temperature);
sum += p[i];
}
for (size_t i = 0; i < n; i++) {
@@ -1116,18 +1205,20 @@ json server_decision_context::format_answer(const server_decision_question & que
const size_t best = std::max_element(probs.begin(), probs.end()) - probs.begin();
answer["choice"] = question.options[best].key;
answer["probabilities"] = probabilities;
- answer["confidence"] = decision_confidence_choice(probs);
+ answer["confidence"] = get_decision_confidence(question.type, probs);
} else {
double expected = 0.0;
json legend = json::object();
for (size_t i = 0; i < n; i++) {
expected += i * probs[i];
- legend[question.options[i].key] = question.options[i].description;
+ const json & description = question.options[i].description;
+ // vllm-decision2 returns structured descriptions as canonical JSON text
+ legend[question.options[i].key] = strict_payload && !description.is_string() ? json(decision_sort_keys(description).dump()) : description;
}
answer["score"] = expected;
answer["legend"] = legend;
answer["probabilities"] = probabilities;
- answer["confidence"] = decision_confidence_score(probs);
+ answer["confidence"] = get_decision_confidence(question.type, probs);
}
return answer;
}
diff --git a/tools/server/server-decision.h b/tools/server/server-decision.h
index 5c7e44f37..3e81098cb 100644
--- a/tools/server/server-decision.h
+++ b/tools/server/server-decision.h
@@ -43,6 +43,7 @@ struct server_decision_context {
case COMMON_DECISION_TYPE_KEV:
case COMMON_DECISION_TYPE_NIMBLE:
case COMMON_DECISION_TYPE_PPLX_DECIDER:
+ case COMMON_DECISION_TYPE_VLLM_DECISION2:
case COMMON_DECISION_TYPE_LFM2_D1:
return true;
default:
@@ -110,9 +111,12 @@ private:
std::shared_ptr<const common_chat_template> tmpl; // the "systemone" template
std::map<std::string, float> temperatures; // "<type>" or "<type>.<n_options bucket>"
+ std::map<size_t, std::vector<float>> score_bias; // added to the scores of a score question, by number of levels
size_t n_options_max = 0;
bool noul_true_first = false; // noul options are [true, false] instead of [false, true]
+ bool noul_criteria_order = false; // noul options are in the order of the criteria keys, if given
bool choice_sorted = false; // choice options are in the order of their keys
+ bool strict_payload = false; // state, instructions and descriptions must be text, an object or an array
// OPENJEV, LEV, NIMBLE, PPLX_DECIDER
std::vector<llama_token> labels;
@@ -148,6 +152,7 @@ private:
server_task & task) const;
float get_temperature(const server_decision_question & question) const;
+ double get_decision_confidence(server_decision_question_type type, const std::vector<double> & probs) const;
};
// group the tasks so that the common prefix of their prompts is evaluated only once
diff --git a/tools/server/server-task.h b/tools/server/server-task.h
index b8d6a3b67..50926ec3c 100644
--- a/tools/server/server-task.h
+++ b/tools/server/server-task.h
@@ -191,6 +191,11 @@ struct server_task {
for (const int32_t marker : markers) {
pos = pos < 0 ? marker : std::min(pos, marker);
}
+ for (size_t i = 0; i < order.size(); i++) {
+ if (order[i] != 0) {
+ return pos < 0 ? (int32_t) i : std::min(pos, (int32_t) i);
+ }
+ }
return pos;
}