Commit 99b95488c for llama.cpp
commit 99b95488cac0f00ce3f05af113a8c1e287753f87
Author: Xuan-Son Nguyen <son@huggingface.co>
Date: Sat Oct 3 02:50:48 2026 +0200
model: add support for clef decision model (text-only) (#29831)
* init support for clef (text only)
* more static graph
* clean up
* nits
* nits 2
* Update gguf-py/gguf/constants.py
Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
---------
Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
diff --git a/common/common.cpp b/common/common.cpp
index 8e997aad4..0269bc895 100644
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -3,6 +3,9 @@
#include "build-info.h"
#include "common.h"
+
+#include "../src/llama-ext.h"
+
#include "fit.h"
#include "log.h"
#include "llama.h"
@@ -1186,6 +1189,7 @@ static const std::map<common_decision_type, std::string> COMMON_DECISION_TYPE_NA
{ COMMON_DECISION_TYPE_KEV, "kev" },
{ COMMON_DECISION_TYPE_NIMBLE, "nimble" },
{ COMMON_DECISION_TYPE_LAYA, "laya" },
+ { COMMON_DECISION_TYPE_CLEF, "clef" },
};
static common_decision_type common_decision_type_from_string(const std::string & str) {
@@ -1261,10 +1265,10 @@ common_init_result::common_init_result(common_params & params, bool model_only)
const llama_vocab * vocab = llama_model_get_vocab(model);
- // this decision model returns a score for each token via the embeddings output
+ // these decision models return a score for each token via the embeddings output
// TODO: maybe improve this in the future
const auto decision_type = common_get_decision_type(model);
- if (decision_type == COMMON_DECISION_TYPE_LAYA || decision_type == COMMON_DECISION_TYPE_KEV) {
+ if (decision_type == COMMON_DECISION_TYPE_LAYA || decision_type == COMMON_DECISION_TYPE_KEV || decision_type == COMMON_DECISION_TYPE_CLEF) {
params.embedding = true;
params.pooling_type = LLAMA_POOLING_TYPE_NONE;
@@ -2210,6 +2214,9 @@ llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) {
if (t.output) {
llama_batch_ext_set_output_logits(res, idx, true);
}
+ if (t.decision_order != 0) {
+ llama_batch_ext_set_decision_order(res, idx, (llama_decision_order) t.decision_order);
+ }
}
return res;
diff --git a/common/common.h b/common/common.h
index f4b72d90a..27be5c058 100644
--- a/common/common.h
+++ b/common/common.h
@@ -960,6 +960,7 @@ enum common_decision_type {
COMMON_DECISION_TYPE_KEV, // dot product of the hidden states of the last token and of one end token per option
COMMON_DECISION_TYPE_NIMBLE, // same as openjev, the prompt lists all the questions of the request
COMMON_DECISION_TYPE_LAYA, // score of one marker token per option, read from the embeddings output
+ COMMON_DECISION_TYPE_CLEF, // all questions in one prompt, score of option i read from the embeddings output at row i
COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported
};
@@ -1062,6 +1063,7 @@ struct common_batch {
bool output;
llama_embd embd; // non-owning view of the data passed to add_embd()/set_embd(), data == NULL if none
std::vector<llama_seq_id> seq_ids_extra; // see add_seq()
+ int32_t decision_order = 0; // see llama_batch_ext_set_decision_order()
};
std::vector<token> tokens; // mirror of the entries, tokens[i] describes batch index i
diff --git a/conversion/__init__.py b/conversion/__init__.py
index d710f7146..6d8ae9c1a 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -42,6 +42,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"ChameleonForConditionalGeneration": "chameleon",
"ChatGLMForConditionalGeneration": "chatglm",
"ChatGLMModel": "chatglm",
+ "ClefModel": "clef",
"CodeShellForCausalLM": "codeshell",
"CogVLMForCausalLM": "cogvlm",
"Cohere2MoeForCausalLM": "command_r",
@@ -296,6 +297,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
MMPROJ_MODEL_MAP: dict[str, str] = {
"AudioFlamingo3ForConditionalGeneration": "ultravox",
+ "ClefModel": "clef",
"CogVLMForCausalLM": "cogvlm",
"DeepseekOCR2ForCausalLM": "deepseek",
"DeepseekOCRForCausalLM": "deepseek",
diff --git a/conversion/clef.py b/conversion/clef.py
new file mode 100644
index 000000000..a6f724394
--- /dev/null
+++ b/conversion/clef.py
@@ -0,0 +1,150 @@
+from __future__ import annotations
+
+import json
+import math
+
+from pathlib import Path
+from typing import Any, Iterable, Iterator, TYPE_CHECKING
+
+import torch
+
+if TYPE_CHECKING:
+ from torch import Tensor
+
+from .base import MmprojModel, ModelBase, gguf, logger
+from .qwen import Qwen3_5TextModel
+
+
+def _is_clef_checkpoint(dir_model: Path) -> bool:
+ return (dir_model / "joint_head_config.json").is_file() and (dir_model / "config.json").is_file()
+
+
+@ModelBase.register_hparams_loader(_is_clef_checkpoint)
+def _load_clef_hparams(dir_model: Path) -> dict[str, Any]:
+ logger.info("gguf: detected Clef checkpoint")
+ hparams = ModelBase.load_hparams(dir_model, False, guess=False)
+ hparams["architectures"] = ["ClefModel"]
+ with open(dir_model / "joint_head_config.json", encoding="utf-8") as f:
+ hparams["decision"] = json.load(f)
+ return hparams
+
+
+@ModelBase.register("ClefModel")
+class ClefModel(Qwen3_5TextModel):
+ model_arch = gguf.MODEL_ARCH.CLEF
+ no_mtp = True # the checkpoint has no MTP head
+
+ # prompt follows joint_schema_model.py of the model repo
+ _SYSTEM_PROMPT = (
+ "Read the complete state and schema. Decide every field jointly. Each answer "
+ "must be exactly one of that field's allowed options."
+ )
+ # torch.nn.LayerNorm default, used by the head
+ _HEAD_NORM_EPS = 1e-5
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ head = self.hparams["decision"]
+ self._n_routing = head["routing_layers"]
+ # the head blocks are named dec.blk.N, routing blocks first
+ self.tensor_map = gguf.get_tensor_name_map(self.model_arch, max(self.block_count, self._n_routing + head["layers"]))
+ self._scales: dict[str, float] = {}
+
+ def set_vocab(self):
+ super().set_vocab()
+ self.gguf_writer.add_chat_template([{"name": "systemone", "template": self._systemone_template()}])
+
+ @classmethod
+ def _systemone_template(cls) -> str:
+ def text(value: str) -> str:
+ return "{{ " + json.dumps(value) + " }}"
+
+ def render(name: str) -> str:
+ # strings are used as is, other values are compact JSON
+ return "{{ " + name + " if " + name + " is string else " + name + " | tojson(separators=[',', ':']) }}"
+
+ # the pieces of the prompt are tokenized one by one, the server gives the text that separates them (sep)
+ # and the text that starts the span of a question or of an option (mark_question, mark_option)
+ # the keys of JSON objects are given in sorted order
+ option = (
+ "{% set d = o.description %}"
+ "{% if q.type == 'noul' and d is none %}"
+ "{% set d = 'The proposition is true or the answer is yes.' if o.key == 'true' else 'The proposition is false or the answer is no.' %}"
+ "{% endif %}"
+ "{{ ({'option_id': o.key} if d is none else {'description': d, 'option_id': o.key}) | tojson(separators=[',', ':']) }}"
+ )
+ return (
+ text(f"<|im_start|>system\n{cls._SYSTEM_PROMPT}<|im_end|>\n<|im_start|>user\nSTATE:\n")
+ + "{{ sep }}" + render("state")
+ + "{{ sep }}" + text("\n\nSCHEMA FIELDS:\n")
+ + "{% for q in questions %}"
+ + "{{ sep }}" + text("\nFIELD ") + "{{ loop.index }}" + text("\nID: ") + "{{ q.id }}"
+ + text("\nTYPE: ") + "{{ q.type }}" + text("\nINSTRUCTION: ")
+ + "{{ sep }}{{ mark_question }}" + render("q.instructions")
+ + "{{ sep }}" + text("\nALLOWED OPTIONS:\n")
+ + "{% for o in q.options %}"
+ + "{{ sep }}" + text("OPTION ") + "{{ loop.index }}" + text(": ")
+ + "{{ sep }}{{ mark_option }}" + option
+ + "{{ sep }}" + text("\n")
+ + "{% endfor %}"
+ + "{{ sep }}" + text("END FIELD\n")
+ + "{% endfor %}"
+ + "{{ sep }}" + text("\n<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\nJOINT SCHEMA DECISIONS:")
+ )
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ head = self.hparams["decision"]
+ self.gguf_writer.add_decision_type(gguf.DecisionType.CLEF)
+ self.gguf_writer.add_decision_routing_block_count(head["routing_layers"])
+ self.gguf_writer.add_decision_block_count(head["layers"])
+ self.gguf_writer.add_decision_head_count(head["heads"])
+ self.gguf_writer.add_layer_norm_eps(self._HEAD_NORM_EPS)
+
+ def get_tensors(self) -> Iterator[tuple[str, Tensor]]:
+ yield from super().get_tensors()
+ from safetensors.torch import load_file
+ for name, data in load_file(self.dir_model / "joint_head.safetensors").items():
+ yield "joint_head." + name, data
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if not name.startswith("joint_head."):
+ yield from super().modify_tensors(data_torch, name, bid)
+ return
+
+ parts = name.split(".")
+
+ # learned scalars, stored as the values used at inference
+ if len(parts) == 2 and data_torch.ndim == 0:
+ value = float(data_torch)
+ if parts[1] == "residual_gate":
+ self._scales[parts[1]] = 1.0 / (1.0 + math.exp(-value))
+ else:
+ self._scales[parts[1]] = math.exp(min(value, math.log(100.0)))
+ if len(self._scales) == 3:
+ scales = [self._scales[k] for k in ("prior_logit_scale", "joint_logit_scale", "residual_gate")]
+ yield self.format_tensor_name(gguf.MODEL_TENSOR.DECISION_SCALES, suffix=""), torch.tensor(scales, dtype=torch.float32)
+ return
+
+ # routing blocks come first
+ if parts[1] == "layers":
+ parts[2] = str(int(parts[2]) + self._n_routing)
+ name = ".".join(parts)
+
+ # nn.MultiheadAttention keeps q, k, v in one tensor
+ for suffix in ("weight", "bias"):
+ if name.endswith(".in_proj_" + suffix):
+ prefix = name[:-len("in_proj_" + suffix)]
+ for x, data in zip("qkv", data_torch.chunk(3, dim=0)):
+ yield self.map_tensor_name(prefix + x + "." + suffix), data
+ return
+
+ yield self.map_tensor_name(name), data_torch
+
+
+@ModelBase.register("ClefModel")
+class ClefVisionModel(MmprojModel):
+ def __init__(self, *args, **kwargs):
+ del args, kwargs
+ raise NotImplementedError(
+ "multimodal input is not supported yet for Clef, requires https://github.com/ggml-org/llama.cpp/pull/29622 to be merged first")
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 4242eb0dc..1e0a6b18a 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -324,6 +324,8 @@ class Keys:
TYPE = "{arch}.decision.type"
# note: single-use-case keys can be hard-coded in cpp code
BLOCK_COUNT = "{arch}.decision.block_count"
+ ROUTING_BLOCK_COUNT = "{arch}.decision.routing_block_count"
+ HEAD_COUNT = "{arch}.decision.head_count"
MAX_HEAD_TOKENS = "{arch}.decision.max_head_tokens"
TEMPERATURE = "{arch}.decision.temperature.{name}" # name: "<type>" or "<type>.<n_opt bucket>"
@@ -537,6 +539,7 @@ class MODEL_ARCH(IntEnum):
QWEN3VLMOE = auto()
QWEN35 = auto()
QWEN35MOE = auto()
+ CLEF = auto()
QWEN4EXP = auto()
PHI2 = auto()
PHI3 = auto()
@@ -861,6 +864,7 @@ class MODEL_TENSOR(IntEnum):
DEC_ATTN_OUT = auto()
DEC_ATTN_REL_B = auto()
DEC_CROSS_ATTN_NORM = auto()
+ DEC_CROSS_ATTN_NORM_KV = auto()
DEC_CROSS_ATTN_Q = auto()
DEC_CROSS_ATTN_K = auto()
DEC_CROSS_ATTN_V = auto()
@@ -885,6 +889,19 @@ class MODEL_TENSOR(IntEnum):
CLS = auto() # classifier
CLS_OUT = auto() # classifier output projection
CLS_NORM = auto()
+ DECISION_HIDDEN_NORM = auto()
+ DECISION_PROJ_MEMORY = auto()
+ DECISION_PROJ_QUESTION = auto()
+ DECISION_PROJ_OPTION_QUESTION = auto()
+ DECISION_PROJ_GLOBAL = auto()
+ DECISION_PROJ_OPTION_CONTEXT = auto()
+ DECISION_PROJ_OPTION_LEXICAL = auto()
+ DECISION_OPTION_SUMMARY_NORM = auto()
+ DECISION_FIELD_NORM = auto()
+ DECISION_OPTION_NORM = auto()
+ DECISION_SCALES = auto()
+ DECISION_SCORER = auto()
+ DECISION_SCORER_OUT = auto()
CONV1D = auto()
CONVNEXT_DW = auto()
CONVNEXT_NORM = auto()
@@ -1301,6 +1318,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
MODEL_ARCH.QWEN3VLMOE: "qwen3vlmoe",
MODEL_ARCH.QWEN35: "qwen35",
MODEL_ARCH.QWEN35MOE: "qwen35moe",
+ MODEL_ARCH.CLEF: "clef",
MODEL_ARCH.QWEN4EXP: "qwen4exp",
MODEL_ARCH.PHI2: "phi2",
MODEL_ARCH.PHI3: "phi3",
@@ -1624,6 +1642,7 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.DEC_ATTN_OUT: "dec.blk.{bid}.attn_o",
MODEL_TENSOR.DEC_ATTN_REL_B: "dec.blk.{bid}.attn_rel_b",
MODEL_TENSOR.DEC_CROSS_ATTN_NORM: "dec.blk.{bid}.cross_attn_norm",
+ MODEL_TENSOR.DEC_CROSS_ATTN_NORM_KV: "dec.blk.{bid}.cross_attn_norm_kv",
MODEL_TENSOR.DEC_CROSS_ATTN_Q: "dec.blk.{bid}.cross_attn_q",
MODEL_TENSOR.DEC_CROSS_ATTN_K: "dec.blk.{bid}.cross_attn_k",
MODEL_TENSOR.DEC_CROSS_ATTN_V: "dec.blk.{bid}.cross_attn_v",
@@ -1648,6 +1667,19 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.CLS: "cls",
MODEL_TENSOR.CLS_OUT: "cls.output",
MODEL_TENSOR.CLS_NORM: "cls.norm",
+ MODEL_TENSOR.DECISION_HIDDEN_NORM: "decision.hidden_norm",
+ MODEL_TENSOR.DECISION_PROJ_MEMORY: "decision.proj_memory",
+ MODEL_TENSOR.DECISION_PROJ_QUESTION: "decision.proj_question",
+ MODEL_TENSOR.DECISION_PROJ_OPTION_QUESTION: "decision.proj_option_question",
+ MODEL_TENSOR.DECISION_PROJ_GLOBAL: "decision.proj_global",
+ MODEL_TENSOR.DECISION_PROJ_OPTION_CONTEXT: "decision.proj_option_context",
+ MODEL_TENSOR.DECISION_PROJ_OPTION_LEXICAL: "decision.proj_option_lexical",
+ MODEL_TENSOR.DECISION_OPTION_SUMMARY_NORM: "decision.option_summary_norm",
+ MODEL_TENSOR.DECISION_FIELD_NORM: "decision.field_norm",
+ MODEL_TENSOR.DECISION_OPTION_NORM: "decision.option_norm",
+ MODEL_TENSOR.DECISION_SCALES: "decision.scales",
+ MODEL_TENSOR.DECISION_SCORER: "decision.scorer",
+ MODEL_TENSOR.DECISION_SCORER_OUT: "decision.scorer_out",
MODEL_TENSOR.CONV1D: "conv1d",
MODEL_TENSOR.CONVNEXT_DW: "convnext.{bid}.dw",
MODEL_TENSOR.CONVNEXT_NORM: "convnext.{bid}.norm",
@@ -2918,6 +2950,60 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
],
+ MODEL_ARCH.CLEF: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_Q_NORM,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_K_NORM,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_POST_NORM,
+ MODEL_TENSOR.ATTN_GATE,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ MODEL_TENSOR.SSM_A,
+ MODEL_TENSOR.SSM_CONV1D,
+ MODEL_TENSOR.SSM_DT,
+ MODEL_TENSOR.SSM_NORM,
+ MODEL_TENSOR.SSM_BETA,
+ MODEL_TENSOR.SSM_ALPHA,
+ MODEL_TENSOR.SSM_OUT,
+ # decision head
+ MODEL_TENSOR.TOKEN_TYPES,
+ MODEL_TENSOR.DEC_ATTN_NORM,
+ MODEL_TENSOR.DEC_ATTN_Q,
+ MODEL_TENSOR.DEC_ATTN_K,
+ MODEL_TENSOR.DEC_ATTN_V,
+ MODEL_TENSOR.DEC_ATTN_OUT,
+ MODEL_TENSOR.DEC_CROSS_ATTN_NORM,
+ MODEL_TENSOR.DEC_CROSS_ATTN_NORM_KV,
+ MODEL_TENSOR.DEC_CROSS_ATTN_Q,
+ MODEL_TENSOR.DEC_CROSS_ATTN_K,
+ MODEL_TENSOR.DEC_CROSS_ATTN_V,
+ MODEL_TENSOR.DEC_CROSS_ATTN_OUT,
+ MODEL_TENSOR.DEC_FFN_NORM,
+ MODEL_TENSOR.DEC_FFN_DOWN,
+ MODEL_TENSOR.DEC_FFN_UP,
+ MODEL_TENSOR.DECISION_HIDDEN_NORM,
+ MODEL_TENSOR.DECISION_PROJ_MEMORY,
+ MODEL_TENSOR.DECISION_PROJ_QUESTION,
+ MODEL_TENSOR.DECISION_PROJ_OPTION_QUESTION,
+ MODEL_TENSOR.DECISION_PROJ_GLOBAL,
+ MODEL_TENSOR.DECISION_PROJ_OPTION_CONTEXT,
+ MODEL_TENSOR.DECISION_PROJ_OPTION_LEXICAL,
+ MODEL_TENSOR.DECISION_OPTION_SUMMARY_NORM,
+ MODEL_TENSOR.DECISION_FIELD_NORM,
+ MODEL_TENSOR.DECISION_OPTION_NORM,
+ MODEL_TENSOR.DECISION_SCALES,
+ MODEL_TENSOR.DECISION_SCORER,
+ MODEL_TENSOR.DECISION_SCORER_OUT,
+ ],
MODEL_ARCH.QWEN35MOE: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT_NORM,
@@ -5921,6 +6007,7 @@ class DecisionType:
LEV = "lev" # same as openjev, noul is read from a rating scale
KEV = "kev" # dot product of the hidden states of the last token and of one end token per option
NIMBLE = "nimble" # same as openjev, the prompt lists all the questions of the request
+ CLEF = "clef" # joint head over all questions, one score per option
class VisionProjectorType:
diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py
index 1dee3fe11..4f1f8e5a5 100644
--- a/gguf-py/gguf/gguf_writer.py
+++ b/gguf-py/gguf/gguf_writer.py
@@ -1346,6 +1346,12 @@ class GGUFWriter:
def add_decision_block_count(self, value: int) -> None:
self.add_uint32(Keys.Decision.BLOCK_COUNT.format(arch=self.arch), value)
+ def add_decision_routing_block_count(self, value: int) -> None:
+ self.add_uint32(Keys.Decision.ROUTING_BLOCK_COUNT.format(arch=self.arch), value)
+
+ def add_decision_head_count(self, value: int) -> None:
+ self.add_uint32(Keys.Decision.HEAD_COUNT.format(arch=self.arch), value)
+
def add_decision_max_head_tokens(self, value: int) -> None:
self.add_uint32(Keys.Decision.MAX_HEAD_TOKENS.format(arch=self.arch), value)
diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py
index c8d52f2fe..cef04d082 100644
--- a/gguf-py/gguf/tensor_mapping.py
+++ b/gguf-py/gguf/tensor_mapping.py
@@ -49,6 +49,7 @@ class TensorNameMap:
MODEL_TENSOR.TOKEN_TYPES: (
"embeddings.token_type_embeddings", # bert nomic-bert
"type_emb", # laya
+ "joint_head.type_embedding", # clef
),
# Normalization of token embeddings
@@ -1181,22 +1182,27 @@ class TensorNameMap:
MODEL_TENSOR.DEC_ATTN_NORM: (
"decoder.block.{bid}.layer.0.layer_norm", # t5
+ "joint_head.layers.{bid}.norm1", # clef
),
MODEL_TENSOR.DEC_ATTN_Q: (
"decoder.block.{bid}.layer.0.SelfAttention.q", # t5
+ "joint_head.layers.{bid}.self_attn.q", # clef
),
MODEL_TENSOR.DEC_ATTN_K: (
"decoder.block.{bid}.layer.0.SelfAttention.k", # t5
+ "joint_head.layers.{bid}.self_attn.k", # clef
),
MODEL_TENSOR.DEC_ATTN_V: (
"decoder.block.{bid}.layer.0.SelfAttention.v", # t5
+ "joint_head.layers.{bid}.self_attn.v", # clef
),
MODEL_TENSOR.DEC_ATTN_OUT: (
"decoder.block.{bid}.layer.0.SelfAttention.o", # t5
+ "joint_head.layers.{bid}.self_attn.out_proj", # clef
),
MODEL_TENSOR.DEC_ATTN_REL_B: (
@@ -1205,22 +1211,32 @@ class TensorNameMap:
MODEL_TENSOR.DEC_CROSS_ATTN_NORM: (
"decoder.block.{bid}.layer.1.layer_norm", # t5
+ "joint_head.layers.{bid}.norm2", # clef
+ "joint_head.evidence_layers.{bid}.query_norm", # clef
),
MODEL_TENSOR.DEC_CROSS_ATTN_Q: (
"decoder.block.{bid}.layer.1.EncDecAttention.q", # t5
+ "joint_head.layers.{bid}.multihead_attn.q", # clef
+ "joint_head.evidence_layers.{bid}.attention.q", # clef
),
MODEL_TENSOR.DEC_CROSS_ATTN_K: (
"decoder.block.{bid}.layer.1.EncDecAttention.k", # t5
+ "joint_head.layers.{bid}.multihead_attn.k", # clef
+ "joint_head.evidence_layers.{bid}.attention.k", # clef
),
MODEL_TENSOR.DEC_CROSS_ATTN_V: (
"decoder.block.{bid}.layer.1.EncDecAttention.v", # t5
+ "joint_head.layers.{bid}.multihead_attn.v", # clef
+ "joint_head.evidence_layers.{bid}.attention.v", # clef
),
MODEL_TENSOR.DEC_CROSS_ATTN_OUT: (
"decoder.block.{bid}.layer.1.EncDecAttention.o", # t5
+ "joint_head.layers.{bid}.multihead_attn.out_proj", # clef
+ "joint_head.evidence_layers.{bid}.attention.out_proj", # clef
),
MODEL_TENSOR.DEC_CROSS_ATTN_REL_B: (
@@ -1229,6 +1245,8 @@ class TensorNameMap:
MODEL_TENSOR.DEC_FFN_NORM: (
"decoder.block.{bid}.layer.2.layer_norm", # t5
+ "joint_head.layers.{bid}.norm3", # clef
+ "joint_head.evidence_layers.{bid}.feedforward_norm", # clef
),
MODEL_TENSOR.DEC_FFN_GATE: (
@@ -1238,16 +1256,72 @@ class TensorNameMap:
MODEL_TENSOR.DEC_FFN_UP: (
"decoder.block.{bid}.layer.2.DenseReluDense.wi", # t5
"decoder.block.{bid}.layer.2.DenseReluDense.wi_1", # flan-t5
+ "joint_head.layers.{bid}.linear1", # clef
+ "joint_head.evidence_layers.{bid}.feedforward.0", # clef
),
MODEL_TENSOR.DEC_FFN_DOWN: (
"decoder.block.{bid}.layer.2.DenseReluDense.wo", # t5
+ "joint_head.layers.{bid}.linear2", # clef
+ "joint_head.evidence_layers.{bid}.feedforward.3", # clef
),
MODEL_TENSOR.DEC_OUTPUT_NORM: (
"decoder.final_layer_norm", # t5
),
+ MODEL_TENSOR.DEC_CROSS_ATTN_NORM_KV: (
+ "joint_head.evidence_layers.{bid}.memory_norm", # clef
+ ),
+
+ MODEL_TENSOR.DECISION_HIDDEN_NORM: (
+ "joint_head.hidden_norm", # clef
+ ),
+
+ MODEL_TENSOR.DECISION_PROJ_MEMORY: (
+ "joint_head.memory_projection", # clef
+ ),
+
+ MODEL_TENSOR.DECISION_PROJ_QUESTION: (
+ "joint_head.question_projection", # clef
+ ),
+
+ MODEL_TENSOR.DECISION_PROJ_OPTION_QUESTION: (
+ "joint_head.option_question_projection", # clef
+ ),
+
+ MODEL_TENSOR.DECISION_PROJ_GLOBAL: (
+ "joint_head.global_projection", # clef
+ ),
+
+ MODEL_TENSOR.DECISION_PROJ_OPTION_CONTEXT: (
+ "joint_head.option_context_projection", # clef
+ ),
+
+ MODEL_TENSOR.DECISION_PROJ_OPTION_LEXICAL: (
+ "joint_head.option_lexical_projection", # clef
+ ),
+
+ MODEL_TENSOR.DECISION_OPTION_SUMMARY_NORM: (
+ "joint_head.option_summary_norm", # clef
+ ),
+
+ MODEL_TENSOR.DECISION_FIELD_NORM: (
+ "joint_head.field_norm", # clef
+ ),
+
+ MODEL_TENSOR.DECISION_OPTION_NORM: (
+ "joint_head.option_norm", # clef
+ ),
+
+ MODEL_TENSOR.DECISION_SCORER: (
+ "joint_head.residual_scorer.0", # clef
+ ),
+
+ MODEL_TENSOR.DECISION_SCORER_OUT: (
+ "joint_head.residual_scorer.3", # clef
+ ),
+
MODEL_TENSOR.ENC_ATTN_NORM: (
"encoder.block.{bid}.layer.0.layer_norm", # t5
),
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index 9f205b942..2af5445df 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -40,6 +40,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
{ LLM_ARCH_QWEN3VLMOE, "qwen3vlmoe" },
{ LLM_ARCH_QWEN35, "qwen35" },
{ LLM_ARCH_QWEN35MOE, "qwen35moe" },
+ { LLM_ARCH_CLEF, "clef" },
{ LLM_ARCH_QWEN4EXP, "qwen4exp" },
{ LLM_ARCH_PHI2, "phi2" },
{ LLM_ARCH_PHI3, "phi3" },
@@ -366,7 +367,9 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_CLASSIFIER_OUTPUT_LABELS, "%s.classifier.output_labels" },
{ LLM_KV_CLASSIFIER_POOLING_TYPE, "%s.classifier.pooling_type" },
- { LLM_KV_DECISION_BLOCK_COUNT, "%s.decision.block_count" },
+ { LLM_KV_DECISION_BLOCK_COUNT, "%s.decision.block_count" },
+ { LLM_KV_DECISION_ROUTING_BLOCK_COUNT, "%s.decision.routing_block_count" },
+ { LLM_KV_DECISION_HEAD_COUNT, "%s.decision.head_count" },
{ LLM_KV_TARGET_LAYERS, "%s.target_layers" },
{ LLM_KV_TARGET_HIDDEN_SIZE, "%s.target_hidden_size" },
@@ -490,6 +493,19 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
{ LLM_TENSOR_CLS, "cls" },
{ LLM_TENSOR_CLS_OUT, "cls.output" },
{ LLM_TENSOR_CLS_NORM, "cls.norm" },
+ { LLM_TENSOR_DECISION_HIDDEN_NORM, "decision.hidden_norm" },
+ { LLM_TENSOR_DECISION_PROJ_MEMORY, "decision.proj_memory" },
+ { LLM_TENSOR_DECISION_PROJ_QUESTION, "decision.proj_question" },
+ { LLM_TENSOR_DECISION_PROJ_OPTION_QUESTION, "decision.proj_option_question" },
+ { LLM_TENSOR_DECISION_PROJ_GLOBAL, "decision.proj_global" },
+ { LLM_TENSOR_DECISION_PROJ_OPTION_CONTEXT, "decision.proj_option_context" },
+ { LLM_TENSOR_DECISION_PROJ_OPTION_LEXICAL, "decision.proj_option_lexical" },
+ { LLM_TENSOR_DECISION_OPTION_SUMMARY_NORM, "decision.option_summary_norm" },
+ { LLM_TENSOR_DECISION_FIELD_NORM, "decision.field_norm" },
+ { LLM_TENSOR_DECISION_OPTION_NORM, "decision.option_norm" },
+ { LLM_TENSOR_DECISION_SCALES, "decision.scales" },
+ { LLM_TENSOR_DECISION_SCORER, "decision.scorer" },
+ { LLM_TENSOR_DECISION_SCORER_OUT, "decision.scorer_out" },
{ LLM_TENSOR_ENC_OUTPUT_NORM, "enc.output_norm" },
{ LLM_TENSOR_FFN_GATE_INP_SHEXP, "blk.%d.ffn_gate_inp_shexp" },
{ LLM_TENSOR_SSM_A_NOSCAN, "blk.%d.ssm_a" },
@@ -606,6 +622,7 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
{ LLM_TENSOR_DEC_ATTN_OUT, "dec.blk.%d.attn_o" },
{ LLM_TENSOR_DEC_ATTN_REL_B, "dec.blk.%d.attn_rel_b" },
{ LLM_TENSOR_DEC_CROSS_ATTN_NORM, "dec.blk.%d.cross_attn_norm" },
+ { LLM_TENSOR_DEC_CROSS_ATTN_NORM_KV, "dec.blk.%d.cross_attn_norm_kv" },
{ LLM_TENSOR_DEC_CROSS_ATTN_Q, "dec.blk.%d.cross_attn_q" },
{ LLM_TENSOR_DEC_CROSS_ATTN_K, "dec.blk.%d.cross_attn_k" },
{ LLM_TENSOR_DEC_CROSS_ATTN_V, "dec.blk.%d.cross_attn_v" },
@@ -739,6 +756,19 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
{LLM_TENSOR_CLS, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_CLS_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_CLS_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
+ {LLM_TENSOR_DECISION_HIDDEN_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
+ {LLM_TENSOR_DECISION_PROJ_MEMORY, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_DECISION_PROJ_QUESTION, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_DECISION_PROJ_OPTION_QUESTION, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_DECISION_PROJ_GLOBAL, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_DECISION_PROJ_OPTION_CONTEXT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_DECISION_PROJ_OPTION_LEXICAL, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_DECISION_OPTION_SUMMARY_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
+ {LLM_TENSOR_DECISION_FIELD_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
+ {LLM_TENSOR_DECISION_OPTION_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
+ {LLM_TENSOR_DECISION_SCALES, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
+ {LLM_TENSOR_DECISION_SCORER, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_DECISION_SCORER_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_DENSE_2_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, // Dense layer output
{LLM_TENSOR_DENSE_3_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, // Dense layer output
{LLM_TENSOR_OUTPUT_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
@@ -908,6 +938,7 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
{LLM_TENSOR_FFN_SUB_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_DEC_ATTN_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_DEC_CROSS_ATTN_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_DEC_CROSS_ATTN_NORM_KV, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_DEC_FFN_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_ENC_ATTN_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_ENC_FFN_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
diff --git a/src/llama-arch.h b/src/llama-arch.h
index ca390fcb3..148d293ce 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -45,6 +45,7 @@ enum llm_arch {
LLM_ARCH_QWEN3VLMOE,
LLM_ARCH_QWEN35,
LLM_ARCH_QWEN35MOE,
+ LLM_ARCH_CLEF,
LLM_ARCH_QWEN4EXP,
LLM_ARCH_PHI2,
LLM_ARCH_PHI3,
@@ -413,6 +414,8 @@ enum llm_kv {
LLM_KV_CLASSIFIER_POOLING_TYPE,
LLM_KV_DECISION_BLOCK_COUNT,
+ LLM_KV_DECISION_ROUTING_BLOCK_COUNT,
+ LLM_KV_DECISION_HEAD_COUNT,
LLM_KV_TARGET_LAYERS,
LLM_KV_TARGET_HIDDEN_SIZE,
@@ -636,6 +639,7 @@ enum llm_tensor {
LLM_TENSOR_DEC_ATTN_OUT,
LLM_TENSOR_DEC_ATTN_REL_B,
LLM_TENSOR_DEC_CROSS_ATTN_NORM,
+ LLM_TENSOR_DEC_CROSS_ATTN_NORM_KV,
LLM_TENSOR_DEC_CROSS_ATTN_Q,
LLM_TENSOR_DEC_CROSS_ATTN_K,
LLM_TENSOR_DEC_CROSS_ATTN_V,
@@ -660,6 +664,19 @@ enum llm_tensor {
LLM_TENSOR_CLS,
LLM_TENSOR_CLS_OUT,
LLM_TENSOR_CLS_NORM,
+ LLM_TENSOR_DECISION_HIDDEN_NORM,
+ LLM_TENSOR_DECISION_PROJ_MEMORY,
+ LLM_TENSOR_DECISION_PROJ_QUESTION,
+ LLM_TENSOR_DECISION_PROJ_OPTION_QUESTION,
+ LLM_TENSOR_DECISION_PROJ_GLOBAL,
+ LLM_TENSOR_DECISION_PROJ_OPTION_CONTEXT,
+ LLM_TENSOR_DECISION_PROJ_OPTION_LEXICAL,
+ LLM_TENSOR_DECISION_OPTION_SUMMARY_NORM,
+ LLM_TENSOR_DECISION_FIELD_NORM,
+ LLM_TENSOR_DECISION_OPTION_NORM,
+ LLM_TENSOR_DECISION_SCALES,
+ LLM_TENSOR_DECISION_SCORER,
+ LLM_TENSOR_DECISION_SCORER_OUT,
LLM_TENSOR_CONV1D,
LLM_TENSOR_CONVNEXT_DW,
LLM_TENSOR_CONVNEXT_NORM,
diff --git a/src/llama-batch.cpp b/src/llama-batch.cpp
index 7a17b3a4e..1b3d70627 100644
--- a/src/llama-batch.cpp
+++ b/src/llama-batch.cpp
@@ -168,6 +168,14 @@ bool llama_batch_allocr::init(
}
}
+ // kept empty if no entry has one
+ for (int32_t i = 0; i < n_tok; ++i) {
+ if (batch_inp.tokens[i].decision_order != 0) {
+ decision_order.resize(n_tok, 0);
+ decision_order[i] = batch_inp.tokens[i].decision_order;
+ }
+ }
+
//
// set up the internal llama_batch to point to our owned arrays
//
@@ -256,6 +264,7 @@ bool llama_batch_allocr::init(
/*.seq_id_unq =*/ this->seq_id_unq.data(),
/*.seq_idx =*/ this->seq_idx.data(),
/*.output =*/ batch.logits,
+ /*.decision_order =*/ decision_order.empty() ? nullptr : decision_order.data(),
/*.data =*/ {},
};
@@ -462,6 +471,7 @@ llama_ubatch llama_batch_allocr::ubatch_reserve(uint32_t n_seq_tokens, uint32_t
/*.seq_id_unq =*/ udata->seq_id_unq.data(),
/*.seq_idx =*/ udata->seq_idx.data(),
/*.output =*/ udata->output.data(),
+ /*.decision_order =*/ nullptr,
/*.data =*/ std::move(udata),
};
@@ -765,6 +775,7 @@ void llama_batch_allocr::clear() {
seq_id .clear();
seq_id_unq .clear();
output .clear();
+ decision_order.clear();
for (auto & cur : seq_pos) {
cur.clear();
@@ -799,6 +810,7 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
udata->seq_id_unq.resize(0);
udata->seq_idx .resize(LLAMA_MAX_SEQ, -1);
udata->output .resize(n_tokens);
+ udata->decision_order.resize(decision_order.empty() ? 0 : n_tokens);
udata->batch_idxs = idxs;
udata->seq_id_data.reserve(n_tokens);
@@ -826,6 +838,10 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
udata->n_seq_id[i] = batch.n_seq_id[idxs[i]];
udata->output[i] = batch.logits[idxs[i]];
+ if (!decision_order.empty()) {
+ udata->decision_order[i] = decision_order[idxs[i]];
+ }
+
for (int s = 0; s < udata->n_seq_id[i]; ++s) {
const llama_seq_id seq_id = batch.seq_id[idxs[i]][s];
@@ -867,6 +883,7 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
/*.seq_id_unq =*/ udata->seq_id_unq.data(),
/*.seq_idx =*/ udata->seq_idx.data(),
/*.output =*/ udata->output.data(),
+ /*.decision_order =*/ udata->decision_order.empty() ? nullptr : udata->decision_order.data(),
/*.data =*/ std::move(udata),
};
@@ -1175,6 +1192,14 @@ bool llama_batch_ext::set_output(int32_t idx, bool output_last) {
return true;
}
+bool llama_batch_ext::set_decision_order(int32_t idx, int32_t order) {
+ if (idx < 0 || idx >= (int32_t) tokens.size()) {
+ return false;
+ }
+ tokens[idx].decision_order = order;
+ return true;
+}
+
// llama_batch_ext C API
llama_batch_ext * llama_batch_ext_init(llama_context * ctx) {
@@ -1243,6 +1268,10 @@ bool llama_batch_ext_set_output_logits(llama_batch_ext * batch, int32_t idx, boo
return batch->set_output(idx, value);
}
+bool llama_batch_ext_set_decision_order(llama_batch_ext * batch, int32_t idx, llama_decision_order order) {
+ return batch->set_decision_order(idx, order);
+}
+
// llama_batch_compat
void llama_batch_compat::init(llama_batch_ext & dst, const llama_batch & batch_inp, size_t n_embd_row) {
diff --git a/src/llama-batch.h b/src/llama-batch.h
index 18e8144c8..32f103bfc 100644
--- a/src/llama-batch.h
+++ b/src/llama-batch.h
@@ -52,6 +52,7 @@ struct llama_ubatch {
llama_seq_id * seq_id_unq; // [n_seqs_unq] | s | seq_id
int32_t * seq_idx; // [LLAMA_MAX_SEQ] | - | seq_idx
int8_t * output; // [n_tokens] | i | -
+ int32_t * decision_order; // [n_tokens], NULL if no entry has one, see llama_batch_ext_set_decision_order()
struct data_t {
std::vector<llama_token> token;
@@ -63,6 +64,7 @@ struct llama_ubatch {
std::vector<int32_t> seq_idx;
std::vector<int8_t> output;
std::vector<int32_t> batch_idxs; // original batch index for each token
+ std::vector<int32_t> decision_order;
std::vector<llama_seq_id> seq_id_data;
};
@@ -96,6 +98,7 @@ struct llama_batch_ext {
bool has_embd = false; // whether embd_off is set
size_t embd_off = 0; // index offset in the embd array
bool output = false; // TODO: have dedicated output flags
+ int32_t decision_order = 0; // see llama_batch_ext_set_decision_order()
std::unordered_set<llama_seq_id> seq_ids;
std::array<llama_pos, GGML_MROPE_SECTIONS> pos = {0, 0, 0, 0};
};
@@ -125,6 +128,7 @@ struct llama_batch_ext {
bool set_token_embd(int32_t idx, llama_embd embd_in);
bool set_token_pos(int32_t idx, const llama_pos * pos_in);
bool set_output(int32_t idx, bool output_last);
+ bool set_decision_order(int32_t idx, int32_t order);
};
// a helper for sanitizing, fulfilling and splitting a batch
@@ -202,6 +206,7 @@ private:
std::vector<llama_seq_id> seq_id_unq;
std::vector<int32_t> seq_idx;
std::vector<int8_t> output;
+ std::vector<int32_t> decision_order; // empty if no entry has one
using pos_set_t = std::set<llama_pos>;
using seq_cpl_t = std::vector<bool>;
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index 4f04db0ee..96b5464e6 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -2406,6 +2406,7 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {
model.arch == LLM_ARCH_BAILINGMOE3 ||
model.arch == LLM_ARCH_QWEN35 ||
model.arch == LLM_ARCH_QWEN35MOE ||
+ model.arch == LLM_ARCH_CLEF ||
model.arch == LLM_ARCH_QWEN4EXP ||
model.arch == LLM_ARCH_DEEPSEEK4 ||
(model.arch == LLM_ARCH_DFLASH && model.hparams.dsv4_hc_mult > 0) ||
diff --git a/src/llama-ext.h b/src/llama-ext.h
index 92a759b7a..fcc78f47e 100644
--- a/src/llama-ext.h
+++ b/src/llama-ext.h
@@ -100,6 +100,20 @@ LLAMA_API void llama_set_embeddings_nextn(struct llama_context * ctx, bool value
// chain multiple trained NextN heads. Default 0 (first head).
LLAMA_API void llama_set_nextn_layer_offset(struct llama_context * ctx, int32_t offset);
+// Marks the entries that a joint decision head (clef) reads, the default is 0
+// See https://github.com/ggml-org/llama.cpp/pull/29831 for details
+// A run of entries with the same value is one span, spans must be separated by entries with value 0
+// An option belongs to the last question before it
+enum llama_decision_order {
+ LLAMA_DECISION_ORDER_NONE = 0, // not read by the head
+ LLAMA_DECISION_ORDER_QUESTION_NOUL = 1, // text of a question
+ LLAMA_DECISION_ORDER_QUESTION_CHOICE = 2,
+ LLAMA_DECISION_ORDER_QUESTION_SCORE = 3,
+ LLAMA_DECISION_ORDER_OPTION = 4, // text of an option
+};
+// The embeddings output has one value per entry: row i is the score of option i
+LLAMA_API bool llama_batch_ext_set_decision_order(struct llama_batch_ext * batch, int32_t idx, enum llama_decision_order order);
+
// mirrors:
// LLAMA_API float * llama_get_embeddings(struct llama_context * ctx);
LLAMA_API float * llama_get_embeddings_nextn(struct llama_context * ctx);
diff --git a/src/llama-kv-cache-dsv4.cpp b/src/llama-kv-cache-dsv4.cpp
index 948d08146..4dbcfb8db 100644
--- a/src/llama-kv-cache-dsv4.cpp
+++ b/src/llama-kv-cache-dsv4.cpp
@@ -164,6 +164,7 @@ static llama_ubatch dsv4_build_raw_write_ubatch(const llama_ubatch & ubatch) {
/*.seq_id_unq =*/ data->seq_id_unq.data(),
/*.seq_idx =*/ data->seq_idx.data(),
/*.output =*/ data->output.data(),
+ /*.decision_order =*/ nullptr,
/*.data =*/ data,
};
diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp
index eeecadac0..e4fd45ec4 100644
--- a/src/llama-model-saver.cpp
+++ b/src/llama-model-saver.cpp
@@ -20,6 +20,7 @@ bool llama_model_saver_supports_arch(llm_arch arch) {
case LLM_ARCH_T5:
case LLM_ARCH_APERTUS:
case LLM_ARCH_STEP35:
+ case LLM_ARCH_CLEF: // the head tensors are not saved
return false;
default:
return true;
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 742bcd5d5..97e0cee70 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -326,6 +326,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_qwen35(params);
case LLM_ARCH_QWEN35MOE:
return new llama_model_qwen35moe(params);
+ case LLM_ARCH_CLEF:
+ return new llama_model_clef(params);
case LLM_ARCH_QWEN4EXP:
return new llama_model_qwen4exp(params);
case LLM_ARCH_MISTRAL3:
@@ -607,7 +609,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
// if a model has fused tensors they need to be separated into "segments", see the comments on ggml_backend_meta_split_state struct
auto get_split_segments = [&](int axis, uint32_t il) -> std::vector<std::pair<int64_t, uint32_t>> {
// TODO: deduplicate condition [TAG_SPLIT_QGATE_QWEN]
- if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE ||
+ if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE || ud->model->arch == LLM_ARCH_CLEF ||
ud->model->arch == LLM_ARCH_QWEN4EXP) {
// fused full attention layers with Q gate tensors that need n_embd doubled:
@@ -762,7 +764,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
GGML_ASSERT(segments.size() == 1);
// some models have Q gate tensors, for those cases the granularity needs to be doubled:
// TODO: deduplicate condition [TAG_SPLIT_QGATE_QWEN]
- if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE ||
+ if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE || ud->model->arch == LLM_ARCH_CLEF ||
ud->model->arch == LLM_ARCH_QWEN4EXP) {
return {std::lcm(2*n_embd_q, blck_size_perf)};
}
@@ -795,7 +797,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_bias)) {
// fused full attention layers need Q gate tensors handled like above:
// TODO: deduplicate condition [TAG_SPLIT_QGATE_QWEN]
- if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE ||
+ if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE || ud->model->arch == LLM_ARCH_CLEF ||
ud->model->arch == LLM_ARCH_QWEN4EXP) {
return {std::lcm(2*n_embd_q, blck_size_perf), granularity_kv};
}
@@ -1961,7 +1963,11 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
}
ggml_tensor * llama_model_base::create_tensor(llama_model_loader & ml, const LLM_TN_IMPL & tn, const std::initializer_list<int64_t> & ne, int flags) {
- const buft_list_t * buft_list_layer = tn.bid == -1 ? nullptr : pimpl->dev_layer.at(tn.bid).buft_list;
+ const buft_list_t * buft_list_layer = nullptr;
+ if (tn.bid != -1) {
+ // blocks that are not model layers (e.g. the blocks of a head) go with the output
+ buft_list_layer = (size_t) tn.bid < pimpl->dev_layer.size() ? pimpl->dev_layer.at(tn.bid).buft_list : pimpl->dev_output.buft_list;
+ }
return ml.create_tensor(
hparams, &pimpl->cpu_buft_list, pimpl->dev_input.buft_list, pimpl->dev_output.buft_list, buft_list_layer,
tn, ne, flags);
@@ -2369,6 +2375,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
case LLM_ARCH_LLADA:
case LLM_ARCH_LLADA_MOE:
case LLM_ARCH_RND1:
+ case LLM_ARCH_CLEF:
{
res = nullptr;
} break;
@@ -3200,6 +3207,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_QWEN3VLMOE:
case LLM_ARCH_QWEN35:
case LLM_ARCH_QWEN35MOE:
+ case LLM_ARCH_CLEF:
case LLM_ARCH_QWEN4EXP:
case LLM_ARCH_QWEN3TTS:
return LLAMA_ROPE_TYPE_IMROPE;
diff --git a/src/models/clef.cpp b/src/models/clef.cpp
new file mode 100644
index 000000000..1911efa7e
--- /dev/null
+++ b/src/models/clef.cpp
@@ -0,0 +1,616 @@
+#include "models.h"
+
+#include "llama-ext.h"
+
+#include <cmath>
+
+void llama_model_clef::load_arch_hparams(llama_model_loader & ml) {
+ llama_model_qwen35::load_arch_hparams(ml);
+
+ ml.get_key(LLM_KV_DECISION_ROUTING_BLOCK_COUNT, n_layer_routing);
+ ml.get_key(LLM_KV_DECISION_BLOCK_COUNT, n_layer_joint);
+ ml.get_key(LLM_KV_DECISION_HEAD_COUNT, n_head_decision);
+
+ if (n_head_decision == 0 || n_layer_routing > LLAMA_MAX_LAYERS || n_layer_joint > LLAMA_MAX_LAYERS) {
+ throw std::runtime_error("invalid size of the decision head");
+ }
+
+ // used by the head
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
+
+ // the output is one score per token, see llama_batch_ext_set_decision_order()
+ hparams.n_embd_out_impl = 1;
+}
+
+void llama_model_clef::load_arch_tensors(llama_model_loader & ml) {
+ llama_model_qwen35::load_arch_tensors(ml);
+
+ LLAMA_LOAD_LOCALS;
+
+ const auto * w_memory = ml.get_weight(tn(LLM_TENSOR_DECISION_PROJ_MEMORY, "weight").str().c_str());
+ const auto * w_ffn = ml.get_weight(tn(LLM_TENSOR_DEC_FFN_UP, "weight", 0).str().c_str());
+ if (w_memory == nullptr || w_ffn == nullptr) {
+ throw std::runtime_error("the decision head is missing");
+ }
+ const int64_t n_embd_h = w_memory->tensor->ne[1];
+ const int64_t n_ff_h = w_ffn->tensor->ne[1];
+
+ if (n_embd_h % n_head_decision != 0) {
+ throw std::runtime_error("invalid width of the decision head");
+ }
+
+ auto load_norm = [&](norm & n, llm_tensor type, int64_t size, int il = -1) {
+ n.w = il < 0 ? create_tensor(tn(type, "weight"), {size}, 0) : create_tensor(tn(type, "weight", il), {size}, 0);
+ n.b = il < 0 ? create_tensor(tn(type, "bias"), {size}, 0) : create_tensor(tn(type, "bias", il), {size}, 0);
+ };
+
+ auto load_attn = [&](attn & a, llm_tensor q, llm_tensor k, llm_tensor v, llm_tensor o, int il) {
+ a.wq = create_tensor(tn(q, "weight", il), {n_embd_h, n_embd_h}, 0);
+ a.bq = create_tensor(tn(q, "bias", il), {n_embd_h}, 0);
+ a.wk = create_tensor(tn(k, "weight", il), {n_embd_h, n_embd_h}, 0);
+ a.bk = create_tensor(tn(k, "bias", il), {n_embd_h}, 0);
+ a.wv = create_tensor(tn(v, "weight", il), {n_embd_h, n_embd_h}, 0);
+ a.bv = create_tensor(tn(v, "bias", il), {n_embd_h}, 0);
+ a.wo = create_tensor(tn(o, "weight", il), {n_embd_h, n_embd_h}, 0);
+ a.bo = create_tensor(tn(o, "bias", il), {n_embd_h}, 0);
+ };
+
+ head_layers.resize(n_layer_routing + n_layer_joint);
+ for (int il = 0; il < (int) head_layers.size(); ++il) {
+ auto & layer = head_layers[il];
+
+ if (il < (int) n_layer_routing) {
+ load_norm(layer.cross_norm_kv, LLM_TENSOR_DEC_CROSS_ATTN_NORM_KV, n_embd_h, il);
+ } else {
+ load_norm(layer.self_norm, LLM_TENSOR_DEC_ATTN_NORM, n_embd_h, il);
+ load_attn(layer.self_attn, LLM_TENSOR_DEC_ATTN_Q, LLM_TENSOR_DEC_ATTN_K, LLM_TENSOR_DEC_ATTN_V, LLM_TENSOR_DEC_ATTN_OUT, il);
+ }
+
+ load_norm(layer.cross_norm, LLM_TENSOR_DEC_CROSS_ATTN_NORM, n_embd_h, il);
+ load_attn(layer.cross_attn, LLM_TENSOR_DEC_CROSS_ATTN_Q, LLM_TENSOR_DEC_CROSS_ATTN_K, LLM_TENSOR_DEC_CROSS_ATTN_V, LLM_TENSOR_DEC_CROSS_ATTN_OUT, il);
+
+ load_norm(layer.ffn_norm, LLM_TENSOR_DEC_FFN_NORM, n_embd_h, il);
+ layer.ffn_up = create_tensor(tn(LLM_TENSOR_DEC_FFN_UP, "weight", il), {n_embd_h, n_ff_h}, 0);
+ layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_DEC_FFN_UP, "bias", il), {n_ff_h}, 0);
+ layer.ffn_down = create_tensor(tn(LLM_TENSOR_DEC_FFN_DOWN, "weight", il), {n_ff_h, n_embd_h}, 0);
+ layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_DEC_FFN_DOWN, "bias", il), {n_embd_h}, 0);
+ }
+
+ load_norm(hidden_norm, LLM_TENSOR_DECISION_HIDDEN_NORM, n_embd);
+ load_norm(option_summary_norm, LLM_TENSOR_DECISION_OPTION_SUMMARY_NORM, n_embd_h);
+ load_norm(field_norm, LLM_TENSOR_DECISION_FIELD_NORM, n_embd_h);
+ load_norm(option_norm, LLM_TENSOR_DECISION_OPTION_NORM, n_embd_h);
+
+ proj_memory = create_tensor(tn(LLM_TENSOR_DECISION_PROJ_MEMORY, "weight"), {n_embd, n_embd_h}, 0);
+ proj_question = create_tensor(tn(LLM_TENSOR_DECISION_PROJ_QUESTION, "weight"), {n_embd, n_embd_h}, 0);
+ proj_option_question = create_tensor(tn(LLM_TENSOR_DECISION_PROJ_OPTION_QUESTION, "weight"), {n_embd, n_embd_h}, 0);
+ proj_global = create_tensor(tn(LLM_TENSOR_DECISION_PROJ_GLOBAL, "weight"), {n_embd, n_embd_h}, 0);
+ proj_option_context = create_tensor(tn(LLM_TENSOR_DECISION_PROJ_OPTION_CONTEXT, "weight"), {n_embd, n_embd_h}, 0);
+ proj_option_lexical = create_tensor(tn(LLM_TENSOR_DECISION_PROJ_OPTION_LEXICAL, "weight"), {n_embd, n_embd_h}, 0);
+
+ scales = create_tensor(tn(LLM_TENSOR_DECISION_SCALES), {3}, 0);
+ type_embd = create_tensor(tn(LLM_TENSOR_TOKEN_TYPES, "weight"), {n_embd_h, 3}, 0);
+
+ scorer = create_tensor(tn(LLM_TENSOR_DECISION_SCORER, "weight"), {4 * n_embd_h, n_embd_h}, 0);
+ scorer_b = create_tensor(tn(LLM_TENSOR_DECISION_SCORER, "bias"), {n_embd_h}, 0);
+ scorer_out = create_tensor(tn(LLM_TENSOR_DECISION_SCORER_OUT, "weight"), {n_embd_h, 1}, 0);
+ scorer_out_b = create_tensor(tn(LLM_TENSOR_DECISION_SCORER_OUT, "bias"), {1}, 0);
+}
+
+std::unique_ptr<llm_graph_context> llama_model_clef::build_arch_graph(const llm_graph_params & params) const {
+ return std::make_unique<graph>(*this, params);
+}
+
+// spans read by the head, [start, end) in ubatch token indices
+struct clef_spans {
+ struct question {
+ int32_t type; // noul, choice, score
+ int32_t start;
+ int32_t end;
+ };
+ struct option {
+ int32_t question;
+ int32_t start;
+ int32_t end;
+ };
+ std::vector<question> questions;
+ std::vector<option> options;
+ bool valid = false;
+};
+
+// see llama_batch_ext_set_decision_order()
+// if the batch has no usable order, returns one empty question with one empty option
+static clef_spans clef_get_spans(const llama_ubatch & ubatch) {
+ clef_spans res;
+ std::vector<bool> has_option;
+
+ // TODO: support multiple sequences
+ bool ok = ubatch.decision_order != nullptr && ubatch.n_seqs_unq == 1;
+
+ const int32_t n_tokens = ubatch.n_tokens;
+ for (int32_t i = 0; ok && i < n_tokens;) {
+ const int32_t order = ubatch.decision_order[i];
+ int32_t end = i + 1;
+ while (end < n_tokens && ubatch.decision_order[end] == order) {
+ end++;
+ }
+ switch (order) {
+ case LLAMA_DECISION_ORDER_NONE:
+ break;
+ case LLAMA_DECISION_ORDER_QUESTION_NOUL:
+ case LLAMA_DECISION_ORDER_QUESTION_CHOICE:
+ case LLAMA_DECISION_ORDER_QUESTION_SCORE:
+ res.questions.push_back({ order - LLAMA_DECISION_ORDER_QUESTION_NOUL, i, end });
+ has_option.push_back(false);
+ break;
+ case LLAMA_DECISION_ORDER_OPTION:
+ ok = !res.questions.empty();
+ if (ok) {
+ res.options.push_back({ (int32_t) res.questions.size() - 1, i, end });
+ has_option.back() = true;
+ }
+ break;
+ default:
+ ok = false;
+ }
+ i = end;
+ }
+
+ // each question needs an option
+ res.valid = ok && !res.questions.empty() && std::find(has_option.begin(), has_option.end(), false) == has_option.end();
+ if (!res.valid) {
+ res.questions.clear();
+ res.options.clear();
+ res.questions.push_back({ 0, 0, 0 });
+ res.options.push_back({ 0, 0, 0 });
+ }
+ return res;
+}
+
+// pooling matrices and indices computed from the spans
+class llama_model_clef::input_decision : public llm_graph_input_i {
+public:
+ input_decision(const llama_ubatch & ubatch) : n_tokens(ubatch.n_tokens) {
+ const auto spans = clef_get_spans(ubatch);
+ n_questions = spans.questions.size();
+ n_options = spans.options.size();
+ }
+
+ void set_input(const llama_ubatch * ubatch) override {
+ GGML_ASSERT(ubatch->token);
+ ggml_backend_tensor_set(tokens, ubatch->token, 0, n_tokens * sizeof(llama_token));
+
+ const auto spans = clef_get_spans(*ubatch);
+ GGML_ASSERT(spans.questions.size() == n_questions && spans.options.size() == n_options);
+
+ // the scores are NaN if the batch has a decision order that cannot be used
+ const float status_data = spans.valid || ubatch->decision_order == nullptr ? 0.0f : NAN;
+ ggml_backend_tensor_set(status, &status_data, 0, ggml_nbytes(status));
+
+ std::vector<float> pool_q_data(n_tokens * n_questions, 0.0f);
+ std::vector<int32_t> types(n_questions);
+ for (size_t i = 0; i < n_questions; i++) {
+ const auto & q = spans.questions[i];
+ for (int32_t j = q.start; j < q.end; j++) {
+ pool_q_data[i * n_tokens + j] = 1.0f / (q.end - q.start);
+ }
+ types[i] = q.type;
+ }
+
+ std::vector<float> pool_o_data(n_tokens * n_options, 0.0f);
+ std::vector<int32_t> owner(n_options);
+ std::vector<float> mask(n_options * n_questions, -INFINITY);
+ for (size_t i = 0; i < n_options; i++) {
+ const auto & o = spans.options[i];
+ for (int32_t j = o.start; j < o.end; j++) {
+ pool_o_data[i * n_tokens + j] = 1.0f / (o.end - o.start);
+ }
+ owner[i] = o.question;
+ mask[o.question * n_options + i] = 0.0f;
+ }
+
+ ggml_backend_tensor_set(pool_q, pool_q_data.data(), 0, ggml_nbytes(pool_q));
+ ggml_backend_tensor_set(pool_o, pool_o_data.data(), 0, ggml_nbytes(pool_o));
+ ggml_backend_tensor_set(question_type, types.data(), 0, ggml_nbytes(question_type));
+ ggml_backend_tensor_set(option_question, owner.data(), 0, ggml_nbytes(option_question));
+ ggml_backend_tensor_set(option_mask, mask.data(), 0, ggml_nbytes(option_mask));
+ }
+
+ bool can_reuse(const llm_graph_params & params) override {
+ // the values are computed again in set_input(), only the shapes must match
+ const auto spans = clef_get_spans(params.ubatch);
+ return spans.questions.size() == n_questions && spans.options.size() == n_options;
+ }
+
+ ggml_tensor * tokens = nullptr; // I32 [n_tokens]
+ ggml_tensor * pool_q = nullptr; // F32 [n_tokens, n_questions], mean over the span of the question
+ ggml_tensor * pool_o = nullptr; // F32 [n_tokens, n_options], mean over the span of the option
+ ggml_tensor * question_type = nullptr; // I32 [n_questions]
+ ggml_tensor * option_question = nullptr; // I32 [n_options]
+ ggml_tensor * option_mask = nullptr; // F32 [n_options, n_questions], 0 if the option belongs to the question, else -inf
+ ggml_tensor * status = nullptr; // F32 [1], added to the scores: 0, or NaN on invalid input
+
+ const int64_t n_tokens;
+ size_t n_questions;
+ size_t n_options;
+};
+
+// the backbone is copied from llama_model_qwen35::graph, without the memory module
+llama_model_clef::graph::graph(const llama_model & model_base, const llm_graph_params & params) :
+ llm_build_delta_net_base(params), model(static_cast<const llama_model_clef &>(model_base)) {
+ const int64_t n_embd_head = hparams.n_embd_head_v();
+
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
+
+ int sections[4];
+ std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);
+
+ ggml_tensor * cur;
+ ggml_tensor * inpL;
+
+ inpL = build_inp_embd(model.tok_embd);
+
+ ggml_tensor * inp_pos = build_inp_pos();
+
+ auto * inp_attn = build_attn_inp_causal();
+
+ auto inp_decision_ptr = std::make_unique<input_decision>(ubatch);
+ auto * inp_decision = inp_decision_ptr.get();
+ res->add_input(std::move(inp_decision_ptr));
+
+ for (int il = 0; il < n_layer; ++il) {
+ ggml_tensor * inpSA = inpL;
+
+ cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il);
+ cb(cur, "attn_norm", il);
+
+ if (hparams.is_recr(il)) {
+ cur = build_layer_attn_linear(cur, il);
+ } else {
+ cur = build_layer_attn(inp_attn, cur, inp_pos, sections, il);
+ }
+
+ cur = ggml_add(ctx0, cur, inpSA);
+ cb(cur, "attn_residual", il);
+
+ ggml_tensor * ffn_residual = cur;
+
+ cur = build_norm(cur, model.layers[il].attn_post_norm, nullptr, LLM_NORM_RMS, il);
+ cb(cur, "attn_post_norm", il);
+
+ cur = build_ffn(cur,
+ model.layers[il].ffn_up, NULL, model.layers[il].ffn_up_s,
+ model.layers[il].ffn_gate, NULL, model.layers[il].ffn_gate_s,
+ model.layers[il].ffn_down, NULL, model.layers[il].ffn_down_s,
+ NULL,
+ LLM_FFN_SILU, LLM_FFN_PAR, il);
+ cb(cur, "ffn_out", il);
+
+ cur = ggml_add(ctx0, cur, ffn_residual);
+ cb(cur, "l_out", il);
+
+ inpL = cur;
+ }
+
+ cur = build_norm(inpL, model.output_norm, nullptr, LLM_NORM_RMS, -1);
+ cb(cur, "result_norm", -1);
+
+ // the head is always evaluated, so that the graph has the same nodes for every batch
+ cur = build_head(cur, inp_decision);
+
+ // row i of the output is the score of option i
+ cur = ggml_pad(ctx0, cur, 0, n_tokens - cur->ne[1], 0, 0);
+ cur = ggml_add(ctx0, cur, inp_decision->status);
+ cb(cur, "result_decision", -1);
+
+ res->t_embd = cur;
+
+ ggml_build_forward_expand(gf, cur);
+}
+
+// same as build_attn_inp_no_cache(), but the mask is causal even if the batch is processed by the encoder path
+llm_graph_input_attn_no_cache * llama_model_clef::graph::build_attn_inp_causal() {
+ llama_cparams cparams_causal = cparams;
+ cparams_causal.causal_attn = true;
+
+ auto inp = std::make_unique<llm_graph_input_attn_no_cache>(hparams, cparams_causal);
+
+ const auto type_mask = cparams.flash_attn ? GGML_TYPE_F16 : GGML_TYPE_F32;
+
+ inp->self_kq_mask = ggml_new_tensor_4d(ctx0, type_mask, n_tokens, n_tokens, 1, 1);
+ ggml_set_input(inp->self_kq_mask);
+ cb(inp->self_kq_mask, "self_kq_mask", -1);
+
+ inp->self_kq_mask_cnv = inp->self_kq_mask;
+
+ return (llm_graph_input_attn_no_cache *) res->add_input(std::move(inp));
+}
+
+ggml_tensor * llama_model_clef::graph::build_layer_attn(
+ llm_graph_input_attn_no_cache * inp,
+ ggml_tensor * cur,
+ ggml_tensor * inp_pos,
+ int * sections,
+ int il) {
+ const int64_t n_embd_head = hparams.n_embd_head_v();
+
+ // the Q projection outputs query + gate
+ auto [Qcur_full, Kcur, Vcur] = build_qkv(model.layers[il], cur,
+ n_embd_head * 2, n_head,
+ n_embd_head, n_head_kv,
+ n_embd_head, n_head_kv,
+ il, false);
+
+ ggml_tensor * Qcur = ggml_view_3d(ctx0, Qcur_full, n_embd_head, n_head, n_tokens,
+ ggml_element_size(Qcur_full) * n_embd_head * 2,
+ ggml_element_size(Qcur_full) * n_embd_head * 2 * n_head, 0);
+
+ Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il);
+ cb(Qcur, "Qcur_normed", il);
+
+ Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
+ Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il);
+ cb(Kcur, "Kcur_normed", il);
+
+ ggml_tensor * gate = ggml_view_3d(ctx0, Qcur_full, n_embd_head, n_head, n_tokens,
+ ggml_element_size(Qcur_full) * n_embd_head * 2,
+ ggml_element_size(Qcur_full) * n_embd_head * 2 * n_head,
+ ggml_element_size(Qcur_full) * n_embd_head);
+ gate = ggml_cont_2d(ctx0, gate, n_embd_head * n_head, n_tokens);
+
+ Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+
+ Qcur = ggml_rope_multi(
+ ctx0, Qcur, inp_pos, nullptr,
+ n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+
+ Kcur = ggml_rope_multi(
+ ctx0, Kcur, inp_pos, nullptr,
+ n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,
+ ext_factor, attn_factor, beta_fast, beta_slow
+ );
+
+ const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;
+
+ cur = build_attn(inp,
+ nullptr, nullptr, nullptr,
+ Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
+ cb(cur, "attn_pregate", il);
+
+ cur = ggml_mul(ctx0, cur, ggml_sigmoid(ctx0, gate));
+
+ cur = build_lora_mm(model.layers[il].wo, cur, model.layers[il].wo_s);
+ cb(cur, "attn_output", il);
+
+ return cur;
+}
+
+// gated delta net over the whole batch, the conv and recurrent states start from zero and are not kept
+ggml_tensor * llama_model_clef::graph::build_layer_attn_linear(
+ ggml_tensor * cur,
+ int il) {
+ const auto & layer = model.layers[il];
+
+ const int64_t d_inner = hparams.ssm_d_inner;
+ const int64_t head_k_dim = hparams.ssm_d_state;
+ const int64_t num_k_heads = hparams.ssm_n_group;
+ const int64_t num_v_heads = hparams.ssm_dt_rank;
+ const int64_t head_v_dim = d_inner / num_v_heads;
+ const int64_t n_seqs = 1;
+
+ ggml_tensor * qkv_mixed = build_lora_mm(layer.wqkv, cur, layer.wqkv_s);
+ qkv_mixed = ggml_reshape_3d(ctx0, qkv_mixed, qkv_mixed->ne[0], n_tokens, n_seqs);
+
+ ggml_tensor * z = build_lora_mm(layer.wqkv_gate, cur, layer.wqkv_gate_s);
+
+ ggml_tensor * beta = build_lora_mm(layer.ssm_beta, cur, layer.ssm_beta_s);
+ beta = ggml_reshape_4d(ctx0, beta, 1, num_v_heads, n_tokens, n_seqs);
+ beta = ggml_sigmoid(ctx0, beta);
+
+ ggml_tensor * alpha = build_lora_mm(layer.ssm_alpha, cur, layer.ssm_alpha_s);
+ alpha = ggml_reshape_3d(ctx0, alpha, num_v_heads, n_tokens, n_seqs);
+
+ ggml_tensor * gate = ggml_softplus(ctx0, ggml_add(ctx0, alpha, layer.ssm_dt));
+ gate = ggml_mul(ctx0, gate, layer.ssm_a);
+ gate = ggml_reshape_4d(ctx0, gate, 1, num_v_heads, n_tokens, n_seqs);
+
+ const int64_t conv_kernel_size = layer.ssm_conv1d->ne[0];
+ const int64_t conv_channels = d_inner + 2 * num_k_heads * head_k_dim;
+
+ ggml_tensor * conv_states = ggml_fill(ctx0, ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, conv_kernel_size - 1, conv_channels, n_seqs), 0.0f);
+ ggml_tensor * conv_input = ggml_concat(ctx0, conv_states, ggml_transpose(ctx0, qkv_mixed), 0);
+
+ ggml_tensor * conv_out = ggml_silu(ctx0, ggml_ssm_conv(ctx0, conv_input, layer.ssm_conv1d));
+ cb(conv_out, "conv_output_silu", il);
+
+ const int64_t qkv_dim = head_k_dim * num_k_heads * 2 + head_v_dim * num_v_heads;
+ const int64_t nb1_qkv = ggml_row_size(conv_out->type, qkv_dim);
+
+ ggml_tensor * q_conv = ggml_view_4d(ctx0, conv_out, head_k_dim, num_k_heads, n_tokens, n_seqs,
+ ggml_row_size(conv_out->type, head_k_dim), nb1_qkv, nb1_qkv * n_tokens,
+ 0);
+ ggml_tensor * k_conv = ggml_view_4d(ctx0, conv_out, head_k_dim, num_k_heads, n_tokens, n_seqs,
+ ggml_row_size(conv_out->type, head_k_dim), nb1_qkv, nb1_qkv * n_tokens,
+ head_k_dim * num_k_heads * ggml_element_size(conv_out));
+ ggml_tensor * v_conv = ggml_view_4d(ctx0, conv_out, head_v_dim, num_v_heads, n_tokens, n_seqs,
+ ggml_row_size(conv_out->type, head_v_dim), nb1_qkv, nb1_qkv * n_tokens,
+ ggml_row_size(conv_out->type, 2 * head_k_dim * num_k_heads));
+
+ q_conv = build_gdn_l2_norm(ctx0, q_conv, hparams.f_norm_rms_eps);
+ k_conv = build_gdn_l2_norm(ctx0, k_conv, hparams.f_norm_rms_eps);
+
+ // note: need explicit repeat only if we are not using the fused GDN
+ if (num_k_heads != num_v_heads && (!cparams.fused_gdn_ar || !cparams.fused_gdn_ch)) {
+ GGML_ASSERT(num_v_heads % num_k_heads == 0);
+ q_conv = ggml_repeat_4d(ctx0, q_conv, head_k_dim, num_v_heads, n_tokens, n_seqs);
+ k_conv = ggml_repeat_4d(ctx0, k_conv, head_k_dim, num_v_heads, n_tokens, n_seqs);
+ }
+
+ ggml_tensor * state = ggml_fill(ctx0, ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, head_v_dim, head_v_dim, num_v_heads, n_seqs), 0.0f);
+
+ ggml_tensor * output = build_delta_net(q_conv, k_conv, v_conv, gate, beta, state, il).first;
+ cb(output, "attn_output", il);
+
+ // gated normalization
+ ggml_tensor * z_4d = ggml_reshape_4d(ctx0, z, head_v_dim, num_v_heads, n_tokens, n_seqs);
+ output = build_norm(output, layer.ssm_norm, nullptr, LLM_NORM_RMS, il);
+ output = ggml_mul(ctx0, output, ggml_silu(ctx0, z_4d));
+
+ output = ggml_reshape_3d(ctx0, output, head_v_dim * num_v_heads, n_tokens, n_seqs);
+
+ cur = build_lora_mm(layer.ssm_out, output, layer.ssm_out_s);
+ cb(cur, "linear_attn_out", il);
+
+ return ggml_reshape_2d(ctx0, cur, n_embd, n_tokens);
+}
+
+ggml_tensor * llama_model_clef::graph::build_head_norm(ggml_tensor * cur, const norm & n) {
+ return build_norm(cur, n.w, n.b, LLM_NORM, -1);
+}
+
+// q: [n_embd_h, n_q], kv: [n_embd_h, n_kv], no mask
+ggml_tensor * llama_model_clef::graph::build_head_attn(ggml_tensor * q, ggml_tensor * kv, const attn & a) {
+ const int64_t n_embd_h = q->ne[0];
+ const int64_t n_head_h = model.n_head_decision;
+ const int64_t d_head = n_embd_h / n_head_h;
+ const int64_t n_q = q->ne[1];
+ const int64_t n_kv = kv->ne[1];
+
+ ggml_tensor * Qcur = ggml_add(ctx0, ggml_mul_mat(ctx0, a.wq, q), a.bq);
+ ggml_tensor * Kcur = ggml_add(ctx0, ggml_mul_mat(ctx0, a.wk, kv), a.bk);
+ ggml_tensor * Vcur = ggml_add(ctx0, ggml_mul_mat(ctx0, a.wv, kv), a.bv);
+
+ Qcur = ggml_permute(ctx0, ggml_reshape_3d(ctx0, Qcur, d_head, n_head_h, n_q), 0, 2, 1, 3); // [d_head, n_q, n_head]
+ Kcur = ggml_permute(ctx0, ggml_reshape_3d(ctx0, Kcur, d_head, n_head_h, n_kv), 0, 2, 1, 3); // [d_head, n_kv, n_head]
+ Vcur = ggml_permute(ctx0, ggml_reshape_3d(ctx0, Vcur, d_head, n_head_h, n_kv), 1, 2, 0, 3); // [n_kv, d_head, n_head]
+ Vcur = ggml_cont(ctx0, Vcur);
+
+ ggml_tensor * kq = ggml_mul_mat(ctx0, Kcur, Qcur); // [n_kv, n_q, n_head]
+ kq = ggml_soft_max_ext(ctx0, kq, nullptr, 1.0f / sqrtf(float(d_head)), 0.0f);
+
+ ggml_tensor * cur = ggml_mul_mat(ctx0, Vcur, kq); // [d_head, n_q, n_head]
+ cur = ggml_cont_2d(ctx0, ggml_permute(ctx0, cur, 0, 2, 1, 3), n_embd_h, n_q);
+
+ return ggml_add(ctx0, ggml_mul_mat(ctx0, a.wo, cur), a.bo);
+}
+
+ggml_tensor * llama_model_clef::graph::build_head_ffn(ggml_tensor * cur, const head_layer & layer) {
+ cur = build_head_norm(cur, layer.ffn_norm);
+ cur = ggml_add(ctx0, ggml_mul_mat(ctx0, layer.ffn_up, cur), layer.ffn_up_b);
+ cur = ggml_gelu_erf(ctx0, cur);
+ return ggml_add(ctx0, ggml_mul_mat(ctx0, layer.ffn_down, cur), layer.ffn_down_b);
+}
+
+// ref: JointSchemaHead in joint_schema_model.py of the model repo
+ggml_tensor * llama_model_clef::graph::build_head(ggml_tensor * hidden, input_decision * inp) {
+ const int64_t n_questions = inp->n_questions;
+ const int64_t n_options = inp->n_options;
+
+ inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
+ inp->pool_q = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, n_questions);
+ inp->pool_o = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_tokens, n_options);
+ inp->question_type = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_questions);
+ inp->option_question = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_options);
+ inp->option_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_options, n_questions);
+ inp->status = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1);
+ for (ggml_tensor * t : { inp->tokens, inp->pool_q, inp->pool_o, inp->question_type, inp->option_question, inp->option_mask, inp->status }) {
+ ggml_set_input(t);
+ }
+
+ hidden = build_head_norm(hidden, model.hidden_norm);
+
+ ggml_tensor * memory = ggml_mul_mat(ctx0, model.proj_memory, hidden); // [n_embd_h, n_tokens]
+
+ const int64_t n_embd_h = memory->ne[0];
+
+ // hidden state of the last token
+ ggml_tensor * global = ggml_view_2d(ctx0, hidden, n_embd, 1, hidden->nb[1], (n_tokens - 1) * hidden->nb[1]);
+
+ // mean of the hidden states over each span
+ ggml_tensor * hidden_t = ggml_cont(ctx0, ggml_transpose(ctx0, hidden));
+ ggml_tensor * question = ggml_mul_mat(ctx0, hidden_t, inp->pool_q); // [n_embd, n_questions]
+ ggml_tensor * opt_ctx = ggml_mul_mat(ctx0, hidden_t, inp->pool_o); // [n_embd, n_options]
+
+ // mean of the output embeddings of the tokens of each option
+ ggml_tensor * lexical = ggml_get_rows(ctx0, model.output, inp->tokens);
+ lexical = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, lexical)), inp->pool_o); // [n_embd, n_options]
+
+ // one query per option
+ ggml_tensor * options = ggml_mul_mat(ctx0, model.proj_option_context, opt_ctx);
+ options = ggml_add(ctx0, options, ggml_mul_mat(ctx0, model.proj_option_lexical, lexical));
+ options = ggml_add(ctx0, options, ggml_get_rows(ctx0, ggml_mul_mat(ctx0, model.proj_option_question, question), inp->option_question));
+
+ // the options read the prompt
+ for (uint32_t il = 0; il < model.n_layer_routing; ++il) {
+ const auto & layer = model.head_layers[il];
+
+ ggml_tensor * cur = build_head_attn(
+ build_head_norm(options, layer.cross_norm),
+ build_head_norm(memory, layer.cross_norm_kv),
+ layer.cross_attn);
+ options = ggml_add(ctx0, options, cur);
+ options = ggml_add(ctx0, options, build_head_ffn(options, layer));
+ }
+ cb(options, "decision_options", -1);
+
+ // one vector per question: its text, a summary of its options, the end of the prompt and its type
+ ggml_tensor * fields = ggml_mul_mat(ctx0, model.proj_question, question); // [n_embd_h, n_questions]
+ {
+ // each question weights its own options
+ ggml_tensor * weights = ggml_mul_mat(ctx0, options, fields); // [n_options, n_questions]
+ weights = ggml_scale(ctx0, weights, 1.0f / sqrtf(float(n_embd_h)));
+ weights = ggml_soft_max(ctx0, ggml_add(ctx0, weights, inp->option_mask));
+
+ ggml_tensor * summary = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, options)), weights);
+
+ fields = ggml_add(ctx0, fields, build_head_norm(summary, model.option_summary_norm));
+ fields = ggml_add(ctx0, fields, ggml_mul_mat(ctx0, model.proj_global, global));
+ fields = ggml_add(ctx0, fields, ggml_get_rows(ctx0, model.type_embd, inp->question_type));
+ }
+
+ // the questions read each other and the prompt
+ for (uint32_t il = model.n_layer_routing; il < model.head_layers.size(); ++il) {
+ const auto & layer = model.head_layers[il];
+
+ ggml_tensor * cur = build_head_norm(fields, layer.self_norm);
+ fields = ggml_add(ctx0, fields, build_head_attn(cur, cur, layer.self_attn));
+
+ cur = build_head_norm(fields, layer.cross_norm);
+ fields = ggml_add(ctx0, fields, build_head_attn(cur, memory, layer.cross_attn));
+
+ fields = ggml_add(ctx0, fields, build_head_ffn(fields, layer));
+ }
+ fields = build_head_norm(fields, model.field_norm);
+ cb(fields, "decision_fields", -1);
+
+ const float eps = 1e-12f;
+
+ // prior: the output embeddings of the option against the question and the end of the prompt
+ ggml_tensor * anchor = ggml_l2_norm(ctx0, ggml_add(ctx0, question, global), eps);
+ anchor = ggml_get_rows(ctx0, anchor, inp->option_question);
+ ggml_tensor * prior = ggml_sum_rows(ctx0, ggml_mul(ctx0, ggml_l2_norm(ctx0, lexical, eps), anchor)); // [1, n_options]
+
+ // joint: each option against the vector of its question
+ options = build_head_norm(options, model.option_norm);
+ ggml_tensor * field = ggml_get_rows(ctx0, fields, inp->option_question); // [n_embd_h, n_options]
+
+ ggml_tensor * cosine = ggml_sum_rows(ctx0, ggml_mul(ctx0, ggml_l2_norm(ctx0, field, eps), ggml_l2_norm(ctx0, options, eps)));
+
+ ggml_tensor * features = ggml_concat(ctx0, field, options, 0);
+ features = ggml_concat(ctx0, features, ggml_mul(ctx0, field, options), 0);
+ features = ggml_concat(ctx0, features, ggml_abs(ctx0, ggml_sub(ctx0, field, options)), 0);
+
+ ggml_tensor * residual = ggml_add(ctx0, ggml_mul_mat(ctx0, model.scorer, features), model.scorer_b);
+ residual = ggml_gelu_erf(ctx0, residual);
+ residual = ggml_add(ctx0, ggml_mul_mat(ctx0, model.scorer_out, residual), model.scorer_out_b); // [1, n_options]
+
+ auto scale = [&](int i) {
+ return ggml_view_1d(ctx0, model.scales, 1, i * ggml_element_size(model.scales));
+ };
+
+ ggml_tensor * joint = ggml_add(ctx0, ggml_mul(ctx0, cosine, scale(1)), residual);
+
+ return ggml_add(ctx0, ggml_mul(ctx0, prior, scale(0)), ggml_mul(ctx0, joint, scale(2)));
+}
diff --git a/src/models/models.h b/src/models/models.h
index 8f04933a6..387a4adcb 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -2391,6 +2391,99 @@ struct llama_model_qwen35 : public llama_model_base {
};
+// Qwen3.5 backbone evaluated without memory, with a joint decision head on top of its hidden states
+struct llama_model_clef : public llama_model_qwen35 {
+ llama_model_clef(const struct llama_model_params & params) : llama_model_qwen35(params) {}
+ void load_arch_hparams(llama_model_loader & ml) override;
+ void load_arch_tensors(llama_model_loader & ml) override;
+
+ struct attn {
+ ggml_tensor * wq = nullptr;
+ ggml_tensor * bq = nullptr;
+ ggml_tensor * wk = nullptr;
+ ggml_tensor * bk = nullptr;
+ ggml_tensor * wv = nullptr;
+ ggml_tensor * bv = nullptr;
+ ggml_tensor * wo = nullptr;
+ ggml_tensor * bo = nullptr;
+ };
+
+ struct norm {
+ ggml_tensor * w = nullptr;
+ ggml_tensor * b = nullptr;
+ };
+
+ // the first n_layer_routing blocks have no self attention
+ struct head_layer {
+ norm self_norm;
+ attn self_attn;
+ norm cross_norm;
+ norm cross_norm_kv; // routing blocks only
+ attn cross_attn;
+ norm ffn_norm;
+ ggml_tensor * ffn_up = nullptr;
+ ggml_tensor * ffn_up_b = nullptr;
+ ggml_tensor * ffn_down = nullptr;
+ ggml_tensor * ffn_down_b = nullptr;
+ };
+
+ uint32_t n_layer_routing = 0;
+ uint32_t n_layer_joint = 0;
+ uint32_t n_head_decision = 0;
+
+ std::vector<head_layer> head_layers; // routing blocks, then joint blocks
+
+ norm hidden_norm;
+ norm option_summary_norm;
+ norm field_norm;
+ norm option_norm;
+
+ ggml_tensor * proj_memory = nullptr;
+ ggml_tensor * proj_question = nullptr;
+ ggml_tensor * proj_option_question = nullptr;
+ ggml_tensor * proj_global = nullptr;
+ ggml_tensor * proj_option_context = nullptr;
+ ggml_tensor * proj_option_lexical = nullptr;
+ ggml_tensor * scales = nullptr; // prior scale, joint scale, residual gate
+ ggml_tensor * scorer = nullptr;
+ ggml_tensor * scorer_b = nullptr;
+ ggml_tensor * scorer_out = nullptr;
+ ggml_tensor * scorer_out_b = nullptr;
+
+ class input_decision;
+
+ struct graph : public llm_build_delta_net_base {
+ graph(const llama_model & model, const llm_graph_params & params);
+ private:
+ llm_graph_input_attn_no_cache * build_attn_inp_causal();
+
+ ggml_tensor * build_layer_attn(
+ llm_graph_input_attn_no_cache * inp_attn,
+ ggml_tensor * cur,
+ ggml_tensor * inp_pos,
+ int * sections,
+ int il);
+
+ ggml_tensor * build_layer_attn_linear(
+ ggml_tensor * cur,
+ int il);
+
+ // hidden: [n_embd, n_tokens], returns one score per option: [1, n_options]
+ ggml_tensor * build_head(
+ ggml_tensor * hidden,
+ input_decision * inp);
+
+ ggml_tensor * build_head_norm(ggml_tensor * cur, const norm & n);
+ ggml_tensor * build_head_attn(ggml_tensor * q, ggml_tensor * kv, const attn & a);
+ ggml_tensor * build_head_ffn (ggml_tensor * cur, const head_layer & layer);
+
+ const llama_model_clef & model;
+ };
+
+ std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
+
struct llama_model_qwen4exp : public llama_model_base {
llama_model_qwen4exp(const struct llama_model_params & params) : llama_model_base(params) {}
diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp
index e3731e6f8..93bf47baa 100644
--- a/tests/test-llama-archs.cpp
+++ b/tests/test-llama-archs.cpp
@@ -685,6 +685,9 @@ static bool arch_supported(const llm_arch arch) {
if (arch == LLM_ARCH_PLM) {
return false; // TODO tensor shapes
}
+ if (arch == LLM_ARCH_CLEF) {
+ return false; // TODO decision head tensors
+ }
if (arch == LLM_ARCH_DEEPSEEK2OCR) {
return false;
}
diff --git a/tools/server/README.md b/tools/server/README.md
index 47953cf11..4a37bc2f3 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -1688,13 +1688,15 @@ Follows the [TypeSafe API](https://docs.typesafe.ai/api), streaming is not suppo
- `score`: An array of 2 to 10 level descriptions, lowest level first.
- `noul`: Optional. An object with the descriptions of `true` and `false`.
-The questions of a request are answered independently, an answer does not depend on the other questions.
+The questions of a request are answered independently, an answer does not depend on the other questions. The exception is clef: it reads all the questions in one prompt and decides them jointly.
-The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for laya. For laya, long questions and options are truncated to the token budget the model was trained with.
+The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for laya and clef. For laya, long questions and options are truncated to the token budget the model was trained with.
+
+For laya and clef, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. A server that runs clef only serves this endpoint, text generation is not available.
*Image input:*
-Image input needs a model that supports it (for example: openjev) and its multimodal projector, see `--mmproj`.
+Image input needs a model that supports it (for example: openjev) and its multimodal projector, see `--mmproj`. Image input is not supported yet for clef.
Images can be given in two ways, and both can be used in the same request:
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 4da504e2e..809be94cd 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -143,6 +143,7 @@ struct server_batch {
llama_pos pos;
bool output;
bool is_prompt; // for stats tracking
+ int32_t decision_order = 0;
};
std::vector<token> tokens;
int32_t n_tokens_alloc = 0;
@@ -202,6 +203,11 @@ struct server_batch {
tokens[idx].output = output;
}
+ void set_decision_order(int32_t idx, int32_t order) {
+ GGML_ASSERT(idx >= 0 && idx < (int32_t)tokens.size());
+ tokens[idx].decision_order = order;
+ }
+
// render the sub-batch [off, off + n_tokens) into view, index i in view is index off + i here
void render(int32_t off, int32_t n_tokens) {
GGML_ASSERT(off >= 0 && off < size());
@@ -217,6 +223,7 @@ struct server_batch {
} else {
view.add(t.token, t.pos, t.id_slot, t.output);
}
+ view.tokens.back().decision_order = t.decision_order;
}
}
};
@@ -452,6 +459,11 @@ struct server_slot {
bool can_batch_with(server_slot & other_slot) const {
GGML_ASSERT(task);
+ // a joint decision head reads the whole batch
+ if (!task->decision.order.empty() || !other_slot.task->decision.order.empty()) {
+ return false;
+ }
+
return task->type == other_slot.task->type
&& inp_embd.size() == other_slot.inp_embd.size()
&& are_lora_equal(lora, other_slot.lora);
@@ -2282,6 +2294,16 @@ private:
return i >= 0 && i < (int32_t) idx.size() ? llama_get_embeddings_ith(slot.ctx_tgt, idx[i]) : nullptr;
};
+ // joint head (decision model): the scores are the first rows
+ for (int32_t i = 0; i < decision.n_scores; i++) {
+ const float * embd = i < (int32_t) idx.size() ? llama_get_embeddings_ith(slot.ctx_tgt, idx[i]) : nullptr;
+ if (embd == nullptr) {
+ send_error(slot, "failed to get embeddings", ERROR_TYPE_SERVER);
+ return;
+ }
+ res->scores.push_back(embd[0]);
+ }
+
const int32_t n_embd_out = llama_model_n_embd_out(model_tgt);
const int32_t n_pointer = n_embd_out / 2;
const float * embd_q = decision.pointer >= 0 ? get_embd(decision.pointer) : nullptr;
@@ -3716,6 +3738,9 @@ private:
/* pos = */ slot.prompt.tokens.pos_next(),
/* output = */ slot.need_embd(),
/* is_prompt = */ true);
+ if (!slot.task->decision.order.empty()) {
+ batch.set_decision_order(batch.size() - 1, slot.task->decision.order[slot.prompt.n_tokens()]);
+ }
slot.prompt.tokens.push_back(cur_tok);
// break at the last user message, or at user messages at least min step past the last checkpoint
@@ -5436,16 +5461,23 @@ void server_routes::init_routes() {
return res;
}
- // one task per variant of each question
+ // one task per variant of each question, or one task for all the questions
auto & rd = res->rd;
{
std::vector<server_task> tasks;
- for (const auto & question : questions) {
- for (size_t variant = 0; variant < decision.n_variants(question); variant++) {
- server_task task = server_task(SERVER_TASK_TYPE_DECISION);
- task.id = rd.get_new_id();
- decision.fill_task(state, questions, question, variant, files, ctx_server.mctx, ctx_server.init_opt, task);
- tasks.push_back(std::move(task));
+ if (decision.is_joint()) {
+ server_task task = server_task(SERVER_TASK_TYPE_DECISION);
+ task.id = rd.get_new_id();
+ decision.fill_task_joint(state, questions, task);
+ tasks.push_back(std::move(task));
+ } else {
+ for (const auto & question : questions) {
+ for (size_t variant = 0; variant < decision.n_variants(question); variant++) {
+ server_task task = server_task(SERVER_TASK_TYPE_DECISION);
+ task.id = rd.get_new_id();
+ decision.fill_task(state, questions, question, variant, files, ctx_server.mctx, ctx_server.init_opt, task);
+ tasks.push_back(std::move(task));
+ }
}
}
if (decision.can_share_prompt()) {
@@ -5466,9 +5498,18 @@ void server_routes::init_routes() {
json answers = json::object();
int32_t n_tokens = 0;
size_t i_result = 0;
+ size_t i_score = 0;
for (const auto & question : questions) {
std::vector<std::vector<float>> scores;
- for (size_t variant = 0; variant < decision.n_variants(question); variant++) {
+ if (decision.is_joint()) {
+ // one result with the scores of all the questions, in order
+ auto * result = dynamic_cast<server_task_result_decision *>(all_results.results[0].get());
+ GGML_ASSERT(result != nullptr && i_score + question.options.size() <= result->scores.size());
+ scores.emplace_back(result->scores.begin() + i_score, result->scores.begin() + i_score + question.options.size());
+ i_score += question.options.size();
+ n_tokens = result->n_tokens;
+ }
+ for (size_t variant = 0; !decision.is_joint() && variant < decision.n_variants(question); variant++) {
auto * result = dynamic_cast<server_task_result_decision *>(all_results.results[i_result++].get());
GGML_ASSERT(result != nullptr);
scores.push_back(result->scores);
diff --git a/tools/server/server-decision.cpp b/tools/server/server-decision.cpp
index 70ba15473..522aa2bed 100644
--- a/tools/server/server-decision.cpp
+++ b/tools/server/server-decision.cpp
@@ -1,5 +1,7 @@
#include "server-decision.h"
+#include "../../src/llama-ext.h" // staging API: llama_decision_order
+
#include <algorithm>
#include <cmath>
#include <regex>
@@ -116,6 +118,10 @@ void server_decision_context::init(const llama_model * model) {
throw std::runtime_error("decision model has no valid max_head_tokens");
}
n_options_max = 255;
+ } else if (model_type == COMMON_DECISION_TYPE_CLEF) {
+ n_options_max = 255;
+ noul_true_first = true;
+ choice_sorted = true;
} else {
throw std::runtime_error("unsupported decision model type: " + type_name);
}
@@ -163,6 +169,11 @@ std::vector<server_decision_question> server_decision_context::parse_questions(c
for (const auto & [key, description] : criteria.items()) {
question.options.push_back({key, description});
}
+ if (choice_sorted) {
+ std::sort(question.options.begin(), question.options.end(), [](const auto & a, const auto & b) {
+ return a.key < b.key;
+ });
+ }
} else if (type_name == "score") {
question.type = SERVER_DECISION_QUESTION_SCORE;
if (!criteria.is_array() || criteria.size() < 2 || criteria.size() > 10) {
@@ -542,6 +553,90 @@ void server_decision_context::fill_task_laya(llama_tokens & tokens, const server
task.decision.column = question.type;
}
+//
+// joint prompt (clef)
+//
+
+// given to the template: text between the pieces of the prompt, and at the start of the span of a question or of an option
+static const std::string CLEF_MARKER = "<<clef:";
+static const std::string CLEF_SEP = "<<clef:sep>>";
+static const std::string CLEF_MARK_QUESTION = "<<clef:question>>";
+static const std::string CLEF_MARK_OPTION = "<<clef:option>>";
+
+void server_decision_context::fill_task_joint(const json & state, const std::vector<server_decision_question> & questions, server_task & task) const {
+ json inp_questions = json::array();
+ for (const auto & question : questions) {
+ json options = json::array();
+ for (const auto & opt : question.options) {
+ options.push_back(json{
+ {"key", opt.key},
+ {"description", opt.description},
+ });
+ }
+ inp_questions.push_back(json{
+ {"id", question.id},
+ {"type", decision_question_type_name(question.type)},
+ {"instructions", question.instructions},
+ {"options", options},
+ });
+ }
+
+ // the template is given raw JSON values with sorted keys, and no marker in the input
+ json inp = json{
+ {"state", state},
+ {"questions", inp_questions},
+ };
+ inp = decision_replace_text(decision_sort_keys(inp), CLEF_MARKER, "<<clef ");
+ inp["sep"] = CLEF_SEP;
+ inp["mark_question"] = CLEF_MARK_QUESTION;
+ inp["mark_option"] = CLEF_MARK_OPTION;
+
+ jinja::context ctx(tmpl->source());
+ jinja::global_from_json(ctx, inp, false);
+ jinja::runtime runtime(ctx);
+ const jinja::value results = runtime.execute(tmpl->prog);
+ const std::string prompt = jinja::runtime::gather_string_parts(results)->as_string().str();
+
+ // the model was trained with the pieces tokenized one by one
+ llama_tokens tokens;
+ size_t i_question = 0;
+ for (std::string piece : string_split(prompt, CLEF_SEP)) {
+ int32_t order = LLAMA_DECISION_ORDER_NONE;
+ if (string_starts_with(piece, CLEF_MARK_QUESTION)) {
+ piece = piece.substr(CLEF_MARK_QUESTION.size());
+ if (i_question >= questions.size()) {
+ throw std::runtime_error("unexpected layout of the decision prompt");
+ }
+ switch (questions[i_question++].type) {
+ case SERVER_DECISION_QUESTION_NOUL: order = LLAMA_DECISION_ORDER_QUESTION_NOUL; break;
+ case SERVER_DECISION_QUESTION_CHOICE: order = LLAMA_DECISION_ORDER_QUESTION_CHOICE; break;
+ case SERVER_DECISION_QUESTION_SCORE: order = LLAMA_DECISION_ORDER_QUESTION_SCORE; break;
+ }
+ } else if (string_starts_with(piece, CLEF_MARK_OPTION)) {
+ piece = piece.substr(CLEF_MARK_OPTION.size());
+ order = LLAMA_DECISION_ORDER_OPTION;
+ task.decision.n_scores++;
+ }
+
+ const llama_tokens piece_tokens = common_tokenize(vocab, piece, false, true);
+ if (order != LLAMA_DECISION_ORDER_NONE && piece_tokens.empty()) {
+ throw std::invalid_argument("the instructions and the options of a question must not be empty");
+ }
+ tokens.insert(tokens.end(), piece_tokens.begin(), piece_tokens.end());
+ task.decision.order.resize(tokens.size(), order);
+ }
+
+ size_t n_options = 0;
+ for (const auto & question : questions) {
+ n_options += question.options.size();
+ }
+ if (i_question != questions.size() || (size_t) task.decision.n_scores != n_options) {
+ throw std::runtime_error("unexpected layout of the decision prompt");
+ }
+
+ task.tokens = server_tokens(tokens, false);
+}
+
//
// answer
//
@@ -609,6 +704,10 @@ json server_decision_context::format_answer(const server_decision_question & que
if (s.size() != n) {
throw std::runtime_error("decision result does not match the number of options");
}
+ // a joint head returns NaN if it could not use the decision order
+ if (std::any_of(s.begin(), s.end(), [](float v) { return std::isnan(v); })) {
+ throw std::runtime_error("the model could not evaluate the decision");
+ }
const float score_max = *std::max_element(s.begin(), s.end());
std::vector<double> p(n);
double sum = 0.0;
diff --git a/tools/server/server-decision.h b/tools/server/server-decision.h
index f443e477e..93480ba26 100644
--- a/tools/server/server-decision.h
+++ b/tools/server/server-decision.h
@@ -48,7 +48,13 @@ struct server_decision_context {
}
}
+ // true if all the questions of a request go in one prompt, see fill_task_joint()
+ bool is_joint() const {
+ return type == COMMON_DECISION_TYPE_CLEF;
+ }
+
// true if the prompt of the model has a place for images
+ // TODO: clef needs token and embedding entries in the same batch, see https://github.com/ggml-org/llama.cpp/pull/29622
bool can_use_images() const {
switch (type) {
case COMMON_DECISION_TYPE_OPENJEV:
@@ -80,6 +86,9 @@ struct server_decision_context {
const mtmd_helper_init_opt & init_opt,
server_task & task) const;
+ // set the prompt of all the questions, the result has the scores of all their options, in order
+ void fill_task_joint(const json & state, const std::vector<server_decision_question> & questions, server_task & task) const;
+
// scores: the raw model outputs of each variant
json format_answer(const server_decision_question & question, const std::vector<std::vector<float>> & scores) const;
@@ -90,6 +99,7 @@ private:
std::map<std::string, float> temperatures; // "<type>" or "<type>.<n_options bucket>"
size_t n_options_max = 0;
bool noul_true_first = false; // noul options are [true, false] instead of [false, true]
+ bool choice_sorted = false; // choice options are in the order of their keys
// OPENJEV, LEV, NIMBLE
std::vector<llama_token> labels;
diff --git a/tools/server/server-task.h b/tools/server/server-task.h
index 8c5fa9a68..752595b0f 100644
--- a/tools/server/server-task.h
+++ b/tools/server/server-task.h
@@ -192,6 +192,11 @@ struct server_task {
}
return pos;
}
+
+ // for a joint head: one value per prompt token, see llama_batch_ext_set_decision_order()
+ // the scores are the first n_scores rows of the embeddings
+ std::vector<int32_t> order;
+ int32_t n_scores = 0;
};
decision decision;
@@ -212,7 +217,7 @@ struct server_task {
case SERVER_TASK_TYPE_RERANK:
return true;
case SERVER_TASK_TYPE_DECISION:
- return !decision.markers.empty();
+ return !decision.markers.empty() || !decision.order.empty();
default:
return false;
}