Commit 74bca3950 for llama.cpp
commit 74bca3950054eed1795f525cb9cf06bfe796492d
Author: Aman Gupta <amangupta052@gmail.com>
Date: Sun Oct 25 09:43:51 2026 +0800
model: DSV4.1 flash (#31086)
* model : add DeepSeek-V4.1-Flash (deepseek41)
* deepseek41 : remove MXFP8, convert the Engram table to BF16
* deepseek41 : compute the compressed-KV read layers once in the cache
* deepseek41 : use get_cells() for the shared-history check, drop llama_kv_cache::seq_pos_tok_le
* deepseek41 : gather the Engram rows in-graph like qwen4exp, drop the host gather
* deepseek41 : keep the Engram hash constants in hparams.ple_*, one module per Engram layer
* deepseek41 : drop the LLAMA_DSV41_KV_MIRROR toggle
* deepseek41 : keep il last in build_hc_mixes
* deepseek41 : sort the top-k rows before the sparse gather
* tests : add deepseek41 to the Metal fusion baseline
---------
Co-authored-by: Daniel Han <danielhanchen@gmail.com>
diff --git a/common/chat.cpp b/common/chat.cpp
index 99ed74faf..227621761 100644
--- a/common/chat.cpp
+++ b/common/chat.cpp
@@ -428,6 +428,41 @@ std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const json & messa
if (message.contains("tool_call_id")) {
msg.tool_call_id = message.at("tool_call_id");
}
+ {
+ json extra = json::object();
+ for (const auto & [k, v] : message.items()) {
+ if (k != "role" && k != "content" && k != "tool_calls" && k != "reasoning_content" &&
+ k != "name" && k != "tool_call_id") {
+ extra[k] = v;
+ }
+ }
+ if (has_tool_calls) {
+ // per tool call: keys besides type/function/id, and function keys besides name/arguments
+ json tc_extra = json::array();
+ bool any = false;
+ for (const auto & tool_call : message.at("tool_calls")) {
+ json e = json::object();
+ for (const auto & [k, v] : tool_call.items()) {
+ if (k != "type" && k != "function" && k != "id") {
+ e[k] = v;
+ }
+ }
+ for (const auto & [k, v] : tool_call.at("function").items()) {
+ if (k != "name" && k != "arguments") {
+ e["function"][k] = v;
+ }
+ }
+ any = any || !e.empty();
+ tc_extra.push_back(e);
+ }
+ if (any) {
+ extra["__tool_call_extras"] = tc_extra;
+ }
+ }
+ if (!extra.empty()) {
+ msg.extra_json = extra.dump();
+ }
+ }
msgs.push_back(msg);
}
@@ -516,6 +551,50 @@ static json render_message_to_json(const std::vector<common_chat_msg> & msgs, co
return messages_inp_normalizer(c).normalize(messages);
}
+// DeepSeek V4.1 reads non-OpenAI message keys and joins content blocks with "\n\n" around images,
+// so keep media markers as separate text parts and restore the extra keys
+static json render_message_to_json_deepseek_v4_1(const std::vector<common_chat_msg> & msgs, const jinja::caps & c) {
+ json messages = json::array();
+ for (const auto & msg : msgs) {
+ json jmsg;
+ if (msg.contains_media() && msg.content.empty()) {
+ common_chat_msg typed = msg;
+ for (auto & part : typed.content_parts) {
+ part.type = "text";
+ }
+ jmsg = typed.to_json_oaicompat(/* concat_typed_text= */ false);
+ } else {
+ jmsg = msg.to_json_oaicompat(/* concat_typed_text= */ false);
+ }
+ if (!msg.extra_json.empty()) {
+ const json extra = json::parse(msg.extra_json);
+ for (const auto & [k, v] : extra.items()) {
+ if (k == "__tool_call_extras") {
+ if (!jmsg.contains("tool_calls")) {
+ continue;
+ }
+ auto & tcs = jmsg.at("tool_calls");
+ for (size_t i = 0; i < tcs.size() && i < v.size(); i++) {
+ for (const auto & [tk, tv] : v[i].items()) {
+ if (tk == "function") {
+ for (const auto & [fk, fv] : tv.items()) {
+ tcs[i]["function"][fk] = fv;
+ }
+ } else {
+ tcs[i][tk] = tv;
+ }
+ }
+ }
+ } else if (!jmsg.contains(k)) {
+ jmsg[k] = v;
+ }
+ }
+ }
+ messages.push_back(std::move(jmsg));
+ }
+ return messages_inp_normalizer(c).normalize(messages);
+}
+
// DEPRECATED: only used in tests
json common_chat_msgs_to_json_oaicompat(const std::vector<common_chat_msg> & msgs, bool concat_typed_text) {
jinja::caps c;
@@ -1208,11 +1287,13 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const
// DeepSeek V3.2/V4 format detection: template defines dsml_token and uses it for tool calls.
// The template source contains the token as a variable assignment, not as a literal in markup.
// V3.2 names the tool call block "function_calls", V4 names it "tool_calls".
- if (src.find("dsml_token") != std::string::npos &&
- src.find("DSML") != std::string::npos &&
- (src.find("function_calls") != std::string::npos ||
- src.find("tool_calls") != std::string::npos)) {
- LOG_DBG("Using specialized template: DeepSeek V3.2/V4\n");
+ // V4.1 tag names have a leading space and its template does not name the dsml_token variable.
+ if ((src.find("dsml_token") != std::string::npos &&
+ src.find("DSML") != std::string::npos &&
+ (src.find("function_calls") != std::string::npos ||
+ src.find("tool_calls") != std::string::npos)) ||
+ src.find("｜DSML｜ calls>") != std::string::npos) {
+ LOG_DBG("Using specialized template: DeepSeek V3.2/V4/V4.1\n");
return common_chat_params_init_deepseek_v3_2;
}
@@ -1313,7 +1394,9 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_
workaround::trim_all_content(trimmed_messages);
messages_to_render = &trimmed_messages;
}
- params.messages = render_message_to_json(*messages_to_render, tmpl.original_caps());
+ params.messages = src.find("｜DSML｜ calls>") != std::string::npos ?
+ render_message_to_json_deepseek_v4_1(*messages_to_render, tmpl.original_caps()) :
+ render_message_to_json(*messages_to_render, tmpl.original_caps());
params.tool_choice = inputs.tool_choice;
params.reasoning_format = inputs.reasoning_format;
params.enable_thinking = inputs.enable_thinking;
diff --git a/common/chat.h b/common/chat.h
index dcff94919..29aaf8b68 100644
--- a/common/chat.h
+++ b/common/chat.h
@@ -94,6 +94,8 @@ struct common_chat_msg {
std::string reasoning_content;
std::string tool_name;
std::string tool_call_id;
+ // non-OpenAI message keys as a JSON object string, only for templates that read them (DeepSeek V4.1)
+ std::string extra_json;
common_json to_json_oaicompat(bool concat_typed_text = false) const;
diff --git a/common/parsers/deepseek.cpp b/common/parsers/deepseek.cpp
index 7ee38ce6d..dac180715 100644
--- a/common/parsers/deepseek.cpp
+++ b/common/parsers/deepseek.cpp
@@ -1,4 +1,5 @@
#include "parsers.h"
+#include "log.h"
// The DeepSeek V4 reference implementation renders consecutive tool results into a single
// user block, ordered by the tool call order of the preceding assistant message (matched
@@ -75,13 +76,30 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ
// non-thinking generation prompt ends with a bare </think> instead of an empty
// <think></think> pair.
const bool is_v4 = tmpl.source().find("function_calls") == std::string::npos;
+ // V4.1: DSML tag names carry a leading space, e.g. <｜DSML｜ calls>
+ const bool is_v41 = is_v4 && tmpl.source().find("｜DSML｜ calls>") != std::string::npos;
std::optional<json> adjusted_messages;
if (is_v4) {
adjusted_messages = deepseek_v4_sort_tool_results(inputs.messages);
}
- auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
+ // V4.1 can declare tools on a system message instead of top-level "tools": parse calls to those too.
+ json tools = inputs.tools;
+ if (is_v41 && !(tools.is_array() && !tools.empty()) && inputs.messages.is_array()) {
+ for (const auto & msg : inputs.messages) {
+ if (msg.value("role", "") == "system" && msg.contains("tools") && msg.at("tools").is_array()) {
+ if (!tools.is_array()) {
+ tools = json::array();
+ }
+ for (const auto & tool : msg.at("tools")) {
+ tools.push_back(tool);
+ }
+ }
+ }
+ }
+
+ auto has_tools = tools.is_array() && !tools.empty();
auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);
@@ -90,17 +108,39 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ
if (is_v4 && has_response_format) {
additional_context = json{ { "response_format", inputs.json_schema } };
}
+ // V4.1 raises on anything but low/high/max or 1-100: map minimal -> low, xhigh -> max, medium -> default
+ if (is_v41 && inputs.extra_context.is_object() && inputs.extra_context.contains("reasoning_effort") &&
+ inputs.extra_context.at("reasoning_effort").is_string()) {
+ const std::string effort = inputs.extra_context.at("reasoning_effort").get<std::string>();
+ json mapped;
+ if (effort == "minimal") {
+ mapped = "low";
+ } else if (effort == "xhigh") {
+ mapped = "max";
+ } else if (effort == "medium") {
+ mapped = nullptr;
+ }
+ if (effort == "minimal" || effort == "xhigh" || effort == "medium") {
+ LOG_DBG("%s: DeepSeek V4.1 reasoning_effort '%s' -> %s\n", __func__, effort.c_str(),
+ mapped.is_null() ? "unset (template default)" : mapped.get<std::string>().c_str());
+ if (!additional_context.has_value()) {
+ additional_context = json::object();
+ }
+ (*additional_context)["reasoning_effort"] = mapped;
+ }
+ }
const std::string DSML = "｜DSML｜";
const std::string THINK_START = "<think>";
const std::string THINK_END = "</think>";
- const std::string TC_BLOCK = is_v4 ? "tool_calls" : "function_calls";
- const std::string FC_START = "<" + DSML + TC_BLOCK + ">";
- const std::string FC_END = "</" + DSML + TC_BLOCK + ">";
- const std::string INVOKE_START = "<" + DSML + "invoke";
- const std::string INVOKE_END = "</" + DSML + "invoke>";
- const std::string PARAM_START = "<" + DSML + "parameter";
- const std::string PARAM_END = "</" + DSML + "parameter>";
+ const std::string TAG_SP = is_v41 ? " " : "";
+ const std::string TC_BLOCK = is_v41 ? "calls" : is_v4 ? "tool_calls" : "function_calls";
+ const std::string FC_START = "<" + DSML + TAG_SP + TC_BLOCK + ">";
+ const std::string FC_END = "</" + DSML + TAG_SP + TC_BLOCK + ">";
+ const std::string INVOKE_START = "<" + DSML + TAG_SP + "invoke";
+ const std::string INVOKE_END = "</" + DSML + TAG_SP + "invoke>";
+ const std::string PARAM_START = "<" + DSML + TAG_SP + "parameter";
+ const std::string PARAM_END = "</" + DSML + TAG_SP + "parameter>";
const std::string GEN_PROMPT = "<｜Assistant｜>";
const std::string TC_SEPARATOR = "\n\n";
@@ -152,7 +192,7 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ
// build tool call section first since we might need it in reasoning
auto tool_choice = p.choice();
if (has_tool_calls) {
- foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
+ foreach_function(tools, [&](size_t tool_index, const json & tool) {
const auto & function = tool.at("function");
std::string name = function.at("name");
@@ -223,7 +263,10 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ
}
if (extract_reasoning && inputs.enable_thinking) {
- reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END);
+ // V4.1 renders a continuation without reasoning as "<｜Assistant｜></think>" even in thinking mode,
+ // so consume the bare THINK_END too or it leaks into the content
+ reasoning = is_v41 ? p.optional((THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END) | p.literal(THINK_END))
+ : p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END);
reasoning_with_tc = THINK_START +
p.reasoning(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START, THINK_END })) +
p.space() + obligatory_tool_calls;
diff --git a/conversion/__init__.py b/conversion/__init__.py
index 35e73b41c..de3e97863 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -70,6 +70,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"LingDSparkModel": "qwen",
"DeepseekV4ForCausalLM": "deepseek",
"DeepseekV4DSparkModel": "deepseek",
+ "DeepseekV41ForCausalLM": "deepseek41",
"DistilBertForMaskedLM": "bert",
"DistilBertForSequenceClassification": "bert",
"DistilBertModel": "bert",
@@ -326,6 +327,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"VllmD3Model": "vllm_d3",
"DeepseekOCR2ForCausalLM": "deepseek",
"DeepseekOCRForCausalLM": "deepseek",
+ "DeepseekV41ForCausalLM": "deepseek41",
"DeepseekV4ForCausalLM": "deepseek",
"Dots3NoteForCausalLM": "dots3",
"Dots3NoteForConditionalGeneration": "dots3",
diff --git a/conversion/deepseek41.py b/conversion/deepseek41.py
new file mode 100644
index 000000000..4b87d5c28
--- /dev/null
+++ b/conversion/deepseek41.py
@@ -0,0 +1,467 @@
+from __future__ import annotations
+
+import json
+import math
+import re
+from pathlib import Path
+from typing import Any, Callable, Iterable, TYPE_CHECKING, cast
+
+import numpy as np
+import torch
+
+if TYPE_CHECKING:
+ from torch import Tensor
+
+from .base import LazyTorchTensor, ModelBase, TextModel, gguf
+from .deepseek import DeepseekV4FlashVisionModel, DeepseekV4Model
+
+
+# Engram hash constants from inference/engram.py, stored in the GGUF: the token map needs the HF tokenizers
+# normalizers and the multipliers numpy's PCG64 stream, neither of which the runtime has.
+
+def _is_prime(n: int) -> bool:
+ if n < 2 or n % 2 == 0:
+ return n == 2
+ return all(n % d for d in range(3, math.isqrt(n) + 1, 2))
+
+
+def engram_primes(n_layers: int, max_ngram_size: int, n_heads: int, vocab_size: int) -> list[list[list[int]]]:
+ """EngramLayout.from_args: [layer][ngram order 2..max][head] bucket moduli, never reused across layers."""
+ seen: set[int] = set()
+ out: list[list[list[int]]] = []
+ for _ in range(n_layers):
+ per_ngram = []
+ for _ in range(max_ngram_size - 1):
+ sizes, cur = [], vocab_size - 1
+ for _ in range(n_heads):
+ cand = cur + 1
+ while not _is_prime(cand) or cand in seen:
+ cand += 1
+ cur = cand
+ seen.add(cur)
+ sizes.append(cur)
+ per_ngram.append(sizes)
+ out.append(per_ngram)
+ return out
+
+
+def engram_multipliers(layer_ids: list[int], max_ngram_size: int, compressed_vocab_size: int) -> list[list[int]]:
+ """compute_hash_multipliers: odd, bounded so that token_id * multiplier cannot overflow int64."""
+ bound = max(1, (np.iinfo(np.int64).max // compressed_vocab_size) // 2)
+ rows = []
+ for layer_id in layer_ids:
+ rng = np.random.default_rng(10007 * layer_id)
+ values = rng.integers(low=0, high=bound, size=(max_ngram_size,), dtype=np.int64)
+ rows.append([int(v) * 2 + 1 for v in values])
+ return rows
+
+
+def engram_compressed_token_map(tokenizer_dir: Path) -> tuple[list[int], int]:
+ """build_compressed_token_map: token id -> compressed id (tokens that normalize alike collapse)."""
+ from tokenizers import Regex, normalizers
+ from transformers import AutoTokenizer
+
+ tokenizer = AutoTokenizer.from_pretrained(tokenizer_dir)
+ sentinel = "\ue000"
+ normalizer = normalizers.Sequence([
+ normalizers.NFKC(),
+ normalizers.NFD(),
+ normalizers.StripAccents(),
+ normalizers.Lowercase(),
+ normalizers.Replace(Regex(r"[ \t\r\n]+"), " "),
+ normalizers.Replace(Regex(r"^ $"), sentinel),
+ normalizers.Strip(),
+ normalizers.Replace(sentinel, " "),
+ ])
+ backend = tokenizer.backend_tokenizer
+ key_to_new: dict[str, int] = {}
+ lookup = [0] * len(tokenizer)
+ for token_id in range(len(tokenizer)):
+ text = backend.decode([token_id], skip_special_tokens=False)
+ if "\ufffd" in text:
+ key = backend.id_to_token(token_id)
+ else:
+ normalized = normalizer.normalize_str(text)
+ key = normalized if normalized else text
+ lookup[token_id] = key_to_new.setdefault(key, len(key_to_new))
+ return lookup, len(key_to_new)
+
+
+class _StreamedBytes:
+ """Raw tensor bytes produced chunk by chunk at write time (an Engram table is ~200 GB in BF16)."""
+ def __init__(self, shape: tuple[int, ...], chunks: Callable[[], Iterable[np.ndarray]]):
+ self.shape = shape
+ self.dtype = np.dtype(np.uint8)
+ self.nbytes = int(np.prod(shape))
+ self._chunks = chunks
+
+ def tofile(self, fout) -> None:
+ written = 0
+ for chunk in self._chunks():
+ chunk = np.ascontiguousarray(chunk, dtype=np.uint8)
+ chunk.tofile(fout)
+ written += chunk.nbytes
+ if written != self.nbytes:
+ raise ValueError(f"streamed {written} bytes, expected {self.nbytes}")
+
+ def byteswap(self, inplace: bool = False):
+ raise NotImplementedError("streamed tensors are written little-endian only")
+
+
+def _e8m0_check(scale_u8: np.ndarray, name: str) -> None:
+ # e4m3 (|x| in [2^-9, 448]) * 2^(e - 127) is exact in BF16 for e in [3, 246]
+ if scale_u8.size and (int(scale_u8.min()) < 3 or int(scale_u8.max()) > 246):
+ raise ValueError(f"{name}: E8M0 scale exponents outside [3, 246], the dequantized values would not be exact")
+
+
+@ModelBase.register("DeepseekV41ForCausalLM")
+class DeepseekV41Model(DeepseekV4Model):
+ """DeepSeek-V4.1 text model, lossless: FP8 (e4m3, 32x32 ue8m0 blocks) linears dequantized to BF16 (F32 with
+ --outtype f32), routed experts copied bit-exact as MXFP4, Engram tables (e4m3 rows, one e8m0 scale per 32) as the
+ exact BF16 lookup (F32 with --outtype f32), router and Engram gate F32. DSpark (mtp.*) is not converted here
+ (--dspark, DeepseekV41DSparkModel); the vision tower goes to an mmproj (--mmproj, DeepseekV41VisionModel).
+ The Engram tables of all layers are written as one per_layer_token_embd (rows concatenated in layer order) with
+ the PLE keys of qwen4exp: absolute head offsets and bucket counts, so the runtime can read it lazily."""
+ model_arch = gguf.MODEL_ARCH.DEEPSEEK41
+ supports_mtp_export = False
+
+ _ENGRAM_EMBED_RE = re.compile(r"layers\.(\d+)\.engram\.embed\.(weight|scale)$")
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.hparams.setdefault("num_hash_layers", 0) # no hash-routed layers in V4.1
+ self._f32 = self.ftype == gguf.LlamaFileType.ALL_F32
+ qc = self.hparams.get("quantization_config") or {}
+ self._fp8_block = tuple(qc.get("weight_block_size") or (32, 32))
+ if qc.get("scale_fmt", "ue8m0") != "ue8m0":
+ raise ValueError(f"unsupported FP8 scale format {qc.get('scale_fmt')!r}")
+ if self.hparams.get("gate_temp", 1.0) != 1.0:
+ raise ValueError("gate_temp != 1 is not supported (the router logits would need dividing by it)")
+
+ # V4.1 does not use the DeepSeek-V4 template; a template shipped with the model is still added by set_vocab
+ self.gguf_writer.remove_key(gguf.Keys.Tokenizer.CHAT_TEMPLATE)
+
+ def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
+ # num_hidden_layers lives in text_config, which is merged only after indexing; MTP stages are skipped anyway
+ return TextModel.index_tensors(self, remote_hf_model_id=remote_hf_model_id)
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ if item[0].startswith("vision."):
+ return None
+ return super().filter_tensors(item)
+
+ def set_gguf_parameters(self):
+ super().set_gguf_parameters()
+ hp = self.hparams
+ w = self.gguf_writer
+ w.add_attention_kv_source_layers(hp["kv_source_layer_ids"])
+ w.add_indexer_source_layers(hp["index_source_layer_ids"])
+ if hp.get("candidate_source_layer_id", -1) >= 0:
+ w.add_indexer_candidate_source_layer(hp["candidate_source_layer_id"])
+ w.add_indexer_candidate_top_k_blocks(hp["candidate_topk_blocks"])
+ w.add_indexer_candidate_block_size(hp["candidate_block_size"])
+
+ # GGUF cannot hold empty arrays: the Engram keys are omitted when the model has no Engram layers
+ layer_ids = list(hp.get("engram_layer_ids") or [])
+ if layer_ids:
+ max_ngram, n_heads = hp["engram_max_ngram_size"], hp["engram_n_heads"]
+ primes = engram_primes(len(layer_ids), max_ngram, n_heads, hp["engram_vocab_size"])
+ for li, layer_primes in enumerate(primes):
+ if sum(p for per in layer_primes for p in per) != hp["engram_num_embeddings"][li]:
+ raise ValueError(f"engram layer {layer_ids[li]}: prime sum != num_embeddings")
+ token_map, cvocab = engram_compressed_token_map(self.dir_model)
+ if cvocab != hp["engram_compressed_vocab_size"] or len(token_map) != hp["vocab_size"]:
+ raise ValueError(f"compressed token map ({len(token_map)} -> {cvocab}) does not match the config")
+ # absolute row offsets into the concatenated table
+ sizes = [p for layer in primes for per in layer for p in per]
+ offsets = [sum(sizes[:i]) for i in range(len(sizes))]
+ w.add_key_value(gguf.Keys.PerLayerEmbedding.LAYERS.format(arch=w.arch), layer_ids,
+ gguf.GGUFValueType.ARRAY, gguf.GGUFValueType.UINT32)
+ w.add_ple_ngram_size(max_ngram)
+ w.add_ple_heads_per_ngram(n_heads)
+ w.add_embedding_length_per_layer_input(hp["engram_head_dim"])
+ w.add_ple_layer_multipliers([m for row in engram_multipliers(layer_ids, max_ngram, cvocab) for m in row])
+ w.add_ple_head_offsets(offsets)
+ w.add_ple_head_vocab_sizes(sizes)
+ w.add_engram_pad_token_id(hp["engram_pad_token_id"])
+ w.add_engram_token_map(token_map)
+
+ def dequant_model(self):
+ bo, bi = self._fp8_block
+
+ def dequant(weight: Tensor, scale: Tensor) -> Tensor:
+ out_f, in_f = weight.shape
+ s = self._e8m0_to_float(scale).repeat_interleave(bo, 0)[:out_f].repeat_interleave(bi, 1)[:, :in_f]
+ return weight.float() * s
+
+ for name in [n for n in self.model_tensors if n.endswith(".scale") and not self._ENGRAM_EMBED_RE.match(n)]:
+ weight_name = name.removesuffix(".scale") + ".weight"
+ if weight_name not in self.model_tensors or self.model_tensors[weight_name]().dtype not in self._float8_dtypes():
+ continue
+ w, s = self.model_tensors[weight_name], self.model_tensors.pop(name)
+ out_f, in_f = w().shape
+ if tuple(s().shape) != (-(-out_f // bo), -(-in_f // bi)):
+ raise ValueError(f"{weight_name}: scale shape {tuple(s().shape)} does not match {(bo, bi)} blocks")
+ _e8m0_check(LazyTorchTensor.to_eager(s()).view(torch.uint8).numpy(), name)
+ self.model_tensors[weight_name] = lambda w=w, s=s: dequant(w(), s())
+ self._dsv4_fp8_dequantized.add(weight_name)
+
+ def _tensor_part(self, name: str) -> Path:
+ index = self.dir_model / "model.safetensors.index.json"
+ if not index.is_file():
+ return self.dir_model / "model.safetensors"
+ with open(index, "r", encoding="utf-8") as f:
+ return self.dir_model / json.load(f)["weight_map"][name]
+
+ def _write_mxfp4_experts(self, bid: int, proj: str, key: gguf.MODEL_TENSOR, n_expert: int | None = None) -> None:
+ n_expert = self.hparams["n_routed_experts"] if n_expert is None else n_expert
+ names = [f"layers.{bid}.ffn.experts.{e}.{proj}" for e in range(n_expert)]
+ pairs = [(self.model_tensors.pop(n + ".weight"), self.model_tensors.pop(n + ".scale")) for n in names]
+ rows, packed_cols = pairs[0][0]().shape
+
+ def chunks() -> Iterable[np.ndarray]:
+ for w, s in pairs:
+ yield self.repack_mxfp4_blocks(LazyTorchTensor.to_eager(w()), LazyTorchTensor.to_eager(s()))
+ data = cast(np.ndarray, _StreamedBytes((len(pairs), rows, packed_cols * 2 // 32 * 17), chunks))
+ self.gguf_writer.add_tensor(self.format_tensor_name(key, bid), data, raw_dtype=gguf.GGMLQuantizationType.MXFP4)
+
+ def _write_engram_tables(self, layer_ids: list[int]) -> None:
+ # official lookup e4m3 * 2^(e8m0 - 127), exact in BF16
+ parts, dim = [], None
+ for bid in layer_ids:
+ wn, sn = f"layers.{bid}.engram.embed.weight", f"layers.{bid}.engram.embed.scale"
+ rows, d = self.model_tensors.pop(wn)().shape
+ if d % 32 != 0 or tuple(self.model_tensors.pop(sn)().shape) != (rows, d // 32) or d != (dim or d):
+ raise ValueError(f"{wn}: expected one e8m0 scale per 32 elements and the same width in every layer")
+ if rows != self.hparams["engram_num_embeddings"][len(parts)]:
+ raise ValueError(f"{wn}: {rows} rows, config says {self.hparams['engram_num_embeddings'][len(parts)]}")
+ dim = d
+ parts.append((wn, sn, rows, self._tensor_part(wn), self._tensor_part(sn)))
+ assert dim is not None
+ name = self.format_tensor_name(gguf.MODEL_TENSOR.PER_LAYER_TOKEN_EMBD)
+ n_rows = sum(p[2] for p in parts)
+
+ def chunks() -> Iterable[np.ndarray]:
+ from safetensors import safe_open
+ for wn, sn, rows, wpart, spart in parts:
+ with safe_open(wpart, framework="pt") as fw, safe_open(spart, framework="pt") as fs:
+ ws, ss = fw.get_slice(wn), fs.get_slice(sn)
+ for r0 in range(0, rows, 1 << 18):
+ q = ws[r0:r0 + (1 << 18)].view(torch.uint8).reshape(-1, 32)
+ e = ss[r0:r0 + (1 << 18)].view(torch.uint8).reshape(-1, 1)
+ _e8m0_check(e.numpy(), sn)
+ v = (q.view(torch.float8_e4m3fn).float() * torch.exp2(e.float() - 127)).reshape(-1, dim)
+ yield (v if self._f32 else v.bfloat16()).view(torch.uint8).numpy()
+ qt = gguf.GGMLQuantizationType.F32 if self._f32 else gguf.GGMLQuantizationType.BF16
+ data = cast(np.ndarray, _StreamedBytes((n_rows, dim * (4 if self._f32 else 2)), chunks))
+ self.gguf_writer.add_tensor(name, data, raw_dtype=qt)
+
+ def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
+ if not self._dsv4_mxfp4_generated:
+ for bid in range(self.block_count):
+ self._write_mxfp4_experts(bid, "w1", gguf.MODEL_TENSOR.FFN_GATE_EXP)
+ self._write_mxfp4_experts(bid, "w2", gguf.MODEL_TENSOR.FFN_DOWN_EXP)
+ self._write_mxfp4_experts(bid, "w3", gguf.MODEL_TENSOR.FFN_UP_EXP)
+ if self.hparams.get("engram_layer_ids"):
+ self._write_engram_tables(list(self.hparams["engram_layer_ids"]))
+ self._dsv4_mxfp4_generated = True
+ return ()
+
+ _LAYER_MAP: dict[str, tuple[gguf.MODEL_TENSOR, str]] = {
+ "attn.indexer.wk.weight": (gguf.MODEL_TENSOR.INDEXER_ATTN_K, ".weight"),
+ "attn.indexer.k_norm.weight": (gguf.MODEL_TENSOR.INDEXER_K_NORM, ".weight"),
+ "engram.wkv.weight": (gguf.MODEL_TENSOR.ENGRAM_KV, ".weight"),
+ }
+
+ def _map_dsv4_tensor_name(self, name: str, bid: int | None) -> tuple[gguf.MODEL_TENSOR, str]:
+ m = re.match(r"layers\.\d+\.(.+)$", name)
+ if m is not None and m.group(1) in self._LAYER_MAP:
+ return self._LAYER_MAP[m.group(1)]
+ return super()._map_dsv4_tensor_name(name, bid)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if name.endswith(".engram.k_weight"):
+ return [] # folded into engram_gate
+ if name.endswith(".engram.q_weight"):
+ # Engram gate weight = q_weight * k_weight (bf16 * bf16 is exact in f32)
+ k = self.model_tensors[name.removesuffix("q_weight") + "k_weight"]
+ return [(self.format_tensor_name(gguf.MODEL_TENSOR.ENGRAM_GATE, bid), data_torch.float() * k().float())]
+ return super().modify_tensors(data_torch, name, bid)
+
+ def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
+ if new_name.endswith((".engram_gate.weight", ".ffn_gate_inp.weight")) or name in self._dsv4_f32_tensors:
+ return gguf.GGMLQuantizationType.F32
+ if n_dims >= 2 and (name in self._dsv4_fp8_dequantized or name in self._dsv4_bf16_tensors):
+ return gguf.GGMLQuantizationType.F32 if self._f32 else gguf.GGMLQuantizationType.BF16
+ return False
+
+
+class DeepseekV41DSparkModel(DeepseekV41Model):
+ """DeepSeek-V4.1 DSpark speculative draft (mtp.0..2 of the checkpoint) as a DFLASH GGUF (`--dspark`).
+
+ Layout = the V4 DSpark draft (conversion/deepseek.py DeepseekV4DSparkModel) with V4.1's differences flagged by
+ `dflash.hyper_connection.single_pass`: stages blk.0..2 (MXFP4 experts bit-exact, dense BF16), mtp.0.main_proj /
+ main_norm -> fc / enc.output_norm, mtp.2.norm -> output_norm, markov_head.embed / head -> markov_w1 / markov_w2,
+ confidence_head.proj -> conf_proj (F32), no hc_head. Token embeddings and the lm head are the target's (shared at
+ runtime through ctx_other). target_layers are the V4.1 ids as-is: the draft reads the attention input of those
+ layers, which is what llama_set_embeddings_layer_inp extracts.
+ """
+ model_arch = gguf.MODEL_ARCH.DFLASH
+ mtp_only = True
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ hp = self.hparams
+ self._n_main = hp["num_hidden_layers"]
+ self._n_stage = hp["num_nextn_predict_layers"]
+ if self._n_stage <= 0 or "dspark_block_size" not in hp:
+ raise ValueError("DeepSeek-V4.1 DSpark export needs num_nextn_predict_layers > 0 and the dspark_* config")
+ # stages keep main-model block ids (mtp.M -> layers.n_main+M); format_tensor_name() maps them back
+ self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self._n_stage)
+
+ def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
+ hp = {**self.hparams, **self.hparams.get("text_config", {})} # text_config is merged only after indexing
+ type(self)._dsv4_main_layers = hp["num_hidden_layers"]
+ type(self)._dsv4_nextn_layers = hp.get("num_nextn_predict_layers", 0)
+ return TextModel.index_tensors(self, remote_hf_model_id=remote_hf_model_id)
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ name, gen = item
+ m = re.match(r"mtp\.(\d+)\.(.+)$", name)
+ if m is None:
+ return None # shared with the target at runtime
+ stage, rest = int(m.group(1)), m.group(2)
+ assert cls._dsv4_main_layers is not None
+ if stage >= cls._dsv4_nextn_layers:
+ raise ValueError(f"Unexpected DSpark stage {stage}")
+ if rest.startswith(("markov_head.", "confidence_head.")):
+ return f"dspark.{rest}", gen
+ return f"layers.{cls._dsv4_main_layers + stage}.{rest}", gen
+
+ def format_tensor_name(self, key: gguf.MODEL_TENSOR, bid: int | None = None, suffix: str = ".weight") -> str:
+ if bid is not None and bid >= self._n_main:
+ bid -= self._n_main
+ return super().format_tensor_name(key, bid, suffix)
+
+ def set_vocab(self):
+ super().set_vocab()
+ self.gguf_writer.add_mask_token_id(self.hparams["dspark_noise_token_id"])
+
+ def set_gguf_parameters(self):
+ hp = self.hparams
+ n = self._n_stage
+ w = self.gguf_writer
+ w.add_block_count(n)
+ w.add_context_length(hp["max_position_embeddings"])
+ w.add_embedding_length(hp["hidden_size"])
+ w.add_feed_forward_length(hp["moe_intermediate_size"])
+ w.add_head_count(hp["num_attention_heads"])
+ w.add_head_count_kv(1)
+ w.add_key_length(hp["head_dim"])
+ w.add_value_length(hp["head_dim"])
+ w.add_rope_dimension_count(hp["qk_rope_head_dim"])
+ # compress-ratio-0 layers: plain rope at rope_theta, no YaRN
+ w.add_rope_freq_base(float(hp["rope_theta"]))
+ w.add_layer_norm_rms_eps(hp["rms_norm_eps"])
+ w.add_q_lora_rank(hp["q_lora_rank"])
+ w.add_sliding_window(hp["sliding_window"])
+ w.add_expert_count(hp["dspark_n_routed_experts"])
+ w.add_expert_used_count(hp["dspark_num_experts_per_tok"])
+ w.add_expert_feed_forward_length(hp["moe_intermediate_size"])
+ w.add_expert_shared_count(hp["n_shared_experts"])
+ w.add_expert_weights_scale(hp["routed_scaling_factor"])
+ w.add_expert_weights_norm(hp["norm_topk_prob"])
+ w.add_expert_gating_func(gguf.ExpertGatingFuncType.SQRTSOFTPLUS)
+ w.add_swiglu_clamp_exp([hp["swiglu_limit"]] * n)
+ w.add_swiglu_clamp_shexp([hp["swiglu_limit"]] * n)
+ w.add_attention_output_group_count(hp["o_groups"])
+ w.add_attention_output_lora_rank(hp["o_lora_rank"])
+ w.add_attention_compress_ratios([0] * n)
+ w.add_hyper_connection_count(hp["hc_mult"])
+ w.add_hyper_connection_sinkhorn_iterations(hp["hc_sinkhorn_iters"])
+ w.add_hyper_connection_epsilon(hp["hc_eps"])
+ w.add_hyper_connection_single_pass(True)
+ w.add_block_size(hp["dspark_block_size"])
+ w.add_target_layers(hp["dspark_target_layer_ids"])
+ w.add_sample_from_anchor(True)
+ w.add_has_confidence_head(True)
+ w.add_file_type(self.ftype)
+
+ def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
+ # no Engram tables in the draft
+ if not self._dsv4_mxfp4_generated:
+ n_expert = self.hparams["dspark_n_routed_experts"]
+ for bid in range(self._n_main, self._n_main + self._n_stage):
+ self._write_mxfp4_experts(bid, "w1", gguf.MODEL_TENSOR.FFN_GATE_EXP, n_expert)
+ self._write_mxfp4_experts(bid, "w2", gguf.MODEL_TENSOR.FFN_DOWN_EXP, n_expert)
+ self._write_mxfp4_experts(bid, "w3", gguf.MODEL_TENSOR.FFN_UP_EXP, n_expert)
+ self._dsv4_mxfp4_generated = True
+ return ()
+
+ _DSPARK_ROOT: dict[str, gguf.MODEL_TENSOR] = {
+ "main_proj.weight": gguf.MODEL_TENSOR.FC,
+ "main_norm.weight": gguf.MODEL_TENSOR.ENC_OUTPUT_NORM,
+ "norm.weight": gguf.MODEL_TENSOR.OUTPUT_NORM,
+ "dspark.markov_head.embed.weight": gguf.MODEL_TENSOR.DSPARK_MARKOV_W1,
+ "dspark.markov_head.head.weight": gguf.MODEL_TENSOR.DSPARK_MARKOV_W2,
+ "dspark.confidence_head.proj.weight": gguf.MODEL_TENSOR.DSPARK_CONF_PROJ,
+ }
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if name.endswith(".ffn.gate.bias_vl"):
+ return []
+ if name in self._DSPARK_ROOT:
+ return [(self.format_tensor_name(self._DSPARK_ROOT[name], None), data_torch)]
+ m = re.match(r"layers\.\d+\.(main_proj\.weight|main_norm\.weight|norm\.weight)$", name)
+ if m is not None:
+ return [(self.format_tensor_name(self._DSPARK_ROOT[m.group(1)], None), data_torch)]
+ return super().modify_tensors(data_torch, name, bid)
+
+ def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
+ if new_name == "conf_proj.weight":
+ return gguf.GGMLQuantizationType.F32
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+
+@ModelBase.register("DeepseekV41ForCausalLM")
+@ModelBase.example("deepseek-ai/DeepSeek-V4.1-Flash-1017")
+class DeepseekV41VisionModel(DeepseekV4FlashVisionModel):
+ """DeepSeek-V4.1 vision tower (projector deepseek41v): the V4-Flash-Vision ViT + 3x3 aligner, with the V4.1 token
+ block ([START] ([IMAGE]*w [NEWLINE])*h [END], no pads) and resize plan. ref: inference/vision.py,
+ inference/image_processor.py of deepseek-ai/DeepSeek-V4.1-Flash-1017. The checkpoint config is HF-style
+ (vision_config); image_start / image_end / image_newline are written like V4's sentinels, there is no image_pad."""
+
+ def get_vision_config(self) -> dict[str, Any] | None:
+ vc = self.global_config.get("vision_config")
+ if not vc or vc.get("num_hidden_layers", 0) == 0:
+ raise ValueError("DeepseekV41VisionModel requires vision_config.num_hidden_layers > 0")
+ return {
+ "num_hidden_layers": vc["num_hidden_layers"],
+ "hidden_size": vc["hidden_size"],
+ "num_attention_heads": vc["num_attention_heads"],
+ "intermediate_size": vc["intermediate_size"],
+ "patch_size": vc["patch_size"],
+ # dynamic resolution; only used for compat / warmup
+ "image_size": vc["patch_size"] * vc["downsample_ratio"] * 16,
+ "rope_theta": vc.get("rope_theta", 10000.0),
+ "downsample_ratio": vc["downsample_ratio"],
+ "min_pixels": vc["min_pixels"],
+ "max_image_tokens": vc["max_image_tokens"],
+ "max_wh_ratio": vc.get("max_wh_ratio"),
+ }
+
+ def set_gguf_parameters(self):
+ assert self.hparams_vision is not None
+ super(DeepseekV4FlashVisionModel, self).set_gguf_parameters()
+ self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.DEEPSEEK41V)
+ # pytorch default eps (vision.py), not the LLM rms_norm_eps (1e-20)
+ self.gguf_writer.add_vision_attention_layernorm_eps(1e-6)
+ self.gguf_writer.add_vision_use_silu(True)
+ self.gguf_writer.add_vision_projector_scale_factor(self.hparams_vision["downsample_ratio"])
+ self.gguf_writer.add_vision_min_pixels(self.hparams_vision["min_pixels"])
+ # hardcoded in clip.cpp (PROJECTOR_TYPE_DEEPSEEK41V); add GGUF keys if a later model differs
+ assert self.hparams_vision["max_image_tokens"] == 1024, self.hparams_vision["max_image_tokens"]
+ assert self.hparams_vision["max_wh_ratio"] is None, self.hparams_vision["max_wh_ratio"]
+ assert self.hparams_vision["rope_theta"] == 10000, self.hparams_vision["rope_theta"]
diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py
index e09616b19..5e263fd16 100755
--- a/convert_hf_to_gguf.py
+++ b/convert_hf_to_gguf.py
@@ -267,11 +267,15 @@ def main() -> None:
sys.exit(1)
if args.dspark:
- if is_mistral_format or model_architecture != "DeepseekV4ForCausalLM":
- logger.error("--dspark is only supported for DeepseekV4ForCausalLM")
+ if is_mistral_format or model_architecture not in ("DeepseekV4ForCausalLM", "DeepseekV41ForCausalLM"):
+ logger.error("--dspark is only supported for DeepseekV4ForCausalLM and DeepseekV41ForCausalLM")
sys.exit(1)
- from conversion.deepseek import DeepseekV4DSparkModel
- model_class = DeepseekV4DSparkModel
+ if model_architecture == "DeepseekV41ForCausalLM":
+ from conversion.deepseek41 import DeepseekV41DSparkModel
+ model_class = DeepseekV41DSparkModel
+ else:
+ from conversion.deepseek import DeepseekV4DSparkModel
+ model_class = DeepseekV4DSparkModel
if args.mtp or args.no_mtp:
if not model_class.supports_mtp_export:
diff --git a/ggml/src/ggml-cpu/unary-ops.cpp b/ggml/src/ggml-cpu/unary-ops.cpp
index 1d8344436..f5d57d6f9 100644
--- a/ggml/src/ggml-cpu/unary-ops.cpp
+++ b/ggml/src/ggml-cpu/unary-ops.cpp
@@ -78,7 +78,7 @@ static inline float op_expm1(float x) {
}
static inline float op_softplus(float x) {
- return (x > 20.0f) ? x : logf(1.0f + expf(x));
+ return (x > 20.0f) ? x : log1pf(expf(x));
}
static inline float op_floor(float x) {
diff --git a/ggml/src/ggml-cuda/topk-moe.cu b/ggml/src/ggml-cuda/topk-moe.cu
index 81d6ec3f7..158b736f0 100644
--- a/ggml/src/ggml-cuda/topk-moe.cu
+++ b/ggml/src/ggml-cuda/topk-moe.cu
@@ -76,7 +76,7 @@ __device__ void sqrt_softplus_warp_inplace(float (&vals)[experts_per_thread], co
for (int i = 0; i < experts_per_thread; i++) {
const int idx = lane + i * WARP_SIZE;
const bool active = !use_limit || (idx < limit);
- vals[i] = active ? sqrtf(vals[i] > 20.0f ? vals[i] : logf(1.0f + expf(vals[i]))) : -INFINITY;
+ vals[i] = active ? sqrtf(vals[i] > 20.0f ? vals[i] : log1pf(expf(vals[i]))) : -INFINITY;
}
}
diff --git a/ggml/src/ggml-cuda/unary.cu b/ggml/src/ggml-cuda/unary.cu
index af839acd7..b3b6aa48d 100644
--- a/ggml/src/ggml-cuda/unary.cu
+++ b/ggml/src/ggml-cuda/unary.cu
@@ -91,7 +91,7 @@ static __device__ __forceinline__ float op_expm1(float x) {
}
static __device__ __forceinline__ float op_softplus(float x) {
- return (x > 20.0f) ? x : logf(1.0f + expf(x));
+ return (x > 20.0f) ? x : log1pf(expf(x));
}
static __device__ __forceinline__ float op_elu(float x) {
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index f941106f2..706cad3ea 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -211,6 +211,7 @@ class Keys:
VALUE_SCALE = "{arch}.attention.value_scale"
COMPRESS_RATIOS = "{arch}.attention.compress_ratios"
COMPRESS_ROPE_FREQ_BASE = "{arch}.attention.compress_rope_freq_base"
+ KV_SOURCE_LAYERS = "{arch}.attention.kv_source_layers"
TEMPERATURE_LENGTH = "{arch}.attention.temperature_length"
KEY_LENGTH_MLA = "{arch}.attention.key_length_mla"
VALUE_LENGTH_MLA = "{arch}.attention.value_length_mla"
@@ -236,6 +237,10 @@ class Keys:
TYPES = "{arch}.attention.indexer.types"
KPOOL = "{arch}.attention.indexer.kpool" # GLM5-Next
KPOOL_SELECT_TAIL = "{arch}.attention.indexer.kpool_select_tail" # GLM5-Next
+ SOURCE_LAYERS = "{arch}.attention.indexer.source_layers"
+ CANDIDATE_SOURCE_LAYER = "{arch}.attention.indexer.candidate_source_layer"
+ CANDIDATE_TOP_K_BLOCKS = "{arch}.attention.indexer.candidate_top_k_blocks"
+ CANDIDATE_BLOCK_SIZE = "{arch}.attention.indexer.candidate_block_size"
class HyperConnection:
COUNT = "{arch}.hyper_connection.count"
@@ -245,6 +250,12 @@ class Keys:
MAGNITUDE = "{arch}.hyper_connection.magnitude"
# absent means the mix projection is full rank (DeepSeek-V4 behaviour)
LOW_RANK = "{arch}.hyper_connection.low_rank"
+ # single-pass mHC: each sublayer collapses with the previous pre-mix (DeepSeek-V4.1)
+ SINGLE_PASS = "{arch}.hyper_connection.single_pass"
+
+ class Engram: # deepseek41; the table uses the PerLayerEmbedding keys
+ PAD_TOKEN_ID = "{arch}.engram.pad_token_id"
+ TOKEN_MAP = "{arch}.engram.token_map"
class PerLayerEmbedding:
LAYERS = "{arch}.ple.layers"
@@ -636,6 +647,7 @@ class MODEL_ARCH(IntEnum):
DEEPSEEK2OCR = auto()
DEEPSEEK32 = auto()
DEEPSEEK4 = auto()
+ DEEPSEEK41 = auto()
CHATGLM = auto()
GLM4 = auto()
GLM4_MOE = auto()
@@ -1317,6 +1329,8 @@ class MODEL_TENSOR(IntEnum):
DSPARK_MARKOV_W1 = auto() # markov head: prev-token embed
DSPARK_MARKOV_W2 = auto() # markov head: bias projection
DSPARK_CONF_PROJ = auto() # confidence head
+ ENGRAM_KV = auto()
+ ENGRAM_GATE = auto()
DFLASH_ATTN_CONV_BASE = auto()
DFLASH_ATTN_CONV_PROJ = auto()
DFLASH_FFN_CONV_BASE = auto()
@@ -1460,6 +1474,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
MODEL_ARCH.DEEPSEEK2OCR: "deepseek2-ocr",
MODEL_ARCH.DEEPSEEK32: "deepseek32",
MODEL_ARCH.DEEPSEEK4: "deepseek4",
+ MODEL_ARCH.DEEPSEEK41: "deepseek41",
MODEL_ARCH.CHATGLM: "chatglm",
MODEL_ARCH.GLM4: "glm4",
MODEL_ARCH.GLM4_MOE: "glm4moe",
@@ -2164,6 +2179,8 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.DSPARK_MARKOV_W1: "markov_w1",
MODEL_TENSOR.DSPARK_MARKOV_W2: "markov_w2",
MODEL_TENSOR.DSPARK_CONF_PROJ: "conf_proj",
+ MODEL_TENSOR.ENGRAM_KV: "blk.{bid}.engram_kv",
+ MODEL_TENSOR.ENGRAM_GATE: "blk.{bid}.engram_gate",
MODEL_TENSOR.DFLASH_ATTN_CONV_BASE: "blk.{bid}.attn_conv_base",
MODEL_TENSOR.DFLASH_ATTN_CONV_PROJ: "blk.{bid}.attn_conv_proj",
MODEL_TENSOR.DFLASH_FFN_CONV_BASE: "blk.{bid}.ffn_conv_base",
@@ -4358,6 +4375,46 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
],
+ MODEL_ARCH.DEEPSEEK41: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_SINKS,
+ MODEL_TENSOR.ATTN_Q_A,
+ MODEL_TENSOR.ATTN_Q_B,
+ MODEL_TENSOR.ATTN_Q_A_NORM,
+ MODEL_TENSOR.ATTN_KV,
+ MODEL_TENSOR.ATTN_KV_NORM,
+ MODEL_TENSOR.ATTN_OUT_A,
+ MODEL_TENSOR.ATTN_OUT_B,
+ MODEL_TENSOR.HC_ATTN_FN,
+ MODEL_TENSOR.HC_ATTN_BASE,
+ MODEL_TENSOR.HC_ATTN_SCALE,
+ MODEL_TENSOR.HC_FFN_FN,
+ MODEL_TENSOR.HC_FFN_BASE,
+ MODEL_TENSOR.HC_FFN_SCALE,
+ MODEL_TENSOR.ATTN_COMPRESSOR_WKV,
+ MODEL_TENSOR.ATTN_COMPRESSOR_WGATE,
+ MODEL_TENSOR.ATTN_COMPRESSOR_NORM,
+ MODEL_TENSOR.INDEXER_ATTN_K,
+ MODEL_TENSOR.INDEXER_K_NORM,
+ MODEL_TENSOR.INDEXER_ATTN_Q_B,
+ MODEL_TENSOR.INDEXER_PROJ,
+ MODEL_TENSOR.PER_LAYER_TOKEN_EMBD,
+ MODEL_TENSOR.ENGRAM_KV,
+ MODEL_TENSOR.ENGRAM_GATE,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ MODEL_TENSOR.FFN_EXP_PROBS_B_VL,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ ],
MODEL_ARCH.ERNIE4_5_MOE: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT_NORM,
@@ -6449,6 +6506,7 @@ class VisionProjectorType:
DEEPSEEKOCR = "deepseekocr"
DEEPSEEKOCR2 = "deepseekocr2"
DEEPSEEK4V = "deepseek4v"
+ DEEPSEEK41V = "deepseek41v"
LFM2A = "lfm2a" # audio
MUSIC_FLAMINGO = "musicflamingo" # audio
GLM4V = "glm4v"
diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py
index 9e49a1514..605013610 100644
--- a/gguf-py/gguf/gguf_writer.py
+++ b/gguf-py/gguf/gguf_writer.py
@@ -1077,6 +1077,28 @@ class GGUFWriter:
def add_attention_compress_rope_freq_base(self, value: float) -> None:
self.add_float32(Keys.Attention.COMPRESS_ROPE_FREQ_BASE.format(arch=self.arch), value)
+ # deepseek41
+ def add_attention_kv_source_layers(self, layers: Sequence[int]) -> None:
+ self.add_key_value(Keys.Attention.KV_SOURCE_LAYERS.format(arch=self.arch), list(layers), GGUFValueType.ARRAY, GGUFValueType.UINT32)
+
+ def add_indexer_source_layers(self, layers: Sequence[int]) -> None:
+ self.add_key_value(Keys.Attention.Indexer.SOURCE_LAYERS.format(arch=self.arch), list(layers), GGUFValueType.ARRAY, GGUFValueType.UINT32)
+
+ def add_indexer_candidate_source_layer(self, layer: int) -> None:
+ self.add_int32(Keys.Attention.Indexer.CANDIDATE_SOURCE_LAYER.format(arch=self.arch), layer)
+
+ def add_indexer_candidate_top_k_blocks(self, value: int) -> None:
+ self.add_uint32(Keys.Attention.Indexer.CANDIDATE_TOP_K_BLOCKS.format(arch=self.arch), value)
+
+ def add_indexer_candidate_block_size(self, value: int) -> None:
+ self.add_uint32(Keys.Attention.Indexer.CANDIDATE_BLOCK_SIZE.format(arch=self.arch), value)
+
+ def add_engram_pad_token_id(self, value: int) -> None:
+ self.add_uint32(Keys.Engram.PAD_TOKEN_ID.format(arch=self.arch), value)
+
+ def add_engram_token_map(self, values: Sequence[int]) -> None:
+ self.add_key_value(Keys.Engram.TOKEN_MAP.format(arch=self.arch), list(values), GGUFValueType.ARRAY, GGUFValueType.INT32)
+
def add_hyper_connection_count(self, count: int) -> None:
self.add_uint32(Keys.HyperConnection.COUNT.format(arch=self.arch), count)
@@ -1092,6 +1114,9 @@ class GGUFWriter:
def add_hyper_connection_low_rank(self, value: int) -> None:
self.add_uint32(Keys.HyperConnection.LOW_RANK.format(arch=self.arch), value)
+ def add_hyper_connection_single_pass(self, value: bool) -> None:
+ self.add_bool(Keys.HyperConnection.SINGLE_PASS.format(arch=self.arch), value)
+
def add_ple_layers(self, values: Sequence[int]) -> None:
self.add_array(Keys.PerLayerEmbedding.LAYERS.format(arch=self.arch), values)
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index 6db74417e..39e29d09a 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -87,6 +87,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
{ LLM_ARCH_DEEPSEEK2OCR, "deepseek2-ocr" },
{ LLM_ARCH_DEEPSEEK32, "deepseek32" },
{ LLM_ARCH_DEEPSEEK4, "deepseek4" },
+ { LLM_ARCH_DEEPSEEK41, "deepseek41" },
{ LLM_ARCH_CHATGLM, "chatglm" },
{ LLM_ARCH_GLM4, "glm4" },
{ LLM_ARCH_GLM4_MOE, "glm4moe" },
@@ -311,6 +312,11 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_ATTENTION_INDEXER_TYPES, "%s.attention.indexer.types" },
{ LLM_KV_ATTENTION_INDEXER_KPOOL, "%s.attention.indexer.kpool" },
{ LLM_KV_ATTENTION_INDEXER_KPOOL_SELECT_TAIL, "%s.attention.indexer.kpool_select_tail" },
+ { LLM_KV_ATTENTION_INDEXER_SOURCE_LAYERS, "%s.attention.indexer.source_layers" },
+ { LLM_KV_ATTENTION_INDEXER_CANDIDATE_SOURCE_LAYER, "%s.attention.indexer.candidate_source_layer" },
+ { LLM_KV_ATTENTION_INDEXER_CANDIDATE_TOP_K_BLOCKS, "%s.attention.indexer.candidate_top_k_blocks" },
+ { LLM_KV_ATTENTION_INDEXER_CANDIDATE_BLOCK_SIZE, "%s.attention.indexer.candidate_block_size" },
+ { LLM_KV_ATTENTION_KV_SOURCE_LAYERS, "%s.attention.kv_source_layers" },
{ LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, "%s.attention.output_group_count" },
{ LLM_KV_ATTENTION_OUTPUT_LORA_RANK, "%s.attention.output_lora_rank" },
{ LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, "%s.attention.compress_rope_freq_base" },
@@ -323,6 +329,10 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_HYPER_CONNECTION_EPSILON, "%s.hyper_connection.epsilon" },
{ LLM_KV_HYPER_CONNECTION_MAGNITUDE, "%s.hyper_connection.magnitude" },
{ LLM_KV_HYPER_CONNECTION_LOW_RANK, "%s.hyper_connection.low_rank" },
+ { LLM_KV_HYPER_CONNECTION_SINGLE_PASS, "%s.hyper_connection.single_pass" },
+
+ { LLM_KV_ENGRAM_PAD_TOKEN_ID, "%s.engram.pad_token_id" },
+ { LLM_KV_ENGRAM_TOKEN_MAP, "%s.engram.token_map" },
{ LLM_KV_PLE_LAYERS, "%s.ple.layers" },
{ LLM_KV_PLE_NGRAM_SIZE, "%s.ple.ngram_size" },
@@ -782,6 +792,8 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
{ LLM_TENSOR_DSPARK_MARKOV_W1, "markov_w1" },
{ LLM_TENSOR_DSPARK_MARKOV_W2, "markov_w2" },
{ LLM_TENSOR_DSPARK_CONF_PROJ, "conf_proj" },
+ { LLM_TENSOR_ENGRAM_KV, "blk.%d.engram_kv" },
+ { LLM_TENSOR_ENGRAM_GATE, "blk.%d.engram_gate" },
{ LLM_TENSOR_DFLASH_ATTN_CONV_BASE, "blk.%d.attn_conv_base" },
{ LLM_TENSOR_DFLASH_ATTN_CONV_PROJ, "blk.%d.attn_conv_proj" },
{ LLM_TENSOR_DFLASH_FFN_CONV_BASE, "blk.%d.ffn_conv_base" },
@@ -1120,6 +1132,8 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
{LLM_TENSOR_DSPARK_MARKOV_W1, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_GET_ROWS}},
{LLM_TENSOR_DSPARK_MARKOV_W2, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_DSPARK_CONF_PROJ, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_ENGRAM_KV, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_ENGRAM_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_DFLASH_ATTN_CONV_BASE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_DFLASH_ATTN_CONV_PROJ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_DFLASH_FFN_CONV_BASE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
@@ -1239,6 +1253,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) {
case LLM_ARCH_QWEN35MOE:
case LLM_ARCH_QWEN4EXP:
case LLM_ARCH_DEEPSEEK4:
+ case LLM_ARCH_DEEPSEEK41:
case LLM_ARCH_MINIMAX_01:
return true;
default:
@@ -1265,6 +1280,7 @@ bool llm_arch_supports_rs_rollback(const llm_arch & arch) {
case LLM_ARCH_QWEN35MOE:
case LLM_ARCH_QWEN4EXP:
case LLM_ARCH_DEEPSEEK4:
+ case LLM_ARCH_DEEPSEEK41:
case LLM_ARCH_GLM5_NEXT:
case LLM_ARCH_NEMOTRON_H:
case LLM_ARCH_NEMOTRON_H_MOE:
@@ -1282,6 +1298,7 @@ bool llm_arch_supports_mixed_batch(const llm_arch & arch) {
switch (arch) {
case LLM_ARCH_COGVLM:
case LLM_ARCH_DEEPSEEK4:
+ case LLM_ARCH_DEEPSEEK41:
case LLM_ARCH_GRANITE_SWITCH:
case LLM_ARCH_EAGLE3:
case LLM_ARCH_DFLASH:
diff --git a/src/llama-arch.h b/src/llama-arch.h
index cdf978fb8..3d0032c0c 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -92,6 +92,7 @@ enum llm_arch {
LLM_ARCH_DEEPSEEK2OCR,
LLM_ARCH_DEEPSEEK32,
LLM_ARCH_DEEPSEEK4,
+ LLM_ARCH_DEEPSEEK41,
LLM_ARCH_CHATGLM,
LLM_ARCH_GLM4,
LLM_ARCH_GLM4_MOE,
@@ -316,6 +317,11 @@ enum llm_kv {
LLM_KV_ATTENTION_INDEXER_TYPES,
LLM_KV_ATTENTION_INDEXER_KPOOL,
LLM_KV_ATTENTION_INDEXER_KPOOL_SELECT_TAIL,
+ LLM_KV_ATTENTION_INDEXER_SOURCE_LAYERS,
+ LLM_KV_ATTENTION_INDEXER_CANDIDATE_SOURCE_LAYER,
+ LLM_KV_ATTENTION_INDEXER_CANDIDATE_TOP_K_BLOCKS,
+ LLM_KV_ATTENTION_INDEXER_CANDIDATE_BLOCK_SIZE,
+ LLM_KV_ATTENTION_KV_SOURCE_LAYERS,
LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT,
LLM_KV_ATTENTION_OUTPUT_LORA_RANK,
LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE,
@@ -328,6 +334,10 @@ enum llm_kv {
LLM_KV_HYPER_CONNECTION_EPSILON,
LLM_KV_HYPER_CONNECTION_MAGNITUDE,
LLM_KV_HYPER_CONNECTION_LOW_RANK,
+ LLM_KV_HYPER_CONNECTION_SINGLE_PASS,
+
+ LLM_KV_ENGRAM_PAD_TOKEN_ID,
+ LLM_KV_ENGRAM_TOKEN_MAP,
LLM_KV_PLE_LAYERS,
LLM_KV_PLE_NGRAM_SIZE,
@@ -788,6 +798,8 @@ enum llm_tensor {
LLM_TENSOR_DSPARK_MARKOV_W1,
LLM_TENSOR_DSPARK_MARKOV_W2,
LLM_TENSOR_DSPARK_CONF_PROJ,
+ LLM_TENSOR_ENGRAM_KV,
+ LLM_TENSOR_ENGRAM_GATE,
LLM_TENSOR_DFLASH_ATTN_CONV_BASE,
LLM_TENSOR_DFLASH_ATTN_CONV_PROJ,
LLM_TENSOR_DFLASH_FFN_CONV_BASE,
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index ca84fc405..d5a66a499 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -1339,8 +1339,9 @@ void llama_context::set_abort_callback(bool (*abort_callback)(void * data), void
void llama_context::set_embeddings(bool value) {
LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value);
- // a pooled context computes the logits only without embeddings, so the toggle changes the graph shape
- if (cparams.embeddings != value && cparams.pooling_type != LLAMA_POOLING_TYPE_NONE) {
+ // the toggle changes the graph shape (pooled: logits only; pooling NONE: the reserve graph outputs all tokens,
+ // which crashed ggml_gallocr on the first /completion), so re-reserve on every toggle
+ if (cparams.embeddings != value) {
sched_need_reserve = true;
}
@@ -2536,6 +2537,7 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {
model.arch == LLM_ARCH_CLEF ||
model.arch == LLM_ARCH_QWEN4EXP ||
model.arch == LLM_ARCH_DEEPSEEK4 ||
+ model.arch == LLM_ARCH_DEEPSEEK41 ||
(model.arch == LLM_ARCH_DFLASH && model.hparams.dsv4_hc_mult > 0) ||
model.arch == LLM_ARCH_NANBEIGE ||
model.arch == LLM_ARCH_MINIMAX_01 ||
@@ -4071,7 +4073,7 @@ llama_context * llama_init_from_model(
}
}
- if ((model->hparams.is_mla() || model->arch == LLM_ARCH_DEEPSEEK4) && params.type_k != params.type_v) {
+ if ((model->hparams.is_mla() || model->arch == LLM_ARCH_DEEPSEEK4 || model->arch == LLM_ARCH_DEEPSEEK41) && params.type_k != params.type_v) {
LLAMA_LOG_ERROR("%s: model does not support different K (%s) and V (%s) cache types\n", __func__, ggml_type_name(params.type_k), ggml_type_name(params.type_v));
return nullptr;
}
diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp
index c81433bed..a22dfba96 100644
--- a/src/llama-graph.cpp
+++ b/src/llama-graph.cpp
@@ -2010,7 +2010,7 @@ ggml_tensor * llm_graph_context::build_ffn(
const float limit = hparams.swiglu_clamp_shexp[il];
constexpr float eps = 1e-6f;
if (limit > eps) {
- if (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_GLM5_NEXT || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0)) {
+ if (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41 || arch == LLM_ARCH_GLM5_NEXT || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0)) {
cur = ggml_swiglu_clamp(ctx0, cur, tmp, limit);
} else {
tmp = ggml_clamp(ctx0, tmp, -limit, limit);
@@ -2340,8 +2340,13 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
ggml_tensor * weights_sum = ggml_sum_rows(ctx0, weights); // [1, n_tokens]
cb(weights_sum, "ffn_moe_weights_sum", il);
- // Avoid division by zero, clamp to smallest number representable by F16
- weights_sum = ggml_clamp(ctx0, weights_sum, 6.103515625e-5, INFINITY);
+ if (arch == LLM_ARCH_DEEPSEEK41) {
+ // model.py Gate.forward: weights / (sum + 1e-20); the sum can fall below the F16 clamp below
+ weights_sum = ggml_scale_bias(ctx0, weights_sum, 1.0f, 1e-20f);
+ } else {
+ // Avoid division by zero, clamp to smallest number representable by F16
+ weights_sum = ggml_clamp(ctx0, weights_sum, 6.103515625e-5, INFINITY);
+ }
cb(weights_sum, "ffn_moe_weights_sum_clamped", il);
weights = ggml_div(ctx0, weights, weights_sum); // [n_expert_used, n_tokens]
@@ -2431,7 +2436,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
const float limit = hparams.swiglu_clamp_exp[il];
constexpr float eps = 1e-6f;
if (limit > eps) {
- if (arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_GLM5_NEXT || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0) || arch == LLM_ARCH_HY_V4) {
+ if (arch == LLM_ARCH_MAPLE || arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41 || arch == LLM_ARCH_GLM5_NEXT || (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0) || arch == LLM_ARCH_HY_V4) {
cur = ggml_swiglu_clamp(ctx0, cur, up, limit);
} else {
up = ggml_clamp(ctx0, up, -limit, limit);
diff --git a/src/llama-hparams.h b/src/llama-hparams.h
index 719d7081d..2b92f5d60 100644
--- a/src/llama-hparams.h
+++ b/src/llama-hparams.h
@@ -10,8 +10,8 @@
// bump if necessary
#define LLAMA_MAX_LAYERS 512
#define LLAMA_MAX_EXPERTS 1024 // Kimi K3
-#define LLAMA_MAX_PLE_NGRAM 8 // qwen4exp
-#define LLAMA_MAX_PLE_HEADS 64 // qwen4exp
+#define LLAMA_MAX_PLE_NGRAM 8 // qwen4exp, deepseek41 (all hash modules)
+#define LLAMA_MAX_PLE_HEADS 64 // qwen4exp, deepseek41 (all hash modules)
enum llama_expert_gating_func_type {
LLAMA_EXPERT_GATING_FUNC_TYPE_NONE = 0,
@@ -326,12 +326,20 @@ struct llama_hparams {
uint32_t dsv4_o_group_count = 0;
uint32_t dsv4_o_lora_rank = 0;
uint32_t dsv4_hc_mult = 0;
+ bool dsv4_hc_single_pass = false; // DeepSeek-V4.1 DSpark draft: lagged pre-mix, no hc_head
uint32_t dsv4_hc_sinkhorn_iters = 0;
uint32_t dsv4_hash_layer_count = 0;
float dsv4_compress_rope_base = 0.0f;
float dsv4_hc_eps = 0.0f;
std::array<uint32_t, LLAMA_MAX_LAYERS> dsv4_compress_ratios;
+ // deepseek41
+ std::array<int32_t, LLAMA_MAX_LAYERS> dsv41_kv_src; // layer whose compressed KV this layer reads, -1 = none
+ std::array<int32_t, LLAMA_MAX_LAYERS> dsv41_topk_src; // layer whose top-k this layer uses, -1 = none
+ int32_t dsv41_cand_src = -1; // -1 = no candidate block selection
+ uint32_t dsv41_cand_topk_blocks = 0;
+ uint32_t dsv41_cand_block_size = 0;
+
// 0 = full rank (DeepSeek-V4)
uint32_t hc_low_rank = 0;
@@ -346,6 +354,10 @@ struct llama_hparams {
uint32_t ple_eos_token_id = 0;
// the id the PLE hash stands in at image positions; 0 makes the loader fall back to EOS
uint32_t ple_image_token_id = 0;
+ // the id the KV cells record at embedding (image) positions for the n-gram history; LLAMA_TOKEN_NULL = none
+ llama_token ple_embd_token = LLAMA_TOKEN_NULL;
+ // hash constant sets, stored back to back: 1 shared by all PLE layers (qwen4exp), or 1 per PLE layer in layer order (deepseek41)
+ uint32_t ple_n_modules = 1;
// the file lists PLE layer indices, so this is never a per-layer gguf array and can hold one bit per layer
std::bitset<LLAMA_MAX_LAYERS> is_ple_impl;
// the hash multipliers reach ~2e13 and have to stay 64-bit
diff --git a/src/llama-kv-cache-dsv4.cpp b/src/llama-kv-cache-dsv4.cpp
index f8caae9d6..e479dc564 100644
--- a/src/llama-kv-cache-dsv4.cpp
+++ b/src/llama-kv-cache-dsv4.cpp
@@ -333,12 +333,24 @@ static void dsv4_state_read_tensor_streams(
}
}
+// serialized compressed-KV layers: all except per-device mirrors (kv_src[il] != il), refilled from their source on read
+static std::vector<uint32_t> dsv4_k_cache_state_layers(const llama_kv_cache * kv, const std::vector<int32_t> & kv_src) {
+ std::vector<uint32_t> res;
+ for (uint32_t il : kv->get_layer_ids()) {
+ if (kv_src[il] == (int32_t) il) {
+ res.push_back(il);
+ }
+ }
+ return res;
+}
+
static void dsv4_state_write_k_cache(
llama_io_write_i & io,
const llama_kv_cache * kv,
llama_seq_id seq_id,
llama_state_seq_flags flags,
- uint32_t n_rows) {
+ uint32_t n_rows,
+ const std::vector<int32_t> & kv_src) {
GGML_UNUSED(flags);
uint32_t s0;
@@ -347,7 +359,8 @@ static void dsv4_state_write_k_cache(
const uint32_t version = DSV4_K_CACHE_STATE_VER;
const uint32_t kv_size = kv->get_size();
- const auto layer_ids = kv->get_layer_ids();
+ // mirrors are never serialized, so the state format does not depend on the split
+ const auto layer_ids = dsv4_k_cache_state_layers(kv, kv_src);
const uint32_t n_layer = layer_ids.size();
if (n_rows > kv_size) {
@@ -369,7 +382,8 @@ static void dsv4_state_read_k_cache(
llama_io_read_i & io,
llama_kv_cache * kv,
llama_seq_id seq_id,
- llama_state_seq_flags flags) {
+ llama_state_seq_flags flags,
+ const std::vector<int32_t> & kv_src) {
GGML_UNUSED(flags);
uint32_t version;
@@ -399,7 +413,7 @@ static void dsv4_state_read_k_cache(
uint32_t s0;
dsv4_state_dst_stream_range(kv->get_n_stream(), seq_id, ns, s0);
- const auto layer_ids = kv->get_layer_ids();
+ const auto layer_ids = dsv4_k_cache_state_layers(kv, kv_src);
if (n_layer_ref != layer_ids.size()) {
throw std::runtime_error("DSV4 K-cache layer count mismatch");
}
@@ -413,6 +427,31 @@ static void dsv4_state_read_k_cache(
dsv4_state_read_tensor_streams(io, kv->get_k_storage(il), kv_size, n_rows_ref, s0, ns);
}
+
+ // refill the mirrors from their source rows just read
+ std::vector<uint8_t> buf;
+ for (uint32_t il : kv->get_layer_ids()) {
+ const int32_t src = kv_src[il];
+ if (src == (int32_t) il) {
+ continue;
+ }
+ ggml_tensor * t_src = kv->get_k_storage(src);
+ ggml_tensor * t_dst = kv->get_k_storage(il);
+ GGML_ASSERT(t_src->type == t_dst->type && t_src->ne[0] == t_dst->ne[0]);
+
+ const size_t row_size = ggml_row_size(t_src->type, t_src->ne[0]);
+ const size_t stream_stride = (size_t) kv_size*row_size;
+ const size_t size = (size_t) n_rows_ref*row_size;
+ if (size == 0) {
+ continue;
+ }
+ buf.resize(size);
+ for (uint32_t s = 0; s < ns; ++s) {
+ const size_t offset = (size_t) (s0 + s)*stream_stride;
+ ggml_backend_tensor_get(t_src, buf.data(), offset, size);
+ ggml_backend_tensor_set(t_dst, buf.data(), offset, size);
+ }
+ }
}
static std::string dsv4_plan_positions(const std::vector<int32_t> & values) {
@@ -432,6 +471,7 @@ static llama_kv_cache_dsv4_context::comp_plan dsv4_build_comp_plan(
const llama_ubatch & ubatch,
uint32_t ratio,
bool overlap,
+ bool pad_blocks,
uint32_t state_size,
uint32_t kv_size,
uint32_t n_stream,
@@ -548,7 +588,7 @@ static llama_kv_cache_dsv4_context::comp_plan dsv4_build_comp_plan(
}
}
- if (ratio == DSV4_CSA_RATIO && !plan.state_pos.empty()) {
+ if (pad_blocks && !plan.state_pos.empty()) {
assert(kv_size > 0);
// Pad each stream to the reserve plan's block count.
@@ -603,7 +643,7 @@ static llama_kv_cache_dsv4_context::comp_plan dsv4_build_comp_plan(
}
}
- if (ratio == DSV4_HCA_RATIO && !plan.state_pos.empty() && plan.state_write_idxs.empty()) {
+ if (!pad_blocks && !plan.state_pos.empty() && plan.state_write_idxs.empty()) {
assert(kv_size > 0);
// the last slot must not be live, or the dummy write would corrupt it;
// a full stream implies a completed block, which implies real writes
@@ -744,6 +784,7 @@ static std::vector<llama_kv_cache_dsv4_context::comp_plan> dsv4_build_comp_plans
const std::vector<llama_ubatch> & ubatches,
uint32_t ratio,
bool overlap,
+ bool pad_blocks,
uint32_t state_size,
uint32_t kv_size,
uint32_t n_stream,
@@ -755,7 +796,7 @@ static std::vector<llama_kv_cache_dsv4_context::comp_plan> dsv4_build_comp_plans
// the first ubatch touching a seq consumes its rollback restore
std::vector<uint32_t> rs(rs_idx);
for (const llama_ubatch & ubatch : ubatches) {
- plans.push_back(dsv4_build_comp_plan(ubatch, ratio, overlap, state_size, kv_size, n_stream, n_rs_seq, rs));
+ plans.push_back(dsv4_build_comp_plan(ubatch, ratio, overlap, pad_blocks, state_size, kv_size, n_stream, n_rs_seq, rs));
for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) {
const llama_seq_id seq_id = ubatch.seq_id_unq[s];
@@ -1056,6 +1097,10 @@ uint32_t llama_dsv4_comp_state::get_ratio() const {
return ratio;
}
+bool llama_dsv4_comp_state::is_overlap() const {
+ return state_size == 2*ratio;
+}
+
uint32_t llama_dsv4_comp_state::get_state_size() const {
return state_size;
}
@@ -1261,6 +1306,23 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4(
v_trans, offload, swa_full, unified_raw, kv_size, n_seq_max, n_ubatch, n_pad,
nullptr, filter_raw, reuse, nullptr);
+ // V4.1: CSA slot = ratio 2, HCA slot = ratio 1, no overlap and no indexer compressor; the index K is stored
+ // next to the compressed KV ([kv | index K]) and only the KV source layers own rows
+ const bool is_v41 = model.arch == LLM_ARCH_DEEPSEEK41;
+ const uint32_t ratio_csa = is_v41 ? 2u : DSV4_CSA_RATIO;
+ const uint32_t ratio_hca = is_v41 ? 1u : DSV4_HCA_RATIO;
+
+ if (is_v41) {
+ const uint32_t n_embd_comp = model.hparams.n_embd_head_k() + model.hparams.indexer_head_size;
+ for (llama_hparams * hp : { &hparams_csa, &hparams_hca }) {
+ std::fill(hp->n_head_kv_arr.begin(), hp->n_head_kv_arr.end(), 1);
+ hp->n_embd_head_k_full = n_embd_comp;
+ hp->n_embd_head_v_full = n_embd_comp;
+ hp->n_embd_head_k_swa = n_embd_comp;
+ hp->n_embd_head_v_swa = n_embd_comp;
+ }
+ }
+
dsv4_make_k_only(hparams_csa);
dsv4_make_k_only(hparams_hca);
@@ -1272,12 +1334,50 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4(
hparams_lid.rope_type = LLAMA_ROPE_TYPE_NEOX;
dsv4_make_k_only(hparams_lid);
+ // V4.1 cross-layer KV sharing: a compressed layer reads the storage of its kv source or, when the source is on
+ // another device (layer split), of a per-device mirror fed with the source's new rows, so the whole cache never
+ // crosses devices. no mirrors without offload
+ const uint32_t n_layer = model.hparams.n_layer();
+ comp_read.assign(n_layer, -1);
+ comp_kv_src.resize(n_layer);
+ for (uint32_t il = 0; il < n_layer; ++il) {
+ const int32_t src = is_v41 ? model.hparams.dsv41_kv_src[il] : (int32_t) il;
+ comp_kv_src[il] = src >= 0 ? src : (int32_t) il;
+ if (model.hparams.dsv4_compress_ratios[il] == 0) {
+ continue;
+ }
+ comp_read[il] = src;
+ if (src == (int32_t) il || !offload || model.dev_layer(src) == model.dev_layer(il)) {
+ continue;
+ }
+ // the first layer of this source on this device holds the mirror
+ comp_read[il] = il;
+ for (int32_t j = src + 1; j < (int32_t) il; ++j) {
+ if (comp_read[j] == j && comp_kv_src[j] == src && model.dev_layer(j) == model.dev_layer(il)) {
+ comp_read[il] = j;
+ break;
+ }
+ }
+ }
+
+ // compressed-KV storage: kv sources and mirrors; compressor state: kv sources only
+ const auto is_comp_owner = [&](int32_t il, uint32_t ratio) {
+ return model.hparams.dsv4_compress_ratios[il] == ratio && comp_read[il] == (int32_t) il;
+ };
+
+ const auto is_state_owner = [&](int32_t il, uint32_t ratio) {
+ if (model.hparams.dsv4_compress_ratios[il] != ratio) {
+ return false;
+ }
+ return !is_v41 || model.hparams.dsv41_kv_src[il] == il;
+ };
+
const layer_filter_cb filter_csa = [&](int32_t il) {
if (filter && !filter(il)) {
return false;
}
- return model.hparams.dsv4_compress_ratios[il] == DSV4_CSA_RATIO;
+ return is_comp_owner(il, ratio_csa);
};
const layer_filter_cb filter_hca = [&](int32_t il) {
@@ -1285,52 +1385,67 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4(
return false;
}
- return model.hparams.dsv4_compress_ratios[il] == DSV4_HCA_RATIO;
+ return is_comp_owner(il, ratio_hca);
+ };
+
+ const layer_filter_cb filter_csa_state = [&](int32_t il) {
+ return (!filter || filter(il)) && is_state_owner(il, ratio_csa);
+ };
+
+ const layer_filter_cb filter_hca_state = [&](int32_t il) {
+ return (!filter || filter(il)) && is_state_owner(il, ratio_hca);
+ };
+
+ const layer_filter_cb filter_lid = [&](int32_t il) {
+ return !is_v41 && filter_csa(il);
};
const bool unified_compressed = false;
LLAMA_LOG_INFO("%s: creating DSV4 CSA compressed KV cache, size = %u cells\n",
- __func__, dsv4_comp_size(kv_size, DSV4_CSA_RATIO));
+ __func__, dsv4_comp_size(kv_size, ratio_csa));
kv_csa = std::make_unique<llama_kv_cache>(
model, hparams_csa, type_k, type_v,
- v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, DSV4_CSA_RATIO), 256u), n_seq_max, n_pad,
+ v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_csa), 256u), n_seq_max, n_pad,
0, LLAMA_SWA_TYPE_NONE, nullptr, filter_csa, nullptr, nullptr);
LLAMA_LOG_INFO("%s: creating DSV4 HCA compressed KV cache, size = %u cells\n",
- __func__, dsv4_comp_size(kv_size, DSV4_HCA_RATIO));
+ __func__, dsv4_comp_size(kv_size, ratio_hca));
kv_hca = std::make_unique<llama_kv_cache>(
model, hparams_hca, type_k, type_v,
- v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, DSV4_HCA_RATIO), 256u), n_seq_max, n_pad,
+ v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_hca), 256u), n_seq_max, n_pad,
0, LLAMA_SWA_TYPE_NONE, nullptr, filter_hca, nullptr, nullptr);
LLAMA_LOG_INFO("%s: creating DSV4 lightning-indexer KV cache, size = %u cells\n",
- __func__, dsv4_comp_size(kv_size, DSV4_CSA_RATIO));
+ __func__, dsv4_comp_size(kv_size, ratio_csa));
kv_lid = std::make_unique<llama_kv_cache>(
model, hparams_lid, type_k, type_v,
- v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, DSV4_CSA_RATIO), 256u), n_seq_max, n_pad,
- 0, LLAMA_SWA_TYPE_NONE, nullptr, filter_csa, nullptr, nullptr);
+ v_trans, offload, unified_compressed, GGML_PAD(dsv4_comp_size(kv_size, ratio_csa), 256u), n_seq_max, n_pad,
+ 0, LLAMA_SWA_TYPE_NONE, nullptr, filter_lid, nullptr, nullptr);
LLAMA_LOG_INFO("%s: creating DSV4 CSA compressor state\n", __func__);
+ const uint32_t csa_state_size = is_v41 ? ratio_csa : 2*ratio_csa;
+ const uint32_t csa_state_embd = is_v41 ? model.hparams.n_embd_head_k() : 2*model.hparams.n_embd_head_k();
+
csa_state = std::make_unique<llama_dsv4_comp_state>(
- model, offload, unified_compressed, n_seq_max, DSV4_CSA_RATIO, 2*DSV4_CSA_RATIO,
- 2*model.hparams.n_embd_head_k(), n_rs_seq, "csa", filter_csa);
+ model, offload, unified_compressed, n_seq_max, ratio_csa, csa_state_size,
+ csa_state_embd, n_rs_seq, "csa", filter_csa_state);
LLAMA_LOG_INFO("%s: creating DSV4 HCA compressor state\n", __func__);
hca_state = std::make_unique<llama_dsv4_comp_state>(
- model, offload, unified_compressed, n_seq_max, DSV4_HCA_RATIO, DSV4_HCA_RATIO,
- model.hparams.n_embd_head_k(), n_rs_seq, "hca", filter_hca);
+ model, offload, unified_compressed, n_seq_max, ratio_hca, ratio_hca,
+ model.hparams.n_embd_head_k(), n_rs_seq, "hca", filter_hca_state);
LLAMA_LOG_INFO("%s: creating DSV4 lightning-indexer compressor state\n", __func__);
lid_state = std::make_unique<llama_dsv4_comp_state>(
model, offload, unified_compressed, n_seq_max, DSV4_CSA_RATIO, 2*DSV4_CSA_RATIO,
- 2*model.hparams.indexer_head_size, n_rs_seq, "lid", filter_csa);
+ 2*model.hparams.indexer_head_size, n_rs_seq, "lid", filter_lid);
// DSV4 attention reads compressed-K / compressor-state rows that the current
// graph does not necessarily overwrite; uninitialized buffer contents would
@@ -1339,12 +1454,76 @@ llama_kv_cache_dsv4::llama_kv_cache_dsv4(
clear_compressed(-1, true);
}
+// Engram hashes the n-gram ending at each token, so a token shared by several sequences (llama-batched prompt)
+// is well-defined only if all of them agree on its predecessors; returns false (and logs) otherwise
+static bool dsv41_shared_history_ok(const llama_batch & batch, const llama_kv_cache * kv, uint32_t n_hist) {
+ if (n_hist == 0 || !batch.token || !batch.n_seq_id) {
+ return true;
+ }
+
+ bool any_shared = false;
+ for (int32_t i = 0; i < batch.n_tokens && !any_shared; ++i) {
+ any_shared = batch.n_seq_id[i] > 1;
+ }
+ if (!any_shared) {
+ return true;
+ }
+
+ std::map<std::pair<llama_seq_id, llama_pos>, llama_token> in_batch;
+ for (int32_t i = 0; i < batch.n_tokens; ++i) {
+ for (int32_t s = 0; s < batch.n_seq_id[i]; ++s) {
+ in_batch[{ batch.seq_id[i][s], batch.pos[i] }] = batch.token[i];
+ }
+ }
+
+ const auto tok_at = [&](llama_seq_id seq_id, llama_pos p) -> llama_token {
+ // batch first: the batch continues the cache
+ auto it = in_batch.upper_bound({ seq_id, p });
+ if (it != in_batch.begin()) {
+ --it;
+ if (it->first.first == seq_id) {
+ return it->second;
+ }
+ }
+ return kv ? kv->get_cells(seq_id).seq_pos_tok_le(seq_id, p) : LLAMA_TOKEN_NULL;
+ };
+
+ for (int32_t i = 0; i < batch.n_tokens; ++i) {
+ if (batch.n_seq_id[i] <= 1) {
+ continue;
+ }
+ for (uint32_t d = 1; d <= n_hist; ++d) {
+ const llama_pos p = batch.pos[i] - (llama_pos) d;
+ if (p < 0) {
+ break;
+ }
+ const llama_token t0 = tok_at(batch.seq_id[i][0], p);
+ for (int32_t s = 1; s < batch.n_seq_id[i]; ++s) {
+ const llama_token ts = tok_at(batch.seq_id[i][s], p);
+ if (ts != t0) {
+ LLAMA_LOG_ERROR("%s: token %d at pos %d is shared by seq %d and seq %d, whose Engram n-gram histories "
+ "differ at pos %d (%d vs %d); decode such tokens per sequence\n", __func__,
+ i, batch.pos[i], batch.seq_id[i][0], batch.seq_id[i][s], p, t0, ts);
+ return false;
+ }
+ }
+ }
+ }
+
+ return true;
+}
+
llama_memory_context_ptr llama_kv_cache_dsv4::init_batch(
llama_batch_allocr & balloc,
uint32_t n_ubatch,
bool embd_all) {
GGML_UNUSED(embd_all);
+ if (hparams_raw.ple_n_heads > 0 &&
+ !dsv41_shared_history_ok(balloc.get_batch(), kv_raw->get_swa(), hparams_raw.ple_ngram_size - 1)) {
+ return std::make_unique<llama_kv_cache_dsv4_context>(LLAMA_MEMORY_STATUS_FAILED_PREPARE);
+ }
+
const bool raw_per_seq = kv_raw->get_base()->get_n_stream() != 1;
const bool comp_per_seq = csa_state->get_n_stream() > 1;
const bool has_coupled = dsv4_batch_has_coupled(balloc.get_batch());
@@ -1475,9 +1654,9 @@ bool llama_kv_cache_dsv4::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1
bool res = true;
res = res & kv_raw->seq_rm(seq_id, p0, -1);
- res = res & kv_csa->seq_rm(seq_id, p0/DSV4_CSA_RATIO, -1);
- res = res & kv_hca->seq_rm(seq_id, p0/DSV4_HCA_RATIO, -1);
- res = res & kv_lid->seq_rm(seq_id, p0/DSV4_CSA_RATIO, -1);
+ res = res & kv_csa->seq_rm(seq_id, p0/csa_state->get_ratio(), -1);
+ res = res & kv_hca->seq_rm(seq_id, p0/hca_state->get_ratio(), -1);
+ res = res & kv_lid->seq_rm(seq_id, p0/lid_state->get_ratio(), -1);
return res;
}
@@ -1613,15 +1792,15 @@ void llama_kv_cache_dsv4::state_write(llama_io_write_i & io, llama_seq_id seq_id
//FIXME : note that we conflate token positions with rows, which is not true for multi-modal case.
const uint32_t n_rows_csa = seq_id >= 0 ?
- dsv4_state_n_used_k_rows(pos_max, DSV4_CSA_RATIO, kv_csa->get_size()) : kv_csa->get_size();
+ dsv4_state_n_used_k_rows(pos_max, csa_state->get_ratio(), kv_csa->get_size()) : kv_csa->get_size();
const uint32_t n_rows_hca = seq_id >= 0 ?
- dsv4_state_n_used_k_rows(pos_max, DSV4_HCA_RATIO, kv_hca->get_size()) : kv_hca->get_size();
+ dsv4_state_n_used_k_rows(pos_max, hca_state->get_ratio(), kv_hca->get_size()) : kv_hca->get_size();
const uint32_t n_rows_lid = seq_id >= 0 ?
- dsv4_state_n_used_k_rows(pos_max, DSV4_CSA_RATIO, kv_lid->get_size()) : kv_lid->get_size();
+ dsv4_state_n_used_k_rows(pos_max, lid_state->get_ratio(), kv_lid->get_size()) : kv_lid->get_size();
- dsv4_state_write_k_cache(io, kv_csa.get(), seq_id, flags, n_rows_csa);
- dsv4_state_write_k_cache(io, kv_hca.get(), seq_id, flags, n_rows_hca);
- dsv4_state_write_k_cache(io, kv_lid.get(), seq_id, flags, n_rows_lid);
+ dsv4_state_write_k_cache(io, kv_csa.get(), seq_id, flags, n_rows_csa, comp_kv_src);
+ dsv4_state_write_k_cache(io, kv_hca.get(), seq_id, flags, n_rows_hca, comp_kv_src);
+ dsv4_state_write_k_cache(io, kv_lid.get(), seq_id, flags, n_rows_lid, comp_kv_src);
}
csa_state->state_write(io, seq_id, flags, rs_idx);
@@ -1659,9 +1838,9 @@ void llama_kv_cache_dsv4::state_read(llama_io_read_i & io, llama_seq_id seq_id,
if (!partial_only) {
clear_compressed(seq_id, true);
- dsv4_state_read_k_cache(io, kv_csa.get(), seq_id, flags);
- dsv4_state_read_k_cache(io, kv_hca.get(), seq_id, flags);
- dsv4_state_read_k_cache(io, kv_lid.get(), seq_id, flags);
+ dsv4_state_read_k_cache(io, kv_csa.get(), seq_id, flags, comp_kv_src);
+ dsv4_state_read_k_cache(io, kv_hca.get(), seq_id, flags, comp_kv_src);
+ dsv4_state_read_k_cache(io, kv_lid.get(), seq_id, flags, comp_kv_src);
}
csa_state->state_read(io, seq_id, flags);
@@ -1704,6 +1883,10 @@ llama_dsv4_comp_state * llama_kv_cache_dsv4::get_lid_state() const {
return lid_state.get();
}
+int32_t llama_kv_cache_dsv4::get_comp_read_layer(int32_t il) const {
+ return comp_read[il];
+}
+
uint32_t llama_kv_cache_dsv4::get_n_rs_seq() const {
return n_rs_seq;
}
@@ -1932,6 +2115,11 @@ void llama_kv_cache_dsv4_raw_context::set_input_kq_mask(ggml_tensor * dst, const
kv_swa->set_input_kq_mask(dst, ubatch, causal_attn);
}
+void llama_kv_cache_dsv4_raw_context::get_prev_tokens(const llama_ubatch & ubatch, uint32_t n, std::vector<llama_token> & res) const {
+ GGML_ASSERT(kv_swa != nullptr);
+ kv_swa->get_prev_tokens(ubatch, n, res);
+}
+
void llama_kv_cache_dsv4_raw_context::set_input_k_rot(ggml_tensor * dst) const {
kv_swa->set_input_k_rot(dst);
}
@@ -2016,7 +2204,8 @@ llama_kv_cache_dsv4_context::llama_kv_cache_dsv4_context(
reserve_plans(true),
status(llama_memory_status_combine(
llama_memory_status_combine(ctx_raw->get_status(), ctx_csa_mem->get_status()),
- llama_memory_status_combine(ctx_hca_mem->get_status(), ctx_lid_mem->get_status()))) {
+ llama_memory_status_combine(ctx_hca_mem->get_status(), ctx_lid_mem->get_status()))),
+ kv(kv) {
}
llama_kv_cache_dsv4_context::llama_kv_cache_dsv4_context(
@@ -2041,7 +2230,8 @@ llama_kv_cache_dsv4_context::llama_kv_cache_dsv4_context(
llama_memory_status_combine(ctx_raw->get_status(), ctx_csa_mem->get_status()),
llama_memory_status_combine(ctx_hca_mem->get_status(), ctx_lid_mem->get_status())),
this->sc_info_csa.empty() && this->sc_info_hca.empty() && this->sc_info_lid.empty() ?
- LLAMA_MEMORY_STATUS_NO_UPDATE : LLAMA_MEMORY_STATUS_SUCCESS)) {
+ LLAMA_MEMORY_STATUS_NO_UPDATE : LLAMA_MEMORY_STATUS_SUCCESS)),
+ kv(kv) {
}
llama_kv_cache_dsv4_context::llama_kv_cache_dsv4_context(
@@ -2052,13 +2242,13 @@ llama_kv_cache_dsv4_context::llama_kv_cache_dsv4_context(
std::vector<llama_ubatch> ubatches,
std::vector<llama_ubatch> ubatches_raw) :
ubatches(std::move(ubatches)),
- plans_csa(dsv4_build_comp_plans(this->ubatches, DSV4_CSA_RATIO, true,
+ plans_csa(dsv4_build_comp_plans(this->ubatches, kv->get_csa_state()->get_ratio(), kv->get_csa_state()->is_overlap(), true,
kv->get_csa_state()->get_state_size(), kv->get_csa()->get_size(), kv->get_csa_state()->get_n_stream(),
kv->get_n_rs_seq(), kv->get_rs_idx())),
- plans_hca(dsv4_build_comp_plans(this->ubatches, DSV4_HCA_RATIO, false,
+ plans_hca(dsv4_build_comp_plans(this->ubatches, kv->get_hca_state()->get_ratio(), kv->get_hca_state()->is_overlap(), false,
kv->get_hca_state()->get_state_size(), kv->get_hca()->get_size(), kv->get_hca_state()->get_n_stream(),
kv->get_n_rs_seq(), kv->get_rs_idx())),
- plans_lid(dsv4_build_comp_plans(this->ubatches, DSV4_CSA_RATIO, true,
+ plans_lid(dsv4_build_comp_plans(this->ubatches, kv->get_lid_state()->get_ratio(), kv->get_lid_state()->is_overlap(), true,
kv->get_lid_state()->get_state_size(), kv->get_lid()->get_size(), kv->get_lid_state()->get_n_stream(),
kv->get_n_rs_seq(), kv->get_rs_idx())),
ctx_raw(std::make_unique<llama_kv_cache_dsv4_raw_context>(
@@ -2086,7 +2276,8 @@ llama_kv_cache_dsv4_context::llama_kv_cache_dsv4_context(
csa_state(kv->get_csa_state()),
hca_state(kv->get_hca_state()),
lid_state(kv->get_lid_state()),
- status(ctx_raw->get_status()) {
+ status(ctx_raw->get_status()),
+ kv(kv) {
kv->reset_rs_idx_for_ubatches(this->ubatches);
}
@@ -2181,6 +2372,10 @@ const llama_dsv4_comp_state * llama_kv_cache_dsv4_context::get_lid_state() const
return lid_state;
}
+int32_t llama_kv_cache_dsv4_context::get_comp_read_layer(int32_t il) const {
+ return kv->get_comp_read_layer(il);
+}
+
const llama_kv_cache_dsv4_context::comp_plan & llama_kv_cache_dsv4_context::get_csa_plan() const {
assert(status == LLAMA_MEMORY_STATUS_SUCCESS);
@@ -2222,7 +2417,7 @@ const llama_kv_cache_dsv4_context::comp_plan & llama_kv_cache_dsv4_context::get_
}
reserve_plan_csa = dsv4_build_reserve_comp_plan(
- ubatch, DSV4_CSA_RATIO, true,
+ ubatch, csa_state->get_ratio(), csa_state->is_overlap(),
csa_state->get_state_size(), get_csa()->get_n_kv(), csa_state->get_n_stream(), csa_state->get_n_rs_seq());
return reserve_plan_csa;
@@ -2236,7 +2431,7 @@ const llama_kv_cache_dsv4_context::comp_plan & llama_kv_cache_dsv4_context::get_
}
reserve_plan_hca = dsv4_build_reserve_comp_plan(
- ubatch, DSV4_HCA_RATIO, false,
+ ubatch, hca_state->get_ratio(), hca_state->is_overlap(),
hca_state->get_state_size(), get_hca()->get_n_kv(), hca_state->get_n_stream(), hca_state->get_n_rs_seq());
return reserve_plan_hca;
@@ -2250,7 +2445,7 @@ const llama_kv_cache_dsv4_context::comp_plan & llama_kv_cache_dsv4_context::get_
}
reserve_plan_lid = dsv4_build_reserve_comp_plan(
- ubatch, DSV4_CSA_RATIO, true,
+ ubatch, lid_state->get_ratio(), lid_state->is_overlap(),
lid_state->get_state_size(), get_lid()->get_n_kv(), lid_state->get_n_stream(), lid_state->get_n_rs_seq());
return reserve_plan_lid;
diff --git a/src/llama-kv-cache-dsv4.h b/src/llama-kv-cache-dsv4.h
index ce39867c0..0368ea59f 100644
--- a/src/llama-kv-cache-dsv4.h
+++ b/src/llama-kv-cache-dsv4.h
@@ -31,6 +31,7 @@ public:
void apply_copies(const stream_copy_info & sc_info) const;
uint32_t get_ratio() const;
+ bool is_overlap() const; // V4 CSA: state holds 2*ratio rows of 2x width
uint32_t get_state_size() const;
uint32_t get_n_stream() const;
uint32_t get_n_rs_seq() const;
@@ -148,6 +149,9 @@ public:
llama_dsv4_comp_state * get_hca_state() const;
llama_dsv4_comp_state * get_lid_state() const;
+ // layer whose compressed-KV storage `il` reads, -1 = no compressed KV
+ int32_t get_comp_read_layer(int32_t il) const;
+
uint32_t get_n_rs_seq() const;
const std::vector<uint32_t> & get_rs_idx() const;
void reset_rs_idx_for_ubatches(const std::vector<llama_ubatch> & ubatches);
@@ -163,6 +167,12 @@ private:
std::vector<uint32_t> rs_idx;
+ // [n_layer] storage layer -> layer whose rows it holds (itself, or the source of a mirror)
+ std::vector<int32_t> comp_kv_src;
+
+ // [n_layer] layer -> layer whose compressed-KV storage it reads (itself, its V4.1 kv source or a mirror), -1 = none
+ std::vector<int32_t> comp_read;
+
std::unique_ptr<llama_kv_cache_iswa> kv_raw;
std::unique_ptr<llama_kv_cache> kv_csa;
std::unique_ptr<llama_kv_cache> kv_hca;
@@ -214,6 +224,9 @@ public:
void set_input_kq_mask(ggml_tensor * dst, const llama_ubatch * ubatch, bool causal_attn) const;
void set_input_k_rot(ggml_tensor * dst) const;
+ // token history of the raw (window) cells, for host-side n-gram hashing (deepseek41 Engram)
+ void get_prev_tokens(const llama_ubatch & ubatch, uint32_t n, std::vector<llama_token> & res) const;
+
private:
size_t i_next = 0;
@@ -371,6 +384,8 @@ public:
const comp_plan & get_hca_plan(const llama_ubatch & ubatch) const;
const comp_plan & get_lid_plan(const llama_ubatch & ubatch) const;
+ int32_t get_comp_read_layer(int32_t il) const;
+
private:
size_t i_next = 0;
@@ -403,4 +418,6 @@ private:
mutable comp_plan reserve_plan_lid;
const llama_memory_status status;
+
+ const llama_kv_cache_dsv4 * kv = nullptr;
};
diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp
index 6f0e3f7f7..4ab280a61 100644
--- a/src/llama-kv-cache.cpp
+++ b/src/llama-kv-cache.cpp
@@ -1168,9 +1168,7 @@ void llama_kv_cache::apply_ubatch(const slot_info & sinfo, const llama_ubatch &
} else if (hparams.ple_n_heads > 0) {
// embd batch (multimodal input) has no token ids, need to pad it with the correct ID for PLE layers
// TODO @ngxson : check if we can do the same as gemma 3n / gemma 4
- ext.tok = hparams.ple_image_token_id != 0
- ? (llama_token) hparams.ple_image_token_id
- : (llama_token) hparams.ple_eos_token_id;
+ ext.tok = hparams.ple_embd_token;
}
cells.ext_set(idx, ext);
@@ -1849,7 +1847,7 @@ void llama_kv_cache::set_input_v_rot(ggml_tensor * dst) const {
}
bool llama_kv_cache::has_cell_ext() const {
- // M-RoPE needs the 2D position, the PLE n-gram hash needs the token id
+ // M-RoPE needs the 2D position, the PLE / Engram n-gram hashes need the token id
return hparams.n_pos_per_embd() > 1 || hparams.ple_n_heads > 0;
}
diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp
index 066e69e91..a963f2480 100644
--- a/src/llama-model-loader.cpp
+++ b/src/llama-model-loader.cpp
@@ -414,6 +414,7 @@ namespace GGUFMeta {
template bool llama_model_loader::get_arr<std::vector<int32_t>>(enum llm_kv kid, std::vector<int32_t> & result, bool required);
template bool llama_model_loader::get_arr<std::array<uint32_t, LLAMA_MAX_LAYERS>>(enum llm_kv kid, std::array<uint32_t, LLAMA_MAX_LAYERS> & result, bool required);
template bool llama_model_loader::get_arr<std::vector<uint32_t>>(enum llm_kv kid, std::vector<uint32_t> & result, bool required);
+ template bool llama_model_loader::get_arr<std::vector<uint64_t>>(enum llm_kv kid, std::vector<uint64_t> & result, bool required);
template bool llama_model_loader::get_arr<std::array<uint64_t, LLAMA_MAX_PLE_NGRAM>>(enum llm_kv kid, std::array<uint64_t, LLAMA_MAX_PLE_NGRAM> & result, bool required);
template bool llama_model_loader::get_arr<std::array<uint64_t, LLAMA_MAX_PLE_HEADS>>(enum llm_kv kid, std::array<uint64_t, LLAMA_MAX_PLE_HEADS> & result, bool required);
@@ -441,6 +442,7 @@ namespace GGUFMeta {
template bool llama_model_loader::get_key<bool> (enum llm_kv kid, bool & result, bool required);
template bool llama_model_loader::get_key<float> (enum llm_kv kid, float & result, bool required);
template bool llama_model_loader::get_key<uint32_t> (enum llm_kv kid, uint32_t & result, bool required);
+ template bool llama_model_loader::get_key<int32_t> (enum llm_kv kid, int32_t & result, bool required);
template bool llama_model_loader::get_key<std::string>(enum llm_kv kid, std::string & result, bool required);
template<>
diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp
index 2135b82b1..752e6361f 100644
--- a/src/llama-model-saver.cpp
+++ b/src/llama-model-saver.cpp
@@ -8,6 +8,7 @@
#include "llama-hparams.h"
#include "llama-model.h"
#include "llama-vocab.h"
+#include "models/models.h"
#include <cstdint>
#include <string>
@@ -122,6 +123,7 @@ void llama_model_saver::add_kv(const enum llm_kv key, const Container & value, c
template void llama_model_saver::add_kv<std::vector<uint32_t>>(const enum llm_kv, const std::vector<uint32_t> &, const bool);
template void llama_model_saver::add_kv<std::vector<float>>(const enum llm_kv, const std::vector<float> &, const bool);
template void llama_model_saver::add_kv<std::vector<uint64_t>>(const enum llm_kv, const std::vector<uint64_t> &, const bool);
+template void llama_model_saver::add_kv<std::vector<int32_t>>(const enum llm_kv, const std::vector<int32_t> &, const bool);
void llama_model_saver::add_kv(const enum llm_kv key, const std::vector<std::string> & value) {
std::vector<const char *> tmp(value.size());
@@ -339,6 +341,32 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_HASH_LAYER_COUNT, hparams.dsv4_hash_layer_count);
add_kv(LLM_KV_HYPER_CONNECTION_LOW_RANK, hparams.hc_low_rank);
+ if (model->arch == LLM_ARCH_DEEPSEEK41) {
+ const auto * m41 = dynamic_cast<const llama_model_deepseek41 *>(model);
+ GGML_ASSERT(m41 != nullptr);
+ std::vector<uint32_t> kv_src;
+ std::vector<uint32_t> idx_src;
+ for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
+ if (hparams.dsv41_kv_src[il] == (int32_t) il) {
+ kv_src.push_back(il);
+ }
+ if (hparams.dsv41_topk_src[il] == (int32_t) il) {
+ idx_src.push_back(il);
+ }
+ }
+ add_kv(LLM_KV_ATTENTION_KV_SOURCE_LAYERS, kv_src);
+ add_kv(LLM_KV_ATTENTION_INDEXER_SOURCE_LAYERS, idx_src);
+ if (hparams.dsv41_cand_src >= 0) {
+ add_kv(LLM_KV_ATTENTION_INDEXER_CANDIDATE_SOURCE_LAYER, hparams.dsv41_cand_src);
+ add_kv(LLM_KV_ATTENTION_INDEXER_CANDIDATE_TOP_K_BLOCKS, hparams.dsv41_cand_topk_blocks);
+ add_kv(LLM_KV_ATTENTION_INDEXER_CANDIDATE_BLOCK_SIZE, hparams.dsv41_cand_block_size);
+ }
+ if (hparams.ple_n_heads > 0) {
+ add_kv(LLM_KV_ENGRAM_PAD_TOKEN_ID, m41->engram_pad_id);
+ add_kv(LLM_KV_ENGRAM_TOKEN_MAP, m41->engram_token_map);
+ }
+ }
+
// the PLE group only means anything whole: write all of it or none
if (hparams.ple_n_heads > 0) {
std::vector<uint32_t> ple_layers;
@@ -355,13 +383,13 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, hparams.ple_head_dim);
add_kv(LLM_KV_PLE_LAYER_MULTIPLIERS, std::vector<uint64_t>(
hparams.ple_layer_multipliers.begin(),
- hparams.ple_layer_multipliers.begin() + hparams.ple_ngram_size));
+ hparams.ple_layer_multipliers.begin() + hparams.ple_n_modules*hparams.ple_ngram_size));
add_kv(LLM_KV_PLE_HEAD_OFFSETS, std::vector<uint64_t>(
hparams.ple_head_offsets.begin(),
- hparams.ple_head_offsets.begin() + hparams.ple_n_heads));
+ hparams.ple_head_offsets.begin() + hparams.ple_n_modules*hparams.ple_n_heads));
add_kv(LLM_KV_PLE_HEAD_VOCAB_SIZES, std::vector<uint64_t>(
hparams.ple_head_vocab_sizes.begin(),
- hparams.ple_head_vocab_sizes.begin() + hparams.ple_n_heads));
+ hparams.ple_head_vocab_sizes.begin() + hparams.ple_n_modules*hparams.ple_n_heads));
}
const float rope_scaling_factor = hparams.rope_freq_scale_train == 1.0f ? 0.0f : 1.0f/hparams.rope_freq_scale_train;
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 65c42f85f..1de65c728 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -212,6 +212,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_dots3note(params);
case LLM_ARCH_DEEPSEEK4:
return new llama_model_deepseek4(params);
+ case LLM_ARCH_DEEPSEEK41:
+ return new llama_model_deepseek41(params);
case LLM_ARCH_GLM_DSA:
return new llama_model_glm_dsa(params);
case LLM_ARCH_MISTRAL4:
@@ -398,7 +400,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
const llama_meta_device_get_split_state_userdata * ud = (const llama_meta_device_get_split_state_userdata *) userdata;
const llama_hparams & hparams = ud->model->hparams;
const std::string tensor_name = tensor->name;
- const bool is_dsv4 = ud->model->arch == LLM_ARCH_DEEPSEEK4 ||
+ const bool is_dsv4 = ud->model->arch == LLM_ARCH_DEEPSEEK4 || ud->model->arch == LLM_ARCH_DEEPSEEK41 ||
(ud->model->arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0);
static const std::regex pattern_q_weight ("blk\\.\\d*\\.attn_q.weight");
@@ -3032,6 +3034,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
nullptr);
}
} break;
+ case LLM_ARCH_DEEPSEEK41:
case LLM_ARCH_DEEPSEEK4:
{
GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE);
@@ -3441,7 +3444,7 @@ int32_t llama_model_n_head_kv(const llama_model * model) {
int32_t llama_model_n_swa(const llama_model * model) {
// dsv4 kv-cache has SWA but it cannot be used as a rollback because of
// other compression ratios, so we return 0 here
- if (model->arch == LLM_ARCH_DEEPSEEK4) {
+ if (model->arch == LLM_ARCH_DEEPSEEK4 || model->arch == LLM_ARCH_DEEPSEEK41) {
return 0;
}
return model->hparams.n_swa;
@@ -3536,6 +3539,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_DEEPSEEK2OCR:
case LLM_ARCH_DEEPSEEK32:
case LLM_ARCH_DEEPSEEK4:
+ case LLM_ARCH_DEEPSEEK41:
case LLM_ARCH_MUSE_GLIMMER:
case LLM_ARCH_MUSE_SPARK:
case LLM_ARCH_PLM:
diff --git a/src/llama-model.h b/src/llama-model.h
index cdf060db2..93164f433 100644
--- a/src/llama-model.h
+++ b/src/llama-model.h
@@ -605,6 +605,10 @@ struct llama_layer {
struct ggml_tensor * indexer_kpool_gate = nullptr;
struct ggml_tensor * indexer_kpool_ape = nullptr;
+ // deepseek41
+ struct ggml_tensor * engram_kv = nullptr;
+ struct ggml_tensor * engram_gate = nullptr; // q_weight*k_weight folded
+
// MSA
struct ggml_tensor * index_q_proj = nullptr;
struct ggml_tensor * index_k_proj = nullptr;
diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp
index b8f96bcef..1348884dd 100644
--- a/src/llama-quant.cpp
+++ b/src/llama-quant.cpp
@@ -316,6 +316,9 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param
// do not quantize the i32 token-id -> expert-id routing table (DeepSeek-V4)
quantize &= name.find("ffn_gate_tid2eid.weight") == std::string::npos;
+ // Engram gate is an elementwise weight (ggml_mul), not a matmul operand
+ quantize &= name.find("engram_gate.weight") == std::string::npos;
+
// these are very small (e.g. 4x4)
quantize &= name.find("altup") == std::string::npos;
quantize &= name.find("laurel") == std::string::npos;
diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp
index 11699909f..7dc5ee3a5 100644
--- a/src/models/deepseek4.cpp
+++ b/src/models/deepseek4.cpp
@@ -8,14 +8,6 @@
#include <stdexcept>
#include <string>
-static float dsv4_rope_attn_factor(float freq_scale, float ext_factor) {
- if (ext_factor == 0.0f) {
- return 1.0f;
- }
-
- return 1.0f / (1.0f + 0.1f*logf(1.0f/freq_scale));
-}
-
void llama_model_deepseek4::load_arch_hparams(llama_model_loader & ml) {
if (hparams.n_layer_nextn > 0) {
const uint32_t n_layer_main = hparams.n_layer_all - hparams.n_layer_nextn;
@@ -358,6 +350,18 @@ ggml_tensor * llama_model_deepseek4::graph::build_hc_pre(
ggml_tensor ** post,
ggml_tensor ** comb,
int il) const {
+ return build_hc_pre(x, build_hc_mixes(x, hc_fn, hc_scale, hc_base, post, comb, false, il), il);
+}
+
+ggml_tensor * llama_model_deepseek4::graph::build_hc_mixes(
+ ggml_tensor * x,
+ ggml_tensor * hc_fn,
+ ggml_tensor * hc_scale,
+ ggml_tensor * hc_base,
+ ggml_tensor ** post,
+ ggml_tensor ** comb,
+ bool f32_src,
+ int il) const {
const int64_t hc = hparams.dsv4_hc_mult;
const int64_t hc_dim = hc*n_embd;
const int64_t hc_mix_dim = (2 + hc)*hc;
@@ -369,6 +373,10 @@ ggml_tensor * llama_model_deepseek4::graph::build_hc_pre(
ggml_tensor * flat = ggml_reshape_2d(ctx0, x, hc_dim, nt);
ggml_tensor * flat_norm = ggml_rms_norm(ctx0, flat, norm_rms_eps);
ggml_tensor * mixes = ggml_mul_mat(ctx0, hc_fn, flat_norm);
+ if (f32_src && !ggml_is_quantized(hc_fn->type)) {
+ // F32 weights (model.py hc_pre runs in fp32): request F32 inputs, the mixes are near-tie sensitive
+ ggml_prec_set_src(mixes, GGML_PREC_F32, 1);
+ }
cb(mixes, "hc_mixes", il);
ggml_tensor * scale_pre = dsv4_view_1d(ctx0, hc_scale, 1, 0);
@@ -404,8 +412,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_hc_pre(
}
cb(*comb, "hc_comb", il);
- ggml_tensor * result = build_hc_pre(x, pre, il);
- return result;
+ return pre;
}
ggml_tensor * llama_model_deepseek4::graph::build_hc_post(
diff --git a/src/models/deepseek41.cpp b/src/models/deepseek41.cpp
new file mode 100644
index 000000000..49e95f54c
--- /dev/null
+++ b/src/models/deepseek41.cpp
@@ -0,0 +1,950 @@
+#include "models.h"
+#include "llama-impl.h"
+
+#include "llama-kv-cache-dsv4.h"
+
+#include <algorithm>
+#include <cinttypes>
+#include <cmath>
+#include <cstdlib>
+#include <stdexcept>
+#include <string>
+
+// ref: deepseek-ai/DeepSeek-V4.1-Flash-1017 inference/model.py, engram.py
+// compressed layers attend over [window | all compressed rows] with top-k as a mask. with flash attention the default
+// is sparse: fused indexer, FA bounded to window + top-k, decode-sized ubatches gather only the selected rows (same
+// keys and mask, different key order). LLAMA_DSV41_SPARSE=0 restores the reference path
+
+static constexpr int64_t DSV41_ENGRAM_DEAD = -1;
+
+static bool dsv41_env_on(const char * name) {
+ const char * v = getenv(name);
+ return !v || atoi(v) != 0;
+}
+
+void llama_model_deepseek41::load_arch_hparams(llama_model_loader & ml) {
+ llama_model_deepseek4::load_arch_hparams(ml);
+
+ hparams.n_embd_out_impl = hparams.n_embd; // no hc_head
+ if (hparams.indexer_top_k == 0) {
+ throw std::runtime_error("DeepSeek-V4.1 indexer top_k must be > 0");
+ }
+
+ // each compressed layer reads the KV / top-k of the latest source layer at or before it
+ std::vector<uint32_t> kv_src;
+ std::vector<uint32_t> idx_src;
+ ml.get_arr(LLM_KV_ATTENTION_KV_SOURCE_LAYERS, kv_src);
+ ml.get_arr(LLM_KV_ATTENTION_INDEXER_SOURCE_LAYERS, idx_src);
+
+ const uint32_t n_layer = hparams.n_layer();
+ const auto out_of_range = [&](const std::vector<uint32_t> & v) {
+ return std::any_of(v.begin(), v.end(), [&](uint32_t il) { return il >= n_layer; });
+ };
+ if (out_of_range(kv_src) || out_of_range(idx_src)) {
+ throw std::runtime_error("DeepSeek-V4.1 kv/index source layer out of range");
+ }
+ hparams.dsv41_kv_src.fill(-1);
+ hparams.dsv41_topk_src.fill(-1);
+ int32_t last_kv = -1;
+ int32_t last_idx = -1;
+ for (uint32_t il = 0; il < n_layer; ++il) {
+ const uint32_t ratio = hparams.dsv4_compress_ratios[il];
+ const bool is_kv = std::find(kv_src.begin(), kv_src.end(), il) != kv_src.end();
+ const bool is_idx = std::find(idx_src.begin(), idx_src.end(), il) != idx_src.end();
+ if ((is_kv || is_idx) && ratio == 0) {
+ throw std::runtime_error(format("DeepSeek-V4.1 source layer %u has compress ratio 0", il));
+ }
+ last_kv = is_kv ? (int32_t) il : last_kv;
+ last_idx = is_idx ? (int32_t) il : last_idx;
+ if (ratio == 0) {
+ continue;
+ }
+ if (ratio > 2 || last_kv < 0 || last_idx < last_kv ||
+ hparams.dsv4_compress_ratios[last_kv] != ratio || hparams.dsv4_compress_ratios[last_idx] != ratio) {
+ throw std::runtime_error(format("DeepSeek-V4.1 layer %u: ratio %u without a matching kv/index source", il, ratio));
+ }
+ hparams.dsv41_kv_src[il] = last_kv;
+ hparams.dsv41_topk_src[il] = last_idx;
+ }
+
+ ml.get_key(LLM_KV_ATTENTION_INDEXER_CANDIDATE_SOURCE_LAYER, hparams.dsv41_cand_src, false);
+ if (hparams.dsv41_cand_src >= 0) {
+ ml.get_key(LLM_KV_ATTENTION_INDEXER_CANDIDATE_TOP_K_BLOCKS, hparams.dsv41_cand_topk_blocks);
+ ml.get_key(LLM_KV_ATTENTION_INDEXER_CANDIDATE_BLOCK_SIZE, hparams.dsv41_cand_block_size);
+ const int32_t cs = hparams.dsv41_cand_src;
+ if ((uint32_t) cs >= n_layer || hparams.dsv41_topk_src[cs] != cs || hparams.dsv41_cand_block_size == 0 ||
+ hparams.dsv41_cand_topk_blocks == 0) {
+ throw std::runtime_error("DeepSeek-V4.1 invalid candidate selection (source must be an index source, block size and top-k blocks > 0)");
+ }
+ for (uint32_t il = cs + 1; il < n_layer; ++il) {
+ if (hparams.dsv41_topk_src[il] == (int32_t) il && hparams.dsv4_compress_ratios[il] != hparams.dsv4_compress_ratios[cs]) {
+ throw std::runtime_error("DeepSeek-V4.1 candidate consumers must share the source's compress ratio");
+ }
+ }
+ }
+
+ // Engram: the PLE n-gram hash of qwen4exp with one set of hash constants per Engram layer, stored back to back
+ std::vector<uint32_t> e_layers;
+ ml.get_arr(LLM_KV_PLE_LAYERS, e_layers, false);
+ if (!e_layers.empty()) {
+ std::vector<uint64_t> mult;
+ std::vector<uint64_t> offsets;
+ std::vector<uint64_t> primes;
+ ml.get_key(LLM_KV_PLE_NGRAM_SIZE, hparams.ple_ngram_size);
+ ml.get_key(LLM_KV_PLE_HEADS_PER_NGRAM, hparams.ple_heads_per_ngram);
+ ml.get_key(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, hparams.ple_head_dim);
+ ml.get_arr(LLM_KV_PLE_LAYER_MULTIPLIERS, mult);
+ ml.get_arr(LLM_KV_PLE_HEAD_OFFSETS, offsets);
+ ml.get_arr(LLM_KV_PLE_HEAD_VOCAB_SIZES, primes);
+ ml.get_key(LLM_KV_ENGRAM_PAD_TOKEN_ID, engram_pad_id);
+ ml.get_arr(LLM_KV_ENGRAM_TOKEN_MAP, engram_token_map);
+
+ const size_t n_e = e_layers.size();
+ const size_t n_gram = hparams.ple_ngram_size;
+ const size_t n_cols = n_gram >= 2 ? (n_gram - 1)*hparams.ple_heads_per_ngram : 0;
+ if (n_cols == 0 || hparams.ple_head_dim == 0 ||
+ mult.size() != n_e*n_gram || primes.size() != n_e*n_cols || offsets.size() != n_e*n_cols ||
+ mult.size() > LLAMA_MAX_PLE_NGRAM || primes.size() > LLAMA_MAX_PLE_HEADS ||
+ engram_pad_id >= engram_token_map.size()) {
+ throw std::runtime_error("DeepSeek-V4.1 engram metadata is inconsistent");
+ }
+ for (size_t e = 0; e < n_e; ++e) {
+ // module e belongs to the e-th Engram layer
+ if (e_layers[e] >= n_layer || (e > 0 && e_layers[e] <= e_layers[e - 1])) {
+ throw std::runtime_error(format("DeepSeek-V4.1 engram layer %u is out of range or out of order", e_layers[e]));
+ }
+ hparams.is_ple_impl.set(e_layers[e]);
+ }
+ for (size_t c = 0; c < n_e*n_cols; ++c) {
+ // a bucket count of 0 traps in the hash; row ids are int32
+ if (primes[c] < 2 || offsets[c] + primes[c] > (uint64_t) INT32_MAX) {
+ throw std::runtime_error(format("DeepSeek-V4.1 engram head %zu: bucket range out of range", c));
+ }
+ hparams.ple_head_offsets[c] = (uint32_t) offsets[c];
+ hparams.ple_head_vocab_sizes[c] = (uint32_t) primes[c];
+ }
+ std::copy(mult.begin(), mult.end(), hparams.ple_layer_multipliers.begin());
+ hparams.ple_n_heads = (uint32_t) n_cols;
+ hparams.ple_n_modules = (uint32_t) n_e;
+ }
+}
+
+void llama_model_deepseek41::load_arch_tensors(llama_model_loader & ml) {
+ LLAMA_LOAD_LOCALS;
+
+ const int64_t q_lora_rank = hparams.n_lora_q;
+ const int64_t n_ff_exp = hparams.n_ff_exp();
+ const int64_t n_expert_shared = hparams.n_expert_shared;
+ const int64_t n_embd_head = hparams.n_embd_head_k();
+ const int64_t o_groups = hparams.dsv4_o_group_count;
+ const int64_t o_lora_rank = hparams.dsv4_o_lora_rank;
+ const int64_t hc_mult = hparams.dsv4_hc_mult;
+ const int64_t hc_dim = hc_mult * n_embd;
+ const int64_t hc_mix_dim = (2 + hc_mult) * hc_mult;
+ const int64_t n_idx_embd = hparams.indexer_head_size;
+ const int64_t n_idx_head = hparams.indexer_n_head;
+
+ tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+ output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+ output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0);
+
+ // all Engram tables in one lazily read tensor, like the qwen4exp n-gram table
+ if (hparams.ple_n_heads > 0) {
+ int64_t n_rows = 0;
+ for (size_t c = 0; c < (size_t) hparams.ple_n_modules*hparams.ple_n_heads; ++c) {
+ n_rows = std::max(n_rows, (int64_t) hparams.ple_head_offsets[c] + hparams.ple_head_vocab_sizes[c]);
+ }
+ const std::string name = tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "weight").str();
+ if (const auto * w = ml.get_weight(name.c_str())) {
+ if (w->tensor->ne[1] < n_rows) {
+ throw std::runtime_error(format("%s has %" PRId64 " rows, too few for the Engram heads (%" PRId64 ")",
+ name.c_str(), w->tensor->ne[1], n_rows));
+ }
+ n_rows = w->tensor->ne[1];
+ }
+ per_layer_tok_embd = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "weight"), {(int64_t) hparams.ple_head_dim, n_rows}, TENSOR_READ_LAZY);
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ auto & layer = layers[i];
+
+ layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
+ layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), {n_head}, 0);
+ layer.wq_a = create_tensor(tn(LLM_TENSOR_ATTN_Q_A, "weight", i), {n_embd, q_lora_rank}, 0);
+ layer.attn_q_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, "weight", i), {q_lora_rank}, 0);
+ layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head * n_embd_head}, 0);
+ layer.wkv = create_tensor(tn(LLM_TENSOR_ATTN_KV, "weight", i), {n_embd, n_embd_head}, 0);
+ layer.attn_kv_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_NORM, "weight", i), {n_embd_head}, 0);
+ layer.wo_a = create_tensor(tn(LLM_TENSOR_ATTN_OUT_A, "weight", i), {n_head * n_embd_head / o_groups, o_lora_rank, o_groups}, TENSOR_ALLOW_RESHAPE);
+ layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT_B, "weight", i), {o_groups * o_lora_rank, n_embd}, 0);
+
+ layer.hc_attn_fn = create_tensor(tn(LLM_TENSOR_HC_ATTN_FN, "weight", i), {hc_dim, hc_mix_dim}, 0);
+ layer.hc_attn_base = create_tensor(tn(LLM_TENSOR_HC_ATTN_BASE, "weight", i), {hc_mix_dim}, 0);
+ layer.hc_attn_scale = create_tensor(tn(LLM_TENSOR_HC_ATTN_SCALE, "weight", i), {3}, 0);
+ layer.hc_ffn_fn = create_tensor(tn(LLM_TENSOR_HC_FFN_FN, "weight", i), {hc_dim, hc_mix_dim}, 0);
+ layer.hc_ffn_base = create_tensor(tn(LLM_TENSOR_HC_FFN_BASE, "weight", i), {hc_mix_dim}, 0);
+ layer.hc_ffn_scale = create_tensor(tn(LLM_TENSOR_HC_FFN_SCALE, "weight", i), {3}, 0);
+
+ if (hparams.dsv41_kv_src[i] == i) {
+ layer.attn_comp_wkv = create_tensor(tn(LLM_TENSOR_ATTN_COMPRESSOR_WKV, "weight", i), {n_embd, n_embd_head}, 0);
+ layer.attn_comp_norm = create_tensor(tn(LLM_TENSOR_ATTN_COMPRESSOR_NORM, "weight", i), {n_embd_head}, 0);
+ if (hparams.dsv4_compress_ratios[i] > 1) {
+ layer.attn_comp_wgate = create_tensor(tn(LLM_TENSOR_ATTN_COMPRESSOR_WGATE, "weight", i), {n_embd, n_embd_head}, 0);
+ }
+ layer.indexer_attn_k = create_tensor(tn(LLM_TENSOR_INDEXER_ATTN_K, "weight", i), {n_embd_head, n_idx_embd}, 0);
+ layer.indexer_k_norm = create_tensor(tn(LLM_TENSOR_INDEXER_K_NORM, "weight", i), {n_idx_embd}, 0);
+ }
+ if (hparams.dsv41_topk_src[i] == i) {
+ layer.indexer_attn_q_b = create_tensor(tn(LLM_TENSOR_INDEXER_ATTN_Q_B, "weight", i), {q_lora_rank, n_idx_head * n_idx_embd}, 0);
+ layer.indexer_proj = create_tensor(tn(LLM_TENSOR_INDEXER_PROJ, "weight", i), {n_embd, n_idx_head}, 0);
+ }
+
+ if (engram_index(i) >= 0) {
+ layer.engram_kv = create_tensor(tn(LLM_TENSOR_ENGRAM_KV, "weight", i), {(int64_t) hparams.ple_n_heads * hparams.ple_head_dim, n_embd * (hc_mult + 1)}, 0);
+ layer.engram_gate = create_tensor(tn(LLM_TENSOR_ENGRAM_GATE, "weight", i), {n_embd, hc_mult}, 0);
+ }
+
+ layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
+ layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0);
+ layer.ffn_exp_probs_b_vl = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B_VL, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED);
+ layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
+
+ layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0);
+ layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);
+ layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0);
+
+ layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0);
+ layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_exp * n_expert_shared, n_embd}, 0);
+ layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, 0);
+ }
+}
+
+std::unique_ptr<llm_graph_context> llama_model_deepseek41::build_arch_graph(const llm_graph_params & params) const {
+ if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) {
+ throw std::runtime_error("DeepSeek-V4.1 has no MTP graph: its DSpark draft is a separate DFLASH GGUF "
+ "(convert_hf_to_gguf.py --dspark), used with --spec-type draft-dspark");
+ }
+ return std::make_unique<graph>(*this, params);
+}
+
+// Engram n-gram hash (engram.py NgramHashState.forward)
+int llama_model_deepseek41::engram_index(int il) const {
+ if (hparams.ple_n_heads == 0 || !hparams.is_ple(il)) {
+ return -1;
+ }
+ int e = 0;
+ for (int j = 0; j < il; ++j) {
+ e += hparams.is_ple(j);
+ }
+ return e;
+}
+
+void llama_model_deepseek41::engram_hash(int e, const llama_ubatch & ubatch, const std::vector<llama_token> & prev, std::vector<int32_t> & out) const {
+ const int64_t n_gram = hparams.ple_ngram_size;
+ const int64_t n_prev = n_gram - 1;
+ const int64_t n_head = hparams.ple_heads_per_ngram;
+ const int64_t n_cols = hparams.ple_n_heads;
+
+ const uint64_t * mult = hparams.ple_layer_multipliers.data() + (size_t) e*n_gram;
+ const uint32_t * primes = hparams.ple_head_vocab_sizes.data() + (size_t) e*n_cols;
+ const int64_t pad = engram_token_map[engram_pad_id];
+ const uint32_t * offs = hparams.ple_head_offsets.data() + (size_t) e*n_cols;
+
+ const auto cid = [&](llama_token t) -> int64_t {
+ return t < 0 || t >= (llama_token) engram_token_map.size() ? DSV41_ENGRAM_DEAD : engram_token_map[t];
+ };
+
+ out.resize(ubatch.n_tokens*n_cols);
+ std::vector<uint64_t> toks(n_gram);
+ for (int64_t i = 0; i < ubatch.n_tokens; ++i) {
+ // shift 0 is the token itself; once a shift is blocked (sequence start or dead token) every longer one is too
+ const bool is_embd = !ubatch.token || (ubatch.is_mixed() && ubatch.type[i]);
+ int64_t c = is_embd ? DSV41_ENGRAM_DEAD : cid(ubatch.token[i]);
+ bool blocked = c == DSV41_ENGRAM_DEAD;
+ toks[0] = blocked ? pad : c;
+ for (int64_t s = 1; s < n_gram; ++s) {
+ c = ubatch.pos[i] - s < 0 ? DSV41_ENGRAM_DEAD : cid(prev[i*n_prev + (n_prev - s)]);
+ blocked = blocked || c == DSV41_ENGRAM_DEAD;
+ toks[s] = blocked ? pad : c;
+ }
+
+ uint64_t rolling = toks[0]*mult[0];
+ for (int64_t o = 1; o < n_gram; ++o) {
+ rolling ^= toks[o]*mult[o];
+ for (int64_t h = 0; h < n_head; ++h) {
+ const int64_t col = (o - 1)*n_head + h;
+ out[i*n_cols + col] = (int32_t) (rolling % primes[col] + offs[col]);
+ }
+ }
+ }
+}
+
+// cand_pin: +1e30 (the reference uses +inf) on the block holding each query's newest compressed row, so a partly
+// filled block always survives; a finite pin keeps fully masked blocks at -inf instead of NaN
+class llm_graph_input_dsv41 : public llm_graph_input_i {
+public:
+ llm_graph_input_dsv41(const llama_model_deepseek41 & model, const llama_kv_cache_dsv4_context * mctx) :
+ model(model), mctx(mctx) {}
+
+ void set_input(const llama_ubatch * ubatch) override;
+ bool can_reuse(const llm_graph_params & params) override;
+
+ std::vector<ggml_tensor *> engram_ids;
+ ggml_tensor * engram_keep = nullptr; // 0 for embedding (image) tokens; only in mixed ubatches
+ ggml_tensor * cand_pin = nullptr;
+ bool cand_csa = false;
+
+ const llama_model_deepseek41 & model;
+ const llama_kv_cache_dsv4_context * mctx;
+
+private:
+ const llama_kv_cache_dsv4_context::comp_plan & cand_plan(const llama_ubatch & ubatch) const {
+ return cand_csa ? mctx->get_csa_plan(ubatch) : mctx->get_hca_plan(ubatch);
+ }
+};
+
+void llm_graph_input_dsv41::set_input(const llama_ubatch * ubatch) {
+ if (!engram_ids.empty()) {
+ // a token shared by several sequences has the same history in each, so its first sequence resolves it
+ std::vector<llama_token> prev;
+ mctx->get_raw()->get_prev_tokens(*ubatch, model.hparams.ple_ngram_size - 1, prev);
+ // hash all Engram layers first so a lazily read table gets one batched readahead before the first gather
+ std::vector<std::vector<int32_t>> ids(engram_ids.size());
+ std::vector<int32_t> all;
+ for (size_t e = 0; e < engram_ids.size(); ++e) {
+ model.engram_hash((int) e, *ubatch, prev, ids[e]);
+ all.insert(all.end(), ids[e].begin(), ids[e].end());
+ }
+ if (model.can_prefetch.count(model.per_layer_tok_embd)) {
+ llama_prefetch_rows(model.per_layer_tok_embd, all.data(), all.size());
+ }
+ for (size_t e = 0; e < engram_ids.size(); ++e) {
+ if (engram_ids[e]->buffer) {
+ ggml_backend_tensor_set(engram_ids[e], ids[e].data(), 0, ids[e].size()*sizeof(int32_t));
+ }
+ }
+ }
+
+ // embedding (image) tokens are hashed DEAD and get no Engram contribution (gate 0)
+ if (engram_keep && engram_keep->buffer) {
+ std::vector<float> keep(ubatch->n_tokens);
+ for (uint32_t i = 0; i < ubatch->n_tokens; ++i) {
+ const bool is_embd = !ubatch->token || (ubatch->is_mixed() && ubatch->type[i]);
+ keep[i] = is_embd ? 0.0f : 1.0f;
+ }
+ ggml_backend_tensor_set(engram_keep, keep.data(), 0, keep.size()*sizeof(float));
+ }
+
+ if (cand_pin && cand_pin->buffer) {
+ const auto & plan = cand_plan(*ubatch);
+ const int64_t n_blk = cand_pin->ne[0];
+ const int64_t bs = model.hparams.dsv41_cand_block_size;
+ GGML_ASSERT((int64_t) plan.n_visible.size() == (int64_t) ubatch->n_tokens);
+ std::vector<float> pin(n_blk*ubatch->n_tokens, 0.0f);
+ for (int64_t i = 0; i < (int64_t) ubatch->n_tokens; ++i) {
+ const int64_t nv = plan.n_visible[i];
+ if (nv > 0) {
+ GGML_ASSERT((nv - 1)/bs < n_blk);
+ pin[i*n_blk + (nv - 1)/bs] = 1e30f;
+ }
+ }
+ ggml_backend_tensor_set(cand_pin, pin.data(), 0, pin.size()*sizeof(float));
+ }
+}
+
+bool llm_graph_input_dsv41::can_reuse(const llm_graph_params & params) {
+ mctx = static_cast<const llama_kv_cache_dsv4_context *>(params.mctx);
+
+ bool res = true;
+ for (auto * t : engram_ids) {
+ res &= t->ne[0] == (int64_t) model.hparams.ple_n_heads*params.ubatch.n_tokens;
+ }
+ res &= (engram_keep != nullptr) == (!engram_ids.empty() && params.ubatch.embd != nullptr);
+ if (engram_keep) {
+ res &= engram_keep->ne[2] == (int64_t) params.ubatch.n_tokens;
+ }
+ if (cand_pin) {
+ const auto & plan = cand_plan(params.ubatch);
+ const int64_t bs = model.hparams.dsv41_cand_block_size;
+ res &= plan.n_kv > 0 && cand_pin->ne[0] == (plan.n_kv + bs - 1)/bs;
+ res &= cand_pin->ne[1]*cand_pin->ne[3] == (int64_t) params.ubatch.n_tokens;
+ }
+ return res;
+}
+
+ggml_tensor * llama_model_deepseek41::graph::build_engram(
+ const llama_model_deepseek41 & model,
+ ggml_tensor * x,
+ ggml_tensor * rows,
+ ggml_tensor * keep,
+ int il) const {
+ const auto & layer = model.layers[il];
+ const int64_t hc = hparams.dsv4_hc_mult;
+ const int64_t nt = x->ne[2];
+
+ GGML_ASSERT(rows->ne[0] == (int64_t) (hparams.ple_head_dim*hparams.ple_n_heads) && rows->ne[1] == nt);
+
+ ggml_tensor * kv = build_lora_mm(layer.engram_kv, rows);
+ cb(kv, "engram_kv", il);
+
+ ggml_tensor * key = ggml_cont(ctx0, ggml_view_3d(ctx0, kv, n_embd, hc, nt, ggml_row_size(kv->type, n_embd), kv->nb[1], 0));
+ ggml_tensor * value = ggml_cont(ctx0, ggml_view_3d(ctx0, kv, n_embd, 1, nt, ggml_row_size(kv->type, n_embd), kv->nb[1],
+ ggml_row_size(kv->type, hc*n_embd)));
+
+ // quantized copy has no binary-op kernel
+ ggml_tensor * egate = ggml_is_quantized(layer.engram_gate->type) ? ggml_cast(ctx0, layer.engram_gate, GGML_TYPE_F32) : layer.engram_gate;
+ ggml_tensor * dot = ggml_mul(ctx0, ggml_mul(ctx0, ggml_rms_norm(ctx0, x, norm_rms_eps), ggml_rms_norm(ctx0, key, norm_rms_eps)), egate);
+ dot = ggml_scale(ctx0, ggml_sum_rows(ctx0, dot), 1.0f/sqrtf((float) n_embd));
+ cb(dot, "engram_dot", il);
+
+ // sigmoid(copysign(sqrt(max(|dot|, 1e-6)), dot)); sign(0) = +1 as in copysign
+ ggml_tensor * mag = ggml_sqrt(ctx0, ggml_clamp(ctx0, ggml_abs(ctx0, dot), 1e-6f, INFINITY));
+ ggml_tensor * sign = ggml_scale_bias(ctx0, ggml_step(ctx0, ggml_neg(ctx0, dot)), -2.0f, 1.0f);
+ ggml_tensor * gate = ggml_sigmoid(ctx0, ggml_mul(ctx0, mag, sign));
+ if (keep) {
+ gate = ggml_mul(ctx0, gate, keep);
+ }
+ cb(gate, "engram_gate", il);
+
+ return ggml_add(ctx0, x, ggml_mul(ctx0, ggml_repeat_4d(ctx0, value, n_embd, hc, nt, 1), gate));
+}
+
+ggml_tensor * llama_model_deepseek41::graph::build_attention41(
+ const llama_model_deepseek41 & model,
+ llm_graph_input_dsv4 * inp_dsv4,
+ ggml_tensor * cur,
+ ggml_tensor * inp_pos,
+ ggml_tensor * cand_pin,
+ int il) {
+ const auto & layer = model.layers[il];
+ llm_graph_input_dsv4_raw * inp_attn = inp_dsv4->get_raw();
+
+ const int64_t n_embd_head = hparams.n_embd_head_k();
+ const int64_t n_embd_head_rope = hparams.n_rot();
+ const int64_t n_embd_head_nope = n_embd_head - n_embd_head_rope;
+ const int64_t n_groups = hparams.dsv4_o_group_count;
+ const int64_t o_lora_rank = hparams.dsv4_o_lora_rank;
+ const int64_t n_idx_embd = hparams.indexer_head_size;
+ const int64_t n_idx_head = hparams.indexer_n_head;
+ const int64_t nt = cur->ne[1];
+ const float kq_scale = 1.0f/sqrtf(float(n_embd_head));
+ const int64_t ratio = hparams.dsv4_compress_ratios[il];
+
+ const bool comp_rope = ratio != 0;
+ const float freq_base_l = comp_rope ? hparams.dsv4_compress_rope_base : freq_base;
+ const float freq_scale_l = comp_rope ? freq_scale : 1.0f;
+ const float ext_factor_l = comp_rope ? ext_factor : 0.0f;
+ const float attn_factor_l = dsv4_rope_attn_factor(freq_scale_l, ext_factor_l);
+ const float beta_fast_l = comp_rope ? beta_fast : 0.0f;
+ const float beta_slow_l = comp_rope ? beta_slow : 0.0f;
+ const int32_t n_ctx_orig_l = comp_rope ? n_ctx_orig : 0;
+
+ const auto rope = [&](ggml_tensor * t, ggml_tensor * pos, int64_t n_nope) {
+ t = ggml_rope_ext(ctx0, t, pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l,
+ freq_base_l, freq_scale_l, ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l);
+ return ggml_rope_set_offset(t, n_nope);
+ };
+
+ ggml_tensor * qr = build_norm(build_lora_mm(layer.wq_a, cur), layer.attn_q_a_norm, nullptr, LLM_NORM_RMS, il);
+ cb(qr, "qr_norm", il);
+
+ ggml_tensor * q = ggml_reshape_3d(ctx0, build_lora_mm(layer.wq_b, qr), n_embd_head, n_head, nt);
+ q = rope(q, inp_pos, n_embd_head_nope);
+ cb(q, "q", il);
+
+ ggml_tensor * kv = build_norm(build_lora_mm(layer.wkv, cur), layer.attn_kv_norm, nullptr, LLM_NORM_RMS, il);
+ kv = rope(ggml_reshape_3d(ctx0, kv, n_embd_head, 1, nt), inp_pos, n_embd_head_nope);
+ cb(kv, "kv", il);
+
+ const auto & ci = ratio == 2 ? inp_dsv4->get_csa() : inp_dsv4->get_hca();
+ const auto * comp_ctx = ratio == 2 ? inp_dsv4->mctx->get_csa() : inp_dsv4->mctx->get_hca();
+ const auto * comp_state = ratio == 2 ? inp_dsv4->mctx->get_csa_state() : inp_dsv4->mctx->get_hca_state();
+
+ const int32_t rd = inp_dsv4->mctx->get_comp_read_layer(il);
+
+ if (ratio != 0) {
+ GGML_ASSERT(hparams.dsv41_kv_src[il] >= 0 && (int64_t) comp_state->get_ratio() == ratio);
+
+ if (hparams.dsv41_kv_src[il] == il && ci.state_pos) {
+ ggml_tensor * state_kv = build_lora_mm(layer.attn_comp_wkv, cur);
+ ggml_tensor * state_score = ratio > 1 ? build_lora_mm(layer.attn_comp_wgate, cur) : ggml_fill(ctx0, state_kv, 0.0f);
+
+ ggml_tensor * restored_kv = comp_state->get_kv_all(ctx0, il);
+ ggml_tensor * restored_score = comp_state->get_score_all(ctx0, il);
+ if (ci.state_restore_src_idxs && ci.state_restore_dst_idxs) {
+ restored_kv = comp_state->cpy_kv (ctx0, ggml_get_rows(ctx0, restored_kv, ci.state_restore_src_idxs), ci.state_restore_dst_idxs, il);
+ restored_score = comp_state->cpy_score(ctx0, ggml_get_rows(ctx0, restored_score, ci.state_restore_src_idxs), ci.state_restore_dst_idxs, il);
+ }
+
+ if (ci.state_write_idxs) {
+ const int64_t n_blocks = ci.state_write_pos->ne[0];
+ const int64_t n_state = comp_state->get_n_rows();
+ GGML_ASSERT(ci.state_read_idxs->ne[0] == ratio*n_blocks);
+
+ ggml_tensor * src_kv = ggml_concat(ctx0, ggml_view_2d(ctx0, restored_kv, restored_kv->ne[0], n_state, restored_kv->nb[1], 0), state_kv, 1);
+ ggml_tensor * src_score = ggml_concat(ctx0, ggml_view_2d(ctx0, restored_score, restored_score->ne[0], n_state, restored_score->nb[1], 0), state_score, 1);
+
+ ggml_tensor * v = ggml_get_rows(ctx0, src_kv, ci.state_read_idxs);
+ ggml_tensor * s = ggml_get_rows(ctx0, src_score, ci.state_read_idxs);
+ v = ggml_cont(ctx0, ggml_permute(ctx0, ggml_reshape_3d(ctx0, v, n_embd_head, ratio, n_blocks), 1, 0, 2, 3));
+ s = ggml_cont(ctx0, ggml_permute(ctx0, ggml_reshape_3d(ctx0, s, n_embd_head, ratio, n_blocks), 1, 0, 2, 3));
+
+ ggml_tensor * comp = ggml_sum_rows(ctx0, ggml_mul(ctx0, v, ggml_soft_max(ctx0, s)));
+ comp = ggml_cont(ctx0, ggml_permute(ctx0, comp, 1, 0, 2, 3));
+ comp = build_norm(comp, layer.attn_comp_norm, nullptr, LLM_NORM_RMS, il);
+ cb(comp, "comp_latent", il);
+
+ // index K from the RoPE-free latent; a latent stands for the first token of its group
+ ggml_tensor * ik = build_norm(build_lora_mm(layer.indexer_attn_k, comp), layer.indexer_k_norm, nullptr, LLM_NORM_RMS, il);
+ ik = rope(ik, ci.state_write_pos, n_idx_embd - n_embd_head_rope);
+ comp = rope(comp, ci.state_write_pos, n_embd_head_nope);
+
+ ggml_tensor * row = ggml_concat(ctx0, comp, ik, 0);
+ ggml_build_forward_expand(gf, comp_ctx->cpy_k(ctx0, row, ci.state_write_idxs, il));
+ comp_row_of[il] = row;
+ }
+
+ if (ci.state_snapshot_src_idxs && ci.state_snapshot_dst_idxs) {
+ ggml_tensor * snap_kv = ggml_concat(ctx0, restored_kv, state_kv, 1);
+ ggml_tensor * snap_score = ggml_concat(ctx0, restored_score, state_score, 1);
+ ggml_build_forward_expand(gf, comp_state->cpy_kv (ctx0, ggml_get_rows(ctx0, snap_kv, ci.state_snapshot_src_idxs), ci.state_snapshot_dst_idxs, il));
+ ggml_build_forward_expand(gf, comp_state->cpy_score(ctx0, ggml_get_rows(ctx0, snap_score, ci.state_snapshot_src_idxs), ci.state_snapshot_dst_idxs, il));
+ }
+ ggml_build_forward_expand(gf, comp_state->cpy_kv (ctx0, ggml_get_rows(ctx0, state_kv, ci.state_persist_src_idxs), ci.state_persist_dst_idxs, il));
+ ggml_build_forward_expand(gf, comp_state->cpy_score(ctx0, ggml_get_rows(ctx0, state_score, ci.state_persist_src_idxs), ci.state_persist_dst_idxs, il));
+ }
+
+ if (rd == il && hparams.dsv41_kv_src[il] != il && comp_row_of[hparams.dsv41_kv_src[il]]) {
+ // first consumer on this device: append the source's new rows to the local mirror
+ ggml_build_forward_expand(gf, comp_ctx->cpy_k(ctx0, comp_row_of[hparams.dsv41_kv_src[il]], ci.state_write_idxs, il));
+ }
+ }
+
+ ggml_tensor * out = nullptr;
+ if (ratio == 0 || !ci.kq_mask) {
+ // no compressed row visible yet: window only
+ out = build_raw_attention(inp_attn, q, kv, layer.attn_sinks, kq_scale, il);
+ } else {
+ const int64_t n_comp = ci.kq_mask->ne[0];
+
+ ggml_tensor * comp_all = comp_ctx->get_k(ctx0, rd);
+ GGML_ASSERT(comp_all->ne[0] == n_embd_head + n_idx_embd && n_comp <= comp_all->ne[2]);
+ const int64_t n_stream = comp_all->ne[3];
+
+ ggml_tensor * top_k = topk_of[hparams.dsv41_topk_src[il]];
+ if (hparams.dsv41_topk_src[il] == il) {
+ ggml_tensor * ik = ggml_view_4d(ctx0, comp_all, n_idx_embd, 1, n_comp, n_stream,
+ comp_all->nb[1], comp_all->nb[2], comp_all->nb[3], ggml_row_size(comp_all->type, n_embd_head));
+ // the fused indexer reads the strided view in place, the unfused path needs it contiguous; quantized
+ // caches are dequantized straight from the strided view (CUDA has no same-type copy of quantized views)
+ const bool fused_idx = sparse && cparams.fused_lid && ci.kq_mask->type == GGML_TYPE_F16;
+ if (ggml_is_quantized(ik->type)) {
+ ik = ggml_cast(ctx0, ik, GGML_TYPE_F32);
+ } else if (!fused_idx) {
+ ik = ggml_cont(ctx0, ik);
+ }
+
+ ggml_tensor * iq = ggml_reshape_3d(ctx0, build_lora_mm(layer.indexer_attn_q_b, qr), n_idx_embd, n_idx_head, nt);
+ iq = rope(iq, inp_pos, n_idx_embd - n_embd_head_rope);
+ iq = ggml_view_4d(ctx0, iq, iq->ne[0], iq->ne[1], nt/n_stream, n_stream, iq->nb[1], iq->nb[2], iq->nb[2]*(nt/n_stream), 0);
+
+ ggml_tensor * iw = ggml_scale(ctx0, build_lora_mm(layer.indexer_proj, cur), 1.0f/sqrtf(float(n_idx_embd*n_idx_head)));
+ iw = ggml_view_4d(ctx0, iw, iw->ne[0], nt/n_stream, 1, n_stream, iw->nb[1], iw->nb[1]*(nt/n_stream), iw->nb[1]*(nt/n_stream), 0);
+
+ // chunk queries so the [n_ih, tokens, n_comp] scores stay near 2^26 elements (at most 64 chunks); the fused
+ // indexer has no such intermediate and runs as one chunk
+ const int64_t n_tok = nt/n_stream;
+ GGML_ASSERT(ci.kq_mask->ne[1] == n_tok);
+ const int64_t chunk = fused_idx ? n_tok :
+ std::max({ (n_tok + 63)/64, (INT64_C(1) << 26)/(n_idx_head*n_comp*n_stream), INT64_C(1) });
+
+ const int32_t cand_src = hparams.dsv41_cand_src;
+ const bool pick_blk = cand_src == il && cand_pin; // level one: pick the candidate blocks
+ const bool use_blk = cand_src >= 0 && cand_src < il && cand_sel; // level two: score inside them
+ const int64_t bs = hparams.dsv41_cand_block_size;
+ const int64_t n_blk = pick_blk || use_blk ? (n_comp + bs - 1)/bs : 0;
+ GGML_ASSERT(!pick_blk || cand_pin->ne[0] == n_blk);
+ GGML_ASSERT(!use_blk || cand_n_comp == n_comp);
+
+ ggml_tensor * ik_p = fused_idx ? nullptr : ggml_permute(ctx0, ik, 0, 2, 1, 3);
+ std::vector<ggml_tensor *> parts_top_k;
+ std::vector<ggml_tensor *> parts_sel;
+ for (int64_t t0 = 0; t0 < n_tok; t0 += chunk) {
+ const int64_t nc = std::min(chunk, n_tok - t0);
+ const auto rows = [&](ggml_tensor * t) {
+ return ggml_view_4d(ctx0, t, t->ne[0], nc, 1, n_stream, t->nb[1], t->nb[2], t->nb[3], t0*t->nb[1]);
+ };
+ ggml_tensor * score = nullptr;
+ if (fused_idx) {
+ score = ggml_lightning_indexer(ctx0, iq, ik, iw, ci.kq_mask);
+ res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, score, il});
+ } else {
+ ggml_tensor * iqc = ggml_view_4d(ctx0, iq, iq->ne[0], iq->ne[1], nc, n_stream, iq->nb[1], iq->nb[2], iq->nb[3], t0*iq->nb[2]);
+
+ ggml_tensor * kq = ggml_mul_mat(ctx0, ik_p, ggml_permute(ctx0, iqc, 0, 2, 1, 3));
+ if (!ggml_is_quantized(ik->type)) {
+ ggml_prec_set_src(kq, GGML_PREC_F32, 1);
+ }
+ kq = ggml_cont(ctx0, ggml_permute(ctx0, kq, 2, 1, 0, 3));
+ score = ggml_sum_rows(ctx0, ggml_mul(ctx0, ggml_relu(ctx0, kq), rows(iw)));
+ ggml_tensor * mask = rows(ci.kq_mask);
+ score = ggml_add(ctx0, ggml_cont(ctx0, ggml_permute(ctx0, score, 2, 1, 0, 3)),
+ mask->type == GGML_TYPE_F32 ? mask : ggml_cast(ctx0, mask, GGML_TYPE_F32));
+ }
+ cb(score, "idx_score", il);
+
+ if (pick_blk) {
+ // best position per block, pin the newest block, keep the top blocks
+ ggml_tensor * s2 = ggml_reshape_2d(ctx0, score, n_comp, nc*n_stream);
+ if (n_blk*bs != n_comp) {
+ s2 = ggml_concat(ctx0, s2, ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_blk*bs - n_comp, nc*n_stream), -INFINITY), 0);
+ }
+ ggml_tensor * bmax = ggml_pool_1d(ctx0, s2, GGML_OP_POOL_MAX, bs, bs, 0);
+ bmax = ggml_add(ctx0, ggml_reshape_4d(ctx0, bmax, n_blk, nc, 1, n_stream), rows(cand_pin));
+ cb(bmax, "cand_block_score", il);
+ parts_sel.push_back(ggml_top_k(ctx0, bmax, std::min<int64_t>(hparams.dsv41_cand_topk_blocks, n_blk)));
+ } else if (use_blk) {
+ // own scores, restricted to the source's candidate blocks: 0 inside them, -inf elsewhere
+ ggml_tensor * bsel = ggml_cont(ctx0, rows(cand_sel));
+ const int64_t k_b = bsel->ne[0];
+ ggml_tensor * bm = ggml_fill(ctx0, ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, 1, n_blk, nc*n_stream, 1), -INFINITY);
+ ggml_tensor * zeros = ggml_fill(ctx0, ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, 1, k_b, nc*n_stream, 1), 0.0f);
+ bm = ggml_set_rows(ctx0, bm, zeros, ggml_reshape_3d(ctx0, bsel, k_b, nc*n_stream, 1));
+ bm = ggml_reshape_2d(ctx0, ggml_repeat_4d(ctx0, bm, bs, n_blk, nc*n_stream, 1), bs*n_blk, nc*n_stream);
+ bm = ggml_reshape_4d(ctx0, ggml_cont(ctx0, ggml_view_2d(ctx0, bm, n_comp, nc*n_stream, bm->nb[1], 0)), n_comp, nc, 1, n_stream);
+ cb(bm, "cand_mask", il);
+ score = ggml_add(ctx0, score, bm);
+ }
+
+ parts_top_k.push_back(ggml_top_k(ctx0, score, std::min<int64_t>(n_comp, hparams.indexer_top_k)));
+ }
+
+ const auto join = [&](std::vector<ggml_tensor *> v) {
+ while (v.size() > 1) {
+ std::vector<ggml_tensor *> w;
+ for (size_t i = 0; i < v.size(); i += 2) {
+ w.push_back(i + 1 < v.size() ? ggml_concat(ctx0, v[i], v[i + 1], 1) : v[i]);
+ }
+ v.swap(w);
+ }
+ return v[0];
+ };
+ if (pick_blk) {
+ cand_sel = join(parts_sel);
+ cand_n_comp = n_comp;
+ }
+ top_k = ggml_cont(ctx0, join(parts_top_k));
+ cb(top_k, "idx_top_k", il);
+ topk_of[il] = top_k;
+ }
+ GGML_ASSERT(top_k && "DeepSeek-V4.1: top-k source layer produced no top-k");
+
+ // a quantized K cache stores the window Hadamard-rotated (as build_raw_attention)
+ ggml_tensor * k_rot = inp_attn->self_k_rot;
+ if (k_rot) {
+ q = llama_mul_mat_hadamard(ctx0, q, k_rot);
+ kv = llama_mul_mat_hadamard(ctx0, kv, k_rot);
+ }
+ ggml_build_forward_expand(gf, q);
+ ggml_build_forward_expand(gf, kv);
+ ggml_build_forward_expand(gf, inp_attn->mctx->cpy_k(ctx0, kv, inp_attn->get_k_idxs(), il));
+
+ ggml_tensor * raw_k = inp_attn->mctx->get_k(ctx0, il);
+ ggml_tensor * raw_mask = inp_attn->get_kq_mask();
+
+ const int32_t topk_src = hparams.dsv41_topk_src[il];
+ const int64_t k_sel = top_k->ne[0];
+ const int64_t n_raw = raw_k->ne[2];
+
+ // few queries per stream: gather each query's top-k rows and attend over [window | selected], one FA stream
+ // per query, instead of a full n_comp mask per layer. same keys and mask as dense, different key order
+ const bool gather = sparse && cparams.flash_attn && raw_mask->type == GGML_TYPE_F16 && ci.kq_mask->type == GGML_TYPE_F16 &&
+ ((n_stream == 1 && nt <= 32) || nt == n_stream);
+
+ if (gather) {
+ // ggml_top_k returns the rows in no particular order (CUB DeviceTopK can vary between runs) and FA sums the
+ // gathered keys in that order: sort the rows once per top-k source so the result is deterministic
+ ggml_tensor *& sel = gather_sel_of[topk_src];
+ if (!sel) {
+ GGML_ASSERT(n_comp <= (INT64_C(1) << 24)); // row ids are exact in F32
+ ggml_tensor * ids = ggml_reshape_2d(ctx0, top_k, k_sel, nt);
+ ggml_tensor * ord = ggml_argsort(ctx0, ggml_cast(ctx0, ids, GGML_TYPE_F32), GGML_SORT_ORDER_ASC);
+ sel = ggml_reshape_2d(ctx0, ggml_get_rows(ctx0, ggml_reshape_3d(ctx0, ids, 1, k_sel, nt), ord), k_sel, nt);
+ cb(sel, "comp_top_k_sorted", il);
+ }
+
+ // built once per (storage, top-k) pair
+ ggml_tensor *& comp_k = gather_k_of[{rd, topk_src}];
+ if (!comp_k) {
+ if (n_stream == 1) {
+ ggml_tensor * src_rows = ggml_view_3d(ctx0, comp_all, n_embd_head, n_comp, 1, comp_all->nb[2], comp_all->nb[3], 0);
+ comp_k = ggml_get_rows(ctx0, src_rows, ggml_reshape_1d(ctx0, sel, k_sel*nt));
+ } else {
+ ggml_tensor * src_rows = ggml_view_3d(ctx0, comp_all, n_embd_head, n_comp, n_stream, comp_all->nb[2], comp_all->nb[3], 0);
+ comp_k = ggml_get_rows(ctx0, src_rows, sel);
+ }
+ comp_k = ggml_reshape_4d(ctx0, comp_k, n_embd_head, 1, k_sel, nt); // F32 (get_rows dequantizes)
+ if (k_rot) {
+ // quantized K cache: the window is stored rotated, the compressed rows are not
+ comp_k = llama_mul_mat_hadamard(ctx0, comp_k, k_rot);
+ }
+ if (comp_k->type != raw_k->type) {
+ comp_k = ggml_cast(ctx0, comp_k, raw_k->type); // exact: rows are raw_k->type values
+ }
+ cb(comp_k, "comp_k_sel", il);
+ }
+
+ if (n_stream == 1 && nt > 1) {
+ if (ggml_is_quantized(raw_k->type)) {
+ // no REPEAT kernel for quantized types: byte-exact copies by doubling concats
+ ggml_tensor * rep = nullptr;
+ ggml_tensor * pw = raw_k;
+ for (int64_t n = nt; n > 0; n >>= 1) {
+ if (n & 1) {
+ rep = rep ? ggml_concat(ctx0, rep, pw, 3) : pw;
+ }
+ if (n > 1) {
+ pw = ggml_concat(ctx0, pw, pw, 3);
+ }
+ }
+ raw_k = rep;
+ } else {
+ raw_k = ggml_repeat_4d(ctx0, raw_k, n_embd_head, 1, n_raw, nt);
+ }
+ }
+ ggml_tensor * k_all = ggml_concat(ctx0, raw_k, comp_k, 2);
+
+ // mask: window row, then comp mask at the selected rows (-inf where top-k picked an invisible row);
+ // one per top-k source, shared by its layers
+ ggml_tensor *& kq_mask = gather_mask_of[topk_src];
+ if (!kq_mask) {
+ ggml_tensor * sel_mask = ggml_get_rows(ctx0, ggml_reshape_3d(ctx0, ci.kq_mask, 1, n_comp, nt), sel);
+ sel_mask = ggml_cast(ctx0, ggml_reshape_4d(ctx0, sel_mask, k_sel, 1, 1, nt), GGML_TYPE_F16);
+ kq_mask = ggml_concat(ctx0, ggml_reshape_4d(ctx0, raw_mask, raw_mask->ne[0], 1, 1, nt), sel_mask, 0);
+ cb(kq_mask, "comp_kq_mask_sel", il);
+ }
+
+ out = build_attn_mha(q, k_all, k_all, nullptr, kq_mask, layer.attn_sinks, nullptr, 0, kq_scale, il);
+ } else {
+ // dense attention over [window | all compressed rows], selection applied as a mask
+ ggml_tensor * comp_k = ggml_view_4d(ctx0, comp_all, n_embd_head, 1, n_comp, n_stream,
+ comp_all->nb[1], comp_all->nb[2], comp_all->nb[3], 0);
+ if (k_rot) {
+ // compressed rows are stored unrotated: rotate in F32, then cast to the cache type (dequantized from
+ // the strided view: CUDA has no same-type quantized copy)
+ comp_k = comp_k->type != GGML_TYPE_F32 ? ggml_cast(ctx0, comp_k, GGML_TYPE_F32) : ggml_cont(ctx0, comp_k);
+ comp_k = llama_mul_mat_hadamard(ctx0, comp_k, k_rot);
+ if (comp_k->type != raw_k->type) {
+ comp_k = ggml_cast(ctx0, comp_k, raw_k->type);
+ }
+ } else if (ggml_is_quantized(comp_k->type)) {
+ // quantized cache without rotation (LLAMA_ATTN_ROT_DISABLE=1): via F32 for the same reason
+ comp_k = ggml_cast(ctx0, ggml_cast(ctx0, comp_k, GGML_TYPE_F32), raw_k->type);
+ } else {
+ comp_k = comp_k->type != raw_k->type ? ggml_cast(ctx0, comp_k, raw_k->type) : ggml_cont(ctx0, comp_k);
+ }
+ cb(comp_k, "comp_k", il);
+ ggml_tensor * k_all = ggml_concat(ctx0, raw_k, comp_k, 2);
+
+ ggml_tensor * sel_mask = build_top_k_mask(ci.kq_mask, top_k, "comp_top_k_mask", il);
+ ggml_tensor * kq_mask = ggml_concat(ctx0, raw_mask, sel_mask, 0);
+ cb(kq_mask, "comp_kq_mask", il);
+
+ // at most window + top-k visible columns per query: lets CUDA FA gather only those
+ const int64_t n_kv_max = sparse ? std::min<int64_t>(raw_mask->ne[0], hparams.n_swa) + k_sel : 0;
+ out = build_attn_mha(q, k_all, k_all, nullptr, kq_mask, layer.attn_sinks, nullptr, n_kv_max, kq_scale, il);
+ }
+ if (k_rot) {
+ out = llama_mul_mat_hadamard(ctx0, out, k_rot);
+ }
+ cb(out, "attn_comp", il);
+ }
+
+ out = ggml_reshape_3d(ctx0, out, n_embd_head, n_head, nt);
+ out = ggml_rope_ext_back(ctx0, out, inp_pos, nullptr, n_embd_head_rope, rope_type, n_ctx_orig_l,
+ freq_base_l, freq_scale_l, ext_factor_l, attn_factor_l, beta_fast_l, beta_slow_l);
+ out = ggml_rope_set_offset(out, n_embd_head_nope);
+ cb(out, "attn_derope", il);
+
+ out = ggml_permute(ctx0, ggml_reshape_3d(ctx0, out, n_head/n_groups*n_embd_head, n_groups, nt), 0, 2, 1, 3);
+ ggml_tensor * oa = ggml_mul_mat(ctx0, layer.wo_a, out);
+ cb(oa, "attn_wo_a", il);
+ oa = ggml_cont_2d(ctx0, ggml_permute(ctx0, oa, 0, 2, 1, 3), o_lora_rank*n_groups, nt);
+
+ out = build_lora_mm(layer.wo_b, oa);
+ cb(out, "attn_out", il);
+
+ return out;
+}
+
+llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_graph_params & params) :
+ llama_model_deepseek4::graph(params) {
+ const auto & model = static_cast<const llama_model_deepseek41 &>(model_base);
+
+ static const bool sparse_env = dsv41_env_on("LLAMA_DSV41_SPARSE");
+ sparse = sparse_env;
+
+ topk_of.assign(n_layer, nullptr);
+ comp_row_of.assign(n_layer, nullptr);
+
+ ggml_tensor * cur;
+
+ ggml_tensor * inp = build_inp_embd(model.tok_embd);
+ ggml_tensor * inp_pos = build_inp_pos();
+ ggml_tensor * inp_out_ids = build_inp_out_ids();
+ llm_graph_input_dsv4 * inp_dsv4 = build_inp_dsv4();
+ ggml_build_forward_expand(gf, inp_dsv4->get_raw()->self_kq_mask);
+
+ auto inp41 = std::make_unique<llm_graph_input_dsv41>(model, inp_dsv4->mctx);
+ // embedding input has no token ids: it skips Engram, like the reference's masked (image) positions
+ // the gathers go right after the token embeddings, so they share the first graph split (the table stays on the host)
+ std::vector<ggml_tensor *> engram_rows;
+ for (int il = 0; ubatch.token && hparams.ple_n_heads > 0 && il < n_layer; ++il) {
+ if (!hparams.is_ple(il)) {
+ continue;
+ }
+ ggml_tensor * ids = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, (int64_t) hparams.ple_n_heads*n_tokens);
+ ggml_set_name(ids, ("engram_ids-" + std::to_string(il)).c_str());
+ ggml_set_input(ids);
+ inp41->engram_ids.push_back(ids);
+
+ ggml_tensor * rows = ggml_get_rows(ctx0, model.per_layer_tok_embd, ids);
+ rows = ggml_reshape_2d(ctx0, rows, hparams.ple_head_dim*hparams.ple_n_heads, n_tokens);
+ cb(rows, "engram_rows", il);
+ ggml_build_forward_expand(gf, rows);
+ engram_rows.push_back(rows);
+ }
+ ggml_tensor * engram_keep = nullptr;
+ if (hparams.ple_n_heads > 0 && ubatch.token && ubatch.embd) {
+ engram_keep = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 1, 1, n_tokens);
+ ggml_set_input(engram_keep);
+ ggml_set_name(engram_keep, "engram_keep");
+ inp41->engram_keep = engram_keep;
+ }
+ ggml_tensor * cand_pin = nullptr;
+ if (hparams.dsv41_cand_src >= 0) {
+ inp41->cand_csa = hparams.dsv4_compress_ratios[hparams.dsv41_cand_src] == 2;
+ const auto & ci = inp41->cand_csa ? inp_dsv4->get_csa() : inp_dsv4->get_hca();
+ if (ci.kq_mask) {
+ const int64_t bs = hparams.dsv41_cand_block_size;
+ cand_pin = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, (ci.kq_mask->ne[0] + bs - 1)/bs, ci.kq_mask->ne[1], 1, ci.kq_mask->ne[3]);
+ ggml_set_input(cand_pin);
+ ggml_set_name(cand_pin, "cand_pin");
+ inp41->cand_pin = cand_pin;
+ }
+ }
+ res->add_input(std::move(inp41));
+
+ const int64_t hc = hparams.dsv4_hc_mult;
+ ggml_tensor * inpL = ggml_repeat_4d(ctx0, ggml_reshape_3d(ctx0, inp, n_embd, 1, n_tokens), n_embd, hc, n_tokens, 1);
+ cb(inpL, "hc_init", -1);
+
+ // single-pass mHC: the pre-mix used to collapse the stream lags one sublayer
+ ggml_tensor * pre_prev = nullptr;
+
+ for (int il = 0; il < n_layer; ++il) {
+ const auto & layer = model.layers[il];
+
+ const int e = model.engram_index(il);
+ if (e >= 0 && !engram_rows.empty()) {
+ inpL = build_engram(model, inpL, engram_rows[e], engram_keep, il);
+ cb(inpL, "engram_out", il);
+ }
+
+ // DSpark target features: mean over the hc copies of the layer input
+ if ((size_t) il < cparams.embeddings_layer_inp.size() && cparams.embeddings_layer_inp[il]) {
+ res->t_layer_inp[il] = build_hc_mean(inpL);
+ cb(res->t_layer_inp[il], "layer_inp", il);
+ ggml_build_forward_expand(gf, res->t_layer_inp[il]);
+ }
+
+ ggml_tensor * residual = inpL;
+ ggml_tensor * post = nullptr;
+ ggml_tensor * comb = nullptr;
+
+ ggml_tensor * attn_pre = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, &post, &comb, true, il);
+
+ // the first layer's attention collapses with the identity pre-mix: copy 0
+ cur = pre_prev ? build_hc_pre(inpL, pre_prev, il) : ggml_cont(ctx0, ggml_view_2d(ctx0, inpL, n_embd, n_tokens, inpL->nb[2], 0));
+ cb(cur, "hc_attn_pre", il);
+
+ cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
+ cb(cur, "attn_norm", il);
+
+ cur = build_attention41(model, inp_dsv4, cur, inp_pos, cand_pin, il);
+
+ inpL = build_hc_post(cur, residual, post, comb, il);
+ cb(inpL, "hc_attn_post", il);
+
+ residual = inpL;
+ pre_prev = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, &post, &comb, true, il);
+
+ cur = build_hc_pre(inpL, attn_pre, il);
+ cb(cur, "hc_ffn_pre", il);
+
+ ggml_build_forward_expand(gf, residual);
+ ggml_build_forward_expand(gf, post);
+ ggml_build_forward_expand(gf, comb);
+
+ cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
+ cb(cur, "ffn_norm", il);
+
+ // F32 router inputs: expert selection is near-tie sensitive
+ ggml_tensor * logits = build_lora_mm(layer.ffn_gate_inp, cur);
+ ggml_prec_set_acc(logits, GGML_PREC_F32);
+ if (!ggml_is_quantized(layer.ffn_gate_inp->type)) {
+ ggml_prec_set_src(logits, GGML_PREC_F32, 1);
+ }
+ cb(logits, "ffn_moe_logits", il);
+
+ // image (embedding) tokens use the vision router bias, as DeepSeek-V4
+ ggml_tensor * exp_probs_b = layer.ffn_exp_probs_b;
+ if (ubatch.embd != nullptr && layer.ffn_exp_probs_b_vl) {
+ exp_probs_b = layer.ffn_exp_probs_b_vl;
+ }
+
+ // the reference normalizes only when it routes to more than one expert
+ ggml_tensor * moe_out = build_moe_ffn(cur,
+ layer.ffn_gate_inp,
+ layer.ffn_up_exps,
+ layer.ffn_gate_exps,
+ layer.ffn_down_exps,
+ exp_probs_b,
+ n_expert, hparams.n_expert_used(),
+ LLM_FFN_SILU, hparams.expert_weights_norm && hparams.n_expert_used() > 1,
+ hparams.expert_weights_scale,
+ (llama_expert_gating_func_type) hparams.expert_gating_func,
+ il, logits);
+ cb(moe_out, "ffn_moe_out", il);
+
+ ggml_tensor * ffn_shexp = build_ffn(cur,
+ layer.ffn_up_shexp, nullptr, nullptr,
+ layer.ffn_gate_shexp, nullptr, nullptr,
+ layer.ffn_down_shexp, nullptr, nullptr,
+ nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il);
+ cb(ffn_shexp, "ffn_shexp", il);
+
+ cur = ggml_add(ctx0, moe_out, ffn_shexp);
+ cb(cur, "ffn_out", il);
+
+ inpL = build_hc_post(cur, residual, post, comb, il);
+ inpL = build_cvec(inpL, il);
+ cb(inpL, "l_out", il);
+ }
+
+ // no hc_head: collapse with the last FFN's pre-mix
+ if (inp_out_ids) {
+ inpL = ggml_reshape_3d(ctx0, ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, inpL, n_embd*hc, n_tokens), inp_out_ids), n_embd, hc, inp_out_ids->ne[0]);
+ pre_prev = ggml_get_rows(ctx0, pre_prev, inp_out_ids);
+ }
+ cur = build_hc_pre(inpL, pre_prev, -1);
+ cb(cur, "hc_out", -1);
+
+ cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1);
+ cb(cur, "result_norm", -1);
+ res->t_embd = cur;
+
+ cur = ggml_mul_mat(ctx0, model.output, cur);
+ cb(cur, "result_output", -1);
+ res->t_logits = cur;
+}
diff --git a/src/models/dflash.cpp b/src/models/dflash.cpp
index e15af6afd..8b6d6abdb 100644
--- a/src/models/dflash.cpp
+++ b/src/models/dflash.cpp
@@ -81,6 +81,7 @@ void llama_model_dflash::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, hparams.dsv4_hc_sinkhorn_iters);
ml.get_key(LLM_KV_HYPER_CONNECTION_EPSILON, hparams.dsv4_hc_eps);
ml.get_arr(LLM_KV_ATTENTION_COMPRESS_RATIOS, hparams.dsv4_compress_ratios, false);
+ ml.get_key(LLM_KV_HYPER_CONNECTION_SINGLE_PASS, hparams.dsv4_hc_single_pass, false);
GGML_ASSERT(hparams.dsv4_o_group_count > 0); // avoid div by zero
@@ -94,6 +95,11 @@ void llama_model_dflash::load_arch_hparams(llama_model_loader & ml) {
}
GGML_ASSERT(hparams.n_swa > 0);
+ if (hparams.dsv4_hc_single_pass) {
+ // DeepSeek-V4.1 DSpark: block queries see the main-KV ring plus the block via the mask row of the
+ // first query, so that row's SWA window must reach window_size positions before the anchor
+ hparams.n_swa += 1;
+ }
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
hparams.set_swa_pattern(0);
for (uint32_t il = 0; il < hparams.n_layer_all; ++il) {
@@ -257,9 +263,12 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) {
const int64_t hc_dim = hc_mult * n_embd;
const int64_t hc_mix_dim = (2 + hc_mult) * hc_mult;
- hc_head_fn = create_tensor(tn(LLM_TENSOR_HC_HEAD_FN, "weight"), {hc_dim, hc_mult}, 0);
- hc_head_base = create_tensor(tn(LLM_TENSOR_HC_HEAD_BASE, "weight"), {hc_mult}, 0);
- hc_head_scale = create_tensor(tn(LLM_TENSOR_HC_HEAD_SCALE, "weight"), {1}, 0);
+ // single-pass mHC: collapse with the last FFN pre-mix, no hc_head
+ if (!hparams.dsv4_hc_single_pass) {
+ hc_head_fn = create_tensor(tn(LLM_TENSOR_HC_HEAD_FN, "weight"), {hc_dim, hc_mult}, 0);
+ hc_head_base = create_tensor(tn(LLM_TENSOR_HC_HEAD_BASE, "weight"), {hc_mult}, 0);
+ hc_head_scale = create_tensor(tn(LLM_TENSOR_HC_HEAD_SCALE, "weight"), {1}, 0);
+ }
for (int i = 0; i < n_layer; ++i) {
auto & layer = layers[i];
@@ -1280,12 +1289,260 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_
}
}
+// DeepSeek-V4.1 DSpark decoder (mtp.0..2), dual-mode by batch type like graph_dsv4:
+// * embd batch -> writes every stage's kv_norm(wkv(main_proj(target features))) into its window ring
+// * token batch -> [anchor, noise...] block through the stages, single-pass mHC, no per-head q norm
+llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const llm_graph_params & params) :
+ llama_model_deepseek41::graph(params) {
+ const int64_t n_embd_inp = hparams.n_embd_inp_enc();
+ const int64_t n_embd_head = hparams.n_embd_head_k();
+ const int64_t n_embd_head_rope = hparams.n_rot();
+ const int64_t n_embd_head_nope = n_embd_head - n_embd_head_rope;
+
+ GGML_ASSERT(hparams.dsv4_hc_single_pass);
+
+ ggml_tensor * inp_pos = build_inp_pos();
+
+ llm_graph_input_attn_k_iswa * inp_attn = build_attn_inp_k_iswa();
+
+ // ratio-0 layers: plain rope at rope_theta, no YaRN (original_seq_len 0)
+ const auto rope = [&](ggml_tensor * t) {
+ t = ggml_rope_ext(ctx0, t, inp_pos, nullptr, n_embd_head_rope, rope_type, 0,
+ freq_base, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+ return ggml_rope_set_offset(t, n_embd_head_nope);
+ };
+
+ ASSERT_EMBD_OR_TOKEN(ubatch);
+ if (ubatch.embd) {
+ auto inp = std::make_unique<llm_graph_input_embd>(n_embd_inp);
+
+ inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd_inp, n_tokens);
+ ggml_set_input(inp->embd);
+
+ ggml_tensor * inp_target = inp->embd;
+ cb(inp_target, "inp_target_features", -1);
+
+ res->add_input(std::move(inp));
+
+ ggml_tensor * inp_g = build_lora_mm(model.fc, inp_target, model.fc_s);
+ inp_g = build_norm(inp_g, model.output_norm_enc, nullptr, LLM_NORM_RMS, -1);
+ cb(inp_g, "main_x", -1);
+
+ for (int il = 0; il < n_layer; ++il) {
+ const auto & layer = model.layers[il];
+
+ ggml_tensor * kv = build_lora_mm(layer.wkv, inp_g);
+ kv = build_norm(kv, layer.attn_kv_norm, nullptr, LLM_NORM_RMS, il);
+ kv = ggml_reshape_3d(ctx0, kv, n_embd_head, 1, n_tokens);
+ kv = rope(kv);
+ cb(kv, "kv_injected", il);
+
+ if (inp_attn->self_k_rot_swa) {
+ kv = llama_mul_mat_hadamard(ctx0, kv, inp_attn->self_k_rot_swa);
+ }
+ ggml_build_forward_expand(gf, inp_attn->mctx->get_swa()->cpy_k(ctx0, kv, inp_attn->get_k_idxs_swa(), il));
+ }
+
+ res->t_embd = inp_g;
+
+ ggml_build_forward_expand(gf, inp_g);
+ return;
+ }
+
+ auto * tok_embd = model.tok_embd;
+ if (tok_embd == nullptr) {
+ GGML_ASSERT(cparams.ctx_other != nullptr);
+ const auto * model_other = llama_get_model(cparams.ctx_other);
+ GGML_ASSERT(model_other->tok_embd != nullptr && "DSpark decoder requires the target model's token embeddings");
+ tok_embd = model_other->tok_embd;
+ }
+
+ auto inp = std::make_unique<llm_graph_input_embd>(n_embd);
+
+ inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
+ ggml_set_input(inp->tokens);
+
+ ggml_tensor * inp_tokens = inp->tokens;
+
+ ggml_tensor * inpL = ggml_get_rows(ctx0, tok_embd, inp->tokens);
+ cb(inpL, "inp_noise_embd", -1);
+
+ res->add_input(std::move(inp));
+
+ const int64_t hc = hparams.dsv4_hc_mult;
+ const int64_t n_groups = hparams.dsv4_o_group_count;
+ const int64_t n_heads_group = n_head / n_groups;
+ const int64_t o_lora_rank = hparams.dsv4_o_lora_rank;
+ const int64_t o_group_dim = n_heads_group*n_embd_head;
+ const float kq_scale = 1.0f/sqrtf(float(n_embd_head));
+
+ GGML_ASSERT(n_embd_head == n_embd_head_v);
+ GGML_ASSERT(n_head % n_groups == 0);
+
+ inpL = ggml_reshape_3d(ctx0, inpL, n_embd, 1, n_tokens);
+ inpL = ggml_repeat_4d(ctx0, inpL, n_embd, hc, n_tokens, 1);
+ cb(inpL, "hc_init", -1);
+
+ // window of a block = window of its first query, see n_swa
+ {
+ ggml_tensor * m = inp_attn->self_kq_mask_swa_cnv;
+ const int64_t n_blocks = ubatch.n_seqs_unq;
+ const int64_t bd = n_blocks > 0 ? n_tokens / n_blocks : 0;
+ if (m && bd > 1 && n_tokens % n_blocks == 0 && m->ne[2] == 1 && m->ne[1] % bd == 0 && ggml_is_contiguous(m)) {
+ ggml_tensor * m4 = ggml_reshape_4d(ctx0, m, m->ne[0], bd, m->ne[1]/bd, m->ne[3]);
+ ggml_tensor * r0 = ggml_view_4d(ctx0, m4, m4->ne[0], 1, m4->ne[2], m4->ne[3], m4->nb[1], m4->nb[2], m4->nb[3], 0);
+ m4 = ggml_repeat(ctx0, r0, m4);
+ inp_attn->self_kq_mask_swa_cnv = ggml_reshape_4d(ctx0, m4, m->ne[0], m->ne[1], 1, m->ne[3]);
+ cb(inp_attn->self_kq_mask_swa_cnv, "dspark_block_mask", -1);
+ }
+ }
+
+ ggml_tensor * pre_prev = nullptr;
+
+ for (int il = 0; il < n_layer; ++il) {
+ const auto & layer = model.layers[il];
+
+ ggml_tensor * residual = inpL;
+ ggml_tensor * post = nullptr;
+ ggml_tensor * comb = nullptr;
+
+ ggml_tensor * attn_pre = build_hc_mixes(inpL, layer.hc_attn_fn, layer.hc_attn_scale, layer.hc_attn_base, &post, &comb, false, il);
+
+ ggml_tensor * cur;
+ if (pre_prev == nullptr) {
+ // identity pre-mix of the first stage: copy 0
+ cur = ggml_cont(ctx0, ggml_view_2d(ctx0, inpL, n_embd, n_tokens, inpL->nb[2], 0));
+ } else {
+ cur = build_hc_pre(inpL, pre_prev, il);
+ }
+ cb(cur, "hc_attn_pre", il);
+
+ cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);
+ cb(cur, "attn_norm", il);
+
+ // non-causal draft context: block queries see the main-KV ring plus every block key
+ {
+ ggml_tensor * qr = build_lora_mm(layer.wq_a, cur);
+ qr = build_norm(qr, layer.attn_q_a_norm, nullptr, LLM_NORM_RMS, il);
+ cb(qr, "qr_norm", il);
+
+ ggml_tensor * q = build_lora_mm(layer.wq_b, qr);
+ q = ggml_reshape_3d(ctx0, q, n_embd_head, n_head, n_tokens);
+ q = rope(q);
+ cb(q, "q", il);
+
+ ggml_tensor * kv = build_lora_mm(layer.wkv, cur);
+ kv = build_norm(kv, layer.attn_kv_norm, nullptr, LLM_NORM_RMS, il);
+ kv = ggml_reshape_3d(ctx0, kv, n_embd_head, 1, n_tokens);
+ kv = rope(kv);
+ cb(kv, "kv", il);
+
+ ggml_tensor * out = build_attn(inp_attn,
+ nullptr, nullptr, nullptr,
+ q, kv, kv,
+ nullptr, layer.attn_sinks, nullptr,
+ kq_scale, il);
+ cb(out, "attn_raw", il);
+
+ out = ggml_reshape_3d(ctx0, out, n_embd_head, n_head, n_tokens);
+ out = ggml_rope_ext_back(ctx0, out, inp_pos, nullptr, n_embd_head_rope, rope_type, 0,
+ freq_base, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+ out = ggml_rope_set_offset(out, n_embd_head_nope);
+ cb(out, "attn_derope", il);
+
+ out = ggml_reshape_3d(ctx0, out, o_group_dim, n_groups, n_tokens);
+ out = ggml_permute(ctx0, out, 0, 2, 1, 3);
+ ggml_tensor * oa = ggml_mul_mat(ctx0, layer.wo_a, out);
+ cb(oa, "attn_wo_a", il);
+ oa = ggml_permute(ctx0, oa, 0, 2, 1, 3);
+ oa = ggml_cont_2d(ctx0, oa, o_lora_rank*n_groups, n_tokens);
+
+ cur = build_lora_mm(layer.wo_b, oa);
+ cb(cur, "attn_out", il);
+ }
+
+ inpL = build_hc_post(cur, residual, post, comb, il);
+ cb(inpL, "hc_attn_post", il);
+
+ residual = inpL;
+ ggml_tensor * ffn_pre = build_hc_mixes(inpL, layer.hc_ffn_fn, layer.hc_ffn_scale, layer.hc_ffn_base, &post, &comb, false, il);
+
+ cur = build_hc_pre(inpL, attn_pre, il);
+ cb(cur, "hc_ffn_pre", il);
+
+ cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
+ cb(cur, "ffn_norm", il);
+
+ ggml_tensor * moe_out = build_moe_ffn(cur,
+ layer.ffn_gate_inp,
+ layer.ffn_up_exps,
+ layer.ffn_gate_exps,
+ layer.ffn_down_exps,
+ layer.ffn_exp_probs_b,
+ n_expert, hparams.n_expert_used(),
+ LLM_FFN_SILU, hparams.expert_weights_norm,
+ hparams.expert_weights_scale,
+ (llama_expert_gating_func_type) hparams.expert_gating_func,
+ il);
+ cb(moe_out, "ffn_moe_out", il);
+
+ ggml_tensor * ffn_shexp = build_ffn(cur,
+ layer.ffn_up_shexp, nullptr, nullptr,
+ layer.ffn_gate_shexp, nullptr, nullptr,
+ layer.ffn_down_shexp, nullptr, nullptr,
+ nullptr, LLM_FFN_SILU, LLM_FFN_PAR, il);
+ cb(ffn_shexp, "ffn_shexp", il);
+
+ cur = ggml_add(ctx0, moe_out, ffn_shexp);
+ cb(cur, "ffn_out", il);
+
+ inpL = build_hc_post(cur, residual, post, comb, il);
+
+ ggml_tensor * l_out = ggml_reshape_2d(ctx0, inpL, n_embd*hc, n_tokens);
+ cb(l_out, "l_out", il);
+ ggml_build_forward_expand(gf, l_out);
+
+ pre_prev = ffn_pre;
+ }
+
+ // no hc_head: collapse with the last FFN's pre-mix
+ ggml_tensor * cur = build_hc_pre(inpL, pre_prev, -1);
+ cb(cur, "hc_out", -1);
+
+ // confidence head takes the un-normed collapsed state
+ res->t_embd = cur;
+
+ cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1);
+ cb(cur, "result_norm", -1);
+
+ auto * output = model.output;
+ auto * output_s = model.output_s;
+ if (output == nullptr) {
+ GGML_ASSERT(cparams.ctx_other != nullptr);
+ const auto * model_other = llama_get_model(cparams.ctx_other);
+ GGML_ASSERT(model_other->output != nullptr && "DSpark decoder requires the target model's output projection");
+ output = model_other->output;
+ output_s = model_other->output_s;
+ }
+
+ cur = build_lora_mm(output, cur, output_s);
+ cb(cur, "result_output", -1);
+ res->t_logits = cur;
+
+ if (model.dspark_markov_w1) {
+ build_dspark_markov_head(*this, model, inp_tokens);
+ }
+}
+
std::unique_ptr<llm_graph_context> llama_model_dflash::build_arch_graph(const llm_graph_params & params) const {
switch (params.gtype) {
case LLM_GRAPH_TYPE_ENCODER:
return std::make_unique<graph<true>>(*this, params);
case LLM_GRAPH_TYPE_DEFAULT:
case LLM_GRAPH_TYPE_DECODER:
+ if (hparams.dsv4_hc_mult > 0 && hparams.dsv4_hc_single_pass) {
+ return std::make_unique<graph_dsv41>(*this, params);
+ }
if (hparams.dsv4_hc_mult > 0) {
return std::make_unique<graph_dsv4>(*this, params);
}
diff --git a/src/models/models.h b/src/models/models.h
index bdcc1a4fa..5fa552026 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -1331,6 +1331,15 @@ struct llama_model_dots3note : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
+// YaRN attention factor that cancels ggml's rope mscale (the DeepSeek-V4 reference applies none)
+static inline float dsv4_rope_attn_factor(float freq_scale, float ext_factor) {
+ if (ext_factor == 0.0f) {
+ return 1.0f;
+ }
+
+ return 1.0f / (1.0f + 0.1f*logf(1.0f/freq_scale));
+}
+
struct llama_model_deepseek4 : public llama_model_base {
llama_model_deepseek4(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1359,6 +1368,17 @@ struct llama_model_deepseek4 : public llama_model_base {
ggml_tensor ** comb,
int il) const;
+ // returns the pre weights and fills post / comb
+ ggml_tensor * build_hc_mixes(
+ ggml_tensor * x,
+ ggml_tensor * hc_fn,
+ ggml_tensor * hc_scale,
+ ggml_tensor * hc_base,
+ ggml_tensor ** post,
+ ggml_tensor ** comb,
+ bool f32_src,
+ int il) const;
+
ggml_tensor * build_hc_post(
ggml_tensor * x,
ggml_tensor * residual,
@@ -1473,6 +1493,57 @@ struct llama_model_deepseek4 : public llama_model_base {
};
+struct llama_model_deepseek41 : public llama_model_deepseek4 {
+ llama_model_deepseek41(const struct llama_model_params & params) : llama_model_deepseek4(params) {}
+ void load_arch_hparams(llama_model_loader & ml) override;
+ void load_arch_tensors(llama_model_loader & ml) override;
+
+ // Engram: the hash constants are hparams.ple_* (one module per Engram layer), hashed on the host (ggml has no
+ // int64 mul/xor/mod); the token map and its pad id are V4.1 only
+ uint32_t engram_pad_id = 0;
+ std::vector<int32_t> engram_token_map;
+
+ int engram_index(int il) const;
+
+ void engram_hash(int e, const llama_ubatch & ubatch, const std::vector<llama_token> & prev, std::vector<int32_t> & out) const;
+
+ struct graph : public llama_model_deepseek4::graph {
+ graph(const llama_model & model, const llm_graph_params & params);
+ // helpers only (no graph built): base of graph_dsv41
+ explicit graph(const llm_graph_params & params) : llama_model_deepseek4::graph(params) {}
+
+ ggml_tensor * build_engram(
+ const llama_model_deepseek41 & model,
+ ggml_tensor * x,
+ ggml_tensor * rows,
+ ggml_tensor * keep,
+ int il) const;
+
+ ggml_tensor * build_attention41(
+ const llama_model_deepseek41 & model,
+ llm_graph_input_dsv4 * inp_dsv4,
+ ggml_tensor * cur,
+ ggml_tensor * inp_pos,
+ ggml_tensor * cand_pin,
+ int il);
+
+ std::vector<ggml_tensor *> topk_of; // top-k of each index source layer
+ std::vector<ggml_tensor *> comp_row_of; // [n_layer] new compressed rows of each kv source (for mirrors)
+ ggml_tensor * cand_sel = nullptr; // candidate blocks of the candidate source layer
+ int64_t cand_n_comp = 0; // compressed rows the candidate source scored
+
+ // sparse path (LLAMA_DSV41_SPARSE=0 disables): fused indexer, FA bounded to window + top-k, decode gather
+ bool sparse = true;
+ // gather path, shared by layers with the same top-k: selected K rows per (storage layer, top-k source),
+ // the sorted top-k and the mask per top-k source (one input copy per split)
+ std::map<std::pair<int32_t, int32_t>, ggml_tensor *> gather_k_of;
+ std::map<int32_t, ggml_tensor *> gather_sel_of;
+ std::map<int32_t, ggml_tensor *> gather_mask_of;
+ };
+
+ std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
struct llama_model_deepseek2ocr : public llama_model_base {
llama_model_deepseek2ocr(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
@@ -1532,6 +1603,11 @@ struct llama_model_dflash : public llama_model_base {
graph_dsv4(const llama_model & model, const llm_graph_params & params);
};
+ // DeepSeek-V4.1 DSpark: single-pass mHC (lagged pre-mix, no hc_head), no per-head q norm
+ struct graph_dsv41 : public llama_model_deepseek41::graph {
+ graph_dsv41(const llama_model & model, const llm_graph_params & params);
+ };
+
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp
index 97e71d3d2..371db0035 100644
--- a/src/models/qwen4exp.cpp
+++ b/src/models/qwen4exp.cpp
@@ -115,6 +115,9 @@ void llama_model_qwen4exp::load_arch_hparams(llama_model_loader & ml) {
hparams.ple_n_heads = (hparams.ple_ngram_size - 1) * hparams.ple_heads_per_ngram;
hparams.ple_head_dim = hparams.n_embd_per_layer;
+ hparams.ple_embd_token = hparams.ple_image_token_id != 0
+ ? (llama_token) hparams.ple_image_token_id
+ : (llama_token) hparams.ple_eos_token_id;
if (hparams.ple_ngram_size < 2 || hparams.ple_ngram_size > LLAMA_MAX_PLE_NGRAM) {
throw std::runtime_error(format("PLE n-gram size %u is out of range", hparams.ple_ngram_size));
}
diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv
index 56044b350..2c57c2a14 100644
--- a/tests/fusion/MTL.csv
+++ b/tests/fusion/MTL.csv
@@ -49,6 +49,8 @@ deepseek32 ,0 ,any ,NORM+MUL+ADD , 2
deepseek32 ,0 ,any ,RMS_NORM+MUL , 9
deepseek4 ,0 ,any ,MUL+ADD , 8
deepseek4 ,0 ,any ,RMS_NORM+MUL , 20
+deepseek41 ,0 ,any ,MUL+ADD , 12
+deepseek41 ,0 ,any ,RMS_NORM+MUL , 29
dots1 ,0 ,any ,ADD+ADD , 1
dots1 ,0 ,any ,MUL+ADD , 2
dots1 ,0 ,any ,RMS_NORM+MUL , 9
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 92808cd8c..1d649d6ca 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -10283,6 +10283,8 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
// im2col 1D
test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {3000, 128, 1, 1}, {3, 128, 1280, 1}, 1, 0, 1, 0, 1, 0, false));
+ // deepseek4v / deepseek41v aligner: 3x3 stride-3 unfold (F.unfold == im2col)
+ test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F32, GGML_TYPE_F32, {48, 33, 1024, 1}, {3, 3, 1024, 1}, 3, 3, 0, 0, 1, 1, true));
test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F32, {3000, 128, 1, 1}, {3, 128, 1280, 1}, 1, 0, 1, 0, 1, 0, false));
test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {3000, 128, 1, 1}, {3, 128, 1280, 1}, 1, 0, 1, 0, 1, 0, false));
test_cases.emplace_back(new test_im2col(GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_F16, {3000, 384, 1, 1}, {3, 384, 384, 1}, 1, 0, 1, 0, 1, 0, false));
diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp
index 49519eeca..0de475132 100644
--- a/tests/test-chat.cpp
+++ b/tests/test-chat.cpp
@@ -149,6 +149,21 @@ static common_chat_templates_ptr read_templates(const std::string & path) {
return common_chat_templates_ptr(common_chat_templates_init(/* model= */ nullptr, read_file(path)));
}
+// DeepSeek V4.1 stand-in: the real template ships in the GGUF; this keeps only the turn markers,
+// the generation prompt, the V4.1 DSML block tag used for detection and the extra keys it reads
+static const char * DEEPSEEK_V4_1_MIN_TEMPLATE = R"({%- for m in messages -%}
+{%- if m.role == 'system' -%}<｜System｜>{{ m.content }}
+{%- elif m.role == 'user' -%}<｜User｜>
+{%- if m.content is string -%}{{ m.content }}{%- else -%}{%- for b in m.content -%}{%- if not loop.first -%}{{ '\n\n' }}{%- endif -%}{{ b.text }}{%- endfor -%}{%- endif -%}
+{%- if m.task -%}<｜{{ m.task }}｜>{%- endif -%}
+{%- elif m.role == 'assistant' -%}<｜Assistant｜></think>{{ m.content }}
+{%- for tc in m.tool_calls or [] -%}<｜DSML｜ calls><｜DSML｜ invoke name="{% if tc.namespace %}{{ tc.namespace }}::{% endif %}{{ tc.function.name }}"></｜DSML｜ invoke></｜DSML｜ calls>{%- endfor -%}
+<｜end▁of▁sentence｜>
+{%- endif -%}
+{%- endfor -%}
+{%- if reasoning_effort is defined and reasoning_effort is not none -%}<｜effort={{ reasoning_effort }}｜>{%- endif -%}
+{%- if add_generation_prompt -%}<｜Assistant｜>{%- if enable_thinking -%}<think>{%- else -%}</think>{%- endif -%}{%- endif -%})";
+
static std::unique_ptr<llama_grammar> build_grammar(const std::string & grammar_str) {
return std::unique_ptr<llama_grammar>(
llama_grammar_init_impl(nullptr, grammar_str.c_str(), "root", false, nullptr, 0, nullptr, 0));
@@ -961,6 +976,7 @@ const common_chat_msg message_user{
/* .reasoning_content = */ "",
/* .tool_name = */ "",
/* .tool_call_id = */ "",
+ /* .extra_json = */ "",
};
const common_chat_msg message_user_parts{
@@ -977,6 +993,7 @@ const common_chat_msg message_user_parts{
"",
/* .tool_name = */ "",
/* .tool_call_id = */ "",
+ /* .extra_json = */ "",
};
static common_chat_msg simple_assist_msg(const std::string & content,
@@ -1486,6 +1503,11 @@ class peg_tester {
template_path_(template_path),
detailed_debug_(detailed_debug) {}
+ peg_tester(const std::string & name, common_chat_templates_ptr tmpls, const bool detailed_debug = false) :
+ tmpls_(std::move(tmpls)),
+ template_path_(name),
+ detailed_debug_(detailed_debug) {}
+
const std::string & template_path() const { return template_path_; }
peg_test_builder test(const std::string & input);
@@ -4381,6 +4403,103 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
.run();
}
+ // DeepSeek V4.1: V4 behaviour, DSML tag names with a leading space
+ {
+ auto tst = peg_tester("deepseek-v4.1-min",
+ common_chat_templates_ptr(common_chat_templates_init(nullptr, DEEPSEEK_V4_1_MIN_TEMPLATE)),
+ detailed_debug);
+
+ tst.test("Hello, world!\nWhat's up?")
+ .enable_thinking(false)
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .expect(message_assist)
+ .run();
+
+ tst.test("I'm\nthinking</think>Hello, world!\nWhat's up?")
+ .enable_thinking(true)
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .expect(message_assist_thoughts)
+ .run();
+
+ tst.test(
+ "Let me check the time</think>\n\n"
+ "<｜DSML｜ calls>\n"
+ "<｜DSML｜ invoke name=\"get_time\">\n"
+ "<｜DSML｜ parameter name=\"city\" string=\"true\">Tokyo</｜DSML｜ parameter>\n"
+ "</｜DSML｜ invoke>\n"
+ "</｜DSML｜ calls>")
+ .enable_thinking(true)
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .tools({ get_time_tool })
+ .expect(message_with_tool_calls_and_reasoning("get_time", R"({"city": "Tokyo"})", "Let me check the time"))
+ .run();
+
+ tst.test(
+ "<｜DSML｜ calls>\n"
+ "<｜DSML｜ invoke name=\"special_function\">\n"
+ "<｜DSML｜ parameter name=\"arg1\" string=\"false\">1</｜DSML｜ parameter>\n"
+ "</｜DSML｜ invoke>\n"
+ "</｜DSML｜ calls>")
+ .enable_thinking(false)
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .tools({ special_function_tool })
+ .expect(message_assist_call)
+ .run();
+
+ tst.test(
+ "Calling both</think>\n\n"
+ "<｜DSML｜ calls>\n"
+ "<｜DSML｜ invoke name=\"get_time\">\n"
+ "<｜DSML｜ parameter name=\"city\" string=\"true\">Paris</｜DSML｜ parameter>\n"
+ "</｜DSML｜ invoke>\n"
+ "<｜DSML｜ invoke name=\"get_weather\">\n"
+ "<｜DSML｜ parameter name=\"city\" string=\"true\">Paris</｜DSML｜ parameter>\n"
+ "</｜DSML｜ invoke>\n"
+ "</｜DSML｜ calls>")
+ .enable_thinking(true)
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .parallel_tool_calls(true)
+ .tools({ get_time_tool, get_weather_tool })
+ .expect(message_with_reasoning_content_and_multiple_tool_calls(
+ "Calling both", "",
+ { { "get_time", R"({"city": "Paris"})" }, { "get_weather", R"({"city": "Paris"})" } }))
+ .run();
+
+ tst.test("world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_content })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
+ tst.test(" thinking</think>Hello, world!\nWhat's up?")
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .enable_thinking(true)
+ .messages({ message_user, message_assist_prefill_reasoning })
+ .add_generation_prompt(false)
+ .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING)
+ .expect_reasoning("I'm thinking")
+ .expect_content("Hello, world!\nWhat's up?")
+ .run();
+
+ tst.test(
+ "Let me check the time\n\n"
+ "<｜DSML｜ calls>\n"
+ "<｜DSML｜ invoke name=\"get_time\">\n"
+ "<｜DSML｜ parameter name=\"city\" string=\"true\">Tokyo</｜DSML｜ parameter>\n"
+ "</｜DSML｜ invoke>\n"
+ "</｜DSML｜ calls>") // no </think> after the TC close because the grammar will immediately constrain it to end
+ .enable_thinking(true)
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .tools({ get_time_tool })
+ .expect_reasoning("Let me check the time")
+ .expect_tool_calls({ { "get_time", R"({"city": "Tokyo"})", {} } })
+ .run();
+ }
+
{
// The DSML separator belongs to the tool call block, not assistant content.
auto tst = peg_tester("models/templates/deepseek-ai-DeepSeek-V4-Flash-0731.jinja", detailed_debug);
@@ -7825,6 +7944,62 @@ static void test_deepseek_v4_tool_result_ordering() {
}
}
+// DeepSeek V4.1: extra message keys, "\n\n" around media markers, reasoning_effort mapping, system-message tools
+static void test_deepseek_v4_1_message_fields() {
+ LOG_DBG("%s\n", __func__);
+
+ common_chat_templates_ptr tmpls(common_chat_templates_init(nullptr, DEEPSEEK_V4_1_MIN_TEMPLATE));
+ auto render = [&](const char * messages_json, bool add_generation_prompt) {
+ common_chat_templates_inputs inputs;
+ inputs.messages = common_chat_msgs_parse_oaicompat(json::parse(messages_json));
+ inputs.add_generation_prompt = add_generation_prompt;
+ inputs.enable_thinking = false;
+ return common_chat_templates_apply(tmpls.get(), inputs).prompt.str();
+ };
+
+ assert_equals(std::string("<｜User｜>q<｜action｜>"),
+ render(R"([{"role": "user", "content": "q", "task": "action"}])", false));
+
+ assert_contains(render(R"([
+ {"role": "user", "content": "q"},
+ {"role": "assistant", "content": "", "tool_calls": [{"type": "function", "namespace": "ns", "function": {"name": "f", "arguments": {"x": 1}}}]}
+ ])", false),
+ "<｜DSML｜ invoke name=\"ns::f\">");
+
+ assert_equals(std::string("<｜User｜>a\n\n<__media__>\n\nb<｜Assistant｜></think>"),
+ render(R"([{"role": "user", "content": [{"type": "text", "text": "a"}, {"type": "media_marker", "text": "<__media__>"}, {"type": "text", "text": "b"}]}])", true));
+
+ for (const auto & [effort, expected] : std::vector<std::pair<std::string, std::string>>{
+ { "minimal", "<｜effort=low｜>" }, { "low", "<｜effort=low｜>" }, { "medium", "" },
+ { "high", "<｜effort=high｜>" }, { "xhigh", "<｜effort=max｜>" }, { "max", "<｜effort=max｜>" } }) {
+ common_chat_templates_inputs inputs;
+ inputs.messages = { message_user };
+ inputs.add_generation_prompt = true;
+ inputs.chat_template_kwargs["reasoning_effort"] = json(effort).dump();
+ assert_equals("<｜User｜>Hey there!" + expected + "<｜Assistant｜><think>",
+ common_chat_templates_apply(tmpls.get(), inputs).prompt.str());
+ }
+
+ {
+ common_chat_templates_inputs inputs;
+ inputs.messages = common_chat_msgs_parse_oaicompat(json::parse(R"([
+ {"role": "system", "content": "s", "tools": [{"type": "function", "function": {"name": "get_time",
+ "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}}}]},
+ {"role": "user", "content": "q"}
+ ])"));
+ inputs.add_generation_prompt = true;
+ inputs.enable_thinking = false;
+ inputs.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;
+ common_chat_session session(tmpls.get(), nullptr, inputs, {});
+ const auto & msg = session.finish(common_chat_input(std::string(
+ "\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name=\"get_time\">\n"
+ "<｜DSML｜ parameter name=\"city\" string=\"true\">Tokyo</｜DSML｜ parameter>\n</｜DSML｜ invoke>\n</｜DSML｜ calls>")));
+ assert_equals(size_t(1), msg.tool_calls.size());
+ assert_equals(std::string("get_time"), msg.tool_calls[0].name);
+ assert_equals(std::string("{\"city\":\"Tokyo\"}"), msg.tool_calls[0].arguments);
+ }
+}
+
static void test_prompt_parts() {
LOG_DBG("%s\n", __func__);
const std::string tmpl_src =
@@ -8175,6 +8350,7 @@ int main(int argc, char ** argv) {
test_translate_gemma();
test_deepseek_v4_thinking_retention();
test_deepseek_v4_tool_result_ordering();
+ test_deepseek_v4_1_message_fields();
test_template_generation_prompt();
test_reasoning_effort_caps();
test_prompt_parts();
diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp
index 1e0b75511..00267716d 100644
--- a/tests/test-llama-archs.cpp
+++ b/tests/test-llama-archs.cpp
@@ -195,6 +195,12 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
n_head = 8;
n_ff = 1024;
n_layer = 4;
+ } else if (arch == LLM_ARCH_DEEPSEEK41) {
+ // window-only, Engram, ratio-2 and ratio-1 sources, top-k reuse and a candidate-masked index layer
+ n_embd = 512;
+ n_head = 8;
+ n_ff = 1024;
+ n_layer = 6;
} else if (arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_LAGUNA) {
n_embd = 160; // exercise per-head tensor split granularity with head size 80
} else if (arch == LLM_ARCH_QWEN3 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_AFMOE) {
@@ -285,11 +291,11 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, n_head_per_layer);
} else {
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head);
- ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, arch == LLM_ARCH_DEEPSEEK4 ? uint32_t(1) : n_head_kv);
+ ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41) ? uint32_t(1) : n_head_kv);
}
ms.add_kv(LLM_KV_ATTENTION_MAX_ALIBI_BIAS, 8.0f);
- if (arch == LLM_ARCH_DEEPSEEK4) {
+ if (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41) {
ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH, n_embd_head);
ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH, n_embd_head);
ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT, n_embd_head/2);
@@ -337,7 +343,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, 1e-5f);
ms.add_kv(LLM_KV_ATTENTION_GROUPNORM_EPS, 1e-5f);
ms.add_kv(LLM_KV_ATTENTION_GROUPNORM_GROUPS, uint32_t(8));
- ms.add_kv(LLM_KV_ATTENTION_Q_LORA_RANK, arch == LLM_ARCH_DEEPSEEK4 ? uint32_t(64) : uint32_t(512));
+ ms.add_kv(LLM_KV_ATTENTION_Q_LORA_RANK, (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41) ? uint32_t(64) : uint32_t(512));
ms.add_kv(LLM_KV_ATTENTION_KV_LORA_RANK, uint32_t(512));
ms.add_kv(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, uint32_t(8));
ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW, n_ctx/8);
@@ -476,6 +482,39 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true);
}
+ if (arch == LLM_ARCH_DEEPSEEK41) {
+ ms.add_kv(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, uint32_t(8));
+ ms.add_kv(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, uint32_t(32));
+ ms.add_kv(LLM_KV_ATTENTION_COMPRESS_RATIOS, std::vector<uint32_t>({0, 0, 2, 2, 1, 1}));
+ ms.add_kv(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, 160000.0f);
+ ms.add_kv(LLM_KV_ATTENTION_KV_SOURCE_LAYERS, std::vector<uint32_t>({2, 4}));
+ ms.add_kv(LLM_KV_ATTENTION_INDEXER_SOURCE_LAYERS, std::vector<uint32_t>({2, 4, 5}));
+ ms.add_kv(LLM_KV_ATTENTION_INDEXER_CANDIDATE_SOURCE_LAYER, int32_t(4));
+ ms.add_kv(LLM_KV_ATTENTION_INDEXER_CANDIDATE_TOP_K_BLOCKS, uint32_t(4096));
+ ms.add_kv(LLM_KV_ATTENTION_INDEXER_CANDIDATE_BLOCK_SIZE, uint32_t(4));
+ ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4));
+ ms.add_kv(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, uint32_t(2));
+ ms.add_kv(LLM_KV_HYPER_CONNECTION_EPSILON, 1.0e-6f);
+ ms.add_kv(LLM_KV_HASH_LAYER_COUNT, uint32_t(0));
+ ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 10.0f);
+ ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 1.0f);
+ ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true);
+ std::vector<int32_t> token_map(n_vocab);
+ for (uint32_t i = 0; i < n_vocab; ++i) {
+ token_map[i] = (int32_t) (i % 61);
+ }
+ ms.add_kv(LLM_KV_PLE_LAYERS, std::vector<uint32_t>({1, 3}));
+ ms.add_kv(LLM_KV_PLE_NGRAM_SIZE, uint32_t(3));
+ ms.add_kv(LLM_KV_PLE_HEADS_PER_NGRAM, uint32_t(2));
+ ms.add_kv(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, uint32_t(64));
+ ms.add_kv(LLM_KV_PLE_LAYER_MULTIPLIERS, std::vector<uint64_t>({ 76632096046245ull, 4839876093313ull, 35959672319349ull,
+ 67716810739261ull, 51510806800915ull, 30921347202721ull }));
+ ms.add_kv(LLM_KV_PLE_HEAD_OFFSETS, std::vector<uint64_t>({0, 101, 204, 311, 420, 533, 660, 791}));
+ ms.add_kv(LLM_KV_PLE_HEAD_VOCAB_SIZES, std::vector<uint64_t>({101, 103, 107, 109, 113, 127, 131, 137}));
+ ms.add_kv(LLM_KV_ENGRAM_PAD_TOKEN_ID, uint32_t(2));
+ ms.add_kv(LLM_KV_ENGRAM_TOKEN_MAP, token_map);
+ }
+
if (arch == LLM_ARCH_HRM_TEXT) {
// 6 cache slots alias 2 physical blocks: 1 low-stack layer + 1 high-stack layer
ms.add_kv(LLM_KV_HRM_LAYERS_PER_STACK, uint32_t(1));
@@ -519,6 +558,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
gating_func = 1;
break;
case LLM_ARCH_DEEPSEEK4:
+ case LLM_ARCH_DEEPSEEK41:
gating_func = 4;
break;
default:
@@ -780,6 +820,7 @@ static bool moe_mandatory(const llm_arch arch) {
case LLM_ARCH_DEEPSEEK32:
case LLM_ARCH_DOTS3NOTE:
case LLM_ARCH_DEEPSEEK4:
+ case LLM_ARCH_DEEPSEEK41:
case LLM_ARCH_GLM4_MOE:
case LLM_ARCH_GLM_DSA:
case LLM_ARCH_EXAONE_MOE:
diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h
index 4ffb7056a..661f0c646 100644
--- a/tools/mtmd/clip-impl.h
+++ b/tools/mtmd/clip-impl.h
@@ -529,6 +529,7 @@ enum projector_type {
PROJECTOR_TYPE_DEEPSEEKOCR,
PROJECTOR_TYPE_DEEPSEEKOCR2,
PROJECTOR_TYPE_DEEPSEEK4V,
+ PROJECTOR_TYPE_DEEPSEEK41V,
PROJECTOR_TYPE_LFM2A,
PROJECTOR_TYPE_D1OMNI_A,
PROJECTOR_TYPE_GLM4V,
@@ -606,6 +607,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_DEEPSEEKOCR, "deepseekocr"},
{ PROJECTOR_TYPE_DEEPSEEKOCR2, "deepseekocr2"},
{ PROJECTOR_TYPE_DEEPSEEK4V, "deepseek4v"},
+ { PROJECTOR_TYPE_DEEPSEEK41V, "deepseek41v"},
{ PROJECTOR_TYPE_LFM2A, "lfm2a"},
{ PROJECTOR_TYPE_D1OMNI_A, "d1omni_a"},
{ PROJECTOR_TYPE_GLM4V, "glm4v"},
@@ -879,6 +881,11 @@ static inline void clip_anyres_unpad(int cur_w, int cur_h, int orig_w, int orig_
}
}
+// deepseek41v: [START] ([IMAGE]*n_llm_w [NEWLINE])*n_llm_h [END], ref: num_image_tokens() in image_processor.py
+static inline int dsv41_n_block_tokens(int n_llm_w, int n_llm_h) {
+ return n_llm_h * (n_llm_w + 1) + 2;
+}
+
// deepseek4v: layout of the LLM token block built from the aligner grid
struct dsv4_block_layout {
int rows; // grid rows, padded to an even count
diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp
index d204708ee..204b16f3e 100644
--- a/tools/mtmd/clip.cpp
+++ b/tools/mtmd/clip.cpp
@@ -1111,6 +1111,7 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
builder = std::make_unique<clip_graph_kimik25>(ctx, img);
} break;
case PROJECTOR_TYPE_DEEPSEEK4V:
+ case PROJECTOR_TYPE_DEEPSEEK41V:
{
builder = std::make_unique<clip_graph_deepseek4v>(ctx, img);
} break;
@@ -1773,6 +1774,30 @@ struct clip_model_loader {
const int warmup_side = (int) std::sqrt((double) std::min(256, hparams.dsv4_max_n_token));
hparams.set_warmup_n_tokens(warmup_side * warmup_side);
} break;
+ case PROJECTOR_TYPE_DEEPSEEK41V:
+ {
+ // ref: load_image in inference/image_processor.py (V4.1), PIL ImageOps.pad on gray 127
+ hparams.image_resize_algo = RESIZE_ALGO_BICUBIC;
+ hparams.image_pad_color = {127, 127, 127};
+ hparams.rope_theta = 10000.0f;
+ get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge);
+ get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
+ // vision_max_n_token = 1024, vision_max_wh_ratio = null (converter asserts both)
+ hparams.dsv4_max_n_token = 1024;
+ hparams.dsv4_max_wh_ratio = 0;
+ const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge;
+ if (hparams.custom_image_min_tokens > 0) {
+ hparams.image_min_pixels = hparams.custom_image_min_tokens * patch_area;
+ }
+ if (hparams.custom_image_max_tokens > 0) {
+ // the cap is on the whole token block (newlines + start/end included)
+ hparams.dsv4_max_n_token = std::max(hparams.custom_image_max_tokens, 16);
+ }
+ hparams.image_max_pixels = hparams.dsv4_max_n_token * patch_area;
+ hparams.image_min_pixels = std::min(hparams.image_min_pixels, hparams.image_max_pixels);
+ const int warmup_side = (int) std::sqrt((double) std::min(256, hparams.dsv4_max_n_token));
+ hparams.set_warmup_n_tokens(warmup_side * warmup_side);
+ } break;
case PROJECTOR_TYPE_GEMMA3:
{
// default value (used by all model sizes in gemma 3 family)
@@ -3065,6 +3090,17 @@ struct clip_model_loader {
model.token_embd_img_end = get_tensor(TN_TOK_IMG_END);
model.token_embd_img_pad = get_tensor(TN_TOK_IMG_PAD);
} break;
+ case PROJECTOR_TYPE_DEEPSEEK41V:
+ {
+ model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
+ model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias"));
+ model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
+ model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
+ // image_start / image_end / image_newline, written into the output block
+ model.image_newline = get_tensor(TN_IMAGE_NEWLINE);
+ model.token_embd_img_start = get_tensor(TN_TOK_IMG_START);
+ model.token_embd_img_end = get_tensor(TN_TOK_IMG_END);
+ } break;
case PROJECTOR_TYPE_PIXTRAL:
{
model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight"));
@@ -4377,7 +4413,7 @@ struct clip_cap clip_get_cap(const char * fname) {
int clip_get_image_max_tokens(const clip_ctx * ctx) {
const auto & hparams = ctx->model.hparams;
- if (ctx->proj_type() == PROJECTOR_TYPE_DEEPSEEK4V) {
+ if (ctx->proj_type() == PROJECTOR_TYPE_DEEPSEEK4V || ctx->proj_type() == PROJECTOR_TYPE_DEEPSEEK41V) {
return hparams.dsv4_max_n_token;
}
if (hparams.image_max_pixels <= 0) {
@@ -4585,6 +4621,13 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
const int n_llm_h = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size;
n_patches = dsv4_get_block_layout(n_llm_w, n_llm_h, img->lead_pad).n_out;
} break;
+ case PROJECTOR_TYPE_DEEPSEEK41V:
+ {
+ const int out_patch_size = params.patch_size * params.n_merge;
+ const int n_llm_w = CLIP_ALIGN(img->nx(), out_patch_size) / out_patch_size;
+ const int n_llm_h = CLIP_ALIGN(img->ny(), out_patch_size) / out_patch_size;
+ n_patches = dsv41_n_block_tokens(n_llm_w, n_llm_h);
+ } break;
case PROJECTOR_TYPE_PADDLEOCR:
case PROJECTOR_TYPE_DOTS_OCR:
case PROJECTOR_TYPE_DOTS3NOTE_V:
@@ -5692,6 +5735,34 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
idx.push_back(idx_end);
set_input_i32("layout_idx", idx);
} break;
+ case PROJECTOR_TYPE_DEEPSEEK41V:
+ {
+ // mrope layout, only the first 2 channels are used
+ const int n_patches_per_row = image_size_width / patch_size;
+ std::vector<int32_t> positions(n_pos * 4, 0);
+ for (int i = 0; i < n_pos; i++) {
+ positions[i] = i / n_patches_per_row;
+ positions[n_pos + i] = i % n_patches_per_row;
+ }
+ set_input_i32("positions", positions);
+
+ // reading-order block: rows [0, n_grid) are the aligner output, then start / end / newline
+ const int n_merge = hparams.n_merge;
+ const int n_llm_w = CLIP_ALIGN(pos_w, n_merge) / n_merge;
+ const int n_llm_h = CLIP_ALIGN(pos_h, n_merge) / n_merge;
+ const int n_grid = n_llm_w * n_llm_h;
+ std::vector<int32_t> idx;
+ idx.reserve(dsv41_n_block_tokens(n_llm_w, n_llm_h));
+ idx.push_back(n_grid); // IMAGE_START
+ for (int r = 0; r < n_llm_h; r++) {
+ for (int c = 0; c < n_llm_w; c++) {
+ idx.push_back(r * n_llm_w + c);
+ }
+ idx.push_back(n_grid + 2); // IMAGE_NEW_LINE
+ }
+ idx.push_back(n_grid + 1); // IMAGE_END
+ set_input_i32("layout_idx", idx);
+ } break;
case PROJECTOR_TYPE_GLM_EDGE:
{
// llava and other models
@@ -6583,6 +6654,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
case PROJECTOR_TYPE_KIMIK25:
case PROJECTOR_TYPE_YASA2:
case PROJECTOR_TYPE_DEEPSEEK4V:
+ case PROJECTOR_TYPE_DEEPSEEK41V:
case PROJECTOR_TYPE_COHERE2V:
return ctx->model.mm_2_w->ne[1];
case PROJECTOR_TYPE_HUNYUANVL:
diff --git a/tools/mtmd/models/deepseek4v.cpp b/tools/mtmd/models/deepseek4v.cpp
index ffe8f59d9..06005d52c 100644
--- a/tools/mtmd/models/deepseek4v.cpp
+++ b/tools/mtmd/models/deepseek4v.cpp
@@ -14,6 +14,9 @@
// the mapping is precomputed on CPU as the "layout_idx" input (see set_input in clip.cpp)
//
// ref: inference/vision.py and inference/image_processor.py in the HF repo
+//
+// DeepSeek-V4.1-Flash (deepseek41v) shares the encoder and aligner; its block is plain reading order
+// [START] ([IMAGE]*n_llm_w [NEWLINE])*n_llm_h [END], no lead pads, no pad row, no IMAGE_PAD tensor
ggml_cgraph * clip_graph_deepseek4v::build() {
const int n_merge = hparams.n_merge;
@@ -52,7 +55,10 @@ ggml_cgraph * clip_graph_deepseek4v::build() {
cur = ggml_pad(ctx0, cur, pad_x, pad_y, 0, 0);
}
- ggml_tensor * kernel = ggml_view_3d(ctx0, cur, n_merge, n_merge, cur->ne[2], 0, 0, 0);
+ // contiguous kernel view: CUDA rejects a zero-stride one and im2col falls back to CPU
+ const size_t esz = ggml_element_size(cur);
+ ggml_tensor * kernel = ggml_view_3d(ctx0, cur, n_merge, n_merge, cur->ne[2],
+ n_merge*esz, n_merge*n_merge*esz, 0);
cur = ggml_im2col(ctx0, kernel, cur, n_merge, n_merge, 0, 0, 1, 1, true, inp->type);
cur = ggml_reshape_2d(ctx0, cur, cur->ne[0], cur->ne[1] * cur->ne[2]);
@@ -73,19 +79,23 @@ ggml_cgraph * clip_graph_deepseek4v::build() {
const int64_t n_grid = cur->ne[1]; // n_llm_w * n_llm_h
// rows n_grid + 0..3, keep in sync with the index computation in set_input
+ const bool is_v41 = proj_type == PROJECTOR_TYPE_DEEPSEEK41V;
ggml_tensor * sentinels[] = {
model.token_embd_img_start,
model.token_embd_img_end,
model.image_newline,
- model.token_embd_img_pad,
+ is_v41 ? nullptr : model.token_embd_img_pad,
};
for (ggml_tensor * tok : sentinels) {
- cur = ggml_concat(ctx0, cur, ggml_reshape_2d(ctx0, tok, n_embd_out, 1), 1);
+ if (tok) {
+ cur = ggml_concat(ctx0, cur, ggml_reshape_2d(ctx0, tok, n_embd_out, 1), 1);
+ }
}
const int n_llm_w = CLIP_ALIGN(n_patches_x, n_merge) / n_merge;
const int n_llm_h = CLIP_ALIGN(n_patches_y, n_merge) / n_merge;
- const int n_out = dsv4_get_block_layout(n_llm_w, n_llm_h, img.lead_pad).n_out;
+ const int n_out = is_v41 ? dsv41_n_block_tokens(n_llm_w, n_llm_h)
+ : dsv4_get_block_layout(n_llm_w, n_llm_h, img.lead_pad).n_out;
GGML_ASSERT(n_grid == n_llm_w * n_llm_h);
ggml_tensor * layout_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_out);
diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp
index cf6e9a8ac..551ea0c60 100644
--- a/tools/mtmd/mtmd-image.cpp
+++ b/tools/mtmd/mtmd-image.cpp
@@ -1454,6 +1454,100 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseek4v::preprocess(const clip
return out;
}
+//
+// DeepSeek-V4.1-Flash (deepseek41v)
+//
+// port of plan_image_grid / load_image from inference/image_processor.py (V4.1)
+// differs from deepseek4v: block = h * (w + 1) + 2 tokens, no even-row rounding, a single solve, no stretch path
+//
+
+void mtmd_image_preprocessor_deepseek41v::plan_grid(int width, int height, int p, int r, int max_n_token, int min_pixels,
+ int & best_width, int & best_height) {
+ auto n_tokens = [&](int bh, int bw) {
+ const int n_llm_h = ((bh / p) + r - 1) / r;
+ const int n_llm_w = ((bw / p) + r - 1) / r;
+ return n_llm_h * (n_llm_w + 1) + 2;
+ };
+ if (min_pixels > 0 && (int64_t) width * height > 0 && (int64_t) width * height < min_pixels) {
+ // python: ratio = (min_pixels / (w * h)) ** 0.5; int(w * ratio)
+ const double ratio = std::sqrt((double) min_pixels / ((double) width * (double) height));
+ width = (int) (width * ratio);
+ height = (int) (height * ratio);
+ }
+ best_width = CLIP_ALIGN(width, p);
+ best_height = CLIP_ALIGN(height, p);
+ if (n_tokens(best_height, best_width) > max_n_token) {
+ // solve_resize_ratio: largest aspect-preserving size whose token block fits
+ const double ratio = (double) height / width;
+ const double max_w_f = std::sqrt((max_n_token - 2) / ratio + 0.25) - 0.5;
+ const double max_h_f = max_w_f * ratio;
+ const int cell = p * r;
+ if (max_w_f < 1.0) {
+ best_height = (max_n_token - 2) / 2 * cell;
+ best_width = cell;
+ } else if (max_h_f < 1.0) {
+ best_height = cell;
+ best_width = (max_n_token - 3) * cell;
+ } else {
+ const double beta = std::min(std::floor(max_w_f) * cell / width, std::floor(max_h_f) * cell / height);
+ best_height = (int) std::floor(height * beta / p) * p;
+ best_width = (int) std::floor(width * beta / p) * p;
+ }
+ GGML_ASSERT(n_tokens(best_height, best_width) <= max_n_token);
+ }
+}
+
+mtmd_image_preproc_out mtmd_image_preprocessor_deepseek41v::preprocess(const clip_image_u8 & img) const {
+ mtmd_image_preproc_out out;
+
+ const int p = hparams.patch_size;
+ const clip_image_size orig = img.get_size();
+ int best_width = 0, best_height = 0;
+ plan_grid(orig.width, orig.height, p, hparams.n_merge, hparams.dsv4_max_n_token, hparams.image_min_pixels,
+ best_width, best_height);
+
+ // PIL ImageOps.pad(color=127): ImageOps.contain then a centered paste; python round is half to even
+ auto round_half_even = [](double x) { return (int) std::nearbyint(x); };
+ int new_w = best_width;
+ int new_h = best_height;
+ const double im_ratio = (double) orig.width / orig.height;
+ const double dest_ratio = (double) best_width / best_height;
+ if (im_ratio != dest_ratio) {
+ if (im_ratio > dest_ratio) {
+ new_h = round_half_even((double) orig.height / orig.width * best_width);
+ } else {
+ new_w = round_half_even((double) orig.width / orig.height * best_height);
+ }
+ }
+ clip_image_u8 resized;
+ img_tool::resize(img, resized, {new_w, new_h}, hparams.image_resize_algo, PAD_NONE);
+
+ clip_image_u8 padded;
+ if (new_w == best_width && new_h == best_height) {
+ padded = std::move(resized);
+ } else {
+ padded.set_size({best_width, best_height}, img.is_placeholder());
+ if (!img.is_placeholder()) {
+ img_tool::fill(padded, hparams.image_pad_color);
+ const int off_x = round_half_even((best_width - new_w) * 0.5);
+ const int off_y = round_half_even((best_height - new_h) * 0.5);
+ img_tool::composite(padded, resized, off_x, off_y);
+ }
+ }
+
+ out.append(hparams, padded);
+
+ // load_image rounds the normalized input to bf16 (round to nearest even)
+ for (auto & e : out.entries) {
+ std::vector<float> buf = e.get_ro_buf();
+ for (float & v : buf) {
+ v = ggml_bf16_to_fp32(ggml_fp32_to_bf16(v));
+ }
+ e.cpy_buf(buf);
+ }
+ return out;
+}
+
mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img) const {
mtmd_image_preproc_out output;
int grid_w = 0;
diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h
index cb66ac2ea..40d5b59cc 100644
--- a/tools/mtmd/mtmd-image.h
+++ b/tools/mtmd/mtmd-image.h
@@ -166,6 +166,15 @@ private:
static void safe_resize(int height, int width, int & best_height, int & best_width, int p, int r, int max_n_token);
};
+// ref: inference/image_processor.py in the HF repo (DeepSeek-V4.1-Flash): plan_image_grid / load_image
+struct mtmd_image_preprocessor_deepseek41v : mtmd_image_preprocessor {
+ mtmd_image_preprocessor_deepseek41v(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
+ mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
+ // target pixel size (multiples of patch_size) for an image of the given size, exposed for tests
+ static void plan_grid(int width, int height, int p, int r, int max_n_token, int min_pixels,
+ int & best_width, int & best_height);
+};
+
// custom llava-uhd slicing logic for MiniCPM-V
struct mtmd_image_preprocessor_minicpmv : mtmd_image_preprocessor_llava_uhd {
using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd;
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index e18f639de..df32b8f36 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -903,6 +903,11 @@ struct mtmd_context {
// no vocab tokens are added; the start/end/newline markers are learned embeddings emitted by the encoder
image_preproc = std::make_unique<mtmd_image_preprocessor_deepseek4v>(ctx_v);
} break;
+ case PROJECTOR_TYPE_DEEPSEEK41V:
+ {
+ // as deepseek4v: markers come from the encoder, the block is decoded causally
+ image_preproc = std::make_unique<mtmd_image_preprocessor_deepseek41v>(ctx_v);
+ } break;
case PROJECTOR_TYPE_DOTS_OCR:
case PROJECTOR_TYPE_DOTS3NOTE_V:
{