Commit 3ad1ba733 for llama.cpp
commit 3ad1ba7336986d98592d3e28cafd1a406715351f
Author: KnightYao <KnightYao@users.noreply.github.com>
Date: Sun Sep 6 23:43:58 2026 +0800
[Model] Support for Spark2_5ForCausalLM implementation (#27868)
* Add Spark3 Model
* rename spark3 -> spark2_5
Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>
Co-authored-by: dongjiang <dongjiang2010@gmail.com>
diff --git a/conversion/__init__.py b/conversion/__init__.py
index 94d6a49fb..4d58bcd10 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -255,6 +255,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"SeedOssForCausalLM": "olmo",
"SmallThinkerForCausalLM": "smallthinker",
"SmolLM3ForCausalLM": "llama",
+ "Spark2_5ForCausalLM": "spark2_5",
"SolarOpenForCausalLM": "glm",
"StableLMEpochForCausalLM": "stablelm",
"StableLmForCausalLM": "stablelm",
diff --git a/conversion/base.py b/conversion/base.py
index c1ecf1c65..dc1083ead 100644
--- a/conversion/base.py
+++ b/conversion/base.py
@@ -1543,6 +1543,9 @@ class TextModel(ModelBase):
if chkhsh == "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7":
# ref: https://huggingface.co/LiquidAI/LFM2.5-8B-A1B
res = "lfm2"
+ if chkhsh == "0a766d034107bc736a3f2dc4968fd62e54a3570f1454443e0c5a4cc6bd7941ed":
+ # ref: https://huggingface.co/XHToken/Spark-X2.5-1.7B
+ res = "spark2_5"
if chkhsh == "0ef9807a4087ebef797fc749390439009c3b9eda9ad1a097abbe738f486c01e5":
# ref: https://huggingface.co/meta-llama/Meta-Llama-3-8B
res = "llama-bpe"
diff --git a/conversion/spark2_5.py b/conversion/spark2_5.py
new file mode 100644
index 000000000..44a0bd262
--- /dev/null
+++ b/conversion/spark2_5.py
@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from torch import Tensor
+
+from .base import ModelBase, TextModel, gguf
+
+
+@ModelBase.register("Spark2_5ForCausalLM")
+@ModelBase.example("XHToken/Spark-X2.5-1.7B")
+class Spark2_5Model(TextModel):
+ model_arch = gguf.MODEL_ARCH.SPARK2_5
+
+ def set_gguf_parameters(self) -> None:
+ super().set_gguf_parameters()
+
+ hparams = self.hparams
+ layer_types = hparams["layer_types"]
+ if len(layer_types) != self.block_count:
+ raise ValueError(
+ f"Spark2_5 layer_types length {len(layer_types)} != num_hidden_layers {self.block_count}"
+ )
+ if any(layer_type not in ("sliding_attention", "full_attention") for layer_type in layer_types):
+ raise ValueError(f"Spark2_5 has unsupported layer_types: {layer_types}")
+ if hparams.get("gate_attn_act_mode") != "sigmoid" or hparams.get("headwise_attn_output_gate") is not True:
+ raise ValueError("Spark2_5 conversion requires head-wise sigmoid attention gates")
+ if hparams.get("hidden_act") != "gelu":
+ raise ValueError(f"Spark2_5 conversion requires GELU, got {hparams.get('hidden_act')!r}")
+
+ self.gguf_writer.add_vocab_size(hparams["vocab_size"])
+ self.gguf_writer.add_sliding_window(hparams["sliding_window"])
+ self.gguf_writer.add_sliding_window_pattern(
+ [layer_type == "sliding_attention" for layer_type in layer_types]
+ )
+
+ head_dim = hparams["head_dim"]
+ full_rope = self.rope_parameters["full_attention"]
+ swa_rope = self.rope_parameters["sliding_attention"]
+ self.gguf_writer.add_rope_dimension_count(
+ int(head_dim * float(full_rope["partial_rotary_factor"]))
+ )
+ self.gguf_writer.add_rope_dimension_count_swa(
+ int(head_dim * float(swa_rope["partial_rotary_factor"]))
+ )
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if name.endswith(".self_attn.q_k_v_proj.weight"):
+ if bid is None:
+ raise ValueError(f"Spark2_5 fused QKV tensor has no block id: {name}")
+ yield self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_QKV, bid), data_torch
+ return
+
+ if name.endswith(".self_attn.g_proj.weight"):
+ if bid is None:
+ raise ValueError(f"Spark2_5 attention gate tensor has no block id: {name}")
+ expected = self.hparams["num_attention_heads"]
+ if data_torch.shape[0] != expected:
+ raise ValueError(
+ f"Spark2_5 layer {bid} attention gate width {data_torch.shape[0]} != head count {expected}"
+ )
+
+ yield from super().modify_tensors(data_torch, name, bid)
diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py
index c4141afa6..6af74cd87 100755
--- a/convert_hf_to_gguf_update.py
+++ b/convert_hf_to_gguf_update.py
@@ -191,6 +191,7 @@ pre_computed_hashes = [
{"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/evilfreelancer/ruGPT3XL", "chkhsh": "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4"},
# lfm2 variants
{"name": "lfm2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B", "chkhsh": "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7"},
+ {"name": "spark2_5", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/XHToken/Spark-X2.5-1.7B", "chkhsh": "0a766d034107bc736a3f2dc4968fd62e54a3570f1454443e0c5a4cc6bd7941ed"},
]
diff --git a/docs/autoparser.md b/docs/autoparser.md
index b5e32621d..2a7ea00b4 100644
--- a/docs/autoparser.md
+++ b/docs/autoparser.md
@@ -514,6 +514,7 @@ The following templates have active tests in `tests/test-chat.cpp`:
| Mistral Small 3.2 | JSON_NATIVE | `[TOOL_CALLS]func[ARGS]{...}` with call ID |
| Devstral | JSON_NATIVE | `[TOOL_CALLS]func[ARGS]{...}` without call ID |
| StepFun 3.5 Flash | TAG_WITH_TAGGED | `<function=X><parameter=Y>` format |
+| Spark2.5 | TAG_WITH_TAGGED | `<tool_call>name<arg_key>...<arg_value>...` format |
## Adding Support for New Templates
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index 399d31f1d..486f3586d 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -619,6 +619,7 @@ class MODEL_ARCH(IntEnum):
PADDLEOCR = auto()
MIMO2 = auto()
STEP35 = auto()
+ SPARK2_5 = auto()
LLAMA_EMBED = auto()
MAINCODER = auto()
KIMI_LINEAR = auto()
@@ -1373,6 +1374,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
MODEL_ARCH.PADDLEOCR: "paddleocr",
MODEL_ARCH.MIMO2: "mimo2",
MODEL_ARCH.STEP35: "step35",
+ MODEL_ARCH.SPARK2_5: "spark2_5",
MODEL_ARCH.LLAMA_EMBED: "llama-embed",
MODEL_ARCH.MAINCODER: "maincoder",
MODEL_ARCH.KIMI_LINEAR: "kimi-linear",
@@ -5231,6 +5233,19 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
],
+ MODEL_ARCH.SPARK2_5: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_GATE,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE,
+ MODEL_TENSOR.FFN_DOWN,
+ MODEL_TENSOR.FFN_UP,
+ ],
MODEL_ARCH.LLAMA_EMBED: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT_NORM,
diff --git a/models/templates/README.md b/models/templates/README.md
index 3a649b8f4..022a5e278 100644
--- a/models/templates/README.md
+++ b/models/templates/README.md
@@ -23,4 +23,6 @@ These templates can be updated with the following commands:
./scripts/get_chat_template.py Qwen/Qwen3-0.6B > models/templates/Qwen-Qwen3-0.6B.jinja
./scripts/get_chat_template.py zai-org/GLM-4.5 > models/templates/zai-org-GLM-4.5.jinja
./scripts/get_chat_template.py deepseek-ai/DeepSeek-V3.1 > models/templates/deepseek-ai-DeepSeek-V3.1.jinja
+./scripts/get_chat_template.py XHToken/Spark-X2.5-1.7B > models/templates/Spark2.5.jinja
+./scripts/get_chat_template.py XHToken/Spark-X2.5-4B > models/templates/Spark2.5.jinja
```
diff --git a/models/templates/Spark2.5.jinja b/models/templates/Spark2.5.jinja
new file mode 100644
index 000000000..54aa34ff2
--- /dev/null
+++ b/models/templates/Spark2.5.jinja
@@ -0,0 +1,110 @@
+{%- if not messages %}
+ {{- raise_exception('No messages provided.') }}
+{%- endif %}
+
+{%- set enable_thinking = enable_thinking | default(true) %}
+
+{#- Render a string or a list of text blocks. -#}
+{%- macro render_content(content, context_name) %}
+ {%- if content is string %}
+ {{- content }}
+ {%- elif content is none or content is undefined %}
+ {{- '' }}
+ {%- elif content is iterable and content is not mapping %}
+ {%- for block in content %}
+ {%- if block.type == 'text' %}
+ {{- block.text }}
+ {%- else %}
+ {{- raise_exception('Unsupported ' ~ context_name ~ ' content block type: ' ~ (block.type | string)) }}
+ {%- endif %}
+ {%- endfor %}
+ {%- else %}
+ {{- raise_exception(context_name ~ ' content must be a string or a list of text blocks') }}
+ {%- endif %}
+{%- endmacro %}
+
+{#- Default system prompt. -#}
+{%- set default_system = 'you are a helpful assistant.' %}
+
+{#- The first message-level system is placed in the initial system block. -#}
+{%- set ns = namespace(initial_system='') %}
+{%- if messages[0].role == 'system' %}
+ {%- set ns.initial_system = render_content(messages[0].content, 'system') %}
+{%- endif %}
+
+{#- System block. -#}
+{{- '<｜start▁of▁sentence｜><|System|>' + '\n' + default_system }}
+{%- if tools %}
+ {{- '## Tools' + '\n' + 'You have access to the following functions:' + '\n' + '<tools>' }}
+ {%- for tool in tools %}
+ {{- '\n' + tool.function | tojson }}
+ {%- endfor %}
+ {{- '\n' + '</tools>' }}
+{%- endif %}
+{%- if ns.initial_system %}
+ {{- '\n\n' + ns.initial_system }}
+{%- endif %}
+{{- '<｜end▁of▁sentence｜>' }}
+
+{#- Conversation turns. -#}
+{%- for message in messages %}
+ {%- if message.role == 'system' %}
+ {#- The first system message was consumed by the initial block. -#}
+ {%- if not loop.first %}
+ {{- '<｜start▁of▁sentence｜><|System|>\n' + render_content(message.content, 'system') + '<｜end▁of▁sentence｜>' }}
+ {%- endif %}
+ {%- elif message.role == 'user' %}
+ {{- '<｜start▁of▁sentence｜><|User|>' + render_content(message.content, 'user') + '<｜end▁of▁sentence｜>' }}
+ {%- elif message.role == 'assistant' %}
+ {%- set assistant_content = render_content(message.content, 'assistant') %}
+ {%- if message.reasoning_content is defined and message.reasoning_content %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- set reasoning_content = '' %}
+ {%- endif %}
+ {{- '<｜start▁of▁sentence｜><|Bot|>' }}
+ {%- if reasoning_content %}
+ {{- '<think>' + reasoning_content + '</think>' }}
+ {%- else %}
+ {{- '</think>' }}
+ {%- endif %}
+ {%- if assistant_content %}
+ {{- assistant_content }}
+ {%- endif %}
+ {%- if message.tool_calls is defined and message.tool_calls is not none %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if tool_call.function.arguments is not mapping %}
+ {{- raise_exception('tool_call.function.arguments must be a dictionary; normalize JSON strings before apply_chat_template') }}
+ {%- endif %}
+ {%- set args = tool_call.function.arguments %}
+ {{- '<tool_call>' + tool_call.function.name }}
+ {%- for k, v in args.items() %}
+ {{- '<arg_key>' ~ k ~ '</arg_key><arg_value>' ~ (v if v is string else v | tojson) ~ '</arg_value>' }}
+ {%- endfor %}
+ {{- '</tool_call>' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<｜end▁of▁sentence｜>' }}
+ {%- elif message.role == 'tool' %}
+ {%- if loop.previtem is undefined or loop.previtem.role != 'tool' %}
+ {{- '<｜start▁of▁sentence｜><|Tool|>' }}
+ {%- endif %}
+ {{- '<tool_response>' ~ message.content ~ '</tool_response>' }}
+ {%- if loop.nextitem is undefined or loop.nextitem.role != 'tool' %}
+ {{- '<｜end▁of▁sentence｜>' }}
+ {%- endif %}
+ {%- else %}
+ {{- raise_exception('Unsupported message role: ' ~ message.role) }}
+ {%- endif %}
+{%- endfor %}
+
+{#- Generation prompt. -#}
+{%- if add_generation_prompt %}
+ {{- '<｜start▁of▁sentence｜><|Bot|>' }}
+ {%- if enable_thinking is defined and enable_thinking %}
+ {{- '<think>' }}
+ {%- endif %}
+ {%- if enable_thinking is defined and not enable_thinking %}
+ {{- '</think>' }}
+ {%- endif %}
+{%- endif %}
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index d06be641a..15f651919 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -146,6 +146,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
{ LLM_ARCH_PADDLEOCR, "paddleocr" },
{ LLM_ARCH_MIMO2, "mimo2" },
{ LLM_ARCH_STEP35, "step35" },
+ { LLM_ARCH_SPARK2_5, "spark2_5" },
{ LLM_ARCH_LLAMA_EMBED, "llama-embed" },
{ LLM_ARCH_MAINCODER, "maincoder" },
{ LLM_ARCH_KIMI_LINEAR, "kimi-linear" },
diff --git a/src/llama-arch.h b/src/llama-arch.h
index 62dfa5d81..f1d173a57 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -147,6 +147,7 @@ enum llm_arch {
LLM_ARCH_PADDLEOCR,
LLM_ARCH_MIMO2,
LLM_ARCH_STEP35,
+ LLM_ARCH_SPARK2_5,
LLM_ARCH_LLAMA_EMBED,
LLM_ARCH_MAINCODER,
LLM_ARCH_KIMI_LINEAR,
diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp
index df2a46d93..66f8bdec3 100644
--- a/src/llama-model-saver.cpp
+++ b/src/llama-model-saver.cpp
@@ -27,6 +27,7 @@ bool llama_model_saver_supports_arch(llm_arch arch) {
case LLM_ARCH_APERTUS:
case LLM_ARCH_MIMO2:
case LLM_ARCH_STEP35:
+ case LLM_ARCH_SPARK2_5:
case LLM_ARCH_MUSE_GLIMMER:
case LLM_ARCH_MELLUM:
case LLM_ARCH_LAGUNA:
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index b837e2765..0e0036781 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -338,6 +338,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_kimi_k3(params);
case LLM_ARCH_STEP35:
return new llama_model_step35(params);
+ case LLM_ARCH_SPARK2_5:
+ return new llama_model_spark2_5(params);
default:
throw std::runtime_error(std::string("unsupported model architecture: '") + llm_arch_name(arch) + "'");
}
@@ -2999,6 +3001,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_QWEN3NEXT:
case LLM_ARCH_MIMO2:
case LLM_ARCH_STEP35:
+ case LLM_ARCH_SPARK2_5:
case LLM_ARCH_TALKIE:
case LLM_ARCH_MELLUM:
return LLAMA_ROPE_TYPE_NEOX;
diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
index c0c34cdd8..a69801f08 100644
--- a/src/llama-vocab.cpp
+++ b/src/llama-vocab.cpp
@@ -325,6 +325,14 @@ struct llm_tokenizer_bpe : llm_tokenizer {
"[!\"#$%&'()*+,\\-./:;<=>?@\\[\\\\\\]^_`{|}~][A-Za-z]+|[^\r\n\\p{L}\\p{P}\\p{S}]?[\\p{L}\\p{M}]+| ?[\\p{P}\\p{S}]+[\r\n]*|\\s*[\r\n]+|\\s+(?!\\S)|\\s+",
};
break;
+ case LLAMA_VOCAB_PRE_TYPE_SPARK2_5:
+ regex_exprs = {
+ "\\p{N}{1,3}",
+ "[一-龥-ゟ゠-ヿ]+",
+ "[!\"#$%&'()*+,\\-./:;<=>?@\\[\\\\\\]^_`{|}~][A-Za-z]+|[^\r\n\\p{L}\\p{P}\\p{S}]?[\\p{L}\\p{M}]+| ?[\\p{P}\\p{S}]+|[\r\n]|\\s+(?!\\S)|\\s+",
+ "\\p{N}",
+ };
+ break;
case LLAMA_VOCAB_PRE_TYPE_YOUTU:
regex_exprs = {
"[가-힣ㄱ-ㆎ]+|[！…“”‘’—：；，、-〿︰-﹏]+|[ㄅ-ㄯ]+|[一-龥-ゟ゠-ヿ]+",
@@ -2170,6 +2178,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
tokenizer_pre == "deepseek-v3") {
pre_type = LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM;
clean_spaces = false;
+ } else if (
+ tokenizer_pre == "spark2_5") {
+ pre_type = LLAMA_VOCAB_PRE_TYPE_SPARK2_5;
+ clean_spaces = false;
} else if (
tokenizer_pre == "youtu") {
pre_type = LLAMA_VOCAB_PRE_TYPE_YOUTU;
diff --git a/src/llama-vocab.h b/src/llama-vocab.h
index e02ea78ff..65293c026 100644
--- a/src/llama-vocab.h
+++ b/src/llama-vocab.h
@@ -66,6 +66,7 @@ enum llama_vocab_pre_type {
LLAMA_VOCAB_PRE_TYPE_MELLUM2 = 55,
LLAMA_VOCAB_PRE_TYPE_LAGUNA = 56,
LLAMA_VOCAB_PRE_TYPE_HY_V4 = 57,
+ LLAMA_VOCAB_PRE_TYPE_SPARK2_5 = 58,
};
struct LLM_KV;
diff --git a/src/models/models.h b/src/models/models.h
index 93a6b3494..50e9a235c 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -2606,3 +2606,16 @@ struct llama_model_step35 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
+
+
+struct llama_model_spark2_5 : public llama_model_base {
+ llama_model_spark2_5(const struct llama_model_params & params) : llama_model_base(params) {}
+ void load_arch_hparams(llama_model_loader & ml) override;
+ void load_arch_tensors(llama_model_loader & ml) override;
+
+ struct graph : public llm_graph_context {
+ graph(const llama_model & model, const llm_graph_params & params);
+ };
+
+ std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
diff --git a/src/models/spark2-5.cpp b/src/models/spark2-5.cpp
new file mode 100644
index 000000000..107448777
--- /dev/null
+++ b/src/models/spark2-5.cpp
@@ -0,0 +1,146 @@
+#include "models.h"
+
+void llama_model_spark2_5::load_arch_hparams(llama_model_loader & ml) {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+ ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
+
+ hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
+ ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);
+
+ hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;
+ hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;
+ ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);
+
+ switch (hparams.n_layer()) {
+ case 28: type = LLM_TYPE_1_7B; break;
+ default: type = LLM_TYPE_UNKNOWN;
+ }
+}
+
+void llama_model_spark2_5::load_arch_tensors(llama_model_loader &) {
+ LLAMA_LOAD_LOCALS;
+
+ tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+
+ output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+ output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
+ if (output == nullptr) {
+ output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ auto & layer = layers[i];
+
+ const int64_t n_head_i = hparams.n_head(i);
+ const int64_t n_head_kv_i = hparams.n_head_kv(i);
+ const int64_t n_embd_q = hparams.n_embd_head_k(i) * n_head_i;
+ const int64_t n_embd_k = hparams.n_embd_head_k(i) * n_head_kv_i;
+ const int64_t n_embd_v = hparams.n_embd_head_v(i) * n_head_kv_i;
+
+ layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
+ create_tensor_qkv(layer, i, n_embd, n_embd_q, n_embd_k, n_embd_v, 0);
+ layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_head_i}, 0);
+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_q, n_embd}, 0);
+
+ layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
+ layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);
+ layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);
+ layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0);
+ }
+}
+
+std::unique_ptr<llm_graph_context> llama_model_spark2_5::build_arch_graph(const llm_graph_params & params) const {
+ return std::make_unique<graph>(*this, params);
+}
+
+llama_model_spark2_5::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
+ const int64_t n_embd_head = hparams.n_embd_head_v();
+
+ GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
+ GGML_ASSERT(hparams.swa_type == LLAMA_SWA_TYPE_STANDARD);
+
+ ggml_tensor * inpL = build_inp_embd(model.tok_embd);
+ ggml_tensor * inp_pos = build_inp_pos();
+ auto * inp_attn = build_attn_inp_kv_iswa();
+ ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+ const float kq_scale = 1.0f / sqrtf(float(n_embd_head));
+
+ for (int il = 0; il < n_layer; ++il) {
+ ggml_tensor * inpSA = inpL;
+ ggml_tensor * cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il);
+ cb(cur, "attn_norm", il);
+
+ const int64_t n_head_i = hparams.n_head(il);
+ const int64_t n_head_kv_i = hparams.n_head_kv(il);
+ const int64_t n_rot_i = hparams.n_rot(il);
+ const float freq_base_i = model.get_rope_freq_base(cparams, il);
+ const float freq_scale_i = model.get_rope_freq_scale(cparams, il);
+
+ ggml_tensor * attn_inp = cur;
+ auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, n_head_i, n_head_kv_i, il);
+
+ Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr,
+ n_rot_i, rope_type, n_ctx_orig, freq_base_i, freq_scale_i,
+ ext_factor, attn_factor, beta_fast, beta_slow);
+ Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr,
+ n_rot_i, rope_type, n_ctx_orig, freq_base_i, freq_scale_i,
+ ext_factor, attn_factor, beta_fast, beta_slow);
+ cb(Qcur, "Qcur_rope", il);
+ cb(Kcur, "Kcur_rope", il);
+
+ cur = build_attn(inp_attn,
+ nullptr, nullptr, nullptr,
+ Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);
+ cb(cur, "attn_out", il);
+
+ ggml_tensor * gate = build_lora_mm(model.layers[il].wqkv_gate, attn_inp);
+ gate = ggml_sigmoid(ctx0, gate);
+ cb(gate, "attn_gate", il);
+
+ const int64_t n_tokens_i = cur->ne[1];
+ cur = ggml_reshape_3d(ctx0, cur, n_embd_head, n_head_i, n_tokens_i);
+ gate = ggml_reshape_3d(ctx0, gate, 1, n_head_i, n_tokens_i);
+ cur = ggml_mul(ctx0, cur, gate);
+ cur = ggml_reshape_2d(ctx0, cur, n_embd_head * n_head_i, n_tokens_i);
+ cb(cur, "attn_gated", il);
+
+ cur = build_lora_mm(model.layers[il].wo, cur, model.layers[il].wo_s);
+ cb(cur, "attn_out_proj", il);
+
+ if (il == n_layer - 1 && inp_out_ids) {
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
+ }
+
+ ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
+ cb(ffn_inp, "ffn_inp", il);
+
+ cur = build_norm(ffn_inp, model.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il);
+ cb(cur, "ffn_norm", il);
+
+ cur = build_ffn(cur,
+ model.layers[il].ffn_up, nullptr, nullptr,
+ model.layers[il].ffn_gate, nullptr, nullptr,
+ model.layers[il].ffn_down, nullptr, nullptr,
+ nullptr,
+ LLM_FFN_GELU, LLM_FFN_PAR, il);
+ cb(cur, "ffn_out", il);
+
+ cur = ggml_add(ctx0, cur, ffn_inp);
+ cur = build_cvec(cur, il);
+ cb(cur, "l_out", il);
+
+ inpL = cur;
+ }
+
+ ggml_tensor * cur = build_norm(inpL, model.output_norm, nullptr, LLM_NORM_RMS, -1);
+ cb(cur, "result_norm", -1);
+ res->t_embd = cur;
+
+ cur = build_lora_mm(model.output, cur);
+ cb(cur, "result_output", -1);
+ res->t_logits = cur;
+
+ ggml_build_forward_expand(gf, cur);
+}
diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp
index 7918f0ffc..f27c91e4d 100644
--- a/tests/test-chat.cpp
+++ b/tests/test-chat.cpp
@@ -4405,6 +4405,100 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
.run();
}
+ // Spark2.5 uses tagged arguments with forced-open thinking.
+ {
+ auto tst = peg_tester("models/templates/Spark2.5.jinja", detailed_debug);
+
+ tst.test("Hello, world!\nWhat's up?")
+ .enable_thinking(false)
+ .expect(message_assist)
+ .expect_reconstruction()
+ .run();
+
+ tst.test("I'm\nthinking</think>Hello, world!\nWhat's up?")
+ .enable_thinking(true)
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .expect(message_assist_thoughts)
+ .expect_reconstruction()
+ .run();
+
+ tst.test(
+ "<tool_call>special_function"
+ "<arg_key>arg1</arg_key><arg_value>1</arg_value>"
+ "</tool_call>")
+ .enable_thinking(false)
+ .tools({ special_function_tool })
+ .expect(message_assist_call)
+ .expect_reconstruction()
+ .run();
+
+ tst.test(
+ "I'm\nthinking</think>"
+ "<tool_call>special_function"
+ "<arg_key>arg1</arg_key><arg_value>1</arg_value>"
+ "</tool_call>")
+ .enable_thinking(true)
+ .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK)
+ .tools({ special_function_tool })
+ .expect(message_assist_call_thoughts)
+ .expect_reconstruction()
+ .run();
+
+ tst.test(
+ "<tool_call>special_function"
+ "<arg_key>arg1</arg_key><arg_value>1</arg_value>"
+ "</tool_call>"
+ "<tool_call>special_function_with_opt"
+ "<arg_key>arg1</arg_key><arg_value>1</arg_value>"
+ "<arg_key>arg2</arg_key><arg_value>2</arg_value>"
+ "</tool_call>")
+ .enable_thinking(false)
+ .parallel_tool_calls(true)
+ .tools({ special_function_tool, special_function_tool_with_optional_param })
+ .expect_tool_calls({
+ { "special_function", R"({"arg1": 1})", {} },
+ { "special_function_with_opt", R"({"arg1": 1, "arg2": 2})", {} },
+ })
+ .expect_reconstruction()
+ .run();
+
+ tst.test(
+ "Preparing updates."
+ "<tool_call>magic_int"
+ "<arg_key>ref</arg_key><arg_value>42</arg_value>"
+ "<arg_key>name</arg_key><arg_value>上海</arg_value>"
+ "</tool_call>"
+ "<tool_call>amount"
+ "<arg_key>orig</arg_key><arg_value>2.5</arg_value>"
+ "</tool_call>"
+ "<tool_call>toggle"
+ "<arg_key>enabled</arg_key><arg_value>true</arg_value>"
+ "</tool_call>"
+ "<tool_call>set_config"
+ "<arg_key>config</arg_key><arg_value>{\"source\": \"spark\", \"options\": {\"strict\": true}}</arg_value>"
+ "</tool_call>"
+ "<tool_call>nested_args"
+ "<arg_key>tags</arg_key><arg_value>[\"alpha\", \"测试\"]</arg_value>"
+ "<arg_key>entries</arg_key><arg_value>[{\"id\": 1, \"label\": \"first\"}, {\"id\": 2, \"label\": \"第二\"}]</arg_value>"
+ "</tool_call>"
+ "<tool_call>empty_args"
+ "</tool_call>")
+ .enable_thinking(false)
+ .parallel_tool_calls(true)
+ .tools({ magic_int_tool, amount_tool, toggle_tool, config_tool, nested_args_tool, empty_args_tool })
+ .expect_content("Preparing updates.")
+ .expect_tool_calls({
+ { "magic_int", R"({"ref": 42, "name": "上海"})", {} },
+ { "amount", R"({"orig": 2.5})", {} },
+ { "toggle", R"({"enabled": true})", {} },
+ { "set_config", R"({"config": {"source": "spark", "options": {"strict": true}}})", {} },
+ { "nested_args", R"({"tags": ["alpha", "测试"], "entries": [{"id": 1, "label": "first"}, {"id": 2, "label": "第二"}]})", {} },
+ { "empty_args", "{}", {} },
+ })
+ .expect_reconstruction()
+ .run();
+ }
+
// Verify the throw path produces a readable error message, not std::out_of_range.
// #20424 introduced effective_input = generation_prompt + input, but the throw
// uses input.substr(result.end) where result.end is in effective_input space.
diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp
index 0f3d1c79a..dbed9846f 100644
--- a/tests/test-llama-archs.cpp
+++ b/tests/test-llama-archs.cpp
@@ -237,7 +237,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, 10000.0f);
// SWA pattern: every 5th layer is full attention (matches E2B layer_types)
ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(5));
- } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 ||
+ } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_SPARK2_5 ||
arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_GRANITE_SWA || arch == LLM_ARCH_DOTS3NOTE) {
std::vector<uint32_t> pattern;
pattern.reserve(n_layer);