Commit 98d67c8d5 for llama.cpp
commit 98d67c8d536dc584897c4adb2510509796517222
Author: varun <varun10221@gmail.com>
Date: Wed Nov 18 06:46:49 2026 -0800
model:add Zaya1 8B (Zyphra) (#32071)
* model:add Zaya1 8B (Zyphra)
- Added the support for Zaya.
- Fixed the SSM_CONV case, so the model runs on cuda without abort.
Please enter the commit message for your changes. Lines starting
* Made the following changes based on the bot reviewer feedback:
1) Added runtime asserts when loading wrong KV head size, unsupported conv kernel size.
2) Removed the static cast usage.
3) Fixed double load of tokenizer.
4) Updated res_scale_in to be input tensor.
* - Moved the res_scale_in back to layer 0 to fix the cuda bug
* i) Removed the EOS addition.Team agreed that updating Tokenizer is the right move.
ii) Fixed formatting
* Fixed the comment to make it clear
---------
Co-authored-by: varun <varun_subramanian@zyphra.com>
diff --git a/conversion/__init__.py b/conversion/__init__.py
index 881111e9f..634cafa24 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -340,6 +340,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"XverseForCausalLM": "xverse",
"YoutuForCausalLM": "deepseek",
"YoutuVLForConditionalGeneration": "deepseek",
+ "ZayaForCausalLM": "zaya1",
"fish_qwen3_omni": "fish_s2",
"modeling_grove_moe.GroveMoeForCausalLM": "grovemoe",
"modeling_sarvam_moe.SarvamMoEForCausalLM": "bailingmoe",
diff --git a/conversion/zaya1.py b/conversion/zaya1.py
new file mode 100644
index 000000000..803fc908a
--- /dev/null
+++ b/conversion/zaya1.py
@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+import json
+
+import torch
+
+from typing import Callable, Iterable, TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from torch import Tensor
+
+from .base import ModelBase, TextModel, gguf
+
+
+@ModelBase.register("ZayaForCausalLM")
+@ModelBase.example("Zyphra/ZAYA1-8B")
+class ZayaModel(TextModel):
+ model_arch = gguf.MODEL_ARCH.ZAYA1
+
+ # checkpoint names without a .weight/.bias suffix
+ _RENAME = {
+ "model.input_hidden_states_scale": "model.input_hidden_states.weight",
+ "model.input_hidden_states_bias": "model.input_hidden_states.bias",
+ }
+
+ # the two kv heads of attn_v: head 0 = current token, head 1 = previous token
+ _V_PARTS = (
+ "self_attn.qkv_proj.v_proj_current.weight",
+ "self_attn.qkv_proj.v_proj_delayed.weight",
+ )
+
+ # (hs + b_hs) * s_hs + (res + b_res) * s_res = hs * s_hs + res * s_res + (b_hs * s_hs + b_res * s_res)
+ _RES_SCALES = {
+ "post_attention_residual_scale.": (gguf.MODEL_TENSOR.ATTN_RES_SCALE_HS, gguf.MODEL_TENSOR.ATTN_RESID_SCALE),
+ "post_mlp_residual_scale.": (gguf.MODEL_TENSOR.FFN_RES_SCALE_HS, gguf.MODEL_TENSOR.FFN_RESID_SCALE),
+ }
+ _RES_PARTS = ("hidden_states_scale", "hidden_states_bias", "residual_scale", "residual_bias")
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ # added tokens overlap the base vocab, so count the token list itself
+ self._vocab = gguf.LlamaHfVocab(self.dir_model)
+ self._n_vocab = self._vocab.vocab_size
+ self._parts: dict[tuple[int, str], dict[str, Tensor]] = {}
+ self.rope_parameters = self.hparams["rope_parameters"]["hybrid"]
+
+ def set_vocab(self):
+ # added special tokens overlap the base vocab, so mark them as control here
+ with open(self.dir_model / "tokenizer.json", "r", encoding="utf-8") as f:
+ added = {t["id"]: t["special"] for t in json.load(f)["added_tokens"]}
+
+ tokens, scores, toktypes = [], [], []
+ for i, (text, score, toktype) in enumerate(self._vocab.all_tokens()):
+ tokens.append(text)
+ scores.append(score)
+ if added.get(i):
+ toktype = gguf.TokenType.CONTROL
+ toktypes.append(toktype)
+
+ self.gguf_writer.add_tokenizer_model("gemma4")
+ self.gguf_writer.add_token_list(tokens)
+ self.gguf_writer.add_token_scores(scores)
+ self.gguf_writer.add_token_types(toktypes)
+
+ special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
+ special_vocab.add_to_gguf(self.gguf_writer)
+ self.gguf_writer.add_add_space_prefix(False)
+
+ def set_gguf_parameters(self):
+ hparams = self.hparams
+ super().set_gguf_parameters()
+
+ self.gguf_writer.add_rope_dimension_count(int(self.rope_parameters["partial_rotary_factor"] * hparams["head_dim"]))
+ self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"])
+ self.gguf_writer.add_ssm_conv_kernel(hparams["cca_time0"])
+ self.gguf_writer.add_router_hidden_size(hparams["router_hidden_size"])
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ name, _ = item
+ # skip the draft head tensors now, they will be added as part of the TiDAR changes
+ if name.startswith(("diffusion_output_layer.", "diffusion_markov_head.")):
+ return None
+
+ return super().filter_tensors(item)
+
+ def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
+ # the router picks one expert, so its matmuls stay F32 like ffn_gate_inp
+ if any(self.match_model_tensor_name(new_name, key, bid) for key in (
+ gguf.MODEL_TENSOR.CCA_CONV_GRP,
+ gguf.MODEL_TENSOR.ROUTER_FC1,
+ gguf.MODEL_TENSOR.ROUTER_FC2,
+ gguf.MODEL_TENSOR.ROUTER_OUT)):
+ return gguf.GGMLQuantizationType.F32
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if bid is not None:
+ local = name.removeprefix(f"model.layers.{bid}.")
+ if local in self._V_PARTS:
+ p = self._parts.setdefault((bid, "v"), {})
+ p[local] = data_torch
+ if len(p) == len(self._V_PARTS):
+ del self._parts[(bid, "v")]
+ v = torch.cat([p[n] for n in self._V_PARTS], dim=0)
+ yield from super().modify_tensors(v, f"model.layers.{bid}.{self._V_PARTS[0]}", bid)
+ return
+ for prefix, (key_hs, key_res) in self._RES_SCALES.items():
+ if local.startswith(prefix) and local[len(prefix):] in self._RES_PARTS:
+ p = self._parts.setdefault((bid, prefix), {})
+ p[local[len(prefix):]] = data_torch
+ if len(p) == len(self._RES_PARTS):
+ del self._parts[(bid, prefix)]
+ s_hs, b_hs, s_res, b_res = (p[n].float() for n in self._RES_PARTS)
+ yield self.format_tensor_name(key_hs, bid, ".weight"), s_hs
+ yield self.format_tensor_name(key_res, bid, ".weight"), s_res
+ yield self.format_tensor_name(key_res, bid, ".bias"), b_hs * s_hs + b_res * s_res
+ return
+
+ name = self._RENAME.get(name, name)
+ if name.endswith("balancing_biases"):
+ name += ".bias"
+ elif not name.endswith((".weight", ".bias")):
+ name += ".weight"
+ new_name = self.map_tensor_name(name)
+
+ def is_tensor(key: gguf.MODEL_TENSOR) -> bool:
+ return self.match_model_tensor_name(new_name, key, bid)
+
+ if any(is_tensor(key) for key in (gguf.MODEL_TENSOR.ATTN_Q, gguf.MODEL_TENSOR.ATTN_K)):
+ # through the base class, which fuses q, k and v with --fuse-qkv
+ yield from super().modify_tensors(data_torch, name, bid)
+ return
+ if is_tensor(gguf.MODEL_TENSOR.TOKEN_EMBD):
+ data_torch = data_torch[:self._n_vocab]
+ elif is_tensor(gguf.MODEL_TENSOR.SSM_CONV1D):
+ data_torch = data_torch.squeeze(1)
+ elif is_tensor(gguf.MODEL_TENSOR.CCA_CONV_GRP):
+ # [n_group*out, in, kernel] -> [kernel, n_group, out, in]
+ n_group = self.hparams["num_attention_heads"] + self.hparams["num_key_value_heads"]
+ n_out, n_in, n_k = data_torch.shape
+ data_torch = data_torch.reshape(n_group, n_out // n_group, n_in, n_k).permute(3, 0, 1, 2).contiguous()
+
+ yield new_name, data_torch
diff --git a/ggml/src/ggml-cuda/ssm-conv.cu b/ggml/src/ggml-cuda/ssm-conv.cu
index f41438b31..0e77a3b22 100644
--- a/ggml/src/ggml-cuda/ssm-conv.cu
+++ b/ggml/src/ggml-cuda/ssm-conv.cu
@@ -148,13 +148,14 @@ static void ssm_conv_f32_cuda(const float * src0, const float * src1, const floa
};
switch (nc) {
+ case 2: launch_kernel(std::integral_constant<int, 2 >{}); break;
case 3: launch_kernel(std::integral_constant<int, 3 >{}); break;
case 4: launch_kernel(std::integral_constant<int, 4 >{}); break;
case 5: launch_kernel(std::integral_constant<int, 5 >{}); break;
case 7: launch_kernel(std::integral_constant<int, 7 >{}); break;
case 9: launch_kernel(std::integral_constant<int, 9 >{}); break;
case 15: launch_kernel(std::integral_constant<int, 15>{}); break;
- default: GGML_ABORT("Only support kernel sizes 3, 4, 5, 7, 9, 15 right now.");
+ default: GGML_ABORT("Only support kernel sizes 2, 3, 4, 5, 7, 9, 15 right now.");
}
}
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index c33a579b0..e9e1866e4 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -143,6 +143,7 @@ class Keys:
DECODER_BLOCK_COUNT = "{arch}.decoder_block_count"
ATTN_LOGIT_SOFTCAPPING = "{arch}.attn_logit_softcapping"
ROUTER_LOGIT_SOFTCAPPING = "{arch}.router_logit_softcapping"
+ ROUTER_HIDDEN_SIZE = "{arch}.router_hidden_size"
FINAL_LOGIT_SOFTCAPPING = "{arch}.final_logit_softcapping"
FINAL_LOGIT_SIGMOID_CAPPING = "{arch}.final_logit_sigmoid_capping"
SWIN_NORM = "{arch}.swin_norm"
@@ -736,6 +737,7 @@ class MODEL_ARCH(IntEnum):
LFM2 = auto()
LFM2MOE = auto()
LFM3 = auto()
+ ZAYA1 = auto()
DREAM = auto()
SMALLTHINKER = auto()
LLADA = auto()
@@ -1533,6 +1535,17 @@ class MODEL_TENSOR(IntEnum):
LILICORR_OUT_HEAD = auto()
LILICORR_IN_HEAD = auto()
LILICORR_ANCHOR_OUT_HEAD = auto()
+ # zaya1
+ CCA_CONV_GRP = auto()
+ CCA_TEMP = auto()
+ RES_SCALE_IN = auto()
+ ATTN_RES_SCALE_HS = auto()
+ FFN_RES_SCALE_HS = auto()
+ ROUTER_EDA = auto()
+ ROUTER_NORM = auto()
+ ROUTER_FC1 = auto()
+ ROUTER_FC2 = auto()
+ ROUTER_OUT = auto()
# lfm2 audio
A_ENC_NORM_CONV = auto()
A_ENC_LINEAR_POS = auto()
@@ -1724,6 +1737,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
MODEL_ARCH.LFM2: "lfm2",
MODEL_ARCH.LFM2MOE: "lfm2moe",
MODEL_ARCH.LFM3: "lfm3",
+ MODEL_ARCH.ZAYA1: "zaya1",
MODEL_ARCH.DREAM: "dream",
MODEL_ARCH.SMALLTHINKER: "smallthinker",
MODEL_ARCH.LLADA: "llada",
@@ -2559,6 +2573,16 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.LILICORR_OUT_HEAD: "lilicorr.out_head",
MODEL_TENSOR.LILICORR_IN_HEAD: "lilicorr.in_head",
MODEL_TENSOR.LILICORR_ANCHOR_OUT_HEAD: "lilicorr.anchor_out_head",
+ MODEL_TENSOR.CCA_CONV_GRP: "blk.{bid}.cca_conv_grp",
+ MODEL_TENSOR.CCA_TEMP: "blk.{bid}.cca_temp",
+ MODEL_TENSOR.RES_SCALE_IN: "res_scale_in",
+ MODEL_TENSOR.ATTN_RES_SCALE_HS: "blk.{bid}.attn_res_scale_hs",
+ MODEL_TENSOR.FFN_RES_SCALE_HS: "blk.{bid}.ffn_res_scale_hs",
+ MODEL_TENSOR.ROUTER_EDA: "blk.{bid}.ffn_router_eda",
+ MODEL_TENSOR.ROUTER_NORM: "blk.{bid}.ffn_router_norm",
+ MODEL_TENSOR.ROUTER_FC1: "blk.{bid}.ffn_router_fc1",
+ MODEL_TENSOR.ROUTER_FC2: "blk.{bid}.ffn_router_fc2",
+ MODEL_TENSOR.ROUTER_OUT: "blk.{bid}.ffn_router_out",
MODEL_TENSOR.D2T: "d2t",
MODEL_TENSOR.ATTN_V_GATE: "blk.{bid}.attn_v_gate",
MODEL_TENSOR.ATTN_V_EXP: "blk.{bid}.attn_v_exps",
@@ -6130,6 +6154,34 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.ATTN_OUT,
MODEL_TENSOR.ATTN_GATE,
],
+ MODEL_ARCH.ZAYA1: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.RES_SCALE_IN,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.SSM_CONV1D,
+ MODEL_TENSOR.CCA_CONV_GRP,
+ MODEL_TENSOR.CCA_TEMP,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_RES_SCALE_HS,
+ MODEL_TENSOR.ATTN_RESID_SCALE,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.ROUTER_EDA,
+ MODEL_TENSOR.ROUTER_NORM,
+ MODEL_TENSOR.ROUTER_FC1,
+ MODEL_TENSOR.ROUTER_FC2,
+ MODEL_TENSOR.ROUTER_OUT,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ MODEL_TENSOR.FFN_GATE_UP_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_RES_SCALE_HS,
+ MODEL_TENSOR.FFN_RESID_SCALE,
+ ],
MODEL_ARCH.SMALLTHINKER: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT_NORM,
diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py
index edf2caf88..eb0bd91d0 100644
--- a/gguf-py/gguf/gguf_writer.py
+++ b/gguf-py/gguf/gguf_writer.py
@@ -885,6 +885,9 @@ class GGUFWriter:
def add_router_logit_softcapping(self, value: float) -> None:
self.add_float32(Keys.LLM.ROUTER_LOGIT_SOFTCAPPING.format(arch=self.arch), value)
+ def add_router_hidden_size(self, value: int) -> None:
+ self.add_uint32(Keys.LLM.ROUTER_HIDDEN_SIZE.format(arch=self.arch), value)
+
def add_final_logit_softcapping(self, value: float) -> None:
self.add_float32(Keys.LLM.FINAL_LOGIT_SOFTCAPPING.format(arch=self.arch), value)
diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py
index 61e4e7ddc..c844a9322 100644
--- a/gguf-py/gguf/tensor_mapping.py
+++ b/gguf-py/gguf/tensor_mapping.py
@@ -228,6 +228,9 @@ class TensorNameMap:
MODEL_TENSOR.LILICORR_ANCHOR_OUT_HEAD: (
"model.lilicorr.anchor_out_head",
),
+ MODEL_TENSOR.RES_SCALE_IN: (
+ "model.input_hidden_states", # zaya1
+ ),
}
block_mappings_cfg: dict[MODEL_TENSOR, tuple[str, ...]] = {
@@ -3350,6 +3353,22 @@ class TensorNameMap:
# architecture-specific block mappings
arch_block_mappings_cfg: dict[MODEL_ARCH, dict[MODEL_TENSOR, tuple[str, ...]]] = {
+ MODEL_ARCH.ZAYA1: {
+ MODEL_TENSOR.ATTN_Q: ("model.layers.{bid}.self_attn.qkv_proj.q_proj",),
+ MODEL_TENSOR.ATTN_K: ("model.layers.{bid}.self_attn.qkv_proj.k_proj",),
+ # the converter joins v_proj_delayed onto v_proj_current, attn_v holds both kv heads
+ MODEL_TENSOR.ATTN_V: ("model.layers.{bid}.self_attn.qkv_proj.v_proj_current",),
+ MODEL_TENSOR.SSM_CONV1D: ("model.layers.{bid}.self_attn.qkv_proj.conv_qk_depthwise",),
+ MODEL_TENSOR.CCA_CONV_GRP: ("model.layers.{bid}.self_attn.qkv_proj.conv_qk_grouped",),
+ MODEL_TENSOR.CCA_TEMP: ("model.layers.{bid}.self_attn.qk_norm.temp",),
+ MODEL_TENSOR.FFN_GATE_INP: ("model.layers.{bid}.mlp.gate.down_proj",),
+ MODEL_TENSOR.FFN_EXP_PROBS_B: ("model.layers.{bid}.mlp.gate.balancing_biases",),
+ MODEL_TENSOR.ROUTER_EDA: ("model.layers.{bid}.mlp.gate.router_states_scale",),
+ MODEL_TENSOR.ROUTER_NORM: ("model.layers.{bid}.mlp.gate.router_mlp.norm",),
+ MODEL_TENSOR.ROUTER_FC1: ("model.layers.{bid}.mlp.gate.router_mlp.fc1",),
+ MODEL_TENSOR.ROUTER_FC2: ("model.layers.{bid}.mlp.gate.router_mlp.fc2",),
+ MODEL_TENSOR.ROUTER_OUT: ("model.layers.{bid}.mlp.gate.router_mlp.out_proj",),
+ },
MODEL_ARCH.ARCTIC: {
MODEL_TENSOR.FFN_NORM: (
"model.layers.{bid}.residual_layernorm",
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index feee62bd4..c50e9f98a 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -139,6 +139,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
{ LLM_ARCH_LFM2, "lfm2" },
{ LLM_ARCH_LFM2MOE, "lfm2moe" },
{ LLM_ARCH_LFM3, "lfm3" },
+ { LLM_ARCH_ZAYA1, "zaya1" },
{ LLM_ARCH_DREAM, "dream" },
{ LLM_ARCH_SMALLTHINKER, "smallthinker" },
{ LLM_ARCH_LLADA, "llada" },
@@ -260,6 +261,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_DECODER_BLOCK_COUNT, "%s.decoder_block_count" },
{ LLM_KV_ATTN_LOGIT_SOFTCAPPING, "%s.attn_logit_softcapping" },
{ LLM_KV_ROUTER_LOGIT_SOFTCAPPING, "%s.router_logit_softcapping" },
+ { LLM_KV_ROUTER_HIDDEN_SIZE, "%s.router_hidden_size" },
{ LLM_KV_FINAL_LOGIT_SOFTCAPPING, "%s.final_logit_softcapping" },
{ LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING, "%s.final_logit_sigmoid_capping" },
{ LLM_KV_SWIN_NORM, "%s.swin_norm" },
@@ -912,6 +914,16 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
{ LLM_TENSOR_LILICORR_OUT_HEAD, "lilicorr.out_head" },
{ LLM_TENSOR_LILICORR_IN_HEAD, "lilicorr.in_head" },
{ LLM_TENSOR_LILICORR_ANCHOR_OUT_HEAD, "lilicorr.anchor_out_head" },
+ { LLM_TENSOR_CCA_CONV_GRP, "blk.%d.cca_conv_grp" },
+ { LLM_TENSOR_CCA_TEMP, "blk.%d.cca_temp" },
+ { LLM_TENSOR_RES_SCALE_IN, "res_scale_in" },
+ { LLM_TENSOR_ATTN_RES_SCALE_HS, "blk.%d.attn_res_scale_hs" },
+ { LLM_TENSOR_FFN_RES_SCALE_HS, "blk.%d.ffn_res_scale_hs" },
+ { LLM_TENSOR_ROUTER_EDA, "blk.%d.ffn_router_eda" },
+ { LLM_TENSOR_ROUTER_NORM, "blk.%d.ffn_router_norm" },
+ { LLM_TENSOR_ROUTER_FC1, "blk.%d.ffn_router_fc1" },
+ { LLM_TENSOR_ROUTER_FC2, "blk.%d.ffn_router_fc2" },
+ { LLM_TENSOR_ROUTER_OUT, "blk.%d.ffn_router_out" },
{ LLM_TENSOR_ATTN_V_GATE, "blk.%d.attn_v_gate" },
{ LLM_TENSOR_ATTN_V_EXPS, "blk.%d.attn_v_exps" },
};
@@ -1317,6 +1329,17 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
{LLM_TENSOR_LILICORR_IN_HEAD, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_LILICORR_ANCHOR_OUT_HEAD, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_CCA_CONV_GRP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_CCA_TEMP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_RES_SCALE_IN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_ATTN_RES_SCALE_HS, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_FFN_RES_SCALE_HS, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_ROUTER_EDA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_ROUTER_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_ROUTER_FC1, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_ROUTER_FC2, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_ROUTER_OUT, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+
{LLM_TENSOR_ATTN_V_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_ATTN_V_EXPS, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT_ID}},
};
@@ -1405,6 +1428,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) {
case LLM_ARCH_LFM2:
case LLM_ARCH_LFM2MOE:
case LLM_ARCH_LFM3:
+ case LLM_ARCH_ZAYA1:
case LLM_ARCH_NEMOTRON_H:
case LLM_ARCH_NEMOTRON_H_MOE:
case LLM_ARCH_METIS16:
@@ -1486,6 +1510,7 @@ bool llm_arch_supports_mixed_batch(const llm_arch & arch) {
bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
switch (arch) {
case LLM_ARCH_CLOVER_MOE:
+ case LLM_ARCH_ZAYA1:
case LLM_ARCH_GROK:
case LLM_ARCH_MPT:
case LLM_ARCH_PLAMO2:
diff --git a/src/llama-arch.h b/src/llama-arch.h
index 4187a0d08..9c517a9fe 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -143,6 +143,7 @@ enum llm_arch {
LLM_ARCH_LFM2,
LLM_ARCH_LFM2MOE,
LLM_ARCH_LFM3,
+ LLM_ARCH_ZAYA1,
LLM_ARCH_DREAM,
LLM_ARCH_SMALLTHINKER,
LLM_ARCH_LLADA,
@@ -264,6 +265,7 @@ enum llm_kv {
LLM_KV_DECODER_BLOCK_COUNT,
LLM_KV_ATTN_LOGIT_SOFTCAPPING,
LLM_KV_ROUTER_LOGIT_SOFTCAPPING,
+ LLM_KV_ROUTER_HIDDEN_SIZE,
LLM_KV_FINAL_LOGIT_SOFTCAPPING,
LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING,
LLM_KV_SWIN_NORM,
@@ -917,6 +919,16 @@ enum llm_tensor {
LLM_TENSOR_LILICORR_OUT_HEAD,
LLM_TENSOR_LILICORR_IN_HEAD,
LLM_TENSOR_LILICORR_ANCHOR_OUT_HEAD,
+ LLM_TENSOR_CCA_CONV_GRP,
+ LLM_TENSOR_CCA_TEMP,
+ LLM_TENSOR_RES_SCALE_IN,
+ LLM_TENSOR_ATTN_RES_SCALE_HS,
+ LLM_TENSOR_FFN_RES_SCALE_HS,
+ LLM_TENSOR_ROUTER_EDA,
+ LLM_TENSOR_ROUTER_NORM,
+ LLM_TENSOR_ROUTER_FC1,
+ LLM_TENSOR_ROUTER_FC2,
+ LLM_TENSOR_ROUTER_OUT,
LLM_TENSOR_ATTN_V_GATE,
LLM_TENSOR_ATTN_V_EXPS,
};
diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cpp
index 66903fbb7..807f7987f 100644
--- a/src/llama-hparams.cpp
+++ b/src/llama-hparams.cpp
@@ -221,6 +221,11 @@ uint32_t llama_hparams::n_embd_r() const {
return n_embd * (n_shortconv_l_cache - 1);
}
+ if (n_cca_qk != 0) {
+ // for ZAYA1 CCA layers: depthwise + grouped conv, each with d_conv - 1 columns of history
+ return 2 * (ssm_d_conv - 1) * n_cca_qk;
+ }
+
if (n_embd_head_kda != 0) {
// for Kimi KDA layers
// Conv state for Q, K, V: 3 * (d_conv - 1) * n_head * head_dim
@@ -244,6 +249,11 @@ uint32_t llama_hparams::n_embd_s() const {
return n_embd * wkv_head_size;
}
+ if (n_cca_qk != 0) {
+ // for ZAYA CCA layers: the delayed value of the previous token
+ return n_embd_v_gqa() / 2;
+ }
+
if (n_embd_head_kda != 0) {
// for Kimi KDA layers
// Full recurrent state: head_dim * head_dim * n_head
diff --git a/src/llama-hparams.h b/src/llama-hparams.h
index b4894956c..642f90118 100644
--- a/src/llama-hparams.h
+++ b/src/llama-hparams.h
@@ -221,6 +221,12 @@ struct llama_hparams {
uint32_t n_embd_head_kda = 0;
bool kda_safe_gate = false;
+ // for ZAYA1 CCA layers: q + k channels in the conv state
+ uint32_t n_cca_qk = 0;
+
+ // for ZAYA1: width of the router MLP
+ uint32_t n_router_hidden = 0;
+
// kimi-k3
uint32_t n_expert_latent = 0; // routed_expert_hidden_size (0 = experts run at n_embd)
uint32_t attn_res_block_size = 0; // 0 = no cross-layer attention residuals
diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp
index 338baa8f7..c01d4dd51 100644
--- a/src/llama-model-saver.cpp
+++ b/src/llama-model-saver.cpp
@@ -241,6 +241,7 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_DECODER_BLOCK_COUNT, hparams.dec_n_layer);
add_kv(LLM_KV_ATTN_LOGIT_SOFTCAPPING, hparams.f_attn_logit_softcapping);
add_kv(LLM_KV_ROUTER_LOGIT_SOFTCAPPING, hparams.f_router_logit_softcapping);
+ add_kv(LLM_KV_ROUTER_HIDDEN_SIZE, hparams.n_router_hidden);
add_kv(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping);
add_kv(LLM_KV_SWIN_NORM, hparams.swin_norm);
add_kv(LLM_KV_RESCALE_EVERY_N_LAYERS, hparams.rescale_every_n_layers);
@@ -578,6 +579,8 @@ void llama_model_saver::add_tensors_from_model() {
add_tensor(model->hc_head_norm);
add_tensor(model->hc_head_down);
add_tensor(model->hc_head_up);
+ add_tensor(model->res_scale_in);
+ add_tensor(model->res_scale_in_b);
// looped architectures alias physical tensors across cache slots; save each
// tensor once. a different tensor with an existing name still asserts below
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 728473fff..ac5266a42 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -332,6 +332,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_lfm2moe(params);
case LLM_ARCH_LFM3:
return new llama_model_lfm3(params);
+ case LLM_ARCH_ZAYA1:
+ return new llama_model_zaya1(params);
case LLM_ARCH_SMALLTHINKER:
return new llama_model_smallthinker(params);
case LLM_ARCH_GROVEMOE:
@@ -3349,7 +3351,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
filter_idx = [&](uint32_t il) {
return il < hparams.n_layer() && !hparams.is_recr(il) && hparams.is_indexer_full(il);
};
- } else if (arch == LLM_ARCH_FALCON_H1 || arch == LLM_ARCH_INKLING2) {
+ } else if (arch == LLM_ARCH_FALCON_H1 || arch == LLM_ARCH_INKLING2 || arch == LLM_ARCH_ZAYA1) {
filter_attn = [&](uint32_t) { return true; };
filter_recr = [&](uint32_t) { return true; };
} else if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) {
@@ -3847,6 +3849,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_LFM2:
case LLM_ARCH_LFM2MOE:
case LLM_ARCH_LFM3:
+ case LLM_ARCH_ZAYA1:
case LLM_ARCH_SMALLTHINKER:
case LLM_ARCH_SEED_OSS:
case LLM_ARCH_GROVEMOE:
diff --git a/src/llama-model.h b/src/llama-model.h
index 27c804eb6..8d2f5629e 100644
--- a/src/llama-model.h
+++ b/src/llama-model.h
@@ -226,6 +226,23 @@ struct llama_layer_shortconv {
struct ggml_tensor * out_proj = nullptr;
};
+struct llama_layer_zaya1 {
+ struct ggml_tensor * cca_conv_grp = nullptr;
+ struct ggml_tensor * cca_conv_grp_b = nullptr;
+ struct ggml_tensor * cca_temp = nullptr;
+
+ struct ggml_tensor * attn_res_hs = nullptr;
+ struct ggml_tensor * ffn_res_hs = nullptr;
+
+ struct ggml_tensor * router_eda = nullptr;
+ struct ggml_tensor * router_norm = nullptr;
+ struct ggml_tensor * router_fc1 = nullptr;
+ struct ggml_tensor * router_fc1_b = nullptr;
+ struct ggml_tensor * router_fc2 = nullptr;
+ struct ggml_tensor * router_fc2_b = nullptr;
+ struct ggml_tensor * router_out = nullptr;
+};
+
struct llama_layer_nextn {
struct ggml_tensor * eh_proj = nullptr;
struct ggml_tensor * eh_proj_s = nullptr;
@@ -667,8 +684,10 @@ struct llama_layer {
// limite
struct ggml_tensor * attn_ve_gate = nullptr;
struct ggml_tensor * attn_xsa_alpha = nullptr;
- struct ggml_tensor * attn_resid_scale = nullptr;
- struct ggml_tensor * ffn_resid_scale = nullptr;
+ struct ggml_tensor * attn_resid_scale = nullptr;
+ struct ggml_tensor * attn_resid_scale_b = nullptr;
+ struct ggml_tensor * ffn_resid_scale = nullptr;
+ struct ggml_tensor * ffn_resid_scale_b = nullptr;
struct ggml_tensor * mudd_down = nullptr;
struct ggml_tensor * mudd_up = nullptr;
struct ggml_tensor * mudd_up_b = nullptr;
@@ -697,6 +716,8 @@ struct llama_layer {
struct llama_layer_shortconv shortconv;
+ struct llama_layer_zaya1 zaya1;
+
struct llama_layer_nextn nextn;
struct llama_layer_switch_lora switch_lora;
@@ -909,6 +930,10 @@ struct llama_model {
struct ggml_tensor * dspark_joint_proj_b = nullptr;
struct ggml_tensor * dspark_mask_embd = nullptr; // per-slot mask embeddings
+ // zaya1
+ struct ggml_tensor * res_scale_in = nullptr;
+ struct ggml_tensor * res_scale_in_b = nullptr;
+
struct ggml_tensor * dflash_selector_prev = nullptr;
struct ggml_tensor * dflash_selector_next = nullptr;
struct ggml_tensor * dflash_selector_hidden = nullptr;
diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp
index a50d3c237..0f3f982b0 100644
--- a/src/llama-quant.cpp
+++ b/src/llama-quant.cpp
@@ -348,6 +348,12 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param
quantize &= name.find("attn_rel_bias.weight") == std::string::npos;
}
+ // zaya1: the top-1 router and the grouped conv kernels stay F32
+ if (arch == LLM_ARCH_ZAYA1) {
+ quantize &= name.find("ffn_router_") == std::string::npos;
+ quantize &= name.find("cca_conv_grp") == std::string::npos;
+ }
+
// do not quantize positional embeddings and token types (BERT)
quantize &= name != LLM_TN(arch)(LLM_TENSOR_POS_EMBD, "weight");
quantize &= name != LLM_TN(arch)(LLM_TENSOR_TOKEN_TYPES, "weight");
diff --git a/src/models/models.h b/src/models/models.h
index 8da3bcf29..8d74c5018 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -2535,6 +2535,22 @@ struct llama_model_lfm3 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
+struct llama_model_zaya1 : public llama_model_base {
+ llama_model_zaya1(const struct llama_model_params & params) : llama_model_base(params) {}
+ void load_arch_hparams(llama_model_loader & ml) override;
+ void load_arch_tensors(llama_model_loader & ml) override;
+
+ struct graph : public llm_graph_context {
+ graph(const llama_model & model, const llm_graph_params & params);
+
+ ggml_tensor * build_cca_block(const llama_model & model, ggml_tensor * cur, ggml_tensor * inp_pos,
+ llm_graph_input_mem_hybrid * inp, int il);
+ ggml_tensor * build_router_moe(const llama_model & model, ggml_tensor * cur, ggml_tensor *& router_state, int il);
+ };
+
+ std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
struct llama_model_smallthinker : public llama_model_base {
llama_model_smallthinker(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
diff --git a/src/models/zaya1.cpp b/src/models/zaya1.cpp
new file mode 100644
index 000000000..2763500f4
--- /dev/null
+++ b/src/models/zaya1.cpp
@@ -0,0 +1,340 @@
+#include "models.h"
+#include "llama-memory-recurrent.h"
+
+void llama_model_zaya1::load_arch_hparams(llama_model_loader & ml) {
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+ ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv);
+ ml.get_key(LLM_KV_ROUTER_HIDDEN_SIZE, hparams.n_router_hidden);
+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);
+
+ if (hparams.ssm_d_conv != 2) {
+ throw std::runtime_error("zaya1: only ssm.conv_kernel == 2 is supported");
+ }
+ // the two kv heads are v_cur and v_del, and the router picks one expert
+ for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
+ if (hparams.n_head_kv(il) != 2 || hparams.n_embd_head_v(il) != hparams.n_embd_head_k(il) || hparams.n_expert_used(il) != 1) {
+ throw std::runtime_error("zaya1: only 2 kv heads of equal k/v size and 1 used expert are supported");
+ }
+ }
+
+ hparams.n_cca_qk = (hparams.n_head() + hparams.n_head_kv()) * hparams.n_embd_head_k();
+
+ std::fill(hparams.is_recr_impl.begin(), hparams.is_recr_impl.end(), true);
+
+ switch (hparams.n_layer()) {
+ case 40: type = LLM_TYPE_8B_A1B; break;
+ default: type = LLM_TYPE_UNKNOWN;
+ }
+}
+
+void llama_model_zaya1::load_arch_tensors(llama_model_loader &) {
+ LLAMA_LOAD_LOCALS;
+
+ const int64_t n_embd_head = n_embd_head_k;
+ const int64_t n_group = n_head + n_head_kv;
+ const int64_t n_qk = hparams.n_cca_qk;
+ const int64_t d_conv = hparams.ssm_d_conv;
+ const int64_t n_ff_exp = hparams.n_ff_exp();
+
+ const int64_t n_router = hparams.n_router_hidden;
+
+ tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+
+ res_scale_in = create_tensor(tn(LLM_TENSOR_RES_SCALE_IN, "weight", 0), {n_embd}, 0);
+ res_scale_in_b = create_tensor(tn(LLM_TENSOR_RES_SCALE_IN, "bias", 0), {n_embd}, 0);
+
+ output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+ output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
+ if (output == nullptr) {
+ output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
+ }
+
+ for (int i = 0; i < n_layer; ++i) {
+ auto & layer = layers[i];
+ auto & z = layer.zaya1;
+
+ layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
+
+ // the two value heads are v_cur and v_del
+ create_tensor_qkv(layer, i, n_embd, n_embd_head * n_head, n_embd_head * n_head_kv, n_embd_v_gqa, 0);
+
+ layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "weight", i), {d_conv, n_qk}, 0);
+ layer.ssm_conv1d_b = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "bias", i), {n_qk}, 0);
+ z.cca_conv_grp = create_tensor(tn(LLM_TENSOR_CCA_CONV_GRP, "weight", i), {n_embd_head, n_embd_head, n_group, d_conv}, 0);
+ z.cca_conv_grp_b = create_tensor(tn(LLM_TENSOR_CCA_CONV_GRP, "bias", i), {n_qk}, 0);
+ z.cca_temp = create_tensor(tn(LLM_TENSOR_CCA_TEMP, "weight", i), {n_head_kv}, 0);
+
+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head * n_head, n_embd}, 0);
+
+ z.attn_res_hs = create_tensor(tn(LLM_TENSOR_ATTN_RES_SCALE_HS, "weight", i), {n_embd}, 0);
+ layer.attn_resid_scale = create_tensor(tn(LLM_TENSOR_ATTN_RESID_SCALE, "weight", i), {n_embd}, 0);
+ layer.attn_resid_scale_b = create_tensor(tn(LLM_TENSOR_ATTN_RESID_SCALE, "bias", i), {n_embd}, 0);
+
+ layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
+
+ layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_router}, 0);
+ layer.ffn_gate_inp_b = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "bias", i), {n_router}, 0);
+ z.router_eda = create_tensor(tn(LLM_TENSOR_ROUTER_EDA, "weight", i), {n_router}, i == 0 ? TENSOR_NOT_REQUIRED : 0);
+ z.router_norm = create_tensor(tn(LLM_TENSOR_ROUTER_NORM, "weight", i), {n_router}, 0);
+ z.router_fc1 = create_tensor(tn(LLM_TENSOR_ROUTER_FC1, "weight", i), {n_router, n_router}, 0);
+ z.router_fc1_b = create_tensor(tn(LLM_TENSOR_ROUTER_FC1, "bias", i), {n_router}, 0);
+ z.router_fc2 = create_tensor(tn(LLM_TENSOR_ROUTER_FC2, "weight", i), {n_router, n_router}, 0);
+ z.router_fc2_b = create_tensor(tn(LLM_TENSOR_ROUTER_FC2, "bias", i), {n_router}, 0);
+ // one extra class for the skip expert
+ z.router_out = create_tensor(tn(LLM_TENSOR_ROUTER_OUT, "weight", i), {n_router, n_expert + 1}, 0);
+ layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert + 1}, 0);
+
+ layer.ffn_gate_up_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", i), {n_embd, 2 * n_ff_exp, n_expert}, 0);
+ layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);
+
+ z.ffn_res_hs = create_tensor(tn(LLM_TENSOR_FFN_RES_SCALE_HS, "weight", i), {n_embd}, 0);
+ layer.ffn_resid_scale = create_tensor(tn(LLM_TENSOR_FFN_RESID_SCALE, "weight", i), {n_embd}, 0);
+ layer.ffn_resid_scale_b = create_tensor(tn(LLM_TENSOR_FFN_RESID_SCALE, "bias", i), {n_embd}, 0);
+ }
+}
+
+std::unique_ptr<llm_graph_context> llama_model_zaya1::build_arch_graph(const llm_graph_params & params) const {
+ return std::make_unique<graph>(*this, params);
+}
+
+static ggml_tensor * zaya1_residual_scale(ggml_context * ctx, ggml_tensor * hs, ggml_tensor * res,
+ ggml_tensor * s_hs, ggml_tensor * s_res, ggml_tensor * b) {
+ return ggml_add(ctx, ggml_mul(ctx, hs, s_hs), ggml_add(ctx, ggml_mul(ctx, res, s_res), b));
+}
+
+// grouped causal conv1d with kernel 2, as one mul_mat per tap broadcast over the groups
+// x: {n_embd_head * n_group, n_t + 1, n_seqs}, w: {n_embd_head (in), n_embd_head (out), n_group, 2}
+// returns {n_embd_head * n_group, n_t, n_seqs} with the bias b added
+static ggml_tensor * zaya1_conv_grouped(ggml_context * ctx, ggml_tensor * x, ggml_tensor * w, ggml_tensor * b) {
+ const int64_t n_embd_head = w->ne[0];
+ const int64_t n_group = w->ne[2];
+ const int64_t n_t = x->ne[1] - 1;
+ const int64_t n_seqs = x->ne[2];
+ GGML_ASSERT(w->ne[3] == 2);
+
+ x = ggml_reshape_4d(ctx, x, n_embd_head, n_group, n_t + 1, n_seqs);
+ x = ggml_permute(ctx, x, 0, 2, 1, 3);
+
+ ggml_tensor * x_prev = ggml_view_4d(ctx, x, n_embd_head, n_t, n_group, n_seqs, x->nb[1], x->nb[2], x->nb[3], 0);
+ ggml_tensor * x_cur = ggml_view_4d(ctx, x, n_embd_head, n_t, n_group, n_seqs, x->nb[1], x->nb[2], x->nb[3], x->nb[1]);
+
+ ggml_tensor * w_prev = ggml_view_3d(ctx, w, n_embd_head, n_embd_head, n_group, w->nb[1], w->nb[2], 0);
+ ggml_tensor * w_cur = ggml_view_3d(ctx, w, n_embd_head, n_embd_head, n_group, w->nb[1], w->nb[2], w->nb[3]);
+
+ ggml_tensor * y = ggml_add(ctx, ggml_mul_mat(ctx, w_prev, x_prev), ggml_mul_mat(ctx, w_cur, x_cur));
+
+ y = ggml_add(ctx, ggml_permute(ctx, y, 0, 2, 1, 3), ggml_reshape_2d(ctx, b, n_embd_head, n_group));
+ return ggml_reshape_3d(ctx, y, n_embd_head * n_group, n_t, n_seqs);
+}
+
+ggml_tensor * llama_model_zaya1::graph::build_cca_block(const llama_model & model, ggml_tensor * cur,
+ ggml_tensor * inp_pos, llm_graph_input_mem_hybrid * inp, int il) {
+ const auto & layer = model.layers[il];
+ const auto & z = layer.zaya1;
+
+ const auto * mctx_cur = inp->get_recr()->mctx;
+ const uint32_t kv_head = mctx_cur->get_head();
+
+ const int64_t n_seq_tokens = ubatch.n_seq_tokens;
+ const int64_t n_seqs = ubatch.n_seqs;
+ GGML_ASSERT(n_seqs != 0);
+ GGML_ASSERT(ubatch.equal_seqs());
+ GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);
+
+ const int64_t n_embd_head = hparams.n_embd_head_k();
+ const int64_t n_head_kv = hparams.n_head_kv(il);
+ const int64_t n_q = n_embd_head * n_head;
+ const int64_t n_qk = hparams.n_cca_qk;
+ const int64_t n_v_half = hparams.n_embd_v_gqa(il) / 2;
+ const int64_t n_hist = 2 * (hparams.ssm_d_conv - 1);
+
+ auto [q_proj, k_proj, v_raw] = build_qkv(layer, cur,
+ n_embd_head, n_head, n_embd_head, n_head_kv, n_embd_head, n_head_kv, il, false);
+ ggml_tensor * qk_raw = ggml_concat(ctx0, q_proj, k_proj, 0);
+ cb(qk_raw, "cca_qk_raw", il);
+ cb(v_raw, "cca_v_raw", il);
+
+ const size_t es = ggml_element_size(qk_raw);
+ const int64_t n_rep = n_head / n_head_kv;
+
+ ggml_tensor * q_raw = ggml_view_4d(ctx0, qk_raw, n_embd_head, n_rep, n_head_kv, n_tokens,
+ n_embd_head * es, n_embd_head * n_rep * es, qk_raw->nb[1], 0);
+ ggml_tensor * k_raw = ggml_view_4d(ctx0, qk_raw, n_embd_head, 1, n_head_kv, n_tokens,
+ n_embd_head * es, n_embd_head * es, qk_raw->nb[1], n_q * es);
+ ggml_tensor * q_res = ggml_scale(ctx0, ggml_add(ctx0, q_raw, k_raw), 0.5f); // {n_embd_head, n_rep, n_head_kv, n_tokens}
+ ggml_tensor * k_res = nullptr;
+ for (int64_t r = 0; r < n_rep; ++r) {
+ ggml_tensor * q_r = ggml_view_4d(ctx0, q_res, n_embd_head, 1, n_head_kv, n_tokens,
+ q_res->nb[1], q_res->nb[2], q_res->nb[3], r * q_res->nb[1]);
+ k_res = k_res ? ggml_add(ctx0, k_res, q_r) : q_r;
+ }
+ k_res = ggml_scale(ctx0, k_res, 1.0f / n_rep);
+ q_res = ggml_reshape_3d(ctx0, q_res, n_embd_head, n_head, n_tokens);
+ k_res = ggml_reshape_3d(ctx0, k_res, n_embd_head, n_head_kv, n_tokens);
+
+ // conv state: the last n_hist raw q/k columns of each sequence
+ ggml_tensor * conv_state = mctx_cur->get_r_l(il);
+ ggml_tensor * conv_rs = build_rs(inp->get_recr(), conv_state, hparams.n_embd_r(), n_seqs);
+ conv_rs = ggml_reshape_3d(ctx0, conv_rs, n_hist, n_qk, n_seqs);
+
+ ggml_tensor * qk = ggml_view_3d(ctx0, qk_raw, n_qk, n_seq_tokens, n_seqs, qk_raw->nb[1], n_seq_tokens * qk_raw->nb[1], 0);
+ qk = ggml_concat(ctx0, conv_rs, ggml_transpose(ctx0, qk), 0); // {n_hist + n_seq_tokens, n_qk, n_seqs}
+
+ // delayed value state: v_del of the previous token
+ ggml_tensor * v_state = mctx_cur->get_s_l(il);
+ ggml_tensor * v_rs = build_rs(inp->get_recr(), v_state, hparams.n_embd_s(), n_seqs);
+ ggml_tensor * v_del_ext = ggml_concat(ctx0,
+ ggml_reshape_3d(ctx0, v_rs, n_v_half, 1, n_seqs),
+ ggml_view_3d(ctx0, v_raw, n_v_half, n_seq_tokens, n_seqs, v_raw->nb[1], n_seq_tokens * v_raw->nb[1], n_v_half * es), 1); // {n_v_half, n_seq_tokens + 1, n_seqs}
+
+ {
+ const size_t conv_row = ggml_row_size(conv_state->type, n_hist * n_qk);
+ const size_t v_row = ggml_row_size(v_state->type, n_v_half);
+
+ ggml_build_forward_expand(gf, ggml_cpy(ctx0,
+ ggml_view_3d(ctx0, qk, n_hist, n_qk, n_seqs, qk->nb[1], qk->nb[2], n_seq_tokens * ggml_element_size(qk)),
+ ggml_view_2d(ctx0, conv_state, n_hist * n_qk, n_seqs, conv_state->nb[1], kv_head * conv_row)));
+ ggml_build_forward_expand(gf, ggml_cpy(ctx0,
+ ggml_view_2d(ctx0, v_del_ext, n_v_half, n_seqs, v_del_ext->nb[2], n_seq_tokens * v_del_ext->nb[1]),
+ ggml_view_2d(ctx0, v_state, n_v_half, n_seqs, v_state->nb[1], kv_head * v_row)));
+ }
+
+ // depthwise conv, then grouped conv
+ ggml_tensor * qk_conv = ggml_ssm_conv(ctx0, qk, layer.ssm_conv1d); // {n_qk, n_seq_tokens + 1, n_seqs}
+ qk_conv = ggml_add(ctx0, qk_conv, layer.ssm_conv1d_b);
+ qk_conv = zaya1_conv_grouped(ctx0, qk_conv, z.cca_conv_grp, z.cca_conv_grp_b); // {n_qk, n_seq_tokens, n_seqs}
+ qk_conv = ggml_reshape_2d(ctx0, qk_conv, n_qk, n_tokens);
+ cb(qk_conv, "cca_qk_conv", il);
+
+ ggml_tensor * Qcur = ggml_view_3d(ctx0, qk_conv, n_embd_head, n_head, n_tokens,
+ n_embd_head * ggml_element_size(qk_conv), qk_conv->nb[1], 0);
+ ggml_tensor * Kcur = ggml_view_3d(ctx0, qk_conv, n_embd_head, n_head_kv, n_tokens,
+ n_embd_head * ggml_element_size(qk_conv), qk_conv->nb[1], n_q * ggml_element_size(qk_conv));
+ Qcur = ggml_add(ctx0, Qcur, q_res);
+ Kcur = ggml_add(ctx0, Kcur, k_res);
+
+ // L2 norm to sqrt(head_dim), then per kv head key temperature
+ const float head_scale = sqrtf((float) n_embd_head);
+ Qcur = ggml_scale(ctx0, ggml_l2_norm(ctx0, Qcur, 1e-6f), head_scale);
+ Kcur = ggml_scale(ctx0, ggml_l2_norm(ctx0, Kcur, 1e-6f), head_scale);
+ Kcur = ggml_mul(ctx0, Kcur, ggml_reshape_2d(ctx0, z.cca_temp, 1, n_head_kv));
+ cb(Qcur, "cca_q", il);
+ cb(Kcur, "cca_k", il);
+
+ Qcur = build_rope(Qcur, inp_pos, il);
+ Kcur = build_rope(Kcur, inp_pos, il);
+
+ // value heads: head 0 = current token, head 1 = previous token
+ ggml_tensor * v_prev = ggml_view_3d(ctx0, v_del_ext, n_v_half, n_seq_tokens, n_seqs,
+ v_del_ext->nb[1], v_del_ext->nb[2], 0);
+ ggml_tensor * v_cur = ggml_view_3d(ctx0, v_raw, n_v_half, n_seq_tokens, n_seqs, v_raw->nb[1], n_seq_tokens * v_raw->nb[1], 0);
+ ggml_tensor * Vcur = ggml_concat(ctx0, v_cur, v_prev, 0);
+ Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+ cb(Vcur, "cca_v", il);
+
+ return build_attn(inp->get_attn(),
+ layer.wo, NULL, layer.wo_s,
+ Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf((float) n_embd_head), il);
+}
+
+ggml_tensor * llama_model_zaya1::graph::build_router_moe(const llama_model & model, ggml_tensor * cur,
+ ggml_tensor *& router_state, int il) {
+ const auto & layer = model.layers[il];
+ const auto & z = layer.zaya1;
+
+ ggml_tensor * r = ggml_add(ctx0, build_lora_mm(layer.ffn_gate_inp, cur), layer.ffn_gate_inp_b);
+ if (router_state) {
+ r = ggml_add(ctx0, r, ggml_mul(ctx0, router_state, z.router_eda));
+ }
+ router_state = r;
+
+ r = build_norm(r, z.router_norm, NULL, LLM_NORM_RMS, il);
+ r = ggml_gelu_erf(ctx0, ggml_add(ctx0, build_lora_mm(z.router_fc1, r), z.router_fc1_b));
+ r = ggml_gelu_erf(ctx0, ggml_add(ctx0, build_lora_mm(z.router_fc2, r), z.router_fc2_b));
+ ggml_tensor * logits = build_lora_mm(z.router_out, r); // {n_expert + 1, n_tokens}
+ ggml_tensor * probs = ggml_soft_max(ctx0, logits);
+ cb(probs, "router_probs", il);
+
+ const int64_t n_rows = cur->ne[1];
+ ggml_tensor * scores = ggml_add(ctx0, probs, layer.ffn_exp_probs_b);
+ ggml_tensor * real = ggml_cont(ctx0, ggml_view_2d(ctx0, scores, n_expert, n_rows, scores->nb[1], 0));
+ ggml_tensor * skip = ggml_view_2d(ctx0, scores, 1, n_rows, scores->nb[1], n_expert * ggml_element_size(scores));
+ ggml_tensor * expert = ggml_argsort_top_k(ctx0, real, 1); // {1, n_rows}
+ ggml_tensor * best = ggml_reshape_2d(ctx0, ggml_get_rows(ctx0, ggml_reshape_3d(ctx0, real, 1, n_expert, n_rows), expert), 1, n_rows);
+ ggml_tensor * keep = ggml_scale_bias(ctx0, ggml_step(ctx0, ggml_sub(ctx0, skip, best)), -1.0f, 1.0f);
+ cb(keep, "router_keep", il);
+
+ ggml_tensor * moe = build_moe_ffn(cur,
+ nullptr,
+ nullptr,
+ nullptr,
+ layer.ffn_down_exps,
+ nullptr,
+ n_expert + 1, n_expert_used,
+ LLM_FFN_SILU, false,
+ 0.0f,
+ LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
+ il,
+ logits,
+ layer.ffn_gate_up_exps,
+ nullptr,
+ nullptr,
+ nullptr,
+ expert);
+
+ return ggml_mul(ctx0, moe, keep);
+}
+
+llama_model_zaya1::graph::graph(const llama_model & model, const llm_graph_params & params) :
+ llm_graph_context(params) {
+ ggml_tensor * cur = build_inp_embd(model.tok_embd);
+ cur = ggml_mul(ctx0, ggml_add(ctx0, cur, model.res_scale_in_b), model.res_scale_in);
+ cb(cur, "inp_scaled", -1);
+
+ auto * inp = build_inp_mem_hybrid();
+
+ ggml_tensor * inp_pos = build_inp_pos();
+ ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+ ggml_tensor * router_state = nullptr;
+
+ for (int il = 0; il < n_layer; ++il) {
+ const auto & layer = model.layers[il];
+ const auto & z = layer.zaya1;
+
+ res->t_layer_inp[il] = cur;
+
+ ggml_tensor * residual = cur;
+
+ cur = build_norm(cur, layer.attn_norm, NULL, LLM_NORM_RMS, il);
+ cur = build_cca_block(model, cur, inp_pos, inp, il);
+ cb(cur, "attn_out", il);
+
+ if (il == n_layer - 1 && inp_out_ids) {
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ residual = ggml_get_rows(ctx0, residual, inp_out_ids);
+ if (router_state) {
+ router_state = ggml_get_rows(ctx0, router_state, inp_out_ids);
+ }
+ }
+
+ residual = zaya1_residual_scale(ctx0, cur, residual, z.attn_res_hs, layer.attn_resid_scale, layer.attn_resid_scale_b);
+
+ cur = build_norm(residual, layer.ffn_norm, NULL, LLM_NORM_RMS, il);
+ cur = build_router_moe(model, cur, router_state, il);
+ cb(cur, "ffn_moe_out", il);
+
+ cur = zaya1_residual_scale(ctx0, cur, residual, z.ffn_res_hs, layer.ffn_resid_scale, layer.ffn_resid_scale_b);
+
+ cur = build_cvec(cur, il);
+ cb(cur, "l_out", il);
+ }
+
+ cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
+ cb(cur, "result_norm", -1);
+ res->t_embd = cur;
+
+ cur = build_lora_mm(model.output, cur);
+ cb(cur, "result_output", -1);
+ res->t_logits = cur;
+}
diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv
index efb8718e7..310934e31 100644
--- a/tests/fusion/MTL.csv
+++ b/tests/fusion/MTL.csv
@@ -348,6 +348,7 @@ xing4_0 ,0 ,any ,MUL+ADD , 2
xing4_0 ,0 ,any ,RMS_NORM+MUL , 9
xing4_0 ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS , 1
xverse ,0 ,any ,RMS_NORM+MUL , 5
+zaya1 ,0 ,any ,RMS_NORM+MUL , 7
zonos3 ,0 ,any ,GATED_DELTA_NET+CPY , 1
zonos3 ,0 ,any ,MUL+ADD , 2
zonos3 ,0 ,any ,RMS_NORM+MUL , 12
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 49ae976a4..fa1ffb853 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -11742,7 +11742,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
}
}
- for (int64_t d_conv : {3, 4, 7, 9}) {
+ for (int64_t d_conv : {2, 3, 4, 7, 9}) {
for (int64_t d_inner: {1024, 1536, 2048}) {
test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {d_conv, d_inner, 1, 1}, {d_conv, d_inner, 1, 1}));
test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {2 * d_conv, d_inner, 1, 1}, {d_conv, d_inner, 1, 1}));
diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp
index a2cba05aa..f10404e8c 100644
--- a/tests/test-llama-archs.cpp
+++ b/tests/test-llama-archs.cpp
@@ -337,7 +337,8 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe, const
if (arch == LLM_ARCH_QWEN3) {
n_head_kv = 1; // MQA coverage
} else if (arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_AFMOE || arch == LLM_ARCH_MAION_CODER ||
- arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_LIMITE || arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_RICURSIVE) {
+ arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_LIMITE || arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_RICURSIVE ||
+ arch == LLM_ARCH_ZAYA1) {
n_head_kv = 2; // GQA coverage
}
if (arch == LLM_ARCH_BEAM) {
@@ -744,7 +745,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe, const
ms.add_kv(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, uint32_t(2));
// with more experts than a ubatch uses, the copy of the used experts in host memory skips some of them
ms.add_kv(LLM_KV_EXPERT_COUNT, uint32_t(host_experts_test(arch) ? 64 : arch == LLM_ARCH_CLOVER_MOE ? 4 : 2));
- ms.add_kv(LLM_KV_EXPERT_USED_COUNT, uint32_t(2));
+ ms.add_kv(LLM_KV_EXPERT_USED_COUNT, uint32_t(arch == LLM_ARCH_ZAYA1 ? 1 : 2)); // ZAYA routes top-1
ms.add_kv(LLM_KV_EXPERT_SHARED_COUNT, uint32_t(1));
uint32_t gating_func = 2;
@@ -799,7 +800,8 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe, const
ms.add_kv(LLM_KV_XIELU_BETA, 1.0f);
ms.add_kv(LLM_KV_XIELU_EPS, 1.0e-7f);
ms.add_kv(LLM_KV_SSM_INNER_SIZE, arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_QWEN4EXP || arch == LLM_ARCH_CLOVER_MOE || arch == LLM_ARCH_ZONOS3 ? 256 : 2*n_embd);
- ms.add_kv(LLM_KV_SSM_CONV_KERNEL, uint32_t(4));
+ ms.add_kv(LLM_KV_ROUTER_HIDDEN_SIZE, n_embd);
+ ms.add_kv(LLM_KV_SSM_CONV_KERNEL, uint32_t(arch == LLM_ARCH_ZAYA1 ? 2 : 4)); // ZAYA CCA convs have kernel 2
ms.add_kv(LLM_KV_SSM_STATE_SIZE, uint32_t(128));
ms.add_kv(LLM_KV_SSM_TIME_STEP_RANK, n_head);
ms.add_kv(LLM_KV_SSM_GROUP_COUNT, arch == LLM_ARCH_PLAMO2 ? 0 : uint32_t(2));
@@ -1074,6 +1076,7 @@ static double check_draft_rollback(llama_model * model, const std::vector<llama_
static bool moe_mandatory(const llm_arch arch) {
switch (arch) {
case LLM_ARCH_FLINT:
+ case LLM_ARCH_ZAYA1:
case LLM_ARCH_LLAMA4:
case LLM_ARCH_MAION_CODER:
case LLM_ARCH_COHERE2MOE: