Commit 98d67c8d5 for llama.cpp

commit 98d67c8d536dc584897c4adb2510509796517222
Author: varun <varun10221@gmail.com>
Date:   Wed Nov 18 06:46:49 2026 -0800

    model:add Zaya1 8B (Zyphra) (#32071)

    * model:add Zaya1 8B (Zyphra)
    - Added the support for Zaya.
    - Fixed the SSM_CONV case, so the model runs on cuda without abort.

     Please enter the commit message for your changes. Lines starting

    * Made the following changes based on the bot reviewer feedback:
    1) Added runtime asserts when loading wrong KV head size, unsupported conv kernel size.
    2) Removed the static cast usage.
    3) Fixed double load of tokenizer.
    4) Updated res_scale_in to be input tensor.

    * - Moved the res_scale_in back to layer 0 to fix the cuda bug

    * i)  Removed the EOS addition.Team agreed that updating Tokenizer is the right move.
    ii) Fixed formatting

    * Fixed the comment to make it clear

    ---------

    Co-authored-by: varun <varun_subramanian@zyphra.com>

diff --git a/conversion/__init__.py b/conversion/__init__.py
index 881111e9f..634cafa24 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -340,6 +340,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "XverseForCausalLM": "xverse",
     "YoutuForCausalLM": "deepseek",
     "YoutuVLForConditionalGeneration": "deepseek",
+    "ZayaForCausalLM": "zaya1",
     "fish_qwen3_omni": "fish_s2",
     "modeling_grove_moe.GroveMoeForCausalLM": "grovemoe",
     "modeling_sarvam_moe.SarvamMoEForCausalLM": "bailingmoe",
diff --git a/conversion/zaya1.py b/conversion/zaya1.py
new file mode 100644
index 000000000..803fc908a
--- /dev/null
+++ b/conversion/zaya1.py
@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+import json
+
+import torch
+
+from typing import Callable, Iterable, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from torch import Tensor
+
+from .base import ModelBase, TextModel, gguf
+
+
+@ModelBase.register("ZayaForCausalLM")
+@ModelBase.example("Zyphra/ZAYA1-8B")
+class ZayaModel(TextModel):
+    model_arch = gguf.MODEL_ARCH.ZAYA1
+
+    # checkpoint names without a .weight/.bias suffix
+    _RENAME = {
+        "model.input_hidden_states_scale": "model.input_hidden_states.weight",
+        "model.input_hidden_states_bias":  "model.input_hidden_states.bias",
+    }
+
+    # the two kv heads of attn_v: head 0 = current token, head 1 = previous token
+    _V_PARTS = (
+        "self_attn.qkv_proj.v_proj_current.weight",
+        "self_attn.qkv_proj.v_proj_delayed.weight",
+    )
+
+    # (hs + b_hs) * s_hs + (res + b_res) * s_res = hs * s_hs + res * s_res + (b_hs * s_hs + b_res * s_res)
+    _RES_SCALES = {
+        "post_attention_residual_scale.": (gguf.MODEL_TENSOR.ATTN_RES_SCALE_HS, gguf.MODEL_TENSOR.ATTN_RESID_SCALE),
+        "post_mlp_residual_scale.":       (gguf.MODEL_TENSOR.FFN_RES_SCALE_HS,  gguf.MODEL_TENSOR.FFN_RESID_SCALE),
+    }
+    _RES_PARTS = ("hidden_states_scale", "hidden_states_bias", "residual_scale", "residual_bias")
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # added tokens overlap the base vocab, so count the token list itself
+        self._vocab = gguf.LlamaHfVocab(self.dir_model)
+        self._n_vocab = self._vocab.vocab_size
+        self._parts: dict[tuple[int, str], dict[str, Tensor]] = {}
+        self.rope_parameters = self.hparams["rope_parameters"]["hybrid"]
+
+    def set_vocab(self):
+        # added special tokens overlap the base vocab, so mark them as control here
+        with open(self.dir_model / "tokenizer.json", "r", encoding="utf-8") as f:
+            added = {t["id"]: t["special"] for t in json.load(f)["added_tokens"]}
+
+        tokens, scores, toktypes = [], [], []
+        for i, (text, score, toktype) in enumerate(self._vocab.all_tokens()):
+            tokens.append(text)
+            scores.append(score)
+            if added.get(i):
+                toktype = gguf.TokenType.CONTROL
+            toktypes.append(toktype)
+
+        self.gguf_writer.add_tokenizer_model("gemma4")
+        self.gguf_writer.add_token_list(tokens)
+        self.gguf_writer.add_token_scores(scores)
+        self.gguf_writer.add_token_types(toktypes)
+
+        special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
+        special_vocab.add_to_gguf(self.gguf_writer)
+        self.gguf_writer.add_add_space_prefix(False)
+
+    def set_gguf_parameters(self):
+        hparams = self.hparams
+        super().set_gguf_parameters()
+
+        self.gguf_writer.add_rope_dimension_count(int(self.rope_parameters["partial_rotary_factor"] * hparams["head_dim"]))
+        self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"])
+        self.gguf_writer.add_ssm_conv_kernel(hparams["cca_time0"])
+        self.gguf_writer.add_router_hidden_size(hparams["router_hidden_size"])
+
+    @classmethod
+    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+        name, _ = item
+        # skip the draft head tensors now, they will be added as part of the TiDAR changes
+        if name.startswith(("diffusion_output_layer.", "diffusion_markov_head.")):
+            return None
+
+        return super().filter_tensors(item)
+
+    def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
+        # the router picks one expert, so its matmuls stay F32 like ffn_gate_inp
+        if any(self.match_model_tensor_name(new_name, key, bid) for key in (
+                gguf.MODEL_TENSOR.CCA_CONV_GRP,
+                gguf.MODEL_TENSOR.ROUTER_FC1,
+                gguf.MODEL_TENSOR.ROUTER_FC2,
+                gguf.MODEL_TENSOR.ROUTER_OUT)):
+            return gguf.GGMLQuantizationType.F32
+        return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        if bid is not None:
+            local = name.removeprefix(f"model.layers.{bid}.")
+            if local in self._V_PARTS:
+                p = self._parts.setdefault((bid, "v"), {})
+                p[local] = data_torch
+                if len(p) == len(self._V_PARTS):
+                    del self._parts[(bid, "v")]
+                    v = torch.cat([p[n] for n in self._V_PARTS], dim=0)
+                    yield from super().modify_tensors(v, f"model.layers.{bid}.{self._V_PARTS[0]}", bid)
+                return
+            for prefix, (key_hs, key_res) in self._RES_SCALES.items():
+                if local.startswith(prefix) and local[len(prefix):] in self._RES_PARTS:
+                    p = self._parts.setdefault((bid, prefix), {})
+                    p[local[len(prefix):]] = data_torch
+                    if len(p) == len(self._RES_PARTS):
+                        del self._parts[(bid, prefix)]
+                        s_hs, b_hs, s_res, b_res = (p[n].float() for n in self._RES_PARTS)
+                        yield self.format_tensor_name(key_hs,  bid, ".weight"), s_hs
+                        yield self.format_tensor_name(key_res, bid, ".weight"), s_res
+                        yield self.format_tensor_name(key_res, bid, ".bias"),   b_hs * s_hs + b_res * s_res
+                    return
+
+        name = self._RENAME.get(name, name)
+        if name.endswith("balancing_biases"):
+            name += ".bias"
+        elif not name.endswith((".weight", ".bias")):
+            name += ".weight"
+        new_name = self.map_tensor_name(name)
+
+        def is_tensor(key: gguf.MODEL_TENSOR) -> bool:
+            return self.match_model_tensor_name(new_name, key, bid)
+
+        if any(is_tensor(key) for key in (gguf.MODEL_TENSOR.ATTN_Q, gguf.MODEL_TENSOR.ATTN_K)):
+            # through the base class, which fuses q, k and v with --fuse-qkv
+            yield from super().modify_tensors(data_torch, name, bid)
+            return
+        if is_tensor(gguf.MODEL_TENSOR.TOKEN_EMBD):
+            data_torch = data_torch[:self._n_vocab]
+        elif is_tensor(gguf.MODEL_TENSOR.SSM_CONV1D):
+            data_torch = data_torch.squeeze(1)
+        elif is_tensor(gguf.MODEL_TENSOR.CCA_CONV_GRP):
+            # [n_group*out, in, kernel] -> [kernel, n_group, out, in]
+            n_group = self.hparams["num_attention_heads"] + self.hparams["num_key_value_heads"]
+            n_out, n_in, n_k = data_torch.shape
+            data_torch = data_torch.reshape(n_group, n_out // n_group, n_in, n_k).permute(3, 0, 1, 2).contiguous()
+
+        yield new_name, data_torch
diff --git a/ggml/src/ggml-cuda/ssm-conv.cu b/ggml/src/ggml-cuda/ssm-conv.cu
index f41438b31..0e77a3b22 100644
--- a/ggml/src/ggml-cuda/ssm-conv.cu
+++ b/ggml/src/ggml-cuda/ssm-conv.cu
@@ -148,13 +148,14 @@ static void ssm_conv_f32_cuda(const float * src0, const float * src1, const floa
     };

     switch (nc) {
+        case 2:  launch_kernel(std::integral_constant<int, 2 >{}); break;
         case 3:  launch_kernel(std::integral_constant<int, 3 >{}); break;
         case 4:  launch_kernel(std::integral_constant<int, 4 >{}); break;
         case 5:  launch_kernel(std::integral_constant<int, 5 >{}); break;
         case 7:  launch_kernel(std::integral_constant<int, 7 >{}); break;
         case 9:  launch_kernel(std::integral_constant<int, 9 >{}); break;
         case 15: launch_kernel(std::integral_constant<int, 15>{}); break;
-        default: GGML_ABORT("Only support kernel sizes 3, 4, 5, 7, 9, 15 right now.");
+        default: GGML_ABORT("Only support kernel sizes 2, 3, 4, 5, 7, 9, 15 right now.");
     }
 }

diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index c33a579b0..e9e1866e4 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -143,6 +143,7 @@ class Keys:
         DECODER_BLOCK_COUNT               = "{arch}.decoder_block_count"
         ATTN_LOGIT_SOFTCAPPING            = "{arch}.attn_logit_softcapping"
         ROUTER_LOGIT_SOFTCAPPING          = "{arch}.router_logit_softcapping"
+        ROUTER_HIDDEN_SIZE                = "{arch}.router_hidden_size"
         FINAL_LOGIT_SOFTCAPPING           = "{arch}.final_logit_softcapping"
         FINAL_LOGIT_SIGMOID_CAPPING       = "{arch}.final_logit_sigmoid_capping"
         SWIN_NORM                         = "{arch}.swin_norm"
@@ -736,6 +737,7 @@ class MODEL_ARCH(IntEnum):
     LFM2             = auto()
     LFM2MOE          = auto()
     LFM3             = auto()
+    ZAYA1            = auto()
     DREAM            = auto()
     SMALLTHINKER     = auto()
     LLADA            = auto()
@@ -1533,6 +1535,17 @@ class MODEL_TENSOR(IntEnum):
     LILICORR_OUT_HEAD             = auto()
     LILICORR_IN_HEAD              = auto()
     LILICORR_ANCHOR_OUT_HEAD      = auto()
+    # zaya1
+    CCA_CONV_GRP           = auto()
+    CCA_TEMP               = auto()
+    RES_SCALE_IN           = auto()
+    ATTN_RES_SCALE_HS      = auto()
+    FFN_RES_SCALE_HS       = auto()
+    ROUTER_EDA             = auto()
+    ROUTER_NORM            = auto()
+    ROUTER_FC1             = auto()
+    ROUTER_FC2             = auto()
+    ROUTER_OUT             = auto()
     # lfm2 audio
     A_ENC_NORM_CONV        = auto()
     A_ENC_LINEAR_POS       = auto()
@@ -1724,6 +1737,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
     MODEL_ARCH.LFM2:             "lfm2",
     MODEL_ARCH.LFM2MOE:          "lfm2moe",
     MODEL_ARCH.LFM3:             "lfm3",
+    MODEL_ARCH.ZAYA1:            "zaya1",
     MODEL_ARCH.DREAM:            "dream",
     MODEL_ARCH.SMALLTHINKER:     "smallthinker",
     MODEL_ARCH.LLADA:            "llada",
@@ -2559,6 +2573,16 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
     MODEL_TENSOR.LILICORR_OUT_HEAD:                     "lilicorr.out_head",
     MODEL_TENSOR.LILICORR_IN_HEAD:                      "lilicorr.in_head",
     MODEL_TENSOR.LILICORR_ANCHOR_OUT_HEAD:              "lilicorr.anchor_out_head",
+    MODEL_TENSOR.CCA_CONV_GRP:              "blk.{bid}.cca_conv_grp",
+    MODEL_TENSOR.CCA_TEMP:                  "blk.{bid}.cca_temp",
+    MODEL_TENSOR.RES_SCALE_IN:              "res_scale_in",
+    MODEL_TENSOR.ATTN_RES_SCALE_HS:         "blk.{bid}.attn_res_scale_hs",
+    MODEL_TENSOR.FFN_RES_SCALE_HS:          "blk.{bid}.ffn_res_scale_hs",
+    MODEL_TENSOR.ROUTER_EDA:                "blk.{bid}.ffn_router_eda",
+    MODEL_TENSOR.ROUTER_NORM:               "blk.{bid}.ffn_router_norm",
+    MODEL_TENSOR.ROUTER_FC1:                "blk.{bid}.ffn_router_fc1",
+    MODEL_TENSOR.ROUTER_FC2:                "blk.{bid}.ffn_router_fc2",
+    MODEL_TENSOR.ROUTER_OUT:                "blk.{bid}.ffn_router_out",
     MODEL_TENSOR.D2T:                       "d2t",
     MODEL_TENSOR.ATTN_V_GATE:               "blk.{bid}.attn_v_gate",
     MODEL_TENSOR.ATTN_V_EXP:                "blk.{bid}.attn_v_exps",
@@ -6130,6 +6154,34 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.ATTN_OUT,
         MODEL_TENSOR.ATTN_GATE,
     ],
+    MODEL_ARCH.ZAYA1: [
+        MODEL_TENSOR.TOKEN_EMBD,
+        MODEL_TENSOR.RES_SCALE_IN,
+        MODEL_TENSOR.OUTPUT_NORM,
+        MODEL_TENSOR.ATTN_NORM,
+        MODEL_TENSOR.ATTN_Q,
+        MODEL_TENSOR.ATTN_K,
+        MODEL_TENSOR.ATTN_V,
+        MODEL_TENSOR.ATTN_QKV,
+        MODEL_TENSOR.SSM_CONV1D,
+        MODEL_TENSOR.CCA_CONV_GRP,
+        MODEL_TENSOR.CCA_TEMP,
+        MODEL_TENSOR.ATTN_OUT,
+        MODEL_TENSOR.ATTN_RES_SCALE_HS,
+        MODEL_TENSOR.ATTN_RESID_SCALE,
+        MODEL_TENSOR.FFN_NORM,
+        MODEL_TENSOR.FFN_GATE_INP,
+        MODEL_TENSOR.ROUTER_EDA,
+        MODEL_TENSOR.ROUTER_NORM,
+        MODEL_TENSOR.ROUTER_FC1,
+        MODEL_TENSOR.ROUTER_FC2,
+        MODEL_TENSOR.ROUTER_OUT,
+        MODEL_TENSOR.FFN_EXP_PROBS_B,
+        MODEL_TENSOR.FFN_GATE_UP_EXP,
+        MODEL_TENSOR.FFN_DOWN_EXP,
+        MODEL_TENSOR.FFN_RES_SCALE_HS,
+        MODEL_TENSOR.FFN_RESID_SCALE,
+    ],
     MODEL_ARCH.SMALLTHINKER: [
         MODEL_TENSOR.TOKEN_EMBD,
         MODEL_TENSOR.OUTPUT_NORM,
diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py
index edf2caf88..eb0bd91d0 100644
--- a/gguf-py/gguf/gguf_writer.py
+++ b/gguf-py/gguf/gguf_writer.py
@@ -885,6 +885,9 @@ class GGUFWriter:
     def add_router_logit_softcapping(self, value: float) -> None:
         self.add_float32(Keys.LLM.ROUTER_LOGIT_SOFTCAPPING.format(arch=self.arch), value)

+    def add_router_hidden_size(self, value: int) -> None:
+        self.add_uint32(Keys.LLM.ROUTER_HIDDEN_SIZE.format(arch=self.arch), value)
+
     def add_final_logit_softcapping(self, value: float) -> None:
         self.add_float32(Keys.LLM.FINAL_LOGIT_SOFTCAPPING.format(arch=self.arch), value)

diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py
index 61e4e7ddc..c844a9322 100644
--- a/gguf-py/gguf/tensor_mapping.py
+++ b/gguf-py/gguf/tensor_mapping.py
@@ -228,6 +228,9 @@ class TensorNameMap:
         MODEL_TENSOR.LILICORR_ANCHOR_OUT_HEAD: (
             "model.lilicorr.anchor_out_head",
         ),
+        MODEL_TENSOR.RES_SCALE_IN: (
+            "model.input_hidden_states",  # zaya1
+        ),
     }

     block_mappings_cfg: dict[MODEL_TENSOR, tuple[str, ...]] = {
@@ -3350,6 +3353,22 @@ class TensorNameMap:

     # architecture-specific block mappings
     arch_block_mappings_cfg: dict[MODEL_ARCH, dict[MODEL_TENSOR, tuple[str, ...]]] = {
+        MODEL_ARCH.ZAYA1: {
+            MODEL_TENSOR.ATTN_Q:        ("model.layers.{bid}.self_attn.qkv_proj.q_proj",),
+            MODEL_TENSOR.ATTN_K:        ("model.layers.{bid}.self_attn.qkv_proj.k_proj",),
+            # the converter joins v_proj_delayed onto v_proj_current, attn_v holds both kv heads
+            MODEL_TENSOR.ATTN_V:        ("model.layers.{bid}.self_attn.qkv_proj.v_proj_current",),
+            MODEL_TENSOR.SSM_CONV1D:    ("model.layers.{bid}.self_attn.qkv_proj.conv_qk_depthwise",),
+            MODEL_TENSOR.CCA_CONV_GRP:  ("model.layers.{bid}.self_attn.qkv_proj.conv_qk_grouped",),
+            MODEL_TENSOR.CCA_TEMP:      ("model.layers.{bid}.self_attn.qk_norm.temp",),
+            MODEL_TENSOR.FFN_GATE_INP:  ("model.layers.{bid}.mlp.gate.down_proj",),
+            MODEL_TENSOR.FFN_EXP_PROBS_B: ("model.layers.{bid}.mlp.gate.balancing_biases",),
+            MODEL_TENSOR.ROUTER_EDA:    ("model.layers.{bid}.mlp.gate.router_states_scale",),
+            MODEL_TENSOR.ROUTER_NORM:   ("model.layers.{bid}.mlp.gate.router_mlp.norm",),
+            MODEL_TENSOR.ROUTER_FC1:    ("model.layers.{bid}.mlp.gate.router_mlp.fc1",),
+            MODEL_TENSOR.ROUTER_FC2:    ("model.layers.{bid}.mlp.gate.router_mlp.fc2",),
+            MODEL_TENSOR.ROUTER_OUT:    ("model.layers.{bid}.mlp.gate.router_mlp.out_proj",),
+        },
         MODEL_ARCH.ARCTIC: {
             MODEL_TENSOR.FFN_NORM: (
                 "model.layers.{bid}.residual_layernorm",
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index feee62bd4..c50e9f98a 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -139,6 +139,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
     { LLM_ARCH_LFM2,             "lfm2"             },
     { LLM_ARCH_LFM2MOE,          "lfm2moe"          },
     { LLM_ARCH_LFM3,             "lfm3"             },
+    { LLM_ARCH_ZAYA1,            "zaya1"            },
     { LLM_ARCH_DREAM,            "dream"            },
     { LLM_ARCH_SMALLTHINKER,     "smallthinker"     },
     { LLM_ARCH_LLADA,            "llada"            },
@@ -260,6 +261,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
     { LLM_KV_DECODER_BLOCK_COUNT,               "%s.decoder_block_count"               },
     { LLM_KV_ATTN_LOGIT_SOFTCAPPING,            "%s.attn_logit_softcapping"            },
     { LLM_KV_ROUTER_LOGIT_SOFTCAPPING,          "%s.router_logit_softcapping"          },
+    { LLM_KV_ROUTER_HIDDEN_SIZE,                "%s.router_hidden_size"                },
     { LLM_KV_FINAL_LOGIT_SOFTCAPPING,           "%s.final_logit_softcapping"           },
     { LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING,       "%s.final_logit_sigmoid_capping"       },
     { LLM_KV_SWIN_NORM,                         "%s.swin_norm"                         },
@@ -912,6 +914,16 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
     { LLM_TENSOR_LILICORR_OUT_HEAD,                      "lilicorr.out_head" },
     { LLM_TENSOR_LILICORR_IN_HEAD,                       "lilicorr.in_head" },
     { LLM_TENSOR_LILICORR_ANCHOR_OUT_HEAD,               "lilicorr.anchor_out_head" },
+    { LLM_TENSOR_CCA_CONV_GRP,                           "blk.%d.cca_conv_grp" },
+    { LLM_TENSOR_CCA_TEMP,                               "blk.%d.cca_temp" },
+    { LLM_TENSOR_RES_SCALE_IN,                           "res_scale_in" },
+    { LLM_TENSOR_ATTN_RES_SCALE_HS,                      "blk.%d.attn_res_scale_hs" },
+    { LLM_TENSOR_FFN_RES_SCALE_HS,                       "blk.%d.ffn_res_scale_hs" },
+    { LLM_TENSOR_ROUTER_EDA,                             "blk.%d.ffn_router_eda" },
+    { LLM_TENSOR_ROUTER_NORM,                            "blk.%d.ffn_router_norm" },
+    { LLM_TENSOR_ROUTER_FC1,                             "blk.%d.ffn_router_fc1" },
+    { LLM_TENSOR_ROUTER_FC2,                             "blk.%d.ffn_router_fc2" },
+    { LLM_TENSOR_ROUTER_OUT,                             "blk.%d.ffn_router_out" },
     { LLM_TENSOR_ATTN_V_GATE,                            "blk.%d.attn_v_gate" },
     { LLM_TENSOR_ATTN_V_EXPS,                            "blk.%d.attn_v_exps" },
 };
@@ -1317,6 +1329,17 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
     {LLM_TENSOR_LILICORR_IN_HEAD,           {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL_MAT}},
     {LLM_TENSOR_LILICORR_ANCHOR_OUT_HEAD,   {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL_MAT}},

+    {LLM_TENSOR_CCA_CONV_GRP,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+    {LLM_TENSOR_CCA_TEMP,                   {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+    {LLM_TENSOR_RES_SCALE_IN,               {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+    {LLM_TENSOR_ATTN_RES_SCALE_HS,          {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+    {LLM_TENSOR_FFN_RES_SCALE_HS,           {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+    {LLM_TENSOR_ROUTER_EDA,                 {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+    {LLM_TENSOR_ROUTER_NORM,                {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+    {LLM_TENSOR_ROUTER_FC1,                 {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+    {LLM_TENSOR_ROUTER_FC2,                 {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+    {LLM_TENSOR_ROUTER_OUT,                 {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+
     {LLM_TENSOR_ATTN_V_GATE,                {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
     {LLM_TENSOR_ATTN_V_EXPS,                {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT_ID}},
 };
@@ -1405,6 +1428,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) {
         case LLM_ARCH_LFM2:
         case LLM_ARCH_LFM2MOE:
         case LLM_ARCH_LFM3:
+        case LLM_ARCH_ZAYA1:
         case LLM_ARCH_NEMOTRON_H:
         case LLM_ARCH_NEMOTRON_H_MOE:
         case LLM_ARCH_METIS16:
@@ -1486,6 +1510,7 @@ bool llm_arch_supports_mixed_batch(const llm_arch & arch) {
 bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
     switch (arch) {
         case LLM_ARCH_CLOVER_MOE:
+        case LLM_ARCH_ZAYA1:
         case LLM_ARCH_GROK:
         case LLM_ARCH_MPT:
         case LLM_ARCH_PLAMO2:
diff --git a/src/llama-arch.h b/src/llama-arch.h
index 4187a0d08..9c517a9fe 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -143,6 +143,7 @@ enum llm_arch {
     LLM_ARCH_LFM2,
     LLM_ARCH_LFM2MOE,
     LLM_ARCH_LFM3,
+    LLM_ARCH_ZAYA1,
     LLM_ARCH_DREAM,
     LLM_ARCH_SMALLTHINKER,
     LLM_ARCH_LLADA,
@@ -264,6 +265,7 @@ enum llm_kv {
     LLM_KV_DECODER_BLOCK_COUNT,
     LLM_KV_ATTN_LOGIT_SOFTCAPPING,
     LLM_KV_ROUTER_LOGIT_SOFTCAPPING,
+    LLM_KV_ROUTER_HIDDEN_SIZE,
     LLM_KV_FINAL_LOGIT_SOFTCAPPING,
     LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING,
     LLM_KV_SWIN_NORM,
@@ -917,6 +919,16 @@ enum llm_tensor {
     LLM_TENSOR_LILICORR_OUT_HEAD,
     LLM_TENSOR_LILICORR_IN_HEAD,
     LLM_TENSOR_LILICORR_ANCHOR_OUT_HEAD,
+    LLM_TENSOR_CCA_CONV_GRP,
+    LLM_TENSOR_CCA_TEMP,
+    LLM_TENSOR_RES_SCALE_IN,
+    LLM_TENSOR_ATTN_RES_SCALE_HS,
+    LLM_TENSOR_FFN_RES_SCALE_HS,
+    LLM_TENSOR_ROUTER_EDA,
+    LLM_TENSOR_ROUTER_NORM,
+    LLM_TENSOR_ROUTER_FC1,
+    LLM_TENSOR_ROUTER_FC2,
+    LLM_TENSOR_ROUTER_OUT,
     LLM_TENSOR_ATTN_V_GATE,
     LLM_TENSOR_ATTN_V_EXPS,
 };
diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cpp
index 66903fbb7..807f7987f 100644
--- a/src/llama-hparams.cpp
+++ b/src/llama-hparams.cpp
@@ -221,6 +221,11 @@ uint32_t llama_hparams::n_embd_r() const {
         return n_embd * (n_shortconv_l_cache - 1);
     }

+    if (n_cca_qk != 0) {
+        // for ZAYA1 CCA layers: depthwise + grouped conv, each with d_conv - 1 columns of history
+        return 2 * (ssm_d_conv - 1) * n_cca_qk;
+    }
+
     if (n_embd_head_kda != 0) {
         // for Kimi KDA layers
         // Conv state for Q, K, V: 3 * (d_conv - 1) * n_head * head_dim
@@ -244,6 +249,11 @@ uint32_t llama_hparams::n_embd_s() const {
         return n_embd * wkv_head_size;
     }

+    if (n_cca_qk != 0) {
+        // for ZAYA CCA layers: the delayed value of the previous token
+        return n_embd_v_gqa() / 2;
+    }
+
     if (n_embd_head_kda != 0) {
         // for Kimi KDA layers
         // Full recurrent state: head_dim * head_dim * n_head
diff --git a/src/llama-hparams.h b/src/llama-hparams.h
index b4894956c..642f90118 100644
--- a/src/llama-hparams.h
+++ b/src/llama-hparams.h
@@ -221,6 +221,12 @@ struct llama_hparams {
     uint32_t n_embd_head_kda = 0;
     bool     kda_safe_gate = false;

+    // for ZAYA1 CCA layers: q + k channels in the conv state
+    uint32_t n_cca_qk = 0;
+
+    // for ZAYA1: width of the router MLP
+    uint32_t n_router_hidden = 0;
+
     // kimi-k3
     uint32_t n_expert_latent      = 0;      // routed_expert_hidden_size (0 = experts run at n_embd)
     uint32_t attn_res_block_size  = 0;      // 0 = no cross-layer attention residuals
diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp
index 338baa8f7..c01d4dd51 100644
--- a/src/llama-model-saver.cpp
+++ b/src/llama-model-saver.cpp
@@ -241,6 +241,7 @@ void llama_model_saver::add_kv_from_model() {
     add_kv(LLM_KV_DECODER_BLOCK_COUNT,               hparams.dec_n_layer);
     add_kv(LLM_KV_ATTN_LOGIT_SOFTCAPPING,            hparams.f_attn_logit_softcapping);
     add_kv(LLM_KV_ROUTER_LOGIT_SOFTCAPPING,          hparams.f_router_logit_softcapping);
+    add_kv(LLM_KV_ROUTER_HIDDEN_SIZE,                hparams.n_router_hidden);
     add_kv(LLM_KV_FINAL_LOGIT_SOFTCAPPING,           hparams.f_final_logit_softcapping);
     add_kv(LLM_KV_SWIN_NORM,                         hparams.swin_norm);
     add_kv(LLM_KV_RESCALE_EVERY_N_LAYERS,            hparams.rescale_every_n_layers);
@@ -578,6 +579,8 @@ void llama_model_saver::add_tensors_from_model() {
     add_tensor(model->hc_head_norm);
     add_tensor(model->hc_head_down);
     add_tensor(model->hc_head_up);
+    add_tensor(model->res_scale_in);
+    add_tensor(model->res_scale_in_b);

     // looped architectures alias physical tensors across cache slots; save each
     // tensor once. a different tensor with an existing name still asserts below
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 728473fff..ac5266a42 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -332,6 +332,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
             return new llama_model_lfm2moe(params);
         case LLM_ARCH_LFM3:
             return new llama_model_lfm3(params);
+        case LLM_ARCH_ZAYA1:
+            return new llama_model_zaya1(params);
         case LLM_ARCH_SMALLTHINKER:
             return new llama_model_smallthinker(params);
         case LLM_ARCH_GROVEMOE:
@@ -3349,7 +3351,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
                         filter_idx = [&](uint32_t il) {
                             return il < hparams.n_layer() && !hparams.is_recr(il) && hparams.is_indexer_full(il);
                         };
-                    } else if (arch == LLM_ARCH_FALCON_H1 || arch == LLM_ARCH_INKLING2) {
+                    } else if (arch == LLM_ARCH_FALCON_H1 || arch == LLM_ARCH_INKLING2 || arch == LLM_ARCH_ZAYA1) {
                         filter_attn = [&](uint32_t) { return true; };
                         filter_recr = [&](uint32_t) { return true; };
                     } else if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) {
@@ -3847,6 +3849,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
         case LLM_ARCH_LFM2:
         case LLM_ARCH_LFM2MOE:
         case LLM_ARCH_LFM3:
+        case LLM_ARCH_ZAYA1:
         case LLM_ARCH_SMALLTHINKER:
         case LLM_ARCH_SEED_OSS:
         case LLM_ARCH_GROVEMOE:
diff --git a/src/llama-model.h b/src/llama-model.h
index 27c804eb6..8d2f5629e 100644
--- a/src/llama-model.h
+++ b/src/llama-model.h
@@ -226,6 +226,23 @@ struct llama_layer_shortconv {
     struct ggml_tensor * out_proj = nullptr;
 };

+struct llama_layer_zaya1 {
+    struct ggml_tensor * cca_conv_grp   = nullptr;
+    struct ggml_tensor * cca_conv_grp_b = nullptr;
+    struct ggml_tensor * cca_temp       = nullptr;
+
+    struct ggml_tensor * attn_res_hs = nullptr;
+    struct ggml_tensor * ffn_res_hs  = nullptr;
+
+    struct ggml_tensor * router_eda     = nullptr;
+    struct ggml_tensor * router_norm    = nullptr;
+    struct ggml_tensor * router_fc1     = nullptr;
+    struct ggml_tensor * router_fc1_b   = nullptr;
+    struct ggml_tensor * router_fc2     = nullptr;
+    struct ggml_tensor * router_fc2_b   = nullptr;
+    struct ggml_tensor * router_out     = nullptr;
+};
+
 struct llama_layer_nextn {
     struct ggml_tensor * eh_proj               = nullptr;
     struct ggml_tensor * eh_proj_s             = nullptr;
@@ -667,8 +684,10 @@ struct llama_layer {
     // limite
     struct ggml_tensor * attn_ve_gate     = nullptr;
     struct ggml_tensor * attn_xsa_alpha   = nullptr;
-    struct ggml_tensor * attn_resid_scale = nullptr;
-    struct ggml_tensor * ffn_resid_scale  = nullptr;
+    struct ggml_tensor * attn_resid_scale   = nullptr;
+    struct ggml_tensor * attn_resid_scale_b = nullptr;
+    struct ggml_tensor * ffn_resid_scale    = nullptr;
+    struct ggml_tensor * ffn_resid_scale_b  = nullptr;
     struct ggml_tensor * mudd_down        = nullptr;
     struct ggml_tensor * mudd_up          = nullptr;
     struct ggml_tensor * mudd_up_b        = nullptr;
@@ -697,6 +716,8 @@ struct llama_layer {

     struct llama_layer_shortconv shortconv;

+    struct llama_layer_zaya1 zaya1;
+
     struct llama_layer_nextn nextn;

     struct llama_layer_switch_lora switch_lora;
@@ -909,6 +930,10 @@ struct llama_model {
     struct ggml_tensor * dspark_joint_proj_b = nullptr;
     struct ggml_tensor * dspark_mask_embd    = nullptr; // per-slot mask embeddings

+    // zaya1
+    struct ggml_tensor * res_scale_in   = nullptr;
+    struct ggml_tensor * res_scale_in_b = nullptr;
+
     struct ggml_tensor * dflash_selector_prev   = nullptr;
     struct ggml_tensor * dflash_selector_next   = nullptr;
     struct ggml_tensor * dflash_selector_hidden = nullptr;
diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp
index a50d3c237..0f3f982b0 100644
--- a/src/llama-quant.cpp
+++ b/src/llama-quant.cpp
@@ -348,6 +348,12 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param
         quantize &= name.find("attn_rel_bias.weight") == std::string::npos;
     }

+    // zaya1: the top-1 router and the grouped conv kernels stay F32
+    if (arch == LLM_ARCH_ZAYA1) {
+        quantize &= name.find("ffn_router_")  == std::string::npos;
+        quantize &= name.find("cca_conv_grp") == std::string::npos;
+    }
+
     // do not quantize positional embeddings and token types (BERT)
     quantize &= name != LLM_TN(arch)(LLM_TENSOR_POS_EMBD,    "weight");
     quantize &= name != LLM_TN(arch)(LLM_TENSOR_TOKEN_TYPES, "weight");
diff --git a/src/models/models.h b/src/models/models.h
index 8da3bcf29..8d74c5018 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -2535,6 +2535,22 @@ struct llama_model_lfm3 : public llama_model_base {
     std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
 };

+struct llama_model_zaya1 : public llama_model_base {
+    llama_model_zaya1(const struct llama_model_params & params) : llama_model_base(params) {}
+    void load_arch_hparams(llama_model_loader & ml) override;
+    void load_arch_tensors(llama_model_loader & ml) override;
+
+    struct graph : public llm_graph_context {
+        graph(const llama_model & model, const llm_graph_params & params);
+
+        ggml_tensor * build_cca_block(const llama_model & model, ggml_tensor * cur, ggml_tensor * inp_pos,
+                llm_graph_input_mem_hybrid * inp, int il);
+        ggml_tensor * build_router_moe(const llama_model & model, ggml_tensor * cur, ggml_tensor *& router_state, int il);
+    };
+
+    std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
 struct llama_model_smallthinker : public llama_model_base {
     llama_model_smallthinker(const struct llama_model_params & params) : llama_model_base(params) {}
     void load_arch_hparams(llama_model_loader & ml) override;
diff --git a/src/models/zaya1.cpp b/src/models/zaya1.cpp
new file mode 100644
index 000000000..2763500f4
--- /dev/null
+++ b/src/models/zaya1.cpp
@@ -0,0 +1,340 @@
+#include "models.h"
+#include "llama-memory-recurrent.h"
+
+void llama_model_zaya1::load_arch_hparams(llama_model_loader & ml) {
+    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+    ml.get_key(LLM_KV_SSM_CONV_KERNEL,             hparams.ssm_d_conv);
+    ml.get_key(LLM_KV_ROUTER_HIDDEN_SIZE,          hparams.n_router_hidden);
+    ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);
+
+    if (hparams.ssm_d_conv != 2) {
+        throw std::runtime_error("zaya1: only ssm.conv_kernel == 2 is supported");
+    }
+    // the two kv heads are v_cur and v_del, and the router picks one expert
+    for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
+        if (hparams.n_head_kv(il) != 2 || hparams.n_embd_head_v(il) != hparams.n_embd_head_k(il) || hparams.n_expert_used(il) != 1) {
+            throw std::runtime_error("zaya1: only 2 kv heads of equal k/v size and 1 used expert are supported");
+        }
+    }
+
+    hparams.n_cca_qk = (hparams.n_head() + hparams.n_head_kv()) * hparams.n_embd_head_k();
+
+    std::fill(hparams.is_recr_impl.begin(), hparams.is_recr_impl.end(), true);
+
+    switch (hparams.n_layer()) {
+        case 40: type = LLM_TYPE_8B_A1B; break;
+        default: type = LLM_TYPE_UNKNOWN;
+    }
+}
+
+void llama_model_zaya1::load_arch_tensors(llama_model_loader &) {
+    LLAMA_LOAD_LOCALS;
+
+    const int64_t n_embd_head = n_embd_head_k;
+    const int64_t n_group     = n_head + n_head_kv;
+    const int64_t n_qk        = hparams.n_cca_qk;
+    const int64_t d_conv      = hparams.ssm_d_conv;
+    const int64_t n_ff_exp    = hparams.n_ff_exp();
+
+    const int64_t n_router    = hparams.n_router_hidden;
+
+    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+
+    res_scale_in   = create_tensor(tn(LLM_TENSOR_RES_SCALE_IN, "weight", 0), {n_embd}, 0);
+    res_scale_in_b = create_tensor(tn(LLM_TENSOR_RES_SCALE_IN, "bias",   0), {n_embd}, 0);
+
+    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
+    if (output == nullptr) {
+        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
+    }
+
+    for (int i = 0; i < n_layer; ++i) {
+        auto & layer = layers[i];
+        auto & z     = layer.zaya1;
+
+        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
+
+        // the two value heads are v_cur and v_del
+        create_tensor_qkv(layer, i, n_embd, n_embd_head * n_head, n_embd_head * n_head_kv, n_embd_v_gqa, 0);
+
+        layer.ssm_conv1d   = create_tensor(tn(LLM_TENSOR_SSM_CONV1D,   "weight", i), {d_conv, n_qk}, 0);
+        layer.ssm_conv1d_b = create_tensor(tn(LLM_TENSOR_SSM_CONV1D,   "bias",   i), {n_qk}, 0);
+        z.cca_conv_grp     = create_tensor(tn(LLM_TENSOR_CCA_CONV_GRP, "weight", i), {n_embd_head, n_embd_head, n_group, d_conv}, 0);
+        z.cca_conv_grp_b   = create_tensor(tn(LLM_TENSOR_CCA_CONV_GRP, "bias",   i), {n_qk}, 0);
+        z.cca_temp         = create_tensor(tn(LLM_TENSOR_CCA_TEMP,     "weight", i), {n_head_kv}, 0);
+
+        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head * n_head, n_embd}, 0);
+
+        z.attn_res_hs            = create_tensor(tn(LLM_TENSOR_ATTN_RES_SCALE_HS, "weight", i), {n_embd}, 0);
+        layer.attn_resid_scale   = create_tensor(tn(LLM_TENSOR_ATTN_RESID_SCALE,  "weight", i), {n_embd}, 0);
+        layer.attn_resid_scale_b = create_tensor(tn(LLM_TENSOR_ATTN_RESID_SCALE,  "bias",   i), {n_embd}, 0);
+
+        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
+
+        layer.ffn_gate_inp   = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_router}, 0);
+        layer.ffn_gate_inp_b = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "bias",   i), {n_router}, 0);
+        z.router_eda   = create_tensor(tn(LLM_TENSOR_ROUTER_EDA,  "weight", i), {n_router}, i == 0 ? TENSOR_NOT_REQUIRED : 0);
+        z.router_norm  = create_tensor(tn(LLM_TENSOR_ROUTER_NORM, "weight", i), {n_router}, 0);
+        z.router_fc1   = create_tensor(tn(LLM_TENSOR_ROUTER_FC1,  "weight", i), {n_router, n_router}, 0);
+        z.router_fc1_b = create_tensor(tn(LLM_TENSOR_ROUTER_FC1,  "bias",   i), {n_router}, 0);
+        z.router_fc2   = create_tensor(tn(LLM_TENSOR_ROUTER_FC2,  "weight", i), {n_router, n_router}, 0);
+        z.router_fc2_b = create_tensor(tn(LLM_TENSOR_ROUTER_FC2,  "bias",   i), {n_router}, 0);
+        // one extra class for the skip expert
+        z.router_out   = create_tensor(tn(LLM_TENSOR_ROUTER_OUT,  "weight", i), {n_router, n_expert + 1}, 0);
+        layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert + 1}, 0);
+
+        layer.ffn_gate_up_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", i), {n_embd, 2 * n_ff_exp, n_expert}, 0);
+        layer.ffn_down_exps    = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS,    "weight", i), {n_ff_exp, n_embd, n_expert}, 0);
+
+        z.ffn_res_hs            = create_tensor(tn(LLM_TENSOR_FFN_RES_SCALE_HS, "weight", i), {n_embd}, 0);
+        layer.ffn_resid_scale   = create_tensor(tn(LLM_TENSOR_FFN_RESID_SCALE,  "weight", i), {n_embd}, 0);
+        layer.ffn_resid_scale_b = create_tensor(tn(LLM_TENSOR_FFN_RESID_SCALE,  "bias",   i), {n_embd}, 0);
+    }
+}
+
+std::unique_ptr<llm_graph_context> llama_model_zaya1::build_arch_graph(const llm_graph_params & params) const {
+    return std::make_unique<graph>(*this, params);
+}
+
+static ggml_tensor * zaya1_residual_scale(ggml_context * ctx, ggml_tensor * hs, ggml_tensor * res,
+        ggml_tensor * s_hs, ggml_tensor * s_res, ggml_tensor * b) {
+    return ggml_add(ctx, ggml_mul(ctx, hs, s_hs), ggml_add(ctx, ggml_mul(ctx, res, s_res), b));
+}
+
+// grouped causal conv1d with kernel 2, as one mul_mat per tap broadcast over the groups
+// x: {n_embd_head * n_group, n_t + 1, n_seqs}, w: {n_embd_head (in), n_embd_head (out), n_group, 2}
+// returns {n_embd_head * n_group, n_t, n_seqs} with the bias b added
+static ggml_tensor * zaya1_conv_grouped(ggml_context * ctx, ggml_tensor * x, ggml_tensor * w, ggml_tensor * b) {
+    const int64_t n_embd_head = w->ne[0];
+    const int64_t n_group     = w->ne[2];
+    const int64_t n_t         = x->ne[1] - 1;
+    const int64_t n_seqs      = x->ne[2];
+    GGML_ASSERT(w->ne[3] == 2);
+
+    x = ggml_reshape_4d(ctx, x, n_embd_head, n_group, n_t + 1, n_seqs);
+    x = ggml_permute(ctx, x, 0, 2, 1, 3);
+
+    ggml_tensor * x_prev = ggml_view_4d(ctx, x, n_embd_head, n_t, n_group, n_seqs, x->nb[1], x->nb[2], x->nb[3], 0);
+    ggml_tensor * x_cur  = ggml_view_4d(ctx, x, n_embd_head, n_t, n_group, n_seqs, x->nb[1], x->nb[2], x->nb[3], x->nb[1]);
+
+    ggml_tensor * w_prev = ggml_view_3d(ctx, w, n_embd_head, n_embd_head, n_group, w->nb[1], w->nb[2], 0);
+    ggml_tensor * w_cur  = ggml_view_3d(ctx, w, n_embd_head, n_embd_head, n_group, w->nb[1], w->nb[2], w->nb[3]);
+
+    ggml_tensor * y = ggml_add(ctx, ggml_mul_mat(ctx, w_prev, x_prev), ggml_mul_mat(ctx, w_cur, x_cur));
+
+    y = ggml_add(ctx, ggml_permute(ctx, y, 0, 2, 1, 3), ggml_reshape_2d(ctx, b, n_embd_head, n_group));
+    return ggml_reshape_3d(ctx, y, n_embd_head * n_group, n_t, n_seqs);
+}
+
+ggml_tensor * llama_model_zaya1::graph::build_cca_block(const llama_model & model, ggml_tensor * cur,
+        ggml_tensor * inp_pos, llm_graph_input_mem_hybrid * inp, int il) {
+    const auto & layer = model.layers[il];
+    const auto & z     = layer.zaya1;
+
+    const auto * mctx_cur = inp->get_recr()->mctx;
+    const uint32_t kv_head = mctx_cur->get_head();
+
+    const int64_t n_seq_tokens = ubatch.n_seq_tokens;
+    const int64_t n_seqs       = ubatch.n_seqs;
+    GGML_ASSERT(n_seqs != 0);
+    GGML_ASSERT(ubatch.equal_seqs());
+    GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);
+
+    const int64_t n_embd_head = hparams.n_embd_head_k();
+    const int64_t n_head_kv   = hparams.n_head_kv(il);
+    const int64_t n_q         = n_embd_head * n_head;
+    const int64_t n_qk        = hparams.n_cca_qk;
+    const int64_t n_v_half    = hparams.n_embd_v_gqa(il) / 2;
+    const int64_t n_hist      = 2 * (hparams.ssm_d_conv - 1);
+
+    auto [q_proj, k_proj, v_raw] = build_qkv(layer, cur,
+            n_embd_head, n_head, n_embd_head, n_head_kv, n_embd_head, n_head_kv, il, false);
+    ggml_tensor * qk_raw = ggml_concat(ctx0, q_proj, k_proj, 0);
+    cb(qk_raw, "cca_qk_raw", il);
+    cb(v_raw,  "cca_v_raw",  il);
+
+    const size_t  es    = ggml_element_size(qk_raw);
+    const int64_t n_rep = n_head / n_head_kv;
+
+    ggml_tensor * q_raw = ggml_view_4d(ctx0, qk_raw, n_embd_head, n_rep, n_head_kv, n_tokens,
+            n_embd_head * es, n_embd_head * n_rep * es, qk_raw->nb[1], 0);
+    ggml_tensor * k_raw = ggml_view_4d(ctx0, qk_raw, n_embd_head, 1, n_head_kv, n_tokens,
+            n_embd_head * es, n_embd_head * es, qk_raw->nb[1], n_q * es);
+    ggml_tensor * q_res = ggml_scale(ctx0, ggml_add(ctx0, q_raw, k_raw), 0.5f); // {n_embd_head, n_rep, n_head_kv, n_tokens}
+    ggml_tensor * k_res = nullptr;
+    for (int64_t r = 0; r < n_rep; ++r) {
+        ggml_tensor * q_r = ggml_view_4d(ctx0, q_res, n_embd_head, 1, n_head_kv, n_tokens,
+                q_res->nb[1], q_res->nb[2], q_res->nb[3], r * q_res->nb[1]);
+        k_res = k_res ? ggml_add(ctx0, k_res, q_r) : q_r;
+    }
+    k_res = ggml_scale(ctx0, k_res, 1.0f / n_rep);
+    q_res = ggml_reshape_3d(ctx0, q_res, n_embd_head, n_head,    n_tokens);
+    k_res = ggml_reshape_3d(ctx0, k_res, n_embd_head, n_head_kv, n_tokens);
+
+    // conv state: the last n_hist raw q/k columns of each sequence
+    ggml_tensor * conv_state = mctx_cur->get_r_l(il);
+    ggml_tensor * conv_rs    = build_rs(inp->get_recr(), conv_state, hparams.n_embd_r(), n_seqs);
+    conv_rs = ggml_reshape_3d(ctx0, conv_rs, n_hist, n_qk, n_seqs);
+
+    ggml_tensor * qk = ggml_view_3d(ctx0, qk_raw, n_qk, n_seq_tokens, n_seqs, qk_raw->nb[1], n_seq_tokens * qk_raw->nb[1], 0);
+    qk = ggml_concat(ctx0, conv_rs, ggml_transpose(ctx0, qk), 0); // {n_hist + n_seq_tokens, n_qk, n_seqs}
+
+    // delayed value state: v_del of the previous token
+    ggml_tensor * v_state = mctx_cur->get_s_l(il);
+    ggml_tensor * v_rs    = build_rs(inp->get_recr(), v_state, hparams.n_embd_s(), n_seqs);
+    ggml_tensor * v_del_ext = ggml_concat(ctx0,
+            ggml_reshape_3d(ctx0, v_rs, n_v_half, 1, n_seqs),
+            ggml_view_3d(ctx0, v_raw, n_v_half, n_seq_tokens, n_seqs, v_raw->nb[1], n_seq_tokens * v_raw->nb[1], n_v_half * es), 1); // {n_v_half, n_seq_tokens + 1, n_seqs}
+
+    {
+        const size_t conv_row = ggml_row_size(conv_state->type, n_hist * n_qk);
+        const size_t v_row    = ggml_row_size(v_state->type, n_v_half);
+
+        ggml_build_forward_expand(gf, ggml_cpy(ctx0,
+                ggml_view_3d(ctx0, qk, n_hist, n_qk, n_seqs, qk->nb[1], qk->nb[2], n_seq_tokens * ggml_element_size(qk)),
+                ggml_view_2d(ctx0, conv_state, n_hist * n_qk, n_seqs, conv_state->nb[1], kv_head * conv_row)));
+        ggml_build_forward_expand(gf, ggml_cpy(ctx0,
+                ggml_view_2d(ctx0, v_del_ext, n_v_half, n_seqs, v_del_ext->nb[2], n_seq_tokens * v_del_ext->nb[1]),
+                ggml_view_2d(ctx0, v_state, n_v_half, n_seqs, v_state->nb[1], kv_head * v_row)));
+    }
+
+    // depthwise conv, then grouped conv
+    ggml_tensor * qk_conv = ggml_ssm_conv(ctx0, qk, layer.ssm_conv1d); // {n_qk, n_seq_tokens + 1, n_seqs}
+    qk_conv = ggml_add(ctx0, qk_conv, layer.ssm_conv1d_b);
+    qk_conv = zaya1_conv_grouped(ctx0, qk_conv, z.cca_conv_grp, z.cca_conv_grp_b); // {n_qk, n_seq_tokens, n_seqs}
+    qk_conv = ggml_reshape_2d(ctx0, qk_conv, n_qk, n_tokens);
+    cb(qk_conv, "cca_qk_conv", il);
+
+    ggml_tensor * Qcur = ggml_view_3d(ctx0, qk_conv, n_embd_head, n_head, n_tokens,
+            n_embd_head * ggml_element_size(qk_conv), qk_conv->nb[1], 0);
+    ggml_tensor * Kcur = ggml_view_3d(ctx0, qk_conv, n_embd_head, n_head_kv, n_tokens,
+            n_embd_head * ggml_element_size(qk_conv), qk_conv->nb[1], n_q * ggml_element_size(qk_conv));
+    Qcur = ggml_add(ctx0, Qcur, q_res);
+    Kcur = ggml_add(ctx0, Kcur, k_res);
+
+    // L2 norm to sqrt(head_dim), then per kv head key temperature
+    const float head_scale = sqrtf((float) n_embd_head);
+    Qcur = ggml_scale(ctx0, ggml_l2_norm(ctx0, Qcur, 1e-6f), head_scale);
+    Kcur = ggml_scale(ctx0, ggml_l2_norm(ctx0, Kcur, 1e-6f), head_scale);
+    Kcur = ggml_mul(ctx0, Kcur, ggml_reshape_2d(ctx0, z.cca_temp, 1, n_head_kv));
+    cb(Qcur, "cca_q", il);
+    cb(Kcur, "cca_k", il);
+
+    Qcur = build_rope(Qcur, inp_pos, il);
+    Kcur = build_rope(Kcur, inp_pos, il);
+
+    // value heads: head 0 = current token, head 1 = previous token
+    ggml_tensor * v_prev = ggml_view_3d(ctx0, v_del_ext, n_v_half, n_seq_tokens, n_seqs,
+            v_del_ext->nb[1], v_del_ext->nb[2], 0);
+    ggml_tensor * v_cur = ggml_view_3d(ctx0, v_raw, n_v_half, n_seq_tokens, n_seqs, v_raw->nb[1], n_seq_tokens * v_raw->nb[1], 0);
+    ggml_tensor * Vcur  = ggml_concat(ctx0, v_cur, v_prev, 0);
+    Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
+    cb(Vcur, "cca_v", il);
+
+    return build_attn(inp->get_attn(),
+            layer.wo, NULL, layer.wo_s,
+            Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf((float) n_embd_head), il);
+}
+
+ggml_tensor * llama_model_zaya1::graph::build_router_moe(const llama_model & model, ggml_tensor * cur,
+        ggml_tensor *& router_state, int il) {
+    const auto & layer = model.layers[il];
+    const auto & z     = layer.zaya1;
+
+    ggml_tensor * r = ggml_add(ctx0, build_lora_mm(layer.ffn_gate_inp, cur), layer.ffn_gate_inp_b);
+    if (router_state) {
+        r = ggml_add(ctx0, r, ggml_mul(ctx0, router_state, z.router_eda));
+    }
+    router_state = r;
+
+    r = build_norm(r, z.router_norm, NULL, LLM_NORM_RMS, il);
+    r = ggml_gelu_erf(ctx0, ggml_add(ctx0, build_lora_mm(z.router_fc1, r), z.router_fc1_b));
+    r = ggml_gelu_erf(ctx0, ggml_add(ctx0, build_lora_mm(z.router_fc2, r), z.router_fc2_b));
+    ggml_tensor * logits = build_lora_mm(z.router_out, r); // {n_expert + 1, n_tokens}
+    ggml_tensor * probs  = ggml_soft_max(ctx0, logits);
+    cb(probs, "router_probs", il);
+
+    const int64_t n_rows = cur->ne[1];
+    ggml_tensor * scores = ggml_add(ctx0, probs, layer.ffn_exp_probs_b);
+    ggml_tensor * real   = ggml_cont(ctx0, ggml_view_2d(ctx0, scores, n_expert, n_rows, scores->nb[1], 0));
+    ggml_tensor * skip   = ggml_view_2d(ctx0, scores, 1, n_rows, scores->nb[1], n_expert * ggml_element_size(scores));
+    ggml_tensor * expert = ggml_argsort_top_k(ctx0, real, 1); // {1, n_rows}
+    ggml_tensor * best   = ggml_reshape_2d(ctx0, ggml_get_rows(ctx0, ggml_reshape_3d(ctx0, real, 1, n_expert, n_rows), expert), 1, n_rows);
+    ggml_tensor * keep   = ggml_scale_bias(ctx0, ggml_step(ctx0, ggml_sub(ctx0, skip, best)), -1.0f, 1.0f);
+    cb(keep, "router_keep", il);
+
+    ggml_tensor * moe = build_moe_ffn(cur,
+            nullptr,
+            nullptr,
+            nullptr,
+            layer.ffn_down_exps,
+            nullptr,
+            n_expert + 1, n_expert_used,
+            LLM_FFN_SILU, false,
+            0.0f,
+            LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX,
+            il,
+            logits,
+            layer.ffn_gate_up_exps,
+            nullptr,
+            nullptr,
+            nullptr,
+            expert);
+
+    return ggml_mul(ctx0, moe, keep);
+}
+
+llama_model_zaya1::graph::graph(const llama_model & model, const llm_graph_params & params) :
+    llm_graph_context(params) {
+    ggml_tensor * cur = build_inp_embd(model.tok_embd);
+    cur = ggml_mul(ctx0, ggml_add(ctx0, cur, model.res_scale_in_b), model.res_scale_in);
+    cb(cur, "inp_scaled", -1);
+
+    auto * inp = build_inp_mem_hybrid();
+
+    ggml_tensor * inp_pos     = build_inp_pos();
+    ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+    ggml_tensor * router_state = nullptr;
+
+    for (int il = 0; il < n_layer; ++il) {
+        const auto & layer = model.layers[il];
+        const auto & z     = layer.zaya1;
+
+        res->t_layer_inp[il] = cur;
+
+        ggml_tensor * residual = cur;
+
+        cur = build_norm(cur, layer.attn_norm, NULL, LLM_NORM_RMS, il);
+        cur = build_cca_block(model, cur, inp_pos, inp, il);
+        cb(cur, "attn_out", il);
+
+        if (il == n_layer - 1 && inp_out_ids) {
+            cur          = ggml_get_rows(ctx0, cur,          inp_out_ids);
+            residual     = ggml_get_rows(ctx0, residual,     inp_out_ids);
+            if (router_state) {
+                router_state = ggml_get_rows(ctx0, router_state, inp_out_ids);
+            }
+        }
+
+        residual = zaya1_residual_scale(ctx0, cur, residual, z.attn_res_hs, layer.attn_resid_scale, layer.attn_resid_scale_b);
+
+        cur = build_norm(residual, layer.ffn_norm, NULL, LLM_NORM_RMS, il);
+        cur = build_router_moe(model, cur, router_state, il);
+        cb(cur, "ffn_moe_out", il);
+
+        cur = zaya1_residual_scale(ctx0, cur, residual, z.ffn_res_hs, layer.ffn_resid_scale, layer.ffn_resid_scale_b);
+
+        cur = build_cvec(cur, il);
+        cb(cur, "l_out", il);
+    }
+
+    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
+    cb(cur, "result_norm", -1);
+    res->t_embd = cur;
+
+    cur = build_lora_mm(model.output, cur);
+    cb(cur, "result_output", -1);
+    res->t_logits = cur;
+}
diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv
index efb8718e7..310934e31 100644
--- a/tests/fusion/MTL.csv
+++ b/tests/fusion/MTL.csv
@@ -348,6 +348,7 @@ xing4_0             ,0   ,any     ,MUL+ADD                     ,      2
 xing4_0             ,0   ,any     ,RMS_NORM+MUL                ,      9
 xing4_0             ,0   ,any     ,UNARY+ADD+ARGSORT+GET_ROWS  ,      1
 xverse              ,0   ,any     ,RMS_NORM+MUL                ,      5
+zaya1               ,0   ,any     ,RMS_NORM+MUL                ,      7
 zonos3              ,0   ,any     ,GATED_DELTA_NET+CPY         ,      1
 zonos3              ,0   ,any     ,MUL+ADD                     ,      2
 zonos3              ,0   ,any     ,RMS_NORM+MUL                ,     12
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 49ae976a4..fa1ffb853 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -11742,7 +11742,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
         }
     }

-    for (int64_t d_conv : {3, 4, 7, 9}) {
+    for (int64_t d_conv : {2, 3, 4, 7, 9}) {
         for (int64_t d_inner: {1024, 1536, 2048}) {
             test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {d_conv, d_inner, 1, 1}, {d_conv, d_inner, 1, 1}));
             test_cases.emplace_back(new test_ssm_conv(GGML_TYPE_F32, {2 * d_conv, d_inner, 1, 1}, {d_conv, d_inner, 1, 1}));
diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp
index a2cba05aa..f10404e8c 100644
--- a/tests/test-llama-archs.cpp
+++ b/tests/test-llama-archs.cpp
@@ -337,7 +337,8 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe, const
     if (arch == LLM_ARCH_QWEN3) {
         n_head_kv = 1; // MQA coverage
     } else if (arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_MUSE_SPARK || arch == LLM_ARCH_AFMOE || arch == LLM_ARCH_MAION_CODER ||
-            arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_LIMITE || arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_RICURSIVE) {
+            arch == LLM_ARCH_KOLIBRI1 || arch == LLM_ARCH_LIMITE || arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_RICURSIVE ||
+            arch == LLM_ARCH_ZAYA1) {
         n_head_kv = 2; // GQA coverage
     }
     if (arch == LLM_ARCH_BEAM) {
@@ -744,7 +745,7 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe, const
         ms.add_kv(LLM_KV_INTERLEAVE_MOE_LAYER_STEP,  uint32_t(2));
         // with more experts than a ubatch uses, the copy of the used experts in host memory skips some of them
         ms.add_kv(LLM_KV_EXPERT_COUNT,               uint32_t(host_experts_test(arch) ? 64 : arch == LLM_ARCH_CLOVER_MOE ? 4 : 2));
-        ms.add_kv(LLM_KV_EXPERT_USED_COUNT,          uint32_t(2));
+        ms.add_kv(LLM_KV_EXPERT_USED_COUNT,          uint32_t(arch == LLM_ARCH_ZAYA1 ? 1 : 2)); // ZAYA routes top-1
         ms.add_kv(LLM_KV_EXPERT_SHARED_COUNT,        uint32_t(1));

         uint32_t gating_func = 2;
@@ -799,7 +800,8 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe, const
     ms.add_kv(LLM_KV_XIELU_BETA,                1.0f);
     ms.add_kv(LLM_KV_XIELU_EPS,                 1.0e-7f);
     ms.add_kv(LLM_KV_SSM_INNER_SIZE,            arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_QWEN4EXP || arch == LLM_ARCH_CLOVER_MOE || arch == LLM_ARCH_ZONOS3 ? 256 : 2*n_embd);
-    ms.add_kv(LLM_KV_SSM_CONV_KERNEL,           uint32_t(4));
+    ms.add_kv(LLM_KV_ROUTER_HIDDEN_SIZE,        n_embd);
+    ms.add_kv(LLM_KV_SSM_CONV_KERNEL,           uint32_t(arch == LLM_ARCH_ZAYA1 ? 2 : 4)); // ZAYA CCA convs have kernel 2
     ms.add_kv(LLM_KV_SSM_STATE_SIZE,            uint32_t(128));
     ms.add_kv(LLM_KV_SSM_TIME_STEP_RANK,        n_head);
     ms.add_kv(LLM_KV_SSM_GROUP_COUNT,           arch == LLM_ARCH_PLAMO2 ? 0 : uint32_t(2));
@@ -1074,6 +1076,7 @@ static double check_draft_rollback(llama_model * model, const std::vector<llama_
 static bool moe_mandatory(const llm_arch arch) {
     switch (arch) {
         case LLM_ARCH_FLINT:
+        case LLM_ARCH_ZAYA1:
         case LLM_ARCH_LLAMA4:
         case LLM_ARCH_MAION_CODER:
         case LLM_ARCH_COHERE2MOE: