Commit 9309bf1bc for llama.cpp
commit 9309bf1bcfe8d52b0fb7ef906474db94041a5e0c
Author: Giulianno <lernexbusiness@gmail.com>
Date: Tue Nov 3 08:24:45 2026 -0700
model : Add support for Metis-1.6 models (#31134)
Assisted-by: Claude
diff --git a/conversion/__init__.py b/conversion/__init__.py
index dcbb97c1f..6cea1ec78 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -197,6 +197,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"MambaLMHeadModel": "mamba",
"MapleForCausalLM": "maple",
"MellumForCausalLM": "mellum",
+ "Metis16ForCausalLM": "metis",
"MiMoV2FlashForCausalLM": "mimo",
"MiMoV2ForCausalLM": "mimo",
"MiniCPM3ForCausalLM": "minicpm",
diff --git a/conversion/base.py b/conversion/base.py
index fd880bbab..9d9d6e875 100644
--- a/conversion/base.py
+++ b/conversion/base.py
@@ -2116,6 +2116,9 @@ class TextModel(ModelBase):
if chkhsh == "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4":
# ref: https://huggingface.co/evilfreelancer/ruGPT3XL
res = "gpt-2"
+ if chkhsh == "df1bc34a6366466bcb36a9a1b1cc0afdb08cfe21c9250602c930f159085390a9":
+ # ref: https://huggingface.co/Lernex/Metis-1.6-Logos
+ res = "gpt-2"
if chkhsh == "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7":
# ref: https://huggingface.co/LiquidAI/LFM2.5-8B-A1B
res = "lfm2"
diff --git a/conversion/metis.py b/conversion/metis.py
new file mode 100644
index 000000000..a073ed4df
--- /dev/null
+++ b/conversion/metis.py
@@ -0,0 +1,204 @@
+from __future__ import annotations
+
+import math
+from typing import Callable, Iterable, TYPE_CHECKING
+
+import torch
+
+if TYPE_CHECKING:
+ from torch import Tensor
+
+from .base import LazyTorchTensor, ModelBase, TextModel, gguf
+
+
+@ModelBase.register("Metis16ForCausalLM")
+class Metis16Model(TextModel):
+ model_arch = gguf.MODEL_ARCH.METIS16
+
+ # read by generate_extra_tensors() or not used at inference: dropped in modify_tensors()
+ _folded = {
+ "pass_embeddings", "lm_head.weight", "depth_budget.multiplier",
+ "depth_memory.pass_embeddings", "depth_memory.anchor_embeddings",
+ "depth_memory.state_write.bias", "depth_memory.metadata_write.bias",
+ }
+ _folded_layer = {
+ "mixer_connection.controller.weight", "mixer_connection.controller.bias", "mixer_connection.mix_logits",
+ "mixer_connection.read_logits", "mixer_connection.write_logits",
+ "moe_connection.controller.weight", "moe_connection.controller.bias", "moe_connection.mix_logits",
+ "moe_connection.read_logits", "moe_connection.write_logits",
+ "moe.k_router.weight", "moe.k_router.bias", "moe.k_budget.multiplier",
+ }
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ hp = self.hparams
+ self.n_pass = hp["num_passes"]
+ self.attn_layers = list(hp["attention_layer_indices"])
+ self._generated: set[str] = set()
+ # one cache slot per pass and layer, while the tensors exist once per physical layer
+ self.block_count = self.n_pass * hp["num_hidden_layers"]
+
+ def set_vocab(self):
+ self._set_vocab_gpt2()
+ # turns end on the first eos id, so llama.cpp must stop on it too
+ eos = self.hparams.get("eos_token_id")
+ if isinstance(eos, list) and eos:
+ self.gguf_writer.add_eot_token_id(eos[0])
+
+ def set_gguf_parameters(self):
+ hp = self.hparams
+ super().set_gguf_parameters()
+
+ n_layer = hp["num_hidden_layers"]
+ self.gguf_writer.add_head_count_kv([hp["num_key_value_heads"] if il % n_layer in self.attn_layers else 0 for il in range(self.block_count)])
+
+ self.gguf_writer.add_expert_feed_forward_length(hp["expert_intermediate_size"])
+ self.gguf_writer.add_expert_shared_feed_forward_length(hp["shared_expert_intermediate_size"])
+ self.gguf_writer.add_expert_shared_count(1)
+ self.gguf_writer.add_moe_latent_size(hp["latent_dim"])
+
+ d_inner = hp["mamba_expand"] * hp["hidden_size"]
+ self.gguf_writer.add_ssm_conv_kernel(hp["mamba_d_conv"])
+ self.gguf_writer.add_ssm_inner_size(d_inner)
+ self.gguf_writer.add_ssm_state_size(hp["mamba_d_state"])
+ self.gguf_writer.add_ssm_time_step_rank(d_inner // hp["mamba_head_dim"])
+ self.gguf_writer.add_ssm_group_count(hp["mamba_n_groups"])
+ self.gguf_writer.add_ssm_norm_epsilon(hp["mamba_norm_eps"])
+
+ self.gguf_writer.add_hyper_connection_count(hp["n_streams"])
+ self.gguf_writer.add_metis_pass_count(self.n_pass)
+ self.gguf_writer.add_metis_memory_length(hp["memory_dim"])
+ self.gguf_writer.add_metis_route_length(hp["route_feature_dim"])
+ self.gguf_writer.add_metis_continuation_logit_limit(hp["continuation_logit_limit"])
+ self.gguf_writer.add_attention_pass_lora_rank(hp["attention_pass_lora_rank"])
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ name, gen = item
+ # the checkpoint leaves these names suffix-less; add the suffix of the GGUF tensor so the mapper keeps it
+ if name.endswith((".mixer.impl.mixer.dt_bias", ".moe.selection_bias")):
+ name += ".bias"
+ elif (name.endswith((".moe.expert_embeddings", ".mixer.pass_gates"))
+ or name in ("depth_memory.stream_gate", "stream_embeddings")):
+ name += ".weight"
+ elif name == "depth_memory.stream_gate_bias":
+ name = "depth_memory.stream_gate.bias"
+ return super().filter_tensors((name, gen))
+
+ def _load(self, name: str) -> Tensor:
+ return LazyTorchTensor.to_eager(self.model_tensors[name]())
+
+ def _emit(self, name: str, data: Tensor) -> tuple[str, Tensor]:
+ self._generated.add(name)
+ return name, data
+
+ def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
+ hp = self.hparams
+ r = hp["route_feature_dim"]
+ n_stream = hp["n_streams"]
+ T = gguf.MODEL_TENSOR
+ passes = self._load("pass_embeddings")[: self.n_pass]
+
+ # the mixes depend only on the weights and the pass, so Sinkhorn runs once here
+ # the controller runs in the checkpoint dtype, as the reference does
+ for il in range(hp["num_hidden_layers"]):
+ for src, (mix, read, write) in (
+ ("mixer_connection", (T.HC_ATTN_MIX, T.HC_ATTN_READ, T.HC_ATTN_WRITE)),
+ ("moe_connection", (T.HC_FFN_MIX, T.HC_FFN_READ, T.HC_FFN_WRITE)),
+ ):
+ prefix = f"layers.{il}.{src}."
+ ctrl_w, ctrl_b = self._load(prefix + "controller.weight"), self._load(prefix + "controller.bias")
+ mixes, reads, writes = [], [], []
+ for p in range(self.n_pass):
+ delta = torch.nn.functional.linear(passes[p].to(ctrl_w.dtype), ctrl_w, ctrl_b).float()
+ m_delta, r_delta, w_delta = torch.split(delta, [n_stream * n_stream, n_stream, n_stream])
+ log_m = self._load(prefix + "mix_logits").float() + m_delta.view(n_stream, n_stream)
+ for _ in range(hp["mhc_sinkhorn_iterations"]):
+ log_m = log_m - torch.logsumexp(log_m, dim=-1, keepdim=True)
+ log_m = log_m - torch.logsumexp(log_m, dim=-2, keepdim=True)
+ mixes.append(log_m.exp())
+ reads.append(torch.softmax(self._load(prefix + "read_logits").float() + r_delta, dim=-1))
+ writes.append(torch.softmax(self._load(prefix + "write_logits").float() + w_delta, dim=-1))
+ act = ctrl_w.dtype
+ yield self._emit(self.format_tensor_name(mix, il), torch.stack(mixes).to(act).float())
+ yield self._emit(self.format_tensor_name(read, il), torch.stack(reads).to(act).float())
+ yield self._emit(self.format_tensor_name(write, il), torch.stack(writes).to(act).float())
+
+ # one bias per memory entry: write biases, pass and anchor embeddings, and the constant routed width
+ meta_w = self._load("depth_memory.metadata_write.weight").float()
+ mean_k = hp["num_experts_per_tok"] / hp["max_routed_k"]
+ base = (self._load("depth_memory.state_write.bias").float() + self._load("depth_memory.metadata_write.bias").float()
+ + mean_k * meta_w[:, r])
+ pass_e = self._load("depth_memory.pass_embeddings").float()
+ anchor_e = self._load("depth_memory.anchor_embeddings").float()
+ n_anchor = len(self.attn_layers) + 1
+ entry_bias = torch.stack([base + pass_e[p] + anchor_e[a] for p in range(self.n_pass) for a in range(n_anchor)])
+ yield self._emit(self.format_tensor_name(T.DEPTH_MEM_ENTRY_BIAS), entry_bias)
+
+ def _split_features(self, data_torch: Tensor, template: str, parts: tuple[str, ...], d: int, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ # the graph reads the concatenated features one block at a time; each block is its own tensor
+ for i, part in enumerate(parts):
+ piece = data_torch[:, i * d:(i + 1) * d] if i < 3 else data_torch[:, 3 * d:]
+ yield from super().modify_tensors(piece, template.format(part), bid)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if name in self._generated:
+ yield name, data_torch
+ return
+ if name in self._folded:
+ return
+ hp = self.hparams
+ d = hp["hidden_size"]
+
+ if bid is None:
+ if name == "stream_embeddings.weight":
+ # the graph keeps the streams innermost: [n_stream, n_embd, n_tokens]
+ yield from super().modify_tensors(data_torch.T.contiguous(), name, bid)
+ return
+ if name == "depth_memory.metadata_write.weight":
+ # metadata input becomes [summary, entropy, confidence, continuation]; entropy arrives unnormalized
+ r = hp["route_feature_dim"]
+ meta_w = data_torch.float()
+ entropy_col = meta_w[:, r + 1:r + 2] / math.log(hp["n_routed_experts"])
+ data_torch = torch.cat((meta_w[:, :r], entropy_col, meta_w[:, r + 2:]), dim=1)
+ elif name == "depth_memory.stream_gate.weight":
+ data_torch = data_torch.float() / math.sqrt(d)
+ elif name == "depth_memory.route_projection.weight":
+ yield from self._split_features(data_torch, "depth_memory.route_{}.weight", ("state", "retr", "diff", "hist"), d, bid)
+ return
+ elif name == "continuation.hidden.weight":
+ yield from self._split_features(data_torch, "continuation.hidden_{}.weight", ("state", "retr", "diff", "route"), d, bid)
+ return
+ yield from super().modify_tensors(data_torch, name, bid)
+ return
+
+ local = name.removeprefix(f"layers.{bid}.")
+ if local in self._folded_layer:
+ return
+
+ if local == "mixer.impl.mixer.A_log":
+ data_torch = -torch.exp(data_torch).unsqueeze(-1)
+ elif local == "mixer.impl.mixer.D":
+ data_torch = data_torch.unsqueeze(-1)
+ elif local == "mixer.impl.mixer.conv1d.weight":
+ data_torch = data_torch.squeeze(1)
+ elif local == "mixer.impl.mixer.norm.weight":
+ data_torch = data_torch.reshape(hp["mamba_n_groups"], -1)
+ elif local == "mixer.pass_gates.weight":
+ # the reference applies tanh(gate) in the checkpoint dtype
+ data_torch = torch.tanh(data_torch[: self.n_pass]).to(torch.bfloat16).float()
+ elif local == "moe.shared_expert.gate_up.weight":
+ gate, up = data_torch.chunk(2, dim=0)
+ yield from super().modify_tensors(gate, name.replace("gate_up", "gate"), bid)
+ yield from super().modify_tensors(up, name.replace("gate_up", "up"), bid)
+ return
+
+ yield from super().modify_tensors(data_torch, name, bid)
+
+ def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
+ # FP32 in the checkpoint, tiny and read as rows, or computed here at full precision: keep FP32
+ if (new_name.startswith(("cont_", "depth_mem_entry_bias", "depth_mem_stream_gate"))
+ or new_name.endswith(("ffn_exp_embd.weight", "ffn_gate_inp.weight", "hc_attn_mix.weight", "hc_attn_read.weight",
+ "hc_attn_write.weight", "hc_ffn_mix.weight", "hc_ffn_read.weight", "hc_ffn_write.weight"))):
+ return gguf.GGMLQuantizationType.F32
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py
index 37df033ef..3b8a33597 100755
--- a/convert_hf_to_gguf_update.py
+++ b/convert_hf_to_gguf_update.py
@@ -198,6 +198,7 @@ pre_computed_hashes = [
# jina-v2-de variants
{"name": "jina-v2-de", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/aari1995/German_Semantic_V3", "chkhsh": "b3d1dd861f1d4c5c0d2569ce36baf3f90fe8a102db3de50dd71ff860d91be3df"},
{"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/evilfreelancer/ruGPT3XL", "chkhsh": "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4"},
+ {"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Lernex/Metis-1.6-Logos", "chkhsh": "df1bc34a6366466bcb36a9a1b1cc0afdb08cfe21c9250602c930f159085390a9"},
# lfm2 variants
{"name": "lfm2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B", "chkhsh": "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7"},
# hrm-text (DFM Mimir) is SPM-style BPE: normalizer maps ' ' -> '▁', merges
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index ad4e675f6..c4c4a4c65 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -203,6 +203,7 @@ class Keys:
ICLR_LORA_RANK = "{arch}.attention.iclr_lora_rank"
VALUE_RESIDUAL_MIX_LORA_RANK = "{arch}.attention.value_residual_mix_lora_rank"
GATE_LORA_RANK = "{arch}.attention.gate_lora_rank"
+ PASS_LORA_RANK = "{arch}.attention.pass_lora_rank"
REL_BUCKETS_COUNT = "{arch}.attention.relative_buckets_count"
SLIDING_WINDOW = "{arch}.attention.sliding_window"
SCALE = "{arch}.attention.scale"
@@ -315,6 +316,12 @@ class Keys:
L_CYCLES = "{arch}.hrm.l_cycles"
PREFIX_LM = "{arch}.hrm.prefix_lm"
+ class Metis:
+ PASS_COUNT = "{arch}.metis.pass_count"
+ MEMORY_LENGTH = "{arch}.metis.memory_length"
+ ROUTE_LENGTH = "{arch}.metis.route_length"
+ CONTINUATION_LOGIT_LIMIT = "{arch}.metis.continuation_logit_limit"
+
class SSM:
CONV_KERNEL = "{arch}.ssm.conv_kernel"
INNER_SIZE = "{arch}.ssm.inner_size"
@@ -322,6 +329,7 @@ class Keys:
TIME_STEP_RANK = "{arch}.ssm.time_step_rank"
GROUP_COUNT = "{arch}.ssm.group_count"
DT_B_C_RMS = "{arch}.ssm.dt_b_c_rms"
+ NORM_EPSILON = "{arch}.ssm.norm_epsilon"
class KDA:
HEAD_DIM = "{arch}.kda.head_dim"
@@ -681,6 +689,7 @@ class MODEL_ARCH(IntEnum):
NEMOTRON = auto()
NEMOTRON_H = auto()
NEMOTRON_H_MOE = auto()
+ METIS16 = auto()
EXAONE = auto()
EXAONE4 = auto()
EXAONE_MOE = auto()
@@ -830,6 +839,36 @@ class MODEL_TENSOR(IntEnum):
FFN_GATE_TID2EID = auto()
MOE_LATENT_DOWN = auto() # nemotron 3 super
MOE_LATENT_UP = auto() # nemotron 3 super
+ STREAM_EMBD = auto()
+ DEPTH_MEM_STATE_WRITE = auto()
+ DEPTH_MEM_META_WRITE = auto()
+ DEPTH_MEM_ENTRY_BIAS = auto()
+ DEPTH_MEM_QUERY = auto()
+ DEPTH_MEM_KEY = auto()
+ DEPTH_MEM_VALUE = auto()
+ DEPTH_MEM_OUT = auto()
+ DEPTH_MEM_ROUTE_STATE = auto()
+ DEPTH_MEM_ROUTE_RETR = auto()
+ DEPTH_MEM_ROUTE_DIFF = auto()
+ DEPTH_MEM_ROUTE_HIST = auto()
+ DEPTH_MEM_ROUTE = auto()
+ DEPTH_MEM_STREAM_GATE = auto()
+ CONT_HIDDEN_STATE = auto()
+ CONT_HIDDEN_RETR = auto()
+ CONT_HIDDEN_DIFF = auto()
+ CONT_HIDDEN_ROUTE = auto()
+ CONT_HIDDEN = auto()
+ CONT_OUTPUT = auto()
+ HC_ATTN_MIX = auto()
+ HC_ATTN_READ = auto()
+ HC_ATTN_WRITE = auto()
+ HC_FFN_MIX = auto()
+ HC_FFN_READ = auto()
+ HC_FFN_WRITE = auto()
+ ATTN_PASS_LORA_DOWN = auto()
+ ATTN_PASS_LORA_UP = auto()
+ ATTN_PASS_GATE = auto()
+ FFN_EXP_EMBD = auto()
FFN_RESID_ALPHA = auto()
FFN_RESID_BETA = auto()
FFN_PRE_EXP_NORM = auto()
@@ -1558,6 +1597,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
MODEL_ARCH.NEMOTRON: "nemotron",
MODEL_ARCH.NEMOTRON_H: "nemotron_h",
MODEL_ARCH.NEMOTRON_H_MOE: "nemotron_h_moe",
+ MODEL_ARCH.METIS16: "metis16",
MODEL_ARCH.EXAONE: "exaone",
MODEL_ARCH.EXAONE4: "exaone4",
MODEL_ARCH.EXAONE_MOE: "exaone-moe",
@@ -1708,6 +1748,36 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.FFN_GATE_TID2EID: "blk.{bid}.ffn_gate_tid2eid",
MODEL_TENSOR.MOE_LATENT_DOWN: "blk.{bid}.ffn_latent_down", # nemotron 3 super
MODEL_TENSOR.MOE_LATENT_UP: "blk.{bid}.ffn_latent_up", # nemotron 3 super
+ MODEL_TENSOR.STREAM_EMBD: "stream_embd",
+ MODEL_TENSOR.DEPTH_MEM_STATE_WRITE: "depth_mem_state_write",
+ MODEL_TENSOR.DEPTH_MEM_META_WRITE: "depth_mem_meta_write",
+ MODEL_TENSOR.DEPTH_MEM_ENTRY_BIAS: "depth_mem_entry_bias",
+ MODEL_TENSOR.DEPTH_MEM_QUERY: "depth_mem_query",
+ MODEL_TENSOR.DEPTH_MEM_KEY: "depth_mem_key",
+ MODEL_TENSOR.DEPTH_MEM_VALUE: "depth_mem_value",
+ MODEL_TENSOR.DEPTH_MEM_OUT: "depth_mem_out",
+ MODEL_TENSOR.DEPTH_MEM_ROUTE_STATE: "depth_mem_route_state",
+ MODEL_TENSOR.DEPTH_MEM_ROUTE_RETR: "depth_mem_route_retr",
+ MODEL_TENSOR.DEPTH_MEM_ROUTE_DIFF: "depth_mem_route_diff",
+ MODEL_TENSOR.DEPTH_MEM_ROUTE_HIST: "depth_mem_route_hist",
+ MODEL_TENSOR.DEPTH_MEM_ROUTE: "depth_mem_route",
+ MODEL_TENSOR.DEPTH_MEM_STREAM_GATE: "depth_mem_stream_gate",
+ MODEL_TENSOR.CONT_HIDDEN_STATE: "cont_hidden_state",
+ MODEL_TENSOR.CONT_HIDDEN_RETR: "cont_hidden_retr",
+ MODEL_TENSOR.CONT_HIDDEN_DIFF: "cont_hidden_diff",
+ MODEL_TENSOR.CONT_HIDDEN_ROUTE: "cont_hidden_route",
+ MODEL_TENSOR.CONT_HIDDEN: "cont_hidden",
+ MODEL_TENSOR.CONT_OUTPUT: "cont_output",
+ MODEL_TENSOR.HC_ATTN_MIX: "blk.{bid}.hc_attn_mix",
+ MODEL_TENSOR.HC_ATTN_READ: "blk.{bid}.hc_attn_read",
+ MODEL_TENSOR.HC_ATTN_WRITE: "blk.{bid}.hc_attn_write",
+ MODEL_TENSOR.HC_FFN_MIX: "blk.{bid}.hc_ffn_mix",
+ MODEL_TENSOR.HC_FFN_READ: "blk.{bid}.hc_ffn_read",
+ MODEL_TENSOR.HC_FFN_WRITE: "blk.{bid}.hc_ffn_write",
+ MODEL_TENSOR.ATTN_PASS_LORA_DOWN: "blk.{bid}.attn_pass_lora_down",
+ MODEL_TENSOR.ATTN_PASS_LORA_UP: "blk.{bid}.attn_pass_lora_up",
+ MODEL_TENSOR.ATTN_PASS_GATE: "blk.{bid}.attn_pass_gate",
+ MODEL_TENSOR.FFN_EXP_EMBD: "blk.{bid}.ffn_exp_embd",
MODEL_TENSOR.FFN_RESID_ALPHA: "blk.{bid}.ffn_resid_alpha",
MODEL_TENSOR.FFN_RESID_BETA: "blk.{bid}.ffn_resid_beta",
MODEL_TENSOR.FFN_PRE_EXP_NORM: "blk.{bid}.ffn_pre_exp_norm",
@@ -4917,6 +4987,61 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.FFN_DOWN,
MODEL_TENSOR.FFN_UP,
],
+ MODEL_ARCH.METIS16: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.STREAM_EMBD,
+ MODEL_TENSOR.DEPTH_MEM_STATE_WRITE,
+ MODEL_TENSOR.DEPTH_MEM_META_WRITE,
+ MODEL_TENSOR.DEPTH_MEM_ENTRY_BIAS,
+ MODEL_TENSOR.DEPTH_MEM_QUERY,
+ MODEL_TENSOR.DEPTH_MEM_KEY,
+ MODEL_TENSOR.DEPTH_MEM_VALUE,
+ MODEL_TENSOR.DEPTH_MEM_OUT,
+ MODEL_TENSOR.DEPTH_MEM_ROUTE_STATE,
+ MODEL_TENSOR.DEPTH_MEM_ROUTE_RETR,
+ MODEL_TENSOR.DEPTH_MEM_ROUTE_DIFF,
+ MODEL_TENSOR.DEPTH_MEM_ROUTE_HIST,
+ MODEL_TENSOR.DEPTH_MEM_ROUTE,
+ MODEL_TENSOR.DEPTH_MEM_STREAM_GATE,
+ MODEL_TENSOR.CONT_HIDDEN_STATE,
+ MODEL_TENSOR.CONT_HIDDEN_RETR,
+ MODEL_TENSOR.CONT_HIDDEN_DIFF,
+ MODEL_TENSOR.CONT_HIDDEN_ROUTE,
+ MODEL_TENSOR.CONT_HIDDEN,
+ MODEL_TENSOR.CONT_OUTPUT,
+ MODEL_TENSOR.HC_ATTN_MIX,
+ MODEL_TENSOR.HC_ATTN_READ,
+ MODEL_TENSOR.HC_ATTN_WRITE,
+ MODEL_TENSOR.HC_FFN_MIX,
+ MODEL_TENSOR.HC_FFN_READ,
+ MODEL_TENSOR.HC_FFN_WRITE,
+ MODEL_TENSOR.ATTN_PASS_LORA_DOWN,
+ MODEL_TENSOR.ATTN_PASS_LORA_UP,
+ MODEL_TENSOR.ATTN_PASS_GATE,
+ MODEL_TENSOR.FFN_EXP_EMBD,
+ MODEL_TENSOR.SSM_IN,
+ MODEL_TENSOR.SSM_CONV1D,
+ MODEL_TENSOR.SSM_DT,
+ MODEL_TENSOR.SSM_A,
+ MODEL_TENSOR.SSM_D,
+ MODEL_TENSOR.SSM_NORM,
+ MODEL_TENSOR.SSM_OUT,
+ MODEL_TENSOR.ATTN_QKV,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.MOE_LATENT_DOWN,
+ MODEL_TENSOR.MOE_LATENT_UP,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ MODEL_TENSOR.FFN_GATE_UP_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_GATE_SHEXP,
+ MODEL_TENSOR.FFN_UP_SHEXP,
+ MODEL_TENSOR.FFN_DOWN_SHEXP,
+ ],
MODEL_ARCH.NEMOTRON_H: [
MODEL_TENSOR.TOKEN_EMBD,
MODEL_TENSOR.OUTPUT_NORM,
diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py
index 56e8938c4..0e7aa5012 100644
--- a/gguf-py/gguf/gguf_writer.py
+++ b/gguf-py/gguf/gguf_writer.py
@@ -975,6 +975,18 @@ class GGUFWriter:
def add_hrm_prefix_lm(self, value: bool) -> None:
self.add_bool(Keys.HRM.PREFIX_LM.format(arch=self.arch), value)
+ def add_metis_pass_count(self, value: int) -> None:
+ self.add_uint32(Keys.Metis.PASS_COUNT.format(arch=self.arch), value)
+
+ def add_metis_memory_length(self, value: int) -> None:
+ self.add_uint32(Keys.Metis.MEMORY_LENGTH.format(arch=self.arch), value)
+
+ def add_metis_route_length(self, value: int) -> None:
+ self.add_uint32(Keys.Metis.ROUTE_LENGTH.format(arch=self.arch), value)
+
+ def add_metis_continuation_logit_limit(self, value: float) -> None:
+ self.add_float32(Keys.Metis.CONTINUATION_LOGIT_LIMIT.format(arch=self.arch), value)
+
def add_adapter_count(self, count: int) -> None:
self.add_uint32(Keys.Adapters.COUNT.format(arch=self.arch), count)
@@ -1086,6 +1098,9 @@ class GGUFWriter:
def add_attention_output_lora_rank(self, length: int) -> None:
self.add_uint32(Keys.Attention.OUTPUT_LORA_RANK.format(arch=self.arch), length)
+ def add_attention_pass_lora_rank(self, length: int) -> None:
+ self.add_uint32(Keys.Attention.PASS_LORA_RANK.format(arch=self.arch), length)
+
def add_attention_compress_ratios(self, values: Sequence[int]) -> None:
self.add_array(Keys.Attention.COMPRESS_RATIOS.format(arch=self.arch), values)
@@ -1267,6 +1282,9 @@ class GGUFWriter:
def add_ssm_dt_b_c_rms(self, value: bool) -> None:
self.add_bool(Keys.SSM.DT_B_C_RMS.format(arch=self.arch), value)
+ def add_ssm_norm_epsilon(self, value: float) -> None:
+ self.add_float32(Keys.SSM.NORM_EPSILON.format(arch=self.arch), value)
+
def add_expert_latent_length(self, value: int) -> None:
self.add_uint32(Keys.LLM.EXPERT_LATENT_LENGTH.format(arch=self.arch), value)
diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py
index 2602bfb61..ff22a60bc 100644
--- a/gguf-py/gguf/tensor_mapping.py
+++ b/gguf-py/gguf/tensor_mapping.py
@@ -35,6 +35,7 @@ class TensorNameMap:
"model.transformer.wte", # llada
"embed_tokens", # qwen3-embedding
"model.embed", # talkie
+ "embedding", # metis16
),
# Masked embeddings
@@ -224,6 +225,7 @@ class TensorNameMap:
block_mappings_cfg: dict[MODEL_TENSOR, tuple[str, ...]] = {
# Attention norm
MODEL_TENSOR.ATTN_NORM: (
+ "layers.{bid}.mixer_connection.norm", # metis16
"model.lilicorr.layers.{bid}.attn_norm", # lilicorr
"gpt_neox.layers.{bid}.input_layernorm", # gptneox
"transformer.h.{bid}.ln_1", # gpt2 gpt-j refact qwen jais exaone
@@ -294,6 +296,7 @@ class TensorNameMap:
"model.layers.{bid}.linear_attn.in_proj_qkv", # qwen3.5
"head.layers.{bid}.self_attn.in_proj", # laya
"layers.{bid}.linear_attn.in_proj_qkv", # qwen3.5 text
+ "layers.{bid}.mixer.qkv", # metis16
),
# Attention query
@@ -401,6 +404,7 @@ class TensorNameMap:
"model.layers.{bid}.self_attn.language_expert_dense", # cogvlm
"model.blocks.{bid}.attn.attn_resid", # talkie
"head.layers.{bid}.self_attn.out_proj", # laya
+ "layers.{bid}.mixer.out", # metis16
),
# Attention output norm
@@ -449,6 +453,7 @@ class TensorNameMap:
# Feed-forward norm
MODEL_TENSOR.FFN_NORM: (
+ "layers.{bid}.moe_connection.norm", # metis16
"model.lilicorr.layers.{bid}.mlp_norm", # lilicorr
"model.lilicorr.feature_mlp.{bid}.norm", # lilicorr
"gpt_neox.layers.{bid}.post_attention_layernorm", # gptneox
@@ -536,6 +541,7 @@ class TensorNameMap:
"backbone.layers.{bid}.mixer.gate", # nemotron-h-moe
"model.layers.{bid}.moe.gate", # step3.5
"model.layers.{bid}.router.proj", # gemma4
+ "layers.{bid}.moe.expert_router", # metis16
),
MODEL_TENSOR.FFN_GATE_INP_SHEXP: (
@@ -543,6 +549,7 @@ class TensorNameMap:
),
MODEL_TENSOR.FFN_EXP_PROBS_B: (
+ "layers.{bid}.moe.selection_bias", # metis16
"model.layers.{bid}.mlp.gate.e_score_correction", # deepseek-v3 dots1
"model.layers.{bid}.mlp.gate.e_score_correction_bias", # muse-spark
"model.layers.{bid}.mlp.moe_statics.e_score_correction", # ernie4.5-moe
@@ -617,6 +624,7 @@ class TensorNameMap:
),
MODEL_TENSOR.FFN_UP_SHEXP: (
+ "layers.{bid}.moe.shared_expert.up", # metis16
"model.layers.{bid}.mlp.shared_expert.up_proj", # qwen2moe
"model.layers.{bid}.mlp.shared_experts.up_proj", # deepseek deepseek2
"model.layers.{bid}.feed_forward.shared_expert.up_proj", # llama4
@@ -670,6 +678,7 @@ class TensorNameMap:
),
MODEL_TENSOR.FFN_GATE_SHEXP: (
+ "layers.{bid}.moe.shared_expert.gate", # metis16
"model.layers.{bid}.mlp.shared_expert.gate_proj", # qwen2moe
"model.layers.{bid}.mlp.shared_experts.gate_proj", # deepseek deepseek2
"model.layers.{bid}.feed_forward.shared_expert.gate_proj", # llama4
@@ -686,18 +695,94 @@ class TensorNameMap:
MODEL_TENSOR.FFN_GATE_UP_EXP: (
"model.layers.{bid}.mlp.experts.gate_up_proj",
"model.layers.{bid}.experts.gate_up_proj", # gemma4
+ "layers.{bid}.moe.local_experts.gate_up", # metis16
),
MODEL_TENSOR.MOE_LATENT_DOWN: (
"backbone.layers.{bid}.mixer.fc1_latent_proj", # nemotron 3 super
"model.layers.{bid}.mlp.pre_expert_proj", # muse-spark
+ "layers.{bid}.moe.latent_down", # metis16
),
MODEL_TENSOR.MOE_LATENT_UP: (
"backbone.layers.{bid}.mixer.fc2_latent_proj", # nemotron 3 super
"model.layers.{bid}.mlp.post_expert_proj", # muse-spark
+ "layers.{bid}.moe.latent_up", # metis16
),
+ MODEL_TENSOR.STREAM_EMBD: (
+ "stream_embeddings", # metis16
+ ),
+ MODEL_TENSOR.DEPTH_MEM_STATE_WRITE: (
+ "depth_memory.state_write", # metis16
+ ),
+ MODEL_TENSOR.DEPTH_MEM_META_WRITE: (
+ "depth_memory.metadata_write", # metis16
+ ),
+ MODEL_TENSOR.DEPTH_MEM_QUERY: (
+ "depth_memory.query", # metis16
+ ),
+ MODEL_TENSOR.DEPTH_MEM_KEY: (
+ "depth_memory.key", # metis16
+ ),
+ MODEL_TENSOR.DEPTH_MEM_VALUE: (
+ "depth_memory.value", # metis16
+ ),
+ MODEL_TENSOR.DEPTH_MEM_OUT: (
+ "depth_memory.output", # metis16
+ ),
+ MODEL_TENSOR.DEPTH_MEM_ROUTE_STATE: (
+ "depth_memory.route_state", # metis16
+ ),
+ MODEL_TENSOR.DEPTH_MEM_ROUTE_RETR: (
+ "depth_memory.route_retr", # metis16
+ ),
+ MODEL_TENSOR.DEPTH_MEM_ROUTE_DIFF: (
+ "depth_memory.route_diff", # metis16
+ ),
+ MODEL_TENSOR.DEPTH_MEM_ROUTE_HIST: (
+ "depth_memory.route_hist", # metis16
+ ),
+ MODEL_TENSOR.DEPTH_MEM_ROUTE: (
+ "depth_memory.route_projection", # metis16
+ ),
+ MODEL_TENSOR.DEPTH_MEM_STREAM_GATE: (
+ "depth_memory.stream_gate", # metis16
+ ),
+ MODEL_TENSOR.CONT_HIDDEN_STATE: (
+ "continuation.hidden_state", # metis16
+ ),
+ MODEL_TENSOR.CONT_HIDDEN_RETR: (
+ "continuation.hidden_retr", # metis16
+ ),
+ MODEL_TENSOR.CONT_HIDDEN_DIFF: (
+ "continuation.hidden_diff", # metis16
+ ),
+ MODEL_TENSOR.CONT_HIDDEN_ROUTE: (
+ "continuation.hidden_route", # metis16
+ ),
+ MODEL_TENSOR.CONT_HIDDEN: (
+ "continuation.hidden", # metis16
+ ),
+ MODEL_TENSOR.CONT_OUTPUT: (
+ "continuation.output", # metis16
+ ),
+ MODEL_TENSOR.ATTN_PASS_LORA_DOWN: (
+ "layers.{bid}.mixer.pass_lora_down", # metis16
+ ),
+ MODEL_TENSOR.ATTN_PASS_LORA_UP: (
+ "layers.{bid}.mixer.pass_lora_up", # metis16
+ ),
+ MODEL_TENSOR.ATTN_PASS_GATE: (
+ "layers.{bid}.mixer.pass_gates", # metis16
+ ),
+ MODEL_TENSOR.FFN_EXP_EMBD: (
+ "layers.{bid}.moe.expert_embeddings", # metis16
+ ),
+
+
+
+
MODEL_TENSOR.FFN_PRE_EXP_NORM: (
"model.layers.{bid}.mlp.pre_expert_norm", # muse-spark
),
@@ -759,6 +844,7 @@ class TensorNameMap:
"model.layers.{bid}.block_sparse_moe.experts.down", # smallthinker
"model.layers.{bid}.moe.down_proj", # step3.5
"model.layers.{bid}.experts.down_proj", # gemma4
+ "layers.{bid}.moe.local_experts.down", # metis16
),
MODEL_TENSOR.FFN_DOWN_SHEXP: (
@@ -771,6 +857,7 @@ class TensorNameMap:
"backbone.layers.{bid}.mixer.shared_experts.down_proj", # nemotron-h-moe
"model.layers.{bid}.block_sparse_moe.shared_experts.down_proj", # kimi
"model.layers.{bid}.share_expert.down_proj", # step3.5
+ "layers.{bid}.moe.shared_expert.down", # metis16
),
MODEL_TENSOR.FFN_DOWN_CHEXP: (
@@ -906,6 +993,7 @@ class TensorNameMap:
"model.layers.{bid}.mamba.in_proj", # jamba falcon-h1 granite-hybrid
"model.layers.layers.{bid}.mixer.in_proj", # plamo2
"model.layers.{bid}.linear_attn.in_proj_qkvz", # qwen3next
+ "layers.{bid}.mixer.impl.mixer.in_proj", # metis16
),
MODEL_TENSOR.SSM_CONV1D: (
@@ -915,6 +1003,7 @@ class TensorNameMap:
"model.layers.layers.{bid}.mixer.conv1d", # plamo2
"model.layers.{bid}.linear_attn.conv1d", # qwen3next
"layers.{bid}.linear_attn.conv1d", # qwen3.5 text
+ "layers.{bid}.mixer.impl.mixer.conv1d", # metis16
),
MODEL_TENSOR.SSM_X: (
@@ -925,6 +1014,7 @@ class TensorNameMap:
),
MODEL_TENSOR.SSM_DT: (
+ "layers.{bid}.mixer.impl.mixer.dt_bias", # metis16
"model.layers.{bid}.dt_proj", # mamba-hf
"backbone.layers.{bid}.mixer.dt_proj", # mamba
"model.layers.{bid}.mamba.dt_proj", # jamba falcon-h1 granite-hybrid
@@ -942,6 +1032,7 @@ class TensorNameMap:
),
MODEL_TENSOR.SSM_A: (
+ "layers.{bid}.mixer.impl.mixer.A_log", # metis16
"model.layers.{bid}.A_log", # mamba-hf
"backbone.layers.{bid}.mixer.A_log", # mamba
"model.layers.{bid}.mamba.A_log", # jamba falcon-h1 granite-hybrid
@@ -965,6 +1056,7 @@ class TensorNameMap:
),
MODEL_TENSOR.SSM_D: (
+ "layers.{bid}.mixer.impl.mixer.D", # metis16
"model.layers.{bid}.D", # mamba-hf
"backbone.layers.{bid}.mixer.D", # mamba
"model.layers.{bid}.mamba.D", # jamba falcon-h1 granite-hybrid
@@ -972,6 +1064,7 @@ class TensorNameMap:
),
MODEL_TENSOR.SSM_NORM: (
+ "layers.{bid}.mixer.impl.mixer.norm", # metis16
"model.layers.{bid}.mamba.norm", # falcon-h1 granite-hybrid
"model.layers.{bid}.linear_attn.norm", # qwen3next
"layers.{bid}.linear_attn.norm", # qwen3.5 text
@@ -987,6 +1080,7 @@ class TensorNameMap:
"model.layers.{bid}.linear_attn.out_proj", # qwen3next
"layers.{bid}.linear_attn.out_proj", # qwen3.5 text
"model.layers.layers.{bid}.mixer.out_proj", # plamo2
+ "layers.{bid}.mixer.impl.mixer.out_proj", # metis16
),
MODEL_TENSOR.SSM_ALPHA: (
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index 525f91031..92a13e985 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -102,6 +102,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
{ LLM_ARCH_NEMOTRON, "nemotron" },
{ LLM_ARCH_NEMOTRON_H, "nemotron_h" },
{ LLM_ARCH_NEMOTRON_H_MOE, "nemotron_h_moe" },
+ { LLM_ARCH_METIS16, "metis16" },
{ LLM_ARCH_EXAONE, "exaone" },
{ LLM_ARCH_EXAONE4, "exaone4" },
{ LLM_ARCH_EXAONE_MOE, "exaone-moe" },
@@ -272,6 +273,10 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_HRM_H_CYCLES, "%s.hrm.h_cycles" },
{ LLM_KV_HRM_L_CYCLES, "%s.hrm.l_cycles" },
{ LLM_KV_HRM_PREFIX_LM, "%s.hrm.prefix_lm" },
+ { LLM_KV_METIS_PASS_COUNT, "%s.metis.pass_count" },
+ { LLM_KV_METIS_MEMORY_LENGTH, "%s.metis.memory_length" },
+ { LLM_KV_METIS_ROUTE_LENGTH, "%s.metis.route_length" },
+ { LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT, "%s.metis.continuation_logit_limit" },
{ LLM_KV_MUDD_FEED_FORWARD_LENGTH, "%s.mudd.feed_forward_length" },
{ LLM_KV_MUDD_TAP_COUNT, "%s.mudd.tap_count" },
{ LLM_KV_MUDD_TAP_INDICES, "%s.mudd.tap_indices" },
@@ -329,6 +334,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_ATTENTION_KV_SOURCE_LAYERS, "%s.attention.kv_source_layers" },
{ LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, "%s.attention.output_group_count" },
{ LLM_KV_ATTENTION_OUTPUT_LORA_RANK, "%s.attention.output_lora_rank" },
+ { LLM_KV_ATTENTION_PASS_LORA_RANK, "%s.attention.pass_lora_rank" },
{ LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, "%s.attention.compress_rope_freq_base" },
{ LLM_KV_ATTENTION_COMPRESS_RATIOS, "%s.attention.compress_ratios" },
{ LLM_KV_ATTENTION_SHARED_KV_LAYERS, "%s.attention.shared_kv_layers" },
@@ -387,6 +393,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_SSM_TIME_STEP_RANK, "%s.ssm.time_step_rank" },
{ LLM_KV_SSM_GROUP_COUNT, "%s.ssm.group_count" },
{ LLM_KV_SSM_DT_B_C_RMS, "%s.ssm.dt_b_c_rms" },
+ { LLM_KV_SSM_NORM_EPS, "%s.ssm.norm_epsilon" },
{ LLM_KV_KDA_HEAD_DIM, "%s.kda.head_dim" },
{ LLM_KV_KDA_SAFE_GATE, "%s.kda.safe_gate" },
@@ -548,6 +555,36 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
{ LLM_TENSOR_FFN_EXP_PROBS_B_VL, "blk.%d.exp_probs_b_vl" },
{ LLM_TENSOR_FFN_LATENT_DOWN, "blk.%d.ffn_latent_down" },
{ LLM_TENSOR_FFN_LATENT_UP, "blk.%d.ffn_latent_up" },
+ { LLM_TENSOR_STREAM_EMBD, "stream_embd" },
+ { LLM_TENSOR_DEPTH_MEM_STATE_WRITE, "depth_mem_state_write" },
+ { LLM_TENSOR_DEPTH_MEM_META_WRITE, "depth_mem_meta_write" },
+ { LLM_TENSOR_DEPTH_MEM_ENTRY_BIAS, "depth_mem_entry_bias" },
+ { LLM_TENSOR_DEPTH_MEM_QUERY, "depth_mem_query" },
+ { LLM_TENSOR_DEPTH_MEM_KEY, "depth_mem_key" },
+ { LLM_TENSOR_DEPTH_MEM_VALUE, "depth_mem_value" },
+ { LLM_TENSOR_DEPTH_MEM_OUT, "depth_mem_out" },
+ { LLM_TENSOR_DEPTH_MEM_ROUTE_STATE, "depth_mem_route_state" },
+ { LLM_TENSOR_DEPTH_MEM_ROUTE_RETR, "depth_mem_route_retr" },
+ { LLM_TENSOR_DEPTH_MEM_ROUTE_DIFF, "depth_mem_route_diff" },
+ { LLM_TENSOR_DEPTH_MEM_ROUTE_HIST, "depth_mem_route_hist" },
+ { LLM_TENSOR_DEPTH_MEM_ROUTE, "depth_mem_route" },
+ { LLM_TENSOR_DEPTH_MEM_STREAM_GATE, "depth_mem_stream_gate" },
+ { LLM_TENSOR_CONT_HIDDEN_STATE, "cont_hidden_state" },
+ { LLM_TENSOR_CONT_HIDDEN_RETR, "cont_hidden_retr" },
+ { LLM_TENSOR_CONT_HIDDEN_DIFF, "cont_hidden_diff" },
+ { LLM_TENSOR_CONT_HIDDEN_ROUTE, "cont_hidden_route" },
+ { LLM_TENSOR_CONT_HIDDEN, "cont_hidden" },
+ { LLM_TENSOR_CONT_OUTPUT, "cont_output" },
+ { LLM_TENSOR_HC_ATTN_MIX, "blk.%d.hc_attn_mix" },
+ { LLM_TENSOR_HC_ATTN_READ, "blk.%d.hc_attn_read" },
+ { LLM_TENSOR_HC_ATTN_WRITE, "blk.%d.hc_attn_write" },
+ { LLM_TENSOR_HC_FFN_MIX, "blk.%d.hc_ffn_mix" },
+ { LLM_TENSOR_HC_FFN_READ, "blk.%d.hc_ffn_read" },
+ { LLM_TENSOR_HC_FFN_WRITE, "blk.%d.hc_ffn_write" },
+ { LLM_TENSOR_ATTN_PASS_LORA_DOWN, "blk.%d.attn_pass_lora_down" },
+ { LLM_TENSOR_ATTN_PASS_LORA_UP, "blk.%d.attn_pass_lora_up" },
+ { LLM_TENSOR_ATTN_PASS_GATE, "blk.%d.attn_pass_gate" },
+ { LLM_TENSOR_FFN_EXP_EMBD, "blk.%d.ffn_exp_embd" },
{ LLM_TENSOR_FFN_PRE_EXP_NORM, "blk.%d.ffn_pre_exp_norm" },
{ LLM_TENSOR_FFN_POST_EXP_NORM, "blk.%d.ffn_post_exp_norm" },
{ LLM_TENSOR_FFN_RESID_ALPHA, "blk.%d.ffn_resid_alpha" },
@@ -1158,6 +1195,36 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
// latent projections feed ggml_mul_mat, the buft probe must use MUL_MAT to keep them on GPU
{LLM_TENSOR_FFN_LATENT_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_FFN_LATENT_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_STREAM_EMBD, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_ADD}},
+ {LLM_TENSOR_DEPTH_MEM_STATE_WRITE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_DEPTH_MEM_META_WRITE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_DEPTH_MEM_ENTRY_BIAS, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_DEPTH_MEM_QUERY, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_DEPTH_MEM_KEY, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_DEPTH_MEM_VALUE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_DEPTH_MEM_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_DEPTH_MEM_ROUTE_STATE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_DEPTH_MEM_ROUTE_RETR, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_DEPTH_MEM_ROUTE_DIFF, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_DEPTH_MEM_ROUTE_HIST, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_DEPTH_MEM_ROUTE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_DEPTH_MEM_STREAM_GATE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_CONT_HIDDEN_STATE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_CONT_HIDDEN_RETR, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_CONT_HIDDEN_DIFF, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_CONT_HIDDEN_ROUTE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_CONT_HIDDEN, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_CONT_OUTPUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_HC_ATTN_MIX, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_HC_ATTN_READ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_HC_ATTN_WRITE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_HC_FFN_MIX, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_HC_FFN_READ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_HC_FFN_WRITE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_ATTN_PASS_LORA_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_ATTN_PASS_LORA_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_ATTN_PASS_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_FFN_EXP_EMBD, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_GET_ROWS}},
{LLM_TENSOR_FFN_PRE_EXP_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_FFN_POST_EXP_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_FFN_RESID_ALPHA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
@@ -1286,6 +1353,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) {
case LLM_ARCH_LFM3:
case LLM_ARCH_NEMOTRON_H:
case LLM_ARCH_NEMOTRON_H_MOE:
+ case LLM_ARCH_METIS16:
case LLM_ARCH_QWEN3NEXT:
case LLM_ARCH_KIMI_LINEAR:
case LLM_ARCH_BAILINGMOE3:
diff --git a/src/llama-arch.h b/src/llama-arch.h
index b98be40cf..f811607b6 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -106,6 +106,7 @@ enum llm_arch {
LLM_ARCH_NEMOTRON,
LLM_ARCH_NEMOTRON_H,
LLM_ARCH_NEMOTRON_H_MOE,
+ LLM_ARCH_METIS16,
LLM_ARCH_EXAONE,
LLM_ARCH_EXAONE4,
LLM_ARCH_EXAONE_MOE,
@@ -276,6 +277,10 @@ enum llm_kv {
LLM_KV_HRM_H_CYCLES,
LLM_KV_HRM_L_CYCLES,
LLM_KV_HRM_PREFIX_LM,
+ LLM_KV_METIS_PASS_COUNT,
+ LLM_KV_METIS_MEMORY_LENGTH,
+ LLM_KV_METIS_ROUTE_LENGTH,
+ LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT,
LLM_KV_REALTIME_DELAY_TOKENS,
LLM_KV_MUDD_FEED_FORWARD_LENGTH,
LLM_KV_MUDD_TAP_COUNT,
@@ -333,6 +338,7 @@ enum llm_kv {
LLM_KV_ATTENTION_KV_SOURCE_LAYERS,
LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT,
LLM_KV_ATTENTION_OUTPUT_LORA_RANK,
+ LLM_KV_ATTENTION_PASS_LORA_RANK,
LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE,
LLM_KV_ATTENTION_COMPRESS_RATIOS,
LLM_KV_ATTENTION_SHARED_KV_LAYERS,
@@ -391,6 +397,7 @@ enum llm_kv {
LLM_KV_SSM_TIME_STEP_RANK,
LLM_KV_SSM_GROUP_COUNT,
LLM_KV_SSM_DT_B_C_RMS,
+ LLM_KV_SSM_NORM_EPS,
LLM_KV_KDA_HEAD_DIM,
LLM_KV_KDA_SAFE_GATE,
@@ -566,6 +573,36 @@ enum llm_tensor {
LLM_TENSOR_FFN_EXP_PROBS_B_VL,
LLM_TENSOR_FFN_LATENT_DOWN,
LLM_TENSOR_FFN_LATENT_UP,
+ LLM_TENSOR_STREAM_EMBD,
+ LLM_TENSOR_DEPTH_MEM_STATE_WRITE,
+ LLM_TENSOR_DEPTH_MEM_META_WRITE,
+ LLM_TENSOR_DEPTH_MEM_ENTRY_BIAS,
+ LLM_TENSOR_DEPTH_MEM_QUERY,
+ LLM_TENSOR_DEPTH_MEM_KEY,
+ LLM_TENSOR_DEPTH_MEM_VALUE,
+ LLM_TENSOR_DEPTH_MEM_OUT,
+ LLM_TENSOR_DEPTH_MEM_ROUTE_STATE,
+ LLM_TENSOR_DEPTH_MEM_ROUTE_RETR,
+ LLM_TENSOR_DEPTH_MEM_ROUTE_DIFF,
+ LLM_TENSOR_DEPTH_MEM_ROUTE_HIST,
+ LLM_TENSOR_DEPTH_MEM_ROUTE,
+ LLM_TENSOR_DEPTH_MEM_STREAM_GATE,
+ LLM_TENSOR_CONT_HIDDEN_STATE,
+ LLM_TENSOR_CONT_HIDDEN_RETR,
+ LLM_TENSOR_CONT_HIDDEN_DIFF,
+ LLM_TENSOR_CONT_HIDDEN_ROUTE,
+ LLM_TENSOR_CONT_HIDDEN,
+ LLM_TENSOR_CONT_OUTPUT,
+ LLM_TENSOR_HC_ATTN_MIX,
+ LLM_TENSOR_HC_ATTN_READ,
+ LLM_TENSOR_HC_ATTN_WRITE,
+ LLM_TENSOR_HC_FFN_MIX,
+ LLM_TENSOR_HC_FFN_READ,
+ LLM_TENSOR_HC_FFN_WRITE,
+ LLM_TENSOR_ATTN_PASS_LORA_DOWN,
+ LLM_TENSOR_ATTN_PASS_LORA_UP,
+ LLM_TENSOR_ATTN_PASS_GATE,
+ LLM_TENSOR_FFN_EXP_EMBD,
LLM_TENSOR_FFN_PRE_EXP_NORM,
LLM_TENSOR_FFN_POST_EXP_NORM,
LLM_TENSOR_FFN_RESID_ALPHA,
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index 0cd795a56..eb9756996 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -2703,6 +2703,7 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {
res = std::max<uint32_t>(n_tokens * 80, 64u * model.n_tensors());
} else if (model.arch == LLM_ARCH_QWEN3NEXT ||
model.arch == LLM_ARCH_KIMI_LINEAR ||
+ model.arch == LLM_ARCH_METIS16 ||
model.arch == LLM_ARCH_BAILINGMOE3 ||
model.arch == LLM_ARCH_QWEN35 ||
model.arch == LLM_ARCH_QWEN35MOE ||
diff --git a/src/llama-hparams.h b/src/llama-hparams.h
index 4d2cb8931..95daf482d 100644
--- a/src/llama-hparams.h
+++ b/src/llama-hparams.h
@@ -209,6 +209,7 @@ struct llama_hparams {
uint32_t ssm_d_state = 0;
uint32_t ssm_dt_rank = 0;
uint32_t ssm_n_group = 0;
+ float ssm_norm_eps = 0.0f; // 0 = use f_norm_rms_eps
// for MiniMax-Text-01 linear attention
uint32_t n_embd_head_la = 0;
@@ -230,6 +231,13 @@ struct llama_hparams {
uint32_t n_hrm_l_cycles = 0;
bool hrm_prefix_lm = false;
+ // metis16 (passes over shared layers, depth memory)
+ uint32_t metis_n_pass = 0;
+ uint32_t metis_n_mem = 0;
+ uint32_t metis_n_route = 0;
+ uint32_t metis_pass_lora_rank = 0;
+ float metis_cont_limit = 0.0f;
+
// limite
float f_xsa_eps = 0.0f;
std::array<float, 3> final_logit_sigmoid_capping = {}; // scale, shift, temperature
diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp
index 36f3850d5..bfa1b547c 100644
--- a/src/llama-model-saver.cpp
+++ b/src/llama-model-saver.cpp
@@ -247,6 +247,10 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_HRM_H_CYCLES, hparams.n_hrm_h_cycles);
add_kv(LLM_KV_HRM_L_CYCLES, hparams.n_hrm_l_cycles);
add_kv(LLM_KV_HRM_PREFIX_LM, hparams.hrm_prefix_lm);
+ add_kv(LLM_KV_METIS_PASS_COUNT, hparams.metis_n_pass);
+ add_kv(LLM_KV_METIS_MEMORY_LENGTH, hparams.metis_n_mem);
+ add_kv(LLM_KV_METIS_ROUTE_LENGTH, hparams.metis_n_route);
+ add_kv(LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT, hparams.metis_cont_limit);
if (hparams.final_logit_sigmoid_capping[0] != 0.0f) {
add_kv(LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING, std::vector<float>(
hparams.final_logit_sigmoid_capping.begin(), hparams.final_logit_sigmoid_capping.end()));
@@ -323,6 +327,7 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, true);
add_kv(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, hparams.dsv4_o_group_count);
add_kv(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, hparams.dsv4_o_lora_rank);
+ add_kv(LLM_KV_ATTENTION_PASS_LORA_RANK, hparams.metis_pass_lora_rank);
add_kv(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, hparams.dsv4_compress_rope_base);
if (model->arch == LLM_ARCH_DEEPSEEK4 || hparams.dsv4_hc_mult > 0) {
// the loader requires one compress ratio per layer, including nextn layers
@@ -419,6 +424,7 @@ void llama_model_saver::add_kv_from_model() {
add_kv(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);
add_kv(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);
add_kv(LLM_KV_SSM_DT_B_C_RMS, hparams.ssm_dt_b_c_rms);
+ add_kv(LLM_KV_SSM_NORM_EPS, hparams.ssm_norm_eps);
add_kv(LLM_KV_KDA_HEAD_DIM, hparams.n_embd_head_kda);
add_kv(LLM_KV_KDA_SAFE_GATE, hparams.kda_safe_gate);
@@ -521,6 +527,32 @@ void llama_model_saver::add_tensors_from_model() {
add_tensor(model->cls_out_b);
add_tensor(model->cls_norm);
add_tensor(model->hrm_z_l_init);
+ add_tensor(model->stream_embd);
+ add_tensor(model->depth_mem_state_write);
+ add_tensor(model->depth_mem_meta_write);
+ add_tensor(model->depth_mem_entry_bias);
+ add_tensor(model->depth_mem_query);
+ add_tensor(model->depth_mem_query_b);
+ add_tensor(model->depth_mem_key);
+ add_tensor(model->depth_mem_key_b);
+ add_tensor(model->depth_mem_value);
+ add_tensor(model->depth_mem_value_b);
+ add_tensor(model->depth_mem_out);
+ add_tensor(model->depth_mem_out_b);
+ add_tensor(model->depth_mem_route_state);
+ add_tensor(model->depth_mem_route_retr);
+ add_tensor(model->depth_mem_route_diff);
+ add_tensor(model->depth_mem_route_hist);
+ add_tensor(model->depth_mem_route_b);
+ add_tensor(model->depth_mem_stream_gate);
+ add_tensor(model->depth_mem_stream_gate_b);
+ add_tensor(model->cont_hidden_state);
+ add_tensor(model->cont_hidden_retr);
+ add_tensor(model->cont_hidden_diff);
+ add_tensor(model->cont_hidden_route);
+ add_tensor(model->cont_hidden_b);
+ add_tensor(model->cont_output);
+ add_tensor(model->cont_output_b);
add_tensor(model->hc_head_fn);
add_tensor(model->hc_head_base);
add_tensor(model->hc_head_scale);
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index 87893bf67..3c4e598f1 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -242,6 +242,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_nemotron_h(params);
case LLM_ARCH_NEMOTRON_H_MOE:
return new llama_model_nemotron_h_moe(params);
+ case LLM_ARCH_METIS16:
+ return new llama_model_metis16(params);
case LLM_ARCH_EXAONE:
return new llama_model_exaone(params);
case LLM_ARCH_EXAONE4:
@@ -515,7 +517,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
};
auto get_tensor_config = [&]() -> tensor_config {
- if (ud->model->arch == LLM_ARCH_HRM_TEXT) {
+ if (ud->model->arch == LLM_ARCH_HRM_TEXT || ud->model->arch == LLM_ARCH_METIS16) {
// aliased cache slots cannot satisfy the meta-split invariants, so replicate all tensors
return {GGML_BACKEND_SPLIT_AXIS_MIRRORED, tensor, 0, 0};
}
@@ -1049,6 +1051,7 @@ const char * llm_type_name(llm_type type) {
case LLM_TYPE_7B: return "7B";
case LLM_TYPE_8B: return "8B";
case LLM_TYPE_9B: return "9B";
+ case LLM_TYPE_10B: return "10B";
case LLM_TYPE_11B: return "11B";
case LLM_TYPE_12B: return "12B";
case LLM_TYPE_13B: return "13B";
@@ -3618,6 +3621,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_WAVTOKENIZER_DEC:
case LLM_ARCH_NEMOTRON_H:
case LLM_ARCH_NEMOTRON_H_MOE:
+ case LLM_ARCH_METIS16:
case LLM_ARCH_KIMI_LINEAR:
case LLM_ARCH_KIMI_K3:
case LLM_ARCH_GLM5_NEXT:
diff --git a/src/llama-model.h b/src/llama-model.h
index da4964c98..45d2b8666 100644
--- a/src/llama-model.h
+++ b/src/llama-model.h
@@ -76,6 +76,7 @@ enum llm_type {
LLM_TYPE_7B,
LLM_TYPE_8B,
LLM_TYPE_9B,
+ LLM_TYPE_10B,
LLM_TYPE_11B,
LLM_TYPE_12B,
LLM_TYPE_13B,
@@ -646,6 +647,18 @@ struct llama_layer {
// gemma4 layer output scale, reused for talkie embedding skip scale
struct ggml_tensor * out_scale = nullptr;
+ // metis16 per-pass hyper-connection mixes, attention pass LoRA, expert route embeddings
+ struct ggml_tensor * hc_attn_mix = nullptr;
+ struct ggml_tensor * hc_attn_read = nullptr;
+ struct ggml_tensor * hc_attn_write = nullptr;
+ struct ggml_tensor * hc_ffn_mix = nullptr;
+ struct ggml_tensor * hc_ffn_read = nullptr;
+ struct ggml_tensor * hc_ffn_write = nullptr;
+ struct ggml_tensor * attn_pass_lora_down = nullptr;
+ struct ggml_tensor * attn_pass_lora_up = nullptr;
+ struct ggml_tensor * attn_pass_gate = nullptr;
+ struct ggml_tensor * ffn_exp_embd = nullptr;
+
// limite
struct ggml_tensor * attn_ve_gate = nullptr;
struct ggml_tensor * attn_xsa_alpha = nullptr;
@@ -779,6 +792,34 @@ struct llama_model {
// hrm-text initial low-cycle state
struct ggml_tensor * hrm_z_l_init = nullptr;
+ // metis16 hyper-connection stream embeddings, depth memory and continuation head
+ struct ggml_tensor * stream_embd = nullptr;
+ struct ggml_tensor * depth_mem_state_write = nullptr;
+ struct ggml_tensor * depth_mem_meta_write = nullptr;
+ struct ggml_tensor * depth_mem_entry_bias = nullptr;
+ struct ggml_tensor * depth_mem_query = nullptr;
+ struct ggml_tensor * depth_mem_query_b = nullptr;
+ struct ggml_tensor * depth_mem_key = nullptr;
+ struct ggml_tensor * depth_mem_key_b = nullptr;
+ struct ggml_tensor * depth_mem_value = nullptr;
+ struct ggml_tensor * depth_mem_value_b = nullptr;
+ struct ggml_tensor * depth_mem_out = nullptr;
+ struct ggml_tensor * depth_mem_out_b = nullptr;
+ struct ggml_tensor * depth_mem_route_state = nullptr;
+ struct ggml_tensor * depth_mem_route_retr = nullptr;
+ struct ggml_tensor * depth_mem_route_diff = nullptr;
+ struct ggml_tensor * depth_mem_route_hist = nullptr;
+ struct ggml_tensor * depth_mem_route_b = nullptr;
+ struct ggml_tensor * depth_mem_stream_gate = nullptr;
+ struct ggml_tensor * depth_mem_stream_gate_b = nullptr;
+ struct ggml_tensor * cont_hidden_state = nullptr;
+ struct ggml_tensor * cont_hidden_retr = nullptr;
+ struct ggml_tensor * cont_hidden_diff = nullptr;
+ struct ggml_tensor * cont_hidden_route = nullptr;
+ struct ggml_tensor * cont_hidden_b = nullptr;
+ struct ggml_tensor * cont_output = nullptr;
+ struct ggml_tensor * cont_output_b = nullptr;
+
// DeepSeek-V4
struct ggml_tensor * hc_head_fn = nullptr;
struct ggml_tensor * hc_head_base = nullptr;
diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp
index 34c247858..d62f7d5e9 100644
--- a/src/llama-quant.cpp
+++ b/src/llama-quant.cpp
@@ -361,6 +361,16 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param
quantize &= name.find("indexer.k_proj.weight") == std::string::npos;
quantize &= name.find("indexer.q_proj.weight") == std::string::npos;
+ // metis16: small control tensors that steer every layer; the stream and hc_ tables (ne[0] = 4) would fall back to F16
+ if (arch == LLM_ARCH_METIS16) {
+ quantize &= name.rfind("depth_mem_", 0) == std::string::npos;
+ quantize &= name.rfind("cont_", 0) == std::string::npos;
+ quantize &= name.find("stream_embd") == std::string::npos;
+ quantize &= name.find("attn_pass_lora") == std::string::npos;
+ quantize &= name.find("ffn_exp_embd") == std::string::npos;
+ quantize &= name.find("hc_") == std::string::npos;
+ }
+
// glm5-next
if (arch == LLM_ARCH_GLM5_NEXT) {
quantize &= name.find("hc_") == std::string::npos;
diff --git a/src/models/mamba-base.cpp b/src/models/mamba-base.cpp
index 370de0f01..63af3113d 100644
--- a/src/models/mamba-base.cpp
+++ b/src/models/mamba-base.cpp
@@ -290,7 +290,11 @@ ggml_tensor * llm_build_mamba_base::build_mamba2_layer(llm_graph_input_rs * inp,
// grouped RMS norm
if (model.layers[il].ssm_norm) {
y = ggml_reshape_4d(ctx0, y, d_inner / n_group, n_group, n_seq_tokens, n_seqs);
- y = build_norm(y, model.layers[il].ssm_norm, NULL, LLM_NORM_RMS, il);
+ if (hparams.ssm_norm_eps > 0.0f) {
+ y = ggml_mul(ctx0, ggml_rms_norm(ctx0, y, hparams.ssm_norm_eps), model.layers[il].ssm_norm);
+ } else {
+ y = build_norm(y, model.layers[il].ssm_norm, NULL, LLM_NORM_RMS, il);
+ }
}
y = ggml_reshape_2d(ctx0, y, d_inner, n_seq_tokens * n_seqs);
diff --git a/src/models/metis16.cpp b/src/models/metis16.cpp
new file mode 100644
index 000000000..4e4b032b9
--- /dev/null
+++ b/src/models/metis16.cpp
@@ -0,0 +1,401 @@
+#include "models.h"
+
+#include <cmath>
+
+void llama_model_metis16::load_arch_hparams(llama_model_loader & ml) {
+ ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv);
+ ml.get_key(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner);
+ ml.get_key(LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state);
+ ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);
+ ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);
+ ml.get_key(LLM_KV_SSM_NORM_EPS, hparams.ssm_norm_eps);
+
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+
+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);
+ ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp);
+ ml.get_key(LLM_KV_MOE_LATENT_SIZE, hparams.moe_latent_size);
+
+ ml.get_key(LLM_KV_HYPER_CONNECTION_COUNT, hparams.dsv4_hc_mult);
+ ml.get_key(LLM_KV_METIS_PASS_COUNT, hparams.metis_n_pass);
+ ml.get_key(LLM_KV_METIS_MEMORY_LENGTH, hparams.metis_n_mem);
+ ml.get_key(LLM_KV_METIS_ROUTE_LENGTH, hparams.metis_n_route);
+ ml.get_key(LLM_KV_ATTENTION_PASS_LORA_RANK, hparams.metis_pass_lora_rank);
+ ml.get_key(LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT, hparams.metis_cont_limit);
+
+ n_pass = hparams.metis_n_pass;
+ n_stream = hparams.dsv4_hc_mult;
+ n_mem = hparams.metis_n_mem;
+ n_route = hparams.metis_n_route;
+ cont_limit = hparams.metis_cont_limit;
+
+ if (n_pass < 1 || hparams.n_layer() % n_pass != 0) {
+ throw std::runtime_error(format("metis16: %u layers do not split into %u passes", hparams.n_layer(), n_pass));
+ }
+ n_layer_phys = hparams.n_layer() / n_pass;
+
+ bool has_attn = false;
+ for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
+ if (hparams.n_head_kv(il) != hparams.n_head_kv(il % n_layer_phys)) {
+ throw std::runtime_error(format("metis16: layer %u does not repeat the layout of layer %u", il, il % n_layer_phys));
+ }
+ hparams.is_recr_impl[il] = hparams.n_head_kv(il) == 0;
+ has_attn = has_attn || !hparams.is_recr_impl[il];
+ }
+ if (!has_attn) {
+ throw std::runtime_error("metis16: at least one attention layer is required");
+ }
+
+ switch (n_layer_phys) {
+ case 12: type = LLM_TYPE_3B; break; // Praxis
+ case 20: type = LLM_TYPE_10B; break; // Logos
+ default: type = LLM_TYPE_UNKNOWN;
+ }
+}
+
+void llama_model_metis16::load_arch_tensors(llama_model_loader &) {
+ LLAMA_LOAD_LOCALS;
+
+ const int64_t d_conv = hparams.ssm_d_conv;
+ const int64_t d_inner = hparams.ssm_d_inner;
+ const int64_t d_state = hparams.ssm_d_state;
+ const int64_t n_ssm_head = hparams.ssm_dt_rank;
+ const int64_t n_group = hparams.ssm_n_group;
+ const int64_t d_in_proj = 2*d_inner + 2*n_group*d_state + n_ssm_head;
+ const int64_t n_latent = hparams.moe_latent_size;
+ const int64_t n_ff_shexp = hparams.n_ff_shexp;
+ const int64_t n_s = n_stream;
+ const int64_t n_p = n_pass;
+ const int64_t n_m = n_mem;
+ const int64_t n_r = n_route;
+
+ int64_t n_attn = 0;
+ for (uint32_t il = 0; il < n_layer_phys; ++il) {
+ n_attn += !hparams.is_recr(il);
+ }
+
+ tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
+ output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+ output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
+ if (output == NULL) {
+ output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
+ }
+
+ stream_embd = create_tensor(tn(LLM_TENSOR_STREAM_EMBD, "weight"), {n_s, n_embd}, 0);
+
+ depth_mem_state_write = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_STATE_WRITE, "weight"), {n_embd, n_m}, 0);
+ depth_mem_meta_write = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_META_WRITE, "weight"), {n_r + 3, n_m}, 0);
+ depth_mem_entry_bias = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ENTRY_BIAS, "weight"), {n_m, n_p*(n_attn + 1)}, 0);
+ depth_mem_query = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_QUERY, "weight"), {n_embd, n_m}, 0);
+ depth_mem_query_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_QUERY, "bias"), {n_m}, 0);
+ depth_mem_key = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_KEY, "weight"), {n_m, n_m}, 0);
+ depth_mem_key_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_KEY, "bias"), {n_m}, 0);
+ depth_mem_value = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_VALUE, "weight"), {n_m, n_m}, 0);
+ depth_mem_value_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_VALUE, "bias"), {n_m}, 0);
+ depth_mem_out = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_OUT, "weight"), {n_m, n_embd}, 0);
+ depth_mem_out_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_OUT, "bias"), {n_embd}, 0);
+ depth_mem_route_state = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE_STATE, "weight"), {n_embd, n_r}, 0);
+ depth_mem_route_retr = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE_RETR, "weight"), {n_embd, n_r}, 0);
+ depth_mem_route_diff = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE_DIFF, "weight"), {n_embd, n_r}, 0);
+ depth_mem_route_hist = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE_HIST, "weight"), {n_r, n_r}, 0);
+ depth_mem_route_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE, "bias"), {n_r}, 0);
+ depth_mem_stream_gate = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_STREAM_GATE, "weight"), {n_embd, n_s}, 0);
+ depth_mem_stream_gate_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_STREAM_GATE, "bias"), {n_s}, 0);
+
+ cont_hidden_state = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN_STATE, "weight"), {n_embd, n_r}, 0);
+ cont_hidden_retr = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN_RETR, "weight"), {n_embd, n_r}, 0);
+ cont_hidden_diff = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN_DIFF, "weight"), {n_embd, n_r}, 0);
+ cont_hidden_route = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN_ROUTE, "weight"), {n_r, n_r}, 0);
+ cont_hidden_b = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN, "bias"), {n_r}, 0);
+ cont_output = create_tensor(tn(LLM_TENSOR_CONT_OUTPUT, "weight"), {n_r, 1}, 0);
+ cont_output_b = create_tensor(tn(LLM_TENSOR_CONT_OUTPUT, "bias"), {1}, 0);
+
+ for (int il = 0; il < (int) n_layer_phys; ++il) {
+ auto & layer = layers[il];
+
+ layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", il), {n_embd}, 0);
+ layer.hc_attn_mix = create_tensor(tn(LLM_TENSOR_HC_ATTN_MIX, "weight", il), {n_s, n_s, n_p}, 0);
+ layer.hc_attn_read = create_tensor(tn(LLM_TENSOR_HC_ATTN_READ, "weight", il), {n_s, n_p}, 0);
+ layer.hc_attn_write = create_tensor(tn(LLM_TENSOR_HC_ATTN_WRITE, "weight", il), {n_s, n_p}, 0);
+ layer.hc_ffn_mix = create_tensor(tn(LLM_TENSOR_HC_FFN_MIX, "weight", il), {n_s, n_s, n_p}, 0);
+ layer.hc_ffn_read = create_tensor(tn(LLM_TENSOR_HC_FFN_READ, "weight", il), {n_s, n_p}, 0);
+ layer.hc_ffn_write = create_tensor(tn(LLM_TENSOR_HC_FFN_WRITE, "weight", il), {n_s, n_p}, 0);
+ layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", il), {n_embd}, 0);
+
+ if (hparams.is_recr(il)) {
+ layer.ssm_in = create_tensor(tn(LLM_TENSOR_SSM_IN, "weight", il), {n_embd, d_in_proj}, 0);
+ layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "weight", il), {d_conv, d_inner + 2*n_group*d_state}, 0);
+ layer.ssm_conv1d_b = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "bias", il), {d_inner + 2*n_group*d_state}, 0);
+ layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", il), {n_ssm_head}, 0);
+ layer.ssm_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", il), {d_inner / n_group, n_group}, 0);
+ layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, "weight", il), {d_inner, n_embd}, 0);
+
+ // no "weight" suffix for these
+ layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, il), {1, n_ssm_head}, 0);
+ layer.ssm_d = create_tensor(tn(LLM_TENSOR_SSM_D, il), {1, n_ssm_head}, 0);
+ } else {
+ const int64_t n_qkv = n_embd_head_k * (hparams.n_head(il) + 2*hparams.n_head_kv(il));
+ const int64_t rank = hparams.metis_pass_lora_rank;
+
+ layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", il), {n_embd, n_qkv}, 0);
+ layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", il), {n_embd_head_k * hparams.n_head(il), n_embd}, 0);
+ layer.attn_pass_lora_down = create_tensor(tn(LLM_TENSOR_ATTN_PASS_LORA_DOWN, "weight", il), {n_embd, rank}, 0);
+ layer.attn_pass_lora_up = create_tensor(tn(LLM_TENSOR_ATTN_PASS_LORA_UP, "weight", il), {rank, n_qkv}, 0);
+ layer.attn_pass_gate = create_tensor(tn(LLM_TENSOR_ATTN_PASS_GATE, "weight", il), {n_p}, 0);
+ }
+
+ const int64_t n_ff_exp = hparams.n_ff_exp(il);
+
+ layer.ffn_latent_down = create_tensor(tn(LLM_TENSOR_FFN_LATENT_DOWN, "weight", il), {n_embd, n_latent}, 0);
+ layer.ffn_latent_up = create_tensor(tn(LLM_TENSOR_FFN_LATENT_UP, "weight", il), {n_latent, n_embd}, 0);
+ layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", il), {n_latent + n_r, n_expert}, 0);
+ layer.ffn_gate_inp_b = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "bias", il), {n_expert}, 0);
+ layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", il), {n_expert}, 0);
+ layer.ffn_exp_embd = create_tensor(tn(LLM_TENSOR_FFN_EXP_EMBD, "weight", il), {n_r, n_expert}, 0);
+ layer.ffn_gate_up_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", il), {n_latent, 2*n_ff_exp, n_expert}, 0);
+ layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", il), {n_ff_exp, n_latent, n_expert}, 0);
+ layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", il), {n_latent, n_ff_shexp}, 0);
+ layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", il), {n_latent, n_ff_shexp}, 0);
+ layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", il), {n_ff_shexp, n_latent}, 0);
+ }
+
+ // the virtual layers of later passes share the physical layer's weights
+ for (int il = n_layer_phys; il < n_layer; ++il) {
+ layers[il] = layers[il % n_layer_phys];
+ }
+}
+
+std::unique_ptr<llm_graph_context> llama_model_metis16::build_arch_graph(const llm_graph_params & params) const {
+ return std::make_unique<graph>(*this, params);
+}
+
+// streams are kept stream-innermost: [n_stream, n_embd, n_tokens]
+ggml_tensor * llama_model_metis16::graph::stream_mean(ggml_tensor * x) const {
+ ggml_tensor * mean = ggml_mean(ctx0, ggml_reshape_2d(ctx0, x, x->ne[0], x->ne[1]*x->ne[2]));
+ return ggml_reshape_2d(ctx0, mean, x->ne[1], x->ne[2]);
+}
+
+// returns the normalized sublayer input
+ggml_tensor * llama_model_metis16::graph::hc_read(ggml_tensor * x, ggml_tensor * mix, ggml_tensor * read, ggml_tensor * norm,
+ int p, ggml_tensor ** mixed, int il) const {
+ const int64_t n_s = x->ne[0];
+ ggml_tensor * x2 = ggml_reshape_2d(ctx0, x, n_s, x->ne[1]*x->ne[2]);
+
+ ggml_tensor * m = ggml_view_2d(ctx0, mix, n_s, n_s, mix->nb[1], p*mix->nb[2]);
+ *mixed = ggml_mul_mat(ctx0, m, x2);
+
+ ggml_tensor * r = ggml_reshape_2d(ctx0, ggml_view_1d(ctx0, read, n_s, p*read->nb[1]), n_s, 1);
+ ggml_tensor * src = ggml_reshape_2d(ctx0, ggml_mul_mat(ctx0, r, x2), x->ne[1], x->ne[2]);
+ return build_norm(src, norm, NULL, LLM_NORM_RMS, il);
+}
+
+ggml_tensor * llama_model_metis16::graph::hc_write(ggml_tensor * mixed, ggml_tensor * write, ggml_tensor * update, int p) const {
+ const int64_t n_s = mixed->ne[0];
+ ggml_tensor * w = ggml_reshape_2d(ctx0, ggml_view_1d(ctx0, write, n_s, p*write->nb[1]), 1, n_s);
+ ggml_tensor * u = ggml_reshape_2d(ctx0, update, 1, update->ne[0]*update->ne[1]);
+ ggml_tensor * out = ggml_add(ctx0, mixed, ggml_mul_mat(ctx0, w, u));
+ return ggml_reshape_3d(ctx0, out, n_s, update->ne[0], update->ne[1]);
+}
+
+// blends the retrieved entries into the streams; returns their summary
+ggml_tensor * llama_model_metis16::graph::mem_read(const llama_model_metis16 & model, ggml_tensor ** x) const {
+ const int64_t n_s = (*x)->ne[0];
+ const int64_t n_m = model.n_mem;
+
+ ggml_tensor * x_e = ggml_cont(ctx0, ggml_permute(ctx0, *x, 1, 0, 2, 3));
+
+ ggml_tensor * q = ggml_mul_mat(ctx0, model.depth_mem_query, ggml_reshape_2d(ctx0, x_e, n_embd, n_s*n_tokens));
+ q = ggml_reshape_3d(ctx0, ggml_add(ctx0, q, model.depth_mem_query_b), n_m, n_s, n_tokens);
+
+ ggml_tensor * kq = ggml_mul_mat(ctx0, bank_k, q);
+ kq = ggml_soft_max_ext(ctx0, kq, nullptr, 1.0f/sqrtf(float(n_m)), 0.0f);
+
+ ggml_tensor * ret = ggml_mul_mat(ctx0, bank_vt, kq);
+ ggml_tensor * proj = ggml_mul_mat(ctx0, model.depth_mem_out, ggml_reshape_2d(ctx0, ret, n_m, n_s*n_tokens));
+ proj = ggml_reshape_3d(ctx0, ggml_add(ctx0, proj, model.depth_mem_out_b), n_embd, n_s, n_tokens);
+
+ ggml_tensor * gate = ggml_sum_rows(ctx0, ggml_mul(ctx0, x_e, model.depth_mem_stream_gate));
+ gate = ggml_sigmoid(ctx0, ggml_add(ctx0, gate, ggml_reshape_2d(ctx0, model.depth_mem_stream_gate_b, 1, n_s)));
+
+ ggml_tensor * proj_s = ggml_cont(ctx0, ggml_permute(ctx0, proj, 1, 0, 2, 3));
+ *x = ggml_add(ctx0, *x, ggml_mul(ctx0, proj_s, ggml_reshape_3d(ctx0, gate, n_s, 1, n_tokens)));
+
+ return stream_mean(proj_s);
+}
+
+void llama_model_metis16::graph::mem_write(const llama_model_metis16 & model, ggml_tensor * x, ggml_tensor * summary,
+ ggml_tensor * entropy, ggml_tensor * confidence, ggml_tensor * cont, int entry) {
+ const int64_t n_m = model.n_mem;
+
+ if (cont == nullptr) {
+ cont = ggml_scale(ctx0, confidence, 0.0f);
+ }
+ ggml_tensor * meta = ggml_concat(ctx0, ggml_concat(ctx0, summary, entropy, 0), ggml_concat(ctx0, confidence, cont, 0), 0);
+
+ ggml_tensor * e = ggml_add(ctx0, ggml_mul_mat(ctx0, model.depth_mem_state_write, stream_mean(x)), ggml_mul_mat(ctx0, model.depth_mem_meta_write, meta));
+ e = ggml_add(ctx0, e, ggml_view_1d(ctx0, model.depth_mem_entry_bias, n_m, entry*model.depth_mem_entry_bias->nb[1]));
+
+ ggml_tensor * k = ggml_add(ctx0, ggml_mul_mat(ctx0, model.depth_mem_key, e), model.depth_mem_key_b);
+ ggml_tensor * v = ggml_add(ctx0, ggml_mul_mat(ctx0, model.depth_mem_value, e), model.depth_mem_value_b);
+ k = ggml_reshape_3d(ctx0, k, n_m, 1, n_tokens);
+ v = ggml_reshape_3d(ctx0, v, 1, n_m, n_tokens);
+ bank_k = bank_k ? ggml_concat(ctx0, bank_k, k, 1) : k;
+ bank_vt = bank_vt ? ggml_concat(ctx0, bank_vt, v, 0) : v;
+}
+
+ggml_tensor * llama_model_metis16::graph::route_features(const llama_model_metis16 & model, ggml_tensor * state,
+ ggml_tensor * summary, ggml_tensor * diff, ggml_tensor * history) const {
+ ggml_tensor * f = ggml_mul_mat(ctx0, model.depth_mem_route_state, state);
+ if (summary) {
+ f = ggml_add(ctx0, f, ggml_mul_mat(ctx0, model.depth_mem_route_retr, summary));
+ }
+ if (diff) {
+ f = ggml_add(ctx0, f, ggml_mul_mat(ctx0, model.depth_mem_route_diff, diff));
+ }
+ if (history) {
+ f = ggml_add(ctx0, f, ggml_mul_mat(ctx0, model.depth_mem_route_hist, history));
+ }
+ return ggml_tanh(ctx0, ggml_add(ctx0, f, model.depth_mem_route_b));
+}
+
+llama_model_metis16::graph::graph(const llama_model_metis16 & model, const llm_graph_params & params) :
+ llm_build_mamba_base(params) {
+ const int64_t n_s = model.n_stream;
+ const int n_p = model.n_pass;
+ const int n_phys = model.n_layer_phys;
+ const int64_t n_r = model.n_route;
+ const int64_t n_hd = hparams.n_embd_head_k();
+ const int64_t n_exp = hparams.n_expert;
+ const int64_t n_used = hparams.n_expert_used();
+
+ ggml_tensor * inpL = build_inp_embd(model.tok_embd);
+ auto * inp = build_inp_mem_hybrid();
+ ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+ ggml_tensor * x = ggml_repeat_4d(ctx0, ggml_reshape_3d(ctx0, inpL, 1, n_embd, n_tokens), n_s, n_embd, n_tokens, 1);
+ x = ggml_add(ctx0, x, model.stream_embd);
+ cb(x, "streams_in", -1);
+
+ ggml_tensor * prev_state = stream_mean(x);
+ ggml_tensor * history = nullptr; // routing history, [n_route, n_tokens]
+ ggml_tensor * cont = nullptr; // depth head reading of the last pass, [1, n_tokens]
+ int n_entry = 0;
+
+ for (int p = 0; p < n_p; ++p) {
+ ggml_tensor * summ = nullptr;
+ ggml_tensor * ent = nullptr;
+ ggml_tensor * conf = nullptr;
+
+ for (int il_p = 0; il_p < n_phys; ++il_p) {
+ const int il = p*n_phys + il_p;
+ const auto & layer = model.layers[il];
+ const bool is_attn = !hparams.is_recr(il);
+ const bool writes = is_attn || il_p == n_phys - 1;
+
+ ggml_tensor * retrieved = bank_k ? mem_read(model, &x) : nullptr;
+ ggml_tensor * feats = route_features(model, stream_mean(x), retrieved, nullptr, history);
+
+ ggml_tensor * mixed = nullptr;
+ ggml_tensor * cur = hc_read(x, layer.hc_attn_mix, layer.hc_attn_read, layer.attn_norm, p, &mixed, il);
+ if (is_attn) {
+ ggml_tensor * qkv = build_lora_mm(layer.wqkv, cur);
+ ggml_tensor * lora = ggml_mul_mat(ctx0, layer.attn_pass_lora_up, ggml_mul_mat(ctx0, layer.attn_pass_lora_down, cur));
+ qkv = ggml_add(ctx0, qkv, ggml_mul(ctx0, lora, ggml_view_1d(ctx0, layer.attn_pass_gate, 1, p*layer.attn_pass_gate->nb[0])));
+
+ const int64_t n_head = hparams.n_head(il);
+ const int64_t n_head_kv = hparams.n_head_kv(il);
+ const size_t es = ggml_element_size(qkv);
+ ggml_tensor * Q = ggml_cont(ctx0, ggml_view_3d(ctx0, qkv, n_hd, n_head, n_tokens, n_hd*es, qkv->nb[1], 0));
+ ggml_tensor * K = ggml_cont(ctx0, ggml_view_3d(ctx0, qkv, n_hd, n_head_kv, n_tokens, n_hd*es, qkv->nb[1], n_hd*n_head*es));
+ ggml_tensor * V = ggml_cont(ctx0, ggml_view_3d(ctx0, qkv, n_hd, n_head_kv, n_tokens, n_hd*es, qkv->nb[1], n_hd*(n_head + n_head_kv)*es));
+
+ cur = build_attn(inp->get_attn(), layer.wo, nullptr, nullptr, Q, K, V, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_hd)), il);
+ } else {
+ cur = build_mamba2_layer(inp->get_recr(), cur, model, ubatch, il);
+ }
+ cb(cur, "mixer_out", il);
+ x = hc_write(mixed, layer.hc_attn_write, cur, p);
+
+ cur = hc_read(x, layer.hc_ffn_mix, layer.hc_ffn_read, layer.ffn_norm, p, &mixed, il);
+ {
+ ggml_tensor * latent = ggml_mul_mat(ctx0, layer.ffn_latent_down, cur);
+ ggml_tensor * logits = ggml_mul_mat(ctx0, layer.ffn_gate_inp, ggml_concat(ctx0, latent, feats, 0));
+ logits = ggml_add(ctx0, logits, layer.ffn_gate_inp_b);
+
+ // the balance bias picks the experts; the weights come from the unbiased logits
+ ggml_tensor * selected = ggml_argsort_top_k(ctx0, ggml_add(ctx0, logits, layer.ffn_exp_probs_b), n_used);
+
+ ggml_tensor * moe_out = build_moe_ffn(latent, nullptr, nullptr, nullptr, layer.ffn_down_exps, nullptr,
+ n_exp, n_used, LLM_FFN_SILU, false, 0.0f, LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX_WEIGHT, il_p,
+ logits, layer.ffn_gate_up_exps, nullptr, nullptr, nullptr, selected);
+ ggml_tensor * shexp = build_ffn(latent,
+ layer.ffn_up_shexp, NULL, NULL,
+ layer.ffn_gate_shexp, NULL, NULL,
+ layer.ffn_down_shexp, NULL, NULL,
+ NULL, LLM_FFN_SILU, LLM_FFN_PAR, il_p);
+ cur = ggml_mul_mat(ctx0, layer.ffn_latent_up, ggml_add(ctx0, moe_out, shexp));
+ cb(cur, "ffn_out", il);
+
+ // routing summary: the chosen experts' identity vectors blended by their weights
+ ggml_tensor * logits3 = ggml_reshape_3d(ctx0, logits, 1, n_exp, n_tokens);
+ ggml_tensor * weights = ggml_soft_max(ctx0, ggml_reshape_2d(ctx0, ggml_get_rows(ctx0, logits3, selected), n_used, n_tokens));
+ ggml_tensor * rows = ggml_get_rows(ctx0, layer.ffn_exp_embd, ggml_reshape_1d(ctx0, ggml_cont(ctx0, selected), n_used*n_tokens));
+ rows = ggml_cont(ctx0, ggml_permute(ctx0, ggml_reshape_3d(ctx0, rows, n_r, n_used, n_tokens), 1, 0, 2, 3));
+ summ = ggml_reshape_2d(ctx0, ggml_mul_mat(ctx0, rows, ggml_reshape_3d(ctx0, weights, n_used, 1, n_tokens)), n_r, n_tokens);
+
+ if (writes) {
+ ggml_tensor * probs = ggml_soft_max(ctx0, logits);
+ ent = ggml_sum_rows(ctx0, ggml_mul(ctx0, probs, ggml_log(ctx0, ggml_clamp(ctx0, probs, 1.0e-9f, 1.0f))));
+ ent = ggml_scale(ctx0, ent, -1.0f);
+ conf = ggml_get_rows(ctx0, ggml_reshape_3d(ctx0, probs, 1, n_exp, n_tokens),
+ ggml_reshape_2d(ctx0, ggml_argmax(ctx0, probs), 1, n_tokens));
+ conf = ggml_reshape_2d(ctx0, conf, 1, n_tokens);
+ }
+ }
+ x = hc_write(mixed, layer.hc_ffn_write, cur, p);
+ cb(x, "streams", il);
+
+ history = history ? ggml_add(ctx0, ggml_scale(ctx0, history, 0.5f), ggml_scale(ctx0, summ, 0.5f)) : ggml_scale(ctx0, summ, 0.5f);
+
+ if (is_attn) {
+ mem_write(model, x, summ, ent, conf, cont, n_entry++);
+ }
+ }
+
+ // end of the pass: one more memory read, the depth head, and an entry recording where the pass left the token
+ ggml_tensor * retrieved = mem_read(model, &x);
+ ggml_tensor * state = stream_mean(x);
+ ggml_tensor * diff = ggml_sub(ctx0, state, prev_state);
+ ggml_tensor * feats = route_features(model, state, retrieved, diff, history);
+
+ ggml_tensor * h = ggml_mul_mat(ctx0, model.cont_hidden_state, state);
+ h = ggml_add(ctx0, h, ggml_mul_mat(ctx0, model.cont_hidden_retr, retrieved));
+ h = ggml_add(ctx0, h, ggml_mul_mat(ctx0, model.cont_hidden_diff, diff));
+ h = ggml_add(ctx0, h, ggml_mul_mat(ctx0, model.cont_hidden_route, feats));
+ h = ggml_silu(ctx0, ggml_add(ctx0, h, model.cont_hidden_b));
+
+ ggml_tensor * logit = ggml_add(ctx0, ggml_mul_mat(ctx0, model.cont_output, h), model.cont_output_b);
+ ggml_tensor * scale = ggml_clamp(ctx0, ggml_sqrt(ctx0, ggml_mean(ctx0, ggml_sqr(ctx0, state))), 1.0e-6f, INFINITY);
+ ggml_tensor * innov = ggml_sqrt(ctx0, ggml_mean(ctx0, ggml_sqr(ctx0, diff)));
+ logit = ggml_add(ctx0, logit, ggml_scale_bias(ctx0, ggml_tanh(ctx0, ggml_div(ctx0, innov, scale)), 1.0f, -0.5f));
+ cont = ggml_sigmoid(ctx0, ggml_clamp(ctx0, logit, -model.cont_limit, model.cont_limit));
+ cb(cont, "continuation", p);
+
+ mem_write(model, x, summ, ent, conf, cont, n_entry++);
+ prev_state = state;
+ }
+
+ ggml_tensor * cur = stream_mean(x);
+ if (inp_out_ids) {
+ cur = ggml_get_rows(ctx0, cur, inp_out_ids);
+ }
+
+ cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
+ cb(cur, "result_norm", -1);
+ res->t_embd = cur;
+
+ cur = build_lora_mm(model.output, cur);
+ cb(cur, "result_output", -1);
+ res->t_logits = cur;
+
+}
diff --git a/src/models/models.h b/src/models/models.h
index 1fc7cae1d..dd3b7b1ed 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -1799,6 +1799,39 @@ struct llama_model_nemotron_h_moe : public llama_model_nemotron_h {
};
+struct llama_model_metis16 : public llama_model_base {
+ llama_model_metis16(const struct llama_model_params & params) : llama_model_base(params) {}
+ void load_arch_hparams(llama_model_loader & ml) override;
+ void load_arch_tensors(llama_model_loader & ml) override;
+
+ uint32_t n_pass = 1;
+ uint32_t n_stream = 1;
+ uint32_t n_mem = 0;
+ uint32_t n_route = 0;
+ uint32_t n_layer_phys = 0;
+ float cont_limit = 0.0f;
+
+ struct graph : public llm_build_mamba_base {
+ graph(const llama_model_metis16 & model, const llm_graph_params & params);
+
+ // memory bank of the current forward: keys [n_mem, n_entry, n_tokens], values [n_entry, n_mem, n_tokens]
+ ggml_tensor * bank_k = nullptr;
+ ggml_tensor * bank_vt = nullptr;
+
+ ggml_tensor * stream_mean(ggml_tensor * x) const;
+ ggml_tensor * hc_read(ggml_tensor * x, ggml_tensor * mix, ggml_tensor * read, ggml_tensor * norm, int p, ggml_tensor ** mixed, int il) const;
+ ggml_tensor * hc_write(ggml_tensor * mixed, ggml_tensor * write, ggml_tensor * update, int p) const;
+ ggml_tensor * mem_read(const llama_model_metis16 & model, ggml_tensor ** x) const;
+ void mem_write(const llama_model_metis16 & model, ggml_tensor * x, ggml_tensor * summary,
+ ggml_tensor * entropy, ggml_tensor * confidence, ggml_tensor * cont, int entry);
+ ggml_tensor * route_features(const llama_model_metis16 & model, ggml_tensor * state, ggml_tensor * summary,
+ ggml_tensor * diff, ggml_tensor * history) const;
+ };
+
+ std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
+
struct llama_model_exaone : public llama_model_base {
llama_model_exaone(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv
index 254b8a03a..eda5119b5 100644
--- a/tests/fusion/MTL.csv
+++ b/tests/fusion/MTL.csv
@@ -188,6 +188,8 @@ maple ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV,
mellum ,0 ,any ,MUL+ADD , 4
mellum ,0 ,any ,RMS_NORM+MUL , 9
mellum ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
+metis16 ,0 ,any ,MUL+ADD , 8
+metis16 ,0 ,any ,RMS_NORM+MUL , 11
mimo2 ,0 ,any ,MUL+ADD , 4
mimo2 ,0 ,any ,RMS_NORM+MUL , 5
minicpm ,0 ,any ,RMS_NORM+MUL , 5
diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp
index 32569dc86..fe949f2c2 100644
--- a/tests/test-llama-archs.cpp
+++ b/tests/test-llama-archs.cpp
@@ -295,6 +295,8 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
n_ff = 192;
} else if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) {
n_layer = 3;
+ } else if (arch == LLM_ARCH_METIS16) {
+ n_layer = 4; // 2 passes over 2 physical layers, one attention and one Mamba-2
} else if (arch == LLM_ARCH_CHAMELEON) {
n_vocab = 10240;
} else if (arch == LLM_ARCH_QWEN3TTS) {
@@ -369,6 +371,15 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head_per_layer);
}
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, n_head_per_layer);
+ } else if (arch == LLM_ARCH_METIS16) {
+ // the layer types repeat in every pass
+ std::vector<uint32_t> n_head_kv_per_layer;
+ n_head_kv_per_layer.reserve(n_layer);
+ for (uint32_t il = 0; il < n_layer; il++) {
+ n_head_kv_per_layer.push_back(il % 2 == 0 ? n_head_kv : 0);
+ }
+ ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head);
+ ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, n_head_kv_per_layer);
} else {
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head);
ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41) ? uint32_t(1) : n_head_kv);
@@ -616,6 +627,17 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 7.0f);
}
+ if (arch == LLM_ARCH_METIS16) {
+ ms.add_kv(LLM_KV_METIS_PASS_COUNT, uint32_t(2));
+ ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4));
+ ms.add_kv(LLM_KV_METIS_MEMORY_LENGTH, uint32_t(64));
+ ms.add_kv(LLM_KV_METIS_ROUTE_LENGTH, uint32_t(32));
+ ms.add_kv(LLM_KV_ATTENTION_PASS_LORA_RANK, uint32_t(8));
+ ms.add_kv(LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT, 4.0f);
+ ms.add_kv(LLM_KV_SSM_NORM_EPS, 1e-5f);
+ ms.add_kv(LLM_KV_MOE_LATENT_SIZE, n_embd/2);
+ }
+
if (arch == LLM_ARCH_LIMITE) {
ms.add_kv(LLM_KV_ATTENTION_SCALE, 0.1f);
ms.add_kv(LLM_KV_ATTENTION_XSA_EPS, 1e-4f);
@@ -972,6 +994,7 @@ static bool moe_mandatory(const llm_arch arch) {
case LLM_ARCH_MAPLE:
case LLM_ARCH_MUSE_SPARK:
case LLM_ARCH_KOLIBRI1:
+ case LLM_ARCH_METIS16:
return true;
default:
return false;