Commit 5c008e000 for llama.cpp
commit 5c008e00085f4d3e394f9a1be649284999de1326
Author: Piotr Wilkin (ilintar) <piotr.wilkin@syndatis.com>
Date: Sun Nov 15 00:51:31 2026 +0100
feat: add Inkling 2 support (text + vision) (#32217)
* convert : add Inkling2 text GGUF conversion
Assisted-by: Codex
* llama : implement Inkling2 text inference
Assisted-by: Codex
* tests : cover Inkling2 text support
Assisted-by: Codex
* conversion : dequantize packed Inkling2 experts in row chunks
Decode per-layer NVFP4 and MXFP8 schemes, validate companions, and use stored scale shapes with a warning for the declared MXFP8 [32,32] grid. Reorder packed w13 rows and their scale rows together. Keep normal GGUF outtype selection and 3D expert tensors.
Stream 128 rows per expert through LazyChunkedTensor. Real w13 (384x4608x4608) needs a 2.25 MiB F32 chunk instead of a 30.375 GiB expanded layer. Real-shaped sparse probes peak below 5.8 MiB of Python/NumPy chunk allocations, plus 5.2 MiB of callback metadata: budget about 16 MiB per projection including mapped pages, excluding runtime, other tensors and OS cache. Full-model chunk metadata is roughly 370 MiB.
Packed synthetic GGUF weights match the independent pinned vLLM decode bit-for-bit. CPU prefill/decode logits remain within 3.1e-6 absolute error, with 100% argmax agreement. Float fixtures, original-name conversion and CPU architecture checks pass. Real checkpoint conversion remains untested.
Assisted-by: Codex
* convert : add Inkling2 vision mmproj support
Assisted-by: Codex
* mtmd : implement Inkling2 still-image encoder
Assisted-by: Codex
* inkling2 : scope image loading
Assisted-by: Codex
* inkling2 : enable flash attention with folded bias and sinks
Assisted-by: Codex
* cuda : route per-head flash attention masks through vec
Assisted-by: Codex
* mtmd : support Inkling2 temporal frame means
Assisted-by: Codex
* style : restore shared image comment
Assisted-by: Codex
* tests : enlarge Inkling2 context fixture
Assisted-by: Codex
* inkling2 : load per-layer expert feed-forward widths
Assisted-by: Codex
* tests : add Inkling2 Metal fusion baselines
Assisted-by: Codex
* inkling2 : group and align registry entries
Assisted-by: Fable
* convert : regenerate the Inkling2 tokenizer hash with the update script
Assisted-by: Fable
* conversion : revert the print_registered_models change
Assisted-by: Fable
* conversion : sort the Inkling2 model map entries
Assisted-by: Fable
* convert : map Inkling2 vision checkpoint names in the tensor table
Assisted-by: Fable
* convert : map Inkling2 text checkpoint names in the tensor table
Assisted-by: Fable
* tests : document the Inkling2 packed expert conversion test
Assisted-by: Fable
* tests : cover Inkling2 checkpoint and transformers tensor names
Assisted-by: Fable
* llama : place Inkling2 fields and list entries by file convention
Assisted-by: Fable
* gguf-py : align Inkling2 tensor mapping comments
Assisted-by: Fable
* mtmd : place Inkling2 entries by file convention
Assisted-by: Fable
* tests : place Inkling2 cases after the existing ones
Assisted-by: Fable
* cuda : index per-head flash attention masks in the tile and mma kernels
Assisted-by: Claude
Claude-Session: https://claude.ai/code/session_01GbYSza1yxgmTz8wdr6uV8Q
* inkling2 : project the relative bias before the gather, drop arch branches
Assisted-by: Claude
Claude-Session: https://claude.ai/code/session_01GbYSza1yxgmTz8wdr6uV8Q
* convert : require Inkling2 hparams, add inkling2 pre-tokenizer
Assisted-by: Claude
Claude-Session: https://claude.ai/code/session_01GbYSza1yxgmTz8wdr6uV8Q
* mtmd : cap Inkling2 image size, drop video frame averaging
Assisted-by: Claude
Claude-Session: https://claude.ai/code/session_01GbYSza1yxgmTz8wdr6uV8Q
* inkling2 : fix CI build, Meta device assert and Windows conversion test
Assisted-by: Claude
* tests : use the default sliding window for the Inkling2 test model
Assisted-by: Claude
* simplify mtmd part
* give _v arch for vision
* remove alpha channel handling, it should be a follow up
* explicit rule about audio + vision models
* add audio support
* rm audio
* rm gguf mtmd rope base
* address review comment
* nits
* do not normalize embd from mtmd
---------
Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
diff --git a/conversion/__init__.py b/conversion/__init__.py
index 10386d5c0..afdd73e4c 100644
--- a/conversion/__init__.py
+++ b/conversion/__init__.py
@@ -145,6 +145,8 @@ TEXT_MODEL_MAP: dict[str, str] = {
"HYV3ForCausalLM": "hunyuan",
"HYV4ForCausalLM": "hy_v4",
"IQuestCoderForCausalLM": "llama",
+ "Inkling2ForCausalLM": "inkling2",
+ "Inkling2ForConditionalGeneration": "inkling2",
"InternLM2ForCausalLM": "internlm",
"InternLM3ForCausalLM": "internlm",
"JAISLMHeadModel": "jais",
@@ -369,6 +371,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
"GraniteSpeechPlusForConditionalGeneration": "granite",
"HunYuanVLForConditionalGeneration": "hunyuan",
"Idefics3ForConditionalGeneration": "smolvlm",
+ "Inkling2ForConditionalGeneration": "inkling2",
"InternVisionModel": "internvl",
"JanusForConditionalGeneration": "januspro",
"KimiK25ForConditionalGeneration": "kimivl",
diff --git a/conversion/base.py b/conversion/base.py
index 58ddb5def..b8461072e 100644
--- a/conversion/base.py
+++ b/conversion/base.py
@@ -2432,6 +2432,9 @@ class TextModel(ModelBase):
if chkhsh == "a9af07a84191f55098b248ae6f3dfe9e32d3190bebe8eafd91c1ddec9bc3449f":
# ref: https://huggingface.co/IFM/K2-Horizon-36B
res = "k2-horizon"
+ if chkhsh == "5635e1c8d621dff7b4825f41f2f97b6ceb39bcf2d5c541db28577080d6efd5e5":
+ # ref: https://huggingface.co/thinkingmachines/Inkling-2-Small
+ res = "inkling2"
if res is None:
logger.warning("\n")
diff --git a/conversion/inkling2.py b/conversion/inkling2.py
new file mode 100644
index 000000000..5b8396378
--- /dev/null
+++ b/conversion/inkling2.py
@@ -0,0 +1,327 @@
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+from functools import partial
+from typing import Callable, Iterable, TYPE_CHECKING, cast
+
+import numpy as np
+import torch
+
+if TYPE_CHECKING:
+ from torch import Tensor
+
+from .base import LazyTorchTensor, ModelBase, MmprojModel, TextModel, gguf, logger
+
+
+def _e4m3_table() -> np.ndarray:
+ codes = np.arange(256, dtype=np.uint8)
+ exponent = ((codes & 0x7F) >> 3).astype(np.int32)
+ mantissa = (codes & 7).astype(np.float32)
+ values = np.where(exponent == 0, np.ldexp(mantissa, -9), np.ldexp(1 + mantissa / 8, exponent - 7))
+ values = np.where(codes & 0x80, -values, values)
+ values[(codes & 0x7F) == 0x7F] = np.nan
+ return values.astype(np.float32)
+
+
+_E4M3 = _e4m3_table()
+_E2M1 = np.array([0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6], dtype=np.float32)
+_EXPERT_CHUNK_ROWS = 128
+
+
+def _dequant_expert_chunk(weight: Callable[[], Tensor], scale: Callable[[], Tensor], amax: Callable[[], Tensor] | None, denominator: Callable[[], Tensor] | None, name: str, expert: int, rows: slice) -> np.ndarray:
+ # Fresh lazy tensors release their mappings after each chunk instead of caching the layer.
+ packed = LazyTorchTensor.to_eager(weight().view(torch.uint8)[expert, rows]).numpy()
+ scales = LazyTorchTensor.to_eager(scale().view(torch.uint8)[expert, rows]).numpy()
+ if amax is not None:
+ block_scale = _E4M3[scales]
+ if np.isnan(block_scale).any():
+ raise ValueError(f"Inkling2: {name}_scale expert {expert}: E4M3FN NaN")
+ values = np.empty((packed.shape[0], packed.shape[1] * 2), dtype=np.float32)
+ values[:, 0::2] = _E2M1[packed & 15]
+ values[:, 1::2] = _E2M1[packed >> 4]
+ values.reshape(values.shape[0], -1, 16)[:] *= block_scale[:, :, None]
+ assert denominator is not None
+ a = LazyTorchTensor.to_eager(amax()[expert]).item()
+ d = LazyTorchTensor.to_eager(denominator()).item()
+ if not np.isfinite(a) or a < 0:
+ raise ValueError(f"Inkling2: {name}_amax expert {expert}: expected finite nonnegative value")
+ if not np.isfinite(d) or d <= 0:
+ raise ValueError(f"Inkling2: {name}_outer_scale_denominator expert {expert}: expected finite positive value")
+ with np.errstate(over="ignore", invalid="ignore"):
+ values *= np.float32(a) / np.float32(d)
+ else:
+ if np.any(scales == 255):
+ raise ValueError(f"Inkling2: {name}_scale expert {expert}: UE8M0 byte 255 is NaN")
+ values = _E4M3[packed]
+ if np.isnan(values).any():
+ raise ValueError(f"Inkling2: {name} expert {expert}: E4M3FN NaN")
+ block_scale = np.ldexp(np.ones(scales.shape, dtype=np.float32), scales.astype(np.int32) - 127)
+ with np.errstate(over="ignore", invalid="ignore"):
+ values.reshape(values.shape[0], -1, 32)[:] *= block_scale[:, :, None]
+ if not np.isfinite(values).all():
+ raise ValueError(f"Inkling2: {name} expert {expert}: non-finite dequantized weight")
+ return values
+
+
+def _is_inkling2(dir_model: Path) -> bool:
+ if not (dir_model / "config.json").is_file():
+ return False
+ with open(dir_model / "config.json", encoding="utf-8") as f:
+ return json.load(f).get("model_type") in ("inkling2", "inkling2_text")
+
+
+@ModelBase.register_hparams_loader(_is_inkling2)
+def _load_inkling2_hparams(dir_model: Path):
+ # AutoConfig injects MXFP8 defaults and can replace the original layer schedule.
+ with open(dir_model / "config.json", encoding="utf-8") as f:
+ return json.load(f)
+
+
+@ModelBase.register("Inkling2ForConditionalGeneration", "Inkling2ForCausalLM")
+@ModelBase.example("thinkingmachines/Inkling-2-Small")
+class Inkling2Model(TextModel):
+ model_arch = gguf.MODEL_ARCH.INKLING2
+
+ def __init__(self, dir_model: Path, *args, **kwargs):
+ self._expert_quant: dict[str, dict] = {}
+ self._packed_experts: dict[str, gguf.LazyChunkedTensor] = {}
+ quant_path = dir_model / "hf_quant_config.json"
+ if quant_path.is_file():
+ with open(quant_path, encoding="utf-8") as f:
+ groups = json.load(f).get("model", {}).get("parameter_groups", [])
+ for group in groups:
+ for name in group.get("tensors", []):
+ item = self.filter_tensors((name, lambda: torch.empty(0)))
+ if item is not None and item[0].endswith((".w13_weight", ".w2_weight")):
+ name = item[0]
+ if name in self._expert_quant:
+ raise ValueError(f"Inkling2: duplicate hf_quant_config.json scheme for {name}")
+ self._expert_quant[name] = group
+ super().__init__(dir_model, *args, **kwargs)
+ logger.warning("Inkling2: converting the text backbone only; vision and MTP tensors are skipped")
+ hp = self.hparams
+ for name, default in (("q_bias", False), ("o_bias", False), ("n_shared_experts", 0), ("use_value_norm_no_gamma", True), ("attn_res_normalize_values", True), ("scoring_func", "sigmoid")):
+ if hp.get(name, default) != default:
+ raise ValueError(f"Inkling2: unsupported {name}={hp[name]!r}")
+ types = hp.get("layer_types")
+ if types is None:
+ local = hp["local_layer_ids"]
+ types = ["hybrid_sliding" if i in local else "hybrid" for i in range(self.block_count)]
+ if len(types) != self.block_count or set(types) - {"hybrid", "hybrid_sliding"}:
+ raise ValueError("Inkling2: invalid layer_types")
+ self.is_swa = [t == "hybrid_sliding" for t in types]
+
+ def dequant_model(self):
+ for name in list(self.model_tensors):
+ if not name.endswith((".w13_weight", ".w2_weight")):
+ continue
+ weight = self.model_tensors[name]
+ tensor = weight()
+ companions = [name + suffix for suffix in ("_scale", "_amax", "_outer_scale_denominator")]
+ if tensor.dtype.is_floating_point and tensor.element_size() > 1:
+ if any(n in self.model_tensors for n in companions):
+ raise ValueError(f"Inkling2: floating-point {name} has packed quantization companions")
+ continue
+ config = self._expert_quant.get(name, {})
+ scheme = config.get("weight_transform")
+ if scheme not in ("nvfp4_nearest", "mxfp8_nearest"):
+ raise ValueError(f"Inkling2: {name}: missing or unsupported hf_quant_config.json scheme {scheme!r}")
+ nvfp4 = scheme == "nvfp4_nearest"
+ if nvfp4 and config.get("weight_scale_format") != "amax_denominator":
+ raise ValueError(f"Inkling2: {name}: expected NVFP4 amax_denominator scale format")
+ required = companions if nvfp4 else companions[:1]
+ for companion in required:
+ if companion not in self.model_tensors:
+ raise ValueError(f"Inkling2: {name}: missing companion {companion}")
+ if not nvfp4 and any(n in self.model_tensors for n in companions[1:]):
+ raise ValueError(f"Inkling2: {name}: MXFP8 must not have amax or outer_scale_denominator")
+ scale = self.model_tensors[companions[0]]
+ scale_tensor = scale()
+ weight_dtype, scale_dtype = (torch.uint8, torch.float8_e4m3fn) if nvfp4 else (torch.float8_e4m3fn, torch.uint8)
+ if tensor.dtype != weight_dtype or scale_tensor.dtype != scale_dtype:
+ raise ValueError(f"Inkling2: {name}: expected {weight_dtype} weights and {scale_dtype} scales, got {tensor.dtype} and {scale_tensor.dtype}")
+ experts = self.hparams["n_routed_experts"]
+ hidden = self.hparams["hidden_size"]
+ inter = self.find_hparam(["moe_intermediate_size", "intermediate_size"])
+ w13 = name.endswith(".w13_weight")
+ rows, cols = (2 * inter, hidden) if w13 else (hidden, inter)
+ shape = (experts, rows, cols)
+ if min(shape) <= 0 or cols % (16 if nvfp4 else 32) or tuple(tensor.shape) != (experts, rows, cols // (2 if nvfp4 else 1)):
+ raise ValueError(f"Inkling2: {name}: invalid packed shape {tuple(tensor.shape)} for logical shape {shape}")
+ ss = tuple(scale_tensor.shape)
+ if len(ss) != 3 or ss[0] != experts or min(ss) <= 0 or rows % ss[1] or cols % ss[2]:
+ raise ValueError(f"Inkling2: {name}_scale: shape {ss} does not divide logical shape {shape}")
+ block = (rows // ss[1], cols // ss[2])
+ if block != (1, 16 if nvfp4 else 32):
+ raise ValueError(f"Inkling2: {name}_scale: unsupported stored block shape {block}")
+ if list(block) != config.get("weight_block_shape"):
+ logger.warning("Inkling2: %s: hf_quant_config.json weight_block_shape %s conflicts with stored weight/scale shapes %s/%s; using %s", name, config.get("weight_block_shape"), tuple(tensor.shape), ss, list(block))
+ amax = denominator = None
+ if nvfp4:
+ amax, denominator = (self.model_tensors[n] for n in companions[1:])
+ for companion, gen, expected in ((companions[1], amax, (experts,)), (companions[2], denominator, (1,))):
+ t = gen()
+ if t.dtype != torch.float32 or tuple(t.shape) != expected:
+ raise ValueError(f"Inkling2: {companion}: expected F32 {expected}, got {t.dtype} {tuple(t.shape)}")
+ chunks: list[Callable[[], np.ndarray]] = []
+ for expert in range(experts):
+ for half in range(2 if w13 else 1):
+ for start in range(0, rows // 2 if w13 else rows, _EXPERT_CHUNK_ROWS):
+ stop = min(start + _EXPERT_CHUNK_ROWS, rows // 2 if w13 else rows)
+ # The same packed row slice selects the weight and its scale groups.
+ row_slice = slice(2 * start + half, 2 * stop + half, 2) if w13 else slice(start, stop)
+ chunks.append(partial(_dequant_expert_chunk, weight, scale, amax, denominator, name, expert, row_slice))
+ self._packed_experts[name] = gguf.LazyChunkedTensor(chunks, shape, np.float32)
+ self.model_tensors[name] = lambda shape=shape: torch.empty(shape, dtype=torch.float32, device="meta")
+ for companion in required:
+ del self.model_tensors[companion]
+ for name, gen in self.model_tensors.items():
+ t = gen()
+ if not t.dtype.is_floating_point or t.element_size() == 1 or name.endswith(("_weight_scale", "_weight_amax", "_weight_outer_scale_denominator")):
+ raise ValueError(f"Inkling2: unexpected packed tensor or orphan companion {name}")
+
+ @classmethod
+ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+ name, gen = item
+ if any(p in name for p in ("visual.", "vision_tower.", "mtp.", "markov_head", "nextn_predict")):
+ return None
+ name = name.replace("model.language_model.", "model.").replace("model.llm.", "model.")
+ # the mapping table cannot shift a block index: attn_queries.{i} feeds block i + 1
+ name = re.sub(r"attn_res\.attn_queries\.(\d+)", lambda m: f"layers.{int(m[1]) + 1}.attention_input.query", name)
+ # the table maps a module, not its bias: gate.weight is the router, gate.bias the score correction
+ name = name.replace("mlp.gate.bias", "mlp.gate.e_score_correction_bias")
+ return name, gen
+
+ def get_vocab_base(self):
+ tokens, types, pre = super().get_vocab_base()
+ n = self.hparams["unpadded_vocab_size"]
+ return tokens[:n], types[:n], pre
+
+ def set_vocab(self):
+ tokens, types, pre = self.get_vocab_base()
+ self.gguf_writer.add_tokenizer_model("gpt2")
+ self.gguf_writer.add_tokenizer_pre(pre)
+ self.gguf_writer.add_token_list(tokens)
+ self.gguf_writer.add_token_types(types)
+ special = gguf.SpecialVocab(self.dir_model, load_merges=True)
+ eos = self.hparams["eos_token_id"]
+ special.special_token_ids["eos"] = eos
+ special.special_token_ids["eot"] = eos
+ special.add_special_token["bos"] = False
+ special.add_to_gguf(self.gguf_writer)
+
+ def set_gguf_parameters(self):
+ hp = self.hparams
+ hp["num_attention_heads"] = [hp.get("swa_num_attention_heads", hp["num_attention_heads"]) if s else hp["num_attention_heads"] for s in self.is_swa]
+ hp["num_key_value_heads"] = [hp.get("swa_num_key_value_heads", hp["num_key_value_heads"]) if s else hp["num_key_value_heads"] for s in self.is_swa]
+ for name in ("n_routed_experts", "num_experts_per_tok"):
+ if name not in hp:
+ raise KeyError(f"Inkling2: missing {name}")
+ super().set_gguf_parameters()
+ w = self.gguf_writer
+ w.add_vocab_size(hp["vocab_size"])
+ w.add_vocab_size_unpadded(hp["unpadded_vocab_size"])
+ w.add_expert_feed_forward_length(self.find_hparam(["moe_intermediate_size", "intermediate_size"]))
+ w.add_key_length_swa(hp.get("swa_head_dim", hp["head_dim"]))
+ w.add_value_length_swa(hp.get("swa_head_dim", hp["head_dim"]))
+ w.add_rope_dimension_count(0)
+ w.add_sliding_window(hp["sliding_window_size"])
+ w.add_sliding_window_pattern(self.is_swa)
+ w.add_shortconv_l_cache(self.find_hparam(["conv_kernel_size", "sconv_kernel_size"]))
+ w.add_expert_weights_scale(hp["route_scale"])
+ w.add_activation_situ_beta(hp["mlp_gate_softcap"])
+ w.add_activation_situ_linear_beta(hp["mlp_up_softcap"])
+ w.add_attn_res_block_size(hp["n_layers_per_attn_res_block"])
+ w.add_final_logit_softcapping(hp["final_logit_softcapping"])
+ w.add_relative_length(hp["d_rel"])
+ w.add_relative_extent(hp["rel_extent"])
+ w.add_qk_clip(hp["qk_l2_clip_threshold"])
+ w.add_relative_clip(hp["rel_l2_clip_threshold"])
+ w.add_log_scale_floor(hp["log_scaling_n_floor"])
+ w.add_log_scale_alpha(hp["log_scaling_alpha"])
+
+ def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
+ if new_name.endswith(("_conv.weight", "_clip.weight", "_res_query.weight", "attn_rel_bias.weight")):
+ return gguf.GGMLQuantizationType.F32
+ return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ if name in self._packed_experts:
+ target = name.replace(".w13_weight", ".gate_up_proj").replace(".w2_weight", ".down_proj")
+ yield self.map_tensor_name(target, try_suffixes=()) + ".weight", cast("Tensor", self._packed_experts[name])
+ return
+ if name.endswith(".w13_weight"):
+ shape = data_torch.shape
+ data_torch = data_torch.reshape(shape[0], shape[1] // 2, 2, shape[2]).transpose(1, 2).reshape(shape)
+ name = name.replace(".w13_weight", ".gate_up_proj")
+ name = name.replace(".w2_weight", ".down_proj")
+ for static, conv in ((".static_weight", ".conv1d.weight"), (".static_w", ".weight")):
+ if name.endswith(static):
+ data_torch = data_torch.flip(0).T.contiguous()
+ name = name.removesuffix(static) + conv
+ if name.endswith((".conv1d.weight", "sconv.weight")):
+ data_torch = data_torch.squeeze(1) if data_torch.ndim == 3 else data_torch
+ elif name.endswith(".rel_logits_proj.proj"):
+ data_torch = data_torch.T.contiguous()
+ # a name the table maps without a suffix is a bare parameter
+ new_name = self.tensor_map.get_name(name)
+ if new_name is not None:
+ suffix = ".bias" if name.endswith(".e_score_correction_bias") else ".weight"
+ yield new_name + suffix, data_torch
+ else:
+ yield from super().modify_tensors(data_torch, name, bid)
+
+
+@ModelBase.register("Inkling2ForConditionalGeneration")
+class Inkling2VisionModel(MmprojModel):
+ def get_vision_config(self):
+ config = self.global_config.get("vision_config")
+ if not config:
+ return None
+ config = dict(config)
+ config["patch_size"] = config.get("native_patch_size", config.get("patch_size", 20))
+ config["spatial_merge_size"] = config.get("adapter_merge_size", config.get("spatial_merge_size", 2))
+ config["intermediate_size"] = config.get("intermediate_size") or int(config["hidden_size"] * config.get("mlp_ratio", 4.0))
+ config["image_size"] = config["posemb_grid_size"] * config["patch_size"]
+ if config.get("num_channels", 3) != 3 or config.get("hidden_act", "gelu") != "gelu":
+ raise ValueError("Inkling2 vision requires RGB patches and exact GELU")
+ return config
+
+ def set_gguf_parameters(self):
+ self.preprocessor_config.setdefault("image_mean", [0.5, 0.5, 0.5])
+ self.preprocessor_config.setdefault("image_std", [0.5, 0.5, 0.5])
+ super().set_gguf_parameters()
+ config = self.hparams
+ rope = config.get("rope_parameters") or {"rope_type": "axial", "rope_theta": 10000.0}
+ if rope["rope_type"] != "axial" or rope["rope_theta"] != 10000.0:
+ raise ValueError("Inkling2 vision requires axial RoPE with theta 10000")
+ w = self.gguf_writer
+ w.add_clip_vision_projector_type(gguf.VisionProjectorType.INKLING2V)
+ w.add_vision_attention_layernorm_eps(config["layer_norm_eps"])
+ w.add_vision_spatial_merge_size(config["spatial_merge_size"])
+
+ @classmethod
+ def filter_tensors(cls, item):
+ name, gen = item
+ if not name.startswith(("model.visual.", "model.vision_tower.")):
+ return None
+ return name, gen
+
+ # mm.{bid} takes the projector layer index, the mapping table cannot derive it from these names
+ _projector_layers = {"model.visual.proj_in": 1, "model.visual.proj_out": 2, "model.vision_tower.merger.linear_1": 1, "model.vision_tower.merger.linear_2": 2}
+
+ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+ module, _, suffix = name.rpartition(".")
+ if module in self._projector_layers:
+ yield self.format_tensor_name(gguf.MODEL_TENSOR.V_MMPROJ, self._projector_layers[module], "." + suffix), data_torch
+ return
+ # lay out the linear patch embedding (HWC patches) as a conv2d for build_inp()
+ if name.endswith(".patch_embedding.weight"):
+ ps = self.hparams["patch_size"]
+ data_torch = data_torch.reshape(data_torch.shape[0], ps, ps, 3).permute(0, 3, 1, 2)
+ # these are bare parameters, they have no .weight
+ if name.endswith(("position_embedding", ".lambda_1", ".lambda_2", ".gamma")):
+ name = name.removesuffix(".gamma") + ".weight"
+ yield from super().modify_tensors(data_torch, name, bid)
diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py
index 4340a9fee..ff5fca4ff 100755
--- a/convert_hf_to_gguf_update.py
+++ b/convert_hf_to_gguf_update.py
@@ -167,6 +167,7 @@ models = [
{"name": "ufakzeka", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/ufakai/ufakzeka-1", },
{"name": "mmbert", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jhu-clsp/mmBERT-base", },
{"name": "k2-horizon", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/IFM/K2-Horizon-36B", },
+ {"name": "inkling2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/thinkingmachines/Inkling-2-Small", },
]
# some models are known to be broken upstream, so we will skip them as exceptions
diff --git a/ggml/src/ggml-cuda/fattn-common.cuh b/ggml/src/ggml-cuda/fattn-common.cuh
index 6667ca400..0e78b9a5b 100644
--- a/ggml/src/ggml-cuda/fattn-common.cuh
+++ b/ggml/src/ggml-cuda/fattn-common.cuh
@@ -40,7 +40,7 @@ typedef void (* fattn_kernel_t)(
const int32_t nb11, const int32_t nb12, const int64_t nb13,
const int32_t nb21, const int32_t nb22, const int64_t nb23,
const int32_t ne31, const int32_t ne32, const int32_t ne33,
- const int32_t nb31, const int32_t nb32, const int64_t nb33);
+ const int32_t nb31, const int64_t nb32, const int64_t nb33);
typedef float (*vec_dot_KQ_t)(
const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8 , const void * __restrict__ Q_ds);
@@ -1170,7 +1170,7 @@ void launch_fattn(
// Optional optimization where the mask is scanned to determine whether part of the calculation can be skipped.
// Only worth the overhead if there is at lease one FATTN_KQ_STRIDE x FATTN_KQ_STRIDE square to be skipped or
// multiple sequences of possibly different lengths, or kv_rows lists that end with padding.
- const bool scan_mask = !use_sparse && mask && n_kv_cols % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1 || kv_rows);
+ const bool scan_mask = !use_sparse && mask && mask->ne[2] == 1 && n_kv_cols % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1 || kv_rows);
if (scan_mask) {
const int64_t s31 = mask->nb[1] / sizeof(half2);
const int64_t s33 = mask->nb[3] / sizeof(half2);
diff --git a/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/ggml/src/ggml-cuda/fattn-mma-f16.cuh
index 07a6db842..f4c096845 100644
--- a/ggml/src/ggml-cuda/fattn-mma-f16.cuh
+++ b/ggml/src/ggml-cuda/fattn-mma-f16.cuh
@@ -1930,7 +1930,7 @@ static __global__ void flash_attn_ext_f16(
const int32_t nb11, const int32_t nb12, const int64_t nb13,
const int32_t nb21, const int32_t nb22, const int64_t nb23,
const int32_t ne31, const int32_t ne32, const int32_t ne33,
- const int32_t nb31, const int32_t nb32, const int64_t nb33) {
+ const int32_t nb31, const int64_t nb32, const int64_t nb33) {
ggml_cuda_pdl_sync(); // TODO optimize placement
#if defined(FLASH_ATTN_AVAILABLE) && (defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE))
const char * GGML_CUDA_RESTRICT Q = Q_ptr;
@@ -2033,7 +2033,7 @@ static __global__ void flash_attn_ext_f16(
const float2 * Q_f2 = (const float2 *) (Q + nb03*sequence + nb02*zt_Q);
const half2 * K_h2 = (const half2 *) (K + nb13*sequence + nb12*z_KV);
const half * mask_h = ncols2 == 1 && !mask ? nullptr :
- (const half *) (mask + nb33*(sequence % ne33));
+ (const half *) (mask + nb33*(sequence % ne33) + nb32*(zt_Q % ne32));
float2 * dstk = ((float2 *) dst) + (sequence*ne01.z*ne02 + zt_Q) * (DV/2);
const half2 * V_h2 = V_is_K_view ? K_h2 : (const half2 *) (V + nb23*sequence + nb22*z_KV);
@@ -2083,7 +2083,7 @@ static __global__ void flash_attn_ext_f16(
const float2 * Q_f2 = (const float2 *) (Q + nb03*sequence + nb02*zt_Q);
const half2 * K_h2 = (const half2 *) (K + nb13*sequence + nb12*z_KV);
const half * mask_h = ncols2 == 1 && !mask ? nullptr :
- (const half *) (mask + nb33*(sequence % ne33));
+ (const half *) (mask + nb33*(sequence % ne33) + nb32*(zt_Q % ne32));
float2 * dstk = ((float2 *) dst) + (sequence*ne01.z*ne02 + zt_Q) * (DV/2);
const half2 * V_h2 = V_is_K_view ? K_h2 : (const half2 *) (V + nb23*sequence + nb22*z_KV);
diff --git a/ggml/src/ggml-cuda/fattn-tile.cuh b/ggml/src/ggml-cuda/fattn-tile.cuh
index ba4628fec..177b2f065 100644
--- a/ggml/src/ggml-cuda/fattn-tile.cuh
+++ b/ggml/src/ggml-cuda/fattn-tile.cuh
@@ -812,7 +812,7 @@ static __global__ void flash_attn_tile(
const int32_t nb11, const int32_t nb12, const int64_t nb13,
const int32_t nb21, const int32_t nb22, const int64_t nb23,
const int32_t ne31, const int32_t ne32, const int32_t ne33,
- const int32_t nb31, const int32_t nb32, const int64_t nb33) {
+ const int32_t nb31, const int64_t nb32, const int64_t nb33) {
#ifdef FLASH_ATTN_AVAILABLE
const char * GGML_CUDA_RESTRICT Q = Q_ptr;
const char * GGML_CUDA_RESTRICT K = K_ptr;
@@ -859,7 +859,7 @@ static __global__ void flash_attn_tile(
const half2 * K_h2 = (const half2 *) (K + nb13*sequence + nb12*(head0 / gqa_ratio));
const half2 * V_h2 = (const half2 *) (V + nb23*sequence + nb22*(head0 / gqa_ratio)); // K and V have same shape
- const half * maskh = mask ? (const half *) (mask + nb33*(sequence % ne33)) : nullptr;
+ const half * maskh = mask ? (const half *) (mask + nb33*(sequence % ne33) + nb32*(head0 % ne32)) : nullptr;
const int stride_K2 = nb11 / sizeof(half2);
const int stride_V2 = nb21 / sizeof(half2);
@@ -1252,7 +1252,8 @@ static void launch_fattn_tile_switch_ncols2(ggml_backend_cuda_context & ctx, ggm
// However, for DKQ == 576, DV == 512 only the kernel variant with GQA optimizations is implemented.
const bool nvidia = GGML_CUDA_CC_IS_NVIDIA(ggml_cuda_info().devices[ggml_cuda_get_device()].cc);
const int gqa_limit = nvidia && gqa_ratio <= 4 && DV <= 256 ? 16 : INT_MAX;
- const bool use_gqa_opt = mask && max_bias == 0.0f && Q->ne[1] <= gqa_limit && K->ne[1] % FATTN_KQ_STRIDE == 0;
+ // a mask with a head dimension is not shared by the Q heads of a K/V head
+ const bool use_gqa_opt = mask && mask->ne[2] == 1 && max_bias == 0.0f && Q->ne[1] <= gqa_limit && K->ne[1] % FATTN_KQ_STRIDE == 0;
if constexpr (DKQ == 320) {
// This branch is only used for Mistral Small 4 which has a GQA ratio of 32.
diff --git a/ggml/src/ggml-cuda/fattn-vec.cuh b/ggml/src/ggml-cuda/fattn-vec.cuh
index 02c3c327f..5077b1ce1 100644
--- a/ggml/src/ggml-cuda/fattn-vec.cuh
+++ b/ggml/src/ggml-cuda/fattn-vec.cuh
@@ -40,7 +40,7 @@ static __global__ void flash_attn_ext_vec(
const int32_t nb11, const int32_t nb12, const int64_t nb13,
const int32_t nb21, const int32_t nb22, const int64_t nb23,
const int32_t ne31, const int32_t ne32, const int32_t ne33,
- const int32_t nb31, const int32_t nb32, const int64_t nb33) {
+ const int32_t nb31, const int64_t nb32, const int64_t nb33) {
ggml_cuda_pdl_lc();
#ifdef FLASH_ATTN_AVAILABLE
const char * GGML_CUDA_RESTRICT Q = Q_ptr;
@@ -112,7 +112,7 @@ static __global__ void flash_attn_ext_vec(
K += nb13*sequence + nb12*(head / gqa_ratio);
V += nb23*sequence + nb22*(head / gqa_ratio);
- const half * maskh = (const half *) (mask + nb33*(sequence % ne33) + nb31*ic0);
+ const half * maskh = mask ? (const half *) (mask + nb33*(sequence % ne33) + nb32*(head % ne32) + nb31*ic0) : nullptr;
const float slope = get_alibi_slope(max_bias, head, n_head_log2, m0, m1);
diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu
index cf44ab4f5..4e4ed086c 100644
--- a/ggml/src/ggml-cuda/fattn.cu
+++ b/ggml/src/ggml-cuda/fattn.cu
@@ -224,7 +224,8 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2(ggml_backend_cuda_con
// Edge cases like no mask, ALiBi, unpadded K/V, or misaligned addresses for large data transfers
// are put into the template specialization without GQA optimizations.
- bool use_gqa_opt = mask && max_bias == 0.0f && n_kv % FATTN_KQ_STRIDE == 0;
+ // a mask with a head dimension is not shared by the Q heads of a K/V head
+ bool use_gqa_opt = mask && mask->ne[2] == 1 && max_bias == 0.0f && n_kv % FATTN_KQ_STRIDE == 0;
for (const ggml_tensor * t : {Q, K, V, mask}) {
if (t == nullptr || ggml_is_quantized(t->type)) {
continue;
@@ -607,8 +608,9 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float));
// The effective batch size for the kernel can be increased by gqa_ratio.
- // The kernel versions without this optimization are also used for ALiBi, if there is no mask, or if the KV cache is not padded,
- bool gqa_opt_applies = gqa_ratio >= 2 && mask && max_bias == 0.0f && n_kv % FATTN_KQ_STRIDE == 0;
+ // The kernel versions without this optimization are also used for ALiBi, if there is no mask, if the mask has a head dimension,
+ // or if the KV cache is not padded,
+ bool gqa_opt_applies = gqa_ratio >= 2 && mask && mask->ne[2] == 1 && max_bias == 0.0f && n_kv % FATTN_KQ_STRIDE == 0;
for (const ggml_tensor * t : {Q, K, V, mask}) {
if (t == nullptr || ggml_is_quantized(t->type)) {
continue;
@@ -677,7 +679,16 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
}
if (mask && mask->ne[2] != 1) {
- return BEST_FATTN_KERNEL_NONE;
+ // the kv_rows mask is compacted per sequence, it has no head dimension
+ if (kv_rows) {
+ return BEST_FATTN_KERNEL_NONE;
+ }
+ // the logit softcap kernels are compiled for these head sizes only
+ float logit_softcap;
+ memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float));
+ if (logit_softcap != 0.0f && Q->ne[0] != 128 && Q->ne[0] != 256) {
+ return BEST_FATTN_KERNEL_NONE;
+ }
}
// only the MMA kernel reads K/V through kv_rows
@@ -722,7 +733,7 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
const int ncols2_max = Q->ne[0] == 320 ? 32 : ((Q->ne[0] == 576 || Q->ne[0] == 192) ? 16 : 8);
int gqa_ratio_eff = 1;
- while (max_bias == 0.0f && gqa_ratio % (2*gqa_ratio_eff) == 0 && gqa_ratio_eff < ncols2_max) {
+ while (max_bias == 0.0f && (!mask || mask->ne[2] == 1) && gqa_ratio % (2*gqa_ratio_eff) == 0 && gqa_ratio_eff < ncols2_max) {
gqa_ratio_eff *= 2;
}
diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py
index ad0bd8e67..ed3ade939 100644
--- a/gguf-py/gguf/constants.py
+++ b/gguf-py/gguf/constants.py
@@ -107,6 +107,7 @@ class Keys:
class LLM:
VOCAB_SIZE = "{arch}.vocab_size"
+ VOCAB_SIZE_UNPADDED = "{arch}.vocab_size_unpadded"
CONTEXT_LENGTH = "{arch}.context_length"
EMBEDDING_LENGTH = "{arch}.embedding_length"
EMBEDDING_LENGTH_OUT = "{arch}.embedding_length_out"
@@ -233,6 +234,12 @@ class Keys:
VALUE_EXPERT_COUNT = "{arch}.attention.value_expert_count"
VALUE_EXPERT_USED_COUNT = "{arch}.attention.value_expert_used_count"
XSA_EPS = "{arch}.attention.xsa_epsilon"
+ RELATIVE_LENGTH = "{arch}.attention.relative_length"
+ RELATIVE_EXTENT = "{arch}.attention.relative_extent"
+ QK_CLIP = "{arch}.attention.qk_clip"
+ RELATIVE_CLIP = "{arch}.attention.relative_clip"
+ LOG_SCALE_FLOOR = "{arch}.attention.log_scale_floor"
+ LOG_SCALE_ALPHA = "{arch}.attention.log_scale_alpha"
class Indexer:
HEAD_COUNT = "{arch}.attention.indexer.head_count"
@@ -761,6 +768,7 @@ class MODEL_ARCH(IntEnum):
K2HORIZON = auto()
KOLIBRI1 = auto()
LIMITE = auto()
+ INKLING2 = auto()
class VISION_PROJECTOR_TYPE(IntEnum):
@@ -790,6 +798,7 @@ class MODEL_TENSOR(IntEnum):
DENSE_2_OUT = auto() # embeddinggemma 2_Dense
DENSE_3_OUT = auto() # embeddinggemma 3_Dense
OUTPUT_NORM = auto()
+ OUTPUT_RES_QUERY = auto()
HC_HEAD_FN = auto()
HC_HEAD_BASE = auto()
HC_HEAD_SCALE = auto()
@@ -814,6 +823,20 @@ class MODEL_TENSOR(IntEnum):
ATTN_GATE = auto()
ATTN_RESID_ALPHA = auto()
ATTN_RESID_BETA = auto()
+ ATTN_REL = auto()
+ ATTN_REL_BIAS = auto()
+ ATTN_Q_CLIP = auto()
+ ATTN_REL_CLIP = auto()
+ ATTN_SINK_PROJ = auto()
+ ATTN_Q_INP_NORM = auto()
+ ATTN_K_INP_NORM = auto()
+ ATTN_V_INP_NORM = auto()
+ ATTN_K_CONV = auto()
+ ATTN_V_CONV = auto()
+ ATTN_K_CONV_PROJ = auto()
+ ATTN_V_CONV_PROJ = auto()
+ ATTN_POST_CONV = auto()
+ ATTN_RES_QUERY = auto()
FFN_GATE_INP = auto()
FFN_GATE_INP_SHEXP = auto()
FFN_NORM = auto()
@@ -844,6 +867,8 @@ class MODEL_TENSOR(IntEnum):
FFN_GATE_TID2EID = auto()
MOE_LATENT_DOWN = auto() # nemotron 3 super
MOE_LATENT_UP = auto() # nemotron 3 super
+ FFN_POST_CONV = auto()
+ FFN_RES_QUERY = auto()
STREAM_EMBD = auto()
DEPTH_MEM_STATE_WRITE = auto()
DEPTH_MEM_META_WRITE = auto()
@@ -1111,6 +1136,7 @@ class MODEL_TENSOR(IntEnum):
V_ENC_EMBD_PATCH = auto()
V_ENC_EMBD_NORM = auto()
V_ENC_EMBD_PATCH_NORM = auto() # allow multiple norms in the same embd, e.g. for gemma4u
+ V_ENC_EMBD_CLS_POS = auto()
V_ENC_EMBD_POS = auto()
V_ENC_ROPE_FREQS = auto()
V_ENC_INPUT_NORM = auto()
@@ -1680,6 +1706,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
MODEL_ARCH.K2HORIZON: "k2-horizon",
MODEL_ARCH.KOLIBRI1: "kolibri1",
MODEL_ARCH.LIMITE: "limite",
+ MODEL_ARCH.INKLING2: "inkling2",
}
VISION_PROJECTOR_TYPE_NAMES: dict[VISION_PROJECTOR_TYPE, str] = {
@@ -1704,6 +1731,7 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.POS_EMBD: "position_embd",
MODEL_TENSOR.OUTPUT_NORM: "output_norm",
MODEL_TENSOR.OUTPUT: "output",
+ MODEL_TENSOR.OUTPUT_RES_QUERY: "output_res_query",
MODEL_TENSOR.HRM_Z_L_INIT: "hrm.z_l_init",
MODEL_TENSOR.DENSE_2_OUT: "dense_2", # embeddinggemma 2_Dense
MODEL_TENSOR.DENSE_3_OUT: "dense_3", # embeddinggemma 2_Dense
@@ -1729,6 +1757,20 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.ATTN_GATE: "blk.{bid}.attn_gate",
MODEL_TENSOR.ATTN_RESID_ALPHA: "blk.{bid}.attn_resid_alpha",
MODEL_TENSOR.ATTN_RESID_BETA: "blk.{bid}.attn_resid_beta",
+ MODEL_TENSOR.ATTN_REL: "blk.{bid}.attn_rel",
+ MODEL_TENSOR.ATTN_REL_BIAS: "blk.{bid}.attn_rel_bias",
+ MODEL_TENSOR.ATTN_Q_CLIP: "blk.{bid}.attn_q_clip",
+ MODEL_TENSOR.ATTN_REL_CLIP: "blk.{bid}.attn_rel_clip",
+ MODEL_TENSOR.ATTN_SINK_PROJ: "blk.{bid}.attn_sink_proj",
+ MODEL_TENSOR.ATTN_Q_INP_NORM: "blk.{bid}.attn_q_inp_norm",
+ MODEL_TENSOR.ATTN_K_INP_NORM: "blk.{bid}.attn_k_inp_norm",
+ MODEL_TENSOR.ATTN_V_INP_NORM: "blk.{bid}.attn_v_inp_norm",
+ MODEL_TENSOR.ATTN_K_CONV: "blk.{bid}.attn_k_conv",
+ MODEL_TENSOR.ATTN_V_CONV: "blk.{bid}.attn_v_conv",
+ MODEL_TENSOR.ATTN_K_CONV_PROJ: "blk.{bid}.attn_k_conv_proj",
+ MODEL_TENSOR.ATTN_V_CONV_PROJ: "blk.{bid}.attn_v_conv_proj",
+ MODEL_TENSOR.ATTN_POST_CONV: "blk.{bid}.attn_post_conv",
+ MODEL_TENSOR.ATTN_RES_QUERY: "blk.{bid}.attn_res_query",
MODEL_TENSOR.ATTN_Q_NORM: "blk.{bid}.attn_q_norm",
MODEL_TENSOR.ATTN_K_NORM: "blk.{bid}.attn_k_norm",
MODEL_TENSOR.ATTN_OUT_NORM: "blk.{bid}.attn_output_norm",
@@ -1763,6 +1805,8 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.FFN_GATE_TID2EID: "blk.{bid}.ffn_gate_tid2eid",
MODEL_TENSOR.MOE_LATENT_DOWN: "blk.{bid}.ffn_latent_down", # nemotron 3 super
MODEL_TENSOR.MOE_LATENT_UP: "blk.{bid}.ffn_latent_up", # nemotron 3 super
+ MODEL_TENSOR.FFN_POST_CONV: "blk.{bid}.ffn_post_conv",
+ MODEL_TENSOR.FFN_RES_QUERY: "blk.{bid}.ffn_res_query",
MODEL_TENSOR.STREAM_EMBD: "stream_embd",
MODEL_TENSOR.DEPTH_MEM_STATE_WRITE: "depth_mem_state_write",
MODEL_TENSOR.DEPTH_MEM_META_WRITE: "depth_mem_meta_write",
@@ -2026,6 +2070,7 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.V_ENC_EMBD_PATCH: "v.patch_embd",
MODEL_TENSOR.V_ENC_EMBD_NORM: "v.norm_embd",
MODEL_TENSOR.V_ENC_EMBD_PATCH_NORM: "v.patch_norm.{bid}",
+ MODEL_TENSOR.V_ENC_EMBD_CLS_POS: "v.class_position_embd",
MODEL_TENSOR.V_ENC_EMBD_POS: "v.position_embd",
MODEL_TENSOR.V_ENC_ROPE_FREQS: "v.rope_freqs",
MODEL_TENSOR.V_ENC_ATTN_QKV: "v.blk.{bid}.attn_qkv",
@@ -2427,6 +2472,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.V_ENC_EMBD_PATCH,
MODEL_TENSOR.V_ENC_EMBD_NORM,
MODEL_TENSOR.V_ENC_EMBD_PATCH_NORM,
+ MODEL_TENSOR.V_ENC_EMBD_CLS_POS,
MODEL_TENSOR.V_ENC_EMBD_POS,
MODEL_TENSOR.V_ENC_ROPE_FREQS,
MODEL_TENSOR.V_ENC_EMBD_IMGNL,
@@ -6745,6 +6791,42 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
MODEL_TENSOR.MUDD_DOWN,
MODEL_TENSOR.MUDD_UP,
],
+ MODEL_ARCH.INKLING2: [
+ MODEL_TENSOR.TOKEN_EMBD,
+ MODEL_TENSOR.TOKEN_EMBD_NORM,
+ MODEL_TENSOR.OUTPUT,
+ MODEL_TENSOR.OUTPUT_NORM,
+ MODEL_TENSOR.OUTPUT_RES_QUERY,
+ MODEL_TENSOR.ATTN_NORM,
+ MODEL_TENSOR.ATTN_Q,
+ MODEL_TENSOR.ATTN_K,
+ MODEL_TENSOR.ATTN_V,
+ MODEL_TENSOR.ATTN_OUT,
+ MODEL_TENSOR.ATTN_GATE,
+ MODEL_TENSOR.ATTN_REL,
+ MODEL_TENSOR.ATTN_REL_BIAS,
+ MODEL_TENSOR.ATTN_Q_CLIP,
+ MODEL_TENSOR.ATTN_REL_CLIP,
+ MODEL_TENSOR.ATTN_SINK_PROJ,
+ MODEL_TENSOR.ATTN_Q_INP_NORM,
+ MODEL_TENSOR.ATTN_K_INP_NORM,
+ MODEL_TENSOR.ATTN_V_INP_NORM,
+ MODEL_TENSOR.ATTN_K_CONV,
+ MODEL_TENSOR.ATTN_V_CONV,
+ MODEL_TENSOR.ATTN_K_CONV_PROJ,
+ MODEL_TENSOR.ATTN_V_CONV_PROJ,
+ MODEL_TENSOR.ATTN_POST_CONV,
+ MODEL_TENSOR.ATTN_RES_QUERY,
+ MODEL_TENSOR.FFN_NORM,
+ MODEL_TENSOR.FFN_GATE_INP,
+ MODEL_TENSOR.FFN_EXP_PROBS_B,
+ MODEL_TENSOR.FFN_GATE_EXP,
+ MODEL_TENSOR.FFN_UP_EXP,
+ MODEL_TENSOR.FFN_GATE_UP_EXP,
+ MODEL_TENSOR.FFN_DOWN_EXP,
+ MODEL_TENSOR.FFN_POST_CONV,
+ MODEL_TENSOR.FFN_RES_QUERY,
+ ],
}
# tensors that will not be serialized
@@ -7091,6 +7173,7 @@ class VisionProjectorType:
D1OMNI_V = "d1omni_v" # lfm2 vision, without separator tokens
D1OMNI_A = "d1omni_a" # lfm2a audio, with a residual block after the projector
MUSE_SPARK_AUDIO = "muse-spark-audio"
+ INKLING2V = "inkling2v"
# Items here are (block size, type size)
diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py
index 364eeaa75..6ca757080 100644
--- a/gguf-py/gguf/gguf_writer.py
+++ b/gguf-py/gguf/gguf_writer.py
@@ -700,6 +700,9 @@ class GGUFWriter:
def add_vocab_size(self, size: int) -> None:
self.add_uint32(Keys.LLM.VOCAB_SIZE.format(arch=self.arch), size)
+ def add_vocab_size_unpadded(self, size: int) -> None:
+ self.add_uint32(Keys.LLM.VOCAB_SIZE_UNPADDED.format(arch=self.arch), size)
+
def add_context_length(self, length: int) -> None:
self.add_uint32(Keys.LLM.CONTEXT_LENGTH.format(arch=self.arch), length)
@@ -1192,6 +1195,24 @@ class GGUFWriter:
def add_xsa_eps(self, value: float) -> None:
self.add_float32(Keys.Attention.XSA_EPS.format(arch=self.arch), value)
+ def add_relative_length(self, length: int) -> None:
+ self.add_uint32(Keys.Attention.RELATIVE_LENGTH.format(arch=self.arch), length)
+
+ def add_relative_extent(self, extent: int) -> None:
+ self.add_uint32(Keys.Attention.RELATIVE_EXTENT.format(arch=self.arch), extent)
+
+ def add_qk_clip(self, value: float) -> None:
+ self.add_float32(Keys.Attention.QK_CLIP.format(arch=self.arch), value)
+
+ def add_relative_clip(self, value: float) -> None:
+ self.add_float32(Keys.Attention.RELATIVE_CLIP.format(arch=self.arch), value)
+
+ def add_log_scale_floor(self, value: int) -> None:
+ self.add_uint32(Keys.Attention.LOG_SCALE_FLOOR.format(arch=self.arch), value)
+
+ def add_log_scale_alpha(self, value: float) -> None:
+ self.add_float32(Keys.Attention.LOG_SCALE_ALPHA.format(arch=self.arch), value)
+
def add_attn_output_scale(self, value: float) -> None:
self.add_float32(Keys.Attention.OUTPUT_SCALE.format(arch=self.arch), value)
diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py
index bb7d6dcae..4ee516afa 100644
--- a/gguf-py/gguf/tensor_mapping.py
+++ b/gguf-py/gguf/tensor_mapping.py
@@ -66,6 +66,8 @@ class TensorNameMap:
"model.layers.0.pre_norm", # rwkv7
"backbone.norm", # wavtokenizer
"model.embedding_norm", # lfm2
+ "model.embed_tokens.embed_norm", # inkling2
+ "model.embed_norm", # inkling2
),
# Position embeddings
@@ -90,6 +92,7 @@ class TensorNameMap:
"model.transformer.ff_out", # llada
"head.decoder", # modern-bert
"embedding_projection", # embeddinggemma2
+ "model.unembed", # inkling2
),
MODEL_TENSOR.DENSE_2_OUT: (
"dense_2_out", # embeddinggemma
@@ -123,6 +126,11 @@ class TensorNameMap:
"model.embedding_norm", # lfm3
),
+ MODEL_TENSOR.OUTPUT_RES_QUERY: (
+ "model.final_input.query", # inkling2
+ "model.attn_res.output_query", # inkling2
+ ),
+
# Rope frequencies
MODEL_TENSOR.ROPE_FREQS: (
"rope.freqs", # llama-pth
@@ -260,6 +268,7 @@ class TensorNameMap:
"model.layers.{bid}.attention_layernorm", # apertus
"model.layers.{bid}.pre_attention_layernorm", # kormo
"head.layers.{bid}.norm1", # laya
+ "model.layers.{bid}.attn_norm", # inkling2
),
# Attention norm 2
@@ -318,6 +327,7 @@ class TensorNameMap:
"layers.{bid}.self_attn.q_proj", # qwen3-embedding
"backbone.layers.{bid}.mixer.q_proj", # nemotron-h
"model.blocks.{bid}.attn.attn_query", # talkie
+ "model.layers.{bid}.attn.wq_du", # inkling2
),
# Attention key
@@ -340,6 +350,7 @@ class TensorNameMap:
"layers.{bid}.self_attn.k_proj", # qwen3-embedding
"backbone.layers.{bid}.mixer.k_proj", # nemotron-h
"model.blocks.{bid}.attn.attn_key", # talkie
+ "model.layers.{bid}.attn.wk_dv", # inkling2
),
# Attention value
@@ -361,6 +372,7 @@ class TensorNameMap:
"layers.{bid}.self_attn.v_proj", # qwen3-embedding
"backbone.layers.{bid}.mixer.v_proj", # nemotron-h
"model.blocks.{bid}.attn.attn_value", # talkie
+ "model.layers.{bid}.attn.wv_dv", # inkling2
),
# Attention output
@@ -405,6 +417,7 @@ class TensorNameMap:
"model.blocks.{bid}.attn.attn_resid", # talkie
"head.layers.{bid}.self_attn.out_proj", # laya
"layers.{bid}.mixer.out", # metis16
+ "model.layers.{bid}.attn.wo_ud", # inkling2
),
# Attention output norm
@@ -449,12 +462,82 @@ class TensorNameMap:
"model.layers.{bid}.self_attn.output_gate", # minimax-01
"model.layers.{bid}.self_attn.linear_gate", # hy-v4
"model.layers.{bid}.self_attn.attn_gate_proj", # k2-horizon
+ "model.layers.{bid}.attn.gate_attn", # inkling2
),
MODEL_TENSOR.ATTN_VRES_LAMBDA: (
"model.layers.{bid}.self_attn.value_res_lambda", # ricursive
),
+ MODEL_TENSOR.ATTN_REL: (
+ "model.layers.{bid}.self_attn.r_proj", # inkling2
+ "model.layers.{bid}.attn.wr_du", # inkling2
+ ),
+
+ MODEL_TENSOR.ATTN_REL_BIAS: (
+ "model.layers.{bid}.self_attn.rel_logits_proj.proj", # inkling2
+ "model.layers.{bid}.attn.rel_logits_proj.proj", # inkling2
+ ),
+
+ MODEL_TENSOR.ATTN_Q_CLIP: (
+ "model.layers.{bid}.self_attn.q_l2_clip.scale", # inkling2
+ "model.layers.{bid}.attn.q_l2_clip.scale", # inkling2
+ ),
+
+ MODEL_TENSOR.ATTN_REL_CLIP: (
+ "model.layers.{bid}.self_attn.rel_l2_clip.scale", # inkling2
+ "model.layers.{bid}.attn.rel_l2_clip.scale", # inkling2
+ ),
+
+ MODEL_TENSOR.ATTN_SINK_PROJ: (
+ "model.layers.{bid}.self_attn.sink_proj", # inkling2
+ "model.layers.{bid}.attn.attn_sink_proj", # inkling2
+ ),
+
+ MODEL_TENSOR.ATTN_Q_INP_NORM: (
+ "model.layers.{bid}.q_input_layernorm", # inkling2
+ "model.layers.{bid}.q_input_norm", # inkling2
+ ),
+
+ MODEL_TENSOR.ATTN_K_INP_NORM: (
+ "model.layers.{bid}.k_input_layernorm", # inkling2
+ "model.layers.{bid}.k_input_norm", # inkling2
+ ),
+
+ MODEL_TENSOR.ATTN_V_INP_NORM: (
+ "model.layers.{bid}.v_input_layernorm", # inkling2
+ "model.layers.{bid}.v_input_norm", # inkling2
+ ),
+
+ MODEL_TENSOR.ATTN_K_CONV: (
+ "model.layers.{bid}.self_attn.k_sconv.conv1d", # inkling2
+ "model.layers.{bid}.attn.k_sconv", # inkling2
+ ),
+
+ MODEL_TENSOR.ATTN_V_CONV: (
+ "model.layers.{bid}.self_attn.v_sconv.conv1d", # inkling2
+ "model.layers.{bid}.attn.v_sconv", # inkling2
+ ),
+
+ MODEL_TENSOR.ATTN_K_CONV_PROJ: (
+ "model.layers.{bid}.self_attn.k_sconv.weight_proj", # inkling2
+ "model.layers.{bid}.attn.k_sconv.weight_proj", # inkling2
+ ),
+
+ MODEL_TENSOR.ATTN_V_CONV_PROJ: (
+ "model.layers.{bid}.self_attn.v_sconv.weight_proj", # inkling2
+ "model.layers.{bid}.attn.v_sconv.weight_proj", # inkling2
+ ),
+
+ MODEL_TENSOR.ATTN_POST_CONV: (
+ "model.layers.{bid}.post_attention_convolution.conv1d", # inkling2
+ "model.layers.{bid}.attn_sconv", # inkling2
+ ),
+
+ MODEL_TENSOR.ATTN_RES_QUERY: (
+ "model.layers.{bid}.attention_input.query", # inkling2
+ ),
+
# Feed-forward norm
MODEL_TENSOR.FFN_NORM: (
"layers.{bid}.moe_connection.norm", # metis16
@@ -484,6 +567,7 @@ class TensorNameMap:
"model.layers.{bid}.pre_mlp_layernorm", # kormo
"layers.{bid}.mlp_norm", # modern-bert
"head.layers.{bid}.norm2", # laya
+ "model.layers.{bid}.mlp_norm", # inkling2
),
# Pre feed-forward norm
@@ -714,6 +798,16 @@ class TensorNameMap:
"layers.{bid}.moe.latent_up", # metis16
),
+ MODEL_TENSOR.FFN_POST_CONV: (
+ "model.layers.{bid}.post_mlp_convolution.conv1d", # inkling2
+ "model.layers.{bid}.mlp_sconv", # inkling2
+ ),
+
+ MODEL_TENSOR.FFN_RES_QUERY: (
+ "model.layers.{bid}.mlp_input.query", # inkling2
+ "model.attn_res.mlp_queries.{bid}", # inkling2
+ ),
+
MODEL_TENSOR.STREAM_EMBD: (
"stream_embeddings", # metis16
),
@@ -1861,6 +1955,8 @@ class TensorNameMap:
"vision_model.radio_model.model.patch_generator.cls_token.token", # Nemotron Nano v2 VL
"vision_model.embeddings.cls_register_token", # Nemotron Omni
"model.vision_model.embeddings.class_embedding", # Deepseek-OCR
+ "model.vision_tower.embeddings.class_embedding", # inkling2
+ "model.visual.backbone.embeddings.class_embedding", # inkling2
),
MODEL_TENSOR.V_ENC_ROPE_FREQS: (
@@ -1893,6 +1989,7 @@ class TensorNameMap:
"vision_model.conv1", # Step3-VL
"model.vision_embedder.patch_dense", # gemma4 unified
"model.vision_tower.patch_embedder.patch_embedding", # muse-glimmer
+ "model.visual.backbone.embeddings.patch_embedding", # inkling2
),
MODEL_TENSOR.V_ENC_EMBD_NORM: (
@@ -1905,6 +2002,11 @@ class TensorNameMap:
"model.vision_embedder.patch_ln{bid}", # gemma4 unified
),
+ MODEL_TENSOR.V_ENC_EMBD_CLS_POS: (
+ "model.vision_tower.embeddings.cls_position_embedding", # inkling2
+ "model.visual.backbone.embeddings.cls_position_embedding", # inkling2
+ ),
+
MODEL_TENSOR.V_ENC_EMBD_POS: (
"model.vision_tower.vision_model.embeddings.position_embedding", # Granite4Vision
"vision_tower.vision_model.embeddings.position_embedding",
@@ -1923,6 +2025,7 @@ class TensorNameMap:
"model.vision_tower.patch_embedder.position_embedding_table", # gemma4
"vision_model.positional_embedding", # Step3-VL
"model.vision_embedder.pos_embedding", # gemma4 unified
+ "model.visual.backbone.embeddings.position_embedding", # inkling2
),
# TODO: I think these should all be moved to mapping_cfg?
@@ -1970,6 +2073,7 @@ class TensorNameMap:
"vision_model.model.layers.{bid}.self_attn.q_proj.linear", # gemma4
"model.qwen2_model.model.model.layers.{bid}.self_attn.q_proj", # Deepseek-OCR-2 qwen2
"model.vision_tower.layers.{bid}.attn.q_proj", # muse-glimmer
+ "model.visual.backbone.layers.{bid}.self_attn.q_proj", # inkling2
),
MODEL_TENSOR.V_ENC_ATTN_Q_NORM: (
@@ -2001,6 +2105,7 @@ class TensorNameMap:
"vision_model.model.layers.{bid}.self_attn.k_proj.linear", # gemma4
"model.qwen2_model.model.model.layers.{bid}.self_attn.k_proj", # Deepseek-OCR-2 qwen2
"model.vision_tower.layers.{bid}.attn.k_proj", # muse-glimmer
+ "model.visual.backbone.layers.{bid}.self_attn.k_proj", # inkling2
),
MODEL_TENSOR.V_ENC_ATTN_K_NORM: (
@@ -2032,6 +2137,7 @@ class TensorNameMap:
"vision_model.model.layers.{bid}.self_attn.v_proj.linear", # gemma4
"model.qwen2_model.model.model.layers.{bid}.self_attn.v_proj", # Deepseek-OCR-2 qwen2
"model.vision_tower.layers.{bid}.attn.v_proj", # muse-glimmer
+ "model.visual.backbone.layers.{bid}.self_attn.v_proj", # inkling2
),
MODEL_TENSOR.V_ENC_INPUT_NORM: (
@@ -2061,6 +2167,8 @@ class TensorNameMap:
"vision_model.transformer.resblocks.{bid}.ln_1", # Step3-VL
"model.qwen2_model.model.model.layers.{bid}.input_layernorm", # Deepseek-OCR-2 qwen2
"model.vision_tower.layers.{bid}.norm1", # muse-glimmer
+ "model.vision_tower.layers.{bid}.layernorm_before", # inkling2
+ "model.visual.backbone.layers.{bid}.ln_1", # inkling2
),
MODEL_TENSOR.V_ENC_ATTN_O: (
@@ -2092,6 +2200,7 @@ class TensorNameMap:
"vision_encoder.blocks.{bid}.attn.proj", # dots3note
"vision_model.transformer.resblocks.{bid}.attn.out_proj", # Step3-VL
"model.vision_tower.layers.{bid}.attn.proj", # muse-glimmer
+ "model.visual.backbone.layers.{bid}.self_attn.out_proj", # inkling2
),
MODEL_TENSOR.V_ENC_ATTN_SINKS: (
@@ -2126,6 +2235,8 @@ class TensorNameMap:
"vision_model.transformer.resblocks.{bid}.ln_2", # Step3-VL
"model.qwen2_model.model.model.layers.{bid}.post_attention_layernorm", # Deepseek-OCR-2 qwen2
"model.vision_tower.layers.{bid}.norm2", # muse-glimmer
+ "model.vision_tower.layers.{bid}.layernorm_after", # inkling2
+ "model.visual.backbone.layers.{bid}.ln_2", # inkling2
),
MODEL_TENSOR.V_ENC_FFN_UP: (
@@ -2156,6 +2267,7 @@ class TensorNameMap:
"vision_model.transformer.resblocks.{bid}.mlp.c_fc", # Step3-VL
"model.qwen2_model.model.model.layers.{bid}.mlp.up_proj", # Deepseek-OCR-2 qwen2
"model.vision_tower.layers.{bid}.mlp.fc1", # muse-glimmer
+ "model.visual.backbone.layers.{bid}.mlp.c_fc", # inkling2
),
MODEL_TENSOR.V_ENC_FFN_GATE: (
@@ -2197,6 +2309,7 @@ class TensorNameMap:
"vision_model.model.layers.{bid}.mlp.down_proj", # gemma4
"vision_model.transformer.resblocks.{bid}.mlp.c_proj", # Step3-VL
"model.vision_tower.layers.{bid}.mlp.fc2", # muse-glimmer
+ "model.visual.backbone.layers.{bid}.mlp.c_proj", # inkling2
),
@@ -2235,6 +2348,8 @@ class TensorNameMap:
"vision_tower.vision_model.encoder.layers.{bid}.ls1", # InternVL
"model.vision_tower.encoder.layer.{bid}.lambda_1", # Intern-S1
"vision_model.transformer.resblocks.{bid}.ls_1", # Step3-VL
+ "model.vision_tower.layers.{bid}.lambda_1", # inkling2
+ "model.visual.backbone.layers.{bid}.ls_1", # inkling2
),
MODEL_TENSOR.V_LAYER_SCALE_2: (
@@ -2242,6 +2357,8 @@ class TensorNameMap:
"vision_tower.vision_model.encoder.layers.{bid}.ls2", # InternVL
"model.vision_tower.encoder.layer.{bid}.lambda_2", # Intern-S1
"vision_model.transformer.resblocks.{bid}.ls_2", # Step3-VL
+ "model.vision_tower.layers.{bid}.lambda_2", # inkling2
+ "model.visual.backbone.layers.{bid}.ls_2", # inkling2
),
MODEL_TENSOR.V_LAYER_OUT_SCALE: (
@@ -2259,6 +2376,8 @@ class TensorNameMap:
"vision_encoder.patch_embed.norm", # dots3note
"vision_model.ln_pre", # Step3-VL
"model.vision_tower.ln_pre", # muse-glimmer
+ "model.vision_tower.pre_layernorm", # inkling2
+ "model.visual.backbone.ln_pre", # inkling2
),
MODEL_TENSOR.V_POST_NORM: (
diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp
index 6dcf34e77..22419390f 100644
--- a/src/llama-arch.cpp
+++ b/src/llama-arch.cpp
@@ -179,6 +179,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
{ LLM_ARCH_BEAM, "beam" },
{ LLM_ARCH_KOLIBRI1, "kolibri1" },
{ LLM_ARCH_RICURSIVE, "ricursive" },
+ { LLM_ARCH_INKLING2, "inkling2" },
{ LLM_ARCH_UNKNOWN, "(unknown)" },
};
@@ -212,6 +213,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_GENERAL_TENSOR_EXTRA_PREC_A4, "general.tensor_extra.prec_a4" },
{ LLM_KV_VOCAB_SIZE, "%s.vocab_size" },
+ { LLM_KV_VOCAB_SIZE_UNPADDED, "%s.vocab_size_unpadded" },
{ LLM_KV_CONTEXT_LENGTH, "%s.context_length" },
{ LLM_KV_EMBEDDING_LENGTH, "%s.embedding_length" },
{ LLM_KV_EMBEDDING_LENGTH_OUT, "%s.embedding_length_out" },
@@ -345,6 +347,12 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
{ LLM_KV_ATTENTION_SHARED_KV_LAYERS, "%s.attention.shared_kv_layers" },
{ LLM_KV_ATTENTION_KV_SOURCE_MAP, "%s.attention.kv_source_map" },
{ LLM_KV_ATTENTION_RECURRENT_LAYERS, "%s.attention.recurrent_layers" },
+ { LLM_KV_ATTENTION_RELATIVE_LENGTH, "%s.attention.relative_length" },
+ { LLM_KV_ATTENTION_RELATIVE_EXTENT, "%s.attention.relative_extent" },
+ { LLM_KV_ATTENTION_QK_CLIP, "%s.attention.qk_clip" },
+ { LLM_KV_ATTENTION_RELATIVE_CLIP, "%s.attention.relative_clip" },
+ { LLM_KV_ATTENTION_LOG_SCALE_FLOOR, "%s.attention.log_scale_floor" },
+ { LLM_KV_ATTENTION_LOG_SCALE_ALPHA, "%s.attention.log_scale_alpha" },
{ LLM_KV_HYPER_CONNECTION_COUNT, "%s.hyper_connection.count" },
{ LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, "%s.hyper_connection.sinkhorn_iterations" },
@@ -521,6 +529,7 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
{ LLM_TENSOR_OUTPUT_NORM, "output_norm" },
{ LLM_TENSOR_OUTPUT_NORM_LFM2, "token_embd_norm" }, // fix for wrong tensor name
{ LLM_TENSOR_OUTPUT, "output" },
+ { LLM_TENSOR_OUTPUT_RES_QUERY, "output_res_query" },
{ LLM_TENSOR_ROPE_FREQS, "rope_freqs" },
{ LLM_TENSOR_HRM_Z_L_INIT, "hrm.z_l_init" },
{ LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
@@ -547,6 +556,20 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
{ LLM_TENSOR_ATTN_GATE, "blk.%d.attn_gate" },
{ LLM_TENSOR_ATTN_RESID_ALPHA, "blk.%d.attn_resid_alpha" },
{ LLM_TENSOR_ATTN_RESID_BETA, "blk.%d.attn_resid_beta" },
+ { LLM_TENSOR_ATTN_REL, "blk.%d.attn_rel" },
+ { LLM_TENSOR_ATTN_REL_BIAS, "blk.%d.attn_rel_bias" },
+ { LLM_TENSOR_ATTN_Q_CLIP, "blk.%d.attn_q_clip" },
+ { LLM_TENSOR_ATTN_REL_CLIP, "blk.%d.attn_rel_clip" },
+ { LLM_TENSOR_ATTN_SINK_PROJ, "blk.%d.attn_sink_proj" },
+ { LLM_TENSOR_ATTN_Q_INP_NORM, "blk.%d.attn_q_inp_norm" },
+ { LLM_TENSOR_ATTN_K_INP_NORM, "blk.%d.attn_k_inp_norm" },
+ { LLM_TENSOR_ATTN_V_INP_NORM, "blk.%d.attn_v_inp_norm" },
+ { LLM_TENSOR_ATTN_K_CONV, "blk.%d.attn_k_conv" },
+ { LLM_TENSOR_ATTN_V_CONV, "blk.%d.attn_v_conv" },
+ { LLM_TENSOR_ATTN_K_CONV_PROJ, "blk.%d.attn_k_conv_proj" },
+ { LLM_TENSOR_ATTN_V_CONV_PROJ, "blk.%d.attn_v_conv_proj" },
+ { LLM_TENSOR_ATTN_POST_CONV, "blk.%d.attn_post_conv" },
+ { LLM_TENSOR_ATTN_RES_QUERY, "blk.%d.attn_res_query" },
{ LLM_TENSOR_FFN_POST_NORM, "blk.%d.post_ffw_norm" },
{ LLM_TENSOR_FFN_POST_NORM_1, "blk.%d.post_ffw_norm_1" },
{ LLM_TENSOR_FFN_POST_NORM_2, "blk.%d.post_ffw_norm_2" },
@@ -560,6 +583,8 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
{ LLM_TENSOR_FFN_EXP_PROBS_B_VL, "blk.%d.exp_probs_b_vl" },
{ LLM_TENSOR_FFN_LATENT_DOWN, "blk.%d.ffn_latent_down" },
{ LLM_TENSOR_FFN_LATENT_UP, "blk.%d.ffn_latent_up" },
+ { LLM_TENSOR_FFN_POST_CONV, "blk.%d.ffn_post_conv" },
+ { LLM_TENSOR_FFN_RES_QUERY, "blk.%d.ffn_res_query" },
{ LLM_TENSOR_STREAM_EMBD, "stream_embd" },
{ LLM_TENSOR_DEPTH_MEM_STATE_WRITE, "depth_mem_state_write" },
{ LLM_TENSOR_DEPTH_MEM_META_WRITE, "depth_mem_meta_write" },
@@ -929,6 +954,7 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
{LLM_TENSOR_DENSE_3_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, // Dense layer output
{LLM_TENSOR_OUTPUT_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
{LLM_TENSOR_OUTPUT_NORM_LFM2, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
+ {LLM_TENSOR_OUTPUT_RES_QUERY, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_DEC_OUTPUT_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
{LLM_TENSOR_ENC_OUTPUT_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
{LLM_TENSOR_ROPE_FREQS, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ROPE}},
@@ -942,6 +968,20 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
{LLM_TENSOR_ATTN_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_ATTN_RESID_ALPHA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_ATTN_RESID_BETA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_ATTN_REL, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_ATTN_REL_BIAS, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_ATTN_Q_CLIP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_ATTN_REL_CLIP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_ATTN_SINK_PROJ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_ATTN_Q_INP_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_ATTN_K_INP_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_ATTN_V_INP_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
+ {LLM_TENSOR_ATTN_K_CONV, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_SSM_CONV}},
+ {LLM_TENSOR_ATTN_V_CONV, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_SSM_CONV}},
+ {LLM_TENSOR_ATTN_K_CONV_PROJ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_ATTN_V_CONV_PROJ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_ATTN_POST_CONV, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_SSM_CONV}},
+ {LLM_TENSOR_ATTN_RES_QUERY, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_FFN_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_FFN_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_FFN_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
@@ -1202,6 +1242,8 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
// latent projections feed ggml_mul_mat, the buft probe must use MUL_MAT to keep them on GPU
{LLM_TENSOR_FFN_LATENT_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_FFN_LATENT_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
+ {LLM_TENSOR_FFN_POST_CONV, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_SSM_CONV}},
+ {LLM_TENSOR_FFN_RES_QUERY, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
{LLM_TENSOR_STREAM_EMBD, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_ADD}},
{LLM_TENSOR_DEPTH_MEM_STATE_WRITE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_DEPTH_MEM_META_WRITE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
@@ -1374,6 +1416,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) {
case LLM_ARCH_DEEPSEEK4:
case LLM_ARCH_DEEPSEEK41:
case LLM_ARCH_MINIMAX_01:
+ case LLM_ARCH_INKLING2:
return true;
default:
return false;
@@ -1409,6 +1452,7 @@ bool llm_arch_supports_rs_rollback(const llm_arch & arch) {
case LLM_ARCH_LFM2MOE:
case LLM_ARCH_LFM3:
case LLM_ARCH_BAILINGMOE3:
+ case LLM_ARCH_INKLING2:
return true;
default:
return false;
@@ -1426,6 +1470,7 @@ bool llm_arch_supports_mixed_batch(const llm_arch & arch) {
case LLM_ARCH_DFLASH:
case LLM_ARCH_GEMMA4_ASSISTANT:
case LLM_ARCH_LIMITE:
+ case LLM_ARCH_INKLING2:
return false;
default:
return true;
@@ -1444,6 +1489,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
case LLM_ARCH_MAMBA2:
case LLM_ARCH_JAMBA:
case LLM_ARCH_FALCON_H1:
+ case LLM_ARCH_INKLING2:
case LLM_ARCH_OLMO2:
case LLM_ARCH_OLMOE:
case LLM_ARCH_DEEPSEEK2:
diff --git a/src/llama-arch.h b/src/llama-arch.h
index 3b8f45d26..74bd6ce67 100644
--- a/src/llama-arch.h
+++ b/src/llama-arch.h
@@ -183,6 +183,7 @@ enum llm_arch {
LLM_ARCH_KOLIBRI1,
LLM_ARCH_LIMITE,
LLM_ARCH_RICURSIVE,
+ LLM_ARCH_INKLING2,
LLM_ARCH_UNKNOWN,
};
@@ -216,6 +217,7 @@ enum llm_kv {
LLM_KV_GENERAL_TENSOR_EXTRA_PREC_A4,
LLM_KV_VOCAB_SIZE,
+ LLM_KV_VOCAB_SIZE_UNPADDED,
LLM_KV_CONTEXT_LENGTH,
LLM_KV_EMBEDDING_LENGTH,
LLM_KV_EMBEDDING_LENGTH_OUT,
@@ -349,6 +351,12 @@ enum llm_kv {
LLM_KV_ATTENTION_SHARED_KV_LAYERS,
LLM_KV_ATTENTION_KV_SOURCE_MAP,
LLM_KV_ATTENTION_RECURRENT_LAYERS,
+ LLM_KV_ATTENTION_RELATIVE_LENGTH,
+ LLM_KV_ATTENTION_RELATIVE_EXTENT,
+ LLM_KV_ATTENTION_QK_CLIP,
+ LLM_KV_ATTENTION_RELATIVE_CLIP,
+ LLM_KV_ATTENTION_LOG_SCALE_FLOOR,
+ LLM_KV_ATTENTION_LOG_SCALE_ALPHA,
LLM_KV_HYPER_CONNECTION_COUNT,
LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS,
@@ -529,6 +537,7 @@ enum llm_tensor {
LLM_TENSOR_OUTPUT,
LLM_TENSOR_OUTPUT_NORM,
LLM_TENSOR_OUTPUT_NORM_LFM2, // fix for wrong tensor name
+ LLM_TENSOR_OUTPUT_RES_QUERY,
LLM_TENSOR_ROPE_FREQS,
LLM_TENSOR_ROPE_FACTORS_LONG,
LLM_TENSOR_ROPE_FACTORS_SHORT,
@@ -547,6 +556,20 @@ enum llm_tensor {
LLM_TENSOR_ATTN_GATE,
LLM_TENSOR_ATTN_RESID_ALPHA,
LLM_TENSOR_ATTN_RESID_BETA,
+ LLM_TENSOR_ATTN_REL,
+ LLM_TENSOR_ATTN_REL_BIAS,
+ LLM_TENSOR_ATTN_Q_CLIP,
+ LLM_TENSOR_ATTN_REL_CLIP,
+ LLM_TENSOR_ATTN_SINK_PROJ,
+ LLM_TENSOR_ATTN_Q_INP_NORM,
+ LLM_TENSOR_ATTN_K_INP_NORM,
+ LLM_TENSOR_ATTN_V_INP_NORM,
+ LLM_TENSOR_ATTN_K_CONV,
+ LLM_TENSOR_ATTN_V_CONV,
+ LLM_TENSOR_ATTN_K_CONV_PROJ,
+ LLM_TENSOR_ATTN_V_CONV_PROJ,
+ LLM_TENSOR_ATTN_POST_CONV,
+ LLM_TENSOR_ATTN_RES_QUERY,
LLM_TENSOR_FFN_GATE_INP,
LLM_TENSOR_FFN_GATE_INP_SHEXP,
LLM_TENSOR_FFN_NORM,
@@ -578,6 +601,8 @@ enum llm_tensor {
LLM_TENSOR_FFN_EXP_PROBS_B_VL,
LLM_TENSOR_FFN_LATENT_DOWN,
LLM_TENSOR_FFN_LATENT_UP,
+ LLM_TENSOR_FFN_POST_CONV,
+ LLM_TENSOR_FFN_RES_QUERY,
LLM_TENSOR_STREAM_EMBD,
LLM_TENSOR_DEPTH_MEM_STATE_WRITE,
LLM_TENSOR_DEPTH_MEM_META_WRITE,
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index 9066e061e..3e931f1cf 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -2918,7 +2918,8 @@ uint32_t llama_context::graph_max_nodes() const {
model.arch == LLM_ARCH_NANBEIGE ||
model.arch == LLM_ARCH_MINIMAX_01 ||
model.arch == LLM_ARCH_MINIMAX_M3 ||
- model.arch == LLM_ARCH_HY_V4) {
+ model.arch == LLM_ARCH_HY_V4 ||
+ model.arch == LLM_ARCH_INKLING2) {
res = std::max<uint32_t>(1024u, 32u * model.n_tensors());
} else if (model.arch == LLM_ARCH_DFLASH && model.hparams.dflash_selector_rank > 0) {
// DFlash2's convolutions and selector are shape work rather than matmuls,
diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp
index a47cb1e5d..01354c36c 100644
--- a/src/llama-graph.cpp
+++ b/src/llama-graph.cpp
@@ -2285,7 +2285,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
if (probs_in == nullptr) {
logits = build_lora_mm(gate_inp, cur); // [n_expert, n_tokens]
- if (gating_op == LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS) {
+ if (gating_op == LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS || hparams.expert_router_f32) {
// the reference computes the router in F32: https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/blob/60d8d70770c6776ff598c94bb586a859a38244f1/inference/model.py#L565
ggml_prec_set_acc(logits, GGML_PREC_F32);
}
@@ -2403,6 +2403,8 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
if (arch == LLM_ARCH_DEEPSEEK41) {
// model.py Gate.forward: weights / (sum + 1e-20); the sum can fall below the F16 clamp below
weights_sum = ggml_scale_bias(ctx0, weights_sum, 1.0f, 1e-20f);
+ } else if (hparams.expert_weights_norm_eps > 0.0f) {
+ weights_sum = ggml_scale_bias(ctx0, weights_sum, 1.0f, hparams.expert_weights_norm_eps);
} else {
// Avoid division by zero, clamp to smallest number representable by F16
weights_sum = ggml_clamp(ctx0, weights_sum, 6.103515625e-5, INFINITY);
@@ -3032,7 +3034,7 @@ ggml_tensor * llm_graph_context::build_attn_mha(
ggml_tensor * cur;
- const bool use_flash_attn = cparams.flash_attn && kq_b == nullptr;
+ const bool use_flash_attn = cparams.flash_attn && kq_b == nullptr && (!sinks || ggml_is_vector(sinks));
GGML_ASSERT(use_flash_attn || kv_rows == nullptr);
if (use_flash_attn) {
GGML_ASSERT(kq_b == nullptr && "Flash attention does not support KQ bias yet");
@@ -3115,8 +3117,16 @@ ggml_tensor * llm_graph_context::build_attn_mha(
cb(kq, "kq_plus_kq_b", il);
}
- kq = ggml_soft_max_ext(ctx0, kq, kq_mask, kq_scale, hparams.f_max_alibi_bias);
- ggml_soft_max_add_sinks(kq, sinks);
+ if (sinks && !ggml_is_vector(sinks)) {
+ // Per-query sinks are extra logits with zero values.
+ const int64_t n_kv = kq->ne[0];
+ kq = ggml_add(ctx0, ggml_scale(ctx0, kq, kq_scale), ggml_cast(ctx0, kq_mask, GGML_TYPE_F32));
+ kq = ggml_soft_max(ctx0, ggml_concat(ctx0, kq, sinks, 0));
+ kq = ggml_cont(ctx0, ggml_view_4d(ctx0, kq, n_kv, kq->ne[1], kq->ne[2], kq->ne[3], kq->nb[1], kq->nb[2], kq->nb[3], 0));
+ } else {
+ kq = ggml_soft_max_ext(ctx0, kq, kq_mask, kq_scale, hparams.f_max_alibi_bias);
+ ggml_soft_max_add_sinks(kq, sinks);
+ }
cb(kq, "kq_soft_max", il);
if (!v_trans) {
@@ -3610,7 +3620,8 @@ ggml_tensor * llm_graph_context::build_attn(
ggml_tensor * sinks,
ggml_tensor * v_mla,
float kq_scale,
- int il) const {
+ int il,
+ ggml_tensor * kq_mask) const {
const bool is_swa = hparams.is_swa(il);
auto * k_rot = is_swa ? inp->self_k_rot_swa[il] : inp->self_k_rot[il];
@@ -3666,7 +3677,9 @@ ggml_tensor * llm_graph_context::build_attn(
ggml_build_forward_expand(gf, mctx_cur->cpy_v(ctx0, v_cur, v_idxs, il));
}
- const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.raw : inp->self_attn_mask.raw;
+ if (!kq_mask) {
+ kq_mask = is_swa ? inp->self_attn_mask_swa.raw : inp->self_attn_mask.raw;
+ }
const auto & kv_rows = is_swa ? inp->self_attn_mask_swa.rows : inp->self_attn_mask.rows;
ggml_tensor * q = q_cur;
diff --git a/src/llama-graph.h b/src/llama-graph.h
index 559149508..2d8c65718 100644
--- a/src/llama-graph.h
+++ b/src/llama-graph.h
@@ -1387,7 +1387,8 @@ struct llm_graph_context {
ggml_tensor * sinks, // [n_head_q]
ggml_tensor * v_mla, // [n_embd_head_v_mla, n_embd_head_v, n_head_v]
float kq_scale,
- int il) const;
+ int il,
+ ggml_tensor * kq_mask = nullptr) const;
llm_graph_input_attn_k_iswa * build_attn_inp_k_iswa() const;
diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cpp
index f05b5d6bb..66903fbb7 100644
--- a/src/llama-hparams.cpp
+++ b/src/llama-hparams.cpp
@@ -211,6 +211,11 @@ uint32_t llama_hparams::n_embd_r() const {
return token_shift_count * n_embd;
}
+ if (n_sconv_channels != 0) {
+ // for Inkling2 models, checked before LFM2 as they set n_shortconv_l_cache too
+ return n_sconv_channels * (n_shortconv_l_cache - 1);
+ }
+
if (n_shortconv_l_cache != 0) {
// for LFM2 models
return n_embd * (n_shortconv_l_cache - 1);
diff --git a/src/llama-hparams.h b/src/llama-hparams.h
index 7b738407d..b4894956c 100644
--- a/src/llama-hparams.h
+++ b/src/llama-hparams.h
@@ -107,6 +107,7 @@ struct llama_hparams {
struct llama_hparams_convnext convnext;
uint32_t n_shortconv_l_cache = 0;
+ uint32_t n_sconv_channels = 0;
std::array<uint32_t, LLAMA_MAX_LAYERS> n_head_arr;
std::array<uint32_t, LLAMA_MAX_LAYERS> n_head_kv_arr;
@@ -136,6 +137,8 @@ struct llama_hparams {
float expert_group_scale = 0.05f;
float expert_weights_scale = 0.0f;
bool expert_weights_norm = false;
+ float expert_weights_norm_eps = 0.0f; // > 0: weights / (sum + eps), else the sum is clamped
+ bool expert_router_f32 = false; // compute the router logits in F32
uint32_t expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_NONE;
uint32_t moe_every_n_layers = 0;
uint32_t moe_latent_size = 0;
@@ -245,6 +248,16 @@ struct llama_hparams {
uint32_t n_mudd_taps = 0;
std::array<int32_t, LLAMA_MAX_LAYERS> mudd_tap_idx; // [n_layer][n_mudd_taps], -1 = no mixing
+ // inkling2
+ uint32_t n_relative = 0;
+ uint32_t n_relative_extent = 0;
+ uint32_t n_vocab_unpadded = 0;
+ uint32_t n_vocab_padded = 0;
+ uint32_t n_log_scale_floor = 0;
+ float f_log_scale_alpha = 0.0f;
+ float f_qk_clip = 0.0f;
+ float f_relative_clip = 0.0f;
+
bool ssm_dt_b_c_rms = false;
float f_clamp_kqv = 0.0f;
diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp
index 4ec7fb89c..3d6fd2a5f 100644
--- a/src/llama-kv-cache.cpp
+++ b/src/llama-kv-cache.cpp
@@ -2022,6 +2022,20 @@ bool llama_kv_cache::has_cell_ext() const {
return hparams.n_pos_per_embd() > 1 || hparams.ple_n_heads > 0;
}
+void llama_kv_cache::set_input_relative_pos(ggml_tensor * dst, const llama_ubatch * ubatch, uint32_t extent) const {
+ GGML_ASSERT(dst->type == GGML_TYPE_I32);
+ const int64_t n_kv = dst->ne[0];
+ std::vector<int32_t> data(n_kv * ubatch->n_tokens);
+ for (uint32_t i = 0; i < ubatch->n_tokens; ++i) {
+ const auto & cells = v_cells.at(seq_to_stream.at(ubatch->seq_id[i][0]));
+ for (int64_t j = 0; j < n_kv; ++j) {
+ const int64_t distance = cells.is_empty(j) ? -1 : int64_t(ubatch->pos[i]) - cells.pos_get(j);
+ data[i*n_kv + j] = distance >= 0 && distance < extent ? distance : extent;
+ }
+ }
+ ggml_backend_tensor_set(dst, data.data(), 0, ggml_nbytes(dst));
+}
+
void llama_kv_cache::get_prev_tokens(const llama_ubatch & ubatch, uint32_t n, std::vector<llama_token> & res) const {
const uint32_t n_tokens = ubatch.n_tokens;
@@ -3125,6 +3139,10 @@ void llama_kv_cache_context::set_input_pos_bucket(ggml_tensor * dst, const llama
kv->set_input_pos_bucket(dst, ubatch);
}
+void llama_kv_cache_context::set_input_relative_pos(ggml_tensor * dst, const llama_ubatch * ubatch, uint32_t extent) const {
+ kv->set_input_relative_pos(dst, ubatch, extent);
+}
+
void llama_kv_cache_context::get_prev_tokens(const llama_ubatch & ubatch, uint32_t n, std::vector<llama_token> & res) const {
kv->get_prev_tokens(ubatch, n, res);
}
diff --git a/src/llama-kv-cache.h b/src/llama-kv-cache.h
index a81cd8498..1b40f6fc4 100644
--- a/src/llama-kv-cache.h
+++ b/src/llama-kv-cache.h
@@ -249,6 +249,7 @@ public:
void set_input_attn_mask (const llm_attn_mask & attn_mask, const llama_ubatch * ubatch, bool causal_attn) const;
void set_input_pos_bucket(ggml_tensor * dst, const llama_ubatch * ubatch) const;
+ void set_input_relative_pos(ggml_tensor * dst, const llama_ubatch * ubatch, uint32_t extent) const;
// true if llama_kv_cell_ext holds information that has to survive a state save/restore
bool has_cell_ext() const;
@@ -453,6 +454,7 @@ public:
void set_input_k_shift (ggml_tensor * dst) const;
void set_input_attn_mask (const llm_attn_mask & attn_mask, const llama_ubatch * ubatch, bool causal_attn) const;
void set_input_pos_bucket(ggml_tensor * dst, const llama_ubatch * ubatch) const;
+ void set_input_relative_pos(ggml_tensor * dst, const llama_ubatch * ubatch, uint32_t extent) const;
// see llama_kv_cache::get_prev_tokens()
void get_prev_tokens(const llama_ubatch & ubatch, uint32_t n, std::vector<llama_token> & res) const;
diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp
index b667bc49f..4e9e23d10 100644
--- a/src/llama-model-saver.cpp
+++ b/src/llama-model-saver.cpp
@@ -193,7 +193,10 @@ void llama_model_saver::add_kv_from_model() {
// add_kv(LLM_KV_GENERAL_SOURCE_URL, ???);
// add_kv(LLM_KV_GENERAL_SOURCE_HF_REPO, ???);
- add_kv(LLM_KV_VOCAB_SIZE, vocab.n_tokens());
+ add_kv(LLM_KV_VOCAB_SIZE, hparams.n_vocab_padded ? hparams.n_vocab_padded : vocab.n_tokens());
+ if (hparams.n_vocab_unpadded > 0) {
+ add_kv(LLM_KV_VOCAB_SIZE_UNPADDED, hparams.n_vocab_unpadded);
+ }
add_kv(LLM_KV_CONTEXT_LENGTH, hparams.n_ctx_train);
add_kv(LLM_KV_EMBEDDING_LENGTH, hparams.n_embd);
if (hparams.n_embd_out_impl > 0) {
@@ -327,6 +330,14 @@ void llama_model_saver::add_kv_from_model() {
}
}
add_kv(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, true);
+ if (hparams.n_relative > 0) {
+ add_kv(LLM_KV_ATTENTION_RELATIVE_LENGTH, hparams.n_relative);
+ add_kv(LLM_KV_ATTENTION_RELATIVE_EXTENT, hparams.n_relative_extent);
+ add_kv(LLM_KV_ATTENTION_QK_CLIP, hparams.f_qk_clip);
+ add_kv(LLM_KV_ATTENTION_RELATIVE_CLIP, hparams.f_relative_clip);
+ add_kv(LLM_KV_ATTENTION_LOG_SCALE_FLOOR, hparams.n_log_scale_floor);
+ add_kv(LLM_KV_ATTENTION_LOG_SCALE_ALPHA, hparams.f_log_scale_alpha);
+ }
add_kv(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, hparams.dsv4_o_group_count);
add_kv(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, hparams.dsv4_o_lora_rank);
add_kv(LLM_KV_ATTENTION_PASS_LORA_RANK, hparams.metis_pass_lora_rank);
@@ -521,6 +532,7 @@ void llama_model_saver::add_tensors_from_model() {
add_tensor(model->output_s);
add_tensor(model->output_in_s);
add_tensor(model->output_res_score);
+ add_tensor(model->output_res_query);
add_tensor(model->nextn_proj_pre);
add_tensor(model->nextn_proj_post);
add_tensor(model->cls);
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index a5435cb63..42924b938 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -318,6 +318,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_openai_moe(params);
case LLM_ARCH_FALCON_H1:
return new llama_model_falcon_h1(params);
+ case LLM_ARCH_INKLING2:
+ return new llama_model_inkling2(params);
case LLM_ARCH_LFM2:
return new llama_model_lfm2(params);
case LLM_ARCH_LFM2MOE:
@@ -3319,7 +3321,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
filter_idx = [&](uint32_t il) {
return il < hparams.n_layer() && !hparams.is_recr(il) && hparams.is_indexer_full(il);
};
- } else if (arch == LLM_ARCH_FALCON_H1) {
+ } else if (arch == LLM_ARCH_FALCON_H1 || arch == LLM_ARCH_INKLING2) {
filter_attn = [&](uint32_t) { return true; };
filter_recr = [&](uint32_t) { return true; };
} else if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) {
@@ -3693,6 +3695,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_KIMI_K31:
case LLM_ARCH_GLM5_NEXT:
case LLM_ARCH_STARCODER:
+ case LLM_ARCH_INKLING2:
return LLAMA_ROPE_TYPE_NONE;
// use what we call a normal RoPE, operating on pairs of consecutive head values
diff --git a/src/llama-model.h b/src/llama-model.h
index 1b7ea1ce7..67250b789 100644
--- a/src/llama-model.h
+++ b/src/llama-model.h
@@ -673,6 +673,24 @@ struct llama_layer {
struct ggml_tensor * mudd_up = nullptr;
struct ggml_tensor * mudd_up_b = nullptr;
+ // inkling2
+ struct ggml_tensor * attn_rel = nullptr;
+ struct ggml_tensor * attn_rel_bias = nullptr;
+ struct ggml_tensor * attn_q_clip = nullptr;
+ struct ggml_tensor * attn_rel_clip = nullptr;
+ struct ggml_tensor * attn_sink_proj = nullptr;
+ struct ggml_tensor * attn_q_inp_norm = nullptr;
+ struct ggml_tensor * attn_k_inp_norm = nullptr;
+ struct ggml_tensor * attn_v_inp_norm = nullptr;
+ struct ggml_tensor * attn_k_conv = nullptr;
+ struct ggml_tensor * attn_v_conv = nullptr;
+ struct ggml_tensor * attn_k_conv_proj = nullptr;
+ struct ggml_tensor * attn_v_conv_proj = nullptr;
+ struct ggml_tensor * attn_post_conv = nullptr;
+ struct ggml_tensor * attn_res_query = nullptr;
+ struct ggml_tensor * ffn_post_conv = nullptr;
+ struct ggml_tensor * ffn_res_query = nullptr;
+
struct llama_layer_posnet posnet;
struct llama_layer_convnext convnext;
@@ -797,6 +815,7 @@ struct llama_model {
struct ggml_tensor * output_norm = nullptr;
struct ggml_tensor * output_res_score = nullptr; // kimi-k3: final cross-layer residual mix
+ struct ggml_tensor * output_res_query = nullptr; // inkling2: query of the final attention-residual mix
struct ggml_tensor * output_norm_b = nullptr;
struct ggml_tensor * output = nullptr;
struct ggml_tensor * output_b = nullptr;
diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp
index 4ae396101..a50d3c237 100644
--- a/src/llama-quant.cpp
+++ b/src/llama-quant.cpp
@@ -340,6 +340,14 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param
quantize &= name.find("mudd_") == std::string::npos;
}
+ // inkling2: short conv kernels, clip scales, residual mix queries and the relative bias projection
+ if (arch == LLM_ARCH_INKLING2) {
+ quantize &= name.find("_conv.weight") == std::string::npos;
+ quantize &= name.find("_clip.weight") == std::string::npos;
+ quantize &= name.find("_res_query.weight") == std::string::npos;
+ quantize &= name.find("attn_rel_bias.weight") == std::string::npos;
+ }
+
// do not quantize positional embeddings and token types (BERT)
quantize &= name != LLM_TN(arch)(LLM_TENSOR_POS_EMBD, "weight");
quantize &= name != LLM_TN(arch)(LLM_TENSOR_TOKEN_TYPES, "weight");
diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp
index e90f1bd11..f2c1854c9 100644
--- a/src/llama-vocab.cpp
+++ b/src/llama-vocab.cpp
@@ -2677,6 +2677,11 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
tokenizer_pre == "talkie") {
pre_type = LLAMA_VOCAB_PRE_TYPE_GPT4O;
clean_spaces = false;
+ } else if (
+ tokenizer_pre == "inkling2") {
+ pre_type = LLAMA_VOCAB_PRE_TYPE_GPT4O;
+ special_bos_id = LLAMA_TOKEN_NULL;
+ clean_spaces = false;
} else if (
tokenizer_pre == "granite-embed-multi-97m") {
pre_type = LLAMA_VOCAB_PRE_TYPE_GRANITE_EMB_MULTI;
diff --git a/src/models/inkling2.cpp b/src/models/inkling2.cpp
new file mode 100644
index 000000000..41eebd190
--- /dev/null
+++ b/src/models/inkling2.cpp
@@ -0,0 +1,315 @@
+#include "models.h"
+#include "../llama-memory-hybrid-iswa.h"
+
+void llama_model_inkling2::load_arch_hparams(llama_model_loader & ml) {
+ ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);
+ ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+ ml.get_key(LLM_KV_ATTENTION_RELATIVE_LENGTH, hparams.n_relative);
+ ml.get_key(LLM_KV_ATTENTION_RELATIVE_EXTENT, hparams.n_relative_extent);
+ ml.get_key(LLM_KV_ATTENTION_QK_CLIP, hparams.f_qk_clip);
+ ml.get_key(LLM_KV_ATTENTION_RELATIVE_CLIP, hparams.f_relative_clip);
+ ml.get_key(LLM_KV_ATTENTION_LOG_SCALE_FLOOR, hparams.n_log_scale_floor);
+ ml.get_key(LLM_KV_ATTENTION_LOG_SCALE_ALPHA, hparams.f_log_scale_alpha);
+ ml.get_key(LLM_KV_ATTN_RES_BLOCK_SIZE, hparams.attn_res_block_size);
+ ml.get_key(LLM_KV_SHORTCONV_L_CACHE, hparams.n_shortconv_l_cache);
+ ml.get_key(LLM_KV_VOCAB_SIZE, hparams.n_vocab_padded);
+ ml.get_key(LLM_KV_VOCAB_SIZE_UNPADDED, hparams.n_vocab_unpadded);
+ ml.get_key(LLM_KV_ACTIVATION_SITU_BETA, hparams.situ_beta);
+ ml.get_key(LLM_KV_ACTIVATION_SITU_LINEAR_BETA, hparams.situ_linear_beta);
+ ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale);
+ ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping);
+ ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
+ ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);
+ uint32_t n_pattern = 0;
+ ml.get_arr_n(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, n_pattern);
+ if (n_pattern != hparams.n_layer() || !hparams.n_relative || !hparams.n_relative_extent || !hparams.n_swa ||
+ !hparams.attn_res_block_size || !hparams.n_log_scale_floor || hparams.n_shortconv_l_cache < 2 ||
+ hparams.n_shortconv_l_cache > 64 || hparams.n_relative_extent > INT32_MAX || hparams.n_swa > INT32_MAX ||
+ !hparams.n_vocab_unpadded || hparams.n_vocab_unpadded > hparams.n_vocab_padded) {
+ throw std::runtime_error("inkling2: invalid dimensions, vocabulary, or layer pattern");
+ }
+ for (float value : {hparams.f_norm_rms_eps, hparams.f_qk_clip, hparams.f_relative_clip, hparams.situ_beta,
+ hparams.situ_linear_beta, hparams.expert_weights_scale, hparams.f_final_logit_softcapping}) {
+ if (!std::isfinite(value) || value <= 0.0f) {
+ throw std::runtime_error("inkling2: normalization, clipping and scaling parameters must be finite and positive");
+ }
+ }
+ if (!std::isfinite(hparams.f_log_scale_alpha) || hparams.f_log_scale_alpha < 0.0f) {
+ throw std::runtime_error("inkling2: invalid logarithmic scale");
+ }
+ hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
+ hparams.expert_router_f32 = true;
+ hparams.expert_weights_norm_eps = 1e-20f;
+ hparams.rope_pattern.fill(false);
+ uint64_t max_channels = 0;
+ for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
+ if (!hparams.n_head_kv(il) || hparams.n_head(il) % hparams.n_head_kv(il) ||
+ hparams.n_embd_head_k(il) != hparams.n_embd_head_v(il) || !hparams.n_expert_used(il) ||
+ hparams.n_expert_used(il) > hparams.n_expert || !hparams.n_ff_exp(il)) {
+ throw std::runtime_error("inkling2: invalid attention or expert dimensions");
+ }
+ max_channels = std::max<uint64_t>(max_channels, 2ull * hparams.n_embd_k_gqa(il) + 2ull * hparams.n_embd);
+ }
+ if (max_channels * (hparams.n_shortconv_l_cache - 1) > UINT32_MAX) {
+ throw std::runtime_error("inkling2: convolution state is too large");
+ }
+ hparams.n_sconv_channels = max_channels;
+ type = LLM_TYPE_UNKNOWN;
+}
+
+void llama_model_inkling2::load_arch_tensors(llama_model_loader &) {
+ LLAMA_LOAD_LOCALS;
+ if (hparams.n_vocab_unpadded != vocab.n_tokens()) {
+ throw std::runtime_error("inkling2: tokenizer size must equal the unpadded vocabulary size");
+ }
+ tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, hparams.n_vocab_padded}, 0);
+ tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {n_embd}, 0);
+ output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, hparams.n_vocab_padded}, 0);
+ output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+ output_res_query = create_tensor(tn(LLM_TENSOR_OUTPUT_RES_QUERY, "weight"), {n_embd}, 0);
+ for (int i = 0; i < n_layer; ++i) {
+ auto & l = layers[i];
+ const int64_t nh = hparams.n_head(i);
+ const int64_t nk = hparams.n_head_kv(i);
+ const int64_t hd = hparams.n_embd_head_k(i);
+ const int64_t nr = hparams.n_relative;
+ const int64_t kernel = hparams.n_shortconv_l_cache;
+ const int64_t extent = hparams.is_swa(i) ? hparams.n_swa : hparams.n_relative_extent;
+ l.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, nh*hd}, 0);
+ l.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, nk*hd}, 0);
+ l.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, nk*hd}, 0);
+ l.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {nh*hd, n_embd}, 0);
+ l.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, nh*hd}, 0);
+ l.attn_rel = create_tensor(tn(LLM_TENSOR_ATTN_REL, "weight", i), {n_embd, nh*nr}, 0);
+ l.attn_rel_bias = create_tensor(tn(LLM_TENSOR_ATTN_REL_BIAS, "weight", i), {nr, extent}, 0);
+ l.attn_q_clip = create_tensor(tn(LLM_TENSOR_ATTN_Q_CLIP, "weight", i), {hd, nh}, 0);
+ l.attn_rel_clip = create_tensor(tn(LLM_TENSOR_ATTN_REL_CLIP, "weight", i), {nr, nh}, 0);
+ l.attn_sink_proj = create_tensor(tn(LLM_TENSOR_ATTN_SINK_PROJ, "weight", i), {n_embd, nh}, 0);
+ l.attn_k_conv = create_tensor(tn(LLM_TENSOR_ATTN_K_CONV, "weight", i), {kernel, nk*hd}, 0);
+ l.attn_v_conv = create_tensor(tn(LLM_TENSOR_ATTN_V_CONV, "weight", i), {kernel, nk*hd}, 0);
+ l.attn_post_conv = create_tensor(tn(LLM_TENSOR_ATTN_POST_CONV, "weight", i), {kernel, n_embd}, 0);
+ l.ffn_post_conv = create_tensor(tn(LLM_TENSOR_FFN_POST_CONV, "weight", i), {kernel, n_embd}, 0);
+ if (hparams.is_swa(i)) {
+ l.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
+ } else {
+ l.attn_q_inp_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_INP_NORM, "weight", i), {n_embd}, 0);
+ l.attn_k_inp_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_INP_NORM, "weight", i), {n_embd}, 0);
+ l.attn_v_inp_norm = create_tensor(tn(LLM_TENSOR_ATTN_V_INP_NORM, "weight", i), {n_embd}, 0);
+ l.attn_k_conv_proj = create_tensor(tn(LLM_TENSOR_ATTN_K_CONV_PROJ, "weight", i), {n_embd, kernel*nk}, 0);
+ l.attn_v_conv_proj = create_tensor(tn(LLM_TENSOR_ATTN_V_CONV_PROJ, "weight", i), {n_embd, kernel*nk}, 0);
+ }
+ if (i > 0) {
+ l.attn_res_query = create_tensor(tn(LLM_TENSOR_ATTN_RES_QUERY, "weight", i), {n_embd, hparams.is_swa(i) ? 1 : 3}, 0);
+ }
+ l.ffn_res_query = create_tensor(tn(LLM_TENSOR_FFN_RES_QUERY, "weight", i), {n_embd}, 0);
+ l.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
+ l.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
+ l.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED);
+ l.ffn_gate_up_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", i), {n_embd, 2*hparams.n_ff_exp(i), n_expert}, 0);
+ l.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {hparams.n_ff_exp(i), n_embd, n_expert}, 0);
+ }
+}
+
+std::unique_ptr<llm_graph_context> llama_model_inkling2::build_arch_graph(const llm_graph_params & params) const {
+ return std::make_unique<graph>(params);
+}
+
+class llm_graph_input_inkling2_pos : public llm_graph_input_i {
+public:
+ const llama_kv_cache_context * cache[2];
+ ggml_tensor * distance[2];
+ uint32_t extent[2];
+
+ bool can_reuse(const llm_graph_params & params) override {
+ const auto * memory = static_cast<const llama_memory_hybrid_iswa_context *>(params.mctx);
+ cache[0] = memory->get_attn()->get_base();
+ cache[1] = memory->get_attn()->get_swa();
+ for (int i = 0; i < 2; ++i) {
+ if (distance[i]->ne[0] != cache[i]->get_n_kv() || distance[i]->ne[1] != params.ubatch.n_tokens) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ void set_input(const llama_ubatch * ubatch) override {
+ for (int i = 0; i < 2; ++i) {
+ if (distance[i]->buffer) {
+ cache[i]->set_input_relative_pos(distance[i], ubatch, extent[i]);
+ }
+ }
+ }
+};
+
+ggml_tensor * llama_model_inkling2::graph::clip(ggml_tensor * cur, ggml_tensor * scale, float threshold) {
+ if (scale) {
+ cur = ggml_mul(ctx0, cur, scale);
+ }
+ auto * norm = ggml_sqrt(ctx0, ggml_sum_rows(ctx0, ggml_sqr(ctx0, cur)));
+ norm = ggml_scale_bias(ctx0, norm, 1.0f, 1e-6f);
+ auto * factor = ggml_div(ctx0, ggml_fill(ctx0, norm, threshold), norm);
+ return ggml_mul(ctx0, cur, ggml_clamp(ctx0, factor, 0.0f, 1.0f));
+}
+
+ggml_tensor * llama_model_inkling2::graph::mix(const std::vector<ggml_tensor *> & blocks, ggml_tensor * cur, ggml_tensor * query) {
+ auto * states = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens);
+ for (auto * block : blocks) {
+ states = ggml_concat(ctx0, ggml_reshape_3d(ctx0, block, n_embd, 1, n_tokens), states, 1);
+ }
+ states = ggml_rms_norm(ctx0, states, 1e-6f);
+ auto * queries = ggml_repeat(ctx0, query, ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_embd, query->ne[1], n_tokens));
+ auto * scores = ggml_mul_mat(ctx0, states, queries);
+ scores = ggml_soft_max_ext(ctx0, scores, nullptr, 1.0f/sqrtf(n_embd), 0.0f);
+ return ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, states)), scores);
+}
+
+ggml_tensor * llama_model_inkling2::graph::convolve(ggml_tensor * cur, ggml_tensor * weight, ggml_tensor * dynamic,
+ ggml_tensor * state, ggml_tensor * history, llm_graph_input_rs * inp, int64_t offset) {
+ const int64_t kernel = hparams.n_shortconv_l_cache;
+ const int64_t channels = cur->ne[0];
+ const int64_t nt = ubatch.n_seq_tokens;
+ const int64_t ns = ubatch.n_seqs;
+ auto * past = ggml_view_3d(ctx0, history, kernel - 1, channels, ns, history->nb[1], history->nb[2], offset*history->nb[1]);
+ auto * x = ggml_transpose(ctx0, ggml_reshape_3d(ctx0, cur, channels, nt, ns));
+ x = ggml_concat(ctx0, past, x, 0);
+ build_conv_state_store(inp, state, x, kernel - 1, offset*(kernel - 1)*ggml_element_size(state));
+ auto * out = ggml_ssm_conv(ctx0, x, weight);
+ out = ggml_reshape_2d(ctx0, out, channels, n_tokens);
+ if (dynamic) {
+ const int64_t heads = dynamic->ne[0]/kernel;
+ dynamic = ggml_reshape_3d(ctx0, dynamic, kernel, heads, n_tokens);
+ for (int64_t lag = 0; lag < kernel; ++lag) {
+ auto * tap = ggml_view_3d(ctx0, dynamic, 1, heads, n_tokens, dynamic->nb[1], dynamic->nb[2], lag*dynamic->nb[0]);
+ auto * window = ggml_view_3d(ctx0, x, nt, channels, ns, x->nb[1], x->nb[2], (kernel - 1 - lag)*x->nb[0]);
+ window = ggml_cont(ctx0, ggml_transpose(ctx0, window));
+ window = ggml_reshape_3d(ctx0, window, channels/heads, heads, n_tokens);
+ auto * term = ggml_reshape_2d(ctx0, ggml_mul(ctx0, window, tap), channels, n_tokens);
+ out = ggml_add(ctx0, out, term);
+ }
+ }
+ return ggml_scale(ctx0, ggml_add(ctx0, cur, out), 1.0f/sqrtf((dynamic ? 2 : 1)*kernel + 1));
+}
+
+llama_model_inkling2::graph::graph(const llm_graph_params & params) : llm_graph_context(params) {
+ auto * inp = build_inp_mem_hybrid_iswa();
+ auto * attn = inp->get_attn();
+ auto * recr = inp->get_recr();
+ auto * positions = build_inp_pos();
+ auto * out_ids = build_inp_out_ids();
+ auto rel = std::make_unique<llm_graph_input_inkling2_pos>();
+ rel->cache[0] = attn->mctx->get_base();
+ rel->cache[1] = attn->mctx->get_swa();
+ rel->extent[0] = hparams.n_relative_extent;
+ rel->extent[1] = hparams.n_swa;
+ ggml_tensor * distances[2];
+ for (int i = 0; i < 2; ++i) {
+ distances[i] = rel->distance[i] = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, rel->cache[i]->get_n_kv(), n_tokens);
+ ggml_set_input(distances[i]);
+ }
+ res->add_input(std::move(rel));
+ auto * tau = ggml_scale_bias(ctx0, ggml_cast(ctx0, positions, GGML_TYPE_F32), 1.0f, 1.0f);
+ tau = ggml_log(ctx0, ggml_clamp(ctx0, ggml_scale(ctx0, tau, 1.0f/hparams.n_log_scale_floor), 1.0f, INFINITY));
+ tau = ggml_reshape_3d(ctx0, ggml_scale_bias(ctx0, tau, hparams.f_log_scale_alpha, 1.0f), 1, 1, n_tokens);
+ auto * cur = build_inp_embd(model.tok_embd);
+ // important: do not normalize raw embeddings input (i.e. encoded image embeddings)
+ if (ubatch.token) {
+ cur = build_norm(cur, model.tok_norm, nullptr, LLM_NORM_RMS, -1);
+ }
+ std::vector<ggml_tensor *> blocks;
+ for (int il = 0; il < n_layer; ++il) {
+ const auto & l = model.layers[il];
+ const bool swa = hparams.is_swa(il);
+ const int64_t nh = hparams.n_head(il);
+ const int64_t nk = hparams.n_head_kv(il);
+ const int64_t hd = hparams.n_embd_head_k(il);
+ const int64_t kv = nk*hd;
+ auto * state = recr->mctx->get_r_l(il);
+ auto * history = build_rs(recr, state, hparams.n_embd_r(), ubatch.n_seqs);
+ history = ggml_reshape_3d(ctx0, history, hparams.n_shortconv_l_cache - 1, hparams.n_sconv_channels, ubatch.n_seqs);
+ auto * residual = cur;
+ ggml_tensor * inputs[3] = {cur, cur, cur};
+ if (l.attn_res_query) {
+ auto * mixed = mix(blocks, cur, l.attn_res_query);
+ for (int j = 0; j < 3; ++j) {
+ inputs[j] = ggml_cont(ctx0, ggml_view_2d(ctx0, mixed, n_embd, n_tokens, mixed->nb[2], (swa ? 0 : j)*mixed->nb[1]));
+ }
+ }
+ if (swa) {
+ inputs[0] = build_norm(inputs[0], l.attn_norm, nullptr, LLM_NORM_RMS, il);
+ inputs[1] = inputs[2] = inputs[0];
+ } else {
+ inputs[0] = build_norm(inputs[0], l.attn_q_inp_norm, nullptr, LLM_NORM_RMS, il);
+ inputs[1] = build_norm(inputs[1], l.attn_k_inp_norm, nullptr, LLM_NORM_RMS, il);
+ inputs[2] = build_norm(inputs[2], l.attn_v_inp_norm, nullptr, LLM_NORM_RMS, il);
+ }
+ auto * q = build_lora_mm(l.wq, inputs[0]);
+ auto * k = convolve(build_lora_mm(l.wk, inputs[1]), l.attn_k_conv,
+ swa ? nullptr : build_lora_mm(l.attn_k_conv_proj, inputs[1]), state, history, recr, 0);
+ auto * v = convolve(build_lora_mm(l.wv, inputs[2]), l.attn_v_conv,
+ swa ? nullptr : build_lora_mm(l.attn_v_conv_proj, inputs[2]), state, history, recr, kv);
+ auto * r = build_lora_mm(l.attn_rel, inputs[0]);
+ q = clip(ggml_reshape_3d(ctx0, q, hd, nh, n_tokens), l.attn_q_clip, hparams.f_qk_clip);
+ k = clip(ggml_reshape_3d(ctx0, k, hd, nk, n_tokens), nullptr, hparams.f_qk_clip);
+ v = ggml_rms_norm(ctx0, ggml_reshape_3d(ctx0, v, hd, nk, n_tokens), hparams.f_norm_rms_eps);
+ r = clip(ggml_reshape_3d(ctx0, r, hparams.n_relative, nh, n_tokens), l.attn_rel_clip, hparams.f_relative_clip);
+ if (!swa) {
+ q = ggml_mul(ctx0, q, tau);
+ r = ggml_mul(ctx0, r, tau);
+ }
+ auto * mask = swa ? attn->self_attn_mask_swa.raw : attn->self_attn_mask.raw;
+ const int64_t streams = mask->ne[3];
+ const int64_t n_kv = distances[swa]->ne[0];
+ auto * sinks = build_lora_mm(l.attn_sink_proj, inputs[0]);
+ // project r on the bias table first, then gather by distance: only the gathered bias is n_kv x n_tokens x nh
+ auto * table = ggml_mul_mat(ctx0, ggml_pad(ctx0, l.attn_rel_bias, 0, 1, 0, 0), r);
+ if (cparams.flash_attn) {
+ // exp(s - t)/(1 + sum(exp(s - t))) = exp(s)/(exp(t) + sum(exp(s))).
+ table = ggml_sub(ctx0, table, ggml_reshape_3d(ctx0, sinks, 1, nh, n_tokens));
+ } else {
+ // build_attn scales the bias along with QK; FA adds the mask after scaling QK.
+ table = ggml_scale(ctx0, table, sqrtf(hd));
+ }
+ table = ggml_cont(ctx0, ggml_permute(ctx0, table, 1, 0, 2, 3));
+ auto * bias = ggml_get_rows(ctx0, table, distances[swa]);
+ bias = ggml_cont(ctx0, ggml_permute(ctx0, ggml_reshape_4d(ctx0, bias, nh, n_kv, n_tokens/streams, streams), 2, 0, 1, 3));
+ ggml_tensor * fa_mask = nullptr;
+ if (cparams.flash_attn) {
+ fa_mask = ggml_cast(ctx0, ggml_add(ctx0, bias, ggml_cast(ctx0, mask, GGML_TYPE_F32)), GGML_TYPE_F16);
+ sinks = ggml_fill(ctx0, ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, nh), 0.0f);
+ bias = nullptr;
+ } else {
+ sinks = ggml_reshape_4d(ctx0, ggml_cont(ctx0, ggml_permute(ctx0,
+ ggml_reshape_3d(ctx0, sinks, nh, n_tokens/streams, streams), 1, 0, 2, 3)), 1, n_tokens/streams, nh, streams);
+ }
+ cur = build_attn(attn, nullptr, nullptr, nullptr, q, k, v, bias, sinks, nullptr, 1.0f/sqrtf(hd), il, fa_mask);
+ cur = ggml_mul(ctx0, cur, ggml_sigmoid(ctx0, build_lora_mm(l.wqkv_gate, inputs[0])));
+ cur = convolve(build_lora_mm(l.wo, cur), l.attn_post_conv, nullptr, state, history, recr, 2*kv);
+ if (il % hparams.attn_res_block_size == 0) {
+ blocks.push_back(residual);
+ } else {
+ cur = ggml_add(ctx0, residual, cur);
+ }
+ residual = cur;
+ cur = ggml_reshape_2d(ctx0, mix(blocks, cur, l.ffn_res_query), n_embd, n_tokens);
+ cur = build_norm(cur, l.ffn_norm, nullptr, LLM_NORM_RMS, il);
+ cur = build_moe_ffn(cur, l.ffn_gate_inp, nullptr, nullptr, l.ffn_down_exps, l.ffn_exp_probs_b,
+ hparams.n_expert, hparams.n_expert_used(il), LLM_FFN_SITU, true, hparams.expert_weights_scale,
+ LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID, il, nullptr, l.ffn_gate_up_exps);
+ cur = convolve(cur, l.ffn_post_conv, nullptr, state, history, recr, 2*kv + n_embd);
+ cur = ggml_add(ctx0, residual, cur);
+ cb(cur, "l_out", il);
+ }
+ cur = ggml_reshape_2d(ctx0, mix(blocks, cur, model.output_res_query), n_embd, n_tokens);
+ cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1);
+ if (out_ids) {
+ cur = ggml_get_rows(ctx0, cur, out_ids);
+ }
+ res->t_embd = cur;
+ cur = build_lora_mm(model.output, cur);
+ ggml_prec_set_acc(cur, GGML_PREC_F32);
+ const float cap = hparams.f_final_logit_softcapping;
+ cur = ggml_scale(ctx0, ggml_tanh(ctx0, ggml_scale(ctx0, cur, 1.0f/cap)), cap);
+ cur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, hparams.n_vocab_unpadded, cur->ne[1], cur->nb[1], 0));
+ res->t_logits = cur;
+ cb(cur, "result_output", -1);
+}
diff --git a/src/models/models.h b/src/models/models.h
index a978c6fab..e5bc8113d 100644
--- a/src/models/models.h
+++ b/src/models/models.h
@@ -2421,6 +2421,21 @@ struct llama_model_falcon_h1 : public llama_model_base {
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
};
+struct llama_model_inkling2 : public llama_model_base {
+ llama_model_inkling2(const struct llama_model_params & params) : llama_model_base(params) {}
+ void load_arch_hparams(llama_model_loader & ml) override;
+ void load_arch_tensors(llama_model_loader & ml) override;
+
+ struct graph : public llm_graph_context {
+ graph(const llm_graph_params & params);
+ ggml_tensor * mix(const std::vector<ggml_tensor *> & blocks, ggml_tensor * cur, ggml_tensor * query);
+ ggml_tensor * clip(ggml_tensor * cur, ggml_tensor * scale, float threshold);
+ ggml_tensor * convolve(ggml_tensor * cur, ggml_tensor * weight, ggml_tensor * dynamic, ggml_tensor * state, ggml_tensor * history, llm_graph_input_rs * inp, int64_t offset);
+ };
+
+ std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
struct llama_model_lfm2 : public llama_model_base {
llama_model_lfm2(const struct llama_model_params & params) : llama_model_base(params) {}
void load_arch_hparams(llama_model_loader & ml) override;
diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv
index 89e884183..401df6930 100644
--- a/tests/fusion/MTL.csv
+++ b/tests/fusion/MTL.csv
@@ -154,6 +154,9 @@ hy_v4 ,0 ,any ,MUL+ADD , 2
hy_v4 ,0 ,any ,NORM+MUL+ADD , 1
hy_v4 ,0 ,any ,RMS_NORM+MUL , 9
hy_v4 ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 1
+inkling2 ,0 ,any ,MUL+ADD , 6
+inkling2 ,0 ,any ,RMS_NORM+MUL , 10
+inkling2 ,0 ,any ,UNARY+ADD+ARGSORT+GET_ROWS+SUM_ROWS+SCALE+DIV+SCALE, 3
internlm2 ,0 ,any ,RMS_NORM+MUL , 5
jais ,0 ,any ,NORM+MUL+ADD , 5
jais2 ,0 ,any ,NORM+MUL+ADD , 5
diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp
index 1ad7cc2ad..44e54fabf 100644
--- a/tests/test-backend-ops.cpp
+++ b/tests/test-backend-ops.cpp
@@ -9028,9 +9028,10 @@ struct test_flash_attn_ext : public test_case {
const bool v_is_view_of_k;
const int64_t n_kv_max;
const int64_t q_head_padding;
+ const int64_t mask_heads;
std::string vars() override {
- return VARS_TO_STR18(hsk, hsv, nh, nr23, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_K, type_V, permute, kv_view, v_is_view_of_k, n_kv_max, q_head_padding);
+ return VARS_TO_STR18(hsk, hsv, nh, nr23, kv, nb, mask, sinks, max_bias, logit_softcap, prec, type_K, type_V, permute, kv_view, v_is_view_of_k, n_kv_max, q_head_padding) + ",mask_heads=" + std::to_string(mask_heads);
}
double max_nmse_err() override {
@@ -9047,9 +9048,9 @@ struct test_flash_attn_ext : public test_case {
test_flash_attn_ext(int64_t hsk = 128, int64_t hsv = 128, int64_t nh = 32, std::array<int64_t, 2> nr23 = {1, 1}, int64_t kv = 96, int64_t nb = 8,
bool mask = true, bool sinks = false, float max_bias = 0.0f, float logit_softcap = 0.0f, ggml_prec prec = GGML_PREC_F32,
ggml_type type_K = GGML_TYPE_F16, ggml_type type_V = GGML_TYPE_F16, std::array<int32_t, 4> permute = {0, 1, 2, 3},
- bool kv_view = true, bool v_is_view_of_k = false, int64_t n_kv_max = 0, int64_t q_head_padding = 0)
+ bool kv_view = true, bool v_is_view_of_k = false, int64_t n_kv_max = 0, int64_t q_head_padding = 0, int64_t mask_heads = 1)
: hsk(hsk), hsv(hsv), nh(nh), nr23(nr23), kv(kv), nb(nb), mask(mask), sinks(sinks), max_bias(max_bias), logit_softcap(logit_softcap), prec(prec),
- type_K(type_K), type_V(type_V), permute(permute), kv_view(kv_view), v_is_view_of_k(v_is_view_of_k), n_kv_max(n_kv_max), q_head_padding(q_head_padding) {}
+ type_K(type_K), type_V(type_V), permute(permute), kv_view(kv_view), v_is_view_of_k(v_is_view_of_k), n_kv_max(n_kv_max), q_head_padding(q_head_padding), mask_heads(mask_heads) {}
ggml_tensor * build_graph(ggml_context * ctx) override {
const int64_t hsk_padded = GGML_PAD(hsk, ggml_blck_size(type_K));
@@ -9101,7 +9102,7 @@ struct test_flash_attn_ext : public test_case {
ggml_tensor * m = nullptr;
if (mask) {
- m = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, kv, nb, 1, nr23[1]);
+ m = ggml_new_tensor_4d(ctx, GGML_TYPE_F16, kv, nb, mask_heads, nr23[1]);
ggml_set_name(m, "m");
}
@@ -9126,7 +9127,21 @@ struct test_flash_attn_ext : public test_case {
// make the sink values more noticeable in order to trigger a test failure when the implementation is wrong
init_tensor_uniform(t, -10.0f, 10.0f);
} else if (strcmp(t->name, "m") == 0) {
- if (n_kv_max > 0) {
+ if (mask_heads > 1) {
+ std::vector<ggml_fp16_t> data(ggml_nelements(t));
+ for (int64_t b = 0; b < t->ne[3]; ++b) {
+ for (int64_t h = 0; h < t->ne[2]; ++h) {
+ for (int64_t q = 0; q < t->ne[1]; ++q) {
+ for (int64_t k = 0; k < t->ne[0]; ++k) {
+ const int64_t i = ((b*t->ne[2] + h)*t->ne[1] + q)*t->ne[0] + k;
+ const float bias = 0.125f*((k + 3*q + 5*h + 7*b) % 17 - 8);
+ data[i] = ggml_fp32_to_fp16(k < (h + 1)*kv/mask_heads ? bias : -INFINITY);
+ }
+ }
+ }
+ }
+ ggml_backend_tensor_set(t, data.data(), 0, data.size()*sizeof(ggml_fp16_t));
+ } else if (n_kv_max > 0) {
init_tensor_kq_mask_sparse(t, n_kv_max);
} else {
init_tensor_kq_mask(t);
@@ -13194,6 +13209,22 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_flash_attn_ext(512, 512, 4, { 4, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_BF16, GGML_TYPE_BF16));
test_cases.emplace_back(new test_flash_attn_ext(576, 512, 4, { 4, 1}, 512, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_BF16, GGML_TYPE_BF16, {0, 1, 2, 3}, true, true));
+ // FLASH_ATTN_EXT: per-head mask (mask->ne[2] > 1) with sinks
+ for (int hs : {64, 80, 128, 256}) {
+ for (int nq : {1, 3, 35}) {
+ for (int mh : {2, 8}) {
+ for (ggml_type tv : {GGML_TYPE_F16, GGML_TYPE_Q8_0}) {
+ test_cases.emplace_back(new test_flash_attn_ext(hs, hs, 2, {4, 2}, 512, nq, true, true, 0, 0, GGML_PREC_F32,
+ GGML_TYPE_F16, tv, {0, 2, 1, 3}, true, false, 0, 0, mh));
+ }
+ }
+ }
+ }
+ test_cases.emplace_back(new test_flash_attn_ext(64, 64, 1, {4, 1}, 512, 1024, true, true, 0, 0, GGML_PREC_F32,
+ GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 0, 0, 4));
+ test_cases.emplace_back(new test_flash_attn_ext(128, 128, 1, {4, 1}, 512, 3, true, true, 8, 10, GGML_PREC_F32,
+ GGML_TYPE_F16, GGML_TYPE_F16, {0, 1, 2, 3}, true, false, 0, 0, 4));
+
// FLASH_ATTN_EXT: large logits
test_cases.emplace_back(new test_flash_attn_ext_large_logits( 64, 64, 16, {4, 1}, 1024, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
test_cases.emplace_back(new test_flash_attn_ext_large_logits(128, 128, 8, {4, 1}, 1024, 75, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp
index 1f4c8a46b..32e77e883 100644
--- a/tests/test-llama-archs.cpp
+++ b/tests/test-llama-archs.cpp
@@ -326,6 +326,11 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe, const
n_vocab = 3072; // TODO: should be 4096, but user code cannot get `n_vocab_out` yet [TAG_LLAMA_N_VOCAB_OUT]
} else if (arch == LLM_ARCH_HRM_TEXT) {
n_layer = 6; // 1 layer per stack x 2 h-cycles x (2 l-cycles + 1) cache slots
+ } else if (arch == LLM_ARCH_INKLING2) {
+ n_embd = 128;
+ n_head = 4;
+ n_ff = 24;
+ n_layer = 3;
}
uint32_t n_head_kv = n_head;
@@ -691,6 +696,31 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe, const
ms.add_kv(LLM_KV_MUDD_TAP_INDICES, std::vector<int32_t>({-1, -1, -1, -1, 0, 2, 1, 3}));
}
+ if (arch == LLM_ARCH_INKLING2) {
+ ms.add_kv(LLM_KV_ATTENTION_RELATIVE_LENGTH, uint32_t(4));
+ ms.add_kv(LLM_KV_ATTENTION_RELATIVE_EXTENT, uint32_t(16));
+ ms.add_kv(LLM_KV_ATTENTION_QK_CLIP, 0.25f);
+ ms.add_kv(LLM_KV_ATTENTION_RELATIVE_CLIP, 0.1f);
+ ms.add_kv(LLM_KV_ATTENTION_LOG_SCALE_FLOOR, uint32_t(4));
+ ms.add_kv(LLM_KV_ATTENTION_LOG_SCALE_ALPHA, 0.1f);
+ ms.add_kv(LLM_KV_ATTN_RES_BLOCK_SIZE, uint32_t(2));
+ ms.add_kv(LLM_KV_SHORTCONV_L_CACHE, uint32_t(4));
+ ms.add_kv(LLM_KV_VOCAB_SIZE, n_vocab + 2);
+ ms.add_kv(LLM_KV_VOCAB_SIZE_UNPADDED, n_vocab);
+ ms.add_kv(LLM_KV_ACTIVATION_SITU_BETA, 0.2f);
+ ms.add_kv(LLM_KV_ACTIVATION_SITU_LINEAR_BETA, 0.3f);
+ ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 8.0f);
+ ms.add_kv(LLM_KV_FINAL_LOGIT_SOFTCAPPING, 30.0f);
+ ms.add_kv(LLM_KV_EXPERT_COUNT, uint32_t(4));
+ ms.add_kv(LLM_KV_EXPERT_USED_COUNT, uint32_t(2));
+ std::vector<bool> pattern(n_layer);
+ for (uint32_t i = 0; i < n_layer; ++i) {
+ pattern[i] = i % 2 == 0;
+ }
+ std::vector<uint8_t> values(pattern.begin(), pattern.end());
+ gguf_set_arr_data(ret.get(), llama_arch_kv_name(arch, LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN).c_str(), GGUF_TYPE_BOOL, values.data(), values.size());
+ }
+
// dummy tokenizer: token ids are derived from fixed-size chunks and detokenized as hex ids
{
std::vector<std::string> tokenizer_list(n_vocab);
@@ -1047,6 +1077,7 @@ static bool moe_mandatory(const llm_arch arch) {
case LLM_ARCH_MUSE_SPARK:
case LLM_ARCH_KOLIBRI1:
case LLM_ARCH_METIS16:
+ case LLM_ARCH_INKLING2:
return true;
default:
return false;
diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp
index 8730fb91e..5dd3d9384 100644
--- a/tests/test-mtmd-impl.cpp
+++ b/tests/test-mtmd-impl.cpp
@@ -267,6 +267,57 @@ MAKE_TEST(test_image_preprocessor_lfm2_torch_resize) {
}
}
+MAKE_TEST(test_image_preprocessor_inkling2v) {
+ mtmd_hparams hparams;
+ hparams.patch_size = 20;
+ hparams.n_merge = 2;
+ for (int c = 0; c < 3; ++c) {
+ hparams.image_mean[c] = hparams.image_std[c] = 0.5f;
+ }
+ mtmd_image_preprocessor_inkling2v preproc(hparams);
+ for (const auto & size : std::vector<mtmd_image_size>{{1, 1}, {40, 40}, {61, 43}, {83, 45}, {161, 21}}) {
+ mtmd_image_u8 image;
+ image.set_size(size, false);
+ for (int y = 0; y < size.height; ++y) {
+ for (int x = 0; x < size.width; ++x) {
+ image.set_pixel(x, y, {uint8_t(x % 256), uint8_t(y % 256), 255});
+ }
+ }
+ const auto result = preproc.preprocess(image);
+ t.assert_equal("one image", size_t(1), result.entries.size());
+ const auto & output = result.entries[0];
+ t.assert_equal("width ceil", ((size.width + 39) / 40) * 40, output.nx());
+ t.assert_equal("height ceil", ((size.height + 39) / 40) * 40, output.ny());
+ const auto & pixels = output.get_ro_buf();
+ bool exact = true;
+ for (int y = 0; y < output.ny(); ++y) {
+ for (int x = 0; x < output.nx(); ++x) {
+ const bool pad = x >= size.width || y >= size.height;
+ const float rgb[3] = {float(x % 256), float(y % 256), 255.0f};
+ for (int c = 0; c < 3; ++c) {
+ const float expected = ((pad ? -1.0f : rgb[c]) - 127.5f) / 127.5f;
+ exact = exact && pixels[3 * (size_t(y) * output.nx() + x) + c] == expected;
+ }
+ }
+ }
+ t.assert_true("pixels and negative padding", exact);
+ image.set_size(size, true);
+ const auto placeholder = preproc.preprocess(image);
+ t.assert_true("placeholder", placeholder.entries[0].is_placeholder());
+ t.assert_equal("placeholder width", output.nx(), placeholder.entries[0].nx());
+ t.assert_equal("placeholder height", output.ny(), placeholder.entries[0].ny());
+ }
+ // an image above image_max_pixels is resized down, the output stays aligned
+ hparams.image_max_pixels = 16 * 40 * 40;
+ mtmd_image_u8 large;
+ large.set_size({1000, 300}, false);
+ large.cpy_buf(std::vector<uint8_t>(1000 * 300 * 3, 17));
+ const auto capped = preproc.preprocess(large);
+ t.assert_true("capped area", capped.entries[0].nx() * capped.entries[0].ny() <= hparams.image_max_pixels);
+ t.assert_equal("capped width aligned", 0, capped.entries[0].nx() % 40);
+ t.assert_equal("capped height aligned", 0, capped.entries[0].ny() % 40);
+}
+
//
// mtmd temporal merge
//
diff --git a/tests/test_conversion_fp8.py b/tests/test_conversion_fp8.py
index be9c3228c..96387ad88 100644
--- a/tests/test_conversion_fp8.py
+++ b/tests/test_conversion_fp8.py
@@ -20,6 +20,7 @@ from safetensors.torch import save_file
from conversion.base import LazyTorchTensor, gguf
from conversion.deepseek import DeepseekV2Model
+from conversion.inkling2 import Inkling2Model, _E4M3, _E2M1
from conversion.llama import LlamaModel
from conversion.nemotron import NemotronHModel
from conversion.phi import PhiMoeModel
@@ -366,5 +367,313 @@ class TestFP8Conversion(unittest.TestCase):
np.testing.assert_allclose(gguf.dequantize(data, dtype), np.full((4, 32), 0.25), atol=1e-4)
+class TestInkling2PackedConversion(unittest.TestCase):
+ """Conversion of the packed Inkling2 expert weights (see _dequant_expert_chunk in conversion/inkling2.py).
+
+ The checkpoint stores `mlp.experts.w13_weight` (gate and up rows) and `mlp.experts.w2_weight` (down rows) packed:
+ - NVFP4: two 4-bit E2M1 codes per byte, one E4M3 scale per 16 weights, times amax[expert] / outer_scale_denominator
+ - MXFP8: one E4M3 byte per weight, one power-of-two scale per 32 weights, stored as the exponent byte (2^(byte - 127))
+
+ What the tests pin:
+ - decode: packed_checkpoint() decodes the same bytes independently, with torch float8 views and a literal E2M1 table
+ - bit-exactness: the converter output must equal that decode bit for bit, for every outtype
+ - row interleave: w13 rows alternate gate, up, gate, up; the GGUF tensor holds all gate rows, then all up rows
+ """
+
+ N_EXPERT, N_EMBD, N_FF = 2, 64, 160
+ E2M1_VALUES = [0, .5, 1, 1.5, 2, 3, 4, 6, -0., -.5, -1, -1.5, -2, -3, -4, -6]
+
+ @staticmethod
+ def gate_then_up(w13):
+ """Reorders interleaved w13 rows (gate, up, gate, up) to the GGUF layout (all gate rows, then all up rows)."""
+ n_expert, n_row, n_col = w13.shape
+ return w13.reshape(n_expert, n_row // 2, 2, n_col).transpose(1, 2).reshape(w13.shape)
+
+ @staticmethod
+ def interleaved(gate_up):
+ """Inverse of gate_then_up()."""
+ n_expert, n_row, n_col = gate_up.shape
+ return gate_up.reshape(n_expert, 2, n_row // 2, n_col).transpose(1, 2).reshape(gate_up.shape)
+
+ @staticmethod
+ def written_bytes(tensor, path):
+ """Streams a lazy tensor to a file like the GGUF writer does and returns the bytes."""
+ with path.open("wb") as f:
+ tensor.tofile(f)
+ return path.read_bytes()
+
+ def nvfp4(self, shape, denominator):
+ """Packs one NVFP4 projection. Returns the checkpoint tensors by name suffix and the decoded weights."""
+ n_expert, n_row, n_col = shape
+ index = torch.arange(n_expert * n_row * n_col).reshape(shape)
+ codes = ((index * 7 + index // n_col) % 16).to(torch.uint8)
+ packed = codes[:, :, ::2] | (codes[:, :, 1::2] << 4) # low nibble first
+ scales = (torch.arange(n_expert * n_row * (n_col // 16)).reshape(n_expert, n_row, n_col // 16) % 126 + 1).to(torch.uint8).view(torch.float8_e4m3fn)
+ amax = torch.linspace(0.7, 1.9, n_expert)
+ denominator = torch.tensor([denominator])
+ decoded = torch.tensor(self.E2M1_VALUES)[codes.long()] * scales.float().repeat_interleave(16, -1)
+ decoded *= (amax / denominator)[:, None, None]
+ return {"_amax": amax, "_outer_scale_denominator": denominator, "": packed, "_scale": scales}, decoded
+
+ def mxfp8(self, shape):
+ """Packs one MXFP8 projection. Returns the checkpoint tensors by name suffix and the decoded weights."""
+ n_expert, n_row, n_col = shape
+ index = torch.arange(n_expert * n_row * n_col).reshape(shape)
+ packed = ((index * 13 + index // n_col) % 256).to(torch.uint8)
+ packed[(packed & 127) == 127] = 126 # 0x7f and 0xff are NaN in E4M3
+ packed = packed.view(torch.float8_e4m3fn)
+ scales = (torch.arange(n_expert * n_row * (n_col // 32)).reshape(n_expert, n_row, n_col // 32) % 7 + 123).to(torch.uint8)
+ scales[0, 0, 0] = 0
+ decoded = packed.float() * torch.exp2(scales.float() - 127).repeat_interleave(32, -1)
+ return {"": packed, "_scale": scales}, decoded
+
+ def packed_checkpoint(self):
+ """Layer 0 is NVFP4, layer 1 is MXFP8. Returns the checkpoint tensors, the hf_quant_config groups and the expected GGUF tensors."""
+ tensors, quant_groups, expected = {}, [], {}
+ w13_shape = (self.N_EXPERT, 2 * self.N_FF, self.N_EMBD)
+ w2_shape = (self.N_EXPERT, self.N_EMBD, self.N_FF)
+ for layer, scheme in enumerate(("nvfp4_nearest", "mxfp8_nearest")):
+ names = []
+ for projection, shape, target in (("w13", w13_shape, "gate_up"), ("w2", w2_shape, "down")):
+ name = f"model.llm.layers.{layer}.mlp.experts.{projection}_weight"
+ names.append(name)
+ if scheme == "nvfp4_nearest":
+ stored, decoded = self.nvfp4(shape, denominator=3.0 if projection == "w13" else 7.0)
+ else:
+ stored, decoded = self.mxfp8(shape)
+ for suffix, tensor in stored.items():
+ tensors[name + suffix] = tensor
+ if projection == "w13":
+ decoded = self.gate_then_up(decoded)
+ expected[f"blk.{layer}.ffn_{target}_exps.weight"] = decoded.numpy()
+ # the MXFP8 group says [32, 32] but the scales are stored per [1, 32] block: the converter follows the stored shape and warns
+ block_shape = [1, 16] if scheme == "nvfp4_nearest" else [32, 32]
+ quant_groups.append({"tensors": names, "weight_transform": scheme, "weight_scale_format": "amax_denominator", "weight_block_shape": block_shape})
+ return tensors, quant_groups, expected
+
+ def converter(self, path, tensors, quant_groups, ftype=gguf.LlamaFileType.ALL_F32, eager=False):
+ """Writes the checkpoint to path and returns an Inkling2Model on it."""
+ hp = {"model_type": "inkling2_text", "architectures": ["Inkling2ForCausalLM"], "num_hidden_layers": 2, "hidden_size": self.N_EMBD, "moe_intermediate_size": self.N_FF, "n_routed_experts": self.N_EXPERT, "layer_types": ["hybrid_sliding", "hybrid"]}
+ (path / "config.json").write_text(json.dumps(hp))
+ (path / "hf_quant_config.json").write_text(json.dumps({"model": {"parameter_groups": quant_groups}}))
+ save_file(tensors, path / "model.safetensors")
+ return Inkling2Model(path, ftype, path / "unused.gguf", hparams=hp, eager=eager)
+
+ def test_lookup_semantics(self):
+ """The converter decode tables agree with torch.float8_e4m3fn; bytes 127 and 255 are NaN; E2M1 code 8 is -0."""
+ reference = torch.arange(256, dtype=torch.uint8).view(torch.float8_e4m3fn).float().numpy()
+ np.testing.assert_array_equal(_E4M3.view(np.uint32)[np.isfinite(reference)], reference.view(np.uint32)[np.isfinite(reference)])
+ np.testing.assert_array_equal(np.flatnonzero(np.isnan(_E4M3)), [127, 255])
+ self.assertTrue(np.signbit(_E2M1[8]))
+
+ def test_mixed_schemes_and_streaming_outtypes(self):
+ """Both schemes convert bit-exact to every outtype, lazily, in chunks of at most 128 rows."""
+ tensors, quant_groups, expected = self.packed_checkpoint()
+ outtypes = (
+ (gguf.LlamaFileType.GUESSED, gguf.GGMLQuantizationType.F16),
+ (gguf.LlamaFileType.ALL_F32, gguf.GGMLQuantizationType.F32),
+ (gguf.LlamaFileType.MOSTLY_F16, gguf.GGMLQuantizationType.F16),
+ (gguf.LlamaFileType.MOSTLY_BF16, gguf.GGMLQuantizationType.BF16),
+ )
+ for eager in (False, True):
+ for ftype, qtype in outtypes:
+ with self.subTest(eager=eager, outtype=qtype), tempfile.TemporaryDirectory() as directory:
+ path = Path(directory)
+ model = self.converter(path, tensors, quant_groups, ftype, eager)
+ with self.assertLogs("hf-to-gguf", level="WARNING") as logs:
+ model.prepare_tensors()
+ self.assertTrue(any("[32, 32]" in m and "[1, 32]" in m for m in logs.output))
+ infos = model.gguf_writer.tensors[0]
+ self.assertEqual(infos.keys(), expected.keys())
+ for name, info in infos.items():
+ self.assertEqual(info.dtype, qtype)
+ self.assertEqual(tuple(info.shape), expected[name].shape)
+ # the tensor must stay lazy: it cannot be materialized as one array, only streamed
+ self.assertIsInstance(info.tensor, gguf.LazyChunkedTensor)
+ with self.assertRaises(TypeError):
+ np.asarray(info.tensor)
+ for load_chunk in info.tensor._chunks:
+ self.assertLessEqual(load_chunk().shape[0], 128)
+ reference = gguf.quantize(expected[name], qtype)
+ self.assertEqual(self.written_bytes(info.tensor, path / "tensor.bin"), reference.tobytes(), name)
+
+ def test_projection_and_companions_in_separate_shards(self):
+ """A packed weight and its scale, amax and denominator can be in different safetensors shards."""
+ tensors, quant_groups, expected = self.packed_checkpoint()
+ name = quant_groups[0]["tensors"][0]
+ tensors = {k: v for k, v in tensors.items() if k.startswith(name)}
+ with tempfile.TemporaryDirectory() as directory:
+ path = Path(directory)
+ self.converter(path, tensors, quant_groups)
+ (path / "model.safetensors").rename(path / "model-00001-of-00002.safetensors")
+ save_file({name: tensors.pop(name)}, path / "model-00001-of-00002.safetensors")
+ save_file(tensors, path / "model-00002-of-00002.safetensors")
+ model = Inkling2Model(path, gguf.LlamaFileType.ALL_F32, path / "unused.gguf")
+ model.prepare_tensors()
+ key = "blk.0.ffn_gate_up_exps.weight"
+ infos = model.gguf_writer.tensors[0]
+ self.assertEqual(list(infos), [key])
+ tensor = infos[key].tensor
+ assert tensor is not None
+ self.assertEqual(self.written_bytes(tensor, path / "tensor.bin"), expected[key].tobytes())
+
+ def test_float_experts_with_stale_quant_config(self):
+ """Float experts convert unchanged even when hf_quant_config.json still lists them as packed."""
+ _, quant_groups, expected = self.packed_checkpoint()
+ tensors = {}
+ for layer in range(2):
+ for projection, target in (("w13", "gate_up"), ("w2", "down")):
+ values = torch.from_numpy(expected[f"blk.{layer}.ffn_{target}_exps.weight"]).double()
+ if projection == "w13":
+ values = self.interleaved(values)
+ tensors[f"model.llm.layers.{layer}.mlp.experts.{projection}_weight"] = values
+ with tempfile.TemporaryDirectory() as directory:
+ model = self.converter(Path(directory), tensors, quant_groups)
+ model.prepare_tensors()
+ for name, info in model.gguf_writer.tensors[0].items():
+ actual = gguf.LazyNumpyTensor.to_eager(info.tensor)
+ np.testing.assert_array_equal(actual.view(np.uint32), expected[name].view(np.uint32))
+
+ def test_invalid_metadata_and_companions(self):
+ """Missing, mistyped or misshapen companions and unknown schemes are rejected before any tensor is written."""
+ cases = ("missing", "dtype", "amax_shape", "den_shape", "grid", "block", "scheme", "orphan", "mxfp8_amax")
+ for case in cases:
+ tensors, quant_groups, _ = self.packed_checkpoint()
+ name = quant_groups[0]["tensors"][0]
+ if case == "missing":
+ del tensors[name + "_amax"]
+ elif case == "dtype":
+ tensors[name + "_scale"] = tensors[name + "_scale"].view(torch.uint8)
+ elif case == "amax_shape":
+ tensors[name + "_amax"] = torch.ones(1)
+ elif case == "den_shape":
+ tensors[name + "_outer_scale_denominator"] = torch.ones(2)
+ elif case == "grid":
+ tensors[name + "_scale"] = tensors[name + "_scale"][:, :, :3].contiguous()
+ elif case == "block":
+ tensors[name + "_scale"] = tensors[name + "_scale"][:, ::2].contiguous()
+ elif case == "scheme":
+ quant_groups[0]["weight_transform"] = "unknown"
+ elif case == "orphan":
+ del tensors[name]
+ else:
+ tensors[quant_groups[1]["tensors"][0] + "_amax"] = torch.ones(2)
+ with self.subTest(case=case), tempfile.TemporaryDirectory() as directory:
+ model = self.converter(Path(directory), tensors, quant_groups)
+ with self.assertRaisesRegex(ValueError, "Inkling2:"):
+ model.prepare_tensors()
+
+ def test_invalid_values_name_expert(self):
+ """NaN codes and bad scales fail when the tensor is written, and the error names the tensor and the expert."""
+ bad_values = (
+ (0, "_scale", 127), (0, "_scale", 255), # NVFP4 scale is E4M3: NaN bytes
+ (1, "", 127), (1, "", 255), # MXFP8 weight is E4M3: NaN bytes
+ (1, "_scale", 255), # MXFP8 scale exponent 255 is NaN
+ (0, "_amax", float("nan")), (0, "_outer_scale_denominator", 0.0),
+ )
+ for layer, suffix, value in bad_values:
+ tensors, quant_groups, _ = self.packed_checkpoint()
+ name = quant_groups[layer]["tensors"][0]
+ target = tensors[name + suffix]
+ if suffix in ("_amax", "_outer_scale_denominator"):
+ target[-1] = value
+ else:
+ target.view(torch.uint8)[1, 0, 0] = value
+ with self.subTest(layer=layer, suffix=suffix, value=value), tempfile.TemporaryDirectory() as directory:
+ path = Path(directory)
+ model = self.converter(path, tensors, quant_groups)
+ model.prepare_tensors()
+ tensor = model.gguf_writer.tensors[0][f"blk.{layer}.ffn_gate_up_exps.weight"].tensor
+ with self.assertRaisesRegex(ValueError, r"w13_weight.*expert [01]"):
+ self.written_bytes(tensor, path / "tensor.bin")
+
+
+class TestInkling2TextNames(unittest.TestCase):
+ """The checkpoint names (model.llm.*) and the transformers names (model.language_model.*) give the same GGUF tensors."""
+
+ # checkpoint name, transformers name, shape; a 4th entry is the transformers shape when it differs
+ NAMES = (
+ ("embed.weight", "embed_tokens.weight", (16, 8)),
+ ("embed_norm.weight", "embed_tokens.embed_norm.weight", (8,)),
+ ("norm.weight", "norm.weight", (8,)),
+ ("unembed.weight", "lm_head.weight", (16, 8)),
+ ("attn_res.attn_queries.0", "layers.1.attention_input.query", (3, 8)),
+ ("attn_res.mlp_queries.0", "layers.0.mlp_input.query", (8,)),
+ ("attn_res.mlp_queries.1", "layers.1.mlp_input.query", (8,)),
+ ("attn_res.output_query", "final_input.query", (8,)),
+ ("layers.0.attn.k_sconv.weight", "layers.0.self_attn.k_sconv.conv1d.weight", (4, 4), (4, 1, 4)),
+ ("layers.1.attn.k_sconv.static_w", "layers.1.self_attn.k_sconv.static_weight", (3, 4)),
+ ) + tuple((f"layers.{i}.{old}", f"layers.{i}.{new}", *shapes) for i in range(2) for old, new, *shapes in (
+ ("attn_norm.weight", "input_layernorm.weight", (8,)),
+ ("mlp_norm.weight", "post_attention_layernorm.weight", (8,)),
+ ("q_input_norm.weight", "q_input_layernorm.weight", (8,)),
+ ("k_input_norm.weight", "k_input_layernorm.weight", (8,)),
+ ("v_input_norm.weight", "v_input_layernorm.weight", (8,)),
+ ("attn.wq_du.weight", "self_attn.q_proj.weight", (8, 8)),
+ ("attn.wk_dv.weight", "self_attn.k_proj.weight", (4, 8)),
+ ("attn.wv_dv.weight", "self_attn.v_proj.weight", (4, 8)),
+ ("attn.wr_du.weight", "self_attn.r_proj.weight", (4, 8)),
+ ("attn.wo_ud.weight", "self_attn.o_proj.weight", (8, 8)),
+ ("attn.gate_attn.weight", "self_attn.gate_proj.weight", (8, 8)),
+ ("attn.attn_sink_proj.weight", "self_attn.sink_proj.weight", (2, 8)),
+ ("attn.q_l2_clip.scale", "self_attn.q_l2_clip.scale", (2, 4)),
+ ("attn.rel_l2_clip.scale", "self_attn.rel_l2_clip.scale", (2, 2)),
+ ("attn.rel_logits_proj.proj", "self_attn.rel_logits_proj.proj", (2, 5)),
+ ("attn.k_sconv.weight_proj", "self_attn.k_sconv.weight_proj.weight", (6, 8)),
+ ("attn.v_sconv.weight", "self_attn.v_sconv.conv1d.weight", (4, 3), (4, 1, 3)),
+ ("attn.v_sconv.weight_proj", "self_attn.v_sconv.weight_proj.weight", (6, 8)),
+ ("attn_sconv.weight", "post_attention_convolution.conv1d.weight", (8, 3), (8, 1, 3)),
+ ("mlp_sconv.weight", "post_mlp_convolution.conv1d.weight", (8, 3), (8, 1, 3)),
+ ("mlp.gate.weight", "mlp.gate.weight", (2, 8)),
+ ("mlp.gate.bias", "mlp.gate.e_score_correction_bias", (2,)),
+ ))
+
+ def convert(self, tensors):
+ with tempfile.TemporaryDirectory() as directory:
+ return self._convert(Path(directory), tensors)
+
+ def _convert(self, path, tensors):
+ hp = {"model_type": "inkling2_text", "architectures": ["Inkling2ForCausalLM"], "num_hidden_layers": 2, "hidden_size": 8, "layer_types": ["hybrid_sliding", "hybrid"]}
+ (path / "config.json").write_text(json.dumps(hp))
+ save_file(tensors, path / "model.safetensors")
+ model = Inkling2Model(path, gguf.LlamaFileType.ALL_F32, path / "unused.gguf", hparams=hp)
+ model.prepare_tensors()
+ # copy: the tensors are views of the mmapped checkpoint, which must be closed before the directory is removed
+ return {name: gguf.LazyNumpyTensor.to_eager(info.tensor).copy() for name, info in model.gguf_writer.tensors[0].items()}
+
+ def test_checkpoint_and_transformers_names(self):
+ checkpoint, transformers = {}, {}
+ for index, (old, new, shape, *new_shape) in enumerate(self.NAMES):
+ tensor = torch.arange(int(np.prod(shape)), dtype=torch.float32).reshape(shape) / 100 + index
+ checkpoint["model.llm." + old] = tensor
+ transformers[new if new.startswith("lm_head.") else "model.language_model." + new] = tensor.reshape(*new_shape) if new_shape else tensor
+ actual = self.convert(checkpoint)
+ self.assertEqual(len(actual), len(self.NAMES))
+ source = {name: tensor.numpy() for name, tensor in checkpoint.items()}
+ for target, expected in (
+ ("token_embd.weight", source["model.llm.embed.weight"]),
+ ("output.weight", source["model.llm.unembed.weight"]),
+ ("output_res_query.weight", source["model.llm.attn_res.output_query"]),
+ ("blk.1.attn_res_query.weight", source["model.llm.attn_res.attn_queries.0"]), # attn_queries.{i} feeds block i + 1
+ ("blk.1.ffn_res_query.weight", source["model.llm.attn_res.mlp_queries.1"]),
+ ("blk.0.attn_q.weight", source["model.llm.layers.0.attn.wq_du.weight"]),
+ ("blk.0.attn_rel.weight", source["model.llm.layers.0.attn.wr_du.weight"]),
+ ("blk.0.attn_rel_bias.weight", source["model.llm.layers.0.attn.rel_logits_proj.proj"].T),
+ ("blk.0.attn_k_conv.weight", source["model.llm.layers.0.attn.k_sconv.weight"]),
+ ("blk.1.attn_k_conv.weight", source["model.llm.layers.1.attn.k_sconv.static_w"][::-1].T), # static taps are [kernel, channels], newest first
+ ("blk.0.attn_k_conv_proj.weight", source["model.llm.layers.0.attn.k_sconv.weight_proj"]),
+ ("blk.0.attn_post_conv.weight", source["model.llm.layers.0.attn_sconv.weight"]),
+ ("blk.0.ffn_post_conv.weight", source["model.llm.layers.0.mlp_sconv.weight"]),
+ ("blk.0.attn_q_inp_norm.weight", source["model.llm.layers.0.q_input_norm.weight"]),
+ ("blk.0.ffn_gate_inp.weight", source["model.llm.layers.0.mlp.gate.weight"]),
+ ("blk.0.exp_probs_b.bias", source["model.llm.layers.0.mlp.gate.bias"]),
+ ):
+ np.testing.assert_array_equal(actual[target], expected, target)
+ renamed = self.convert(transformers)
+ self.assertEqual(renamed.keys(), actual.keys())
+ for name in actual:
+ np.testing.assert_array_equal(renamed[name], actual[name], name)
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt
index 44ad17733..fd54e2e72 100644
--- a/tools/mtmd/CMakeLists.txt
+++ b/tools/mtmd/CMakeLists.txt
@@ -54,6 +54,7 @@ add_library(mtmd
models/granite-speech.cpp
models/granite4-vision.cpp
models/hunyuanvl.cpp
+ models/inkling2v.cpp
models/internvl.cpp
models/kimivl.cpp
models/kimik25.cpp
diff --git a/tools/mtmd/README-dev.md b/tools/mtmd/README-dev.md
index b24c393a2..736bc9575 100644
--- a/tools/mtmd/README-dev.md
+++ b/tools/mtmd/README-dev.md
@@ -37,6 +37,7 @@ Rules of thumb:
- Like the pp/tg graphs of `llama_context`, the subcontexts share one scheduler: each warmup reserves its worst-case graph on it, so the compute buffers fit the largest one, and switching modality re-plans inside them
- A model declares its runtime needs (e.g. `max_nodes`, `flash_attn_auto`) in `load_hparams`, the subcontext applies them
- Models and backends must outlive the subcontexts, `mtmd_context` declares them first for that reason
+- Models having both audio and vision encoders must have different arch names (projector types), postfixed by `a` and `v`. See Gemma 4 as an example (`gemma4a` / `gemma4v`)
- Locking happens only at the public API boundary (`mtmd.cpp`): a new public function that runs a graph must take `compute_mutex`. The internals (subcontexts, preprocessors, `mtmd_audio_log_mel()`) assume it is held and never lock
## Pipeline
diff --git a/tools/mtmd/models/inkling2v.cpp b/tools/mtmd/models/inkling2v.cpp
new file mode 100644
index 000000000..d5bab6f0a
--- /dev/null
+++ b/tools/mtmd/models/inkling2v.cpp
@@ -0,0 +1,33 @@
+#include "models.h"
+
+// ViT with a CLS token, learned position embeddings and 2D RoPE; the patches are merged before the projector
+ggml_cgraph * mtmd_graph_inkling2v::build() {
+ const int n_pos = n_patches + 1;
+ auto * pos_w = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos);
+ ggml_set_name(pos_w, "pos_w");
+ ggml_set_input(pos_w);
+ auto * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos);
+ ggml_set_name(pos_h, "pos_h");
+ ggml_set_input(pos_h);
+
+ auto * cur = build_inp();
+ cb(cur, "patch_embd", -1);
+ cur = ggml_add(ctx0, cur, resize_position_embeddings(GGML_SCALE_MODE_BILINEAR));
+ auto * cls = ggml_add(ctx0, model.class_embedding, model.class_position_embedding);
+ cur = ggml_concat(ctx0, cls, cur, 1);
+ cb(cur, "embeddings", -1);
+
+ auto add_pos = [&](ggml_tensor * x, const mtmd_layer &) {
+ return build_rope_2d(ctx0, x, pos_w, pos_h, hparams.rope_theta, false);
+ };
+ cur = build_vit(cur, n_pos, NORM_TYPE_NORMAL, hparams.ffn_op, nullptr, add_pos);
+ cb(cur, "encoder_out", -1);
+
+ // remove CLS token
+ cur = ggml_view_2d(ctx0, cur, n_embd, n_patches, cur->nb[1], cur->nb[1]);
+ cur = build_patch_merge_permute(cur, hparams.n_merge);
+ cur = build_ffn(cur, model.mm_1_w, model.mm_1_b, nullptr, nullptr, model.mm_2_w, model.mm_2_b, FFN_GELU_ERF, -1);
+ cb(cur, "projected", -1);
+ ggml_build_forward_expand(gf, cur);
+ return gf;
+}
diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h
index 4f68462ba..58a83378a 100644
--- a/tools/mtmd/models/models.h
+++ b/tools/mtmd/models/models.h
@@ -628,3 +628,8 @@ struct mtmd_graph_muse_glimmer : mtmd_graph {
mtmd_graph_muse_glimmer(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
ggml_cgraph * build() override;
};
+
+struct mtmd_graph_inkling2v : mtmd_graph {
+ using mtmd_graph::mtmd_graph;
+ ggml_cgraph * build() override;
+};
diff --git a/tools/mtmd/mtmd-graph.cpp b/tools/mtmd/mtmd-graph.cpp
index 738837152..01c4c0ac6 100644
--- a/tools/mtmd/mtmd-graph.cpp
+++ b/tools/mtmd/mtmd-graph.cpp
@@ -825,6 +825,10 @@ std::unique_ptr<mtmd_graph> mtmd_get_graph_builder(mtmd_subcontext * ctx, const
{
builder = std::make_unique<mtmd_graph_minimax_m3>(ctx, img);
} break;
+ case PROJECTOR_TYPE_INKLING2V:
+ {
+ builder = std::make_unique<mtmd_graph_inkling2v>(ctx, img);
+ } break;
case PROJECTOR_TYPE_MUSE_GLIMMER:
{
builder = std::make_unique<mtmd_graph_muse_glimmer>(ctx, img);
diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp
index 33ce243d5..c69930f85 100644
--- a/tools/mtmd/mtmd-image.cpp
+++ b/tools/mtmd/mtmd-image.cpp
@@ -2145,6 +2145,49 @@ mtmd_image_preproc_out mtmd_image_preprocessor_muse_glimmer::preprocess(const mt
return output;
}
+//
+// mtmd_image_preprocessor_inkling2v
+//
+
+mtmd_image_preproc_out mtmd_image_preprocessor_inkling2v::preprocess(const mtmd_image_u8 & input) const {
+ const int align = hparams.patch_size * hparams.n_merge;
+ auto size = input.get_size();
+ mtmd_image_u8 resized;
+ const bool too_large = hparams.image_max_pixels > 0 &&
+ int64_t(MTMD_ALIGN(size.width, align)) * MTMD_ALIGN(size.height, align) > hparams.image_max_pixels;
+ if (too_large) {
+ // the reference does not resize; this only bounds the number of image tokens
+ img_tool::calc_size_opt opts;
+ opts.align_size = align;
+ opts.max_pixels = hparams.image_max_pixels;
+ size = img_tool::calc_size_preserved_ratio(size, opts);
+ img_tool::resize(input, resized, size, hparams.image_resize_algo, PAD_NONE);
+ }
+ const mtmd_image_u8 & img = too_large ? resized : input;
+ const int width = MTMD_ALIGN(size.width, align);
+ const int height = MTMD_ALIGN(size.height, align);
+ mtmd_image_f32 padded;
+ padded.set_size({width, height}, img.is_placeholder(), false);
+ if (!img.is_placeholder()) {
+ std::vector<float> pixels(size_t(width) * height * 3, -1.0f);
+ const auto & src = img.get_ro_buf();
+ for (int y = 0; y < size.height; ++y) {
+ std::copy_n(src.begin() + size_t(y) * size.width * 3, size_t(size.width) * 3, pixels.begin() + size_t(y) * width * 3);
+ }
+ padded.cpy_buf(pixels);
+ // Match the reference's fused rescale and normalization, including negative padding.
+ float mean[3], std[3];
+ for (int c = 0; c < 3; ++c) {
+ mean[c] = hparams.image_mean[c] * 255.0f;
+ std[c] = hparams.image_std[c] * 255.0f;
+ }
+ padded.normalize(mean, std);
+ }
+ mtmd_image_preproc_out result;
+ result.append(hparams, padded, false);
+ return result;
+}
+
//#define MTMD_DEBUG_FUNCTIONS
#ifdef MTMD_DEBUG_FUNCTIONS
diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h
index 3e0b127e9..b11f001ab 100644
--- a/tools/mtmd/mtmd-image.h
+++ b/tools/mtmd/mtmd-image.h
@@ -352,3 +352,9 @@ struct mtmd_image_preprocessor_muse_glimmer : mtmd_image_preprocessor {
mtmd_image_preprocessor_muse_glimmer(const mtmd_model * model) : mtmd_image_preprocessor(model) {}
mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override;
};
+
+// pad bottom-right to a multiple of patch_size*n_merge; resize only if the image is above image_max_pixels
+struct mtmd_image_preprocessor_inkling2v : mtmd_image_preprocessor {
+ using mtmd_image_preprocessor::mtmd_image_preprocessor;
+ mtmd_image_preproc_out preprocess(const mtmd_image_u8 & img) const override;
+};
diff --git a/tools/mtmd/mtmd-impl.h b/tools/mtmd/mtmd-impl.h
index 9d447a05e..e2157f767 100644
--- a/tools/mtmd/mtmd-impl.h
+++ b/tools/mtmd/mtmd-impl.h
@@ -124,6 +124,7 @@
#define TN_POS_EMBD "%s.position_embd.weight"
#define TN_CLASS_EMBD "v.class_embd"
+#define TN_CLASS_POS_EMBD "v.class_position_embd.weight"
#define TN_PATCH_EMBD "v.patch_embd.weight" // not rename tensor with ".0" postfix for backward compat
#define TN_PATCH_EMBD_1 "v.patch_embd.weight.1"
#define TN_PATCH_BIAS "v.patch_embd.bias"
@@ -577,6 +578,7 @@ enum projector_type {
PROJECTOR_TYPE_MUSE_SPARK_AUDIO,
PROJECTOR_TYPE_CHATTERBOX,
PROJECTOR_TYPE_CHATTERBOX_SPKENC,
+ PROJECTOR_TYPE_INKLING2V,
PROJECTOR_TYPE_UNKNOWN,
};
@@ -664,6 +666,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
{ PROJECTOR_TYPE_MUSE_SPARK_AUDIO, "muse-spark-audio"},
{ PROJECTOR_TYPE_CHATTERBOX, "chatterbox"},
{ PROJECTOR_TYPE_CHATTERBOX_SPKENC, "chatterbox_spkenc"},
+ { PROJECTOR_TYPE_INKLING2V, "inkling2v"},
};
static projector_type projector_type_from_string(const std::string & str) {
diff --git a/tools/mtmd/mtmd-model.cpp b/tools/mtmd/mtmd-model.cpp
index da74ed383..c4e5bbd54 100644
--- a/tools/mtmd/mtmd-model.cpp
+++ b/tools/mtmd/mtmd-model.cpp
@@ -784,6 +784,15 @@ struct mtmd_model_loader {
}
hparams.warmup_audio_size = hparams.audio_samples_per_token();
} break;
+ case PROJECTOR_TYPE_INKLING2V:
+ {
+ hparams.ffn_op = FFN_GELU_ERF;
+ log_ffn_op = "gelu_erf";
+ get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge);
+ hparams.rope_theta = 10000.0f;
+ hparams.set_limit_image_tokens(1, 4096);
+ hparams.set_warmup_n_tokens(32*32); // avoid OOM on warmup
+ } break;
case PROJECTOR_TYPE_MUSE_GLIMMER:
{
hparams.n_merge = 2; // pixel-shuffle downsample after the ViT
@@ -1904,6 +1913,18 @@ struct mtmd_model_loader {
model.mm_fc_w = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "weight"));
model.mm_fc_b = get_tensor(mtmd_format(TN_MM_AUDIO_FC, "bias"));
} break;
+ case PROJECTOR_TYPE_INKLING2V:
+ {
+ // these come from the generic path above, where they are optional
+ if (!model.class_embedding || !model.position_embeddings || !model.pre_ln_w || !model.pre_ln_b) {
+ throw std::runtime_error(mtmd_format("%s: missing class_embd, position_embd or pre_ln\n", __func__));
+ }
+ model.class_position_embedding = get_tensor(TN_CLASS_POS_EMBD);
+ model.mm_1_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "weight"));
+ model.mm_1_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 1, "bias"), false);
+ model.mm_2_w = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "weight"));
+ model.mm_2_b = get_tensor(mtmd_format(TN_LLAVA_PROJ, 2, "bias"), false);
+ } break;
case PROJECTOR_TYPE_MUSE_GLIMMER:
{
// 3-linear MLP: fc -> erf-GELU -> proj -> erf-GELU -> vision_proj (into LLM residual dim)
@@ -3806,6 +3827,7 @@ int mtmd_model::n_output_tokens_x(const mtmd_image_f32 * img) const {
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
case PROJECTOR_TYPE_MINICPM_OCR_1_0:
+ case PROJECTOR_TYPE_INKLING2V:
return img->nx() / (params.patch_size * params.n_merge);
case PROJECTOR_TYPE_DEEPSEEKOCR:
case PROJECTOR_TYPE_DEEPSEEKOCR2:
@@ -3840,6 +3862,7 @@ int mtmd_model::n_output_tokens_y(const mtmd_image_f32 * img) const {
case PROJECTOR_TYPE_MINICPMV4_6:
case PROJECTOR_TYPE_MINICPMV4_7:
case PROJECTOR_TYPE_MINICPM_OCR_1_0:
+ case PROJECTOR_TYPE_INKLING2V:
return img->ny() / (params.patch_size * params.n_merge);
default:
break;
@@ -3931,6 +3954,7 @@ int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const {
n_patches = x_patch * y_patch;
} break;
case PROJECTOR_TYPE_STEP3VL:
+ case PROJECTOR_TYPE_INKLING2V:
{
int x_patch = img->nx() / (params.patch_size * params.n_merge);
int y_patch = img->ny() / (params.patch_size * params.n_merge);
@@ -4247,6 +4271,7 @@ int mtmd_model::n_mmproj_embd() const {
case PROJECTOR_TYPE_MINIMAX_M3:
return mm_merger_fc2_b->ne[0];
case PROJECTOR_TYPE_MUSE_GLIMMER:
+ case PROJECTOR_TYPE_INKLING2V:
return mm_2_w->ne[1];
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
diff --git a/tools/mtmd/mtmd-model.h b/tools/mtmd/mtmd-model.h
index 0d975ba36..b27b1c0d8 100644
--- a/tools/mtmd/mtmd-model.h
+++ b/tools/mtmd/mtmd-model.h
@@ -898,6 +898,7 @@ struct mtmd_model {
// embeddings
ggml_tensor * class_embedding = nullptr;
+ ggml_tensor * class_position_embedding = nullptr;
ggml_tensor * patch_embeddings_0 = nullptr;
ggml_tensor * patch_embeddings_1 = nullptr; // additional patch kernel; interpretation is model-specific (Qwen2VL, Nemotron Omni)
ggml_tensor * patch_bias = nullptr;
diff --git a/tools/mtmd/mtmd-subcontext.cpp b/tools/mtmd/mtmd-subcontext.cpp
index 7f13b7f62..798d875bb 100644
--- a/tools/mtmd/mtmd-subcontext.cpp
+++ b/tools/mtmd/mtmd-subcontext.cpp
@@ -800,6 +800,19 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
tail.resize(hparams.n_embd, 0.0f);
set_input_f32("inp_stem_tail", tail);
} break;
+ case PROJECTOR_TYPE_INKLING2V:
+ {
+ // CLS is at position 0, the patches are 1-indexed
+ std::vector<int32_t> positions_w{0}, positions_h{0};
+ for (int y = 0; y < pos_h; ++y) {
+ for (int x = 0; x < pos_w; ++x) {
+ positions_w.push_back(x + 1);
+ positions_h.push_back(y + 1);
+ }
+ }
+ set_input_i32("pos_w", positions_w);
+ set_input_i32("pos_h", positions_h);
+ } break;
case PROJECTOR_TYPE_MUSE_GLIMMER:
{
const int grid_w = pos_w; // image_size_width / patch_size
diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp
index 9b45d3de3..929f34620 100644
--- a/tools/mtmd/mtmd.cpp
+++ b/tools/mtmd/mtmd.cpp
@@ -905,6 +905,13 @@ struct mtmd_context {
img_end = "</img>";
image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(models.v.get());
} break;
+ case PROJECTOR_TYPE_INKLING2V:
+ {
+ // image goes in its own message block (always user role)
+ img_beg = "<|end_message|><|message_user|><|content_image|>";
+ img_end = "<|end_message|><|message_user|><|content_text|>";
+ image_preproc = std::make_unique<mtmd_image_preprocessor_inkling2v>(models.v.get());
+ } break;
case PROJECTOR_TYPE_MUSE_GLIMMER:
{
img_beg = "<|image_start|>";